Subversion Repositories Kolibri OS

Rev

Rev 5078 | Rev 5271 | Go to most recent revision | Details | Compare with Previous | Last modification | View Log | RSS feed

Rev Author Line No. Line
5078 serge 1 
/*
2 
 * Copyright 2012 Advanced Micro Devices, Inc.
3 
 *
4 
 * Permission is hereby granted, free of charge, to any person obtaining a
5 
 * copy of this software and associated documentation files (the "Software"),
6 
 * to deal in the Software without restriction, including without limitation
7 
 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 
 * and/or sell copies of the Software, and to permit persons to whom the
9 
 * Software is furnished to do so, subject to the following conditions:
10 
 *
11 
 * The above copyright notice and this permission notice shall be included in
12 
 * all copies or substantial portions of the Software.
13 
 *
14 
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
17 
 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 
 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 
 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 
 * OTHER DEALINGS IN THE SOFTWARE.
21 
 *
22 
 * Authors: Alex Deucher
23 
 */
24 
#include
25 
#include
26 
#include
27 
#include "drmP.h"
28 
#include "radeon.h"
29 
#include "radeon_asic.h"
30 
#include "cikd.h"
31 
#include "atom.h"
32 
#include "cik_blit_shaders.h"
33 
#include "radeon_ucode.h"
34 
#include "clearstate_ci.h"
35 
 
36 
MODULE_FIRMWARE("radeon/BONAIRE_pfp.bin");
37 
MODULE_FIRMWARE("radeon/BONAIRE_me.bin");
38 
MODULE_FIRMWARE("radeon/BONAIRE_ce.bin");
39 
MODULE_FIRMWARE("radeon/BONAIRE_mec.bin");
40 
MODULE_FIRMWARE("radeon/BONAIRE_mc.bin");
41 
MODULE_FIRMWARE("radeon/BONAIRE_mc2.bin");
42 
MODULE_FIRMWARE("radeon/BONAIRE_rlc.bin");
43 
MODULE_FIRMWARE("radeon/BONAIRE_sdma.bin");
44 
MODULE_FIRMWARE("radeon/BONAIRE_smc.bin");
45 
 
46 
MODULE_FIRMWARE("radeon/bonaire_pfp.bin");
47 
MODULE_FIRMWARE("radeon/bonaire_me.bin");
48 
MODULE_FIRMWARE("radeon/bonaire_ce.bin");
49 
MODULE_FIRMWARE("radeon/bonaire_mec.bin");
50 
MODULE_FIRMWARE("radeon/bonaire_mc.bin");
51 
MODULE_FIRMWARE("radeon/bonaire_rlc.bin");
52 
MODULE_FIRMWARE("radeon/bonaire_sdma.bin");
53 
MODULE_FIRMWARE("radeon/bonaire_smc.bin");
54 
 
55 
MODULE_FIRMWARE("radeon/HAWAII_pfp.bin");
56 
MODULE_FIRMWARE("radeon/HAWAII_me.bin");
57 
MODULE_FIRMWARE("radeon/HAWAII_ce.bin");
58 
MODULE_FIRMWARE("radeon/HAWAII_mec.bin");
59 
MODULE_FIRMWARE("radeon/HAWAII_mc.bin");
60 
MODULE_FIRMWARE("radeon/HAWAII_mc2.bin");
61 
MODULE_FIRMWARE("radeon/HAWAII_rlc.bin");
62 
MODULE_FIRMWARE("radeon/HAWAII_sdma.bin");
63 
MODULE_FIRMWARE("radeon/HAWAII_smc.bin");
64 
 
65 
MODULE_FIRMWARE("radeon/hawaii_pfp.bin");
66 
MODULE_FIRMWARE("radeon/hawaii_me.bin");
67 
MODULE_FIRMWARE("radeon/hawaii_ce.bin");
68 
MODULE_FIRMWARE("radeon/hawaii_mec.bin");
69 
MODULE_FIRMWARE("radeon/hawaii_mc.bin");
70 
MODULE_FIRMWARE("radeon/hawaii_rlc.bin");
71 
MODULE_FIRMWARE("radeon/hawaii_sdma.bin");
72 
MODULE_FIRMWARE("radeon/hawaii_smc.bin");
73 
 
74 
MODULE_FIRMWARE("radeon/KAVERI_pfp.bin");
75 
MODULE_FIRMWARE("radeon/KAVERI_me.bin");
76 
MODULE_FIRMWARE("radeon/KAVERI_ce.bin");
77 
MODULE_FIRMWARE("radeon/KAVERI_mec.bin");
78 
MODULE_FIRMWARE("radeon/KAVERI_rlc.bin");
79 
MODULE_FIRMWARE("radeon/KAVERI_sdma.bin");
80 
 
81 
MODULE_FIRMWARE("radeon/kaveri_pfp.bin");
82 
MODULE_FIRMWARE("radeon/kaveri_me.bin");
83 
MODULE_FIRMWARE("radeon/kaveri_ce.bin");
84 
MODULE_FIRMWARE("radeon/kaveri_mec.bin");
85 
MODULE_FIRMWARE("radeon/kaveri_mec2.bin");
86 
MODULE_FIRMWARE("radeon/kaveri_rlc.bin");
87 
MODULE_FIRMWARE("radeon/kaveri_sdma.bin");
88 
 
89 
MODULE_FIRMWARE("radeon/KABINI_pfp.bin");
90 
MODULE_FIRMWARE("radeon/KABINI_me.bin");
91 
MODULE_FIRMWARE("radeon/KABINI_ce.bin");
92 
MODULE_FIRMWARE("radeon/KABINI_mec.bin");
93 
MODULE_FIRMWARE("radeon/KABINI_rlc.bin");
94 
MODULE_FIRMWARE("radeon/KABINI_sdma.bin");
95 
 
96 
MODULE_FIRMWARE("radeon/kabini_pfp.bin");
97 
MODULE_FIRMWARE("radeon/kabini_me.bin");
98 
MODULE_FIRMWARE("radeon/kabini_ce.bin");
99 
MODULE_FIRMWARE("radeon/kabini_mec.bin");
100 
MODULE_FIRMWARE("radeon/kabini_rlc.bin");
101 
MODULE_FIRMWARE("radeon/kabini_sdma.bin");
102 
 
103 
MODULE_FIRMWARE("radeon/MULLINS_pfp.bin");
104 
MODULE_FIRMWARE("radeon/MULLINS_me.bin");
105 
MODULE_FIRMWARE("radeon/MULLINS_ce.bin");
106 
MODULE_FIRMWARE("radeon/MULLINS_mec.bin");
107 
MODULE_FIRMWARE("radeon/MULLINS_rlc.bin");
108 
MODULE_FIRMWARE("radeon/MULLINS_sdma.bin");
109 
 
110 
MODULE_FIRMWARE("radeon/mullins_pfp.bin");
111 
MODULE_FIRMWARE("radeon/mullins_me.bin");
112 
MODULE_FIRMWARE("radeon/mullins_ce.bin");
113 
MODULE_FIRMWARE("radeon/mullins_mec.bin");
114 
MODULE_FIRMWARE("radeon/mullins_rlc.bin");
115 
MODULE_FIRMWARE("radeon/mullins_sdma.bin");
116 
 
117 
extern int r600_ih_ring_alloc(struct radeon_device *rdev);
118 
extern void r600_ih_ring_fini(struct radeon_device *rdev);
119 
extern void evergreen_mc_stop(struct radeon_device *rdev, struct evergreen_mc_save *save);
120 
extern void evergreen_mc_resume(struct radeon_device *rdev, struct evergreen_mc_save *save);
121 
extern bool evergreen_is_display_hung(struct radeon_device *rdev);
122 
extern void sumo_rlc_fini(struct radeon_device *rdev);
123 
extern int sumo_rlc_init(struct radeon_device *rdev);
124 
extern void si_vram_gtt_location(struct radeon_device *rdev, struct radeon_mc *mc);
125 
extern void si_rlc_reset(struct radeon_device *rdev);
126 
extern void si_init_uvd_internal_cg(struct radeon_device *rdev);
127 
static u32 cik_get_cu_active_bitmap(struct radeon_device *rdev, u32 se, u32 sh);
128 
extern int cik_sdma_resume(struct radeon_device *rdev);
129 
extern void cik_sdma_enable(struct radeon_device *rdev, bool enable);
130 
extern void cik_sdma_fini(struct radeon_device *rdev);
131 
extern void vce_v2_0_enable_mgcg(struct radeon_device *rdev, bool enable);
132 
static void cik_rlc_stop(struct radeon_device *rdev);
133 
static void cik_pcie_gen3_enable(struct radeon_device *rdev);
134 
static void cik_program_aspm(struct radeon_device *rdev);
135 
static void cik_init_pg(struct radeon_device *rdev);
136 
static void cik_init_cg(struct radeon_device *rdev);
137 
static void cik_fini_pg(struct radeon_device *rdev);
138 
static void cik_fini_cg(struct radeon_device *rdev);
139 
static void cik_enable_gui_idle_interrupt(struct radeon_device *rdev,
140 
					  bool enable);
141 
 
142 
/* get temperature in millidegrees */
143 
int ci_get_temp(struct radeon_device *rdev)
144 
{
145 
	u32 temp;
146 
	int actual_temp = 0;
147 
 
148 
	temp = (RREG32_SMC(CG_MULT_THERMAL_STATUS) & CTF_TEMP_MASK) >>
149 
		CTF_TEMP_SHIFT;
150 
 
151 
	if (temp & 0x200)
152 
		actual_temp = 255;
153 
	else
154 
		actual_temp = temp & 0x1ff;
155 
 
156 
	actual_temp = actual_temp * 1000;
157 
 
158 
	return actual_temp;
159 
}
160 
 
161 
/* get temperature in millidegrees */
162 
int kv_get_temp(struct radeon_device *rdev)
163 
{
164 
	u32 temp;
165 
	int actual_temp = 0;
166 
 
167 
	temp = RREG32_SMC(0xC0300E0C);
168 
 
169 
	if (temp)
170 
		actual_temp = (temp / 8) - 49;
171 
	else
172 
		actual_temp = 0;
173 
 
174 
	actual_temp = actual_temp * 1000;
175 
 
176 
	return actual_temp;
177 
}
178 
 
179 
/*
180 
 * Indirect registers accessor
181 
 */
182 
u32 cik_pciep_rreg(struct radeon_device *rdev, u32 reg)
183 
{
184 
	unsigned long flags;
185 
	u32 r;
186 
 
187 
	spin_lock_irqsave(&rdev->pciep_idx_lock, flags);
188 
	WREG32(PCIE_INDEX, reg);
189 
	(void)RREG32(PCIE_INDEX);
190 
	r = RREG32(PCIE_DATA);
191 
	spin_unlock_irqrestore(&rdev->pciep_idx_lock, flags);
192 
	return r;
193 
}
194 
 
195 
void cik_pciep_wreg(struct radeon_device *rdev, u32 reg, u32 v)
196 
{
197 
	unsigned long flags;
198 
 
199 
	spin_lock_irqsave(&rdev->pciep_idx_lock, flags);
200 
	WREG32(PCIE_INDEX, reg);
201 
	(void)RREG32(PCIE_INDEX);
202 
	WREG32(PCIE_DATA, v);
203 
	(void)RREG32(PCIE_DATA);
204 
	spin_unlock_irqrestore(&rdev->pciep_idx_lock, flags);
205 
}
206 
 
207 
static const u32 spectre_rlc_save_restore_register_list[] =
208 
{
209 
	(0x0e00 << 16) | (0xc12c >> 2),
210 
	0x00000000,
211 
	(0x0e00 << 16) | (0xc140 >> 2),
212 
	0x00000000,
213 
	(0x0e00 << 16) | (0xc150 >> 2),
214 
	0x00000000,
215 
	(0x0e00 << 16) | (0xc15c >> 2),
216 
	0x00000000,
217 
	(0x0e00 << 16) | (0xc168 >> 2),
218 
	0x00000000,
219 
	(0x0e00 << 16) | (0xc170 >> 2),
220 
	0x00000000,
221 
	(0x0e00 << 16) | (0xc178 >> 2),
222 
	0x00000000,
223 
	(0x0e00 << 16) | (0xc204 >> 2),
224 
	0x00000000,
225 
	(0x0e00 << 16) | (0xc2b4 >> 2),
226 
	0x00000000,
227 
	(0x0e00 << 16) | (0xc2b8 >> 2),
228 
	0x00000000,
229 
	(0x0e00 << 16) | (0xc2bc >> 2),
230 
	0x00000000,
231 
	(0x0e00 << 16) | (0xc2c0 >> 2),
232 
	0x00000000,
233 
	(0x0e00 << 16) | (0x8228 >> 2),
234 
	0x00000000,
235 
	(0x0e00 << 16) | (0x829c >> 2),
236 
	0x00000000,
237 
	(0x0e00 << 16) | (0x869c >> 2),
238 
	0x00000000,
239 
	(0x0600 << 16) | (0x98f4 >> 2),
240 
	0x00000000,
241 
	(0x0e00 << 16) | (0x98f8 >> 2),
242 
	0x00000000,
243 
	(0x0e00 << 16) | (0x9900 >> 2),
244 
	0x00000000,
245 
	(0x0e00 << 16) | (0xc260 >> 2),
246 
	0x00000000,
247 
	(0x0e00 << 16) | (0x90e8 >> 2),
248 
	0x00000000,
249 
	(0x0e00 << 16) | (0x3c000 >> 2),
250 
	0x00000000,
251 
	(0x0e00 << 16) | (0x3c00c >> 2),
252 
	0x00000000,
253 
	(0x0e00 << 16) | (0x8c1c >> 2),
254 
	0x00000000,
255 
	(0x0e00 << 16) | (0x9700 >> 2),
256 
	0x00000000,
257 
	(0x0e00 << 16) | (0xcd20 >> 2),
258 
	0x00000000,
259 
	(0x4e00 << 16) | (0xcd20 >> 2),
260 
	0x00000000,
261 
	(0x5e00 << 16) | (0xcd20 >> 2),
262 
	0x00000000,
263 
	(0x6e00 << 16) | (0xcd20 >> 2),
264 
	0x00000000,
265 
	(0x7e00 << 16) | (0xcd20 >> 2),
266 
	0x00000000,
267 
	(0x8e00 << 16) | (0xcd20 >> 2),
268 
	0x00000000,
269 
	(0x9e00 << 16) | (0xcd20 >> 2),
270 
	0x00000000,
271 
	(0xae00 << 16) | (0xcd20 >> 2),
272 
	0x00000000,
273 
	(0xbe00 << 16) | (0xcd20 >> 2),
274 
	0x00000000,
275 
	(0x0e00 << 16) | (0x89bc >> 2),
276 
	0x00000000,
277 
	(0x0e00 << 16) | (0x8900 >> 2),
278 
	0x00000000,
279 
	0x3,
280 
	(0x0e00 << 16) | (0xc130 >> 2),
281 
	0x00000000,
282 
	(0x0e00 << 16) | (0xc134 >> 2),
283 
	0x00000000,
284 
	(0x0e00 << 16) | (0xc1fc >> 2),
285 
	0x00000000,
286 
	(0x0e00 << 16) | (0xc208 >> 2),
287 
	0x00000000,
288 
	(0x0e00 << 16) | (0xc264 >> 2),
289 
	0x00000000,
290 
	(0x0e00 << 16) | (0xc268 >> 2),
291 
	0x00000000,
292 
	(0x0e00 << 16) | (0xc26c >> 2),
293 
	0x00000000,
294 
	(0x0e00 << 16) | (0xc270 >> 2),
295 
	0x00000000,
296 
	(0x0e00 << 16) | (0xc274 >> 2),
297 
	0x00000000,
298 
	(0x0e00 << 16) | (0xc278 >> 2),
299 
	0x00000000,
300 
	(0x0e00 << 16) | (0xc27c >> 2),
301 
	0x00000000,
302 
	(0x0e00 << 16) | (0xc280 >> 2),
303 
	0x00000000,
304 
	(0x0e00 << 16) | (0xc284 >> 2),
305 
	0x00000000,
306 
	(0x0e00 << 16) | (0xc288 >> 2),
307 
	0x00000000,
308 
	(0x0e00 << 16) | (0xc28c >> 2),
309 
	0x00000000,
310 
	(0x0e00 << 16) | (0xc290 >> 2),
311 
	0x00000000,
312 
	(0x0e00 << 16) | (0xc294 >> 2),
313 
	0x00000000,
314 
	(0x0e00 << 16) | (0xc298 >> 2),
315 
	0x00000000,
316 
	(0x0e00 << 16) | (0xc29c >> 2),
317 
	0x00000000,
318 
	(0x0e00 << 16) | (0xc2a0 >> 2),
319 
	0x00000000,
320 
	(0x0e00 << 16) | (0xc2a4 >> 2),
321 
	0x00000000,
322 
	(0x0e00 << 16) | (0xc2a8 >> 2),
323 
	0x00000000,
324 
	(0x0e00 << 16) | (0xc2ac  >> 2),
325 
	0x00000000,
326 
	(0x0e00 << 16) | (0xc2b0 >> 2),
327 
	0x00000000,
328 
	(0x0e00 << 16) | (0x301d0 >> 2),
329 
	0x00000000,
330 
	(0x0e00 << 16) | (0x30238 >> 2),
331 
	0x00000000,
332 
	(0x0e00 << 16) | (0x30250 >> 2),
333 
	0x00000000,
334 
	(0x0e00 << 16) | (0x30254 >> 2),
335 
	0x00000000,
336 
	(0x0e00 << 16) | (0x30258 >> 2),
337 
	0x00000000,
338 
	(0x0e00 << 16) | (0x3025c >> 2),
339 
	0x00000000,
340 
	(0x4e00 << 16) | (0xc900 >> 2),
341 
	0x00000000,
342 
	(0x5e00 << 16) | (0xc900 >> 2),
343 
	0x00000000,
344 
	(0x6e00 << 16) | (0xc900 >> 2),
345 
	0x00000000,
346 
	(0x7e00 << 16) | (0xc900 >> 2),
347 
	0x00000000,
348 
	(0x8e00 << 16) | (0xc900 >> 2),
349 
	0x00000000,
350 
	(0x9e00 << 16) | (0xc900 >> 2),
351 
	0x00000000,
352 
	(0xae00 << 16) | (0xc900 >> 2),
353 
	0x00000000,
354 
	(0xbe00 << 16) | (0xc900 >> 2),
355 
	0x00000000,
356 
	(0x4e00 << 16) | (0xc904 >> 2),
357 
	0x00000000,
358 
	(0x5e00 << 16) | (0xc904 >> 2),
359 
	0x00000000,
360 
	(0x6e00 << 16) | (0xc904 >> 2),
361 
	0x00000000,
362 
	(0x7e00 << 16) | (0xc904 >> 2),
363 
	0x00000000,
364 
	(0x8e00 << 16) | (0xc904 >> 2),
365 
	0x00000000,
366 
	(0x9e00 << 16) | (0xc904 >> 2),
367 
	0x00000000,
368 
	(0xae00 << 16) | (0xc904 >> 2),
369 
	0x00000000,
370 
	(0xbe00 << 16) | (0xc904 >> 2),
371 
	0x00000000,
372 
	(0x4e00 << 16) | (0xc908 >> 2),
373 
	0x00000000,
374 
	(0x5e00 << 16) | (0xc908 >> 2),
375 
	0x00000000,
376 
	(0x6e00 << 16) | (0xc908 >> 2),
377 
	0x00000000,
378 
	(0x7e00 << 16) | (0xc908 >> 2),
379 
	0x00000000,
380 
	(0x8e00 << 16) | (0xc908 >> 2),
381 
	0x00000000,
382 
	(0x9e00 << 16) | (0xc908 >> 2),
383 
	0x00000000,
384 
	(0xae00 << 16) | (0xc908 >> 2),
385 
	0x00000000,
386 
	(0xbe00 << 16) | (0xc908 >> 2),
387 
	0x00000000,
388 
	(0x4e00 << 16) | (0xc90c >> 2),
389 
	0x00000000,
390 
	(0x5e00 << 16) | (0xc90c >> 2),
391 
	0x00000000,
392 
	(0x6e00 << 16) | (0xc90c >> 2),
393 
	0x00000000,
394 
	(0x7e00 << 16) | (0xc90c >> 2),
395 
	0x00000000,
396 
	(0x8e00 << 16) | (0xc90c >> 2),
397 
	0x00000000,
398 
	(0x9e00 << 16) | (0xc90c >> 2),
399 
	0x00000000,
400 
	(0xae00 << 16) | (0xc90c >> 2),
401 
	0x00000000,
402 
	(0xbe00 << 16) | (0xc90c >> 2),
403 
	0x00000000,
404 
	(0x4e00 << 16) | (0xc910 >> 2),
405 
	0x00000000,
406 
	(0x5e00 << 16) | (0xc910 >> 2),
407 
	0x00000000,
408 
	(0x6e00 << 16) | (0xc910 >> 2),
409 
	0x00000000,
410 
	(0x7e00 << 16) | (0xc910 >> 2),
411 
	0x00000000,
412 
	(0x8e00 << 16) | (0xc910 >> 2),
413 
	0x00000000,
414 
	(0x9e00 << 16) | (0xc910 >> 2),
415 
	0x00000000,
416 
	(0xae00 << 16) | (0xc910 >> 2),
417 
	0x00000000,
418 
	(0xbe00 << 16) | (0xc910 >> 2),
419 
	0x00000000,
420 
	(0x0e00 << 16) | (0xc99c >> 2),
421 
	0x00000000,
422 
	(0x0e00 << 16) | (0x9834 >> 2),
423 
	0x00000000,
424 
	(0x0000 << 16) | (0x30f00 >> 2),
425 
	0x00000000,
426 
	(0x0001 << 16) | (0x30f00 >> 2),
427 
	0x00000000,
428 
	(0x0000 << 16) | (0x30f04 >> 2),
429 
	0x00000000,
430 
	(0x0001 << 16) | (0x30f04 >> 2),
431 
	0x00000000,
432 
	(0x0000 << 16) | (0x30f08 >> 2),
433 
	0x00000000,
434 
	(0x0001 << 16) | (0x30f08 >> 2),
435 
	0x00000000,
436 
	(0x0000 << 16) | (0x30f0c >> 2),
437 
	0x00000000,
438 
	(0x0001 << 16) | (0x30f0c >> 2),
439 
	0x00000000,
440 
	(0x0600 << 16) | (0x9b7c >> 2),
441 
	0x00000000,
442 
	(0x0e00 << 16) | (0x8a14 >> 2),
443 
	0x00000000,
444 
	(0x0e00 << 16) | (0x8a18 >> 2),
445 
	0x00000000,
446 
	(0x0600 << 16) | (0x30a00 >> 2),
447 
	0x00000000,
448 
	(0x0e00 << 16) | (0x8bf0 >> 2),
449 
	0x00000000,
450 
	(0x0e00 << 16) | (0x8bcc >> 2),
451 
	0x00000000,
452 
	(0x0e00 << 16) | (0x8b24 >> 2),
453 
	0x00000000,
454 
	(0x0e00 << 16) | (0x30a04 >> 2),
455 
	0x00000000,
456 
	(0x0600 << 16) | (0x30a10 >> 2),
457 
	0x00000000,
458 
	(0x0600 << 16) | (0x30a14 >> 2),
459 
	0x00000000,
460 
	(0x0600 << 16) | (0x30a18 >> 2),
461 
	0x00000000,
462 
	(0x0600 << 16) | (0x30a2c >> 2),
463 
	0x00000000,
464 
	(0x0e00 << 16) | (0xc700 >> 2),
465 
	0x00000000,
466 
	(0x0e00 << 16) | (0xc704 >> 2),
467 
	0x00000000,
468 
	(0x0e00 << 16) | (0xc708 >> 2),
469 
	0x00000000,
470 
	(0x0e00 << 16) | (0xc768 >> 2),
471 
	0x00000000,
472 
	(0x0400 << 16) | (0xc770 >> 2),
473 
	0x00000000,
474 
	(0x0400 << 16) | (0xc774 >> 2),
475 
	0x00000000,
476 
	(0x0400 << 16) | (0xc778 >> 2),
477 
	0x00000000,
478 
	(0x0400 << 16) | (0xc77c >> 2),
479 
	0x00000000,
480 
	(0x0400 << 16) | (0xc780 >> 2),
481 
	0x00000000,
482 
	(0x0400 << 16) | (0xc784 >> 2),
483 
	0x00000000,
484 
	(0x0400 << 16) | (0xc788 >> 2),
485 
	0x00000000,
486 
	(0x0400 << 16) | (0xc78c >> 2),
487 
	0x00000000,
488 
	(0x0400 << 16) | (0xc798 >> 2),
489 
	0x00000000,
490 
	(0x0400 << 16) | (0xc79c >> 2),
491 
	0x00000000,
492 
	(0x0400 << 16) | (0xc7a0 >> 2),
493 
	0x00000000,
494 
	(0x0400 << 16) | (0xc7a4 >> 2),
495 
	0x00000000,
496 
	(0x0400 << 16) | (0xc7a8 >> 2),
497 
	0x00000000,
498 
	(0x0400 << 16) | (0xc7ac >> 2),
499 
	0x00000000,
500 
	(0x0400 << 16) | (0xc7b0 >> 2),
501 
	0x00000000,
502 
	(0x0400 << 16) | (0xc7b4 >> 2),
503 
	0x00000000,
504 
	(0x0e00 << 16) | (0x9100 >> 2),
505 
	0x00000000,
506 
	(0x0e00 << 16) | (0x3c010 >> 2),
507 
	0x00000000,
508 
	(0x0e00 << 16) | (0x92a8 >> 2),
509 
	0x00000000,
510 
	(0x0e00 << 16) | (0x92ac >> 2),
511 
	0x00000000,
512 
	(0x0e00 << 16) | (0x92b4 >> 2),
513 
	0x00000000,
514 
	(0x0e00 << 16) | (0x92b8 >> 2),
515 
	0x00000000,
516 
	(0x0e00 << 16) | (0x92bc >> 2),
517 
	0x00000000,
518 
	(0x0e00 << 16) | (0x92c0 >> 2),
519 
	0x00000000,
520 
	(0x0e00 << 16) | (0x92c4 >> 2),
521 
	0x00000000,
522 
	(0x0e00 << 16) | (0x92c8 >> 2),
523 
	0x00000000,
524 
	(0x0e00 << 16) | (0x92cc >> 2),
525 
	0x00000000,
526 
	(0x0e00 << 16) | (0x92d0 >> 2),
527 
	0x00000000,
528 
	(0x0e00 << 16) | (0x8c00 >> 2),
529 
	0x00000000,
530 
	(0x0e00 << 16) | (0x8c04 >> 2),
531 
	0x00000000,
532 
	(0x0e00 << 16) | (0x8c20 >> 2),
533 
	0x00000000,
534 
	(0x0e00 << 16) | (0x8c38 >> 2),
535 
	0x00000000,
536 
	(0x0e00 << 16) | (0x8c3c >> 2),
537 
	0x00000000,
538 
	(0x0e00 << 16) | (0xae00 >> 2),
539 
	0x00000000,
540 
	(0x0e00 << 16) | (0x9604 >> 2),
541 
	0x00000000,
542 
	(0x0e00 << 16) | (0xac08 >> 2),
543 
	0x00000000,
544 
	(0x0e00 << 16) | (0xac0c >> 2),
545 
	0x00000000,
546 
	(0x0e00 << 16) | (0xac10 >> 2),
547 
	0x00000000,
548 
	(0x0e00 << 16) | (0xac14 >> 2),
549 
	0x00000000,
550 
	(0x0e00 << 16) | (0xac58 >> 2),
551 
	0x00000000,
552 
	(0x0e00 << 16) | (0xac68 >> 2),
553 
	0x00000000,
554 
	(0x0e00 << 16) | (0xac6c >> 2),
555 
	0x00000000,
556 
	(0x0e00 << 16) | (0xac70 >> 2),
557 
	0x00000000,
558 
	(0x0e00 << 16) | (0xac74 >> 2),
559 
	0x00000000,
560 
	(0x0e00 << 16) | (0xac78 >> 2),
561 
	0x00000000,
562 
	(0x0e00 << 16) | (0xac7c >> 2),
563 
	0x00000000,
564 
	(0x0e00 << 16) | (0xac80 >> 2),
565 
	0x00000000,
566 
	(0x0e00 << 16) | (0xac84 >> 2),
567 
	0x00000000,
568 
	(0x0e00 << 16) | (0xac88 >> 2),
569 
	0x00000000,
570 
	(0x0e00 << 16) | (0xac8c >> 2),
571 
	0x00000000,
572 
	(0x0e00 << 16) | (0x970c >> 2),
573 
	0x00000000,
574 
	(0x0e00 << 16) | (0x9714 >> 2),
575 
	0x00000000,
576 
	(0x0e00 << 16) | (0x9718 >> 2),
577 
	0x00000000,
578 
	(0x0e00 << 16) | (0x971c >> 2),
579 
	0x00000000,
580 
	(0x0e00 << 16) | (0x31068 >> 2),
581 
	0x00000000,
582 
	(0x4e00 << 16) | (0x31068 >> 2),
583 
	0x00000000,
584 
	(0x5e00 << 16) | (0x31068 >> 2),
585 
	0x00000000,
586 
	(0x6e00 << 16) | (0x31068 >> 2),
587 
	0x00000000,
588 
	(0x7e00 << 16) | (0x31068 >> 2),
589 
	0x00000000,
590 
	(0x8e00 << 16) | (0x31068 >> 2),
591 
	0x00000000,
592 
	(0x9e00 << 16) | (0x31068 >> 2),
593 
	0x00000000,
594 
	(0xae00 << 16) | (0x31068 >> 2),
595 
	0x00000000,
596 
	(0xbe00 << 16) | (0x31068 >> 2),
597 
	0x00000000,
598 
	(0x0e00 << 16) | (0xcd10 >> 2),
599 
	0x00000000,
600 
	(0x0e00 << 16) | (0xcd14 >> 2),
601 
	0x00000000,
602 
	(0x0e00 << 16) | (0x88b0 >> 2),
603 
	0x00000000,
604 
	(0x0e00 << 16) | (0x88b4 >> 2),
605 
	0x00000000,
606 
	(0x0e00 << 16) | (0x88b8 >> 2),
607 
	0x00000000,
608 
	(0x0e00 << 16) | (0x88bc >> 2),
609 
	0x00000000,
610 
	(0x0400 << 16) | (0x89c0 >> 2),
611 
	0x00000000,
612 
	(0x0e00 << 16) | (0x88c4 >> 2),
613 
	0x00000000,
614 
	(0x0e00 << 16) | (0x88c8 >> 2),
615 
	0x00000000,
616 
	(0x0e00 << 16) | (0x88d0 >> 2),
617 
	0x00000000,
618 
	(0x0e00 << 16) | (0x88d4 >> 2),
619 
	0x00000000,
620 
	(0x0e00 << 16) | (0x88d8 >> 2),
621 
	0x00000000,
622 
	(0x0e00 << 16) | (0x8980 >> 2),
623 
	0x00000000,
624 
	(0x0e00 << 16) | (0x30938 >> 2),
625 
	0x00000000,
626 
	(0x0e00 << 16) | (0x3093c >> 2),
627 
	0x00000000,
628 
	(0x0e00 << 16) | (0x30940 >> 2),
629 
	0x00000000,
630 
	(0x0e00 << 16) | (0x89a0 >> 2),
631 
	0x00000000,
632 
	(0x0e00 << 16) | (0x30900 >> 2),
633 
	0x00000000,
634 
	(0x0e00 << 16) | (0x30904 >> 2),
635 
	0x00000000,
636 
	(0x0e00 << 16) | (0x89b4 >> 2),
637 
	0x00000000,
638 
	(0x0e00 << 16) | (0x3c210 >> 2),
639 
	0x00000000,
640 
	(0x0e00 << 16) | (0x3c214 >> 2),
641 
	0x00000000,
642 
	(0x0e00 << 16) | (0x3c218 >> 2),
643 
	0x00000000,
644 
	(0x0e00 << 16) | (0x8904 >> 2),
645 
	0x00000000,
646 
	0x5,
647 
	(0x0e00 << 16) | (0x8c28 >> 2),
648 
	(0x0e00 << 16) | (0x8c2c >> 2),
649 
	(0x0e00 << 16) | (0x8c30 >> 2),
650 
	(0x0e00 << 16) | (0x8c34 >> 2),
651 
	(0x0e00 << 16) | (0x9600 >> 2),
652 
};
653 
 
654 
static const u32 kalindi_rlc_save_restore_register_list[] =
655 
{
656 
	(0x0e00 << 16) | (0xc12c >> 2),
657 
	0x00000000,
658 
	(0x0e00 << 16) | (0xc140 >> 2),
659 
	0x00000000,
660 
	(0x0e00 << 16) | (0xc150 >> 2),
661 
	0x00000000,
662 
	(0x0e00 << 16) | (0xc15c >> 2),
663 
	0x00000000,
664 
	(0x0e00 << 16) | (0xc168 >> 2),
665 
	0x00000000,
666 
	(0x0e00 << 16) | (0xc170 >> 2),
667 
	0x00000000,
668 
	(0x0e00 << 16) | (0xc204 >> 2),
669 
	0x00000000,
670 
	(0x0e00 << 16) | (0xc2b4 >> 2),
671 
	0x00000000,
672 
	(0x0e00 << 16) | (0xc2b8 >> 2),
673 
	0x00000000,
674 
	(0x0e00 << 16) | (0xc2bc >> 2),
675 
	0x00000000,
676 
	(0x0e00 << 16) | (0xc2c0 >> 2),
677 
	0x00000000,
678 
	(0x0e00 << 16) | (0x8228 >> 2),
679 
	0x00000000,
680 
	(0x0e00 << 16) | (0x829c >> 2),
681 
	0x00000000,
682 
	(0x0e00 << 16) | (0x869c >> 2),
683 
	0x00000000,
684 
	(0x0600 << 16) | (0x98f4 >> 2),
685 
	0x00000000,
686 
	(0x0e00 << 16) | (0x98f8 >> 2),
687 
	0x00000000,
688 
	(0x0e00 << 16) | (0x9900 >> 2),
689 
	0x00000000,
690 
	(0x0e00 << 16) | (0xc260 >> 2),
691 
	0x00000000,
692 
	(0x0e00 << 16) | (0x90e8 >> 2),
693 
	0x00000000,
694 
	(0x0e00 << 16) | (0x3c000 >> 2),
695 
	0x00000000,
696 
	(0x0e00 << 16) | (0x3c00c >> 2),
697 
	0x00000000,
698 
	(0x0e00 << 16) | (0x8c1c >> 2),
699 
	0x00000000,
700 
	(0x0e00 << 16) | (0x9700 >> 2),
701 
	0x00000000,
702 
	(0x0e00 << 16) | (0xcd20 >> 2),
703 
	0x00000000,
704 
	(0x4e00 << 16) | (0xcd20 >> 2),
705 
	0x00000000,
706 
	(0x5e00 << 16) | (0xcd20 >> 2),
707 
	0x00000000,
708 
	(0x6e00 << 16) | (0xcd20 >> 2),
709 
	0x00000000,
710 
	(0x7e00 << 16) | (0xcd20 >> 2),
711 
	0x00000000,
712 
	(0x0e00 << 16) | (0x89bc >> 2),
713 
	0x00000000,
714 
	(0x0e00 << 16) | (0x8900 >> 2),
715 
	0x00000000,
716 
	0x3,
717 
	(0x0e00 << 16) | (0xc130 >> 2),
718 
	0x00000000,
719 
	(0x0e00 << 16) | (0xc134 >> 2),
720 
	0x00000000,
721 
	(0x0e00 << 16) | (0xc1fc >> 2),
722 
	0x00000000,
723 
	(0x0e00 << 16) | (0xc208 >> 2),
724 
	0x00000000,
725 
	(0x0e00 << 16) | (0xc264 >> 2),
726 
	0x00000000,
727 
	(0x0e00 << 16) | (0xc268 >> 2),
728 
	0x00000000,
729 
	(0x0e00 << 16) | (0xc26c >> 2),
730 
	0x00000000,
731 
	(0x0e00 << 16) | (0xc270 >> 2),
732 
	0x00000000,
733 
	(0x0e00 << 16) | (0xc274 >> 2),
734 
	0x00000000,
735 
	(0x0e00 << 16) | (0xc28c >> 2),
736 
	0x00000000,
737 
	(0x0e00 << 16) | (0xc290 >> 2),
738 
	0x00000000,
739 
	(0x0e00 << 16) | (0xc294 >> 2),
740 
	0x00000000,
741 
	(0x0e00 << 16) | (0xc298 >> 2),
742 
	0x00000000,
743 
	(0x0e00 << 16) | (0xc2a0 >> 2),
744 
	0x00000000,
745 
	(0x0e00 << 16) | (0xc2a4 >> 2),
746 
	0x00000000,
747 
	(0x0e00 << 16) | (0xc2a8 >> 2),
748 
	0x00000000,
749 
	(0x0e00 << 16) | (0xc2ac >> 2),
750 
	0x00000000,
751 
	(0x0e00 << 16) | (0x301d0 >> 2),
752 
	0x00000000,
753 
	(0x0e00 << 16) | (0x30238 >> 2),
754 
	0x00000000,
755 
	(0x0e00 << 16) | (0x30250 >> 2),
756 
	0x00000000,
757 
	(0x0e00 << 16) | (0x30254 >> 2),
758 
	0x00000000,
759 
	(0x0e00 << 16) | (0x30258 >> 2),
760 
	0x00000000,
761 
	(0x0e00 << 16) | (0x3025c >> 2),
762 
	0x00000000,
763 
	(0x4e00 << 16) | (0xc900 >> 2),
764 
	0x00000000,
765 
	(0x5e00 << 16) | (0xc900 >> 2),
766 
	0x00000000,
767 
	(0x6e00 << 16) | (0xc900 >> 2),
768 
	0x00000000,
769 
	(0x7e00 << 16) | (0xc900 >> 2),
770 
	0x00000000,
771 
	(0x4e00 << 16) | (0xc904 >> 2),
772 
	0x00000000,
773 
	(0x5e00 << 16) | (0xc904 >> 2),
774 
	0x00000000,
775 
	(0x6e00 << 16) | (0xc904 >> 2),
776 
	0x00000000,
777 
	(0x7e00 << 16) | (0xc904 >> 2),
778 
	0x00000000,
779 
	(0x4e00 << 16) | (0xc908 >> 2),
780 
	0x00000000,
781 
	(0x5e00 << 16) | (0xc908 >> 2),
782 
	0x00000000,
783 
	(0x6e00 << 16) | (0xc908 >> 2),
784 
	0x00000000,
785 
	(0x7e00 << 16) | (0xc908 >> 2),
786 
	0x00000000,
787 
	(0x4e00 << 16) | (0xc90c >> 2),
788 
	0x00000000,
789 
	(0x5e00 << 16) | (0xc90c >> 2),
790 
	0x00000000,
791 
	(0x6e00 << 16) | (0xc90c >> 2),
792 
	0x00000000,
793 
	(0x7e00 << 16) | (0xc90c >> 2),
794 
	0x00000000,
795 
	(0x4e00 << 16) | (0xc910 >> 2),
796 
	0x00000000,
797 
	(0x5e00 << 16) | (0xc910 >> 2),
798 
	0x00000000,
799 
	(0x6e00 << 16) | (0xc910 >> 2),
800 
	0x00000000,
801 
	(0x7e00 << 16) | (0xc910 >> 2),
802 
	0x00000000,
803 
	(0x0e00 << 16) | (0xc99c >> 2),
804 
	0x00000000,
805 
	(0x0e00 << 16) | (0x9834 >> 2),
806 
	0x00000000,
807 
	(0x0000 << 16) | (0x30f00 >> 2),
808 
	0x00000000,
809 
	(0x0000 << 16) | (0x30f04 >> 2),
810 
	0x00000000,
811 
	(0x0000 << 16) | (0x30f08 >> 2),
812 
	0x00000000,
813 
	(0x0000 << 16) | (0x30f0c >> 2),
814 
	0x00000000,
815 
	(0x0600 << 16) | (0x9b7c >> 2),
816 
	0x00000000,
817 
	(0x0e00 << 16) | (0x8a14 >> 2),
818 
	0x00000000,
819 
	(0x0e00 << 16) | (0x8a18 >> 2),
820 
	0x00000000,
821 
	(0x0600 << 16) | (0x30a00 >> 2),
822 
	0x00000000,
823 
	(0x0e00 << 16) | (0x8bf0 >> 2),
824 
	0x00000000,
825 
	(0x0e00 << 16) | (0x8bcc >> 2),
826 
	0x00000000,
827 
	(0x0e00 << 16) | (0x8b24 >> 2),
828 
	0x00000000,
829 
	(0x0e00 << 16) | (0x30a04 >> 2),
830 
	0x00000000,
831 
	(0x0600 << 16) | (0x30a10 >> 2),
832 
	0x00000000,
833 
	(0x0600 << 16) | (0x30a14 >> 2),
834 
	0x00000000,
835 
	(0x0600 << 16) | (0x30a18 >> 2),
836 
	0x00000000,
837 
	(0x0600 << 16) | (0x30a2c >> 2),
838 
	0x00000000,
839 
	(0x0e00 << 16) | (0xc700 >> 2),
840 
	0x00000000,
841 
	(0x0e00 << 16) | (0xc704 >> 2),
842 
	0x00000000,
843 
	(0x0e00 << 16) | (0xc708 >> 2),
844 
	0x00000000,
845 
	(0x0e00 << 16) | (0xc768 >> 2),
846 
	0x00000000,
847 
	(0x0400 << 16) | (0xc770 >> 2),
848 
	0x00000000,
849 
	(0x0400 << 16) | (0xc774 >> 2),
850 
	0x00000000,
851 
	(0x0400 << 16) | (0xc798 >> 2),
852 
	0x00000000,
853 
	(0x0400 << 16) | (0xc79c >> 2),
854 
	0x00000000,
855 
	(0x0e00 << 16) | (0x9100 >> 2),
856 
	0x00000000,
857 
	(0x0e00 << 16) | (0x3c010 >> 2),
858 
	0x00000000,
859 
	(0x0e00 << 16) | (0x8c00 >> 2),
860 
	0x00000000,
861 
	(0x0e00 << 16) | (0x8c04 >> 2),
862 
	0x00000000,
863 
	(0x0e00 << 16) | (0x8c20 >> 2),
864 
	0x00000000,
865 
	(0x0e00 << 16) | (0x8c38 >> 2),
866 
	0x00000000,
867 
	(0x0e00 << 16) | (0x8c3c >> 2),
868 
	0x00000000,
869 
	(0x0e00 << 16) | (0xae00 >> 2),
870 
	0x00000000,
871 
	(0x0e00 << 16) | (0x9604 >> 2),
872 
	0x00000000,
873 
	(0x0e00 << 16) | (0xac08 >> 2),
874 
	0x00000000,
875 
	(0x0e00 << 16) | (0xac0c >> 2),
876 
	0x00000000,
877 
	(0x0e00 << 16) | (0xac10 >> 2),
878 
	0x00000000,
879 
	(0x0e00 << 16) | (0xac14 >> 2),
880 
	0x00000000,
881 
	(0x0e00 << 16) | (0xac58 >> 2),
882 
	0x00000000,
883 
	(0x0e00 << 16) | (0xac68 >> 2),
884 
	0x00000000,
885 
	(0x0e00 << 16) | (0xac6c >> 2),
886 
	0x00000000,
887 
	(0x0e00 << 16) | (0xac70 >> 2),
888 
	0x00000000,
889 
	(0x0e00 << 16) | (0xac74 >> 2),
890 
	0x00000000,
891 
	(0x0e00 << 16) | (0xac78 >> 2),
892 
	0x00000000,
893 
	(0x0e00 << 16) | (0xac7c >> 2),
894 
	0x00000000,
895 
	(0x0e00 << 16) | (0xac80 >> 2),
896 
	0x00000000,
897 
	(0x0e00 << 16) | (0xac84 >> 2),
898 
	0x00000000,
899 
	(0x0e00 << 16) | (0xac88 >> 2),
900 
	0x00000000,
901 
	(0x0e00 << 16) | (0xac8c >> 2),
902 
	0x00000000,
903 
	(0x0e00 << 16) | (0x970c >> 2),
904 
	0x00000000,
905 
	(0x0e00 << 16) | (0x9714 >> 2),
906 
	0x00000000,
907 
	(0x0e00 << 16) | (0x9718 >> 2),
908 
	0x00000000,
909 
	(0x0e00 << 16) | (0x971c >> 2),
910 
	0x00000000,
911 
	(0x0e00 << 16) | (0x31068 >> 2),
912 
	0x00000000,
913 
	(0x4e00 << 16) | (0x31068 >> 2),
914 
	0x00000000,
915 
	(0x5e00 << 16) | (0x31068 >> 2),
916 
	0x00000000,
917 
	(0x6e00 << 16) | (0x31068 >> 2),
918 
	0x00000000,
919 
	(0x7e00 << 16) | (0x31068 >> 2),
920 
	0x00000000,
921 
	(0x0e00 << 16) | (0xcd10 >> 2),
922 
	0x00000000,
923 
	(0x0e00 << 16) | (0xcd14 >> 2),
924 
	0x00000000,
925 
	(0x0e00 << 16) | (0x88b0 >> 2),
926 
	0x00000000,
927 
	(0x0e00 << 16) | (0x88b4 >> 2),
928 
	0x00000000,
929 
	(0x0e00 << 16) | (0x88b8 >> 2),
930 
	0x00000000,
931 
	(0x0e00 << 16) | (0x88bc >> 2),
932 
	0x00000000,
933 
	(0x0400 << 16) | (0x89c0 >> 2),
934 
	0x00000000,
935 
	(0x0e00 << 16) | (0x88c4 >> 2),
936 
	0x00000000,
937 
	(0x0e00 << 16) | (0x88c8 >> 2),
938 
	0x00000000,
939 
	(0x0e00 << 16) | (0x88d0 >> 2),
940 
	0x00000000,
941 
	(0x0e00 << 16) | (0x88d4 >> 2),
942 
	0x00000000,
943 
	(0x0e00 << 16) | (0x88d8 >> 2),
944 
	0x00000000,
945 
	(0x0e00 << 16) | (0x8980 >> 2),
946 
	0x00000000,
947 
	(0x0e00 << 16) | (0x30938 >> 2),
948 
	0x00000000,
949 
	(0x0e00 << 16) | (0x3093c >> 2),
950 
	0x00000000,
951 
	(0x0e00 << 16) | (0x30940 >> 2),
952 
	0x00000000,
953 
	(0x0e00 << 16) | (0x89a0 >> 2),
954 
	0x00000000,
955 
	(0x0e00 << 16) | (0x30900 >> 2),
956 
	0x00000000,
957 
	(0x0e00 << 16) | (0x30904 >> 2),
958 
	0x00000000,
959 
	(0x0e00 << 16) | (0x89b4 >> 2),
960 
	0x00000000,
961 
	(0x0e00 << 16) | (0x3e1fc >> 2),
962 
	0x00000000,
963 
	(0x0e00 << 16) | (0x3c210 >> 2),
964 
	0x00000000,
965 
	(0x0e00 << 16) | (0x3c214 >> 2),
966 
	0x00000000,
967 
	(0x0e00 << 16) | (0x3c218 >> 2),
968 
	0x00000000,
969 
	(0x0e00 << 16) | (0x8904 >> 2),
970 
	0x00000000,
971 
	0x5,
972 
	(0x0e00 << 16) | (0x8c28 >> 2),
973 
	(0x0e00 << 16) | (0x8c2c >> 2),
974 
	(0x0e00 << 16) | (0x8c30 >> 2),
975 
	(0x0e00 << 16) | (0x8c34 >> 2),
976 
	(0x0e00 << 16) | (0x9600 >> 2),
977 
};
978 
 
979 
static const u32 bonaire_golden_spm_registers[] =
980 
{
981 
	0x30800, 0xe0ffffff, 0xe0000000
982 
};
983 
 
984 
static const u32 bonaire_golden_common_registers[] =
985 
{
986 
	0xc770, 0xffffffff, 0x00000800,
987 
	0xc774, 0xffffffff, 0x00000800,
988 
	0xc798, 0xffffffff, 0x00007fbf,
989 
	0xc79c, 0xffffffff, 0x00007faf
990 
};
991 
 
992 
static const u32 bonaire_golden_registers[] =
993 
{
994 
	0x3354, 0x00000333, 0x00000333,
995 
	0x3350, 0x000c0fc0, 0x00040200,
996 
	0x9a10, 0x00010000, 0x00058208,
997 
	0x3c000, 0xffff1fff, 0x00140000,
998 
	0x3c200, 0xfdfc0fff, 0x00000100,
999 
	0x3c234, 0x40000000, 0x40000200,
1000 
	0x9830, 0xffffffff, 0x00000000,
1001 
	0x9834, 0xf00fffff, 0x00000400,
1002 
	0x9838, 0x0002021c, 0x00020200,
1003 
	0xc78, 0x00000080, 0x00000000,
1004 
	0x5bb0, 0x000000f0, 0x00000070,
1005 
	0x5bc0, 0xf0311fff, 0x80300000,
1006 
	0x98f8, 0x73773777, 0x12010001,
1007 
	0x350c, 0x00810000, 0x408af000,
1008 
	0x7030, 0x31000111, 0x00000011,
1009 
	0x2f48, 0x73773777, 0x12010001,
1010 
	0x220c, 0x00007fb6, 0x0021a1b1,
1011 
	0x2210, 0x00007fb6, 0x002021b1,
1012 
	0x2180, 0x00007fb6, 0x00002191,
1013 
	0x2218, 0x00007fb6, 0x002121b1,
1014 
	0x221c, 0x00007fb6, 0x002021b1,
1015 
	0x21dc, 0x00007fb6, 0x00002191,
1016 
	0x21e0, 0x00007fb6, 0x00002191,
1017 
	0x3628, 0x0000003f, 0x0000000a,
1018 
	0x362c, 0x0000003f, 0x0000000a,
1019 
	0x2ae4, 0x00073ffe, 0x000022a2,
1020 
	0x240c, 0x000007ff, 0x00000000,
1021 
	0x8a14, 0xf000003f, 0x00000007,
1022 
	0x8bf0, 0x00002001, 0x00000001,
1023 
	0x8b24, 0xffffffff, 0x00ffffff,
1024 
	0x30a04, 0x0000ff0f, 0x00000000,
1025 
	0x28a4c, 0x07ffffff, 0x06000000,
1026 
	0x4d8, 0x00000fff, 0x00000100,
1027 
	0x3e78, 0x00000001, 0x00000002,
1028 
	0x9100, 0x03000000, 0x0362c688,
1029 
	0x8c00, 0x000000ff, 0x00000001,
1030 
	0xe40, 0x00001fff, 0x00001fff,
1031 
	0x9060, 0x0000007f, 0x00000020,
1032 
	0x9508, 0x00010000, 0x00010000,
1033 
	0xac14, 0x000003ff, 0x000000f3,
1034 
	0xac0c, 0xffffffff, 0x00001032
1035 
};
1036 
 
1037 
static const u32 bonaire_mgcg_cgcg_init[] =
1038 
{
1039 
	0xc420, 0xffffffff, 0xfffffffc,
1040 
	0x30800, 0xffffffff, 0xe0000000,
1041 
	0x3c2a0, 0xffffffff, 0x00000100,
1042 
	0x3c208, 0xffffffff, 0x00000100,
1043 
	0x3c2c0, 0xffffffff, 0xc0000100,
1044 
	0x3c2c8, 0xffffffff, 0xc0000100,
1045 
	0x3c2c4, 0xffffffff, 0xc0000100,
1046 
	0x55e4, 0xffffffff, 0x00600100,
1047 
	0x3c280, 0xffffffff, 0x00000100,
1048 
	0x3c214, 0xffffffff, 0x06000100,
1049 
	0x3c220, 0xffffffff, 0x00000100,
1050 
	0x3c218, 0xffffffff, 0x06000100,
1051 
	0x3c204, 0xffffffff, 0x00000100,
1052 
	0x3c2e0, 0xffffffff, 0x00000100,
1053 
	0x3c224, 0xffffffff, 0x00000100,
1054 
	0x3c200, 0xffffffff, 0x00000100,
1055 
	0x3c230, 0xffffffff, 0x00000100,
1056 
	0x3c234, 0xffffffff, 0x00000100,
1057 
	0x3c250, 0xffffffff, 0x00000100,
1058 
	0x3c254, 0xffffffff, 0x00000100,
1059 
	0x3c258, 0xffffffff, 0x00000100,
1060 
	0x3c25c, 0xffffffff, 0x00000100,
1061 
	0x3c260, 0xffffffff, 0x00000100,
1062 
	0x3c27c, 0xffffffff, 0x00000100,
1063 
	0x3c278, 0xffffffff, 0x00000100,
1064 
	0x3c210, 0xffffffff, 0x06000100,
1065 
	0x3c290, 0xffffffff, 0x00000100,
1066 
	0x3c274, 0xffffffff, 0x00000100,
1067 
	0x3c2b4, 0xffffffff, 0x00000100,
1068 
	0x3c2b0, 0xffffffff, 0x00000100,
1069 
	0x3c270, 0xffffffff, 0x00000100,
1070 
	0x30800, 0xffffffff, 0xe0000000,
1071 
	0x3c020, 0xffffffff, 0x00010000,
1072 
	0x3c024, 0xffffffff, 0x00030002,
1073 
	0x3c028, 0xffffffff, 0x00040007,
1074 
	0x3c02c, 0xffffffff, 0x00060005,
1075 
	0x3c030, 0xffffffff, 0x00090008,
1076 
	0x3c034, 0xffffffff, 0x00010000,
1077 
	0x3c038, 0xffffffff, 0x00030002,
1078 
	0x3c03c, 0xffffffff, 0x00040007,
1079 
	0x3c040, 0xffffffff, 0x00060005,
1080 
	0x3c044, 0xffffffff, 0x00090008,
1081 
	0x3c048, 0xffffffff, 0x00010000,
1082 
	0x3c04c, 0xffffffff, 0x00030002,
1083 
	0x3c050, 0xffffffff, 0x00040007,
1084 
	0x3c054, 0xffffffff, 0x00060005,
1085 
	0x3c058, 0xffffffff, 0x00090008,
1086 
	0x3c05c, 0xffffffff, 0x00010000,
1087 
	0x3c060, 0xffffffff, 0x00030002,
1088 
	0x3c064, 0xffffffff, 0x00040007,
1089 
	0x3c068, 0xffffffff, 0x00060005,
1090 
	0x3c06c, 0xffffffff, 0x00090008,
1091 
	0x3c070, 0xffffffff, 0x00010000,
1092 
	0x3c074, 0xffffffff, 0x00030002,
1093 
	0x3c078, 0xffffffff, 0x00040007,
1094 
	0x3c07c, 0xffffffff, 0x00060005,
1095 
	0x3c080, 0xffffffff, 0x00090008,
1096 
	0x3c084, 0xffffffff, 0x00010000,
1097 
	0x3c088, 0xffffffff, 0x00030002,
1098 
	0x3c08c, 0xffffffff, 0x00040007,
1099 
	0x3c090, 0xffffffff, 0x00060005,
1100 
	0x3c094, 0xffffffff, 0x00090008,
1101 
	0x3c098, 0xffffffff, 0x00010000,
1102 
	0x3c09c, 0xffffffff, 0x00030002,
1103 
	0x3c0a0, 0xffffffff, 0x00040007,
1104 
	0x3c0a4, 0xffffffff, 0x00060005,
1105 
	0x3c0a8, 0xffffffff, 0x00090008,
1106 
	0x3c000, 0xffffffff, 0x96e00200,
1107 
	0x8708, 0xffffffff, 0x00900100,
1108 
	0xc424, 0xffffffff, 0x0020003f,
1109 
	0x38, 0xffffffff, 0x0140001c,
1110 
	0x3c, 0x000f0000, 0x000f0000,
1111 
	0x220, 0xffffffff, 0xC060000C,
1112 
	0x224, 0xc0000fff, 0x00000100,
1113 
	0xf90, 0xffffffff, 0x00000100,
1114 
	0xf98, 0x00000101, 0x00000000,
1115 
	0x20a8, 0xffffffff, 0x00000104,
1116 
	0x55e4, 0xff000fff, 0x00000100,
1117 
	0x30cc, 0xc0000fff, 0x00000104,
1118 
	0xc1e4, 0x00000001, 0x00000001,
1119 
	0xd00c, 0xff000ff0, 0x00000100,
1120 
	0xd80c, 0xff000ff0, 0x00000100
1121 
};
1122 
 
1123 
static const u32 spectre_golden_spm_registers[] =
1124 
{
1125 
	0x30800, 0xe0ffffff, 0xe0000000
1126 
};
1127 
 
1128 
static const u32 spectre_golden_common_registers[] =
1129 
{
1130 
	0xc770, 0xffffffff, 0x00000800,
1131 
	0xc774, 0xffffffff, 0x00000800,
1132 
	0xc798, 0xffffffff, 0x00007fbf,
1133 
	0xc79c, 0xffffffff, 0x00007faf
1134 
};
1135 
 
1136 
static const u32 spectre_golden_registers[] =
1137 
{
1138 
	0x3c000, 0xffff1fff, 0x96940200,
1139 
	0x3c00c, 0xffff0001, 0xff000000,
1140 
	0x3c200, 0xfffc0fff, 0x00000100,
1141 
	0x6ed8, 0x00010101, 0x00010000,
1142 
	0x9834, 0xf00fffff, 0x00000400,
1143 
	0x9838, 0xfffffffc, 0x00020200,
1144 
	0x5bb0, 0x000000f0, 0x00000070,
1145 
	0x5bc0, 0xf0311fff, 0x80300000,
1146 
	0x98f8, 0x73773777, 0x12010001,
1147 
	0x9b7c, 0x00ff0000, 0x00fc0000,
1148 
	0x2f48, 0x73773777, 0x12010001,
1149 
	0x8a14, 0xf000003f, 0x00000007,
1150 
	0x8b24, 0xffffffff, 0x00ffffff,
1151 
	0x28350, 0x3f3f3fff, 0x00000082,
1152 
	0x28354, 0x0000003f, 0x00000000,
1153 
	0x3e78, 0x00000001, 0x00000002,
1154 
	0x913c, 0xffff03df, 0x00000004,
1155 
	0xc768, 0x00000008, 0x00000008,
1156 
	0x8c00, 0x000008ff, 0x00000800,
1157 
	0x9508, 0x00010000, 0x00010000,
1158 
	0xac0c, 0xffffffff, 0x54763210,
1159 
	0x214f8, 0x01ff01ff, 0x00000002,
1160 
	0x21498, 0x007ff800, 0x00200000,
1161 
	0x2015c, 0xffffffff, 0x00000f40,
1162 
	0x30934, 0xffffffff, 0x00000001
1163 
};
1164 
 
1165 
static const u32 spectre_mgcg_cgcg_init[] =
1166 
{
1167 
	0xc420, 0xffffffff, 0xfffffffc,
1168 
	0x30800, 0xffffffff, 0xe0000000,
1169 
	0x3c2a0, 0xffffffff, 0x00000100,
1170 
	0x3c208, 0xffffffff, 0x00000100,
1171 
	0x3c2c0, 0xffffffff, 0x00000100,
1172 
	0x3c2c8, 0xffffffff, 0x00000100,
1173 
	0x3c2c4, 0xffffffff, 0x00000100,
1174 
	0x55e4, 0xffffffff, 0x00600100,
1175 
	0x3c280, 0xffffffff, 0x00000100,
1176 
	0x3c214, 0xffffffff, 0x06000100,
1177 
	0x3c220, 0xffffffff, 0x00000100,
1178 
	0x3c218, 0xffffffff, 0x06000100,
1179 
	0x3c204, 0xffffffff, 0x00000100,
1180 
	0x3c2e0, 0xffffffff, 0x00000100,
1181 
	0x3c224, 0xffffffff, 0x00000100,
1182 
	0x3c200, 0xffffffff, 0x00000100,
1183 
	0x3c230, 0xffffffff, 0x00000100,
1184 
	0x3c234, 0xffffffff, 0x00000100,
1185 
	0x3c250, 0xffffffff, 0x00000100,
1186 
	0x3c254, 0xffffffff, 0x00000100,
1187 
	0x3c258, 0xffffffff, 0x00000100,
1188 
	0x3c25c, 0xffffffff, 0x00000100,
1189 
	0x3c260, 0xffffffff, 0x00000100,
1190 
	0x3c27c, 0xffffffff, 0x00000100,
1191 
	0x3c278, 0xffffffff, 0x00000100,
1192 
	0x3c210, 0xffffffff, 0x06000100,
1193 
	0x3c290, 0xffffffff, 0x00000100,
1194 
	0x3c274, 0xffffffff, 0x00000100,
1195 
	0x3c2b4, 0xffffffff, 0x00000100,
1196 
	0x3c2b0, 0xffffffff, 0x00000100,
1197 
	0x3c270, 0xffffffff, 0x00000100,
1198 
	0x30800, 0xffffffff, 0xe0000000,
1199 
	0x3c020, 0xffffffff, 0x00010000,
1200 
	0x3c024, 0xffffffff, 0x00030002,
1201 
	0x3c028, 0xffffffff, 0x00040007,
1202 
	0x3c02c, 0xffffffff, 0x00060005,
1203 
	0x3c030, 0xffffffff, 0x00090008,
1204 
	0x3c034, 0xffffffff, 0x00010000,
1205 
	0x3c038, 0xffffffff, 0x00030002,
1206 
	0x3c03c, 0xffffffff, 0x00040007,
1207 
	0x3c040, 0xffffffff, 0x00060005,
1208 
	0x3c044, 0xffffffff, 0x00090008,
1209 
	0x3c048, 0xffffffff, 0x00010000,
1210 
	0x3c04c, 0xffffffff, 0x00030002,
1211 
	0x3c050, 0xffffffff, 0x00040007,
1212 
	0x3c054, 0xffffffff, 0x00060005,
1213 
	0x3c058, 0xffffffff, 0x00090008,
1214 
	0x3c05c, 0xffffffff, 0x00010000,
1215 
	0x3c060, 0xffffffff, 0x00030002,
1216 
	0x3c064, 0xffffffff, 0x00040007,
1217 
	0x3c068, 0xffffffff, 0x00060005,
1218 
	0x3c06c, 0xffffffff, 0x00090008,
1219 
	0x3c070, 0xffffffff, 0x00010000,
1220 
	0x3c074, 0xffffffff, 0x00030002,
1221 
	0x3c078, 0xffffffff, 0x00040007,
1222 
	0x3c07c, 0xffffffff, 0x00060005,
1223 
	0x3c080, 0xffffffff, 0x00090008,
1224 
	0x3c084, 0xffffffff, 0x00010000,
1225 
	0x3c088, 0xffffffff, 0x00030002,
1226 
	0x3c08c, 0xffffffff, 0x00040007,
1227 
	0x3c090, 0xffffffff, 0x00060005,
1228 
	0x3c094, 0xffffffff, 0x00090008,
1229 
	0x3c098, 0xffffffff, 0x00010000,
1230 
	0x3c09c, 0xffffffff, 0x00030002,
1231 
	0x3c0a0, 0xffffffff, 0x00040007,
1232 
	0x3c0a4, 0xffffffff, 0x00060005,
1233 
	0x3c0a8, 0xffffffff, 0x00090008,
1234 
	0x3c0ac, 0xffffffff, 0x00010000,
1235 
	0x3c0b0, 0xffffffff, 0x00030002,
1236 
	0x3c0b4, 0xffffffff, 0x00040007,
1237 
	0x3c0b8, 0xffffffff, 0x00060005,
1238 
	0x3c0bc, 0xffffffff, 0x00090008,
1239 
	0x3c000, 0xffffffff, 0x96e00200,
1240 
	0x8708, 0xffffffff, 0x00900100,
1241 
	0xc424, 0xffffffff, 0x0020003f,
1242 
	0x38, 0xffffffff, 0x0140001c,
1243 
	0x3c, 0x000f0000, 0x000f0000,
1244 
	0x220, 0xffffffff, 0xC060000C,
1245 
	0x224, 0xc0000fff, 0x00000100,
1246 
	0xf90, 0xffffffff, 0x00000100,
1247 
	0xf98, 0x00000101, 0x00000000,
1248 
	0x20a8, 0xffffffff, 0x00000104,
1249 
	0x55e4, 0xff000fff, 0x00000100,
1250 
	0x30cc, 0xc0000fff, 0x00000104,
1251 
	0xc1e4, 0x00000001, 0x00000001,
1252 
	0xd00c, 0xff000ff0, 0x00000100,
1253 
	0xd80c, 0xff000ff0, 0x00000100
1254 
};
1255 
 
1256 
static const u32 kalindi_golden_spm_registers[] =
1257 
{
1258 
	0x30800, 0xe0ffffff, 0xe0000000
1259 
};
1260 
 
1261 
static const u32 kalindi_golden_common_registers[] =
1262 
{
1263 
	0xc770, 0xffffffff, 0x00000800,
1264 
	0xc774, 0xffffffff, 0x00000800,
1265 
	0xc798, 0xffffffff, 0x00007fbf,
1266 
	0xc79c, 0xffffffff, 0x00007faf
1267 
};
1268 
 
1269 
static const u32 kalindi_golden_registers[] =
1270 
{
1271 
	0x3c000, 0xffffdfff, 0x6e944040,
1272 
	0x55e4, 0xff607fff, 0xfc000100,
1273 
	0x3c220, 0xff000fff, 0x00000100,
1274 
	0x3c224, 0xff000fff, 0x00000100,
1275 
	0x3c200, 0xfffc0fff, 0x00000100,
1276 
	0x6ed8, 0x00010101, 0x00010000,
1277 
	0x9830, 0xffffffff, 0x00000000,
1278 
	0x9834, 0xf00fffff, 0x00000400,
1279 
	0x5bb0, 0x000000f0, 0x00000070,
1280 
	0x5bc0, 0xf0311fff, 0x80300000,
1281 
	0x98f8, 0x73773777, 0x12010001,
1282 
	0x98fc, 0xffffffff, 0x00000010,
1283 
	0x9b7c, 0x00ff0000, 0x00fc0000,
1284 
	0x8030, 0x00001f0f, 0x0000100a,
1285 
	0x2f48, 0x73773777, 0x12010001,
1286 
	0x2408, 0x000fffff, 0x000c007f,
1287 
	0x8a14, 0xf000003f, 0x00000007,
1288 
	0x8b24, 0x3fff3fff, 0x00ffcfff,
1289 
	0x30a04, 0x0000ff0f, 0x00000000,
1290 
	0x28a4c, 0x07ffffff, 0x06000000,
1291 
	0x4d8, 0x00000fff, 0x00000100,
1292 
	0x3e78, 0x00000001, 0x00000002,
1293 
	0xc768, 0x00000008, 0x00000008,
1294 
	0x8c00, 0x000000ff, 0x00000003,
1295 
	0x214f8, 0x01ff01ff, 0x00000002,
1296 
	0x21498, 0x007ff800, 0x00200000,
1297 
	0x2015c, 0xffffffff, 0x00000f40,
1298 
	0x88c4, 0x001f3ae3, 0x00000082,
1299 
	0x88d4, 0x0000001f, 0x00000010,
1300 
	0x30934, 0xffffffff, 0x00000000
1301 
};
1302 
 
1303 
static const u32 kalindi_mgcg_cgcg_init[] =
1304 
{
1305 
	0xc420, 0xffffffff, 0xfffffffc,
1306 
	0x30800, 0xffffffff, 0xe0000000,
1307 
	0x3c2a0, 0xffffffff, 0x00000100,
1308 
	0x3c208, 0xffffffff, 0x00000100,
1309 
	0x3c2c0, 0xffffffff, 0x00000100,
1310 
	0x3c2c8, 0xffffffff, 0x00000100,
1311 
	0x3c2c4, 0xffffffff, 0x00000100,
1312 
	0x55e4, 0xffffffff, 0x00600100,
1313 
	0x3c280, 0xffffffff, 0x00000100,
1314 
	0x3c214, 0xffffffff, 0x06000100,
1315 
	0x3c220, 0xffffffff, 0x00000100,
1316 
	0x3c218, 0xffffffff, 0x06000100,
1317 
	0x3c204, 0xffffffff, 0x00000100,
1318 
	0x3c2e0, 0xffffffff, 0x00000100,
1319 
	0x3c224, 0xffffffff, 0x00000100,
1320 
	0x3c200, 0xffffffff, 0x00000100,
1321 
	0x3c230, 0xffffffff, 0x00000100,
1322 
	0x3c234, 0xffffffff, 0x00000100,
1323 
	0x3c250, 0xffffffff, 0x00000100,
1324 
	0x3c254, 0xffffffff, 0x00000100,
1325 
	0x3c258, 0xffffffff, 0x00000100,
1326 
	0x3c25c, 0xffffffff, 0x00000100,
1327 
	0x3c260, 0xffffffff, 0x00000100,
1328 
	0x3c27c, 0xffffffff, 0x00000100,
1329 
	0x3c278, 0xffffffff, 0x00000100,
1330 
	0x3c210, 0xffffffff, 0x06000100,
1331 
	0x3c290, 0xffffffff, 0x00000100,
1332 
	0x3c274, 0xffffffff, 0x00000100,
1333 
	0x3c2b4, 0xffffffff, 0x00000100,
1334 
	0x3c2b0, 0xffffffff, 0x00000100,
1335 
	0x3c270, 0xffffffff, 0x00000100,
1336 
	0x30800, 0xffffffff, 0xe0000000,
1337 
	0x3c020, 0xffffffff, 0x00010000,
1338 
	0x3c024, 0xffffffff, 0x00030002,
1339 
	0x3c028, 0xffffffff, 0x00040007,
1340 
	0x3c02c, 0xffffffff, 0x00060005,
1341 
	0x3c030, 0xffffffff, 0x00090008,
1342 
	0x3c034, 0xffffffff, 0x00010000,
1343 
	0x3c038, 0xffffffff, 0x00030002,
1344 
	0x3c03c, 0xffffffff, 0x00040007,
1345 
	0x3c040, 0xffffffff, 0x00060005,
1346 
	0x3c044, 0xffffffff, 0x00090008,
1347 
	0x3c000, 0xffffffff, 0x96e00200,
1348 
	0x8708, 0xffffffff, 0x00900100,
1349 
	0xc424, 0xffffffff, 0x0020003f,
1350 
	0x38, 0xffffffff, 0x0140001c,
1351 
	0x3c, 0x000f0000, 0x000f0000,
1352 
	0x220, 0xffffffff, 0xC060000C,
1353 
	0x224, 0xc0000fff, 0x00000100,
1354 
	0x20a8, 0xffffffff, 0x00000104,
1355 
	0x55e4, 0xff000fff, 0x00000100,
1356 
	0x30cc, 0xc0000fff, 0x00000104,
1357 
	0xc1e4, 0x00000001, 0x00000001,
1358 
	0xd00c, 0xff000ff0, 0x00000100,
1359 
	0xd80c, 0xff000ff0, 0x00000100
1360 
};
1361 
 
1362 
static const u32 hawaii_golden_spm_registers[] =
1363 
{
1364 
	0x30800, 0xe0ffffff, 0xe0000000
1365 
};
1366 
 
1367 
static const u32 hawaii_golden_common_registers[] =
1368 
{
1369 
	0x30800, 0xffffffff, 0xe0000000,
1370 
	0x28350, 0xffffffff, 0x3a00161a,
1371 
	0x28354, 0xffffffff, 0x0000002e,
1372 
	0x9a10, 0xffffffff, 0x00018208,
1373 
	0x98f8, 0xffffffff, 0x12011003
1374 
};
1375 
 
1376 
static const u32 hawaii_golden_registers[] =
1377 
{
1378 
	0x3354, 0x00000333, 0x00000333,
1379 
	0x9a10, 0x00010000, 0x00058208,
1380 
	0x9830, 0xffffffff, 0x00000000,
1381 
	0x9834, 0xf00fffff, 0x00000400,
1382 
	0x9838, 0x0002021c, 0x00020200,
1383 
	0xc78, 0x00000080, 0x00000000,
1384 
	0x5bb0, 0x000000f0, 0x00000070,
1385 
	0x5bc0, 0xf0311fff, 0x80300000,
1386 
	0x350c, 0x00810000, 0x408af000,
1387 
	0x7030, 0x31000111, 0x00000011,
1388 
	0x2f48, 0x73773777, 0x12010001,
1389 
	0x2120, 0x0000007f, 0x0000001b,
1390 
	0x21dc, 0x00007fb6, 0x00002191,
1391 
	0x3628, 0x0000003f, 0x0000000a,
1392 
	0x362c, 0x0000003f, 0x0000000a,
1393 
	0x2ae4, 0x00073ffe, 0x000022a2,
1394 
	0x240c, 0x000007ff, 0x00000000,
1395 
	0x8bf0, 0x00002001, 0x00000001,
1396 
	0x8b24, 0xffffffff, 0x00ffffff,
1397 
	0x30a04, 0x0000ff0f, 0x00000000,
1398 
	0x28a4c, 0x07ffffff, 0x06000000,
1399 
	0x3e78, 0x00000001, 0x00000002,
1400 
	0xc768, 0x00000008, 0x00000008,
1401 
	0xc770, 0x00000f00, 0x00000800,
1402 
	0xc774, 0x00000f00, 0x00000800,
1403 
	0xc798, 0x00ffffff, 0x00ff7fbf,
1404 
	0xc79c, 0x00ffffff, 0x00ff7faf,
1405 
	0x8c00, 0x000000ff, 0x00000800,
1406 
	0xe40, 0x00001fff, 0x00001fff,
1407 
	0x9060, 0x0000007f, 0x00000020,
1408 
	0x9508, 0x00010000, 0x00010000,
1409 
	0xae00, 0x00100000, 0x000ff07c,
1410 
	0xac14, 0x000003ff, 0x0000000f,
1411 
	0xac10, 0xffffffff, 0x7564fdec,
1412 
	0xac0c, 0xffffffff, 0x3120b9a8,
1413 
	0xac08, 0x20000000, 0x0f9c0000
1414 
};
1415 
 
1416 
static const u32 hawaii_mgcg_cgcg_init[] =
1417 
{
1418 
	0xc420, 0xffffffff, 0xfffffffd,
1419 
	0x30800, 0xffffffff, 0xe0000000,
1420 
	0x3c2a0, 0xffffffff, 0x00000100,
1421 
	0x3c208, 0xffffffff, 0x00000100,
1422 
	0x3c2c0, 0xffffffff, 0x00000100,
1423 
	0x3c2c8, 0xffffffff, 0x00000100,
1424 
	0x3c2c4, 0xffffffff, 0x00000100,
1425 
	0x55e4, 0xffffffff, 0x00200100,
1426 
	0x3c280, 0xffffffff, 0x00000100,
1427 
	0x3c214, 0xffffffff, 0x06000100,
1428 
	0x3c220, 0xffffffff, 0x00000100,
1429 
	0x3c218, 0xffffffff, 0x06000100,
1430 
	0x3c204, 0xffffffff, 0x00000100,
1431 
	0x3c2e0, 0xffffffff, 0x00000100,
1432 
	0x3c224, 0xffffffff, 0x00000100,
1433 
	0x3c200, 0xffffffff, 0x00000100,
1434 
	0x3c230, 0xffffffff, 0x00000100,
1435 
	0x3c234, 0xffffffff, 0x00000100,
1436 
	0x3c250, 0xffffffff, 0x00000100,
1437 
	0x3c254, 0xffffffff, 0x00000100,
1438 
	0x3c258, 0xffffffff, 0x00000100,
1439 
	0x3c25c, 0xffffffff, 0x00000100,
1440 
	0x3c260, 0xffffffff, 0x00000100,
1441 
	0x3c27c, 0xffffffff, 0x00000100,
1442 
	0x3c278, 0xffffffff, 0x00000100,
1443 
	0x3c210, 0xffffffff, 0x06000100,
1444 
	0x3c290, 0xffffffff, 0x00000100,
1445 
	0x3c274, 0xffffffff, 0x00000100,
1446 
	0x3c2b4, 0xffffffff, 0x00000100,
1447 
	0x3c2b0, 0xffffffff, 0x00000100,
1448 
	0x3c270, 0xffffffff, 0x00000100,
1449 
	0x30800, 0xffffffff, 0xe0000000,
1450 
	0x3c020, 0xffffffff, 0x00010000,
1451 
	0x3c024, 0xffffffff, 0x00030002,
1452 
	0x3c028, 0xffffffff, 0x00040007,
1453 
	0x3c02c, 0xffffffff, 0x00060005,
1454 
	0x3c030, 0xffffffff, 0x00090008,
1455 
	0x3c034, 0xffffffff, 0x00010000,
1456 
	0x3c038, 0xffffffff, 0x00030002,
1457 
	0x3c03c, 0xffffffff, 0x00040007,
1458 
	0x3c040, 0xffffffff, 0x00060005,
1459 
	0x3c044, 0xffffffff, 0x00090008,
1460 
	0x3c048, 0xffffffff, 0x00010000,
1461 
	0x3c04c, 0xffffffff, 0x00030002,
1462 
	0x3c050, 0xffffffff, 0x00040007,
1463 
	0x3c054, 0xffffffff, 0x00060005,
1464 
	0x3c058, 0xffffffff, 0x00090008,
1465 
	0x3c05c, 0xffffffff, 0x00010000,
1466 
	0x3c060, 0xffffffff, 0x00030002,
1467 
	0x3c064, 0xffffffff, 0x00040007,
1468 
	0x3c068, 0xffffffff, 0x00060005,
1469 
	0x3c06c, 0xffffffff, 0x00090008,
1470 
	0x3c070, 0xffffffff, 0x00010000,
1471 
	0x3c074, 0xffffffff, 0x00030002,
1472 
	0x3c078, 0xffffffff, 0x00040007,
1473 
	0x3c07c, 0xffffffff, 0x00060005,
1474 
	0x3c080, 0xffffffff, 0x00090008,
1475 
	0x3c084, 0xffffffff, 0x00010000,
1476 
	0x3c088, 0xffffffff, 0x00030002,
1477 
	0x3c08c, 0xffffffff, 0x00040007,
1478 
	0x3c090, 0xffffffff, 0x00060005,
1479 
	0x3c094, 0xffffffff, 0x00090008,
1480 
	0x3c098, 0xffffffff, 0x00010000,
1481 
	0x3c09c, 0xffffffff, 0x00030002,
1482 
	0x3c0a0, 0xffffffff, 0x00040007,
1483 
	0x3c0a4, 0xffffffff, 0x00060005,
1484 
	0x3c0a8, 0xffffffff, 0x00090008,
1485 
	0x3c0ac, 0xffffffff, 0x00010000,
1486 
	0x3c0b0, 0xffffffff, 0x00030002,
1487 
	0x3c0b4, 0xffffffff, 0x00040007,
1488 
	0x3c0b8, 0xffffffff, 0x00060005,
1489 
	0x3c0bc, 0xffffffff, 0x00090008,
1490 
	0x3c0c0, 0xffffffff, 0x00010000,
1491 
	0x3c0c4, 0xffffffff, 0x00030002,
1492 
	0x3c0c8, 0xffffffff, 0x00040007,
1493 
	0x3c0cc, 0xffffffff, 0x00060005,
1494 
	0x3c0d0, 0xffffffff, 0x00090008,
1495 
	0x3c0d4, 0xffffffff, 0x00010000,
1496 
	0x3c0d8, 0xffffffff, 0x00030002,
1497 
	0x3c0dc, 0xffffffff, 0x00040007,
1498 
	0x3c0e0, 0xffffffff, 0x00060005,
1499 
	0x3c0e4, 0xffffffff, 0x00090008,
1500 
	0x3c0e8, 0xffffffff, 0x00010000,
1501 
	0x3c0ec, 0xffffffff, 0x00030002,
1502 
	0x3c0f0, 0xffffffff, 0x00040007,
1503 
	0x3c0f4, 0xffffffff, 0x00060005,
1504 
	0x3c0f8, 0xffffffff, 0x00090008,
1505 
	0xc318, 0xffffffff, 0x00020200,
1506 
	0x3350, 0xffffffff, 0x00000200,
1507 
	0x15c0, 0xffffffff, 0x00000400,
1508 
	0x55e8, 0xffffffff, 0x00000000,
1509 
	0x2f50, 0xffffffff, 0x00000902,
1510 
	0x3c000, 0xffffffff, 0x96940200,
1511 
	0x8708, 0xffffffff, 0x00900100,
1512 
	0xc424, 0xffffffff, 0x0020003f,
1513 
	0x38, 0xffffffff, 0x0140001c,
1514 
	0x3c, 0x000f0000, 0x000f0000,
1515 
	0x220, 0xffffffff, 0xc060000c,
1516 
	0x224, 0xc0000fff, 0x00000100,
1517 
	0xf90, 0xffffffff, 0x00000100,
1518 
	0xf98, 0x00000101, 0x00000000,
1519 
	0x20a8, 0xffffffff, 0x00000104,
1520 
	0x55e4, 0xff000fff, 0x00000100,
1521 
	0x30cc, 0xc0000fff, 0x00000104,
1522 
	0xc1e4, 0x00000001, 0x00000001,
1523 
	0xd00c, 0xff000ff0, 0x00000100,
1524 
	0xd80c, 0xff000ff0, 0x00000100
1525 
};
1526 
 
1527 
static const u32 godavari_golden_registers[] =
1528 
{
1529 
	0x55e4, 0xff607fff, 0xfc000100,
1530 
	0x6ed8, 0x00010101, 0x00010000,
1531 
	0x9830, 0xffffffff, 0x00000000,
1532 
	0x98302, 0xf00fffff, 0x00000400,
1533 
	0x6130, 0xffffffff, 0x00010000,
1534 
	0x5bb0, 0x000000f0, 0x00000070,
1535 
	0x5bc0, 0xf0311fff, 0x80300000,
1536 
	0x98f8, 0x73773777, 0x12010001,
1537 
	0x98fc, 0xffffffff, 0x00000010,
1538 
	0x8030, 0x00001f0f, 0x0000100a,
1539 
	0x2f48, 0x73773777, 0x12010001,
1540 
	0x2408, 0x000fffff, 0x000c007f,
1541 
	0x8a14, 0xf000003f, 0x00000007,
1542 
	0x8b24, 0xffffffff, 0x00ff0fff,
1543 
	0x30a04, 0x0000ff0f, 0x00000000,
1544 
	0x28a4c, 0x07ffffff, 0x06000000,
1545 
	0x4d8, 0x00000fff, 0x00000100,
1546 
	0xd014, 0x00010000, 0x00810001,
1547 
	0xd814, 0x00010000, 0x00810001,
1548 
	0x3e78, 0x00000001, 0x00000002,
1549 
	0xc768, 0x00000008, 0x00000008,
1550 
	0xc770, 0x00000f00, 0x00000800,
1551 
	0xc774, 0x00000f00, 0x00000800,
1552 
	0xc798, 0x00ffffff, 0x00ff7fbf,
1553 
	0xc79c, 0x00ffffff, 0x00ff7faf,
1554 
	0x8c00, 0x000000ff, 0x00000001,
1555 
	0x214f8, 0x01ff01ff, 0x00000002,
1556 
	0x21498, 0x007ff800, 0x00200000,
1557 
	0x2015c, 0xffffffff, 0x00000f40,
1558 
	0x88c4, 0x001f3ae3, 0x00000082,
1559 
	0x88d4, 0x0000001f, 0x00000010,
1560 
	0x30934, 0xffffffff, 0x00000000
1561 
};
1562 
 
1563 
 
1564 
static void cik_init_golden_registers(struct radeon_device *rdev)
1565 
{
1566 
	switch (rdev->family) {
1567 
	case CHIP_BONAIRE:
1568 
		radeon_program_register_sequence(rdev,
1569 
						 bonaire_mgcg_cgcg_init,
1570 
						 (const u32)ARRAY_SIZE(bonaire_mgcg_cgcg_init));
1571 
		radeon_program_register_sequence(rdev,
1572 
						 bonaire_golden_registers,
1573 
						 (const u32)ARRAY_SIZE(bonaire_golden_registers));
1574 
		radeon_program_register_sequence(rdev,
1575 
						 bonaire_golden_common_registers,
1576 
						 (const u32)ARRAY_SIZE(bonaire_golden_common_registers));
1577 
		radeon_program_register_sequence(rdev,
1578 
						 bonaire_golden_spm_registers,
1579 
						 (const u32)ARRAY_SIZE(bonaire_golden_spm_registers));
1580 
		break;
1581 
	case CHIP_KABINI:
1582 
		radeon_program_register_sequence(rdev,
1583 
						 kalindi_mgcg_cgcg_init,
1584 
						 (const u32)ARRAY_SIZE(kalindi_mgcg_cgcg_init));
1585 
		radeon_program_register_sequence(rdev,
1586 
						 kalindi_golden_registers,
1587 
						 (const u32)ARRAY_SIZE(kalindi_golden_registers));
1588 
		radeon_program_register_sequence(rdev,
1589 
						 kalindi_golden_common_registers,
1590 
						 (const u32)ARRAY_SIZE(kalindi_golden_common_registers));
1591 
		radeon_program_register_sequence(rdev,
1592 
						 kalindi_golden_spm_registers,
1593 
						 (const u32)ARRAY_SIZE(kalindi_golden_spm_registers));
1594 
		break;
1595 
	case CHIP_MULLINS:
1596 
		radeon_program_register_sequence(rdev,
1597 
						 kalindi_mgcg_cgcg_init,
1598 
						 (const u32)ARRAY_SIZE(kalindi_mgcg_cgcg_init));
1599 
		radeon_program_register_sequence(rdev,
1600 
						 godavari_golden_registers,
1601 
						 (const u32)ARRAY_SIZE(godavari_golden_registers));
1602 
		radeon_program_register_sequence(rdev,
1603 
						 kalindi_golden_common_registers,
1604 
						 (const u32)ARRAY_SIZE(kalindi_golden_common_registers));
1605 
		radeon_program_register_sequence(rdev,
1606 
						 kalindi_golden_spm_registers,
1607 
						 (const u32)ARRAY_SIZE(kalindi_golden_spm_registers));
1608 
		break;
1609 
	case CHIP_KAVERI:
1610 
		radeon_program_register_sequence(rdev,
1611 
						 spectre_mgcg_cgcg_init,
1612 
						 (const u32)ARRAY_SIZE(spectre_mgcg_cgcg_init));
1613 
		radeon_program_register_sequence(rdev,
1614 
						 spectre_golden_registers,
1615 
						 (const u32)ARRAY_SIZE(spectre_golden_registers));
1616 
		radeon_program_register_sequence(rdev,
1617 
						 spectre_golden_common_registers,
1618 
						 (const u32)ARRAY_SIZE(spectre_golden_common_registers));
1619 
		radeon_program_register_sequence(rdev,
1620 
						 spectre_golden_spm_registers,
1621 
						 (const u32)ARRAY_SIZE(spectre_golden_spm_registers));
1622 
		break;
1623 
	case CHIP_HAWAII:
1624 
		radeon_program_register_sequence(rdev,
1625 
						 hawaii_mgcg_cgcg_init,
1626 
						 (const u32)ARRAY_SIZE(hawaii_mgcg_cgcg_init));
1627 
		radeon_program_register_sequence(rdev,
1628 
						 hawaii_golden_registers,
1629 
						 (const u32)ARRAY_SIZE(hawaii_golden_registers));
1630 
		radeon_program_register_sequence(rdev,
1631 
						 hawaii_golden_common_registers,
1632 
						 (const u32)ARRAY_SIZE(hawaii_golden_common_registers));
1633 
		radeon_program_register_sequence(rdev,
1634 
						 hawaii_golden_spm_registers,
1635 
						 (const u32)ARRAY_SIZE(hawaii_golden_spm_registers));
1636 
		break;
1637 
	default:
1638 
		break;
1639 
	}
1640 
}
1641 
 
1642 
/**
1643 
 * cik_get_xclk - get the xclk
1644 
 *
1645 
 * @rdev: radeon_device pointer
1646 
 *
1647 
 * Returns the reference clock used by the gfx engine
1648 
 * (CIK).
1649 
 */
1650 
u32 cik_get_xclk(struct radeon_device *rdev)
1651 
{
1652 
        u32 reference_clock = rdev->clock.spll.reference_freq;
1653 
 
1654 
	if (rdev->flags & RADEON_IS_IGP) {
1655 
		if (RREG32_SMC(GENERAL_PWRMGT) & GPU_COUNTER_CLK)
1656 
			return reference_clock / 2;
1657 
	} else {
1658 
		if (RREG32_SMC(CG_CLKPIN_CNTL) & XTALIN_DIVIDE)
1659 
			return reference_clock / 4;
1660 
	}
1661 
	return reference_clock;
1662 
}
1663 
 
1664 
/**
1665 
 * cik_mm_rdoorbell - read a doorbell dword
1666 
 *
1667 
 * @rdev: radeon_device pointer
1668 
 * @index: doorbell index
1669 
 *
1670 
 * Returns the value in the doorbell aperture at the
1671 
 * requested doorbell index (CIK).
1672 
 */
1673 
u32 cik_mm_rdoorbell(struct radeon_device *rdev, u32 index)
1674 
{
1675 
	if (index < rdev->doorbell.num_doorbells) {
1676 
		return readl(rdev->doorbell.ptr + index);
1677 
	} else {
1678 
		DRM_ERROR("reading beyond doorbell aperture: 0x%08x!\n", index);
1679 
		return 0;
1680 
	}
1681 
}
1682 
 
1683 
/**
1684 
 * cik_mm_wdoorbell - write a doorbell dword
1685 
 *
1686 
 * @rdev: radeon_device pointer
1687 
 * @index: doorbell index
1688 
 * @v: value to write
1689 
 *
1690 
 * Writes @v to the doorbell aperture at the
1691 
 * requested doorbell index (CIK).
1692 
 */
1693 
void cik_mm_wdoorbell(struct radeon_device *rdev, u32 index, u32 v)
1694 
{
1695 
	if (index < rdev->doorbell.num_doorbells) {
1696 
		writel(v, rdev->doorbell.ptr + index);
1697 
	} else {
1698 
		DRM_ERROR("writing beyond doorbell aperture: 0x%08x!\n", index);
1699 
	}
1700 
}
1701 
 
1702 
#define BONAIRE_IO_MC_REGS_SIZE 36
1703 
 
1704 
static const u32 bonaire_io_mc_regs[BONAIRE_IO_MC_REGS_SIZE][2] =
1705 
{
1706 
	{0x00000070, 0x04400000},
1707 
	{0x00000071, 0x80c01803},
1708 
	{0x00000072, 0x00004004},
1709 
	{0x00000073, 0x00000100},
1710 
	{0x00000074, 0x00ff0000},
1711 
	{0x00000075, 0x34000000},
1712 
	{0x00000076, 0x08000014},
1713 
	{0x00000077, 0x00cc08ec},
1714 
	{0x00000078, 0x00000400},
1715 
	{0x00000079, 0x00000000},
1716 
	{0x0000007a, 0x04090000},
1717 
	{0x0000007c, 0x00000000},
1718 
	{0x0000007e, 0x4408a8e8},
1719 
	{0x0000007f, 0x00000304},
1720 
	{0x00000080, 0x00000000},
1721 
	{0x00000082, 0x00000001},
1722 
	{0x00000083, 0x00000002},
1723 
	{0x00000084, 0xf3e4f400},
1724 
	{0x00000085, 0x052024e3},
1725 
	{0x00000087, 0x00000000},
1726 
	{0x00000088, 0x01000000},
1727 
	{0x0000008a, 0x1c0a0000},
1728 
	{0x0000008b, 0xff010000},
1729 
	{0x0000008d, 0xffffefff},
1730 
	{0x0000008e, 0xfff3efff},
1731 
	{0x0000008f, 0xfff3efbf},
1732 
	{0x00000092, 0xf7ffffff},
1733 
	{0x00000093, 0xffffff7f},
1734 
	{0x00000095, 0x00101101},
1735 
	{0x00000096, 0x00000fff},
1736 
	{0x00000097, 0x00116fff},
1737 
	{0x00000098, 0x60010000},
1738 
	{0x00000099, 0x10010000},
1739 
	{0x0000009a, 0x00006000},
1740 
	{0x0000009b, 0x00001000},
1741 
	{0x0000009f, 0x00b48000}
1742 
};
1743 
 
1744 
#define HAWAII_IO_MC_REGS_SIZE 22
1745 
 
1746 
static const u32 hawaii_io_mc_regs[HAWAII_IO_MC_REGS_SIZE][2] =
1747 
{
1748 
	{0x0000007d, 0x40000000},
1749 
	{0x0000007e, 0x40180304},
1750 
	{0x0000007f, 0x0000ff00},
1751 
	{0x00000081, 0x00000000},
1752 
	{0x00000083, 0x00000800},
1753 
	{0x00000086, 0x00000000},
1754 
	{0x00000087, 0x00000100},
1755 
	{0x00000088, 0x00020100},
1756 
	{0x00000089, 0x00000000},
1757 
	{0x0000008b, 0x00040000},
1758 
	{0x0000008c, 0x00000100},
1759 
	{0x0000008e, 0xff010000},
1760 
	{0x00000090, 0xffffefff},
1761 
	{0x00000091, 0xfff3efff},
1762 
	{0x00000092, 0xfff3efbf},
1763 
	{0x00000093, 0xf7ffffff},
1764 
	{0x00000094, 0xffffff7f},
1765 
	{0x00000095, 0x00000fff},
1766 
	{0x00000096, 0x00116fff},
1767 
	{0x00000097, 0x60010000},
1768 
	{0x00000098, 0x10010000},
1769 
	{0x0000009f, 0x00c79000}
1770 
};
1771 
 
1772 
 
1773 
/**
1774 
 * cik_srbm_select - select specific register instances
1775 
 *
1776 
 * @rdev: radeon_device pointer
1777 
 * @me: selected ME (micro engine)
1778 
 * @pipe: pipe
1779 
 * @queue: queue
1780 
 * @vmid: VMID
1781 
 *
1782 
 * Switches the currently active registers instances.  Some
1783 
 * registers are instanced per VMID, others are instanced per
1784 
 * me/pipe/queue combination.
1785 
 */
1786 
static void cik_srbm_select(struct radeon_device *rdev,
1787 
			    u32 me, u32 pipe, u32 queue, u32 vmid)
1788 
{
1789 
	u32 srbm_gfx_cntl = (PIPEID(pipe & 0x3) |
1790 
			     MEID(me & 0x3) |
1791 
			     VMID(vmid & 0xf) |
1792 
			     QUEUEID(queue & 0x7));
1793 
	WREG32(SRBM_GFX_CNTL, srbm_gfx_cntl);
1794 
}
1795 
 
1796 
/* ucode loading */
1797 
/**
1798 
 * ci_mc_load_microcode - load MC ucode into the hw
1799 
 *
1800 
 * @rdev: radeon_device pointer
1801 
 *
1802 
 * Load the GDDR MC ucode into the hw (CIK).
1803 
 * Returns 0 on success, error on failure.
1804 
 */
1805 
int ci_mc_load_microcode(struct radeon_device *rdev)
1806 
{
1807 
	const __be32 *fw_data = NULL;
1808 
	const __le32 *new_fw_data = NULL;
1809 
	u32 running, blackout = 0;
1810 
	u32 *io_mc_regs = NULL;
1811 
	const __le32 *new_io_mc_regs = NULL;
1812 
	int i, regs_size, ucode_size;
1813 
 
1814 
	if (!rdev->mc_fw)
1815 
		return -EINVAL;
1816 
 
1817 
	if (rdev->new_fw) {
1818 
		const struct mc_firmware_header_v1_0 *hdr =
1819 
			(const struct mc_firmware_header_v1_0 *)rdev->mc_fw->data;
1820 
 
1821 
		radeon_ucode_print_mc_hdr(&hdr->header);
1822 
 
1823 
		regs_size = le32_to_cpu(hdr->io_debug_size_bytes) / (4 * 2);
1824 
		new_io_mc_regs = (const __le32 *)
1825 
			(rdev->mc_fw->data + le32_to_cpu(hdr->io_debug_array_offset_bytes));
1826 
		ucode_size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4;
1827 
		new_fw_data = (const __le32 *)
1828 
			(rdev->mc_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
1829 
	} else {
1830 
	ucode_size = rdev->mc_fw->size / 4;
1831 
 
1832 
	switch (rdev->family) {
1833 
	case CHIP_BONAIRE:
1834 
		io_mc_regs = (u32 *)&bonaire_io_mc_regs;
1835 
		regs_size = BONAIRE_IO_MC_REGS_SIZE;
1836 
		break;
1837 
	case CHIP_HAWAII:
1838 
		io_mc_regs = (u32 *)&hawaii_io_mc_regs;
1839 
		regs_size = HAWAII_IO_MC_REGS_SIZE;
1840 
		break;
1841 
	default:
1842 
		return -EINVAL;
1843 
	}
1844 
		fw_data = (const __be32 *)rdev->mc_fw->data;
1845 
	}
1846 
 
1847 
	running = RREG32(MC_SEQ_SUP_CNTL) & RUN_MASK;
1848 
 
1849 
	if (running == 0) {
1850 
		if (running) {
1851 
			blackout = RREG32(MC_SHARED_BLACKOUT_CNTL);
1852 
			WREG32(MC_SHARED_BLACKOUT_CNTL, blackout | 1);
1853 
		}
1854 
 
1855 
		/* reset the engine and set to writable */
1856 
		WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
1857 
		WREG32(MC_SEQ_SUP_CNTL, 0x00000010);
1858 
 
1859 
		/* load mc io regs */
1860 
		for (i = 0; i < regs_size; i++) {
1861 
			if (rdev->new_fw) {
1862 
				WREG32(MC_SEQ_IO_DEBUG_INDEX, le32_to_cpup(new_io_mc_regs++));
1863 
				WREG32(MC_SEQ_IO_DEBUG_DATA, le32_to_cpup(new_io_mc_regs++));
1864 
			} else {
1865 
			WREG32(MC_SEQ_IO_DEBUG_INDEX, io_mc_regs[(i << 1)]);
1866 
			WREG32(MC_SEQ_IO_DEBUG_DATA, io_mc_regs[(i << 1) + 1]);
1867 
		}
1868 
		}
1869 
		/* load the MC ucode */
1870 
		for (i = 0; i < ucode_size; i++) {
1871 
			if (rdev->new_fw)
1872 
				WREG32(MC_SEQ_SUP_PGM, le32_to_cpup(new_fw_data++));
1873 
			else
1874 
			WREG32(MC_SEQ_SUP_PGM, be32_to_cpup(fw_data++));
1875 
		}
1876 
 
1877 
		/* put the engine back into the active state */
1878 
		WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
1879 
		WREG32(MC_SEQ_SUP_CNTL, 0x00000004);
1880 
		WREG32(MC_SEQ_SUP_CNTL, 0x00000001);
1881 
 
1882 
		/* wait for training to complete */
1883 
		for (i = 0; i < rdev->usec_timeout; i++) {
1884 
			if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D0)
1885 
				break;
1886 
			udelay(1);
1887 
		}
1888 
		for (i = 0; i < rdev->usec_timeout; i++) {
1889 
			if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D1)
1890 
				break;
1891 
			udelay(1);
1892 
		}
1893 
 
1894 
		if (running)
1895 
			WREG32(MC_SHARED_BLACKOUT_CNTL, blackout);
1896 
	}
1897 
 
1898 
	return 0;
1899 
}
1900 
 
1901 
/**
1902 
 * cik_init_microcode - load ucode images from disk
1903 
 *
1904 
 * @rdev: radeon_device pointer
1905 
 *
1906 
 * Use the firmware interface to load the ucode images into
1907 
 * the driver (not loaded into hw).
1908 
 * Returns 0 on success, error on failure.
1909 
 */
1910 
static int cik_init_microcode(struct radeon_device *rdev)
1911 
{
1912 
	const char *chip_name;
1913 
	const char *new_chip_name;
1914 
	size_t pfp_req_size, me_req_size, ce_req_size,
1915 
		mec_req_size, rlc_req_size, mc_req_size = 0,
1916 
		sdma_req_size, smc_req_size = 0, mc2_req_size = 0;
1917 
	char fw_name[30];
1918 
	int new_fw = 0;
1919 
	int err;
1920 
	int num_fw;
1921 
 
1922 
	DRM_DEBUG("\n");
1923 
 
1924 
	switch (rdev->family) {
1925 
	case CHIP_BONAIRE:
1926 
		chip_name = "BONAIRE";
1927 
		new_chip_name = "bonaire";
1928 
		pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
1929 
		me_req_size = CIK_ME_UCODE_SIZE * 4;
1930 
		ce_req_size = CIK_CE_UCODE_SIZE * 4;
1931 
		mec_req_size = CIK_MEC_UCODE_SIZE * 4;
1932 
		rlc_req_size = BONAIRE_RLC_UCODE_SIZE * 4;
1933 
		mc_req_size = BONAIRE_MC_UCODE_SIZE * 4;
1934 
		mc2_req_size = BONAIRE_MC2_UCODE_SIZE * 4;
1935 
		sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
1936 
		smc_req_size = ALIGN(BONAIRE_SMC_UCODE_SIZE, 4);
1937 
		num_fw = 8;
1938 
		break;
1939 
	case CHIP_HAWAII:
1940 
		chip_name = "HAWAII";
1941 
		new_chip_name = "hawaii";
1942 
		pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
1943 
		me_req_size = CIK_ME_UCODE_SIZE * 4;
1944 
		ce_req_size = CIK_CE_UCODE_SIZE * 4;
1945 
		mec_req_size = CIK_MEC_UCODE_SIZE * 4;
1946 
		rlc_req_size = BONAIRE_RLC_UCODE_SIZE * 4;
1947 
		mc_req_size = HAWAII_MC_UCODE_SIZE * 4;
1948 
		mc2_req_size = HAWAII_MC2_UCODE_SIZE * 4;
1949 
		sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
1950 
		smc_req_size = ALIGN(HAWAII_SMC_UCODE_SIZE, 4);
1951 
		num_fw = 8;
1952 
		break;
1953 
	case CHIP_KAVERI:
1954 
		chip_name = "KAVERI";
1955 
		new_chip_name = "kaveri";
1956 
		pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
1957 
		me_req_size = CIK_ME_UCODE_SIZE * 4;
1958 
		ce_req_size = CIK_CE_UCODE_SIZE * 4;
1959 
		mec_req_size = CIK_MEC_UCODE_SIZE * 4;
1960 
		rlc_req_size = KV_RLC_UCODE_SIZE * 4;
1961 
		sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
1962 
		num_fw = 7;
1963 
		break;
1964 
	case CHIP_KABINI:
1965 
		chip_name = "KABINI";
1966 
		new_chip_name = "kabini";
1967 
		pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
1968 
		me_req_size = CIK_ME_UCODE_SIZE * 4;
1969 
		ce_req_size = CIK_CE_UCODE_SIZE * 4;
1970 
		mec_req_size = CIK_MEC_UCODE_SIZE * 4;
1971 
		rlc_req_size = KB_RLC_UCODE_SIZE * 4;
1972 
		sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
1973 
		num_fw = 6;
1974 
		break;
1975 
	case CHIP_MULLINS:
1976 
		chip_name = "MULLINS";
1977 
		new_chip_name = "mullins";
1978 
		pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
1979 
		me_req_size = CIK_ME_UCODE_SIZE * 4;
1980 
		ce_req_size = CIK_CE_UCODE_SIZE * 4;
1981 
		mec_req_size = CIK_MEC_UCODE_SIZE * 4;
1982 
		rlc_req_size = ML_RLC_UCODE_SIZE * 4;
1983 
		sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
1984 
		num_fw = 6;
1985 
		break;
1986 
	default: BUG();
1987 
	}
1988 
 
1989 
	DRM_INFO("Loading %s Microcode\n", new_chip_name);
1990 
 
1991 
	snprintf(fw_name, sizeof(fw_name), "radeon/%s_pfp.bin", new_chip_name);
1992 
	err = request_firmware(&rdev->pfp_fw, fw_name, rdev->dev);
1993 
	if (err) {
1994 
	snprintf(fw_name, sizeof(fw_name), "radeon/%s_pfp.bin", chip_name);
1995 
	err = request_firmware(&rdev->pfp_fw, fw_name, rdev->dev);
1996 
	if (err)
1997 
		goto out;
1998 
	if (rdev->pfp_fw->size != pfp_req_size) {
1999 
		printk(KERN_ERR
2000 
		       "cik_cp: Bogus length %zu in firmware \"%s\"\n",
2001 
		       rdev->pfp_fw->size, fw_name);
2002 
		err = -EINVAL;
2003 
		goto out;
2004 
	}
2005 
	} else {
2006 
		err = radeon_ucode_validate(rdev->pfp_fw);
2007 
		if (err) {
2008 
			printk(KERN_ERR
2009 
			       "cik_fw: validation failed for firmware \"%s\"\n",
2010 
			       fw_name);
2011 
			goto out;
2012 
		} else {
2013 
			new_fw++;
2014 
		}
2015 
	}
2016 
 
2017 
	snprintf(fw_name, sizeof(fw_name), "radeon/%s_me.bin", new_chip_name);
2018 
	err = request_firmware(&rdev->me_fw, fw_name, rdev->dev);
2019 
	if (err) {
2020 
	snprintf(fw_name, sizeof(fw_name), "radeon/%s_me.bin", chip_name);
2021 
	err = request_firmware(&rdev->me_fw, fw_name, rdev->dev);
2022 
	if (err)
2023 
		goto out;
2024 
	if (rdev->me_fw->size != me_req_size) {
2025 
		printk(KERN_ERR
2026 
		       "cik_cp: Bogus length %zu in firmware \"%s\"\n",
2027 
		       rdev->me_fw->size, fw_name);
2028 
		err = -EINVAL;
2029 
	}
2030 
	} else {
2031 
		err = radeon_ucode_validate(rdev->me_fw);
2032 
		if (err) {
2033 
			printk(KERN_ERR
2034 
			       "cik_fw: validation failed for firmware \"%s\"\n",
2035 
			       fw_name);
2036 
			goto out;
2037 
		} else {
2038 
			new_fw++;
2039 
		}
2040 
	}
2041 
 
2042 
	snprintf(fw_name, sizeof(fw_name), "radeon/%s_ce.bin", new_chip_name);
2043 
	err = request_firmware(&rdev->ce_fw, fw_name, rdev->dev);
2044 
	if (err) {
2045 
	snprintf(fw_name, sizeof(fw_name), "radeon/%s_ce.bin", chip_name);
2046 
	err = request_firmware(&rdev->ce_fw, fw_name, rdev->dev);
2047 
	if (err)
2048 
		goto out;
2049 
	if (rdev->ce_fw->size != ce_req_size) {
2050 
		printk(KERN_ERR
2051 
		       "cik_cp: Bogus length %zu in firmware \"%s\"\n",
2052 
		       rdev->ce_fw->size, fw_name);
2053 
		err = -EINVAL;
2054 
	}
2055 
	} else {
2056 
		err = radeon_ucode_validate(rdev->ce_fw);
2057 
		if (err) {
2058 
			printk(KERN_ERR
2059 
			       "cik_fw: validation failed for firmware \"%s\"\n",
2060 
			       fw_name);
2061 
			goto out;
2062 
		} else {
2063 
			new_fw++;
2064 
		}
2065 
	}
2066 
 
2067 
	snprintf(fw_name, sizeof(fw_name), "radeon/%s_mec.bin", new_chip_name);
2068 
	err = request_firmware(&rdev->mec_fw, fw_name, rdev->dev);
2069 
	if (err) {
2070 
	snprintf(fw_name, sizeof(fw_name), "radeon/%s_mec.bin", chip_name);
2071 
	err = request_firmware(&rdev->mec_fw, fw_name, rdev->dev);
2072 
	if (err)
2073 
		goto out;
2074 
	if (rdev->mec_fw->size != mec_req_size) {
2075 
		printk(KERN_ERR
2076 
		       "cik_cp: Bogus length %zu in firmware \"%s\"\n",
2077 
		       rdev->mec_fw->size, fw_name);
2078 
		err = -EINVAL;
2079 
	}
2080 
	} else {
2081 
		err = radeon_ucode_validate(rdev->mec_fw);
2082 
		if (err) {
2083 
			printk(KERN_ERR
2084 
			       "cik_fw: validation failed for firmware \"%s\"\n",
2085 
			       fw_name);
2086 
			goto out;
2087 
		} else {
2088 
			new_fw++;
2089 
		}
2090 
	}
2091 
 
2092 
	if (rdev->family == CHIP_KAVERI) {
2093 
		snprintf(fw_name, sizeof(fw_name), "radeon/%s_mec2.bin", new_chip_name);
2094 
		err = request_firmware(&rdev->mec2_fw, fw_name, rdev->dev);
2095 
		if (err) {
2096 
			goto out;
2097 
		} else {
2098 
			err = radeon_ucode_validate(rdev->mec2_fw);
2099 
			if (err) {
2100 
				goto out;
2101 
			} else {
2102 
				new_fw++;
2103 
			}
2104 
		}
2105 
	}
2106 
 
2107 
	snprintf(fw_name, sizeof(fw_name), "radeon/%s_rlc.bin", new_chip_name);
2108 
	err = request_firmware(&rdev->rlc_fw, fw_name, rdev->dev);
2109 
	if (err) {
2110 
	snprintf(fw_name, sizeof(fw_name), "radeon/%s_rlc.bin", chip_name);
2111 
	err = request_firmware(&rdev->rlc_fw, fw_name, rdev->dev);
2112 
	if (err)
2113 
		goto out;
2114 
	if (rdev->rlc_fw->size != rlc_req_size) {
2115 
		printk(KERN_ERR
2116 
		       "cik_rlc: Bogus length %zu in firmware \"%s\"\n",
2117 
		       rdev->rlc_fw->size, fw_name);
2118 
		err = -EINVAL;
2119 
	}
2120 
	} else {
2121 
		err = radeon_ucode_validate(rdev->rlc_fw);
2122 
		if (err) {
2123 
			printk(KERN_ERR
2124 
			       "cik_fw: validation failed for firmware \"%s\"\n",
2125 
			       fw_name);
2126 
			goto out;
2127 
		} else {
2128 
			new_fw++;
2129 
		}
2130 
	}
2131 
 
2132 
	snprintf(fw_name, sizeof(fw_name), "radeon/%s_sdma.bin", new_chip_name);
2133 
	err = request_firmware(&rdev->sdma_fw, fw_name, rdev->dev);
2134 
	if (err) {
2135 
	snprintf(fw_name, sizeof(fw_name), "radeon/%s_sdma.bin", chip_name);
2136 
	err = request_firmware(&rdev->sdma_fw, fw_name, rdev->dev);
2137 
	if (err)
2138 
		goto out;
2139 
	if (rdev->sdma_fw->size != sdma_req_size) {
2140 
		printk(KERN_ERR
2141 
		       "cik_sdma: Bogus length %zu in firmware \"%s\"\n",
2142 
		       rdev->sdma_fw->size, fw_name);
2143 
		err = -EINVAL;
2144 
	}
2145 
	} else {
2146 
		err = radeon_ucode_validate(rdev->sdma_fw);
2147 
		if (err) {
2148 
			printk(KERN_ERR
2149 
			       "cik_fw: validation failed for firmware \"%s\"\n",
2150 
			       fw_name);
2151 
			goto out;
2152 
		} else {
2153 
			new_fw++;
2154 
		}
2155 
	}
2156 
 
2157 
	/* No SMC, MC ucode on APUs */
2158 
	if (!(rdev->flags & RADEON_IS_IGP)) {
2159 
		snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc.bin", new_chip_name);
2160 
		err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev);
2161 
		if (err) {
2162 
		snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc2.bin", chip_name);
2163 
		err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev);
2164 
		if (err) {
2165 
			snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc.bin", chip_name);
2166 
			err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev);
2167 
			if (err)
2168 
				goto out;
2169 
		}
2170 
		if ((rdev->mc_fw->size != mc_req_size) &&
2171 
		    (rdev->mc_fw->size != mc2_req_size)){
2172 
			printk(KERN_ERR
2173 
			       "cik_mc: Bogus length %zu in firmware \"%s\"\n",
2174 
			       rdev->mc_fw->size, fw_name);
2175 
			err = -EINVAL;
2176 
		}
2177 
		DRM_INFO("%s: %zu bytes\n", fw_name, rdev->mc_fw->size);
2178 
		} else {
2179 
			err = radeon_ucode_validate(rdev->mc_fw);
2180 
			if (err) {
2181 
				printk(KERN_ERR
2182 
				       "cik_fw: validation failed for firmware \"%s\"\n",
2183 
				       fw_name);
2184 
				goto out;
2185 
			} else {
2186 
				new_fw++;
2187 
			}
2188 
		}
2189 
 
2190 
		snprintf(fw_name, sizeof(fw_name), "radeon/%s_smc.bin", new_chip_name);
2191 
		err = request_firmware(&rdev->smc_fw, fw_name, rdev->dev);
2192 
		if (err) {
2193 
		snprintf(fw_name, sizeof(fw_name), "radeon/%s_smc.bin", chip_name);
2194 
		err = request_firmware(&rdev->smc_fw, fw_name, rdev->dev);
2195 
		if (err) {
2196 
			printk(KERN_ERR
2197 
			       "smc: error loading firmware \"%s\"\n",
2198 
			       fw_name);
2199 
			release_firmware(rdev->smc_fw);
2200 
			rdev->smc_fw = NULL;
2201 
			err = 0;
2202 
		} else if (rdev->smc_fw->size != smc_req_size) {
2203 
			printk(KERN_ERR
2204 
			       "cik_smc: Bogus length %zu in firmware \"%s\"\n",
2205 
			       rdev->smc_fw->size, fw_name);
2206 
			err = -EINVAL;
2207 
		}
2208 
		} else {
2209 
			err = radeon_ucode_validate(rdev->smc_fw);
2210 
			if (err) {
2211 
				printk(KERN_ERR
2212 
				       "cik_fw: validation failed for firmware \"%s\"\n",
2213 
				       fw_name);
2214 
				goto out;
2215 
			} else {
2216 
				new_fw++;
2217 
			}
2218 
		}
2219 
	}
2220 
 
2221 
	if (new_fw == 0) {
2222 
		rdev->new_fw = false;
2223 
	} else if (new_fw < num_fw) {
2224 
		printk(KERN_ERR "ci_fw: mixing new and old firmware!\n");
2225 
		err = -EINVAL;
2226 
	} else {
2227 
		rdev->new_fw = true;
2228 
	}
2229 
 
2230 
out:
2231 
	if (err) {
2232 
		if (err != -EINVAL)
2233 
			printk(KERN_ERR
2234 
			       "cik_cp: Failed to load firmware \"%s\"\n",
2235 
			       fw_name);
2236 
		release_firmware(rdev->pfp_fw);
2237 
		rdev->pfp_fw = NULL;
2238 
		release_firmware(rdev->me_fw);
2239 
		rdev->me_fw = NULL;
2240 
		release_firmware(rdev->ce_fw);
2241 
		rdev->ce_fw = NULL;
2242 
		release_firmware(rdev->mec_fw);
2243 
		rdev->mec_fw = NULL;
2244 
		release_firmware(rdev->mec2_fw);
2245 
		rdev->mec2_fw = NULL;
2246 
		release_firmware(rdev->rlc_fw);
2247 
		rdev->rlc_fw = NULL;
2248 
		release_firmware(rdev->sdma_fw);
2249 
		rdev->sdma_fw = NULL;
2250 
		release_firmware(rdev->mc_fw);
2251 
		rdev->mc_fw = NULL;
2252 
		release_firmware(rdev->smc_fw);
2253 
		rdev->smc_fw = NULL;
2254 
	}
2255 
	return err;
2256 
}
2257 
 
2258 
/*
2259 
 * Core functions
2260 
 */
2261 
/**
2262 
 * cik_tiling_mode_table_init - init the hw tiling table
2263 
 *
2264 
 * @rdev: radeon_device pointer
2265 
 *
2266 
 * Starting with SI, the tiling setup is done globally in a
2267 
 * set of 32 tiling modes.  Rather than selecting each set of
2268 
 * parameters per surface as on older asics, we just select
2269 
 * which index in the tiling table we want to use, and the
2270 
 * surface uses those parameters (CIK).
2271 
 */
2272 
static void cik_tiling_mode_table_init(struct radeon_device *rdev)
2273 
{
2274 
	const u32 num_tile_mode_states = 32;
2275 
	const u32 num_secondary_tile_mode_states = 16;
2276 
	u32 reg_offset, gb_tile_moden, split_equal_to_row_size;
2277 
	u32 num_pipe_configs;
2278 
	u32 num_rbs = rdev->config.cik.max_backends_per_se *
2279 
		rdev->config.cik.max_shader_engines;
2280 
 
2281 
	switch (rdev->config.cik.mem_row_size_in_kb) {
2282 
	case 1:
2283 
		split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_1KB;
2284 
		break;
2285 
	case 2:
2286 
	default:
2287 
		split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_2KB;
2288 
		break;
2289 
	case 4:
2290 
		split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_4KB;
2291 
		break;
2292 
	}
2293 
 
2294 
	num_pipe_configs = rdev->config.cik.max_tile_pipes;
2295 
	if (num_pipe_configs > 8)
2296 
		num_pipe_configs = 16;
2297 
 
2298 
	if (num_pipe_configs == 16) {
2299 
		for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2300 
			switch (reg_offset) {
2301 
			case 0:
2302 
				gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2303 
						 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2304 
						 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2305 
						 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2306 
				break;
2307 
			case 1:
2308 
				gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2309 
						 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2310 
						 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2311 
						 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2312 
				break;
2313 
			case 2:
2314 
				gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2315 
						 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2316 
						 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2317 
						 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2318 
				break;
2319 
			case 3:
2320 
				gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2321 
						 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2322 
						 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2323 
						 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2324 
				break;
2325 
			case 4:
2326 
				gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2327 
						 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2328 
						 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2329 
						 TILE_SPLIT(split_equal_to_row_size));
2330 
				break;
2331 
			case 5:
2332 
				gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2333 
						 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2334 
						 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2335 
				break;
2336 
			case 6:
2337 
				gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2338 
						 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2339 
						 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2340 
						 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2341 
				break;
2342 
			case 7:
2343 
				gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2344 
						 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2345 
						 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2346 
						 TILE_SPLIT(split_equal_to_row_size));
2347 
				break;
2348 
			case 8:
2349 
				gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2350 
						 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16));
2351 
				break;
2352 
			case 9:
2353 
				gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2354 
						 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2355 
						 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2356 
				break;
2357 
			case 10:
2358 
				gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2359 
						 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2360 
						 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2361 
						 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2362 
				break;
2363 
			case 11:
2364 
				gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2365 
						 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2366 
						 PIPE_CONFIG(ADDR_SURF_P16_32x32_8x16) |
2367 
						 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2368 
				break;
2369 
			case 12:
2370 
				gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2371 
						 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2372 
						 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2373 
						 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2374 
				break;
2375 
			case 13:
2376 
				gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2377 
						 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2378 
						 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2379 
				break;
2380 
			case 14:
2381 
				gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2382 
						 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2383 
						 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2384 
						 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2385 
				break;
2386 
			case 16:
2387 
				gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2388 
						 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2389 
						 PIPE_CONFIG(ADDR_SURF_P16_32x32_8x16) |
2390 
						 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2391 
				break;
2392 
			case 17:
2393 
				gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2394 
						 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2395 
						 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2396 
						 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2397 
				break;
2398 
			case 27:
2399 
				gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2400 
						 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2401 
						 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2402 
				break;
2403 
			case 28:
2404 
				gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2405 
						 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2406 
						 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2407 
						 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2408 
				break;
2409 
			case 29:
2410 
				gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2411 
						 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2412 
						 PIPE_CONFIG(ADDR_SURF_P16_32x32_8x16) |
2413 
						 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2414 
				break;
2415 
			case 30:
2416 
				gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2417 
						 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2418 
						 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2419 
						 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2420 
				break;
2421 
			default:
2422 
				gb_tile_moden = 0;
2423 
				break;
2424 
			}
2425 
			rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
2426 
			WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2427 
		}
2428 
		for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) {
2429 
			switch (reg_offset) {
2430 
			case 0:
2431 
				gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2432 
						 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2433 
						 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2434 
						 NUM_BANKS(ADDR_SURF_16_BANK));
2435 
				break;
2436 
			case 1:
2437 
				gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2438 
						 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2439 
						 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2440 
						 NUM_BANKS(ADDR_SURF_16_BANK));
2441 
				break;
2442 
			case 2:
2443 
				gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2444 
						 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2445 
						 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2446 
						 NUM_BANKS(ADDR_SURF_16_BANK));
2447 
				break;
2448 
			case 3:
2449 
				gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2450 
						 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2451 
						 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2452 
						 NUM_BANKS(ADDR_SURF_16_BANK));
2453 
				break;
2454 
			case 4:
2455 
				gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2456 
						 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2457 
						 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2458 
						 NUM_BANKS(ADDR_SURF_8_BANK));
2459 
				break;
2460 
			case 5:
2461 
				gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2462 
						 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2463 
						 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2464 
						 NUM_BANKS(ADDR_SURF_4_BANK));
2465 
				break;
2466 
			case 6:
2467 
				gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2468 
						 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2469 
						 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2470 
						 NUM_BANKS(ADDR_SURF_2_BANK));
2471 
				break;
2472 
			case 8:
2473 
				gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2474 
						 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2475 
						 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2476 
						 NUM_BANKS(ADDR_SURF_16_BANK));
2477 
				break;
2478 
			case 9:
2479 
				gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2480 
						 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2481 
						 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2482 
						 NUM_BANKS(ADDR_SURF_16_BANK));
2483 
				break;
2484 
			case 10:
2485 
				gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2486 
						 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2487 
						 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2488 
						 NUM_BANKS(ADDR_SURF_16_BANK));
2489 
				break;
2490 
			case 11:
2491 
				gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2492 
						 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2493 
						 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2494 
						 NUM_BANKS(ADDR_SURF_8_BANK));
2495 
				break;
2496 
			case 12:
2497 
				gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2498 
						 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2499 
						 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2500 
						 NUM_BANKS(ADDR_SURF_4_BANK));
2501 
				break;
2502 
			case 13:
2503 
				gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2504 
						 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2505 
						 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2506 
						 NUM_BANKS(ADDR_SURF_2_BANK));
2507 
				break;
2508 
			case 14:
2509 
				gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2510 
						 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2511 
						 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2512 
						 NUM_BANKS(ADDR_SURF_2_BANK));
2513 
				break;
2514 
			default:
2515 
				gb_tile_moden = 0;
2516 
				break;
2517 
			}
2518 
			rdev->config.cik.macrotile_mode_array[reg_offset] = gb_tile_moden;
2519 
			WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2520 
		}
2521 
	} else if (num_pipe_configs == 8) {
2522 
		for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2523 
			switch (reg_offset) {
2524 
			case 0:
2525 
				gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2526 
						 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2527 
						 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2528 
						 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2529 
				break;
2530 
			case 1:
2531 
				gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2532 
						 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2533 
						 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2534 
						 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2535 
				break;
2536 
			case 2:
2537 
				gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2538 
						 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2539 
						 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2540 
						 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2541 
				break;
2542 
			case 3:
2543 
				gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2544 
						 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2545 
						 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2546 
						 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2547 
				break;
2548 
			case 4:
2549 
				gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2550 
						 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2551 
						 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2552 
						 TILE_SPLIT(split_equal_to_row_size));
2553 
				break;
2554 
			case 5:
2555 
				gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2556 
						 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2557 
						 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2558 
				break;
2559 
			case 6:
2560 
				gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2561 
						 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2562 
						 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2563 
						 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2564 
				break;
2565 
			case 7:
2566 
				gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2567 
						 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2568 
						 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2569 
						 TILE_SPLIT(split_equal_to_row_size));
2570 
				break;
2571 
			case 8:
2572 
				gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2573 
						 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16));
2574 
				break;
2575 
			case 9:
2576 
				gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2577 
						 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2578 
						 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2579 
				break;
2580 
			case 10:
2581 
				gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2582 
						 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2583 
						 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2584 
						 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2585 
				break;
2586 
			case 11:
2587 
				gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2588 
						 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2589 
						 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2590 
						 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2591 
				break;
2592 
			case 12:
2593 
				gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2594 
						 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2595 
						 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2596 
						 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2597 
				break;
2598 
			case 13:
2599 
				gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2600 
						 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2601 
						 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2602 
				break;
2603 
			case 14:
2604 
				gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2605 
						 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2606 
						 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2607 
						 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2608 
				break;
2609 
			case 16:
2610 
				gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2611 
						 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2612 
						 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2613 
						 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2614 
				break;
2615 
			case 17:
2616 
				gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2617 
						 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2618 
						 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2619 
						 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2620 
				break;
2621 
			case 27:
2622 
				gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2623 
						 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2624 
						 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2625 
				break;
2626 
			case 28:
2627 
				gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2628 
						 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2629 
						 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2630 
						 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2631 
				break;
2632 
			case 29:
2633 
				gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2634 
						 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2635 
						 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2636 
						 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2637 
				break;
2638 
			case 30:
2639 
				gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2640 
						 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2641 
						 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2642 
						 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2643 
				break;
2644 
			default:
2645 
				gb_tile_moden = 0;
2646 
				break;
2647 
			}
2648 
			rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
2649 
			WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2650 
		}
2651 
		for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) {
2652 
			switch (reg_offset) {
2653 
			case 0:
2654 
				gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2655 
						 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2656 
						 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2657 
						 NUM_BANKS(ADDR_SURF_16_BANK));
2658 
				break;
2659 
			case 1:
2660 
				gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2661 
						 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2662 
						 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2663 
						 NUM_BANKS(ADDR_SURF_16_BANK));
2664 
				break;
2665 
			case 2:
2666 
				gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2667 
						 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2668 
						 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2669 
						 NUM_BANKS(ADDR_SURF_16_BANK));
2670 
				break;
2671 
			case 3:
2672 
				gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2673 
						 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2674 
						 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2675 
						 NUM_BANKS(ADDR_SURF_16_BANK));
2676 
				break;
2677 
			case 4:
2678 
				gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2679 
						 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2680 
						 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2681 
						 NUM_BANKS(ADDR_SURF_8_BANK));
2682 
				break;
2683 
			case 5:
2684 
				gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2685 
						 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2686 
						 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2687 
						 NUM_BANKS(ADDR_SURF_4_BANK));
2688 
				break;
2689 
			case 6:
2690 
				gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2691 
						 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2692 
						 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2693 
						 NUM_BANKS(ADDR_SURF_2_BANK));
2694 
				break;
2695 
			case 8:
2696 
				gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2697 
						 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
2698 
						 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2699 
						 NUM_BANKS(ADDR_SURF_16_BANK));
2700 
				break;
2701 
			case 9:
2702 
				gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2703 
						 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2704 
						 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2705 
						 NUM_BANKS(ADDR_SURF_16_BANK));
2706 
				break;
2707 
			case 10:
2708 
				gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2709 
						 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2710 
						 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2711 
						 NUM_BANKS(ADDR_SURF_16_BANK));
2712 
				break;
2713 
			case 11:
2714 
				gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2715 
						 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2716 
						 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2717 
						 NUM_BANKS(ADDR_SURF_16_BANK));
2718 
				break;
2719 
			case 12:
2720 
				gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2721 
						 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2722 
						 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2723 
						 NUM_BANKS(ADDR_SURF_8_BANK));
2724 
				break;
2725 
			case 13:
2726 
				gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2727 
						 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2728 
						 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2729 
						 NUM_BANKS(ADDR_SURF_4_BANK));
2730 
				break;
2731 
			case 14:
2732 
				gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2733 
						 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2734 
						 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2735 
						 NUM_BANKS(ADDR_SURF_2_BANK));
2736 
				break;
2737 
			default:
2738 
				gb_tile_moden = 0;
2739 
				break;
2740 
			}
2741 
			rdev->config.cik.macrotile_mode_array[reg_offset] = gb_tile_moden;
2742 
			WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2743 
		}
2744 
	} else if (num_pipe_configs == 4) {
2745 
		if (num_rbs == 4) {
2746 
			for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2747 
				switch (reg_offset) {
2748 
				case 0:
2749 
					gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2750 
							 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2751 
							 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2752 
							 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2753 
					break;
2754 
				case 1:
2755 
					gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2756 
							 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2757 
							 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2758 
							 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2759 
					break;
2760 
				case 2:
2761 
					gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2762 
							 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2763 
							 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2764 
							 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2765 
					break;
2766 
				case 3:
2767 
					gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2768 
							 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2769 
							 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2770 
							 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2771 
					break;
2772 
				case 4:
2773 
					gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2774 
							 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2775 
							 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2776 
							 TILE_SPLIT(split_equal_to_row_size));
2777 
					break;
2778 
				case 5:
2779 
					gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2780 
							 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2781 
							 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2782 
					break;
2783 
				case 6:
2784 
					gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2785 
							 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2786 
							 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2787 
							 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2788 
					break;
2789 
				case 7:
2790 
					gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2791 
							 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2792 
							 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2793 
							 TILE_SPLIT(split_equal_to_row_size));
2794 
					break;
2795 
				case 8:
2796 
					gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2797 
							 PIPE_CONFIG(ADDR_SURF_P4_16x16));
2798 
					break;
2799 
				case 9:
2800 
					gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2801 
							 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2802 
							 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2803 
					break;
2804 
				case 10:
2805 
					gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2806 
							 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2807 
							 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2808 
							 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2809 
					break;
2810 
				case 11:
2811 
					gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2812 
							 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2813 
							 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2814 
							 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2815 
					break;
2816 
				case 12:
2817 
					gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2818 
							 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2819 
							 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2820 
							 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2821 
					break;
2822 
				case 13:
2823 
					gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2824 
							 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2825 
							 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2826 
					break;
2827 
				case 14:
2828 
					gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2829 
							 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2830 
							 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2831 
							 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2832 
					break;
2833 
				case 16:
2834 
					gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2835 
							 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2836 
							 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2837 
							 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2838 
					break;
2839 
				case 17:
2840 
					gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2841 
							 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2842 
							 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2843 
							 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2844 
					break;
2845 
				case 27:
2846 
					gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2847 
							 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2848 
							 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2849 
					break;
2850 
				case 28:
2851 
					gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2852 
							 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2853 
							 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2854 
							 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2855 
					break;
2856 
				case 29:
2857 
					gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2858 
							 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2859 
							 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2860 
							 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2861 
					break;
2862 
				case 30:
2863 
					gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2864 
							 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2865 
							 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2866 
							 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2867 
					break;
2868 
				default:
2869 
					gb_tile_moden = 0;
2870 
					break;
2871 
				}
2872 
				rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
2873 
				WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2874 
			}
2875 
		} else if (num_rbs < 4) {
2876 
			for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2877 
				switch (reg_offset) {
2878 
				case 0:
2879 
					gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2880 
							 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2881 
							 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2882 
							 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2883 
					break;
2884 
				case 1:
2885 
					gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2886 
							 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2887 
							 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2888 
							 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2889 
					break;
2890 
				case 2:
2891 
					gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2892 
							 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2893 
							 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2894 
							 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2895 
					break;
2896 
				case 3:
2897 
					gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2898 
							 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2899 
							 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2900 
							 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2901 
					break;
2902 
				case 4:
2903 
					gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2904 
							 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2905 
							 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2906 
							 TILE_SPLIT(split_equal_to_row_size));
2907 
					break;
2908 
				case 5:
2909 
					gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2910 
							 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2911 
							 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2912 
					break;
2913 
				case 6:
2914 
					gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2915 
							 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2916 
							 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2917 
							 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2918 
					break;
2919 
				case 7:
2920 
					gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2921 
							 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2922 
							 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2923 
							 TILE_SPLIT(split_equal_to_row_size));
2924 
					break;
2925 
				case 8:
2926 
					gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2927 
						 PIPE_CONFIG(ADDR_SURF_P4_8x16));
2928 
					break;
2929 
				case 9:
2930 
					gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2931 
							 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2932 
							 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2933 
					break;
2934 
				case 10:
2935 
					gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2936 
							 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2937 
							 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2938 
							 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2939 
					break;
2940 
				case 11:
2941 
					gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2942 
							 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2943 
							 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2944 
							 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2945 
					break;
2946 
				case 12:
2947 
					gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2948 
							 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2949 
							 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2950 
							 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2951 
					break;
2952 
				case 13:
2953 
					gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2954 
							 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2955 
							 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2956 
					break;
2957 
				case 14:
2958 
					gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2959 
							 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2960 
							 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2961 
							 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2962 
					break;
2963 
				case 16:
2964 
					gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2965 
							 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2966 
							 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2967 
							 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2968 
					break;
2969 
				case 17:
2970 
					gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2971 
							 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2972 
							 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2973 
							 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2974 
					break;
2975 
				case 27:
2976 
					gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2977 
							 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2978 
							 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2979 
					break;
2980 
				case 28:
2981 
					gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2982 
							 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2983 
							 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2984 
							 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2985 
					break;
2986 
				case 29:
2987 
					gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2988 
							 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2989 
							 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2990 
							 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2991 
					break;
2992 
				case 30:
2993 
					gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2994 
							 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2995 
							 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2996 
							 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2997 
					break;
2998 
				default:
2999 
					gb_tile_moden = 0;
3000 
					break;
3001 
				}
3002 
				rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
3003 
				WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
3004 
			}
3005 
		}
3006 
		for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) {
3007 
			switch (reg_offset) {
3008 
			case 0:
3009 
				gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3010 
						 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
3011 
						 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3012 
						 NUM_BANKS(ADDR_SURF_16_BANK));
3013 
				break;
3014 
			case 1:
3015 
				gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3016 
						 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
3017 
						 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3018 
						 NUM_BANKS(ADDR_SURF_16_BANK));
3019 
				break;
3020 
			case 2:
3021 
				gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3022 
						 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3023 
						 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3024 
						 NUM_BANKS(ADDR_SURF_16_BANK));
3025 
				break;
3026 
			case 3:
3027 
				gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3028 
						 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3029 
						 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3030 
						 NUM_BANKS(ADDR_SURF_16_BANK));
3031 
				break;
3032 
			case 4:
3033 
				gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3034 
						 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3035 
						 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3036 
						 NUM_BANKS(ADDR_SURF_16_BANK));
3037 
				break;
3038 
			case 5:
3039 
				gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3040 
						 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3041 
						 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3042 
						 NUM_BANKS(ADDR_SURF_8_BANK));
3043 
				break;
3044 
			case 6:
3045 
				gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3046 
						 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3047 
						 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
3048 
						 NUM_BANKS(ADDR_SURF_4_BANK));
3049 
				break;
3050 
			case 8:
3051 
				gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
3052 
						 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
3053 
						 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3054 
						 NUM_BANKS(ADDR_SURF_16_BANK));
3055 
				break;
3056 
			case 9:
3057 
				gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
3058 
						 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
3059 
						 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3060 
						 NUM_BANKS(ADDR_SURF_16_BANK));
3061 
				break;
3062 
			case 10:
3063 
				gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3064 
						 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
3065 
						 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3066 
						 NUM_BANKS(ADDR_SURF_16_BANK));
3067 
				break;
3068 
			case 11:
3069 
				gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3070 
						 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
3071 
						 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3072 
						 NUM_BANKS(ADDR_SURF_16_BANK));
3073 
				break;
3074 
			case 12:
3075 
				gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3076 
						 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3077 
						 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3078 
						 NUM_BANKS(ADDR_SURF_16_BANK));
3079 
				break;
3080 
			case 13:
3081 
				gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3082 
						 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3083 
						 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3084 
						 NUM_BANKS(ADDR_SURF_8_BANK));
3085 
				break;
3086 
			case 14:
3087 
				gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3088 
						 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3089 
						 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
3090 
						 NUM_BANKS(ADDR_SURF_4_BANK));
3091 
				break;
3092 
			default:
3093 
				gb_tile_moden = 0;
3094 
				break;
3095 
			}
3096 
			rdev->config.cik.macrotile_mode_array[reg_offset] = gb_tile_moden;
3097 
			WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), gb_tile_moden);
3098 
		}
3099 
	} else if (num_pipe_configs == 2) {
3100 
		for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
3101 
			switch (reg_offset) {
3102 
			case 0:
3103 
				gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3104 
						 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
3105 
						 PIPE_CONFIG(ADDR_SURF_P2) |
3106 
						 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
3107 
				break;
3108 
			case 1:
3109 
				gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3110 
						 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
3111 
						 PIPE_CONFIG(ADDR_SURF_P2) |
3112 
						 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
3113 
				break;
3114 
			case 2:
3115 
				gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3116 
						 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
3117 
						 PIPE_CONFIG(ADDR_SURF_P2) |
3118 
						 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
3119 
				break;
3120 
			case 3:
3121 
				gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3122 
						 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
3123 
						 PIPE_CONFIG(ADDR_SURF_P2) |
3124 
						 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
3125 
				break;
3126 
			case 4:
3127 
				gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3128 
						 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
3129 
						 PIPE_CONFIG(ADDR_SURF_P2) |
3130 
						 TILE_SPLIT(split_equal_to_row_size));
3131 
				break;
3132 
			case 5:
3133 
				gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
3134 
						 PIPE_CONFIG(ADDR_SURF_P2) |
3135 
						 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
3136 
				break;
3137 
			case 6:
3138 
				gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
3139 
						 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
3140 
						 PIPE_CONFIG(ADDR_SURF_P2) |
3141 
						 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
3142 
				break;
3143 
			case 7:
3144 
				gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
3145 
						 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
3146 
						 PIPE_CONFIG(ADDR_SURF_P2) |
3147 
						 TILE_SPLIT(split_equal_to_row_size));
3148 
				break;
3149 
			case 8:
3150 
				gb_tile_moden = ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
3151 
						PIPE_CONFIG(ADDR_SURF_P2);
3152 
				break;
3153 
			case 9:
3154 
				gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
3155 
						 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
3156 
						 PIPE_CONFIG(ADDR_SURF_P2));
3157 
				break;
3158 
			case 10:
3159 
				gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3160 
						 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
3161 
						 PIPE_CONFIG(ADDR_SURF_P2) |
3162 
						 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3163 
				break;
3164 
			case 11:
3165 
				gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
3166 
						 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
3167 
						 PIPE_CONFIG(ADDR_SURF_P2) |
3168 
						 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3169 
				break;
3170 
			case 12:
3171 
				gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
3172 
						 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
3173 
						 PIPE_CONFIG(ADDR_SURF_P2) |
3174 
						 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3175 
				break;
3176 
			case 13:
3177 
				gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
3178 
						 PIPE_CONFIG(ADDR_SURF_P2) |
3179 
						 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
3180 
				break;
3181 
			case 14:
3182 
				gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3183 
						 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
3184 
						 PIPE_CONFIG(ADDR_SURF_P2) |
3185 
						 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3186 
				break;
3187 
			case 16:
3188 
				gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
3189 
						 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
3190 
						 PIPE_CONFIG(ADDR_SURF_P2) |
3191 
						 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3192 
				break;
3193 
			case 17:
3194 
				gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
3195 
						 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
3196 
						 PIPE_CONFIG(ADDR_SURF_P2) |
3197 
						 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3198 
				break;
3199 
			case 27:
3200 
				gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
3201 
						 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
3202 
						 PIPE_CONFIG(ADDR_SURF_P2));
3203 
				break;
3204 
			case 28:
3205 
				gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
3206 
						 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
3207 
						 PIPE_CONFIG(ADDR_SURF_P2) |
3208 
						 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3209 
				break;
3210 
			case 29:
3211 
				gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
3212 
						 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
3213 
						 PIPE_CONFIG(ADDR_SURF_P2) |
3214 
						 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3215 
				break;
3216 
			case 30:
3217 
				gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
3218 
						 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
3219 
						 PIPE_CONFIG(ADDR_SURF_P2) |
3220 
						 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3221 
				break;
3222 
			default:
3223 
				gb_tile_moden = 0;
3224 
				break;
3225 
			}
3226 
			rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
3227 
			WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
3228 
		}
3229 
		for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) {
3230 
			switch (reg_offset) {
3231 
			case 0:
3232 
				gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
3233 
						 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
3234 
						 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3235 
						 NUM_BANKS(ADDR_SURF_16_BANK));
3236 
				break;
3237 
			case 1:
3238 
				gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
3239 
						 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
3240 
						 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3241 
						 NUM_BANKS(ADDR_SURF_16_BANK));
3242 
				break;
3243 
			case 2:
3244 
				gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3245 
						 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
3246 
						 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3247 
						 NUM_BANKS(ADDR_SURF_16_BANK));
3248 
				break;
3249 
			case 3:
3250 
				gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3251 
						 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3252 
						 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3253 
						 NUM_BANKS(ADDR_SURF_16_BANK));
3254 
				break;
3255 
			case 4:
3256 
				gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3257 
						 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3258 
						 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3259 
						 NUM_BANKS(ADDR_SURF_16_BANK));
3260 
				break;
3261 
			case 5:
3262 
				gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3263 
						 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3264 
						 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3265 
						 NUM_BANKS(ADDR_SURF_16_BANK));
3266 
				break;
3267 
			case 6:
3268 
				gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3269 
						 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3270 
						 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3271 
						 NUM_BANKS(ADDR_SURF_8_BANK));
3272 
				break;
3273 
			case 8:
3274 
				gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) |
3275 
						 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
3276 
						 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3277 
						 NUM_BANKS(ADDR_SURF_16_BANK));
3278 
				break;
3279 
			case 9:
3280 
				gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) |
3281 
						 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
3282 
						 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3283 
						 NUM_BANKS(ADDR_SURF_16_BANK));
3284 
				break;
3285 
			case 10:
3286 
				gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
3287 
						 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
3288 
						 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3289 
						 NUM_BANKS(ADDR_SURF_16_BANK));
3290 
				break;
3291 
			case 11:
3292 
				gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
3293 
						 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
3294 
						 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3295 
						 NUM_BANKS(ADDR_SURF_16_BANK));
3296 
				break;
3297 
			case 12:
3298 
				gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3299 
						 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
3300 
						 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3301 
						 NUM_BANKS(ADDR_SURF_16_BANK));
3302 
				break;
3303 
			case 13:
3304 
				gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3305 
						 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3306 
						 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3307 
						 NUM_BANKS(ADDR_SURF_16_BANK));
3308 
				break;
3309 
			case 14:
3310 
				gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3311 
						 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3312 
						 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3313 
						 NUM_BANKS(ADDR_SURF_8_BANK));
3314 
				break;
3315 
			default:
3316 
				gb_tile_moden = 0;
3317 
				break;
3318 
			}
3319 
			rdev->config.cik.macrotile_mode_array[reg_offset] = gb_tile_moden;
3320 
			WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), gb_tile_moden);
3321 
		}
3322 
	} else
3323 
		DRM_ERROR("unknown num pipe config: 0x%x\n", num_pipe_configs);
3324 
}
3325 
 
3326 
/**
3327 
 * cik_select_se_sh - select which SE, SH to address
3328 
 *
3329 
 * @rdev: radeon_device pointer
3330 
 * @se_num: shader engine to address
3331 
 * @sh_num: sh block to address
3332 
 *
3333 
 * Select which SE, SH combinations to address. Certain
3334 
 * registers are instanced per SE or SH.  0xffffffff means
3335 
 * broadcast to all SEs or SHs (CIK).
3336 
 */
3337 
static void cik_select_se_sh(struct radeon_device *rdev,
3338 
			     u32 se_num, u32 sh_num)
3339 
{
3340 
	u32 data = INSTANCE_BROADCAST_WRITES;
3341 
 
3342 
	if ((se_num == 0xffffffff) && (sh_num == 0xffffffff))
3343 
		data |= SH_BROADCAST_WRITES | SE_BROADCAST_WRITES;
3344 
	else if (se_num == 0xffffffff)
3345 
		data |= SE_BROADCAST_WRITES | SH_INDEX(sh_num);
3346 
	else if (sh_num == 0xffffffff)
3347 
		data |= SH_BROADCAST_WRITES | SE_INDEX(se_num);
3348 
	else
3349 
		data |= SH_INDEX(sh_num) | SE_INDEX(se_num);
3350 
	WREG32(GRBM_GFX_INDEX, data);
3351 
}
3352 
 
3353 
/**
3354 
 * cik_create_bitmask - create a bitmask
3355 
 *
3356 
 * @bit_width: length of the mask
3357 
 *
3358 
 * create a variable length bit mask (CIK).
3359 
 * Returns the bitmask.
3360 
 */
3361 
static u32 cik_create_bitmask(u32 bit_width)
3362 
{
3363 
	u32 i, mask = 0;
3364 
 
3365 
	for (i = 0; i < bit_width; i++) {
3366 
		mask <<= 1;
3367 
		mask |= 1;
3368 
	}
3369 
	return mask;
3370 
}
3371 
 
3372 
/**
3373 
 * cik_get_rb_disabled - computes the mask of disabled RBs
3374 
 *
3375 
 * @rdev: radeon_device pointer
3376 
 * @max_rb_num: max RBs (render backends) for the asic
3377 
 * @se_num: number of SEs (shader engines) for the asic
3378 
 * @sh_per_se: number of SH blocks per SE for the asic
3379 
 *
3380 
 * Calculates the bitmask of disabled RBs (CIK).
3381 
 * Returns the disabled RB bitmask.
3382 
 */
3383 
static u32 cik_get_rb_disabled(struct radeon_device *rdev,
3384 
			      u32 max_rb_num_per_se,
3385 
			      u32 sh_per_se)
3386 
{
3387 
	u32 data, mask;
3388 
 
3389 
	data = RREG32(CC_RB_BACKEND_DISABLE);
3390 
	if (data & 1)
3391 
		data &= BACKEND_DISABLE_MASK;
3392 
	else
3393 
		data = 0;
3394 
	data |= RREG32(GC_USER_RB_BACKEND_DISABLE);
3395 
 
3396 
	data >>= BACKEND_DISABLE_SHIFT;
3397 
 
3398 
	mask = cik_create_bitmask(max_rb_num_per_se / sh_per_se);
3399 
 
3400 
	return data & mask;
3401 
}
3402 
 
3403 
/**
3404 
 * cik_setup_rb - setup the RBs on the asic
3405 
 *
3406 
 * @rdev: radeon_device pointer
3407 
 * @se_num: number of SEs (shader engines) for the asic
3408 
 * @sh_per_se: number of SH blocks per SE for the asic
3409 
 * @max_rb_num: max RBs (render backends) for the asic
3410 
 *
3411 
 * Configures per-SE/SH RB registers (CIK).
3412 
 */
3413 
static void cik_setup_rb(struct radeon_device *rdev,
3414 
			 u32 se_num, u32 sh_per_se,
3415 
			 u32 max_rb_num_per_se)
3416 
{
3417 
	int i, j;
3418 
	u32 data, mask;
3419 
	u32 disabled_rbs = 0;
3420 
	u32 enabled_rbs = 0;
3421 
 
3422 
	for (i = 0; i < se_num; i++) {
3423 
		for (j = 0; j < sh_per_se; j++) {
3424 
			cik_select_se_sh(rdev, i, j);
3425 
			data = cik_get_rb_disabled(rdev, max_rb_num_per_se, sh_per_se);
3426 
			if (rdev->family == CHIP_HAWAII)
3427 
				disabled_rbs |= data << ((i * sh_per_se + j) * HAWAII_RB_BITMAP_WIDTH_PER_SH);
3428 
			else
3429 
				disabled_rbs |= data << ((i * sh_per_se + j) * CIK_RB_BITMAP_WIDTH_PER_SH);
3430 
		}
3431 
	}
3432 
	cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
3433 
 
3434 
	mask = 1;
3435 
	for (i = 0; i < max_rb_num_per_se * se_num; i++) {
3436 
		if (!(disabled_rbs & mask))
3437 
			enabled_rbs |= mask;
3438 
		mask <<= 1;
3439 
	}
3440 
 
3441 
	rdev->config.cik.backend_enable_mask = enabled_rbs;
3442 
 
3443 
	for (i = 0; i < se_num; i++) {
3444 
		cik_select_se_sh(rdev, i, 0xffffffff);
3445 
		data = 0;
3446 
		for (j = 0; j < sh_per_se; j++) {
3447 
			switch (enabled_rbs & 3) {
3448 
			case 0:
3449 
				if (j == 0)
3450 
					data |= PKR_MAP(RASTER_CONFIG_RB_MAP_3);
3451 
				else
3452 
					data |= PKR_MAP(RASTER_CONFIG_RB_MAP_0);
3453 
				break;
3454 
			case 1:
3455 
				data |= (RASTER_CONFIG_RB_MAP_0 << (i * sh_per_se + j) * 2);
3456 
				break;
3457 
			case 2:
3458 
				data |= (RASTER_CONFIG_RB_MAP_3 << (i * sh_per_se + j) * 2);
3459 
				break;
3460 
			case 3:
3461 
			default:
3462 
				data |= (RASTER_CONFIG_RB_MAP_2 << (i * sh_per_se + j) * 2);
3463 
				break;
3464 
			}
3465 
			enabled_rbs >>= 2;
3466 
		}
3467 
		WREG32(PA_SC_RASTER_CONFIG, data);
3468 
	}
3469 
	cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
3470 
}
3471 
 
3472 
/**
3473 
 * cik_gpu_init - setup the 3D engine
3474 
 *
3475 
 * @rdev: radeon_device pointer
3476 
 *
3477 
 * Configures the 3D engine and tiling configuration
3478 
 * registers so that the 3D engine is usable.
3479 
 */
3480 
static void cik_gpu_init(struct radeon_device *rdev)
3481 
{
3482 
	u32 gb_addr_config = RREG32(GB_ADDR_CONFIG);
3483 
	u32 mc_shared_chmap, mc_arb_ramcfg;
3484 
	u32 hdp_host_path_cntl;
3485 
	u32 tmp;
3486 
	int i, j;
3487 
 
3488 
	switch (rdev->family) {
3489 
	case CHIP_BONAIRE:
3490 
		rdev->config.cik.max_shader_engines = 2;
3491 
		rdev->config.cik.max_tile_pipes = 4;
3492 
		rdev->config.cik.max_cu_per_sh = 7;
3493 
		rdev->config.cik.max_sh_per_se = 1;
3494 
		rdev->config.cik.max_backends_per_se = 2;
3495 
		rdev->config.cik.max_texture_channel_caches = 4;
3496 
		rdev->config.cik.max_gprs = 256;
3497 
		rdev->config.cik.max_gs_threads = 32;
3498 
		rdev->config.cik.max_hw_contexts = 8;
3499 
 
3500 
		rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
3501 
		rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
3502 
		rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
3503 
		rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
3504 
		gb_addr_config = BONAIRE_GB_ADDR_CONFIG_GOLDEN;
3505 
		break;
3506 
	case CHIP_HAWAII:
3507 
		rdev->config.cik.max_shader_engines = 4;
3508 
		rdev->config.cik.max_tile_pipes = 16;
3509 
		rdev->config.cik.max_cu_per_sh = 11;
3510 
		rdev->config.cik.max_sh_per_se = 1;
3511 
		rdev->config.cik.max_backends_per_se = 4;
3512 
		rdev->config.cik.max_texture_channel_caches = 16;
3513 
		rdev->config.cik.max_gprs = 256;
3514 
		rdev->config.cik.max_gs_threads = 32;
3515 
		rdev->config.cik.max_hw_contexts = 8;
3516 
 
3517 
		rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
3518 
		rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
3519 
		rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
3520 
		rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
3521 
		gb_addr_config = HAWAII_GB_ADDR_CONFIG_GOLDEN;
3522 
		break;
3523 
	case CHIP_KAVERI:
3524 
		rdev->config.cik.max_shader_engines = 1;
3525 
		rdev->config.cik.max_tile_pipes = 4;
3526 
		if ((rdev->pdev->device == 0x1304) ||
3527 
		    (rdev->pdev->device == 0x1305) ||
3528 
		    (rdev->pdev->device == 0x130C) ||
3529 
		    (rdev->pdev->device == 0x130F) ||
3530 
		    (rdev->pdev->device == 0x1310) ||
3531 
		    (rdev->pdev->device == 0x1311) ||
3532 
		    (rdev->pdev->device == 0x131C)) {
3533 
			rdev->config.cik.max_cu_per_sh = 8;
3534 
			rdev->config.cik.max_backends_per_se = 2;
3535 
		} else if ((rdev->pdev->device == 0x1309) ||
3536 
			   (rdev->pdev->device == 0x130A) ||
3537 
			   (rdev->pdev->device == 0x130D) ||
3538 
			   (rdev->pdev->device == 0x1313) ||
3539 
			   (rdev->pdev->device == 0x131D)) {
3540 
			rdev->config.cik.max_cu_per_sh = 6;
3541 
			rdev->config.cik.max_backends_per_se = 2;
3542 
		} else if ((rdev->pdev->device == 0x1306) ||
3543 
			   (rdev->pdev->device == 0x1307) ||
3544 
			   (rdev->pdev->device == 0x130B) ||
3545 
			   (rdev->pdev->device == 0x130E) ||
3546 
			   (rdev->pdev->device == 0x1315) ||
3547 
			   (rdev->pdev->device == 0x1318) ||
3548 
			   (rdev->pdev->device == 0x131B)) {
3549 
			rdev->config.cik.max_cu_per_sh = 4;
3550 
			rdev->config.cik.max_backends_per_se = 1;
3551 
		} else {
3552 
			rdev->config.cik.max_cu_per_sh = 3;
3553 
			rdev->config.cik.max_backends_per_se = 1;
3554 
		}
3555 
		rdev->config.cik.max_sh_per_se = 1;
3556 
		rdev->config.cik.max_texture_channel_caches = 4;
3557 
		rdev->config.cik.max_gprs = 256;
3558 
		rdev->config.cik.max_gs_threads = 16;
3559 
		rdev->config.cik.max_hw_contexts = 8;
3560 
 
3561 
		rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
3562 
		rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
3563 
		rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
3564 
		rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
3565 
		gb_addr_config = BONAIRE_GB_ADDR_CONFIG_GOLDEN;
3566 
		break;
3567 
	case CHIP_KABINI:
3568 
	case CHIP_MULLINS:
3569 
	default:
3570 
		rdev->config.cik.max_shader_engines = 1;
3571 
		rdev->config.cik.max_tile_pipes = 2;
3572 
		rdev->config.cik.max_cu_per_sh = 2;
3573 
		rdev->config.cik.max_sh_per_se = 1;
3574 
		rdev->config.cik.max_backends_per_se = 1;
3575 
		rdev->config.cik.max_texture_channel_caches = 2;
3576 
		rdev->config.cik.max_gprs = 256;
3577 
		rdev->config.cik.max_gs_threads = 16;
3578 
		rdev->config.cik.max_hw_contexts = 8;
3579 
 
3580 
		rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
3581 
		rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
3582 
		rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
3583 
		rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
3584 
		gb_addr_config = BONAIRE_GB_ADDR_CONFIG_GOLDEN;
3585 
		break;
3586 
	}
3587 
 
3588 
	/* Initialize HDP */
3589 
	for (i = 0, j = 0; i < 32; i++, j += 0x18) {
3590 
		WREG32((0x2c14 + j), 0x00000000);
3591 
		WREG32((0x2c18 + j), 0x00000000);
3592 
		WREG32((0x2c1c + j), 0x00000000);
3593 
		WREG32((0x2c20 + j), 0x00000000);
3594 
		WREG32((0x2c24 + j), 0x00000000);
3595 
	}
3596 
 
3597 
	WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
3598 
 
3599 
	WREG32(BIF_FB_EN, FB_READ_EN | FB_WRITE_EN);
3600 
 
3601 
	mc_shared_chmap = RREG32(MC_SHARED_CHMAP);
3602 
	mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
3603 
 
3604 
	rdev->config.cik.num_tile_pipes = rdev->config.cik.max_tile_pipes;
3605 
	rdev->config.cik.mem_max_burst_length_bytes = 256;
3606 
	tmp = (mc_arb_ramcfg & NOOFCOLS_MASK) >> NOOFCOLS_SHIFT;
3607 
	rdev->config.cik.mem_row_size_in_kb = (4 * (1 << (8 + tmp))) / 1024;
3608 
	if (rdev->config.cik.mem_row_size_in_kb > 4)
3609 
		rdev->config.cik.mem_row_size_in_kb = 4;
3610 
	/* XXX use MC settings? */
3611 
	rdev->config.cik.shader_engine_tile_size = 32;
3612 
	rdev->config.cik.num_gpus = 1;
3613 
	rdev->config.cik.multi_gpu_tile_size = 64;
3614 
 
3615 
	/* fix up row size */
3616 
	gb_addr_config &= ~ROW_SIZE_MASK;
3617 
	switch (rdev->config.cik.mem_row_size_in_kb) {
3618 
	case 1:
3619 
	default:
3620 
		gb_addr_config |= ROW_SIZE(0);
3621 
		break;
3622 
	case 2:
3623 
		gb_addr_config |= ROW_SIZE(1);
3624 
		break;
3625 
	case 4:
3626 
		gb_addr_config |= ROW_SIZE(2);
3627 
		break;
3628 
	}
3629 
 
3630 
	/* setup tiling info dword.  gb_addr_config is not adequate since it does
3631 
	 * not have bank info, so create a custom tiling dword.
3632 
	 * bits 3:0   num_pipes
3633 
	 * bits 7:4   num_banks
3634 
	 * bits 11:8  group_size
3635 
	 * bits 15:12 row_size
3636 
	 */
3637 
	rdev->config.cik.tile_config = 0;
3638 
	switch (rdev->config.cik.num_tile_pipes) {
3639 
	case 1:
3640 
		rdev->config.cik.tile_config |= (0 << 0);
3641 
		break;
3642 
	case 2:
3643 
		rdev->config.cik.tile_config |= (1 << 0);
3644 
		break;
3645 
	case 4:
3646 
		rdev->config.cik.tile_config |= (2 << 0);
3647 
		break;
3648 
	case 8:
3649 
	default:
3650 
		/* XXX what about 12? */
3651 
		rdev->config.cik.tile_config |= (3 << 0);
3652 
		break;
3653 
	}
3654 
	rdev->config.cik.tile_config |=
3655 
		((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT) << 4;
3656 
	rdev->config.cik.tile_config |=
3657 
		((gb_addr_config & PIPE_INTERLEAVE_SIZE_MASK) >> PIPE_INTERLEAVE_SIZE_SHIFT) << 8;
3658 
	rdev->config.cik.tile_config |=
3659 
		((gb_addr_config & ROW_SIZE_MASK) >> ROW_SIZE_SHIFT) << 12;
3660 
 
3661 
	WREG32(GB_ADDR_CONFIG, gb_addr_config);
3662 
	WREG32(HDP_ADDR_CONFIG, gb_addr_config);
3663 
	WREG32(DMIF_ADDR_CALC, gb_addr_config);
3664 
	WREG32(SDMA0_TILING_CONFIG + SDMA0_REGISTER_OFFSET, gb_addr_config & 0x70);
3665 
	WREG32(SDMA0_TILING_CONFIG + SDMA1_REGISTER_OFFSET, gb_addr_config & 0x70);
3666 
	WREG32(UVD_UDEC_ADDR_CONFIG, gb_addr_config);
3667 
	WREG32(UVD_UDEC_DB_ADDR_CONFIG, gb_addr_config);
3668 
	WREG32(UVD_UDEC_DBW_ADDR_CONFIG, gb_addr_config);
3669 
 
3670 
	cik_tiling_mode_table_init(rdev);
3671 
 
3672 
	cik_setup_rb(rdev, rdev->config.cik.max_shader_engines,
3673 
		     rdev->config.cik.max_sh_per_se,
3674 
		     rdev->config.cik.max_backends_per_se);
3675 
 
3676 
	rdev->config.cik.active_cus = 0;
3677 
	for (i = 0; i < rdev->config.cik.max_shader_engines; i++) {
3678 
		for (j = 0; j < rdev->config.cik.max_sh_per_se; j++) {
3679 
				rdev->config.cik.active_cus +=
3680 
					hweight32(cik_get_cu_active_bitmap(rdev, i, j));
3681 
			}
3682 
		}
3683 
 
3684 
	/* set HW defaults for 3D engine */
3685 
	WREG32(CP_MEQ_THRESHOLDS, MEQ1_START(0x30) | MEQ2_START(0x60));
3686 
 
3687 
	WREG32(SX_DEBUG_1, 0x20);
3688 
 
3689 
	WREG32(TA_CNTL_AUX, 0x00010000);
3690 
 
3691 
	tmp = RREG32(SPI_CONFIG_CNTL);
3692 
	tmp |= 0x03000000;
3693 
	WREG32(SPI_CONFIG_CNTL, tmp);
3694 
 
3695 
	WREG32(SQ_CONFIG, 1);
3696 
 
3697 
	WREG32(DB_DEBUG, 0);
3698 
 
3699 
	tmp = RREG32(DB_DEBUG2) & ~0xf00fffff;
3700 
	tmp |= 0x00000400;
3701 
	WREG32(DB_DEBUG2, tmp);
3702 
 
3703 
	tmp = RREG32(DB_DEBUG3) & ~0x0002021c;
3704 
	tmp |= 0x00020200;
3705 
	WREG32(DB_DEBUG3, tmp);
3706 
 
3707 
	tmp = RREG32(CB_HW_CONTROL) & ~0x00010000;
3708 
	tmp |= 0x00018208;
3709 
	WREG32(CB_HW_CONTROL, tmp);
3710 
 
3711 
	WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4));
3712 
 
3713 
	WREG32(PA_SC_FIFO_SIZE, (SC_FRONTEND_PRIM_FIFO_SIZE(rdev->config.cik.sc_prim_fifo_size_frontend) |
3714 
				 SC_BACKEND_PRIM_FIFO_SIZE(rdev->config.cik.sc_prim_fifo_size_backend) |
3715 
				 SC_HIZ_TILE_FIFO_SIZE(rdev->config.cik.sc_hiz_tile_fifo_size) |
3716 
				 SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.cik.sc_earlyz_tile_fifo_size)));
3717 
 
3718 
	WREG32(VGT_NUM_INSTANCES, 1);
3719 
 
3720 
	WREG32(CP_PERFMON_CNTL, 0);
3721 
 
3722 
	WREG32(SQ_CONFIG, 0);
3723 
 
3724 
	WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) |
3725 
					  FORCE_EOV_MAX_REZ_CNT(255)));
3726 
 
3727 
	WREG32(VGT_CACHE_INVALIDATION, CACHE_INVALIDATION(VC_AND_TC) |
3728 
	       AUTO_INVLD_EN(ES_AND_GS_AUTO));
3729 
 
3730 
	WREG32(VGT_GS_VERTEX_REUSE, 16);
3731 
	WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
3732 
 
3733 
	tmp = RREG32(HDP_MISC_CNTL);
3734 
	tmp |= HDP_FLUSH_INVALIDATE_CACHE;
3735 
	WREG32(HDP_MISC_CNTL, tmp);
3736 
 
3737 
	hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
3738 
	WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
3739 
 
3740 
	WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3));
3741 
	WREG32(PA_SC_ENHANCE, ENABLE_PA_SC_OUT_OF_ORDER);
3742 
 
3743 
	udelay(50);
3744 
}
3745 
 
3746 
/*
3747 
 * GPU scratch registers helpers function.
3748 
 */
3749 
/**
3750 
 * cik_scratch_init - setup driver info for CP scratch regs
3751 
 *
3752 
 * @rdev: radeon_device pointer
3753 
 *
3754 
 * Set up the number and offset of the CP scratch registers.
3755 
 * NOTE: use of CP scratch registers is a legacy inferface and
3756 
 * is not used by default on newer asics (r6xx+).  On newer asics,
3757 
 * memory buffers are used for fences rather than scratch regs.
3758 
 */
3759 
static void cik_scratch_init(struct radeon_device *rdev)
3760 
{
3761 
	int i;
3762 
 
3763 
	rdev->scratch.num_reg = 7;
3764 
	rdev->scratch.reg_base = SCRATCH_REG0;
3765 
	for (i = 0; i < rdev->scratch.num_reg; i++) {
3766 
		rdev->scratch.free[i] = true;
3767 
		rdev->scratch.reg[i] = rdev->scratch.reg_base + (i * 4);
3768 
	}
3769 
}
3770 
 
3771 
/**
3772 
 * cik_ring_test - basic gfx ring test
3773 
 *
3774 
 * @rdev: radeon_device pointer
3775 
 * @ring: radeon_ring structure holding ring information
3776 
 *
3777 
 * Allocate a scratch register and write to it using the gfx ring (CIK).
3778 
 * Provides a basic gfx ring test to verify that the ring is working.
3779 
 * Used by cik_cp_gfx_resume();
3780 
 * Returns 0 on success, error on failure.
3781 
 */
3782 
int cik_ring_test(struct radeon_device *rdev, struct radeon_ring *ring)
3783 
{
3784 
	uint32_t scratch;
3785 
	uint32_t tmp = 0;
3786 
	unsigned i;
3787 
	int r;
3788 
 
3789 
	r = radeon_scratch_get(rdev, &scratch);
3790 
	if (r) {
3791 
		DRM_ERROR("radeon: cp failed to get scratch reg (%d).\n", r);
3792 
		return r;
3793 
	}
3794 
	WREG32(scratch, 0xCAFEDEAD);
3795 
	r = radeon_ring_lock(rdev, ring, 3);
3796 
	if (r) {
3797 
		DRM_ERROR("radeon: cp failed to lock ring %d (%d).\n", ring->idx, r);
3798 
		radeon_scratch_free(rdev, scratch);
3799 
		return r;
3800 
	}
3801 
	radeon_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG, 1));
3802 
	radeon_ring_write(ring, ((scratch - PACKET3_SET_UCONFIG_REG_START) >> 2));
3803 
	radeon_ring_write(ring, 0xDEADBEEF);
3804 
	radeon_ring_unlock_commit(rdev, ring, false);
3805 
 
3806 
	for (i = 0; i < rdev->usec_timeout; i++) {
3807 
		tmp = RREG32(scratch);
3808 
		if (tmp == 0xDEADBEEF)
3809 
			break;
3810 
		DRM_UDELAY(1);
3811 
	}
3812 
	if (i < rdev->usec_timeout) {
3813 
		DRM_INFO("ring test on %d succeeded in %d usecs\n", ring->idx, i);
3814 
	} else {
3815 
		DRM_ERROR("radeon: ring %d test failed (scratch(0x%04X)=0x%08X)\n",
3816 
			  ring->idx, scratch, tmp);
3817 
		r = -EINVAL;
3818 
	}
3819 
	radeon_scratch_free(rdev, scratch);
3820 
	return r;
3821 
}
3822 
 
3823 
/**
3824 
 * cik_hdp_flush_cp_ring_emit - emit an hdp flush on the cp
3825 
 *
3826 
 * @rdev: radeon_device pointer
3827 
 * @ridx: radeon ring index
3828 
 *
3829 
 * Emits an hdp flush on the cp.
3830 
 */
3831 
static void cik_hdp_flush_cp_ring_emit(struct radeon_device *rdev,
3832 
				       int ridx)
3833 
{
3834 
	struct radeon_ring *ring = &rdev->ring[ridx];
3835 
	u32 ref_and_mask;
3836 
 
3837 
	switch (ring->idx) {
3838 
	case CAYMAN_RING_TYPE_CP1_INDEX:
3839 
	case CAYMAN_RING_TYPE_CP2_INDEX:
3840 
	default:
3841 
		switch (ring->me) {
3842 
		case 0:
3843 
			ref_and_mask = CP2 << ring->pipe;
3844 
			break;
3845 
		case 1:
3846 
			ref_and_mask = CP6 << ring->pipe;
3847 
			break;
3848 
		default:
3849 
			return;
3850 
		}
3851 
		break;
3852 
	case RADEON_RING_TYPE_GFX_INDEX:
3853 
		ref_and_mask = CP0;
3854 
		break;
3855 
	}
3856 
 
3857 
	radeon_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5));
3858 
	radeon_ring_write(ring, (WAIT_REG_MEM_OPERATION(1) | /* write, wait, write */
3859 
				 WAIT_REG_MEM_FUNCTION(3) |  /* == */
3860 
				 WAIT_REG_MEM_ENGINE(1)));   /* pfp */
3861 
	radeon_ring_write(ring, GPU_HDP_FLUSH_REQ >> 2);
3862 
	radeon_ring_write(ring, GPU_HDP_FLUSH_DONE >> 2);
3863 
	radeon_ring_write(ring, ref_and_mask);
3864 
	radeon_ring_write(ring, ref_and_mask);
3865 
	radeon_ring_write(ring, 0x20); /* poll interval */
3866 
}
3867 
 
3868 
/**
3869 
 * cik_fence_gfx_ring_emit - emit a fence on the gfx ring
3870 
 *
3871 
 * @rdev: radeon_device pointer
3872 
 * @fence: radeon fence object
3873 
 *
3874 
 * Emits a fence sequnce number on the gfx ring and flushes
3875 
 * GPU caches.
3876 
 */
3877 
void cik_fence_gfx_ring_emit(struct radeon_device *rdev,
3878 
			     struct radeon_fence *fence)
3879 
{
3880 
	struct radeon_ring *ring = &rdev->ring[fence->ring];
3881 
	u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
3882 
 
3883 
	/* EVENT_WRITE_EOP - flush caches, send int */
3884 
	radeon_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE_EOP, 4));
3885 
	radeon_ring_write(ring, (EOP_TCL1_ACTION_EN |
3886 
				 EOP_TC_ACTION_EN |
3887 
				 EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) |
3888 
				 EVENT_INDEX(5)));
3889 
	radeon_ring_write(ring, addr & 0xfffffffc);
3890 
	radeon_ring_write(ring, (upper_32_bits(addr) & 0xffff) | DATA_SEL(1) | INT_SEL(2));
3891 
	radeon_ring_write(ring, fence->seq);
3892 
	radeon_ring_write(ring, 0);
3893 
}
3894 
 
3895 
/**
3896 
 * cik_fence_compute_ring_emit - emit a fence on the compute ring
3897 
 *
3898 
 * @rdev: radeon_device pointer
3899 
 * @fence: radeon fence object
3900 
 *
3901 
 * Emits a fence sequnce number on the compute ring and flushes
3902 
 * GPU caches.
3903 
 */
3904 
void cik_fence_compute_ring_emit(struct radeon_device *rdev,
3905 
				 struct radeon_fence *fence)
3906 
{
3907 
	struct radeon_ring *ring = &rdev->ring[fence->ring];
3908 
	u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
3909 
 
3910 
	/* RELEASE_MEM - flush caches, send int */
3911 
	radeon_ring_write(ring, PACKET3(PACKET3_RELEASE_MEM, 5));
3912 
	radeon_ring_write(ring, (EOP_TCL1_ACTION_EN |
3913 
				 EOP_TC_ACTION_EN |
3914 
				 EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) |
3915 
				 EVENT_INDEX(5)));
3916 
	radeon_ring_write(ring, DATA_SEL(1) | INT_SEL(2));
3917 
	radeon_ring_write(ring, addr & 0xfffffffc);
3918 
	radeon_ring_write(ring, upper_32_bits(addr));
3919 
	radeon_ring_write(ring, fence->seq);
3920 
	radeon_ring_write(ring, 0);
3921 
}
3922 
 
3923 
/**
3924 
 * cik_semaphore_ring_emit - emit a semaphore on the CP ring
3925 
 *
3926 
 * @rdev: radeon_device pointer
3927 
 * @ring: radeon ring buffer object
3928 
 * @semaphore: radeon semaphore object
3929 
 * @emit_wait: Is this a sempahore wait?
3930 
 *
3931 
 * Emits a semaphore signal/wait packet to the CP ring and prevents the PFP
3932 
 * from running ahead of semaphore waits.
3933 
 */
3934 
bool cik_semaphore_ring_emit(struct radeon_device *rdev,
3935 
			     struct radeon_ring *ring,
3936 
			     struct radeon_semaphore *semaphore,
3937 
			     bool emit_wait)
3938 
{
3939 
	uint64_t addr = semaphore->gpu_addr;
3940 
	unsigned sel = emit_wait ? PACKET3_SEM_SEL_WAIT : PACKET3_SEM_SEL_SIGNAL;
3941 
 
3942 
	radeon_ring_write(ring, PACKET3(PACKET3_MEM_SEMAPHORE, 1));
3943 
	radeon_ring_write(ring, lower_32_bits(addr));
3944 
	radeon_ring_write(ring, (upper_32_bits(addr) & 0xffff) | sel);
3945 
 
3946 
	if (emit_wait && ring->idx == RADEON_RING_TYPE_GFX_INDEX) {
3947 
		/* Prevent the PFP from running ahead of the semaphore wait */
3948 
		radeon_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0));
3949 
		radeon_ring_write(ring, 0x0);
3950 
	}
3951 
 
3952 
	return true;
3953 
}
3954 
 
3955 
/**
3956 
 * cik_copy_cpdma - copy pages using the CP DMA engine
3957 
 *
3958 
 * @rdev: radeon_device pointer
3959 
 * @src_offset: src GPU address
3960 
 * @dst_offset: dst GPU address
3961 
 * @num_gpu_pages: number of GPU pages to xfer
3962 
 * @fence: radeon fence object
3963 
 *
3964 
 * Copy GPU paging using the CP DMA engine (CIK+).
3965 
 * Used by the radeon ttm implementation to move pages if
3966 
 * registered as the asic copy callback.
3967 
 */
3968 
int cik_copy_cpdma(struct radeon_device *rdev,
3969 
		   uint64_t src_offset, uint64_t dst_offset,
3970 
		   unsigned num_gpu_pages,
3971 
		   struct radeon_fence **fence)
3972 
{
3973 
	struct radeon_semaphore *sem = NULL;
3974 
	int ring_index = rdev->asic->copy.blit_ring_index;
3975 
	struct radeon_ring *ring = &rdev->ring[ring_index];
3976 
	u32 size_in_bytes, cur_size_in_bytes, control;
3977 
	int i, num_loops;
3978 
	int r = 0;
3979 
 
3980 
	r = radeon_semaphore_create(rdev, &sem);
3981 
	if (r) {
3982 
		DRM_ERROR("radeon: moving bo (%d).\n", r);
3983 
		return r;
3984 
	}
3985 
 
3986 
	size_in_bytes = (num_gpu_pages << RADEON_GPU_PAGE_SHIFT);
3987 
	num_loops = DIV_ROUND_UP(size_in_bytes, 0x1fffff);
3988 
	r = radeon_ring_lock(rdev, ring, num_loops * 7 + 18);
3989 
	if (r) {
3990 
		DRM_ERROR("radeon: moving bo (%d).\n", r);
3991 
		radeon_semaphore_free(rdev, &sem, NULL);
3992 
		return r;
3993 
	}
3994 
 
3995 
	radeon_semaphore_sync_to(sem, *fence);
3996 
	radeon_semaphore_sync_rings(rdev, sem, ring->idx);
3997 
 
3998 
	for (i = 0; i < num_loops; i++) {
3999 
		cur_size_in_bytes = size_in_bytes;
4000 
		if (cur_size_in_bytes > 0x1fffff)
4001 
			cur_size_in_bytes = 0x1fffff;
4002 
		size_in_bytes -= cur_size_in_bytes;
4003 
		control = 0;
4004 
		if (size_in_bytes == 0)
4005 
			control |= PACKET3_DMA_DATA_CP_SYNC;
4006 
		radeon_ring_write(ring, PACKET3(PACKET3_DMA_DATA, 5));
4007 
		radeon_ring_write(ring, control);
4008 
		radeon_ring_write(ring, lower_32_bits(src_offset));
4009 
		radeon_ring_write(ring, upper_32_bits(src_offset));
4010 
		radeon_ring_write(ring, lower_32_bits(dst_offset));
4011 
		radeon_ring_write(ring, upper_32_bits(dst_offset));
4012 
		radeon_ring_write(ring, cur_size_in_bytes);
4013 
		src_offset += cur_size_in_bytes;
4014 
		dst_offset += cur_size_in_bytes;
4015 
	}
4016 
 
4017 
	r = radeon_fence_emit(rdev, fence, ring->idx);
4018 
	if (r) {
4019 
		radeon_ring_unlock_undo(rdev, ring);
4020 
		radeon_semaphore_free(rdev, &sem, NULL);
4021 
		return r;
4022 
	}
4023 
 
4024 
	radeon_ring_unlock_commit(rdev, ring, false);
4025 
	radeon_semaphore_free(rdev, &sem, *fence);
4026 
 
4027 
	return r;
4028 
}
4029 
 
4030 
/*
4031 
 * IB stuff
4032 
 */
4033 
/**
4034 
 * cik_ring_ib_execute - emit an IB (Indirect Buffer) on the gfx ring
4035 
 *
4036 
 * @rdev: radeon_device pointer
4037 
 * @ib: radeon indirect buffer object
4038 
 *
4039 
 * Emits an DE (drawing engine) or CE (constant engine) IB
4040 
 * on the gfx ring.  IBs are usually generated by userspace
4041 
 * acceleration drivers and submitted to the kernel for
4042 
 * sheduling on the ring.  This function schedules the IB
4043 
 * on the gfx ring for execution by the GPU.
4044 
 */
4045 
void cik_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
4046 
{
4047 
	struct radeon_ring *ring = &rdev->ring[ib->ring];
4048 
	u32 header, control = INDIRECT_BUFFER_VALID;
4049 
 
4050 
	if (ib->is_const_ib) {
4051 
		/* set switch buffer packet before const IB */
4052 
		radeon_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0));
4053 
		radeon_ring_write(ring, 0);
4054 
 
4055 
		header = PACKET3(PACKET3_INDIRECT_BUFFER_CONST, 2);
4056 
	} else {
4057 
		u32 next_rptr;
4058 
		if (ring->rptr_save_reg) {
4059 
			next_rptr = ring->wptr + 3 + 4;
4060 
			radeon_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG, 1));
4061 
			radeon_ring_write(ring, ((ring->rptr_save_reg -
4062 
						  PACKET3_SET_UCONFIG_REG_START) >> 2));
4063 
			radeon_ring_write(ring, next_rptr);
4064 
		} else if (rdev->wb.enabled) {
4065 
			next_rptr = ring->wptr + 5 + 4;
4066 
			radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
4067 
			radeon_ring_write(ring, WRITE_DATA_DST_SEL(1));
4068 
			radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
4069 
			radeon_ring_write(ring, upper_32_bits(ring->next_rptr_gpu_addr));
4070 
			radeon_ring_write(ring, next_rptr);
4071 
		}
4072 
 
4073 
		header = PACKET3(PACKET3_INDIRECT_BUFFER, 2);
4074 
	}
4075 
 
4076 
	control |= ib->length_dw |
4077 
		(ib->vm ? (ib->vm->id << 24) : 0);
4078 
 
4079 
	radeon_ring_write(ring, header);
4080 
	radeon_ring_write(ring,
4081 
#ifdef __BIG_ENDIAN
4082 
			  (2 << 0) |
4083 
#endif
4084 
			  (ib->gpu_addr & 0xFFFFFFFC));
4085 
	radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFFFF);
4086 
	radeon_ring_write(ring, control);
4087 
}
4088 
 
4089 
/**
4090 
 * cik_ib_test - basic gfx ring IB test
4091 
 *
4092 
 * @rdev: radeon_device pointer
4093 
 * @ring: radeon_ring structure holding ring information
4094 
 *
4095 
 * Allocate an IB and execute it on the gfx ring (CIK).
4096 
 * Provides a basic gfx ring test to verify that IBs are working.
4097 
 * Returns 0 on success, error on failure.
4098 
 */
4099 
int cik_ib_test(struct radeon_device *rdev, struct radeon_ring *ring)
4100 
{
4101 
	struct radeon_ib ib;
4102 
	uint32_t scratch;
4103 
	uint32_t tmp = 0;
4104 
	unsigned i;
4105 
	int r;
4106 
 
4107 
	r = radeon_scratch_get(rdev, &scratch);
4108 
	if (r) {
4109 
		DRM_ERROR("radeon: failed to get scratch reg (%d).\n", r);
4110 
		return r;
4111 
	}
4112 
	WREG32(scratch, 0xCAFEDEAD);
4113 
	r = radeon_ib_get(rdev, ring->idx, &ib, NULL, 256);
4114 
	if (r) {
4115 
		DRM_ERROR("radeon: failed to get ib (%d).\n", r);
4116 
		radeon_scratch_free(rdev, scratch);
4117 
		return r;
4118 
	}
4119 
	ib.ptr[0] = PACKET3(PACKET3_SET_UCONFIG_REG, 1);
4120 
	ib.ptr[1] = ((scratch - PACKET3_SET_UCONFIG_REG_START) >> 2);
4121 
	ib.ptr[2] = 0xDEADBEEF;
4122 
	ib.length_dw = 3;
4123 
	r = radeon_ib_schedule(rdev, &ib, NULL, false);
4124 
	if (r) {
4125 
		radeon_scratch_free(rdev, scratch);
4126 
		radeon_ib_free(rdev, &ib);
4127 
		DRM_ERROR("radeon: failed to schedule ib (%d).\n", r);
4128 
		return r;
4129 
	}
4130 
	r = radeon_fence_wait(ib.fence, false);
4131 
	if (r) {
4132 
		DRM_ERROR("radeon: fence wait failed (%d).\n", r);
4133 
		radeon_scratch_free(rdev, scratch);
4134 
		radeon_ib_free(rdev, &ib);
4135 
		return r;
4136 
	}
4137 
	for (i = 0; i < rdev->usec_timeout; i++) {
4138 
		tmp = RREG32(scratch);
4139 
		if (tmp == 0xDEADBEEF)
4140 
			break;
4141 
		DRM_UDELAY(1);
4142 
	}
4143 
	if (i < rdev->usec_timeout) {
4144 
		DRM_INFO("ib test on ring %d succeeded in %u usecs\n", ib.fence->ring, i);
4145 
	} else {
4146 
		DRM_ERROR("radeon: ib test failed (scratch(0x%04X)=0x%08X)\n",
4147 
			  scratch, tmp);
4148 
		r = -EINVAL;
4149 
	}
4150 
	radeon_scratch_free(rdev, scratch);
4151 
	radeon_ib_free(rdev, &ib);
4152 
	return r;
4153 
}
4154 
 
4155 
/*
4156 
 * CP.
4157 
 * On CIK, gfx and compute now have independant command processors.
4158 
 *
4159 
 * GFX
4160 
 * Gfx consists of a single ring and can process both gfx jobs and
4161 
 * compute jobs.  The gfx CP consists of three microengines (ME):
4162 
 * PFP - Pre-Fetch Parser
4163 
 * ME - Micro Engine
4164 
 * CE - Constant Engine
4165 
 * The PFP and ME make up what is considered the Drawing Engine (DE).
4166 
 * The CE is an asynchronous engine used for updating buffer desciptors
4167 
 * used by the DE so that they can be loaded into cache in parallel
4168 
 * while the DE is processing state update packets.
4169 
 *
4170 
 * Compute
4171 
 * The compute CP consists of two microengines (ME):
4172 
 * MEC1 - Compute MicroEngine 1
4173 
 * MEC2 - Compute MicroEngine 2
4174 
 * Each MEC supports 4 compute pipes and each pipe supports 8 queues.
4175 
 * The queues are exposed to userspace and are programmed directly
4176 
 * by the compute runtime.
4177 
 */
4178 
/**
4179 
 * cik_cp_gfx_enable - enable/disable the gfx CP MEs
4180 
 *
4181 
 * @rdev: radeon_device pointer
4182 
 * @enable: enable or disable the MEs
4183 
 *
4184 
 * Halts or unhalts the gfx MEs.
4185 
 */
4186 
static void cik_cp_gfx_enable(struct radeon_device *rdev, bool enable)
4187 
{
4188 
	if (enable)
4189 
		WREG32(CP_ME_CNTL, 0);
4190 
	else {
4191 
		if (rdev->asic->copy.copy_ring_index == RADEON_RING_TYPE_GFX_INDEX)
4192 
			radeon_ttm_set_active_vram_size(rdev, rdev->mc.visible_vram_size);
4193 
		WREG32(CP_ME_CNTL, (CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT));
4194 
		rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
4195 
	}
4196 
	udelay(50);
4197 
}
4198 
 
4199 
/**
4200 
 * cik_cp_gfx_load_microcode - load the gfx CP ME ucode
4201 
 *
4202 
 * @rdev: radeon_device pointer
4203 
 *
4204 
 * Loads the gfx PFP, ME, and CE ucode.
4205 
 * Returns 0 for success, -EINVAL if the ucode is not available.
4206 
 */
4207 
static int cik_cp_gfx_load_microcode(struct radeon_device *rdev)
4208 
{
4209 
	int i;
4210 
 
4211 
	if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw)
4212 
		return -EINVAL;
4213 
 
4214 
	cik_cp_gfx_enable(rdev, false);
4215 
 
4216 
	if (rdev->new_fw) {
4217 
		const struct gfx_firmware_header_v1_0 *pfp_hdr =
4218 
			(const struct gfx_firmware_header_v1_0 *)rdev->pfp_fw->data;
4219 
		const struct gfx_firmware_header_v1_0 *ce_hdr =
4220 
			(const struct gfx_firmware_header_v1_0 *)rdev->ce_fw->data;
4221 
		const struct gfx_firmware_header_v1_0 *me_hdr =
4222 
			(const struct gfx_firmware_header_v1_0 *)rdev->me_fw->data;
4223 
		const __le32 *fw_data;
4224 
		u32 fw_size;
4225 
 
4226 
		radeon_ucode_print_gfx_hdr(&pfp_hdr->header);
4227 
		radeon_ucode_print_gfx_hdr(&ce_hdr->header);
4228 
		radeon_ucode_print_gfx_hdr(&me_hdr->header);
4229 
 
4230 
		/* PFP */
4231 
		fw_data = (const __le32 *)
4232 
			(rdev->pfp_fw->data + le32_to_cpu(pfp_hdr->header.ucode_array_offset_bytes));
4233 
		fw_size = le32_to_cpu(pfp_hdr->header.ucode_size_bytes) / 4;
4234 
		WREG32(CP_PFP_UCODE_ADDR, 0);
4235 
		for (i = 0; i < fw_size; i++)
4236 
			WREG32(CP_PFP_UCODE_DATA, le32_to_cpup(fw_data++));
4237 
		WREG32(CP_PFP_UCODE_ADDR, 0);
4238 
 
4239 
		/* CE */
4240 
		fw_data = (const __le32 *)
4241 
			(rdev->ce_fw->data + le32_to_cpu(ce_hdr->header.ucode_array_offset_bytes));
4242 
		fw_size = le32_to_cpu(ce_hdr->header.ucode_size_bytes) / 4;
4243 
		WREG32(CP_CE_UCODE_ADDR, 0);
4244 
		for (i = 0; i < fw_size; i++)
4245 
			WREG32(CP_CE_UCODE_DATA, le32_to_cpup(fw_data++));
4246 
		WREG32(CP_CE_UCODE_ADDR, 0);
4247 
 
4248 
		/* ME */
4249 
		fw_data = (const __be32 *)
4250 
			(rdev->me_fw->data + le32_to_cpu(me_hdr->header.ucode_array_offset_bytes));
4251 
		fw_size = le32_to_cpu(me_hdr->header.ucode_size_bytes) / 4;
4252 
		WREG32(CP_ME_RAM_WADDR, 0);
4253 
		for (i = 0; i < fw_size; i++)
4254 
			WREG32(CP_ME_RAM_DATA, le32_to_cpup(fw_data++));
4255 
		WREG32(CP_ME_RAM_WADDR, 0);
4256 
	} else {
4257 
		const __be32 *fw_data;
4258 
 
4259 
	/* PFP */
4260 
	fw_data = (const __be32 *)rdev->pfp_fw->data;
4261 
	WREG32(CP_PFP_UCODE_ADDR, 0);
4262 
	for (i = 0; i < CIK_PFP_UCODE_SIZE; i++)
4263 
		WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
4264 
	WREG32(CP_PFP_UCODE_ADDR, 0);
4265 
 
4266 
	/* CE */
4267 
	fw_data = (const __be32 *)rdev->ce_fw->data;
4268 
	WREG32(CP_CE_UCODE_ADDR, 0);
4269 
	for (i = 0; i < CIK_CE_UCODE_SIZE; i++)
4270 
		WREG32(CP_CE_UCODE_DATA, be32_to_cpup(fw_data++));
4271 
	WREG32(CP_CE_UCODE_ADDR, 0);
4272 
 
4273 
	/* ME */
4274 
	fw_data = (const __be32 *)rdev->me_fw->data;
4275 
	WREG32(CP_ME_RAM_WADDR, 0);
4276 
	for (i = 0; i < CIK_ME_UCODE_SIZE; i++)
4277 
		WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
4278 
	WREG32(CP_ME_RAM_WADDR, 0);
4279 
	}
4280 
 
4281 
	WREG32(CP_PFP_UCODE_ADDR, 0);
4282 
	WREG32(CP_CE_UCODE_ADDR, 0);
4283 
	WREG32(CP_ME_RAM_WADDR, 0);
4284 
	WREG32(CP_ME_RAM_RADDR, 0);
4285 
	return 0;
4286 
}
4287 
 
4288 
/**
4289 
 * cik_cp_gfx_start - start the gfx ring
4290 
 *
4291 
 * @rdev: radeon_device pointer
4292 
 *
4293 
 * Enables the ring and loads the clear state context and other
4294 
 * packets required to init the ring.
4295 
 * Returns 0 for success, error for failure.
4296 
 */
4297 
static int cik_cp_gfx_start(struct radeon_device *rdev)
4298 
{
4299 
	struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
4300 
	int r, i;
4301 
 
4302 
	/* init the CP */
4303 
	WREG32(CP_MAX_CONTEXT, rdev->config.cik.max_hw_contexts - 1);
4304 
	WREG32(CP_ENDIAN_SWAP, 0);
4305 
	WREG32(CP_DEVICE_ID, 1);
4306 
 
4307 
	cik_cp_gfx_enable(rdev, true);
4308 
 
4309 
	r = radeon_ring_lock(rdev, ring, cik_default_size + 17);
4310 
	if (r) {
4311 
		DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
4312 
		return r;
4313 
	}
4314 
 
4315 
	/* init the CE partitions.  CE only used for gfx on CIK */
4316 
	radeon_ring_write(ring, PACKET3(PACKET3_SET_BASE, 2));
4317 
	radeon_ring_write(ring, PACKET3_BASE_INDEX(CE_PARTITION_BASE));
4318 
	radeon_ring_write(ring, 0xc000);
4319 
	radeon_ring_write(ring, 0xc000);
4320 
 
4321 
	/* setup clear context state */
4322 
	radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
4323 
	radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
4324 
 
4325 
	radeon_ring_write(ring, PACKET3(PACKET3_CONTEXT_CONTROL, 1));
4326 
	radeon_ring_write(ring, 0x80000000);
4327 
	radeon_ring_write(ring, 0x80000000);
4328 
 
4329 
	for (i = 0; i < cik_default_size; i++)
4330 
		radeon_ring_write(ring, cik_default_state[i]);
4331 
 
4332 
	radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
4333 
	radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
4334 
 
4335 
	/* set clear context state */
4336 
	radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
4337 
	radeon_ring_write(ring, 0);
4338 
 
4339 
	radeon_ring_write(ring, PACKET3(PACKET3_SET_CONTEXT_REG, 2));
4340 
	radeon_ring_write(ring, 0x00000316);
4341 
	radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
4342 
	radeon_ring_write(ring, 0x00000010); /* VGT_OUT_DEALLOC_CNTL */
4343 
 
4344 
	radeon_ring_unlock_commit(rdev, ring, false);
4345 
 
4346 
	return 0;
4347 
}
4348 
 
4349 
/**
4350 
 * cik_cp_gfx_fini - stop the gfx ring
4351 
 *
4352 
 * @rdev: radeon_device pointer
4353 
 *
4354 
 * Stop the gfx ring and tear down the driver ring
4355 
 * info.
4356 
 */
4357 
static void cik_cp_gfx_fini(struct radeon_device *rdev)
4358 
{
4359 
	cik_cp_gfx_enable(rdev, false);
4360 
	radeon_ring_fini(rdev, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]);
4361 
}
4362 
 
4363 
/**
4364 
 * cik_cp_gfx_resume - setup the gfx ring buffer registers
4365 
 *
4366 
 * @rdev: radeon_device pointer
4367 
 *
4368 
 * Program the location and size of the gfx ring buffer
4369 
 * and test it to make sure it's working.
4370 
 * Returns 0 for success, error for failure.
4371 
 */
4372 
static int cik_cp_gfx_resume(struct radeon_device *rdev)
4373 
{
4374 
	struct radeon_ring *ring;
4375 
	u32 tmp;
4376 
	u32 rb_bufsz;
4377 
	u64 rb_addr;
4378 
	int r;
4379 
 
4380 
	WREG32(CP_SEM_WAIT_TIMER, 0x0);
4381 
	if (rdev->family != CHIP_HAWAII)
4382 
		WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0);
4383 
 
4384 
	/* Set the write pointer delay */
4385 
	WREG32(CP_RB_WPTR_DELAY, 0);
4386 
 
4387 
	/* set the RB to use vmid 0 */
4388 
	WREG32(CP_RB_VMID, 0);
4389 
 
4390 
	WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
4391 
 
4392 
	/* ring 0 - compute and gfx */
4393 
	/* Set ring buffer size */
4394 
	ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
4395 
	rb_bufsz = order_base_2(ring->ring_size / 8);
4396 
	tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
4397 
#ifdef __BIG_ENDIAN
4398 
	tmp |= BUF_SWAP_32BIT;
4399 
#endif
4400 
	WREG32(CP_RB0_CNTL, tmp);
4401 
 
4402 
	/* Initialize the ring buffer's read and write pointers */
4403 
	WREG32(CP_RB0_CNTL, tmp | RB_RPTR_WR_ENA);
4404 
	ring->wptr = 0;
4405 
	WREG32(CP_RB0_WPTR, ring->wptr);
4406 
 
4407 
	/* set the wb address wether it's enabled or not */
4408 
	WREG32(CP_RB0_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC);
4409 
	WREG32(CP_RB0_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF);
4410 
 
4411 
	/* scratch register shadowing is no longer supported */
4412 
	WREG32(SCRATCH_UMSK, 0);
4413 
 
4414 
	if (!rdev->wb.enabled)
4415 
		tmp |= RB_NO_UPDATE;
4416 
 
4417 
	mdelay(1);
4418 
	WREG32(CP_RB0_CNTL, tmp);
4419 
 
4420 
	rb_addr = ring->gpu_addr >> 8;
4421 
	WREG32(CP_RB0_BASE, rb_addr);
4422 
	WREG32(CP_RB0_BASE_HI, upper_32_bits(rb_addr));
4423 
 
4424 
	/* start the ring */
4425 
	cik_cp_gfx_start(rdev);
4426 
	rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = true;
4427 
	r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]);
4428 
	if (r) {
4429 
		rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
4430 
		return r;
4431 
	}
4432 
 
4433 
	if (rdev->asic->copy.copy_ring_index == RADEON_RING_TYPE_GFX_INDEX)
4434 
		radeon_ttm_set_active_vram_size(rdev, rdev->mc.real_vram_size);
4435 
 
4436 
	return 0;
4437 
}
4438 
 
4439 
u32 cik_gfx_get_rptr(struct radeon_device *rdev,
4440 
		     struct radeon_ring *ring)
4441 
{
4442 
	u32 rptr;
4443 
 
4444 
	if (rdev->wb.enabled)
4445 
		rptr = rdev->wb.wb[ring->rptr_offs/4];
4446 
	else
4447 
		rptr = RREG32(CP_RB0_RPTR);
4448 
 
4449 
	return rptr;
4450 
}
4451 
 
4452 
u32 cik_gfx_get_wptr(struct radeon_device *rdev,
4453 
		     struct radeon_ring *ring)
4454 
{
4455 
	u32 wptr;
4456 
 
4457 
	wptr = RREG32(CP_RB0_WPTR);
4458 
 
4459 
	return wptr;
4460 
}
4461 
 
4462 
void cik_gfx_set_wptr(struct radeon_device *rdev,
4463 
		      struct radeon_ring *ring)
4464 
{
4465 
	WREG32(CP_RB0_WPTR, ring->wptr);
4466 
	(void)RREG32(CP_RB0_WPTR);
4467 
}
4468 
 
4469 
u32 cik_compute_get_rptr(struct radeon_device *rdev,
4470 
			 struct radeon_ring *ring)
4471 
{
4472 
	u32 rptr;
4473 
 
4474 
	if (rdev->wb.enabled) {
4475 
		rptr = rdev->wb.wb[ring->rptr_offs/4];
4476 
	} else {
4477 
		mutex_lock(&rdev->srbm_mutex);
4478 
		cik_srbm_select(rdev, ring->me, ring->pipe, ring->queue, 0);
4479 
		rptr = RREG32(CP_HQD_PQ_RPTR);
4480 
		cik_srbm_select(rdev, 0, 0, 0, 0);
4481 
		mutex_unlock(&rdev->srbm_mutex);
4482 
	}
4483 
 
4484 
	return rptr;
4485 
}
4486 
 
4487 
u32 cik_compute_get_wptr(struct radeon_device *rdev,
4488 
			 struct radeon_ring *ring)
4489 
{
4490 
	u32 wptr;
4491 
 
4492 
	if (rdev->wb.enabled) {
4493 
		/* XXX check if swapping is necessary on BE */
4494 
		wptr = rdev->wb.wb[ring->wptr_offs/4];
4495 
	} else {
4496 
		mutex_lock(&rdev->srbm_mutex);
4497 
		cik_srbm_select(rdev, ring->me, ring->pipe, ring->queue, 0);
4498 
		wptr = RREG32(CP_HQD_PQ_WPTR);
4499 
		cik_srbm_select(rdev, 0, 0, 0, 0);
4500 
		mutex_unlock(&rdev->srbm_mutex);
4501 
	}
4502 
 
4503 
	return wptr;
4504 
}
4505 
 
4506 
void cik_compute_set_wptr(struct radeon_device *rdev,
4507 
			  struct radeon_ring *ring)
4508 
{
4509 
	/* XXX check if swapping is necessary on BE */
4510 
	rdev->wb.wb[ring->wptr_offs/4] = ring->wptr;
4511 
	WDOORBELL32(ring->doorbell_index, ring->wptr);
4512 
}
4513 
 
4514 
/**
4515 
 * cik_cp_compute_enable - enable/disable the compute CP MEs
4516 
 *
4517 
 * @rdev: radeon_device pointer
4518 
 * @enable: enable or disable the MEs
4519 
 *
4520 
 * Halts or unhalts the compute MEs.
4521 
 */
4522 
static void cik_cp_compute_enable(struct radeon_device *rdev, bool enable)
4523 
{
4524 
	if (enable)
4525 
		WREG32(CP_MEC_CNTL, 0);
4526 
	else {
4527 
		WREG32(CP_MEC_CNTL, (MEC_ME1_HALT | MEC_ME2_HALT));
4528 
		rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
4529 
		rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
4530 
	}
4531 
	udelay(50);
4532 
}
4533 
 
4534 
/**
4535 
 * cik_cp_compute_load_microcode - load the compute CP ME ucode
4536 
 *
4537 
 * @rdev: radeon_device pointer
4538 
 *
4539 
 * Loads the compute MEC1&2 ucode.
4540 
 * Returns 0 for success, -EINVAL if the ucode is not available.
4541 
 */
4542 
static int cik_cp_compute_load_microcode(struct radeon_device *rdev)
4543 
{
4544 
	int i;
4545 
 
4546 
	if (!rdev->mec_fw)
4547 
		return -EINVAL;
4548 
 
4549 
	cik_cp_compute_enable(rdev, false);
4550 
 
4551 
	if (rdev->new_fw) {
4552 
		const struct gfx_firmware_header_v1_0 *mec_hdr =
4553 
			(const struct gfx_firmware_header_v1_0 *)rdev->mec_fw->data;
4554 
		const __le32 *fw_data;
4555 
		u32 fw_size;
4556 
 
4557 
		radeon_ucode_print_gfx_hdr(&mec_hdr->header);
4558 
 
4559 
		/* MEC1 */
4560 
		fw_data = (const __le32 *)
4561 
			(rdev->mec_fw->data + le32_to_cpu(mec_hdr->header.ucode_array_offset_bytes));
4562 
		fw_size = le32_to_cpu(mec_hdr->header.ucode_size_bytes) / 4;
4563 
		WREG32(CP_MEC_ME1_UCODE_ADDR, 0);
4564 
		for (i = 0; i < fw_size; i++)
4565 
			WREG32(CP_MEC_ME1_UCODE_DATA, le32_to_cpup(fw_data++));
4566 
		WREG32(CP_MEC_ME1_UCODE_ADDR, 0);
4567 
 
4568 
		/* MEC2 */
4569 
		if (rdev->family == CHIP_KAVERI) {
4570 
			const struct gfx_firmware_header_v1_0 *mec2_hdr =
4571 
				(const struct gfx_firmware_header_v1_0 *)rdev->mec2_fw->data;
4572 
 
4573 
			fw_data = (const __le32 *)
4574 
				(rdev->mec2_fw->data +
4575 
				 le32_to_cpu(mec2_hdr->header.ucode_array_offset_bytes));
4576 
			fw_size = le32_to_cpu(mec2_hdr->header.ucode_size_bytes) / 4;
4577 
			WREG32(CP_MEC_ME2_UCODE_ADDR, 0);
4578 
			for (i = 0; i < fw_size; i++)
4579 
				WREG32(CP_MEC_ME2_UCODE_DATA, le32_to_cpup(fw_data++));
4580 
			WREG32(CP_MEC_ME2_UCODE_ADDR, 0);
4581 
		}
4582 
	} else {
4583 
		const __be32 *fw_data;
4584 
 
4585 
	/* MEC1 */
4586 
	fw_data = (const __be32 *)rdev->mec_fw->data;
4587 
	WREG32(CP_MEC_ME1_UCODE_ADDR, 0);
4588 
	for (i = 0; i < CIK_MEC_UCODE_SIZE; i++)
4589 
		WREG32(CP_MEC_ME1_UCODE_DATA, be32_to_cpup(fw_data++));
4590 
	WREG32(CP_MEC_ME1_UCODE_ADDR, 0);
4591 
 
4592 
	if (rdev->family == CHIP_KAVERI) {
4593 
		/* MEC2 */
4594 
		fw_data = (const __be32 *)rdev->mec_fw->data;
4595 
		WREG32(CP_MEC_ME2_UCODE_ADDR, 0);
4596 
		for (i = 0; i < CIK_MEC_UCODE_SIZE; i++)
4597 
			WREG32(CP_MEC_ME2_UCODE_DATA, be32_to_cpup(fw_data++));
4598 
		WREG32(CP_MEC_ME2_UCODE_ADDR, 0);
4599 
	}
4600 
	}
4601 
 
4602 
	return 0;
4603 
}
4604 
 
4605 
/**
4606 
 * cik_cp_compute_start - start the compute queues
4607 
 *
4608 
 * @rdev: radeon_device pointer
4609 
 *
4610 
 * Enable the compute queues.
4611 
 * Returns 0 for success, error for failure.
4612 
 */
4613 
static int cik_cp_compute_start(struct radeon_device *rdev)
4614 
{
4615 
	cik_cp_compute_enable(rdev, true);
4616 
 
4617 
	return 0;
4618 
}
4619 
 
4620 
/**
4621 
 * cik_cp_compute_fini - stop the compute queues
4622 
 *
4623 
 * @rdev: radeon_device pointer
4624 
 *
4625 
 * Stop the compute queues and tear down the driver queue
4626 
 * info.
4627 
 */
4628 
static void cik_cp_compute_fini(struct radeon_device *rdev)
4629 
{
4630 
	int i, idx, r;
4631 
 
4632 
	cik_cp_compute_enable(rdev, false);
4633 
 
4634 
	for (i = 0; i < 2; i++) {
4635 
		if (i == 0)
4636 
			idx = CAYMAN_RING_TYPE_CP1_INDEX;
4637 
		else
4638 
			idx = CAYMAN_RING_TYPE_CP2_INDEX;
4639 
 
4640 
		if (rdev->ring[idx].mqd_obj) {
4641 
			r = radeon_bo_reserve(rdev->ring[idx].mqd_obj, false);
4642 
			if (unlikely(r != 0))
4643 
				dev_warn(rdev->dev, "(%d) reserve MQD bo failed\n", r);
4644 
 
4645 
			radeon_bo_unpin(rdev->ring[idx].mqd_obj);
4646 
			radeon_bo_unreserve(rdev->ring[idx].mqd_obj);
4647 
 
4648 
			radeon_bo_unref(&rdev->ring[idx].mqd_obj);
4649 
			rdev->ring[idx].mqd_obj = NULL;
4650 
		}
4651 
	}
4652 
}
4653 
 
4654 
static void cik_mec_fini(struct radeon_device *rdev)
4655 
{
4656 
	int r;
4657 
 
4658 
	if (rdev->mec.hpd_eop_obj) {
4659 
		r = radeon_bo_reserve(rdev->mec.hpd_eop_obj, false);
4660 
		if (unlikely(r != 0))
4661 
			dev_warn(rdev->dev, "(%d) reserve HPD EOP bo failed\n", r);
4662 
		radeon_bo_unpin(rdev->mec.hpd_eop_obj);
4663 
		radeon_bo_unreserve(rdev->mec.hpd_eop_obj);
4664 
 
4665 
		radeon_bo_unref(&rdev->mec.hpd_eop_obj);
4666 
		rdev->mec.hpd_eop_obj = NULL;
4667 
	}
4668 
}
4669 
 
4670 
#define MEC_HPD_SIZE 2048
4671 
 
4672 
static int cik_mec_init(struct radeon_device *rdev)
4673 
{
4674 
	int r;
4675 
	u32 *hpd;
4676 
 
4677 
	/*
4678 
	 * KV:    2 MEC, 4 Pipes/MEC, 8 Queues/Pipe - 64 Queues total
4679 
	 * CI/KB: 1 MEC, 4 Pipes/MEC, 8 Queues/Pipe - 32 Queues total
4680 
	 */
4681 
	if (rdev->family == CHIP_KAVERI)
4682 
		rdev->mec.num_mec = 2;
4683 
	else
4684 
		rdev->mec.num_mec = 1;
4685 
	rdev->mec.num_pipe = 4;
4686 
	rdev->mec.num_queue = rdev->mec.num_mec * rdev->mec.num_pipe * 8;
4687 
 
4688 
	if (rdev->mec.hpd_eop_obj == NULL) {
4689 
		r = radeon_bo_create(rdev,
4690 
				     rdev->mec.num_mec *rdev->mec.num_pipe * MEC_HPD_SIZE * 2,
4691 
				     PAGE_SIZE, true,
4692 
				     RADEON_GEM_DOMAIN_GTT, 0, NULL,
4693 
				     &rdev->mec.hpd_eop_obj);
4694 
		if (r) {
4695 
			dev_warn(rdev->dev, "(%d) create HDP EOP bo failed\n", r);
4696 
			return r;
4697 
		}
4698 
	}
4699 
 
4700 
	r = radeon_bo_reserve(rdev->mec.hpd_eop_obj, false);
4701 
	if (unlikely(r != 0)) {
4702 
		cik_mec_fini(rdev);
4703 
		return r;
4704 
	}
4705 
	r = radeon_bo_pin(rdev->mec.hpd_eop_obj, RADEON_GEM_DOMAIN_GTT,
4706 
			  &rdev->mec.hpd_eop_gpu_addr);
4707 
	if (r) {
4708 
		dev_warn(rdev->dev, "(%d) pin HDP EOP bo failed\n", r);
4709 
		cik_mec_fini(rdev);
4710 
		return r;
4711 
	}
4712 
	r = radeon_bo_kmap(rdev->mec.hpd_eop_obj, (void **)&hpd);
4713 
	if (r) {
4714 
		dev_warn(rdev->dev, "(%d) map HDP EOP bo failed\n", r);
4715 
		cik_mec_fini(rdev);
4716 
		return r;
4717 
	}
4718 
 
4719 
	/* clear memory.  Not sure if this is required or not */
4720 
	memset(hpd, 0, rdev->mec.num_mec *rdev->mec.num_pipe * MEC_HPD_SIZE * 2);
4721 
 
4722 
	radeon_bo_kunmap(rdev->mec.hpd_eop_obj);
4723 
	radeon_bo_unreserve(rdev->mec.hpd_eop_obj);
4724 
 
4725 
	return 0;
4726 
}
4727 
 
4728 
struct hqd_registers
4729 
{
4730 
	u32 cp_mqd_base_addr;
4731 
	u32 cp_mqd_base_addr_hi;
4732 
	u32 cp_hqd_active;
4733 
	u32 cp_hqd_vmid;
4734 
	u32 cp_hqd_persistent_state;
4735 
	u32 cp_hqd_pipe_priority;
4736 
	u32 cp_hqd_queue_priority;
4737 
	u32 cp_hqd_quantum;
4738 
	u32 cp_hqd_pq_base;
4739 
	u32 cp_hqd_pq_base_hi;
4740 
	u32 cp_hqd_pq_rptr;
4741 
	u32 cp_hqd_pq_rptr_report_addr;
4742 
	u32 cp_hqd_pq_rptr_report_addr_hi;
4743 
	u32 cp_hqd_pq_wptr_poll_addr;
4744 
	u32 cp_hqd_pq_wptr_poll_addr_hi;
4745 
	u32 cp_hqd_pq_doorbell_control;
4746 
	u32 cp_hqd_pq_wptr;
4747 
	u32 cp_hqd_pq_control;
4748 
	u32 cp_hqd_ib_base_addr;
4749 
	u32 cp_hqd_ib_base_addr_hi;
4750 
	u32 cp_hqd_ib_rptr;
4751 
	u32 cp_hqd_ib_control;
4752 
	u32 cp_hqd_iq_timer;
4753 
	u32 cp_hqd_iq_rptr;
4754 
	u32 cp_hqd_dequeue_request;
4755 
	u32 cp_hqd_dma_offload;
4756 
	u32 cp_hqd_sema_cmd;
4757 
	u32 cp_hqd_msg_type;
4758 
	u32 cp_hqd_atomic0_preop_lo;
4759 
	u32 cp_hqd_atomic0_preop_hi;
4760 
	u32 cp_hqd_atomic1_preop_lo;
4761 
	u32 cp_hqd_atomic1_preop_hi;
4762 
	u32 cp_hqd_hq_scheduler0;
4763 
	u32 cp_hqd_hq_scheduler1;
4764 
	u32 cp_mqd_control;
4765 
};
4766 
 
4767 
struct bonaire_mqd
4768 
{
4769 
	u32 header;
4770 
	u32 dispatch_initiator;
4771 
	u32 dimensions[3];
4772 
	u32 start_idx[3];
4773 
	u32 num_threads[3];
4774 
	u32 pipeline_stat_enable;
4775 
	u32 perf_counter_enable;
4776 
	u32 pgm[2];
4777 
	u32 tba[2];
4778 
	u32 tma[2];
4779 
	u32 pgm_rsrc[2];
4780 
	u32 vmid;
4781 
	u32 resource_limits;
4782 
	u32 static_thread_mgmt01[2];
4783 
	u32 tmp_ring_size;
4784 
	u32 static_thread_mgmt23[2];
4785 
	u32 restart[3];
4786 
	u32 thread_trace_enable;
4787 
	u32 reserved1;
4788 
	u32 user_data[16];
4789 
	u32 vgtcs_invoke_count[2];
4790 
	struct hqd_registers queue_state;
4791 
	u32 dequeue_cntr;
4792 
	u32 interrupt_queue[64];
4793 
};
4794 
 
4795 
/**
4796 
 * cik_cp_compute_resume - setup the compute queue registers
4797 
 *
4798 
 * @rdev: radeon_device pointer
4799 
 *
4800 
 * Program the compute queues and test them to make sure they
4801 
 * are working.
4802 
 * Returns 0 for success, error for failure.
4803 
 */
4804 
static int cik_cp_compute_resume(struct radeon_device *rdev)
4805 
{
5179 serge 4806 
	int r, i, j, idx;
5078 serge 4807 
	u32 tmp;
4808 
	bool use_doorbell = true;
4809 
	u64 hqd_gpu_addr;
4810 
	u64 mqd_gpu_addr;
4811 
	u64 eop_gpu_addr;
4812 
	u64 wb_gpu_addr;
4813 
	u32 *buf;
4814 
	struct bonaire_mqd *mqd;
4815 
 
4816 
	r = cik_cp_compute_start(rdev);
4817 
	if (r)
4818 
		return r;
4819 
 
4820 
	/* fix up chicken bits */
4821 
	tmp = RREG32(CP_CPF_DEBUG);
4822 
	tmp |= (1 << 23);
4823 
	WREG32(CP_CPF_DEBUG, tmp);
4824 
 
4825 
	/* init the pipes */
4826 
	mutex_lock(&rdev->srbm_mutex);
4827 
	for (i = 0; i < (rdev->mec.num_pipe * rdev->mec.num_mec); i++) {
4828 
		int me = (i < 4) ? 1 : 2;
4829 
		int pipe = (i < 4) ? i : (i - 4);
4830 
 
4831 
		eop_gpu_addr = rdev->mec.hpd_eop_gpu_addr + (i * MEC_HPD_SIZE * 2);
4832 
 
4833 
		cik_srbm_select(rdev, me, pipe, 0, 0);
4834 
 
4835 
		/* write the EOP addr */
4836 
		WREG32(CP_HPD_EOP_BASE_ADDR, eop_gpu_addr >> 8);
4837 
		WREG32(CP_HPD_EOP_BASE_ADDR_HI, upper_32_bits(eop_gpu_addr) >> 8);
4838 
 
4839 
		/* set the VMID assigned */
4840 
		WREG32(CP_HPD_EOP_VMID, 0);
4841 
 
4842 
		/* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */
4843 
		tmp = RREG32(CP_HPD_EOP_CONTROL);
4844 
		tmp &= ~EOP_SIZE_MASK;
4845 
		tmp |= order_base_2(MEC_HPD_SIZE / 8);
4846 
		WREG32(CP_HPD_EOP_CONTROL, tmp);
4847 
	}
4848 
	cik_srbm_select(rdev, 0, 0, 0, 0);
4849 
	mutex_unlock(&rdev->srbm_mutex);
4850 
 
4851 
	/* init the queues.  Just two for now. */
4852 
	for (i = 0; i < 2; i++) {
4853 
		if (i == 0)
4854 
			idx = CAYMAN_RING_TYPE_CP1_INDEX;
4855 
		else
4856 
			idx = CAYMAN_RING_TYPE_CP2_INDEX;
4857 
 
4858 
		if (rdev->ring[idx].mqd_obj == NULL) {
4859 
			r = radeon_bo_create(rdev,
4860 
					     sizeof(struct bonaire_mqd),
4861 
					     PAGE_SIZE, true,
4862 
					     RADEON_GEM_DOMAIN_GTT, 0, NULL,
4863 
					     &rdev->ring[idx].mqd_obj);
4864 
			if (r) {
4865 
				dev_warn(rdev->dev, "(%d) create MQD bo failed\n", r);
4866 
				return r;
4867 
			}
4868 
		}
4869 
 
4870 
		r = radeon_bo_reserve(rdev->ring[idx].mqd_obj, false);
4871 
		if (unlikely(r != 0)) {
4872 
			cik_cp_compute_fini(rdev);
4873 
			return r;
4874 
		}
4875 
		r = radeon_bo_pin(rdev->ring[idx].mqd_obj, RADEON_GEM_DOMAIN_GTT,
4876 
				  &mqd_gpu_addr);
4877 
		if (r) {
4878 
			dev_warn(rdev->dev, "(%d) pin MQD bo failed\n", r);
4879 
			cik_cp_compute_fini(rdev);
4880 
			return r;
4881 
		}
4882 
		r = radeon_bo_kmap(rdev->ring[idx].mqd_obj, (void **)&buf);
4883 
		if (r) {
4884 
			dev_warn(rdev->dev, "(%d) map MQD bo failed\n", r);
4885 
			cik_cp_compute_fini(rdev);
4886 
			return r;
4887 
		}
4888 
 
4889 
		/* init the mqd struct */
4890 
		memset(buf, 0, sizeof(struct bonaire_mqd));
4891 
 
4892 
		mqd = (struct bonaire_mqd *)buf;
4893 
		mqd->header = 0xC0310800;
4894 
		mqd->static_thread_mgmt01[0] = 0xffffffff;
4895 
		mqd->static_thread_mgmt01[1] = 0xffffffff;
4896 
		mqd->static_thread_mgmt23[0] = 0xffffffff;
4897 
		mqd->static_thread_mgmt23[1] = 0xffffffff;
4898 
 
4899 
		mutex_lock(&rdev->srbm_mutex);
4900 
		cik_srbm_select(rdev, rdev->ring[idx].me,
4901 
				rdev->ring[idx].pipe,
4902 
				rdev->ring[idx].queue, 0);
4903 
 
4904 
		/* disable wptr polling */
4905 
		tmp = RREG32(CP_PQ_WPTR_POLL_CNTL);
4906 
		tmp &= ~WPTR_POLL_EN;
4907 
		WREG32(CP_PQ_WPTR_POLL_CNTL, tmp);
4908 
 
4909 
		/* enable doorbell? */
4910 
		mqd->queue_state.cp_hqd_pq_doorbell_control =
4911 
			RREG32(CP_HQD_PQ_DOORBELL_CONTROL);
4912 
		if (use_doorbell)
4913 
			mqd->queue_state.cp_hqd_pq_doorbell_control |= DOORBELL_EN;
4914 
		else
4915 
			mqd->queue_state.cp_hqd_pq_doorbell_control &= ~DOORBELL_EN;
4916 
		WREG32(CP_HQD_PQ_DOORBELL_CONTROL,
4917 
		       mqd->queue_state.cp_hqd_pq_doorbell_control);
4918 
 
4919 
		/* disable the queue if it's active */
4920 
		mqd->queue_state.cp_hqd_dequeue_request = 0;
4921 
		mqd->queue_state.cp_hqd_pq_rptr = 0;
4922 
		mqd->queue_state.cp_hqd_pq_wptr= 0;
4923 
		if (RREG32(CP_HQD_ACTIVE) & 1) {
4924 
			WREG32(CP_HQD_DEQUEUE_REQUEST, 1);
5179 serge 4925 
			for (j = 0; j < rdev->usec_timeout; j++) {
5078 serge 4926 
				if (!(RREG32(CP_HQD_ACTIVE) & 1))
4927 
					break;
4928 
				udelay(1);
4929 
			}
4930 
			WREG32(CP_HQD_DEQUEUE_REQUEST, mqd->queue_state.cp_hqd_dequeue_request);
4931 
			WREG32(CP_HQD_PQ_RPTR, mqd->queue_state.cp_hqd_pq_rptr);
4932 
			WREG32(CP_HQD_PQ_WPTR, mqd->queue_state.cp_hqd_pq_wptr);
4933 
		}
4934 
 
4935 
		/* set the pointer to the MQD */
4936 
		mqd->queue_state.cp_mqd_base_addr = mqd_gpu_addr & 0xfffffffc;
4937 
		mqd->queue_state.cp_mqd_base_addr_hi = upper_32_bits(mqd_gpu_addr);
4938 
		WREG32(CP_MQD_BASE_ADDR, mqd->queue_state.cp_mqd_base_addr);
4939 
		WREG32(CP_MQD_BASE_ADDR_HI, mqd->queue_state.cp_mqd_base_addr_hi);
4940 
		/* set MQD vmid to 0 */
4941 
		mqd->queue_state.cp_mqd_control = RREG32(CP_MQD_CONTROL);
4942 
		mqd->queue_state.cp_mqd_control &= ~MQD_VMID_MASK;
4943 
		WREG32(CP_MQD_CONTROL, mqd->queue_state.cp_mqd_control);
4944 
 
4945 
		/* set the pointer to the HQD, this is similar CP_RB0_BASE/_HI */
4946 
		hqd_gpu_addr = rdev->ring[idx].gpu_addr >> 8;
4947 
		mqd->queue_state.cp_hqd_pq_base = hqd_gpu_addr;
4948 
		mqd->queue_state.cp_hqd_pq_base_hi = upper_32_bits(hqd_gpu_addr);
4949 
		WREG32(CP_HQD_PQ_BASE, mqd->queue_state.cp_hqd_pq_base);
4950 
		WREG32(CP_HQD_PQ_BASE_HI, mqd->queue_state.cp_hqd_pq_base_hi);
4951 
 
4952 
		/* set up the HQD, this is similar to CP_RB0_CNTL */
4953 
		mqd->queue_state.cp_hqd_pq_control = RREG32(CP_HQD_PQ_CONTROL);
4954 
		mqd->queue_state.cp_hqd_pq_control &=
4955 
			~(QUEUE_SIZE_MASK | RPTR_BLOCK_SIZE_MASK);
4956 
 
4957 
		mqd->queue_state.cp_hqd_pq_control |=
4958 
			order_base_2(rdev->ring[idx].ring_size / 8);
4959 
		mqd->queue_state.cp_hqd_pq_control |=
4960 
			(order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8);
4961 
#ifdef __BIG_ENDIAN
4962 
		mqd->queue_state.cp_hqd_pq_control |= BUF_SWAP_32BIT;
4963 
#endif
4964 
		mqd->queue_state.cp_hqd_pq_control &=
4965 
			~(UNORD_DISPATCH | ROQ_PQ_IB_FLIP | PQ_VOLATILE);
4966 
		mqd->queue_state.cp_hqd_pq_control |=
4967 
			PRIV_STATE | KMD_QUEUE; /* assuming kernel queue control */
4968 
		WREG32(CP_HQD_PQ_CONTROL, mqd->queue_state.cp_hqd_pq_control);
4969 
 
4970 
		/* only used if CP_PQ_WPTR_POLL_CNTL.WPTR_POLL_EN=1 */
4971 
		if (i == 0)
4972 
			wb_gpu_addr = rdev->wb.gpu_addr + CIK_WB_CP1_WPTR_OFFSET;
4973 
		else
4974 
			wb_gpu_addr = rdev->wb.gpu_addr + CIK_WB_CP2_WPTR_OFFSET;
4975 
		mqd->queue_state.cp_hqd_pq_wptr_poll_addr = wb_gpu_addr & 0xfffffffc;
4976 
		mqd->queue_state.cp_hqd_pq_wptr_poll_addr_hi = upper_32_bits(wb_gpu_addr) & 0xffff;
4977 
		WREG32(CP_HQD_PQ_WPTR_POLL_ADDR, mqd->queue_state.cp_hqd_pq_wptr_poll_addr);
4978 
		WREG32(CP_HQD_PQ_WPTR_POLL_ADDR_HI,
4979 
		       mqd->queue_state.cp_hqd_pq_wptr_poll_addr_hi);
4980 
 
4981 
		/* set the wb address wether it's enabled or not */
4982 
		if (i == 0)
4983 
			wb_gpu_addr = rdev->wb.gpu_addr + RADEON_WB_CP1_RPTR_OFFSET;
4984 
		else
4985 
			wb_gpu_addr = rdev->wb.gpu_addr + RADEON_WB_CP2_RPTR_OFFSET;
4986 
		mqd->queue_state.cp_hqd_pq_rptr_report_addr = wb_gpu_addr & 0xfffffffc;
4987 
		mqd->queue_state.cp_hqd_pq_rptr_report_addr_hi =
4988 
			upper_32_bits(wb_gpu_addr) & 0xffff;
4989 
		WREG32(CP_HQD_PQ_RPTR_REPORT_ADDR,
4990 
		       mqd->queue_state.cp_hqd_pq_rptr_report_addr);
4991 
		WREG32(CP_HQD_PQ_RPTR_REPORT_ADDR_HI,
4992 
		       mqd->queue_state.cp_hqd_pq_rptr_report_addr_hi);
4993 
 
4994 
		/* enable the doorbell if requested */
4995 
		if (use_doorbell) {
4996 
			mqd->queue_state.cp_hqd_pq_doorbell_control =
4997 
				RREG32(CP_HQD_PQ_DOORBELL_CONTROL);
4998 
			mqd->queue_state.cp_hqd_pq_doorbell_control &= ~DOORBELL_OFFSET_MASK;
4999 
			mqd->queue_state.cp_hqd_pq_doorbell_control |=
5000 
				DOORBELL_OFFSET(rdev->ring[idx].doorbell_index);
5001 
			mqd->queue_state.cp_hqd_pq_doorbell_control |= DOORBELL_EN;
5002 
			mqd->queue_state.cp_hqd_pq_doorbell_control &=
5003 
				~(DOORBELL_SOURCE | DOORBELL_HIT);
5004 
 
5005 
		} else {
5006 
			mqd->queue_state.cp_hqd_pq_doorbell_control = 0;
5007 
		}
5008 
		WREG32(CP_HQD_PQ_DOORBELL_CONTROL,
5009 
		       mqd->queue_state.cp_hqd_pq_doorbell_control);
5010 
 
5011 
		/* read and write pointers, similar to CP_RB0_WPTR/_RPTR */
5012 
		rdev->ring[idx].wptr = 0;
5013 
		mqd->queue_state.cp_hqd_pq_wptr = rdev->ring[idx].wptr;
5014 
		WREG32(CP_HQD_PQ_WPTR, mqd->queue_state.cp_hqd_pq_wptr);
5015 
		mqd->queue_state.cp_hqd_pq_rptr = RREG32(CP_HQD_PQ_RPTR);
5016 
 
5017 
		/* set the vmid for the queue */
5018 
		mqd->queue_state.cp_hqd_vmid = 0;
5019 
		WREG32(CP_HQD_VMID, mqd->queue_state.cp_hqd_vmid);
5020 
 
5021 
		/* activate the queue */
5022 
		mqd->queue_state.cp_hqd_active = 1;
5023 
		WREG32(CP_HQD_ACTIVE, mqd->queue_state.cp_hqd_active);
5024 
 
5025 
		cik_srbm_select(rdev, 0, 0, 0, 0);
5026 
		mutex_unlock(&rdev->srbm_mutex);
5027 
 
5028 
		radeon_bo_kunmap(rdev->ring[idx].mqd_obj);
5029 
		radeon_bo_unreserve(rdev->ring[idx].mqd_obj);
5030 
 
5031 
		rdev->ring[idx].ready = true;
5032 
		r = radeon_ring_test(rdev, idx, &rdev->ring[idx]);
5033 
		if (r)
5034 
			rdev->ring[idx].ready = false;
5035 
	}
5036 
 
5037 
	return 0;
5038 
}
5039 
 
5040 
static void cik_cp_enable(struct radeon_device *rdev, bool enable)
5041 
{
5042 
	cik_cp_gfx_enable(rdev, enable);
5043 
	cik_cp_compute_enable(rdev, enable);
5044 
}
5045 
 
5046 
static int cik_cp_load_microcode(struct radeon_device *rdev)
5047 
{
5048 
	int r;
5049 
 
5050 
	r = cik_cp_gfx_load_microcode(rdev);
5051 
	if (r)
5052 
		return r;
5053 
	r = cik_cp_compute_load_microcode(rdev);
5054 
	if (r)
5055 
		return r;
5056 
 
5057 
	return 0;
5058 
}
5059 
 
5060 
static void cik_cp_fini(struct radeon_device *rdev)
5061 
{
5062 
	cik_cp_gfx_fini(rdev);
5063 
	cik_cp_compute_fini(rdev);
5064 
}
5065 
 
5066 
static int cik_cp_resume(struct radeon_device *rdev)
5067 
{
5068 
	int r;
5069 
 
5070 
	cik_enable_gui_idle_interrupt(rdev, false);
5071 
 
5072 
	r = cik_cp_load_microcode(rdev);
5073 
	if (r)
5074 
		return r;
5075 
 
5076 
	r = cik_cp_gfx_resume(rdev);
5077 
	if (r)
5078 
		return r;
5079 
	r = cik_cp_compute_resume(rdev);
5080 
	if (r)
5081 
		return r;
5082 
 
5083 
	cik_enable_gui_idle_interrupt(rdev, true);
5084 
 
5085 
	return 0;
5086 
}
5087 
 
5088 
static void cik_print_gpu_status_regs(struct radeon_device *rdev)
5089 
{
5090 
	dev_info(rdev->dev, "  GRBM_STATUS=0x%08X\n",
5091 
		RREG32(GRBM_STATUS));
5092 
	dev_info(rdev->dev, "  GRBM_STATUS2=0x%08X\n",
5093 
		RREG32(GRBM_STATUS2));
5094 
	dev_info(rdev->dev, "  GRBM_STATUS_SE0=0x%08X\n",
5095 
		RREG32(GRBM_STATUS_SE0));
5096 
	dev_info(rdev->dev, "  GRBM_STATUS_SE1=0x%08X\n",
5097 
		RREG32(GRBM_STATUS_SE1));
5098 
	dev_info(rdev->dev, "  GRBM_STATUS_SE2=0x%08X\n",
5099 
		RREG32(GRBM_STATUS_SE2));
5100 
	dev_info(rdev->dev, "  GRBM_STATUS_SE3=0x%08X\n",
5101 
		RREG32(GRBM_STATUS_SE3));
5102 
	dev_info(rdev->dev, "  SRBM_STATUS=0x%08X\n",
5103 
		RREG32(SRBM_STATUS));
5104 
	dev_info(rdev->dev, "  SRBM_STATUS2=0x%08X\n",
5105 
		RREG32(SRBM_STATUS2));
5106 
	dev_info(rdev->dev, "  SDMA0_STATUS_REG   = 0x%08X\n",
5107 
		RREG32(SDMA0_STATUS_REG + SDMA0_REGISTER_OFFSET));
5108 
	dev_info(rdev->dev, "  SDMA1_STATUS_REG   = 0x%08X\n",
5109 
		 RREG32(SDMA0_STATUS_REG + SDMA1_REGISTER_OFFSET));
5110 
	dev_info(rdev->dev, "  CP_STAT = 0x%08x\n", RREG32(CP_STAT));
5111 
	dev_info(rdev->dev, "  CP_STALLED_STAT1 = 0x%08x\n",
5112 
		 RREG32(CP_STALLED_STAT1));
5113 
	dev_info(rdev->dev, "  CP_STALLED_STAT2 = 0x%08x\n",
5114 
		 RREG32(CP_STALLED_STAT2));
5115 
	dev_info(rdev->dev, "  CP_STALLED_STAT3 = 0x%08x\n",
5116 
		 RREG32(CP_STALLED_STAT3));
5117 
	dev_info(rdev->dev, "  CP_CPF_BUSY_STAT = 0x%08x\n",
5118 
		 RREG32(CP_CPF_BUSY_STAT));
5119 
	dev_info(rdev->dev, "  CP_CPF_STALLED_STAT1 = 0x%08x\n",
5120 
		 RREG32(CP_CPF_STALLED_STAT1));
5121 
	dev_info(rdev->dev, "  CP_CPF_STATUS = 0x%08x\n", RREG32(CP_CPF_STATUS));
5122 
	dev_info(rdev->dev, "  CP_CPC_BUSY_STAT = 0x%08x\n", RREG32(CP_CPC_BUSY_STAT));
5123 
	dev_info(rdev->dev, "  CP_CPC_STALLED_STAT1 = 0x%08x\n",
5124 
		 RREG32(CP_CPC_STALLED_STAT1));
5125 
	dev_info(rdev->dev, "  CP_CPC_STATUS = 0x%08x\n", RREG32(CP_CPC_STATUS));
5126 
}
5127 
 
5128 
/**
5129 
 * cik_gpu_check_soft_reset - check which blocks are busy
5130 
 *
5131 
 * @rdev: radeon_device pointer
5132 
 *
5133 
 * Check which blocks are busy and return the relevant reset
5134 
 * mask to be used by cik_gpu_soft_reset().
5135 
 * Returns a mask of the blocks to be reset.
5136 
 */
5137 
u32 cik_gpu_check_soft_reset(struct radeon_device *rdev)
5138 
{
5139 
	u32 reset_mask = 0;
5140 
	u32 tmp;
5141 
 
5142 
	/* GRBM_STATUS */
5143 
	tmp = RREG32(GRBM_STATUS);
5144 
	if (tmp & (PA_BUSY | SC_BUSY |
5145 
		   BCI_BUSY | SX_BUSY |
5146 
		   TA_BUSY | VGT_BUSY |
5147 
		   DB_BUSY | CB_BUSY |
5148 
		   GDS_BUSY | SPI_BUSY |
5149 
		   IA_BUSY | IA_BUSY_NO_DMA))
5150 
		reset_mask |= RADEON_RESET_GFX;
5151 
 
5152 
	if (tmp & (CP_BUSY | CP_COHERENCY_BUSY))
5153 
		reset_mask |= RADEON_RESET_CP;
5154 
 
5155 
	/* GRBM_STATUS2 */
5156 
	tmp = RREG32(GRBM_STATUS2);
5157 
	if (tmp & RLC_BUSY)
5158 
		reset_mask |= RADEON_RESET_RLC;
5159 
 
5160 
	/* SDMA0_STATUS_REG */
5161 
	tmp = RREG32(SDMA0_STATUS_REG + SDMA0_REGISTER_OFFSET);
5162 
	if (!(tmp & SDMA_IDLE))
5163 
		reset_mask |= RADEON_RESET_DMA;
5164 
 
5165 
	/* SDMA1_STATUS_REG */
5166 
	tmp = RREG32(SDMA0_STATUS_REG + SDMA1_REGISTER_OFFSET);
5167 
	if (!(tmp & SDMA_IDLE))
5168 
		reset_mask |= RADEON_RESET_DMA1;
5169 
 
5170 
	/* SRBM_STATUS2 */
5171 
	tmp = RREG32(SRBM_STATUS2);
5172 
	if (tmp & SDMA_BUSY)
5173 
		reset_mask |= RADEON_RESET_DMA;
5174 
 
5175 
	if (tmp & SDMA1_BUSY)
5176 
		reset_mask |= RADEON_RESET_DMA1;
5177 
 
5178 
	/* SRBM_STATUS */
5179 
	tmp = RREG32(SRBM_STATUS);
5180 
 
5181 
	if (tmp & IH_BUSY)
5182 
		reset_mask |= RADEON_RESET_IH;
5183 
 
5184 
	if (tmp & SEM_BUSY)
5185 
		reset_mask |= RADEON_RESET_SEM;
5186 
 
5187 
	if (tmp & GRBM_RQ_PENDING)
5188 
		reset_mask |= RADEON_RESET_GRBM;
5189 
 
5190 
	if (tmp & VMC_BUSY)
5191 
		reset_mask |= RADEON_RESET_VMC;
5192 
 
5193 
	if (tmp & (MCB_BUSY | MCB_NON_DISPLAY_BUSY |
5194 
		   MCC_BUSY | MCD_BUSY))
5195 
		reset_mask |= RADEON_RESET_MC;
5196 
 
5197 
	if (evergreen_is_display_hung(rdev))
5198 
		reset_mask |= RADEON_RESET_DISPLAY;
5199 
 
5200 
	/* Skip MC reset as it's mostly likely not hung, just busy */
5201 
	if (reset_mask & RADEON_RESET_MC) {
5202 
		DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask);
5203 
		reset_mask &= ~RADEON_RESET_MC;
5204 
	}
5205 
 
5206 
	return reset_mask;
5207 
}
5208 
 
5209 
/**
5210 
 * cik_gpu_soft_reset - soft reset GPU
5211 
 *
5212 
 * @rdev: radeon_device pointer
5213 
 * @reset_mask: mask of which blocks to reset
5214 
 *
5215 
 * Soft reset the blocks specified in @reset_mask.
5216 
 */
5217 
static void cik_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask)
5218 
{
5219 
	struct evergreen_mc_save save;
5220 
	u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
5221 
	u32 tmp;
5222 
 
5223 
	if (reset_mask == 0)
5224 
		return;
5225 
 
5226 
	dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask);
5227 
 
5228 
	cik_print_gpu_status_regs(rdev);
5229 
	dev_info(rdev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_ADDR   0x%08X\n",
5230 
		 RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR));
5231 
	dev_info(rdev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
5232 
		 RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS));
5233 
 
5234 
	/* disable CG/PG */
5235 
	cik_fini_pg(rdev);
5236 
	cik_fini_cg(rdev);
5237 
 
5238 
	/* stop the rlc */
5239 
	cik_rlc_stop(rdev);
5240 
 
5241 
	/* Disable GFX parsing/prefetching */
5242 
	WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT);
5243 
 
5244 
	/* Disable MEC parsing/prefetching */
5245 
	WREG32(CP_MEC_CNTL, MEC_ME1_HALT | MEC_ME2_HALT);
5246 
 
5247 
	if (reset_mask & RADEON_RESET_DMA) {
5248 
		/* sdma0 */
5249 
		tmp = RREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET);
5250 
		tmp |= SDMA_HALT;
5251 
		WREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET, tmp);
5252 
	}
5253 
	if (reset_mask & RADEON_RESET_DMA1) {
5254 
		/* sdma1 */
5255 
		tmp = RREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET);
5256 
		tmp |= SDMA_HALT;
5257 
		WREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET, tmp);
5258 
	}
5259 
 
5260 
	evergreen_mc_stop(rdev, &save);
5261 
	if (evergreen_mc_wait_for_idle(rdev)) {
5262 
		dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
5263 
	}
5264 
 
5265 
	if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE | RADEON_RESET_CP))
5266 
		grbm_soft_reset = SOFT_RESET_CP | SOFT_RESET_GFX;
5267 
 
5268 
	if (reset_mask & RADEON_RESET_CP) {
5269 
		grbm_soft_reset |= SOFT_RESET_CP;
5270 
 
5271 
		srbm_soft_reset |= SOFT_RESET_GRBM;
5272 
	}
5273 
 
5274 
	if (reset_mask & RADEON_RESET_DMA)
5275 
		srbm_soft_reset |= SOFT_RESET_SDMA;
5276 
 
5277 
	if (reset_mask & RADEON_RESET_DMA1)
5278 
		srbm_soft_reset |= SOFT_RESET_SDMA1;
5279 
 
5280 
	if (reset_mask & RADEON_RESET_DISPLAY)
5281 
		srbm_soft_reset |= SOFT_RESET_DC;
5282 
 
5283 
	if (reset_mask & RADEON_RESET_RLC)
5284 
		grbm_soft_reset |= SOFT_RESET_RLC;
5285 
 
5286 
	if (reset_mask & RADEON_RESET_SEM)
5287 
		srbm_soft_reset |= SOFT_RESET_SEM;
5288 
 
5289 
	if (reset_mask & RADEON_RESET_IH)
5290 
		srbm_soft_reset |= SOFT_RESET_IH;
5291 
 
5292 
	if (reset_mask & RADEON_RESET_GRBM)
5293 
		srbm_soft_reset |= SOFT_RESET_GRBM;
5294 
 
5295 
	if (reset_mask & RADEON_RESET_VMC)
5296 
		srbm_soft_reset |= SOFT_RESET_VMC;
5297 
 
5298 
	if (!(rdev->flags & RADEON_IS_IGP)) {
5299 
		if (reset_mask & RADEON_RESET_MC)
5300 
			srbm_soft_reset |= SOFT_RESET_MC;
5301 
	}
5302 
 
5303 
	if (grbm_soft_reset) {
5304 
		tmp = RREG32(GRBM_SOFT_RESET);
5305 
		tmp |= grbm_soft_reset;
5306 
		dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
5307 
		WREG32(GRBM_SOFT_RESET, tmp);
5308 
		tmp = RREG32(GRBM_SOFT_RESET);
5309 
 
5310 
		udelay(50);
5311 
 
5312 
		tmp &= ~grbm_soft_reset;
5313 
		WREG32(GRBM_SOFT_RESET, tmp);
5314 
		tmp = RREG32(GRBM_SOFT_RESET);
5315 
	}
5316 
 
5317 
	if (srbm_soft_reset) {
5318 
		tmp = RREG32(SRBM_SOFT_RESET);
5319 
		tmp |= srbm_soft_reset;
5320 
		dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
5321 
		WREG32(SRBM_SOFT_RESET, tmp);
5322 
		tmp = RREG32(SRBM_SOFT_RESET);
5323 
 
5324 
		udelay(50);
5325 
 
5326 
		tmp &= ~srbm_soft_reset;
5327 
		WREG32(SRBM_SOFT_RESET, tmp);
5328 
		tmp = RREG32(SRBM_SOFT_RESET);
5329 
	}
5330 
 
5331 
	/* Wait a little for things to settle down */
5332 
	udelay(50);
5333 
 
5334 
	evergreen_mc_resume(rdev, &save);
5335 
	udelay(50);
5336 
 
5337 
	cik_print_gpu_status_regs(rdev);
5338 
}
5339 
 
5340 
struct kv_reset_save_regs {
5341 
	u32 gmcon_reng_execute;
5342 
	u32 gmcon_misc;
5343 
	u32 gmcon_misc3;
5344 
};
5345 
 
5346 
static void kv_save_regs_for_reset(struct radeon_device *rdev,
5347 
				   struct kv_reset_save_regs *save)
5348 
{
5349 
	save->gmcon_reng_execute = RREG32(GMCON_RENG_EXECUTE);
5350 
	save->gmcon_misc = RREG32(GMCON_MISC);
5351 
	save->gmcon_misc3 = RREG32(GMCON_MISC3);
5352 
 
5353 
	WREG32(GMCON_RENG_EXECUTE, save->gmcon_reng_execute & ~RENG_EXECUTE_ON_PWR_UP);
5354 
	WREG32(GMCON_MISC, save->gmcon_misc & ~(RENG_EXECUTE_ON_REG_UPDATE |
5355 
						STCTRL_STUTTER_EN));
5356 
}
5357 
 
5358 
static void kv_restore_regs_for_reset(struct radeon_device *rdev,
5359 
				      struct kv_reset_save_regs *save)
5360 
{
5361 
	int i;
5362 
 
5363 
	WREG32(GMCON_PGFSM_WRITE, 0);
5364 
	WREG32(GMCON_PGFSM_CONFIG, 0x200010ff);
5365 
 
5366 
	for (i = 0; i < 5; i++)
5367 
		WREG32(GMCON_PGFSM_WRITE, 0);
5368 
 
5369 
	WREG32(GMCON_PGFSM_WRITE, 0);
5370 
	WREG32(GMCON_PGFSM_CONFIG, 0x300010ff);
5371 
 
5372 
	for (i = 0; i < 5; i++)
5373 
		WREG32(GMCON_PGFSM_WRITE, 0);
5374 
 
5375 
	WREG32(GMCON_PGFSM_WRITE, 0x210000);
5376 
	WREG32(GMCON_PGFSM_CONFIG, 0xa00010ff);
5377 
 
5378 
	for (i = 0; i < 5; i++)
5379 
		WREG32(GMCON_PGFSM_WRITE, 0);
5380 
 
5381 
	WREG32(GMCON_PGFSM_WRITE, 0x21003);
5382 
	WREG32(GMCON_PGFSM_CONFIG, 0xb00010ff);
5383 
 
5384 
	for (i = 0; i < 5; i++)
5385 
		WREG32(GMCON_PGFSM_WRITE, 0);
5386 
 
5387 
	WREG32(GMCON_PGFSM_WRITE, 0x2b00);
5388 
	WREG32(GMCON_PGFSM_CONFIG, 0xc00010ff);
5389 
 
5390 
	for (i = 0; i < 5; i++)
5391 
		WREG32(GMCON_PGFSM_WRITE, 0);
5392 
 
5393 
	WREG32(GMCON_PGFSM_WRITE, 0);
5394 
	WREG32(GMCON_PGFSM_CONFIG, 0xd00010ff);
5395 
 
5396 
	for (i = 0; i < 5; i++)
5397 
		WREG32(GMCON_PGFSM_WRITE, 0);
5398 
 
5399 
	WREG32(GMCON_PGFSM_WRITE, 0x420000);
5400 
	WREG32(GMCON_PGFSM_CONFIG, 0x100010ff);
5401 
 
5402 
	for (i = 0; i < 5; i++)
5403 
		WREG32(GMCON_PGFSM_WRITE, 0);
5404 
 
5405 
	WREG32(GMCON_PGFSM_WRITE, 0x120202);
5406 
	WREG32(GMCON_PGFSM_CONFIG, 0x500010ff);
5407 
 
5408 
	for (i = 0; i < 5; i++)
5409 
		WREG32(GMCON_PGFSM_WRITE, 0);
5410 
 
5411 
	WREG32(GMCON_PGFSM_WRITE, 0x3e3e36);
5412 
	WREG32(GMCON_PGFSM_CONFIG, 0x600010ff);
5413 
 
5414 
	for (i = 0; i < 5; i++)
5415 
		WREG32(GMCON_PGFSM_WRITE, 0);
5416 
 
5417 
	WREG32(GMCON_PGFSM_WRITE, 0x373f3e);
5418 
	WREG32(GMCON_PGFSM_CONFIG, 0x700010ff);
5419 
 
5420 
	for (i = 0; i < 5; i++)
5421 
		WREG32(GMCON_PGFSM_WRITE, 0);
5422 
 
5423 
	WREG32(GMCON_PGFSM_WRITE, 0x3e1332);
5424 
	WREG32(GMCON_PGFSM_CONFIG, 0xe00010ff);
5425 
 
5426 
	WREG32(GMCON_MISC3, save->gmcon_misc3);
5427 
	WREG32(GMCON_MISC, save->gmcon_misc);
5428 
	WREG32(GMCON_RENG_EXECUTE, save->gmcon_reng_execute);
5429 
}
5430 
 
5431 
static void cik_gpu_pci_config_reset(struct radeon_device *rdev)
5432 
{
5433 
	struct evergreen_mc_save save;
5434 
	struct kv_reset_save_regs kv_save = { 0 };
5435 
	u32 tmp, i;
5436 
 
5437 
	dev_info(rdev->dev, "GPU pci config reset\n");
5438 
 
5439 
	/* disable dpm? */
5440 
 
5441 
	/* disable cg/pg */
5442 
	cik_fini_pg(rdev);
5443 
	cik_fini_cg(rdev);
5444 
 
5445 
	/* Disable GFX parsing/prefetching */
5446 
	WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT);
5447 
 
5448 
	/* Disable MEC parsing/prefetching */
5449 
	WREG32(CP_MEC_CNTL, MEC_ME1_HALT | MEC_ME2_HALT);
5450 
 
5451 
	/* sdma0 */
5452 
	tmp = RREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET);
5453 
	tmp |= SDMA_HALT;
5454 
	WREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET, tmp);
5455 
	/* sdma1 */
5456 
	tmp = RREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET);
5457 
	tmp |= SDMA_HALT;
5458 
	WREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET, tmp);
5459 
	/* XXX other engines? */
5460 
 
5461 
	/* halt the rlc, disable cp internal ints */
5462 
	cik_rlc_stop(rdev);
5463 
 
5464 
	udelay(50);
5465 
 
5466 
	/* disable mem access */
5467 
	evergreen_mc_stop(rdev, &save);
5468 
	if (evergreen_mc_wait_for_idle(rdev)) {
5469 
		dev_warn(rdev->dev, "Wait for MC idle timed out !\n");
5470 
	}
5471 
 
5472 
	if (rdev->flags & RADEON_IS_IGP)
5473 
		kv_save_regs_for_reset(rdev, &kv_save);
5474 
 
5475 
	/* disable BM */
5476 
	pci_clear_master(rdev->pdev);
5477 
	/* reset */
5478 
	radeon_pci_config_reset(rdev);
5479 
 
5480 
	udelay(100);
5481 
 
5482 
	/* wait for asic to come out of reset */
5483 
	for (i = 0; i < rdev->usec_timeout; i++) {
5484 
		if (RREG32(CONFIG_MEMSIZE) != 0xffffffff)
5485 
			break;
5486 
		udelay(1);
5487 
	}
5488 
 
5489 
	/* does asic init need to be run first??? */
5490 
	if (rdev->flags & RADEON_IS_IGP)
5491 
		kv_restore_regs_for_reset(rdev, &kv_save);
5492 
}
5493 
 
5494 
/**
5495 
 * cik_asic_reset - soft reset GPU
5496 
 *
5497 
 * @rdev: radeon_device pointer
5498 
 *
5499 
 * Look up which blocks are hung and attempt
5500 
 * to reset them.
5501 
 * Returns 0 for success.
5502 
 */
5503 
int cik_asic_reset(struct radeon_device *rdev)
5504 
{
5505 
	u32 reset_mask;
5506 
 
5507 
	reset_mask = cik_gpu_check_soft_reset(rdev);
5508 
 
5509 
	if (reset_mask)
5510 
		r600_set_bios_scratch_engine_hung(rdev, true);
5511 
 
5512 
	/* try soft reset */
5513 
	cik_gpu_soft_reset(rdev, reset_mask);
5514 
 
5515 
	reset_mask = cik_gpu_check_soft_reset(rdev);
5516 
 
5517 
	/* try pci config reset */
5518 
	if (reset_mask && radeon_hard_reset)
5519 
		cik_gpu_pci_config_reset(rdev);
5520 
 
5521 
	reset_mask = cik_gpu_check_soft_reset(rdev);
5522 
 
5523 
	if (!reset_mask)
5524 
		r600_set_bios_scratch_engine_hung(rdev, false);
5525 
 
5526 
	return 0;
5527 
}
5528 
 
5529 
/**
5530 
 * cik_gfx_is_lockup - check if the 3D engine is locked up
5531 
 *
5532 
 * @rdev: radeon_device pointer
5533 
 * @ring: radeon_ring structure holding ring information
5534 
 *
5535 
 * Check if the 3D engine is locked up (CIK).
5536 
 * Returns true if the engine is locked, false if not.
5537 
 */
5538 
bool cik_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
5539 
{
5540 
	u32 reset_mask = cik_gpu_check_soft_reset(rdev);
5541 
 
5542 
	if (!(reset_mask & (RADEON_RESET_GFX |
5543 
			    RADEON_RESET_COMPUTE |
5544 
			    RADEON_RESET_CP))) {
5545 
		radeon_ring_lockup_update(rdev, ring);
5546 
		return false;
5547 
	}
5548 
	return radeon_ring_test_lockup(rdev, ring);
5549 
}
5550 
 
5551 
/* MC */
5552 
/**
5553 
 * cik_mc_program - program the GPU memory controller
5554 
 *
5555 
 * @rdev: radeon_device pointer
5556 
 *
5557 
 * Set the location of vram, gart, and AGP in the GPU's
5558 
 * physical address space (CIK).
5559 
 */
5560 
static void cik_mc_program(struct radeon_device *rdev)
5561 
{
5562 
	struct evergreen_mc_save save;
5563 
	u32 tmp;
5564 
	int i, j;
5565 
 
5566 
	/* Initialize HDP */
5567 
	for (i = 0, j = 0; i < 32; i++, j += 0x18) {
5568 
		WREG32((0x2c14 + j), 0x00000000);
5569 
		WREG32((0x2c18 + j), 0x00000000);
5570 
		WREG32((0x2c1c + j), 0x00000000);
5571 
		WREG32((0x2c20 + j), 0x00000000);
5572 
		WREG32((0x2c24 + j), 0x00000000);
5573 
	}
5574 
	WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0);
5575 
 
5576 
	evergreen_mc_stop(rdev, &save);
5577 
	if (radeon_mc_wait_for_idle(rdev)) {
5578 
		dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
5579 
	}
5580 
	/* Lockout access through VGA aperture*/
5581 
	WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE);
5582 
	/* Update configuration */
5583 
	WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
5584 
	       rdev->mc.vram_start >> 12);
5585 
	WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
5586 
	       rdev->mc.vram_end >> 12);
5587 
	WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR,
5588 
	       rdev->vram_scratch.gpu_addr >> 12);
5589 
	tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16;
5590 
	tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF);
5591 
	WREG32(MC_VM_FB_LOCATION, tmp);
5592 
	/* XXX double check these! */
5593 
	WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8));
5594 
	WREG32(HDP_NONSURFACE_INFO, (2 << 7) | (1 << 30));
5595 
	WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF);
5596 
	WREG32(MC_VM_AGP_BASE, 0);
5597 
	WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF);
5598 
	WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF);
5599 
	if (radeon_mc_wait_for_idle(rdev)) {
5600 
		dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
5601 
	}
5602 
	evergreen_mc_resume(rdev, &save);
5603 
	/* we need to own VRAM, so turn off the VGA renderer here
5604 
	 * to stop it overwriting our objects */
5605 
	rv515_vga_render_disable(rdev);
5606 
}
5607 
 
5608 
/**
5609 
 * cik_mc_init - initialize the memory controller driver params
5610 
 *
5611 
 * @rdev: radeon_device pointer
5612 
 *
5613 
 * Look up the amount of vram, vram width, and decide how to place
5614 
 * vram and gart within the GPU's physical address space (CIK).
5615 
 * Returns 0 for success.
5616 
 */
5617 
static int cik_mc_init(struct radeon_device *rdev)
5618 
{
5619 
	u32 tmp;
5620 
	int chansize, numchan;
5621 
 
5622 
	/* Get VRAM informations */
5623 
	rdev->mc.vram_is_ddr = true;
5624 
	tmp = RREG32(MC_ARB_RAMCFG);
5625 
	if (tmp & CHANSIZE_MASK) {
5626 
		chansize = 64;
5627 
	} else {
5628 
		chansize = 32;
5629 
	}
5630 
	tmp = RREG32(MC_SHARED_CHMAP);
5631 
	switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
5632 
	case 0:
5633 
	default:
5634 
		numchan = 1;
5635 
		break;
5636 
	case 1:
5637 
		numchan = 2;
5638 
		break;
5639 
	case 2:
5640 
		numchan = 4;
5641 
		break;
5642 
	case 3:
5643 
		numchan = 8;
5644 
		break;
5645 
	case 4:
5646 
		numchan = 3;
5647 
		break;
5648 
	case 5:
5649 
		numchan = 6;
5650 
		break;
5651 
	case 6:
5652 
		numchan = 10;
5653 
		break;
5654 
	case 7:
5655 
		numchan = 12;
5656 
		break;
5657 
	case 8:
5658 
		numchan = 16;
5659 
		break;
5660 
	}
5661 
	rdev->mc.vram_width = numchan * chansize;
5662 
	/* Could aper size report 0 ? */
5663 
	rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0);
5664 
	rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0);
5665 
	/* size in MB on si */
5666 
	rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
5667 
	rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
5668 
	rdev->mc.visible_vram_size = rdev->mc.aper_size;
5669 
	si_vram_gtt_location(rdev, &rdev->mc);
5670 
	radeon_update_bandwidth_info(rdev);
5671 
 
5672 
	return 0;
5673 
}
5674 
 
5675 
/*
5676 
 * GART
5677 
 * VMID 0 is the physical GPU addresses as used by the kernel.
5678 
 * VMIDs 1-15 are used for userspace clients and are handled
5679 
 * by the radeon vm/hsa code.
5680 
 */
5681 
/**
5682 
 * cik_pcie_gart_tlb_flush - gart tlb flush callback
5683 
 *
5684 
 * @rdev: radeon_device pointer
5685 
 *
5686 
 * Flush the TLB for the VMID 0 page table (CIK).
5687 
 */
5688 
void cik_pcie_gart_tlb_flush(struct radeon_device *rdev)
5689 
{
5690 
	/* flush hdp cache */
5691 
	WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0);
5692 
 
5693 
	/* bits 0-15 are the VM contexts0-15 */
5694 
	WREG32(VM_INVALIDATE_REQUEST, 0x1);
5695 
}
5696 
 
5697 
/**
5698 
 * cik_pcie_gart_enable - gart enable
5699 
 *
5700 
 * @rdev: radeon_device pointer
5701 
 *
5702 
 * This sets up the TLBs, programs the page tables for VMID0,
5703 
 * sets up the hw for VMIDs 1-15 which are allocated on
5704 
 * demand, and sets up the global locations for the LDS, GDS,
5705 
 * and GPUVM for FSA64 clients (CIK).
5706 
 * Returns 0 for success, errors for failure.
5707 
 */
5708 
static int cik_pcie_gart_enable(struct radeon_device *rdev)
5709 
{
5710 
	int r, i;
5711 
 
5712 
	if (rdev->gart.robj == NULL) {
5713 
		dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
5714 
		return -EINVAL;
5715 
	}
5716 
	r = radeon_gart_table_vram_pin(rdev);
5717 
	if (r)
5718 
		return r;
5719 
	/* Setup TLB control */
5720 
	WREG32(MC_VM_MX_L1_TLB_CNTL,
5721 
	       (0xA << 7) |
5722 
	       ENABLE_L1_TLB |
5723 
	       ENABLE_L1_FRAGMENT_PROCESSING |
5724 
	       SYSTEM_ACCESS_MODE_NOT_IN_SYS |
5725 
	       ENABLE_ADVANCED_DRIVER_MODEL |
5726 
	       SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
5727 
	/* Setup L2 cache */
5728 
	WREG32(VM_L2_CNTL, ENABLE_L2_CACHE |
5729 
	       ENABLE_L2_FRAGMENT_PROCESSING |
5730 
	       ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
5731 
	       ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
5732 
	       EFFECTIVE_L2_QUEUE_SIZE(7) |
5733 
	       CONTEXT1_IDENTITY_ACCESS_MODE(1));
5734 
	WREG32(VM_L2_CNTL2, INVALIDATE_ALL_L1_TLBS | INVALIDATE_L2_CACHE);
5735 
	WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
5736 
	       BANK_SELECT(4) |
5737 
	       L2_CACHE_BIGK_FRAGMENT_SIZE(4));
5738 
	/* setup context0 */
5739 
	WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
5740 
	WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
5741 
	WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
5742 
	WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
5743 
			(u32)(rdev->dummy_page.addr >> 12));
5744 
	WREG32(VM_CONTEXT0_CNTL2, 0);
5745 
	WREG32(VM_CONTEXT0_CNTL, (ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
5746 
				  RANGE_PROTECTION_FAULT_ENABLE_DEFAULT));
5747 
 
5748 
	WREG32(0x15D4, 0);
5749 
	WREG32(0x15D8, 0);
5750 
	WREG32(0x15DC, 0);
5751 
 
5752 
	/* restore context1-15 */
5753 
	/* set vm size, must be a multiple of 4 */
5754 
	WREG32(VM_CONTEXT1_PAGE_TABLE_START_ADDR, 0);
5755 
	WREG32(VM_CONTEXT1_PAGE_TABLE_END_ADDR, rdev->vm_manager.max_pfn);
5756 
	for (i = 1; i < 16; i++) {
5757 
		if (i < 8)
5758 
			WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (i << 2),
5759 
			       rdev->vm_manager.saved_table_addr[i]);
5760 
		else
5761 
			WREG32(VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((i - 8) << 2),
5762 
			       rdev->vm_manager.saved_table_addr[i]);
5763 
	}
5764 
 
5765 
	/* enable context1-15 */
5766 
	WREG32(VM_CONTEXT1_PROTECTION_FAULT_DEFAULT_ADDR,
5767 
	       (u32)(rdev->dummy_page.addr >> 12));
5768 
	WREG32(VM_CONTEXT1_CNTL2, 4);
5769 
	WREG32(VM_CONTEXT1_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(1) |
5770 
				PAGE_TABLE_BLOCK_SIZE(radeon_vm_block_size - 9) |
5771 
				RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
5772 
				RANGE_PROTECTION_FAULT_ENABLE_DEFAULT |
5773 
				DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
5774 
				DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT |
5775 
				PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT |
5776 
				PDE0_PROTECTION_FAULT_ENABLE_DEFAULT |
5777 
				VALID_PROTECTION_FAULT_ENABLE_INTERRUPT |
5778 
				VALID_PROTECTION_FAULT_ENABLE_DEFAULT |
5779 
				READ_PROTECTION_FAULT_ENABLE_INTERRUPT |
5780 
				READ_PROTECTION_FAULT_ENABLE_DEFAULT |
5781 
				WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT |
5782 
				WRITE_PROTECTION_FAULT_ENABLE_DEFAULT);
5783 
 
5784 
	if (rdev->family == CHIP_KAVERI) {
5785 
		u32 tmp = RREG32(CHUB_CONTROL);
5786 
		tmp &= ~BYPASS_VM;
5787 
		WREG32(CHUB_CONTROL, tmp);
5788 
	}
5789 
 
5790 
	/* XXX SH_MEM regs */
5791 
	/* where to put LDS, scratch, GPUVM in FSA64 space */
5792 
	mutex_lock(&rdev->srbm_mutex);
5793 
	for (i = 0; i < 16; i++) {
5794 
		cik_srbm_select(rdev, 0, 0, 0, i);
5795 
		/* CP and shaders */
5796 
		WREG32(SH_MEM_CONFIG, 0);
5797 
		WREG32(SH_MEM_APE1_BASE, 1);
5798 
		WREG32(SH_MEM_APE1_LIMIT, 0);
5799 
		WREG32(SH_MEM_BASES, 0);
5800 
		/* SDMA GFX */
5801 
		WREG32(SDMA0_GFX_VIRTUAL_ADDR + SDMA0_REGISTER_OFFSET, 0);
5802 
		WREG32(SDMA0_GFX_APE1_CNTL + SDMA0_REGISTER_OFFSET, 0);
5803 
		WREG32(SDMA0_GFX_VIRTUAL_ADDR + SDMA1_REGISTER_OFFSET, 0);
5804 
		WREG32(SDMA0_GFX_APE1_CNTL + SDMA1_REGISTER_OFFSET, 0);
5805 
		/* XXX SDMA RLC - todo */
5806 
	}
5807 
	cik_srbm_select(rdev, 0, 0, 0, 0);
5808 
	mutex_unlock(&rdev->srbm_mutex);
5809 
 
5810 
	cik_pcie_gart_tlb_flush(rdev);
5811 
	DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
5812 
		 (unsigned)(rdev->mc.gtt_size >> 20),
5813 
		 (unsigned long long)rdev->gart.table_addr);
5814 
	rdev->gart.ready = true;
5815 
	return 0;
5816 
}
5817 
 
5818 
/**
5819 
 * cik_pcie_gart_disable - gart disable
5820 
 *
5821 
 * @rdev: radeon_device pointer
5822 
 *
5823 
 * This disables all VM page table (CIK).
5824 
 */
5825 
static void cik_pcie_gart_disable(struct radeon_device *rdev)
5826 
{
5827 
	unsigned i;
5828 
 
5829 
	for (i = 1; i < 16; ++i) {
5830 
		uint32_t reg;
5831 
		if (i < 8)
5832 
			reg = VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (i << 2);
5833 
		else
5834 
			reg = VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((i - 8) << 2);
5835 
		rdev->vm_manager.saved_table_addr[i] = RREG32(reg);
5836 
	}
5837 
 
5838 
	/* Disable all tables */
5839 
	WREG32(VM_CONTEXT0_CNTL, 0);
5840 
	WREG32(VM_CONTEXT1_CNTL, 0);
5841 
	/* Setup TLB control */
5842 
	WREG32(MC_VM_MX_L1_TLB_CNTL, SYSTEM_ACCESS_MODE_NOT_IN_SYS |
5843 
	       SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
5844 
	/* Setup L2 cache */
5845 
	WREG32(VM_L2_CNTL,
5846 
	       ENABLE_L2_FRAGMENT_PROCESSING |
5847 
	       ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
5848 
	       ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
5849 
	       EFFECTIVE_L2_QUEUE_SIZE(7) |
5850 
	       CONTEXT1_IDENTITY_ACCESS_MODE(1));
5851 
	WREG32(VM_L2_CNTL2, 0);
5852 
	WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
5853 
	       L2_CACHE_BIGK_FRAGMENT_SIZE(6));
5854 
	radeon_gart_table_vram_unpin(rdev);
5855 
}
5856 
 
5857 
/**
5858 
 * cik_pcie_gart_fini - vm fini callback
5859 
 *
5860 
 * @rdev: radeon_device pointer
5861 
 *
5862 
 * Tears down the driver GART/VM setup (CIK).
5863 
 */
5864 
static void cik_pcie_gart_fini(struct radeon_device *rdev)
5865 
{
5866 
	cik_pcie_gart_disable(rdev);
5867 
	radeon_gart_table_vram_free(rdev);
5868 
	radeon_gart_fini(rdev);
5869 
}
5870 
 
5871 
/* vm parser */
5872 
/**
5873 
 * cik_ib_parse - vm ib_parse callback
5874 
 *
5875 
 * @rdev: radeon_device pointer
5876 
 * @ib: indirect buffer pointer
5877 
 *
5878 
 * CIK uses hw IB checking so this is a nop (CIK).
5879 
 */
5880 
int cik_ib_parse(struct radeon_device *rdev, struct radeon_ib *ib)
5881 
{
5882 
	return 0;
5883 
}
5884 
 
5885 
/*
5886 
 * vm
5887 
 * VMID 0 is the physical GPU addresses as used by the kernel.
5888 
 * VMIDs 1-15 are used for userspace clients and are handled
5889 
 * by the radeon vm/hsa code.
5890 
 */
5891 
/**
5892 
 * cik_vm_init - cik vm init callback
5893 
 *
5894 
 * @rdev: radeon_device pointer
5895 
 *
5896 
 * Inits cik specific vm parameters (number of VMs, base of vram for
5897 
 * VMIDs 1-15) (CIK).
5898 
 * Returns 0 for success.
5899 
 */
5900 
int cik_vm_init(struct radeon_device *rdev)
5901 
{
5902 
	/* number of VMs */
5903 
	rdev->vm_manager.nvm = 16;
5904 
	/* base offset of vram pages */
5905 
	if (rdev->flags & RADEON_IS_IGP) {
5906 
		u64 tmp = RREG32(MC_VM_FB_OFFSET);
5907 
		tmp <<= 22;
5908 
		rdev->vm_manager.vram_base_offset = tmp;
5909 
	} else
5910 
		rdev->vm_manager.vram_base_offset = 0;
5911 
 
5912 
	return 0;
5913 
}
5914 
 
5915 
/**
5916 
 * cik_vm_fini - cik vm fini callback
5917 
 *
5918 
 * @rdev: radeon_device pointer
5919 
 *
5920 
 * Tear down any asic specific VM setup (CIK).
5921 
 */
5922 
void cik_vm_fini(struct radeon_device *rdev)
5923 
{
5924 
}
5925 
 
5926 
/**
5927 
 * cik_vm_decode_fault - print human readable fault info
5928 
 *
5929 
 * @rdev: radeon_device pointer
5930 
 * @status: VM_CONTEXT1_PROTECTION_FAULT_STATUS register value
5931 
 * @addr: VM_CONTEXT1_PROTECTION_FAULT_ADDR register value
5932 
 *
5933 
 * Print human readable fault information (CIK).
5934 
 */
5935 
static void cik_vm_decode_fault(struct radeon_device *rdev,
5936 
				u32 status, u32 addr, u32 mc_client)
5937 
{
5938 
	u32 mc_id;
5939 
	u32 vmid = (status & FAULT_VMID_MASK) >> FAULT_VMID_SHIFT;
5940 
	u32 protections = (status & PROTECTIONS_MASK) >> PROTECTIONS_SHIFT;
5941 
	char block[5] = { mc_client >> 24, (mc_client >> 16) & 0xff,
5942 
		(mc_client >> 8) & 0xff, mc_client & 0xff, 0 };
5943 
 
5944 
	if (rdev->family == CHIP_HAWAII)
5945 
		mc_id = (status & HAWAII_MEMORY_CLIENT_ID_MASK) >> MEMORY_CLIENT_ID_SHIFT;
5946 
	else
5947 
		mc_id = (status & MEMORY_CLIENT_ID_MASK) >> MEMORY_CLIENT_ID_SHIFT;
5948 
 
5949 
	printk("VM fault (0x%02x, vmid %d) at page %u, %s from '%s' (0x%08x) (%d)\n",
5950 
	       protections, vmid, addr,
5951 
	       (status & MEMORY_CLIENT_RW_MASK) ? "write" : "read",
5952 
	       block, mc_client, mc_id);
5953 
}
5954 
 
5955 
/**
5956 
 * cik_vm_flush - cik vm flush using the CP
5957 
 *
5958 
 * @rdev: radeon_device pointer
5959 
 *
5960 
 * Update the page table base and flush the VM TLB
5961 
 * using the CP (CIK).
5962 
 */
5963 
void cik_vm_flush(struct radeon_device *rdev, int ridx, struct radeon_vm *vm)
5964 
{
5965 
	struct radeon_ring *ring = &rdev->ring[ridx];
5966 
	int usepfp = (ridx == RADEON_RING_TYPE_GFX_INDEX);
5967 
 
5968 
	if (vm == NULL)
5969 
		return;
5970 
 
5971 
	radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5972 
	radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(usepfp) |
5973 
				 WRITE_DATA_DST_SEL(0)));
5974 
	if (vm->id < 8) {
5975 
		radeon_ring_write(ring,
5976 
				  (VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (vm->id << 2)) >> 2);
5977 
	} else {
5978 
		radeon_ring_write(ring,
5979 
				  (VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((vm->id - 8) << 2)) >> 2);
5980 
	}
5981 
	radeon_ring_write(ring, 0);
5982 
	radeon_ring_write(ring, vm->pd_gpu_addr >> 12);
5983 
 
5984 
	/* update SH_MEM_* regs */
5985 
	radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5986 
	radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(usepfp) |
5987 
				 WRITE_DATA_DST_SEL(0)));
5988 
	radeon_ring_write(ring, SRBM_GFX_CNTL >> 2);
5989 
	radeon_ring_write(ring, 0);
5990 
	radeon_ring_write(ring, VMID(vm->id));
5991 
 
5992 
	radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 6));
5993 
	radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(usepfp) |
5994 
				 WRITE_DATA_DST_SEL(0)));
5995 
	radeon_ring_write(ring, SH_MEM_BASES >> 2);
5996 
	radeon_ring_write(ring, 0);
5997 
 
5998 
	radeon_ring_write(ring, 0); /* SH_MEM_BASES */
5999 
	radeon_ring_write(ring, 0); /* SH_MEM_CONFIG */
6000 
	radeon_ring_write(ring, 1); /* SH_MEM_APE1_BASE */
6001 
	radeon_ring_write(ring, 0); /* SH_MEM_APE1_LIMIT */
6002 
 
6003 
	radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
6004 
	radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(usepfp) |
6005 
				 WRITE_DATA_DST_SEL(0)));
6006 
	radeon_ring_write(ring, SRBM_GFX_CNTL >> 2);
6007 
	radeon_ring_write(ring, 0);
6008 
	radeon_ring_write(ring, VMID(0));
6009 
 
6010 
	/* HDP flush */
6011 
	cik_hdp_flush_cp_ring_emit(rdev, ridx);
6012 
 
6013 
	/* bits 0-15 are the VM contexts0-15 */
6014 
	radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
6015 
	radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(usepfp) |
6016 
				 WRITE_DATA_DST_SEL(0)));
6017 
	radeon_ring_write(ring, VM_INVALIDATE_REQUEST >> 2);
6018 
	radeon_ring_write(ring, 0);
6019 
	radeon_ring_write(ring, 1 << vm->id);
6020 
 
6021 
	/* compute doesn't have PFP */
6022 
	if (usepfp) {
6023 
		/* sync PFP to ME, otherwise we might get invalid PFP reads */
6024 
		radeon_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0));
6025 
		radeon_ring_write(ring, 0x0);
6026 
	}
6027 
}
6028 
 
6029 
/*
6030 
 * RLC
6031 
 * The RLC is a multi-purpose microengine that handles a
6032 
 * variety of functions, the most important of which is
6033 
 * the interrupt controller.
6034 
 */
6035 
static void cik_enable_gui_idle_interrupt(struct radeon_device *rdev,
6036 
					  bool enable)
6037 
{
6038 
	u32 tmp = RREG32(CP_INT_CNTL_RING0);
6039 
 
6040 
	if (enable)
6041 
		tmp |= (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
6042 
	else
6043 
		tmp &= ~(CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
6044 
	WREG32(CP_INT_CNTL_RING0, tmp);
6045 
}
6046 
 
6047 
static void cik_enable_lbpw(struct radeon_device *rdev, bool enable)
6048 
{
6049 
	u32 tmp;
6050 
 
6051 
	tmp = RREG32(RLC_LB_CNTL);
6052 
	if (enable)
6053 
		tmp |= LOAD_BALANCE_ENABLE;
6054 
	else
6055 
		tmp &= ~LOAD_BALANCE_ENABLE;
6056 
	WREG32(RLC_LB_CNTL, tmp);
6057 
}
6058 
 
6059 
static void cik_wait_for_rlc_serdes(struct radeon_device *rdev)
6060 
{
6061 
	u32 i, j, k;
6062 
	u32 mask;
6063 
 
6064 
	for (i = 0; i < rdev->config.cik.max_shader_engines; i++) {
6065 
		for (j = 0; j < rdev->config.cik.max_sh_per_se; j++) {
6066 
			cik_select_se_sh(rdev, i, j);
6067 
			for (k = 0; k < rdev->usec_timeout; k++) {
6068 
				if (RREG32(RLC_SERDES_CU_MASTER_BUSY) == 0)
6069 
					break;
6070 
				udelay(1);
6071 
			}
6072 
		}
6073 
	}
6074 
	cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
6075 
 
6076 
	mask = SE_MASTER_BUSY_MASK | GC_MASTER_BUSY | TC0_MASTER_BUSY | TC1_MASTER_BUSY;
6077 
	for (k = 0; k < rdev->usec_timeout; k++) {
6078 
		if ((RREG32(RLC_SERDES_NONCU_MASTER_BUSY) & mask) == 0)
6079 
			break;
6080 
		udelay(1);
6081 
	}
6082 
}
6083 
 
6084 
static void cik_update_rlc(struct radeon_device *rdev, u32 rlc)
6085 
{
6086 
	u32 tmp;
6087 
 
6088 
	tmp = RREG32(RLC_CNTL);
6089 
	if (tmp != rlc)
6090 
		WREG32(RLC_CNTL, rlc);
6091 
}
6092 
 
6093 
static u32 cik_halt_rlc(struct radeon_device *rdev)
6094 
{
6095 
	u32 data, orig;
6096 
 
6097 
	orig = data = RREG32(RLC_CNTL);
6098 
 
6099 
	if (data & RLC_ENABLE) {
6100 
		u32 i;
6101 
 
6102 
		data &= ~RLC_ENABLE;
6103 
		WREG32(RLC_CNTL, data);
6104 
 
6105 
		for (i = 0; i < rdev->usec_timeout; i++) {
6106 
			if ((RREG32(RLC_GPM_STAT) & RLC_GPM_BUSY) == 0)
6107 
				break;
6108 
			udelay(1);
6109 
		}
6110 
 
6111 
		cik_wait_for_rlc_serdes(rdev);
6112 
	}
6113 
 
6114 
	return orig;
6115 
}
6116 
 
6117 
void cik_enter_rlc_safe_mode(struct radeon_device *rdev)
6118 
{
6119 
	u32 tmp, i, mask;
6120 
 
6121 
	tmp = REQ | MESSAGE(MSG_ENTER_RLC_SAFE_MODE);
6122 
	WREG32(RLC_GPR_REG2, tmp);
6123 
 
6124 
	mask = GFX_POWER_STATUS | GFX_CLOCK_STATUS;
6125 
	for (i = 0; i < rdev->usec_timeout; i++) {
6126 
		if ((RREG32(RLC_GPM_STAT) & mask) == mask)
6127 
			break;
6128 
		udelay(1);
6129 
	}
6130 
 
6131 
	for (i = 0; i < rdev->usec_timeout; i++) {
6132 
		if ((RREG32(RLC_GPR_REG2) & REQ) == 0)
6133 
			break;
6134 
		udelay(1);
6135 
	}
6136 
}
6137 
 
6138 
void cik_exit_rlc_safe_mode(struct radeon_device *rdev)
6139 
{
6140 
	u32 tmp;
6141 
 
6142 
	tmp = REQ | MESSAGE(MSG_EXIT_RLC_SAFE_MODE);
6143 
	WREG32(RLC_GPR_REG2, tmp);
6144 
}
6145 
 
6146 
/**
6147 
 * cik_rlc_stop - stop the RLC ME
6148 
 *
6149 
 * @rdev: radeon_device pointer
6150 
 *
6151 
 * Halt the RLC ME (MicroEngine) (CIK).
6152 
 */
6153 
static void cik_rlc_stop(struct radeon_device *rdev)
6154 
{
6155 
	WREG32(RLC_CNTL, 0);
6156 
 
6157 
	cik_enable_gui_idle_interrupt(rdev, false);
6158 
 
6159 
	cik_wait_for_rlc_serdes(rdev);
6160 
}
6161 
 
6162 
/**
6163 
 * cik_rlc_start - start the RLC ME
6164 
 *
6165 
 * @rdev: radeon_device pointer
6166 
 *
6167 
 * Unhalt the RLC ME (MicroEngine) (CIK).
6168 
 */
6169 
static void cik_rlc_start(struct radeon_device *rdev)
6170 
{
6171 
	WREG32(RLC_CNTL, RLC_ENABLE);
6172 
 
6173 
	cik_enable_gui_idle_interrupt(rdev, true);
6174 
 
6175 
	udelay(50);
6176 
}
6177 
 
6178 
/**
6179 
 * cik_rlc_resume - setup the RLC hw
6180 
 *
6181 
 * @rdev: radeon_device pointer
6182 
 *
6183 
 * Initialize the RLC registers, load the ucode,
6184 
 * and start the RLC (CIK).
6185 
 * Returns 0 for success, -EINVAL if the ucode is not available.
6186 
 */
6187 
static int cik_rlc_resume(struct radeon_device *rdev)
6188 
{
6189 
	u32 i, size, tmp;
6190 
 
6191 
	if (!rdev->rlc_fw)
6192 
		return -EINVAL;
6193 
 
6194 
	cik_rlc_stop(rdev);
6195 
 
6196 
	/* disable CG */
6197 
	tmp = RREG32(RLC_CGCG_CGLS_CTRL) & 0xfffffffc;
6198 
	WREG32(RLC_CGCG_CGLS_CTRL, tmp);
6199 
 
6200 
	si_rlc_reset(rdev);
6201 
 
6202 
	cik_init_pg(rdev);
6203 
 
6204 
	cik_init_cg(rdev);
6205 
 
6206 
	WREG32(RLC_LB_CNTR_INIT, 0);
6207 
	WREG32(RLC_LB_CNTR_MAX, 0x00008000);
6208 
 
6209 
	cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
6210 
	WREG32(RLC_LB_INIT_CU_MASK, 0xffffffff);
6211 
	WREG32(RLC_LB_PARAMS, 0x00600408);
6212 
	WREG32(RLC_LB_CNTL, 0x80000004);
6213 
 
6214 
	WREG32(RLC_MC_CNTL, 0);
6215 
	WREG32(RLC_UCODE_CNTL, 0);
6216 
 
6217 
	if (rdev->new_fw) {
6218 
		const struct rlc_firmware_header_v1_0 *hdr =
6219 
			(const struct rlc_firmware_header_v1_0 *)rdev->rlc_fw->data;
6220 
		const __le32 *fw_data = (const __le32 *)
6221 
			(rdev->rlc_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
6222 
 
6223 
		radeon_ucode_print_rlc_hdr(&hdr->header);
6224 
 
6225 
		size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4;
6226 
		WREG32(RLC_GPM_UCODE_ADDR, 0);
6227 
		for (i = 0; i < size; i++)
6228 
			WREG32(RLC_GPM_UCODE_DATA, le32_to_cpup(fw_data++));
6229 
		WREG32(RLC_GPM_UCODE_ADDR, 0);
6230 
	} else {
6231 
		const __be32 *fw_data;
6232 
 
6233 
		switch (rdev->family) {
6234 
		case CHIP_BONAIRE:
6235 
		case CHIP_HAWAII:
6236 
		default:
6237 
			size = BONAIRE_RLC_UCODE_SIZE;
6238 
			break;
6239 
		case CHIP_KAVERI:
6240 
			size = KV_RLC_UCODE_SIZE;
6241 
			break;
6242 
		case CHIP_KABINI:
6243 
			size = KB_RLC_UCODE_SIZE;
6244 
			break;
6245 
		case CHIP_MULLINS:
6246 
			size = ML_RLC_UCODE_SIZE;
6247 
			break;
6248 
		}
6249 
 
6250 
	fw_data = (const __be32 *)rdev->rlc_fw->data;
6251 
		WREG32(RLC_GPM_UCODE_ADDR, 0);
6252 
	for (i = 0; i < size; i++)
6253 
		WREG32(RLC_GPM_UCODE_DATA, be32_to_cpup(fw_data++));
6254 
	WREG32(RLC_GPM_UCODE_ADDR, 0);
6255 
	}
6256 
 
6257 
	/* XXX - find out what chips support lbpw */
6258 
	cik_enable_lbpw(rdev, false);
6259 
 
6260 
	if (rdev->family == CHIP_BONAIRE)
6261 
		WREG32(RLC_DRIVER_DMA_STATUS, 0);
6262 
 
6263 
	cik_rlc_start(rdev);
6264 
 
6265 
	return 0;
6266 
}
6267 
 
6268 
static void cik_enable_cgcg(struct radeon_device *rdev, bool enable)
6269 
{
6270 
	u32 data, orig, tmp, tmp2;
6271 
 
6272 
	orig = data = RREG32(RLC_CGCG_CGLS_CTRL);
6273 
 
6274 
	if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGCG)) {
6275 
		cik_enable_gui_idle_interrupt(rdev, true);
6276 
 
6277 
		tmp = cik_halt_rlc(rdev);
6278 
 
6279 
		cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
6280 
		WREG32(RLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff);
6281 
		WREG32(RLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff);
6282 
		tmp2 = BPM_ADDR_MASK | CGCG_OVERRIDE_0 | CGLS_ENABLE;
6283 
		WREG32(RLC_SERDES_WR_CTRL, tmp2);
6284 
 
6285 
		cik_update_rlc(rdev, tmp);
6286 
 
6287 
		data |= CGCG_EN | CGLS_EN;
6288 
	} else {
6289 
		cik_enable_gui_idle_interrupt(rdev, false);
6290 
 
6291 
		RREG32(CB_CGTT_SCLK_CTRL);
6292 
		RREG32(CB_CGTT_SCLK_CTRL);
6293 
		RREG32(CB_CGTT_SCLK_CTRL);
6294 
		RREG32(CB_CGTT_SCLK_CTRL);
6295 
 
6296 
		data &= ~(CGCG_EN | CGLS_EN);
6297 
	}
6298 
 
6299 
	if (orig != data)
6300 
		WREG32(RLC_CGCG_CGLS_CTRL, data);
6301 
 
6302 
}
6303 
 
6304 
static void cik_enable_mgcg(struct radeon_device *rdev, bool enable)
6305 
{
6306 
	u32 data, orig, tmp = 0;
6307 
 
6308 
	if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGCG)) {
6309 
		if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGLS) {
6310 
			if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CP_LS) {
6311 
				orig = data = RREG32(CP_MEM_SLP_CNTL);
6312 
				data |= CP_MEM_LS_EN;
6313 
				if (orig != data)
6314 
					WREG32(CP_MEM_SLP_CNTL, data);
6315 
			}
6316 
		}
6317 
 
6318 
		orig = data = RREG32(RLC_CGTT_MGCG_OVERRIDE);
6319 
		data &= 0xfffffffd;
6320 
		if (orig != data)
6321 
			WREG32(RLC_CGTT_MGCG_OVERRIDE, data);
6322 
 
6323 
		tmp = cik_halt_rlc(rdev);
6324 
 
6325 
		cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
6326 
		WREG32(RLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff);
6327 
		WREG32(RLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff);
6328 
		data = BPM_ADDR_MASK | MGCG_OVERRIDE_0;
6329 
		WREG32(RLC_SERDES_WR_CTRL, data);
6330 
 
6331 
		cik_update_rlc(rdev, tmp);
6332 
 
6333 
		if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGTS) {
6334 
			orig = data = RREG32(CGTS_SM_CTRL_REG);
6335 
			data &= ~SM_MODE_MASK;
6336 
			data |= SM_MODE(0x2);
6337 
			data |= SM_MODE_ENABLE;
6338 
			data &= ~CGTS_OVERRIDE;
6339 
			if ((rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGLS) &&
6340 
			    (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGTS_LS))
6341 
				data &= ~CGTS_LS_OVERRIDE;
6342 
			data &= ~ON_MONITOR_ADD_MASK;
6343 
			data |= ON_MONITOR_ADD_EN;
6344 
			data |= ON_MONITOR_ADD(0x96);
6345 
			if (orig != data)
6346 
				WREG32(CGTS_SM_CTRL_REG, data);
6347 
		}
6348 
	} else {
6349 
		orig = data = RREG32(RLC_CGTT_MGCG_OVERRIDE);
6350 
		data |= 0x00000002;
6351 
		if (orig != data)
6352 
			WREG32(RLC_CGTT_MGCG_OVERRIDE, data);
6353 
 
6354 
		data = RREG32(RLC_MEM_SLP_CNTL);
6355 
		if (data & RLC_MEM_LS_EN) {
6356 
			data &= ~RLC_MEM_LS_EN;
6357 
			WREG32(RLC_MEM_SLP_CNTL, data);
6358 
		}
6359 
 
6360 
		data = RREG32(CP_MEM_SLP_CNTL);
6361 
		if (data & CP_MEM_LS_EN) {
6362 
			data &= ~CP_MEM_LS_EN;
6363 
			WREG32(CP_MEM_SLP_CNTL, data);
6364 
		}
6365 
 
6366 
		orig = data = RREG32(CGTS_SM_CTRL_REG);
6367 
		data |= CGTS_OVERRIDE | CGTS_LS_OVERRIDE;
6368 
		if (orig != data)
6369 
			WREG32(CGTS_SM_CTRL_REG, data);
6370 
 
6371 
		tmp = cik_halt_rlc(rdev);
6372 
 
6373 
		cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
6374 
		WREG32(RLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff);
6375 
		WREG32(RLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff);
6376 
		data = BPM_ADDR_MASK | MGCG_OVERRIDE_1;
6377 
		WREG32(RLC_SERDES_WR_CTRL, data);
6378 
 
6379 
		cik_update_rlc(rdev, tmp);
6380 
	}
6381 
}
6382 
 
6383 
static const u32 mc_cg_registers[] =
6384 
{
6385 
	MC_HUB_MISC_HUB_CG,
6386 
	MC_HUB_MISC_SIP_CG,
6387 
	MC_HUB_MISC_VM_CG,
6388 
	MC_XPB_CLK_GAT,
6389 
	ATC_MISC_CG,
6390 
	MC_CITF_MISC_WR_CG,
6391 
	MC_CITF_MISC_RD_CG,
6392 
	MC_CITF_MISC_VM_CG,
6393 
	VM_L2_CG,
6394 
};
6395 
 
6396 
static void cik_enable_mc_ls(struct radeon_device *rdev,
6397 
			     bool enable)
6398 
{
6399 
	int i;
6400 
	u32 orig, data;
6401 
 
6402 
	for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) {
6403 
		orig = data = RREG32(mc_cg_registers[i]);
6404 
		if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_MC_LS))
6405 
			data |= MC_LS_ENABLE;
6406 
		else
6407 
			data &= ~MC_LS_ENABLE;
6408 
		if (data != orig)
6409 
			WREG32(mc_cg_registers[i], data);
6410 
	}
6411 
}
6412 
 
6413 
static void cik_enable_mc_mgcg(struct radeon_device *rdev,
6414 
			       bool enable)
6415 
{
6416 
	int i;
6417 
	u32 orig, data;
6418 
 
6419 
	for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) {
6420 
		orig = data = RREG32(mc_cg_registers[i]);
6421 
		if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_MC_MGCG))
6422 
			data |= MC_CG_ENABLE;
6423 
		else
6424 
			data &= ~MC_CG_ENABLE;
6425 
		if (data != orig)
6426 
			WREG32(mc_cg_registers[i], data);
6427 
	}
6428 
}
6429 
 
6430 
static void cik_enable_sdma_mgcg(struct radeon_device *rdev,
6431 
				 bool enable)
6432 
{
6433 
	u32 orig, data;
6434 
 
6435 
	if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_SDMA_MGCG)) {
6436 
		WREG32(SDMA0_CLK_CTRL + SDMA0_REGISTER_OFFSET, 0x00000100);
6437 
		WREG32(SDMA0_CLK_CTRL + SDMA1_REGISTER_OFFSET, 0x00000100);
6438 
	} else {
6439 
		orig = data = RREG32(SDMA0_CLK_CTRL + SDMA0_REGISTER_OFFSET);
6440 
		data |= 0xff000000;
6441 
		if (data != orig)
6442 
			WREG32(SDMA0_CLK_CTRL + SDMA0_REGISTER_OFFSET, data);
6443 
 
6444 
		orig = data = RREG32(SDMA0_CLK_CTRL + SDMA1_REGISTER_OFFSET);
6445 
		data |= 0xff000000;
6446 
		if (data != orig)
6447 
			WREG32(SDMA0_CLK_CTRL + SDMA1_REGISTER_OFFSET, data);
6448 
	}
6449 
}
6450 
 
6451 
static void cik_enable_sdma_mgls(struct radeon_device *rdev,
6452 
				 bool enable)
6453 
{
6454 
	u32 orig, data;
6455 
 
6456 
	if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_SDMA_LS)) {
6457 
		orig = data = RREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET);
6458 
		data |= 0x100;
6459 
		if (orig != data)
6460 
			WREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET, data);
6461 
 
6462 
		orig = data = RREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET);
6463 
		data |= 0x100;
6464 
		if (orig != data)
6465 
			WREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET, data);
6466 
	} else {
6467 
		orig = data = RREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET);
6468 
		data &= ~0x100;
6469 
		if (orig != data)
6470 
			WREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET, data);
6471 
 
6472 
		orig = data = RREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET);
6473 
		data &= ~0x100;
6474 
		if (orig != data)
6475 
			WREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET, data);
6476 
	}
6477 
}
6478 
 
6479 
static void cik_enable_uvd_mgcg(struct radeon_device *rdev,
6480 
				bool enable)
6481 
{
6482 
	u32 orig, data;
6483 
 
6484 
	if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_UVD_MGCG)) {
6485 
		data = RREG32_UVD_CTX(UVD_CGC_MEM_CTRL);
6486 
		data = 0xfff;
6487 
		WREG32_UVD_CTX(UVD_CGC_MEM_CTRL, data);
6488 
 
6489 
		orig = data = RREG32(UVD_CGC_CTRL);
6490 
		data |= DCM;
6491 
		if (orig != data)
6492 
			WREG32(UVD_CGC_CTRL, data);
6493 
	} else {
6494 
		data = RREG32_UVD_CTX(UVD_CGC_MEM_CTRL);
6495 
		data &= ~0xfff;
6496 
		WREG32_UVD_CTX(UVD_CGC_MEM_CTRL, data);
6497 
 
6498 
		orig = data = RREG32(UVD_CGC_CTRL);
6499 
		data &= ~DCM;
6500 
		if (orig != data)
6501 
			WREG32(UVD_CGC_CTRL, data);
6502 
	}
6503 
}
6504 
 
6505 
static void cik_enable_bif_mgls(struct radeon_device *rdev,
6506 
			       bool enable)
6507 
{
6508 
	u32 orig, data;
6509 
 
6510 
	orig = data = RREG32_PCIE_PORT(PCIE_CNTL2);
6511 
 
6512 
	if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_BIF_LS))
6513 
		data |= SLV_MEM_LS_EN | MST_MEM_LS_EN |
6514 
			REPLAY_MEM_LS_EN | SLV_MEM_AGGRESSIVE_LS_EN;
6515 
	else
6516 
		data &= ~(SLV_MEM_LS_EN | MST_MEM_LS_EN |
6517 
			  REPLAY_MEM_LS_EN | SLV_MEM_AGGRESSIVE_LS_EN);
6518 
 
6519 
	if (orig != data)
6520 
		WREG32_PCIE_PORT(PCIE_CNTL2, data);
6521 
}
6522 
 
6523 
static void cik_enable_hdp_mgcg(struct radeon_device *rdev,
6524 
				bool enable)
6525 
{
6526 
	u32 orig, data;
6527 
 
6528 
	orig = data = RREG32(HDP_HOST_PATH_CNTL);
6529 
 
6530 
	if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_HDP_MGCG))
6531 
		data &= ~CLOCK_GATING_DIS;
6532 
	else
6533 
		data |= CLOCK_GATING_DIS;
6534 
 
6535 
	if (orig != data)
6536 
		WREG32(HDP_HOST_PATH_CNTL, data);
6537 
}
6538 
 
6539 
static void cik_enable_hdp_ls(struct radeon_device *rdev,
6540 
			      bool enable)
6541 
{
6542 
	u32 orig, data;
6543 
 
6544 
	orig = data = RREG32(HDP_MEM_POWER_LS);
6545 
 
6546 
	if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_HDP_LS))
6547 
		data |= HDP_LS_ENABLE;
6548 
	else
6549 
		data &= ~HDP_LS_ENABLE;
6550 
 
6551 
	if (orig != data)
6552 
		WREG32(HDP_MEM_POWER_LS, data);
6553 
}
6554 
 
6555 
void cik_update_cg(struct radeon_device *rdev,
6556 
		   u32 block, bool enable)
6557 
{
6558 
 
6559 
	if (block & RADEON_CG_BLOCK_GFX) {
6560 
		cik_enable_gui_idle_interrupt(rdev, false);
6561 
		/* order matters! */
6562 
		if (enable) {
6563 
			cik_enable_mgcg(rdev, true);
6564 
			cik_enable_cgcg(rdev, true);
6565 
		} else {
6566 
			cik_enable_cgcg(rdev, false);
6567 
			cik_enable_mgcg(rdev, false);
6568 
		}
6569 
		cik_enable_gui_idle_interrupt(rdev, true);
6570 
	}
6571 
 
6572 
	if (block & RADEON_CG_BLOCK_MC) {
6573 
		if (!(rdev->flags & RADEON_IS_IGP)) {
6574 
			cik_enable_mc_mgcg(rdev, enable);
6575 
			cik_enable_mc_ls(rdev, enable);
6576 
		}
6577 
	}
6578 
 
6579 
	if (block & RADEON_CG_BLOCK_SDMA) {
6580 
		cik_enable_sdma_mgcg(rdev, enable);
6581 
		cik_enable_sdma_mgls(rdev, enable);
6582 
	}
6583 
 
6584 
	if (block & RADEON_CG_BLOCK_BIF) {
6585 
		cik_enable_bif_mgls(rdev, enable);
6586 
	}
6587 
 
6588 
	if (block & RADEON_CG_BLOCK_UVD) {
6589 
		if (rdev->has_uvd)
6590 
			cik_enable_uvd_mgcg(rdev, enable);
6591 
	}
6592 
 
6593 
	if (block & RADEON_CG_BLOCK_HDP) {
6594 
		cik_enable_hdp_mgcg(rdev, enable);
6595 
		cik_enable_hdp_ls(rdev, enable);
6596 
	}
6597 
 
6598 
	if (block & RADEON_CG_BLOCK_VCE) {
6599 
		vce_v2_0_enable_mgcg(rdev, enable);
6600 
    }
6601 
}
6602 
 
6603 
static void cik_init_cg(struct radeon_device *rdev)
6604 
{
6605 
 
6606 
	cik_update_cg(rdev, RADEON_CG_BLOCK_GFX, true);
6607 
 
6608 
	if (rdev->has_uvd)
6609 
		si_init_uvd_internal_cg(rdev);
6610 
 
6611 
	cik_update_cg(rdev, (RADEON_CG_BLOCK_MC |
6612 
			     RADEON_CG_BLOCK_SDMA |
6613 
			     RADEON_CG_BLOCK_BIF |
6614 
			     RADEON_CG_BLOCK_UVD |
6615 
			     RADEON_CG_BLOCK_HDP), true);
6616 
}
6617 
 
6618 
static void cik_fini_cg(struct radeon_device *rdev)
6619 
{
6620 
	cik_update_cg(rdev, (RADEON_CG_BLOCK_MC |
6621 
			     RADEON_CG_BLOCK_SDMA |
6622 
			     RADEON_CG_BLOCK_BIF |
6623 
			     RADEON_CG_BLOCK_UVD |
6624 
			     RADEON_CG_BLOCK_HDP), false);
6625 
 
6626 
	cik_update_cg(rdev, RADEON_CG_BLOCK_GFX, false);
6627 
}
6628 
 
6629 
static void cik_enable_sck_slowdown_on_pu(struct radeon_device *rdev,
6630 
					  bool enable)
6631 
{
6632 
	u32 data, orig;
6633 
 
6634 
	orig = data = RREG32(RLC_PG_CNTL);
6635 
	if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_RLC_SMU_HS))
6636 
		data |= SMU_CLK_SLOWDOWN_ON_PU_ENABLE;
6637 
	else
6638 
		data &= ~SMU_CLK_SLOWDOWN_ON_PU_ENABLE;
6639 
	if (orig != data)
6640 
		WREG32(RLC_PG_CNTL, data);
6641 
}
6642 
 
6643 
static void cik_enable_sck_slowdown_on_pd(struct radeon_device *rdev,
6644 
					  bool enable)
6645 
{
6646 
	u32 data, orig;
6647 
 
6648 
	orig = data = RREG32(RLC_PG_CNTL);
6649 
	if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_RLC_SMU_HS))
6650 
		data |= SMU_CLK_SLOWDOWN_ON_PD_ENABLE;
6651 
	else
6652 
		data &= ~SMU_CLK_SLOWDOWN_ON_PD_ENABLE;
6653 
	if (orig != data)
6654 
		WREG32(RLC_PG_CNTL, data);
6655 
}
6656 
 
6657 
static void cik_enable_cp_pg(struct radeon_device *rdev, bool enable)
6658 
{
6659 
	u32 data, orig;
6660 
 
6661 
	orig = data = RREG32(RLC_PG_CNTL);
6662 
	if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_CP))
6663 
		data &= ~DISABLE_CP_PG;
6664 
	else
6665 
		data |= DISABLE_CP_PG;
6666 
	if (orig != data)
6667 
		WREG32(RLC_PG_CNTL, data);
6668 
}
6669 
 
6670 
static void cik_enable_gds_pg(struct radeon_device *rdev, bool enable)
6671 
{
6672 
	u32 data, orig;
6673 
 
6674 
	orig = data = RREG32(RLC_PG_CNTL);
6675 
	if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GDS))
6676 
		data &= ~DISABLE_GDS_PG;
6677 
	else
6678 
		data |= DISABLE_GDS_PG;
6679 
	if (orig != data)
6680 
		WREG32(RLC_PG_CNTL, data);
6681 
}
6682 
 
6683 
#define CP_ME_TABLE_SIZE    96
6684 
#define CP_ME_TABLE_OFFSET  2048
6685 
#define CP_MEC_TABLE_OFFSET 4096
6686 
 
6687 
void cik_init_cp_pg_table(struct radeon_device *rdev)
6688 
{
6689 
	volatile u32 *dst_ptr;
6690 
	int me, i, max_me = 4;
6691 
	u32 bo_offset = 0;
6692 
	u32 table_offset, table_size;
6693 
 
6694 
	if (rdev->family == CHIP_KAVERI)
6695 
		max_me = 5;
6696 
 
6697 
	if (rdev->rlc.cp_table_ptr == NULL)
6698 
		return;
6699 
 
6700 
	/* write the cp table buffer */
6701 
	dst_ptr = rdev->rlc.cp_table_ptr;
6702 
	for (me = 0; me < max_me; me++) {
6703 
		if (rdev->new_fw) {
6704 
			const __le32 *fw_data;
6705 
			const struct gfx_firmware_header_v1_0 *hdr;
6706 
 
6707 
			if (me == 0) {
6708 
				hdr = (const struct gfx_firmware_header_v1_0 *)rdev->ce_fw->data;
6709 
				fw_data = (const __le32 *)
6710 
					(rdev->ce_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
6711 
				table_offset = le32_to_cpu(hdr->jt_offset);
6712 
				table_size = le32_to_cpu(hdr->jt_size);
6713 
			} else if (me == 1) {
6714 
				hdr = (const struct gfx_firmware_header_v1_0 *)rdev->pfp_fw->data;
6715 
				fw_data = (const __le32 *)
6716 
					(rdev->pfp_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
6717 
				table_offset = le32_to_cpu(hdr->jt_offset);
6718 
				table_size = le32_to_cpu(hdr->jt_size);
6719 
			} else if (me == 2) {
6720 
				hdr = (const struct gfx_firmware_header_v1_0 *)rdev->me_fw->data;
6721 
				fw_data = (const __le32 *)
6722 
					(rdev->me_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
6723 
				table_offset = le32_to_cpu(hdr->jt_offset);
6724 
				table_size = le32_to_cpu(hdr->jt_size);
6725 
			} else if (me == 3) {
6726 
				hdr = (const struct gfx_firmware_header_v1_0 *)rdev->mec_fw->data;
6727 
				fw_data = (const __le32 *)
6728 
					(rdev->mec_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
6729 
				table_offset = le32_to_cpu(hdr->jt_offset);
6730 
				table_size = le32_to_cpu(hdr->jt_size);
6731 
			} else {
6732 
				hdr = (const struct gfx_firmware_header_v1_0 *)rdev->mec2_fw->data;
6733 
				fw_data = (const __le32 *)
6734 
					(rdev->mec2_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
6735 
				table_offset = le32_to_cpu(hdr->jt_offset);
6736 
				table_size = le32_to_cpu(hdr->jt_size);
6737 
			}
6738 
 
6739 
			for (i = 0; i < table_size; i ++) {
6740 
				dst_ptr[bo_offset + i] =
6741 
					cpu_to_le32(le32_to_cpu(fw_data[table_offset + i]));
6742 
			}
6743 
			bo_offset += table_size;
6744 
		} else {
6745 
			const __be32 *fw_data;
6746 
			table_size = CP_ME_TABLE_SIZE;
6747 
 
6748 
		if (me == 0) {
6749 
			fw_data = (const __be32 *)rdev->ce_fw->data;
6750 
			table_offset = CP_ME_TABLE_OFFSET;
6751 
		} else if (me == 1) {
6752 
			fw_data = (const __be32 *)rdev->pfp_fw->data;
6753 
			table_offset = CP_ME_TABLE_OFFSET;
6754 
		} else if (me == 2) {
6755 
			fw_data = (const __be32 *)rdev->me_fw->data;
6756 
			table_offset = CP_ME_TABLE_OFFSET;
6757 
		} else {
6758 
			fw_data = (const __be32 *)rdev->mec_fw->data;
6759 
			table_offset = CP_MEC_TABLE_OFFSET;
6760 
		}
6761 
 
6762 
			for (i = 0; i < table_size; i ++) {
6763 
				dst_ptr[bo_offset + i] =
6764 
					cpu_to_le32(be32_to_cpu(fw_data[table_offset + i]));
6765 
			}
6766 
			bo_offset += table_size;
6767 
		}
6768 
	}
6769 
}
6770 
 
6771 
static void cik_enable_gfx_cgpg(struct radeon_device *rdev,
6772 
				bool enable)
6773 
{
6774 
	u32 data, orig;
6775 
 
6776 
	if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG)) {
6777 
		orig = data = RREG32(RLC_PG_CNTL);
6778 
		data |= GFX_PG_ENABLE;
6779 
		if (orig != data)
6780 
			WREG32(RLC_PG_CNTL, data);
6781 
 
6782 
		orig = data = RREG32(RLC_AUTO_PG_CTRL);
6783 
		data |= AUTO_PG_EN;
6784 
		if (orig != data)
6785 
			WREG32(RLC_AUTO_PG_CTRL, data);
6786 
	} else {
6787 
		orig = data = RREG32(RLC_PG_CNTL);
6788 
		data &= ~GFX_PG_ENABLE;
6789 
		if (orig != data)
6790 
			WREG32(RLC_PG_CNTL, data);
6791 
 
6792 
		orig = data = RREG32(RLC_AUTO_PG_CTRL);
6793 
		data &= ~AUTO_PG_EN;
6794 
		if (orig != data)
6795 
			WREG32(RLC_AUTO_PG_CTRL, data);
6796 
 
6797 
		data = RREG32(DB_RENDER_CONTROL);
6798 
	}
6799 
}
6800 
 
6801 
static u32 cik_get_cu_active_bitmap(struct radeon_device *rdev, u32 se, u32 sh)
6802 
{
6803 
	u32 mask = 0, tmp, tmp1;
6804 
	int i;
6805 
 
6806 
	cik_select_se_sh(rdev, se, sh);
6807 
	tmp = RREG32(CC_GC_SHADER_ARRAY_CONFIG);
6808 
	tmp1 = RREG32(GC_USER_SHADER_ARRAY_CONFIG);
6809 
	cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
6810 
 
6811 
	tmp &= 0xffff0000;
6812 
 
6813 
	tmp |= tmp1;
6814 
	tmp >>= 16;
6815 
 
6816 
	for (i = 0; i < rdev->config.cik.max_cu_per_sh; i ++) {
6817 
		mask <<= 1;
6818 
		mask |= 1;
6819 
	}
6820 
 
6821 
	return (~tmp) & mask;
6822 
}
6823 
 
6824 
static void cik_init_ao_cu_mask(struct radeon_device *rdev)
6825 
{
6826 
	u32 i, j, k, active_cu_number = 0;
6827 
	u32 mask, counter, cu_bitmap;
6828 
	u32 tmp = 0;
6829 
 
6830 
	for (i = 0; i < rdev->config.cik.max_shader_engines; i++) {
6831 
		for (j = 0; j < rdev->config.cik.max_sh_per_se; j++) {
6832 
			mask = 1;
6833 
			cu_bitmap = 0;
6834 
			counter = 0;
6835 
			for (k = 0; k < rdev->config.cik.max_cu_per_sh; k ++) {
6836 
				if (cik_get_cu_active_bitmap(rdev, i, j) & mask) {
6837 
					if (counter < 2)
6838 
						cu_bitmap |= mask;
6839 
					counter ++;
6840 
				}
6841 
				mask <<= 1;
6842 
			}
6843 
 
6844 
			active_cu_number += counter;
6845 
			tmp |= (cu_bitmap << (i * 16 + j * 8));
6846 
		}
6847 
	}
6848 
 
6849 
	WREG32(RLC_PG_AO_CU_MASK, tmp);
6850 
 
6851 
	tmp = RREG32(RLC_MAX_PG_CU);
6852 
	tmp &= ~MAX_PU_CU_MASK;
6853 
	tmp |= MAX_PU_CU(active_cu_number);
6854 
	WREG32(RLC_MAX_PG_CU, tmp);
6855 
}
6856 
 
6857 
static void cik_enable_gfx_static_mgpg(struct radeon_device *rdev,
6858 
				       bool enable)
6859 
{
6860 
	u32 data, orig;
6861 
 
6862 
	orig = data = RREG32(RLC_PG_CNTL);
6863 
	if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_SMG))
6864 
		data |= STATIC_PER_CU_PG_ENABLE;
6865 
	else
6866 
		data &= ~STATIC_PER_CU_PG_ENABLE;
6867 
	if (orig != data)
6868 
		WREG32(RLC_PG_CNTL, data);
6869 
}
6870 
 
6871 
static void cik_enable_gfx_dynamic_mgpg(struct radeon_device *rdev,
6872 
					bool enable)
6873 
{
6874 
	u32 data, orig;
6875 
 
6876 
	orig = data = RREG32(RLC_PG_CNTL);
6877 
	if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_DMG))
6878 
		data |= DYN_PER_CU_PG_ENABLE;
6879 
	else
6880 
		data &= ~DYN_PER_CU_PG_ENABLE;
6881 
	if (orig != data)
6882 
		WREG32(RLC_PG_CNTL, data);
6883 
}
6884 
 
6885 
#define RLC_SAVE_AND_RESTORE_STARTING_OFFSET 0x90
6886 
#define RLC_CLEAR_STATE_DESCRIPTOR_OFFSET    0x3D
6887 
 
6888 
static void cik_init_gfx_cgpg(struct radeon_device *rdev)
6889 
{
6890 
	u32 data, orig;
6891 
	u32 i;
6892 
 
6893 
	if (rdev->rlc.cs_data) {
6894 
		WREG32(RLC_GPM_SCRATCH_ADDR, RLC_CLEAR_STATE_DESCRIPTOR_OFFSET);
6895 
		WREG32(RLC_GPM_SCRATCH_DATA, upper_32_bits(rdev->rlc.clear_state_gpu_addr));
6896 
		WREG32(RLC_GPM_SCRATCH_DATA, lower_32_bits(rdev->rlc.clear_state_gpu_addr));
6897 
		WREG32(RLC_GPM_SCRATCH_DATA, rdev->rlc.clear_state_size);
6898 
	} else {
6899 
		WREG32(RLC_GPM_SCRATCH_ADDR, RLC_CLEAR_STATE_DESCRIPTOR_OFFSET);
6900 
		for (i = 0; i < 3; i++)
6901 
			WREG32(RLC_GPM_SCRATCH_DATA, 0);
6902 
	}
6903 
	if (rdev->rlc.reg_list) {
6904 
		WREG32(RLC_GPM_SCRATCH_ADDR, RLC_SAVE_AND_RESTORE_STARTING_OFFSET);
6905 
		for (i = 0; i < rdev->rlc.reg_list_size; i++)
6906 
			WREG32(RLC_GPM_SCRATCH_DATA, rdev->rlc.reg_list[i]);
6907 
	}
6908 
 
6909 
	orig = data = RREG32(RLC_PG_CNTL);
6910 
	data |= GFX_PG_SRC;
6911 
	if (orig != data)
6912 
		WREG32(RLC_PG_CNTL, data);
6913 
 
6914 
	WREG32(RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
6915 
	WREG32(RLC_CP_TABLE_RESTORE, rdev->rlc.cp_table_gpu_addr >> 8);
6916 
 
6917 
	data = RREG32(CP_RB_WPTR_POLL_CNTL);
6918 
	data &= ~IDLE_POLL_COUNT_MASK;
6919 
	data |= IDLE_POLL_COUNT(0x60);
6920 
	WREG32(CP_RB_WPTR_POLL_CNTL, data);
6921 
 
6922 
	data = 0x10101010;
6923 
	WREG32(RLC_PG_DELAY, data);
6924 
 
6925 
	data = RREG32(RLC_PG_DELAY_2);
6926 
	data &= ~0xff;
6927 
	data |= 0x3;
6928 
	WREG32(RLC_PG_DELAY_2, data);
6929 
 
6930 
	data = RREG32(RLC_AUTO_PG_CTRL);
6931 
	data &= ~GRBM_REG_SGIT_MASK;
6932 
	data |= GRBM_REG_SGIT(0x700);
6933 
	WREG32(RLC_AUTO_PG_CTRL, data);
6934 
 
6935 
}
6936 
 
6937 
static void cik_update_gfx_pg(struct radeon_device *rdev, bool enable)
6938 
{
6939 
	cik_enable_gfx_cgpg(rdev, enable);
6940 
	cik_enable_gfx_static_mgpg(rdev, enable);
6941 
	cik_enable_gfx_dynamic_mgpg(rdev, enable);
6942 
}
6943 
 
6944 
u32 cik_get_csb_size(struct radeon_device *rdev)
6945 
{
6946 
	u32 count = 0;
6947 
	const struct cs_section_def *sect = NULL;
6948 
	const struct cs_extent_def *ext = NULL;
6949 
 
6950 
	if (rdev->rlc.cs_data == NULL)
6951 
		return 0;
6952 
 
6953 
	/* begin clear state */
6954 
	count += 2;
6955 
	/* context control state */
6956 
	count += 3;
6957 
 
6958 
	for (sect = rdev->rlc.cs_data; sect->section != NULL; ++sect) {
6959 
		for (ext = sect->section; ext->extent != NULL; ++ext) {
6960 
			if (sect->id == SECT_CONTEXT)
6961 
				count += 2 + ext->reg_count;
6962 
			else
6963 
				return 0;
6964 
		}
6965 
	}
6966 
	/* pa_sc_raster_config/pa_sc_raster_config1 */
6967 
	count += 4;
6968 
	/* end clear state */
6969 
	count += 2;
6970 
	/* clear state */
6971 
	count += 2;
6972 
 
6973 
	return count;
6974 
}
6975 
 
6976 
void cik_get_csb_buffer(struct radeon_device *rdev, volatile u32 *buffer)
6977 
{
6978 
	u32 count = 0, i;
6979 
	const struct cs_section_def *sect = NULL;
6980 
	const struct cs_extent_def *ext = NULL;
6981 
 
6982 
	if (rdev->rlc.cs_data == NULL)
6983 
		return;
6984 
	if (buffer == NULL)
6985 
		return;
6986 
 
6987 
	buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
6988 
	buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
6989 
 
6990 
	buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CONTEXT_CONTROL, 1));
6991 
	buffer[count++] = cpu_to_le32(0x80000000);
6992 
	buffer[count++] = cpu_to_le32(0x80000000);
6993 
 
6994 
	for (sect = rdev->rlc.cs_data; sect->section != NULL; ++sect) {
6995 
		for (ext = sect->section; ext->extent != NULL; ++ext) {
6996 
			if (sect->id == SECT_CONTEXT) {
6997 
				buffer[count++] =
6998 
					cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, ext->reg_count));
6999 
				buffer[count++] = cpu_to_le32(ext->reg_index - 0xa000);
7000 
				for (i = 0; i < ext->reg_count; i++)
7001 
					buffer[count++] = cpu_to_le32(ext->extent[i]);
7002 
			} else {
7003 
				return;
7004 
			}
7005 
		}
7006 
	}
7007 
 
7008 
	buffer[count++] = cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, 2));
7009 
	buffer[count++] = cpu_to_le32(PA_SC_RASTER_CONFIG - PACKET3_SET_CONTEXT_REG_START);
7010 
	switch (rdev->family) {
7011 
	case CHIP_BONAIRE:
7012 
		buffer[count++] = cpu_to_le32(0x16000012);
7013 
		buffer[count++] = cpu_to_le32(0x00000000);
7014 
		break;
7015 
	case CHIP_KAVERI:
7016 
		buffer[count++] = cpu_to_le32(0x00000000); /* XXX */
7017 
		buffer[count++] = cpu_to_le32(0x00000000);
7018 
		break;
7019 
	case CHIP_KABINI:
7020 
	case CHIP_MULLINS:
7021 
		buffer[count++] = cpu_to_le32(0x00000000); /* XXX */
7022 
		buffer[count++] = cpu_to_le32(0x00000000);
7023 
		break;
7024 
	case CHIP_HAWAII:
7025 
		buffer[count++] = cpu_to_le32(0x3a00161a);
7026 
		buffer[count++] = cpu_to_le32(0x0000002e);
7027 
		break;
7028 
	default:
7029 
		buffer[count++] = cpu_to_le32(0x00000000);
7030 
		buffer[count++] = cpu_to_le32(0x00000000);
7031 
		break;
7032 
	}
7033 
 
7034 
	buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
7035 
	buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_END_CLEAR_STATE);
7036 
 
7037 
	buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CLEAR_STATE, 0));
7038 
	buffer[count++] = cpu_to_le32(0);
7039 
}
7040 
 
7041 
static void cik_init_pg(struct radeon_device *rdev)
7042 
{
7043 
	if (rdev->pg_flags) {
7044 
		cik_enable_sck_slowdown_on_pu(rdev, true);
7045 
		cik_enable_sck_slowdown_on_pd(rdev, true);
7046 
		if (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG) {
7047 
			cik_init_gfx_cgpg(rdev);
7048 
			cik_enable_cp_pg(rdev, true);
7049 
			cik_enable_gds_pg(rdev, true);
7050 
		}
7051 
		cik_init_ao_cu_mask(rdev);
7052 
		cik_update_gfx_pg(rdev, true);
7053 
	}
7054 
}
7055 
 
7056 
static void cik_fini_pg(struct radeon_device *rdev)
7057 
{
7058 
	if (rdev->pg_flags) {
7059 
		cik_update_gfx_pg(rdev, false);
7060 
		if (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG) {
7061 
			cik_enable_cp_pg(rdev, false);
7062 
			cik_enable_gds_pg(rdev, false);
7063 
		}
7064 
	}
7065 
}
7066 
 
7067 
/*
7068 
 * Interrupts
7069 
 * Starting with r6xx, interrupts are handled via a ring buffer.
7070 
 * Ring buffers are areas of GPU accessible memory that the GPU
7071 
 * writes interrupt vectors into and the host reads vectors out of.
7072 
 * There is a rptr (read pointer) that determines where the
7073 
 * host is currently reading, and a wptr (write pointer)
7074 
 * which determines where the GPU has written.  When the
7075 
 * pointers are equal, the ring is idle.  When the GPU
7076 
 * writes vectors to the ring buffer, it increments the
7077 
 * wptr.  When there is an interrupt, the host then starts
7078 
 * fetching commands and processing them until the pointers are
7079 
 * equal again at which point it updates the rptr.
7080 
 */
7081 
 
7082 
/**
7083 
 * cik_enable_interrupts - Enable the interrupt ring buffer
7084 
 *
7085 
 * @rdev: radeon_device pointer
7086 
 *
7087 
 * Enable the interrupt ring buffer (CIK).
7088 
 */
7089 
static void cik_enable_interrupts(struct radeon_device *rdev)
7090 
{
7091 
	u32 ih_cntl = RREG32(IH_CNTL);
7092 
	u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
7093 
 
7094 
	ih_cntl |= ENABLE_INTR;
7095 
	ih_rb_cntl |= IH_RB_ENABLE;
7096 
	WREG32(IH_CNTL, ih_cntl);
7097 
	WREG32(IH_RB_CNTL, ih_rb_cntl);
7098 
	rdev->ih.enabled = true;
7099 
}
7100 
 
7101 
/**
7102 
 * cik_disable_interrupts - Disable the interrupt ring buffer
7103 
 *
7104 
 * @rdev: radeon_device pointer
7105 
 *
7106 
 * Disable the interrupt ring buffer (CIK).
7107 
 */
7108 
static void cik_disable_interrupts(struct radeon_device *rdev)
7109 
{
7110 
	u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
7111 
	u32 ih_cntl = RREG32(IH_CNTL);
7112 
 
7113 
	ih_rb_cntl &= ~IH_RB_ENABLE;
7114 
	ih_cntl &= ~ENABLE_INTR;
7115 
	WREG32(IH_RB_CNTL, ih_rb_cntl);
7116 
	WREG32(IH_CNTL, ih_cntl);
7117 
	/* set rptr, wptr to 0 */
7118 
	WREG32(IH_RB_RPTR, 0);
7119 
	WREG32(IH_RB_WPTR, 0);
7120 
	rdev->ih.enabled = false;
7121 
	rdev->ih.rptr = 0;
7122 
}
7123 
 
7124 
/**
7125 
 * cik_disable_interrupt_state - Disable all interrupt sources
7126 
 *
7127 
 * @rdev: radeon_device pointer
7128 
 *
7129 
 * Clear all interrupt enable bits used by the driver (CIK).
7130 
 */
7131 
static void cik_disable_interrupt_state(struct radeon_device *rdev)
7132 
{
7133 
	u32 tmp;
7134 
 
7135 
	/* gfx ring */
7136 
	tmp = RREG32(CP_INT_CNTL_RING0) &
7137 
		(CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
7138 
	WREG32(CP_INT_CNTL_RING0, tmp);
7139 
	/* sdma */
7140 
	tmp = RREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET) & ~TRAP_ENABLE;
7141 
	WREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET, tmp);
7142 
	tmp = RREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET) & ~TRAP_ENABLE;
7143 
	WREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET, tmp);
7144 
	/* compute queues */
7145 
	WREG32(CP_ME1_PIPE0_INT_CNTL, 0);
7146 
	WREG32(CP_ME1_PIPE1_INT_CNTL, 0);
7147 
	WREG32(CP_ME1_PIPE2_INT_CNTL, 0);
7148 
	WREG32(CP_ME1_PIPE3_INT_CNTL, 0);
7149 
	WREG32(CP_ME2_PIPE0_INT_CNTL, 0);
7150 
	WREG32(CP_ME2_PIPE1_INT_CNTL, 0);
7151 
	WREG32(CP_ME2_PIPE2_INT_CNTL, 0);
7152 
	WREG32(CP_ME2_PIPE3_INT_CNTL, 0);
7153 
	/* grbm */
7154 
	WREG32(GRBM_INT_CNTL, 0);
7155 
	/* vline/vblank, etc. */
7156 
	WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
7157 
	WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
7158 
	if (rdev->num_crtc >= 4) {
7159 
		WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
7160 
		WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
7161 
	}
7162 
	if (rdev->num_crtc >= 6) {
7163 
		WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
7164 
		WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
7165 
	}
7166 
	/* pflip */
7167 
	if (rdev->num_crtc >= 2) {
7168 
		WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
7169 
		WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
7170 
	}
7171 
	if (rdev->num_crtc >= 4) {
7172 
		WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
7173 
		WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
7174 
	}
7175 
	if (rdev->num_crtc >= 6) {
7176 
		WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
7177 
		WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
7178 
	}
7179 
 
7180 
	/* dac hotplug */
7181 
	WREG32(DAC_AUTODETECT_INT_CONTROL, 0);
7182 
 
7183 
	/* digital hotplug */
7184 
	tmp = RREG32(DC_HPD1_INT_CONTROL) & DC_HPDx_INT_POLARITY;
7185 
	WREG32(DC_HPD1_INT_CONTROL, tmp);
7186 
	tmp = RREG32(DC_HPD2_INT_CONTROL) & DC_HPDx_INT_POLARITY;
7187 
	WREG32(DC_HPD2_INT_CONTROL, tmp);
7188 
	tmp = RREG32(DC_HPD3_INT_CONTROL) & DC_HPDx_INT_POLARITY;
7189 
	WREG32(DC_HPD3_INT_CONTROL, tmp);
7190 
	tmp = RREG32(DC_HPD4_INT_CONTROL) & DC_HPDx_INT_POLARITY;
7191 
	WREG32(DC_HPD4_INT_CONTROL, tmp);
7192 
	tmp = RREG32(DC_HPD5_INT_CONTROL) & DC_HPDx_INT_POLARITY;
7193 
	WREG32(DC_HPD5_INT_CONTROL, tmp);
7194 
	tmp = RREG32(DC_HPD6_INT_CONTROL) & DC_HPDx_INT_POLARITY;
7195 
	WREG32(DC_HPD6_INT_CONTROL, tmp);
7196 
 
7197 
}
7198 
 
7199 
/**
7200 
 * cik_irq_init - init and enable the interrupt ring
7201 
 *
7202 
 * @rdev: radeon_device pointer
7203 
 *
7204 
 * Allocate a ring buffer for the interrupt controller,
7205 
 * enable the RLC, disable interrupts, enable the IH
7206 
 * ring buffer and enable it (CIK).
7207 
 * Called at device load and reume.
7208 
 * Returns 0 for success, errors for failure.
7209 
 */
7210 
static int cik_irq_init(struct radeon_device *rdev)
7211 
{
7212 
	int ret = 0;
7213 
	int rb_bufsz;
7214 
	u32 interrupt_cntl, ih_cntl, ih_rb_cntl;
7215 
 
7216 
	/* allocate ring */
7217 
	ret = r600_ih_ring_alloc(rdev);
7218 
	if (ret)
7219 
		return ret;
7220 
 
7221 
	/* disable irqs */
7222 
	cik_disable_interrupts(rdev);
7223 
 
7224 
	/* init rlc */
7225 
	ret = cik_rlc_resume(rdev);
7226 
	if (ret) {
7227 
		r600_ih_ring_fini(rdev);
7228 
		return ret;
7229 
	}
7230 
 
7231 
	/* setup interrupt control */
7232 
	/* XXX this should actually be a bus address, not an MC address. same on older asics */
7233 
	WREG32(INTERRUPT_CNTL2, rdev->ih.gpu_addr >> 8);
7234 
	interrupt_cntl = RREG32(INTERRUPT_CNTL);
7235 
	/* IH_DUMMY_RD_OVERRIDE=0 - dummy read disabled with msi, enabled without msi
7236 
	 * IH_DUMMY_RD_OVERRIDE=1 - dummy read controlled by IH_DUMMY_RD_EN
7237 
	 */
7238 
	interrupt_cntl &= ~IH_DUMMY_RD_OVERRIDE;
7239 
	/* IH_REQ_NONSNOOP_EN=1 if ring is in non-cacheable memory, e.g., vram */
7240 
	interrupt_cntl &= ~IH_REQ_NONSNOOP_EN;
7241 
	WREG32(INTERRUPT_CNTL, interrupt_cntl);
7242 
 
7243 
	WREG32(IH_RB_BASE, rdev->ih.gpu_addr >> 8);
7244 
	rb_bufsz = order_base_2(rdev->ih.ring_size / 4);
7245 
 
7246 
	ih_rb_cntl = (IH_WPTR_OVERFLOW_ENABLE |
7247 
		      IH_WPTR_OVERFLOW_CLEAR |
7248 
		      (rb_bufsz << 1));
7249 
 
7250 
	if (rdev->wb.enabled)
7251 
		ih_rb_cntl |= IH_WPTR_WRITEBACK_ENABLE;
7252 
 
7253 
	/* set the writeback address whether it's enabled or not */
7254 
	WREG32(IH_RB_WPTR_ADDR_LO, (rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFFFFFFFC);
7255 
	WREG32(IH_RB_WPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFF);
7256 
 
7257 
	WREG32(IH_RB_CNTL, ih_rb_cntl);
7258 
 
7259 
	/* set rptr, wptr to 0 */
7260 
	WREG32(IH_RB_RPTR, 0);
7261 
	WREG32(IH_RB_WPTR, 0);
7262 
 
7263 
	/* Default settings for IH_CNTL (disabled at first) */
7264 
	ih_cntl = MC_WRREQ_CREDIT(0x10) | MC_WR_CLEAN_CNT(0x10) | MC_VMID(0);
7265 
	/* RPTR_REARM only works if msi's are enabled */
7266 
	if (rdev->msi_enabled)
7267 
		ih_cntl |= RPTR_REARM;
7268 
	WREG32(IH_CNTL, ih_cntl);
7269 
 
7270 
	/* force the active interrupt state to all disabled */
7271 
	cik_disable_interrupt_state(rdev);
7272 
 
7273 
//   pci_set_master(rdev->pdev);
7274 
 
7275 
	/* enable irqs */
7276 
	cik_enable_interrupts(rdev);
7277 
 
7278 
	return ret;
7279 
}
7280 
 
7281 
/**
7282 
 * cik_irq_set - enable/disable interrupt sources
7283 
 *
7284 
 * @rdev: radeon_device pointer
7285 
 *
7286 
 * Enable interrupt sources on the GPU (vblanks, hpd,
7287 
 * etc.) (CIK).
7288 
 * Returns 0 for success, errors for failure.
7289 
 */
7290 
int cik_irq_set(struct radeon_device *rdev)
7291 
{
7292 
	u32 cp_int_cntl;
7293 
	u32 cp_m1p0, cp_m1p1, cp_m1p2, cp_m1p3;
7294 
	u32 cp_m2p0, cp_m2p1, cp_m2p2, cp_m2p3;
7295 
	u32 crtc1 = 0, crtc2 = 0, crtc3 = 0, crtc4 = 0, crtc5 = 0, crtc6 = 0;
7296 
	u32 hpd1, hpd2, hpd3, hpd4, hpd5, hpd6;
7297 
	u32 grbm_int_cntl = 0;
7298 
	u32 dma_cntl, dma_cntl1;
7299 
	u32 thermal_int;
7300 
 
7301 
	if (!rdev->irq.installed) {
7302 
		WARN(1, "Can't enable IRQ/MSI because no handler is installed\n");
7303 
		return -EINVAL;
7304 
	}
7305 
	/* don't enable anything if the ih is disabled */
7306 
	if (!rdev->ih.enabled) {
7307 
		cik_disable_interrupts(rdev);
7308 
		/* force the active interrupt state to all disabled */
7309 
		cik_disable_interrupt_state(rdev);
7310 
		return 0;
7311 
	}
7312 
 
7313 
	cp_int_cntl = RREG32(CP_INT_CNTL_RING0) &
7314 
		(CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
7315 
	cp_int_cntl |= PRIV_INSTR_INT_ENABLE | PRIV_REG_INT_ENABLE;
7316 
 
7317 
	hpd1 = RREG32(DC_HPD1_INT_CONTROL) & ~DC_HPDx_INT_EN;
7318 
	hpd2 = RREG32(DC_HPD2_INT_CONTROL) & ~DC_HPDx_INT_EN;
7319 
	hpd3 = RREG32(DC_HPD3_INT_CONTROL) & ~DC_HPDx_INT_EN;
7320 
	hpd4 = RREG32(DC_HPD4_INT_CONTROL) & ~DC_HPDx_INT_EN;
7321 
	hpd5 = RREG32(DC_HPD5_INT_CONTROL) & ~DC_HPDx_INT_EN;
7322 
	hpd6 = RREG32(DC_HPD6_INT_CONTROL) & ~DC_HPDx_INT_EN;
7323 
 
7324 
	dma_cntl = RREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET) & ~TRAP_ENABLE;
7325 
	dma_cntl1 = RREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET) & ~TRAP_ENABLE;
7326 
 
7327 
	cp_m1p0 = RREG32(CP_ME1_PIPE0_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
7328 
	cp_m1p1 = RREG32(CP_ME1_PIPE1_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
7329 
	cp_m1p2 = RREG32(CP_ME1_PIPE2_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
7330 
	cp_m1p3 = RREG32(CP_ME1_PIPE3_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
7331 
	cp_m2p0 = RREG32(CP_ME2_PIPE0_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
7332 
	cp_m2p1 = RREG32(CP_ME2_PIPE1_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
7333 
	cp_m2p2 = RREG32(CP_ME2_PIPE2_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
7334 
	cp_m2p3 = RREG32(CP_ME2_PIPE3_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
7335 
 
7336 
	if (rdev->flags & RADEON_IS_IGP)
7337 
		thermal_int = RREG32_SMC(CG_THERMAL_INT_CTRL) &
7338 
			~(THERM_INTH_MASK | THERM_INTL_MASK);
7339 
	else
7340 
		thermal_int = RREG32_SMC(CG_THERMAL_INT) &
7341 
			~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW);
7342 
 
7343 
	/* enable CP interrupts on all rings */
7344 
	if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
7345 
		DRM_DEBUG("cik_irq_set: sw int gfx\n");
7346 
		cp_int_cntl |= TIME_STAMP_INT_ENABLE;
7347 
	}
7348 
	if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP1_INDEX])) {
7349 
		struct radeon_ring *ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
7350 
		DRM_DEBUG("si_irq_set: sw int cp1\n");
7351 
		if (ring->me == 1) {
7352 
			switch (ring->pipe) {
7353 
			case 0:
7354 
				cp_m1p0 |= TIME_STAMP_INT_ENABLE;
7355 
				break;
7356 
			case 1:
7357 
				cp_m1p1 |= TIME_STAMP_INT_ENABLE;
7358 
				break;
7359 
			case 2:
7360 
				cp_m1p2 |= TIME_STAMP_INT_ENABLE;
7361 
				break;
7362 
			case 3:
7363 
				cp_m1p2 |= TIME_STAMP_INT_ENABLE;
7364 
				break;
7365 
			default:
7366 
				DRM_DEBUG("si_irq_set: sw int cp1 invalid pipe %d\n", ring->pipe);
7367 
				break;
7368 
			}
7369 
		} else if (ring->me == 2) {
7370 
			switch (ring->pipe) {
7371 
			case 0:
7372 
				cp_m2p0 |= TIME_STAMP_INT_ENABLE;
7373 
				break;
7374 
			case 1:
7375 
				cp_m2p1 |= TIME_STAMP_INT_ENABLE;
7376 
				break;
7377 
			case 2:
7378 
				cp_m2p2 |= TIME_STAMP_INT_ENABLE;
7379 
				break;
7380 
			case 3:
7381 
				cp_m2p2 |= TIME_STAMP_INT_ENABLE;
7382 
				break;
7383 
			default:
7384 
				DRM_DEBUG("si_irq_set: sw int cp1 invalid pipe %d\n", ring->pipe);
7385 
				break;
7386 
			}
7387 
		} else {
7388 
			DRM_DEBUG("si_irq_set: sw int cp1 invalid me %d\n", ring->me);
7389 
		}
7390 
	}
7391 
	if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP2_INDEX])) {
7392 
		struct radeon_ring *ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
7393 
		DRM_DEBUG("si_irq_set: sw int cp2\n");
7394 
		if (ring->me == 1) {
7395 
			switch (ring->pipe) {
7396 
			case 0:
7397 
				cp_m1p0 |= TIME_STAMP_INT_ENABLE;
7398 
				break;
7399 
			case 1:
7400 
				cp_m1p1 |= TIME_STAMP_INT_ENABLE;
7401 
				break;
7402 
			case 2:
7403 
				cp_m1p2 |= TIME_STAMP_INT_ENABLE;
7404 
				break;
7405 
			case 3:
7406 
				cp_m1p2 |= TIME_STAMP_INT_ENABLE;
7407 
				break;
7408 
			default:
7409 
				DRM_DEBUG("si_irq_set: sw int cp2 invalid pipe %d\n", ring->pipe);
7410 
				break;
7411 
			}
7412 
		} else if (ring->me == 2) {
7413 
			switch (ring->pipe) {
7414 
			case 0:
7415 
				cp_m2p0 |= TIME_STAMP_INT_ENABLE;
7416 
				break;
7417 
			case 1:
7418 
				cp_m2p1 |= TIME_STAMP_INT_ENABLE;
7419 
				break;
7420 
			case 2:
7421 
				cp_m2p2 |= TIME_STAMP_INT_ENABLE;
7422 
				break;
7423 
			case 3:
7424 
				cp_m2p2 |= TIME_STAMP_INT_ENABLE;
7425 
				break;
7426 
			default:
7427 
				DRM_DEBUG("si_irq_set: sw int cp2 invalid pipe %d\n", ring->pipe);
7428 
				break;
7429 
			}
7430 
		} else {
7431 
			DRM_DEBUG("si_irq_set: sw int cp2 invalid me %d\n", ring->me);
7432 
		}
7433 
	}
7434 
 
7435 
	if (atomic_read(&rdev->irq.ring_int[R600_RING_TYPE_DMA_INDEX])) {
7436 
		DRM_DEBUG("cik_irq_set: sw int dma\n");
7437 
		dma_cntl |= TRAP_ENABLE;
7438 
	}
7439 
 
7440 
	if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_DMA1_INDEX])) {
7441 
		DRM_DEBUG("cik_irq_set: sw int dma1\n");
7442 
		dma_cntl1 |= TRAP_ENABLE;
7443 
	}
7444 
 
7445 
	if (rdev->irq.crtc_vblank_int[0] ||
7446 
	    atomic_read(&rdev->irq.pflip[0])) {
7447 
		DRM_DEBUG("cik_irq_set: vblank 0\n");
7448 
		crtc1 |= VBLANK_INTERRUPT_MASK;
7449 
	}
7450 
	if (rdev->irq.crtc_vblank_int[1] ||
7451 
	    atomic_read(&rdev->irq.pflip[1])) {
7452 
		DRM_DEBUG("cik_irq_set: vblank 1\n");
7453 
		crtc2 |= VBLANK_INTERRUPT_MASK;
7454 
	}
7455 
	if (rdev->irq.crtc_vblank_int[2] ||
7456 
	    atomic_read(&rdev->irq.pflip[2])) {
7457 
		DRM_DEBUG("cik_irq_set: vblank 2\n");
7458 
		crtc3 |= VBLANK_INTERRUPT_MASK;
7459 
	}
7460 
	if (rdev->irq.crtc_vblank_int[3] ||
7461 
	    atomic_read(&rdev->irq.pflip[3])) {
7462 
		DRM_DEBUG("cik_irq_set: vblank 3\n");
7463 
		crtc4 |= VBLANK_INTERRUPT_MASK;
7464 
	}
7465 
	if (rdev->irq.crtc_vblank_int[4] ||
7466 
	    atomic_read(&rdev->irq.pflip[4])) {
7467 
		DRM_DEBUG("cik_irq_set: vblank 4\n");
7468 
		crtc5 |= VBLANK_INTERRUPT_MASK;
7469 
	}
7470 
	if (rdev->irq.crtc_vblank_int[5] ||
7471 
	    atomic_read(&rdev->irq.pflip[5])) {
7472 
		DRM_DEBUG("cik_irq_set: vblank 5\n");
7473 
		crtc6 |= VBLANK_INTERRUPT_MASK;
7474 
	}
7475 
	if (rdev->irq.hpd[0]) {
7476 
		DRM_DEBUG("cik_irq_set: hpd 1\n");
7477 
		hpd1 |= DC_HPDx_INT_EN;
7478 
	}
7479 
	if (rdev->irq.hpd[1]) {
7480 
		DRM_DEBUG("cik_irq_set: hpd 2\n");
7481 
		hpd2 |= DC_HPDx_INT_EN;
7482 
	}
7483 
	if (rdev->irq.hpd[2]) {
7484 
		DRM_DEBUG("cik_irq_set: hpd 3\n");
7485 
		hpd3 |= DC_HPDx_INT_EN;
7486 
	}
7487 
	if (rdev->irq.hpd[3]) {
7488 
		DRM_DEBUG("cik_irq_set: hpd 4\n");
7489 
		hpd4 |= DC_HPDx_INT_EN;
7490 
	}
7491 
	if (rdev->irq.hpd[4]) {
7492 
		DRM_DEBUG("cik_irq_set: hpd 5\n");
7493 
		hpd5 |= DC_HPDx_INT_EN;
7494 
	}
7495 
	if (rdev->irq.hpd[5]) {
7496 
		DRM_DEBUG("cik_irq_set: hpd 6\n");
7497 
		hpd6 |= DC_HPDx_INT_EN;
7498 
	}
7499 
 
7500 
	if (rdev->irq.dpm_thermal) {
7501 
		DRM_DEBUG("dpm thermal\n");
7502 
		if (rdev->flags & RADEON_IS_IGP)
7503 
			thermal_int |= THERM_INTH_MASK | THERM_INTL_MASK;
7504 
		else
7505 
			thermal_int |= THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW;
7506 
	}
7507 
 
7508 
	WREG32(CP_INT_CNTL_RING0, cp_int_cntl);
7509 
 
7510 
	WREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET, dma_cntl);
7511 
	WREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET, dma_cntl1);
7512 
 
7513 
	WREG32(CP_ME1_PIPE0_INT_CNTL, cp_m1p0);
7514 
	WREG32(CP_ME1_PIPE1_INT_CNTL, cp_m1p1);
7515 
	WREG32(CP_ME1_PIPE2_INT_CNTL, cp_m1p2);
7516 
	WREG32(CP_ME1_PIPE3_INT_CNTL, cp_m1p3);
7517 
	WREG32(CP_ME2_PIPE0_INT_CNTL, cp_m2p0);
7518 
	WREG32(CP_ME2_PIPE1_INT_CNTL, cp_m2p1);
7519 
	WREG32(CP_ME2_PIPE2_INT_CNTL, cp_m2p2);
7520 
	WREG32(CP_ME2_PIPE3_INT_CNTL, cp_m2p3);
7521 
 
7522 
	WREG32(GRBM_INT_CNTL, grbm_int_cntl);
7523 
 
7524 
	WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, crtc1);
7525 
	WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, crtc2);
7526 
	if (rdev->num_crtc >= 4) {
7527 
		WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, crtc3);
7528 
		WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, crtc4);
7529 
	}
7530 
	if (rdev->num_crtc >= 6) {
7531 
		WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, crtc5);
7532 
		WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, crtc6);
7533 
	}
7534 
 
7535 
	if (rdev->num_crtc >= 2) {
7536 
		WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET,
7537 
		       GRPH_PFLIP_INT_MASK);
7538 
		WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET,
7539 
		       GRPH_PFLIP_INT_MASK);
7540 
	}
7541 
	if (rdev->num_crtc >= 4) {
7542 
		WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET,
7543 
		       GRPH_PFLIP_INT_MASK);
7544 
		WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET,
7545 
		       GRPH_PFLIP_INT_MASK);
7546 
	}
7547 
	if (rdev->num_crtc >= 6) {
7548 
		WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET,
7549 
		       GRPH_PFLIP_INT_MASK);
7550 
		WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET,
7551 
		       GRPH_PFLIP_INT_MASK);
7552 
	}
7553 
 
7554 
	WREG32(DC_HPD1_INT_CONTROL, hpd1);
7555 
	WREG32(DC_HPD2_INT_CONTROL, hpd2);
7556 
	WREG32(DC_HPD3_INT_CONTROL, hpd3);
7557 
	WREG32(DC_HPD4_INT_CONTROL, hpd4);
7558 
	WREG32(DC_HPD5_INT_CONTROL, hpd5);
7559 
	WREG32(DC_HPD6_INT_CONTROL, hpd6);
7560 
 
7561 
	if (rdev->flags & RADEON_IS_IGP)
7562 
		WREG32_SMC(CG_THERMAL_INT_CTRL, thermal_int);
7563 
	else
7564 
		WREG32_SMC(CG_THERMAL_INT, thermal_int);
7565 
 
7566 
	return 0;
7567 
}
7568 
 
7569 
/**
7570 
 * cik_irq_ack - ack interrupt sources
7571 
 *
7572 
 * @rdev: radeon_device pointer
7573 
 *
7574 
 * Ack interrupt sources on the GPU (vblanks, hpd,
7575 
 * etc.) (CIK).  Certain interrupts sources are sw
7576 
 * generated and do not require an explicit ack.
7577 
 */
7578 
static inline void cik_irq_ack(struct radeon_device *rdev)
7579 
{
7580 
	u32 tmp;
7581 
 
7582 
	rdev->irq.stat_regs.cik.disp_int = RREG32(DISP_INTERRUPT_STATUS);
7583 
	rdev->irq.stat_regs.cik.disp_int_cont = RREG32(DISP_INTERRUPT_STATUS_CONTINUE);
7584 
	rdev->irq.stat_regs.cik.disp_int_cont2 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE2);
7585 
	rdev->irq.stat_regs.cik.disp_int_cont3 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE3);
7586 
	rdev->irq.stat_regs.cik.disp_int_cont4 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE4);
7587 
	rdev->irq.stat_regs.cik.disp_int_cont5 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE5);
7588 
	rdev->irq.stat_regs.cik.disp_int_cont6 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE6);
7589 
 
7590 
	rdev->irq.stat_regs.cik.d1grph_int = RREG32(GRPH_INT_STATUS +
7591 
		EVERGREEN_CRTC0_REGISTER_OFFSET);
7592 
	rdev->irq.stat_regs.cik.d2grph_int = RREG32(GRPH_INT_STATUS +
7593 
		EVERGREEN_CRTC1_REGISTER_OFFSET);
7594 
	if (rdev->num_crtc >= 4) {
7595 
		rdev->irq.stat_regs.cik.d3grph_int = RREG32(GRPH_INT_STATUS +
7596 
			EVERGREEN_CRTC2_REGISTER_OFFSET);
7597 
		rdev->irq.stat_regs.cik.d4grph_int = RREG32(GRPH_INT_STATUS +
7598 
			EVERGREEN_CRTC3_REGISTER_OFFSET);
7599 
	}
7600 
	if (rdev->num_crtc >= 6) {
7601 
		rdev->irq.stat_regs.cik.d5grph_int = RREG32(GRPH_INT_STATUS +
7602 
			EVERGREEN_CRTC4_REGISTER_OFFSET);
7603 
		rdev->irq.stat_regs.cik.d6grph_int = RREG32(GRPH_INT_STATUS +
7604 
			EVERGREEN_CRTC5_REGISTER_OFFSET);
7605 
	}
7606 
 
7607 
	if (rdev->irq.stat_regs.cik.d1grph_int & GRPH_PFLIP_INT_OCCURRED)
7608 
		WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET,
7609 
		       GRPH_PFLIP_INT_CLEAR);
7610 
	if (rdev->irq.stat_regs.cik.d2grph_int & GRPH_PFLIP_INT_OCCURRED)
7611 
		WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET,
7612 
		       GRPH_PFLIP_INT_CLEAR);
7613 
	if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VBLANK_INTERRUPT)
7614 
		WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VBLANK_ACK);
7615 
	if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VLINE_INTERRUPT)
7616 
		WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VLINE_ACK);
7617 
	if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VBLANK_INTERRUPT)
7618 
		WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VBLANK_ACK);
7619 
	if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VLINE_INTERRUPT)
7620 
		WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VLINE_ACK);
7621 
 
7622 
	if (rdev->num_crtc >= 4) {
7623 
		if (rdev->irq.stat_regs.cik.d3grph_int & GRPH_PFLIP_INT_OCCURRED)
7624 
			WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET,
7625 
			       GRPH_PFLIP_INT_CLEAR);
7626 
		if (rdev->irq.stat_regs.cik.d4grph_int & GRPH_PFLIP_INT_OCCURRED)
7627 
			WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET,
7628 
			       GRPH_PFLIP_INT_CLEAR);
7629 
		if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT)
7630 
			WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VBLANK_ACK);
7631 
		if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VLINE_INTERRUPT)
7632 
			WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VLINE_ACK);
7633 
		if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT)
7634 
			WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VBLANK_ACK);
7635 
		if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VLINE_INTERRUPT)
7636 
			WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VLINE_ACK);
7637 
	}
7638 
 
7639 
	if (rdev->num_crtc >= 6) {
7640 
		if (rdev->irq.stat_regs.cik.d5grph_int & GRPH_PFLIP_INT_OCCURRED)
7641 
			WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET,
7642 
			       GRPH_PFLIP_INT_CLEAR);
7643 
		if (rdev->irq.stat_regs.cik.d6grph_int & GRPH_PFLIP_INT_OCCURRED)
7644 
			WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET,
7645 
			       GRPH_PFLIP_INT_CLEAR);
7646 
		if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT)
7647 
			WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VBLANK_ACK);
7648 
		if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VLINE_INTERRUPT)
7649 
			WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VLINE_ACK);
7650 
		if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT)
7651 
			WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VBLANK_ACK);
7652 
		if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VLINE_INTERRUPT)
7653 
			WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VLINE_ACK);
7654 
	}
7655 
 
7656 
	if (rdev->irq.stat_regs.cik.disp_int & DC_HPD1_INTERRUPT) {
7657 
		tmp = RREG32(DC_HPD1_INT_CONTROL);
7658 
		tmp |= DC_HPDx_INT_ACK;
7659 
		WREG32(DC_HPD1_INT_CONTROL, tmp);
7660 
	}
7661 
	if (rdev->irq.stat_regs.cik.disp_int_cont & DC_HPD2_INTERRUPT) {
7662 
		tmp = RREG32(DC_HPD2_INT_CONTROL);
7663 
		tmp |= DC_HPDx_INT_ACK;
7664 
		WREG32(DC_HPD2_INT_CONTROL, tmp);
7665 
	}
7666 
	if (rdev->irq.stat_regs.cik.disp_int_cont2 & DC_HPD3_INTERRUPT) {
7667 
		tmp = RREG32(DC_HPD3_INT_CONTROL);
7668 
		tmp |= DC_HPDx_INT_ACK;
7669 
		WREG32(DC_HPD3_INT_CONTROL, tmp);
7670 
	}
7671 
	if (rdev->irq.stat_regs.cik.disp_int_cont3 & DC_HPD4_INTERRUPT) {
7672 
		tmp = RREG32(DC_HPD4_INT_CONTROL);
7673 
		tmp |= DC_HPDx_INT_ACK;
7674 
		WREG32(DC_HPD4_INT_CONTROL, tmp);
7675 
	}
7676 
	if (rdev->irq.stat_regs.cik.disp_int_cont4 & DC_HPD5_INTERRUPT) {
7677 
		tmp = RREG32(DC_HPD5_INT_CONTROL);
7678 
		tmp |= DC_HPDx_INT_ACK;
7679 
		WREG32(DC_HPD5_INT_CONTROL, tmp);
7680 
	}
7681 
	if (rdev->irq.stat_regs.cik.disp_int_cont5 & DC_HPD6_INTERRUPT) {
7682 
		tmp = RREG32(DC_HPD5_INT_CONTROL);
7683 
		tmp |= DC_HPDx_INT_ACK;
7684 
		WREG32(DC_HPD6_INT_CONTROL, tmp);
7685 
	}
7686 
}
7687 
 
7688 
/**
7689 
 * cik_irq_disable - disable interrupts
7690 
 *
7691 
 * @rdev: radeon_device pointer
7692 
 *
7693 
 * Disable interrupts on the hw (CIK).
7694 
 */
7695 
static void cik_irq_disable(struct radeon_device *rdev)
7696 
{
7697 
	cik_disable_interrupts(rdev);
7698 
	/* Wait and acknowledge irq */
7699 
	mdelay(1);
7700 
	cik_irq_ack(rdev);
7701 
	cik_disable_interrupt_state(rdev);
7702 
}
7703 
 
7704 
/**
7705 
 * cik_irq_disable - disable interrupts for suspend
7706 
 *
7707 
 * @rdev: radeon_device pointer
7708 
 *
7709 
 * Disable interrupts and stop the RLC (CIK).
7710 
 * Used for suspend.
7711 
 */
7712 
static void cik_irq_suspend(struct radeon_device *rdev)
7713 
{
7714 
	cik_irq_disable(rdev);
7715 
	cik_rlc_stop(rdev);
7716 
}
7717 
 
7718 
/**
7719 
 * cik_irq_fini - tear down interrupt support
7720 
 *
7721 
 * @rdev: radeon_device pointer
7722 
 *
7723 
 * Disable interrupts on the hw and free the IH ring
7724 
 * buffer (CIK).
7725 
 * Used for driver unload.
7726 
 */
7727 
static void cik_irq_fini(struct radeon_device *rdev)
7728 
{
7729 
	cik_irq_suspend(rdev);
7730 
	r600_ih_ring_fini(rdev);
7731 
}
7732 
 
7733 
/**
7734 
 * cik_get_ih_wptr - get the IH ring buffer wptr
7735 
 *
7736 
 * @rdev: radeon_device pointer
7737 
 *
7738 
 * Get the IH ring buffer wptr from either the register
7739 
 * or the writeback memory buffer (CIK).  Also check for
7740 
 * ring buffer overflow and deal with it.
7741 
 * Used by cik_irq_process().
7742 
 * Returns the value of the wptr.
7743 
 */
7744 
static inline u32 cik_get_ih_wptr(struct radeon_device *rdev)
7745 
{
7746 
	u32 wptr, tmp;
7747 
 
7748 
	if (rdev->wb.enabled)
7749 
		wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]);
7750 
	else
7751 
		wptr = RREG32(IH_RB_WPTR);
7752 
 
7753 
	if (wptr & RB_OVERFLOW) {
5179 serge 7754 
		wptr &= ~RB_OVERFLOW;
5078 serge 7755 
		/* When a ring buffer overflow happen start parsing interrupt
7756 
		 * from the last not overwritten vector (wptr + 16). Hopefully
7757 
		 * this should allow us to catchup.
7758 
		 */
5179 serge 7759 
		dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, 0x%08X, 0x%08X)\n",
7760 
			 wptr, rdev->ih.rptr, (wptr + 16) & rdev->ih.ptr_mask);
5078 serge 7761 
		rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask;
7762 
		tmp = RREG32(IH_RB_CNTL);
7763 
		tmp |= IH_WPTR_OVERFLOW_CLEAR;
7764 
		WREG32(IH_RB_CNTL, tmp);
7765 
	}
7766 
	return (wptr & rdev->ih.ptr_mask);
7767 
}
7768 
 
7769 
/*        CIK IV Ring
7770 
 * Each IV ring entry is 128 bits:
7771 
 * [7:0]    - interrupt source id
7772 
 * [31:8]   - reserved
7773 
 * [59:32]  - interrupt source data
7774 
 * [63:60]  - reserved
7775 
 * [71:64]  - RINGID
7776 
 *            CP:
7777 
 *            ME_ID [1:0], PIPE_ID[1:0], QUEUE_ID[2:0]
7778 
 *            QUEUE_ID - for compute, which of the 8 queues owned by the dispatcher
7779 
 *                     - for gfx, hw shader state (0=PS...5=LS, 6=CS)
7780 
 *            ME_ID - 0 = gfx, 1 = first 4 CS pipes, 2 = second 4 CS pipes
7781 
 *            PIPE_ID - ME0 0=3D
7782 
 *                    - ME1&2 compute dispatcher (4 pipes each)
7783 
 *            SDMA:
7784 
 *            INSTANCE_ID [1:0], QUEUE_ID[1:0]
7785 
 *            INSTANCE_ID - 0 = sdma0, 1 = sdma1
7786 
 *            QUEUE_ID - 0 = gfx, 1 = rlc0, 2 = rlc1
7787 
 * [79:72]  - VMID
7788 
 * [95:80]  - PASID
7789 
 * [127:96] - reserved
7790 
 */
7791 
/**
7792 
 * cik_irq_process - interrupt handler
7793 
 *
7794 
 * @rdev: radeon_device pointer
7795 
 *
7796 
 * Interrupt hander (CIK).  Walk the IH ring,
7797 
 * ack interrupts and schedule work to handle
7798 
 * interrupt events.
7799 
 * Returns irq process return code.
7800 
 */
7801 
int cik_irq_process(struct radeon_device *rdev)
7802 
{
7803 
	struct radeon_ring *cp1_ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
7804 
	struct radeon_ring *cp2_ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
7805 
	u32 wptr;
7806 
	u32 rptr;
7807 
	u32 src_id, src_data, ring_id;
7808 
	u8 me_id, pipe_id, queue_id;
7809 
	u32 ring_index;
7810 
	bool queue_hotplug = false;
7811 
	bool queue_reset = false;
7812 
	u32 addr, status, mc_client;
7813 
	bool queue_thermal = false;
7814 
 
7815 
	if (!rdev->ih.enabled || rdev->shutdown)
7816 
		return IRQ_NONE;
7817 
 
7818 
	wptr = cik_get_ih_wptr(rdev);
7819 
 
7820 
restart_ih:
7821 
	/* is somebody else already processing irqs? */
7822 
	if (atomic_xchg(&rdev->ih.lock, 1))
7823 
		return IRQ_NONE;
7824 
 
7825 
	rptr = rdev->ih.rptr;
7826 
	DRM_DEBUG("cik_irq_process start: rptr %d, wptr %d\n", rptr, wptr);
7827 
 
7828 
	/* Order reading of wptr vs. reading of IH ring data */
7829 
	rmb();
7830 
 
7831 
	/* display interrupts */
7832 
	cik_irq_ack(rdev);
7833 
 
7834 
	while (rptr != wptr) {
7835 
		/* wptr/rptr are in bytes! */
7836 
		ring_index = rptr / 4;
7837 
		src_id =  le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff;
7838 
		src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff;
7839 
		ring_id = le32_to_cpu(rdev->ih.ring[ring_index + 2]) & 0xff;
7840 
 
7841 
		switch (src_id) {
7842 
		case 1: /* D1 vblank/vline */
7843 
			switch (src_data) {
7844 
			case 0: /* D1 vblank */
7845 
				if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VBLANK_INTERRUPT) {
7846 
					if (rdev->irq.crtc_vblank_int[0]) {
7847 
						rdev->pm.vblank_sync = true;
7848 
					}
7849 
					rdev->irq.stat_regs.cik.disp_int &= ~LB_D1_VBLANK_INTERRUPT;
7850 
					DRM_DEBUG("IH: D1 vblank\n");
7851 
				}
7852 
				break;
7853 
			case 1: /* D1 vline */
7854 
				if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VLINE_INTERRUPT) {
7855 
					rdev->irq.stat_regs.cik.disp_int &= ~LB_D1_VLINE_INTERRUPT;
7856 
					DRM_DEBUG("IH: D1 vline\n");
7857 
				}
7858 
				break;
7859 
			default:
7860 
				DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7861 
				break;
7862 
			}
7863 
			break;
7864 
		case 2: /* D2 vblank/vline */
7865 
			switch (src_data) {
7866 
			case 0: /* D2 vblank */
7867 
				if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VBLANK_INTERRUPT) {
7868 
					if (rdev->irq.crtc_vblank_int[1]) {
7869 
						rdev->pm.vblank_sync = true;
7870 
					}
7871 
					rdev->irq.stat_regs.cik.disp_int_cont &= ~LB_D2_VBLANK_INTERRUPT;
7872 
					DRM_DEBUG("IH: D2 vblank\n");
7873 
				}
7874 
				break;
7875 
			case 1: /* D2 vline */
7876 
				if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VLINE_INTERRUPT) {
7877 
					rdev->irq.stat_regs.cik.disp_int_cont &= ~LB_D2_VLINE_INTERRUPT;
7878 
					DRM_DEBUG("IH: D2 vline\n");
7879 
				}
7880 
				break;
7881 
			default:
7882 
				DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7883 
				break;
7884 
			}
7885 
			break;
7886 
		case 3: /* D3 vblank/vline */
7887 
			switch (src_data) {
7888 
			case 0: /* D3 vblank */
7889 
				if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT) {
7890 
					if (rdev->irq.crtc_vblank_int[2]) {
7891 
						rdev->pm.vblank_sync = true;
7892 
					}
7893 
					rdev->irq.stat_regs.cik.disp_int_cont2 &= ~LB_D3_VBLANK_INTERRUPT;
7894 
					DRM_DEBUG("IH: D3 vblank\n");
7895 
				}
7896 
				break;
7897 
			case 1: /* D3 vline */
7898 
				if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VLINE_INTERRUPT) {
7899 
					rdev->irq.stat_regs.cik.disp_int_cont2 &= ~LB_D3_VLINE_INTERRUPT;
7900 
					DRM_DEBUG("IH: D3 vline\n");
7901 
				}
7902 
				break;
7903 
			default:
7904 
				DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7905 
				break;
7906 
			}
7907 
			break;
7908 
		case 4: /* D4 vblank/vline */
7909 
			switch (src_data) {
7910 
			case 0: /* D4 vblank */
7911 
				if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT) {
7912 
					if (rdev->irq.crtc_vblank_int[3]) {
7913 
						rdev->pm.vblank_sync = true;
7914 
					}
7915 
					rdev->irq.stat_regs.cik.disp_int_cont3 &= ~LB_D4_VBLANK_INTERRUPT;
7916 
					DRM_DEBUG("IH: D4 vblank\n");
7917 
				}
7918 
				break;
7919 
			case 1: /* D4 vline */
7920 
				if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VLINE_INTERRUPT) {
7921 
					rdev->irq.stat_regs.cik.disp_int_cont3 &= ~LB_D4_VLINE_INTERRUPT;
7922 
					DRM_DEBUG("IH: D4 vline\n");
7923 
				}
7924 
				break;
7925 
			default:
7926 
				DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7927 
				break;
7928 
			}
7929 
			break;
7930 
		case 5: /* D5 vblank/vline */
7931 
			switch (src_data) {
7932 
			case 0: /* D5 vblank */
7933 
				if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT) {
7934 
					if (rdev->irq.crtc_vblank_int[4]) {
7935 
						rdev->pm.vblank_sync = true;
7936 
					}
7937 
					rdev->irq.stat_regs.cik.disp_int_cont4 &= ~LB_D5_VBLANK_INTERRUPT;
7938 
					DRM_DEBUG("IH: D5 vblank\n");
7939 
				}
7940 
				break;
7941 
			case 1: /* D5 vline */
7942 
				if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VLINE_INTERRUPT) {
7943 
					rdev->irq.stat_regs.cik.disp_int_cont4 &= ~LB_D5_VLINE_INTERRUPT;
7944 
					DRM_DEBUG("IH: D5 vline\n");
7945 
				}
7946 
				break;
7947 
			default:
7948 
				DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7949 
				break;
7950 
			}
7951 
			break;
7952 
		case 6: /* D6 vblank/vline */
7953 
			switch (src_data) {
7954 
			case 0: /* D6 vblank */
7955 
				if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT) {
7956 
					if (rdev->irq.crtc_vblank_int[5]) {
7957 
						rdev->pm.vblank_sync = true;
7958 
					}
7959 
					rdev->irq.stat_regs.cik.disp_int_cont5 &= ~LB_D6_VBLANK_INTERRUPT;
7960 
					DRM_DEBUG("IH: D6 vblank\n");
7961 
				}
7962 
				break;
7963 
			case 1: /* D6 vline */
7964 
				if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VLINE_INTERRUPT) {
7965 
					rdev->irq.stat_regs.cik.disp_int_cont5 &= ~LB_D6_VLINE_INTERRUPT;
7966 
					DRM_DEBUG("IH: D6 vline\n");
7967 
				}
7968 
				break;
7969 
			default:
7970 
				DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7971 
				break;
7972 
			}
7973 
			break;
7974 
		case 8: /* D1 page flip */
7975 
		case 10: /* D2 page flip */
7976 
		case 12: /* D3 page flip */
7977 
		case 14: /* D4 page flip */
7978 
		case 16: /* D5 page flip */
7979 
		case 18: /* D6 page flip */
7980 
			DRM_DEBUG("IH: D%d flip\n", ((src_id - 8) >> 1) + 1);
7981 
			break;
7982 
		case 42: /* HPD hotplug */
7983 
			switch (src_data) {
7984 
			case 0:
7985 
				if (rdev->irq.stat_regs.cik.disp_int & DC_HPD1_INTERRUPT) {
7986 
					rdev->irq.stat_regs.cik.disp_int &= ~DC_HPD1_INTERRUPT;
7987 
					queue_hotplug = true;
7988 
					DRM_DEBUG("IH: HPD1\n");
7989 
				}
7990 
				break;
7991 
			case 1:
7992 
				if (rdev->irq.stat_regs.cik.disp_int_cont & DC_HPD2_INTERRUPT) {
7993 
					rdev->irq.stat_regs.cik.disp_int_cont &= ~DC_HPD2_INTERRUPT;
7994 
					queue_hotplug = true;
7995 
					DRM_DEBUG("IH: HPD2\n");
7996 
				}
7997 
				break;
7998 
			case 2:
7999 
				if (rdev->irq.stat_regs.cik.disp_int_cont2 & DC_HPD3_INTERRUPT) {
8000 
					rdev->irq.stat_regs.cik.disp_int_cont2 &= ~DC_HPD3_INTERRUPT;
8001 
					queue_hotplug = true;
8002 
					DRM_DEBUG("IH: HPD3\n");
8003 
				}
8004 
				break;
8005 
			case 3:
8006 
				if (rdev->irq.stat_regs.cik.disp_int_cont3 & DC_HPD4_INTERRUPT) {
8007 
					rdev->irq.stat_regs.cik.disp_int_cont3 &= ~DC_HPD4_INTERRUPT;
8008 
					queue_hotplug = true;
8009 
					DRM_DEBUG("IH: HPD4\n");
8010 
				}
8011 
				break;
8012 
			case 4:
8013 
				if (rdev->irq.stat_regs.cik.disp_int_cont4 & DC_HPD5_INTERRUPT) {
8014 
					rdev->irq.stat_regs.cik.disp_int_cont4 &= ~DC_HPD5_INTERRUPT;
8015 
					queue_hotplug = true;
8016 
					DRM_DEBUG("IH: HPD5\n");
8017 
				}
8018 
				break;
8019 
			case 5:
8020 
				if (rdev->irq.stat_regs.cik.disp_int_cont5 & DC_HPD6_INTERRUPT) {
8021 
					rdev->irq.stat_regs.cik.disp_int_cont5 &= ~DC_HPD6_INTERRUPT;
8022 
					queue_hotplug = true;
8023 
					DRM_DEBUG("IH: HPD6\n");
8024 
				}
8025 
				break;
8026 
			default:
8027 
				DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
8028 
				break;
8029 
			}
8030 
			break;
8031 
		case 124: /* UVD */
8032 
			DRM_DEBUG("IH: UVD int: 0x%08x\n", src_data);
8033 
			radeon_fence_process(rdev, R600_RING_TYPE_UVD_INDEX);
8034 
			break;
8035 
		case 146:
8036 
		case 147:
8037 
			addr = RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR);
8038 
			status = RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS);
8039 
			mc_client = RREG32(VM_CONTEXT1_PROTECTION_FAULT_MCCLIENT);
8040 
			/* reset addr and status */
8041 
			WREG32_P(VM_CONTEXT1_CNTL2, 1, ~1);
8042 
			if (addr == 0x0 && status == 0x0)
8043 
				break;
8044 
			dev_err(rdev->dev, "GPU fault detected: %d 0x%08x\n", src_id, src_data);
8045 
			dev_err(rdev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_ADDR   0x%08X\n",
8046 
				addr);
8047 
			dev_err(rdev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
8048 
				status);
8049 
			cik_vm_decode_fault(rdev, status, addr, mc_client);
8050 
			break;
8051 
		case 167: /* VCE */
8052 
			DRM_DEBUG("IH: VCE int: 0x%08x\n", src_data);
8053 
			switch (src_data) {
8054 
			case 0:
8055 
				radeon_fence_process(rdev, TN_RING_TYPE_VCE1_INDEX);
8056 
				break;
8057 
			case 1:
8058 
				radeon_fence_process(rdev, TN_RING_TYPE_VCE2_INDEX);
8059 
				break;
8060 
			default:
8061 
				DRM_ERROR("Unhandled interrupt: %d %d\n", src_id, src_data);
8062 
				break;
8063 
			}
8064 
			break;
8065 
		case 176: /* GFX RB CP_INT */
8066 
		case 177: /* GFX IB CP_INT */
8067 
			radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
8068 
			break;
8069 
		case 181: /* CP EOP event */
8070 
			DRM_DEBUG("IH: CP EOP\n");
8071 
			/* XXX check the bitfield order! */
8072 
			me_id = (ring_id & 0x60) >> 5;
8073 
			pipe_id = (ring_id & 0x18) >> 3;
8074 
			queue_id = (ring_id & 0x7) >> 0;
8075 
			switch (me_id) {
8076 
			case 0:
8077 
				radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
8078 
				break;
8079 
			case 1:
8080 
			case 2:
8081 
				if ((cp1_ring->me == me_id) & (cp1_ring->pipe == pipe_id))
8082 
					radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
8083 
				if ((cp2_ring->me == me_id) & (cp2_ring->pipe == pipe_id))
8084 
					radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
8085 
				break;
8086 
			}
8087 
			break;
8088 
		case 184: /* CP Privileged reg access */
8089 
			DRM_ERROR("Illegal register access in command stream\n");
8090 
			/* XXX check the bitfield order! */
8091 
			me_id = (ring_id & 0x60) >> 5;
8092 
			pipe_id = (ring_id & 0x18) >> 3;
8093 
			queue_id = (ring_id & 0x7) >> 0;
8094 
			switch (me_id) {
8095 
			case 0:
8096 
				/* This results in a full GPU reset, but all we need to do is soft
8097 
				 * reset the CP for gfx
8098 
				 */
8099 
				queue_reset = true;
8100 
				break;
8101 
			case 1:
8102 
				/* XXX compute */
8103 
				queue_reset = true;
8104 
				break;
8105 
			case 2:
8106 
				/* XXX compute */
8107 
				queue_reset = true;
8108 
				break;
8109 
			}
8110 
			break;
8111 
		case 185: /* CP Privileged inst */
8112 
			DRM_ERROR("Illegal instruction in command stream\n");
8113 
			/* XXX check the bitfield order! */
8114 
			me_id = (ring_id & 0x60) >> 5;
8115 
			pipe_id = (ring_id & 0x18) >> 3;
8116 
			queue_id = (ring_id & 0x7) >> 0;
8117 
			switch (me_id) {
8118 
			case 0:
8119 
				/* This results in a full GPU reset, but all we need to do is soft
8120 
				 * reset the CP for gfx
8121 
				 */
8122 
				queue_reset = true;
8123 
				break;
8124 
			case 1:
8125 
				/* XXX compute */
8126 
				queue_reset = true;
8127 
				break;
8128 
			case 2:
8129 
				/* XXX compute */
8130 
				queue_reset = true;
8131 
				break;
8132 
			}
8133 
			break;
8134 
		case 224: /* SDMA trap event */
8135 
			/* XXX check the bitfield order! */
8136 
			me_id = (ring_id & 0x3) >> 0;
8137 
			queue_id = (ring_id & 0xc) >> 2;
8138 
			DRM_DEBUG("IH: SDMA trap\n");
8139 
			switch (me_id) {
8140 
			case 0:
8141 
				switch (queue_id) {
8142 
				case 0:
8143 
					radeon_fence_process(rdev, R600_RING_TYPE_DMA_INDEX);
8144 
					break;
8145 
				case 1:
8146 
					/* XXX compute */
8147 
					break;
8148 
				case 2:
8149 
					/* XXX compute */
8150 
					break;
8151 
				}
8152 
				break;
8153 
			case 1:
8154 
				switch (queue_id) {
8155 
				case 0:
8156 
					radeon_fence_process(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
8157 
					break;
8158 
				case 1:
8159 
					/* XXX compute */
8160 
					break;
8161 
				case 2:
8162 
					/* XXX compute */
8163 
					break;
8164 
				}
8165 
				break;
8166 
			}
8167 
			break;
8168 
		case 230: /* thermal low to high */
8169 
			DRM_DEBUG("IH: thermal low to high\n");
8170 
			rdev->pm.dpm.thermal.high_to_low = false;
8171 
			queue_thermal = true;
8172 
			break;
8173 
		case 231: /* thermal high to low */
8174 
			DRM_DEBUG("IH: thermal high to low\n");
8175 
			rdev->pm.dpm.thermal.high_to_low = true;
8176 
			queue_thermal = true;
8177 
			break;
8178 
		case 233: /* GUI IDLE */
8179 
			DRM_DEBUG("IH: GUI idle\n");
8180 
			break;
8181 
		case 241: /* SDMA Privileged inst */
8182 
		case 247: /* SDMA Privileged inst */
8183 
			DRM_ERROR("Illegal instruction in SDMA command stream\n");
8184 
			/* XXX check the bitfield order! */
8185 
			me_id = (ring_id & 0x3) >> 0;
8186 
			queue_id = (ring_id & 0xc) >> 2;
8187 
			switch (me_id) {
8188 
			case 0:
8189 
				switch (queue_id) {
8190 
				case 0:
8191 
					queue_reset = true;
8192 
					break;
8193 
				case 1:
8194 
					/* XXX compute */
8195 
					queue_reset = true;
8196 
					break;
8197 
				case 2:
8198 
					/* XXX compute */
8199 
					queue_reset = true;
8200 
					break;
8201 
				}
8202 
				break;
8203 
			case 1:
8204 
				switch (queue_id) {
8205 
				case 0:
8206 
					queue_reset = true;
8207 
					break;
8208 
				case 1:
8209 
					/* XXX compute */
8210 
					queue_reset = true;
8211 
					break;
8212 
				case 2:
8213 
					/* XXX compute */
8214 
					queue_reset = true;
8215 
					break;
8216 
				}
8217 
				break;
8218 
			}
8219 
			break;
8220 
		default:
8221 
			DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
8222 
			break;
8223 
		}
8224 
 
8225 
		/* wptr/rptr are in bytes! */
8226 
		rptr += 16;
8227 
		rptr &= rdev->ih.ptr_mask;
5179 serge 8228 
		WREG32(IH_RB_RPTR, rptr);
5078 serge 8229 
	}
8230 
	rdev->ih.rptr = rptr;
8231 
	atomic_set(&rdev->ih.lock, 0);
8232 
 
8233 
	/* make sure wptr hasn't changed while processing */
8234 
	wptr = cik_get_ih_wptr(rdev);
8235 
	if (wptr != rptr)
8236 
		goto restart_ih;
8237 
 
8238 
	return IRQ_HANDLED;
8239 
}
8240 
 
8241 
/*
8242 
 * startup/shutdown callbacks
8243 
 */
8244 
/**
8245 
 * cik_startup - program the asic to a functional state
8246 
 *
8247 
 * @rdev: radeon_device pointer
8248 
 *
8249 
 * Programs the asic to a functional state (CIK).
8250 
 * Called by cik_init() and cik_resume().
8251 
 * Returns 0 for success, error for failure.
8252 
 */
8253 
static int cik_startup(struct radeon_device *rdev)
8254 
{
8255 
	struct radeon_ring *ring;
8256 
	u32 nop;
8257 
	int r;
8258 
 
8259 
	/* enable pcie gen2/3 link */
8260 
	cik_pcie_gen3_enable(rdev);
8261 
	/* enable aspm */
8262 
	cik_program_aspm(rdev);
8263 
 
8264 
	/* scratch needs to be initialized before MC */
8265 
	r = r600_vram_scratch_init(rdev);
8266 
	if (r)
8267 
		return r;
8268 
 
8269 
	cik_mc_program(rdev);
8270 
 
8271 
	if (!(rdev->flags & RADEON_IS_IGP) && !rdev->pm.dpm_enabled) {
8272 
		r = ci_mc_load_microcode(rdev);
8273 
		if (r) {
8274 
			DRM_ERROR("Failed to load MC firmware!\n");
8275 
			return r;
8276 
		}
8277 
	}
8278 
 
8279 
	r = cik_pcie_gart_enable(rdev);
8280 
	if (r)
8281 
		return r;
8282 
	cik_gpu_init(rdev);
8283 
 
8284 
	/* allocate rlc buffers */
8285 
	if (rdev->flags & RADEON_IS_IGP) {
8286 
		if (rdev->family == CHIP_KAVERI) {
8287 
			rdev->rlc.reg_list = spectre_rlc_save_restore_register_list;
8288 
			rdev->rlc.reg_list_size =
8289 
				(u32)ARRAY_SIZE(spectre_rlc_save_restore_register_list);
8290 
		} else {
8291 
			rdev->rlc.reg_list = kalindi_rlc_save_restore_register_list;
8292 
			rdev->rlc.reg_list_size =
8293 
				(u32)ARRAY_SIZE(kalindi_rlc_save_restore_register_list);
8294 
		}
8295 
	}
8296 
	rdev->rlc.cs_data = ci_cs_data;
8297 
	rdev->rlc.cp_table_size = CP_ME_TABLE_SIZE * 5 * 4;
8298 
	r = sumo_rlc_init(rdev);
8299 
	if (r) {
8300 
		DRM_ERROR("Failed to init rlc BOs!\n");
8301 
		return r;
8302 
	}
8303 
 
8304 
	/* allocate wb buffer */
8305 
	r = radeon_wb_init(rdev);
8306 
	if (r)
8307 
		return r;
8308 
 
8309 
	/* allocate mec buffers */
8310 
	r = cik_mec_init(rdev);
8311 
	if (r) {
8312 
		DRM_ERROR("Failed to init MEC BOs!\n");
8313 
		return r;
8314 
	}
8315 
 
8316 
	r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
8317 
	if (r) {
8318 
		dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
8319 
		return r;
8320 
	}
8321 
 
8322 
	r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
8323 
	if (r) {
8324 
		dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
8325 
		return r;
8326 
	}
8327 
 
8328 
	r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
8329 
	if (r) {
8330 
		dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
8331 
		return r;
8332 
	}
8333 
 
8334 
	r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX);
8335 
	if (r) {
8336 
		dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
8337 
		return r;
8338 
	}
8339 
 
8340 
	r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
8341 
	if (r) {
8342 
		dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
8343 
		return r;
8344 
	}
8345 
 
8346 
	r = radeon_uvd_resume(rdev);
8347 
	if (!r) {
8348 
		r = uvd_v4_2_resume(rdev);
8349 
		if (!r) {
8350 
			r = radeon_fence_driver_start_ring(rdev,
8351 
							   R600_RING_TYPE_UVD_INDEX);
8352 
			if (r)
8353 
				dev_err(rdev->dev, "UVD fences init error (%d).\n", r);
8354 
		}
8355 
	}
8356 
	if (r)
8357 
		rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0;
8358 
 
8359 
	if (r) {
8360 
		dev_err(rdev->dev, "VCE init error (%d).\n", r);
8361 
		rdev->ring[TN_RING_TYPE_VCE1_INDEX].ring_size = 0;
8362 
		rdev->ring[TN_RING_TYPE_VCE2_INDEX].ring_size = 0;
8363 
	}
8364 
 
8365 
	/* Enable IRQ */
8366 
	if (!rdev->irq.installed) {
8367 
		r = radeon_irq_kms_init(rdev);
8368 
		if (r)
8369 
			return r;
8370 
	}
8371 
 
8372 
	r = cik_irq_init(rdev);
8373 
	if (r) {
8374 
		DRM_ERROR("radeon: IH init failed (%d).\n", r);
8375 
//       radeon_irq_kms_fini(rdev);
8376 
		return r;
8377 
	}
8378 
	cik_irq_set(rdev);
8379 
 
8380 
	if (rdev->family == CHIP_HAWAII) {
8381 
		if (rdev->new_fw)
8382 
			nop = PACKET3(PACKET3_NOP, 0x3FFF);
8383 
		else
8384 
			nop = RADEON_CP_PACKET2;
8385 
	} else {
8386 
		nop = PACKET3(PACKET3_NOP, 0x3FFF);
8387 
	}
8388 
 
8389 
	ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
8390 
	r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
8391 
			     nop);
8392 
	if (r)
8393 
		return r;
8394 
 
8395 
	/* set up the compute queues */
8396 
	/* type-2 packets are deprecated on MEC, use type-3 instead */
8397 
	ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
8398 
	r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP1_RPTR_OFFSET,
8399 
			     nop);
8400 
	if (r)
8401 
		return r;
8402 
	ring->me = 1; /* first MEC */
8403 
	ring->pipe = 0; /* first pipe */
8404 
	ring->queue = 0; /* first queue */
8405 
	ring->wptr_offs = CIK_WB_CP1_WPTR_OFFSET;
8406 
 
8407 
	/* type-2 packets are deprecated on MEC, use type-3 instead */
8408 
	ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
8409 
	r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP2_RPTR_OFFSET,
8410 
			     nop);
8411 
	if (r)
8412 
		return r;
8413 
	/* dGPU only have 1 MEC */
8414 
	ring->me = 1; /* first MEC */
8415 
	ring->pipe = 0; /* first pipe */
8416 
	ring->queue = 1; /* second queue */
8417 
	ring->wptr_offs = CIK_WB_CP2_WPTR_OFFSET;
8418 
 
8419 
	ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
8420 
	r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET,
8421 
			     SDMA_PACKET(SDMA_OPCODE_NOP, 0, 0));
8422 
	if (r)
8423 
		return r;
8424 
 
8425 
	ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
8426 
	r = radeon_ring_init(rdev, ring, ring->ring_size, CAYMAN_WB_DMA1_RPTR_OFFSET,
8427 
			     SDMA_PACKET(SDMA_OPCODE_NOP, 0, 0));
8428 
	if (r)
8429 
		return r;
8430 
 
8431 
	r = cik_cp_resume(rdev);
8432 
	if (r)
8433 
		return r;
8434 
 
8435 
	r = cik_sdma_resume(rdev);
8436 
	if (r)
8437 
		return r;
8438 
 
8439 
	ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
8440 
	if (ring->ring_size) {
8441 
		r = radeon_ring_init(rdev, ring, ring->ring_size, 0,
8442 
				     RADEON_CP_PACKET2);
8443 
		if (!r)
8444 
			r = uvd_v1_0_init(rdev);
8445 
		if (r)
8446 
			DRM_ERROR("radeon: failed initializing UVD (%d).\n", r);
8447 
	}
8448 
	r = radeon_ib_pool_init(rdev);
8449 
	if (r) {
8450 
		dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
8451 
		return r;
8452 
	}
8453 
 
8454 
	r = radeon_vm_manager_init(rdev);
8455 
	if (r) {
8456 
		dev_err(rdev->dev, "vm manager initialization failed (%d).\n", r);
8457 
		return r;
8458 
	}
8459 
 
8460 
	return 0;
8461 
}
8462 
 
8463 
 
8464 
/* Plan is to move initialization in that function and use
8465 
 * helper function so that radeon_device_init pretty much
8466 
 * do nothing more than calling asic specific function. This
8467 
 * should also allow to remove a bunch of callback function
8468 
 * like vram_info.
8469 
 */
8470 
/**
8471 
 * cik_init - asic specific driver and hw init
8472 
 *
8473 
 * @rdev: radeon_device pointer
8474 
 *
8475 
 * Setup asic specific driver variables and program the hw
8476 
 * to a functional state (CIK).
8477 
 * Called at driver startup.
8478 
 * Returns 0 for success, errors for failure.
8479 
 */
8480 
int cik_init(struct radeon_device *rdev)
8481 
{
8482 
	struct radeon_ring *ring;
8483 
	int r;
8484 
 
8485 
	/* Read BIOS */
8486 
	if (!radeon_get_bios(rdev)) {
8487 
		if (ASIC_IS_AVIVO(rdev))
8488 
			return -EINVAL;
8489 
	}
8490 
	/* Must be an ATOMBIOS */
8491 
	if (!rdev->is_atom_bios) {
8492 
		dev_err(rdev->dev, "Expecting atombios for cayman GPU\n");
8493 
		return -EINVAL;
8494 
	}
8495 
	r = radeon_atombios_init(rdev);
8496 
	if (r)
8497 
		return r;
8498 
 
8499 
	/* Post card if necessary */
8500 
	if (!radeon_card_posted(rdev)) {
8501 
		if (!rdev->bios) {
8502 
			dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
8503 
			return -EINVAL;
8504 
		}
8505 
		DRM_INFO("GPU not posted. posting now...\n");
8506 
		atom_asic_init(rdev->mode_info.atom_context);
8507 
	}
8508 
	/* init golden registers */
8509 
	cik_init_golden_registers(rdev);
8510 
	/* Initialize scratch registers */
8511 
	cik_scratch_init(rdev);
8512 
	/* Initialize surface registers */
8513 
	radeon_surface_init(rdev);
8514 
	/* Initialize clocks */
8515 
	radeon_get_clock_info(rdev->ddev);
8516 
 
8517 
	/* Fence driver */
8518 
	r = radeon_fence_driver_init(rdev);
8519 
	if (r)
8520 
		return r;
8521 
 
8522 
	/* initialize memory controller */
8523 
	r = cik_mc_init(rdev);
8524 
	if (r)
8525 
		return r;
8526 
	/* Memory manager */
8527 
	r = radeon_bo_init(rdev);
8528 
	if (r)
8529 
		return r;
8530 
 
8531 
	if (rdev->flags & RADEON_IS_IGP) {
8532 
		if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw ||
8533 
		    !rdev->mec_fw || !rdev->sdma_fw || !rdev->rlc_fw) {
8534 
			r = cik_init_microcode(rdev);
8535 
			if (r) {
8536 
				DRM_ERROR("Failed to load firmware!\n");
8537 
				return r;
8538 
			}
8539 
		}
8540 
	} else {
8541 
		if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw ||
8542 
		    !rdev->mec_fw || !rdev->sdma_fw || !rdev->rlc_fw ||
8543 
		    !rdev->mc_fw) {
8544 
			r = cik_init_microcode(rdev);
8545 
			if (r) {
8546 
				DRM_ERROR("Failed to load firmware!\n");
8547 
				return r;
8548 
			}
8549 
		}
8550 
	}
8551 
 
8552 
	/* Initialize power management */
8553 
	radeon_pm_init(rdev);
8554 
 
8555 
	ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
8556 
	ring->ring_obj = NULL;
8557 
	r600_ring_init(rdev, ring, 1024 * 1024);
8558 
 
8559 
	ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
8560 
	ring->ring_obj = NULL;
8561 
	r600_ring_init(rdev, ring, 1024 * 1024);
8562 
	r = radeon_doorbell_get(rdev, &ring->doorbell_index);
8563 
	if (r)
8564 
		return r;
8565 
 
8566 
	ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
8567 
	ring->ring_obj = NULL;
8568 
	r600_ring_init(rdev, ring, 1024 * 1024);
8569 
	r = radeon_doorbell_get(rdev, &ring->doorbell_index);
8570 
	if (r)
8571 
		return r;
8572 
 
8573 
	ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
8574 
	ring->ring_obj = NULL;
8575 
	r600_ring_init(rdev, ring, 256 * 1024);
8576 
 
8577 
	ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
8578 
	ring->ring_obj = NULL;
8579 
	r600_ring_init(rdev, ring, 256 * 1024);
8580 
 
8581 
	r = radeon_uvd_init(rdev);
8582 
	if (!r) {
8583 
		ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
8584 
		ring->ring_obj = NULL;
8585 
		r600_ring_init(rdev, ring, 4096);
8586 
	}
8587 
	rdev->ih.ring_obj = NULL;
8588 
	r600_ih_ring_init(rdev, 64 * 1024);
8589 
 
8590 
	r = r600_pcie_gart_init(rdev);
8591 
	if (r)
8592 
		return r;
8593 
 
8594 
	rdev->accel_working = true;
8595 
	r = cik_startup(rdev);
8596 
	if (r) {
8597 
		dev_err(rdev->dev, "disabling GPU acceleration\n");
8598 
        rdev->accel_working = false;
8599 
	}
8600 
 
8601 
	/* Don't start up if the MC ucode is missing.
8602 
	 * The default clocks and voltages before the MC ucode
8603 
	 * is loaded are not suffient for advanced operations.
8604 
	 */
8605 
	if (!rdev->mc_fw && !(rdev->flags & RADEON_IS_IGP)) {
8606 
		DRM_ERROR("radeon: MC ucode required for NI+.\n");
8607 
		return -EINVAL;
8608 
	}
8609 
 
8610 
	return 0;
8611 
}
8612 
 
8613 
/**
8614 
 * cik_fini - asic specific driver and hw fini
8615 
 *
8616 
 * @rdev: radeon_device pointer
8617 
 *
8618 
 * Tear down the asic specific driver variables and program the hw
8619 
 * to an idle state (CIK).
8620 
 * Called at driver unload.
8621 
 */
8622 
void cik_fini(struct radeon_device *rdev)
8623 
{
8624 
    kfree(rdev->bios);
8625 
	rdev->bios = NULL;
8626 
}
8627 
 
8628 
void dce8_program_fmt(struct drm_encoder *encoder)
8629 
{
8630 
	struct drm_device *dev = encoder->dev;
8631 
	struct radeon_device *rdev = dev->dev_private;
8632 
	struct radeon_encoder *radeon_encoder = to_radeon_encoder(encoder);
8633 
	struct radeon_crtc *radeon_crtc = to_radeon_crtc(encoder->crtc);
8634 
	struct drm_connector *connector = radeon_get_connector_for_encoder(encoder);
8635 
	int bpc = 0;
8636 
	u32 tmp = 0;
8637 
	enum radeon_connector_dither dither = RADEON_FMT_DITHER_DISABLE;
8638 
 
8639 
	if (connector) {
8640 
		struct radeon_connector *radeon_connector = to_radeon_connector(connector);
8641 
		bpc = radeon_get_monitor_bpc(connector);
8642 
		dither = radeon_connector->dither;
8643 
	}
8644 
 
8645 
	/* LVDS/eDP FMT is set up by atom */
8646 
	if (radeon_encoder->devices & ATOM_DEVICE_LCD_SUPPORT)
8647 
		return;
8648 
 
8649 
	/* not needed for analog */
8650 
	if ((radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC1) ||
8651 
	    (radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC2))
8652 
		return;
8653 
 
8654 
	if (bpc == 0)
8655 
		return;
8656 
 
8657 
	switch (bpc) {
8658 
	case 6:
8659 
		if (dither == RADEON_FMT_DITHER_ENABLE)
8660 
			/* XXX sort out optimal dither settings */
8661 
			tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
8662 
				FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH(0));
8663 
		else
8664 
			tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH(0));
8665 
		break;
8666 
	case 8:
8667 
		if (dither == RADEON_FMT_DITHER_ENABLE)
8668 
			/* XXX sort out optimal dither settings */
8669 
			tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
8670 
				FMT_RGB_RANDOM_ENABLE |
8671 
				FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH(1));
8672 
		else
8673 
			tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH(1));
8674 
		break;
8675 
	case 10:
8676 
		if (dither == RADEON_FMT_DITHER_ENABLE)
8677 
			/* XXX sort out optimal dither settings */
8678 
			tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
8679 
				FMT_RGB_RANDOM_ENABLE |
8680 
				FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH(2));
8681 
		else
8682 
			tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH(2));
8683 
		break;
8684 
	default:
8685 
		/* not needed */
8686 
		break;
8687 
	}
8688 
 
8689 
	WREG32(FMT_BIT_DEPTH_CONTROL + radeon_crtc->crtc_offset, tmp);
8690 
}
8691 
 
8692 
/* display watermark setup */
8693 
/**
8694 
 * dce8_line_buffer_adjust - Set up the line buffer
8695 
 *
8696 
 * @rdev: radeon_device pointer
8697 
 * @radeon_crtc: the selected display controller
8698 
 * @mode: the current display mode on the selected display
8699 
 * controller
8700 
 *
8701 
 * Setup up the line buffer allocation for
8702 
 * the selected display controller (CIK).
8703 
 * Returns the line buffer size in pixels.
8704 
 */
8705 
static u32 dce8_line_buffer_adjust(struct radeon_device *rdev,
8706 
				   struct radeon_crtc *radeon_crtc,
8707 
				   struct drm_display_mode *mode)
8708 
{
8709 
	u32 tmp, buffer_alloc, i;
8710 
	u32 pipe_offset = radeon_crtc->crtc_id * 0x20;
8711 
	/*
8712 
	 * Line Buffer Setup
8713 
	 * There are 6 line buffers, one for each display controllers.
8714 
	 * There are 3 partitions per LB. Select the number of partitions
8715 
	 * to enable based on the display width.  For display widths larger
8716 
	 * than 4096, you need use to use 2 display controllers and combine
8717 
	 * them using the stereo blender.
8718 
	 */
8719 
	if (radeon_crtc->base.enabled && mode) {
8720 
		if (mode->crtc_hdisplay < 1920) {
8721 
			tmp = 1;
8722 
			buffer_alloc = 2;
8723 
		} else if (mode->crtc_hdisplay < 2560) {
8724 
			tmp = 2;
8725 
			buffer_alloc = 2;
8726 
		} else if (mode->crtc_hdisplay < 4096) {
8727 
			tmp = 0;
8728 
			buffer_alloc = (rdev->flags & RADEON_IS_IGP) ? 2 : 4;
8729 
		} else {
8730 
			DRM_DEBUG_KMS("Mode too big for LB!\n");
8731 
			tmp = 0;
8732 
			buffer_alloc = (rdev->flags & RADEON_IS_IGP) ? 2 : 4;
8733 
		}
8734 
	} else {
8735 
		tmp = 1;
8736 
		buffer_alloc = 0;
8737 
	}
8738 
 
8739 
	WREG32(LB_MEMORY_CTRL + radeon_crtc->crtc_offset,
8740 
	       LB_MEMORY_CONFIG(tmp) | LB_MEMORY_SIZE(0x6B0));
8741 
 
8742 
	WREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset,
8743 
	       DMIF_BUFFERS_ALLOCATED(buffer_alloc));
8744 
	for (i = 0; i < rdev->usec_timeout; i++) {
8745 
		if (RREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset) &
8746 
		    DMIF_BUFFERS_ALLOCATED_COMPLETED)
8747 
			break;
8748 
		udelay(1);
8749 
	}
8750 
 
8751 
	if (radeon_crtc->base.enabled && mode) {
8752 
		switch (tmp) {
8753 
		case 0:
8754 
		default:
8755 
			return 4096 * 2;
8756 
		case 1:
8757 
			return 1920 * 2;
8758 
		case 2:
8759 
			return 2560 * 2;
8760 
		}
8761 
	}
8762 
 
8763 
	/* controller not enabled, so no lb used */
8764 
	return 0;
8765 
}
8766 
 
8767 
/**
8768 
 * cik_get_number_of_dram_channels - get the number of dram channels
8769 
 *
8770 
 * @rdev: radeon_device pointer
8771 
 *
8772 
 * Look up the number of video ram channels (CIK).
8773 
 * Used for display watermark bandwidth calculations
8774 
 * Returns the number of dram channels
8775 
 */
8776 
static u32 cik_get_number_of_dram_channels(struct radeon_device *rdev)
8777 
{
8778 
	u32 tmp = RREG32(MC_SHARED_CHMAP);
8779 
 
8780 
	switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
8781 
	case 0:
8782 
	default:
8783 
		return 1;
8784 
	case 1:
8785 
		return 2;
8786 
	case 2:
8787 
		return 4;
8788 
	case 3:
8789 
		return 8;
8790 
	case 4:
8791 
		return 3;
8792 
	case 5:
8793 
		return 6;
8794 
	case 6:
8795 
		return 10;
8796 
	case 7:
8797 
		return 12;
8798 
	case 8:
8799 
		return 16;
8800 
	}
8801 
}
8802 
 
8803 
struct dce8_wm_params {
8804 
	u32 dram_channels; /* number of dram channels */
8805 
	u32 yclk;          /* bandwidth per dram data pin in kHz */
8806 
	u32 sclk;          /* engine clock in kHz */
8807 
	u32 disp_clk;      /* display clock in kHz */
8808 
	u32 src_width;     /* viewport width */
8809 
	u32 active_time;   /* active display time in ns */
8810 
	u32 blank_time;    /* blank time in ns */
8811 
	bool interlaced;    /* mode is interlaced */
8812 
	fixed20_12 vsc;    /* vertical scale ratio */
8813 
	u32 num_heads;     /* number of active crtcs */
8814 
	u32 bytes_per_pixel; /* bytes per pixel display + overlay */
8815 
	u32 lb_size;       /* line buffer allocated to pipe */
8816 
	u32 vtaps;         /* vertical scaler taps */
8817 
};
8818 
 
8819 
/**
8820 
 * dce8_dram_bandwidth - get the dram bandwidth
8821 
 *
8822 
 * @wm: watermark calculation data
8823 
 *
8824 
 * Calculate the raw dram bandwidth (CIK).
8825 
 * Used for display watermark bandwidth calculations
8826 
 * Returns the dram bandwidth in MBytes/s
8827 
 */
8828 
static u32 dce8_dram_bandwidth(struct dce8_wm_params *wm)
8829 
{
8830 
	/* Calculate raw DRAM Bandwidth */
8831 
	fixed20_12 dram_efficiency; /* 0.7 */
8832 
	fixed20_12 yclk, dram_channels, bandwidth;
8833 
	fixed20_12 a;
8834 
 
8835 
	a.full = dfixed_const(1000);
8836 
	yclk.full = dfixed_const(wm->yclk);
8837 
	yclk.full = dfixed_div(yclk, a);
8838 
	dram_channels.full = dfixed_const(wm->dram_channels * 4);
8839 
	a.full = dfixed_const(10);
8840 
	dram_efficiency.full = dfixed_const(7);
8841 
	dram_efficiency.full = dfixed_div(dram_efficiency, a);
8842 
	bandwidth.full = dfixed_mul(dram_channels, yclk);
8843 
	bandwidth.full = dfixed_mul(bandwidth, dram_efficiency);
8844 
 
8845 
	return dfixed_trunc(bandwidth);
8846 
}
8847 
 
8848 
/**
8849 
 * dce8_dram_bandwidth_for_display - get the dram bandwidth for display
8850 
 *
8851 
 * @wm: watermark calculation data
8852 
 *
8853 
 * Calculate the dram bandwidth used for display (CIK).
8854 
 * Used for display watermark bandwidth calculations
8855 
 * Returns the dram bandwidth for display in MBytes/s
8856 
 */
8857 
static u32 dce8_dram_bandwidth_for_display(struct dce8_wm_params *wm)
8858 
{
8859 
	/* Calculate DRAM Bandwidth and the part allocated to display. */
8860 
	fixed20_12 disp_dram_allocation; /* 0.3 to 0.7 */
8861 
	fixed20_12 yclk, dram_channels, bandwidth;
8862 
	fixed20_12 a;
8863 
 
8864 
	a.full = dfixed_const(1000);
8865 
	yclk.full = dfixed_const(wm->yclk);
8866 
	yclk.full = dfixed_div(yclk, a);
8867 
	dram_channels.full = dfixed_const(wm->dram_channels * 4);
8868 
	a.full = dfixed_const(10);
8869 
	disp_dram_allocation.full = dfixed_const(3); /* XXX worse case value 0.3 */
8870 
	disp_dram_allocation.full = dfixed_div(disp_dram_allocation, a);
8871 
	bandwidth.full = dfixed_mul(dram_channels, yclk);
8872 
	bandwidth.full = dfixed_mul(bandwidth, disp_dram_allocation);
8873 
 
8874 
	return dfixed_trunc(bandwidth);
8875 
}
8876 
 
8877 
/**
8878 
 * dce8_data_return_bandwidth - get the data return bandwidth
8879 
 *
8880 
 * @wm: watermark calculation data
8881 
 *
8882 
 * Calculate the data return bandwidth used for display (CIK).
8883 
 * Used for display watermark bandwidth calculations
8884 
 * Returns the data return bandwidth in MBytes/s
8885 
 */
8886 
static u32 dce8_data_return_bandwidth(struct dce8_wm_params *wm)
8887 
{
8888 
	/* Calculate the display Data return Bandwidth */
8889 
	fixed20_12 return_efficiency; /* 0.8 */
8890 
	fixed20_12 sclk, bandwidth;
8891 
	fixed20_12 a;
8892 
 
8893 
	a.full = dfixed_const(1000);
8894 
	sclk.full = dfixed_const(wm->sclk);
8895 
	sclk.full = dfixed_div(sclk, a);
8896 
	a.full = dfixed_const(10);
8897 
	return_efficiency.full = dfixed_const(8);
8898 
	return_efficiency.full = dfixed_div(return_efficiency, a);
8899 
	a.full = dfixed_const(32);
8900 
	bandwidth.full = dfixed_mul(a, sclk);
8901 
	bandwidth.full = dfixed_mul(bandwidth, return_efficiency);
8902 
 
8903 
	return dfixed_trunc(bandwidth);
8904 
}
8905 
 
8906 
/**
8907 
 * dce8_dmif_request_bandwidth - get the dmif bandwidth
8908 
 *
8909 
 * @wm: watermark calculation data
8910 
 *
8911 
 * Calculate the dmif bandwidth used for display (CIK).
8912 
 * Used for display watermark bandwidth calculations
8913 
 * Returns the dmif bandwidth in MBytes/s
8914 
 */
8915 
static u32 dce8_dmif_request_bandwidth(struct dce8_wm_params *wm)
8916 
{
8917 
	/* Calculate the DMIF Request Bandwidth */
8918 
	fixed20_12 disp_clk_request_efficiency; /* 0.8 */
8919 
	fixed20_12 disp_clk, bandwidth;
8920 
	fixed20_12 a, b;
8921 
 
8922 
	a.full = dfixed_const(1000);
8923 
	disp_clk.full = dfixed_const(wm->disp_clk);
8924 
	disp_clk.full = dfixed_div(disp_clk, a);
8925 
	a.full = dfixed_const(32);
8926 
	b.full = dfixed_mul(a, disp_clk);
8927 
 
8928 
	a.full = dfixed_const(10);
8929 
	disp_clk_request_efficiency.full = dfixed_const(8);
8930 
	disp_clk_request_efficiency.full = dfixed_div(disp_clk_request_efficiency, a);
8931 
 
8932 
	bandwidth.full = dfixed_mul(b, disp_clk_request_efficiency);
8933 
 
8934 
	return dfixed_trunc(bandwidth);
8935 
}
8936 
 
8937 
/**
8938 
 * dce8_available_bandwidth - get the min available bandwidth
8939 
 *
8940 
 * @wm: watermark calculation data
8941 
 *
8942 
 * Calculate the min available bandwidth used for display (CIK).
8943 
 * Used for display watermark bandwidth calculations
8944 
 * Returns the min available bandwidth in MBytes/s
8945 
 */
8946 
static u32 dce8_available_bandwidth(struct dce8_wm_params *wm)
8947 
{
8948 
	/* Calculate the Available bandwidth. Display can use this temporarily but not in average. */
8949 
	u32 dram_bandwidth = dce8_dram_bandwidth(wm);
8950 
	u32 data_return_bandwidth = dce8_data_return_bandwidth(wm);
8951 
	u32 dmif_req_bandwidth = dce8_dmif_request_bandwidth(wm);
8952 
 
8953 
	return min(dram_bandwidth, min(data_return_bandwidth, dmif_req_bandwidth));
8954 
}
8955 
 
8956 
/**
8957 
 * dce8_average_bandwidth - get the average available bandwidth
8958 
 *
8959 
 * @wm: watermark calculation data
8960 
 *
8961 
 * Calculate the average available bandwidth used for display (CIK).
8962 
 * Used for display watermark bandwidth calculations
8963 
 * Returns the average available bandwidth in MBytes/s
8964 
 */
8965 
static u32 dce8_average_bandwidth(struct dce8_wm_params *wm)
8966 
{
8967 
	/* Calculate the display mode Average Bandwidth
8968 
	 * DisplayMode should contain the source and destination dimensions,
8969 
	 * timing, etc.
8970 
	 */
8971 
	fixed20_12 bpp;
8972 
	fixed20_12 line_time;
8973 
	fixed20_12 src_width;
8974 
	fixed20_12 bandwidth;
8975 
	fixed20_12 a;
8976 
 
8977 
	a.full = dfixed_const(1000);
8978 
	line_time.full = dfixed_const(wm->active_time + wm->blank_time);
8979 
	line_time.full = dfixed_div(line_time, a);
8980 
	bpp.full = dfixed_const(wm->bytes_per_pixel);
8981 
	src_width.full = dfixed_const(wm->src_width);
8982 
	bandwidth.full = dfixed_mul(src_width, bpp);
8983 
	bandwidth.full = dfixed_mul(bandwidth, wm->vsc);
8984 
	bandwidth.full = dfixed_div(bandwidth, line_time);
8985 
 
8986 
	return dfixed_trunc(bandwidth);
8987 
}
8988 
 
8989 
/**
8990 
 * dce8_latency_watermark - get the latency watermark
8991 
 *
8992 
 * @wm: watermark calculation data
8993 
 *
8994 
 * Calculate the latency watermark (CIK).
8995 
 * Used for display watermark bandwidth calculations
8996 
 * Returns the latency watermark in ns
8997 
 */
8998 
static u32 dce8_latency_watermark(struct dce8_wm_params *wm)
8999 
{
9000 
	/* First calculate the latency in ns */
9001 
	u32 mc_latency = 2000; /* 2000 ns. */
9002 
	u32 available_bandwidth = dce8_available_bandwidth(wm);
9003 
	u32 worst_chunk_return_time = (512 * 8 * 1000) / available_bandwidth;
9004 
	u32 cursor_line_pair_return_time = (128 * 4 * 1000) / available_bandwidth;
9005 
	u32 dc_latency = 40000000 / wm->disp_clk; /* dc pipe latency */
9006 
	u32 other_heads_data_return_time = ((wm->num_heads + 1) * worst_chunk_return_time) +
9007 
		(wm->num_heads * cursor_line_pair_return_time);
9008 
	u32 latency = mc_latency + other_heads_data_return_time + dc_latency;
9009 
	u32 max_src_lines_per_dst_line, lb_fill_bw, line_fill_time;
9010 
	u32 tmp, dmif_size = 12288;
9011 
	fixed20_12 a, b, c;
9012 
 
9013 
	if (wm->num_heads == 0)
9014 
		return 0;
9015 
 
9016 
	a.full = dfixed_const(2);
9017 
	b.full = dfixed_const(1);
9018 
	if ((wm->vsc.full > a.full) ||
9019 
	    ((wm->vsc.full > b.full) && (wm->vtaps >= 3)) ||
9020 
	    (wm->vtaps >= 5) ||
9021 
	    ((wm->vsc.full >= a.full) && wm->interlaced))
9022 
		max_src_lines_per_dst_line = 4;
9023 
	else
9024 
		max_src_lines_per_dst_line = 2;
9025 
 
9026 
	a.full = dfixed_const(available_bandwidth);
9027 
	b.full = dfixed_const(wm->num_heads);
9028 
	a.full = dfixed_div(a, b);
9029 
 
9030 
	b.full = dfixed_const(mc_latency + 512);
9031 
	c.full = dfixed_const(wm->disp_clk);
9032 
	b.full = dfixed_div(b, c);
9033 
 
9034 
	c.full = dfixed_const(dmif_size);
9035 
	b.full = dfixed_div(c, b);
9036 
 
9037 
	tmp = min(dfixed_trunc(a), dfixed_trunc(b));
9038 
 
9039 
	b.full = dfixed_const(1000);
9040 
	c.full = dfixed_const(wm->disp_clk);
9041 
	b.full = dfixed_div(c, b);
9042 
	c.full = dfixed_const(wm->bytes_per_pixel);
9043 
	b.full = dfixed_mul(b, c);
9044 
 
9045 
	lb_fill_bw = min(tmp, dfixed_trunc(b));
9046 
 
9047 
	a.full = dfixed_const(max_src_lines_per_dst_line * wm->src_width * wm->bytes_per_pixel);
9048 
	b.full = dfixed_const(1000);
9049 
	c.full = dfixed_const(lb_fill_bw);
9050 
	b.full = dfixed_div(c, b);
9051 
	a.full = dfixed_div(a, b);
9052 
	line_fill_time = dfixed_trunc(a);
9053 
 
9054 
	if (line_fill_time < wm->active_time)
9055 
		return latency;
9056 
	else
9057 
		return latency + (line_fill_time - wm->active_time);
9058 
 
9059 
}
9060 
 
9061 
/**
9062 
 * dce8_average_bandwidth_vs_dram_bandwidth_for_display - check
9063 
 * average and available dram bandwidth
9064 
 *
9065 
 * @wm: watermark calculation data
9066 
 *
9067 
 * Check if the display average bandwidth fits in the display
9068 
 * dram bandwidth (CIK).
9069 
 * Used for display watermark bandwidth calculations
9070 
 * Returns true if the display fits, false if not.
9071 
 */
9072 
static bool dce8_average_bandwidth_vs_dram_bandwidth_for_display(struct dce8_wm_params *wm)
9073 
{
9074 
	if (dce8_average_bandwidth(wm) <=
9075 
	    (dce8_dram_bandwidth_for_display(wm) / wm->num_heads))
9076 
		return true;
9077 
	else
9078 
		return false;
9079 
}
9080 
 
9081 
/**
9082 
 * dce8_average_bandwidth_vs_available_bandwidth - check
9083 
 * average and available bandwidth
9084 
 *
9085 
 * @wm: watermark calculation data
9086 
 *
9087 
 * Check if the display average bandwidth fits in the display
9088 
 * available bandwidth (CIK).
9089 
 * Used for display watermark bandwidth calculations
9090 
 * Returns true if the display fits, false if not.
9091 
 */
9092 
static bool dce8_average_bandwidth_vs_available_bandwidth(struct dce8_wm_params *wm)
9093 
{
9094 
	if (dce8_average_bandwidth(wm) <=
9095 
	    (dce8_available_bandwidth(wm) / wm->num_heads))
9096 
		return true;
9097 
	else
9098 
		return false;
9099 
}
9100 
 
9101 
/**
9102 
 * dce8_check_latency_hiding - check latency hiding
9103 
 *
9104 
 * @wm: watermark calculation data
9105 
 *
9106 
 * Check latency hiding (CIK).
9107 
 * Used for display watermark bandwidth calculations
9108 
 * Returns true if the display fits, false if not.
9109 
 */
9110 
static bool dce8_check_latency_hiding(struct dce8_wm_params *wm)
9111 
{
9112 
	u32 lb_partitions = wm->lb_size / wm->src_width;
9113 
	u32 line_time = wm->active_time + wm->blank_time;
9114 
	u32 latency_tolerant_lines;
9115 
	u32 latency_hiding;
9116 
	fixed20_12 a;
9117 
 
9118 
	a.full = dfixed_const(1);
9119 
	if (wm->vsc.full > a.full)
9120 
		latency_tolerant_lines = 1;
9121 
	else {
9122 
		if (lb_partitions <= (wm->vtaps + 1))
9123 
			latency_tolerant_lines = 1;
9124 
		else
9125 
			latency_tolerant_lines = 2;
9126 
	}
9127 
 
9128 
	latency_hiding = (latency_tolerant_lines * line_time + wm->blank_time);
9129 
 
9130 
	if (dce8_latency_watermark(wm) <= latency_hiding)
9131 
		return true;
9132 
	else
9133 
		return false;
9134 
}
9135 
 
9136 
/**
9137 
 * dce8_program_watermarks - program display watermarks
9138 
 *
9139 
 * @rdev: radeon_device pointer
9140 
 * @radeon_crtc: the selected display controller
9141 
 * @lb_size: line buffer size
9142 
 * @num_heads: number of display controllers in use
9143 
 *
9144 
 * Calculate and program the display watermarks for the
9145 
 * selected display controller (CIK).
9146 
 */
9147 
static void dce8_program_watermarks(struct radeon_device *rdev,
9148 
				    struct radeon_crtc *radeon_crtc,
9149 
				    u32 lb_size, u32 num_heads)
9150 
{
9151 
	struct drm_display_mode *mode = &radeon_crtc->base.mode;
9152 
	struct dce8_wm_params wm_low, wm_high;
9153 
	u32 pixel_period;
9154 
	u32 line_time = 0;
9155 
	u32 latency_watermark_a = 0, latency_watermark_b = 0;
9156 
	u32 tmp, wm_mask;
9157 
 
9158 
	if (radeon_crtc->base.enabled && num_heads && mode) {
9159 
		pixel_period = 1000000 / (u32)mode->clock;
9160 
		line_time = min((u32)mode->crtc_htotal * pixel_period, (u32)65535);
9161 
 
9162 
		/* watermark for high clocks */
9163 
		if ((rdev->pm.pm_method == PM_METHOD_DPM) &&
9164 
		    rdev->pm.dpm_enabled) {
9165 
			wm_high.yclk =
9166 
				radeon_dpm_get_mclk(rdev, false) * 10;
9167 
			wm_high.sclk =
9168 
				radeon_dpm_get_sclk(rdev, false) * 10;
9169 
		} else {
9170 
			wm_high.yclk = rdev->pm.current_mclk * 10;
9171 
			wm_high.sclk = rdev->pm.current_sclk * 10;
9172 
		}
9173 
 
9174 
		wm_high.disp_clk = mode->clock;
9175 
		wm_high.src_width = mode->crtc_hdisplay;
9176 
		wm_high.active_time = mode->crtc_hdisplay * pixel_period;
9177 
		wm_high.blank_time = line_time - wm_high.active_time;
9178 
		wm_high.interlaced = false;
9179 
		if (mode->flags & DRM_MODE_FLAG_INTERLACE)
9180 
			wm_high.interlaced = true;
9181 
		wm_high.vsc = radeon_crtc->vsc;
9182 
		wm_high.vtaps = 1;
9183 
		if (radeon_crtc->rmx_type != RMX_OFF)
9184 
			wm_high.vtaps = 2;
9185 
		wm_high.bytes_per_pixel = 4; /* XXX: get this from fb config */
9186 
		wm_high.lb_size = lb_size;
9187 
		wm_high.dram_channels = cik_get_number_of_dram_channels(rdev);
9188 
		wm_high.num_heads = num_heads;
9189 
 
9190 
		/* set for high clocks */
9191 
		latency_watermark_a = min(dce8_latency_watermark(&wm_high), (u32)65535);
9192 
 
9193 
		/* possibly force display priority to high */
9194 
		/* should really do this at mode validation time... */
9195 
		if (!dce8_average_bandwidth_vs_dram_bandwidth_for_display(&wm_high) ||
9196 
		    !dce8_average_bandwidth_vs_available_bandwidth(&wm_high) ||
9197 
		    !dce8_check_latency_hiding(&wm_high) ||
9198 
		    (rdev->disp_priority == 2)) {
9199 
			DRM_DEBUG_KMS("force priority to high\n");
9200 
		}
9201 
 
9202 
		/* watermark for low clocks */
9203 
		if ((rdev->pm.pm_method == PM_METHOD_DPM) &&
9204 
		    rdev->pm.dpm_enabled) {
9205 
			wm_low.yclk =
9206 
				radeon_dpm_get_mclk(rdev, true) * 10;
9207 
			wm_low.sclk =
9208 
				radeon_dpm_get_sclk(rdev, true) * 10;
9209 
		} else {
9210 
			wm_low.yclk = rdev->pm.current_mclk * 10;
9211 
			wm_low.sclk = rdev->pm.current_sclk * 10;
9212 
		}
9213 
 
9214 
		wm_low.disp_clk = mode->clock;
9215 
		wm_low.src_width = mode->crtc_hdisplay;
9216 
		wm_low.active_time = mode->crtc_hdisplay * pixel_period;
9217 
		wm_low.blank_time = line_time - wm_low.active_time;
9218 
		wm_low.interlaced = false;
9219 
		if (mode->flags & DRM_MODE_FLAG_INTERLACE)
9220 
			wm_low.interlaced = true;
9221 
		wm_low.vsc = radeon_crtc->vsc;
9222 
		wm_low.vtaps = 1;
9223 
		if (radeon_crtc->rmx_type != RMX_OFF)
9224 
			wm_low.vtaps = 2;
9225 
		wm_low.bytes_per_pixel = 4; /* XXX: get this from fb config */
9226 
		wm_low.lb_size = lb_size;
9227 
		wm_low.dram_channels = cik_get_number_of_dram_channels(rdev);
9228 
		wm_low.num_heads = num_heads;
9229 
 
9230 
		/* set for low clocks */
9231 
		latency_watermark_b = min(dce8_latency_watermark(&wm_low), (u32)65535);
9232 
 
9233 
		/* possibly force display priority to high */
9234 
		/* should really do this at mode validation time... */
9235 
		if (!dce8_average_bandwidth_vs_dram_bandwidth_for_display(&wm_low) ||
9236 
		    !dce8_average_bandwidth_vs_available_bandwidth(&wm_low) ||
9237 
		    !dce8_check_latency_hiding(&wm_low) ||
9238 
		    (rdev->disp_priority == 2)) {
9239 
			DRM_DEBUG_KMS("force priority to high\n");
9240 
		}
9241 
	}
9242 
 
9243 
	/* select wm A */
9244 
	wm_mask = RREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset);
9245 
	tmp = wm_mask;
9246 
	tmp &= ~LATENCY_WATERMARK_MASK(3);
9247 
	tmp |= LATENCY_WATERMARK_MASK(1);
9248 
	WREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset, tmp);
9249 
	WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset,
9250 
	       (LATENCY_LOW_WATERMARK(latency_watermark_a) |
9251 
		LATENCY_HIGH_WATERMARK(line_time)));
9252 
	/* select wm B */
9253 
	tmp = RREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset);
9254 
	tmp &= ~LATENCY_WATERMARK_MASK(3);
9255 
	tmp |= LATENCY_WATERMARK_MASK(2);
9256 
	WREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset, tmp);
9257 
	WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset,
9258 
	       (LATENCY_LOW_WATERMARK(latency_watermark_b) |
9259 
		LATENCY_HIGH_WATERMARK(line_time)));
9260 
	/* restore original selection */
9261 
	WREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset, wm_mask);
9262 
 
9263 
	/* save values for DPM */
9264 
	radeon_crtc->line_time = line_time;
9265 
	radeon_crtc->wm_high = latency_watermark_a;
9266 
	radeon_crtc->wm_low = latency_watermark_b;
9267 
}
9268 
 
9269 
/**
9270 
 * dce8_bandwidth_update - program display watermarks
9271 
 *
9272 
 * @rdev: radeon_device pointer
9273 
 *
9274 
 * Calculate and program the display watermarks and line
9275 
 * buffer allocation (CIK).
9276 
 */
9277 
void dce8_bandwidth_update(struct radeon_device *rdev)
9278 
{
9279 
	struct drm_display_mode *mode = NULL;
9280 
	u32 num_heads = 0, lb_size;
9281 
	int i;
9282 
 
9283 
	radeon_update_display_priority(rdev);
9284 
 
9285 
	for (i = 0; i < rdev->num_crtc; i++) {
9286 
		if (rdev->mode_info.crtcs[i]->base.enabled)
9287 
			num_heads++;
9288 
	}
9289 
	for (i = 0; i < rdev->num_crtc; i++) {
9290 
		mode = &rdev->mode_info.crtcs[i]->base.mode;
9291 
		lb_size = dce8_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i], mode);
9292 
		dce8_program_watermarks(rdev, rdev->mode_info.crtcs[i], lb_size, num_heads);
9293 
	}
9294 
}
9295 
 
9296 
/**
9297 
 * cik_get_gpu_clock_counter - return GPU clock counter snapshot
9298 
 *
9299 
 * @rdev: radeon_device pointer
9300 
 *
9301 
 * Fetches a GPU clock counter snapshot (SI).
9302 
 * Returns the 64 bit clock counter snapshot.
9303 
 */
9304 
uint64_t cik_get_gpu_clock_counter(struct radeon_device *rdev)
9305 
{
9306 
	uint64_t clock;
9307 
 
9308 
	mutex_lock(&rdev->gpu_clock_mutex);
9309 
	WREG32(RLC_CAPTURE_GPU_CLOCK_COUNT, 1);
9310 
	clock = (uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_LSB) |
9311 
	        ((uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_MSB) << 32ULL);
9312 
	mutex_unlock(&rdev->gpu_clock_mutex);
9313 
	return clock;
9314 
}
9315 
 
9316 
static int cik_set_uvd_clock(struct radeon_device *rdev, u32 clock,
9317 
                              u32 cntl_reg, u32 status_reg)
9318 
{
9319 
	int r, i;
9320 
	struct atom_clock_dividers dividers;
9321 
	uint32_t tmp;
9322 
 
9323 
	r = radeon_atom_get_clock_dividers(rdev, COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK,
9324 
					   clock, false, ÷rs);
9325 
	if (r)
9326 
		return r;
9327 
 
9328 
	tmp = RREG32_SMC(cntl_reg);
9329 
	tmp &= ~(DCLK_DIR_CNTL_EN|DCLK_DIVIDER_MASK);
9330 
	tmp |= dividers.post_divider;
9331 
	WREG32_SMC(cntl_reg, tmp);
9332 
 
9333 
	for (i = 0; i < 100; i++) {
9334 
		if (RREG32_SMC(status_reg) & DCLK_STATUS)
9335 
			break;
9336 
		mdelay(10);
9337 
	}
9338 
	if (i == 100)
9339 
		return -ETIMEDOUT;
9340 
 
9341 
	return 0;
9342 
}
9343 
 
9344 
int cik_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
9345 
{
9346 
	int r = 0;
9347 
 
9348 
	r = cik_set_uvd_clock(rdev, vclk, CG_VCLK_CNTL, CG_VCLK_STATUS);
9349 
	if (r)
9350 
		return r;
9351 
 
9352 
	r = cik_set_uvd_clock(rdev, dclk, CG_DCLK_CNTL, CG_DCLK_STATUS);
9353 
	return r;
9354 
}
9355 
 
9356 
int cik_set_vce_clocks(struct radeon_device *rdev, u32 evclk, u32 ecclk)
9357 
{
9358 
	int r, i;
9359 
	struct atom_clock_dividers dividers;
9360 
	u32 tmp;
9361 
 
9362 
	r = radeon_atom_get_clock_dividers(rdev, COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK,
9363 
					   ecclk, false, ÷rs);
9364 
	if (r)
9365 
		return r;
9366 
 
9367 
	for (i = 0; i < 100; i++) {
9368 
		if (RREG32_SMC(CG_ECLK_STATUS) & ECLK_STATUS)
9369 
			break;
9370 
		mdelay(10);
9371 
	}
9372 
	if (i == 100)
9373 
		return -ETIMEDOUT;
9374 
 
9375 
	tmp = RREG32_SMC(CG_ECLK_CNTL);
9376 
	tmp &= ~(ECLK_DIR_CNTL_EN|ECLK_DIVIDER_MASK);
9377 
	tmp |= dividers.post_divider;
9378 
	WREG32_SMC(CG_ECLK_CNTL, tmp);
9379 
 
9380 
	for (i = 0; i < 100; i++) {
9381 
		if (RREG32_SMC(CG_ECLK_STATUS) & ECLK_STATUS)
9382 
			break;
9383 
		mdelay(10);
9384 
	}
9385 
	if (i == 100)
9386 
		return -ETIMEDOUT;
9387 
 
9388 
	return 0;
9389 
}
9390 
 
9391 
static void cik_pcie_gen3_enable(struct radeon_device *rdev)
9392 
{
9393 
	struct pci_dev *root = rdev->pdev->bus->self;
9394 
	int bridge_pos, gpu_pos;
9395 
	u32 speed_cntl, mask, current_data_rate;
9396 
	int ret, i;
9397 
	u16 tmp16;
9398 
 
9399 
	if (radeon_pcie_gen2 == 0)
9400 
		return;
9401 
 
9402 
	if (rdev->flags & RADEON_IS_IGP)
9403 
		return;
9404 
 
9405 
	if (!(rdev->flags & RADEON_IS_PCIE))
9406 
		return;
9407 
 
9408 
	ret = drm_pcie_get_speed_cap_mask(rdev->ddev, &mask);
9409 
	if (ret != 0)
9410 
		return;
9411 
 
9412 
	if (!(mask & (DRM_PCIE_SPEED_50 | DRM_PCIE_SPEED_80)))
9413 
		return;
9414 
 
9415 
	speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
9416 
	current_data_rate = (speed_cntl & LC_CURRENT_DATA_RATE_MASK) >>
9417 
		LC_CURRENT_DATA_RATE_SHIFT;
9418 
	if (mask & DRM_PCIE_SPEED_80) {
9419 
		if (current_data_rate == 2) {
9420 
			DRM_INFO("PCIE gen 3 link speeds already enabled\n");
9421 
			return;
9422 
		}
9423 
		DRM_INFO("enabling PCIE gen 3 link speeds, disable with radeon.pcie_gen2=0\n");
9424 
	} else if (mask & DRM_PCIE_SPEED_50) {
9425 
		if (current_data_rate == 1) {
9426 
			DRM_INFO("PCIE gen 2 link speeds already enabled\n");
9427 
			return;
9428 
		}
9429 
		DRM_INFO("enabling PCIE gen 2 link speeds, disable with radeon.pcie_gen2=0\n");
9430 
	}
9431 
 
9432 
	bridge_pos = pci_pcie_cap(root);
9433 
	if (!bridge_pos)
9434 
		return;
9435 
 
9436 
	gpu_pos = pci_pcie_cap(rdev->pdev);
9437 
	if (!gpu_pos)
9438 
		return;
9439 
 
9440 
	if (mask & DRM_PCIE_SPEED_80) {
9441 
		/* re-try equalization if gen3 is not already enabled */
9442 
		if (current_data_rate != 2) {
9443 
			u16 bridge_cfg, gpu_cfg;
9444 
			u16 bridge_cfg2, gpu_cfg2;
9445 
			u32 max_lw, current_lw, tmp;
9446 
 
9447 
			pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg);
9448 
			pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg);
9449 
 
9450 
			tmp16 = bridge_cfg | PCI_EXP_LNKCTL_HAWD;
9451 
			pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16);
9452 
 
9453 
			tmp16 = gpu_cfg | PCI_EXP_LNKCTL_HAWD;
9454 
			pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16);
9455 
 
9456 
			tmp = RREG32_PCIE_PORT(PCIE_LC_STATUS1);
9457 
			max_lw = (tmp & LC_DETECTED_LINK_WIDTH_MASK) >> LC_DETECTED_LINK_WIDTH_SHIFT;
9458 
			current_lw = (tmp & LC_OPERATING_LINK_WIDTH_MASK) >> LC_OPERATING_LINK_WIDTH_SHIFT;
9459 
 
9460 
			if (current_lw < max_lw) {
9461 
				tmp = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
9462 
				if (tmp & LC_RENEGOTIATION_SUPPORT) {
9463 
					tmp &= ~(LC_LINK_WIDTH_MASK | LC_UPCONFIGURE_DIS);
9464 
					tmp |= (max_lw << LC_LINK_WIDTH_SHIFT);
9465 
					tmp |= LC_UPCONFIGURE_SUPPORT | LC_RENEGOTIATE_EN | LC_RECONFIG_NOW;
9466 
					WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, tmp);
9467 
				}
9468 
			}
9469 
 
9470 
			for (i = 0; i < 10; i++) {
9471 
				/* check status */
9472 
				pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_DEVSTA, &tmp16);
9473 
				if (tmp16 & PCI_EXP_DEVSTA_TRPND)
9474 
					break;
9475 
 
9476 
				pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg);
9477 
				pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg);
9478 
 
9479 
				pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &bridge_cfg2);
9480 
				pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &gpu_cfg2);
9481 
 
9482 
				tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
9483 
				tmp |= LC_SET_QUIESCE;
9484 
				WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
9485 
 
9486 
				tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
9487 
				tmp |= LC_REDO_EQ;
9488 
				WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
9489 
 
9490 
				mdelay(100);
9491 
 
9492 
				/* linkctl */
9493 
				pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &tmp16);
9494 
				tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
9495 
				tmp16 |= (bridge_cfg & PCI_EXP_LNKCTL_HAWD);
9496 
				pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16);
9497 
 
9498 
				pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &tmp16);
9499 
				tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
9500 
				tmp16 |= (gpu_cfg & PCI_EXP_LNKCTL_HAWD);
9501 
				pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16);
9502 
 
9503 
				/* linkctl2 */
9504 
				pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &tmp16);
9505 
				tmp16 &= ~((1 << 4) | (7 << 9));
9506 
				tmp16 |= (bridge_cfg2 & ((1 << 4) | (7 << 9)));
9507 
				pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, tmp16);
9508 
 
9509 
				pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16);
9510 
				tmp16 &= ~((1 << 4) | (7 << 9));
9511 
				tmp16 |= (gpu_cfg2 & ((1 << 4) | (7 << 9)));
9512 
				pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16);
9513 
 
9514 
				tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
9515 
				tmp &= ~LC_SET_QUIESCE;
9516 
				WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
9517 
			}
9518 
		}
9519 
	}
9520 
 
9521 
	/* set the link speed */
9522 
	speed_cntl |= LC_FORCE_EN_SW_SPEED_CHANGE | LC_FORCE_DIS_HW_SPEED_CHANGE;
9523 
	speed_cntl &= ~LC_FORCE_DIS_SW_SPEED_CHANGE;
9524 
	WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
9525 
 
9526 
	pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16);
9527 
	tmp16 &= ~0xf;
9528 
	if (mask & DRM_PCIE_SPEED_80)
9529 
		tmp16 |= 3; /* gen3 */
9530 
	else if (mask & DRM_PCIE_SPEED_50)
9531 
		tmp16 |= 2; /* gen2 */
9532 
	else
9533 
		tmp16 |= 1; /* gen1 */
9534 
	pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16);
9535 
 
9536 
	speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
9537 
	speed_cntl |= LC_INITIATE_LINK_SPEED_CHANGE;
9538 
	WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
9539 
 
9540 
	for (i = 0; i < rdev->usec_timeout; i++) {
9541 
		speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
9542 
		if ((speed_cntl & LC_INITIATE_LINK_SPEED_CHANGE) == 0)
9543 
			break;
9544 
		udelay(1);
9545 
	}
9546 
}
9547 
 
9548 
static void cik_program_aspm(struct radeon_device *rdev)
9549 
{
9550 
	u32 data, orig;
9551 
	bool disable_l0s = false, disable_l1 = false, disable_plloff_in_l1 = false;
9552 
	bool disable_clkreq = false;
9553 
 
9554 
	if (radeon_aspm == 0)
9555 
		return;
9556 
 
9557 
	/* XXX double check IGPs */
9558 
	if (rdev->flags & RADEON_IS_IGP)
9559 
		return;
9560 
 
9561 
	if (!(rdev->flags & RADEON_IS_PCIE))
9562 
		return;
9563 
 
9564 
	orig = data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL);
9565 
	data &= ~LC_XMIT_N_FTS_MASK;
9566 
	data |= LC_XMIT_N_FTS(0x24) | LC_XMIT_N_FTS_OVERRIDE_EN;
9567 
	if (orig != data)
9568 
		WREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL, data);
9569 
 
9570 
	orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL3);
9571 
	data |= LC_GO_TO_RECOVERY;
9572 
	if (orig != data)
9573 
		WREG32_PCIE_PORT(PCIE_LC_CNTL3, data);
9574 
 
9575 
	orig = data = RREG32_PCIE_PORT(PCIE_P_CNTL);
9576 
	data |= P_IGNORE_EDB_ERR;
9577 
	if (orig != data)
9578 
		WREG32_PCIE_PORT(PCIE_P_CNTL, data);
9579 
 
9580 
	orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL);
9581 
	data &= ~(LC_L0S_INACTIVITY_MASK | LC_L1_INACTIVITY_MASK);
9582 
	data |= LC_PMI_TO_L1_DIS;
9583 
	if (!disable_l0s)
9584 
		data |= LC_L0S_INACTIVITY(7);
9585 
 
9586 
	if (!disable_l1) {
9587 
		data |= LC_L1_INACTIVITY(7);
9588 
		data &= ~LC_PMI_TO_L1_DIS;
9589 
		if (orig != data)
9590 
			WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
9591 
 
9592 
		if (!disable_plloff_in_l1) {
9593 
			bool clk_req_support;
9594 
 
9595 
			orig = data = RREG32_PCIE_PORT(PB0_PIF_PWRDOWN_0);
9596 
			data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
9597 
			data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
9598 
			if (orig != data)
9599 
				WREG32_PCIE_PORT(PB0_PIF_PWRDOWN_0, data);
9600 
 
9601 
			orig = data = RREG32_PCIE_PORT(PB0_PIF_PWRDOWN_1);
9602 
			data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
9603 
			data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
9604 
			if (orig != data)
9605 
				WREG32_PCIE_PORT(PB0_PIF_PWRDOWN_1, data);
9606 
 
9607 
			orig = data = RREG32_PCIE_PORT(PB1_PIF_PWRDOWN_0);
9608 
			data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
9609 
			data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
9610 
			if (orig != data)
9611 
				WREG32_PCIE_PORT(PB1_PIF_PWRDOWN_0, data);
9612 
 
9613 
			orig = data = RREG32_PCIE_PORT(PB1_PIF_PWRDOWN_1);
9614 
			data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
9615 
			data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
9616 
			if (orig != data)
9617 
				WREG32_PCIE_PORT(PB1_PIF_PWRDOWN_1, data);
9618 
 
9619 
			orig = data = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
9620 
			data &= ~LC_DYN_LANES_PWR_STATE_MASK;
9621 
			data |= LC_DYN_LANES_PWR_STATE(3);
9622 
			if (orig != data)
9623 
				WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, data);
9624 
 
9625 
			if (!disable_clkreq) {
9626 
				struct pci_dev *root = rdev->pdev->bus->self;
9627 
				u32 lnkcap;
9628 
 
9629 
				clk_req_support = false;
9630 
				pcie_capability_read_dword(root, PCI_EXP_LNKCAP, &lnkcap);
9631 
				if (lnkcap & PCI_EXP_LNKCAP_CLKPM)
9632 
					clk_req_support = true;
9633 
			} else {
9634 
				clk_req_support = false;
9635 
			}
9636 
 
9637 
			if (clk_req_support) {
9638 
				orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL2);
9639 
				data |= LC_ALLOW_PDWN_IN_L1 | LC_ALLOW_PDWN_IN_L23;
9640 
				if (orig != data)
9641 
					WREG32_PCIE_PORT(PCIE_LC_CNTL2, data);
9642 
 
9643 
				orig = data = RREG32_SMC(THM_CLK_CNTL);
9644 
				data &= ~(CMON_CLK_SEL_MASK | TMON_CLK_SEL_MASK);
9645 
				data |= CMON_CLK_SEL(1) | TMON_CLK_SEL(1);
9646 
				if (orig != data)
9647 
					WREG32_SMC(THM_CLK_CNTL, data);
9648 
 
9649 
				orig = data = RREG32_SMC(MISC_CLK_CTRL);
9650 
				data &= ~(DEEP_SLEEP_CLK_SEL_MASK | ZCLK_SEL_MASK);
9651 
				data |= DEEP_SLEEP_CLK_SEL(1) | ZCLK_SEL(1);
9652 
				if (orig != data)
9653 
					WREG32_SMC(MISC_CLK_CTRL, data);
9654 
 
9655 
				orig = data = RREG32_SMC(CG_CLKPIN_CNTL);
9656 
				data &= ~BCLK_AS_XCLK;
9657 
				if (orig != data)
9658 
					WREG32_SMC(CG_CLKPIN_CNTL, data);
9659 
 
9660 
				orig = data = RREG32_SMC(CG_CLKPIN_CNTL_2);
9661 
				data &= ~FORCE_BIF_REFCLK_EN;
9662 
				if (orig != data)
9663 
					WREG32_SMC(CG_CLKPIN_CNTL_2, data);
9664 
 
9665 
				orig = data = RREG32_SMC(MPLL_BYPASSCLK_SEL);
9666 
				data &= ~MPLL_CLKOUT_SEL_MASK;
9667 
				data |= MPLL_CLKOUT_SEL(4);
9668 
				if (orig != data)
9669 
					WREG32_SMC(MPLL_BYPASSCLK_SEL, data);
9670 
			}
9671 
		}
9672 
	} else {
9673 
		if (orig != data)
9674 
			WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
9675 
	}
9676 
 
9677 
	orig = data = RREG32_PCIE_PORT(PCIE_CNTL2);
9678 
	data |= SLV_MEM_LS_EN | MST_MEM_LS_EN | REPLAY_MEM_LS_EN;
9679 
	if (orig != data)
9680 
		WREG32_PCIE_PORT(PCIE_CNTL2, data);
9681 
 
9682 
	if (!disable_l0s) {
9683 
		data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL);
9684 
		if((data & LC_N_FTS_MASK) == LC_N_FTS_MASK) {
9685 
			data = RREG32_PCIE_PORT(PCIE_LC_STATUS1);
9686 
			if ((data & LC_REVERSE_XMIT) && (data & LC_REVERSE_RCVR)) {
9687 
				orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL);
9688 
				data &= ~LC_L0S_INACTIVITY_MASK;
9689 
				if (orig != data)
9690 
					WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
9691 
			}
9692 
		}
9693 
	}
9694 
}