Rev 4539 | Rev 5060 | Go to most recent revision | Details | Compare with Previous | Last modification | View Log | RSS feed
Rev | Author | Line No. | Line |
---|---|---|---|
2332 | Serge | 1 | /* |
2 | * Copyright © 2008-2010 Intel Corporation |
||
3 | * |
||
4 | * Permission is hereby granted, free of charge, to any person obtaining a |
||
5 | * copy of this software and associated documentation files (the "Software"), |
||
6 | * to deal in the Software without restriction, including without limitation |
||
7 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, |
||
8 | * and/or sell copies of the Software, and to permit persons to whom the |
||
9 | * Software is furnished to do so, subject to the following conditions: |
||
10 | * |
||
11 | * The above copyright notice and this permission notice (including the next |
||
12 | * paragraph) shall be included in all copies or substantial portions of the |
||
13 | * Software. |
||
14 | * |
||
15 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR |
||
16 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, |
||
17 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL |
||
18 | * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER |
||
19 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING |
||
20 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS |
||
21 | * IN THE SOFTWARE. |
||
22 | * |
||
23 | * Authors: |
||
24 | * Eric Anholt |
||
25 | * Zou Nan hai |
||
26 | * Xiang Hai hao |
||
27 | * |
||
28 | */ |
||
29 | |||
3031 | serge | 30 | #include |
2332 | Serge | 31 | #include "i915_drv.h" |
3031 | serge | 32 | #include |
2351 | Serge | 33 | #include "i915_trace.h" |
2332 | Serge | 34 | #include "intel_drv.h" |
35 | |||
36 | static inline int ring_space(struct intel_ring_buffer *ring) |
||
37 | { |
||
3243 | Serge | 38 | int space = (ring->head & HEAD_ADDR) - (ring->tail + I915_RING_FREE_SPACE); |
2332 | Serge | 39 | if (space < 0) |
40 | space += ring->size; |
||
41 | return space; |
||
42 | } |
||
43 | |||
4560 | Serge | 44 | void __intel_ring_advance(struct intel_ring_buffer *ring) |
45 | { |
||
46 | struct drm_i915_private *dev_priv = ring->dev->dev_private; |
||
47 | |||
48 | ring->tail &= ring->size - 1; |
||
49 | if (dev_priv->gpu_error.stop_rings & intel_ring_flag(ring)) |
||
50 | return; |
||
51 | ring->write_tail(ring, ring->tail); |
||
52 | } |
||
53 | |||
3031 | serge | 54 | static int |
55 | gen2_render_ring_flush(struct intel_ring_buffer *ring, |
||
56 | u32 invalidate_domains, |
||
57 | u32 flush_domains) |
||
2332 | Serge | 58 | { |
3031 | serge | 59 | u32 cmd; |
60 | int ret; |
||
2332 | Serge | 61 | |
3031 | serge | 62 | cmd = MI_FLUSH; |
63 | if (((invalidate_domains|flush_domains) & I915_GEM_DOMAIN_RENDER) == 0) |
||
64 | cmd |= MI_NO_WRITE_FLUSH; |
||
2332 | Serge | 65 | |
3031 | serge | 66 | if (invalidate_domains & I915_GEM_DOMAIN_SAMPLER) |
67 | cmd |= MI_READ_FLUSH; |
||
2332 | Serge | 68 | |
3031 | serge | 69 | ret = intel_ring_begin(ring, 2); |
70 | if (ret) |
||
71 | return ret; |
||
72 | |||
73 | intel_ring_emit(ring, cmd); |
||
74 | intel_ring_emit(ring, MI_NOOP); |
||
75 | intel_ring_advance(ring); |
||
76 | |||
77 | return 0; |
||
2332 | Serge | 78 | } |
79 | |||
80 | static int |
||
3031 | serge | 81 | gen4_render_ring_flush(struct intel_ring_buffer *ring, |
2332 | Serge | 82 | u32 invalidate_domains, |
83 | u32 flush_domains) |
||
84 | { |
||
85 | struct drm_device *dev = ring->dev; |
||
86 | u32 cmd; |
||
87 | int ret; |
||
88 | |||
89 | /* |
||
90 | * read/write caches: |
||
91 | * |
||
92 | * I915_GEM_DOMAIN_RENDER is always invalidated, but is |
||
93 | * only flushed if MI_NO_WRITE_FLUSH is unset. On 965, it is |
||
94 | * also flushed at 2d versus 3d pipeline switches. |
||
95 | * |
||
96 | * read-only caches: |
||
97 | * |
||
98 | * I915_GEM_DOMAIN_SAMPLER is flushed on pre-965 if |
||
99 | * MI_READ_FLUSH is set, and is always flushed on 965. |
||
100 | * |
||
101 | * I915_GEM_DOMAIN_COMMAND may not exist? |
||
102 | * |
||
103 | * I915_GEM_DOMAIN_INSTRUCTION, which exists on 965, is |
||
104 | * invalidated when MI_EXE_FLUSH is set. |
||
105 | * |
||
106 | * I915_GEM_DOMAIN_VERTEX, which exists on 965, is |
||
107 | * invalidated with every MI_FLUSH. |
||
108 | * |
||
109 | * TLBs: |
||
110 | * |
||
111 | * On 965, TLBs associated with I915_GEM_DOMAIN_COMMAND |
||
112 | * and I915_GEM_DOMAIN_CPU in are invalidated at PTE write and |
||
113 | * I915_GEM_DOMAIN_RENDER and I915_GEM_DOMAIN_SAMPLER |
||
114 | * are flushed at any MI_FLUSH. |
||
115 | */ |
||
116 | |||
117 | cmd = MI_FLUSH | MI_NO_WRITE_FLUSH; |
||
3031 | serge | 118 | if ((invalidate_domains|flush_domains) & I915_GEM_DOMAIN_RENDER) |
2332 | Serge | 119 | cmd &= ~MI_NO_WRITE_FLUSH; |
120 | if (invalidate_domains & I915_GEM_DOMAIN_INSTRUCTION) |
||
121 | cmd |= MI_EXE_FLUSH; |
||
122 | |||
123 | if (invalidate_domains & I915_GEM_DOMAIN_COMMAND && |
||
124 | (IS_G4X(dev) || IS_GEN5(dev))) |
||
125 | cmd |= MI_INVALIDATE_ISP; |
||
126 | |||
127 | ret = intel_ring_begin(ring, 2); |
||
128 | if (ret) |
||
129 | return ret; |
||
130 | |||
131 | intel_ring_emit(ring, cmd); |
||
132 | intel_ring_emit(ring, MI_NOOP); |
||
133 | intel_ring_advance(ring); |
||
134 | |||
135 | return 0; |
||
136 | } |
||
137 | |||
2342 | Serge | 138 | /** |
139 | * Emits a PIPE_CONTROL with a non-zero post-sync operation, for |
||
140 | * implementing two workarounds on gen6. From section 1.4.7.1 |
||
141 | * "PIPE_CONTROL" of the Sandy Bridge PRM volume 2 part 1: |
||
142 | * |
||
143 | * [DevSNB-C+{W/A}] Before any depth stall flush (including those |
||
144 | * produced by non-pipelined state commands), software needs to first |
||
145 | * send a PIPE_CONTROL with no bits set except Post-Sync Operation != |
||
146 | * 0. |
||
147 | * |
||
148 | * [Dev-SNB{W/A}]: Before a PIPE_CONTROL with Write Cache Flush Enable |
||
149 | * =1, a PIPE_CONTROL with any non-zero post-sync-op is required. |
||
150 | * |
||
151 | * And the workaround for these two requires this workaround first: |
||
152 | * |
||
153 | * [Dev-SNB{W/A}]: Pipe-control with CS-stall bit set must be sent |
||
154 | * BEFORE the pipe-control with a post-sync op and no write-cache |
||
155 | * flushes. |
||
156 | * |
||
157 | * And this last workaround is tricky because of the requirements on |
||
158 | * that bit. From section 1.4.7.2.3 "Stall" of the Sandy Bridge PRM |
||
159 | * volume 2 part 1: |
||
160 | * |
||
161 | * "1 of the following must also be set: |
||
162 | * - Render Target Cache Flush Enable ([12] of DW1) |
||
163 | * - Depth Cache Flush Enable ([0] of DW1) |
||
164 | * - Stall at Pixel Scoreboard ([1] of DW1) |
||
165 | * - Depth Stall ([13] of DW1) |
||
166 | * - Post-Sync Operation ([13] of DW1) |
||
167 | * - Notify Enable ([8] of DW1)" |
||
168 | * |
||
169 | * The cache flushes require the workaround flush that triggered this |
||
170 | * one, so we can't use it. Depth stall would trigger the same. |
||
171 | * Post-sync nonzero is what triggered this second workaround, so we |
||
172 | * can't use that one either. Notify enable is IRQs, which aren't |
||
173 | * really our business. That leaves only stall at scoreboard. |
||
174 | */ |
||
175 | static int |
||
176 | intel_emit_post_sync_nonzero_flush(struct intel_ring_buffer *ring) |
||
177 | { |
||
4104 | Serge | 178 | u32 scratch_addr = ring->scratch.gtt_offset + 128; |
2342 | Serge | 179 | int ret; |
180 | |||
181 | |||
182 | ret = intel_ring_begin(ring, 6); |
||
183 | if (ret) |
||
184 | return ret; |
||
185 | |||
186 | intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(5)); |
||
187 | intel_ring_emit(ring, PIPE_CONTROL_CS_STALL | |
||
188 | PIPE_CONTROL_STALL_AT_SCOREBOARD); |
||
189 | intel_ring_emit(ring, scratch_addr | PIPE_CONTROL_GLOBAL_GTT); /* address */ |
||
190 | intel_ring_emit(ring, 0); /* low dword */ |
||
191 | intel_ring_emit(ring, 0); /* high dword */ |
||
192 | intel_ring_emit(ring, MI_NOOP); |
||
193 | intel_ring_advance(ring); |
||
194 | |||
195 | ret = intel_ring_begin(ring, 6); |
||
196 | if (ret) |
||
197 | return ret; |
||
198 | |||
199 | intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(5)); |
||
200 | intel_ring_emit(ring, PIPE_CONTROL_QW_WRITE); |
||
201 | intel_ring_emit(ring, scratch_addr | PIPE_CONTROL_GLOBAL_GTT); /* address */ |
||
202 | intel_ring_emit(ring, 0); |
||
203 | intel_ring_emit(ring, 0); |
||
204 | intel_ring_emit(ring, MI_NOOP); |
||
205 | intel_ring_advance(ring); |
||
206 | |||
207 | return 0; |
||
208 | } |
||
209 | |||
210 | static int |
||
211 | gen6_render_ring_flush(struct intel_ring_buffer *ring, |
||
212 | u32 invalidate_domains, u32 flush_domains) |
||
213 | { |
||
214 | u32 flags = 0; |
||
4104 | Serge | 215 | u32 scratch_addr = ring->scratch.gtt_offset + 128; |
2342 | Serge | 216 | int ret; |
217 | |||
218 | /* Force SNB workarounds for PIPE_CONTROL flushes */ |
||
3031 | serge | 219 | ret = intel_emit_post_sync_nonzero_flush(ring); |
220 | if (ret) |
||
221 | return ret; |
||
2342 | Serge | 222 | |
223 | /* Just flush everything. Experiments have shown that reducing the |
||
224 | * number of bits based on the write domains has little performance |
||
225 | * impact. |
||
226 | */ |
||
3031 | serge | 227 | if (flush_domains) { |
228 | flags |= PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH; |
||
229 | flags |= PIPE_CONTROL_DEPTH_CACHE_FLUSH; |
||
230 | /* |
||
231 | * Ensure that any following seqno writes only happen |
||
232 | * when the render cache is indeed flushed. |
||
233 | */ |
||
234 | flags |= PIPE_CONTROL_CS_STALL; |
||
235 | } |
||
236 | if (invalidate_domains) { |
||
237 | flags |= PIPE_CONTROL_TLB_INVALIDATE; |
||
238 | flags |= PIPE_CONTROL_INSTRUCTION_CACHE_INVALIDATE; |
||
239 | flags |= PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE; |
||
240 | flags |= PIPE_CONTROL_VF_CACHE_INVALIDATE; |
||
241 | flags |= PIPE_CONTROL_CONST_CACHE_INVALIDATE; |
||
242 | flags |= PIPE_CONTROL_STATE_CACHE_INVALIDATE; |
||
243 | /* |
||
244 | * TLB invalidate requires a post-sync write. |
||
245 | */ |
||
3243 | Serge | 246 | flags |= PIPE_CONTROL_QW_WRITE | PIPE_CONTROL_CS_STALL; |
3031 | serge | 247 | } |
248 | |||
249 | ret = intel_ring_begin(ring, 4); |
||
250 | if (ret) |
||
251 | return ret; |
||
252 | |||
253 | intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(4)); |
||
254 | intel_ring_emit(ring, flags); |
||
255 | intel_ring_emit(ring, scratch_addr | PIPE_CONTROL_GLOBAL_GTT); |
||
256 | intel_ring_emit(ring, 0); |
||
257 | intel_ring_advance(ring); |
||
258 | |||
259 | return 0; |
||
260 | } |
||
261 | |||
262 | static int |
||
263 | gen7_render_ring_cs_stall_wa(struct intel_ring_buffer *ring) |
||
264 | { |
||
265 | int ret; |
||
266 | |||
267 | ret = intel_ring_begin(ring, 4); |
||
268 | if (ret) |
||
269 | return ret; |
||
270 | |||
271 | intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(4)); |
||
272 | intel_ring_emit(ring, PIPE_CONTROL_CS_STALL | |
||
273 | PIPE_CONTROL_STALL_AT_SCOREBOARD); |
||
274 | intel_ring_emit(ring, 0); |
||
275 | intel_ring_emit(ring, 0); |
||
276 | intel_ring_advance(ring); |
||
277 | |||
278 | return 0; |
||
279 | } |
||
280 | |||
4104 | Serge | 281 | static int gen7_ring_fbc_flush(struct intel_ring_buffer *ring, u32 value) |
282 | { |
||
283 | int ret; |
||
284 | |||
285 | if (!ring->fbc_dirty) |
||
286 | return 0; |
||
287 | |||
4560 | Serge | 288 | ret = intel_ring_begin(ring, 6); |
4104 | Serge | 289 | if (ret) |
290 | return ret; |
||
291 | /* WaFbcNukeOn3DBlt:ivb/hsw */ |
||
292 | intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(1)); |
||
293 | intel_ring_emit(ring, MSG_FBC_REND_STATE); |
||
294 | intel_ring_emit(ring, value); |
||
4560 | Serge | 295 | intel_ring_emit(ring, MI_STORE_REGISTER_MEM(1) | MI_SRM_LRM_GLOBAL_GTT); |
296 | intel_ring_emit(ring, MSG_FBC_REND_STATE); |
||
297 | intel_ring_emit(ring, ring->scratch.gtt_offset + 256); |
||
4104 | Serge | 298 | intel_ring_advance(ring); |
299 | |||
300 | ring->fbc_dirty = false; |
||
301 | return 0; |
||
302 | } |
||
303 | |||
3031 | serge | 304 | static int |
305 | gen7_render_ring_flush(struct intel_ring_buffer *ring, |
||
306 | u32 invalidate_domains, u32 flush_domains) |
||
307 | { |
||
308 | u32 flags = 0; |
||
4104 | Serge | 309 | u32 scratch_addr = ring->scratch.gtt_offset + 128; |
3031 | serge | 310 | int ret; |
311 | |||
312 | /* |
||
313 | * Ensure that any following seqno writes only happen when the render |
||
314 | * cache is indeed flushed. |
||
315 | * |
||
316 | * Workaround: 4th PIPE_CONTROL command (except the ones with only |
||
317 | * read-cache invalidate bits set) must have the CS_STALL bit set. We |
||
318 | * don't try to be clever and just set it unconditionally. |
||
319 | */ |
||
320 | flags |= PIPE_CONTROL_CS_STALL; |
||
321 | |||
322 | /* Just flush everything. Experiments have shown that reducing the |
||
323 | * number of bits based on the write domains has little performance |
||
324 | * impact. |
||
325 | */ |
||
326 | if (flush_domains) { |
||
2342 | Serge | 327 | flags |= PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH; |
3031 | serge | 328 | flags |= PIPE_CONTROL_DEPTH_CACHE_FLUSH; |
329 | } |
||
330 | if (invalidate_domains) { |
||
331 | flags |= PIPE_CONTROL_TLB_INVALIDATE; |
||
2342 | Serge | 332 | flags |= PIPE_CONTROL_INSTRUCTION_CACHE_INVALIDATE; |
333 | flags |= PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE; |
||
334 | flags |= PIPE_CONTROL_VF_CACHE_INVALIDATE; |
||
335 | flags |= PIPE_CONTROL_CONST_CACHE_INVALIDATE; |
||
336 | flags |= PIPE_CONTROL_STATE_CACHE_INVALIDATE; |
||
3031 | serge | 337 | /* |
338 | * TLB invalidate requires a post-sync write. |
||
339 | */ |
||
340 | flags |= PIPE_CONTROL_QW_WRITE; |
||
3480 | Serge | 341 | flags |= PIPE_CONTROL_GLOBAL_GTT_IVB; |
2342 | Serge | 342 | |
3031 | serge | 343 | /* Workaround: we must issue a pipe_control with CS-stall bit |
344 | * set before a pipe_control command that has the state cache |
||
345 | * invalidate bit set. */ |
||
346 | gen7_render_ring_cs_stall_wa(ring); |
||
347 | } |
||
348 | |||
349 | ret = intel_ring_begin(ring, 4); |
||
2342 | Serge | 350 | if (ret) |
351 | return ret; |
||
352 | |||
3031 | serge | 353 | intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(4)); |
2342 | Serge | 354 | intel_ring_emit(ring, flags); |
3480 | Serge | 355 | intel_ring_emit(ring, scratch_addr); |
3031 | serge | 356 | intel_ring_emit(ring, 0); |
2342 | Serge | 357 | intel_ring_advance(ring); |
358 | |||
4560 | Serge | 359 | if (!invalidate_domains && flush_domains) |
4104 | Serge | 360 | return gen7_ring_fbc_flush(ring, FBC_REND_NUKE); |
361 | |||
2342 | Serge | 362 | return 0; |
363 | } |
||
364 | |||
4560 | Serge | 365 | static int |
366 | gen8_render_ring_flush(struct intel_ring_buffer *ring, |
||
367 | u32 invalidate_domains, u32 flush_domains) |
||
368 | { |
||
369 | u32 flags = 0; |
||
370 | u32 scratch_addr = ring->scratch.gtt_offset + 128; |
||
371 | int ret; |
||
372 | |||
373 | flags |= PIPE_CONTROL_CS_STALL; |
||
374 | |||
375 | if (flush_domains) { |
||
376 | flags |= PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH; |
||
377 | flags |= PIPE_CONTROL_DEPTH_CACHE_FLUSH; |
||
378 | } |
||
379 | if (invalidate_domains) { |
||
380 | flags |= PIPE_CONTROL_TLB_INVALIDATE; |
||
381 | flags |= PIPE_CONTROL_INSTRUCTION_CACHE_INVALIDATE; |
||
382 | flags |= PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE; |
||
383 | flags |= PIPE_CONTROL_VF_CACHE_INVALIDATE; |
||
384 | flags |= PIPE_CONTROL_CONST_CACHE_INVALIDATE; |
||
385 | flags |= PIPE_CONTROL_STATE_CACHE_INVALIDATE; |
||
386 | flags |= PIPE_CONTROL_QW_WRITE; |
||
387 | flags |= PIPE_CONTROL_GLOBAL_GTT_IVB; |
||
388 | } |
||
389 | |||
390 | ret = intel_ring_begin(ring, 6); |
||
391 | if (ret) |
||
392 | return ret; |
||
393 | |||
394 | intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(6)); |
||
395 | intel_ring_emit(ring, flags); |
||
396 | intel_ring_emit(ring, scratch_addr); |
||
397 | intel_ring_emit(ring, 0); |
||
398 | intel_ring_emit(ring, 0); |
||
399 | intel_ring_emit(ring, 0); |
||
400 | intel_ring_advance(ring); |
||
401 | |||
402 | return 0; |
||
403 | |||
404 | } |
||
405 | |||
2332 | Serge | 406 | static void ring_write_tail(struct intel_ring_buffer *ring, |
407 | u32 value) |
||
408 | { |
||
409 | drm_i915_private_t *dev_priv = ring->dev->dev_private; |
||
410 | I915_WRITE_TAIL(ring, value); |
||
411 | } |
||
412 | |||
413 | u32 intel_ring_get_active_head(struct intel_ring_buffer *ring) |
||
414 | { |
||
415 | drm_i915_private_t *dev_priv = ring->dev->dev_private; |
||
416 | u32 acthd_reg = INTEL_INFO(ring->dev)->gen >= 4 ? |
||
417 | RING_ACTHD(ring->mmio_base) : ACTHD; |
||
418 | |||
419 | return I915_READ(acthd_reg); |
||
420 | } |
||
421 | |||
4104 | Serge | 422 | static void ring_setup_phys_status_page(struct intel_ring_buffer *ring) |
423 | { |
||
424 | struct drm_i915_private *dev_priv = ring->dev->dev_private; |
||
425 | u32 addr; |
||
426 | |||
427 | addr = dev_priv->status_page_dmah->busaddr; |
||
428 | if (INTEL_INFO(ring->dev)->gen >= 4) |
||
429 | addr |= (dev_priv->status_page_dmah->busaddr >> 28) & 0xf0; |
||
430 | I915_WRITE(HWS_PGA, addr); |
||
431 | } |
||
432 | |||
2332 | Serge | 433 | static int init_ring_common(struct intel_ring_buffer *ring) |
434 | { |
||
3031 | serge | 435 | struct drm_device *dev = ring->dev; |
436 | drm_i915_private_t *dev_priv = dev->dev_private; |
||
2332 | Serge | 437 | struct drm_i915_gem_object *obj = ring->obj; |
3031 | serge | 438 | int ret = 0; |
2332 | Serge | 439 | u32 head; |
440 | |||
4560 | Serge | 441 | gen6_gt_force_wake_get(dev_priv, FORCEWAKE_ALL); |
3031 | serge | 442 | |
4104 | Serge | 443 | if (I915_NEED_GFX_HWS(dev)) |
444 | intel_ring_setup_status_page(ring); |
||
445 | else |
||
446 | ring_setup_phys_status_page(ring); |
||
447 | |||
2332 | Serge | 448 | /* Stop the ring if it's running. */ |
449 | I915_WRITE_CTL(ring, 0); |
||
450 | I915_WRITE_HEAD(ring, 0); |
||
451 | ring->write_tail(ring, 0); |
||
452 | |||
453 | head = I915_READ_HEAD(ring) & HEAD_ADDR; |
||
454 | |||
455 | /* G45 ring initialization fails to reset head to zero */ |
||
456 | if (head != 0) { |
||
457 | DRM_DEBUG_KMS("%s head not reset to zero " |
||
458 | "ctl %08x head %08x tail %08x start %08x\n", |
||
459 | ring->name, |
||
460 | I915_READ_CTL(ring), |
||
461 | I915_READ_HEAD(ring), |
||
462 | I915_READ_TAIL(ring), |
||
463 | I915_READ_START(ring)); |
||
464 | |||
465 | I915_WRITE_HEAD(ring, 0); |
||
466 | |||
467 | if (I915_READ_HEAD(ring) & HEAD_ADDR) { |
||
468 | DRM_ERROR("failed to set %s head to zero " |
||
469 | "ctl %08x head %08x tail %08x start %08x\n", |
||
470 | ring->name, |
||
471 | I915_READ_CTL(ring), |
||
472 | I915_READ_HEAD(ring), |
||
473 | I915_READ_TAIL(ring), |
||
474 | I915_READ_START(ring)); |
||
475 | } |
||
476 | } |
||
477 | |||
3031 | serge | 478 | /* Initialize the ring. This must happen _after_ we've cleared the ring |
479 | * registers with the above sequence (the readback of the HEAD registers |
||
480 | * also enforces ordering), otherwise the hw might lose the new ring |
||
481 | * register values. */ |
||
4104 | Serge | 482 | I915_WRITE_START(ring, i915_gem_obj_ggtt_offset(obj)); |
2332 | Serge | 483 | I915_WRITE_CTL(ring, |
484 | ((ring->size - PAGE_SIZE) & RING_NR_PAGES) |
||
3031 | serge | 485 | | RING_VALID); |
2332 | Serge | 486 | |
487 | /* If the head is still not zero, the ring is dead */ |
||
3031 | serge | 488 | if (wait_for((I915_READ_CTL(ring) & RING_VALID) != 0 && |
4104 | Serge | 489 | I915_READ_START(ring) == i915_gem_obj_ggtt_offset(obj) && |
3031 | serge | 490 | (I915_READ_HEAD(ring) & HEAD_ADDR) == 0, 50)) { |
2332 | Serge | 491 | DRM_ERROR("%s initialization failed " |
492 | "ctl %08x head %08x tail %08x start %08x\n", |
||
493 | ring->name, |
||
494 | I915_READ_CTL(ring), |
||
495 | I915_READ_HEAD(ring), |
||
496 | I915_READ_TAIL(ring), |
||
497 | I915_READ_START(ring)); |
||
3031 | serge | 498 | ret = -EIO; |
499 | goto out; |
||
2332 | Serge | 500 | } |
501 | |||
3031 | serge | 502 | ring->head = I915_READ_HEAD(ring); |
503 | ring->tail = I915_READ_TAIL(ring) & TAIL_ADDR; |
||
504 | ring->space = ring_space(ring); |
||
505 | ring->last_retired_head = -1; |
||
2332 | Serge | 506 | |
4104 | Serge | 507 | memset(&ring->hangcheck, 0, sizeof(ring->hangcheck)); |
508 | |||
3031 | serge | 509 | out: |
4560 | Serge | 510 | gen6_gt_force_wake_put(dev_priv, FORCEWAKE_ALL); |
2332 | Serge | 511 | |
3031 | serge | 512 | return ret; |
2332 | Serge | 513 | } |
514 | |||
515 | static int |
||
516 | init_pipe_control(struct intel_ring_buffer *ring) |
||
517 | { |
||
518 | int ret; |
||
519 | |||
4104 | Serge | 520 | if (ring->scratch.obj) |
2332 | Serge | 521 | return 0; |
522 | |||
4104 | Serge | 523 | ring->scratch.obj = i915_gem_alloc_object(ring->dev, 4096); |
524 | if (ring->scratch.obj == NULL) { |
||
2332 | Serge | 525 | DRM_ERROR("Failed to allocate seqno page\n"); |
526 | ret = -ENOMEM; |
||
527 | goto err; |
||
528 | } |
||
529 | |||
4104 | Serge | 530 | i915_gem_object_set_cache_level(ring->scratch.obj, I915_CACHE_LLC); |
2332 | Serge | 531 | |
4104 | Serge | 532 | ret = i915_gem_obj_ggtt_pin(ring->scratch.obj, 4096, true, false); |
2332 | Serge | 533 | if (ret) |
534 | goto err_unref; |
||
535 | |||
4104 | Serge | 536 | ring->scratch.gtt_offset = i915_gem_obj_ggtt_offset(ring->scratch.obj); |
4539 | Serge | 537 | ring->scratch.cpu_page = (void*)MapIoMem((addr_t)sg_page(ring->scratch.obj->pages->sgl),4096, PG_SW|0x100); |
4104 | Serge | 538 | if (ring->scratch.cpu_page == NULL) { |
539 | ret = -ENOMEM; |
||
2332 | Serge | 540 | goto err_unpin; |
4104 | Serge | 541 | } |
2332 | Serge | 542 | |
3480 | Serge | 543 | DRM_DEBUG_DRIVER("%s pipe control offset: 0x%08x\n", |
4104 | Serge | 544 | ring->name, ring->scratch.gtt_offset); |
2332 | Serge | 545 | return 0; |
546 | |||
547 | err_unpin: |
||
4104 | Serge | 548 | i915_gem_object_unpin(ring->scratch.obj); |
2332 | Serge | 549 | err_unref: |
4104 | Serge | 550 | drm_gem_object_unreference(&ring->scratch.obj->base); |
2332 | Serge | 551 | err: |
552 | return ret; |
||
553 | } |
||
554 | |||
555 | static int init_render_ring(struct intel_ring_buffer *ring) |
||
556 | { |
||
557 | struct drm_device *dev = ring->dev; |
||
558 | struct drm_i915_private *dev_priv = dev->dev_private; |
||
559 | int ret = init_ring_common(ring); |
||
560 | |||
3243 | Serge | 561 | if (INTEL_INFO(dev)->gen > 3) |
3031 | serge | 562 | I915_WRITE(MI_MODE, _MASKED_BIT_ENABLE(VS_TIMER_DISPATCH)); |
3243 | Serge | 563 | |
564 | /* We need to disable the AsyncFlip performance optimisations in order |
||
565 | * to use MI_WAIT_FOR_EVENT within the CS. It should already be |
||
566 | * programmed to '1' on all products. |
||
4104 | Serge | 567 | * |
568 | * WaDisableAsyncFlipPerfMode:snb,ivb,hsw,vlv |
||
3243 | Serge | 569 | */ |
570 | if (INTEL_INFO(dev)->gen >= 6) |
||
571 | I915_WRITE(MI_MODE, _MASKED_BIT_ENABLE(ASYNC_FLIP_PERF_DISABLE)); |
||
572 | |||
573 | /* Required for the hardware to program scanline values for waiting */ |
||
574 | if (INTEL_INFO(dev)->gen == 6) |
||
575 | I915_WRITE(GFX_MODE, |
||
576 | _MASKED_BIT_ENABLE(GFX_TLB_INVALIDATE_ALWAYS)); |
||
577 | |||
2332 | Serge | 578 | if (IS_GEN7(dev)) |
579 | I915_WRITE(GFX_MODE_GEN7, |
||
3031 | serge | 580 | _MASKED_BIT_DISABLE(GFX_TLB_INVALIDATE_ALWAYS) | |
581 | _MASKED_BIT_ENABLE(GFX_REPLAY_MODE)); |
||
2332 | Serge | 582 | |
2342 | Serge | 583 | if (INTEL_INFO(dev)->gen >= 5) { |
2339 | Serge | 584 | ret = init_pipe_control(ring); |
2332 | Serge | 585 | if (ret) |
586 | return ret; |
||
587 | } |
||
588 | |||
3031 | serge | 589 | if (IS_GEN6(dev)) { |
590 | /* From the Sandybridge PRM, volume 1 part 3, page 24: |
||
591 | * "If this bit is set, STCunit will have LRA as replacement |
||
592 | * policy. [...] This bit must be reset. LRA replacement |
||
593 | * policy is not supported." |
||
594 | */ |
||
595 | I915_WRITE(CACHE_MODE_0, |
||
596 | _MASKED_BIT_DISABLE(CM0_STC_EVICT_DISABLE_LRA_SNB)); |
||
597 | |||
598 | /* This is not explicitly set for GEN6, so read the register. |
||
599 | * see intel_ring_mi_set_context() for why we care. |
||
600 | * TODO: consider explicitly setting the bit for GEN5 |
||
601 | */ |
||
602 | ring->itlb_before_ctx_switch = |
||
603 | !!(I915_READ(GFX_MODE) & GFX_TLB_INVALIDATE_ALWAYS); |
||
2342 | Serge | 604 | } |
605 | |||
3031 | serge | 606 | if (INTEL_INFO(dev)->gen >= 6) |
607 | I915_WRITE(INSTPM, _MASKED_BIT_ENABLE(INSTPM_FORCE_ORDERING)); |
||
608 | |||
4560 | Serge | 609 | if (HAS_L3_DPF(dev)) |
610 | I915_WRITE_IMR(ring, ~GT_PARITY_ERROR(dev)); |
||
3031 | serge | 611 | |
2332 | Serge | 612 | return ret; |
613 | } |
||
614 | |||
615 | static void render_ring_cleanup(struct intel_ring_buffer *ring) |
||
616 | { |
||
3480 | Serge | 617 | struct drm_device *dev = ring->dev; |
618 | |||
4104 | Serge | 619 | if (ring->scratch.obj == NULL) |
2332 | Serge | 620 | return; |
621 | |||
4104 | Serge | 622 | if (INTEL_INFO(dev)->gen >= 5) { |
623 | // kunmap(sg_page(ring->scratch.obj->pages->sgl)); |
||
624 | i915_gem_object_unpin(ring->scratch.obj); |
||
625 | } |
||
626 | |||
627 | drm_gem_object_unreference(&ring->scratch.obj->base); |
||
628 | ring->scratch.obj = NULL; |
||
2332 | Serge | 629 | } |
630 | |||
631 | static void |
||
2342 | Serge | 632 | update_mboxes(struct intel_ring_buffer *ring, |
633 | u32 mmio_offset) |
||
2332 | Serge | 634 | { |
4104 | Serge | 635 | /* NB: In order to be able to do semaphore MBOX updates for varying number |
636 | * of rings, it's easiest if we round up each individual update to a |
||
637 | * multiple of 2 (since ring updates must always be a multiple of 2) |
||
638 | * even though the actual update only requires 3 dwords. |
||
639 | */ |
||
640 | #define MBOX_UPDATE_DWORDS 4 |
||
3243 | Serge | 641 | intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(1)); |
2342 | Serge | 642 | intel_ring_emit(ring, mmio_offset); |
4560 | Serge | 643 | intel_ring_emit(ring, ring->outstanding_lazy_seqno); |
4104 | Serge | 644 | intel_ring_emit(ring, MI_NOOP); |
2332 | Serge | 645 | } |
646 | |||
2342 | Serge | 647 | /** |
648 | * gen6_add_request - Update the semaphore mailbox registers |
||
649 | * |
||
650 | * @ring - ring that is adding a request |
||
651 | * @seqno - return seqno stuck into the ring |
||
652 | * |
||
653 | * Update the mailbox registers in the *other* rings with the current seqno. |
||
654 | * This acts like a signal in the canonical semaphore. |
||
655 | */ |
||
2332 | Serge | 656 | static int |
3243 | Serge | 657 | gen6_add_request(struct intel_ring_buffer *ring) |
2332 | Serge | 658 | { |
4104 | Serge | 659 | struct drm_device *dev = ring->dev; |
660 | struct drm_i915_private *dev_priv = dev->dev_private; |
||
661 | struct intel_ring_buffer *useless; |
||
4560 | Serge | 662 | int i, ret, num_dwords = 4; |
2332 | Serge | 663 | |
4560 | Serge | 664 | if (i915_semaphore_is_enabled(dev)) |
665 | num_dwords += ((I915_NUM_RINGS-1) * MBOX_UPDATE_DWORDS); |
||
666 | #undef MBOX_UPDATE_DWORDS |
||
667 | |||
668 | ret = intel_ring_begin(ring, num_dwords); |
||
2332 | Serge | 669 | if (ret) |
670 | return ret; |
||
671 | |||
4560 | Serge | 672 | if (i915_semaphore_is_enabled(dev)) { |
4104 | Serge | 673 | for_each_ring(useless, dev_priv, i) { |
674 | u32 mbox_reg = ring->signal_mbox[i]; |
||
675 | if (mbox_reg != GEN6_NOSYNC) |
||
676 | update_mboxes(ring, mbox_reg); |
||
677 | } |
||
4560 | Serge | 678 | } |
2332 | Serge | 679 | |
680 | intel_ring_emit(ring, MI_STORE_DWORD_INDEX); |
||
681 | intel_ring_emit(ring, I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT); |
||
4560 | Serge | 682 | intel_ring_emit(ring, ring->outstanding_lazy_seqno); |
2332 | Serge | 683 | intel_ring_emit(ring, MI_USER_INTERRUPT); |
4560 | Serge | 684 | __intel_ring_advance(ring); |
2332 | Serge | 685 | |
686 | return 0; |
||
687 | } |
||
688 | |||
3480 | Serge | 689 | static inline bool i915_gem_has_seqno_wrapped(struct drm_device *dev, |
690 | u32 seqno) |
||
691 | { |
||
692 | struct drm_i915_private *dev_priv = dev->dev_private; |
||
693 | return dev_priv->last_seqno < seqno; |
||
694 | } |
||
695 | |||
2342 | Serge | 696 | /** |
697 | * intel_ring_sync - sync the waiter to the signaller on seqno |
||
698 | * |
||
699 | * @waiter - ring that is waiting |
||
700 | * @signaller - ring which has, or will signal |
||
701 | * @seqno - seqno which the waiter will block on |
||
702 | */ |
||
703 | static int |
||
3031 | serge | 704 | gen6_ring_sync(struct intel_ring_buffer *waiter, |
2342 | Serge | 705 | struct intel_ring_buffer *signaller, |
2332 | Serge | 706 | u32 seqno) |
707 | { |
||
708 | int ret; |
||
2342 | Serge | 709 | u32 dw1 = MI_SEMAPHORE_MBOX | |
710 | MI_SEMAPHORE_COMPARE | |
||
711 | MI_SEMAPHORE_REGISTER; |
||
2332 | Serge | 712 | |
3031 | serge | 713 | /* Throughout all of the GEM code, seqno passed implies our current |
714 | * seqno is >= the last seqno executed. However for hardware the |
||
715 | * comparison is strictly greater than. |
||
716 | */ |
||
717 | seqno -= 1; |
||
718 | |||
719 | WARN_ON(signaller->semaphore_register[waiter->id] == |
||
720 | MI_SEMAPHORE_SYNC_INVALID); |
||
721 | |||
2342 | Serge | 722 | ret = intel_ring_begin(waiter, 4); |
2332 | Serge | 723 | if (ret) |
724 | return ret; |
||
725 | |||
3480 | Serge | 726 | /* If seqno wrap happened, omit the wait with no-ops */ |
727 | if (likely(!i915_gem_has_seqno_wrapped(waiter->dev, seqno))) { |
||
3031 | serge | 728 | intel_ring_emit(waiter, |
3480 | Serge | 729 | dw1 | |
730 | signaller->semaphore_register[waiter->id]); |
||
2342 | Serge | 731 | intel_ring_emit(waiter, seqno); |
732 | intel_ring_emit(waiter, 0); |
||
733 | intel_ring_emit(waiter, MI_NOOP); |
||
3480 | Serge | 734 | } else { |
735 | intel_ring_emit(waiter, MI_NOOP); |
||
736 | intel_ring_emit(waiter, MI_NOOP); |
||
737 | intel_ring_emit(waiter, MI_NOOP); |
||
738 | intel_ring_emit(waiter, MI_NOOP); |
||
739 | } |
||
2342 | Serge | 740 | intel_ring_advance(waiter); |
2332 | Serge | 741 | |
742 | return 0; |
||
743 | } |
||
744 | |||
745 | #define PIPE_CONTROL_FLUSH(ring__, addr__) \ |
||
746 | do { \ |
||
2342 | Serge | 747 | intel_ring_emit(ring__, GFX_OP_PIPE_CONTROL(4) | PIPE_CONTROL_QW_WRITE | \ |
748 | PIPE_CONTROL_DEPTH_STALL); \ |
||
2332 | Serge | 749 | intel_ring_emit(ring__, (addr__) | PIPE_CONTROL_GLOBAL_GTT); \ |
750 | intel_ring_emit(ring__, 0); \ |
||
751 | intel_ring_emit(ring__, 0); \ |
||
752 | } while (0) |
||
753 | |||
754 | static int |
||
3243 | Serge | 755 | pc_render_add_request(struct intel_ring_buffer *ring) |
2332 | Serge | 756 | { |
4104 | Serge | 757 | u32 scratch_addr = ring->scratch.gtt_offset + 128; |
2332 | Serge | 758 | int ret; |
759 | |||
760 | /* For Ironlake, MI_USER_INTERRUPT was deprecated and apparently |
||
761 | * incoherent with writes to memory, i.e. completely fubar, |
||
762 | * so we need to use PIPE_NOTIFY instead. |
||
763 | * |
||
764 | * However, we also need to workaround the qword write |
||
765 | * incoherence by flushing the 6 PIPE_NOTIFY buffers out to |
||
766 | * memory before requesting an interrupt. |
||
767 | */ |
||
768 | ret = intel_ring_begin(ring, 32); |
||
769 | if (ret) |
||
770 | return ret; |
||
771 | |||
2342 | Serge | 772 | intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(4) | PIPE_CONTROL_QW_WRITE | |
773 | PIPE_CONTROL_WRITE_FLUSH | |
||
774 | PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE); |
||
4104 | Serge | 775 | intel_ring_emit(ring, ring->scratch.gtt_offset | PIPE_CONTROL_GLOBAL_GTT); |
4560 | Serge | 776 | intel_ring_emit(ring, ring->outstanding_lazy_seqno); |
2332 | Serge | 777 | intel_ring_emit(ring, 0); |
778 | PIPE_CONTROL_FLUSH(ring, scratch_addr); |
||
779 | scratch_addr += 128; /* write to separate cachelines */ |
||
780 | PIPE_CONTROL_FLUSH(ring, scratch_addr); |
||
781 | scratch_addr += 128; |
||
782 | PIPE_CONTROL_FLUSH(ring, scratch_addr); |
||
783 | scratch_addr += 128; |
||
784 | PIPE_CONTROL_FLUSH(ring, scratch_addr); |
||
785 | scratch_addr += 128; |
||
786 | PIPE_CONTROL_FLUSH(ring, scratch_addr); |
||
787 | scratch_addr += 128; |
||
788 | PIPE_CONTROL_FLUSH(ring, scratch_addr); |
||
3031 | serge | 789 | |
2342 | Serge | 790 | intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(4) | PIPE_CONTROL_QW_WRITE | |
791 | PIPE_CONTROL_WRITE_FLUSH | |
||
792 | PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE | |
||
2332 | Serge | 793 | PIPE_CONTROL_NOTIFY); |
4104 | Serge | 794 | intel_ring_emit(ring, ring->scratch.gtt_offset | PIPE_CONTROL_GLOBAL_GTT); |
4560 | Serge | 795 | intel_ring_emit(ring, ring->outstanding_lazy_seqno); |
2332 | Serge | 796 | intel_ring_emit(ring, 0); |
4560 | Serge | 797 | __intel_ring_advance(ring); |
2332 | Serge | 798 | |
799 | return 0; |
||
800 | } |
||
801 | |||
802 | static u32 |
||
3031 | serge | 803 | gen6_ring_get_seqno(struct intel_ring_buffer *ring, bool lazy_coherency) |
2342 | Serge | 804 | { |
805 | /* Workaround to force correct ordering between irq and seqno writes on |
||
806 | * ivb (and maybe also on snb) by reading from a CS register (like |
||
807 | * ACTHD) before reading the status page. */ |
||
3031 | serge | 808 | if (!lazy_coherency) |
2342 | Serge | 809 | intel_ring_get_active_head(ring); |
810 | return intel_read_status_page(ring, I915_GEM_HWS_INDEX); |
||
811 | } |
||
812 | |||
813 | static u32 |
||
3031 | serge | 814 | ring_get_seqno(struct intel_ring_buffer *ring, bool lazy_coherency) |
2332 | Serge | 815 | { |
816 | return intel_read_status_page(ring, I915_GEM_HWS_INDEX); |
||
817 | } |
||
818 | |||
3480 | Serge | 819 | static void |
820 | ring_set_seqno(struct intel_ring_buffer *ring, u32 seqno) |
||
821 | { |
||
822 | intel_write_status_page(ring, I915_GEM_HWS_INDEX, seqno); |
||
823 | } |
||
824 | |||
2332 | Serge | 825 | static u32 |
3031 | serge | 826 | pc_render_get_seqno(struct intel_ring_buffer *ring, bool lazy_coherency) |
2332 | Serge | 827 | { |
4104 | Serge | 828 | return ring->scratch.cpu_page[0]; |
2332 | Serge | 829 | } |
830 | |||
3480 | Serge | 831 | static void |
832 | pc_render_set_seqno(struct intel_ring_buffer *ring, u32 seqno) |
||
833 | { |
||
4104 | Serge | 834 | ring->scratch.cpu_page[0] = seqno; |
3480 | Serge | 835 | } |
836 | |||
3031 | serge | 837 | static bool |
838 | gen5_ring_get_irq(struct intel_ring_buffer *ring) |
||
2332 | Serge | 839 | { |
3031 | serge | 840 | struct drm_device *dev = ring->dev; |
841 | drm_i915_private_t *dev_priv = dev->dev_private; |
||
842 | unsigned long flags; |
||
843 | |||
844 | if (!dev->irq_enabled) |
||
845 | return false; |
||
846 | |||
847 | spin_lock_irqsave(&dev_priv->irq_lock, flags); |
||
4104 | Serge | 848 | if (ring->irq_refcount++ == 0) |
849 | ilk_enable_gt_irq(dev_priv, ring->irq_enable_mask); |
||
3031 | serge | 850 | spin_unlock_irqrestore(&dev_priv->irq_lock, flags); |
851 | |||
852 | return true; |
||
2332 | Serge | 853 | } |
854 | |||
855 | static void |
||
3031 | serge | 856 | gen5_ring_put_irq(struct intel_ring_buffer *ring) |
2332 | Serge | 857 | { |
3031 | serge | 858 | struct drm_device *dev = ring->dev; |
859 | drm_i915_private_t *dev_priv = dev->dev_private; |
||
860 | unsigned long flags; |
||
861 | |||
862 | spin_lock_irqsave(&dev_priv->irq_lock, flags); |
||
4104 | Serge | 863 | if (--ring->irq_refcount == 0) |
864 | ilk_disable_gt_irq(dev_priv, ring->irq_enable_mask); |
||
3031 | serge | 865 | spin_unlock_irqrestore(&dev_priv->irq_lock, flags); |
2332 | Serge | 866 | } |
867 | |||
3031 | serge | 868 | static bool |
869 | i9xx_ring_get_irq(struct intel_ring_buffer *ring) |
||
2332 | Serge | 870 | { |
3031 | serge | 871 | struct drm_device *dev = ring->dev; |
872 | drm_i915_private_t *dev_priv = dev->dev_private; |
||
873 | unsigned long flags; |
||
874 | |||
875 | if (!dev->irq_enabled) |
||
876 | return false; |
||
877 | |||
878 | spin_lock_irqsave(&dev_priv->irq_lock, flags); |
||
879 | if (ring->irq_refcount++ == 0) { |
||
880 | dev_priv->irq_mask &= ~ring->irq_enable_mask; |
||
2332 | Serge | 881 | I915_WRITE(IMR, dev_priv->irq_mask); |
882 | POSTING_READ(IMR); |
||
3031 | serge | 883 | } |
884 | spin_unlock_irqrestore(&dev_priv->irq_lock, flags); |
||
885 | |||
886 | return true; |
||
2332 | Serge | 887 | } |
888 | |||
889 | static void |
||
3031 | serge | 890 | i9xx_ring_put_irq(struct intel_ring_buffer *ring) |
2332 | Serge | 891 | { |
3031 | serge | 892 | struct drm_device *dev = ring->dev; |
893 | drm_i915_private_t *dev_priv = dev->dev_private; |
||
894 | unsigned long flags; |
||
895 | |||
896 | spin_lock_irqsave(&dev_priv->irq_lock, flags); |
||
897 | if (--ring->irq_refcount == 0) { |
||
898 | dev_priv->irq_mask |= ring->irq_enable_mask; |
||
2332 | Serge | 899 | I915_WRITE(IMR, dev_priv->irq_mask); |
900 | POSTING_READ(IMR); |
||
3031 | serge | 901 | } |
902 | spin_unlock_irqrestore(&dev_priv->irq_lock, flags); |
||
2332 | Serge | 903 | } |
904 | |||
905 | static bool |
||
3031 | serge | 906 | i8xx_ring_get_irq(struct intel_ring_buffer *ring) |
2332 | Serge | 907 | { |
908 | struct drm_device *dev = ring->dev; |
||
909 | drm_i915_private_t *dev_priv = dev->dev_private; |
||
3031 | serge | 910 | unsigned long flags; |
2332 | Serge | 911 | |
912 | if (!dev->irq_enabled) |
||
913 | return false; |
||
914 | |||
3031 | serge | 915 | spin_lock_irqsave(&dev_priv->irq_lock, flags); |
2332 | Serge | 916 | if (ring->irq_refcount++ == 0) { |
3031 | serge | 917 | dev_priv->irq_mask &= ~ring->irq_enable_mask; |
918 | I915_WRITE16(IMR, dev_priv->irq_mask); |
||
919 | POSTING_READ16(IMR); |
||
2332 | Serge | 920 | } |
3031 | serge | 921 | spin_unlock_irqrestore(&dev_priv->irq_lock, flags); |
2332 | Serge | 922 | |
923 | return true; |
||
924 | } |
||
925 | |||
926 | static void |
||
3031 | serge | 927 | i8xx_ring_put_irq(struct intel_ring_buffer *ring) |
2332 | Serge | 928 | { |
929 | struct drm_device *dev = ring->dev; |
||
930 | drm_i915_private_t *dev_priv = dev->dev_private; |
||
3031 | serge | 931 | unsigned long flags; |
2332 | Serge | 932 | |
3031 | serge | 933 | spin_lock_irqsave(&dev_priv->irq_lock, flags); |
2332 | Serge | 934 | if (--ring->irq_refcount == 0) { |
3031 | serge | 935 | dev_priv->irq_mask |= ring->irq_enable_mask; |
936 | I915_WRITE16(IMR, dev_priv->irq_mask); |
||
937 | POSTING_READ16(IMR); |
||
2332 | Serge | 938 | } |
3031 | serge | 939 | spin_unlock_irqrestore(&dev_priv->irq_lock, flags); |
2332 | Serge | 940 | } |
941 | |||
942 | void intel_ring_setup_status_page(struct intel_ring_buffer *ring) |
||
943 | { |
||
944 | struct drm_device *dev = ring->dev; |
||
945 | drm_i915_private_t *dev_priv = ring->dev->dev_private; |
||
946 | u32 mmio = 0; |
||
947 | |||
948 | /* The ring status page addresses are no longer next to the rest of |
||
949 | * the ring registers as of gen7. |
||
950 | */ |
||
951 | if (IS_GEN7(dev)) { |
||
952 | switch (ring->id) { |
||
3031 | serge | 953 | case RCS: |
2332 | Serge | 954 | mmio = RENDER_HWS_PGA_GEN7; |
955 | break; |
||
3031 | serge | 956 | case BCS: |
2332 | Serge | 957 | mmio = BLT_HWS_PGA_GEN7; |
958 | break; |
||
3031 | serge | 959 | case VCS: |
2332 | Serge | 960 | mmio = BSD_HWS_PGA_GEN7; |
961 | break; |
||
4104 | Serge | 962 | case VECS: |
963 | mmio = VEBOX_HWS_PGA_GEN7; |
||
964 | break; |
||
2332 | Serge | 965 | } |
966 | } else if (IS_GEN6(ring->dev)) { |
||
967 | mmio = RING_HWS_PGA_GEN6(ring->mmio_base); |
||
968 | } else { |
||
4560 | Serge | 969 | /* XXX: gen8 returns to sanity */ |
2332 | Serge | 970 | mmio = RING_HWS_PGA(ring->mmio_base); |
971 | } |
||
972 | |||
973 | I915_WRITE(mmio, (u32)ring->status_page.gfx_addr); |
||
974 | POSTING_READ(mmio); |
||
3746 | Serge | 975 | |
4104 | Serge | 976 | /* Flush the TLB for this page */ |
977 | if (INTEL_INFO(dev)->gen >= 6) { |
||
978 | u32 reg = RING_INSTPM(ring->mmio_base); |
||
979 | I915_WRITE(reg, |
||
980 | _MASKED_BIT_ENABLE(INSTPM_TLB_INVALIDATE | |
||
981 | INSTPM_SYNC_FLUSH)); |
||
982 | if (wait_for((I915_READ(reg) & INSTPM_SYNC_FLUSH) == 0, |
||
983 | 1000)) |
||
984 | DRM_ERROR("%s: wait for SyncFlush to complete for TLB invalidation timed out\n", |
||
985 | ring->name); |
||
986 | } |
||
2332 | Serge | 987 | } |
988 | |||
989 | static int |
||
990 | bsd_ring_flush(struct intel_ring_buffer *ring, |
||
991 | u32 invalidate_domains, |
||
992 | u32 flush_domains) |
||
993 | { |
||
994 | int ret; |
||
995 | |||
996 | ret = intel_ring_begin(ring, 2); |
||
997 | if (ret) |
||
998 | return ret; |
||
999 | |||
1000 | intel_ring_emit(ring, MI_FLUSH); |
||
1001 | intel_ring_emit(ring, MI_NOOP); |
||
1002 | intel_ring_advance(ring); |
||
1003 | return 0; |
||
1004 | } |
||
1005 | |||
1006 | static int |
||
3243 | Serge | 1007 | i9xx_add_request(struct intel_ring_buffer *ring) |
2332 | Serge | 1008 | { |
1009 | int ret; |
||
1010 | |||
1011 | ret = intel_ring_begin(ring, 4); |
||
1012 | if (ret) |
||
1013 | return ret; |
||
1014 | |||
1015 | intel_ring_emit(ring, MI_STORE_DWORD_INDEX); |
||
1016 | intel_ring_emit(ring, I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT); |
||
4560 | Serge | 1017 | intel_ring_emit(ring, ring->outstanding_lazy_seqno); |
2332 | Serge | 1018 | intel_ring_emit(ring, MI_USER_INTERRUPT); |
4560 | Serge | 1019 | __intel_ring_advance(ring); |
2332 | Serge | 1020 | |
1021 | return 0; |
||
1022 | } |
||
1023 | |||
1024 | static bool |
||
3031 | serge | 1025 | gen6_ring_get_irq(struct intel_ring_buffer *ring) |
2332 | Serge | 1026 | { |
1027 | struct drm_device *dev = ring->dev; |
||
1028 | drm_i915_private_t *dev_priv = dev->dev_private; |
||
3031 | serge | 1029 | unsigned long flags; |
2332 | Serge | 1030 | |
1031 | if (!dev->irq_enabled) |
||
1032 | return false; |
||
1033 | |||
3031 | serge | 1034 | spin_lock_irqsave(&dev_priv->irq_lock, flags); |
2332 | Serge | 1035 | if (ring->irq_refcount++ == 0) { |
4560 | Serge | 1036 | if (HAS_L3_DPF(dev) && ring->id == RCS) |
4104 | Serge | 1037 | I915_WRITE_IMR(ring, |
1038 | ~(ring->irq_enable_mask | |
||
4560 | Serge | 1039 | GT_PARITY_ERROR(dev))); |
3031 | serge | 1040 | else |
1041 | I915_WRITE_IMR(ring, ~ring->irq_enable_mask); |
||
4104 | Serge | 1042 | ilk_enable_gt_irq(dev_priv, ring->irq_enable_mask); |
2332 | Serge | 1043 | } |
3031 | serge | 1044 | spin_unlock_irqrestore(&dev_priv->irq_lock, flags); |
2332 | Serge | 1045 | |
2351 | Serge | 1046 | return true; |
2332 | Serge | 1047 | } |
1048 | |||
1049 | static void |
||
3031 | serge | 1050 | gen6_ring_put_irq(struct intel_ring_buffer *ring) |
2332 | Serge | 1051 | { |
1052 | struct drm_device *dev = ring->dev; |
||
1053 | drm_i915_private_t *dev_priv = dev->dev_private; |
||
3031 | serge | 1054 | unsigned long flags; |
2332 | Serge | 1055 | |
3031 | serge | 1056 | spin_lock_irqsave(&dev_priv->irq_lock, flags); |
2332 | Serge | 1057 | if (--ring->irq_refcount == 0) { |
4560 | Serge | 1058 | if (HAS_L3_DPF(dev) && ring->id == RCS) |
1059 | I915_WRITE_IMR(ring, ~GT_PARITY_ERROR(dev)); |
||
3031 | serge | 1060 | else |
1061 | I915_WRITE_IMR(ring, ~0); |
||
4104 | Serge | 1062 | ilk_disable_gt_irq(dev_priv, ring->irq_enable_mask); |
2332 | Serge | 1063 | } |
3031 | serge | 1064 | spin_unlock_irqrestore(&dev_priv->irq_lock, flags); |
2332 | Serge | 1065 | } |
1066 | |||
4104 | Serge | 1067 | static bool |
1068 | hsw_vebox_get_irq(struct intel_ring_buffer *ring) |
||
1069 | { |
||
1070 | struct drm_device *dev = ring->dev; |
||
1071 | struct drm_i915_private *dev_priv = dev->dev_private; |
||
1072 | unsigned long flags; |
||
1073 | |||
1074 | if (!dev->irq_enabled) |
||
1075 | return false; |
||
1076 | |||
1077 | spin_lock_irqsave(&dev_priv->irq_lock, flags); |
||
1078 | if (ring->irq_refcount++ == 0) { |
||
1079 | I915_WRITE_IMR(ring, ~ring->irq_enable_mask); |
||
1080 | snb_enable_pm_irq(dev_priv, ring->irq_enable_mask); |
||
1081 | } |
||
1082 | spin_unlock_irqrestore(&dev_priv->irq_lock, flags); |
||
1083 | |||
1084 | return true; |
||
1085 | } |
||
1086 | |||
1087 | static void |
||
1088 | hsw_vebox_put_irq(struct intel_ring_buffer *ring) |
||
1089 | { |
||
1090 | struct drm_device *dev = ring->dev; |
||
1091 | struct drm_i915_private *dev_priv = dev->dev_private; |
||
1092 | unsigned long flags; |
||
1093 | |||
1094 | if (!dev->irq_enabled) |
||
1095 | return; |
||
1096 | |||
1097 | spin_lock_irqsave(&dev_priv->irq_lock, flags); |
||
1098 | if (--ring->irq_refcount == 0) { |
||
1099 | I915_WRITE_IMR(ring, ~0); |
||
1100 | snb_disable_pm_irq(dev_priv, ring->irq_enable_mask); |
||
1101 | } |
||
1102 | spin_unlock_irqrestore(&dev_priv->irq_lock, flags); |
||
1103 | } |
||
1104 | |||
4560 | Serge | 1105 | static bool |
1106 | gen8_ring_get_irq(struct intel_ring_buffer *ring) |
||
1107 | { |
||
1108 | struct drm_device *dev = ring->dev; |
||
1109 | struct drm_i915_private *dev_priv = dev->dev_private; |
||
1110 | unsigned long flags; |
||
1111 | |||
1112 | if (!dev->irq_enabled) |
||
1113 | return false; |
||
1114 | |||
1115 | spin_lock_irqsave(&dev_priv->irq_lock, flags); |
||
1116 | if (ring->irq_refcount++ == 0) { |
||
1117 | if (HAS_L3_DPF(dev) && ring->id == RCS) { |
||
1118 | I915_WRITE_IMR(ring, |
||
1119 | ~(ring->irq_enable_mask | |
||
1120 | GT_RENDER_L3_PARITY_ERROR_INTERRUPT)); |
||
1121 | } else { |
||
1122 | I915_WRITE_IMR(ring, ~ring->irq_enable_mask); |
||
1123 | } |
||
1124 | POSTING_READ(RING_IMR(ring->mmio_base)); |
||
1125 | } |
||
1126 | spin_unlock_irqrestore(&dev_priv->irq_lock, flags); |
||
1127 | |||
1128 | return true; |
||
1129 | } |
||
1130 | |||
1131 | static void |
||
1132 | gen8_ring_put_irq(struct intel_ring_buffer *ring) |
||
1133 | { |
||
1134 | struct drm_device *dev = ring->dev; |
||
1135 | struct drm_i915_private *dev_priv = dev->dev_private; |
||
1136 | unsigned long flags; |
||
1137 | |||
1138 | spin_lock_irqsave(&dev_priv->irq_lock, flags); |
||
1139 | if (--ring->irq_refcount == 0) { |
||
1140 | if (HAS_L3_DPF(dev) && ring->id == RCS) { |
||
1141 | I915_WRITE_IMR(ring, |
||
1142 | ~GT_RENDER_L3_PARITY_ERROR_INTERRUPT); |
||
1143 | } else { |
||
1144 | I915_WRITE_IMR(ring, ~0); |
||
1145 | } |
||
1146 | POSTING_READ(RING_IMR(ring->mmio_base)); |
||
1147 | } |
||
1148 | spin_unlock_irqrestore(&dev_priv->irq_lock, flags); |
||
1149 | } |
||
1150 | |||
2332 | Serge | 1151 | static int |
3243 | Serge | 1152 | i965_dispatch_execbuffer(struct intel_ring_buffer *ring, |
1153 | u32 offset, u32 length, |
||
1154 | unsigned flags) |
||
2332 | Serge | 1155 | { |
1156 | int ret; |
||
1157 | |||
1158 | ret = intel_ring_begin(ring, 2); |
||
1159 | if (ret) |
||
1160 | return ret; |
||
1161 | |||
1162 | intel_ring_emit(ring, |
||
3031 | serge | 1163 | MI_BATCH_BUFFER_START | |
1164 | MI_BATCH_GTT | |
||
3243 | Serge | 1165 | (flags & I915_DISPATCH_SECURE ? 0 : MI_BATCH_NON_SECURE_I965)); |
2332 | Serge | 1166 | intel_ring_emit(ring, offset); |
1167 | intel_ring_advance(ring); |
||
1168 | |||
1169 | return 0; |
||
1170 | } |
||
1171 | |||
3243 | Serge | 1172 | /* Just userspace ABI convention to limit the wa batch bo to a resonable size */ |
1173 | #define I830_BATCH_LIMIT (256*1024) |
||
2332 | Serge | 1174 | static int |
3031 | serge | 1175 | i830_dispatch_execbuffer(struct intel_ring_buffer *ring, |
3243 | Serge | 1176 | u32 offset, u32 len, |
1177 | unsigned flags) |
||
2332 | Serge | 1178 | { |
1179 | int ret; |
||
1180 | |||
3243 | Serge | 1181 | if (flags & I915_DISPATCH_PINNED) { |
2332 | Serge | 1182 | ret = intel_ring_begin(ring, 4); |
1183 | if (ret) |
||
1184 | return ret; |
||
1185 | |||
1186 | intel_ring_emit(ring, MI_BATCH_BUFFER); |
||
3243 | Serge | 1187 | intel_ring_emit(ring, offset | (flags & I915_DISPATCH_SECURE ? 0 : MI_BATCH_NON_SECURE)); |
2332 | Serge | 1188 | intel_ring_emit(ring, offset + len - 8); |
3243 | Serge | 1189 | intel_ring_emit(ring, MI_NOOP); |
1190 | intel_ring_advance(ring); |
||
1191 | } else { |
||
4104 | Serge | 1192 | u32 cs_offset = ring->scratch.gtt_offset; |
3243 | Serge | 1193 | |
1194 | if (len > I830_BATCH_LIMIT) |
||
1195 | return -ENOSPC; |
||
1196 | |||
1197 | ret = intel_ring_begin(ring, 9+3); |
||
1198 | if (ret) |
||
1199 | return ret; |
||
1200 | /* Blit the batch (which has now all relocs applied) to the stable batch |
||
1201 | * scratch bo area (so that the CS never stumbles over its tlb |
||
1202 | * invalidation bug) ... */ |
||
1203 | intel_ring_emit(ring, XY_SRC_COPY_BLT_CMD | |
||
1204 | XY_SRC_COPY_BLT_WRITE_ALPHA | |
||
1205 | XY_SRC_COPY_BLT_WRITE_RGB); |
||
1206 | intel_ring_emit(ring, BLT_DEPTH_32 | BLT_ROP_GXCOPY | 4096); |
||
2332 | Serge | 1207 | intel_ring_emit(ring, 0); |
3243 | Serge | 1208 | intel_ring_emit(ring, (DIV_ROUND_UP(len, 4096) << 16) | 1024); |
1209 | intel_ring_emit(ring, cs_offset); |
||
1210 | intel_ring_emit(ring, 0); |
||
1211 | intel_ring_emit(ring, 4096); |
||
1212 | intel_ring_emit(ring, offset); |
||
1213 | intel_ring_emit(ring, MI_FLUSH); |
||
1214 | |||
1215 | /* ... and execute it. */ |
||
1216 | intel_ring_emit(ring, MI_BATCH_BUFFER); |
||
1217 | intel_ring_emit(ring, cs_offset | (flags & I915_DISPATCH_SECURE ? 0 : MI_BATCH_NON_SECURE)); |
||
1218 | intel_ring_emit(ring, cs_offset + len - 8); |
||
3031 | serge | 1219 | intel_ring_advance(ring); |
3243 | Serge | 1220 | } |
3031 | serge | 1221 | |
1222 | return 0; |
||
1223 | } |
||
1224 | |||
1225 | static int |
||
1226 | i915_dispatch_execbuffer(struct intel_ring_buffer *ring, |
||
3243 | Serge | 1227 | u32 offset, u32 len, |
1228 | unsigned flags) |
||
3031 | serge | 1229 | { |
1230 | int ret; |
||
1231 | |||
2332 | Serge | 1232 | ret = intel_ring_begin(ring, 2); |
1233 | if (ret) |
||
1234 | return ret; |
||
1235 | |||
3031 | serge | 1236 | intel_ring_emit(ring, MI_BATCH_BUFFER_START | MI_BATCH_GTT); |
3243 | Serge | 1237 | intel_ring_emit(ring, offset | (flags & I915_DISPATCH_SECURE ? 0 : MI_BATCH_NON_SECURE)); |
2332 | Serge | 1238 | intel_ring_advance(ring); |
1239 | |||
1240 | return 0; |
||
1241 | } |
||
1242 | |||
1243 | static void cleanup_status_page(struct intel_ring_buffer *ring) |
||
1244 | { |
||
1245 | struct drm_i915_gem_object *obj; |
||
1246 | |||
1247 | obj = ring->status_page.obj; |
||
1248 | if (obj == NULL) |
||
1249 | return; |
||
1250 | |||
3031 | serge | 1251 | // kunmap(sg_page(obj->pages->sgl)); |
2344 | Serge | 1252 | i915_gem_object_unpin(obj); |
1253 | drm_gem_object_unreference(&obj->base); |
||
2332 | Serge | 1254 | ring->status_page.obj = NULL; |
1255 | } |
||
1256 | |||
1257 | static int init_status_page(struct intel_ring_buffer *ring) |
||
1258 | { |
||
1259 | struct drm_device *dev = ring->dev; |
||
1260 | struct drm_i915_gem_object *obj; |
||
1261 | int ret; |
||
1262 | |||
1263 | obj = i915_gem_alloc_object(dev, 4096); |
||
1264 | if (obj == NULL) { |
||
1265 | DRM_ERROR("Failed to allocate status page\n"); |
||
1266 | ret = -ENOMEM; |
||
1267 | goto err; |
||
1268 | } |
||
1269 | |||
2352 | Serge | 1270 | i915_gem_object_set_cache_level(obj, I915_CACHE_LLC); |
2332 | Serge | 1271 | |
4104 | Serge | 1272 | ret = i915_gem_obj_ggtt_pin(obj, 4096, true, false); |
2332 | Serge | 1273 | if (ret != 0) { |
1274 | goto err_unref; |
||
1275 | } |
||
1276 | |||
4104 | Serge | 1277 | ring->status_page.gfx_addr = i915_gem_obj_ggtt_offset(obj); |
4539 | Serge | 1278 | ring->status_page.page_addr = (void*)MapIoMem((addr_t)sg_page(obj->pages->sgl),4096,PG_SW|0x100); |
2332 | Serge | 1279 | if (ring->status_page.page_addr == NULL) { |
3031 | serge | 1280 | ret = -ENOMEM; |
2332 | Serge | 1281 | goto err_unpin; |
1282 | } |
||
1283 | ring->status_page.obj = obj; |
||
1284 | memset(ring->status_page.page_addr, 0, PAGE_SIZE); |
||
1285 | |||
1286 | DRM_DEBUG_DRIVER("%s hws offset: 0x%08x\n", |
||
1287 | ring->name, ring->status_page.gfx_addr); |
||
1288 | |||
1289 | return 0; |
||
1290 | |||
1291 | err_unpin: |
||
2344 | Serge | 1292 | i915_gem_object_unpin(obj); |
2332 | Serge | 1293 | err_unref: |
2344 | Serge | 1294 | drm_gem_object_unreference(&obj->base); |
2332 | Serge | 1295 | err: |
1296 | return ret; |
||
1297 | } |
||
1298 | |||
4104 | Serge | 1299 | static int init_phys_status_page(struct intel_ring_buffer *ring) |
3243 | Serge | 1300 | { |
1301 | struct drm_i915_private *dev_priv = ring->dev->dev_private; |
||
1302 | |||
1303 | if (!dev_priv->status_page_dmah) { |
||
1304 | dev_priv->status_page_dmah = |
||
1305 | drm_pci_alloc(ring->dev, PAGE_SIZE, PAGE_SIZE); |
||
1306 | if (!dev_priv->status_page_dmah) |
||
1307 | return -ENOMEM; |
||
1308 | } |
||
1309 | |||
1310 | ring->status_page.page_addr = dev_priv->status_page_dmah->vaddr; |
||
1311 | memset(ring->status_page.page_addr, 0, PAGE_SIZE); |
||
1312 | |||
1313 | return 0; |
||
1314 | } |
||
1315 | |||
3031 | serge | 1316 | static int intel_init_ring_buffer(struct drm_device *dev, |
2332 | Serge | 1317 | struct intel_ring_buffer *ring) |
1318 | { |
||
2340 | Serge | 1319 | struct drm_i915_gem_object *obj; |
3031 | serge | 1320 | struct drm_i915_private *dev_priv = dev->dev_private; |
2332 | Serge | 1321 | int ret; |
2340 | Serge | 1322 | |
2332 | Serge | 1323 | ring->dev = dev; |
1324 | INIT_LIST_HEAD(&ring->active_list); |
||
1325 | INIT_LIST_HEAD(&ring->request_list); |
||
3031 | serge | 1326 | ring->size = 32 * PAGE_SIZE; |
3243 | Serge | 1327 | memset(ring->sync_seqno, 0, sizeof(ring->sync_seqno)); |
2332 | Serge | 1328 | |
2352 | Serge | 1329 | init_waitqueue_head(&ring->irq_queue); |
2332 | Serge | 1330 | |
1331 | if (I915_NEED_GFX_HWS(dev)) { |
||
2340 | Serge | 1332 | ret = init_status_page(ring); |
1333 | if (ret) |
||
1334 | return ret; |
||
3243 | Serge | 1335 | } else { |
1336 | BUG_ON(ring->id != RCS); |
||
4104 | Serge | 1337 | ret = init_phys_status_page(ring); |
3243 | Serge | 1338 | if (ret) |
1339 | return ret; |
||
2332 | Serge | 1340 | } |
1341 | |||
3480 | Serge | 1342 | obj = NULL; |
4371 | Serge | 1343 | if (!HAS_LLC(dev)) |
1344 | obj = i915_gem_object_create_stolen(dev, ring->size); |
||
3480 | Serge | 1345 | if (obj == NULL) |
4560 | Serge | 1346 | obj = i915_gem_alloc_object(dev, ring->size); |
2332 | Serge | 1347 | if (obj == NULL) { |
1348 | DRM_ERROR("Failed to allocate ringbuffer\n"); |
||
1349 | ret = -ENOMEM; |
||
1350 | goto err_hws; |
||
1351 | } |
||
1352 | |||
1353 | ring->obj = obj; |
||
1354 | |||
4104 | Serge | 1355 | ret = i915_gem_obj_ggtt_pin(obj, PAGE_SIZE, true, false); |
2332 | Serge | 1356 | if (ret) |
1357 | goto err_unref; |
||
1358 | |||
3031 | serge | 1359 | ret = i915_gem_object_set_to_gtt_domain(obj, true); |
1360 | if (ret) |
||
1361 | goto err_unpin; |
||
2332 | Serge | 1362 | |
3031 | serge | 1363 | ring->virtual_start = |
4104 | Serge | 1364 | ioremap_wc(dev_priv->gtt.mappable_base + i915_gem_obj_ggtt_offset(obj), |
3031 | serge | 1365 | ring->size); |
1366 | if (ring->virtual_start == NULL) { |
||
2332 | Serge | 1367 | DRM_ERROR("Failed to map ringbuffer.\n"); |
1368 | ret = -EINVAL; |
||
1369 | goto err_unpin; |
||
1370 | } |
||
1371 | |||
1372 | ret = ring->init(ring); |
||
1373 | if (ret) |
||
1374 | goto err_unmap; |
||
1375 | |||
1376 | /* Workaround an erratum on the i830 which causes a hang if |
||
1377 | * the TAIL pointer points to within the last 2 cachelines |
||
1378 | * of the buffer. |
||
1379 | */ |
||
1380 | ring->effective_size = ring->size; |
||
3031 | serge | 1381 | if (IS_I830(ring->dev) || IS_845G(ring->dev)) |
2332 | Serge | 1382 | ring->effective_size -= 128; |
2340 | Serge | 1383 | |
2332 | Serge | 1384 | return 0; |
1385 | |||
1386 | err_unmap: |
||
3480 | Serge | 1387 | iounmap(ring->virtual_start); |
2332 | Serge | 1388 | err_unpin: |
2344 | Serge | 1389 | i915_gem_object_unpin(obj); |
2332 | Serge | 1390 | err_unref: |
2344 | Serge | 1391 | drm_gem_object_unreference(&obj->base); |
2332 | Serge | 1392 | ring->obj = NULL; |
1393 | err_hws: |
||
1394 | // cleanup_status_page(ring); |
||
1395 | return ret; |
||
1396 | } |
||
1397 | |||
1398 | void intel_cleanup_ring_buffer(struct intel_ring_buffer *ring) |
||
1399 | { |
||
1400 | struct drm_i915_private *dev_priv; |
||
1401 | int ret; |
||
1402 | |||
1403 | if (ring->obj == NULL) |
||
1404 | return; |
||
1405 | |||
1406 | /* Disable the ring buffer. The ring must be idle at this point */ |
||
1407 | dev_priv = ring->dev->dev_private; |
||
3243 | Serge | 1408 | ret = intel_ring_idle(ring); |
4560 | Serge | 1409 | if (ret && !i915_reset_in_progress(&dev_priv->gpu_error)) |
2332 | Serge | 1410 | DRM_ERROR("failed to quiesce %s whilst cleaning up: %d\n", |
1411 | ring->name, ret); |
||
1412 | |||
1413 | I915_WRITE_CTL(ring, 0); |
||
1414 | |||
3480 | Serge | 1415 | iounmap(ring->virtual_start); |
2332 | Serge | 1416 | |
2344 | Serge | 1417 | i915_gem_object_unpin(ring->obj); |
1418 | drm_gem_object_unreference(&ring->obj->base); |
||
2332 | Serge | 1419 | ring->obj = NULL; |
4560 | Serge | 1420 | ring->preallocated_lazy_request = NULL; |
1421 | ring->outstanding_lazy_seqno = 0; |
||
2332 | Serge | 1422 | |
1423 | if (ring->cleanup) |
||
1424 | ring->cleanup(ring); |
||
1425 | |||
1426 | // cleanup_status_page(ring); |
||
1427 | } |
||
1428 | |||
3031 | serge | 1429 | static int intel_ring_wait_seqno(struct intel_ring_buffer *ring, u32 seqno) |
2332 | Serge | 1430 | { |
3031 | serge | 1431 | int ret; |
2332 | Serge | 1432 | |
3031 | serge | 1433 | ret = i915_wait_seqno(ring, seqno); |
1434 | if (!ret) |
||
1435 | i915_gem_retire_requests_ring(ring); |
||
1436 | |||
1437 | return ret; |
||
1438 | } |
||
1439 | |||
1440 | static int intel_ring_wait_request(struct intel_ring_buffer *ring, int n) |
||
1441 | { |
||
1442 | struct drm_i915_gem_request *request; |
||
1443 | u32 seqno = 0; |
||
1444 | int ret; |
||
1445 | |||
1446 | i915_gem_retire_requests_ring(ring); |
||
1447 | |||
1448 | if (ring->last_retired_head != -1) { |
||
1449 | ring->head = ring->last_retired_head; |
||
1450 | ring->last_retired_head = -1; |
||
2332 | Serge | 1451 | ring->space = ring_space(ring); |
1452 | if (ring->space >= n) |
||
1453 | return 0; |
||
1454 | } |
||
1455 | |||
3031 | serge | 1456 | list_for_each_entry(request, &ring->request_list, list) { |
1457 | int space; |
||
2344 | Serge | 1458 | |
3031 | serge | 1459 | if (request->tail == -1) |
1460 | continue; |
||
1461 | |||
3243 | Serge | 1462 | space = request->tail - (ring->tail + I915_RING_FREE_SPACE); |
3031 | serge | 1463 | if (space < 0) |
1464 | space += ring->size; |
||
1465 | if (space >= n) { |
||
1466 | seqno = request->seqno; |
||
1467 | break; |
||
1468 | } |
||
1469 | |||
1470 | /* Consume this request in case we need more space than |
||
1471 | * is available and so need to prevent a race between |
||
1472 | * updating last_retired_head and direct reads of |
||
1473 | * I915_RING_HEAD. It also provides a nice sanity check. |
||
1474 | */ |
||
1475 | request->tail = -1; |
||
1476 | } |
||
1477 | |||
1478 | if (seqno == 0) |
||
1479 | return -ENOSPC; |
||
1480 | |||
1481 | ret = intel_ring_wait_seqno(ring, seqno); |
||
1482 | if (ret) |
||
1483 | return ret; |
||
1484 | |||
1485 | if (WARN_ON(ring->last_retired_head == -1)) |
||
1486 | return -ENOSPC; |
||
1487 | |||
1488 | ring->head = ring->last_retired_head; |
||
1489 | ring->last_retired_head = -1; |
||
1490 | ring->space = ring_space(ring); |
||
1491 | if (WARN_ON(ring->space < n)) |
||
1492 | return -ENOSPC; |
||
1493 | |||
1494 | return 0; |
||
1495 | } |
||
1496 | |||
3243 | Serge | 1497 | static int ring_wait_for_space(struct intel_ring_buffer *ring, int n) |
3031 | serge | 1498 | { |
1499 | struct drm_device *dev = ring->dev; |
||
1500 | struct drm_i915_private *dev_priv = dev->dev_private; |
||
1501 | unsigned long end; |
||
1502 | int ret; |
||
1503 | |||
1504 | ret = intel_ring_wait_request(ring, n); |
||
1505 | if (ret != -ENOSPC) |
||
1506 | return ret; |
||
1507 | |||
4560 | Serge | 1508 | /* force the tail write in case we have been skipping them */ |
1509 | __intel_ring_advance(ring); |
||
1510 | |||
3243 | Serge | 1511 | trace_i915_ring_wait_begin(ring); |
3031 | serge | 1512 | /* With GEM the hangcheck timer should kick us out of the loop, |
1513 | * leaving it early runs the risk of corrupting GEM state (due |
||
1514 | * to running on almost untested codepaths). But on resume |
||
1515 | * timers don't work yet, so prevent a complete hang in that |
||
1516 | * case by choosing an insanely large timeout. */ |
||
1517 | end = GetTimerTicks() + 60 * HZ; |
||
1518 | |||
2332 | Serge | 1519 | do { |
1520 | ring->head = I915_READ_HEAD(ring); |
||
1521 | ring->space = ring_space(ring); |
||
1522 | if (ring->space >= n) { |
||
2351 | Serge | 1523 | trace_i915_ring_wait_end(ring); |
2332 | Serge | 1524 | return 0; |
1525 | } |
||
1526 | |||
1527 | msleep(1); |
||
3031 | serge | 1528 | |
3480 | Serge | 1529 | ret = i915_gem_check_wedge(&dev_priv->gpu_error, |
1530 | dev_priv->mm.interruptible); |
||
3031 | serge | 1531 | if (ret) |
1532 | return ret; |
||
1533 | } while (!time_after(GetTimerTicks(), end)); |
||
2351 | Serge | 1534 | trace_i915_ring_wait_end(ring); |
2332 | Serge | 1535 | return -EBUSY; |
1536 | } |
||
1537 | |||
3243 | Serge | 1538 | static int intel_wrap_ring_buffer(struct intel_ring_buffer *ring) |
1539 | { |
||
1540 | uint32_t __iomem *virt; |
||
1541 | int rem = ring->size - ring->tail; |
||
1542 | |||
1543 | if (ring->space < rem) { |
||
1544 | int ret = ring_wait_for_space(ring, rem); |
||
1545 | if (ret) |
||
1546 | return ret; |
||
1547 | } |
||
1548 | |||
1549 | virt = ring->virtual_start + ring->tail; |
||
1550 | rem /= 4; |
||
1551 | while (rem--) |
||
1552 | iowrite32(MI_NOOP, virt++); |
||
1553 | |||
1554 | ring->tail = 0; |
||
1555 | ring->space = ring_space(ring); |
||
1556 | |||
1557 | return 0; |
||
1558 | } |
||
1559 | |||
1560 | int intel_ring_idle(struct intel_ring_buffer *ring) |
||
1561 | { |
||
1562 | u32 seqno; |
||
1563 | int ret; |
||
1564 | |||
1565 | /* We need to add any requests required to flush the objects and ring */ |
||
4560 | Serge | 1566 | if (ring->outstanding_lazy_seqno) { |
4104 | Serge | 1567 | ret = i915_add_request(ring, NULL); |
3243 | Serge | 1568 | if (ret) |
1569 | return ret; |
||
1570 | } |
||
1571 | |||
1572 | /* Wait upon the last request to be completed */ |
||
1573 | if (list_empty(&ring->request_list)) |
||
1574 | return 0; |
||
1575 | |||
1576 | seqno = list_entry(ring->request_list.prev, |
||
1577 | struct drm_i915_gem_request, |
||
1578 | list)->seqno; |
||
1579 | |||
1580 | return i915_wait_seqno(ring, seqno); |
||
1581 | } |
||
1582 | |||
1583 | static int |
||
1584 | intel_ring_alloc_seqno(struct intel_ring_buffer *ring) |
||
1585 | { |
||
4560 | Serge | 1586 | if (ring->outstanding_lazy_seqno) |
3243 | Serge | 1587 | return 0; |
1588 | |||
4560 | Serge | 1589 | if (ring->preallocated_lazy_request == NULL) { |
1590 | struct drm_i915_gem_request *request; |
||
1591 | |||
1592 | request = kmalloc(sizeof(*request), GFP_KERNEL); |
||
1593 | if (request == NULL) |
||
1594 | return -ENOMEM; |
||
1595 | |||
1596 | ring->preallocated_lazy_request = request; |
||
1597 | } |
||
1598 | |||
1599 | return i915_gem_get_seqno(ring->dev, &ring->outstanding_lazy_seqno); |
||
3243 | Serge | 1600 | } |
1601 | |||
4560 | Serge | 1602 | static int __intel_ring_prepare(struct intel_ring_buffer *ring, |
3480 | Serge | 1603 | int bytes) |
1604 | { |
||
1605 | int ret; |
||
1606 | |||
1607 | if (unlikely(ring->tail + bytes > ring->effective_size)) { |
||
1608 | ret = intel_wrap_ring_buffer(ring); |
||
1609 | if (unlikely(ret)) |
||
1610 | return ret; |
||
1611 | } |
||
1612 | |||
1613 | if (unlikely(ring->space < bytes)) { |
||
1614 | ret = ring_wait_for_space(ring, bytes); |
||
1615 | if (unlikely(ret)) |
||
1616 | return ret; |
||
1617 | } |
||
1618 | |||
1619 | return 0; |
||
1620 | } |
||
1621 | |||
2332 | Serge | 1622 | int intel_ring_begin(struct intel_ring_buffer *ring, |
1623 | int num_dwords) |
||
1624 | { |
||
3031 | serge | 1625 | drm_i915_private_t *dev_priv = ring->dev->dev_private; |
2332 | Serge | 1626 | int ret; |
1627 | |||
3480 | Serge | 1628 | ret = i915_gem_check_wedge(&dev_priv->gpu_error, |
1629 | dev_priv->mm.interruptible); |
||
3031 | serge | 1630 | if (ret) |
1631 | return ret; |
||
2332 | Serge | 1632 | |
4560 | Serge | 1633 | ret = __intel_ring_prepare(ring, num_dwords * sizeof(uint32_t)); |
1634 | if (ret) |
||
1635 | return ret; |
||
1636 | |||
3243 | Serge | 1637 | /* Preallocate the olr before touching the ring */ |
1638 | ret = intel_ring_alloc_seqno(ring); |
||
1639 | if (ret) |
||
1640 | return ret; |
||
1641 | |||
4560 | Serge | 1642 | ring->space -= num_dwords * sizeof(uint32_t); |
1643 | return 0; |
||
3480 | Serge | 1644 | } |
2332 | Serge | 1645 | |
3480 | Serge | 1646 | void intel_ring_init_seqno(struct intel_ring_buffer *ring, u32 seqno) |
1647 | { |
||
1648 | struct drm_i915_private *dev_priv = ring->dev->dev_private; |
||
1649 | |||
4560 | Serge | 1650 | BUG_ON(ring->outstanding_lazy_seqno); |
3480 | Serge | 1651 | |
1652 | if (INTEL_INFO(ring->dev)->gen >= 6) { |
||
1653 | I915_WRITE(RING_SYNC_0(ring->mmio_base), 0); |
||
1654 | I915_WRITE(RING_SYNC_1(ring->mmio_base), 0); |
||
4104 | Serge | 1655 | if (HAS_VEBOX(ring->dev)) |
1656 | I915_WRITE(RING_SYNC_2(ring->mmio_base), 0); |
||
2332 | Serge | 1657 | } |
1658 | |||
3480 | Serge | 1659 | ring->set_seqno(ring, seqno); |
4104 | Serge | 1660 | ring->hangcheck.seqno = seqno; |
2332 | Serge | 1661 | } |
1662 | |||
1663 | static void gen6_bsd_ring_write_tail(struct intel_ring_buffer *ring, |
||
1664 | u32 value) |
||
1665 | { |
||
1666 | drm_i915_private_t *dev_priv = ring->dev->dev_private; |
||
1667 | |||
1668 | /* Every tail move must follow the sequence below */ |
||
3031 | serge | 1669 | |
1670 | /* Disable notification that the ring is IDLE. The GT |
||
1671 | * will then assume that it is busy and bring it out of rc6. |
||
1672 | */ |
||
2332 | Serge | 1673 | I915_WRITE(GEN6_BSD_SLEEP_PSMI_CONTROL, |
3031 | serge | 1674 | _MASKED_BIT_ENABLE(GEN6_BSD_SLEEP_MSG_DISABLE)); |
2332 | Serge | 1675 | |
3031 | serge | 1676 | /* Clear the context id. Here be magic! */ |
1677 | I915_WRITE64(GEN6_BSD_RNCID, 0x0); |
||
1678 | |||
1679 | /* Wait for the ring not to be idle, i.e. for it to wake up. */ |
||
2332 | Serge | 1680 | if (wait_for((I915_READ(GEN6_BSD_SLEEP_PSMI_CONTROL) & |
3031 | serge | 1681 | GEN6_BSD_SLEEP_INDICATOR) == 0, |
2332 | Serge | 1682 | 50)) |
3031 | serge | 1683 | DRM_ERROR("timed out waiting for the BSD ring to wake up\n"); |
2332 | Serge | 1684 | |
3031 | serge | 1685 | /* Now that the ring is fully powered up, update the tail */ |
2332 | Serge | 1686 | I915_WRITE_TAIL(ring, value); |
3031 | serge | 1687 | POSTING_READ(RING_TAIL(ring->mmio_base)); |
1688 | |||
1689 | /* Let the ring send IDLE messages to the GT again, |
||
1690 | * and so let it sleep to conserve power when idle. |
||
1691 | */ |
||
2332 | Serge | 1692 | I915_WRITE(GEN6_BSD_SLEEP_PSMI_CONTROL, |
3031 | serge | 1693 | _MASKED_BIT_DISABLE(GEN6_BSD_SLEEP_MSG_DISABLE)); |
2332 | Serge | 1694 | } |
1695 | |||
4104 | Serge | 1696 | static int gen6_bsd_ring_flush(struct intel_ring_buffer *ring, |
2332 | Serge | 1697 | u32 invalidate, u32 flush) |
1698 | { |
||
1699 | uint32_t cmd; |
||
1700 | int ret; |
||
1701 | |||
1702 | ret = intel_ring_begin(ring, 4); |
||
1703 | if (ret) |
||
1704 | return ret; |
||
1705 | |||
1706 | cmd = MI_FLUSH_DW; |
||
4560 | Serge | 1707 | if (INTEL_INFO(ring->dev)->gen >= 8) |
1708 | cmd += 1; |
||
3243 | Serge | 1709 | /* |
1710 | * Bspec vol 1c.5 - video engine command streamer: |
||
1711 | * "If ENABLED, all TLBs will be invalidated once the flush |
||
1712 | * operation is complete. This bit is only valid when the |
||
1713 | * Post-Sync Operation field is a value of 1h or 3h." |
||
1714 | */ |
||
2332 | Serge | 1715 | if (invalidate & I915_GEM_GPU_DOMAINS) |
3243 | Serge | 1716 | cmd |= MI_INVALIDATE_TLB | MI_INVALIDATE_BSD | |
1717 | MI_FLUSH_DW_STORE_INDEX | MI_FLUSH_DW_OP_STOREDW; |
||
2332 | Serge | 1718 | intel_ring_emit(ring, cmd); |
3243 | Serge | 1719 | intel_ring_emit(ring, I915_GEM_HWS_SCRATCH_ADDR | MI_FLUSH_DW_USE_GTT); |
4560 | Serge | 1720 | if (INTEL_INFO(ring->dev)->gen >= 8) { |
1721 | intel_ring_emit(ring, 0); /* upper addr */ |
||
1722 | intel_ring_emit(ring, 0); /* value */ |
||
1723 | } else { |
||
2332 | Serge | 1724 | intel_ring_emit(ring, 0); |
1725 | intel_ring_emit(ring, MI_NOOP); |
||
4560 | Serge | 1726 | } |
2332 | Serge | 1727 | intel_ring_advance(ring); |
1728 | return 0; |
||
1729 | } |
||
1730 | |||
1731 | static int |
||
4560 | Serge | 1732 | gen8_ring_dispatch_execbuffer(struct intel_ring_buffer *ring, |
1733 | u32 offset, u32 len, |
||
1734 | unsigned flags) |
||
1735 | { |
||
1736 | struct drm_i915_private *dev_priv = ring->dev->dev_private; |
||
1737 | bool ppgtt = dev_priv->mm.aliasing_ppgtt != NULL && |
||
1738 | !(flags & I915_DISPATCH_SECURE); |
||
1739 | int ret; |
||
1740 | |||
1741 | ret = intel_ring_begin(ring, 4); |
||
1742 | if (ret) |
||
1743 | return ret; |
||
1744 | |||
1745 | /* FIXME(BDW): Address space and security selectors. */ |
||
1746 | intel_ring_emit(ring, MI_BATCH_BUFFER_START_GEN8 | (ppgtt<<8)); |
||
1747 | intel_ring_emit(ring, offset); |
||
1748 | intel_ring_emit(ring, 0); |
||
1749 | intel_ring_emit(ring, MI_NOOP); |
||
1750 | intel_ring_advance(ring); |
||
1751 | |||
1752 | return 0; |
||
1753 | } |
||
1754 | |||
1755 | static int |
||
3243 | Serge | 1756 | hsw_ring_dispatch_execbuffer(struct intel_ring_buffer *ring, |
1757 | u32 offset, u32 len, |
||
1758 | unsigned flags) |
||
1759 | { |
||
1760 | int ret; |
||
1761 | |||
1762 | ret = intel_ring_begin(ring, 2); |
||
1763 | if (ret) |
||
1764 | return ret; |
||
1765 | |||
1766 | intel_ring_emit(ring, |
||
1767 | MI_BATCH_BUFFER_START | MI_BATCH_PPGTT_HSW | |
||
1768 | (flags & I915_DISPATCH_SECURE ? 0 : MI_BATCH_NON_SECURE_HSW)); |
||
1769 | /* bit0-7 is the length on GEN6+ */ |
||
1770 | intel_ring_emit(ring, offset); |
||
1771 | intel_ring_advance(ring); |
||
1772 | |||
1773 | return 0; |
||
1774 | } |
||
1775 | |||
1776 | static int |
||
2332 | Serge | 1777 | gen6_ring_dispatch_execbuffer(struct intel_ring_buffer *ring, |
3243 | Serge | 1778 | u32 offset, u32 len, |
1779 | unsigned flags) |
||
2332 | Serge | 1780 | { |
1781 | int ret; |
||
1782 | |||
1783 | ret = intel_ring_begin(ring, 2); |
||
1784 | if (ret) |
||
1785 | return ret; |
||
1786 | |||
3243 | Serge | 1787 | intel_ring_emit(ring, |
1788 | MI_BATCH_BUFFER_START | |
||
1789 | (flags & I915_DISPATCH_SECURE ? 0 : MI_BATCH_NON_SECURE_I965)); |
||
2332 | Serge | 1790 | /* bit0-7 is the length on GEN6+ */ |
1791 | intel_ring_emit(ring, offset); |
||
1792 | intel_ring_advance(ring); |
||
1793 | |||
1794 | return 0; |
||
1795 | } |
||
1796 | |||
1797 | /* Blitter support (SandyBridge+) */ |
||
1798 | |||
4104 | Serge | 1799 | static int gen6_ring_flush(struct intel_ring_buffer *ring, |
2332 | Serge | 1800 | u32 invalidate, u32 flush) |
1801 | { |
||
4104 | Serge | 1802 | struct drm_device *dev = ring->dev; |
2332 | Serge | 1803 | uint32_t cmd; |
1804 | int ret; |
||
1805 | |||
3031 | serge | 1806 | ret = intel_ring_begin(ring, 4); |
2332 | Serge | 1807 | if (ret) |
1808 | return ret; |
||
1809 | |||
1810 | cmd = MI_FLUSH_DW; |
||
4560 | Serge | 1811 | if (INTEL_INFO(ring->dev)->gen >= 8) |
1812 | cmd += 1; |
||
3243 | Serge | 1813 | /* |
1814 | * Bspec vol 1c.3 - blitter engine command streamer: |
||
1815 | * "If ENABLED, all TLBs will be invalidated once the flush |
||
1816 | * operation is complete. This bit is only valid when the |
||
1817 | * Post-Sync Operation field is a value of 1h or 3h." |
||
1818 | */ |
||
2332 | Serge | 1819 | if (invalidate & I915_GEM_DOMAIN_RENDER) |
3243 | Serge | 1820 | cmd |= MI_INVALIDATE_TLB | MI_FLUSH_DW_STORE_INDEX | |
1821 | MI_FLUSH_DW_OP_STOREDW; |
||
2332 | Serge | 1822 | intel_ring_emit(ring, cmd); |
3243 | Serge | 1823 | intel_ring_emit(ring, I915_GEM_HWS_SCRATCH_ADDR | MI_FLUSH_DW_USE_GTT); |
4560 | Serge | 1824 | if (INTEL_INFO(ring->dev)->gen >= 8) { |
1825 | intel_ring_emit(ring, 0); /* upper addr */ |
||
1826 | intel_ring_emit(ring, 0); /* value */ |
||
1827 | } else { |
||
2332 | Serge | 1828 | intel_ring_emit(ring, 0); |
1829 | intel_ring_emit(ring, MI_NOOP); |
||
4560 | Serge | 1830 | } |
2332 | Serge | 1831 | intel_ring_advance(ring); |
4104 | Serge | 1832 | |
4560 | Serge | 1833 | if (IS_GEN7(dev) && !invalidate && flush) |
4104 | Serge | 1834 | return gen7_ring_fbc_flush(ring, FBC_REND_CACHE_CLEAN); |
1835 | |||
2332 | Serge | 1836 | return 0; |
1837 | } |
||
1838 | |||
1839 | int intel_init_render_ring_buffer(struct drm_device *dev) |
||
1840 | { |
||
1841 | drm_i915_private_t *dev_priv = dev->dev_private; |
||
1842 | struct intel_ring_buffer *ring = &dev_priv->ring[RCS]; |
||
2340 | Serge | 1843 | |
3031 | serge | 1844 | ring->name = "render ring"; |
1845 | ring->id = RCS; |
||
1846 | ring->mmio_base = RENDER_RING_BASE; |
||
1847 | |||
2332 | Serge | 1848 | if (INTEL_INFO(dev)->gen >= 6) { |
2339 | Serge | 1849 | ring->add_request = gen6_add_request; |
3031 | serge | 1850 | ring->flush = gen7_render_ring_flush; |
1851 | if (INTEL_INFO(dev)->gen == 6) |
||
2342 | Serge | 1852 | ring->flush = gen6_render_ring_flush; |
4560 | Serge | 1853 | if (INTEL_INFO(dev)->gen >= 8) { |
1854 | ring->flush = gen8_render_ring_flush; |
||
1855 | ring->irq_get = gen8_ring_get_irq; |
||
1856 | ring->irq_put = gen8_ring_put_irq; |
||
1857 | } else { |
||
3031 | serge | 1858 | ring->irq_get = gen6_ring_get_irq; |
1859 | ring->irq_put = gen6_ring_put_irq; |
||
4560 | Serge | 1860 | } |
4104 | Serge | 1861 | ring->irq_enable_mask = GT_RENDER_USER_INTERRUPT; |
2342 | Serge | 1862 | ring->get_seqno = gen6_ring_get_seqno; |
3480 | Serge | 1863 | ring->set_seqno = ring_set_seqno; |
3031 | serge | 1864 | ring->sync_to = gen6_ring_sync; |
4104 | Serge | 1865 | ring->semaphore_register[RCS] = MI_SEMAPHORE_SYNC_INVALID; |
1866 | ring->semaphore_register[VCS] = MI_SEMAPHORE_SYNC_RV; |
||
1867 | ring->semaphore_register[BCS] = MI_SEMAPHORE_SYNC_RB; |
||
1868 | ring->semaphore_register[VECS] = MI_SEMAPHORE_SYNC_RVE; |
||
1869 | ring->signal_mbox[RCS] = GEN6_NOSYNC; |
||
1870 | ring->signal_mbox[VCS] = GEN6_VRSYNC; |
||
1871 | ring->signal_mbox[BCS] = GEN6_BRSYNC; |
||
1872 | ring->signal_mbox[VECS] = GEN6_VERSYNC; |
||
2332 | Serge | 1873 | } else if (IS_GEN5(dev)) { |
2339 | Serge | 1874 | ring->add_request = pc_render_add_request; |
3031 | serge | 1875 | ring->flush = gen4_render_ring_flush; |
2342 | Serge | 1876 | ring->get_seqno = pc_render_get_seqno; |
3480 | Serge | 1877 | ring->set_seqno = pc_render_set_seqno; |
3031 | serge | 1878 | ring->irq_get = gen5_ring_get_irq; |
1879 | ring->irq_put = gen5_ring_put_irq; |
||
4104 | Serge | 1880 | ring->irq_enable_mask = GT_RENDER_USER_INTERRUPT | |
1881 | GT_RENDER_PIPECTL_NOTIFY_INTERRUPT; |
||
3031 | serge | 1882 | } else { |
1883 | ring->add_request = i9xx_add_request; |
||
1884 | if (INTEL_INFO(dev)->gen < 4) |
||
1885 | ring->flush = gen2_render_ring_flush; |
||
1886 | else |
||
1887 | ring->flush = gen4_render_ring_flush; |
||
1888 | ring->get_seqno = ring_get_seqno; |
||
3480 | Serge | 1889 | ring->set_seqno = ring_set_seqno; |
3031 | serge | 1890 | if (IS_GEN2(dev)) { |
1891 | ring->irq_get = i8xx_ring_get_irq; |
||
1892 | ring->irq_put = i8xx_ring_put_irq; |
||
1893 | } else { |
||
1894 | ring->irq_get = i9xx_ring_get_irq; |
||
1895 | ring->irq_put = i9xx_ring_put_irq; |
||
1896 | } |
||
1897 | ring->irq_enable_mask = I915_USER_INTERRUPT; |
||
2332 | Serge | 1898 | } |
3031 | serge | 1899 | ring->write_tail = ring_write_tail; |
3243 | Serge | 1900 | if (IS_HASWELL(dev)) |
1901 | ring->dispatch_execbuffer = hsw_ring_dispatch_execbuffer; |
||
4560 | Serge | 1902 | else if (IS_GEN8(dev)) |
1903 | ring->dispatch_execbuffer = gen8_ring_dispatch_execbuffer; |
||
3243 | Serge | 1904 | else if (INTEL_INFO(dev)->gen >= 6) |
3031 | serge | 1905 | ring->dispatch_execbuffer = gen6_ring_dispatch_execbuffer; |
1906 | else if (INTEL_INFO(dev)->gen >= 4) |
||
1907 | ring->dispatch_execbuffer = i965_dispatch_execbuffer; |
||
1908 | else if (IS_I830(dev) || IS_845G(dev)) |
||
1909 | ring->dispatch_execbuffer = i830_dispatch_execbuffer; |
||
1910 | else |
||
1911 | ring->dispatch_execbuffer = i915_dispatch_execbuffer; |
||
1912 | ring->init = init_render_ring; |
||
1913 | ring->cleanup = render_ring_cleanup; |
||
2332 | Serge | 1914 | |
3243 | Serge | 1915 | /* Workaround batchbuffer to combat CS tlb bug. */ |
1916 | if (HAS_BROKEN_CS_TLB(dev)) { |
||
1917 | struct drm_i915_gem_object *obj; |
||
1918 | int ret; |
||
3031 | serge | 1919 | |
3243 | Serge | 1920 | obj = i915_gem_alloc_object(dev, I830_BATCH_LIMIT); |
1921 | if (obj == NULL) { |
||
1922 | DRM_ERROR("Failed to allocate batch bo\n"); |
||
1923 | return -ENOMEM; |
||
1924 | } |
||
1925 | |||
4104 | Serge | 1926 | ret = i915_gem_obj_ggtt_pin(obj, 0, true, false); |
3243 | Serge | 1927 | if (ret != 0) { |
1928 | drm_gem_object_unreference(&obj->base); |
||
1929 | DRM_ERROR("Failed to ping batch bo\n"); |
||
1930 | return ret; |
||
1931 | } |
||
1932 | |||
4104 | Serge | 1933 | ring->scratch.obj = obj; |
1934 | ring->scratch.gtt_offset = i915_gem_obj_ggtt_offset(obj); |
||
2332 | Serge | 1935 | } |
2340 | Serge | 1936 | |
2332 | Serge | 1937 | return intel_init_ring_buffer(dev, ring); |
1938 | } |
||
1939 | |||
3243 | Serge | 1940 | #if 0 |
1941 | int intel_render_ring_init_dri(struct drm_device *dev, u64 start, u32 size) |
||
1942 | { |
||
1943 | drm_i915_private_t *dev_priv = dev->dev_private; |
||
1944 | struct intel_ring_buffer *ring = &dev_priv->ring[RCS]; |
||
1945 | int ret; |
||
2332 | Serge | 1946 | |
3243 | Serge | 1947 | ring->name = "render ring"; |
1948 | ring->id = RCS; |
||
1949 | ring->mmio_base = RENDER_RING_BASE; |
||
1950 | |||
1951 | if (INTEL_INFO(dev)->gen >= 6) { |
||
1952 | /* non-kms not supported on gen6+ */ |
||
1953 | return -ENODEV; |
||
1954 | } |
||
1955 | |||
1956 | /* Note: gem is not supported on gen5/ilk without kms (the corresponding |
||
1957 | * gem_init ioctl returns with -ENODEV). Hence we do not need to set up |
||
1958 | * the special gen5 functions. */ |
||
1959 | ring->add_request = i9xx_add_request; |
||
1960 | if (INTEL_INFO(dev)->gen < 4) |
||
1961 | ring->flush = gen2_render_ring_flush; |
||
1962 | else |
||
1963 | ring->flush = gen4_render_ring_flush; |
||
1964 | ring->get_seqno = ring_get_seqno; |
||
3480 | Serge | 1965 | ring->set_seqno = ring_set_seqno; |
3243 | Serge | 1966 | if (IS_GEN2(dev)) { |
1967 | ring->irq_get = i8xx_ring_get_irq; |
||
1968 | ring->irq_put = i8xx_ring_put_irq; |
||
1969 | } else { |
||
1970 | ring->irq_get = i9xx_ring_get_irq; |
||
1971 | ring->irq_put = i9xx_ring_put_irq; |
||
1972 | } |
||
1973 | ring->irq_enable_mask = I915_USER_INTERRUPT; |
||
1974 | ring->write_tail = ring_write_tail; |
||
1975 | if (INTEL_INFO(dev)->gen >= 4) |
||
1976 | ring->dispatch_execbuffer = i965_dispatch_execbuffer; |
||
1977 | else if (IS_I830(dev) || IS_845G(dev)) |
||
1978 | ring->dispatch_execbuffer = i830_dispatch_execbuffer; |
||
1979 | else |
||
1980 | ring->dispatch_execbuffer = i915_dispatch_execbuffer; |
||
1981 | ring->init = init_render_ring; |
||
1982 | ring->cleanup = render_ring_cleanup; |
||
1983 | |||
1984 | ring->dev = dev; |
||
1985 | INIT_LIST_HEAD(&ring->active_list); |
||
1986 | INIT_LIST_HEAD(&ring->request_list); |
||
1987 | |||
1988 | ring->size = size; |
||
1989 | ring->effective_size = ring->size; |
||
1990 | if (IS_I830(ring->dev) || IS_845G(ring->dev)) |
||
1991 | ring->effective_size -= 128; |
||
1992 | |||
1993 | ring->virtual_start = ioremap_wc(start, size); |
||
1994 | if (ring->virtual_start == NULL) { |
||
1995 | DRM_ERROR("can not ioremap virtual address for" |
||
1996 | " ring buffer\n"); |
||
1997 | return -ENOMEM; |
||
1998 | } |
||
1999 | |||
2000 | if (!I915_NEED_GFX_HWS(dev)) { |
||
4104 | Serge | 2001 | ret = init_phys_status_page(ring); |
3243 | Serge | 2002 | if (ret) |
2003 | return ret; |
||
2004 | } |
||
2005 | |||
2006 | return 0; |
||
2007 | } |
||
2008 | #endif |
||
2009 | |||
2332 | Serge | 2010 | int intel_init_bsd_ring_buffer(struct drm_device *dev) |
2011 | { |
||
2012 | drm_i915_private_t *dev_priv = dev->dev_private; |
||
2013 | struct intel_ring_buffer *ring = &dev_priv->ring[VCS]; |
||
2014 | |||
3031 | serge | 2015 | ring->name = "bsd ring"; |
2016 | ring->id = VCS; |
||
2332 | Serge | 2017 | |
3031 | serge | 2018 | ring->write_tail = ring_write_tail; |
4560 | Serge | 2019 | if (INTEL_INFO(dev)->gen >= 6) { |
3031 | serge | 2020 | ring->mmio_base = GEN6_BSD_RING_BASE; |
2021 | /* gen6 bsd needs a special wa for tail updates */ |
||
2022 | if (IS_GEN6(dev)) |
||
2023 | ring->write_tail = gen6_bsd_ring_write_tail; |
||
4104 | Serge | 2024 | ring->flush = gen6_bsd_ring_flush; |
3031 | serge | 2025 | ring->add_request = gen6_add_request; |
2026 | ring->get_seqno = gen6_ring_get_seqno; |
||
3480 | Serge | 2027 | ring->set_seqno = ring_set_seqno; |
4560 | Serge | 2028 | if (INTEL_INFO(dev)->gen >= 8) { |
2029 | ring->irq_enable_mask = |
||
2030 | GT_RENDER_USER_INTERRUPT << GEN8_VCS1_IRQ_SHIFT; |
||
2031 | ring->irq_get = gen8_ring_get_irq; |
||
2032 | ring->irq_put = gen8_ring_put_irq; |
||
2033 | ring->dispatch_execbuffer = |
||
2034 | gen8_ring_dispatch_execbuffer; |
||
2035 | } else { |
||
4104 | Serge | 2036 | ring->irq_enable_mask = GT_BSD_USER_INTERRUPT; |
3031 | serge | 2037 | ring->irq_get = gen6_ring_get_irq; |
2038 | ring->irq_put = gen6_ring_put_irq; |
||
4560 | Serge | 2039 | ring->dispatch_execbuffer = |
2040 | gen6_ring_dispatch_execbuffer; |
||
2041 | } |
||
3031 | serge | 2042 | ring->sync_to = gen6_ring_sync; |
4104 | Serge | 2043 | ring->semaphore_register[RCS] = MI_SEMAPHORE_SYNC_VR; |
2044 | ring->semaphore_register[VCS] = MI_SEMAPHORE_SYNC_INVALID; |
||
2045 | ring->semaphore_register[BCS] = MI_SEMAPHORE_SYNC_VB; |
||
2046 | ring->semaphore_register[VECS] = MI_SEMAPHORE_SYNC_VVE; |
||
2047 | ring->signal_mbox[RCS] = GEN6_RVSYNC; |
||
2048 | ring->signal_mbox[VCS] = GEN6_NOSYNC; |
||
2049 | ring->signal_mbox[BCS] = GEN6_BVSYNC; |
||
2050 | ring->signal_mbox[VECS] = GEN6_VEVSYNC; |
||
3031 | serge | 2051 | } else { |
2052 | ring->mmio_base = BSD_RING_BASE; |
||
2053 | ring->flush = bsd_ring_flush; |
||
2054 | ring->add_request = i9xx_add_request; |
||
2055 | ring->get_seqno = ring_get_seqno; |
||
3480 | Serge | 2056 | ring->set_seqno = ring_set_seqno; |
3031 | serge | 2057 | if (IS_GEN5(dev)) { |
4104 | Serge | 2058 | ring->irq_enable_mask = ILK_BSD_USER_INTERRUPT; |
3031 | serge | 2059 | ring->irq_get = gen5_ring_get_irq; |
2060 | ring->irq_put = gen5_ring_put_irq; |
||
2061 | } else { |
||
2062 | ring->irq_enable_mask = I915_BSD_USER_INTERRUPT; |
||
2063 | ring->irq_get = i9xx_ring_get_irq; |
||
2064 | ring->irq_put = i9xx_ring_put_irq; |
||
2065 | } |
||
2066 | ring->dispatch_execbuffer = i965_dispatch_execbuffer; |
||
2067 | } |
||
2068 | ring->init = init_ring_common; |
||
2069 | |||
2332 | Serge | 2070 | return intel_init_ring_buffer(dev, ring); |
2071 | } |
||
2072 | |||
2073 | int intel_init_blt_ring_buffer(struct drm_device *dev) |
||
2074 | { |
||
2075 | drm_i915_private_t *dev_priv = dev->dev_private; |
||
2076 | struct intel_ring_buffer *ring = &dev_priv->ring[BCS]; |
||
2077 | |||
3031 | serge | 2078 | ring->name = "blitter ring"; |
2079 | ring->id = BCS; |
||
2332 | Serge | 2080 | |
3031 | serge | 2081 | ring->mmio_base = BLT_RING_BASE; |
2082 | ring->write_tail = ring_write_tail; |
||
4104 | Serge | 2083 | ring->flush = gen6_ring_flush; |
3031 | serge | 2084 | ring->add_request = gen6_add_request; |
2085 | ring->get_seqno = gen6_ring_get_seqno; |
||
3480 | Serge | 2086 | ring->set_seqno = ring_set_seqno; |
4560 | Serge | 2087 | if (INTEL_INFO(dev)->gen >= 8) { |
2088 | ring->irq_enable_mask = |
||
2089 | GT_RENDER_USER_INTERRUPT << GEN8_BCS_IRQ_SHIFT; |
||
2090 | ring->irq_get = gen8_ring_get_irq; |
||
2091 | ring->irq_put = gen8_ring_put_irq; |
||
2092 | ring->dispatch_execbuffer = gen8_ring_dispatch_execbuffer; |
||
2093 | } else { |
||
4104 | Serge | 2094 | ring->irq_enable_mask = GT_BLT_USER_INTERRUPT; |
3031 | serge | 2095 | ring->irq_get = gen6_ring_get_irq; |
2096 | ring->irq_put = gen6_ring_put_irq; |
||
2097 | ring->dispatch_execbuffer = gen6_ring_dispatch_execbuffer; |
||
4560 | Serge | 2098 | } |
3031 | serge | 2099 | ring->sync_to = gen6_ring_sync; |
4104 | Serge | 2100 | ring->semaphore_register[RCS] = MI_SEMAPHORE_SYNC_BR; |
2101 | ring->semaphore_register[VCS] = MI_SEMAPHORE_SYNC_BV; |
||
2102 | ring->semaphore_register[BCS] = MI_SEMAPHORE_SYNC_INVALID; |
||
2103 | ring->semaphore_register[VECS] = MI_SEMAPHORE_SYNC_BVE; |
||
2104 | ring->signal_mbox[RCS] = GEN6_RBSYNC; |
||
2105 | ring->signal_mbox[VCS] = GEN6_VBSYNC; |
||
2106 | ring->signal_mbox[BCS] = GEN6_NOSYNC; |
||
2107 | ring->signal_mbox[VECS] = GEN6_VEBSYNC; |
||
3031 | serge | 2108 | ring->init = init_ring_common; |
2109 | |||
2332 | Serge | 2110 | return intel_init_ring_buffer(dev, ring); |
2111 | } |
||
3031 | serge | 2112 | |
4104 | Serge | 2113 | int intel_init_vebox_ring_buffer(struct drm_device *dev) |
2114 | { |
||
2115 | drm_i915_private_t *dev_priv = dev->dev_private; |
||
2116 | struct intel_ring_buffer *ring = &dev_priv->ring[VECS]; |
||
2117 | |||
2118 | ring->name = "video enhancement ring"; |
||
2119 | ring->id = VECS; |
||
2120 | |||
2121 | ring->mmio_base = VEBOX_RING_BASE; |
||
2122 | ring->write_tail = ring_write_tail; |
||
2123 | ring->flush = gen6_ring_flush; |
||
2124 | ring->add_request = gen6_add_request; |
||
2125 | ring->get_seqno = gen6_ring_get_seqno; |
||
2126 | ring->set_seqno = ring_set_seqno; |
||
4560 | Serge | 2127 | |
2128 | if (INTEL_INFO(dev)->gen >= 8) { |
||
2129 | ring->irq_enable_mask = |
||
2130 | GT_RENDER_USER_INTERRUPT << GEN8_VECS_IRQ_SHIFT; |
||
2131 | ring->irq_get = gen8_ring_get_irq; |
||
2132 | ring->irq_put = gen8_ring_put_irq; |
||
2133 | ring->dispatch_execbuffer = gen8_ring_dispatch_execbuffer; |
||
2134 | } else { |
||
4104 | Serge | 2135 | ring->irq_enable_mask = PM_VEBOX_USER_INTERRUPT; |
2136 | ring->irq_get = hsw_vebox_get_irq; |
||
2137 | ring->irq_put = hsw_vebox_put_irq; |
||
2138 | ring->dispatch_execbuffer = gen6_ring_dispatch_execbuffer; |
||
4560 | Serge | 2139 | } |
4104 | Serge | 2140 | ring->sync_to = gen6_ring_sync; |
2141 | ring->semaphore_register[RCS] = MI_SEMAPHORE_SYNC_VER; |
||
2142 | ring->semaphore_register[VCS] = MI_SEMAPHORE_SYNC_VEV; |
||
2143 | ring->semaphore_register[BCS] = MI_SEMAPHORE_SYNC_VEB; |
||
2144 | ring->semaphore_register[VECS] = MI_SEMAPHORE_SYNC_INVALID; |
||
2145 | ring->signal_mbox[RCS] = GEN6_RVESYNC; |
||
2146 | ring->signal_mbox[VCS] = GEN6_VVESYNC; |
||
2147 | ring->signal_mbox[BCS] = GEN6_BVESYNC; |
||
2148 | ring->signal_mbox[VECS] = GEN6_NOSYNC; |
||
2149 | ring->init = init_ring_common; |
||
2150 | |||
2151 | return intel_init_ring_buffer(dev, ring); |
||
2152 | } |
||
2153 | |||
3031 | serge | 2154 | int |
2155 | intel_ring_flush_all_caches(struct intel_ring_buffer *ring) |
||
2156 | { |
||
2157 | int ret; |
||
2158 | |||
2159 | if (!ring->gpu_caches_dirty) |
||
2160 | return 0; |
||
2161 | |||
2162 | ret = ring->flush(ring, 0, I915_GEM_GPU_DOMAINS); |
||
2163 | if (ret) |
||
2164 | return ret; |
||
2165 | |||
2166 | trace_i915_gem_ring_flush(ring, 0, I915_GEM_GPU_DOMAINS); |
||
2167 | |||
2168 | ring->gpu_caches_dirty = false; |
||
2169 | return 0; |
||
2170 | } |
||
2171 | |||
2172 | int |
||
2173 | intel_ring_invalidate_all_caches(struct intel_ring_buffer *ring) |
||
2174 | { |
||
2175 | uint32_t flush_domains; |
||
2176 | int ret; |
||
2177 | |||
2178 | flush_domains = 0; |
||
2179 | if (ring->gpu_caches_dirty) |
||
2180 | flush_domains = I915_GEM_GPU_DOMAINS; |
||
2181 | |||
2182 | ret = ring->flush(ring, I915_GEM_GPU_DOMAINS, flush_domains); |
||
2183 | if (ret) |
||
2184 | return ret; |
||
2185 | |||
2186 | trace_i915_gem_ring_flush(ring, I915_GEM_GPU_DOMAINS, flush_domains); |
||
2187 | |||
2188 | ring->gpu_caches_dirty = false; |
||
2189 | return 0; |
||
2190 | }><>><>><>>>8)); |