Rev 5078 | Rev 6104 | Go to most recent revision | Show entire file | Regard whitespace | Details | Blame | Last modification | View Log | RSS feed
Rev 5078 | Rev 5271 | ||
---|---|---|---|
Line 27... | Line 27... | ||
27 | * Authors: |
27 | * Authors: |
28 | * Jerome Glisse |
28 | * Jerome Glisse |
29 | * Dave Airlie |
29 | * Dave Airlie |
30 | */ |
30 | */ |
31 | #include |
31 | #include |
32 | #include |
32 | #include |
33 | #include |
33 | #include |
34 | #include |
- | |
35 | #include |
34 | #include |
36 | #include |
35 | #include |
37 | #include |
36 | #include |
38 | #include "radeon_reg.h" |
37 | #include "radeon_reg.h" |
39 | #include "radeon.h" |
38 | #include "radeon.h" |
Line 109... | Line 108... | ||
109 | */ |
108 | */ |
110 | int radeon_fence_emit(struct radeon_device *rdev, |
109 | int radeon_fence_emit(struct radeon_device *rdev, |
111 | struct radeon_fence **fence, |
110 | struct radeon_fence **fence, |
112 | int ring) |
111 | int ring) |
113 | { |
112 | { |
- | 113 | u64 seq = ++rdev->fence_drv[ring].sync_seq[ring]; |
|
- | 114 | ||
114 | /* we are protected by the ring emission mutex */ |
115 | /* we are protected by the ring emission mutex */ |
115 | *fence = kmalloc(sizeof(struct radeon_fence), GFP_KERNEL); |
116 | *fence = kmalloc(sizeof(struct radeon_fence), GFP_KERNEL); |
116 | if ((*fence) == NULL) { |
117 | if ((*fence) == NULL) { |
117 | return -ENOMEM; |
118 | return -ENOMEM; |
118 | } |
119 | } |
119 | kref_init(&((*fence)->kref)); |
- | |
120 | (*fence)->rdev = rdev; |
120 | (*fence)->rdev = rdev; |
121 | (*fence)->seq = ++rdev->fence_drv[ring].sync_seq[ring]; |
121 | (*fence)->seq = seq; |
122 | (*fence)->ring = ring; |
122 | (*fence)->ring = ring; |
- | 123 | (*fence)->is_vm_update = false; |
|
- | 124 | fence_init(&(*fence)->base, &radeon_fence_ops, |
|
- | 125 | &rdev->fence_queue.lock, rdev->fence_context + ring, seq); |
|
123 | radeon_fence_ring_emit(rdev, ring, *fence); |
126 | radeon_fence_ring_emit(rdev, ring, *fence); |
124 | trace_radeon_fence_emit(rdev->ddev, ring, (*fence)->seq); |
127 | trace_radeon_fence_emit(rdev->ddev, ring, (*fence)->seq); |
125 | return 0; |
128 | return 0; |
126 | } |
129 | } |
Line 127... | Line 130... | ||
127 | 130 | ||
- | 131 | /** |
|
- | 132 | * radeon_fence_check_signaled - callback from fence_queue |
|
- | 133 | * |
|
- | 134 | * this function is called with fence_queue lock held, which is also used |
|
- | 135 | * for the fence locking itself, so unlocked variants are used for |
|
- | 136 | * fence_signal, and remove_wait_queue. |
|
- | 137 | */ |
|
- | 138 | static int radeon_fence_check_signaled(wait_queue_t *wait, unsigned mode, int flags, void *key) |
|
- | 139 | { |
|
- | 140 | struct radeon_fence *fence; |
|
- | 141 | u64 seq; |
|
- | 142 | ||
- | 143 | fence = container_of(wait, struct radeon_fence, fence_wake); |
|
- | 144 | ||
- | 145 | /* |
|
- | 146 | * We cannot use radeon_fence_process here because we're already |
|
- | 147 | * in the waitqueue, in a call from wake_up_all. |
|
- | 148 | */ |
|
- | 149 | seq = atomic64_read(&fence->rdev->fence_drv[fence->ring].last_seq); |
|
- | 150 | if (seq >= fence->seq) { |
|
- | 151 | int ret = fence_signal_locked(&fence->base); |
|
- | 152 | ||
- | 153 | if (!ret) |
|
- | 154 | FENCE_TRACE(&fence->base, "signaled from irq context\n"); |
|
- | 155 | else |
|
- | 156 | FENCE_TRACE(&fence->base, "was already signaled\n"); |
|
- | 157 | ||
- | 158 | radeon_irq_kms_sw_irq_put(fence->rdev, fence->ring); |
|
- | 159 | // __remove_wait_queue(&fence->rdev->fence_queue, &fence->fence_wake); |
|
- | 160 | fence_put(&fence->base); |
|
- | 161 | } else |
|
- | 162 | FENCE_TRACE(&fence->base, "pending\n"); |
|
- | 163 | return 0; |
|
- | 164 | } |
|
- | 165 | ||
128 | /** |
166 | /** |
129 | * radeon_fence_process - process a fence |
167 | * radeon_fence_activity - check for fence activity |
130 | * |
168 | * |
131 | * @rdev: radeon_device pointer |
169 | * @rdev: radeon_device pointer |
132 | * @ring: ring index the fence is associated with |
170 | * @ring: ring index the fence is associated with |
133 | * |
171 | * |
- | 172 | * Checks the current fence value and calculates the last |
|
134 | * Checks the current fence value and wakes the fence queue |
173 | * signalled fence value. Returns true if activity occured |
135 | * if the sequence number has increased (all asics). |
174 | * on the ring, and the fence_queue should be waken up. |
136 | */ |
175 | */ |
137 | void radeon_fence_process(struct radeon_device *rdev, int ring) |
176 | static bool radeon_fence_activity(struct radeon_device *rdev, int ring) |
138 | { |
177 | { |
139 | uint64_t seq, last_seq, last_emitted; |
178 | uint64_t seq, last_seq, last_emitted; |
140 | unsigned count_loop = 0; |
179 | unsigned count_loop = 0; |
Line 188... | Line 227... | ||
188 | */ |
227 | */ |
189 | break; |
228 | break; |
190 | } |
229 | } |
191 | } while (atomic64_xchg(&rdev->fence_drv[ring].last_seq, seq) > seq); |
230 | } while (atomic64_xchg(&rdev->fence_drv[ring].last_seq, seq) > seq); |
Line 192... | Line 231... | ||
192 | 231 | ||
193 | if (wake) |
232 | // if (seq < last_emitted) |
- | 233 | // radeon_fence_schedule_check(rdev, ring); |
|
- | 234 | ||
194 | wake_up_all(&rdev->fence_queue); |
235 | return wake; |
Line 195... | Line 236... | ||
195 | } |
236 | } |
196 | 237 | ||
197 | /** |
238 | /** |
198 | * radeon_fence_destroy - destroy a fence |
239 | * radeon_fence_check_lockup - check for hardware lockup |
199 | * |
240 | * |
- | 241 | * @work: delayed work item |
|
200 | * @kref: fence kref |
242 | * |
201 | * |
243 | * Checks for fence activity and if there is none probe |
202 | * Frees the fence object (all asics). |
244 | * the hardware if a lockup occured. |
203 | */ |
245 | */ |
- | 246 | static void radeon_fence_check_lockup(struct work_struct *work) |
|
204 | static void radeon_fence_destroy(struct kref *kref) |
247 | { |
- | 248 | struct radeon_fence_driver *fence_drv; |
|
Line 205... | Line 249... | ||
205 | { |
249 | struct radeon_device *rdev; |
- | 250 | int ring; |
|
- | 251 | ||
- | 252 | fence_drv = container_of(work, struct radeon_fence_driver, |
|
- | 253 | lockup_work.work); |
|
- | 254 | rdev = fence_drv->rdev; |
|
- | 255 | ring = fence_drv - &rdev->fence_drv[0]; |
|
- | 256 | ||
206 | struct radeon_fence *fence; |
257 | // if (!down_read_trylock(&rdev->exclusive_lock)) { |
- | 258 | // /* just reschedule the check if a reset is going on */ |
|
- | 259 | // radeon_fence_schedule_check(rdev, ring); |
|
- | 260 | // return; |
|
- | 261 | // } |
|
- | 262 | ||
- | 263 | if (fence_drv->delayed_irq && rdev->ddev->irq_enabled) { |
|
- | 264 | unsigned long irqflags; |
|
- | 265 | ||
- | 266 | fence_drv->delayed_irq = false; |
|
- | 267 | spin_lock_irqsave(&rdev->irq.lock, irqflags); |
|
- | 268 | radeon_irq_set(rdev); |
|
- | 269 | spin_unlock_irqrestore(&rdev->irq.lock, irqflags); |
|
- | 270 | } |
|
- | 271 | ||
- | 272 | if (radeon_fence_activity(rdev, ring)) |
|
- | 273 | wake_up_all(&rdev->fence_queue); |
|
- | 274 | ||
- | 275 | else if (radeon_ring_is_lockup(rdev, ring, &rdev->ring[ring])) { |
|
- | 276 | ||
- | 277 | /* good news we believe it's a lockup */ |
|
- | 278 | dev_warn(rdev->dev, "GPU lockup (current fence id " |
|
- | 279 | "0x%016llx last fence id 0x%016llx on ring %d)\n", |
|
- | 280 | (uint64_t)atomic64_read(&fence_drv->last_seq), |
|
- | 281 | fence_drv->sync_seq[ring], ring); |
|
- | 282 | ||
- | 283 | /* remember that we need an reset */ |
|
- | 284 | rdev->needs_reset = true; |
|
- | 285 | wake_up_all(&rdev->fence_queue); |
|
- | 286 | } |
|
- | 287 | // up_read(&rdev->exclusive_lock); |
|
- | 288 | } |
|
- | 289 | ||
- | 290 | /** |
|
- | 291 | * radeon_fence_process - process a fence |
|
- | 292 | * |
|
- | 293 | * @rdev: radeon_device pointer |
|
- | 294 | * @ring: ring index the fence is associated with |
|
- | 295 | * |
|
- | 296 | * Checks the current fence value and wakes the fence queue |
|
- | 297 | * if the sequence number has increased (all asics). |
|
- | 298 | */ |
|
- | 299 | void radeon_fence_process(struct radeon_device *rdev, int ring) |
|
207 | 300 | { |
|
Line 208... | Line 301... | ||
208 | fence = container_of(kref, struct radeon_fence, kref); |
301 | if (radeon_fence_activity(rdev, ring)) |
209 | kfree(fence); |
302 | wake_up_all(&rdev->fence_queue); |
210 | } |
303 | } |
Line 235... | Line 328... | ||
235 | return true; |
328 | return true; |
236 | } |
329 | } |
237 | return false; |
330 | return false; |
238 | } |
331 | } |
Line -... | Line 332... | ||
- | 332 | ||
- | 333 | static bool radeon_fence_is_signaled(struct fence *f) |
|
- | 334 | { |
|
- | 335 | struct radeon_fence *fence = to_radeon_fence(f); |
|
- | 336 | struct radeon_device *rdev = fence->rdev; |
|
- | 337 | unsigned ring = fence->ring; |
|
- | 338 | u64 seq = fence->seq; |
|
- | 339 | ||
- | 340 | if (atomic64_read(&rdev->fence_drv[ring].last_seq) >= seq) { |
|
- | 341 | return true; |
|
- | 342 | } |
|
- | 343 | ||
- | 344 | // if (down_read_trylock(&rdev->exclusive_lock)) |
|
- | 345 | { |
|
- | 346 | radeon_fence_process(rdev, ring); |
|
- | 347 | // up_read(&rdev->exclusive_lock); |
|
- | 348 | ||
- | 349 | if (atomic64_read(&rdev->fence_drv[ring].last_seq) >= seq) { |
|
- | 350 | return true; |
|
- | 351 | } |
|
- | 352 | } |
|
- | 353 | return false; |
|
- | 354 | } |
|
- | 355 | ||
- | 356 | /** |
|
- | 357 | * radeon_fence_enable_signaling - enable signalling on fence |
|
- | 358 | * @fence: fence |
|
- | 359 | * |
|
- | 360 | * This function is called with fence_queue lock held, and adds a callback |
|
- | 361 | * to fence_queue that checks if this fence is signaled, and if so it |
|
- | 362 | * signals the fence and removes itself. |
|
- | 363 | */ |
|
- | 364 | static bool radeon_fence_enable_signaling(struct fence *f) |
|
- | 365 | { |
|
- | 366 | struct radeon_fence *fence = to_radeon_fence(f); |
|
- | 367 | struct radeon_device *rdev = fence->rdev; |
|
- | 368 | ||
- | 369 | if (atomic64_read(&rdev->fence_drv[fence->ring].last_seq) >= fence->seq) |
|
- | 370 | return false; |
|
- | 371 | ||
- | 372 | // if (down_read_trylock(&rdev->exclusive_lock)) |
|
- | 373 | { |
|
- | 374 | radeon_irq_kms_sw_irq_get(rdev, fence->ring); |
|
- | 375 | ||
- | 376 | // if (radeon_fence_activity(rdev, fence->ring)) |
|
- | 377 | // wake_up_all_locked(&rdev->fence_queue); |
|
- | 378 | ||
- | 379 | /* did fence get signaled after we enabled the sw irq? */ |
|
- | 380 | if (atomic64_read(&rdev->fence_drv[fence->ring].last_seq) >= fence->seq) { |
|
- | 381 | radeon_irq_kms_sw_irq_put(rdev, fence->ring); |
|
- | 382 | // up_read(&rdev->exclusive_lock); |
|
- | 383 | return false; |
|
- | 384 | } |
|
- | 385 | ||
- | 386 | // up_read(&rdev->exclusive_lock); |
|
- | 387 | // } else { |
|
- | 388 | /* we're probably in a lockup, lets not fiddle too much */ |
|
- | 389 | // if (radeon_irq_kms_sw_irq_get_delayed(rdev, fence->ring)) |
|
- | 390 | // rdev->fence_drv[fence->ring].delayed_irq = true; |
|
- | 391 | // radeon_fence_schedule_check(rdev, fence->ring); |
|
- | 392 | } |
|
- | 393 | ||
- | 394 | // fence->fence_wake.flags = 0; |
|
- | 395 | // fence->fence_wake.private = NULL; |
|
- | 396 | fence->fence_wake.func = radeon_fence_check_signaled; |
|
- | 397 | __add_wait_queue(&rdev->fence_queue, &fence->fence_wake); |
|
- | 398 | fence_get(f); |
|
- | 399 | ||
- | 400 | FENCE_TRACE(&fence->base, "armed on ring %i!\n", fence->ring); |
|
- | 401 | return true; |
|
- | 402 | } |
|
239 | 403 | ||
240 | /** |
404 | /** |
241 | * radeon_fence_signaled - check if a fence has signaled |
405 | * radeon_fence_signaled - check if a fence has signaled |
242 | * |
406 | * |
243 | * @fence: radeon fence object |
407 | * @fence: radeon fence object |
244 | * |
408 | * |
245 | * Check if the requested fence has signaled (all asics). |
409 | * Check if the requested fence has signaled (all asics). |
246 | * Returns true if the fence has signaled or false if it has not. |
410 | * Returns true if the fence has signaled or false if it has not. |
247 | */ |
411 | */ |
248 | bool radeon_fence_signaled(struct radeon_fence *fence) |
412 | bool radeon_fence_signaled(struct radeon_fence *fence) |
249 | { |
413 | { |
250 | if (!fence) { |
- | |
251 | return true; |
- | |
252 | } |
- | |
253 | if (fence->seq == RADEON_FENCE_SIGNALED_SEQ) { |
414 | if (!fence) |
254 | return true; |
415 | return true; |
255 | } |
416 | |
- | 417 | if (radeon_fence_seq_signaled(fence->rdev, fence->seq, fence->ring)) { |
|
- | 418 | int ret; |
|
256 | if (radeon_fence_seq_signaled(fence->rdev, fence->seq, fence->ring)) { |
419 | |
- | 420 | ret = fence_signal(&fence->base); |
|
- | 421 | if (!ret) |
|
257 | fence->seq = RADEON_FENCE_SIGNALED_SEQ; |
422 | FENCE_TRACE(&fence->base, "signaled from radeon_fence_signaled\n"); |
258 | return true; |
423 | return true; |
259 | } |
424 | } |
260 | return false; |
425 | return false; |
Line 281... | Line 446... | ||
281 | } |
446 | } |
282 | return false; |
447 | return false; |
283 | } |
448 | } |
Line 284... | Line 449... | ||
284 | 449 | ||
285 | /** |
450 | /** |
286 | * radeon_fence_wait_seq - wait for a specific sequence numbers |
451 | * radeon_fence_wait_seq_timeout - wait for a specific sequence numbers |
287 | * |
452 | * |
288 | * @rdev: radeon device pointer |
453 | * @rdev: radeon device pointer |
289 | * @target_seq: sequence number(s) we want to wait for |
454 | * @target_seq: sequence number(s) we want to wait for |
- | 455 | * @intr: use interruptable sleep |
|
290 | * @intr: use interruptable sleep |
456 | * @timeout: maximum time to wait, or MAX_SCHEDULE_TIMEOUT for infinite wait |
291 | * |
457 | * |
292 | * Wait for the requested sequence number(s) to be written by any ring |
458 | * Wait for the requested sequence number(s) to be written by any ring |
293 | * (all asics). Sequnce number array is indexed by ring id. |
459 | * (all asics). Sequnce number array is indexed by ring id. |
294 | * @intr selects whether to use interruptable (true) or non-interruptable |
460 | * @intr selects whether to use interruptable (true) or non-interruptable |
295 | * (false) sleep when waiting for the sequence number. Helper function |
461 | * (false) sleep when waiting for the sequence number. Helper function |
- | 462 | * for radeon_fence_wait_*(). |
|
296 | * for radeon_fence_wait_*(). |
463 | * Returns remaining time if the sequence number has passed, 0 when |
297 | * Returns 0 if the sequence number has passed, error for all other cases. |
464 | * the wait timeout, or an error for all other cases. |
298 | * -EDEADLK is returned when a GPU lockup has been detected. |
465 | * -EDEADLK is returned when a GPU lockup has been detected. |
299 | */ |
466 | */ |
- | 467 | static long radeon_fence_wait_seq_timeout(struct radeon_device *rdev, |
|
300 | static int radeon_fence_wait_seq(struct radeon_device *rdev, u64 *target_seq, |
468 | u64 *target_seq, bool intr, |
301 | bool intr) |
469 | long timeout) |
302 | { |
- | |
303 | uint64_t last_seq[RADEON_NUM_RINGS]; |
470 | { |
304 | bool signaled; |
471 | long r; |
Line 305... | Line 472... | ||
305 | int i, r; |
472 | int i; |
- | 473 | ||
Line 306... | Line 474... | ||
306 | 474 | if (radeon_fence_any_seq_signaled(rdev, target_seq)) |
|
307 | while (!radeon_fence_any_seq_signaled(rdev, target_seq)) { |
475 | return timeout; |
308 | 476 | ||
309 | /* Save current sequence values, used to check for GPU lockups */ |
477 | /* enable IRQs and tracing */ |
Line 310... | Line -... | ||
310 | for (i = 0; i < RADEON_NUM_RINGS; ++i) { |
- | |
311 | if (!target_seq[i]) |
478 | for (i = 0; i < RADEON_NUM_RINGS; ++i) { |
312 | continue; |
479 | if (!target_seq[i]) |
313 | 480 | continue; |
|
Line 314... | Line 481... | ||
314 | last_seq[i] = atomic64_read(&rdev->fence_drv[i].last_seq); |
481 | |
315 | trace_radeon_fence_wait_begin(rdev->ddev, i, target_seq[i]); |
482 | trace_radeon_fence_wait_begin(rdev->ddev, i, target_seq[i]); |
316 | radeon_irq_kms_sw_irq_get(rdev, i); |
483 | radeon_irq_kms_sw_irq_get(rdev, i); |
317 | } |
484 | } |
318 | 485 | ||
319 | if (intr) { |
486 | if (intr) { |
320 | r = wait_event_interruptible_timeout(rdev->fence_queue, ( |
487 | r = wait_event_interruptible_timeout(rdev->fence_queue, ( |
321 | (signaled = radeon_fence_any_seq_signaled(rdev, target_seq)) |
488 | radeon_fence_any_seq_signaled(rdev, target_seq) |
322 | || rdev->needs_reset), RADEON_FENCE_JIFFIES_TIMEOUT); |
489 | || rdev->needs_reset), timeout); |
Line -... | Line 490... | ||
- | 490 | } else { |
|
- | 491 | r = wait_event_timeout(rdev->fence_queue, ( |
|
- | 492 | radeon_fence_any_seq_signaled(rdev, target_seq) |
|
323 | } else { |
493 | || rdev->needs_reset), timeout); |
324 | r = wait_event_timeout(rdev->fence_queue, ( |
494 | } |
325 | (signaled = radeon_fence_any_seq_signaled(rdev, target_seq)) |
495 | |
Line 326... | Line 496... | ||
326 | || rdev->needs_reset), RADEON_FENCE_JIFFIES_TIMEOUT); |
496 | if (rdev->needs_reset) |
327 | } |
497 | r = -EDEADLK; |
328 | 498 | ||
Line 329... | Line -... | ||
329 | for (i = 0; i < RADEON_NUM_RINGS; ++i) { |
- | |
330 | if (!target_seq[i]) |
499 | for (i = 0; i < RADEON_NUM_RINGS; ++i) { |
331 | continue; |
- | |
332 | - | ||
333 | radeon_irq_kms_sw_irq_put(rdev, i); |
- | |
334 | trace_radeon_fence_wait_end(rdev->ddev, i, target_seq[i]); |
- | |
335 | } |
- | |
336 | - | ||
337 | if (unlikely(r < 0)) |
- | |
338 | return r; |
- | |
339 | - | ||
340 | if (unlikely(!signaled)) { |
- | |
341 | if (rdev->needs_reset) |
- | |
342 | return -EDEADLK; |
- | |
343 | - | ||
344 | /* we were interrupted for some reason and fence |
- | |
345 | * isn't signaled yet, resume waiting */ |
- | |
346 | if (r) |
- | |
347 | continue; |
- | |
348 | - | ||
349 | for (i = 0; i < RADEON_NUM_RINGS; ++i) { |
- | |
350 | if (!target_seq[i]) |
- | |
351 | continue; |
- | |
352 | - | ||
353 | if (last_seq[i] != atomic64_read(&rdev->fence_drv[i].last_seq)) |
- | |
354 | break; |
- | |
355 | } |
- | |
356 | - | ||
357 | if (i != RADEON_NUM_RINGS) |
- | |
358 | continue; |
- | |
359 | - | ||
360 | for (i = 0; i < RADEON_NUM_RINGS; ++i) { |
- | |
361 | if (!target_seq[i]) |
- | |
362 | continue; |
- | |
363 | - | ||
364 | if (radeon_ring_is_lockup(rdev, i, &rdev->ring[i])) |
- | |
365 | break; |
- | |
366 | } |
- | |
367 | - | ||
368 | if (i < RADEON_NUM_RINGS) { |
- | |
369 | /* good news we believe it's a lockup */ |
- | |
370 | dev_warn(rdev->dev, "GPU lockup (waiting for " |
- | |
371 | "0x%016llx last fence id 0x%016llx on" |
- | |
372 | " ring %d)\n", |
- | |
373 | target_seq[i], last_seq[i], i); |
- | |
374 | - | ||
375 | /* remember that we need an reset */ |
500 | if (!target_seq[i]) |
Line 376... | Line 501... | ||
376 | rdev->needs_reset = true; |
501 | continue; |
377 | wake_up_all(&rdev->fence_queue); |
502 | |
378 | return -EDEADLK; |
503 | radeon_irq_kms_sw_irq_put(rdev, i); |
379 | } |
504 | trace_radeon_fence_wait_end(rdev->ddev, i, target_seq[i]); |
380 | } |
505 | } |
381 | } |
506 | |
382 | return 0; |
507 | return r; |
383 | } |
508 | } |
384 | 509 | ||
385 | /** |
510 | /** |
386 | * radeon_fence_wait - wait for a fence to signal |
511 | * radeon_fence_wait - wait for a fence to signal |
387 | * |
512 | * |
388 | * @fence: radeon fence object |
513 | * @fence: radeon fence object |
389 | * @intr: use interruptable sleep |
514 | * @intr: use interruptible sleep |
390 | * |
515 | * |
Line -... | Line 516... | ||
- | 516 | * Wait for the requested fence to signal (all asics). |
|
391 | * Wait for the requested fence to signal (all asics). |
517 | * @intr selects whether to use interruptable (true) or non-interruptable |
- | 518 | * (false) sleep when waiting for the fence. |
|
392 | * @intr selects whether to use interruptable (true) or non-interruptable |
519 | * Returns 0 if the fence has passed, error for all other cases. |
393 | * (false) sleep when waiting for the fence. |
520 | */ |
394 | * Returns 0 if the fence has passed, error for all other cases. |
521 | int radeon_fence_wait(struct radeon_fence *fence, bool intr) |
- | 522 | { |
|
- | 523 | uint64_t seq[RADEON_NUM_RINGS] = {}; |
|
Line 395... | Line 524... | ||
395 | */ |
524 | long r; |
396 | int radeon_fence_wait(struct radeon_fence *fence, bool intr) |
- | |
397 | { |
- | |
398 | uint64_t seq[RADEON_NUM_RINGS] = {}; |
- | |
399 | int r; |
525 | |
400 | 526 | /* |
|
401 | if (fence == NULL) { |
527 | * This function should not be called on !radeon fences. |
- | 528 | * If this is the case, it would mean this function can |
|
Line -... | Line 529... | ||
- | 529 | * also be called on radeon fences belonging to another card. |
|
- | 530 | * exclusive_lock is not held in that case. |
|
402 | WARN(1, "Querying an invalid fence : %p !\n", fence); |
531 | */ |
403 | return -EINVAL; |
532 | if (WARN_ON_ONCE(!to_radeon_fence(&fence->base))) |
404 | } |
533 | return fence_wait(&fence->base, intr); |
Line 405... | Line 534... | ||
405 | 534 | ||
406 | seq[fence->ring] = fence->seq; |
535 | seq[fence->ring] = fence->seq; |
Line 432... | Line 561... | ||
432 | struct radeon_fence **fences, |
561 | struct radeon_fence **fences, |
433 | bool intr) |
562 | bool intr) |
434 | { |
563 | { |
435 | uint64_t seq[RADEON_NUM_RINGS]; |
564 | uint64_t seq[RADEON_NUM_RINGS]; |
436 | unsigned i, num_rings = 0; |
565 | unsigned i, num_rings = 0; |
437 | int r; |
566 | long r; |
Line 438... | Line 567... | ||
438 | 567 | ||
439 | for (i = 0; i < RADEON_NUM_RINGS; ++i) { |
568 | for (i = 0; i < RADEON_NUM_RINGS; ++i) { |
Line 440... | Line 569... | ||
440 | seq[i] = 0; |
569 | seq[i] = 0; |
441 | 570 | ||
442 | if (!fences[i]) { |
571 | if (!fences[i]) { |
Line 443... | Line 572... | ||
443 | continue; |
572 | continue; |
444 | } |
573 | } |
445 | - | ||
446 | seq[i] = fences[i]->seq; |
- | |
447 | ++num_rings; |
- | |
448 | - | ||
449 | /* test if something was allready signaled */ |
574 | |
Line 450... | Line 575... | ||
450 | if (seq[i] == RADEON_FENCE_SIGNALED_SEQ) |
575 | seq[i] = fences[i]->seq; |
451 | return 0; |
576 | ++num_rings; |
452 | } |
577 | } |
Line 453... | Line 578... | ||
453 | 578 | ||
454 | /* nothing to wait for ? */ |
579 | /* nothing to wait for ? */ |
455 | if (num_rings == 0) |
580 | if (num_rings == 0) |
456 | return -ENOENT; |
581 | return -ENOENT; |
457 | 582 | ||
458 | r = radeon_fence_wait_seq(rdev, seq, intr); |
583 | r = radeon_fence_wait_seq_timeout(rdev, seq, intr, MAX_SCHEDULE_TIMEOUT); |
Line 473... | Line 598... | ||
473 | * Caller must hold ring lock. |
598 | * Caller must hold ring lock. |
474 | */ |
599 | */ |
475 | int radeon_fence_wait_next(struct radeon_device *rdev, int ring) |
600 | int radeon_fence_wait_next(struct radeon_device *rdev, int ring) |
476 | { |
601 | { |
477 | uint64_t seq[RADEON_NUM_RINGS] = {}; |
602 | uint64_t seq[RADEON_NUM_RINGS] = {}; |
- | 603 | long r; |
|
Line 478... | Line 604... | ||
478 | 604 | ||
479 | seq[ring] = atomic64_read(&rdev->fence_drv[ring].last_seq) + 1ULL; |
605 | seq[ring] = atomic64_read(&rdev->fence_drv[ring].last_seq) + 1ULL; |
480 | if (seq[ring] >= rdev->fence_drv[ring].sync_seq[ring]) { |
606 | if (seq[ring] >= rdev->fence_drv[ring].sync_seq[ring]) { |
481 | /* nothing to wait for, last_seq is |
607 | /* nothing to wait for, last_seq is |
482 | already the last emited fence */ |
608 | already the last emited fence */ |
483 | return -ENOENT; |
609 | return -ENOENT; |
484 | } |
610 | } |
- | 611 | r = radeon_fence_wait_seq_timeout(rdev, seq, false, MAX_SCHEDULE_TIMEOUT); |
|
- | 612 | if (r < 0) |
|
- | 613 | return r; |
|
485 | return radeon_fence_wait_seq(rdev, seq, false); |
614 | return 0; |
Line 486... | Line 615... | ||
486 | } |
615 | } |
487 | 616 | ||
488 | /** |
617 | /** |
Line 496... | Line 625... | ||
496 | * Caller must hold ring lock. |
625 | * Caller must hold ring lock. |
497 | */ |
626 | */ |
498 | int radeon_fence_wait_empty(struct radeon_device *rdev, int ring) |
627 | int radeon_fence_wait_empty(struct radeon_device *rdev, int ring) |
499 | { |
628 | { |
500 | uint64_t seq[RADEON_NUM_RINGS] = {}; |
629 | uint64_t seq[RADEON_NUM_RINGS] = {}; |
501 | int r; |
630 | long r; |
Line 502... | Line 631... | ||
502 | 631 | ||
503 | seq[ring] = rdev->fence_drv[ring].sync_seq[ring]; |
632 | seq[ring] = rdev->fence_drv[ring].sync_seq[ring]; |
504 | if (!seq[ring]) |
633 | if (!seq[ring]) |
Line 505... | Line 634... | ||
505 | return 0; |
634 | return 0; |
506 | 635 | ||
507 | r = radeon_fence_wait_seq(rdev, seq, false); |
636 | r = radeon_fence_wait_seq_timeout(rdev, seq, false, MAX_SCHEDULE_TIMEOUT); |
508 | if (r) { |
637 | if (r < 0) { |
Line 509... | Line 638... | ||
509 | if (r == -EDEADLK) |
638 | if (r == -EDEADLK) |
510 | return -EDEADLK; |
639 | return -EDEADLK; |
511 | 640 | ||
512 | dev_err(rdev->dev, "error waiting for ring[%d] to become idle (%d)\n", |
641 | dev_err(rdev->dev, "error waiting for ring[%d] to become idle (%ld)\n", |
513 | ring, r); |
642 | ring, r); |
Line 523... | Line 652... | ||
523 | * Take a reference on a fence (all asics). |
652 | * Take a reference on a fence (all asics). |
524 | * Returns the fence. |
653 | * Returns the fence. |
525 | */ |
654 | */ |
526 | struct radeon_fence *radeon_fence_ref(struct radeon_fence *fence) |
655 | struct radeon_fence *radeon_fence_ref(struct radeon_fence *fence) |
527 | { |
656 | { |
528 | kref_get(&fence->kref); |
657 | fence_get(&fence->base); |
529 | return fence; |
658 | return fence; |
530 | } |
659 | } |
Line 531... | Line 660... | ||
531 | 660 | ||
532 | /** |
661 | /** |
Line 540... | Line 669... | ||
540 | { |
669 | { |
541 | struct radeon_fence *tmp = *fence; |
670 | struct radeon_fence *tmp = *fence; |
Line 542... | Line 671... | ||
542 | 671 | ||
543 | *fence = NULL; |
672 | *fence = NULL; |
544 | if (tmp) { |
673 | if (tmp) { |
545 | kref_put(&tmp->kref, radeon_fence_destroy); |
674 | fence_put(&tmp->base); |
546 | } |
675 | } |
Line 547... | Line 676... | ||
547 | } |
676 | } |
548 | 677 | ||
Line 709... | Line 838... | ||
709 | rdev->fence_drv[ring].gpu_addr = 0; |
838 | rdev->fence_drv[ring].gpu_addr = 0; |
710 | for (i = 0; i < RADEON_NUM_RINGS; ++i) |
839 | for (i = 0; i < RADEON_NUM_RINGS; ++i) |
711 | rdev->fence_drv[ring].sync_seq[i] = 0; |
840 | rdev->fence_drv[ring].sync_seq[i] = 0; |
712 | atomic64_set(&rdev->fence_drv[ring].last_seq, 0); |
841 | atomic64_set(&rdev->fence_drv[ring].last_seq, 0); |
713 | rdev->fence_drv[ring].initialized = false; |
842 | rdev->fence_drv[ring].initialized = false; |
- | 843 | INIT_DELAYED_WORK(&rdev->fence_drv[ring].lockup_work, |
|
- | 844 | radeon_fence_check_lockup); |
|
- | 845 | rdev->fence_drv[ring].rdev = rdev; |
|
714 | } |
846 | } |
Line 715... | Line 847... | ||
715 | 847 | ||
716 | /** |
848 | /** |
717 | * radeon_fence_driver_init - init the fence driver |
849 | * radeon_fence_driver_init - init the fence driver |
Line 756... | Line 888... | ||
756 | if (!rdev->fence_drv[ring].initialized) |
888 | if (!rdev->fence_drv[ring].initialized) |
757 | continue; |
889 | continue; |
758 | r = radeon_fence_wait_empty(rdev, ring); |
890 | r = radeon_fence_wait_empty(rdev, ring); |
759 | if (r) { |
891 | if (r) { |
760 | /* no need to trigger GPU reset as we are unloading */ |
892 | /* no need to trigger GPU reset as we are unloading */ |
761 | radeon_fence_driver_force_completion(rdev); |
893 | radeon_fence_driver_force_completion(rdev, ring); |
762 | } |
894 | } |
763 | wake_up_all(&rdev->fence_queue); |
895 | wake_up_all(&rdev->fence_queue); |
764 | radeon_scratch_free(rdev, rdev->fence_drv[ring].scratch_reg); |
896 | radeon_scratch_free(rdev, rdev->fence_drv[ring].scratch_reg); |
765 | rdev->fence_drv[ring].initialized = false; |
897 | rdev->fence_drv[ring].initialized = false; |
766 | } |
898 | } |
Line 769... | Line 901... | ||
769 | 901 | ||
770 | /** |
902 | /** |
771 | * radeon_fence_driver_force_completion - force all fence waiter to complete |
903 | * radeon_fence_driver_force_completion - force all fence waiter to complete |
772 | * |
904 | * |
- | 905 | * @rdev: radeon device pointer |
|
773 | * @rdev: radeon device pointer |
906 | * @ring: the ring to complete |
774 | * |
907 | * |
775 | * In case of GPU reset failure make sure no process keep waiting on fence |
908 | * In case of GPU reset failure make sure no process keep waiting on fence |
776 | * that will never complete. |
909 | * that will never complete. |
777 | */ |
910 | */ |
778 | void radeon_fence_driver_force_completion(struct radeon_device *rdev) |
911 | void radeon_fence_driver_force_completion(struct radeon_device *rdev, int ring) |
779 | { |
- | |
780 | int ring; |
- | |
781 | - | ||
782 | for (ring = 0; ring < RADEON_NUM_RINGS; ring++) { |
912 | { |
783 | if (!rdev->fence_drv[ring].initialized) |
- | |
784 | continue; |
913 | if (rdev->fence_drv[ring].initialized) { |
785 | radeon_fence_write(rdev, rdev->fence_drv[ring].sync_seq[ring], ring); |
914 | radeon_fence_write(rdev, rdev->fence_drv[ring].sync_seq[ring], ring); |
786 | } |
915 | } |
Line 831... | Line 960... | ||
831 | struct radeon_device *rdev = dev->dev_private; |
960 | struct radeon_device *rdev = dev->dev_private; |
Line 832... | Line 961... | ||
832 | 961 | ||
833 | down_read(&rdev->exclusive_lock); |
962 | down_read(&rdev->exclusive_lock); |
834 | seq_printf(m, "%d\n", rdev->needs_reset); |
963 | seq_printf(m, "%d\n", rdev->needs_reset); |
- | 964 | rdev->needs_reset = true; |
|
835 | rdev->needs_reset = true; |
965 | wake_up_all(&rdev->fence_queue); |
Line 836... | Line 966... | ||
836 | up_read(&rdev->exclusive_lock); |
966 | up_read(&rdev->exclusive_lock); |
837 | 967 | ||
Line 850... | Line 980... | ||
850 | return radeon_debugfs_add_files(rdev, radeon_debugfs_fence_list, 2); |
980 | return radeon_debugfs_add_files(rdev, radeon_debugfs_fence_list, 2); |
851 | #else |
981 | #else |
852 | return 0; |
982 | return 0; |
853 | #endif |
983 | #endif |
854 | }>>>>>>>=>>>>>>>>>>=>> |
984 | } |
- | 985 | ||
- | 986 | static const char *radeon_fence_get_driver_name(struct fence *fence) |
|
- | 987 | { |
|
- | 988 | return "radeon"; |
|
- | 989 | } |
|
- | 990 | ||
- | 991 | static const char *radeon_fence_get_timeline_name(struct fence *f) |
|
- | 992 | { |
|
- | 993 | struct radeon_fence *fence = to_radeon_fence(f); |
|
- | 994 | switch (fence->ring) { |
|
- | 995 | case RADEON_RING_TYPE_GFX_INDEX: return "radeon.gfx"; |
|
- | 996 | case CAYMAN_RING_TYPE_CP1_INDEX: return "radeon.cp1"; |
|
- | 997 | case CAYMAN_RING_TYPE_CP2_INDEX: return "radeon.cp2"; |
|
- | 998 | case R600_RING_TYPE_DMA_INDEX: return "radeon.dma"; |
|
- | 999 | case CAYMAN_RING_TYPE_DMA1_INDEX: return "radeon.dma1"; |
|
- | 1000 | case R600_RING_TYPE_UVD_INDEX: return "radeon.uvd"; |
|
- | 1001 | case TN_RING_TYPE_VCE1_INDEX: return "radeon.vce1"; |
|
- | 1002 | case TN_RING_TYPE_VCE2_INDEX: return "radeon.vce2"; |
|
- | 1003 | default: WARN_ON_ONCE(1); return "radeon.unk"; |
|
- | 1004 | } |
|
- | 1005 | } |
|
- | 1006 | ||
- | 1007 | static inline bool radeon_test_signaled(struct radeon_fence *fence) |
|
- | 1008 | { |
|
- | 1009 | return test_bit(FENCE_FLAG_SIGNALED_BIT, &fence->base.flags); |
|
- | 1010 | } |
|
- | 1011 | ||
- | 1012 | static signed long radeon_fence_default_wait(struct fence *f, bool intr, |
|
- | 1013 | signed long t) |
|
- | 1014 | { |
|
- | 1015 | struct radeon_fence *fence = to_radeon_fence(f); |
|
- | 1016 | struct radeon_device *rdev = fence->rdev; |
|
- | 1017 | bool signaled; |
|
- | 1018 | ||
- | 1019 | fence_enable_sw_signaling(&fence->base); |
|
- | 1020 | ||
- | 1021 | /* |
|
- | 1022 | * This function has to return -EDEADLK, but cannot hold |
|
- | 1023 | * exclusive_lock during the wait because some callers |
|
- | 1024 | * may already hold it. This means checking needs_reset without |
|
- | 1025 | * lock, and not fiddling with any gpu internals. |
|
- | 1026 | * |
|
- | 1027 | * The callback installed with fence_enable_sw_signaling will |
|
- | 1028 | * run before our wait_event_*timeout call, so we will see |
|
- | 1029 | * both the signaled fence and the changes to needs_reset. |
|
- | 1030 | */ |
|
- | 1031 | ||
- | 1032 | if (intr) |
|
- | 1033 | t = wait_event_interruptible_timeout(rdev->fence_queue, |
|
- | 1034 | ((signaled = radeon_test_signaled(fence)) || |
|
- | 1035 | rdev->needs_reset), t); |
|
- | 1036 | else |
|
- | 1037 | t = wait_event_timeout(rdev->fence_queue, |
|
- | 1038 | ((signaled = radeon_test_signaled(fence)) || |
|
- | 1039 | rdev->needs_reset), t); |
|
- | 1040 | ||
- | 1041 | if (t > 0 && !signaled) |
|
- | 1042 | return -EDEADLK; |
|
- | 1043 | return t; |
|
- | 1044 | } |
|
- | 1045 | ||
- | 1046 | const struct fence_ops radeon_fence_ops = { |
|
- | 1047 | .get_driver_name = radeon_fence_get_driver_name, |
|
- | 1048 | .get_timeline_name = radeon_fence_get_timeline_name, |
|
- | 1049 | .enable_signaling = radeon_fence_enable_signaling, |
|
- | 1050 | .signaled = radeon_fence_is_signaled, |
|
- | 1051 | .wait = radeon_fence_default_wait, |
|
- | 1052 | .release = NULL, |
|
- | 1053 | };>>>>>>=>>>>>>>>>>>=>> |