| 37,6 → 37,7 |
|---|
| #include <drm/drmP.h> |
| #include "radeon_reg.h" |
| #include "radeon.h" |
| #include "radeon_trace.h" |
| |
| /* |
| * Fences |
| 120,7 → 121,7 |
|---|
| (*fence)->seq = ++rdev->fence_drv[ring].sync_seq[ring]; |
| (*fence)->ring = ring; |
| radeon_fence_ring_emit(rdev, ring, *fence); |
| // trace_radeon_fence_emit(rdev->ddev, (*fence)->seq); |
| trace_radeon_fence_emit(rdev->ddev, ring, (*fence)->seq); |
| return 0; |
| } |
| |
| 189,11 → 190,9 |
|---|
| } |
| } while (atomic64_xchg(&rdev->fence_drv[ring].last_seq, seq) > seq); |
| |
| if (wake) { |
| rdev->fence_drv[ring].last_activity = GetTimerTicks(); |
| if (wake) |
| wake_up_all(&rdev->fence_queue); |
| } |
| } |
| |
| /** |
| * radeon_fence_destroy - destroy a fence |
| 211,13 → 210,13 |
|---|
| } |
| |
| /** |
| * radeon_fence_seq_signaled - check if a fence sequeuce number has signaled |
| * radeon_fence_seq_signaled - check if a fence sequence number has signaled |
| * |
| * @rdev: radeon device pointer |
| * @seq: sequence number |
| * @ring: ring index the fence is associated with |
| * |
| * Check if the last singled fence sequnce number is >= the requested |
| * Check if the last signaled fence sequnce number is >= the requested |
| * sequence number (all asics). |
| * Returns true if the fence has signaled (current fence value |
| * is >= requested value) or false if it has not (current fence |
| 262,115 → 261,124 |
|---|
| } |
| |
| /** |
| * radeon_fence_wait_seq - wait for a specific sequence number |
| * radeon_fence_any_seq_signaled - check if any sequence number is signaled |
| * |
| * @rdev: radeon device pointer |
| * @target_seq: sequence number we want to wait for |
| * @ring: ring index the fence is associated with |
| * @seq: sequence numbers |
| * |
| * Check if the last signaled fence sequnce number is >= the requested |
| * sequence number (all asics). |
| * Returns true if any has signaled (current value is >= requested value) |
| * or false if it has not. Helper function for radeon_fence_wait_seq. |
| */ |
| static bool radeon_fence_any_seq_signaled(struct radeon_device *rdev, u64 *seq) |
| { |
| unsigned i; |
| |
| for (i = 0; i < RADEON_NUM_RINGS; ++i) { |
| if (seq[i] && radeon_fence_seq_signaled(rdev, seq[i], i)) |
| return true; |
| } |
| return false; |
| } |
| |
| /** |
| * radeon_fence_wait_seq - wait for a specific sequence numbers |
| * |
| * @rdev: radeon device pointer |
| * @target_seq: sequence number(s) we want to wait for |
| * @intr: use interruptable sleep |
| * @lock_ring: whether the ring should be locked or not |
| * |
| * Wait for the requested sequence number to be written (all asics). |
| * Wait for the requested sequence number(s) to be written by any ring |
| * (all asics). Sequnce number array is indexed by ring id. |
| * @intr selects whether to use interruptable (true) or non-interruptable |
| * (false) sleep when waiting for the sequence number. Helper function |
| * for radeon_fence_wait(), et al. |
| * for radeon_fence_wait_*(). |
| * Returns 0 if the sequence number has passed, error for all other cases. |
| * -EDEADLK is returned when a GPU lockup has been detected and the ring is |
| * marked as not ready so no further jobs get scheduled until a successful |
| * reset. |
| * -EDEADLK is returned when a GPU lockup has been detected. |
| */ |
| static int radeon_fence_wait_seq(struct radeon_device *rdev, u64 target_seq, |
| unsigned ring, bool intr, bool lock_ring) |
| static int radeon_fence_wait_seq(struct radeon_device *rdev, u64 *target_seq, |
| bool intr) |
| { |
| unsigned long timeout, last_activity; |
| uint64_t seq; |
| unsigned i; |
| uint64_t last_seq[RADEON_NUM_RINGS]; |
| bool signaled; |
| int r; |
| int i, r; |
| |
| while (target_seq > atomic64_read(&rdev->fence_drv[ring].last_seq)) { |
| if (!rdev->ring[ring].ready) { |
| return -EBUSY; |
| } |
| while (!radeon_fence_any_seq_signaled(rdev, target_seq)) { |
| |
| timeout = GetTimerTicks() - RADEON_FENCE_JIFFIES_TIMEOUT; |
| if (time_after(rdev->fence_drv[ring].last_activity, timeout)) { |
| /* the normal case, timeout is somewhere before last_activity */ |
| timeout = rdev->fence_drv[ring].last_activity - timeout; |
| } else { |
| /* either jiffies wrapped around, or no fence was signaled in the last 500ms |
| * anyway we will just wait for the minimum amount and then check for a lockup |
| */ |
| timeout = 1; |
| /* Save current sequence values, used to check for GPU lockups */ |
| for (i = 0; i < RADEON_NUM_RINGS; ++i) { |
| if (!target_seq[i]) |
| continue; |
| |
| last_seq[i] = atomic64_read(&rdev->fence_drv[i].last_seq); |
| trace_radeon_fence_wait_begin(rdev->ddev, i, target_seq[i]); |
| radeon_irq_kms_sw_irq_get(rdev, i); |
| } |
| seq = atomic64_read(&rdev->fence_drv[ring].last_seq); |
| /* Save current last activity valuee, used to check for GPU lockups */ |
| last_activity = rdev->fence_drv[ring].last_activity; |
| |
| // trace_radeon_fence_wait_begin(rdev->ddev, seq); |
| radeon_irq_kms_sw_irq_get(rdev, ring); |
| if (intr) { |
| r = wait_event_interruptible_timeout(rdev->fence_queue, |
| (signaled = radeon_fence_seq_signaled(rdev, target_seq, ring)), |
| timeout); |
| r = wait_event_interruptible_timeout(rdev->fence_queue, ( |
| (signaled = radeon_fence_any_seq_signaled(rdev, target_seq)) |
| || rdev->needs_reset), RADEON_FENCE_JIFFIES_TIMEOUT); |
| } else { |
| r = wait_event_timeout(rdev->fence_queue, |
| (signaled = radeon_fence_seq_signaled(rdev, target_seq, ring)), |
| timeout); |
| r = wait_event_timeout(rdev->fence_queue, ( |
| (signaled = radeon_fence_any_seq_signaled(rdev, target_seq)) |
| || rdev->needs_reset), RADEON_FENCE_JIFFIES_TIMEOUT); |
| } |
| radeon_irq_kms_sw_irq_put(rdev, ring); |
| if (unlikely(r < 0)) { |
| return r; |
| |
| for (i = 0; i < RADEON_NUM_RINGS; ++i) { |
| if (!target_seq[i]) |
| continue; |
| |
| radeon_irq_kms_sw_irq_put(rdev, i); |
| trace_radeon_fence_wait_end(rdev->ddev, i, target_seq[i]); |
| } |
| // trace_radeon_fence_wait_end(rdev->ddev, seq); |
| |
| if (unlikely(r < 0)) |
| return r; |
| |
| if (unlikely(!signaled)) { |
| if (rdev->needs_reset) |
| return -EDEADLK; |
| |
| /* we were interrupted for some reason and fence |
| * isn't signaled yet, resume waiting */ |
| if (r) { |
| if (r) |
| continue; |
| } |
| |
| /* check if sequence value has changed since last_activity */ |
| if (seq != atomic64_read(&rdev->fence_drv[ring].last_seq)) { |
| for (i = 0; i < RADEON_NUM_RINGS; ++i) { |
| if (!target_seq[i]) |
| continue; |
| } |
| |
| if (lock_ring) { |
| mutex_lock(&rdev->ring_lock); |
| if (last_seq[i] != atomic64_read(&rdev->fence_drv[i].last_seq)) |
| break; |
| } |
| |
| /* test if somebody else has already decided that this is a lockup */ |
| if (last_activity != rdev->fence_drv[ring].last_activity) { |
| if (lock_ring) { |
| mutex_unlock(&rdev->ring_lock); |
| } |
| if (i != RADEON_NUM_RINGS) |
| continue; |
| |
| for (i = 0; i < RADEON_NUM_RINGS; ++i) { |
| if (!target_seq[i]) |
| continue; |
| |
| if (radeon_ring_is_lockup(rdev, i, &rdev->ring[i])) |
| break; |
| } |
| |
| if (radeon_ring_is_lockup(rdev, ring, &rdev->ring[ring])) { |
| if (i < RADEON_NUM_RINGS) { |
| /* good news we believe it's a lockup */ |
| dev_warn(rdev->dev, "GPU lockup (waiting for 0x%016llx last fence id 0x%016llx)\n", |
| target_seq, seq); |
| dev_warn(rdev->dev, "GPU lockup (waiting for " |
| "0x%016llx last fence id 0x%016llx on" |
| " ring %d)\n", |
| target_seq[i], last_seq[i], i); |
| |
| /* change last activity so nobody else think there is a lockup */ |
| for (i = 0; i < RADEON_NUM_RINGS; ++i) { |
| rdev->fence_drv[i].last_activity = GetTimerTicks(); |
| } |
| |
| /* mark the ring as not ready any more */ |
| rdev->ring[ring].ready = false; |
| if (lock_ring) { |
| mutex_unlock(&rdev->ring_lock); |
| } |
| /* remember that we need an reset */ |
| rdev->needs_reset = true; |
| wake_up_all(&rdev->fence_queue); |
| return -EDEADLK; |
| } |
| |
| if (lock_ring) { |
| mutex_unlock(&rdev->ring_lock); |
| } |
| } |
| } |
| return 0; |
| } |
| |
| 387,6 → 395,7 |
|---|
| */ |
| int radeon_fence_wait(struct radeon_fence *fence, bool intr) |
| { |
| uint64_t seq[RADEON_NUM_RINGS] = {}; |
| int r; |
| |
| if (fence == NULL) { |
| 394,147 → 403,15 |
|---|
| return -EINVAL; |
| } |
| |
| r = radeon_fence_wait_seq(fence->rdev, fence->seq, |
| fence->ring, intr, true); |
| if (r) { |
| return r; |
| } |
| fence->seq = RADEON_FENCE_SIGNALED_SEQ; |
| seq[fence->ring] = fence->seq; |
| if (seq[fence->ring] == RADEON_FENCE_SIGNALED_SEQ) |
| return 0; |
| } |
| |
| static bool radeon_fence_any_seq_signaled(struct radeon_device *rdev, u64 *seq) |
| { |
| unsigned i; |
| |
| for (i = 0; i < RADEON_NUM_RINGS; ++i) { |
| if (seq[i] && radeon_fence_seq_signaled(rdev, seq[i], i)) { |
| return true; |
| } |
| } |
| return false; |
| } |
| |
| /** |
| * radeon_fence_wait_any_seq - wait for a sequence number on any ring |
| * |
| * @rdev: radeon device pointer |
| * @target_seq: sequence number(s) we want to wait for |
| * @intr: use interruptable sleep |
| * |
| * Wait for the requested sequence number(s) to be written by any ring |
| * (all asics). Sequnce number array is indexed by ring id. |
| * @intr selects whether to use interruptable (true) or non-interruptable |
| * (false) sleep when waiting for the sequence number. Helper function |
| * for radeon_fence_wait_any(), et al. |
| * Returns 0 if the sequence number has passed, error for all other cases. |
| */ |
| static int radeon_fence_wait_any_seq(struct radeon_device *rdev, |
| u64 *target_seq, bool intr) |
| { |
| unsigned long timeout, last_activity, tmp; |
| unsigned i, ring = RADEON_NUM_RINGS; |
| bool signaled; |
| int r; |
| |
| for (i = 0, last_activity = 0; i < RADEON_NUM_RINGS; ++i) { |
| if (!target_seq[i]) { |
| continue; |
| } |
| |
| /* use the most recent one as indicator */ |
| if (time_after(rdev->fence_drv[i].last_activity, last_activity)) { |
| last_activity = rdev->fence_drv[i].last_activity; |
| } |
| |
| /* For lockup detection just pick the lowest ring we are |
| * actively waiting for |
| */ |
| if (i < ring) { |
| ring = i; |
| } |
| } |
| |
| /* nothing to wait for ? */ |
| if (ring == RADEON_NUM_RINGS) { |
| return -ENOENT; |
| } |
| |
| while (!radeon_fence_any_seq_signaled(rdev, target_seq)) { |
| timeout = GetTimerTicks() - RADEON_FENCE_JIFFIES_TIMEOUT; |
| if (time_after(last_activity, timeout)) { |
| /* the normal case, timeout is somewhere before last_activity */ |
| timeout = last_activity - timeout; |
| } else { |
| /* either jiffies wrapped around, or no fence was signaled in the last 500ms |
| * anyway we will just wait for the minimum amount and then check for a lockup |
| */ |
| timeout = 1; |
| } |
| |
| // trace_radeon_fence_wait_begin(rdev->ddev, target_seq[ring]); |
| for (i = 0; i < RADEON_NUM_RINGS; ++i) { |
| if (target_seq[i]) { |
| radeon_irq_kms_sw_irq_get(rdev, i); |
| } |
| } |
| if (intr) { |
| r = wait_event_interruptible_timeout(rdev->fence_queue, |
| (signaled = radeon_fence_any_seq_signaled(rdev, target_seq)), |
| timeout); |
| } else { |
| r = wait_event_timeout(rdev->fence_queue, |
| (signaled = radeon_fence_any_seq_signaled(rdev, target_seq)), |
| timeout); |
| } |
| for (i = 0; i < RADEON_NUM_RINGS; ++i) { |
| if (target_seq[i]) { |
| radeon_irq_kms_sw_irq_put(rdev, i); |
| } |
| } |
| if (unlikely(r < 0)) { |
| r = radeon_fence_wait_seq(fence->rdev, seq, intr); |
| if (r) |
| return r; |
| } |
| // trace_radeon_fence_wait_end(rdev->ddev, seq); |
| |
| if (unlikely(!signaled)) { |
| /* we were interrupted for some reason and fence |
| * isn't signaled yet, resume waiting */ |
| if (r) { |
| continue; |
| } |
| |
| mutex_lock(&rdev->ring_lock); |
| for (i = 0, tmp = 0; i < RADEON_NUM_RINGS; ++i) { |
| if (time_after(rdev->fence_drv[i].last_activity, tmp)) { |
| tmp = rdev->fence_drv[i].last_activity; |
| } |
| } |
| /* test if somebody else has already decided that this is a lockup */ |
| if (last_activity != tmp) { |
| last_activity = tmp; |
| mutex_unlock(&rdev->ring_lock); |
| continue; |
| } |
| |
| if (radeon_ring_is_lockup(rdev, ring, &rdev->ring[ring])) { |
| /* good news we believe it's a lockup */ |
| dev_warn(rdev->dev, "GPU lockup (waiting for 0x%016llx)\n", |
| target_seq[ring]); |
| |
| /* change last activity so nobody else think there is a lockup */ |
| for (i = 0; i < RADEON_NUM_RINGS; ++i) { |
| rdev->fence_drv[i].last_activity = GetTimerTicks(); |
| } |
| |
| /* mark the ring as not ready any more */ |
| rdev->ring[ring].ready = false; |
| mutex_unlock(&rdev->ring_lock); |
| return -EDEADLK; |
| } |
| mutex_unlock(&rdev->ring_lock); |
| } |
| } |
| fence->seq = RADEON_FENCE_SIGNALED_SEQ; |
| return 0; |
| } |
| |
| 556,7 → 433,7 |
|---|
| bool intr) |
| { |
| uint64_t seq[RADEON_NUM_RINGS]; |
| unsigned i; |
| unsigned i, num_rings = 0; |
| int r; |
| |
| for (i = 0; i < RADEON_NUM_RINGS; ++i) { |
| 566,15 → 443,19 |
|---|
| continue; |
| } |
| |
| if (fences[i]->seq == RADEON_FENCE_SIGNALED_SEQ) { |
| /* something was allready signaled */ |
| seq[i] = fences[i]->seq; |
| ++num_rings; |
| |
| /* test if something was allready signaled */ |
| if (seq[i] == RADEON_FENCE_SIGNALED_SEQ) |
| return 0; |
| } |
| |
| seq[i] = fences[i]->seq; |
| } |
| /* nothing to wait for ? */ |
| if (num_rings == 0) |
| return -ENOENT; |
| |
| r = radeon_fence_wait_any_seq(rdev, seq, intr); |
| r = radeon_fence_wait_seq(rdev, seq, intr); |
| if (r) { |
| return r; |
| } |
| 582,7 → 463,7 |
|---|
| } |
| |
| /** |
| * radeon_fence_wait_next_locked - wait for the next fence to signal |
| * radeon_fence_wait_next - wait for the next fence to signal |
| * |
| * @rdev: radeon device pointer |
| * @ring: ring index the fence is associated with |
| 591,21 → 472,21 |
|---|
| * Returns 0 if the next fence has passed, error for all other cases. |
| * Caller must hold ring lock. |
| */ |
| int radeon_fence_wait_next_locked(struct radeon_device *rdev, int ring) |
| int radeon_fence_wait_next(struct radeon_device *rdev, int ring) |
| { |
| uint64_t seq; |
| uint64_t seq[RADEON_NUM_RINGS] = {}; |
| |
| seq = atomic64_read(&rdev->fence_drv[ring].last_seq) + 1ULL; |
| if (seq >= rdev->fence_drv[ring].sync_seq[ring]) { |
| seq[ring] = atomic64_read(&rdev->fence_drv[ring].last_seq) + 1ULL; |
| if (seq[ring] >= rdev->fence_drv[ring].sync_seq[ring]) { |
| /* nothing to wait for, last_seq is |
| already the last emited fence */ |
| return -ENOENT; |
| } |
| return radeon_fence_wait_seq(rdev, seq, ring, false, false); |
| return radeon_fence_wait_seq(rdev, seq, false); |
| } |
| |
| /** |
| * radeon_fence_wait_empty_locked - wait for all fences to signal |
| * radeon_fence_wait_empty - wait for all fences to signal |
| * |
| * @rdev: radeon device pointer |
| * @ring: ring index the fence is associated with |
| 614,16 → 495,20 |
|---|
| * Returns 0 if the fences have passed, error for all other cases. |
| * Caller must hold ring lock. |
| */ |
| int radeon_fence_wait_empty_locked(struct radeon_device *rdev, int ring) |
| int radeon_fence_wait_empty(struct radeon_device *rdev, int ring) |
| { |
| uint64_t seq = rdev->fence_drv[ring].sync_seq[ring]; |
| uint64_t seq[RADEON_NUM_RINGS] = {}; |
| int r; |
| |
| r = radeon_fence_wait_seq(rdev, seq, ring, false, false); |
| seq[ring] = rdev->fence_drv[ring].sync_seq[ring]; |
| if (!seq[ring]) |
| return 0; |
| |
| r = radeon_fence_wait_seq(rdev, seq, false); |
| if (r) { |
| if (r == -EDEADLK) { |
| if (r == -EDEADLK) |
| return -EDEADLK; |
| } |
| |
| dev_err(rdev->dev, "error waiting for ring[%d] to become idle (%d)\n", |
| ring, r); |
| } |
| 825,7 → 710,6 |
|---|
| for (i = 0; i < RADEON_NUM_RINGS; ++i) |
| rdev->fence_drv[ring].sync_seq[i] = 0; |
| atomic64_set(&rdev->fence_drv[ring].last_seq, 0); |
| rdev->fence_drv[ring].last_activity = GetTimerTicks(); |
| rdev->fence_drv[ring].initialized = false; |
| } |
| |
| 871,7 → 755,7 |
|---|
| for (ring = 0; ring < RADEON_NUM_RINGS; ring++) { |
| if (!rdev->fence_drv[ring].initialized) |
| continue; |
| r = radeon_fence_wait_empty_locked(rdev, ring); |
| r = radeon_fence_wait_empty(rdev, ring); |
| if (r) { |
| /* no need to trigger GPU reset as we are unloading */ |
| radeon_fence_driver_force_completion(rdev); |
| 918,6 → 802,8 |
|---|
| if (!rdev->fence_drv[i].initialized) |
| continue; |
| |
| radeon_fence_process(rdev, i); |
| |
| seq_printf(m, "--- ring %d ---\n", i); |
| seq_printf(m, "Last signaled fence 0x%016llx\n", |
| (unsigned long long)atomic64_read(&rdev->fence_drv[i].last_seq)); |
| 933,8 → 819,28 |
|---|
| return 0; |
| } |
| |
| /** |
| * radeon_debugfs_gpu_reset - manually trigger a gpu reset |
| * |
| * Manually trigger a gpu reset at the next fence wait. |
| */ |
| static int radeon_debugfs_gpu_reset(struct seq_file *m, void *data) |
| { |
| struct drm_info_node *node = (struct drm_info_node *) m->private; |
| struct drm_device *dev = node->minor->dev; |
| struct radeon_device *rdev = dev->dev_private; |
| |
| down_read(&rdev->exclusive_lock); |
| seq_printf(m, "%d\n", rdev->needs_reset); |
| rdev->needs_reset = true; |
| up_read(&rdev->exclusive_lock); |
| |
| return 0; |
| } |
| |
| static struct drm_info_list radeon_debugfs_fence_list[] = { |
| {"radeon_fence_info", &radeon_debugfs_fence_info, 0, NULL}, |
| {"radeon_gpu_reset", &radeon_debugfs_gpu_reset, 0, NULL} |
| }; |
| #endif |
| |
| 941,7 → 847,7 |
|---|
| int radeon_debugfs_fence_init(struct radeon_device *rdev) |
| { |
| #if defined(CONFIG_DEBUG_FS) |
| return radeon_debugfs_add_files(rdev, radeon_debugfs_fence_list, 1); |
| return radeon_debugfs_add_files(rdev, radeon_debugfs_fence_list, 2); |
| #else |
| return 0; |
| #endif |