WebSVN – Kolibri OS – Diff – /drivers/video/drm/radeon/radeon_fence.c

 #include
 //#include
 #include
 #include
 #include
-#include "drmP.h"
+#include
-#include "drm.h"
 #include "radeon_reg.h"
 #include "radeon.h"
+/*
+ * Fences
+ * Fences mark an event in the GPUs pipeline and are used
+ * for GPU/CPU synchronization.  When the fence is written,
+ * it is expected that all buffers associated with that fence
+ * are no longer in use by the associated ring on the GPU and
+ * that the the relevant GPU caches have been flushed.  Whether
+ * we use a scratch register or memory location depends on the asic
+ * and whether writeback is enabled.
+ */
+/**
+ * radeon_fence_write - write a fence value
+ *
+ * @rdev: radeon_device pointer
+ * @seq: sequence number to write
+ * @ring: ring index the fence is associated with
+ *
+ * Writes a fence value to memory or a scratch register (all asics).
+ */
-static void radeon_fence_write(struct radeon_device *rdev, u32 seq)
+static void radeon_fence_write(struct radeon_device *rdev, u32 seq, int ring)
-{
-	if (rdev->wb.enabled) {
-		u32 scratch_index;
-		if (rdev->wb.use_event)
+{
-			scratch_index = R600_WB_EVENT_OFFSET + rdev->fence_drv.scratch_reg - rdev->scratch.reg_base;
-		else
+	struct radeon_fence_driver *drv = &rdev->fence_drv[ring];
-			scratch_index = RADEON_WB_SCRATCH_OFFSET + rdev->fence_drv.scratch_reg - rdev->scratch.reg_base;
+	if (likely(rdev->wb.enabled || !drv->scratch_reg)) {
-		rdev->wb.wb[scratch_index/4] = cpu_to_le32(seq);;
+		*drv->cpu_addr = cpu_to_le32(seq);
-	} else
+	} else {
-		WREG32(rdev->fence_drv.scratch_reg, seq);
+		WREG32(drv->scratch_reg, seq);
-}
-static u32 radeon_fence_read(struct radeon_device *rdev)
-{
-	u32 seq;
-	if (rdev->wb.enabled) {
-		u32 scratch_index;
-		if (rdev->wb.use_event)
-			scratch_index = R600_WB_EVENT_OFFSET + rdev->fence_drv.scratch_reg - rdev->scratch.reg_base;
-		else
-			scratch_index = RADEON_WB_SCRATCH_OFFSET + rdev->fence_drv.scratch_reg - rdev->scratch.reg_base;
-		seq = le32_to_cpu(rdev->wb.wb[scratch_index/4]);
-	} else
-		seq = RREG32(rdev->fence_drv.scratch_reg);
 	return seq;
+}
+/**
+ * radeon_fence_read - read a fence value
+ *
+ * @rdev: radeon_device pointer
+ * @ring: ring index the fence is associated with
+ *
+ * Reads a fence value from memory or a scratch register (all asics).
-}
+ * Returns the value of the fence read from memory or register.
  */
+static u32 radeon_fence_read(struct radeon_device *rdev, int ring)
-int radeon_fence_emit(struct radeon_device *rdev, struct radeon_fence *fence)
+{
-{
+	struct radeon_fence_driver *drv = &rdev->fence_drv[ring];
-	unsigned long irq_flags;
+	u32 seq = 0;
-	write_lock_irqsave(&rdev->fence_drv.lock, irq_flags);
+	if (likely(rdev->wb.enabled || !drv->scratch_reg)) {
-	if (fence->emited) {
+		seq = le32_to_cpu(*drv->cpu_addr);
+	} else {
+		seq = RREG32(drv->scratch_reg);
+	}
+	return seq;
-		write_unlock_irqrestore(&rdev->fence_drv.lock, irq_flags);
+}
+/**
-		return 0;
+ * radeon_fence_emit - emit a fence on the requested ring
-	}
+ *
-	fence->seq = atomic_add_return(1, &rdev->fence_drv.seq);
+ * @rdev: radeon_device pointer
+ * @fence: radeon fence object
+ * @ring: ring index the fence is associated with
-	if (!rdev->cp.ready)
+ *
-		/* FIXME: cp is not running assume everythings is done right
+ * Emits a fence command on the requested ring (all asics).
+ * Returns 0 on success, -ENOMEM on failure.
+ */
-		 * away
+int radeon_fence_emit(struct radeon_device *rdev,
-		 */
+		      struct radeon_fence **fence,
+		      int ring)
+{
+	/* we are protected by the ring emission mutex */
-		radeon_fence_write(rdev, fence->seq);
+	*fence = kmalloc(sizeof(struct radeon_fence), GFP_KERNEL);
-	else
+	if ((*fence) == NULL) {
-		radeon_fence_ring_emit(rdev, fence);
+		return -ENOMEM;
+	}
+	kref_init(&((*fence)->kref));
+	(*fence)->rdev = rdev;
-//   trace_radeon_fence_emit(rdev->ddev, fence->seq);
+	(*fence)->seq = ++rdev->fence_drv[ring].sync_seq[ring];
-	fence->emited = true;
+	(*fence)->ring = ring;
-	list_move_tail(&fence->list, &rdev->fence_drv.emited);
+	radeon_fence_ring_emit(rdev, ring, *fence);
+//   trace_radeon_fence_emit(rdev->ddev, (*fence)->seq);
+	return 0;
+}
+/**
+ * radeon_fence_process - process a fence
+ *
+ * @rdev: radeon_device pointer
+ * @ring: ring index the fence is associated with
-	write_unlock_irqrestore(&rdev->fence_drv.lock, irq_flags);
+ *
-	return 0;
+ * Checks the current fence value and wakes the fence queue
-}
+ * if the sequence number has increased (all asics).
+ */
-static bool radeon_fence_poll_locked(struct radeon_device *rdev)
 void radeon_fence_process(struct radeon_device *rdev, int ring)
-	struct radeon_fence *fence;
-	struct list_head *i, *n;
+{
-	uint32_t seq;
+	uint64_t seq, last_seq, last_emitted;
-	bool wake = false;
+	unsigned count_loop = 0;
-	unsigned long cjiffies;
+	bool wake = false;
-	seq = radeon_fence_read(rdev);
+	/* Note there is a scenario here for an infinite loop but it's
-	if (seq != rdev->fence_drv.last_seq) {
+	 * very unlikely to happen. For it to happen, the current polling
-		rdev->fence_drv.last_seq = seq;
+	 * process need to be interrupted by another process and another
-        rdev->fence_drv.last_jiffies = GetTimerTicks();
+	 * process needs to update the last_seq btw the atomic read and
-        rdev->fence_drv.last_timeout = RADEON_FENCE_JIFFIES_TIMEOUT;
+	 * xchg of the current process.
-	} else {
+	 *
-        cjiffies = GetTimerTicks();
+	 * More over for this to go in infinite loop there need to be
-		if (time_after(cjiffies, rdev->fence_drv.last_jiffies)) {
+	 * continuously new fence signaled ie radeon_fence_read needs
-			cjiffies -= rdev->fence_drv.last_jiffies;
+	 * to return a different value each time for both the currently
-			if (time_after(rdev->fence_drv.last_timeout, cjiffies)) {
+	 * polling process and the other process that xchg the last_seq
-				/* update the timeout */
+	 * btw atomic read and xchg of the current process. And the
-				rdev->fence_drv.last_timeout -= cjiffies;
+	 * value the other process set as last seq must be higher than
-			} else {
-				/* the 500ms timeout is elapsed we should test
+	 * the seq value we just read. Which means that current process
-				 * for GPU lockup
+	 * need to be interrupted after radeon_fence_read and before
-				 */
+	 * atomic xchg.
-				rdev->fence_drv.last_timeout = 1;
+	 *
-			}
+	 * To be even more safe we count the number of time we loop and
-		} else {
+	 * we bail after 10 loop just accepting the fact that we might
-			/* wrap around update last jiffies, we will just wait
+	 * have temporarly set the last_seq not to the true real last
-			 * a little longer
-			 */
-			rdev->fence_drv.last_jiffies = cjiffies;
+	 * seq but to an older one.
-	}
+	 */
-		return false;
+	last_seq = atomic64_read(&rdev->fence_drv[ring].last_seq);
-	}
+	do {
+		last_emitted = rdev->fence_drv[ring].sync_seq[ring];
+		seq = radeon_fence_read(rdev, ring);
+		seq |= last_seq & 0xffffffff00000000LL;
+		if (seq < last_seq) {
-	n = NULL;
+			seq &= 0xffffffff;
-	list_for_each(i, &rdev->fence_drv.emited) {
+			seq |= last_emitted & 0xffffffff00000000LL;
-		fence = list_entry(i, struct radeon_fence, list);
-		if (fence->seq == seq) {
+		}
-			n = i;
-			break;
-		}
-	}
-	/* all fence previous to this one are considered as signaled */
-	if (n) {
-        kevent_t event;
-        event.code = -1;
-		i = n;
-		do {
-			n = i->prev;
+		if (seq <= last_seq || seq > last_emitted) {
-			list_move_tail(i, &rdev->fence_drv.signaled);
+			break;
-			fence = list_entry(i, struct radeon_fence, list);
 			fence->signaled = true;
+		/* If we loop over we don't want to return without
+		 * checking if a fence is signaled as it means that the
+		 * seq we just read is different from the previous on.
+		 */
+		wake = true;
+		last_seq = seq;
+		if ((count_loop++) > 10) {
+			/* We looped over too many time leave with the
+			 * fact that we might have set an older fence
+			 * seq then the current real last seq as signaled
+			 * by the hw.
+			 */
+			break;
+		}
-//            dbgprintf("fence %x done\n", fence);
+	} while (atomic64_xchg(&rdev->fence_drv[ring].last_seq, seq) > seq);
-            RaiseEvent(fence->evnt, 0, &event);
 			i = n;
+	if (wake) {
+		rdev->fence_drv[ring].last_activity = GetTimerTicks();
+		wake_up_all(&rdev->fence_queue);
+	}
-		} while (i != &rdev->fence_drv.emited);
+}
+/**
-		wake = true;
+ * radeon_fence_destroy - destroy a fence
+ *
-	return wake;
+ * @kref: fence kref
-}
-int radeon_fence_create(struct radeon_device *rdev, struct radeon_fence **fence)
-{
-	unsigned long irq_flags;
-	*fence = kmalloc(sizeof(struct radeon_fence), GFP_KERNEL);
-	if ((*fence) == NULL) {
-		return -ENOMEM;
-	}
-    (*fence)->evnt = CreateEvent(NULL, MANUAL_DESTROY);
-//	kref_init(&((*fence)->kref));
-	(*fence)->rdev = rdev;
+ *
-	(*fence)->emited = false;
-	(*fence)->signaled = false;
+ * Frees the fence object (all asics).
-	(*fence)->seq = 0;
+ */
+static void radeon_fence_destroy(struct kref *kref)
+{
+	struct radeon_fence *fence;
+	fence = container_of(kref, struct radeon_fence, kref);
+	kfree(fence);
+}
+/**
+ * radeon_fence_seq_signaled - check if a fence sequeuce number has signaled
+ *
+ * @rdev: radeon device pointer
+ * @seq: sequence number
+ * @ring: ring index the fence is associated with
+ *
+ * Check if the last singled fence sequnce number is >= the requested
+ * sequence number (all asics).
+ * Returns true if the fence has signaled (current fence value
+ * is >= requested value) or false if it has not (current fence
+ * value is < the requested value.  Helper function for
+ * radeon_fence_signaled().
+ */
+static bool radeon_fence_seq_signaled(struct radeon_device *rdev,
+				      u64 seq, unsigned ring)
+{
+	if (atomic64_read(&rdev->fence_drv[ring].last_seq) >= seq) {
+		return true;
+	}
+	/* poll new last sequence at least once */
+	radeon_fence_process(rdev, ring);
+	if (atomic64_read(&rdev->fence_drv[ring].last_seq) >= seq) {
+		return true;
+	}
+	return false;
+}
 	INIT_LIST_HEAD(&(*fence)->list);
 /**
-	write_lock_irqsave(&rdev->fence_drv.lock, irq_flags);
-	list_add_tail(&(*fence)->list, &rdev->fence_drv.created);
-	write_unlock_irqrestore(&rdev->fence_drv.lock, irq_flags);
-	return 0;
+ * radeon_fence_signaled - check if a fence has signaled
+ *
+ * @fence: radeon fence object
+ *
+ * Check if the requested fence has signaled (all asics).
+ * Returns true if the fence has signaled or false if it has not.
+				 */
+bool radeon_fence_signaled(struct radeon_fence *fence)
 bool radeon_fence_signaled(struct radeon_fence *fence)
+	if (!fence) {
+		return true;
+	}
+	if (fence->seq == RADEON_FENCE_SIGNALED_SEQ) {
+		return true;
+	}
+	if (radeon_fence_seq_signaled(fence->rdev, fence->seq, fence->ring)) {
+		fence->seq = RADEON_FENCE_SIGNALED_SEQ;
+		return true;
+	}
+	return false;
+}
+/**
+ * radeon_fence_wait_seq - wait for a specific sequence number
+ *
+ * @rdev: radeon device pointer
+ * @target_seq: sequence number we want to wait for
+ * @ring: ring index the fence is associated with
+ * @intr: use interruptable sleep
+ * @lock_ring: whether the ring should be locked or not
+ *
+ * Wait for the requested sequence number to be written (all asics).
+ * @intr selects whether to use interruptable (true) or non-interruptable
+ * (false) sleep when waiting for the sequence number.  Helper function
+ * for radeon_fence_wait(), et al.
+ * Returns 0 if the sequence number has passed, error for all other cases.
+ * -EDEADLK is returned when a GPU lockup has been detected and the ring is
+ * marked as not ready so no further jobs get scheduled until a successful
+ * reset.
+ */
+static int radeon_fence_wait_seq(struct radeon_device *rdev, u64 target_seq,
+				 unsigned ring, bool intr, bool lock_ring)
+{
+	unsigned long timeout, last_activity;
+	uint64_t seq;
+	unsigned i;
+	bool signaled;
+	int r;
+	while (target_seq > atomic64_read(&rdev->fence_drv[ring].last_seq)) {
+		if (!rdev->ring[ring].ready) {
+			return -EBUSY;
+			}
+		timeout = GetTimerTicks() - RADEON_FENCE_JIFFIES_TIMEOUT;
+		if (time_after(rdev->fence_drv[ring].last_activity, timeout)) {
+			/* the normal case, timeout is somewhere before last_activity */
+			timeout = rdev->fence_drv[ring].last_activity - timeout;
+		} else {
+			/* either jiffies wrapped around, or no fence was signaled in the last 500ms
+			 * anyway we will just wait for the minimum amount and then check for a lockup
+			 */
+			timeout = 1;
+	}
+		seq = atomic64_read(&rdev->fence_drv[ring].last_seq);
+		/* Save current last activity valuee, used to check for GPU lockups */
+		last_activity = rdev->fence_drv[ring].last_activity;
+//		trace_radeon_fence_wait_begin(rdev->ddev, seq);
+		radeon_irq_kms_sw_irq_get(rdev, ring);
+//       if (intr) {
+//           r = wait_event_interruptible_timeout(rdev->fence_queue,
+//               (signaled = radeon_fence_seq_signaled(rdev, target_seq, ring)),
+//               timeout);
+//                } else {
+//           r = wait_event_timeout(rdev->fence_queue,
+//               (signaled = radeon_fence_seq_signaled(rdev, target_seq, ring)),
+//               timeout);
+//   }
+        delay(1);
+		radeon_irq_kms_sw_irq_put(rdev, ring);
+//       if (unlikely(r < 0)) {
+//           return r;
+//       }
+//		trace_radeon_fence_wait_end(rdev->ddev, seq);
+		if (unlikely(!signaled)) {
+			/* we were interrupted for some reason and fence
+			 * isn't signaled yet, resume waiting */
+			if (r) {
+				continue;
+	}
+			/* check if sequence value has changed since last_activity */
+			if (seq != atomic64_read(&rdev->fence_drv[ring].last_seq)) {
+				continue;
+			}
+			if (lock_ring) {
+				mutex_lock(&rdev->ring_lock);
+	}
+			/* test if somebody else has already decided that this is a lockup */
+			if (last_activity != rdev->fence_drv[ring].last_activity) {
+				if (lock_ring) {
+					mutex_unlock(&rdev->ring_lock);
+				}
+				continue;
+			}
+			if (radeon_ring_is_lockup(rdev, ring, &rdev->ring[ring])) {
+				/* good news we believe it's a lockup */
+				dev_warn(rdev->dev, "GPU lockup (waiting for 0x%016llx last fence id 0x%016llx)\n",
+					 target_seq, seq);
+				/* change last activity so nobody else think there is a lockup */
+				for (i = 0; i < RADEON_NUM_RINGS; ++i) {
+					rdev->fence_drv[i].last_activity = jiffies;
-{
-	unsigned long irq_flags;
+				}
-	bool signaled = false;
-	if (!fence)
-		return true;
+				/* mark the ring as not ready any more */
-	if (fence->rdev->gpu_lockup)
-		return true;
+				rdev->ring[ring].ready = false;
-	write_lock_irqsave(&fence->rdev->fence_drv.lock, irq_flags);
-	signaled = fence->signaled;
-	/* if we are shuting down report all fence as signaled */
-	if (fence->rdev->shutdown) {
+				if (lock_ring) {
-		signaled = true;
-	}
+					mutex_unlock(&rdev->ring_lock);
 	if (!fence->emited) {
+				return -EDEADLK;
+			}
+			if (lock_ring) {
+				mutex_unlock(&rdev->ring_lock);
+			}
+		}
+	}
+	return 0;
+}
-		WARN(1, "Querying an unemited fence : %p !\n", fence);
+/**
-		signaled = true;
+ * radeon_fence_wait - wait for a fence to signal
-	}
-	if (!signaled) {
-		radeon_fence_poll_locked(fence->rdev);
 		signaled = fence->signaled;
  * @fence: radeon fence object
-	write_unlock_irqrestore(&fence->rdev->fence_drv.lock, irq_flags);
+ * @intr: use interruptable sleep
+ *
+ * Wait for the requested fence to signal (all asics).
+ * @intr selects whether to use interruptable (true) or non-interruptable
+ * (false) sleep when waiting for the fence.
+ * Returns 0 if the fence has passed, error for all other cases.
+ */
-	return signaled;
+int radeon_fence_wait(struct radeon_fence *fence, bool intr)
+{
-int radeon_fence_wait(struct radeon_fence *fence, bool intr)
+	int r;
+	if (fence == NULL) {
+		WARN(1, "Querying an invalid fence : %p !\n", fence);
+		return -EINVAL;
+	}
+	r = radeon_fence_wait_seq(fence->rdev, fence->seq,
+				  fence->ring, intr, true);
+	if (r) {
+		return r;
+	}
-	struct radeon_device *rdev;
+	fence->seq = RADEON_FENCE_SIGNALED_SEQ;
-	unsigned long irq_flags, timeout;
+		return 0;
-	u32 seq;
+}
-	int r;
-	if (fence == NULL) {
-		WARN(1, "Querying an invalid fence : %p !\n", fence);
-		return 0;
-	}
-	rdev = fence->rdev;
+static bool radeon_fence_any_seq_signaled(struct radeon_device *rdev, u64 *seq)
+{
-	if (radeon_fence_signaled(fence)) {
+	unsigned i;
+	for (i = 0; i < RADEON_NUM_RINGS; ++i) {
+		if (seq[i] && radeon_fence_seq_signaled(rdev, seq[i], i)) {
+			return true;
+		}
+	}
+	return false;
+}
+/**
+ * radeon_fence_wait_any_seq - wait for a sequence number on any ring
+ *
+ * @rdev: radeon device pointer
+ * @target_seq: sequence number(s) we want to wait for
+ * @intr: use interruptable sleep
+ *
+ * Wait for the requested sequence number(s) to be written by any ring
-		return 0;
+ * (all asics).  Sequnce number array is indexed by ring id.
-	}
+ * @intr selects whether to use interruptable (true) or non-interruptable
-	timeout = rdev->fence_drv.last_timeout;
+ * (false) sleep when waiting for the sequence number.  Helper function
-retry:
+ * for radeon_fence_wait_any(), et al.
-	/* save current sequence used to check for GPU lockup */
-	seq = rdev->fence_drv.last_seq;
-//   trace_radeon_fence_wait_begin(rdev->ddev, seq);
-	if (intr) {
+ * Returns 0 if the sequence number has passed, error for all other cases.
+ */
+static int radeon_fence_wait_any_seq(struct radeon_device *rdev,
+				     u64 *target_seq, bool intr)
+{
+	unsigned long timeout, last_activity, tmp;
-		radeon_irq_kms_sw_irq_get(rdev);
+	unsigned i, ring = RADEON_NUM_RINGS;
+	bool signaled;
+	int r;
+	for (i = 0, last_activity = 0; i < RADEON_NUM_RINGS; ++i) {
+		if (!target_seq[i]) {
-//       r = wait_event_interruptible_timeout(rdev->fence_drv.queue,
+			continue;
+		}
-//               radeon_fence_signaled(fence), timeout);
+		/* use the most recent one as indicator */
+		if (time_after(rdev->fence_drv[i].last_activity, last_activity)) {
+			last_activity = rdev->fence_drv[i].last_activity;
+	}
+		/* For lockup detection just pick the lowest ring we are
+		 * actively waiting for
+		 */
+		if (i < ring) {
+			ring = i;
+		}
+	}
+	/* nothing to wait for ? */
+	if (ring == RADEON_NUM_RINGS) {
+		return -ENOENT;
+	}
+	while (!radeon_fence_any_seq_signaled(rdev, target_seq)) {
+        timeout = GetTimerTicks() - RADEON_FENCE_JIFFIES_TIMEOUT;
+		if (time_after(last_activity, timeout)) {
+			/* the normal case, timeout is somewhere before last_activity */
+			timeout = last_activity - timeout;
+		} else {
+			/* either jiffies wrapped around, or no fence was signaled in the last 500ms
+			 * anyway we will just wait for the minimum amount and then check for a lockup
+			 */
+			timeout = 1;
+		}
+//		trace_radeon_fence_wait_begin(rdev->ddev, target_seq[ring]);
+		for (i = 0; i < RADEON_NUM_RINGS; ++i) {
+			if (target_seq[i]) {
 				radeon_irq_kms_sw_irq_get(rdev, i);
+			}
-        WaitEvent(fence->evnt);
+		}
-		radeon_irq_kms_sw_irq_put(rdev);
+//        WaitEvent(fence->evnt);
-		if (unlikely(r < 0)) {
 			return r;
-		}
+		r = 1;
+		for (i = 0; i < RADEON_NUM_RINGS; ++i) {
+			if (target_seq[i]) {
+				radeon_irq_kms_sw_irq_put(rdev, i);
+			}
+		}
+		if (unlikely(r < 0)) {
+			return r;
+		}
+//   trace_radeon_fence_wait_end(rdev->ddev, seq);
+		if (unlikely(!signaled)) {
-	} else {
+			/* we were interrupted for some reason and fence
-		radeon_irq_kms_sw_irq_get(rdev);
+			 * isn't signaled yet, resume waiting */
-//       r = wait_event_timeout(rdev->fence_drv.queue,
-//            radeon_fence_signaled(fence), timeout);
+		if (r) {
-        WaitEvent(fence->evnt);
+				continue;
+			}
-		radeon_irq_kms_sw_irq_put(rdev);
-	}
+			mutex_lock(&rdev->ring_lock);
+			for (i = 0, tmp = 0; i < RADEON_NUM_RINGS; ++i) {
-//   trace_radeon_fence_wait_end(rdev->ddev, seq);
+				if (time_after(rdev->fence_drv[i].last_activity, tmp)) {
-	if (unlikely(!radeon_fence_signaled(fence))) {
+					tmp = rdev->fence_drv[i].last_activity;
+				}
-		/* we were interrupted for some reason and fence isn't
+			}
-		 * isn't signaled yet, resume wait
-		 */
-		if (r) {
-			timeout = r;
-			goto retry;
-		}
+			/* test if somebody else has already decided that this is a lockup */
-		/* don't protect read access to rdev->fence_drv.last_seq
+			if (last_activity != tmp) {
-		 * if we experiencing a lockup the value doesn't change
+				last_activity = tmp;
+				mutex_unlock(&rdev->ring_lock);
+				continue;
-		 */
+		}
-		if (seq == rdev->fence_drv.last_seq && radeon_gpu_is_lockup(rdev)) {
-			/* good news we believe it's a lockup */
-			WARN(1, "GPU lockup (waiting for 0x%08X last fence id 0x%08X)\n",
-			     fence->seq, seq);
-			/* FIXME: what should we do ? marking everyone
-			 * as signaled for now
+			if (radeon_ring_is_lockup(rdev, ring, &rdev->ring[ring])) {
-			 */
+			/* good news we believe it's a lockup */
-			rdev->gpu_lockup = true;
+				dev_warn(rdev->dev, "GPU lockup (waiting for 0x%016llx)\n",
-//           r = radeon_gpu_reset(rdev);
+					 target_seq[ring]);
+				/* change last activity so nobody else think there is a lockup */
-//           if (r)
+				for (i = 0; i < RADEON_NUM_RINGS; ++i) {
+					rdev->fence_drv[i].last_activity = GetTimerTicks();
+				}
+				/* mark the ring as not ready any more */
+				rdev->ring[ring].ready = false;
+				mutex_unlock(&rdev->ring_lock);
+				return -EDEADLK;
+			}
+			mutex_unlock(&rdev->ring_lock);
+			}
-//               return r;
+	}
+    return 0;
+}
             return true;
+/**
-//           radeon_fence_write(rdev, fence->seq);
+ * radeon_fence_wait_any - wait for a fence to signal on any ring
 //           rdev->gpu_lockup = false;
+ * @rdev: radeon device pointer
+ * @fences: radeon fence object(s)
+ * @intr: use interruptable sleep
-			}
+ *
+ * Wait for any requested fence to signal (all asics).  Fence
+ * array is indexed by ring id.  @intr selects whether to use
+ * interruptable (true) or non-interruptable (false) sleep when
+ * waiting for the fences. Used by the suballocator.
+ * Returns 0 if any fence has passed, error for all other cases.
-        timeout = RADEON_FENCE_JIFFIES_TIMEOUT;
+ */
-		write_lock_irqsave(&rdev->fence_drv.lock, irq_flags);
+int radeon_fence_wait_any(struct radeon_device *rdev,
-        rdev->fence_drv.last_timeout = RADEON_FENCE_JIFFIES_TIMEOUT;
+			  struct radeon_fence **fences,
-        rdev->fence_drv.last_jiffies = GetTimerTicks();
+			  bool intr)
+{
+	uint64_t seq[RADEON_NUM_RINGS];
-		write_unlock_irqrestore(&rdev->fence_drv.lock, irq_flags);
+	unsigned i;
+	int r;
-		goto retry;
 	for (i = 0; i < RADEON_NUM_RINGS; ++i) {
-	return 0;
-}
-#if 0
-int radeon_fence_wait_next(struct radeon_device *rdev)
-{
-	unsigned long irq_flags;
+		seq[i] = 0;
 	struct radeon_fence *fence;
+		if (!fences[i]) {
+			continue;
+		}
+		if (fences[i]->seq == RADEON_FENCE_SIGNALED_SEQ) {
+			/* something was allready signaled */
+		return 0;
+	}
+		seq[i] = fences[i]->seq;
-	int r;
+	}
-	if (rdev->gpu_lockup) {
-		return 0;
-	}
+	r = radeon_fence_wait_any_seq(rdev, seq, intr);
+	if (r) {
+		return r;
+	}
+	return 0;
+}
+/**
+ * radeon_fence_wait_next_locked - wait for the next fence to signal
+ *
+ * @rdev: radeon device pointer
+ * @ring: ring index the fence is associated with
+ *
+ * Wait for the next fence on the requested ring to signal (all asics).
+ * Returns 0 if the next fence has passed, error for all other cases.
+ * Caller must hold ring lock.
+ */
+int radeon_fence_wait_next_locked(struct radeon_device *rdev, int ring)
+{
+	uint64_t seq;
+	seq = atomic64_read(&rdev->fence_drv[ring].last_seq) + 1ULL;
+	if (seq >= rdev->fence_drv[ring].sync_seq[ring]) {
+		/* nothing to wait for, last_seq is
+		   already the last emited fence */
+		return -ENOENT;
+	}
-	write_lock_irqsave(&rdev->fence_drv.lock, irq_flags);
+	return radeon_fence_wait_seq(rdev, seq, ring, false, false);
+}
+/**
+ * radeon_fence_wait_empty_locked - wait for all fences to signal
-	if (list_empty(&rdev->fence_drv.emited)) {
+ *
-		write_unlock_irqrestore(&rdev->fence_drv.lock, irq_flags);
+ * @rdev: radeon device pointer
-		return 0;
+ * @ring: ring index the fence is associated with
-	}
+ *
-	fence = list_entry(rdev->fence_drv.emited.next,
+ * Wait for all fences on the requested ring to signal (all asics).
+ * Returns 0 if the fences have passed, error for all other cases.
-			   struct radeon_fence, list);
+ * Caller must hold ring lock.
-	radeon_fence_ref(fence);
+ */
-	write_unlock_irqrestore(&rdev->fence_drv.lock, irq_flags);
-	r = radeon_fence_wait(fence, false);
-	radeon_fence_unref(&fence);
-	return r;
-}
-int radeon_fence_wait_last(struct radeon_device *rdev)
 void radeon_fence_wait_empty_locked(struct radeon_device *rdev, int ring)
+{
+	uint64_t seq = rdev->fence_drv[ring].sync_seq[ring];
+	while(1) {
+	int r;
+		r = radeon_fence_wait_seq(rdev, seq, ring, false, false);
+		if (r == -EDEADLK) {
+			mutex_unlock(&rdev->ring_lock);
-	unsigned long irq_flags;
+			r = radeon_gpu_reset(rdev);
-	struct radeon_fence *fence;
+			mutex_lock(&rdev->ring_lock);
-	int r;
+			if (!r)
 				continue;
 	if (rdev->gpu_lockup) {
-		return 0;
+		if (r) {
+			dev_err(rdev->dev, "error waiting for ring to become"
-	}
+				" idle (%d)\n", r);
+		}
+		return;
+	}
+}
 	write_lock_irqsave(&rdev->fence_drv.lock, irq_flags);
-	if (list_empty(&rdev->fence_drv.emited)) {
+/**
+ * radeon_fence_ref - take a ref on a fence
+ *
+ * @fence: radeon fence object
+ *
+ * Take a reference on a fence (all asics).
+ * Returns the fence.
+ */
+struct radeon_fence *radeon_fence_ref(struct radeon_fence *fence)
+{
+	kref_get(&fence->kref);
+	return fence;
+}
+/**
+ * radeon_fence_unref - remove a ref on a fence
+ *
-		write_unlock_irqrestore(&rdev->fence_drv.lock, irq_flags);
+ * @fence: radeon fence object
+ *
+ * Remove a reference on a fence (all asics).
+ */
+void radeon_fence_unref(struct radeon_fence **fence)
+{
+    struct radeon_fence *tmp = *fence;
+    *fence = NULL;
+	if (tmp) {
+		kref_put(&tmp->kref, radeon_fence_destroy);
+	}
+}
+/**
+ * radeon_fence_count_emitted - get the count of emitted fences
+ *
+ * @rdev: radeon device pointer
+ * @ring: ring index the fence is associated with
+ *
+ * Get the number of fences emitted on the requested ring (all asics).
+ * Returns the number of emitted fences on the ring.  Used by the
+ * dynpm code to ring track activity.
+ */
+unsigned radeon_fence_count_emitted(struct radeon_device *rdev, int ring)
+{
+	uint64_t emitted;
+	/* We are not protected by ring lock when reading the last sequence
+	 * but it's ok to report slightly wrong fence count here.
+	 */
+	radeon_fence_process(rdev, ring);
-		return 0;
+	emitted = rdev->fence_drv[ring].sync_seq[ring]
+		- atomic64_read(&rdev->fence_drv[ring].last_seq);
-	}
+	/* to avoid 32bits warp around */
+	if (emitted > 0x10000000) {
+		emitted = 0x10000000;
-	fence = list_entry(rdev->fence_drv.emited.prev,
+	}
-			   struct radeon_fence, list);
+	return (unsigned)emitted;
+}
-	radeon_fence_ref(fence);
+/**
-	write_unlock_irqrestore(&rdev->fence_drv.lock, irq_flags);
+ * radeon_fence_need_sync - do we need a semaphore
-	r = radeon_fence_wait(fence, false);
+ *
-	radeon_fence_unref(&fence);
-	return r;
+ * @fence: radeon fence object
+ * @dst_ring: which ring to check against
+ *
  * Check if the fence needs to be synced against another ring
+ * (all asics).  If so, we need to emit a semaphore.
+ * Returns true if we need to sync with another ring, false if
+ * not.
+ */
+bool radeon_fence_need_sync(struct radeon_fence *fence, int dst_ring)
+{
+	struct radeon_fence_driver *fdrv;
+	if (!fence) {
+		return false;
 struct radeon_fence *radeon_fence_ref(struct radeon_fence *fence)
-{
+	if (fence->ring == dst_ring) {
+		return false;
+	}
-	kref_get(&fence->kref);
+	/* we are protected by the ring mutex */
+	fdrv = &fence->rdev->fence_drv[dst_ring];
+	if (fence->seq <= fdrv->sync_seq[fence->ring]) {
+		return false;
+	}
-	return fence;
+	return true;
 }
+/**
+ * radeon_fence_note_sync - record the sync point
+ *
+ * @fence: radeon fence object
+ * @dst_ring: which ring to check against
 #endif
+ * Note the sequence number at which point the fence will
+ * be synced with the requested ring (all asics).
+ */
+void radeon_fence_note_sync(struct radeon_fence *fence, int dst_ring)
+{
+	struct radeon_fence_driver *dst, *src;
+	unsigned i;
+	if (!fence) {
+		return;
+	}
+	if (fence->ring == dst_ring) {
-void radeon_fence_unref(struct radeon_fence **fence)
+		return;
-{
+	}
     unsigned long irq_flags;
-	struct radeon_fence *tmp = *fence;
+	/* we are protected by the ring mutex */
+	src = &fence->rdev->fence_drv[fence->ring];
+	dst = &fence->rdev->fence_drv[dst_ring];
+	for (i = 0; i < RADEON_NUM_RINGS; ++i) {
+		if (i == dst_ring) {
+			continue;
 	*fence = NULL;
 		dst->sync_seq[i] = max(dst->sync_seq[i], src->sync_seq[i]);
-    if(tmp)
+	}
         write_lock_irqsave(&tmp->rdev->fence_drv.lock, irq_flags);
-        list_del(&tmp->list);
-        tmp->emited = false;
+/**
-        write_unlock_irqrestore(&tmp->rdev->fence_drv.lock, irq_flags);
+ * radeon_fence_driver_start_ring - make the fence driver
+ * ready for use on the requested ring.
-    };
+ *
-}
+ * @rdev: radeon device pointer
+ * @ring: ring index to start the fence driver on
-void radeon_fence_process(struct radeon_device *rdev)
+ *
+ * Make the fence driver ready for processing (all asics).
-{
+ * Not all asics have all rings, so each asic will only
-	unsigned long irq_flags;
+ * start the fence driver on the rings it has.
-	bool wake;
+ * Returns 0 for success, errors for failure.
+ */
+int radeon_fence_driver_start_ring(struct radeon_device *rdev, int ring)
+{
+	uint64_t index;
+	int r;
+	radeon_scratch_free(rdev, rdev->fence_drv[ring].scratch_reg);
+	if (rdev->wb.use_event) {
+		rdev->fence_drv[ring].scratch_reg = 0;
+		index = R600_WB_EVENT_OFFSET + ring * 4;
+	} else {
+		r = radeon_scratch_get(rdev, &rdev->fence_drv[ring].scratch_reg);
+	if (r) {
+		dev_err(rdev->dev, "fence failed to get scratch register\n");
+		return r;
+	}
+		index = RADEON_WB_SCRATCH_OFFSET +
+			rdev->fence_drv[ring].scratch_reg -
+			rdev->scratch.reg_base;
+	}
+	rdev->fence_drv[ring].cpu_addr = &rdev->wb.wb[index/4];
+	rdev->fence_drv[ring].gpu_addr = rdev->wb.gpu_addr + index;
+	radeon_fence_write(rdev, atomic64_read(&rdev->fence_drv[ring].last_seq), ring);
+	rdev->fence_drv[ring].initialized = true;
+	dev_info(rdev->dev, "fence driver on ring %d use gpu addr 0x%016llx and cpu addr 0x%p\n",
+		 ring, rdev->fence_drv[ring].gpu_addr, rdev->fence_drv[ring].cpu_addr);
+    return 0;
+}
+/**
+ * radeon_fence_driver_init_ring - init the fence driver
+ * for the requested ring.
+ *
+ * @rdev: radeon device pointer
+ * @ring: ring index to start the fence driver on
+ *
+ * Init the fence driver for the requested ring (all asics).
+ * Helper function for radeon_fence_driver_init().
+ */
+static void radeon_fence_driver_init_ring(struct radeon_device *rdev, int ring)
+{
+	int i;
+	rdev->fence_drv[ring].scratch_reg = -1;
+	rdev->fence_drv[ring].cpu_addr = NULL;
+	rdev->fence_drv[ring].gpu_addr = 0;
+	for (i = 0; i < RADEON_NUM_RINGS; ++i)
+		rdev->fence_drv[ring].sync_seq[i] = 0;
+	atomic64_set(&rdev->fence_drv[ring].last_seq, 0);
+	rdev->fence_drv[ring].last_activity = jiffies;
+	rdev->fence_drv[ring].initialized = false;
+}
+/**
+ * radeon_fence_driver_init - init the fence driver
+ * for all possible rings.
+ *
+ * @rdev: radeon device pointer
+ *
+ * Init the fence driver for all possible rings (all asics).
+ * Not all asics have all rings, so each asic will only
+ * start the fence driver on the rings it has using
+ * radeon_fence_driver_start_ring().
+ * Returns 0 for success.
+ */
+int radeon_fence_driver_init(struct radeon_device *rdev)
+{
+	int ring;
+	init_waitqueue_head(&rdev->fence_queue);
+	for (ring = 0; ring < RADEON_NUM_RINGS; ring++) {
+		radeon_fence_driver_init_ring(rdev, ring);
+	}
+	if (radeon_debugfs_fence_init(rdev)) {
 		dev_err(rdev->dev, "fence debugfs file creation failed\n");
 	write_lock_irqsave(&rdev->fence_drv.lock, irq_flags);
-	wake = radeon_fence_poll_locked(rdev);
+	return 0;
 	write_unlock_irqrestore(&rdev->fence_drv.lock, irq_flags);
 /**
-int radeon_fence_driver_init(struct radeon_device *rdev)
+ * radeon_fence_driver_fini - tear down the fence driver
  * for all possible rings.
 	unsigned long irq_flags;
-	int r;
+ * @rdev: radeon device pointer
+ *
+ * Tear down the fence driver for all possible rings (all asics).
+ */
+void radeon_fence_driver_fini(struct radeon_device *rdev)
+{
+	int ring;
-	write_lock_irqsave(&rdev->fence_drv.lock, irq_flags);
-	r = radeon_scratch_get(rdev, &rdev->fence_drv.scratch_reg);
+	mutex_lock(&rdev->ring_lock);
-	if (r) {
+	for (ring = 0; ring < RADEON_NUM_RINGS; ring++) {
+		if (!rdev->fence_drv[ring].initialized)
+			continue;
-		dev_err(rdev->dev, "fence failed to get scratch register\n");
+		radeon_fence_wait_empty_locked(rdev, ring);
-		write_unlock_irqrestore(&rdev->fence_drv.lock, irq_flags);
+		wake_up_all(&rdev->fence_queue);
-		return r;
+		radeon_scratch_free(rdev, rdev->fence_drv[ring].scratch_reg);
+		rdev->fence_drv[ring].initialized = false;
+	}
-	radeon_fence_write(rdev, 0);
+	mutex_unlock(&rdev->ring_lock);
 	atomic_set(&rdev->fence_drv.seq, 0);
 	INIT_LIST_HEAD(&rdev->fence_drv.created);

Subversion Repositories Kolibri OS

(root)/drivers/video/drm/radeon/radeon_fence.c @ 3031 – Rev 2005 → 2997