Subversion Repositories Kolibri OS

Rev

Rev 2005 | Rev 3031 | Go to most recent revision | Details | Compare with Previous | Last modification | View Log | RSS feed

Rev Author Line No. Line
1125 serge 1
/*
2
 * Copyright 2009 Jerome Glisse.
3
 * All Rights Reserved.
4
 *
5
 * Permission is hereby granted, free of charge, to any person obtaining a
6
 * copy of this software and associated documentation files (the
7
 * "Software"), to deal in the Software without restriction, including
8
 * without limitation the rights to use, copy, modify, merge, publish,
9
 * distribute, sub license, and/or sell copies of the Software, and to
10
 * permit persons to whom the Software is furnished to do so, subject to
11
 * the following conditions:
12
 *
13
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
14
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
15
 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
16
 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
17
 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
18
 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
19
 * USE OR OTHER DEALINGS IN THE SOFTWARE.
20
 *
21
 * The above copyright notice and this permission notice (including the
22
 * next paragraph) shall be included in all copies or substantial portions
23
 * of the Software.
24
 *
25
 */
26
/*
27
 * Authors:
28
 *    Jerome Glisse 
29
 *    Dave Airlie
30
 */
31
#include 
32
#include 
2004 serge 33
//#include 
1125 serge 34
#include 
35
#include 
1986 serge 36
#include 
2997 Serge 37
#include 
1125 serge 38
#include "radeon_reg.h"
39
#include "radeon.h"
40
 
2997 Serge 41
/*
42
 * Fences
43
 * Fences mark an event in the GPUs pipeline and are used
44
 * for GPU/CPU synchronization.  When the fence is written,
45
 * it is expected that all buffers associated with that fence
46
 * are no longer in use by the associated ring on the GPU and
47
 * that the the relevant GPU caches have been flushed.  Whether
48
 * we use a scratch register or memory location depends on the asic
49
 * and whether writeback is enabled.
50
 */
51
 
52
/**
53
 * radeon_fence_write - write a fence value
54
 *
55
 * @rdev: radeon_device pointer
56
 * @seq: sequence number to write
57
 * @ring: ring index the fence is associated with
58
 *
59
 * Writes a fence value to memory or a scratch register (all asics).
60
 */
61
static void radeon_fence_write(struct radeon_device *rdev, u32 seq, int ring)
2004 serge 62
{
2997 Serge 63
	struct radeon_fence_driver *drv = &rdev->fence_drv[ring];
64
	if (likely(rdev->wb.enabled || !drv->scratch_reg)) {
65
		*drv->cpu_addr = cpu_to_le32(seq);
66
	} else {
67
		WREG32(drv->scratch_reg, seq);
68
	}
2004 serge 69
}
70
 
2997 Serge 71
/**
72
 * radeon_fence_read - read a fence value
73
 *
74
 * @rdev: radeon_device pointer
75
 * @ring: ring index the fence is associated with
76
 *
77
 * Reads a fence value from memory or a scratch register (all asics).
78
 * Returns the value of the fence read from memory or register.
79
 */
80
static u32 radeon_fence_read(struct radeon_device *rdev, int ring)
2004 serge 81
{
2997 Serge 82
	struct radeon_fence_driver *drv = &rdev->fence_drv[ring];
83
	u32 seq = 0;
2004 serge 84
 
2997 Serge 85
	if (likely(rdev->wb.enabled || !drv->scratch_reg)) {
86
		seq = le32_to_cpu(*drv->cpu_addr);
87
	} else {
88
		seq = RREG32(drv->scratch_reg);
89
	}
2004 serge 90
	return seq;
91
}
92
 
2997 Serge 93
/**
94
 * radeon_fence_emit - emit a fence on the requested ring
95
 *
96
 * @rdev: radeon_device pointer
97
 * @fence: radeon fence object
98
 * @ring: ring index the fence is associated with
99
 *
100
 * Emits a fence command on the requested ring (all asics).
101
 * Returns 0 on success, -ENOMEM on failure.
102
 */
103
int radeon_fence_emit(struct radeon_device *rdev,
104
		      struct radeon_fence **fence,
105
		      int ring)
1125 serge 106
{
2997 Serge 107
	/* we are protected by the ring emission mutex */
108
	*fence = kmalloc(sizeof(struct radeon_fence), GFP_KERNEL);
109
	if ((*fence) == NULL) {
110
		return -ENOMEM;
1125 serge 111
	}
2997 Serge 112
	kref_init(&((*fence)->kref));
113
	(*fence)->rdev = rdev;
114
	(*fence)->seq = ++rdev->fence_drv[ring].sync_seq[ring];
115
	(*fence)->ring = ring;
116
	radeon_fence_ring_emit(rdev, ring, *fence);
117
//   trace_radeon_fence_emit(rdev->ddev, (*fence)->seq);
1125 serge 118
	return 0;
119
}
120
 
2997 Serge 121
/**
122
 * radeon_fence_process - process a fence
123
 *
124
 * @rdev: radeon_device pointer
125
 * @ring: ring index the fence is associated with
126
 *
127
 * Checks the current fence value and wakes the fence queue
128
 * if the sequence number has increased (all asics).
129
 */
130
void radeon_fence_process(struct radeon_device *rdev, int ring)
1125 serge 131
{
2997 Serge 132
	uint64_t seq, last_seq, last_emitted;
133
	unsigned count_loop = 0;
1125 serge 134
	bool wake = false;
135
 
2997 Serge 136
	/* Note there is a scenario here for an infinite loop but it's
137
	 * very unlikely to happen. For it to happen, the current polling
138
	 * process need to be interrupted by another process and another
139
	 * process needs to update the last_seq btw the atomic read and
140
	 * xchg of the current process.
141
	 *
142
	 * More over for this to go in infinite loop there need to be
143
	 * continuously new fence signaled ie radeon_fence_read needs
144
	 * to return a different value each time for both the currently
145
	 * polling process and the other process that xchg the last_seq
146
	 * btw atomic read and xchg of the current process. And the
147
	 * value the other process set as last seq must be higher than
148
	 * the seq value we just read. Which means that current process
149
	 * need to be interrupted after radeon_fence_read and before
150
	 * atomic xchg.
151
	 *
152
	 * To be even more safe we count the number of time we loop and
153
	 * we bail after 10 loop just accepting the fact that we might
154
	 * have temporarly set the last_seq not to the true real last
155
	 * seq but to an older one.
156
	 */
157
	last_seq = atomic64_read(&rdev->fence_drv[ring].last_seq);
158
	do {
159
		last_emitted = rdev->fence_drv[ring].sync_seq[ring];
160
		seq = radeon_fence_read(rdev, ring);
161
		seq |= last_seq & 0xffffffff00000000LL;
162
		if (seq < last_seq) {
163
			seq &= 0xffffffff;
164
			seq |= last_emitted & 0xffffffff00000000LL;
165
		}
166
 
167
		if (seq <= last_seq || seq > last_emitted) {
168
			break;
169
		}
170
		/* If we loop over we don't want to return without
171
		 * checking if a fence is signaled as it means that the
172
		 * seq we just read is different from the previous on.
173
		 */
174
		wake = true;
175
		last_seq = seq;
176
		if ((count_loop++) > 10) {
177
			/* We looped over too many time leave with the
178
			 * fact that we might have set an older fence
179
			 * seq then the current real last seq as signaled
180
			 * by the hw.
1963 serge 181
			 */
1125 serge 182
			break;
183
		}
2997 Serge 184
	} while (atomic64_xchg(&rdev->fence_drv[ring].last_seq, seq) > seq);
185
 
186
	if (wake) {
187
		rdev->fence_drv[ring].last_activity = GetTimerTicks();
188
		wake_up_all(&rdev->fence_queue);
1125 serge 189
	}
2997 Serge 190
}
191
 
192
/**
193
 * radeon_fence_destroy - destroy a fence
194
 *
195
 * @kref: fence kref
196
 *
197
 * Frees the fence object (all asics).
198
 */
199
static void radeon_fence_destroy(struct kref *kref)
200
{
201
	struct radeon_fence *fence;
202
 
203
	fence = container_of(kref, struct radeon_fence, kref);
204
	kfree(fence);
205
}
206
 
207
/**
208
 * radeon_fence_seq_signaled - check if a fence sequeuce number has signaled
209
 *
210
 * @rdev: radeon device pointer
211
 * @seq: sequence number
212
 * @ring: ring index the fence is associated with
213
 *
214
 * Check if the last singled fence sequnce number is >= the requested
215
 * sequence number (all asics).
216
 * Returns true if the fence has signaled (current fence value
217
 * is >= requested value) or false if it has not (current fence
218
 * value is < the requested value.  Helper function for
219
 * radeon_fence_signaled().
220
 */
221
static bool radeon_fence_seq_signaled(struct radeon_device *rdev,
222
				      u64 seq, unsigned ring)
223
{
224
	if (atomic64_read(&rdev->fence_drv[ring].last_seq) >= seq) {
225
		return true;
1125 serge 226
	}
2997 Serge 227
	/* poll new last sequence at least once */
228
	radeon_fence_process(rdev, ring);
229
	if (atomic64_read(&rdev->fence_drv[ring].last_seq) >= seq) {
230
		return true;
231
	}
232
	return false;
1125 serge 233
}
234
 
2997 Serge 235
/**
236
 * radeon_fence_signaled - check if a fence has signaled
237
 *
238
 * @fence: radeon fence object
239
 *
240
 * Check if the requested fence has signaled (all asics).
241
 * Returns true if the fence has signaled or false if it has not.
242
				 */
243
bool radeon_fence_signaled(struct radeon_fence *fence)
244
{
245
	if (!fence) {
246
		return true;
247
	}
248
	if (fence->seq == RADEON_FENCE_SIGNALED_SEQ) {
249
		return true;
250
	}
251
	if (radeon_fence_seq_signaled(fence->rdev, fence->seq, fence->ring)) {
252
		fence->seq = RADEON_FENCE_SIGNALED_SEQ;
253
		return true;
254
	}
255
	return false;
256
}
1125 serge 257
 
2997 Serge 258
/**
259
 * radeon_fence_wait_seq - wait for a specific sequence number
260
 *
261
 * @rdev: radeon device pointer
262
 * @target_seq: sequence number we want to wait for
263
 * @ring: ring index the fence is associated with
264
 * @intr: use interruptable sleep
265
 * @lock_ring: whether the ring should be locked or not
266
 *
267
 * Wait for the requested sequence number to be written (all asics).
268
 * @intr selects whether to use interruptable (true) or non-interruptable
269
 * (false) sleep when waiting for the sequence number.  Helper function
270
 * for radeon_fence_wait(), et al.
271
 * Returns 0 if the sequence number has passed, error for all other cases.
272
 * -EDEADLK is returned when a GPU lockup has been detected and the ring is
273
 * marked as not ready so no further jobs get scheduled until a successful
274
 * reset.
275
 */
276
static int radeon_fence_wait_seq(struct radeon_device *rdev, u64 target_seq,
277
				 unsigned ring, bool intr, bool lock_ring)
1125 serge 278
{
2997 Serge 279
	unsigned long timeout, last_activity;
280
	uint64_t seq;
281
	unsigned i;
282
	bool signaled;
283
	int r;
1125 serge 284
 
2997 Serge 285
	while (target_seq > atomic64_read(&rdev->fence_drv[ring].last_seq)) {
286
		if (!rdev->ring[ring].ready) {
287
			return -EBUSY;
288
			}
289
 
290
		timeout = GetTimerTicks() - RADEON_FENCE_JIFFIES_TIMEOUT;
291
		if (time_after(rdev->fence_drv[ring].last_activity, timeout)) {
292
			/* the normal case, timeout is somewhere before last_activity */
293
			timeout = rdev->fence_drv[ring].last_activity - timeout;
294
		} else {
295
			/* either jiffies wrapped around, or no fence was signaled in the last 500ms
296
			 * anyway we will just wait for the minimum amount and then check for a lockup
297
			 */
298
			timeout = 1;
1125 serge 299
	}
2997 Serge 300
		seq = atomic64_read(&rdev->fence_drv[ring].last_seq);
301
		/* Save current last activity valuee, used to check for GPU lockups */
302
		last_activity = rdev->fence_drv[ring].last_activity;
2005 serge 303
 
2997 Serge 304
//		trace_radeon_fence_wait_begin(rdev->ddev, seq);
305
		radeon_irq_kms_sw_irq_get(rdev, ring);
306
//       if (intr) {
307
//           r = wait_event_interruptible_timeout(rdev->fence_queue,
308
//               (signaled = radeon_fence_seq_signaled(rdev, target_seq, ring)),
309
//               timeout);
310
//                } else {
311
//           r = wait_event_timeout(rdev->fence_queue,
312
//               (signaled = radeon_fence_seq_signaled(rdev, target_seq, ring)),
313
//               timeout);
314
//   }
315
        delay(1);
1125 serge 316
 
2997 Serge 317
		radeon_irq_kms_sw_irq_put(rdev, ring);
318
//       if (unlikely(r < 0)) {
319
//           return r;
320
//       }
321
//		trace_radeon_fence_wait_end(rdev->ddev, seq);
1125 serge 322
 
2997 Serge 323
		if (unlikely(!signaled)) {
324
			/* we were interrupted for some reason and fence
325
			 * isn't signaled yet, resume waiting */
326
			if (r) {
327
				continue;
328
	}
1125 serge 329
 
2997 Serge 330
			/* check if sequence value has changed since last_activity */
331
			if (seq != atomic64_read(&rdev->fence_drv[ring].last_seq)) {
332
				continue;
333
			}
1125 serge 334
 
2997 Serge 335
			if (lock_ring) {
336
				mutex_lock(&rdev->ring_lock);
337
	}
1404 serge 338
 
2997 Serge 339
			/* test if somebody else has already decided that this is a lockup */
340
			if (last_activity != rdev->fence_drv[ring].last_activity) {
341
				if (lock_ring) {
342
					mutex_unlock(&rdev->ring_lock);
343
				}
344
				continue;
345
			}
1404 serge 346
 
2997 Serge 347
			if (radeon_ring_is_lockup(rdev, ring, &rdev->ring[ring])) {
348
				/* good news we believe it's a lockup */
349
				dev_warn(rdev->dev, "GPU lockup (waiting for 0x%016llx last fence id 0x%016llx)\n",
350
					 target_seq, seq);
351
 
352
				/* change last activity so nobody else think there is a lockup */
353
				for (i = 0; i < RADEON_NUM_RINGS; ++i) {
354
					rdev->fence_drv[i].last_activity = jiffies;
355
				}
356
 
357
				/* mark the ring as not ready any more */
358
				rdev->ring[ring].ready = false;
359
				if (lock_ring) {
360
					mutex_unlock(&rdev->ring_lock);
361
				}
362
				return -EDEADLK;
363
			}
364
 
365
			if (lock_ring) {
366
				mutex_unlock(&rdev->ring_lock);
367
			}
368
		}
1125 serge 369
	}
2997 Serge 370
	return 0;
1125 serge 371
}
372
 
2997 Serge 373
/**
374
 * radeon_fence_wait - wait for a fence to signal
375
 *
376
 * @fence: radeon fence object
377
 * @intr: use interruptable sleep
378
 *
379
 * Wait for the requested fence to signal (all asics).
380
 * @intr selects whether to use interruptable (true) or non-interruptable
381
 * (false) sleep when waiting for the fence.
382
 * Returns 0 if the fence has passed, error for all other cases.
383
 */
1179 serge 384
int radeon_fence_wait(struct radeon_fence *fence, bool intr)
385
{
1125 serge 386
	int r;
387
 
388
	if (fence == NULL) {
389
		WARN(1, "Querying an invalid fence : %p !\n", fence);
2997 Serge 390
		return -EINVAL;
1125 serge 391
	}
2997 Serge 392
 
393
	r = radeon_fence_wait_seq(fence->rdev, fence->seq,
394
				  fence->ring, intr, true);
395
	if (r) {
396
		return r;
397
	}
398
	fence->seq = RADEON_FENCE_SIGNALED_SEQ;
1125 serge 399
		return 0;
2997 Serge 400
}
401
 
402
static bool radeon_fence_any_seq_signaled(struct radeon_device *rdev, u64 *seq)
403
{
404
	unsigned i;
405
 
406
	for (i = 0; i < RADEON_NUM_RINGS; ++i) {
407
		if (seq[i] && radeon_fence_seq_signaled(rdev, seq[i], i)) {
408
			return true;
409
		}
1125 serge 410
	}
2997 Serge 411
	return false;
412
}
2005 serge 413
 
2997 Serge 414
/**
415
 * radeon_fence_wait_any_seq - wait for a sequence number on any ring
416
 *
417
 * @rdev: radeon device pointer
418
 * @target_seq: sequence number(s) we want to wait for
419
 * @intr: use interruptable sleep
420
 *
421
 * Wait for the requested sequence number(s) to be written by any ring
422
 * (all asics).  Sequnce number array is indexed by ring id.
423
 * @intr selects whether to use interruptable (true) or non-interruptable
424
 * (false) sleep when waiting for the sequence number.  Helper function
425
 * for radeon_fence_wait_any(), et al.
426
 * Returns 0 if the sequence number has passed, error for all other cases.
427
 */
428
static int radeon_fence_wait_any_seq(struct radeon_device *rdev,
429
				     u64 *target_seq, bool intr)
430
{
431
	unsigned long timeout, last_activity, tmp;
432
	unsigned i, ring = RADEON_NUM_RINGS;
433
	bool signaled;
434
	int r;
2005 serge 435
 
2997 Serge 436
	for (i = 0, last_activity = 0; i < RADEON_NUM_RINGS; ++i) {
437
		if (!target_seq[i]) {
438
			continue;
1963 serge 439
		}
2005 serge 440
 
2997 Serge 441
		/* use the most recent one as indicator */
442
		if (time_after(rdev->fence_drv[i].last_activity, last_activity)) {
443
			last_activity = rdev->fence_drv[i].last_activity;
444
	}
2005 serge 445
 
2997 Serge 446
		/* For lockup detection just pick the lowest ring we are
447
		 * actively waiting for
448
		 */
449
		if (i < ring) {
450
			ring = i;
451
		}
1125 serge 452
	}
2997 Serge 453
 
454
	/* nothing to wait for ? */
455
	if (ring == RADEON_NUM_RINGS) {
456
		return -ENOENT;
457
	}
458
 
459
	while (!radeon_fence_any_seq_signaled(rdev, target_seq)) {
460
        timeout = GetTimerTicks() - RADEON_FENCE_JIFFIES_TIMEOUT;
461
		if (time_after(last_activity, timeout)) {
462
			/* the normal case, timeout is somewhere before last_activity */
463
			timeout = last_activity - timeout;
464
		} else {
465
			/* either jiffies wrapped around, or no fence was signaled in the last 500ms
466
			 * anyway we will just wait for the minimum amount and then check for a lockup
467
			 */
468
			timeout = 1;
469
		}
470
 
471
//		trace_radeon_fence_wait_begin(rdev->ddev, target_seq[ring]);
472
		for (i = 0; i < RADEON_NUM_RINGS; ++i) {
473
			if (target_seq[i]) {
474
				radeon_irq_kms_sw_irq_get(rdev, i);
475
			}
476
		}
477
 
478
//        WaitEvent(fence->evnt);
479
 
480
		r = 1;
481
 
482
		for (i = 0; i < RADEON_NUM_RINGS; ++i) {
483
			if (target_seq[i]) {
484
				radeon_irq_kms_sw_irq_put(rdev, i);
485
			}
486
		}
487
		if (unlikely(r < 0)) {
488
			return r;
489
		}
2004 serge 490
//   trace_radeon_fence_wait_end(rdev->ddev, seq);
2997 Serge 491
 
492
		if (unlikely(!signaled)) {
493
			/* we were interrupted for some reason and fence
494
			 * isn't signaled yet, resume waiting */
1963 serge 495
		if (r) {
2997 Serge 496
				continue;
497
			}
498
 
499
			mutex_lock(&rdev->ring_lock);
500
			for (i = 0, tmp = 0; i < RADEON_NUM_RINGS; ++i) {
501
				if (time_after(rdev->fence_drv[i].last_activity, tmp)) {
502
					tmp = rdev->fence_drv[i].last_activity;
503
				}
504
			}
505
			/* test if somebody else has already decided that this is a lockup */
506
			if (last_activity != tmp) {
507
				last_activity = tmp;
508
				mutex_unlock(&rdev->ring_lock);
509
				continue;
1125 serge 510
		}
2997 Serge 511
 
512
			if (radeon_ring_is_lockup(rdev, ring, &rdev->ring[ring])) {
1963 serge 513
			/* good news we believe it's a lockup */
2997 Serge 514
				dev_warn(rdev->dev, "GPU lockup (waiting for 0x%016llx)\n",
515
					 target_seq[ring]);
2005 serge 516
 
2997 Serge 517
				/* change last activity so nobody else think there is a lockup */
518
				for (i = 0; i < RADEON_NUM_RINGS; ++i) {
519
					rdev->fence_drv[i].last_activity = GetTimerTicks();
520
				}
521
 
522
				/* mark the ring as not ready any more */
523
				rdev->ring[ring].ready = false;
524
				mutex_unlock(&rdev->ring_lock);
525
				return -EDEADLK;
1125 serge 526
			}
2997 Serge 527
			mutex_unlock(&rdev->ring_lock);
528
			}
1125 serge 529
	}
2997 Serge 530
    return 0;
1125 serge 531
}
532
 
2997 Serge 533
/**
534
 * radeon_fence_wait_any - wait for a fence to signal on any ring
535
 *
536
 * @rdev: radeon device pointer
537
 * @fences: radeon fence object(s)
538
 * @intr: use interruptable sleep
539
 *
540
 * Wait for any requested fence to signal (all asics).  Fence
541
 * array is indexed by ring id.  @intr selects whether to use
542
 * interruptable (true) or non-interruptable (false) sleep when
543
 * waiting for the fences. Used by the suballocator.
544
 * Returns 0 if any fence has passed, error for all other cases.
545
 */
546
int radeon_fence_wait_any(struct radeon_device *rdev,
547
			  struct radeon_fence **fences,
548
			  bool intr)
1125 serge 549
{
2997 Serge 550
	uint64_t seq[RADEON_NUM_RINGS];
551
	unsigned i;
1125 serge 552
	int r;
553
 
2997 Serge 554
	for (i = 0; i < RADEON_NUM_RINGS; ++i) {
555
		seq[i] = 0;
556
 
557
		if (!fences[i]) {
558
			continue;
559
		}
560
 
561
		if (fences[i]->seq == RADEON_FENCE_SIGNALED_SEQ) {
562
			/* something was allready signaled */
1125 serge 563
		return 0;
564
	}
2997 Serge 565
 
566
		seq[i] = fences[i]->seq;
1125 serge 567
	}
2997 Serge 568
 
569
	r = radeon_fence_wait_any_seq(rdev, seq, intr);
570
	if (r) {
571
		return r;
572
	}
573
	return 0;
1125 serge 574
}
575
 
2997 Serge 576
/**
577
 * radeon_fence_wait_next_locked - wait for the next fence to signal
578
 *
579
 * @rdev: radeon device pointer
580
 * @ring: ring index the fence is associated with
581
 *
582
 * Wait for the next fence on the requested ring to signal (all asics).
583
 * Returns 0 if the next fence has passed, error for all other cases.
584
 * Caller must hold ring lock.
585
 */
586
int radeon_fence_wait_next_locked(struct radeon_device *rdev, int ring)
1125 serge 587
{
2997 Serge 588
	uint64_t seq;
1125 serge 589
 
2997 Serge 590
	seq = atomic64_read(&rdev->fence_drv[ring].last_seq) + 1ULL;
591
	if (seq >= rdev->fence_drv[ring].sync_seq[ring]) {
592
		/* nothing to wait for, last_seq is
593
		   already the last emited fence */
594
		return -ENOENT;
1125 serge 595
	}
2997 Serge 596
	return radeon_fence_wait_seq(rdev, seq, ring, false, false);
597
}
598
 
599
/**
600
 * radeon_fence_wait_empty_locked - wait for all fences to signal
601
 *
602
 * @rdev: radeon device pointer
603
 * @ring: ring index the fence is associated with
604
 *
605
 * Wait for all fences on the requested ring to signal (all asics).
606
 * Returns 0 if the fences have passed, error for all other cases.
607
 * Caller must hold ring lock.
608
 */
609
void radeon_fence_wait_empty_locked(struct radeon_device *rdev, int ring)
610
{
611
	uint64_t seq = rdev->fence_drv[ring].sync_seq[ring];
612
 
613
	while(1) {
614
	int r;
615
		r = radeon_fence_wait_seq(rdev, seq, ring, false, false);
616
		if (r == -EDEADLK) {
617
			mutex_unlock(&rdev->ring_lock);
618
			r = radeon_gpu_reset(rdev);
619
			mutex_lock(&rdev->ring_lock);
620
			if (!r)
621
				continue;
1125 serge 622
	}
2997 Serge 623
		if (r) {
624
			dev_err(rdev->dev, "error waiting for ring to become"
625
				" idle (%d)\n", r);
626
		}
627
		return;
628
	}
1125 serge 629
}
630
 
2997 Serge 631
/**
632
 * radeon_fence_ref - take a ref on a fence
633
 *
634
 * @fence: radeon fence object
635
 *
636
 * Take a reference on a fence (all asics).
637
 * Returns the fence.
638
 */
1125 serge 639
struct radeon_fence *radeon_fence_ref(struct radeon_fence *fence)
640
{
641
	kref_get(&fence->kref);
642
	return fence;
643
}
644
 
2997 Serge 645
/**
646
 * radeon_fence_unref - remove a ref on a fence
647
 *
648
 * @fence: radeon fence object
649
 *
650
 * Remove a reference on a fence (all asics).
651
 */
1125 serge 652
void radeon_fence_unref(struct radeon_fence **fence)
653
{
2997 Serge 654
    struct radeon_fence *tmp = *fence;
1125 serge 655
 
2997 Serge 656
    *fence = NULL;
657
	if (tmp) {
658
		kref_put(&tmp->kref, radeon_fence_destroy);
659
	}
660
}
2005 serge 661
 
2997 Serge 662
/**
663
 * radeon_fence_count_emitted - get the count of emitted fences
664
 *
665
 * @rdev: radeon device pointer
666
 * @ring: ring index the fence is associated with
667
 *
668
 * Get the number of fences emitted on the requested ring (all asics).
669
 * Returns the number of emitted fences on the ring.  Used by the
670
 * dynpm code to ring track activity.
671
 */
672
unsigned radeon_fence_count_emitted(struct radeon_device *rdev, int ring)
673
{
674
	uint64_t emitted;
675
 
676
	/* We are not protected by ring lock when reading the last sequence
677
	 * but it's ok to report slightly wrong fence count here.
678
	 */
679
	radeon_fence_process(rdev, ring);
680
	emitted = rdev->fence_drv[ring].sync_seq[ring]
681
		- atomic64_read(&rdev->fence_drv[ring].last_seq);
682
	/* to avoid 32bits warp around */
683
	if (emitted > 0x10000000) {
684
		emitted = 0x10000000;
685
	}
686
	return (unsigned)emitted;
1125 serge 687
}
688
 
2997 Serge 689
/**
690
 * radeon_fence_need_sync - do we need a semaphore
691
 *
692
 * @fence: radeon fence object
693
 * @dst_ring: which ring to check against
694
 *
695
 * Check if the fence needs to be synced against another ring
696
 * (all asics).  If so, we need to emit a semaphore.
697
 * Returns true if we need to sync with another ring, false if
698
 * not.
699
 */
700
bool radeon_fence_need_sync(struct radeon_fence *fence, int dst_ring)
1125 serge 701
{
2997 Serge 702
	struct radeon_fence_driver *fdrv;
1125 serge 703
 
2997 Serge 704
	if (!fence) {
705
		return false;
706
	}
707
 
708
	if (fence->ring == dst_ring) {
709
		return false;
710
	}
711
 
712
	/* we are protected by the ring mutex */
713
	fdrv = &fence->rdev->fence_drv[dst_ring];
714
	if (fence->seq <= fdrv->sync_seq[fence->ring]) {
715
		return false;
716
	}
717
 
718
	return true;
1125 serge 719
}
720
 
2997 Serge 721
/**
722
 * radeon_fence_note_sync - record the sync point
723
 *
724
 * @fence: radeon fence object
725
 * @dst_ring: which ring to check against
726
 *
727
 * Note the sequence number at which point the fence will
728
 * be synced with the requested ring (all asics).
729
 */
730
void radeon_fence_note_sync(struct radeon_fence *fence, int dst_ring)
1125 serge 731
{
2997 Serge 732
	struct radeon_fence_driver *dst, *src;
733
	unsigned i;
734
 
735
	if (!fence) {
736
		return;
737
	}
738
 
739
	if (fence->ring == dst_ring) {
740
		return;
741
	}
742
 
743
	/* we are protected by the ring mutex */
744
	src = &fence->rdev->fence_drv[fence->ring];
745
	dst = &fence->rdev->fence_drv[dst_ring];
746
	for (i = 0; i < RADEON_NUM_RINGS; ++i) {
747
		if (i == dst_ring) {
748
			continue;
749
		}
750
		dst->sync_seq[i] = max(dst->sync_seq[i], src->sync_seq[i]);
751
	}
752
}
753
 
754
/**
755
 * radeon_fence_driver_start_ring - make the fence driver
756
 * ready for use on the requested ring.
757
 *
758
 * @rdev: radeon device pointer
759
 * @ring: ring index to start the fence driver on
760
 *
761
 * Make the fence driver ready for processing (all asics).
762
 * Not all asics have all rings, so each asic will only
763
 * start the fence driver on the rings it has.
764
 * Returns 0 for success, errors for failure.
765
 */
766
int radeon_fence_driver_start_ring(struct radeon_device *rdev, int ring)
767
{
768
	uint64_t index;
1125 serge 769
	int r;
770
 
2997 Serge 771
	radeon_scratch_free(rdev, rdev->fence_drv[ring].scratch_reg);
772
	if (rdev->wb.use_event) {
773
		rdev->fence_drv[ring].scratch_reg = 0;
774
		index = R600_WB_EVENT_OFFSET + ring * 4;
775
	} else {
776
		r = radeon_scratch_get(rdev, &rdev->fence_drv[ring].scratch_reg);
1125 serge 777
	if (r) {
1404 serge 778
		dev_err(rdev->dev, "fence failed to get scratch register\n");
1125 serge 779
		return r;
780
	}
2997 Serge 781
		index = RADEON_WB_SCRATCH_OFFSET +
782
			rdev->fence_drv[ring].scratch_reg -
783
			rdev->scratch.reg_base;
784
	}
785
	rdev->fence_drv[ring].cpu_addr = &rdev->wb.wb[index/4];
786
	rdev->fence_drv[ring].gpu_addr = rdev->wb.gpu_addr + index;
787
	radeon_fence_write(rdev, atomic64_read(&rdev->fence_drv[ring].last_seq), ring);
788
	rdev->fence_drv[ring].initialized = true;
789
	dev_info(rdev->dev, "fence driver on ring %d use gpu addr 0x%016llx and cpu addr 0x%p\n",
790
		 ring, rdev->fence_drv[ring].gpu_addr, rdev->fence_drv[ring].cpu_addr);
2004 serge 791
    return 0;
1125 serge 792
}
793
 
2997 Serge 794
/**
795
 * radeon_fence_driver_init_ring - init the fence driver
796
 * for the requested ring.
797
 *
798
 * @rdev: radeon device pointer
799
 * @ring: ring index to start the fence driver on
800
 *
801
 * Init the fence driver for the requested ring (all asics).
802
 * Helper function for radeon_fence_driver_init().
803
 */
804
static void radeon_fence_driver_init_ring(struct radeon_device *rdev, int ring)
805
{
806
	int i;
1125 serge 807
 
2997 Serge 808
	rdev->fence_drv[ring].scratch_reg = -1;
809
	rdev->fence_drv[ring].cpu_addr = NULL;
810
	rdev->fence_drv[ring].gpu_addr = 0;
811
	for (i = 0; i < RADEON_NUM_RINGS; ++i)
812
		rdev->fence_drv[ring].sync_seq[i] = 0;
813
	atomic64_set(&rdev->fence_drv[ring].last_seq, 0);
814
	rdev->fence_drv[ring].last_activity = jiffies;
815
	rdev->fence_drv[ring].initialized = false;
816
}
817
 
818
/**
819
 * radeon_fence_driver_init - init the fence driver
820
 * for all possible rings.
821
 *
822
 * @rdev: radeon device pointer
823
 *
824
 * Init the fence driver for all possible rings (all asics).
825
 * Not all asics have all rings, so each asic will only
826
 * start the fence driver on the rings it has using
827
 * radeon_fence_driver_start_ring().
828
 * Returns 0 for success.
829
 */
830
int radeon_fence_driver_init(struct radeon_device *rdev)
831
{
832
	int ring;
833
 
834
	init_waitqueue_head(&rdev->fence_queue);
835
	for (ring = 0; ring < RADEON_NUM_RINGS; ring++) {
836
		radeon_fence_driver_init_ring(rdev, ring);
837
	}
838
	if (radeon_debugfs_fence_init(rdev)) {
839
		dev_err(rdev->dev, "fence debugfs file creation failed\n");
840
	}
841
	return 0;
842
}
843
 
844
/**
845
 * radeon_fence_driver_fini - tear down the fence driver
846
 * for all possible rings.
847
 *
848
 * @rdev: radeon device pointer
849
 *
850
 * Tear down the fence driver for all possible rings (all asics).
851
 */
852
void radeon_fence_driver_fini(struct radeon_device *rdev)
853
{
854
	int ring;
855
 
856
	mutex_lock(&rdev->ring_lock);
857
	for (ring = 0; ring < RADEON_NUM_RINGS; ring++) {
858
		if (!rdev->fence_drv[ring].initialized)
859
			continue;
860
		radeon_fence_wait_empty_locked(rdev, ring);
861
		wake_up_all(&rdev->fence_queue);
862
		radeon_scratch_free(rdev, rdev->fence_drv[ring].scratch_reg);
863
		rdev->fence_drv[ring].initialized = false;
864
	}
865
	mutex_unlock(&rdev->ring_lock);
866
}
867
 
868
 
1125 serge 869
/*
870
 * Fence debugfs
871
 */
872
#if defined(CONFIG_DEBUG_FS)
873
static int radeon_debugfs_fence_info(struct seq_file *m, void *data)
874
{
875
	struct drm_info_node *node = (struct drm_info_node *)m->private;
876
	struct drm_device *dev = node->minor->dev;
877
	struct radeon_device *rdev = dev->dev_private;
2997 Serge 878
	int i, j;
1125 serge 879
 
2997 Serge 880
	for (i = 0; i < RADEON_NUM_RINGS; ++i) {
881
		if (!rdev->fence_drv[i].initialized)
882
			continue;
883
 
884
		seq_printf(m, "--- ring %d ---\n", i);
885
		seq_printf(m, "Last signaled fence 0x%016llx\n",
886
			   (unsigned long long)atomic64_read(&rdev->fence_drv[i].last_seq));
887
		seq_printf(m, "Last emitted        0x%016llx\n",
888
			   rdev->fence_drv[i].sync_seq[i]);
889
 
890
		for (j = 0; j < RADEON_NUM_RINGS; ++j) {
891
			if (i != j && rdev->fence_drv[j].initialized)
892
				seq_printf(m, "Last sync to ring %d 0x%016llx\n",
893
					   j, rdev->fence_drv[i].sync_seq[j]);
894
		}
1125 serge 895
	}
896
	return 0;
897
}
898
 
899
static struct drm_info_list radeon_debugfs_fence_list[] = {
900
	{"radeon_fence_info", &radeon_debugfs_fence_info, 0, NULL},
901
};
902
#endif
903
 
904
int radeon_debugfs_fence_init(struct radeon_device *rdev)
905
{
906
#if defined(CONFIG_DEBUG_FS)
907
	return radeon_debugfs_add_files(rdev, radeon_debugfs_fence_list, 1);
908
#else
909
	return 0;
910
#endif
911
}