Subversion Repositories Kolibri OS

Rev

Rev 5354 | Rev 6937 | Go to most recent revision | Only display areas with differences | Regard whitespace | Details | Blame | Last modification | View Log | RSS feed

Rev 5354 Rev 6084
1
#ifndef _INTEL_RINGBUFFER_H_
1
#ifndef _INTEL_RINGBUFFER_H_
2
#define _INTEL_RINGBUFFER_H_
2
#define _INTEL_RINGBUFFER_H_
3
 
3
 
4
#include 
4
#include 
-
 
5
#include "i915_gem_batch_pool.h"
5
 
6
 
6
#define I915_CMD_HASH_ORDER 9
7
#define I915_CMD_HASH_ORDER 9
7
 
8
 
8
/* Early gen2 devices have a cacheline of just 32 bytes, using 64 is overkill,
9
/* Early gen2 devices have a cacheline of just 32 bytes, using 64 is overkill,
9
 * but keeps the logic simple. Indeed, the whole purpose of this macro is just
10
 * but keeps the logic simple. Indeed, the whole purpose of this macro is just
10
 * to give some inclination as to some of the magic values used in the various
11
 * to give some inclination as to some of the magic values used in the various
11
 * workarounds!
12
 * workarounds!
12
 */
13
 */
13
#define CACHELINE_BYTES 64
14
#define CACHELINE_BYTES 64
-
 
15
#define CACHELINE_DWORDS (CACHELINE_BYTES / sizeof(uint32_t))
14
 
16
 
15
/*
17
/*
16
 * Gen2 BSpec "1. Programming Environment" / 1.4.4.6 "Ring Buffer Use"
18
 * Gen2 BSpec "1. Programming Environment" / 1.4.4.6 "Ring Buffer Use"
17
 * Gen3 BSpec "vol1c Memory Interface Functions" / 2.3.4.5 "Ring Buffer Use"
19
 * Gen3 BSpec "vol1c Memory Interface Functions" / 2.3.4.5 "Ring Buffer Use"
18
 * Gen4+ BSpec "vol1c Memory Interface and Command Stream" / 5.3.4.5 "Ring Buffer Use"
20
 * Gen4+ BSpec "vol1c Memory Interface and Command Stream" / 5.3.4.5 "Ring Buffer Use"
19
 *
21
 *
20
 * "If the Ring Buffer Head Pointer and the Tail Pointer are on the same
22
 * "If the Ring Buffer Head Pointer and the Tail Pointer are on the same
21
 * cacheline, the Head Pointer must not be greater than the Tail
23
 * cacheline, the Head Pointer must not be greater than the Tail
22
 * Pointer."
24
 * Pointer."
23
 */
25
 */
24
#define I915_RING_FREE_SPACE 64
26
#define I915_RING_FREE_SPACE 64
25
 
27
 
26
struct  intel_hw_status_page {
28
struct  intel_hw_status_page {
27
	u32		*page_addr;
29
	u32		*page_addr;
28
	unsigned int	gfx_addr;
30
	unsigned int	gfx_addr;
29
	struct		drm_i915_gem_object *obj;
31
	struct		drm_i915_gem_object *obj;
30
};
32
};
31
 
33
 
32
#define I915_READ_TAIL(ring) I915_READ(RING_TAIL((ring)->mmio_base))
34
#define I915_READ_TAIL(ring) I915_READ(RING_TAIL((ring)->mmio_base))
33
#define I915_WRITE_TAIL(ring, val) I915_WRITE(RING_TAIL((ring)->mmio_base), val)
35
#define I915_WRITE_TAIL(ring, val) I915_WRITE(RING_TAIL((ring)->mmio_base), val)
34
 
36
 
35
#define I915_READ_START(ring) I915_READ(RING_START((ring)->mmio_base))
37
#define I915_READ_START(ring) I915_READ(RING_START((ring)->mmio_base))
36
#define I915_WRITE_START(ring, val) I915_WRITE(RING_START((ring)->mmio_base), val)
38
#define I915_WRITE_START(ring, val) I915_WRITE(RING_START((ring)->mmio_base), val)
37
 
39
 
38
#define I915_READ_HEAD(ring)  I915_READ(RING_HEAD((ring)->mmio_base))
40
#define I915_READ_HEAD(ring)  I915_READ(RING_HEAD((ring)->mmio_base))
39
#define I915_WRITE_HEAD(ring, val) I915_WRITE(RING_HEAD((ring)->mmio_base), val)
41
#define I915_WRITE_HEAD(ring, val) I915_WRITE(RING_HEAD((ring)->mmio_base), val)
40
 
42
 
41
#define I915_READ_CTL(ring) I915_READ(RING_CTL((ring)->mmio_base))
43
#define I915_READ_CTL(ring) I915_READ(RING_CTL((ring)->mmio_base))
42
#define I915_WRITE_CTL(ring, val) I915_WRITE(RING_CTL((ring)->mmio_base), val)
44
#define I915_WRITE_CTL(ring, val) I915_WRITE(RING_CTL((ring)->mmio_base), val)
43
 
45
 
44
#define I915_READ_IMR(ring) I915_READ(RING_IMR((ring)->mmio_base))
46
#define I915_READ_IMR(ring) I915_READ(RING_IMR((ring)->mmio_base))
45
#define I915_WRITE_IMR(ring, val) I915_WRITE(RING_IMR((ring)->mmio_base), val)
47
#define I915_WRITE_IMR(ring, val) I915_WRITE(RING_IMR((ring)->mmio_base), val)
46
 
48
 
47
#define I915_READ_MODE(ring) I915_READ(RING_MI_MODE((ring)->mmio_base))
49
#define I915_READ_MODE(ring) I915_READ(RING_MI_MODE((ring)->mmio_base))
48
#define I915_WRITE_MODE(ring, val) I915_WRITE(RING_MI_MODE((ring)->mmio_base), val)
50
#define I915_WRITE_MODE(ring, val) I915_WRITE(RING_MI_MODE((ring)->mmio_base), val)
49
 
51
 
50
/* seqno size is actually only a uint32, but since we plan to use MI_FLUSH_DW to
52
/* seqno size is actually only a uint32, but since we plan to use MI_FLUSH_DW to
51
 * do the writes, and that must have qw aligned offsets, simply pretend it's 8b.
53
 * do the writes, and that must have qw aligned offsets, simply pretend it's 8b.
52
 */
54
 */
53
#define i915_semaphore_seqno_size sizeof(uint64_t)
55
#define i915_semaphore_seqno_size sizeof(uint64_t)
54
#define GEN8_SIGNAL_OFFSET(__ring, to)			     \
56
#define GEN8_SIGNAL_OFFSET(__ring, to)			     \
55
	(i915_gem_obj_ggtt_offset(dev_priv->semaphore_obj) + \
57
	(i915_gem_obj_ggtt_offset(dev_priv->semaphore_obj) + \
56
	((__ring)->id * I915_NUM_RINGS * i915_semaphore_seqno_size) +	\
58
	((__ring)->id * I915_NUM_RINGS * i915_semaphore_seqno_size) +	\
57
	(i915_semaphore_seqno_size * (to)))
59
	(i915_semaphore_seqno_size * (to)))
58
 
60
 
59
#define GEN8_WAIT_OFFSET(__ring, from)			     \
61
#define GEN8_WAIT_OFFSET(__ring, from)			     \
60
	(i915_gem_obj_ggtt_offset(dev_priv->semaphore_obj) + \
62
	(i915_gem_obj_ggtt_offset(dev_priv->semaphore_obj) + \
61
	((from) * I915_NUM_RINGS * i915_semaphore_seqno_size) + \
63
	((from) * I915_NUM_RINGS * i915_semaphore_seqno_size) + \
62
	(i915_semaphore_seqno_size * (__ring)->id))
64
	(i915_semaphore_seqno_size * (__ring)->id))
63
 
65
 
64
#define GEN8_RING_SEMAPHORE_INIT do { \
66
#define GEN8_RING_SEMAPHORE_INIT do { \
65
	if (!dev_priv->semaphore_obj) { \
67
	if (!dev_priv->semaphore_obj) { \
66
		break; \
68
		break; \
67
	} \
69
	} \
68
	ring->semaphore.signal_ggtt[RCS] = GEN8_SIGNAL_OFFSET(ring, RCS); \
70
	ring->semaphore.signal_ggtt[RCS] = GEN8_SIGNAL_OFFSET(ring, RCS); \
69
	ring->semaphore.signal_ggtt[VCS] = GEN8_SIGNAL_OFFSET(ring, VCS); \
71
	ring->semaphore.signal_ggtt[VCS] = GEN8_SIGNAL_OFFSET(ring, VCS); \
70
	ring->semaphore.signal_ggtt[BCS] = GEN8_SIGNAL_OFFSET(ring, BCS); \
72
	ring->semaphore.signal_ggtt[BCS] = GEN8_SIGNAL_OFFSET(ring, BCS); \
71
	ring->semaphore.signal_ggtt[VECS] = GEN8_SIGNAL_OFFSET(ring, VECS); \
73
	ring->semaphore.signal_ggtt[VECS] = GEN8_SIGNAL_OFFSET(ring, VECS); \
72
	ring->semaphore.signal_ggtt[VCS2] = GEN8_SIGNAL_OFFSET(ring, VCS2); \
74
	ring->semaphore.signal_ggtt[VCS2] = GEN8_SIGNAL_OFFSET(ring, VCS2); \
73
	ring->semaphore.signal_ggtt[ring->id] = MI_SEMAPHORE_SYNC_INVALID; \
75
	ring->semaphore.signal_ggtt[ring->id] = MI_SEMAPHORE_SYNC_INVALID; \
74
	} while(0)
76
	} while(0)
75
 
77
 
76
enum intel_ring_hangcheck_action {
78
enum intel_ring_hangcheck_action {
77
	HANGCHECK_IDLE = 0,
79
	HANGCHECK_IDLE = 0,
78
	HANGCHECK_WAIT,
80
	HANGCHECK_WAIT,
79
	HANGCHECK_ACTIVE,
81
	HANGCHECK_ACTIVE,
80
	HANGCHECK_ACTIVE_LOOP,
82
	HANGCHECK_ACTIVE_LOOP,
81
	HANGCHECK_KICK,
83
	HANGCHECK_KICK,
82
	HANGCHECK_HUNG,
84
	HANGCHECK_HUNG,
83
};
85
};
84
 
86
 
85
#define HANGCHECK_SCORE_RING_HUNG 31
87
#define HANGCHECK_SCORE_RING_HUNG 31
86
 
88
 
87
struct intel_ring_hangcheck {
89
struct intel_ring_hangcheck {
88
	u64 acthd;
90
	u64 acthd;
89
	u64 max_acthd;
91
	u64 max_acthd;
90
	u32 seqno;
92
	u32 seqno;
91
	int score;
93
	int score;
92
	enum intel_ring_hangcheck_action action;
94
	enum intel_ring_hangcheck_action action;
93
	int deadlock;
95
	int deadlock;
94
};
96
};
95
 
97
 
96
struct intel_ringbuffer {
98
struct intel_ringbuffer {
97
	struct drm_i915_gem_object *obj;
99
	struct drm_i915_gem_object *obj;
98
	void		__iomem *virtual_start;
100
	void __iomem *virtual_start;
99
 
101
 
100
	struct intel_engine_cs *ring;
102
	struct intel_engine_cs *ring;
101
 
-
 
102
	/*
-
 
103
	 * FIXME: This backpointer is an artifact of the history of how the
-
 
104
	 * execlist patches came into being. It will get removed once the basic
-
 
105
	 * code has landed.
-
 
106
	 */
-
 
107
	struct intel_context *FIXME_lrc_ctx;
-
 
108
 
103
 
109
	u32		head;
104
	u32 head;
110
	u32		tail;
105
	u32 tail;
111
	int		space;
106
	int space;
112
	int		size;
107
	int size;
113
	int		effective_size;
108
	int effective_size;
-
 
109
	int reserved_size;
-
 
110
	int reserved_tail;
-
 
111
	bool reserved_in_use;
114
 
112
 
115
	/** We track the position of the requests in the ring buffer, and
113
	/** We track the position of the requests in the ring buffer, and
116
	 * when each is retired we increment last_retired_head as the GPU
114
	 * when each is retired we increment last_retired_head as the GPU
117
	 * must have finished processing the request and so we know we
115
	 * must have finished processing the request and so we know we
118
	 * can advance the ringbuffer up to that position.
116
	 * can advance the ringbuffer up to that position.
119
	 *
117
	 *
120
	 * last_retired_head is set to -1 after the value is consumed so
118
	 * last_retired_head is set to -1 after the value is consumed so
121
	 * we can detect new retirements.
119
	 * we can detect new retirements.
122
	 */
120
	 */
123
	u32		last_retired_head;
121
	u32 last_retired_head;
124
};
122
};
-
 
123
 
-
 
124
struct	intel_context;
-
 
125
struct drm_i915_reg_descriptor;
-
 
126
 
-
 
127
/*
-
 
128
 * we use a single page to load ctx workarounds so all of these
-
 
129
 * values are referred in terms of dwords
-
 
130
 *
-
 
131
 * struct i915_wa_ctx_bb:
-
 
132
 *  offset: specifies batch starting position, also helpful in case
-
 
133
 *    if we want to have multiple batches at different offsets based on
-
 
134
 *    some criteria. It is not a requirement at the moment but provides
-
 
135
 *    an option for future use.
-
 
136
 *  size: size of the batch in DWORDS
-
 
137
 */
-
 
138
struct  i915_ctx_workarounds {
-
 
139
	struct i915_wa_ctx_bb {
-
 
140
		u32 offset;
-
 
141
		u32 size;
-
 
142
	} indirect_ctx, per_ctx;
-
 
143
	struct drm_i915_gem_object *obj;
-
 
144
};
125
 
145
 
126
struct  intel_engine_cs {
146
struct  intel_engine_cs {
127
	const char	*name;
147
	const char	*name;
128
	enum intel_ring_id {
148
	enum intel_ring_id {
129
		RCS = 0x0,
149
		RCS = 0x0,
130
		VCS,
150
		VCS,
131
		BCS,
151
		BCS,
132
		VECS,
152
		VECS,
133
		VCS2
153
		VCS2
134
	} id;
154
	} id;
135
#define I915_NUM_RINGS 5
155
#define I915_NUM_RINGS 5
136
#define LAST_USER_RING (VECS + 1)
156
#define LAST_USER_RING (VECS + 1)
137
	u32		mmio_base;
157
	u32		mmio_base;
138
	struct		drm_device *dev;
158
	struct		drm_device *dev;
139
	struct intel_ringbuffer *buffer;
159
	struct intel_ringbuffer *buffer;
-
 
160
 
-
 
161
	/*
-
 
162
	 * A pool of objects to use as shadow copies of client batch buffers
-
 
163
	 * when the command parser is enabled. Prevents the client from
-
 
164
	 * modifying the batch contents after software parsing.
-
 
165
	 */
-
 
166
	struct i915_gem_batch_pool batch_pool;
140
 
167
 
-
 
168
	struct intel_hw_status_page status_page;
141
	struct intel_hw_status_page status_page;
169
	struct i915_ctx_workarounds wa_ctx;
142
 
170
 
143
	unsigned irq_refcount; /* protected by dev_priv->irq_lock */
171
	unsigned irq_refcount; /* protected by dev_priv->irq_lock */
144
	u32		irq_enable_mask;	/* bitmask to enable ring interrupt */
172
	u32		irq_enable_mask;	/* bitmask to enable ring interrupt */
145
	u32		trace_irq_seqno;
173
	struct drm_i915_gem_request *trace_irq_req;
146
	bool __must_check (*irq_get)(struct intel_engine_cs *ring);
174
	bool __must_check (*irq_get)(struct intel_engine_cs *ring);
147
	void		(*irq_put)(struct intel_engine_cs *ring);
175
	void		(*irq_put)(struct intel_engine_cs *ring);
148
 
176
 
149
	int		(*init)(struct intel_engine_cs *ring);
177
	int		(*init_hw)(struct intel_engine_cs *ring);
150
 
-
 
151
	int		(*init_context)(struct intel_engine_cs *ring,
178
 
152
					struct intel_context *ctx);
179
	int		(*init_context)(struct drm_i915_gem_request *req);
153
 
180
 
154
	void		(*write_tail)(struct intel_engine_cs *ring,
181
	void		(*write_tail)(struct intel_engine_cs *ring,
155
				      u32 value);
182
				      u32 value);
156
	int __must_check (*flush)(struct intel_engine_cs *ring,
183
	int __must_check (*flush)(struct drm_i915_gem_request *req,
157
				  u32	invalidate_domains,
184
				  u32	invalidate_domains,
158
				  u32	flush_domains);
185
				  u32	flush_domains);
159
	int		(*add_request)(struct intel_engine_cs *ring);
186
	int		(*add_request)(struct drm_i915_gem_request *req);
160
	/* Some chipsets are not quite as coherent as advertised and need
187
	/* Some chipsets are not quite as coherent as advertised and need
161
	 * an expensive kick to force a true read of the up-to-date seqno.
188
	 * an expensive kick to force a true read of the up-to-date seqno.
162
	 * However, the up-to-date seqno is not always required and the last
189
	 * However, the up-to-date seqno is not always required and the last
163
	 * seen value is good enough. Note that the seqno will always be
190
	 * seen value is good enough. Note that the seqno will always be
164
	 * monotonic, even if not coherent.
191
	 * monotonic, even if not coherent.
165
	 */
192
	 */
166
	u32		(*get_seqno)(struct intel_engine_cs *ring,
193
	u32		(*get_seqno)(struct intel_engine_cs *ring,
167
				     bool lazy_coherency);
194
				     bool lazy_coherency);
168
	void		(*set_seqno)(struct intel_engine_cs *ring,
195
	void		(*set_seqno)(struct intel_engine_cs *ring,
169
				     u32 seqno);
196
				     u32 seqno);
170
	int		(*dispatch_execbuffer)(struct intel_engine_cs *ring,
197
	int		(*dispatch_execbuffer)(struct drm_i915_gem_request *req,
171
					       u64 offset, u32 length,
198
					       u64 offset, u32 length,
172
					       unsigned flags);
199
					       unsigned dispatch_flags);
173
#define I915_DISPATCH_SECURE 0x1
200
#define I915_DISPATCH_SECURE 0x1
174
#define I915_DISPATCH_PINNED 0x2
201
#define I915_DISPATCH_PINNED 0x2
-
 
202
#define I915_DISPATCH_RS     0x4
175
	void		(*cleanup)(struct intel_engine_cs *ring);
203
	void		(*cleanup)(struct intel_engine_cs *ring);
176
 
204
 
177
	/* GEN8 signal/wait table - never trust comments!
205
	/* GEN8 signal/wait table - never trust comments!
178
	 *	  signal to	signal to    signal to   signal to      signal to
206
	 *	  signal to	signal to    signal to   signal to      signal to
179
	 *	    RCS		   VCS          BCS        VECS		 VCS2
207
	 *	    RCS		   VCS          BCS        VECS		 VCS2
180
	 *      --------------------------------------------------------------------
208
	 *      --------------------------------------------------------------------
181
	 *  RCS | NOP (0x00) | VCS (0x08) | BCS (0x10) | VECS (0x18) | VCS2 (0x20) |
209
	 *  RCS | NOP (0x00) | VCS (0x08) | BCS (0x10) | VECS (0x18) | VCS2 (0x20) |
182
	 *	|-------------------------------------------------------------------
210
	 *	|-------------------------------------------------------------------
183
	 *  VCS | RCS (0x28) | NOP (0x30) | BCS (0x38) | VECS (0x40) | VCS2 (0x48) |
211
	 *  VCS | RCS (0x28) | NOP (0x30) | BCS (0x38) | VECS (0x40) | VCS2 (0x48) |
184
	 *	|-------------------------------------------------------------------
212
	 *	|-------------------------------------------------------------------
185
	 *  BCS | RCS (0x50) | VCS (0x58) | NOP (0x60) | VECS (0x68) | VCS2 (0x70) |
213
	 *  BCS | RCS (0x50) | VCS (0x58) | NOP (0x60) | VECS (0x68) | VCS2 (0x70) |
186
	 *	|-------------------------------------------------------------------
214
	 *	|-------------------------------------------------------------------
187
	 * VECS | RCS (0x78) | VCS (0x80) | BCS (0x88) |  NOP (0x90) | VCS2 (0x98) |
215
	 * VECS | RCS (0x78) | VCS (0x80) | BCS (0x88) |  NOP (0x90) | VCS2 (0x98) |
188
	 *	|-------------------------------------------------------------------
216
	 *	|-------------------------------------------------------------------
189
	 * VCS2 | RCS (0xa0) | VCS (0xa8) | BCS (0xb0) | VECS (0xb8) | NOP  (0xc0) |
217
	 * VCS2 | RCS (0xa0) | VCS (0xa8) | BCS (0xb0) | VECS (0xb8) | NOP  (0xc0) |
190
	 *	|-------------------------------------------------------------------
218
	 *	|-------------------------------------------------------------------
191
	 *
219
	 *
192
	 * Generalization:
220
	 * Generalization:
193
	 *  f(x, y) := (x->id * NUM_RINGS * seqno_size) + (seqno_size * y->id)
221
	 *  f(x, y) := (x->id * NUM_RINGS * seqno_size) + (seqno_size * y->id)
194
	 *  ie. transpose of g(x, y)
222
	 *  ie. transpose of g(x, y)
195
	 *
223
	 *
196
	 *	 sync from	sync from    sync from    sync from	sync from
224
	 *	 sync from	sync from    sync from    sync from	sync from
197
	 *	    RCS		   VCS          BCS        VECS		 VCS2
225
	 *	    RCS		   VCS          BCS        VECS		 VCS2
198
	 *      --------------------------------------------------------------------
226
	 *      --------------------------------------------------------------------
199
	 *  RCS | NOP (0x00) | VCS (0x28) | BCS (0x50) | VECS (0x78) | VCS2 (0xa0) |
227
	 *  RCS | NOP (0x00) | VCS (0x28) | BCS (0x50) | VECS (0x78) | VCS2 (0xa0) |
200
	 *	|-------------------------------------------------------------------
228
	 *	|-------------------------------------------------------------------
201
	 *  VCS | RCS (0x08) | NOP (0x30) | BCS (0x58) | VECS (0x80) | VCS2 (0xa8) |
229
	 *  VCS | RCS (0x08) | NOP (0x30) | BCS (0x58) | VECS (0x80) | VCS2 (0xa8) |
202
	 *	|-------------------------------------------------------------------
230
	 *	|-------------------------------------------------------------------
203
	 *  BCS | RCS (0x10) | VCS (0x38) | NOP (0x60) | VECS (0x88) | VCS2 (0xb0) |
231
	 *  BCS | RCS (0x10) | VCS (0x38) | NOP (0x60) | VECS (0x88) | VCS2 (0xb0) |
204
	 *	|-------------------------------------------------------------------
232
	 *	|-------------------------------------------------------------------
205
	 * VECS | RCS (0x18) | VCS (0x40) | BCS (0x68) |  NOP (0x90) | VCS2 (0xb8) |
233
	 * VECS | RCS (0x18) | VCS (0x40) | BCS (0x68) |  NOP (0x90) | VCS2 (0xb8) |
206
	 *	|-------------------------------------------------------------------
234
	 *	|-------------------------------------------------------------------
207
	 * VCS2 | RCS (0x20) | VCS (0x48) | BCS (0x70) | VECS (0x98) |  NOP (0xc0) |
235
	 * VCS2 | RCS (0x20) | VCS (0x48) | BCS (0x70) | VECS (0x98) |  NOP (0xc0) |
208
	 *	|-------------------------------------------------------------------
236
	 *	|-------------------------------------------------------------------
209
	 *
237
	 *
210
	 * Generalization:
238
	 * Generalization:
211
	 *  g(x, y) := (y->id * NUM_RINGS * seqno_size) + (seqno_size * x->id)
239
	 *  g(x, y) := (y->id * NUM_RINGS * seqno_size) + (seqno_size * x->id)
212
	 *  ie. transpose of f(x, y)
240
	 *  ie. transpose of f(x, y)
213
	 */
241
	 */
214
	struct {
242
	struct {
215
		u32	sync_seqno[I915_NUM_RINGS-1];
243
		u32	sync_seqno[I915_NUM_RINGS-1];
216
 
244
 
217
		union {
245
		union {
218
		struct {
246
			struct {
219
	/* our mbox written by others */
247
				/* our mbox written by others */
220
			u32		wait[I915_NUM_RINGS];
248
				u32		wait[I915_NUM_RINGS];
221
	/* mboxes this ring signals to */
249
				/* mboxes this ring signals to */
222
			u32		signal[I915_NUM_RINGS];
250
				u32		signal[I915_NUM_RINGS];
223
		} mbox;
251
			} mbox;
224
			u64		signal_ggtt[I915_NUM_RINGS];
252
			u64		signal_ggtt[I915_NUM_RINGS];
225
		};
253
		};
226
 
254
 
227
		/* AKA wait() */
255
		/* AKA wait() */
228
		int	(*sync_to)(struct intel_engine_cs *ring,
256
		int	(*sync_to)(struct drm_i915_gem_request *to_req,
229
				   struct intel_engine_cs *to,
257
				   struct intel_engine_cs *from,
230
				   u32 seqno);
258
				   u32 seqno);
231
		int	(*signal)(struct intel_engine_cs *signaller,
259
		int	(*signal)(struct drm_i915_gem_request *signaller_req,
232
				  /* num_dwords needed by caller */
260
				  /* num_dwords needed by caller */
233
				  unsigned int num_dwords);
261
				  unsigned int num_dwords);
234
	} semaphore;
262
	} semaphore;
235
 
263
 
236
	/* Execlists */
264
	/* Execlists */
237
	spinlock_t execlist_lock;
265
	spinlock_t execlist_lock;
238
	struct list_head execlist_queue;
266
	struct list_head execlist_queue;
239
	struct list_head execlist_retired_req_list;
267
	struct list_head execlist_retired_req_list;
240
	u8 next_context_status_buffer;
268
	u8 next_context_status_buffer;
241
	u32             irq_keep_mask; /* bitmask for interrupts that should not be masked */
269
	u32             irq_keep_mask; /* bitmask for interrupts that should not be masked */
242
	int		(*emit_request)(struct intel_ringbuffer *ringbuf);
270
	int		(*emit_request)(struct drm_i915_gem_request *request);
243
	int		(*emit_flush)(struct intel_ringbuffer *ringbuf,
271
	int		(*emit_flush)(struct drm_i915_gem_request *request,
244
				      u32 invalidate_domains,
272
				      u32 invalidate_domains,
245
				      u32 flush_domains);
273
				      u32 flush_domains);
246
	int		(*emit_bb_start)(struct intel_ringbuffer *ringbuf,
274
	int		(*emit_bb_start)(struct drm_i915_gem_request *req,
247
					 u64 offset, unsigned flags);
275
					 u64 offset, unsigned dispatch_flags);
248
 
276
 
249
	/**
277
	/**
250
	 * List of objects currently involved in rendering from the
278
	 * List of objects currently involved in rendering from the
251
	 * ringbuffer.
279
	 * ringbuffer.
252
	 *
280
	 *
253
	 * Includes buffers having the contents of their GPU caches
281
	 * Includes buffers having the contents of their GPU caches
254
	 * flushed, not necessarily primitives.  last_rendering_seqno
282
	 * flushed, not necessarily primitives.  last_read_req
255
	 * represents when the rendering involved will be completed.
283
	 * represents when the rendering involved will be completed.
256
	 *
284
	 *
257
	 * A reference is held on the buffer while on this list.
285
	 * A reference is held on the buffer while on this list.
258
	 */
286
	 */
259
	struct list_head active_list;
287
	struct list_head active_list;
260
 
288
 
261
	/**
289
	/**
262
	 * List of breadcrumbs associated with GPU requests currently
290
	 * List of breadcrumbs associated with GPU requests currently
263
	 * outstanding.
291
	 * outstanding.
264
	 */
292
	 */
265
	struct list_head request_list;
293
	struct list_head request_list;
266
 
294
 
267
	/**
295
	/**
268
	 * Do we have some not yet emitted requests outstanding?
296
	 * Seqno of request most recently submitted to request_list.
-
 
297
	 * Used exclusively by hang checker to avoid grabbing lock while
-
 
298
	 * inspecting request list.
269
	 */
299
	 */
270
	struct drm_i915_gem_request *preallocated_lazy_request;
-
 
271
	u32 outstanding_lazy_seqno;
300
	u32 last_submitted_seqno;
-
 
301
 
272
	bool gpu_caches_dirty;
302
	bool gpu_caches_dirty;
273
	bool fbc_dirty;
-
 
274
 
303
 
275
	wait_queue_head_t irq_queue;
304
	wait_queue_head_t irq_queue;
276
 
305
 
277
	struct intel_context *default_context;
306
	struct intel_context *default_context;
278
	struct intel_context *last_context;
307
	struct intel_context *last_context;
279
 
308
 
280
	struct intel_ring_hangcheck hangcheck;
309
	struct intel_ring_hangcheck hangcheck;
281
 
310
 
282
	struct {
311
	struct {
283
		struct drm_i915_gem_object *obj;
312
		struct drm_i915_gem_object *obj;
284
		u32 gtt_offset;
313
		u32 gtt_offset;
285
		volatile u32 *cpu_page;
314
		volatile u32 *cpu_page;
286
	} scratch;
315
	} scratch;
287
 
316
 
288
	bool needs_cmd_parser;
317
	bool needs_cmd_parser;
289
 
318
 
290
	/*
319
	/*
291
	 * Table of commands the command parser needs to know about
320
	 * Table of commands the command parser needs to know about
292
	 * for this ring.
321
	 * for this ring.
293
	 */
322
	 */
294
	DECLARE_HASHTABLE(cmd_hash, I915_CMD_HASH_ORDER);
323
	DECLARE_HASHTABLE(cmd_hash, I915_CMD_HASH_ORDER);
295
 
324
 
296
	/*
325
	/*
297
	 * Table of registers allowed in commands that read/write registers.
326
	 * Table of registers allowed in commands that read/write registers.
298
	 */
327
	 */
299
	const u32 *reg_table;
328
	const struct drm_i915_reg_descriptor *reg_table;
300
	int reg_count;
329
	int reg_count;
301
 
330
 
302
	/*
331
	/*
303
	 * Table of registers allowed in commands that read/write registers, but
332
	 * Table of registers allowed in commands that read/write registers, but
304
	 * only from the DRM master.
333
	 * only from the DRM master.
305
	 */
334
	 */
306
	const u32 *master_reg_table;
335
	const struct drm_i915_reg_descriptor *master_reg_table;
307
	int master_reg_count;
336
	int master_reg_count;
308
 
337
 
309
	/*
338
	/*
310
	 * Returns the bitmask for the length field of the specified command.
339
	 * Returns the bitmask for the length field of the specified command.
311
	 * Return 0 for an unrecognized/invalid command.
340
	 * Return 0 for an unrecognized/invalid command.
312
	 *
341
	 *
313
	 * If the command parser finds an entry for a command in the ring's
342
	 * If the command parser finds an entry for a command in the ring's
314
	 * cmd_tables, it gets the command's length based on the table entry.
343
	 * cmd_tables, it gets the command's length based on the table entry.
315
	 * If not, it calls this function to determine the per-ring length field
344
	 * If not, it calls this function to determine the per-ring length field
316
	 * encoding for the command (i.e. certain opcode ranges use certain bits
345
	 * encoding for the command (i.e. certain opcode ranges use certain bits
317
	 * to encode the command length in the header).
346
	 * to encode the command length in the header).
318
	 */
347
	 */
319
	u32 (*get_cmd_length_mask)(u32 cmd_header);
348
	u32 (*get_cmd_length_mask)(u32 cmd_header);
320
};
349
};
321
 
350
 
322
bool intel_ring_initialized(struct intel_engine_cs *ring);
351
bool intel_ring_initialized(struct intel_engine_cs *ring);
323
 
352
 
324
static inline unsigned
353
static inline unsigned
325
intel_ring_flag(struct intel_engine_cs *ring)
354
intel_ring_flag(struct intel_engine_cs *ring)
326
{
355
{
327
	return 1 << ring->id;
356
	return 1 << ring->id;
328
}
357
}
329
 
358
 
330
static inline u32
359
static inline u32
331
intel_ring_sync_index(struct intel_engine_cs *ring,
360
intel_ring_sync_index(struct intel_engine_cs *ring,
332
		      struct intel_engine_cs *other)
361
		      struct intel_engine_cs *other)
333
{
362
{
334
	int idx;
363
	int idx;
335
 
364
 
336
	/*
365
	/*
337
	 * rcs -> 0 = vcs, 1 = bcs, 2 = vecs, 3 = vcs2;
366
	 * rcs -> 0 = vcs, 1 = bcs, 2 = vecs, 3 = vcs2;
338
	 * vcs -> 0 = bcs, 1 = vecs, 2 = vcs2, 3 = rcs;
367
	 * vcs -> 0 = bcs, 1 = vecs, 2 = vcs2, 3 = rcs;
339
	 * bcs -> 0 = vecs, 1 = vcs2. 2 = rcs, 3 = vcs;
368
	 * bcs -> 0 = vecs, 1 = vcs2. 2 = rcs, 3 = vcs;
340
	 * vecs -> 0 = vcs2, 1 = rcs, 2 = vcs, 3 = bcs;
369
	 * vecs -> 0 = vcs2, 1 = rcs, 2 = vcs, 3 = bcs;
341
	 * vcs2 -> 0 = rcs, 1 = vcs, 2 = bcs, 3 = vecs;
370
	 * vcs2 -> 0 = rcs, 1 = vcs, 2 = bcs, 3 = vecs;
342
	 */
371
	 */
343
 
372
 
344
	idx = (other - ring) - 1;
373
	idx = (other - ring) - 1;
345
	if (idx < 0)
374
	if (idx < 0)
346
		idx += I915_NUM_RINGS;
375
		idx += I915_NUM_RINGS;
347
 
376
 
348
	return idx;
377
	return idx;
349
}
378
}
-
 
379
 
-
 
380
static inline void
-
 
381
intel_flush_status_page(struct intel_engine_cs *ring, int reg)
-
 
382
{
-
 
383
	drm_clflush_virt_range(&ring->status_page.page_addr[reg],
-
 
384
			       sizeof(uint32_t));
-
 
385
}
350
 
386
 
351
static inline u32
387
static inline u32
352
intel_read_status_page(struct intel_engine_cs *ring,
388
intel_read_status_page(struct intel_engine_cs *ring,
353
		       int reg)
389
		       int reg)
354
{
390
{
355
	/* Ensure that the compiler doesn't optimize away the load. */
391
	/* Ensure that the compiler doesn't optimize away the load. */
356
	barrier();
392
	barrier();
357
	return ring->status_page.page_addr[reg];
393
	return ring->status_page.page_addr[reg];
358
}
394
}
359
 
395
 
360
static inline void
396
static inline void
361
intel_write_status_page(struct intel_engine_cs *ring,
397
intel_write_status_page(struct intel_engine_cs *ring,
362
			int reg, u32 value)
398
			int reg, u32 value)
363
{
399
{
364
	ring->status_page.page_addr[reg] = value;
400
	ring->status_page.page_addr[reg] = value;
365
}
401
}
366
 
402
 
367
/**
403
/**
368
 * Reads a dword out of the status page, which is written to from the command
404
 * Reads a dword out of the status page, which is written to from the command
369
 * queue by automatic updates, MI_REPORT_HEAD, MI_STORE_DATA_INDEX, or
405
 * queue by automatic updates, MI_REPORT_HEAD, MI_STORE_DATA_INDEX, or
370
 * MI_STORE_DATA_IMM.
406
 * MI_STORE_DATA_IMM.
371
 *
407
 *
372
 * The following dwords have a reserved meaning:
408
 * The following dwords have a reserved meaning:
373
 * 0x00: ISR copy, updated when an ISR bit not set in the HWSTAM changes.
409
 * 0x00: ISR copy, updated when an ISR bit not set in the HWSTAM changes.
374
 * 0x04: ring 0 head pointer
410
 * 0x04: ring 0 head pointer
375
 * 0x05: ring 1 head pointer (915-class)
411
 * 0x05: ring 1 head pointer (915-class)
376
 * 0x06: ring 2 head pointer (915-class)
412
 * 0x06: ring 2 head pointer (915-class)
377
 * 0x10-0x1b: Context status DWords (GM45)
413
 * 0x10-0x1b: Context status DWords (GM45)
378
 * 0x1f: Last written status offset. (GM45)
414
 * 0x1f: Last written status offset. (GM45)
-
 
415
 * 0x20-0x2f: Reserved (Gen6+)
379
 *
416
 *
380
 * The area from dword 0x20 to 0x3ff is available for driver usage.
417
 * The area from dword 0x30 to 0x3ff is available for driver usage.
381
 */
418
 */
382
#define I915_GEM_HWS_INDEX		0x20
419
#define I915_GEM_HWS_INDEX		0x30
383
#define I915_GEM_HWS_SCRATCH_INDEX	0x30
420
#define I915_GEM_HWS_SCRATCH_INDEX	0x40
384
#define I915_GEM_HWS_SCRATCH_ADDR (I915_GEM_HWS_SCRATCH_INDEX << MI_STORE_DWORD_INDEX_SHIFT)
421
#define I915_GEM_HWS_SCRATCH_ADDR (I915_GEM_HWS_SCRATCH_INDEX << MI_STORE_DWORD_INDEX_SHIFT)
-
 
422
 
385
 
423
struct intel_ringbuffer *
386
void intel_unpin_ringbuffer_obj(struct intel_ringbuffer *ringbuf);
424
intel_engine_create_ringbuffer(struct intel_engine_cs *engine, int size);
387
int intel_pin_and_map_ringbuffer_obj(struct drm_device *dev,
425
int intel_pin_and_map_ringbuffer_obj(struct drm_device *dev,
388
				     struct intel_ringbuffer *ringbuf);
426
				     struct intel_ringbuffer *ringbuf);
389
void intel_destroy_ringbuffer_obj(struct intel_ringbuffer *ringbuf);
-
 
390
int intel_alloc_ringbuffer_obj(struct drm_device *dev,
427
void intel_unpin_ringbuffer_obj(struct intel_ringbuffer *ringbuf);
391
			       struct intel_ringbuffer *ringbuf);
428
void intel_ringbuffer_free(struct intel_ringbuffer *ring);
392
 
429
 
393
void intel_stop_ring_buffer(struct intel_engine_cs *ring);
430
void intel_stop_ring_buffer(struct intel_engine_cs *ring);
394
void intel_cleanup_ring_buffer(struct intel_engine_cs *ring);
431
void intel_cleanup_ring_buffer(struct intel_engine_cs *ring);
-
 
432
 
-
 
433
int intel_ring_alloc_request_extras(struct drm_i915_gem_request *request);
395
 
434
 
396
int __must_check intel_ring_begin(struct intel_engine_cs *ring, int n);
435
int __must_check intel_ring_begin(struct drm_i915_gem_request *req, int n);
397
int __must_check intel_ring_cacheline_align(struct intel_engine_cs *ring);
436
int __must_check intel_ring_cacheline_align(struct drm_i915_gem_request *req);
398
static inline void intel_ring_emit(struct intel_engine_cs *ring,
437
static inline void intel_ring_emit(struct intel_engine_cs *ring,
399
				   u32 data)
438
				   u32 data)
400
{
439
{
401
	struct intel_ringbuffer *ringbuf = ring->buffer;
440
	struct intel_ringbuffer *ringbuf = ring->buffer;
402
	iowrite32(data, ringbuf->virtual_start + ringbuf->tail);
441
	iowrite32(data, ringbuf->virtual_start + ringbuf->tail);
403
	ringbuf->tail += 4;
442
	ringbuf->tail += 4;
404
}
443
}
405
static inline void intel_ring_advance(struct intel_engine_cs *ring)
444
static inline void intel_ring_advance(struct intel_engine_cs *ring)
406
{
445
{
407
	struct intel_ringbuffer *ringbuf = ring->buffer;
446
	struct intel_ringbuffer *ringbuf = ring->buffer;
408
	ringbuf->tail &= ringbuf->size - 1;
447
	ringbuf->tail &= ringbuf->size - 1;
409
}
448
}
410
int __intel_ring_space(int head, int tail, int size);
449
int __intel_ring_space(int head, int tail, int size);
-
 
450
void intel_ring_update_space(struct intel_ringbuffer *ringbuf);
411
int intel_ring_space(struct intel_ringbuffer *ringbuf);
451
int intel_ring_space(struct intel_ringbuffer *ringbuf);
412
bool intel_ring_stopped(struct intel_engine_cs *ring);
452
bool intel_ring_stopped(struct intel_engine_cs *ring);
413
void __intel_ring_advance(struct intel_engine_cs *ring);
-
 
414
 
453
 
415
int __must_check intel_ring_idle(struct intel_engine_cs *ring);
454
int __must_check intel_ring_idle(struct intel_engine_cs *ring);
416
void intel_ring_init_seqno(struct intel_engine_cs *ring, u32 seqno);
455
void intel_ring_init_seqno(struct intel_engine_cs *ring, u32 seqno);
417
int intel_ring_flush_all_caches(struct intel_engine_cs *ring);
456
int intel_ring_flush_all_caches(struct drm_i915_gem_request *req);
418
int intel_ring_invalidate_all_caches(struct intel_engine_cs *ring);
457
int intel_ring_invalidate_all_caches(struct drm_i915_gem_request *req);
419
 
458
 
420
void intel_fini_pipe_control(struct intel_engine_cs *ring);
459
void intel_fini_pipe_control(struct intel_engine_cs *ring);
421
int intel_init_pipe_control(struct intel_engine_cs *ring);
460
int intel_init_pipe_control(struct intel_engine_cs *ring);
422
 
461
 
423
int intel_init_render_ring_buffer(struct drm_device *dev);
462
int intel_init_render_ring_buffer(struct drm_device *dev);
424
int intel_init_bsd_ring_buffer(struct drm_device *dev);
463
int intel_init_bsd_ring_buffer(struct drm_device *dev);
425
int intel_init_bsd2_ring_buffer(struct drm_device *dev);
464
int intel_init_bsd2_ring_buffer(struct drm_device *dev);
426
int intel_init_blt_ring_buffer(struct drm_device *dev);
465
int intel_init_blt_ring_buffer(struct drm_device *dev);
427
int intel_init_vebox_ring_buffer(struct drm_device *dev);
466
int intel_init_vebox_ring_buffer(struct drm_device *dev);
428
 
467
 
429
u64 intel_ring_get_active_head(struct intel_engine_cs *ring);
468
u64 intel_ring_get_active_head(struct intel_engine_cs *ring);
430
void intel_ring_setup_status_page(struct intel_engine_cs *ring);
-
 
431
 
469
 
432
int init_workarounds_ring(struct intel_engine_cs *ring);
470
int init_workarounds_ring(struct intel_engine_cs *ring);
433
 
471
 
434
static inline u32 intel_ring_get_tail(struct intel_ringbuffer *ringbuf)
472
static inline u32 intel_ring_get_tail(struct intel_ringbuffer *ringbuf)
435
{
473
{
436
	return ringbuf->tail;
474
	return ringbuf->tail;
437
}
475
}
438
 
-
 
439
static inline u32 intel_ring_get_seqno(struct intel_engine_cs *ring)
476
 
440
{
477
/*
-
 
478
 * Arbitrary size for largest possible 'add request' sequence. The code paths
-
 
479
 * are complex and variable. Empirical measurement shows that the worst case
-
 
480
 * is ILK at 136 words. Reserving too much is better than reserving too little
441
	BUG_ON(ring->outstanding_lazy_seqno == 0);
481
 * as that allows for corner cases that might have been missed. So the figure
442
	return ring->outstanding_lazy_seqno;
482
 * has been rounded up to 160 words.
-
 
483
 */
-
 
484
#define MIN_SPACE_FOR_ADD_REQUEST	160
443
}
485
 
-
 
486
/*
-
 
487
 * Reserve space in the ring to guarantee that the i915_add_request() call
444
 
488
 * will always have sufficient room to do its stuff. The request creation
-
 
489
 * code calls this automatically.
-
 
490
 */
-
 
491
void intel_ring_reserved_space_reserve(struct intel_ringbuffer *ringbuf, int size);
-
 
492
/* Cancel the reservation, e.g. because the request is being discarded. */
445
static inline void i915_trace_irq_get(struct intel_engine_cs *ring, u32 seqno)
493
void intel_ring_reserved_space_cancel(struct intel_ringbuffer *ringbuf);
-
 
494
/* Use the reserved space - for use by i915_add_request() only. */
446
{
495
void intel_ring_reserved_space_use(struct intel_ringbuffer *ringbuf);
447
	if (ring->trace_irq_seqno == 0 && ring->irq_get(ring))
496
/* Finish with the reserved space - for use by i915_add_request() only. */
-
 
497
void intel_ring_reserved_space_end(struct intel_ringbuffer *ringbuf);
-
 
498
 
448
		ring->trace_irq_seqno = seqno;
499
/* Legacy ringbuffer specific portion of reservation code: */
449
}
500
int intel_ring_reserve_space(struct drm_i915_gem_request *request);
450
 
501
 
451
#endif /* _INTEL_RINGBUFFER_H_ */
502
#endif /* _INTEL_RINGBUFFER_H_ */