Subversion Repositories Kolibri OS

Rev

Rev 4539 | Rev 5060 | Go to most recent revision | Details | Compare with Previous | Last modification | View Log | RSS feed

Rev Author Line No. Line
2332 Serge 1
/*
2
 * Copyright © 2008-2010 Intel Corporation
3
 *
4
 * Permission is hereby granted, free of charge, to any person obtaining a
5
 * copy of this software and associated documentation files (the "Software"),
6
 * to deal in the Software without restriction, including without limitation
7
 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8
 * and/or sell copies of the Software, and to permit persons to whom the
9
 * Software is furnished to do so, subject to the following conditions:
10
 *
11
 * The above copyright notice and this permission notice (including the next
12
 * paragraph) shall be included in all copies or substantial portions of the
13
 * Software.
14
 *
15
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
18
 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20
 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21
 * IN THE SOFTWARE.
22
 *
23
 * Authors:
24
 *    Eric Anholt 
25
 *    Zou Nan hai 
26
 *    Xiang Hai hao
27
 *
28
 */
29
 
3031 serge 30
#include 
2332 Serge 31
#include "i915_drv.h"
3031 serge 32
#include 
2351 Serge 33
#include "i915_trace.h"
2332 Serge 34
#include "intel_drv.h"
35
 
36
static inline int ring_space(struct intel_ring_buffer *ring)
37
{
3243 Serge 38
	int space = (ring->head & HEAD_ADDR) - (ring->tail + I915_RING_FREE_SPACE);
2332 Serge 39
	if (space < 0)
40
		space += ring->size;
41
	return space;
42
}
43
 
4560 Serge 44
void __intel_ring_advance(struct intel_ring_buffer *ring)
45
{
46
	struct drm_i915_private *dev_priv = ring->dev->dev_private;
47
 
48
	ring->tail &= ring->size - 1;
49
	if (dev_priv->gpu_error.stop_rings & intel_ring_flag(ring))
50
		return;
51
	ring->write_tail(ring, ring->tail);
52
}
53
 
3031 serge 54
static int
55
gen2_render_ring_flush(struct intel_ring_buffer *ring,
56
		       u32	invalidate_domains,
57
		       u32	flush_domains)
2332 Serge 58
{
3031 serge 59
	u32 cmd;
60
	int ret;
2332 Serge 61
 
3031 serge 62
	cmd = MI_FLUSH;
63
	if (((invalidate_domains|flush_domains) & I915_GEM_DOMAIN_RENDER) == 0)
64
		cmd |= MI_NO_WRITE_FLUSH;
2332 Serge 65
 
3031 serge 66
	if (invalidate_domains & I915_GEM_DOMAIN_SAMPLER)
67
		cmd |= MI_READ_FLUSH;
2332 Serge 68
 
3031 serge 69
	ret = intel_ring_begin(ring, 2);
70
	if (ret)
71
		return ret;
72
 
73
	intel_ring_emit(ring, cmd);
74
	intel_ring_emit(ring, MI_NOOP);
75
	intel_ring_advance(ring);
76
 
77
	return 0;
2332 Serge 78
}
79
 
80
static int
3031 serge 81
gen4_render_ring_flush(struct intel_ring_buffer *ring,
2332 Serge 82
		  u32	invalidate_domains,
83
		  u32	flush_domains)
84
{
85
	struct drm_device *dev = ring->dev;
86
	u32 cmd;
87
	int ret;
88
 
89
	/*
90
	 * read/write caches:
91
	 *
92
	 * I915_GEM_DOMAIN_RENDER is always invalidated, but is
93
	 * only flushed if MI_NO_WRITE_FLUSH is unset.  On 965, it is
94
	 * also flushed at 2d versus 3d pipeline switches.
95
	 *
96
	 * read-only caches:
97
	 *
98
	 * I915_GEM_DOMAIN_SAMPLER is flushed on pre-965 if
99
	 * MI_READ_FLUSH is set, and is always flushed on 965.
100
	 *
101
	 * I915_GEM_DOMAIN_COMMAND may not exist?
102
	 *
103
	 * I915_GEM_DOMAIN_INSTRUCTION, which exists on 965, is
104
	 * invalidated when MI_EXE_FLUSH is set.
105
	 *
106
	 * I915_GEM_DOMAIN_VERTEX, which exists on 965, is
107
	 * invalidated with every MI_FLUSH.
108
	 *
109
	 * TLBs:
110
	 *
111
	 * On 965, TLBs associated with I915_GEM_DOMAIN_COMMAND
112
	 * and I915_GEM_DOMAIN_CPU in are invalidated at PTE write and
113
	 * I915_GEM_DOMAIN_RENDER and I915_GEM_DOMAIN_SAMPLER
114
	 * are flushed at any MI_FLUSH.
115
	 */
116
 
117
	cmd = MI_FLUSH | MI_NO_WRITE_FLUSH;
3031 serge 118
	if ((invalidate_domains|flush_domains) & I915_GEM_DOMAIN_RENDER)
2332 Serge 119
		cmd &= ~MI_NO_WRITE_FLUSH;
120
	if (invalidate_domains & I915_GEM_DOMAIN_INSTRUCTION)
121
		cmd |= MI_EXE_FLUSH;
122
 
123
	if (invalidate_domains & I915_GEM_DOMAIN_COMMAND &&
124
	    (IS_G4X(dev) || IS_GEN5(dev)))
125
		cmd |= MI_INVALIDATE_ISP;
126
 
127
	ret = intel_ring_begin(ring, 2);
128
	if (ret)
129
		return ret;
130
 
131
	intel_ring_emit(ring, cmd);
132
	intel_ring_emit(ring, MI_NOOP);
133
	intel_ring_advance(ring);
134
 
135
	return 0;
136
}
137
 
2342 Serge 138
/**
139
 * Emits a PIPE_CONTROL with a non-zero post-sync operation, for
140
 * implementing two workarounds on gen6.  From section 1.4.7.1
141
 * "PIPE_CONTROL" of the Sandy Bridge PRM volume 2 part 1:
142
 *
143
 * [DevSNB-C+{W/A}] Before any depth stall flush (including those
144
 * produced by non-pipelined state commands), software needs to first
145
 * send a PIPE_CONTROL with no bits set except Post-Sync Operation !=
146
 * 0.
147
 *
148
 * [Dev-SNB{W/A}]: Before a PIPE_CONTROL with Write Cache Flush Enable
149
 * =1, a PIPE_CONTROL with any non-zero post-sync-op is required.
150
 *
151
 * And the workaround for these two requires this workaround first:
152
 *
153
 * [Dev-SNB{W/A}]: Pipe-control with CS-stall bit set must be sent
154
 * BEFORE the pipe-control with a post-sync op and no write-cache
155
 * flushes.
156
 *
157
 * And this last workaround is tricky because of the requirements on
158
 * that bit.  From section 1.4.7.2.3 "Stall" of the Sandy Bridge PRM
159
 * volume 2 part 1:
160
 *
161
 *     "1 of the following must also be set:
162
 *      - Render Target Cache Flush Enable ([12] of DW1)
163
 *      - Depth Cache Flush Enable ([0] of DW1)
164
 *      - Stall at Pixel Scoreboard ([1] of DW1)
165
 *      - Depth Stall ([13] of DW1)
166
 *      - Post-Sync Operation ([13] of DW1)
167
 *      - Notify Enable ([8] of DW1)"
168
 *
169
 * The cache flushes require the workaround flush that triggered this
170
 * one, so we can't use it.  Depth stall would trigger the same.
171
 * Post-sync nonzero is what triggered this second workaround, so we
172
 * can't use that one either.  Notify enable is IRQs, which aren't
173
 * really our business.  That leaves only stall at scoreboard.
174
 */
175
static int
176
intel_emit_post_sync_nonzero_flush(struct intel_ring_buffer *ring)
177
{
4104 Serge 178
	u32 scratch_addr = ring->scratch.gtt_offset + 128;
2342 Serge 179
	int ret;
180
 
181
 
182
	ret = intel_ring_begin(ring, 6);
183
	if (ret)
184
		return ret;
185
 
186
	intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(5));
187
	intel_ring_emit(ring, PIPE_CONTROL_CS_STALL |
188
			PIPE_CONTROL_STALL_AT_SCOREBOARD);
189
	intel_ring_emit(ring, scratch_addr | PIPE_CONTROL_GLOBAL_GTT); /* address */
190
	intel_ring_emit(ring, 0); /* low dword */
191
	intel_ring_emit(ring, 0); /* high dword */
192
	intel_ring_emit(ring, MI_NOOP);
193
	intel_ring_advance(ring);
194
 
195
	ret = intel_ring_begin(ring, 6);
196
	if (ret)
197
		return ret;
198
 
199
	intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(5));
200
	intel_ring_emit(ring, PIPE_CONTROL_QW_WRITE);
201
	intel_ring_emit(ring, scratch_addr | PIPE_CONTROL_GLOBAL_GTT); /* address */
202
	intel_ring_emit(ring, 0);
203
	intel_ring_emit(ring, 0);
204
	intel_ring_emit(ring, MI_NOOP);
205
	intel_ring_advance(ring);
206
 
207
	return 0;
208
}
209
 
210
static int
211
gen6_render_ring_flush(struct intel_ring_buffer *ring,
212
                         u32 invalidate_domains, u32 flush_domains)
213
{
214
	u32 flags = 0;
4104 Serge 215
	u32 scratch_addr = ring->scratch.gtt_offset + 128;
2342 Serge 216
	int ret;
217
 
218
	/* Force SNB workarounds for PIPE_CONTROL flushes */
3031 serge 219
	ret = intel_emit_post_sync_nonzero_flush(ring);
220
	if (ret)
221
		return ret;
2342 Serge 222
 
223
	/* Just flush everything.  Experiments have shown that reducing the
224
	 * number of bits based on the write domains has little performance
225
	 * impact.
226
	 */
3031 serge 227
	if (flush_domains) {
228
		flags |= PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH;
229
		flags |= PIPE_CONTROL_DEPTH_CACHE_FLUSH;
230
		/*
231
		 * Ensure that any following seqno writes only happen
232
		 * when the render cache is indeed flushed.
233
		 */
234
		flags |= PIPE_CONTROL_CS_STALL;
235
	}
236
	if (invalidate_domains) {
237
		flags |= PIPE_CONTROL_TLB_INVALIDATE;
238
		flags |= PIPE_CONTROL_INSTRUCTION_CACHE_INVALIDATE;
239
		flags |= PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE;
240
		flags |= PIPE_CONTROL_VF_CACHE_INVALIDATE;
241
		flags |= PIPE_CONTROL_CONST_CACHE_INVALIDATE;
242
		flags |= PIPE_CONTROL_STATE_CACHE_INVALIDATE;
243
		/*
244
		 * TLB invalidate requires a post-sync write.
245
		 */
3243 Serge 246
		flags |= PIPE_CONTROL_QW_WRITE | PIPE_CONTROL_CS_STALL;
3031 serge 247
	}
248
 
249
	ret = intel_ring_begin(ring, 4);
250
	if (ret)
251
		return ret;
252
 
253
	intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(4));
254
	intel_ring_emit(ring, flags);
255
	intel_ring_emit(ring, scratch_addr | PIPE_CONTROL_GLOBAL_GTT);
256
	intel_ring_emit(ring, 0);
257
	intel_ring_advance(ring);
258
 
259
	return 0;
260
}
261
 
262
static int
263
gen7_render_ring_cs_stall_wa(struct intel_ring_buffer *ring)
264
{
265
	int ret;
266
 
267
	ret = intel_ring_begin(ring, 4);
268
	if (ret)
269
		return ret;
270
 
271
	intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(4));
272
	intel_ring_emit(ring, PIPE_CONTROL_CS_STALL |
273
			      PIPE_CONTROL_STALL_AT_SCOREBOARD);
274
	intel_ring_emit(ring, 0);
275
	intel_ring_emit(ring, 0);
276
	intel_ring_advance(ring);
277
 
278
	return 0;
279
}
280
 
4104 Serge 281
static int gen7_ring_fbc_flush(struct intel_ring_buffer *ring, u32 value)
282
{
283
	int ret;
284
 
285
	if (!ring->fbc_dirty)
286
		return 0;
287
 
4560 Serge 288
	ret = intel_ring_begin(ring, 6);
4104 Serge 289
	if (ret)
290
		return ret;
291
	/* WaFbcNukeOn3DBlt:ivb/hsw */
292
	intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(1));
293
	intel_ring_emit(ring, MSG_FBC_REND_STATE);
294
	intel_ring_emit(ring, value);
4560 Serge 295
	intel_ring_emit(ring, MI_STORE_REGISTER_MEM(1) | MI_SRM_LRM_GLOBAL_GTT);
296
	intel_ring_emit(ring, MSG_FBC_REND_STATE);
297
	intel_ring_emit(ring, ring->scratch.gtt_offset + 256);
4104 Serge 298
	intel_ring_advance(ring);
299
 
300
	ring->fbc_dirty = false;
301
	return 0;
302
}
303
 
3031 serge 304
static int
305
gen7_render_ring_flush(struct intel_ring_buffer *ring,
306
		       u32 invalidate_domains, u32 flush_domains)
307
{
308
	u32 flags = 0;
4104 Serge 309
	u32 scratch_addr = ring->scratch.gtt_offset + 128;
3031 serge 310
	int ret;
311
 
312
	/*
313
	 * Ensure that any following seqno writes only happen when the render
314
	 * cache is indeed flushed.
315
	 *
316
	 * Workaround: 4th PIPE_CONTROL command (except the ones with only
317
	 * read-cache invalidate bits set) must have the CS_STALL bit set. We
318
	 * don't try to be clever and just set it unconditionally.
319
	 */
320
	flags |= PIPE_CONTROL_CS_STALL;
321
 
322
	/* Just flush everything.  Experiments have shown that reducing the
323
	 * number of bits based on the write domains has little performance
324
	 * impact.
325
	 */
326
	if (flush_domains) {
2342 Serge 327
	flags |= PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH;
3031 serge 328
		flags |= PIPE_CONTROL_DEPTH_CACHE_FLUSH;
329
	}
330
	if (invalidate_domains) {
331
		flags |= PIPE_CONTROL_TLB_INVALIDATE;
2342 Serge 332
	flags |= PIPE_CONTROL_INSTRUCTION_CACHE_INVALIDATE;
333
	flags |= PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE;
334
	flags |= PIPE_CONTROL_VF_CACHE_INVALIDATE;
335
	flags |= PIPE_CONTROL_CONST_CACHE_INVALIDATE;
336
	flags |= PIPE_CONTROL_STATE_CACHE_INVALIDATE;
3031 serge 337
		/*
338
		 * TLB invalidate requires a post-sync write.
339
		 */
340
		flags |= PIPE_CONTROL_QW_WRITE;
3480 Serge 341
		flags |= PIPE_CONTROL_GLOBAL_GTT_IVB;
2342 Serge 342
 
3031 serge 343
		/* Workaround: we must issue a pipe_control with CS-stall bit
344
		 * set before a pipe_control command that has the state cache
345
		 * invalidate bit set. */
346
		gen7_render_ring_cs_stall_wa(ring);
347
	}
348
 
349
	ret = intel_ring_begin(ring, 4);
2342 Serge 350
	if (ret)
351
		return ret;
352
 
3031 serge 353
	intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(4));
2342 Serge 354
	intel_ring_emit(ring, flags);
3480 Serge 355
	intel_ring_emit(ring, scratch_addr);
3031 serge 356
	intel_ring_emit(ring, 0);
2342 Serge 357
	intel_ring_advance(ring);
358
 
4560 Serge 359
	if (!invalidate_domains && flush_domains)
4104 Serge 360
		return gen7_ring_fbc_flush(ring, FBC_REND_NUKE);
361
 
2342 Serge 362
	return 0;
363
}
364
 
4560 Serge 365
static int
366
gen8_render_ring_flush(struct intel_ring_buffer *ring,
367
		       u32 invalidate_domains, u32 flush_domains)
368
{
369
	u32 flags = 0;
370
	u32 scratch_addr = ring->scratch.gtt_offset + 128;
371
	int ret;
372
 
373
	flags |= PIPE_CONTROL_CS_STALL;
374
 
375
	if (flush_domains) {
376
		flags |= PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH;
377
		flags |= PIPE_CONTROL_DEPTH_CACHE_FLUSH;
378
	}
379
	if (invalidate_domains) {
380
		flags |= PIPE_CONTROL_TLB_INVALIDATE;
381
		flags |= PIPE_CONTROL_INSTRUCTION_CACHE_INVALIDATE;
382
		flags |= PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE;
383
		flags |= PIPE_CONTROL_VF_CACHE_INVALIDATE;
384
		flags |= PIPE_CONTROL_CONST_CACHE_INVALIDATE;
385
		flags |= PIPE_CONTROL_STATE_CACHE_INVALIDATE;
386
		flags |= PIPE_CONTROL_QW_WRITE;
387
		flags |= PIPE_CONTROL_GLOBAL_GTT_IVB;
388
	}
389
 
390
	ret = intel_ring_begin(ring, 6);
391
	if (ret)
392
		return ret;
393
 
394
	intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(6));
395
	intel_ring_emit(ring, flags);
396
	intel_ring_emit(ring, scratch_addr);
397
	intel_ring_emit(ring, 0);
398
	intel_ring_emit(ring, 0);
399
	intel_ring_emit(ring, 0);
400
	intel_ring_advance(ring);
401
 
402
	return 0;
403
 
404
}
405
 
2332 Serge 406
static void ring_write_tail(struct intel_ring_buffer *ring,
407
			    u32 value)
408
{
409
	drm_i915_private_t *dev_priv = ring->dev->dev_private;
410
	I915_WRITE_TAIL(ring, value);
411
}
412
 
413
u32 intel_ring_get_active_head(struct intel_ring_buffer *ring)
414
{
415
	drm_i915_private_t *dev_priv = ring->dev->dev_private;
416
	u32 acthd_reg = INTEL_INFO(ring->dev)->gen >= 4 ?
417
			RING_ACTHD(ring->mmio_base) : ACTHD;
418
 
419
	return I915_READ(acthd_reg);
420
}
421
 
4104 Serge 422
static void ring_setup_phys_status_page(struct intel_ring_buffer *ring)
423
{
424
	struct drm_i915_private *dev_priv = ring->dev->dev_private;
425
	u32 addr;
426
 
427
	addr = dev_priv->status_page_dmah->busaddr;
428
	if (INTEL_INFO(ring->dev)->gen >= 4)
429
		addr |= (dev_priv->status_page_dmah->busaddr >> 28) & 0xf0;
430
	I915_WRITE(HWS_PGA, addr);
431
}
432
 
2332 Serge 433
static int init_ring_common(struct intel_ring_buffer *ring)
434
{
3031 serge 435
	struct drm_device *dev = ring->dev;
436
	drm_i915_private_t *dev_priv = dev->dev_private;
2332 Serge 437
	struct drm_i915_gem_object *obj = ring->obj;
3031 serge 438
	int ret = 0;
2332 Serge 439
	u32 head;
440
 
4560 Serge 441
	gen6_gt_force_wake_get(dev_priv, FORCEWAKE_ALL);
3031 serge 442
 
4104 Serge 443
	if (I915_NEED_GFX_HWS(dev))
444
		intel_ring_setup_status_page(ring);
445
	else
446
		ring_setup_phys_status_page(ring);
447
 
2332 Serge 448
	/* Stop the ring if it's running. */
449
	I915_WRITE_CTL(ring, 0);
450
	I915_WRITE_HEAD(ring, 0);
451
	ring->write_tail(ring, 0);
452
 
453
	head = I915_READ_HEAD(ring) & HEAD_ADDR;
454
 
455
	/* G45 ring initialization fails to reset head to zero */
456
	if (head != 0) {
457
		DRM_DEBUG_KMS("%s head not reset to zero "
458
			      "ctl %08x head %08x tail %08x start %08x\n",
459
			      ring->name,
460
			      I915_READ_CTL(ring),
461
			      I915_READ_HEAD(ring),
462
			      I915_READ_TAIL(ring),
463
			      I915_READ_START(ring));
464
 
465
		I915_WRITE_HEAD(ring, 0);
466
 
467
		if (I915_READ_HEAD(ring) & HEAD_ADDR) {
468
			DRM_ERROR("failed to set %s head to zero "
469
				  "ctl %08x head %08x tail %08x start %08x\n",
470
				  ring->name,
471
				  I915_READ_CTL(ring),
472
				  I915_READ_HEAD(ring),
473
				  I915_READ_TAIL(ring),
474
				  I915_READ_START(ring));
475
		}
476
	}
477
 
3031 serge 478
	/* Initialize the ring. This must happen _after_ we've cleared the ring
479
	 * registers with the above sequence (the readback of the HEAD registers
480
	 * also enforces ordering), otherwise the hw might lose the new ring
481
	 * register values. */
4104 Serge 482
	I915_WRITE_START(ring, i915_gem_obj_ggtt_offset(obj));
2332 Serge 483
	I915_WRITE_CTL(ring,
484
			((ring->size - PAGE_SIZE) & RING_NR_PAGES)
3031 serge 485
			| RING_VALID);
2332 Serge 486
 
487
	/* If the head is still not zero, the ring is dead */
3031 serge 488
	if (wait_for((I915_READ_CTL(ring) & RING_VALID) != 0 &&
4104 Serge 489
		     I915_READ_START(ring) == i915_gem_obj_ggtt_offset(obj) &&
3031 serge 490
		     (I915_READ_HEAD(ring) & HEAD_ADDR) == 0, 50)) {
2332 Serge 491
		DRM_ERROR("%s initialization failed "
492
				"ctl %08x head %08x tail %08x start %08x\n",
493
				ring->name,
494
				I915_READ_CTL(ring),
495
				I915_READ_HEAD(ring),
496
				I915_READ_TAIL(ring),
497
				I915_READ_START(ring));
3031 serge 498
		ret = -EIO;
499
		goto out;
2332 Serge 500
	}
501
 
3031 serge 502
		ring->head = I915_READ_HEAD(ring);
503
		ring->tail = I915_READ_TAIL(ring) & TAIL_ADDR;
504
		ring->space = ring_space(ring);
505
		ring->last_retired_head = -1;
2332 Serge 506
 
4104 Serge 507
	memset(&ring->hangcheck, 0, sizeof(ring->hangcheck));
508
 
3031 serge 509
out:
4560 Serge 510
	gen6_gt_force_wake_put(dev_priv, FORCEWAKE_ALL);
2332 Serge 511
 
3031 serge 512
	return ret;
2332 Serge 513
}
514
 
515
static int
516
init_pipe_control(struct intel_ring_buffer *ring)
517
{
518
	int ret;
519
 
4104 Serge 520
	if (ring->scratch.obj)
2332 Serge 521
		return 0;
522
 
4104 Serge 523
	ring->scratch.obj = i915_gem_alloc_object(ring->dev, 4096);
524
	if (ring->scratch.obj == NULL) {
2332 Serge 525
		DRM_ERROR("Failed to allocate seqno page\n");
526
		ret = -ENOMEM;
527
		goto err;
528
	}
529
 
4104 Serge 530
	i915_gem_object_set_cache_level(ring->scratch.obj, I915_CACHE_LLC);
2332 Serge 531
 
4104 Serge 532
	ret = i915_gem_obj_ggtt_pin(ring->scratch.obj, 4096, true, false);
2332 Serge 533
	if (ret)
534
		goto err_unref;
535
 
4104 Serge 536
	ring->scratch.gtt_offset = i915_gem_obj_ggtt_offset(ring->scratch.obj);
4539 Serge 537
    ring->scratch.cpu_page = (void*)MapIoMem((addr_t)sg_page(ring->scratch.obj->pages->sgl),4096, PG_SW|0x100);
4104 Serge 538
	if (ring->scratch.cpu_page == NULL) {
539
		ret = -ENOMEM;
2332 Serge 540
		goto err_unpin;
4104 Serge 541
	}
2332 Serge 542
 
3480 Serge 543
	DRM_DEBUG_DRIVER("%s pipe control offset: 0x%08x\n",
4104 Serge 544
			 ring->name, ring->scratch.gtt_offset);
2332 Serge 545
	return 0;
546
 
547
err_unpin:
4104 Serge 548
	i915_gem_object_unpin(ring->scratch.obj);
2332 Serge 549
err_unref:
4104 Serge 550
	drm_gem_object_unreference(&ring->scratch.obj->base);
2332 Serge 551
err:
552
	return ret;
553
}
554
 
555
static int init_render_ring(struct intel_ring_buffer *ring)
556
{
557
	struct drm_device *dev = ring->dev;
558
	struct drm_i915_private *dev_priv = dev->dev_private;
559
	int ret = init_ring_common(ring);
560
 
3243 Serge 561
	if (INTEL_INFO(dev)->gen > 3)
3031 serge 562
		I915_WRITE(MI_MODE, _MASKED_BIT_ENABLE(VS_TIMER_DISPATCH));
3243 Serge 563
 
564
	/* We need to disable the AsyncFlip performance optimisations in order
565
	 * to use MI_WAIT_FOR_EVENT within the CS. It should already be
566
	 * programmed to '1' on all products.
4104 Serge 567
	 *
568
	 * WaDisableAsyncFlipPerfMode:snb,ivb,hsw,vlv
3243 Serge 569
	 */
570
	if (INTEL_INFO(dev)->gen >= 6)
571
		I915_WRITE(MI_MODE, _MASKED_BIT_ENABLE(ASYNC_FLIP_PERF_DISABLE));
572
 
573
	/* Required for the hardware to program scanline values for waiting */
574
	if (INTEL_INFO(dev)->gen == 6)
575
		I915_WRITE(GFX_MODE,
576
			   _MASKED_BIT_ENABLE(GFX_TLB_INVALIDATE_ALWAYS));
577
 
2332 Serge 578
		if (IS_GEN7(dev))
579
			I915_WRITE(GFX_MODE_GEN7,
3031 serge 580
				   _MASKED_BIT_DISABLE(GFX_TLB_INVALIDATE_ALWAYS) |
581
				   _MASKED_BIT_ENABLE(GFX_REPLAY_MODE));
2332 Serge 582
 
2342 Serge 583
	if (INTEL_INFO(dev)->gen >= 5) {
2339 Serge 584
		ret = init_pipe_control(ring);
2332 Serge 585
		if (ret)
586
			return ret;
587
	}
588
 
3031 serge 589
	if (IS_GEN6(dev)) {
590
		/* From the Sandybridge PRM, volume 1 part 3, page 24:
591
		 * "If this bit is set, STCunit will have LRA as replacement
592
		 *  policy. [...] This bit must be reset.  LRA replacement
593
		 *  policy is not supported."
594
		 */
595
		I915_WRITE(CACHE_MODE_0,
596
			   _MASKED_BIT_DISABLE(CM0_STC_EVICT_DISABLE_LRA_SNB));
597
 
598
		/* This is not explicitly set for GEN6, so read the register.
599
		 * see intel_ring_mi_set_context() for why we care.
600
		 * TODO: consider explicitly setting the bit for GEN5
601
		 */
602
		ring->itlb_before_ctx_switch =
603
			!!(I915_READ(GFX_MODE) & GFX_TLB_INVALIDATE_ALWAYS);
2342 Serge 604
	}
605
 
3031 serge 606
	if (INTEL_INFO(dev)->gen >= 6)
607
		I915_WRITE(INSTPM, _MASKED_BIT_ENABLE(INSTPM_FORCE_ORDERING));
608
 
4560 Serge 609
	if (HAS_L3_DPF(dev))
610
		I915_WRITE_IMR(ring, ~GT_PARITY_ERROR(dev));
3031 serge 611
 
2332 Serge 612
	return ret;
613
}
614
 
615
static void render_ring_cleanup(struct intel_ring_buffer *ring)
616
{
3480 Serge 617
	struct drm_device *dev = ring->dev;
618
 
4104 Serge 619
	if (ring->scratch.obj == NULL)
2332 Serge 620
		return;
621
 
4104 Serge 622
	if (INTEL_INFO(dev)->gen >= 5) {
623
//       kunmap(sg_page(ring->scratch.obj->pages->sgl));
624
		i915_gem_object_unpin(ring->scratch.obj);
625
	}
626
 
627
	drm_gem_object_unreference(&ring->scratch.obj->base);
628
	ring->scratch.obj = NULL;
2332 Serge 629
}
630
 
631
static void
2342 Serge 632
update_mboxes(struct intel_ring_buffer *ring,
633
	    u32 mmio_offset)
2332 Serge 634
{
4104 Serge 635
/* NB: In order to be able to do semaphore MBOX updates for varying number
636
 * of rings, it's easiest if we round up each individual update to a
637
 * multiple of 2 (since ring updates must always be a multiple of 2)
638
 * even though the actual update only requires 3 dwords.
639
 */
640
#define MBOX_UPDATE_DWORDS 4
3243 Serge 641
	intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(1));
2342 Serge 642
	intel_ring_emit(ring, mmio_offset);
4560 Serge 643
	intel_ring_emit(ring, ring->outstanding_lazy_seqno);
4104 Serge 644
	intel_ring_emit(ring, MI_NOOP);
2332 Serge 645
}
646
 
2342 Serge 647
/**
648
 * gen6_add_request - Update the semaphore mailbox registers
649
 *
650
 * @ring - ring that is adding a request
651
 * @seqno - return seqno stuck into the ring
652
 *
653
 * Update the mailbox registers in the *other* rings with the current seqno.
654
 * This acts like a signal in the canonical semaphore.
655
 */
2332 Serge 656
static int
3243 Serge 657
gen6_add_request(struct intel_ring_buffer *ring)
2332 Serge 658
{
4104 Serge 659
	struct drm_device *dev = ring->dev;
660
	struct drm_i915_private *dev_priv = dev->dev_private;
661
	struct intel_ring_buffer *useless;
4560 Serge 662
	int i, ret, num_dwords = 4;
2332 Serge 663
 
4560 Serge 664
	if (i915_semaphore_is_enabled(dev))
665
		num_dwords += ((I915_NUM_RINGS-1) * MBOX_UPDATE_DWORDS);
666
#undef MBOX_UPDATE_DWORDS
667
 
668
	ret = intel_ring_begin(ring, num_dwords);
2332 Serge 669
	if (ret)
670
		return ret;
671
 
4560 Serge 672
	if (i915_semaphore_is_enabled(dev)) {
4104 Serge 673
	for_each_ring(useless, dev_priv, i) {
674
		u32 mbox_reg = ring->signal_mbox[i];
675
		if (mbox_reg != GEN6_NOSYNC)
676
			update_mboxes(ring, mbox_reg);
677
	}
4560 Serge 678
	}
2332 Serge 679
 
680
	intel_ring_emit(ring, MI_STORE_DWORD_INDEX);
681
	intel_ring_emit(ring, I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
4560 Serge 682
	intel_ring_emit(ring, ring->outstanding_lazy_seqno);
2332 Serge 683
	intel_ring_emit(ring, MI_USER_INTERRUPT);
4560 Serge 684
	__intel_ring_advance(ring);
2332 Serge 685
 
686
	return 0;
687
}
688
 
3480 Serge 689
static inline bool i915_gem_has_seqno_wrapped(struct drm_device *dev,
690
					      u32 seqno)
691
{
692
	struct drm_i915_private *dev_priv = dev->dev_private;
693
	return dev_priv->last_seqno < seqno;
694
}
695
 
2342 Serge 696
/**
697
 * intel_ring_sync - sync the waiter to the signaller on seqno
698
 *
699
 * @waiter - ring that is waiting
700
 * @signaller - ring which has, or will signal
701
 * @seqno - seqno which the waiter will block on
702
 */
703
static int
3031 serge 704
gen6_ring_sync(struct intel_ring_buffer *waiter,
2342 Serge 705
		struct intel_ring_buffer *signaller,
2332 Serge 706
		u32 seqno)
707
{
708
	int ret;
2342 Serge 709
	u32 dw1 = MI_SEMAPHORE_MBOX |
710
		  MI_SEMAPHORE_COMPARE |
711
		  MI_SEMAPHORE_REGISTER;
2332 Serge 712
 
3031 serge 713
	/* Throughout all of the GEM code, seqno passed implies our current
714
	 * seqno is >= the last seqno executed. However for hardware the
715
	 * comparison is strictly greater than.
716
	 */
717
	seqno -= 1;
718
 
719
	WARN_ON(signaller->semaphore_register[waiter->id] ==
720
		MI_SEMAPHORE_SYNC_INVALID);
721
 
2342 Serge 722
	ret = intel_ring_begin(waiter, 4);
2332 Serge 723
	if (ret)
724
		return ret;
725
 
3480 Serge 726
	/* If seqno wrap happened, omit the wait with no-ops */
727
	if (likely(!i915_gem_has_seqno_wrapped(waiter->dev, seqno))) {
3031 serge 728
	intel_ring_emit(waiter,
3480 Serge 729
				dw1 |
730
				signaller->semaphore_register[waiter->id]);
2342 Serge 731
	intel_ring_emit(waiter, seqno);
732
	intel_ring_emit(waiter, 0);
733
	intel_ring_emit(waiter, MI_NOOP);
3480 Serge 734
	} else {
735
		intel_ring_emit(waiter, MI_NOOP);
736
		intel_ring_emit(waiter, MI_NOOP);
737
		intel_ring_emit(waiter, MI_NOOP);
738
		intel_ring_emit(waiter, MI_NOOP);
739
	}
2342 Serge 740
	intel_ring_advance(waiter);
2332 Serge 741
 
742
	return 0;
743
}
744
 
745
#define PIPE_CONTROL_FLUSH(ring__, addr__)					\
746
do {									\
2342 Serge 747
	intel_ring_emit(ring__, GFX_OP_PIPE_CONTROL(4) | PIPE_CONTROL_QW_WRITE |		\
748
		 PIPE_CONTROL_DEPTH_STALL);				\
2332 Serge 749
	intel_ring_emit(ring__, (addr__) | PIPE_CONTROL_GLOBAL_GTT);			\
750
	intel_ring_emit(ring__, 0);							\
751
	intel_ring_emit(ring__, 0);							\
752
} while (0)
753
 
754
static int
3243 Serge 755
pc_render_add_request(struct intel_ring_buffer *ring)
2332 Serge 756
{
4104 Serge 757
	u32 scratch_addr = ring->scratch.gtt_offset + 128;
2332 Serge 758
	int ret;
759
 
760
	/* For Ironlake, MI_USER_INTERRUPT was deprecated and apparently
761
	 * incoherent with writes to memory, i.e. completely fubar,
762
	 * so we need to use PIPE_NOTIFY instead.
763
	 *
764
	 * However, we also need to workaround the qword write
765
	 * incoherence by flushing the 6 PIPE_NOTIFY buffers out to
766
	 * memory before requesting an interrupt.
767
	 */
768
	ret = intel_ring_begin(ring, 32);
769
	if (ret)
770
		return ret;
771
 
2342 Serge 772
	intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(4) | PIPE_CONTROL_QW_WRITE |
773
			PIPE_CONTROL_WRITE_FLUSH |
774
			PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE);
4104 Serge 775
	intel_ring_emit(ring, ring->scratch.gtt_offset | PIPE_CONTROL_GLOBAL_GTT);
4560 Serge 776
	intel_ring_emit(ring, ring->outstanding_lazy_seqno);
2332 Serge 777
	intel_ring_emit(ring, 0);
778
	PIPE_CONTROL_FLUSH(ring, scratch_addr);
779
	scratch_addr += 128; /* write to separate cachelines */
780
	PIPE_CONTROL_FLUSH(ring, scratch_addr);
781
	scratch_addr += 128;
782
	PIPE_CONTROL_FLUSH(ring, scratch_addr);
783
	scratch_addr += 128;
784
	PIPE_CONTROL_FLUSH(ring, scratch_addr);
785
	scratch_addr += 128;
786
	PIPE_CONTROL_FLUSH(ring, scratch_addr);
787
	scratch_addr += 128;
788
	PIPE_CONTROL_FLUSH(ring, scratch_addr);
3031 serge 789
 
2342 Serge 790
	intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(4) | PIPE_CONTROL_QW_WRITE |
791
			PIPE_CONTROL_WRITE_FLUSH |
792
			PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE |
2332 Serge 793
			PIPE_CONTROL_NOTIFY);
4104 Serge 794
	intel_ring_emit(ring, ring->scratch.gtt_offset | PIPE_CONTROL_GLOBAL_GTT);
4560 Serge 795
	intel_ring_emit(ring, ring->outstanding_lazy_seqno);
2332 Serge 796
	intel_ring_emit(ring, 0);
4560 Serge 797
	__intel_ring_advance(ring);
2332 Serge 798
 
799
	return 0;
800
}
801
 
802
static u32
3031 serge 803
gen6_ring_get_seqno(struct intel_ring_buffer *ring, bool lazy_coherency)
2342 Serge 804
{
805
	/* Workaround to force correct ordering between irq and seqno writes on
806
	 * ivb (and maybe also on snb) by reading from a CS register (like
807
	 * ACTHD) before reading the status page. */
3031 serge 808
	if (!lazy_coherency)
2342 Serge 809
		intel_ring_get_active_head(ring);
810
	return intel_read_status_page(ring, I915_GEM_HWS_INDEX);
811
}
812
 
813
static u32
3031 serge 814
ring_get_seqno(struct intel_ring_buffer *ring, bool lazy_coherency)
2332 Serge 815
{
816
	return intel_read_status_page(ring, I915_GEM_HWS_INDEX);
817
}
818
 
3480 Serge 819
static void
820
ring_set_seqno(struct intel_ring_buffer *ring, u32 seqno)
821
{
822
	intel_write_status_page(ring, I915_GEM_HWS_INDEX, seqno);
823
}
824
 
2332 Serge 825
static u32
3031 serge 826
pc_render_get_seqno(struct intel_ring_buffer *ring, bool lazy_coherency)
2332 Serge 827
{
4104 Serge 828
	return ring->scratch.cpu_page[0];
2332 Serge 829
}
830
 
3480 Serge 831
static void
832
pc_render_set_seqno(struct intel_ring_buffer *ring, u32 seqno)
833
{
4104 Serge 834
	ring->scratch.cpu_page[0] = seqno;
3480 Serge 835
}
836
 
3031 serge 837
static bool
838
gen5_ring_get_irq(struct intel_ring_buffer *ring)
2332 Serge 839
{
3031 serge 840
	struct drm_device *dev = ring->dev;
841
	drm_i915_private_t *dev_priv = dev->dev_private;
842
	unsigned long flags;
843
 
844
	if (!dev->irq_enabled)
845
		return false;
846
 
847
	spin_lock_irqsave(&dev_priv->irq_lock, flags);
4104 Serge 848
	if (ring->irq_refcount++ == 0)
849
		ilk_enable_gt_irq(dev_priv, ring->irq_enable_mask);
3031 serge 850
	spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
851
 
852
	return true;
2332 Serge 853
}
854
 
855
static void
3031 serge 856
gen5_ring_put_irq(struct intel_ring_buffer *ring)
2332 Serge 857
{
3031 serge 858
	struct drm_device *dev = ring->dev;
859
	drm_i915_private_t *dev_priv = dev->dev_private;
860
	unsigned long flags;
861
 
862
	spin_lock_irqsave(&dev_priv->irq_lock, flags);
4104 Serge 863
	if (--ring->irq_refcount == 0)
864
		ilk_disable_gt_irq(dev_priv, ring->irq_enable_mask);
3031 serge 865
	spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
2332 Serge 866
}
867
 
3031 serge 868
static bool
869
i9xx_ring_get_irq(struct intel_ring_buffer *ring)
2332 Serge 870
{
3031 serge 871
	struct drm_device *dev = ring->dev;
872
	drm_i915_private_t *dev_priv = dev->dev_private;
873
	unsigned long flags;
874
 
875
	if (!dev->irq_enabled)
876
		return false;
877
 
878
	spin_lock_irqsave(&dev_priv->irq_lock, flags);
879
	if (ring->irq_refcount++ == 0) {
880
		dev_priv->irq_mask &= ~ring->irq_enable_mask;
2332 Serge 881
	I915_WRITE(IMR, dev_priv->irq_mask);
882
	POSTING_READ(IMR);
3031 serge 883
	}
884
	spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
885
 
886
	return true;
2332 Serge 887
}
888
 
889
static void
3031 serge 890
i9xx_ring_put_irq(struct intel_ring_buffer *ring)
2332 Serge 891
{
3031 serge 892
	struct drm_device *dev = ring->dev;
893
	drm_i915_private_t *dev_priv = dev->dev_private;
894
	unsigned long flags;
895
 
896
	spin_lock_irqsave(&dev_priv->irq_lock, flags);
897
	if (--ring->irq_refcount == 0) {
898
		dev_priv->irq_mask |= ring->irq_enable_mask;
2332 Serge 899
	I915_WRITE(IMR, dev_priv->irq_mask);
900
	POSTING_READ(IMR);
3031 serge 901
	}
902
	spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
2332 Serge 903
}
904
 
905
static bool
3031 serge 906
i8xx_ring_get_irq(struct intel_ring_buffer *ring)
2332 Serge 907
{
908
	struct drm_device *dev = ring->dev;
909
	drm_i915_private_t *dev_priv = dev->dev_private;
3031 serge 910
	unsigned long flags;
2332 Serge 911
 
912
	if (!dev->irq_enabled)
913
		return false;
914
 
3031 serge 915
	spin_lock_irqsave(&dev_priv->irq_lock, flags);
2332 Serge 916
	if (ring->irq_refcount++ == 0) {
3031 serge 917
		dev_priv->irq_mask &= ~ring->irq_enable_mask;
918
		I915_WRITE16(IMR, dev_priv->irq_mask);
919
		POSTING_READ16(IMR);
2332 Serge 920
	}
3031 serge 921
	spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
2332 Serge 922
 
923
	return true;
924
}
925
 
926
static void
3031 serge 927
i8xx_ring_put_irq(struct intel_ring_buffer *ring)
2332 Serge 928
{
929
	struct drm_device *dev = ring->dev;
930
	drm_i915_private_t *dev_priv = dev->dev_private;
3031 serge 931
	unsigned long flags;
2332 Serge 932
 
3031 serge 933
	spin_lock_irqsave(&dev_priv->irq_lock, flags);
2332 Serge 934
	if (--ring->irq_refcount == 0) {
3031 serge 935
		dev_priv->irq_mask |= ring->irq_enable_mask;
936
		I915_WRITE16(IMR, dev_priv->irq_mask);
937
		POSTING_READ16(IMR);
2332 Serge 938
	}
3031 serge 939
	spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
2332 Serge 940
}
941
 
942
void intel_ring_setup_status_page(struct intel_ring_buffer *ring)
943
{
944
	struct drm_device *dev = ring->dev;
945
	drm_i915_private_t *dev_priv = ring->dev->dev_private;
946
	u32 mmio = 0;
947
 
948
	/* The ring status page addresses are no longer next to the rest of
949
	 * the ring registers as of gen7.
950
	 */
951
	if (IS_GEN7(dev)) {
952
		switch (ring->id) {
3031 serge 953
		case RCS:
2332 Serge 954
			mmio = RENDER_HWS_PGA_GEN7;
955
			break;
3031 serge 956
		case BCS:
2332 Serge 957
			mmio = BLT_HWS_PGA_GEN7;
958
			break;
3031 serge 959
		case VCS:
2332 Serge 960
			mmio = BSD_HWS_PGA_GEN7;
961
			break;
4104 Serge 962
		case VECS:
963
			mmio = VEBOX_HWS_PGA_GEN7;
964
			break;
2332 Serge 965
		}
966
	} else if (IS_GEN6(ring->dev)) {
967
		mmio = RING_HWS_PGA_GEN6(ring->mmio_base);
968
	} else {
4560 Serge 969
		/* XXX: gen8 returns to sanity */
2332 Serge 970
		mmio = RING_HWS_PGA(ring->mmio_base);
971
	}
972
 
973
	I915_WRITE(mmio, (u32)ring->status_page.gfx_addr);
974
	POSTING_READ(mmio);
3746 Serge 975
 
4104 Serge 976
	/* Flush the TLB for this page */
977
	if (INTEL_INFO(dev)->gen >= 6) {
978
		u32 reg = RING_INSTPM(ring->mmio_base);
979
		I915_WRITE(reg,
980
			   _MASKED_BIT_ENABLE(INSTPM_TLB_INVALIDATE |
981
					      INSTPM_SYNC_FLUSH));
982
		if (wait_for((I915_READ(reg) & INSTPM_SYNC_FLUSH) == 0,
983
			     1000))
984
			DRM_ERROR("%s: wait for SyncFlush to complete for TLB invalidation timed out\n",
985
				  ring->name);
986
	}
2332 Serge 987
}
988
 
989
static int
990
bsd_ring_flush(struct intel_ring_buffer *ring,
991
	       u32     invalidate_domains,
992
	       u32     flush_domains)
993
{
994
	int ret;
995
 
996
	ret = intel_ring_begin(ring, 2);
997
	if (ret)
998
		return ret;
999
 
1000
	intel_ring_emit(ring, MI_FLUSH);
1001
	intel_ring_emit(ring, MI_NOOP);
1002
	intel_ring_advance(ring);
1003
	return 0;
1004
}
1005
 
1006
static int
3243 Serge 1007
i9xx_add_request(struct intel_ring_buffer *ring)
2332 Serge 1008
{
1009
	int ret;
1010
 
1011
	ret = intel_ring_begin(ring, 4);
1012
	if (ret)
1013
		return ret;
1014
 
1015
	intel_ring_emit(ring, MI_STORE_DWORD_INDEX);
1016
	intel_ring_emit(ring, I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
4560 Serge 1017
	intel_ring_emit(ring, ring->outstanding_lazy_seqno);
2332 Serge 1018
	intel_ring_emit(ring, MI_USER_INTERRUPT);
4560 Serge 1019
	__intel_ring_advance(ring);
2332 Serge 1020
 
1021
	return 0;
1022
}
1023
 
1024
static bool
3031 serge 1025
gen6_ring_get_irq(struct intel_ring_buffer *ring)
2332 Serge 1026
{
1027
	struct drm_device *dev = ring->dev;
1028
	drm_i915_private_t *dev_priv = dev->dev_private;
3031 serge 1029
	unsigned long flags;
2332 Serge 1030
 
1031
	if (!dev->irq_enabled)
1032
	       return false;
1033
 
3031 serge 1034
	spin_lock_irqsave(&dev_priv->irq_lock, flags);
2332 Serge 1035
	if (ring->irq_refcount++ == 0) {
4560 Serge 1036
		if (HAS_L3_DPF(dev) && ring->id == RCS)
4104 Serge 1037
			I915_WRITE_IMR(ring,
1038
				       ~(ring->irq_enable_mask |
4560 Serge 1039
					 GT_PARITY_ERROR(dev)));
3031 serge 1040
		else
1041
			I915_WRITE_IMR(ring, ~ring->irq_enable_mask);
4104 Serge 1042
		ilk_enable_gt_irq(dev_priv, ring->irq_enable_mask);
2332 Serge 1043
	}
3031 serge 1044
	spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
2332 Serge 1045
 
2351 Serge 1046
    return true;
2332 Serge 1047
}
1048
 
1049
static void
3031 serge 1050
gen6_ring_put_irq(struct intel_ring_buffer *ring)
2332 Serge 1051
{
1052
	struct drm_device *dev = ring->dev;
1053
	drm_i915_private_t *dev_priv = dev->dev_private;
3031 serge 1054
	unsigned long flags;
2332 Serge 1055
 
3031 serge 1056
	spin_lock_irqsave(&dev_priv->irq_lock, flags);
2332 Serge 1057
	if (--ring->irq_refcount == 0) {
4560 Serge 1058
		if (HAS_L3_DPF(dev) && ring->id == RCS)
1059
			I915_WRITE_IMR(ring, ~GT_PARITY_ERROR(dev));
3031 serge 1060
		else
1061
			I915_WRITE_IMR(ring, ~0);
4104 Serge 1062
		ilk_disable_gt_irq(dev_priv, ring->irq_enable_mask);
2332 Serge 1063
	}
3031 serge 1064
	spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
2332 Serge 1065
}
1066
 
4104 Serge 1067
static bool
1068
hsw_vebox_get_irq(struct intel_ring_buffer *ring)
1069
{
1070
	struct drm_device *dev = ring->dev;
1071
	struct drm_i915_private *dev_priv = dev->dev_private;
1072
	unsigned long flags;
1073
 
1074
	if (!dev->irq_enabled)
1075
		return false;
1076
 
1077
	spin_lock_irqsave(&dev_priv->irq_lock, flags);
1078
	if (ring->irq_refcount++ == 0) {
1079
		I915_WRITE_IMR(ring, ~ring->irq_enable_mask);
1080
		snb_enable_pm_irq(dev_priv, ring->irq_enable_mask);
1081
	}
1082
	spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
1083
 
1084
	return true;
1085
}
1086
 
1087
static void
1088
hsw_vebox_put_irq(struct intel_ring_buffer *ring)
1089
{
1090
	struct drm_device *dev = ring->dev;
1091
	struct drm_i915_private *dev_priv = dev->dev_private;
1092
	unsigned long flags;
1093
 
1094
	if (!dev->irq_enabled)
1095
		return;
1096
 
1097
	spin_lock_irqsave(&dev_priv->irq_lock, flags);
1098
	if (--ring->irq_refcount == 0) {
1099
		I915_WRITE_IMR(ring, ~0);
1100
		snb_disable_pm_irq(dev_priv, ring->irq_enable_mask);
1101
	}
1102
	spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
1103
}
1104
 
4560 Serge 1105
static bool
1106
gen8_ring_get_irq(struct intel_ring_buffer *ring)
1107
{
1108
	struct drm_device *dev = ring->dev;
1109
	struct drm_i915_private *dev_priv = dev->dev_private;
1110
	unsigned long flags;
1111
 
1112
	if (!dev->irq_enabled)
1113
		return false;
1114
 
1115
	spin_lock_irqsave(&dev_priv->irq_lock, flags);
1116
	if (ring->irq_refcount++ == 0) {
1117
		if (HAS_L3_DPF(dev) && ring->id == RCS) {
1118
			I915_WRITE_IMR(ring,
1119
				       ~(ring->irq_enable_mask |
1120
					 GT_RENDER_L3_PARITY_ERROR_INTERRUPT));
1121
		} else {
1122
			I915_WRITE_IMR(ring, ~ring->irq_enable_mask);
1123
		}
1124
		POSTING_READ(RING_IMR(ring->mmio_base));
1125
	}
1126
	spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
1127
 
1128
	return true;
1129
}
1130
 
1131
static void
1132
gen8_ring_put_irq(struct intel_ring_buffer *ring)
1133
{
1134
	struct drm_device *dev = ring->dev;
1135
	struct drm_i915_private *dev_priv = dev->dev_private;
1136
	unsigned long flags;
1137
 
1138
	spin_lock_irqsave(&dev_priv->irq_lock, flags);
1139
	if (--ring->irq_refcount == 0) {
1140
		if (HAS_L3_DPF(dev) && ring->id == RCS) {
1141
			I915_WRITE_IMR(ring,
1142
				       ~GT_RENDER_L3_PARITY_ERROR_INTERRUPT);
1143
		} else {
1144
			I915_WRITE_IMR(ring, ~0);
1145
		}
1146
		POSTING_READ(RING_IMR(ring->mmio_base));
1147
	}
1148
	spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
1149
}
1150
 
2332 Serge 1151
static int
3243 Serge 1152
i965_dispatch_execbuffer(struct intel_ring_buffer *ring,
1153
			 u32 offset, u32 length,
1154
			 unsigned flags)
2332 Serge 1155
{
1156
	int ret;
1157
 
1158
	ret = intel_ring_begin(ring, 2);
1159
	if (ret)
1160
		return ret;
1161
 
1162
	intel_ring_emit(ring,
3031 serge 1163
			MI_BATCH_BUFFER_START |
1164
			MI_BATCH_GTT |
3243 Serge 1165
			(flags & I915_DISPATCH_SECURE ? 0 : MI_BATCH_NON_SECURE_I965));
2332 Serge 1166
	intel_ring_emit(ring, offset);
1167
	intel_ring_advance(ring);
1168
 
1169
	return 0;
1170
}
1171
 
3243 Serge 1172
/* Just userspace ABI convention to limit the wa batch bo to a resonable size */
1173
#define I830_BATCH_LIMIT (256*1024)
2332 Serge 1174
static int
3031 serge 1175
i830_dispatch_execbuffer(struct intel_ring_buffer *ring,
3243 Serge 1176
				u32 offset, u32 len,
1177
				unsigned flags)
2332 Serge 1178
{
1179
	int ret;
1180
 
3243 Serge 1181
	if (flags & I915_DISPATCH_PINNED) {
2332 Serge 1182
		ret = intel_ring_begin(ring, 4);
1183
		if (ret)
1184
			return ret;
1185
 
1186
		intel_ring_emit(ring, MI_BATCH_BUFFER);
3243 Serge 1187
		intel_ring_emit(ring, offset | (flags & I915_DISPATCH_SECURE ? 0 : MI_BATCH_NON_SECURE));
2332 Serge 1188
		intel_ring_emit(ring, offset + len - 8);
3243 Serge 1189
		intel_ring_emit(ring, MI_NOOP);
1190
		intel_ring_advance(ring);
1191
	} else {
4104 Serge 1192
		u32 cs_offset = ring->scratch.gtt_offset;
3243 Serge 1193
 
1194
		if (len > I830_BATCH_LIMIT)
1195
			return -ENOSPC;
1196
 
1197
		ret = intel_ring_begin(ring, 9+3);
1198
		if (ret)
1199
			return ret;
1200
		/* Blit the batch (which has now all relocs applied) to the stable batch
1201
		 * scratch bo area (so that the CS never stumbles over its tlb
1202
		 * invalidation bug) ... */
1203
		intel_ring_emit(ring, XY_SRC_COPY_BLT_CMD |
1204
				XY_SRC_COPY_BLT_WRITE_ALPHA |
1205
				XY_SRC_COPY_BLT_WRITE_RGB);
1206
		intel_ring_emit(ring, BLT_DEPTH_32 | BLT_ROP_GXCOPY | 4096);
2332 Serge 1207
		intel_ring_emit(ring, 0);
3243 Serge 1208
		intel_ring_emit(ring, (DIV_ROUND_UP(len, 4096) << 16) | 1024);
1209
		intel_ring_emit(ring, cs_offset);
1210
		intel_ring_emit(ring, 0);
1211
		intel_ring_emit(ring, 4096);
1212
		intel_ring_emit(ring, offset);
1213
		intel_ring_emit(ring, MI_FLUSH);
1214
 
1215
		/* ... and execute it. */
1216
		intel_ring_emit(ring, MI_BATCH_BUFFER);
1217
		intel_ring_emit(ring, cs_offset | (flags & I915_DISPATCH_SECURE ? 0 : MI_BATCH_NON_SECURE));
1218
		intel_ring_emit(ring, cs_offset + len - 8);
3031 serge 1219
	intel_ring_advance(ring);
3243 Serge 1220
	}
3031 serge 1221
 
1222
	return 0;
1223
}
1224
 
1225
static int
1226
i915_dispatch_execbuffer(struct intel_ring_buffer *ring,
3243 Serge 1227
			 u32 offset, u32 len,
1228
			 unsigned flags)
3031 serge 1229
{
1230
	int ret;
1231
 
2332 Serge 1232
		ret = intel_ring_begin(ring, 2);
1233
		if (ret)
1234
			return ret;
1235
 
3031 serge 1236
	intel_ring_emit(ring, MI_BATCH_BUFFER_START | MI_BATCH_GTT);
3243 Serge 1237
	intel_ring_emit(ring, offset | (flags & I915_DISPATCH_SECURE ? 0 : MI_BATCH_NON_SECURE));
2332 Serge 1238
	intel_ring_advance(ring);
1239
 
1240
	return 0;
1241
}
1242
 
1243
static void cleanup_status_page(struct intel_ring_buffer *ring)
1244
{
1245
	struct drm_i915_gem_object *obj;
1246
 
1247
	obj = ring->status_page.obj;
1248
	if (obj == NULL)
1249
		return;
1250
 
3031 serge 1251
//   kunmap(sg_page(obj->pages->sgl));
2344 Serge 1252
    i915_gem_object_unpin(obj);
1253
	drm_gem_object_unreference(&obj->base);
2332 Serge 1254
	ring->status_page.obj = NULL;
1255
}
1256
 
1257
static int init_status_page(struct intel_ring_buffer *ring)
1258
{
1259
	struct drm_device *dev = ring->dev;
1260
	struct drm_i915_gem_object *obj;
1261
	int ret;
1262
 
1263
	obj = i915_gem_alloc_object(dev, 4096);
1264
	if (obj == NULL) {
1265
		DRM_ERROR("Failed to allocate status page\n");
1266
		ret = -ENOMEM;
1267
		goto err;
1268
	}
1269
 
2352 Serge 1270
	i915_gem_object_set_cache_level(obj, I915_CACHE_LLC);
2332 Serge 1271
 
4104 Serge 1272
	ret = i915_gem_obj_ggtt_pin(obj, 4096, true, false);
2332 Serge 1273
	if (ret != 0) {
1274
		goto err_unref;
1275
	}
1276
 
4104 Serge 1277
	ring->status_page.gfx_addr = i915_gem_obj_ggtt_offset(obj);
4539 Serge 1278
    ring->status_page.page_addr = (void*)MapIoMem((addr_t)sg_page(obj->pages->sgl),4096,PG_SW|0x100);
2332 Serge 1279
	if (ring->status_page.page_addr == NULL) {
3031 serge 1280
		ret = -ENOMEM;
2332 Serge 1281
		goto err_unpin;
1282
	}
1283
	ring->status_page.obj = obj;
1284
	memset(ring->status_page.page_addr, 0, PAGE_SIZE);
1285
 
1286
	DRM_DEBUG_DRIVER("%s hws offset: 0x%08x\n",
1287
			ring->name, ring->status_page.gfx_addr);
1288
 
1289
	return 0;
1290
 
1291
err_unpin:
2344 Serge 1292
	i915_gem_object_unpin(obj);
2332 Serge 1293
err_unref:
2344 Serge 1294
	drm_gem_object_unreference(&obj->base);
2332 Serge 1295
err:
1296
	return ret;
1297
}
1298
 
4104 Serge 1299
static int init_phys_status_page(struct intel_ring_buffer *ring)
3243 Serge 1300
{
1301
    struct drm_i915_private *dev_priv = ring->dev->dev_private;
1302
 
1303
    if (!dev_priv->status_page_dmah) {
1304
        dev_priv->status_page_dmah =
1305
            drm_pci_alloc(ring->dev, PAGE_SIZE, PAGE_SIZE);
1306
        if (!dev_priv->status_page_dmah)
1307
            return -ENOMEM;
1308
    }
1309
 
1310
    ring->status_page.page_addr = dev_priv->status_page_dmah->vaddr;
1311
    memset(ring->status_page.page_addr, 0, PAGE_SIZE);
1312
 
1313
    return 0;
1314
}
1315
 
3031 serge 1316
static int intel_init_ring_buffer(struct drm_device *dev,
2332 Serge 1317
			   struct intel_ring_buffer *ring)
1318
{
2340 Serge 1319
	struct drm_i915_gem_object *obj;
3031 serge 1320
	struct drm_i915_private *dev_priv = dev->dev_private;
2332 Serge 1321
	int ret;
2340 Serge 1322
 
2332 Serge 1323
	ring->dev = dev;
1324
	INIT_LIST_HEAD(&ring->active_list);
1325
	INIT_LIST_HEAD(&ring->request_list);
3031 serge 1326
	ring->size = 32 * PAGE_SIZE;
3243 Serge 1327
	memset(ring->sync_seqno, 0, sizeof(ring->sync_seqno));
2332 Serge 1328
 
2352 Serge 1329
	init_waitqueue_head(&ring->irq_queue);
2332 Serge 1330
 
1331
	if (I915_NEED_GFX_HWS(dev)) {
2340 Serge 1332
       ret = init_status_page(ring);
1333
       if (ret)
1334
           return ret;
3243 Serge 1335
	} else {
1336
		BUG_ON(ring->id != RCS);
4104 Serge 1337
		ret = init_phys_status_page(ring);
3243 Serge 1338
		if (ret)
1339
			return ret;
2332 Serge 1340
	}
1341
 
3480 Serge 1342
	obj = NULL;
4371 Serge 1343
	if (!HAS_LLC(dev))
1344
		obj = i915_gem_object_create_stolen(dev, ring->size);
3480 Serge 1345
	if (obj == NULL)
4560 Serge 1346
        obj = i915_gem_alloc_object(dev, ring->size);
2332 Serge 1347
	if (obj == NULL) {
1348
		DRM_ERROR("Failed to allocate ringbuffer\n");
1349
		ret = -ENOMEM;
1350
		goto err_hws;
1351
	}
1352
 
1353
	ring->obj = obj;
1354
 
4104 Serge 1355
	ret = i915_gem_obj_ggtt_pin(obj, PAGE_SIZE, true, false);
2332 Serge 1356
	if (ret)
1357
		goto err_unref;
1358
 
3031 serge 1359
	ret = i915_gem_object_set_to_gtt_domain(obj, true);
1360
	if (ret)
1361
		goto err_unpin;
2332 Serge 1362
 
3031 serge 1363
	ring->virtual_start =
4104 Serge 1364
		ioremap_wc(dev_priv->gtt.mappable_base + i915_gem_obj_ggtt_offset(obj),
3031 serge 1365
			   ring->size);
1366
	if (ring->virtual_start == NULL) {
2332 Serge 1367
		DRM_ERROR("Failed to map ringbuffer.\n");
1368
		ret = -EINVAL;
1369
		goto err_unpin;
1370
	}
1371
 
1372
	ret = ring->init(ring);
1373
	if (ret)
1374
		goto err_unmap;
1375
 
1376
	/* Workaround an erratum on the i830 which causes a hang if
1377
	 * the TAIL pointer points to within the last 2 cachelines
1378
	 * of the buffer.
1379
	 */
1380
	ring->effective_size = ring->size;
3031 serge 1381
	if (IS_I830(ring->dev) || IS_845G(ring->dev))
2332 Serge 1382
		ring->effective_size -= 128;
2340 Serge 1383
 
2332 Serge 1384
	return 0;
1385
 
1386
err_unmap:
3480 Serge 1387
	iounmap(ring->virtual_start);
2332 Serge 1388
err_unpin:
2344 Serge 1389
	i915_gem_object_unpin(obj);
2332 Serge 1390
err_unref:
2344 Serge 1391
	drm_gem_object_unreference(&obj->base);
2332 Serge 1392
	ring->obj = NULL;
1393
err_hws:
1394
//   cleanup_status_page(ring);
1395
	return ret;
1396
}
1397
 
1398
void intel_cleanup_ring_buffer(struct intel_ring_buffer *ring)
1399
{
1400
	struct drm_i915_private *dev_priv;
1401
	int ret;
1402
 
1403
	if (ring->obj == NULL)
1404
		return;
1405
 
1406
	/* Disable the ring buffer. The ring must be idle at this point */
1407
	dev_priv = ring->dev->dev_private;
3243 Serge 1408
	ret = intel_ring_idle(ring);
4560 Serge 1409
	if (ret && !i915_reset_in_progress(&dev_priv->gpu_error))
2332 Serge 1410
		DRM_ERROR("failed to quiesce %s whilst cleaning up: %d\n",
1411
			  ring->name, ret);
1412
 
1413
	I915_WRITE_CTL(ring, 0);
1414
 
3480 Serge 1415
	iounmap(ring->virtual_start);
2332 Serge 1416
 
2344 Serge 1417
    i915_gem_object_unpin(ring->obj);
1418
	drm_gem_object_unreference(&ring->obj->base);
2332 Serge 1419
	ring->obj = NULL;
4560 Serge 1420
	ring->preallocated_lazy_request = NULL;
1421
	ring->outstanding_lazy_seqno = 0;
2332 Serge 1422
 
1423
	if (ring->cleanup)
1424
		ring->cleanup(ring);
1425
 
1426
//   cleanup_status_page(ring);
1427
}
1428
 
3031 serge 1429
static int intel_ring_wait_seqno(struct intel_ring_buffer *ring, u32 seqno)
2332 Serge 1430
{
3031 serge 1431
	int ret;
2332 Serge 1432
 
3031 serge 1433
	ret = i915_wait_seqno(ring, seqno);
1434
	if (!ret)
1435
		i915_gem_retire_requests_ring(ring);
1436
 
1437
	return ret;
1438
}
1439
 
1440
static int intel_ring_wait_request(struct intel_ring_buffer *ring, int n)
1441
{
1442
	struct drm_i915_gem_request *request;
1443
	u32 seqno = 0;
1444
	int ret;
1445
 
1446
	i915_gem_retire_requests_ring(ring);
1447
 
1448
	if (ring->last_retired_head != -1) {
1449
		ring->head = ring->last_retired_head;
1450
		ring->last_retired_head = -1;
2332 Serge 1451
		ring->space = ring_space(ring);
1452
		if (ring->space >= n)
1453
			return 0;
1454
	}
1455
 
3031 serge 1456
	list_for_each_entry(request, &ring->request_list, list) {
1457
		int space;
2344 Serge 1458
 
3031 serge 1459
		if (request->tail == -1)
1460
			continue;
1461
 
3243 Serge 1462
		space = request->tail - (ring->tail + I915_RING_FREE_SPACE);
3031 serge 1463
		if (space < 0)
1464
			space += ring->size;
1465
		if (space >= n) {
1466
			seqno = request->seqno;
1467
			break;
1468
		}
1469
 
1470
		/* Consume this request in case we need more space than
1471
		 * is available and so need to prevent a race between
1472
		 * updating last_retired_head and direct reads of
1473
		 * I915_RING_HEAD. It also provides a nice sanity check.
1474
		 */
1475
		request->tail = -1;
1476
	}
1477
 
1478
	if (seqno == 0)
1479
		return -ENOSPC;
1480
 
1481
	ret = intel_ring_wait_seqno(ring, seqno);
1482
	if (ret)
1483
		return ret;
1484
 
1485
	if (WARN_ON(ring->last_retired_head == -1))
1486
		return -ENOSPC;
1487
 
1488
	ring->head = ring->last_retired_head;
1489
	ring->last_retired_head = -1;
1490
	ring->space = ring_space(ring);
1491
	if (WARN_ON(ring->space < n))
1492
		return -ENOSPC;
1493
 
1494
	return 0;
1495
}
1496
 
3243 Serge 1497
static int ring_wait_for_space(struct intel_ring_buffer *ring, int n)
3031 serge 1498
{
1499
	struct drm_device *dev = ring->dev;
1500
	struct drm_i915_private *dev_priv = dev->dev_private;
1501
	unsigned long end;
1502
	int ret;
1503
 
1504
	ret = intel_ring_wait_request(ring, n);
1505
	if (ret != -ENOSPC)
1506
		return ret;
1507
 
4560 Serge 1508
	/* force the tail write in case we have been skipping them */
1509
	__intel_ring_advance(ring);
1510
 
3243 Serge 1511
	trace_i915_ring_wait_begin(ring);
3031 serge 1512
	/* With GEM the hangcheck timer should kick us out of the loop,
1513
	 * leaving it early runs the risk of corrupting GEM state (due
1514
	 * to running on almost untested codepaths). But on resume
1515
	 * timers don't work yet, so prevent a complete hang in that
1516
	 * case by choosing an insanely large timeout. */
1517
	end = GetTimerTicks() + 60 * HZ;
1518
 
2332 Serge 1519
	do {
1520
		ring->head = I915_READ_HEAD(ring);
1521
		ring->space = ring_space(ring);
1522
		if (ring->space >= n) {
2351 Serge 1523
			trace_i915_ring_wait_end(ring);
2332 Serge 1524
			return 0;
1525
		}
1526
 
1527
		msleep(1);
3031 serge 1528
 
3480 Serge 1529
		ret = i915_gem_check_wedge(&dev_priv->gpu_error,
1530
					   dev_priv->mm.interruptible);
3031 serge 1531
		if (ret)
1532
			return ret;
1533
    } while (!time_after(GetTimerTicks(), end));
2351 Serge 1534
	trace_i915_ring_wait_end(ring);
2332 Serge 1535
	return -EBUSY;
1536
}
1537
 
3243 Serge 1538
static int intel_wrap_ring_buffer(struct intel_ring_buffer *ring)
1539
{
1540
	uint32_t __iomem *virt;
1541
	int rem = ring->size - ring->tail;
1542
 
1543
	if (ring->space < rem) {
1544
		int ret = ring_wait_for_space(ring, rem);
1545
		if (ret)
1546
			return ret;
1547
	}
1548
 
1549
	virt = ring->virtual_start + ring->tail;
1550
	rem /= 4;
1551
	while (rem--)
1552
		iowrite32(MI_NOOP, virt++);
1553
 
1554
	ring->tail = 0;
1555
	ring->space = ring_space(ring);
1556
 
1557
	return 0;
1558
}
1559
 
1560
int intel_ring_idle(struct intel_ring_buffer *ring)
1561
{
1562
	u32 seqno;
1563
	int ret;
1564
 
1565
	/* We need to add any requests required to flush the objects and ring */
4560 Serge 1566
	if (ring->outstanding_lazy_seqno) {
4104 Serge 1567
		ret = i915_add_request(ring, NULL);
3243 Serge 1568
		if (ret)
1569
			return ret;
1570
	}
1571
 
1572
	/* Wait upon the last request to be completed */
1573
	if (list_empty(&ring->request_list))
1574
		return 0;
1575
 
1576
	seqno = list_entry(ring->request_list.prev,
1577
			   struct drm_i915_gem_request,
1578
			   list)->seqno;
1579
 
1580
	return i915_wait_seqno(ring, seqno);
1581
}
1582
 
1583
static int
1584
intel_ring_alloc_seqno(struct intel_ring_buffer *ring)
1585
{
4560 Serge 1586
	if (ring->outstanding_lazy_seqno)
3243 Serge 1587
		return 0;
1588
 
4560 Serge 1589
	if (ring->preallocated_lazy_request == NULL) {
1590
		struct drm_i915_gem_request *request;
1591
 
1592
		request = kmalloc(sizeof(*request), GFP_KERNEL);
1593
		if (request == NULL)
1594
			return -ENOMEM;
1595
 
1596
		ring->preallocated_lazy_request = request;
1597
	}
1598
 
1599
	return i915_gem_get_seqno(ring->dev, &ring->outstanding_lazy_seqno);
3243 Serge 1600
}
1601
 
4560 Serge 1602
static int __intel_ring_prepare(struct intel_ring_buffer *ring,
3480 Serge 1603
			      int bytes)
1604
{
1605
	int ret;
1606
 
1607
	if (unlikely(ring->tail + bytes > ring->effective_size)) {
1608
		ret = intel_wrap_ring_buffer(ring);
1609
		if (unlikely(ret))
1610
			return ret;
1611
	}
1612
 
1613
	if (unlikely(ring->space < bytes)) {
1614
		ret = ring_wait_for_space(ring, bytes);
1615
		if (unlikely(ret))
1616
			return ret;
1617
	}
1618
 
1619
	return 0;
1620
}
1621
 
2332 Serge 1622
int intel_ring_begin(struct intel_ring_buffer *ring,
1623
		     int num_dwords)
1624
{
3031 serge 1625
	drm_i915_private_t *dev_priv = ring->dev->dev_private;
2332 Serge 1626
	int ret;
1627
 
3480 Serge 1628
	ret = i915_gem_check_wedge(&dev_priv->gpu_error,
1629
				   dev_priv->mm.interruptible);
3031 serge 1630
	if (ret)
1631
		return ret;
2332 Serge 1632
 
4560 Serge 1633
	ret = __intel_ring_prepare(ring, num_dwords * sizeof(uint32_t));
1634
	if (ret)
1635
		return ret;
1636
 
3243 Serge 1637
	/* Preallocate the olr before touching the ring */
1638
	ret = intel_ring_alloc_seqno(ring);
1639
	if (ret)
1640
		return ret;
1641
 
4560 Serge 1642
	ring->space -= num_dwords * sizeof(uint32_t);
1643
	return 0;
3480 Serge 1644
}
2332 Serge 1645
 
3480 Serge 1646
void intel_ring_init_seqno(struct intel_ring_buffer *ring, u32 seqno)
1647
{
1648
	struct drm_i915_private *dev_priv = ring->dev->dev_private;
1649
 
4560 Serge 1650
	BUG_ON(ring->outstanding_lazy_seqno);
3480 Serge 1651
 
1652
	if (INTEL_INFO(ring->dev)->gen >= 6) {
1653
		I915_WRITE(RING_SYNC_0(ring->mmio_base), 0);
1654
		I915_WRITE(RING_SYNC_1(ring->mmio_base), 0);
4104 Serge 1655
		if (HAS_VEBOX(ring->dev))
1656
			I915_WRITE(RING_SYNC_2(ring->mmio_base), 0);
2332 Serge 1657
	}
1658
 
3480 Serge 1659
	ring->set_seqno(ring, seqno);
4104 Serge 1660
	ring->hangcheck.seqno = seqno;
2332 Serge 1661
}
1662
 
1663
static void gen6_bsd_ring_write_tail(struct intel_ring_buffer *ring,
1664
				     u32 value)
1665
{
1666
       drm_i915_private_t *dev_priv = ring->dev->dev_private;
1667
 
1668
       /* Every tail move must follow the sequence below */
3031 serge 1669
 
1670
	/* Disable notification that the ring is IDLE. The GT
1671
	 * will then assume that it is busy and bring it out of rc6.
1672
	 */
2332 Serge 1673
       I915_WRITE(GEN6_BSD_SLEEP_PSMI_CONTROL,
3031 serge 1674
		   _MASKED_BIT_ENABLE(GEN6_BSD_SLEEP_MSG_DISABLE));
2332 Serge 1675
 
3031 serge 1676
	/* Clear the context id. Here be magic! */
1677
	I915_WRITE64(GEN6_BSD_RNCID, 0x0);
1678
 
1679
	/* Wait for the ring not to be idle, i.e. for it to wake up. */
2332 Serge 1680
       if (wait_for((I915_READ(GEN6_BSD_SLEEP_PSMI_CONTROL) &
3031 serge 1681
		      GEN6_BSD_SLEEP_INDICATOR) == 0,
2332 Serge 1682
                       50))
3031 serge 1683
		DRM_ERROR("timed out waiting for the BSD ring to wake up\n");
2332 Serge 1684
 
3031 serge 1685
	/* Now that the ring is fully powered up, update the tail */
2332 Serge 1686
       I915_WRITE_TAIL(ring, value);
3031 serge 1687
	POSTING_READ(RING_TAIL(ring->mmio_base));
1688
 
1689
	/* Let the ring send IDLE messages to the GT again,
1690
	 * and so let it sleep to conserve power when idle.
1691
	 */
2332 Serge 1692
       I915_WRITE(GEN6_BSD_SLEEP_PSMI_CONTROL,
3031 serge 1693
		   _MASKED_BIT_DISABLE(GEN6_BSD_SLEEP_MSG_DISABLE));
2332 Serge 1694
}
1695
 
4104 Serge 1696
static int gen6_bsd_ring_flush(struct intel_ring_buffer *ring,
2332 Serge 1697
			   u32 invalidate, u32 flush)
1698
{
1699
	uint32_t cmd;
1700
	int ret;
1701
 
1702
	ret = intel_ring_begin(ring, 4);
1703
	if (ret)
1704
		return ret;
1705
 
1706
	cmd = MI_FLUSH_DW;
4560 Serge 1707
	if (INTEL_INFO(ring->dev)->gen >= 8)
1708
		cmd += 1;
3243 Serge 1709
	/*
1710
	 * Bspec vol 1c.5 - video engine command streamer:
1711
	 * "If ENABLED, all TLBs will be invalidated once the flush
1712
	 * operation is complete. This bit is only valid when the
1713
	 * Post-Sync Operation field is a value of 1h or 3h."
1714
	 */
2332 Serge 1715
	if (invalidate & I915_GEM_GPU_DOMAINS)
3243 Serge 1716
		cmd |= MI_INVALIDATE_TLB | MI_INVALIDATE_BSD |
1717
			MI_FLUSH_DW_STORE_INDEX | MI_FLUSH_DW_OP_STOREDW;
2332 Serge 1718
	intel_ring_emit(ring, cmd);
3243 Serge 1719
	intel_ring_emit(ring, I915_GEM_HWS_SCRATCH_ADDR | MI_FLUSH_DW_USE_GTT);
4560 Serge 1720
	if (INTEL_INFO(ring->dev)->gen >= 8) {
1721
		intel_ring_emit(ring, 0); /* upper addr */
1722
		intel_ring_emit(ring, 0); /* value */
1723
	} else  {
2332 Serge 1724
	intel_ring_emit(ring, 0);
1725
	intel_ring_emit(ring, MI_NOOP);
4560 Serge 1726
	}
2332 Serge 1727
	intel_ring_advance(ring);
1728
	return 0;
1729
}
1730
 
1731
static int
4560 Serge 1732
gen8_ring_dispatch_execbuffer(struct intel_ring_buffer *ring,
1733
			      u32 offset, u32 len,
1734
			      unsigned flags)
1735
{
1736
	struct drm_i915_private *dev_priv = ring->dev->dev_private;
1737
	bool ppgtt = dev_priv->mm.aliasing_ppgtt != NULL &&
1738
		!(flags & I915_DISPATCH_SECURE);
1739
	int ret;
1740
 
1741
	ret = intel_ring_begin(ring, 4);
1742
	if (ret)
1743
		return ret;
1744
 
1745
	/* FIXME(BDW): Address space and security selectors. */
1746
	intel_ring_emit(ring, MI_BATCH_BUFFER_START_GEN8 | (ppgtt<<8));
1747
	intel_ring_emit(ring, offset);
1748
	intel_ring_emit(ring, 0);
1749
	intel_ring_emit(ring, MI_NOOP);
1750
	intel_ring_advance(ring);
1751
 
1752
	return 0;
1753
}
1754
 
1755
static int
3243 Serge 1756
hsw_ring_dispatch_execbuffer(struct intel_ring_buffer *ring,
1757
			      u32 offset, u32 len,
1758
			      unsigned flags)
1759
{
1760
	int ret;
1761
 
1762
	ret = intel_ring_begin(ring, 2);
1763
	if (ret)
1764
		return ret;
1765
 
1766
	intel_ring_emit(ring,
1767
			MI_BATCH_BUFFER_START | MI_BATCH_PPGTT_HSW |
1768
			(flags & I915_DISPATCH_SECURE ? 0 : MI_BATCH_NON_SECURE_HSW));
1769
	/* bit0-7 is the length on GEN6+ */
1770
	intel_ring_emit(ring, offset);
1771
	intel_ring_advance(ring);
1772
 
1773
	return 0;
1774
}
1775
 
1776
static int
2332 Serge 1777
gen6_ring_dispatch_execbuffer(struct intel_ring_buffer *ring,
3243 Serge 1778
			      u32 offset, u32 len,
1779
			      unsigned flags)
2332 Serge 1780
{
1781
       int ret;
1782
 
1783
       ret = intel_ring_begin(ring, 2);
1784
       if (ret)
1785
	       return ret;
1786
 
3243 Serge 1787
	intel_ring_emit(ring,
1788
			MI_BATCH_BUFFER_START |
1789
			(flags & I915_DISPATCH_SECURE ? 0 : MI_BATCH_NON_SECURE_I965));
2332 Serge 1790
       /* bit0-7 is the length on GEN6+ */
1791
       intel_ring_emit(ring, offset);
1792
       intel_ring_advance(ring);
1793
 
1794
       return 0;
1795
}
1796
 
1797
/* Blitter support (SandyBridge+) */
1798
 
4104 Serge 1799
static int gen6_ring_flush(struct intel_ring_buffer *ring,
2332 Serge 1800
			  u32 invalidate, u32 flush)
1801
{
4104 Serge 1802
	struct drm_device *dev = ring->dev;
2332 Serge 1803
	uint32_t cmd;
1804
	int ret;
1805
 
3031 serge 1806
	ret = intel_ring_begin(ring, 4);
2332 Serge 1807
	if (ret)
1808
		return ret;
1809
 
1810
	cmd = MI_FLUSH_DW;
4560 Serge 1811
	if (INTEL_INFO(ring->dev)->gen >= 8)
1812
		cmd += 1;
3243 Serge 1813
	/*
1814
	 * Bspec vol 1c.3 - blitter engine command streamer:
1815
	 * "If ENABLED, all TLBs will be invalidated once the flush
1816
	 * operation is complete. This bit is only valid when the
1817
	 * Post-Sync Operation field is a value of 1h or 3h."
1818
	 */
2332 Serge 1819
	if (invalidate & I915_GEM_DOMAIN_RENDER)
3243 Serge 1820
		cmd |= MI_INVALIDATE_TLB | MI_FLUSH_DW_STORE_INDEX |
1821
			MI_FLUSH_DW_OP_STOREDW;
2332 Serge 1822
	intel_ring_emit(ring, cmd);
3243 Serge 1823
	intel_ring_emit(ring, I915_GEM_HWS_SCRATCH_ADDR | MI_FLUSH_DW_USE_GTT);
4560 Serge 1824
	if (INTEL_INFO(ring->dev)->gen >= 8) {
1825
		intel_ring_emit(ring, 0); /* upper addr */
1826
		intel_ring_emit(ring, 0); /* value */
1827
	} else  {
2332 Serge 1828
	intel_ring_emit(ring, 0);
1829
	intel_ring_emit(ring, MI_NOOP);
4560 Serge 1830
	}
2332 Serge 1831
	intel_ring_advance(ring);
4104 Serge 1832
 
4560 Serge 1833
	if (IS_GEN7(dev) && !invalidate && flush)
4104 Serge 1834
		return gen7_ring_fbc_flush(ring, FBC_REND_CACHE_CLEAN);
1835
 
2332 Serge 1836
	return 0;
1837
}
1838
 
1839
int intel_init_render_ring_buffer(struct drm_device *dev)
1840
{
1841
	drm_i915_private_t *dev_priv = dev->dev_private;
1842
	struct intel_ring_buffer *ring = &dev_priv->ring[RCS];
2340 Serge 1843
 
3031 serge 1844
	ring->name = "render ring";
1845
	ring->id = RCS;
1846
	ring->mmio_base = RENDER_RING_BASE;
1847
 
2332 Serge 1848
	if (INTEL_INFO(dev)->gen >= 6) {
2339 Serge 1849
       ring->add_request = gen6_add_request;
3031 serge 1850
		ring->flush = gen7_render_ring_flush;
1851
		if (INTEL_INFO(dev)->gen == 6)
2342 Serge 1852
		ring->flush = gen6_render_ring_flush;
4560 Serge 1853
		if (INTEL_INFO(dev)->gen >= 8) {
1854
			ring->flush = gen8_render_ring_flush;
1855
			ring->irq_get = gen8_ring_get_irq;
1856
			ring->irq_put = gen8_ring_put_irq;
1857
		} else {
3031 serge 1858
		ring->irq_get = gen6_ring_get_irq;
1859
		ring->irq_put = gen6_ring_put_irq;
4560 Serge 1860
		}
4104 Serge 1861
		ring->irq_enable_mask = GT_RENDER_USER_INTERRUPT;
2342 Serge 1862
		ring->get_seqno = gen6_ring_get_seqno;
3480 Serge 1863
		ring->set_seqno = ring_set_seqno;
3031 serge 1864
		ring->sync_to = gen6_ring_sync;
4104 Serge 1865
		ring->semaphore_register[RCS] = MI_SEMAPHORE_SYNC_INVALID;
1866
		ring->semaphore_register[VCS] = MI_SEMAPHORE_SYNC_RV;
1867
		ring->semaphore_register[BCS] = MI_SEMAPHORE_SYNC_RB;
1868
		ring->semaphore_register[VECS] = MI_SEMAPHORE_SYNC_RVE;
1869
		ring->signal_mbox[RCS] = GEN6_NOSYNC;
1870
		ring->signal_mbox[VCS] = GEN6_VRSYNC;
1871
		ring->signal_mbox[BCS] = GEN6_BRSYNC;
1872
		ring->signal_mbox[VECS] = GEN6_VERSYNC;
2332 Serge 1873
	} else if (IS_GEN5(dev)) {
2339 Serge 1874
       ring->add_request = pc_render_add_request;
3031 serge 1875
		ring->flush = gen4_render_ring_flush;
2342 Serge 1876
		ring->get_seqno = pc_render_get_seqno;
3480 Serge 1877
		ring->set_seqno = pc_render_set_seqno;
3031 serge 1878
		ring->irq_get = gen5_ring_get_irq;
1879
		ring->irq_put = gen5_ring_put_irq;
4104 Serge 1880
		ring->irq_enable_mask = GT_RENDER_USER_INTERRUPT |
1881
					GT_RENDER_PIPECTL_NOTIFY_INTERRUPT;
3031 serge 1882
	} else {
1883
		ring->add_request = i9xx_add_request;
1884
		if (INTEL_INFO(dev)->gen < 4)
1885
			ring->flush = gen2_render_ring_flush;
1886
		else
1887
			ring->flush = gen4_render_ring_flush;
1888
		ring->get_seqno = ring_get_seqno;
3480 Serge 1889
		ring->set_seqno = ring_set_seqno;
3031 serge 1890
		if (IS_GEN2(dev)) {
1891
			ring->irq_get = i8xx_ring_get_irq;
1892
			ring->irq_put = i8xx_ring_put_irq;
1893
		} else {
1894
			ring->irq_get = i9xx_ring_get_irq;
1895
			ring->irq_put = i9xx_ring_put_irq;
1896
		}
1897
		ring->irq_enable_mask = I915_USER_INTERRUPT;
2332 Serge 1898
	}
3031 serge 1899
	ring->write_tail = ring_write_tail;
3243 Serge 1900
	if (IS_HASWELL(dev))
1901
		ring->dispatch_execbuffer = hsw_ring_dispatch_execbuffer;
4560 Serge 1902
	else if (IS_GEN8(dev))
1903
		ring->dispatch_execbuffer = gen8_ring_dispatch_execbuffer;
3243 Serge 1904
	else if (INTEL_INFO(dev)->gen >= 6)
3031 serge 1905
		ring->dispatch_execbuffer = gen6_ring_dispatch_execbuffer;
1906
	else if (INTEL_INFO(dev)->gen >= 4)
1907
		ring->dispatch_execbuffer = i965_dispatch_execbuffer;
1908
	else if (IS_I830(dev) || IS_845G(dev))
1909
		ring->dispatch_execbuffer = i830_dispatch_execbuffer;
1910
	else
1911
		ring->dispatch_execbuffer = i915_dispatch_execbuffer;
1912
	ring->init = init_render_ring;
1913
	ring->cleanup = render_ring_cleanup;
2332 Serge 1914
 
3243 Serge 1915
	/* Workaround batchbuffer to combat CS tlb bug. */
1916
	if (HAS_BROKEN_CS_TLB(dev)) {
1917
		struct drm_i915_gem_object *obj;
1918
		int ret;
3031 serge 1919
 
3243 Serge 1920
		obj = i915_gem_alloc_object(dev, I830_BATCH_LIMIT);
1921
		if (obj == NULL) {
1922
			DRM_ERROR("Failed to allocate batch bo\n");
1923
			return -ENOMEM;
1924
		}
1925
 
4104 Serge 1926
		ret = i915_gem_obj_ggtt_pin(obj, 0, true, false);
3243 Serge 1927
		if (ret != 0) {
1928
			drm_gem_object_unreference(&obj->base);
1929
			DRM_ERROR("Failed to ping batch bo\n");
1930
			return ret;
1931
		}
1932
 
4104 Serge 1933
		ring->scratch.obj = obj;
1934
		ring->scratch.gtt_offset = i915_gem_obj_ggtt_offset(obj);
2332 Serge 1935
	}
2340 Serge 1936
 
2332 Serge 1937
	return intel_init_ring_buffer(dev, ring);
1938
}
1939
 
3243 Serge 1940
#if 0
1941
int intel_render_ring_init_dri(struct drm_device *dev, u64 start, u32 size)
1942
{
1943
	drm_i915_private_t *dev_priv = dev->dev_private;
1944
	struct intel_ring_buffer *ring = &dev_priv->ring[RCS];
1945
	int ret;
2332 Serge 1946
 
3243 Serge 1947
	ring->name = "render ring";
1948
	ring->id = RCS;
1949
	ring->mmio_base = RENDER_RING_BASE;
1950
 
1951
	if (INTEL_INFO(dev)->gen >= 6) {
1952
		/* non-kms not supported on gen6+ */
1953
		return -ENODEV;
1954
	}
1955
 
1956
	/* Note: gem is not supported on gen5/ilk without kms (the corresponding
1957
	 * gem_init ioctl returns with -ENODEV). Hence we do not need to set up
1958
	 * the special gen5 functions. */
1959
	ring->add_request = i9xx_add_request;
1960
	if (INTEL_INFO(dev)->gen < 4)
1961
		ring->flush = gen2_render_ring_flush;
1962
	else
1963
		ring->flush = gen4_render_ring_flush;
1964
	ring->get_seqno = ring_get_seqno;
3480 Serge 1965
	ring->set_seqno = ring_set_seqno;
3243 Serge 1966
	if (IS_GEN2(dev)) {
1967
		ring->irq_get = i8xx_ring_get_irq;
1968
		ring->irq_put = i8xx_ring_put_irq;
1969
	} else {
1970
		ring->irq_get = i9xx_ring_get_irq;
1971
		ring->irq_put = i9xx_ring_put_irq;
1972
	}
1973
	ring->irq_enable_mask = I915_USER_INTERRUPT;
1974
	ring->write_tail = ring_write_tail;
1975
	if (INTEL_INFO(dev)->gen >= 4)
1976
		ring->dispatch_execbuffer = i965_dispatch_execbuffer;
1977
	else if (IS_I830(dev) || IS_845G(dev))
1978
		ring->dispatch_execbuffer = i830_dispatch_execbuffer;
1979
	else
1980
		ring->dispatch_execbuffer = i915_dispatch_execbuffer;
1981
	ring->init = init_render_ring;
1982
	ring->cleanup = render_ring_cleanup;
1983
 
1984
	ring->dev = dev;
1985
	INIT_LIST_HEAD(&ring->active_list);
1986
	INIT_LIST_HEAD(&ring->request_list);
1987
 
1988
	ring->size = size;
1989
	ring->effective_size = ring->size;
1990
	if (IS_I830(ring->dev) || IS_845G(ring->dev))
1991
		ring->effective_size -= 128;
1992
 
1993
	ring->virtual_start = ioremap_wc(start, size);
1994
	if (ring->virtual_start == NULL) {
1995
		DRM_ERROR("can not ioremap virtual address for"
1996
			  " ring buffer\n");
1997
		return -ENOMEM;
1998
	}
1999
 
2000
	if (!I915_NEED_GFX_HWS(dev)) {
4104 Serge 2001
		ret = init_phys_status_page(ring);
3243 Serge 2002
		if (ret)
2003
			return ret;
2004
	}
2005
 
2006
	return 0;
2007
}
2008
#endif
2009
 
2332 Serge 2010
int intel_init_bsd_ring_buffer(struct drm_device *dev)
2011
{
2012
	drm_i915_private_t *dev_priv = dev->dev_private;
2013
	struct intel_ring_buffer *ring = &dev_priv->ring[VCS];
2014
 
3031 serge 2015
	ring->name = "bsd ring";
2016
	ring->id = VCS;
2332 Serge 2017
 
3031 serge 2018
	ring->write_tail = ring_write_tail;
4560 Serge 2019
	if (INTEL_INFO(dev)->gen >= 6) {
3031 serge 2020
		ring->mmio_base = GEN6_BSD_RING_BASE;
2021
		/* gen6 bsd needs a special wa for tail updates */
2022
		if (IS_GEN6(dev))
2023
			ring->write_tail = gen6_bsd_ring_write_tail;
4104 Serge 2024
		ring->flush = gen6_bsd_ring_flush;
3031 serge 2025
		ring->add_request = gen6_add_request;
2026
		ring->get_seqno = gen6_ring_get_seqno;
3480 Serge 2027
		ring->set_seqno = ring_set_seqno;
4560 Serge 2028
		if (INTEL_INFO(dev)->gen >= 8) {
2029
			ring->irq_enable_mask =
2030
				GT_RENDER_USER_INTERRUPT << GEN8_VCS1_IRQ_SHIFT;
2031
			ring->irq_get = gen8_ring_get_irq;
2032
			ring->irq_put = gen8_ring_put_irq;
2033
			ring->dispatch_execbuffer =
2034
				gen8_ring_dispatch_execbuffer;
2035
		} else {
4104 Serge 2036
		ring->irq_enable_mask = GT_BSD_USER_INTERRUPT;
3031 serge 2037
		ring->irq_get = gen6_ring_get_irq;
2038
		ring->irq_put = gen6_ring_put_irq;
4560 Serge 2039
			ring->dispatch_execbuffer =
2040
				gen6_ring_dispatch_execbuffer;
2041
		}
3031 serge 2042
		ring->sync_to = gen6_ring_sync;
4104 Serge 2043
		ring->semaphore_register[RCS] = MI_SEMAPHORE_SYNC_VR;
2044
		ring->semaphore_register[VCS] = MI_SEMAPHORE_SYNC_INVALID;
2045
		ring->semaphore_register[BCS] = MI_SEMAPHORE_SYNC_VB;
2046
		ring->semaphore_register[VECS] = MI_SEMAPHORE_SYNC_VVE;
2047
		ring->signal_mbox[RCS] = GEN6_RVSYNC;
2048
		ring->signal_mbox[VCS] = GEN6_NOSYNC;
2049
		ring->signal_mbox[BCS] = GEN6_BVSYNC;
2050
		ring->signal_mbox[VECS] = GEN6_VEVSYNC;
3031 serge 2051
	} else {
2052
		ring->mmio_base = BSD_RING_BASE;
2053
		ring->flush = bsd_ring_flush;
2054
		ring->add_request = i9xx_add_request;
2055
		ring->get_seqno = ring_get_seqno;
3480 Serge 2056
		ring->set_seqno = ring_set_seqno;
3031 serge 2057
		if (IS_GEN5(dev)) {
4104 Serge 2058
			ring->irq_enable_mask = ILK_BSD_USER_INTERRUPT;
3031 serge 2059
			ring->irq_get = gen5_ring_get_irq;
2060
			ring->irq_put = gen5_ring_put_irq;
2061
		} else {
2062
			ring->irq_enable_mask = I915_BSD_USER_INTERRUPT;
2063
			ring->irq_get = i9xx_ring_get_irq;
2064
			ring->irq_put = i9xx_ring_put_irq;
2065
		}
2066
		ring->dispatch_execbuffer = i965_dispatch_execbuffer;
2067
	}
2068
	ring->init = init_ring_common;
2069
 
2332 Serge 2070
	return intel_init_ring_buffer(dev, ring);
2071
}
2072
 
2073
int intel_init_blt_ring_buffer(struct drm_device *dev)
2074
{
2075
	drm_i915_private_t *dev_priv = dev->dev_private;
2076
	struct intel_ring_buffer *ring = &dev_priv->ring[BCS];
2077
 
3031 serge 2078
	ring->name = "blitter ring";
2079
	ring->id = BCS;
2332 Serge 2080
 
3031 serge 2081
	ring->mmio_base = BLT_RING_BASE;
2082
	ring->write_tail = ring_write_tail;
4104 Serge 2083
	ring->flush = gen6_ring_flush;
3031 serge 2084
	ring->add_request = gen6_add_request;
2085
	ring->get_seqno = gen6_ring_get_seqno;
3480 Serge 2086
	ring->set_seqno = ring_set_seqno;
4560 Serge 2087
	if (INTEL_INFO(dev)->gen >= 8) {
2088
		ring->irq_enable_mask =
2089
			GT_RENDER_USER_INTERRUPT << GEN8_BCS_IRQ_SHIFT;
2090
		ring->irq_get = gen8_ring_get_irq;
2091
		ring->irq_put = gen8_ring_put_irq;
2092
		ring->dispatch_execbuffer = gen8_ring_dispatch_execbuffer;
2093
	} else {
4104 Serge 2094
	ring->irq_enable_mask = GT_BLT_USER_INTERRUPT;
3031 serge 2095
	ring->irq_get = gen6_ring_get_irq;
2096
	ring->irq_put = gen6_ring_put_irq;
2097
	ring->dispatch_execbuffer = gen6_ring_dispatch_execbuffer;
4560 Serge 2098
	}
3031 serge 2099
	ring->sync_to = gen6_ring_sync;
4104 Serge 2100
	ring->semaphore_register[RCS] = MI_SEMAPHORE_SYNC_BR;
2101
	ring->semaphore_register[VCS] = MI_SEMAPHORE_SYNC_BV;
2102
	ring->semaphore_register[BCS] = MI_SEMAPHORE_SYNC_INVALID;
2103
	ring->semaphore_register[VECS] = MI_SEMAPHORE_SYNC_BVE;
2104
	ring->signal_mbox[RCS] = GEN6_RBSYNC;
2105
	ring->signal_mbox[VCS] = GEN6_VBSYNC;
2106
	ring->signal_mbox[BCS] = GEN6_NOSYNC;
2107
	ring->signal_mbox[VECS] = GEN6_VEBSYNC;
3031 serge 2108
	ring->init = init_ring_common;
2109
 
2332 Serge 2110
	return intel_init_ring_buffer(dev, ring);
2111
}
3031 serge 2112
 
4104 Serge 2113
int intel_init_vebox_ring_buffer(struct drm_device *dev)
2114
{
2115
	drm_i915_private_t *dev_priv = dev->dev_private;
2116
	struct intel_ring_buffer *ring = &dev_priv->ring[VECS];
2117
 
2118
	ring->name = "video enhancement ring";
2119
	ring->id = VECS;
2120
 
2121
	ring->mmio_base = VEBOX_RING_BASE;
2122
	ring->write_tail = ring_write_tail;
2123
	ring->flush = gen6_ring_flush;
2124
	ring->add_request = gen6_add_request;
2125
	ring->get_seqno = gen6_ring_get_seqno;
2126
	ring->set_seqno = ring_set_seqno;
4560 Serge 2127
 
2128
	if (INTEL_INFO(dev)->gen >= 8) {
2129
		ring->irq_enable_mask =
2130
			GT_RENDER_USER_INTERRUPT << GEN8_VECS_IRQ_SHIFT;
2131
		ring->irq_get = gen8_ring_get_irq;
2132
		ring->irq_put = gen8_ring_put_irq;
2133
		ring->dispatch_execbuffer = gen8_ring_dispatch_execbuffer;
2134
	} else {
4104 Serge 2135
	ring->irq_enable_mask = PM_VEBOX_USER_INTERRUPT;
2136
	ring->irq_get = hsw_vebox_get_irq;
2137
	ring->irq_put = hsw_vebox_put_irq;
2138
	ring->dispatch_execbuffer = gen6_ring_dispatch_execbuffer;
4560 Serge 2139
	}
4104 Serge 2140
	ring->sync_to = gen6_ring_sync;
2141
	ring->semaphore_register[RCS] = MI_SEMAPHORE_SYNC_VER;
2142
	ring->semaphore_register[VCS] = MI_SEMAPHORE_SYNC_VEV;
2143
	ring->semaphore_register[BCS] = MI_SEMAPHORE_SYNC_VEB;
2144
	ring->semaphore_register[VECS] = MI_SEMAPHORE_SYNC_INVALID;
2145
	ring->signal_mbox[RCS] = GEN6_RVESYNC;
2146
	ring->signal_mbox[VCS] = GEN6_VVESYNC;
2147
	ring->signal_mbox[BCS] = GEN6_BVESYNC;
2148
	ring->signal_mbox[VECS] = GEN6_NOSYNC;
2149
	ring->init = init_ring_common;
2150
 
2151
	return intel_init_ring_buffer(dev, ring);
2152
}
2153
 
3031 serge 2154
int
2155
intel_ring_flush_all_caches(struct intel_ring_buffer *ring)
2156
{
2157
	int ret;
2158
 
2159
	if (!ring->gpu_caches_dirty)
2160
		return 0;
2161
 
2162
	ret = ring->flush(ring, 0, I915_GEM_GPU_DOMAINS);
2163
	if (ret)
2164
		return ret;
2165
 
2166
	trace_i915_gem_ring_flush(ring, 0, I915_GEM_GPU_DOMAINS);
2167
 
2168
	ring->gpu_caches_dirty = false;
2169
	return 0;
2170
}
2171
 
2172
int
2173
intel_ring_invalidate_all_caches(struct intel_ring_buffer *ring)
2174
{
2175
	uint32_t flush_domains;
2176
	int ret;
2177
 
2178
	flush_domains = 0;
2179
	if (ring->gpu_caches_dirty)
2180
		flush_domains = I915_GEM_GPU_DOMAINS;
2181
 
2182
	ret = ring->flush(ring, I915_GEM_GPU_DOMAINS, flush_domains);
2183
	if (ret)
2184
		return ret;
2185
 
2186
	trace_i915_gem_ring_flush(ring, I915_GEM_GPU_DOMAINS, flush_domains);
2187
 
2188
	ring->gpu_caches_dirty = false;
2189
	return 0;
2190
}