WebSVN – Kolibri OS – Blame – /drivers/video/drm/i915/intel_ringbuffer.c

Rev	Author	Line No.	Line
2332	Serge	1	/*
		2	* Copyright © 2008-2010 Intel Corporation
		3	*
		4	* Permission is hereby granted, free of charge, to any person obtaining a
		5	* copy of this software and associated documentation files (the "Software"),
		6	* to deal in the Software without restriction, including without limitation
		7	* the rights to use, copy, modify, merge, publish, distribute, sublicense,
		8	* and/or sell copies of the Software, and to permit persons to whom the
		9	* Software is furnished to do so, subject to the following conditions:
		10	*
		11	* The above copyright notice and this permission notice (including the next
		12	* paragraph) shall be included in all copies or substantial portions of the
		13	* Software.
		14	*
		15	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
		16	* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
		17	* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
		18	* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
		19	* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
		20	* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
		21	* IN THE SOFTWARE.
		22	*
		23	* Authors:
		24	* Eric Anholt
		25	* Zou Nan hai
		26	* Xiang Hai hao
		27	*
		28	*/
		29
3031	serge	30	#include
2332	Serge	31	#include "i915_drv.h"
3031	serge	32	#include
2351	Serge	33	#include "i915_trace.h"
2332	Serge	34	#include "intel_drv.h"
		35
5354	serge	36	bool
		37	intel_ring_initialized(struct intel_engine_cs *ring)
		38	{
		39	struct drm_device *dev = ring->dev;
5060	serge	40
5354	serge	41	if (!dev)
		42	return false;
		43
		44	if (i915.enable_execlists) {
		45	struct intel_context *dctx = ring->default_context;
		46	struct intel_ringbuffer *ringbuf = dctx->engine[ring->id].ringbuf;
		47
		48	return ringbuf->obj;
		49	} else
		50	return ring->buffer && ring->buffer->obj;
		51	}
		52
		53	int __intel_ring_space(int head, int tail, int size)
2332	Serge	54	{
6084	serge	55	int space = head - tail;
		56	if (space <= 0)
5060	serge	57	space += size;
6084	serge	58	return space - I915_RING_FREE_SPACE;
2332	Serge	59	}
		60
6084	serge	61	void intel_ring_update_space(struct intel_ringbuffer *ringbuf)
		62	{
		63	if (ringbuf->last_retired_head != -1) {
		64	ringbuf->head = ringbuf->last_retired_head;
		65	ringbuf->last_retired_head = -1;
		66	}
		67
		68	ringbuf->space = __intel_ring_space(ringbuf->head & HEAD_ADDR,
		69	ringbuf->tail, ringbuf->size);
		70	}
		71
5354	serge	72	int intel_ring_space(struct intel_ringbuffer *ringbuf)
4560	Serge	73	{
6084	serge	74	intel_ring_update_space(ringbuf);
		75	return ringbuf->space;
5060	serge	76	}
		77
5354	serge	78	bool intel_ring_stopped(struct intel_engine_cs *ring)
5060	serge	79	{
4560	Serge	80	struct drm_i915_private *dev_priv = ring->dev->dev_private;
5060	serge	81	return dev_priv->gpu_error.stop_rings & intel_ring_flag(ring);
		82	}
4560	Serge	83
6084	serge	84	static void __intel_ring_advance(struct intel_engine_cs *ring)
5060	serge	85	{
		86	struct intel_ringbuffer *ringbuf = ring->buffer;
		87	ringbuf->tail &= ringbuf->size - 1;
		88	if (intel_ring_stopped(ring))
4560	Serge	89	return;
5060	serge	90	ring->write_tail(ring, ringbuf->tail);
4560	Serge	91	}
		92
3031	serge	93	static int
6084	serge	94	gen2_render_ring_flush(struct drm_i915_gem_request *req,
3031	serge	95	u32 invalidate_domains,
		96	u32 flush_domains)
2332	Serge	97	{
6084	serge	98	struct intel_engine_cs *ring = req->ring;
3031	serge	99	u32 cmd;
		100	int ret;
2332	Serge	101
3031	serge	102	cmd = MI_FLUSH;
		103	if (((invalidate_domains\|flush_domains) & I915_GEM_DOMAIN_RENDER) == 0)
		104	cmd \|= MI_NO_WRITE_FLUSH;
2332	Serge	105
3031	serge	106	if (invalidate_domains & I915_GEM_DOMAIN_SAMPLER)
		107	cmd \|= MI_READ_FLUSH;
2332	Serge	108
6084	serge	109	ret = intel_ring_begin(req, 2);
3031	serge	110	if (ret)
		111	return ret;
		112
		113	intel_ring_emit(ring, cmd);
		114	intel_ring_emit(ring, MI_NOOP);
		115	intel_ring_advance(ring);
		116
		117	return 0;
2332	Serge	118	}
		119
		120	static int
6084	serge	121	gen4_render_ring_flush(struct drm_i915_gem_request *req,
		122	u32 invalidate_domains,
		123	u32 flush_domains)
2332	Serge	124	{
6084	serge	125	struct intel_engine_cs *ring = req->ring;
2332	Serge	126	struct drm_device *dev = ring->dev;
		127	u32 cmd;
		128	int ret;
		129
		130	/*
		131	* read/write caches:
		132	*
		133	* I915_GEM_DOMAIN_RENDER is always invalidated, but is
		134	* only flushed if MI_NO_WRITE_FLUSH is unset. On 965, it is
		135	* also flushed at 2d versus 3d pipeline switches.
		136	*
		137	* read-only caches:
		138	*
		139	* I915_GEM_DOMAIN_SAMPLER is flushed on pre-965 if
		140	* MI_READ_FLUSH is set, and is always flushed on 965.
		141	*
		142	* I915_GEM_DOMAIN_COMMAND may not exist?
		143	*
		144	* I915_GEM_DOMAIN_INSTRUCTION, which exists on 965, is
		145	* invalidated when MI_EXE_FLUSH is set.
		146	*
		147	* I915_GEM_DOMAIN_VERTEX, which exists on 965, is
		148	* invalidated with every MI_FLUSH.
		149	*
		150	* TLBs:
		151	*
		152	* On 965, TLBs associated with I915_GEM_DOMAIN_COMMAND
		153	* and I915_GEM_DOMAIN_CPU in are invalidated at PTE write and
		154	* I915_GEM_DOMAIN_RENDER and I915_GEM_DOMAIN_SAMPLER
		155	* are flushed at any MI_FLUSH.
		156	*/
		157
		158	cmd = MI_FLUSH \| MI_NO_WRITE_FLUSH;
3031	serge	159	if ((invalidate_domains\|flush_domains) & I915_GEM_DOMAIN_RENDER)
2332	Serge	160	cmd &= ~MI_NO_WRITE_FLUSH;
		161	if (invalidate_domains & I915_GEM_DOMAIN_INSTRUCTION)
		162	cmd \|= MI_EXE_FLUSH;
		163
		164	if (invalidate_domains & I915_GEM_DOMAIN_COMMAND &&
		165	(IS_G4X(dev) \|\| IS_GEN5(dev)))
		166	cmd \|= MI_INVALIDATE_ISP;
		167
6084	serge	168	ret = intel_ring_begin(req, 2);
2332	Serge	169	if (ret)
		170	return ret;
		171
		172	intel_ring_emit(ring, cmd);
		173	intel_ring_emit(ring, MI_NOOP);
		174	intel_ring_advance(ring);
		175
		176	return 0;
		177	}
		178
2342	Serge	179	/**
		180	* Emits a PIPE_CONTROL with a non-zero post-sync operation, for
		181	* implementing two workarounds on gen6. From section 1.4.7.1
		182	* "PIPE_CONTROL" of the Sandy Bridge PRM volume 2 part 1:
		183	*
		184	* [DevSNB-C+{W/A}] Before any depth stall flush (including those
		185	* produced by non-pipelined state commands), software needs to first
		186	* send a PIPE_CONTROL with no bits set except Post-Sync Operation !=
		187	* 0.
		188	*
		189	* [Dev-SNB{W/A}]: Before a PIPE_CONTROL with Write Cache Flush Enable
		190	* =1, a PIPE_CONTROL with any non-zero post-sync-op is required.
		191	*
		192	* And the workaround for these two requires this workaround first:
		193	*
		194	* [Dev-SNB{W/A}]: Pipe-control with CS-stall bit set must be sent
		195	* BEFORE the pipe-control with a post-sync op and no write-cache
		196	* flushes.
		197	*
		198	* And this last workaround is tricky because of the requirements on
		199	* that bit. From section 1.4.7.2.3 "Stall" of the Sandy Bridge PRM
		200	* volume 2 part 1:
		201	*
		202	* "1 of the following must also be set:
		203	* - Render Target Cache Flush Enable ([12] of DW1)
		204	* - Depth Cache Flush Enable ([0] of DW1)
		205	* - Stall at Pixel Scoreboard ([1] of DW1)
		206	* - Depth Stall ([13] of DW1)
		207	* - Post-Sync Operation ([13] of DW1)
		208	* - Notify Enable ([8] of DW1)"
		209	*
		210	* The cache flushes require the workaround flush that triggered this
		211	* one, so we can't use it. Depth stall would trigger the same.
		212	* Post-sync nonzero is what triggered this second workaround, so we
		213	* can't use that one either. Notify enable is IRQs, which aren't
		214	* really our business. That leaves only stall at scoreboard.
		215	*/
		216	static int
6084	serge	217	intel_emit_post_sync_nonzero_flush(struct drm_i915_gem_request *req)
2342	Serge	218	{
6084	serge	219	struct intel_engine_cs *ring = req->ring;
5060	serge	220	u32 scratch_addr = ring->scratch.gtt_offset + 2 * CACHELINE_BYTES;
2342	Serge	221	int ret;
		222
6084	serge	223	ret = intel_ring_begin(req, 6);
2342	Serge	224	if (ret)
		225	return ret;
		226
		227	intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(5));
		228	intel_ring_emit(ring, PIPE_CONTROL_CS_STALL \|
		229	PIPE_CONTROL_STALL_AT_SCOREBOARD);
		230	intel_ring_emit(ring, scratch_addr \| PIPE_CONTROL_GLOBAL_GTT); /* address */
		231	intel_ring_emit(ring, 0); /* low dword */
		232	intel_ring_emit(ring, 0); /* high dword */
		233	intel_ring_emit(ring, MI_NOOP);
		234	intel_ring_advance(ring);
		235
6084	serge	236	ret = intel_ring_begin(req, 6);
2342	Serge	237	if (ret)
		238	return ret;
		239
		240	intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(5));
		241	intel_ring_emit(ring, PIPE_CONTROL_QW_WRITE);
		242	intel_ring_emit(ring, scratch_addr \| PIPE_CONTROL_GLOBAL_GTT); /* address */
		243	intel_ring_emit(ring, 0);
		244	intel_ring_emit(ring, 0);
		245	intel_ring_emit(ring, MI_NOOP);
		246	intel_ring_advance(ring);
		247
		248	return 0;
		249	}
		250
		251	static int
6084	serge	252	gen6_render_ring_flush(struct drm_i915_gem_request *req,
		253	u32 invalidate_domains, u32 flush_domains)
2342	Serge	254	{
6084	serge	255	struct intel_engine_cs *ring = req->ring;
2342	Serge	256	u32 flags = 0;
5060	serge	257	u32 scratch_addr = ring->scratch.gtt_offset + 2 * CACHELINE_BYTES;
2342	Serge	258	int ret;
		259
		260	/* Force SNB workarounds for PIPE_CONTROL flushes */
6084	serge	261	ret = intel_emit_post_sync_nonzero_flush(req);
3031	serge	262	if (ret)
		263	return ret;
2342	Serge	264
		265	/* Just flush everything. Experiments have shown that reducing the
		266	* number of bits based on the write domains has little performance
		267	* impact.
		268	*/
3031	serge	269	if (flush_domains) {
		270	flags \|= PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH;
		271	flags \|= PIPE_CONTROL_DEPTH_CACHE_FLUSH;
		272	/*
		273	* Ensure that any following seqno writes only happen
		274	* when the render cache is indeed flushed.
		275	*/
		276	flags \|= PIPE_CONTROL_CS_STALL;
		277	}
		278	if (invalidate_domains) {
		279	flags \|= PIPE_CONTROL_TLB_INVALIDATE;
		280	flags \|= PIPE_CONTROL_INSTRUCTION_CACHE_INVALIDATE;
		281	flags \|= PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE;
		282	flags \|= PIPE_CONTROL_VF_CACHE_INVALIDATE;
		283	flags \|= PIPE_CONTROL_CONST_CACHE_INVALIDATE;
		284	flags \|= PIPE_CONTROL_STATE_CACHE_INVALIDATE;
		285	/*
		286	* TLB invalidate requires a post-sync write.
		287	*/
3243	Serge	288	flags \|= PIPE_CONTROL_QW_WRITE \| PIPE_CONTROL_CS_STALL;
3031	serge	289	}
		290
6084	serge	291	ret = intel_ring_begin(req, 4);
3031	serge	292	if (ret)
		293	return ret;
		294
		295	intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(4));
		296	intel_ring_emit(ring, flags);
		297	intel_ring_emit(ring, scratch_addr \| PIPE_CONTROL_GLOBAL_GTT);
		298	intel_ring_emit(ring, 0);
		299	intel_ring_advance(ring);
		300
		301	return 0;
		302	}
		303
		304	static int
6084	serge	305	gen7_render_ring_cs_stall_wa(struct drm_i915_gem_request *req)
3031	serge	306	{
6084	serge	307	struct intel_engine_cs *ring = req->ring;
3031	serge	308	int ret;
		309
6084	serge	310	ret = intel_ring_begin(req, 4);
3031	serge	311	if (ret)
		312	return ret;
		313
		314	intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(4));
		315	intel_ring_emit(ring, PIPE_CONTROL_CS_STALL \|
		316	PIPE_CONTROL_STALL_AT_SCOREBOARD);
		317	intel_ring_emit(ring, 0);
		318	intel_ring_emit(ring, 0);
		319	intel_ring_advance(ring);
		320
		321	return 0;
		322	}
		323
		324	static int
6084	serge	325	gen7_render_ring_flush(struct drm_i915_gem_request *req,
3031	serge	326	u32 invalidate_domains, u32 flush_domains)
		327	{
6084	serge	328	struct intel_engine_cs *ring = req->ring;
3031	serge	329	u32 flags = 0;
5060	serge	330	u32 scratch_addr = ring->scratch.gtt_offset + 2 * CACHELINE_BYTES;
3031	serge	331	int ret;
		332
		333	/*
		334	* Ensure that any following seqno writes only happen when the render
		335	* cache is indeed flushed.
		336	*
		337	* Workaround: 4th PIPE_CONTROL command (except the ones with only
		338	* read-cache invalidate bits set) must have the CS_STALL bit set. We
		339	* don't try to be clever and just set it unconditionally.
		340	*/
		341	flags \|= PIPE_CONTROL_CS_STALL;
		342
		343	/* Just flush everything. Experiments have shown that reducing the
		344	* number of bits based on the write domains has little performance
		345	* impact.
		346	*/
		347	if (flush_domains) {
6084	serge	348	flags \|= PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH;
3031	serge	349	flags \|= PIPE_CONTROL_DEPTH_CACHE_FLUSH;
6320	serge	350	flags \|= PIPE_CONTROL_DC_FLUSH_ENABLE;
6084	serge	351	flags \|= PIPE_CONTROL_FLUSH_ENABLE;
3031	serge	352	}
		353	if (invalidate_domains) {
		354	flags \|= PIPE_CONTROL_TLB_INVALIDATE;
6084	serge	355	flags \|= PIPE_CONTROL_INSTRUCTION_CACHE_INVALIDATE;
		356	flags \|= PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE;
		357	flags \|= PIPE_CONTROL_VF_CACHE_INVALIDATE;
		358	flags \|= PIPE_CONTROL_CONST_CACHE_INVALIDATE;
		359	flags \|= PIPE_CONTROL_STATE_CACHE_INVALIDATE;
5354	serge	360	flags \|= PIPE_CONTROL_MEDIA_STATE_CLEAR;
3031	serge	361	/*
		362	* TLB invalidate requires a post-sync write.
		363	*/
		364	flags \|= PIPE_CONTROL_QW_WRITE;
3480	Serge	365	flags \|= PIPE_CONTROL_GLOBAL_GTT_IVB;
2342	Serge	366
5354	serge	367	flags \|= PIPE_CONTROL_STALL_AT_SCOREBOARD;
		368
3031	serge	369	/* Workaround: we must issue a pipe_control with CS-stall bit
		370	* set before a pipe_control command that has the state cache
		371	* invalidate bit set. */
6084	serge	372	gen7_render_ring_cs_stall_wa(req);
3031	serge	373	}
		374
6084	serge	375	ret = intel_ring_begin(req, 4);
2342	Serge	376	if (ret)
		377	return ret;
		378
3031	serge	379	intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(4));
2342	Serge	380	intel_ring_emit(ring, flags);
3480	Serge	381	intel_ring_emit(ring, scratch_addr);
3031	serge	382	intel_ring_emit(ring, 0);
2342	Serge	383	intel_ring_advance(ring);
		384
		385	return 0;
		386	}
		387
4560	Serge	388	static int
6084	serge	389	gen8_emit_pipe_control(struct drm_i915_gem_request *req,
5060	serge	390	u32 flags, u32 scratch_addr)
		391	{
6084	serge	392	struct intel_engine_cs *ring = req->ring;
5060	serge	393	int ret;
		394
6084	serge	395	ret = intel_ring_begin(req, 6);
5060	serge	396	if (ret)
		397	return ret;
		398
		399	intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(6));
		400	intel_ring_emit(ring, flags);
		401	intel_ring_emit(ring, scratch_addr);
		402	intel_ring_emit(ring, 0);
		403	intel_ring_emit(ring, 0);
		404	intel_ring_emit(ring, 0);
		405	intel_ring_advance(ring);
		406
		407	return 0;
		408	}
		409
		410	static int
6084	serge	411	gen8_render_ring_flush(struct drm_i915_gem_request *req,
4560	Serge	412	u32 invalidate_domains, u32 flush_domains)
		413	{
		414	u32 flags = 0;
6084	serge	415	u32 scratch_addr = req->ring->scratch.gtt_offset + 2 * CACHELINE_BYTES;
4560	Serge	416	int ret;
		417
		418	flags \|= PIPE_CONTROL_CS_STALL;
		419
		420	if (flush_domains) {
		421	flags \|= PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH;
		422	flags \|= PIPE_CONTROL_DEPTH_CACHE_FLUSH;
6320	serge	423	flags \|= PIPE_CONTROL_DC_FLUSH_ENABLE;
6084	serge	424	flags \|= PIPE_CONTROL_FLUSH_ENABLE;
4560	Serge	425	}
		426	if (invalidate_domains) {
		427	flags \|= PIPE_CONTROL_TLB_INVALIDATE;
		428	flags \|= PIPE_CONTROL_INSTRUCTION_CACHE_INVALIDATE;
		429	flags \|= PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE;
		430	flags \|= PIPE_CONTROL_VF_CACHE_INVALIDATE;
		431	flags \|= PIPE_CONTROL_CONST_CACHE_INVALIDATE;
		432	flags \|= PIPE_CONTROL_STATE_CACHE_INVALIDATE;
		433	flags \|= PIPE_CONTROL_QW_WRITE;
		434	flags \|= PIPE_CONTROL_GLOBAL_GTT_IVB;
		435
5060	serge	436	/* WaCsStallBeforeStateCacheInvalidate:bdw,chv */
6084	serge	437	ret = gen8_emit_pipe_control(req,
5060	serge	438	PIPE_CONTROL_CS_STALL \|
		439	PIPE_CONTROL_STALL_AT_SCOREBOARD,
		440	0);
6084	serge	441	if (ret)
		442	return ret;
5060	serge	443	}
4560	Serge	444
6084	serge	445	return gen8_emit_pipe_control(req, flags, scratch_addr);
4560	Serge	446	}
		447
5060	serge	448	static void ring_write_tail(struct intel_engine_cs *ring,
2332	Serge	449	u32 value)
		450	{
5060	serge	451	struct drm_i915_private *dev_priv = ring->dev->dev_private;
2332	Serge	452	I915_WRITE_TAIL(ring, value);
		453	}
		454
5060	serge	455	u64 intel_ring_get_active_head(struct intel_engine_cs *ring)
2332	Serge	456	{
5060	serge	457	struct drm_i915_private *dev_priv = ring->dev->dev_private;
		458	u64 acthd;
2332	Serge	459
5060	serge	460	if (INTEL_INFO(ring->dev)->gen >= 8)
		461	acthd = I915_READ64_2x32(RING_ACTHD(ring->mmio_base),
		462	RING_ACTHD_UDW(ring->mmio_base));
		463	else if (INTEL_INFO(ring->dev)->gen >= 4)
		464	acthd = I915_READ(RING_ACTHD(ring->mmio_base));
		465	else
		466	acthd = I915_READ(ACTHD);
		467
		468	return acthd;
2332	Serge	469	}
		470
5060	serge	471	static void ring_setup_phys_status_page(struct intel_engine_cs *ring)
4104	Serge	472	{
		473	struct drm_i915_private *dev_priv = ring->dev->dev_private;
		474	u32 addr;
		475
		476	addr = dev_priv->status_page_dmah->busaddr;
		477	if (INTEL_INFO(ring->dev)->gen >= 4)
		478	addr \|= (dev_priv->status_page_dmah->busaddr >> 28) & 0xf0;
		479	I915_WRITE(HWS_PGA, addr);
		480	}
		481
6084	serge	482	static void intel_ring_setup_status_page(struct intel_engine_cs *ring)
		483	{
		484	struct drm_device *dev = ring->dev;
		485	struct drm_i915_private *dev_priv = ring->dev->dev_private;
		486	u32 mmio = 0;
		487
		488	/* The ring status page addresses are no longer next to the rest of
		489	* the ring registers as of gen7.
		490	*/
		491	if (IS_GEN7(dev)) {
		492	switch (ring->id) {
		493	case RCS:
		494	mmio = RENDER_HWS_PGA_GEN7;
		495	break;
		496	case BCS:
		497	mmio = BLT_HWS_PGA_GEN7;
		498	break;
		499	/*
		500	* VCS2 actually doesn't exist on Gen7. Only shut up
		501	* gcc switch check warning
		502	*/
		503	case VCS2:
		504	case VCS:
		505	mmio = BSD_HWS_PGA_GEN7;
		506	break;
		507	case VECS:
		508	mmio = VEBOX_HWS_PGA_GEN7;
		509	break;
		510	}
		511	} else if (IS_GEN6(ring->dev)) {
		512	mmio = RING_HWS_PGA_GEN6(ring->mmio_base);
		513	} else {
		514	/* XXX: gen8 returns to sanity */
		515	mmio = RING_HWS_PGA(ring->mmio_base);
		516	}
		517
		518	I915_WRITE(mmio, (u32)ring->status_page.gfx_addr);
		519	POSTING_READ(mmio);
		520
		521	/*
		522	* Flush the TLB for this page
		523	*
		524	* FIXME: These two bits have disappeared on gen8, so a question
		525	* arises: do we still need this and if so how should we go about
		526	* invalidating the TLB?
		527	*/
		528	if (INTEL_INFO(dev)->gen >= 6 && INTEL_INFO(dev)->gen < 8) {
		529	u32 reg = RING_INSTPM(ring->mmio_base);
		530
		531	/* ring should be idle before issuing a sync flush*/
		532	WARN_ON((I915_READ_MODE(ring) & MODE_IDLE) == 0);
		533
		534	I915_WRITE(reg,
		535	_MASKED_BIT_ENABLE(INSTPM_TLB_INVALIDATE \|
		536	INSTPM_SYNC_FLUSH));
		537	if (wait_for((I915_READ(reg) & INSTPM_SYNC_FLUSH) == 0,
		538	1000))
		539	DRM_ERROR("%s: wait for SyncFlush to complete for TLB invalidation timed out\n",
		540	ring->name);
		541	}
		542	}
		543
5060	serge	544	static bool stop_ring(struct intel_engine_cs *ring)
2332	Serge	545	{
5060	serge	546	struct drm_i915_private *dev_priv = to_i915(ring->dev);
2332	Serge	547
5060	serge	548	if (!IS_GEN2(ring->dev)) {
		549	I915_WRITE_MODE(ring, _MASKED_BIT_ENABLE(STOP_RING));
5354	serge	550	if (wait_for((I915_READ_MODE(ring) & MODE_IDLE) != 0, 1000)) {
		551	DRM_ERROR("%s : timed out trying to stop ring\n", ring->name);
		552	/* Sometimes we observe that the idle flag is not
		553	* set even though the ring is empty. So double
		554	* check before giving up.
		555	*/
		556	if (I915_READ_HEAD(ring) != I915_READ_TAIL(ring))
6084	serge	557	return false;
5060	serge	558	}
		559	}
3031	serge	560
2332	Serge	561	I915_WRITE_CTL(ring, 0);
		562	I915_WRITE_HEAD(ring, 0);
		563	ring->write_tail(ring, 0);
		564
5060	serge	565	if (!IS_GEN2(ring->dev)) {
		566	(void)I915_READ_CTL(ring);
		567	I915_WRITE_MODE(ring, _MASKED_BIT_DISABLE(STOP_RING));
		568	}
2332	Serge	569
5060	serge	570	return (I915_READ_HEAD(ring) & HEAD_ADDR) == 0;
		571	}
		572
		573	static int init_ring_common(struct intel_engine_cs *ring)
		574	{
		575	struct drm_device *dev = ring->dev;
		576	struct drm_i915_private *dev_priv = dev->dev_private;
		577	struct intel_ringbuffer *ringbuf = ring->buffer;
		578	struct drm_i915_gem_object *obj = ringbuf->obj;
		579	int ret = 0;
		580
6084	serge	581	intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
5060	serge	582
		583	if (!stop_ring(ring)) {
		584	/* G45 ring initialization often fails to reset head to zero */
2332	Serge	585	DRM_DEBUG_KMS("%s head not reset to zero "
		586	"ctl %08x head %08x tail %08x start %08x\n",
		587	ring->name,
		588	I915_READ_CTL(ring),
		589	I915_READ_HEAD(ring),
		590	I915_READ_TAIL(ring),
		591	I915_READ_START(ring));
		592
5060	serge	593	if (!stop_ring(ring)) {
2332	Serge	594	DRM_ERROR("failed to set %s head to zero "
		595	"ctl %08x head %08x tail %08x start %08x\n",
		596	ring->name,
		597	I915_READ_CTL(ring),
		598	I915_READ_HEAD(ring),
		599	I915_READ_TAIL(ring),
		600	I915_READ_START(ring));
5060	serge	601	ret = -EIO;
		602	goto out;
2332	Serge	603	}
		604	}
		605
5060	serge	606	if (I915_NEED_GFX_HWS(dev))
		607	intel_ring_setup_status_page(ring);
		608	else
		609	ring_setup_phys_status_page(ring);
		610
		611	/* Enforce ordering by reading HEAD register back */
		612	I915_READ_HEAD(ring);
		613
3031	serge	614	/* Initialize the ring. This must happen _after_ we've cleared the ring
		615	* registers with the above sequence (the readback of the HEAD registers
		616	* also enforces ordering), otherwise the hw might lose the new ring
		617	* register values. */
4104	Serge	618	I915_WRITE_START(ring, i915_gem_obj_ggtt_offset(obj));
5354	serge	619
		620	/* WaClearRingBufHeadRegAtInit:ctg,elk */
		621	if (I915_READ_HEAD(ring))
		622	DRM_DEBUG("%s initialization failed [head=%08x], fudging\n",
		623	ring->name, I915_READ_HEAD(ring));
		624	I915_WRITE_HEAD(ring, 0);
		625	(void)I915_READ_HEAD(ring);
		626
2332	Serge	627	I915_WRITE_CTL(ring,
5060	serge	628	((ringbuf->size - PAGE_SIZE) & RING_NR_PAGES)
3031	serge	629	\| RING_VALID);
2332	Serge	630
		631	/* If the head is still not zero, the ring is dead */
3031	serge	632	if (wait_for((I915_READ_CTL(ring) & RING_VALID) != 0 &&
4104	Serge	633	I915_READ_START(ring) == i915_gem_obj_ggtt_offset(obj) &&
3031	serge	634	(I915_READ_HEAD(ring) & HEAD_ADDR) == 0, 50)) {
2332	Serge	635	DRM_ERROR("%s initialization failed "
5060	serge	636	"ctl %08x (valid? %d) head %08x tail %08x start %08x [expected %08lx]\n",
6084	serge	637	ring->name,
5060	serge	638	I915_READ_CTL(ring), I915_READ_CTL(ring) & RING_VALID,
		639	I915_READ_HEAD(ring), I915_READ_TAIL(ring),
		640	I915_READ_START(ring), (unsigned long)i915_gem_obj_ggtt_offset(obj));
3031	serge	641	ret = -EIO;
		642	goto out;
2332	Serge	643	}
		644
6084	serge	645	ringbuf->last_retired_head = -1;
		646	ringbuf->head = I915_READ_HEAD(ring);
		647	ringbuf->tail = I915_READ_TAIL(ring) & TAIL_ADDR;
		648	intel_ring_update_space(ringbuf);
5060	serge	649
4104	Serge	650	memset(&ring->hangcheck, 0, sizeof(ring->hangcheck));
		651
3031	serge	652	out:
6084	serge	653	intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
2332	Serge	654
3031	serge	655	return ret;
2332	Serge	656	}
		657
5354	serge	658	void
		659	intel_fini_pipe_control(struct intel_engine_cs *ring)
2332	Serge	660	{
5354	serge	661	struct drm_device *dev = ring->dev;
		662
		663	if (ring->scratch.obj == NULL)
		664	return;
		665
		666	if (INTEL_INFO(dev)->gen >= 5) {
		667	kunmap(sg_page(ring->scratch.obj->pages->sgl));
		668	i915_gem_object_ggtt_unpin(ring->scratch.obj);
		669	}
		670
		671	drm_gem_object_unreference(&ring->scratch.obj->base);
		672	ring->scratch.obj = NULL;
		673	}
		674
		675	int
		676	intel_init_pipe_control(struct intel_engine_cs *ring)
		677	{
2332	Serge	678	int ret;
		679
6084	serge	680	WARN_ON(ring->scratch.obj);
2332	Serge	681
4104	Serge	682	ring->scratch.obj = i915_gem_alloc_object(ring->dev, 4096);
		683	if (ring->scratch.obj == NULL) {
2332	Serge	684	DRM_ERROR("Failed to allocate seqno page\n");
		685	ret = -ENOMEM;
		686	goto err;
		687	}
		688
5060	serge	689	ret = i915_gem_object_set_cache_level(ring->scratch.obj, I915_CACHE_LLC);
		690	if (ret)
		691	goto err_unref;
2332	Serge	692
5060	serge	693	ret = i915_gem_obj_ggtt_pin(ring->scratch.obj, 4096, 0);
2332	Serge	694	if (ret)
		695	goto err_unref;
		696
4104	Serge	697	ring->scratch.gtt_offset = i915_gem_obj_ggtt_offset(ring->scratch.obj);
5354	serge	698	ring->scratch.cpu_page = kmap(sg_page(ring->scratch.obj->pages->sgl));
4104	Serge	699	if (ring->scratch.cpu_page == NULL) {
		700	ret = -ENOMEM;
2332	Serge	701	goto err_unpin;
4104	Serge	702	}
2332	Serge	703
3480	Serge	704	DRM_DEBUG_DRIVER("%s pipe control offset: 0x%08x\n",
4104	Serge	705	ring->name, ring->scratch.gtt_offset);
2332	Serge	706	return 0;
		707
		708	err_unpin:
5060	serge	709	i915_gem_object_ggtt_unpin(ring->scratch.obj);
2332	Serge	710	err_unref:
4104	Serge	711	drm_gem_object_unreference(&ring->scratch.obj->base);
2332	Serge	712	err:
		713	return ret;
		714	}
		715
6084	serge	716	static int intel_ring_workarounds_emit(struct drm_i915_gem_request *req)
5354	serge	717	{
		718	int ret, i;
6084	serge	719	struct intel_engine_cs *ring = req->ring;
5354	serge	720	struct drm_device *dev = ring->dev;
		721	struct drm_i915_private *dev_priv = dev->dev_private;
		722	struct i915_workarounds *w = &dev_priv->workarounds;
		723
6084	serge	724	if (w->count == 0)
5354	serge	725	return 0;
		726
		727	ring->gpu_caches_dirty = true;
6084	serge	728	ret = intel_ring_flush_all_caches(req);
5354	serge	729	if (ret)
		730	return ret;
		731
6084	serge	732	ret = intel_ring_begin(req, (w->count * 2 + 2));
5354	serge	733	if (ret)
		734	return ret;
		735
		736	intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(w->count));
		737	for (i = 0; i < w->count; i++) {
		738	intel_ring_emit(ring, w->reg[i].addr);
		739	intel_ring_emit(ring, w->reg[i].value);
		740	}
		741	intel_ring_emit(ring, MI_NOOP);
		742
		743	intel_ring_advance(ring);
		744
		745	ring->gpu_caches_dirty = true;
6084	serge	746	ret = intel_ring_flush_all_caches(req);
5354	serge	747	if (ret)
		748	return ret;
		749
		750	DRM_DEBUG_DRIVER("Number of Workarounds emitted: %d\n", w->count);
		751
		752	return 0;
		753	}
		754
6084	serge	755	static int intel_rcs_ctx_init(struct drm_i915_gem_request *req)
		756	{
		757	int ret;
		758
		759	ret = intel_ring_workarounds_emit(req);
		760	if (ret != 0)
		761	return ret;
		762
		763	ret = i915_gem_render_state_init(req);
		764	if (ret)
		765	DRM_ERROR("init render state: %d\n", ret);
		766
		767	return ret;
		768	}
		769
5354	serge	770	static int wa_add(struct drm_i915_private *dev_priv,
		771	const u32 addr, const u32 mask, const u32 val)
		772	{
		773	const u32 idx = dev_priv->workarounds.count;
		774
		775	if (WARN_ON(idx >= I915_MAX_WA_REGS))
		776	return -ENOSPC;
		777
		778	dev_priv->workarounds.reg[idx].addr = addr;
		779	dev_priv->workarounds.reg[idx].value = val;
		780	dev_priv->workarounds.reg[idx].mask = mask;
		781
		782	dev_priv->workarounds.count++;
		783
		784	return 0;
		785	}
		786
6084	serge	787	#define WA_REG(addr, mask, val) do { \
5354	serge	788	const int r = wa_add(dev_priv, (addr), (mask), (val)); \
		789	if (r) \
		790	return r; \
6084	serge	791	} while (0)
5354	serge	792
		793	#define WA_SET_BIT_MASKED(addr, mask) \
		794	WA_REG(addr, (mask), _MASKED_BIT_ENABLE(mask))
		795
		796	#define WA_CLR_BIT_MASKED(addr, mask) \
		797	WA_REG(addr, (mask), _MASKED_BIT_DISABLE(mask))
		798
		799	#define WA_SET_FIELD_MASKED(addr, mask, value) \
		800	WA_REG(addr, mask, _MASKED_FIELD(mask, value))
		801
		802	#define WA_SET_BIT(addr, mask) WA_REG(addr, mask, I915_READ(addr) \| (mask))
		803	#define WA_CLR_BIT(addr, mask) WA_REG(addr, mask, I915_READ(addr) & ~(mask))
		804
		805	#define WA_WRITE(addr, val) WA_REG(addr, 0xffffffff, val)
		806
6084	serge	807	static int gen8_init_workarounds(struct intel_engine_cs *ring)
5354	serge	808	{
		809	struct drm_device *dev = ring->dev;
		810	struct drm_i915_private *dev_priv = dev->dev_private;
		811
6084	serge	812	WA_SET_BIT_MASKED(INSTPM, INSTPM_FORCE_ORDERING);
5354	serge	813
6084	serge	814	/* WaDisableAsyncFlipPerfMode:bdw,chv */
		815	WA_SET_BIT_MASKED(MI_MODE, ASYNC_FLIP_PERF_DISABLE);
5354	serge	816
6084	serge	817	/* WaDisablePartialInstShootdown:bdw,chv */
		818	WA_SET_BIT_MASKED(GEN8_ROW_CHICKEN,
		819	PARTIAL_INSTRUCTION_SHOOTDOWN_DISABLE);
5354	serge	820
		821	/* Use Force Non-Coherent whenever executing a 3D context. This is a
		822	* workaround for for a possible hang in the unlikely event a TLB
		823	* invalidation occurs during a PSD flush.
		824	*/
6084	serge	825	/* WaForceEnableNonCoherent:bdw,chv */
		826	/* WaHdcDisableFetchWhenMasked:bdw,chv */
5354	serge	827	WA_SET_BIT_MASKED(HDC_CHICKEN0,
6084	serge	828	HDC_DONOT_FETCH_MEM_WHEN_MASKED \|
		829	HDC_FORCE_NON_COHERENT);
5354	serge	830
6084	serge	831	/* From the Haswell PRM, Command Reference: Registers, CACHE_MODE_0:
		832	* "The Hierarchical Z RAW Stall Optimization allows non-overlapping
		833	* polygons in the same 8x4 pixel/sample area to be processed without
		834	* stalling waiting for the earlier ones to write to Hierarchical Z
		835	* buffer."
		836	*
		837	* This optimization is off by default for BDW and CHV; turn it on.
		838	*/
		839	WA_CLR_BIT_MASKED(CACHE_MODE_0_GEN7, HIZ_RAW_STALL_OPT_DISABLE);
5354	serge	840
6084	serge	841	/* Wa4x4STCOptimizationDisable:bdw,chv */
		842	WA_SET_BIT_MASKED(CACHE_MODE_1, GEN8_4x4_STC_OPTIMIZATION_DISABLE);
		843
5354	serge	844	/*
		845	* BSpec recommends 8x4 when MSAA is used,
		846	* however in practice 16x4 seems fastest.
		847	*
		848	* Note that PS/WM thread counts depend on the WIZ hashing
		849	* disable bit, which we don't touch here, but it's good
		850	* to keep in mind (see 3DSTATE_PS and 3DSTATE_WM).
		851	*/
		852	WA_SET_FIELD_MASKED(GEN7_GT_MODE,
		853	GEN6_WIZ_HASHING_MASK,
		854	GEN6_WIZ_HASHING_16x4);
		855
		856	return 0;
		857	}
		858
6084	serge	859	static int bdw_init_workarounds(struct intel_engine_cs *ring)
		860	{
		861	int ret;
		862	struct drm_device *dev = ring->dev;
		863	struct drm_i915_private *dev_priv = dev->dev_private;
		864
		865	ret = gen8_init_workarounds(ring);
		866	if (ret)
		867	return ret;
		868
		869	/* WaDisableThreadStallDopClockGating:bdw (pre-production) */
		870	WA_SET_BIT_MASKED(GEN8_ROW_CHICKEN, STALL_DOP_GATING_DISABLE);
		871
		872	/* WaDisableDopClockGating:bdw */
		873	WA_SET_BIT_MASKED(GEN7_ROW_CHICKEN2,
		874	DOP_CLOCK_GATING_DISABLE);
		875
		876	WA_SET_BIT_MASKED(HALF_SLICE_CHICKEN3,
		877	GEN8_SAMPLER_POWER_BYPASS_DIS);
		878
		879	WA_SET_BIT_MASKED(HDC_CHICKEN0,
		880	/* WaForceContextSaveRestoreNonCoherent:bdw */
		881	HDC_FORCE_CONTEXT_SAVE_RESTORE_NON_COHERENT \|
		882	/* WaDisableFenceDestinationToSLM:bdw (pre-prod) */
		883	(IS_BDW_GT3(dev) ? HDC_FENCE_DEST_SLM_DISABLE : 0));
		884
		885	return 0;
		886	}
		887
5354	serge	888	static int chv_init_workarounds(struct intel_engine_cs *ring)
		889	{
6084	serge	890	int ret;
5354	serge	891	struct drm_device *dev = ring->dev;
		892	struct drm_i915_private *dev_priv = dev->dev_private;
		893
6084	serge	894	ret = gen8_init_workarounds(ring);
		895	if (ret)
		896	return ret;
		897
5354	serge	898	/* WaDisableThreadStallDopClockGating:chv */
6084	serge	899	WA_SET_BIT_MASKED(GEN8_ROW_CHICKEN, STALL_DOP_GATING_DISABLE);
		900
		901	/* Improve HiZ throughput on CHV. */
		902	WA_SET_BIT_MASKED(HIZ_CHICKEN, CHV_HZ_8X8_MODE_IN_1X);
		903
		904	return 0;
		905	}
		906
		907	static int gen9_init_workarounds(struct intel_engine_cs *ring)
		908	{
		909	struct drm_device *dev = ring->dev;
		910	struct drm_i915_private *dev_priv = dev->dev_private;
		911	uint32_t tmp;
		912
		913	/* WaEnableLbsSlaRetryTimerDecrement:skl */
		914	I915_WRITE(BDW_SCRATCH1, I915_READ(BDW_SCRATCH1) \|
		915	GEN9_LBS_SLA_RETRY_TIMER_DECREMENT_ENABLE);
		916
		917	/* WaDisableKillLogic:bxt,skl */
		918	I915_WRITE(GAM_ECOCHK, I915_READ(GAM_ECOCHK) \|
		919	ECOCHK_DIS_TLB);
		920
		921	/* WaDisablePartialInstShootdown:skl,bxt */
5354	serge	922	WA_SET_BIT_MASKED(GEN8_ROW_CHICKEN,
6084	serge	923	PARTIAL_INSTRUCTION_SHOOTDOWN_DISABLE);
5354	serge	924
6084	serge	925	/* Syncing dependencies between camera and graphics:skl,bxt */
		926	WA_SET_BIT_MASKED(HALF_SLICE_CHICKEN3,
		927	GEN9_DISABLE_OCL_OOB_SUPPRESS_LOGIC);
		928
		929	if ((IS_SKYLAKE(dev) && (INTEL_REVID(dev) == SKL_REVID_A0 \|\|
		930	INTEL_REVID(dev) == SKL_REVID_B0)) \|\|
		931	(IS_BROXTON(dev) && INTEL_REVID(dev) < BXT_REVID_B0)) {
		932	/* WaDisableDgMirrorFixInHalfSliceChicken5:skl,bxt */
		933	WA_CLR_BIT_MASKED(GEN9_HALF_SLICE_CHICKEN5,
		934	GEN9_DG_MIRROR_FIX_ENABLE);
		935	}
		936
		937	if ((IS_SKYLAKE(dev) && INTEL_REVID(dev) <= SKL_REVID_B0) \|\|
		938	(IS_BROXTON(dev) && INTEL_REVID(dev) < BXT_REVID_B0)) {
		939	/* WaSetDisablePixMaskCammingAndRhwoInCommonSliceChicken:skl,bxt */
		940	WA_SET_BIT_MASKED(GEN7_COMMON_SLICE_CHICKEN1,
		941	GEN9_RHWO_OPTIMIZATION_DISABLE);
		942	/*
		943	* WA also requires GEN9_SLICE_COMMON_ECO_CHICKEN0[14:14] to be set
		944	* but we do that in per ctx batchbuffer as there is an issue
		945	* with this register not getting restored on ctx restore
		946	*/
		947	}
		948
		949	if ((IS_SKYLAKE(dev) && INTEL_REVID(dev) >= SKL_REVID_C0) \|\|
		950	IS_BROXTON(dev)) {
		951	/* WaEnableYV12BugFixInHalfSliceChicken7:skl,bxt */
		952	WA_SET_BIT_MASKED(GEN9_HALF_SLICE_CHICKEN7,
		953	GEN9_ENABLE_YV12_BUGFIX);
		954	}
		955
		956	/* Wa4x4STCOptimizationDisable:skl,bxt */
		957	/* WaDisablePartialResolveInVc:skl,bxt */
		958	WA_SET_BIT_MASKED(CACHE_MODE_1, (GEN8_4x4_STC_OPTIMIZATION_DISABLE \|
		959	GEN9_PARTIAL_RESOLVE_IN_VC_DISABLE));
		960
		961	/* WaCcsTlbPrefetchDisable:skl,bxt */
		962	WA_CLR_BIT_MASKED(GEN9_HALF_SLICE_CHICKEN5,
		963	GEN9_CCS_TLB_PREFETCH_ENABLE);
		964
		965	/* WaDisableMaskBasedCammingInRCC:skl,bxt */
		966	if ((IS_SKYLAKE(dev) && INTEL_REVID(dev) == SKL_REVID_C0) \|\|
		967	(IS_BROXTON(dev) && INTEL_REVID(dev) < BXT_REVID_B0))
		968	WA_SET_BIT_MASKED(SLICE_ECO_CHICKEN0,
		969	PIXEL_MASK_CAMMING_DISABLE);
		970
		971	/* WaForceContextSaveRestoreNonCoherent:skl,bxt */
		972	tmp = HDC_FORCE_CONTEXT_SAVE_RESTORE_NON_COHERENT;
		973	if ((IS_SKYLAKE(dev) && INTEL_REVID(dev) == SKL_REVID_F0) \|\|
		974	(IS_BROXTON(dev) && INTEL_REVID(dev) >= BXT_REVID_B0))
		975	tmp \|= HDC_FORCE_CSR_NON_COHERENT_OVR_DISABLE;
		976	WA_SET_BIT_MASKED(HDC_CHICKEN0, tmp);
		977
		978	/* WaDisableSamplerPowerBypassForSOPingPong:skl,bxt */
		979	if (IS_SKYLAKE(dev) \|\|
		980	(IS_BROXTON(dev) && INTEL_REVID(dev) <= BXT_REVID_B0)) {
		981	WA_SET_BIT_MASKED(HALF_SLICE_CHICKEN3,
		982	GEN8_SAMPLER_POWER_BYPASS_DIS);
		983	}
		984
		985	/* WaDisableSTUnitPowerOptimization:skl,bxt */
		986	WA_SET_BIT_MASKED(HALF_SLICE_CHICKEN2, GEN8_ST_PO_DISABLE);
		987
		988	return 0;
		989	}
		990
		991	static int skl_tune_iz_hashing(struct intel_engine_cs *ring)
		992	{
		993	struct drm_device *dev = ring->dev;
		994	struct drm_i915_private *dev_priv = dev->dev_private;
		995	u8 vals[3] = { 0, 0, 0 };
		996	unsigned int i;
		997
		998	for (i = 0; i < 3; i++) {
		999	u8 ss;
		1000
		1001	/*
		1002	* Only consider slices where one, and only one, subslice has 7
		1003	* EUs
		1004	*/
		1005	if (hweight8(dev_priv->info.subslice_7eu[i]) != 1)
		1006	continue;
		1007
		1008	/*
		1009	* subslice_7eu[i] != 0 (because of the check above) and
		1010	* ss_max == 4 (maximum number of subslices possible per slice)
		1011	*
		1012	* -> 0 <= ss <= 3;
		1013	*/
		1014	ss = ffs(dev_priv->info.subslice_7eu[i]) - 1;
		1015	vals[i] = 3 - ss;
		1016	}
		1017
		1018	if (vals[0] == 0 && vals[1] == 0 && vals[2] == 0)
		1019	return 0;
		1020
		1021	/* Tune IZ hashing. See intel_device_info_runtime_init() */
		1022	WA_SET_FIELD_MASKED(GEN7_GT_MODE,
		1023	GEN9_IZ_HASHING_MASK(2) \|
		1024	GEN9_IZ_HASHING_MASK(1) \|
		1025	GEN9_IZ_HASHING_MASK(0),
		1026	GEN9_IZ_HASHING(2, vals[2]) \|
		1027	GEN9_IZ_HASHING(1, vals[1]) \|
		1028	GEN9_IZ_HASHING(0, vals[0]));
		1029
		1030	return 0;
		1031	}
		1032
		1033	static int skl_init_workarounds(struct intel_engine_cs *ring)
		1034	{
		1035	int ret;
		1036	struct drm_device *dev = ring->dev;
		1037	struct drm_i915_private *dev_priv = dev->dev_private;
		1038
		1039	ret = gen9_init_workarounds(ring);
		1040	if (ret)
		1041	return ret;
		1042
		1043	if (INTEL_REVID(dev) <= SKL_REVID_D0) {
		1044	/* WaDisableHDCInvalidation:skl */
		1045	I915_WRITE(GAM_ECOCHK, I915_READ(GAM_ECOCHK) \|
		1046	BDW_DISABLE_HDC_INVALIDATION);
		1047
		1048	/* WaDisableChickenBitTSGBarrierAckForFFSliceCS:skl */
		1049	I915_WRITE(FF_SLICE_CS_CHICKEN2,
		1050	_MASKED_BIT_ENABLE(GEN9_TSG_BARRIER_ACK_DISABLE));
		1051	}
		1052
		1053	/* GEN8_L3SQCREG4 has a dependency with WA batch so any new changes
		1054	* involving this register should also be added to WA batch as required.
5354	serge	1055	*/
6084	serge	1056	if (INTEL_REVID(dev) <= SKL_REVID_E0)
		1057	/* WaDisableLSQCROPERFforOCL:skl */
		1058	I915_WRITE(GEN8_L3SQCREG4, I915_READ(GEN8_L3SQCREG4) \|
		1059	GEN8_LQSC_RO_PERF_DIS);
5354	serge	1060
6084	serge	1061	/* WaEnableGapsTsvCreditFix:skl */
		1062	if (IS_SKYLAKE(dev) && (INTEL_REVID(dev) >= SKL_REVID_C0)) {
		1063	I915_WRITE(GEN8_GARBCNTL, (I915_READ(GEN8_GARBCNTL) \|
		1064	GEN9_GAPS_TSV_CREDIT_DISABLE));
		1065	}
		1066
		1067	/* WaDisablePowerCompilerClockGating:skl */
		1068	if (INTEL_REVID(dev) == SKL_REVID_B0)
		1069	WA_SET_BIT_MASKED(HIZ_CHICKEN,
		1070	BDW_HIZ_POWER_COMPILER_CLOCK_GATING_DISABLE);
		1071
		1072	if (INTEL_REVID(dev) <= SKL_REVID_D0) {
		1073	/*
		1074	*Use Force Non-Coherent whenever executing a 3D context. This
		1075	* is a workaround for a possible hang in the unlikely event
		1076	* a TLB invalidation occurs during a PSD flush.
		1077	*/
		1078	/* WaForceEnableNonCoherent:skl */
		1079	WA_SET_BIT_MASKED(HDC_CHICKEN0,
		1080	HDC_FORCE_NON_COHERENT);
		1081	}
		1082
		1083	if (INTEL_REVID(dev) == SKL_REVID_C0 \|\|
		1084	INTEL_REVID(dev) == SKL_REVID_D0)
		1085	/* WaBarrierPerformanceFixDisable:skl */
		1086	WA_SET_BIT_MASKED(HDC_CHICKEN0,
		1087	HDC_FENCE_DEST_SLM_DISABLE \|
		1088	HDC_BARRIER_PERFORMANCE_DISABLE);
		1089
		1090	/* WaDisableSbeCacheDispatchPortSharing:skl */
		1091	if (INTEL_REVID(dev) <= SKL_REVID_F0) {
		1092	WA_SET_BIT_MASKED(
		1093	GEN7_HALF_SLICE_CHICKEN1,
		1094	GEN7_SBE_SS_CACHE_DISPATCH_PORT_SHARING_DISABLE);
		1095	}
		1096
		1097	return skl_tune_iz_hashing(ring);
		1098	}
		1099
		1100	static int bxt_init_workarounds(struct intel_engine_cs *ring)
		1101	{
		1102	int ret;
		1103	struct drm_device *dev = ring->dev;
		1104	struct drm_i915_private *dev_priv = dev->dev_private;
		1105
		1106	ret = gen9_init_workarounds(ring);
		1107	if (ret)
		1108	return ret;
		1109
		1110	/* WaStoreMultiplePTEenable:bxt */
		1111	/* This is a requirement according to Hardware specification */
		1112	if (INTEL_REVID(dev) == BXT_REVID_A0)
		1113	I915_WRITE(TILECTL, I915_READ(TILECTL) \| TILECTL_TLBPF);
		1114
		1115	/* WaSetClckGatingDisableMedia:bxt */
		1116	if (INTEL_REVID(dev) == BXT_REVID_A0) {
		1117	I915_WRITE(GEN7_MISCCPCTL, (I915_READ(GEN7_MISCCPCTL) &
		1118	~GEN8_DOP_CLOCK_GATE_MEDIA_ENABLE));
		1119	}
		1120
		1121	/* WaDisableThreadStallDopClockGating:bxt */
		1122	WA_SET_BIT_MASKED(GEN8_ROW_CHICKEN,
		1123	STALL_DOP_GATING_DISABLE);
		1124
		1125	/* WaDisableSbeCacheDispatchPortSharing:bxt */
		1126	if (INTEL_REVID(dev) <= BXT_REVID_B0) {
		1127	WA_SET_BIT_MASKED(
		1128	GEN7_HALF_SLICE_CHICKEN1,
		1129	GEN7_SBE_SS_CACHE_DISPATCH_PORT_SHARING_DISABLE);
		1130	}
		1131
5354	serge	1132	return 0;
		1133	}
		1134
		1135	int init_workarounds_ring(struct intel_engine_cs *ring)
		1136	{
		1137	struct drm_device *dev = ring->dev;
		1138	struct drm_i915_private *dev_priv = dev->dev_private;
		1139
		1140	WARN_ON(ring->id != RCS);
		1141
		1142	dev_priv->workarounds.count = 0;
		1143
		1144	if (IS_BROADWELL(dev))
		1145	return bdw_init_workarounds(ring);
		1146
		1147	if (IS_CHERRYVIEW(dev))
		1148	return chv_init_workarounds(ring);
		1149
6084	serge	1150	if (IS_SKYLAKE(dev))
		1151	return skl_init_workarounds(ring);
		1152
		1153	if (IS_BROXTON(dev))
		1154	return bxt_init_workarounds(ring);
		1155
5354	serge	1156	return 0;
		1157	}
		1158
5060	serge	1159	static int init_render_ring(struct intel_engine_cs *ring)
2332	Serge	1160	{
		1161	struct drm_device *dev = ring->dev;
		1162	struct drm_i915_private *dev_priv = dev->dev_private;
		1163	int ret = init_ring_common(ring);
5060	serge	1164	if (ret)
		1165	return ret;
2332	Serge	1166
5060	serge	1167	/* WaTimedSingleVertexDispatch:cl,bw,ctg,elk,ilk,snb */
		1168	if (INTEL_INFO(dev)->gen >= 4 && INTEL_INFO(dev)->gen < 7)
3031	serge	1169	I915_WRITE(MI_MODE, _MASKED_BIT_ENABLE(VS_TIMER_DISPATCH));
3243	Serge	1170
		1171	/* We need to disable the AsyncFlip performance optimisations in order
		1172	* to use MI_WAIT_FOR_EVENT within the CS. It should already be
		1173	* programmed to '1' on all products.
4104	Serge	1174	*
6084	serge	1175	* WaDisableAsyncFlipPerfMode:snb,ivb,hsw,vlv
3243	Serge	1176	*/
6084	serge	1177	if (INTEL_INFO(dev)->gen >= 6 && INTEL_INFO(dev)->gen < 8)
3243	Serge	1178	I915_WRITE(MI_MODE, _MASKED_BIT_ENABLE(ASYNC_FLIP_PERF_DISABLE));
		1179
		1180	/* Required for the hardware to program scanline values for waiting */
5060	serge	1181	/* WaEnableFlushTlbInvalidationMode:snb */
3243	Serge	1182	if (INTEL_INFO(dev)->gen == 6)
		1183	I915_WRITE(GFX_MODE,
5060	serge	1184	_MASKED_BIT_ENABLE(GFX_TLB_INVALIDATE_EXPLICIT));
3243	Serge	1185
5060	serge	1186	/* WaBCSVCSTlbInvalidationMode:ivb,vlv,hsw */
6084	serge	1187	if (IS_GEN7(dev))
		1188	I915_WRITE(GFX_MODE_GEN7,
5060	serge	1189	_MASKED_BIT_ENABLE(GFX_TLB_INVALIDATE_EXPLICIT) \|
6084	serge	1190	_MASKED_BIT_ENABLE(GFX_REPLAY_MODE));
2332	Serge	1191
3031	serge	1192	if (IS_GEN6(dev)) {
		1193	/* From the Sandybridge PRM, volume 1 part 3, page 24:
		1194	* "If this bit is set, STCunit will have LRA as replacement
		1195	* policy. [...] This bit must be reset. LRA replacement
		1196	* policy is not supported."
		1197	*/
		1198	I915_WRITE(CACHE_MODE_0,
		1199	_MASKED_BIT_DISABLE(CM0_STC_EVICT_DISABLE_LRA_SNB));
2342	Serge	1200	}
		1201
6084	serge	1202	if (INTEL_INFO(dev)->gen >= 6 && INTEL_INFO(dev)->gen < 8)
3031	serge	1203	I915_WRITE(INSTPM, _MASKED_BIT_ENABLE(INSTPM_FORCE_ORDERING));
		1204
4560	Serge	1205	if (HAS_L3_DPF(dev))
		1206	I915_WRITE_IMR(ring, ~GT_PARITY_ERROR(dev));
3031	serge	1207
5354	serge	1208	return init_workarounds_ring(ring);
2332	Serge	1209	}
		1210
5060	serge	1211	static void render_ring_cleanup(struct intel_engine_cs *ring)
2332	Serge	1212	{
3480	Serge	1213	struct drm_device *dev = ring->dev;
5128	serge	1214	struct drm_i915_private *dev_priv = dev->dev_private;
3480	Serge	1215
5128	serge	1216	if (dev_priv->semaphore_obj) {
		1217	i915_gem_object_ggtt_unpin(dev_priv->semaphore_obj);
		1218	drm_gem_object_unreference(&dev_priv->semaphore_obj->base);
		1219	dev_priv->semaphore_obj = NULL;
		1220	}
		1221
5354	serge	1222	intel_fini_pipe_control(ring);
2332	Serge	1223	}
		1224
6084	serge	1225	static int gen8_rcs_signal(struct drm_i915_gem_request *signaller_req,
5060	serge	1226	unsigned int num_dwords)
2332	Serge	1227	{
5060	serge	1228	#define MBOX_UPDATE_DWORDS 8
6084	serge	1229	struct intel_engine_cs *signaller = signaller_req->ring;
5060	serge	1230	struct drm_device *dev = signaller->dev;
		1231	struct drm_i915_private *dev_priv = dev->dev_private;
		1232	struct intel_engine_cs *waiter;
		1233	int i, ret, num_rings;
		1234
		1235	num_rings = hweight32(INTEL_INFO(dev)->ring_mask);
		1236	num_dwords += (num_rings-1) * MBOX_UPDATE_DWORDS;
		1237	#undef MBOX_UPDATE_DWORDS
		1238
6084	serge	1239	ret = intel_ring_begin(signaller_req, num_dwords);
5060	serge	1240	if (ret)
		1241	return ret;
		1242
		1243	for_each_ring(waiter, dev_priv, i) {
6084	serge	1244	u32 seqno;
5060	serge	1245	u64 gtt_offset = signaller->semaphore.signal_ggtt[i];
		1246	if (gtt_offset == MI_SEMAPHORE_SYNC_INVALID)
		1247	continue;
		1248
6084	serge	1249	seqno = i915_gem_request_get_seqno(signaller_req);
5060	serge	1250	intel_ring_emit(signaller, GFX_OP_PIPE_CONTROL(6));
		1251	intel_ring_emit(signaller, PIPE_CONTROL_GLOBAL_GTT_IVB \|
		1252	PIPE_CONTROL_QW_WRITE \|
		1253	PIPE_CONTROL_FLUSH_ENABLE);
		1254	intel_ring_emit(signaller, lower_32_bits(gtt_offset));
		1255	intel_ring_emit(signaller, upper_32_bits(gtt_offset));
6084	serge	1256	intel_ring_emit(signaller, seqno);
5060	serge	1257	intel_ring_emit(signaller, 0);
		1258	intel_ring_emit(signaller, MI_SEMAPHORE_SIGNAL \|
		1259	MI_SEMAPHORE_TARGET(waiter->id));
		1260	intel_ring_emit(signaller, 0);
		1261	}
		1262
		1263	return 0;
2332	Serge	1264	}
		1265
6084	serge	1266	static int gen8_xcs_signal(struct drm_i915_gem_request *signaller_req,
5060	serge	1267	unsigned int num_dwords)
		1268	{
		1269	#define MBOX_UPDATE_DWORDS 6
6084	serge	1270	struct intel_engine_cs *signaller = signaller_req->ring;
5060	serge	1271	struct drm_device *dev = signaller->dev;
		1272	struct drm_i915_private *dev_priv = dev->dev_private;
		1273	struct intel_engine_cs *waiter;
		1274	int i, ret, num_rings;
		1275
		1276	num_rings = hweight32(INTEL_INFO(dev)->ring_mask);
		1277	num_dwords += (num_rings-1) * MBOX_UPDATE_DWORDS;
		1278	#undef MBOX_UPDATE_DWORDS
		1279
6084	serge	1280	ret = intel_ring_begin(signaller_req, num_dwords);
5060	serge	1281	if (ret)
		1282	return ret;
		1283
		1284	for_each_ring(waiter, dev_priv, i) {
6084	serge	1285	u32 seqno;
5060	serge	1286	u64 gtt_offset = signaller->semaphore.signal_ggtt[i];
		1287	if (gtt_offset == MI_SEMAPHORE_SYNC_INVALID)
		1288	continue;
		1289
6084	serge	1290	seqno = i915_gem_request_get_seqno(signaller_req);
5060	serge	1291	intel_ring_emit(signaller, (MI_FLUSH_DW + 1) \|
		1292	MI_FLUSH_DW_OP_STOREDW);
		1293	intel_ring_emit(signaller, lower_32_bits(gtt_offset) \|
		1294	MI_FLUSH_DW_USE_GTT);
		1295	intel_ring_emit(signaller, upper_32_bits(gtt_offset));
6084	serge	1296	intel_ring_emit(signaller, seqno);
5060	serge	1297	intel_ring_emit(signaller, MI_SEMAPHORE_SIGNAL \|
		1298	MI_SEMAPHORE_TARGET(waiter->id));
		1299	intel_ring_emit(signaller, 0);
		1300	}
		1301
		1302	return 0;
		1303	}
		1304
6084	serge	1305	static int gen6_signal(struct drm_i915_gem_request *signaller_req,
5060	serge	1306	unsigned int num_dwords)
		1307	{
6084	serge	1308	struct intel_engine_cs *signaller = signaller_req->ring;
5060	serge	1309	struct drm_device *dev = signaller->dev;
		1310	struct drm_i915_private *dev_priv = dev->dev_private;
		1311	struct intel_engine_cs *useless;
		1312	int i, ret, num_rings;
		1313
		1314	#define MBOX_UPDATE_DWORDS 3
		1315	num_rings = hweight32(INTEL_INFO(dev)->ring_mask);
		1316	num_dwords += round_up((num_rings-1) * MBOX_UPDATE_DWORDS, 2);
		1317	#undef MBOX_UPDATE_DWORDS
		1318
6084	serge	1319	ret = intel_ring_begin(signaller_req, num_dwords);
5060	serge	1320	if (ret)
		1321	return ret;
		1322
		1323	for_each_ring(useless, dev_priv, i) {
		1324	u32 mbox_reg = signaller->semaphore.mbox.signal[i];
		1325	if (mbox_reg != GEN6_NOSYNC) {
6084	serge	1326	u32 seqno = i915_gem_request_get_seqno(signaller_req);
5060	serge	1327	intel_ring_emit(signaller, MI_LOAD_REGISTER_IMM(1));
		1328	intel_ring_emit(signaller, mbox_reg);
6084	serge	1329	intel_ring_emit(signaller, seqno);
5060	serge	1330	}
		1331	}
		1332
		1333	/* If num_dwords was rounded, make sure the tail pointer is correct */
		1334	if (num_rings % 2 == 0)
		1335	intel_ring_emit(signaller, MI_NOOP);
		1336
		1337	return 0;
		1338	}
		1339
2342	Serge	1340	/**
		1341	* gen6_add_request - Update the semaphore mailbox registers
		1342	*
6084	serge	1343	* @request - request to write to the ring
2342	Serge	1344	*
		1345	* Update the mailbox registers in the other rings with the current seqno.
		1346	* This acts like a signal in the canonical semaphore.
		1347	*/
2332	Serge	1348	static int
6084	serge	1349	gen6_add_request(struct drm_i915_gem_request *req)
2332	Serge	1350	{
6084	serge	1351	struct intel_engine_cs *ring = req->ring;
5060	serge	1352	int ret;
2332	Serge	1353
5060	serge	1354	if (ring->semaphore.signal)
6084	serge	1355	ret = ring->semaphore.signal(req, 4);
5060	serge	1356	else
6084	serge	1357	ret = intel_ring_begin(req, 4);
4560	Serge	1358
2332	Serge	1359	if (ret)
		1360	return ret;
		1361
		1362	intel_ring_emit(ring, MI_STORE_DWORD_INDEX);
		1363	intel_ring_emit(ring, I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
6084	serge	1364	intel_ring_emit(ring, i915_gem_request_get_seqno(req));
2332	Serge	1365	intel_ring_emit(ring, MI_USER_INTERRUPT);
4560	Serge	1366	__intel_ring_advance(ring);
2332	Serge	1367
		1368	return 0;
		1369	}
		1370
3480	Serge	1371	static inline bool i915_gem_has_seqno_wrapped(struct drm_device *dev,
		1372	u32 seqno)
		1373	{
		1374	struct drm_i915_private *dev_priv = dev->dev_private;
		1375	return dev_priv->last_seqno < seqno;
		1376	}
		1377
2342	Serge	1378	/**
		1379	* intel_ring_sync - sync the waiter to the signaller on seqno
		1380	*
		1381	* @waiter - ring that is waiting
		1382	* @signaller - ring which has, or will signal
		1383	* @seqno - seqno which the waiter will block on
		1384	*/
5060	serge	1385
2342	Serge	1386	static int
6084	serge	1387	gen8_ring_sync(struct drm_i915_gem_request *waiter_req,
5060	serge	1388	struct intel_engine_cs *signaller,
		1389	u32 seqno)
		1390	{
6084	serge	1391	struct intel_engine_cs *waiter = waiter_req->ring;
5060	serge	1392	struct drm_i915_private *dev_priv = waiter->dev->dev_private;
		1393	int ret;
		1394
6084	serge	1395	ret = intel_ring_begin(waiter_req, 4);
5060	serge	1396	if (ret)
		1397	return ret;
		1398
		1399	intel_ring_emit(waiter, MI_SEMAPHORE_WAIT \|
		1400	MI_SEMAPHORE_GLOBAL_GTT \|
		1401	MI_SEMAPHORE_POLL \|
		1402	MI_SEMAPHORE_SAD_GTE_SDD);
		1403	intel_ring_emit(waiter, seqno);
		1404	intel_ring_emit(waiter,
		1405	lower_32_bits(GEN8_WAIT_OFFSET(waiter, signaller->id)));
		1406	intel_ring_emit(waiter,
		1407	upper_32_bits(GEN8_WAIT_OFFSET(waiter, signaller->id)));
		1408	intel_ring_advance(waiter);
		1409	return 0;
		1410	}
		1411
		1412	static int
6084	serge	1413	gen6_ring_sync(struct drm_i915_gem_request *waiter_req,
5060	serge	1414	struct intel_engine_cs *signaller,
6084	serge	1415	u32 seqno)
2332	Serge	1416	{
6084	serge	1417	struct intel_engine_cs *waiter = waiter_req->ring;
2342	Serge	1418	u32 dw1 = MI_SEMAPHORE_MBOX \|
		1419	MI_SEMAPHORE_COMPARE \|
		1420	MI_SEMAPHORE_REGISTER;
5060	serge	1421	u32 wait_mbox = signaller->semaphore.mbox.wait[waiter->id];
		1422	int ret;
2332	Serge	1423
3031	serge	1424	/* Throughout all of the GEM code, seqno passed implies our current
		1425	* seqno is >= the last seqno executed. However for hardware the
		1426	* comparison is strictly greater than.
		1427	*/
		1428	seqno -= 1;
		1429
5060	serge	1430	WARN_ON(wait_mbox == MI_SEMAPHORE_SYNC_INVALID);
3031	serge	1431
6084	serge	1432	ret = intel_ring_begin(waiter_req, 4);
2332	Serge	1433	if (ret)
		1434	return ret;
		1435
3480	Serge	1436	/* If seqno wrap happened, omit the wait with no-ops */
		1437	if (likely(!i915_gem_has_seqno_wrapped(waiter->dev, seqno))) {
5060	serge	1438	intel_ring_emit(waiter, dw1 \| wait_mbox);
6084	serge	1439	intel_ring_emit(waiter, seqno);
		1440	intel_ring_emit(waiter, 0);
		1441	intel_ring_emit(waiter, MI_NOOP);
3480	Serge	1442	} else {
		1443	intel_ring_emit(waiter, MI_NOOP);
		1444	intel_ring_emit(waiter, MI_NOOP);
		1445	intel_ring_emit(waiter, MI_NOOP);
		1446	intel_ring_emit(waiter, MI_NOOP);
		1447	}
2342	Serge	1448	intel_ring_advance(waiter);
2332	Serge	1449
		1450	return 0;
		1451	}
		1452
		1453	#define PIPE_CONTROL_FLUSH(ring__, addr__) \
		1454	do { \
2342	Serge	1455	intel_ring_emit(ring__, GFX_OP_PIPE_CONTROL(4) \| PIPE_CONTROL_QW_WRITE \| \
		1456	PIPE_CONTROL_DEPTH_STALL); \
2332	Serge	1457	intel_ring_emit(ring__, (addr__) \| PIPE_CONTROL_GLOBAL_GTT); \
		1458	intel_ring_emit(ring__, 0); \
		1459	intel_ring_emit(ring__, 0); \
		1460	} while (0)
		1461
		1462	static int
6084	serge	1463	pc_render_add_request(struct drm_i915_gem_request *req)
2332	Serge	1464	{
6084	serge	1465	struct intel_engine_cs *ring = req->ring;
5060	serge	1466	u32 scratch_addr = ring->scratch.gtt_offset + 2 * CACHELINE_BYTES;
2332	Serge	1467	int ret;
		1468
		1469	/* For Ironlake, MI_USER_INTERRUPT was deprecated and apparently
		1470	* incoherent with writes to memory, i.e. completely fubar,
		1471	* so we need to use PIPE_NOTIFY instead.
		1472	*
		1473	* However, we also need to workaround the qword write
		1474	* incoherence by flushing the 6 PIPE_NOTIFY buffers out to
		1475	* memory before requesting an interrupt.
		1476	*/
6084	serge	1477	ret = intel_ring_begin(req, 32);
2332	Serge	1478	if (ret)
		1479	return ret;
		1480
2342	Serge	1481	intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(4) \| PIPE_CONTROL_QW_WRITE \|
		1482	PIPE_CONTROL_WRITE_FLUSH \|
		1483	PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE);
4104	Serge	1484	intel_ring_emit(ring, ring->scratch.gtt_offset \| PIPE_CONTROL_GLOBAL_GTT);
6084	serge	1485	intel_ring_emit(ring, i915_gem_request_get_seqno(req));
2332	Serge	1486	intel_ring_emit(ring, 0);
		1487	PIPE_CONTROL_FLUSH(ring, scratch_addr);
5060	serge	1488	scratch_addr += 2 * CACHELINE_BYTES; /* write to separate cachelines */
2332	Serge	1489	PIPE_CONTROL_FLUSH(ring, scratch_addr);
5060	serge	1490	scratch_addr += 2 * CACHELINE_BYTES;
2332	Serge	1491	PIPE_CONTROL_FLUSH(ring, scratch_addr);
5060	serge	1492	scratch_addr += 2 * CACHELINE_BYTES;
2332	Serge	1493	PIPE_CONTROL_FLUSH(ring, scratch_addr);
5060	serge	1494	scratch_addr += 2 * CACHELINE_BYTES;
2332	Serge	1495	PIPE_CONTROL_FLUSH(ring, scratch_addr);
5060	serge	1496	scratch_addr += 2 * CACHELINE_BYTES;
2332	Serge	1497	PIPE_CONTROL_FLUSH(ring, scratch_addr);
3031	serge	1498
2342	Serge	1499	intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(4) \| PIPE_CONTROL_QW_WRITE \|
		1500	PIPE_CONTROL_WRITE_FLUSH \|
		1501	PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE \|
2332	Serge	1502	PIPE_CONTROL_NOTIFY);
4104	Serge	1503	intel_ring_emit(ring, ring->scratch.gtt_offset \| PIPE_CONTROL_GLOBAL_GTT);
6084	serge	1504	intel_ring_emit(ring, i915_gem_request_get_seqno(req));
2332	Serge	1505	intel_ring_emit(ring, 0);
4560	Serge	1506	__intel_ring_advance(ring);
2332	Serge	1507
		1508	return 0;
		1509	}
		1510
		1511	static u32
5060	serge	1512	gen6_ring_get_seqno(struct intel_engine_cs *ring, bool lazy_coherency)
2342	Serge	1513	{
		1514	/* Workaround to force correct ordering between irq and seqno writes on
		1515	* ivb (and maybe also on snb) by reading from a CS register (like
		1516	* ACTHD) before reading the status page. */
5060	serge	1517	if (!lazy_coherency) {
		1518	struct drm_i915_private *dev_priv = ring->dev->dev_private;
		1519	POSTING_READ(RING_ACTHD(ring->mmio_base));
		1520	}
		1521
2342	Serge	1522	return intel_read_status_page(ring, I915_GEM_HWS_INDEX);
		1523	}
		1524
		1525	static u32
5060	serge	1526	ring_get_seqno(struct intel_engine_cs *ring, bool lazy_coherency)
2332	Serge	1527	{
		1528	return intel_read_status_page(ring, I915_GEM_HWS_INDEX);
		1529	}
		1530
3480	Serge	1531	static void
5060	serge	1532	ring_set_seqno(struct intel_engine_cs *ring, u32 seqno)
3480	Serge	1533	{
		1534	intel_write_status_page(ring, I915_GEM_HWS_INDEX, seqno);
		1535	}
		1536
2332	Serge	1537	static u32
5060	serge	1538	pc_render_get_seqno(struct intel_engine_cs *ring, bool lazy_coherency)
2332	Serge	1539	{
4104	Serge	1540	return ring->scratch.cpu_page[0];
2332	Serge	1541	}
		1542
3480	Serge	1543	static void
5060	serge	1544	pc_render_set_seqno(struct intel_engine_cs *ring, u32 seqno)
3480	Serge	1545	{
4104	Serge	1546	ring->scratch.cpu_page[0] = seqno;
3480	Serge	1547	}
		1548
3031	serge	1549	static bool
5060	serge	1550	gen5_ring_get_irq(struct intel_engine_cs *ring)
2332	Serge	1551	{
3031	serge	1552	struct drm_device *dev = ring->dev;
5060	serge	1553	struct drm_i915_private *dev_priv = dev->dev_private;
3031	serge	1554	unsigned long flags;
		1555
5354	serge	1556	if (WARN_ON(!intel_irqs_enabled(dev_priv)))
3031	serge	1557	return false;
		1558
		1559	spin_lock_irqsave(&dev_priv->irq_lock, flags);
4104	Serge	1560	if (ring->irq_refcount++ == 0)
5060	serge	1561	gen5_enable_gt_irq(dev_priv, ring->irq_enable_mask);
3031	serge	1562	spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
		1563
		1564	return true;
2332	Serge	1565	}
		1566
		1567	static void
5060	serge	1568	gen5_ring_put_irq(struct intel_engine_cs *ring)
2332	Serge	1569	{
3031	serge	1570	struct drm_device *dev = ring->dev;
5060	serge	1571	struct drm_i915_private *dev_priv = dev->dev_private;
3031	serge	1572	unsigned long flags;
		1573
		1574	spin_lock_irqsave(&dev_priv->irq_lock, flags);
4104	Serge	1575	if (--ring->irq_refcount == 0)
5060	serge	1576	gen5_disable_gt_irq(dev_priv, ring->irq_enable_mask);
3031	serge	1577	spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
2332	Serge	1578	}
		1579
3031	serge	1580	static bool
5060	serge	1581	i9xx_ring_get_irq(struct intel_engine_cs *ring)
2332	Serge	1582	{
3031	serge	1583	struct drm_device *dev = ring->dev;
5060	serge	1584	struct drm_i915_private *dev_priv = dev->dev_private;
3031	serge	1585	unsigned long flags;
		1586
5354	serge	1587	if (!intel_irqs_enabled(dev_priv))
3031	serge	1588	return false;
		1589
		1590	spin_lock_irqsave(&dev_priv->irq_lock, flags);
		1591	if (ring->irq_refcount++ == 0) {
		1592	dev_priv->irq_mask &= ~ring->irq_enable_mask;
6084	serge	1593	I915_WRITE(IMR, dev_priv->irq_mask);
		1594	POSTING_READ(IMR);
3031	serge	1595	}
		1596	spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
		1597
		1598	return true;
2332	Serge	1599	}
		1600
		1601	static void
5060	serge	1602	i9xx_ring_put_irq(struct intel_engine_cs *ring)
2332	Serge	1603	{
3031	serge	1604	struct drm_device *dev = ring->dev;
5060	serge	1605	struct drm_i915_private *dev_priv = dev->dev_private;
3031	serge	1606	unsigned long flags;
		1607
		1608	spin_lock_irqsave(&dev_priv->irq_lock, flags);
		1609	if (--ring->irq_refcount == 0) {
		1610	dev_priv->irq_mask \|= ring->irq_enable_mask;
6084	serge	1611	I915_WRITE(IMR, dev_priv->irq_mask);
		1612	POSTING_READ(IMR);
3031	serge	1613	}
		1614	spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
2332	Serge	1615	}
		1616
		1617	static bool
5060	serge	1618	i8xx_ring_get_irq(struct intel_engine_cs *ring)
2332	Serge	1619	{
		1620	struct drm_device *dev = ring->dev;
5060	serge	1621	struct drm_i915_private *dev_priv = dev->dev_private;
3031	serge	1622	unsigned long flags;
2332	Serge	1623
5354	serge	1624	if (!intel_irqs_enabled(dev_priv))
2332	Serge	1625	return false;
		1626
3031	serge	1627	spin_lock_irqsave(&dev_priv->irq_lock, flags);
2332	Serge	1628	if (ring->irq_refcount++ == 0) {
3031	serge	1629	dev_priv->irq_mask &= ~ring->irq_enable_mask;
		1630	I915_WRITE16(IMR, dev_priv->irq_mask);
		1631	POSTING_READ16(IMR);
2332	Serge	1632	}
3031	serge	1633	spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
2332	Serge	1634
		1635	return true;
		1636	}
		1637
		1638	static void
5060	serge	1639	i8xx_ring_put_irq(struct intel_engine_cs *ring)
2332	Serge	1640	{
		1641	struct drm_device *dev = ring->dev;
5060	serge	1642	struct drm_i915_private *dev_priv = dev->dev_private;
3031	serge	1643	unsigned long flags;
2332	Serge	1644
3031	serge	1645	spin_lock_irqsave(&dev_priv->irq_lock, flags);
2332	Serge	1646	if (--ring->irq_refcount == 0) {
3031	serge	1647	dev_priv->irq_mask \|= ring->irq_enable_mask;
		1648	I915_WRITE16(IMR, dev_priv->irq_mask);
		1649	POSTING_READ16(IMR);
2332	Serge	1650	}
3031	serge	1651	spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
2332	Serge	1652	}
		1653
		1654	static int
6084	serge	1655	bsd_ring_flush(struct drm_i915_gem_request *req,
2332	Serge	1656	u32 invalidate_domains,
		1657	u32 flush_domains)
		1658	{
6084	serge	1659	struct intel_engine_cs *ring = req->ring;
2332	Serge	1660	int ret;
		1661
6084	serge	1662	ret = intel_ring_begin(req, 2);
2332	Serge	1663	if (ret)
		1664	return ret;
		1665
		1666	intel_ring_emit(ring, MI_FLUSH);
		1667	intel_ring_emit(ring, MI_NOOP);
		1668	intel_ring_advance(ring);
		1669	return 0;
		1670	}
		1671
		1672	static int
6084	serge	1673	i9xx_add_request(struct drm_i915_gem_request *req)
2332	Serge	1674	{
6084	serge	1675	struct intel_engine_cs *ring = req->ring;
2332	Serge	1676	int ret;
		1677
6084	serge	1678	ret = intel_ring_begin(req, 4);
2332	Serge	1679	if (ret)
		1680	return ret;
		1681
		1682	intel_ring_emit(ring, MI_STORE_DWORD_INDEX);
		1683	intel_ring_emit(ring, I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
6084	serge	1684	intel_ring_emit(ring, i915_gem_request_get_seqno(req));
2332	Serge	1685	intel_ring_emit(ring, MI_USER_INTERRUPT);
4560	Serge	1686	__intel_ring_advance(ring);
2332	Serge	1687
		1688	return 0;
		1689	}
		1690
		1691	static bool
5060	serge	1692	gen6_ring_get_irq(struct intel_engine_cs *ring)
2332	Serge	1693	{
		1694	struct drm_device *dev = ring->dev;
5060	serge	1695	struct drm_i915_private *dev_priv = dev->dev_private;
3031	serge	1696	unsigned long flags;
2332	Serge	1697
5354	serge	1698	if (WARN_ON(!intel_irqs_enabled(dev_priv)))
6084	serge	1699	return false;
2332	Serge	1700
3031	serge	1701	spin_lock_irqsave(&dev_priv->irq_lock, flags);
2332	Serge	1702	if (ring->irq_refcount++ == 0) {
4560	Serge	1703	if (HAS_L3_DPF(dev) && ring->id == RCS)
4104	Serge	1704	I915_WRITE_IMR(ring,
		1705	~(ring->irq_enable_mask \|
4560	Serge	1706	GT_PARITY_ERROR(dev)));
3031	serge	1707	else
		1708	I915_WRITE_IMR(ring, ~ring->irq_enable_mask);
5060	serge	1709	gen5_enable_gt_irq(dev_priv, ring->irq_enable_mask);
2332	Serge	1710	}
3031	serge	1711	spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
2332	Serge	1712
6084	serge	1713	return true;
2332	Serge	1714	}
		1715
		1716	static void
5060	serge	1717	gen6_ring_put_irq(struct intel_engine_cs *ring)
2332	Serge	1718	{
		1719	struct drm_device *dev = ring->dev;
5060	serge	1720	struct drm_i915_private *dev_priv = dev->dev_private;
3031	serge	1721	unsigned long flags;
2332	Serge	1722
3031	serge	1723	spin_lock_irqsave(&dev_priv->irq_lock, flags);
2332	Serge	1724	if (--ring->irq_refcount == 0) {
4560	Serge	1725	if (HAS_L3_DPF(dev) && ring->id == RCS)
		1726	I915_WRITE_IMR(ring, ~GT_PARITY_ERROR(dev));
3031	serge	1727	else
		1728	I915_WRITE_IMR(ring, ~0);
5060	serge	1729	gen5_disable_gt_irq(dev_priv, ring->irq_enable_mask);
2332	Serge	1730	}
3031	serge	1731	spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
2332	Serge	1732	}
		1733
4104	Serge	1734	static bool
5060	serge	1735	hsw_vebox_get_irq(struct intel_engine_cs *ring)
4104	Serge	1736	{
		1737	struct drm_device *dev = ring->dev;
		1738	struct drm_i915_private *dev_priv = dev->dev_private;
		1739	unsigned long flags;
		1740
5354	serge	1741	if (WARN_ON(!intel_irqs_enabled(dev_priv)))
4104	Serge	1742	return false;
		1743
		1744	spin_lock_irqsave(&dev_priv->irq_lock, flags);
		1745	if (ring->irq_refcount++ == 0) {
		1746	I915_WRITE_IMR(ring, ~ring->irq_enable_mask);
5060	serge	1747	gen6_enable_pm_irq(dev_priv, ring->irq_enable_mask);
4104	Serge	1748	}
		1749	spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
		1750
		1751	return true;
		1752	}
		1753
		1754	static void
5060	serge	1755	hsw_vebox_put_irq(struct intel_engine_cs *ring)
4104	Serge	1756	{
		1757	struct drm_device *dev = ring->dev;
		1758	struct drm_i915_private *dev_priv = dev->dev_private;
		1759	unsigned long flags;
		1760
		1761	spin_lock_irqsave(&dev_priv->irq_lock, flags);
		1762	if (--ring->irq_refcount == 0) {
		1763	I915_WRITE_IMR(ring, ~0);
5060	serge	1764	gen6_disable_pm_irq(dev_priv, ring->irq_enable_mask);
4104	Serge	1765	}
		1766	spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
		1767	}
		1768
4560	Serge	1769	static bool
5060	serge	1770	gen8_ring_get_irq(struct intel_engine_cs *ring)
4560	Serge	1771	{
		1772	struct drm_device *dev = ring->dev;
		1773	struct drm_i915_private *dev_priv = dev->dev_private;
		1774	unsigned long flags;
		1775
5354	serge	1776	if (WARN_ON(!intel_irqs_enabled(dev_priv)))
4560	Serge	1777	return false;
		1778
		1779	spin_lock_irqsave(&dev_priv->irq_lock, flags);
		1780	if (ring->irq_refcount++ == 0) {
		1781	if (HAS_L3_DPF(dev) && ring->id == RCS) {
		1782	I915_WRITE_IMR(ring,
		1783	~(ring->irq_enable_mask \|
		1784	GT_RENDER_L3_PARITY_ERROR_INTERRUPT));
		1785	} else {
		1786	I915_WRITE_IMR(ring, ~ring->irq_enable_mask);
		1787	}
		1788	POSTING_READ(RING_IMR(ring->mmio_base));
		1789	}
		1790	spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
		1791
		1792	return true;
		1793	}
		1794
		1795	static void
5060	serge	1796	gen8_ring_put_irq(struct intel_engine_cs *ring)
4560	Serge	1797	{
		1798	struct drm_device *dev = ring->dev;
		1799	struct drm_i915_private *dev_priv = dev->dev_private;
		1800	unsigned long flags;
		1801
		1802	spin_lock_irqsave(&dev_priv->irq_lock, flags);
		1803	if (--ring->irq_refcount == 0) {
		1804	if (HAS_L3_DPF(dev) && ring->id == RCS) {
		1805	I915_WRITE_IMR(ring,
		1806	~GT_RENDER_L3_PARITY_ERROR_INTERRUPT);
		1807	} else {
		1808	I915_WRITE_IMR(ring, ~0);
		1809	}
		1810	POSTING_READ(RING_IMR(ring->mmio_base));
		1811	}
		1812	spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
		1813	}
		1814
2332	Serge	1815	static int
6084	serge	1816	i965_dispatch_execbuffer(struct drm_i915_gem_request *req,
5060	serge	1817	u64 offset, u32 length,
6084	serge	1818	unsigned dispatch_flags)
2332	Serge	1819	{
6084	serge	1820	struct intel_engine_cs *ring = req->ring;
2332	Serge	1821	int ret;
		1822
6084	serge	1823	ret = intel_ring_begin(req, 2);
2332	Serge	1824	if (ret)
		1825	return ret;
		1826
		1827	intel_ring_emit(ring,
3031	serge	1828	MI_BATCH_BUFFER_START \|
		1829	MI_BATCH_GTT \|
6084	serge	1830	(dispatch_flags & I915_DISPATCH_SECURE ?
		1831
2332	Serge	1832	intel_ring_emit(ring, offset);
		1833	intel_ring_advance(ring);
		1834
		1835	return 0;
		1836	}
		1837
3243	Serge	1838	/* Just userspace ABI convention to limit the wa batch bo to a resonable size */
		1839	#define I830_BATCH_LIMIT (256*1024)
5128	serge	1840	#define I830_TLB_ENTRIES (2)
		1841	#define I830_WA_SIZE max(I830_TLB_ENTRIES*4096, I830_BATCH_LIMIT)
2332	Serge	1842	static int
6084	serge	1843	i830_dispatch_execbuffer(struct drm_i915_gem_request *req,
		1844	u64 offset, u32 len,
		1845	unsigned dispatch_flags)
2332	Serge	1846	{
6084	serge	1847	struct intel_engine_cs *ring = req->ring;
5128	serge	1848	u32 cs_offset = ring->scratch.gtt_offset;
2332	Serge	1849	int ret;
		1850
6084	serge	1851	ret = intel_ring_begin(req, 6);
		1852	if (ret)
		1853	return ret;
2332	Serge	1854
5128	serge	1855	/* Evict the invalid PTE TLBs */
		1856	intel_ring_emit(ring, COLOR_BLT_CMD \| BLT_WRITE_RGBA);
		1857	intel_ring_emit(ring, BLT_DEPTH_32 \| BLT_ROP_COLOR_COPY \| 4096);
		1858	intel_ring_emit(ring, I830_TLB_ENTRIES << 16 \| 4); /* load each page */
		1859	intel_ring_emit(ring, cs_offset);
		1860	intel_ring_emit(ring, 0xdeadbeef);
6084	serge	1861	intel_ring_emit(ring, MI_NOOP);
		1862	intel_ring_advance(ring);
3243	Serge	1863
6084	serge	1864	if ((dispatch_flags & I915_DISPATCH_PINNED) == 0) {
3243	Serge	1865	if (len > I830_BATCH_LIMIT)
		1866	return -ENOSPC;
		1867
6084	serge	1868	ret = intel_ring_begin(req, 6 + 2);
3243	Serge	1869	if (ret)
		1870	return ret;
5128	serge	1871
		1872	/* Blit the batch (which has now all relocs applied) to the
		1873	* stable batch scratch bo area (so that the CS never
		1874	* stumbles over its tlb invalidation bug) ...
		1875	*/
		1876	intel_ring_emit(ring, SRC_COPY_BLT_CMD \| BLT_WRITE_RGBA);
		1877	intel_ring_emit(ring, BLT_DEPTH_32 \| BLT_ROP_SRC_COPY \| 4096);
5139	serge	1878	intel_ring_emit(ring, DIV_ROUND_UP(len, 4096) << 16 \| 4096);
3243	Serge	1879	intel_ring_emit(ring, cs_offset);
		1880	intel_ring_emit(ring, 4096);
		1881	intel_ring_emit(ring, offset);
5128	serge	1882
3243	Serge	1883	intel_ring_emit(ring, MI_FLUSH);
5128	serge	1884	intel_ring_emit(ring, MI_NOOP);
		1885	intel_ring_advance(ring);
3243	Serge	1886
		1887	/* ... and execute it. */
5128	serge	1888	offset = cs_offset;
		1889	}
		1890
6084	serge	1891	ret = intel_ring_begin(req, 4);
5128	serge	1892	if (ret)
		1893	return ret;
		1894
6084	serge	1895	intel_ring_emit(ring, MI_BATCH_BUFFER);
		1896	intel_ring_emit(ring, offset \| (dispatch_flags & I915_DISPATCH_SECURE ?
		1897
5128	serge	1898	intel_ring_emit(ring, offset + len - 8);
		1899	intel_ring_emit(ring, MI_NOOP);
3031	serge	1900	intel_ring_advance(ring);
		1901
		1902	return 0;
		1903	}
		1904
		1905	static int
6084	serge	1906	i915_dispatch_execbuffer(struct drm_i915_gem_request *req,
5060	serge	1907	u64 offset, u32 len,
6084	serge	1908	unsigned dispatch_flags)
3031	serge	1909	{
6084	serge	1910	struct intel_engine_cs *ring = req->ring;
3031	serge	1911	int ret;
		1912
6084	serge	1913	ret = intel_ring_begin(req, 2);
		1914	if (ret)
		1915	return ret;
2332	Serge	1916
3031	serge	1917	intel_ring_emit(ring, MI_BATCH_BUFFER_START \| MI_BATCH_GTT);
6084	serge	1918	intel_ring_emit(ring, offset \| (dispatch_flags & I915_DISPATCH_SECURE ?
		1919
2332	Serge	1920	intel_ring_advance(ring);
		1921
		1922	return 0;
		1923	}
		1924
5060	serge	1925	static void cleanup_status_page(struct intel_engine_cs *ring)
2332	Serge	1926	{
		1927	struct drm_i915_gem_object *obj;
		1928
		1929	obj = ring->status_page.obj;
		1930	if (obj == NULL)
		1931	return;
		1932
5354	serge	1933	kunmap(sg_page(obj->pages->sgl));
5060	serge	1934	i915_gem_object_ggtt_unpin(obj);
2344	Serge	1935	drm_gem_object_unreference(&obj->base);
2332	Serge	1936	ring->status_page.obj = NULL;
		1937	}
		1938
5060	serge	1939	static int init_status_page(struct intel_engine_cs *ring)
2332	Serge	1940	{
		1941	struct drm_i915_gem_object *obj;
5060	serge	1942
		1943	if ((obj = ring->status_page.obj) == NULL) {
		1944	unsigned flags;
6084	serge	1945	int ret;
2332	Serge	1946
5060	serge	1947	obj = i915_gem_alloc_object(ring->dev, 4096);
6084	serge	1948	if (obj == NULL) {
		1949	DRM_ERROR("Failed to allocate status page\n");
5060	serge	1950	return -ENOMEM;
6084	serge	1951	}
2332	Serge	1952
6084	serge	1953	ret = i915_gem_object_set_cache_level(obj, I915_CACHE_LLC);
		1954	if (ret)
		1955	goto err_unref;
2332	Serge	1956
5060	serge	1957	flags = 0;
		1958	if (!HAS_LLC(ring->dev))
		1959	/* On g33, we cannot place HWS above 256MiB, so
		1960	* restrict its pinning to the low mappable arena.
		1961	* Though this restriction is not documented for
		1962	* gen4, gen5, or byt, they also behave similarly
		1963	* and hang if the HWS is placed at the top of the
		1964	* GTT. To generalise, it appears that all !llc
		1965	* platforms have issues with us placing the HWS
		1966	* above the mappable region (even though we never
		1967	* actualy map it).
		1968	*/
		1969	flags \|= PIN_MAPPABLE;
		1970	ret = i915_gem_obj_ggtt_pin(obj, 4096, flags);
		1971	if (ret) {
		1972	err_unref:
		1973	drm_gem_object_unreference(&obj->base);
		1974	return ret;
		1975	}
		1976
		1977	ring->status_page.obj = obj;
2332	Serge	1978	}
		1979
4104	Serge	1980	ring->status_page.gfx_addr = i915_gem_obj_ggtt_offset(obj);
5354	serge	1981	ring->status_page.page_addr = kmap(sg_page(obj->pages->sgl));
2332	Serge	1982	memset(ring->status_page.page_addr, 0, PAGE_SIZE);
		1983
		1984	DRM_DEBUG_DRIVER("%s hws offset: 0x%08x\n",
		1985	ring->name, ring->status_page.gfx_addr);
		1986
		1987	return 0;
		1988	}
		1989
5060	serge	1990	static int init_phys_status_page(struct intel_engine_cs *ring)
3243	Serge	1991	{
6084	serge	1992	struct drm_i915_private *dev_priv = ring->dev->dev_private;
3243	Serge	1993
6084	serge	1994	if (!dev_priv->status_page_dmah) {
		1995	dev_priv->status_page_dmah =
		1996	drm_pci_alloc(ring->dev, PAGE_SIZE, PAGE_SIZE);
		1997	if (!dev_priv->status_page_dmah)
		1998	return -ENOMEM;
		1999	}
3243	Serge	2000
6084	serge	2001	ring->status_page.page_addr = dev_priv->status_page_dmah->vaddr;
		2002	memset(ring->status_page.page_addr, 0, PAGE_SIZE);
3243	Serge	2003
6084	serge	2004	return 0;
3243	Serge	2005	}
		2006
5354	serge	2007	void intel_unpin_ringbuffer_obj(struct intel_ringbuffer *ringbuf)
2332	Serge	2008	{
5060	serge	2009	iounmap(ringbuf->virtual_start);
5354	serge	2010	ringbuf->virtual_start = NULL;
5060	serge	2011	i915_gem_object_ggtt_unpin(ringbuf->obj);
		2012	}
		2013
5354	serge	2014	int intel_pin_and_map_ringbuffer_obj(struct drm_device *dev,
6084	serge	2015	struct intel_ringbuffer *ringbuf)
5060	serge	2016	{
		2017	struct drm_i915_private *dev_priv = to_i915(dev);
5354	serge	2018	struct drm_i915_gem_object *obj = ringbuf->obj;
2332	Serge	2019	int ret;
2340	Serge	2020
5354	serge	2021	ret = i915_gem_obj_ggtt_pin(obj, PAGE_SIZE, PIN_MAPPABLE);
		2022	if (ret)
		2023	return ret;
		2024
		2025	ret = i915_gem_object_set_to_gtt_domain(obj, true);
		2026	if (ret) {
		2027	i915_gem_object_ggtt_unpin(obj);
		2028	return ret;
		2029	}
		2030
		2031	ringbuf->virtual_start = ioremap_wc(dev_priv->gtt.mappable_base +
		2032	i915_gem_obj_ggtt_offset(obj), ringbuf->size);
		2033	if (ringbuf->virtual_start == NULL) {
		2034	i915_gem_object_ggtt_unpin(obj);
		2035	return -EINVAL;
		2036	}
		2037
6084	serge	2038	return 0;
5354	serge	2039	}
2332	Serge	2040
6084	serge	2041	static void intel_destroy_ringbuffer_obj(struct intel_ringbuffer *ringbuf)
5354	serge	2042	{
		2043	drm_gem_object_unreference(&ringbuf->obj->base);
		2044	ringbuf->obj = NULL;
		2045	}
		2046
6084	serge	2047	static int intel_alloc_ringbuffer_obj(struct drm_device *dev,
		2048	struct intel_ringbuffer *ringbuf)
5354	serge	2049	{
		2050	struct drm_i915_gem_object *obj;
		2051
3480	Serge	2052	obj = NULL;
4371	Serge	2053	if (!HAS_LLC(dev))
5060	serge	2054	obj = i915_gem_object_create_stolen(dev, ringbuf->size);
3480	Serge	2055	if (obj == NULL)
5060	serge	2056	obj = i915_gem_alloc_object(dev, ringbuf->size);
		2057	if (obj == NULL)
		2058	return -ENOMEM;
2332	Serge	2059
5060	serge	2060	/* mark ring buffers as read-only from GPU side by default */
		2061	obj->gt_ro = 1;
2332	Serge	2062
5354	serge	2063	ringbuf->obj = obj;
2332	Serge	2064
5060	serge	2065	return 0;
		2066	}
		2067
6084	serge	2068	struct intel_ringbuffer *
		2069	intel_engine_create_ringbuffer(struct intel_engine_cs *engine, int size)
		2070	{
		2071	struct intel_ringbuffer *ring;
		2072	int ret;
		2073
		2074	ring = kzalloc(sizeof(*ring), GFP_KERNEL);
		2075	if (ring == NULL)
		2076	return ERR_PTR(-ENOMEM);
		2077
		2078	ring->ring = engine;
		2079
		2080	ring->size = size;
		2081	/* Workaround an erratum on the i830 which causes a hang if
		2082	* the TAIL pointer points to within the last 2 cachelines
		2083	* of the buffer.
		2084	*/
		2085	ring->effective_size = size;
		2086	if (IS_I830(engine->dev) \|\| IS_845G(engine->dev))
		2087	ring->effective_size -= 2 * CACHELINE_BYTES;
		2088
		2089	ring->last_retired_head = -1;
		2090	intel_ring_update_space(ring);
		2091
		2092	ret = intel_alloc_ringbuffer_obj(engine->dev, ring);
		2093	if (ret) {
		2094	DRM_ERROR("Failed to allocate ringbuffer %s: %d\n",
		2095	engine->name, ret);
		2096	kfree(ring);
		2097	return ERR_PTR(ret);
		2098	}
		2099
		2100	return ring;
		2101	}
		2102
		2103	void
		2104	intel_ringbuffer_free(struct intel_ringbuffer *ring)
		2105	{
		2106	intel_destroy_ringbuffer_obj(ring);
		2107	kfree(ring);
		2108	}
		2109
5060	serge	2110	static int intel_init_ring_buffer(struct drm_device *dev,
		2111	struct intel_engine_cs *ring)
		2112	{
6084	serge	2113	struct intel_ringbuffer *ringbuf;
5060	serge	2114	int ret;
		2115
6084	serge	2116	WARN_ON(ring->buffer);
5060	serge	2117
		2118	ring->dev = dev;
		2119	INIT_LIST_HEAD(&ring->active_list);
		2120	INIT_LIST_HEAD(&ring->request_list);
5354	serge	2121	INIT_LIST_HEAD(&ring->execlist_queue);
6084	serge	2122	i915_gem_batch_pool_init(dev, &ring->batch_pool);
5060	serge	2123	memset(ring->semaphore.sync_seqno, 0, sizeof(ring->semaphore.sync_seqno));
		2124
		2125	init_waitqueue_head(&ring->irq_queue);
		2126
6084	serge	2127	ringbuf = intel_engine_create_ringbuffer(ring, 32 * PAGE_SIZE);
		2128	if (IS_ERR(ringbuf))
		2129	return PTR_ERR(ringbuf);
		2130	ring->buffer = ringbuf;
		2131
5060	serge	2132	if (I915_NEED_GFX_HWS(dev)) {
		2133	ret = init_status_page(ring);
		2134	if (ret)
		2135	goto error;
		2136	} else {
		2137	BUG_ON(ring->id != RCS);
		2138	ret = init_phys_status_page(ring);
6084	serge	2139	if (ret)
5060	serge	2140	goto error;
		2141	}
2332	Serge	2142
6084	serge	2143	ret = intel_pin_and_map_ringbuffer_obj(dev, ringbuf);
5060	serge	2144	if (ret) {
6084	serge	2145	DRM_ERROR("Failed to pin and map ringbuffer %s: %d\n",
		2146	ring->name, ret);
		2147	intel_destroy_ringbuffer_obj(ringbuf);
5060	serge	2148	goto error;
		2149	}
		2150
		2151	ret = i915_cmd_parser_init_ring(ring);
		2152	if (ret)
		2153	goto error;
		2154
2332	Serge	2155	return 0;
		2156
5060	serge	2157	error:
6084	serge	2158	intel_ringbuffer_free(ringbuf);
5060	serge	2159	ring->buffer = NULL;
2332	Serge	2160	return ret;
		2161	}
		2162
5060	serge	2163	void intel_cleanup_ring_buffer(struct intel_engine_cs *ring)
2332	Serge	2164	{
5354	serge	2165	struct drm_i915_private *dev_priv;
2332	Serge	2166
5060	serge	2167	if (!intel_ring_initialized(ring))
2332	Serge	2168	return;
		2169
5354	serge	2170	dev_priv = to_i915(ring->dev);
		2171
5060	serge	2172	intel_stop_ring_buffer(ring);
		2173	WARN_ON(!IS_GEN2(ring->dev) && (I915_READ_MODE(ring) & MODE_IDLE) == 0);
2332	Serge	2174
6084	serge	2175	intel_unpin_ringbuffer_obj(ring->buffer);
		2176	intel_ringbuffer_free(ring->buffer);
		2177	ring->buffer = NULL;
2332	Serge	2178
		2179	if (ring->cleanup)
		2180	ring->cleanup(ring);
		2181
6084	serge	2182	cleanup_status_page(ring);
2332	Serge	2183
5060	serge	2184	i915_cmd_parser_fini_ring(ring);
6084	serge	2185	i915_gem_batch_pool_fini(&ring->batch_pool);
3031	serge	2186	}
		2187
6084	serge	2188	static int ring_wait_for_space(struct intel_engine_cs *ring, int n)
3031	serge	2189	{
5060	serge	2190	struct intel_ringbuffer *ringbuf = ring->buffer;
3031	serge	2191	struct drm_i915_gem_request *request;
6084	serge	2192	unsigned space;
3031	serge	2193	int ret;
		2194
6084	serge	2195	if (intel_ring_space(ringbuf) >= n)
		2196	return 0;
3031	serge	2197
6084	serge	2198	/* The whole point of reserving space is to not wait! */
		2199	WARN_ON(ringbuf->reserved_in_use);
2332	Serge	2200
3031	serge	2201	list_for_each_entry(request, &ring->request_list, list) {
6084	serge	2202	space = __intel_ring_space(request->postfix, ringbuf->tail,
		2203	ringbuf->size);
		2204	if (space >= n)
3031	serge	2205	break;
		2206	}
		2207
6084	serge	2208	if (WARN_ON(&request->list == &ring->request_list))
3031	serge	2209	return -ENOSPC;
		2210
6084	serge	2211	ret = i915_wait_request(request);
3031	serge	2212	if (ret)
		2213	return ret;
		2214
6084	serge	2215	ringbuf->space = space;
3031	serge	2216	return 0;
		2217	}
		2218
6084	serge	2219	static void __wrap_ring_buffer(struct intel_ringbuffer *ringbuf)
3031	serge	2220	{
3243	Serge	2221	uint32_t __iomem *virt;
5060	serge	2222	int rem = ringbuf->size - ringbuf->tail;
3243	Serge	2223
5060	serge	2224	virt = ringbuf->virtual_start + ringbuf->tail;
3243	Serge	2225	rem /= 4;
		2226	while (rem--)
		2227	iowrite32(MI_NOOP, virt++);
		2228
5060	serge	2229	ringbuf->tail = 0;
6084	serge	2230	intel_ring_update_space(ringbuf);
3243	Serge	2231	}
		2232
5060	serge	2233	int intel_ring_idle(struct intel_engine_cs *ring)
3243	Serge	2234	{
6084	serge	2235	struct drm_i915_gem_request *req;
3243	Serge	2236
		2237	/* Wait upon the last request to be completed */
		2238	if (list_empty(&ring->request_list))
		2239	return 0;
		2240
6084	serge	2241	req = list_entry(ring->request_list.prev,
		2242	struct drm_i915_gem_request,
		2243	list);
3243	Serge	2244
6084	serge	2245	/* Make sure we do not trigger any retires */
		2246	return __i915_wait_request(req,
		2247	atomic_read(&to_i915(ring->dev)->gpu_error.reset_counter),
		2248	to_i915(ring->dev)->mm.interruptible,
		2249	NULL, NULL);
3243	Serge	2250	}
		2251
6084	serge	2252	int intel_ring_alloc_request_extras(struct drm_i915_gem_request *request)
3243	Serge	2253	{
6084	serge	2254	request->ringbuf = request->ring->buffer;
		2255	return 0;
		2256	}
3243	Serge	2257
6084	serge	2258	int intel_ring_reserve_space(struct drm_i915_gem_request *request)
		2259	{
		2260	/*
		2261	* The first call merely notes the reserve request and is common for
		2262	* all back ends. The subsequent localised _begin() call actually
		2263	* ensures that the reservation is available. Without the begin, if
		2264	* the request creator immediately submitted the request without
		2265	* adding any commands to it then there might not actually be
		2266	* sufficient room for the submission commands.
		2267	*/
		2268	intel_ring_reserved_space_reserve(request->ringbuf, MIN_SPACE_FOR_ADD_REQUEST);
4560	Serge	2269
6084	serge	2270	return intel_ring_begin(request, 0);
		2271	}
4560	Serge	2272
6084	serge	2273	void intel_ring_reserved_space_reserve(struct intel_ringbuffer *ringbuf, int size)
		2274	{
		2275	WARN_ON(ringbuf->reserved_size);
		2276	WARN_ON(ringbuf->reserved_in_use);
		2277
		2278	ringbuf->reserved_size = size;
		2279	}
		2280
		2281	void intel_ring_reserved_space_cancel(struct intel_ringbuffer *ringbuf)
		2282	{
		2283	WARN_ON(ringbuf->reserved_in_use);
		2284
		2285	ringbuf->reserved_size = 0;
		2286	ringbuf->reserved_in_use = false;
		2287	}
		2288
		2289	void intel_ring_reserved_space_use(struct intel_ringbuffer *ringbuf)
		2290	{
		2291	WARN_ON(ringbuf->reserved_in_use);
		2292
		2293	ringbuf->reserved_in_use = true;
		2294	ringbuf->reserved_tail = ringbuf->tail;
		2295	}
		2296
		2297	void intel_ring_reserved_space_end(struct intel_ringbuffer *ringbuf)
		2298	{
		2299	WARN_ON(!ringbuf->reserved_in_use);
		2300	if (ringbuf->tail > ringbuf->reserved_tail) {
		2301	WARN(ringbuf->tail > ringbuf->reserved_tail + ringbuf->reserved_size,
		2302	"request reserved size too small: %d vs %d!\n",
		2303	ringbuf->tail - ringbuf->reserved_tail, ringbuf->reserved_size);
		2304	} else {
		2305	/*
		2306	* The ring was wrapped while the reserved space was in use.
		2307	* That means that some unknown amount of the ring tail was
		2308	* no-op filled and skipped. Thus simply adding the ring size
		2309	* to the tail and doing the above space check will not work.
		2310	* Rather than attempt to track how much tail was skipped,
		2311	* it is much simpler to say that also skipping the sanity
		2312	* check every once in a while is not a big issue.
		2313	*/
4560	Serge	2314	}
		2315
6084	serge	2316	ringbuf->reserved_size = 0;
		2317	ringbuf->reserved_in_use = false;
3243	Serge	2318	}
		2319
6084	serge	2320	static int __intel_ring_prepare(struct intel_engine_cs *ring, int bytes)
3480	Serge	2321	{
5060	serge	2322	struct intel_ringbuffer *ringbuf = ring->buffer;
6084	serge	2323	int remain_usable = ringbuf->effective_size - ringbuf->tail;
		2324	int remain_actual = ringbuf->size - ringbuf->tail;
		2325	int ret, total_bytes, wait_bytes = 0;
		2326	bool need_wrap = false;
3480	Serge	2327
6084	serge	2328	if (ringbuf->reserved_in_use)
		2329	total_bytes = bytes;
		2330	else
		2331	total_bytes = bytes + ringbuf->reserved_size;
		2332
		2333	if (unlikely(bytes > remain_usable)) {
		2334	/*
		2335	* Not enough space for the basic request. So need to flush
		2336	* out the remainder and then wait for base + reserved.
		2337	*/
		2338	wait_bytes = remain_actual + total_bytes;
		2339	need_wrap = true;
		2340	} else {
		2341	if (unlikely(total_bytes > remain_usable)) {
		2342	/*
		2343	* The base request will fit but the reserved space
		2344	* falls off the end. So only need to to wait for the
		2345	* reserved size after flushing out the remainder.
		2346	*/
		2347	wait_bytes = remain_actual + ringbuf->reserved_size;
		2348	need_wrap = true;
		2349	} else if (total_bytes > ringbuf->space) {
		2350	/* No wrapping required, just waiting. */
		2351	wait_bytes = total_bytes;
		2352	}
3480	Serge	2353	}
		2354
6084	serge	2355	if (wait_bytes) {
		2356	ret = ring_wait_for_space(ring, wait_bytes);
3480	Serge	2357	if (unlikely(ret))
		2358	return ret;
6084	serge	2359
		2360	if (need_wrap)
		2361	__wrap_ring_buffer(ringbuf);
3480	Serge	2362	}
		2363
		2364	return 0;
		2365	}
		2366
6084	serge	2367	int intel_ring_begin(struct drm_i915_gem_request *req,
2332	Serge	2368	int num_dwords)
		2369	{
6084	serge	2370	struct intel_engine_cs *ring;
		2371	struct drm_i915_private *dev_priv;
2332	Serge	2372	int ret;
		2373
6084	serge	2374	WARN_ON(req == NULL);
		2375	ring = req->ring;
		2376	dev_priv = ring->dev->dev_private;
		2377
3480	Serge	2378	ret = i915_gem_check_wedge(&dev_priv->gpu_error,
		2379	dev_priv->mm.interruptible);
3031	serge	2380	if (ret)
		2381	return ret;
2332	Serge	2382
4560	Serge	2383	ret = __intel_ring_prepare(ring, num_dwords * sizeof(uint32_t));
		2384	if (ret)
		2385	return ret;
		2386
5060	serge	2387	ring->buffer->space -= num_dwords * sizeof(uint32_t);
4560	Serge	2388	return 0;
3480	Serge	2389	}
2332	Serge	2390
5060	serge	2391	/* Align the ring tail to a cacheline boundary */
6084	serge	2392	int intel_ring_cacheline_align(struct drm_i915_gem_request *req)
3480	Serge	2393	{
6084	serge	2394	struct intel_engine_cs *ring = req->ring;
5060	serge	2395	int num_dwords = (ring->buffer->tail & (CACHELINE_BYTES - 1)) / sizeof(uint32_t);
		2396	int ret;
3480	Serge	2397
5060	serge	2398	if (num_dwords == 0)
		2399	return 0;
		2400
		2401	num_dwords = CACHELINE_BYTES / sizeof(uint32_t) - num_dwords;
6084	serge	2402	ret = intel_ring_begin(req, num_dwords);
5060	serge	2403	if (ret)
		2404	return ret;
		2405
		2406	while (num_dwords--)
		2407	intel_ring_emit(ring, MI_NOOP);
		2408
		2409	intel_ring_advance(ring);
		2410
		2411	return 0;
		2412	}
		2413
		2414	void intel_ring_init_seqno(struct intel_engine_cs *ring, u32 seqno)
		2415	{
		2416	struct drm_device *dev = ring->dev;
		2417	struct drm_i915_private *dev_priv = dev->dev_private;
		2418
		2419	if (INTEL_INFO(dev)->gen == 6 \|\| INTEL_INFO(dev)->gen == 7) {
3480	Serge	2420	I915_WRITE(RING_SYNC_0(ring->mmio_base), 0);
		2421	I915_WRITE(RING_SYNC_1(ring->mmio_base), 0);
5060	serge	2422	if (HAS_VEBOX(dev))
4104	Serge	2423	I915_WRITE(RING_SYNC_2(ring->mmio_base), 0);
2332	Serge	2424	}
		2425
3480	Serge	2426	ring->set_seqno(ring, seqno);
4104	Serge	2427	ring->hangcheck.seqno = seqno;
2332	Serge	2428	}
		2429
5060	serge	2430	static void gen6_bsd_ring_write_tail(struct intel_engine_cs *ring,
2332	Serge	2431	u32 value)
		2432	{
5060	serge	2433	struct drm_i915_private *dev_priv = ring->dev->dev_private;
2332	Serge	2434
		2435	/* Every tail move must follow the sequence below */
3031	serge	2436
		2437	/* Disable notification that the ring is IDLE. The GT
		2438	* will then assume that it is busy and bring it out of rc6.
		2439	*/
6084	serge	2440	I915_WRITE(GEN6_BSD_SLEEP_PSMI_CONTROL,
3031	serge	2441	_MASKED_BIT_ENABLE(GEN6_BSD_SLEEP_MSG_DISABLE));
2332	Serge	2442
3031	serge	2443	/* Clear the context id. Here be magic! */
		2444	I915_WRITE64(GEN6_BSD_RNCID, 0x0);
		2445
		2446	/* Wait for the ring not to be idle, i.e. for it to wake up. */
6084	serge	2447	if (wait_for((I915_READ(GEN6_BSD_SLEEP_PSMI_CONTROL) &
3031	serge	2448	GEN6_BSD_SLEEP_INDICATOR) == 0,
6084	serge	2449	50))
3031	serge	2450	DRM_ERROR("timed out waiting for the BSD ring to wake up\n");
2332	Serge	2451
3031	serge	2452	/* Now that the ring is fully powered up, update the tail */
6084	serge	2453	I915_WRITE_TAIL(ring, value);
3031	serge	2454	POSTING_READ(RING_TAIL(ring->mmio_base));
		2455
		2456	/* Let the ring send IDLE messages to the GT again,
		2457	* and so let it sleep to conserve power when idle.
		2458	*/
6084	serge	2459	I915_WRITE(GEN6_BSD_SLEEP_PSMI_CONTROL,
3031	serge	2460	_MASKED_BIT_DISABLE(GEN6_BSD_SLEEP_MSG_DISABLE));
2332	Serge	2461	}
		2462
6084	serge	2463	static int gen6_bsd_ring_flush(struct drm_i915_gem_request *req,
		2464	u32 invalidate, u32 flush)
2332	Serge	2465	{
6084	serge	2466	struct intel_engine_cs *ring = req->ring;
2332	Serge	2467	uint32_t cmd;
		2468	int ret;
		2469
6084	serge	2470	ret = intel_ring_begin(req, 4);
2332	Serge	2471	if (ret)
		2472	return ret;
		2473
		2474	cmd = MI_FLUSH_DW;
4560	Serge	2475	if (INTEL_INFO(ring->dev)->gen >= 8)
		2476	cmd += 1;
6084	serge	2477
		2478	/* We always require a command barrier so that subsequent
		2479	* commands, such as breadcrumb interrupts, are strictly ordered
		2480	* wrt the contents of the write cache being flushed to memory
		2481	* (and thus being coherent from the CPU).
		2482	*/
		2483	cmd \|= MI_FLUSH_DW_STORE_INDEX \| MI_FLUSH_DW_OP_STOREDW;
		2484
3243	Serge	2485	/*
		2486	* Bspec vol 1c.5 - video engine command streamer:
		2487	* "If ENABLED, all TLBs will be invalidated once the flush
		2488	* operation is complete. This bit is only valid when the
		2489	* Post-Sync Operation field is a value of 1h or 3h."
		2490	*/
2332	Serge	2491	if (invalidate & I915_GEM_GPU_DOMAINS)
6084	serge	2492	cmd \|= MI_INVALIDATE_TLB \| MI_INVALIDATE_BSD;
		2493
2332	Serge	2494	intel_ring_emit(ring, cmd);
3243	Serge	2495	intel_ring_emit(ring, I915_GEM_HWS_SCRATCH_ADDR \| MI_FLUSH_DW_USE_GTT);
4560	Serge	2496	if (INTEL_INFO(ring->dev)->gen >= 8) {
		2497	intel_ring_emit(ring, 0); /* upper addr */
		2498	intel_ring_emit(ring, 0); /* value */
		2499	} else {
6084	serge	2500	intel_ring_emit(ring, 0);
		2501	intel_ring_emit(ring, MI_NOOP);
4560	Serge	2502	}
2332	Serge	2503	intel_ring_advance(ring);
		2504	return 0;
		2505	}
		2506
		2507	static int
6084	serge	2508	gen8_ring_dispatch_execbuffer(struct drm_i915_gem_request *req,
5060	serge	2509	u64 offset, u32 len,
6084	serge	2510	unsigned dispatch_flags)
4560	Serge	2511	{
6084	serge	2512	struct intel_engine_cs *ring = req->ring;
		2513	bool ppgtt = USES_PPGTT(ring->dev) &&
		2514	!(dispatch_flags & I915_DISPATCH_SECURE);
4560	Serge	2515	int ret;
		2516
6084	serge	2517	ret = intel_ring_begin(req, 4);
4560	Serge	2518	if (ret)
		2519	return ret;
		2520
		2521	/* FIXME(BDW): Address space and security selectors. */
6084	serge	2522	intel_ring_emit(ring, MI_BATCH_BUFFER_START_GEN8 \| (ppgtt<<8) \|
		2523	(dispatch_flags & I915_DISPATCH_RS ?
		2524	MI_BATCH_RESOURCE_STREAMER : 0));
5060	serge	2525	intel_ring_emit(ring, lower_32_bits(offset));
		2526	intel_ring_emit(ring, upper_32_bits(offset));
4560	Serge	2527	intel_ring_emit(ring, MI_NOOP);
		2528	intel_ring_advance(ring);
		2529
		2530	return 0;
		2531	}
		2532
		2533	static int
6084	serge	2534	hsw_ring_dispatch_execbuffer(struct drm_i915_gem_request *req,
		2535	u64 offset, u32 len,
		2536	unsigned dispatch_flags)
3243	Serge	2537	{
6084	serge	2538	struct intel_engine_cs *ring = req->ring;
3243	Serge	2539	int ret;
		2540
6084	serge	2541	ret = intel_ring_begin(req, 2);
3243	Serge	2542	if (ret)
		2543	return ret;
		2544
		2545	intel_ring_emit(ring,
5354	serge	2546	MI_BATCH_BUFFER_START \|
6084	serge	2547	(dispatch_flags & I915_DISPATCH_SECURE ?
		2548
		2549	(dispatch_flags & I915_DISPATCH_RS ?
		2550	MI_BATCH_RESOURCE_STREAMER : 0));
3243	Serge	2551	/* bit0-7 is the length on GEN6+ */
		2552	intel_ring_emit(ring, offset);
		2553	intel_ring_advance(ring);
		2554
		2555	return 0;
		2556	}
		2557
		2558	static int
6084	serge	2559	gen6_ring_dispatch_execbuffer(struct drm_i915_gem_request *req,
5060	serge	2560	u64 offset, u32 len,
6084	serge	2561	unsigned dispatch_flags)
2332	Serge	2562	{
6084	serge	2563	struct intel_engine_cs *ring = req->ring;
		2564	int ret;
2332	Serge	2565
6084	serge	2566	ret = intel_ring_begin(req, 2);
		2567	if (ret)
		2568	return ret;
2332	Serge	2569
3243	Serge	2570	intel_ring_emit(ring,
		2571	MI_BATCH_BUFFER_START \|
6084	serge	2572	(dispatch_flags & I915_DISPATCH_SECURE ?
		2573
		2574	/* bit0-7 is the length on GEN6+ */
		2575	intel_ring_emit(ring, offset);
		2576	intel_ring_advance(ring);
2332	Serge	2577
6084	serge	2578	return 0;
2332	Serge	2579	}
		2580
		2581	/* Blitter support (SandyBridge+) */
		2582
6084	serge	2583	static int gen6_ring_flush(struct drm_i915_gem_request *req,
		2584	u32 invalidate, u32 flush)
2332	Serge	2585	{
6084	serge	2586	struct intel_engine_cs *ring = req->ring;
4104	Serge	2587	struct drm_device *dev = ring->dev;
2332	Serge	2588	uint32_t cmd;
		2589	int ret;
		2590
6084	serge	2591	ret = intel_ring_begin(req, 4);
2332	Serge	2592	if (ret)
		2593	return ret;
		2594
		2595	cmd = MI_FLUSH_DW;
6084	serge	2596	if (INTEL_INFO(dev)->gen >= 8)
4560	Serge	2597	cmd += 1;
6084	serge	2598
		2599	/* We always require a command barrier so that subsequent
		2600	* commands, such as breadcrumb interrupts, are strictly ordered
		2601	* wrt the contents of the write cache being flushed to memory
		2602	* (and thus being coherent from the CPU).
		2603	*/
		2604	cmd \|= MI_FLUSH_DW_STORE_INDEX \| MI_FLUSH_DW_OP_STOREDW;
		2605
3243	Serge	2606	/*
		2607	* Bspec vol 1c.3 - blitter engine command streamer:
		2608	* "If ENABLED, all TLBs will be invalidated once the flush
		2609	* operation is complete. This bit is only valid when the
		2610	* Post-Sync Operation field is a value of 1h or 3h."
		2611	*/
2332	Serge	2612	if (invalidate & I915_GEM_DOMAIN_RENDER)
6084	serge	2613	cmd \|= MI_INVALIDATE_TLB;
2332	Serge	2614	intel_ring_emit(ring, cmd);
3243	Serge	2615	intel_ring_emit(ring, I915_GEM_HWS_SCRATCH_ADDR \| MI_FLUSH_DW_USE_GTT);
6084	serge	2616	if (INTEL_INFO(dev)->gen >= 8) {
4560	Serge	2617	intel_ring_emit(ring, 0); /* upper addr */
		2618	intel_ring_emit(ring, 0); /* value */
		2619	} else {
6084	serge	2620	intel_ring_emit(ring, 0);
		2621	intel_ring_emit(ring, MI_NOOP);
4560	Serge	2622	}
2332	Serge	2623	intel_ring_advance(ring);
4104	Serge	2624
2332	Serge	2625	return 0;
		2626	}
		2627
		2628	int intel_init_render_ring_buffer(struct drm_device *dev)
		2629	{
5060	serge	2630	struct drm_i915_private *dev_priv = dev->dev_private;
		2631	struct intel_engine_cs *ring = &dev_priv->ring[RCS];
		2632	struct drm_i915_gem_object *obj;
		2633	int ret;
2340	Serge	2634
3031	serge	2635	ring->name = "render ring";
		2636	ring->id = RCS;
		2637	ring->mmio_base = RENDER_RING_BASE;
		2638
5060	serge	2639	if (INTEL_INFO(dev)->gen >= 8) {
		2640	if (i915_semaphore_is_enabled(dev)) {
		2641	obj = i915_gem_alloc_object(dev, 4096);
		2642	if (obj == NULL) {
		2643	DRM_ERROR("Failed to allocate semaphore bo. Disabling semaphores\n");
		2644	i915.semaphores = 0;
		2645	} else {
		2646	i915_gem_object_set_cache_level(obj, I915_CACHE_LLC);
		2647	ret = i915_gem_obj_ggtt_pin(obj, 0, PIN_NONBLOCK);
		2648	if (ret != 0) {
		2649	drm_gem_object_unreference(&obj->base);
		2650	DRM_ERROR("Failed to pin semaphore bo. Disabling semaphores\n");
		2651	i915.semaphores = 0;
		2652	} else
		2653	dev_priv->semaphore_obj = obj;
		2654	}
		2655	}
5354	serge	2656
6084	serge	2657	ring->init_context = intel_rcs_ctx_init;
5060	serge	2658	ring->add_request = gen6_add_request;
		2659	ring->flush = gen8_render_ring_flush;
		2660	ring->irq_get = gen8_ring_get_irq;
		2661	ring->irq_put = gen8_ring_put_irq;
		2662	ring->irq_enable_mask = GT_RENDER_USER_INTERRUPT;
		2663	ring->get_seqno = gen6_ring_get_seqno;
		2664	ring->set_seqno = ring_set_seqno;
		2665	if (i915_semaphore_is_enabled(dev)) {
		2666	WARN_ON(!dev_priv->semaphore_obj);
		2667	ring->semaphore.sync_to = gen8_ring_sync;
		2668	ring->semaphore.signal = gen8_rcs_signal;
		2669	GEN8_RING_SEMAPHORE_INIT;
		2670	}
		2671	} else if (INTEL_INFO(dev)->gen >= 6) {
6084	serge	2672	ring->init_context = intel_rcs_ctx_init;
		2673	ring->add_request = gen6_add_request;
3031	serge	2674	ring->flush = gen7_render_ring_flush;
		2675	if (INTEL_INFO(dev)->gen == 6)
6084	serge	2676	ring->flush = gen6_render_ring_flush;
3031	serge	2677	ring->irq_get = gen6_ring_get_irq;
		2678	ring->irq_put = gen6_ring_put_irq;
4104	Serge	2679	ring->irq_enable_mask = GT_RENDER_USER_INTERRUPT;
2342	Serge	2680	ring->get_seqno = gen6_ring_get_seqno;
3480	Serge	2681	ring->set_seqno = ring_set_seqno;
5060	serge	2682	if (i915_semaphore_is_enabled(dev)) {
6084	serge	2683	ring->semaphore.sync_to = gen6_ring_sync;
		2684	ring->semaphore.signal = gen6_signal;
		2685	/*
5060	serge	2686	* The current semaphore is only applied on pre-gen8
		2687	* platform. And there is no VCS2 ring on the pre-gen8
		2688	* platform. So the semaphore between RCS and VCS2 is
		2689	* initialized as INVALID. Gen8 will initialize the
		2690	* sema between VCS2 and RCS later.
6084	serge	2691	*/
		2692	ring->semaphore.mbox.wait[RCS] = MI_SEMAPHORE_SYNC_INVALID;
		2693	ring->semaphore.mbox.wait[VCS] = MI_SEMAPHORE_SYNC_RV;
		2694	ring->semaphore.mbox.wait[BCS] = MI_SEMAPHORE_SYNC_RB;
		2695	ring->semaphore.mbox.wait[VECS] = MI_SEMAPHORE_SYNC_RVE;
		2696	ring->semaphore.mbox.wait[VCS2] = MI_SEMAPHORE_SYNC_INVALID;
		2697	ring->semaphore.mbox.signal[RCS] = GEN6_NOSYNC;
		2698	ring->semaphore.mbox.signal[VCS] = GEN6_VRSYNC;
		2699	ring->semaphore.mbox.signal[BCS] = GEN6_BRSYNC;
		2700	ring->semaphore.mbox.signal[VECS] = GEN6_VERSYNC;
		2701	ring->semaphore.mbox.signal[VCS2] = GEN6_NOSYNC;
5060	serge	2702	}
2332	Serge	2703	} else if (IS_GEN5(dev)) {
6084	serge	2704	ring->add_request = pc_render_add_request;
3031	serge	2705	ring->flush = gen4_render_ring_flush;
2342	Serge	2706	ring->get_seqno = pc_render_get_seqno;
3480	Serge	2707	ring->set_seqno = pc_render_set_seqno;
3031	serge	2708	ring->irq_get = gen5_ring_get_irq;
		2709	ring->irq_put = gen5_ring_put_irq;
4104	Serge	2710	ring->irq_enable_mask = GT_RENDER_USER_INTERRUPT \|
		2711	GT_RENDER_PIPECTL_NOTIFY_INTERRUPT;
3031	serge	2712	} else {
		2713	ring->add_request = i9xx_add_request;
		2714	if (INTEL_INFO(dev)->gen < 4)
		2715	ring->flush = gen2_render_ring_flush;
		2716	else
		2717	ring->flush = gen4_render_ring_flush;
		2718	ring->get_seqno = ring_get_seqno;
3480	Serge	2719	ring->set_seqno = ring_set_seqno;
3031	serge	2720	if (IS_GEN2(dev)) {
		2721	ring->irq_get = i8xx_ring_get_irq;
		2722	ring->irq_put = i8xx_ring_put_irq;
		2723	} else {
		2724	ring->irq_get = i9xx_ring_get_irq;
		2725	ring->irq_put = i9xx_ring_put_irq;
		2726	}
		2727	ring->irq_enable_mask = I915_USER_INTERRUPT;
2332	Serge	2728	}
3031	serge	2729	ring->write_tail = ring_write_tail;
5060	serge	2730
3243	Serge	2731	if (IS_HASWELL(dev))
		2732	ring->dispatch_execbuffer = hsw_ring_dispatch_execbuffer;
4560	Serge	2733	else if (IS_GEN8(dev))
		2734	ring->dispatch_execbuffer = gen8_ring_dispatch_execbuffer;
3243	Serge	2735	else if (INTEL_INFO(dev)->gen >= 6)
3031	serge	2736	ring->dispatch_execbuffer = gen6_ring_dispatch_execbuffer;
		2737	else if (INTEL_INFO(dev)->gen >= 4)
		2738	ring->dispatch_execbuffer = i965_dispatch_execbuffer;
		2739	else if (IS_I830(dev) \|\| IS_845G(dev))
		2740	ring->dispatch_execbuffer = i830_dispatch_execbuffer;
		2741	else
		2742	ring->dispatch_execbuffer = i915_dispatch_execbuffer;
6084	serge	2743	ring->init_hw = init_render_ring;
3031	serge	2744	ring->cleanup = render_ring_cleanup;
2332	Serge	2745
3243	Serge	2746	/* Workaround batchbuffer to combat CS tlb bug. */
		2747	if (HAS_BROKEN_CS_TLB(dev)) {
5128	serge	2748	obj = i915_gem_alloc_object(dev, I830_WA_SIZE);
3243	Serge	2749	if (obj == NULL) {
		2750	DRM_ERROR("Failed to allocate batch bo\n");
		2751	return -ENOMEM;
		2752	}
		2753
5060	serge	2754	ret = i915_gem_obj_ggtt_pin(obj, 0, 0);
3243	Serge	2755	if (ret != 0) {
		2756	drm_gem_object_unreference(&obj->base);
		2757	DRM_ERROR("Failed to ping batch bo\n");
		2758	return ret;
		2759	}
		2760
4104	Serge	2761	ring->scratch.obj = obj;
		2762	ring->scratch.gtt_offset = i915_gem_obj_ggtt_offset(obj);
2332	Serge	2763	}
2340	Serge	2764
6084	serge	2765	ret = intel_init_ring_buffer(dev, ring);
		2766	if (ret)
		2767	return ret;
		2768
		2769	if (INTEL_INFO(dev)->gen >= 5) {
		2770	ret = intel_init_pipe_control(ring);
		2771	if (ret)
		2772	return ret;
		2773	}
		2774
		2775	return 0;
2332	Serge	2776	}
		2777
		2778	int intel_init_bsd_ring_buffer(struct drm_device *dev)
		2779	{
5060	serge	2780	struct drm_i915_private *dev_priv = dev->dev_private;
		2781	struct intel_engine_cs *ring = &dev_priv->ring[VCS];
2332	Serge	2782
3031	serge	2783	ring->name = "bsd ring";
		2784	ring->id = VCS;
2332	Serge	2785
3031	serge	2786	ring->write_tail = ring_write_tail;
4560	Serge	2787	if (INTEL_INFO(dev)->gen >= 6) {
3031	serge	2788	ring->mmio_base = GEN6_BSD_RING_BASE;
		2789	/* gen6 bsd needs a special wa for tail updates */
		2790	if (IS_GEN6(dev))
		2791	ring->write_tail = gen6_bsd_ring_write_tail;
4104	Serge	2792	ring->flush = gen6_bsd_ring_flush;
3031	serge	2793	ring->add_request = gen6_add_request;
		2794	ring->get_seqno = gen6_ring_get_seqno;
3480	Serge	2795	ring->set_seqno = ring_set_seqno;
4560	Serge	2796	if (INTEL_INFO(dev)->gen >= 8) {
		2797	ring->irq_enable_mask =
		2798	GT_RENDER_USER_INTERRUPT << GEN8_VCS1_IRQ_SHIFT;
		2799	ring->irq_get = gen8_ring_get_irq;
		2800	ring->irq_put = gen8_ring_put_irq;
		2801	ring->dispatch_execbuffer =
		2802	gen8_ring_dispatch_execbuffer;
5060	serge	2803	if (i915_semaphore_is_enabled(dev)) {
		2804	ring->semaphore.sync_to = gen8_ring_sync;
		2805	ring->semaphore.signal = gen8_xcs_signal;
		2806	GEN8_RING_SEMAPHORE_INIT;
		2807	}
4560	Serge	2808	} else {
6084	serge	2809	ring->irq_enable_mask = GT_BSD_USER_INTERRUPT;
		2810	ring->irq_get = gen6_ring_get_irq;
		2811	ring->irq_put = gen6_ring_put_irq;
4560	Serge	2812	ring->dispatch_execbuffer =
		2813	gen6_ring_dispatch_execbuffer;
5060	serge	2814	if (i915_semaphore_is_enabled(dev)) {
6084	serge	2815	ring->semaphore.sync_to = gen6_ring_sync;
		2816	ring->semaphore.signal = gen6_signal;
		2817	ring->semaphore.mbox.wait[RCS] = MI_SEMAPHORE_SYNC_VR;
		2818	ring->semaphore.mbox.wait[VCS] = MI_SEMAPHORE_SYNC_INVALID;
		2819	ring->semaphore.mbox.wait[BCS] = MI_SEMAPHORE_SYNC_VB;
		2820	ring->semaphore.mbox.wait[VECS] = MI_SEMAPHORE_SYNC_VVE;
		2821	ring->semaphore.mbox.wait[VCS2] = MI_SEMAPHORE_SYNC_INVALID;
		2822	ring->semaphore.mbox.signal[RCS] = GEN6_RVSYNC;
		2823	ring->semaphore.mbox.signal[VCS] = GEN6_NOSYNC;
		2824	ring->semaphore.mbox.signal[BCS] = GEN6_BVSYNC;
		2825	ring->semaphore.mbox.signal[VECS] = GEN6_VEVSYNC;
		2826	ring->semaphore.mbox.signal[VCS2] = GEN6_NOSYNC;
5060	serge	2827	}
4560	Serge	2828	}
3031	serge	2829	} else {
		2830	ring->mmio_base = BSD_RING_BASE;
		2831	ring->flush = bsd_ring_flush;
		2832	ring->add_request = i9xx_add_request;
		2833	ring->get_seqno = ring_get_seqno;
3480	Serge	2834	ring->set_seqno = ring_set_seqno;
3031	serge	2835	if (IS_GEN5(dev)) {
4104	Serge	2836	ring->irq_enable_mask = ILK_BSD_USER_INTERRUPT;
3031	serge	2837	ring->irq_get = gen5_ring_get_irq;
		2838	ring->irq_put = gen5_ring_put_irq;
		2839	} else {
		2840	ring->irq_enable_mask = I915_BSD_USER_INTERRUPT;
		2841	ring->irq_get = i9xx_ring_get_irq;
		2842	ring->irq_put = i9xx_ring_put_irq;
		2843	}
		2844	ring->dispatch_execbuffer = i965_dispatch_execbuffer;
		2845	}
6084	serge	2846	ring->init_hw = init_ring_common;
3031	serge	2847
2332	Serge	2848	return intel_init_ring_buffer(dev, ring);
		2849	}
		2850
5060	serge	2851	/**
6084	serge	2852	* Initialize the second BSD ring (eg. Broadwell GT3, Skylake GT3)
5060	serge	2853	*/
		2854	int intel_init_bsd2_ring_buffer(struct drm_device *dev)
		2855	{
		2856	struct drm_i915_private *dev_priv = dev->dev_private;
		2857	struct intel_engine_cs *ring = &dev_priv->ring[VCS2];
		2858
		2859	ring->name = "bsd2 ring";
		2860	ring->id = VCS2;
		2861
		2862	ring->write_tail = ring_write_tail;
		2863	ring->mmio_base = GEN8_BSD2_RING_BASE;
		2864	ring->flush = gen6_bsd_ring_flush;
		2865	ring->add_request = gen6_add_request;
		2866	ring->get_seqno = gen6_ring_get_seqno;
		2867	ring->set_seqno = ring_set_seqno;
		2868	ring->irq_enable_mask =
		2869	GT_RENDER_USER_INTERRUPT << GEN8_VCS2_IRQ_SHIFT;
		2870	ring->irq_get = gen8_ring_get_irq;
		2871	ring->irq_put = gen8_ring_put_irq;
		2872	ring->dispatch_execbuffer =
		2873	gen8_ring_dispatch_execbuffer;
		2874	if (i915_semaphore_is_enabled(dev)) {
		2875	ring->semaphore.sync_to = gen8_ring_sync;
		2876	ring->semaphore.signal = gen8_xcs_signal;
		2877	GEN8_RING_SEMAPHORE_INIT;
		2878	}
6084	serge	2879	ring->init_hw = init_ring_common;
5060	serge	2880
		2881	return intel_init_ring_buffer(dev, ring);
		2882	}
		2883
2332	Serge	2884	int intel_init_blt_ring_buffer(struct drm_device *dev)
		2885	{
5060	serge	2886	struct drm_i915_private *dev_priv = dev->dev_private;
		2887	struct intel_engine_cs *ring = &dev_priv->ring[BCS];
2332	Serge	2888
3031	serge	2889	ring->name = "blitter ring";
		2890	ring->id = BCS;
2332	Serge	2891
3031	serge	2892	ring->mmio_base = BLT_RING_BASE;
		2893	ring->write_tail = ring_write_tail;
4104	Serge	2894	ring->flush = gen6_ring_flush;
3031	serge	2895	ring->add_request = gen6_add_request;
		2896	ring->get_seqno = gen6_ring_get_seqno;
3480	Serge	2897	ring->set_seqno = ring_set_seqno;
4560	Serge	2898	if (INTEL_INFO(dev)->gen >= 8) {
		2899	ring->irq_enable_mask =
		2900	GT_RENDER_USER_INTERRUPT << GEN8_BCS_IRQ_SHIFT;
		2901	ring->irq_get = gen8_ring_get_irq;
		2902	ring->irq_put = gen8_ring_put_irq;
		2903	ring->dispatch_execbuffer = gen8_ring_dispatch_execbuffer;
5060	serge	2904	if (i915_semaphore_is_enabled(dev)) {
		2905	ring->semaphore.sync_to = gen8_ring_sync;
		2906	ring->semaphore.signal = gen8_xcs_signal;
		2907	GEN8_RING_SEMAPHORE_INIT;
		2908	}
4560	Serge	2909	} else {
6084	serge	2910	ring->irq_enable_mask = GT_BLT_USER_INTERRUPT;
		2911	ring->irq_get = gen6_ring_get_irq;
		2912	ring->irq_put = gen6_ring_put_irq;
		2913	ring->dispatch_execbuffer = gen6_ring_dispatch_execbuffer;
5060	serge	2914	if (i915_semaphore_is_enabled(dev)) {
		2915	ring->semaphore.signal = gen6_signal;
6084	serge	2916	ring->semaphore.sync_to = gen6_ring_sync;
		2917	/*
5060	serge	2918	* The current semaphore is only applied on pre-gen8
		2919	* platform. And there is no VCS2 ring on the pre-gen8
		2920	* platform. So the semaphore between BCS and VCS2 is
		2921	* initialized as INVALID. Gen8 will initialize the
		2922	* sema between BCS and VCS2 later.
6084	serge	2923	*/
		2924	ring->semaphore.mbox.wait[RCS] = MI_SEMAPHORE_SYNC_BR;
		2925	ring->semaphore.mbox.wait[VCS] = MI_SEMAPHORE_SYNC_BV;
		2926	ring->semaphore.mbox.wait[BCS] = MI_SEMAPHORE_SYNC_INVALID;
		2927	ring->semaphore.mbox.wait[VECS] = MI_SEMAPHORE_SYNC_BVE;
		2928	ring->semaphore.mbox.wait[VCS2] = MI_SEMAPHORE_SYNC_INVALID;
		2929	ring->semaphore.mbox.signal[RCS] = GEN6_RBSYNC;
		2930	ring->semaphore.mbox.signal[VCS] = GEN6_VBSYNC;
		2931	ring->semaphore.mbox.signal[BCS] = GEN6_NOSYNC;
		2932	ring->semaphore.mbox.signal[VECS] = GEN6_VEBSYNC;
		2933	ring->semaphore.mbox.signal[VCS2] = GEN6_NOSYNC;
5060	serge	2934	}
4560	Serge	2935	}
6084	serge	2936	ring->init_hw = init_ring_common;
3031	serge	2937
2332	Serge	2938	return intel_init_ring_buffer(dev, ring);
		2939	}
3031	serge	2940
4104	Serge	2941	int intel_init_vebox_ring_buffer(struct drm_device *dev)
		2942	{
5060	serge	2943	struct drm_i915_private *dev_priv = dev->dev_private;
		2944	struct intel_engine_cs *ring = &dev_priv->ring[VECS];
4104	Serge	2945
		2946	ring->name = "video enhancement ring";
		2947	ring->id = VECS;
		2948
		2949	ring->mmio_base = VEBOX_RING_BASE;
		2950	ring->write_tail = ring_write_tail;
		2951	ring->flush = gen6_ring_flush;
		2952	ring->add_request = gen6_add_request;
		2953	ring->get_seqno = gen6_ring_get_seqno;
		2954	ring->set_seqno = ring_set_seqno;
4560	Serge	2955
		2956	if (INTEL_INFO(dev)->gen >= 8) {
		2957	ring->irq_enable_mask =
		2958	GT_RENDER_USER_INTERRUPT << GEN8_VECS_IRQ_SHIFT;
		2959	ring->irq_get = gen8_ring_get_irq;
		2960	ring->irq_put = gen8_ring_put_irq;
		2961	ring->dispatch_execbuffer = gen8_ring_dispatch_execbuffer;
5060	serge	2962	if (i915_semaphore_is_enabled(dev)) {
		2963	ring->semaphore.sync_to = gen8_ring_sync;
		2964	ring->semaphore.signal = gen8_xcs_signal;
		2965	GEN8_RING_SEMAPHORE_INIT;
		2966	}
4560	Serge	2967	} else {
6084	serge	2968	ring->irq_enable_mask = PM_VEBOX_USER_INTERRUPT;
		2969	ring->irq_get = hsw_vebox_get_irq;
		2970	ring->irq_put = hsw_vebox_put_irq;
		2971	ring->dispatch_execbuffer = gen6_ring_dispatch_execbuffer;
5060	serge	2972	if (i915_semaphore_is_enabled(dev)) {
6084	serge	2973	ring->semaphore.sync_to = gen6_ring_sync;
		2974	ring->semaphore.signal = gen6_signal;
		2975	ring->semaphore.mbox.wait[RCS] = MI_SEMAPHORE_SYNC_VER;
		2976	ring->semaphore.mbox.wait[VCS] = MI_SEMAPHORE_SYNC_VEV;
		2977	ring->semaphore.mbox.wait[BCS] = MI_SEMAPHORE_SYNC_VEB;
		2978	ring->semaphore.mbox.wait[VECS] = MI_SEMAPHORE_SYNC_INVALID;
		2979	ring->semaphore.mbox.wait[VCS2] = MI_SEMAPHORE_SYNC_INVALID;
		2980	ring->semaphore.mbox.signal[RCS] = GEN6_RVESYNC;
		2981	ring->semaphore.mbox.signal[VCS] = GEN6_VVESYNC;
		2982	ring->semaphore.mbox.signal[BCS] = GEN6_BVESYNC;
		2983	ring->semaphore.mbox.signal[VECS] = GEN6_NOSYNC;
		2984	ring->semaphore.mbox.signal[VCS2] = GEN6_NOSYNC;
5060	serge	2985	}
4560	Serge	2986	}
6084	serge	2987	ring->init_hw = init_ring_common;
4104	Serge	2988
		2989	return intel_init_ring_buffer(dev, ring);
		2990	}
		2991
3031	serge	2992	int
6084	serge	2993	intel_ring_flush_all_caches(struct drm_i915_gem_request *req)
3031	serge	2994	{
6084	serge	2995	struct intel_engine_cs *ring = req->ring;
3031	serge	2996	int ret;
		2997
		2998	if (!ring->gpu_caches_dirty)
		2999	return 0;
		3000
6084	serge	3001	ret = ring->flush(req, 0, I915_GEM_GPU_DOMAINS);
3031	serge	3002	if (ret)
		3003	return ret;
		3004
6084	serge	3005	trace_i915_gem_ring_flush(req, 0, I915_GEM_GPU_DOMAINS);
3031	serge	3006
		3007	ring->gpu_caches_dirty = false;
		3008	return 0;
		3009	}
		3010
		3011	int
6084	serge	3012	intel_ring_invalidate_all_caches(struct drm_i915_gem_request *req)
3031	serge	3013	{
6084	serge	3014	struct intel_engine_cs *ring = req->ring;
3031	serge	3015	uint32_t flush_domains;
		3016	int ret;
		3017
		3018	flush_domains = 0;
		3019	if (ring->gpu_caches_dirty)
		3020	flush_domains = I915_GEM_GPU_DOMAINS;
		3021
6084	serge	3022	ret = ring->flush(req, I915_GEM_GPU_DOMAINS, flush_domains);
3031	serge	3023	if (ret)
		3024	return ret;
		3025
6084	serge	3026	trace_i915_gem_ring_flush(req, I915_GEM_GPU_DOMAINS, flush_domains);
3031	serge	3027
		3028	ring->gpu_caches_dirty = false;
		3029	return 0;
		3030	}
5060	serge	3031
		3032	void
		3033	intel_stop_ring_buffer(struct intel_engine_cs *ring)
		3034	{
		3035	int ret;
		3036
		3037	if (!intel_ring_initialized(ring))
		3038	return;
		3039
		3040	ret = intel_ring_idle(ring);
		3041	if (ret && !i915_reset_in_progress(&to_i915(ring->dev)->gpu_error))
		3042	DRM_ERROR("failed to quiesce %s whilst cleaning up: %d\n",
		3043	ring->name, ret);
		3044
		3045	stop_ring(ring);
		3046	}

Subversion Repositories Kolibri OS

(root)/drivers/video/drm/i915/intel_ringbuffer.c – Rev 6320