WebSVN – Kolibri OS – Blame – /drivers/video/drm/i915/intel_ringbuffer.c

Rev	Author	Line No.	Line
2332	Serge	1	/*
		2	* Copyright © 2008-2010 Intel Corporation
		3	*
		4	* Permission is hereby granted, free of charge, to any person obtaining a
		5	* copy of this software and associated documentation files (the "Software"),
		6	* to deal in the Software without restriction, including without limitation
		7	* the rights to use, copy, modify, merge, publish, distribute, sublicense,
		8	* and/or sell copies of the Software, and to permit persons to whom the
		9	* Software is furnished to do so, subject to the following conditions:
		10	*
		11	* The above copyright notice and this permission notice (including the next
		12	* paragraph) shall be included in all copies or substantial portions of the
		13	* Software.
		14	*
		15	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
		16	* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
		17	* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
		18	* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
		19	* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
		20	* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
		21	* IN THE SOFTWARE.
		22	*
		23	* Authors:
		24	* Eric Anholt
		25	* Zou Nan hai
		26	* Xiang Hai hao
		27	*
		28	*/
		29
3031	serge	30	#include
2332	Serge	31	#include "i915_drv.h"
3031	serge	32	#include
2351	Serge	33	#include "i915_trace.h"
2332	Serge	34	#include "intel_drv.h"
		35
5354	serge	36	bool
		37	intel_ring_initialized(struct intel_engine_cs *ring)
		38	{
		39	struct drm_device *dev = ring->dev;
5060	serge	40
5354	serge	41	if (!dev)
		42	return false;
		43
		44	if (i915.enable_execlists) {
		45	struct intel_context *dctx = ring->default_context;
		46	struct intel_ringbuffer *ringbuf = dctx->engine[ring->id].ringbuf;
		47
		48	return ringbuf->obj;
		49	} else
		50	return ring->buffer && ring->buffer->obj;
		51	}
		52
		53	int __intel_ring_space(int head, int tail, int size)
2332	Serge	54	{
6084	serge	55	int space = head - tail;
		56	if (space <= 0)
5060	serge	57	space += size;
6084	serge	58	return space - I915_RING_FREE_SPACE;
2332	Serge	59	}
		60
6084	serge	61	void intel_ring_update_space(struct intel_ringbuffer *ringbuf)
		62	{
		63	if (ringbuf->last_retired_head != -1) {
		64	ringbuf->head = ringbuf->last_retired_head;
		65	ringbuf->last_retired_head = -1;
		66	}
		67
		68	ringbuf->space = __intel_ring_space(ringbuf->head & HEAD_ADDR,
		69	ringbuf->tail, ringbuf->size);
		70	}
		71
5354	serge	72	int intel_ring_space(struct intel_ringbuffer *ringbuf)
4560	Serge	73	{
6084	serge	74	intel_ring_update_space(ringbuf);
		75	return ringbuf->space;
5060	serge	76	}
		77
5354	serge	78	bool intel_ring_stopped(struct intel_engine_cs *ring)
5060	serge	79	{
4560	Serge	80	struct drm_i915_private *dev_priv = ring->dev->dev_private;
5060	serge	81	return dev_priv->gpu_error.stop_rings & intel_ring_flag(ring);
		82	}
4560	Serge	83
6084	serge	84	static void __intel_ring_advance(struct intel_engine_cs *ring)
5060	serge	85	{
		86	struct intel_ringbuffer *ringbuf = ring->buffer;
		87	ringbuf->tail &= ringbuf->size - 1;
		88	if (intel_ring_stopped(ring))
4560	Serge	89	return;
5060	serge	90	ring->write_tail(ring, ringbuf->tail);
4560	Serge	91	}
		92
3031	serge	93	static int
6084	serge	94	gen2_render_ring_flush(struct drm_i915_gem_request *req,
3031	serge	95	u32 invalidate_domains,
		96	u32 flush_domains)
2332	Serge	97	{
6084	serge	98	struct intel_engine_cs *ring = req->ring;
3031	serge	99	u32 cmd;
		100	int ret;
2332	Serge	101
3031	serge	102	cmd = MI_FLUSH;
		103	if (((invalidate_domains\|flush_domains) & I915_GEM_DOMAIN_RENDER) == 0)
		104	cmd \|= MI_NO_WRITE_FLUSH;
2332	Serge	105
3031	serge	106	if (invalidate_domains & I915_GEM_DOMAIN_SAMPLER)
		107	cmd \|= MI_READ_FLUSH;
2332	Serge	108
6084	serge	109	ret = intel_ring_begin(req, 2);
3031	serge	110	if (ret)
		111	return ret;
		112
		113	intel_ring_emit(ring, cmd);
		114	intel_ring_emit(ring, MI_NOOP);
		115	intel_ring_advance(ring);
		116
		117	return 0;
2332	Serge	118	}
		119
		120	static int
6084	serge	121	gen4_render_ring_flush(struct drm_i915_gem_request *req,
		122	u32 invalidate_domains,
		123	u32 flush_domains)
2332	Serge	124	{
6084	serge	125	struct intel_engine_cs *ring = req->ring;
2332	Serge	126	struct drm_device *dev = ring->dev;
		127	u32 cmd;
		128	int ret;
		129
		130	/*
		131	* read/write caches:
		132	*
		133	* I915_GEM_DOMAIN_RENDER is always invalidated, but is
		134	* only flushed if MI_NO_WRITE_FLUSH is unset. On 965, it is
		135	* also flushed at 2d versus 3d pipeline switches.
		136	*
		137	* read-only caches:
		138	*
		139	* I915_GEM_DOMAIN_SAMPLER is flushed on pre-965 if
		140	* MI_READ_FLUSH is set, and is always flushed on 965.
		141	*
		142	* I915_GEM_DOMAIN_COMMAND may not exist?
		143	*
		144	* I915_GEM_DOMAIN_INSTRUCTION, which exists on 965, is
		145	* invalidated when MI_EXE_FLUSH is set.
		146	*
		147	* I915_GEM_DOMAIN_VERTEX, which exists on 965, is
		148	* invalidated with every MI_FLUSH.
		149	*
		150	* TLBs:
		151	*
		152	* On 965, TLBs associated with I915_GEM_DOMAIN_COMMAND
		153	* and I915_GEM_DOMAIN_CPU in are invalidated at PTE write and
		154	* I915_GEM_DOMAIN_RENDER and I915_GEM_DOMAIN_SAMPLER
		155	* are flushed at any MI_FLUSH.
		156	*/
		157
		158	cmd = MI_FLUSH \| MI_NO_WRITE_FLUSH;
3031	serge	159	if ((invalidate_domains\|flush_domains) & I915_GEM_DOMAIN_RENDER)
2332	Serge	160	cmd &= ~MI_NO_WRITE_FLUSH;
		161	if (invalidate_domains & I915_GEM_DOMAIN_INSTRUCTION)
		162	cmd \|= MI_EXE_FLUSH;
		163
		164	if (invalidate_domains & I915_GEM_DOMAIN_COMMAND &&
		165	(IS_G4X(dev) \|\| IS_GEN5(dev)))
		166	cmd \|= MI_INVALIDATE_ISP;
		167
6084	serge	168	ret = intel_ring_begin(req, 2);
2332	Serge	169	if (ret)
		170	return ret;
		171
		172	intel_ring_emit(ring, cmd);
		173	intel_ring_emit(ring, MI_NOOP);
		174	intel_ring_advance(ring);
		175
		176	return 0;
		177	}
		178
2342	Serge	179	/**
		180	* Emits a PIPE_CONTROL with a non-zero post-sync operation, for
		181	* implementing two workarounds on gen6. From section 1.4.7.1
		182	* "PIPE_CONTROL" of the Sandy Bridge PRM volume 2 part 1:
		183	*
		184	* [DevSNB-C+{W/A}] Before any depth stall flush (including those
		185	* produced by non-pipelined state commands), software needs to first
		186	* send a PIPE_CONTROL with no bits set except Post-Sync Operation !=
		187	* 0.
		188	*
		189	* [Dev-SNB{W/A}]: Before a PIPE_CONTROL with Write Cache Flush Enable
		190	* =1, a PIPE_CONTROL with any non-zero post-sync-op is required.
		191	*
		192	* And the workaround for these two requires this workaround first:
		193	*
		194	* [Dev-SNB{W/A}]: Pipe-control with CS-stall bit set must be sent
		195	* BEFORE the pipe-control with a post-sync op and no write-cache
		196	* flushes.
		197	*
		198	* And this last workaround is tricky because of the requirements on
		199	* that bit. From section 1.4.7.2.3 "Stall" of the Sandy Bridge PRM
		200	* volume 2 part 1:
		201	*
		202	* "1 of the following must also be set:
		203	* - Render Target Cache Flush Enable ([12] of DW1)
		204	* - Depth Cache Flush Enable ([0] of DW1)
		205	* - Stall at Pixel Scoreboard ([1] of DW1)
		206	* - Depth Stall ([13] of DW1)
		207	* - Post-Sync Operation ([13] of DW1)
		208	* - Notify Enable ([8] of DW1)"
		209	*
		210	* The cache flushes require the workaround flush that triggered this
		211	* one, so we can't use it. Depth stall would trigger the same.
		212	* Post-sync nonzero is what triggered this second workaround, so we
		213	* can't use that one either. Notify enable is IRQs, which aren't
		214	* really our business. That leaves only stall at scoreboard.
		215	*/
		216	static int
6084	serge	217	intel_emit_post_sync_nonzero_flush(struct drm_i915_gem_request *req)
2342	Serge	218	{
6084	serge	219	struct intel_engine_cs *ring = req->ring;
5060	serge	220	u32 scratch_addr = ring->scratch.gtt_offset + 2 * CACHELINE_BYTES;
2342	Serge	221	int ret;
		222
6084	serge	223	ret = intel_ring_begin(req, 6);
2342	Serge	224	if (ret)
		225	return ret;
		226
		227	intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(5));
		228	intel_ring_emit(ring, PIPE_CONTROL_CS_STALL \|
		229	PIPE_CONTROL_STALL_AT_SCOREBOARD);
		230	intel_ring_emit(ring, scratch_addr \| PIPE_CONTROL_GLOBAL_GTT); /* address */
		231	intel_ring_emit(ring, 0); /* low dword */
		232	intel_ring_emit(ring, 0); /* high dword */
		233	intel_ring_emit(ring, MI_NOOP);
		234	intel_ring_advance(ring);
		235
6084	serge	236	ret = intel_ring_begin(req, 6);
2342	Serge	237	if (ret)
		238	return ret;
		239
		240	intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(5));
		241	intel_ring_emit(ring, PIPE_CONTROL_QW_WRITE);
		242	intel_ring_emit(ring, scratch_addr \| PIPE_CONTROL_GLOBAL_GTT); /* address */
		243	intel_ring_emit(ring, 0);
		244	intel_ring_emit(ring, 0);
		245	intel_ring_emit(ring, MI_NOOP);
		246	intel_ring_advance(ring);
		247
		248	return 0;
		249	}
		250
		251	static int
6084	serge	252	gen6_render_ring_flush(struct drm_i915_gem_request *req,
		253	u32 invalidate_domains, u32 flush_domains)
2342	Serge	254	{
6084	serge	255	struct intel_engine_cs *ring = req->ring;
2342	Serge	256	u32 flags = 0;
5060	serge	257	u32 scratch_addr = ring->scratch.gtt_offset + 2 * CACHELINE_BYTES;
2342	Serge	258	int ret;
		259
		260	/* Force SNB workarounds for PIPE_CONTROL flushes */
6084	serge	261	ret = intel_emit_post_sync_nonzero_flush(req);
3031	serge	262	if (ret)
		263	return ret;
2342	Serge	264
		265	/* Just flush everything. Experiments have shown that reducing the
		266	* number of bits based on the write domains has little performance
		267	* impact.
		268	*/
3031	serge	269	if (flush_domains) {
		270	flags \|= PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH;
		271	flags \|= PIPE_CONTROL_DEPTH_CACHE_FLUSH;
		272	/*
		273	* Ensure that any following seqno writes only happen
		274	* when the render cache is indeed flushed.
		275	*/
		276	flags \|= PIPE_CONTROL_CS_STALL;
		277	}
		278	if (invalidate_domains) {
		279	flags \|= PIPE_CONTROL_TLB_INVALIDATE;
		280	flags \|= PIPE_CONTROL_INSTRUCTION_CACHE_INVALIDATE;
		281	flags \|= PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE;
		282	flags \|= PIPE_CONTROL_VF_CACHE_INVALIDATE;
		283	flags \|= PIPE_CONTROL_CONST_CACHE_INVALIDATE;
		284	flags \|= PIPE_CONTROL_STATE_CACHE_INVALIDATE;
		285	/*
		286	* TLB invalidate requires a post-sync write.
		287	*/
3243	Serge	288	flags \|= PIPE_CONTROL_QW_WRITE \| PIPE_CONTROL_CS_STALL;
3031	serge	289	}
		290
6084	serge	291	ret = intel_ring_begin(req, 4);
3031	serge	292	if (ret)
		293	return ret;
		294
		295	intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(4));
		296	intel_ring_emit(ring, flags);
		297	intel_ring_emit(ring, scratch_addr \| PIPE_CONTROL_GLOBAL_GTT);
		298	intel_ring_emit(ring, 0);
		299	intel_ring_advance(ring);
		300
		301	return 0;
		302	}
		303
		304	static int
6084	serge	305	gen7_render_ring_cs_stall_wa(struct drm_i915_gem_request *req)
3031	serge	306	{
6084	serge	307	struct intel_engine_cs *ring = req->ring;
3031	serge	308	int ret;
		309
6084	serge	310	ret = intel_ring_begin(req, 4);
3031	serge	311	if (ret)
		312	return ret;
		313
		314	intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(4));
		315	intel_ring_emit(ring, PIPE_CONTROL_CS_STALL \|
		316	PIPE_CONTROL_STALL_AT_SCOREBOARD);
		317	intel_ring_emit(ring, 0);
		318	intel_ring_emit(ring, 0);
		319	intel_ring_advance(ring);
		320
		321	return 0;
		322	}
		323
		324	static int
6084	serge	325	gen7_render_ring_flush(struct drm_i915_gem_request *req,
3031	serge	326	u32 invalidate_domains, u32 flush_domains)
		327	{
6084	serge	328	struct intel_engine_cs *ring = req->ring;
3031	serge	329	u32 flags = 0;
5060	serge	330	u32 scratch_addr = ring->scratch.gtt_offset + 2 * CACHELINE_BYTES;
3031	serge	331	int ret;
		332
		333	/*
		334	* Ensure that any following seqno writes only happen when the render
		335	* cache is indeed flushed.
		336	*
		337	* Workaround: 4th PIPE_CONTROL command (except the ones with only
		338	* read-cache invalidate bits set) must have the CS_STALL bit set. We
		339	* don't try to be clever and just set it unconditionally.
		340	*/
		341	flags \|= PIPE_CONTROL_CS_STALL;
		342
		343	/* Just flush everything. Experiments have shown that reducing the
		344	* number of bits based on the write domains has little performance
		345	* impact.
		346	*/
		347	if (flush_domains) {
6084	serge	348	flags \|= PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH;
3031	serge	349	flags \|= PIPE_CONTROL_DEPTH_CACHE_FLUSH;
6320	serge	350	flags \|= PIPE_CONTROL_DC_FLUSH_ENABLE;
6084	serge	351	flags \|= PIPE_CONTROL_FLUSH_ENABLE;
3031	serge	352	}
		353	if (invalidate_domains) {
		354	flags \|= PIPE_CONTROL_TLB_INVALIDATE;
6084	serge	355	flags \|= PIPE_CONTROL_INSTRUCTION_CACHE_INVALIDATE;
		356	flags \|= PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE;
		357	flags \|= PIPE_CONTROL_VF_CACHE_INVALIDATE;
		358	flags \|= PIPE_CONTROL_CONST_CACHE_INVALIDATE;
		359	flags \|= PIPE_CONTROL_STATE_CACHE_INVALIDATE;
5354	serge	360	flags \|= PIPE_CONTROL_MEDIA_STATE_CLEAR;
3031	serge	361	/*
		362	* TLB invalidate requires a post-sync write.
		363	*/
		364	flags \|= PIPE_CONTROL_QW_WRITE;
3480	Serge	365	flags \|= PIPE_CONTROL_GLOBAL_GTT_IVB;
2342	Serge	366
5354	serge	367	flags \|= PIPE_CONTROL_STALL_AT_SCOREBOARD;
		368
3031	serge	369	/* Workaround: we must issue a pipe_control with CS-stall bit
		370	* set before a pipe_control command that has the state cache
		371	* invalidate bit set. */
6084	serge	372	gen7_render_ring_cs_stall_wa(req);
3031	serge	373	}
		374
6084	serge	375	ret = intel_ring_begin(req, 4);
2342	Serge	376	if (ret)
		377	return ret;
		378
3031	serge	379	intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(4));
2342	Serge	380	intel_ring_emit(ring, flags);
3480	Serge	381	intel_ring_emit(ring, scratch_addr);
3031	serge	382	intel_ring_emit(ring, 0);
2342	Serge	383	intel_ring_advance(ring);
		384
		385	return 0;
		386	}
		387
4560	Serge	388	static int
6084	serge	389	gen8_emit_pipe_control(struct drm_i915_gem_request *req,
5060	serge	390	u32 flags, u32 scratch_addr)
		391	{
6084	serge	392	struct intel_engine_cs *ring = req->ring;
5060	serge	393	int ret;
		394
6084	serge	395	ret = intel_ring_begin(req, 6);
5060	serge	396	if (ret)
		397	return ret;
		398
		399	intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(6));
		400	intel_ring_emit(ring, flags);
		401	intel_ring_emit(ring, scratch_addr);
		402	intel_ring_emit(ring, 0);
		403	intel_ring_emit(ring, 0);
		404	intel_ring_emit(ring, 0);
		405	intel_ring_advance(ring);
		406
		407	return 0;
		408	}
		409
		410	static int
6084	serge	411	gen8_render_ring_flush(struct drm_i915_gem_request *req,
4560	Serge	412	u32 invalidate_domains, u32 flush_domains)
		413	{
		414	u32 flags = 0;
6084	serge	415	u32 scratch_addr = req->ring->scratch.gtt_offset + 2 * CACHELINE_BYTES;
4560	Serge	416	int ret;
		417
		418	flags \|= PIPE_CONTROL_CS_STALL;
		419
		420	if (flush_domains) {
		421	flags \|= PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH;
		422	flags \|= PIPE_CONTROL_DEPTH_CACHE_FLUSH;
6320	serge	423	flags \|= PIPE_CONTROL_DC_FLUSH_ENABLE;
6084	serge	424	flags \|= PIPE_CONTROL_FLUSH_ENABLE;
4560	Serge	425	}
		426	if (invalidate_domains) {
		427	flags \|= PIPE_CONTROL_TLB_INVALIDATE;
		428	flags \|= PIPE_CONTROL_INSTRUCTION_CACHE_INVALIDATE;
		429	flags \|= PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE;
		430	flags \|= PIPE_CONTROL_VF_CACHE_INVALIDATE;
		431	flags \|= PIPE_CONTROL_CONST_CACHE_INVALIDATE;
		432	flags \|= PIPE_CONTROL_STATE_CACHE_INVALIDATE;
		433	flags \|= PIPE_CONTROL_QW_WRITE;
		434	flags \|= PIPE_CONTROL_GLOBAL_GTT_IVB;
		435
5060	serge	436	/* WaCsStallBeforeStateCacheInvalidate:bdw,chv */
6084	serge	437	ret = gen8_emit_pipe_control(req,
5060	serge	438	PIPE_CONTROL_CS_STALL \|
		439	PIPE_CONTROL_STALL_AT_SCOREBOARD,
		440	0);
6084	serge	441	if (ret)
		442	return ret;
5060	serge	443	}
4560	Serge	444
6084	serge	445	return gen8_emit_pipe_control(req, flags, scratch_addr);
4560	Serge	446	}
		447
5060	serge	448	static void ring_write_tail(struct intel_engine_cs *ring,
2332	Serge	449	u32 value)
		450	{
5060	serge	451	struct drm_i915_private *dev_priv = ring->dev->dev_private;
2332	Serge	452	I915_WRITE_TAIL(ring, value);
		453	}
		454
5060	serge	455	u64 intel_ring_get_active_head(struct intel_engine_cs *ring)
2332	Serge	456	{
5060	serge	457	struct drm_i915_private *dev_priv = ring->dev->dev_private;
		458	u64 acthd;
2332	Serge	459
5060	serge	460	if (INTEL_INFO(ring->dev)->gen >= 8)
		461	acthd = I915_READ64_2x32(RING_ACTHD(ring->mmio_base),
		462	RING_ACTHD_UDW(ring->mmio_base));
		463	else if (INTEL_INFO(ring->dev)->gen >= 4)
		464	acthd = I915_READ(RING_ACTHD(ring->mmio_base));
		465	else
		466	acthd = I915_READ(ACTHD);
		467
		468	return acthd;
2332	Serge	469	}
		470
5060	serge	471	static void ring_setup_phys_status_page(struct intel_engine_cs *ring)
4104	Serge	472	{
		473	struct drm_i915_private *dev_priv = ring->dev->dev_private;
		474	u32 addr;
		475
		476	addr = dev_priv->status_page_dmah->busaddr;
		477	if (INTEL_INFO(ring->dev)->gen >= 4)
		478	addr \|= (dev_priv->status_page_dmah->busaddr >> 28) & 0xf0;
		479	I915_WRITE(HWS_PGA, addr);
		480	}
		481
6084	serge	482	static void intel_ring_setup_status_page(struct intel_engine_cs *ring)
		483	{
		484	struct drm_device *dev = ring->dev;
		485	struct drm_i915_private *dev_priv = ring->dev->dev_private;
		486	u32 mmio = 0;
		487
		488	/* The ring status page addresses are no longer next to the rest of
		489	* the ring registers as of gen7.
		490	*/
		491	if (IS_GEN7(dev)) {
		492	switch (ring->id) {
		493	case RCS:
		494	mmio = RENDER_HWS_PGA_GEN7;
		495	break;
		496	case BCS:
		497	mmio = BLT_HWS_PGA_GEN7;
		498	break;
		499	/*
		500	* VCS2 actually doesn't exist on Gen7. Only shut up
		501	* gcc switch check warning
		502	*/
		503	case VCS2:
		504	case VCS:
		505	mmio = BSD_HWS_PGA_GEN7;
		506	break;
		507	case VECS:
		508	mmio = VEBOX_HWS_PGA_GEN7;
		509	break;
		510	}
		511	} else if (IS_GEN6(ring->dev)) {
		512	mmio = RING_HWS_PGA_GEN6(ring->mmio_base);
		513	} else {
		514	/* XXX: gen8 returns to sanity */
		515	mmio = RING_HWS_PGA(ring->mmio_base);
		516	}
		517
		518	I915_WRITE(mmio, (u32)ring->status_page.gfx_addr);
		519	POSTING_READ(mmio);
		520
		521	/*
		522	* Flush the TLB for this page
		523	*
		524	* FIXME: These two bits have disappeared on gen8, so a question
		525	* arises: do we still need this and if so how should we go about
		526	* invalidating the TLB?
		527	*/
		528	if (INTEL_INFO(dev)->gen >= 6 && INTEL_INFO(dev)->gen < 8) {
		529	u32 reg = RING_INSTPM(ring->mmio_base);
		530
		531	/* ring should be idle before issuing a sync flush*/
		532	WARN_ON((I915_READ_MODE(ring) & MODE_IDLE) == 0);
		533
		534	I915_WRITE(reg,
		535	_MASKED_BIT_ENABLE(INSTPM_TLB_INVALIDATE \|
		536	INSTPM_SYNC_FLUSH));
		537	if (wait_for((I915_READ(reg) & INSTPM_SYNC_FLUSH) == 0,
		538	1000))
		539	DRM_ERROR("%s: wait for SyncFlush to complete for TLB invalidation timed out\n",
		540	ring->name);
		541	}
		542	}
		543
5060	serge	544	static bool stop_ring(struct intel_engine_cs *ring)
2332	Serge	545	{
5060	serge	546	struct drm_i915_private *dev_priv = to_i915(ring->dev);
2332	Serge	547
5060	serge	548	if (!IS_GEN2(ring->dev)) {
		549	I915_WRITE_MODE(ring, _MASKED_BIT_ENABLE(STOP_RING));
5354	serge	550	if (wait_for((I915_READ_MODE(ring) & MODE_IDLE) != 0, 1000)) {
		551	DRM_ERROR("%s : timed out trying to stop ring\n", ring->name);
		552	/* Sometimes we observe that the idle flag is not
		553	* set even though the ring is empty. So double
		554	* check before giving up.
		555	*/
		556	if (I915_READ_HEAD(ring) != I915_READ_TAIL(ring))
6084	serge	557	return false;
5060	serge	558	}
		559	}
3031	serge	560
2332	Serge	561	I915_WRITE_CTL(ring, 0);
		562	I915_WRITE_HEAD(ring, 0);
		563	ring->write_tail(ring, 0);
		564
5060	serge	565	if (!IS_GEN2(ring->dev)) {
		566	(void)I915_READ_CTL(ring);
		567	I915_WRITE_MODE(ring, _MASKED_BIT_DISABLE(STOP_RING));
		568	}
2332	Serge	569
5060	serge	570	return (I915_READ_HEAD(ring) & HEAD_ADDR) == 0;
		571	}
		572
		573	static int init_ring_common(struct intel_engine_cs *ring)
		574	{
		575	struct drm_device *dev = ring->dev;
		576	struct drm_i915_private *dev_priv = dev->dev_private;
		577	struct intel_ringbuffer *ringbuf = ring->buffer;
		578	struct drm_i915_gem_object *obj = ringbuf->obj;
		579	int ret = 0;
		580
6084	serge	581	intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
5060	serge	582
		583	if (!stop_ring(ring)) {
		584	/* G45 ring initialization often fails to reset head to zero */
2332	Serge	585	DRM_DEBUG_KMS("%s head not reset to zero "
		586	"ctl %08x head %08x tail %08x start %08x\n",
		587	ring->name,
		588	I915_READ_CTL(ring),
		589	I915_READ_HEAD(ring),
		590	I915_READ_TAIL(ring),
		591	I915_READ_START(ring));
		592
5060	serge	593	if (!stop_ring(ring)) {
2332	Serge	594	DRM_ERROR("failed to set %s head to zero "
		595	"ctl %08x head %08x tail %08x start %08x\n",
		596	ring->name,
		597	I915_READ_CTL(ring),
		598	I915_READ_HEAD(ring),
		599	I915_READ_TAIL(ring),
		600	I915_READ_START(ring));
5060	serge	601	ret = -EIO;
		602	goto out;
2332	Serge	603	}
		604	}
		605
5060	serge	606	if (I915_NEED_GFX_HWS(dev))
		607	intel_ring_setup_status_page(ring);
		608	else
		609	ring_setup_phys_status_page(ring);
		610
		611	/* Enforce ordering by reading HEAD register back */
		612	I915_READ_HEAD(ring);
		613
3031	serge	614	/* Initialize the ring. This must happen _after_ we've cleared the ring
		615	* registers with the above sequence (the readback of the HEAD registers
		616	* also enforces ordering), otherwise the hw might lose the new ring
		617	* register values. */
4104	Serge	618	I915_WRITE_START(ring, i915_gem_obj_ggtt_offset(obj));
5354	serge	619
		620	/* WaClearRingBufHeadRegAtInit:ctg,elk */
		621	if (I915_READ_HEAD(ring))
		622	DRM_DEBUG("%s initialization failed [head=%08x], fudging\n",
		623	ring->name, I915_READ_HEAD(ring));
		624	I915_WRITE_HEAD(ring, 0);
		625	(void)I915_READ_HEAD(ring);
		626
2332	Serge	627	I915_WRITE_CTL(ring,
5060	serge	628	((ringbuf->size - PAGE_SIZE) & RING_NR_PAGES)
3031	serge	629	\| RING_VALID);
2332	Serge	630
		631	/* If the head is still not zero, the ring is dead */
3031	serge	632	if (wait_for((I915_READ_CTL(ring) & RING_VALID) != 0 &&
4104	Serge	633	I915_READ_START(ring) == i915_gem_obj_ggtt_offset(obj) &&
3031	serge	634	(I915_READ_HEAD(ring) & HEAD_ADDR) == 0, 50)) {
2332	Serge	635	DRM_ERROR("%s initialization failed "
5060	serge	636	"ctl %08x (valid? %d) head %08x tail %08x start %08x [expected %08lx]\n",
6084	serge	637	ring->name,
5060	serge	638	I915_READ_CTL(ring), I915_READ_CTL(ring) & RING_VALID,
		639	I915_READ_HEAD(ring), I915_READ_TAIL(ring),
		640	I915_READ_START(ring), (unsigned long)i915_gem_obj_ggtt_offset(obj));
3031	serge	641	ret = -EIO;
		642	goto out;
2332	Serge	643	}
		644
6084	serge	645	ringbuf->last_retired_head = -1;
		646	ringbuf->head = I915_READ_HEAD(ring);
		647	ringbuf->tail = I915_READ_TAIL(ring) & TAIL_ADDR;
		648	intel_ring_update_space(ringbuf);
5060	serge	649
4104	Serge	650	memset(&ring->hangcheck, 0, sizeof(ring->hangcheck));
		651
3031	serge	652	out:
6084	serge	653	intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
2332	Serge	654
3031	serge	655	return ret;
2332	Serge	656	}
		657
5354	serge	658	void
		659	intel_fini_pipe_control(struct intel_engine_cs *ring)
2332	Serge	660	{
5354	serge	661	struct drm_device *dev = ring->dev;
		662
		663	if (ring->scratch.obj == NULL)
		664	return;
		665
		666	if (INTEL_INFO(dev)->gen >= 5) {
		667	kunmap(sg_page(ring->scratch.obj->pages->sgl));
		668	i915_gem_object_ggtt_unpin(ring->scratch.obj);
		669	}
		670
		671	drm_gem_object_unreference(&ring->scratch.obj->base);
		672	ring->scratch.obj = NULL;
		673	}
		674
		675	int
		676	intel_init_pipe_control(struct intel_engine_cs *ring)
		677	{
2332	Serge	678	int ret;
		679
6084	serge	680	WARN_ON(ring->scratch.obj);
2332	Serge	681
4104	Serge	682	ring->scratch.obj = i915_gem_alloc_object(ring->dev, 4096);
		683	if (ring->scratch.obj == NULL) {
2332	Serge	684	DRM_ERROR("Failed to allocate seqno page\n");
		685	ret = -ENOMEM;
		686	goto err;
		687	}
		688
5060	serge	689	ret = i915_gem_object_set_cache_level(ring->scratch.obj, I915_CACHE_LLC);
		690	if (ret)
		691	goto err_unref;
2332	Serge	692
5060	serge	693	ret = i915_gem_obj_ggtt_pin(ring->scratch.obj, 4096, 0);
2332	Serge	694	if (ret)
		695	goto err_unref;
		696
4104	Serge	697	ring->scratch.gtt_offset = i915_gem_obj_ggtt_offset(ring->scratch.obj);
5354	serge	698	ring->scratch.cpu_page = kmap(sg_page(ring->scratch.obj->pages->sgl));
4104	Serge	699	if (ring->scratch.cpu_page == NULL) {
		700	ret = -ENOMEM;
2332	Serge	701	goto err_unpin;
4104	Serge	702	}
2332	Serge	703
3480	Serge	704	DRM_DEBUG_DRIVER("%s pipe control offset: 0x%08x\n",
4104	Serge	705	ring->name, ring->scratch.gtt_offset);
2332	Serge	706	return 0;
		707
		708	err_unpin:
5060	serge	709	i915_gem_object_ggtt_unpin(ring->scratch.obj);
2332	Serge	710	err_unref:
4104	Serge	711	drm_gem_object_unreference(&ring->scratch.obj->base);
2332	Serge	712	err:
		713	return ret;
		714	}
		715
6084	serge	716	static int intel_ring_workarounds_emit(struct drm_i915_gem_request *req)
5354	serge	717	{
		718	int ret, i;
6084	serge	719	struct intel_engine_cs *ring = req->ring;
5354	serge	720	struct drm_device *dev = ring->dev;
		721	struct drm_i915_private *dev_priv = dev->dev_private;
		722	struct i915_workarounds *w = &dev_priv->workarounds;
		723
6084	serge	724	if (w->count == 0)
5354	serge	725	return 0;
		726
		727	ring->gpu_caches_dirty = true;
6084	serge	728	ret = intel_ring_flush_all_caches(req);
5354	serge	729	if (ret)
		730	return ret;
		731
6084	serge	732	ret = intel_ring_begin(req, (w->count * 2 + 2));
5354	serge	733	if (ret)
		734	return ret;
		735
		736	intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(w->count));
		737	for (i = 0; i < w->count; i++) {
		738	intel_ring_emit(ring, w->reg[i].addr);
		739	intel_ring_emit(ring, w->reg[i].value);
		740	}
		741	intel_ring_emit(ring, MI_NOOP);
		742
		743	intel_ring_advance(ring);
		744
		745	ring->gpu_caches_dirty = true;
6084	serge	746	ret = intel_ring_flush_all_caches(req);
5354	serge	747	if (ret)
		748	return ret;
		749
		750	DRM_DEBUG_DRIVER("Number of Workarounds emitted: %d\n", w->count);
		751
		752	return 0;
		753	}
		754
6084	serge	755	static int intel_rcs_ctx_init(struct drm_i915_gem_request *req)
		756	{
		757	int ret;
		758
		759	ret = intel_ring_workarounds_emit(req);
		760	if (ret != 0)
		761	return ret;
		762
		763	ret = i915_gem_render_state_init(req);
		764	if (ret)
		765	DRM_ERROR("init render state: %d\n", ret);
		766
		767	return ret;
		768	}
		769
5354	serge	770	static int wa_add(struct drm_i915_private *dev_priv,
		771	const u32 addr, const u32 mask, const u32 val)
		772	{
		773	const u32 idx = dev_priv->workarounds.count;
		774
		775	if (WARN_ON(idx >= I915_MAX_WA_REGS))
		776	return -ENOSPC;
		777
		778	dev_priv->workarounds.reg[idx].addr = addr;
		779	dev_priv->workarounds.reg[idx].value = val;
		780	dev_priv->workarounds.reg[idx].mask = mask;
		781
		782	dev_priv->workarounds.count++;
		783
		784	return 0;
		785	}
		786
6084	serge	787	#define WA_REG(addr, mask, val) do { \
5354	serge	788	const int r = wa_add(dev_priv, (addr), (mask), (val)); \
		789	if (r) \
		790	return r; \
6084	serge	791	} while (0)
5354	serge	792
		793	#define WA_SET_BIT_MASKED(addr, mask) \
		794	WA_REG(addr, (mask), _MASKED_BIT_ENABLE(mask))
		795
		796	#define WA_CLR_BIT_MASKED(addr, mask) \
		797	WA_REG(addr, (mask), _MASKED_BIT_DISABLE(mask))
		798
		799	#define WA_SET_FIELD_MASKED(addr, mask, value) \
		800	WA_REG(addr, mask, _MASKED_FIELD(mask, value))
		801
		802	#define WA_SET_BIT(addr, mask) WA_REG(addr, mask, I915_READ(addr) \| (mask))
		803	#define WA_CLR_BIT(addr, mask) WA_REG(addr, mask, I915_READ(addr) & ~(mask))
		804
		805	#define WA_WRITE(addr, val) WA_REG(addr, 0xffffffff, val)
		806
6084	serge	807	static int gen8_init_workarounds(struct intel_engine_cs *ring)
5354	serge	808	{
		809	struct drm_device *dev = ring->dev;
		810	struct drm_i915_private *dev_priv = dev->dev_private;
		811
6084	serge	812	WA_SET_BIT_MASKED(INSTPM, INSTPM_FORCE_ORDERING);
5354	serge	813
6084	serge	814	/* WaDisableAsyncFlipPerfMode:bdw,chv */
		815	WA_SET_BIT_MASKED(MI_MODE, ASYNC_FLIP_PERF_DISABLE);
5354	serge	816
6084	serge	817	/* WaDisablePartialInstShootdown:bdw,chv */
		818	WA_SET_BIT_MASKED(GEN8_ROW_CHICKEN,
		819	PARTIAL_INSTRUCTION_SHOOTDOWN_DISABLE);
5354	serge	820
		821	/* Use Force Non-Coherent whenever executing a 3D context. This is a
		822	* workaround for for a possible hang in the unlikely event a TLB
		823	* invalidation occurs during a PSD flush.
		824	*/
6084	serge	825	/* WaForceEnableNonCoherent:bdw,chv */
		826	/* WaHdcDisableFetchWhenMasked:bdw,chv */
5354	serge	827	WA_SET_BIT_MASKED(HDC_CHICKEN0,
6084	serge	828	HDC_DONOT_FETCH_MEM_WHEN_MASKED \|
		829	HDC_FORCE_NON_COHERENT);
5354	serge	830
6084	serge	831	/* From the Haswell PRM, Command Reference: Registers, CACHE_MODE_0:
		832	* "The Hierarchical Z RAW Stall Optimization allows non-overlapping
		833	* polygons in the same 8x4 pixel/sample area to be processed without
		834	* stalling waiting for the earlier ones to write to Hierarchical Z
		835	* buffer."
		836	*
		837	* This optimization is off by default for BDW and CHV; turn it on.
		838	*/
		839	WA_CLR_BIT_MASKED(CACHE_MODE_0_GEN7, HIZ_RAW_STALL_OPT_DISABLE);
5354	serge	840
6084	serge	841	/* Wa4x4STCOptimizationDisable:bdw,chv */
		842	WA_SET_BIT_MASKED(CACHE_MODE_1, GEN8_4x4_STC_OPTIMIZATION_DISABLE);
		843
5354	serge	844	/*
		845	* BSpec recommends 8x4 when MSAA is used,
		846	* however in practice 16x4 seems fastest.
		847	*
		848	* Note that PS/WM thread counts depend on the WIZ hashing
		849	* disable bit, which we don't touch here, but it's good
		850	* to keep in mind (see 3DSTATE_PS and 3DSTATE_WM).
		851	*/
		852	WA_SET_FIELD_MASKED(GEN7_GT_MODE,
		853	GEN6_WIZ_HASHING_MASK,
		854	GEN6_WIZ_HASHING_16x4);
		855
		856	return 0;
		857	}
		858
6084	serge	859	static int bdw_init_workarounds(struct intel_engine_cs *ring)
		860	{
		861	int ret;
		862	struct drm_device *dev = ring->dev;
		863	struct drm_i915_private *dev_priv = dev->dev_private;
		864
		865	ret = gen8_init_workarounds(ring);
		866	if (ret)
		867	return ret;
		868
		869	/* WaDisableThreadStallDopClockGating:bdw (pre-production) */
		870	WA_SET_BIT_MASKED(GEN8_ROW_CHICKEN, STALL_DOP_GATING_DISABLE);
		871
		872	/* WaDisableDopClockGating:bdw */
		873	WA_SET_BIT_MASKED(GEN7_ROW_CHICKEN2,
		874	DOP_CLOCK_GATING_DISABLE);
		875
		876	WA_SET_BIT_MASKED(HALF_SLICE_CHICKEN3,
		877	GEN8_SAMPLER_POWER_BYPASS_DIS);
		878
		879	WA_SET_BIT_MASKED(HDC_CHICKEN0,
		880	/* WaForceContextSaveRestoreNonCoherent:bdw */
		881	HDC_FORCE_CONTEXT_SAVE_RESTORE_NON_COHERENT \|
		882	/* WaDisableFenceDestinationToSLM:bdw (pre-prod) */
		883	(IS_BDW_GT3(dev) ? HDC_FENCE_DEST_SLM_DISABLE : 0));
		884
		885	return 0;
		886	}
		887
5354	serge	888	static int chv_init_workarounds(struct intel_engine_cs *ring)
		889	{
6084	serge	890	int ret;
5354	serge	891	struct drm_device *dev = ring->dev;
		892	struct drm_i915_private *dev_priv = dev->dev_private;
		893
6084	serge	894	ret = gen8_init_workarounds(ring);
		895	if (ret)
		896	return ret;
		897
5354	serge	898	/* WaDisableThreadStallDopClockGating:chv */
6084	serge	899	WA_SET_BIT_MASKED(GEN8_ROW_CHICKEN, STALL_DOP_GATING_DISABLE);
		900
		901	/* Improve HiZ throughput on CHV. */
		902	WA_SET_BIT_MASKED(HIZ_CHICKEN, CHV_HZ_8X8_MODE_IN_1X);
		903
		904	return 0;
		905	}
		906
		907	static int gen9_init_workarounds(struct intel_engine_cs *ring)
		908	{
		909	struct drm_device *dev = ring->dev;
		910	struct drm_i915_private *dev_priv = dev->dev_private;
		911	uint32_t tmp;
		912
		913	/* WaEnableLbsSlaRetryTimerDecrement:skl */
		914	I915_WRITE(BDW_SCRATCH1, I915_READ(BDW_SCRATCH1) \|
		915	GEN9_LBS_SLA_RETRY_TIMER_DECREMENT_ENABLE);
		916
		917	/* WaDisableKillLogic:bxt,skl */
		918	I915_WRITE(GAM_ECOCHK, I915_READ(GAM_ECOCHK) \|
		919	ECOCHK_DIS_TLB);
		920
		921	/* WaDisablePartialInstShootdown:skl,bxt */
5354	serge	922	WA_SET_BIT_MASKED(GEN8_ROW_CHICKEN,
6084	serge	923	PARTIAL_INSTRUCTION_SHOOTDOWN_DISABLE);
5354	serge	924
6084	serge	925	/* Syncing dependencies between camera and graphics:skl,bxt */
		926	WA_SET_BIT_MASKED(HALF_SLICE_CHICKEN3,
		927	GEN9_DISABLE_OCL_OOB_SUPPRESS_LOGIC);
		928
		929	if ((IS_SKYLAKE(dev) && (INTEL_REVID(dev) == SKL_REVID_A0 \|\|
		930	INTEL_REVID(dev) == SKL_REVID_B0)) \|\|
		931	(IS_BROXTON(dev) && INTEL_REVID(dev) < BXT_REVID_B0)) {
		932	/* WaDisableDgMirrorFixInHalfSliceChicken5:skl,bxt */
		933	WA_CLR_BIT_MASKED(GEN9_HALF_SLICE_CHICKEN5,
		934	GEN9_DG_MIRROR_FIX_ENABLE);
		935	}
		936
		937	if ((IS_SKYLAKE(dev) && INTEL_REVID(dev) <= SKL_REVID_B0) \|\|
		938	(IS_BROXTON(dev) && INTEL_REVID(dev) < BXT_REVID_B0)) {
		939	/* WaSetDisablePixMaskCammingAndRhwoInCommonSliceChicken:skl,bxt */
		940	WA_SET_BIT_MASKED(GEN7_COMMON_SLICE_CHICKEN1,
		941	GEN9_RHWO_OPTIMIZATION_DISABLE);
		942	/*
		943	* WA also requires GEN9_SLICE_COMMON_ECO_CHICKEN0[14:14] to be set
		944	* but we do that in per ctx batchbuffer as there is an issue
		945	* with this register not getting restored on ctx restore
		946	*/
		947	}
		948
		949	if ((IS_SKYLAKE(dev) && INTEL_REVID(dev) >= SKL_REVID_C0) \|\|
		950	IS_BROXTON(dev)) {
		951	/* WaEnableYV12BugFixInHalfSliceChicken7:skl,bxt */
		952	WA_SET_BIT_MASKED(GEN9_HALF_SLICE_CHICKEN7,
		953	GEN9_ENABLE_YV12_BUGFIX);
		954	}
		955
		956	/* Wa4x4STCOptimizationDisable:skl,bxt */
		957	/* WaDisablePartialResolveInVc:skl,bxt */
		958	WA_SET_BIT_MASKED(CACHE_MODE_1, (GEN8_4x4_STC_OPTIMIZATION_DISABLE \|
		959	GEN9_PARTIAL_RESOLVE_IN_VC_DISABLE));
		960
		961	/* WaCcsTlbPrefetchDisable:skl,bxt */
		962	WA_CLR_BIT_MASKED(GEN9_HALF_SLICE_CHICKEN5,
		963	GEN9_CCS_TLB_PREFETCH_ENABLE);
		964
		965	/* WaDisableMaskBasedCammingInRCC:skl,bxt */
		966	if ((IS_SKYLAKE(dev) && INTEL_REVID(dev) == SKL_REVID_C0) \|\|
		967	(IS_BROXTON(dev) && INTEL_REVID(dev) < BXT_REVID_B0))
		968	WA_SET_BIT_MASKED(SLICE_ECO_CHICKEN0,
		969	PIXEL_MASK_CAMMING_DISABLE);
		970
		971	/* WaForceContextSaveRestoreNonCoherent:skl,bxt */
		972	tmp = HDC_FORCE_CONTEXT_SAVE_RESTORE_NON_COHERENT;
		973	if ((IS_SKYLAKE(dev) && INTEL_REVID(dev) == SKL_REVID_F0) \|\|
		974	(IS_BROXTON(dev) && INTEL_REVID(dev) >= BXT_REVID_B0))
		975	tmp \|= HDC_FORCE_CSR_NON_COHERENT_OVR_DISABLE;
		976	WA_SET_BIT_MASKED(HDC_CHICKEN0, tmp);
		977
		978	/* WaDisableSamplerPowerBypassForSOPingPong:skl,bxt */
		979	if (IS_SKYLAKE(dev) \|\|
		980	(IS_BROXTON(dev) && INTEL_REVID(dev) <= BXT_REVID_B0)) {
		981	WA_SET_BIT_MASKED(HALF_SLICE_CHICKEN3,
		982	GEN8_SAMPLER_POWER_BYPASS_DIS);
		983	}
		984
		985	/* WaDisableSTUnitPowerOptimization:skl,bxt */
		986	WA_SET_BIT_MASKED(HALF_SLICE_CHICKEN2, GEN8_ST_PO_DISABLE);
		987
		988	return 0;
		989	}
		990
		991	static int skl_tune_iz_hashing(struct intel_engine_cs *ring)
		992	{
		993	struct drm_device *dev = ring->dev;
		994	struct drm_i915_private *dev_priv = dev->dev_private;
		995	u8 vals[3] = { 0, 0, 0 };
		996	unsigned int i;
		997
		998	for (i = 0; i < 3; i++) {
		999	u8 ss;
		1000
		1001	/*
		1002	* Only consider slices where one, and only one, subslice has 7
		1003	* EUs
		1004	*/
		1005	if (hweight8(dev_priv->info.subslice_7eu[i]) != 1)
		1006	continue;
		1007
		1008	/*
		1009	* subslice_7eu[i] != 0 (because of the check above) and
		1010	* ss_max == 4 (maximum number of subslices possible per slice)
		1011	*
		1012	* -> 0 <= ss <= 3;
		1013	*/
		1014	ss = ffs(dev_priv->info.subslice_7eu[i]) - 1;
		1015	vals[i] = 3 - ss;
		1016	}
		1017
		1018	if (vals[0] == 0 && vals[1] == 0 && vals[2] == 0)
		1019	return 0;
		1020
		1021	/* Tune IZ hashing. See intel_device_info_runtime_init() */
		1022	WA_SET_FIELD_MASKED(GEN7_GT_MODE,
		1023	GEN9_IZ_HASHING_MASK(2) \|
		1024	GEN9_IZ_HASHING_MASK(1) \|
		1025	GEN9_IZ_HASHING_MASK(0),
		1026	GEN9_IZ_HASHING(2, vals[2]) \|
		1027	GEN9_IZ_HASHING(1, vals[1]) \|
		1028	GEN9_IZ_HASHING(0, vals[0]));
		1029
		1030	return 0;
		1031	}
		1032
		1033	static int skl_init_workarounds(struct intel_engine_cs *ring)
		1034	{
		1035	int ret;
		1036	struct drm_device *dev = ring->dev;
		1037	struct drm_i915_private *dev_priv = dev->dev_private;
		1038
		1039	ret = gen9_init_workarounds(ring);
		1040	if (ret)
		1041	return ret;
		1042
		1043	if (INTEL_REVID(dev) <= SKL_REVID_D0) {
		1044	/* WaDisableHDCInvalidation:skl */
		1045	I915_WRITE(GAM_ECOCHK, I915_READ(GAM_ECOCHK) \|
		1046	BDW_DISABLE_HDC_INVALIDATION);
		1047
		1048	/* WaDisableChickenBitTSGBarrierAckForFFSliceCS:skl */
		1049	I915_WRITE(FF_SLICE_CS_CHICKEN2,
		1050	_MASKED_BIT_ENABLE(GEN9_TSG_BARRIER_ACK_DISABLE));
		1051	}
		1052
		1053	/* GEN8_L3SQCREG4 has a dependency with WA batch so any new changes
		1054	* involving this register should also be added to WA batch as required.
5354	serge	1055	*/
6084	serge	1056	if (INTEL_REVID(dev) <= SKL_REVID_E0)
		1057	/* WaDisableLSQCROPERFforOCL:skl */
		1058	I915_WRITE(GEN8_L3SQCREG4, I915_READ(GEN8_L3SQCREG4) \|
		1059	GEN8_LQSC_RO_PERF_DIS);
5354	serge	1060
6084	serge	1061	/* WaEnableGapsTsvCreditFix:skl */
		1062	if (IS_SKYLAKE(dev) && (INTEL_REVID(dev) >= SKL_REVID_C0)) {
		1063	I915_WRITE(GEN8_GARBCNTL, (I915_READ(GEN8_GARBCNTL) \|
		1064	GEN9_GAPS_TSV_CREDIT_DISABLE));
		1065	}
		1066
		1067	/* WaDisablePowerCompilerClockGating:skl */
		1068	if (INTEL_REVID(dev) == SKL_REVID_B0)
		1069	WA_SET_BIT_MASKED(HIZ_CHICKEN,
		1070	BDW_HIZ_POWER_COMPILER_CLOCK_GATING_DISABLE);
		1071
		1072	if (INTEL_REVID(dev) <= SKL_REVID_D0) {
		1073	/*
		1074	*Use Force Non-Coherent whenever executing a 3D context. This
		1075	* is a workaround for a possible hang in the unlikely event
		1076	* a TLB invalidation occurs during a PSD flush.
		1077	*/
		1078	/* WaForceEnableNonCoherent:skl */
		1079	WA_SET_BIT_MASKED(HDC_CHICKEN0,
		1080	HDC_FORCE_NON_COHERENT);
		1081	}
		1082
		1083	if (INTEL_REVID(dev) == SKL_REVID_C0 \|\|
		1084	INTEL_REVID(dev) == SKL_REVID_D0)
		1085	/* WaBarrierPerformanceFixDisable:skl */
		1086	WA_SET_BIT_MASKED(HDC_CHICKEN0,
		1087	HDC_FENCE_DEST_SLM_DISABLE \|
		1088	HDC_BARRIER_PERFORMANCE_DISABLE);
		1089
		1090	/* WaDisableSbeCacheDispatchPortSharing:skl */
		1091	if (INTEL_REVID(dev) <= SKL_REVID_F0) {
		1092	WA_SET_BIT_MASKED(
		1093	GEN7_HALF_SLICE_CHICKEN1,
		1094	GEN7_SBE_SS_CACHE_DISPATCH_PORT_SHARING_DISABLE);
		1095	}
		1096
		1097	return skl_tune_iz_hashing(ring);
		1098	}
		1099
		1100	static int bxt_init_workarounds(struct intel_engine_cs *ring)
		1101	{
		1102	int ret;
		1103	struct drm_device *dev = ring->dev;
		1104	struct drm_i915_private *dev_priv = dev->dev_private;
		1105
		1106	ret = gen9_init_workarounds(ring);
		1107	if (ret)
		1108	return ret;
		1109
		1110	/* WaStoreMultiplePTEenable:bxt */
		1111	/* This is a requirement according to Hardware specification */
		1112	if (INTEL_REVID(dev) == BXT_REVID_A0)
		1113	I915_WRITE(TILECTL, I915_READ(TILECTL) \| TILECTL_TLBPF);
		1114
		1115	/* WaSetClckGatingDisableMedia:bxt */
		1116	if (INTEL_REVID(dev) == BXT_REVID_A0) {
		1117	I915_WRITE(GEN7_MISCCPCTL, (I915_READ(GEN7_MISCCPCTL) &
		1118	~GEN8_DOP_CLOCK_GATE_MEDIA_ENABLE));
		1119	}
		1120
		1121	/* WaDisableThreadStallDopClockGating:bxt */
		1122	WA_SET_BIT_MASKED(GEN8_ROW_CHICKEN,
		1123	STALL_DOP_GATING_DISABLE);
		1124
		1125	/* WaDisableSbeCacheDispatchPortSharing:bxt */
		1126	if (INTEL_REVID(dev) <= BXT_REVID_B0) {
		1127	WA_SET_BIT_MASKED(
		1128	GEN7_HALF_SLICE_CHICKEN1,
		1129	GEN7_SBE_SS_CACHE_DISPATCH_PORT_SHARING_DISABLE);
		1130	}
		1131
5354	serge	1132	return 0;
		1133	}
		1134
		1135	int init_workarounds_ring(struct intel_engine_cs *ring)
		1136	{
		1137	struct drm_device *dev = ring->dev;
		1138	struct drm_i915_private *dev_priv = dev->dev_private;
		1139
		1140	WARN_ON(ring->id != RCS);
		1141
		1142	dev_priv->workarounds.count = 0;
		1143
		1144	if (IS_BROADWELL(dev))
		1145	return bdw_init_workarounds(ring);
		1146
		1147	if (IS_CHERRYVIEW(dev))
		1148	return chv_init_workarounds(ring);
		1149
6084	serge	1150	if (IS_SKYLAKE(dev))
		1151	return skl_init_workarounds(ring);
		1152
		1153	if (IS_BROXTON(dev))
		1154	return bxt_init_workarounds(ring);
		1155
5354	serge	1156	return 0;
		1157	}
		1158
5060	serge	1159	static int init_render_ring(struct intel_engine_cs *ring)
2332	Serge	1160	{
		1161	struct drm_device *dev = ring->dev;
		1162	struct drm_i915_private *dev_priv = dev->dev_private;
		1163	int ret = init_ring_common(ring);
5060	serge	1164	if (ret)
		1165	return ret;
2332	Serge	1166
5060	serge	1167	/* WaTimedSingleVertexDispatch:cl,bw,ctg,elk,ilk,snb */
		1168	if (INTEL_INFO(dev)->gen >= 4 && INTEL_INFO(dev)->gen < 7)
3031	serge	1169	I915_WRITE(MI_MODE, _MASKED_BIT_ENABLE(VS_TIMER_DISPATCH));
3243	Serge	1170
		1171	/* We need to disable the AsyncFlip performance optimisations in order
		1172	* to use MI_WAIT_FOR_EVENT within the CS. It should already be
		1173	* programmed to '1' on all products.
4104	Serge	1174	*
6084	serge	1175	* WaDisableAsyncFlipPerfMode:snb,ivb,hsw,vlv
3243	Serge	1176	*/
6084	serge	1177	if (INTEL_INFO(dev)->gen >= 6 && INTEL_INFO(dev)->gen < 8)
3243	Serge	1178	I915_WRITE(MI_MODE, _MASKED_BIT_ENABLE(ASYNC_FLIP_PERF_DISABLE));
		1179
		1180	/* Required for the hardware to program scanline values for waiting */
5060	serge	1181	/* WaEnableFlushTlbInvalidationMode:snb */
3243	Serge	1182	if (INTEL_INFO(dev)->gen == 6)
		1183	I915_WRITE(GFX_MODE,
5060	serge	1184	_MASKED_BIT_ENABLE(GFX_TLB_INVALIDATE_EXPLICIT));
3243	Serge	1185
5060	serge	1186	/* WaBCSVCSTlbInvalidationMode:ivb,vlv,hsw */
6084	serge	1187	if (IS_GEN7(dev))
		1188	I915_WRITE(GFX_MODE_GEN7,
5060	serge	1189	_MASKED_BIT_ENABLE(GFX_TLB_INVALIDATE_EXPLICIT) \|
6084	serge	1190	_MASKED_BIT_ENABLE(GFX_REPLAY_MODE));
2332	Serge	1191
3031	serge	1192	if (IS_GEN6(dev)) {
		1193	/* From the Sandybridge PRM, volume 1 part 3, page 24:
		1194	* "If this bit is set, STCunit will have LRA as replacement
		1195	* policy. [...] This bit must be reset. LRA replacement
		1196	* policy is not supported."
		1197	*/
		1198	I915_WRITE(CACHE_MODE_0,
		1199	_MASKED_BIT_DISABLE(CM0_STC_EVICT_DISABLE_LRA_SNB));
2342	Serge	1200	}
		1201
6084	serge	1202	if (INTEL_INFO(dev)->gen >= 6 && INTEL_INFO(dev)->gen < 8)
3031	serge	1203	I915_WRITE(INSTPM, _MASKED_BIT_ENABLE(INSTPM_FORCE_ORDERING));
		1204
4560	Serge	1205	if (HAS_L3_DPF(dev))
		1206	I915_WRITE_IMR(ring, ~GT_PARITY_ERROR(dev));
3031	serge	1207
5354	serge	1208	return init_workarounds_ring(ring);
2332	Serge	1209	}
		1210
5060	serge	1211	static void render_ring_cleanup(struct intel_engine_cs *ring)
2332	Serge	1212	{
3480	Serge	1213	struct drm_device *dev = ring->dev;
5128	serge	1214	struct drm_i915_private *dev_priv = dev->dev_private;
3480	Serge	1215
5128	serge	1216	if (dev_priv->semaphore_obj) {
		1217	i915_gem_object_ggtt_unpin(dev_priv->semaphore_obj);
		1218	drm_gem_object_unreference(&dev_priv->semaphore_obj->base);
		1219	dev_priv->semaphore_obj = NULL;
		1220	}
		1221
5354	serge	1222	intel_fini_pipe_control(ring);
2332	Serge	1223	}
		1224
6084	serge	1225	static int gen8_rcs_signal(struct drm_i915_gem_request *signaller_req,
5060	serge	1226	unsigned int num_dwords)
2332	Serge	1227	{
5060	serge	1228	#define MBOX_UPDATE_DWORDS 8
6084	serge	1229	struct intel_engine_cs *signaller = signaller_req->ring;
5060	serge	1230	struct drm_device *dev = signaller->dev;
		1231	struct drm_i915_private *dev_priv = dev->dev_private;
		1232	struct intel_engine_cs *waiter;
		1233	int i, ret, num_rings;
		1234
		1235	num_rings = hweight32(INTEL_INFO(dev)->ring_mask);
		1236	num_dwords += (num_rings-1) * MBOX_UPDATE_DWORDS;
		1237	#undef MBOX_UPDATE_DWORDS
		1238
6084	serge	1239	ret = intel_ring_begin(signaller_req, num_dwords);
5060	serge	1240	if (ret)
		1241	return ret;
		1242
		1243	for_each_ring(waiter, dev_priv, i) {
6084	serge	1244	u32 seqno;
5060	serge	1245	u64 gtt_offset = signaller->semaphore.signal_ggtt[i];
		1246	if (gtt_offset == MI_SEMAPHORE_SYNC_INVALID)
		1247	continue;
		1248
6084	serge	1249	seqno = i915_gem_request_get_seqno(signaller_req);
5060	serge	1250	intel_ring_emit(signaller, GFX_OP_PIPE_CONTROL(6));
		1251	intel_ring_emit(signaller, PIPE_CONTROL_GLOBAL_GTT_IVB \|
		1252	PIPE_CONTROL_QW_WRITE \|
		1253	PIPE_CONTROL_FLUSH_ENABLE);
		1254	intel_ring_emit(signaller, lower_32_bits(gtt_offset));
		1255	intel_ring_emit(signaller, upper_32_bits(gtt_offset));
6084	serge	1256	intel_ring_emit(signaller, seqno);
5060	serge	1257	intel_ring_emit(signaller, 0);
		1258	intel_ring_emit(signaller, MI_SEMAPHORE_SIGNAL \|
		1259	MI_SEMAPHORE_TARGET(waiter->id));
		1260	intel_ring_emit(signaller, 0);
		1261	}
		1262
		1263	return 0;
2332	Serge	1264	}
		1265
6084	serge	1266	static int gen8_xcs_signal(struct drm_i915_gem_request *signaller_req,
5060	serge	1267	unsigned int num_dwords)
		1268	{
		1269	#define MBOX_UPDATE_DWORDS 6
6084	serge	1270	struct intel_engine_cs *signaller = signaller_req->ring;
5060	serge	1271	struct drm_device *dev = signaller->dev;
		1272	struct drm_i915_private *dev_priv = dev->dev_private;
		1273	struct intel_engine_cs *waiter;
		1274	int i, ret, num_rings;
		1275
		1276	num_rings = hweight32(INTEL_INFO(dev)->ring_mask);
		1277	num_dwords += (num_rings-1) * MBOX_UPDATE_DWORDS;
		1278	#undef MBOX_UPDATE_DWORDS
		1279
6084	serge	1280	ret = intel_ring_begin(signaller_req, num_dwords);
5060	serge	1281	if (ret)
		1282	return ret;
		1283
		1284	for_each_ring(waiter, dev_priv, i) {
6084	serge	1285	u32 seqno;
5060	serge	1286	u64 gtt_offset = signaller->semaphore.signal_ggtt[i];
		1287	if (gtt_offset == MI_SEMAPHORE_SYNC_INVALID)
		1288	continue;
		1289
6084	serge	1290	seqno = i915_gem_request_get_seqno(signaller_req);
5060	serge	1291	intel_ring_emit(signaller, (MI_FLUSH_DW + 1) \|
		1292	MI_FLUSH_DW_OP_STOREDW);
		1293	intel_ring_emit(signaller, lower_32_bits(gtt_offset) \|
		1294	MI_FLUSH_DW_USE_GTT);
		1295	intel_ring_emit(signaller, upper_32_bits(gtt_offset));
6084	serge	1296	intel_ring_emit(signaller, seqno);
5060	serge	1297	intel_ring_emit(signaller, MI_SEMAPHORE_SIGNAL \|
		1298	MI_SEMAPHORE_TARGET(waiter->id));
		1299	intel_ring_emit(signaller, 0);
		1300	}
		1301
		1302	return 0;
		1303	}
		1304
6084	serge	1305	static int gen6_signal(struct drm_i915_gem_request *signaller_req,
5060	serge	1306	unsigned int num_dwords)
		1307	{
6084	serge	1308	struct intel_engine_cs *signaller = signaller_req->ring;
5060	serge	1309	struct drm_device *dev = signaller->dev;
		1310	struct drm_i915_private *dev_priv = dev->dev_private;
		1311	struct intel_engine_cs *useless;
		1312	int i, ret, num_rings;
		1313
		1314	#define MBOX_UPDATE_DWORDS 3
		1315	num_rings = hweight32(INTEL_INFO(dev)->ring_mask);
		1316	num_dwords += round_up((num_rings-1) * MBOX_UPDATE_DWORDS, 2);
		1317	#undef MBOX_UPDATE_DWORDS
		1318
6084	serge	1319	ret = intel_ring_begin(signaller_req, num_dwords);
5060	serge	1320	if (ret)
		1321	return ret;
		1322
		1323	for_each_ring(useless, dev_priv, i) {
		1324	u32 mbox_reg = signaller->semaphore.mbox.signal[i];
		1325	if (mbox_reg != GEN6_NOSYNC) {
6084	serge	1326	u32 seqno = i915_gem_request_get_seqno(signaller_req);
5060	serge	1327	intel_ring_emit(signaller, MI_LOAD_REGISTER_IMM(1));
		1328	intel_ring_emit(signaller, mbox_reg);
6084	serge	1329	intel_ring_emit(signaller, seqno);
5060	serge	1330	}
		1331	}
		1332
		1333	/* If num_dwords was rounded, make sure the tail pointer is correct */
		1334	if (num_rings % 2 == 0)
		1335	intel_ring_emit(signaller, MI_NOOP);
		1336
		1337	return 0;
		1338	}
		1339
2342	Serge	1340	/**
		1341	* gen6_add_request - Update the semaphore mailbox registers
		1342	*
6084	serge	1343	* @request - request to write to the ring
2342	Serge	1344	*
		1345	* Update the mailbox registers in the other rings with the current seqno.
		1346	* This acts like a signal in the canonical semaphore.
		1347	*/
2332	Serge	1348	static int
6084	serge	1349	gen6_add_request(struct drm_i915_gem_request *req)
2332	Serge	1350	{
6084	serge	1351	struct intel_engine_cs *ring = req->ring;
5060	serge	1352	int ret;
2332	Serge	1353
5060	serge	1354	if (ring->semaphore.signal)
6084	serge	1355	ret = ring->semaphore.signal(req, 4);
5060	serge	1356	else
6084	serge	1357	ret = intel_ring_begin(req, 4);
4560	Serge	1358
2332	Serge	1359	if (ret)
		1360	return ret;
		1361
		1362	intel_ring_emit(ring, MI_STORE_DWORD_INDEX);
		1363	intel_ring_emit(ring, I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
6084	serge	1364	intel_ring_emit(ring, i915_gem_request_get_seqno(req));
2332	Serge	1365	intel_ring_emit(ring, MI_USER_INTERRUPT);
4560	Serge	1366	__intel_ring_advance(ring);
2332	Serge	1367
		1368	return 0;
		1369	}
		1370
3480	Serge	1371	static inline bool i915_gem_has_seqno_wrapped(struct drm_device *dev,
		1372	u32 seqno)
		1373	{
		1374	struct drm_i915_private *dev_priv = dev->dev_private;
		1375	return dev_priv->last_seqno < seqno;
		1376	}
		1377
2342	Serge	1378	/**
		1379	* intel_ring_sync - sync the waiter to the signaller on seqno
		1380	*
		1381	* @waiter - ring that is waiting
		1382	* @signaller - ring which has, or will signal
		1383	* @seqno - seqno which the waiter will block on
		1384	*/
5060	serge	1385
2342	Serge	1386	static int
6084	serge	1387	gen8_ring_sync(struct drm_i915_gem_request *waiter_req,
5060	serge	1388	struct intel_engine_cs *signaller,
		1389	u32 seqno)
		1390	{
6084	serge	1391	struct intel_engine_cs *waiter = waiter_req->ring;
5060	serge	1392	struct drm_i915_private *dev_priv = waiter->dev->dev_private;
		1393	int ret;
		1394
6084	serge	1395	ret = intel_ring_begin(waiter_req, 4);
5060	serge	1396	if (ret)
		1397	return ret;
		1398
		1399	intel_ring_emit(waiter, MI_SEMAPHORE_WAIT \|
		1400	MI_SEMAPHORE_GLOBAL_GTT \|
		1401	MI_SEMAPHORE_POLL \|
		1402	MI_SEMAPHORE_SAD_GTE_SDD);
		1403	intel_ring_emit(waiter, seqno);
		1404	intel_ring_emit(waiter,
		1405	lower_32_bits(GEN8_WAIT_OFFSET(waiter, signaller->id)));
		1406	intel_ring_emit(waiter,
		1407	upper_32_bits(GEN8_WAIT_OFFSET(waiter, signaller->id)));
		1408	intel_ring_advance(waiter);
		1409	return 0;
		1410	}
		1411
		1412	static int
6084	serge	1413	gen6_ring_sync(struct drm_i915_gem_request *waiter_req,
5060	serge	1414	struct intel_engine_cs *signaller,
6084	serge	1415	u32 seqno)
2332	Serge	1416	{
6084	serge	1417	struct intel_engine_cs *waiter = waiter_req->ring;
2342	Serge	1418	u32 dw1 = MI_SEMAPHORE_MBOX \|
		1419	MI_SEMAPHORE_COMPARE \|
		1420	MI_SEMAPHORE_REGISTER;
5060	serge	1421	u32 wait_mbox = signaller->semaphore.mbox.wait[waiter->id];
		1422	int ret;
2332	Serge	1423
3031	serge	1424	/* Throughout all of the GEM code, seqno passed implies our current
		1425	* seqno is >= the last seqno executed. However for hardware the
		1426	* comparison is strictly greater than.
		1427	*/
		1428	seqno -= 1;
		1429
5060	serge	1430	WARN_ON(wait_mbox == MI_SEMAPHORE_SYNC_INVALID);
3031	serge	1431
6084	serge	1432	ret = intel_ring_begin(waiter_req, 4);
2332	Serge	1433	if (ret)
		1434	return ret;
		1435
3480	Serge	1436	/* If seqno wrap happened, omit the wait with no-ops */
		1437	if (likely(!i915_gem_has_seqno_wrapped(waiter->dev, seqno))) {
5060	serge	1438	intel_ring_emit(waiter, dw1 \| wait_mbox);
6084	serge	1439	intel_ring_emit(waiter, seqno);
		1440	intel_ring_emit(waiter, 0);
		1441	intel_ring_emit(waiter, MI_NOOP);
3480	Serge	1442	} else {
		1443	intel_ring_emit(waiter, MI_NOOP);
		1444	intel_ring_emit(waiter, MI_NOOP);
		1445	intel_ring_emit(waiter, MI_NOOP);
		1446	intel_ring_emit(waiter, MI_NOOP);
		1447	}
2342	Serge	1448	intel_ring_advance(waiter);
2332	Serge	1449
		1450	return 0;
		1451	}
		1452
		1453	#define PIPE_CONTROL_FLUSH(ring__, addr__) \
		1454	do { \
2342	Serge	1455	intel_ring_emit(ring__, GFX_OP_PIPE_CONTROL(4) \| PIPE_CONTROL_QW_WRITE \| \
		1456	PIPE_CONTROL_DEPTH_STALL); \
2332	Serge	1457	intel_ring_emit(ring__, (addr__) \| PIPE_CONTROL_GLOBAL_GTT); \
		1458	intel_ring_emit(ring__, 0); \
		1459	intel_ring_emit(ring__, 0); \
		1460	} while (0)
		1461
		1462	static int
6084	serge	1463	pc_render_add_request(struct drm_i915_gem_request *req)
2332	Serge	1464	{
6084	serge	1465	struct intel_engine_cs *ring = req->ring;
5060	serge	1466	u32 scratch_addr = ring->scratch.gtt_offset + 2 * CACHELINE_BYTES;
2332	Serge	1467	int ret;
		1468
		1469	/* For Ironlake, MI_USER_INTERRUPT was deprecated and apparently
		1470	* incoherent with writes to memory, i.e. completely fubar,
		1471	* so we need to use PIPE_NOTIFY instead.
		1472	*
		1473	* However, we also need to workaround the qword write
		1474	* incoherence by flushing the 6 PIPE_NOTIFY buffers out to
		1475	* memory before requesting an interrupt.
		1476	*/
6084	serge	1477	ret = intel_ring_begin(req, 32);
2332	Serge	1478	if (ret)
		1479	return ret;
		1480
2342	Serge	1481	intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(4) \| PIPE_CONTROL_QW_WRITE \|
		1482	PIPE_CONTROL_WRITE_FLUSH \|
		1483	PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE);
4104	Serge	1484	intel_ring_emit(ring, ring->scratch.gtt_offset \| PIPE_CONTROL_GLOBAL_GTT);
6084	serge	1485	intel_ring_emit(ring, i915_gem_request_get_seqno(req));
2332	Serge	1486	intel_ring_emit(ring, 0);
		1487	PIPE_CONTROL_FLUSH(ring, scratch_addr);
5060	serge	1488	scratch_addr += 2 * CACHELINE_BYTES; /* write to separate cachelines */
2332	Serge	1489	PIPE_CONTROL_FLUSH(ring, scratch_addr);
5060	serge	1490	scratch_addr += 2 * CACHELINE_BYTES;
2332	Serge	1491	PIPE_CONTROL_FLUSH(ring, scratch_addr);
5060	serge	1492	scratch_addr += 2 * CACHELINE_BYTES;
2332	Serge	1493	PIPE_CONTROL_FLUSH(ring, scratch_addr);
5060	serge	1494	scratch_addr += 2 * CACHELINE_BYTES;
2332	Serge	1495	PIPE_CONTROL_FLUSH(ring, scratch_addr);
5060	serge	1496	scratch_addr += 2 * CACHELINE_BYTES;
2332	Serge	1497	PIPE_CONTROL_FLUSH(ring, scratch_addr);
3031	serge	1498
2342	Serge	1499	intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(4) \| PIPE_CONTROL_QW_WRITE \|
		1500	PIPE_CONTROL_WRITE_FLUSH \|
		1501	PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE \|
2332	Serge	1502	PIPE_CONTROL_NOTIFY);
4104	Serge	1503	intel_ring_emit(ring, ring->scratch.gtt_offset \| PIPE_CONTROL_GLOBAL_GTT);
6084	serge	1504	intel_ring_emit(ring, i915_gem_request_get_seqno(req));
2332	Serge	1505	intel_ring_emit(ring, 0);
4560	Serge	1506	__intel_ring_advance(ring);
2332	Serge	1507
		1508	return 0;
		1509	}
		1510
		1511	static u32
5060	serge	1512	gen6_ring_get_seqno(struct intel_engine_cs *ring, bool lazy_coherency)
2342	Serge	1513	{
		1514	/* Workaround to force correct ordering between irq and seqno writes on
		1515	* ivb (and maybe also on snb) by reading from a CS register (like
		1516	* ACTHD) before reading the status page. */
5060	serge	1517	if (!lazy_coherency) {
		1518	struct drm_i915_private *dev_priv = ring->dev->dev_private;
		1519	POSTING_READ(RING_ACTHD(ring->mmio_base));
		1520	}
		1521
2342	Serge	1522	return intel_read_status_page(ring, I915_GEM_HWS_INDEX);
		1523	}
		1524
		1525	static u32
5060	serge	1526	ring_get_seqno(struct intel_engine_cs *ring, bool lazy_coherency)
2332	Serge	1527	{
		1528	return intel_read_status_page(ring, I915_GEM_HWS_INDEX);
		1529	}
		1530
3480	Serge	1531	static void
5060	serge	1532	ring_set_seqno(struct intel_engine_cs *ring, u32 seqno)
3480	Serge	1533	{
		1534	intel_write_status_page(ring, I915_GEM_HWS_INDEX, seqno);
		1535	}
		1536
2332	Serge	1537	static u32
5060	serge	1538	pc_render_get_seqno(struct intel_engine_cs *ring, bool lazy_coherency)
2332	Serge	1539	{
4104	Serge	1540	return ring->scratch.cpu_page[0];
2332	Serge	1541	}
		1542
3480	Serge	1543	static void
5060	serge	1544	pc_render_set_seqno(struct intel_engine_cs *ring, u32 seqno)
3480	Serge	1545	{
4104	Serge	1546	ring->scratch.cpu_page[0] = seqno;
3480	Serge	1547	}
		1548
3031	serge	1549	static bool
5060	serge	1550	gen5_ring_get_irq(struct intel_engine_cs *ring)
2332	Serge	1551	{
3031	serge	1552	struct drm_device *dev = ring->dev;
5060	serge	1553	struct drm_i915_private *dev_priv = dev->dev_private;
3031	serge	1554	unsigned long flags;
		1555
5354	serge	1556	if (WARN_ON(!intel_irqs_enabled(dev_priv)))
3031	serge	1557	return false;
		1558
		1559	spin_lock_irqsave(&dev_priv->irq_lock, flags);
4104	Serge	1560	if (ring->irq_refcount++ == 0)
5060	serge	1561	gen5_enable_gt_irq(dev_priv, ring->irq_enable_mask);
3031	serge	1562	spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
		1563
		1564	return true;
2332	Serge	1565	}
		1566
		1567	static void
5060	serge	1568	gen5_ring_put_irq(struct intel_engine_cs *ring)
2332	Serge	1569	{
3031	serge	1570	struct drm_device *dev = ring->dev;
5060	serge	1571	struct drm_i915_private *dev_priv = dev->dev_private;
3031	serge	1572	unsigned long flags;
		1573
		1574	spin_lock_irqsave(&dev_priv->irq_lock, flags);
4104	Serge	1575	if (--ring->irq_refcount == 0)
5060	serge	1576	gen5_disable_gt_irq(dev_priv, ring->irq_enable_mask);
3031	serge	1577	spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
2332	Serge	1578	}
		1579
3031	serge	1580	static bool
5060	serge	1581	i9xx_ring_get_irq(struct intel_engine_cs *ring)
2332	Serge	1582	{
3031	serge	1583	struct drm_device *dev = ring->dev;
5060	serge	1584	struct drm_i915_private *dev_priv = dev->dev_private;
3031	serge	1585	unsigned long flags;
		1586
5354	serge	1587	if (!intel_irqs_enabled(dev_priv))
3031	serge	1588	return false;
		1589
		1590	spin_lock_irqsave(&dev_priv->irq_lock, flags);
		1591	if (ring->irq_refcount++ == 0) {
		1592	dev_priv->irq_mask &= ~ring->irq_enable_mask;
6084	serge	1593	I915_WRITE(IMR, dev_priv->irq_mask);
		1594	POSTING_READ(IMR);
3031	serge	1595	}
		1596	spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
		1597
		1598	return true;
2332	Serge	1599	}
		1600
		1601	static void
5060	serge	1602	i9xx_ring_put_irq(struct intel_engine_cs *ring)
2332	Serge	1603	{
3031	serge	1604	struct drm_device *dev = ring->dev;
5060	serge	1605	struct drm_i915_private *dev_priv = dev->dev_private;
3031	serge	1606	unsigned long flags;
		1607
		1608	spin_lock_irqsave(&dev_priv->irq_lock, flags);
		1609	if (--ring->irq_refcount == 0) {
		1610	dev_priv->irq_mask \|= ring->irq_enable_mask;
6084	serge	1611	I915_WRITE(IMR, dev_priv->irq_mask);
		1612	POSTING_READ(IMR);
3031	serge	1613	}
		1614	spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
2332	Serge	1615	}
		1616
		1617	static bool
5060	serge	1618	i8xx_ring_get_irq(struct intel_engine_cs *ring)
2332	Serge	1619	{
		1620	struct drm_device *dev = ring->dev;
5060	serge	1621	struct drm_i915_private *dev_priv = dev->dev_private;
3031	serge	1622	unsigned long flags;
2332	Serge	1623
5354	serge	1624	if (!intel_irqs_enabled(dev_priv))
2332	Serge	1625	return false;
		1626
3031	serge	1627	spin_lock_irqsave(&dev_priv->irq_lock, flags);
2332	Serge	1628	if (ring->irq_refcount++ == 0) {
3031	serge	1629	dev_priv->irq_mask &= ~ring->irq_enable_mask;
		1630	I915_WRITE16(IMR, dev_priv->irq_mask);
		1631	POSTING_READ16(IMR);
2332	Serge	1632	}
3031	serge	1633	spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
2332	Serge	1634
		1635	return true;
		1636	}
		1637
		1638	static void
5060	serge	1639	i8xx_ring_put_irq(struct intel_engine_cs *ring)
2332	Serge	1640	{
		1641	struct drm_device *dev = ring->dev;
5060	serge	1642	struct drm_i915_private *dev_priv = dev->dev_private;
3031	serge	1643	unsigned long flags;
2332	Serge	1644
3031	serge	1645	spin_lock_irqsave(&dev_priv->irq_lock, flags);
2332	Serge	1646	if (--ring->irq_refcount == 0) {
3031	serge	1647	dev_priv->irq_mask \|= ring->irq_enable_mask;
		1648	I915_WRITE16(IMR, dev_priv->irq_mask);
		1649	POSTING_READ16(IMR);
2332	Serge	1650	}
3031	serge	1651	spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
2332	Serge	1652	}
		1653
		1654	static int
6084	serge	1655	bsd_ring_flush(struct drm_i915_gem_request *req,
2332	Serge	1656	u32 invalidate_domains,
		1657	u32 flush_domains)
		1658	{
6084	serge	1659	struct intel_engine_cs *ring = req->ring;
2332	Serge	1660	int ret;
		1661
6084	serge	1662	ret = intel_ring_begin(req, 2);
2332	Serge	1663	if (ret)
		1664	return ret;
		1665
		1666	intel_ring_emit(ring, MI_FLUSH);
		1667	intel_ring_emit(ring, MI_NOOP);
		1668	intel_ring_advance(ring);
		1669	return 0;
		1670	}
		1671
		1672	static int
6084	serge	1673	i9xx_add_request(struct drm_i915_gem_request *req)
2332	Serge	1674	{
6084	serge	1675	struct intel_engine_cs *ring = req->ring;
2332	Serge	1676	int ret;
		1677
6084	serge	1678	ret = intel_ring_begin(req, 4);
2332	Serge	1679	if (ret)
		1680	return ret;
		1681
		1682	intel_ring_emit(ring, MI_STORE_DWORD_INDEX);
		1683	intel_ring_emit(ring, I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
6084	serge	1684	intel_ring_emit(ring, i915_gem_request_get_seqno(req));
2332	Serge	1685	intel_ring_emit(ring, MI_USER_INTERRUPT);
4560	Serge	1686	__intel_ring_advance(ring);
2332	Serge	1687
		1688	return 0;
		1689	}
		1690
		1691	static bool
5060	serge	1692	gen6_ring_get_irq(struct intel_engine_cs *ring)
2332	Serge	1693	{
		1694	struct drm_device *dev = ring->dev;
5060	serge	1695	struct drm_i915_private *dev_priv = dev->dev_private;
3031	serge	1696	unsigned long flags;
2332	Serge	1697
5354	serge	1698	if (WARN_ON(!intel_irqs_enabled(dev_priv)))
6084	serge	1699	return false;
2332	Serge	1700
3031	serge	1701	spin_lock_irqsave(&dev_priv->irq_lock, flags);
2332	Serge	1702	if (ring->irq_refcount++ == 0) {
4560	Serge	1703	if (HAS_L3_DPF(dev) && ring->id == RCS)
4104	Serge	1704	I915_WRITE_IMR(ring,
		1705	~(ring->irq_enable_mask \|
4560	Serge	1706	GT_PARITY_ERROR(dev)));
3031	serge	1707	else
		1708	I915_WRITE_IMR(ring, ~ring->irq_enable_mask);
5060	serge	1709	gen5_enable_gt_irq(dev_priv, ring->irq_enable_mask);
2332	Serge	1710	}
3031	serge	1711	spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
2332	Serge	1712
6084	serge	1713	return true;
2332	Serge	1714	}
		1715
		1716	static void
5060	serge	1717	gen6_ring_put_irq(struct intel_engine_cs *ring)
2332	Serge	1718	{
		1719	struct drm_device *dev = ring->dev;
5060	serge	1720	struct drm_i915_private *dev_priv = dev->dev_private;
3031	serge	1721	unsigned long flags;
2332	Serge	1722
3031	serge	1723	spin_lock_irqsave(&dev_priv->irq_lock, flags);
2332	Serge	1724	if (--ring->irq_refcount == 0) {
4560	Serge	1725	if (HAS_L3_DPF(dev) && ring->id == RCS)
		1726	I915_WRITE_IMR(ring, ~GT_PARITY_ERROR(dev));
3031	serge	1727	else
		1728	I915_WRITE_IMR(ring, ~0);
5060	serge	1729	gen5_disable_gt_irq(dev_priv, ring->irq_enable_mask);
2332	Serge	1730	}
3031	serge	1731	spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
2332	Serge	1732	}
		1733
4104	Serge	1734	static bool
5060	serge	1735	hsw_vebox_get_irq(struct intel_engine_cs *ring)
4104	Serge	1736	{
		1737	struct drm_device *dev = ring->dev;
		1738	struct drm_i915_private *dev_priv = dev->dev_private;
		1739	unsigned long flags;
		1740
5354	serge	1741	if (WARN_ON(!intel_irqs_enabled(dev_priv)))
4104	Serge	1742	return false;
		1743
		1744	spin_lock_irqsave(&dev_priv->irq_lock, flags);
		1745	if (ring->irq_refcount++ == 0) {
		1746	I915_WRITE_IMR(ring, ~ring->irq_enable_mask);
5060	serge	1747	gen6_enable_pm_irq(dev_priv, ring->irq_enable_mask);
4104	Serge	1748	}
		1749	spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
		1750
		1751	return true;
		1752	}
		1753
		1754	static void
5060	serge	1755	hsw_vebox_put_irq(struct intel_engine_cs *ring)
4104	Serge	1756	{
		1757	struct drm_device *dev = ring->dev;
		1758	struct drm_i915_private *dev_priv = dev->dev_private;
		1759	unsigned long flags;
		1760
		1761	spin_lock_irqsave(&dev_priv->irq_lock, flags);
		1762	if (--ring->irq_refcount == 0) {
		1763	I915_WRITE_IMR(ring, ~0);
5060	serge	1764	gen6_disable_pm_irq(dev_priv, ring->irq_enable_mask);
4104	Serge	1765	}
		1766	spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
		1767	}
		1768
4560	Serge	1769	static bool
5060	serge	1770	gen8_ring_get_irq(struct intel_engine_cs *ring)
4560	Serge	1771	{
		1772	struct drm_device *dev = ring->dev;
		1773	struct drm_i915_private *dev_priv = dev->dev_private;
		1774	unsigned long flags;
		1775
5354	serge	1776	if (WARN_ON(!intel_irqs_enabled(dev_priv)))
4560	Serge	1777	return false;
		1778
		1779	spin_lock_irqsave(&dev_priv->irq_lock, flags);
		1780	if (ring->irq_refcount++ == 0) {
		1781	if (HAS_L3_DPF(dev) && ring->id == RCS) {
		1782	I915_WRITE_IMR(ring,
		1783	~(ring->irq_enable_mask \|
		1784	GT_RENDER_L3_PARITY_ERROR_INTERRUPT));
		1785	} else {
		1786	I915_WRITE_IMR(ring, ~ring->irq_enable_mask);
		1787	}
		1788	POSTING_READ(RING_IMR(ring->mmio_base));
		1789	}
		1790	spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
		1791
		1792	return true;
		1793	}
		1794
		1795	static void
5060	serge	1796	gen8_ring_put_irq(struct intel_engine_cs *ring)
4560	Serge	1797	{
		1798	struct drm_device *dev = ring->dev;
		1799	struct drm_i915_private *dev_priv = dev->dev_private;
		1800	unsigned long flags;
		1801
		1802	spin_lock_irqsave(&dev_priv->irq_lock, flags);
		1803	if (--ring->irq_refcount == 0) {
		1804	if (HAS_L3_DPF(dev) && ring->id == RCS) {
		1805	I915_WRITE_IMR(ring,
		1806	~GT_RENDER_L3_PARITY_ERROR_INTERRUPT);
		1807	} else {
		1808	I915_WRITE_IMR(ring, ~0);
		1809	}
		1810	POSTING_READ(RING_IMR(ring->mmio_base));
		1811	}
		1812	spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
		1813	}
		1814
2332	Serge	1815	static int
6084	serge	1816	i965_dispatch_execbuffer(struct drm_i915_gem_request *req,
5060	serge	1817	u64 offset, u32 length,
6084	serge	1818	unsigned dispatch_flags)
2332	Serge	1819	{
6084	serge	1820	struct intel_engine_cs *ring = req->ring;
2332	Serge	1821	int ret;
		1822
6084	serge	1823	ret = intel_ring_begin(req, 2);
2332	Serge	1824	if (ret)
		1825	return ret;
		1826
		1827	intel_ring_emit(ring,
3031	serge	1828	MI_BATCH_BUFFER_START \|
		1829	MI_BATCH_GTT \|
6084	serge	1830	(dispatch_flags & I915_DISPATCH_SECURE ?
		1831
2332	Serge	1832	intel_ring_emit(ring, offset);
		1833	intel_ring_advance(ring);
		1834
		1835	return 0;
		1836	}
		1837
3243	Serge	1838	/* Just userspace ABI convention to limit the wa batch bo to a resonable size */
		1839	#define I830_BATCH_LIMIT (256*1024)
5128	serge	1840	#define I830_TLB_ENTRIES (2)
		1841	#define I830_WA_SIZE max(I830_TLB_ENTRIES*4096, I830_BATCH_LIMIT)
2332	Serge	1842	static int
6084	serge	1843	i830_dispatch_execbuffer(struct drm_i915_gem_request *req,
		1844	u64 offset, u32 len,
		1845	unsigned dispatch_flags)
2332	Serge	1846	{
6084	serge	1847	struct intel_engine_cs *ring = req->ring;
5128	serge	1848	u32 cs_offset = ring->scratch.gtt_offset;
2332	Serge	1849	int ret;
		1850
6084	serge	1851	ret = intel_ring_begin(req, 6);
		1852	if (ret)
		1853	return ret;
2332	Serge	1854
5128	serge	1855	/* Evict the invalid PTE TLBs */
		1856	intel_ring_emit(ring, COLOR_BLT_CMD \| BLT_WRITE_RGBA);
		1857	intel_ring_emit(ring, BLT_DEPTH_32 \| BLT_ROP_COLOR_COPY \| 4096);
		1858	intel_ring_emit(ring, I830_TLB_ENTRIES << 16 \| 4); /* load each page */
		1859	intel_ring_emit(ring, cs_offset);
		1860	intel_ring_emit(ring, 0xdeadbeef);
6084	serge	1861	intel_ring_emit(ring, MI_NOOP);
		1862	intel_ring_advance(ring);
3243	Serge	1863
6084	serge	1864	if ((dispatch_flags & I915_DISPATCH_PINNED) == 0) {
3243	Serge	1865	if (len > I830_BATCH_LIMIT)
		1866	return -ENOSPC;
		1867
6084	serge	1868	ret = intel_ring_begin(req, 6 + 2);
3243	Serge	1869	if (ret)
		1870	return ret;
5128	serge	1871
		1872	/* Blit the batch (which has now all relocs applied) to the
		1873	* stable batch scratch bo area (so that the CS never
		1874	* stumbles over its tlb invalidation bug) ...
		1875	*/
		1876	intel_ring_emit(ring, SRC_COPY_BLT_CMD \| BLT_WRITE_RGBA);
		1877	intel_ring_emit(ring, BLT_DEPTH_32 \| BLT_ROP_SRC_COPY \| 4096);
5139	serge	1878	intel_ring_emit(ring, DIV_ROUND_UP(len, 4096) << 16 \| 4096);
3243	Serge	1879	intel_ring_emit(ring, cs_offset);
		1880	intel_ring_emit(ring, 4096);
		1881	intel_ring_emit(ring, offset);
5128	serge	1882
3243	Serge	1883	intel_ring_emit(ring, MI_FLUSH);
5128	serge	1884	intel_ring_emit(ring, MI_NOOP);
		1885	intel_ring_advance(ring);
3243	Serge	1886
		1887	/* ... and execute it. */
5128	serge	1888	offset = cs_offset;
		1889	}
		1890
6084	serge	1891	ret = intel_ring_begin(req, 4);
5128	serge	1892	if (ret)
		1893	return ret;
		1894
6084	serge	1895	intel_ring_emit(ring, MI_BATCH_BUFFER);
		1896	intel_ring_emit(ring, offset \| (dispatch_flags & I915_DISPATCH_SECURE ?
		1897
5128	serge	1898	intel_ring_emit(ring, offset + len - 8);
		1899	intel_ring_emit(ring, MI_NOOP);
3031	serge	1900	intel_ring_advance(ring);
		1901
		1902	return 0;
		1903	}
		1904
		1905	static int
6084	serge	1906	i915_dispatch_execbuffer(struct drm_i915_gem_request *req,
5060	serge	1907	u64 offset, u32 len,
6084	serge	1908	unsigned dispatch_flags)
3031	serge	1909	{
6084	serge	1910	struct intel_engine_cs *ring = req->ring;
3031	serge	1911	int ret;
		1912
6084	serge	1913	ret = intel_ring_begin(req, 2);
		1914	if (ret)
		1915	return ret;
2332	Serge	1916
3031	serge	1917	intel_ring_emit(ring, MI_BATCH_BUFFER_START \| MI_BATCH_GTT);
6084	serge	1918	intel_ring_emit(ring, offset \| (dispatch_flags & I915_DISPATCH_SECURE ?
		1919
2332	Serge	1920	intel_ring_advance(ring);
		1921
		1922	return 0;
		1923	}
		1924
6660	serge	1925	static void cleanup_phys_status_page(struct intel_engine_cs *ring)
		1926	{
		1927	struct drm_i915_private *dev_priv = to_i915(ring->dev);
		1928
		1929	if (!dev_priv->status_page_dmah)
		1930	return;
		1931
		1932	drm_pci_free(ring->dev, dev_priv->status_page_dmah);
		1933	ring->status_page.page_addr = NULL;
		1934	}
		1935
5060	serge	1936	static void cleanup_status_page(struct intel_engine_cs *ring)
2332	Serge	1937	{
		1938	struct drm_i915_gem_object *obj;
		1939
		1940	obj = ring->status_page.obj;
		1941	if (obj == NULL)
		1942	return;
		1943
5354	serge	1944	kunmap(sg_page(obj->pages->sgl));
5060	serge	1945	i915_gem_object_ggtt_unpin(obj);
2344	Serge	1946	drm_gem_object_unreference(&obj->base);
2332	Serge	1947	ring->status_page.obj = NULL;
		1948	}
		1949
5060	serge	1950	static int init_status_page(struct intel_engine_cs *ring)
2332	Serge	1951	{
6660	serge	1952	struct drm_i915_gem_object *obj = ring->status_page.obj;
5060	serge	1953
6660	serge	1954	if (obj == NULL) {
5060	serge	1955	unsigned flags;
6084	serge	1956	int ret;
2332	Serge	1957
5060	serge	1958	obj = i915_gem_alloc_object(ring->dev, 4096);
6084	serge	1959	if (obj == NULL) {
		1960	DRM_ERROR("Failed to allocate status page\n");
5060	serge	1961	return -ENOMEM;
6084	serge	1962	}
2332	Serge	1963
6084	serge	1964	ret = i915_gem_object_set_cache_level(obj, I915_CACHE_LLC);
		1965	if (ret)
		1966	goto err_unref;
2332	Serge	1967
5060	serge	1968	flags = 0;
		1969	if (!HAS_LLC(ring->dev))
		1970	/* On g33, we cannot place HWS above 256MiB, so
		1971	* restrict its pinning to the low mappable arena.
		1972	* Though this restriction is not documented for
		1973	* gen4, gen5, or byt, they also behave similarly
		1974	* and hang if the HWS is placed at the top of the
		1975	* GTT. To generalise, it appears that all !llc
		1976	* platforms have issues with us placing the HWS
		1977	* above the mappable region (even though we never
		1978	* actualy map it).
		1979	*/
		1980	flags \|= PIN_MAPPABLE;
		1981	ret = i915_gem_obj_ggtt_pin(obj, 4096, flags);
		1982	if (ret) {
		1983	err_unref:
		1984	drm_gem_object_unreference(&obj->base);
		1985	return ret;
		1986	}
		1987
		1988	ring->status_page.obj = obj;
2332	Serge	1989	}
		1990
4104	Serge	1991	ring->status_page.gfx_addr = i915_gem_obj_ggtt_offset(obj);
5354	serge	1992	ring->status_page.page_addr = kmap(sg_page(obj->pages->sgl));
2332	Serge	1993	memset(ring->status_page.page_addr, 0, PAGE_SIZE);
		1994
		1995	DRM_DEBUG_DRIVER("%s hws offset: 0x%08x\n",
		1996	ring->name, ring->status_page.gfx_addr);
		1997
		1998	return 0;
		1999	}
		2000
5060	serge	2001	static int init_phys_status_page(struct intel_engine_cs *ring)
3243	Serge	2002	{
6084	serge	2003	struct drm_i915_private *dev_priv = ring->dev->dev_private;
3243	Serge	2004
6084	serge	2005	if (!dev_priv->status_page_dmah) {
		2006	dev_priv->status_page_dmah =
		2007	drm_pci_alloc(ring->dev, PAGE_SIZE, PAGE_SIZE);
		2008	if (!dev_priv->status_page_dmah)
		2009	return -ENOMEM;
		2010	}
3243	Serge	2011
6084	serge	2012	ring->status_page.page_addr = dev_priv->status_page_dmah->vaddr;
		2013	memset(ring->status_page.page_addr, 0, PAGE_SIZE);
3243	Serge	2014
6084	serge	2015	return 0;
3243	Serge	2016	}
		2017
5354	serge	2018	void intel_unpin_ringbuffer_obj(struct intel_ringbuffer *ringbuf)
2332	Serge	2019	{
5060	serge	2020	iounmap(ringbuf->virtual_start);
5354	serge	2021	ringbuf->virtual_start = NULL;
5060	serge	2022	i915_gem_object_ggtt_unpin(ringbuf->obj);
		2023	}
		2024
5354	serge	2025	int intel_pin_and_map_ringbuffer_obj(struct drm_device *dev,
6084	serge	2026	struct intel_ringbuffer *ringbuf)
5060	serge	2027	{
		2028	struct drm_i915_private *dev_priv = to_i915(dev);
5354	serge	2029	struct drm_i915_gem_object *obj = ringbuf->obj;
2332	Serge	2030	int ret;
2340	Serge	2031
5354	serge	2032	ret = i915_gem_obj_ggtt_pin(obj, PAGE_SIZE, PIN_MAPPABLE);
		2033	if (ret)
		2034	return ret;
		2035
		2036	ret = i915_gem_object_set_to_gtt_domain(obj, true);
		2037	if (ret) {
		2038	i915_gem_object_ggtt_unpin(obj);
		2039	return ret;
		2040	}
		2041
		2042	ringbuf->virtual_start = ioremap_wc(dev_priv->gtt.mappable_base +
		2043	i915_gem_obj_ggtt_offset(obj), ringbuf->size);
		2044	if (ringbuf->virtual_start == NULL) {
		2045	i915_gem_object_ggtt_unpin(obj);
		2046	return -EINVAL;
		2047	}
		2048
6084	serge	2049	return 0;
5354	serge	2050	}
2332	Serge	2051
6084	serge	2052	static void intel_destroy_ringbuffer_obj(struct intel_ringbuffer *ringbuf)
5354	serge	2053	{
		2054	drm_gem_object_unreference(&ringbuf->obj->base);
		2055	ringbuf->obj = NULL;
		2056	}
		2057
6084	serge	2058	static int intel_alloc_ringbuffer_obj(struct drm_device *dev,
		2059	struct intel_ringbuffer *ringbuf)
5354	serge	2060	{
		2061	struct drm_i915_gem_object *obj;
		2062
3480	Serge	2063	obj = NULL;
4371	Serge	2064	if (!HAS_LLC(dev))
5060	serge	2065	obj = i915_gem_object_create_stolen(dev, ringbuf->size);
3480	Serge	2066	if (obj == NULL)
5060	serge	2067	obj = i915_gem_alloc_object(dev, ringbuf->size);
		2068	if (obj == NULL)
		2069	return -ENOMEM;
2332	Serge	2070
5060	serge	2071	/* mark ring buffers as read-only from GPU side by default */
		2072	obj->gt_ro = 1;
2332	Serge	2073
5354	serge	2074	ringbuf->obj = obj;
2332	Serge	2075
5060	serge	2076	return 0;
		2077	}
		2078
6084	serge	2079	struct intel_ringbuffer *
		2080	intel_engine_create_ringbuffer(struct intel_engine_cs *engine, int size)
		2081	{
		2082	struct intel_ringbuffer *ring;
		2083	int ret;
		2084
		2085	ring = kzalloc(sizeof(*ring), GFP_KERNEL);
		2086	if (ring == NULL)
		2087	return ERR_PTR(-ENOMEM);
		2088
		2089	ring->ring = engine;
		2090
		2091	ring->size = size;
		2092	/* Workaround an erratum on the i830 which causes a hang if
		2093	* the TAIL pointer points to within the last 2 cachelines
		2094	* of the buffer.
		2095	*/
		2096	ring->effective_size = size;
		2097	if (IS_I830(engine->dev) \|\| IS_845G(engine->dev))
		2098	ring->effective_size -= 2 * CACHELINE_BYTES;
		2099
		2100	ring->last_retired_head = -1;
		2101	intel_ring_update_space(ring);
		2102
		2103	ret = intel_alloc_ringbuffer_obj(engine->dev, ring);
		2104	if (ret) {
		2105	DRM_ERROR("Failed to allocate ringbuffer %s: %d\n",
		2106	engine->name, ret);
		2107	kfree(ring);
		2108	return ERR_PTR(ret);
		2109	}
		2110
		2111	return ring;
		2112	}
		2113
		2114	void
		2115	intel_ringbuffer_free(struct intel_ringbuffer *ring)
		2116	{
		2117	intel_destroy_ringbuffer_obj(ring);
		2118	kfree(ring);
		2119	}
		2120
5060	serge	2121	static int intel_init_ring_buffer(struct drm_device *dev,
		2122	struct intel_engine_cs *ring)
		2123	{
6084	serge	2124	struct intel_ringbuffer *ringbuf;
5060	serge	2125	int ret;
		2126
6084	serge	2127	WARN_ON(ring->buffer);
5060	serge	2128
		2129	ring->dev = dev;
		2130	INIT_LIST_HEAD(&ring->active_list);
		2131	INIT_LIST_HEAD(&ring->request_list);
5354	serge	2132	INIT_LIST_HEAD(&ring->execlist_queue);
6084	serge	2133	i915_gem_batch_pool_init(dev, &ring->batch_pool);
5060	serge	2134	memset(ring->semaphore.sync_seqno, 0, sizeof(ring->semaphore.sync_seqno));
		2135
		2136	init_waitqueue_head(&ring->irq_queue);
		2137
6084	serge	2138	ringbuf = intel_engine_create_ringbuffer(ring, 32 * PAGE_SIZE);
		2139	if (IS_ERR(ringbuf))
		2140	return PTR_ERR(ringbuf);
		2141	ring->buffer = ringbuf;
		2142
5060	serge	2143	if (I915_NEED_GFX_HWS(dev)) {
		2144	ret = init_status_page(ring);
		2145	if (ret)
		2146	goto error;
		2147	} else {
6660	serge	2148	WARN_ON(ring->id != RCS);
5060	serge	2149	ret = init_phys_status_page(ring);
6084	serge	2150	if (ret)
5060	serge	2151	goto error;
		2152	}
2332	Serge	2153
6084	serge	2154	ret = intel_pin_and_map_ringbuffer_obj(dev, ringbuf);
5060	serge	2155	if (ret) {
6084	serge	2156	DRM_ERROR("Failed to pin and map ringbuffer %s: %d\n",
		2157	ring->name, ret);
		2158	intel_destroy_ringbuffer_obj(ringbuf);
5060	serge	2159	goto error;
		2160	}
		2161
		2162	ret = i915_cmd_parser_init_ring(ring);
		2163	if (ret)
		2164	goto error;
		2165
2332	Serge	2166	return 0;
		2167
5060	serge	2168	error:
6084	serge	2169	intel_ringbuffer_free(ringbuf);
5060	serge	2170	ring->buffer = NULL;
2332	Serge	2171	return ret;
		2172	}
		2173
5060	serge	2174	void intel_cleanup_ring_buffer(struct intel_engine_cs *ring)
2332	Serge	2175	{
5354	serge	2176	struct drm_i915_private *dev_priv;
2332	Serge	2177
5060	serge	2178	if (!intel_ring_initialized(ring))
2332	Serge	2179	return;
		2180
5354	serge	2181	dev_priv = to_i915(ring->dev);
		2182
5060	serge	2183	intel_stop_ring_buffer(ring);
		2184	WARN_ON(!IS_GEN2(ring->dev) && (I915_READ_MODE(ring) & MODE_IDLE) == 0);
2332	Serge	2185
6084	serge	2186	intel_unpin_ringbuffer_obj(ring->buffer);
		2187	intel_ringbuffer_free(ring->buffer);
		2188	ring->buffer = NULL;
2332	Serge	2189
		2190	if (ring->cleanup)
		2191	ring->cleanup(ring);
		2192
6660	serge	2193	if (I915_NEED_GFX_HWS(ring->dev)) {
6084	serge	2194	cleanup_status_page(ring);
6660	serge	2195	} else {
		2196	WARN_ON(ring->id != RCS);
		2197	cleanup_phys_status_page(ring);
		2198	}
2332	Serge	2199
5060	serge	2200	i915_cmd_parser_fini_ring(ring);
6084	serge	2201	i915_gem_batch_pool_fini(&ring->batch_pool);
3031	serge	2202	}
		2203
6084	serge	2204	static int ring_wait_for_space(struct intel_engine_cs *ring, int n)
3031	serge	2205	{
5060	serge	2206	struct intel_ringbuffer *ringbuf = ring->buffer;
3031	serge	2207	struct drm_i915_gem_request *request;
6084	serge	2208	unsigned space;
3031	serge	2209	int ret;
		2210
6084	serge	2211	if (intel_ring_space(ringbuf) >= n)
		2212	return 0;
3031	serge	2213
6084	serge	2214	/* The whole point of reserving space is to not wait! */
		2215	WARN_ON(ringbuf->reserved_in_use);
2332	Serge	2216
3031	serge	2217	list_for_each_entry(request, &ring->request_list, list) {
6084	serge	2218	space = __intel_ring_space(request->postfix, ringbuf->tail,
		2219	ringbuf->size);
		2220	if (space >= n)
3031	serge	2221	break;
		2222	}
		2223
6084	serge	2224	if (WARN_ON(&request->list == &ring->request_list))
3031	serge	2225	return -ENOSPC;
		2226
6084	serge	2227	ret = i915_wait_request(request);
3031	serge	2228	if (ret)
		2229	return ret;
		2230
6084	serge	2231	ringbuf->space = space;
3031	serge	2232	return 0;
		2233	}
		2234
6084	serge	2235	static void __wrap_ring_buffer(struct intel_ringbuffer *ringbuf)
3031	serge	2236	{
3243	Serge	2237	uint32_t __iomem *virt;
5060	serge	2238	int rem = ringbuf->size - ringbuf->tail;
3243	Serge	2239
5060	serge	2240	virt = ringbuf->virtual_start + ringbuf->tail;
3243	Serge	2241	rem /= 4;
		2242	while (rem--)
		2243	iowrite32(MI_NOOP, virt++);
		2244
5060	serge	2245	ringbuf->tail = 0;
6084	serge	2246	intel_ring_update_space(ringbuf);
3243	Serge	2247	}
		2248
5060	serge	2249	int intel_ring_idle(struct intel_engine_cs *ring)
3243	Serge	2250	{
6084	serge	2251	struct drm_i915_gem_request *req;
3243	Serge	2252
		2253	/* Wait upon the last request to be completed */
		2254	if (list_empty(&ring->request_list))
		2255	return 0;
		2256
6084	serge	2257	req = list_entry(ring->request_list.prev,
		2258	struct drm_i915_gem_request,
		2259	list);
3243	Serge	2260
6084	serge	2261	/* Make sure we do not trigger any retires */
		2262	return __i915_wait_request(req,
		2263	atomic_read(&to_i915(ring->dev)->gpu_error.reset_counter),
		2264	to_i915(ring->dev)->mm.interruptible,
		2265	NULL, NULL);
3243	Serge	2266	}
		2267
6084	serge	2268	int intel_ring_alloc_request_extras(struct drm_i915_gem_request *request)
3243	Serge	2269	{
6084	serge	2270	request->ringbuf = request->ring->buffer;
		2271	return 0;
		2272	}
3243	Serge	2273
6084	serge	2274	int intel_ring_reserve_space(struct drm_i915_gem_request *request)
		2275	{
		2276	/*
		2277	* The first call merely notes the reserve request and is common for
		2278	* all back ends. The subsequent localised _begin() call actually
		2279	* ensures that the reservation is available. Without the begin, if
		2280	* the request creator immediately submitted the request without
		2281	* adding any commands to it then there might not actually be
		2282	* sufficient room for the submission commands.
		2283	*/
		2284	intel_ring_reserved_space_reserve(request->ringbuf, MIN_SPACE_FOR_ADD_REQUEST);
4560	Serge	2285
6084	serge	2286	return intel_ring_begin(request, 0);
		2287	}
4560	Serge	2288
6084	serge	2289	void intel_ring_reserved_space_reserve(struct intel_ringbuffer *ringbuf, int size)
		2290	{
		2291	WARN_ON(ringbuf->reserved_size);
		2292	WARN_ON(ringbuf->reserved_in_use);
		2293
		2294	ringbuf->reserved_size = size;
		2295	}
		2296
		2297	void intel_ring_reserved_space_cancel(struct intel_ringbuffer *ringbuf)
		2298	{
		2299	WARN_ON(ringbuf->reserved_in_use);
		2300
		2301	ringbuf->reserved_size = 0;
		2302	ringbuf->reserved_in_use = false;
		2303	}
		2304
		2305	void intel_ring_reserved_space_use(struct intel_ringbuffer *ringbuf)
		2306	{
		2307	WARN_ON(ringbuf->reserved_in_use);
		2308
		2309	ringbuf->reserved_in_use = true;
		2310	ringbuf->reserved_tail = ringbuf->tail;
		2311	}
		2312
		2313	void intel_ring_reserved_space_end(struct intel_ringbuffer *ringbuf)
		2314	{
		2315	WARN_ON(!ringbuf->reserved_in_use);
		2316	if (ringbuf->tail > ringbuf->reserved_tail) {
		2317	WARN(ringbuf->tail > ringbuf->reserved_tail + ringbuf->reserved_size,
		2318	"request reserved size too small: %d vs %d!\n",
		2319	ringbuf->tail - ringbuf->reserved_tail, ringbuf->reserved_size);
		2320	} else {
		2321	/*
		2322	* The ring was wrapped while the reserved space was in use.
		2323	* That means that some unknown amount of the ring tail was
		2324	* no-op filled and skipped. Thus simply adding the ring size
		2325	* to the tail and doing the above space check will not work.
		2326	* Rather than attempt to track how much tail was skipped,
		2327	* it is much simpler to say that also skipping the sanity
		2328	* check every once in a while is not a big issue.
		2329	*/
4560	Serge	2330	}
		2331
6084	serge	2332	ringbuf->reserved_size = 0;
		2333	ringbuf->reserved_in_use = false;
3243	Serge	2334	}
		2335
6084	serge	2336	static int __intel_ring_prepare(struct intel_engine_cs *ring, int bytes)
3480	Serge	2337	{
5060	serge	2338	struct intel_ringbuffer *ringbuf = ring->buffer;
6084	serge	2339	int remain_usable = ringbuf->effective_size - ringbuf->tail;
		2340	int remain_actual = ringbuf->size - ringbuf->tail;
		2341	int ret, total_bytes, wait_bytes = 0;
		2342	bool need_wrap = false;
3480	Serge	2343
6084	serge	2344	if (ringbuf->reserved_in_use)
		2345	total_bytes = bytes;
		2346	else
		2347	total_bytes = bytes + ringbuf->reserved_size;
		2348
		2349	if (unlikely(bytes > remain_usable)) {
		2350	/*
		2351	* Not enough space for the basic request. So need to flush
		2352	* out the remainder and then wait for base + reserved.
		2353	*/
		2354	wait_bytes = remain_actual + total_bytes;
		2355	need_wrap = true;
		2356	} else {
		2357	if (unlikely(total_bytes > remain_usable)) {
		2358	/*
		2359	* The base request will fit but the reserved space
6660	serge	2360	* falls off the end. So don't need an immediate wrap
		2361	* and only need to effectively wait for the reserved
		2362	* size space from the start of ringbuffer.
6084	serge	2363	*/
		2364	wait_bytes = remain_actual + ringbuf->reserved_size;
		2365	} else if (total_bytes > ringbuf->space) {
		2366	/* No wrapping required, just waiting. */
		2367	wait_bytes = total_bytes;
		2368	}
3480	Serge	2369	}
		2370
6084	serge	2371	if (wait_bytes) {
		2372	ret = ring_wait_for_space(ring, wait_bytes);
3480	Serge	2373	if (unlikely(ret))
		2374	return ret;
6084	serge	2375
		2376	if (need_wrap)
		2377	__wrap_ring_buffer(ringbuf);
3480	Serge	2378	}
		2379
		2380	return 0;
		2381	}
		2382
6084	serge	2383	int intel_ring_begin(struct drm_i915_gem_request *req,
2332	Serge	2384	int num_dwords)
		2385	{
6084	serge	2386	struct intel_engine_cs *ring;
		2387	struct drm_i915_private *dev_priv;
2332	Serge	2388	int ret;
		2389
6084	serge	2390	WARN_ON(req == NULL);
		2391	ring = req->ring;
		2392	dev_priv = ring->dev->dev_private;
		2393
3480	Serge	2394	ret = i915_gem_check_wedge(&dev_priv->gpu_error,
		2395	dev_priv->mm.interruptible);
3031	serge	2396	if (ret)
		2397	return ret;
2332	Serge	2398
4560	Serge	2399	ret = __intel_ring_prepare(ring, num_dwords * sizeof(uint32_t));
		2400	if (ret)
		2401	return ret;
		2402
5060	serge	2403	ring->buffer->space -= num_dwords * sizeof(uint32_t);
4560	Serge	2404	return 0;
3480	Serge	2405	}
2332	Serge	2406
5060	serge	2407	/* Align the ring tail to a cacheline boundary */
6084	serge	2408	int intel_ring_cacheline_align(struct drm_i915_gem_request *req)
3480	Serge	2409	{
6084	serge	2410	struct intel_engine_cs *ring = req->ring;
5060	serge	2411	int num_dwords = (ring->buffer->tail & (CACHELINE_BYTES - 1)) / sizeof(uint32_t);
		2412	int ret;
3480	Serge	2413
5060	serge	2414	if (num_dwords == 0)
		2415	return 0;
		2416
		2417	num_dwords = CACHELINE_BYTES / sizeof(uint32_t) - num_dwords;
6084	serge	2418	ret = intel_ring_begin(req, num_dwords);
5060	serge	2419	if (ret)
		2420	return ret;
		2421
		2422	while (num_dwords--)
		2423	intel_ring_emit(ring, MI_NOOP);
		2424
		2425	intel_ring_advance(ring);
		2426
		2427	return 0;
		2428	}
		2429
		2430	void intel_ring_init_seqno(struct intel_engine_cs *ring, u32 seqno)
		2431	{
		2432	struct drm_device *dev = ring->dev;
		2433	struct drm_i915_private *dev_priv = dev->dev_private;
		2434
		2435	if (INTEL_INFO(dev)->gen == 6 \|\| INTEL_INFO(dev)->gen == 7) {
3480	Serge	2436	I915_WRITE(RING_SYNC_0(ring->mmio_base), 0);
		2437	I915_WRITE(RING_SYNC_1(ring->mmio_base), 0);
5060	serge	2438	if (HAS_VEBOX(dev))
4104	Serge	2439	I915_WRITE(RING_SYNC_2(ring->mmio_base), 0);
2332	Serge	2440	}
		2441
3480	Serge	2442	ring->set_seqno(ring, seqno);
4104	Serge	2443	ring->hangcheck.seqno = seqno;
2332	Serge	2444	}
		2445
5060	serge	2446	static void gen6_bsd_ring_write_tail(struct intel_engine_cs *ring,
2332	Serge	2447	u32 value)
		2448	{
5060	serge	2449	struct drm_i915_private *dev_priv = ring->dev->dev_private;
2332	Serge	2450
		2451	/* Every tail move must follow the sequence below */
3031	serge	2452
		2453	/* Disable notification that the ring is IDLE. The GT
		2454	* will then assume that it is busy and bring it out of rc6.
		2455	*/
6084	serge	2456	I915_WRITE(GEN6_BSD_SLEEP_PSMI_CONTROL,
3031	serge	2457	_MASKED_BIT_ENABLE(GEN6_BSD_SLEEP_MSG_DISABLE));
2332	Serge	2458
3031	serge	2459	/* Clear the context id. Here be magic! */
		2460	I915_WRITE64(GEN6_BSD_RNCID, 0x0);
		2461
		2462	/* Wait for the ring not to be idle, i.e. for it to wake up. */
6084	serge	2463	if (wait_for((I915_READ(GEN6_BSD_SLEEP_PSMI_CONTROL) &
3031	serge	2464	GEN6_BSD_SLEEP_INDICATOR) == 0,
6084	serge	2465	50))
3031	serge	2466	DRM_ERROR("timed out waiting for the BSD ring to wake up\n");
2332	Serge	2467
3031	serge	2468	/* Now that the ring is fully powered up, update the tail */
6084	serge	2469	I915_WRITE_TAIL(ring, value);
3031	serge	2470	POSTING_READ(RING_TAIL(ring->mmio_base));
		2471
		2472	/* Let the ring send IDLE messages to the GT again,
		2473	* and so let it sleep to conserve power when idle.
		2474	*/
6084	serge	2475	I915_WRITE(GEN6_BSD_SLEEP_PSMI_CONTROL,
3031	serge	2476	_MASKED_BIT_DISABLE(GEN6_BSD_SLEEP_MSG_DISABLE));
2332	Serge	2477	}
		2478
6084	serge	2479	static int gen6_bsd_ring_flush(struct drm_i915_gem_request *req,
		2480	u32 invalidate, u32 flush)
2332	Serge	2481	{
6084	serge	2482	struct intel_engine_cs *ring = req->ring;
2332	Serge	2483	uint32_t cmd;
		2484	int ret;
		2485
6084	serge	2486	ret = intel_ring_begin(req, 4);
2332	Serge	2487	if (ret)
		2488	return ret;
		2489
		2490	cmd = MI_FLUSH_DW;
4560	Serge	2491	if (INTEL_INFO(ring->dev)->gen >= 8)
		2492	cmd += 1;
6084	serge	2493
		2494	/* We always require a command barrier so that subsequent
		2495	* commands, such as breadcrumb interrupts, are strictly ordered
		2496	* wrt the contents of the write cache being flushed to memory
		2497	* (and thus being coherent from the CPU).
		2498	*/
		2499	cmd \|= MI_FLUSH_DW_STORE_INDEX \| MI_FLUSH_DW_OP_STOREDW;
		2500
3243	Serge	2501	/*
		2502	* Bspec vol 1c.5 - video engine command streamer:
		2503	* "If ENABLED, all TLBs will be invalidated once the flush
		2504	* operation is complete. This bit is only valid when the
		2505	* Post-Sync Operation field is a value of 1h or 3h."
		2506	*/
2332	Serge	2507	if (invalidate & I915_GEM_GPU_DOMAINS)
6084	serge	2508	cmd \|= MI_INVALIDATE_TLB \| MI_INVALIDATE_BSD;
		2509
2332	Serge	2510	intel_ring_emit(ring, cmd);
3243	Serge	2511	intel_ring_emit(ring, I915_GEM_HWS_SCRATCH_ADDR \| MI_FLUSH_DW_USE_GTT);
4560	Serge	2512	if (INTEL_INFO(ring->dev)->gen >= 8) {
		2513	intel_ring_emit(ring, 0); /* upper addr */
		2514	intel_ring_emit(ring, 0); /* value */
		2515	} else {
6084	serge	2516	intel_ring_emit(ring, 0);
		2517	intel_ring_emit(ring, MI_NOOP);
4560	Serge	2518	}
2332	Serge	2519	intel_ring_advance(ring);
		2520	return 0;
		2521	}
		2522
		2523	static int
6084	serge	2524	gen8_ring_dispatch_execbuffer(struct drm_i915_gem_request *req,
5060	serge	2525	u64 offset, u32 len,
6084	serge	2526	unsigned dispatch_flags)
4560	Serge	2527	{
6084	serge	2528	struct intel_engine_cs *ring = req->ring;
		2529	bool ppgtt = USES_PPGTT(ring->dev) &&
		2530	!(dispatch_flags & I915_DISPATCH_SECURE);
4560	Serge	2531	int ret;
		2532
6084	serge	2533	ret = intel_ring_begin(req, 4);
4560	Serge	2534	if (ret)
		2535	return ret;
		2536
		2537	/* FIXME(BDW): Address space and security selectors. */
6084	serge	2538	intel_ring_emit(ring, MI_BATCH_BUFFER_START_GEN8 \| (ppgtt<<8) \|
		2539	(dispatch_flags & I915_DISPATCH_RS ?
		2540	MI_BATCH_RESOURCE_STREAMER : 0));
5060	serge	2541	intel_ring_emit(ring, lower_32_bits(offset));
		2542	intel_ring_emit(ring, upper_32_bits(offset));
4560	Serge	2543	intel_ring_emit(ring, MI_NOOP);
		2544	intel_ring_advance(ring);
		2545
		2546	return 0;
		2547	}
		2548
		2549	static int
6084	serge	2550	hsw_ring_dispatch_execbuffer(struct drm_i915_gem_request *req,
		2551	u64 offset, u32 len,
		2552	unsigned dispatch_flags)
3243	Serge	2553	{
6084	serge	2554	struct intel_engine_cs *ring = req->ring;
3243	Serge	2555	int ret;
		2556
6084	serge	2557	ret = intel_ring_begin(req, 2);
3243	Serge	2558	if (ret)
		2559	return ret;
		2560
		2561	intel_ring_emit(ring,
5354	serge	2562	MI_BATCH_BUFFER_START \|
6084	serge	2563	(dispatch_flags & I915_DISPATCH_SECURE ?
		2564
		2565	(dispatch_flags & I915_DISPATCH_RS ?
		2566	MI_BATCH_RESOURCE_STREAMER : 0));
3243	Serge	2567	/* bit0-7 is the length on GEN6+ */
		2568	intel_ring_emit(ring, offset);
		2569	intel_ring_advance(ring);
		2570
		2571	return 0;
		2572	}
		2573
		2574	static int
6084	serge	2575	gen6_ring_dispatch_execbuffer(struct drm_i915_gem_request *req,
5060	serge	2576	u64 offset, u32 len,
6084	serge	2577	unsigned dispatch_flags)
2332	Serge	2578	{
6084	serge	2579	struct intel_engine_cs *ring = req->ring;
		2580	int ret;
2332	Serge	2581
6084	serge	2582	ret = intel_ring_begin(req, 2);
		2583	if (ret)
		2584	return ret;
2332	Serge	2585
3243	Serge	2586	intel_ring_emit(ring,
		2587	MI_BATCH_BUFFER_START \|
6084	serge	2588	(dispatch_flags & I915_DISPATCH_SECURE ?
		2589
		2590	/* bit0-7 is the length on GEN6+ */
		2591	intel_ring_emit(ring, offset);
		2592	intel_ring_advance(ring);
2332	Serge	2593
6084	serge	2594	return 0;
2332	Serge	2595	}
		2596
		2597	/* Blitter support (SandyBridge+) */
		2598
6084	serge	2599	static int gen6_ring_flush(struct drm_i915_gem_request *req,
		2600	u32 invalidate, u32 flush)
2332	Serge	2601	{
6084	serge	2602	struct intel_engine_cs *ring = req->ring;
4104	Serge	2603	struct drm_device *dev = ring->dev;
2332	Serge	2604	uint32_t cmd;
		2605	int ret;
		2606
6084	serge	2607	ret = intel_ring_begin(req, 4);
2332	Serge	2608	if (ret)
		2609	return ret;
		2610
		2611	cmd = MI_FLUSH_DW;
6084	serge	2612	if (INTEL_INFO(dev)->gen >= 8)
4560	Serge	2613	cmd += 1;
6084	serge	2614
		2615	/* We always require a command barrier so that subsequent
		2616	* commands, such as breadcrumb interrupts, are strictly ordered
		2617	* wrt the contents of the write cache being flushed to memory
		2618	* (and thus being coherent from the CPU).
		2619	*/
		2620	cmd \|= MI_FLUSH_DW_STORE_INDEX \| MI_FLUSH_DW_OP_STOREDW;
		2621
3243	Serge	2622	/*
		2623	* Bspec vol 1c.3 - blitter engine command streamer:
		2624	* "If ENABLED, all TLBs will be invalidated once the flush
		2625	* operation is complete. This bit is only valid when the
		2626	* Post-Sync Operation field is a value of 1h or 3h."
		2627	*/
2332	Serge	2628	if (invalidate & I915_GEM_DOMAIN_RENDER)
6084	serge	2629	cmd \|= MI_INVALIDATE_TLB;
2332	Serge	2630	intel_ring_emit(ring, cmd);
3243	Serge	2631	intel_ring_emit(ring, I915_GEM_HWS_SCRATCH_ADDR \| MI_FLUSH_DW_USE_GTT);
6084	serge	2632	if (INTEL_INFO(dev)->gen >= 8) {
4560	Serge	2633	intel_ring_emit(ring, 0); /* upper addr */
		2634	intel_ring_emit(ring, 0); /* value */
		2635	} else {
6084	serge	2636	intel_ring_emit(ring, 0);
		2637	intel_ring_emit(ring, MI_NOOP);
4560	Serge	2638	}
2332	Serge	2639	intel_ring_advance(ring);
4104	Serge	2640
2332	Serge	2641	return 0;
		2642	}
		2643
		2644	int intel_init_render_ring_buffer(struct drm_device *dev)
		2645	{
5060	serge	2646	struct drm_i915_private *dev_priv = dev->dev_private;
		2647	struct intel_engine_cs *ring = &dev_priv->ring[RCS];
		2648	struct drm_i915_gem_object *obj;
		2649	int ret;
2340	Serge	2650
3031	serge	2651	ring->name = "render ring";
		2652	ring->id = RCS;
		2653	ring->mmio_base = RENDER_RING_BASE;
		2654
5060	serge	2655	if (INTEL_INFO(dev)->gen >= 8) {
		2656	if (i915_semaphore_is_enabled(dev)) {
		2657	obj = i915_gem_alloc_object(dev, 4096);
		2658	if (obj == NULL) {
		2659	DRM_ERROR("Failed to allocate semaphore bo. Disabling semaphores\n");
		2660	i915.semaphores = 0;
		2661	} else {
		2662	i915_gem_object_set_cache_level(obj, I915_CACHE_LLC);
		2663	ret = i915_gem_obj_ggtt_pin(obj, 0, PIN_NONBLOCK);
		2664	if (ret != 0) {
		2665	drm_gem_object_unreference(&obj->base);
		2666	DRM_ERROR("Failed to pin semaphore bo. Disabling semaphores\n");
		2667	i915.semaphores = 0;
		2668	} else
		2669	dev_priv->semaphore_obj = obj;
		2670	}
		2671	}
5354	serge	2672
6084	serge	2673	ring->init_context = intel_rcs_ctx_init;
5060	serge	2674	ring->add_request = gen6_add_request;
		2675	ring->flush = gen8_render_ring_flush;
		2676	ring->irq_get = gen8_ring_get_irq;
		2677	ring->irq_put = gen8_ring_put_irq;
		2678	ring->irq_enable_mask = GT_RENDER_USER_INTERRUPT;
		2679	ring->get_seqno = gen6_ring_get_seqno;
		2680	ring->set_seqno = ring_set_seqno;
		2681	if (i915_semaphore_is_enabled(dev)) {
		2682	WARN_ON(!dev_priv->semaphore_obj);
		2683	ring->semaphore.sync_to = gen8_ring_sync;
		2684	ring->semaphore.signal = gen8_rcs_signal;
		2685	GEN8_RING_SEMAPHORE_INIT;
		2686	}
		2687	} else if (INTEL_INFO(dev)->gen >= 6) {
6084	serge	2688	ring->init_context = intel_rcs_ctx_init;
		2689	ring->add_request = gen6_add_request;
3031	serge	2690	ring->flush = gen7_render_ring_flush;
		2691	if (INTEL_INFO(dev)->gen == 6)
6084	serge	2692	ring->flush = gen6_render_ring_flush;
3031	serge	2693	ring->irq_get = gen6_ring_get_irq;
		2694	ring->irq_put = gen6_ring_put_irq;
4104	Serge	2695	ring->irq_enable_mask = GT_RENDER_USER_INTERRUPT;
2342	Serge	2696	ring->get_seqno = gen6_ring_get_seqno;
3480	Serge	2697	ring->set_seqno = ring_set_seqno;
5060	serge	2698	if (i915_semaphore_is_enabled(dev)) {
6084	serge	2699	ring->semaphore.sync_to = gen6_ring_sync;
		2700	ring->semaphore.signal = gen6_signal;
		2701	/*
5060	serge	2702	* The current semaphore is only applied on pre-gen8
		2703	* platform. And there is no VCS2 ring on the pre-gen8
		2704	* platform. So the semaphore between RCS and VCS2 is
		2705	* initialized as INVALID. Gen8 will initialize the
		2706	* sema between VCS2 and RCS later.
6084	serge	2707	*/
		2708	ring->semaphore.mbox.wait[RCS] = MI_SEMAPHORE_SYNC_INVALID;
		2709	ring->semaphore.mbox.wait[VCS] = MI_SEMAPHORE_SYNC_RV;
		2710	ring->semaphore.mbox.wait[BCS] = MI_SEMAPHORE_SYNC_RB;
		2711	ring->semaphore.mbox.wait[VECS] = MI_SEMAPHORE_SYNC_RVE;
		2712	ring->semaphore.mbox.wait[VCS2] = MI_SEMAPHORE_SYNC_INVALID;
		2713	ring->semaphore.mbox.signal[RCS] = GEN6_NOSYNC;
		2714	ring->semaphore.mbox.signal[VCS] = GEN6_VRSYNC;
		2715	ring->semaphore.mbox.signal[BCS] = GEN6_BRSYNC;
		2716	ring->semaphore.mbox.signal[VECS] = GEN6_VERSYNC;
		2717	ring->semaphore.mbox.signal[VCS2] = GEN6_NOSYNC;
5060	serge	2718	}
2332	Serge	2719	} else if (IS_GEN5(dev)) {
6084	serge	2720	ring->add_request = pc_render_add_request;
3031	serge	2721	ring->flush = gen4_render_ring_flush;
2342	Serge	2722	ring->get_seqno = pc_render_get_seqno;
3480	Serge	2723	ring->set_seqno = pc_render_set_seqno;
3031	serge	2724	ring->irq_get = gen5_ring_get_irq;
		2725	ring->irq_put = gen5_ring_put_irq;
4104	Serge	2726	ring->irq_enable_mask = GT_RENDER_USER_INTERRUPT \|
		2727	GT_RENDER_PIPECTL_NOTIFY_INTERRUPT;
3031	serge	2728	} else {
		2729	ring->add_request = i9xx_add_request;
		2730	if (INTEL_INFO(dev)->gen < 4)
		2731	ring->flush = gen2_render_ring_flush;
		2732	else
		2733	ring->flush = gen4_render_ring_flush;
		2734	ring->get_seqno = ring_get_seqno;
3480	Serge	2735	ring->set_seqno = ring_set_seqno;
3031	serge	2736	if (IS_GEN2(dev)) {
		2737	ring->irq_get = i8xx_ring_get_irq;
		2738	ring->irq_put = i8xx_ring_put_irq;
		2739	} else {
		2740	ring->irq_get = i9xx_ring_get_irq;
		2741	ring->irq_put = i9xx_ring_put_irq;
		2742	}
		2743	ring->irq_enable_mask = I915_USER_INTERRUPT;
2332	Serge	2744	}
3031	serge	2745	ring->write_tail = ring_write_tail;
5060	serge	2746
3243	Serge	2747	if (IS_HASWELL(dev))
		2748	ring->dispatch_execbuffer = hsw_ring_dispatch_execbuffer;
4560	Serge	2749	else if (IS_GEN8(dev))
		2750	ring->dispatch_execbuffer = gen8_ring_dispatch_execbuffer;
3243	Serge	2751	else if (INTEL_INFO(dev)->gen >= 6)
3031	serge	2752	ring->dispatch_execbuffer = gen6_ring_dispatch_execbuffer;
		2753	else if (INTEL_INFO(dev)->gen >= 4)
		2754	ring->dispatch_execbuffer = i965_dispatch_execbuffer;
		2755	else if (IS_I830(dev) \|\| IS_845G(dev))
		2756	ring->dispatch_execbuffer = i830_dispatch_execbuffer;
		2757	else
		2758	ring->dispatch_execbuffer = i915_dispatch_execbuffer;
6084	serge	2759	ring->init_hw = init_render_ring;
3031	serge	2760	ring->cleanup = render_ring_cleanup;
2332	Serge	2761
3243	Serge	2762	/* Workaround batchbuffer to combat CS tlb bug. */
		2763	if (HAS_BROKEN_CS_TLB(dev)) {
5128	serge	2764	obj = i915_gem_alloc_object(dev, I830_WA_SIZE);
3243	Serge	2765	if (obj == NULL) {
		2766	DRM_ERROR("Failed to allocate batch bo\n");
		2767	return -ENOMEM;
		2768	}
		2769
5060	serge	2770	ret = i915_gem_obj_ggtt_pin(obj, 0, 0);
3243	Serge	2771	if (ret != 0) {
		2772	drm_gem_object_unreference(&obj->base);
		2773	DRM_ERROR("Failed to ping batch bo\n");
		2774	return ret;
		2775	}
		2776
4104	Serge	2777	ring->scratch.obj = obj;
		2778	ring->scratch.gtt_offset = i915_gem_obj_ggtt_offset(obj);
2332	Serge	2779	}
2340	Serge	2780
6084	serge	2781	ret = intel_init_ring_buffer(dev, ring);
		2782	if (ret)
		2783	return ret;
		2784
		2785	if (INTEL_INFO(dev)->gen >= 5) {
		2786	ret = intel_init_pipe_control(ring);
		2787	if (ret)
		2788	return ret;
		2789	}
		2790
		2791	return 0;
2332	Serge	2792	}
		2793
		2794	int intel_init_bsd_ring_buffer(struct drm_device *dev)
		2795	{
5060	serge	2796	struct drm_i915_private *dev_priv = dev->dev_private;
		2797	struct intel_engine_cs *ring = &dev_priv->ring[VCS];
2332	Serge	2798
3031	serge	2799	ring->name = "bsd ring";
		2800	ring->id = VCS;
2332	Serge	2801
3031	serge	2802	ring->write_tail = ring_write_tail;
4560	Serge	2803	if (INTEL_INFO(dev)->gen >= 6) {
3031	serge	2804	ring->mmio_base = GEN6_BSD_RING_BASE;
		2805	/* gen6 bsd needs a special wa for tail updates */
		2806	if (IS_GEN6(dev))
		2807	ring->write_tail = gen6_bsd_ring_write_tail;
4104	Serge	2808	ring->flush = gen6_bsd_ring_flush;
3031	serge	2809	ring->add_request = gen6_add_request;
		2810	ring->get_seqno = gen6_ring_get_seqno;
3480	Serge	2811	ring->set_seqno = ring_set_seqno;
4560	Serge	2812	if (INTEL_INFO(dev)->gen >= 8) {
		2813	ring->irq_enable_mask =
		2814	GT_RENDER_USER_INTERRUPT << GEN8_VCS1_IRQ_SHIFT;
		2815	ring->irq_get = gen8_ring_get_irq;
		2816	ring->irq_put = gen8_ring_put_irq;
		2817	ring->dispatch_execbuffer =
		2818	gen8_ring_dispatch_execbuffer;
5060	serge	2819	if (i915_semaphore_is_enabled(dev)) {
		2820	ring->semaphore.sync_to = gen8_ring_sync;
		2821	ring->semaphore.signal = gen8_xcs_signal;
		2822	GEN8_RING_SEMAPHORE_INIT;
		2823	}
4560	Serge	2824	} else {
6084	serge	2825	ring->irq_enable_mask = GT_BSD_USER_INTERRUPT;
		2826	ring->irq_get = gen6_ring_get_irq;
		2827	ring->irq_put = gen6_ring_put_irq;
4560	Serge	2828	ring->dispatch_execbuffer =
		2829	gen6_ring_dispatch_execbuffer;
5060	serge	2830	if (i915_semaphore_is_enabled(dev)) {
6084	serge	2831	ring->semaphore.sync_to = gen6_ring_sync;
		2832	ring->semaphore.signal = gen6_signal;
		2833	ring->semaphore.mbox.wait[RCS] = MI_SEMAPHORE_SYNC_VR;
		2834	ring->semaphore.mbox.wait[VCS] = MI_SEMAPHORE_SYNC_INVALID;
		2835	ring->semaphore.mbox.wait[BCS] = MI_SEMAPHORE_SYNC_VB;
		2836	ring->semaphore.mbox.wait[VECS] = MI_SEMAPHORE_SYNC_VVE;
		2837	ring->semaphore.mbox.wait[VCS2] = MI_SEMAPHORE_SYNC_INVALID;
		2838	ring->semaphore.mbox.signal[RCS] = GEN6_RVSYNC;
		2839	ring->semaphore.mbox.signal[VCS] = GEN6_NOSYNC;
		2840	ring->semaphore.mbox.signal[BCS] = GEN6_BVSYNC;
		2841	ring->semaphore.mbox.signal[VECS] = GEN6_VEVSYNC;
		2842	ring->semaphore.mbox.signal[VCS2] = GEN6_NOSYNC;
5060	serge	2843	}
4560	Serge	2844	}
3031	serge	2845	} else {
		2846	ring->mmio_base = BSD_RING_BASE;
		2847	ring->flush = bsd_ring_flush;
		2848	ring->add_request = i9xx_add_request;
		2849	ring->get_seqno = ring_get_seqno;
3480	Serge	2850	ring->set_seqno = ring_set_seqno;
3031	serge	2851	if (IS_GEN5(dev)) {
4104	Serge	2852	ring->irq_enable_mask = ILK_BSD_USER_INTERRUPT;
3031	serge	2853	ring->irq_get = gen5_ring_get_irq;
		2854	ring->irq_put = gen5_ring_put_irq;
		2855	} else {
		2856	ring->irq_enable_mask = I915_BSD_USER_INTERRUPT;
		2857	ring->irq_get = i9xx_ring_get_irq;
		2858	ring->irq_put = i9xx_ring_put_irq;
		2859	}
		2860	ring->dispatch_execbuffer = i965_dispatch_execbuffer;
		2861	}
6084	serge	2862	ring->init_hw = init_ring_common;
3031	serge	2863
2332	Serge	2864	return intel_init_ring_buffer(dev, ring);
		2865	}
		2866
5060	serge	2867	/**
6084	serge	2868	* Initialize the second BSD ring (eg. Broadwell GT3, Skylake GT3)
5060	serge	2869	*/
		2870	int intel_init_bsd2_ring_buffer(struct drm_device *dev)
		2871	{
		2872	struct drm_i915_private *dev_priv = dev->dev_private;
		2873	struct intel_engine_cs *ring = &dev_priv->ring[VCS2];
		2874
		2875	ring->name = "bsd2 ring";
		2876	ring->id = VCS2;
		2877
		2878	ring->write_tail = ring_write_tail;
		2879	ring->mmio_base = GEN8_BSD2_RING_BASE;
		2880	ring->flush = gen6_bsd_ring_flush;
		2881	ring->add_request = gen6_add_request;
		2882	ring->get_seqno = gen6_ring_get_seqno;
		2883	ring->set_seqno = ring_set_seqno;
		2884	ring->irq_enable_mask =
		2885	GT_RENDER_USER_INTERRUPT << GEN8_VCS2_IRQ_SHIFT;
		2886	ring->irq_get = gen8_ring_get_irq;
		2887	ring->irq_put = gen8_ring_put_irq;
		2888	ring->dispatch_execbuffer =
		2889	gen8_ring_dispatch_execbuffer;
		2890	if (i915_semaphore_is_enabled(dev)) {
		2891	ring->semaphore.sync_to = gen8_ring_sync;
		2892	ring->semaphore.signal = gen8_xcs_signal;
		2893	GEN8_RING_SEMAPHORE_INIT;
		2894	}
6084	serge	2895	ring->init_hw = init_ring_common;
5060	serge	2896
		2897	return intel_init_ring_buffer(dev, ring);
		2898	}
		2899
2332	Serge	2900	int intel_init_blt_ring_buffer(struct drm_device *dev)
		2901	{
5060	serge	2902	struct drm_i915_private *dev_priv = dev->dev_private;
		2903	struct intel_engine_cs *ring = &dev_priv->ring[BCS];
2332	Serge	2904
3031	serge	2905	ring->name = "blitter ring";
		2906	ring->id = BCS;
2332	Serge	2907
3031	serge	2908	ring->mmio_base = BLT_RING_BASE;
		2909	ring->write_tail = ring_write_tail;
4104	Serge	2910	ring->flush = gen6_ring_flush;
3031	serge	2911	ring->add_request = gen6_add_request;
		2912	ring->get_seqno = gen6_ring_get_seqno;
3480	Serge	2913	ring->set_seqno = ring_set_seqno;
4560	Serge	2914	if (INTEL_INFO(dev)->gen >= 8) {
		2915	ring->irq_enable_mask =
		2916	GT_RENDER_USER_INTERRUPT << GEN8_BCS_IRQ_SHIFT;
		2917	ring->irq_get = gen8_ring_get_irq;
		2918	ring->irq_put = gen8_ring_put_irq;
		2919	ring->dispatch_execbuffer = gen8_ring_dispatch_execbuffer;
5060	serge	2920	if (i915_semaphore_is_enabled(dev)) {
		2921	ring->semaphore.sync_to = gen8_ring_sync;
		2922	ring->semaphore.signal = gen8_xcs_signal;
		2923	GEN8_RING_SEMAPHORE_INIT;
		2924	}
4560	Serge	2925	} else {
6084	serge	2926	ring->irq_enable_mask = GT_BLT_USER_INTERRUPT;
		2927	ring->irq_get = gen6_ring_get_irq;
		2928	ring->irq_put = gen6_ring_put_irq;
		2929	ring->dispatch_execbuffer = gen6_ring_dispatch_execbuffer;
5060	serge	2930	if (i915_semaphore_is_enabled(dev)) {
		2931	ring->semaphore.signal = gen6_signal;
6084	serge	2932	ring->semaphore.sync_to = gen6_ring_sync;
		2933	/*
5060	serge	2934	* The current semaphore is only applied on pre-gen8
		2935	* platform. And there is no VCS2 ring on the pre-gen8
		2936	* platform. So the semaphore between BCS and VCS2 is
		2937	* initialized as INVALID. Gen8 will initialize the
		2938	* sema between BCS and VCS2 later.
6084	serge	2939	*/
		2940	ring->semaphore.mbox.wait[RCS] = MI_SEMAPHORE_SYNC_BR;
		2941	ring->semaphore.mbox.wait[VCS] = MI_SEMAPHORE_SYNC_BV;
		2942	ring->semaphore.mbox.wait[BCS] = MI_SEMAPHORE_SYNC_INVALID;
		2943	ring->semaphore.mbox.wait[VECS] = MI_SEMAPHORE_SYNC_BVE;
		2944	ring->semaphore.mbox.wait[VCS2] = MI_SEMAPHORE_SYNC_INVALID;
		2945	ring->semaphore.mbox.signal[RCS] = GEN6_RBSYNC;
		2946	ring->semaphore.mbox.signal[VCS] = GEN6_VBSYNC;
		2947	ring->semaphore.mbox.signal[BCS] = GEN6_NOSYNC;
		2948	ring->semaphore.mbox.signal[VECS] = GEN6_VEBSYNC;
		2949	ring->semaphore.mbox.signal[VCS2] = GEN6_NOSYNC;
5060	serge	2950	}
4560	Serge	2951	}
6084	serge	2952	ring->init_hw = init_ring_common;
3031	serge	2953
2332	Serge	2954	return intel_init_ring_buffer(dev, ring);
		2955	}
3031	serge	2956
4104	Serge	2957	int intel_init_vebox_ring_buffer(struct drm_device *dev)
		2958	{
5060	serge	2959	struct drm_i915_private *dev_priv = dev->dev_private;
		2960	struct intel_engine_cs *ring = &dev_priv->ring[VECS];
4104	Serge	2961
		2962	ring->name = "video enhancement ring";
		2963	ring->id = VECS;
		2964
		2965	ring->mmio_base = VEBOX_RING_BASE;
		2966	ring->write_tail = ring_write_tail;
		2967	ring->flush = gen6_ring_flush;
		2968	ring->add_request = gen6_add_request;
		2969	ring->get_seqno = gen6_ring_get_seqno;
		2970	ring->set_seqno = ring_set_seqno;
4560	Serge	2971
		2972	if (INTEL_INFO(dev)->gen >= 8) {
		2973	ring->irq_enable_mask =
		2974	GT_RENDER_USER_INTERRUPT << GEN8_VECS_IRQ_SHIFT;
		2975	ring->irq_get = gen8_ring_get_irq;
		2976	ring->irq_put = gen8_ring_put_irq;
		2977	ring->dispatch_execbuffer = gen8_ring_dispatch_execbuffer;
5060	serge	2978	if (i915_semaphore_is_enabled(dev)) {
		2979	ring->semaphore.sync_to = gen8_ring_sync;
		2980	ring->semaphore.signal = gen8_xcs_signal;
		2981	GEN8_RING_SEMAPHORE_INIT;
		2982	}
4560	Serge	2983	} else {
6084	serge	2984	ring->irq_enable_mask = PM_VEBOX_USER_INTERRUPT;
		2985	ring->irq_get = hsw_vebox_get_irq;
		2986	ring->irq_put = hsw_vebox_put_irq;
		2987	ring->dispatch_execbuffer = gen6_ring_dispatch_execbuffer;
5060	serge	2988	if (i915_semaphore_is_enabled(dev)) {
6084	serge	2989	ring->semaphore.sync_to = gen6_ring_sync;
		2990	ring->semaphore.signal = gen6_signal;
		2991	ring->semaphore.mbox.wait[RCS] = MI_SEMAPHORE_SYNC_VER;
		2992	ring->semaphore.mbox.wait[VCS] = MI_SEMAPHORE_SYNC_VEV;
		2993	ring->semaphore.mbox.wait[BCS] = MI_SEMAPHORE_SYNC_VEB;
		2994	ring->semaphore.mbox.wait[VECS] = MI_SEMAPHORE_SYNC_INVALID;
		2995	ring->semaphore.mbox.wait[VCS2] = MI_SEMAPHORE_SYNC_INVALID;
		2996	ring->semaphore.mbox.signal[RCS] = GEN6_RVESYNC;
		2997	ring->semaphore.mbox.signal[VCS] = GEN6_VVESYNC;
		2998	ring->semaphore.mbox.signal[BCS] = GEN6_BVESYNC;
		2999	ring->semaphore.mbox.signal[VECS] = GEN6_NOSYNC;
		3000	ring->semaphore.mbox.signal[VCS2] = GEN6_NOSYNC;
5060	serge	3001	}
4560	Serge	3002	}
6084	serge	3003	ring->init_hw = init_ring_common;
4104	Serge	3004
		3005	return intel_init_ring_buffer(dev, ring);
		3006	}
		3007
3031	serge	3008	int
6084	serge	3009	intel_ring_flush_all_caches(struct drm_i915_gem_request *req)
3031	serge	3010	{
6084	serge	3011	struct intel_engine_cs *ring = req->ring;
3031	serge	3012	int ret;
		3013
		3014	if (!ring->gpu_caches_dirty)
		3015	return 0;
		3016
6084	serge	3017	ret = ring->flush(req, 0, I915_GEM_GPU_DOMAINS);
3031	serge	3018	if (ret)
		3019	return ret;
		3020
6084	serge	3021	trace_i915_gem_ring_flush(req, 0, I915_GEM_GPU_DOMAINS);
3031	serge	3022
		3023	ring->gpu_caches_dirty = false;
		3024	return 0;
		3025	}
		3026
		3027	int
6084	serge	3028	intel_ring_invalidate_all_caches(struct drm_i915_gem_request *req)
3031	serge	3029	{
6084	serge	3030	struct intel_engine_cs *ring = req->ring;
3031	serge	3031	uint32_t flush_domains;
		3032	int ret;
		3033
		3034	flush_domains = 0;
		3035	if (ring->gpu_caches_dirty)
		3036	flush_domains = I915_GEM_GPU_DOMAINS;
		3037
6084	serge	3038	ret = ring->flush(req, I915_GEM_GPU_DOMAINS, flush_domains);
3031	serge	3039	if (ret)
		3040	return ret;
		3041
6084	serge	3042	trace_i915_gem_ring_flush(req, I915_GEM_GPU_DOMAINS, flush_domains);
3031	serge	3043
		3044	ring->gpu_caches_dirty = false;
		3045	return 0;
		3046	}
5060	serge	3047
		3048	void
		3049	intel_stop_ring_buffer(struct intel_engine_cs *ring)
		3050	{
		3051	int ret;
		3052
		3053	if (!intel_ring_initialized(ring))
		3054	return;
		3055
		3056	ret = intel_ring_idle(ring);
		3057	if (ret && !i915_reset_in_progress(&to_i915(ring->dev)->gpu_error))
		3058	DRM_ERROR("failed to quiesce %s whilst cleaning up: %d\n",
		3059	ring->name, ret);
		3060
		3061	stop_ring(ring);
		3062	}

Subversion Repositories Kolibri OS

(root)/drivers/video/drm/i915/intel_ringbuffer.c @ 4557 – Rev 6660