WebSVN – Kolibri OS – Blame – /drivers/video/drm/i915/intel_ringbuffer.c

Rev	Author	Line No.	Line
2332	Serge	1	/*
		2	* Copyright © 2008-2010 Intel Corporation
		3	*
		4	* Permission is hereby granted, free of charge, to any person obtaining a
		5	* copy of this software and associated documentation files (the "Software"),
		6	* to deal in the Software without restriction, including without limitation
		7	* the rights to use, copy, modify, merge, publish, distribute, sublicense,
		8	* and/or sell copies of the Software, and to permit persons to whom the
		9	* Software is furnished to do so, subject to the following conditions:
		10	*
		11	* The above copyright notice and this permission notice (including the next
		12	* paragraph) shall be included in all copies or substantial portions of the
		13	* Software.
		14	*
		15	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
		16	* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
		17	* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
		18	* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
		19	* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
		20	* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
		21	* IN THE SOFTWARE.
		22	*
		23	* Authors:
		24	* Eric Anholt
		25	* Zou Nan hai
		26	* Xiang Hai hao
		27	*
		28	*/
		29
3031	serge	30	#include
2332	Serge	31	#include "i915_drv.h"
3031	serge	32	#include
2351	Serge	33	#include "i915_trace.h"
2332	Serge	34	#include "intel_drv.h"
		35
5060	serge	36	/* Early gen2 devices have a cacheline of just 32 bytes, using 64 is overkill,
		37	* but keeps the logic simple. Indeed, the whole purpose of this macro is just
		38	* to give some inclination as to some of the magic values used in the various
		39	* workarounds!
		40	*/
		41	#define CACHELINE_BYTES 64
		42
		43	static inline int __ring_space(int head, int tail, int size)
2332	Serge	44	{
5060	serge	45	int space = head - (tail + I915_RING_FREE_SPACE);
2332	Serge	46	if (space < 0)
5060	serge	47	space += size;
2332	Serge	48	return space;
		49	}
		50
5060	serge	51	static inline int ring_space(struct intel_ringbuffer *ringbuf)
4560	Serge	52	{
5060	serge	53	return __ring_space(ringbuf->head & HEAD_ADDR, ringbuf->tail, ringbuf->size);
		54	}
		55
		56	static bool intel_ring_stopped(struct intel_engine_cs *ring)
		57	{
4560	Serge	58	struct drm_i915_private *dev_priv = ring->dev->dev_private;
5060	serge	59	return dev_priv->gpu_error.stop_rings & intel_ring_flag(ring);
		60	}
4560	Serge	61
5060	serge	62	void __intel_ring_advance(struct intel_engine_cs *ring)
		63	{
		64	struct intel_ringbuffer *ringbuf = ring->buffer;
		65	ringbuf->tail &= ringbuf->size - 1;
		66	if (intel_ring_stopped(ring))
4560	Serge	67	return;
5060	serge	68	ring->write_tail(ring, ringbuf->tail);
4560	Serge	69	}
		70
3031	serge	71	static int
5060	serge	72	gen2_render_ring_flush(struct intel_engine_cs *ring,
3031	serge	73	u32 invalidate_domains,
		74	u32 flush_domains)
2332	Serge	75	{
3031	serge	76	u32 cmd;
		77	int ret;
2332	Serge	78
3031	serge	79	cmd = MI_FLUSH;
		80	if (((invalidate_domains\|flush_domains) & I915_GEM_DOMAIN_RENDER) == 0)
		81	cmd \|= MI_NO_WRITE_FLUSH;
2332	Serge	82
3031	serge	83	if (invalidate_domains & I915_GEM_DOMAIN_SAMPLER)
		84	cmd \|= MI_READ_FLUSH;
2332	Serge	85
3031	serge	86	ret = intel_ring_begin(ring, 2);
		87	if (ret)
		88	return ret;
		89
		90	intel_ring_emit(ring, cmd);
		91	intel_ring_emit(ring, MI_NOOP);
		92	intel_ring_advance(ring);
		93
		94	return 0;
2332	Serge	95	}
		96
		97	static int
5060	serge	98	gen4_render_ring_flush(struct intel_engine_cs *ring,
2332	Serge	99	u32 invalidate_domains,
		100	u32 flush_domains)
		101	{
		102	struct drm_device *dev = ring->dev;
		103	u32 cmd;
		104	int ret;
		105
		106	/*
		107	* read/write caches:
		108	*
		109	* I915_GEM_DOMAIN_RENDER is always invalidated, but is
		110	* only flushed if MI_NO_WRITE_FLUSH is unset. On 965, it is
		111	* also flushed at 2d versus 3d pipeline switches.
		112	*
		113	* read-only caches:
		114	*
		115	* I915_GEM_DOMAIN_SAMPLER is flushed on pre-965 if
		116	* MI_READ_FLUSH is set, and is always flushed on 965.
		117	*
		118	* I915_GEM_DOMAIN_COMMAND may not exist?
		119	*
		120	* I915_GEM_DOMAIN_INSTRUCTION, which exists on 965, is
		121	* invalidated when MI_EXE_FLUSH is set.
		122	*
		123	* I915_GEM_DOMAIN_VERTEX, which exists on 965, is
		124	* invalidated with every MI_FLUSH.
		125	*
		126	* TLBs:
		127	*
		128	* On 965, TLBs associated with I915_GEM_DOMAIN_COMMAND
		129	* and I915_GEM_DOMAIN_CPU in are invalidated at PTE write and
		130	* I915_GEM_DOMAIN_RENDER and I915_GEM_DOMAIN_SAMPLER
		131	* are flushed at any MI_FLUSH.
		132	*/
		133
		134	cmd = MI_FLUSH \| MI_NO_WRITE_FLUSH;
3031	serge	135	if ((invalidate_domains\|flush_domains) & I915_GEM_DOMAIN_RENDER)
2332	Serge	136	cmd &= ~MI_NO_WRITE_FLUSH;
		137	if (invalidate_domains & I915_GEM_DOMAIN_INSTRUCTION)
		138	cmd \|= MI_EXE_FLUSH;
		139
		140	if (invalidate_domains & I915_GEM_DOMAIN_COMMAND &&
		141	(IS_G4X(dev) \|\| IS_GEN5(dev)))
		142	cmd \|= MI_INVALIDATE_ISP;
		143
		144	ret = intel_ring_begin(ring, 2);
		145	if (ret)
		146	return ret;
		147
		148	intel_ring_emit(ring, cmd);
		149	intel_ring_emit(ring, MI_NOOP);
		150	intel_ring_advance(ring);
		151
		152	return 0;
		153	}
		154
2342	Serge	155	/**
		156	* Emits a PIPE_CONTROL with a non-zero post-sync operation, for
		157	* implementing two workarounds on gen6. From section 1.4.7.1
		158	* "PIPE_CONTROL" of the Sandy Bridge PRM volume 2 part 1:
		159	*
		160	* [DevSNB-C+{W/A}] Before any depth stall flush (including those
		161	* produced by non-pipelined state commands), software needs to first
		162	* send a PIPE_CONTROL with no bits set except Post-Sync Operation !=
		163	* 0.
		164	*
		165	* [Dev-SNB{W/A}]: Before a PIPE_CONTROL with Write Cache Flush Enable
		166	* =1, a PIPE_CONTROL with any non-zero post-sync-op is required.
		167	*
		168	* And the workaround for these two requires this workaround first:
		169	*
		170	* [Dev-SNB{W/A}]: Pipe-control with CS-stall bit set must be sent
		171	* BEFORE the pipe-control with a post-sync op and no write-cache
		172	* flushes.
		173	*
		174	* And this last workaround is tricky because of the requirements on
		175	* that bit. From section 1.4.7.2.3 "Stall" of the Sandy Bridge PRM
		176	* volume 2 part 1:
		177	*
		178	* "1 of the following must also be set:
		179	* - Render Target Cache Flush Enable ([12] of DW1)
		180	* - Depth Cache Flush Enable ([0] of DW1)
		181	* - Stall at Pixel Scoreboard ([1] of DW1)
		182	* - Depth Stall ([13] of DW1)
		183	* - Post-Sync Operation ([13] of DW1)
		184	* - Notify Enable ([8] of DW1)"
		185	*
		186	* The cache flushes require the workaround flush that triggered this
		187	* one, so we can't use it. Depth stall would trigger the same.
		188	* Post-sync nonzero is what triggered this second workaround, so we
		189	* can't use that one either. Notify enable is IRQs, which aren't
		190	* really our business. That leaves only stall at scoreboard.
		191	*/
		192	static int
5060	serge	193	intel_emit_post_sync_nonzero_flush(struct intel_engine_cs *ring)
2342	Serge	194	{
5060	serge	195	u32 scratch_addr = ring->scratch.gtt_offset + 2 * CACHELINE_BYTES;
2342	Serge	196	int ret;
		197
		198
		199	ret = intel_ring_begin(ring, 6);
		200	if (ret)
		201	return ret;
		202
		203	intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(5));
		204	intel_ring_emit(ring, PIPE_CONTROL_CS_STALL \|
		205	PIPE_CONTROL_STALL_AT_SCOREBOARD);
		206	intel_ring_emit(ring, scratch_addr \| PIPE_CONTROL_GLOBAL_GTT); /* address */
		207	intel_ring_emit(ring, 0); /* low dword */
		208	intel_ring_emit(ring, 0); /* high dword */
		209	intel_ring_emit(ring, MI_NOOP);
		210	intel_ring_advance(ring);
		211
		212	ret = intel_ring_begin(ring, 6);
		213	if (ret)
		214	return ret;
		215
		216	intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(5));
		217	intel_ring_emit(ring, PIPE_CONTROL_QW_WRITE);
		218	intel_ring_emit(ring, scratch_addr \| PIPE_CONTROL_GLOBAL_GTT); /* address */
		219	intel_ring_emit(ring, 0);
		220	intel_ring_emit(ring, 0);
		221	intel_ring_emit(ring, MI_NOOP);
		222	intel_ring_advance(ring);
		223
		224	return 0;
		225	}
		226
		227	static int
5060	serge	228	gen6_render_ring_flush(struct intel_engine_cs *ring,
2342	Serge	229	u32 invalidate_domains, u32 flush_domains)
		230	{
		231	u32 flags = 0;
5060	serge	232	u32 scratch_addr = ring->scratch.gtt_offset + 2 * CACHELINE_BYTES;
2342	Serge	233	int ret;
		234
		235	/* Force SNB workarounds for PIPE_CONTROL flushes */
3031	serge	236	ret = intel_emit_post_sync_nonzero_flush(ring);
		237	if (ret)
		238	return ret;
2342	Serge	239
		240	/* Just flush everything. Experiments have shown that reducing the
		241	* number of bits based on the write domains has little performance
		242	* impact.
		243	*/
3031	serge	244	if (flush_domains) {
		245	flags \|= PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH;
		246	flags \|= PIPE_CONTROL_DEPTH_CACHE_FLUSH;
		247	/*
		248	* Ensure that any following seqno writes only happen
		249	* when the render cache is indeed flushed.
		250	*/
		251	flags \|= PIPE_CONTROL_CS_STALL;
		252	}
		253	if (invalidate_domains) {
		254	flags \|= PIPE_CONTROL_TLB_INVALIDATE;
		255	flags \|= PIPE_CONTROL_INSTRUCTION_CACHE_INVALIDATE;
		256	flags \|= PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE;
		257	flags \|= PIPE_CONTROL_VF_CACHE_INVALIDATE;
		258	flags \|= PIPE_CONTROL_CONST_CACHE_INVALIDATE;
		259	flags \|= PIPE_CONTROL_STATE_CACHE_INVALIDATE;
		260	/*
		261	* TLB invalidate requires a post-sync write.
		262	*/
3243	Serge	263	flags \|= PIPE_CONTROL_QW_WRITE \| PIPE_CONTROL_CS_STALL;
3031	serge	264	}
		265
		266	ret = intel_ring_begin(ring, 4);
		267	if (ret)
		268	return ret;
		269
		270	intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(4));
		271	intel_ring_emit(ring, flags);
		272	intel_ring_emit(ring, scratch_addr \| PIPE_CONTROL_GLOBAL_GTT);
		273	intel_ring_emit(ring, 0);
		274	intel_ring_advance(ring);
		275
		276	return 0;
		277	}
		278
		279	static int
5060	serge	280	gen7_render_ring_cs_stall_wa(struct intel_engine_cs *ring)
3031	serge	281	{
		282	int ret;
		283
		284	ret = intel_ring_begin(ring, 4);
		285	if (ret)
		286	return ret;
		287
		288	intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(4));
		289	intel_ring_emit(ring, PIPE_CONTROL_CS_STALL \|
		290	PIPE_CONTROL_STALL_AT_SCOREBOARD);
		291	intel_ring_emit(ring, 0);
		292	intel_ring_emit(ring, 0);
		293	intel_ring_advance(ring);
		294
		295	return 0;
		296	}
		297
5060	serge	298	static int gen7_ring_fbc_flush(struct intel_engine_cs *ring, u32 value)
4104	Serge	299	{
		300	int ret;
		301
		302	if (!ring->fbc_dirty)
		303	return 0;
		304
4560	Serge	305	ret = intel_ring_begin(ring, 6);
4104	Serge	306	if (ret)
		307	return ret;
		308	/* WaFbcNukeOn3DBlt:ivb/hsw */
		309	intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(1));
		310	intel_ring_emit(ring, MSG_FBC_REND_STATE);
		311	intel_ring_emit(ring, value);
4560	Serge	312	intel_ring_emit(ring, MI_STORE_REGISTER_MEM(1) \| MI_SRM_LRM_GLOBAL_GTT);
		313	intel_ring_emit(ring, MSG_FBC_REND_STATE);
		314	intel_ring_emit(ring, ring->scratch.gtt_offset + 256);
4104	Serge	315	intel_ring_advance(ring);
		316
		317	ring->fbc_dirty = false;
		318	return 0;
		319	}
		320
3031	serge	321	static int
5060	serge	322	gen7_render_ring_flush(struct intel_engine_cs *ring,
3031	serge	323	u32 invalidate_domains, u32 flush_domains)
		324	{
		325	u32 flags = 0;
5060	serge	326	u32 scratch_addr = ring->scratch.gtt_offset + 2 * CACHELINE_BYTES;
3031	serge	327	int ret;
		328
		329	/*
		330	* Ensure that any following seqno writes only happen when the render
		331	* cache is indeed flushed.
		332	*
		333	* Workaround: 4th PIPE_CONTROL command (except the ones with only
		334	* read-cache invalidate bits set) must have the CS_STALL bit set. We
		335	* don't try to be clever and just set it unconditionally.
		336	*/
		337	flags \|= PIPE_CONTROL_CS_STALL;
		338
		339	/* Just flush everything. Experiments have shown that reducing the
		340	* number of bits based on the write domains has little performance
		341	* impact.
		342	*/
		343	if (flush_domains) {
2342	Serge	344	flags \|= PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH;
3031	serge	345	flags \|= PIPE_CONTROL_DEPTH_CACHE_FLUSH;
		346	}
		347	if (invalidate_domains) {
		348	flags \|= PIPE_CONTROL_TLB_INVALIDATE;
2342	Serge	349	flags \|= PIPE_CONTROL_INSTRUCTION_CACHE_INVALIDATE;
		350	flags \|= PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE;
		351	flags \|= PIPE_CONTROL_VF_CACHE_INVALIDATE;
		352	flags \|= PIPE_CONTROL_CONST_CACHE_INVALIDATE;
		353	flags \|= PIPE_CONTROL_STATE_CACHE_INVALIDATE;
3031	serge	354	/*
		355	* TLB invalidate requires a post-sync write.
		356	*/
		357	flags \|= PIPE_CONTROL_QW_WRITE;
3480	Serge	358	flags \|= PIPE_CONTROL_GLOBAL_GTT_IVB;
2342	Serge	359
3031	serge	360	/* Workaround: we must issue a pipe_control with CS-stall bit
		361	* set before a pipe_control command that has the state cache
		362	* invalidate bit set. */
		363	gen7_render_ring_cs_stall_wa(ring);
		364	}
		365
		366	ret = intel_ring_begin(ring, 4);
2342	Serge	367	if (ret)
		368	return ret;
		369
3031	serge	370	intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(4));
2342	Serge	371	intel_ring_emit(ring, flags);
3480	Serge	372	intel_ring_emit(ring, scratch_addr);
3031	serge	373	intel_ring_emit(ring, 0);
2342	Serge	374	intel_ring_advance(ring);
		375
4560	Serge	376	if (!invalidate_domains && flush_domains)
4104	Serge	377	return gen7_ring_fbc_flush(ring, FBC_REND_NUKE);
		378
2342	Serge	379	return 0;
		380	}
		381
4560	Serge	382	static int
5060	serge	383	gen8_emit_pipe_control(struct intel_engine_cs *ring,
		384	u32 flags, u32 scratch_addr)
		385	{
		386	int ret;
		387
		388	ret = intel_ring_begin(ring, 6);
		389	if (ret)
		390	return ret;
		391
		392	intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(6));
		393	intel_ring_emit(ring, flags);
		394	intel_ring_emit(ring, scratch_addr);
		395	intel_ring_emit(ring, 0);
		396	intel_ring_emit(ring, 0);
		397	intel_ring_emit(ring, 0);
		398	intel_ring_advance(ring);
		399
		400	return 0;
		401	}
		402
		403	static int
		404	gen8_render_ring_flush(struct intel_engine_cs *ring,
4560	Serge	405	u32 invalidate_domains, u32 flush_domains)
		406	{
		407	u32 flags = 0;
5060	serge	408	u32 scratch_addr = ring->scratch.gtt_offset + 2 * CACHELINE_BYTES;
4560	Serge	409	int ret;
		410
		411	flags \|= PIPE_CONTROL_CS_STALL;
		412
		413	if (flush_domains) {
		414	flags \|= PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH;
		415	flags \|= PIPE_CONTROL_DEPTH_CACHE_FLUSH;
		416	}
		417	if (invalidate_domains) {
		418	flags \|= PIPE_CONTROL_TLB_INVALIDATE;
		419	flags \|= PIPE_CONTROL_INSTRUCTION_CACHE_INVALIDATE;
		420	flags \|= PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE;
		421	flags \|= PIPE_CONTROL_VF_CACHE_INVALIDATE;
		422	flags \|= PIPE_CONTROL_CONST_CACHE_INVALIDATE;
		423	flags \|= PIPE_CONTROL_STATE_CACHE_INVALIDATE;
		424	flags \|= PIPE_CONTROL_QW_WRITE;
		425	flags \|= PIPE_CONTROL_GLOBAL_GTT_IVB;
		426
5060	serge	427	/* WaCsStallBeforeStateCacheInvalidate:bdw,chv */
		428	ret = gen8_emit_pipe_control(ring,
		429	PIPE_CONTROL_CS_STALL \|
		430	PIPE_CONTROL_STALL_AT_SCOREBOARD,
		431	0);
4560	Serge	432	if (ret)
		433	return ret;
5060	serge	434	}
4560	Serge	435
5060	serge	436	return gen8_emit_pipe_control(ring, flags, scratch_addr);
4560	Serge	437	}
		438
5060	serge	439	static void ring_write_tail(struct intel_engine_cs *ring,
2332	Serge	440	u32 value)
		441	{
5060	serge	442	struct drm_i915_private *dev_priv = ring->dev->dev_private;
2332	Serge	443	I915_WRITE_TAIL(ring, value);
		444	}
		445
5060	serge	446	u64 intel_ring_get_active_head(struct intel_engine_cs *ring)
2332	Serge	447	{
5060	serge	448	struct drm_i915_private *dev_priv = ring->dev->dev_private;
		449	u64 acthd;
2332	Serge	450
5060	serge	451	if (INTEL_INFO(ring->dev)->gen >= 8)
		452	acthd = I915_READ64_2x32(RING_ACTHD(ring->mmio_base),
		453	RING_ACTHD_UDW(ring->mmio_base));
		454	else if (INTEL_INFO(ring->dev)->gen >= 4)
		455	acthd = I915_READ(RING_ACTHD(ring->mmio_base));
		456	else
		457	acthd = I915_READ(ACTHD);
		458
		459	return acthd;
2332	Serge	460	}
		461
5060	serge	462	static void ring_setup_phys_status_page(struct intel_engine_cs *ring)
4104	Serge	463	{
		464	struct drm_i915_private *dev_priv = ring->dev->dev_private;
		465	u32 addr;
		466
		467	addr = dev_priv->status_page_dmah->busaddr;
		468	if (INTEL_INFO(ring->dev)->gen >= 4)
		469	addr \|= (dev_priv->status_page_dmah->busaddr >> 28) & 0xf0;
		470	I915_WRITE(HWS_PGA, addr);
		471	}
		472
5060	serge	473	static bool stop_ring(struct intel_engine_cs *ring)
2332	Serge	474	{
5060	serge	475	struct drm_i915_private *dev_priv = to_i915(ring->dev);
2332	Serge	476
5060	serge	477	if (!IS_GEN2(ring->dev)) {
		478	I915_WRITE_MODE(ring, _MASKED_BIT_ENABLE(STOP_RING));
		479	if (wait_for_atomic((I915_READ_MODE(ring) & MODE_IDLE) != 0, 1000)) {
		480	DRM_ERROR("%s :timed out trying to stop ring\n", ring->name);
		481	return false;
		482	}
		483	}
3031	serge	484
2332	Serge	485	I915_WRITE_CTL(ring, 0);
		486	I915_WRITE_HEAD(ring, 0);
		487	ring->write_tail(ring, 0);
		488
5060	serge	489	if (!IS_GEN2(ring->dev)) {
		490	(void)I915_READ_CTL(ring);
		491	I915_WRITE_MODE(ring, _MASKED_BIT_DISABLE(STOP_RING));
		492	}
2332	Serge	493
5060	serge	494	return (I915_READ_HEAD(ring) & HEAD_ADDR) == 0;
		495	}
		496
		497	static int init_ring_common(struct intel_engine_cs *ring)
		498	{
		499	struct drm_device *dev = ring->dev;
		500	struct drm_i915_private *dev_priv = dev->dev_private;
		501	struct intel_ringbuffer *ringbuf = ring->buffer;
		502	struct drm_i915_gem_object *obj = ringbuf->obj;
		503	int ret = 0;
		504
		505	gen6_gt_force_wake_get(dev_priv, FORCEWAKE_ALL);
		506
		507	if (!stop_ring(ring)) {
		508	/* G45 ring initialization often fails to reset head to zero */
2332	Serge	509	DRM_DEBUG_KMS("%s head not reset to zero "
		510	"ctl %08x head %08x tail %08x start %08x\n",
		511	ring->name,
		512	I915_READ_CTL(ring),
		513	I915_READ_HEAD(ring),
		514	I915_READ_TAIL(ring),
		515	I915_READ_START(ring));
		516
5060	serge	517	if (!stop_ring(ring)) {
2332	Serge	518	DRM_ERROR("failed to set %s head to zero "
		519	"ctl %08x head %08x tail %08x start %08x\n",
		520	ring->name,
		521	I915_READ_CTL(ring),
		522	I915_READ_HEAD(ring),
		523	I915_READ_TAIL(ring),
		524	I915_READ_START(ring));
5060	serge	525	ret = -EIO;
		526	goto out;
2332	Serge	527	}
		528	}
		529
5060	serge	530	if (I915_NEED_GFX_HWS(dev))
		531	intel_ring_setup_status_page(ring);
		532	else
		533	ring_setup_phys_status_page(ring);
		534
		535	/* Enforce ordering by reading HEAD register back */
		536	I915_READ_HEAD(ring);
		537
3031	serge	538	/* Initialize the ring. This must happen _after_ we've cleared the ring
		539	* registers with the above sequence (the readback of the HEAD registers
		540	* also enforces ordering), otherwise the hw might lose the new ring
		541	* register values. */
4104	Serge	542	I915_WRITE_START(ring, i915_gem_obj_ggtt_offset(obj));
2332	Serge	543	I915_WRITE_CTL(ring,
5060	serge	544	((ringbuf->size - PAGE_SIZE) & RING_NR_PAGES)
3031	serge	545	\| RING_VALID);
2332	Serge	546
		547	/* If the head is still not zero, the ring is dead */
3031	serge	548	if (wait_for((I915_READ_CTL(ring) & RING_VALID) != 0 &&
4104	Serge	549	I915_READ_START(ring) == i915_gem_obj_ggtt_offset(obj) &&
3031	serge	550	(I915_READ_HEAD(ring) & HEAD_ADDR) == 0, 50)) {
2332	Serge	551	DRM_ERROR("%s initialization failed "
5060	serge	552	"ctl %08x (valid? %d) head %08x tail %08x start %08x [expected %08lx]\n",
2332	Serge	553	ring->name,
5060	serge	554	I915_READ_CTL(ring), I915_READ_CTL(ring) & RING_VALID,
		555	I915_READ_HEAD(ring), I915_READ_TAIL(ring),
		556	I915_READ_START(ring), (unsigned long)i915_gem_obj_ggtt_offset(obj));
3031	serge	557	ret = -EIO;
		558	goto out;
2332	Serge	559	}
		560
		561
5060	serge	562	ringbuf->head = I915_READ_HEAD(ring);
		563	ringbuf->tail = I915_READ_TAIL(ring) & TAIL_ADDR;
		564	ringbuf->space = ring_space(ringbuf);
		565	ringbuf->last_retired_head = -1;
		566
4104	Serge	567	memset(&ring->hangcheck, 0, sizeof(ring->hangcheck));
		568
3031	serge	569	out:
4560	Serge	570	gen6_gt_force_wake_put(dev_priv, FORCEWAKE_ALL);
2332	Serge	571
3031	serge	572	return ret;
2332	Serge	573	}
		574
		575	static int
5060	serge	576	init_pipe_control(struct intel_engine_cs *ring)
2332	Serge	577	{
		578	int ret;
		579
4104	Serge	580	if (ring->scratch.obj)
2332	Serge	581	return 0;
		582
4104	Serge	583	ring->scratch.obj = i915_gem_alloc_object(ring->dev, 4096);
		584	if (ring->scratch.obj == NULL) {
2332	Serge	585	DRM_ERROR("Failed to allocate seqno page\n");
		586	ret = -ENOMEM;
		587	goto err;
		588	}
		589
5060	serge	590	ret = i915_gem_object_set_cache_level(ring->scratch.obj, I915_CACHE_LLC);
		591	if (ret)
		592	goto err_unref;
2332	Serge	593
5060	serge	594	ret = i915_gem_obj_ggtt_pin(ring->scratch.obj, 4096, 0);
2332	Serge	595	if (ret)
		596	goto err_unref;
		597
4104	Serge	598	ring->scratch.gtt_offset = i915_gem_obj_ggtt_offset(ring->scratch.obj);
4539	Serge	599	ring->scratch.cpu_page = (void*)MapIoMem((addr_t)sg_page(ring->scratch.obj->pages->sgl),4096, PG_SW\|0x100);
4104	Serge	600	if (ring->scratch.cpu_page == NULL) {
		601	ret = -ENOMEM;
2332	Serge	602	goto err_unpin;
4104	Serge	603	}
2332	Serge	604
3480	Serge	605	DRM_DEBUG_DRIVER("%s pipe control offset: 0x%08x\n",
4104	Serge	606	ring->name, ring->scratch.gtt_offset);
2332	Serge	607	return 0;
		608
		609	err_unpin:
5060	serge	610	i915_gem_object_ggtt_unpin(ring->scratch.obj);
2332	Serge	611	err_unref:
4104	Serge	612	drm_gem_object_unreference(&ring->scratch.obj->base);
2332	Serge	613	err:
		614	return ret;
		615	}
		616
5060	serge	617	static int init_render_ring(struct intel_engine_cs *ring)
2332	Serge	618	{
		619	struct drm_device *dev = ring->dev;
		620	struct drm_i915_private *dev_priv = dev->dev_private;
		621	int ret = init_ring_common(ring);
5060	serge	622	if (ret)
		623	return ret;
2332	Serge	624
5060	serge	625	/* WaTimedSingleVertexDispatch:cl,bw,ctg,elk,ilk,snb */
		626	if (INTEL_INFO(dev)->gen >= 4 && INTEL_INFO(dev)->gen < 7)
3031	serge	627	I915_WRITE(MI_MODE, _MASKED_BIT_ENABLE(VS_TIMER_DISPATCH));
3243	Serge	628
		629	/* We need to disable the AsyncFlip performance optimisations in order
		630	* to use MI_WAIT_FOR_EVENT within the CS. It should already be
		631	* programmed to '1' on all products.
4104	Serge	632	*
5060	serge	633	* WaDisableAsyncFlipPerfMode:snb,ivb,hsw,vlv,bdw,chv
3243	Serge	634	*/
		635	if (INTEL_INFO(dev)->gen >= 6)
		636	I915_WRITE(MI_MODE, _MASKED_BIT_ENABLE(ASYNC_FLIP_PERF_DISABLE));
		637
		638	/* Required for the hardware to program scanline values for waiting */
5060	serge	639	/* WaEnableFlushTlbInvalidationMode:snb */
3243	Serge	640	if (INTEL_INFO(dev)->gen == 6)
		641	I915_WRITE(GFX_MODE,
5060	serge	642	_MASKED_BIT_ENABLE(GFX_TLB_INVALIDATE_EXPLICIT));
3243	Serge	643
5060	serge	644	/* WaBCSVCSTlbInvalidationMode:ivb,vlv,hsw */
2332	Serge	645	if (IS_GEN7(dev))
		646	I915_WRITE(GFX_MODE_GEN7,
5060	serge	647	_MASKED_BIT_ENABLE(GFX_TLB_INVALIDATE_EXPLICIT) \|
3031	serge	648	_MASKED_BIT_ENABLE(GFX_REPLAY_MODE));
2332	Serge	649
2342	Serge	650	if (INTEL_INFO(dev)->gen >= 5) {
2339	Serge	651	ret = init_pipe_control(ring);
2332	Serge	652	if (ret)
		653	return ret;
		654	}
		655
3031	serge	656	if (IS_GEN6(dev)) {
		657	/* From the Sandybridge PRM, volume 1 part 3, page 24:
		658	* "If this bit is set, STCunit will have LRA as replacement
		659	* policy. [...] This bit must be reset. LRA replacement
		660	* policy is not supported."
		661	*/
		662	I915_WRITE(CACHE_MODE_0,
		663	_MASKED_BIT_DISABLE(CM0_STC_EVICT_DISABLE_LRA_SNB));
2342	Serge	664	}
		665
3031	serge	666	if (INTEL_INFO(dev)->gen >= 6)
		667	I915_WRITE(INSTPM, _MASKED_BIT_ENABLE(INSTPM_FORCE_ORDERING));
		668
4560	Serge	669	if (HAS_L3_DPF(dev))
		670	I915_WRITE_IMR(ring, ~GT_PARITY_ERROR(dev));
3031	serge	671
2332	Serge	672	return ret;
		673	}
		674
5060	serge	675	static void render_ring_cleanup(struct intel_engine_cs *ring)
2332	Serge	676	{
3480	Serge	677	struct drm_device *dev = ring->dev;
		678
4104	Serge	679	if (ring->scratch.obj == NULL)
2332	Serge	680	return;
		681
4104	Serge	682	if (INTEL_INFO(dev)->gen >= 5) {
5060	serge	683	// kunmap(sg_page(ring->scratch.obj->pages->sgl));
		684	i915_gem_object_ggtt_unpin(ring->scratch.obj);
4104	Serge	685	}
		686
		687	drm_gem_object_unreference(&ring->scratch.obj->base);
		688	ring->scratch.obj = NULL;
2332	Serge	689	}
		690
5060	serge	691	static int gen8_rcs_signal(struct intel_engine_cs *signaller,
		692	unsigned int num_dwords)
2332	Serge	693	{
5060	serge	694	#define MBOX_UPDATE_DWORDS 8
		695	struct drm_device *dev = signaller->dev;
		696	struct drm_i915_private *dev_priv = dev->dev_private;
		697	struct intel_engine_cs *waiter;
		698	int i, ret, num_rings;
		699
		700	num_rings = hweight32(INTEL_INFO(dev)->ring_mask);
		701	num_dwords += (num_rings-1) * MBOX_UPDATE_DWORDS;
		702	#undef MBOX_UPDATE_DWORDS
		703
		704	ret = intel_ring_begin(signaller, num_dwords);
		705	if (ret)
		706	return ret;
		707
		708	for_each_ring(waiter, dev_priv, i) {
		709	u64 gtt_offset = signaller->semaphore.signal_ggtt[i];
		710	if (gtt_offset == MI_SEMAPHORE_SYNC_INVALID)
		711	continue;
		712
		713	intel_ring_emit(signaller, GFX_OP_PIPE_CONTROL(6));
		714	intel_ring_emit(signaller, PIPE_CONTROL_GLOBAL_GTT_IVB \|
		715	PIPE_CONTROL_QW_WRITE \|
		716	PIPE_CONTROL_FLUSH_ENABLE);
		717	intel_ring_emit(signaller, lower_32_bits(gtt_offset));
		718	intel_ring_emit(signaller, upper_32_bits(gtt_offset));
		719	intel_ring_emit(signaller, signaller->outstanding_lazy_seqno);
		720	intel_ring_emit(signaller, 0);
		721	intel_ring_emit(signaller, MI_SEMAPHORE_SIGNAL \|
		722	MI_SEMAPHORE_TARGET(waiter->id));
		723	intel_ring_emit(signaller, 0);
		724	}
		725
		726	return 0;
2332	Serge	727	}
		728
5060	serge	729	static int gen8_xcs_signal(struct intel_engine_cs *signaller,
		730	unsigned int num_dwords)
		731	{
		732	#define MBOX_UPDATE_DWORDS 6
		733	struct drm_device *dev = signaller->dev;
		734	struct drm_i915_private *dev_priv = dev->dev_private;
		735	struct intel_engine_cs *waiter;
		736	int i, ret, num_rings;
		737
		738	num_rings = hweight32(INTEL_INFO(dev)->ring_mask);
		739	num_dwords += (num_rings-1) * MBOX_UPDATE_DWORDS;
		740	#undef MBOX_UPDATE_DWORDS
		741
		742	ret = intel_ring_begin(signaller, num_dwords);
		743	if (ret)
		744	return ret;
		745
		746	for_each_ring(waiter, dev_priv, i) {
		747	u64 gtt_offset = signaller->semaphore.signal_ggtt[i];
		748	if (gtt_offset == MI_SEMAPHORE_SYNC_INVALID)
		749	continue;
		750
		751	intel_ring_emit(signaller, (MI_FLUSH_DW + 1) \|
		752	MI_FLUSH_DW_OP_STOREDW);
		753	intel_ring_emit(signaller, lower_32_bits(gtt_offset) \|
		754	MI_FLUSH_DW_USE_GTT);
		755	intel_ring_emit(signaller, upper_32_bits(gtt_offset));
		756	intel_ring_emit(signaller, signaller->outstanding_lazy_seqno);
		757	intel_ring_emit(signaller, MI_SEMAPHORE_SIGNAL \|
		758	MI_SEMAPHORE_TARGET(waiter->id));
		759	intel_ring_emit(signaller, 0);
		760	}
		761
		762	return 0;
		763	}
		764
		765	static int gen6_signal(struct intel_engine_cs *signaller,
		766	unsigned int num_dwords)
		767	{
		768	struct drm_device *dev = signaller->dev;
		769	struct drm_i915_private *dev_priv = dev->dev_private;
		770	struct intel_engine_cs *useless;
		771	int i, ret, num_rings;
		772
		773	#define MBOX_UPDATE_DWORDS 3
		774	num_rings = hweight32(INTEL_INFO(dev)->ring_mask);
		775	num_dwords += round_up((num_rings-1) * MBOX_UPDATE_DWORDS, 2);
		776	#undef MBOX_UPDATE_DWORDS
		777
		778	ret = intel_ring_begin(signaller, num_dwords);
		779	if (ret)
		780	return ret;
		781
		782	for_each_ring(useless, dev_priv, i) {
		783	u32 mbox_reg = signaller->semaphore.mbox.signal[i];
		784	if (mbox_reg != GEN6_NOSYNC) {
		785	intel_ring_emit(signaller, MI_LOAD_REGISTER_IMM(1));
		786	intel_ring_emit(signaller, mbox_reg);
		787	intel_ring_emit(signaller, signaller->outstanding_lazy_seqno);
		788	}
		789	}
		790
		791	/* If num_dwords was rounded, make sure the tail pointer is correct */
		792	if (num_rings % 2 == 0)
		793	intel_ring_emit(signaller, MI_NOOP);
		794
		795	return 0;
		796	}
		797
2342	Serge	798	/**
		799	* gen6_add_request - Update the semaphore mailbox registers
		800	*
		801	* @ring - ring that is adding a request
		802	* @seqno - return seqno stuck into the ring
		803	*
		804	* Update the mailbox registers in the other rings with the current seqno.
		805	* This acts like a signal in the canonical semaphore.
		806	*/
2332	Serge	807	static int
5060	serge	808	gen6_add_request(struct intel_engine_cs *ring)
2332	Serge	809	{
5060	serge	810	int ret;
2332	Serge	811
5060	serge	812	if (ring->semaphore.signal)
		813	ret = ring->semaphore.signal(ring, 4);
		814	else
		815	ret = intel_ring_begin(ring, 4);
4560	Serge	816
2332	Serge	817	if (ret)
		818	return ret;
		819
		820	intel_ring_emit(ring, MI_STORE_DWORD_INDEX);
		821	intel_ring_emit(ring, I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
4560	Serge	822	intel_ring_emit(ring, ring->outstanding_lazy_seqno);
2332	Serge	823	intel_ring_emit(ring, MI_USER_INTERRUPT);
4560	Serge	824	__intel_ring_advance(ring);
2332	Serge	825
		826	return 0;
		827	}
		828
3480	Serge	829	static inline bool i915_gem_has_seqno_wrapped(struct drm_device *dev,
		830	u32 seqno)
		831	{
		832	struct drm_i915_private *dev_priv = dev->dev_private;
		833	return dev_priv->last_seqno < seqno;
		834	}
		835
2342	Serge	836	/**
		837	* intel_ring_sync - sync the waiter to the signaller on seqno
		838	*
		839	* @waiter - ring that is waiting
		840	* @signaller - ring which has, or will signal
		841	* @seqno - seqno which the waiter will block on
		842	*/
5060	serge	843
2342	Serge	844	static int
5060	serge	845	gen8_ring_sync(struct intel_engine_cs *waiter,
		846	struct intel_engine_cs *signaller,
		847	u32 seqno)
		848	{
		849	struct drm_i915_private *dev_priv = waiter->dev->dev_private;
		850	int ret;
		851
		852	ret = intel_ring_begin(waiter, 4);
		853	if (ret)
		854	return ret;
		855
		856	intel_ring_emit(waiter, MI_SEMAPHORE_WAIT \|
		857	MI_SEMAPHORE_GLOBAL_GTT \|
		858	MI_SEMAPHORE_POLL \|
		859	MI_SEMAPHORE_SAD_GTE_SDD);
		860	intel_ring_emit(waiter, seqno);
		861	intel_ring_emit(waiter,
		862	lower_32_bits(GEN8_WAIT_OFFSET(waiter, signaller->id)));
		863	intel_ring_emit(waiter,
		864	upper_32_bits(GEN8_WAIT_OFFSET(waiter, signaller->id)));
		865	intel_ring_advance(waiter);
		866	return 0;
		867	}
		868
		869	static int
		870	gen6_ring_sync(struct intel_engine_cs *waiter,
		871	struct intel_engine_cs *signaller,
2332	Serge	872	u32 seqno)
		873	{
2342	Serge	874	u32 dw1 = MI_SEMAPHORE_MBOX \|
		875	MI_SEMAPHORE_COMPARE \|
		876	MI_SEMAPHORE_REGISTER;
5060	serge	877	u32 wait_mbox = signaller->semaphore.mbox.wait[waiter->id];
		878	int ret;
2332	Serge	879
3031	serge	880	/* Throughout all of the GEM code, seqno passed implies our current
		881	* seqno is >= the last seqno executed. However for hardware the
		882	* comparison is strictly greater than.
		883	*/
		884	seqno -= 1;
		885
5060	serge	886	WARN_ON(wait_mbox == MI_SEMAPHORE_SYNC_INVALID);
3031	serge	887
2342	Serge	888	ret = intel_ring_begin(waiter, 4);
2332	Serge	889	if (ret)
		890	return ret;
		891
3480	Serge	892	/* If seqno wrap happened, omit the wait with no-ops */
		893	if (likely(!i915_gem_has_seqno_wrapped(waiter->dev, seqno))) {
5060	serge	894	intel_ring_emit(waiter, dw1 \| wait_mbox);
2342	Serge	895	intel_ring_emit(waiter, seqno);
		896	intel_ring_emit(waiter, 0);
		897	intel_ring_emit(waiter, MI_NOOP);
3480	Serge	898	} else {
		899	intel_ring_emit(waiter, MI_NOOP);
		900	intel_ring_emit(waiter, MI_NOOP);
		901	intel_ring_emit(waiter, MI_NOOP);
		902	intel_ring_emit(waiter, MI_NOOP);
		903	}
2342	Serge	904	intel_ring_advance(waiter);
2332	Serge	905
		906	return 0;
		907	}
		908
		909	#define PIPE_CONTROL_FLUSH(ring__, addr__) \
		910	do { \
2342	Serge	911	intel_ring_emit(ring__, GFX_OP_PIPE_CONTROL(4) \| PIPE_CONTROL_QW_WRITE \| \
		912	PIPE_CONTROL_DEPTH_STALL); \
2332	Serge	913	intel_ring_emit(ring__, (addr__) \| PIPE_CONTROL_GLOBAL_GTT); \
		914	intel_ring_emit(ring__, 0); \
		915	intel_ring_emit(ring__, 0); \
		916	} while (0)
		917
		918	static int
5060	serge	919	pc_render_add_request(struct intel_engine_cs *ring)
2332	Serge	920	{
5060	serge	921	u32 scratch_addr = ring->scratch.gtt_offset + 2 * CACHELINE_BYTES;
2332	Serge	922	int ret;
		923
		924	/* For Ironlake, MI_USER_INTERRUPT was deprecated and apparently
		925	* incoherent with writes to memory, i.e. completely fubar,
		926	* so we need to use PIPE_NOTIFY instead.
		927	*
		928	* However, we also need to workaround the qword write
		929	* incoherence by flushing the 6 PIPE_NOTIFY buffers out to
		930	* memory before requesting an interrupt.
		931	*/
		932	ret = intel_ring_begin(ring, 32);
		933	if (ret)
		934	return ret;
		935
2342	Serge	936	intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(4) \| PIPE_CONTROL_QW_WRITE \|
		937	PIPE_CONTROL_WRITE_FLUSH \|
		938	PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE);
4104	Serge	939	intel_ring_emit(ring, ring->scratch.gtt_offset \| PIPE_CONTROL_GLOBAL_GTT);
4560	Serge	940	intel_ring_emit(ring, ring->outstanding_lazy_seqno);
2332	Serge	941	intel_ring_emit(ring, 0);
		942	PIPE_CONTROL_FLUSH(ring, scratch_addr);
5060	serge	943	scratch_addr += 2 * CACHELINE_BYTES; /* write to separate cachelines */
2332	Serge	944	PIPE_CONTROL_FLUSH(ring, scratch_addr);
5060	serge	945	scratch_addr += 2 * CACHELINE_BYTES;
2332	Serge	946	PIPE_CONTROL_FLUSH(ring, scratch_addr);
5060	serge	947	scratch_addr += 2 * CACHELINE_BYTES;
2332	Serge	948	PIPE_CONTROL_FLUSH(ring, scratch_addr);
5060	serge	949	scratch_addr += 2 * CACHELINE_BYTES;
2332	Serge	950	PIPE_CONTROL_FLUSH(ring, scratch_addr);
5060	serge	951	scratch_addr += 2 * CACHELINE_BYTES;
2332	Serge	952	PIPE_CONTROL_FLUSH(ring, scratch_addr);
3031	serge	953
2342	Serge	954	intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(4) \| PIPE_CONTROL_QW_WRITE \|
		955	PIPE_CONTROL_WRITE_FLUSH \|
		956	PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE \|
2332	Serge	957	PIPE_CONTROL_NOTIFY);
4104	Serge	958	intel_ring_emit(ring, ring->scratch.gtt_offset \| PIPE_CONTROL_GLOBAL_GTT);
4560	Serge	959	intel_ring_emit(ring, ring->outstanding_lazy_seqno);
2332	Serge	960	intel_ring_emit(ring, 0);
4560	Serge	961	__intel_ring_advance(ring);
2332	Serge	962
		963	return 0;
		964	}
		965
		966	static u32
5060	serge	967	gen6_ring_get_seqno(struct intel_engine_cs *ring, bool lazy_coherency)
2342	Serge	968	{
		969	/* Workaround to force correct ordering between irq and seqno writes on
		970	* ivb (and maybe also on snb) by reading from a CS register (like
		971	* ACTHD) before reading the status page. */
5060	serge	972	if (!lazy_coherency) {
		973	struct drm_i915_private *dev_priv = ring->dev->dev_private;
		974	POSTING_READ(RING_ACTHD(ring->mmio_base));
		975	}
		976
2342	Serge	977	return intel_read_status_page(ring, I915_GEM_HWS_INDEX);
		978	}
		979
		980	static u32
5060	serge	981	ring_get_seqno(struct intel_engine_cs *ring, bool lazy_coherency)
2332	Serge	982	{
		983	return intel_read_status_page(ring, I915_GEM_HWS_INDEX);
		984	}
		985
3480	Serge	986	static void
5060	serge	987	ring_set_seqno(struct intel_engine_cs *ring, u32 seqno)
3480	Serge	988	{
		989	intel_write_status_page(ring, I915_GEM_HWS_INDEX, seqno);
		990	}
		991
2332	Serge	992	static u32
5060	serge	993	pc_render_get_seqno(struct intel_engine_cs *ring, bool lazy_coherency)
2332	Serge	994	{
4104	Serge	995	return ring->scratch.cpu_page[0];
2332	Serge	996	}
		997
3480	Serge	998	static void
5060	serge	999	pc_render_set_seqno(struct intel_engine_cs *ring, u32 seqno)
3480	Serge	1000	{
4104	Serge	1001	ring->scratch.cpu_page[0] = seqno;
3480	Serge	1002	}
		1003
3031	serge	1004	static bool
5060	serge	1005	gen5_ring_get_irq(struct intel_engine_cs *ring)
2332	Serge	1006	{
3031	serge	1007	struct drm_device *dev = ring->dev;
5060	serge	1008	struct drm_i915_private *dev_priv = dev->dev_private;
3031	serge	1009	unsigned long flags;
		1010
		1011	if (!dev->irq_enabled)
		1012	return false;
		1013
		1014	spin_lock_irqsave(&dev_priv->irq_lock, flags);
4104	Serge	1015	if (ring->irq_refcount++ == 0)
5060	serge	1016	gen5_enable_gt_irq(dev_priv, ring->irq_enable_mask);
3031	serge	1017	spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
		1018
		1019	return true;
2332	Serge	1020	}
		1021
		1022	static void
5060	serge	1023	gen5_ring_put_irq(struct intel_engine_cs *ring)
2332	Serge	1024	{
3031	serge	1025	struct drm_device *dev = ring->dev;
5060	serge	1026	struct drm_i915_private *dev_priv = dev->dev_private;
3031	serge	1027	unsigned long flags;
		1028
		1029	spin_lock_irqsave(&dev_priv->irq_lock, flags);
4104	Serge	1030	if (--ring->irq_refcount == 0)
5060	serge	1031	gen5_disable_gt_irq(dev_priv, ring->irq_enable_mask);
3031	serge	1032	spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
2332	Serge	1033	}
		1034
3031	serge	1035	static bool
5060	serge	1036	i9xx_ring_get_irq(struct intel_engine_cs *ring)
2332	Serge	1037	{
3031	serge	1038	struct drm_device *dev = ring->dev;
5060	serge	1039	struct drm_i915_private *dev_priv = dev->dev_private;
3031	serge	1040	unsigned long flags;
		1041
		1042	if (!dev->irq_enabled)
		1043	return false;
		1044
		1045	spin_lock_irqsave(&dev_priv->irq_lock, flags);
		1046	if (ring->irq_refcount++ == 0) {
		1047	dev_priv->irq_mask &= ~ring->irq_enable_mask;
2332	Serge	1048	I915_WRITE(IMR, dev_priv->irq_mask);
		1049	POSTING_READ(IMR);
3031	serge	1050	}
		1051	spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
		1052
		1053	return true;
2332	Serge	1054	}
		1055
		1056	static void
5060	serge	1057	i9xx_ring_put_irq(struct intel_engine_cs *ring)
2332	Serge	1058	{
3031	serge	1059	struct drm_device *dev = ring->dev;
5060	serge	1060	struct drm_i915_private *dev_priv = dev->dev_private;
3031	serge	1061	unsigned long flags;
		1062
		1063	spin_lock_irqsave(&dev_priv->irq_lock, flags);
		1064	if (--ring->irq_refcount == 0) {
		1065	dev_priv->irq_mask \|= ring->irq_enable_mask;
2332	Serge	1066	I915_WRITE(IMR, dev_priv->irq_mask);
		1067	POSTING_READ(IMR);
3031	serge	1068	}
		1069	spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
2332	Serge	1070	}
		1071
		1072	static bool
5060	serge	1073	i8xx_ring_get_irq(struct intel_engine_cs *ring)
2332	Serge	1074	{
		1075	struct drm_device *dev = ring->dev;
5060	serge	1076	struct drm_i915_private *dev_priv = dev->dev_private;
3031	serge	1077	unsigned long flags;
2332	Serge	1078
		1079	if (!dev->irq_enabled)
		1080	return false;
		1081
3031	serge	1082	spin_lock_irqsave(&dev_priv->irq_lock, flags);
2332	Serge	1083	if (ring->irq_refcount++ == 0) {
3031	serge	1084	dev_priv->irq_mask &= ~ring->irq_enable_mask;
		1085	I915_WRITE16(IMR, dev_priv->irq_mask);
		1086	POSTING_READ16(IMR);
2332	Serge	1087	}
3031	serge	1088	spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
2332	Serge	1089
		1090	return true;
		1091	}
		1092
		1093	static void
5060	serge	1094	i8xx_ring_put_irq(struct intel_engine_cs *ring)
2332	Serge	1095	{
		1096	struct drm_device *dev = ring->dev;
5060	serge	1097	struct drm_i915_private *dev_priv = dev->dev_private;
3031	serge	1098	unsigned long flags;
2332	Serge	1099
3031	serge	1100	spin_lock_irqsave(&dev_priv->irq_lock, flags);
2332	Serge	1101	if (--ring->irq_refcount == 0) {
3031	serge	1102	dev_priv->irq_mask \|= ring->irq_enable_mask;
		1103	I915_WRITE16(IMR, dev_priv->irq_mask);
		1104	POSTING_READ16(IMR);
2332	Serge	1105	}
3031	serge	1106	spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
2332	Serge	1107	}
		1108
5060	serge	1109	void intel_ring_setup_status_page(struct intel_engine_cs *ring)
2332	Serge	1110	{
		1111	struct drm_device *dev = ring->dev;
5060	serge	1112	struct drm_i915_private *dev_priv = ring->dev->dev_private;
2332	Serge	1113	u32 mmio = 0;
		1114
		1115	/* The ring status page addresses are no longer next to the rest of
		1116	* the ring registers as of gen7.
		1117	*/
		1118	if (IS_GEN7(dev)) {
		1119	switch (ring->id) {
3031	serge	1120	case RCS:
2332	Serge	1121	mmio = RENDER_HWS_PGA_GEN7;
		1122	break;
3031	serge	1123	case BCS:
2332	Serge	1124	mmio = BLT_HWS_PGA_GEN7;
		1125	break;
5060	serge	1126	/*
		1127	* VCS2 actually doesn't exist on Gen7. Only shut up
		1128	* gcc switch check warning
		1129	*/
		1130	case VCS2:
3031	serge	1131	case VCS:
2332	Serge	1132	mmio = BSD_HWS_PGA_GEN7;
		1133	break;
4104	Serge	1134	case VECS:
		1135	mmio = VEBOX_HWS_PGA_GEN7;
		1136	break;
2332	Serge	1137	}
		1138	} else if (IS_GEN6(ring->dev)) {
		1139	mmio = RING_HWS_PGA_GEN6(ring->mmio_base);
		1140	} else {
4560	Serge	1141	/* XXX: gen8 returns to sanity */
2332	Serge	1142	mmio = RING_HWS_PGA(ring->mmio_base);
		1143	}
		1144
		1145	I915_WRITE(mmio, (u32)ring->status_page.gfx_addr);
		1146	POSTING_READ(mmio);
3746	Serge	1147
5060	serge	1148	/*
		1149	* Flush the TLB for this page
		1150	*
		1151	* FIXME: These two bits have disappeared on gen8, so a question
		1152	* arises: do we still need this and if so how should we go about
		1153	* invalidating the TLB?
		1154	*/
		1155	if (INTEL_INFO(dev)->gen >= 6 && INTEL_INFO(dev)->gen < 8) {
4104	Serge	1156	u32 reg = RING_INSTPM(ring->mmio_base);
5060	serge	1157
		1158	/* ring should be idle before issuing a sync flush*/
		1159	WARN_ON((I915_READ_MODE(ring) & MODE_IDLE) == 0);
		1160
4104	Serge	1161	I915_WRITE(reg,
		1162	_MASKED_BIT_ENABLE(INSTPM_TLB_INVALIDATE \|
		1163	INSTPM_SYNC_FLUSH));
		1164	if (wait_for((I915_READ(reg) & INSTPM_SYNC_FLUSH) == 0,
		1165	1000))
		1166	DRM_ERROR("%s: wait for SyncFlush to complete for TLB invalidation timed out\n",
		1167	ring->name);
		1168	}
2332	Serge	1169	}
		1170
		1171	static int
5060	serge	1172	bsd_ring_flush(struct intel_engine_cs *ring,
2332	Serge	1173	u32 invalidate_domains,
		1174	u32 flush_domains)
		1175	{
		1176	int ret;
		1177
		1178	ret = intel_ring_begin(ring, 2);
		1179	if (ret)
		1180	return ret;
		1181
		1182	intel_ring_emit(ring, MI_FLUSH);
		1183	intel_ring_emit(ring, MI_NOOP);
		1184	intel_ring_advance(ring);
		1185	return 0;
		1186	}
		1187
		1188	static int
5060	serge	1189	i9xx_add_request(struct intel_engine_cs *ring)
2332	Serge	1190	{
		1191	int ret;
		1192
		1193	ret = intel_ring_begin(ring, 4);
		1194	if (ret)
		1195	return ret;
		1196
		1197	intel_ring_emit(ring, MI_STORE_DWORD_INDEX);
		1198	intel_ring_emit(ring, I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
4560	Serge	1199	intel_ring_emit(ring, ring->outstanding_lazy_seqno);
2332	Serge	1200	intel_ring_emit(ring, MI_USER_INTERRUPT);
4560	Serge	1201	__intel_ring_advance(ring);
2332	Serge	1202
		1203	return 0;
		1204	}
		1205
		1206	static bool
5060	serge	1207	gen6_ring_get_irq(struct intel_engine_cs *ring)
2332	Serge	1208	{
		1209	struct drm_device *dev = ring->dev;
5060	serge	1210	struct drm_i915_private *dev_priv = dev->dev_private;
3031	serge	1211	unsigned long flags;
2332	Serge	1212
		1213	if (!dev->irq_enabled)
		1214	return false;
		1215
3031	serge	1216	spin_lock_irqsave(&dev_priv->irq_lock, flags);
2332	Serge	1217	if (ring->irq_refcount++ == 0) {
4560	Serge	1218	if (HAS_L3_DPF(dev) && ring->id == RCS)
4104	Serge	1219	I915_WRITE_IMR(ring,
		1220	~(ring->irq_enable_mask \|
4560	Serge	1221	GT_PARITY_ERROR(dev)));
3031	serge	1222	else
		1223	I915_WRITE_IMR(ring, ~ring->irq_enable_mask);
5060	serge	1224	gen5_enable_gt_irq(dev_priv, ring->irq_enable_mask);
2332	Serge	1225	}
3031	serge	1226	spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
2332	Serge	1227
2351	Serge	1228	return true;
2332	Serge	1229	}
		1230
		1231	static void
5060	serge	1232	gen6_ring_put_irq(struct intel_engine_cs *ring)
2332	Serge	1233	{
		1234	struct drm_device *dev = ring->dev;
5060	serge	1235	struct drm_i915_private *dev_priv = dev->dev_private;
3031	serge	1236	unsigned long flags;
2332	Serge	1237
3031	serge	1238	spin_lock_irqsave(&dev_priv->irq_lock, flags);
2332	Serge	1239	if (--ring->irq_refcount == 0) {
4560	Serge	1240	if (HAS_L3_DPF(dev) && ring->id == RCS)
		1241	I915_WRITE_IMR(ring, ~GT_PARITY_ERROR(dev));
3031	serge	1242	else
		1243	I915_WRITE_IMR(ring, ~0);
5060	serge	1244	gen5_disable_gt_irq(dev_priv, ring->irq_enable_mask);
2332	Serge	1245	}
3031	serge	1246	spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
2332	Serge	1247	}
		1248
4104	Serge	1249	static bool
5060	serge	1250	hsw_vebox_get_irq(struct intel_engine_cs *ring)
4104	Serge	1251	{
		1252	struct drm_device *dev = ring->dev;
		1253	struct drm_i915_private *dev_priv = dev->dev_private;
		1254	unsigned long flags;
		1255
		1256	if (!dev->irq_enabled)
		1257	return false;
		1258
		1259	spin_lock_irqsave(&dev_priv->irq_lock, flags);
		1260	if (ring->irq_refcount++ == 0) {
		1261	I915_WRITE_IMR(ring, ~ring->irq_enable_mask);
5060	serge	1262	gen6_enable_pm_irq(dev_priv, ring->irq_enable_mask);
4104	Serge	1263	}
		1264	spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
		1265
		1266	return true;
		1267	}
		1268
		1269	static void
5060	serge	1270	hsw_vebox_put_irq(struct intel_engine_cs *ring)
4104	Serge	1271	{
		1272	struct drm_device *dev = ring->dev;
		1273	struct drm_i915_private *dev_priv = dev->dev_private;
		1274	unsigned long flags;
		1275
		1276	if (!dev->irq_enabled)
		1277	return;
		1278
		1279	spin_lock_irqsave(&dev_priv->irq_lock, flags);
		1280	if (--ring->irq_refcount == 0) {
		1281	I915_WRITE_IMR(ring, ~0);
5060	serge	1282	gen6_disable_pm_irq(dev_priv, ring->irq_enable_mask);
4104	Serge	1283	}
		1284	spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
		1285	}
		1286
4560	Serge	1287	static bool
5060	serge	1288	gen8_ring_get_irq(struct intel_engine_cs *ring)
4560	Serge	1289	{
		1290	struct drm_device *dev = ring->dev;
		1291	struct drm_i915_private *dev_priv = dev->dev_private;
		1292	unsigned long flags;
		1293
		1294	if (!dev->irq_enabled)
		1295	return false;
		1296
		1297	spin_lock_irqsave(&dev_priv->irq_lock, flags);
		1298	if (ring->irq_refcount++ == 0) {
		1299	if (HAS_L3_DPF(dev) && ring->id == RCS) {
		1300	I915_WRITE_IMR(ring,
		1301	~(ring->irq_enable_mask \|
		1302	GT_RENDER_L3_PARITY_ERROR_INTERRUPT));
		1303	} else {
		1304	I915_WRITE_IMR(ring, ~ring->irq_enable_mask);
		1305	}
		1306	POSTING_READ(RING_IMR(ring->mmio_base));
		1307	}
		1308	spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
		1309
		1310	return true;
		1311	}
		1312
		1313	static void
5060	serge	1314	gen8_ring_put_irq(struct intel_engine_cs *ring)
4560	Serge	1315	{
		1316	struct drm_device *dev = ring->dev;
		1317	struct drm_i915_private *dev_priv = dev->dev_private;
		1318	unsigned long flags;
		1319
		1320	spin_lock_irqsave(&dev_priv->irq_lock, flags);
		1321	if (--ring->irq_refcount == 0) {
		1322	if (HAS_L3_DPF(dev) && ring->id == RCS) {
		1323	I915_WRITE_IMR(ring,
		1324	~GT_RENDER_L3_PARITY_ERROR_INTERRUPT);
		1325	} else {
		1326	I915_WRITE_IMR(ring, ~0);
		1327	}
		1328	POSTING_READ(RING_IMR(ring->mmio_base));
		1329	}
		1330	spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
		1331	}
		1332
2332	Serge	1333	static int
5060	serge	1334	i965_dispatch_execbuffer(struct intel_engine_cs *ring,
		1335	u64 offset, u32 length,
3243	Serge	1336	unsigned flags)
2332	Serge	1337	{
		1338	int ret;
		1339
		1340	ret = intel_ring_begin(ring, 2);
		1341	if (ret)
		1342	return ret;
		1343
		1344	intel_ring_emit(ring,
3031	serge	1345	MI_BATCH_BUFFER_START \|
		1346	MI_BATCH_GTT \|
3243	Serge	1347	(flags & I915_DISPATCH_SECURE ? 0 : MI_BATCH_NON_SECURE_I965));
2332	Serge	1348	intel_ring_emit(ring, offset);
		1349	intel_ring_advance(ring);
		1350
		1351	return 0;
		1352	}
		1353
3243	Serge	1354	/* Just userspace ABI convention to limit the wa batch bo to a resonable size */
		1355	#define I830_BATCH_LIMIT (256*1024)
2332	Serge	1356	static int
5060	serge	1357	i830_dispatch_execbuffer(struct intel_engine_cs *ring,
		1358	u64 offset, u32 len,
3243	Serge	1359	unsigned flags)
2332	Serge	1360	{
		1361	int ret;
		1362
3243	Serge	1363	if (flags & I915_DISPATCH_PINNED) {
2332	Serge	1364	ret = intel_ring_begin(ring, 4);
		1365	if (ret)
		1366	return ret;
		1367
		1368	intel_ring_emit(ring, MI_BATCH_BUFFER);
3243	Serge	1369	intel_ring_emit(ring, offset \| (flags & I915_DISPATCH_SECURE ? 0 : MI_BATCH_NON_SECURE));
2332	Serge	1370	intel_ring_emit(ring, offset + len - 8);
3243	Serge	1371	intel_ring_emit(ring, MI_NOOP);
		1372	intel_ring_advance(ring);
		1373	} else {
4104	Serge	1374	u32 cs_offset = ring->scratch.gtt_offset;
3243	Serge	1375
		1376	if (len > I830_BATCH_LIMIT)
		1377	return -ENOSPC;
		1378
		1379	ret = intel_ring_begin(ring, 9+3);
		1380	if (ret)
		1381	return ret;
		1382	/* Blit the batch (which has now all relocs applied) to the stable batch
		1383	* scratch bo area (so that the CS never stumbles over its tlb
		1384	* invalidation bug) ... */
		1385	intel_ring_emit(ring, XY_SRC_COPY_BLT_CMD \|
		1386	XY_SRC_COPY_BLT_WRITE_ALPHA \|
		1387	XY_SRC_COPY_BLT_WRITE_RGB);
		1388	intel_ring_emit(ring, BLT_DEPTH_32 \| BLT_ROP_GXCOPY \| 4096);
2332	Serge	1389	intel_ring_emit(ring, 0);
3243	Serge	1390	intel_ring_emit(ring, (DIV_ROUND_UP(len, 4096) << 16) \| 1024);
		1391	intel_ring_emit(ring, cs_offset);
		1392	intel_ring_emit(ring, 0);
		1393	intel_ring_emit(ring, 4096);
		1394	intel_ring_emit(ring, offset);
		1395	intel_ring_emit(ring, MI_FLUSH);
		1396
		1397	/* ... and execute it. */
		1398	intel_ring_emit(ring, MI_BATCH_BUFFER);
		1399	intel_ring_emit(ring, cs_offset \| (flags & I915_DISPATCH_SECURE ? 0 : MI_BATCH_NON_SECURE));
		1400	intel_ring_emit(ring, cs_offset + len - 8);
3031	serge	1401	intel_ring_advance(ring);
3243	Serge	1402	}
3031	serge	1403
		1404	return 0;
		1405	}
		1406
		1407	static int
5060	serge	1408	i915_dispatch_execbuffer(struct intel_engine_cs *ring,
		1409	u64 offset, u32 len,
3243	Serge	1410	unsigned flags)
3031	serge	1411	{
		1412	int ret;
		1413
2332	Serge	1414	ret = intel_ring_begin(ring, 2);
		1415	if (ret)
		1416	return ret;
		1417
3031	serge	1418	intel_ring_emit(ring, MI_BATCH_BUFFER_START \| MI_BATCH_GTT);
3243	Serge	1419	intel_ring_emit(ring, offset \| (flags & I915_DISPATCH_SECURE ? 0 : MI_BATCH_NON_SECURE));
2332	Serge	1420	intel_ring_advance(ring);
		1421
		1422	return 0;
		1423	}
		1424
5060	serge	1425	static void cleanup_status_page(struct intel_engine_cs *ring)
2332	Serge	1426	{
		1427	struct drm_i915_gem_object *obj;
		1428
		1429	obj = ring->status_page.obj;
		1430	if (obj == NULL)
		1431	return;
		1432
5060	serge	1433	// kunmap(sg_page(obj->pages->sgl));
		1434	i915_gem_object_ggtt_unpin(obj);
2344	Serge	1435	drm_gem_object_unreference(&obj->base);
2332	Serge	1436	ring->status_page.obj = NULL;
		1437	}
		1438
5060	serge	1439	static int init_status_page(struct intel_engine_cs *ring)
2332	Serge	1440	{
		1441	struct drm_i915_gem_object *obj;
5060	serge	1442
		1443	if ((obj = ring->status_page.obj) == NULL) {
		1444	unsigned flags;
2332	Serge	1445	int ret;
		1446
5060	serge	1447	obj = i915_gem_alloc_object(ring->dev, 4096);
2332	Serge	1448	if (obj == NULL) {
		1449	DRM_ERROR("Failed to allocate status page\n");
5060	serge	1450	return -ENOMEM;
2332	Serge	1451	}
		1452
5060	serge	1453	ret = i915_gem_object_set_cache_level(obj, I915_CACHE_LLC);
		1454	if (ret)
		1455	goto err_unref;
2332	Serge	1456
5060	serge	1457	flags = 0;
		1458	if (!HAS_LLC(ring->dev))
		1459	/* On g33, we cannot place HWS above 256MiB, so
		1460	* restrict its pinning to the low mappable arena.
		1461	* Though this restriction is not documented for
		1462	* gen4, gen5, or byt, they also behave similarly
		1463	* and hang if the HWS is placed at the top of the
		1464	* GTT. To generalise, it appears that all !llc
		1465	* platforms have issues with us placing the HWS
		1466	* above the mappable region (even though we never
		1467	* actualy map it).
		1468	*/
		1469	flags \|= PIN_MAPPABLE;
		1470	ret = i915_gem_obj_ggtt_pin(obj, 4096, flags);
		1471	if (ret) {
		1472	err_unref:
		1473	drm_gem_object_unreference(&obj->base);
		1474	return ret;
		1475	}
		1476
		1477	ring->status_page.obj = obj;
2332	Serge	1478	}
		1479
4104	Serge	1480	ring->status_page.gfx_addr = i915_gem_obj_ggtt_offset(obj);
4539	Serge	1481	ring->status_page.page_addr = (void*)MapIoMem((addr_t)sg_page(obj->pages->sgl),4096,PG_SW\|0x100);
2332	Serge	1482	memset(ring->status_page.page_addr, 0, PAGE_SIZE);
		1483
		1484	DRM_DEBUG_DRIVER("%s hws offset: 0x%08x\n",
		1485	ring->name, ring->status_page.gfx_addr);
		1486
		1487	return 0;
		1488	}
		1489
5060	serge	1490	static int init_phys_status_page(struct intel_engine_cs *ring)
3243	Serge	1491	{
		1492	struct drm_i915_private *dev_priv = ring->dev->dev_private;
		1493
		1494	if (!dev_priv->status_page_dmah) {
		1495	dev_priv->status_page_dmah =
		1496	drm_pci_alloc(ring->dev, PAGE_SIZE, PAGE_SIZE);
		1497	if (!dev_priv->status_page_dmah)
		1498	return -ENOMEM;
		1499	}
		1500
		1501	ring->status_page.page_addr = dev_priv->status_page_dmah->vaddr;
		1502	memset(ring->status_page.page_addr, 0, PAGE_SIZE);
		1503
		1504	return 0;
		1505	}
		1506
5060	serge	1507	static void intel_destroy_ringbuffer_obj(struct intel_ringbuffer *ringbuf)
2332	Serge	1508	{
5060	serge	1509	if (!ringbuf->obj)
		1510	return;
		1511
		1512	iounmap(ringbuf->virtual_start);
		1513	i915_gem_object_ggtt_unpin(ringbuf->obj);
		1514	drm_gem_object_unreference(&ringbuf->obj->base);
		1515	ringbuf->obj = NULL;
		1516	}
		1517
		1518	static int intel_alloc_ringbuffer_obj(struct drm_device *dev,
		1519	struct intel_ringbuffer *ringbuf)
		1520	{
		1521	struct drm_i915_private *dev_priv = to_i915(dev);
2340	Serge	1522	struct drm_i915_gem_object *obj;
2332	Serge	1523	int ret;
2340	Serge	1524
5060	serge	1525	if (ringbuf->obj)
		1526	return 0;
2332	Serge	1527
3480	Serge	1528	obj = NULL;
4371	Serge	1529	if (!HAS_LLC(dev))
5060	serge	1530	obj = i915_gem_object_create_stolen(dev, ringbuf->size);
3480	Serge	1531	if (obj == NULL)
5060	serge	1532	obj = i915_gem_alloc_object(dev, ringbuf->size);
		1533	if (obj == NULL)
		1534	return -ENOMEM;
2332	Serge	1535
5060	serge	1536	/* mark ring buffers as read-only from GPU side by default */
		1537	obj->gt_ro = 1;
2332	Serge	1538
5060	serge	1539	ret = i915_gem_obj_ggtt_pin(obj, PAGE_SIZE, PIN_MAPPABLE);
2332	Serge	1540	if (ret)
		1541	goto err_unref;
		1542
3031	serge	1543	ret = i915_gem_object_set_to_gtt_domain(obj, true);
		1544	if (ret)
		1545	goto err_unpin;
2332	Serge	1546
5060	serge	1547	ringbuf->virtual_start =
4104	Serge	1548	ioremap_wc(dev_priv->gtt.mappable_base + i915_gem_obj_ggtt_offset(obj),
5060	serge	1549	ringbuf->size);
		1550	if (ringbuf->virtual_start == NULL) {
2332	Serge	1551	ret = -EINVAL;
		1552	goto err_unpin;
		1553	}
		1554
5060	serge	1555	ringbuf->obj = obj;
		1556	return 0;
		1557
		1558	err_unpin:
		1559	i915_gem_object_ggtt_unpin(obj);
		1560	err_unref:
		1561	drm_gem_object_unreference(&obj->base);
		1562	return ret;
		1563	}
		1564
		1565	static int intel_init_ring_buffer(struct drm_device *dev,
		1566	struct intel_engine_cs *ring)
		1567	{
		1568	struct intel_ringbuffer *ringbuf = ring->buffer;
		1569	int ret;
		1570
		1571	if (ringbuf == NULL) {
		1572	ringbuf = kzalloc(sizeof(*ringbuf), GFP_KERNEL);
		1573	if (!ringbuf)
		1574	return -ENOMEM;
		1575	ring->buffer = ringbuf;
		1576	}
		1577
		1578	ring->dev = dev;
		1579	INIT_LIST_HEAD(&ring->active_list);
		1580	INIT_LIST_HEAD(&ring->request_list);
		1581	ringbuf->size = 32 * PAGE_SIZE;
		1582	memset(ring->semaphore.sync_seqno, 0, sizeof(ring->semaphore.sync_seqno));
		1583
		1584	init_waitqueue_head(&ring->irq_queue);
		1585
		1586	if (I915_NEED_GFX_HWS(dev)) {
		1587	ret = init_status_page(ring);
		1588	if (ret)
		1589	goto error;
		1590	} else {
		1591	BUG_ON(ring->id != RCS);
		1592	ret = init_phys_status_page(ring);
2332	Serge	1593	if (ret)
5060	serge	1594	goto error;
		1595	}
2332	Serge	1596
5060	serge	1597	ret = intel_alloc_ringbuffer_obj(dev, ringbuf);
		1598	if (ret) {
		1599	DRM_ERROR("Failed to allocate ringbuffer %s: %d\n", ring->name, ret);
		1600	goto error;
		1601	}
		1602
2332	Serge	1603	/* Workaround an erratum on the i830 which causes a hang if
		1604	* the TAIL pointer points to within the last 2 cachelines
		1605	* of the buffer.
		1606	*/
5060	serge	1607	ringbuf->effective_size = ringbuf->size;
		1608	if (IS_I830(dev) \|\| IS_845G(dev))
		1609	ringbuf->effective_size -= 2 * CACHELINE_BYTES;
2340	Serge	1610
5060	serge	1611	ret = i915_cmd_parser_init_ring(ring);
		1612	if (ret)
		1613	goto error;
		1614
		1615	ret = ring->init(ring);
		1616	if (ret)
		1617	goto error;
		1618
2332	Serge	1619	return 0;
		1620
5060	serge	1621	error:
		1622	kfree(ringbuf);
		1623	ring->buffer = NULL;
2332	Serge	1624	return ret;
		1625	}
		1626
5060	serge	1627	void intel_cleanup_ring_buffer(struct intel_engine_cs *ring)
2332	Serge	1628	{
5060	serge	1629	struct drm_i915_private *dev_priv = to_i915(ring->dev);
		1630	struct intel_ringbuffer *ringbuf = ring->buffer;
2332	Serge	1631
5060	serge	1632	if (!intel_ring_initialized(ring))
2332	Serge	1633	return;
		1634
5060	serge	1635	intel_stop_ring_buffer(ring);
		1636	WARN_ON(!IS_GEN2(ring->dev) && (I915_READ_MODE(ring) & MODE_IDLE) == 0);
2332	Serge	1637
5060	serge	1638	intel_destroy_ringbuffer_obj(ringbuf);
4560	Serge	1639	ring->preallocated_lazy_request = NULL;
		1640	ring->outstanding_lazy_seqno = 0;
2332	Serge	1641
		1642	if (ring->cleanup)
		1643	ring->cleanup(ring);
		1644
5060	serge	1645	// cleanup_status_page(ring);
2332	Serge	1646
5060	serge	1647	i915_cmd_parser_fini_ring(ring);
2332	Serge	1648
5060	serge	1649	kfree(ringbuf);
		1650	ring->buffer = NULL;
3031	serge	1651	}
		1652
5060	serge	1653	static int intel_ring_wait_request(struct intel_engine_cs *ring, int n)
3031	serge	1654	{
5060	serge	1655	struct intel_ringbuffer *ringbuf = ring->buffer;
3031	serge	1656	struct drm_i915_gem_request *request;
		1657	u32 seqno = 0;
		1658	int ret;
		1659
5060	serge	1660	if (ringbuf->last_retired_head != -1) {
		1661	ringbuf->head = ringbuf->last_retired_head;
		1662	ringbuf->last_retired_head = -1;
3031	serge	1663
5060	serge	1664	ringbuf->space = ring_space(ringbuf);
		1665	if (ringbuf->space >= n)
2332	Serge	1666	return 0;
		1667	}
		1668
3031	serge	1669	list_for_each_entry(request, &ring->request_list, list) {
5060	serge	1670	if (__ring_space(request->tail, ringbuf->tail, ringbuf->size) >= n) {
3031	serge	1671	seqno = request->seqno;
		1672	break;
		1673	}
		1674	}
		1675
		1676	if (seqno == 0)
		1677	return -ENOSPC;
		1678
5060	serge	1679	ret = i915_wait_seqno(ring, seqno);
3031	serge	1680	if (ret)
		1681	return ret;
		1682
5060	serge	1683	i915_gem_retire_requests_ring(ring);
		1684	ringbuf->head = ringbuf->last_retired_head;
		1685	ringbuf->last_retired_head = -1;
3031	serge	1686
5060	serge	1687	ringbuf->space = ring_space(ringbuf);
3031	serge	1688	return 0;
		1689	}
		1690
5060	serge	1691	static int ring_wait_for_space(struct intel_engine_cs *ring, int n)
3031	serge	1692	{
		1693	struct drm_device *dev = ring->dev;
		1694	struct drm_i915_private *dev_priv = dev->dev_private;
5060	serge	1695	struct intel_ringbuffer *ringbuf = ring->buffer;
3031	serge	1696	unsigned long end;
		1697	int ret;
		1698
		1699	ret = intel_ring_wait_request(ring, n);
		1700	if (ret != -ENOSPC)
		1701	return ret;
		1702
4560	Serge	1703	/* force the tail write in case we have been skipping them */
		1704	__intel_ring_advance(ring);
		1705
3031	serge	1706	/* With GEM the hangcheck timer should kick us out of the loop,
		1707	* leaving it early runs the risk of corrupting GEM state (due
		1708	* to running on almost untested codepaths). But on resume
		1709	* timers don't work yet, so prevent a complete hang in that
		1710	* case by choosing an insanely large timeout. */
5060	serge	1711	end = jiffies + 60 * HZ;
3031	serge	1712
5060	serge	1713	trace_i915_ring_wait_begin(ring);
2332	Serge	1714	do {
5060	serge	1715	ringbuf->head = I915_READ_HEAD(ring);
		1716	ringbuf->space = ring_space(ringbuf);
		1717	if (ringbuf->space >= n) {
		1718	ret = 0;
		1719	break;
2332	Serge	1720	}
		1721
5060	serge	1722
2332	Serge	1723	msleep(1);
3031	serge	1724
3480	Serge	1725	ret = i915_gem_check_wedge(&dev_priv->gpu_error,
		1726	dev_priv->mm.interruptible);
3031	serge	1727	if (ret)
5060	serge	1728	break;
		1729
		1730	if (time_after(jiffies, end)) {
		1731	ret = -EBUSY;
		1732	break;
		1733	}
		1734	} while (1);
		1735	trace_i915_ring_wait_end(ring);
3031	serge	1736	return ret;
2332	Serge	1737	}
		1738
5060	serge	1739	static int intel_wrap_ring_buffer(struct intel_engine_cs *ring)
3243	Serge	1740	{
		1741	uint32_t __iomem *virt;
5060	serge	1742	struct intel_ringbuffer *ringbuf = ring->buffer;
		1743	int rem = ringbuf->size - ringbuf->tail;
3243	Serge	1744
5060	serge	1745	if (ringbuf->space < rem) {
3243	Serge	1746	int ret = ring_wait_for_space(ring, rem);
		1747	if (ret)
		1748	return ret;
		1749	}
		1750
5060	serge	1751	virt = ringbuf->virtual_start + ringbuf->tail;
3243	Serge	1752	rem /= 4;
		1753	while (rem--)
		1754	iowrite32(MI_NOOP, virt++);
		1755
5060	serge	1756	ringbuf->tail = 0;
		1757	ringbuf->space = ring_space(ringbuf);
3243	Serge	1758
		1759	return 0;
		1760	}
		1761
5060	serge	1762	int intel_ring_idle(struct intel_engine_cs *ring)
3243	Serge	1763	{
		1764	u32 seqno;
		1765	int ret;
		1766
		1767	/* We need to add any requests required to flush the objects and ring */
4560	Serge	1768	if (ring->outstanding_lazy_seqno) {
4104	Serge	1769	ret = i915_add_request(ring, NULL);
3243	Serge	1770	if (ret)
		1771	return ret;
		1772	}
		1773
		1774	/* Wait upon the last request to be completed */
		1775	if (list_empty(&ring->request_list))
		1776	return 0;
		1777
		1778	seqno = list_entry(ring->request_list.prev,
		1779	struct drm_i915_gem_request,
		1780	list)->seqno;
		1781
		1782	return i915_wait_seqno(ring, seqno);
		1783	}
		1784
		1785	static int
5060	serge	1786	intel_ring_alloc_seqno(struct intel_engine_cs *ring)
3243	Serge	1787	{
4560	Serge	1788	if (ring->outstanding_lazy_seqno)
3243	Serge	1789	return 0;
		1790
4560	Serge	1791	if (ring->preallocated_lazy_request == NULL) {
		1792	struct drm_i915_gem_request *request;
		1793
		1794	request = kmalloc(sizeof(*request), GFP_KERNEL);
		1795	if (request == NULL)
		1796	return -ENOMEM;
		1797
		1798	ring->preallocated_lazy_request = request;
		1799	}
		1800
		1801	return i915_gem_get_seqno(ring->dev, &ring->outstanding_lazy_seqno);
3243	Serge	1802	}
		1803
5060	serge	1804	static int __intel_ring_prepare(struct intel_engine_cs *ring,
3480	Serge	1805	int bytes)
		1806	{
5060	serge	1807	struct intel_ringbuffer *ringbuf = ring->buffer;
3480	Serge	1808	int ret;
		1809
5060	serge	1810	if (unlikely(ringbuf->tail + bytes > ringbuf->effective_size)) {
3480	Serge	1811	ret = intel_wrap_ring_buffer(ring);
		1812	if (unlikely(ret))
		1813	return ret;
		1814	}
		1815
5060	serge	1816	if (unlikely(ringbuf->space < bytes)) {
3480	Serge	1817	ret = ring_wait_for_space(ring, bytes);
		1818	if (unlikely(ret))
		1819	return ret;
		1820	}
		1821
		1822	return 0;
		1823	}
		1824
5060	serge	1825	int intel_ring_begin(struct intel_engine_cs *ring,
2332	Serge	1826	int num_dwords)
		1827	{
5060	serge	1828	struct drm_i915_private *dev_priv = ring->dev->dev_private;
2332	Serge	1829	int ret;
		1830
3480	Serge	1831	ret = i915_gem_check_wedge(&dev_priv->gpu_error,
		1832	dev_priv->mm.interruptible);
3031	serge	1833	if (ret)
		1834	return ret;
2332	Serge	1835
4560	Serge	1836	ret = __intel_ring_prepare(ring, num_dwords * sizeof(uint32_t));
		1837	if (ret)
		1838	return ret;
		1839
3243	Serge	1840	/* Preallocate the olr before touching the ring */
		1841	ret = intel_ring_alloc_seqno(ring);
		1842	if (ret)
		1843	return ret;
		1844
5060	serge	1845	ring->buffer->space -= num_dwords * sizeof(uint32_t);
4560	Serge	1846	return 0;
3480	Serge	1847	}
2332	Serge	1848
5060	serge	1849	/* Align the ring tail to a cacheline boundary */
		1850	int intel_ring_cacheline_align(struct intel_engine_cs *ring)
3480	Serge	1851	{
5060	serge	1852	int num_dwords = (ring->buffer->tail & (CACHELINE_BYTES - 1)) / sizeof(uint32_t);
		1853	int ret;
3480	Serge	1854
5060	serge	1855	if (num_dwords == 0)
		1856	return 0;
		1857
		1858	num_dwords = CACHELINE_BYTES / sizeof(uint32_t) - num_dwords;
		1859	ret = intel_ring_begin(ring, num_dwords);
		1860	if (ret)
		1861	return ret;
		1862
		1863	while (num_dwords--)
		1864	intel_ring_emit(ring, MI_NOOP);
		1865
		1866	intel_ring_advance(ring);
		1867
		1868	return 0;
		1869	}
		1870
		1871	void intel_ring_init_seqno(struct intel_engine_cs *ring, u32 seqno)
		1872	{
		1873	struct drm_device *dev = ring->dev;
		1874	struct drm_i915_private *dev_priv = dev->dev_private;
		1875
4560	Serge	1876	BUG_ON(ring->outstanding_lazy_seqno);
3480	Serge	1877
5060	serge	1878	if (INTEL_INFO(dev)->gen == 6 \|\| INTEL_INFO(dev)->gen == 7) {
3480	Serge	1879	I915_WRITE(RING_SYNC_0(ring->mmio_base), 0);
		1880	I915_WRITE(RING_SYNC_1(ring->mmio_base), 0);
5060	serge	1881	if (HAS_VEBOX(dev))
4104	Serge	1882	I915_WRITE(RING_SYNC_2(ring->mmio_base), 0);
2332	Serge	1883	}
		1884
3480	Serge	1885	ring->set_seqno(ring, seqno);
4104	Serge	1886	ring->hangcheck.seqno = seqno;
2332	Serge	1887	}
		1888
5060	serge	1889	static void gen6_bsd_ring_write_tail(struct intel_engine_cs *ring,
2332	Serge	1890	u32 value)
		1891	{
5060	serge	1892	struct drm_i915_private *dev_priv = ring->dev->dev_private;
2332	Serge	1893
		1894	/* Every tail move must follow the sequence below */
3031	serge	1895
		1896	/* Disable notification that the ring is IDLE. The GT
		1897	* will then assume that it is busy and bring it out of rc6.
		1898	*/
2332	Serge	1899	I915_WRITE(GEN6_BSD_SLEEP_PSMI_CONTROL,
3031	serge	1900	_MASKED_BIT_ENABLE(GEN6_BSD_SLEEP_MSG_DISABLE));
2332	Serge	1901
3031	serge	1902	/* Clear the context id. Here be magic! */
		1903	I915_WRITE64(GEN6_BSD_RNCID, 0x0);
		1904
		1905	/* Wait for the ring not to be idle, i.e. for it to wake up. */
2332	Serge	1906	if (wait_for((I915_READ(GEN6_BSD_SLEEP_PSMI_CONTROL) &
3031	serge	1907	GEN6_BSD_SLEEP_INDICATOR) == 0,
2332	Serge	1908	50))
3031	serge	1909	DRM_ERROR("timed out waiting for the BSD ring to wake up\n");
2332	Serge	1910
3031	serge	1911	/* Now that the ring is fully powered up, update the tail */
2332	Serge	1912	I915_WRITE_TAIL(ring, value);
3031	serge	1913	POSTING_READ(RING_TAIL(ring->mmio_base));
		1914
		1915	/* Let the ring send IDLE messages to the GT again,
		1916	* and so let it sleep to conserve power when idle.
		1917	*/
2332	Serge	1918	I915_WRITE(GEN6_BSD_SLEEP_PSMI_CONTROL,
3031	serge	1919	_MASKED_BIT_DISABLE(GEN6_BSD_SLEEP_MSG_DISABLE));
2332	Serge	1920	}
		1921
5060	serge	1922	static int gen6_bsd_ring_flush(struct intel_engine_cs *ring,
2332	Serge	1923	u32 invalidate, u32 flush)
		1924	{
		1925	uint32_t cmd;
		1926	int ret;
		1927
		1928	ret = intel_ring_begin(ring, 4);
		1929	if (ret)
		1930	return ret;
		1931
		1932	cmd = MI_FLUSH_DW;
4560	Serge	1933	if (INTEL_INFO(ring->dev)->gen >= 8)
		1934	cmd += 1;
3243	Serge	1935	/*
		1936	* Bspec vol 1c.5 - video engine command streamer:
		1937	* "If ENABLED, all TLBs will be invalidated once the flush
		1938	* operation is complete. This bit is only valid when the
		1939	* Post-Sync Operation field is a value of 1h or 3h."
		1940	*/
2332	Serge	1941	if (invalidate & I915_GEM_GPU_DOMAINS)
3243	Serge	1942	cmd \|= MI_INVALIDATE_TLB \| MI_INVALIDATE_BSD \|
		1943	MI_FLUSH_DW_STORE_INDEX \| MI_FLUSH_DW_OP_STOREDW;
2332	Serge	1944	intel_ring_emit(ring, cmd);
3243	Serge	1945	intel_ring_emit(ring, I915_GEM_HWS_SCRATCH_ADDR \| MI_FLUSH_DW_USE_GTT);
4560	Serge	1946	if (INTEL_INFO(ring->dev)->gen >= 8) {
		1947	intel_ring_emit(ring, 0); /* upper addr */
		1948	intel_ring_emit(ring, 0); /* value */
		1949	} else {
2332	Serge	1950	intel_ring_emit(ring, 0);
		1951	intel_ring_emit(ring, MI_NOOP);
4560	Serge	1952	}
2332	Serge	1953	intel_ring_advance(ring);
		1954	return 0;
		1955	}
		1956
		1957	static int
5060	serge	1958	gen8_ring_dispatch_execbuffer(struct intel_engine_cs *ring,
		1959	u64 offset, u32 len,
4560	Serge	1960	unsigned flags)
		1961	{
		1962	struct drm_i915_private *dev_priv = ring->dev->dev_private;
		1963	bool ppgtt = dev_priv->mm.aliasing_ppgtt != NULL &&
		1964	!(flags & I915_DISPATCH_SECURE);
		1965	int ret;
		1966
		1967	ret = intel_ring_begin(ring, 4);
		1968	if (ret)
		1969	return ret;
		1970
		1971	/* FIXME(BDW): Address space and security selectors. */
		1972	intel_ring_emit(ring, MI_BATCH_BUFFER_START_GEN8 \| (ppgtt<<8));
5060	serge	1973	intel_ring_emit(ring, lower_32_bits(offset));
		1974	intel_ring_emit(ring, upper_32_bits(offset));
4560	Serge	1975	intel_ring_emit(ring, MI_NOOP);
		1976	intel_ring_advance(ring);
		1977
		1978	return 0;
		1979	}
		1980
		1981	static int
5060	serge	1982	hsw_ring_dispatch_execbuffer(struct intel_engine_cs *ring,
		1983	u64 offset, u32 len,
3243	Serge	1984	unsigned flags)
		1985	{
		1986	int ret;
		1987
		1988	ret = intel_ring_begin(ring, 2);
		1989	if (ret)
		1990	return ret;
		1991
		1992	intel_ring_emit(ring,
		1993	MI_BATCH_BUFFER_START \| MI_BATCH_PPGTT_HSW \|
		1994	(flags & I915_DISPATCH_SECURE ? 0 : MI_BATCH_NON_SECURE_HSW));
		1995	/* bit0-7 is the length on GEN6+ */
		1996	intel_ring_emit(ring, offset);
		1997	intel_ring_advance(ring);
		1998
		1999	return 0;
		2000	}
		2001
		2002	static int
5060	serge	2003	gen6_ring_dispatch_execbuffer(struct intel_engine_cs *ring,
		2004	u64 offset, u32 len,
3243	Serge	2005	unsigned flags)
2332	Serge	2006	{
		2007	int ret;
		2008
		2009	ret = intel_ring_begin(ring, 2);
		2010	if (ret)
		2011	return ret;
		2012
3243	Serge	2013	intel_ring_emit(ring,
		2014	MI_BATCH_BUFFER_START \|
		2015	(flags & I915_DISPATCH_SECURE ? 0 : MI_BATCH_NON_SECURE_I965));
2332	Serge	2016	/* bit0-7 is the length on GEN6+ */
		2017	intel_ring_emit(ring, offset);
		2018	intel_ring_advance(ring);
		2019
		2020	return 0;
		2021	}
		2022
		2023	/* Blitter support (SandyBridge+) */
		2024
5060	serge	2025	static int gen6_ring_flush(struct intel_engine_cs *ring,
2332	Serge	2026	u32 invalidate, u32 flush)
		2027	{
4104	Serge	2028	struct drm_device *dev = ring->dev;
2332	Serge	2029	uint32_t cmd;
		2030	int ret;
		2031
3031	serge	2032	ret = intel_ring_begin(ring, 4);
2332	Serge	2033	if (ret)
		2034	return ret;
		2035
		2036	cmd = MI_FLUSH_DW;
4560	Serge	2037	if (INTEL_INFO(ring->dev)->gen >= 8)
		2038	cmd += 1;
3243	Serge	2039	/*
		2040	* Bspec vol 1c.3 - blitter engine command streamer:
		2041	* "If ENABLED, all TLBs will be invalidated once the flush
		2042	* operation is complete. This bit is only valid when the
		2043	* Post-Sync Operation field is a value of 1h or 3h."
		2044	*/
2332	Serge	2045	if (invalidate & I915_GEM_DOMAIN_RENDER)
3243	Serge	2046	cmd \|= MI_INVALIDATE_TLB \| MI_FLUSH_DW_STORE_INDEX \|
		2047	MI_FLUSH_DW_OP_STOREDW;
2332	Serge	2048	intel_ring_emit(ring, cmd);
3243	Serge	2049	intel_ring_emit(ring, I915_GEM_HWS_SCRATCH_ADDR \| MI_FLUSH_DW_USE_GTT);
4560	Serge	2050	if (INTEL_INFO(ring->dev)->gen >= 8) {
		2051	intel_ring_emit(ring, 0); /* upper addr */
		2052	intel_ring_emit(ring, 0); /* value */
		2053	} else {
2332	Serge	2054	intel_ring_emit(ring, 0);
		2055	intel_ring_emit(ring, MI_NOOP);
4560	Serge	2056	}
2332	Serge	2057	intel_ring_advance(ring);
4104	Serge	2058
4560	Serge	2059	if (IS_GEN7(dev) && !invalidate && flush)
4104	Serge	2060	return gen7_ring_fbc_flush(ring, FBC_REND_CACHE_CLEAN);
		2061
2332	Serge	2062	return 0;
		2063	}
		2064
		2065	int intel_init_render_ring_buffer(struct drm_device *dev)
		2066	{
5060	serge	2067	struct drm_i915_private *dev_priv = dev->dev_private;
		2068	struct intel_engine_cs *ring = &dev_priv->ring[RCS];
		2069	struct drm_i915_gem_object *obj;
		2070	int ret;
2340	Serge	2071
3031	serge	2072	ring->name = "render ring";
		2073	ring->id = RCS;
		2074	ring->mmio_base = RENDER_RING_BASE;
		2075
5060	serge	2076	if (INTEL_INFO(dev)->gen >= 8) {
		2077	if (i915_semaphore_is_enabled(dev)) {
		2078	obj = i915_gem_alloc_object(dev, 4096);
		2079	if (obj == NULL) {
		2080	DRM_ERROR("Failed to allocate semaphore bo. Disabling semaphores\n");
		2081	i915.semaphores = 0;
		2082	} else {
		2083	i915_gem_object_set_cache_level(obj, I915_CACHE_LLC);
		2084	ret = i915_gem_obj_ggtt_pin(obj, 0, PIN_NONBLOCK);
		2085	if (ret != 0) {
		2086	drm_gem_object_unreference(&obj->base);
		2087	DRM_ERROR("Failed to pin semaphore bo. Disabling semaphores\n");
		2088	i915.semaphores = 0;
		2089	} else
		2090	dev_priv->semaphore_obj = obj;
		2091	}
		2092	}
		2093	ring->add_request = gen6_add_request;
		2094	ring->flush = gen8_render_ring_flush;
		2095	ring->irq_get = gen8_ring_get_irq;
		2096	ring->irq_put = gen8_ring_put_irq;
		2097	ring->irq_enable_mask = GT_RENDER_USER_INTERRUPT;
		2098	ring->get_seqno = gen6_ring_get_seqno;
		2099	ring->set_seqno = ring_set_seqno;
		2100	if (i915_semaphore_is_enabled(dev)) {
		2101	WARN_ON(!dev_priv->semaphore_obj);
		2102	ring->semaphore.sync_to = gen8_ring_sync;
		2103	ring->semaphore.signal = gen8_rcs_signal;
		2104	GEN8_RING_SEMAPHORE_INIT;
		2105	}
		2106	} else if (INTEL_INFO(dev)->gen >= 6) {
2339	Serge	2107	ring->add_request = gen6_add_request;
3031	serge	2108	ring->flush = gen7_render_ring_flush;
		2109	if (INTEL_INFO(dev)->gen == 6)
2342	Serge	2110	ring->flush = gen6_render_ring_flush;
3031	serge	2111	ring->irq_get = gen6_ring_get_irq;
		2112	ring->irq_put = gen6_ring_put_irq;
4104	Serge	2113	ring->irq_enable_mask = GT_RENDER_USER_INTERRUPT;
2342	Serge	2114	ring->get_seqno = gen6_ring_get_seqno;
3480	Serge	2115	ring->set_seqno = ring_set_seqno;
5060	serge	2116	if (i915_semaphore_is_enabled(dev)) {
		2117	ring->semaphore.sync_to = gen6_ring_sync;
		2118	ring->semaphore.signal = gen6_signal;
		2119	/*
		2120	* The current semaphore is only applied on pre-gen8
		2121	* platform. And there is no VCS2 ring on the pre-gen8
		2122	* platform. So the semaphore between RCS and VCS2 is
		2123	* initialized as INVALID. Gen8 will initialize the
		2124	* sema between VCS2 and RCS later.
		2125	*/
		2126	ring->semaphore.mbox.wait[RCS] = MI_SEMAPHORE_SYNC_INVALID;
		2127	ring->semaphore.mbox.wait[VCS] = MI_SEMAPHORE_SYNC_RV;
		2128	ring->semaphore.mbox.wait[BCS] = MI_SEMAPHORE_SYNC_RB;
		2129	ring->semaphore.mbox.wait[VECS] = MI_SEMAPHORE_SYNC_RVE;
		2130	ring->semaphore.mbox.wait[VCS2] = MI_SEMAPHORE_SYNC_INVALID;
		2131	ring->semaphore.mbox.signal[RCS] = GEN6_NOSYNC;
		2132	ring->semaphore.mbox.signal[VCS] = GEN6_VRSYNC;
		2133	ring->semaphore.mbox.signal[BCS] = GEN6_BRSYNC;
		2134	ring->semaphore.mbox.signal[VECS] = GEN6_VERSYNC;
		2135	ring->semaphore.mbox.signal[VCS2] = GEN6_NOSYNC;
		2136	}
2332	Serge	2137	} else if (IS_GEN5(dev)) {
2339	Serge	2138	ring->add_request = pc_render_add_request;
3031	serge	2139	ring->flush = gen4_render_ring_flush;
2342	Serge	2140	ring->get_seqno = pc_render_get_seqno;
3480	Serge	2141	ring->set_seqno = pc_render_set_seqno;
3031	serge	2142	ring->irq_get = gen5_ring_get_irq;
		2143	ring->irq_put = gen5_ring_put_irq;
4104	Serge	2144	ring->irq_enable_mask = GT_RENDER_USER_INTERRUPT \|
		2145	GT_RENDER_PIPECTL_NOTIFY_INTERRUPT;
3031	serge	2146	} else {
		2147	ring->add_request = i9xx_add_request;
		2148	if (INTEL_INFO(dev)->gen < 4)
		2149	ring->flush = gen2_render_ring_flush;
		2150	else
		2151	ring->flush = gen4_render_ring_flush;
		2152	ring->get_seqno = ring_get_seqno;
3480	Serge	2153	ring->set_seqno = ring_set_seqno;
3031	serge	2154	if (IS_GEN2(dev)) {
		2155	ring->irq_get = i8xx_ring_get_irq;
		2156	ring->irq_put = i8xx_ring_put_irq;
		2157	} else {
		2158	ring->irq_get = i9xx_ring_get_irq;
		2159	ring->irq_put = i9xx_ring_put_irq;
		2160	}
		2161	ring->irq_enable_mask = I915_USER_INTERRUPT;
2332	Serge	2162	}
3031	serge	2163	ring->write_tail = ring_write_tail;
5060	serge	2164
3243	Serge	2165	if (IS_HASWELL(dev))
		2166	ring->dispatch_execbuffer = hsw_ring_dispatch_execbuffer;
4560	Serge	2167	else if (IS_GEN8(dev))
		2168	ring->dispatch_execbuffer = gen8_ring_dispatch_execbuffer;
3243	Serge	2169	else if (INTEL_INFO(dev)->gen >= 6)
3031	serge	2170	ring->dispatch_execbuffer = gen6_ring_dispatch_execbuffer;
		2171	else if (INTEL_INFO(dev)->gen >= 4)
		2172	ring->dispatch_execbuffer = i965_dispatch_execbuffer;
		2173	else if (IS_I830(dev) \|\| IS_845G(dev))
		2174	ring->dispatch_execbuffer = i830_dispatch_execbuffer;
		2175	else
		2176	ring->dispatch_execbuffer = i915_dispatch_execbuffer;
		2177	ring->init = init_render_ring;
		2178	ring->cleanup = render_ring_cleanup;
2332	Serge	2179
3243	Serge	2180	/* Workaround batchbuffer to combat CS tlb bug. */
		2181	if (HAS_BROKEN_CS_TLB(dev)) {
		2182	obj = i915_gem_alloc_object(dev, I830_BATCH_LIMIT);
		2183	if (obj == NULL) {
		2184	DRM_ERROR("Failed to allocate batch bo\n");
		2185	return -ENOMEM;
		2186	}
		2187
5060	serge	2188	ret = i915_gem_obj_ggtt_pin(obj, 0, 0);
3243	Serge	2189	if (ret != 0) {
		2190	drm_gem_object_unreference(&obj->base);
		2191	DRM_ERROR("Failed to ping batch bo\n");
		2192	return ret;
		2193	}
		2194
4104	Serge	2195	ring->scratch.obj = obj;
		2196	ring->scratch.gtt_offset = i915_gem_obj_ggtt_offset(obj);
2332	Serge	2197	}
2340	Serge	2198
2332	Serge	2199	return intel_init_ring_buffer(dev, ring);
		2200	}
		2201
3243	Serge	2202	#if 0
		2203	int intel_render_ring_init_dri(struct drm_device *dev, u64 start, u32 size)
		2204	{
5060	serge	2205	struct drm_i915_private *dev_priv = dev->dev_private;
		2206	struct intel_engine_cs *ring = &dev_priv->ring[RCS];
		2207	struct intel_ringbuffer *ringbuf = ring->buffer;
3243	Serge	2208	int ret;
2332	Serge	2209
5060	serge	2210	if (ringbuf == NULL) {
		2211	ringbuf = kzalloc(sizeof(*ringbuf), GFP_KERNEL);
		2212	if (!ringbuf)
		2213	return -ENOMEM;
		2214	ring->buffer = ringbuf;
		2215	}
		2216
3243	Serge	2217	ring->name = "render ring";
		2218	ring->id = RCS;
		2219	ring->mmio_base = RENDER_RING_BASE;
		2220
		2221	if (INTEL_INFO(dev)->gen >= 6) {
		2222	/* non-kms not supported on gen6+ */
5060	serge	2223	ret = -ENODEV;
		2224	goto err_ringbuf;
3243	Serge	2225	}
		2226
		2227	/* Note: gem is not supported on gen5/ilk without kms (the corresponding
		2228	* gem_init ioctl returns with -ENODEV). Hence we do not need to set up
		2229	* the special gen5 functions. */
		2230	ring->add_request = i9xx_add_request;
		2231	if (INTEL_INFO(dev)->gen < 4)
		2232	ring->flush = gen2_render_ring_flush;
		2233	else
		2234	ring->flush = gen4_render_ring_flush;
		2235	ring->get_seqno = ring_get_seqno;
3480	Serge	2236	ring->set_seqno = ring_set_seqno;
3243	Serge	2237	if (IS_GEN2(dev)) {
		2238	ring->irq_get = i8xx_ring_get_irq;
		2239	ring->irq_put = i8xx_ring_put_irq;
		2240	} else {
		2241	ring->irq_get = i9xx_ring_get_irq;
		2242	ring->irq_put = i9xx_ring_put_irq;
		2243	}
		2244	ring->irq_enable_mask = I915_USER_INTERRUPT;
		2245	ring->write_tail = ring_write_tail;
		2246	if (INTEL_INFO(dev)->gen >= 4)
		2247	ring->dispatch_execbuffer = i965_dispatch_execbuffer;
		2248	else if (IS_I830(dev) \|\| IS_845G(dev))
		2249	ring->dispatch_execbuffer = i830_dispatch_execbuffer;
		2250	else
		2251	ring->dispatch_execbuffer = i915_dispatch_execbuffer;
		2252	ring->init = init_render_ring;
		2253	ring->cleanup = render_ring_cleanup;
		2254
		2255	ring->dev = dev;
		2256	INIT_LIST_HEAD(&ring->active_list);
		2257	INIT_LIST_HEAD(&ring->request_list);
		2258
5060	serge	2259	ringbuf->size = size;
		2260	ringbuf->effective_size = ringbuf->size;
3243	Serge	2261	if (IS_I830(ring->dev) \|\| IS_845G(ring->dev))
5060	serge	2262	ringbuf->effective_size -= 2 * CACHELINE_BYTES;
3243	Serge	2263
5060	serge	2264	ringbuf->virtual_start = ioremap_wc(start, size);
		2265	if (ringbuf->virtual_start == NULL) {
3243	Serge	2266	DRM_ERROR("can not ioremap virtual address for"
		2267	" ring buffer\n");
5060	serge	2268	ret = -ENOMEM;
		2269	goto err_ringbuf;
3243	Serge	2270	}
		2271
		2272	if (!I915_NEED_GFX_HWS(dev)) {
4104	Serge	2273	ret = init_phys_status_page(ring);
3243	Serge	2274	if (ret)
5060	serge	2275	goto err_vstart;
3243	Serge	2276	}
		2277
		2278	return 0;
5060	serge	2279
		2280	err_vstart:
		2281	iounmap(ringbuf->virtual_start);
		2282	err_ringbuf:
		2283	kfree(ringbuf);
		2284	ring->buffer = NULL;
		2285	return ret;
3243	Serge	2286	}
		2287	#endif
		2288
2332	Serge	2289	int intel_init_bsd_ring_buffer(struct drm_device *dev)
		2290	{
5060	serge	2291	struct drm_i915_private *dev_priv = dev->dev_private;
		2292	struct intel_engine_cs *ring = &dev_priv->ring[VCS];
2332	Serge	2293
3031	serge	2294	ring->name = "bsd ring";
		2295	ring->id = VCS;
2332	Serge	2296
3031	serge	2297	ring->write_tail = ring_write_tail;
4560	Serge	2298	if (INTEL_INFO(dev)->gen >= 6) {
3031	serge	2299	ring->mmio_base = GEN6_BSD_RING_BASE;
		2300	/* gen6 bsd needs a special wa for tail updates */
		2301	if (IS_GEN6(dev))
		2302	ring->write_tail = gen6_bsd_ring_write_tail;
4104	Serge	2303	ring->flush = gen6_bsd_ring_flush;
3031	serge	2304	ring->add_request = gen6_add_request;
		2305	ring->get_seqno = gen6_ring_get_seqno;
3480	Serge	2306	ring->set_seqno = ring_set_seqno;
4560	Serge	2307	if (INTEL_INFO(dev)->gen >= 8) {
		2308	ring->irq_enable_mask =
		2309	GT_RENDER_USER_INTERRUPT << GEN8_VCS1_IRQ_SHIFT;
		2310	ring->irq_get = gen8_ring_get_irq;
		2311	ring->irq_put = gen8_ring_put_irq;
		2312	ring->dispatch_execbuffer =
		2313	gen8_ring_dispatch_execbuffer;
5060	serge	2314	if (i915_semaphore_is_enabled(dev)) {
		2315	ring->semaphore.sync_to = gen8_ring_sync;
		2316	ring->semaphore.signal = gen8_xcs_signal;
		2317	GEN8_RING_SEMAPHORE_INIT;
		2318	}
4560	Serge	2319	} else {
4104	Serge	2320	ring->irq_enable_mask = GT_BSD_USER_INTERRUPT;
3031	serge	2321	ring->irq_get = gen6_ring_get_irq;
		2322	ring->irq_put = gen6_ring_put_irq;
4560	Serge	2323	ring->dispatch_execbuffer =
		2324	gen6_ring_dispatch_execbuffer;
5060	serge	2325	if (i915_semaphore_is_enabled(dev)) {
		2326	ring->semaphore.sync_to = gen6_ring_sync;
		2327	ring->semaphore.signal = gen6_signal;
		2328	ring->semaphore.mbox.wait[RCS] = MI_SEMAPHORE_SYNC_VR;
		2329	ring->semaphore.mbox.wait[VCS] = MI_SEMAPHORE_SYNC_INVALID;
		2330	ring->semaphore.mbox.wait[BCS] = MI_SEMAPHORE_SYNC_VB;
		2331	ring->semaphore.mbox.wait[VECS] = MI_SEMAPHORE_SYNC_VVE;
		2332	ring->semaphore.mbox.wait[VCS2] = MI_SEMAPHORE_SYNC_INVALID;
		2333	ring->semaphore.mbox.signal[RCS] = GEN6_RVSYNC;
		2334	ring->semaphore.mbox.signal[VCS] = GEN6_NOSYNC;
		2335	ring->semaphore.mbox.signal[BCS] = GEN6_BVSYNC;
		2336	ring->semaphore.mbox.signal[VECS] = GEN6_VEVSYNC;
		2337	ring->semaphore.mbox.signal[VCS2] = GEN6_NOSYNC;
		2338	}
4560	Serge	2339	}
3031	serge	2340	} else {
		2341	ring->mmio_base = BSD_RING_BASE;
		2342	ring->flush = bsd_ring_flush;
		2343	ring->add_request = i9xx_add_request;
		2344	ring->get_seqno = ring_get_seqno;
3480	Serge	2345	ring->set_seqno = ring_set_seqno;
3031	serge	2346	if (IS_GEN5(dev)) {
4104	Serge	2347	ring->irq_enable_mask = ILK_BSD_USER_INTERRUPT;
3031	serge	2348	ring->irq_get = gen5_ring_get_irq;
		2349	ring->irq_put = gen5_ring_put_irq;
		2350	} else {
		2351	ring->irq_enable_mask = I915_BSD_USER_INTERRUPT;
		2352	ring->irq_get = i9xx_ring_get_irq;
		2353	ring->irq_put = i9xx_ring_put_irq;
		2354	}
		2355	ring->dispatch_execbuffer = i965_dispatch_execbuffer;
		2356	}
		2357	ring->init = init_ring_common;
		2358
2332	Serge	2359	return intel_init_ring_buffer(dev, ring);
		2360	}
		2361
5060	serge	2362	/**
		2363	* Initialize the second BSD ring for Broadwell GT3.
		2364	* It is noted that this only exists on Broadwell GT3.
		2365	*/
		2366	int intel_init_bsd2_ring_buffer(struct drm_device *dev)
		2367	{
		2368	struct drm_i915_private *dev_priv = dev->dev_private;
		2369	struct intel_engine_cs *ring = &dev_priv->ring[VCS2];
		2370
		2371	if ((INTEL_INFO(dev)->gen != 8)) {
		2372	DRM_ERROR("No dual-BSD ring on non-BDW machine\n");
		2373	return -EINVAL;
		2374	}
		2375
		2376	ring->name = "bsd2 ring";
		2377	ring->id = VCS2;
		2378
		2379	ring->write_tail = ring_write_tail;
		2380	ring->mmio_base = GEN8_BSD2_RING_BASE;
		2381	ring->flush = gen6_bsd_ring_flush;
		2382	ring->add_request = gen6_add_request;
		2383	ring->get_seqno = gen6_ring_get_seqno;
		2384	ring->set_seqno = ring_set_seqno;
		2385	ring->irq_enable_mask =
		2386	GT_RENDER_USER_INTERRUPT << GEN8_VCS2_IRQ_SHIFT;
		2387	ring->irq_get = gen8_ring_get_irq;
		2388	ring->irq_put = gen8_ring_put_irq;
		2389	ring->dispatch_execbuffer =
		2390	gen8_ring_dispatch_execbuffer;
		2391	if (i915_semaphore_is_enabled(dev)) {
		2392	ring->semaphore.sync_to = gen8_ring_sync;
		2393	ring->semaphore.signal = gen8_xcs_signal;
		2394	GEN8_RING_SEMAPHORE_INIT;
		2395	}
		2396	ring->init = init_ring_common;
		2397
		2398	return intel_init_ring_buffer(dev, ring);
		2399	}
		2400
2332	Serge	2401	int intel_init_blt_ring_buffer(struct drm_device *dev)
		2402	{
5060	serge	2403	struct drm_i915_private *dev_priv = dev->dev_private;
		2404	struct intel_engine_cs *ring = &dev_priv->ring[BCS];
2332	Serge	2405
3031	serge	2406	ring->name = "blitter ring";
		2407	ring->id = BCS;
2332	Serge	2408
3031	serge	2409	ring->mmio_base = BLT_RING_BASE;
		2410	ring->write_tail = ring_write_tail;
4104	Serge	2411	ring->flush = gen6_ring_flush;
3031	serge	2412	ring->add_request = gen6_add_request;
		2413	ring->get_seqno = gen6_ring_get_seqno;
3480	Serge	2414	ring->set_seqno = ring_set_seqno;
4560	Serge	2415	if (INTEL_INFO(dev)->gen >= 8) {
		2416	ring->irq_enable_mask =
		2417	GT_RENDER_USER_INTERRUPT << GEN8_BCS_IRQ_SHIFT;
		2418	ring->irq_get = gen8_ring_get_irq;
		2419	ring->irq_put = gen8_ring_put_irq;
		2420	ring->dispatch_execbuffer = gen8_ring_dispatch_execbuffer;
5060	serge	2421	if (i915_semaphore_is_enabled(dev)) {
		2422	ring->semaphore.sync_to = gen8_ring_sync;
		2423	ring->semaphore.signal = gen8_xcs_signal;
		2424	GEN8_RING_SEMAPHORE_INIT;
		2425	}
4560	Serge	2426	} else {
4104	Serge	2427	ring->irq_enable_mask = GT_BLT_USER_INTERRUPT;
3031	serge	2428	ring->irq_get = gen6_ring_get_irq;
		2429	ring->irq_put = gen6_ring_put_irq;
		2430	ring->dispatch_execbuffer = gen6_ring_dispatch_execbuffer;
5060	serge	2431	if (i915_semaphore_is_enabled(dev)) {
		2432	ring->semaphore.signal = gen6_signal;
		2433	ring->semaphore.sync_to = gen6_ring_sync;
		2434	/*
		2435	* The current semaphore is only applied on pre-gen8
		2436	* platform. And there is no VCS2 ring on the pre-gen8
		2437	* platform. So the semaphore between BCS and VCS2 is
		2438	* initialized as INVALID. Gen8 will initialize the
		2439	* sema between BCS and VCS2 later.
		2440	*/
		2441	ring->semaphore.mbox.wait[RCS] = MI_SEMAPHORE_SYNC_BR;
		2442	ring->semaphore.mbox.wait[VCS] = MI_SEMAPHORE_SYNC_BV;
		2443	ring->semaphore.mbox.wait[BCS] = MI_SEMAPHORE_SYNC_INVALID;
		2444	ring->semaphore.mbox.wait[VECS] = MI_SEMAPHORE_SYNC_BVE;
		2445	ring->semaphore.mbox.wait[VCS2] = MI_SEMAPHORE_SYNC_INVALID;
		2446	ring->semaphore.mbox.signal[RCS] = GEN6_RBSYNC;
		2447	ring->semaphore.mbox.signal[VCS] = GEN6_VBSYNC;
		2448	ring->semaphore.mbox.signal[BCS] = GEN6_NOSYNC;
		2449	ring->semaphore.mbox.signal[VECS] = GEN6_VEBSYNC;
		2450	ring->semaphore.mbox.signal[VCS2] = GEN6_NOSYNC;
		2451	}
4560	Serge	2452	}
3031	serge	2453	ring->init = init_ring_common;
		2454
2332	Serge	2455	return intel_init_ring_buffer(dev, ring);
		2456	}
3031	serge	2457
4104	Serge	2458	int intel_init_vebox_ring_buffer(struct drm_device *dev)
		2459	{
5060	serge	2460	struct drm_i915_private *dev_priv = dev->dev_private;
		2461	struct intel_engine_cs *ring = &dev_priv->ring[VECS];
4104	Serge	2462
		2463	ring->name = "video enhancement ring";
		2464	ring->id = VECS;
		2465
		2466	ring->mmio_base = VEBOX_RING_BASE;
		2467	ring->write_tail = ring_write_tail;
		2468	ring->flush = gen6_ring_flush;
		2469	ring->add_request = gen6_add_request;
		2470	ring->get_seqno = gen6_ring_get_seqno;
		2471	ring->set_seqno = ring_set_seqno;
4560	Serge	2472
		2473	if (INTEL_INFO(dev)->gen >= 8) {
		2474	ring->irq_enable_mask =
		2475	GT_RENDER_USER_INTERRUPT << GEN8_VECS_IRQ_SHIFT;
		2476	ring->irq_get = gen8_ring_get_irq;
		2477	ring->irq_put = gen8_ring_put_irq;
		2478	ring->dispatch_execbuffer = gen8_ring_dispatch_execbuffer;
5060	serge	2479	if (i915_semaphore_is_enabled(dev)) {
		2480	ring->semaphore.sync_to = gen8_ring_sync;
		2481	ring->semaphore.signal = gen8_xcs_signal;
		2482	GEN8_RING_SEMAPHORE_INIT;
		2483	}
4560	Serge	2484	} else {
4104	Serge	2485	ring->irq_enable_mask = PM_VEBOX_USER_INTERRUPT;
		2486	ring->irq_get = hsw_vebox_get_irq;
		2487	ring->irq_put = hsw_vebox_put_irq;
		2488	ring->dispatch_execbuffer = gen6_ring_dispatch_execbuffer;
5060	serge	2489	if (i915_semaphore_is_enabled(dev)) {
		2490	ring->semaphore.sync_to = gen6_ring_sync;
		2491	ring->semaphore.signal = gen6_signal;
		2492	ring->semaphore.mbox.wait[RCS] = MI_SEMAPHORE_SYNC_VER;
		2493	ring->semaphore.mbox.wait[VCS] = MI_SEMAPHORE_SYNC_VEV;
		2494	ring->semaphore.mbox.wait[BCS] = MI_SEMAPHORE_SYNC_VEB;
		2495	ring->semaphore.mbox.wait[VECS] = MI_SEMAPHORE_SYNC_INVALID;
		2496	ring->semaphore.mbox.wait[VCS2] = MI_SEMAPHORE_SYNC_INVALID;
		2497	ring->semaphore.mbox.signal[RCS] = GEN6_RVESYNC;
		2498	ring->semaphore.mbox.signal[VCS] = GEN6_VVESYNC;
		2499	ring->semaphore.mbox.signal[BCS] = GEN6_BVESYNC;
		2500	ring->semaphore.mbox.signal[VECS] = GEN6_NOSYNC;
		2501	ring->semaphore.mbox.signal[VCS2] = GEN6_NOSYNC;
		2502	}
4560	Serge	2503	}
4104	Serge	2504	ring->init = init_ring_common;
		2505
		2506	return intel_init_ring_buffer(dev, ring);
		2507	}
		2508
3031	serge	2509	int
5060	serge	2510	intel_ring_flush_all_caches(struct intel_engine_cs *ring)
3031	serge	2511	{
		2512	int ret;
		2513
		2514	if (!ring->gpu_caches_dirty)
		2515	return 0;
		2516
		2517	ret = ring->flush(ring, 0, I915_GEM_GPU_DOMAINS);
		2518	if (ret)
		2519	return ret;
		2520
		2521	trace_i915_gem_ring_flush(ring, 0, I915_GEM_GPU_DOMAINS);
		2522
		2523	ring->gpu_caches_dirty = false;
		2524	return 0;
		2525	}
		2526
		2527	int
5060	serge	2528	intel_ring_invalidate_all_caches(struct intel_engine_cs *ring)
3031	serge	2529	{
		2530	uint32_t flush_domains;
		2531	int ret;
		2532
		2533	flush_domains = 0;
		2534	if (ring->gpu_caches_dirty)
		2535	flush_domains = I915_GEM_GPU_DOMAINS;
		2536
		2537	ret = ring->flush(ring, I915_GEM_GPU_DOMAINS, flush_domains);
		2538	if (ret)
		2539	return ret;
		2540
		2541	trace_i915_gem_ring_flush(ring, I915_GEM_GPU_DOMAINS, flush_domains);
		2542
		2543	ring->gpu_caches_dirty = false;
		2544	return 0;
		2545	}
5060	serge	2546
		2547	void
		2548	intel_stop_ring_buffer(struct intel_engine_cs *ring)
		2549	{
		2550	int ret;
		2551
		2552	if (!intel_ring_initialized(ring))
		2553	return;
		2554
		2555	ret = intel_ring_idle(ring);
		2556	if (ret && !i915_reset_in_progress(&to_i915(ring->dev)->gpu_error))
		2557	DRM_ERROR("failed to quiesce %s whilst cleaning up: %d\n",
		2558	ring->name, ret);
		2559
		2560	stop_ring(ring);
		2561	}

Subversion Repositories Kolibri OS

(root)/drivers/video/drm/i915/intel_ringbuffer.c @ 5354 – Rev 5060