WebSVN – Kolibri OS – Blame – /drivers/video/drm/i915/intel_ringbuffer.c

Rev	Author	Line No.	Line
2332	Serge	1	/*
		2	* Copyright © 2008-2010 Intel Corporation
		3	*
		4	* Permission is hereby granted, free of charge, to any person obtaining a
		5	* copy of this software and associated documentation files (the "Software"),
		6	* to deal in the Software without restriction, including without limitation
		7	* the rights to use, copy, modify, merge, publish, distribute, sublicense,
		8	* and/or sell copies of the Software, and to permit persons to whom the
		9	* Software is furnished to do so, subject to the following conditions:
		10	*
		11	* The above copyright notice and this permission notice (including the next
		12	* paragraph) shall be included in all copies or substantial portions of the
		13	* Software.
		14	*
		15	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
		16	* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
		17	* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
		18	* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
		19	* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
		20	* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
		21	* IN THE SOFTWARE.
		22	*
		23	* Authors:
		24	* Eric Anholt
		25	* Zou Nan hai
		26	* Xiang Hai hao
		27	*
		28	*/
		29	#define iowrite32(v, addr) writel((v), (addr))
		30	#define ioread32(addr) readl(addr)
		31
3031	serge	32	#include
2332	Serge	33	#include "i915_drv.h"
3031	serge	34	#include
2351	Serge	35	#include "i915_trace.h"
2332	Serge	36	#include "intel_drv.h"
		37
2342	Serge	38	/*
		39	* 965+ support PIPE_CONTROL commands, which provide finer grained control
		40	* over cache flushing.
		41	*/
		42	struct pipe_control {
		43	struct drm_i915_gem_object *obj;
		44	volatile u32 *cpu_page;
		45	u32 gtt_offset;
		46	};
		47
2332	Serge	48	static inline int ring_space(struct intel_ring_buffer *ring)
		49	{
		50	int space = (ring->head & HEAD_ADDR) - (ring->tail + 8);
		51	if (space < 0)
		52	space += ring->size;
		53	return space;
		54	}
		55
3031	serge	56	static int
		57	gen2_render_ring_flush(struct intel_ring_buffer *ring,
		58	u32 invalidate_domains,
		59	u32 flush_domains)
2332	Serge	60	{
3031	serge	61	u32 cmd;
		62	int ret;
2332	Serge	63
3031	serge	64	cmd = MI_FLUSH;
		65	if (((invalidate_domains\|flush_domains) & I915_GEM_DOMAIN_RENDER) == 0)
		66	cmd \|= MI_NO_WRITE_FLUSH;
2332	Serge	67
3031	serge	68	if (invalidate_domains & I915_GEM_DOMAIN_SAMPLER)
		69	cmd \|= MI_READ_FLUSH;
2332	Serge	70
3031	serge	71	ret = intel_ring_begin(ring, 2);
		72	if (ret)
		73	return ret;
		74
		75	intel_ring_emit(ring, cmd);
		76	intel_ring_emit(ring, MI_NOOP);
		77	intel_ring_advance(ring);
		78
		79	return 0;
2332	Serge	80	}
		81
		82	static int
3031	serge	83	gen4_render_ring_flush(struct intel_ring_buffer *ring,
2332	Serge	84	u32 invalidate_domains,
		85	u32 flush_domains)
		86	{
		87	struct drm_device *dev = ring->dev;
		88	u32 cmd;
		89	int ret;
		90
		91	/*
		92	* read/write caches:
		93	*
		94	* I915_GEM_DOMAIN_RENDER is always invalidated, but is
		95	* only flushed if MI_NO_WRITE_FLUSH is unset. On 965, it is
		96	* also flushed at 2d versus 3d pipeline switches.
		97	*
		98	* read-only caches:
		99	*
		100	* I915_GEM_DOMAIN_SAMPLER is flushed on pre-965 if
		101	* MI_READ_FLUSH is set, and is always flushed on 965.
		102	*
		103	* I915_GEM_DOMAIN_COMMAND may not exist?
		104	*
		105	* I915_GEM_DOMAIN_INSTRUCTION, which exists on 965, is
		106	* invalidated when MI_EXE_FLUSH is set.
		107	*
		108	* I915_GEM_DOMAIN_VERTEX, which exists on 965, is
		109	* invalidated with every MI_FLUSH.
		110	*
		111	* TLBs:
		112	*
		113	* On 965, TLBs associated with I915_GEM_DOMAIN_COMMAND
		114	* and I915_GEM_DOMAIN_CPU in are invalidated at PTE write and
		115	* I915_GEM_DOMAIN_RENDER and I915_GEM_DOMAIN_SAMPLER
		116	* are flushed at any MI_FLUSH.
		117	*/
		118
		119	cmd = MI_FLUSH \| MI_NO_WRITE_FLUSH;
3031	serge	120	if ((invalidate_domains\|flush_domains) & I915_GEM_DOMAIN_RENDER)
2332	Serge	121	cmd &= ~MI_NO_WRITE_FLUSH;
		122	if (invalidate_domains & I915_GEM_DOMAIN_INSTRUCTION)
		123	cmd \|= MI_EXE_FLUSH;
		124
		125	if (invalidate_domains & I915_GEM_DOMAIN_COMMAND &&
		126	(IS_G4X(dev) \|\| IS_GEN5(dev)))
		127	cmd \|= MI_INVALIDATE_ISP;
		128
		129	ret = intel_ring_begin(ring, 2);
		130	if (ret)
		131	return ret;
		132
		133	intel_ring_emit(ring, cmd);
		134	intel_ring_emit(ring, MI_NOOP);
		135	intel_ring_advance(ring);
		136
		137	return 0;
		138	}
		139
2342	Serge	140	/**
		141	* Emits a PIPE_CONTROL with a non-zero post-sync operation, for
		142	* implementing two workarounds on gen6. From section 1.4.7.1
		143	* "PIPE_CONTROL" of the Sandy Bridge PRM volume 2 part 1:
		144	*
		145	* [DevSNB-C+{W/A}] Before any depth stall flush (including those
		146	* produced by non-pipelined state commands), software needs to first
		147	* send a PIPE_CONTROL with no bits set except Post-Sync Operation !=
		148	* 0.
		149	*
		150	* [Dev-SNB{W/A}]: Before a PIPE_CONTROL with Write Cache Flush Enable
		151	* =1, a PIPE_CONTROL with any non-zero post-sync-op is required.
		152	*
		153	* And the workaround for these two requires this workaround first:
		154	*
		155	* [Dev-SNB{W/A}]: Pipe-control with CS-stall bit set must be sent
		156	* BEFORE the pipe-control with a post-sync op and no write-cache
		157	* flushes.
		158	*
		159	* And this last workaround is tricky because of the requirements on
		160	* that bit. From section 1.4.7.2.3 "Stall" of the Sandy Bridge PRM
		161	* volume 2 part 1:
		162	*
		163	* "1 of the following must also be set:
		164	* - Render Target Cache Flush Enable ([12] of DW1)
		165	* - Depth Cache Flush Enable ([0] of DW1)
		166	* - Stall at Pixel Scoreboard ([1] of DW1)
		167	* - Depth Stall ([13] of DW1)
		168	* - Post-Sync Operation ([13] of DW1)
		169	* - Notify Enable ([8] of DW1)"
		170	*
		171	* The cache flushes require the workaround flush that triggered this
		172	* one, so we can't use it. Depth stall would trigger the same.
		173	* Post-sync nonzero is what triggered this second workaround, so we
		174	* can't use that one either. Notify enable is IRQs, which aren't
		175	* really our business. That leaves only stall at scoreboard.
		176	*/
		177	static int
		178	intel_emit_post_sync_nonzero_flush(struct intel_ring_buffer *ring)
		179	{
		180	struct pipe_control *pc = ring->private;
		181	u32 scratch_addr = pc->gtt_offset + 128;
		182	int ret;
		183
		184
		185	ret = intel_ring_begin(ring, 6);
		186	if (ret)
		187	return ret;
		188
		189	intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(5));
		190	intel_ring_emit(ring, PIPE_CONTROL_CS_STALL \|
		191	PIPE_CONTROL_STALL_AT_SCOREBOARD);
		192	intel_ring_emit(ring, scratch_addr \| PIPE_CONTROL_GLOBAL_GTT); /* address */
		193	intel_ring_emit(ring, 0); /* low dword */
		194	intel_ring_emit(ring, 0); /* high dword */
		195	intel_ring_emit(ring, MI_NOOP);
		196	intel_ring_advance(ring);
		197
		198	ret = intel_ring_begin(ring, 6);
		199	if (ret)
		200	return ret;
		201
		202	intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(5));
		203	intel_ring_emit(ring, PIPE_CONTROL_QW_WRITE);
		204	intel_ring_emit(ring, scratch_addr \| PIPE_CONTROL_GLOBAL_GTT); /* address */
		205	intel_ring_emit(ring, 0);
		206	intel_ring_emit(ring, 0);
		207	intel_ring_emit(ring, MI_NOOP);
		208	intel_ring_advance(ring);
		209
		210	return 0;
		211	}
		212
		213	static int
		214	gen6_render_ring_flush(struct intel_ring_buffer *ring,
		215	u32 invalidate_domains, u32 flush_domains)
		216	{
		217	u32 flags = 0;
		218	struct pipe_control *pc = ring->private;
		219	u32 scratch_addr = pc->gtt_offset + 128;
		220	int ret;
		221
		222	/* Force SNB workarounds for PIPE_CONTROL flushes */
3031	serge	223	ret = intel_emit_post_sync_nonzero_flush(ring);
		224	if (ret)
		225	return ret;
2342	Serge	226
		227	/* Just flush everything. Experiments have shown that reducing the
		228	* number of bits based on the write domains has little performance
		229	* impact.
		230	*/
3031	serge	231	if (flush_domains) {
		232	flags \|= PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH;
		233	flags \|= PIPE_CONTROL_DEPTH_CACHE_FLUSH;
		234	/*
		235	* Ensure that any following seqno writes only happen
		236	* when the render cache is indeed flushed.
		237	*/
		238	flags \|= PIPE_CONTROL_CS_STALL;
		239	}
		240	if (invalidate_domains) {
		241	flags \|= PIPE_CONTROL_TLB_INVALIDATE;
		242	flags \|= PIPE_CONTROL_INSTRUCTION_CACHE_INVALIDATE;
		243	flags \|= PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE;
		244	flags \|= PIPE_CONTROL_VF_CACHE_INVALIDATE;
		245	flags \|= PIPE_CONTROL_CONST_CACHE_INVALIDATE;
		246	flags \|= PIPE_CONTROL_STATE_CACHE_INVALIDATE;
		247	/*
		248	* TLB invalidate requires a post-sync write.
		249	*/
		250	flags \|= PIPE_CONTROL_QW_WRITE;
		251	}
		252
		253	ret = intel_ring_begin(ring, 4);
		254	if (ret)
		255	return ret;
		256
		257	intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(4));
		258	intel_ring_emit(ring, flags);
		259	intel_ring_emit(ring, scratch_addr \| PIPE_CONTROL_GLOBAL_GTT);
		260	intel_ring_emit(ring, 0);
		261	intel_ring_advance(ring);
		262
		263	return 0;
		264	}
		265
		266	static int
		267	gen7_render_ring_cs_stall_wa(struct intel_ring_buffer *ring)
		268	{
		269	int ret;
		270
		271	ret = intel_ring_begin(ring, 4);
		272	if (ret)
		273	return ret;
		274
		275	intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(4));
		276	intel_ring_emit(ring, PIPE_CONTROL_CS_STALL \|
		277	PIPE_CONTROL_STALL_AT_SCOREBOARD);
		278	intel_ring_emit(ring, 0);
		279	intel_ring_emit(ring, 0);
		280	intel_ring_advance(ring);
		281
		282	return 0;
		283	}
		284
		285	static int
		286	gen7_render_ring_flush(struct intel_ring_buffer *ring,
		287	u32 invalidate_domains, u32 flush_domains)
		288	{
		289	u32 flags = 0;
		290	struct pipe_control *pc = ring->private;
		291	u32 scratch_addr = pc->gtt_offset + 128;
		292	int ret;
		293
		294	/*
		295	* Ensure that any following seqno writes only happen when the render
		296	* cache is indeed flushed.
		297	*
		298	* Workaround: 4th PIPE_CONTROL command (except the ones with only
		299	* read-cache invalidate bits set) must have the CS_STALL bit set. We
		300	* don't try to be clever and just set it unconditionally.
		301	*/
		302	flags \|= PIPE_CONTROL_CS_STALL;
		303
		304	/* Just flush everything. Experiments have shown that reducing the
		305	* number of bits based on the write domains has little performance
		306	* impact.
		307	*/
		308	if (flush_domains) {
2342	Serge	309	flags \|= PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH;
3031	serge	310	flags \|= PIPE_CONTROL_DEPTH_CACHE_FLUSH;
		311	}
		312	if (invalidate_domains) {
		313	flags \|= PIPE_CONTROL_TLB_INVALIDATE;
2342	Serge	314	flags \|= PIPE_CONTROL_INSTRUCTION_CACHE_INVALIDATE;
		315	flags \|= PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE;
		316	flags \|= PIPE_CONTROL_VF_CACHE_INVALIDATE;
		317	flags \|= PIPE_CONTROL_CONST_CACHE_INVALIDATE;
		318	flags \|= PIPE_CONTROL_STATE_CACHE_INVALIDATE;
3031	serge	319	/*
		320	* TLB invalidate requires a post-sync write.
		321	*/
		322	flags \|= PIPE_CONTROL_QW_WRITE;
2342	Serge	323
3031	serge	324	/* Workaround: we must issue a pipe_control with CS-stall bit
		325	* set before a pipe_control command that has the state cache
		326	* invalidate bit set. */
		327	gen7_render_ring_cs_stall_wa(ring);
		328	}
		329
		330	ret = intel_ring_begin(ring, 4);
2342	Serge	331	if (ret)
		332	return ret;
		333
3031	serge	334	intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(4));
2342	Serge	335	intel_ring_emit(ring, flags);
		336	intel_ring_emit(ring, scratch_addr \| PIPE_CONTROL_GLOBAL_GTT);
3031	serge	337	intel_ring_emit(ring, 0);
2342	Serge	338	intel_ring_advance(ring);
		339
		340	return 0;
		341	}
		342
2332	Serge	343	static void ring_write_tail(struct intel_ring_buffer *ring,
		344	u32 value)
		345	{
		346	drm_i915_private_t *dev_priv = ring->dev->dev_private;
		347	I915_WRITE_TAIL(ring, value);
		348	}
		349
		350	u32 intel_ring_get_active_head(struct intel_ring_buffer *ring)
		351	{
		352	drm_i915_private_t *dev_priv = ring->dev->dev_private;
		353	u32 acthd_reg = INTEL_INFO(ring->dev)->gen >= 4 ?
		354	RING_ACTHD(ring->mmio_base) : ACTHD;
		355
		356	return I915_READ(acthd_reg);
		357	}
		358
		359	static int init_ring_common(struct intel_ring_buffer *ring)
		360	{
3031	serge	361	struct drm_device *dev = ring->dev;
		362	drm_i915_private_t *dev_priv = dev->dev_private;
2332	Serge	363	struct drm_i915_gem_object *obj = ring->obj;
3031	serge	364	int ret = 0;
2332	Serge	365	u32 head;
		366
3031	serge	367	if (HAS_FORCE_WAKE(dev))
		368	gen6_gt_force_wake_get(dev_priv);
		369
2332	Serge	370	/* Stop the ring if it's running. */
		371	I915_WRITE_CTL(ring, 0);
		372	I915_WRITE_HEAD(ring, 0);
		373	ring->write_tail(ring, 0);
		374
		375	head = I915_READ_HEAD(ring) & HEAD_ADDR;
		376
		377	/* G45 ring initialization fails to reset head to zero */
		378	if (head != 0) {
		379	DRM_DEBUG_KMS("%s head not reset to zero "
		380	"ctl %08x head %08x tail %08x start %08x\n",
		381	ring->name,
		382	I915_READ_CTL(ring),
		383	I915_READ_HEAD(ring),
		384	I915_READ_TAIL(ring),
		385	I915_READ_START(ring));
		386
		387	I915_WRITE_HEAD(ring, 0);
		388
		389	if (I915_READ_HEAD(ring) & HEAD_ADDR) {
		390	DRM_ERROR("failed to set %s head to zero "
		391	"ctl %08x head %08x tail %08x start %08x\n",
		392	ring->name,
		393	I915_READ_CTL(ring),
		394	I915_READ_HEAD(ring),
		395	I915_READ_TAIL(ring),
		396	I915_READ_START(ring));
		397	}
		398	}
		399
3031	serge	400	/* Initialize the ring. This must happen _after_ we've cleared the ring
		401	* registers with the above sequence (the readback of the HEAD registers
		402	* also enforces ordering), otherwise the hw might lose the new ring
		403	* register values. */
		404	I915_WRITE_START(ring, obj->gtt_offset);
2332	Serge	405	I915_WRITE_CTL(ring,
		406	((ring->size - PAGE_SIZE) & RING_NR_PAGES)
3031	serge	407	\| RING_VALID);
2332	Serge	408
		409	/* If the head is still not zero, the ring is dead */
3031	serge	410	if (wait_for((I915_READ_CTL(ring) & RING_VALID) != 0 &&
		411	I915_READ_START(ring) == obj->gtt_offset &&
		412	(I915_READ_HEAD(ring) & HEAD_ADDR) == 0, 50)) {
2332	Serge	413	DRM_ERROR("%s initialization failed "
		414	"ctl %08x head %08x tail %08x start %08x\n",
		415	ring->name,
		416	I915_READ_CTL(ring),
		417	I915_READ_HEAD(ring),
		418	I915_READ_TAIL(ring),
		419	I915_READ_START(ring));
3031	serge	420	ret = -EIO;
		421	goto out;
2332	Serge	422	}
		423
3031	serge	424	ring->head = I915_READ_HEAD(ring);
		425	ring->tail = I915_READ_TAIL(ring) & TAIL_ADDR;
		426	ring->space = ring_space(ring);
		427	ring->last_retired_head = -1;
2332	Serge	428
3031	serge	429	out:
		430	if (HAS_FORCE_WAKE(dev))
		431	gen6_gt_force_wake_put(dev_priv);
2332	Serge	432
3031	serge	433	return ret;
2332	Serge	434	}
		435
		436	static int
		437	init_pipe_control(struct intel_ring_buffer *ring)
		438	{
		439	struct pipe_control *pc;
		440	struct drm_i915_gem_object *obj;
		441	int ret;
		442
		443	if (ring->private)
		444	return 0;
		445
		446	pc = kmalloc(sizeof(*pc), GFP_KERNEL);
		447	if (!pc)
		448	return -ENOMEM;
		449
		450	obj = i915_gem_alloc_object(ring->dev, 4096);
		451	if (obj == NULL) {
		452	DRM_ERROR("Failed to allocate seqno page\n");
		453	ret = -ENOMEM;
		454	goto err;
		455	}
		456
2352	Serge	457	i915_gem_object_set_cache_level(obj, I915_CACHE_LLC);
2332	Serge	458
3031	serge	459	ret = i915_gem_object_pin(obj, 4096, true, false);
2332	Serge	460	if (ret)
		461	goto err_unref;
		462
		463	pc->gtt_offset = obj->gtt_offset;
3031	serge	464	pc->cpu_page = (void*)MapIoMem((addr_t)obj->pages.page[0], 4096, PG_SW);
2332	Serge	465	if (pc->cpu_page == NULL)
		466	goto err_unpin;
		467
		468	pc->obj = obj;
		469	ring->private = pc;
		470	return 0;
		471
		472	err_unpin:
2344	Serge	473	i915_gem_object_unpin(obj);
2332	Serge	474	err_unref:
2344	Serge	475	drm_gem_object_unreference(&obj->base);
2332	Serge	476	err:
		477	kfree(pc);
		478	return ret;
		479	}
		480
		481	static void
		482	cleanup_pipe_control(struct intel_ring_buffer *ring)
		483	{
		484	struct pipe_control *pc = ring->private;
		485	struct drm_i915_gem_object *obj;
		486
		487	if (!ring->private)
		488	return;
		489
		490	obj = pc->obj;
2339	Serge	491	// kunmap(obj->pages[0]);
2344	Serge	492	i915_gem_object_unpin(obj);
		493	drm_gem_object_unreference(&obj->base);
2332	Serge	494
		495	kfree(pc);
		496	ring->private = NULL;
		497	}
		498
		499	static int init_render_ring(struct intel_ring_buffer *ring)
		500	{
		501	struct drm_device *dev = ring->dev;
		502	struct drm_i915_private *dev_priv = dev->dev_private;
		503	int ret = init_ring_common(ring);
		504
		505	if (INTEL_INFO(dev)->gen > 3) {
3031	serge	506	I915_WRITE(MI_MODE, _MASKED_BIT_ENABLE(VS_TIMER_DISPATCH));
2332	Serge	507	if (IS_GEN7(dev))
		508	I915_WRITE(GFX_MODE_GEN7,
3031	serge	509	_MASKED_BIT_DISABLE(GFX_TLB_INVALIDATE_ALWAYS) \|
		510	_MASKED_BIT_ENABLE(GFX_REPLAY_MODE));
2332	Serge	511	}
		512
2342	Serge	513	if (INTEL_INFO(dev)->gen >= 5) {
2339	Serge	514	ret = init_pipe_control(ring);
2332	Serge	515	if (ret)
		516	return ret;
		517	}
		518
3031	serge	519	if (IS_GEN6(dev)) {
		520	/* From the Sandybridge PRM, volume 1 part 3, page 24:
		521	* "If this bit is set, STCunit will have LRA as replacement
		522	* policy. [...] This bit must be reset. LRA replacement
		523	* policy is not supported."
		524	*/
		525	I915_WRITE(CACHE_MODE_0,
		526	_MASKED_BIT_DISABLE(CM0_STC_EVICT_DISABLE_LRA_SNB));
		527
		528	/* This is not explicitly set for GEN6, so read the register.
		529	* see intel_ring_mi_set_context() for why we care.
		530	* TODO: consider explicitly setting the bit for GEN5
		531	*/
		532	ring->itlb_before_ctx_switch =
		533	!!(I915_READ(GFX_MODE) & GFX_TLB_INVALIDATE_ALWAYS);
2342	Serge	534	}
		535
3031	serge	536	if (INTEL_INFO(dev)->gen >= 6)
		537	I915_WRITE(INSTPM, _MASKED_BIT_ENABLE(INSTPM_FORCE_ORDERING));
		538
		539	if (HAS_L3_GPU_CACHE(dev))
		540	I915_WRITE_IMR(ring, ~GEN6_RENDER_L3_PARITY_ERROR);
		541
2332	Serge	542	return ret;
		543	}
		544
		545	static void render_ring_cleanup(struct intel_ring_buffer *ring)
		546	{
		547	if (!ring->private)
		548	return;
		549
		550	cleanup_pipe_control(ring);
		551	}
		552
		553	static void
2342	Serge	554	update_mboxes(struct intel_ring_buffer *ring,
		555	u32 seqno,
		556	u32 mmio_offset)
2332	Serge	557	{
2342	Serge	558	intel_ring_emit(ring, MI_SEMAPHORE_MBOX \|
		559	MI_SEMAPHORE_GLOBAL_GTT \|
2332	Serge	560	MI_SEMAPHORE_REGISTER \|
		561	MI_SEMAPHORE_UPDATE);
		562	intel_ring_emit(ring, seqno);
2342	Serge	563	intel_ring_emit(ring, mmio_offset);
2332	Serge	564	}
		565
2342	Serge	566	/**
		567	* gen6_add_request - Update the semaphore mailbox registers
		568	*
		569	* @ring - ring that is adding a request
		570	* @seqno - return seqno stuck into the ring
		571	*
		572	* Update the mailbox registers in the other rings with the current seqno.
		573	* This acts like a signal in the canonical semaphore.
		574	*/
2332	Serge	575	static int
		576	gen6_add_request(struct intel_ring_buffer *ring,
2342	Serge	577	u32 *seqno)
2332	Serge	578	{
2342	Serge	579	u32 mbox1_reg;
		580	u32 mbox2_reg;
2332	Serge	581	int ret;
		582
		583	ret = intel_ring_begin(ring, 10);
		584	if (ret)
		585	return ret;
		586
2342	Serge	587	mbox1_reg = ring->signal_mbox[0];
		588	mbox2_reg = ring->signal_mbox[1];
2332	Serge	589
3031	serge	590	*seqno = i915_gem_next_request_seqno(ring);
2342	Serge	591
		592	update_mboxes(ring, *seqno, mbox1_reg);
		593	update_mboxes(ring, *seqno, mbox2_reg);
2332	Serge	594	intel_ring_emit(ring, MI_STORE_DWORD_INDEX);
		595	intel_ring_emit(ring, I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
2342	Serge	596	intel_ring_emit(ring, *seqno);
2332	Serge	597	intel_ring_emit(ring, MI_USER_INTERRUPT);
		598	intel_ring_advance(ring);
		599
		600	return 0;
		601	}
		602
2342	Serge	603	/**
		604	* intel_ring_sync - sync the waiter to the signaller on seqno
		605	*
		606	* @waiter - ring that is waiting
		607	* @signaller - ring which has, or will signal
		608	* @seqno - seqno which the waiter will block on
		609	*/
		610	static int
3031	serge	611	gen6_ring_sync(struct intel_ring_buffer *waiter,
2342	Serge	612	struct intel_ring_buffer *signaller,
2332	Serge	613	u32 seqno)
		614	{
		615	int ret;
2342	Serge	616	u32 dw1 = MI_SEMAPHORE_MBOX \|
		617	MI_SEMAPHORE_COMPARE \|
		618	MI_SEMAPHORE_REGISTER;
2332	Serge	619
3031	serge	620	/* Throughout all of the GEM code, seqno passed implies our current
		621	* seqno is >= the last seqno executed. However for hardware the
		622	* comparison is strictly greater than.
		623	*/
		624	seqno -= 1;
		625
		626	WARN_ON(signaller->semaphore_register[waiter->id] ==
		627	MI_SEMAPHORE_SYNC_INVALID);
		628
2342	Serge	629	ret = intel_ring_begin(waiter, 4);
2332	Serge	630	if (ret)
		631	return ret;
		632
3031	serge	633	intel_ring_emit(waiter,
		634	dw1 \| signaller->semaphore_register[waiter->id]);
2342	Serge	635	intel_ring_emit(waiter, seqno);
		636	intel_ring_emit(waiter, 0);
		637	intel_ring_emit(waiter, MI_NOOP);
		638	intel_ring_advance(waiter);
2332	Serge	639
		640	return 0;
		641	}
		642
		643	#define PIPE_CONTROL_FLUSH(ring__, addr__) \
		644	do { \
2342	Serge	645	intel_ring_emit(ring__, GFX_OP_PIPE_CONTROL(4) \| PIPE_CONTROL_QW_WRITE \| \
		646	PIPE_CONTROL_DEPTH_STALL); \
2332	Serge	647	intel_ring_emit(ring__, (addr__) \| PIPE_CONTROL_GLOBAL_GTT); \
		648	intel_ring_emit(ring__, 0); \
		649	intel_ring_emit(ring__, 0); \
		650	} while (0)
		651
		652	static int
		653	pc_render_add_request(struct intel_ring_buffer *ring,
		654	u32 *result)
		655	{
3031	serge	656	u32 seqno = i915_gem_next_request_seqno(ring);
2332	Serge	657	struct pipe_control *pc = ring->private;
		658	u32 scratch_addr = pc->gtt_offset + 128;
		659	int ret;
		660
		661	/* For Ironlake, MI_USER_INTERRUPT was deprecated and apparently
		662	* incoherent with writes to memory, i.e. completely fubar,
		663	* so we need to use PIPE_NOTIFY instead.
		664	*
		665	* However, we also need to workaround the qword write
		666	* incoherence by flushing the 6 PIPE_NOTIFY buffers out to
		667	* memory before requesting an interrupt.
		668	*/
		669	ret = intel_ring_begin(ring, 32);
		670	if (ret)
		671	return ret;
		672
2342	Serge	673	intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(4) \| PIPE_CONTROL_QW_WRITE \|
		674	PIPE_CONTROL_WRITE_FLUSH \|
		675	PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE);
2332	Serge	676	intel_ring_emit(ring, pc->gtt_offset \| PIPE_CONTROL_GLOBAL_GTT);
		677	intel_ring_emit(ring, seqno);
		678	intel_ring_emit(ring, 0);
		679	PIPE_CONTROL_FLUSH(ring, scratch_addr);
		680	scratch_addr += 128; /* write to separate cachelines */
		681	PIPE_CONTROL_FLUSH(ring, scratch_addr);
		682	scratch_addr += 128;
		683	PIPE_CONTROL_FLUSH(ring, scratch_addr);
		684	scratch_addr += 128;
		685	PIPE_CONTROL_FLUSH(ring, scratch_addr);
		686	scratch_addr += 128;
		687	PIPE_CONTROL_FLUSH(ring, scratch_addr);
		688	scratch_addr += 128;
		689	PIPE_CONTROL_FLUSH(ring, scratch_addr);
3031	serge	690
2342	Serge	691	intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(4) \| PIPE_CONTROL_QW_WRITE \|
		692	PIPE_CONTROL_WRITE_FLUSH \|
		693	PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE \|
2332	Serge	694	PIPE_CONTROL_NOTIFY);
		695	intel_ring_emit(ring, pc->gtt_offset \| PIPE_CONTROL_GLOBAL_GTT);
		696	intel_ring_emit(ring, seqno);
		697	intel_ring_emit(ring, 0);
		698	intel_ring_advance(ring);
		699
		700	*result = seqno;
		701	return 0;
		702	}
		703
		704	static u32
3031	serge	705	gen6_ring_get_seqno(struct intel_ring_buffer *ring, bool lazy_coherency)
2342	Serge	706	{
		707	/* Workaround to force correct ordering between irq and seqno writes on
		708	* ivb (and maybe also on snb) by reading from a CS register (like
		709	* ACTHD) before reading the status page. */
3031	serge	710	if (!lazy_coherency)
2342	Serge	711	intel_ring_get_active_head(ring);
		712	return intel_read_status_page(ring, I915_GEM_HWS_INDEX);
		713	}
		714
		715	static u32
3031	serge	716	ring_get_seqno(struct intel_ring_buffer *ring, bool lazy_coherency)
2332	Serge	717	{
		718	return intel_read_status_page(ring, I915_GEM_HWS_INDEX);
		719	}
		720
		721	static u32
3031	serge	722	pc_render_get_seqno(struct intel_ring_buffer *ring, bool lazy_coherency)
2332	Serge	723	{
		724	struct pipe_control *pc = ring->private;
		725	return pc->cpu_page[0];
		726	}
		727
3031	serge	728	static bool
		729	gen5_ring_get_irq(struct intel_ring_buffer *ring)
2332	Serge	730	{
3031	serge	731	struct drm_device *dev = ring->dev;
		732	drm_i915_private_t *dev_priv = dev->dev_private;
		733	unsigned long flags;
		734
		735	if (!dev->irq_enabled)
		736	return false;
		737
		738	spin_lock_irqsave(&dev_priv->irq_lock, flags);
		739	if (ring->irq_refcount++ == 0) {
		740	dev_priv->gt_irq_mask &= ~ring->irq_enable_mask;
2332	Serge	741	I915_WRITE(GTIMR, dev_priv->gt_irq_mask);
		742	POSTING_READ(GTIMR);
3031	serge	743	}
		744	spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
		745
		746	return true;
2332	Serge	747	}
		748
		749	static void
3031	serge	750	gen5_ring_put_irq(struct intel_ring_buffer *ring)
2332	Serge	751	{
3031	serge	752	struct drm_device *dev = ring->dev;
		753	drm_i915_private_t *dev_priv = dev->dev_private;
		754	unsigned long flags;
		755
		756	spin_lock_irqsave(&dev_priv->irq_lock, flags);
		757	if (--ring->irq_refcount == 0) {
		758	dev_priv->gt_irq_mask \|= ring->irq_enable_mask;
2332	Serge	759	I915_WRITE(GTIMR, dev_priv->gt_irq_mask);
		760	POSTING_READ(GTIMR);
3031	serge	761	}
		762	spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
2332	Serge	763	}
		764
3031	serge	765	static bool
		766	i9xx_ring_get_irq(struct intel_ring_buffer *ring)
2332	Serge	767	{
3031	serge	768	struct drm_device *dev = ring->dev;
		769	drm_i915_private_t *dev_priv = dev->dev_private;
		770	unsigned long flags;
		771
		772	if (!dev->irq_enabled)
		773	return false;
		774
		775	spin_lock_irqsave(&dev_priv->irq_lock, flags);
		776	if (ring->irq_refcount++ == 0) {
		777	dev_priv->irq_mask &= ~ring->irq_enable_mask;
2332	Serge	778	I915_WRITE(IMR, dev_priv->irq_mask);
		779	POSTING_READ(IMR);
3031	serge	780	}
		781	spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
		782
		783	return true;
2332	Serge	784	}
		785
		786	static void
3031	serge	787	i9xx_ring_put_irq(struct intel_ring_buffer *ring)
2332	Serge	788	{
3031	serge	789	struct drm_device *dev = ring->dev;
		790	drm_i915_private_t *dev_priv = dev->dev_private;
		791	unsigned long flags;
		792
		793	spin_lock_irqsave(&dev_priv->irq_lock, flags);
		794	if (--ring->irq_refcount == 0) {
		795	dev_priv->irq_mask \|= ring->irq_enable_mask;
2332	Serge	796	I915_WRITE(IMR, dev_priv->irq_mask);
		797	POSTING_READ(IMR);
3031	serge	798	}
		799	spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
2332	Serge	800	}
		801
		802	static bool
3031	serge	803	i8xx_ring_get_irq(struct intel_ring_buffer *ring)
2332	Serge	804	{
		805	struct drm_device *dev = ring->dev;
		806	drm_i915_private_t *dev_priv = dev->dev_private;
3031	serge	807	unsigned long flags;
2332	Serge	808
		809	if (!dev->irq_enabled)
		810	return false;
		811
3031	serge	812	spin_lock_irqsave(&dev_priv->irq_lock, flags);
2332	Serge	813	if (ring->irq_refcount++ == 0) {
3031	serge	814	dev_priv->irq_mask &= ~ring->irq_enable_mask;
		815	I915_WRITE16(IMR, dev_priv->irq_mask);
		816	POSTING_READ16(IMR);
2332	Serge	817	}
3031	serge	818	spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
2332	Serge	819
		820	return true;
		821	}
		822
		823	static void
3031	serge	824	i8xx_ring_put_irq(struct intel_ring_buffer *ring)
2332	Serge	825	{
		826	struct drm_device *dev = ring->dev;
		827	drm_i915_private_t *dev_priv = dev->dev_private;
3031	serge	828	unsigned long flags;
2332	Serge	829
3031	serge	830	spin_lock_irqsave(&dev_priv->irq_lock, flags);
2332	Serge	831	if (--ring->irq_refcount == 0) {
3031	serge	832	dev_priv->irq_mask \|= ring->irq_enable_mask;
		833	I915_WRITE16(IMR, dev_priv->irq_mask);
		834	POSTING_READ16(IMR);
2332	Serge	835	}
3031	serge	836	spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
2332	Serge	837	}
		838
		839	void intel_ring_setup_status_page(struct intel_ring_buffer *ring)
		840	{
		841	struct drm_device *dev = ring->dev;
		842	drm_i915_private_t *dev_priv = ring->dev->dev_private;
		843	u32 mmio = 0;
		844
		845	/* The ring status page addresses are no longer next to the rest of
		846	* the ring registers as of gen7.
		847	*/
		848	if (IS_GEN7(dev)) {
		849	switch (ring->id) {
3031	serge	850	case RCS:
2332	Serge	851	mmio = RENDER_HWS_PGA_GEN7;
		852	break;
3031	serge	853	case BCS:
2332	Serge	854	mmio = BLT_HWS_PGA_GEN7;
		855	break;
3031	serge	856	case VCS:
2332	Serge	857	mmio = BSD_HWS_PGA_GEN7;
		858	break;
		859	}
		860	} else if (IS_GEN6(ring->dev)) {
		861	mmio = RING_HWS_PGA_GEN6(ring->mmio_base);
		862	} else {
		863	mmio = RING_HWS_PGA(ring->mmio_base);
		864	}
		865
		866	I915_WRITE(mmio, (u32)ring->status_page.gfx_addr);
		867	POSTING_READ(mmio);
		868	}
		869
		870	static int
		871	bsd_ring_flush(struct intel_ring_buffer *ring,
		872	u32 invalidate_domains,
		873	u32 flush_domains)
		874	{
		875	int ret;
		876
		877	ret = intel_ring_begin(ring, 2);
		878	if (ret)
		879	return ret;
		880
		881	intel_ring_emit(ring, MI_FLUSH);
		882	intel_ring_emit(ring, MI_NOOP);
		883	intel_ring_advance(ring);
		884	return 0;
		885	}
		886
		887	static int
3031	serge	888	i9xx_add_request(struct intel_ring_buffer *ring,
2332	Serge	889	u32 *result)
		890	{
		891	u32 seqno;
		892	int ret;
		893
		894	ret = intel_ring_begin(ring, 4);
		895	if (ret)
		896	return ret;
		897
3031	serge	898	seqno = i915_gem_next_request_seqno(ring);
2332	Serge	899
		900	intel_ring_emit(ring, MI_STORE_DWORD_INDEX);
		901	intel_ring_emit(ring, I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
		902	intel_ring_emit(ring, seqno);
		903	intel_ring_emit(ring, MI_USER_INTERRUPT);
		904	intel_ring_advance(ring);
		905
		906	*result = seqno;
		907	return 0;
		908	}
		909
		910	static bool
3031	serge	911	gen6_ring_get_irq(struct intel_ring_buffer *ring)
2332	Serge	912	{
		913	struct drm_device *dev = ring->dev;
		914	drm_i915_private_t *dev_priv = dev->dev_private;
3031	serge	915	unsigned long flags;
2332	Serge	916
		917	if (!dev->irq_enabled)
		918	return false;
		919
2342	Serge	920	/* It looks like we need to prevent the gt from suspending while waiting
		921	* for an notifiy irq, otherwise irqs seem to get lost on at least the
		922	* blt/bsd rings on ivb. */
		923	gen6_gt_force_wake_get(dev_priv);
		924
3031	serge	925	spin_lock_irqsave(&dev_priv->irq_lock, flags);
2332	Serge	926	if (ring->irq_refcount++ == 0) {
3031	serge	927	if (HAS_L3_GPU_CACHE(dev) && ring->id == RCS)
		928	I915_WRITE_IMR(ring, ~(ring->irq_enable_mask \|
		929	GEN6_RENDER_L3_PARITY_ERROR));
		930	else
		931	I915_WRITE_IMR(ring, ~ring->irq_enable_mask);
		932	dev_priv->gt_irq_mask &= ~ring->irq_enable_mask;
		933	I915_WRITE(GTIMR, dev_priv->gt_irq_mask);
		934	POSTING_READ(GTIMR);
2332	Serge	935	}
3031	serge	936	spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
2332	Serge	937
2351	Serge	938	return true;
2332	Serge	939	}
		940
		941	static void
3031	serge	942	gen6_ring_put_irq(struct intel_ring_buffer *ring)
2332	Serge	943	{
		944	struct drm_device *dev = ring->dev;
		945	drm_i915_private_t *dev_priv = dev->dev_private;
3031	serge	946	unsigned long flags;
2332	Serge	947
3031	serge	948	spin_lock_irqsave(&dev_priv->irq_lock, flags);
2332	Serge	949	if (--ring->irq_refcount == 0) {
3031	serge	950	if (HAS_L3_GPU_CACHE(dev) && ring->id == RCS)
		951	I915_WRITE_IMR(ring, ~GEN6_RENDER_L3_PARITY_ERROR);
		952	else
		953	I915_WRITE_IMR(ring, ~0);
		954	dev_priv->gt_irq_mask \|= ring->irq_enable_mask;
		955	I915_WRITE(GTIMR, dev_priv->gt_irq_mask);
		956	POSTING_READ(GTIMR);
2332	Serge	957	}
3031	serge	958	spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
2342	Serge	959
		960	gen6_gt_force_wake_put(dev_priv);
2332	Serge	961	}
		962
		963	static int
3031	serge	964	i965_dispatch_execbuffer(struct intel_ring_buffer *ring, u32 offset, u32 length)
2332	Serge	965	{
		966	int ret;
		967
		968	ret = intel_ring_begin(ring, 2);
		969	if (ret)
		970	return ret;
		971
		972	intel_ring_emit(ring,
3031	serge	973	MI_BATCH_BUFFER_START \|
		974	MI_BATCH_GTT \|
2332	Serge	975	MI_BATCH_NON_SECURE_I965);
		976	intel_ring_emit(ring, offset);
		977	intel_ring_advance(ring);
		978
		979	return 0;
		980	}
		981
		982	static int
3031	serge	983	i830_dispatch_execbuffer(struct intel_ring_buffer *ring,
2332	Serge	984	u32 offset, u32 len)
		985	{
		986	int ret;
		987
		988	ret = intel_ring_begin(ring, 4);
		989	if (ret)
		990	return ret;
		991
		992	intel_ring_emit(ring, MI_BATCH_BUFFER);
		993	intel_ring_emit(ring, offset \| MI_BATCH_NON_SECURE);
		994	intel_ring_emit(ring, offset + len - 8);
		995	intel_ring_emit(ring, 0);
3031	serge	996	intel_ring_advance(ring);
		997
		998	return 0;
		999	}
		1000
		1001	static int
		1002	i915_dispatch_execbuffer(struct intel_ring_buffer *ring,
		1003	u32 offset, u32 len)
		1004	{
		1005	int ret;
		1006
2332	Serge	1007	ret = intel_ring_begin(ring, 2);
		1008	if (ret)
		1009	return ret;
		1010
3031	serge	1011	intel_ring_emit(ring, MI_BATCH_BUFFER_START \| MI_BATCH_GTT);
2332	Serge	1012	intel_ring_emit(ring, offset \| MI_BATCH_NON_SECURE);
		1013	intel_ring_advance(ring);
		1014
		1015	return 0;
		1016	}
		1017
		1018	static void cleanup_status_page(struct intel_ring_buffer *ring)
		1019	{
		1020	struct drm_i915_gem_object *obj;
		1021
		1022	obj = ring->status_page.obj;
		1023	if (obj == NULL)
		1024	return;
		1025
3031	serge	1026	// kunmap(sg_page(obj->pages->sgl));
2344	Serge	1027	i915_gem_object_unpin(obj);
		1028	drm_gem_object_unreference(&obj->base);
2332	Serge	1029	ring->status_page.obj = NULL;
		1030	}
		1031
		1032	static int init_status_page(struct intel_ring_buffer *ring)
		1033	{
		1034	struct drm_device *dev = ring->dev;
		1035	struct drm_i915_gem_object *obj;
		1036	int ret;
		1037
		1038	obj = i915_gem_alloc_object(dev, 4096);
		1039	if (obj == NULL) {
		1040	DRM_ERROR("Failed to allocate status page\n");
		1041	ret = -ENOMEM;
		1042	goto err;
		1043	}
		1044
2352	Serge	1045	i915_gem_object_set_cache_level(obj, I915_CACHE_LLC);
2332	Serge	1046
3031	serge	1047	ret = i915_gem_object_pin(obj, 4096, true, false);
2332	Serge	1048	if (ret != 0) {
		1049	goto err_unref;
		1050	}
		1051
		1052	ring->status_page.gfx_addr = obj->gtt_offset;
3031	serge	1053	ring->status_page.page_addr = (void*)MapIoMem(obj->pages.page[0],4096,PG_SW);
2332	Serge	1054	if (ring->status_page.page_addr == NULL) {
3031	serge	1055	ret = -ENOMEM;
2332	Serge	1056	goto err_unpin;
		1057	}
		1058	ring->status_page.obj = obj;
		1059	memset(ring->status_page.page_addr, 0, PAGE_SIZE);
		1060
		1061	intel_ring_setup_status_page(ring);
		1062	DRM_DEBUG_DRIVER("%s hws offset: 0x%08x\n",
		1063	ring->name, ring->status_page.gfx_addr);
		1064
		1065	return 0;
		1066
		1067	err_unpin:
2344	Serge	1068	i915_gem_object_unpin(obj);
2332	Serge	1069	err_unref:
2344	Serge	1070	drm_gem_object_unreference(&obj->base);
2332	Serge	1071	err:
		1072	return ret;
		1073	}
		1074
3031	serge	1075	static int intel_init_ring_buffer(struct drm_device *dev,
2332	Serge	1076	struct intel_ring_buffer *ring)
		1077	{
2340	Serge	1078	struct drm_i915_gem_object *obj;
3031	serge	1079	struct drm_i915_private *dev_priv = dev->dev_private;
2332	Serge	1080	int ret;
2340	Serge	1081
2332	Serge	1082	ring->dev = dev;
		1083	INIT_LIST_HEAD(&ring->active_list);
		1084	INIT_LIST_HEAD(&ring->request_list);
3031	serge	1085	ring->size = 32 * PAGE_SIZE;
2332	Serge	1086
2352	Serge	1087	init_waitqueue_head(&ring->irq_queue);
2332	Serge	1088
		1089	if (I915_NEED_GFX_HWS(dev)) {
2340	Serge	1090	ret = init_status_page(ring);
		1091	if (ret)
		1092	return ret;
2332	Serge	1093	}
		1094
		1095	obj = i915_gem_alloc_object(dev, ring->size);
		1096	if (obj == NULL) {
		1097	DRM_ERROR("Failed to allocate ringbuffer\n");
		1098	ret = -ENOMEM;
		1099	goto err_hws;
		1100	}
		1101
		1102	ring->obj = obj;
		1103
3031	serge	1104	ret = i915_gem_object_pin(obj, PAGE_SIZE, true, false);
2332	Serge	1105	if (ret)
		1106	goto err_unref;
		1107
3031	serge	1108	ret = i915_gem_object_set_to_gtt_domain(obj, true);
		1109	if (ret)
		1110	goto err_unpin;
2332	Serge	1111
3031	serge	1112	ring->virtual_start =
		1113	ioremap(dev_priv->mm.gtt->gma_bus_addr + obj->gtt_offset,
		1114	ring->size);
		1115	if (ring->virtual_start == NULL) {
2332	Serge	1116	DRM_ERROR("Failed to map ringbuffer.\n");
		1117	ret = -EINVAL;
		1118	goto err_unpin;
		1119	}
		1120
		1121	ret = ring->init(ring);
		1122	if (ret)
		1123	goto err_unmap;
		1124
		1125	/* Workaround an erratum on the i830 which causes a hang if
		1126	* the TAIL pointer points to within the last 2 cachelines
		1127	* of the buffer.
		1128	*/
		1129	ring->effective_size = ring->size;
3031	serge	1130	if (IS_I830(ring->dev) \|\| IS_845G(ring->dev))
2332	Serge	1131	ring->effective_size -= 128;
2340	Serge	1132
2332	Serge	1133	return 0;
		1134
		1135	err_unmap:
		1136	FreeKernelSpace(ring->virtual_start);
		1137	err_unpin:
2344	Serge	1138	i915_gem_object_unpin(obj);
2332	Serge	1139	err_unref:
2344	Serge	1140	drm_gem_object_unreference(&obj->base);
2332	Serge	1141	ring->obj = NULL;
		1142	err_hws:
		1143	// cleanup_status_page(ring);
		1144	return ret;
		1145	}
		1146
		1147	void intel_cleanup_ring_buffer(struct intel_ring_buffer *ring)
		1148	{
		1149	struct drm_i915_private *dev_priv;
		1150	int ret;
		1151
		1152	if (ring->obj == NULL)
		1153	return;
		1154
		1155	/* Disable the ring buffer. The ring must be idle at this point */
		1156	dev_priv = ring->dev->dev_private;
		1157	ret = intel_wait_ring_idle(ring);
		1158	if (ret)
		1159	DRM_ERROR("failed to quiesce %s whilst cleaning up: %d\n",
		1160	ring->name, ret);
		1161
		1162	I915_WRITE_CTL(ring, 0);
		1163
		1164	// drm_core_ioremapfree(&ring->map, ring->dev);
		1165
2344	Serge	1166	i915_gem_object_unpin(ring->obj);
		1167	drm_gem_object_unreference(&ring->obj->base);
2332	Serge	1168	ring->obj = NULL;
		1169
		1170	if (ring->cleanup)
		1171	ring->cleanup(ring);
		1172
		1173	// cleanup_status_page(ring);
		1174	}
		1175
		1176	static int intel_wrap_ring_buffer(struct intel_ring_buffer *ring)
		1177	{
3031	serge	1178	uint32_t __iomem *virt;
2332	Serge	1179	int rem = ring->size - ring->tail;
		1180
2340	Serge	1181	ENTER();
		1182
2332	Serge	1183	if (ring->space < rem) {
		1184	int ret = intel_wait_ring_buffer(ring, rem);
		1185	if (ret)
		1186	return ret;
		1187	}
		1188
3031	serge	1189	virt = ring->virtual_start + ring->tail;
		1190	rem /= 4;
		1191	while (rem--)
		1192	iowrite32(MI_NOOP, virt++);
2332	Serge	1193
		1194	ring->tail = 0;
		1195	ring->space = ring_space(ring);
		1196
2340	Serge	1197	LEAVE();
2332	Serge	1198	return 0;
		1199	}
		1200
3031	serge	1201	static int intel_ring_wait_seqno(struct intel_ring_buffer *ring, u32 seqno)
2332	Serge	1202	{
3031	serge	1203	int ret;
2332	Serge	1204
3031	serge	1205	ret = i915_wait_seqno(ring, seqno);
		1206	if (!ret)
		1207	i915_gem_retire_requests_ring(ring);
		1208
		1209	return ret;
		1210	}
		1211
		1212	static int intel_ring_wait_request(struct intel_ring_buffer *ring, int n)
		1213	{
		1214	struct drm_i915_gem_request *request;
		1215	u32 seqno = 0;
		1216	int ret;
		1217
		1218	i915_gem_retire_requests_ring(ring);
		1219
		1220	if (ring->last_retired_head != -1) {
		1221	ring->head = ring->last_retired_head;
		1222	ring->last_retired_head = -1;
2332	Serge	1223	ring->space = ring_space(ring);
		1224	if (ring->space >= n)
		1225	return 0;
		1226	}
		1227
3031	serge	1228	list_for_each_entry(request, &ring->request_list, list) {
		1229	int space;
2344	Serge	1230
3031	serge	1231	if (request->tail == -1)
		1232	continue;
		1233
		1234	space = request->tail - (ring->tail + 8);
		1235	if (space < 0)
		1236	space += ring->size;
		1237	if (space >= n) {
		1238	seqno = request->seqno;
		1239	break;
		1240	}
		1241
		1242	/* Consume this request in case we need more space than
		1243	* is available and so need to prevent a race between
		1244	* updating last_retired_head and direct reads of
		1245	* I915_RING_HEAD. It also provides a nice sanity check.
		1246	*/
		1247	request->tail = -1;
		1248	}
		1249
		1250	if (seqno == 0)
		1251	return -ENOSPC;
		1252
		1253	ret = intel_ring_wait_seqno(ring, seqno);
		1254	if (ret)
		1255	return ret;
		1256
		1257	if (WARN_ON(ring->last_retired_head == -1))
		1258	return -ENOSPC;
		1259
		1260	ring->head = ring->last_retired_head;
		1261	ring->last_retired_head = -1;
		1262	ring->space = ring_space(ring);
		1263	if (WARN_ON(ring->space < n))
		1264	return -ENOSPC;
		1265
		1266	return 0;
		1267	}
		1268
		1269	int intel_wait_ring_buffer(struct intel_ring_buffer *ring, int n)
		1270	{
		1271	struct drm_device *dev = ring->dev;
		1272	struct drm_i915_private *dev_priv = dev->dev_private;
		1273	unsigned long end;
		1274	int ret;
		1275
		1276	ret = intel_ring_wait_request(ring, n);
		1277	if (ret != -ENOSPC)
		1278	return ret;
		1279
		1280
		1281	/* With GEM the hangcheck timer should kick us out of the loop,
		1282	* leaving it early runs the risk of corrupting GEM state (due
		1283	* to running on almost untested codepaths). But on resume
		1284	* timers don't work yet, so prevent a complete hang in that
		1285	* case by choosing an insanely large timeout. */
		1286	end = GetTimerTicks() + 60 * HZ;
		1287
2332	Serge	1288	do {
		1289	ring->head = I915_READ_HEAD(ring);
		1290	ring->space = ring_space(ring);
		1291	if (ring->space >= n) {
2351	Serge	1292	trace_i915_ring_wait_end(ring);
2332	Serge	1293	return 0;
		1294	}
		1295
		1296	msleep(1);
3031	serge	1297
		1298	ret = i915_gem_check_wedge(dev_priv, dev_priv->mm.interruptible);
		1299	if (ret)
		1300	return ret;
		1301	} while (!time_after(GetTimerTicks(), end));
2351	Serge	1302	trace_i915_ring_wait_end(ring);
2332	Serge	1303	return -EBUSY;
		1304	}
		1305
		1306	int intel_ring_begin(struct intel_ring_buffer *ring,
		1307	int num_dwords)
		1308	{
3031	serge	1309	drm_i915_private_t *dev_priv = ring->dev->dev_private;
2332	Serge	1310	int n = 4*num_dwords;
		1311	int ret;
		1312
3031	serge	1313	ret = i915_gem_check_wedge(dev_priv, dev_priv->mm.interruptible);
		1314	if (ret)
		1315	return ret;
2332	Serge	1316
		1317	if (unlikely(ring->tail + n > ring->effective_size)) {
		1318	ret = intel_wrap_ring_buffer(ring);
		1319	if (unlikely(ret))
		1320	return ret;
		1321	}
		1322
		1323	if (unlikely(ring->space < n)) {
		1324	ret = intel_wait_ring_buffer(ring, n);
		1325	if (unlikely(ret))
		1326	return ret;
		1327	}
		1328
		1329	ring->space -= n;
		1330	return 0;
		1331	}
		1332
		1333	void intel_ring_advance(struct intel_ring_buffer *ring)
		1334	{
3031	serge	1335	struct drm_i915_private *dev_priv = ring->dev->dev_private;
		1336
2332	Serge	1337	ring->tail &= ring->size - 1;
3031	serge	1338	if (dev_priv->stop_rings & intel_ring_flag(ring))
		1339	return;
2332	Serge	1340	ring->write_tail(ring, ring->tail);
		1341	}
		1342
		1343
		1344	static void gen6_bsd_ring_write_tail(struct intel_ring_buffer *ring,
		1345	u32 value)
		1346	{
		1347	drm_i915_private_t *dev_priv = ring->dev->dev_private;
		1348
		1349	/* Every tail move must follow the sequence below */
3031	serge	1350
		1351	/* Disable notification that the ring is IDLE. The GT
		1352	* will then assume that it is busy and bring it out of rc6.
		1353	*/
2332	Serge	1354	I915_WRITE(GEN6_BSD_SLEEP_PSMI_CONTROL,
3031	serge	1355	_MASKED_BIT_ENABLE(GEN6_BSD_SLEEP_MSG_DISABLE));
2332	Serge	1356
3031	serge	1357	/* Clear the context id. Here be magic! */
		1358	I915_WRITE64(GEN6_BSD_RNCID, 0x0);
		1359
		1360	/* Wait for the ring not to be idle, i.e. for it to wake up. */
2332	Serge	1361	if (wait_for((I915_READ(GEN6_BSD_SLEEP_PSMI_CONTROL) &
3031	serge	1362	GEN6_BSD_SLEEP_INDICATOR) == 0,
2332	Serge	1363	50))
3031	serge	1364	DRM_ERROR("timed out waiting for the BSD ring to wake up\n");
2332	Serge	1365
3031	serge	1366	/* Now that the ring is fully powered up, update the tail */
2332	Serge	1367	I915_WRITE_TAIL(ring, value);
3031	serge	1368	POSTING_READ(RING_TAIL(ring->mmio_base));
		1369
		1370	/* Let the ring send IDLE messages to the GT again,
		1371	* and so let it sleep to conserve power when idle.
		1372	*/
2332	Serge	1373	I915_WRITE(GEN6_BSD_SLEEP_PSMI_CONTROL,
3031	serge	1374	_MASKED_BIT_DISABLE(GEN6_BSD_SLEEP_MSG_DISABLE));
2332	Serge	1375	}
		1376
		1377	static int gen6_ring_flush(struct intel_ring_buffer *ring,
		1378	u32 invalidate, u32 flush)
		1379	{
		1380	uint32_t cmd;
		1381	int ret;
		1382
		1383	ret = intel_ring_begin(ring, 4);
		1384	if (ret)
		1385	return ret;
		1386
		1387	cmd = MI_FLUSH_DW;
		1388	if (invalidate & I915_GEM_GPU_DOMAINS)
		1389	cmd \|= MI_INVALIDATE_TLB \| MI_INVALIDATE_BSD;
		1390	intel_ring_emit(ring, cmd);
		1391	intel_ring_emit(ring, 0);
		1392	intel_ring_emit(ring, 0);
		1393	intel_ring_emit(ring, MI_NOOP);
		1394	intel_ring_advance(ring);
		1395	return 0;
		1396	}
		1397
		1398	static int
		1399	gen6_ring_dispatch_execbuffer(struct intel_ring_buffer *ring,
		1400	u32 offset, u32 len)
		1401	{
		1402	int ret;
		1403
		1404	ret = intel_ring_begin(ring, 2);
		1405	if (ret)
		1406	return ret;
		1407
		1408	intel_ring_emit(ring, MI_BATCH_BUFFER_START \| MI_BATCH_NON_SECURE_I965);
		1409	/* bit0-7 is the length on GEN6+ */
		1410	intel_ring_emit(ring, offset);
		1411	intel_ring_advance(ring);
		1412
		1413	return 0;
		1414	}
		1415
		1416	/* Blitter support (SandyBridge+) */
		1417
		1418	static int blt_ring_flush(struct intel_ring_buffer *ring,
		1419	u32 invalidate, u32 flush)
		1420	{
		1421	uint32_t cmd;
		1422	int ret;
		1423
3031	serge	1424	ret = intel_ring_begin(ring, 4);
2332	Serge	1425	if (ret)
		1426	return ret;
		1427
		1428	cmd = MI_FLUSH_DW;
		1429	if (invalidate & I915_GEM_DOMAIN_RENDER)
		1430	cmd \|= MI_INVALIDATE_TLB;
		1431	intel_ring_emit(ring, cmd);
		1432	intel_ring_emit(ring, 0);
		1433	intel_ring_emit(ring, 0);
		1434	intel_ring_emit(ring, MI_NOOP);
		1435	intel_ring_advance(ring);
		1436	return 0;
		1437	}
		1438
		1439	int intel_init_render_ring_buffer(struct drm_device *dev)
		1440	{
		1441	drm_i915_private_t *dev_priv = dev->dev_private;
		1442	struct intel_ring_buffer *ring = &dev_priv->ring[RCS];
2340	Serge	1443
3031	serge	1444	ring->name = "render ring";
		1445	ring->id = RCS;
		1446	ring->mmio_base = RENDER_RING_BASE;
		1447
2332	Serge	1448	if (INTEL_INFO(dev)->gen >= 6) {
2339	Serge	1449	ring->add_request = gen6_add_request;
3031	serge	1450	ring->flush = gen7_render_ring_flush;
		1451	if (INTEL_INFO(dev)->gen == 6)
2342	Serge	1452	ring->flush = gen6_render_ring_flush;
3031	serge	1453	ring->irq_get = gen6_ring_get_irq;
		1454	ring->irq_put = gen6_ring_put_irq;
		1455	ring->irq_enable_mask = GT_USER_INTERRUPT;
2342	Serge	1456	ring->get_seqno = gen6_ring_get_seqno;
3031	serge	1457	ring->sync_to = gen6_ring_sync;
		1458	ring->semaphore_register[0] = MI_SEMAPHORE_SYNC_INVALID;
		1459	ring->semaphore_register[1] = MI_SEMAPHORE_SYNC_RV;
		1460	ring->semaphore_register[2] = MI_SEMAPHORE_SYNC_RB;
		1461	ring->signal_mbox[0] = GEN6_VRSYNC;
		1462	ring->signal_mbox[1] = GEN6_BRSYNC;
2332	Serge	1463	} else if (IS_GEN5(dev)) {
2339	Serge	1464	ring->add_request = pc_render_add_request;
3031	serge	1465	ring->flush = gen4_render_ring_flush;
2342	Serge	1466	ring->get_seqno = pc_render_get_seqno;
3031	serge	1467	ring->irq_get = gen5_ring_get_irq;
		1468	ring->irq_put = gen5_ring_put_irq;
		1469	ring->irq_enable_mask = GT_USER_INTERRUPT \| GT_PIPE_NOTIFY;
		1470	} else {
		1471	ring->add_request = i9xx_add_request;
		1472	if (INTEL_INFO(dev)->gen < 4)
		1473	ring->flush = gen2_render_ring_flush;
		1474	else
		1475	ring->flush = gen4_render_ring_flush;
		1476	ring->get_seqno = ring_get_seqno;
		1477	if (IS_GEN2(dev)) {
		1478	ring->irq_get = i8xx_ring_get_irq;
		1479	ring->irq_put = i8xx_ring_put_irq;
		1480	} else {
		1481	ring->irq_get = i9xx_ring_get_irq;
		1482	ring->irq_put = i9xx_ring_put_irq;
		1483	}
		1484	ring->irq_enable_mask = I915_USER_INTERRUPT;
2332	Serge	1485	}
3031	serge	1486	ring->write_tail = ring_write_tail;
		1487	if (INTEL_INFO(dev)->gen >= 6)
		1488	ring->dispatch_execbuffer = gen6_ring_dispatch_execbuffer;
		1489	else if (INTEL_INFO(dev)->gen >= 4)
		1490	ring->dispatch_execbuffer = i965_dispatch_execbuffer;
		1491	else if (IS_I830(dev) \|\| IS_845G(dev))
		1492	ring->dispatch_execbuffer = i830_dispatch_execbuffer;
		1493	else
		1494	ring->dispatch_execbuffer = i915_dispatch_execbuffer;
		1495	ring->init = init_render_ring;
		1496	ring->cleanup = render_ring_cleanup;
2332	Serge	1497
3031	serge	1498
2332	Serge	1499	if (!I915_NEED_GFX_HWS(dev)) {
		1500	ring->status_page.page_addr = dev_priv->status_page_dmah->vaddr;
		1501	memset(ring->status_page.page_addr, 0, PAGE_SIZE);
		1502	}
2340	Serge	1503
2332	Serge	1504	return intel_init_ring_buffer(dev, ring);
		1505	}
		1506
		1507
		1508	int intel_init_bsd_ring_buffer(struct drm_device *dev)
		1509	{
		1510	drm_i915_private_t *dev_priv = dev->dev_private;
		1511	struct intel_ring_buffer *ring = &dev_priv->ring[VCS];
		1512
3031	serge	1513	ring->name = "bsd ring";
		1514	ring->id = VCS;
2332	Serge	1515
3031	serge	1516	ring->write_tail = ring_write_tail;
		1517	if (IS_GEN6(dev) \|\| IS_GEN7(dev)) {
		1518	ring->mmio_base = GEN6_BSD_RING_BASE;
		1519	/* gen6 bsd needs a special wa for tail updates */
		1520	if (IS_GEN6(dev))
		1521	ring->write_tail = gen6_bsd_ring_write_tail;
		1522	ring->flush = gen6_ring_flush;
		1523	ring->add_request = gen6_add_request;
		1524	ring->get_seqno = gen6_ring_get_seqno;
		1525	ring->irq_enable_mask = GEN6_BSD_USER_INTERRUPT;
		1526	ring->irq_get = gen6_ring_get_irq;
		1527	ring->irq_put = gen6_ring_put_irq;
		1528	ring->dispatch_execbuffer = gen6_ring_dispatch_execbuffer;
		1529	ring->sync_to = gen6_ring_sync;
		1530	ring->semaphore_register[0] = MI_SEMAPHORE_SYNC_VR;
		1531	ring->semaphore_register[1] = MI_SEMAPHORE_SYNC_INVALID;
		1532	ring->semaphore_register[2] = MI_SEMAPHORE_SYNC_VB;
		1533	ring->signal_mbox[0] = GEN6_RVSYNC;
		1534	ring->signal_mbox[1] = GEN6_BVSYNC;
		1535	} else {
		1536	ring->mmio_base = BSD_RING_BASE;
		1537	ring->flush = bsd_ring_flush;
		1538	ring->add_request = i9xx_add_request;
		1539	ring->get_seqno = ring_get_seqno;
		1540	if (IS_GEN5(dev)) {
		1541	ring->irq_enable_mask = GT_BSD_USER_INTERRUPT;
		1542	ring->irq_get = gen5_ring_get_irq;
		1543	ring->irq_put = gen5_ring_put_irq;
		1544	} else {
		1545	ring->irq_enable_mask = I915_BSD_USER_INTERRUPT;
		1546	ring->irq_get = i9xx_ring_get_irq;
		1547	ring->irq_put = i9xx_ring_put_irq;
		1548	}
		1549	ring->dispatch_execbuffer = i965_dispatch_execbuffer;
		1550	}
		1551	ring->init = init_ring_common;
		1552
		1553
2332	Serge	1554	return intel_init_ring_buffer(dev, ring);
		1555	}
		1556
		1557	int intel_init_blt_ring_buffer(struct drm_device *dev)
		1558	{
		1559	drm_i915_private_t *dev_priv = dev->dev_private;
		1560	struct intel_ring_buffer *ring = &dev_priv->ring[BCS];
		1561
3031	serge	1562	ring->name = "blitter ring";
		1563	ring->id = BCS;
2332	Serge	1564
3031	serge	1565	ring->mmio_base = BLT_RING_BASE;
		1566	ring->write_tail = ring_write_tail;
		1567	ring->flush = blt_ring_flush;
		1568	ring->add_request = gen6_add_request;
		1569	ring->get_seqno = gen6_ring_get_seqno;
		1570	ring->irq_enable_mask = GEN6_BLITTER_USER_INTERRUPT;
		1571	ring->irq_get = gen6_ring_get_irq;
		1572	ring->irq_put = gen6_ring_put_irq;
		1573	ring->dispatch_execbuffer = gen6_ring_dispatch_execbuffer;
		1574	ring->sync_to = gen6_ring_sync;
		1575	ring->semaphore_register[0] = MI_SEMAPHORE_SYNC_BR;
		1576	ring->semaphore_register[1] = MI_SEMAPHORE_SYNC_BV;
		1577	ring->semaphore_register[2] = MI_SEMAPHORE_SYNC_INVALID;
		1578	ring->signal_mbox[0] = GEN6_RBSYNC;
		1579	ring->signal_mbox[1] = GEN6_VBSYNC;
		1580	ring->init = init_ring_common;
		1581
2332	Serge	1582	return intel_init_ring_buffer(dev, ring);
		1583	}
3031	serge	1584
		1585	int
		1586	intel_ring_flush_all_caches(struct intel_ring_buffer *ring)
		1587	{
		1588	int ret;
		1589
		1590	if (!ring->gpu_caches_dirty)
		1591	return 0;
		1592
		1593	ret = ring->flush(ring, 0, I915_GEM_GPU_DOMAINS);
		1594	if (ret)
		1595	return ret;
		1596
		1597	trace_i915_gem_ring_flush(ring, 0, I915_GEM_GPU_DOMAINS);
		1598
		1599	ring->gpu_caches_dirty = false;
		1600	return 0;
		1601	}
		1602
		1603	int
		1604	intel_ring_invalidate_all_caches(struct intel_ring_buffer *ring)
		1605	{
		1606	uint32_t flush_domains;
		1607	int ret;
		1608
		1609	flush_domains = 0;
		1610	if (ring->gpu_caches_dirty)
		1611	flush_domains = I915_GEM_GPU_DOMAINS;
		1612
		1613	ret = ring->flush(ring, I915_GEM_GPU_DOMAINS, flush_domains);
		1614	if (ret)
		1615	return ret;
		1616
		1617	trace_i915_gem_ring_flush(ring, I915_GEM_GPU_DOMAINS, flush_domains);
		1618
		1619	ring->gpu_caches_dirty = false;
		1620	return 0;
		1621	}

Subversion Repositories Kolibri OS

(root)/drivers/video/drm/i915/intel_ringbuffer.c – Rev 3031