WebSVN – Kolibri OS – Blame – /drivers/video/drm/i915/intel_ringbuffer.c

Rev	Author	Line No.	Line
2332	Serge	1	/*
		2	* Copyright © 2008-2010 Intel Corporation
		3	*
		4	* Permission is hereby granted, free of charge, to any person obtaining a
		5	* copy of this software and associated documentation files (the "Software"),
		6	* to deal in the Software without restriction, including without limitation
		7	* the rights to use, copy, modify, merge, publish, distribute, sublicense,
		8	* and/or sell copies of the Software, and to permit persons to whom the
		9	* Software is furnished to do so, subject to the following conditions:
		10	*
		11	* The above copyright notice and this permission notice (including the next
		12	* paragraph) shall be included in all copies or substantial portions of the
		13	* Software.
		14	*
		15	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
		16	* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
		17	* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
		18	* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
		19	* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
		20	* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
		21	* IN THE SOFTWARE.
		22	*
		23	* Authors:
		24	* Eric Anholt
		25	* Zou Nan hai
		26	* Xiang Hai hao
		27	*
		28	*/
		29	#define iowrite32(v, addr) writel((v), (addr))
		30	#define ioread32(addr) readl(addr)
		31
3031	serge	32	#include
2332	Serge	33	#include "i915_drv.h"
3031	serge	34	#include
2351	Serge	35	#include "i915_trace.h"
2332	Serge	36	#include "intel_drv.h"
		37
2342	Serge	38	/*
		39	* 965+ support PIPE_CONTROL commands, which provide finer grained control
		40	* over cache flushing.
		41	*/
		42	struct pipe_control {
		43	struct drm_i915_gem_object *obj;
		44	volatile u32 *cpu_page;
		45	u32 gtt_offset;
		46	};
		47
2332	Serge	48	static inline int ring_space(struct intel_ring_buffer *ring)
		49	{
3243	Serge	50	int space = (ring->head & HEAD_ADDR) - (ring->tail + I915_RING_FREE_SPACE);
2332	Serge	51	if (space < 0)
		52	space += ring->size;
		53	return space;
		54	}
		55
3031	serge	56	static int
		57	gen2_render_ring_flush(struct intel_ring_buffer *ring,
		58	u32 invalidate_domains,
		59	u32 flush_domains)
2332	Serge	60	{
3031	serge	61	u32 cmd;
		62	int ret;
2332	Serge	63
3031	serge	64	cmd = MI_FLUSH;
		65	if (((invalidate_domains\|flush_domains) & I915_GEM_DOMAIN_RENDER) == 0)
		66	cmd \|= MI_NO_WRITE_FLUSH;
2332	Serge	67
3031	serge	68	if (invalidate_domains & I915_GEM_DOMAIN_SAMPLER)
		69	cmd \|= MI_READ_FLUSH;
2332	Serge	70
3031	serge	71	ret = intel_ring_begin(ring, 2);
		72	if (ret)
		73	return ret;
		74
		75	intel_ring_emit(ring, cmd);
		76	intel_ring_emit(ring, MI_NOOP);
		77	intel_ring_advance(ring);
		78
		79	return 0;
2332	Serge	80	}
		81
		82	static int
3031	serge	83	gen4_render_ring_flush(struct intel_ring_buffer *ring,
2332	Serge	84	u32 invalidate_domains,
		85	u32 flush_domains)
		86	{
		87	struct drm_device *dev = ring->dev;
		88	u32 cmd;
		89	int ret;
		90
		91	/*
		92	* read/write caches:
		93	*
		94	* I915_GEM_DOMAIN_RENDER is always invalidated, but is
		95	* only flushed if MI_NO_WRITE_FLUSH is unset. On 965, it is
		96	* also flushed at 2d versus 3d pipeline switches.
		97	*
		98	* read-only caches:
		99	*
		100	* I915_GEM_DOMAIN_SAMPLER is flushed on pre-965 if
		101	* MI_READ_FLUSH is set, and is always flushed on 965.
		102	*
		103	* I915_GEM_DOMAIN_COMMAND may not exist?
		104	*
		105	* I915_GEM_DOMAIN_INSTRUCTION, which exists on 965, is
		106	* invalidated when MI_EXE_FLUSH is set.
		107	*
		108	* I915_GEM_DOMAIN_VERTEX, which exists on 965, is
		109	* invalidated with every MI_FLUSH.
		110	*
		111	* TLBs:
		112	*
		113	* On 965, TLBs associated with I915_GEM_DOMAIN_COMMAND
		114	* and I915_GEM_DOMAIN_CPU in are invalidated at PTE write and
		115	* I915_GEM_DOMAIN_RENDER and I915_GEM_DOMAIN_SAMPLER
		116	* are flushed at any MI_FLUSH.
		117	*/
		118
		119	cmd = MI_FLUSH \| MI_NO_WRITE_FLUSH;
3031	serge	120	if ((invalidate_domains\|flush_domains) & I915_GEM_DOMAIN_RENDER)
2332	Serge	121	cmd &= ~MI_NO_WRITE_FLUSH;
		122	if (invalidate_domains & I915_GEM_DOMAIN_INSTRUCTION)
		123	cmd \|= MI_EXE_FLUSH;
		124
		125	if (invalidate_domains & I915_GEM_DOMAIN_COMMAND &&
		126	(IS_G4X(dev) \|\| IS_GEN5(dev)))
		127	cmd \|= MI_INVALIDATE_ISP;
		128
		129	ret = intel_ring_begin(ring, 2);
		130	if (ret)
		131	return ret;
		132
		133	intel_ring_emit(ring, cmd);
		134	intel_ring_emit(ring, MI_NOOP);
		135	intel_ring_advance(ring);
		136
		137	return 0;
		138	}
		139
2342	Serge	140	/**
		141	* Emits a PIPE_CONTROL with a non-zero post-sync operation, for
		142	* implementing two workarounds on gen6. From section 1.4.7.1
		143	* "PIPE_CONTROL" of the Sandy Bridge PRM volume 2 part 1:
		144	*
		145	* [DevSNB-C+{W/A}] Before any depth stall flush (including those
		146	* produced by non-pipelined state commands), software needs to first
		147	* send a PIPE_CONTROL with no bits set except Post-Sync Operation !=
		148	* 0.
		149	*
		150	* [Dev-SNB{W/A}]: Before a PIPE_CONTROL with Write Cache Flush Enable
		151	* =1, a PIPE_CONTROL with any non-zero post-sync-op is required.
		152	*
		153	* And the workaround for these two requires this workaround first:
		154	*
		155	* [Dev-SNB{W/A}]: Pipe-control with CS-stall bit set must be sent
		156	* BEFORE the pipe-control with a post-sync op and no write-cache
		157	* flushes.
		158	*
		159	* And this last workaround is tricky because of the requirements on
		160	* that bit. From section 1.4.7.2.3 "Stall" of the Sandy Bridge PRM
		161	* volume 2 part 1:
		162	*
		163	* "1 of the following must also be set:
		164	* - Render Target Cache Flush Enable ([12] of DW1)
		165	* - Depth Cache Flush Enable ([0] of DW1)
		166	* - Stall at Pixel Scoreboard ([1] of DW1)
		167	* - Depth Stall ([13] of DW1)
		168	* - Post-Sync Operation ([13] of DW1)
		169	* - Notify Enable ([8] of DW1)"
		170	*
		171	* The cache flushes require the workaround flush that triggered this
		172	* one, so we can't use it. Depth stall would trigger the same.
		173	* Post-sync nonzero is what triggered this second workaround, so we
		174	* can't use that one either. Notify enable is IRQs, which aren't
		175	* really our business. That leaves only stall at scoreboard.
		176	*/
		177	static int
		178	intel_emit_post_sync_nonzero_flush(struct intel_ring_buffer *ring)
		179	{
		180	struct pipe_control *pc = ring->private;
		181	u32 scratch_addr = pc->gtt_offset + 128;
		182	int ret;
		183
		184
		185	ret = intel_ring_begin(ring, 6);
		186	if (ret)
		187	return ret;
		188
		189	intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(5));
		190	intel_ring_emit(ring, PIPE_CONTROL_CS_STALL \|
		191	PIPE_CONTROL_STALL_AT_SCOREBOARD);
		192	intel_ring_emit(ring, scratch_addr \| PIPE_CONTROL_GLOBAL_GTT); /* address */
		193	intel_ring_emit(ring, 0); /* low dword */
		194	intel_ring_emit(ring, 0); /* high dword */
		195	intel_ring_emit(ring, MI_NOOP);
		196	intel_ring_advance(ring);
		197
		198	ret = intel_ring_begin(ring, 6);
		199	if (ret)
		200	return ret;
		201
		202	intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(5));
		203	intel_ring_emit(ring, PIPE_CONTROL_QW_WRITE);
		204	intel_ring_emit(ring, scratch_addr \| PIPE_CONTROL_GLOBAL_GTT); /* address */
		205	intel_ring_emit(ring, 0);
		206	intel_ring_emit(ring, 0);
		207	intel_ring_emit(ring, MI_NOOP);
		208	intel_ring_advance(ring);
		209
		210	return 0;
		211	}
		212
		213	static int
		214	gen6_render_ring_flush(struct intel_ring_buffer *ring,
		215	u32 invalidate_domains, u32 flush_domains)
		216	{
		217	u32 flags = 0;
		218	struct pipe_control *pc = ring->private;
		219	u32 scratch_addr = pc->gtt_offset + 128;
		220	int ret;
		221
		222	/* Force SNB workarounds for PIPE_CONTROL flushes */
3031	serge	223	ret = intel_emit_post_sync_nonzero_flush(ring);
		224	if (ret)
		225	return ret;
2342	Serge	226
		227	/* Just flush everything. Experiments have shown that reducing the
		228	* number of bits based on the write domains has little performance
		229	* impact.
		230	*/
3031	serge	231	if (flush_domains) {
		232	flags \|= PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH;
		233	flags \|= PIPE_CONTROL_DEPTH_CACHE_FLUSH;
		234	/*
		235	* Ensure that any following seqno writes only happen
		236	* when the render cache is indeed flushed.
		237	*/
		238	flags \|= PIPE_CONTROL_CS_STALL;
		239	}
		240	if (invalidate_domains) {
		241	flags \|= PIPE_CONTROL_TLB_INVALIDATE;
		242	flags \|= PIPE_CONTROL_INSTRUCTION_CACHE_INVALIDATE;
		243	flags \|= PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE;
		244	flags \|= PIPE_CONTROL_VF_CACHE_INVALIDATE;
		245	flags \|= PIPE_CONTROL_CONST_CACHE_INVALIDATE;
		246	flags \|= PIPE_CONTROL_STATE_CACHE_INVALIDATE;
		247	/*
		248	* TLB invalidate requires a post-sync write.
		249	*/
3243	Serge	250	flags \|= PIPE_CONTROL_QW_WRITE \| PIPE_CONTROL_CS_STALL;
3031	serge	251	}
		252
		253	ret = intel_ring_begin(ring, 4);
		254	if (ret)
		255	return ret;
		256
		257	intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(4));
		258	intel_ring_emit(ring, flags);
		259	intel_ring_emit(ring, scratch_addr \| PIPE_CONTROL_GLOBAL_GTT);
		260	intel_ring_emit(ring, 0);
		261	intel_ring_advance(ring);
		262
		263	return 0;
		264	}
		265
		266	static int
		267	gen7_render_ring_cs_stall_wa(struct intel_ring_buffer *ring)
		268	{
		269	int ret;
		270
		271	ret = intel_ring_begin(ring, 4);
		272	if (ret)
		273	return ret;
		274
		275	intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(4));
		276	intel_ring_emit(ring, PIPE_CONTROL_CS_STALL \|
		277	PIPE_CONTROL_STALL_AT_SCOREBOARD);
		278	intel_ring_emit(ring, 0);
		279	intel_ring_emit(ring, 0);
		280	intel_ring_advance(ring);
		281
		282	return 0;
		283	}
		284
		285	static int
		286	gen7_render_ring_flush(struct intel_ring_buffer *ring,
		287	u32 invalidate_domains, u32 flush_domains)
		288	{
		289	u32 flags = 0;
		290	struct pipe_control *pc = ring->private;
		291	u32 scratch_addr = pc->gtt_offset + 128;
		292	int ret;
		293
		294	/*
		295	* Ensure that any following seqno writes only happen when the render
		296	* cache is indeed flushed.
		297	*
		298	* Workaround: 4th PIPE_CONTROL command (except the ones with only
		299	* read-cache invalidate bits set) must have the CS_STALL bit set. We
		300	* don't try to be clever and just set it unconditionally.
		301	*/
		302	flags \|= PIPE_CONTROL_CS_STALL;
		303
		304	/* Just flush everything. Experiments have shown that reducing the
		305	* number of bits based on the write domains has little performance
		306	* impact.
		307	*/
		308	if (flush_domains) {
2342	Serge	309	flags \|= PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH;
3031	serge	310	flags \|= PIPE_CONTROL_DEPTH_CACHE_FLUSH;
		311	}
		312	if (invalidate_domains) {
		313	flags \|= PIPE_CONTROL_TLB_INVALIDATE;
2342	Serge	314	flags \|= PIPE_CONTROL_INSTRUCTION_CACHE_INVALIDATE;
		315	flags \|= PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE;
		316	flags \|= PIPE_CONTROL_VF_CACHE_INVALIDATE;
		317	flags \|= PIPE_CONTROL_CONST_CACHE_INVALIDATE;
		318	flags \|= PIPE_CONTROL_STATE_CACHE_INVALIDATE;
3031	serge	319	/*
		320	* TLB invalidate requires a post-sync write.
		321	*/
		322	flags \|= PIPE_CONTROL_QW_WRITE;
3480	Serge	323	flags \|= PIPE_CONTROL_GLOBAL_GTT_IVB;
2342	Serge	324
3031	serge	325	/* Workaround: we must issue a pipe_control with CS-stall bit
		326	* set before a pipe_control command that has the state cache
		327	* invalidate bit set. */
		328	gen7_render_ring_cs_stall_wa(ring);
		329	}
		330
		331	ret = intel_ring_begin(ring, 4);
2342	Serge	332	if (ret)
		333	return ret;
		334
3031	serge	335	intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(4));
2342	Serge	336	intel_ring_emit(ring, flags);
3480	Serge	337	intel_ring_emit(ring, scratch_addr);
3031	serge	338	intel_ring_emit(ring, 0);
2342	Serge	339	intel_ring_advance(ring);
		340
		341	return 0;
		342	}
		343
2332	Serge	344	static void ring_write_tail(struct intel_ring_buffer *ring,
		345	u32 value)
		346	{
		347	drm_i915_private_t *dev_priv = ring->dev->dev_private;
		348	I915_WRITE_TAIL(ring, value);
		349	}
		350
		351	u32 intel_ring_get_active_head(struct intel_ring_buffer *ring)
		352	{
		353	drm_i915_private_t *dev_priv = ring->dev->dev_private;
		354	u32 acthd_reg = INTEL_INFO(ring->dev)->gen >= 4 ?
		355	RING_ACTHD(ring->mmio_base) : ACTHD;
		356
		357	return I915_READ(acthd_reg);
		358	}
		359
		360	static int init_ring_common(struct intel_ring_buffer *ring)
		361	{
3031	serge	362	struct drm_device *dev = ring->dev;
		363	drm_i915_private_t *dev_priv = dev->dev_private;
2332	Serge	364	struct drm_i915_gem_object *obj = ring->obj;
3031	serge	365	int ret = 0;
2332	Serge	366	u32 head;
		367
3031	serge	368	if (HAS_FORCE_WAKE(dev))
		369	gen6_gt_force_wake_get(dev_priv);
		370
2332	Serge	371	/* Stop the ring if it's running. */
		372	I915_WRITE_CTL(ring, 0);
		373	I915_WRITE_HEAD(ring, 0);
		374	ring->write_tail(ring, 0);
		375
		376	head = I915_READ_HEAD(ring) & HEAD_ADDR;
		377
		378	/* G45 ring initialization fails to reset head to zero */
		379	if (head != 0) {
		380	DRM_DEBUG_KMS("%s head not reset to zero "
		381	"ctl %08x head %08x tail %08x start %08x\n",
		382	ring->name,
		383	I915_READ_CTL(ring),
		384	I915_READ_HEAD(ring),
		385	I915_READ_TAIL(ring),
		386	I915_READ_START(ring));
		387
		388	I915_WRITE_HEAD(ring, 0);
		389
		390	if (I915_READ_HEAD(ring) & HEAD_ADDR) {
		391	DRM_ERROR("failed to set %s head to zero "
		392	"ctl %08x head %08x tail %08x start %08x\n",
		393	ring->name,
		394	I915_READ_CTL(ring),
		395	I915_READ_HEAD(ring),
		396	I915_READ_TAIL(ring),
		397	I915_READ_START(ring));
		398	}
		399	}
		400
3031	serge	401	/* Initialize the ring. This must happen _after_ we've cleared the ring
		402	* registers with the above sequence (the readback of the HEAD registers
		403	* also enforces ordering), otherwise the hw might lose the new ring
		404	* register values. */
		405	I915_WRITE_START(ring, obj->gtt_offset);
2332	Serge	406	I915_WRITE_CTL(ring,
		407	((ring->size - PAGE_SIZE) & RING_NR_PAGES)
3031	serge	408	\| RING_VALID);
2332	Serge	409
		410	/* If the head is still not zero, the ring is dead */
3031	serge	411	if (wait_for((I915_READ_CTL(ring) & RING_VALID) != 0 &&
		412	I915_READ_START(ring) == obj->gtt_offset &&
		413	(I915_READ_HEAD(ring) & HEAD_ADDR) == 0, 50)) {
2332	Serge	414	DRM_ERROR("%s initialization failed "
		415	"ctl %08x head %08x tail %08x start %08x\n",
		416	ring->name,
		417	I915_READ_CTL(ring),
		418	I915_READ_HEAD(ring),
		419	I915_READ_TAIL(ring),
		420	I915_READ_START(ring));
3031	serge	421	ret = -EIO;
		422	goto out;
2332	Serge	423	}
		424
3031	serge	425	ring->head = I915_READ_HEAD(ring);
		426	ring->tail = I915_READ_TAIL(ring) & TAIL_ADDR;
		427	ring->space = ring_space(ring);
		428	ring->last_retired_head = -1;
2332	Serge	429
3031	serge	430	out:
		431	if (HAS_FORCE_WAKE(dev))
		432	gen6_gt_force_wake_put(dev_priv);
2332	Serge	433
3031	serge	434	return ret;
2332	Serge	435	}
		436
		437	static int
		438	init_pipe_control(struct intel_ring_buffer *ring)
		439	{
		440	struct pipe_control *pc;
		441	struct drm_i915_gem_object *obj;
		442	int ret;
		443
		444	if (ring->private)
		445	return 0;
		446
		447	pc = kmalloc(sizeof(*pc), GFP_KERNEL);
		448	if (!pc)
		449	return -ENOMEM;
		450
		451	obj = i915_gem_alloc_object(ring->dev, 4096);
		452	if (obj == NULL) {
		453	DRM_ERROR("Failed to allocate seqno page\n");
		454	ret = -ENOMEM;
		455	goto err;
		456	}
		457
2352	Serge	458	i915_gem_object_set_cache_level(obj, I915_CACHE_LLC);
2332	Serge	459
3031	serge	460	ret = i915_gem_object_pin(obj, 4096, true, false);
2332	Serge	461	if (ret)
		462	goto err_unref;
		463
		464	pc->gtt_offset = obj->gtt_offset;
3243	Serge	465	pc->cpu_page = (void*)MapIoMem((addr_t)sg_page(obj->pages->sgl),4096, PG_SW);
2332	Serge	466	if (pc->cpu_page == NULL)
		467	goto err_unpin;
		468
3480	Serge	469	DRM_DEBUG_DRIVER("%s pipe control offset: 0x%08x\n",
		470	ring->name, pc->gtt_offset);
		471
2332	Serge	472	pc->obj = obj;
		473	ring->private = pc;
		474	return 0;
		475
		476	err_unpin:
2344	Serge	477	i915_gem_object_unpin(obj);
2332	Serge	478	err_unref:
2344	Serge	479	drm_gem_object_unreference(&obj->base);
2332	Serge	480	err:
		481	kfree(pc);
		482	return ret;
		483	}
		484
		485	static void
		486	cleanup_pipe_control(struct intel_ring_buffer *ring)
		487	{
		488	struct pipe_control *pc = ring->private;
		489	struct drm_i915_gem_object *obj;
		490
		491	if (!ring->private)
		492	return;
		493
		494	obj = pc->obj;
2339	Serge	495	// kunmap(obj->pages[0]);
2344	Serge	496	i915_gem_object_unpin(obj);
		497	drm_gem_object_unreference(&obj->base);
2332	Serge	498
		499	kfree(pc);
		500	ring->private = NULL;
		501	}
		502
		503	static int init_render_ring(struct intel_ring_buffer *ring)
		504	{
		505	struct drm_device *dev = ring->dev;
		506	struct drm_i915_private *dev_priv = dev->dev_private;
		507	int ret = init_ring_common(ring);
		508
3243	Serge	509	if (INTEL_INFO(dev)->gen > 3)
3031	serge	510	I915_WRITE(MI_MODE, _MASKED_BIT_ENABLE(VS_TIMER_DISPATCH));
3243	Serge	511
		512	/* We need to disable the AsyncFlip performance optimisations in order
		513	* to use MI_WAIT_FOR_EVENT within the CS. It should already be
		514	* programmed to '1' on all products.
		515	*/
		516	if (INTEL_INFO(dev)->gen >= 6)
		517	I915_WRITE(MI_MODE, _MASKED_BIT_ENABLE(ASYNC_FLIP_PERF_DISABLE));
		518
		519	/* Required for the hardware to program scanline values for waiting */
		520	if (INTEL_INFO(dev)->gen == 6)
		521	I915_WRITE(GFX_MODE,
		522	_MASKED_BIT_ENABLE(GFX_TLB_INVALIDATE_ALWAYS));
		523
2332	Serge	524	if (IS_GEN7(dev))
		525	I915_WRITE(GFX_MODE_GEN7,
3031	serge	526	_MASKED_BIT_DISABLE(GFX_TLB_INVALIDATE_ALWAYS) \|
		527	_MASKED_BIT_ENABLE(GFX_REPLAY_MODE));
2332	Serge	528
2342	Serge	529	if (INTEL_INFO(dev)->gen >= 5) {
2339	Serge	530	ret = init_pipe_control(ring);
2332	Serge	531	if (ret)
		532	return ret;
		533	}
		534
3031	serge	535	if (IS_GEN6(dev)) {
		536	/* From the Sandybridge PRM, volume 1 part 3, page 24:
		537	* "If this bit is set, STCunit will have LRA as replacement
		538	* policy. [...] This bit must be reset. LRA replacement
		539	* policy is not supported."
		540	*/
		541	I915_WRITE(CACHE_MODE_0,
		542	_MASKED_BIT_DISABLE(CM0_STC_EVICT_DISABLE_LRA_SNB));
		543
		544	/* This is not explicitly set for GEN6, so read the register.
		545	* see intel_ring_mi_set_context() for why we care.
		546	* TODO: consider explicitly setting the bit for GEN5
		547	*/
		548	ring->itlb_before_ctx_switch =
		549	!!(I915_READ(GFX_MODE) & GFX_TLB_INVALIDATE_ALWAYS);
2342	Serge	550	}
		551
3031	serge	552	if (INTEL_INFO(dev)->gen >= 6)
		553	I915_WRITE(INSTPM, _MASKED_BIT_ENABLE(INSTPM_FORCE_ORDERING));
		554
		555	if (HAS_L3_GPU_CACHE(dev))
		556	I915_WRITE_IMR(ring, ~GEN6_RENDER_L3_PARITY_ERROR);
		557
2332	Serge	558	return ret;
		559	}
		560
		561	static void render_ring_cleanup(struct intel_ring_buffer *ring)
		562	{
3480	Serge	563	struct drm_device *dev = ring->dev;
		564
2332	Serge	565	if (!ring->private)
		566	return;
		567
		568	cleanup_pipe_control(ring);
		569	}
		570
		571	static void
2342	Serge	572	update_mboxes(struct intel_ring_buffer *ring,
		573	u32 mmio_offset)
2332	Serge	574	{
3243	Serge	575	intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(1));
2342	Serge	576	intel_ring_emit(ring, mmio_offset);
3243	Serge	577	intel_ring_emit(ring, ring->outstanding_lazy_request);
2332	Serge	578	}
		579
2342	Serge	580	/**
		581	* gen6_add_request - Update the semaphore mailbox registers
		582	*
		583	* @ring - ring that is adding a request
		584	* @seqno - return seqno stuck into the ring
		585	*
		586	* Update the mailbox registers in the other rings with the current seqno.
		587	* This acts like a signal in the canonical semaphore.
		588	*/
2332	Serge	589	static int
3243	Serge	590	gen6_add_request(struct intel_ring_buffer *ring)
2332	Serge	591	{
2342	Serge	592	u32 mbox1_reg;
		593	u32 mbox2_reg;
2332	Serge	594	int ret;
		595
		596	ret = intel_ring_begin(ring, 10);
		597	if (ret)
		598	return ret;
		599
2342	Serge	600	mbox1_reg = ring->signal_mbox[0];
		601	mbox2_reg = ring->signal_mbox[1];
2332	Serge	602
3243	Serge	603	update_mboxes(ring, mbox1_reg);
		604	update_mboxes(ring, mbox2_reg);
2332	Serge	605	intel_ring_emit(ring, MI_STORE_DWORD_INDEX);
		606	intel_ring_emit(ring, I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
3243	Serge	607	intel_ring_emit(ring, ring->outstanding_lazy_request);
2332	Serge	608	intel_ring_emit(ring, MI_USER_INTERRUPT);
		609	intel_ring_advance(ring);
		610
		611	return 0;
		612	}
		613
3480	Serge	614	static inline bool i915_gem_has_seqno_wrapped(struct drm_device *dev,
		615	u32 seqno)
		616	{
		617	struct drm_i915_private *dev_priv = dev->dev_private;
		618	return dev_priv->last_seqno < seqno;
		619	}
		620
2342	Serge	621	/**
		622	* intel_ring_sync - sync the waiter to the signaller on seqno
		623	*
		624	* @waiter - ring that is waiting
		625	* @signaller - ring which has, or will signal
		626	* @seqno - seqno which the waiter will block on
		627	*/
		628	static int
3031	serge	629	gen6_ring_sync(struct intel_ring_buffer *waiter,
2342	Serge	630	struct intel_ring_buffer *signaller,
2332	Serge	631	u32 seqno)
		632	{
		633	int ret;
2342	Serge	634	u32 dw1 = MI_SEMAPHORE_MBOX \|
		635	MI_SEMAPHORE_COMPARE \|
		636	MI_SEMAPHORE_REGISTER;
2332	Serge	637
3031	serge	638	/* Throughout all of the GEM code, seqno passed implies our current
		639	* seqno is >= the last seqno executed. However for hardware the
		640	* comparison is strictly greater than.
		641	*/
		642	seqno -= 1;
		643
		644	WARN_ON(signaller->semaphore_register[waiter->id] ==
		645	MI_SEMAPHORE_SYNC_INVALID);
		646
2342	Serge	647	ret = intel_ring_begin(waiter, 4);
2332	Serge	648	if (ret)
		649	return ret;
		650
3480	Serge	651	/* If seqno wrap happened, omit the wait with no-ops */
		652	if (likely(!i915_gem_has_seqno_wrapped(waiter->dev, seqno))) {
3031	serge	653	intel_ring_emit(waiter,
3480	Serge	654	dw1 \|
		655	signaller->semaphore_register[waiter->id]);
2342	Serge	656	intel_ring_emit(waiter, seqno);
		657	intel_ring_emit(waiter, 0);
		658	intel_ring_emit(waiter, MI_NOOP);
3480	Serge	659	} else {
		660	intel_ring_emit(waiter, MI_NOOP);
		661	intel_ring_emit(waiter, MI_NOOP);
		662	intel_ring_emit(waiter, MI_NOOP);
		663	intel_ring_emit(waiter, MI_NOOP);
		664	}
2342	Serge	665	intel_ring_advance(waiter);
2332	Serge	666
		667	return 0;
		668	}
		669
		670	#define PIPE_CONTROL_FLUSH(ring__, addr__) \
		671	do { \
2342	Serge	672	intel_ring_emit(ring__, GFX_OP_PIPE_CONTROL(4) \| PIPE_CONTROL_QW_WRITE \| \
		673	PIPE_CONTROL_DEPTH_STALL); \
2332	Serge	674	intel_ring_emit(ring__, (addr__) \| PIPE_CONTROL_GLOBAL_GTT); \
		675	intel_ring_emit(ring__, 0); \
		676	intel_ring_emit(ring__, 0); \
		677	} while (0)
		678
		679	static int
3243	Serge	680	pc_render_add_request(struct intel_ring_buffer *ring)
2332	Serge	681	{
		682	struct pipe_control *pc = ring->private;
		683	u32 scratch_addr = pc->gtt_offset + 128;
		684	int ret;
		685
		686	/* For Ironlake, MI_USER_INTERRUPT was deprecated and apparently
		687	* incoherent with writes to memory, i.e. completely fubar,
		688	* so we need to use PIPE_NOTIFY instead.
		689	*
		690	* However, we also need to workaround the qword write
		691	* incoherence by flushing the 6 PIPE_NOTIFY buffers out to
		692	* memory before requesting an interrupt.
		693	*/
		694	ret = intel_ring_begin(ring, 32);
		695	if (ret)
		696	return ret;
		697
2342	Serge	698	intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(4) \| PIPE_CONTROL_QW_WRITE \|
		699	PIPE_CONTROL_WRITE_FLUSH \|
		700	PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE);
2332	Serge	701	intel_ring_emit(ring, pc->gtt_offset \| PIPE_CONTROL_GLOBAL_GTT);
3243	Serge	702	intel_ring_emit(ring, ring->outstanding_lazy_request);
2332	Serge	703	intel_ring_emit(ring, 0);
		704	PIPE_CONTROL_FLUSH(ring, scratch_addr);
		705	scratch_addr += 128; /* write to separate cachelines */
		706	PIPE_CONTROL_FLUSH(ring, scratch_addr);
		707	scratch_addr += 128;
		708	PIPE_CONTROL_FLUSH(ring, scratch_addr);
		709	scratch_addr += 128;
		710	PIPE_CONTROL_FLUSH(ring, scratch_addr);
		711	scratch_addr += 128;
		712	PIPE_CONTROL_FLUSH(ring, scratch_addr);
		713	scratch_addr += 128;
		714	PIPE_CONTROL_FLUSH(ring, scratch_addr);
3031	serge	715
2342	Serge	716	intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(4) \| PIPE_CONTROL_QW_WRITE \|
		717	PIPE_CONTROL_WRITE_FLUSH \|
		718	PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE \|
2332	Serge	719	PIPE_CONTROL_NOTIFY);
		720	intel_ring_emit(ring, pc->gtt_offset \| PIPE_CONTROL_GLOBAL_GTT);
3243	Serge	721	intel_ring_emit(ring, ring->outstanding_lazy_request);
2332	Serge	722	intel_ring_emit(ring, 0);
		723	intel_ring_advance(ring);
		724
		725	return 0;
		726	}
		727
		728	static u32
3031	serge	729	gen6_ring_get_seqno(struct intel_ring_buffer *ring, bool lazy_coherency)
2342	Serge	730	{
		731	/* Workaround to force correct ordering between irq and seqno writes on
		732	* ivb (and maybe also on snb) by reading from a CS register (like
		733	* ACTHD) before reading the status page. */
3031	serge	734	if (!lazy_coherency)
2342	Serge	735	intel_ring_get_active_head(ring);
		736	return intel_read_status_page(ring, I915_GEM_HWS_INDEX);
		737	}
		738
		739	static u32
3031	serge	740	ring_get_seqno(struct intel_ring_buffer *ring, bool lazy_coherency)
2332	Serge	741	{
		742	return intel_read_status_page(ring, I915_GEM_HWS_INDEX);
		743	}
		744
3480	Serge	745	static void
		746	ring_set_seqno(struct intel_ring_buffer *ring, u32 seqno)
		747	{
		748	intel_write_status_page(ring, I915_GEM_HWS_INDEX, seqno);
		749	}
		750
2332	Serge	751	static u32
3031	serge	752	pc_render_get_seqno(struct intel_ring_buffer *ring, bool lazy_coherency)
2332	Serge	753	{
		754	struct pipe_control *pc = ring->private;
		755	return pc->cpu_page[0];
		756	}
		757
3480	Serge	758	static void
		759	pc_render_set_seqno(struct intel_ring_buffer *ring, u32 seqno)
		760	{
		761	struct pipe_control *pc = ring->private;
		762	pc->cpu_page[0] = seqno;
		763	}
		764
3031	serge	765	static bool
		766	gen5_ring_get_irq(struct intel_ring_buffer *ring)
2332	Serge	767	{
3031	serge	768	struct drm_device *dev = ring->dev;
		769	drm_i915_private_t *dev_priv = dev->dev_private;
		770	unsigned long flags;
		771
		772	if (!dev->irq_enabled)
		773	return false;
		774
		775	spin_lock_irqsave(&dev_priv->irq_lock, flags);
		776	if (ring->irq_refcount++ == 0) {
		777	dev_priv->gt_irq_mask &= ~ring->irq_enable_mask;
2332	Serge	778	I915_WRITE(GTIMR, dev_priv->gt_irq_mask);
		779	POSTING_READ(GTIMR);
3031	serge	780	}
		781	spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
		782
		783	return true;
2332	Serge	784	}
		785
		786	static void
3031	serge	787	gen5_ring_put_irq(struct intel_ring_buffer *ring)
2332	Serge	788	{
3031	serge	789	struct drm_device *dev = ring->dev;
		790	drm_i915_private_t *dev_priv = dev->dev_private;
		791	unsigned long flags;
		792
		793	spin_lock_irqsave(&dev_priv->irq_lock, flags);
		794	if (--ring->irq_refcount == 0) {
		795	dev_priv->gt_irq_mask \|= ring->irq_enable_mask;
2332	Serge	796	I915_WRITE(GTIMR, dev_priv->gt_irq_mask);
		797	POSTING_READ(GTIMR);
3031	serge	798	}
		799	spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
2332	Serge	800	}
		801
3031	serge	802	static bool
		803	i9xx_ring_get_irq(struct intel_ring_buffer *ring)
2332	Serge	804	{
3031	serge	805	struct drm_device *dev = ring->dev;
		806	drm_i915_private_t *dev_priv = dev->dev_private;
		807	unsigned long flags;
		808
		809	if (!dev->irq_enabled)
		810	return false;
		811
		812	spin_lock_irqsave(&dev_priv->irq_lock, flags);
		813	if (ring->irq_refcount++ == 0) {
		814	dev_priv->irq_mask &= ~ring->irq_enable_mask;
2332	Serge	815	I915_WRITE(IMR, dev_priv->irq_mask);
		816	POSTING_READ(IMR);
3031	serge	817	}
		818	spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
		819
		820	return true;
2332	Serge	821	}
		822
		823	static void
3031	serge	824	i9xx_ring_put_irq(struct intel_ring_buffer *ring)
2332	Serge	825	{
3031	serge	826	struct drm_device *dev = ring->dev;
		827	drm_i915_private_t *dev_priv = dev->dev_private;
		828	unsigned long flags;
		829
		830	spin_lock_irqsave(&dev_priv->irq_lock, flags);
		831	if (--ring->irq_refcount == 0) {
		832	dev_priv->irq_mask \|= ring->irq_enable_mask;
2332	Serge	833	I915_WRITE(IMR, dev_priv->irq_mask);
		834	POSTING_READ(IMR);
3031	serge	835	}
		836	spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
2332	Serge	837	}
		838
		839	static bool
3031	serge	840	i8xx_ring_get_irq(struct intel_ring_buffer *ring)
2332	Serge	841	{
		842	struct drm_device *dev = ring->dev;
		843	drm_i915_private_t *dev_priv = dev->dev_private;
3031	serge	844	unsigned long flags;
2332	Serge	845
		846	if (!dev->irq_enabled)
		847	return false;
		848
3031	serge	849	spin_lock_irqsave(&dev_priv->irq_lock, flags);
2332	Serge	850	if (ring->irq_refcount++ == 0) {
3031	serge	851	dev_priv->irq_mask &= ~ring->irq_enable_mask;
		852	I915_WRITE16(IMR, dev_priv->irq_mask);
		853	POSTING_READ16(IMR);
2332	Serge	854	}
3031	serge	855	spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
2332	Serge	856
		857	return true;
		858	}
		859
		860	static void
3031	serge	861	i8xx_ring_put_irq(struct intel_ring_buffer *ring)
2332	Serge	862	{
		863	struct drm_device *dev = ring->dev;
		864	drm_i915_private_t *dev_priv = dev->dev_private;
3031	serge	865	unsigned long flags;
2332	Serge	866
3031	serge	867	spin_lock_irqsave(&dev_priv->irq_lock, flags);
2332	Serge	868	if (--ring->irq_refcount == 0) {
3031	serge	869	dev_priv->irq_mask \|= ring->irq_enable_mask;
		870	I915_WRITE16(IMR, dev_priv->irq_mask);
		871	POSTING_READ16(IMR);
2332	Serge	872	}
3031	serge	873	spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
2332	Serge	874	}
		875
		876	void intel_ring_setup_status_page(struct intel_ring_buffer *ring)
		877	{
		878	struct drm_device *dev = ring->dev;
		879	drm_i915_private_t *dev_priv = ring->dev->dev_private;
		880	u32 mmio = 0;
		881
		882	/* The ring status page addresses are no longer next to the rest of
		883	* the ring registers as of gen7.
		884	*/
		885	if (IS_GEN7(dev)) {
		886	switch (ring->id) {
3031	serge	887	case RCS:
2332	Serge	888	mmio = RENDER_HWS_PGA_GEN7;
		889	break;
3031	serge	890	case BCS:
2332	Serge	891	mmio = BLT_HWS_PGA_GEN7;
		892	break;
3031	serge	893	case VCS:
2332	Serge	894	mmio = BSD_HWS_PGA_GEN7;
		895	break;
		896	}
		897	} else if (IS_GEN6(ring->dev)) {
		898	mmio = RING_HWS_PGA_GEN6(ring->mmio_base);
		899	} else {
		900	mmio = RING_HWS_PGA(ring->mmio_base);
		901	}
		902
		903	I915_WRITE(mmio, (u32)ring->status_page.gfx_addr);
		904	POSTING_READ(mmio);
		905	}
		906
		907	static int
		908	bsd_ring_flush(struct intel_ring_buffer *ring,
		909	u32 invalidate_domains,
		910	u32 flush_domains)
		911	{
		912	int ret;
		913
		914	ret = intel_ring_begin(ring, 2);
		915	if (ret)
		916	return ret;
		917
		918	intel_ring_emit(ring, MI_FLUSH);
		919	intel_ring_emit(ring, MI_NOOP);
		920	intel_ring_advance(ring);
		921	return 0;
		922	}
		923
		924	static int
3243	Serge	925	i9xx_add_request(struct intel_ring_buffer *ring)
2332	Serge	926	{
		927	int ret;
		928
		929	ret = intel_ring_begin(ring, 4);
		930	if (ret)
		931	return ret;
		932
		933	intel_ring_emit(ring, MI_STORE_DWORD_INDEX);
		934	intel_ring_emit(ring, I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
3243	Serge	935	intel_ring_emit(ring, ring->outstanding_lazy_request);
2332	Serge	936	intel_ring_emit(ring, MI_USER_INTERRUPT);
		937	intel_ring_advance(ring);
		938
		939	return 0;
		940	}
		941
		942	static bool
3031	serge	943	gen6_ring_get_irq(struct intel_ring_buffer *ring)
2332	Serge	944	{
		945	struct drm_device *dev = ring->dev;
		946	drm_i915_private_t *dev_priv = dev->dev_private;
3031	serge	947	unsigned long flags;
2332	Serge	948
		949	if (!dev->irq_enabled)
		950	return false;
		951
2342	Serge	952	/* It looks like we need to prevent the gt from suspending while waiting
		953	* for an notifiy irq, otherwise irqs seem to get lost on at least the
		954	* blt/bsd rings on ivb. */
		955	gen6_gt_force_wake_get(dev_priv);
		956
3031	serge	957	spin_lock_irqsave(&dev_priv->irq_lock, flags);
2332	Serge	958	if (ring->irq_refcount++ == 0) {
3031	serge	959	if (HAS_L3_GPU_CACHE(dev) && ring->id == RCS)
		960	I915_WRITE_IMR(ring, ~(ring->irq_enable_mask \|
		961	GEN6_RENDER_L3_PARITY_ERROR));
		962	else
		963	I915_WRITE_IMR(ring, ~ring->irq_enable_mask);
		964	dev_priv->gt_irq_mask &= ~ring->irq_enable_mask;
		965	I915_WRITE(GTIMR, dev_priv->gt_irq_mask);
		966	POSTING_READ(GTIMR);
2332	Serge	967	}
3031	serge	968	spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
2332	Serge	969
2351	Serge	970	return true;
2332	Serge	971	}
		972
		973	static void
3031	serge	974	gen6_ring_put_irq(struct intel_ring_buffer *ring)
2332	Serge	975	{
		976	struct drm_device *dev = ring->dev;
		977	drm_i915_private_t *dev_priv = dev->dev_private;
3031	serge	978	unsigned long flags;
2332	Serge	979
3031	serge	980	spin_lock_irqsave(&dev_priv->irq_lock, flags);
2332	Serge	981	if (--ring->irq_refcount == 0) {
3031	serge	982	if (HAS_L3_GPU_CACHE(dev) && ring->id == RCS)
		983	I915_WRITE_IMR(ring, ~GEN6_RENDER_L3_PARITY_ERROR);
		984	else
		985	I915_WRITE_IMR(ring, ~0);
		986	dev_priv->gt_irq_mask \|= ring->irq_enable_mask;
		987	I915_WRITE(GTIMR, dev_priv->gt_irq_mask);
		988	POSTING_READ(GTIMR);
2332	Serge	989	}
3031	serge	990	spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
2342	Serge	991
		992	gen6_gt_force_wake_put(dev_priv);
2332	Serge	993	}
		994
		995	static int
3243	Serge	996	i965_dispatch_execbuffer(struct intel_ring_buffer *ring,
		997	u32 offset, u32 length,
		998	unsigned flags)
2332	Serge	999	{
		1000	int ret;
		1001
		1002	ret = intel_ring_begin(ring, 2);
		1003	if (ret)
		1004	return ret;
		1005
		1006	intel_ring_emit(ring,
3031	serge	1007	MI_BATCH_BUFFER_START \|
		1008	MI_BATCH_GTT \|
3243	Serge	1009	(flags & I915_DISPATCH_SECURE ? 0 : MI_BATCH_NON_SECURE_I965));
2332	Serge	1010	intel_ring_emit(ring, offset);
		1011	intel_ring_advance(ring);
		1012
		1013	return 0;
		1014	}
		1015
3243	Serge	1016	/* Just userspace ABI convention to limit the wa batch bo to a resonable size */
		1017	#define I830_BATCH_LIMIT (256*1024)
2332	Serge	1018	static int
3031	serge	1019	i830_dispatch_execbuffer(struct intel_ring_buffer *ring,
3243	Serge	1020	u32 offset, u32 len,
		1021	unsigned flags)
2332	Serge	1022	{
		1023	int ret;
		1024
3243	Serge	1025	if (flags & I915_DISPATCH_PINNED) {
2332	Serge	1026	ret = intel_ring_begin(ring, 4);
		1027	if (ret)
		1028	return ret;
		1029
		1030	intel_ring_emit(ring, MI_BATCH_BUFFER);
3243	Serge	1031	intel_ring_emit(ring, offset \| (flags & I915_DISPATCH_SECURE ? 0 : MI_BATCH_NON_SECURE));
2332	Serge	1032	intel_ring_emit(ring, offset + len - 8);
3243	Serge	1033	intel_ring_emit(ring, MI_NOOP);
		1034	intel_ring_advance(ring);
		1035	} else {
		1036	struct drm_i915_gem_object *obj = ring->private;
		1037	u32 cs_offset = obj->gtt_offset;
		1038
		1039	if (len > I830_BATCH_LIMIT)
		1040	return -ENOSPC;
		1041
		1042	ret = intel_ring_begin(ring, 9+3);
		1043	if (ret)
		1044	return ret;
		1045	/* Blit the batch (which has now all relocs applied) to the stable batch
		1046	* scratch bo area (so that the CS never stumbles over its tlb
		1047	* invalidation bug) ... */
		1048	intel_ring_emit(ring, XY_SRC_COPY_BLT_CMD \|
		1049	XY_SRC_COPY_BLT_WRITE_ALPHA \|
		1050	XY_SRC_COPY_BLT_WRITE_RGB);
		1051	intel_ring_emit(ring, BLT_DEPTH_32 \| BLT_ROP_GXCOPY \| 4096);
2332	Serge	1052	intel_ring_emit(ring, 0);
3243	Serge	1053	intel_ring_emit(ring, (DIV_ROUND_UP(len, 4096) << 16) \| 1024);
		1054	intel_ring_emit(ring, cs_offset);
		1055	intel_ring_emit(ring, 0);
		1056	intel_ring_emit(ring, 4096);
		1057	intel_ring_emit(ring, offset);
		1058	intel_ring_emit(ring, MI_FLUSH);
		1059
		1060	/* ... and execute it. */
		1061	intel_ring_emit(ring, MI_BATCH_BUFFER);
		1062	intel_ring_emit(ring, cs_offset \| (flags & I915_DISPATCH_SECURE ? 0 : MI_BATCH_NON_SECURE));
		1063	intel_ring_emit(ring, cs_offset + len - 8);
3031	serge	1064	intel_ring_advance(ring);
3243	Serge	1065	}
3031	serge	1066
		1067	return 0;
		1068	}
		1069
		1070	static int
		1071	i915_dispatch_execbuffer(struct intel_ring_buffer *ring,
3243	Serge	1072	u32 offset, u32 len,
		1073	unsigned flags)
3031	serge	1074	{
		1075	int ret;
		1076
2332	Serge	1077	ret = intel_ring_begin(ring, 2);
		1078	if (ret)
		1079	return ret;
		1080
3031	serge	1081	intel_ring_emit(ring, MI_BATCH_BUFFER_START \| MI_BATCH_GTT);
3243	Serge	1082	intel_ring_emit(ring, offset \| (flags & I915_DISPATCH_SECURE ? 0 : MI_BATCH_NON_SECURE));
2332	Serge	1083	intel_ring_advance(ring);
		1084
		1085	return 0;
		1086	}
		1087
		1088	static void cleanup_status_page(struct intel_ring_buffer *ring)
		1089	{
		1090	struct drm_i915_gem_object *obj;
		1091
		1092	obj = ring->status_page.obj;
		1093	if (obj == NULL)
		1094	return;
		1095
3031	serge	1096	// kunmap(sg_page(obj->pages->sgl));
2344	Serge	1097	i915_gem_object_unpin(obj);
		1098	drm_gem_object_unreference(&obj->base);
2332	Serge	1099	ring->status_page.obj = NULL;
		1100	}
		1101
		1102	static int init_status_page(struct intel_ring_buffer *ring)
		1103	{
		1104	struct drm_device *dev = ring->dev;
		1105	struct drm_i915_gem_object *obj;
		1106	int ret;
		1107
		1108	obj = i915_gem_alloc_object(dev, 4096);
		1109	if (obj == NULL) {
		1110	DRM_ERROR("Failed to allocate status page\n");
		1111	ret = -ENOMEM;
		1112	goto err;
		1113	}
		1114
2352	Serge	1115	i915_gem_object_set_cache_level(obj, I915_CACHE_LLC);
2332	Serge	1116
3031	serge	1117	ret = i915_gem_object_pin(obj, 4096, true, false);
2332	Serge	1118	if (ret != 0) {
		1119	goto err_unref;
		1120	}
		1121
		1122	ring->status_page.gfx_addr = obj->gtt_offset;
3243	Serge	1123	ring->status_page.page_addr = (void*)MapIoMem((addr_t)sg_page(obj->pages->sgl),4096,PG_SW);
2332	Serge	1124	if (ring->status_page.page_addr == NULL) {
3031	serge	1125	ret = -ENOMEM;
2332	Serge	1126	goto err_unpin;
		1127	}
		1128	ring->status_page.obj = obj;
		1129	memset(ring->status_page.page_addr, 0, PAGE_SIZE);
		1130
		1131	intel_ring_setup_status_page(ring);
		1132	DRM_DEBUG_DRIVER("%s hws offset: 0x%08x\n",
		1133	ring->name, ring->status_page.gfx_addr);
		1134
		1135	return 0;
		1136
		1137	err_unpin:
2344	Serge	1138	i915_gem_object_unpin(obj);
2332	Serge	1139	err_unref:
2344	Serge	1140	drm_gem_object_unreference(&obj->base);
2332	Serge	1141	err:
		1142	return ret;
		1143	}
		1144
3243	Serge	1145	static int init_phys_hws_pga(struct intel_ring_buffer *ring)
		1146	{
		1147	struct drm_i915_private *dev_priv = ring->dev->dev_private;
		1148	u32 addr;
		1149
		1150	if (!dev_priv->status_page_dmah) {
		1151	dev_priv->status_page_dmah =
		1152	drm_pci_alloc(ring->dev, PAGE_SIZE, PAGE_SIZE);
		1153	if (!dev_priv->status_page_dmah)
		1154	return -ENOMEM;
		1155	}
		1156
		1157	addr = dev_priv->status_page_dmah->busaddr;
		1158	if (INTEL_INFO(ring->dev)->gen >= 4)
		1159	addr \|= (dev_priv->status_page_dmah->busaddr >> 28) & 0xf0;
		1160	I915_WRITE(HWS_PGA, addr);
		1161
		1162	ring->status_page.page_addr = dev_priv->status_page_dmah->vaddr;
		1163	memset(ring->status_page.page_addr, 0, PAGE_SIZE);
		1164
		1165	return 0;
		1166	}
		1167
3031	serge	1168	static int intel_init_ring_buffer(struct drm_device *dev,
2332	Serge	1169	struct intel_ring_buffer *ring)
		1170	{
2340	Serge	1171	struct drm_i915_gem_object *obj;
3031	serge	1172	struct drm_i915_private *dev_priv = dev->dev_private;
2332	Serge	1173	int ret;
2340	Serge	1174
2332	Serge	1175	ring->dev = dev;
		1176	INIT_LIST_HEAD(&ring->active_list);
		1177	INIT_LIST_HEAD(&ring->request_list);
3031	serge	1178	ring->size = 32 * PAGE_SIZE;
3243	Serge	1179	memset(ring->sync_seqno, 0, sizeof(ring->sync_seqno));
2332	Serge	1180
2352	Serge	1181	init_waitqueue_head(&ring->irq_queue);
2332	Serge	1182
		1183	if (I915_NEED_GFX_HWS(dev)) {
2340	Serge	1184	ret = init_status_page(ring);
		1185	if (ret)
		1186	return ret;
3243	Serge	1187	} else {
		1188	BUG_ON(ring->id != RCS);
		1189	ret = init_phys_hws_pga(ring);
		1190	if (ret)
		1191	return ret;
2332	Serge	1192	}
		1193
3480	Serge	1194	obj = NULL;
		1195	if (!HAS_LLC(dev))
		1196	obj = i915_gem_object_create_stolen(dev, ring->size);
		1197	if (obj == NULL)
2332	Serge	1198	obj = i915_gem_alloc_object(dev, ring->size);
		1199	if (obj == NULL) {
		1200	DRM_ERROR("Failed to allocate ringbuffer\n");
		1201	ret = -ENOMEM;
		1202	goto err_hws;
		1203	}
		1204
		1205	ring->obj = obj;
		1206
3031	serge	1207	ret = i915_gem_object_pin(obj, PAGE_SIZE, true, false);
2332	Serge	1208	if (ret)
		1209	goto err_unref;
		1210
3031	serge	1211	ret = i915_gem_object_set_to_gtt_domain(obj, true);
		1212	if (ret)
		1213	goto err_unpin;
2332	Serge	1214
3031	serge	1215	ring->virtual_start =
3480	Serge	1216	ioremap_wc(dev_priv->gtt.mappable_base + obj->gtt_offset,
3031	serge	1217	ring->size);
		1218	if (ring->virtual_start == NULL) {
2332	Serge	1219	DRM_ERROR("Failed to map ringbuffer.\n");
		1220	ret = -EINVAL;
		1221	goto err_unpin;
		1222	}
		1223
		1224	ret = ring->init(ring);
		1225	if (ret)
		1226	goto err_unmap;
		1227
		1228	/* Workaround an erratum on the i830 which causes a hang if
		1229	* the TAIL pointer points to within the last 2 cachelines
		1230	* of the buffer.
		1231	*/
		1232	ring->effective_size = ring->size;
3031	serge	1233	if (IS_I830(ring->dev) \|\| IS_845G(ring->dev))
2332	Serge	1234	ring->effective_size -= 128;
2340	Serge	1235
2332	Serge	1236	return 0;
		1237
		1238	err_unmap:
3480	Serge	1239	iounmap(ring->virtual_start);
2332	Serge	1240	err_unpin:
2344	Serge	1241	i915_gem_object_unpin(obj);
2332	Serge	1242	err_unref:
2344	Serge	1243	drm_gem_object_unreference(&obj->base);
2332	Serge	1244	ring->obj = NULL;
		1245	err_hws:
		1246	// cleanup_status_page(ring);
		1247	return ret;
		1248	}
		1249
		1250	void intel_cleanup_ring_buffer(struct intel_ring_buffer *ring)
		1251	{
		1252	struct drm_i915_private *dev_priv;
		1253	int ret;
		1254
		1255	if (ring->obj == NULL)
		1256	return;
		1257
		1258	/* Disable the ring buffer. The ring must be idle at this point */
		1259	dev_priv = ring->dev->dev_private;
3243	Serge	1260	ret = intel_ring_idle(ring);
2332	Serge	1261	if (ret)
		1262	DRM_ERROR("failed to quiesce %s whilst cleaning up: %d\n",
		1263	ring->name, ret);
		1264
		1265	I915_WRITE_CTL(ring, 0);
		1266
3480	Serge	1267	iounmap(ring->virtual_start);
2332	Serge	1268
2344	Serge	1269	i915_gem_object_unpin(ring->obj);
		1270	drm_gem_object_unreference(&ring->obj->base);
2332	Serge	1271	ring->obj = NULL;
		1272
		1273	if (ring->cleanup)
		1274	ring->cleanup(ring);
		1275
		1276	// cleanup_status_page(ring);
		1277	}
		1278
3031	serge	1279	static int intel_ring_wait_seqno(struct intel_ring_buffer *ring, u32 seqno)
2332	Serge	1280	{
3031	serge	1281	int ret;
2332	Serge	1282
3031	serge	1283	ret = i915_wait_seqno(ring, seqno);
		1284	if (!ret)
		1285	i915_gem_retire_requests_ring(ring);
		1286
		1287	return ret;
		1288	}
		1289
		1290	static int intel_ring_wait_request(struct intel_ring_buffer *ring, int n)
		1291	{
		1292	struct drm_i915_gem_request *request;
		1293	u32 seqno = 0;
		1294	int ret;
		1295
		1296	i915_gem_retire_requests_ring(ring);
		1297
		1298	if (ring->last_retired_head != -1) {
		1299	ring->head = ring->last_retired_head;
		1300	ring->last_retired_head = -1;
2332	Serge	1301	ring->space = ring_space(ring);
		1302	if (ring->space >= n)
		1303	return 0;
		1304	}
		1305
3031	serge	1306	list_for_each_entry(request, &ring->request_list, list) {
		1307	int space;
2344	Serge	1308
3031	serge	1309	if (request->tail == -1)
		1310	continue;
		1311
3243	Serge	1312	space = request->tail - (ring->tail + I915_RING_FREE_SPACE);
3031	serge	1313	if (space < 0)
		1314	space += ring->size;
		1315	if (space >= n) {
		1316	seqno = request->seqno;
		1317	break;
		1318	}
		1319
		1320	/* Consume this request in case we need more space than
		1321	* is available and so need to prevent a race between
		1322	* updating last_retired_head and direct reads of
		1323	* I915_RING_HEAD. It also provides a nice sanity check.
		1324	*/
		1325	request->tail = -1;
		1326	}
		1327
		1328	if (seqno == 0)
		1329	return -ENOSPC;
		1330
		1331	ret = intel_ring_wait_seqno(ring, seqno);
		1332	if (ret)
		1333	return ret;
		1334
		1335	if (WARN_ON(ring->last_retired_head == -1))
		1336	return -ENOSPC;
		1337
		1338	ring->head = ring->last_retired_head;
		1339	ring->last_retired_head = -1;
		1340	ring->space = ring_space(ring);
		1341	if (WARN_ON(ring->space < n))
		1342	return -ENOSPC;
		1343
		1344	return 0;
		1345	}
		1346
3243	Serge	1347	static int ring_wait_for_space(struct intel_ring_buffer *ring, int n)
3031	serge	1348	{
		1349	struct drm_device *dev = ring->dev;
		1350	struct drm_i915_private *dev_priv = dev->dev_private;
		1351	unsigned long end;
		1352	int ret;
		1353
		1354	ret = intel_ring_wait_request(ring, n);
		1355	if (ret != -ENOSPC)
		1356	return ret;
		1357
3243	Serge	1358	trace_i915_ring_wait_begin(ring);
3031	serge	1359	/* With GEM the hangcheck timer should kick us out of the loop,
		1360	* leaving it early runs the risk of corrupting GEM state (due
		1361	* to running on almost untested codepaths). But on resume
		1362	* timers don't work yet, so prevent a complete hang in that
		1363	* case by choosing an insanely large timeout. */
		1364	end = GetTimerTicks() + 60 * HZ;
		1365
2332	Serge	1366	do {
		1367	ring->head = I915_READ_HEAD(ring);
		1368	ring->space = ring_space(ring);
		1369	if (ring->space >= n) {
2351	Serge	1370	trace_i915_ring_wait_end(ring);
2332	Serge	1371	return 0;
		1372	}
		1373
		1374	msleep(1);
3031	serge	1375
3480	Serge	1376	ret = i915_gem_check_wedge(&dev_priv->gpu_error,
		1377	dev_priv->mm.interruptible);
3031	serge	1378	if (ret)
		1379	return ret;
		1380	} while (!time_after(GetTimerTicks(), end));
2351	Serge	1381	trace_i915_ring_wait_end(ring);
2332	Serge	1382	return -EBUSY;
		1383	}
		1384
3243	Serge	1385	static int intel_wrap_ring_buffer(struct intel_ring_buffer *ring)
		1386	{
		1387	uint32_t __iomem *virt;
		1388	int rem = ring->size - ring->tail;
		1389
		1390	if (ring->space < rem) {
		1391	int ret = ring_wait_for_space(ring, rem);
		1392	if (ret)
		1393	return ret;
		1394	}
		1395
		1396	virt = ring->virtual_start + ring->tail;
		1397	rem /= 4;
		1398	while (rem--)
		1399	iowrite32(MI_NOOP, virt++);
		1400
		1401	ring->tail = 0;
		1402	ring->space = ring_space(ring);
		1403
		1404	return 0;
		1405	}
		1406
		1407	int intel_ring_idle(struct intel_ring_buffer *ring)
		1408	{
		1409	u32 seqno;
		1410	int ret;
		1411
		1412	/* We need to add any requests required to flush the objects and ring */
		1413	if (ring->outstanding_lazy_request) {
		1414	ret = i915_add_request(ring, NULL, NULL);
		1415	if (ret)
		1416	return ret;
		1417	}
		1418
		1419	/* Wait upon the last request to be completed */
		1420	if (list_empty(&ring->request_list))
		1421	return 0;
		1422
		1423	seqno = list_entry(ring->request_list.prev,
		1424	struct drm_i915_gem_request,
		1425	list)->seqno;
		1426
		1427	return i915_wait_seqno(ring, seqno);
		1428	}
		1429
		1430	static int
		1431	intel_ring_alloc_seqno(struct intel_ring_buffer *ring)
		1432	{
		1433	if (ring->outstanding_lazy_request)
		1434	return 0;
		1435
		1436	return i915_gem_get_seqno(ring->dev, &ring->outstanding_lazy_request);
		1437	}
		1438
3480	Serge	1439	static int __intel_ring_begin(struct intel_ring_buffer *ring,
		1440	int bytes)
		1441	{
		1442	int ret;
		1443
		1444	if (unlikely(ring->tail + bytes > ring->effective_size)) {
		1445	ret = intel_wrap_ring_buffer(ring);
		1446	if (unlikely(ret))
		1447	return ret;
		1448	}
		1449
		1450	if (unlikely(ring->space < bytes)) {
		1451	ret = ring_wait_for_space(ring, bytes);
		1452	if (unlikely(ret))
		1453	return ret;
		1454	}
		1455
		1456	ring->space -= bytes;
		1457	return 0;
		1458	}
		1459
2332	Serge	1460	int intel_ring_begin(struct intel_ring_buffer *ring,
		1461	int num_dwords)
		1462	{
3031	serge	1463	drm_i915_private_t *dev_priv = ring->dev->dev_private;
2332	Serge	1464	int ret;
		1465
3480	Serge	1466	ret = i915_gem_check_wedge(&dev_priv->gpu_error,
		1467	dev_priv->mm.interruptible);
3031	serge	1468	if (ret)
		1469	return ret;
2332	Serge	1470
3243	Serge	1471	/* Preallocate the olr before touching the ring */
		1472	ret = intel_ring_alloc_seqno(ring);
		1473	if (ret)
		1474	return ret;
		1475
3480	Serge	1476	return __intel_ring_begin(ring, num_dwords * sizeof(uint32_t));
		1477	}
2332	Serge	1478
3480	Serge	1479	void intel_ring_init_seqno(struct intel_ring_buffer *ring, u32 seqno)
		1480	{
		1481	struct drm_i915_private *dev_priv = ring->dev->dev_private;
		1482
		1483	BUG_ON(ring->outstanding_lazy_request);
		1484
		1485	if (INTEL_INFO(ring->dev)->gen >= 6) {
		1486	I915_WRITE(RING_SYNC_0(ring->mmio_base), 0);
		1487	I915_WRITE(RING_SYNC_1(ring->mmio_base), 0);
2332	Serge	1488	}
		1489
3480	Serge	1490	ring->set_seqno(ring, seqno);
2332	Serge	1491	}
		1492
		1493	void intel_ring_advance(struct intel_ring_buffer *ring)
		1494	{
3031	serge	1495	struct drm_i915_private *dev_priv = ring->dev->dev_private;
		1496
2332	Serge	1497	ring->tail &= ring->size - 1;
3480	Serge	1498	if (dev_priv->gpu_error.stop_rings & intel_ring_flag(ring))
3031	serge	1499	return;
2332	Serge	1500	ring->write_tail(ring, ring->tail);
		1501	}
		1502
		1503
		1504	static void gen6_bsd_ring_write_tail(struct intel_ring_buffer *ring,
		1505	u32 value)
		1506	{
		1507	drm_i915_private_t *dev_priv = ring->dev->dev_private;
		1508
		1509	/* Every tail move must follow the sequence below */
3031	serge	1510
		1511	/* Disable notification that the ring is IDLE. The GT
		1512	* will then assume that it is busy and bring it out of rc6.
		1513	*/
2332	Serge	1514	I915_WRITE(GEN6_BSD_SLEEP_PSMI_CONTROL,
3031	serge	1515	_MASKED_BIT_ENABLE(GEN6_BSD_SLEEP_MSG_DISABLE));
2332	Serge	1516
3031	serge	1517	/* Clear the context id. Here be magic! */
		1518	I915_WRITE64(GEN6_BSD_RNCID, 0x0);
		1519
		1520	/* Wait for the ring not to be idle, i.e. for it to wake up. */
2332	Serge	1521	if (wait_for((I915_READ(GEN6_BSD_SLEEP_PSMI_CONTROL) &
3031	serge	1522	GEN6_BSD_SLEEP_INDICATOR) == 0,
2332	Serge	1523	50))
3031	serge	1524	DRM_ERROR("timed out waiting for the BSD ring to wake up\n");
2332	Serge	1525
3031	serge	1526	/* Now that the ring is fully powered up, update the tail */
2332	Serge	1527	I915_WRITE_TAIL(ring, value);
3031	serge	1528	POSTING_READ(RING_TAIL(ring->mmio_base));
		1529
		1530	/* Let the ring send IDLE messages to the GT again,
		1531	* and so let it sleep to conserve power when idle.
		1532	*/
2332	Serge	1533	I915_WRITE(GEN6_BSD_SLEEP_PSMI_CONTROL,
3031	serge	1534	_MASKED_BIT_DISABLE(GEN6_BSD_SLEEP_MSG_DISABLE));
2332	Serge	1535	}
		1536
		1537	static int gen6_ring_flush(struct intel_ring_buffer *ring,
		1538	u32 invalidate, u32 flush)
		1539	{
		1540	uint32_t cmd;
		1541	int ret;
		1542
		1543	ret = intel_ring_begin(ring, 4);
		1544	if (ret)
		1545	return ret;
		1546
		1547	cmd = MI_FLUSH_DW;
3243	Serge	1548	/*
		1549	* Bspec vol 1c.5 - video engine command streamer:
		1550	* "If ENABLED, all TLBs will be invalidated once the flush
		1551	* operation is complete. This bit is only valid when the
		1552	* Post-Sync Operation field is a value of 1h or 3h."
		1553	*/
2332	Serge	1554	if (invalidate & I915_GEM_GPU_DOMAINS)
3243	Serge	1555	cmd \|= MI_INVALIDATE_TLB \| MI_INVALIDATE_BSD \|
		1556	MI_FLUSH_DW_STORE_INDEX \| MI_FLUSH_DW_OP_STOREDW;
2332	Serge	1557	intel_ring_emit(ring, cmd);
3243	Serge	1558	intel_ring_emit(ring, I915_GEM_HWS_SCRATCH_ADDR \| MI_FLUSH_DW_USE_GTT);
2332	Serge	1559	intel_ring_emit(ring, 0);
		1560	intel_ring_emit(ring, MI_NOOP);
		1561	intel_ring_advance(ring);
		1562	return 0;
		1563	}
		1564
		1565	static int
3243	Serge	1566	hsw_ring_dispatch_execbuffer(struct intel_ring_buffer *ring,
		1567	u32 offset, u32 len,
		1568	unsigned flags)
		1569	{
		1570	int ret;
		1571
		1572	ret = intel_ring_begin(ring, 2);
		1573	if (ret)
		1574	return ret;
		1575
		1576	intel_ring_emit(ring,
		1577	MI_BATCH_BUFFER_START \| MI_BATCH_PPGTT_HSW \|
		1578	(flags & I915_DISPATCH_SECURE ? 0 : MI_BATCH_NON_SECURE_HSW));
		1579	/* bit0-7 is the length on GEN6+ */
		1580	intel_ring_emit(ring, offset);
		1581	intel_ring_advance(ring);
		1582
		1583	return 0;
		1584	}
		1585
		1586	static int
2332	Serge	1587	gen6_ring_dispatch_execbuffer(struct intel_ring_buffer *ring,
3243	Serge	1588	u32 offset, u32 len,
		1589	unsigned flags)
2332	Serge	1590	{
		1591	int ret;
		1592
		1593	ret = intel_ring_begin(ring, 2);
		1594	if (ret)
		1595	return ret;
		1596
3243	Serge	1597	intel_ring_emit(ring,
		1598	MI_BATCH_BUFFER_START \|
		1599	(flags & I915_DISPATCH_SECURE ? 0 : MI_BATCH_NON_SECURE_I965));
2332	Serge	1600	/* bit0-7 is the length on GEN6+ */
		1601	intel_ring_emit(ring, offset);
		1602	intel_ring_advance(ring);
		1603
		1604	return 0;
		1605	}
		1606
		1607	/* Blitter support (SandyBridge+) */
		1608
		1609	static int blt_ring_flush(struct intel_ring_buffer *ring,
		1610	u32 invalidate, u32 flush)
		1611	{
		1612	uint32_t cmd;
		1613	int ret;
		1614
3031	serge	1615	ret = intel_ring_begin(ring, 4);
2332	Serge	1616	if (ret)
		1617	return ret;
		1618
		1619	cmd = MI_FLUSH_DW;
3243	Serge	1620	/*
		1621	* Bspec vol 1c.3 - blitter engine command streamer:
		1622	* "If ENABLED, all TLBs will be invalidated once the flush
		1623	* operation is complete. This bit is only valid when the
		1624	* Post-Sync Operation field is a value of 1h or 3h."
		1625	*/
2332	Serge	1626	if (invalidate & I915_GEM_DOMAIN_RENDER)
3243	Serge	1627	cmd \|= MI_INVALIDATE_TLB \| MI_FLUSH_DW_STORE_INDEX \|
		1628	MI_FLUSH_DW_OP_STOREDW;
2332	Serge	1629	intel_ring_emit(ring, cmd);
3243	Serge	1630	intel_ring_emit(ring, I915_GEM_HWS_SCRATCH_ADDR \| MI_FLUSH_DW_USE_GTT);
2332	Serge	1631	intel_ring_emit(ring, 0);
		1632	intel_ring_emit(ring, MI_NOOP);
		1633	intel_ring_advance(ring);
		1634	return 0;
		1635	}
		1636
		1637	int intel_init_render_ring_buffer(struct drm_device *dev)
		1638	{
		1639	drm_i915_private_t *dev_priv = dev->dev_private;
		1640	struct intel_ring_buffer *ring = &dev_priv->ring[RCS];
2340	Serge	1641
3031	serge	1642	ring->name = "render ring";
		1643	ring->id = RCS;
		1644	ring->mmio_base = RENDER_RING_BASE;
		1645
2332	Serge	1646	if (INTEL_INFO(dev)->gen >= 6) {
2339	Serge	1647	ring->add_request = gen6_add_request;
3031	serge	1648	ring->flush = gen7_render_ring_flush;
		1649	if (INTEL_INFO(dev)->gen == 6)
2342	Serge	1650	ring->flush = gen6_render_ring_flush;
3031	serge	1651	ring->irq_get = gen6_ring_get_irq;
		1652	ring->irq_put = gen6_ring_put_irq;
		1653	ring->irq_enable_mask = GT_USER_INTERRUPT;
2342	Serge	1654	ring->get_seqno = gen6_ring_get_seqno;
3480	Serge	1655	ring->set_seqno = ring_set_seqno;
3031	serge	1656	ring->sync_to = gen6_ring_sync;
		1657	ring->semaphore_register[0] = MI_SEMAPHORE_SYNC_INVALID;
		1658	ring->semaphore_register[1] = MI_SEMAPHORE_SYNC_RV;
		1659	ring->semaphore_register[2] = MI_SEMAPHORE_SYNC_RB;
		1660	ring->signal_mbox[0] = GEN6_VRSYNC;
		1661	ring->signal_mbox[1] = GEN6_BRSYNC;
2332	Serge	1662	} else if (IS_GEN5(dev)) {
2339	Serge	1663	ring->add_request = pc_render_add_request;
3031	serge	1664	ring->flush = gen4_render_ring_flush;
2342	Serge	1665	ring->get_seqno = pc_render_get_seqno;
3480	Serge	1666	ring->set_seqno = pc_render_set_seqno;
3031	serge	1667	ring->irq_get = gen5_ring_get_irq;
		1668	ring->irq_put = gen5_ring_put_irq;
		1669	ring->irq_enable_mask = GT_USER_INTERRUPT \| GT_PIPE_NOTIFY;
		1670	} else {
		1671	ring->add_request = i9xx_add_request;
		1672	if (INTEL_INFO(dev)->gen < 4)
		1673	ring->flush = gen2_render_ring_flush;
		1674	else
		1675	ring->flush = gen4_render_ring_flush;
		1676	ring->get_seqno = ring_get_seqno;
3480	Serge	1677	ring->set_seqno = ring_set_seqno;
3031	serge	1678	if (IS_GEN2(dev)) {
		1679	ring->irq_get = i8xx_ring_get_irq;
		1680	ring->irq_put = i8xx_ring_put_irq;
		1681	} else {
		1682	ring->irq_get = i9xx_ring_get_irq;
		1683	ring->irq_put = i9xx_ring_put_irq;
		1684	}
		1685	ring->irq_enable_mask = I915_USER_INTERRUPT;
2332	Serge	1686	}
3031	serge	1687	ring->write_tail = ring_write_tail;
3243	Serge	1688	if (IS_HASWELL(dev))
		1689	ring->dispatch_execbuffer = hsw_ring_dispatch_execbuffer;
		1690	else if (INTEL_INFO(dev)->gen >= 6)
3031	serge	1691	ring->dispatch_execbuffer = gen6_ring_dispatch_execbuffer;
		1692	else if (INTEL_INFO(dev)->gen >= 4)
		1693	ring->dispatch_execbuffer = i965_dispatch_execbuffer;
		1694	else if (IS_I830(dev) \|\| IS_845G(dev))
		1695	ring->dispatch_execbuffer = i830_dispatch_execbuffer;
		1696	else
		1697	ring->dispatch_execbuffer = i915_dispatch_execbuffer;
		1698	ring->init = init_render_ring;
		1699	ring->cleanup = render_ring_cleanup;
2332	Serge	1700
3243	Serge	1701	/* Workaround batchbuffer to combat CS tlb bug. */
		1702	if (HAS_BROKEN_CS_TLB(dev)) {
		1703	struct drm_i915_gem_object *obj;
		1704	int ret;
3031	serge	1705
3243	Serge	1706	obj = i915_gem_alloc_object(dev, I830_BATCH_LIMIT);
		1707	if (obj == NULL) {
		1708	DRM_ERROR("Failed to allocate batch bo\n");
		1709	return -ENOMEM;
		1710	}
		1711
		1712	ret = i915_gem_object_pin(obj, 0, true, false);
		1713	if (ret != 0) {
		1714	drm_gem_object_unreference(&obj->base);
		1715	DRM_ERROR("Failed to ping batch bo\n");
		1716	return ret;
		1717	}
		1718
		1719	ring->private = obj;
2332	Serge	1720	}
2340	Serge	1721
2332	Serge	1722	return intel_init_ring_buffer(dev, ring);
		1723	}
		1724
3243	Serge	1725	#if 0
		1726	int intel_render_ring_init_dri(struct drm_device *dev, u64 start, u32 size)
		1727	{
		1728	drm_i915_private_t *dev_priv = dev->dev_private;
		1729	struct intel_ring_buffer *ring = &dev_priv->ring[RCS];
		1730	int ret;
2332	Serge	1731
3243	Serge	1732	ring->name = "render ring";
		1733	ring->id = RCS;
		1734	ring->mmio_base = RENDER_RING_BASE;
		1735
		1736	if (INTEL_INFO(dev)->gen >= 6) {
		1737	/* non-kms not supported on gen6+ */
		1738	return -ENODEV;
		1739	}
		1740
		1741	/* Note: gem is not supported on gen5/ilk without kms (the corresponding
		1742	* gem_init ioctl returns with -ENODEV). Hence we do not need to set up
		1743	* the special gen5 functions. */
		1744	ring->add_request = i9xx_add_request;
		1745	if (INTEL_INFO(dev)->gen < 4)
		1746	ring->flush = gen2_render_ring_flush;
		1747	else
		1748	ring->flush = gen4_render_ring_flush;
		1749	ring->get_seqno = ring_get_seqno;
3480	Serge	1750	ring->set_seqno = ring_set_seqno;
3243	Serge	1751	if (IS_GEN2(dev)) {
		1752	ring->irq_get = i8xx_ring_get_irq;
		1753	ring->irq_put = i8xx_ring_put_irq;
		1754	} else {
		1755	ring->irq_get = i9xx_ring_get_irq;
		1756	ring->irq_put = i9xx_ring_put_irq;
		1757	}
		1758	ring->irq_enable_mask = I915_USER_INTERRUPT;
		1759	ring->write_tail = ring_write_tail;
		1760	if (INTEL_INFO(dev)->gen >= 4)
		1761	ring->dispatch_execbuffer = i965_dispatch_execbuffer;
		1762	else if (IS_I830(dev) \|\| IS_845G(dev))
		1763	ring->dispatch_execbuffer = i830_dispatch_execbuffer;
		1764	else
		1765	ring->dispatch_execbuffer = i915_dispatch_execbuffer;
		1766	ring->init = init_render_ring;
		1767	ring->cleanup = render_ring_cleanup;
		1768
		1769	ring->dev = dev;
		1770	INIT_LIST_HEAD(&ring->active_list);
		1771	INIT_LIST_HEAD(&ring->request_list);
		1772
		1773	ring->size = size;
		1774	ring->effective_size = ring->size;
		1775	if (IS_I830(ring->dev) \|\| IS_845G(ring->dev))
		1776	ring->effective_size -= 128;
		1777
		1778	ring->virtual_start = ioremap_wc(start, size);
		1779	if (ring->virtual_start == NULL) {
		1780	DRM_ERROR("can not ioremap virtual address for"
		1781	" ring buffer\n");
		1782	return -ENOMEM;
		1783	}
		1784
		1785	if (!I915_NEED_GFX_HWS(dev)) {
		1786	ret = init_phys_hws_pga(ring);
		1787	if (ret)
		1788	return ret;
		1789	}
		1790
		1791	return 0;
		1792	}
		1793	#endif
		1794
2332	Serge	1795	int intel_init_bsd_ring_buffer(struct drm_device *dev)
		1796	{
		1797	drm_i915_private_t *dev_priv = dev->dev_private;
		1798	struct intel_ring_buffer *ring = &dev_priv->ring[VCS];
		1799
3031	serge	1800	ring->name = "bsd ring";
		1801	ring->id = VCS;
2332	Serge	1802
3031	serge	1803	ring->write_tail = ring_write_tail;
		1804	if (IS_GEN6(dev) \|\| IS_GEN7(dev)) {
		1805	ring->mmio_base = GEN6_BSD_RING_BASE;
		1806	/* gen6 bsd needs a special wa for tail updates */
		1807	if (IS_GEN6(dev))
		1808	ring->write_tail = gen6_bsd_ring_write_tail;
		1809	ring->flush = gen6_ring_flush;
		1810	ring->add_request = gen6_add_request;
		1811	ring->get_seqno = gen6_ring_get_seqno;
3480	Serge	1812	ring->set_seqno = ring_set_seqno;
3031	serge	1813	ring->irq_enable_mask = GEN6_BSD_USER_INTERRUPT;
		1814	ring->irq_get = gen6_ring_get_irq;
		1815	ring->irq_put = gen6_ring_put_irq;
		1816	ring->dispatch_execbuffer = gen6_ring_dispatch_execbuffer;
		1817	ring->sync_to = gen6_ring_sync;
		1818	ring->semaphore_register[0] = MI_SEMAPHORE_SYNC_VR;
		1819	ring->semaphore_register[1] = MI_SEMAPHORE_SYNC_INVALID;
		1820	ring->semaphore_register[2] = MI_SEMAPHORE_SYNC_VB;
		1821	ring->signal_mbox[0] = GEN6_RVSYNC;
		1822	ring->signal_mbox[1] = GEN6_BVSYNC;
		1823	} else {
		1824	ring->mmio_base = BSD_RING_BASE;
		1825	ring->flush = bsd_ring_flush;
		1826	ring->add_request = i9xx_add_request;
		1827	ring->get_seqno = ring_get_seqno;
3480	Serge	1828	ring->set_seqno = ring_set_seqno;
3031	serge	1829	if (IS_GEN5(dev)) {
		1830	ring->irq_enable_mask = GT_BSD_USER_INTERRUPT;
		1831	ring->irq_get = gen5_ring_get_irq;
		1832	ring->irq_put = gen5_ring_put_irq;
		1833	} else {
		1834	ring->irq_enable_mask = I915_BSD_USER_INTERRUPT;
		1835	ring->irq_get = i9xx_ring_get_irq;
		1836	ring->irq_put = i9xx_ring_put_irq;
		1837	}
		1838	ring->dispatch_execbuffer = i965_dispatch_execbuffer;
		1839	}
		1840	ring->init = init_ring_common;
		1841
2332	Serge	1842	return intel_init_ring_buffer(dev, ring);
		1843	}
		1844
		1845	int intel_init_blt_ring_buffer(struct drm_device *dev)
		1846	{
		1847	drm_i915_private_t *dev_priv = dev->dev_private;
		1848	struct intel_ring_buffer *ring = &dev_priv->ring[BCS];
		1849
3031	serge	1850	ring->name = "blitter ring";
		1851	ring->id = BCS;
2332	Serge	1852
3031	serge	1853	ring->mmio_base = BLT_RING_BASE;
		1854	ring->write_tail = ring_write_tail;
		1855	ring->flush = blt_ring_flush;
		1856	ring->add_request = gen6_add_request;
		1857	ring->get_seqno = gen6_ring_get_seqno;
3480	Serge	1858	ring->set_seqno = ring_set_seqno;
3031	serge	1859	ring->irq_enable_mask = GEN6_BLITTER_USER_INTERRUPT;
		1860	ring->irq_get = gen6_ring_get_irq;
		1861	ring->irq_put = gen6_ring_put_irq;
		1862	ring->dispatch_execbuffer = gen6_ring_dispatch_execbuffer;
		1863	ring->sync_to = gen6_ring_sync;
		1864	ring->semaphore_register[0] = MI_SEMAPHORE_SYNC_BR;
		1865	ring->semaphore_register[1] = MI_SEMAPHORE_SYNC_BV;
		1866	ring->semaphore_register[2] = MI_SEMAPHORE_SYNC_INVALID;
		1867	ring->signal_mbox[0] = GEN6_RBSYNC;
		1868	ring->signal_mbox[1] = GEN6_VBSYNC;
		1869	ring->init = init_ring_common;
		1870
2332	Serge	1871	return intel_init_ring_buffer(dev, ring);
		1872	}
3031	serge	1873
		1874	int
		1875	intel_ring_flush_all_caches(struct intel_ring_buffer *ring)
		1876	{
		1877	int ret;
		1878
		1879	if (!ring->gpu_caches_dirty)
		1880	return 0;
		1881
		1882	ret = ring->flush(ring, 0, I915_GEM_GPU_DOMAINS);
		1883	if (ret)
		1884	return ret;
		1885
		1886	trace_i915_gem_ring_flush(ring, 0, I915_GEM_GPU_DOMAINS);
		1887
		1888	ring->gpu_caches_dirty = false;
		1889	return 0;
		1890	}
		1891
		1892	int
		1893	intel_ring_invalidate_all_caches(struct intel_ring_buffer *ring)
		1894	{
		1895	uint32_t flush_domains;
		1896	int ret;
		1897
		1898	flush_domains = 0;
		1899	if (ring->gpu_caches_dirty)
		1900	flush_domains = I915_GEM_GPU_DOMAINS;
		1901
		1902	ret = ring->flush(ring, I915_GEM_GPU_DOMAINS, flush_domains);
		1903	if (ret)
		1904	return ret;
		1905
		1906	trace_i915_gem_ring_flush(ring, I915_GEM_GPU_DOMAINS, flush_domains);
		1907
		1908	ring->gpu_caches_dirty = false;
		1909	return 0;
		1910	}

Subversion Repositories Kolibri OS

(root)/drivers/video/drm/i915/intel_ringbuffer.c – Rev 3480