WebSVN – Kolibri OS – Blame – /drivers/video/drm/i915/intel_ringbuffer.c

Rev	Author	Line No.	Line
2332	Serge	1	/*
		2	* Copyright © 2008-2010 Intel Corporation
		3	*
		4	* Permission is hereby granted, free of charge, to any person obtaining a
		5	* copy of this software and associated documentation files (the "Software"),
		6	* to deal in the Software without restriction, including without limitation
		7	* the rights to use, copy, modify, merge, publish, distribute, sublicense,
		8	* and/or sell copies of the Software, and to permit persons to whom the
		9	* Software is furnished to do so, subject to the following conditions:
		10	*
		11	* The above copyright notice and this permission notice (including the next
		12	* paragraph) shall be included in all copies or substantial portions of the
		13	* Software.
		14	*
		15	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
		16	* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
		17	* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
		18	* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
		19	* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
		20	* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
		21	* IN THE SOFTWARE.
		22	*
		23	* Authors:
		24	* Eric Anholt
		25	* Zou Nan hai
		26	* Xiang Hai hao
		27	*
		28	*/
		29
3031	serge	30	#include
2332	Serge	31	#include "i915_drv.h"
3031	serge	32	#include
2351	Serge	33	#include "i915_trace.h"
2332	Serge	34	#include "intel_drv.h"
		35
2342	Serge	36	/*
		37	* 965+ support PIPE_CONTROL commands, which provide finer grained control
		38	* over cache flushing.
		39	*/
		40	struct pipe_control {
		41	struct drm_i915_gem_object *obj;
		42	volatile u32 *cpu_page;
		43	u32 gtt_offset;
		44	};
		45
2332	Serge	46	static inline int ring_space(struct intel_ring_buffer *ring)
		47	{
3243	Serge	48	int space = (ring->head & HEAD_ADDR) - (ring->tail + I915_RING_FREE_SPACE);
2332	Serge	49	if (space < 0)
		50	space += ring->size;
		51	return space;
		52	}
		53
3031	serge	54	static int
		55	gen2_render_ring_flush(struct intel_ring_buffer *ring,
		56	u32 invalidate_domains,
		57	u32 flush_domains)
2332	Serge	58	{
3031	serge	59	u32 cmd;
		60	int ret;
2332	Serge	61
3031	serge	62	cmd = MI_FLUSH;
		63	if (((invalidate_domains\|flush_domains) & I915_GEM_DOMAIN_RENDER) == 0)
		64	cmd \|= MI_NO_WRITE_FLUSH;
2332	Serge	65
3031	serge	66	if (invalidate_domains & I915_GEM_DOMAIN_SAMPLER)
		67	cmd \|= MI_READ_FLUSH;
2332	Serge	68
3031	serge	69	ret = intel_ring_begin(ring, 2);
		70	if (ret)
		71	return ret;
		72
		73	intel_ring_emit(ring, cmd);
		74	intel_ring_emit(ring, MI_NOOP);
		75	intel_ring_advance(ring);
		76
		77	return 0;
2332	Serge	78	}
		79
		80	static int
3031	serge	81	gen4_render_ring_flush(struct intel_ring_buffer *ring,
2332	Serge	82	u32 invalidate_domains,
		83	u32 flush_domains)
		84	{
		85	struct drm_device *dev = ring->dev;
		86	u32 cmd;
		87	int ret;
		88
		89	/*
		90	* read/write caches:
		91	*
		92	* I915_GEM_DOMAIN_RENDER is always invalidated, but is
		93	* only flushed if MI_NO_WRITE_FLUSH is unset. On 965, it is
		94	* also flushed at 2d versus 3d pipeline switches.
		95	*
		96	* read-only caches:
		97	*
		98	* I915_GEM_DOMAIN_SAMPLER is flushed on pre-965 if
		99	* MI_READ_FLUSH is set, and is always flushed on 965.
		100	*
		101	* I915_GEM_DOMAIN_COMMAND may not exist?
		102	*
		103	* I915_GEM_DOMAIN_INSTRUCTION, which exists on 965, is
		104	* invalidated when MI_EXE_FLUSH is set.
		105	*
		106	* I915_GEM_DOMAIN_VERTEX, which exists on 965, is
		107	* invalidated with every MI_FLUSH.
		108	*
		109	* TLBs:
		110	*
		111	* On 965, TLBs associated with I915_GEM_DOMAIN_COMMAND
		112	* and I915_GEM_DOMAIN_CPU in are invalidated at PTE write and
		113	* I915_GEM_DOMAIN_RENDER and I915_GEM_DOMAIN_SAMPLER
		114	* are flushed at any MI_FLUSH.
		115	*/
		116
		117	cmd = MI_FLUSH \| MI_NO_WRITE_FLUSH;
3031	serge	118	if ((invalidate_domains\|flush_domains) & I915_GEM_DOMAIN_RENDER)
2332	Serge	119	cmd &= ~MI_NO_WRITE_FLUSH;
		120	if (invalidate_domains & I915_GEM_DOMAIN_INSTRUCTION)
		121	cmd \|= MI_EXE_FLUSH;
		122
		123	if (invalidate_domains & I915_GEM_DOMAIN_COMMAND &&
		124	(IS_G4X(dev) \|\| IS_GEN5(dev)))
		125	cmd \|= MI_INVALIDATE_ISP;
		126
		127	ret = intel_ring_begin(ring, 2);
		128	if (ret)
		129	return ret;
		130
		131	intel_ring_emit(ring, cmd);
		132	intel_ring_emit(ring, MI_NOOP);
		133	intel_ring_advance(ring);
		134
		135	return 0;
		136	}
		137
2342	Serge	138	/**
		139	* Emits a PIPE_CONTROL with a non-zero post-sync operation, for
		140	* implementing two workarounds on gen6. From section 1.4.7.1
		141	* "PIPE_CONTROL" of the Sandy Bridge PRM volume 2 part 1:
		142	*
		143	* [DevSNB-C+{W/A}] Before any depth stall flush (including those
		144	* produced by non-pipelined state commands), software needs to first
		145	* send a PIPE_CONTROL with no bits set except Post-Sync Operation !=
		146	* 0.
		147	*
		148	* [Dev-SNB{W/A}]: Before a PIPE_CONTROL with Write Cache Flush Enable
		149	* =1, a PIPE_CONTROL with any non-zero post-sync-op is required.
		150	*
		151	* And the workaround for these two requires this workaround first:
		152	*
		153	* [Dev-SNB{W/A}]: Pipe-control with CS-stall bit set must be sent
		154	* BEFORE the pipe-control with a post-sync op and no write-cache
		155	* flushes.
		156	*
		157	* And this last workaround is tricky because of the requirements on
		158	* that bit. From section 1.4.7.2.3 "Stall" of the Sandy Bridge PRM
		159	* volume 2 part 1:
		160	*
		161	* "1 of the following must also be set:
		162	* - Render Target Cache Flush Enable ([12] of DW1)
		163	* - Depth Cache Flush Enable ([0] of DW1)
		164	* - Stall at Pixel Scoreboard ([1] of DW1)
		165	* - Depth Stall ([13] of DW1)
		166	* - Post-Sync Operation ([13] of DW1)
		167	* - Notify Enable ([8] of DW1)"
		168	*
		169	* The cache flushes require the workaround flush that triggered this
		170	* one, so we can't use it. Depth stall would trigger the same.
		171	* Post-sync nonzero is what triggered this second workaround, so we
		172	* can't use that one either. Notify enable is IRQs, which aren't
		173	* really our business. That leaves only stall at scoreboard.
		174	*/
		175	static int
		176	intel_emit_post_sync_nonzero_flush(struct intel_ring_buffer *ring)
		177	{
		178	struct pipe_control *pc = ring->private;
		179	u32 scratch_addr = pc->gtt_offset + 128;
		180	int ret;
		181
		182
		183	ret = intel_ring_begin(ring, 6);
		184	if (ret)
		185	return ret;
		186
		187	intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(5));
		188	intel_ring_emit(ring, PIPE_CONTROL_CS_STALL \|
		189	PIPE_CONTROL_STALL_AT_SCOREBOARD);
		190	intel_ring_emit(ring, scratch_addr \| PIPE_CONTROL_GLOBAL_GTT); /* address */
		191	intel_ring_emit(ring, 0); /* low dword */
		192	intel_ring_emit(ring, 0); /* high dword */
		193	intel_ring_emit(ring, MI_NOOP);
		194	intel_ring_advance(ring);
		195
		196	ret = intel_ring_begin(ring, 6);
		197	if (ret)
		198	return ret;
		199
		200	intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(5));
		201	intel_ring_emit(ring, PIPE_CONTROL_QW_WRITE);
		202	intel_ring_emit(ring, scratch_addr \| PIPE_CONTROL_GLOBAL_GTT); /* address */
		203	intel_ring_emit(ring, 0);
		204	intel_ring_emit(ring, 0);
		205	intel_ring_emit(ring, MI_NOOP);
		206	intel_ring_advance(ring);
		207
		208	return 0;
		209	}
		210
		211	static int
		212	gen6_render_ring_flush(struct intel_ring_buffer *ring,
		213	u32 invalidate_domains, u32 flush_domains)
		214	{
		215	u32 flags = 0;
		216	struct pipe_control *pc = ring->private;
		217	u32 scratch_addr = pc->gtt_offset + 128;
		218	int ret;
		219
		220	/* Force SNB workarounds for PIPE_CONTROL flushes */
3031	serge	221	ret = intel_emit_post_sync_nonzero_flush(ring);
		222	if (ret)
		223	return ret;
2342	Serge	224
		225	/* Just flush everything. Experiments have shown that reducing the
		226	* number of bits based on the write domains has little performance
		227	* impact.
		228	*/
3031	serge	229	if (flush_domains) {
		230	flags \|= PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH;
		231	flags \|= PIPE_CONTROL_DEPTH_CACHE_FLUSH;
		232	/*
		233	* Ensure that any following seqno writes only happen
		234	* when the render cache is indeed flushed.
		235	*/
		236	flags \|= PIPE_CONTROL_CS_STALL;
		237	}
		238	if (invalidate_domains) {
		239	flags \|= PIPE_CONTROL_TLB_INVALIDATE;
		240	flags \|= PIPE_CONTROL_INSTRUCTION_CACHE_INVALIDATE;
		241	flags \|= PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE;
		242	flags \|= PIPE_CONTROL_VF_CACHE_INVALIDATE;
		243	flags \|= PIPE_CONTROL_CONST_CACHE_INVALIDATE;
		244	flags \|= PIPE_CONTROL_STATE_CACHE_INVALIDATE;
		245	/*
		246	* TLB invalidate requires a post-sync write.
		247	*/
3243	Serge	248	flags \|= PIPE_CONTROL_QW_WRITE \| PIPE_CONTROL_CS_STALL;
3031	serge	249	}
		250
		251	ret = intel_ring_begin(ring, 4);
		252	if (ret)
		253	return ret;
		254
		255	intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(4));
		256	intel_ring_emit(ring, flags);
		257	intel_ring_emit(ring, scratch_addr \| PIPE_CONTROL_GLOBAL_GTT);
		258	intel_ring_emit(ring, 0);
		259	intel_ring_advance(ring);
		260
		261	return 0;
		262	}
		263
		264	static int
		265	gen7_render_ring_cs_stall_wa(struct intel_ring_buffer *ring)
		266	{
		267	int ret;
		268
		269	ret = intel_ring_begin(ring, 4);
		270	if (ret)
		271	return ret;
		272
		273	intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(4));
		274	intel_ring_emit(ring, PIPE_CONTROL_CS_STALL \|
		275	PIPE_CONTROL_STALL_AT_SCOREBOARD);
		276	intel_ring_emit(ring, 0);
		277	intel_ring_emit(ring, 0);
		278	intel_ring_advance(ring);
		279
		280	return 0;
		281	}
		282
		283	static int
		284	gen7_render_ring_flush(struct intel_ring_buffer *ring,
		285	u32 invalidate_domains, u32 flush_domains)
		286	{
		287	u32 flags = 0;
		288	struct pipe_control *pc = ring->private;
		289	u32 scratch_addr = pc->gtt_offset + 128;
		290	int ret;
		291
		292	/*
		293	* Ensure that any following seqno writes only happen when the render
		294	* cache is indeed flushed.
		295	*
		296	* Workaround: 4th PIPE_CONTROL command (except the ones with only
		297	* read-cache invalidate bits set) must have the CS_STALL bit set. We
		298	* don't try to be clever and just set it unconditionally.
		299	*/
		300	flags \|= PIPE_CONTROL_CS_STALL;
		301
		302	/* Just flush everything. Experiments have shown that reducing the
		303	* number of bits based on the write domains has little performance
		304	* impact.
		305	*/
		306	if (flush_domains) {
2342	Serge	307	flags \|= PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH;
3031	serge	308	flags \|= PIPE_CONTROL_DEPTH_CACHE_FLUSH;
		309	}
		310	if (invalidate_domains) {
		311	flags \|= PIPE_CONTROL_TLB_INVALIDATE;
2342	Serge	312	flags \|= PIPE_CONTROL_INSTRUCTION_CACHE_INVALIDATE;
		313	flags \|= PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE;
		314	flags \|= PIPE_CONTROL_VF_CACHE_INVALIDATE;
		315	flags \|= PIPE_CONTROL_CONST_CACHE_INVALIDATE;
		316	flags \|= PIPE_CONTROL_STATE_CACHE_INVALIDATE;
3031	serge	317	/*
		318	* TLB invalidate requires a post-sync write.
		319	*/
		320	flags \|= PIPE_CONTROL_QW_WRITE;
3480	Serge	321	flags \|= PIPE_CONTROL_GLOBAL_GTT_IVB;
2342	Serge	322
3031	serge	323	/* Workaround: we must issue a pipe_control with CS-stall bit
		324	* set before a pipe_control command that has the state cache
		325	* invalidate bit set. */
		326	gen7_render_ring_cs_stall_wa(ring);
		327	}
		328
		329	ret = intel_ring_begin(ring, 4);
2342	Serge	330	if (ret)
		331	return ret;
		332
3031	serge	333	intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(4));
2342	Serge	334	intel_ring_emit(ring, flags);
3480	Serge	335	intel_ring_emit(ring, scratch_addr);
3031	serge	336	intel_ring_emit(ring, 0);
2342	Serge	337	intel_ring_advance(ring);
		338
		339	return 0;
		340	}
		341
2332	Serge	342	static void ring_write_tail(struct intel_ring_buffer *ring,
		343	u32 value)
		344	{
		345	drm_i915_private_t *dev_priv = ring->dev->dev_private;
		346	I915_WRITE_TAIL(ring, value);
		347	}
		348
		349	u32 intel_ring_get_active_head(struct intel_ring_buffer *ring)
		350	{
		351	drm_i915_private_t *dev_priv = ring->dev->dev_private;
		352	u32 acthd_reg = INTEL_INFO(ring->dev)->gen >= 4 ?
		353	RING_ACTHD(ring->mmio_base) : ACTHD;
		354
		355	return I915_READ(acthd_reg);
		356	}
		357
		358	static int init_ring_common(struct intel_ring_buffer *ring)
		359	{
3031	serge	360	struct drm_device *dev = ring->dev;
		361	drm_i915_private_t *dev_priv = dev->dev_private;
2332	Serge	362	struct drm_i915_gem_object *obj = ring->obj;
3031	serge	363	int ret = 0;
2332	Serge	364	u32 head;
		365
3031	serge	366	if (HAS_FORCE_WAKE(dev))
		367	gen6_gt_force_wake_get(dev_priv);
		368
2332	Serge	369	/* Stop the ring if it's running. */
		370	I915_WRITE_CTL(ring, 0);
		371	I915_WRITE_HEAD(ring, 0);
		372	ring->write_tail(ring, 0);
		373
		374	head = I915_READ_HEAD(ring) & HEAD_ADDR;
		375
		376	/* G45 ring initialization fails to reset head to zero */
		377	if (head != 0) {
		378	DRM_DEBUG_KMS("%s head not reset to zero "
		379	"ctl %08x head %08x tail %08x start %08x\n",
		380	ring->name,
		381	I915_READ_CTL(ring),
		382	I915_READ_HEAD(ring),
		383	I915_READ_TAIL(ring),
		384	I915_READ_START(ring));
		385
		386	I915_WRITE_HEAD(ring, 0);
		387
		388	if (I915_READ_HEAD(ring) & HEAD_ADDR) {
		389	DRM_ERROR("failed to set %s head to zero "
		390	"ctl %08x head %08x tail %08x start %08x\n",
		391	ring->name,
		392	I915_READ_CTL(ring),
		393	I915_READ_HEAD(ring),
		394	I915_READ_TAIL(ring),
		395	I915_READ_START(ring));
		396	}
		397	}
		398
3031	serge	399	/* Initialize the ring. This must happen _after_ we've cleared the ring
		400	* registers with the above sequence (the readback of the HEAD registers
		401	* also enforces ordering), otherwise the hw might lose the new ring
		402	* register values. */
		403	I915_WRITE_START(ring, obj->gtt_offset);
2332	Serge	404	I915_WRITE_CTL(ring,
		405	((ring->size - PAGE_SIZE) & RING_NR_PAGES)
3031	serge	406	\| RING_VALID);
2332	Serge	407
		408	/* If the head is still not zero, the ring is dead */
3031	serge	409	if (wait_for((I915_READ_CTL(ring) & RING_VALID) != 0 &&
		410	I915_READ_START(ring) == obj->gtt_offset &&
		411	(I915_READ_HEAD(ring) & HEAD_ADDR) == 0, 50)) {
2332	Serge	412	DRM_ERROR("%s initialization failed "
		413	"ctl %08x head %08x tail %08x start %08x\n",
		414	ring->name,
		415	I915_READ_CTL(ring),
		416	I915_READ_HEAD(ring),
		417	I915_READ_TAIL(ring),
		418	I915_READ_START(ring));
3031	serge	419	ret = -EIO;
		420	goto out;
2332	Serge	421	}
		422
3031	serge	423	ring->head = I915_READ_HEAD(ring);
		424	ring->tail = I915_READ_TAIL(ring) & TAIL_ADDR;
		425	ring->space = ring_space(ring);
		426	ring->last_retired_head = -1;
2332	Serge	427
3031	serge	428	out:
		429	if (HAS_FORCE_WAKE(dev))
		430	gen6_gt_force_wake_put(dev_priv);
2332	Serge	431
3031	serge	432	return ret;
2332	Serge	433	}
		434
		435	static int
		436	init_pipe_control(struct intel_ring_buffer *ring)
		437	{
		438	struct pipe_control *pc;
		439	struct drm_i915_gem_object *obj;
		440	int ret;
		441
		442	if (ring->private)
		443	return 0;
		444
		445	pc = kmalloc(sizeof(*pc), GFP_KERNEL);
		446	if (!pc)
		447	return -ENOMEM;
		448
		449	obj = i915_gem_alloc_object(ring->dev, 4096);
		450	if (obj == NULL) {
		451	DRM_ERROR("Failed to allocate seqno page\n");
		452	ret = -ENOMEM;
		453	goto err;
		454	}
		455
2352	Serge	456	i915_gem_object_set_cache_level(obj, I915_CACHE_LLC);
2332	Serge	457
3031	serge	458	ret = i915_gem_object_pin(obj, 4096, true, false);
2332	Serge	459	if (ret)
		460	goto err_unref;
		461
		462	pc->gtt_offset = obj->gtt_offset;
3243	Serge	463	pc->cpu_page = (void*)MapIoMem((addr_t)sg_page(obj->pages->sgl),4096, PG_SW);
2332	Serge	464	if (pc->cpu_page == NULL)
		465	goto err_unpin;
		466
3480	Serge	467	DRM_DEBUG_DRIVER("%s pipe control offset: 0x%08x\n",
		468	ring->name, pc->gtt_offset);
		469
2332	Serge	470	pc->obj = obj;
		471	ring->private = pc;
		472	return 0;
		473
		474	err_unpin:
2344	Serge	475	i915_gem_object_unpin(obj);
2332	Serge	476	err_unref:
2344	Serge	477	drm_gem_object_unreference(&obj->base);
2332	Serge	478	err:
		479	kfree(pc);
		480	return ret;
		481	}
		482
		483	static void
		484	cleanup_pipe_control(struct intel_ring_buffer *ring)
		485	{
		486	struct pipe_control *pc = ring->private;
		487	struct drm_i915_gem_object *obj;
		488
		489	if (!ring->private)
		490	return;
		491
		492	obj = pc->obj;
2339	Serge	493	// kunmap(obj->pages[0]);
2344	Serge	494	i915_gem_object_unpin(obj);
		495	drm_gem_object_unreference(&obj->base);
2332	Serge	496
		497	kfree(pc);
		498	ring->private = NULL;
		499	}
		500
		501	static int init_render_ring(struct intel_ring_buffer *ring)
		502	{
		503	struct drm_device *dev = ring->dev;
		504	struct drm_i915_private *dev_priv = dev->dev_private;
		505	int ret = init_ring_common(ring);
		506
3746	Serge	507	ENTER();
		508
3243	Serge	509	if (INTEL_INFO(dev)->gen > 3)
3031	serge	510	I915_WRITE(MI_MODE, _MASKED_BIT_ENABLE(VS_TIMER_DISPATCH));
3243	Serge	511
		512	/* We need to disable the AsyncFlip performance optimisations in order
		513	* to use MI_WAIT_FOR_EVENT within the CS. It should already be
		514	* programmed to '1' on all products.
		515	*/
		516	if (INTEL_INFO(dev)->gen >= 6)
		517	I915_WRITE(MI_MODE, _MASKED_BIT_ENABLE(ASYNC_FLIP_PERF_DISABLE));
		518
		519	/* Required for the hardware to program scanline values for waiting */
		520	if (INTEL_INFO(dev)->gen == 6)
		521	I915_WRITE(GFX_MODE,
		522	_MASKED_BIT_ENABLE(GFX_TLB_INVALIDATE_ALWAYS));
		523
2332	Serge	524	if (IS_GEN7(dev))
		525	I915_WRITE(GFX_MODE_GEN7,
3031	serge	526	_MASKED_BIT_DISABLE(GFX_TLB_INVALIDATE_ALWAYS) \|
		527	_MASKED_BIT_ENABLE(GFX_REPLAY_MODE));
2332	Serge	528
2342	Serge	529	if (INTEL_INFO(dev)->gen >= 5) {
2339	Serge	530	ret = init_pipe_control(ring);
2332	Serge	531	if (ret)
		532	return ret;
		533	}
		534
3031	serge	535	if (IS_GEN6(dev)) {
		536	/* From the Sandybridge PRM, volume 1 part 3, page 24:
		537	* "If this bit is set, STCunit will have LRA as replacement
		538	* policy. [...] This bit must be reset. LRA replacement
		539	* policy is not supported."
		540	*/
		541	I915_WRITE(CACHE_MODE_0,
		542	_MASKED_BIT_DISABLE(CM0_STC_EVICT_DISABLE_LRA_SNB));
		543
		544	/* This is not explicitly set for GEN6, so read the register.
		545	* see intel_ring_mi_set_context() for why we care.
		546	* TODO: consider explicitly setting the bit for GEN5
		547	*/
		548	ring->itlb_before_ctx_switch =
		549	!!(I915_READ(GFX_MODE) & GFX_TLB_INVALIDATE_ALWAYS);
2342	Serge	550	}
		551
3031	serge	552	if (INTEL_INFO(dev)->gen >= 6)
		553	I915_WRITE(INSTPM, _MASKED_BIT_ENABLE(INSTPM_FORCE_ORDERING));
		554
		555	if (HAS_L3_GPU_CACHE(dev))
		556	I915_WRITE_IMR(ring, ~GEN6_RENDER_L3_PARITY_ERROR);
		557
3746	Serge	558	LEAVE();
		559
2332	Serge	560	return ret;
		561	}
		562
		563	static void render_ring_cleanup(struct intel_ring_buffer *ring)
		564	{
3480	Serge	565	struct drm_device *dev = ring->dev;
		566
2332	Serge	567	if (!ring->private)
		568	return;
		569
		570	cleanup_pipe_control(ring);
		571	}
		572
		573	static void
2342	Serge	574	update_mboxes(struct intel_ring_buffer *ring,
		575	u32 mmio_offset)
2332	Serge	576	{
3243	Serge	577	intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(1));
2342	Serge	578	intel_ring_emit(ring, mmio_offset);
3243	Serge	579	intel_ring_emit(ring, ring->outstanding_lazy_request);
2332	Serge	580	}
		581
2342	Serge	582	/**
		583	* gen6_add_request - Update the semaphore mailbox registers
		584	*
		585	* @ring - ring that is adding a request
		586	* @seqno - return seqno stuck into the ring
		587	*
		588	* Update the mailbox registers in the other rings with the current seqno.
		589	* This acts like a signal in the canonical semaphore.
		590	*/
2332	Serge	591	static int
3243	Serge	592	gen6_add_request(struct intel_ring_buffer *ring)
2332	Serge	593	{
2342	Serge	594	u32 mbox1_reg;
		595	u32 mbox2_reg;
2332	Serge	596	int ret;
		597
		598	ret = intel_ring_begin(ring, 10);
		599	if (ret)
		600	return ret;
		601
2342	Serge	602	mbox1_reg = ring->signal_mbox[0];
		603	mbox2_reg = ring->signal_mbox[1];
2332	Serge	604
3243	Serge	605	update_mboxes(ring, mbox1_reg);
		606	update_mboxes(ring, mbox2_reg);
2332	Serge	607	intel_ring_emit(ring, MI_STORE_DWORD_INDEX);
		608	intel_ring_emit(ring, I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
3243	Serge	609	intel_ring_emit(ring, ring->outstanding_lazy_request);
2332	Serge	610	intel_ring_emit(ring, MI_USER_INTERRUPT);
		611	intel_ring_advance(ring);
		612
		613	return 0;
		614	}
		615
3480	Serge	616	static inline bool i915_gem_has_seqno_wrapped(struct drm_device *dev,
		617	u32 seqno)
		618	{
		619	struct drm_i915_private *dev_priv = dev->dev_private;
		620	return dev_priv->last_seqno < seqno;
		621	}
		622
2342	Serge	623	/**
		624	* intel_ring_sync - sync the waiter to the signaller on seqno
		625	*
		626	* @waiter - ring that is waiting
		627	* @signaller - ring which has, or will signal
		628	* @seqno - seqno which the waiter will block on
		629	*/
		630	static int
3031	serge	631	gen6_ring_sync(struct intel_ring_buffer *waiter,
2342	Serge	632	struct intel_ring_buffer *signaller,
2332	Serge	633	u32 seqno)
		634	{
		635	int ret;
2342	Serge	636	u32 dw1 = MI_SEMAPHORE_MBOX \|
		637	MI_SEMAPHORE_COMPARE \|
		638	MI_SEMAPHORE_REGISTER;
2332	Serge	639
3031	serge	640	/* Throughout all of the GEM code, seqno passed implies our current
		641	* seqno is >= the last seqno executed. However for hardware the
		642	* comparison is strictly greater than.
		643	*/
		644	seqno -= 1;
		645
		646	WARN_ON(signaller->semaphore_register[waiter->id] ==
		647	MI_SEMAPHORE_SYNC_INVALID);
		648
2342	Serge	649	ret = intel_ring_begin(waiter, 4);
2332	Serge	650	if (ret)
		651	return ret;
		652
3480	Serge	653	/* If seqno wrap happened, omit the wait with no-ops */
		654	if (likely(!i915_gem_has_seqno_wrapped(waiter->dev, seqno))) {
3031	serge	655	intel_ring_emit(waiter,
3480	Serge	656	dw1 \|
		657	signaller->semaphore_register[waiter->id]);
2342	Serge	658	intel_ring_emit(waiter, seqno);
		659	intel_ring_emit(waiter, 0);
		660	intel_ring_emit(waiter, MI_NOOP);
3480	Serge	661	} else {
		662	intel_ring_emit(waiter, MI_NOOP);
		663	intel_ring_emit(waiter, MI_NOOP);
		664	intel_ring_emit(waiter, MI_NOOP);
		665	intel_ring_emit(waiter, MI_NOOP);
		666	}
2342	Serge	667	intel_ring_advance(waiter);
2332	Serge	668
		669	return 0;
		670	}
		671
		672	#define PIPE_CONTROL_FLUSH(ring__, addr__) \
		673	do { \
2342	Serge	674	intel_ring_emit(ring__, GFX_OP_PIPE_CONTROL(4) \| PIPE_CONTROL_QW_WRITE \| \
		675	PIPE_CONTROL_DEPTH_STALL); \
2332	Serge	676	intel_ring_emit(ring__, (addr__) \| PIPE_CONTROL_GLOBAL_GTT); \
		677	intel_ring_emit(ring__, 0); \
		678	intel_ring_emit(ring__, 0); \
		679	} while (0)
		680
		681	static int
3243	Serge	682	pc_render_add_request(struct intel_ring_buffer *ring)
2332	Serge	683	{
		684	struct pipe_control *pc = ring->private;
		685	u32 scratch_addr = pc->gtt_offset + 128;
		686	int ret;
		687
		688	/* For Ironlake, MI_USER_INTERRUPT was deprecated and apparently
		689	* incoherent with writes to memory, i.e. completely fubar,
		690	* so we need to use PIPE_NOTIFY instead.
		691	*
		692	* However, we also need to workaround the qword write
		693	* incoherence by flushing the 6 PIPE_NOTIFY buffers out to
		694	* memory before requesting an interrupt.
		695	*/
		696	ret = intel_ring_begin(ring, 32);
		697	if (ret)
		698	return ret;
		699
2342	Serge	700	intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(4) \| PIPE_CONTROL_QW_WRITE \|
		701	PIPE_CONTROL_WRITE_FLUSH \|
		702	PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE);
2332	Serge	703	intel_ring_emit(ring, pc->gtt_offset \| PIPE_CONTROL_GLOBAL_GTT);
3243	Serge	704	intel_ring_emit(ring, ring->outstanding_lazy_request);
2332	Serge	705	intel_ring_emit(ring, 0);
		706	PIPE_CONTROL_FLUSH(ring, scratch_addr);
		707	scratch_addr += 128; /* write to separate cachelines */
		708	PIPE_CONTROL_FLUSH(ring, scratch_addr);
		709	scratch_addr += 128;
		710	PIPE_CONTROL_FLUSH(ring, scratch_addr);
		711	scratch_addr += 128;
		712	PIPE_CONTROL_FLUSH(ring, scratch_addr);
		713	scratch_addr += 128;
		714	PIPE_CONTROL_FLUSH(ring, scratch_addr);
		715	scratch_addr += 128;
		716	PIPE_CONTROL_FLUSH(ring, scratch_addr);
3031	serge	717
2342	Serge	718	intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(4) \| PIPE_CONTROL_QW_WRITE \|
		719	PIPE_CONTROL_WRITE_FLUSH \|
		720	PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE \|
2332	Serge	721	PIPE_CONTROL_NOTIFY);
		722	intel_ring_emit(ring, pc->gtt_offset \| PIPE_CONTROL_GLOBAL_GTT);
3243	Serge	723	intel_ring_emit(ring, ring->outstanding_lazy_request);
2332	Serge	724	intel_ring_emit(ring, 0);
		725	intel_ring_advance(ring);
		726
		727	return 0;
		728	}
		729
		730	static u32
3031	serge	731	gen6_ring_get_seqno(struct intel_ring_buffer *ring, bool lazy_coherency)
2342	Serge	732	{
		733	/* Workaround to force correct ordering between irq and seqno writes on
		734	* ivb (and maybe also on snb) by reading from a CS register (like
		735	* ACTHD) before reading the status page. */
3031	serge	736	if (!lazy_coherency)
2342	Serge	737	intel_ring_get_active_head(ring);
		738	return intel_read_status_page(ring, I915_GEM_HWS_INDEX);
		739	}
		740
		741	static u32
3031	serge	742	ring_get_seqno(struct intel_ring_buffer *ring, bool lazy_coherency)
2332	Serge	743	{
		744	return intel_read_status_page(ring, I915_GEM_HWS_INDEX);
		745	}
		746
3480	Serge	747	static void
		748	ring_set_seqno(struct intel_ring_buffer *ring, u32 seqno)
		749	{
		750	intel_write_status_page(ring, I915_GEM_HWS_INDEX, seqno);
		751	}
		752
2332	Serge	753	static u32
3031	serge	754	pc_render_get_seqno(struct intel_ring_buffer *ring, bool lazy_coherency)
2332	Serge	755	{
		756	struct pipe_control *pc = ring->private;
		757	return pc->cpu_page[0];
		758	}
		759
3480	Serge	760	static void
		761	pc_render_set_seqno(struct intel_ring_buffer *ring, u32 seqno)
		762	{
		763	struct pipe_control *pc = ring->private;
		764	pc->cpu_page[0] = seqno;
		765	}
		766
3031	serge	767	static bool
		768	gen5_ring_get_irq(struct intel_ring_buffer *ring)
2332	Serge	769	{
3031	serge	770	struct drm_device *dev = ring->dev;
		771	drm_i915_private_t *dev_priv = dev->dev_private;
		772	unsigned long flags;
		773
		774	if (!dev->irq_enabled)
		775	return false;
		776
		777	spin_lock_irqsave(&dev_priv->irq_lock, flags);
		778	if (ring->irq_refcount++ == 0) {
		779	dev_priv->gt_irq_mask &= ~ring->irq_enable_mask;
2332	Serge	780	I915_WRITE(GTIMR, dev_priv->gt_irq_mask);
		781	POSTING_READ(GTIMR);
3031	serge	782	}
		783	spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
		784
		785	return true;
2332	Serge	786	}
		787
		788	static void
3031	serge	789	gen5_ring_put_irq(struct intel_ring_buffer *ring)
2332	Serge	790	{
3031	serge	791	struct drm_device *dev = ring->dev;
		792	drm_i915_private_t *dev_priv = dev->dev_private;
		793	unsigned long flags;
		794
		795	spin_lock_irqsave(&dev_priv->irq_lock, flags);
		796	if (--ring->irq_refcount == 0) {
		797	dev_priv->gt_irq_mask \|= ring->irq_enable_mask;
2332	Serge	798	I915_WRITE(GTIMR, dev_priv->gt_irq_mask);
		799	POSTING_READ(GTIMR);
3031	serge	800	}
		801	spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
2332	Serge	802	}
		803
3031	serge	804	static bool
		805	i9xx_ring_get_irq(struct intel_ring_buffer *ring)
2332	Serge	806	{
3031	serge	807	struct drm_device *dev = ring->dev;
		808	drm_i915_private_t *dev_priv = dev->dev_private;
		809	unsigned long flags;
		810
		811	if (!dev->irq_enabled)
		812	return false;
		813
		814	spin_lock_irqsave(&dev_priv->irq_lock, flags);
		815	if (ring->irq_refcount++ == 0) {
		816	dev_priv->irq_mask &= ~ring->irq_enable_mask;
2332	Serge	817	I915_WRITE(IMR, dev_priv->irq_mask);
		818	POSTING_READ(IMR);
3031	serge	819	}
		820	spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
		821
		822	return true;
2332	Serge	823	}
		824
		825	static void
3031	serge	826	i9xx_ring_put_irq(struct intel_ring_buffer *ring)
2332	Serge	827	{
3031	serge	828	struct drm_device *dev = ring->dev;
		829	drm_i915_private_t *dev_priv = dev->dev_private;
		830	unsigned long flags;
		831
		832	spin_lock_irqsave(&dev_priv->irq_lock, flags);
		833	if (--ring->irq_refcount == 0) {
		834	dev_priv->irq_mask \|= ring->irq_enable_mask;
2332	Serge	835	I915_WRITE(IMR, dev_priv->irq_mask);
		836	POSTING_READ(IMR);
3031	serge	837	}
		838	spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
2332	Serge	839	}
		840
		841	static bool
3031	serge	842	i8xx_ring_get_irq(struct intel_ring_buffer *ring)
2332	Serge	843	{
		844	struct drm_device *dev = ring->dev;
		845	drm_i915_private_t *dev_priv = dev->dev_private;
3031	serge	846	unsigned long flags;
2332	Serge	847
		848	if (!dev->irq_enabled)
		849	return false;
		850
3031	serge	851	spin_lock_irqsave(&dev_priv->irq_lock, flags);
2332	Serge	852	if (ring->irq_refcount++ == 0) {
3031	serge	853	dev_priv->irq_mask &= ~ring->irq_enable_mask;
		854	I915_WRITE16(IMR, dev_priv->irq_mask);
		855	POSTING_READ16(IMR);
2332	Serge	856	}
3031	serge	857	spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
2332	Serge	858
		859	return true;
		860	}
		861
		862	static void
3031	serge	863	i8xx_ring_put_irq(struct intel_ring_buffer *ring)
2332	Serge	864	{
		865	struct drm_device *dev = ring->dev;
		866	drm_i915_private_t *dev_priv = dev->dev_private;
3031	serge	867	unsigned long flags;
2332	Serge	868
3031	serge	869	spin_lock_irqsave(&dev_priv->irq_lock, flags);
2332	Serge	870	if (--ring->irq_refcount == 0) {
3031	serge	871	dev_priv->irq_mask \|= ring->irq_enable_mask;
		872	I915_WRITE16(IMR, dev_priv->irq_mask);
		873	POSTING_READ16(IMR);
2332	Serge	874	}
3031	serge	875	spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
2332	Serge	876	}
		877
		878	void intel_ring_setup_status_page(struct intel_ring_buffer *ring)
		879	{
		880	struct drm_device *dev = ring->dev;
		881	drm_i915_private_t *dev_priv = ring->dev->dev_private;
		882	u32 mmio = 0;
		883
3746	Serge	884	ENTER();
		885
2332	Serge	886	/* The ring status page addresses are no longer next to the rest of
		887	* the ring registers as of gen7.
		888	*/
		889	if (IS_GEN7(dev)) {
		890	switch (ring->id) {
3031	serge	891	case RCS:
2332	Serge	892	mmio = RENDER_HWS_PGA_GEN7;
		893	break;
3031	serge	894	case BCS:
2332	Serge	895	mmio = BLT_HWS_PGA_GEN7;
		896	break;
3031	serge	897	case VCS:
2332	Serge	898	mmio = BSD_HWS_PGA_GEN7;
		899	break;
		900	}
		901	} else if (IS_GEN6(ring->dev)) {
		902	mmio = RING_HWS_PGA_GEN6(ring->mmio_base);
		903	} else {
		904	mmio = RING_HWS_PGA(ring->mmio_base);
		905	}
		906
		907	I915_WRITE(mmio, (u32)ring->status_page.gfx_addr);
		908	POSTING_READ(mmio);
3746	Serge	909	LEAVE();
		910
2332	Serge	911	}
		912
		913	static int
		914	bsd_ring_flush(struct intel_ring_buffer *ring,
		915	u32 invalidate_domains,
		916	u32 flush_domains)
		917	{
		918	int ret;
		919
		920	ret = intel_ring_begin(ring, 2);
		921	if (ret)
		922	return ret;
		923
		924	intel_ring_emit(ring, MI_FLUSH);
		925	intel_ring_emit(ring, MI_NOOP);
		926	intel_ring_advance(ring);
		927	return 0;
		928	}
		929
		930	static int
3243	Serge	931	i9xx_add_request(struct intel_ring_buffer *ring)
2332	Serge	932	{
		933	int ret;
		934
		935	ret = intel_ring_begin(ring, 4);
		936	if (ret)
		937	return ret;
		938
		939	intel_ring_emit(ring, MI_STORE_DWORD_INDEX);
		940	intel_ring_emit(ring, I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
3243	Serge	941	intel_ring_emit(ring, ring->outstanding_lazy_request);
2332	Serge	942	intel_ring_emit(ring, MI_USER_INTERRUPT);
		943	intel_ring_advance(ring);
		944
		945	return 0;
		946	}
		947
		948	static bool
3031	serge	949	gen6_ring_get_irq(struct intel_ring_buffer *ring)
2332	Serge	950	{
		951	struct drm_device *dev = ring->dev;
		952	drm_i915_private_t *dev_priv = dev->dev_private;
3031	serge	953	unsigned long flags;
2332	Serge	954
		955	if (!dev->irq_enabled)
		956	return false;
		957
2342	Serge	958	/* It looks like we need to prevent the gt from suspending while waiting
		959	* for an notifiy irq, otherwise irqs seem to get lost on at least the
		960	* blt/bsd rings on ivb. */
		961	gen6_gt_force_wake_get(dev_priv);
		962
3031	serge	963	spin_lock_irqsave(&dev_priv->irq_lock, flags);
2332	Serge	964	if (ring->irq_refcount++ == 0) {
3031	serge	965	if (HAS_L3_GPU_CACHE(dev) && ring->id == RCS)
		966	I915_WRITE_IMR(ring, ~(ring->irq_enable_mask \|
		967	GEN6_RENDER_L3_PARITY_ERROR));
		968	else
		969	I915_WRITE_IMR(ring, ~ring->irq_enable_mask);
		970	dev_priv->gt_irq_mask &= ~ring->irq_enable_mask;
		971	I915_WRITE(GTIMR, dev_priv->gt_irq_mask);
		972	POSTING_READ(GTIMR);
2332	Serge	973	}
3031	serge	974	spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
2332	Serge	975
2351	Serge	976	return true;
2332	Serge	977	}
		978
		979	static void
3031	serge	980	gen6_ring_put_irq(struct intel_ring_buffer *ring)
2332	Serge	981	{
		982	struct drm_device *dev = ring->dev;
		983	drm_i915_private_t *dev_priv = dev->dev_private;
3031	serge	984	unsigned long flags;
2332	Serge	985
3031	serge	986	spin_lock_irqsave(&dev_priv->irq_lock, flags);
2332	Serge	987	if (--ring->irq_refcount == 0) {
3031	serge	988	if (HAS_L3_GPU_CACHE(dev) && ring->id == RCS)
		989	I915_WRITE_IMR(ring, ~GEN6_RENDER_L3_PARITY_ERROR);
		990	else
		991	I915_WRITE_IMR(ring, ~0);
		992	dev_priv->gt_irq_mask \|= ring->irq_enable_mask;
		993	I915_WRITE(GTIMR, dev_priv->gt_irq_mask);
		994	POSTING_READ(GTIMR);
2332	Serge	995	}
3031	serge	996	spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
2342	Serge	997
		998	gen6_gt_force_wake_put(dev_priv);
2332	Serge	999	}
		1000
		1001	static int
3243	Serge	1002	i965_dispatch_execbuffer(struct intel_ring_buffer *ring,
		1003	u32 offset, u32 length,
		1004	unsigned flags)
2332	Serge	1005	{
		1006	int ret;
		1007
		1008	ret = intel_ring_begin(ring, 2);
		1009	if (ret)
		1010	return ret;
		1011
		1012	intel_ring_emit(ring,
3031	serge	1013	MI_BATCH_BUFFER_START \|
		1014	MI_BATCH_GTT \|
3243	Serge	1015	(flags & I915_DISPATCH_SECURE ? 0 : MI_BATCH_NON_SECURE_I965));
2332	Serge	1016	intel_ring_emit(ring, offset);
		1017	intel_ring_advance(ring);
		1018
		1019	return 0;
		1020	}
		1021
3243	Serge	1022	/* Just userspace ABI convention to limit the wa batch bo to a resonable size */
		1023	#define I830_BATCH_LIMIT (256*1024)
2332	Serge	1024	static int
3031	serge	1025	i830_dispatch_execbuffer(struct intel_ring_buffer *ring,
3243	Serge	1026	u32 offset, u32 len,
		1027	unsigned flags)
2332	Serge	1028	{
		1029	int ret;
		1030
3243	Serge	1031	if (flags & I915_DISPATCH_PINNED) {
2332	Serge	1032	ret = intel_ring_begin(ring, 4);
		1033	if (ret)
		1034	return ret;
		1035
		1036	intel_ring_emit(ring, MI_BATCH_BUFFER);
3243	Serge	1037	intel_ring_emit(ring, offset \| (flags & I915_DISPATCH_SECURE ? 0 : MI_BATCH_NON_SECURE));
2332	Serge	1038	intel_ring_emit(ring, offset + len - 8);
3243	Serge	1039	intel_ring_emit(ring, MI_NOOP);
		1040	intel_ring_advance(ring);
		1041	} else {
		1042	struct drm_i915_gem_object *obj = ring->private;
		1043	u32 cs_offset = obj->gtt_offset;
		1044
		1045	if (len > I830_BATCH_LIMIT)
		1046	return -ENOSPC;
		1047
		1048	ret = intel_ring_begin(ring, 9+3);
		1049	if (ret)
		1050	return ret;
		1051	/* Blit the batch (which has now all relocs applied) to the stable batch
		1052	* scratch bo area (so that the CS never stumbles over its tlb
		1053	* invalidation bug) ... */
		1054	intel_ring_emit(ring, XY_SRC_COPY_BLT_CMD \|
		1055	XY_SRC_COPY_BLT_WRITE_ALPHA \|
		1056	XY_SRC_COPY_BLT_WRITE_RGB);
		1057	intel_ring_emit(ring, BLT_DEPTH_32 \| BLT_ROP_GXCOPY \| 4096);
2332	Serge	1058	intel_ring_emit(ring, 0);
3243	Serge	1059	intel_ring_emit(ring, (DIV_ROUND_UP(len, 4096) << 16) \| 1024);
		1060	intel_ring_emit(ring, cs_offset);
		1061	intel_ring_emit(ring, 0);
		1062	intel_ring_emit(ring, 4096);
		1063	intel_ring_emit(ring, offset);
		1064	intel_ring_emit(ring, MI_FLUSH);
		1065
		1066	/* ... and execute it. */
		1067	intel_ring_emit(ring, MI_BATCH_BUFFER);
		1068	intel_ring_emit(ring, cs_offset \| (flags & I915_DISPATCH_SECURE ? 0 : MI_BATCH_NON_SECURE));
		1069	intel_ring_emit(ring, cs_offset + len - 8);
3031	serge	1070	intel_ring_advance(ring);
3243	Serge	1071	}
3031	serge	1072
		1073	return 0;
		1074	}
		1075
		1076	static int
		1077	i915_dispatch_execbuffer(struct intel_ring_buffer *ring,
3243	Serge	1078	u32 offset, u32 len,
		1079	unsigned flags)
3031	serge	1080	{
		1081	int ret;
		1082
2332	Serge	1083	ret = intel_ring_begin(ring, 2);
		1084	if (ret)
		1085	return ret;
		1086
3031	serge	1087	intel_ring_emit(ring, MI_BATCH_BUFFER_START \| MI_BATCH_GTT);
3243	Serge	1088	intel_ring_emit(ring, offset \| (flags & I915_DISPATCH_SECURE ? 0 : MI_BATCH_NON_SECURE));
2332	Serge	1089	intel_ring_advance(ring);
		1090
		1091	return 0;
		1092	}
		1093
		1094	static void cleanup_status_page(struct intel_ring_buffer *ring)
		1095	{
		1096	struct drm_i915_gem_object *obj;
		1097
		1098	obj = ring->status_page.obj;
		1099	if (obj == NULL)
		1100	return;
		1101
3031	serge	1102	// kunmap(sg_page(obj->pages->sgl));
2344	Serge	1103	i915_gem_object_unpin(obj);
		1104	drm_gem_object_unreference(&obj->base);
2332	Serge	1105	ring->status_page.obj = NULL;
		1106	}
		1107
		1108	static int init_status_page(struct intel_ring_buffer *ring)
		1109	{
		1110	struct drm_device *dev = ring->dev;
		1111	struct drm_i915_gem_object *obj;
		1112	int ret;
		1113
		1114	obj = i915_gem_alloc_object(dev, 4096);
		1115	if (obj == NULL) {
		1116	DRM_ERROR("Failed to allocate status page\n");
		1117	ret = -ENOMEM;
		1118	goto err;
		1119	}
		1120
2352	Serge	1121	i915_gem_object_set_cache_level(obj, I915_CACHE_LLC);
2332	Serge	1122
3031	serge	1123	ret = i915_gem_object_pin(obj, 4096, true, false);
2332	Serge	1124	if (ret != 0) {
		1125	goto err_unref;
		1126	}
		1127
		1128	ring->status_page.gfx_addr = obj->gtt_offset;
3243	Serge	1129	ring->status_page.page_addr = (void*)MapIoMem((addr_t)sg_page(obj->pages->sgl),4096,PG_SW);
2332	Serge	1130	if (ring->status_page.page_addr == NULL) {
3031	serge	1131	ret = -ENOMEM;
2332	Serge	1132	goto err_unpin;
		1133	}
		1134	ring->status_page.obj = obj;
		1135	memset(ring->status_page.page_addr, 0, PAGE_SIZE);
		1136
		1137	intel_ring_setup_status_page(ring);
		1138	DRM_DEBUG_DRIVER("%s hws offset: 0x%08x\n",
		1139	ring->name, ring->status_page.gfx_addr);
		1140
		1141	return 0;
		1142
		1143	err_unpin:
2344	Serge	1144	i915_gem_object_unpin(obj);
2332	Serge	1145	err_unref:
2344	Serge	1146	drm_gem_object_unreference(&obj->base);
2332	Serge	1147	err:
		1148	return ret;
		1149	}
		1150
3243	Serge	1151	static int init_phys_hws_pga(struct intel_ring_buffer *ring)
		1152	{
		1153	struct drm_i915_private *dev_priv = ring->dev->dev_private;
		1154	u32 addr;
		1155
		1156	if (!dev_priv->status_page_dmah) {
		1157	dev_priv->status_page_dmah =
		1158	drm_pci_alloc(ring->dev, PAGE_SIZE, PAGE_SIZE);
		1159	if (!dev_priv->status_page_dmah)
		1160	return -ENOMEM;
		1161	}
		1162
		1163	addr = dev_priv->status_page_dmah->busaddr;
		1164	if (INTEL_INFO(ring->dev)->gen >= 4)
		1165	addr \|= (dev_priv->status_page_dmah->busaddr >> 28) & 0xf0;
		1166	I915_WRITE(HWS_PGA, addr);
		1167
		1168	ring->status_page.page_addr = dev_priv->status_page_dmah->vaddr;
		1169	memset(ring->status_page.page_addr, 0, PAGE_SIZE);
		1170
		1171	return 0;
		1172	}
		1173
3031	serge	1174	static int intel_init_ring_buffer(struct drm_device *dev,
2332	Serge	1175	struct intel_ring_buffer *ring)
		1176	{
2340	Serge	1177	struct drm_i915_gem_object *obj;
3031	serge	1178	struct drm_i915_private *dev_priv = dev->dev_private;
2332	Serge	1179	int ret;
2340	Serge	1180
2332	Serge	1181	ring->dev = dev;
		1182	INIT_LIST_HEAD(&ring->active_list);
		1183	INIT_LIST_HEAD(&ring->request_list);
3031	serge	1184	ring->size = 32 * PAGE_SIZE;
3243	Serge	1185	memset(ring->sync_seqno, 0, sizeof(ring->sync_seqno));
2332	Serge	1186
2352	Serge	1187	init_waitqueue_head(&ring->irq_queue);
2332	Serge	1188
		1189	if (I915_NEED_GFX_HWS(dev)) {
2340	Serge	1190	ret = init_status_page(ring);
		1191	if (ret)
		1192	return ret;
3243	Serge	1193	} else {
		1194	BUG_ON(ring->id != RCS);
		1195	ret = init_phys_hws_pga(ring);
		1196	if (ret)
		1197	return ret;
2332	Serge	1198	}
		1199
3480	Serge	1200	obj = NULL;
		1201	if (!HAS_LLC(dev))
		1202	obj = i915_gem_object_create_stolen(dev, ring->size);
		1203	if (obj == NULL)
2332	Serge	1204	obj = i915_gem_alloc_object(dev, ring->size);
		1205	if (obj == NULL) {
		1206	DRM_ERROR("Failed to allocate ringbuffer\n");
		1207	ret = -ENOMEM;
		1208	goto err_hws;
		1209	}
		1210
		1211	ring->obj = obj;
		1212
3031	serge	1213	ret = i915_gem_object_pin(obj, PAGE_SIZE, true, false);
2332	Serge	1214	if (ret)
		1215	goto err_unref;
		1216
3031	serge	1217	ret = i915_gem_object_set_to_gtt_domain(obj, true);
		1218	if (ret)
		1219	goto err_unpin;
2332	Serge	1220
3031	serge	1221	ring->virtual_start =
3480	Serge	1222	ioremap_wc(dev_priv->gtt.mappable_base + obj->gtt_offset,
3031	serge	1223	ring->size);
		1224	if (ring->virtual_start == NULL) {
2332	Serge	1225	DRM_ERROR("Failed to map ringbuffer.\n");
		1226	ret = -EINVAL;
		1227	goto err_unpin;
		1228	}
		1229
		1230	ret = ring->init(ring);
		1231	if (ret)
		1232	goto err_unmap;
		1233
		1234	/* Workaround an erratum on the i830 which causes a hang if
		1235	* the TAIL pointer points to within the last 2 cachelines
		1236	* of the buffer.
		1237	*/
		1238	ring->effective_size = ring->size;
3031	serge	1239	if (IS_I830(ring->dev) \|\| IS_845G(ring->dev))
2332	Serge	1240	ring->effective_size -= 128;
2340	Serge	1241
2332	Serge	1242	return 0;
		1243
		1244	err_unmap:
3480	Serge	1245	iounmap(ring->virtual_start);
2332	Serge	1246	err_unpin:
2344	Serge	1247	i915_gem_object_unpin(obj);
2332	Serge	1248	err_unref:
2344	Serge	1249	drm_gem_object_unreference(&obj->base);
2332	Serge	1250	ring->obj = NULL;
		1251	err_hws:
		1252	// cleanup_status_page(ring);
		1253	return ret;
		1254	}
		1255
		1256	void intel_cleanup_ring_buffer(struct intel_ring_buffer *ring)
		1257	{
		1258	struct drm_i915_private *dev_priv;
		1259	int ret;
		1260
		1261	if (ring->obj == NULL)
		1262	return;
		1263
		1264	/* Disable the ring buffer. The ring must be idle at this point */
		1265	dev_priv = ring->dev->dev_private;
3243	Serge	1266	ret = intel_ring_idle(ring);
2332	Serge	1267	if (ret)
		1268	DRM_ERROR("failed to quiesce %s whilst cleaning up: %d\n",
		1269	ring->name, ret);
		1270
		1271	I915_WRITE_CTL(ring, 0);
		1272
3480	Serge	1273	iounmap(ring->virtual_start);
2332	Serge	1274
2344	Serge	1275	i915_gem_object_unpin(ring->obj);
		1276	drm_gem_object_unreference(&ring->obj->base);
2332	Serge	1277	ring->obj = NULL;
		1278
		1279	if (ring->cleanup)
		1280	ring->cleanup(ring);
		1281
		1282	// cleanup_status_page(ring);
		1283	}
		1284
3031	serge	1285	static int intel_ring_wait_seqno(struct intel_ring_buffer *ring, u32 seqno)
2332	Serge	1286	{
3031	serge	1287	int ret;
2332	Serge	1288
3031	serge	1289	ret = i915_wait_seqno(ring, seqno);
		1290	if (!ret)
		1291	i915_gem_retire_requests_ring(ring);
		1292
		1293	return ret;
		1294	}
		1295
		1296	static int intel_ring_wait_request(struct intel_ring_buffer *ring, int n)
		1297	{
		1298	struct drm_i915_gem_request *request;
		1299	u32 seqno = 0;
		1300	int ret;
		1301
		1302	i915_gem_retire_requests_ring(ring);
		1303
		1304	if (ring->last_retired_head != -1) {
		1305	ring->head = ring->last_retired_head;
		1306	ring->last_retired_head = -1;
2332	Serge	1307	ring->space = ring_space(ring);
		1308	if (ring->space >= n)
		1309	return 0;
		1310	}
		1311
3031	serge	1312	list_for_each_entry(request, &ring->request_list, list) {
		1313	int space;
2344	Serge	1314
3031	serge	1315	if (request->tail == -1)
		1316	continue;
		1317
3243	Serge	1318	space = request->tail - (ring->tail + I915_RING_FREE_SPACE);
3031	serge	1319	if (space < 0)
		1320	space += ring->size;
		1321	if (space >= n) {
		1322	seqno = request->seqno;
		1323	break;
		1324	}
		1325
		1326	/* Consume this request in case we need more space than
		1327	* is available and so need to prevent a race between
		1328	* updating last_retired_head and direct reads of
		1329	* I915_RING_HEAD. It also provides a nice sanity check.
		1330	*/
		1331	request->tail = -1;
		1332	}
		1333
		1334	if (seqno == 0)
		1335	return -ENOSPC;
		1336
		1337	ret = intel_ring_wait_seqno(ring, seqno);
		1338	if (ret)
		1339	return ret;
		1340
		1341	if (WARN_ON(ring->last_retired_head == -1))
		1342	return -ENOSPC;
		1343
		1344	ring->head = ring->last_retired_head;
		1345	ring->last_retired_head = -1;
		1346	ring->space = ring_space(ring);
		1347	if (WARN_ON(ring->space < n))
		1348	return -ENOSPC;
		1349
		1350	return 0;
		1351	}
		1352
3243	Serge	1353	static int ring_wait_for_space(struct intel_ring_buffer *ring, int n)
3031	serge	1354	{
		1355	struct drm_device *dev = ring->dev;
		1356	struct drm_i915_private *dev_priv = dev->dev_private;
		1357	unsigned long end;
		1358	int ret;
		1359
		1360	ret = intel_ring_wait_request(ring, n);
		1361	if (ret != -ENOSPC)
		1362	return ret;
		1363
3243	Serge	1364	trace_i915_ring_wait_begin(ring);
3031	serge	1365	/* With GEM the hangcheck timer should kick us out of the loop,
		1366	* leaving it early runs the risk of corrupting GEM state (due
		1367	* to running on almost untested codepaths). But on resume
		1368	* timers don't work yet, so prevent a complete hang in that
		1369	* case by choosing an insanely large timeout. */
		1370	end = GetTimerTicks() + 60 * HZ;
		1371
2332	Serge	1372	do {
		1373	ring->head = I915_READ_HEAD(ring);
		1374	ring->space = ring_space(ring);
		1375	if (ring->space >= n) {
2351	Serge	1376	trace_i915_ring_wait_end(ring);
2332	Serge	1377	return 0;
		1378	}
		1379
		1380	msleep(1);
3031	serge	1381
3480	Serge	1382	ret = i915_gem_check_wedge(&dev_priv->gpu_error,
		1383	dev_priv->mm.interruptible);
3031	serge	1384	if (ret)
		1385	return ret;
		1386	} while (!time_after(GetTimerTicks(), end));
2351	Serge	1387	trace_i915_ring_wait_end(ring);
2332	Serge	1388	return -EBUSY;
		1389	}
		1390
3243	Serge	1391	static int intel_wrap_ring_buffer(struct intel_ring_buffer *ring)
		1392	{
		1393	uint32_t __iomem *virt;
		1394	int rem = ring->size - ring->tail;
		1395
		1396	if (ring->space < rem) {
		1397	int ret = ring_wait_for_space(ring, rem);
		1398	if (ret)
		1399	return ret;
		1400	}
		1401
		1402	virt = ring->virtual_start + ring->tail;
		1403	rem /= 4;
		1404	while (rem--)
		1405	iowrite32(MI_NOOP, virt++);
		1406
		1407	ring->tail = 0;
		1408	ring->space = ring_space(ring);
		1409
		1410	return 0;
		1411	}
		1412
		1413	int intel_ring_idle(struct intel_ring_buffer *ring)
		1414	{
		1415	u32 seqno;
		1416	int ret;
		1417
		1418	/* We need to add any requests required to flush the objects and ring */
		1419	if (ring->outstanding_lazy_request) {
		1420	ret = i915_add_request(ring, NULL, NULL);
		1421	if (ret)
		1422	return ret;
		1423	}
		1424
		1425	/* Wait upon the last request to be completed */
		1426	if (list_empty(&ring->request_list))
		1427	return 0;
		1428
		1429	seqno = list_entry(ring->request_list.prev,
		1430	struct drm_i915_gem_request,
		1431	list)->seqno;
		1432
		1433	return i915_wait_seqno(ring, seqno);
		1434	}
		1435
		1436	static int
		1437	intel_ring_alloc_seqno(struct intel_ring_buffer *ring)
		1438	{
		1439	if (ring->outstanding_lazy_request)
		1440	return 0;
		1441
		1442	return i915_gem_get_seqno(ring->dev, &ring->outstanding_lazy_request);
		1443	}
		1444
3480	Serge	1445	static int __intel_ring_begin(struct intel_ring_buffer *ring,
		1446	int bytes)
		1447	{
		1448	int ret;
		1449
		1450	if (unlikely(ring->tail + bytes > ring->effective_size)) {
		1451	ret = intel_wrap_ring_buffer(ring);
		1452	if (unlikely(ret))
		1453	return ret;
		1454	}
		1455
		1456	if (unlikely(ring->space < bytes)) {
		1457	ret = ring_wait_for_space(ring, bytes);
		1458	if (unlikely(ret))
		1459	return ret;
		1460	}
		1461
		1462	ring->space -= bytes;
		1463	return 0;
		1464	}
		1465
2332	Serge	1466	int intel_ring_begin(struct intel_ring_buffer *ring,
		1467	int num_dwords)
		1468	{
3031	serge	1469	drm_i915_private_t *dev_priv = ring->dev->dev_private;
2332	Serge	1470	int ret;
		1471
3480	Serge	1472	ret = i915_gem_check_wedge(&dev_priv->gpu_error,
		1473	dev_priv->mm.interruptible);
3031	serge	1474	if (ret)
		1475	return ret;
2332	Serge	1476
3243	Serge	1477	/* Preallocate the olr before touching the ring */
		1478	ret = intel_ring_alloc_seqno(ring);
		1479	if (ret)
		1480	return ret;
		1481
3480	Serge	1482	return __intel_ring_begin(ring, num_dwords * sizeof(uint32_t));
		1483	}
2332	Serge	1484
3480	Serge	1485	void intel_ring_init_seqno(struct intel_ring_buffer *ring, u32 seqno)
		1486	{
		1487	struct drm_i915_private *dev_priv = ring->dev->dev_private;
		1488
		1489	BUG_ON(ring->outstanding_lazy_request);
		1490
		1491	if (INTEL_INFO(ring->dev)->gen >= 6) {
		1492	I915_WRITE(RING_SYNC_0(ring->mmio_base), 0);
		1493	I915_WRITE(RING_SYNC_1(ring->mmio_base), 0);
2332	Serge	1494	}
		1495
3480	Serge	1496	ring->set_seqno(ring, seqno);
2332	Serge	1497	}
		1498
		1499	void intel_ring_advance(struct intel_ring_buffer *ring)
		1500	{
3031	serge	1501	struct drm_i915_private *dev_priv = ring->dev->dev_private;
		1502
2332	Serge	1503	ring->tail &= ring->size - 1;
3480	Serge	1504	if (dev_priv->gpu_error.stop_rings & intel_ring_flag(ring))
3031	serge	1505	return;
2332	Serge	1506	ring->write_tail(ring, ring->tail);
		1507	}
		1508
		1509
		1510	static void gen6_bsd_ring_write_tail(struct intel_ring_buffer *ring,
		1511	u32 value)
		1512	{
		1513	drm_i915_private_t *dev_priv = ring->dev->dev_private;
		1514
		1515	/* Every tail move must follow the sequence below */
3031	serge	1516
		1517	/* Disable notification that the ring is IDLE. The GT
		1518	* will then assume that it is busy and bring it out of rc6.
		1519	*/
2332	Serge	1520	I915_WRITE(GEN6_BSD_SLEEP_PSMI_CONTROL,
3031	serge	1521	_MASKED_BIT_ENABLE(GEN6_BSD_SLEEP_MSG_DISABLE));
2332	Serge	1522
3031	serge	1523	/* Clear the context id. Here be magic! */
		1524	I915_WRITE64(GEN6_BSD_RNCID, 0x0);
		1525
		1526	/* Wait for the ring not to be idle, i.e. for it to wake up. */
2332	Serge	1527	if (wait_for((I915_READ(GEN6_BSD_SLEEP_PSMI_CONTROL) &
3031	serge	1528	GEN6_BSD_SLEEP_INDICATOR) == 0,
2332	Serge	1529	50))
3031	serge	1530	DRM_ERROR("timed out waiting for the BSD ring to wake up\n");
2332	Serge	1531
3031	serge	1532	/* Now that the ring is fully powered up, update the tail */
2332	Serge	1533	I915_WRITE_TAIL(ring, value);
3031	serge	1534	POSTING_READ(RING_TAIL(ring->mmio_base));
		1535
		1536	/* Let the ring send IDLE messages to the GT again,
		1537	* and so let it sleep to conserve power when idle.
		1538	*/
2332	Serge	1539	I915_WRITE(GEN6_BSD_SLEEP_PSMI_CONTROL,
3031	serge	1540	_MASKED_BIT_DISABLE(GEN6_BSD_SLEEP_MSG_DISABLE));
2332	Serge	1541	}
		1542
		1543	static int gen6_ring_flush(struct intel_ring_buffer *ring,
		1544	u32 invalidate, u32 flush)
		1545	{
		1546	uint32_t cmd;
		1547	int ret;
		1548
		1549	ret = intel_ring_begin(ring, 4);
		1550	if (ret)
		1551	return ret;
		1552
		1553	cmd = MI_FLUSH_DW;
3243	Serge	1554	/*
		1555	* Bspec vol 1c.5 - video engine command streamer:
		1556	* "If ENABLED, all TLBs will be invalidated once the flush
		1557	* operation is complete. This bit is only valid when the
		1558	* Post-Sync Operation field is a value of 1h or 3h."
		1559	*/
2332	Serge	1560	if (invalidate & I915_GEM_GPU_DOMAINS)
3243	Serge	1561	cmd \|= MI_INVALIDATE_TLB \| MI_INVALIDATE_BSD \|
		1562	MI_FLUSH_DW_STORE_INDEX \| MI_FLUSH_DW_OP_STOREDW;
2332	Serge	1563	intel_ring_emit(ring, cmd);
3243	Serge	1564	intel_ring_emit(ring, I915_GEM_HWS_SCRATCH_ADDR \| MI_FLUSH_DW_USE_GTT);
2332	Serge	1565	intel_ring_emit(ring, 0);
		1566	intel_ring_emit(ring, MI_NOOP);
		1567	intel_ring_advance(ring);
		1568	return 0;
		1569	}
		1570
		1571	static int
3243	Serge	1572	hsw_ring_dispatch_execbuffer(struct intel_ring_buffer *ring,
		1573	u32 offset, u32 len,
		1574	unsigned flags)
		1575	{
		1576	int ret;
		1577
		1578	ret = intel_ring_begin(ring, 2);
		1579	if (ret)
		1580	return ret;
		1581
		1582	intel_ring_emit(ring,
		1583	MI_BATCH_BUFFER_START \| MI_BATCH_PPGTT_HSW \|
		1584	(flags & I915_DISPATCH_SECURE ? 0 : MI_BATCH_NON_SECURE_HSW));
		1585	/* bit0-7 is the length on GEN6+ */
		1586	intel_ring_emit(ring, offset);
		1587	intel_ring_advance(ring);
		1588
		1589	return 0;
		1590	}
		1591
		1592	static int
2332	Serge	1593	gen6_ring_dispatch_execbuffer(struct intel_ring_buffer *ring,
3243	Serge	1594	u32 offset, u32 len,
		1595	unsigned flags)
2332	Serge	1596	{
		1597	int ret;
		1598
		1599	ret = intel_ring_begin(ring, 2);
		1600	if (ret)
		1601	return ret;
		1602
3243	Serge	1603	intel_ring_emit(ring,
		1604	MI_BATCH_BUFFER_START \|
		1605	(flags & I915_DISPATCH_SECURE ? 0 : MI_BATCH_NON_SECURE_I965));
2332	Serge	1606	/* bit0-7 is the length on GEN6+ */
		1607	intel_ring_emit(ring, offset);
		1608	intel_ring_advance(ring);
		1609
		1610	return 0;
		1611	}
		1612
		1613	/* Blitter support (SandyBridge+) */
		1614
		1615	static int blt_ring_flush(struct intel_ring_buffer *ring,
		1616	u32 invalidate, u32 flush)
		1617	{
		1618	uint32_t cmd;
		1619	int ret;
		1620
3031	serge	1621	ret = intel_ring_begin(ring, 4);
2332	Serge	1622	if (ret)
		1623	return ret;
		1624
		1625	cmd = MI_FLUSH_DW;
3243	Serge	1626	/*
		1627	* Bspec vol 1c.3 - blitter engine command streamer:
		1628	* "If ENABLED, all TLBs will be invalidated once the flush
		1629	* operation is complete. This bit is only valid when the
		1630	* Post-Sync Operation field is a value of 1h or 3h."
		1631	*/
2332	Serge	1632	if (invalidate & I915_GEM_DOMAIN_RENDER)
3243	Serge	1633	cmd \|= MI_INVALIDATE_TLB \| MI_FLUSH_DW_STORE_INDEX \|
		1634	MI_FLUSH_DW_OP_STOREDW;
2332	Serge	1635	intel_ring_emit(ring, cmd);
3243	Serge	1636	intel_ring_emit(ring, I915_GEM_HWS_SCRATCH_ADDR \| MI_FLUSH_DW_USE_GTT);
2332	Serge	1637	intel_ring_emit(ring, 0);
		1638	intel_ring_emit(ring, MI_NOOP);
		1639	intel_ring_advance(ring);
		1640	return 0;
		1641	}
		1642
		1643	int intel_init_render_ring_buffer(struct drm_device *dev)
		1644	{
		1645	drm_i915_private_t *dev_priv = dev->dev_private;
		1646	struct intel_ring_buffer *ring = &dev_priv->ring[RCS];
2340	Serge	1647
3031	serge	1648	ring->name = "render ring";
		1649	ring->id = RCS;
		1650	ring->mmio_base = RENDER_RING_BASE;
		1651
2332	Serge	1652	if (INTEL_INFO(dev)->gen >= 6) {
2339	Serge	1653	ring->add_request = gen6_add_request;
3031	serge	1654	ring->flush = gen7_render_ring_flush;
		1655	if (INTEL_INFO(dev)->gen == 6)
2342	Serge	1656	ring->flush = gen6_render_ring_flush;
3031	serge	1657	ring->irq_get = gen6_ring_get_irq;
		1658	ring->irq_put = gen6_ring_put_irq;
		1659	ring->irq_enable_mask = GT_USER_INTERRUPT;
2342	Serge	1660	ring->get_seqno = gen6_ring_get_seqno;
3480	Serge	1661	ring->set_seqno = ring_set_seqno;
3031	serge	1662	ring->sync_to = gen6_ring_sync;
		1663	ring->semaphore_register[0] = MI_SEMAPHORE_SYNC_INVALID;
		1664	ring->semaphore_register[1] = MI_SEMAPHORE_SYNC_RV;
		1665	ring->semaphore_register[2] = MI_SEMAPHORE_SYNC_RB;
		1666	ring->signal_mbox[0] = GEN6_VRSYNC;
		1667	ring->signal_mbox[1] = GEN6_BRSYNC;
2332	Serge	1668	} else if (IS_GEN5(dev)) {
2339	Serge	1669	ring->add_request = pc_render_add_request;
3031	serge	1670	ring->flush = gen4_render_ring_flush;
2342	Serge	1671	ring->get_seqno = pc_render_get_seqno;
3480	Serge	1672	ring->set_seqno = pc_render_set_seqno;
3031	serge	1673	ring->irq_get = gen5_ring_get_irq;
		1674	ring->irq_put = gen5_ring_put_irq;
		1675	ring->irq_enable_mask = GT_USER_INTERRUPT \| GT_PIPE_NOTIFY;
		1676	} else {
		1677	ring->add_request = i9xx_add_request;
		1678	if (INTEL_INFO(dev)->gen < 4)
		1679	ring->flush = gen2_render_ring_flush;
		1680	else
		1681	ring->flush = gen4_render_ring_flush;
		1682	ring->get_seqno = ring_get_seqno;
3480	Serge	1683	ring->set_seqno = ring_set_seqno;
3031	serge	1684	if (IS_GEN2(dev)) {
		1685	ring->irq_get = i8xx_ring_get_irq;
		1686	ring->irq_put = i8xx_ring_put_irq;
		1687	} else {
		1688	ring->irq_get = i9xx_ring_get_irq;
		1689	ring->irq_put = i9xx_ring_put_irq;
		1690	}
		1691	ring->irq_enable_mask = I915_USER_INTERRUPT;
2332	Serge	1692	}
3031	serge	1693	ring->write_tail = ring_write_tail;
3243	Serge	1694	if (IS_HASWELL(dev))
		1695	ring->dispatch_execbuffer = hsw_ring_dispatch_execbuffer;
		1696	else if (INTEL_INFO(dev)->gen >= 6)
3031	serge	1697	ring->dispatch_execbuffer = gen6_ring_dispatch_execbuffer;
		1698	else if (INTEL_INFO(dev)->gen >= 4)
		1699	ring->dispatch_execbuffer = i965_dispatch_execbuffer;
		1700	else if (IS_I830(dev) \|\| IS_845G(dev))
		1701	ring->dispatch_execbuffer = i830_dispatch_execbuffer;
		1702	else
		1703	ring->dispatch_execbuffer = i915_dispatch_execbuffer;
		1704	ring->init = init_render_ring;
		1705	ring->cleanup = render_ring_cleanup;
2332	Serge	1706
3243	Serge	1707	/* Workaround batchbuffer to combat CS tlb bug. */
		1708	if (HAS_BROKEN_CS_TLB(dev)) {
		1709	struct drm_i915_gem_object *obj;
		1710	int ret;
3031	serge	1711
3243	Serge	1712	obj = i915_gem_alloc_object(dev, I830_BATCH_LIMIT);
		1713	if (obj == NULL) {
		1714	DRM_ERROR("Failed to allocate batch bo\n");
		1715	return -ENOMEM;
		1716	}
		1717
		1718	ret = i915_gem_object_pin(obj, 0, true, false);
		1719	if (ret != 0) {
		1720	drm_gem_object_unreference(&obj->base);
		1721	DRM_ERROR("Failed to ping batch bo\n");
		1722	return ret;
		1723	}
		1724
		1725	ring->private = obj;
2332	Serge	1726	}
2340	Serge	1727
2332	Serge	1728	return intel_init_ring_buffer(dev, ring);
		1729	}
		1730
3243	Serge	1731	#if 0
		1732	int intel_render_ring_init_dri(struct drm_device *dev, u64 start, u32 size)
		1733	{
		1734	drm_i915_private_t *dev_priv = dev->dev_private;
		1735	struct intel_ring_buffer *ring = &dev_priv->ring[RCS];
		1736	int ret;
2332	Serge	1737
3243	Serge	1738	ring->name = "render ring";
		1739	ring->id = RCS;
		1740	ring->mmio_base = RENDER_RING_BASE;
		1741
		1742	if (INTEL_INFO(dev)->gen >= 6) {
		1743	/* non-kms not supported on gen6+ */
		1744	return -ENODEV;
		1745	}
		1746
		1747	/* Note: gem is not supported on gen5/ilk without kms (the corresponding
		1748	* gem_init ioctl returns with -ENODEV). Hence we do not need to set up
		1749	* the special gen5 functions. */
		1750	ring->add_request = i9xx_add_request;
		1751	if (INTEL_INFO(dev)->gen < 4)
		1752	ring->flush = gen2_render_ring_flush;
		1753	else
		1754	ring->flush = gen4_render_ring_flush;
		1755	ring->get_seqno = ring_get_seqno;
3480	Serge	1756	ring->set_seqno = ring_set_seqno;
3243	Serge	1757	if (IS_GEN2(dev)) {
		1758	ring->irq_get = i8xx_ring_get_irq;
		1759	ring->irq_put = i8xx_ring_put_irq;
		1760	} else {
		1761	ring->irq_get = i9xx_ring_get_irq;
		1762	ring->irq_put = i9xx_ring_put_irq;
		1763	}
		1764	ring->irq_enable_mask = I915_USER_INTERRUPT;
		1765	ring->write_tail = ring_write_tail;
		1766	if (INTEL_INFO(dev)->gen >= 4)
		1767	ring->dispatch_execbuffer = i965_dispatch_execbuffer;
		1768	else if (IS_I830(dev) \|\| IS_845G(dev))
		1769	ring->dispatch_execbuffer = i830_dispatch_execbuffer;
		1770	else
		1771	ring->dispatch_execbuffer = i915_dispatch_execbuffer;
		1772	ring->init = init_render_ring;
		1773	ring->cleanup = render_ring_cleanup;
		1774
		1775	ring->dev = dev;
		1776	INIT_LIST_HEAD(&ring->active_list);
		1777	INIT_LIST_HEAD(&ring->request_list);
		1778
		1779	ring->size = size;
		1780	ring->effective_size = ring->size;
		1781	if (IS_I830(ring->dev) \|\| IS_845G(ring->dev))
		1782	ring->effective_size -= 128;
		1783
		1784	ring->virtual_start = ioremap_wc(start, size);
		1785	if (ring->virtual_start == NULL) {
		1786	DRM_ERROR("can not ioremap virtual address for"
		1787	" ring buffer\n");
		1788	return -ENOMEM;
		1789	}
		1790
		1791	if (!I915_NEED_GFX_HWS(dev)) {
		1792	ret = init_phys_hws_pga(ring);
		1793	if (ret)
		1794	return ret;
		1795	}
		1796
		1797	return 0;
		1798	}
		1799	#endif
		1800
2332	Serge	1801	int intel_init_bsd_ring_buffer(struct drm_device *dev)
		1802	{
		1803	drm_i915_private_t *dev_priv = dev->dev_private;
		1804	struct intel_ring_buffer *ring = &dev_priv->ring[VCS];
		1805
3031	serge	1806	ring->name = "bsd ring";
		1807	ring->id = VCS;
2332	Serge	1808
3031	serge	1809	ring->write_tail = ring_write_tail;
		1810	if (IS_GEN6(dev) \|\| IS_GEN7(dev)) {
		1811	ring->mmio_base = GEN6_BSD_RING_BASE;
		1812	/* gen6 bsd needs a special wa for tail updates */
		1813	if (IS_GEN6(dev))
		1814	ring->write_tail = gen6_bsd_ring_write_tail;
		1815	ring->flush = gen6_ring_flush;
		1816	ring->add_request = gen6_add_request;
		1817	ring->get_seqno = gen6_ring_get_seqno;
3480	Serge	1818	ring->set_seqno = ring_set_seqno;
3031	serge	1819	ring->irq_enable_mask = GEN6_BSD_USER_INTERRUPT;
		1820	ring->irq_get = gen6_ring_get_irq;
		1821	ring->irq_put = gen6_ring_put_irq;
		1822	ring->dispatch_execbuffer = gen6_ring_dispatch_execbuffer;
		1823	ring->sync_to = gen6_ring_sync;
		1824	ring->semaphore_register[0] = MI_SEMAPHORE_SYNC_VR;
		1825	ring->semaphore_register[1] = MI_SEMAPHORE_SYNC_INVALID;
		1826	ring->semaphore_register[2] = MI_SEMAPHORE_SYNC_VB;
		1827	ring->signal_mbox[0] = GEN6_RVSYNC;
		1828	ring->signal_mbox[1] = GEN6_BVSYNC;
		1829	} else {
		1830	ring->mmio_base = BSD_RING_BASE;
		1831	ring->flush = bsd_ring_flush;
		1832	ring->add_request = i9xx_add_request;
		1833	ring->get_seqno = ring_get_seqno;
3480	Serge	1834	ring->set_seqno = ring_set_seqno;
3031	serge	1835	if (IS_GEN5(dev)) {
		1836	ring->irq_enable_mask = GT_BSD_USER_INTERRUPT;
		1837	ring->irq_get = gen5_ring_get_irq;
		1838	ring->irq_put = gen5_ring_put_irq;
		1839	} else {
		1840	ring->irq_enable_mask = I915_BSD_USER_INTERRUPT;
		1841	ring->irq_get = i9xx_ring_get_irq;
		1842	ring->irq_put = i9xx_ring_put_irq;
		1843	}
		1844	ring->dispatch_execbuffer = i965_dispatch_execbuffer;
		1845	}
		1846	ring->init = init_ring_common;
		1847
2332	Serge	1848	return intel_init_ring_buffer(dev, ring);
		1849	}
		1850
		1851	int intel_init_blt_ring_buffer(struct drm_device *dev)
		1852	{
		1853	drm_i915_private_t *dev_priv = dev->dev_private;
		1854	struct intel_ring_buffer *ring = &dev_priv->ring[BCS];
		1855
3031	serge	1856	ring->name = "blitter ring";
		1857	ring->id = BCS;
2332	Serge	1858
3031	serge	1859	ring->mmio_base = BLT_RING_BASE;
		1860	ring->write_tail = ring_write_tail;
		1861	ring->flush = blt_ring_flush;
		1862	ring->add_request = gen6_add_request;
		1863	ring->get_seqno = gen6_ring_get_seqno;
3480	Serge	1864	ring->set_seqno = ring_set_seqno;
3031	serge	1865	ring->irq_enable_mask = GEN6_BLITTER_USER_INTERRUPT;
		1866	ring->irq_get = gen6_ring_get_irq;
		1867	ring->irq_put = gen6_ring_put_irq;
		1868	ring->dispatch_execbuffer = gen6_ring_dispatch_execbuffer;
		1869	ring->sync_to = gen6_ring_sync;
		1870	ring->semaphore_register[0] = MI_SEMAPHORE_SYNC_BR;
		1871	ring->semaphore_register[1] = MI_SEMAPHORE_SYNC_BV;
		1872	ring->semaphore_register[2] = MI_SEMAPHORE_SYNC_INVALID;
		1873	ring->signal_mbox[0] = GEN6_RBSYNC;
		1874	ring->signal_mbox[1] = GEN6_VBSYNC;
		1875	ring->init = init_ring_common;
		1876
2332	Serge	1877	return intel_init_ring_buffer(dev, ring);
		1878	}
3031	serge	1879
		1880	int
		1881	intel_ring_flush_all_caches(struct intel_ring_buffer *ring)
		1882	{
		1883	int ret;
		1884
		1885	if (!ring->gpu_caches_dirty)
		1886	return 0;
		1887
		1888	ret = ring->flush(ring, 0, I915_GEM_GPU_DOMAINS);
		1889	if (ret)
		1890	return ret;
		1891
		1892	trace_i915_gem_ring_flush(ring, 0, I915_GEM_GPU_DOMAINS);
		1893
		1894	ring->gpu_caches_dirty = false;
		1895	return 0;
		1896	}
		1897
		1898	int
		1899	intel_ring_invalidate_all_caches(struct intel_ring_buffer *ring)
		1900	{
		1901	uint32_t flush_domains;
		1902	int ret;
		1903
		1904	flush_domains = 0;
		1905	if (ring->gpu_caches_dirty)
		1906	flush_domains = I915_GEM_GPU_DOMAINS;
		1907
		1908	ret = ring->flush(ring, I915_GEM_GPU_DOMAINS, flush_domains);
		1909	if (ret)
		1910	return ret;
		1911
		1912	trace_i915_gem_ring_flush(ring, I915_GEM_GPU_DOMAINS, flush_domains);
		1913
		1914	ring->gpu_caches_dirty = false;
		1915	return 0;
		1916	}

Subversion Repositories Kolibri OS

(root)/drivers/video/drm/i915/intel_ringbuffer.c – Rev 3746