WebSVN – Kolibri OS – Diff – /drivers/video/drm/i915/intel_lrc.c

 	FAULT_AND_HALT, /* Debug only */
 	FAULT_AND_STREAM,
 	FAULT_AND_CONTINUE /* Unsupported */
 };
 #define GEN8_CTX_ID_SHIFT 32
-#define CTX_RCS_INDIRECT_CTX_OFFSET_DEFAULT  0x17
+#define GEN8_CTX_RCS_INDIRECT_CTX_OFFSET_DEFAULT	0x17
+#define GEN9_CTX_RCS_INDIRECT_CTX_OFFSET_DEFAULT	0x26
+static int intel_lr_context_pin(struct intel_context *ctx,
-static int intel_lr_context_pin(struct drm_i915_gem_request *rq);
+				struct intel_engine_cs *engine);
 static void lrc_setup_hardware_status_page(struct intel_engine_cs *ring,
 		struct drm_i915_gem_object *default_ctx_obj);
 		return 1;
 	return 0;
-}
-/**
- * intel_execlists_ctx_id() - get the Execlists Context ID
- * @ctx_obj: Logical Ring Context backing object.
- *
- * Do not confuse with ctx->id! Unfortunately we have a name overload
- * here: the old context ID we pass to userspace as a handler so that
- * they can refer to a context, and the new context ID we pass to the
+}
- * ELSP so that the GPU can inform us of the context status via
- * interrupts.
- *
- * Return: 20-bits globally unique context ID.
- */
-u32 intel_execlists_ctx_id(struct drm_i915_gem_object *ctx_obj)
-{
-	u32 lrca = i915_gem_obj_ggtt_offset(ctx_obj) +
-			LRC_PPHWSP_PN * PAGE_SIZE;
-	/* LRCA is required to be 4K aligned so the more significant 20 bits
-	 * are globally unique */
-	return lrca >> 12;
-}
 static void
-static bool disable_lite_restore_wa(struct intel_engine_cs *ring)
+logical_ring_init_platform_invariants(struct intel_engine_cs *ring)
 {
 	struct drm_device *dev = ring->dev;
+	ring->disable_lite_restore_wa = (IS_SKL_REVID(dev, 0, SKL_REVID_B0) ||
+					IS_BXT_REVID(dev, 0, BXT_REVID_A1)) &&
+					(ring->id == VCS || ring->id == VCS2);
+	ring->ctx_desc_template = GEN8_CTX_VALID;
+	ring->ctx_desc_template |= GEN8_CTX_ADDRESSING_MODE(dev) <<
+				   GEN8_CTX_ADDRESSING_MODE_SHIFT;
+	if (IS_GEN8(dev))
+		ring->ctx_desc_template |= GEN8_CTX_L3LLC_COHERENT;
+	ring->ctx_desc_template |= GEN8_CTX_PRIVILEGE;
+	/* TODO: WaDisableLiteRestore when we start using semaphore
+	 * signalling between Command Streamers */
+	/* ring->ctx_desc_template |= GEN8_CTX_FORCE_RESTORE; */
+	/* WaEnableForceRestoreInCtxtDescForVCS:skl */
-	return (IS_SKL_REVID(dev, 0, SKL_REVID_B0) ||
+	/* WaEnableForceRestoreInCtxtDescForVCS:bxt */
+	if (ring->disable_lite_restore_wa)
+		ring->ctx_desc_template |= GEN8_CTX_FORCE_RESTORE;
+}
+/**
+ * intel_lr_context_descriptor_update() - calculate & cache the descriptor
+ * 					  descriptor for a pinned context
+ *
+ * @ctx: Context to work on
+ * @ring: Engine the descriptor will be used with
+ *
+ * The context descriptor encodes various attributes of a context,
+ * including its GTT address and some flags. Because it's fairly
+ * expensive to calculate, we'll just do it once and cache the result,
+ * which remains valid until the context is unpinned.
+ *
+ * This is what a descriptor looks like, from LSB to MSB:
+ *    bits 0-11:    flags, GEN8_CTX_* (cached in ctx_desc_template)
+ *    bits 12-31:    LRCA, GTT address of (the HWSP of) this context
-		IS_BXT_REVID(dev, 0, BXT_REVID_A1)) &&
+ *    bits 32-51:    ctx ID, a globally unique tag (the LRCA again!)
-	       (ring->id == VCS || ring->id == VCS2);
+ *    bits 52-63:    reserved, may encode the engine ID (for GuC)
  */
-uint64_t intel_lr_context_descriptor(struct intel_context *ctx,
+static void
-				     struct intel_engine_cs *ring)
-{
+intel_lr_context_descriptor_update(struct intel_context *ctx,
-	struct drm_i915_gem_object *ctx_obj = ctx->engine[ring->id].state;
+				   struct intel_engine_cs *ring)
-	uint64_t desc;
-	uint64_t lrca = i915_gem_obj_ggtt_offset(ctx_obj) +
+{
-			LRC_PPHWSP_PN * PAGE_SIZE;
+	uint64_t lrca, desc;
-	WARN_ON(lrca & 0xFFFFFFFF00000FFFULL);
-	desc = GEN8_CTX_VALID;
-	desc |= GEN8_CTX_ADDRESSING_MODE(dev) << GEN8_CTX_ADDRESSING_MODE_SHIFT;
-	if (IS_GEN8(ctx_obj->base.dev))
+	lrca = ctx->engine[ring->id].lrc_vma->node.start +
-		desc |= GEN8_CTX_L3LLC_COHERENT;
+	       LRC_PPHWSP_PN * PAGE_SIZE;
-	desc |= GEN8_CTX_PRIVILEGE;
-	desc |= lrca;
-	desc |= (u64)intel_execlists_ctx_id(ctx_obj) << GEN8_CTX_ID_SHIFT;
+	desc = ring->ctx_desc_template;			   /* bits  0-11 */
+	desc |= lrca;					   /* bits 12-31 */
+	desc |= (lrca >> PAGE_SHIFT) << GEN8_CTX_ID_SHIFT; /* bits 32-51 */
+	ctx->engine[ring->id].lrc_desc = desc;
+}
+uint64_t intel_lr_context_descriptor(struct intel_context *ctx,
+				     struct intel_engine_cs *ring)
+{
+	return ctx->engine[ring->id].lrc_desc;
+}
-	/* TODO: WaDisableLiteRestore when we start using semaphore
+/**
+ * intel_execlists_ctx_id() - get the Execlists Context ID
+ * @ctx: Context to get the ID for
+ * @ring: Engine to get the ID for
+ *
+ * Do not confuse with ctx->id! Unfortunately we have a name overload
+ * here: the old context ID we pass to userspace as a handler so that
+ * they can refer to a context, and the new context ID we pass to the
+ * ELSP so that the GPU can inform us of the context status via
+ * interrupts.
+ *
-	 * signalling between Command Streamers */
+ * The context ID is a portion of the context descriptor, so we can
-	/* desc |= GEN8_CTX_FORCE_RESTORE; */
+ * just extract the required part from the cached descriptor.
+ *
-	/* WaEnableForceRestoreInCtxtDescForVCS:skl */
+ * Return: 20-bits globally unique context ID.
 static int execlists_update_context(struct drm_i915_gem_request *rq)
+{
 	struct intel_engine_cs *ring = rq->ring;
 	struct i915_hw_ppgtt *ppgtt = rq->ctx->ppgtt;
-	struct drm_i915_gem_object *ctx_obj = rq->ctx->engine[ring->id].state;
-	struct drm_i915_gem_object *rb_obj = rq->ringbuf->obj;
-	struct page *page;
-	uint32_t *reg_state;
-	BUG_ON(!ctx_obj);
-	WARN_ON(!i915_gem_obj_is_pinned(ctx_obj));
-	WARN_ON(!i915_gem_obj_is_pinned(rb_obj));
-	page = i915_gem_object_get_dirty_page(ctx_obj, LRC_STATE_PN);
-	reg_state = kmap_atomic(page);
+	uint32_t *reg_state = rq->ctx->engine[ring->id].lrc_reg_state;
 	reg_state[CTX_RING_TAIL+1] = rq->tail;
-	reg_state[CTX_RING_BUFFER_START+1] = i915_gem_obj_ggtt_offset(rb_obj);
+	reg_state[CTX_RING_TAIL+1] = rq->tail;
 	if (ppgtt && !USES_FULL_48BIT_PPGTT(ppgtt->base.dev)) {
 		ASSIGN_CTX_PDP(ppgtt, reg_state, 2);
 		ASSIGN_CTX_PDP(ppgtt, reg_state, 1);
 		ASSIGN_CTX_PDP(ppgtt, reg_state, 0);
+	}
-	kunmap_atomic(reg_state);
 	return 0;
+}
 			req0 = cursor;
 		} else if (req0->ctx == cursor->ctx) {
 			/* Same ctx: ignore first request, as second request
 			 * will update tail past first request's workload */
 			cursor->elsp_submitted = req0->elsp_submitted;
-			list_del(&req0->execlist_link);
-			list_add_tail(&req0->execlist_link,
+			list_move_tail(&req0->execlist_link,
-				&ring->execlist_retired_req_list);
+				       &ring->execlist_retired_req_list);
 			req0 = cursor;
 		} else {
 			req1 = cursor;
 			break;
 	head_req = list_first_entry_or_null(&ring->execlist_queue,
 					    struct drm_i915_gem_request,
 					    execlist_link);
-	if (head_req != NULL) {
-		struct drm_i915_gem_object *ctx_obj =
-				head_req->ctx->engine[ring->id].state;
+	if (head_req != NULL) {
-		if (intel_execlists_ctx_id(ctx_obj) == request_id) {
+		if (intel_execlists_ctx_id(head_req->ctx, ring) == request_id) {
 			WARN(head_req->elsp_submitted == 0,
 			     "Never submitted head request\n");
-			if (--head_req->elsp_submitted <= 0) {
-				list_del(&head_req->execlist_link);
+			if (--head_req->elsp_submitted <= 0) {
-				list_add_tail(&head_req->execlist_link,
+				list_move_tail(&head_req->execlist_link,
-					&ring->execlist_retired_req_list);
+					       &ring->execlist_retired_req_list);
 				return true;
+			}
+		}
+	}
+	return false;
+}
+static void get_context_status(struct intel_engine_cs *ring,
+			       u8 read_pointer,
+			       u32 *status, u32 *context_id)
+{
+	struct drm_i915_private *dev_priv = ring->dev->dev_private;
+	if (WARN_ON(read_pointer >= GEN8_CSB_ENTRIES))
+		return;
 	*status = I915_READ(RING_CONTEXT_STATUS_BUF_LO(ring, read_pointer));
-	return false;
+	*context_id = I915_READ(RING_CONTEXT_STATUS_BUF_HI(ring, read_pointer));
+}
 /**
 	u32 submit_contexts = 0;
 	status_pointer = I915_READ(RING_CONTEXT_STATUS_PTR(ring));
 	read_pointer = ring->next_context_status_buffer;
-	write_pointer = status_pointer & GEN8_CSB_PTR_MASK;
+	write_pointer = GEN8_CSB_WRITE_PTR(status_pointer);
 	if (read_pointer > write_pointer)
 		write_pointer += GEN8_CSB_ENTRIES;
 	spin_lock(&ring->execlist_lock);
 	while (read_pointer < write_pointer) {
 		read_pointer++;
-		status = I915_READ(RING_CONTEXT_STATUS_BUF_LO(ring, read_pointer % GEN8_CSB_ENTRIES));
+		get_context_status(ring, ++read_pointer % GEN8_CSB_ENTRIES,
 			if (execlists_check_remove_request(ring, status_id))
 				submit_contexts++;
+		}
+	}
-	if (disable_lite_restore_wa(ring)) {
+	if (ring->disable_lite_restore_wa) {
 		/* Prevent a ctx to preempt itself */
 		if ((status & GEN8_CTX_STATUS_ACTIVE_IDLE) &&
 		    (submit_contexts != 0))
 			execlists_context_unqueue(ring);
 	} else if (submit_contexts != 0) {
 		execlists_context_unqueue(ring);
+	}
+	spin_unlock(&ring->execlist_lock);
-	spin_unlock(&ring->execlist_lock);
+	if (unlikely(submit_contexts > 2))
+		DRM_ERROR("More than two context complete events?\n");
 	ring->next_context_status_buffer = write_pointer % GEN8_CSB_ENTRIES;
-	WARN(submit_contexts > 2, "More than two context complete events?\n");
-	ring->next_context_status_buffer = write_pointer % GEN8_CSB_ENTRIES;
+	/* Update the read pointer to the old write pointer. Manual ringbuffer
-	I915_WRITE(RING_CONTEXT_STATUS_PTR(ring),
+	 * management ftw  */
-		   _MASKED_FIELD(GEN8_CSB_PTR_MASK << 8,
+	I915_WRITE(RING_CONTEXT_STATUS_PTR(ring),
-				 ((u32)ring->next_context_status_buffer &
+		   _MASKED_FIELD(GEN8_CSB_READ_PTR_MASK,
-				  GEN8_CSB_PTR_MASK) << 8));
+				 ring->next_context_status_buffer << 8));
+}
 static int execlists_context_queue(struct drm_i915_gem_request *request)
 {
 	struct intel_engine_cs *ring = request->ring;
 	struct drm_i915_gem_request *cursor;
 					   execlist_link);
 		if (request->ctx == tail_req->ctx) {
 			WARN(tail_req->elsp_submitted != 0,
-				"More than 2 already-submitted reqs queued\n");
-			list_del(&tail_req->execlist_link);
+				"More than 2 already-submitted reqs queued\n");
-			list_add_tail(&tail_req->execlist_link,
+			list_move_tail(&tail_req->execlist_link,
-				&ring->execlist_retired_req_list);
+				       &ring->execlist_retired_req_list);
+		}
+	}
 	return logical_ring_invalidate_all_caches(req);
+}
 int intel_logical_ring_alloc_request_extras(struct drm_i915_gem_request *request)
 {
-	int ret;
+	int ret = 0;
+	request->ringbuf = request->ctx->engine[request->ring->id].ringbuf;
+	if (i915.enable_guc_submission) {
+		/*
+		 * Check that the GuC has space for the request before
+		 * going any further, as the i915_add_request() call
+		 * later on mustn't fail ...
-	request->ringbuf = request->ctx->engine[request->ring->id].ringbuf;
+		 */
 		struct intel_guc *guc = &request->i915->guc;
 	if (request->ctx != request->ring->default_context) {
-		ret = intel_lr_context_pin(request);
+		ret = i915_guc_wq_check_space(guc->execbuf_client);
+		if (ret)
+			return ret;
+	}
-		if (ret)
-			return ret;
+	if (request->ctx != request->i915->kernel_context)
 		ret = intel_lr_context_pin(request->ctx, request->ring);
-	return 0;
+	return ret;
  * The tail is updated in our logical ringbuffer struct, not in the actual context. What
  * really happens during submission is that the context and current tail will be placed
  * on a queue waiting for the ELSP to be ready to accept a new context submission. At that
  * point, the tail *inside* the context is updated and the ELSP written to.
  */
-static void
+static int
 intel_logical_ring_advance_and_submit(struct drm_i915_gem_request *request)
+{
-	struct intel_engine_cs *ring = request->ring;
+	struct intel_ringbuffer *ringbuf = request->ringbuf;
 	struct drm_i915_private *dev_priv = request->i915;
+	struct intel_engine_cs *engine = request->ring;
+	intel_logical_ring_advance(ringbuf);
+	request->tail = ringbuf->tail;
+	/*
+	 * Here we add two extra NOOPs as padding to avoid
+	 * lite restore of a context with HEAD==TAIL.
+	 *
+	 * Caller must reserve WA_TAIL_DWORDS for us!
-	intel_logical_ring_advance(request->ringbuf);
+	 */
+	intel_logical_ring_emit(ringbuf, MI_NOOP);
+	intel_logical_ring_emit(ringbuf, MI_NOOP);
-	request->tail = request->ringbuf->tail;
+	intel_logical_ring_advance(ringbuf);
+	if (intel_ring_stopped(engine))
+		return 0;
+	if (engine->last_context != request->ctx) {
+		if (engine->last_context)
+			intel_lr_context_unpin(engine->last_context, engine);
+		if (request->ctx != request->i915->kernel_context) {
+			intel_lr_context_pin(request->ctx, engine);
+			engine->last_context = request->ctx;
+		} else {
 			engine->last_context = NULL;
 	if (intel_ring_stopped(ring))
 		return;
+	if (dev_priv->guc.execbuf_client)
+		i915_guc_submit(dev_priv->guc.execbuf_client, request);
-	if (dev_priv->guc.execbuf_client)
+	else
-		i915_guc_submit(dev_priv->guc.execbuf_client, request);
+		execlists_context_queue(request);
 	else
-		execlists_context_queue(request);
+	return 0;
 	list_for_each_entry_safe(req, tmp, &retired_list, execlist_link) {
 		struct intel_context *ctx = req->ctx;
 		struct drm_i915_gem_object *ctx_obj =
 				ctx->engine[ring->id].state;
-		if (ctx_obj && (ctx != ring->default_context))
+		if (ctx_obj && (ctx != req->i915->kernel_context))
+			intel_lr_context_unpin(ctx, ring);
 			intel_lr_context_unpin(req);
 		list_del(&req->execlist_link);
 		i915_gem_request_unreference(req);
+	}
 	ring->gpu_caches_dirty = false;
 	return 0;
 }
-static int intel_lr_context_do_pin(struct intel_engine_cs *ring,
-		struct drm_i915_gem_object *ctx_obj,
+static int intel_lr_context_do_pin(struct intel_context *ctx,
-		struct intel_ringbuffer *ringbuf)
+				   struct intel_engine_cs *ring)
+{
+	struct drm_device *dev = ring->dev;
+	struct drm_i915_private *dev_priv = dev->dev_private;
+	struct drm_i915_gem_object *ctx_obj = ctx->engine[ring->id].state;
+	struct intel_ringbuffer *ringbuf = ctx->engine[ring->id].ringbuf;
-	struct drm_device *dev = ring->dev;
+	struct page *lrc_state_page;
-	struct drm_i915_private *dev_priv = dev->dev_private;
+	uint32_t *lrc_reg_state;
+	int ret;
 	int ret = 0;
 	WARN_ON(!mutex_is_locked(&ring->dev->struct_mutex));
 	WARN_ON(!mutex_is_locked(&ring->dev->struct_mutex));
 	ret = i915_gem_obj_ggtt_pin(ctx_obj, GEN8_LR_CONTEXT_ALIGN,
+			PIN_OFFSET_BIAS | GUC_WOPCM_TOP);
+	if (ret)
+		return ret;
+	lrc_state_page = i915_gem_object_get_dirty_page(ctx_obj, LRC_STATE_PN);
+	if (WARN_ON(!lrc_state_page)) {
-			PIN_OFFSET_BIAS | GUC_WOPCM_TOP);
+		ret = -ENODEV;
-	if (ret)
+		goto unpin_ctx_obj;
 		return ret;
+	ret = intel_pin_and_map_ringbuffer_obj(ring->dev, ringbuf);
+	if (ret)
+		goto unpin_ctx_obj;
 	ctx->engine[ring->id].lrc_vma = i915_gem_obj_to_ggtt(ctx_obj);
-	ret = intel_pin_and_map_ringbuffer_obj(ring->dev, ringbuf);
+	intel_lr_context_descriptor_update(ctx, ring);
-	if (ret)
+	lrc_reg_state = kmap(lrc_state_page);
-		goto unpin_ctx_obj;
+	lrc_reg_state[CTX_RING_BUFFER_START+1] = ringbuf->vma->node.start;
 	i915_gem_object_ggtt_unpin(ctx_obj);
 	return ret;
 }
 static int intel_lr_context_pin(struct intel_context *ctx,
-static int intel_lr_context_pin(struct drm_i915_gem_request *rq)
+				struct intel_engine_cs *engine)
-{
-	int ret = 0;
-	struct intel_engine_cs *ring = rq->ring;
-	struct drm_i915_gem_object *ctx_obj = rq->ctx->engine[ring->id].state;
+{
-	struct intel_ringbuffer *ringbuf = rq->ringbuf;
+	int ret = 0;
-	if (rq->ctx->engine[ring->id].pin_count++ == 0) {
+	if (ctx->engine[engine->id].pin_count++ == 0) {
+		ret = intel_lr_context_do_pin(ctx, engine);
+		if (ret)
-		ret = intel_lr_context_do_pin(ring, ctx_obj, ringbuf);
+			goto reset_pin_count;
 		if (ret)
-			goto reset_pin_count;
+		i915_gem_context_reference(ctx);
 	}
 	return ret;
 reset_pin_count:
+	ctx->engine[engine->id].pin_count = 0;
-	rq->ctx->engine[ring->id].pin_count = 0;
+	return ret;
-	return ret;
 }
-void intel_lr_context_unpin(struct drm_i915_gem_request *rq)
-{
-	struct intel_engine_cs *ring = rq->ring;
+void intel_lr_context_unpin(struct intel_context *ctx,
+			    struct intel_engine_cs *engine)
-	struct drm_i915_gem_object *ctx_obj = rq->ctx->engine[ring->id].state;
+{
-	struct intel_ringbuffer *ringbuf = rq->ringbuf;
+	struct drm_i915_gem_object *ctx_obj = ctx->engine[engine->id].state;
+	WARN_ON(!mutex_is_locked(&ctx->i915->dev->struct_mutex));
+	if (--ctx->engine[engine->id].pin_count == 0) {
+//		kunmap(kmap_to_page(ctx->engine[engine->id].lrc_reg_state));
+		intel_unpin_ringbuffer_obj(ctx->engine[engine->id].ringbuf);
-	if (ctx_obj) {
+		i915_gem_object_ggtt_unpin(ctx_obj);
-		WARN_ON(!mutex_is_locked(&ring->dev->struct_mutex));
+		ctx->engine[engine->id].lrc_vma = NULL;
-		if (--rq->ctx->engine[ring->id].pin_count == 0) {
+		ctx->engine[engine->id].lrc_desc = 0;
-			intel_unpin_ringbuffer_obj(ringbuf);
+		ctx->engine[engine->id].lrc_reg_state = NULL;
 	struct intel_ringbuffer *ringbuf = req->ringbuf;
 	struct drm_device *dev = ring->dev;
 	struct drm_i915_private *dev_priv = dev->dev_private;
 	struct i915_workarounds *w = &dev_priv->workarounds;
-	if (WARN_ON_ONCE(w->count == 0))
+	if (w->count == 0)
 		return 0;
 	ring->gpu_caches_dirty = true;
 	struct drm_device *dev = ring->dev;
 	struct drm_i915_private *dev_priv = dev->dev_private;
 	u8 next_context_status_buffer_hw;
 	lrc_setup_hardware_status_page(ring,
-				ring->default_context->engine[ring->id].state);
+				dev_priv->kernel_context->engine[ring->id].state);
 	I915_WRITE_IMR(ring, ~(ring->irq_enable_mask | ring->irq_keep_mask));
 	 * Effectively, these states have been observed:
+	 *
 	 *      | Suspend-to-idle (freeze) | Suspend-to-RAM (mem) |
 	 * BDW  | CSB regs not reset       | CSB regs reset       |
 	 * CHT  | CSB regs not reset       | CSB regs not reset   |
+	 * SKL  |         ?                |         ?            |
+	 * BXT  |         ?                |         ?            |
 	 */
-	next_context_status_buffer_hw = (I915_READ(RING_CONTEXT_STATUS_PTR(ring))
+	next_context_status_buffer_hw =
-						   & GEN8_CSB_PTR_MASK);
+		GEN8_CSB_WRITE_PTR(I915_READ(RING_CONTEXT_STATUS_PTR(ring)));
 	/*
 	 * When the CSB registers are reset (also after power-up / gpu reset),
 	 * CSB write pointer is set to all 1's, which is not valid, use '5' in
 				  u32 flush_domains)
+{
 	struct intel_ringbuffer *ringbuf = request->ringbuf;
 	struct intel_engine_cs *ring = ringbuf->ring;
 	u32 scratch_addr = ring->scratch.gtt_offset + 2 * CACHELINE_BYTES;
-	bool vf_flush_wa;
+	bool vf_flush_wa = false;
 	u32 flags = 0;
 	int ret;
 		flags |= PIPE_CONTROL_VF_CACHE_INVALIDATE;
 		flags |= PIPE_CONTROL_CONST_CACHE_INVALIDATE;
 		flags |= PIPE_CONTROL_STATE_CACHE_INVALIDATE;
 		flags |= PIPE_CONTROL_QW_WRITE;
 		flags |= PIPE_CONTROL_GLOBAL_GTT_IVB;
-	}
-	/*
+		/*
-	 * On GEN9+ Before VF_CACHE_INVALIDATE we need to emit a NULL pipe
+		 * On GEN9: before VF_CACHE_INVALIDATE we need to emit a NULL
-	 * control.
+		 * pipe control.
-	 */
+		 */
-	vf_flush_wa = INTEL_INFO(ring->dev)->gen >= 9 &&
+		if (IS_GEN9(ring->dev))
+			vf_flush_wa = true;
 		      flags & PIPE_CONTROL_VF_CACHE_INVALIDATE;
 	ret = intel_logical_ring_begin(request, vf_flush_wa ? 12 : 6);
 	/* See bxt_a_get_seqno() explaining the reason for the clflush. */
 	intel_flush_status_page(ring, I915_GEM_HWS_INDEX);
-}
-static int gen8_emit_request(struct drm_i915_gem_request *request)
-{
-	struct intel_ringbuffer *ringbuf = request->ringbuf;
-	struct intel_engine_cs *ring = ringbuf->ring;
-	u32 cmd;
 	int ret;
-	/*
+/*
-	 * Reserve space for 2 NOOPs at the end of each request to be
+ * Reserve space for 2 NOOPs at the end of each request to be
-	 * used as a workaround for not being allowed to do lite
+ * used as a workaround for not being allowed to do lite
+ * restore with HEAD==TAIL (WaIdleLiteRestore).
+ */
+#define WA_TAIL_DWORDS 2
+static inline u32 hws_seqno_address(struct intel_engine_cs *engine)
+{
+	return engine->status_page.gfx_addr + I915_GEM_HWS_INDEX_ADDR;
+}
+static int gen8_emit_request(struct drm_i915_gem_request *request)
+{
+	struct intel_ringbuffer *ringbuf = request->ringbuf;
-	 * restore with HEAD==TAIL (WaIdleLiteRestore).
+	int ret;
 	 */
-	ret = intel_logical_ring_begin(request, 8);
+	ret = intel_logical_ring_begin(request, 6 + WA_TAIL_DWORDS);
 	if (ret)
 		return ret;
 	cmd = MI_STORE_DWORD_IMM_GEN4;
+	/* w/a: bit 5 needs to be zero for MI_FLUSH_DW address. */
+	BUILD_BUG_ON(I915_GEM_HWS_INDEX_ADDR & (1 << 5));
-	cmd |= MI_GLOBAL_GTT;
+	intel_logical_ring_emit(ringbuf,
-	intel_logical_ring_emit(ringbuf, cmd);
+				(MI_FLUSH_DW + 1) | MI_FLUSH_DW_OP_STOREDW);
 	intel_logical_ring_emit(ringbuf,
-				(ring->status_page.gfx_addr +
+				hws_seqno_address(request->ring) |
-				(I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT)));
+				MI_FLUSH_DW_USE_GTT);
 	intel_logical_ring_emit(ringbuf, 0);
+	intel_logical_ring_emit(ringbuf, i915_gem_request_get_seqno(request));
+	intel_logical_ring_emit(ringbuf, MI_USER_INTERRUPT);
-	intel_logical_ring_emit(ringbuf, i915_gem_request_get_seqno(request));
+	intel_logical_ring_emit(ringbuf, MI_NOOP);
+	return intel_logical_ring_advance_and_submit(request);
+}
+static int gen8_emit_request_render(struct drm_i915_gem_request *request)
+{
+	struct intel_ringbuffer *ringbuf = request->ringbuf;
+	int ret;
-	intel_logical_ring_emit(ringbuf, MI_USER_INTERRUPT);
+	ret = intel_logical_ring_begin(request, 8 + WA_TAIL_DWORDS);
+	if (ret)
+		return ret;
-	intel_logical_ring_emit(ringbuf, MI_NOOP);
+	/* We're using qword write, seqno should be aligned to 8 bytes. */
-	intel_logical_ring_advance_and_submit(request);
+	BUILD_BUG_ON(I915_GEM_HWS_INDEX & 1);
+	/* w/a for post sync ops following a GPGPU operation we
+	 * need a prior CS_STALL, which is emitted by the flush
+	 * following the batch.
+	 */
+	intel_logical_ring_emit(ringbuf, GFX_OP_PIPE_CONTROL(6));
+	intel_logical_ring_emit(ringbuf,
+				(PIPE_CONTROL_GLOBAL_GTT_IVB |
+				 PIPE_CONTROL_CS_STALL |
+				 PIPE_CONTROL_QW_WRITE));
+	intel_logical_ring_emit(ringbuf, hws_seqno_address(request->ring));
 	intel_logical_ring_emit(ringbuf, 0);
-	/*
-	 * Here we add two extra NOOPs as padding to avoid
+	intel_logical_ring_emit(ringbuf, i915_gem_request_get_seqno(request));
-	 * lite restore of a context with HEAD==TAIL.
-	 */
-	intel_logical_ring_emit(ringbuf, MI_NOOP);
+	/* We're thrashing one dword of HWS. */
-	intel_logical_ring_emit(ringbuf, MI_NOOP);
+	intel_logical_ring_emit(ringbuf, 0);
-	intel_logical_ring_advance(ringbuf);
+	intel_logical_ring_emit(ringbuf, MI_USER_INTERRUPT);
 	intel_logical_ring_emit(ringbuf, MI_NOOP);
 	if (ring->status_page.obj) {
 		kunmap(sg_page(ring->status_page.obj->pages->sgl));
 		ring->status_page.obj = NULL;
+	}
+	ring->disable_lite_restore_wa = false;
+	ring->ctx_desc_template = 0;
 	lrc_destroy_wa_ctx_obj(ring);
 	ring->dev = NULL;
+}
-}
+static void
 logical_ring_default_vfuncs(struct drm_device *dev,
+			    struct intel_engine_cs *ring)
+{
+	/* Default vfuncs which can be overriden by each engine. */
+	ring->init_hw = gen8_init_common_ring;
+	ring->emit_request = gen8_emit_request;
+	ring->emit_flush = gen8_emit_flush;
+	ring->irq_get = gen8_logical_ring_get_irq;
+	ring->irq_put = gen8_logical_ring_put_irq;
+	ring->emit_bb_start = gen8_emit_bb_start;
+	if (IS_BXT_REVID(dev, 0, BXT_REVID_A1)) {
+		ring->get_seqno = bxt_a_get_seqno;
+		ring->set_seqno = bxt_a_set_seqno;
+	} else {
+		ring->get_seqno = gen8_get_seqno;
+		ring->set_seqno = gen8_set_seqno;
+	}
+}
+static inline void
+logical_ring_default_irqs(struct intel_engine_cs *ring, unsigned shift)
+{
+	ring->irq_enable_mask = GT_RENDER_USER_INTERRUPT << shift;
+	ring->irq_keep_mask = GT_CONTEXT_SWITCH_INTERRUPT << shift;
+}
+static int
+logical_ring_init(struct drm_device *dev, struct intel_engine_cs *ring)
 static int logical_ring_init(struct drm_device *dev, struct intel_engine_cs *ring)
 	struct intel_context *dctx = to_i915(dev)->kernel_context;
 	int ret;
 	INIT_LIST_HEAD(&ring->buffers);
 	INIT_LIST_HEAD(&ring->execlist_queue);
 	INIT_LIST_HEAD(&ring->execlist_retired_req_list);
 	spin_lock_init(&ring->execlist_lock);
+	logical_ring_init_platform_invariants(ring);
 	ret = i915_cmd_parser_init_ring(ring);
 	if (ret)
 		goto error;
-	ret = intel_lr_context_deferred_alloc(ring->default_context, ring);
+	ret = intel_lr_context_deferred_alloc(dctx, ring);
 	if (ret)
 		goto error;
-	/* As this is the default context, always pin it */
-	ret = intel_lr_context_do_pin(
 			ring,
-			ring->default_context->engine[ring->id].state,
+	/* As this is the default context, always pin it */
-			ring->default_context->engine[ring->id].ringbuf);
+	ret = intel_lr_context_do_pin(dctx, ring);
 	if (ret) {
 		DRM_ERROR(
 	struct intel_engine_cs *ring = &dev_priv->ring[RCS];
 	int ret;
 	ring->name = "render ring";
+	ring->id = RCS;
+	ring->exec_id = I915_EXEC_RENDER;
-	ring->id = RCS;
+	ring->guc_id = GUC_RENDER_ENGINE;
-	ring->mmio_base = RENDER_RING_BASE;
+	ring->mmio_base = RENDER_RING_BASE;
-	ring->irq_enable_mask =
-		GT_RENDER_USER_INTERRUPT << GEN8_RCS_IRQ_SHIFT;
-	ring->irq_keep_mask =
-		GT_CONTEXT_SWITCH_INTERRUPT << GEN8_RCS_IRQ_SHIFT;
+	logical_ring_default_irqs(ring, GEN8_RCS_IRQ_SHIFT);
 	if (HAS_L3_DPF(dev))
+		ring->irq_keep_mask |= GT_RENDER_L3_PARITY_ERROR_INTERRUPT;
+	logical_ring_default_vfuncs(dev, ring);
 		ring->irq_keep_mask |= GT_RENDER_L3_PARITY_ERROR_INTERRUPT;
 	/* Override some for render ring. */
 	if (INTEL_INFO(dev)->gen >= 9)
 		ring->init_hw = gen9_init_render_ring;
 	else
 		ring->init_hw = gen8_init_render_ring;
-	ring->init_context = gen8_init_rcs_context;
-	ring->cleanup = intel_fini_pipe_control;
-	if (IS_BXT_REVID(dev, 0, BXT_REVID_A1)) {
-		ring->get_seqno = bxt_a_get_seqno;
-		ring->set_seqno = bxt_a_set_seqno;
-	} else {
-		ring->get_seqno = gen8_get_seqno;
-		ring->set_seqno = gen8_set_seqno;
 	ring->init_context = gen8_init_rcs_context;
-	ring->emit_request = gen8_emit_request;
-	ring->emit_flush = gen8_emit_flush_render;
-	ring->irq_get = gen8_logical_ring_get_irq;
+	ring->cleanup = intel_fini_pipe_control;
-	ring->irq_put = gen8_logical_ring_put_irq;
+	ring->emit_flush = gen8_emit_flush_render;
-	ring->emit_bb_start = gen8_emit_bb_start;
+	ring->emit_request = gen8_emit_request_render;
 	struct drm_i915_private *dev_priv = dev->dev_private;
 	struct intel_engine_cs *ring = &dev_priv->ring[VCS];
 	ring->name = "bsd ring";
+	ring->id = VCS;
+	ring->exec_id = I915_EXEC_BSD;
-	ring->id = VCS;
+	ring->guc_id = GUC_VIDEO_ENGINE;
-	ring->mmio_base = GEN6_BSD_RING_BASE;
-	ring->irq_enable_mask =
-		GT_RENDER_USER_INTERRUPT << GEN8_VCS1_IRQ_SHIFT;
-	ring->irq_keep_mask =
-		GT_CONTEXT_SWITCH_INTERRUPT << GEN8_VCS1_IRQ_SHIFT;
-	ring->init_hw = gen8_init_common_ring;
-	if (IS_BXT_REVID(dev, 0, BXT_REVID_A1)) {
-		ring->get_seqno = bxt_a_get_seqno;
-		ring->set_seqno = bxt_a_set_seqno;
-	} else {
-		ring->get_seqno = gen8_get_seqno;
-		ring->set_seqno = gen8_set_seqno;
+	ring->mmio_base = GEN6_BSD_RING_BASE;
-	}
-	ring->emit_request = gen8_emit_request;
-	ring->emit_flush = gen8_emit_flush;
-	ring->irq_get = gen8_logical_ring_get_irq;
-	ring->irq_put = gen8_logical_ring_put_irq;
+	logical_ring_default_irqs(ring, GEN8_VCS1_IRQ_SHIFT);
-	ring->emit_bb_start = gen8_emit_bb_start;
+	logical_ring_default_vfuncs(dev, ring);
 	return logical_ring_init(dev, ring);
+}
 static int logical_bsd2_ring_init(struct drm_device *dev)
+{
+	struct drm_i915_private *dev_priv = dev->dev_private;
+	struct intel_engine_cs *ring = &dev_priv->ring[VCS2];
 	struct drm_i915_private *dev_priv = dev->dev_private;
-	struct intel_engine_cs *ring = &dev_priv->ring[VCS2];
-	ring->name = "bds2 ring";
-	ring->id = VCS2;
-	ring->mmio_base = GEN8_BSD2_RING_BASE;
-	ring->irq_enable_mask =
-		GT_RENDER_USER_INTERRUPT << GEN8_VCS2_IRQ_SHIFT;
-	ring->irq_keep_mask =
+	ring->name = "bsd2 ring";
-		GT_CONTEXT_SWITCH_INTERRUPT << GEN8_VCS2_IRQ_SHIFT;
+	ring->id = VCS2;
-	ring->init_hw = gen8_init_common_ring;
-	ring->get_seqno = gen8_get_seqno;
-	ring->set_seqno = gen8_set_seqno;
+	ring->exec_id = I915_EXEC_BSD;
-	ring->emit_request = gen8_emit_request;
+	ring->guc_id = GUC_VIDEO_ENGINE2;
-	ring->emit_flush = gen8_emit_flush;
+	ring->mmio_base = GEN8_BSD2_RING_BASE;
 	struct drm_i915_private *dev_priv = dev->dev_private;
 	struct intel_engine_cs *ring = &dev_priv->ring[BCS];
 	ring->name = "blitter ring";
+	ring->id = BCS;
+	ring->exec_id = I915_EXEC_BLT;
-	ring->id = BCS;
+	ring->guc_id = GUC_BLITTER_ENGINE;
-	ring->mmio_base = BLT_RING_BASE;
-	ring->irq_enable_mask =
-		GT_RENDER_USER_INTERRUPT << GEN8_BCS_IRQ_SHIFT;
-	ring->irq_keep_mask =
-		GT_CONTEXT_SWITCH_INTERRUPT << GEN8_BCS_IRQ_SHIFT;
-	ring->init_hw = gen8_init_common_ring;
-	if (IS_BXT_REVID(dev, 0, BXT_REVID_A1)) {
+	ring->mmio_base = BLT_RING_BASE;
-		ring->get_seqno = bxt_a_get_seqno;
-		ring->set_seqno = bxt_a_set_seqno;
-	} else {
-		ring->get_seqno = gen8_get_seqno;
-		ring->set_seqno = gen8_set_seqno;
-	}
-	ring->emit_request = gen8_emit_request;
-	ring->emit_flush = gen8_emit_flush;
-	ring->irq_get = gen8_logical_ring_get_irq;
-	ring->irq_put = gen8_logical_ring_put_irq;
+	logical_ring_default_irqs(ring, GEN8_BCS_IRQ_SHIFT);
-	ring->emit_bb_start = gen8_emit_bb_start;
+	logical_ring_default_vfuncs(dev, ring);
 	struct drm_i915_private *dev_priv = dev->dev_private;
 	struct intel_engine_cs *ring = &dev_priv->ring[VECS];
 	ring->name = "video enhancement ring";
+	ring->id = VECS;
+	ring->exec_id = I915_EXEC_VEBOX;
-	ring->id = VECS;
+	ring->guc_id = GUC_VIDEOENHANCE_ENGINE;
-	ring->mmio_base = VEBOX_RING_BASE;
-	ring->irq_enable_mask =
-		GT_RENDER_USER_INTERRUPT << GEN8_VECS_IRQ_SHIFT;
-	ring->irq_keep_mask =
-		GT_CONTEXT_SWITCH_INTERRUPT << GEN8_VECS_IRQ_SHIFT;
-	ring->init_hw = gen8_init_common_ring;
-	if (IS_BXT_REVID(dev, 0, BXT_REVID_A1)) {
-		ring->get_seqno = bxt_a_get_seqno;
-		ring->set_seqno = bxt_a_set_seqno;
-	} else {
-		ring->get_seqno = gen8_get_seqno;
-		ring->set_seqno = gen8_set_seqno;
+	ring->mmio_base = VEBOX_RING_BASE;
-	}
-	ring->emit_request = gen8_emit_request;
-	ring->emit_flush = gen8_emit_flush;
-	ring->irq_get = gen8_logical_ring_get_irq;
-	ring->irq_put = gen8_logical_ring_put_irq;
+	logical_ring_default_irqs(ring, GEN8_VECS_IRQ_SHIFT);
-	ring->emit_bb_start = gen8_emit_bb_start;
+	logical_ring_default_vfuncs(dev, ring);
+	}
 	return rpcs;
+}
+static u32 intel_lr_indirect_ctx_offset(struct intel_engine_cs *ring)
+{
+	u32 indirect_ctx_offset;
+	switch (INTEL_INFO(ring->dev)->gen) {
+	default:
+		MISSING_CASE(INTEL_INFO(ring->dev)->gen);
+		/* fall through */
+	case 9:
+		indirect_ctx_offset =
+			GEN9_CTX_RCS_INDIRECT_CTX_OFFSET_DEFAULT;
+		break;
+	case 8:
+		indirect_ctx_offset =
+			GEN8_CTX_RCS_INDIRECT_CTX_OFFSET_DEFAULT;
+		break;
+	}
+	return indirect_ctx_offset;
+}
 static int
 populate_lr_context(struct intel_context *ctx, struct drm_i915_gem_object *ctx_obj,
 		    struct intel_engine_cs *ring, struct intel_ringbuffer *ringbuf)
 	reg_state[CTX_LRI_HEADER_0] =
 		MI_LOAD_REGISTER_IMM(ring->id == RCS ? 14 : 11) | MI_LRI_FORCE_POSTED;
 	ASSIGN_CTX_REG(reg_state, CTX_CONTEXT_CONTROL, RING_CONTEXT_CONTROL(ring),
-		_MASKED_BIT_ENABLE(CTX_CTRL_INHIBIT_SYN_CTX_SWITCH |
+		       _MASKED_BIT_ENABLE(CTX_CTRL_INHIBIT_SYN_CTX_SWITCH |
-				   CTX_CTRL_ENGINE_CTX_RESTORE_INHIBIT |
+					  CTX_CTRL_ENGINE_CTX_RESTORE_INHIBIT |
+					  (HAS_RESOURCE_STREAMER(dev) ?
-					  CTX_CTRL_RS_CTX_ENABLE));
+					    CTX_CTRL_RS_CTX_ENABLE : 0)));
 	ASSIGN_CTX_REG(reg_state, CTX_RING_HEAD, RING_HEAD(ring->mmio_base), 0);
 	ASSIGN_CTX_REG(reg_state, CTX_RING_TAIL, RING_TAIL(ring->mmio_base), 0);
 	/* Ring buffer start address is not known until the buffer is pinned.
 	 * It is written to the context image in execlists_update_context()
 	 */
 			reg_state[CTX_RCS_INDIRECT_CTX+1] =
 				(ggtt_offset + wa_ctx->indirect_ctx.offset * sizeof(uint32_t)) |
 				(wa_ctx->indirect_ctx.size / CACHELINE_DWORDS);
 			reg_state[CTX_RCS_INDIRECT_CTX_OFFSET+1] =
-				CTX_RCS_INDIRECT_CTX_OFFSET_DEFAULT << 6;
+				intel_lr_indirect_ctx_offset(ring) << 6;
 			reg_state[CTX_BB_PER_CTX_PTR+1] =
 				(ggtt_offset + wa_ctx->per_ctx.offset * sizeof(uint32_t)) |
  */
 void intel_lr_context_free(struct intel_context *ctx)
+{
 	int i;
+	for (i = I915_NUM_RINGS; --i >= 0; ) {
-	for (i = 0; i < I915_NUM_RINGS; i++) {
+		struct intel_ringbuffer *ringbuf = ctx->engine[i].ringbuf;
 		struct drm_i915_gem_object *ctx_obj = ctx->engine[i].state;
-		if (ctx_obj) {
-			struct intel_ringbuffer *ringbuf =
-					ctx->engine[i].ringbuf;
+		if (!ctx_obj)
-			struct intel_engine_cs *ring = ringbuf->ring;
+			continue;
-			if (ctx == ring->default_context) {
+		if (ctx == ctx->i915->kernel_context) {
+			intel_unpin_ringbuffer_obj(ringbuf);
-				intel_unpin_ringbuffer_obj(ringbuf);
+			i915_gem_object_ggtt_unpin(ctx_obj);
 				i915_gem_object_ggtt_unpin(ctx_obj);
-			WARN_ON(ctx->engine[ring->id].pin_count);
+		WARN_ON(ctx->engine[i].pin_count);
-			intel_ringbuffer_free(ringbuf);
+		intel_ringbuffer_free(ringbuf);
-			drm_gem_object_unreference(&ctx_obj->base);
+		drm_gem_object_unreference(&ctx_obj->base);
+	}
+}
+/**
+ * intel_lr_context_size() - return the size of the context for an engine
+ * @ring: which engine to find the context size for
+ *
+ * Each engine may require a different amount of space for a context image,
+ * so when allocating (or copying) an image, this function can be used to
+ * find the right size for the specific engine.
+ *
+ * Return: size (in bytes) of an engine-specific context image
+ *
-		}
+ * Note: this size includes the HWSP, which is part of the context image
  * in LRC mode, but does not include the "shared data page" used with
  * GuC submission. The caller should account for this if using the GuC.
  */
 	int ret;
 	WARN_ON(ctx->legacy_hw_ctx.rcs_state != NULL);
 	WARN_ON(ctx->engine[ring->id].state);
-	context_size = round_up(get_lr_context_size(ring), 4096);
+	context_size = round_up(intel_lr_context_size(ring), 4096);
+	}
 	ctx->engine[ring->id].ringbuf = ringbuf;
 	ctx->engine[ring->id].state = ctx_obj;
-	if (ctx != ring->default_context && ring->init_context) {
+	if (ctx != ctx->i915->kernel_context && ring->init_context) {
 		struct drm_i915_gem_request *req;
-		ret = i915_gem_request_alloc(ring,
+		req = i915_gem_request_alloc(ring, ctx);
-			ctx, &req);
-		if (ret) {
+		if (IS_ERR(req)) {
-			DRM_ERROR("ring create req: %d\n",
+			ret = PTR_ERR(req);
-				ret);
+			DRM_ERROR("ring create req: %d\n", ret);
 			goto error_ringbuf;

Subversion Repositories Kolibri OS

(root)/drivers/video/drm/i915/intel_lrc.c – Rev 6937 → 7144