/*
* Copyright © 2011 Intel Corporation
*
* Permission is hereby granted, free of charge, to any person obtaining a
* copy of this software and associated documentation files (the "Software"),
* to deal in the Software without restriction, including without limitation
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
* and/or sell copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice (including the next
* paragraph) shall be included in all copies or substantial portions of the
* Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
* IN THE SOFTWARE.
*/
/**
* @file gen7_sol_state.c
*
* Controls the stream output logic (SOL) stage of the gen7 hardware, which is
* used to implement GL_EXT_transform_feedback.
*/
#include "brw_context.h"
#include "brw_state.h"
#include "brw_defines.h"
#include "intel_batchbuffer.h"
#include "intel_buffer_objects.h"
#include "main/transformfeedback.h"
static void
upload_3dstate_so_buffers(struct brw_context *brw)
{
struct gl_context *ctx = &brw->ctx;
/* BRW_NEW_TRANSFORM_FEEDBACK */
struct gl_transform_feedback_object *xfb_obj =
ctx->TransformFeedback.CurrentObject;
const struct gl_transform_feedback_info *linked_xfb_info =
&xfb_obj->shader_program->LinkedTransformFeedback;
int i;
/* Set up the up to 4 output buffers. These are the ranges defined in the
* gl_transform_feedback_object.
*/
for (i = 0; i < 4; i++) {
struct intel_buffer_object *bufferobj =
intel_buffer_object(xfb_obj->Buffers[i]);
drm_intel_bo *bo;
uint32_t start, end;
uint32_t stride;
if (!xfb_obj->Buffers[i]) {
/* The pitch of 0 in this command indicates that the buffer is
* unbound and won't be written to.
*/
BEGIN_BATCH(4);
OUT_BATCH(_3DSTATE_SO_BUFFER << 16 | (4 - 2));
OUT_BATCH((i << SO_BUFFER_INDEX_SHIFT));
OUT_BATCH(0);
OUT_BATCH(0);
ADVANCE_BATCH();
continue;
}
stride = linked_xfb_info->BufferStride[i] * 4;
start = xfb_obj->Offset[i];
end = ALIGN(start + xfb_obj->Size[i], 4);
bo = intel_bufferobj_buffer(brw, bufferobj, start, end - start);
BEGIN_BATCH(4);
OUT_BATCH(_3DSTATE_SO_BUFFER << 16 | (4 - 2));
OUT_BATCH((i << SO_BUFFER_INDEX_SHIFT) | stride);
OUT_RELOC(bo, I915_GEM_DOMAIN_RENDER, I915_GEM_DOMAIN_RENDER, start);
OUT_RELOC(bo, I915_GEM_DOMAIN_RENDER, I915_GEM_DOMAIN_RENDER, end);
ADVANCE_BATCH();
}
}
/**
* Outputs the 3DSTATE_SO_DECL_LIST command.
*
* The data output is a series of 64-bit entries containing a SO_DECL per
* stream. We only have one stream of rendering coming out of the GS unit, so
* we only emit stream 0 (low 16 bits) SO_DECLs.
*/
void
gen7_upload_3dstate_so_decl_list(struct brw_context *brw,
const struct brw_vue_map *vue_map)
{
struct gl_context *ctx = &brw->ctx;
/* BRW_NEW_TRANSFORM_FEEDBACK */
struct gl_transform_feedback_object *xfb_obj =
ctx->TransformFeedback.CurrentObject;
const struct gl_transform_feedback_info *linked_xfb_info =
&xfb_obj->shader_program->LinkedTransformFeedback;
uint16_t so_decl[MAX_VERTEX_STREAMS][128];
int buffer_mask[MAX_VERTEX_STREAMS] = {0, 0, 0, 0};
int next_offset[MAX_VERTEX_STREAMS] = {0, 0, 0, 0};
int decls[MAX_VERTEX_STREAMS] = {0, 0, 0, 0};
int max_decls = 0;
STATIC_ASSERT(ARRAY_SIZE(so_decl[0]) >= MAX_PROGRAM_OUTPUTS);
memset(so_decl
, 0, sizeof(so_decl
));
/* Construct the list of SO_DECLs to be emitted. The formatting of the
* command is feels strange -- each dword pair contains a SO_DECL per stream.
*/
for (int i = 0; i < linked_xfb_info->NumOutputs; i++) {
int buffer = linked_xfb_info->Outputs[i].OutputBuffer;
uint16_t decl = 0;
int varying = linked_xfb_info->Outputs[i].OutputRegister;
const unsigned components = linked_xfb_info->Outputs[i].NumComponents;
unsigned component_mask = (1 << components) - 1;
unsigned stream_id = linked_xfb_info->Outputs[i].StreamId;
assert(stream_id
< MAX_VERTEX_STREAMS
);
/* gl_PointSize is stored in VARYING_SLOT_PSIZ.w
* gl_Layer is stored in VARYING_SLOT_PSIZ.y
* gl_ViewportIndex is stored in VARYING_SLOT_PSIZ.z
*/
if (varying == VARYING_SLOT_PSIZ) {
component_mask <<= 3;
} else if (varying == VARYING_SLOT_LAYER) {
component_mask <<= 1;
} else if (varying == VARYING_SLOT_VIEWPORT) {
component_mask <<= 2;
} else {
component_mask <<= linked_xfb_info->Outputs[i].ComponentOffset;
}
buffer_mask[stream_id] |= 1 << buffer;
decl |= buffer << SO_DECL_OUTPUT_BUFFER_SLOT_SHIFT;
if (varying == VARYING_SLOT_LAYER || varying == VARYING_SLOT_VIEWPORT) {
decl |= vue_map->varying_to_slot[VARYING_SLOT_PSIZ] <<
SO_DECL_REGISTER_INDEX_SHIFT;
} else {
assert(vue_map
->varying_to_slot
[varying
] >= 0); decl |= vue_map->varying_to_slot[varying] <<
SO_DECL_REGISTER_INDEX_SHIFT;
}
decl |= component_mask << SO_DECL_COMPONENT_MASK_SHIFT;
/* Mesa doesn't store entries for gl_SkipComponents in the Outputs[]
* array. Instead, it simply increments DstOffset for the following
* input by the number of components that should be skipped.
*
* Our hardware is unusual in that it requires us to program SO_DECLs
* for fake "hole" components, rather than simply taking the offset
* for each real varying. Each hole can have size 1, 2, 3, or 4; we
* program as many size = 4 holes as we can, then a final hole to
* accommodate the final 1, 2, or 3 remaining.
*/
int skip_components =
linked_xfb_info->Outputs[i].DstOffset - next_offset[buffer];
next_offset[buffer] += skip_components;
while (skip_components >= 4) {
so_decl[stream_id][decls[stream_id]++] = SO_DECL_HOLE_FLAG | 0xf;
skip_components -= 4;
}
if (skip_components > 0)
so_decl[stream_id][decls[stream_id]++] =
SO_DECL_HOLE_FLAG | ((1 << skip_components) - 1);
assert(linked_xfb_info
->Outputs
[i
].
DstOffset == next_offset
[buffer
]);
next_offset[buffer] += components;
so_decl[stream_id][decls[stream_id]++] = decl;
if (decls[stream_id] > max_decls)
max_decls = decls[stream_id];
}
BEGIN_BATCH(max_decls * 2 + 3);
OUT_BATCH(_3DSTATE_SO_DECL_LIST << 16 | (max_decls * 2 + 1));
OUT_BATCH((buffer_mask[0] << SO_STREAM_TO_BUFFER_SELECTS_0_SHIFT) |
(buffer_mask[1] << SO_STREAM_TO_BUFFER_SELECTS_1_SHIFT) |
(buffer_mask[2] << SO_STREAM_TO_BUFFER_SELECTS_2_SHIFT) |
(buffer_mask[3] << SO_STREAM_TO_BUFFER_SELECTS_3_SHIFT));
OUT_BATCH((decls[0] << SO_NUM_ENTRIES_0_SHIFT) |
(decls[1] << SO_NUM_ENTRIES_1_SHIFT) |
(decls[2] << SO_NUM_ENTRIES_2_SHIFT) |
(decls[3] << SO_NUM_ENTRIES_3_SHIFT));
for (int i = 0; i < max_decls; i++) {
/* Stream 1 | Stream 0 */
OUT_BATCH(((uint32_t) so_decl[1][i]) << 16 | so_decl[0][i]);
/* Stream 3 | Stream 2 */
OUT_BATCH(((uint32_t) so_decl[3][i]) << 16 | so_decl[2][i]);
}
ADVANCE_BATCH();
}
static void
upload_3dstate_streamout(struct brw_context *brw, bool active,
const struct brw_vue_map *vue_map)
{
struct gl_context *ctx = &brw->ctx;
/* BRW_NEW_TRANSFORM_FEEDBACK */
struct gl_transform_feedback_object *xfb_obj =
ctx->TransformFeedback.CurrentObject;
uint32_t dw1 = 0, dw2 = 0;
int i;
if (active) {
int urb_entry_read_offset = 0;
int urb_entry_read_length = (vue_map->num_slots + 1) / 2 -
urb_entry_read_offset;
dw1 |= SO_FUNCTION_ENABLE;
dw1 |= SO_STATISTICS_ENABLE;
/* _NEW_LIGHT */
if (ctx->Light.ProvokingVertex != GL_FIRST_VERTEX_CONVENTION)
dw1 |= SO_REORDER_TRAILING;
for (i = 0; i < 4; i++) {
if (xfb_obj->Buffers[i]) {
dw1 |= SO_BUFFER_ENABLE(i);
}
}
/* We always read the whole vertex. This could be reduced at some
* point by reading less and offsetting the register index in the
* SO_DECLs.
*/
dw2 |= SET_FIELD(urb_entry_read_offset, SO_STREAM_0_VERTEX_READ_OFFSET);
dw2 |= SET_FIELD(urb_entry_read_length - 1, SO_STREAM_0_VERTEX_READ_LENGTH);
dw2 |= SET_FIELD(urb_entry_read_offset, SO_STREAM_1_VERTEX_READ_OFFSET);
dw2 |= SET_FIELD(urb_entry_read_length - 1, SO_STREAM_1_VERTEX_READ_LENGTH);
dw2 |= SET_FIELD(urb_entry_read_offset, SO_STREAM_2_VERTEX_READ_OFFSET);
dw2 |= SET_FIELD(urb_entry_read_length - 1, SO_STREAM_2_VERTEX_READ_LENGTH);
dw2 |= SET_FIELD(urb_entry_read_offset, SO_STREAM_3_VERTEX_READ_OFFSET);
dw2 |= SET_FIELD(urb_entry_read_length - 1, SO_STREAM_3_VERTEX_READ_LENGTH);
}
BEGIN_BATCH(3);
OUT_BATCH(_3DSTATE_STREAMOUT << 16 | (3 - 2));
OUT_BATCH(dw1);
OUT_BATCH(dw2);
ADVANCE_BATCH();
}
static void
upload_sol_state(struct brw_context *brw)
{
struct gl_context *ctx = &brw->ctx;
/* BRW_NEW_TRANSFORM_FEEDBACK */
bool active = _mesa_is_xfb_active_and_unpaused(ctx);
if (active) {
upload_3dstate_so_buffers(brw);
/* BRW_NEW_VUE_MAP_GEOM_OUT */
gen7_upload_3dstate_so_decl_list(brw, &brw->vue_map_geom_out);
}
/* Finally, set up the SOL stage. This command must always follow updates to
* the nonpipelined SOL state (3DSTATE_SO_BUFFER, 3DSTATE_SO_DECL_LIST) or
* MMIO register updates (current performed by the kernel at each batch
* emit).
*/
upload_3dstate_streamout(brw, active, &brw->vue_map_geom_out);
}
const struct brw_tracked_state gen7_sol_state = {
.dirty = {
.mesa = _NEW_LIGHT,
.brw = BRW_NEW_BATCH |
BRW_NEW_VUE_MAP_GEOM_OUT |
BRW_NEW_TRANSFORM_FEEDBACK,
},
.emit = upload_sol_state,
};
/**
* Tally the number of primitives generated so far.
*
* The buffer contains a series of pairs:
* (<start0, start1, start2, start3>, <end0, end1, end2, end3>) ;
* (<start0, start1, start2, start3>, <end0, end1, end2, end3>) ;
*
* For each stream, we subtract the pair of values (end - start) to get the
* number of primitives generated during one section. We accumulate these
* values, adding them up to get the total number of primitives generated.
*/
static void
gen7_tally_prims_generated(struct brw_context *brw,
struct brw_transform_feedback_object *obj)
{
/* If the current batch is still contributing to the number of primitives
* generated, flush it now so the results will be present when mapped.
*/
if (drm_intel_bo_references(brw->batch.bo, obj->prim_count_bo))
intel_batchbuffer_flush(brw);
if (unlikely(brw->perf_debug && drm_intel_bo_busy(obj->prim_count_bo)))
perf_debug("Stalling for # of transform feedback primitives written.\n");
drm_intel_bo_map(obj->prim_count_bo, false);
uint64_t *prim_counts = obj->prim_count_bo->virtual;
assert(obj
->prim_count_buffer_index
% (2 * BRW_MAX_XFB_STREAMS
) == 0); int pairs = obj->prim_count_buffer_index / (2 * BRW_MAX_XFB_STREAMS);
for (int i = 0; i < pairs; i++) {
for (int s = 0; s < BRW_MAX_XFB_STREAMS; s++) {
obj->prims_generated[s] +=
prim_counts[BRW_MAX_XFB_STREAMS + s] - prim_counts[s];
}
prim_counts += 2 * BRW_MAX_XFB_STREAMS; /* move to the next pair */
}
drm_intel_bo_unmap(obj->prim_count_bo);
/* We've already gathered up the old data; we can safely overwrite it now. */
obj->prim_count_buffer_index = 0;
}
/**
* Store the SO_NUM_PRIMS_WRITTEN counters for each stream (4 uint64_t values)
* to prim_count_bo.
*
* If prim_count_bo is out of space, gather up the results so far into
* prims_generated[] and allocate a new buffer with enough space.
*
* The number of primitives written is used to compute the number of vertices
* written to a transform feedback stream, which is required to implement
* DrawTransformFeedback().
*/
static void
gen7_save_primitives_written_counters(struct brw_context *brw,
struct brw_transform_feedback_object *obj)
{
const int streams = BRW_MAX_XFB_STREAMS;
/* Check if there's enough space for a new pair of four values. */
if (obj->prim_count_bo != NULL &&
obj->prim_count_buffer_index + 2 * streams >= 4096 / sizeof(uint64_t)) {
/* Gather up the results so far and release the BO. */
gen7_tally_prims_generated(brw, obj);
}
/* Flush any drawing so that the counters have the right values. */
intel_batchbuffer_emit_mi_flush(brw);
/* Emit MI_STORE_REGISTER_MEM commands to write the values. */
for (int i = 0; i < streams; i++) {
brw_store_register_mem64(brw, obj->prim_count_bo,
GEN7_SO_NUM_PRIMS_WRITTEN(i),
obj->prim_count_buffer_index + i);
}
/* Update where to write data to. */
obj->prim_count_buffer_index += streams;
}
/**
* Compute the number of vertices written by this transform feedback operation.
*/
static void
brw_compute_xfb_vertices_written(struct brw_context *brw,
struct brw_transform_feedback_object *obj)
{
if (obj->vertices_written_valid || !obj->base.EndedAnytime)
return;
unsigned vertices_per_prim = 0;
switch (obj->primitive_mode) {
case GL_POINTS:
vertices_per_prim = 1;
break;
case GL_LINES:
vertices_per_prim = 2;
break;
case GL_TRIANGLES:
vertices_per_prim = 3;
break;
default:
unreachable("Invalid transform feedback primitive mode.");
}
/* Get the number of primitives generated. */
gen7_tally_prims_generated(brw, obj);
for (int i = 0; i < BRW_MAX_XFB_STREAMS; i++) {
obj->vertices_written[i] = vertices_per_prim * obj->prims_generated[i];
}
obj->vertices_written_valid = true;
}
/**
* GetTransformFeedbackVertexCount() driver hook.
*
* Returns the number of vertices written to a particular stream by the last
* Begin/EndTransformFeedback block. Used to implement DrawTransformFeedback().
*/
GLsizei
brw_get_transform_feedback_vertex_count(struct gl_context *ctx,
struct gl_transform_feedback_object *obj,
GLuint stream)
{
struct brw_context *brw = brw_context(ctx);
struct brw_transform_feedback_object *brw_obj =
(struct brw_transform_feedback_object *) obj;
assert(stream
< BRW_MAX_XFB_STREAMS
);
brw_compute_xfb_vertices_written(brw, brw_obj);
return brw_obj->vertices_written[stream];
}
void
gen7_begin_transform_feedback(struct gl_context *ctx, GLenum mode,
struct gl_transform_feedback_object *obj)
{
struct brw_context *brw = brw_context(ctx);
struct brw_transform_feedback_object *brw_obj =
(struct brw_transform_feedback_object *) obj;
/* Reset the SO buffer offsets to 0. */
if (brw->gen >= 8) {
brw_obj->zero_offsets = true;
} else {
intel_batchbuffer_flush(brw);
brw->batch.needs_sol_reset = true;
}
/* We're about to lose the information needed to compute the number of
* vertices written during the last Begin/EndTransformFeedback section,
* so we can't delay it any further.
*/
brw_compute_xfb_vertices_written(brw, brw_obj);
/* No primitives have been generated yet. */
for (int i = 0; i < BRW_MAX_XFB_STREAMS; i++) {
brw_obj->prims_generated[i] = 0;
}
/* Store the starting value of the SO_NUM_PRIMS_WRITTEN counters. */
gen7_save_primitives_written_counters(brw, brw_obj);
brw_obj->primitive_mode = mode;
}
void
gen7_end_transform_feedback(struct gl_context *ctx,
struct gl_transform_feedback_object *obj)
{
/* After EndTransformFeedback, it's likely that the client program will try
* to draw using the contents of the transform feedback buffer as vertex
* input. In order for this to work, we need to flush the data through at
* least the GS stage of the pipeline, and flush out the render cache. For
* simplicity, just do a full flush.
*/
struct brw_context *brw = brw_context(ctx);
struct brw_transform_feedback_object *brw_obj =
(struct brw_transform_feedback_object *) obj;
/* Store the ending value of the SO_NUM_PRIMS_WRITTEN counters. */
gen7_save_primitives_written_counters(brw, brw_obj);
/* EndTransformFeedback() means that we need to update the number of
* vertices written. Since it's only necessary if DrawTransformFeedback()
* is called and it means mapping a buffer object, we delay computing it
* until it's absolutely necessary to try and avoid stalls.
*/
brw_obj->vertices_written_valid = false;
}
void
gen7_pause_transform_feedback(struct gl_context *ctx,
struct gl_transform_feedback_object *obj)
{
struct brw_context *brw = brw_context(ctx);
struct brw_transform_feedback_object *brw_obj =
(struct brw_transform_feedback_object *) obj;
/* Flush any drawing so that the counters have the right values. */
intel_batchbuffer_emit_mi_flush(brw);
/* Save the SOL buffer offset register values. */
if (brw->gen < 8) {
for (int i = 0; i < 4; i++) {
BEGIN_BATCH(3);
OUT_BATCH(MI_STORE_REGISTER_MEM | (3 - 2));
OUT_BATCH(GEN7_SO_WRITE_OFFSET(i));
OUT_RELOC(brw_obj->offset_bo,
I915_GEM_DOMAIN_INSTRUCTION, I915_GEM_DOMAIN_INSTRUCTION,
i * sizeof(uint32_t));
ADVANCE_BATCH();
}
}
/* Store the temporary ending value of the SO_NUM_PRIMS_WRITTEN counters.
* While this operation is paused, other transform feedback actions may
* occur, which will contribute to the counters. We need to exclude that
* from our counts.
*/
gen7_save_primitives_written_counters(brw, brw_obj);
}
void
gen7_resume_transform_feedback(struct gl_context *ctx,
struct gl_transform_feedback_object *obj)
{
struct brw_context *brw = brw_context(ctx);
struct brw_transform_feedback_object *brw_obj =
(struct brw_transform_feedback_object *) obj;
/* Reload the SOL buffer offset registers. */
if (brw->gen < 8) {
for (int i = 0; i < 4; i++) {
BEGIN_BATCH(3);
OUT_BATCH(GEN7_MI_LOAD_REGISTER_MEM | (3 - 2));
OUT_BATCH(GEN7_SO_WRITE_OFFSET(i));
OUT_RELOC(brw_obj->offset_bo,
I915_GEM_DOMAIN_INSTRUCTION, I915_GEM_DOMAIN_INSTRUCTION,
i * sizeof(uint32_t));
ADVANCE_BATCH();
}
}
/* Store the new starting value of the SO_NUM_PRIMS_WRITTEN counters. */
gen7_save_primitives_written_counters(brw, brw_obj);
}