41,6 → 41,16 |
return space; |
} |
|
void __intel_ring_advance(struct intel_ring_buffer *ring) |
{ |
struct drm_i915_private *dev_priv = ring->dev->dev_private; |
|
ring->tail &= ring->size - 1; |
if (dev_priv->gpu_error.stop_rings & intel_ring_flag(ring)) |
return; |
ring->write_tail(ring, ring->tail); |
} |
|
static int |
gen2_render_ring_flush(struct intel_ring_buffer *ring, |
u32 invalidate_domains, |
275,14 → 285,16 |
if (!ring->fbc_dirty) |
return 0; |
|
ret = intel_ring_begin(ring, 4); |
ret = intel_ring_begin(ring, 6); |
if (ret) |
return ret; |
intel_ring_emit(ring, MI_NOOP); |
/* WaFbcNukeOn3DBlt:ivb/hsw */ |
intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(1)); |
intel_ring_emit(ring, MSG_FBC_REND_STATE); |
intel_ring_emit(ring, value); |
intel_ring_emit(ring, MI_STORE_REGISTER_MEM(1) | MI_SRM_LRM_GLOBAL_GTT); |
intel_ring_emit(ring, MSG_FBC_REND_STATE); |
intel_ring_emit(ring, ring->scratch.gtt_offset + 256); |
intel_ring_advance(ring); |
|
ring->fbc_dirty = false; |
344,12 → 356,53 |
intel_ring_emit(ring, 0); |
intel_ring_advance(ring); |
|
if (flush_domains) |
if (!invalidate_domains && flush_domains) |
return gen7_ring_fbc_flush(ring, FBC_REND_NUKE); |
|
return 0; |
} |
|
static int |
gen8_render_ring_flush(struct intel_ring_buffer *ring, |
u32 invalidate_domains, u32 flush_domains) |
{ |
u32 flags = 0; |
u32 scratch_addr = ring->scratch.gtt_offset + 128; |
int ret; |
|
flags |= PIPE_CONTROL_CS_STALL; |
|
if (flush_domains) { |
flags |= PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH; |
flags |= PIPE_CONTROL_DEPTH_CACHE_FLUSH; |
} |
if (invalidate_domains) { |
flags |= PIPE_CONTROL_TLB_INVALIDATE; |
flags |= PIPE_CONTROL_INSTRUCTION_CACHE_INVALIDATE; |
flags |= PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE; |
flags |= PIPE_CONTROL_VF_CACHE_INVALIDATE; |
flags |= PIPE_CONTROL_CONST_CACHE_INVALIDATE; |
flags |= PIPE_CONTROL_STATE_CACHE_INVALIDATE; |
flags |= PIPE_CONTROL_QW_WRITE; |
flags |= PIPE_CONTROL_GLOBAL_GTT_IVB; |
} |
|
ret = intel_ring_begin(ring, 6); |
if (ret) |
return ret; |
|
intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(6)); |
intel_ring_emit(ring, flags); |
intel_ring_emit(ring, scratch_addr); |
intel_ring_emit(ring, 0); |
intel_ring_emit(ring, 0); |
intel_ring_emit(ring, 0); |
intel_ring_advance(ring); |
|
return 0; |
|
} |
|
static void ring_write_tail(struct intel_ring_buffer *ring, |
u32 value) |
{ |
385,8 → 438,7 |
int ret = 0; |
u32 head; |
|
if (HAS_FORCE_WAKE(dev)) |
gen6_gt_force_wake_get(dev_priv); |
gen6_gt_force_wake_get(dev_priv, FORCEWAKE_ALL); |
|
if (I915_NEED_GFX_HWS(dev)) |
intel_ring_setup_status_page(ring); |
455,8 → 507,7 |
memset(&ring->hangcheck, 0, sizeof(ring->hangcheck)); |
|
out: |
if (HAS_FORCE_WAKE(dev)) |
gen6_gt_force_wake_put(dev_priv); |
gen6_gt_force_wake_put(dev_priv, FORCEWAKE_ALL); |
|
return ret; |
} |
555,8 → 606,8 |
if (INTEL_INFO(dev)->gen >= 6) |
I915_WRITE(INSTPM, _MASKED_BIT_ENABLE(INSTPM_FORCE_ORDERING)); |
|
if (HAS_L3_GPU_CACHE(dev)) |
I915_WRITE_IMR(ring, ~GT_RENDER_L3_PARITY_ERROR_INTERRUPT); |
if (HAS_L3_DPF(dev)) |
I915_WRITE_IMR(ring, ~GT_PARITY_ERROR(dev)); |
|
return ret; |
} |
589,7 → 640,7 |
#define MBOX_UPDATE_DWORDS 4 |
intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(1)); |
intel_ring_emit(ring, mmio_offset); |
intel_ring_emit(ring, ring->outstanding_lazy_request); |
intel_ring_emit(ring, ring->outstanding_lazy_seqno); |
intel_ring_emit(ring, MI_NOOP); |
} |
|
608,26 → 659,29 |
struct drm_device *dev = ring->dev; |
struct drm_i915_private *dev_priv = dev->dev_private; |
struct intel_ring_buffer *useless; |
int i, ret; |
int i, ret, num_dwords = 4; |
|
ret = intel_ring_begin(ring, ((I915_NUM_RINGS-1) * |
MBOX_UPDATE_DWORDS) + |
4); |
if (i915_semaphore_is_enabled(dev)) |
num_dwords += ((I915_NUM_RINGS-1) * MBOX_UPDATE_DWORDS); |
#undef MBOX_UPDATE_DWORDS |
|
ret = intel_ring_begin(ring, num_dwords); |
if (ret) |
return ret; |
#undef MBOX_UPDATE_DWORDS |
|
if (i915_semaphore_is_enabled(dev)) { |
for_each_ring(useless, dev_priv, i) { |
u32 mbox_reg = ring->signal_mbox[i]; |
if (mbox_reg != GEN6_NOSYNC) |
update_mboxes(ring, mbox_reg); |
} |
} |
|
intel_ring_emit(ring, MI_STORE_DWORD_INDEX); |
intel_ring_emit(ring, I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT); |
intel_ring_emit(ring, ring->outstanding_lazy_request); |
intel_ring_emit(ring, ring->outstanding_lazy_seqno); |
intel_ring_emit(ring, MI_USER_INTERRUPT); |
intel_ring_advance(ring); |
__intel_ring_advance(ring); |
|
return 0; |
} |
719,7 → 773,7 |
PIPE_CONTROL_WRITE_FLUSH | |
PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE); |
intel_ring_emit(ring, ring->scratch.gtt_offset | PIPE_CONTROL_GLOBAL_GTT); |
intel_ring_emit(ring, ring->outstanding_lazy_request); |
intel_ring_emit(ring, ring->outstanding_lazy_seqno); |
intel_ring_emit(ring, 0); |
PIPE_CONTROL_FLUSH(ring, scratch_addr); |
scratch_addr += 128; /* write to separate cachelines */ |
738,9 → 792,9 |
PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE | |
PIPE_CONTROL_NOTIFY); |
intel_ring_emit(ring, ring->scratch.gtt_offset | PIPE_CONTROL_GLOBAL_GTT); |
intel_ring_emit(ring, ring->outstanding_lazy_request); |
intel_ring_emit(ring, ring->outstanding_lazy_seqno); |
intel_ring_emit(ring, 0); |
intel_ring_advance(ring); |
__intel_ring_advance(ring); |
|
return 0; |
} |
912,6 → 966,7 |
} else if (IS_GEN6(ring->dev)) { |
mmio = RING_HWS_PGA_GEN6(ring->mmio_base); |
} else { |
/* XXX: gen8 returns to sanity */ |
mmio = RING_HWS_PGA(ring->mmio_base); |
} |
|
959,9 → 1014,9 |
|
intel_ring_emit(ring, MI_STORE_DWORD_INDEX); |
intel_ring_emit(ring, I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT); |
intel_ring_emit(ring, ring->outstanding_lazy_request); |
intel_ring_emit(ring, ring->outstanding_lazy_seqno); |
intel_ring_emit(ring, MI_USER_INTERRUPT); |
intel_ring_advance(ring); |
__intel_ring_advance(ring); |
|
return 0; |
} |
976,17 → 1031,12 |
if (!dev->irq_enabled) |
return false; |
|
/* It looks like we need to prevent the gt from suspending while waiting |
* for an notifiy irq, otherwise irqs seem to get lost on at least the |
* blt/bsd rings on ivb. */ |
gen6_gt_force_wake_get(dev_priv); |
|
spin_lock_irqsave(&dev_priv->irq_lock, flags); |
if (ring->irq_refcount++ == 0) { |
if (HAS_L3_GPU_CACHE(dev) && ring->id == RCS) |
if (HAS_L3_DPF(dev) && ring->id == RCS) |
I915_WRITE_IMR(ring, |
~(ring->irq_enable_mask | |
GT_RENDER_L3_PARITY_ERROR_INTERRUPT)); |
GT_PARITY_ERROR(dev))); |
else |
I915_WRITE_IMR(ring, ~ring->irq_enable_mask); |
ilk_enable_gt_irq(dev_priv, ring->irq_enable_mask); |
1005,16 → 1055,13 |
|
spin_lock_irqsave(&dev_priv->irq_lock, flags); |
if (--ring->irq_refcount == 0) { |
if (HAS_L3_GPU_CACHE(dev) && ring->id == RCS) |
I915_WRITE_IMR(ring, |
~GT_RENDER_L3_PARITY_ERROR_INTERRUPT); |
if (HAS_L3_DPF(dev) && ring->id == RCS) |
I915_WRITE_IMR(ring, ~GT_PARITY_ERROR(dev)); |
else |
I915_WRITE_IMR(ring, ~0); |
ilk_disable_gt_irq(dev_priv, ring->irq_enable_mask); |
} |
spin_unlock_irqrestore(&dev_priv->irq_lock, flags); |
|
gen6_gt_force_wake_put(dev_priv); |
} |
|
static bool |
1055,6 → 1102,52 |
spin_unlock_irqrestore(&dev_priv->irq_lock, flags); |
} |
|
static bool |
gen8_ring_get_irq(struct intel_ring_buffer *ring) |
{ |
struct drm_device *dev = ring->dev; |
struct drm_i915_private *dev_priv = dev->dev_private; |
unsigned long flags; |
|
if (!dev->irq_enabled) |
return false; |
|
spin_lock_irqsave(&dev_priv->irq_lock, flags); |
if (ring->irq_refcount++ == 0) { |
if (HAS_L3_DPF(dev) && ring->id == RCS) { |
I915_WRITE_IMR(ring, |
~(ring->irq_enable_mask | |
GT_RENDER_L3_PARITY_ERROR_INTERRUPT)); |
} else { |
I915_WRITE_IMR(ring, ~ring->irq_enable_mask); |
} |
POSTING_READ(RING_IMR(ring->mmio_base)); |
} |
spin_unlock_irqrestore(&dev_priv->irq_lock, flags); |
|
return true; |
} |
|
static void |
gen8_ring_put_irq(struct intel_ring_buffer *ring) |
{ |
struct drm_device *dev = ring->dev; |
struct drm_i915_private *dev_priv = dev->dev_private; |
unsigned long flags; |
|
spin_lock_irqsave(&dev_priv->irq_lock, flags); |
if (--ring->irq_refcount == 0) { |
if (HAS_L3_DPF(dev) && ring->id == RCS) { |
I915_WRITE_IMR(ring, |
~GT_RENDER_L3_PARITY_ERROR_INTERRUPT); |
} else { |
I915_WRITE_IMR(ring, ~0); |
} |
POSTING_READ(RING_IMR(ring->mmio_base)); |
} |
spin_unlock_irqrestore(&dev_priv->irq_lock, flags); |
} |
|
static int |
i965_dispatch_execbuffer(struct intel_ring_buffer *ring, |
u32 offset, u32 length, |
1313,7 → 1406,7 |
/* Disable the ring buffer. The ring must be idle at this point */ |
dev_priv = ring->dev->dev_private; |
ret = intel_ring_idle(ring); |
if (ret) |
if (ret && !i915_reset_in_progress(&dev_priv->gpu_error)) |
DRM_ERROR("failed to quiesce %s whilst cleaning up: %d\n", |
ring->name, ret); |
|
1324,6 → 1417,8 |
i915_gem_object_unpin(ring->obj); |
drm_gem_object_unreference(&ring->obj->base); |
ring->obj = NULL; |
ring->preallocated_lazy_request = NULL; |
ring->outstanding_lazy_seqno = 0; |
|
if (ring->cleanup) |
ring->cleanup(ring); |
1410,6 → 1505,9 |
if (ret != -ENOSPC) |
return ret; |
|
/* force the tail write in case we have been skipping them */ |
__intel_ring_advance(ring); |
|
trace_i915_ring_wait_begin(ring); |
/* With GEM the hangcheck timer should kick us out of the loop, |
* leaving it early runs the risk of corrupting GEM state (due |
1465,7 → 1563,7 |
int ret; |
|
/* We need to add any requests required to flush the objects and ring */ |
if (ring->outstanding_lazy_request) { |
if (ring->outstanding_lazy_seqno) { |
ret = i915_add_request(ring, NULL); |
if (ret) |
return ret; |
1485,13 → 1583,23 |
static int |
intel_ring_alloc_seqno(struct intel_ring_buffer *ring) |
{ |
if (ring->outstanding_lazy_request) |
if (ring->outstanding_lazy_seqno) |
return 0; |
|
return i915_gem_get_seqno(ring->dev, &ring->outstanding_lazy_request); |
if (ring->preallocated_lazy_request == NULL) { |
struct drm_i915_gem_request *request; |
|
request = kmalloc(sizeof(*request), GFP_KERNEL); |
if (request == NULL) |
return -ENOMEM; |
|
ring->preallocated_lazy_request = request; |
} |
|
static int __intel_ring_begin(struct intel_ring_buffer *ring, |
return i915_gem_get_seqno(ring->dev, &ring->outstanding_lazy_seqno); |
} |
|
static int __intel_ring_prepare(struct intel_ring_buffer *ring, |
int bytes) |
{ |
int ret; |
1508,7 → 1616,6 |
return ret; |
} |
|
ring->space -= bytes; |
return 0; |
} |
|
1523,12 → 1630,17 |
if (ret) |
return ret; |
|
ret = __intel_ring_prepare(ring, num_dwords * sizeof(uint32_t)); |
if (ret) |
return ret; |
|
/* Preallocate the olr before touching the ring */ |
ret = intel_ring_alloc_seqno(ring); |
if (ret) |
return ret; |
|
return __intel_ring_begin(ring, num_dwords * sizeof(uint32_t)); |
ring->space -= num_dwords * sizeof(uint32_t); |
return 0; |
} |
|
void intel_ring_init_seqno(struct intel_ring_buffer *ring, u32 seqno) |
1535,7 → 1647,7 |
{ |
struct drm_i915_private *dev_priv = ring->dev->dev_private; |
|
BUG_ON(ring->outstanding_lazy_request); |
BUG_ON(ring->outstanding_lazy_seqno); |
|
if (INTEL_INFO(ring->dev)->gen >= 6) { |
I915_WRITE(RING_SYNC_0(ring->mmio_base), 0); |
1548,17 → 1660,6 |
ring->hangcheck.seqno = seqno; |
} |
|
void intel_ring_advance(struct intel_ring_buffer *ring) |
{ |
struct drm_i915_private *dev_priv = ring->dev->dev_private; |
|
ring->tail &= ring->size - 1; |
if (dev_priv->gpu_error.stop_rings & intel_ring_flag(ring)) |
return; |
ring->write_tail(ring, ring->tail); |
} |
|
|
static void gen6_bsd_ring_write_tail(struct intel_ring_buffer *ring, |
u32 value) |
{ |
1603,6 → 1704,8 |
return ret; |
|
cmd = MI_FLUSH_DW; |
if (INTEL_INFO(ring->dev)->gen >= 8) |
cmd += 1; |
/* |
* Bspec vol 1c.5 - video engine command streamer: |
* "If ENABLED, all TLBs will be invalidated once the flush |
1614,13 → 1717,42 |
MI_FLUSH_DW_STORE_INDEX | MI_FLUSH_DW_OP_STOREDW; |
intel_ring_emit(ring, cmd); |
intel_ring_emit(ring, I915_GEM_HWS_SCRATCH_ADDR | MI_FLUSH_DW_USE_GTT); |
if (INTEL_INFO(ring->dev)->gen >= 8) { |
intel_ring_emit(ring, 0); /* upper addr */ |
intel_ring_emit(ring, 0); /* value */ |
} else { |
intel_ring_emit(ring, 0); |
intel_ring_emit(ring, MI_NOOP); |
} |
intel_ring_advance(ring); |
return 0; |
} |
|
static int |
gen8_ring_dispatch_execbuffer(struct intel_ring_buffer *ring, |
u32 offset, u32 len, |
unsigned flags) |
{ |
struct drm_i915_private *dev_priv = ring->dev->dev_private; |
bool ppgtt = dev_priv->mm.aliasing_ppgtt != NULL && |
!(flags & I915_DISPATCH_SECURE); |
int ret; |
|
ret = intel_ring_begin(ring, 4); |
if (ret) |
return ret; |
|
/* FIXME(BDW): Address space and security selectors. */ |
intel_ring_emit(ring, MI_BATCH_BUFFER_START_GEN8 | (ppgtt<<8)); |
intel_ring_emit(ring, offset); |
intel_ring_emit(ring, 0); |
intel_ring_emit(ring, MI_NOOP); |
intel_ring_advance(ring); |
|
return 0; |
} |
|
static int |
hsw_ring_dispatch_execbuffer(struct intel_ring_buffer *ring, |
u32 offset, u32 len, |
unsigned flags) |
1676,6 → 1808,8 |
return ret; |
|
cmd = MI_FLUSH_DW; |
if (INTEL_INFO(ring->dev)->gen >= 8) |
cmd += 1; |
/* |
* Bspec vol 1c.3 - blitter engine command streamer: |
* "If ENABLED, all TLBs will be invalidated once the flush |
1687,11 → 1821,16 |
MI_FLUSH_DW_OP_STOREDW; |
intel_ring_emit(ring, cmd); |
intel_ring_emit(ring, I915_GEM_HWS_SCRATCH_ADDR | MI_FLUSH_DW_USE_GTT); |
if (INTEL_INFO(ring->dev)->gen >= 8) { |
intel_ring_emit(ring, 0); /* upper addr */ |
intel_ring_emit(ring, 0); /* value */ |
} else { |
intel_ring_emit(ring, 0); |
intel_ring_emit(ring, MI_NOOP); |
} |
intel_ring_advance(ring); |
|
if (IS_GEN7(dev) && flush) |
if (IS_GEN7(dev) && !invalidate && flush) |
return gen7_ring_fbc_flush(ring, FBC_REND_CACHE_CLEAN); |
|
return 0; |
1711,8 → 1850,14 |
ring->flush = gen7_render_ring_flush; |
if (INTEL_INFO(dev)->gen == 6) |
ring->flush = gen6_render_ring_flush; |
if (INTEL_INFO(dev)->gen >= 8) { |
ring->flush = gen8_render_ring_flush; |
ring->irq_get = gen8_ring_get_irq; |
ring->irq_put = gen8_ring_put_irq; |
} else { |
ring->irq_get = gen6_ring_get_irq; |
ring->irq_put = gen6_ring_put_irq; |
} |
ring->irq_enable_mask = GT_RENDER_USER_INTERRUPT; |
ring->get_seqno = gen6_ring_get_seqno; |
ring->set_seqno = ring_set_seqno; |
1754,6 → 1899,8 |
ring->write_tail = ring_write_tail; |
if (IS_HASWELL(dev)) |
ring->dispatch_execbuffer = hsw_ring_dispatch_execbuffer; |
else if (IS_GEN8(dev)) |
ring->dispatch_execbuffer = gen8_ring_dispatch_execbuffer; |
else if (INTEL_INFO(dev)->gen >= 6) |
ring->dispatch_execbuffer = gen6_ring_dispatch_execbuffer; |
else if (INTEL_INFO(dev)->gen >= 4) |
1869,7 → 2016,7 |
ring->id = VCS; |
|
ring->write_tail = ring_write_tail; |
if (IS_GEN6(dev) || IS_GEN7(dev)) { |
if (INTEL_INFO(dev)->gen >= 6) { |
ring->mmio_base = GEN6_BSD_RING_BASE; |
/* gen6 bsd needs a special wa for tail updates */ |
if (IS_GEN6(dev)) |
1878,10 → 2025,20 |
ring->add_request = gen6_add_request; |
ring->get_seqno = gen6_ring_get_seqno; |
ring->set_seqno = ring_set_seqno; |
if (INTEL_INFO(dev)->gen >= 8) { |
ring->irq_enable_mask = |
GT_RENDER_USER_INTERRUPT << GEN8_VCS1_IRQ_SHIFT; |
ring->irq_get = gen8_ring_get_irq; |
ring->irq_put = gen8_ring_put_irq; |
ring->dispatch_execbuffer = |
gen8_ring_dispatch_execbuffer; |
} else { |
ring->irq_enable_mask = GT_BSD_USER_INTERRUPT; |
ring->irq_get = gen6_ring_get_irq; |
ring->irq_put = gen6_ring_put_irq; |
ring->dispatch_execbuffer = gen6_ring_dispatch_execbuffer; |
ring->dispatch_execbuffer = |
gen6_ring_dispatch_execbuffer; |
} |
ring->sync_to = gen6_ring_sync; |
ring->semaphore_register[RCS] = MI_SEMAPHORE_SYNC_VR; |
ring->semaphore_register[VCS] = MI_SEMAPHORE_SYNC_INVALID; |
1927,10 → 2084,18 |
ring->add_request = gen6_add_request; |
ring->get_seqno = gen6_ring_get_seqno; |
ring->set_seqno = ring_set_seqno; |
if (INTEL_INFO(dev)->gen >= 8) { |
ring->irq_enable_mask = |
GT_RENDER_USER_INTERRUPT << GEN8_BCS_IRQ_SHIFT; |
ring->irq_get = gen8_ring_get_irq; |
ring->irq_put = gen8_ring_put_irq; |
ring->dispatch_execbuffer = gen8_ring_dispatch_execbuffer; |
} else { |
ring->irq_enable_mask = GT_BLT_USER_INTERRUPT; |
ring->irq_get = gen6_ring_get_irq; |
ring->irq_put = gen6_ring_put_irq; |
ring->dispatch_execbuffer = gen6_ring_dispatch_execbuffer; |
} |
ring->sync_to = gen6_ring_sync; |
ring->semaphore_register[RCS] = MI_SEMAPHORE_SYNC_BR; |
ring->semaphore_register[VCS] = MI_SEMAPHORE_SYNC_BV; |
1959,10 → 2124,19 |
ring->add_request = gen6_add_request; |
ring->get_seqno = gen6_ring_get_seqno; |
ring->set_seqno = ring_set_seqno; |
|
if (INTEL_INFO(dev)->gen >= 8) { |
ring->irq_enable_mask = |
GT_RENDER_USER_INTERRUPT << GEN8_VECS_IRQ_SHIFT; |
ring->irq_get = gen8_ring_get_irq; |
ring->irq_put = gen8_ring_put_irq; |
ring->dispatch_execbuffer = gen8_ring_dispatch_execbuffer; |
} else { |
ring->irq_enable_mask = PM_VEBOX_USER_INTERRUPT; |
ring->irq_get = hsw_vebox_get_irq; |
ring->irq_put = hsw_vebox_put_irq; |
ring->dispatch_execbuffer = gen6_ring_dispatch_execbuffer; |
} |
ring->sync_to = gen6_ring_sync; |
ring->semaphore_register[RCS] = MI_SEMAPHORE_SYNC_VER; |
ring->semaphore_register[VCS] = MI_SEMAPHORE_SYNC_VEV; |