64,7 → 64,8 |
__raw_posting_read(dev_priv, ECOBUS); |
} |
|
static void __gen6_gt_force_wake_get(struct drm_i915_private *dev_priv) |
static void __gen6_gt_force_wake_get(struct drm_i915_private *dev_priv, |
int fw_engine) |
{ |
if (wait_for_atomic((__raw_i915_read32(dev_priv, FORCEWAKE_ACK) & 1) == 0, |
FORCEWAKE_ACK_TIMEOUT_MS)) |
89,11 → 90,12 |
__raw_posting_read(dev_priv, ECOBUS); |
} |
|
static void __gen6_gt_force_wake_mt_get(struct drm_i915_private *dev_priv) |
static void __gen6_gt_force_wake_mt_get(struct drm_i915_private *dev_priv, |
int fw_engine) |
{ |
u32 forcewake_ack; |
|
if (IS_HASWELL(dev_priv->dev)) |
if (IS_HASWELL(dev_priv->dev) || IS_GEN8(dev_priv->dev)) |
forcewake_ack = FORCEWAKE_ACK_HSW; |
else |
forcewake_ack = FORCEWAKE_MT_ACK; |
112,6 → 114,7 |
DRM_ERROR("Timed out waiting for forcewake to ack request.\n"); |
|
/* WaRsForcewakeWaitTC0:ivb,hsw */ |
if (INTEL_INFO(dev_priv->dev)->gen < 8) |
__gen6_gt_wait_for_thread_c0(dev_priv); |
} |
|
120,12 → 123,12 |
u32 gtfifodbg; |
|
gtfifodbg = __raw_i915_read32(dev_priv, GTFIFODBG); |
if (WARN(gtfifodbg & GT_FIFO_CPU_ERROR_MASK, |
"MMIO read or write has been dropped %x\n", gtfifodbg)) |
__raw_i915_write32(dev_priv, GTFIFODBG, GT_FIFO_CPU_ERROR_MASK); |
if (WARN(gtfifodbg, "GT wake FIFO error 0x%x\n", gtfifodbg)) |
__raw_i915_write32(dev_priv, GTFIFODBG, gtfifodbg); |
} |
|
static void __gen6_gt_force_wake_put(struct drm_i915_private *dev_priv) |
static void __gen6_gt_force_wake_put(struct drm_i915_private *dev_priv, |
int fw_engine) |
{ |
__raw_i915_write32(dev_priv, FORCEWAKE, 0); |
/* something from same cacheline, but !FORCEWAKE */ |
133,7 → 136,8 |
gen6_gt_check_fifodbg(dev_priv); |
} |
|
static void __gen6_gt_force_wake_mt_put(struct drm_i915_private *dev_priv) |
static void __gen6_gt_force_wake_mt_put(struct drm_i915_private *dev_priv, |
int fw_engine) |
{ |
__raw_i915_write32(dev_priv, FORCEWAKE_MT, |
_MASKED_BIT_DISABLE(FORCEWAKE_KERNEL)); |
146,12 → 150,19 |
{ |
int ret = 0; |
|
/* On VLV, FIFO will be shared by both SW and HW. |
* So, we need to read the FREE_ENTRIES everytime */ |
if (IS_VALLEYVIEW(dev_priv->dev)) |
dev_priv->uncore.fifo_count = |
__raw_i915_read32(dev_priv, GTFIFOCTL) & |
GT_FIFO_FREE_ENTRIES_MASK; |
|
if (dev_priv->uncore.fifo_count < GT_FIFO_NUM_RESERVED_ENTRIES) { |
int loop = 500; |
u32 fifo = __raw_i915_read32(dev_priv, GT_FIFO_FREE_ENTRIES); |
u32 fifo = __raw_i915_read32(dev_priv, GTFIFOCTL) & GT_FIFO_FREE_ENTRIES_MASK; |
while (fifo <= GT_FIFO_NUM_RESERVED_ENTRIES && loop--) { |
udelay(10); |
fifo = __raw_i915_read32(dev_priv, GT_FIFO_FREE_ENTRIES); |
fifo = __raw_i915_read32(dev_priv, GTFIFOCTL) & GT_FIFO_FREE_ENTRIES_MASK; |
} |
if (WARN_ON(loop < 0 && fifo <= GT_FIFO_NUM_RESERVED_ENTRIES)) |
++ret; |
170,40 → 181,126 |
__raw_posting_read(dev_priv, FORCEWAKE_ACK_VLV); |
} |
|
static void vlv_force_wake_get(struct drm_i915_private *dev_priv) |
static void __vlv_force_wake_get(struct drm_i915_private *dev_priv, |
int fw_engine) |
{ |
if (wait_for_atomic((__raw_i915_read32(dev_priv, FORCEWAKE_ACK_VLV) & FORCEWAKE_KERNEL) == 0, |
/* Check for Render Engine */ |
if (FORCEWAKE_RENDER & fw_engine) { |
if (wait_for_atomic((__raw_i915_read32(dev_priv, |
FORCEWAKE_ACK_VLV) & |
FORCEWAKE_KERNEL) == 0, |
FORCEWAKE_ACK_TIMEOUT_MS)) |
DRM_ERROR("Timed out waiting for forcewake old ack to clear.\n"); |
DRM_ERROR("Timed out: Render forcewake old ack to clear.\n"); |
|
__raw_i915_write32(dev_priv, FORCEWAKE_VLV, |
_MASKED_BIT_ENABLE(FORCEWAKE_KERNEL)); |
|
if (wait_for_atomic((__raw_i915_read32(dev_priv, |
FORCEWAKE_ACK_VLV) & |
FORCEWAKE_KERNEL), |
FORCEWAKE_ACK_TIMEOUT_MS)) |
DRM_ERROR("Timed out: waiting for Render to ack.\n"); |
} |
|
/* Check for Media Engine */ |
if (FORCEWAKE_MEDIA & fw_engine) { |
if (wait_for_atomic((__raw_i915_read32(dev_priv, |
FORCEWAKE_ACK_MEDIA_VLV) & |
FORCEWAKE_KERNEL) == 0, |
FORCEWAKE_ACK_TIMEOUT_MS)) |
DRM_ERROR("Timed out: Media forcewake old ack to clear.\n"); |
|
__raw_i915_write32(dev_priv, FORCEWAKE_MEDIA_VLV, |
_MASKED_BIT_ENABLE(FORCEWAKE_KERNEL)); |
|
if (wait_for_atomic((__raw_i915_read32(dev_priv, FORCEWAKE_ACK_VLV) & FORCEWAKE_KERNEL), |
FORCEWAKE_ACK_TIMEOUT_MS)) |
DRM_ERROR("Timed out waiting for GT to ack forcewake request.\n"); |
|
if (wait_for_atomic((__raw_i915_read32(dev_priv, FORCEWAKE_ACK_MEDIA_VLV) & |
if (wait_for_atomic((__raw_i915_read32(dev_priv, |
FORCEWAKE_ACK_MEDIA_VLV) & |
FORCEWAKE_KERNEL), |
FORCEWAKE_ACK_TIMEOUT_MS)) |
DRM_ERROR("Timed out waiting for media to ack forcewake request.\n"); |
DRM_ERROR("Timed out: waiting for media to ack.\n"); |
} |
|
/* WaRsForcewakeWaitTC0:vlv */ |
__gen6_gt_wait_for_thread_c0(dev_priv); |
|
} |
|
static void vlv_force_wake_put(struct drm_i915_private *dev_priv) |
static void __vlv_force_wake_put(struct drm_i915_private *dev_priv, |
int fw_engine) |
{ |
|
/* Check for Render Engine */ |
if (FORCEWAKE_RENDER & fw_engine) |
__raw_i915_write32(dev_priv, FORCEWAKE_VLV, |
_MASKED_BIT_DISABLE(FORCEWAKE_KERNEL)); |
|
|
/* Check for Media Engine */ |
if (FORCEWAKE_MEDIA & fw_engine) |
__raw_i915_write32(dev_priv, FORCEWAKE_MEDIA_VLV, |
_MASKED_BIT_DISABLE(FORCEWAKE_KERNEL)); |
|
/* The below doubles as a POSTING_READ */ |
gen6_gt_check_fifodbg(dev_priv); |
|
} |
|
void vlv_force_wake_get(struct drm_i915_private *dev_priv, |
int fw_engine) |
{ |
unsigned long irqflags; |
|
spin_lock_irqsave(&dev_priv->uncore.lock, irqflags); |
if (FORCEWAKE_RENDER & fw_engine) { |
if (dev_priv->uncore.fw_rendercount++ == 0) |
dev_priv->uncore.funcs.force_wake_get(dev_priv, |
FORCEWAKE_RENDER); |
} |
if (FORCEWAKE_MEDIA & fw_engine) { |
if (dev_priv->uncore.fw_mediacount++ == 0) |
dev_priv->uncore.funcs.force_wake_get(dev_priv, |
FORCEWAKE_MEDIA); |
} |
|
spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags); |
} |
|
void vlv_force_wake_put(struct drm_i915_private *dev_priv, |
int fw_engine) |
{ |
unsigned long irqflags; |
|
spin_lock_irqsave(&dev_priv->uncore.lock, irqflags); |
|
if (FORCEWAKE_RENDER & fw_engine) { |
WARN_ON(dev_priv->uncore.fw_rendercount == 0); |
if (--dev_priv->uncore.fw_rendercount == 0) |
dev_priv->uncore.funcs.force_wake_put(dev_priv, |
FORCEWAKE_RENDER); |
} |
|
if (FORCEWAKE_MEDIA & fw_engine) { |
WARN_ON(dev_priv->uncore.fw_mediacount == 0); |
if (--dev_priv->uncore.fw_mediacount == 0) |
dev_priv->uncore.funcs.force_wake_put(dev_priv, |
FORCEWAKE_MEDIA); |
} |
|
spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags); |
} |
|
static void gen6_force_wake_work(struct work_struct *work) |
{ |
struct drm_i915_private *dev_priv = |
container_of(work, typeof(*dev_priv), uncore.force_wake_work.work); |
unsigned long irqflags; |
|
spin_lock_irqsave(&dev_priv->uncore.lock, irqflags); |
if (--dev_priv->uncore.forcewake_count == 0) |
dev_priv->uncore.funcs.force_wake_put(dev_priv, FORCEWAKE_ALL); |
spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags); |
} |
|
static void intel_uncore_forcewake_reset(struct drm_device *dev) |
{ |
struct drm_i915_private *dev_priv = dev->dev_private; |
223,66 → 320,47 |
|
if (HAS_FPGA_DBG_UNCLAIMED(dev)) |
__raw_i915_write32(dev_priv, FPGA_DBG, FPGA_DBG_RM_NOCLAIM); |
|
if (IS_HASWELL(dev) && |
(__raw_i915_read32(dev_priv, HSW_EDRAM_PRESENT) == 1)) { |
/* The docs do not explain exactly how the calculation can be |
* made. It is somewhat guessable, but for now, it's always |
* 128MB. |
* NB: We can't write IDICR yet because we do not have gt funcs |
* set up */ |
dev_priv->ellc_size = 128; |
DRM_INFO("Found %zuMB of eLLC\n", dev_priv->ellc_size); |
} |
|
void intel_uncore_init(struct drm_device *dev) |
{ |
struct drm_i915_private *dev_priv = dev->dev_private; |
/* clear out old GT FIFO errors */ |
if (IS_GEN6(dev) || IS_GEN7(dev)) |
__raw_i915_write32(dev_priv, GTFIFODBG, |
__raw_i915_read32(dev_priv, GTFIFODBG)); |
|
if (IS_VALLEYVIEW(dev)) { |
dev_priv->uncore.funcs.force_wake_get = vlv_force_wake_get; |
dev_priv->uncore.funcs.force_wake_put = vlv_force_wake_put; |
} else if (IS_HASWELL(dev)) { |
dev_priv->uncore.funcs.force_wake_get = __gen6_gt_force_wake_mt_get; |
dev_priv->uncore.funcs.force_wake_put = __gen6_gt_force_wake_mt_put; |
} else if (IS_IVYBRIDGE(dev)) { |
u32 ecobus; |
|
/* IVB configs may use multi-threaded forcewake */ |
|
/* A small trick here - if the bios hasn't configured |
* MT forcewake, and if the device is in RC6, then |
* force_wake_mt_get will not wake the device and the |
* ECOBUS read will return zero. Which will be |
* (correctly) interpreted by the test below as MT |
* forcewake being disabled. |
*/ |
mutex_lock(&dev->struct_mutex); |
__gen6_gt_force_wake_mt_get(dev_priv); |
ecobus = __raw_i915_read32(dev_priv, ECOBUS); |
__gen6_gt_force_wake_mt_put(dev_priv); |
mutex_unlock(&dev->struct_mutex); |
|
if (ecobus & FORCEWAKE_MT_ENABLE) { |
dev_priv->uncore.funcs.force_wake_get = |
__gen6_gt_force_wake_mt_get; |
dev_priv->uncore.funcs.force_wake_put = |
__gen6_gt_force_wake_mt_put; |
} else { |
DRM_INFO("No MT forcewake available on Ivybridge, this can result in issues\n"); |
DRM_INFO("when using vblank-synced partial screen updates.\n"); |
dev_priv->uncore.funcs.force_wake_get = |
__gen6_gt_force_wake_get; |
dev_priv->uncore.funcs.force_wake_put = |
__gen6_gt_force_wake_put; |
} |
} else if (IS_GEN6(dev)) { |
dev_priv->uncore.funcs.force_wake_get = |
__gen6_gt_force_wake_get; |
dev_priv->uncore.funcs.force_wake_put = |
__gen6_gt_force_wake_put; |
} |
|
intel_uncore_forcewake_reset(dev); |
} |
|
void intel_uncore_sanitize(struct drm_device *dev) |
{ |
intel_uncore_forcewake_reset(dev); |
struct drm_i915_private *dev_priv = dev->dev_private; |
u32 reg_val; |
|
/* BIOS often leaves RC6 enabled, but disable it for hw init */ |
intel_disable_gt_powersave(dev); |
|
/* Turn off power gate, require especially for the BIOS less system */ |
if (IS_VALLEYVIEW(dev)) { |
|
mutex_lock(&dev_priv->rps.hw_lock); |
reg_val = vlv_punit_read(dev_priv, PUNIT_REG_PWRGT_STATUS); |
|
if (reg_val & (RENDER_PWRGT | MEDIA_PWRGT | DISP2D_PWRGT)) |
vlv_punit_write(dev_priv, PUNIT_REG_PWRGT_CTRL, 0x0); |
|
mutex_unlock(&dev_priv->rps.hw_lock); |
|
} |
} |
|
/* |
* Generally this is called implicitly by the register read function. However, |
290,13 → 368,22 |
* be called at the beginning of the sequence followed by a call to |
* gen6_gt_force_wake_put() at the end of the sequence. |
*/ |
void gen6_gt_force_wake_get(struct drm_i915_private *dev_priv) |
void gen6_gt_force_wake_get(struct drm_i915_private *dev_priv, int fw_engine) |
{ |
unsigned long irqflags; |
|
if (!dev_priv->uncore.funcs.force_wake_get) |
return; |
|
intel_runtime_pm_get(dev_priv); |
|
/* Redirect to VLV specific routine */ |
if (IS_VALLEYVIEW(dev_priv->dev)) |
return vlv_force_wake_get(dev_priv, fw_engine); |
|
spin_lock_irqsave(&dev_priv->uncore.lock, irqflags); |
if (dev_priv->uncore.forcewake_count++ == 0) |
dev_priv->uncore.funcs.force_wake_get(dev_priv); |
dev_priv->uncore.funcs.force_wake_get(dev_priv, FORCEWAKE_ALL); |
spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags); |
} |
|
303,21 → 390,33 |
/* |
* see gen6_gt_force_wake_get() |
*/ |
void gen6_gt_force_wake_put(struct drm_i915_private *dev_priv) |
void gen6_gt_force_wake_put(struct drm_i915_private *dev_priv, int fw_engine) |
{ |
unsigned long irqflags; |
|
if (!dev_priv->uncore.funcs.force_wake_put) |
return; |
|
/* Redirect to VLV specific routine */ |
if (IS_VALLEYVIEW(dev_priv->dev)) |
return vlv_force_wake_put(dev_priv, fw_engine); |
|
|
spin_lock_irqsave(&dev_priv->uncore.lock, irqflags); |
if (--dev_priv->uncore.forcewake_count == 0) |
dev_priv->uncore.funcs.force_wake_put(dev_priv); |
if (--dev_priv->uncore.forcewake_count == 0) { |
dev_priv->uncore.forcewake_count++; |
mod_delayed_work(dev_priv->wq, |
&dev_priv->uncore.force_wake_work, |
1); |
} |
spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags); |
|
intel_runtime_pm_put(dev_priv); |
} |
|
/* We give fast paths for the really cool registers */ |
#define NEEDS_FORCE_WAKE(dev_priv, reg) \ |
((HAS_FORCE_WAKE((dev_priv)->dev)) && \ |
((reg) < 0x40000) && \ |
((reg) != FORCEWAKE)) |
((reg) < 0x40000 && (reg) != FORCEWAKE) |
|
static void |
ilk_dummy_write(struct drm_i915_private *dev_priv) |
331,8 → 430,7 |
static void |
hsw_unclaimed_reg_clear(struct drm_i915_private *dev_priv, u32 reg) |
{ |
if (HAS_FPGA_DBG_UNCLAIMED(dev_priv->dev) && |
(__raw_i915_read32(dev_priv, FPGA_DBG) & FPGA_DBG_RM_NOCLAIM)) { |
if (__raw_i915_read32(dev_priv, FPGA_DBG) & FPGA_DBG_RM_NOCLAIM) { |
DRM_ERROR("Unknown unclaimed register before writing to %x\n", |
reg); |
__raw_i915_write32(dev_priv, FPGA_DBG, FPGA_DBG_RM_NOCLAIM); |
342,51 → 440,167 |
static void |
hsw_unclaimed_reg_check(struct drm_i915_private *dev_priv, u32 reg) |
{ |
if (HAS_FPGA_DBG_UNCLAIMED(dev_priv->dev) && |
(__raw_i915_read32(dev_priv, FPGA_DBG) & FPGA_DBG_RM_NOCLAIM)) { |
if (__raw_i915_read32(dev_priv, FPGA_DBG) & FPGA_DBG_RM_NOCLAIM) { |
DRM_ERROR("Unclaimed write to %x\n", reg); |
__raw_i915_write32(dev_priv, FPGA_DBG, FPGA_DBG_RM_NOCLAIM); |
} |
} |
|
#define __i915_read(x) \ |
u##x i915_read##x(struct drm_i915_private *dev_priv, u32 reg, bool trace) { \ |
static void |
assert_device_not_suspended(struct drm_i915_private *dev_priv) |
{ |
WARN(HAS_RUNTIME_PM(dev_priv->dev) && dev_priv->pm.suspended, |
"Device suspended\n"); |
} |
|
#define REG_READ_HEADER(x) \ |
unsigned long irqflags; \ |
u##x val = 0; \ |
spin_lock_irqsave(&dev_priv->uncore.lock, irqflags); \ |
if (dev_priv->info->gen == 5) \ |
spin_lock_irqsave(&dev_priv->uncore.lock, irqflags) |
|
#define REG_READ_FOOTER \ |
spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags); \ |
trace_i915_reg_rw(false, reg, val, sizeof(val), trace); \ |
return val |
|
#define __gen4_read(x) \ |
static u##x \ |
gen4_read##x(struct drm_i915_private *dev_priv, off_t reg, bool trace) { \ |
REG_READ_HEADER(x); \ |
val = __raw_i915_read##x(dev_priv, reg); \ |
REG_READ_FOOTER; \ |
} |
|
#define __gen5_read(x) \ |
static u##x \ |
gen5_read##x(struct drm_i915_private *dev_priv, off_t reg, bool trace) { \ |
REG_READ_HEADER(x); \ |
ilk_dummy_write(dev_priv); \ |
val = __raw_i915_read##x(dev_priv, reg); \ |
REG_READ_FOOTER; \ |
} |
|
#define __gen6_read(x) \ |
static u##x \ |
gen6_read##x(struct drm_i915_private *dev_priv, off_t reg, bool trace) { \ |
REG_READ_HEADER(x); \ |
if (NEEDS_FORCE_WAKE((dev_priv), (reg))) { \ |
if (dev_priv->uncore.forcewake_count == 0) \ |
dev_priv->uncore.funcs.force_wake_get(dev_priv); \ |
dev_priv->uncore.funcs.force_wake_get(dev_priv, \ |
FORCEWAKE_ALL); \ |
val = __raw_i915_read##x(dev_priv, reg); \ |
if (dev_priv->uncore.forcewake_count == 0) \ |
dev_priv->uncore.funcs.force_wake_put(dev_priv); \ |
dev_priv->uncore.funcs.force_wake_put(dev_priv, \ |
FORCEWAKE_ALL); \ |
} else { \ |
val = __raw_i915_read##x(dev_priv, reg); \ |
} \ |
spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags); \ |
trace_i915_reg_rw(false, reg, val, sizeof(val), trace); \ |
return val; \ |
REG_READ_FOOTER; \ |
} |
|
__i915_read(8) |
__i915_read(16) |
__i915_read(32) |
__i915_read(64) |
#undef __i915_read |
#define __vlv_read(x) \ |
static u##x \ |
vlv_read##x(struct drm_i915_private *dev_priv, off_t reg, bool trace) { \ |
unsigned fwengine = 0; \ |
unsigned *fwcount; \ |
REG_READ_HEADER(x); \ |
if (FORCEWAKE_VLV_RENDER_RANGE_OFFSET(reg)) { \ |
fwengine = FORCEWAKE_RENDER; \ |
fwcount = &dev_priv->uncore.fw_rendercount; \ |
} \ |
else if (FORCEWAKE_VLV_MEDIA_RANGE_OFFSET(reg)) { \ |
fwengine = FORCEWAKE_MEDIA; \ |
fwcount = &dev_priv->uncore.fw_mediacount; \ |
} \ |
if (fwengine != 0) { \ |
if ((*fwcount)++ == 0) \ |
(dev_priv)->uncore.funcs.force_wake_get(dev_priv, \ |
fwengine); \ |
val = __raw_i915_read##x(dev_priv, reg); \ |
if (--(*fwcount) == 0) \ |
(dev_priv)->uncore.funcs.force_wake_put(dev_priv, \ |
fwengine); \ |
} else { \ |
val = __raw_i915_read##x(dev_priv, reg); \ |
} \ |
REG_READ_FOOTER; \ |
} |
|
#define __i915_write(x) \ |
void i915_write##x(struct drm_i915_private *dev_priv, u32 reg, u##x val, bool trace) { \ |
|
__vlv_read(8) |
__vlv_read(16) |
__vlv_read(32) |
__vlv_read(64) |
__gen6_read(8) |
__gen6_read(16) |
__gen6_read(32) |
__gen6_read(64) |
__gen5_read(8) |
__gen5_read(16) |
__gen5_read(32) |
__gen5_read(64) |
__gen4_read(8) |
__gen4_read(16) |
__gen4_read(32) |
__gen4_read(64) |
|
#undef __vlv_read |
#undef __gen6_read |
#undef __gen5_read |
#undef __gen4_read |
#undef REG_READ_FOOTER |
#undef REG_READ_HEADER |
|
#define REG_WRITE_HEADER \ |
unsigned long irqflags; \ |
trace_i915_reg_rw(true, reg, val, sizeof(val), trace); \ |
spin_lock_irqsave(&dev_priv->uncore.lock, irqflags) |
|
#define REG_WRITE_FOOTER \ |
spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags) |
|
#define __gen4_write(x) \ |
static void \ |
gen4_write##x(struct drm_i915_private *dev_priv, off_t reg, u##x val, bool trace) { \ |
REG_WRITE_HEADER; \ |
__raw_i915_write##x(dev_priv, reg, val); \ |
REG_WRITE_FOOTER; \ |
} |
|
#define __gen5_write(x) \ |
static void \ |
gen5_write##x(struct drm_i915_private *dev_priv, off_t reg, u##x val, bool trace) { \ |
REG_WRITE_HEADER; \ |
ilk_dummy_write(dev_priv); \ |
__raw_i915_write##x(dev_priv, reg, val); \ |
REG_WRITE_FOOTER; \ |
} |
|
#define __gen6_write(x) \ |
static void \ |
gen6_write##x(struct drm_i915_private *dev_priv, off_t reg, u##x val, bool trace) { \ |
u32 __fifo_ret = 0; \ |
trace_i915_reg_rw(true, reg, val, sizeof(val), trace); \ |
spin_lock_irqsave(&dev_priv->uncore.lock, irqflags); \ |
REG_WRITE_HEADER; \ |
if (NEEDS_FORCE_WAKE((dev_priv), (reg))) { \ |
__fifo_ret = __gen6_gt_wait_for_fifo(dev_priv); \ |
} \ |
if (dev_priv->info->gen == 5) \ |
ilk_dummy_write(dev_priv); \ |
assert_device_not_suspended(dev_priv); \ |
__raw_i915_write##x(dev_priv, reg, val); \ |
if (unlikely(__fifo_ret)) { \ |
gen6_gt_check_fifodbg(dev_priv); \ |
} \ |
REG_WRITE_FOOTER; \ |
} |
|
#define __hsw_write(x) \ |
static void \ |
hsw_write##x(struct drm_i915_private *dev_priv, off_t reg, u##x val, bool trace) { \ |
u32 __fifo_ret = 0; \ |
REG_WRITE_HEADER; \ |
if (NEEDS_FORCE_WAKE((dev_priv), (reg))) { \ |
__fifo_ret = __gen6_gt_wait_for_fifo(dev_priv); \ |
} \ |
assert_device_not_suspended(dev_priv); \ |
hsw_unclaimed_reg_clear(dev_priv, reg); \ |
__raw_i915_write##x(dev_priv, reg, val); \ |
if (unlikely(__fifo_ret)) { \ |
393,20 → 607,205 |
gen6_gt_check_fifodbg(dev_priv); \ |
} \ |
hsw_unclaimed_reg_check(dev_priv, reg); \ |
spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags); \ |
REG_WRITE_FOOTER; \ |
} |
__i915_write(8) |
__i915_write(16) |
__i915_write(32) |
__i915_write(64) |
#undef __i915_write |
|
static const u32 gen8_shadowed_regs[] = { |
FORCEWAKE_MT, |
GEN6_RPNSWREQ, |
GEN6_RC_VIDEO_FREQ, |
RING_TAIL(RENDER_RING_BASE), |
RING_TAIL(GEN6_BSD_RING_BASE), |
RING_TAIL(VEBOX_RING_BASE), |
RING_TAIL(BLT_RING_BASE), |
/* TODO: Other registers are not yet used */ |
}; |
|
static bool is_gen8_shadowed(struct drm_i915_private *dev_priv, u32 reg) |
{ |
int i; |
for (i = 0; i < ARRAY_SIZE(gen8_shadowed_regs); i++) |
if (reg == gen8_shadowed_regs[i]) |
return true; |
|
return false; |
} |
|
#define __gen8_write(x) \ |
static void \ |
gen8_write##x(struct drm_i915_private *dev_priv, off_t reg, u##x val, bool trace) { \ |
bool __needs_put = reg < 0x40000 && !is_gen8_shadowed(dev_priv, reg); \ |
REG_WRITE_HEADER; \ |
if (__needs_put) { \ |
dev_priv->uncore.funcs.force_wake_get(dev_priv, \ |
FORCEWAKE_ALL); \ |
} \ |
__raw_i915_write##x(dev_priv, reg, val); \ |
if (__needs_put) { \ |
dev_priv->uncore.funcs.force_wake_put(dev_priv, \ |
FORCEWAKE_ALL); \ |
} \ |
REG_WRITE_FOOTER; \ |
} |
|
__gen8_write(8) |
__gen8_write(16) |
__gen8_write(32) |
__gen8_write(64) |
__hsw_write(8) |
__hsw_write(16) |
__hsw_write(32) |
__hsw_write(64) |
__gen6_write(8) |
__gen6_write(16) |
__gen6_write(32) |
__gen6_write(64) |
__gen5_write(8) |
__gen5_write(16) |
__gen5_write(32) |
__gen5_write(64) |
__gen4_write(8) |
__gen4_write(16) |
__gen4_write(32) |
__gen4_write(64) |
|
#undef __gen8_write |
#undef __hsw_write |
#undef __gen6_write |
#undef __gen5_write |
#undef __gen4_write |
#undef REG_WRITE_FOOTER |
#undef REG_WRITE_HEADER |
|
void intel_uncore_init(struct drm_device *dev) |
{ |
struct drm_i915_private *dev_priv = dev->dev_private; |
|
INIT_DELAYED_WORK(&dev_priv->uncore.force_wake_work, |
gen6_force_wake_work); |
|
if (IS_VALLEYVIEW(dev)) { |
dev_priv->uncore.funcs.force_wake_get = __vlv_force_wake_get; |
dev_priv->uncore.funcs.force_wake_put = __vlv_force_wake_put; |
} else if (IS_HASWELL(dev) || IS_GEN8(dev)) { |
dev_priv->uncore.funcs.force_wake_get = __gen6_gt_force_wake_mt_get; |
dev_priv->uncore.funcs.force_wake_put = __gen6_gt_force_wake_mt_put; |
} else if (IS_IVYBRIDGE(dev)) { |
u32 ecobus; |
|
/* IVB configs may use multi-threaded forcewake */ |
|
/* A small trick here - if the bios hasn't configured |
* MT forcewake, and if the device is in RC6, then |
* force_wake_mt_get will not wake the device and the |
* ECOBUS read will return zero. Which will be |
* (correctly) interpreted by the test below as MT |
* forcewake being disabled. |
*/ |
mutex_lock(&dev->struct_mutex); |
__gen6_gt_force_wake_mt_get(dev_priv, FORCEWAKE_ALL); |
ecobus = __raw_i915_read32(dev_priv, ECOBUS); |
__gen6_gt_force_wake_mt_put(dev_priv, FORCEWAKE_ALL); |
mutex_unlock(&dev->struct_mutex); |
|
if (ecobus & FORCEWAKE_MT_ENABLE) { |
dev_priv->uncore.funcs.force_wake_get = |
__gen6_gt_force_wake_mt_get; |
dev_priv->uncore.funcs.force_wake_put = |
__gen6_gt_force_wake_mt_put; |
} else { |
DRM_INFO("No MT forcewake available on Ivybridge, this can result in issues\n"); |
DRM_INFO("when using vblank-synced partial screen updates.\n"); |
dev_priv->uncore.funcs.force_wake_get = |
__gen6_gt_force_wake_get; |
dev_priv->uncore.funcs.force_wake_put = |
__gen6_gt_force_wake_put; |
} |
} else if (IS_GEN6(dev)) { |
dev_priv->uncore.funcs.force_wake_get = |
__gen6_gt_force_wake_get; |
dev_priv->uncore.funcs.force_wake_put = |
__gen6_gt_force_wake_put; |
} |
|
switch (INTEL_INFO(dev)->gen) { |
default: |
dev_priv->uncore.funcs.mmio_writeb = gen8_write8; |
dev_priv->uncore.funcs.mmio_writew = gen8_write16; |
dev_priv->uncore.funcs.mmio_writel = gen8_write32; |
dev_priv->uncore.funcs.mmio_writeq = gen8_write64; |
dev_priv->uncore.funcs.mmio_readb = gen6_read8; |
dev_priv->uncore.funcs.mmio_readw = gen6_read16; |
dev_priv->uncore.funcs.mmio_readl = gen6_read32; |
dev_priv->uncore.funcs.mmio_readq = gen6_read64; |
break; |
case 7: |
case 6: |
if (IS_HASWELL(dev)) { |
dev_priv->uncore.funcs.mmio_writeb = hsw_write8; |
dev_priv->uncore.funcs.mmio_writew = hsw_write16; |
dev_priv->uncore.funcs.mmio_writel = hsw_write32; |
dev_priv->uncore.funcs.mmio_writeq = hsw_write64; |
} else { |
dev_priv->uncore.funcs.mmio_writeb = gen6_write8; |
dev_priv->uncore.funcs.mmio_writew = gen6_write16; |
dev_priv->uncore.funcs.mmio_writel = gen6_write32; |
dev_priv->uncore.funcs.mmio_writeq = gen6_write64; |
} |
|
if (IS_VALLEYVIEW(dev)) { |
dev_priv->uncore.funcs.mmio_readb = vlv_read8; |
dev_priv->uncore.funcs.mmio_readw = vlv_read16; |
dev_priv->uncore.funcs.mmio_readl = vlv_read32; |
dev_priv->uncore.funcs.mmio_readq = vlv_read64; |
} else { |
dev_priv->uncore.funcs.mmio_readb = gen6_read8; |
dev_priv->uncore.funcs.mmio_readw = gen6_read16; |
dev_priv->uncore.funcs.mmio_readl = gen6_read32; |
dev_priv->uncore.funcs.mmio_readq = gen6_read64; |
} |
break; |
case 5: |
dev_priv->uncore.funcs.mmio_writeb = gen5_write8; |
dev_priv->uncore.funcs.mmio_writew = gen5_write16; |
dev_priv->uncore.funcs.mmio_writel = gen5_write32; |
dev_priv->uncore.funcs.mmio_writeq = gen5_write64; |
dev_priv->uncore.funcs.mmio_readb = gen5_read8; |
dev_priv->uncore.funcs.mmio_readw = gen5_read16; |
dev_priv->uncore.funcs.mmio_readl = gen5_read32; |
dev_priv->uncore.funcs.mmio_readq = gen5_read64; |
break; |
case 4: |
case 3: |
case 2: |
dev_priv->uncore.funcs.mmio_writeb = gen4_write8; |
dev_priv->uncore.funcs.mmio_writew = gen4_write16; |
dev_priv->uncore.funcs.mmio_writel = gen4_write32; |
dev_priv->uncore.funcs.mmio_writeq = gen4_write64; |
dev_priv->uncore.funcs.mmio_readb = gen4_read8; |
dev_priv->uncore.funcs.mmio_readw = gen4_read16; |
dev_priv->uncore.funcs.mmio_readl = gen4_read32; |
dev_priv->uncore.funcs.mmio_readq = gen4_read64; |
break; |
} |
} |
|
void intel_uncore_fini(struct drm_device *dev) |
{ |
struct drm_i915_private *dev_priv = dev->dev_private; |
|
// flush_delayed_work(&dev_priv->uncore.force_wake_work); |
|
/* Paranoia: make sure we have disabled everything before we exit. */ |
intel_uncore_sanitize(dev); |
} |
|
static const struct register_whitelist { |
uint64_t offset; |
uint32_t size; |
uint32_t gen_bitmask; /* support gens, 0x10 for 4, 0x30 for 4 and 5, etc. */ |
} whitelist[] = { |
{ RING_TIMESTAMP(RENDER_RING_BASE), 8, 0xF0 }, |
{ RING_TIMESTAMP(RENDER_RING_BASE), 8, 0x1F0 }, |
}; |
|
int i915_reg_read_ioctl(struct drm_device *dev, |
447,33 → 846,37 |
return 0; |
} |
|
static int i8xx_do_reset(struct drm_device *dev) |
int i915_get_reset_stats_ioctl(struct drm_device *dev, |
void *data, struct drm_file *file) |
{ |
struct drm_i915_private *dev_priv = dev->dev_private; |
struct drm_i915_reset_stats *args = data; |
struct i915_ctx_hang_stats *hs; |
int ret; |
|
if (IS_I85X(dev)) |
return -ENODEV; |
if (args->flags || args->pad) |
return -EINVAL; |
|
I915_WRITE(D_STATE, I915_READ(D_STATE) | DSTATE_GFX_RESET_I830); |
POSTING_READ(D_STATE); |
if (args->ctx_id == DEFAULT_CONTEXT_ID) |
return -EPERM; |
|
if (IS_I830(dev) || IS_845G(dev)) { |
I915_WRITE(DEBUG_RESET_I830, |
DEBUG_RESET_DISPLAY | |
DEBUG_RESET_RENDER | |
DEBUG_RESET_FULL); |
POSTING_READ(DEBUG_RESET_I830); |
msleep(1); |
ret = mutex_lock_interruptible(&dev->struct_mutex); |
if (ret) |
return ret; |
|
I915_WRITE(DEBUG_RESET_I830, 0); |
POSTING_READ(DEBUG_RESET_I830); |
hs = i915_gem_context_get_hang_stats(dev, file, args->ctx_id); |
if (IS_ERR(hs)) { |
mutex_unlock(&dev->struct_mutex); |
return PTR_ERR(hs); |
} |
|
msleep(1); |
args->reset_count = i915_reset_count(&dev_priv->gpu_error); |
|
I915_WRITE(D_STATE, I915_READ(D_STATE) & ~DSTATE_GFX_RESET_I830); |
POSTING_READ(D_STATE); |
args->batch_active = hs->batch_active; |
args->batch_pending = hs->batch_pending; |
|
mutex_unlock(&dev->struct_mutex); |
|
return 0; |
} |
|
560,12 → 963,12 |
|
/* If reset with a user forcewake, try to restore, otherwise turn it off */ |
if (dev_priv->uncore.forcewake_count) |
dev_priv->uncore.funcs.force_wake_get(dev_priv); |
dev_priv->uncore.funcs.force_wake_get(dev_priv, FORCEWAKE_ALL); |
else |
dev_priv->uncore.funcs.force_wake_put(dev_priv); |
dev_priv->uncore.funcs.force_wake_put(dev_priv, FORCEWAKE_ALL); |
|
/* Restore fifo count */ |
dev_priv->uncore.fifo_count = __raw_i915_read32(dev_priv, GT_FIFO_FREE_ENTRIES); |
dev_priv->uncore.fifo_count = __raw_i915_read32(dev_priv, GTFIFOCTL) & GT_FIFO_FREE_ENTRIES_MASK; |
|
spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags); |
return ret; |
574,24 → 977,15 |
int intel_gpu_reset(struct drm_device *dev) |
{ |
switch (INTEL_INFO(dev)->gen) { |
case 8: |
case 7: |
case 6: return gen6_do_reset(dev); |
case 5: return ironlake_do_reset(dev); |
case 4: return i965_do_reset(dev); |
case 2: return i8xx_do_reset(dev); |
default: return -ENODEV; |
} |
} |
|
void intel_uncore_clear_errors(struct drm_device *dev) |
{ |
struct drm_i915_private *dev_priv = dev->dev_private; |
|
/* XXX needs spinlock around caller's grouping */ |
if (HAS_FPGA_DBG_UNCLAIMED(dev)) |
__raw_i915_write32(dev_priv, FPGA_DBG, FPGA_DBG_RM_NOCLAIM); |
} |
|
void intel_uncore_check_errors(struct drm_device *dev) |
{ |
struct drm_i915_private *dev_priv = dev->dev_private; |