Subversion Repositories Kolibri OS

Compare Revisions

Regard whitespace Rev 5077 → Rev 5078

/drivers/video/drm/radeon/evergreen.c
33,10 → 33,8
#include "avivod.h"
#include "evergreen_reg.h"
#include "evergreen_blit_shaders.h"
#include "radeon_ucode.h"
 
#define EVERGREEN_PFP_UCODE_SIZE 1120
#define EVERGREEN_PM4_UCODE_SIZE 1376
 
static const u32 crtc_offsets[6] =
{
EVERGREEN_CRTC0_REGISTER_OFFSET,
47,12 → 45,109
EVERGREEN_CRTC5_REGISTER_OFFSET
};
 
#include "clearstate_evergreen.h"
 
static const u32 sumo_rlc_save_restore_register_list[] =
{
0x98fc,
0x9830,
0x9834,
0x9838,
0x9870,
0x9874,
0x8a14,
0x8b24,
0x8bcc,
0x8b10,
0x8d00,
0x8d04,
0x8c00,
0x8c04,
0x8c08,
0x8c0c,
0x8d8c,
0x8c20,
0x8c24,
0x8c28,
0x8c18,
0x8c1c,
0x8cf0,
0x8e2c,
0x8e38,
0x8c30,
0x9508,
0x9688,
0x9608,
0x960c,
0x9610,
0x9614,
0x88c4,
0x88d4,
0xa008,
0x900c,
0x9100,
0x913c,
0x98f8,
0x98f4,
0x9b7c,
0x3f8c,
0x8950,
0x8954,
0x8a18,
0x8b28,
0x9144,
0x9148,
0x914c,
0x3f90,
0x3f94,
0x915c,
0x9160,
0x9178,
0x917c,
0x9180,
0x918c,
0x9190,
0x9194,
0x9198,
0x919c,
0x91a8,
0x91ac,
0x91b0,
0x91b4,
0x91b8,
0x91c4,
0x91c8,
0x91cc,
0x91d0,
0x91d4,
0x91e0,
0x91e4,
0x91ec,
0x91f0,
0x91f4,
0x9200,
0x9204,
0x929c,
0x9150,
0x802c,
};
 
static void evergreen_gpu_init(struct radeon_device *rdev);
void evergreen_fini(struct radeon_device *rdev);
void evergreen_pcie_gen2_enable(struct radeon_device *rdev);
void evergreen_program_aspm(struct radeon_device *rdev);
extern void cayman_cp_int_cntl_setup(struct radeon_device *rdev,
int ring, u32 cp_int_cntl);
extern void cayman_vm_decode_fault(struct radeon_device *rdev,
u32 status, u32 addr);
void cik_init_cp_pg_table(struct radeon_device *rdev);
 
extern u32 si_get_csb_size(struct radeon_device *rdev);
extern void si_get_csb_buffer(struct radeon_device *rdev, volatile u32 *buffer);
extern u32 cik_get_csb_size(struct radeon_device *rdev);
extern void cik_get_csb_buffer(struct radeon_device *rdev, volatile u32 *buffer);
extern void rv770_set_clk_bypass_mode(struct radeon_device *rdev);
 
static const u32 evergreen_golden_registers[] =
{
0x3f90, 0xffff0000, 0xff000000,
94,7 → 189,7
0x8c1c, 0xffffffff, 0x00001010,
0x28350, 0xffffffff, 0x00000000,
0xa008, 0xffffffff, 0x00010000,
0x5cc, 0xffffffff, 0x00000001,
0x5c4, 0xffffffff, 0x00000001,
0x9508, 0xffffffff, 0x00000002,
0x913c, 0x0000000f, 0x0000000a
};
381,7 → 476,7
0x8c1c, 0xffffffff, 0x00001010,
0x28350, 0xffffffff, 0x00000000,
0xa008, 0xffffffff, 0x00010000,
0x5cc, 0xffffffff, 0x00000001,
0x5c4, 0xffffffff, 0x00000001,
0x9508, 0xffffffff, 0x00000002
};
 
540,7 → 635,7
static const u32 supersumo_golden_registers[] =
{
0x5eb4, 0xffffffff, 0x00000002,
0x5cc, 0xffffffff, 0x00000001,
0x5c4, 0xffffffff, 0x00000001,
0x7030, 0xffffffff, 0x00000011,
0x7c30, 0xffffffff, 0x00000011,
0x6104, 0x01000300, 0x00000000,
624,7 → 719,7
static const u32 wrestler_golden_registers[] =
{
0x5eb4, 0xffffffff, 0x00000002,
0x5cc, 0xffffffff, 0x00000001,
0x5c4, 0xffffffff, 0x00000001,
0x7030, 0xffffffff, 0x00000011,
0x7c30, 0xffffffff, 0x00000011,
0x6104, 0x01000300, 0x00000000,
1080,25 → 1175,74
 
void evergreen_fix_pci_max_read_req_size(struct radeon_device *rdev)
{
u16 ctl, v;
int err;
int readrq;
u16 v;
 
err = pcie_capability_read_word(rdev->pdev, PCI_EXP_DEVCTL, &ctl);
if (err)
return;
 
v = (ctl & PCI_EXP_DEVCTL_READRQ) >> 12;
 
readrq = pcie_get_readrq(rdev->pdev);
v = ffs(readrq) - 8;
/* if bios or OS sets MAX_READ_REQUEST_SIZE to an invalid value, fix it
* to avoid hangs or perfomance issues
*/
if ((v == 0) || (v == 6) || (v == 7)) {
ctl &= ~PCI_EXP_DEVCTL_READRQ;
ctl |= (2 << 12);
pcie_capability_write_word(rdev->pdev, PCI_EXP_DEVCTL, ctl);
if ((v == 0) || (v == 6) || (v == 7))
pcie_set_readrq(rdev->pdev, 512);
}
 
void dce4_program_fmt(struct drm_encoder *encoder)
{
struct drm_device *dev = encoder->dev;
struct radeon_device *rdev = dev->dev_private;
struct radeon_encoder *radeon_encoder = to_radeon_encoder(encoder);
struct radeon_crtc *radeon_crtc = to_radeon_crtc(encoder->crtc);
struct drm_connector *connector = radeon_get_connector_for_encoder(encoder);
int bpc = 0;
u32 tmp = 0;
enum radeon_connector_dither dither = RADEON_FMT_DITHER_DISABLE;
 
if (connector) {
struct radeon_connector *radeon_connector = to_radeon_connector(connector);
bpc = radeon_get_monitor_bpc(connector);
dither = radeon_connector->dither;
}
 
/* LVDS/eDP FMT is set up by atom */
if (radeon_encoder->devices & ATOM_DEVICE_LCD_SUPPORT)
return;
 
/* not needed for analog */
if ((radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC1) ||
(radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC2))
return;
 
if (bpc == 0)
return;
 
switch (bpc) {
case 6:
if (dither == RADEON_FMT_DITHER_ENABLE)
/* XXX sort out optimal dither settings */
tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
FMT_SPATIAL_DITHER_EN);
else
tmp |= FMT_TRUNCATE_EN;
break;
case 8:
if (dither == RADEON_FMT_DITHER_ENABLE)
/* XXX sort out optimal dither settings */
tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
FMT_RGB_RANDOM_ENABLE |
FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH);
else
tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH);
break;
case 10:
default:
/* not needed */
break;
}
 
WREG32(FMT_BIT_DEPTH_CONTROL + radeon_crtc->crtc_offset, tmp);
}
 
static bool dce4_is_in_vblank(struct radeon_device *rdev, int crtc)
{
if (RREG32(EVERGREEN_CRTC_STATUS + crtc_offsets[crtc]) & EVERGREEN_CRTC_V_BLANK)
1156,7 → 1300,6
}
}
 
 
/**
* evergreen_page_flip - pageflip callback.
*
1170,7 → 1313,7
* double buffered update to take place.
* Returns the current update pending status.
*/
u32 evergreen_page_flip(struct radeon_device *rdev, int crtc_id, u64 crtc_base)
void evergreen_page_flip(struct radeon_device *rdev, int crtc_id, u64 crtc_base)
{
struct radeon_crtc *radeon_crtc = rdev->mode_info.crtcs[crtc_id];
u32 tmp = RREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset);
1202,9 → 1345,23
/* Unlock the lock, so double-buffering can take place inside vblank */
tmp &= ~EVERGREEN_GRPH_UPDATE_LOCK;
WREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset, tmp);
}
 
/**
* evergreen_page_flip_pending - check if page flip is still pending
*
* @rdev: radeon_device pointer
* @crtc_id: crtc to check
*
* Returns the current update pending status.
*/
bool evergreen_page_flip_pending(struct radeon_device *rdev, int crtc_id)
{
struct radeon_crtc *radeon_crtc = rdev->mode_info.crtcs[crtc_id];
 
/* Return current update_pending status: */
return RREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset) & EVERGREEN_GRPH_SURFACE_UPDATE_PENDING;
return !!(RREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset) &
EVERGREEN_GRPH_SURFACE_UPDATE_PENDING);
}
 
/* get temperature in millidegrees */
1388,8 → 1545,8
struct radeon_voltage *voltage = &ps->clock_info[req_cm_idx].voltage;
 
if (voltage->type == VOLTAGE_SW) {
/* 0xff01 is a flag rather then an actual voltage */
if (voltage->voltage == 0xff01)
/* 0xff0x are flags rather then an actual voltage */
if ((voltage->voltage & 0xff00) == 0xff00)
return;
if (voltage->voltage && (voltage->voltage != rdev->pm.current_vddc)) {
radeon_atom_set_voltage(rdev, voltage->voltage, SET_VOLTAGE_TYPE_ASIC_VDDC);
1409,8 → 1566,8
voltage = &rdev->pm.power_state[req_ps_idx].
clock_info[rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx].voltage;
 
/* 0xff01 is a flag rather then an actual voltage */
if (voltage->vddci == 0xff01)
/* 0xff0x are flags rather then an actual voltage */
if ((voltage->vddci & 0xff00) == 0xff00)
return;
if (voltage->vddci && (voltage->vddci != rdev->pm.current_vddci)) {
radeon_atom_set_voltage(rdev, voltage->vddci, SET_VOLTAGE_TYPE_ASIC_VDDCI);
1689,7 → 1846,8
struct drm_display_mode *mode,
struct drm_display_mode *other_mode)
{
u32 tmp;
u32 tmp, buffer_alloc, i;
u32 pipe_offset = radeon_crtc->crtc_id * 0x20;
/*
* Line Buffer Setup
* There are 3 line buffers, each one shared by 2 display controllers.
1712,12 → 1870,17
* non-linked crtcs for maximum line buffer allocation.
*/
if (radeon_crtc->base.enabled && mode) {
if (other_mode)
if (other_mode) {
tmp = 0; /* 1/2 */
else
buffer_alloc = 1;
} else {
tmp = 2; /* whole */
} else
buffer_alloc = 2;
}
} else {
tmp = 0;
buffer_alloc = 0;
}
 
/* second controller of the pair uses second half of the lb */
if (radeon_crtc->crtc_id % 2)
1724,6 → 1887,17
tmp += 4;
WREG32(DC_LB_MEMORY_SPLIT + radeon_crtc->crtc_offset, tmp);
 
if (ASIC_IS_DCE41(rdev) || ASIC_IS_DCE5(rdev)) {
WREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset,
DMIF_BUFFERS_ALLOCATED(buffer_alloc));
for (i = 0; i < rdev->usec_timeout; i++) {
if (RREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset) &
DMIF_BUFFERS_ALLOCATED_COMPLETED)
break;
udelay(1);
}
}
 
if (radeon_crtc->base.enabled && mode) {
switch (tmp) {
case 0:
2007,7 → 2181,8
u32 lb_size, u32 num_heads)
{
struct drm_display_mode *mode = &radeon_crtc->base.mode;
struct evergreen_wm_params wm;
struct evergreen_wm_params wm_low, wm_high;
u32 dram_channels;
u32 pixel_period;
u32 line_time = 0;
u32 latency_watermark_a = 0, latency_watermark_b = 0;
2023,39 → 2198,81
line_time = min((u32)mode->crtc_htotal * pixel_period, (u32)65535);
priority_a_cnt = 0;
priority_b_cnt = 0;
dram_channels = evergreen_get_number_of_dram_channels(rdev);
 
wm.yclk = rdev->pm.current_mclk * 10;
wm.sclk = rdev->pm.current_sclk * 10;
wm.disp_clk = mode->clock;
wm.src_width = mode->crtc_hdisplay;
wm.active_time = mode->crtc_hdisplay * pixel_period;
wm.blank_time = line_time - wm.active_time;
wm.interlaced = false;
/* watermark for high clocks */
if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) {
wm_high.yclk =
radeon_dpm_get_mclk(rdev, false) * 10;
wm_high.sclk =
radeon_dpm_get_sclk(rdev, false) * 10;
} else {
wm_high.yclk = rdev->pm.current_mclk * 10;
wm_high.sclk = rdev->pm.current_sclk * 10;
}
 
wm_high.disp_clk = mode->clock;
wm_high.src_width = mode->crtc_hdisplay;
wm_high.active_time = mode->crtc_hdisplay * pixel_period;
wm_high.blank_time = line_time - wm_high.active_time;
wm_high.interlaced = false;
if (mode->flags & DRM_MODE_FLAG_INTERLACE)
wm.interlaced = true;
wm.vsc = radeon_crtc->vsc;
wm.vtaps = 1;
wm_high.interlaced = true;
wm_high.vsc = radeon_crtc->vsc;
wm_high.vtaps = 1;
if (radeon_crtc->rmx_type != RMX_OFF)
wm.vtaps = 2;
wm.bytes_per_pixel = 4; /* XXX: get this from fb config */
wm.lb_size = lb_size;
wm.dram_channels = evergreen_get_number_of_dram_channels(rdev);
wm.num_heads = num_heads;
wm_high.vtaps = 2;
wm_high.bytes_per_pixel = 4; /* XXX: get this from fb config */
wm_high.lb_size = lb_size;
wm_high.dram_channels = dram_channels;
wm_high.num_heads = num_heads;
 
/* watermark for low clocks */
if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) {
wm_low.yclk =
radeon_dpm_get_mclk(rdev, true) * 10;
wm_low.sclk =
radeon_dpm_get_sclk(rdev, true) * 10;
} else {
wm_low.yclk = rdev->pm.current_mclk * 10;
wm_low.sclk = rdev->pm.current_sclk * 10;
}
 
wm_low.disp_clk = mode->clock;
wm_low.src_width = mode->crtc_hdisplay;
wm_low.active_time = mode->crtc_hdisplay * pixel_period;
wm_low.blank_time = line_time - wm_low.active_time;
wm_low.interlaced = false;
if (mode->flags & DRM_MODE_FLAG_INTERLACE)
wm_low.interlaced = true;
wm_low.vsc = radeon_crtc->vsc;
wm_low.vtaps = 1;
if (radeon_crtc->rmx_type != RMX_OFF)
wm_low.vtaps = 2;
wm_low.bytes_per_pixel = 4; /* XXX: get this from fb config */
wm_low.lb_size = lb_size;
wm_low.dram_channels = dram_channels;
wm_low.num_heads = num_heads;
 
/* set for high clocks */
latency_watermark_a = min(evergreen_latency_watermark(&wm), (u32)65535);
latency_watermark_a = min(evergreen_latency_watermark(&wm_high), (u32)65535);
/* set for low clocks */
/* wm.yclk = low clk; wm.sclk = low clk */
latency_watermark_b = min(evergreen_latency_watermark(&wm), (u32)65535);
latency_watermark_b = min(evergreen_latency_watermark(&wm_low), (u32)65535);
 
/* possibly force display priority to high */
/* should really do this at mode validation time... */
if (!evergreen_average_bandwidth_vs_dram_bandwidth_for_display(&wm) ||
!evergreen_average_bandwidth_vs_available_bandwidth(&wm) ||
!evergreen_check_latency_hiding(&wm) ||
if (!evergreen_average_bandwidth_vs_dram_bandwidth_for_display(&wm_high) ||
!evergreen_average_bandwidth_vs_available_bandwidth(&wm_high) ||
!evergreen_check_latency_hiding(&wm_high) ||
(rdev->disp_priority == 2)) {
DRM_DEBUG_KMS("force priority to high\n");
DRM_DEBUG_KMS("force priority a to high\n");
priority_a_cnt |= PRIORITY_ALWAYS_ON;
}
if (!evergreen_average_bandwidth_vs_dram_bandwidth_for_display(&wm_low) ||
!evergreen_average_bandwidth_vs_available_bandwidth(&wm_low) ||
!evergreen_check_latency_hiding(&wm_low) ||
(rdev->disp_priority == 2)) {
DRM_DEBUG_KMS("force priority b to high\n");
priority_b_cnt |= PRIORITY_ALWAYS_ON;
}
 
2108,6 → 2325,10
WREG32(PRIORITY_A_CNT + radeon_crtc->crtc_offset, priority_a_cnt);
WREG32(PRIORITY_B_CNT + radeon_crtc->crtc_offset, priority_b_cnt);
 
/* save values for DPM */
radeon_crtc->line_time = line_time;
radeon_crtc->wm_high = latency_watermark_a;
radeon_crtc->wm_low = latency_watermark_b;
}
 
/**
2203,7 → 2424,6
r = radeon_gart_table_vram_pin(rdev);
if (r)
return r;
radeon_gart_restore(rdev);
/* Setup L2 cache */
WREG32(VM_L2_CNTL, ENABLE_L2_CACHE | ENABLE_L2_FRAGMENT_PROCESSING |
ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
2421,8 → 2641,9
for (i = 0; i < rdev->num_crtc; i++) {
if (save->crtc_enabled[i]) {
tmp = RREG32(EVERGREEN_MASTER_UPDATE_MODE + crtc_offsets[i]);
if ((tmp & 0x3) != 0) {
tmp &= ~0x3;
if ((tmp & 0x7) != 3) {
tmp &= ~0x7;
tmp |= 0x3;
WREG32(EVERGREEN_MASTER_UPDATE_MODE + crtc_offsets[i], tmp);
}
tmp = RREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i]);
2455,7 → 2676,7
if (save->crtc_enabled[i]) {
if (ASIC_IS_DCE6(rdev)) {
tmp = RREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i]);
tmp |= EVERGREEN_CRTC_BLANK_DATA_EN;
tmp &= ~EVERGREEN_CRTC_BLANK_DATA_EN;
WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
WREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i], tmp);
WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
2648,7 → 2869,7
radeon_ring_write(ring, PACKET3_ME_INITIALIZE_DEVICE_ID(1));
radeon_ring_write(ring, 0);
radeon_ring_write(ring, 0);
radeon_ring_unlock_commit(rdev, ring);
radeon_ring_unlock_commit(rdev, ring, false);
 
cp_me = 0xff;
WREG32(CP_ME_CNTL, cp_me);
2691,7 → 2912,7
radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
radeon_ring_write(ring, 0x00000010); /* */
 
radeon_ring_unlock_commit(rdev, ring);
radeon_ring_unlock_commit(rdev, ring, false);
 
return 0;
}
2716,8 → 2937,8
RREG32(GRBM_SOFT_RESET);
 
/* Set ring buffer size */
rb_bufsz = drm_order(ring->ring_size / 8);
tmp = (drm_order(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
rb_bufsz = order_base_2(ring->ring_size / 8);
tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
#ifdef __BIG_ENDIAN
tmp |= BUF_SWAP_32BIT;
#endif
2753,8 → 2974,6
WREG32(CP_RB_BASE, ring->gpu_addr >> 8);
WREG32(CP_DEBUG, (1 << 27) | (1 << 28));
 
ring->rptr = RREG32(CP_RB_RPTR);
 
evergreen_cp_start(rdev);
ring->ready = true;
r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, ring);
2944,7 → 3163,7
rdev->config.evergreen.sx_max_export_size = 256;
rdev->config.evergreen.sx_max_export_pos_size = 64;
rdev->config.evergreen.sx_max_export_smx_size = 192;
rdev->config.evergreen.max_hw_contexts = 8;
rdev->config.evergreen.max_hw_contexts = 4;
rdev->config.evergreen.sq_num_cf_insts = 2;
 
rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3091,10 → 3310,8
u32 efuse_straps_4;
u32 efuse_straps_3;
 
WREG32(RCU_IND_INDEX, 0x204);
efuse_straps_4 = RREG32(RCU_IND_DATA);
WREG32(RCU_IND_INDEX, 0x203);
efuse_straps_3 = RREG32(RCU_IND_DATA);
efuse_straps_4 = RREG32_RCU(0x204);
efuse_straps_3 = RREG32_RCU(0x203);
tmp = (((efuse_straps_4 & 0xf) << 4) |
((efuse_straps_3 & 0xf0000000) >> 28));
} else {
3120,6 → 3337,18
disabled_rb_mask &= ~(1 << i);
}
 
for (i = 0; i < rdev->config.evergreen.num_ses; i++) {
u32 simd_disable_bitmap;
 
WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
simd_disable_bitmap = (RREG32(CC_GC_SHADER_PIPE_CONFIG) & 0xffff0000) >> 16;
simd_disable_bitmap |= 0xffffffff << rdev->config.evergreen.max_simds;
tmp <<= 16;
tmp |= simd_disable_bitmap;
}
rdev->config.evergreen.active_simds = hweight32(~tmp);
 
WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_BROADCAST_WRITES);
WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_BROADCAST_WRITES);
 
3450,7 → 3679,7
return true;
}
 
static u32 evergreen_gpu_check_soft_reset(struct radeon_device *rdev)
u32 evergreen_gpu_check_soft_reset(struct radeon_device *rdev)
{
u32 reset_mask = 0;
u32 tmp;
3633,6 → 3862,48
evergreen_print_gpu_status_regs(rdev);
}
 
void evergreen_gpu_pci_config_reset(struct radeon_device *rdev)
{
struct evergreen_mc_save save;
u32 tmp, i;
 
dev_info(rdev->dev, "GPU pci config reset\n");
 
/* disable dpm? */
 
/* Disable CP parsing/prefetching */
WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT);
udelay(50);
/* Disable DMA */
tmp = RREG32(DMA_RB_CNTL);
tmp &= ~DMA_RB_ENABLE;
WREG32(DMA_RB_CNTL, tmp);
/* XXX other engines? */
 
/* halt the rlc */
r600_rlc_stop(rdev);
 
udelay(50);
 
/* set mclk/sclk to bypass */
rv770_set_clk_bypass_mode(rdev);
/* disable BM */
pci_clear_master(rdev->pdev);
/* disable mem access */
evergreen_mc_stop(rdev, &save);
if (evergreen_mc_wait_for_idle(rdev)) {
dev_warn(rdev->dev, "Wait for MC idle timed out !\n");
}
/* reset */
radeon_pci_config_reset(rdev);
/* wait for asic to come out of reset */
for (i = 0; i < rdev->usec_timeout; i++) {
if (RREG32(CONFIG_MEMSIZE) != 0xffffffff)
break;
udelay(1);
}
}
 
int evergreen_asic_reset(struct radeon_device *rdev)
{
u32 reset_mask;
3642,10 → 3913,17
if (reset_mask)
r600_set_bios_scratch_engine_hung(rdev, true);
 
/* try soft reset */
evergreen_gpu_soft_reset(rdev, reset_mask);
 
reset_mask = evergreen_gpu_check_soft_reset(rdev);
 
/* try pci config reset */
if (reset_mask && radeon_hard_reset)
evergreen_gpu_pci_config_reset(rdev);
 
reset_mask = evergreen_gpu_check_soft_reset(rdev);
 
if (!reset_mask)
r600_set_bios_scratch_engine_hung(rdev, false);
 
3668,36 → 3946,356
if (!(reset_mask & (RADEON_RESET_GFX |
RADEON_RESET_COMPUTE |
RADEON_RESET_CP))) {
radeon_ring_lockup_update(ring);
radeon_ring_lockup_update(rdev, ring);
return false;
}
/* force CP activities */
radeon_ring_force_activity(rdev, ring);
return radeon_ring_test_lockup(rdev, ring);
}
 
/**
* evergreen_dma_is_lockup - Check if the DMA engine is locked up
*
* @rdev: radeon_device pointer
* @ring: radeon_ring structure holding ring information
*
* Check if the async DMA engine is locked up.
* Returns true if the engine appears to be locked up, false if not.
/*
* RLC
*/
bool evergreen_dma_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
#define RLC_SAVE_RESTORE_LIST_END_MARKER 0x00000000
#define RLC_CLEAR_STATE_END_MARKER 0x00000001
 
void sumo_rlc_fini(struct radeon_device *rdev)
{
u32 reset_mask = evergreen_gpu_check_soft_reset(rdev);
int r;
 
if (!(reset_mask & RADEON_RESET_DMA)) {
radeon_ring_lockup_update(ring);
return false;
/* save restore block */
if (rdev->rlc.save_restore_obj) {
r = radeon_bo_reserve(rdev->rlc.save_restore_obj, false);
if (unlikely(r != 0))
dev_warn(rdev->dev, "(%d) reserve RLC sr bo failed\n", r);
radeon_bo_unpin(rdev->rlc.save_restore_obj);
radeon_bo_unreserve(rdev->rlc.save_restore_obj);
 
radeon_bo_unref(&rdev->rlc.save_restore_obj);
rdev->rlc.save_restore_obj = NULL;
}
/* force ring activities */
radeon_ring_force_activity(rdev, ring);
return radeon_ring_test_lockup(rdev, ring);
 
/* clear state block */
if (rdev->rlc.clear_state_obj) {
r = radeon_bo_reserve(rdev->rlc.clear_state_obj, false);
if (unlikely(r != 0))
dev_warn(rdev->dev, "(%d) reserve RLC c bo failed\n", r);
radeon_bo_unpin(rdev->rlc.clear_state_obj);
radeon_bo_unreserve(rdev->rlc.clear_state_obj);
 
radeon_bo_unref(&rdev->rlc.clear_state_obj);
rdev->rlc.clear_state_obj = NULL;
}
 
/* clear state block */
if (rdev->rlc.cp_table_obj) {
r = radeon_bo_reserve(rdev->rlc.cp_table_obj, false);
if (unlikely(r != 0))
dev_warn(rdev->dev, "(%d) reserve RLC cp table bo failed\n", r);
radeon_bo_unpin(rdev->rlc.cp_table_obj);
radeon_bo_unreserve(rdev->rlc.cp_table_obj);
 
radeon_bo_unref(&rdev->rlc.cp_table_obj);
rdev->rlc.cp_table_obj = NULL;
}
}
 
#define CP_ME_TABLE_SIZE 96
 
int sumo_rlc_init(struct radeon_device *rdev)
{
const u32 *src_ptr;
volatile u32 *dst_ptr;
u32 dws, data, i, j, k, reg_num;
u32 reg_list_num, reg_list_hdr_blk_index, reg_list_blk_index = 0;
u64 reg_list_mc_addr;
const struct cs_section_def *cs_data;
int r;
 
src_ptr = rdev->rlc.reg_list;
dws = rdev->rlc.reg_list_size;
if (rdev->family >= CHIP_BONAIRE) {
dws += (5 * 16) + 48 + 48 + 64;
}
cs_data = rdev->rlc.cs_data;
 
if (src_ptr) {
/* save restore block */
if (rdev->rlc.save_restore_obj == NULL) {
r = radeon_bo_create(rdev, dws * 4, PAGE_SIZE, true,
RADEON_GEM_DOMAIN_VRAM, 0, NULL,
&rdev->rlc.save_restore_obj);
if (r) {
dev_warn(rdev->dev, "(%d) create RLC sr bo failed\n", r);
return r;
}
}
 
r = radeon_bo_reserve(rdev->rlc.save_restore_obj, false);
if (unlikely(r != 0)) {
sumo_rlc_fini(rdev);
return r;
}
r = radeon_bo_pin(rdev->rlc.save_restore_obj, RADEON_GEM_DOMAIN_VRAM,
&rdev->rlc.save_restore_gpu_addr);
if (r) {
radeon_bo_unreserve(rdev->rlc.save_restore_obj);
dev_warn(rdev->dev, "(%d) pin RLC sr bo failed\n", r);
sumo_rlc_fini(rdev);
return r;
}
 
r = radeon_bo_kmap(rdev->rlc.save_restore_obj, (void **)&rdev->rlc.sr_ptr);
if (r) {
dev_warn(rdev->dev, "(%d) map RLC sr bo failed\n", r);
sumo_rlc_fini(rdev);
return r;
}
/* write the sr buffer */
dst_ptr = rdev->rlc.sr_ptr;
if (rdev->family >= CHIP_TAHITI) {
/* SI */
for (i = 0; i < rdev->rlc.reg_list_size; i++)
dst_ptr[i] = cpu_to_le32(src_ptr[i]);
} else {
/* ON/LN/TN */
/* format:
* dw0: (reg2 << 16) | reg1
* dw1: reg1 save space
* dw2: reg2 save space
*/
for (i = 0; i < dws; i++) {
data = src_ptr[i] >> 2;
i++;
if (i < dws)
data |= (src_ptr[i] >> 2) << 16;
j = (((i - 1) * 3) / 2);
dst_ptr[j] = cpu_to_le32(data);
}
j = ((i * 3) / 2);
dst_ptr[j] = cpu_to_le32(RLC_SAVE_RESTORE_LIST_END_MARKER);
}
radeon_bo_kunmap(rdev->rlc.save_restore_obj);
radeon_bo_unreserve(rdev->rlc.save_restore_obj);
}
 
if (cs_data) {
/* clear state block */
if (rdev->family >= CHIP_BONAIRE) {
rdev->rlc.clear_state_size = dws = cik_get_csb_size(rdev);
} else if (rdev->family >= CHIP_TAHITI) {
rdev->rlc.clear_state_size = si_get_csb_size(rdev);
dws = rdev->rlc.clear_state_size + (256 / 4);
} else {
reg_list_num = 0;
dws = 0;
for (i = 0; cs_data[i].section != NULL; i++) {
for (j = 0; cs_data[i].section[j].extent != NULL; j++) {
reg_list_num++;
dws += cs_data[i].section[j].reg_count;
}
}
reg_list_blk_index = (3 * reg_list_num + 2);
dws += reg_list_blk_index;
rdev->rlc.clear_state_size = dws;
}
 
if (rdev->rlc.clear_state_obj == NULL) {
r = radeon_bo_create(rdev, dws * 4, PAGE_SIZE, true,
RADEON_GEM_DOMAIN_VRAM, 0, NULL,
&rdev->rlc.clear_state_obj);
if (r) {
dev_warn(rdev->dev, "(%d) create RLC c bo failed\n", r);
sumo_rlc_fini(rdev);
return r;
}
}
r = radeon_bo_reserve(rdev->rlc.clear_state_obj, false);
if (unlikely(r != 0)) {
sumo_rlc_fini(rdev);
return r;
}
r = radeon_bo_pin(rdev->rlc.clear_state_obj, RADEON_GEM_DOMAIN_VRAM,
&rdev->rlc.clear_state_gpu_addr);
if (r) {
radeon_bo_unreserve(rdev->rlc.clear_state_obj);
dev_warn(rdev->dev, "(%d) pin RLC c bo failed\n", r);
sumo_rlc_fini(rdev);
return r;
}
 
r = radeon_bo_kmap(rdev->rlc.clear_state_obj, (void **)&rdev->rlc.cs_ptr);
if (r) {
dev_warn(rdev->dev, "(%d) map RLC c bo failed\n", r);
sumo_rlc_fini(rdev);
return r;
}
/* set up the cs buffer */
dst_ptr = rdev->rlc.cs_ptr;
if (rdev->family >= CHIP_BONAIRE) {
cik_get_csb_buffer(rdev, dst_ptr);
} else if (rdev->family >= CHIP_TAHITI) {
reg_list_mc_addr = rdev->rlc.clear_state_gpu_addr + 256;
dst_ptr[0] = cpu_to_le32(upper_32_bits(reg_list_mc_addr));
dst_ptr[1] = cpu_to_le32(lower_32_bits(reg_list_mc_addr));
dst_ptr[2] = cpu_to_le32(rdev->rlc.clear_state_size);
si_get_csb_buffer(rdev, &dst_ptr[(256/4)]);
} else {
reg_list_hdr_blk_index = 0;
reg_list_mc_addr = rdev->rlc.clear_state_gpu_addr + (reg_list_blk_index * 4);
data = upper_32_bits(reg_list_mc_addr);
dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(data);
reg_list_hdr_blk_index++;
for (i = 0; cs_data[i].section != NULL; i++) {
for (j = 0; cs_data[i].section[j].extent != NULL; j++) {
reg_num = cs_data[i].section[j].reg_count;
data = reg_list_mc_addr & 0xffffffff;
dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(data);
reg_list_hdr_blk_index++;
 
data = (cs_data[i].section[j].reg_index * 4) & 0xffffffff;
dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(data);
reg_list_hdr_blk_index++;
 
data = 0x08000000 | (reg_num * 4);
dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(data);
reg_list_hdr_blk_index++;
 
for (k = 0; k < reg_num; k++) {
data = cs_data[i].section[j].extent[k];
dst_ptr[reg_list_blk_index + k] = cpu_to_le32(data);
}
reg_list_mc_addr += reg_num * 4;
reg_list_blk_index += reg_num;
}
}
dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(RLC_CLEAR_STATE_END_MARKER);
}
radeon_bo_kunmap(rdev->rlc.clear_state_obj);
radeon_bo_unreserve(rdev->rlc.clear_state_obj);
}
 
if (rdev->rlc.cp_table_size) {
if (rdev->rlc.cp_table_obj == NULL) {
r = radeon_bo_create(rdev, rdev->rlc.cp_table_size,
PAGE_SIZE, true,
RADEON_GEM_DOMAIN_VRAM, 0, NULL,
&rdev->rlc.cp_table_obj);
if (r) {
dev_warn(rdev->dev, "(%d) create RLC cp table bo failed\n", r);
sumo_rlc_fini(rdev);
return r;
}
}
 
r = radeon_bo_reserve(rdev->rlc.cp_table_obj, false);
if (unlikely(r != 0)) {
dev_warn(rdev->dev, "(%d) reserve RLC cp table bo failed\n", r);
sumo_rlc_fini(rdev);
return r;
}
r = radeon_bo_pin(rdev->rlc.cp_table_obj, RADEON_GEM_DOMAIN_VRAM,
&rdev->rlc.cp_table_gpu_addr);
if (r) {
radeon_bo_unreserve(rdev->rlc.cp_table_obj);
dev_warn(rdev->dev, "(%d) pin RLC cp_table bo failed\n", r);
sumo_rlc_fini(rdev);
return r;
}
r = radeon_bo_kmap(rdev->rlc.cp_table_obj, (void **)&rdev->rlc.cp_table_ptr);
if (r) {
dev_warn(rdev->dev, "(%d) map RLC cp table bo failed\n", r);
sumo_rlc_fini(rdev);
return r;
}
 
cik_init_cp_pg_table(rdev);
 
radeon_bo_kunmap(rdev->rlc.cp_table_obj);
radeon_bo_unreserve(rdev->rlc.cp_table_obj);
 
}
 
return 0;
}
 
static void evergreen_rlc_start(struct radeon_device *rdev)
{
u32 mask = RLC_ENABLE;
 
if (rdev->flags & RADEON_IS_IGP) {
mask |= GFX_POWER_GATING_ENABLE | GFX_POWER_GATING_SRC;
}
 
WREG32(RLC_CNTL, mask);
}
 
int evergreen_rlc_resume(struct radeon_device *rdev)
{
u32 i;
const __be32 *fw_data;
 
if (!rdev->rlc_fw)
return -EINVAL;
 
r600_rlc_stop(rdev);
 
WREG32(RLC_HB_CNTL, 0);
 
if (rdev->flags & RADEON_IS_IGP) {
if (rdev->family == CHIP_ARUBA) {
u32 always_on_bitmap =
3 | (3 << (16 * rdev->config.cayman.max_shader_engines));
/* find out the number of active simds */
u32 tmp = (RREG32(CC_GC_SHADER_PIPE_CONFIG) & 0xffff0000) >> 16;
tmp |= 0xffffffff << rdev->config.cayman.max_simds_per_se;
tmp = hweight32(~tmp);
if (tmp == rdev->config.cayman.max_simds_per_se) {
WREG32(TN_RLC_LB_ALWAYS_ACTIVE_SIMD_MASK, always_on_bitmap);
WREG32(TN_RLC_LB_PARAMS, 0x00601004);
WREG32(TN_RLC_LB_INIT_SIMD_MASK, 0xffffffff);
WREG32(TN_RLC_LB_CNTR_INIT, 0x00000000);
WREG32(TN_RLC_LB_CNTR_MAX, 0x00002000);
}
} else {
WREG32(RLC_HB_WPTR_LSB_ADDR, 0);
WREG32(RLC_HB_WPTR_MSB_ADDR, 0);
}
WREG32(TN_RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
WREG32(TN_RLC_CLEAR_STATE_RESTORE_BASE, rdev->rlc.clear_state_gpu_addr >> 8);
} else {
WREG32(RLC_HB_BASE, 0);
WREG32(RLC_HB_RPTR, 0);
WREG32(RLC_HB_WPTR, 0);
WREG32(RLC_HB_WPTR_LSB_ADDR, 0);
WREG32(RLC_HB_WPTR_MSB_ADDR, 0);
}
WREG32(RLC_MC_CNTL, 0);
WREG32(RLC_UCODE_CNTL, 0);
 
fw_data = (const __be32 *)rdev->rlc_fw->data;
if (rdev->family >= CHIP_ARUBA) {
for (i = 0; i < ARUBA_RLC_UCODE_SIZE; i++) {
WREG32(RLC_UCODE_ADDR, i);
WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
}
} else if (rdev->family >= CHIP_CAYMAN) {
for (i = 0; i < CAYMAN_RLC_UCODE_SIZE; i++) {
WREG32(RLC_UCODE_ADDR, i);
WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
}
} else {
for (i = 0; i < EVERGREEN_RLC_UCODE_SIZE; i++) {
WREG32(RLC_UCODE_ADDR, i);
WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
}
}
WREG32(RLC_UCODE_ADDR, 0);
 
evergreen_rlc_start(rdev);
 
return 0;
}
 
/* Interrupts */
 
u32 evergreen_get_vblank_counter(struct radeon_device *rdev, int crtc)
3746,8 → 4344,8
WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
}
 
/* only one DAC on DCE6 */
if (!ASIC_IS_DCE6(rdev))
/* only one DAC on DCE5 */
if (!ASIC_IS_DCE5(rdev))
WREG32(DACA_AUTODETECT_INT_CONTROL, 0);
WREG32(DACB_AUTODETECT_INT_CONTROL, 0);
 
3773,9 → 4371,9
u32 crtc1 = 0, crtc2 = 0, crtc3 = 0, crtc4 = 0, crtc5 = 0, crtc6 = 0;
u32 hpd1, hpd2, hpd3, hpd4, hpd5, hpd6;
u32 grbm_int_cntl = 0;
u32 grph1 = 0, grph2 = 0, grph3 = 0, grph4 = 0, grph5 = 0, grph6 = 0;
u32 afmt1 = 0, afmt2 = 0, afmt3 = 0, afmt4 = 0, afmt5 = 0, afmt6 = 0;
u32 dma_cntl, dma_cntl1 = 0;
u32 thermal_int = 0;
 
if (!rdev->irq.installed) {
WARN(1, "Can't enable IRQ/MSI because no handler is installed\n");
3795,6 → 4393,12
hpd4 = RREG32(DC_HPD4_INT_CONTROL) & ~DC_HPDx_INT_EN;
hpd5 = RREG32(DC_HPD5_INT_CONTROL) & ~DC_HPDx_INT_EN;
hpd6 = RREG32(DC_HPD6_INT_CONTROL) & ~DC_HPDx_INT_EN;
if (rdev->family == CHIP_ARUBA)
thermal_int = RREG32(TN_CG_THERMAL_INT_CTRL) &
~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW);
else
thermal_int = RREG32(CG_THERMAL_INT) &
~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW);
 
afmt1 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
afmt2 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
3840,6 → 4444,11
}
}
 
if (rdev->irq.dpm_thermal) {
DRM_DEBUG("dpm thermal\n");
thermal_int |= THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW;
}
 
if (rdev->irq.crtc_vblank_int[0] ||
atomic_read(&rdev->irq.pflip[0])) {
DRM_DEBUG("evergreen_irq_set: vblank 0\n");
3944,15 → 4553,21
WREG32(INT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, crtc6);
}
 
WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, grph1);
WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, grph2);
WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET,
GRPH_PFLIP_INT_MASK);
WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET,
GRPH_PFLIP_INT_MASK);
if (rdev->num_crtc >= 4) {
WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, grph3);
WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, grph4);
WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET,
GRPH_PFLIP_INT_MASK);
WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET,
GRPH_PFLIP_INT_MASK);
}
if (rdev->num_crtc >= 6) {
WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, grph5);
WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, grph6);
WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET,
GRPH_PFLIP_INT_MASK);
WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET,
GRPH_PFLIP_INT_MASK);
}
 
WREG32(DC_HPD1_INT_CONTROL, hpd1);
3961,6 → 4576,10
WREG32(DC_HPD4_INT_CONTROL, hpd4);
WREG32(DC_HPD5_INT_CONTROL, hpd5);
WREG32(DC_HPD6_INT_CONTROL, hpd6);
if (rdev->family == CHIP_ARUBA)
WREG32(TN_CG_THERMAL_INT_CTRL, thermal_int);
else
WREG32(CG_THERMAL_INT, thermal_int);
 
WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, afmt1);
WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, afmt2);
4140,6 → 4759,7
tmp = RREG32(IH_RB_CNTL);
tmp |= IH_WPTR_OVERFLOW_CLEAR;
WREG32(IH_RB_CNTL, tmp);
wptr &= ~RB_OVERFLOW;
}
return (wptr & rdev->ih.ptr_mask);
}
4152,6 → 4772,8
u32 ring_index;
bool queue_hotplug = false;
bool queue_hdmi = false;
bool queue_thermal = false;
u32 status, addr;
 
if (!rdev->ih.enabled || rdev->shutdown)
return IRQ_NONE;
4335,6 → 4957,14
break;
}
break;
case 8: /* D1 page flip */
case 10: /* D2 page flip */
case 12: /* D3 page flip */
case 14: /* D4 page flip */
case 16: /* D5 page flip */
case 18: /* D6 page flip */
DRM_DEBUG("IH: D%d flip\n", ((src_id - 8) >> 1) + 1);
break;
case 42: /* HPD hotplug */
switch (src_data) {
case 0:
4438,13 → 5068,18
break;
case 146:
case 147:
addr = RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR);
status = RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS);
/* reset addr and status */
WREG32_P(VM_CONTEXT1_CNTL2, 1, ~1);
if (addr == 0x0 && status == 0x0)
break;
dev_err(rdev->dev, "GPU fault detected: %d 0x%08x\n", src_id, src_data);
dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR));
addr);
dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS));
/* reset addr and status */
WREG32_P(VM_CONTEXT1_CNTL2, 1, ~1);
status);
cayman_vm_decode_fault(rdev, status, addr);
break;
case 176: /* CP_INT in ring buffer */
case 177: /* CP_INT in IB1 */
4473,6 → 5108,16
DRM_DEBUG("IH: DMA trap\n");
radeon_fence_process(rdev, R600_RING_TYPE_DMA_INDEX);
break;
case 230: /* thermal low to high */
DRM_DEBUG("IH: thermal low to high\n");
rdev->pm.dpm.thermal.high_to_low = false;
queue_thermal = true;
break;
case 231: /* thermal high to low */
DRM_DEBUG("IH: thermal high to low\n");
rdev->pm.dpm.thermal.high_to_low = true;
queue_thermal = true;
break;
case 233: /* GUI IDLE */
DRM_DEBUG("IH: GUI idle\n");
break;
4503,143 → 5148,6
return IRQ_HANDLED;
}
 
/**
* evergreen_dma_fence_ring_emit - emit a fence on the DMA ring
*
* @rdev: radeon_device pointer
* @fence: radeon fence object
*
* Add a DMA fence packet to the ring to write
* the fence seq number and DMA trap packet to generate
* an interrupt if needed (evergreen-SI).
*/
void evergreen_dma_fence_ring_emit(struct radeon_device *rdev,
struct radeon_fence *fence)
{
struct radeon_ring *ring = &rdev->ring[fence->ring];
u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
/* write the fence */
radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_FENCE, 0, 0));
radeon_ring_write(ring, addr & 0xfffffffc);
radeon_ring_write(ring, (upper_32_bits(addr) & 0xff));
radeon_ring_write(ring, fence->seq);
/* generate an interrupt */
radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_TRAP, 0, 0));
/* flush HDP */
radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_SRBM_WRITE, 0, 0));
radeon_ring_write(ring, (0xf << 16) | (HDP_MEM_COHERENCY_FLUSH_CNTL >> 2));
radeon_ring_write(ring, 1);
}
 
/**
* evergreen_dma_ring_ib_execute - schedule an IB on the DMA engine
*
* @rdev: radeon_device pointer
* @ib: IB object to schedule
*
* Schedule an IB in the DMA ring (evergreen).
*/
void evergreen_dma_ring_ib_execute(struct radeon_device *rdev,
struct radeon_ib *ib)
{
struct radeon_ring *ring = &rdev->ring[ib->ring];
 
if (rdev->wb.enabled) {
u32 next_rptr = ring->wptr + 4;
while ((next_rptr & 7) != 5)
next_rptr++;
next_rptr += 3;
radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_WRITE, 0, 1));
radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
radeon_ring_write(ring, upper_32_bits(ring->next_rptr_gpu_addr) & 0xff);
radeon_ring_write(ring, next_rptr);
}
 
/* The indirect buffer packet must end on an 8 DW boundary in the DMA ring.
* Pad as necessary with NOPs.
*/
while ((ring->wptr & 7) != 5)
radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_NOP, 0, 0));
radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_INDIRECT_BUFFER, 0, 0));
radeon_ring_write(ring, (ib->gpu_addr & 0xFFFFFFE0));
radeon_ring_write(ring, (ib->length_dw << 12) | (upper_32_bits(ib->gpu_addr) & 0xFF));
 
}
 
/**
* evergreen_copy_dma - copy pages using the DMA engine
*
* @rdev: radeon_device pointer
* @src_offset: src GPU address
* @dst_offset: dst GPU address
* @num_gpu_pages: number of GPU pages to xfer
* @fence: radeon fence object
*
* Copy GPU paging using the DMA engine (evergreen-cayman).
* Used by the radeon ttm implementation to move pages if
* registered as the asic copy callback.
*/
int evergreen_copy_dma(struct radeon_device *rdev,
uint64_t src_offset, uint64_t dst_offset,
unsigned num_gpu_pages,
struct radeon_fence **fence)
{
struct radeon_semaphore *sem = NULL;
int ring_index = rdev->asic->copy.dma_ring_index;
struct radeon_ring *ring = &rdev->ring[ring_index];
u32 size_in_dw, cur_size_in_dw;
int i, num_loops;
int r = 0;
 
r = radeon_semaphore_create(rdev, &sem);
if (r) {
DRM_ERROR("radeon: moving bo (%d).\n", r);
return r;
}
 
size_in_dw = (num_gpu_pages << RADEON_GPU_PAGE_SHIFT) / 4;
num_loops = DIV_ROUND_UP(size_in_dw, 0xfffff);
r = radeon_ring_lock(rdev, ring, num_loops * 5 + 11);
if (r) {
DRM_ERROR("radeon: moving bo (%d).\n", r);
radeon_semaphore_free(rdev, &sem, NULL);
return r;
}
 
if (radeon_fence_need_sync(*fence, ring->idx)) {
radeon_semaphore_sync_rings(rdev, sem, (*fence)->ring,
ring->idx);
radeon_fence_note_sync(*fence, ring->idx);
} else {
radeon_semaphore_free(rdev, &sem, NULL);
}
 
for (i = 0; i < num_loops; i++) {
cur_size_in_dw = size_in_dw;
if (cur_size_in_dw > 0xFFFFF)
cur_size_in_dw = 0xFFFFF;
size_in_dw -= cur_size_in_dw;
radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_COPY, 0, cur_size_in_dw));
radeon_ring_write(ring, dst_offset & 0xfffffffc);
radeon_ring_write(ring, src_offset & 0xfffffffc);
radeon_ring_write(ring, upper_32_bits(dst_offset) & 0xff);
radeon_ring_write(ring, upper_32_bits(src_offset) & 0xff);
src_offset += cur_size_in_dw * 4;
dst_offset += cur_size_in_dw * 4;
}
 
r = radeon_fence_emit(rdev, fence, ring->idx);
if (r) {
radeon_ring_unlock_undo(rdev, ring);
return r;
}
 
radeon_ring_unlock_commit(rdev, ring);
radeon_semaphore_free(rdev, &sem, *fence);
 
return r;
}
 
static int evergreen_startup(struct radeon_device *rdev)
{
struct radeon_ring *ring;
4647,35 → 5155,24
 
/* enable pcie gen2 link */
evergreen_pcie_gen2_enable(rdev);
/* enable aspm */
evergreen_program_aspm(rdev);
 
if (ASIC_IS_DCE5(rdev)) {
if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw || !rdev->mc_fw) {
r = ni_init_microcode(rdev);
if (r) {
DRM_ERROR("Failed to load firmware!\n");
/* scratch needs to be initialized before MC */
r = r600_vram_scratch_init(rdev);
if (r)
return r;
}
}
 
evergreen_mc_program(rdev);
 
if (ASIC_IS_DCE5(rdev) && !rdev->pm.dpm_enabled) {
r = ni_mc_load_microcode(rdev);
if (r) {
DRM_ERROR("Failed to load MC firmware!\n");
return r;
}
} else {
if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw) {
r = r600_init_microcode(rdev);
if (r) {
DRM_ERROR("Failed to load firmware!\n");
return r;
}
}
}
 
r = r600_vram_scratch_init(rdev);
if (r)
return r;
 
evergreen_mc_program(rdev);
if (rdev->flags & RADEON_IS_AGP) {
evergreen_agp_enable(rdev);
} else {
4685,12 → 5182,18
}
evergreen_gpu_init(rdev);
 
r = evergreen_blit_init(rdev);
/* allocate rlc buffers */
if (rdev->flags & RADEON_IS_IGP) {
rdev->rlc.reg_list = sumo_rlc_save_restore_register_list;
rdev->rlc.reg_list_size =
(u32)ARRAY_SIZE(sumo_rlc_save_restore_register_list);
rdev->rlc.cs_data = evergreen_cs_data;
r = sumo_rlc_init(rdev);
if (r) {
// r600_blit_fini(rdev);
rdev->asic->copy.copy = NULL;
dev_warn(rdev->dev, "failed blitter (%d) falling back to memcpy\n", r);
DRM_ERROR("Failed to init rlc BOs!\n");
return r;
}
}
 
/* allocate wb buffer */
r = radeon_wb_init(rdev);
4717,8 → 5220,8
// dev_err(rdev->dev, "UVD fences init error (%d).\n", r);
// }
 
// if (r)
// rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0;
if (r)
rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0;
 
/* Enable IRQ */
if (!rdev->irq.installed) {
4737,15 → 5240,13
 
ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
R600_CP_RB_RPTR, R600_CP_RB_WPTR,
0, 0xfffff, RADEON_CP_PACKET2);
RADEON_CP_PACKET2);
if (r)
return r;
 
ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET,
DMA_RB_RPTR, DMA_RB_WPTR,
2, 0x3fffc, DMA_PACKET(DMA_PACKET_NOP, 0, 0));
DMA_PACKET(DMA_PACKET_NOP, 0, 0));
if (r)
return r;
 
4759,19 → 5260,7
if (r)
return r;
 
ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
if (ring->ring_size) {
r = radeon_ring_init(rdev, ring, ring->ring_size,
R600_WB_UVD_RPTR_OFFSET,
UVD_RBC_RB_RPTR, UVD_RBC_RB_WPTR,
0, 0xfffff, RADEON_CP_PACKET2);
if (!r)
r = r600_uvd_init(rdev);
 
if (r)
DRM_ERROR("radeon: error initializing UVD (%d).\n", r);
}
 
r = radeon_ib_pool_init(rdev);
if (r) {
dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
4783,30 → 5272,7
 
 
 
#if 0
 
int evergreen_copy_blit(struct radeon_device *rdev,
uint64_t src_offset, uint64_t dst_offset,
unsigned num_pages, struct radeon_fence *fence)
{
int r;
 
mutex_lock(&rdev->r600_blit.mutex);
rdev->r600_blit.vb_ib = NULL;
r = evergreen_blit_prepare_copy(rdev, num_pages * RADEON_GPU_PAGE_SIZE);
if (r) {
if (rdev->r600_blit.vb_ib)
radeon_ib_free(rdev, &rdev->r600_blit.vb_ib);
mutex_unlock(&rdev->r600_blit.mutex);
return r;
}
evergreen_kms_blit_copy(rdev, src_offset, dst_offset, num_pages * RADEON_GPU_PAGE_SIZE);
evergreen_blit_done_copy(rdev, fence);
mutex_unlock(&rdev->r600_blit.mutex);
return 0;
}
#endif
 
/* Plan is to move initialization in that function and use
* helper function so that radeon_device_init pretty much
* do nothing more than calling asic specific function. This
4871,6 → 5337,27
if (r)
return r;
 
if (ASIC_IS_DCE5(rdev)) {
if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw || !rdev->mc_fw) {
r = ni_init_microcode(rdev);
if (r) {
DRM_ERROR("Failed to load firmware!\n");
return r;
}
}
} else {
if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw) {
r = r600_init_microcode(rdev);
if (r) {
DRM_ERROR("Failed to load firmware!\n");
return r;
}
}
}
 
/* Initialize power management */
radeon_pm_init(rdev);
 
rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ring_obj = NULL;
r600_ring_init(rdev, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX], 1024 * 1024);
 
4975,3 → 5462,153
WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
}
}
 
void evergreen_program_aspm(struct radeon_device *rdev)
{
u32 data, orig;
u32 pcie_lc_cntl, pcie_lc_cntl_old;
bool disable_l0s, disable_l1 = false, disable_plloff_in_l1 = false;
/* fusion_platform = true
* if the system is a fusion system
* (APU or DGPU in a fusion system).
* todo: check if the system is a fusion platform.
*/
bool fusion_platform = false;
 
if (radeon_aspm == 0)
return;
 
if (!(rdev->flags & RADEON_IS_PCIE))
return;
 
switch (rdev->family) {
case CHIP_CYPRESS:
case CHIP_HEMLOCK:
case CHIP_JUNIPER:
case CHIP_REDWOOD:
case CHIP_CEDAR:
case CHIP_SUMO:
case CHIP_SUMO2:
case CHIP_PALM:
case CHIP_ARUBA:
disable_l0s = true;
break;
default:
disable_l0s = false;
break;
}
 
if (rdev->flags & RADEON_IS_IGP)
fusion_platform = true; /* XXX also dGPUs in a fusion system */
 
data = orig = RREG32_PIF_PHY0(PB0_PIF_PAIRING);
if (fusion_platform)
data &= ~MULTI_PIF;
else
data |= MULTI_PIF;
if (data != orig)
WREG32_PIF_PHY0(PB0_PIF_PAIRING, data);
 
data = orig = RREG32_PIF_PHY1(PB1_PIF_PAIRING);
if (fusion_platform)
data &= ~MULTI_PIF;
else
data |= MULTI_PIF;
if (data != orig)
WREG32_PIF_PHY1(PB1_PIF_PAIRING, data);
 
pcie_lc_cntl = pcie_lc_cntl_old = RREG32_PCIE_PORT(PCIE_LC_CNTL);
pcie_lc_cntl &= ~(LC_L0S_INACTIVITY_MASK | LC_L1_INACTIVITY_MASK);
if (!disable_l0s) {
if (rdev->family >= CHIP_BARTS)
pcie_lc_cntl |= LC_L0S_INACTIVITY(7);
else
pcie_lc_cntl |= LC_L0S_INACTIVITY(3);
}
 
if (!disable_l1) {
if (rdev->family >= CHIP_BARTS)
pcie_lc_cntl |= LC_L1_INACTIVITY(7);
else
pcie_lc_cntl |= LC_L1_INACTIVITY(8);
 
if (!disable_plloff_in_l1) {
data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0);
data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
if (data != orig)
WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0, data);
 
data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1);
data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
if (data != orig)
WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1, data);
 
data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0);
data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
if (data != orig)
WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0, data);
 
data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1);
data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
if (data != orig)
WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1, data);
 
if (rdev->family >= CHIP_BARTS) {
data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0);
data &= ~PLL_RAMP_UP_TIME_0_MASK;
data |= PLL_RAMP_UP_TIME_0(4);
if (data != orig)
WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0, data);
 
data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1);
data &= ~PLL_RAMP_UP_TIME_1_MASK;
data |= PLL_RAMP_UP_TIME_1(4);
if (data != orig)
WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1, data);
 
data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0);
data &= ~PLL_RAMP_UP_TIME_0_MASK;
data |= PLL_RAMP_UP_TIME_0(4);
if (data != orig)
WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0, data);
 
data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1);
data &= ~PLL_RAMP_UP_TIME_1_MASK;
data |= PLL_RAMP_UP_TIME_1(4);
if (data != orig)
WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1, data);
}
 
data = orig = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
data &= ~LC_DYN_LANES_PWR_STATE_MASK;
data |= LC_DYN_LANES_PWR_STATE(3);
if (data != orig)
WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, data);
 
if (rdev->family >= CHIP_BARTS) {
data = orig = RREG32_PIF_PHY0(PB0_PIF_CNTL);
data &= ~LS2_EXIT_TIME_MASK;
data |= LS2_EXIT_TIME(1);
if (data != orig)
WREG32_PIF_PHY0(PB0_PIF_CNTL, data);
 
data = orig = RREG32_PIF_PHY1(PB1_PIF_CNTL);
data &= ~LS2_EXIT_TIME_MASK;
data |= LS2_EXIT_TIME(1);
if (data != orig)
WREG32_PIF_PHY1(PB1_PIF_CNTL, data);
}
}
}
 
/* evergreen parts only */
if (rdev->family < CHIP_BARTS)
pcie_lc_cntl |= LC_PMI_TO_L1_DIS;
 
if (pcie_lc_cntl != pcie_lc_cntl_old)
WREG32_PCIE_PORT(PCIE_LC_CNTL, pcie_lc_cntl);
}