diff options
Diffstat (limited to 'drivers/gpu/drm/radeon/evergreen.c')
| -rw-r--r-- | drivers/gpu/drm/radeon/evergreen.c | 1268 |
1 files changed, 959 insertions, 309 deletions
diff --git a/drivers/gpu/drm/radeon/evergreen.c b/drivers/gpu/drm/radeon/evergreen.c index 0f89ce3d02b..15e4f28015e 100644 --- a/drivers/gpu/drm/radeon/evergreen.c +++ b/drivers/gpu/drm/radeon/evergreen.c @@ -33,9 +33,7 @@ #include "avivod.h" #include "evergreen_reg.h" #include "evergreen_blit_shaders.h" - -#define EVERGREEN_PFP_UCODE_SIZE 1120 -#define EVERGREEN_PM4_UCODE_SIZE 1376 +#include "radeon_ucode.h" static const u32 crtc_offsets[6] = { @@ -47,11 +45,108 @@ static const u32 crtc_offsets[6] = EVERGREEN_CRTC5_REGISTER_OFFSET }; +#include "clearstate_evergreen.h" + +static const u32 sumo_rlc_save_restore_register_list[] = +{ + 0x98fc, + 0x9830, + 0x9834, + 0x9838, + 0x9870, + 0x9874, + 0x8a14, + 0x8b24, + 0x8bcc, + 0x8b10, + 0x8d00, + 0x8d04, + 0x8c00, + 0x8c04, + 0x8c08, + 0x8c0c, + 0x8d8c, + 0x8c20, + 0x8c24, + 0x8c28, + 0x8c18, + 0x8c1c, + 0x8cf0, + 0x8e2c, + 0x8e38, + 0x8c30, + 0x9508, + 0x9688, + 0x9608, + 0x960c, + 0x9610, + 0x9614, + 0x88c4, + 0x88d4, + 0xa008, + 0x900c, + 0x9100, + 0x913c, + 0x98f8, + 0x98f4, + 0x9b7c, + 0x3f8c, + 0x8950, + 0x8954, + 0x8a18, + 0x8b28, + 0x9144, + 0x9148, + 0x914c, + 0x3f90, + 0x3f94, + 0x915c, + 0x9160, + 0x9178, + 0x917c, + 0x9180, + 0x918c, + 0x9190, + 0x9194, + 0x9198, + 0x919c, + 0x91a8, + 0x91ac, + 0x91b0, + 0x91b4, + 0x91b8, + 0x91c4, + 0x91c8, + 0x91cc, + 0x91d0, + 0x91d4, + 0x91e0, + 0x91e4, + 0x91ec, + 0x91f0, + 0x91f4, + 0x9200, + 0x9204, + 0x929c, + 0x9150, + 0x802c, +}; + static void evergreen_gpu_init(struct radeon_device *rdev); void evergreen_fini(struct radeon_device *rdev); void evergreen_pcie_gen2_enable(struct radeon_device *rdev); +void evergreen_program_aspm(struct radeon_device *rdev); extern void cayman_cp_int_cntl_setup(struct radeon_device *rdev, int ring, u32 cp_int_cntl); +extern void cayman_vm_decode_fault(struct radeon_device *rdev, + u32 status, u32 addr); +void cik_init_cp_pg_table(struct radeon_device *rdev); + +extern u32 si_get_csb_size(struct radeon_device *rdev); +extern void si_get_csb_buffer(struct radeon_device *rdev, volatile u32 *buffer); +extern u32 cik_get_csb_size(struct radeon_device *rdev); +extern void cik_get_csb_buffer(struct radeon_device *rdev, volatile u32 *buffer); +extern void rv770_set_clk_bypass_mode(struct radeon_device *rdev); static const u32 evergreen_golden_registers[] = { @@ -94,7 +189,7 @@ static const u32 evergreen_golden_registers[] = 0x8c1c, 0xffffffff, 0x00001010, 0x28350, 0xffffffff, 0x00000000, 0xa008, 0xffffffff, 0x00010000, - 0x5cc, 0xffffffff, 0x00000001, + 0x5c4, 0xffffffff, 0x00000001, 0x9508, 0xffffffff, 0x00000002, 0x913c, 0x0000000f, 0x0000000a }; @@ -381,7 +476,7 @@ static const u32 cedar_golden_registers[] = 0x8c1c, 0xffffffff, 0x00001010, 0x28350, 0xffffffff, 0x00000000, 0xa008, 0xffffffff, 0x00010000, - 0x5cc, 0xffffffff, 0x00000001, + 0x5c4, 0xffffffff, 0x00000001, 0x9508, 0xffffffff, 0x00000002 }; @@ -540,7 +635,7 @@ static const u32 juniper_mgcg_init[] = static const u32 supersumo_golden_registers[] = { 0x5eb4, 0xffffffff, 0x00000002, - 0x5cc, 0xffffffff, 0x00000001, + 0x5c4, 0xffffffff, 0x00000001, 0x7030, 0xffffffff, 0x00000011, 0x7c30, 0xffffffff, 0x00000011, 0x6104, 0x01000300, 0x00000000, @@ -624,7 +719,7 @@ static const u32 sumo_golden_registers[] = static const u32 wrestler_golden_registers[] = { 0x5eb4, 0xffffffff, 0x00000002, - 0x5cc, 0xffffffff, 0x00000001, + 0x5c4, 0xffffffff, 0x00000001, 0x7030, 0xffffffff, 0x00000011, 0x7c30, 0xffffffff, 0x00000011, 0x6104, 0x01000300, 0x00000000, @@ -1080,23 +1175,72 @@ int evergreen_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk) void evergreen_fix_pci_max_read_req_size(struct radeon_device *rdev) { - u16 ctl, v; - int err; - - err = pcie_capability_read_word(rdev->pdev, PCI_EXP_DEVCTL, &ctl); - if (err) - return; - - v = (ctl & PCI_EXP_DEVCTL_READRQ) >> 12; + int readrq; + u16 v; + readrq = pcie_get_readrq(rdev->pdev); + v = ffs(readrq) - 8; /* if bios or OS sets MAX_READ_REQUEST_SIZE to an invalid value, fix it * to avoid hangs or perfomance issues */ - if ((v == 0) || (v == 6) || (v == 7)) { - ctl &= ~PCI_EXP_DEVCTL_READRQ; - ctl |= (2 << 12); - pcie_capability_write_word(rdev->pdev, PCI_EXP_DEVCTL, ctl); + if ((v == 0) || (v == 6) || (v == 7)) + pcie_set_readrq(rdev->pdev, 512); +} + +void dce4_program_fmt(struct drm_encoder *encoder) +{ + struct drm_device *dev = encoder->dev; + struct radeon_device *rdev = dev->dev_private; + struct radeon_encoder *radeon_encoder = to_radeon_encoder(encoder); + struct radeon_crtc *radeon_crtc = to_radeon_crtc(encoder->crtc); + struct drm_connector *connector = radeon_get_connector_for_encoder(encoder); + int bpc = 0; + u32 tmp = 0; + enum radeon_connector_dither dither = RADEON_FMT_DITHER_DISABLE; + + if (connector) { + struct radeon_connector *radeon_connector = to_radeon_connector(connector); + bpc = radeon_get_monitor_bpc(connector); + dither = radeon_connector->dither; } + + /* LVDS/eDP FMT is set up by atom */ + if (radeon_encoder->devices & ATOM_DEVICE_LCD_SUPPORT) + return; + + /* not needed for analog */ + if ((radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC1) || + (radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC2)) + return; + + if (bpc == 0) + return; + + switch (bpc) { + case 6: + if (dither == RADEON_FMT_DITHER_ENABLE) + /* XXX sort out optimal dither settings */ + tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE | + FMT_SPATIAL_DITHER_EN); + else + tmp |= FMT_TRUNCATE_EN; + break; + case 8: + if (dither == RADEON_FMT_DITHER_ENABLE) + /* XXX sort out optimal dither settings */ + tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE | + FMT_RGB_RANDOM_ENABLE | + FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH); + else + tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH); + break; + case 10: + default: + /* not needed */ + break; + } + + WREG32(FMT_BIT_DEPTH_CONTROL + radeon_crtc->crtc_offset, tmp); } static bool dce4_is_in_vblank(struct radeon_device *rdev, int crtc) @@ -1157,36 +1301,6 @@ void dce4_wait_for_vblank(struct radeon_device *rdev, int crtc) } /** - * radeon_irq_kms_pflip_irq_get - pre-pageflip callback. - * - * @rdev: radeon_device pointer - * @crtc: crtc to prepare for pageflip on - * - * Pre-pageflip callback (evergreen+). - * Enables the pageflip irq (vblank irq). - */ -void evergreen_pre_page_flip(struct radeon_device *rdev, int crtc) -{ - /* enable the pflip int */ - radeon_irq_kms_pflip_irq_get(rdev, crtc); -} - -/** - * evergreen_post_page_flip - pos-pageflip callback. - * - * @rdev: radeon_device pointer - * @crtc: crtc to cleanup pageflip on - * - * Post-pageflip callback (evergreen+). - * Disables the pageflip irq (vblank irq). - */ -void evergreen_post_page_flip(struct radeon_device *rdev, int crtc) -{ - /* disable the pflip int */ - radeon_irq_kms_pflip_irq_put(rdev, crtc); -} - -/** * evergreen_page_flip - pageflip callback. * * @rdev: radeon_device pointer @@ -1199,7 +1313,7 @@ void evergreen_post_page_flip(struct radeon_device *rdev, int crtc) * double buffered update to take place. * Returns the current update pending status. */ -u32 evergreen_page_flip(struct radeon_device *rdev, int crtc_id, u64 crtc_base) +void evergreen_page_flip(struct radeon_device *rdev, int crtc_id, u64 crtc_base) { struct radeon_crtc *radeon_crtc = rdev->mode_info.crtcs[crtc_id]; u32 tmp = RREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset); @@ -1231,9 +1345,23 @@ u32 evergreen_page_flip(struct radeon_device *rdev, int crtc_id, u64 crtc_base) /* Unlock the lock, so double-buffering can take place inside vblank */ tmp &= ~EVERGREEN_GRPH_UPDATE_LOCK; WREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset, tmp); +} + +/** + * evergreen_page_flip_pending - check if page flip is still pending + * + * @rdev: radeon_device pointer + * @crtc_id: crtc to check + * + * Returns the current update pending status. + */ +bool evergreen_page_flip_pending(struct radeon_device *rdev, int crtc_id) +{ + struct radeon_crtc *radeon_crtc = rdev->mode_info.crtcs[crtc_id]; /* Return current update_pending status: */ - return RREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset) & EVERGREEN_GRPH_SURFACE_UPDATE_PENDING; + return !!(RREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset) & + EVERGREEN_GRPH_SURFACE_UPDATE_PENDING); } /* get temperature in millidegrees */ @@ -1417,8 +1545,8 @@ void evergreen_pm_misc(struct radeon_device *rdev) struct radeon_voltage *voltage = &ps->clock_info[req_cm_idx].voltage; if (voltage->type == VOLTAGE_SW) { - /* 0xff01 is a flag rather then an actual voltage */ - if (voltage->voltage == 0xff01) + /* 0xff0x are flags rather then an actual voltage */ + if ((voltage->voltage & 0xff00) == 0xff00) return; if (voltage->voltage && (voltage->voltage != rdev->pm.current_vddc)) { radeon_atom_set_voltage(rdev, voltage->voltage, SET_VOLTAGE_TYPE_ASIC_VDDC); @@ -1438,8 +1566,8 @@ void evergreen_pm_misc(struct radeon_device *rdev) voltage = &rdev->pm.power_state[req_ps_idx]. clock_info[rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx].voltage; - /* 0xff01 is a flag rather then an actual voltage */ - if (voltage->vddci == 0xff01) + /* 0xff0x are flags rather then an actual voltage */ + if ((voltage->vddci & 0xff00) == 0xff00) return; if (voltage->vddci && (voltage->vddci != rdev->pm.current_vddci)) { radeon_atom_set_voltage(rdev, voltage->vddci, SET_VOLTAGE_TYPE_ASIC_VDDCI); @@ -1536,7 +1664,7 @@ bool evergreen_hpd_sense(struct radeon_device *rdev, enum radeon_hpd_id hpd) case RADEON_HPD_6: if (RREG32(DC_HPD6_INT_STATUS) & DC_HPDx_SENSE) connected = true; - break; + break; default: break; } @@ -1718,7 +1846,8 @@ static u32 evergreen_line_buffer_adjust(struct radeon_device *rdev, struct drm_display_mode *mode, struct drm_display_mode *other_mode) { - u32 tmp; + u32 tmp, buffer_alloc, i; + u32 pipe_offset = radeon_crtc->crtc_id * 0x20; /* * Line Buffer Setup * There are 3 line buffers, each one shared by 2 display controllers. @@ -1741,18 +1870,34 @@ static u32 evergreen_line_buffer_adjust(struct radeon_device *rdev, * non-linked crtcs for maximum line buffer allocation. */ if (radeon_crtc->base.enabled && mode) { - if (other_mode) + if (other_mode) { tmp = 0; /* 1/2 */ - else + buffer_alloc = 1; + } else { tmp = 2; /* whole */ - } else + buffer_alloc = 2; + } + } else { tmp = 0; + buffer_alloc = 0; + } /* second controller of the pair uses second half of the lb */ if (radeon_crtc->crtc_id % 2) tmp += 4; WREG32(DC_LB_MEMORY_SPLIT + radeon_crtc->crtc_offset, tmp); + if (ASIC_IS_DCE41(rdev) || ASIC_IS_DCE5(rdev)) { + WREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset, + DMIF_BUFFERS_ALLOCATED(buffer_alloc)); + for (i = 0; i < rdev->usec_timeout; i++) { + if (RREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset) & + DMIF_BUFFERS_ALLOCATED_COMPLETED) + break; + udelay(1); + } + } + if (radeon_crtc->base.enabled && mode) { switch (tmp) { case 0: @@ -2036,7 +2181,8 @@ static void evergreen_program_watermarks(struct radeon_device *rdev, u32 lb_size, u32 num_heads) { struct drm_display_mode *mode = &radeon_crtc->base.mode; - struct evergreen_wm_params wm; + struct evergreen_wm_params wm_low, wm_high; + u32 dram_channels; u32 pixel_period; u32 line_time = 0; u32 latency_watermark_a = 0, latency_watermark_b = 0; @@ -2052,39 +2198,81 @@ static void evergreen_program_watermarks(struct radeon_device *rdev, line_time = min((u32)mode->crtc_htotal * pixel_period, (u32)65535); priority_a_cnt = 0; priority_b_cnt = 0; + dram_channels = evergreen_get_number_of_dram_channels(rdev); + + /* watermark for high clocks */ + if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) { + wm_high.yclk = + radeon_dpm_get_mclk(rdev, false) * 10; + wm_high.sclk = + radeon_dpm_get_sclk(rdev, false) * 10; + } else { + wm_high.yclk = rdev->pm.current_mclk * 10; + wm_high.sclk = rdev->pm.current_sclk * 10; + } - wm.yclk = rdev->pm.current_mclk * 10; - wm.sclk = rdev->pm.current_sclk * 10; - wm.disp_clk = mode->clock; - wm.src_width = mode->crtc_hdisplay; - wm.active_time = mode->crtc_hdisplay * pixel_period; - wm.blank_time = line_time - wm.active_time; - wm.interlaced = false; + wm_high.disp_clk = mode->clock; + wm_high.src_width = mode->crtc_hdisplay; + wm_high.active_time = mode->crtc_hdisplay * pixel_period; + wm_high.blank_time = line_time - wm_high.active_time; + wm_high.interlaced = false; if (mode->flags & DRM_MODE_FLAG_INTERLACE) - wm.interlaced = true; - wm.vsc = radeon_crtc->vsc; - wm.vtaps = 1; + wm_high.interlaced = true; + wm_high.vsc = radeon_crtc->vsc; + wm_high.vtaps = 1; if (radeon_crtc->rmx_type != RMX_OFF) - wm.vtaps = 2; - wm.bytes_per_pixel = 4; /* XXX: get this from fb config */ - wm.lb_size = lb_size; - wm.dram_channels = evergreen_get_number_of_dram_channels(rdev); - wm.num_heads = num_heads; + wm_high.vtaps = 2; + wm_high.bytes_per_pixel = 4; /* XXX: get this from fb config */ + wm_high.lb_size = lb_size; + wm_high.dram_channels = dram_channels; + wm_high.num_heads = num_heads; + + /* watermark for low clocks */ + if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) { + wm_low.yclk = + radeon_dpm_get_mclk(rdev, true) * 10; + wm_low.sclk = + radeon_dpm_get_sclk(rdev, true) * 10; + } else { + wm_low.yclk = rdev->pm.current_mclk * 10; + wm_low.sclk = rdev->pm.current_sclk * 10; + } + + wm_low.disp_clk = mode->clock; + wm_low.src_width = mode->crtc_hdisplay; + wm_low.active_time = mode->crtc_hdisplay * pixel_period; + wm_low.blank_time = line_time - wm_low.active_time; + wm_low.interlaced = false; + if (mode->flags & DRM_MODE_FLAG_INTERLACE) + wm_low.interlaced = true; + wm_low.vsc = radeon_crtc->vsc; + wm_low.vtaps = 1; + if (radeon_crtc->rmx_type != RMX_OFF) + wm_low.vtaps = 2; + wm_low.bytes_per_pixel = 4; /* XXX: get this from fb config */ + wm_low.lb_size = lb_size; + wm_low.dram_channels = dram_channels; + wm_low.num_heads = num_heads; /* set for high clocks */ - latency_watermark_a = min(evergreen_latency_watermark(&wm), (u32)65535); + latency_watermark_a = min(evergreen_latency_watermark(&wm_high), (u32)65535); /* set for low clocks */ - /* wm.yclk = low clk; wm.sclk = low clk */ - latency_watermark_b = min(evergreen_latency_watermark(&wm), (u32)65535); + latency_watermark_b = min(evergreen_latency_watermark(&wm_low), (u32)65535); /* possibly force display priority to high */ /* should really do this at mode validation time... */ - if (!evergreen_average_bandwidth_vs_dram_bandwidth_for_display(&wm) || - !evergreen_average_bandwidth_vs_available_bandwidth(&wm) || - !evergreen_check_latency_hiding(&wm) || + if (!evergreen_average_bandwidth_vs_dram_bandwidth_for_display(&wm_high) || + !evergreen_average_bandwidth_vs_available_bandwidth(&wm_high) || + !evergreen_check_latency_hiding(&wm_high) || (rdev->disp_priority == 2)) { - DRM_DEBUG_KMS("force priority to high\n"); + DRM_DEBUG_KMS("force priority a to high\n"); priority_a_cnt |= PRIORITY_ALWAYS_ON; + } + if (!evergreen_average_bandwidth_vs_dram_bandwidth_for_display(&wm_low) || + !evergreen_average_bandwidth_vs_available_bandwidth(&wm_low) || + !evergreen_check_latency_hiding(&wm_low) || + (rdev->disp_priority == 2)) { + DRM_DEBUG_KMS("force priority b to high\n"); priority_b_cnt |= PRIORITY_ALWAYS_ON; } @@ -2137,6 +2325,10 @@ static void evergreen_program_watermarks(struct radeon_device *rdev, WREG32(PRIORITY_A_CNT + radeon_crtc->crtc_offset, priority_a_cnt); WREG32(PRIORITY_B_CNT + radeon_crtc->crtc_offset, priority_b_cnt); + /* save values for DPM */ + radeon_crtc->line_time = line_time; + radeon_crtc->wm_high = latency_watermark_a; + radeon_crtc->wm_low = latency_watermark_b; } /** @@ -2450,8 +2642,9 @@ void evergreen_mc_resume(struct radeon_device *rdev, struct evergreen_mc_save *s for (i = 0; i < rdev->num_crtc; i++) { if (save->crtc_enabled[i]) { tmp = RREG32(EVERGREEN_MASTER_UPDATE_MODE + crtc_offsets[i]); - if ((tmp & 0x3) != 0) { - tmp &= ~0x3; + if ((tmp & 0x7) != 3) { + tmp &= ~0x7; + tmp |= 0x3; WREG32(EVERGREEN_MASTER_UPDATE_MODE + crtc_offsets[i], tmp); } tmp = RREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i]); @@ -2745,8 +2938,8 @@ static int evergreen_cp_resume(struct radeon_device *rdev) RREG32(GRBM_SOFT_RESET); /* Set ring buffer size */ - rb_bufsz = drm_order(ring->ring_size / 8); - tmp = (drm_order(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz; + rb_bufsz = order_base_2(ring->ring_size / 8); + tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz; #ifdef __BIG_ENDIAN tmp |= BUF_SWAP_32BIT; #endif @@ -2782,8 +2975,6 @@ static int evergreen_cp_resume(struct radeon_device *rdev) WREG32(CP_RB_BASE, ring->gpu_addr >> 8); WREG32(CP_DEBUG, (1 << 27) | (1 << 28)); - ring->rptr = RREG32(CP_RB_RPTR); - evergreen_cp_start(rdev); ring->ready = true; r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, ring); @@ -2973,7 +3164,7 @@ static void evergreen_gpu_init(struct radeon_device *rdev) rdev->config.evergreen.sx_max_export_size = 256; rdev->config.evergreen.sx_max_export_pos_size = 64; rdev->config.evergreen.sx_max_export_smx_size = 192; - rdev->config.evergreen.max_hw_contexts = 8; + rdev->config.evergreen.max_hw_contexts = 4; rdev->config.evergreen.sq_num_cf_insts = 2; rdev->config.evergreen.sc_prim_fifo_size = 0x40; @@ -3120,10 +3311,8 @@ static void evergreen_gpu_init(struct radeon_device *rdev) u32 efuse_straps_4; u32 efuse_straps_3; - WREG32(RCU_IND_INDEX, 0x204); - efuse_straps_4 = RREG32(RCU_IND_DATA); - WREG32(RCU_IND_INDEX, 0x203); - efuse_straps_3 = RREG32(RCU_IND_DATA); + efuse_straps_4 = RREG32_RCU(0x204); + efuse_straps_3 = RREG32_RCU(0x203); tmp = (((efuse_straps_4 & 0xf) << 4) | ((efuse_straps_3 & 0xf0000000) >> 28)); } else { @@ -3149,6 +3338,18 @@ static void evergreen_gpu_init(struct radeon_device *rdev) disabled_rb_mask &= ~(1 << i); } + for (i = 0; i < rdev->config.evergreen.num_ses; i++) { + u32 simd_disable_bitmap; + + WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i)); + WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i)); + simd_disable_bitmap = (RREG32(CC_GC_SHADER_PIPE_CONFIG) & 0xffff0000) >> 16; + simd_disable_bitmap |= 0xffffffff << rdev->config.evergreen.max_simds; + tmp <<= 16; + tmp |= simd_disable_bitmap; + } + rdev->config.evergreen.active_simds = hweight32(~tmp); + WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_BROADCAST_WRITES); WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_BROADCAST_WRITES); @@ -3479,7 +3680,7 @@ bool evergreen_is_display_hung(struct radeon_device *rdev) return true; } -static u32 evergreen_gpu_check_soft_reset(struct radeon_device *rdev) +u32 evergreen_gpu_check_soft_reset(struct radeon_device *rdev) { u32 reset_mask = 0; u32 tmp; @@ -3662,6 +3863,48 @@ static void evergreen_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask) evergreen_print_gpu_status_regs(rdev); } +void evergreen_gpu_pci_config_reset(struct radeon_device *rdev) +{ + struct evergreen_mc_save save; + u32 tmp, i; + + dev_info(rdev->dev, "GPU pci config reset\n"); + + /* disable dpm? */ + + /* Disable CP parsing/prefetching */ + WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT); + udelay(50); + /* Disable DMA */ + tmp = RREG32(DMA_RB_CNTL); + tmp &= ~DMA_RB_ENABLE; + WREG32(DMA_RB_CNTL, tmp); + /* XXX other engines? */ + + /* halt the rlc */ + r600_rlc_stop(rdev); + + udelay(50); + + /* set mclk/sclk to bypass */ + rv770_set_clk_bypass_mode(rdev); + /* disable BM */ + pci_clear_master(rdev->pdev); + /* disable mem access */ + evergreen_mc_stop(rdev, &save); + if (evergreen_mc_wait_for_idle(rdev)) { + dev_warn(rdev->dev, "Wait for MC idle timed out !\n"); + } + /* reset */ + radeon_pci_config_reset(rdev); + /* wait for asic to come out of reset */ + for (i = 0; i < rdev->usec_timeout; i++) { + if (RREG32(CONFIG_MEMSIZE) != 0xffffffff) + break; + udelay(1); + } +} + int evergreen_asic_reset(struct radeon_device *rdev) { u32 reset_mask; @@ -3671,10 +3914,17 @@ int evergreen_asic_reset(struct radeon_device *rdev) if (reset_mask) r600_set_bios_scratch_engine_hung(rdev, true); + /* try soft reset */ evergreen_gpu_soft_reset(rdev, reset_mask); reset_mask = evergreen_gpu_check_soft_reset(rdev); + /* try pci config reset */ + if (reset_mask && radeon_hard_reset) + evergreen_gpu_pci_config_reset(rdev); + + reset_mask = evergreen_gpu_check_soft_reset(rdev); + if (!reset_mask) r600_set_bios_scratch_engine_hung(rdev, false); @@ -3697,34 +3947,350 @@ bool evergreen_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *rin if (!(reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE | RADEON_RESET_CP))) { - radeon_ring_lockup_update(ring); + radeon_ring_lockup_update(rdev, ring); return false; } - /* force CP activities */ - radeon_ring_force_activity(rdev, ring); return radeon_ring_test_lockup(rdev, ring); } -/** - * evergreen_dma_is_lockup - Check if the DMA engine is locked up - * - * @rdev: radeon_device pointer - * @ring: radeon_ring structure holding ring information - * - * Check if the async DMA engine is locked up. - * Returns true if the engine appears to be locked up, false if not. +/* + * RLC */ -bool evergreen_dma_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring) +#define RLC_SAVE_RESTORE_LIST_END_MARKER 0x00000000 +#define RLC_CLEAR_STATE_END_MARKER 0x00000001 + +void sumo_rlc_fini(struct radeon_device *rdev) { - u32 reset_mask = evergreen_gpu_check_soft_reset(rdev); + int r; - if (!(reset_mask & RADEON_RESET_DMA)) { - radeon_ring_lockup_update(ring); - return false; + /* save restore block */ + if (rdev->rlc.save_restore_obj) { + r = radeon_bo_reserve(rdev->rlc.save_restore_obj, false); + if (unlikely(r != 0)) + dev_warn(rdev->dev, "(%d) reserve RLC sr bo failed\n", r); + radeon_bo_unpin(rdev->rlc.save_restore_obj); + radeon_bo_unreserve(rdev->rlc.save_restore_obj); + + radeon_bo_unref(&rdev->rlc.save_restore_obj); + rdev->rlc.save_restore_obj = NULL; + } + + /* clear state block */ + if (rdev->rlc.clear_state_obj) { + r = radeon_bo_reserve(rdev->rlc.clear_state_obj, false); + if (unlikely(r != 0)) + dev_warn(rdev->dev, "(%d) reserve RLC c bo failed\n", r); + radeon_bo_unpin(rdev->rlc.clear_state_obj); + radeon_bo_unreserve(rdev->rlc.clear_state_obj); + + radeon_bo_unref(&rdev->rlc.clear_state_obj); + rdev->rlc.clear_state_obj = NULL; + } + + /* clear state block */ + if (rdev->rlc.cp_table_obj) { + r = radeon_bo_reserve(rdev->rlc.cp_table_obj, false); + if (unlikely(r != 0)) + dev_warn(rdev->dev, "(%d) reserve RLC cp table bo failed\n", r); + radeon_bo_unpin(rdev->rlc.cp_table_obj); + radeon_bo_unreserve(rdev->rlc.cp_table_obj); + + radeon_bo_unref(&rdev->rlc.cp_table_obj); + rdev->rlc.cp_table_obj = NULL; } - /* force ring activities */ - radeon_ring_force_activity(rdev, ring); - return radeon_ring_test_lockup(rdev, ring); +} + +#define CP_ME_TABLE_SIZE 96 + +int sumo_rlc_init(struct radeon_device *rdev) +{ + const u32 *src_ptr; + volatile u32 *dst_ptr; + u32 dws, data, i, j, k, reg_num; + u32 reg_list_num, reg_list_hdr_blk_index, reg_list_blk_index = 0; + u64 reg_list_mc_addr; + const struct cs_section_def *cs_data; + int r; + + src_ptr = rdev->rlc.reg_list; + dws = rdev->rlc.reg_list_size; + if (rdev->family >= CHIP_BONAIRE) { + dws += (5 * 16) + 48 + 48 + 64; + } + cs_data = rdev->rlc.cs_data; + + if (src_ptr) { + /* save restore block */ + if (rdev->rlc.save_restore_obj == NULL) { + r = radeon_bo_create(rdev, dws * 4, PAGE_SIZE, true, + RADEON_GEM_DOMAIN_VRAM, NULL, &rdev->rlc.save_restore_obj); + if (r) { + dev_warn(rdev->dev, "(%d) create RLC sr bo failed\n", r); + return r; + } + } + + r = radeon_bo_reserve(rdev->rlc.save_restore_obj, false); + if (unlikely(r != 0)) { + sumo_rlc_fini(rdev); + return r; + } + r = radeon_bo_pin(rdev->rlc.save_restore_obj, RADEON_GEM_DOMAIN_VRAM, + &rdev->rlc.save_restore_gpu_addr); + if (r) { + radeon_bo_unreserve(rdev->rlc.save_restore_obj); + dev_warn(rdev->dev, "(%d) pin RLC sr bo failed\n", r); + sumo_rlc_fini(rdev); + return r; + } + + r = radeon_bo_kmap(rdev->rlc.save_restore_obj, (void **)&rdev->rlc.sr_ptr); + if (r) { + dev_warn(rdev->dev, "(%d) map RLC sr bo failed\n", r); + sumo_rlc_fini(rdev); + return r; + } + /* write the sr buffer */ + dst_ptr = rdev->rlc.sr_ptr; + if (rdev->family >= CHIP_TAHITI) { + /* SI */ + for (i = 0; i < rdev->rlc.reg_list_size; i++) + dst_ptr[i] = cpu_to_le32(src_ptr[i]); + } else { + /* ON/LN/TN */ + /* format: + * dw0: (reg2 << 16) | reg1 + * dw1: reg1 save space + * dw2: reg2 save space + */ + for (i = 0; i < dws; i++) { + data = src_ptr[i] >> 2; + i++; + if (i < dws) + data |= (src_ptr[i] >> 2) << 16; + j = (((i - 1) * 3) / 2); + dst_ptr[j] = cpu_to_le32(data); + } + j = ((i * 3) / 2); + dst_ptr[j] = cpu_to_le32(RLC_SAVE_RESTORE_LIST_END_MARKER); + } + radeon_bo_kunmap(rdev->rlc.save_restore_obj); + radeon_bo_unreserve(rdev->rlc.save_restore_obj); + } + + if (cs_data) { + /* clear state block */ + if (rdev->family >= CHIP_BONAIRE) { + rdev->rlc.clear_state_size = dws = cik_get_csb_size(rdev); + } else if (rdev->family >= CHIP_TAHITI) { + rdev->rlc.clear_state_size = si_get_csb_size(rdev); + dws = rdev->rlc.clear_state_size + (256 / 4); + } else { + reg_list_num = 0; + dws = 0; + for (i = 0; cs_data[i].section != NULL; i++) { + for (j = 0; cs_data[i].section[j].extent != NULL; j++) { + reg_list_num++; + dws += cs_data[i].section[j].reg_count; + } + } + reg_list_blk_index = (3 * reg_list_num + 2); + dws += reg_list_blk_index; + rdev->rlc.clear_state_size = dws; + } + + if (rdev->rlc.clear_state_obj == NULL) { + r = radeon_bo_create(rdev, dws * 4, PAGE_SIZE, true, + RADEON_GEM_DOMAIN_VRAM, NULL, &rdev->rlc.clear_state_obj); + if (r) { + dev_warn(rdev->dev, "(%d) create RLC c bo failed\n", r); + sumo_rlc_fini(rdev); + return r; + } + } + r = radeon_bo_reserve(rdev->rlc.clear_state_obj, false); + if (unlikely(r != 0)) { + sumo_rlc_fini(rdev); + return r; + } + r = radeon_bo_pin(rdev->rlc.clear_state_obj, RADEON_GEM_DOMAIN_VRAM, + &rdev->rlc.clear_state_gpu_addr); + if (r) { + radeon_bo_unreserve(rdev->rlc.clear_state_obj); + dev_warn(rdev->dev, "(%d) pin RLC c bo failed\n", r); + sumo_rlc_fini(rdev); + return r; + } + + r = radeon_bo_kmap(rdev->rlc.clear_state_obj, (void **)&rdev->rlc.cs_ptr); + if (r) { + dev_warn(rdev->dev, "(%d) map RLC c bo failed\n", r); + sumo_rlc_fini(rdev); + return r; + } + /* set up the cs buffer */ + dst_ptr = rdev->rlc.cs_ptr; + if (rdev->family >= CHIP_BONAIRE) { + cik_get_csb_buffer(rdev, dst_ptr); + } else if (rdev->family >= CHIP_TAHITI) { + reg_list_mc_addr = rdev->rlc.clear_state_gpu_addr + 256; + dst_ptr[0] = cpu_to_le32(upper_32_bits(reg_list_mc_addr)); + dst_ptr[1] = cpu_to_le32(lower_32_bits(reg_list_mc_addr)); + dst_ptr[2] = cpu_to_le32(rdev->rlc.clear_state_size); + si_get_csb_buffer(rdev, &dst_ptr[(256/4)]); + } else { + reg_list_hdr_blk_index = 0; + reg_list_mc_addr = rdev->rlc.clear_state_gpu_addr + (reg_list_blk_index * 4); + data = upper_32_bits(reg_list_mc_addr); + dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(data); + reg_list_hdr_blk_index++; + for (i = 0; cs_data[i].section != NULL; i++) { + for (j = 0; cs_data[i].section[j].extent != NULL; j++) { + reg_num = cs_data[i].section[j].reg_count; + data = reg_list_mc_addr & 0xffffffff; + dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(data); + reg_list_hdr_blk_index++; + + data = (cs_data[i].section[j].reg_index * 4) & 0xffffffff; + dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(data); + reg_list_hdr_blk_index++; + + data = 0x08000000 | (reg_num * 4); + dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(data); + reg_list_hdr_blk_index++; + + for (k = 0; k < reg_num; k++) { + data = cs_data[i].section[j].extent[k]; + dst_ptr[reg_list_blk_index + k] = cpu_to_le32(data); + } + reg_list_mc_addr += reg_num * 4; + reg_list_blk_index += reg_num; + } + } + dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(RLC_CLEAR_STATE_END_MARKER); + } + radeon_bo_kunmap(rdev->rlc.clear_state_obj); + radeon_bo_unreserve(rdev->rlc.clear_state_obj); + } + + if (rdev->rlc.cp_table_size) { + if (rdev->rlc.cp_table_obj == NULL) { + r = radeon_bo_create(rdev, rdev->rlc.cp_table_size, PAGE_SIZE, true, + RADEON_GEM_DOMAIN_VRAM, NULL, &rdev->rlc.cp_table_obj); + if (r) { + dev_warn(rdev->dev, "(%d) create RLC cp table bo failed\n", r); + sumo_rlc_fini(rdev); + return r; + } + } + + r = radeon_bo_reserve(rdev->rlc.cp_table_obj, false); + if (unlikely(r != 0)) { + dev_warn(rdev->dev, "(%d) reserve RLC cp table bo failed\n", r); + sumo_rlc_fini(rdev); + return r; + } + r = radeon_bo_pin(rdev->rlc.cp_table_obj, RADEON_GEM_DOMAIN_VRAM, + &rdev->rlc.cp_table_gpu_addr); + if (r) { + radeon_bo_unreserve(rdev->rlc.cp_table_obj); + dev_warn(rdev->dev, "(%d) pin RLC cp_table bo failed\n", r); + sumo_rlc_fini(rdev); + return r; + } + r = radeon_bo_kmap(rdev->rlc.cp_table_obj, (void **)&rdev->rlc.cp_table_ptr); + if (r) { + dev_warn(rdev->dev, "(%d) map RLC cp table bo failed\n", r); + sumo_rlc_fini(rdev); + return r; + } + + cik_init_cp_pg_table(rdev); + + radeon_bo_kunmap(rdev->rlc.cp_table_obj); + radeon_bo_unreserve(rdev->rlc.cp_table_obj); + + } + + return 0; +} + +static void evergreen_rlc_start(struct radeon_device *rdev) +{ + u32 mask = RLC_ENABLE; + + if (rdev->flags & RADEON_IS_IGP) { + mask |= GFX_POWER_GATING_ENABLE | GFX_POWER_GATING_SRC; + } + + WREG32(RLC_CNTL, mask); +} + +int evergreen_rlc_resume(struct radeon_device *rdev) +{ + u32 i; + const __be32 *fw_data; + + if (!rdev->rlc_fw) + return -EINVAL; + + r600_rlc_stop(rdev); + + WREG32(RLC_HB_CNTL, 0); + + if (rdev->flags & RADEON_IS_IGP) { + if (rdev->family == CHIP_ARUBA) { + u32 always_on_bitmap = + 3 | (3 << (16 * rdev->config.cayman.max_shader_engines)); + /* find out the number of active simds */ + u32 tmp = (RREG32(CC_GC_SHADER_PIPE_CONFIG) & 0xffff0000) >> 16; + tmp |= 0xffffffff << rdev->config.cayman.max_simds_per_se; + tmp = hweight32(~tmp); + if (tmp == rdev->config.cayman.max_simds_per_se) { + WREG32(TN_RLC_LB_ALWAYS_ACTIVE_SIMD_MASK, always_on_bitmap); + WREG32(TN_RLC_LB_PARAMS, 0x00601004); + WREG32(TN_RLC_LB_INIT_SIMD_MASK, 0xffffffff); + WREG32(TN_RLC_LB_CNTR_INIT, 0x00000000); + WREG32(TN_RLC_LB_CNTR_MAX, 0x00002000); + } + } else { + WREG32(RLC_HB_WPTR_LSB_ADDR, 0); + WREG32(RLC_HB_WPTR_MSB_ADDR, 0); + } + WREG32(TN_RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8); + WREG32(TN_RLC_CLEAR_STATE_RESTORE_BASE, rdev->rlc.clear_state_gpu_addr >> 8); + } else { + WREG32(RLC_HB_BASE, 0); + WREG32(RLC_HB_RPTR, 0); + WREG32(RLC_HB_WPTR, 0); + WREG32(RLC_HB_WPTR_LSB_ADDR, 0); + WREG32(RLC_HB_WPTR_MSB_ADDR, 0); + } + WREG32(RLC_MC_CNTL, 0); + WREG32(RLC_UCODE_CNTL, 0); + + fw_data = (const __be32 *)rdev->rlc_fw->data; + if (rdev->family >= CHIP_ARUBA) { + for (i = 0; i < ARUBA_RLC_UCODE_SIZE; i++) { + WREG32(RLC_UCODE_ADDR, i); + WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++)); + } + } else if (rdev->family >= CHIP_CAYMAN) { + for (i = 0; i < CAYMAN_RLC_UCODE_SIZE; i++) { + WREG32(RLC_UCODE_ADDR, i); + WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++)); + } + } else { + for (i = 0; i < EVERGREEN_RLC_UCODE_SIZE; i++) { + WREG32(RLC_UCODE_ADDR, i); + WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++)); + } + } + WREG32(RLC_UCODE_ADDR, 0); + + evergreen_rlc_start(rdev); + + return 0; } /* Interrupts */ @@ -3775,8 +4341,8 @@ void evergreen_disable_interrupt_state(struct radeon_device *rdev) WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, 0); } - /* only one DAC on DCE6 */ - if (!ASIC_IS_DCE6(rdev)) + /* only one DAC on DCE5 */ + if (!ASIC_IS_DCE5(rdev)) WREG32(DACA_AUTODETECT_INT_CONTROL, 0); WREG32(DACB_AUTODETECT_INT_CONTROL, 0); @@ -3802,9 +4368,9 @@ int evergreen_irq_set(struct radeon_device *rdev) u32 crtc1 = 0, crtc2 = 0, crtc3 = 0, crtc4 = 0, crtc5 = 0, crtc6 = 0; u32 hpd1, hpd2, hpd3, hpd4, hpd5, hpd6; u32 grbm_int_cntl = 0; - u32 grph1 = 0, grph2 = 0, grph3 = 0, grph4 = 0, grph5 = 0, grph6 = 0; u32 afmt1 = 0, afmt2 = 0, afmt3 = 0, afmt4 = 0, afmt5 = 0, afmt6 = 0; u32 dma_cntl, dma_cntl1 = 0; + u32 thermal_int = 0; if (!rdev->irq.installed) { WARN(1, "Can't enable IRQ/MSI because no handler is installed\n"); @@ -3824,6 +4390,12 @@ int evergreen_irq_set(struct radeon_device *rdev) hpd4 = RREG32(DC_HPD4_INT_CONTROL) & ~DC_HPDx_INT_EN; hpd5 = RREG32(DC_HPD5_INT_CONTROL) & ~DC_HPDx_INT_EN; hpd6 = RREG32(DC_HPD6_INT_CONTROL) & ~DC_HPDx_INT_EN; + if (rdev->family == CHIP_ARUBA) + thermal_int = RREG32(TN_CG_THERMAL_INT_CTRL) & + ~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW); + else + thermal_int = RREG32(CG_THERMAL_INT) & + ~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW); afmt1 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK; afmt2 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK; @@ -3869,6 +4441,11 @@ int evergreen_irq_set(struct radeon_device *rdev) } } + if (rdev->irq.dpm_thermal) { + DRM_DEBUG("dpm thermal\n"); + thermal_int |= THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW; + } + if (rdev->irq.crtc_vblank_int[0] || atomic_read(&rdev->irq.pflip[0])) { DRM_DEBUG("evergreen_irq_set: vblank 0\n"); @@ -3973,15 +4550,21 @@ int evergreen_irq_set(struct radeon_device *rdev) WREG32(INT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, crtc6); } - WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, grph1); - WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, grph2); + WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, + GRPH_PFLIP_INT_MASK); + WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, + GRPH_PFLIP_INT_MASK); if (rdev->num_crtc >= 4) { - WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, grph3); - WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, grph4); + WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, + GRPH_PFLIP_INT_MASK); + WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, + GRPH_PFLIP_INT_MASK); } if (rdev->num_crtc >= 6) { - WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, grph5); - WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, grph6); + WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, + GRPH_PFLIP_INT_MASK); + WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, + GRPH_PFLIP_INT_MASK); } WREG32(DC_HPD1_INT_CONTROL, hpd1); @@ -3990,6 +4573,10 @@ int evergreen_irq_set(struct radeon_device *rdev) WREG32(DC_HPD4_INT_CONTROL, hpd4); WREG32(DC_HPD5_INT_CONTROL, hpd5); WREG32(DC_HPD6_INT_CONTROL, hpd6); + if (rdev->family == CHIP_ARUBA) + WREG32(TN_CG_THERMAL_INT_CTRL, thermal_int); + else + WREG32(CG_THERMAL_INT, thermal_int); WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, afmt1); WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, afmt2); @@ -4169,6 +4756,7 @@ static u32 evergreen_get_ih_wptr(struct radeon_device *rdev) tmp = RREG32(IH_RB_CNTL); tmp |= IH_WPTR_OVERFLOW_CLEAR; WREG32(IH_RB_CNTL, tmp); + wptr &= ~RB_OVERFLOW; } return (wptr & rdev->ih.ptr_mask); } @@ -4181,6 +4769,8 @@ int evergreen_irq_process(struct radeon_device *rdev) u32 ring_index; bool queue_hotplug = false; bool queue_hdmi = false; + bool queue_thermal = false; + u32 status, addr; if (!rdev->ih.enabled || rdev->shutdown) return IRQ_NONE; @@ -4218,7 +4808,7 @@ restart_ih: wake_up(&rdev->irq.vblank_queue); } if (atomic_read(&rdev->irq.pflip[0])) - radeon_crtc_handle_flip(rdev, 0); + radeon_crtc_handle_vblank(rdev, 0); rdev->irq.stat_regs.evergreen.disp_int &= ~LB_D1_VBLANK_INTERRUPT; DRM_DEBUG("IH: D1 vblank\n"); } @@ -4244,7 +4834,7 @@ restart_ih: wake_up(&rdev->irq.vblank_queue); } if (atomic_read(&rdev->irq.pflip[1])) - radeon_crtc_handle_flip(rdev, 1); + radeon_crtc_handle_vblank(rdev, 1); rdev->irq.stat_regs.evergreen.disp_int_cont &= ~LB_D2_VBLANK_INTERRUPT; DRM_DEBUG("IH: D2 vblank\n"); } @@ -4270,7 +4860,7 @@ restart_ih: wake_up(&rdev->irq.vblank_queue); } if (atomic_read(&rdev->irq.pflip[2])) - radeon_crtc_handle_flip(rdev, 2); + radeon_crtc_handle_vblank(rdev, 2); rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~LB_D3_VBLANK_INTERRUPT; DRM_DEBUG("IH: D3 vblank\n"); } @@ -4296,7 +4886,7 @@ restart_ih: wake_up(&rdev->irq.vblank_queue); } if (atomic_read(&rdev->irq.pflip[3])) - radeon_crtc_handle_flip(rdev, 3); + radeon_crtc_handle_vblank(rdev, 3); rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~LB_D4_VBLANK_INTERRUPT; DRM_DEBUG("IH: D4 vblank\n"); } @@ -4322,7 +4912,7 @@ restart_ih: wake_up(&rdev->irq.vblank_queue); } if (atomic_read(&rdev->irq.pflip[4])) - radeon_crtc_handle_flip(rdev, 4); + radeon_crtc_handle_vblank(rdev, 4); rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~LB_D5_VBLANK_INTERRUPT; DRM_DEBUG("IH: D5 vblank\n"); } @@ -4348,7 +4938,7 @@ restart_ih: wake_up(&rdev->irq.vblank_queue); } if (atomic_read(&rdev->irq.pflip[5])) - radeon_crtc_handle_flip(rdev, 5); + radeon_crtc_handle_vblank(rdev, 5); rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~LB_D6_VBLANK_INTERRUPT; DRM_DEBUG("IH: D6 vblank\n"); } @@ -4364,6 +4954,15 @@ restart_ih: break; } break; + case 8: /* D1 page flip */ + case 10: /* D2 page flip */ + case 12: /* D3 page flip */ + case 14: /* D4 page flip */ + case 16: /* D5 page flip */ + case 18: /* D6 page flip */ + DRM_DEBUG("IH: D%d flip\n", ((src_id - 8) >> 1) + 1); + radeon_crtc_handle_flip(rdev, (src_id - 8) >> 1); + break; case 42: /* HPD hotplug */ switch (src_data) { case 0: @@ -4467,13 +5066,18 @@ restart_ih: break; case 146: case 147: + addr = RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR); + status = RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS); + /* reset addr and status */ + WREG32_P(VM_CONTEXT1_CNTL2, 1, ~1); + if (addr == 0x0 && status == 0x0) + break; dev_err(rdev->dev, "GPU fault detected: %d 0x%08x\n", src_id, src_data); dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n", - RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR)); + addr); dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n", - RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS)); - /* reset addr and status */ - WREG32_P(VM_CONTEXT1_CNTL2, 1, ~1); + status); + cayman_vm_decode_fault(rdev, status, addr); break; case 176: /* CP_INT in ring buffer */ case 177: /* CP_INT in IB1 */ @@ -4502,6 +5106,16 @@ restart_ih: DRM_DEBUG("IH: DMA trap\n"); radeon_fence_process(rdev, R600_RING_TYPE_DMA_INDEX); break; + case 230: /* thermal low to high */ + DRM_DEBUG("IH: thermal low to high\n"); + rdev->pm.dpm.thermal.high_to_low = false; + queue_thermal = true; + break; + case 231: /* thermal high to low */ + DRM_DEBUG("IH: thermal high to low\n"); + rdev->pm.dpm.thermal.high_to_low = true; + queue_thermal = true; + break; case 233: /* GUI IDLE */ DRM_DEBUG("IH: GUI idle\n"); break; @@ -4524,6 +5138,8 @@ restart_ih: schedule_work(&rdev->hotplug_work); if (queue_hdmi) schedule_work(&rdev->audio_work); + if (queue_thermal && rdev->pm.dpm_enabled) + schedule_work(&rdev->pm.dpm.thermal.work); rdev->ih.rptr = rptr; WREG32(IH_RB_RPTR, rdev->ih.rptr); atomic_set(&rdev->ih.lock, 0); @@ -4536,143 +5152,6 @@ restart_ih: return IRQ_HANDLED; } -/** - * evergreen_dma_fence_ring_emit - emit a fence on the DMA ring - * - * @rdev: radeon_device pointer - * @fence: radeon fence object - * - * Add a DMA fence packet to the ring to write - * the fence seq number and DMA trap packet to generate - * an interrupt if needed (evergreen-SI). - */ -void evergreen_dma_fence_ring_emit(struct radeon_device *rdev, - struct radeon_fence *fence) -{ - struct radeon_ring *ring = &rdev->ring[fence->ring]; - u64 addr = rdev->fence_drv[fence->ring].gpu_addr; - /* write the fence */ - radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_FENCE, 0, 0)); - radeon_ring_write(ring, addr & 0xfffffffc); - radeon_ring_write(ring, (upper_32_bits(addr) & 0xff)); - radeon_ring_write(ring, fence->seq); - /* generate an interrupt */ - radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_TRAP, 0, 0)); - /* flush HDP */ - radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_SRBM_WRITE, 0, 0)); - radeon_ring_write(ring, (0xf << 16) | (HDP_MEM_COHERENCY_FLUSH_CNTL >> 2)); - radeon_ring_write(ring, 1); -} - -/** - * evergreen_dma_ring_ib_execute - schedule an IB on the DMA engine - * - * @rdev: radeon_device pointer - * @ib: IB object to schedule - * - * Schedule an IB in the DMA ring (evergreen). - */ -void evergreen_dma_ring_ib_execute(struct radeon_device *rdev, - struct radeon_ib *ib) -{ - struct radeon_ring *ring = &rdev->ring[ib->ring]; - - if (rdev->wb.enabled) { - u32 next_rptr = ring->wptr + 4; - while ((next_rptr & 7) != 5) - next_rptr++; - next_rptr += 3; - radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_WRITE, 0, 1)); - radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc); - radeon_ring_write(ring, upper_32_bits(ring->next_rptr_gpu_addr) & 0xff); - radeon_ring_write(ring, next_rptr); - } - - /* The indirect buffer packet must end on an 8 DW boundary in the DMA ring. - * Pad as necessary with NOPs. - */ - while ((ring->wptr & 7) != 5) - radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_NOP, 0, 0)); - radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_INDIRECT_BUFFER, 0, 0)); - radeon_ring_write(ring, (ib->gpu_addr & 0xFFFFFFE0)); - radeon_ring_write(ring, (ib->length_dw << 12) | (upper_32_bits(ib->gpu_addr) & 0xFF)); - -} - -/** - * evergreen_copy_dma - copy pages using the DMA engine - * - * @rdev: radeon_device pointer - * @src_offset: src GPU address - * @dst_offset: dst GPU address - * @num_gpu_pages: number of GPU pages to xfer - * @fence: radeon fence object - * - * Copy GPU paging using the DMA engine (evergreen-cayman). - * Used by the radeon ttm implementation to move pages if - * registered as the asic copy callback. - */ -int evergreen_copy_dma(struct radeon_device *rdev, - uint64_t src_offset, uint64_t dst_offset, - unsigned num_gpu_pages, - struct radeon_fence **fence) -{ - struct radeon_semaphore *sem = NULL; - int ring_index = rdev->asic->copy.dma_ring_index; - struct radeon_ring *ring = &rdev->ring[ring_index]; - u32 size_in_dw, cur_size_in_dw; - int i, num_loops; - int r = 0; - - r = radeon_semaphore_create(rdev, &sem); - if (r) { - DRM_ERROR("radeon: moving bo (%d).\n", r); - return r; - } - - size_in_dw = (num_gpu_pages << RADEON_GPU_PAGE_SHIFT) / 4; - num_loops = DIV_ROUND_UP(size_in_dw, 0xfffff); - r = radeon_ring_lock(rdev, ring, num_loops * 5 + 11); - if (r) { - DRM_ERROR("radeon: moving bo (%d).\n", r); - radeon_semaphore_free(rdev, &sem, NULL); - return r; - } - - if (radeon_fence_need_sync(*fence, ring->idx)) { - radeon_semaphore_sync_rings(rdev, sem, (*fence)->ring, - ring->idx); - radeon_fence_note_sync(*fence, ring->idx); - } else { - radeon_semaphore_free(rdev, &sem, NULL); - } - - for (i = 0; i < num_loops; i++) { - cur_size_in_dw = size_in_dw; - if (cur_size_in_dw > 0xFFFFF) - cur_size_in_dw = 0xFFFFF; - size_in_dw -= cur_size_in_dw; - radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_COPY, 0, cur_size_in_dw)); - radeon_ring_write(ring, dst_offset & 0xfffffffc); - radeon_ring_write(ring, src_offset & 0xfffffffc); - radeon_ring_write(ring, upper_32_bits(dst_offset) & 0xff); - radeon_ring_write(ring, upper_32_bits(src_offset) & 0xff); - src_offset += cur_size_in_dw * 4; - dst_offset += cur_size_in_dw * 4; - } - - r = radeon_fence_emit(rdev, fence, ring->idx); - if (r) { - radeon_ring_unlock_undo(rdev, ring); - return r; - } - - radeon_ring_unlock_commit(rdev, ring); - radeon_semaphore_free(rdev, &sem, *fence); - - return r; -} - static int evergreen_startup(struct radeon_device *rdev) { struct radeon_ring *ring; @@ -4680,35 +5159,24 @@ static int evergreen_startup(struct radeon_device *rdev) /* enable pcie gen2 link */ evergreen_pcie_gen2_enable(rdev); + /* enable aspm */ + evergreen_program_aspm(rdev); - if (ASIC_IS_DCE5(rdev)) { - if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw || !rdev->mc_fw) { - r = ni_init_microcode(rdev); - if (r) { - DRM_ERROR("Failed to load firmware!\n"); - return r; - } - } + /* scratch needs to be initialized before MC */ + r = r600_vram_scratch_init(rdev); + if (r) + return r; + + evergreen_mc_program(rdev); + + if (ASIC_IS_DCE5(rdev) && !rdev->pm.dpm_enabled) { r = ni_mc_load_microcode(rdev); if (r) { DRM_ERROR("Failed to load MC firmware!\n"); return r; } - } else { - if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw) { - r = r600_init_microcode(rdev); - if (r) { - DRM_ERROR("Failed to load firmware!\n"); - return r; - } - } } - r = r600_vram_scratch_init(rdev); - if (r) - return r; - - evergreen_mc_program(rdev); if (rdev->flags & RADEON_IS_AGP) { evergreen_agp_enable(rdev); } else { @@ -4718,11 +5186,17 @@ static int evergreen_startup(struct radeon_device *rdev) } evergreen_gpu_init(rdev); - r = evergreen_blit_init(rdev); - if (r) { - r600_blit_fini(rdev); - rdev->asic->copy.copy = NULL; - dev_warn(rdev->dev, "failed blitter (%d) falling back to memcpy\n", r); + /* allocate rlc buffers */ + if (rdev->flags & RADEON_IS_IGP) { + rdev->rlc.reg_list = sumo_rlc_save_restore_register_list; + rdev->rlc.reg_list_size = + (u32)ARRAY_SIZE(sumo_rlc_save_restore_register_list); + rdev->rlc.cs_data = evergreen_cs_data; + r = sumo_rlc_init(rdev); + if (r) { + DRM_ERROR("Failed to init rlc BOs!\n"); + return r; + } } /* allocate wb buffer */ @@ -4742,7 +5216,7 @@ static int evergreen_startup(struct radeon_device *rdev) return r; } - r = rv770_uvd_resume(rdev); + r = uvd_v2_2_resume(rdev); if (!r) { r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_UVD_INDEX); @@ -4770,15 +5244,13 @@ static int evergreen_startup(struct radeon_device *rdev) ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]; r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET, - R600_CP_RB_RPTR, R600_CP_RB_WPTR, - 0, 0xfffff, RADEON_CP_PACKET2); + RADEON_CP_PACKET2); if (r) return r; ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX]; r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET, - DMA_RB_RPTR, DMA_RB_WPTR, - 2, 0x3fffc, DMA_PACKET(DMA_PACKET_NOP, 0, 0)); + DMA_PACKET(DMA_PACKET_NOP, 0, 0)); if (r) return r; @@ -4794,12 +5266,10 @@ static int evergreen_startup(struct radeon_device *rdev) ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX]; if (ring->ring_size) { - r = radeon_ring_init(rdev, ring, ring->ring_size, - R600_WB_UVD_RPTR_OFFSET, - UVD_RBC_RB_RPTR, UVD_RBC_RB_WPTR, - 0, 0xfffff, RADEON_CP_PACKET2); + r = radeon_ring_init(rdev, ring, ring->ring_size, 0, + RADEON_CP_PACKET2); if (!r) - r = r600_uvd_init(rdev); + r = uvd_v1_0_init(rdev); if (r) DRM_ERROR("radeon: error initializing UVD (%d).\n", r); @@ -4839,6 +5309,9 @@ int evergreen_resume(struct radeon_device *rdev) /* init golden registers */ evergreen_init_golden_registers(rdev); + if (rdev->pm.pm_method == PM_METHOD_DPM) + radeon_pm_resume(rdev); + rdev->accel_working = true; r = evergreen_startup(rdev); if (r) { @@ -4853,11 +5326,12 @@ int evergreen_resume(struct radeon_device *rdev) int evergreen_suspend(struct radeon_device *rdev) { + radeon_pm_suspend(rdev); r600_audio_fini(rdev); + uvd_v1_0_fini(rdev); radeon_uvd_suspend(rdev); r700_cp_stop(rdev); r600_dma_stop(rdev); - r600_uvd_rbc_stop(rdev); evergreen_irq_suspend(rdev); radeon_wb_disable(rdev); evergreen_pcie_gart_disable(rdev); @@ -4929,6 +5403,27 @@ int evergreen_init(struct radeon_device *rdev) if (r) return r; + if (ASIC_IS_DCE5(rdev)) { + if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw || !rdev->mc_fw) { + r = ni_init_microcode(rdev); + if (r) { + DRM_ERROR("Failed to load firmware!\n"); + return r; + } + } + } else { + if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw) { + r = r600_init_microcode(rdev); + if (r) { + DRM_ERROR("Failed to load firmware!\n"); + return r; + } + } + } + + /* Initialize power management */ + radeon_pm_init(rdev); + rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ring_obj = NULL; r600_ring_init(rdev, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX], 1024 * 1024); @@ -4956,6 +5451,8 @@ int evergreen_init(struct radeon_device *rdev) r700_cp_fini(rdev); r600_dma_fini(rdev); r600_irq_fini(rdev); + if (rdev->flags & RADEON_IS_IGP) + sumo_rlc_fini(rdev); radeon_wb_fini(rdev); radeon_ib_pool_fini(rdev); radeon_irq_kms_fini(rdev); @@ -4979,16 +5476,19 @@ int evergreen_init(struct radeon_device *rdev) void evergreen_fini(struct radeon_device *rdev) { + radeon_pm_fini(rdev); r600_audio_fini(rdev); - r600_blit_fini(rdev); r700_cp_fini(rdev); r600_dma_fini(rdev); r600_irq_fini(rdev); + if (rdev->flags & RADEON_IS_IGP) + sumo_rlc_fini(rdev); radeon_wb_fini(rdev); radeon_ib_pool_fini(rdev); radeon_irq_kms_fini(rdev); - evergreen_pcie_gart_fini(rdev); + uvd_v1_0_fini(rdev); radeon_uvd_fini(rdev); + evergreen_pcie_gart_fini(rdev); r600_vram_scratch_fini(rdev); radeon_gem_fini(rdev); radeon_fence_driver_fini(rdev); @@ -5061,3 +5561,153 @@ void evergreen_pcie_gen2_enable(struct radeon_device *rdev) WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl); } } + +void evergreen_program_aspm(struct radeon_device *rdev) +{ + u32 data, orig; + u32 pcie_lc_cntl, pcie_lc_cntl_old; + bool disable_l0s, disable_l1 = false, disable_plloff_in_l1 = false; + /* fusion_platform = true + * if the system is a fusion system + * (APU or DGPU in a fusion system). + * todo: check if the system is a fusion platform. + */ + bool fusion_platform = false; + + if (radeon_aspm == 0) + return; + + if (!(rdev->flags & RADEON_IS_PCIE)) + return; + + switch (rdev->family) { + case CHIP_CYPRESS: + case CHIP_HEMLOCK: + case CHIP_JUNIPER: + case CHIP_REDWOOD: + case CHIP_CEDAR: + case CHIP_SUMO: + case CHIP_SUMO2: + case CHIP_PALM: + case CHIP_ARUBA: + disable_l0s = true; + break; + default: + disable_l0s = false; + break; + } + + if (rdev->flags & RADEON_IS_IGP) + fusion_platform = true; /* XXX also dGPUs in a fusion system */ + + data = orig = RREG32_PIF_PHY0(PB0_PIF_PAIRING); + if (fusion_platform) + data &= ~MULTI_PIF; + else + data |= MULTI_PIF; + if (data != orig) + WREG32_PIF_PHY0(PB0_PIF_PAIRING, data); + + data = orig = RREG32_PIF_PHY1(PB1_PIF_PAIRING); + if (fusion_platform) + data &= ~MULTI_PIF; + else + data |= MULTI_PIF; + if (data != orig) + WREG32_PIF_PHY1(PB1_PIF_PAIRING, data); + + pcie_lc_cntl = pcie_lc_cntl_old = RREG32_PCIE_PORT(PCIE_LC_CNTL); + pcie_lc_cntl &= ~(LC_L0S_INACTIVITY_MASK | LC_L1_INACTIVITY_MASK); + if (!disable_l0s) { + if (rdev->family >= CHIP_BARTS) + pcie_lc_cntl |= LC_L0S_INACTIVITY(7); + else + pcie_lc_cntl |= LC_L0S_INACTIVITY(3); + } + + if (!disable_l1) { + if (rdev->family >= CHIP_BARTS) + pcie_lc_cntl |= LC_L1_INACTIVITY(7); + else + pcie_lc_cntl |= LC_L1_INACTIVITY(8); + + if (!disable_plloff_in_l1) { + data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0); + data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK); + data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7); + if (data != orig) + WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0, data); + + data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1); + data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK); + data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7); + if (data != orig) + WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1, data); + + data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0); + data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK); + data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7); + if (data != orig) + WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0, data); + + data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1); + data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK); + data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7); + if (data != orig) + WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1, data); + + if (rdev->family >= CHIP_BARTS) { + data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0); + data &= ~PLL_RAMP_UP_TIME_0_MASK; + data |= PLL_RAMP_UP_TIME_0(4); + if (data != orig) + WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0, data); + + data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1); + data &= ~PLL_RAMP_UP_TIME_1_MASK; + data |= PLL_RAMP_UP_TIME_1(4); + if (data != orig) + WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1, data); + + data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0); + data &= ~PLL_RAMP_UP_TIME_0_MASK; + data |= PLL_RAMP_UP_TIME_0(4); + if (data != orig) + WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0, data); + + data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1); + data &= ~PLL_RAMP_UP_TIME_1_MASK; + data |= PLL_RAMP_UP_TIME_1(4); + if (data != orig) + WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1, data); + } + + data = orig = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL); + data &= ~LC_DYN_LANES_PWR_STATE_MASK; + data |= LC_DYN_LANES_PWR_STATE(3); + if (data != orig) + WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, data); + + if (rdev->family >= CHIP_BARTS) { + data = orig = RREG32_PIF_PHY0(PB0_PIF_CNTL); + data &= ~LS2_EXIT_TIME_MASK; + data |= LS2_EXIT_TIME(1); + if (data != orig) + WREG32_PIF_PHY0(PB0_PIF_CNTL, data); + + data = orig = RREG32_PIF_PHY1(PB1_PIF_CNTL); + data &= ~LS2_EXIT_TIME_MASK; + data |= LS2_EXIT_TIME(1); + if (data != orig) + WREG32_PIF_PHY1(PB1_PIF_CNTL, data); + } + } + } + + /* evergreen parts only */ + if (rdev->family < CHIP_BARTS) + pcie_lc_cntl |= LC_PMI_TO_L1_DIS; + + if (pcie_lc_cntl != pcie_lc_cntl_old) + WREG32_PCIE_PORT(PCIE_LC_CNTL, pcie_lc_cntl); +} |