diff options
Diffstat (limited to 'drivers/gpu/drm/radeon/radeon_device.c')
| -rw-r--r-- | drivers/gpu/drm/radeon/radeon_device.c | 1015 |
1 files changed, 850 insertions, 165 deletions
diff --git a/drivers/gpu/drm/radeon/radeon_device.c b/drivers/gpu/drm/radeon/radeon_device.c index 49f7cb7e226..697add2cd4e 100644 --- a/drivers/gpu/drm/radeon/radeon_device.c +++ b/drivers/gpu/drm/radeon/radeon_device.c @@ -89,11 +89,76 @@ static const char radeon_family_name[][16] = { "TURKS", "CAICOS", "CAYMAN", + "ARUBA", + "TAHITI", + "PITCAIRN", + "VERDE", + "OLAND", + "HAINAN", + "BONAIRE", + "KAVERI", + "KABINI", + "HAWAII", + "MULLINS", "LAST", }; -/* - * Clear GPU surface registers. +bool radeon_is_px(struct drm_device *dev) +{ + struct radeon_device *rdev = dev->dev_private; + + if (rdev->flags & RADEON_IS_PX) + return true; + return false; +} + +/** + * radeon_program_register_sequence - program an array of registers. + * + * @rdev: radeon_device pointer + * @registers: pointer to the register array + * @array_size: size of the register array + * + * Programs an array or registers with and and or masks. + * This is a helper for setting golden registers. + */ +void radeon_program_register_sequence(struct radeon_device *rdev, + const u32 *registers, + const u32 array_size) +{ + u32 tmp, reg, and_mask, or_mask; + int i; + + if (array_size % 3) + return; + + for (i = 0; i < array_size; i +=3) { + reg = registers[i + 0]; + and_mask = registers[i + 1]; + or_mask = registers[i + 2]; + + if (and_mask == 0xffffffff) { + tmp = or_mask; + } else { + tmp = RREG32(reg); + tmp &= ~and_mask; + tmp |= or_mask; + } + WREG32(reg, tmp); + } +} + +void radeon_pci_config_reset(struct radeon_device *rdev) +{ + pci_write_config_dword(rdev->pdev, 0x7c, RADEON_ASIC_RESET_DATA); +} + +/** + * radeon_surface_init - Clear GPU surface registers. + * + * @rdev: radeon_device pointer + * + * Clear GPU surface registers (r1xx-r5xx). */ void radeon_surface_init(struct radeon_device *rdev) { @@ -115,6 +180,13 @@ void radeon_surface_init(struct radeon_device *rdev) /* * GPU scratch registers helpers function. */ +/** + * radeon_scratch_init - Init scratch register driver information. + * + * @rdev: radeon_device pointer + * + * Init CP scratch register driver information (r1xx-r5xx) + */ void radeon_scratch_init(struct radeon_device *rdev) { int i; @@ -132,6 +204,15 @@ void radeon_scratch_init(struct radeon_device *rdev) } } +/** + * radeon_scratch_get - Allocate a scratch register + * + * @rdev: radeon_device pointer + * @reg: scratch register mmio offset + * + * Allocate a CP scratch register for use by the driver (all asics). + * Returns 0 on success or -EINVAL on failure. + */ int radeon_scratch_get(struct radeon_device *rdev, uint32_t *reg) { int i; @@ -146,6 +227,14 @@ int radeon_scratch_get(struct radeon_device *rdev, uint32_t *reg) return -EINVAL; } +/** + * radeon_scratch_free - Free a scratch register + * + * @rdev: radeon_device pointer + * @reg: scratch register mmio offset + * + * Free a CP scratch register allocated for use by the driver (all asics) + */ void radeon_scratch_free(struct radeon_device *rdev, uint32_t reg) { int i; @@ -158,62 +247,169 @@ void radeon_scratch_free(struct radeon_device *rdev, uint32_t reg) } } -void radeon_wb_disable(struct radeon_device *rdev) +/* + * GPU doorbell aperture helpers function. + */ +/** + * radeon_doorbell_init - Init doorbell driver information. + * + * @rdev: radeon_device pointer + * + * Init doorbell driver information (CIK) + * Returns 0 on success, error on failure. + */ +static int radeon_doorbell_init(struct radeon_device *rdev) { - int r; + /* doorbell bar mapping */ + rdev->doorbell.base = pci_resource_start(rdev->pdev, 2); + rdev->doorbell.size = pci_resource_len(rdev->pdev, 2); - if (rdev->wb.wb_obj) { - r = radeon_bo_reserve(rdev->wb.wb_obj, false); - if (unlikely(r != 0)) - return; - radeon_bo_kunmap(rdev->wb.wb_obj); - radeon_bo_unpin(rdev->wb.wb_obj); - radeon_bo_unreserve(rdev->wb.wb_obj); + rdev->doorbell.num_doorbells = min_t(u32, rdev->doorbell.size / sizeof(u32), RADEON_MAX_DOORBELLS); + if (rdev->doorbell.num_doorbells == 0) + return -EINVAL; + + rdev->doorbell.ptr = ioremap(rdev->doorbell.base, rdev->doorbell.num_doorbells * sizeof(u32)); + if (rdev->doorbell.ptr == NULL) { + return -ENOMEM; + } + DRM_INFO("doorbell mmio base: 0x%08X\n", (uint32_t)rdev->doorbell.base); + DRM_INFO("doorbell mmio size: %u\n", (unsigned)rdev->doorbell.size); + + memset(&rdev->doorbell.used, 0, sizeof(rdev->doorbell.used)); + + return 0; +} + +/** + * radeon_doorbell_fini - Tear down doorbell driver information. + * + * @rdev: radeon_device pointer + * + * Tear down doorbell driver information (CIK) + */ +static void radeon_doorbell_fini(struct radeon_device *rdev) +{ + iounmap(rdev->doorbell.ptr); + rdev->doorbell.ptr = NULL; +} + +/** + * radeon_doorbell_get - Allocate a doorbell entry + * + * @rdev: radeon_device pointer + * @doorbell: doorbell index + * + * Allocate a doorbell for use by the driver (all asics). + * Returns 0 on success or -EINVAL on failure. + */ +int radeon_doorbell_get(struct radeon_device *rdev, u32 *doorbell) +{ + unsigned long offset = find_first_zero_bit(rdev->doorbell.used, rdev->doorbell.num_doorbells); + if (offset < rdev->doorbell.num_doorbells) { + __set_bit(offset, rdev->doorbell.used); + *doorbell = offset; + return 0; + } else { + return -EINVAL; } +} + +/** + * radeon_doorbell_free - Free a doorbell entry + * + * @rdev: radeon_device pointer + * @doorbell: doorbell index + * + * Free a doorbell allocated for use by the driver (all asics) + */ +void radeon_doorbell_free(struct radeon_device *rdev, u32 doorbell) +{ + if (doorbell < rdev->doorbell.num_doorbells) + __clear_bit(doorbell, rdev->doorbell.used); +} + +/* + * radeon_wb_*() + * Writeback is the the method by which the the GPU updates special pages + * in memory with the status of certain GPU events (fences, ring pointers, + * etc.). + */ + +/** + * radeon_wb_disable - Disable Writeback + * + * @rdev: radeon_device pointer + * + * Disables Writeback (all asics). Used for suspend. + */ +void radeon_wb_disable(struct radeon_device *rdev) +{ rdev->wb.enabled = false; } +/** + * radeon_wb_fini - Disable Writeback and free memory + * + * @rdev: radeon_device pointer + * + * Disables Writeback and frees the Writeback memory (all asics). + * Used at driver shutdown. + */ void radeon_wb_fini(struct radeon_device *rdev) { radeon_wb_disable(rdev); if (rdev->wb.wb_obj) { + if (!radeon_bo_reserve(rdev->wb.wb_obj, false)) { + radeon_bo_kunmap(rdev->wb.wb_obj); + radeon_bo_unpin(rdev->wb.wb_obj); + radeon_bo_unreserve(rdev->wb.wb_obj); + } radeon_bo_unref(&rdev->wb.wb_obj); rdev->wb.wb = NULL; rdev->wb.wb_obj = NULL; } } +/** + * radeon_wb_init- Init Writeback driver info and allocate memory + * + * @rdev: radeon_device pointer + * + * Disables Writeback and frees the Writeback memory (all asics). + * Used at driver startup. + * Returns 0 on success or an -error on failure. + */ int radeon_wb_init(struct radeon_device *rdev) { int r; if (rdev->wb.wb_obj == NULL) { r = radeon_bo_create(rdev, RADEON_GPU_PAGE_SIZE, PAGE_SIZE, true, - RADEON_GEM_DOMAIN_GTT, &rdev->wb.wb_obj); + RADEON_GEM_DOMAIN_GTT, NULL, &rdev->wb.wb_obj); if (r) { dev_warn(rdev->dev, "(%d) create WB bo failed\n", r); return r; } - } - r = radeon_bo_reserve(rdev->wb.wb_obj, false); - if (unlikely(r != 0)) { - radeon_wb_fini(rdev); - return r; - } - r = radeon_bo_pin(rdev->wb.wb_obj, RADEON_GEM_DOMAIN_GTT, - &rdev->wb.gpu_addr); - if (r) { + r = radeon_bo_reserve(rdev->wb.wb_obj, false); + if (unlikely(r != 0)) { + radeon_wb_fini(rdev); + return r; + } + r = radeon_bo_pin(rdev->wb.wb_obj, RADEON_GEM_DOMAIN_GTT, + &rdev->wb.gpu_addr); + if (r) { + radeon_bo_unreserve(rdev->wb.wb_obj); + dev_warn(rdev->dev, "(%d) pin WB bo failed\n", r); + radeon_wb_fini(rdev); + return r; + } + r = radeon_bo_kmap(rdev->wb.wb_obj, (void **)&rdev->wb.wb); radeon_bo_unreserve(rdev->wb.wb_obj); - dev_warn(rdev->dev, "(%d) pin WB bo failed\n", r); - radeon_wb_fini(rdev); - return r; - } - r = radeon_bo_kmap(rdev->wb.wb_obj, (void **)&rdev->wb.wb); - radeon_bo_unreserve(rdev->wb.wb_obj); - if (r) { - dev_warn(rdev->dev, "(%d) map WB bo failed\n", r); - radeon_wb_fini(rdev); - return r; + if (r) { + dev_warn(rdev->dev, "(%d) map WB bo failed\n", r); + radeon_wb_fini(rdev); + return r; + } } /* clear wb memory */ @@ -221,9 +417,9 @@ int radeon_wb_init(struct radeon_device *rdev) /* disable event_write fences */ rdev->wb.use_event = false; /* disabled via module param */ - if (radeon_no_wb == 1) + if (radeon_no_wb == 1) { rdev->wb.enabled = false; - else { + } else { if (rdev->flags & RADEON_IS_AGP) { /* often unreliable on AGP */ rdev->wb.enabled = false; @@ -233,12 +429,13 @@ int radeon_wb_init(struct radeon_device *rdev) } else { rdev->wb.enabled = true; /* event_write fences are only available on r600+ */ - if (rdev->family >= CHIP_R600) + if (rdev->family >= CHIP_R600) { rdev->wb.use_event = true; + } } } - /* always use writeback/events on NI */ - if (ASIC_IS_DCE5(rdev)) { + /* always use writeback/events on NI, APUs */ + if (rdev->family >= CHIP_PALM) { rdev->wb.enabled = true; rdev->wb.use_event = true; } @@ -291,8 +488,10 @@ int radeon_wb_init(struct radeon_device *rdev) */ void radeon_vram_location(struct radeon_device *rdev, struct radeon_mc *mc, u64 base) { + uint64_t limit = (uint64_t)radeon_vram_limit << 20; + mc->vram_start = base; - if (mc->mc_vram_size > (0xFFFFFFFF - base + 1)) { + if (mc->mc_vram_size > (rdev->mc.mc_mask - base + 1)) { dev_warn(rdev->dev, "limiting VRAM to PCI aperture size\n"); mc->real_vram_size = mc->aper_size; mc->mc_vram_size = mc->aper_size; @@ -304,8 +503,8 @@ void radeon_vram_location(struct radeon_device *rdev, struct radeon_mc *mc, u64 mc->mc_vram_size = mc->aper_size; } mc->vram_end = mc->vram_start + mc->mc_vram_size - 1; - if (radeon_vram_limit && radeon_vram_limit < mc->real_vram_size) - mc->real_vram_size = radeon_vram_limit; + if (limit && limit < mc->real_vram_size) + mc->real_vram_size = limit; dev_info(rdev->dev, "VRAM: %lluM 0x%016llX - 0x%016llX (%lluM used)\n", mc->mc_vram_size >> 20, mc->vram_start, mc->vram_end, mc->real_vram_size >> 20); @@ -327,7 +526,7 @@ void radeon_gtt_location(struct radeon_device *rdev, struct radeon_mc *mc) { u64 size_af, size_bf; - size_af = ((0xFFFFFFFF - mc->vram_end) + mc->gtt_base_align) & ~mc->gtt_base_align; + size_af = ((rdev->mc.mc_mask - mc->vram_end) + mc->gtt_base_align) & ~mc->gtt_base_align; size_bf = mc->vram_start & ~mc->gtt_base_align; if (size_bf > size_af) { if (mc->gtt_size > size_bf) { @@ -350,26 +549,40 @@ void radeon_gtt_location(struct radeon_device *rdev, struct radeon_mc *mc) /* * GPU helpers function. */ +/** + * radeon_card_posted - check if the hw has already been initialized + * + * @rdev: radeon_device pointer + * + * Check if the asic has been initialized (all asics). + * Used at driver startup. + * Returns true if initialized or false if not. + */ bool radeon_card_posted(struct radeon_device *rdev) { uint32_t reg; - if (efi_enabled && rdev->pdev->subsystem_vendor == PCI_VENDOR_ID_APPLE) + /* required for EFI mode on macbook2,1 which uses an r5xx asic */ + if (efi_enabled(EFI_BOOT) && + (rdev->pdev->subsystem_vendor == PCI_VENDOR_ID_APPLE) && + (rdev->family < CHIP_R600)) return false; + if (ASIC_IS_NODCE(rdev)) + goto check_memsize; + /* first check CRTCs */ - if (ASIC_IS_DCE41(rdev)) { + if (ASIC_IS_DCE4(rdev)) { reg = RREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET) | RREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET); - if (reg & EVERGREEN_CRTC_MASTER_EN) - return true; - } else if (ASIC_IS_DCE4(rdev)) { - reg = RREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET) | - RREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET) | - RREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET) | - RREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET) | - RREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET) | - RREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET); + if (rdev->num_crtc >= 4) { + reg |= RREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET) | + RREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET); + } + if (rdev->num_crtc >= 6) { + reg |= RREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET) | + RREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET); + } if (reg & EVERGREEN_CRTC_MASTER_EN) return true; } else if (ASIC_IS_AVIVO(rdev)) { @@ -386,6 +599,7 @@ bool radeon_card_posted(struct radeon_device *rdev) } } +check_memsize: /* then check MEM_SIZE, in case the crtcs are off */ if (rdev->family >= CHIP_R600) reg = RREG32(R600_CONFIG_MEMSIZE); @@ -399,6 +613,14 @@ bool radeon_card_posted(struct radeon_device *rdev) } +/** + * radeon_update_bandwidth_info - update display bandwidth params + * + * @rdev: radeon_device pointer + * + * Used when sclk/mclk are switched or display modes are set. + * params are used to calculate display watermarks (all asics) + */ void radeon_update_bandwidth_info(struct radeon_device *rdev) { fixed20_12 a; @@ -419,6 +641,15 @@ void radeon_update_bandwidth_info(struct radeon_device *rdev) } } +/** + * radeon_boot_test_post_card - check and possibly initialize the hw + * + * @rdev: radeon_device pointer + * + * Check if the asic is initialized and if not, attempt to initialize + * it (all asics). + * Returns true if initialized or false if not. + */ bool radeon_boot_test_post_card(struct radeon_device *rdev) { if (radeon_card_posted(rdev)) @@ -437,6 +668,16 @@ bool radeon_boot_test_post_card(struct radeon_device *rdev) } } +/** + * radeon_dummy_page_init - init dummy page used by the driver + * + * @rdev: radeon_device pointer + * + * Allocate the dummy page used by the driver (all asics). + * This dummy page is used by the driver as a filler for gart entries + * when pages are taken out of the GART + * Returns 0 on sucess, -ENOMEM on failure. + */ int radeon_dummy_page_init(struct radeon_device *rdev) { if (rdev->dummy_page.page) @@ -455,6 +696,13 @@ int radeon_dummy_page_init(struct radeon_device *rdev) return 0; } +/** + * radeon_dummy_page_fini - free dummy page used by the driver + * + * @rdev: radeon_device pointer + * + * Frees the dummy page used by the driver (all asics). + */ void radeon_dummy_page_fini(struct radeon_device *rdev) { if (rdev->dummy_page.page == NULL) @@ -467,6 +715,23 @@ void radeon_dummy_page_fini(struct radeon_device *rdev) /* ATOM accessor methods */ +/* + * ATOM is an interpreted byte code stored in tables in the vbios. The + * driver registers callbacks to access registers and the interpreter + * in the driver parses the tables and executes then to program specific + * actions (set display modes, asic init, etc.). See radeon_atombios.c, + * atombios.h, and atom.c + */ + +/** + * cail_pll_read - read PLL register + * + * @info: atom card_info pointer + * @reg: PLL register offset + * + * Provides a PLL register accessor for the atom interpreter (r4xx+). + * Returns the value of the PLL register. + */ static uint32_t cail_pll_read(struct card_info *info, uint32_t reg) { struct radeon_device *rdev = info->dev->dev_private; @@ -476,6 +741,15 @@ static uint32_t cail_pll_read(struct card_info *info, uint32_t reg) return r; } +/** + * cail_pll_write - write PLL register + * + * @info: atom card_info pointer + * @reg: PLL register offset + * @val: value to write to the pll register + * + * Provides a PLL register accessor for the atom interpreter (r4xx+). + */ static void cail_pll_write(struct card_info *info, uint32_t reg, uint32_t val) { struct radeon_device *rdev = info->dev->dev_private; @@ -483,6 +757,15 @@ static void cail_pll_write(struct card_info *info, uint32_t reg, uint32_t val) rdev->pll_wreg(rdev, reg, val); } +/** + * cail_mc_read - read MC (Memory Controller) register + * + * @info: atom card_info pointer + * @reg: MC register offset + * + * Provides an MC register accessor for the atom interpreter (r4xx+). + * Returns the value of the MC register. + */ static uint32_t cail_mc_read(struct card_info *info, uint32_t reg) { struct radeon_device *rdev = info->dev->dev_private; @@ -492,6 +775,15 @@ static uint32_t cail_mc_read(struct card_info *info, uint32_t reg) return r; } +/** + * cail_mc_write - write MC (Memory Controller) register + * + * @info: atom card_info pointer + * @reg: MC register offset + * @val: value to write to the pll register + * + * Provides a MC register accessor for the atom interpreter (r4xx+). + */ static void cail_mc_write(struct card_info *info, uint32_t reg, uint32_t val) { struct radeon_device *rdev = info->dev->dev_private; @@ -499,6 +791,15 @@ static void cail_mc_write(struct card_info *info, uint32_t reg, uint32_t val) rdev->mc_wreg(rdev, reg, val); } +/** + * cail_reg_write - write MMIO register + * + * @info: atom card_info pointer + * @reg: MMIO register offset + * @val: value to write to the pll register + * + * Provides a MMIO register accessor for the atom interpreter (r4xx+). + */ static void cail_reg_write(struct card_info *info, uint32_t reg, uint32_t val) { struct radeon_device *rdev = info->dev->dev_private; @@ -506,6 +807,15 @@ static void cail_reg_write(struct card_info *info, uint32_t reg, uint32_t val) WREG32(reg*4, val); } +/** + * cail_reg_read - read MMIO register + * + * @info: atom card_info pointer + * @reg: MMIO register offset + * + * Provides an MMIO register accessor for the atom interpreter (r4xx+). + * Returns the value of the MMIO register. + */ static uint32_t cail_reg_read(struct card_info *info, uint32_t reg) { struct radeon_device *rdev = info->dev->dev_private; @@ -515,6 +825,15 @@ static uint32_t cail_reg_read(struct card_info *info, uint32_t reg) return r; } +/** + * cail_ioreg_write - write IO register + * + * @info: atom card_info pointer + * @reg: IO register offset + * @val: value to write to the pll register + * + * Provides a IO register accessor for the atom interpreter (r4xx+). + */ static void cail_ioreg_write(struct card_info *info, uint32_t reg, uint32_t val) { struct radeon_device *rdev = info->dev->dev_private; @@ -522,6 +841,15 @@ static void cail_ioreg_write(struct card_info *info, uint32_t reg, uint32_t val) WREG32_IO(reg*4, val); } +/** + * cail_ioreg_read - read IO register + * + * @info: atom card_info pointer + * @reg: IO register offset + * + * Provides an IO register accessor for the atom interpreter (r4xx+). + * Returns the value of the IO register. + */ static uint32_t cail_ioreg_read(struct card_info *info, uint32_t reg) { struct radeon_device *rdev = info->dev->dev_private; @@ -531,6 +859,16 @@ static uint32_t cail_ioreg_read(struct card_info *info, uint32_t reg) return r; } +/** + * radeon_atombios_init - init the driver info and callbacks for atombios + * + * @rdev: radeon_device pointer + * + * Initializes the driver info and register access callbacks for the + * ATOM interpreter (r4xx+). + * Returns 0 on sucess, -ENOMEM on failure. + * Called at driver startup. + */ int radeon_atombios_init(struct radeon_device *rdev) { struct card_info *atom_card_info = @@ -558,32 +896,81 @@ int radeon_atombios_init(struct radeon_device *rdev) atom_card_info->pll_write = cail_pll_write; rdev->mode_info.atom_context = atom_parse(atom_card_info, rdev->bios); + if (!rdev->mode_info.atom_context) { + radeon_atombios_fini(rdev); + return -ENOMEM; + } + mutex_init(&rdev->mode_info.atom_context->mutex); radeon_atom_initialize_bios_scratch_regs(rdev->ddev); atom_allocate_fb_scratch(rdev->mode_info.atom_context); return 0; } +/** + * radeon_atombios_fini - free the driver info and callbacks for atombios + * + * @rdev: radeon_device pointer + * + * Frees the driver info and register access callbacks for the ATOM + * interpreter (r4xx+). + * Called at driver shutdown. + */ void radeon_atombios_fini(struct radeon_device *rdev) { if (rdev->mode_info.atom_context) { kfree(rdev->mode_info.atom_context->scratch); - kfree(rdev->mode_info.atom_context); } + kfree(rdev->mode_info.atom_context); + rdev->mode_info.atom_context = NULL; kfree(rdev->mode_info.atom_card_info); + rdev->mode_info.atom_card_info = NULL; } +/* COMBIOS */ +/* + * COMBIOS is the bios format prior to ATOM. It provides + * command tables similar to ATOM, but doesn't have a unified + * parser. See radeon_combios.c + */ + +/** + * radeon_combios_init - init the driver info for combios + * + * @rdev: radeon_device pointer + * + * Initializes the driver info for combios (r1xx-r3xx). + * Returns 0 on sucess. + * Called at driver startup. + */ int radeon_combios_init(struct radeon_device *rdev) { radeon_combios_initialize_bios_scratch_regs(rdev->ddev); return 0; } +/** + * radeon_combios_fini - free the driver info for combios + * + * @rdev: radeon_device pointer + * + * Frees the driver info for combios (r1xx-r3xx). + * Called at driver shutdown. + */ void radeon_combios_fini(struct radeon_device *rdev) { } -/* if we get transitioned to only one device, tak VGA back */ +/* if we get transitioned to only one device, take VGA back */ +/** + * radeon_vga_set_decode - enable/disable vga decode + * + * @cookie: radeon_device pointer + * @state: enable/disable vga decode + * + * Enable/disable vga decode (all asics). + * Returns VGA resource flags. + */ static unsigned int radeon_vga_set_decode(void *cookie, bool state) { struct radeon_device *rdev = cookie; @@ -595,55 +982,61 @@ static unsigned int radeon_vga_set_decode(void *cookie, bool state) return VGA_RSRC_NORMAL_IO | VGA_RSRC_NORMAL_MEM; } -void radeon_check_arguments(struct radeon_device *rdev) +/** + * radeon_check_pot_argument - check that argument is a power of two + * + * @arg: value to check + * + * Validates that a certain argument is a power of two (all asics). + * Returns true if argument is valid. + */ +static bool radeon_check_pot_argument(int arg) +{ + return (arg & (arg - 1)) == 0; +} + +/** + * radeon_check_arguments - validate module params + * + * @rdev: radeon_device pointer + * + * Validates certain module parameters and updates + * the associated values used by the driver (all asics). + */ +static void radeon_check_arguments(struct radeon_device *rdev) { /* vramlimit must be a power of two */ - switch (radeon_vram_limit) { - case 0: - case 4: - case 8: - case 16: - case 32: - case 64: - case 128: - case 256: - case 512: - case 1024: - case 2048: - case 4096: - break; - default: + if (!radeon_check_pot_argument(radeon_vram_limit)) { dev_warn(rdev->dev, "vram limit (%d) must be a power of 2\n", radeon_vram_limit); radeon_vram_limit = 0; - break; } - radeon_vram_limit = radeon_vram_limit << 20; + + if (radeon_gart_size == -1) { + /* default to a larger gart size on newer asics */ + if (rdev->family >= CHIP_RV770) + radeon_gart_size = 1024; + else + radeon_gart_size = 512; + } /* gtt size must be power of two and greater or equal to 32M */ - switch (radeon_gart_size) { - case 4: - case 8: - case 16: - dev_warn(rdev->dev, "gart size (%d) too small forcing to 512M\n", + if (radeon_gart_size < 32) { + dev_warn(rdev->dev, "gart size (%d) too small\n", radeon_gart_size); - radeon_gart_size = 512; - break; - case 32: - case 64: - case 128: - case 256: - case 512: - case 1024: - case 2048: - case 4096: - break; - default: + if (rdev->family >= CHIP_RV770) + radeon_gart_size = 1024; + else + radeon_gart_size = 512; + } else if (!radeon_check_pot_argument(radeon_gart_size)) { dev_warn(rdev->dev, "gart size (%d) must be a power of 2\n", radeon_gart_size); - radeon_gart_size = 512; - break; + if (rdev->family >= CHIP_RV770) + radeon_gart_size = 1024; + else + radeon_gart_size = 512; } - rdev->mc.gtt_size = radeon_gart_size * 1024 * 1024; + rdev->mc.gtt_size = (uint64_t)radeon_gart_size << 20; + /* AGP mode can only be -1, 1, 2, 4, 8 */ switch (radeon_agpmode) { case -1: @@ -659,40 +1052,144 @@ void radeon_check_arguments(struct radeon_device *rdev) radeon_agpmode = 0; break; } + + if (!radeon_check_pot_argument(radeon_vm_size)) { + dev_warn(rdev->dev, "VM size (%d) must be a power of 2\n", + radeon_vm_size); + radeon_vm_size = 4; + } + + if (radeon_vm_size < 1) { + dev_warn(rdev->dev, "VM size (%d) to small, min is 1GB\n", + radeon_vm_size); + radeon_vm_size = 4; + } + + /* + * Max GPUVM size for Cayman, SI and CI are 40 bits. + */ + if (radeon_vm_size > 1024) { + dev_warn(rdev->dev, "VM size (%d) too large, max is 1TB\n", + radeon_vm_size); + radeon_vm_size = 4; + } + + /* defines number of bits in page table versus page directory, + * a page is 4KB so we have 12 bits offset, minimum 9 bits in the + * page table and the remaining bits are in the page directory */ + if (radeon_vm_block_size < 9) { + dev_warn(rdev->dev, "VM page table size (%d) too small\n", + radeon_vm_block_size); + radeon_vm_block_size = 9; + } + + if (radeon_vm_block_size > 24 || + (radeon_vm_size * 1024) < (1ull << radeon_vm_block_size)) { + dev_warn(rdev->dev, "VM page table size (%d) too large\n", + radeon_vm_block_size); + radeon_vm_block_size = 9; + } +} + +/** + * radeon_switcheroo_quirk_long_wakeup - return true if longer d3 delay is + * needed for waking up. + * + * @pdev: pci dev pointer + */ +static bool radeon_switcheroo_quirk_long_wakeup(struct pci_dev *pdev) +{ + + /* 6600m in a macbook pro */ + if (pdev->subsystem_vendor == PCI_VENDOR_ID_APPLE && + pdev->subsystem_device == 0x00e2) { + printk(KERN_INFO "radeon: quirking longer d3 wakeup delay\n"); + return true; + } + + return false; } +/** + * radeon_switcheroo_set_state - set switcheroo state + * + * @pdev: pci dev pointer + * @state: vga switcheroo state + * + * Callback for the switcheroo driver. Suspends or resumes the + * the asics before or after it is powered up using ACPI methods. + */ static void radeon_switcheroo_set_state(struct pci_dev *pdev, enum vga_switcheroo_state state) { struct drm_device *dev = pci_get_drvdata(pdev); - pm_message_t pmm = { .event = PM_EVENT_SUSPEND }; + + if (radeon_is_px(dev) && state == VGA_SWITCHEROO_OFF) + return; + if (state == VGA_SWITCHEROO_ON) { + unsigned d3_delay = dev->pdev->d3_delay; + printk(KERN_INFO "radeon: switched on\n"); /* don't suspend or resume card normally */ dev->switch_power_state = DRM_SWITCH_POWER_CHANGING; - radeon_resume_kms(dev); + + if (d3_delay < 20 && radeon_switcheroo_quirk_long_wakeup(pdev)) + dev->pdev->d3_delay = 20; + + radeon_resume_kms(dev, true, true); + + dev->pdev->d3_delay = d3_delay; + dev->switch_power_state = DRM_SWITCH_POWER_ON; drm_kms_helper_poll_enable(dev); } else { printk(KERN_INFO "radeon: switched off\n"); drm_kms_helper_poll_disable(dev); dev->switch_power_state = DRM_SWITCH_POWER_CHANGING; - radeon_suspend_kms(dev, pmm); + radeon_suspend_kms(dev, true, true); dev->switch_power_state = DRM_SWITCH_POWER_OFF; } } +/** + * radeon_switcheroo_can_switch - see if switcheroo state can change + * + * @pdev: pci dev pointer + * + * Callback for the switcheroo driver. Check of the switcheroo + * state can be changed. + * Returns true if the state can be changed, false if not. + */ static bool radeon_switcheroo_can_switch(struct pci_dev *pdev) { struct drm_device *dev = pci_get_drvdata(pdev); - bool can_switch; - spin_lock(&dev->count_lock); - can_switch = (dev->open_count == 0); - spin_unlock(&dev->count_lock); - return can_switch; + /* + * FIXME: open_count is protected by drm_global_mutex but that would lead to + * locking inversion with the driver load path. And the access here is + * completely racy anyway. So don't bother with locking for now. + */ + return dev->open_count == 0; } +static const struct vga_switcheroo_client_ops radeon_switcheroo_ops = { + .set_gpu_state = radeon_switcheroo_set_state, + .reprobe = NULL, + .can_switch = radeon_switcheroo_can_switch, +}; +/** + * radeon_device_init - initialize the driver + * + * @rdev: radeon_device pointer + * @pdev: drm dev pointer + * @pdev: pci dev pointer + * @flags: driver flags + * + * Initializes the driver info and hw (all asics). + * Returns 0 for success or an error on failure. + * Called at driver startup. + */ int radeon_device_init(struct radeon_device *rdev, struct drm_device *ddev, struct pci_dev *pdev, @@ -700,6 +1197,7 @@ int radeon_device_init(struct radeon_device *rdev, { int r, i; int dma_bits; + bool runtime = false; rdev->shutdown = false; rdev->dev = &pdev->dev; @@ -709,9 +1207,12 @@ int radeon_device_init(struct radeon_device *rdev, rdev->family = flags & RADEON_FAMILY_MASK; rdev->is_atom_bios = false; rdev->usec_timeout = RADEON_MAX_USEC_TIMEOUT; - rdev->mc.gtt_size = radeon_gart_size * 1024 * 1024; - rdev->gpu_lockup = false; + rdev->mc.gtt_size = 512 * 1024 * 1024; rdev->accel_working = false; + /* set up ring ids */ + for (i = 0; i < RADEON_NUM_RINGS; i++) { + rdev->ring[i].idx = i; + } DRM_INFO("initializing kernel modesetting (%s 0x%04X:0x%04X 0x%04X:0x%04X).\n", radeon_family_name[rdev->family], pdev->vendor, pdev->device, @@ -719,32 +1220,30 @@ int radeon_device_init(struct radeon_device *rdev, /* mutex initialization are all done here so we * can recall function without having locking issues */ - radeon_mutex_init(&rdev->cs_mutex); - radeon_mutex_init(&rdev->ib_pool.mutex); - for (i = 0; i < RADEON_NUM_RINGS; ++i) - mutex_init(&rdev->ring[i].mutex); + mutex_init(&rdev->ring_lock); mutex_init(&rdev->dc_hw_i2c_mutex); - if (rdev->family >= CHIP_R600) - spin_lock_init(&rdev->ih.lock); + atomic_set(&rdev->ih.lock, 0); mutex_init(&rdev->gem.mutex); mutex_init(&rdev->pm.mutex); - mutex_init(&rdev->vram_mutex); - rwlock_init(&rdev->fence_lock); - rwlock_init(&rdev->semaphore_drv.lock); - INIT_LIST_HEAD(&rdev->gem.objects); + mutex_init(&rdev->gpu_clock_mutex); + mutex_init(&rdev->srbm_mutex); + init_rwsem(&rdev->pm.mclk_lock); + init_rwsem(&rdev->exclusive_lock); init_waitqueue_head(&rdev->irq.vblank_queue); - init_waitqueue_head(&rdev->irq.idle_queue); - INIT_LIST_HEAD(&rdev->semaphore_drv.bo); - /* initialize vm here */ - rdev->vm_manager.use_bitmap = 1; - rdev->vm_manager.max_pfn = 1 << 20; - INIT_LIST_HEAD(&rdev->vm_manager.lru_vm); + r = radeon_gem_init(rdev); + if (r) + return r; + + radeon_check_arguments(rdev); + /* Adjust VM size here. + * Max GPUVM size for cayman+ is 40 bits. + */ + rdev->vm_manager.max_pfn = radeon_vm_size << 18; /* Set asic functions */ r = radeon_asic_init(rdev); if (r) return r; - radeon_check_arguments(rdev); /* all of the newer IGP chips have an internal gart * However some rs4xx report as AGP, so remove that here. @@ -758,6 +1257,17 @@ int radeon_device_init(struct radeon_device *rdev, radeon_agp_disable(rdev); } + /* Set the internal MC address mask + * This is the max address of the GPU's + * internal address space. + */ + if (rdev->family >= CHIP_CAYMAN) + rdev->mc.mc_mask = 0xffffffffffULL; /* 40 bit MC */ + else if (rdev->family >= CHIP_CEDAR) + rdev->mc.mc_mask = 0xfffffffffULL; /* 36 bit MC */ + else + rdev->mc.mc_mask = 0xffffffffULL; /* 32 bit MC */ + /* set DMA mask + need_dma32 flags. * PCIE - can handle 40-bits. * IGP - can handle 40-bits @@ -768,7 +1278,7 @@ int radeon_device_init(struct radeon_device *rdev, if (rdev->flags & RADEON_IS_AGP) rdev->need_dma32 = true; if ((rdev->flags & RADEON_IS_PCI) && - (rdev->family < CHIP_RS400)) + (rdev->family <= CHIP_RS740)) rdev->need_dma32 = true; dma_bits = rdev->need_dma32 ? 32 : 40; @@ -786,8 +1296,25 @@ int radeon_device_init(struct radeon_device *rdev, /* Registers mapping */ /* TODO: block userspace mapping of io register */ - rdev->rmmio_base = pci_resource_start(rdev->pdev, 2); - rdev->rmmio_size = pci_resource_len(rdev->pdev, 2); + spin_lock_init(&rdev->mmio_idx_lock); + spin_lock_init(&rdev->smc_idx_lock); + spin_lock_init(&rdev->pll_idx_lock); + spin_lock_init(&rdev->mc_idx_lock); + spin_lock_init(&rdev->pcie_idx_lock); + spin_lock_init(&rdev->pciep_idx_lock); + spin_lock_init(&rdev->pif_idx_lock); + spin_lock_init(&rdev->cg_idx_lock); + spin_lock_init(&rdev->uvd_idx_lock); + spin_lock_init(&rdev->rcu_idx_lock); + spin_lock_init(&rdev->didt_idx_lock); + spin_lock_init(&rdev->end_idx_lock); + if (rdev->family >= CHIP_BONAIRE) { + rdev->rmmio_base = pci_resource_start(rdev->pdev, 5); + rdev->rmmio_size = pci_resource_len(rdev->pdev, 5); + } else { + rdev->rmmio_base = pci_resource_start(rdev->pdev, 2); + rdev->rmmio_size = pci_resource_len(rdev->pdev, 2); + } rdev->rmmio = ioremap(rdev->rmmio_base, rdev->rmmio_size); if (rdev->rmmio == NULL) { return -ENOMEM; @@ -795,6 +1322,10 @@ int radeon_device_init(struct radeon_device *rdev, DRM_INFO("register mmio base: 0x%08X\n", (uint32_t)rdev->rmmio_base); DRM_INFO("register mmio size: %u\n", (unsigned)rdev->rmmio_size); + /* doorbell bar mapping */ + if (rdev->family >= CHIP_BONAIRE) + radeon_doorbell_init(rdev); + /* io port mapping */ for (i = 0; i < DEVICE_COUNT_RESOURCE; i++) { if (pci_resource_flags(rdev->pdev, i) & IORESOURCE_IO) { @@ -810,15 +1341,26 @@ int radeon_device_init(struct radeon_device *rdev, /* this will fail for cards that aren't VGA class devices, just * ignore it */ vga_client_register(rdev->pdev, rdev, NULL, radeon_vga_set_decode); - vga_switcheroo_register_client(rdev->pdev, - radeon_switcheroo_set_state, - NULL, - radeon_switcheroo_can_switch); + + if (rdev->flags & RADEON_IS_PX) + runtime = true; + vga_switcheroo_register_client(rdev->pdev, &radeon_switcheroo_ops, runtime); + if (runtime) + vga_switcheroo_init_domain_pm_ops(rdev->dev, &rdev->vga_pm_domain); r = radeon_init(rdev); if (r) return r; + r = radeon_ib_ring_tests(rdev); + if (r) + DRM_ERROR("ib ring test failed (%d).\n", r); + + r = radeon_gem_debugfs_init(rdev); + if (r) { + DRM_ERROR("registering gem debugfs failed (%d).\n", r); + } + if (rdev->flags & RADEON_IS_AGP && !rdev->accel_working) { /* Acceleration not working on AGP card try again * with fallback to PCI or PCIE GART @@ -830,20 +1372,38 @@ int radeon_device_init(struct radeon_device *rdev, if (r) return r; } + if ((radeon_testing & 1)) { - radeon_test_moves(rdev); + if (rdev->accel_working) + radeon_test_moves(rdev); + else + DRM_INFO("radeon: acceleration disabled, skipping move tests\n"); } if ((radeon_testing & 2)) { - radeon_test_syncing(rdev); + if (rdev->accel_working) + radeon_test_syncing(rdev); + else + DRM_INFO("radeon: acceleration disabled, skipping sync tests\n"); } if (radeon_benchmarking) { - radeon_benchmark(rdev, radeon_benchmarking); + if (rdev->accel_working) + radeon_benchmark(rdev, radeon_benchmarking); + else + DRM_INFO("radeon: acceleration disabled, skipping benchmarks\n"); } return 0; } static void radeon_debugfs_remove_files(struct radeon_device *rdev); +/** + * radeon_device_fini - tear down the driver + * + * @rdev: radeon_device pointer + * + * Tear down the driver info (all asics). + * Called at driver shutdown. + */ void radeon_device_fini(struct radeon_device *rdev) { DRM_INFO("radeon: finishing device.\n"); @@ -858,6 +1418,8 @@ void radeon_device_fini(struct radeon_device *rdev) rdev->rio_mem = NULL; iounmap(rdev->rmmio); rdev->rmmio = NULL; + if (rdev->family >= CHIP_BONAIRE) + radeon_doorbell_fini(rdev); radeon_debugfs_remove_files(rdev); } @@ -865,19 +1427,28 @@ void radeon_device_fini(struct radeon_device *rdev) /* * Suspend & resume. */ -int radeon_suspend_kms(struct drm_device *dev, pm_message_t state) +/** + * radeon_suspend_kms - initiate device suspend + * + * @pdev: drm dev pointer + * @state: suspend state + * + * Puts the hw in the suspend state (all asics). + * Returns 0 for success or an error on failure. + * Called at driver suspend. + */ +int radeon_suspend_kms(struct drm_device *dev, bool suspend, bool fbcon) { struct radeon_device *rdev; struct drm_crtc *crtc; struct drm_connector *connector; int i, r; + bool force_completion = false; if (dev == NULL || dev->dev_private == NULL) { return -ENODEV; } - if (state.event == PM_EVENT_PRETHAW) { - return 0; - } + rdev = dev->dev_private; if (dev->switch_power_state == DRM_SWITCH_POWER_OFF) @@ -892,7 +1463,7 @@ int radeon_suspend_kms(struct drm_device *dev, pm_message_t state) /* unpin the front buffers */ list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) { - struct radeon_framebuffer *rfb = to_radeon_framebuffer(crtc->fb); + struct radeon_framebuffer *rfb = to_radeon_framebuffer(crtc->primary->fb); struct radeon_bo *robj; if (rfb == NULL || rfb->obj == NULL) { @@ -910,13 +1481,21 @@ int radeon_suspend_kms(struct drm_device *dev, pm_message_t state) } /* evict vram memory */ radeon_bo_evict_vram(rdev); + /* wait for gpu to finish processing current batch */ - for (i = 0; i < RADEON_NUM_RINGS; i++) - radeon_fence_wait_last(rdev, i); + for (i = 0; i < RADEON_NUM_RINGS; i++) { + r = radeon_fence_wait_empty(rdev, i); + if (r) { + /* delay GPU reset to resume */ + force_completion = true; + } + } + if (force_completion) { + radeon_fence_driver_force_completion(rdev); + } radeon_save_bios_scratch_regs(rdev); - radeon_pm_suspend(rdev); radeon_suspend(rdev); radeon_hpd_fini(rdev); /* evict remaining vram memory */ @@ -925,89 +1504,195 @@ int radeon_suspend_kms(struct drm_device *dev, pm_message_t state) radeon_agp_suspend(rdev); pci_save_state(dev->pdev); - if (state.event == PM_EVENT_SUSPEND) { + if (suspend) { /* Shut down the device */ pci_disable_device(dev->pdev); pci_set_power_state(dev->pdev, PCI_D3hot); } - console_lock(); - radeon_fbdev_set_suspend(rdev, 1); - console_unlock(); + + if (fbcon) { + console_lock(); + radeon_fbdev_set_suspend(rdev, 1); + console_unlock(); + } return 0; } -int radeon_resume_kms(struct drm_device *dev) +/** + * radeon_resume_kms - initiate device resume + * + * @pdev: drm dev pointer + * + * Bring the hw back to operating state (all asics). + * Returns 0 for success or an error on failure. + * Called at driver resume. + */ +int radeon_resume_kms(struct drm_device *dev, bool resume, bool fbcon) { struct drm_connector *connector; struct radeon_device *rdev = dev->dev_private; + int r; if (dev->switch_power_state == DRM_SWITCH_POWER_OFF) return 0; - console_lock(); - pci_set_power_state(dev->pdev, PCI_D0); - pci_restore_state(dev->pdev); - if (pci_enable_device(dev->pdev)) { - console_unlock(); - return -1; + if (fbcon) { + console_lock(); + } + if (resume) { + pci_set_power_state(dev->pdev, PCI_D0); + pci_restore_state(dev->pdev); + if (pci_enable_device(dev->pdev)) { + if (fbcon) + console_unlock(); + return -1; + } } - pci_set_master(dev->pdev); /* resume AGP if in use */ radeon_agp_resume(rdev); radeon_resume(rdev); - radeon_pm_resume(rdev); - radeon_restore_bios_scratch_regs(rdev); - radeon_fbdev_set_suspend(rdev, 0); - console_unlock(); + r = radeon_ib_ring_tests(rdev); + if (r) + DRM_ERROR("ib ring test failed (%d).\n", r); + + if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) { + /* do dpm late init */ + r = radeon_pm_late_init(rdev); + if (r) { + rdev->pm.dpm_enabled = false; + DRM_ERROR("radeon_pm_late_init failed, disabling dpm\n"); + } + } else { + /* resume old pm late */ + radeon_pm_resume(rdev); + } + + radeon_restore_bios_scratch_regs(rdev); /* init dig PHYs, disp eng pll */ if (rdev->is_atom_bios) { radeon_atom_encoder_init(rdev); - radeon_atom_dcpll_init(rdev); + radeon_atom_disp_eng_pll_init(rdev); + /* turn on the BL */ + if (rdev->mode_info.bl_encoder) { + u8 bl_level = radeon_get_backlight_level(rdev, + rdev->mode_info.bl_encoder); + radeon_set_backlight_level(rdev, rdev->mode_info.bl_encoder, + bl_level); + } } /* reset hpd state */ radeon_hpd_init(rdev); /* blat the mode back in */ - drm_helper_resume_force_mode(dev); - /* turn on display hw */ - list_for_each_entry(connector, &dev->mode_config.connector_list, head) { - drm_helper_connector_dpms(connector, DRM_MODE_DPMS_ON); + if (fbcon) { + drm_helper_resume_force_mode(dev); + /* turn on display hw */ + list_for_each_entry(connector, &dev->mode_config.connector_list, head) { + drm_helper_connector_dpms(connector, DRM_MODE_DPMS_ON); + } } drm_kms_helper_poll_enable(dev); + + /* set the power state here in case we are a PX system or headless */ + if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) + radeon_pm_compute_clocks(rdev); + + if (fbcon) { + radeon_fbdev_set_suspend(rdev, 0); + console_unlock(); + } + return 0; } +/** + * radeon_gpu_reset - reset the asic + * + * @rdev: radeon device pointer + * + * Attempt the reset the GPU if it has hung (all asics). + * Returns 0 for success or an error on failure. + */ int radeon_gpu_reset(struct radeon_device *rdev) { - int r; + unsigned ring_sizes[RADEON_NUM_RINGS]; + uint32_t *ring_data[RADEON_NUM_RINGS]; + + bool saved = false; + + int i, r; int resched; - /* Prevent CS ioctl from interfering */ - radeon_mutex_lock(&rdev->cs_mutex); + down_write(&rdev->exclusive_lock); + + if (!rdev->needs_reset) { + up_write(&rdev->exclusive_lock); + return 0; + } + + rdev->needs_reset = false; radeon_save_bios_scratch_regs(rdev); /* block TTM */ resched = ttm_bo_lock_delayed_workqueue(&rdev->mman.bdev); + radeon_pm_suspend(rdev); radeon_suspend(rdev); + for (i = 0; i < RADEON_NUM_RINGS; ++i) { + ring_sizes[i] = radeon_ring_backup(rdev, &rdev->ring[i], + &ring_data[i]); + if (ring_sizes[i]) { + saved = true; + dev_info(rdev->dev, "Saved %d dwords of commands " + "on ring %d.\n", ring_sizes[i], i); + } + } + +retry: r = radeon_asic_reset(rdev); if (!r) { - dev_info(rdev->dev, "GPU reset succeed\n"); + dev_info(rdev->dev, "GPU reset succeeded, trying to resume\n"); radeon_resume(rdev); - radeon_restore_bios_scratch_regs(rdev); - drm_helper_resume_force_mode(rdev->ddev); - ttm_bo_unlock_delayed_workqueue(&rdev->mman.bdev, resched); } - radeon_mutex_unlock(&rdev->cs_mutex); + radeon_restore_bios_scratch_regs(rdev); + + if (!r) { + for (i = 0; i < RADEON_NUM_RINGS; ++i) { + radeon_ring_restore(rdev, &rdev->ring[i], + ring_sizes[i], ring_data[i]); + ring_sizes[i] = 0; + ring_data[i] = NULL; + } + + r = radeon_ib_ring_tests(rdev); + if (r) { + dev_err(rdev->dev, "ib ring test failed (%d).\n", r); + if (saved) { + saved = false; + radeon_suspend(rdev); + goto retry; + } + } + } else { + radeon_fence_driver_force_completion(rdev); + for (i = 0; i < RADEON_NUM_RINGS; ++i) { + kfree(ring_data[i]); + } + } + + radeon_pm_resume(rdev); + drm_helper_resume_force_mode(rdev->ddev); + ttm_bo_unlock_delayed_workqueue(&rdev->mman.bdev, resched); if (r) { /* bad news, how to tell it to userspace ? */ dev_info(rdev->dev, "GPU reset failed\n"); } + up_write(&rdev->exclusive_lock); return r; } |