diff options
Diffstat (limited to 'drivers/gpu/drm/radeon/radeon_ttm.c')
| -rw-r--r-- | drivers/gpu/drm/radeon/radeon_ttm.c | 328 |
1 files changed, 223 insertions, 105 deletions
diff --git a/drivers/gpu/drm/radeon/radeon_ttm.c b/drivers/gpu/drm/radeon/radeon_ttm.c index c421e77ace7..c8a8a5144ec 100644 --- a/drivers/gpu/drm/radeon/radeon_ttm.c +++ b/drivers/gpu/drm/radeon/radeon_ttm.c @@ -38,12 +38,15 @@ #include <drm/radeon_drm.h> #include <linux/seq_file.h> #include <linux/slab.h> +#include <linux/swiotlb.h> +#include <linux/debugfs.h> #include "radeon_reg.h" #include "radeon.h" #define DRM_FILE_PAGE_OFFSET (0x100000000ULL >> PAGE_SHIFT) static int radeon_ttm_debugfs_init(struct radeon_device *rdev); +static void radeon_ttm_debugfs_fini(struct radeon_device *rdev); static struct radeon_device *radeon_get_rdev(struct ttm_bo_device *bdev) { @@ -141,7 +144,7 @@ static int radeon_init_mem_type(struct ttm_bo_device *bdev, uint32_t type, man->flags = TTM_MEMTYPE_FLAG_MAPPABLE | TTM_MEMTYPE_FLAG_CMA; #if __OS_HAS_AGP if (rdev->flags & RADEON_IS_AGP) { - if (!(drm_core_has_AGP(rdev->ddev) && rdev->ddev->agp)) { + if (!rdev->ddev->agp) { DRM_ERROR("AGP is not enabled for memory type %u\n", (unsigned)type); return -EINVAL; @@ -202,7 +205,9 @@ static void radeon_evict_flags(struct ttm_buffer_object *bo, static int radeon_verify_access(struct ttm_buffer_object *bo, struct file *filp) { - return 0; + struct radeon_bo *rbo = container_of(bo, struct radeon_bo, tbo); + + return drm_vma_node_verify_access(&rbo->gem_base.vma_node, filp); } static void radeon_move_null(struct ttm_buffer_object *bo, @@ -216,20 +221,17 @@ static void radeon_move_null(struct ttm_buffer_object *bo, } static int radeon_move_blit(struct ttm_buffer_object *bo, - bool evict, int no_wait_reserve, bool no_wait_gpu, + bool evict, bool no_wait_gpu, struct ttm_mem_reg *new_mem, struct ttm_mem_reg *old_mem) { struct radeon_device *rdev; uint64_t old_start, new_start; struct radeon_fence *fence; - int r, i; + int r, ridx; rdev = radeon_get_rdev(bo->bdev); - r = radeon_fence_create(rdev, &fence, rdev->copy_ring); - if (unlikely(r)) { - return r; - } + ridx = radeon_copy_ring_index(rdev); old_start = old_mem->start << PAGE_SHIFT; new_start = new_mem->start << PAGE_SHIFT; @@ -255,7 +257,7 @@ static int radeon_move_blit(struct ttm_buffer_object *bo, DRM_ERROR("Unknown placement %d\n", old_mem->mem_type); return -EINVAL; } - if (!rdev->ring[rdev->copy_ring].ready) { + if (!rdev->ring[ridx].ready) { DRM_ERROR("Trying to move memory with ring turned off.\n"); return -EINVAL; } @@ -263,48 +265,20 @@ static int radeon_move_blit(struct ttm_buffer_object *bo, BUILD_BUG_ON((PAGE_SIZE % RADEON_GPU_PAGE_SIZE) != 0); /* sync other rings */ - if (rdev->family >= CHIP_R600) { - for (i = 0; i < RADEON_NUM_RINGS; ++i) { - /* no need to sync to our own or unused rings */ - if (i == rdev->copy_ring || !rdev->ring[i].ready) - continue; - - if (!fence->semaphore) { - r = radeon_semaphore_create(rdev, &fence->semaphore); - /* FIXME: handle semaphore error */ - if (r) - continue; - } - - r = radeon_ring_lock(rdev, &rdev->ring[i], 3); - /* FIXME: handle ring lock error */ - if (r) - continue; - radeon_semaphore_emit_signal(rdev, i, fence->semaphore); - radeon_ring_unlock_commit(rdev, &rdev->ring[i]); - - r = radeon_ring_lock(rdev, &rdev->ring[rdev->copy_ring], 3); - /* FIXME: handle ring lock error */ - if (r) - continue; - radeon_semaphore_emit_wait(rdev, rdev->copy_ring, fence->semaphore); - radeon_ring_unlock_commit(rdev, &rdev->ring[rdev->copy_ring]); - } - } - + fence = bo->sync_obj; r = radeon_copy(rdev, old_start, new_start, new_mem->num_pages * (PAGE_SIZE / RADEON_GPU_PAGE_SIZE), /* GPU pages */ - fence); + &fence); /* FIXME: handle copy error */ - r = ttm_bo_move_accel_cleanup(bo, (void *)fence, NULL, - evict, no_wait_reserve, no_wait_gpu, new_mem); + r = ttm_bo_move_accel_cleanup(bo, (void *)fence, + evict, no_wait_gpu, new_mem); radeon_fence_unref(&fence); return r; } static int radeon_move_vram_ram(struct ttm_buffer_object *bo, bool evict, bool interruptible, - bool no_wait_reserve, bool no_wait_gpu, + bool no_wait_gpu, struct ttm_mem_reg *new_mem) { struct radeon_device *rdev; @@ -325,7 +299,7 @@ static int radeon_move_vram_ram(struct ttm_buffer_object *bo, placement.busy_placement = &placements; placements = TTM_PL_MASK_CACHING | TTM_PL_FLAG_TT; r = ttm_bo_mem_space(bo, &placement, &tmp_mem, - interruptible, no_wait_reserve, no_wait_gpu); + interruptible, no_wait_gpu); if (unlikely(r)) { return r; } @@ -339,11 +313,11 @@ static int radeon_move_vram_ram(struct ttm_buffer_object *bo, if (unlikely(r)) { goto out_cleanup; } - r = radeon_move_blit(bo, true, no_wait_reserve, no_wait_gpu, &tmp_mem, old_mem); + r = radeon_move_blit(bo, true, no_wait_gpu, &tmp_mem, old_mem); if (unlikely(r)) { goto out_cleanup; } - r = ttm_bo_move_ttm(bo, true, no_wait_reserve, no_wait_gpu, new_mem); + r = ttm_bo_move_ttm(bo, true, no_wait_gpu, new_mem); out_cleanup: ttm_bo_mem_put(bo, &tmp_mem); return r; @@ -351,7 +325,7 @@ out_cleanup: static int radeon_move_ram_vram(struct ttm_buffer_object *bo, bool evict, bool interruptible, - bool no_wait_reserve, bool no_wait_gpu, + bool no_wait_gpu, struct ttm_mem_reg *new_mem) { struct radeon_device *rdev; @@ -371,15 +345,16 @@ static int radeon_move_ram_vram(struct ttm_buffer_object *bo, placement.num_busy_placement = 1; placement.busy_placement = &placements; placements = TTM_PL_MASK_CACHING | TTM_PL_FLAG_TT; - r = ttm_bo_mem_space(bo, &placement, &tmp_mem, interruptible, no_wait_reserve, no_wait_gpu); + r = ttm_bo_mem_space(bo, &placement, &tmp_mem, + interruptible, no_wait_gpu); if (unlikely(r)) { return r; } - r = ttm_bo_move_ttm(bo, true, no_wait_reserve, no_wait_gpu, &tmp_mem); + r = ttm_bo_move_ttm(bo, true, no_wait_gpu, &tmp_mem); if (unlikely(r)) { goto out_cleanup; } - r = radeon_move_blit(bo, true, no_wait_reserve, no_wait_gpu, new_mem, old_mem); + r = radeon_move_blit(bo, true, no_wait_gpu, new_mem, old_mem); if (unlikely(r)) { goto out_cleanup; } @@ -390,7 +365,7 @@ out_cleanup: static int radeon_bo_move(struct ttm_buffer_object *bo, bool evict, bool interruptible, - bool no_wait_reserve, bool no_wait_gpu, + bool no_wait_gpu, struct ttm_mem_reg *new_mem) { struct radeon_device *rdev; @@ -410,7 +385,8 @@ static int radeon_bo_move(struct ttm_buffer_object *bo, radeon_move_null(bo, new_mem); return 0; } - if (!rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready || rdev->asic->copy == NULL) { + if (!rdev->ring[radeon_copy_ring_index(rdev)].ready || + rdev->asic->copy.copy == NULL) { /* use memcpy */ goto memcpy; } @@ -418,20 +394,26 @@ static int radeon_bo_move(struct ttm_buffer_object *bo, if (old_mem->mem_type == TTM_PL_VRAM && new_mem->mem_type == TTM_PL_SYSTEM) { r = radeon_move_vram_ram(bo, evict, interruptible, - no_wait_reserve, no_wait_gpu, new_mem); + no_wait_gpu, new_mem); } else if (old_mem->mem_type == TTM_PL_SYSTEM && new_mem->mem_type == TTM_PL_VRAM) { r = radeon_move_ram_vram(bo, evict, interruptible, - no_wait_reserve, no_wait_gpu, new_mem); + no_wait_gpu, new_mem); } else { - r = radeon_move_blit(bo, evict, no_wait_reserve, no_wait_gpu, new_mem, old_mem); + r = radeon_move_blit(bo, evict, no_wait_gpu, new_mem, old_mem); } if (r) { memcpy: - r = ttm_bo_move_memcpy(bo, evict, no_wait_reserve, no_wait_gpu, new_mem); + r = ttm_bo_move_memcpy(bo, evict, no_wait_gpu, new_mem); + if (r) { + return r; + } } - return r; + + /* update statistics */ + atomic64_add((u64)bo->num_pages << PAGE_SHIFT, &rdev->num_bytes_moved); + return 0; } static int radeon_ttm_io_mem_reserve(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem) @@ -501,13 +483,12 @@ static void radeon_ttm_io_mem_free(struct ttm_bo_device *bdev, struct ttm_mem_re { } -static int radeon_sync_obj_wait(void *sync_obj, void *sync_arg, - bool lazy, bool interruptible) +static int radeon_sync_obj_wait(void *sync_obj, bool lazy, bool interruptible) { return radeon_fence_wait((struct radeon_fence *)sync_obj, interruptible); } -static int radeon_sync_obj_flush(void *sync_obj, void *sync_arg) +static int radeon_sync_obj_flush(void *sync_obj) { return 0; } @@ -522,7 +503,7 @@ static void *radeon_sync_obj_ref(void *sync_obj) return radeon_fence_ref((struct radeon_fence *)sync_obj); } -static bool radeon_sync_obj_signaled(void *sync_obj, void *sync_arg) +static bool radeon_sync_obj_signaled(void *sync_obj) { return radeon_fence_signaled((struct radeon_fence *)sync_obj); } @@ -579,7 +560,7 @@ static struct ttm_backend_func radeon_backend_func = { .destroy = &radeon_ttm_backend_destroy, }; -struct ttm_tt *radeon_ttm_tt_create(struct ttm_bo_device *bdev, +static struct ttm_tt *radeon_ttm_tt_create(struct ttm_bo_device *bdev, unsigned long size, uint32_t page_flags, struct page *dummy_read_page) { @@ -613,10 +594,18 @@ static int radeon_ttm_tt_populate(struct ttm_tt *ttm) struct radeon_ttm_tt *gtt = (void *)ttm; unsigned i; int r; + bool slave = !!(ttm->page_flags & TTM_PAGE_FLAG_SG); if (ttm->state != tt_unpopulated) return 0; + if (slave && ttm->sg) { + drm_prime_sg_to_page_addr_arrays(ttm->sg, ttm->pages, + gtt->ttm.dma_address, ttm->num_pages); + ttm->state = tt_unbound; + return 0; + } + rdev = radeon_get_rdev(ttm->bdev); #if __OS_HAS_AGP if (rdev->flags & RADEON_IS_AGP) { @@ -657,6 +646,10 @@ static void radeon_ttm_tt_unpopulate(struct ttm_tt *ttm) struct radeon_device *rdev; struct radeon_ttm_tt *gtt = (void *)ttm; unsigned i; + bool slave = !!(ttm->page_flags & TTM_PAGE_FLAG_SG); + + if (slave) + return; rdev = radeon_get_rdev(ttm->bdev); #if __OS_HAS_AGP @@ -714,7 +707,9 @@ int radeon_ttm_init(struct radeon_device *rdev) /* No others user of address space so set it to 0 */ r = ttm_bo_device_init(&rdev->mman.bdev, rdev->mman.bo_global_ref.ref.object, - &radeon_bo_driver, DRM_FILE_PAGE_OFFSET, + &radeon_bo_driver, + rdev->ddev->anon_inode->i_mapping, + DRM_FILE_PAGE_OFFSET, rdev->need_dma32); if (r) { DRM_ERROR("failed initializing buffer object driver(%d).\n", r); @@ -727,9 +722,12 @@ int radeon_ttm_init(struct radeon_device *rdev) DRM_ERROR("Failed initializing VRAM heap.\n"); return r; } + /* Change the size here instead of the init above so only lpfn is affected */ + radeon_ttm_set_active_vram_size(rdev, rdev->mc.visible_vram_size); + r = radeon_bo_create(rdev, 256 * 1024, PAGE_SIZE, true, - RADEON_GEM_DOMAIN_VRAM, - &rdev->stollen_vga_memory); + RADEON_GEM_DOMAIN_VRAM, + NULL, &rdev->stollen_vga_memory); if (r) { return r; } @@ -743,7 +741,7 @@ int radeon_ttm_init(struct radeon_device *rdev) return r; } DRM_INFO("radeon: %uM of VRAM memory ready\n", - (unsigned)rdev->mc.real_vram_size / (1024 * 1024)); + (unsigned) (rdev->mc.real_vram_size / (1024 * 1024))); r = ttm_bo_init_mm(&rdev->mman.bdev, TTM_PL_TT, rdev->mc.gtt_size >> PAGE_SHIFT); if (r) { @@ -752,9 +750,6 @@ int radeon_ttm_init(struct radeon_device *rdev) } DRM_INFO("radeon: %uM of GTT memory ready.\n", (unsigned)(rdev->mc.gtt_size / (1024 * 1024))); - if (unlikely(rdev->mman.bdev.dev_mapping == NULL)) { - rdev->mman.bdev.dev_mapping = rdev->ddev->dev_mapping; - } r = radeon_ttm_debugfs_init(rdev); if (r) { @@ -770,6 +765,7 @@ void radeon_ttm_fini(struct radeon_device *rdev) if (!rdev->mman.initialized) return; + radeon_ttm_debugfs_fini(rdev); if (rdev->stollen_vga_memory) { r = radeon_bo_reserve(rdev->stollen_vga_memory, false); if (r == 0) { @@ -815,9 +811,9 @@ static int radeon_ttm_fault(struct vm_area_struct *vma, struct vm_fault *vmf) return VM_FAULT_NOPAGE; } rdev = radeon_get_rdev(bo->bdev); - mutex_lock(&rdev->vram_mutex); + down_read(&rdev->pm.mclk_lock); r = ttm_vm_ops->fault(vma, vmf); - mutex_unlock(&rdev->vram_mutex); + up_read(&rdev->pm.mclk_lock); return r; } @@ -849,16 +845,15 @@ int radeon_mmap(struct file *filp, struct vm_area_struct *vma) return 0; } - -#define RADEON_DEBUGFS_MEM_TYPES 2 - #if defined(CONFIG_DEBUG_FS) + static int radeon_mm_dump_table(struct seq_file *m, void *data) { struct drm_info_node *node = (struct drm_info_node *)m->private; - struct drm_mm *mm = (struct drm_mm *)node->info_ent->data; + unsigned ttm_pl = *(int *)node->info_ent->data; struct drm_device *dev = node->minor->dev; struct radeon_device *rdev = dev->dev_private; + struct drm_mm *mm = (struct drm_mm *)rdev->mman.bdev.man[ttm_pl].priv; int ret; struct ttm_bo_global *glob = rdev->mman.bdev.glob; @@ -867,46 +862,169 @@ static int radeon_mm_dump_table(struct seq_file *m, void *data) spin_unlock(&glob->lru_lock); return ret; } + +static int ttm_pl_vram = TTM_PL_VRAM; +static int ttm_pl_tt = TTM_PL_TT; + +static struct drm_info_list radeon_ttm_debugfs_list[] = { + {"radeon_vram_mm", radeon_mm_dump_table, 0, &ttm_pl_vram}, + {"radeon_gtt_mm", radeon_mm_dump_table, 0, &ttm_pl_tt}, + {"ttm_page_pool", ttm_page_alloc_debugfs, 0, NULL}, +#ifdef CONFIG_SWIOTLB + {"ttm_dma_page_pool", ttm_dma_page_alloc_debugfs, 0, NULL} #endif +}; -static int radeon_ttm_debugfs_init(struct radeon_device *rdev) +static int radeon_ttm_vram_open(struct inode *inode, struct file *filep) { -#if defined(CONFIG_DEBUG_FS) - static struct drm_info_list radeon_mem_types_list[RADEON_DEBUGFS_MEM_TYPES+2]; - static char radeon_mem_types_names[RADEON_DEBUGFS_MEM_TYPES+2][32]; - unsigned i; + struct radeon_device *rdev = inode->i_private; + i_size_write(inode, rdev->mc.mc_vram_size); + filep->private_data = inode->i_private; + return 0; +} - for (i = 0; i < RADEON_DEBUGFS_MEM_TYPES; i++) { - if (i == 0) - sprintf(radeon_mem_types_names[i], "radeon_vram_mm"); - else - sprintf(radeon_mem_types_names[i], "radeon_gtt_mm"); - radeon_mem_types_list[i].name = radeon_mem_types_names[i]; - radeon_mem_types_list[i].show = &radeon_mm_dump_table; - radeon_mem_types_list[i].driver_features = 0; - if (i == 0) - radeon_mem_types_list[i].data = rdev->mman.bdev.man[TTM_PL_VRAM].priv; - else - radeon_mem_types_list[i].data = rdev->mman.bdev.man[TTM_PL_TT].priv; +static ssize_t radeon_ttm_vram_read(struct file *f, char __user *buf, + size_t size, loff_t *pos) +{ + struct radeon_device *rdev = f->private_data; + ssize_t result = 0; + int r; + + if (size & 0x3 || *pos & 0x3) + return -EINVAL; + + while (size) { + unsigned long flags; + uint32_t value; + + if (*pos >= rdev->mc.mc_vram_size) + return result; + + spin_lock_irqsave(&rdev->mmio_idx_lock, flags); + WREG32(RADEON_MM_INDEX, ((uint32_t)*pos) | 0x80000000); + if (rdev->family >= CHIP_CEDAR) + WREG32(EVERGREEN_MM_INDEX_HI, *pos >> 31); + value = RREG32(RADEON_MM_DATA); + spin_unlock_irqrestore(&rdev->mmio_idx_lock, flags); + + r = put_user(value, (uint32_t *)buf); + if (r) + return r; + result += 4; + buf += 4; + *pos += 4; + size -= 4; } - /* Add ttm page pool to debugfs */ - sprintf(radeon_mem_types_names[i], "ttm_page_pool"); - radeon_mem_types_list[i].name = radeon_mem_types_names[i]; - radeon_mem_types_list[i].show = &ttm_page_alloc_debugfs; - radeon_mem_types_list[i].driver_features = 0; - radeon_mem_types_list[i++].data = NULL; -#ifdef CONFIG_SWIOTLB - if (swiotlb_nr_tbl()) { - sprintf(radeon_mem_types_names[i], "ttm_dma_page_pool"); - radeon_mem_types_list[i].name = radeon_mem_types_names[i]; - radeon_mem_types_list[i].show = &ttm_dma_page_alloc_debugfs; - radeon_mem_types_list[i].driver_features = 0; - radeon_mem_types_list[i++].data = NULL; + + return result; +} + +static const struct file_operations radeon_ttm_vram_fops = { + .owner = THIS_MODULE, + .open = radeon_ttm_vram_open, + .read = radeon_ttm_vram_read, + .llseek = default_llseek +}; + +static int radeon_ttm_gtt_open(struct inode *inode, struct file *filep) +{ + struct radeon_device *rdev = inode->i_private; + i_size_write(inode, rdev->mc.gtt_size); + filep->private_data = inode->i_private; + return 0; +} + +static ssize_t radeon_ttm_gtt_read(struct file *f, char __user *buf, + size_t size, loff_t *pos) +{ + struct radeon_device *rdev = f->private_data; + ssize_t result = 0; + int r; + + while (size) { + loff_t p = *pos / PAGE_SIZE; + unsigned off = *pos & ~PAGE_MASK; + size_t cur_size = min_t(size_t, size, PAGE_SIZE - off); + struct page *page; + void *ptr; + + if (p >= rdev->gart.num_cpu_pages) + return result; + + page = rdev->gart.pages[p]; + if (page) { + ptr = kmap(page); + ptr += off; + + r = copy_to_user(buf, ptr, cur_size); + kunmap(rdev->gart.pages[p]); + } else + r = clear_user(buf, cur_size); + + if (r) + return -EFAULT; + + result += cur_size; + buf += cur_size; + *pos += cur_size; + size -= cur_size; } + + return result; +} + +static const struct file_operations radeon_ttm_gtt_fops = { + .owner = THIS_MODULE, + .open = radeon_ttm_gtt_open, + .read = radeon_ttm_gtt_read, + .llseek = default_llseek +}; + #endif - return radeon_debugfs_add_files(rdev, radeon_mem_types_list, i); +static int radeon_ttm_debugfs_init(struct radeon_device *rdev) +{ +#if defined(CONFIG_DEBUG_FS) + unsigned count; + + struct drm_minor *minor = rdev->ddev->primary; + struct dentry *ent, *root = minor->debugfs_root; + + ent = debugfs_create_file("radeon_vram", S_IFREG | S_IRUGO, root, + rdev, &radeon_ttm_vram_fops); + if (IS_ERR(ent)) + return PTR_ERR(ent); + rdev->mman.vram = ent; + + ent = debugfs_create_file("radeon_gtt", S_IFREG | S_IRUGO, root, + rdev, &radeon_ttm_gtt_fops); + if (IS_ERR(ent)) + return PTR_ERR(ent); + rdev->mman.gtt = ent; + + count = ARRAY_SIZE(radeon_ttm_debugfs_list); + +#ifdef CONFIG_SWIOTLB + if (!swiotlb_nr_tbl()) + --count; #endif + + return radeon_debugfs_add_files(rdev, radeon_ttm_debugfs_list, count); +#else + return 0; +#endif +} + +static void radeon_ttm_debugfs_fini(struct radeon_device *rdev) +{ +#if defined(CONFIG_DEBUG_FS) + + debugfs_remove(rdev->mman.vram); + rdev->mman.vram = NULL; + + debugfs_remove(rdev->mman.gtt); + rdev->mman.gtt = NULL; +#endif } |