diff options
Diffstat (limited to 'drivers/gpu/drm/ttm/ttm_tt.c')
| -rw-r--r-- | drivers/gpu/drm/ttm/ttm_tt.c | 491 |
1 files changed, 152 insertions, 339 deletions
diff --git a/drivers/gpu/drm/ttm/ttm_tt.c b/drivers/gpu/drm/ttm/ttm_tt.c index 9c2b1cc5dba..75f31909004 100644 --- a/drivers/gpu/drm/ttm/ttm_tt.c +++ b/drivers/gpu/drm/ttm/ttm_tt.c @@ -28,191 +28,68 @@ * Authors: Thomas Hellstrom <thellstrom-at-vmware-dot-com> */ -#include <linux/vmalloc.h> +#define pr_fmt(fmt) "[TTM] " fmt + #include <linux/sched.h> #include <linux/highmem.h> #include <linux/pagemap.h> +#include <linux/shmem_fs.h> #include <linux/file.h> #include <linux/swap.h> -#include "drm_cache.h" -#include "ttm/ttm_module.h" -#include "ttm/ttm_bo_driver.h" -#include "ttm/ttm_placement.h" - -static int ttm_tt_swapin(struct ttm_tt *ttm); +#include <linux/slab.h> +#include <linux/export.h> +#include <drm/drm_cache.h> +#include <drm/drm_mem_util.h> +#include <drm/ttm/ttm_module.h> +#include <drm/ttm/ttm_bo_driver.h> +#include <drm/ttm/ttm_placement.h> +#include <drm/ttm/ttm_page_alloc.h> /** * Allocates storage for pointers to the pages that back the ttm. - * - * Uses kmalloc if possible. Otherwise falls back to vmalloc. */ static void ttm_tt_alloc_page_directory(struct ttm_tt *ttm) { - unsigned long size = ttm->num_pages * sizeof(*ttm->pages); - ttm->pages = NULL; - - if (size <= PAGE_SIZE) - ttm->pages = kzalloc(size, GFP_KERNEL); - - if (!ttm->pages) { - ttm->pages = vmalloc_user(size); - if (ttm->pages) - ttm->page_flags |= TTM_PAGE_FLAG_VMALLOC; - } -} - -static void ttm_tt_free_page_directory(struct ttm_tt *ttm) -{ - if (ttm->page_flags & TTM_PAGE_FLAG_VMALLOC) { - vfree(ttm->pages); - ttm->page_flags &= ~TTM_PAGE_FLAG_VMALLOC; - } else { - kfree(ttm->pages); - } - ttm->pages = NULL; -} - -static struct page *ttm_tt_alloc_page(unsigned page_flags) -{ - gfp_t gfp_flags = GFP_USER; - - if (page_flags & TTM_PAGE_FLAG_ZERO_ALLOC) - gfp_flags |= __GFP_ZERO; - - if (page_flags & TTM_PAGE_FLAG_DMA32) - gfp_flags |= __GFP_DMA32; - else - gfp_flags |= __GFP_HIGHMEM; - - return alloc_page(gfp_flags); -} - -static void ttm_tt_free_user_pages(struct ttm_tt *ttm) -{ - int write; - int dirty; - struct page *page; - int i; - struct ttm_backend *be = ttm->be; - - BUG_ON(!(ttm->page_flags & TTM_PAGE_FLAG_USER)); - write = ((ttm->page_flags & TTM_PAGE_FLAG_WRITE) != 0); - dirty = ((ttm->page_flags & TTM_PAGE_FLAG_USER_DIRTY) != 0); - - if (be) - be->func->clear(be); - - for (i = 0; i < ttm->num_pages; ++i) { - page = ttm->pages[i]; - if (page == NULL) - continue; - - if (page == ttm->dummy_read_page) { - BUG_ON(write); - continue; - } - - if (write && dirty && !PageReserved(page)) - set_page_dirty_lock(page); - - ttm->pages[i] = NULL; - ttm_mem_global_free(ttm->glob->mem_glob, PAGE_SIZE); - put_page(page); - } - ttm->state = tt_unpopulated; - ttm->first_himem_page = ttm->num_pages; - ttm->last_lomem_page = -1; -} - -static struct page *__ttm_tt_get_page(struct ttm_tt *ttm, int index) -{ - struct page *p; - struct ttm_mem_global *mem_glob = ttm->glob->mem_glob; - int ret; - - while (NULL == (p = ttm->pages[index])) { - p = ttm_tt_alloc_page(ttm->page_flags); - - if (!p) - return NULL; - - ret = ttm_mem_global_alloc_page(mem_glob, p, false, false); - if (unlikely(ret != 0)) - goto out_err; - - if (PageHighMem(p)) - ttm->pages[--ttm->first_himem_page] = p; - else - ttm->pages[++ttm->last_lomem_page] = p; - } - return p; -out_err: - put_page(p); - return NULL; + ttm->pages = drm_calloc_large(ttm->num_pages, sizeof(void*)); } -struct page *ttm_tt_get_page(struct ttm_tt *ttm, int index) +static void ttm_dma_tt_alloc_page_directory(struct ttm_dma_tt *ttm) { - int ret; - - if (unlikely(ttm->page_flags & TTM_PAGE_FLAG_SWAPPED)) { - ret = ttm_tt_swapin(ttm); - if (unlikely(ret != 0)) - return NULL; - } - return __ttm_tt_get_page(ttm, index); + ttm->ttm.pages = drm_calloc_large(ttm->ttm.num_pages, sizeof(void*)); + ttm->dma_address = drm_calloc_large(ttm->ttm.num_pages, + sizeof(*ttm->dma_address)); } -int ttm_tt_populate(struct ttm_tt *ttm) +#ifdef CONFIG_X86 +static inline int ttm_tt_set_page_caching(struct page *p, + enum ttm_caching_state c_old, + enum ttm_caching_state c_new) { - struct page *page; - unsigned long i; - struct ttm_backend *be; - int ret; + int ret = 0; - if (ttm->state != tt_unpopulated) + if (PageHighMem(p)) return 0; - if (unlikely(ttm->page_flags & TTM_PAGE_FLAG_SWAPPED)) { - ret = ttm_tt_swapin(ttm); - if (unlikely(ret != 0)) - return ret; - } - - be = ttm->be; + if (c_old != tt_cached) { + /* p isn't in the default caching state, set it to + * writeback first to free its current memtype. */ - for (i = 0; i < ttm->num_pages; ++i) { - page = __ttm_tt_get_page(ttm, i); - if (!page) - return -ENOMEM; + ret = set_pages_wb(p, 1); + if (ret) + return ret; } - be->func->populate(be, ttm->num_pages, ttm->pages, - ttm->dummy_read_page); - ttm->state = tt_unbound; - return 0; -} -EXPORT_SYMBOL(ttm_tt_populate); - -#ifdef CONFIG_X86 -static inline int ttm_tt_set_page_caching(struct page *p, - enum ttm_caching_state c_state) -{ - if (PageHighMem(p)) - return 0; + if (c_new == tt_wc) + ret = set_memory_wc((unsigned long) page_address(p), 1); + else if (c_new == tt_uncached) + ret = set_pages_uc(p, 1); - switch (c_state) { - case tt_cached: - return set_pages_wb(p, 1); - case tt_wc: - return set_memory_wc((unsigned long) page_address(p), 1); - default: - return set_pages_uc(p, 1); - } + return ret; } #else /* CONFIG_X86 */ static inline int ttm_tt_set_page_caching(struct page *p, - enum ttm_caching_state c_state) + enum ttm_caching_state c_old, + enum ttm_caching_state c_new) { return 0; } @@ -233,10 +110,10 @@ static int ttm_tt_set_caching(struct ttm_tt *ttm, if (ttm->caching_state == c_state) return 0; - if (c_state != tt_cached) { - ret = ttm_tt_populate(ttm); - if (unlikely(ret != 0)) - return ret; + if (ttm->state == tt_unpopulated) { + /* Change caching but don't populate */ + ttm->caching_state = c_state; + return 0; } if (ttm->caching_state == tt_cached) @@ -245,7 +122,9 @@ static int ttm_tt_set_caching(struct ttm_tt *ttm, for (i = 0; i < ttm->num_pages; ++i) { cur_page = ttm->pages[i]; if (likely(cur_page != NULL)) { - ret = ttm_tt_set_page_caching(cur_page, c_state); + ret = ttm_tt_set_page_caching(cur_page, + ttm->caching_state, + c_state); if (unlikely(ret != 0)) goto out_err; } @@ -259,7 +138,7 @@ out_err: for (j = 0; j < i; ++j) { cur_page = ttm->pages[j]; if (likely(cur_page != NULL)) { - (void)ttm_tt_set_page_caching(cur_page, + (void)ttm_tt_set_page_caching(cur_page, c_state, ttm->caching_state); } } @@ -282,146 +161,99 @@ int ttm_tt_set_placement_caching(struct ttm_tt *ttm, uint32_t placement) } EXPORT_SYMBOL(ttm_tt_set_placement_caching); -static void ttm_tt_free_alloced_pages(struct ttm_tt *ttm) -{ - int i; - struct page *cur_page; - struct ttm_backend *be = ttm->be; - - if (be) - be->func->clear(be); - (void)ttm_tt_set_caching(ttm, tt_cached); - for (i = 0; i < ttm->num_pages; ++i) { - cur_page = ttm->pages[i]; - ttm->pages[i] = NULL; - if (cur_page) { - if (page_count(cur_page) != 1) - printk(KERN_ERR TTM_PFX - "Erroneous page count. " - "Leaking pages.\n"); - ttm_mem_global_free_page(ttm->glob->mem_glob, - cur_page); - __free_page(cur_page); - } - } - ttm->state = tt_unpopulated; - ttm->first_himem_page = ttm->num_pages; - ttm->last_lomem_page = -1; -} - void ttm_tt_destroy(struct ttm_tt *ttm) { - struct ttm_backend *be; - if (unlikely(ttm == NULL)) return; - be = ttm->be; - if (likely(be != NULL)) { - be->func->destroy(be); - ttm->be = NULL; + if (ttm->state == tt_bound) { + ttm_tt_unbind(ttm); } - if (likely(ttm->pages != NULL)) { - if (ttm->page_flags & TTM_PAGE_FLAG_USER) - ttm_tt_free_user_pages(ttm); - else - ttm_tt_free_alloced_pages(ttm); + if (ttm->state == tt_unbound) + ttm_tt_unpopulate(ttm); - ttm_tt_free_page_directory(ttm); - } - - if (!(ttm->page_flags & TTM_PAGE_FLAG_PERSISTANT_SWAP) && + if (!(ttm->page_flags & TTM_PAGE_FLAG_PERSISTENT_SWAP) && ttm->swap_storage) fput(ttm->swap_storage); - kfree(ttm); + ttm->swap_storage = NULL; + ttm->func->destroy(ttm); } -int ttm_tt_set_user(struct ttm_tt *ttm, - struct task_struct *tsk, - unsigned long start, unsigned long num_pages) +int ttm_tt_init(struct ttm_tt *ttm, struct ttm_bo_device *bdev, + unsigned long size, uint32_t page_flags, + struct page *dummy_read_page) { - struct mm_struct *mm = tsk->mm; - int ret; - int write = (ttm->page_flags & TTM_PAGE_FLAG_WRITE) != 0; - struct ttm_mem_global *mem_glob = ttm->glob->mem_glob; - - BUG_ON(num_pages != ttm->num_pages); - BUG_ON((ttm->page_flags & TTM_PAGE_FLAG_USER) == 0); - - /** - * Account user pages as lowmem pages for now. - */ - - ret = ttm_mem_global_alloc(mem_glob, num_pages * PAGE_SIZE, - false, false); - if (unlikely(ret != 0)) - return ret; - - down_read(&mm->mmap_sem); - ret = get_user_pages(tsk, mm, start, num_pages, - write, 0, ttm->pages, NULL); - up_read(&mm->mmap_sem); + ttm->bdev = bdev; + ttm->glob = bdev->glob; + ttm->num_pages = (size + PAGE_SIZE - 1) >> PAGE_SHIFT; + ttm->caching_state = tt_cached; + ttm->page_flags = page_flags; + ttm->dummy_read_page = dummy_read_page; + ttm->state = tt_unpopulated; + ttm->swap_storage = NULL; - if (ret != num_pages && write) { - ttm_tt_free_user_pages(ttm); - ttm_mem_global_free(mem_glob, num_pages * PAGE_SIZE); + ttm_tt_alloc_page_directory(ttm); + if (!ttm->pages) { + ttm_tt_destroy(ttm); + pr_err("Failed allocating page table\n"); return -ENOMEM; } - - ttm->tsk = tsk; - ttm->start = start; - ttm->state = tt_unbound; - return 0; } +EXPORT_SYMBOL(ttm_tt_init); -struct ttm_tt *ttm_tt_create(struct ttm_bo_device *bdev, unsigned long size, - uint32_t page_flags, struct page *dummy_read_page) +void ttm_tt_fini(struct ttm_tt *ttm) { - struct ttm_bo_driver *bo_driver = bdev->driver; - struct ttm_tt *ttm; - - if (!bo_driver) - return NULL; + drm_free_large(ttm->pages); + ttm->pages = NULL; +} +EXPORT_SYMBOL(ttm_tt_fini); - ttm = kzalloc(sizeof(*ttm), GFP_KERNEL); - if (!ttm) - return NULL; +int ttm_dma_tt_init(struct ttm_dma_tt *ttm_dma, struct ttm_bo_device *bdev, + unsigned long size, uint32_t page_flags, + struct page *dummy_read_page) +{ + struct ttm_tt *ttm = &ttm_dma->ttm; + ttm->bdev = bdev; ttm->glob = bdev->glob; ttm->num_pages = (size + PAGE_SIZE - 1) >> PAGE_SHIFT; - ttm->first_himem_page = ttm->num_pages; - ttm->last_lomem_page = -1; ttm->caching_state = tt_cached; ttm->page_flags = page_flags; - ttm->dummy_read_page = dummy_read_page; + ttm->state = tt_unpopulated; + ttm->swap_storage = NULL; - ttm_tt_alloc_page_directory(ttm); - if (!ttm->pages) { + INIT_LIST_HEAD(&ttm_dma->pages_list); + ttm_dma_tt_alloc_page_directory(ttm_dma); + if (!ttm->pages || !ttm_dma->dma_address) { ttm_tt_destroy(ttm); - printk(KERN_ERR TTM_PFX "Failed allocating page table\n"); - return NULL; - } - ttm->be = bo_driver->create_ttm_backend_entry(bdev); - if (!ttm->be) { - ttm_tt_destroy(ttm); - printk(KERN_ERR TTM_PFX "Failed creating ttm backend entry\n"); - return NULL; + pr_err("Failed allocating page table\n"); + return -ENOMEM; } - ttm->state = tt_unpopulated; - return ttm; + return 0; +} +EXPORT_SYMBOL(ttm_dma_tt_init); + +void ttm_dma_tt_fini(struct ttm_dma_tt *ttm_dma) +{ + struct ttm_tt *ttm = &ttm_dma->ttm; + + drm_free_large(ttm->pages); + ttm->pages = NULL; + drm_free_large(ttm_dma->dma_address); + ttm_dma->dma_address = NULL; } +EXPORT_SYMBOL(ttm_dma_tt_fini); void ttm_tt_unbind(struct ttm_tt *ttm) { int ret; - struct ttm_backend *be = ttm->be; if (ttm->state == tt_bound) { - ret = be->func->unbind(be); + ret = ttm->func->unbind(ttm); BUG_ON(ret); ttm->state = tt_unbound; } @@ -430,7 +262,6 @@ void ttm_tt_unbind(struct ttm_tt *ttm) int ttm_tt_bind(struct ttm_tt *ttm, struct ttm_mem_reg *bo_mem) { int ret = 0; - struct ttm_backend *be; if (!ttm) return -EINVAL; @@ -438,149 +269,131 @@ int ttm_tt_bind(struct ttm_tt *ttm, struct ttm_mem_reg *bo_mem) if (ttm->state == tt_bound) return 0; - be = ttm->be; - - ret = ttm_tt_populate(ttm); + ret = ttm->bdev->driver->ttm_tt_populate(ttm); if (ret) return ret; - ret = be->func->bind(be, bo_mem); - if (ret) { - printk(KERN_ERR TTM_PFX "Couldn't bind backend.\n"); + ret = ttm->func->bind(ttm, bo_mem); + if (unlikely(ret != 0)) return ret; - } ttm->state = tt_bound; - if (ttm->page_flags & TTM_PAGE_FLAG_USER) - ttm->page_flags |= TTM_PAGE_FLAG_USER_DIRTY; return 0; } EXPORT_SYMBOL(ttm_tt_bind); -static int ttm_tt_swapin(struct ttm_tt *ttm) +int ttm_tt_swapin(struct ttm_tt *ttm) { struct address_space *swap_space; struct file *swap_storage; struct page *from_page; struct page *to_page; - void *from_virtual; - void *to_virtual; int i; - int ret; - - if (ttm->page_flags & TTM_PAGE_FLAG_USER) { - ret = ttm_tt_set_user(ttm, ttm->tsk, ttm->start, - ttm->num_pages); - if (unlikely(ret != 0)) - return ret; - - ttm->page_flags &= ~TTM_PAGE_FLAG_SWAPPED; - return 0; - } + int ret = -ENOMEM; swap_storage = ttm->swap_storage; BUG_ON(swap_storage == NULL); - swap_space = swap_storage->f_path.dentry->d_inode->i_mapping; + swap_space = file_inode(swap_storage)->i_mapping; for (i = 0; i < ttm->num_pages; ++i) { - from_page = read_mapping_page(swap_space, i, NULL); - if (IS_ERR(from_page)) + from_page = shmem_read_mapping_page(swap_space, i); + if (IS_ERR(from_page)) { + ret = PTR_ERR(from_page); goto out_err; - to_page = __ttm_tt_get_page(ttm, i); + } + to_page = ttm->pages[i]; if (unlikely(to_page == NULL)) goto out_err; - preempt_disable(); - from_virtual = kmap_atomic(from_page, KM_USER0); - to_virtual = kmap_atomic(to_page, KM_USER1); - memcpy(to_virtual, from_virtual, PAGE_SIZE); - kunmap_atomic(to_virtual, KM_USER1); - kunmap_atomic(from_virtual, KM_USER0); - preempt_enable(); + copy_highpage(to_page, from_page); page_cache_release(from_page); } - if (!(ttm->page_flags & TTM_PAGE_FLAG_PERSISTANT_SWAP)) + if (!(ttm->page_flags & TTM_PAGE_FLAG_PERSISTENT_SWAP)) fput(swap_storage); ttm->swap_storage = NULL; ttm->page_flags &= ~TTM_PAGE_FLAG_SWAPPED; return 0; out_err: - ttm_tt_free_alloced_pages(ttm); - return -ENOMEM; + return ret; } -int ttm_tt_swapout(struct ttm_tt *ttm, struct file *persistant_swap_storage) +int ttm_tt_swapout(struct ttm_tt *ttm, struct file *persistent_swap_storage) { struct address_space *swap_space; struct file *swap_storage; struct page *from_page; struct page *to_page; - void *from_virtual; - void *to_virtual; int i; + int ret = -ENOMEM; BUG_ON(ttm->state != tt_unbound && ttm->state != tt_unpopulated); BUG_ON(ttm->caching_state != tt_cached); - /* - * For user buffers, just unpin the pages, as there should be - * vma references. - */ - - if (ttm->page_flags & TTM_PAGE_FLAG_USER) { - ttm_tt_free_user_pages(ttm); - ttm->page_flags |= TTM_PAGE_FLAG_SWAPPED; - ttm->swap_storage = NULL; - return 0; - } - - if (!persistant_swap_storage) { + if (!persistent_swap_storage) { swap_storage = shmem_file_setup("ttm swap", ttm->num_pages << PAGE_SHIFT, 0); if (unlikely(IS_ERR(swap_storage))) { - printk(KERN_ERR "Failed allocating swap storage.\n"); - return -ENOMEM; + pr_err("Failed allocating swap storage\n"); + return PTR_ERR(swap_storage); } } else - swap_storage = persistant_swap_storage; + swap_storage = persistent_swap_storage; - swap_space = swap_storage->f_path.dentry->d_inode->i_mapping; + swap_space = file_inode(swap_storage)->i_mapping; for (i = 0; i < ttm->num_pages; ++i) { from_page = ttm->pages[i]; if (unlikely(from_page == NULL)) continue; - to_page = read_mapping_page(swap_space, i, NULL); - if (unlikely(to_page == NULL)) + to_page = shmem_read_mapping_page(swap_space, i); + if (unlikely(IS_ERR(to_page))) { + ret = PTR_ERR(to_page); goto out_err; - - preempt_disable(); - from_virtual = kmap_atomic(from_page, KM_USER0); - to_virtual = kmap_atomic(to_page, KM_USER1); - memcpy(to_virtual, from_virtual, PAGE_SIZE); - kunmap_atomic(to_virtual, KM_USER1); - kunmap_atomic(from_virtual, KM_USER0); - preempt_enable(); + } + copy_highpage(to_page, from_page); set_page_dirty(to_page); mark_page_accessed(to_page); page_cache_release(to_page); } - ttm_tt_free_alloced_pages(ttm); + ttm_tt_unpopulate(ttm); ttm->swap_storage = swap_storage; ttm->page_flags |= TTM_PAGE_FLAG_SWAPPED; - if (persistant_swap_storage) - ttm->page_flags |= TTM_PAGE_FLAG_PERSISTANT_SWAP; + if (persistent_swap_storage) + ttm->page_flags |= TTM_PAGE_FLAG_PERSISTENT_SWAP; return 0; out_err: - if (!persistant_swap_storage) + if (!persistent_swap_storage) fput(swap_storage); - return -ENOMEM; + return ret; +} + +static void ttm_tt_clear_mapping(struct ttm_tt *ttm) +{ + pgoff_t i; + struct page **page = ttm->pages; + + if (ttm->page_flags & TTM_PAGE_FLAG_SG) + return; + + for (i = 0; i < ttm->num_pages; ++i) { + (*page)->mapping = NULL; + (*page++)->index = 0; + } +} + +void ttm_tt_unpopulate(struct ttm_tt *ttm) +{ + if (ttm->state == tt_unpopulated) + return; + + ttm_tt_clear_mapping(ttm); + ttm->bdev->driver->ttm_tt_unpopulate(ttm); } |