diff options
Diffstat (limited to 'drivers/gpu/drm/ttm/ttm_page_alloc.c')
| -rw-r--r-- | drivers/gpu/drm/ttm/ttm_page_alloc.c | 298 |
1 files changed, 187 insertions, 111 deletions
diff --git a/drivers/gpu/drm/ttm/ttm_page_alloc.c b/drivers/gpu/drm/ttm/ttm_page_alloc.c index b1e02fffd3c..863bef9f923 100644 --- a/drivers/gpu/drm/ttm/ttm_page_alloc.c +++ b/drivers/gpu/drm/ttm/ttm_page_alloc.c @@ -30,6 +30,9 @@ * - Use page->lru to keep a free list * - doesn't track currently in use pages */ + +#define pr_fmt(fmt) "[TTM] " fmt + #include <linux/list.h> #include <linux/spinlock.h> #include <linux/highmem.h> @@ -38,11 +41,12 @@ #include <linux/mm.h> #include <linux/seq_file.h> /* for seq_printf */ #include <linux/slab.h> +#include <linux/dma-mapping.h> -#include <asm/atomic.h> +#include <linux/atomic.h> -#include "ttm/ttm_bo_driver.h" -#include "ttm/ttm_page_alloc.h" +#include <drm/ttm/ttm_bo_driver.h> +#include <drm/ttm/ttm_page_alloc.h> #ifdef TTM_HAS_AGP #include <asm/agp.h> @@ -166,18 +170,13 @@ static ssize_t ttm_pool_store(struct kobject *kobj, m->options.small = val; else if (attr == &ttm_page_pool_alloc_size) { if (val > NUM_PAGES_TO_ALLOC*8) { - printk(KERN_ERR TTM_PFX - "Setting allocation size to %lu " - "is not allowed. Recommended size is " - "%lu\n", + pr_err("Setting allocation size to %lu is not allowed. Recommended size is %lu\n", NUM_PAGES_TO_ALLOC*(PAGE_SIZE >> 7), NUM_PAGES_TO_ALLOC*(PAGE_SIZE >> 10)); return size; } else if (val > NUM_PAGES_TO_ALLOC) { - printk(KERN_WARNING TTM_PFX - "Setting allocation size to " - "larger than %lu is not recommended.\n", - NUM_PAGES_TO_ALLOC*(PAGE_SIZE >> 10)); + pr_warn("Setting allocation size to larger than %lu is not recommended\n", + NUM_PAGES_TO_ALLOC*(PAGE_SIZE >> 10)); } m->options.alloc_size = val; } @@ -278,8 +277,7 @@ static void ttm_pages_put(struct page *pages[], unsigned npages) { unsigned i; if (set_pages_array_wb(pages, npages)) - printk(KERN_ERR TTM_PFX "Failed to set %d pages to wb!\n", - npages); + pr_err("Failed to set %d pages to wb!\n", npages); for (i = 0; i < npages; ++i) __free_page(pages[i]); } @@ -314,8 +312,7 @@ static int ttm_page_pool_free(struct ttm_page_pool *pool, unsigned nr_free) pages_to_free = kmalloc(npages_to_free * sizeof(struct page *), GFP_KERNEL); if (!pages_to_free) { - printk(KERN_ERR TTM_PFX - "Failed to allocate memory for pool free operation.\n"); + pr_err("Failed to allocate memory for pool free operation\n"); return 0; } @@ -354,7 +351,7 @@ restart: if (nr_free) goto restart; - /* Not allowed to fall tough or break because + /* Not allowed to fall through or break because * following context is inside spinlock while we are * outside here. */ @@ -380,26 +377,26 @@ out: return nr_free; } -/* Get good estimation how many pages are free in pools */ -static int ttm_pool_get_num_unused_pages(void) -{ - unsigned i; - int total = 0; - for (i = 0; i < NUM_POOLS; ++i) - total += _manager->pools[i].npages; - - return total; -} - /** * Callback for mm to request pool to reduce number of page held. + * + * XXX: (dchinner) Deadlock warning! + * + * ttm_page_pool_free() does memory allocation using GFP_KERNEL. that means + * this can deadlock when called a sc->gfp_mask that is not equal to + * GFP_KERNEL. + * + * This code is crying out for a shrinker per pool.... */ -static int ttm_pool_mm_shrink(struct shrinker *shrink, int shrink_pages, gfp_t gfp_mask) +static unsigned long +ttm_pool_shrink_scan(struct shrinker *shrink, struct shrink_control *sc) { static atomic_t start_pool = ATOMIC_INIT(0); unsigned i; unsigned pool_offset = atomic_add_return(1, &start_pool); struct ttm_page_pool *pool; + int shrink_pages = sc->nr_to_scan; + unsigned long freed = 0; pool_offset = pool_offset % NUM_POOLS; /* select start pool in round robin fashion */ @@ -409,14 +406,28 @@ static int ttm_pool_mm_shrink(struct shrinker *shrink, int shrink_pages, gfp_t g break; pool = &_manager->pools[(i + pool_offset)%NUM_POOLS]; shrink_pages = ttm_page_pool_free(pool, nr_free); + freed += nr_free - shrink_pages; } - /* return estimated number of unused pages in pool */ - return ttm_pool_get_num_unused_pages(); + return freed; +} + + +static unsigned long +ttm_pool_shrink_count(struct shrinker *shrink, struct shrink_control *sc) +{ + unsigned i; + unsigned long count = 0; + + for (i = 0; i < NUM_POOLS; ++i) + count += _manager->pools[i].npages; + + return count; } static void ttm_pool_mm_shrink_init(struct ttm_pool_manager *manager) { - manager->mm_shrink.shrink = &ttm_pool_mm_shrink; + manager->mm_shrink.count_objects = ttm_pool_shrink_count; + manager->mm_shrink.scan_objects = ttm_pool_shrink_scan; manager->mm_shrink.seeks = 1; register_shrinker(&manager->mm_shrink); } @@ -435,16 +446,12 @@ static int ttm_set_pages_caching(struct page **pages, case tt_uncached: r = set_pages_array_uc(pages, cpages); if (r) - printk(KERN_ERR TTM_PFX - "Failed to set %d pages to uc!\n", - cpages); + pr_err("Failed to set %d pages to uc!\n", cpages); break; case tt_wc: r = set_pages_array_wc(pages, cpages); if (r) - printk(KERN_ERR TTM_PFX - "Failed to set %d pages to wc!\n", - cpages); + pr_err("Failed to set %d pages to wc!\n", cpages); break; default: break; @@ -489,8 +496,7 @@ static int ttm_alloc_new_pages(struct list_head *pages, gfp_t gfp_flags, caching_array = kmalloc(max_cpages*sizeof(struct page *), GFP_KERNEL); if (!caching_array) { - printk(KERN_ERR TTM_PFX - "Unable to allocate table for new pages."); + pr_err("Unable to allocate table for new pages\n"); return -ENOMEM; } @@ -498,7 +504,7 @@ static int ttm_alloc_new_pages(struct list_head *pages, gfp_t gfp_flags, p = alloc_page(gfp_flags); if (!p) { - printk(KERN_ERR TTM_PFX "Unable to get page %u.\n", i); + pr_err("Unable to get page %u\n", i); /* store already allocated pages in the pool after * setting the caching state */ @@ -553,7 +559,7 @@ out: } /** - * Fill the given pool if there isn't enough pages and requested number of + * Fill the given pool if there aren't enough pages and the requested number of * pages is small. */ static void ttm_page_pool_fill_locked(struct ttm_page_pool *pool, @@ -573,8 +579,8 @@ static void ttm_page_pool_fill_locked(struct ttm_page_pool *pool, pool->fill_lock = true; - /* If allocation request is small and there is not enough - * pages in pool we fill the pool first */ + /* If allocation request is small and there are not enough + * pages in a pool we fill the pool up first. */ if (count < _manager->options.small && count > pool->npages) { struct list_head new_pages; @@ -596,8 +602,7 @@ static void ttm_page_pool_fill_locked(struct ttm_page_pool *pool, ++pool->nrefills; pool->npages += alloc_size; } else { - printk(KERN_ERR TTM_PFX - "Failed to fill pool (%p).", pool); + pr_err("Failed to fill pool (%p)\n", pool); /* If we have any pages left put them to the pool. */ list_for_each_entry(p, &pool->list, lru) { ++cpages; @@ -611,13 +616,15 @@ static void ttm_page_pool_fill_locked(struct ttm_page_pool *pool, } /** - * Cut count nubmer of pages from the pool and put them to return list + * Cut 'count' number of pages from the pool and put them on the return list. * - * @return count of pages still to allocate to fill the request. + * @return count of pages still required to fulfill the request. */ static unsigned ttm_page_pool_get_pages(struct ttm_page_pool *pool, - struct list_head *pages, int ttm_flags, - enum ttm_caching_state cstate, unsigned count) + struct list_head *pages, + int ttm_flags, + enum ttm_caching_state cstate, + unsigned count) { unsigned long irq_flags; struct list_head *p; @@ -634,7 +641,7 @@ static unsigned ttm_page_pool_get_pages(struct ttm_page_pool *pool, goto out; } /* find the last pages to include for requested number of pages. Split - * pool to begin and halves to reduce search space. */ + * pool to begin and halve it to reduce search space. */ if (count <= pool->npages/2) { i = 0; list_for_each(p, &pool->list) { @@ -648,7 +655,7 @@ static unsigned ttm_page_pool_get_pages(struct ttm_page_pool *pool, break; } } - /* Cut count number of pages from pool */ + /* Cut 'count' number of pages from the pool */ list_cut_position(pages, &pool->list, p); pool->npages -= count; count = 0; @@ -657,16 +664,63 @@ out: return count; } +/* Put all pages in pages list to correct pool to wait for reuse */ +static void ttm_put_pages(struct page **pages, unsigned npages, int flags, + enum ttm_caching_state cstate) +{ + unsigned long irq_flags; + struct ttm_page_pool *pool = ttm_get_pool(flags, cstate); + unsigned i; + + if (pool == NULL) { + /* No pool for this memory type so free the pages */ + for (i = 0; i < npages; i++) { + if (pages[i]) { + if (page_count(pages[i]) != 1) + pr_err("Erroneous page count. Leaking pages.\n"); + __free_page(pages[i]); + pages[i] = NULL; + } + } + return; + } + + spin_lock_irqsave(&pool->lock, irq_flags); + for (i = 0; i < npages; i++) { + if (pages[i]) { + if (page_count(pages[i]) != 1) + pr_err("Erroneous page count. Leaking pages.\n"); + list_add_tail(&pages[i]->lru, &pool->list); + pages[i] = NULL; + pool->npages++; + } + } + /* Check that we don't go over the pool limit */ + npages = 0; + if (pool->npages > _manager->options.max_size) { + npages = pool->npages - _manager->options.max_size; + /* free at least NUM_PAGES_TO_ALLOC number of pages + * to reduce calls to set_memory_wb */ + if (npages < NUM_PAGES_TO_ALLOC) + npages = NUM_PAGES_TO_ALLOC; + } + spin_unlock_irqrestore(&pool->lock, irq_flags); + if (npages) + ttm_page_pool_free(pool, npages); +} + /* * On success pages list will hold count number of correctly * cached pages. */ -int ttm_get_pages(struct list_head *pages, int flags, - enum ttm_caching_state cstate, unsigned count) +static int ttm_get_pages(struct page **pages, unsigned npages, int flags, + enum ttm_caching_state cstate) { struct ttm_page_pool *pool = ttm_get_pool(flags, cstate); + struct list_head plist; struct page *p = NULL; gfp_t gfp_flags = GFP_USER; + unsigned count; int r; /* set zero flag for page allocation if required */ @@ -680,96 +734,62 @@ int ttm_get_pages(struct list_head *pages, int flags, else gfp_flags |= GFP_HIGHUSER; - for (r = 0; r < count; ++r) { + for (r = 0; r < npages; ++r) { p = alloc_page(gfp_flags); if (!p) { - printk(KERN_ERR TTM_PFX - "Unable to allocate page."); + pr_err("Unable to allocate page\n"); return -ENOMEM; } - list_add(&p->lru, pages); + pages[r] = p; } return 0; } - /* combine zero flag to pool flags */ gfp_flags |= pool->gfp_flags; /* First we take pages from the pool */ - count = ttm_page_pool_get_pages(pool, pages, flags, cstate, count); + INIT_LIST_HEAD(&plist); + npages = ttm_page_pool_get_pages(pool, &plist, flags, cstate, npages); + count = 0; + list_for_each_entry(p, &plist, lru) { + pages[count++] = p; + } /* clear the pages coming from the pool if requested */ if (flags & TTM_PAGE_FLAG_ZERO_ALLOC) { - list_for_each_entry(p, pages, lru) { - clear_page(page_address(p)); + list_for_each_entry(p, &plist, lru) { + if (PageHighMem(p)) + clear_highpage(p); + else + clear_page(page_address(p)); } } /* If pool didn't have enough pages allocate new one. */ - if (count > 0) { + if (npages > 0) { /* ttm_alloc_new_pages doesn't reference pool so we can run * multiple requests in parallel. **/ - r = ttm_alloc_new_pages(pages, gfp_flags, flags, cstate, count); + INIT_LIST_HEAD(&plist); + r = ttm_alloc_new_pages(&plist, gfp_flags, flags, cstate, npages); + list_for_each_entry(p, &plist, lru) { + pages[count++] = p; + } if (r) { /* If there is any pages in the list put them back to * the pool. */ - printk(KERN_ERR TTM_PFX - "Failed to allocate extra pages " - "for large request."); - ttm_put_pages(pages, 0, flags, cstate); + pr_err("Failed to allocate extra pages for large request\n"); + ttm_put_pages(pages, count, flags, cstate); return r; } } - return 0; } -/* Put all pages in pages list to correct pool to wait for reuse */ -void ttm_put_pages(struct list_head *pages, unsigned page_count, int flags, - enum ttm_caching_state cstate) -{ - unsigned long irq_flags; - struct ttm_page_pool *pool = ttm_get_pool(flags, cstate); - struct page *p, *tmp; - - if (pool == NULL) { - /* No pool for this memory type so free the pages */ - - list_for_each_entry_safe(p, tmp, pages, lru) { - __free_page(p); - } - /* Make the pages list empty */ - INIT_LIST_HEAD(pages); - return; - } - if (page_count == 0) { - list_for_each_entry_safe(p, tmp, pages, lru) { - ++page_count; - } - } - - spin_lock_irqsave(&pool->lock, irq_flags); - list_splice_init(pages, &pool->list); - pool->npages += page_count; - /* Check that we don't go over the pool limit */ - page_count = 0; - if (pool->npages > _manager->options.max_size) { - page_count = pool->npages - _manager->options.max_size; - /* free at least NUM_PAGES_TO_ALLOC number of pages - * to reduce calls to set_memory_wb */ - if (page_count < NUM_PAGES_TO_ALLOC) - page_count = NUM_PAGES_TO_ALLOC; - } - spin_unlock_irqrestore(&pool->lock, irq_flags); - if (page_count) - ttm_page_pool_free(pool, page_count); -} - static void ttm_page_pool_init_locked(struct ttm_page_pool *pool, int flags, char *name) { @@ -787,7 +807,7 @@ int ttm_page_alloc_init(struct ttm_mem_global *glob, unsigned max_pages) WARN_ON(_manager); - printk(KERN_INFO TTM_PFX "Initializing pool allocator.\n"); + pr_info("Initializing pool allocator\n"); _manager = kzalloc(sizeof(*_manager), GFP_KERNEL); @@ -822,7 +842,7 @@ void ttm_page_alloc_fini(void) { int i; - printk(KERN_INFO TTM_PFX "Finalizing pool allocator.\n"); + pr_info("Finalizing pool allocator\n"); ttm_pool_mm_shrink_fini(_manager); for (i = 0; i < NUM_POOLS; ++i) @@ -832,6 +852,62 @@ void ttm_page_alloc_fini(void) _manager = NULL; } +int ttm_pool_populate(struct ttm_tt *ttm) +{ + struct ttm_mem_global *mem_glob = ttm->glob->mem_glob; + unsigned i; + int ret; + + if (ttm->state != tt_unpopulated) + return 0; + + for (i = 0; i < ttm->num_pages; ++i) { + ret = ttm_get_pages(&ttm->pages[i], 1, + ttm->page_flags, + ttm->caching_state); + if (ret != 0) { + ttm_pool_unpopulate(ttm); + return -ENOMEM; + } + + ret = ttm_mem_global_alloc_page(mem_glob, ttm->pages[i], + false, false); + if (unlikely(ret != 0)) { + ttm_pool_unpopulate(ttm); + return -ENOMEM; + } + } + + if (unlikely(ttm->page_flags & TTM_PAGE_FLAG_SWAPPED)) { + ret = ttm_tt_swapin(ttm); + if (unlikely(ret != 0)) { + ttm_pool_unpopulate(ttm); + return ret; + } + } + + ttm->state = tt_unbound; + return 0; +} +EXPORT_SYMBOL(ttm_pool_populate); + +void ttm_pool_unpopulate(struct ttm_tt *ttm) +{ + unsigned i; + + for (i = 0; i < ttm->num_pages; ++i) { + if (ttm->pages[i]) { + ttm_mem_global_free_page(ttm->glob->mem_glob, + ttm->pages[i]); + ttm_put_pages(&ttm->pages[i], 1, + ttm->page_flags, + ttm->caching_state); + } + } + ttm->state = tt_unpopulated; +} +EXPORT_SYMBOL(ttm_pool_unpopulate); + int ttm_page_alloc_debugfs(struct seq_file *m, void *data) { struct ttm_page_pool *p; |