diff options
Diffstat (limited to 'drivers/gpu/drm/ttm')
| -rw-r--r-- | drivers/gpu/drm/ttm/Makefile | 2 | ||||
| -rw-r--r-- | drivers/gpu/drm/ttm/ttm_agp_backend.c | 115 | ||||
| -rw-r--r-- | drivers/gpu/drm/ttm/ttm_bo.c | 857 | ||||
| -rw-r--r-- | drivers/gpu/drm/ttm/ttm_bo_manager.c | 62 | ||||
| -rw-r--r-- | drivers/gpu/drm/ttm/ttm_bo_util.c | 110 | ||||
| -rw-r--r-- | drivers/gpu/drm/ttm/ttm_bo_vm.c | 383 | ||||
| -rw-r--r-- | drivers/gpu/drm/ttm/ttm_execbuf_util.c | 132 | ||||
| -rw-r--r-- | drivers/gpu/drm/ttm/ttm_lock.c | 14 | ||||
| -rw-r--r-- | drivers/gpu/drm/ttm/ttm_memory.c | 21 | ||||
| -rw-r--r-- | drivers/gpu/drm/ttm/ttm_module.c | 4 | ||||
| -rw-r--r-- | drivers/gpu/drm/ttm/ttm_object.c | 448 | ||||
| -rw-r--r-- | drivers/gpu/drm/ttm/ttm_page_alloc.c | 298 | ||||
| -rw-r--r-- | drivers/gpu/drm/ttm/ttm_page_alloc_dma.c | 1147 | ||||
| -rw-r--r-- | drivers/gpu/drm/ttm/ttm_tt.c | 408 |
14 files changed, 2572 insertions, 1429 deletions
diff --git a/drivers/gpu/drm/ttm/Makefile b/drivers/gpu/drm/ttm/Makefile index f3cf6f02c99..b433b9f040c 100644 --- a/drivers/gpu/drm/ttm/Makefile +++ b/drivers/gpu/drm/ttm/Makefile @@ -5,6 +5,6 @@ ccflags-y := -Iinclude/drm ttm-y := ttm_agp_backend.o ttm_memory.o ttm_tt.o ttm_bo.o \ ttm_bo_util.o ttm_bo_vm.o ttm_module.o \ ttm_object.o ttm_lock.o ttm_execbuf_util.o ttm_page_alloc.o \ - ttm_bo_manager.o + ttm_bo_manager.o ttm_page_alloc_dma.o obj-$(CONFIG_DRM_TTM) += ttm.o diff --git a/drivers/gpu/drm/ttm/ttm_agp_backend.c b/drivers/gpu/drm/ttm/ttm_agp_backend.c index f999e36f30b..764be36397f 100644 --- a/drivers/gpu/drm/ttm/ttm_agp_backend.c +++ b/drivers/gpu/drm/ttm/ttm_agp_backend.c @@ -29,10 +29,13 @@ * Keith Packard. */ -#include "ttm/ttm_module.h" -#include "ttm/ttm_bo_driver.h" +#define pr_fmt(fmt) "[TTM] " fmt + +#include <drm/ttm/ttm_module.h> +#include <drm/ttm/ttm_bo_driver.h> +#include <drm/ttm/ttm_page_alloc.h> #ifdef TTM_HAS_AGP -#include "ttm/ttm_placement.h" +#include <drm/ttm/ttm_placement.h> #include <linux/agp_backend.h> #include <linux/module.h> #include <linux/slab.h> @@ -40,99 +43,77 @@ #include <asm/agp.h> struct ttm_agp_backend { - struct ttm_backend backend; + struct ttm_tt ttm; struct agp_memory *mem; struct agp_bridge_data *bridge; }; -static int ttm_agp_populate(struct ttm_backend *backend, - unsigned long num_pages, struct page **pages, - struct page *dummy_read_page) +static int ttm_agp_bind(struct ttm_tt *ttm, struct ttm_mem_reg *bo_mem) { - struct ttm_agp_backend *agp_be = - container_of(backend, struct ttm_agp_backend, backend); - struct page **cur_page, **last_page = pages + num_pages; + struct ttm_agp_backend *agp_be = container_of(ttm, struct ttm_agp_backend, ttm); + struct drm_mm_node *node = bo_mem->mm_node; struct agp_memory *mem; + int ret, cached = (bo_mem->placement & TTM_PL_FLAG_CACHED); + unsigned i; - mem = agp_allocate_memory(agp_be->bridge, num_pages, AGP_USER_MEMORY); + mem = agp_allocate_memory(agp_be->bridge, ttm->num_pages, AGP_USER_MEMORY); if (unlikely(mem == NULL)) return -ENOMEM; mem->page_count = 0; - for (cur_page = pages; cur_page < last_page; ++cur_page) { - struct page *page = *cur_page; + for (i = 0; i < ttm->num_pages; i++) { + struct page *page = ttm->pages[i]; + if (!page) - page = dummy_read_page; + page = ttm->dummy_read_page; mem->pages[mem->page_count++] = page; } agp_be->mem = mem; - return 0; -} - -static int ttm_agp_bind(struct ttm_backend *backend, struct ttm_mem_reg *bo_mem) -{ - struct ttm_agp_backend *agp_be = - container_of(backend, struct ttm_agp_backend, backend); - struct drm_mm_node *node = bo_mem->mm_node; - struct agp_memory *mem = agp_be->mem; - int cached = (bo_mem->placement & TTM_PL_FLAG_CACHED); - int ret; mem->is_flushed = 1; mem->type = (cached) ? AGP_USER_CACHED_MEMORY : AGP_USER_MEMORY; ret = agp_bind_memory(mem, node->start); if (ret) - printk(KERN_ERR TTM_PFX "AGP Bind memory failed.\n"); + pr_err("AGP Bind memory failed\n"); return ret; } -static int ttm_agp_unbind(struct ttm_backend *backend) +static int ttm_agp_unbind(struct ttm_tt *ttm) { - struct ttm_agp_backend *agp_be = - container_of(backend, struct ttm_agp_backend, backend); - - if (agp_be->mem->is_bound) - return agp_unbind_memory(agp_be->mem); - else - return 0; -} + struct ttm_agp_backend *agp_be = container_of(ttm, struct ttm_agp_backend, ttm); -static void ttm_agp_clear(struct ttm_backend *backend) -{ - struct ttm_agp_backend *agp_be = - container_of(backend, struct ttm_agp_backend, backend); - struct agp_memory *mem = agp_be->mem; - - if (mem) { - ttm_agp_unbind(backend); - agp_free_memory(mem); + if (agp_be->mem) { + if (agp_be->mem->is_bound) + return agp_unbind_memory(agp_be->mem); + agp_free_memory(agp_be->mem); + agp_be->mem = NULL; } - agp_be->mem = NULL; + return 0; } -static void ttm_agp_destroy(struct ttm_backend *backend) +static void ttm_agp_destroy(struct ttm_tt *ttm) { - struct ttm_agp_backend *agp_be = - container_of(backend, struct ttm_agp_backend, backend); + struct ttm_agp_backend *agp_be = container_of(ttm, struct ttm_agp_backend, ttm); if (agp_be->mem) - ttm_agp_clear(backend); + ttm_agp_unbind(ttm); + ttm_tt_fini(ttm); kfree(agp_be); } static struct ttm_backend_func ttm_agp_func = { - .populate = ttm_agp_populate, - .clear = ttm_agp_clear, .bind = ttm_agp_bind, .unbind = ttm_agp_unbind, .destroy = ttm_agp_destroy, }; -struct ttm_backend *ttm_agp_backend_init(struct ttm_bo_device *bdev, - struct agp_bridge_data *bridge) +struct ttm_tt *ttm_agp_tt_create(struct ttm_bo_device *bdev, + struct agp_bridge_data *bridge, + unsigned long size, uint32_t page_flags, + struct page *dummy_read_page) { struct ttm_agp_backend *agp_be; @@ -142,10 +123,30 @@ struct ttm_backend *ttm_agp_backend_init(struct ttm_bo_device *bdev, agp_be->mem = NULL; agp_be->bridge = bridge; - agp_be->backend.func = &ttm_agp_func; - agp_be->backend.bdev = bdev; - return &agp_be->backend; + agp_be->ttm.func = &ttm_agp_func; + + if (ttm_tt_init(&agp_be->ttm, bdev, size, page_flags, dummy_read_page)) { + kfree(agp_be); + return NULL; + } + + return &agp_be->ttm; +} +EXPORT_SYMBOL(ttm_agp_tt_create); + +int ttm_agp_tt_populate(struct ttm_tt *ttm) +{ + if (ttm->state != tt_unpopulated) + return 0; + + return ttm_pool_populate(ttm); +} +EXPORT_SYMBOL(ttm_agp_tt_populate); + +void ttm_agp_tt_unpopulate(struct ttm_tt *ttm) +{ + ttm_pool_unpopulate(ttm); } -EXPORT_SYMBOL(ttm_agp_backend_init); +EXPORT_SYMBOL(ttm_agp_tt_unpopulate); #endif diff --git a/drivers/gpu/drm/ttm/ttm_bo.c b/drivers/gpu/drm/ttm/ttm_bo.c index af61fc29e84..4ab9f7171c4 100644 --- a/drivers/gpu/drm/ttm/ttm_bo.c +++ b/drivers/gpu/drm/ttm/ttm_bo.c @@ -28,22 +28,23 @@ * Authors: Thomas Hellstrom <thellstrom-at-vmware-dot-com> */ -#include "ttm/ttm_module.h" -#include "ttm/ttm_bo_driver.h" -#include "ttm/ttm_placement.h" +#define pr_fmt(fmt) "[TTM] " fmt + +#include <drm/ttm/ttm_module.h> +#include <drm/ttm/ttm_bo_driver.h> +#include <drm/ttm/ttm_placement.h> #include <linux/jiffies.h> #include <linux/slab.h> #include <linux/sched.h> #include <linux/mm.h> #include <linux/file.h> #include <linux/module.h> -#include <asm/atomic.h> +#include <linux/atomic.h> #define TTM_ASSERT_LOCKED(param) #define TTM_DEBUG(fmt, arg...) #define TTM_BO_HASH_ORDER 13 -static int ttm_bo_setup_vm(struct ttm_buffer_object *bo); static int ttm_bo_swapout(struct ttm_mem_shrink *shrink); static void ttm_bo_global_kobj_release(struct kobject *kobj); @@ -68,15 +69,13 @@ static void ttm_mem_type_debug(struct ttm_bo_device *bdev, int mem_type) { struct ttm_mem_type_manager *man = &bdev->man[mem_type]; - printk(KERN_ERR TTM_PFX " has_type: %d\n", man->has_type); - printk(KERN_ERR TTM_PFX " use_type: %d\n", man->use_type); - printk(KERN_ERR TTM_PFX " flags: 0x%08X\n", man->flags); - printk(KERN_ERR TTM_PFX " gpu_offset: 0x%08lX\n", man->gpu_offset); - printk(KERN_ERR TTM_PFX " size: %llu\n", man->size); - printk(KERN_ERR TTM_PFX " available_caching: 0x%08X\n", - man->available_caching); - printk(KERN_ERR TTM_PFX " default_caching: 0x%08X\n", - man->default_caching); + pr_err(" has_type: %d\n", man->has_type); + pr_err(" use_type: %d\n", man->use_type); + pr_err(" flags: 0x%08X\n", man->flags); + pr_err(" gpu_offset: 0x%08lX\n", man->gpu_offset); + pr_err(" size: %llu\n", man->size); + pr_err(" available_caching: 0x%08X\n", man->available_caching); + pr_err(" default_caching: 0x%08X\n", man->default_caching); if (mem_type != TTM_PL_SYSTEM) (*man->func->debug)(man, TTM_PFX); } @@ -86,16 +85,16 @@ static void ttm_bo_mem_space_debug(struct ttm_buffer_object *bo, { int i, ret, mem_type; - printk(KERN_ERR TTM_PFX "No space for %p (%lu pages, %luK, %luM)\n", - bo, bo->mem.num_pages, bo->mem.size >> 10, - bo->mem.size >> 20); + pr_err("No space for %p (%lu pages, %luK, %luM)\n", + bo, bo->mem.num_pages, bo->mem.size >> 10, + bo->mem.size >> 20); for (i = 0; i < placement->num_placement; i++) { ret = ttm_mem_type_from_flags(placement->placement[i], &mem_type); if (ret) return; - printk(KERN_ERR TTM_PFX " placement[%d]=0x%08X (%d)\n", - i, placement->placement[i], mem_type); + pr_err(" placement[%d]=0x%08X (%d)\n", + i, placement->placement[i], mem_type); ttm_mem_type_debug(bo->bdev, mem_type); } } @@ -137,6 +136,7 @@ static void ttm_bo_release_list(struct kref *list_kref) struct ttm_buffer_object *bo = container_of(list_kref, struct ttm_buffer_object, list_kref); struct ttm_bo_device *bdev = bo->bdev; + size_t acc_size = bo->acc_size; BUG_ON(atomic_read(&bo->list_kref.refcount)); BUG_ON(atomic_read(&bo->kref.refcount)); @@ -149,32 +149,23 @@ static void ttm_bo_release_list(struct kref *list_kref) if (bo->ttm) ttm_tt_destroy(bo->ttm); atomic_dec(&bo->glob->bo_count); + if (bo->resv == &bo->ttm_resv) + reservation_object_fini(&bo->ttm_resv); + mutex_destroy(&bo->wu_mutex); if (bo->destroy) bo->destroy(bo); else { - ttm_mem_global_free(bdev->glob->mem_glob, bo->acc_size); kfree(bo); } + ttm_mem_global_free(bdev->glob->mem_glob, acc_size); } -int ttm_bo_wait_unreserved(struct ttm_buffer_object *bo, bool interruptible) -{ - if (interruptible) { - return wait_event_interruptible(bo->event_queue, - atomic_read(&bo->reserved) == 0); - } else { - wait_event(bo->event_queue, atomic_read(&bo->reserved) == 0); - return 0; - } -} -EXPORT_SYMBOL(ttm_bo_wait_unreserved); - void ttm_bo_add_to_lru(struct ttm_buffer_object *bo) { struct ttm_bo_device *bdev = bo->bdev; struct ttm_mem_type_manager *man; - BUG_ON(!atomic_read(&bo->reserved)); + lockdep_assert_held(&bo->resv->lock.base); if (!(bo->mem.placement & TTM_PL_FLAG_NO_EVICT)) { @@ -190,6 +181,7 @@ void ttm_bo_add_to_lru(struct ttm_buffer_object *bo) } } } +EXPORT_SYMBOL(ttm_bo_add_to_lru); int ttm_bo_del_from_lru(struct ttm_buffer_object *bo) { @@ -212,61 +204,6 @@ int ttm_bo_del_from_lru(struct ttm_buffer_object *bo) return put_count; } -int ttm_bo_reserve_locked(struct ttm_buffer_object *bo, - bool interruptible, - bool no_wait, bool use_sequence, uint32_t sequence) -{ - struct ttm_bo_global *glob = bo->glob; - int ret; - - while (unlikely(atomic_cmpxchg(&bo->reserved, 0, 1) != 0)) { - /** - * Deadlock avoidance for multi-bo reserving. - */ - if (use_sequence && bo->seq_valid) { - /** - * We've already reserved this one. - */ - if (unlikely(sequence == bo->val_seq)) - return -EDEADLK; - /** - * Already reserved by a thread that will not back - * off for us. We need to back off. - */ - if (unlikely(sequence - bo->val_seq < (1 << 31))) - return -EAGAIN; - } - - if (no_wait) - return -EBUSY; - - spin_unlock(&glob->lru_lock); - ret = ttm_bo_wait_unreserved(bo, interruptible); - spin_lock(&glob->lru_lock); - - if (unlikely(ret)) - return ret; - } - - if (use_sequence) { - /** - * Wake up waiters that may need to recheck for deadlock, - * if we decreased the sequence number. - */ - if (unlikely((bo->val_seq - sequence < (1 << 31)) - || !bo->seq_valid)) - wake_up_all(&bo->event_queue); - - bo->val_seq = sequence; - bo->seq_valid = true; - } else { - bo->seq_valid = false; - } - - return 0; -} -EXPORT_SYMBOL(ttm_bo_reserve); - static void ttm_bo_ref_bug(struct kref *list_kref) { BUG(); @@ -279,42 +216,16 @@ void ttm_bo_list_ref_sub(struct ttm_buffer_object *bo, int count, (never_free) ? ttm_bo_ref_bug : ttm_bo_release_list); } -int ttm_bo_reserve(struct ttm_buffer_object *bo, - bool interruptible, - bool no_wait, bool use_sequence, uint32_t sequence) +void ttm_bo_del_sub_from_lru(struct ttm_buffer_object *bo) { - struct ttm_bo_global *glob = bo->glob; - int put_count = 0; - int ret; - - spin_lock(&glob->lru_lock); - ret = ttm_bo_reserve_locked(bo, interruptible, no_wait, use_sequence, - sequence); - if (likely(ret == 0)) - put_count = ttm_bo_del_from_lru(bo); - spin_unlock(&glob->lru_lock); + int put_count; + spin_lock(&bo->glob->lru_lock); + put_count = ttm_bo_del_from_lru(bo); + spin_unlock(&bo->glob->lru_lock); ttm_bo_list_ref_sub(bo, put_count, true); - - return ret; -} - -void ttm_bo_unreserve_locked(struct ttm_buffer_object *bo) -{ - ttm_bo_add_to_lru(bo); - atomic_set(&bo->reserved, 0); - wake_up_all(&bo->event_queue); } - -void ttm_bo_unreserve(struct ttm_buffer_object *bo) -{ - struct ttm_bo_global *glob = bo->glob; - - spin_lock(&glob->lru_lock); - ttm_bo_unreserve_locked(bo); - spin_unlock(&glob->lru_lock); -} -EXPORT_SYMBOL(ttm_bo_unreserve); +EXPORT_SYMBOL(ttm_bo_del_sub_from_lru); /* * Call bo->mutex locked. @@ -337,27 +248,23 @@ static int ttm_bo_add_ttm(struct ttm_buffer_object *bo, bool zero_alloc) if (zero_alloc) page_flags |= TTM_PAGE_FLAG_ZERO_ALLOC; case ttm_bo_type_kernel: - bo->ttm = ttm_tt_create(bdev, bo->num_pages << PAGE_SHIFT, - page_flags, glob->dummy_read_page); + bo->ttm = bdev->driver->ttm_tt_create(bdev, bo->num_pages << PAGE_SHIFT, + page_flags, glob->dummy_read_page); if (unlikely(bo->ttm == NULL)) ret = -ENOMEM; break; - case ttm_bo_type_user: - bo->ttm = ttm_tt_create(bdev, bo->num_pages << PAGE_SHIFT, - page_flags | TTM_PAGE_FLAG_USER, - glob->dummy_read_page); + case ttm_bo_type_sg: + bo->ttm = bdev->driver->ttm_tt_create(bdev, bo->num_pages << PAGE_SHIFT, + page_flags | TTM_PAGE_FLAG_SG, + glob->dummy_read_page); if (unlikely(bo->ttm == NULL)) { ret = -ENOMEM; break; } - - ret = ttm_tt_set_user(bo->ttm, current, - bo->buffer_start, bo->num_pages); - if (unlikely(ret != 0)) - ttm_tt_destroy(bo->ttm); + bo->ttm->sg = bo->sg; break; default: - printk(KERN_ERR TTM_PFX "Illegal buffer object type\n"); + pr_err("Illegal buffer object type\n"); ret = -EINVAL; break; } @@ -368,7 +275,7 @@ static int ttm_bo_add_ttm(struct ttm_buffer_object *bo, bool zero_alloc) static int ttm_bo_handle_move_mem(struct ttm_buffer_object *bo, struct ttm_mem_reg *mem, bool evict, bool interruptible, - bool no_wait_reserve, bool no_wait_gpu) + bool no_wait_gpu) { struct ttm_bo_device *bdev = bo->bdev; bool old_is_pci = ttm_mem_reg_is_pci(bdev, &bo->mem); @@ -390,10 +297,13 @@ static int ttm_bo_handle_move_mem(struct ttm_buffer_object *bo, * Create and bind a ttm if required. */ - if (!(new_man->flags & TTM_MEMTYPE_FLAG_FIXED) && (bo->ttm == NULL)) { - ret = ttm_bo_add_ttm(bo, false); - if (ret) - goto out_err; + if (!(new_man->flags & TTM_MEMTYPE_FLAG_FIXED)) { + if (bo->ttm == NULL) { + bool zero = !(old_man->flags & TTM_MEMTYPE_FLAG_FIXED); + ret = ttm_bo_add_ttm(bo, zero); + if (ret) + goto out_err; + } ret = ttm_tt_set_placement_caching(bo->ttm, mem->placement); if (ret) @@ -406,11 +316,12 @@ static int ttm_bo_handle_move_mem(struct ttm_buffer_object *bo, } if (bo->mem.mem_type == TTM_PL_SYSTEM) { + if (bdev->driver->move_notify) + bdev->driver->move_notify(bo, mem); bo->mem = *mem; mem->mm_node = NULL; goto moved; } - } if (bdev->driver->move_notify) @@ -418,21 +329,33 @@ static int ttm_bo_handle_move_mem(struct ttm_buffer_object *bo, if (!(old_man->flags & TTM_MEMTYPE_FLAG_FIXED) && !(new_man->flags & TTM_MEMTYPE_FLAG_FIXED)) - ret = ttm_bo_move_ttm(bo, evict, no_wait_reserve, no_wait_gpu, mem); + ret = ttm_bo_move_ttm(bo, evict, no_wait_gpu, mem); else if (bdev->driver->move) ret = bdev->driver->move(bo, evict, interruptible, - no_wait_reserve, no_wait_gpu, mem); + no_wait_gpu, mem); else - ret = ttm_bo_move_memcpy(bo, evict, no_wait_reserve, no_wait_gpu, mem); + ret = ttm_bo_move_memcpy(bo, evict, no_wait_gpu, mem); + + if (ret) { + if (bdev->driver->move_notify) { + struct ttm_mem_reg tmp_mem = *mem; + *mem = bo->mem; + bo->mem = tmp_mem; + bdev->driver->move_notify(bo, mem); + bo->mem = *mem; + *mem = tmp_mem; + } - if (ret) goto out_err; + } moved: if (bo->evicted) { - ret = bdev->driver->invalidate_caches(bdev, bo->mem.placement); - if (ret) - printk(KERN_ERR TTM_PFX "Can not flush read caches\n"); + if (bdev->driver->invalidate_caches) { + ret = bdev->driver->invalidate_caches(bdev, bo->mem.placement); + if (ret) + pr_err("Can not flush read caches\n"); + } bo->evicted = false; } @@ -466,6 +389,9 @@ out_err: static void ttm_bo_cleanup_memtype_use(struct ttm_buffer_object *bo) { + if (bo->bdev->driver->move_notify) + bo->bdev->driver->move_notify(bo, NULL); + if (bo->ttm) { ttm_tt_unbind(bo->ttm); ttm_tt_destroy(bo->ttm); @@ -473,41 +399,24 @@ static void ttm_bo_cleanup_memtype_use(struct ttm_buffer_object *bo) } ttm_bo_mem_put(bo, &bo->mem); - atomic_set(&bo->reserved, 0); - - /* - * Make processes trying to reserve really pick it up. - */ - smp_mb__after_atomic_dec(); - wake_up_all(&bo->event_queue); + ww_mutex_unlock (&bo->resv->lock); } static void ttm_bo_cleanup_refs_or_queue(struct ttm_buffer_object *bo) { struct ttm_bo_device *bdev = bo->bdev; struct ttm_bo_global *glob = bo->glob; - struct ttm_bo_driver *driver; + struct ttm_bo_driver *driver = bdev->driver; void *sync_obj = NULL; - void *sync_obj_arg; int put_count; int ret; + spin_lock(&glob->lru_lock); + ret = __ttm_bo_reserve(bo, false, true, false, 0); + spin_lock(&bdev->fence_lock); (void) ttm_bo_wait(bo, false, false, true); - if (!bo->sync_obj) { - - spin_lock(&glob->lru_lock); - - /** - * Lock inversion between bo:reserve and bdev::fence_lock here, - * but that's OK, since we're only trylocking. - */ - - ret = ttm_bo_reserve_locked(bo, false, true, false, 0); - - if (unlikely(ret == -EBUSY)) - goto queue; - + if (!ret && !bo->sync_obj) { spin_unlock(&bdev->fence_lock); put_count = ttm_bo_del_from_lru(bo); @@ -517,22 +426,32 @@ static void ttm_bo_cleanup_refs_or_queue(struct ttm_buffer_object *bo) ttm_bo_list_ref_sub(bo, put_count, true); return; - } else { - spin_lock(&glob->lru_lock); } -queue: - driver = bdev->driver; if (bo->sync_obj) sync_obj = driver->sync_obj_ref(bo->sync_obj); - sync_obj_arg = bo->sync_obj_arg; + spin_unlock(&bdev->fence_lock); + + if (!ret) { + + /* + * Make NO_EVICT bos immediately available to + * shrinkers, now that they are queued for + * destruction. + */ + if (bo->mem.placement & TTM_PL_FLAG_NO_EVICT) { + bo->mem.placement &= ~TTM_PL_FLAG_NO_EVICT; + ttm_bo_add_to_lru(bo); + } + + __ttm_bo_unreserve(bo); + } kref_get(&bo->list_kref); list_add_tail(&bo->ddestroy, &bdev->ddestroy); spin_unlock(&glob->lru_lock); - spin_unlock(&bdev->fence_lock); if (sync_obj) { - driver->sync_obj_flush(sync_obj, sync_obj_arg); + driver->sync_obj_flush(sync_obj); driver->sync_obj_unref(&sync_obj); } schedule_delayed_work(&bdev->wq, @@ -540,55 +459,82 @@ queue: } /** - * function ttm_bo_cleanup_refs + * function ttm_bo_cleanup_refs_and_unlock * If bo idle, remove from delayed- and lru lists, and unref. * If not idle, do nothing. * + * Must be called with lru_lock and reservation held, this function + * will drop both before returning. + * * @interruptible Any sleeps should occur interruptibly. - * @no_wait_reserve Never wait for reserve. Return -EBUSY instead. * @no_wait_gpu Never wait for gpu. Return -EBUSY instead. */ -static int ttm_bo_cleanup_refs(struct ttm_buffer_object *bo, - bool interruptible, - bool no_wait_reserve, - bool no_wait_gpu) +static int ttm_bo_cleanup_refs_and_unlock(struct ttm_buffer_object *bo, + bool interruptible, + bool no_wait_gpu) { struct ttm_bo_device *bdev = bo->bdev; + struct ttm_bo_driver *driver = bdev->driver; struct ttm_bo_global *glob = bo->glob; int put_count; - int ret = 0; + int ret; -retry: spin_lock(&bdev->fence_lock); - ret = ttm_bo_wait(bo, false, interruptible, no_wait_gpu); - spin_unlock(&bdev->fence_lock); + ret = ttm_bo_wait(bo, false, false, true); - if (unlikely(ret != 0)) - return ret; + if (ret && !no_wait_gpu) { + void *sync_obj; - spin_lock(&glob->lru_lock); - ret = ttm_bo_reserve_locked(bo, interruptible, - no_wait_reserve, false, 0); + /* + * Take a reference to the fence and unreserve, + * at this point the buffer should be dead, so + * no new sync objects can be attached. + */ + sync_obj = driver->sync_obj_ref(bo->sync_obj); + spin_unlock(&bdev->fence_lock); - if (unlikely(ret != 0) || list_empty(&bo->ddestroy)) { + __ttm_bo_unreserve(bo); spin_unlock(&glob->lru_lock); - return ret; - } - /** - * We can re-check for sync object without taking - * the bo::lock since setting the sync object requires - * also bo::reserved. A busy object at this point may - * be caused by another thread recently starting an accelerated - * eviction. - */ + ret = driver->sync_obj_wait(sync_obj, false, interruptible); + driver->sync_obj_unref(&sync_obj); + if (ret) + return ret; + + /* + * remove sync_obj with ttm_bo_wait, the wait should be + * finished, and no new wait object should have been added. + */ + spin_lock(&bdev->fence_lock); + ret = ttm_bo_wait(bo, false, false, true); + WARN_ON(ret); + spin_unlock(&bdev->fence_lock); + if (ret) + return ret; + + spin_lock(&glob->lru_lock); + ret = __ttm_bo_reserve(bo, false, true, false, 0); + + /* + * We raced, and lost, someone else holds the reservation now, + * and is probably busy in ttm_bo_cleanup_memtype_use. + * + * Even if it's not the case, because we finished waiting any + * delayed destruction would succeed, so just return success + * here. + */ + if (ret) { + spin_unlock(&glob->lru_lock); + return 0; + } + } else + spin_unlock(&bdev->fence_lock); - if (unlikely(bo->sync_obj)) { - atomic_set(&bo->reserved, 0); - wake_up_all(&bo->event_queue); + if (ret || unlikely(list_empty(&bo->ddestroy))) { + __ttm_bo_unreserve(bo); spin_unlock(&glob->lru_lock); - goto retry; + return ret; } put_count = ttm_bo_del_from_lru(bo); @@ -631,9 +577,20 @@ static int ttm_bo_delayed_delete(struct ttm_bo_device *bdev, bool remove_all) kref_get(&nentry->list_kref); } - spin_unlock(&glob->lru_lock); - ret = ttm_bo_cleanup_refs(entry, false, !remove_all, - !remove_all); + ret = __ttm_bo_reserve(entry, false, true, false, 0); + if (remove_all && ret) { + spin_unlock(&glob->lru_lock); + ret = __ttm_bo_reserve(entry, false, false, + false, 0); + spin_lock(&glob->lru_lock); + } + + if (!ret) + ret = ttm_bo_cleanup_refs_and_unlock(entry, false, + !remove_all); + else + spin_unlock(&glob->lru_lock); + kref_put(&entry->list_kref, ttm_bo_release_list); entry = nentry; @@ -671,29 +628,20 @@ static void ttm_bo_release(struct kref *kref) struct ttm_bo_device *bdev = bo->bdev; struct ttm_mem_type_manager *man = &bdev->man[bo->mem.mem_type]; - if (likely(bo->vm_node != NULL)) { - rb_erase(&bo->vm_rb, &bdev->addr_space_rb); - drm_mm_put_block(bo->vm_node); - bo->vm_node = NULL; - } - write_unlock(&bdev->vm_lock); + drm_vma_offset_remove(&bdev->vma_manager, &bo->vma_node); ttm_mem_io_lock(man, false); ttm_mem_io_free_vm(bo); ttm_mem_io_unlock(man); ttm_bo_cleanup_refs_or_queue(bo); kref_put(&bo->list_kref, ttm_bo_release_list); - write_lock(&bdev->vm_lock); } void ttm_bo_unref(struct ttm_buffer_object **p_bo) { struct ttm_buffer_object *bo = *p_bo; - struct ttm_bo_device *bdev = bo->bdev; *p_bo = NULL; - write_lock(&bdev->vm_lock); kref_put(&bo->kref, ttm_bo_release); - write_unlock(&bdev->vm_lock); } EXPORT_SYMBOL(ttm_bo_unref); @@ -712,7 +660,7 @@ void ttm_bo_unlock_delayed_workqueue(struct ttm_bo_device *bdev, int resched) EXPORT_SYMBOL(ttm_bo_unlock_delayed_workqueue); static int ttm_bo_evict(struct ttm_buffer_object *bo, bool interruptible, - bool no_wait_reserve, bool no_wait_gpu) + bool no_wait_gpu) { struct ttm_bo_device *bdev = bo->bdev; struct ttm_mem_reg evict_mem; @@ -725,14 +673,12 @@ static int ttm_bo_evict(struct ttm_buffer_object *bo, bool interruptible, if (unlikely(ret != 0)) { if (ret != -ERESTARTSYS) { - printk(KERN_ERR TTM_PFX - "Failed to expire sync object before " - "buffer eviction.\n"); + pr_err("Failed to expire sync object before buffer eviction\n"); } goto out; } - BUG_ON(!atomic_read(&bo->reserved)); + lockdep_assert_held(&bo->resv->lock.base); evict_mem = bo->mem; evict_mem.mm_node = NULL; @@ -745,22 +691,21 @@ static int ttm_bo_evict(struct ttm_buffer_object *bo, bool interruptible, placement.num_busy_placement = 0; bdev->driver->evict_flags(bo, &placement); ret = ttm_bo_mem_space(bo, &placement, &evict_mem, interruptible, - no_wait_reserve, no_wait_gpu); + no_wait_gpu); if (ret) { if (ret != -ERESTARTSYS) { - printk(KERN_ERR TTM_PFX - "Failed to find memory space for " - "buffer 0x%p eviction.\n", bo); + pr_err("Failed to find memory space for buffer 0x%p eviction\n", + bo); ttm_bo_mem_space_debug(bo, &placement); } goto out; } ret = ttm_bo_handle_move_mem(bo, &evict_mem, true, interruptible, - no_wait_reserve, no_wait_gpu); + no_wait_gpu); if (ret) { if (ret != -ERESTARTSYS) - printk(KERN_ERR TTM_PFX "Buffer eviction failed\n"); + pr_err("Buffer eviction failed\n"); ttm_bo_mem_put(bo, &evict_mem); goto out; } @@ -771,52 +716,33 @@ out: static int ttm_mem_evict_first(struct ttm_bo_device *bdev, uint32_t mem_type, - bool interruptible, bool no_wait_reserve, + bool interruptible, bool no_wait_gpu) { struct ttm_bo_global *glob = bdev->glob; struct ttm_mem_type_manager *man = &bdev->man[mem_type]; struct ttm_buffer_object *bo; - int ret, put_count = 0; + int ret = -EBUSY, put_count; -retry: spin_lock(&glob->lru_lock); - if (list_empty(&man->lru)) { - spin_unlock(&glob->lru_lock); - return -EBUSY; + list_for_each_entry(bo, &man->lru, lru) { + ret = __ttm_bo_reserve(bo, false, true, false, 0); + if (!ret) + break; } - bo = list_first_entry(&man->lru, struct ttm_buffer_object, lru); - kref_get(&bo->list_kref); - - if (!list_empty(&bo->ddestroy)) { + if (ret) { spin_unlock(&glob->lru_lock); - ret = ttm_bo_cleanup_refs(bo, interruptible, - no_wait_reserve, no_wait_gpu); - kref_put(&bo->list_kref, ttm_bo_release_list); - - if (likely(ret == 0 || ret == -ERESTARTSYS)) - return ret; - - goto retry; + return ret; } - ret = ttm_bo_reserve_locked(bo, false, no_wait_reserve, false, 0); - - if (unlikely(ret == -EBUSY)) { - spin_unlock(&glob->lru_lock); - if (likely(!no_wait_gpu)) - ret = ttm_bo_wait_unreserved(bo, interruptible); + kref_get(&bo->list_kref); + if (!list_empty(&bo->ddestroy)) { + ret = ttm_bo_cleanup_refs_and_unlock(bo, interruptible, + no_wait_gpu); kref_put(&bo->list_kref, ttm_bo_release_list); - - /** - * We *need* to retry after releasing the lru lock. - */ - - if (unlikely(ret != 0)) - return ret; - goto retry; + return ret; } put_count = ttm_bo_del_from_lru(bo); @@ -826,7 +752,7 @@ retry: ttm_bo_list_ref_sub(bo, put_count, true); - ret = ttm_bo_evict(bo, interruptible, no_wait_reserve, no_wait_gpu); + ret = ttm_bo_evict(bo, interruptible, no_wait_gpu); ttm_bo_unreserve(bo); kref_put(&bo->list_kref, ttm_bo_release_list); @@ -851,7 +777,6 @@ static int ttm_bo_mem_force_space(struct ttm_buffer_object *bo, struct ttm_placement *placement, struct ttm_mem_reg *mem, bool interruptible, - bool no_wait_reserve, bool no_wait_gpu) { struct ttm_bo_device *bdev = bo->bdev; @@ -864,8 +789,8 @@ static int ttm_bo_mem_force_space(struct ttm_buffer_object *bo, return ret; if (mem->mm_node) break; - ret = ttm_mem_evict_first(bdev, mem_type, interruptible, - no_wait_reserve, no_wait_gpu); + ret = ttm_mem_evict_first(bdev, mem_type, + interruptible, no_wait_gpu); if (unlikely(ret != 0)) return ret; } while (1); @@ -901,16 +826,12 @@ static uint32_t ttm_bo_select_caching(struct ttm_mem_type_manager *man, } static bool ttm_bo_mt_compatible(struct ttm_mem_type_manager *man, - bool disallow_fixed, uint32_t mem_type, uint32_t proposed_placement, uint32_t *masked_placement) { uint32_t cur_flags = ttm_bo_type_flags(mem_type); - if ((man->flags & TTM_MEMTYPE_FLAG_FIXED) && disallow_fixed) - return false; - if ((cur_flags & proposed_placement & TTM_PL_MASK_MEM) == 0) return false; @@ -934,7 +855,7 @@ static bool ttm_bo_mt_compatible(struct ttm_mem_type_manager *man, int ttm_bo_mem_space(struct ttm_buffer_object *bo, struct ttm_placement *placement, struct ttm_mem_reg *mem, - bool interruptible, bool no_wait_reserve, + bool interruptible, bool no_wait_gpu) { struct ttm_bo_device *bdev = bo->bdev; @@ -955,7 +876,6 @@ int ttm_bo_mem_space(struct ttm_buffer_object *bo, man = &bdev->man[mem_type]; type_ok = ttm_bo_mt_compatible(man, - bo->type == ttm_bo_type_user, mem_type, placement->placement[i], &cur_flags); @@ -1003,7 +923,6 @@ int ttm_bo_mem_space(struct ttm_buffer_object *bo, if (!man->has_type) continue; if (!ttm_bo_mt_compatible(man, - bo->type == ttm_bo_type_user, mem_type, placement->busy_placement[i], &cur_flags)) @@ -1027,7 +946,7 @@ int ttm_bo_mem_space(struct ttm_buffer_object *bo, } ret = ttm_bo_mem_force_space(bo, mem_type, placement, mem, - interruptible, no_wait_reserve, no_wait_gpu); + interruptible, no_wait_gpu); if (ret == 0 && mem->mm_node) { mem->placement = cur_flags; return 0; @@ -1040,26 +959,16 @@ int ttm_bo_mem_space(struct ttm_buffer_object *bo, } EXPORT_SYMBOL(ttm_bo_mem_space); -int ttm_bo_wait_cpu(struct ttm_buffer_object *bo, bool no_wait) -{ - if ((atomic_read(&bo->cpu_writers) > 0) && no_wait) - return -EBUSY; - - return wait_event_interruptible(bo->event_queue, - atomic_read(&bo->cpu_writers) == 0); -} -EXPORT_SYMBOL(ttm_bo_wait_cpu); - -int ttm_bo_move_buffer(struct ttm_buffer_object *bo, +static int ttm_bo_move_buffer(struct ttm_buffer_object *bo, struct ttm_placement *placement, - bool interruptible, bool no_wait_reserve, + bool interruptible, bool no_wait_gpu) { int ret = 0; struct ttm_mem_reg mem; struct ttm_bo_device *bdev = bo->bdev; - BUG_ON(!atomic_read(&bo->reserved)); + lockdep_assert_held(&bo->resv->lock.base); /* * FIXME: It's possible to pipeline buffer moves. @@ -1079,44 +988,55 @@ int ttm_bo_move_buffer(struct ttm_buffer_object *bo, /* * Determine where to move the buffer. */ - ret = ttm_bo_mem_space(bo, placement, &mem, interruptible, no_wait_reserve, no_wait_gpu); + ret = ttm_bo_mem_space(bo, placement, &mem, + interruptible, no_wait_gpu); if (ret) goto out_unlock; - ret = ttm_bo_handle_move_mem(bo, &mem, false, interruptible, no_wait_reserve, no_wait_gpu); + ret = ttm_bo_handle_move_mem(bo, &mem, false, + interruptible, no_wait_gpu); out_unlock: if (ret && mem.mm_node) ttm_bo_mem_put(bo, &mem); return ret; } -static int ttm_bo_mem_compat(struct ttm_placement *placement, - struct ttm_mem_reg *mem) +static bool ttm_bo_mem_compat(struct ttm_placement *placement, + struct ttm_mem_reg *mem, + uint32_t *new_flags) { int i; if (mem->mm_node && placement->lpfn != 0 && (mem->start < placement->fpfn || mem->start + mem->num_pages > placement->lpfn)) - return -1; + return false; for (i = 0; i < placement->num_placement; i++) { - if ((placement->placement[i] & mem->placement & - TTM_PL_MASK_CACHING) && - (placement->placement[i] & mem->placement & - TTM_PL_MASK_MEM)) - return i; + *new_flags = placement->placement[i]; + if ((*new_flags & mem->placement & TTM_PL_MASK_CACHING) && + (*new_flags & mem->placement & TTM_PL_MASK_MEM)) + return true; } - return -1; + + for (i = 0; i < placement->num_busy_placement; i++) { + *new_flags = placement->busy_placement[i]; + if ((*new_flags & mem->placement & TTM_PL_MASK_CACHING) && + (*new_flags & mem->placement & TTM_PL_MASK_MEM)) + return true; + } + + return false; } int ttm_bo_validate(struct ttm_buffer_object *bo, struct ttm_placement *placement, - bool interruptible, bool no_wait_reserve, + bool interruptible, bool no_wait_gpu) { int ret; + uint32_t new_flags; - BUG_ON(!atomic_read(&bo->reserved)); + lockdep_assert_held(&bo->resv->lock.base); /* Check that range is valid */ if (placement->lpfn || placement->fpfn) if (placement->fpfn > placement->lpfn || @@ -1125,9 +1045,9 @@ int ttm_bo_validate(struct ttm_buffer_object *bo, /* * Check whether we need to move buffer. */ - ret = ttm_bo_mem_compat(placement, &bo->mem); - if (ret < 0) { - ret = ttm_bo_move_buffer(bo, placement, interruptible, no_wait_reserve, no_wait_gpu); + if (!ttm_bo_mem_compat(placement, &bo->mem, &new_flags)) { + ret = ttm_bo_move_buffer(bo, placement, interruptible, + no_wait_gpu); if (ret) return ret; } else { @@ -1135,7 +1055,7 @@ int ttm_bo_validate(struct ttm_buffer_object *bo, * Use the access and other non-mapping-related flag bits from * the compatible memory placement flags to the active flags */ - ttm_flag_masked(&bo->mem.placement, placement->placement[ret], + ttm_flag_masked(&bo->mem.placement, new_flags, ~TTM_PL_MASK_MEMTYPE); } /* @@ -1165,23 +1085,35 @@ int ttm_bo_init(struct ttm_bo_device *bdev, enum ttm_bo_type type, struct ttm_placement *placement, uint32_t page_alignment, - unsigned long buffer_start, bool interruptible, - struct file *persistant_swap_storage, + struct file *persistent_swap_storage, size_t acc_size, + struct sg_table *sg, void (*destroy) (struct ttm_buffer_object *)) { int ret = 0; unsigned long num_pages; + struct ttm_mem_global *mem_glob = bdev->glob->mem_glob; + bool locked; + + ret = ttm_mem_global_alloc(mem_glob, acc_size, false, false); + if (ret) { + pr_err("Out of kernel memory\n"); + if (destroy) + (*destroy)(bo); + else + kfree(bo); + return -ENOMEM; + } - size += buffer_start & ~PAGE_MASK; num_pages = (size + PAGE_SIZE - 1) >> PAGE_SHIFT; if (num_pages == 0) { - printk(KERN_ERR TTM_PFX "Illegal buffer object size.\n"); + pr_err("Illegal buffer object size\n"); if (destroy) (*destroy)(bo); else kfree(bo); + ttm_mem_global_free(mem_glob, acc_size); return -EINVAL; } bo->destroy = destroy; @@ -1189,12 +1121,11 @@ int ttm_bo_init(struct ttm_bo_device *bdev, kref_init(&bo->kref); kref_init(&bo->list_kref); atomic_set(&bo->cpu_writers, 0); - atomic_set(&bo->reserved, 1); - init_waitqueue_head(&bo->event_queue); INIT_LIST_HEAD(&bo->lru); INIT_LIST_HEAD(&bo->ddestroy); INIT_LIST_HEAD(&bo->swap); INIT_LIST_HEAD(&bo->io_reserve_lru); + mutex_init(&bo->wu_mutex); bo->bdev = bdev; bo->glob = bdev->glob; bo->type = type; @@ -1206,87 +1137,99 @@ int ttm_bo_init(struct ttm_bo_device *bdev, bo->mem.page_alignment = page_alignment; bo->mem.bus.io_reserved_vm = false; bo->mem.bus.io_reserved_count = 0; - bo->buffer_start = buffer_start & PAGE_MASK; bo->priv_flags = 0; bo->mem.placement = (TTM_PL_FLAG_SYSTEM | TTM_PL_FLAG_CACHED); - bo->seq_valid = false; - bo->persistant_swap_storage = persistant_swap_storage; + bo->persistent_swap_storage = persistent_swap_storage; bo->acc_size = acc_size; + bo->sg = sg; + bo->resv = &bo->ttm_resv; + reservation_object_init(bo->resv); atomic_inc(&bo->glob->bo_count); + drm_vma_node_reset(&bo->vma_node); ret = ttm_bo_check_placement(bo, placement); - if (unlikely(ret != 0)) - goto out_err; /* * For ttm_bo_type_device buffers, allocate * address space from the device. */ - if (bo->type == ttm_bo_type_device) { - ret = ttm_bo_setup_vm(bo); - if (ret) - goto out_err; - } + if (likely(!ret) && + (bo->type == ttm_bo_type_device || + bo->type == ttm_bo_type_sg)) + ret = drm_vma_offset_add(&bdev->vma_manager, &bo->vma_node, + bo->mem.num_pages); - ret = ttm_bo_validate(bo, placement, interruptible, false, false); - if (ret) - goto out_err; + locked = ww_mutex_trylock(&bo->resv->lock); + WARN_ON(!locked); - ttm_bo_unreserve(bo); - return 0; + if (likely(!ret)) + ret = ttm_bo_validate(bo, placement, interruptible, false); -out_err: ttm_bo_unreserve(bo); - ttm_bo_unref(&bo); + + if (unlikely(ret)) + ttm_bo_unref(&bo); return ret; } EXPORT_SYMBOL(ttm_bo_init); -static inline size_t ttm_bo_size(struct ttm_bo_global *glob, - unsigned long num_pages) +size_t ttm_bo_acc_size(struct ttm_bo_device *bdev, + unsigned long bo_size, + unsigned struct_size) { - size_t page_array_size = (num_pages * sizeof(void *) + PAGE_SIZE - 1) & - PAGE_MASK; + unsigned npages = (PAGE_ALIGN(bo_size)) >> PAGE_SHIFT; + size_t size = 0; + + size += ttm_round_pot(struct_size); + size += PAGE_ALIGN(npages * sizeof(void *)); + size += ttm_round_pot(sizeof(struct ttm_tt)); + return size; +} +EXPORT_SYMBOL(ttm_bo_acc_size); - return glob->ttm_bo_size + 2 * page_array_size; +size_t ttm_bo_dma_acc_size(struct ttm_bo_device *bdev, + unsigned long bo_size, + unsigned struct_size) +{ + unsigned npages = (PAGE_ALIGN(bo_size)) >> PAGE_SHIFT; + size_t size = 0; + + size += ttm_round_pot(struct_size); + size += PAGE_ALIGN(npages * sizeof(void *)); + size += PAGE_ALIGN(npages * sizeof(dma_addr_t)); + size += ttm_round_pot(sizeof(struct ttm_dma_tt)); + return size; } +EXPORT_SYMBOL(ttm_bo_dma_acc_size); int ttm_bo_create(struct ttm_bo_device *bdev, unsigned long size, enum ttm_bo_type type, struct ttm_placement *placement, uint32_t page_alignment, - unsigned long buffer_start, bool interruptible, - struct file *persistant_swap_storage, + struct file *persistent_swap_storage, struct ttm_buffer_object **p_bo) { struct ttm_buffer_object *bo; - struct ttm_mem_global *mem_glob = bdev->glob->mem_glob; + size_t acc_size; int ret; - size_t acc_size = - ttm_bo_size(bdev->glob, (size + PAGE_SIZE - 1) >> PAGE_SHIFT); - ret = ttm_mem_global_alloc(mem_glob, acc_size, false, false); - if (unlikely(ret != 0)) - return ret; - bo = kzalloc(sizeof(*bo), GFP_KERNEL); - - if (unlikely(bo == NULL)) { - ttm_mem_global_free(mem_glob, acc_size); + if (unlikely(bo == NULL)) return -ENOMEM; - } + acc_size = ttm_bo_acc_size(bdev, size, sizeof(struct ttm_buffer_object)); ret = ttm_bo_init(bdev, bo, size, type, placement, page_alignment, - buffer_start, interruptible, - persistant_swap_storage, acc_size, NULL); + interruptible, persistent_swap_storage, acc_size, + NULL, NULL); if (likely(ret == 0)) *p_bo = bo; return ret; } +EXPORT_SYMBOL(ttm_bo_create); static int ttm_bo_force_list_clean(struct ttm_bo_device *bdev, unsigned mem_type, bool allow_errors) @@ -1302,13 +1245,12 @@ static int ttm_bo_force_list_clean(struct ttm_bo_device *bdev, spin_lock(&glob->lru_lock); while (!list_empty(&man->lru)) { spin_unlock(&glob->lru_lock); - ret = ttm_mem_evict_first(bdev, mem_type, false, false, false); + ret = ttm_mem_evict_first(bdev, mem_type, false, false); if (ret) { if (allow_errors) { return ret; } else { - printk(KERN_ERR TTM_PFX - "Cleanup eviction failed\n"); + pr_err("Cleanup eviction failed\n"); } } spin_lock(&glob->lru_lock); @@ -1323,14 +1265,14 @@ int ttm_bo_clean_mm(struct ttm_bo_device *bdev, unsigned mem_type) int ret = -EINVAL; if (mem_type >= TTM_NUM_MEM_TYPES) { - printk(KERN_ERR TTM_PFX "Illegal memory type %d\n", mem_type); + pr_err("Illegal memory type %d\n", mem_type); return ret; } man = &bdev->man[mem_type]; if (!man->has_type) { - printk(KERN_ERR TTM_PFX "Trying to take down uninitialized " - "memory manager type %u\n", mem_type); + pr_err("Trying to take down uninitialized memory manager type %u\n", + mem_type); return ret; } @@ -1353,16 +1295,12 @@ int ttm_bo_evict_mm(struct ttm_bo_device *bdev, unsigned mem_type) struct ttm_mem_type_manager *man = &bdev->man[mem_type]; if (mem_type == 0 || mem_type >= TTM_NUM_MEM_TYPES) { - printk(KERN_ERR TTM_PFX - "Illegal memory manager memory type %u.\n", - mem_type); + pr_err("Illegal memory manager memory type %u\n", mem_type); return -EINVAL; } if (!man->has_type) { - printk(KERN_ERR TTM_PFX - "Memory type %u has not been initialized.\n", - mem_type); + pr_err("Memory type %u has not been initialized\n", mem_type); return 0; } @@ -1447,18 +1385,10 @@ int ttm_bo_global_init(struct drm_global_reference *ref) ttm_mem_init_shrink(&glob->shrink, ttm_bo_swapout); ret = ttm_mem_register_shrink(glob->mem_glob, &glob->shrink); if (unlikely(ret != 0)) { - printk(KERN_ERR TTM_PFX - "Could not register buffer object swapout.\n"); + pr_err("Could not register buffer object swapout\n"); goto out_no_shrink; } - glob->ttm_bo_extra_size = - ttm_round_pot(sizeof(struct ttm_tt)) + - ttm_round_pot(sizeof(struct ttm_backend)); - - glob->ttm_bo_size = glob->ttm_bo_extra_size + - ttm_round_pot(sizeof(struct ttm_buffer_object)); - atomic_set(&glob->bo_count, 0); ret = kobject_init_and_add( @@ -1488,9 +1418,8 @@ int ttm_bo_device_release(struct ttm_bo_device *bdev) man->use_type = false; if ((i != TTM_PL_SYSTEM) && ttm_bo_clean_mm(bdev, i)) { ret = -EBUSY; - printk(KERN_ERR TTM_PFX - "DRM memory manager type %d " - "is not clean.\n", i); + pr_err("DRM memory manager type %d is not clean\n", + i); } man->has_type = false; } @@ -1513,10 +1442,7 @@ int ttm_bo_device_release(struct ttm_bo_device *bdev) TTM_DEBUG("Swap list was clean\n"); spin_unlock(&glob->lru_lock); - BUG_ON(!drm_mm_clean(&bdev->addr_space_mm)); - write_lock(&bdev->vm_lock); - drm_mm_takedown(&bdev->addr_space_mm); - write_unlock(&bdev->vm_lock); + drm_vma_offset_manager_destroy(&bdev->vma_manager); return ret; } @@ -1525,12 +1451,12 @@ EXPORT_SYMBOL(ttm_bo_device_release); int ttm_bo_device_init(struct ttm_bo_device *bdev, struct ttm_bo_global *glob, struct ttm_bo_driver *driver, + struct address_space *mapping, uint64_t file_page_offset, bool need_dma32) { int ret = -EINVAL; - rwlock_init(&bdev->vm_lock); bdev->driver = driver; memset(bdev->man, 0, sizeof(bdev->man)); @@ -1543,15 +1469,11 @@ int ttm_bo_device_init(struct ttm_bo_device *bdev, if (unlikely(ret != 0)) goto out_no_sys; - bdev->addr_space_rb = RB_ROOT; - ret = drm_mm_init(&bdev->addr_space_mm, file_page_offset, 0x10000000); - if (unlikely(ret != 0)) - goto out_no_addr_mm; - + drm_vma_offset_manager_init(&bdev->vma_manager, file_page_offset, + 0x10000000); INIT_DELAYED_WORK(&bdev->wq, ttm_bo_delayed_workqueue); - bdev->nice_mode = true; INIT_LIST_HEAD(&bdev->ddestroy); - bdev->dev_mapping = NULL; + bdev->dev_mapping = mapping; bdev->glob = glob; bdev->need_dma32 = need_dma32; bdev->val_seq = 0; @@ -1561,8 +1483,6 @@ int ttm_bo_device_init(struct ttm_bo_device *bdev, mutex_unlock(&glob->device_list_mutex); return 0; -out_no_addr_mm: - ttm_bo_clean_mm(bdev, 0); out_no_sys: return ret; } @@ -1592,12 +1512,8 @@ bool ttm_mem_reg_is_pci(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem) void ttm_bo_unmap_virtual_locked(struct ttm_buffer_object *bo) { struct ttm_bo_device *bdev = bo->bdev; - loff_t offset = (loff_t) bo->addr_space_offset; - loff_t holelen = ((loff_t) bo->mem.num_pages) << PAGE_SHIFT; - if (!bdev->dev_mapping) - return; - unmap_mapping_range(bdev->dev_mapping, offset, holelen, 1); + drm_vma_node_unmap(&bo->vma_node, bdev->dev_mapping); ttm_mem_io_free_vm(bo); } @@ -1614,78 +1530,6 @@ void ttm_bo_unmap_virtual(struct ttm_buffer_object *bo) EXPORT_SYMBOL(ttm_bo_unmap_virtual); -static void ttm_bo_vm_insert_rb(struct ttm_buffer_object *bo) -{ - struct ttm_bo_device *bdev = bo->bdev; - struct rb_node **cur = &bdev->addr_space_rb.rb_node; - struct rb_node *parent = NULL; - struct ttm_buffer_object *cur_bo; - unsigned long offset = bo->vm_node->start; - unsigned long cur_offset; - - while (*cur) { - parent = *cur; - cur_bo = rb_entry(parent, struct ttm_buffer_object, vm_rb); - cur_offset = cur_bo->vm_node->start; - if (offset < cur_offset) - cur = &parent->rb_left; - else if (offset > cur_offset) - cur = &parent->rb_right; - else - BUG(); - } - - rb_link_node(&bo->vm_rb, parent, cur); - rb_insert_color(&bo->vm_rb, &bdev->addr_space_rb); -} - -/** - * ttm_bo_setup_vm: - * - * @bo: the buffer to allocate address space for - * - * Allocate address space in the drm device so that applications - * can mmap the buffer and access the contents. This only - * applies to ttm_bo_type_device objects as others are not - * placed in the drm device address space. - */ - -static int ttm_bo_setup_vm(struct ttm_buffer_object *bo) -{ - struct ttm_bo_device *bdev = bo->bdev; - int ret; - -retry_pre_get: - ret = drm_mm_pre_get(&bdev->addr_space_mm); - if (unlikely(ret != 0)) - return ret; - - write_lock(&bdev->vm_lock); - bo->vm_node = drm_mm_search_free(&bdev->addr_space_mm, - bo->mem.num_pages, 0, 0); - - if (unlikely(bo->vm_node == NULL)) { - ret = -ENOMEM; - goto out_unlock; - } - - bo->vm_node = drm_mm_get_block_atomic(bo->vm_node, - bo->mem.num_pages, 0); - - if (unlikely(bo->vm_node == NULL)) { - write_unlock(&bdev->vm_lock); - goto retry_pre_get; - } - - ttm_bo_vm_insert_rb(bo); - write_unlock(&bdev->vm_lock); - bo->addr_space_offset = ((uint64_t) bo->vm_node->start) << PAGE_SHIFT; - - return 0; -out_unlock: - write_unlock(&bdev->vm_lock); - return ret; -} int ttm_bo_wait(struct ttm_buffer_object *bo, bool lazy, bool interruptible, bool no_wait) @@ -1693,7 +1537,6 @@ int ttm_bo_wait(struct ttm_buffer_object *bo, struct ttm_bo_driver *driver = bo->bdev->driver; struct ttm_bo_device *bdev = bo->bdev; void *sync_obj; - void *sync_obj_arg; int ret = 0; if (likely(bo->sync_obj == NULL)) @@ -1701,7 +1544,7 @@ int ttm_bo_wait(struct ttm_buffer_object *bo, while (bo->sync_obj) { - if (driver->sync_obj_signaled(bo->sync_obj, bo->sync_obj_arg)) { + if (driver->sync_obj_signaled(bo->sync_obj)) { void *tmp_obj = bo->sync_obj; bo->sync_obj = NULL; clear_bit(TTM_BO_PRIV_FLAG_MOVING, &bo->priv_flags); @@ -1715,9 +1558,8 @@ int ttm_bo_wait(struct ttm_buffer_object *bo, return -EBUSY; sync_obj = driver->sync_obj_ref(bo->sync_obj); - sync_obj_arg = bo->sync_obj_arg; spin_unlock(&bdev->fence_lock); - ret = driver->sync_obj_wait(sync_obj, sync_obj_arg, + ret = driver->sync_obj_wait(sync_obj, lazy, interruptible); if (unlikely(ret != 0)) { driver->sync_obj_unref(&sync_obj); @@ -1725,8 +1567,7 @@ int ttm_bo_wait(struct ttm_buffer_object *bo, return ret; } spin_lock(&bdev->fence_lock); - if (likely(bo->sync_obj == sync_obj && - bo->sync_obj_arg == sync_obj_arg)) { + if (likely(bo->sync_obj == sync_obj)) { void *tmp_obj = bo->sync_obj; bo->sync_obj = NULL; clear_bit(TTM_BO_PRIV_FLAG_MOVING, @@ -1769,8 +1610,7 @@ EXPORT_SYMBOL(ttm_bo_synccpu_write_grab); void ttm_bo_synccpu_write_release(struct ttm_buffer_object *bo) { - if (atomic_dec_and_test(&bo->cpu_writers)) - wake_up_all(&bo->event_queue); + atomic_dec(&bo->cpu_writers); } EXPORT_SYMBOL(ttm_bo_synccpu_write_release); @@ -1789,39 +1629,25 @@ static int ttm_bo_swapout(struct ttm_mem_shrink *shrink) uint32_t swap_placement = (TTM_PL_FLAG_CACHED | TTM_PL_FLAG_SYSTEM); spin_lock(&glob->lru_lock); - while (ret == -EBUSY) { - if (unlikely(list_empty(&glob->swap_lru))) { - spin_unlock(&glob->lru_lock); - return -EBUSY; - } - - bo = list_first_entry(&glob->swap_lru, - struct ttm_buffer_object, swap); - kref_get(&bo->list_kref); + list_for_each_entry(bo, &glob->swap_lru, swap) { + ret = __ttm_bo_reserve(bo, false, true, false, 0); + if (!ret) + break; + } - if (!list_empty(&bo->ddestroy)) { - spin_unlock(&glob->lru_lock); - (void) ttm_bo_cleanup_refs(bo, false, false, false); - kref_put(&bo->list_kref, ttm_bo_release_list); - continue; - } + if (ret) { + spin_unlock(&glob->lru_lock); + return ret; + } - /** - * Reserve buffer. Since we unlock while sleeping, we need - * to re-check that nobody removed us from the swap-list while - * we slept. - */ + kref_get(&bo->list_kref); - ret = ttm_bo_reserve_locked(bo, false, true, false, 0); - if (unlikely(ret == -EBUSY)) { - spin_unlock(&glob->lru_lock); - ttm_bo_wait_unreserved(bo, false); - kref_put(&bo->list_kref, ttm_bo_release_list); - spin_lock(&glob->lru_lock); - } + if (!list_empty(&bo->ddestroy)) { + ret = ttm_bo_cleanup_refs_and_unlock(bo, false, false); + kref_put(&bo->list_kref, ttm_bo_release_list); + return ret; } - BUG_ON(ret != 0); put_count = ttm_bo_del_from_lru(bo); spin_unlock(&glob->lru_lock); @@ -1847,7 +1673,7 @@ static int ttm_bo_swapout(struct ttm_mem_shrink *shrink) evict_mem.mem_type = TTM_PL_SYSTEM; ret = ttm_bo_handle_move_mem(bo, &evict_mem, true, - false, false, false); + false, false); if (unlikely(ret != 0)) goto out; } @@ -1862,7 +1688,7 @@ static int ttm_bo_swapout(struct ttm_mem_shrink *shrink) if (bo->bdev->driver->swap_notify) bo->bdev->driver->swap_notify(bo); - ret = ttm_tt_swapout(bo->ttm, bo->persistant_swap_storage); + ret = ttm_tt_swapout(bo->ttm, bo->persistent_swap_storage); out: /** @@ -1871,8 +1697,7 @@ out: * already swapped buffer. */ - atomic_set(&bo->reserved, 0); - wake_up_all(&bo->event_queue); + __ttm_bo_unreserve(bo); kref_put(&bo->list_kref, ttm_bo_release_list); return ret; } @@ -1883,3 +1708,35 @@ void ttm_bo_swapout_all(struct ttm_bo_device *bdev) ; } EXPORT_SYMBOL(ttm_bo_swapout_all); + +/** + * ttm_bo_wait_unreserved - interruptible wait for a buffer object to become + * unreserved + * + * @bo: Pointer to buffer + */ +int ttm_bo_wait_unreserved(struct ttm_buffer_object *bo) +{ + int ret; + + /* + * In the absense of a wait_unlocked API, + * Use the bo::wu_mutex to avoid triggering livelocks due to + * concurrent use of this function. Note that this use of + * bo::wu_mutex can go away if we change locking order to + * mmap_sem -> bo::reserve. + */ + ret = mutex_lock_interruptible(&bo->wu_mutex); + if (unlikely(ret != 0)) + return -ERESTARTSYS; + if (!ww_mutex_is_locked(&bo->resv->lock)) + goto out_unlock; + ret = __ttm_bo_reserve(bo, true, false, false, NULL); + if (unlikely(ret != 0)) + goto out_unlock; + __ttm_bo_unreserve(bo); + +out_unlock: + mutex_unlock(&bo->wu_mutex); + return ret; +} diff --git a/drivers/gpu/drm/ttm/ttm_bo_manager.c b/drivers/gpu/drm/ttm/ttm_bo_manager.c index 038e947d00f..bd850c9f4bc 100644 --- a/drivers/gpu/drm/ttm/ttm_bo_manager.c +++ b/drivers/gpu/drm/ttm/ttm_bo_manager.c @@ -28,10 +28,10 @@ * Authors: Thomas Hellstrom <thellstrom-at-vmware-dot-com> */ -#include "ttm/ttm_module.h" -#include "ttm/ttm_bo_driver.h" -#include "ttm/ttm_placement.h" -#include "drm_mm.h" +#include <drm/ttm/ttm_module.h> +#include <drm/ttm/ttm_bo_driver.h> +#include <drm/ttm/ttm_placement.h> +#include <drm/drm_mm.h> #include <linux/slab.h> #include <linux/spinlock.h> #include <linux/module.h> @@ -55,34 +55,36 @@ static int ttm_bo_man_get_node(struct ttm_mem_type_manager *man, struct ttm_range_manager *rman = (struct ttm_range_manager *) man->priv; struct drm_mm *mm = &rman->mm; struct drm_mm_node *node = NULL; + enum drm_mm_allocator_flags aflags = DRM_MM_CREATE_DEFAULT; unsigned long lpfn; int ret; lpfn = placement->lpfn; if (!lpfn) lpfn = man->size; - do { - ret = drm_mm_pre_get(mm); - if (unlikely(ret)) - return ret; - spin_lock(&rman->lock); - node = drm_mm_search_free_in_range(mm, - mem->num_pages, mem->page_alignment, - placement->fpfn, lpfn, 1); - if (unlikely(node == NULL)) { - spin_unlock(&rman->lock); - return 0; - } - node = drm_mm_get_block_atomic_range(node, mem->num_pages, - mem->page_alignment, - placement->fpfn, - lpfn); - spin_unlock(&rman->lock); - } while (node == NULL); + node = kzalloc(sizeof(*node), GFP_KERNEL); + if (!node) + return -ENOMEM; + + if (bo->mem.placement & TTM_PL_FLAG_TOPDOWN) + aflags = DRM_MM_CREATE_TOP; + + spin_lock(&rman->lock); + ret = drm_mm_insert_node_in_range_generic(mm, node, mem->num_pages, + mem->page_alignment, 0, + placement->fpfn, lpfn, + DRM_MM_SEARCH_BEST, + aflags); + spin_unlock(&rman->lock); + + if (unlikely(ret)) { + kfree(node); + } else { + mem->mm_node = node; + mem->start = node->start; + } - mem->mm_node = node; - mem->start = node->start; return 0; } @@ -93,8 +95,10 @@ static void ttm_bo_man_put_node(struct ttm_mem_type_manager *man, if (mem->mm_node) { spin_lock(&rman->lock); - drm_mm_put_block(mem->mm_node); + drm_mm_remove_node(mem->mm_node); spin_unlock(&rman->lock); + + kfree(mem->mm_node); mem->mm_node = NULL; } } @@ -103,18 +107,12 @@ static int ttm_bo_man_init(struct ttm_mem_type_manager *man, unsigned long p_size) { struct ttm_range_manager *rman; - int ret; rman = kzalloc(sizeof(*rman), GFP_KERNEL); if (!rman) return -ENOMEM; - ret = drm_mm_init(&rman->mm, 0, p_size); - if (ret) { - kfree(rman); - return ret; - } - + drm_mm_init(&rman->mm, 0, p_size); spin_lock_init(&rman->lock); man->priv = rman; return 0; diff --git a/drivers/gpu/drm/ttm/ttm_bo_util.c b/drivers/gpu/drm/ttm/ttm_bo_util.c index 77dbf408c0d..1df856f7856 100644 --- a/drivers/gpu/drm/ttm/ttm_bo_util.c +++ b/drivers/gpu/drm/ttm/ttm_bo_util.c @@ -28,8 +28,9 @@ * Authors: Thomas Hellstrom <thellstrom-at-vmware-dot-com> */ -#include "ttm/ttm_bo_driver.h" -#include "ttm/ttm_placement.h" +#include <drm/ttm/ttm_bo_driver.h> +#include <drm/ttm/ttm_placement.h> +#include <drm/drm_vma_manager.h> #include <linux/io.h> #include <linux/highmem.h> #include <linux/wait.h> @@ -43,7 +44,7 @@ void ttm_bo_free_old_node(struct ttm_buffer_object *bo) } int ttm_bo_move_ttm(struct ttm_buffer_object *bo, - bool evict, bool no_wait_reserve, + bool evict, bool no_wait_gpu, struct ttm_mem_reg *new_mem) { struct ttm_tt *ttm = bo->ttm; @@ -86,6 +87,7 @@ int ttm_mem_io_lock(struct ttm_mem_type_manager *man, bool interruptible) mutex_lock(&man->io_reserve_mutex); return 0; } +EXPORT_SYMBOL(ttm_mem_io_lock); void ttm_mem_io_unlock(struct ttm_mem_type_manager *man) { @@ -94,6 +96,7 @@ void ttm_mem_io_unlock(struct ttm_mem_type_manager *man) mutex_unlock(&man->io_reserve_mutex); } +EXPORT_SYMBOL(ttm_mem_io_unlock); static int ttm_mem_io_evict(struct ttm_mem_type_manager *man) { @@ -111,8 +114,9 @@ static int ttm_mem_io_evict(struct ttm_mem_type_manager *man) return 0; } -static int ttm_mem_io_reserve(struct ttm_bo_device *bdev, - struct ttm_mem_reg *mem) + +int ttm_mem_io_reserve(struct ttm_bo_device *bdev, + struct ttm_mem_reg *mem) { struct ttm_mem_type_manager *man = &bdev->man[mem->mem_type]; int ret = 0; @@ -134,9 +138,10 @@ retry: } return ret; } +EXPORT_SYMBOL(ttm_mem_io_reserve); -static void ttm_mem_io_free(struct ttm_bo_device *bdev, - struct ttm_mem_reg *mem) +void ttm_mem_io_free(struct ttm_bo_device *bdev, + struct ttm_mem_reg *mem) { struct ttm_mem_type_manager *man = &bdev->man[mem->mem_type]; @@ -149,6 +154,7 @@ static void ttm_mem_io_free(struct ttm_bo_device *bdev, bdev->driver->io_mem_free(bdev, mem); } +EXPORT_SYMBOL(ttm_mem_io_free); int ttm_mem_io_reserve_vm(struct ttm_buffer_object *bo) { @@ -181,7 +187,7 @@ void ttm_mem_io_free_vm(struct ttm_buffer_object *bo) } } -int ttm_mem_reg_ioremap(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem, +static int ttm_mem_reg_ioremap(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem, void **virtual) { struct ttm_mem_type_manager *man = &bdev->man[mem->mem_type]; @@ -213,7 +219,7 @@ int ttm_mem_reg_ioremap(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem, return 0; } -void ttm_mem_reg_iounmap(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem, +static void ttm_mem_reg_iounmap(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem, void *virtual) { struct ttm_mem_type_manager *man; @@ -244,7 +250,7 @@ static int ttm_copy_io_ttm_page(struct ttm_tt *ttm, void *src, unsigned long page, pgprot_t prot) { - struct page *d = ttm_tt_get_page(ttm, page); + struct page *d = ttm->pages[page]; void *dst; if (!d) @@ -281,7 +287,7 @@ static int ttm_copy_ttm_io_page(struct ttm_tt *ttm, void *dst, unsigned long page, pgprot_t prot) { - struct page *s = ttm_tt_get_page(ttm, page); + struct page *s = ttm->pages[page]; void *src; if (!s) @@ -314,14 +320,14 @@ static int ttm_copy_ttm_io_page(struct ttm_tt *ttm, void *dst, } int ttm_bo_move_memcpy(struct ttm_buffer_object *bo, - bool evict, bool no_wait_reserve, bool no_wait_gpu, + bool evict, bool no_wait_gpu, struct ttm_mem_reg *new_mem) { struct ttm_bo_device *bdev = bo->bdev; struct ttm_mem_type_manager *man = &bdev->man[new_mem->mem_type]; struct ttm_tt *ttm = bo->ttm; struct ttm_mem_reg *old_mem = &bo->mem; - struct ttm_mem_reg old_copy; + struct ttm_mem_reg old_copy = *old_mem; void *old_iomap; void *new_iomap; int ret; @@ -337,10 +343,30 @@ int ttm_bo_move_memcpy(struct ttm_buffer_object *bo, if (ret) goto out; + /* + * Single TTM move. NOP. + */ if (old_iomap == NULL && new_iomap == NULL) goto out2; - if (old_iomap == NULL && ttm == NULL) + + /* + * Don't move nonexistent data. Clear destination instead. + */ + if (old_iomap == NULL && + (ttm == NULL || (ttm->state == tt_unpopulated && + !(ttm->page_flags & TTM_PAGE_FLAG_SWAPPED)))) { + memset_io(new_iomap, 0, new_mem->num_pages*PAGE_SIZE); goto out2; + } + + /* + * TTM might be null for moves within the same region. + */ + if (ttm && ttm->state == tt_unpopulated) { + ret = ttm->bdev->driver->ttm_tt_populate(ttm); + if (ret) + goto out1; + } add = 0; dir = 1; @@ -384,7 +410,12 @@ out1: ttm_mem_reg_iounmap(bdev, old_mem, new_iomap); out: ttm_mem_reg_iounmap(bdev, &old_copy, old_iomap); - ttm_bo_mem_put(bo, &old_copy); + + /* + * On error, keep the mm node! + */ + if (!ret) + ttm_bo_mem_put(bo, &old_copy); return ret; } EXPORT_SYMBOL(ttm_bo_move_memcpy); @@ -415,8 +446,9 @@ static int ttm_buffer_object_transfer(struct ttm_buffer_object *bo, struct ttm_buffer_object *fbo; struct ttm_bo_device *bdev = bo->bdev; struct ttm_bo_driver *driver = bdev->driver; + int ret; - fbo = kzalloc(sizeof(*fbo), GFP_KERNEL); + fbo = kmalloc(sizeof(*fbo), GFP_KERNEL); if (!fbo) return -ENOMEM; @@ -427,18 +459,27 @@ static int ttm_buffer_object_transfer(struct ttm_buffer_object *bo, * TODO: Explicit member copy would probably be better here. */ - init_waitqueue_head(&fbo->event_queue); INIT_LIST_HEAD(&fbo->ddestroy); INIT_LIST_HEAD(&fbo->lru); INIT_LIST_HEAD(&fbo->swap); INIT_LIST_HEAD(&fbo->io_reserve_lru); - fbo->vm_node = NULL; + drm_vma_node_reset(&fbo->vma_node); atomic_set(&fbo->cpu_writers, 0); - fbo->sync_obj = driver->sync_obj_ref(bo->sync_obj); + spin_lock(&bdev->fence_lock); + if (bo->sync_obj) + fbo->sync_obj = driver->sync_obj_ref(bo->sync_obj); + else + fbo->sync_obj = NULL; + spin_unlock(&bdev->fence_lock); kref_init(&fbo->list_kref); kref_init(&fbo->kref); fbo->destroy = &ttm_transfered_destroy; + fbo->acc_size = 0; + fbo->resv = &fbo->ttm_resv; + reservation_object_init(fbo->resv); + ret = ww_mutex_trylock(&fbo->resv->lock); + WARN_ON(!ret); *new_obj = fbo; return 0; @@ -465,7 +506,7 @@ pgprot_t ttm_io_prot(uint32_t caching_flags, pgprot_t tmp) else tmp = pgprot_noncached(tmp); #endif -#if defined(__sparc__) +#if defined(__sparc__) || defined(__mips__) if (!(caching_flags & TTM_PL_FLAG_CACHED)) tmp = pgprot_noncached(tmp); #endif @@ -502,10 +543,16 @@ static int ttm_bo_kmap_ttm(struct ttm_buffer_object *bo, { struct ttm_mem_reg *mem = &bo->mem; pgprot_t prot; struct ttm_tt *ttm = bo->ttm; - struct page *d; - int i; + int ret; BUG_ON(!ttm); + + if (ttm->state == tt_unpopulated) { + ret = ttm->bdev->driver->ttm_tt_populate(ttm); + if (ret) + return ret; + } + if (num_pages == 1 && (mem->placement & TTM_PL_FLAG_CACHED)) { /* * We're mapping a single page, and the desired @@ -513,18 +560,9 @@ static int ttm_bo_kmap_ttm(struct ttm_buffer_object *bo, */ map->bo_kmap_type = ttm_bo_map_kmap; - map->page = ttm_tt_get_page(ttm, start_page); + map->page = ttm->pages[start_page]; map->virtual = kmap(map->page); } else { - /* - * Populate the part we're mapping; - */ - for (i = start_page; i < start_page + num_pages; ++i) { - d = ttm_tt_get_page(ttm, i); - if (!d) - return -ENOMEM; - } - /* * We need to use vmap to get the desired page protection * or to make the buffer object look contiguous. @@ -556,7 +594,7 @@ int ttm_bo_kmap(struct ttm_buffer_object *bo, if (start_page > bo->num_pages) return -EINVAL; #if 0 - if (num_pages > 1 && !DRM_SUSER(DRM_CURPROC)) + if (num_pages > 1 && !capable(CAP_SYS_ADMIN)) return -EPERM; #endif (void) ttm_mem_io_lock(man, false); @@ -607,8 +645,7 @@ EXPORT_SYMBOL(ttm_bo_kunmap); int ttm_bo_move_accel_cleanup(struct ttm_buffer_object *bo, void *sync_obj, - void *sync_obj_arg, - bool evict, bool no_wait_reserve, + bool evict, bool no_wait_gpu, struct ttm_mem_reg *new_mem) { @@ -626,7 +663,6 @@ int ttm_bo_move_accel_cleanup(struct ttm_buffer_object *bo, bo->sync_obj = NULL; } bo->sync_obj = driver->sync_obj_ref(sync_obj); - bo->sync_obj_arg = sync_obj_arg; if (evict) { ret = ttm_bo_wait(bo, false, false, false); spin_unlock(&bdev->fence_lock); @@ -635,13 +671,13 @@ int ttm_bo_move_accel_cleanup(struct ttm_buffer_object *bo, if (ret) return ret; - ttm_bo_free_old_node(bo); if ((man->flags & TTM_MEMTYPE_FLAG_FIXED) && (bo->ttm != NULL)) { ttm_tt_unbind(bo->ttm); ttm_tt_destroy(bo->ttm); bo->ttm = NULL; } + ttm_bo_free_old_node(bo); } else { /** * This should help pipeline ordinary buffer moves. diff --git a/drivers/gpu/drm/ttm/ttm_bo_vm.c b/drivers/gpu/drm/ttm/ttm_bo_vm.c index 221b924aceb..0ce48e5a9cb 100644 --- a/drivers/gpu/drm/ttm/ttm_bo_vm.c +++ b/drivers/gpu/drm/ttm/ttm_bo_vm.c @@ -28,9 +28,12 @@ * Authors: Thomas Hellstrom <thellstrom-at-vmware-dot-com> */ +#define pr_fmt(fmt) "[TTM] " fmt + #include <ttm/ttm_module.h> #include <ttm/ttm_bo_driver.h> #include <ttm/ttm_placement.h> +#include <drm/drm_vma_manager.h> #include <linux/mm.h> #include <linux/rbtree.h> #include <linux/module.h> @@ -38,35 +41,49 @@ #define TTM_BO_VM_NUM_PREFAULT 16 -static struct ttm_buffer_object *ttm_bo_vm_lookup_rb(struct ttm_bo_device *bdev, - unsigned long page_start, - unsigned long num_pages) +static int ttm_bo_vm_fault_idle(struct ttm_buffer_object *bo, + struct vm_area_struct *vma, + struct vm_fault *vmf) { - struct rb_node *cur = bdev->addr_space_rb.rb_node; - unsigned long cur_offset; - struct ttm_buffer_object *bo; - struct ttm_buffer_object *best_bo = NULL; - - while (likely(cur != NULL)) { - bo = rb_entry(cur, struct ttm_buffer_object, vm_rb); - cur_offset = bo->vm_node->start; - if (page_start >= cur_offset) { - cur = cur->rb_right; - best_bo = bo; - if (page_start == cur_offset) - break; - } else - cur = cur->rb_left; - } + struct ttm_bo_device *bdev = bo->bdev; + int ret = 0; + + spin_lock(&bdev->fence_lock); + if (likely(!test_bit(TTM_BO_PRIV_FLAG_MOVING, &bo->priv_flags))) + goto out_unlock; + + /* + * Quick non-stalling check for idle. + */ + ret = ttm_bo_wait(bo, false, false, true); + if (likely(ret == 0)) + goto out_unlock; + + /* + * If possible, avoid waiting for GPU with mmap_sem + * held. + */ + if (vmf->flags & FAULT_FLAG_ALLOW_RETRY) { + ret = VM_FAULT_RETRY; + if (vmf->flags & FAULT_FLAG_RETRY_NOWAIT) + goto out_unlock; - if (unlikely(best_bo == NULL)) - return NULL; + up_read(&vma->vm_mm->mmap_sem); + (void) ttm_bo_wait(bo, false, true, false); + goto out_unlock; + } - if (unlikely((best_bo->vm_node->start + best_bo->num_pages) < - (page_start + num_pages))) - return NULL; + /* + * Ordinary wait. + */ + ret = ttm_bo_wait(bo, false, true, false); + if (unlikely(ret != 0)) + ret = (ret != -ERESTARTSYS) ? VM_FAULT_SIGBUS : + VM_FAULT_NOPAGE; - return best_bo; +out_unlock: + spin_unlock(&bdev->fence_lock); + return ret; } static int ttm_bo_vm_fault(struct vm_area_struct *vma, struct vm_fault *vmf) @@ -85,27 +102,51 @@ static int ttm_bo_vm_fault(struct vm_area_struct *vma, struct vm_fault *vmf) int retval = VM_FAULT_NOPAGE; struct ttm_mem_type_manager *man = &bdev->man[bo->mem.mem_type]; + struct vm_area_struct cvma; /* * Work around locking order reversal in fault / nopfn * between mmap_sem and bo_reserve: Perform a trylock operation - * for reserve, and if it fails, retry the fault after scheduling. + * for reserve, and if it fails, retry the fault after waiting + * for the buffer to become unreserved. */ - - ret = ttm_bo_reserve(bo, true, true, false, 0); + ret = ttm_bo_reserve(bo, true, true, false, NULL); if (unlikely(ret != 0)) { - if (ret == -EBUSY) - set_need_resched(); + if (ret != -EBUSY) + return VM_FAULT_NOPAGE; + + if (vmf->flags & FAULT_FLAG_ALLOW_RETRY) { + if (!(vmf->flags & FAULT_FLAG_RETRY_NOWAIT)) { + up_read(&vma->vm_mm->mmap_sem); + (void) ttm_bo_wait_unreserved(bo); + } + + return VM_FAULT_RETRY; + } + + /* + * If we'd want to change locking order to + * mmap_sem -> bo::reserve, we'd use a blocking reserve here + * instead of retrying the fault... + */ return VM_FAULT_NOPAGE; } + /* + * Refuse to fault imported pages. This should be handled + * (if at all) by redirecting mmap to the exporter. + */ + if (bo->ttm && (bo->ttm->page_flags & TTM_PAGE_FLAG_SG)) { + retval = VM_FAULT_SIGBUS; + goto out_unlock; + } + if (bdev->driver->fault_reserve_notify) { ret = bdev->driver->fault_reserve_notify(bo); switch (ret) { case 0: break; case -EBUSY: - set_need_resched(); case -ERESTARTSYS: retval = VM_FAULT_NOPAGE; goto out_unlock; @@ -119,18 +160,11 @@ static int ttm_bo_vm_fault(struct vm_area_struct *vma, struct vm_fault *vmf) * Wait for buffer data in transit, due to a pipelined * move. */ - - spin_lock(&bdev->fence_lock); - if (test_bit(TTM_BO_PRIV_FLAG_MOVING, &bo->priv_flags)) { - ret = ttm_bo_wait(bo, false, true, false); - spin_unlock(&bdev->fence_lock); - if (unlikely(ret != 0)) { - retval = (ret != -ERESTARTSYS) ? - VM_FAULT_SIGBUS : VM_FAULT_NOPAGE; - goto out_unlock; - } - } else - spin_unlock(&bdev->fence_lock); + ret = ttm_bo_vm_fault_idle(bo, vma, vmf); + if (unlikely(ret != 0)) { + retval = ret; + goto out_unlock; + } ret = ttm_mem_io_lock(man, true); if (unlikely(ret != 0)) { @@ -144,9 +178,9 @@ static int ttm_bo_vm_fault(struct vm_area_struct *vma, struct vm_fault *vmf) } page_offset = ((address - vma->vm_start) >> PAGE_SHIFT) + - bo->vm_node->start - vma->vm_pgoff; - page_last = ((vma->vm_end - vma->vm_start) >> PAGE_SHIFT) + - bo->vm_node->start - vma->vm_pgoff; + vma->vm_pgoff - drm_vma_node_start(&bo->vma_node); + page_last = vma_pages(vma) + vma->vm_pgoff - + drm_vma_node_start(&bo->vma_node); if (unlikely(page_offset >= bo->num_pages)) { retval = VM_FAULT_SIGBUS; @@ -154,48 +188,55 @@ static int ttm_bo_vm_fault(struct vm_area_struct *vma, struct vm_fault *vmf) } /* - * Strictly, we're not allowed to modify vma->vm_page_prot here, - * since the mmap_sem is only held in read mode. However, we - * modify only the caching bits of vma->vm_page_prot and - * consider those bits protected by - * the bo->mutex, as we should be the only writers. - * There shouldn't really be any readers of these bits except - * within vm_insert_mixed()? fork? - * - * TODO: Add a list of vmas to the bo, and change the - * vma->vm_page_prot when the object changes caching policy, with - * the correct locks held. + * Make a local vma copy to modify the page_prot member + * and vm_flags if necessary. The vma parameter is protected + * by mmap_sem in write mode. */ + cvma = *vma; + cvma.vm_page_prot = vm_get_page_prot(cvma.vm_flags); + if (bo->mem.bus.is_iomem) { - vma->vm_page_prot = ttm_io_prot(bo->mem.placement, - vma->vm_page_prot); + cvma.vm_page_prot = ttm_io_prot(bo->mem.placement, + cvma.vm_page_prot); } else { ttm = bo->ttm; - vma->vm_page_prot = (bo->mem.placement & TTM_PL_FLAG_CACHED) ? - vm_get_page_prot(vma->vm_flags) : - ttm_io_prot(bo->mem.placement, vma->vm_page_prot); + if (!(bo->mem.placement & TTM_PL_FLAG_CACHED)) + cvma.vm_page_prot = ttm_io_prot(bo->mem.placement, + cvma.vm_page_prot); + + /* Allocate all page at once, most common usage */ + if (ttm->bdev->driver->ttm_tt_populate(ttm)) { + retval = VM_FAULT_OOM; + goto out_io_unlock; + } } /* * Speculatively prefault a number of pages. Only error on * first page. */ - for (i = 0; i < TTM_BO_VM_NUM_PREFAULT; ++i) { if (bo->mem.bus.is_iomem) pfn = ((bo->mem.bus.base + bo->mem.bus.offset) >> PAGE_SHIFT) + page_offset; else { - page = ttm_tt_get_page(ttm, page_offset); + page = ttm->pages[page_offset]; if (unlikely(!page && i == 0)) { retval = VM_FAULT_OOM; goto out_io_unlock; } else if (unlikely(!page)) { break; } + page->mapping = vma->vm_file->f_mapping; + page->index = drm_vma_node_start(&bo->vma_node) + + page_offset; pfn = page_to_pfn(page); } - ret = vm_insert_mixed(vma, address, pfn); + if (vma->vm_flags & VM_MIXEDMAP) + ret = vm_insert_mixed(&cvma, address, pfn); + else + ret = vm_insert_pfn(&cvma, address, pfn); + /* * Somebody beat us to this PTE or prefaulting to * an already populated PTE, or prefaulting error. @@ -225,6 +266,8 @@ static void ttm_bo_vm_open(struct vm_area_struct *vma) struct ttm_buffer_object *bo = (struct ttm_buffer_object *)vma->vm_private_data; + WARN_ON(bo->bdev->dev_mapping != vma->vm_file->f_mapping); + (void)ttm_bo_reference(bo); } @@ -242,6 +285,30 @@ static const struct vm_operations_struct ttm_bo_vm_ops = { .close = ttm_bo_vm_close }; +static struct ttm_buffer_object *ttm_bo_vm_lookup(struct ttm_bo_device *bdev, + unsigned long offset, + unsigned long pages) +{ + struct drm_vma_offset_node *node; + struct ttm_buffer_object *bo = NULL; + + drm_vma_offset_lock_lookup(&bdev->vma_manager); + + node = drm_vma_offset_lookup_locked(&bdev->vma_manager, offset, pages); + if (likely(node)) { + bo = container_of(node, struct ttm_buffer_object, vma_node); + if (!kref_get_unless_zero(&bo->kref)) + bo = NULL; + } + + drm_vma_offset_unlock_lookup(&bdev->vma_manager); + + if (!bo) + pr_err("Could not find buffer object to map\n"); + + return bo; +} + int ttm_bo_mmap(struct file *filp, struct vm_area_struct *vma, struct ttm_bo_device *bdev) { @@ -249,18 +316,9 @@ int ttm_bo_mmap(struct file *filp, struct vm_area_struct *vma, struct ttm_buffer_object *bo; int ret; - read_lock(&bdev->vm_lock); - bo = ttm_bo_vm_lookup_rb(bdev, vma->vm_pgoff, - (vma->vm_end - vma->vm_start) >> PAGE_SHIFT); - if (likely(bo != NULL)) - ttm_bo_reference(bo); - read_unlock(&bdev->vm_lock); - - if (unlikely(bo == NULL)) { - printk(KERN_ERR TTM_PFX - "Could not find buffer object to map.\n"); + bo = ttm_bo_vm_lookup(bdev, vma->vm_pgoff, vma_pages(vma)); + if (unlikely(!bo)) return -EINVAL; - } driver = bo->bdev->driver; if (unlikely(!driver->verify_access)) { @@ -279,7 +337,16 @@ int ttm_bo_mmap(struct file *filp, struct vm_area_struct *vma, */ vma->vm_private_data = bo; - vma->vm_flags |= VM_RESERVED | VM_IO | VM_MIXEDMAP | VM_DONTEXPAND; + + /* + * We'd like to use VM_PFNMAP on shared mappings, where + * (vma->vm_flags & VM_SHARED) != 0, for performance reasons, + * but for some reason VM_PFNMAP + x86 PAT + write-combine is very + * bad for performance. Until that has been sorted out, use + * VM_MIXEDMAP on all mappings. See freedesktop.org bug #75719 + */ + vma->vm_flags |= VM_MIXEDMAP; + vma->vm_flags |= VM_IO | VM_DONTEXPAND | VM_DONTDUMP; return 0; out_unref: ttm_bo_unref(&bo); @@ -294,166 +361,8 @@ int ttm_fbdev_mmap(struct vm_area_struct *vma, struct ttm_buffer_object *bo) vma->vm_ops = &ttm_bo_vm_ops; vma->vm_private_data = ttm_bo_reference(bo); - vma->vm_flags |= VM_RESERVED | VM_IO | VM_MIXEDMAP | VM_DONTEXPAND; + vma->vm_flags |= VM_MIXEDMAP; + vma->vm_flags |= VM_IO | VM_DONTEXPAND; return 0; } EXPORT_SYMBOL(ttm_fbdev_mmap); - - -ssize_t ttm_bo_io(struct ttm_bo_device *bdev, struct file *filp, - const char __user *wbuf, char __user *rbuf, size_t count, - loff_t *f_pos, bool write) -{ - struct ttm_buffer_object *bo; - struct ttm_bo_driver *driver; - struct ttm_bo_kmap_obj map; - unsigned long dev_offset = (*f_pos >> PAGE_SHIFT); - unsigned long kmap_offset; - unsigned long kmap_end; - unsigned long kmap_num; - size_t io_size; - unsigned int page_offset; - char *virtual; - int ret; - bool no_wait = false; - bool dummy; - - read_lock(&bdev->vm_lock); - bo = ttm_bo_vm_lookup_rb(bdev, dev_offset, 1); - if (likely(bo != NULL)) - ttm_bo_reference(bo); - read_unlock(&bdev->vm_lock); - - if (unlikely(bo == NULL)) - return -EFAULT; - - driver = bo->bdev->driver; - if (unlikely(!driver->verify_access)) { - ret = -EPERM; - goto out_unref; - } - - ret = driver->verify_access(bo, filp); - if (unlikely(ret != 0)) - goto out_unref; - - kmap_offset = dev_offset - bo->vm_node->start; - if (unlikely(kmap_offset >= bo->num_pages)) { - ret = -EFBIG; - goto out_unref; - } - - page_offset = *f_pos & ~PAGE_MASK; - io_size = bo->num_pages - kmap_offset; - io_size = (io_size << PAGE_SHIFT) - page_offset; - if (count < io_size) - io_size = count; - - kmap_end = (*f_pos + count - 1) >> PAGE_SHIFT; - kmap_num = kmap_end - kmap_offset + 1; - - ret = ttm_bo_reserve(bo, true, no_wait, false, 0); - - switch (ret) { - case 0: - break; - case -EBUSY: - ret = -EAGAIN; - goto out_unref; - default: - goto out_unref; - } - - ret = ttm_bo_kmap(bo, kmap_offset, kmap_num, &map); - if (unlikely(ret != 0)) { - ttm_bo_unreserve(bo); - goto out_unref; - } - - virtual = ttm_kmap_obj_virtual(&map, &dummy); - virtual += page_offset; - - if (write) - ret = copy_from_user(virtual, wbuf, io_size); - else - ret = copy_to_user(rbuf, virtual, io_size); - - ttm_bo_kunmap(&map); - ttm_bo_unreserve(bo); - ttm_bo_unref(&bo); - - if (unlikely(ret != 0)) - return -EFBIG; - - *f_pos += io_size; - - return io_size; -out_unref: - ttm_bo_unref(&bo); - return ret; -} - -ssize_t ttm_bo_fbdev_io(struct ttm_buffer_object *bo, const char __user *wbuf, - char __user *rbuf, size_t count, loff_t *f_pos, - bool write) -{ - struct ttm_bo_kmap_obj map; - unsigned long kmap_offset; - unsigned long kmap_end; - unsigned long kmap_num; - size_t io_size; - unsigned int page_offset; - char *virtual; - int ret; - bool no_wait = false; - bool dummy; - - kmap_offset = (*f_pos >> PAGE_SHIFT); - if (unlikely(kmap_offset >= bo->num_pages)) - return -EFBIG; - - page_offset = *f_pos & ~PAGE_MASK; - io_size = bo->num_pages - kmap_offset; - io_size = (io_size << PAGE_SHIFT) - page_offset; - if (count < io_size) - io_size = count; - - kmap_end = (*f_pos + count - 1) >> PAGE_SHIFT; - kmap_num = kmap_end - kmap_offset + 1; - - ret = ttm_bo_reserve(bo, true, no_wait, false, 0); - - switch (ret) { - case 0: - break; - case -EBUSY: - return -EAGAIN; - default: - return ret; - } - - ret = ttm_bo_kmap(bo, kmap_offset, kmap_num, &map); - if (unlikely(ret != 0)) { - ttm_bo_unreserve(bo); - return ret; - } - - virtual = ttm_kmap_obj_virtual(&map, &dummy); - virtual += page_offset; - - if (write) - ret = copy_from_user(virtual, wbuf, io_size); - else - ret = copy_to_user(rbuf, virtual, io_size); - - ttm_bo_kunmap(&map); - ttm_bo_unreserve(bo); - ttm_bo_unref(&bo); - - if (unlikely(ret != 0)) - return ret; - - *f_pos += io_size; - - return io_size; -} diff --git a/drivers/gpu/drm/ttm/ttm_execbuf_util.c b/drivers/gpu/drm/ttm/ttm_execbuf_util.c index 3832fe10b4d..e8dac875852 100644 --- a/drivers/gpu/drm/ttm/ttm_execbuf_util.c +++ b/drivers/gpu/drm/ttm/ttm_execbuf_util.c @@ -25,9 +25,9 @@ * **************************************************************************/ -#include "ttm/ttm_execbuf_util.h" -#include "ttm/ttm_bo_driver.h" -#include "ttm/ttm_placement.h" +#include <drm/ttm/ttm_execbuf_util.h> +#include <drm/ttm/ttm_bo_driver.h> +#include <drm/ttm/ttm_placement.h> #include <linux/wait.h> #include <linux/sched.h> #include <linux/module.h> @@ -41,14 +41,12 @@ static void ttm_eu_backoff_reservation_locked(struct list_head *list) if (!entry->reserved) continue; + entry->reserved = false; if (entry->removed) { ttm_bo_add_to_lru(bo); entry->removed = false; - } - entry->reserved = false; - atomic_set(&bo->reserved, 0); - wake_up_all(&bo->event_queue); + __ttm_bo_unreserve(bo); } } @@ -82,23 +80,8 @@ static void ttm_eu_list_ref_sub(struct list_head *list) } } -static int ttm_eu_wait_unreserved_locked(struct list_head *list, - struct ttm_buffer_object *bo) -{ - struct ttm_bo_global *glob = bo->glob; - int ret; - - ttm_eu_del_from_lru_locked(list); - spin_unlock(&glob->lru_lock); - ret = ttm_bo_wait_unreserved(bo, true); - spin_lock(&glob->lru_lock); - if (unlikely(ret != 0)) - ttm_eu_backoff_reservation_locked(list); - return ret; -} - - -void ttm_eu_backoff_reservation(struct list_head *list) +void ttm_eu_backoff_reservation(struct ww_acquire_ctx *ticket, + struct list_head *list) { struct ttm_validate_buffer *entry; struct ttm_bo_global *glob; @@ -110,6 +93,8 @@ void ttm_eu_backoff_reservation(struct list_head *list) glob = entry->bo->glob; spin_lock(&glob->lru_lock); ttm_eu_backoff_reservation_locked(list); + if (ticket) + ww_acquire_fini(ticket); spin_unlock(&glob->lru_lock); } EXPORT_SYMBOL(ttm_eu_backoff_reservation); @@ -126,12 +111,12 @@ EXPORT_SYMBOL(ttm_eu_backoff_reservation); * buffers in different orders. */ -int ttm_eu_reserve_buffers(struct list_head *list) +int ttm_eu_reserve_buffers(struct ww_acquire_ctx *ticket, + struct list_head *list) { struct ttm_bo_global *glob; struct ttm_validate_buffer *entry; int ret; - uint32_t val_seq; if (list_empty(list)) return 0; @@ -145,62 +130,77 @@ int ttm_eu_reserve_buffers(struct list_head *list) entry = list_first_entry(list, struct ttm_validate_buffer, head); glob = entry->bo->glob; + if (ticket) + ww_acquire_init(ticket, &reservation_ww_class); retry: - spin_lock(&glob->lru_lock); - val_seq = entry->bo->bdev->val_seq++; - list_for_each_entry(entry, list, head) { struct ttm_buffer_object *bo = entry->bo; -retry_this_bo: - ret = ttm_bo_reserve_locked(bo, true, true, true, val_seq); - switch (ret) { - case 0: - break; - case -EBUSY: - ret = ttm_eu_wait_unreserved_locked(list, bo); - if (unlikely(ret != 0)) { - spin_unlock(&glob->lru_lock); - ttm_eu_list_ref_sub(list); - return ret; - } - goto retry_this_bo; - case -EAGAIN: + /* already slowpath reserved? */ + if (entry->reserved) + continue; + + ret = __ttm_bo_reserve(bo, true, (ticket == NULL), true, + ticket); + + if (ret == -EDEADLK) { + /* uh oh, we lost out, drop every reservation and try + * to only reserve this buffer, then start over if + * this succeeds. + */ + BUG_ON(ticket == NULL); + spin_lock(&glob->lru_lock); ttm_eu_backoff_reservation_locked(list); spin_unlock(&glob->lru_lock); ttm_eu_list_ref_sub(list); - ret = ttm_bo_wait_unreserved(bo, true); - if (unlikely(ret != 0)) - return ret; + ret = ww_mutex_lock_slow_interruptible(&bo->resv->lock, + ticket); + if (unlikely(ret != 0)) { + if (ret == -EINTR) + ret = -ERESTARTSYS; + goto err_fini; + } + + entry->reserved = true; + if (unlikely(atomic_read(&bo->cpu_writers) > 0)) { + ret = -EBUSY; + goto err; + } goto retry; - default: - ttm_eu_backoff_reservation_locked(list); - spin_unlock(&glob->lru_lock); - ttm_eu_list_ref_sub(list); - return ret; - } + } else if (ret) + goto err; entry->reserved = true; if (unlikely(atomic_read(&bo->cpu_writers) > 0)) { - ttm_eu_backoff_reservation_locked(list); - spin_unlock(&glob->lru_lock); - ttm_eu_list_ref_sub(list); - ret = ttm_bo_wait_cpu(bo, false); - if (ret) - return ret; - goto retry; + ret = -EBUSY; + goto err; } } + if (ticket) + ww_acquire_done(ticket); + spin_lock(&glob->lru_lock); ttm_eu_del_from_lru_locked(list); spin_unlock(&glob->lru_lock); ttm_eu_list_ref_sub(list); - return 0; + +err: + spin_lock(&glob->lru_lock); + ttm_eu_backoff_reservation_locked(list); + spin_unlock(&glob->lru_lock); + ttm_eu_list_ref_sub(list); +err_fini: + if (ticket) { + ww_acquire_done(ticket); + ww_acquire_fini(ticket); + } + return ret; } EXPORT_SYMBOL(ttm_eu_reserve_buffers); -void ttm_eu_fence_buffer_objects(struct list_head *list, void *sync_obj) +void ttm_eu_fence_buffer_objects(struct ww_acquire_ctx *ticket, + struct list_head *list, void *sync_obj) { struct ttm_validate_buffer *entry; struct ttm_buffer_object *bo; @@ -216,19 +216,21 @@ void ttm_eu_fence_buffer_objects(struct list_head *list, void *sync_obj) driver = bdev->driver; glob = bo->glob; - spin_lock(&bdev->fence_lock); spin_lock(&glob->lru_lock); + spin_lock(&bdev->fence_lock); list_for_each_entry(entry, list, head) { bo = entry->bo; entry->old_sync_obj = bo->sync_obj; bo->sync_obj = driver->sync_obj_ref(sync_obj); - bo->sync_obj_arg = entry->new_sync_obj_arg; - ttm_bo_unreserve_locked(bo); + ttm_bo_add_to_lru(bo); + __ttm_bo_unreserve(bo); entry->reserved = false; } - spin_unlock(&glob->lru_lock); spin_unlock(&bdev->fence_lock); + spin_unlock(&glob->lru_lock); + if (ticket) + ww_acquire_fini(ticket); list_for_each_entry(entry, list, head) { if (entry->old_sync_obj) diff --git a/drivers/gpu/drm/ttm/ttm_lock.c b/drivers/gpu/drm/ttm/ttm_lock.c index de41e55a944..6a954544727 100644 --- a/drivers/gpu/drm/ttm/ttm_lock.c +++ b/drivers/gpu/drm/ttm/ttm_lock.c @@ -28,9 +28,9 @@ * Authors: Thomas Hellstrom <thellstrom-at-vmware-dot-com> */ -#include "ttm/ttm_lock.h" -#include "ttm/ttm_module.h" -#include <asm/atomic.h> +#include <drm/ttm/ttm_lock.h> +#include <drm/ttm/ttm_module.h> +#include <linux/atomic.h> #include <linux/errno.h> #include <linux/wait.h> #include <linux/sched.h> @@ -186,14 +186,6 @@ int ttm_write_lock(struct ttm_lock *lock, bool interruptible) } EXPORT_SYMBOL(ttm_write_lock); -void ttm_write_lock_downgrade(struct ttm_lock *lock) -{ - spin_lock(&lock->lock); - lock->rw = 1; - wake_up_all(&lock->queue); - spin_unlock(&lock->lock); -} - static int __ttm_vt_unlock(struct ttm_lock *lock) { int ret = 0; diff --git a/drivers/gpu/drm/ttm/ttm_memory.c b/drivers/gpu/drm/ttm/ttm_memory.c index e70ddd82dc0..dbc2def887c 100644 --- a/drivers/gpu/drm/ttm/ttm_memory.c +++ b/drivers/gpu/drm/ttm/ttm_memory.c @@ -25,9 +25,11 @@ * **************************************************************************/ -#include "ttm/ttm_memory.h" -#include "ttm/ttm_module.h" -#include "ttm/ttm_page_alloc.h" +#define pr_fmt(fmt) "[TTM] " fmt + +#include <drm/ttm/ttm_memory.h> +#include <drm/ttm/ttm_module.h> +#include <drm/ttm/ttm_page_alloc.h> #include <linux/spinlock.h> #include <linux/sched.h> #include <linux/wait.h> @@ -74,9 +76,8 @@ static void ttm_mem_zone_kobj_release(struct kobject *kobj) struct ttm_mem_zone *zone = container_of(kobj, struct ttm_mem_zone, kobj); - printk(KERN_INFO TTM_PFX - "Zone %7s: Used memory at exit: %llu kiB.\n", - zone->name, (unsigned long long) zone->used_mem >> 10); + pr_info("Zone %7s: Used memory at exit: %llu kiB\n", + zone->name, (unsigned long long)zone->used_mem >> 10); kfree(zone); } @@ -366,7 +367,6 @@ int ttm_mem_global_init(struct ttm_mem_global *glob) spin_lock_init(&glob->lock); glob->swap_queue = create_singlethread_workqueue("ttm_swap"); INIT_WORK(&glob->work, ttm_shrink_work); - init_waitqueue_head(&glob->queue); ret = kobject_init_and_add( &glob->kobj, &ttm_mem_glob_kobj_type, ttm_get_kobj(), "memory_accounting"); if (unlikely(ret != 0)) { @@ -390,11 +390,11 @@ int ttm_mem_global_init(struct ttm_mem_global *glob) #endif for (i = 0; i < glob->num_zones; ++i) { zone = glob->zones[i]; - printk(KERN_INFO TTM_PFX - "Zone %7s: Available graphics memory: %llu kiB.\n", - zone->name, (unsigned long long) zone->max_mem >> 10); + pr_info("Zone %7s: Available graphics memory: %llu kiB\n", + zone->name, (unsigned long long)zone->max_mem >> 10); } ttm_page_alloc_init(glob, glob->zone_kernel->max_mem/(2*PAGE_SIZE)); + ttm_dma_page_alloc_init(glob, glob->zone_kernel->max_mem/(2*PAGE_SIZE)); return 0; out_no_zone: ttm_mem_global_release(glob); @@ -409,6 +409,7 @@ void ttm_mem_global_release(struct ttm_mem_global *glob) /* let the page allocator first stop the shrink work. */ ttm_page_alloc_fini(); + ttm_dma_page_alloc_fini(); flush_workqueue(glob->swap_queue); destroy_workqueue(glob->swap_queue); diff --git a/drivers/gpu/drm/ttm/ttm_module.c b/drivers/gpu/drm/ttm/ttm_module.c index 902d7cf9fb4..d7f92fe9d90 100644 --- a/drivers/gpu/drm/ttm/ttm_module.c +++ b/drivers/gpu/drm/ttm/ttm_module.c @@ -31,8 +31,8 @@ #include <linux/module.h> #include <linux/device.h> #include <linux/sched.h> -#include "ttm/ttm_module.h" -#include "drm_sysfs.h" +#include <drm/ttm/ttm_module.h> +#include <drm/drm_sysfs.h> static DECLARE_WAIT_QUEUE_HEAD(exit_q); atomic_t device_released; diff --git a/drivers/gpu/drm/ttm/ttm_object.c b/drivers/gpu/drm/ttm/ttm_object.c index 75e9d6f86ba..d2a05335278 100644 --- a/drivers/gpu/drm/ttm/ttm_object.c +++ b/drivers/gpu/drm/ttm/ttm_object.c @@ -1,6 +1,6 @@ /************************************************************************** * - * Copyright (c) 2009 VMware, Inc., Palo Alto, CA., USA + * Copyright (c) 2009-2013 VMware, Inc., Palo Alto, CA., USA * All Rights Reserved. * * Permission is hereby granted, free of charge, to any person obtaining a @@ -26,6 +26,12 @@ **************************************************************************/ /* * Authors: Thomas Hellstrom <thellstrom-at-vmware-dot-com> + * + * While no substantial code is shared, the prime code is inspired by + * drm_prime.c, with + * Authors: + * Dave Airlie <airlied@redhat.com> + * Rob Clark <rob.clark@linaro.org> */ /** @file ttm_ref_object.c * @@ -34,6 +40,7 @@ * and release on file close. */ + /** * struct ttm_object_file * @@ -49,17 +56,19 @@ * for fast lookup of ref objects given a base object. */ -#include "ttm/ttm_object.h" -#include "ttm/ttm_module.h" +#define pr_fmt(fmt) "[TTM] " fmt + +#include <drm/ttm/ttm_object.h> +#include <drm/ttm/ttm_module.h> #include <linux/list.h> #include <linux/spinlock.h> #include <linux/slab.h> #include <linux/module.h> -#include <asm/atomic.h> +#include <linux/atomic.h> struct ttm_object_file { struct ttm_object_device *tdev; - rwlock_t lock; + spinlock_t lock; struct list_head ref_list; struct drm_open_hash ref_hash[TTM_REF_NUM]; struct kref refcount; @@ -78,10 +87,13 @@ struct ttm_object_file { */ struct ttm_object_device { - rwlock_t object_lock; + spinlock_t object_lock; struct drm_open_hash object_hash; atomic_t object_count; struct ttm_mem_global *mem_glob; + struct dma_buf_ops ops; + void (*dmabuf_release)(struct dma_buf *dma_buf); + size_t dma_buf_size; }; /** @@ -106,6 +118,7 @@ struct ttm_object_device { */ struct ttm_ref_object { + struct rcu_head rcu_head; struct drm_hash_item hash; struct list_head head; struct kref kref; @@ -114,6 +127,8 @@ struct ttm_ref_object { struct ttm_object_file *tfile; }; +static void ttm_prime_dmabuf_release(struct dma_buf *dma_buf); + static inline struct ttm_object_file * ttm_object_file_ref(struct ttm_object_file *tfile) { @@ -155,12 +170,12 @@ int ttm_base_object_init(struct ttm_object_file *tfile, base->refcount_release = refcount_release; base->ref_obj_release = ref_obj_release; base->object_type = object_type; - write_lock(&tdev->object_lock); kref_init(&base->refcount); - ret = drm_ht_just_insert_please(&tdev->object_hash, - &base->hash, - (unsigned long)base, 31, 0, 0); - write_unlock(&tdev->object_lock); + spin_lock(&tdev->object_lock); + ret = drm_ht_just_insert_please_rcu(&tdev->object_hash, + &base->hash, + (unsigned long)base, 31, 0, 0); + spin_unlock(&tdev->object_lock); if (unlikely(ret != 0)) goto out_err0; @@ -172,7 +187,9 @@ int ttm_base_object_init(struct ttm_object_file *tfile, return 0; out_err1: - (void)drm_ht_remove_item(&tdev->object_hash, &base->hash); + spin_lock(&tdev->object_lock); + (void)drm_ht_remove_item_rcu(&tdev->object_hash, &base->hash); + spin_unlock(&tdev->object_lock); out_err0: return ret; } @@ -184,63 +201,120 @@ static void ttm_release_base(struct kref *kref) container_of(kref, struct ttm_base_object, refcount); struct ttm_object_device *tdev = base->tfile->tdev; - (void)drm_ht_remove_item(&tdev->object_hash, &base->hash); - write_unlock(&tdev->object_lock); - if (base->refcount_release) { - ttm_object_file_unref(&base->tfile); + spin_lock(&tdev->object_lock); + (void)drm_ht_remove_item_rcu(&tdev->object_hash, &base->hash); + spin_unlock(&tdev->object_lock); + + /* + * Note: We don't use synchronize_rcu() here because it's far + * too slow. It's up to the user to free the object using + * call_rcu() or ttm_base_object_kfree(). + */ + + ttm_object_file_unref(&base->tfile); + if (base->refcount_release) base->refcount_release(&base); - } - write_lock(&tdev->object_lock); } void ttm_base_object_unref(struct ttm_base_object **p_base) { struct ttm_base_object *base = *p_base; - struct ttm_object_device *tdev = base->tfile->tdev; *p_base = NULL; - /* - * Need to take the lock here to avoid racing with - * users trying to look up the object. - */ - - write_lock(&tdev->object_lock); - (void)kref_put(&base->refcount, &ttm_release_base); - write_unlock(&tdev->object_lock); + kref_put(&base->refcount, ttm_release_base); } EXPORT_SYMBOL(ttm_base_object_unref); struct ttm_base_object *ttm_base_object_lookup(struct ttm_object_file *tfile, uint32_t key) { - struct ttm_object_device *tdev = tfile->tdev; - struct ttm_base_object *base; + struct ttm_base_object *base = NULL; struct drm_hash_item *hash; + struct drm_open_hash *ht = &tfile->ref_hash[TTM_REF_USAGE]; int ret; - read_lock(&tdev->object_lock); - ret = drm_ht_find_item(&tdev->object_hash, key, &hash); + rcu_read_lock(); + ret = drm_ht_find_item_rcu(ht, key, &hash); if (likely(ret == 0)) { - base = drm_hash_entry(hash, struct ttm_base_object, hash); - kref_get(&base->refcount); + base = drm_hash_entry(hash, struct ttm_ref_object, hash)->obj; + if (!kref_get_unless_zero(&base->refcount)) + base = NULL; } - read_unlock(&tdev->object_lock); + rcu_read_unlock(); - if (unlikely(ret != 0)) - return NULL; + return base; +} +EXPORT_SYMBOL(ttm_base_object_lookup); - if (tfile != base->tfile && !base->shareable) { - printk(KERN_ERR TTM_PFX - "Attempted access of non-shareable object.\n"); - ttm_base_object_unref(&base); - return NULL; +struct ttm_base_object * +ttm_base_object_lookup_for_ref(struct ttm_object_device *tdev, uint32_t key) +{ + struct ttm_base_object *base = NULL; + struct drm_hash_item *hash; + struct drm_open_hash *ht = &tdev->object_hash; + int ret; + + rcu_read_lock(); + ret = drm_ht_find_item_rcu(ht, key, &hash); + + if (likely(ret == 0)) { + base = drm_hash_entry(hash, struct ttm_base_object, hash); + if (!kref_get_unless_zero(&base->refcount)) + base = NULL; } + rcu_read_unlock(); return base; } -EXPORT_SYMBOL(ttm_base_object_lookup); +EXPORT_SYMBOL(ttm_base_object_lookup_for_ref); + +/** + * ttm_ref_object_exists - Check whether a caller has a valid ref object + * (has opened) a base object. + * + * @tfile: Pointer to a struct ttm_object_file identifying the caller. + * @base: Pointer to a struct base object. + * + * Checks wether the caller identified by @tfile has put a valid USAGE + * reference object on the base object identified by @base. + */ +bool ttm_ref_object_exists(struct ttm_object_file *tfile, + struct ttm_base_object *base) +{ + struct drm_open_hash *ht = &tfile->ref_hash[TTM_REF_USAGE]; + struct drm_hash_item *hash; + struct ttm_ref_object *ref; + + rcu_read_lock(); + if (unlikely(drm_ht_find_item_rcu(ht, base->hash.key, &hash) != 0)) + goto out_false; + + /* + * Verify that the ref object is really pointing to our base object. + * Our base object could actually be dead, and the ref object pointing + * to another base object with the same handle. + */ + ref = drm_hash_entry(hash, struct ttm_ref_object, hash); + if (unlikely(base != ref->obj)) + goto out_false; + + /* + * Verify that the ref->obj pointer was actually valid! + */ + rmb(); + if (unlikely(atomic_read(&ref->kref.refcount) == 0)) + goto out_false; + + rcu_read_unlock(); + return true; + + out_false: + rcu_read_unlock(); + return false; +} +EXPORT_SYMBOL(ttm_ref_object_exists); int ttm_ref_object_add(struct ttm_object_file *tfile, struct ttm_base_object *base, @@ -252,21 +326,25 @@ int ttm_ref_object_add(struct ttm_object_file *tfile, struct ttm_mem_global *mem_glob = tfile->tdev->mem_glob; int ret = -EINVAL; + if (base->tfile != tfile && !base->shareable) + return -EPERM; + if (existed != NULL) *existed = true; while (ret == -EINVAL) { - read_lock(&tfile->lock); - ret = drm_ht_find_item(ht, base->hash.key, &hash); + rcu_read_lock(); + ret = drm_ht_find_item_rcu(ht, base->hash.key, &hash); if (ret == 0) { ref = drm_hash_entry(hash, struct ttm_ref_object, hash); - kref_get(&ref->kref); - read_unlock(&tfile->lock); - break; + if (kref_get_unless_zero(&ref->kref)) { + rcu_read_unlock(); + break; + } } - read_unlock(&tfile->lock); + rcu_read_unlock(); ret = ttm_mem_global_alloc(mem_glob, sizeof(*ref), false, false); if (unlikely(ret != 0)) @@ -283,19 +361,19 @@ int ttm_ref_object_add(struct ttm_object_file *tfile, ref->ref_type = ref_type; kref_init(&ref->kref); - write_lock(&tfile->lock); - ret = drm_ht_insert_item(ht, &ref->hash); + spin_lock(&tfile->lock); + ret = drm_ht_insert_item_rcu(ht, &ref->hash); if (likely(ret == 0)) { list_add_tail(&ref->head, &tfile->ref_list); kref_get(&base->refcount); - write_unlock(&tfile->lock); + spin_unlock(&tfile->lock); if (existed != NULL) *existed = false; break; } - write_unlock(&tfile->lock); + spin_unlock(&tfile->lock); BUG_ON(ret != -EINVAL); ttm_mem_global_free(mem_glob, sizeof(*ref)); @@ -316,17 +394,17 @@ static void ttm_ref_object_release(struct kref *kref) struct ttm_mem_global *mem_glob = tfile->tdev->mem_glob; ht = &tfile->ref_hash[ref->ref_type]; - (void)drm_ht_remove_item(ht, &ref->hash); + (void)drm_ht_remove_item_rcu(ht, &ref->hash); list_del(&ref->head); - write_unlock(&tfile->lock); + spin_unlock(&tfile->lock); if (ref->ref_type != TTM_REF_USAGE && base->ref_obj_release) base->ref_obj_release(base, ref->ref_type); ttm_base_object_unref(&ref->obj); ttm_mem_global_free(mem_glob, sizeof(*ref)); - kfree(ref); - write_lock(&tfile->lock); + kfree_rcu(ref, rcu_head); + spin_lock(&tfile->lock); } int ttm_ref_object_base_unref(struct ttm_object_file *tfile, @@ -337,15 +415,15 @@ int ttm_ref_object_base_unref(struct ttm_object_file *tfile, struct drm_hash_item *hash; int ret; - write_lock(&tfile->lock); + spin_lock(&tfile->lock); ret = drm_ht_find_item(ht, key, &hash); if (unlikely(ret != 0)) { - write_unlock(&tfile->lock); + spin_unlock(&tfile->lock); return -EINVAL; } ref = drm_hash_entry(hash, struct ttm_ref_object, hash); kref_put(&ref->kref, ttm_ref_object_release); - write_unlock(&tfile->lock); + spin_unlock(&tfile->lock); return 0; } EXPORT_SYMBOL(ttm_ref_object_base_unref); @@ -358,7 +436,7 @@ void ttm_object_file_release(struct ttm_object_file **p_tfile) struct ttm_object_file *tfile = *p_tfile; *p_tfile = NULL; - write_lock(&tfile->lock); + spin_lock(&tfile->lock); /* * Since we release the lock within the loop, we have to @@ -374,7 +452,7 @@ void ttm_object_file_release(struct ttm_object_file **p_tfile) for (i = 0; i < TTM_REF_NUM; ++i) drm_ht_remove(&tfile->ref_hash[i]); - write_unlock(&tfile->lock); + spin_unlock(&tfile->lock); ttm_object_file_unref(&tfile); } EXPORT_SYMBOL(ttm_object_file_release); @@ -390,7 +468,7 @@ struct ttm_object_file *ttm_object_file_init(struct ttm_object_device *tdev, if (unlikely(tfile == NULL)) return NULL; - rwlock_init(&tfile->lock); + spin_lock_init(&tfile->lock); tfile->tdev = tdev; kref_init(&tfile->refcount); INIT_LIST_HEAD(&tfile->ref_list); @@ -414,9 +492,10 @@ out_err: } EXPORT_SYMBOL(ttm_object_file_init); -struct ttm_object_device *ttm_object_device_init(struct ttm_mem_global - *mem_glob, - unsigned int hash_order) +struct ttm_object_device * +ttm_object_device_init(struct ttm_mem_global *mem_glob, + unsigned int hash_order, + const struct dma_buf_ops *ops) { struct ttm_object_device *tdev = kmalloc(sizeof(*tdev), GFP_KERNEL); int ret; @@ -425,13 +504,20 @@ struct ttm_object_device *ttm_object_device_init(struct ttm_mem_global return NULL; tdev->mem_glob = mem_glob; - rwlock_init(&tdev->object_lock); + spin_lock_init(&tdev->object_lock); atomic_set(&tdev->object_count, 0); ret = drm_ht_create(&tdev->object_hash, hash_order); + if (ret != 0) + goto out_no_object_hash; - if (likely(ret == 0)) - return tdev; + tdev->ops = *ops; + tdev->dmabuf_release = tdev->ops.release; + tdev->ops.release = ttm_prime_dmabuf_release; + tdev->dma_buf_size = ttm_round_pot(sizeof(struct dma_buf)) + + ttm_round_pot(sizeof(struct file)); + return tdev; +out_no_object_hash: kfree(tdev); return NULL; } @@ -443,10 +529,232 @@ void ttm_object_device_release(struct ttm_object_device **p_tdev) *p_tdev = NULL; - write_lock(&tdev->object_lock); + spin_lock(&tdev->object_lock); drm_ht_remove(&tdev->object_hash); - write_unlock(&tdev->object_lock); + spin_unlock(&tdev->object_lock); kfree(tdev); } EXPORT_SYMBOL(ttm_object_device_release); + +/** + * get_dma_buf_unless_doomed - get a dma_buf reference if possible. + * + * @dma_buf: Non-refcounted pointer to a struct dma-buf. + * + * Obtain a file reference from a lookup structure that doesn't refcount + * the file, but synchronizes with its release method to make sure it has + * not been freed yet. See for example kref_get_unless_zero documentation. + * Returns true if refcounting succeeds, false otherwise. + * + * Nobody really wants this as a public API yet, so let it mature here + * for some time... + */ +static bool __must_check get_dma_buf_unless_doomed(struct dma_buf *dmabuf) +{ + return atomic_long_inc_not_zero(&dmabuf->file->f_count) != 0L; +} + +/** + * ttm_prime_refcount_release - refcount release method for a prime object. + * + * @p_base: Pointer to ttm_base_object pointer. + * + * This is a wrapper that calls the refcount_release founction of the + * underlying object. At the same time it cleans up the prime object. + * This function is called when all references to the base object we + * derive from are gone. + */ +static void ttm_prime_refcount_release(struct ttm_base_object **p_base) +{ + struct ttm_base_object *base = *p_base; + struct ttm_prime_object *prime; + + *p_base = NULL; + prime = container_of(base, struct ttm_prime_object, base); + BUG_ON(prime->dma_buf != NULL); + mutex_destroy(&prime->mutex); + if (prime->refcount_release) + prime->refcount_release(&base); +} + +/** + * ttm_prime_dmabuf_release - Release method for the dma-bufs we export + * + * @dma_buf: + * + * This function first calls the dma_buf release method the driver + * provides. Then it cleans up our dma_buf pointer used for lookup, + * and finally releases the reference the dma_buf has on our base + * object. + */ +static void ttm_prime_dmabuf_release(struct dma_buf *dma_buf) +{ + struct ttm_prime_object *prime = + (struct ttm_prime_object *) dma_buf->priv; + struct ttm_base_object *base = &prime->base; + struct ttm_object_device *tdev = base->tfile->tdev; + + if (tdev->dmabuf_release) + tdev->dmabuf_release(dma_buf); + mutex_lock(&prime->mutex); + if (prime->dma_buf == dma_buf) + prime->dma_buf = NULL; + mutex_unlock(&prime->mutex); + ttm_mem_global_free(tdev->mem_glob, tdev->dma_buf_size); + ttm_base_object_unref(&base); +} + +/** + * ttm_prime_fd_to_handle - Get a base object handle from a prime fd + * + * @tfile: A struct ttm_object_file identifying the caller. + * @fd: The prime / dmabuf fd. + * @handle: The returned handle. + * + * This function returns a handle to an object that previously exported + * a dma-buf. Note that we don't handle imports yet, because we simply + * have no consumers of that implementation. + */ +int ttm_prime_fd_to_handle(struct ttm_object_file *tfile, + int fd, u32 *handle) +{ + struct ttm_object_device *tdev = tfile->tdev; + struct dma_buf *dma_buf; + struct ttm_prime_object *prime; + struct ttm_base_object *base; + int ret; + + dma_buf = dma_buf_get(fd); + if (IS_ERR(dma_buf)) + return PTR_ERR(dma_buf); + + if (dma_buf->ops != &tdev->ops) + return -ENOSYS; + + prime = (struct ttm_prime_object *) dma_buf->priv; + base = &prime->base; + *handle = base->hash.key; + ret = ttm_ref_object_add(tfile, base, TTM_REF_USAGE, NULL); + + dma_buf_put(dma_buf); + + return ret; +} +EXPORT_SYMBOL_GPL(ttm_prime_fd_to_handle); + +/** + * ttm_prime_handle_to_fd - Return a dma_buf fd from a ttm prime object + * + * @tfile: Struct ttm_object_file identifying the caller. + * @handle: Handle to the object we're exporting from. + * @flags: flags for dma-buf creation. We just pass them on. + * @prime_fd: The returned file descriptor. + * + */ +int ttm_prime_handle_to_fd(struct ttm_object_file *tfile, + uint32_t handle, uint32_t flags, + int *prime_fd) +{ + struct ttm_object_device *tdev = tfile->tdev; + struct ttm_base_object *base; + struct dma_buf *dma_buf; + struct ttm_prime_object *prime; + int ret; + + base = ttm_base_object_lookup(tfile, handle); + if (unlikely(base == NULL || + base->object_type != ttm_prime_type)) { + ret = -ENOENT; + goto out_unref; + } + + prime = container_of(base, struct ttm_prime_object, base); + if (unlikely(!base->shareable)) { + ret = -EPERM; + goto out_unref; + } + + ret = mutex_lock_interruptible(&prime->mutex); + if (unlikely(ret != 0)) { + ret = -ERESTARTSYS; + goto out_unref; + } + + dma_buf = prime->dma_buf; + if (!dma_buf || !get_dma_buf_unless_doomed(dma_buf)) { + + /* + * Need to create a new dma_buf, with memory accounting. + */ + ret = ttm_mem_global_alloc(tdev->mem_glob, tdev->dma_buf_size, + false, true); + if (unlikely(ret != 0)) { + mutex_unlock(&prime->mutex); + goto out_unref; + } + + dma_buf = dma_buf_export(prime, &tdev->ops, + prime->size, flags); + if (IS_ERR(dma_buf)) { + ret = PTR_ERR(dma_buf); + ttm_mem_global_free(tdev->mem_glob, + tdev->dma_buf_size); + mutex_unlock(&prime->mutex); + goto out_unref; + } + + /* + * dma_buf has taken the base object reference + */ + base = NULL; + prime->dma_buf = dma_buf; + } + mutex_unlock(&prime->mutex); + + ret = dma_buf_fd(dma_buf, flags); + if (ret >= 0) { + *prime_fd = ret; + ret = 0; + } else + dma_buf_put(dma_buf); + +out_unref: + if (base) + ttm_base_object_unref(&base); + return ret; +} +EXPORT_SYMBOL_GPL(ttm_prime_handle_to_fd); + +/** + * ttm_prime_object_init - Initialize a ttm_prime_object + * + * @tfile: struct ttm_object_file identifying the caller + * @size: The size of the dma_bufs we export. + * @prime: The object to be initialized. + * @shareable: See ttm_base_object_init + * @type: See ttm_base_object_init + * @refcount_release: See ttm_base_object_init + * @ref_obj_release: See ttm_base_object_init + * + * Initializes an object which is compatible with the drm_prime model + * for data sharing between processes and devices. + */ +int ttm_prime_object_init(struct ttm_object_file *tfile, size_t size, + struct ttm_prime_object *prime, bool shareable, + enum ttm_object_type type, + void (*refcount_release) (struct ttm_base_object **), + void (*ref_obj_release) (struct ttm_base_object *, + enum ttm_ref_type ref_type)) +{ + mutex_init(&prime->mutex); + prime->size = PAGE_ALIGN(size); + prime->real_type = type; + prime->dma_buf = NULL; + prime->refcount_release = refcount_release; + return ttm_base_object_init(tfile, &prime->base, shareable, + ttm_prime_type, + ttm_prime_refcount_release, + ref_obj_release); +} +EXPORT_SYMBOL(ttm_prime_object_init); diff --git a/drivers/gpu/drm/ttm/ttm_page_alloc.c b/drivers/gpu/drm/ttm/ttm_page_alloc.c index b1e02fffd3c..863bef9f923 100644 --- a/drivers/gpu/drm/ttm/ttm_page_alloc.c +++ b/drivers/gpu/drm/ttm/ttm_page_alloc.c @@ -30,6 +30,9 @@ * - Use page->lru to keep a free list * - doesn't track currently in use pages */ + +#define pr_fmt(fmt) "[TTM] " fmt + #include <linux/list.h> #include <linux/spinlock.h> #include <linux/highmem.h> @@ -38,11 +41,12 @@ #include <linux/mm.h> #include <linux/seq_file.h> /* for seq_printf */ #include <linux/slab.h> +#include <linux/dma-mapping.h> -#include <asm/atomic.h> +#include <linux/atomic.h> -#include "ttm/ttm_bo_driver.h" -#include "ttm/ttm_page_alloc.h" +#include <drm/ttm/ttm_bo_driver.h> +#include <drm/ttm/ttm_page_alloc.h> #ifdef TTM_HAS_AGP #include <asm/agp.h> @@ -166,18 +170,13 @@ static ssize_t ttm_pool_store(struct kobject *kobj, m->options.small = val; else if (attr == &ttm_page_pool_alloc_size) { if (val > NUM_PAGES_TO_ALLOC*8) { - printk(KERN_ERR TTM_PFX - "Setting allocation size to %lu " - "is not allowed. Recommended size is " - "%lu\n", + pr_err("Setting allocation size to %lu is not allowed. Recommended size is %lu\n", NUM_PAGES_TO_ALLOC*(PAGE_SIZE >> 7), NUM_PAGES_TO_ALLOC*(PAGE_SIZE >> 10)); return size; } else if (val > NUM_PAGES_TO_ALLOC) { - printk(KERN_WARNING TTM_PFX - "Setting allocation size to " - "larger than %lu is not recommended.\n", - NUM_PAGES_TO_ALLOC*(PAGE_SIZE >> 10)); + pr_warn("Setting allocation size to larger than %lu is not recommended\n", + NUM_PAGES_TO_ALLOC*(PAGE_SIZE >> 10)); } m->options.alloc_size = val; } @@ -278,8 +277,7 @@ static void ttm_pages_put(struct page *pages[], unsigned npages) { unsigned i; if (set_pages_array_wb(pages, npages)) - printk(KERN_ERR TTM_PFX "Failed to set %d pages to wb!\n", - npages); + pr_err("Failed to set %d pages to wb!\n", npages); for (i = 0; i < npages; ++i) __free_page(pages[i]); } @@ -314,8 +312,7 @@ static int ttm_page_pool_free(struct ttm_page_pool *pool, unsigned nr_free) pages_to_free = kmalloc(npages_to_free * sizeof(struct page *), GFP_KERNEL); if (!pages_to_free) { - printk(KERN_ERR TTM_PFX - "Failed to allocate memory for pool free operation.\n"); + pr_err("Failed to allocate memory for pool free operation\n"); return 0; } @@ -354,7 +351,7 @@ restart: if (nr_free) goto restart; - /* Not allowed to fall tough or break because + /* Not allowed to fall through or break because * following context is inside spinlock while we are * outside here. */ @@ -380,26 +377,26 @@ out: return nr_free; } -/* Get good estimation how many pages are free in pools */ -static int ttm_pool_get_num_unused_pages(void) -{ - unsigned i; - int total = 0; - for (i = 0; i < NUM_POOLS; ++i) - total += _manager->pools[i].npages; - - return total; -} - /** * Callback for mm to request pool to reduce number of page held. + * + * XXX: (dchinner) Deadlock warning! + * + * ttm_page_pool_free() does memory allocation using GFP_KERNEL. that means + * this can deadlock when called a sc->gfp_mask that is not equal to + * GFP_KERNEL. + * + * This code is crying out for a shrinker per pool.... */ -static int ttm_pool_mm_shrink(struct shrinker *shrink, int shrink_pages, gfp_t gfp_mask) +static unsigned long +ttm_pool_shrink_scan(struct shrinker *shrink, struct shrink_control *sc) { static atomic_t start_pool = ATOMIC_INIT(0); unsigned i; unsigned pool_offset = atomic_add_return(1, &start_pool); struct ttm_page_pool *pool; + int shrink_pages = sc->nr_to_scan; + unsigned long freed = 0; pool_offset = pool_offset % NUM_POOLS; /* select start pool in round robin fashion */ @@ -409,14 +406,28 @@ static int ttm_pool_mm_shrink(struct shrinker *shrink, int shrink_pages, gfp_t g break; pool = &_manager->pools[(i + pool_offset)%NUM_POOLS]; shrink_pages = ttm_page_pool_free(pool, nr_free); + freed += nr_free - shrink_pages; } - /* return estimated number of unused pages in pool */ - return ttm_pool_get_num_unused_pages(); + return freed; +} + + +static unsigned long +ttm_pool_shrink_count(struct shrinker *shrink, struct shrink_control *sc) +{ + unsigned i; + unsigned long count = 0; + + for (i = 0; i < NUM_POOLS; ++i) + count += _manager->pools[i].npages; + + return count; } static void ttm_pool_mm_shrink_init(struct ttm_pool_manager *manager) { - manager->mm_shrink.shrink = &ttm_pool_mm_shrink; + manager->mm_shrink.count_objects = ttm_pool_shrink_count; + manager->mm_shrink.scan_objects = ttm_pool_shrink_scan; manager->mm_shrink.seeks = 1; register_shrinker(&manager->mm_shrink); } @@ -435,16 +446,12 @@ static int ttm_set_pages_caching(struct page **pages, case tt_uncached: r = set_pages_array_uc(pages, cpages); if (r) - printk(KERN_ERR TTM_PFX - "Failed to set %d pages to uc!\n", - cpages); + pr_err("Failed to set %d pages to uc!\n", cpages); break; case tt_wc: r = set_pages_array_wc(pages, cpages); if (r) - printk(KERN_ERR TTM_PFX - "Failed to set %d pages to wc!\n", - cpages); + pr_err("Failed to set %d pages to wc!\n", cpages); break; default: break; @@ -489,8 +496,7 @@ static int ttm_alloc_new_pages(struct list_head *pages, gfp_t gfp_flags, caching_array = kmalloc(max_cpages*sizeof(struct page *), GFP_KERNEL); if (!caching_array) { - printk(KERN_ERR TTM_PFX - "Unable to allocate table for new pages."); + pr_err("Unable to allocate table for new pages\n"); return -ENOMEM; } @@ -498,7 +504,7 @@ static int ttm_alloc_new_pages(struct list_head *pages, gfp_t gfp_flags, p = alloc_page(gfp_flags); if (!p) { - printk(KERN_ERR TTM_PFX "Unable to get page %u.\n", i); + pr_err("Unable to get page %u\n", i); /* store already allocated pages in the pool after * setting the caching state */ @@ -553,7 +559,7 @@ out: } /** - * Fill the given pool if there isn't enough pages and requested number of + * Fill the given pool if there aren't enough pages and the requested number of * pages is small. */ static void ttm_page_pool_fill_locked(struct ttm_page_pool *pool, @@ -573,8 +579,8 @@ static void ttm_page_pool_fill_locked(struct ttm_page_pool *pool, pool->fill_lock = true; - /* If allocation request is small and there is not enough - * pages in pool we fill the pool first */ + /* If allocation request is small and there are not enough + * pages in a pool we fill the pool up first. */ if (count < _manager->options.small && count > pool->npages) { struct list_head new_pages; @@ -596,8 +602,7 @@ static void ttm_page_pool_fill_locked(struct ttm_page_pool *pool, ++pool->nrefills; pool->npages += alloc_size; } else { - printk(KERN_ERR TTM_PFX - "Failed to fill pool (%p).", pool); + pr_err("Failed to fill pool (%p)\n", pool); /* If we have any pages left put them to the pool. */ list_for_each_entry(p, &pool->list, lru) { ++cpages; @@ -611,13 +616,15 @@ static void ttm_page_pool_fill_locked(struct ttm_page_pool *pool, } /** - * Cut count nubmer of pages from the pool and put them to return list + * Cut 'count' number of pages from the pool and put them on the return list. * - * @return count of pages still to allocate to fill the request. + * @return count of pages still required to fulfill the request. */ static unsigned ttm_page_pool_get_pages(struct ttm_page_pool *pool, - struct list_head *pages, int ttm_flags, - enum ttm_caching_state cstate, unsigned count) + struct list_head *pages, + int ttm_flags, + enum ttm_caching_state cstate, + unsigned count) { unsigned long irq_flags; struct list_head *p; @@ -634,7 +641,7 @@ static unsigned ttm_page_pool_get_pages(struct ttm_page_pool *pool, goto out; } /* find the last pages to include for requested number of pages. Split - * pool to begin and halves to reduce search space. */ + * pool to begin and halve it to reduce search space. */ if (count <= pool->npages/2) { i = 0; list_for_each(p, &pool->list) { @@ -648,7 +655,7 @@ static unsigned ttm_page_pool_get_pages(struct ttm_page_pool *pool, break; } } - /* Cut count number of pages from pool */ + /* Cut 'count' number of pages from the pool */ list_cut_position(pages, &pool->list, p); pool->npages -= count; count = 0; @@ -657,16 +664,63 @@ out: return count; } +/* Put all pages in pages list to correct pool to wait for reuse */ +static void ttm_put_pages(struct page **pages, unsigned npages, int flags, + enum ttm_caching_state cstate) +{ + unsigned long irq_flags; + struct ttm_page_pool *pool = ttm_get_pool(flags, cstate); + unsigned i; + + if (pool == NULL) { + /* No pool for this memory type so free the pages */ + for (i = 0; i < npages; i++) { + if (pages[i]) { + if (page_count(pages[i]) != 1) + pr_err("Erroneous page count. Leaking pages.\n"); + __free_page(pages[i]); + pages[i] = NULL; + } + } + return; + } + + spin_lock_irqsave(&pool->lock, irq_flags); + for (i = 0; i < npages; i++) { + if (pages[i]) { + if (page_count(pages[i]) != 1) + pr_err("Erroneous page count. Leaking pages.\n"); + list_add_tail(&pages[i]->lru, &pool->list); + pages[i] = NULL; + pool->npages++; + } + } + /* Check that we don't go over the pool limit */ + npages = 0; + if (pool->npages > _manager->options.max_size) { + npages = pool->npages - _manager->options.max_size; + /* free at least NUM_PAGES_TO_ALLOC number of pages + * to reduce calls to set_memory_wb */ + if (npages < NUM_PAGES_TO_ALLOC) + npages = NUM_PAGES_TO_ALLOC; + } + spin_unlock_irqrestore(&pool->lock, irq_flags); + if (npages) + ttm_page_pool_free(pool, npages); +} + /* * On success pages list will hold count number of correctly * cached pages. */ -int ttm_get_pages(struct list_head *pages, int flags, - enum ttm_caching_state cstate, unsigned count) +static int ttm_get_pages(struct page **pages, unsigned npages, int flags, + enum ttm_caching_state cstate) { struct ttm_page_pool *pool = ttm_get_pool(flags, cstate); + struct list_head plist; struct page *p = NULL; gfp_t gfp_flags = GFP_USER; + unsigned count; int r; /* set zero flag for page allocation if required */ @@ -680,96 +734,62 @@ int ttm_get_pages(struct list_head *pages, int flags, else gfp_flags |= GFP_HIGHUSER; - for (r = 0; r < count; ++r) { + for (r = 0; r < npages; ++r) { p = alloc_page(gfp_flags); if (!p) { - printk(KERN_ERR TTM_PFX - "Unable to allocate page."); + pr_err("Unable to allocate page\n"); return -ENOMEM; } - list_add(&p->lru, pages); + pages[r] = p; } return 0; } - /* combine zero flag to pool flags */ gfp_flags |= pool->gfp_flags; /* First we take pages from the pool */ - count = ttm_page_pool_get_pages(pool, pages, flags, cstate, count); + INIT_LIST_HEAD(&plist); + npages = ttm_page_pool_get_pages(pool, &plist, flags, cstate, npages); + count = 0; + list_for_each_entry(p, &plist, lru) { + pages[count++] = p; + } /* clear the pages coming from the pool if requested */ if (flags & TTM_PAGE_FLAG_ZERO_ALLOC) { - list_for_each_entry(p, pages, lru) { - clear_page(page_address(p)); + list_for_each_entry(p, &plist, lru) { + if (PageHighMem(p)) + clear_highpage(p); + else + clear_page(page_address(p)); } } /* If pool didn't have enough pages allocate new one. */ - if (count > 0) { + if (npages > 0) { /* ttm_alloc_new_pages doesn't reference pool so we can run * multiple requests in parallel. **/ - r = ttm_alloc_new_pages(pages, gfp_flags, flags, cstate, count); + INIT_LIST_HEAD(&plist); + r = ttm_alloc_new_pages(&plist, gfp_flags, flags, cstate, npages); + list_for_each_entry(p, &plist, lru) { + pages[count++] = p; + } if (r) { /* If there is any pages in the list put them back to * the pool. */ - printk(KERN_ERR TTM_PFX - "Failed to allocate extra pages " - "for large request."); - ttm_put_pages(pages, 0, flags, cstate); + pr_err("Failed to allocate extra pages for large request\n"); + ttm_put_pages(pages, count, flags, cstate); return r; } } - return 0; } -/* Put all pages in pages list to correct pool to wait for reuse */ -void ttm_put_pages(struct list_head *pages, unsigned page_count, int flags, - enum ttm_caching_state cstate) -{ - unsigned long irq_flags; - struct ttm_page_pool *pool = ttm_get_pool(flags, cstate); - struct page *p, *tmp; - - if (pool == NULL) { - /* No pool for this memory type so free the pages */ - - list_for_each_entry_safe(p, tmp, pages, lru) { - __free_page(p); - } - /* Make the pages list empty */ - INIT_LIST_HEAD(pages); - return; - } - if (page_count == 0) { - list_for_each_entry_safe(p, tmp, pages, lru) { - ++page_count; - } - } - - spin_lock_irqsave(&pool->lock, irq_flags); - list_splice_init(pages, &pool->list); - pool->npages += page_count; - /* Check that we don't go over the pool limit */ - page_count = 0; - if (pool->npages > _manager->options.max_size) { - page_count = pool->npages - _manager->options.max_size; - /* free at least NUM_PAGES_TO_ALLOC number of pages - * to reduce calls to set_memory_wb */ - if (page_count < NUM_PAGES_TO_ALLOC) - page_count = NUM_PAGES_TO_ALLOC; - } - spin_unlock_irqrestore(&pool->lock, irq_flags); - if (page_count) - ttm_page_pool_free(pool, page_count); -} - static void ttm_page_pool_init_locked(struct ttm_page_pool *pool, int flags, char *name) { @@ -787,7 +807,7 @@ int ttm_page_alloc_init(struct ttm_mem_global *glob, unsigned max_pages) WARN_ON(_manager); - printk(KERN_INFO TTM_PFX "Initializing pool allocator.\n"); + pr_info("Initializing pool allocator\n"); _manager = kzalloc(sizeof(*_manager), GFP_KERNEL); @@ -822,7 +842,7 @@ void ttm_page_alloc_fini(void) { int i; - printk(KERN_INFO TTM_PFX "Finalizing pool allocator.\n"); + pr_info("Finalizing pool allocator\n"); ttm_pool_mm_shrink_fini(_manager); for (i = 0; i < NUM_POOLS; ++i) @@ -832,6 +852,62 @@ void ttm_page_alloc_fini(void) _manager = NULL; } +int ttm_pool_populate(struct ttm_tt *ttm) +{ + struct ttm_mem_global *mem_glob = ttm->glob->mem_glob; + unsigned i; + int ret; + + if (ttm->state != tt_unpopulated) + return 0; + + for (i = 0; i < ttm->num_pages; ++i) { + ret = ttm_get_pages(&ttm->pages[i], 1, + ttm->page_flags, + ttm->caching_state); + if (ret != 0) { + ttm_pool_unpopulate(ttm); + return -ENOMEM; + } + + ret = ttm_mem_global_alloc_page(mem_glob, ttm->pages[i], + false, false); + if (unlikely(ret != 0)) { + ttm_pool_unpopulate(ttm); + return -ENOMEM; + } + } + + if (unlikely(ttm->page_flags & TTM_PAGE_FLAG_SWAPPED)) { + ret = ttm_tt_swapin(ttm); + if (unlikely(ret != 0)) { + ttm_pool_unpopulate(ttm); + return ret; + } + } + + ttm->state = tt_unbound; + return 0; +} +EXPORT_SYMBOL(ttm_pool_populate); + +void ttm_pool_unpopulate(struct ttm_tt *ttm) +{ + unsigned i; + + for (i = 0; i < ttm->num_pages; ++i) { + if (ttm->pages[i]) { + ttm_mem_global_free_page(ttm->glob->mem_glob, + ttm->pages[i]); + ttm_put_pages(&ttm->pages[i], 1, + ttm->page_flags, + ttm->caching_state); + } + } + ttm->state = tt_unpopulated; +} +EXPORT_SYMBOL(ttm_pool_unpopulate); + int ttm_page_alloc_debugfs(struct seq_file *m, void *data) { struct ttm_page_pool *p; diff --git a/drivers/gpu/drm/ttm/ttm_page_alloc_dma.c b/drivers/gpu/drm/ttm/ttm_page_alloc_dma.c new file mode 100644 index 00000000000..fb8259f6983 --- /dev/null +++ b/drivers/gpu/drm/ttm/ttm_page_alloc_dma.c @@ -0,0 +1,1147 @@ +/* + * Copyright 2011 (c) Oracle Corp. + + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sub license, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice (including the + * next paragraph) shall be included in all copies or substantial portions + * of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + * + * Author: Konrad Rzeszutek Wilk <konrad.wilk@oracle.com> + */ + +/* + * A simple DMA pool losely based on dmapool.c. It has certain advantages + * over the DMA pools: + * - Pool collects resently freed pages for reuse (and hooks up to + * the shrinker). + * - Tracks currently in use pages + * - Tracks whether the page is UC, WB or cached (and reverts to WB + * when freed). + */ + +#if defined(CONFIG_SWIOTLB) || defined(CONFIG_INTEL_IOMMU) +#define pr_fmt(fmt) "[TTM] " fmt + +#include <linux/dma-mapping.h> +#include <linux/list.h> +#include <linux/seq_file.h> /* for seq_printf */ +#include <linux/slab.h> +#include <linux/spinlock.h> +#include <linux/highmem.h> +#include <linux/mm_types.h> +#include <linux/module.h> +#include <linux/mm.h> +#include <linux/atomic.h> +#include <linux/device.h> +#include <linux/kthread.h> +#include <drm/ttm/ttm_bo_driver.h> +#include <drm/ttm/ttm_page_alloc.h> +#ifdef TTM_HAS_AGP +#include <asm/agp.h> +#endif + +#define NUM_PAGES_TO_ALLOC (PAGE_SIZE/sizeof(struct page *)) +#define SMALL_ALLOCATION 4 +#define FREE_ALL_PAGES (~0U) +/* times are in msecs */ +#define IS_UNDEFINED (0) +#define IS_WC (1<<1) +#define IS_UC (1<<2) +#define IS_CACHED (1<<3) +#define IS_DMA32 (1<<4) + +enum pool_type { + POOL_IS_UNDEFINED, + POOL_IS_WC = IS_WC, + POOL_IS_UC = IS_UC, + POOL_IS_CACHED = IS_CACHED, + POOL_IS_WC_DMA32 = IS_WC | IS_DMA32, + POOL_IS_UC_DMA32 = IS_UC | IS_DMA32, + POOL_IS_CACHED_DMA32 = IS_CACHED | IS_DMA32, +}; +/* + * The pool structure. There are usually six pools: + * - generic (not restricted to DMA32): + * - write combined, uncached, cached. + * - dma32 (up to 2^32 - so up 4GB): + * - write combined, uncached, cached. + * for each 'struct device'. The 'cached' is for pages that are actively used. + * The other ones can be shrunk by the shrinker API if neccessary. + * @pools: The 'struct device->dma_pools' link. + * @type: Type of the pool + * @lock: Protects the inuse_list and free_list from concurrnet access. Must be + * used with irqsave/irqrestore variants because pool allocator maybe called + * from delayed work. + * @inuse_list: Pool of pages that are in use. The order is very important and + * it is in the order that the TTM pages that are put back are in. + * @free_list: Pool of pages that are free to be used. No order requirements. + * @dev: The device that is associated with these pools. + * @size: Size used during DMA allocation. + * @npages_free: Count of available pages for re-use. + * @npages_in_use: Count of pages that are in use. + * @nfrees: Stats when pool is shrinking. + * @nrefills: Stats when the pool is grown. + * @gfp_flags: Flags to pass for alloc_page. + * @name: Name of the pool. + * @dev_name: Name derieved from dev - similar to how dev_info works. + * Used during shutdown as the dev_info during release is unavailable. + */ +struct dma_pool { + struct list_head pools; /* The 'struct device->dma_pools link */ + enum pool_type type; + spinlock_t lock; + struct list_head inuse_list; + struct list_head free_list; + struct device *dev; + unsigned size; + unsigned npages_free; + unsigned npages_in_use; + unsigned long nfrees; /* Stats when shrunk. */ + unsigned long nrefills; /* Stats when grown. */ + gfp_t gfp_flags; + char name[13]; /* "cached dma32" */ + char dev_name[64]; /* Constructed from dev */ +}; + +/* + * The accounting page keeping track of the allocated page along with + * the DMA address. + * @page_list: The link to the 'page_list' in 'struct dma_pool'. + * @vaddr: The virtual address of the page + * @dma: The bus address of the page. If the page is not allocated + * via the DMA API, it will be -1. + */ +struct dma_page { + struct list_head page_list; + void *vaddr; + struct page *p; + dma_addr_t dma; +}; + +/* + * Limits for the pool. They are handled without locks because only place where + * they may change is in sysfs store. They won't have immediate effect anyway + * so forcing serialization to access them is pointless. + */ + +struct ttm_pool_opts { + unsigned alloc_size; + unsigned max_size; + unsigned small; +}; + +/* + * Contains the list of all of the 'struct device' and their corresponding + * DMA pools. Guarded by _mutex->lock. + * @pools: The link to 'struct ttm_pool_manager->pools' + * @dev: The 'struct device' associated with the 'pool' + * @pool: The 'struct dma_pool' associated with the 'dev' + */ +struct device_pools { + struct list_head pools; + struct device *dev; + struct dma_pool *pool; +}; + +/* + * struct ttm_pool_manager - Holds memory pools for fast allocation + * + * @lock: Lock used when adding/removing from pools + * @pools: List of 'struct device' and 'struct dma_pool' tuples. + * @options: Limits for the pool. + * @npools: Total amount of pools in existence. + * @shrinker: The structure used by [un|]register_shrinker + */ +struct ttm_pool_manager { + struct mutex lock; + struct list_head pools; + struct ttm_pool_opts options; + unsigned npools; + struct shrinker mm_shrink; + struct kobject kobj; +}; + +static struct ttm_pool_manager *_manager; + +static struct attribute ttm_page_pool_max = { + .name = "pool_max_size", + .mode = S_IRUGO | S_IWUSR +}; +static struct attribute ttm_page_pool_small = { + .name = "pool_small_allocation", + .mode = S_IRUGO | S_IWUSR +}; +static struct attribute ttm_page_pool_alloc_size = { + .name = "pool_allocation_size", + .mode = S_IRUGO | S_IWUSR +}; + +static struct attribute *ttm_pool_attrs[] = { + &ttm_page_pool_max, + &ttm_page_pool_small, + &ttm_page_pool_alloc_size, + NULL +}; + +static void ttm_pool_kobj_release(struct kobject *kobj) +{ + struct ttm_pool_manager *m = + container_of(kobj, struct ttm_pool_manager, kobj); + kfree(m); +} + +static ssize_t ttm_pool_store(struct kobject *kobj, struct attribute *attr, + const char *buffer, size_t size) +{ + struct ttm_pool_manager *m = + container_of(kobj, struct ttm_pool_manager, kobj); + int chars; + unsigned val; + chars = sscanf(buffer, "%u", &val); + if (chars == 0) + return size; + + /* Convert kb to number of pages */ + val = val / (PAGE_SIZE >> 10); + + if (attr == &ttm_page_pool_max) + m->options.max_size = val; + else if (attr == &ttm_page_pool_small) + m->options.small = val; + else if (attr == &ttm_page_pool_alloc_size) { + if (val > NUM_PAGES_TO_ALLOC*8) { + pr_err("Setting allocation size to %lu is not allowed. Recommended size is %lu\n", + NUM_PAGES_TO_ALLOC*(PAGE_SIZE >> 7), + NUM_PAGES_TO_ALLOC*(PAGE_SIZE >> 10)); + return size; + } else if (val > NUM_PAGES_TO_ALLOC) { + pr_warn("Setting allocation size to larger than %lu is not recommended\n", + NUM_PAGES_TO_ALLOC*(PAGE_SIZE >> 10)); + } + m->options.alloc_size = val; + } + + return size; +} + +static ssize_t ttm_pool_show(struct kobject *kobj, struct attribute *attr, + char *buffer) +{ + struct ttm_pool_manager *m = + container_of(kobj, struct ttm_pool_manager, kobj); + unsigned val = 0; + + if (attr == &ttm_page_pool_max) + val = m->options.max_size; + else if (attr == &ttm_page_pool_small) + val = m->options.small; + else if (attr == &ttm_page_pool_alloc_size) + val = m->options.alloc_size; + + val = val * (PAGE_SIZE >> 10); + + return snprintf(buffer, PAGE_SIZE, "%u\n", val); +} + +static const struct sysfs_ops ttm_pool_sysfs_ops = { + .show = &ttm_pool_show, + .store = &ttm_pool_store, +}; + +static struct kobj_type ttm_pool_kobj_type = { + .release = &ttm_pool_kobj_release, + .sysfs_ops = &ttm_pool_sysfs_ops, + .default_attrs = ttm_pool_attrs, +}; + +#ifndef CONFIG_X86 +static int set_pages_array_wb(struct page **pages, int addrinarray) +{ +#ifdef TTM_HAS_AGP + int i; + + for (i = 0; i < addrinarray; i++) + unmap_page_from_agp(pages[i]); +#endif + return 0; +} + +static int set_pages_array_wc(struct page **pages, int addrinarray) +{ +#ifdef TTM_HAS_AGP + int i; + + for (i = 0; i < addrinarray; i++) + map_page_into_agp(pages[i]); +#endif + return 0; +} + +static int set_pages_array_uc(struct page **pages, int addrinarray) +{ +#ifdef TTM_HAS_AGP + int i; + + for (i = 0; i < addrinarray; i++) + map_page_into_agp(pages[i]); +#endif + return 0; +} +#endif /* for !CONFIG_X86 */ + +static int ttm_set_pages_caching(struct dma_pool *pool, + struct page **pages, unsigned cpages) +{ + int r = 0; + /* Set page caching */ + if (pool->type & IS_UC) { + r = set_pages_array_uc(pages, cpages); + if (r) + pr_err("%s: Failed to set %d pages to uc!\n", + pool->dev_name, cpages); + } + if (pool->type & IS_WC) { + r = set_pages_array_wc(pages, cpages); + if (r) + pr_err("%s: Failed to set %d pages to wc!\n", + pool->dev_name, cpages); + } + return r; +} + +static void __ttm_dma_free_page(struct dma_pool *pool, struct dma_page *d_page) +{ + dma_addr_t dma = d_page->dma; + dma_free_coherent(pool->dev, pool->size, d_page->vaddr, dma); + + kfree(d_page); + d_page = NULL; +} +static struct dma_page *__ttm_dma_alloc_page(struct dma_pool *pool) +{ + struct dma_page *d_page; + + d_page = kmalloc(sizeof(struct dma_page), GFP_KERNEL); + if (!d_page) + return NULL; + + d_page->vaddr = dma_alloc_coherent(pool->dev, pool->size, + &d_page->dma, + pool->gfp_flags); + if (d_page->vaddr) + d_page->p = virt_to_page(d_page->vaddr); + else { + kfree(d_page); + d_page = NULL; + } + return d_page; +} +static enum pool_type ttm_to_type(int flags, enum ttm_caching_state cstate) +{ + enum pool_type type = IS_UNDEFINED; + + if (flags & TTM_PAGE_FLAG_DMA32) + type |= IS_DMA32; + if (cstate == tt_cached) + type |= IS_CACHED; + else if (cstate == tt_uncached) + type |= IS_UC; + else + type |= IS_WC; + + return type; +} + +static void ttm_pool_update_free_locked(struct dma_pool *pool, + unsigned freed_pages) +{ + pool->npages_free -= freed_pages; + pool->nfrees += freed_pages; + +} + +/* set memory back to wb and free the pages. */ +static void ttm_dma_pages_put(struct dma_pool *pool, struct list_head *d_pages, + struct page *pages[], unsigned npages) +{ + struct dma_page *d_page, *tmp; + + /* Don't set WB on WB page pool. */ + if (npages && !(pool->type & IS_CACHED) && + set_pages_array_wb(pages, npages)) + pr_err("%s: Failed to set %d pages to wb!\n", + pool->dev_name, npages); + + list_for_each_entry_safe(d_page, tmp, d_pages, page_list) { + list_del(&d_page->page_list); + __ttm_dma_free_page(pool, d_page); + } +} + +static void ttm_dma_page_put(struct dma_pool *pool, struct dma_page *d_page) +{ + /* Don't set WB on WB page pool. */ + if (!(pool->type & IS_CACHED) && set_pages_array_wb(&d_page->p, 1)) + pr_err("%s: Failed to set %d pages to wb!\n", + pool->dev_name, 1); + + list_del(&d_page->page_list); + __ttm_dma_free_page(pool, d_page); +} + +/* + * Free pages from pool. + * + * To prevent hogging the ttm_swap process we only free NUM_PAGES_TO_ALLOC + * number of pages in one go. + * + * @pool: to free the pages from + * @nr_free: If set to true will free all pages in pool + **/ +static unsigned ttm_dma_page_pool_free(struct dma_pool *pool, unsigned nr_free) +{ + unsigned long irq_flags; + struct dma_page *dma_p, *tmp; + struct page **pages_to_free; + struct list_head d_pages; + unsigned freed_pages = 0, + npages_to_free = nr_free; + + if (NUM_PAGES_TO_ALLOC < nr_free) + npages_to_free = NUM_PAGES_TO_ALLOC; +#if 0 + if (nr_free > 1) { + pr_debug("%s: (%s:%d) Attempting to free %d (%d) pages\n", + pool->dev_name, pool->name, current->pid, + npages_to_free, nr_free); + } +#endif + pages_to_free = kmalloc(npages_to_free * sizeof(struct page *), + GFP_KERNEL); + + if (!pages_to_free) { + pr_err("%s: Failed to allocate memory for pool free operation\n", + pool->dev_name); + return 0; + } + INIT_LIST_HEAD(&d_pages); +restart: + spin_lock_irqsave(&pool->lock, irq_flags); + + /* We picking the oldest ones off the list */ + list_for_each_entry_safe_reverse(dma_p, tmp, &pool->free_list, + page_list) { + if (freed_pages >= npages_to_free) + break; + + /* Move the dma_page from one list to another. */ + list_move(&dma_p->page_list, &d_pages); + + pages_to_free[freed_pages++] = dma_p->p; + /* We can only remove NUM_PAGES_TO_ALLOC at a time. */ + if (freed_pages >= NUM_PAGES_TO_ALLOC) { + + ttm_pool_update_free_locked(pool, freed_pages); + /** + * Because changing page caching is costly + * we unlock the pool to prevent stalling. + */ + spin_unlock_irqrestore(&pool->lock, irq_flags); + + ttm_dma_pages_put(pool, &d_pages, pages_to_free, + freed_pages); + + INIT_LIST_HEAD(&d_pages); + + if (likely(nr_free != FREE_ALL_PAGES)) + nr_free -= freed_pages; + + if (NUM_PAGES_TO_ALLOC >= nr_free) + npages_to_free = nr_free; + else + npages_to_free = NUM_PAGES_TO_ALLOC; + + freed_pages = 0; + + /* free all so restart the processing */ + if (nr_free) + goto restart; + + /* Not allowed to fall through or break because + * following context is inside spinlock while we are + * outside here. + */ + goto out; + + } + } + + /* remove range of pages from the pool */ + if (freed_pages) { + ttm_pool_update_free_locked(pool, freed_pages); + nr_free -= freed_pages; + } + + spin_unlock_irqrestore(&pool->lock, irq_flags); + + if (freed_pages) + ttm_dma_pages_put(pool, &d_pages, pages_to_free, freed_pages); +out: + kfree(pages_to_free); + return nr_free; +} + +static void ttm_dma_free_pool(struct device *dev, enum pool_type type) +{ + struct device_pools *p; + struct dma_pool *pool; + + if (!dev) + return; + + mutex_lock(&_manager->lock); + list_for_each_entry_reverse(p, &_manager->pools, pools) { + if (p->dev != dev) + continue; + pool = p->pool; + if (pool->type != type) + continue; + + list_del(&p->pools); + kfree(p); + _manager->npools--; + break; + } + list_for_each_entry_reverse(pool, &dev->dma_pools, pools) { + if (pool->type != type) + continue; + /* Takes a spinlock.. */ + ttm_dma_page_pool_free(pool, FREE_ALL_PAGES); + WARN_ON(((pool->npages_in_use + pool->npages_free) != 0)); + /* This code path is called after _all_ references to the + * struct device has been dropped - so nobody should be + * touching it. In case somebody is trying to _add_ we are + * guarded by the mutex. */ + list_del(&pool->pools); + kfree(pool); + break; + } + mutex_unlock(&_manager->lock); +} + +/* + * On free-ing of the 'struct device' this deconstructor is run. + * Albeit the pool might have already been freed earlier. + */ +static void ttm_dma_pool_release(struct device *dev, void *res) +{ + struct dma_pool *pool = *(struct dma_pool **)res; + + if (pool) + ttm_dma_free_pool(dev, pool->type); +} + +static int ttm_dma_pool_match(struct device *dev, void *res, void *match_data) +{ + return *(struct dma_pool **)res == match_data; +} + +static struct dma_pool *ttm_dma_pool_init(struct device *dev, gfp_t flags, + enum pool_type type) +{ + char *n[] = {"wc", "uc", "cached", " dma32", "unknown",}; + enum pool_type t[] = {IS_WC, IS_UC, IS_CACHED, IS_DMA32, IS_UNDEFINED}; + struct device_pools *sec_pool = NULL; + struct dma_pool *pool = NULL, **ptr; + unsigned i; + int ret = -ENODEV; + char *p; + + if (!dev) + return NULL; + + ptr = devres_alloc(ttm_dma_pool_release, sizeof(*ptr), GFP_KERNEL); + if (!ptr) + return NULL; + + ret = -ENOMEM; + + pool = kmalloc_node(sizeof(struct dma_pool), GFP_KERNEL, + dev_to_node(dev)); + if (!pool) + goto err_mem; + + sec_pool = kmalloc_node(sizeof(struct device_pools), GFP_KERNEL, + dev_to_node(dev)); + if (!sec_pool) + goto err_mem; + + INIT_LIST_HEAD(&sec_pool->pools); + sec_pool->dev = dev; + sec_pool->pool = pool; + + INIT_LIST_HEAD(&pool->free_list); + INIT_LIST_HEAD(&pool->inuse_list); + INIT_LIST_HEAD(&pool->pools); + spin_lock_init(&pool->lock); + pool->dev = dev; + pool->npages_free = pool->npages_in_use = 0; + pool->nfrees = 0; + pool->gfp_flags = flags; + pool->size = PAGE_SIZE; + pool->type = type; + pool->nrefills = 0; + p = pool->name; + for (i = 0; i < 5; i++) { + if (type & t[i]) { + p += snprintf(p, sizeof(pool->name) - (p - pool->name), + "%s", n[i]); + } + } + *p = 0; + /* We copy the name for pr_ calls b/c when dma_pool_destroy is called + * - the kobj->name has already been deallocated.*/ + snprintf(pool->dev_name, sizeof(pool->dev_name), "%s %s", + dev_driver_string(dev), dev_name(dev)); + mutex_lock(&_manager->lock); + /* You can get the dma_pool from either the global: */ + list_add(&sec_pool->pools, &_manager->pools); + _manager->npools++; + /* or from 'struct device': */ + list_add(&pool->pools, &dev->dma_pools); + mutex_unlock(&_manager->lock); + + *ptr = pool; + devres_add(dev, ptr); + + return pool; +err_mem: + devres_free(ptr); + kfree(sec_pool); + kfree(pool); + return ERR_PTR(ret); +} + +static struct dma_pool *ttm_dma_find_pool(struct device *dev, + enum pool_type type) +{ + struct dma_pool *pool, *tmp, *found = NULL; + + if (type == IS_UNDEFINED) + return found; + + /* NB: We iterate on the 'struct dev' which has no spinlock, but + * it does have a kref which we have taken. The kref is taken during + * graphic driver loading - in the drm_pci_init it calls either + * pci_dev_get or pci_register_driver which both end up taking a kref + * on 'struct device'. + * + * On teardown, the graphic drivers end up quiescing the TTM (put_pages) + * and calls the dev_res deconstructors: ttm_dma_pool_release. The nice + * thing is at that point of time there are no pages associated with the + * driver so this function will not be called. + */ + list_for_each_entry_safe(pool, tmp, &dev->dma_pools, pools) { + if (pool->type != type) + continue; + found = pool; + break; + } + return found; +} + +/* + * Free pages the pages that failed to change the caching state. If there + * are pages that have changed their caching state already put them to the + * pool. + */ +static void ttm_dma_handle_caching_state_failure(struct dma_pool *pool, + struct list_head *d_pages, + struct page **failed_pages, + unsigned cpages) +{ + struct dma_page *d_page, *tmp; + struct page *p; + unsigned i = 0; + + p = failed_pages[0]; + if (!p) + return; + /* Find the failed page. */ + list_for_each_entry_safe(d_page, tmp, d_pages, page_list) { + if (d_page->p != p) + continue; + /* .. and then progress over the full list. */ + list_del(&d_page->page_list); + __ttm_dma_free_page(pool, d_page); + if (++i < cpages) + p = failed_pages[i]; + else + break; + } + +} + +/* + * Allocate 'count' pages, and put 'need' number of them on the + * 'pages' and as well on the 'dma_address' starting at 'dma_offset' offset. + * The full list of pages should also be on 'd_pages'. + * We return zero for success, and negative numbers as errors. + */ +static int ttm_dma_pool_alloc_new_pages(struct dma_pool *pool, + struct list_head *d_pages, + unsigned count) +{ + struct page **caching_array; + struct dma_page *dma_p; + struct page *p; + int r = 0; + unsigned i, cpages; + unsigned max_cpages = min(count, + (unsigned)(PAGE_SIZE/sizeof(struct page *))); + + /* allocate array for page caching change */ + caching_array = kmalloc(max_cpages*sizeof(struct page *), GFP_KERNEL); + + if (!caching_array) { + pr_err("%s: Unable to allocate table for new pages\n", + pool->dev_name); + return -ENOMEM; + } + + if (count > 1) { + pr_debug("%s: (%s:%d) Getting %d pages\n", + pool->dev_name, pool->name, current->pid, count); + } + + for (i = 0, cpages = 0; i < count; ++i) { + dma_p = __ttm_dma_alloc_page(pool); + if (!dma_p) { + pr_err("%s: Unable to get page %u\n", + pool->dev_name, i); + + /* store already allocated pages in the pool after + * setting the caching state */ + if (cpages) { + r = ttm_set_pages_caching(pool, caching_array, + cpages); + if (r) + ttm_dma_handle_caching_state_failure( + pool, d_pages, caching_array, + cpages); + } + r = -ENOMEM; + goto out; + } + p = dma_p->p; +#ifdef CONFIG_HIGHMEM + /* gfp flags of highmem page should never be dma32 so we + * we should be fine in such case + */ + if (!PageHighMem(p)) +#endif + { + caching_array[cpages++] = p; + if (cpages == max_cpages) { + /* Note: Cannot hold the spinlock */ + r = ttm_set_pages_caching(pool, caching_array, + cpages); + if (r) { + ttm_dma_handle_caching_state_failure( + pool, d_pages, caching_array, + cpages); + goto out; + } + cpages = 0; + } + } + list_add(&dma_p->page_list, d_pages); + } + + if (cpages) { + r = ttm_set_pages_caching(pool, caching_array, cpages); + if (r) + ttm_dma_handle_caching_state_failure(pool, d_pages, + caching_array, cpages); + } +out: + kfree(caching_array); + return r; +} + +/* + * @return count of pages still required to fulfill the request. + */ +static int ttm_dma_page_pool_fill_locked(struct dma_pool *pool, + unsigned long *irq_flags) +{ + unsigned count = _manager->options.small; + int r = pool->npages_free; + + if (count > pool->npages_free) { + struct list_head d_pages; + + INIT_LIST_HEAD(&d_pages); + + spin_unlock_irqrestore(&pool->lock, *irq_flags); + + /* Returns how many more are neccessary to fulfill the + * request. */ + r = ttm_dma_pool_alloc_new_pages(pool, &d_pages, count); + + spin_lock_irqsave(&pool->lock, *irq_flags); + if (!r) { + /* Add the fresh to the end.. */ + list_splice(&d_pages, &pool->free_list); + ++pool->nrefills; + pool->npages_free += count; + r = count; + } else { + struct dma_page *d_page; + unsigned cpages = 0; + + pr_err("%s: Failed to fill %s pool (r:%d)!\n", + pool->dev_name, pool->name, r); + + list_for_each_entry(d_page, &d_pages, page_list) { + cpages++; + } + list_splice_tail(&d_pages, &pool->free_list); + pool->npages_free += cpages; + r = cpages; + } + } + return r; +} + +/* + * @return count of pages still required to fulfill the request. + * The populate list is actually a stack (not that is matters as TTM + * allocates one page at a time. + */ +static int ttm_dma_pool_get_pages(struct dma_pool *pool, + struct ttm_dma_tt *ttm_dma, + unsigned index) +{ + struct dma_page *d_page; + struct ttm_tt *ttm = &ttm_dma->ttm; + unsigned long irq_flags; + int count, r = -ENOMEM; + + spin_lock_irqsave(&pool->lock, irq_flags); + count = ttm_dma_page_pool_fill_locked(pool, &irq_flags); + if (count) { + d_page = list_first_entry(&pool->free_list, struct dma_page, page_list); + ttm->pages[index] = d_page->p; + ttm_dma->dma_address[index] = d_page->dma; + list_move_tail(&d_page->page_list, &ttm_dma->pages_list); + r = 0; + pool->npages_in_use += 1; + pool->npages_free -= 1; + } + spin_unlock_irqrestore(&pool->lock, irq_flags); + return r; +} + +/* + * On success pages list will hold count number of correctly + * cached pages. On failure will hold the negative return value (-ENOMEM, etc). + */ +int ttm_dma_populate(struct ttm_dma_tt *ttm_dma, struct device *dev) +{ + struct ttm_tt *ttm = &ttm_dma->ttm; + struct ttm_mem_global *mem_glob = ttm->glob->mem_glob; + struct dma_pool *pool; + enum pool_type type; + unsigned i; + gfp_t gfp_flags; + int ret; + + if (ttm->state != tt_unpopulated) + return 0; + + type = ttm_to_type(ttm->page_flags, ttm->caching_state); + if (ttm->page_flags & TTM_PAGE_FLAG_DMA32) + gfp_flags = GFP_USER | GFP_DMA32; + else + gfp_flags = GFP_HIGHUSER; + if (ttm->page_flags & TTM_PAGE_FLAG_ZERO_ALLOC) + gfp_flags |= __GFP_ZERO; + + pool = ttm_dma_find_pool(dev, type); + if (!pool) { + pool = ttm_dma_pool_init(dev, gfp_flags, type); + if (IS_ERR_OR_NULL(pool)) { + return -ENOMEM; + } + } + + INIT_LIST_HEAD(&ttm_dma->pages_list); + for (i = 0; i < ttm->num_pages; ++i) { + ret = ttm_dma_pool_get_pages(pool, ttm_dma, i); + if (ret != 0) { + ttm_dma_unpopulate(ttm_dma, dev); + return -ENOMEM; + } + + ret = ttm_mem_global_alloc_page(mem_glob, ttm->pages[i], + false, false); + if (unlikely(ret != 0)) { + ttm_dma_unpopulate(ttm_dma, dev); + return -ENOMEM; + } + } + + if (unlikely(ttm->page_flags & TTM_PAGE_FLAG_SWAPPED)) { + ret = ttm_tt_swapin(ttm); + if (unlikely(ret != 0)) { + ttm_dma_unpopulate(ttm_dma, dev); + return ret; + } + } + + ttm->state = tt_unbound; + return 0; +} +EXPORT_SYMBOL_GPL(ttm_dma_populate); + +/* Put all pages in pages list to correct pool to wait for reuse */ +void ttm_dma_unpopulate(struct ttm_dma_tt *ttm_dma, struct device *dev) +{ + struct ttm_tt *ttm = &ttm_dma->ttm; + struct dma_pool *pool; + struct dma_page *d_page, *next; + enum pool_type type; + bool is_cached = false; + unsigned count = 0, i, npages = 0; + unsigned long irq_flags; + + type = ttm_to_type(ttm->page_flags, ttm->caching_state); + pool = ttm_dma_find_pool(dev, type); + if (!pool) + return; + + is_cached = (ttm_dma_find_pool(pool->dev, + ttm_to_type(ttm->page_flags, tt_cached)) == pool); + + /* make sure pages array match list and count number of pages */ + list_for_each_entry(d_page, &ttm_dma->pages_list, page_list) { + ttm->pages[count] = d_page->p; + count++; + } + + spin_lock_irqsave(&pool->lock, irq_flags); + pool->npages_in_use -= count; + if (is_cached) { + pool->nfrees += count; + } else { + pool->npages_free += count; + list_splice(&ttm_dma->pages_list, &pool->free_list); + npages = count; + if (pool->npages_free > _manager->options.max_size) { + npages = pool->npages_free - _manager->options.max_size; + /* free at least NUM_PAGES_TO_ALLOC number of pages + * to reduce calls to set_memory_wb */ + if (npages < NUM_PAGES_TO_ALLOC) + npages = NUM_PAGES_TO_ALLOC; + } + } + spin_unlock_irqrestore(&pool->lock, irq_flags); + + if (is_cached) { + list_for_each_entry_safe(d_page, next, &ttm_dma->pages_list, page_list) { + ttm_mem_global_free_page(ttm->glob->mem_glob, + d_page->p); + ttm_dma_page_put(pool, d_page); + } + } else { + for (i = 0; i < count; i++) { + ttm_mem_global_free_page(ttm->glob->mem_glob, + ttm->pages[i]); + } + } + + INIT_LIST_HEAD(&ttm_dma->pages_list); + for (i = 0; i < ttm->num_pages; i++) { + ttm->pages[i] = NULL; + ttm_dma->dma_address[i] = 0; + } + + /* shrink pool if necessary (only on !is_cached pools)*/ + if (npages) + ttm_dma_page_pool_free(pool, npages); + ttm->state = tt_unpopulated; +} +EXPORT_SYMBOL_GPL(ttm_dma_unpopulate); + +/** + * Callback for mm to request pool to reduce number of page held. + * + * XXX: (dchinner) Deadlock warning! + * + * ttm_dma_page_pool_free() does GFP_KERNEL memory allocation, and so attention + * needs to be paid to sc->gfp_mask to determine if this can be done or not. + * GFP_KERNEL memory allocation in a GFP_ATOMIC reclaim context woul dbe really + * bad. + * + * I'm getting sadder as I hear more pathetical whimpers about needing per-pool + * shrinkers + */ +static unsigned long +ttm_dma_pool_shrink_scan(struct shrinker *shrink, struct shrink_control *sc) +{ + static atomic_t start_pool = ATOMIC_INIT(0); + unsigned idx = 0; + unsigned pool_offset = atomic_add_return(1, &start_pool); + unsigned shrink_pages = sc->nr_to_scan; + struct device_pools *p; + unsigned long freed = 0; + + if (list_empty(&_manager->pools)) + return SHRINK_STOP; + + mutex_lock(&_manager->lock); + pool_offset = pool_offset % _manager->npools; + list_for_each_entry(p, &_manager->pools, pools) { + unsigned nr_free; + + if (!p->dev) + continue; + if (shrink_pages == 0) + break; + /* Do it in round-robin fashion. */ + if (++idx < pool_offset) + continue; + nr_free = shrink_pages; + shrink_pages = ttm_dma_page_pool_free(p->pool, nr_free); + freed += nr_free - shrink_pages; + + pr_debug("%s: (%s:%d) Asked to shrink %d, have %d more to go\n", + p->pool->dev_name, p->pool->name, current->pid, + nr_free, shrink_pages); + } + mutex_unlock(&_manager->lock); + return freed; +} + +static unsigned long +ttm_dma_pool_shrink_count(struct shrinker *shrink, struct shrink_control *sc) +{ + struct device_pools *p; + unsigned long count = 0; + + mutex_lock(&_manager->lock); + list_for_each_entry(p, &_manager->pools, pools) + count += p->pool->npages_free; + mutex_unlock(&_manager->lock); + return count; +} + +static void ttm_dma_pool_mm_shrink_init(struct ttm_pool_manager *manager) +{ + manager->mm_shrink.count_objects = ttm_dma_pool_shrink_count; + manager->mm_shrink.scan_objects = &ttm_dma_pool_shrink_scan; + manager->mm_shrink.seeks = 1; + register_shrinker(&manager->mm_shrink); +} + +static void ttm_dma_pool_mm_shrink_fini(struct ttm_pool_manager *manager) +{ + unregister_shrinker(&manager->mm_shrink); +} + +int ttm_dma_page_alloc_init(struct ttm_mem_global *glob, unsigned max_pages) +{ + int ret = -ENOMEM; + + WARN_ON(_manager); + + pr_info("Initializing DMA pool allocator\n"); + + _manager = kzalloc(sizeof(*_manager), GFP_KERNEL); + if (!_manager) + goto err; + + mutex_init(&_manager->lock); + INIT_LIST_HEAD(&_manager->pools); + + _manager->options.max_size = max_pages; + _manager->options.small = SMALL_ALLOCATION; + _manager->options.alloc_size = NUM_PAGES_TO_ALLOC; + + /* This takes care of auto-freeing the _manager */ + ret = kobject_init_and_add(&_manager->kobj, &ttm_pool_kobj_type, + &glob->kobj, "dma_pool"); + if (unlikely(ret != 0)) { + kobject_put(&_manager->kobj); + goto err; + } + ttm_dma_pool_mm_shrink_init(_manager); + return 0; +err: + return ret; +} + +void ttm_dma_page_alloc_fini(void) +{ + struct device_pools *p, *t; + + pr_info("Finalizing DMA pool allocator\n"); + ttm_dma_pool_mm_shrink_fini(_manager); + + list_for_each_entry_safe_reverse(p, t, &_manager->pools, pools) { + dev_dbg(p->dev, "(%s:%d) Freeing.\n", p->pool->name, + current->pid); + WARN_ON(devres_destroy(p->dev, ttm_dma_pool_release, + ttm_dma_pool_match, p->pool)); + ttm_dma_free_pool(p->dev, p->pool->type); + } + kobject_put(&_manager->kobj); + _manager = NULL; +} + +int ttm_dma_page_alloc_debugfs(struct seq_file *m, void *data) +{ + struct device_pools *p; + struct dma_pool *pool = NULL; + char *h[] = {"pool", "refills", "pages freed", "inuse", "available", + "name", "virt", "busaddr"}; + + if (!_manager) { + seq_printf(m, "No pool allocator running.\n"); + return 0; + } + seq_printf(m, "%13s %12s %13s %8s %8s %8s\n", + h[0], h[1], h[2], h[3], h[4], h[5]); + mutex_lock(&_manager->lock); + list_for_each_entry(p, &_manager->pools, pools) { + struct device *dev = p->dev; + if (!dev) + continue; + pool = p->pool; + seq_printf(m, "%13s %12ld %13ld %8d %8d %8s\n", + pool->name, pool->nrefills, + pool->nfrees, pool->npages_in_use, + pool->npages_free, + pool->dev_name); + } + mutex_unlock(&_manager->lock); + return 0; +} +EXPORT_SYMBOL_GPL(ttm_dma_page_alloc_debugfs); + +#endif diff --git a/drivers/gpu/drm/ttm/ttm_tt.c b/drivers/gpu/drm/ttm/ttm_tt.c index af789dc869b..75f31909004 100644 --- a/drivers/gpu/drm/ttm/ttm_tt.c +++ b/drivers/gpu/drm/ttm/ttm_tt.c @@ -28,148 +28,38 @@ * Authors: Thomas Hellstrom <thellstrom-at-vmware-dot-com> */ +#define pr_fmt(fmt) "[TTM] " fmt + #include <linux/sched.h> #include <linux/highmem.h> #include <linux/pagemap.h> +#include <linux/shmem_fs.h> #include <linux/file.h> #include <linux/swap.h> #include <linux/slab.h> -#include "drm_cache.h" -#include "drm_mem_util.h" -#include "ttm/ttm_module.h" -#include "ttm/ttm_bo_driver.h" -#include "ttm/ttm_placement.h" -#include "ttm/ttm_page_alloc.h" - -static int ttm_tt_swapin(struct ttm_tt *ttm); +#include <linux/export.h> +#include <drm/drm_cache.h> +#include <drm/drm_mem_util.h> +#include <drm/ttm/ttm_module.h> +#include <drm/ttm/ttm_bo_driver.h> +#include <drm/ttm/ttm_placement.h> +#include <drm/ttm/ttm_page_alloc.h> /** * Allocates storage for pointers to the pages that back the ttm. */ static void ttm_tt_alloc_page_directory(struct ttm_tt *ttm) { - ttm->pages = drm_calloc_large(ttm->num_pages, sizeof(*ttm->pages)); + ttm->pages = drm_calloc_large(ttm->num_pages, sizeof(void*)); } -static void ttm_tt_free_page_directory(struct ttm_tt *ttm) +static void ttm_dma_tt_alloc_page_directory(struct ttm_dma_tt *ttm) { - drm_free_large(ttm->pages); - ttm->pages = NULL; + ttm->ttm.pages = drm_calloc_large(ttm->ttm.num_pages, sizeof(void*)); + ttm->dma_address = drm_calloc_large(ttm->ttm.num_pages, + sizeof(*ttm->dma_address)); } -static void ttm_tt_free_user_pages(struct ttm_tt *ttm) -{ - int write; - int dirty; - struct page *page; - int i; - struct ttm_backend *be = ttm->be; - - BUG_ON(!(ttm->page_flags & TTM_PAGE_FLAG_USER)); - write = ((ttm->page_flags & TTM_PAGE_FLAG_WRITE) != 0); - dirty = ((ttm->page_flags & TTM_PAGE_FLAG_USER_DIRTY) != 0); - - if (be) - be->func->clear(be); - - for (i = 0; i < ttm->num_pages; ++i) { - page = ttm->pages[i]; - if (page == NULL) - continue; - - if (page == ttm->dummy_read_page) { - BUG_ON(write); - continue; - } - - if (write && dirty && !PageReserved(page)) - set_page_dirty_lock(page); - - ttm->pages[i] = NULL; - ttm_mem_global_free(ttm->glob->mem_glob, PAGE_SIZE); - put_page(page); - } - ttm->state = tt_unpopulated; - ttm->first_himem_page = ttm->num_pages; - ttm->last_lomem_page = -1; -} - -static struct page *__ttm_tt_get_page(struct ttm_tt *ttm, int index) -{ - struct page *p; - struct list_head h; - struct ttm_mem_global *mem_glob = ttm->glob->mem_glob; - int ret; - - while (NULL == (p = ttm->pages[index])) { - - INIT_LIST_HEAD(&h); - - ret = ttm_get_pages(&h, ttm->page_flags, ttm->caching_state, 1); - - if (ret != 0) - return NULL; - - p = list_first_entry(&h, struct page, lru); - - ret = ttm_mem_global_alloc_page(mem_glob, p, false, false); - if (unlikely(ret != 0)) - goto out_err; - - if (PageHighMem(p)) - ttm->pages[--ttm->first_himem_page] = p; - else - ttm->pages[++ttm->last_lomem_page] = p; - } - return p; -out_err: - put_page(p); - return NULL; -} - -struct page *ttm_tt_get_page(struct ttm_tt *ttm, int index) -{ - int ret; - - if (unlikely(ttm->page_flags & TTM_PAGE_FLAG_SWAPPED)) { - ret = ttm_tt_swapin(ttm); - if (unlikely(ret != 0)) - return NULL; - } - return __ttm_tt_get_page(ttm, index); -} - -int ttm_tt_populate(struct ttm_tt *ttm) -{ - struct page *page; - unsigned long i; - struct ttm_backend *be; - int ret; - - if (ttm->state != tt_unpopulated) - return 0; - - if (unlikely(ttm->page_flags & TTM_PAGE_FLAG_SWAPPED)) { - ret = ttm_tt_swapin(ttm); - if (unlikely(ret != 0)) - return ret; - } - - be = ttm->be; - - for (i = 0; i < ttm->num_pages; ++i) { - page = __ttm_tt_get_page(ttm, i); - if (!page) - return -ENOMEM; - } - - be->func->populate(be, ttm->num_pages, ttm->pages, - ttm->dummy_read_page); - ttm->state = tt_unbound; - return 0; -} -EXPORT_SYMBOL(ttm_tt_populate); - #ifdef CONFIG_X86 static inline int ttm_tt_set_page_caching(struct page *p, enum ttm_caching_state c_old, @@ -271,152 +161,99 @@ int ttm_tt_set_placement_caching(struct ttm_tt *ttm, uint32_t placement) } EXPORT_SYMBOL(ttm_tt_set_placement_caching); -static void ttm_tt_free_alloced_pages(struct ttm_tt *ttm) -{ - int i; - unsigned count = 0; - struct list_head h; - struct page *cur_page; - struct ttm_backend *be = ttm->be; - - INIT_LIST_HEAD(&h); - - if (be) - be->func->clear(be); - for (i = 0; i < ttm->num_pages; ++i) { - - cur_page = ttm->pages[i]; - ttm->pages[i] = NULL; - if (cur_page) { - if (page_count(cur_page) != 1) - printk(KERN_ERR TTM_PFX - "Erroneous page count. " - "Leaking pages.\n"); - ttm_mem_global_free_page(ttm->glob->mem_glob, - cur_page); - list_add(&cur_page->lru, &h); - count++; - } - } - ttm_put_pages(&h, count, ttm->page_flags, ttm->caching_state); - ttm->state = tt_unpopulated; - ttm->first_himem_page = ttm->num_pages; - ttm->last_lomem_page = -1; -} - void ttm_tt_destroy(struct ttm_tt *ttm) { - struct ttm_backend *be; - if (unlikely(ttm == NULL)) return; - be = ttm->be; - if (likely(be != NULL)) { - be->func->destroy(be); - ttm->be = NULL; + if (ttm->state == tt_bound) { + ttm_tt_unbind(ttm); } - if (likely(ttm->pages != NULL)) { - if (ttm->page_flags & TTM_PAGE_FLAG_USER) - ttm_tt_free_user_pages(ttm); - else - ttm_tt_free_alloced_pages(ttm); - - ttm_tt_free_page_directory(ttm); - } + if (ttm->state == tt_unbound) + ttm_tt_unpopulate(ttm); - if (!(ttm->page_flags & TTM_PAGE_FLAG_PERSISTANT_SWAP) && + if (!(ttm->page_flags & TTM_PAGE_FLAG_PERSISTENT_SWAP) && ttm->swap_storage) fput(ttm->swap_storage); - kfree(ttm); + ttm->swap_storage = NULL; + ttm->func->destroy(ttm); } -int ttm_tt_set_user(struct ttm_tt *ttm, - struct task_struct *tsk, - unsigned long start, unsigned long num_pages) +int ttm_tt_init(struct ttm_tt *ttm, struct ttm_bo_device *bdev, + unsigned long size, uint32_t page_flags, + struct page *dummy_read_page) { - struct mm_struct *mm = tsk->mm; - int ret; - int write = (ttm->page_flags & TTM_PAGE_FLAG_WRITE) != 0; - struct ttm_mem_global *mem_glob = ttm->glob->mem_glob; - - BUG_ON(num_pages != ttm->num_pages); - BUG_ON((ttm->page_flags & TTM_PAGE_FLAG_USER) == 0); - - /** - * Account user pages as lowmem pages for now. - */ - - ret = ttm_mem_global_alloc(mem_glob, num_pages * PAGE_SIZE, - false, false); - if (unlikely(ret != 0)) - return ret; - - down_read(&mm->mmap_sem); - ret = get_user_pages(tsk, mm, start, num_pages, - write, 0, ttm->pages, NULL); - up_read(&mm->mmap_sem); + ttm->bdev = bdev; + ttm->glob = bdev->glob; + ttm->num_pages = (size + PAGE_SIZE - 1) >> PAGE_SHIFT; + ttm->caching_state = tt_cached; + ttm->page_flags = page_flags; + ttm->dummy_read_page = dummy_read_page; + ttm->state = tt_unpopulated; + ttm->swap_storage = NULL; - if (ret != num_pages && write) { - ttm_tt_free_user_pages(ttm); - ttm_mem_global_free(mem_glob, num_pages * PAGE_SIZE); + ttm_tt_alloc_page_directory(ttm); + if (!ttm->pages) { + ttm_tt_destroy(ttm); + pr_err("Failed allocating page table\n"); return -ENOMEM; } - - ttm->tsk = tsk; - ttm->start = start; - ttm->state = tt_unbound; - return 0; } +EXPORT_SYMBOL(ttm_tt_init); -struct ttm_tt *ttm_tt_create(struct ttm_bo_device *bdev, unsigned long size, - uint32_t page_flags, struct page *dummy_read_page) +void ttm_tt_fini(struct ttm_tt *ttm) { - struct ttm_bo_driver *bo_driver = bdev->driver; - struct ttm_tt *ttm; - - if (!bo_driver) - return NULL; + drm_free_large(ttm->pages); + ttm->pages = NULL; +} +EXPORT_SYMBOL(ttm_tt_fini); - ttm = kzalloc(sizeof(*ttm), GFP_KERNEL); - if (!ttm) - return NULL; +int ttm_dma_tt_init(struct ttm_dma_tt *ttm_dma, struct ttm_bo_device *bdev, + unsigned long size, uint32_t page_flags, + struct page *dummy_read_page) +{ + struct ttm_tt *ttm = &ttm_dma->ttm; + ttm->bdev = bdev; ttm->glob = bdev->glob; ttm->num_pages = (size + PAGE_SIZE - 1) >> PAGE_SHIFT; - ttm->first_himem_page = ttm->num_pages; - ttm->last_lomem_page = -1; ttm->caching_state = tt_cached; ttm->page_flags = page_flags; - ttm->dummy_read_page = dummy_read_page; + ttm->state = tt_unpopulated; + ttm->swap_storage = NULL; - ttm_tt_alloc_page_directory(ttm); - if (!ttm->pages) { + INIT_LIST_HEAD(&ttm_dma->pages_list); + ttm_dma_tt_alloc_page_directory(ttm_dma); + if (!ttm->pages || !ttm_dma->dma_address) { ttm_tt_destroy(ttm); - printk(KERN_ERR TTM_PFX "Failed allocating page table\n"); - return NULL; - } - ttm->be = bo_driver->create_ttm_backend_entry(bdev); - if (!ttm->be) { - ttm_tt_destroy(ttm); - printk(KERN_ERR TTM_PFX "Failed creating ttm backend entry\n"); - return NULL; + pr_err("Failed allocating page table\n"); + return -ENOMEM; } - ttm->state = tt_unpopulated; - return ttm; + return 0; } +EXPORT_SYMBOL(ttm_dma_tt_init); + +void ttm_dma_tt_fini(struct ttm_dma_tt *ttm_dma) +{ + struct ttm_tt *ttm = &ttm_dma->ttm; + + drm_free_large(ttm->pages); + ttm->pages = NULL; + drm_free_large(ttm_dma->dma_address); + ttm_dma->dma_address = NULL; +} +EXPORT_SYMBOL(ttm_dma_tt_fini); void ttm_tt_unbind(struct ttm_tt *ttm) { int ret; - struct ttm_backend *be = ttm->be; if (ttm->state == tt_bound) { - ret = be->func->unbind(be); + ret = ttm->func->unbind(ttm); BUG_ON(ret); ttm->state = tt_unbound; } @@ -425,7 +262,6 @@ void ttm_tt_unbind(struct ttm_tt *ttm) int ttm_tt_bind(struct ttm_tt *ttm, struct ttm_mem_reg *bo_mem) { int ret = 0; - struct ttm_backend *be; if (!ttm) return -EINVAL; @@ -433,151 +269,131 @@ int ttm_tt_bind(struct ttm_tt *ttm, struct ttm_mem_reg *bo_mem) if (ttm->state == tt_bound) return 0; - be = ttm->be; - - ret = ttm_tt_populate(ttm); + ret = ttm->bdev->driver->ttm_tt_populate(ttm); if (ret) return ret; - ret = be->func->bind(be, bo_mem); + ret = ttm->func->bind(ttm, bo_mem); if (unlikely(ret != 0)) return ret; ttm->state = tt_bound; - if (ttm->page_flags & TTM_PAGE_FLAG_USER) - ttm->page_flags |= TTM_PAGE_FLAG_USER_DIRTY; return 0; } EXPORT_SYMBOL(ttm_tt_bind); -static int ttm_tt_swapin(struct ttm_tt *ttm) +int ttm_tt_swapin(struct ttm_tt *ttm) { struct address_space *swap_space; struct file *swap_storage; struct page *from_page; struct page *to_page; - void *from_virtual; - void *to_virtual; int i; int ret = -ENOMEM; - if (ttm->page_flags & TTM_PAGE_FLAG_USER) { - ret = ttm_tt_set_user(ttm, ttm->tsk, ttm->start, - ttm->num_pages); - if (unlikely(ret != 0)) - return ret; - - ttm->page_flags &= ~TTM_PAGE_FLAG_SWAPPED; - return 0; - } - swap_storage = ttm->swap_storage; BUG_ON(swap_storage == NULL); - swap_space = swap_storage->f_path.dentry->d_inode->i_mapping; + swap_space = file_inode(swap_storage)->i_mapping; for (i = 0; i < ttm->num_pages; ++i) { - from_page = read_mapping_page(swap_space, i, NULL); + from_page = shmem_read_mapping_page(swap_space, i); if (IS_ERR(from_page)) { ret = PTR_ERR(from_page); goto out_err; } - to_page = __ttm_tt_get_page(ttm, i); + to_page = ttm->pages[i]; if (unlikely(to_page == NULL)) goto out_err; - preempt_disable(); - from_virtual = kmap_atomic(from_page, KM_USER0); - to_virtual = kmap_atomic(to_page, KM_USER1); - memcpy(to_virtual, from_virtual, PAGE_SIZE); - kunmap_atomic(to_virtual, KM_USER1); - kunmap_atomic(from_virtual, KM_USER0); - preempt_enable(); + copy_highpage(to_page, from_page); page_cache_release(from_page); } - if (!(ttm->page_flags & TTM_PAGE_FLAG_PERSISTANT_SWAP)) + if (!(ttm->page_flags & TTM_PAGE_FLAG_PERSISTENT_SWAP)) fput(swap_storage); ttm->swap_storage = NULL; ttm->page_flags &= ~TTM_PAGE_FLAG_SWAPPED; return 0; out_err: - ttm_tt_free_alloced_pages(ttm); return ret; } -int ttm_tt_swapout(struct ttm_tt *ttm, struct file *persistant_swap_storage) +int ttm_tt_swapout(struct ttm_tt *ttm, struct file *persistent_swap_storage) { struct address_space *swap_space; struct file *swap_storage; struct page *from_page; struct page *to_page; - void *from_virtual; - void *to_virtual; int i; int ret = -ENOMEM; BUG_ON(ttm->state != tt_unbound && ttm->state != tt_unpopulated); BUG_ON(ttm->caching_state != tt_cached); - /* - * For user buffers, just unpin the pages, as there should be - * vma references. - */ - - if (ttm->page_flags & TTM_PAGE_FLAG_USER) { - ttm_tt_free_user_pages(ttm); - ttm->page_flags |= TTM_PAGE_FLAG_SWAPPED; - ttm->swap_storage = NULL; - return 0; - } - - if (!persistant_swap_storage) { + if (!persistent_swap_storage) { swap_storage = shmem_file_setup("ttm swap", ttm->num_pages << PAGE_SHIFT, 0); if (unlikely(IS_ERR(swap_storage))) { - printk(KERN_ERR "Failed allocating swap storage.\n"); + pr_err("Failed allocating swap storage\n"); return PTR_ERR(swap_storage); } } else - swap_storage = persistant_swap_storage; + swap_storage = persistent_swap_storage; - swap_space = swap_storage->f_path.dentry->d_inode->i_mapping; + swap_space = file_inode(swap_storage)->i_mapping; for (i = 0; i < ttm->num_pages; ++i) { from_page = ttm->pages[i]; if (unlikely(from_page == NULL)) continue; - to_page = read_mapping_page(swap_space, i, NULL); + to_page = shmem_read_mapping_page(swap_space, i); if (unlikely(IS_ERR(to_page))) { ret = PTR_ERR(to_page); goto out_err; } - preempt_disable(); - from_virtual = kmap_atomic(from_page, KM_USER0); - to_virtual = kmap_atomic(to_page, KM_USER1); - memcpy(to_virtual, from_virtual, PAGE_SIZE); - kunmap_atomic(to_virtual, KM_USER1); - kunmap_atomic(from_virtual, KM_USER0); - preempt_enable(); + copy_highpage(to_page, from_page); set_page_dirty(to_page); mark_page_accessed(to_page); page_cache_release(to_page); } - ttm_tt_free_alloced_pages(ttm); + ttm_tt_unpopulate(ttm); ttm->swap_storage = swap_storage; ttm->page_flags |= TTM_PAGE_FLAG_SWAPPED; - if (persistant_swap_storage) - ttm->page_flags |= TTM_PAGE_FLAG_PERSISTANT_SWAP; + if (persistent_swap_storage) + ttm->page_flags |= TTM_PAGE_FLAG_PERSISTENT_SWAP; return 0; out_err: - if (!persistant_swap_storage) + if (!persistent_swap_storage) fput(swap_storage); return ret; } + +static void ttm_tt_clear_mapping(struct ttm_tt *ttm) +{ + pgoff_t i; + struct page **page = ttm->pages; + + if (ttm->page_flags & TTM_PAGE_FLAG_SG) + return; + + for (i = 0; i < ttm->num_pages; ++i) { + (*page)->mapping = NULL; + (*page++)->index = 0; + } +} + +void ttm_tt_unpopulate(struct ttm_tt *ttm) +{ + if (ttm->state == tt_unpopulated) + return; + + ttm_tt_clear_mapping(ttm); + ttm->bdev->driver->ttm_tt_unpopulate(ttm); +} |