diff options
Diffstat (limited to 'drivers/gpu/drm/ttm')
| -rw-r--r-- | drivers/gpu/drm/ttm/Makefile | 6 | ||||
| -rw-r--r-- | drivers/gpu/drm/ttm/ttm_agp_backend.c | 13 | ||||
| -rw-r--r-- | drivers/gpu/drm/ttm/ttm_bo.c | 755 | ||||
| -rw-r--r-- | drivers/gpu/drm/ttm/ttm_bo_manager.c | 62 | ||||
| -rw-r--r-- | drivers/gpu/drm/ttm/ttm_bo_util.c | 76 | ||||
| -rw-r--r-- | drivers/gpu/drm/ttm/ttm_bo_vm.c | 374 | ||||
| -rw-r--r-- | drivers/gpu/drm/ttm/ttm_execbuf_util.c | 132 | ||||
| -rw-r--r-- | drivers/gpu/drm/ttm/ttm_lock.c | 12 | ||||
| -rw-r--r-- | drivers/gpu/drm/ttm/ttm_memory.c | 19 | ||||
| -rw-r--r-- | drivers/gpu/drm/ttm/ttm_module.c | 4 | ||||
| -rw-r--r-- | drivers/gpu/drm/ttm/ttm_object.c | 444 | ||||
| -rw-r--r-- | drivers/gpu/drm/ttm/ttm_page_alloc.c | 108 | ||||
| -rw-r--r-- | drivers/gpu/drm/ttm/ttm_page_alloc_dma.c | 123 | ||||
| -rw-r--r-- | drivers/gpu/drm/ttm/ttm_tt.c | 74 |
14 files changed, 1138 insertions, 1064 deletions
diff --git a/drivers/gpu/drm/ttm/Makefile b/drivers/gpu/drm/ttm/Makefile index b2b33dde2af..b433b9f040c 100644 --- a/drivers/gpu/drm/ttm/Makefile +++ b/drivers/gpu/drm/ttm/Makefile @@ -5,10 +5,6 @@ ccflags-y := -Iinclude/drm ttm-y := ttm_agp_backend.o ttm_memory.o ttm_tt.o ttm_bo.o \ ttm_bo_util.o ttm_bo_vm.o ttm_module.o \ ttm_object.o ttm_lock.o ttm_execbuf_util.o ttm_page_alloc.o \ - ttm_bo_manager.o - -ifeq ($(CONFIG_SWIOTLB),y) -ttm-y += ttm_page_alloc_dma.o -endif + ttm_bo_manager.o ttm_page_alloc_dma.o obj-$(CONFIG_DRM_TTM) += ttm.o diff --git a/drivers/gpu/drm/ttm/ttm_agp_backend.c b/drivers/gpu/drm/ttm/ttm_agp_backend.c index 747c1413fc9..764be36397f 100644 --- a/drivers/gpu/drm/ttm/ttm_agp_backend.c +++ b/drivers/gpu/drm/ttm/ttm_agp_backend.c @@ -29,11 +29,13 @@ * Keith Packard. */ -#include "ttm/ttm_module.h" -#include "ttm/ttm_bo_driver.h" -#include "ttm/ttm_page_alloc.h" +#define pr_fmt(fmt) "[TTM] " fmt + +#include <drm/ttm/ttm_module.h> +#include <drm/ttm/ttm_bo_driver.h> +#include <drm/ttm/ttm_page_alloc.h> #ifdef TTM_HAS_AGP -#include "ttm/ttm_placement.h" +#include <drm/ttm/ttm_placement.h> #include <linux/agp_backend.h> #include <linux/module.h> #include <linux/slab.h> @@ -74,7 +76,7 @@ static int ttm_agp_bind(struct ttm_tt *ttm, struct ttm_mem_reg *bo_mem) ret = agp_bind_memory(mem, node->start); if (ret) - printk(KERN_ERR TTM_PFX "AGP Bind memory failed.\n"); + pr_err("AGP Bind memory failed\n"); return ret; } @@ -124,6 +126,7 @@ struct ttm_tt *ttm_agp_tt_create(struct ttm_bo_device *bdev, agp_be->ttm.func = &ttm_agp_func; if (ttm_tt_init(&agp_be->ttm, bdev, size, page_flags, dummy_read_page)) { + kfree(agp_be); return NULL; } diff --git a/drivers/gpu/drm/ttm/ttm_bo.c b/drivers/gpu/drm/ttm/ttm_bo.c index 7c3a57de818..4ab9f7171c4 100644 --- a/drivers/gpu/drm/ttm/ttm_bo.c +++ b/drivers/gpu/drm/ttm/ttm_bo.c @@ -28,9 +28,11 @@ * Authors: Thomas Hellstrom <thellstrom-at-vmware-dot-com> */ -#include "ttm/ttm_module.h" -#include "ttm/ttm_bo_driver.h" -#include "ttm/ttm_placement.h" +#define pr_fmt(fmt) "[TTM] " fmt + +#include <drm/ttm/ttm_module.h> +#include <drm/ttm/ttm_bo_driver.h> +#include <drm/ttm/ttm_placement.h> #include <linux/jiffies.h> #include <linux/slab.h> #include <linux/sched.h> @@ -43,7 +45,6 @@ #define TTM_DEBUG(fmt, arg...) #define TTM_BO_HASH_ORDER 13 -static int ttm_bo_setup_vm(struct ttm_buffer_object *bo); static int ttm_bo_swapout(struct ttm_mem_shrink *shrink); static void ttm_bo_global_kobj_release(struct kobject *kobj); @@ -68,15 +69,13 @@ static void ttm_mem_type_debug(struct ttm_bo_device *bdev, int mem_type) { struct ttm_mem_type_manager *man = &bdev->man[mem_type]; - printk(KERN_ERR TTM_PFX " has_type: %d\n", man->has_type); - printk(KERN_ERR TTM_PFX " use_type: %d\n", man->use_type); - printk(KERN_ERR TTM_PFX " flags: 0x%08X\n", man->flags); - printk(KERN_ERR TTM_PFX " gpu_offset: 0x%08lX\n", man->gpu_offset); - printk(KERN_ERR TTM_PFX " size: %llu\n", man->size); - printk(KERN_ERR TTM_PFX " available_caching: 0x%08X\n", - man->available_caching); - printk(KERN_ERR TTM_PFX " default_caching: 0x%08X\n", - man->default_caching); + pr_err(" has_type: %d\n", man->has_type); + pr_err(" use_type: %d\n", man->use_type); + pr_err(" flags: 0x%08X\n", man->flags); + pr_err(" gpu_offset: 0x%08lX\n", man->gpu_offset); + pr_err(" size: %llu\n", man->size); + pr_err(" available_caching: 0x%08X\n", man->available_caching); + pr_err(" default_caching: 0x%08X\n", man->default_caching); if (mem_type != TTM_PL_SYSTEM) (*man->func->debug)(man, TTM_PFX); } @@ -86,16 +85,16 @@ static void ttm_bo_mem_space_debug(struct ttm_buffer_object *bo, { int i, ret, mem_type; - printk(KERN_ERR TTM_PFX "No space for %p (%lu pages, %luK, %luM)\n", - bo, bo->mem.num_pages, bo->mem.size >> 10, - bo->mem.size >> 20); + pr_err("No space for %p (%lu pages, %luK, %luM)\n", + bo, bo->mem.num_pages, bo->mem.size >> 10, + bo->mem.size >> 20); for (i = 0; i < placement->num_placement; i++) { ret = ttm_mem_type_from_flags(placement->placement[i], &mem_type); if (ret) return; - printk(KERN_ERR TTM_PFX " placement[%d]=0x%08X (%d)\n", - i, placement->placement[i], mem_type); + pr_err(" placement[%d]=0x%08X (%d)\n", + i, placement->placement[i], mem_type); ttm_mem_type_debug(bo->bdev, mem_type); } } @@ -150,6 +149,9 @@ static void ttm_bo_release_list(struct kref *list_kref) if (bo->ttm) ttm_tt_destroy(bo->ttm); atomic_dec(&bo->glob->bo_count); + if (bo->resv == &bo->ttm_resv) + reservation_object_fini(&bo->ttm_resv); + mutex_destroy(&bo->wu_mutex); if (bo->destroy) bo->destroy(bo); else { @@ -158,24 +160,12 @@ static void ttm_bo_release_list(struct kref *list_kref) ttm_mem_global_free(bdev->glob->mem_glob, acc_size); } -int ttm_bo_wait_unreserved(struct ttm_buffer_object *bo, bool interruptible) -{ - if (interruptible) { - return wait_event_interruptible(bo->event_queue, - atomic_read(&bo->reserved) == 0); - } else { - wait_event(bo->event_queue, atomic_read(&bo->reserved) == 0); - return 0; - } -} -EXPORT_SYMBOL(ttm_bo_wait_unreserved); - void ttm_bo_add_to_lru(struct ttm_buffer_object *bo) { struct ttm_bo_device *bdev = bo->bdev; struct ttm_mem_type_manager *man; - BUG_ON(!atomic_read(&bo->reserved)); + lockdep_assert_held(&bo->resv->lock.base); if (!(bo->mem.placement & TTM_PL_FLAG_NO_EVICT)) { @@ -191,6 +181,7 @@ void ttm_bo_add_to_lru(struct ttm_buffer_object *bo) } } } +EXPORT_SYMBOL(ttm_bo_add_to_lru); int ttm_bo_del_from_lru(struct ttm_buffer_object *bo) { @@ -213,61 +204,6 @@ int ttm_bo_del_from_lru(struct ttm_buffer_object *bo) return put_count; } -int ttm_bo_reserve_locked(struct ttm_buffer_object *bo, - bool interruptible, - bool no_wait, bool use_sequence, uint32_t sequence) -{ - struct ttm_bo_global *glob = bo->glob; - int ret; - - while (unlikely(atomic_cmpxchg(&bo->reserved, 0, 1) != 0)) { - /** - * Deadlock avoidance for multi-bo reserving. - */ - if (use_sequence && bo->seq_valid) { - /** - * We've already reserved this one. - */ - if (unlikely(sequence == bo->val_seq)) - return -EDEADLK; - /** - * Already reserved by a thread that will not back - * off for us. We need to back off. - */ - if (unlikely(sequence - bo->val_seq < (1 << 31))) - return -EAGAIN; - } - - if (no_wait) - return -EBUSY; - - spin_unlock(&glob->lru_lock); - ret = ttm_bo_wait_unreserved(bo, interruptible); - spin_lock(&glob->lru_lock); - - if (unlikely(ret)) - return ret; - } - - if (use_sequence) { - /** - * Wake up waiters that may need to recheck for deadlock, - * if we decreased the sequence number. - */ - if (unlikely((bo->val_seq - sequence < (1 << 31)) - || !bo->seq_valid)) - wake_up_all(&bo->event_queue); - - bo->val_seq = sequence; - bo->seq_valid = true; - } else { - bo->seq_valid = false; - } - - return 0; -} -EXPORT_SYMBOL(ttm_bo_reserve); - static void ttm_bo_ref_bug(struct kref *list_kref) { BUG(); @@ -280,42 +216,16 @@ void ttm_bo_list_ref_sub(struct ttm_buffer_object *bo, int count, (never_free) ? ttm_bo_ref_bug : ttm_bo_release_list); } -int ttm_bo_reserve(struct ttm_buffer_object *bo, - bool interruptible, - bool no_wait, bool use_sequence, uint32_t sequence) +void ttm_bo_del_sub_from_lru(struct ttm_buffer_object *bo) { - struct ttm_bo_global *glob = bo->glob; - int put_count = 0; - int ret; - - spin_lock(&glob->lru_lock); - ret = ttm_bo_reserve_locked(bo, interruptible, no_wait, use_sequence, - sequence); - if (likely(ret == 0)) - put_count = ttm_bo_del_from_lru(bo); - spin_unlock(&glob->lru_lock); + int put_count; + spin_lock(&bo->glob->lru_lock); + put_count = ttm_bo_del_from_lru(bo); + spin_unlock(&bo->glob->lru_lock); ttm_bo_list_ref_sub(bo, put_count, true); - - return ret; } - -void ttm_bo_unreserve_locked(struct ttm_buffer_object *bo) -{ - ttm_bo_add_to_lru(bo); - atomic_set(&bo->reserved, 0); - wake_up_all(&bo->event_queue); -} - -void ttm_bo_unreserve(struct ttm_buffer_object *bo) -{ - struct ttm_bo_global *glob = bo->glob; - - spin_lock(&glob->lru_lock); - ttm_bo_unreserve_locked(bo); - spin_unlock(&glob->lru_lock); -} -EXPORT_SYMBOL(ttm_bo_unreserve); +EXPORT_SYMBOL(ttm_bo_del_sub_from_lru); /* * Call bo->mutex locked. @@ -343,8 +253,18 @@ static int ttm_bo_add_ttm(struct ttm_buffer_object *bo, bool zero_alloc) if (unlikely(bo->ttm == NULL)) ret = -ENOMEM; break; + case ttm_bo_type_sg: + bo->ttm = bdev->driver->ttm_tt_create(bdev, bo->num_pages << PAGE_SHIFT, + page_flags | TTM_PAGE_FLAG_SG, + glob->dummy_read_page); + if (unlikely(bo->ttm == NULL)) { + ret = -ENOMEM; + break; + } + bo->ttm->sg = bo->sg; + break; default: - printk(KERN_ERR TTM_PFX "Illegal buffer object type\n"); + pr_err("Illegal buffer object type\n"); ret = -EINVAL; break; } @@ -355,7 +275,7 @@ static int ttm_bo_add_ttm(struct ttm_buffer_object *bo, bool zero_alloc) static int ttm_bo_handle_move_mem(struct ttm_buffer_object *bo, struct ttm_mem_reg *mem, bool evict, bool interruptible, - bool no_wait_reserve, bool no_wait_gpu) + bool no_wait_gpu) { struct ttm_bo_device *bdev = bo->bdev; bool old_is_pci = ttm_mem_reg_is_pci(bdev, &bo->mem); @@ -409,12 +329,12 @@ static int ttm_bo_handle_move_mem(struct ttm_buffer_object *bo, if (!(old_man->flags & TTM_MEMTYPE_FLAG_FIXED) && !(new_man->flags & TTM_MEMTYPE_FLAG_FIXED)) - ret = ttm_bo_move_ttm(bo, evict, no_wait_reserve, no_wait_gpu, mem); + ret = ttm_bo_move_ttm(bo, evict, no_wait_gpu, mem); else if (bdev->driver->move) ret = bdev->driver->move(bo, evict, interruptible, - no_wait_reserve, no_wait_gpu, mem); + no_wait_gpu, mem); else - ret = ttm_bo_move_memcpy(bo, evict, no_wait_reserve, no_wait_gpu, mem); + ret = ttm_bo_move_memcpy(bo, evict, no_wait_gpu, mem); if (ret) { if (bdev->driver->move_notify) { @@ -423,6 +343,7 @@ static int ttm_bo_handle_move_mem(struct ttm_buffer_object *bo, bo->mem = tmp_mem; bdev->driver->move_notify(bo, mem); bo->mem = *mem; + *mem = tmp_mem; } goto out_err; @@ -430,9 +351,11 @@ static int ttm_bo_handle_move_mem(struct ttm_buffer_object *bo, moved: if (bo->evicted) { - ret = bdev->driver->invalidate_caches(bdev, bo->mem.placement); - if (ret) - printk(KERN_ERR TTM_PFX "Can not flush read caches\n"); + if (bdev->driver->invalidate_caches) { + ret = bdev->driver->invalidate_caches(bdev, bo->mem.placement); + if (ret) + pr_err("Can not flush read caches\n"); + } bo->evicted = false; } @@ -476,41 +399,24 @@ static void ttm_bo_cleanup_memtype_use(struct ttm_buffer_object *bo) } ttm_bo_mem_put(bo, &bo->mem); - atomic_set(&bo->reserved, 0); - - /* - * Make processes trying to reserve really pick it up. - */ - smp_mb__after_atomic_dec(); - wake_up_all(&bo->event_queue); + ww_mutex_unlock (&bo->resv->lock); } static void ttm_bo_cleanup_refs_or_queue(struct ttm_buffer_object *bo) { struct ttm_bo_device *bdev = bo->bdev; struct ttm_bo_global *glob = bo->glob; - struct ttm_bo_driver *driver; + struct ttm_bo_driver *driver = bdev->driver; void *sync_obj = NULL; - void *sync_obj_arg; int put_count; int ret; + spin_lock(&glob->lru_lock); + ret = __ttm_bo_reserve(bo, false, true, false, 0); + spin_lock(&bdev->fence_lock); (void) ttm_bo_wait(bo, false, false, true); - if (!bo->sync_obj) { - - spin_lock(&glob->lru_lock); - - /** - * Lock inversion between bo:reserve and bdev::fence_lock here, - * but that's OK, since we're only trylocking. - */ - - ret = ttm_bo_reserve_locked(bo, false, true, false, 0); - - if (unlikely(ret == -EBUSY)) - goto queue; - + if (!ret && !bo->sync_obj) { spin_unlock(&bdev->fence_lock); put_count = ttm_bo_del_from_lru(bo); @@ -520,22 +426,32 @@ static void ttm_bo_cleanup_refs_or_queue(struct ttm_buffer_object *bo) ttm_bo_list_ref_sub(bo, put_count, true); return; - } else { - spin_lock(&glob->lru_lock); } -queue: - driver = bdev->driver; if (bo->sync_obj) sync_obj = driver->sync_obj_ref(bo->sync_obj); - sync_obj_arg = bo->sync_obj_arg; + spin_unlock(&bdev->fence_lock); + + if (!ret) { + + /* + * Make NO_EVICT bos immediately available to + * shrinkers, now that they are queued for + * destruction. + */ + if (bo->mem.placement & TTM_PL_FLAG_NO_EVICT) { + bo->mem.placement &= ~TTM_PL_FLAG_NO_EVICT; + ttm_bo_add_to_lru(bo); + } + + __ttm_bo_unreserve(bo); + } kref_get(&bo->list_kref); list_add_tail(&bo->ddestroy, &bdev->ddestroy); spin_unlock(&glob->lru_lock); - spin_unlock(&bdev->fence_lock); if (sync_obj) { - driver->sync_obj_flush(sync_obj, sync_obj_arg); + driver->sync_obj_flush(sync_obj); driver->sync_obj_unref(&sync_obj); } schedule_delayed_work(&bdev->wq, @@ -543,61 +459,82 @@ queue: } /** - * function ttm_bo_cleanup_refs + * function ttm_bo_cleanup_refs_and_unlock * If bo idle, remove from delayed- and lru lists, and unref. * If not idle, do nothing. * + * Must be called with lru_lock and reservation held, this function + * will drop both before returning. + * * @interruptible Any sleeps should occur interruptibly. - * @no_wait_reserve Never wait for reserve. Return -EBUSY instead. * @no_wait_gpu Never wait for gpu. Return -EBUSY instead. */ -static int ttm_bo_cleanup_refs(struct ttm_buffer_object *bo, - bool interruptible, - bool no_wait_reserve, - bool no_wait_gpu) +static int ttm_bo_cleanup_refs_and_unlock(struct ttm_buffer_object *bo, + bool interruptible, + bool no_wait_gpu) { struct ttm_bo_device *bdev = bo->bdev; + struct ttm_bo_driver *driver = bdev->driver; struct ttm_bo_global *glob = bo->glob; int put_count; - int ret = 0; + int ret; -retry: spin_lock(&bdev->fence_lock); - ret = ttm_bo_wait(bo, false, interruptible, no_wait_gpu); - spin_unlock(&bdev->fence_lock); + ret = ttm_bo_wait(bo, false, false, true); - if (unlikely(ret != 0)) - return ret; + if (ret && !no_wait_gpu) { + void *sync_obj; - spin_lock(&glob->lru_lock); + /* + * Take a reference to the fence and unreserve, + * at this point the buffer should be dead, so + * no new sync objects can be attached. + */ + sync_obj = driver->sync_obj_ref(bo->sync_obj); + spin_unlock(&bdev->fence_lock); - if (unlikely(list_empty(&bo->ddestroy))) { + __ttm_bo_unreserve(bo); spin_unlock(&glob->lru_lock); - return 0; - } - ret = ttm_bo_reserve_locked(bo, interruptible, - no_wait_reserve, false, 0); + ret = driver->sync_obj_wait(sync_obj, false, interruptible); + driver->sync_obj_unref(&sync_obj); + if (ret) + return ret; - if (unlikely(ret != 0)) { - spin_unlock(&glob->lru_lock); - return ret; - } + /* + * remove sync_obj with ttm_bo_wait, the wait should be + * finished, and no new wait object should have been added. + */ + spin_lock(&bdev->fence_lock); + ret = ttm_bo_wait(bo, false, false, true); + WARN_ON(ret); + spin_unlock(&bdev->fence_lock); + if (ret) + return ret; - /** - * We can re-check for sync object without taking - * the bo::lock since setting the sync object requires - * also bo::reserved. A busy object at this point may - * be caused by another thread recently starting an accelerated - * eviction. - */ + spin_lock(&glob->lru_lock); + ret = __ttm_bo_reserve(bo, false, true, false, 0); - if (unlikely(bo->sync_obj)) { - atomic_set(&bo->reserved, 0); - wake_up_all(&bo->event_queue); + /* + * We raced, and lost, someone else holds the reservation now, + * and is probably busy in ttm_bo_cleanup_memtype_use. + * + * Even if it's not the case, because we finished waiting any + * delayed destruction would succeed, so just return success + * here. + */ + if (ret) { + spin_unlock(&glob->lru_lock); + return 0; + } + } else + spin_unlock(&bdev->fence_lock); + + if (ret || unlikely(list_empty(&bo->ddestroy))) { + __ttm_bo_unreserve(bo); spin_unlock(&glob->lru_lock); - goto retry; + return ret; } put_count = ttm_bo_del_from_lru(bo); @@ -640,9 +577,20 @@ static int ttm_bo_delayed_delete(struct ttm_bo_device *bdev, bool remove_all) kref_get(&nentry->list_kref); } - spin_unlock(&glob->lru_lock); - ret = ttm_bo_cleanup_refs(entry, false, !remove_all, - !remove_all); + ret = __ttm_bo_reserve(entry, false, true, false, 0); + if (remove_all && ret) { + spin_unlock(&glob->lru_lock); + ret = __ttm_bo_reserve(entry, false, false, + false, 0); + spin_lock(&glob->lru_lock); + } + + if (!ret) + ret = ttm_bo_cleanup_refs_and_unlock(entry, false, + !remove_all); + else + spin_unlock(&glob->lru_lock); + kref_put(&entry->list_kref, ttm_bo_release_list); entry = nentry; @@ -680,29 +628,20 @@ static void ttm_bo_release(struct kref *kref) struct ttm_bo_device *bdev = bo->bdev; struct ttm_mem_type_manager *man = &bdev->man[bo->mem.mem_type]; - if (likely(bo->vm_node != NULL)) { - rb_erase(&bo->vm_rb, &bdev->addr_space_rb); - drm_mm_put_block(bo->vm_node); - bo->vm_node = NULL; - } - write_unlock(&bdev->vm_lock); + drm_vma_offset_remove(&bdev->vma_manager, &bo->vma_node); ttm_mem_io_lock(man, false); ttm_mem_io_free_vm(bo); ttm_mem_io_unlock(man); ttm_bo_cleanup_refs_or_queue(bo); kref_put(&bo->list_kref, ttm_bo_release_list); - write_lock(&bdev->vm_lock); } void ttm_bo_unref(struct ttm_buffer_object **p_bo) { struct ttm_buffer_object *bo = *p_bo; - struct ttm_bo_device *bdev = bo->bdev; *p_bo = NULL; - write_lock(&bdev->vm_lock); kref_put(&bo->kref, ttm_bo_release); - write_unlock(&bdev->vm_lock); } EXPORT_SYMBOL(ttm_bo_unref); @@ -721,7 +660,7 @@ void ttm_bo_unlock_delayed_workqueue(struct ttm_bo_device *bdev, int resched) EXPORT_SYMBOL(ttm_bo_unlock_delayed_workqueue); static int ttm_bo_evict(struct ttm_buffer_object *bo, bool interruptible, - bool no_wait_reserve, bool no_wait_gpu) + bool no_wait_gpu) { struct ttm_bo_device *bdev = bo->bdev; struct ttm_mem_reg evict_mem; @@ -734,14 +673,12 @@ static int ttm_bo_evict(struct ttm_buffer_object *bo, bool interruptible, if (unlikely(ret != 0)) { if (ret != -ERESTARTSYS) { - printk(KERN_ERR TTM_PFX - "Failed to expire sync object before " - "buffer eviction.\n"); + pr_err("Failed to expire sync object before buffer eviction\n"); } goto out; } - BUG_ON(!atomic_read(&bo->reserved)); + lockdep_assert_held(&bo->resv->lock.base); evict_mem = bo->mem; evict_mem.mm_node = NULL; @@ -754,22 +691,21 @@ static int ttm_bo_evict(struct ttm_buffer_object *bo, bool interruptible, placement.num_busy_placement = 0; bdev->driver->evict_flags(bo, &placement); ret = ttm_bo_mem_space(bo, &placement, &evict_mem, interruptible, - no_wait_reserve, no_wait_gpu); + no_wait_gpu); if (ret) { if (ret != -ERESTARTSYS) { - printk(KERN_ERR TTM_PFX - "Failed to find memory space for " - "buffer 0x%p eviction.\n", bo); + pr_err("Failed to find memory space for buffer 0x%p eviction\n", + bo); ttm_bo_mem_space_debug(bo, &placement); } goto out; } ret = ttm_bo_handle_move_mem(bo, &evict_mem, true, interruptible, - no_wait_reserve, no_wait_gpu); + no_wait_gpu); if (ret) { if (ret != -ERESTARTSYS) - printk(KERN_ERR TTM_PFX "Buffer eviction failed\n"); + pr_err("Buffer eviction failed\n"); ttm_bo_mem_put(bo, &evict_mem); goto out; } @@ -780,52 +716,33 @@ out: static int ttm_mem_evict_first(struct ttm_bo_device *bdev, uint32_t mem_type, - bool interruptible, bool no_wait_reserve, + bool interruptible, bool no_wait_gpu) { struct ttm_bo_global *glob = bdev->glob; struct ttm_mem_type_manager *man = &bdev->man[mem_type]; struct ttm_buffer_object *bo; - int ret, put_count = 0; + int ret = -EBUSY, put_count; -retry: spin_lock(&glob->lru_lock); - if (list_empty(&man->lru)) { - spin_unlock(&glob->lru_lock); - return -EBUSY; + list_for_each_entry(bo, &man->lru, lru) { + ret = __ttm_bo_reserve(bo, false, true, false, 0); + if (!ret) + break; } - bo = list_first_entry(&man->lru, struct ttm_buffer_object, lru); - kref_get(&bo->list_kref); - - if (!list_empty(&bo->ddestroy)) { + if (ret) { spin_unlock(&glob->lru_lock); - ret = ttm_bo_cleanup_refs(bo, interruptible, - no_wait_reserve, no_wait_gpu); - kref_put(&bo->list_kref, ttm_bo_release_list); - - if (likely(ret == 0 || ret == -ERESTARTSYS)) - return ret; - - goto retry; + return ret; } - ret = ttm_bo_reserve_locked(bo, false, no_wait_reserve, false, 0); - - if (unlikely(ret == -EBUSY)) { - spin_unlock(&glob->lru_lock); - if (likely(!no_wait_gpu)) - ret = ttm_bo_wait_unreserved(bo, interruptible); + kref_get(&bo->list_kref); + if (!list_empty(&bo->ddestroy)) { + ret = ttm_bo_cleanup_refs_and_unlock(bo, interruptible, + no_wait_gpu); kref_put(&bo->list_kref, ttm_bo_release_list); - - /** - * We *need* to retry after releasing the lru lock. - */ - - if (unlikely(ret != 0)) - return ret; - goto retry; + return ret; } put_count = ttm_bo_del_from_lru(bo); @@ -835,7 +752,7 @@ retry: ttm_bo_list_ref_sub(bo, put_count, true); - ret = ttm_bo_evict(bo, interruptible, no_wait_reserve, no_wait_gpu); + ret = ttm_bo_evict(bo, interruptible, no_wait_gpu); ttm_bo_unreserve(bo); kref_put(&bo->list_kref, ttm_bo_release_list); @@ -860,7 +777,6 @@ static int ttm_bo_mem_force_space(struct ttm_buffer_object *bo, struct ttm_placement *placement, struct ttm_mem_reg *mem, bool interruptible, - bool no_wait_reserve, bool no_wait_gpu) { struct ttm_bo_device *bdev = bo->bdev; @@ -873,8 +789,8 @@ static int ttm_bo_mem_force_space(struct ttm_buffer_object *bo, return ret; if (mem->mm_node) break; - ret = ttm_mem_evict_first(bdev, mem_type, interruptible, - no_wait_reserve, no_wait_gpu); + ret = ttm_mem_evict_first(bdev, mem_type, + interruptible, no_wait_gpu); if (unlikely(ret != 0)) return ret; } while (1); @@ -939,7 +855,7 @@ static bool ttm_bo_mt_compatible(struct ttm_mem_type_manager *man, int ttm_bo_mem_space(struct ttm_buffer_object *bo, struct ttm_placement *placement, struct ttm_mem_reg *mem, - bool interruptible, bool no_wait_reserve, + bool interruptible, bool no_wait_gpu) { struct ttm_bo_device *bdev = bo->bdev; @@ -1030,7 +946,7 @@ int ttm_bo_mem_space(struct ttm_buffer_object *bo, } ret = ttm_bo_mem_force_space(bo, mem_type, placement, mem, - interruptible, no_wait_reserve, no_wait_gpu); + interruptible, no_wait_gpu); if (ret == 0 && mem->mm_node) { mem->placement = cur_flags; return 0; @@ -1043,26 +959,16 @@ int ttm_bo_mem_space(struct ttm_buffer_object *bo, } EXPORT_SYMBOL(ttm_bo_mem_space); -int ttm_bo_wait_cpu(struct ttm_buffer_object *bo, bool no_wait) -{ - if ((atomic_read(&bo->cpu_writers) > 0) && no_wait) - return -EBUSY; - - return wait_event_interruptible(bo->event_queue, - atomic_read(&bo->cpu_writers) == 0); -} -EXPORT_SYMBOL(ttm_bo_wait_cpu); - -int ttm_bo_move_buffer(struct ttm_buffer_object *bo, +static int ttm_bo_move_buffer(struct ttm_buffer_object *bo, struct ttm_placement *placement, - bool interruptible, bool no_wait_reserve, + bool interruptible, bool no_wait_gpu) { int ret = 0; struct ttm_mem_reg mem; struct ttm_bo_device *bdev = bo->bdev; - BUG_ON(!atomic_read(&bo->reserved)); + lockdep_assert_held(&bo->resv->lock.base); /* * FIXME: It's possible to pipeline buffer moves. @@ -1082,44 +988,55 @@ int ttm_bo_move_buffer(struct ttm_buffer_object *bo, /* * Determine where to move the buffer. */ - ret = ttm_bo_mem_space(bo, placement, &mem, interruptible, no_wait_reserve, no_wait_gpu); + ret = ttm_bo_mem_space(bo, placement, &mem, + interruptible, no_wait_gpu); if (ret) goto out_unlock; - ret = ttm_bo_handle_move_mem(bo, &mem, false, interruptible, no_wait_reserve, no_wait_gpu); + ret = ttm_bo_handle_move_mem(bo, &mem, false, + interruptible, no_wait_gpu); out_unlock: if (ret && mem.mm_node) ttm_bo_mem_put(bo, &mem); return ret; } -static int ttm_bo_mem_compat(struct ttm_placement *placement, - struct ttm_mem_reg *mem) +static bool ttm_bo_mem_compat(struct ttm_placement *placement, + struct ttm_mem_reg *mem, + uint32_t *new_flags) { int i; if (mem->mm_node && placement->lpfn != 0 && (mem->start < placement->fpfn || mem->start + mem->num_pages > placement->lpfn)) - return -1; + return false; for (i = 0; i < placement->num_placement; i++) { - if ((placement->placement[i] & mem->placement & - TTM_PL_MASK_CACHING) && - (placement->placement[i] & mem->placement & - TTM_PL_MASK_MEM)) - return i; + *new_flags = placement->placement[i]; + if ((*new_flags & mem->placement & TTM_PL_MASK_CACHING) && + (*new_flags & mem->placement & TTM_PL_MASK_MEM)) + return true; + } + + for (i = 0; i < placement->num_busy_placement; i++) { + *new_flags = placement->busy_placement[i]; + if ((*new_flags & mem->placement & TTM_PL_MASK_CACHING) && + (*new_flags & mem->placement & TTM_PL_MASK_MEM)) + return true; } - return -1; + + return false; } int ttm_bo_validate(struct ttm_buffer_object *bo, struct ttm_placement *placement, - bool interruptible, bool no_wait_reserve, + bool interruptible, bool no_wait_gpu) { int ret; + uint32_t new_flags; - BUG_ON(!atomic_read(&bo->reserved)); + lockdep_assert_held(&bo->resv->lock.base); /* Check that range is valid */ if (placement->lpfn || placement->fpfn) if (placement->fpfn > placement->lpfn || @@ -1128,9 +1045,9 @@ int ttm_bo_validate(struct ttm_buffer_object *bo, /* * Check whether we need to move buffer. */ - ret = ttm_bo_mem_compat(placement, &bo->mem); - if (ret < 0) { - ret = ttm_bo_move_buffer(bo, placement, interruptible, no_wait_reserve, no_wait_gpu); + if (!ttm_bo_mem_compat(placement, &bo->mem, &new_flags)) { + ret = ttm_bo_move_buffer(bo, placement, interruptible, + no_wait_gpu); if (ret) return ret; } else { @@ -1138,7 +1055,7 @@ int ttm_bo_validate(struct ttm_buffer_object *bo, * Use the access and other non-mapping-related flag bits from * the compatible memory placement flags to the active flags */ - ttm_flag_masked(&bo->mem.placement, placement->placement[ret], + ttm_flag_masked(&bo->mem.placement, new_flags, ~TTM_PL_MASK_MEMTYPE); } /* @@ -1168,19 +1085,20 @@ int ttm_bo_init(struct ttm_bo_device *bdev, enum ttm_bo_type type, struct ttm_placement *placement, uint32_t page_alignment, - unsigned long buffer_start, bool interruptible, struct file *persistent_swap_storage, size_t acc_size, + struct sg_table *sg, void (*destroy) (struct ttm_buffer_object *)) { int ret = 0; unsigned long num_pages; struct ttm_mem_global *mem_glob = bdev->glob->mem_glob; + bool locked; ret = ttm_mem_global_alloc(mem_glob, acc_size, false, false); if (ret) { - printk(KERN_ERR TTM_PFX "Out of kernel memory.\n"); + pr_err("Out of kernel memory\n"); if (destroy) (*destroy)(bo); else @@ -1188,14 +1106,14 @@ int ttm_bo_init(struct ttm_bo_device *bdev, return -ENOMEM; } - size += buffer_start & ~PAGE_MASK; num_pages = (size + PAGE_SIZE - 1) >> PAGE_SHIFT; if (num_pages == 0) { - printk(KERN_ERR TTM_PFX "Illegal buffer object size.\n"); + pr_err("Illegal buffer object size\n"); if (destroy) (*destroy)(bo); else kfree(bo); + ttm_mem_global_free(mem_glob, acc_size); return -EINVAL; } bo->destroy = destroy; @@ -1203,12 +1121,11 @@ int ttm_bo_init(struct ttm_bo_device *bdev, kref_init(&bo->kref); kref_init(&bo->list_kref); atomic_set(&bo->cpu_writers, 0); - atomic_set(&bo->reserved, 1); - init_waitqueue_head(&bo->event_queue); INIT_LIST_HEAD(&bo->lru); INIT_LIST_HEAD(&bo->ddestroy); INIT_LIST_HEAD(&bo->swap); INIT_LIST_HEAD(&bo->io_reserve_lru); + mutex_init(&bo->wu_mutex); bo->bdev = bdev; bo->glob = bdev->glob; bo->type = type; @@ -1220,38 +1137,38 @@ int ttm_bo_init(struct ttm_bo_device *bdev, bo->mem.page_alignment = page_alignment; bo->mem.bus.io_reserved_vm = false; bo->mem.bus.io_reserved_count = 0; - bo->buffer_start = buffer_start & PAGE_MASK; bo->priv_flags = 0; bo->mem.placement = (TTM_PL_FLAG_SYSTEM | TTM_PL_FLAG_CACHED); - bo->seq_valid = false; bo->persistent_swap_storage = persistent_swap_storage; bo->acc_size = acc_size; + bo->sg = sg; + bo->resv = &bo->ttm_resv; + reservation_object_init(bo->resv); atomic_inc(&bo->glob->bo_count); + drm_vma_node_reset(&bo->vma_node); ret = ttm_bo_check_placement(bo, placement); - if (unlikely(ret != 0)) - goto out_err; /* * For ttm_bo_type_device buffers, allocate * address space from the device. */ - if (bo->type == ttm_bo_type_device) { - ret = ttm_bo_setup_vm(bo); - if (ret) - goto out_err; - } + if (likely(!ret) && + (bo->type == ttm_bo_type_device || + bo->type == ttm_bo_type_sg)) + ret = drm_vma_offset_add(&bdev->vma_manager, &bo->vma_node, + bo->mem.num_pages); - ret = ttm_bo_validate(bo, placement, interruptible, false, false); - if (ret) - goto out_err; + locked = ww_mutex_trylock(&bo->resv->lock); + WARN_ON(!locked); - ttm_bo_unreserve(bo); - return 0; + if (likely(!ret)) + ret = ttm_bo_validate(bo, placement, interruptible, false); -out_err: ttm_bo_unreserve(bo); - ttm_bo_unref(&bo); + + if (unlikely(ret)) + ttm_bo_unref(&bo); return ret; } @@ -1291,31 +1208,22 @@ int ttm_bo_create(struct ttm_bo_device *bdev, enum ttm_bo_type type, struct ttm_placement *placement, uint32_t page_alignment, - unsigned long buffer_start, bool interruptible, struct file *persistent_swap_storage, struct ttm_buffer_object **p_bo) { struct ttm_buffer_object *bo; - struct ttm_mem_global *mem_glob = bdev->glob->mem_glob; size_t acc_size; int ret; - acc_size = ttm_bo_acc_size(bdev, size, sizeof(struct ttm_buffer_object)); - ret = ttm_mem_global_alloc(mem_glob, acc_size, false, false); - if (unlikely(ret != 0)) - return ret; - bo = kzalloc(sizeof(*bo), GFP_KERNEL); - - if (unlikely(bo == NULL)) { - ttm_mem_global_free(mem_glob, acc_size); + if (unlikely(bo == NULL)) return -ENOMEM; - } + acc_size = ttm_bo_acc_size(bdev, size, sizeof(struct ttm_buffer_object)); ret = ttm_bo_init(bdev, bo, size, type, placement, page_alignment, - buffer_start, interruptible, - persistent_swap_storage, acc_size, NULL); + interruptible, persistent_swap_storage, acc_size, + NULL, NULL); if (likely(ret == 0)) *p_bo = bo; @@ -1337,13 +1245,12 @@ static int ttm_bo_force_list_clean(struct ttm_bo_device *bdev, spin_lock(&glob->lru_lock); while (!list_empty(&man->lru)) { spin_unlock(&glob->lru_lock); - ret = ttm_mem_evict_first(bdev, mem_type, false, false, false); + ret = ttm_mem_evict_first(bdev, mem_type, false, false); if (ret) { if (allow_errors) { return ret; } else { - printk(KERN_ERR TTM_PFX - "Cleanup eviction failed\n"); + pr_err("Cleanup eviction failed\n"); } } spin_lock(&glob->lru_lock); @@ -1358,14 +1265,14 @@ int ttm_bo_clean_mm(struct ttm_bo_device *bdev, unsigned mem_type) int ret = -EINVAL; if (mem_type >= TTM_NUM_MEM_TYPES) { - printk(KERN_ERR TTM_PFX "Illegal memory type %d\n", mem_type); + pr_err("Illegal memory type %d\n", mem_type); return ret; } man = &bdev->man[mem_type]; if (!man->has_type) { - printk(KERN_ERR TTM_PFX "Trying to take down uninitialized " - "memory manager type %u\n", mem_type); + pr_err("Trying to take down uninitialized memory manager type %u\n", + mem_type); return ret; } @@ -1388,16 +1295,12 @@ int ttm_bo_evict_mm(struct ttm_bo_device *bdev, unsigned mem_type) struct ttm_mem_type_manager *man = &bdev->man[mem_type]; if (mem_type == 0 || mem_type >= TTM_NUM_MEM_TYPES) { - printk(KERN_ERR TTM_PFX - "Illegal memory manager memory type %u.\n", - mem_type); + pr_err("Illegal memory manager memory type %u\n", mem_type); return -EINVAL; } if (!man->has_type) { - printk(KERN_ERR TTM_PFX - "Memory type %u has not been initialized.\n", - mem_type); + pr_err("Memory type %u has not been initialized\n", mem_type); return 0; } @@ -1482,8 +1385,7 @@ int ttm_bo_global_init(struct drm_global_reference *ref) ttm_mem_init_shrink(&glob->shrink, ttm_bo_swapout); ret = ttm_mem_register_shrink(glob->mem_glob, &glob->shrink); if (unlikely(ret != 0)) { - printk(KERN_ERR TTM_PFX - "Could not register buffer object swapout.\n"); + pr_err("Could not register buffer object swapout\n"); goto out_no_shrink; } @@ -1516,9 +1418,8 @@ int ttm_bo_device_release(struct ttm_bo_device *bdev) man->use_type = false; if ((i != TTM_PL_SYSTEM) && ttm_bo_clean_mm(bdev, i)) { ret = -EBUSY; - printk(KERN_ERR TTM_PFX - "DRM memory manager type %d " - "is not clean.\n", i); + pr_err("DRM memory manager type %d is not clean\n", + i); } man->has_type = false; } @@ -1541,10 +1442,7 @@ int ttm_bo_device_release(struct ttm_bo_device *bdev) TTM_DEBUG("Swap list was clean\n"); spin_unlock(&glob->lru_lock); - BUG_ON(!drm_mm_clean(&bdev->addr_space_mm)); - write_lock(&bdev->vm_lock); - drm_mm_takedown(&bdev->addr_space_mm); - write_unlock(&bdev->vm_lock); + drm_vma_offset_manager_destroy(&bdev->vma_manager); return ret; } @@ -1553,12 +1451,12 @@ EXPORT_SYMBOL(ttm_bo_device_release); int ttm_bo_device_init(struct ttm_bo_device *bdev, struct ttm_bo_global *glob, struct ttm_bo_driver *driver, + struct address_space *mapping, uint64_t file_page_offset, bool need_dma32) { int ret = -EINVAL; - rwlock_init(&bdev->vm_lock); bdev->driver = driver; memset(bdev->man, 0, sizeof(bdev->man)); @@ -1571,15 +1469,11 @@ int ttm_bo_device_init(struct ttm_bo_device *bdev, if (unlikely(ret != 0)) goto out_no_sys; - bdev->addr_space_rb = RB_ROOT; - ret = drm_mm_init(&bdev->addr_space_mm, file_page_offset, 0x10000000); - if (unlikely(ret != 0)) - goto out_no_addr_mm; - + drm_vma_offset_manager_init(&bdev->vma_manager, file_page_offset, + 0x10000000); INIT_DELAYED_WORK(&bdev->wq, ttm_bo_delayed_workqueue); - bdev->nice_mode = true; INIT_LIST_HEAD(&bdev->ddestroy); - bdev->dev_mapping = NULL; + bdev->dev_mapping = mapping; bdev->glob = glob; bdev->need_dma32 = need_dma32; bdev->val_seq = 0; @@ -1589,8 +1483,6 @@ int ttm_bo_device_init(struct ttm_bo_device *bdev, mutex_unlock(&glob->device_list_mutex); return 0; -out_no_addr_mm: - ttm_bo_clean_mm(bdev, 0); out_no_sys: return ret; } @@ -1620,12 +1512,8 @@ bool ttm_mem_reg_is_pci(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem) void ttm_bo_unmap_virtual_locked(struct ttm_buffer_object *bo) { struct ttm_bo_device *bdev = bo->bdev; - loff_t offset = (loff_t) bo->addr_space_offset; - loff_t holelen = ((loff_t) bo->mem.num_pages) << PAGE_SHIFT; - if (!bdev->dev_mapping) - return; - unmap_mapping_range(bdev->dev_mapping, offset, holelen, 1); + drm_vma_node_unmap(&bo->vma_node, bdev->dev_mapping); ttm_mem_io_free_vm(bo); } @@ -1642,78 +1530,6 @@ void ttm_bo_unmap_virtual(struct ttm_buffer_object *bo) EXPORT_SYMBOL(ttm_bo_unmap_virtual); -static void ttm_bo_vm_insert_rb(struct ttm_buffer_object *bo) -{ - struct ttm_bo_device *bdev = bo->bdev; - struct rb_node **cur = &bdev->addr_space_rb.rb_node; - struct rb_node *parent = NULL; - struct ttm_buffer_object *cur_bo; - unsigned long offset = bo->vm_node->start; - unsigned long cur_offset; - - while (*cur) { - parent = *cur; - cur_bo = rb_entry(parent, struct ttm_buffer_object, vm_rb); - cur_offset = cur_bo->vm_node->start; - if (offset < cur_offset) - cur = &parent->rb_left; - else if (offset > cur_offset) - cur = &parent->rb_right; - else - BUG(); - } - - rb_link_node(&bo->vm_rb, parent, cur); - rb_insert_color(&bo->vm_rb, &bdev->addr_space_rb); -} - -/** - * ttm_bo_setup_vm: - * - * @bo: the buffer to allocate address space for - * - * Allocate address space in the drm device so that applications - * can mmap the buffer and access the contents. This only - * applies to ttm_bo_type_device objects as others are not - * placed in the drm device address space. - */ - -static int ttm_bo_setup_vm(struct ttm_buffer_object *bo) -{ - struct ttm_bo_device *bdev = bo->bdev; - int ret; - -retry_pre_get: - ret = drm_mm_pre_get(&bdev->addr_space_mm); - if (unlikely(ret != 0)) - return ret; - - write_lock(&bdev->vm_lock); - bo->vm_node = drm_mm_search_free(&bdev->addr_space_mm, - bo->mem.num_pages, 0, 0); - - if (unlikely(bo->vm_node == NULL)) { - ret = -ENOMEM; - goto out_unlock; - } - - bo->vm_node = drm_mm_get_block_atomic(bo->vm_node, - bo->mem.num_pages, 0); - - if (unlikely(bo->vm_node == NULL)) { - write_unlock(&bdev->vm_lock); - goto retry_pre_get; - } - - ttm_bo_vm_insert_rb(bo); - write_unlock(&bdev->vm_lock); - bo->addr_space_offset = ((uint64_t) bo->vm_node->start) << PAGE_SHIFT; - - return 0; -out_unlock: - write_unlock(&bdev->vm_lock); - return ret; -} int ttm_bo_wait(struct ttm_buffer_object *bo, bool lazy, bool interruptible, bool no_wait) @@ -1721,7 +1537,6 @@ int ttm_bo_wait(struct ttm_buffer_object *bo, struct ttm_bo_driver *driver = bo->bdev->driver; struct ttm_bo_device *bdev = bo->bdev; void *sync_obj; - void *sync_obj_arg; int ret = 0; if (likely(bo->sync_obj == NULL)) @@ -1729,7 +1544,7 @@ int ttm_bo_wait(struct ttm_buffer_object *bo, while (bo->sync_obj) { - if (driver->sync_obj_signaled(bo->sync_obj, bo->sync_obj_arg)) { + if (driver->sync_obj_signaled(bo->sync_obj)) { void *tmp_obj = bo->sync_obj; bo->sync_obj = NULL; clear_bit(TTM_BO_PRIV_FLAG_MOVING, &bo->priv_flags); @@ -1743,9 +1558,8 @@ int ttm_bo_wait(struct ttm_buffer_object *bo, return -EBUSY; sync_obj = driver->sync_obj_ref(bo->sync_obj); - sync_obj_arg = bo->sync_obj_arg; spin_unlock(&bdev->fence_lock); - ret = driver->sync_obj_wait(sync_obj, sync_obj_arg, + ret = driver->sync_obj_wait(sync_obj, lazy, interruptible); if (unlikely(ret != 0)) { driver->sync_obj_unref(&sync_obj); @@ -1753,8 +1567,7 @@ int ttm_bo_wait(struct ttm_buffer_object *bo, return ret; } spin_lock(&bdev->fence_lock); - if (likely(bo->sync_obj == sync_obj && - bo->sync_obj_arg == sync_obj_arg)) { + if (likely(bo->sync_obj == sync_obj)) { void *tmp_obj = bo->sync_obj; bo->sync_obj = NULL; clear_bit(TTM_BO_PRIV_FLAG_MOVING, @@ -1797,8 +1610,7 @@ EXPORT_SYMBOL(ttm_bo_synccpu_write_grab); void ttm_bo_synccpu_write_release(struct ttm_buffer_object *bo) { - if (atomic_dec_and_test(&bo->cpu_writers)) - wake_up_all(&bo->event_queue); + atomic_dec(&bo->cpu_writers); } EXPORT_SYMBOL(ttm_bo_synccpu_write_release); @@ -1817,39 +1629,25 @@ static int ttm_bo_swapout(struct ttm_mem_shrink *shrink) uint32_t swap_placement = (TTM_PL_FLAG_CACHED | TTM_PL_FLAG_SYSTEM); spin_lock(&glob->lru_lock); - while (ret == -EBUSY) { - if (unlikely(list_empty(&glob->swap_lru))) { - spin_unlock(&glob->lru_lock); - return -EBUSY; - } - - bo = list_first_entry(&glob->swap_lru, - struct ttm_buffer_object, swap); - kref_get(&bo->list_kref); + list_for_each_entry(bo, &glob->swap_lru, swap) { + ret = __ttm_bo_reserve(bo, false, true, false, 0); + if (!ret) + break; + } - if (!list_empty(&bo->ddestroy)) { - spin_unlock(&glob->lru_lock); - (void) ttm_bo_cleanup_refs(bo, false, false, false); - kref_put(&bo->list_kref, ttm_bo_release_list); - continue; - } + if (ret) { + spin_unlock(&glob->lru_lock); + return ret; + } - /** - * Reserve buffer. Since we unlock while sleeping, we need - * to re-check that nobody removed us from the swap-list while - * we slept. - */ + kref_get(&bo->list_kref); - ret = ttm_bo_reserve_locked(bo, false, true, false, 0); - if (unlikely(ret == -EBUSY)) { - spin_unlock(&glob->lru_lock); - ttm_bo_wait_unreserved(bo, false); - kref_put(&bo->list_kref, ttm_bo_release_list); - spin_lock(&glob->lru_lock); - } + if (!list_empty(&bo->ddestroy)) { + ret = ttm_bo_cleanup_refs_and_unlock(bo, false, false); + kref_put(&bo->list_kref, ttm_bo_release_list); + return ret; } - BUG_ON(ret != 0); put_count = ttm_bo_del_from_lru(bo); spin_unlock(&glob->lru_lock); @@ -1875,7 +1673,7 @@ static int ttm_bo_swapout(struct ttm_mem_shrink *shrink) evict_mem.mem_type = TTM_PL_SYSTEM; ret = ttm_bo_handle_move_mem(bo, &evict_mem, true, - false, false, false); + false, false); if (unlikely(ret != 0)) goto out; } @@ -1899,8 +1697,7 @@ out: * already swapped buffer. */ - atomic_set(&bo->reserved, 0); - wake_up_all(&bo->event_queue); + __ttm_bo_unreserve(bo); kref_put(&bo->list_kref, ttm_bo_release_list); return ret; } @@ -1911,3 +1708,35 @@ void ttm_bo_swapout_all(struct ttm_bo_device *bdev) ; } EXPORT_SYMBOL(ttm_bo_swapout_all); + +/** + * ttm_bo_wait_unreserved - interruptible wait for a buffer object to become + * unreserved + * + * @bo: Pointer to buffer + */ +int ttm_bo_wait_unreserved(struct ttm_buffer_object *bo) +{ + int ret; + + /* + * In the absense of a wait_unlocked API, + * Use the bo::wu_mutex to avoid triggering livelocks due to + * concurrent use of this function. Note that this use of + * bo::wu_mutex can go away if we change locking order to + * mmap_sem -> bo::reserve. + */ + ret = mutex_lock_interruptible(&bo->wu_mutex); + if (unlikely(ret != 0)) + return -ERESTARTSYS; + if (!ww_mutex_is_locked(&bo->resv->lock)) + goto out_unlock; + ret = __ttm_bo_reserve(bo, true, false, false, NULL); + if (unlikely(ret != 0)) + goto out_unlock; + __ttm_bo_unreserve(bo); + +out_unlock: + mutex_unlock(&bo->wu_mutex); + return ret; +} diff --git a/drivers/gpu/drm/ttm/ttm_bo_manager.c b/drivers/gpu/drm/ttm/ttm_bo_manager.c index 038e947d00f..bd850c9f4bc 100644 --- a/drivers/gpu/drm/ttm/ttm_bo_manager.c +++ b/drivers/gpu/drm/ttm/ttm_bo_manager.c @@ -28,10 +28,10 @@ * Authors: Thomas Hellstrom <thellstrom-at-vmware-dot-com> */ -#include "ttm/ttm_module.h" -#include "ttm/ttm_bo_driver.h" -#include "ttm/ttm_placement.h" -#include "drm_mm.h" +#include <drm/ttm/ttm_module.h> +#include <drm/ttm/ttm_bo_driver.h> +#include <drm/ttm/ttm_placement.h> +#include <drm/drm_mm.h> #include <linux/slab.h> #include <linux/spinlock.h> #include <linux/module.h> @@ -55,34 +55,36 @@ static int ttm_bo_man_get_node(struct ttm_mem_type_manager *man, struct ttm_range_manager *rman = (struct ttm_range_manager *) man->priv; struct drm_mm *mm = &rman->mm; struct drm_mm_node *node = NULL; + enum drm_mm_allocator_flags aflags = DRM_MM_CREATE_DEFAULT; unsigned long lpfn; int ret; lpfn = placement->lpfn; if (!lpfn) lpfn = man->size; - do { - ret = drm_mm_pre_get(mm); - if (unlikely(ret)) - return ret; - spin_lock(&rman->lock); - node = drm_mm_search_free_in_range(mm, - mem->num_pages, mem->page_alignment, - placement->fpfn, lpfn, 1); - if (unlikely(node == NULL)) { - spin_unlock(&rman->lock); - return 0; - } - node = drm_mm_get_block_atomic_range(node, mem->num_pages, - mem->page_alignment, - placement->fpfn, - lpfn); - spin_unlock(&rman->lock); - } while (node == NULL); + node = kzalloc(sizeof(*node), GFP_KERNEL); + if (!node) + return -ENOMEM; + + if (bo->mem.placement & TTM_PL_FLAG_TOPDOWN) + aflags = DRM_MM_CREATE_TOP; + + spin_lock(&rman->lock); + ret = drm_mm_insert_node_in_range_generic(mm, node, mem->num_pages, + mem->page_alignment, 0, + placement->fpfn, lpfn, + DRM_MM_SEARCH_BEST, + aflags); + spin_unlock(&rman->lock); + + if (unlikely(ret)) { + kfree(node); + } else { + mem->mm_node = node; + mem->start = node->start; + } - mem->mm_node = node; - mem->start = node->start; return 0; } @@ -93,8 +95,10 @@ static void ttm_bo_man_put_node(struct ttm_mem_type_manager *man, if (mem->mm_node) { spin_lock(&rman->lock); - drm_mm_put_block(mem->mm_node); + drm_mm_remove_node(mem->mm_node); spin_unlock(&rman->lock); + + kfree(mem->mm_node); mem->mm_node = NULL; } } @@ -103,18 +107,12 @@ static int ttm_bo_man_init(struct ttm_mem_type_manager *man, unsigned long p_size) { struct ttm_range_manager *rman; - int ret; rman = kzalloc(sizeof(*rman), GFP_KERNEL); if (!rman) return -ENOMEM; - ret = drm_mm_init(&rman->mm, 0, p_size); - if (ret) { - kfree(rman); - return ret; - } - + drm_mm_init(&rman->mm, 0, p_size); spin_lock_init(&rman->lock); man->priv = rman; return 0; diff --git a/drivers/gpu/drm/ttm/ttm_bo_util.c b/drivers/gpu/drm/ttm/ttm_bo_util.c index f8187ead7b3..1df856f7856 100644 --- a/drivers/gpu/drm/ttm/ttm_bo_util.c +++ b/drivers/gpu/drm/ttm/ttm_bo_util.c @@ -28,8 +28,9 @@ * Authors: Thomas Hellstrom <thellstrom-at-vmware-dot-com> */ -#include "ttm/ttm_bo_driver.h" -#include "ttm/ttm_placement.h" +#include <drm/ttm/ttm_bo_driver.h> +#include <drm/ttm/ttm_placement.h> +#include <drm/drm_vma_manager.h> #include <linux/io.h> #include <linux/highmem.h> #include <linux/wait.h> @@ -43,7 +44,7 @@ void ttm_bo_free_old_node(struct ttm_buffer_object *bo) } int ttm_bo_move_ttm(struct ttm_buffer_object *bo, - bool evict, bool no_wait_reserve, + bool evict, bool no_wait_gpu, struct ttm_mem_reg *new_mem) { struct ttm_tt *ttm = bo->ttm; @@ -86,6 +87,7 @@ int ttm_mem_io_lock(struct ttm_mem_type_manager *man, bool interruptible) mutex_lock(&man->io_reserve_mutex); return 0; } +EXPORT_SYMBOL(ttm_mem_io_lock); void ttm_mem_io_unlock(struct ttm_mem_type_manager *man) { @@ -94,6 +96,7 @@ void ttm_mem_io_unlock(struct ttm_mem_type_manager *man) mutex_unlock(&man->io_reserve_mutex); } +EXPORT_SYMBOL(ttm_mem_io_unlock); static int ttm_mem_io_evict(struct ttm_mem_type_manager *man) { @@ -111,8 +114,9 @@ static int ttm_mem_io_evict(struct ttm_mem_type_manager *man) return 0; } -static int ttm_mem_io_reserve(struct ttm_bo_device *bdev, - struct ttm_mem_reg *mem) + +int ttm_mem_io_reserve(struct ttm_bo_device *bdev, + struct ttm_mem_reg *mem) { struct ttm_mem_type_manager *man = &bdev->man[mem->mem_type]; int ret = 0; @@ -134,9 +138,10 @@ retry: } return ret; } +EXPORT_SYMBOL(ttm_mem_io_reserve); -static void ttm_mem_io_free(struct ttm_bo_device *bdev, - struct ttm_mem_reg *mem) +void ttm_mem_io_free(struct ttm_bo_device *bdev, + struct ttm_mem_reg *mem) { struct ttm_mem_type_manager *man = &bdev->man[mem->mem_type]; @@ -149,6 +154,7 @@ static void ttm_mem_io_free(struct ttm_bo_device *bdev, bdev->driver->io_mem_free(bdev, mem); } +EXPORT_SYMBOL(ttm_mem_io_free); int ttm_mem_io_reserve_vm(struct ttm_buffer_object *bo) { @@ -181,7 +187,7 @@ void ttm_mem_io_free_vm(struct ttm_buffer_object *bo) } } -int ttm_mem_reg_ioremap(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem, +static int ttm_mem_reg_ioremap(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem, void **virtual) { struct ttm_mem_type_manager *man = &bdev->man[mem->mem_type]; @@ -213,7 +219,7 @@ int ttm_mem_reg_ioremap(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem, return 0; } -void ttm_mem_reg_iounmap(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem, +static void ttm_mem_reg_iounmap(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem, void *virtual) { struct ttm_mem_type_manager *man; @@ -314,7 +320,7 @@ static int ttm_copy_ttm_io_page(struct ttm_tt *ttm, void *dst, } int ttm_bo_move_memcpy(struct ttm_buffer_object *bo, - bool evict, bool no_wait_reserve, bool no_wait_gpu, + bool evict, bool no_wait_gpu, struct ttm_mem_reg *new_mem) { struct ttm_bo_device *bdev = bo->bdev; @@ -337,12 +343,26 @@ int ttm_bo_move_memcpy(struct ttm_buffer_object *bo, if (ret) goto out; + /* + * Single TTM move. NOP. + */ if (old_iomap == NULL && new_iomap == NULL) goto out2; - if (old_iomap == NULL && ttm == NULL) + + /* + * Don't move nonexistent data. Clear destination instead. + */ + if (old_iomap == NULL && + (ttm == NULL || (ttm->state == tt_unpopulated && + !(ttm->page_flags & TTM_PAGE_FLAG_SWAPPED)))) { + memset_io(new_iomap, 0, new_mem->num_pages*PAGE_SIZE); goto out2; + } - if (ttm->state == tt_unpopulated) { + /* + * TTM might be null for moves within the same region. + */ + if (ttm && ttm->state == tt_unpopulated) { ret = ttm->bdev->driver->ttm_tt_populate(ttm); if (ret) goto out1; @@ -390,7 +410,12 @@ out1: ttm_mem_reg_iounmap(bdev, old_mem, new_iomap); out: ttm_mem_reg_iounmap(bdev, &old_copy, old_iomap); - ttm_bo_mem_put(bo, &old_copy); + + /* + * On error, keep the mm node! + */ + if (!ret) + ttm_bo_mem_put(bo, &old_copy); return ret; } EXPORT_SYMBOL(ttm_bo_move_memcpy); @@ -421,8 +446,9 @@ static int ttm_buffer_object_transfer(struct ttm_buffer_object *bo, struct ttm_buffer_object *fbo; struct ttm_bo_device *bdev = bo->bdev; struct ttm_bo_driver *driver = bdev->driver; + int ret; - fbo = kzalloc(sizeof(*fbo), GFP_KERNEL); + fbo = kmalloc(sizeof(*fbo), GFP_KERNEL); if (!fbo) return -ENOMEM; @@ -433,19 +459,27 @@ static int ttm_buffer_object_transfer(struct ttm_buffer_object *bo, * TODO: Explicit member copy would probably be better here. */ - init_waitqueue_head(&fbo->event_queue); INIT_LIST_HEAD(&fbo->ddestroy); INIT_LIST_HEAD(&fbo->lru); INIT_LIST_HEAD(&fbo->swap); INIT_LIST_HEAD(&fbo->io_reserve_lru); - fbo->vm_node = NULL; + drm_vma_node_reset(&fbo->vma_node); atomic_set(&fbo->cpu_writers, 0); - fbo->sync_obj = driver->sync_obj_ref(bo->sync_obj); + spin_lock(&bdev->fence_lock); + if (bo->sync_obj) + fbo->sync_obj = driver->sync_obj_ref(bo->sync_obj); + else + fbo->sync_obj = NULL; + spin_unlock(&bdev->fence_lock); kref_init(&fbo->list_kref); kref_init(&fbo->kref); fbo->destroy = &ttm_transfered_destroy; fbo->acc_size = 0; + fbo->resv = &fbo->ttm_resv; + reservation_object_init(fbo->resv); + ret = ww_mutex_trylock(&fbo->resv->lock); + WARN_ON(!ret); *new_obj = fbo; return 0; @@ -472,7 +506,7 @@ pgprot_t ttm_io_prot(uint32_t caching_flags, pgprot_t tmp) else tmp = pgprot_noncached(tmp); #endif -#if defined(__sparc__) +#if defined(__sparc__) || defined(__mips__) if (!(caching_flags & TTM_PL_FLAG_CACHED)) tmp = pgprot_noncached(tmp); #endif @@ -560,7 +594,7 @@ int ttm_bo_kmap(struct ttm_buffer_object *bo, if (start_page > bo->num_pages) return -EINVAL; #if 0 - if (num_pages > 1 && !DRM_SUSER(DRM_CURPROC)) + if (num_pages > 1 && !capable(CAP_SYS_ADMIN)) return -EPERM; #endif (void) ttm_mem_io_lock(man, false); @@ -611,8 +645,7 @@ EXPORT_SYMBOL(ttm_bo_kunmap); int ttm_bo_move_accel_cleanup(struct ttm_buffer_object *bo, void *sync_obj, - void *sync_obj_arg, - bool evict, bool no_wait_reserve, + bool evict, bool no_wait_gpu, struct ttm_mem_reg *new_mem) { @@ -630,7 +663,6 @@ int ttm_bo_move_accel_cleanup(struct ttm_buffer_object *bo, bo->sync_obj = NULL; } bo->sync_obj = driver->sync_obj_ref(sync_obj); - bo->sync_obj_arg = sync_obj_arg; if (evict) { ret = ttm_bo_wait(bo, false, false, false); spin_unlock(&bdev->fence_lock); diff --git a/drivers/gpu/drm/ttm/ttm_bo_vm.c b/drivers/gpu/drm/ttm/ttm_bo_vm.c index 54412848de8..0ce48e5a9cb 100644 --- a/drivers/gpu/drm/ttm/ttm_bo_vm.c +++ b/drivers/gpu/drm/ttm/ttm_bo_vm.c @@ -28,9 +28,12 @@ * Authors: Thomas Hellstrom <thellstrom-at-vmware-dot-com> */ +#define pr_fmt(fmt) "[TTM] " fmt + #include <ttm/ttm_module.h> #include <ttm/ttm_bo_driver.h> #include <ttm/ttm_placement.h> +#include <drm/drm_vma_manager.h> #include <linux/mm.h> #include <linux/rbtree.h> #include <linux/module.h> @@ -38,35 +41,49 @@ #define TTM_BO_VM_NUM_PREFAULT 16 -static struct ttm_buffer_object *ttm_bo_vm_lookup_rb(struct ttm_bo_device *bdev, - unsigned long page_start, - unsigned long num_pages) +static int ttm_bo_vm_fault_idle(struct ttm_buffer_object *bo, + struct vm_area_struct *vma, + struct vm_fault *vmf) { - struct rb_node *cur = bdev->addr_space_rb.rb_node; - unsigned long cur_offset; - struct ttm_buffer_object *bo; - struct ttm_buffer_object *best_bo = NULL; - - while (likely(cur != NULL)) { - bo = rb_entry(cur, struct ttm_buffer_object, vm_rb); - cur_offset = bo->vm_node->start; - if (page_start >= cur_offset) { - cur = cur->rb_right; - best_bo = bo; - if (page_start == cur_offset) - break; - } else - cur = cur->rb_left; - } + struct ttm_bo_device *bdev = bo->bdev; + int ret = 0; + + spin_lock(&bdev->fence_lock); + if (likely(!test_bit(TTM_BO_PRIV_FLAG_MOVING, &bo->priv_flags))) + goto out_unlock; - if (unlikely(best_bo == NULL)) - return NULL; + /* + * Quick non-stalling check for idle. + */ + ret = ttm_bo_wait(bo, false, false, true); + if (likely(ret == 0)) + goto out_unlock; - if (unlikely((best_bo->vm_node->start + best_bo->num_pages) < - (page_start + num_pages))) - return NULL; + /* + * If possible, avoid waiting for GPU with mmap_sem + * held. + */ + if (vmf->flags & FAULT_FLAG_ALLOW_RETRY) { + ret = VM_FAULT_RETRY; + if (vmf->flags & FAULT_FLAG_RETRY_NOWAIT) + goto out_unlock; - return best_bo; + up_read(&vma->vm_mm->mmap_sem); + (void) ttm_bo_wait(bo, false, true, false); + goto out_unlock; + } + + /* + * Ordinary wait. + */ + ret = ttm_bo_wait(bo, false, true, false); + if (unlikely(ret != 0)) + ret = (ret != -ERESTARTSYS) ? VM_FAULT_SIGBUS : + VM_FAULT_NOPAGE; + +out_unlock: + spin_unlock(&bdev->fence_lock); + return ret; } static int ttm_bo_vm_fault(struct vm_area_struct *vma, struct vm_fault *vmf) @@ -85,27 +102,51 @@ static int ttm_bo_vm_fault(struct vm_area_struct *vma, struct vm_fault *vmf) int retval = VM_FAULT_NOPAGE; struct ttm_mem_type_manager *man = &bdev->man[bo->mem.mem_type]; + struct vm_area_struct cvma; /* * Work around locking order reversal in fault / nopfn * between mmap_sem and bo_reserve: Perform a trylock operation - * for reserve, and if it fails, retry the fault after scheduling. + * for reserve, and if it fails, retry the fault after waiting + * for the buffer to become unreserved. */ - - ret = ttm_bo_reserve(bo, true, true, false, 0); + ret = ttm_bo_reserve(bo, true, true, false, NULL); if (unlikely(ret != 0)) { - if (ret == -EBUSY) - set_need_resched(); + if (ret != -EBUSY) + return VM_FAULT_NOPAGE; + + if (vmf->flags & FAULT_FLAG_ALLOW_RETRY) { + if (!(vmf->flags & FAULT_FLAG_RETRY_NOWAIT)) { + up_read(&vma->vm_mm->mmap_sem); + (void) ttm_bo_wait_unreserved(bo); + } + + return VM_FAULT_RETRY; + } + + /* + * If we'd want to change locking order to + * mmap_sem -> bo::reserve, we'd use a blocking reserve here + * instead of retrying the fault... + */ return VM_FAULT_NOPAGE; } + /* + * Refuse to fault imported pages. This should be handled + * (if at all) by redirecting mmap to the exporter. + */ + if (bo->ttm && (bo->ttm->page_flags & TTM_PAGE_FLAG_SG)) { + retval = VM_FAULT_SIGBUS; + goto out_unlock; + } + if (bdev->driver->fault_reserve_notify) { ret = bdev->driver->fault_reserve_notify(bo); switch (ret) { case 0: break; case -EBUSY: - set_need_resched(); case -ERESTARTSYS: retval = VM_FAULT_NOPAGE; goto out_unlock; @@ -119,18 +160,11 @@ static int ttm_bo_vm_fault(struct vm_area_struct *vma, struct vm_fault *vmf) * Wait for buffer data in transit, due to a pipelined * move. */ - - spin_lock(&bdev->fence_lock); - if (test_bit(TTM_BO_PRIV_FLAG_MOVING, &bo->priv_flags)) { - ret = ttm_bo_wait(bo, false, true, false); - spin_unlock(&bdev->fence_lock); - if (unlikely(ret != 0)) { - retval = (ret != -ERESTARTSYS) ? - VM_FAULT_SIGBUS : VM_FAULT_NOPAGE; - goto out_unlock; - } - } else - spin_unlock(&bdev->fence_lock); + ret = ttm_bo_vm_fault_idle(bo, vma, vmf); + if (unlikely(ret != 0)) { + retval = ret; + goto out_unlock; + } ret = ttm_mem_io_lock(man, true); if (unlikely(ret != 0)) { @@ -144,9 +178,9 @@ static int ttm_bo_vm_fault(struct vm_area_struct *vma, struct vm_fault *vmf) } page_offset = ((address - vma->vm_start) >> PAGE_SHIFT) + - bo->vm_node->start - vma->vm_pgoff; - page_last = ((vma->vm_end - vma->vm_start) >> PAGE_SHIFT) + - bo->vm_node->start - vma->vm_pgoff; + vma->vm_pgoff - drm_vma_node_start(&bo->vma_node); + page_last = vma_pages(vma) + vma->vm_pgoff - + drm_vma_node_start(&bo->vma_node); if (unlikely(page_offset >= bo->num_pages)) { retval = VM_FAULT_SIGBUS; @@ -154,26 +188,21 @@ static int ttm_bo_vm_fault(struct vm_area_struct *vma, struct vm_fault *vmf) } /* - * Strictly, we're not allowed to modify vma->vm_page_prot here, - * since the mmap_sem is only held in read mode. However, we - * modify only the caching bits of vma->vm_page_prot and - * consider those bits protected by - * the bo->mutex, as we should be the only writers. - * There shouldn't really be any readers of these bits except - * within vm_insert_mixed()? fork? - * - * TODO: Add a list of vmas to the bo, and change the - * vma->vm_page_prot when the object changes caching policy, with - * the correct locks held. + * Make a local vma copy to modify the page_prot member + * and vm_flags if necessary. The vma parameter is protected + * by mmap_sem in write mode. */ + cvma = *vma; + cvma.vm_page_prot = vm_get_page_prot(cvma.vm_flags); + if (bo->mem.bus.is_iomem) { - vma->vm_page_prot = ttm_io_prot(bo->mem.placement, - vma->vm_page_prot); + cvma.vm_page_prot = ttm_io_prot(bo->mem.placement, + cvma.vm_page_prot); } else { ttm = bo->ttm; - vma->vm_page_prot = (bo->mem.placement & TTM_PL_FLAG_CACHED) ? - vm_get_page_prot(vma->vm_flags) : - ttm_io_prot(bo->mem.placement, vma->vm_page_prot); + if (!(bo->mem.placement & TTM_PL_FLAG_CACHED)) + cvma.vm_page_prot = ttm_io_prot(bo->mem.placement, + cvma.vm_page_prot); /* Allocate all page at once, most common usage */ if (ttm->bdev->driver->ttm_tt_populate(ttm)) { @@ -197,10 +226,17 @@ static int ttm_bo_vm_fault(struct vm_area_struct *vma, struct vm_fault *vmf) } else if (unlikely(!page)) { break; } + page->mapping = vma->vm_file->f_mapping; + page->index = drm_vma_node_start(&bo->vma_node) + + page_offset; pfn = page_to_pfn(page); } - ret = vm_insert_mixed(vma, address, pfn); + if (vma->vm_flags & VM_MIXEDMAP) + ret = vm_insert_mixed(&cvma, address, pfn); + else + ret = vm_insert_pfn(&cvma, address, pfn); + /* * Somebody beat us to this PTE or prefaulting to * an already populated PTE, or prefaulting error. @@ -230,6 +266,8 @@ static void ttm_bo_vm_open(struct vm_area_struct *vma) struct ttm_buffer_object *bo = (struct ttm_buffer_object *)vma->vm_private_data; + WARN_ON(bo->bdev->dev_mapping != vma->vm_file->f_mapping); + (void)ttm_bo_reference(bo); } @@ -247,6 +285,30 @@ static const struct vm_operations_struct ttm_bo_vm_ops = { .close = ttm_bo_vm_close }; +static struct ttm_buffer_object *ttm_bo_vm_lookup(struct ttm_bo_device *bdev, + unsigned long offset, + unsigned long pages) +{ + struct drm_vma_offset_node *node; + struct ttm_buffer_object *bo = NULL; + + drm_vma_offset_lock_lookup(&bdev->vma_manager); + + node = drm_vma_offset_lookup_locked(&bdev->vma_manager, offset, pages); + if (likely(node)) { + bo = container_of(node, struct ttm_buffer_object, vma_node); + if (!kref_get_unless_zero(&bo->kref)) + bo = NULL; + } + + drm_vma_offset_unlock_lookup(&bdev->vma_manager); + + if (!bo) + pr_err("Could not find buffer object to map\n"); + + return bo; +} + int ttm_bo_mmap(struct file *filp, struct vm_area_struct *vma, struct ttm_bo_device *bdev) { @@ -254,18 +316,9 @@ int ttm_bo_mmap(struct file *filp, struct vm_area_struct *vma, struct ttm_buffer_object *bo; int ret; - read_lock(&bdev->vm_lock); - bo = ttm_bo_vm_lookup_rb(bdev, vma->vm_pgoff, - (vma->vm_end - vma->vm_start) >> PAGE_SHIFT); - if (likely(bo != NULL)) - ttm_bo_reference(bo); - read_unlock(&bdev->vm_lock); - - if (unlikely(bo == NULL)) { - printk(KERN_ERR TTM_PFX - "Could not find buffer object to map.\n"); + bo = ttm_bo_vm_lookup(bdev, vma->vm_pgoff, vma_pages(vma)); + if (unlikely(!bo)) return -EINVAL; - } driver = bo->bdev->driver; if (unlikely(!driver->verify_access)) { @@ -284,7 +337,16 @@ int ttm_bo_mmap(struct file *filp, struct vm_area_struct *vma, */ vma->vm_private_data = bo; - vma->vm_flags |= VM_RESERVED | VM_IO | VM_MIXEDMAP | VM_DONTEXPAND; + + /* + * We'd like to use VM_PFNMAP on shared mappings, where + * (vma->vm_flags & VM_SHARED) != 0, for performance reasons, + * but for some reason VM_PFNMAP + x86 PAT + write-combine is very + * bad for performance. Until that has been sorted out, use + * VM_MIXEDMAP on all mappings. See freedesktop.org bug #75719 + */ + vma->vm_flags |= VM_MIXEDMAP; + vma->vm_flags |= VM_IO | VM_DONTEXPAND | VM_DONTDUMP; return 0; out_unref: ttm_bo_unref(&bo); @@ -299,166 +361,8 @@ int ttm_fbdev_mmap(struct vm_area_struct *vma, struct ttm_buffer_object *bo) vma->vm_ops = &ttm_bo_vm_ops; vma->vm_private_data = ttm_bo_reference(bo); - vma->vm_flags |= VM_RESERVED | VM_IO | VM_MIXEDMAP | VM_DONTEXPAND; + vma->vm_flags |= VM_MIXEDMAP; + vma->vm_flags |= VM_IO | VM_DONTEXPAND; return 0; } EXPORT_SYMBOL(ttm_fbdev_mmap); - - -ssize_t ttm_bo_io(struct ttm_bo_device *bdev, struct file *filp, - const char __user *wbuf, char __user *rbuf, size_t count, - loff_t *f_pos, bool write) -{ - struct ttm_buffer_object *bo; - struct ttm_bo_driver *driver; - struct ttm_bo_kmap_obj map; - unsigned long dev_offset = (*f_pos >> PAGE_SHIFT); - unsigned long kmap_offset; - unsigned long kmap_end; - unsigned long kmap_num; - size_t io_size; - unsigned int page_offset; - char *virtual; - int ret; - bool no_wait = false; - bool dummy; - - read_lock(&bdev->vm_lock); - bo = ttm_bo_vm_lookup_rb(bdev, dev_offset, 1); - if (likely(bo != NULL)) - ttm_bo_reference(bo); - read_unlock(&bdev->vm_lock); - - if (unlikely(bo == NULL)) - return -EFAULT; - - driver = bo->bdev->driver; - if (unlikely(!driver->verify_access)) { - ret = -EPERM; - goto out_unref; - } - - ret = driver->verify_access(bo, filp); - if (unlikely(ret != 0)) - goto out_unref; - - kmap_offset = dev_offset - bo->vm_node->start; - if (unlikely(kmap_offset >= bo->num_pages)) { - ret = -EFBIG; - goto out_unref; - } - - page_offset = *f_pos & ~PAGE_MASK; - io_size = bo->num_pages - kmap_offset; - io_size = (io_size << PAGE_SHIFT) - page_offset; - if (count < io_size) - io_size = count; - - kmap_end = (*f_pos + count - 1) >> PAGE_SHIFT; - kmap_num = kmap_end - kmap_offset + 1; - - ret = ttm_bo_reserve(bo, true, no_wait, false, 0); - - switch (ret) { - case 0: - break; - case -EBUSY: - ret = -EAGAIN; - goto out_unref; - default: - goto out_unref; - } - - ret = ttm_bo_kmap(bo, kmap_offset, kmap_num, &map); - if (unlikely(ret != 0)) { - ttm_bo_unreserve(bo); - goto out_unref; - } - - virtual = ttm_kmap_obj_virtual(&map, &dummy); - virtual += page_offset; - - if (write) - ret = copy_from_user(virtual, wbuf, io_size); - else - ret = copy_to_user(rbuf, virtual, io_size); - - ttm_bo_kunmap(&map); - ttm_bo_unreserve(bo); - ttm_bo_unref(&bo); - - if (unlikely(ret != 0)) - return -EFBIG; - - *f_pos += io_size; - - return io_size; -out_unref: - ttm_bo_unref(&bo); - return ret; -} - -ssize_t ttm_bo_fbdev_io(struct ttm_buffer_object *bo, const char __user *wbuf, - char __user *rbuf, size_t count, loff_t *f_pos, - bool write) -{ - struct ttm_bo_kmap_obj map; - unsigned long kmap_offset; - unsigned long kmap_end; - unsigned long kmap_num; - size_t io_size; - unsigned int page_offset; - char *virtual; - int ret; - bool no_wait = false; - bool dummy; - - kmap_offset = (*f_pos >> PAGE_SHIFT); - if (unlikely(kmap_offset >= bo->num_pages)) - return -EFBIG; - - page_offset = *f_pos & ~PAGE_MASK; - io_size = bo->num_pages - kmap_offset; - io_size = (io_size << PAGE_SHIFT) - page_offset; - if (count < io_size) - io_size = count; - - kmap_end = (*f_pos + count - 1) >> PAGE_SHIFT; - kmap_num = kmap_end - kmap_offset + 1; - - ret = ttm_bo_reserve(bo, true, no_wait, false, 0); - - switch (ret) { - case 0: - break; - case -EBUSY: - return -EAGAIN; - default: - return ret; - } - - ret = ttm_bo_kmap(bo, kmap_offset, kmap_num, &map); - if (unlikely(ret != 0)) { - ttm_bo_unreserve(bo); - return ret; - } - - virtual = ttm_kmap_obj_virtual(&map, &dummy); - virtual += page_offset; - - if (write) - ret = copy_from_user(virtual, wbuf, io_size); - else - ret = copy_to_user(rbuf, virtual, io_size); - - ttm_bo_kunmap(&map); - ttm_bo_unreserve(bo); - ttm_bo_unref(&bo); - - if (unlikely(ret != 0)) - return ret; - - *f_pos += io_size; - - return io_size; -} diff --git a/drivers/gpu/drm/ttm/ttm_execbuf_util.c b/drivers/gpu/drm/ttm/ttm_execbuf_util.c index 3832fe10b4d..e8dac875852 100644 --- a/drivers/gpu/drm/ttm/ttm_execbuf_util.c +++ b/drivers/gpu/drm/ttm/ttm_execbuf_util.c @@ -25,9 +25,9 @@ * **************************************************************************/ -#include "ttm/ttm_execbuf_util.h" -#include "ttm/ttm_bo_driver.h" -#include "ttm/ttm_placement.h" +#include <drm/ttm/ttm_execbuf_util.h> +#include <drm/ttm/ttm_bo_driver.h> +#include <drm/ttm/ttm_placement.h> #include <linux/wait.h> #include <linux/sched.h> #include <linux/module.h> @@ -41,14 +41,12 @@ static void ttm_eu_backoff_reservation_locked(struct list_head *list) if (!entry->reserved) continue; + entry->reserved = false; if (entry->removed) { ttm_bo_add_to_lru(bo); entry->removed = false; - } - entry->reserved = false; - atomic_set(&bo->reserved, 0); - wake_up_all(&bo->event_queue); + __ttm_bo_unreserve(bo); } } @@ -82,23 +80,8 @@ static void ttm_eu_list_ref_sub(struct list_head *list) } } -static int ttm_eu_wait_unreserved_locked(struct list_head *list, - struct ttm_buffer_object *bo) -{ - struct ttm_bo_global *glob = bo->glob; - int ret; - - ttm_eu_del_from_lru_locked(list); - spin_unlock(&glob->lru_lock); - ret = ttm_bo_wait_unreserved(bo, true); - spin_lock(&glob->lru_lock); - if (unlikely(ret != 0)) - ttm_eu_backoff_reservation_locked(list); - return ret; -} - - -void ttm_eu_backoff_reservation(struct list_head *list) +void ttm_eu_backoff_reservation(struct ww_acquire_ctx *ticket, + struct list_head *list) { struct ttm_validate_buffer *entry; struct ttm_bo_global *glob; @@ -110,6 +93,8 @@ void ttm_eu_backoff_reservation(struct list_head *list) glob = entry->bo->glob; spin_lock(&glob->lru_lock); ttm_eu_backoff_reservation_locked(list); + if (ticket) + ww_acquire_fini(ticket); spin_unlock(&glob->lru_lock); } EXPORT_SYMBOL(ttm_eu_backoff_reservation); @@ -126,12 +111,12 @@ EXPORT_SYMBOL(ttm_eu_backoff_reservation); * buffers in different orders. */ -int ttm_eu_reserve_buffers(struct list_head *list) +int ttm_eu_reserve_buffers(struct ww_acquire_ctx *ticket, + struct list_head *list) { struct ttm_bo_global *glob; struct ttm_validate_buffer *entry; int ret; - uint32_t val_seq; if (list_empty(list)) return 0; @@ -145,62 +130,77 @@ int ttm_eu_reserve_buffers(struct list_head *list) entry = list_first_entry(list, struct ttm_validate_buffer, head); glob = entry->bo->glob; + if (ticket) + ww_acquire_init(ticket, &reservation_ww_class); retry: - spin_lock(&glob->lru_lock); - val_seq = entry->bo->bdev->val_seq++; - list_for_each_entry(entry, list, head) { struct ttm_buffer_object *bo = entry->bo; -retry_this_bo: - ret = ttm_bo_reserve_locked(bo, true, true, true, val_seq); - switch (ret) { - case 0: - break; - case -EBUSY: - ret = ttm_eu_wait_unreserved_locked(list, bo); - if (unlikely(ret != 0)) { - spin_unlock(&glob->lru_lock); - ttm_eu_list_ref_sub(list); - return ret; - } - goto retry_this_bo; - case -EAGAIN: + /* already slowpath reserved? */ + if (entry->reserved) + continue; + + ret = __ttm_bo_reserve(bo, true, (ticket == NULL), true, + ticket); + + if (ret == -EDEADLK) { + /* uh oh, we lost out, drop every reservation and try + * to only reserve this buffer, then start over if + * this succeeds. + */ + BUG_ON(ticket == NULL); + spin_lock(&glob->lru_lock); ttm_eu_backoff_reservation_locked(list); spin_unlock(&glob->lru_lock); ttm_eu_list_ref_sub(list); - ret = ttm_bo_wait_unreserved(bo, true); - if (unlikely(ret != 0)) - return ret; + ret = ww_mutex_lock_slow_interruptible(&bo->resv->lock, + ticket); + if (unlikely(ret != 0)) { + if (ret == -EINTR) + ret = -ERESTARTSYS; + goto err_fini; + } + + entry->reserved = true; + if (unlikely(atomic_read(&bo->cpu_writers) > 0)) { + ret = -EBUSY; + goto err; + } goto retry; - default: - ttm_eu_backoff_reservation_locked(list); - spin_unlock(&glob->lru_lock); - ttm_eu_list_ref_sub(list); - return ret; - } + } else if (ret) + goto err; entry->reserved = true; if (unlikely(atomic_read(&bo->cpu_writers) > 0)) { - ttm_eu_backoff_reservation_locked(list); - spin_unlock(&glob->lru_lock); - ttm_eu_list_ref_sub(list); - ret = ttm_bo_wait_cpu(bo, false); - if (ret) - return ret; - goto retry; + ret = -EBUSY; + goto err; } } + if (ticket) + ww_acquire_done(ticket); + spin_lock(&glob->lru_lock); ttm_eu_del_from_lru_locked(list); spin_unlock(&glob->lru_lock); ttm_eu_list_ref_sub(list); - return 0; + +err: + spin_lock(&glob->lru_lock); + ttm_eu_backoff_reservation_locked(list); + spin_unlock(&glob->lru_lock); + ttm_eu_list_ref_sub(list); +err_fini: + if (ticket) { + ww_acquire_done(ticket); + ww_acquire_fini(ticket); + } + return ret; } EXPORT_SYMBOL(ttm_eu_reserve_buffers); -void ttm_eu_fence_buffer_objects(struct list_head *list, void *sync_obj) +void ttm_eu_fence_buffer_objects(struct ww_acquire_ctx *ticket, + struct list_head *list, void *sync_obj) { struct ttm_validate_buffer *entry; struct ttm_buffer_object *bo; @@ -216,19 +216,21 @@ void ttm_eu_fence_buffer_objects(struct list_head *list, void *sync_obj) driver = bdev->driver; glob = bo->glob; - spin_lock(&bdev->fence_lock); spin_lock(&glob->lru_lock); + spin_lock(&bdev->fence_lock); list_for_each_entry(entry, list, head) { bo = entry->bo; entry->old_sync_obj = bo->sync_obj; bo->sync_obj = driver->sync_obj_ref(sync_obj); - bo->sync_obj_arg = entry->new_sync_obj_arg; - ttm_bo_unreserve_locked(bo); + ttm_bo_add_to_lru(bo); + __ttm_bo_unreserve(bo); entry->reserved = false; } - spin_unlock(&glob->lru_lock); spin_unlock(&bdev->fence_lock); + spin_unlock(&glob->lru_lock); + if (ticket) + ww_acquire_fini(ticket); list_for_each_entry(entry, list, head) { if (entry->old_sync_obj) diff --git a/drivers/gpu/drm/ttm/ttm_lock.c b/drivers/gpu/drm/ttm/ttm_lock.c index 075daf44bce..6a954544727 100644 --- a/drivers/gpu/drm/ttm/ttm_lock.c +++ b/drivers/gpu/drm/ttm/ttm_lock.c @@ -28,8 +28,8 @@ * Authors: Thomas Hellstrom <thellstrom-at-vmware-dot-com> */ -#include "ttm/ttm_lock.h" -#include "ttm/ttm_module.h" +#include <drm/ttm/ttm_lock.h> +#include <drm/ttm/ttm_module.h> #include <linux/atomic.h> #include <linux/errno.h> #include <linux/wait.h> @@ -186,14 +186,6 @@ int ttm_write_lock(struct ttm_lock *lock, bool interruptible) } EXPORT_SYMBOL(ttm_write_lock); -void ttm_write_lock_downgrade(struct ttm_lock *lock) -{ - spin_lock(&lock->lock); - lock->rw = 1; - wake_up_all(&lock->queue); - spin_unlock(&lock->lock); -} - static int __ttm_vt_unlock(struct ttm_lock *lock) { int ret = 0; diff --git a/drivers/gpu/drm/ttm/ttm_memory.c b/drivers/gpu/drm/ttm/ttm_memory.c index 9eba8e9a4e9..dbc2def887c 100644 --- a/drivers/gpu/drm/ttm/ttm_memory.c +++ b/drivers/gpu/drm/ttm/ttm_memory.c @@ -25,9 +25,11 @@ * **************************************************************************/ -#include "ttm/ttm_memory.h" -#include "ttm/ttm_module.h" -#include "ttm/ttm_page_alloc.h" +#define pr_fmt(fmt) "[TTM] " fmt + +#include <drm/ttm/ttm_memory.h> +#include <drm/ttm/ttm_module.h> +#include <drm/ttm/ttm_page_alloc.h> #include <linux/spinlock.h> #include <linux/sched.h> #include <linux/wait.h> @@ -74,9 +76,8 @@ static void ttm_mem_zone_kobj_release(struct kobject *kobj) struct ttm_mem_zone *zone = container_of(kobj, struct ttm_mem_zone, kobj); - printk(KERN_INFO TTM_PFX - "Zone %7s: Used memory at exit: %llu kiB.\n", - zone->name, (unsigned long long) zone->used_mem >> 10); + pr_info("Zone %7s: Used memory at exit: %llu kiB\n", + zone->name, (unsigned long long)zone->used_mem >> 10); kfree(zone); } @@ -366,7 +367,6 @@ int ttm_mem_global_init(struct ttm_mem_global *glob) spin_lock_init(&glob->lock); glob->swap_queue = create_singlethread_workqueue("ttm_swap"); INIT_WORK(&glob->work, ttm_shrink_work); - init_waitqueue_head(&glob->queue); ret = kobject_init_and_add( &glob->kobj, &ttm_mem_glob_kobj_type, ttm_get_kobj(), "memory_accounting"); if (unlikely(ret != 0)) { @@ -390,9 +390,8 @@ int ttm_mem_global_init(struct ttm_mem_global *glob) #endif for (i = 0; i < glob->num_zones; ++i) { zone = glob->zones[i]; - printk(KERN_INFO TTM_PFX - "Zone %7s: Available graphics memory: %llu kiB.\n", - zone->name, (unsigned long long) zone->max_mem >> 10); + pr_info("Zone %7s: Available graphics memory: %llu kiB\n", + zone->name, (unsigned long long)zone->max_mem >> 10); } ttm_page_alloc_init(glob, glob->zone_kernel->max_mem/(2*PAGE_SIZE)); ttm_dma_page_alloc_init(glob, glob->zone_kernel->max_mem/(2*PAGE_SIZE)); diff --git a/drivers/gpu/drm/ttm/ttm_module.c b/drivers/gpu/drm/ttm/ttm_module.c index 902d7cf9fb4..d7f92fe9d90 100644 --- a/drivers/gpu/drm/ttm/ttm_module.c +++ b/drivers/gpu/drm/ttm/ttm_module.c @@ -31,8 +31,8 @@ #include <linux/module.h> #include <linux/device.h> #include <linux/sched.h> -#include "ttm/ttm_module.h" -#include "drm_sysfs.h" +#include <drm/ttm/ttm_module.h> +#include <drm/drm_sysfs.h> static DECLARE_WAIT_QUEUE_HEAD(exit_q); atomic_t device_released; diff --git a/drivers/gpu/drm/ttm/ttm_object.c b/drivers/gpu/drm/ttm/ttm_object.c index 93577f2e295..d2a05335278 100644 --- a/drivers/gpu/drm/ttm/ttm_object.c +++ b/drivers/gpu/drm/ttm/ttm_object.c @@ -1,6 +1,6 @@ /************************************************************************** * - * Copyright (c) 2009 VMware, Inc., Palo Alto, CA., USA + * Copyright (c) 2009-2013 VMware, Inc., Palo Alto, CA., USA * All Rights Reserved. * * Permission is hereby granted, free of charge, to any person obtaining a @@ -26,6 +26,12 @@ **************************************************************************/ /* * Authors: Thomas Hellstrom <thellstrom-at-vmware-dot-com> + * + * While no substantial code is shared, the prime code is inspired by + * drm_prime.c, with + * Authors: + * Dave Airlie <airlied@redhat.com> + * Rob Clark <rob.clark@linaro.org> */ /** @file ttm_ref_object.c * @@ -34,6 +40,7 @@ * and release on file close. */ + /** * struct ttm_object_file * @@ -49,8 +56,10 @@ * for fast lookup of ref objects given a base object. */ -#include "ttm/ttm_object.h" -#include "ttm/ttm_module.h" +#define pr_fmt(fmt) "[TTM] " fmt + +#include <drm/ttm/ttm_object.h> +#include <drm/ttm/ttm_module.h> #include <linux/list.h> #include <linux/spinlock.h> #include <linux/slab.h> @@ -59,7 +68,7 @@ struct ttm_object_file { struct ttm_object_device *tdev; - rwlock_t lock; + spinlock_t lock; struct list_head ref_list; struct drm_open_hash ref_hash[TTM_REF_NUM]; struct kref refcount; @@ -78,10 +87,13 @@ struct ttm_object_file { */ struct ttm_object_device { - rwlock_t object_lock; + spinlock_t object_lock; struct drm_open_hash object_hash; atomic_t object_count; struct ttm_mem_global *mem_glob; + struct dma_buf_ops ops; + void (*dmabuf_release)(struct dma_buf *dma_buf); + size_t dma_buf_size; }; /** @@ -106,6 +118,7 @@ struct ttm_object_device { */ struct ttm_ref_object { + struct rcu_head rcu_head; struct drm_hash_item hash; struct list_head head; struct kref kref; @@ -114,6 +127,8 @@ struct ttm_ref_object { struct ttm_object_file *tfile; }; +static void ttm_prime_dmabuf_release(struct dma_buf *dma_buf); + static inline struct ttm_object_file * ttm_object_file_ref(struct ttm_object_file *tfile) { @@ -155,12 +170,12 @@ int ttm_base_object_init(struct ttm_object_file *tfile, base->refcount_release = refcount_release; base->ref_obj_release = ref_obj_release; base->object_type = object_type; - write_lock(&tdev->object_lock); kref_init(&base->refcount); - ret = drm_ht_just_insert_please(&tdev->object_hash, - &base->hash, - (unsigned long)base, 31, 0, 0); - write_unlock(&tdev->object_lock); + spin_lock(&tdev->object_lock); + ret = drm_ht_just_insert_please_rcu(&tdev->object_hash, + &base->hash, + (unsigned long)base, 31, 0, 0); + spin_unlock(&tdev->object_lock); if (unlikely(ret != 0)) goto out_err0; @@ -172,7 +187,9 @@ int ttm_base_object_init(struct ttm_object_file *tfile, return 0; out_err1: - (void)drm_ht_remove_item(&tdev->object_hash, &base->hash); + spin_lock(&tdev->object_lock); + (void)drm_ht_remove_item_rcu(&tdev->object_hash, &base->hash); + spin_unlock(&tdev->object_lock); out_err0: return ret; } @@ -184,63 +201,120 @@ static void ttm_release_base(struct kref *kref) container_of(kref, struct ttm_base_object, refcount); struct ttm_object_device *tdev = base->tfile->tdev; - (void)drm_ht_remove_item(&tdev->object_hash, &base->hash); - write_unlock(&tdev->object_lock); - if (base->refcount_release) { - ttm_object_file_unref(&base->tfile); + spin_lock(&tdev->object_lock); + (void)drm_ht_remove_item_rcu(&tdev->object_hash, &base->hash); + spin_unlock(&tdev->object_lock); + + /* + * Note: We don't use synchronize_rcu() here because it's far + * too slow. It's up to the user to free the object using + * call_rcu() or ttm_base_object_kfree(). + */ + + ttm_object_file_unref(&base->tfile); + if (base->refcount_release) base->refcount_release(&base); - } - write_lock(&tdev->object_lock); } void ttm_base_object_unref(struct ttm_base_object **p_base) { struct ttm_base_object *base = *p_base; - struct ttm_object_device *tdev = base->tfile->tdev; *p_base = NULL; - /* - * Need to take the lock here to avoid racing with - * users trying to look up the object. - */ - - write_lock(&tdev->object_lock); kref_put(&base->refcount, ttm_release_base); - write_unlock(&tdev->object_lock); } EXPORT_SYMBOL(ttm_base_object_unref); struct ttm_base_object *ttm_base_object_lookup(struct ttm_object_file *tfile, uint32_t key) { - struct ttm_object_device *tdev = tfile->tdev; - struct ttm_base_object *base; + struct ttm_base_object *base = NULL; struct drm_hash_item *hash; + struct drm_open_hash *ht = &tfile->ref_hash[TTM_REF_USAGE]; int ret; - read_lock(&tdev->object_lock); - ret = drm_ht_find_item(&tdev->object_hash, key, &hash); + rcu_read_lock(); + ret = drm_ht_find_item_rcu(ht, key, &hash); if (likely(ret == 0)) { - base = drm_hash_entry(hash, struct ttm_base_object, hash); - kref_get(&base->refcount); + base = drm_hash_entry(hash, struct ttm_ref_object, hash)->obj; + if (!kref_get_unless_zero(&base->refcount)) + base = NULL; } - read_unlock(&tdev->object_lock); + rcu_read_unlock(); - if (unlikely(ret != 0)) - return NULL; + return base; +} +EXPORT_SYMBOL(ttm_base_object_lookup); - if (tfile != base->tfile && !base->shareable) { - printk(KERN_ERR TTM_PFX - "Attempted access of non-shareable object.\n"); - ttm_base_object_unref(&base); - return NULL; +struct ttm_base_object * +ttm_base_object_lookup_for_ref(struct ttm_object_device *tdev, uint32_t key) +{ + struct ttm_base_object *base = NULL; + struct drm_hash_item *hash; + struct drm_open_hash *ht = &tdev->object_hash; + int ret; + + rcu_read_lock(); + ret = drm_ht_find_item_rcu(ht, key, &hash); + + if (likely(ret == 0)) { + base = drm_hash_entry(hash, struct ttm_base_object, hash); + if (!kref_get_unless_zero(&base->refcount)) + base = NULL; } + rcu_read_unlock(); return base; } -EXPORT_SYMBOL(ttm_base_object_lookup); +EXPORT_SYMBOL(ttm_base_object_lookup_for_ref); + +/** + * ttm_ref_object_exists - Check whether a caller has a valid ref object + * (has opened) a base object. + * + * @tfile: Pointer to a struct ttm_object_file identifying the caller. + * @base: Pointer to a struct base object. + * + * Checks wether the caller identified by @tfile has put a valid USAGE + * reference object on the base object identified by @base. + */ +bool ttm_ref_object_exists(struct ttm_object_file *tfile, + struct ttm_base_object *base) +{ + struct drm_open_hash *ht = &tfile->ref_hash[TTM_REF_USAGE]; + struct drm_hash_item *hash; + struct ttm_ref_object *ref; + + rcu_read_lock(); + if (unlikely(drm_ht_find_item_rcu(ht, base->hash.key, &hash) != 0)) + goto out_false; + + /* + * Verify that the ref object is really pointing to our base object. + * Our base object could actually be dead, and the ref object pointing + * to another base object with the same handle. + */ + ref = drm_hash_entry(hash, struct ttm_ref_object, hash); + if (unlikely(base != ref->obj)) + goto out_false; + + /* + * Verify that the ref->obj pointer was actually valid! + */ + rmb(); + if (unlikely(atomic_read(&ref->kref.refcount) == 0)) + goto out_false; + + rcu_read_unlock(); + return true; + + out_false: + rcu_read_unlock(); + return false; +} +EXPORT_SYMBOL(ttm_ref_object_exists); int ttm_ref_object_add(struct ttm_object_file *tfile, struct ttm_base_object *base, @@ -252,21 +326,25 @@ int ttm_ref_object_add(struct ttm_object_file *tfile, struct ttm_mem_global *mem_glob = tfile->tdev->mem_glob; int ret = -EINVAL; + if (base->tfile != tfile && !base->shareable) + return -EPERM; + if (existed != NULL) *existed = true; while (ret == -EINVAL) { - read_lock(&tfile->lock); - ret = drm_ht_find_item(ht, base->hash.key, &hash); + rcu_read_lock(); + ret = drm_ht_find_item_rcu(ht, base->hash.key, &hash); if (ret == 0) { ref = drm_hash_entry(hash, struct ttm_ref_object, hash); - kref_get(&ref->kref); - read_unlock(&tfile->lock); - break; + if (kref_get_unless_zero(&ref->kref)) { + rcu_read_unlock(); + break; + } } - read_unlock(&tfile->lock); + rcu_read_unlock(); ret = ttm_mem_global_alloc(mem_glob, sizeof(*ref), false, false); if (unlikely(ret != 0)) @@ -283,19 +361,19 @@ int ttm_ref_object_add(struct ttm_object_file *tfile, ref->ref_type = ref_type; kref_init(&ref->kref); - write_lock(&tfile->lock); - ret = drm_ht_insert_item(ht, &ref->hash); + spin_lock(&tfile->lock); + ret = drm_ht_insert_item_rcu(ht, &ref->hash); if (likely(ret == 0)) { list_add_tail(&ref->head, &tfile->ref_list); kref_get(&base->refcount); - write_unlock(&tfile->lock); + spin_unlock(&tfile->lock); if (existed != NULL) *existed = false; break; } - write_unlock(&tfile->lock); + spin_unlock(&tfile->lock); BUG_ON(ret != -EINVAL); ttm_mem_global_free(mem_glob, sizeof(*ref)); @@ -316,17 +394,17 @@ static void ttm_ref_object_release(struct kref *kref) struct ttm_mem_global *mem_glob = tfile->tdev->mem_glob; ht = &tfile->ref_hash[ref->ref_type]; - (void)drm_ht_remove_item(ht, &ref->hash); + (void)drm_ht_remove_item_rcu(ht, &ref->hash); list_del(&ref->head); - write_unlock(&tfile->lock); + spin_unlock(&tfile->lock); if (ref->ref_type != TTM_REF_USAGE && base->ref_obj_release) base->ref_obj_release(base, ref->ref_type); ttm_base_object_unref(&ref->obj); ttm_mem_global_free(mem_glob, sizeof(*ref)); - kfree(ref); - write_lock(&tfile->lock); + kfree_rcu(ref, rcu_head); + spin_lock(&tfile->lock); } int ttm_ref_object_base_unref(struct ttm_object_file *tfile, @@ -337,15 +415,15 @@ int ttm_ref_object_base_unref(struct ttm_object_file *tfile, struct drm_hash_item *hash; int ret; - write_lock(&tfile->lock); + spin_lock(&tfile->lock); ret = drm_ht_find_item(ht, key, &hash); if (unlikely(ret != 0)) { - write_unlock(&tfile->lock); + spin_unlock(&tfile->lock); return -EINVAL; } ref = drm_hash_entry(hash, struct ttm_ref_object, hash); kref_put(&ref->kref, ttm_ref_object_release); - write_unlock(&tfile->lock); + spin_unlock(&tfile->lock); return 0; } EXPORT_SYMBOL(ttm_ref_object_base_unref); @@ -358,7 +436,7 @@ void ttm_object_file_release(struct ttm_object_file **p_tfile) struct ttm_object_file *tfile = *p_tfile; *p_tfile = NULL; - write_lock(&tfile->lock); + spin_lock(&tfile->lock); /* * Since we release the lock within the loop, we have to @@ -374,7 +452,7 @@ void ttm_object_file_release(struct ttm_object_file **p_tfile) for (i = 0; i < TTM_REF_NUM; ++i) drm_ht_remove(&tfile->ref_hash[i]); - write_unlock(&tfile->lock); + spin_unlock(&tfile->lock); ttm_object_file_unref(&tfile); } EXPORT_SYMBOL(ttm_object_file_release); @@ -390,7 +468,7 @@ struct ttm_object_file *ttm_object_file_init(struct ttm_object_device *tdev, if (unlikely(tfile == NULL)) return NULL; - rwlock_init(&tfile->lock); + spin_lock_init(&tfile->lock); tfile->tdev = tdev; kref_init(&tfile->refcount); INIT_LIST_HEAD(&tfile->ref_list); @@ -414,9 +492,10 @@ out_err: } EXPORT_SYMBOL(ttm_object_file_init); -struct ttm_object_device *ttm_object_device_init(struct ttm_mem_global - *mem_glob, - unsigned int hash_order) +struct ttm_object_device * +ttm_object_device_init(struct ttm_mem_global *mem_glob, + unsigned int hash_order, + const struct dma_buf_ops *ops) { struct ttm_object_device *tdev = kmalloc(sizeof(*tdev), GFP_KERNEL); int ret; @@ -425,13 +504,20 @@ struct ttm_object_device *ttm_object_device_init(struct ttm_mem_global return NULL; tdev->mem_glob = mem_glob; - rwlock_init(&tdev->object_lock); + spin_lock_init(&tdev->object_lock); atomic_set(&tdev->object_count, 0); ret = drm_ht_create(&tdev->object_hash, hash_order); + if (ret != 0) + goto out_no_object_hash; - if (likely(ret == 0)) - return tdev; + tdev->ops = *ops; + tdev->dmabuf_release = tdev->ops.release; + tdev->ops.release = ttm_prime_dmabuf_release; + tdev->dma_buf_size = ttm_round_pot(sizeof(struct dma_buf)) + + ttm_round_pot(sizeof(struct file)); + return tdev; +out_no_object_hash: kfree(tdev); return NULL; } @@ -443,10 +529,232 @@ void ttm_object_device_release(struct ttm_object_device **p_tdev) *p_tdev = NULL; - write_lock(&tdev->object_lock); + spin_lock(&tdev->object_lock); drm_ht_remove(&tdev->object_hash); - write_unlock(&tdev->object_lock); + spin_unlock(&tdev->object_lock); kfree(tdev); } EXPORT_SYMBOL(ttm_object_device_release); + +/** + * get_dma_buf_unless_doomed - get a dma_buf reference if possible. + * + * @dma_buf: Non-refcounted pointer to a struct dma-buf. + * + * Obtain a file reference from a lookup structure that doesn't refcount + * the file, but synchronizes with its release method to make sure it has + * not been freed yet. See for example kref_get_unless_zero documentation. + * Returns true if refcounting succeeds, false otherwise. + * + * Nobody really wants this as a public API yet, so let it mature here + * for some time... + */ +static bool __must_check get_dma_buf_unless_doomed(struct dma_buf *dmabuf) +{ + return atomic_long_inc_not_zero(&dmabuf->file->f_count) != 0L; +} + +/** + * ttm_prime_refcount_release - refcount release method for a prime object. + * + * @p_base: Pointer to ttm_base_object pointer. + * + * This is a wrapper that calls the refcount_release founction of the + * underlying object. At the same time it cleans up the prime object. + * This function is called when all references to the base object we + * derive from are gone. + */ +static void ttm_prime_refcount_release(struct ttm_base_object **p_base) +{ + struct ttm_base_object *base = *p_base; + struct ttm_prime_object *prime; + + *p_base = NULL; + prime = container_of(base, struct ttm_prime_object, base); + BUG_ON(prime->dma_buf != NULL); + mutex_destroy(&prime->mutex); + if (prime->refcount_release) + prime->refcount_release(&base); +} + +/** + * ttm_prime_dmabuf_release - Release method for the dma-bufs we export + * + * @dma_buf: + * + * This function first calls the dma_buf release method the driver + * provides. Then it cleans up our dma_buf pointer used for lookup, + * and finally releases the reference the dma_buf has on our base + * object. + */ +static void ttm_prime_dmabuf_release(struct dma_buf *dma_buf) +{ + struct ttm_prime_object *prime = + (struct ttm_prime_object *) dma_buf->priv; + struct ttm_base_object *base = &prime->base; + struct ttm_object_device *tdev = base->tfile->tdev; + + if (tdev->dmabuf_release) + tdev->dmabuf_release(dma_buf); + mutex_lock(&prime->mutex); + if (prime->dma_buf == dma_buf) + prime->dma_buf = NULL; + mutex_unlock(&prime->mutex); + ttm_mem_global_free(tdev->mem_glob, tdev->dma_buf_size); + ttm_base_object_unref(&base); +} + +/** + * ttm_prime_fd_to_handle - Get a base object handle from a prime fd + * + * @tfile: A struct ttm_object_file identifying the caller. + * @fd: The prime / dmabuf fd. + * @handle: The returned handle. + * + * This function returns a handle to an object that previously exported + * a dma-buf. Note that we don't handle imports yet, because we simply + * have no consumers of that implementation. + */ +int ttm_prime_fd_to_handle(struct ttm_object_file *tfile, + int fd, u32 *handle) +{ + struct ttm_object_device *tdev = tfile->tdev; + struct dma_buf *dma_buf; + struct ttm_prime_object *prime; + struct ttm_base_object *base; + int ret; + + dma_buf = dma_buf_get(fd); + if (IS_ERR(dma_buf)) + return PTR_ERR(dma_buf); + + if (dma_buf->ops != &tdev->ops) + return -ENOSYS; + + prime = (struct ttm_prime_object *) dma_buf->priv; + base = &prime->base; + *handle = base->hash.key; + ret = ttm_ref_object_add(tfile, base, TTM_REF_USAGE, NULL); + + dma_buf_put(dma_buf); + + return ret; +} +EXPORT_SYMBOL_GPL(ttm_prime_fd_to_handle); + +/** + * ttm_prime_handle_to_fd - Return a dma_buf fd from a ttm prime object + * + * @tfile: Struct ttm_object_file identifying the caller. + * @handle: Handle to the object we're exporting from. + * @flags: flags for dma-buf creation. We just pass them on. + * @prime_fd: The returned file descriptor. + * + */ +int ttm_prime_handle_to_fd(struct ttm_object_file *tfile, + uint32_t handle, uint32_t flags, + int *prime_fd) +{ + struct ttm_object_device *tdev = tfile->tdev; + struct ttm_base_object *base; + struct dma_buf *dma_buf; + struct ttm_prime_object *prime; + int ret; + + base = ttm_base_object_lookup(tfile, handle); + if (unlikely(base == NULL || + base->object_type != ttm_prime_type)) { + ret = -ENOENT; + goto out_unref; + } + + prime = container_of(base, struct ttm_prime_object, base); + if (unlikely(!base->shareable)) { + ret = -EPERM; + goto out_unref; + } + + ret = mutex_lock_interruptible(&prime->mutex); + if (unlikely(ret != 0)) { + ret = -ERESTARTSYS; + goto out_unref; + } + + dma_buf = prime->dma_buf; + if (!dma_buf || !get_dma_buf_unless_doomed(dma_buf)) { + + /* + * Need to create a new dma_buf, with memory accounting. + */ + ret = ttm_mem_global_alloc(tdev->mem_glob, tdev->dma_buf_size, + false, true); + if (unlikely(ret != 0)) { + mutex_unlock(&prime->mutex); + goto out_unref; + } + + dma_buf = dma_buf_export(prime, &tdev->ops, + prime->size, flags); + if (IS_ERR(dma_buf)) { + ret = PTR_ERR(dma_buf); + ttm_mem_global_free(tdev->mem_glob, + tdev->dma_buf_size); + mutex_unlock(&prime->mutex); + goto out_unref; + } + + /* + * dma_buf has taken the base object reference + */ + base = NULL; + prime->dma_buf = dma_buf; + } + mutex_unlock(&prime->mutex); + + ret = dma_buf_fd(dma_buf, flags); + if (ret >= 0) { + *prime_fd = ret; + ret = 0; + } else + dma_buf_put(dma_buf); + +out_unref: + if (base) + ttm_base_object_unref(&base); + return ret; +} +EXPORT_SYMBOL_GPL(ttm_prime_handle_to_fd); + +/** + * ttm_prime_object_init - Initialize a ttm_prime_object + * + * @tfile: struct ttm_object_file identifying the caller + * @size: The size of the dma_bufs we export. + * @prime: The object to be initialized. + * @shareable: See ttm_base_object_init + * @type: See ttm_base_object_init + * @refcount_release: See ttm_base_object_init + * @ref_obj_release: See ttm_base_object_init + * + * Initializes an object which is compatible with the drm_prime model + * for data sharing between processes and devices. + */ +int ttm_prime_object_init(struct ttm_object_file *tfile, size_t size, + struct ttm_prime_object *prime, bool shareable, + enum ttm_object_type type, + void (*refcount_release) (struct ttm_base_object **), + void (*ref_obj_release) (struct ttm_base_object *, + enum ttm_ref_type ref_type)) +{ + mutex_init(&prime->mutex); + prime->size = PAGE_ALIGN(size); + prime->real_type = type; + prime->dma_buf = NULL; + prime->refcount_release = refcount_release; + return ttm_base_object_init(tfile, &prime->base, shareable, + ttm_prime_type, + ttm_prime_refcount_release, + ref_obj_release); +} +EXPORT_SYMBOL(ttm_prime_object_init); diff --git a/drivers/gpu/drm/ttm/ttm_page_alloc.c b/drivers/gpu/drm/ttm/ttm_page_alloc.c index 499debda791..863bef9f923 100644 --- a/drivers/gpu/drm/ttm/ttm_page_alloc.c +++ b/drivers/gpu/drm/ttm/ttm_page_alloc.c @@ -30,6 +30,9 @@ * - Use page->lru to keep a free list * - doesn't track currently in use pages */ + +#define pr_fmt(fmt) "[TTM] " fmt + #include <linux/list.h> #include <linux/spinlock.h> #include <linux/highmem.h> @@ -42,8 +45,8 @@ #include <linux/atomic.h> -#include "ttm/ttm_bo_driver.h" -#include "ttm/ttm_page_alloc.h" +#include <drm/ttm/ttm_bo_driver.h> +#include <drm/ttm/ttm_page_alloc.h> #ifdef TTM_HAS_AGP #include <asm/agp.h> @@ -167,18 +170,13 @@ static ssize_t ttm_pool_store(struct kobject *kobj, m->options.small = val; else if (attr == &ttm_page_pool_alloc_size) { if (val > NUM_PAGES_TO_ALLOC*8) { - printk(KERN_ERR TTM_PFX - "Setting allocation size to %lu " - "is not allowed. Recommended size is " - "%lu\n", + pr_err("Setting allocation size to %lu is not allowed. Recommended size is %lu\n", NUM_PAGES_TO_ALLOC*(PAGE_SIZE >> 7), NUM_PAGES_TO_ALLOC*(PAGE_SIZE >> 10)); return size; } else if (val > NUM_PAGES_TO_ALLOC) { - printk(KERN_WARNING TTM_PFX - "Setting allocation size to " - "larger than %lu is not recommended.\n", - NUM_PAGES_TO_ALLOC*(PAGE_SIZE >> 10)); + pr_warn("Setting allocation size to larger than %lu is not recommended\n", + NUM_PAGES_TO_ALLOC*(PAGE_SIZE >> 10)); } m->options.alloc_size = val; } @@ -279,8 +277,7 @@ static void ttm_pages_put(struct page *pages[], unsigned npages) { unsigned i; if (set_pages_array_wb(pages, npages)) - printk(KERN_ERR TTM_PFX "Failed to set %d pages to wb!\n", - npages); + pr_err("Failed to set %d pages to wb!\n", npages); for (i = 0; i < npages; ++i) __free_page(pages[i]); } @@ -315,8 +312,7 @@ static int ttm_page_pool_free(struct ttm_page_pool *pool, unsigned nr_free) pages_to_free = kmalloc(npages_to_free * sizeof(struct page *), GFP_KERNEL); if (!pages_to_free) { - printk(KERN_ERR TTM_PFX - "Failed to allocate memory for pool free operation.\n"); + pr_err("Failed to allocate memory for pool free operation\n"); return 0; } @@ -381,28 +377,26 @@ out: return nr_free; } -/* Get good estimation how many pages are free in pools */ -static int ttm_pool_get_num_unused_pages(void) -{ - unsigned i; - int total = 0; - for (i = 0; i < NUM_POOLS; ++i) - total += _manager->pools[i].npages; - - return total; -} - /** * Callback for mm to request pool to reduce number of page held. + * + * XXX: (dchinner) Deadlock warning! + * + * ttm_page_pool_free() does memory allocation using GFP_KERNEL. that means + * this can deadlock when called a sc->gfp_mask that is not equal to + * GFP_KERNEL. + * + * This code is crying out for a shrinker per pool.... */ -static int ttm_pool_mm_shrink(struct shrinker *shrink, - struct shrink_control *sc) +static unsigned long +ttm_pool_shrink_scan(struct shrinker *shrink, struct shrink_control *sc) { static atomic_t start_pool = ATOMIC_INIT(0); unsigned i; unsigned pool_offset = atomic_add_return(1, &start_pool); struct ttm_page_pool *pool; int shrink_pages = sc->nr_to_scan; + unsigned long freed = 0; pool_offset = pool_offset % NUM_POOLS; /* select start pool in round robin fashion */ @@ -412,14 +406,28 @@ static int ttm_pool_mm_shrink(struct shrinker *shrink, break; pool = &_manager->pools[(i + pool_offset)%NUM_POOLS]; shrink_pages = ttm_page_pool_free(pool, nr_free); + freed += nr_free - shrink_pages; } - /* return estimated number of unused pages in pool */ - return ttm_pool_get_num_unused_pages(); + return freed; +} + + +static unsigned long +ttm_pool_shrink_count(struct shrinker *shrink, struct shrink_control *sc) +{ + unsigned i; + unsigned long count = 0; + + for (i = 0; i < NUM_POOLS; ++i) + count += _manager->pools[i].npages; + + return count; } static void ttm_pool_mm_shrink_init(struct ttm_pool_manager *manager) { - manager->mm_shrink.shrink = &ttm_pool_mm_shrink; + manager->mm_shrink.count_objects = ttm_pool_shrink_count; + manager->mm_shrink.scan_objects = ttm_pool_shrink_scan; manager->mm_shrink.seeks = 1; register_shrinker(&manager->mm_shrink); } @@ -438,16 +446,12 @@ static int ttm_set_pages_caching(struct page **pages, case tt_uncached: r = set_pages_array_uc(pages, cpages); if (r) - printk(KERN_ERR TTM_PFX - "Failed to set %d pages to uc!\n", - cpages); + pr_err("Failed to set %d pages to uc!\n", cpages); break; case tt_wc: r = set_pages_array_wc(pages, cpages); if (r) - printk(KERN_ERR TTM_PFX - "Failed to set %d pages to wc!\n", - cpages); + pr_err("Failed to set %d pages to wc!\n", cpages); break; default: break; @@ -492,8 +496,7 @@ static int ttm_alloc_new_pages(struct list_head *pages, gfp_t gfp_flags, caching_array = kmalloc(max_cpages*sizeof(struct page *), GFP_KERNEL); if (!caching_array) { - printk(KERN_ERR TTM_PFX - "Unable to allocate table for new pages."); + pr_err("Unable to allocate table for new pages\n"); return -ENOMEM; } @@ -501,7 +504,7 @@ static int ttm_alloc_new_pages(struct list_head *pages, gfp_t gfp_flags, p = alloc_page(gfp_flags); if (!p) { - printk(KERN_ERR TTM_PFX "Unable to get page %u.\n", i); + pr_err("Unable to get page %u\n", i); /* store already allocated pages in the pool after * setting the caching state */ @@ -599,8 +602,7 @@ static void ttm_page_pool_fill_locked(struct ttm_page_pool *pool, ++pool->nrefills; pool->npages += alloc_size; } else { - printk(KERN_ERR TTM_PFX - "Failed to fill pool (%p).", pool); + pr_err("Failed to fill pool (%p)\n", pool); /* If we have any pages left put them to the pool. */ list_for_each_entry(p, &pool->list, lru) { ++cpages; @@ -675,9 +677,7 @@ static void ttm_put_pages(struct page **pages, unsigned npages, int flags, for (i = 0; i < npages; i++) { if (pages[i]) { if (page_count(pages[i]) != 1) - printk(KERN_ERR TTM_PFX - "Erroneous page count. " - "Leaking pages.\n"); + pr_err("Erroneous page count. Leaking pages.\n"); __free_page(pages[i]); pages[i] = NULL; } @@ -689,9 +689,7 @@ static void ttm_put_pages(struct page **pages, unsigned npages, int flags, for (i = 0; i < npages; i++) { if (pages[i]) { if (page_count(pages[i]) != 1) - printk(KERN_ERR TTM_PFX - "Erroneous page count. " - "Leaking pages.\n"); + pr_err("Erroneous page count. Leaking pages.\n"); list_add_tail(&pages[i]->lru, &pool->list); pages[i] = NULL; pool->npages++; @@ -740,8 +738,7 @@ static int ttm_get_pages(struct page **pages, unsigned npages, int flags, p = alloc_page(gfp_flags); if (!p) { - printk(KERN_ERR TTM_PFX - "Unable to allocate page."); + pr_err("Unable to allocate page\n"); return -ENOMEM; } @@ -764,7 +761,10 @@ static int ttm_get_pages(struct page **pages, unsigned npages, int flags, /* clear the pages coming from the pool if requested */ if (flags & TTM_PAGE_FLAG_ZERO_ALLOC) { list_for_each_entry(p, &plist, lru) { - clear_page(page_address(p)); + if (PageHighMem(p)) + clear_highpage(p); + else + clear_page(page_address(p)); } } @@ -781,9 +781,7 @@ static int ttm_get_pages(struct page **pages, unsigned npages, int flags, if (r) { /* If there is any pages in the list put them back to * the pool. */ - printk(KERN_ERR TTM_PFX - "Failed to allocate extra pages " - "for large request."); + pr_err("Failed to allocate extra pages for large request\n"); ttm_put_pages(pages, count, flags, cstate); return r; } @@ -809,7 +807,7 @@ int ttm_page_alloc_init(struct ttm_mem_global *glob, unsigned max_pages) WARN_ON(_manager); - printk(KERN_INFO TTM_PFX "Initializing pool allocator.\n"); + pr_info("Initializing pool allocator\n"); _manager = kzalloc(sizeof(*_manager), GFP_KERNEL); @@ -844,7 +842,7 @@ void ttm_page_alloc_fini(void) { int i; - printk(KERN_INFO TTM_PFX "Finalizing pool allocator.\n"); + pr_info("Finalizing pool allocator\n"); ttm_pool_mm_shrink_fini(_manager); for (i = 0; i < NUM_POOLS; ++i) diff --git a/drivers/gpu/drm/ttm/ttm_page_alloc_dma.c b/drivers/gpu/drm/ttm/ttm_page_alloc_dma.c index 0c46d8cdc6e..fb8259f6983 100644 --- a/drivers/gpu/drm/ttm/ttm_page_alloc_dma.c +++ b/drivers/gpu/drm/ttm/ttm_page_alloc_dma.c @@ -33,6 +33,9 @@ * when freed). */ +#if defined(CONFIG_SWIOTLB) || defined(CONFIG_INTEL_IOMMU) +#define pr_fmt(fmt) "[TTM] " fmt + #include <linux/dma-mapping.h> #include <linux/list.h> #include <linux/seq_file.h> /* for seq_printf */ @@ -45,8 +48,8 @@ #include <linux/atomic.h> #include <linux/device.h> #include <linux/kthread.h> -#include "ttm/ttm_bo_driver.h" -#include "ttm/ttm_page_alloc.h" +#include <drm/ttm/ttm_bo_driver.h> +#include <drm/ttm/ttm_page_alloc.h> #ifdef TTM_HAS_AGP #include <asm/agp.h> #endif @@ -221,18 +224,13 @@ static ssize_t ttm_pool_store(struct kobject *kobj, struct attribute *attr, m->options.small = val; else if (attr == &ttm_page_pool_alloc_size) { if (val > NUM_PAGES_TO_ALLOC*8) { - printk(KERN_ERR TTM_PFX - "Setting allocation size to %lu " - "is not allowed. Recommended size is " - "%lu\n", + pr_err("Setting allocation size to %lu is not allowed. Recommended size is %lu\n", NUM_PAGES_TO_ALLOC*(PAGE_SIZE >> 7), NUM_PAGES_TO_ALLOC*(PAGE_SIZE >> 10)); return size; } else if (val > NUM_PAGES_TO_ALLOC) { - printk(KERN_WARNING TTM_PFX - "Setting allocation size to " - "larger than %lu is not recommended.\n", - NUM_PAGES_TO_ALLOC*(PAGE_SIZE >> 10)); + pr_warn("Setting allocation size to larger than %lu is not recommended\n", + NUM_PAGES_TO_ALLOC*(PAGE_SIZE >> 10)); } m->options.alloc_size = val; } @@ -313,15 +311,13 @@ static int ttm_set_pages_caching(struct dma_pool *pool, if (pool->type & IS_UC) { r = set_pages_array_uc(pages, cpages); if (r) - pr_err(TTM_PFX - "%s: Failed to set %d pages to uc!\n", + pr_err("%s: Failed to set %d pages to uc!\n", pool->dev_name, cpages); } if (pool->type & IS_WC) { r = set_pages_array_wc(pages, cpages); if (r) - pr_err(TTM_PFX - "%s: Failed to set %d pages to wc!\n", + pr_err("%s: Failed to set %d pages to wc!\n", pool->dev_name, cpages); } return r; @@ -387,8 +383,8 @@ static void ttm_dma_pages_put(struct dma_pool *pool, struct list_head *d_pages, /* Don't set WB on WB page pool. */ if (npages && !(pool->type & IS_CACHED) && set_pages_array_wb(pages, npages)) - pr_err(TTM_PFX "%s: Failed to set %d pages to wb!\n", - pool->dev_name, npages); + pr_err("%s: Failed to set %d pages to wb!\n", + pool->dev_name, npages); list_for_each_entry_safe(d_page, tmp, d_pages, page_list) { list_del(&d_page->page_list); @@ -400,8 +396,8 @@ static void ttm_dma_page_put(struct dma_pool *pool, struct dma_page *d_page) { /* Don't set WB on WB page pool. */ if (!(pool->type & IS_CACHED) && set_pages_array_wb(&d_page->p, 1)) - pr_err(TTM_PFX "%s: Failed to set %d pages to wb!\n", - pool->dev_name, 1); + pr_err("%s: Failed to set %d pages to wb!\n", + pool->dev_name, 1); list_del(&d_page->page_list); __ttm_dma_free_page(pool, d_page); @@ -430,17 +426,16 @@ static unsigned ttm_dma_page_pool_free(struct dma_pool *pool, unsigned nr_free) #if 0 if (nr_free > 1) { pr_debug("%s: (%s:%d) Attempting to free %d (%d) pages\n", - pool->dev_name, pool->name, current->pid, - npages_to_free, nr_free); + pool->dev_name, pool->name, current->pid, + npages_to_free, nr_free); } #endif pages_to_free = kmalloc(npages_to_free * sizeof(struct page *), GFP_KERNEL); if (!pages_to_free) { - pr_err(TTM_PFX - "%s: Failed to allocate memory for pool free operation.\n", - pool->dev_name); + pr_err("%s: Failed to allocate memory for pool free operation\n", + pool->dev_name); return 0; } INIT_LIST_HEAD(&d_pages); @@ -723,23 +718,21 @@ static int ttm_dma_pool_alloc_new_pages(struct dma_pool *pool, caching_array = kmalloc(max_cpages*sizeof(struct page *), GFP_KERNEL); if (!caching_array) { - pr_err(TTM_PFX - "%s: Unable to allocate table for new pages.", - pool->dev_name); + pr_err("%s: Unable to allocate table for new pages\n", + pool->dev_name); return -ENOMEM; } if (count > 1) { pr_debug("%s: (%s:%d) Getting %d pages\n", - pool->dev_name, pool->name, current->pid, - count); + pool->dev_name, pool->name, current->pid, count); } for (i = 0, cpages = 0; i < count; ++i) { dma_p = __ttm_dma_alloc_page(pool); if (!dma_p) { - pr_err(TTM_PFX "%s: Unable to get page %u.\n", - pool->dev_name, i); + pr_err("%s: Unable to get page %u\n", + pool->dev_name, i); /* store already allocated pages in the pool after * setting the caching state */ @@ -821,8 +814,8 @@ static int ttm_dma_page_pool_fill_locked(struct dma_pool *pool, struct dma_page *d_page; unsigned cpages = 0; - pr_err(TTM_PFX "%s: Failed to fill %s pool (r:%d)!\n", - pool->dev_name, pool->name, r); + pr_err("%s: Failed to fill %s pool (r:%d)!\n", + pool->dev_name, pool->name, r); list_for_each_entry(d_page, &d_pages, page_list) { cpages++; @@ -926,19 +919,6 @@ int ttm_dma_populate(struct ttm_dma_tt *ttm_dma, struct device *dev) } EXPORT_SYMBOL_GPL(ttm_dma_populate); -/* Get good estimation how many pages are free in pools */ -static int ttm_dma_pool_get_num_unused_pages(void) -{ - struct device_pools *p; - unsigned total = 0; - - mutex_lock(&_manager->lock); - list_for_each_entry(p, &_manager->pools, pools) - total += p->pool->npages_free; - mutex_unlock(&_manager->lock); - return total; -} - /* Put all pages in pages list to correct pool to wait for reuse */ void ttm_dma_unpopulate(struct ttm_dma_tt *ttm_dma, struct device *dev) { @@ -1010,18 +990,29 @@ EXPORT_SYMBOL_GPL(ttm_dma_unpopulate); /** * Callback for mm to request pool to reduce number of page held. + * + * XXX: (dchinner) Deadlock warning! + * + * ttm_dma_page_pool_free() does GFP_KERNEL memory allocation, and so attention + * needs to be paid to sc->gfp_mask to determine if this can be done or not. + * GFP_KERNEL memory allocation in a GFP_ATOMIC reclaim context woul dbe really + * bad. + * + * I'm getting sadder as I hear more pathetical whimpers about needing per-pool + * shrinkers */ -static int ttm_dma_pool_mm_shrink(struct shrinker *shrink, - struct shrink_control *sc) +static unsigned long +ttm_dma_pool_shrink_scan(struct shrinker *shrink, struct shrink_control *sc) { static atomic_t start_pool = ATOMIC_INIT(0); unsigned idx = 0; unsigned pool_offset = atomic_add_return(1, &start_pool); unsigned shrink_pages = sc->nr_to_scan; struct device_pools *p; + unsigned long freed = 0; if (list_empty(&_manager->pools)) - return 0; + return SHRINK_STOP; mutex_lock(&_manager->lock); pool_offset = pool_offset % _manager->npools; @@ -1037,18 +1028,33 @@ static int ttm_dma_pool_mm_shrink(struct shrinker *shrink, continue; nr_free = shrink_pages; shrink_pages = ttm_dma_page_pool_free(p->pool, nr_free); + freed += nr_free - shrink_pages; + pr_debug("%s: (%s:%d) Asked to shrink %d, have %d more to go\n", - p->pool->dev_name, p->pool->name, current->pid, nr_free, - shrink_pages); + p->pool->dev_name, p->pool->name, current->pid, + nr_free, shrink_pages); } mutex_unlock(&_manager->lock); - /* return estimated number of unused pages in pool */ - return ttm_dma_pool_get_num_unused_pages(); + return freed; +} + +static unsigned long +ttm_dma_pool_shrink_count(struct shrinker *shrink, struct shrink_control *sc) +{ + struct device_pools *p; + unsigned long count = 0; + + mutex_lock(&_manager->lock); + list_for_each_entry(p, &_manager->pools, pools) + count += p->pool->npages_free; + mutex_unlock(&_manager->lock); + return count; } static void ttm_dma_pool_mm_shrink_init(struct ttm_pool_manager *manager) { - manager->mm_shrink.shrink = &ttm_dma_pool_mm_shrink; + manager->mm_shrink.count_objects = ttm_dma_pool_shrink_count; + manager->mm_shrink.scan_objects = &ttm_dma_pool_shrink_scan; manager->mm_shrink.seeks = 1; register_shrinker(&manager->mm_shrink); } @@ -1064,11 +1070,11 @@ int ttm_dma_page_alloc_init(struct ttm_mem_global *glob, unsigned max_pages) WARN_ON(_manager); - printk(KERN_INFO TTM_PFX "Initializing DMA pool allocator.\n"); + pr_info("Initializing DMA pool allocator\n"); _manager = kzalloc(sizeof(*_manager), GFP_KERNEL); if (!_manager) - goto err_manager; + goto err; mutex_init(&_manager->lock); INIT_LIST_HEAD(&_manager->pools); @@ -1086,9 +1092,6 @@ int ttm_dma_page_alloc_init(struct ttm_mem_global *glob, unsigned max_pages) } ttm_dma_pool_mm_shrink_init(_manager); return 0; -err_manager: - kfree(_manager); - _manager = NULL; err: return ret; } @@ -1097,7 +1100,7 @@ void ttm_dma_page_alloc_fini(void) { struct device_pools *p, *t; - printk(KERN_INFO TTM_PFX "Finalizing DMA pool allocator.\n"); + pr_info("Finalizing DMA pool allocator\n"); ttm_dma_pool_mm_shrink_fini(_manager); list_for_each_entry_safe_reverse(p, t, &_manager->pools, pools) { @@ -1140,3 +1143,5 @@ int ttm_dma_page_alloc_debugfs(struct seq_file *m, void *data) return 0; } EXPORT_SYMBOL_GPL(ttm_dma_page_alloc_debugfs); + +#endif diff --git a/drivers/gpu/drm/ttm/ttm_tt.c b/drivers/gpu/drm/ttm/ttm_tt.c index 2f75d203a2b..75f31909004 100644 --- a/drivers/gpu/drm/ttm/ttm_tt.c +++ b/drivers/gpu/drm/ttm/ttm_tt.c @@ -28,6 +28,8 @@ * Authors: Thomas Hellstrom <thellstrom-at-vmware-dot-com> */ +#define pr_fmt(fmt) "[TTM] " fmt + #include <linux/sched.h> #include <linux/highmem.h> #include <linux/pagemap.h> @@ -36,12 +38,12 @@ #include <linux/swap.h> #include <linux/slab.h> #include <linux/export.h> -#include "drm_cache.h" -#include "drm_mem_util.h" -#include "ttm/ttm_module.h" -#include "ttm/ttm_bo_driver.h" -#include "ttm/ttm_placement.h" -#include "ttm/ttm_page_alloc.h" +#include <drm/drm_cache.h> +#include <drm/drm_mem_util.h> +#include <drm/ttm/ttm_module.h> +#include <drm/ttm/ttm_bo_driver.h> +#include <drm/ttm/ttm_placement.h> +#include <drm/ttm/ttm_page_alloc.h> /** * Allocates storage for pointers to the pages that back the ttm. @@ -168,9 +170,8 @@ void ttm_tt_destroy(struct ttm_tt *ttm) ttm_tt_unbind(ttm); } - if (likely(ttm->pages != NULL)) { - ttm->bdev->driver->ttm_tt_unpopulate(ttm); - } + if (ttm->state == tt_unbound) + ttm_tt_unpopulate(ttm); if (!(ttm->page_flags & TTM_PAGE_FLAG_PERSISTENT_SWAP) && ttm->swap_storage) @@ -196,7 +197,7 @@ int ttm_tt_init(struct ttm_tt *ttm, struct ttm_bo_device *bdev, ttm_tt_alloc_page_directory(ttm); if (!ttm->pages) { ttm_tt_destroy(ttm); - printk(KERN_ERR TTM_PFX "Failed allocating page table\n"); + pr_err("Failed allocating page table\n"); return -ENOMEM; } return 0; @@ -229,7 +230,7 @@ int ttm_dma_tt_init(struct ttm_dma_tt *ttm_dma, struct ttm_bo_device *bdev, ttm_dma_tt_alloc_page_directory(ttm_dma); if (!ttm->pages || !ttm_dma->dma_address) { ttm_tt_destroy(ttm); - printk(KERN_ERR TTM_PFX "Failed allocating page table\n"); + pr_err("Failed allocating page table\n"); return -ENOMEM; } return 0; @@ -288,15 +289,13 @@ int ttm_tt_swapin(struct ttm_tt *ttm) struct file *swap_storage; struct page *from_page; struct page *to_page; - void *from_virtual; - void *to_virtual; int i; int ret = -ENOMEM; swap_storage = ttm->swap_storage; BUG_ON(swap_storage == NULL); - swap_space = swap_storage->f_path.dentry->d_inode->i_mapping; + swap_space = file_inode(swap_storage)->i_mapping; for (i = 0; i < ttm->num_pages; ++i) { from_page = shmem_read_mapping_page(swap_space, i); @@ -308,13 +307,7 @@ int ttm_tt_swapin(struct ttm_tt *ttm) if (unlikely(to_page == NULL)) goto out_err; - preempt_disable(); - from_virtual = kmap_atomic(from_page, KM_USER0); - to_virtual = kmap_atomic(to_page, KM_USER1); - memcpy(to_virtual, from_virtual, PAGE_SIZE); - kunmap_atomic(to_virtual, KM_USER1); - kunmap_atomic(from_virtual, KM_USER0); - preempt_enable(); + copy_highpage(to_page, from_page); page_cache_release(from_page); } @@ -334,8 +327,6 @@ int ttm_tt_swapout(struct ttm_tt *ttm, struct file *persistent_swap_storage) struct file *swap_storage; struct page *from_page; struct page *to_page; - void *from_virtual; - void *to_virtual; int i; int ret = -ENOMEM; @@ -347,13 +338,13 @@ int ttm_tt_swapout(struct ttm_tt *ttm, struct file *persistent_swap_storage) ttm->num_pages << PAGE_SHIFT, 0); if (unlikely(IS_ERR(swap_storage))) { - printk(KERN_ERR "Failed allocating swap storage.\n"); + pr_err("Failed allocating swap storage\n"); return PTR_ERR(swap_storage); } } else swap_storage = persistent_swap_storage; - swap_space = swap_storage->f_path.dentry->d_inode->i_mapping; + swap_space = file_inode(swap_storage)->i_mapping; for (i = 0; i < ttm->num_pages; ++i) { from_page = ttm->pages[i]; @@ -364,19 +355,13 @@ int ttm_tt_swapout(struct ttm_tt *ttm, struct file *persistent_swap_storage) ret = PTR_ERR(to_page); goto out_err; } - preempt_disable(); - from_virtual = kmap_atomic(from_page, KM_USER0); - to_virtual = kmap_atomic(to_page, KM_USER1); - memcpy(to_virtual, from_virtual, PAGE_SIZE); - kunmap_atomic(to_virtual, KM_USER1); - kunmap_atomic(from_virtual, KM_USER0); - preempt_enable(); + copy_highpage(to_page, from_page); set_page_dirty(to_page); mark_page_accessed(to_page); page_cache_release(to_page); } - ttm->bdev->driver->ttm_tt_unpopulate(ttm); + ttm_tt_unpopulate(ttm); ttm->swap_storage = swap_storage; ttm->page_flags |= TTM_PAGE_FLAG_SWAPPED; if (persistent_swap_storage) @@ -389,3 +374,26 @@ out_err: return ret; } + +static void ttm_tt_clear_mapping(struct ttm_tt *ttm) +{ + pgoff_t i; + struct page **page = ttm->pages; + + if (ttm->page_flags & TTM_PAGE_FLAG_SG) + return; + + for (i = 0; i < ttm->num_pages; ++i) { + (*page)->mapping = NULL; + (*page++)->index = 0; + } +} + +void ttm_tt_unpopulate(struct ttm_tt *ttm) +{ + if (ttm->state == tt_unpopulated) + return; + + ttm_tt_clear_mapping(ttm); + ttm->bdev->driver->ttm_tt_unpopulate(ttm); +} |