diff options
Diffstat (limited to 'drivers/gpu/drm/i915/i915_gem.c')
| -rw-r--r-- | drivers/gpu/drm/i915/i915_gem.c | 1156 |
1 files changed, 648 insertions, 508 deletions
diff --git a/drivers/gpu/drm/i915/i915_gem.c b/drivers/gpu/drm/i915/i915_gem.c index cf4ffbee1c0..29e97c07542 100644 --- a/drivers/gpu/drm/i915/i915_gem.c +++ b/drivers/gpu/drm/i915/i915_gem.c @@ -37,7 +37,9 @@ #include <linux/intel-gtt.h> static uint32_t i915_gem_get_gtt_alignment(struct drm_gem_object *obj); -static int i915_gem_object_flush_gpu_write_domain(struct drm_gem_object *obj); + +static int i915_gem_object_flush_gpu_write_domain(struct drm_gem_object *obj, + bool pipelined); static void i915_gem_object_flush_gtt_write_domain(struct drm_gem_object *obj); static void i915_gem_object_flush_cpu_write_domain(struct drm_gem_object *obj); static int i915_gem_object_set_to_cpu_domain(struct drm_gem_object *obj, @@ -46,7 +48,8 @@ static int i915_gem_object_set_cpu_read_domain_range(struct drm_gem_object *obj, uint64_t offset, uint64_t size); static void i915_gem_object_set_to_full_cpu_read_domain(struct drm_gem_object *obj); -static int i915_gem_object_wait_rendering(struct drm_gem_object *obj); +static int i915_gem_object_wait_rendering(struct drm_gem_object *obj, + bool interruptible); static int i915_gem_object_bind_to_gtt(struct drm_gem_object *obj, unsigned alignment); static void i915_gem_clear_fence_reg(struct drm_gem_object *obj); @@ -55,9 +58,111 @@ static int i915_gem_phys_pwrite(struct drm_device *dev, struct drm_gem_object *o struct drm_file *file_priv); static void i915_gem_free_object_tail(struct drm_gem_object *obj); +static int +i915_gem_object_get_pages(struct drm_gem_object *obj, + gfp_t gfpmask); + +static void +i915_gem_object_put_pages(struct drm_gem_object *obj); + static LIST_HEAD(shrink_list); static DEFINE_SPINLOCK(shrink_list_lock); +/* some bookkeeping */ +static void i915_gem_info_add_obj(struct drm_i915_private *dev_priv, + size_t size) +{ + dev_priv->mm.object_count++; + dev_priv->mm.object_memory += size; +} + +static void i915_gem_info_remove_obj(struct drm_i915_private *dev_priv, + size_t size) +{ + dev_priv->mm.object_count--; + dev_priv->mm.object_memory -= size; +} + +static void i915_gem_info_add_gtt(struct drm_i915_private *dev_priv, + size_t size) +{ + dev_priv->mm.gtt_count++; + dev_priv->mm.gtt_memory += size; +} + +static void i915_gem_info_remove_gtt(struct drm_i915_private *dev_priv, + size_t size) +{ + dev_priv->mm.gtt_count--; + dev_priv->mm.gtt_memory -= size; +} + +static void i915_gem_info_add_pin(struct drm_i915_private *dev_priv, + size_t size) +{ + dev_priv->mm.pin_count++; + dev_priv->mm.pin_memory += size; +} + +static void i915_gem_info_remove_pin(struct drm_i915_private *dev_priv, + size_t size) +{ + dev_priv->mm.pin_count--; + dev_priv->mm.pin_memory -= size; +} + +int +i915_gem_check_is_wedged(struct drm_device *dev) +{ + struct drm_i915_private *dev_priv = dev->dev_private; + struct completion *x = &dev_priv->error_completion; + unsigned long flags; + int ret; + + if (!atomic_read(&dev_priv->mm.wedged)) + return 0; + + ret = wait_for_completion_interruptible(x); + if (ret) + return ret; + + /* Success, we reset the GPU! */ + if (!atomic_read(&dev_priv->mm.wedged)) + return 0; + + /* GPU is hung, bump the completion count to account for + * the token we just consumed so that we never hit zero and + * end up waiting upon a subsequent completion event that + * will never happen. + */ + spin_lock_irqsave(&x->wait.lock, flags); + x->done++; + spin_unlock_irqrestore(&x->wait.lock, flags); + return -EIO; +} + +static int i915_mutex_lock_interruptible(struct drm_device *dev) +{ + struct drm_i915_private *dev_priv = dev->dev_private; + int ret; + + ret = i915_gem_check_is_wedged(dev); + if (ret) + return ret; + + ret = mutex_lock_interruptible(&dev->struct_mutex); + if (ret) + return ret; + + if (atomic_read(&dev_priv->mm.wedged)) { + mutex_unlock(&dev->struct_mutex); + return -EAGAIN; + } + + WARN_ON(i915_verify_lists(dev)); + return 0; +} + static inline bool i915_gem_object_is_inactive(struct drm_i915_gem_object *obj_priv) { @@ -66,7 +171,8 @@ i915_gem_object_is_inactive(struct drm_i915_gem_object *obj_priv) obj_priv->pin_count == 0; } -int i915_gem_do_init(struct drm_device *dev, unsigned long start, +int i915_gem_do_init(struct drm_device *dev, + unsigned long start, unsigned long end) { drm_i915_private_t *dev_priv = dev->dev_private; @@ -80,7 +186,7 @@ int i915_gem_do_init(struct drm_device *dev, unsigned long start, drm_mm_init(&dev_priv->mm.gtt_space, start, end - start); - dev->gtt_total = (uint32_t) (end - start); + dev_priv->mm.gtt_total = end - start; return 0; } @@ -103,14 +209,16 @@ int i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data, struct drm_file *file_priv) { + struct drm_i915_private *dev_priv = dev->dev_private; struct drm_i915_gem_get_aperture *args = data; if (!(dev->driver->driver_features & DRIVER_GEM)) return -ENODEV; - args->aper_size = dev->gtt_total; - args->aper_available_size = (args->aper_size - - atomic_read(&dev->pin_memory)); + mutex_lock(&dev->struct_mutex); + args->aper_size = dev_priv->mm.gtt_total; + args->aper_available_size = args->aper_size - dev_priv->mm.pin_memory; + mutex_unlock(&dev->struct_mutex); return 0; } @@ -265,7 +373,9 @@ i915_gem_shmem_pread_fast(struct drm_device *dev, struct drm_gem_object *obj, user_data = (char __user *) (uintptr_t) args->data_ptr; remain = args->size; - mutex_lock(&dev->struct_mutex); + ret = i915_mutex_lock_interruptible(dev); + if (ret) + return ret; ret = i915_gem_object_get_pages(obj, 0); if (ret != 0) @@ -384,7 +494,9 @@ i915_gem_shmem_pread_slow(struct drm_device *dev, struct drm_gem_object *obj, do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj); - mutex_lock(&dev->struct_mutex); + ret = i915_mutex_lock_interruptible(dev); + if (ret) + goto fail_put_user_pages; ret = i915_gem_object_get_pages_or_evict(obj); if (ret) @@ -464,21 +576,27 @@ i915_gem_pread_ioctl(struct drm_device *dev, void *data, struct drm_i915_gem_pread *args = data; struct drm_gem_object *obj; struct drm_i915_gem_object *obj_priv; - int ret; + int ret = 0; obj = drm_gem_object_lookup(dev, file_priv, args->handle); if (obj == NULL) return -ENOENT; obj_priv = to_intel_bo(obj); - /* Bounds check source. - * - * XXX: This could use review for overflow issues... - */ - if (args->offset > obj->size || args->size > obj->size || - args->offset + args->size > obj->size) { - drm_gem_object_unreference_unlocked(obj); - return -EINVAL; + /* Bounds check source. */ + if (args->offset > obj->size || args->size > obj->size - args->offset) { + ret = -EINVAL; + goto out; + } + + if (args->size == 0) + goto out; + + if (!access_ok(VERIFY_WRITE, + (char __user *)(uintptr_t)args->data_ptr, + args->size)) { + ret = -EFAULT; + goto out; } if (i915_gem_object_needs_bit17_swizzle(obj)) { @@ -490,8 +608,8 @@ i915_gem_pread_ioctl(struct drm_device *dev, void *data, file_priv); } +out: drm_gem_object_unreference_unlocked(obj); - return ret; } @@ -580,11 +698,11 @@ i915_gem_gtt_pwrite_fast(struct drm_device *dev, struct drm_gem_object *obj, user_data = (char __user *) (uintptr_t) args->data_ptr; remain = args->size; - if (!access_ok(VERIFY_READ, user_data, remain)) - return -EFAULT; + ret = i915_mutex_lock_interruptible(dev); + if (ret) + return ret; - mutex_lock(&dev->struct_mutex); ret = i915_gem_object_pin(obj, 0); if (ret) { mutex_unlock(&dev->struct_mutex); @@ -679,7 +797,10 @@ i915_gem_gtt_pwrite_slow(struct drm_device *dev, struct drm_gem_object *obj, goto out_unpin_pages; } - mutex_lock(&dev->struct_mutex); + ret = i915_mutex_lock_interruptible(dev); + if (ret) + goto out_unpin_pages; + ret = i915_gem_object_pin(obj, 0); if (ret) goto out_unlock; @@ -753,7 +874,9 @@ i915_gem_shmem_pwrite_fast(struct drm_device *dev, struct drm_gem_object *obj, user_data = (char __user *) (uintptr_t) args->data_ptr; remain = args->size; - mutex_lock(&dev->struct_mutex); + ret = i915_mutex_lock_interruptible(dev); + if (ret) + return ret; ret = i915_gem_object_get_pages(obj, 0); if (ret != 0) @@ -849,7 +972,9 @@ i915_gem_shmem_pwrite_slow(struct drm_device *dev, struct drm_gem_object *obj, do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj); - mutex_lock(&dev->struct_mutex); + ret = i915_mutex_lock_interruptible(dev); + if (ret) + goto fail_put_user_pages; ret = i915_gem_object_get_pages_or_evict(obj); if (ret) @@ -934,14 +1059,20 @@ i915_gem_pwrite_ioctl(struct drm_device *dev, void *data, return -ENOENT; obj_priv = to_intel_bo(obj); - /* Bounds check destination. - * - * XXX: This could use review for overflow issues... - */ - if (args->offset > obj->size || args->size > obj->size || - args->offset + args->size > obj->size) { - drm_gem_object_unreference_unlocked(obj); - return -EINVAL; + /* Bounds check destination. */ + if (args->offset > obj->size || args->size > obj->size - args->offset) { + ret = -EINVAL; + goto out; + } + + if (args->size == 0) + goto out; + + if (!access_ok(VERIFY_READ, + (char __user *)(uintptr_t)args->data_ptr, + args->size)) { + ret = -EFAULT; + goto out; } /* We can only do the GTT pwrite on untiled buffers, as otherwise @@ -953,7 +1084,7 @@ i915_gem_pwrite_ioctl(struct drm_device *dev, void *data, if (obj_priv->phys_obj) ret = i915_gem_phys_pwrite(dev, obj, args, file_priv); else if (obj_priv->tiling_mode == I915_TILING_NONE && - dev->gtt_total != 0 && + obj_priv->gtt_space && obj->write_domain != I915_GEM_DOMAIN_CPU) { ret = i915_gem_gtt_pwrite_fast(dev, obj, args, file_priv); if (ret == -EFAULT) { @@ -975,8 +1106,8 @@ i915_gem_pwrite_ioctl(struct drm_device *dev, void *data, DRM_INFO("pwrite failed %d\n", ret); #endif +out: drm_gem_object_unreference_unlocked(obj); - return ret; } @@ -1017,14 +1148,14 @@ i915_gem_set_domain_ioctl(struct drm_device *dev, void *data, return -ENOENT; obj_priv = to_intel_bo(obj); - mutex_lock(&dev->struct_mutex); + ret = i915_mutex_lock_interruptible(dev); + if (ret) { + drm_gem_object_unreference_unlocked(obj); + return ret; + } intel_mark_busy(dev, obj); -#if WATCH_BUF - DRM_INFO("set_domain_ioctl %p(%zd), %08x %08x\n", - obj, obj->size, read_domains, write_domain); -#endif if (read_domains & I915_GEM_DOMAIN_GTT) { ret = i915_gem_object_set_to_gtt_domain(obj, write_domain != 0); @@ -1048,7 +1179,6 @@ i915_gem_set_domain_ioctl(struct drm_device *dev, void *data, ret = i915_gem_object_set_to_cpu_domain(obj, write_domain != 0); } - /* Maintain LRU order of "inactive" objects */ if (ret == 0 && i915_gem_object_is_inactive(obj_priv)) list_move_tail(&obj_priv->list, &dev_priv->mm.inactive_list); @@ -1067,27 +1197,23 @@ i915_gem_sw_finish_ioctl(struct drm_device *dev, void *data, { struct drm_i915_gem_sw_finish *args = data; struct drm_gem_object *obj; - struct drm_i915_gem_object *obj_priv; int ret = 0; if (!(dev->driver->driver_features & DRIVER_GEM)) return -ENODEV; - mutex_lock(&dev->struct_mutex); obj = drm_gem_object_lookup(dev, file_priv, args->handle); - if (obj == NULL) { - mutex_unlock(&dev->struct_mutex); + if (obj == NULL) return -ENOENT; - } -#if WATCH_BUF - DRM_INFO("%s: sw_finish %d (%p %zd)\n", - __func__, args->handle, obj, obj->size); -#endif - obj_priv = to_intel_bo(obj); + ret = i915_mutex_lock_interruptible(dev); + if (ret) { + drm_gem_object_unreference_unlocked(obj); + return ret; + } /* Pinned buffers may be scanout, so flush the cache */ - if (obj_priv->pin_count) + if (to_intel_bo(obj)->pin_count) i915_gem_object_flush_cpu_write_domain(obj); drm_gem_object_unreference(obj); @@ -1179,7 +1305,7 @@ int i915_gem_fault(struct vm_area_struct *vma, struct vm_fault *vmf) /* Need a new fence register? */ if (obj_priv->tiling_mode != I915_TILING_NONE) { - ret = i915_gem_object_get_fence_reg(obj); + ret = i915_gem_object_get_fence_reg(obj, true); if (ret) goto unlock; } @@ -1244,7 +1370,7 @@ i915_gem_create_mmap_offset(struct drm_gem_object *obj) obj->size / PAGE_SIZE, 0, 0); if (!list->file_offset_node) { DRM_ERROR("failed to allocate offset for bo %d\n", obj->name); - ret = -ENOMEM; + ret = -ENOSPC; goto out_free_list; } @@ -1256,9 +1382,9 @@ i915_gem_create_mmap_offset(struct drm_gem_object *obj) } list->hash.key = list->file_offset_node->start; - if (drm_ht_insert_item(&mm->offset_hash, &list->hash)) { + ret = drm_ht_insert_item(&mm->offset_hash, &list->hash); + if (ret) { DRM_ERROR("failed to add to map hash\n"); - ret = -ENOMEM; goto out_free_mm; } @@ -1343,14 +1469,14 @@ i915_gem_get_gtt_alignment(struct drm_gem_object *obj) * Minimum alignment is 4k (GTT page size), but might be greater * if a fence register is needed for the object. */ - if (IS_I965G(dev) || obj_priv->tiling_mode == I915_TILING_NONE) + if (INTEL_INFO(dev)->gen >= 4 || obj_priv->tiling_mode == I915_TILING_NONE) return 4096; /* * Previous chips need to be aligned to the size of the smallest * fence register that can contain the object. */ - if (IS_I9XX(dev)) + if (INTEL_INFO(dev)->gen == 3) start = 1024*1024; else start = 512*1024; @@ -1392,7 +1518,11 @@ i915_gem_mmap_gtt_ioctl(struct drm_device *dev, void *data, if (obj == NULL) return -ENOENT; - mutex_lock(&dev->struct_mutex); + ret = i915_mutex_lock_interruptible(dev); + if (ret) { + drm_gem_object_unreference_unlocked(obj); + return ret; + } obj_priv = to_intel_bo(obj); @@ -1434,7 +1564,7 @@ i915_gem_mmap_gtt_ioctl(struct drm_device *dev, void *data, return 0; } -void +static void i915_gem_object_put_pages(struct drm_gem_object *obj) { struct drm_i915_gem_object *obj_priv = to_intel_bo(obj); @@ -1468,13 +1598,24 @@ i915_gem_object_put_pages(struct drm_gem_object *obj) obj_priv->pages = NULL; } +static uint32_t +i915_gem_next_request_seqno(struct drm_device *dev, + struct intel_ring_buffer *ring) +{ + drm_i915_private_t *dev_priv = dev->dev_private; + + ring->outstanding_lazy_request = true; + return dev_priv->next_seqno; +} + static void -i915_gem_object_move_to_active(struct drm_gem_object *obj, uint32_t seqno, +i915_gem_object_move_to_active(struct drm_gem_object *obj, struct intel_ring_buffer *ring) { struct drm_device *dev = obj->dev; - drm_i915_private_t *dev_priv = dev->dev_private; struct drm_i915_gem_object *obj_priv = to_intel_bo(obj); + uint32_t seqno = i915_gem_next_request_seqno(dev, ring); + BUG_ON(ring == NULL); obj_priv->ring = ring; @@ -1483,10 +1624,9 @@ i915_gem_object_move_to_active(struct drm_gem_object *obj, uint32_t seqno, drm_gem_object_reference(obj); obj_priv->active = 1; } + /* Move from whatever list we were on to the tail of execution. */ - spin_lock(&dev_priv->mm.active_list_lock); list_move_tail(&obj_priv->list, &ring->active_list); - spin_unlock(&dev_priv->mm.active_list_lock); obj_priv->last_rendering_seqno = seqno; } @@ -1536,9 +1676,8 @@ i915_gem_object_move_to_inactive(struct drm_gem_object *obj) drm_i915_private_t *dev_priv = dev->dev_private; struct drm_i915_gem_object *obj_priv = to_intel_bo(obj); - i915_verify_inactive(dev, __FILE__, __LINE__); if (obj_priv->pin_count != 0) - list_del_init(&obj_priv->list); + list_move_tail(&obj_priv->list, &dev_priv->mm.pinned_list); else list_move_tail(&obj_priv->list, &dev_priv->mm.inactive_list); @@ -1550,12 +1689,12 @@ i915_gem_object_move_to_inactive(struct drm_gem_object *obj) obj_priv->active = 0; drm_gem_object_unreference(obj); } - i915_verify_inactive(dev, __FILE__, __LINE__); + WARN_ON(i915_verify_lists(dev)); } static void i915_gem_process_flushing_list(struct drm_device *dev, - uint32_t flush_domains, uint32_t seqno, + uint32_t flush_domains, struct intel_ring_buffer *ring) { drm_i915_private_t *dev_priv = dev->dev_private; @@ -1566,14 +1705,13 @@ i915_gem_process_flushing_list(struct drm_device *dev, gpu_write_list) { struct drm_gem_object *obj = &obj_priv->base; - if ((obj->write_domain & flush_domains) == - obj->write_domain && - obj_priv->ring->ring_flag == ring->ring_flag) { + if (obj->write_domain & flush_domains && + obj_priv->ring == ring) { uint32_t old_write_domain = obj->write_domain; obj->write_domain = 0; list_del_init(&obj_priv->gpu_write_list); - i915_gem_object_move_to_active(obj, seqno, ring); + i915_gem_object_move_to_active(obj, ring); /* update the fence lru list */ if (obj_priv->fence_reg != I915_FENCE_REG_NONE) { @@ -1591,23 +1729,27 @@ i915_gem_process_flushing_list(struct drm_device *dev, } uint32_t -i915_add_request(struct drm_device *dev, struct drm_file *file_priv, - uint32_t flush_domains, struct intel_ring_buffer *ring) +i915_add_request(struct drm_device *dev, + struct drm_file *file, + struct drm_i915_gem_request *request, + struct intel_ring_buffer *ring) { drm_i915_private_t *dev_priv = dev->dev_private; - struct drm_i915_file_private *i915_file_priv = NULL; - struct drm_i915_gem_request *request; + struct drm_i915_file_private *file_priv = NULL; uint32_t seqno; int was_empty; - if (file_priv != NULL) - i915_file_priv = file_priv->driver_priv; + if (file != NULL) + file_priv = file->driver_priv; - request = kzalloc(sizeof(*request), GFP_KERNEL); - if (request == NULL) - return 0; + if (request == NULL) { + request = kzalloc(sizeof(*request), GFP_KERNEL); + if (request == NULL) + return 0; + } - seqno = ring->add_request(dev, ring, file_priv, flush_domains); + seqno = ring->add_request(dev, ring, 0); + ring->outstanding_lazy_request = false; request->seqno = seqno; request->ring = ring; @@ -1615,23 +1757,20 @@ i915_add_request(struct drm_device *dev, struct drm_file *file_priv, was_empty = list_empty(&ring->request_list); list_add_tail(&request->list, &ring->request_list); - if (i915_file_priv) { + if (file_priv) { + spin_lock(&file_priv->mm.lock); + request->file_priv = file_priv; list_add_tail(&request->client_list, - &i915_file_priv->mm.request_list); - } else { - INIT_LIST_HEAD(&request->client_list); + &file_priv->mm.request_list); + spin_unlock(&file_priv->mm.lock); } - /* Associate any objects on the flushing list matching the write - * domain we're flushing with our flush. - */ - if (flush_domains != 0) - i915_gem_process_flushing_list(dev, flush_domains, seqno, ring); - if (!dev_priv->mm.suspended) { - mod_timer(&dev_priv->hangcheck_timer, jiffies + DRM_I915_HANGCHECK_PERIOD); + mod_timer(&dev_priv->hangcheck_timer, + jiffies + msecs_to_jiffies(DRM_I915_HANGCHECK_PERIOD)); if (was_empty) - queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, HZ); + queue_delayed_work(dev_priv->wq, + &dev_priv->mm.retire_work, HZ); } return seqno; } @@ -1642,91 +1781,105 @@ i915_add_request(struct drm_device *dev, struct drm_file *file_priv, * Ensures that all commands in the ring are finished * before signalling the CPU */ -static uint32_t +static void i915_retire_commands(struct drm_device *dev, struct intel_ring_buffer *ring) { uint32_t flush_domains = 0; /* The sampler always gets flushed on i965 (sigh) */ - if (IS_I965G(dev)) + if (INTEL_INFO(dev)->gen >= 4) flush_domains |= I915_GEM_DOMAIN_SAMPLER; ring->flush(dev, ring, I915_GEM_DOMAIN_COMMAND, flush_domains); - return flush_domains; } -/** - * Moves buffers associated only with the given active seqno from the active - * to inactive list, potentially freeing them. - */ -static void -i915_gem_retire_request(struct drm_device *dev, - struct drm_i915_gem_request *request) +static inline void +i915_gem_request_remove_from_client(struct drm_i915_gem_request *request) { - drm_i915_private_t *dev_priv = dev->dev_private; + struct drm_i915_file_private *file_priv = request->file_priv; - trace_i915_gem_request_retire(dev, request->seqno); + if (!file_priv) + return; - /* Move any buffers on the active list that are no longer referenced - * by the ringbuffer to the flushing/inactive lists as appropriate. - */ - spin_lock(&dev_priv->mm.active_list_lock); - while (!list_empty(&request->ring->active_list)) { - struct drm_gem_object *obj; + spin_lock(&file_priv->mm.lock); + list_del(&request->client_list); + request->file_priv = NULL; + spin_unlock(&file_priv->mm.lock); +} + +static void i915_gem_reset_ring_lists(struct drm_i915_private *dev_priv, + struct intel_ring_buffer *ring) +{ + while (!list_empty(&ring->request_list)) { + struct drm_i915_gem_request *request; + + request = list_first_entry(&ring->request_list, + struct drm_i915_gem_request, + list); + + list_del(&request->list); + i915_gem_request_remove_from_client(request); + kfree(request); + } + + while (!list_empty(&ring->active_list)) { struct drm_i915_gem_object *obj_priv; - obj_priv = list_first_entry(&request->ring->active_list, + obj_priv = list_first_entry(&ring->active_list, struct drm_i915_gem_object, list); - obj = &obj_priv->base; - - /* If the seqno being retired doesn't match the oldest in the - * list, then the oldest in the list must still be newer than - * this seqno. - */ - if (obj_priv->last_rendering_seqno != request->seqno) - goto out; -#if WATCH_LRU - DRM_INFO("%s: retire %d moves to inactive list %p\n", - __func__, request->seqno, obj); -#endif - - if (obj->write_domain != 0) - i915_gem_object_move_to_flushing(obj); - else { - /* Take a reference on the object so it won't be - * freed while the spinlock is held. The list - * protection for this spinlock is safe when breaking - * the lock like this since the next thing we do - * is just get the head of the list again. - */ - drm_gem_object_reference(obj); - i915_gem_object_move_to_inactive(obj); - spin_unlock(&dev_priv->mm.active_list_lock); - drm_gem_object_unreference(obj); - spin_lock(&dev_priv->mm.active_list_lock); - } + obj_priv->base.write_domain = 0; + list_del_init(&obj_priv->gpu_write_list); + i915_gem_object_move_to_inactive(&obj_priv->base); } -out: - spin_unlock(&dev_priv->mm.active_list_lock); } -/** - * Returns true if seq1 is later than seq2. - */ -bool -i915_seqno_passed(uint32_t seq1, uint32_t seq2) +void i915_gem_reset(struct drm_device *dev) { - return (int32_t)(seq1 - seq2) >= 0; -} + struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_gem_object *obj_priv; + int i; -uint32_t -i915_get_gem_seqno(struct drm_device *dev, - struct intel_ring_buffer *ring) -{ - return ring->get_gem_seqno(dev, ring); + i915_gem_reset_ring_lists(dev_priv, &dev_priv->render_ring); + if (HAS_BSD(dev)) + i915_gem_reset_ring_lists(dev_priv, &dev_priv->bsd_ring); + + /* Remove anything from the flushing lists. The GPU cache is likely + * to be lost on reset along with the data, so simply move the + * lost bo to the inactive list. + */ + while (!list_empty(&dev_priv->mm.flushing_list)) { + obj_priv = list_first_entry(&dev_priv->mm.flushing_list, + struct drm_i915_gem_object, + list); + + obj_priv->base.write_domain = 0; + list_del_init(&obj_priv->gpu_write_list); + i915_gem_object_move_to_inactive(&obj_priv->base); + } + + /* Move everything out of the GPU domains to ensure we do any + * necessary invalidation upon reuse. + */ + list_for_each_entry(obj_priv, + &dev_priv->mm.inactive_list, + list) + { + obj_priv->base.read_domains &= ~I915_GEM_GPU_DOMAINS; + } + + /* The fence registers are invalidated so clear them out */ + for (i = 0; i < 16; i++) { + struct drm_i915_fence_reg *reg; + + reg = &dev_priv->fence_regs[i]; + if (!reg->obj) + continue; + + i915_gem_clear_fence_reg(reg->obj); + } } /** @@ -1739,38 +1892,58 @@ i915_gem_retire_requests_ring(struct drm_device *dev, drm_i915_private_t *dev_priv = dev->dev_private; uint32_t seqno; - if (!ring->status_page.page_addr - || list_empty(&ring->request_list)) + if (!ring->status_page.page_addr || + list_empty(&ring->request_list)) return; - seqno = i915_get_gem_seqno(dev, ring); + WARN_ON(i915_verify_lists(dev)); + seqno = ring->get_seqno(dev, ring); while (!list_empty(&ring->request_list)) { struct drm_i915_gem_request *request; - uint32_t retiring_seqno; request = list_first_entry(&ring->request_list, struct drm_i915_gem_request, list); - retiring_seqno = request->seqno; - if (i915_seqno_passed(seqno, retiring_seqno) || - atomic_read(&dev_priv->mm.wedged)) { - i915_gem_retire_request(dev, request); + if (!i915_seqno_passed(seqno, request->seqno)) + break; + + trace_i915_gem_request_retire(dev, request->seqno); + + list_del(&request->list); + i915_gem_request_remove_from_client(request); + kfree(request); + } + + /* Move any buffers on the active list that are no longer referenced + * by the ringbuffer to the flushing/inactive lists as appropriate. + */ + while (!list_empty(&ring->active_list)) { + struct drm_gem_object *obj; + struct drm_i915_gem_object *obj_priv; + + obj_priv = list_first_entry(&ring->active_list, + struct drm_i915_gem_object, + list); - list_del(&request->list); - list_del(&request->client_list); - kfree(request); - } else + if (!i915_seqno_passed(seqno, obj_priv->last_rendering_seqno)) break; + + obj = &obj_priv->base; + if (obj->write_domain != 0) + i915_gem_object_move_to_flushing(obj); + else + i915_gem_object_move_to_inactive(obj); } if (unlikely (dev_priv->trace_irq_seqno && i915_seqno_passed(dev_priv->trace_irq_seqno, seqno))) { - ring->user_irq_put(dev, ring); dev_priv->trace_irq_seqno = 0; } + + WARN_ON(i915_verify_lists(dev)); } void @@ -1797,7 +1970,7 @@ i915_gem_retire_requests(struct drm_device *dev) i915_gem_retire_requests_ring(dev, &dev_priv->bsd_ring); } -void +static void i915_gem_retire_work_handler(struct work_struct *work) { drm_i915_private_t *dev_priv; @@ -1807,7 +1980,12 @@ i915_gem_retire_work_handler(struct work_struct *work) mm.retire_work.work); dev = dev_priv->dev; - mutex_lock(&dev->struct_mutex); + /* Come back later if the device is busy... */ + if (!mutex_trylock(&dev->struct_mutex)) { + queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, HZ); + return; + } + i915_gem_retire_requests(dev); if (!dev_priv->mm.suspended && @@ -1820,7 +1998,7 @@ i915_gem_retire_work_handler(struct work_struct *work) int i915_do_wait_request(struct drm_device *dev, uint32_t seqno, - int interruptible, struct intel_ring_buffer *ring) + bool interruptible, struct intel_ring_buffer *ring) { drm_i915_private_t *dev_priv = dev->dev_private; u32 ier; @@ -1829,9 +2007,16 @@ i915_do_wait_request(struct drm_device *dev, uint32_t seqno, BUG_ON(seqno == 0); if (atomic_read(&dev_priv->mm.wedged)) - return -EIO; + return -EAGAIN; + + if (ring->outstanding_lazy_request) { + seqno = i915_add_request(dev, NULL, NULL, ring); + if (seqno == 0) + return -ENOMEM; + } + BUG_ON(seqno == dev_priv->next_seqno); - if (!i915_seqno_passed(ring->get_gem_seqno(dev, ring), seqno)) { + if (!i915_seqno_passed(ring->get_seqno(dev, ring), seqno)) { if (HAS_PCH_SPLIT(dev)) ier = I915_READ(DEIER) | I915_READ(GTIER); else @@ -1850,12 +2035,12 @@ i915_do_wait_request(struct drm_device *dev, uint32_t seqno, if (interruptible) ret = wait_event_interruptible(ring->irq_queue, i915_seqno_passed( - ring->get_gem_seqno(dev, ring), seqno) + ring->get_seqno(dev, ring), seqno) || atomic_read(&dev_priv->mm.wedged)); else wait_event(ring->irq_queue, i915_seqno_passed( - ring->get_gem_seqno(dev, ring), seqno) + ring->get_seqno(dev, ring), seqno) || atomic_read(&dev_priv->mm.wedged)); ring->user_irq_put(dev, ring); @@ -1864,11 +2049,12 @@ i915_do_wait_request(struct drm_device *dev, uint32_t seqno, trace_i915_gem_request_wait_end(dev, seqno); } if (atomic_read(&dev_priv->mm.wedged)) - ret = -EIO; + ret = -EAGAIN; if (ret && ret != -ERESTARTSYS) - DRM_ERROR("%s returns %d (awaiting %d at %d)\n", - __func__, ret, seqno, ring->get_gem_seqno(dev, ring)); + DRM_ERROR("%s returns %d (awaiting %d at %d, next %d)\n", + __func__, ret, seqno, ring->get_seqno(dev, ring), + dev_priv->next_seqno); /* Directly dispatch request retiring. While we have the work queue * to handle this, the waiter on a request often wants an associated @@ -1887,27 +2073,44 @@ i915_do_wait_request(struct drm_device *dev, uint32_t seqno, */ static int i915_wait_request(struct drm_device *dev, uint32_t seqno, - struct intel_ring_buffer *ring) + struct intel_ring_buffer *ring) { return i915_do_wait_request(dev, seqno, 1, ring); } static void +i915_gem_flush_ring(struct drm_device *dev, + struct drm_file *file_priv, + struct intel_ring_buffer *ring, + uint32_t invalidate_domains, + uint32_t flush_domains) +{ + ring->flush(dev, ring, invalidate_domains, flush_domains); + i915_gem_process_flushing_list(dev, flush_domains, ring); +} + +static void i915_gem_flush(struct drm_device *dev, + struct drm_file *file_priv, uint32_t invalidate_domains, - uint32_t flush_domains) + uint32_t flush_domains, + uint32_t flush_rings) { drm_i915_private_t *dev_priv = dev->dev_private; + if (flush_domains & I915_GEM_DOMAIN_CPU) drm_agp_chipset_flush(dev); - dev_priv->render_ring.flush(dev, &dev_priv->render_ring, - invalidate_domains, - flush_domains); - if (HAS_BSD(dev)) - dev_priv->bsd_ring.flush(dev, &dev_priv->bsd_ring, - invalidate_domains, - flush_domains); + if ((flush_domains | invalidate_domains) & I915_GEM_GPU_DOMAINS) { + if (flush_rings & RING_RENDER) + i915_gem_flush_ring(dev, file_priv, + &dev_priv->render_ring, + invalidate_domains, flush_domains); + if (flush_rings & RING_BSD) + i915_gem_flush_ring(dev, file_priv, + &dev_priv->bsd_ring, + invalidate_domains, flush_domains); + } } /** @@ -1915,7 +2118,8 @@ i915_gem_flush(struct drm_device *dev, * safe to unbind from the GTT or access from the CPU. */ static int -i915_gem_object_wait_rendering(struct drm_gem_object *obj) +i915_gem_object_wait_rendering(struct drm_gem_object *obj, + bool interruptible) { struct drm_device *dev = obj->dev; struct drm_i915_gem_object *obj_priv = to_intel_bo(obj); @@ -1930,13 +2134,11 @@ i915_gem_object_wait_rendering(struct drm_gem_object *obj) * it. */ if (obj_priv->active) { -#if WATCH_BUF - DRM_INFO("%s: object %p wait for seqno %08x\n", - __func__, obj, obj_priv->last_rendering_seqno); -#endif - ret = i915_wait_request(dev, - obj_priv->last_rendering_seqno, obj_priv->ring); - if (ret != 0) + ret = i915_do_wait_request(dev, + obj_priv->last_rendering_seqno, + interruptible, + obj_priv->ring); + if (ret) return ret; } @@ -1950,14 +2152,10 @@ int i915_gem_object_unbind(struct drm_gem_object *obj) { struct drm_device *dev = obj->dev; - drm_i915_private_t *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = dev->dev_private; struct drm_i915_gem_object *obj_priv = to_intel_bo(obj); int ret = 0; -#if WATCH_BUF - DRM_INFO("%s:%d %p\n", __func__, __LINE__, obj); - DRM_INFO("gtt_space %p\n", obj_priv->gtt_space); -#endif if (obj_priv->gtt_space == NULL) return 0; @@ -1982,33 +2180,26 @@ i915_gem_object_unbind(struct drm_gem_object *obj) * should be safe and we need to cleanup or else we might * cause memory corruption through use-after-free. */ + if (ret) { + i915_gem_clflush_object(obj); + obj->read_domains = obj->write_domain = I915_GEM_DOMAIN_CPU; + } /* release the fence reg _after_ flushing */ if (obj_priv->fence_reg != I915_FENCE_REG_NONE) i915_gem_clear_fence_reg(obj); - if (obj_priv->agp_mem != NULL) { - drm_unbind_agp(obj_priv->agp_mem); - drm_free_agp(obj_priv->agp_mem, obj->size / PAGE_SIZE); - obj_priv->agp_mem = NULL; - } + drm_unbind_agp(obj_priv->agp_mem); + drm_free_agp(obj_priv->agp_mem, obj->size / PAGE_SIZE); i915_gem_object_put_pages(obj); BUG_ON(obj_priv->pages_refcount); - if (obj_priv->gtt_space) { - atomic_dec(&dev->gtt_count); - atomic_sub(obj->size, &dev->gtt_memory); + i915_gem_info_remove_gtt(dev_priv, obj->size); + list_del_init(&obj_priv->list); - drm_mm_put_block(obj_priv->gtt_space); - obj_priv->gtt_space = NULL; - } - - /* Remove ourselves from the LRU list if present. */ - spin_lock(&dev_priv->mm.active_list_lock); - if (!list_empty(&obj_priv->list)) - list_del_init(&obj_priv->list); - spin_unlock(&dev_priv->mm.active_list_lock); + drm_mm_put_block(obj_priv->gtt_space); + obj_priv->gtt_space = NULL; if (i915_gem_object_is_purgeable(obj_priv)) i915_gem_object_truncate(obj); @@ -2018,48 +2209,45 @@ i915_gem_object_unbind(struct drm_gem_object *obj) return ret; } +static int i915_ring_idle(struct drm_device *dev, + struct intel_ring_buffer *ring) +{ + i915_gem_flush_ring(dev, NULL, ring, + I915_GEM_GPU_DOMAINS, I915_GEM_GPU_DOMAINS); + return i915_wait_request(dev, + i915_gem_next_request_seqno(dev, ring), + ring); +} + int i915_gpu_idle(struct drm_device *dev) { drm_i915_private_t *dev_priv = dev->dev_private; bool lists_empty; - uint32_t seqno1, seqno2; int ret; - spin_lock(&dev_priv->mm.active_list_lock); lists_empty = (list_empty(&dev_priv->mm.flushing_list) && list_empty(&dev_priv->render_ring.active_list) && (!HAS_BSD(dev) || list_empty(&dev_priv->bsd_ring.active_list))); - spin_unlock(&dev_priv->mm.active_list_lock); - if (lists_empty) return 0; /* Flush everything onto the inactive list. */ - i915_gem_flush(dev, I915_GEM_GPU_DOMAINS, I915_GEM_GPU_DOMAINS); - seqno1 = i915_add_request(dev, NULL, I915_GEM_GPU_DOMAINS, - &dev_priv->render_ring); - if (seqno1 == 0) - return -ENOMEM; - ret = i915_wait_request(dev, seqno1, &dev_priv->render_ring); + ret = i915_ring_idle(dev, &dev_priv->render_ring); + if (ret) + return ret; if (HAS_BSD(dev)) { - seqno2 = i915_add_request(dev, NULL, I915_GEM_GPU_DOMAINS, - &dev_priv->bsd_ring); - if (seqno2 == 0) - return -ENOMEM; - - ret = i915_wait_request(dev, seqno2, &dev_priv->bsd_ring); + ret = i915_ring_idle(dev, &dev_priv->bsd_ring); if (ret) return ret; } - - return ret; + return 0; } -int +static int i915_gem_object_get_pages(struct drm_gem_object *obj, gfp_t gfpmask) { @@ -2239,7 +2427,8 @@ static void i830_write_fence_reg(struct drm_i915_fence_reg *reg) I915_WRITE(FENCE_REG_830_0 + (regnum * 4), val); } -static int i915_find_fence_reg(struct drm_device *dev) +static int i915_find_fence_reg(struct drm_device *dev, + bool interruptible) { struct drm_i915_fence_reg *reg = NULL; struct drm_i915_gem_object *obj_priv = NULL; @@ -2284,7 +2473,7 @@ static int i915_find_fence_reg(struct drm_device *dev) * private reference to obj like the other callers of put_fence_reg * (set_tiling ioctl) do. */ drm_gem_object_reference(obj); - ret = i915_gem_object_put_fence_reg(obj); + ret = i915_gem_object_put_fence_reg(obj, interruptible); drm_gem_object_unreference(obj); if (ret != 0) return ret; @@ -2306,7 +2495,8 @@ static int i915_find_fence_reg(struct drm_device *dev) * and tiling format. */ int -i915_gem_object_get_fence_reg(struct drm_gem_object *obj) +i915_gem_object_get_fence_reg(struct drm_gem_object *obj, + bool interruptible) { struct drm_device *dev = obj->dev; struct drm_i915_private *dev_priv = dev->dev_private; @@ -2341,7 +2531,7 @@ i915_gem_object_get_fence_reg(struct drm_gem_object *obj) break; } - ret = i915_find_fence_reg(dev); + ret = i915_find_fence_reg(dev, interruptible); if (ret < 0) return ret; @@ -2400,7 +2590,7 @@ i915_gem_clear_fence_reg(struct drm_gem_object *obj) I915_WRITE64(FENCE_REG_965_0 + (obj_priv->fence_reg * 8), 0); break; case 3: - if (obj_priv->fence_reg > 8) + if (obj_priv->fence_reg >= 8) fence_reg = FENCE_REG_945_8 + (obj_priv->fence_reg - 8) * 4; else case 2: @@ -2419,15 +2609,19 @@ i915_gem_clear_fence_reg(struct drm_gem_object *obj) * i915_gem_object_put_fence_reg - waits on outstanding fenced access * to the buffer to finish, and then resets the fence register. * @obj: tiled object holding a fence register. + * @bool: whether the wait upon the fence is interruptible * * Zeroes out the fence register itself and clears out the associated * data structures in dev_priv and obj_priv. */ int -i915_gem_object_put_fence_reg(struct drm_gem_object *obj) +i915_gem_object_put_fence_reg(struct drm_gem_object *obj, + bool interruptible) { struct drm_device *dev = obj->dev; + struct drm_i915_private *dev_priv = dev->dev_private; struct drm_i915_gem_object *obj_priv = to_intel_bo(obj); + struct drm_i915_fence_reg *reg; if (obj_priv->fence_reg == I915_FENCE_REG_NONE) return 0; @@ -2442,20 +2636,23 @@ i915_gem_object_put_fence_reg(struct drm_gem_object *obj) * therefore we must wait for any outstanding access to complete * before clearing the fence. */ - if (!IS_I965G(dev)) { + reg = &dev_priv->fence_regs[obj_priv->fence_reg]; + if (reg->gpu) { int ret; - ret = i915_gem_object_flush_gpu_write_domain(obj); - if (ret != 0) + ret = i915_gem_object_flush_gpu_write_domain(obj, true); + if (ret) return ret; - ret = i915_gem_object_wait_rendering(obj); - if (ret != 0) + ret = i915_gem_object_wait_rendering(obj, interruptible); + if (ret) return ret; + + reg->gpu = false; } i915_gem_object_flush_gtt_write_domain(obj); - i915_gem_clear_fence_reg (obj); + i915_gem_clear_fence_reg(obj); return 0; } @@ -2488,7 +2685,7 @@ i915_gem_object_bind_to_gtt(struct drm_gem_object *obj, unsigned alignment) /* If the object is bigger than the entire aperture, reject it early * before evicting everything in a vain attempt to find space. */ - if (obj->size > dev->gtt_total) { + if (obj->size > dev_priv->mm.gtt_total) { DRM_ERROR("Attempting to bind an object larger than the aperture\n"); return -E2BIG; } @@ -2506,9 +2703,6 @@ i915_gem_object_bind_to_gtt(struct drm_gem_object *obj, unsigned alignment) /* If the gtt is empty and we're still having trouble * fitting our object in, we're out of memory. */ -#if WATCH_LRU - DRM_INFO("%s: GTT full, evicting something\n", __func__); -#endif ret = i915_gem_evict_something(dev, obj->size, alignment); if (ret) return ret; @@ -2516,10 +2710,6 @@ i915_gem_object_bind_to_gtt(struct drm_gem_object *obj, unsigned alignment) goto search_free; } -#if WATCH_BUF - DRM_INFO("Binding object of size %zd at 0x%08x\n", - obj->size, obj_priv->gtt_offset); -#endif ret = i915_gem_object_get_pages(obj, gfpmask); if (ret) { drm_mm_put_block(obj_priv->gtt_space); @@ -2564,11 +2754,10 @@ i915_gem_object_bind_to_gtt(struct drm_gem_object *obj, unsigned alignment) goto search_free; } - atomic_inc(&dev->gtt_count); - atomic_add(obj->size, &dev->gtt_memory); /* keep track of bounds object by adding it to the inactive list */ list_add_tail(&obj_priv->list, &dev_priv->mm.inactive_list); + i915_gem_info_add_gtt(dev_priv, obj->size); /* Assert that the object is not currently in any GPU domain. As it * wasn't in the GTT, there shouldn't be any way it could have been in @@ -2601,25 +2790,30 @@ i915_gem_clflush_object(struct drm_gem_object *obj) /** Flushes any GPU write domain for the object if it's dirty. */ static int -i915_gem_object_flush_gpu_write_domain(struct drm_gem_object *obj) +i915_gem_object_flush_gpu_write_domain(struct drm_gem_object *obj, + bool pipelined) { struct drm_device *dev = obj->dev; uint32_t old_write_domain; - struct drm_i915_gem_object *obj_priv = to_intel_bo(obj); if ((obj->write_domain & I915_GEM_GPU_DOMAINS) == 0) return 0; /* Queue the GPU write cache flushing we need. */ old_write_domain = obj->write_domain; - i915_gem_flush(dev, 0, obj->write_domain); - if (i915_add_request(dev, NULL, obj->write_domain, obj_priv->ring) == 0) - return -ENOMEM; + i915_gem_flush_ring(dev, NULL, + to_intel_bo(obj)->ring, + 0, obj->write_domain); + BUG_ON(obj->write_domain); trace_i915_gem_object_change_domain(obj, obj->read_domains, old_write_domain); - return 0; + + if (pipelined) + return 0; + + return i915_gem_object_wait_rendering(obj, true); } /** Flushes the GTT write domain for the object if it's dirty. */ @@ -2663,26 +2857,6 @@ i915_gem_object_flush_cpu_write_domain(struct drm_gem_object *obj) old_write_domain); } -int -i915_gem_object_flush_write_domain(struct drm_gem_object *obj) -{ - int ret = 0; - - switch (obj->write_domain) { - case I915_GEM_DOMAIN_GTT: - i915_gem_object_flush_gtt_write_domain(obj); - break; - case I915_GEM_DOMAIN_CPU: - i915_gem_object_flush_cpu_write_domain(obj); - break; - default: - ret = i915_gem_object_flush_gpu_write_domain(obj); - break; - } - - return ret; -} - /** * Moves a single object to the GTT read, and possibly write domain. * @@ -2700,32 +2874,28 @@ i915_gem_object_set_to_gtt_domain(struct drm_gem_object *obj, int write) if (obj_priv->gtt_space == NULL) return -EINVAL; - ret = i915_gem_object_flush_gpu_write_domain(obj); + ret = i915_gem_object_flush_gpu_write_domain(obj, false); if (ret != 0) return ret; - /* Wait on any GPU rendering and flushing to occur. */ - ret = i915_gem_object_wait_rendering(obj); - if (ret != 0) - return ret; + i915_gem_object_flush_cpu_write_domain(obj); + + if (write) { + ret = i915_gem_object_wait_rendering(obj, true); + if (ret) + return ret; + } old_write_domain = obj->write_domain; old_read_domains = obj->read_domains; - /* If we're writing through the GTT domain, then CPU and GPU caches - * will need to be invalidated at next use. - */ - if (write) - obj->read_domains &= I915_GEM_DOMAIN_GTT; - - i915_gem_object_flush_cpu_write_domain(obj); - /* It should now be out of any other write domains, and we can update * the domain values for our changes. */ BUG_ON((obj->write_domain & ~I915_GEM_DOMAIN_GTT) != 0); obj->read_domains |= I915_GEM_DOMAIN_GTT; if (write) { + obj->read_domains = I915_GEM_DOMAIN_GTT; obj->write_domain = I915_GEM_DOMAIN_GTT; obj_priv->dirty = 1; } @@ -2742,51 +2912,36 @@ i915_gem_object_set_to_gtt_domain(struct drm_gem_object *obj, int write) * wait, as in modesetting process we're not supposed to be interrupted. */ int -i915_gem_object_set_to_display_plane(struct drm_gem_object *obj) +i915_gem_object_set_to_display_plane(struct drm_gem_object *obj, + bool pipelined) { - struct drm_device *dev = obj->dev; struct drm_i915_gem_object *obj_priv = to_intel_bo(obj); - uint32_t old_write_domain, old_read_domains; + uint32_t old_read_domains; int ret; /* Not valid to be called on unbound objects. */ if (obj_priv->gtt_space == NULL) return -EINVAL; - ret = i915_gem_object_flush_gpu_write_domain(obj); + ret = i915_gem_object_flush_gpu_write_domain(obj, true); if (ret) return ret; - /* Wait on any GPU rendering and flushing to occur. */ - if (obj_priv->active) { -#if WATCH_BUF - DRM_INFO("%s: object %p wait for seqno %08x\n", - __func__, obj, obj_priv->last_rendering_seqno); -#endif - ret = i915_do_wait_request(dev, - obj_priv->last_rendering_seqno, - 0, - obj_priv->ring); - if (ret != 0) + /* Currently, we are always called from an non-interruptible context. */ + if (!pipelined) { + ret = i915_gem_object_wait_rendering(obj, false); + if (ret) return ret; } i915_gem_object_flush_cpu_write_domain(obj); - old_write_domain = obj->write_domain; old_read_domains = obj->read_domains; - - /* It should now be out of any other write domains, and we can update - * the domain values for our changes. - */ - BUG_ON((obj->write_domain & ~I915_GEM_DOMAIN_GTT) != 0); - obj->read_domains = I915_GEM_DOMAIN_GTT; - obj->write_domain = I915_GEM_DOMAIN_GTT; - obj_priv->dirty = 1; + obj->read_domains |= I915_GEM_DOMAIN_GTT; trace_i915_gem_object_change_domain(obj, old_read_domains, - old_write_domain); + obj->write_domain); return 0; } @@ -2803,12 +2958,7 @@ i915_gem_object_set_to_cpu_domain(struct drm_gem_object *obj, int write) uint32_t old_write_domain, old_read_domains; int ret; - ret = i915_gem_object_flush_gpu_write_domain(obj); - if (ret) - return ret; - - /* Wait on any GPU rendering and flushing to occur. */ - ret = i915_gem_object_wait_rendering(obj); + ret = i915_gem_object_flush_gpu_write_domain(obj, false); if (ret != 0) return ret; @@ -2819,6 +2969,12 @@ i915_gem_object_set_to_cpu_domain(struct drm_gem_object *obj, int write) */ i915_gem_object_set_to_full_cpu_read_domain(obj); + if (write) { + ret = i915_gem_object_wait_rendering(obj, true); + if (ret) + return ret; + } + old_write_domain = obj->write_domain; old_read_domains = obj->read_domains; @@ -2838,7 +2994,7 @@ i915_gem_object_set_to_cpu_domain(struct drm_gem_object *obj, int write) * need to be invalidated at next use. */ if (write) { - obj->read_domains &= I915_GEM_DOMAIN_CPU; + obj->read_domains = I915_GEM_DOMAIN_CPU; obj->write_domain = I915_GEM_DOMAIN_CPU; } @@ -2964,7 +3120,7 @@ static void i915_gem_object_set_to_gpu_domain(struct drm_gem_object *obj) { struct drm_device *dev = obj->dev; - drm_i915_private_t *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = dev->dev_private; struct drm_i915_gem_object *obj_priv = to_intel_bo(obj); uint32_t invalidate_domains = 0; uint32_t flush_domains = 0; @@ -2975,12 +3131,6 @@ i915_gem_object_set_to_gpu_domain(struct drm_gem_object *obj) intel_mark_busy(dev, obj); -#if WATCH_BUF - DRM_INFO("%s: object %p read %08x -> %08x write %08x -> %08x\n", - __func__, obj, - obj->read_domains, obj->pending_read_domains, - obj->write_domain, obj->pending_write_domain); -#endif /* * If the object isn't moving to a new write domain, * let the object stay in multiple read domains @@ -3007,13 +3157,8 @@ i915_gem_object_set_to_gpu_domain(struct drm_gem_object *obj) * stale data. That is, any new read domains. */ invalidate_domains |= obj->pending_read_domains & ~obj->read_domains; - if ((flush_domains | invalidate_domains) & I915_GEM_DOMAIN_CPU) { -#if WATCH_BUF - DRM_INFO("%s: CPU domain flush %08x invalidate %08x\n", - __func__, flush_domains, invalidate_domains); -#endif + if ((flush_domains | invalidate_domains) & I915_GEM_DOMAIN_CPU) i915_gem_clflush_object(obj); - } old_read_domains = obj->read_domains; @@ -3027,21 +3172,10 @@ i915_gem_object_set_to_gpu_domain(struct drm_gem_object *obj) obj->pending_write_domain = obj->write_domain; obj->read_domains = obj->pending_read_domains; - if (flush_domains & I915_GEM_GPU_DOMAINS) { - if (obj_priv->ring == &dev_priv->render_ring) - dev_priv->flush_rings |= FLUSH_RENDER_RING; - else if (obj_priv->ring == &dev_priv->bsd_ring) - dev_priv->flush_rings |= FLUSH_BSD_RING; - } - dev->invalidate_domains |= invalidate_domains; dev->flush_domains |= flush_domains; -#if WATCH_BUF - DRM_INFO("%s: read %08x write %08x invalidate %08x flush %08x\n", - __func__, - obj->read_domains, obj->write_domain, - dev->invalidate_domains, dev->flush_domains); -#endif + if (obj_priv->ring) + dev_priv->mm.flush_rings |= obj_priv->ring->id; trace_i915_gem_object_change_domain(obj, old_read_domains, @@ -3104,12 +3238,7 @@ i915_gem_object_set_cpu_read_domain_range(struct drm_gem_object *obj, if (offset == 0 && size == obj->size) return i915_gem_object_set_to_cpu_domain(obj, 0); - ret = i915_gem_object_flush_gpu_write_domain(obj); - if (ret) - return ret; - - /* Wait on any GPU rendering and flushing to occur. */ - ret = i915_gem_object_wait_rendering(obj); + ret = i915_gem_object_flush_gpu_write_domain(obj, false); if (ret != 0) return ret; i915_gem_object_flush_gtt_write_domain(obj); @@ -3196,11 +3325,13 @@ i915_gem_object_pin_and_relocate(struct drm_gem_object *obj, * properly handle blits to/from tiled surfaces. */ if (need_fence) { - ret = i915_gem_object_get_fence_reg(obj); + ret = i915_gem_object_get_fence_reg(obj, true); if (ret != 0) { i915_gem_object_unpin(obj); return ret; } + + dev_priv->fence_regs[obj_priv->fence_reg].gpu = true; } entry->offset = obj_priv->gtt_offset; @@ -3258,6 +3389,8 @@ i915_gem_object_pin_and_relocate(struct drm_gem_object *obj, (int) reloc->offset, reloc->read_domains, reloc->write_domain); + drm_gem_object_unreference(target_obj); + i915_gem_object_unpin(obj); return -EINVAL; } if (reloc->write_domain & I915_GEM_DOMAIN_CPU || @@ -3333,7 +3466,7 @@ i915_gem_object_pin_and_relocate(struct drm_gem_object *obj, if (ret != 0) { drm_gem_object_unreference(target_obj); i915_gem_object_unpin(obj); - return -EINVAL; + return ret; } /* Map the page containing the relocation we're going to @@ -3348,11 +3481,6 @@ i915_gem_object_pin_and_relocate(struct drm_gem_object *obj, (reloc_offset & (PAGE_SIZE - 1))); reloc_val = target_obj_priv->gtt_offset + reloc->delta; -#if WATCH_BUF - DRM_INFO("Applied relocation: %p@0x%08x %08x -> %08x\n", - obj, (unsigned int) reloc->offset, - readl(reloc_entry), reloc_val); -#endif writel(reloc_val, reloc_entry); io_mapping_unmap_atomic(reloc_page, KM_USER0); @@ -3364,10 +3492,6 @@ i915_gem_object_pin_and_relocate(struct drm_gem_object *obj, drm_gem_object_unreference(target_obj); } -#if WATCH_BUF - if (0) - i915_gem_dump_object(obj, 128, __func__, ~0); -#endif return 0; } @@ -3382,28 +3506,48 @@ i915_gem_object_pin_and_relocate(struct drm_gem_object *obj, * relatively low latency when blocking on a particular request to finish. */ static int -i915_gem_ring_throttle(struct drm_device *dev, struct drm_file *file_priv) +i915_gem_ring_throttle(struct drm_device *dev, struct drm_file *file) { - struct drm_i915_file_private *i915_file_priv = file_priv->driver_priv; - int ret = 0; + struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_file_private *file_priv = file->driver_priv; unsigned long recent_enough = jiffies - msecs_to_jiffies(20); + struct drm_i915_gem_request *request; + struct intel_ring_buffer *ring = NULL; + u32 seqno = 0; + int ret; - mutex_lock(&dev->struct_mutex); - while (!list_empty(&i915_file_priv->mm.request_list)) { - struct drm_i915_gem_request *request; - - request = list_first_entry(&i915_file_priv->mm.request_list, - struct drm_i915_gem_request, - client_list); - + spin_lock(&file_priv->mm.lock); + list_for_each_entry(request, &file_priv->mm.request_list, client_list) { if (time_after_eq(request->emitted_jiffies, recent_enough)) break; - ret = i915_wait_request(dev, request->seqno, request->ring); - if (ret != 0) - break; + ring = request->ring; + seqno = request->seqno; } - mutex_unlock(&dev->struct_mutex); + spin_unlock(&file_priv->mm.lock); + + if (seqno == 0) + return 0; + + ret = 0; + if (!i915_seqno_passed(ring->get_seqno(dev, ring), seqno)) { + /* And wait for the seqno passing without holding any locks and + * causing extra latency for others. This is safe as the irq + * generation is designed to be run atomically and so is + * lockless. + */ + ring->user_irq_get(dev, ring); + ret = wait_event_interruptible(ring->irq_queue, + i915_seqno_passed(ring->get_seqno(dev, ring), seqno) + || atomic_read(&dev_priv->mm.wedged)); + ring->user_irq_put(dev, ring); + + if (ret == 0 && atomic_read(&dev_priv->mm.wedged)) + ret = -EIO; + } + + if (ret == 0) + queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, 0); return ret; } @@ -3539,8 +3683,7 @@ i915_gem_wait_for_pending_flip(struct drm_device *dev, return ret; } - -int +static int i915_gem_do_execbuffer(struct drm_device *dev, void *data, struct drm_file *file_priv, struct drm_i915_gem_execbuffer2 *args, @@ -3552,13 +3695,18 @@ i915_gem_do_execbuffer(struct drm_device *dev, void *data, struct drm_i915_gem_object *obj_priv; struct drm_clip_rect *cliprects = NULL; struct drm_i915_gem_relocation_entry *relocs = NULL; - int ret = 0, ret2, i, pinned = 0; + struct drm_i915_gem_request *request = NULL; + int ret, ret2, i, pinned = 0; uint64_t exec_offset; - uint32_t seqno, flush_domains, reloc_index; + uint32_t reloc_index; int pin_tries, flips; struct intel_ring_buffer *ring = NULL; + ret = i915_gem_check_is_wedged(dev); + if (ret) + return ret; + #if WATCH_EXEC DRM_INFO("buffers_ptr %d buffer_count %d len %08x\n", (int) args->buffers_ptr, args->buffer_count, args->batch_len); @@ -3605,20 +3753,20 @@ i915_gem_do_execbuffer(struct drm_device *dev, void *data, } } + request = kzalloc(sizeof(*request), GFP_KERNEL); + if (request == NULL) { + ret = -ENOMEM; + goto pre_mutex_err; + } + ret = i915_gem_get_relocs_from_user(exec_list, args->buffer_count, &relocs); if (ret != 0) goto pre_mutex_err; - mutex_lock(&dev->struct_mutex); - - i915_verify_inactive(dev, __FILE__, __LINE__); - - if (atomic_read(&dev_priv->mm.wedged)) { - mutex_unlock(&dev->struct_mutex); - ret = -EIO; + ret = i915_mutex_lock_interruptible(dev); + if (ret) goto pre_mutex_err; - } if (dev_priv->mm.suspended) { mutex_unlock(&dev->struct_mutex); @@ -3698,15 +3846,16 @@ i915_gem_do_execbuffer(struct drm_device *dev, void *data, pinned+1, args->buffer_count, total_size, num_fences, ret); - DRM_ERROR("%d objects [%d pinned], " - "%d object bytes [%d pinned], " - "%d/%d gtt bytes\n", - atomic_read(&dev->object_count), - atomic_read(&dev->pin_count), - atomic_read(&dev->object_memory), - atomic_read(&dev->pin_memory), - atomic_read(&dev->gtt_memory), - dev->gtt_total); + DRM_ERROR("%u objects [%u pinned, %u GTT], " + "%zu object bytes [%zu pinned], " + "%zu /%zu gtt bytes\n", + dev_priv->mm.object_count, + dev_priv->mm.pin_count, + dev_priv->mm.gtt_count, + dev_priv->mm.object_memory, + dev_priv->mm.pin_memory, + dev_priv->mm.gtt_memory, + dev_priv->mm.gtt_total); } goto err; } @@ -3739,15 +3888,13 @@ i915_gem_do_execbuffer(struct drm_device *dev, void *data, goto err; } - i915_verify_inactive(dev, __FILE__, __LINE__); - /* Zero the global flush/invalidate flags. These * will be modified as new domains are computed * for each object */ dev->invalidate_domains = 0; dev->flush_domains = 0; - dev_priv->flush_rings = 0; + dev_priv->mm.flush_rings = 0; for (i = 0; i < args->buffer_count; i++) { struct drm_gem_object *obj = object_list[i]; @@ -3756,8 +3903,6 @@ i915_gem_do_execbuffer(struct drm_device *dev, void *data, i915_gem_object_set_to_gpu_domain(obj); } - i915_verify_inactive(dev, __FILE__, __LINE__); - if (dev->invalidate_domains | dev->flush_domains) { #if WATCH_EXEC DRM_INFO("%s: invalidate_domains %08x flush_domains %08x\n", @@ -3765,17 +3910,10 @@ i915_gem_do_execbuffer(struct drm_device *dev, void *data, dev->invalidate_domains, dev->flush_domains); #endif - i915_gem_flush(dev, + i915_gem_flush(dev, file_priv, dev->invalidate_domains, - dev->flush_domains); - if (dev_priv->flush_rings & FLUSH_RENDER_RING) - (void)i915_add_request(dev, file_priv, - dev->flush_domains, - &dev_priv->render_ring); - if (dev_priv->flush_rings & FLUSH_BSD_RING) - (void)i915_add_request(dev, file_priv, - dev->flush_domains, - &dev_priv->bsd_ring); + dev->flush_domains, + dev_priv->mm.flush_rings); } for (i = 0; i < args->buffer_count; i++) { @@ -3787,16 +3925,12 @@ i915_gem_do_execbuffer(struct drm_device *dev, void *data, if (obj->write_domain) list_move_tail(&obj_priv->gpu_write_list, &dev_priv->mm.gpu_write_list); - else - list_del_init(&obj_priv->gpu_write_list); trace_i915_gem_object_change_domain(obj, obj->read_domains, old_write_domain); } - i915_verify_inactive(dev, __FILE__, __LINE__); - #if WATCH_COHERENCY for (i = 0; i < args->buffer_count; i++) { i915_gem_object_check_coherency(object_list[i], @@ -3823,33 +3957,17 @@ i915_gem_do_execbuffer(struct drm_device *dev, void *data, * Ensure that the commands in the batch buffer are * finished before the interrupt fires */ - flush_domains = i915_retire_commands(dev, ring); - - i915_verify_inactive(dev, __FILE__, __LINE__); + i915_retire_commands(dev, ring); - /* - * Get a seqno representing the execution of the current buffer, - * which we can wait on. We would like to mitigate these interrupts, - * likely by only creating seqnos occasionally (so that we have - * *some* interrupts representing completion of buffers that we can - * wait on when trying to clear up gtt space). - */ - seqno = i915_add_request(dev, file_priv, flush_domains, ring); - BUG_ON(seqno == 0); for (i = 0; i < args->buffer_count; i++) { struct drm_gem_object *obj = object_list[i]; obj_priv = to_intel_bo(obj); - i915_gem_object_move_to_active(obj, seqno, ring); -#if WATCH_LRU - DRM_INFO("%s: move to exec list %p\n", __func__, obj); -#endif + i915_gem_object_move_to_active(obj, ring); } -#if WATCH_LRU - i915_dump_lru(dev, __func__); -#endif - i915_verify_inactive(dev, __FILE__, __LINE__); + i915_add_request(dev, file_priv, request, ring); + request = NULL; err: for (i = 0; i < pinned; i++) @@ -3882,6 +4000,7 @@ pre_mutex_err: drm_free_large(object_list); kfree(cliprects); + kfree(request); return ret; } @@ -3938,7 +4057,7 @@ i915_gem_execbuffer(struct drm_device *dev, void *data, exec2_list[i].relocs_ptr = exec_list[i].relocs_ptr; exec2_list[i].alignment = exec_list[i].alignment; exec2_list[i].offset = exec_list[i].offset; - if (!IS_I965G(dev)) + if (INTEL_INFO(dev)->gen < 4) exec2_list[i].flags = EXEC_OBJECT_NEEDS_FENCE; else exec2_list[i].flags = 0; @@ -4035,12 +4154,12 @@ int i915_gem_object_pin(struct drm_gem_object *obj, uint32_t alignment) { struct drm_device *dev = obj->dev; + struct drm_i915_private *dev_priv = dev->dev_private; struct drm_i915_gem_object *obj_priv = to_intel_bo(obj); int ret; BUG_ON(obj_priv->pin_count == DRM_I915_GEM_OBJECT_MAX_PIN_COUNT); - - i915_verify_inactive(dev, __FILE__, __LINE__); + WARN_ON(i915_verify_lists(dev)); if (obj_priv->gtt_space != NULL) { if (alignment == 0) @@ -4068,14 +4187,13 @@ i915_gem_object_pin(struct drm_gem_object *obj, uint32_t alignment) * remove it from the inactive list */ if (obj_priv->pin_count == 1) { - atomic_inc(&dev->pin_count); - atomic_add(obj->size, &dev->pin_memory); - if (!obj_priv->active && - (obj->write_domain & I915_GEM_GPU_DOMAINS) == 0) - list_del_init(&obj_priv->list); + i915_gem_info_add_pin(dev_priv, obj->size); + if (!obj_priv->active) + list_move_tail(&obj_priv->list, + &dev_priv->mm.pinned_list); } - i915_verify_inactive(dev, __FILE__, __LINE__); + WARN_ON(i915_verify_lists(dev)); return 0; } @@ -4086,7 +4204,7 @@ i915_gem_object_unpin(struct drm_gem_object *obj) drm_i915_private_t *dev_priv = dev->dev_private; struct drm_i915_gem_object *obj_priv = to_intel_bo(obj); - i915_verify_inactive(dev, __FILE__, __LINE__); + WARN_ON(i915_verify_lists(dev)); obj_priv->pin_count--; BUG_ON(obj_priv->pin_count < 0); BUG_ON(obj_priv->gtt_space == NULL); @@ -4096,14 +4214,12 @@ i915_gem_object_unpin(struct drm_gem_object *obj) * the inactive list */ if (obj_priv->pin_count == 0) { - if (!obj_priv->active && - (obj->write_domain & I915_GEM_GPU_DOMAINS) == 0) + if (!obj_priv->active) list_move_tail(&obj_priv->list, &dev_priv->mm.inactive_list); - atomic_dec(&dev->pin_count); - atomic_sub(obj->size, &dev->pin_memory); + i915_gem_info_remove_pin(dev_priv, obj->size); } - i915_verify_inactive(dev, __FILE__, __LINE__); + WARN_ON(i915_verify_lists(dev)); } int @@ -4115,17 +4231,20 @@ i915_gem_pin_ioctl(struct drm_device *dev, void *data, struct drm_i915_gem_object *obj_priv; int ret; - mutex_lock(&dev->struct_mutex); - obj = drm_gem_object_lookup(dev, file_priv, args->handle); if (obj == NULL) { DRM_ERROR("Bad handle in i915_gem_pin_ioctl(): %d\n", args->handle); - mutex_unlock(&dev->struct_mutex); return -ENOENT; } obj_priv = to_intel_bo(obj); + ret = i915_mutex_lock_interruptible(dev); + if (ret) { + drm_gem_object_unreference_unlocked(obj); + return ret; + } + if (obj_priv->madv != I915_MADV_WILLNEED) { DRM_ERROR("Attempting to pin a purgeable buffer\n"); drm_gem_object_unreference(obj); @@ -4170,18 +4289,23 @@ i915_gem_unpin_ioctl(struct drm_device *dev, void *data, struct drm_i915_gem_pin *args = data; struct drm_gem_object *obj; struct drm_i915_gem_object *obj_priv; - - mutex_lock(&dev->struct_mutex); + int ret; obj = drm_gem_object_lookup(dev, file_priv, args->handle); if (obj == NULL) { DRM_ERROR("Bad handle in i915_gem_unpin_ioctl(): %d\n", args->handle); - mutex_unlock(&dev->struct_mutex); return -ENOENT; } obj_priv = to_intel_bo(obj); + + ret = i915_mutex_lock_interruptible(dev); + if (ret) { + drm_gem_object_unreference_unlocked(obj); + return ret; + } + if (obj_priv->pin_filp != file_priv) { DRM_ERROR("Not pinned by caller in i915_gem_pin_ioctl(): %d\n", args->handle); @@ -4207,6 +4331,7 @@ i915_gem_busy_ioctl(struct drm_device *dev, void *data, struct drm_i915_gem_busy *args = data; struct drm_gem_object *obj; struct drm_i915_gem_object *obj_priv; + int ret; obj = drm_gem_object_lookup(dev, file_priv, args->handle); if (obj == NULL) { @@ -4215,7 +4340,11 @@ i915_gem_busy_ioctl(struct drm_device *dev, void *data, return -ENOENT; } - mutex_lock(&dev->struct_mutex); + ret = i915_mutex_lock_interruptible(dev); + if (ret) { + drm_gem_object_unreference_unlocked(obj); + return ret; + } /* Count all active objects as busy, even if they are currently not used * by the gpu. Users of this interface expect objects to eventually @@ -4230,10 +4359,10 @@ i915_gem_busy_ioctl(struct drm_device *dev, void *data, * use this buffer rather sooner than later, so issuing the required * flush earlier is beneficial. */ - if (obj->write_domain) { - i915_gem_flush(dev, 0, obj->write_domain); - (void)i915_add_request(dev, file_priv, obj->write_domain, obj_priv->ring); - } + if (obj->write_domain & I915_GEM_GPU_DOMAINS) + i915_gem_flush_ring(dev, file_priv, + obj_priv->ring, + 0, obj->write_domain); /* Update the active list for the hardware's current position. * Otherwise this only updates on a delayed timer or when irqs @@ -4264,6 +4393,7 @@ i915_gem_madvise_ioctl(struct drm_device *dev, void *data, struct drm_i915_gem_madvise *args = data; struct drm_gem_object *obj; struct drm_i915_gem_object *obj_priv; + int ret; switch (args->madv) { case I915_MADV_DONTNEED: @@ -4279,10 +4409,14 @@ i915_gem_madvise_ioctl(struct drm_device *dev, void *data, args->handle); return -ENOENT; } - - mutex_lock(&dev->struct_mutex); obj_priv = to_intel_bo(obj); + ret = i915_mutex_lock_interruptible(dev); + if (ret) { + drm_gem_object_unreference_unlocked(obj); + return ret; + } + if (obj_priv->pin_count) { drm_gem_object_unreference(obj); mutex_unlock(&dev->struct_mutex); @@ -4310,6 +4444,7 @@ i915_gem_madvise_ioctl(struct drm_device *dev, void *data, struct drm_gem_object * i915_gem_alloc_object(struct drm_device *dev, size_t size) { + struct drm_i915_private *dev_priv = dev->dev_private; struct drm_i915_gem_object *obj; obj = kzalloc(sizeof(*obj), GFP_KERNEL); @@ -4321,6 +4456,8 @@ struct drm_gem_object * i915_gem_alloc_object(struct drm_device *dev, return NULL; } + i915_gem_info_add_obj(dev_priv, size); + obj->base.write_domain = I915_GEM_DOMAIN_CPU; obj->base.read_domains = I915_GEM_DOMAIN_CPU; @@ -4361,6 +4498,7 @@ static void i915_gem_free_object_tail(struct drm_gem_object *obj) i915_gem_free_mmap_offset(obj); drm_gem_object_release(obj); + i915_gem_info_remove_obj(dev_priv, obj->size); kfree(obj_priv->page_cpu_valid); kfree(obj_priv->bit_17); @@ -4419,7 +4557,7 @@ i915_gem_idle(struct drm_device *dev) * And not confound mm.suspended! */ dev_priv->mm.suspended = 1; - del_timer(&dev_priv->hangcheck_timer); + del_timer_sync(&dev_priv->hangcheck_timer); i915_kernel_lost_context(dev); i915_gem_cleanup_ringbuffer(dev); @@ -4499,28 +4637,18 @@ i915_gem_init_ringbuffer(struct drm_device *dev) drm_i915_private_t *dev_priv = dev->dev_private; int ret; - dev_priv->render_ring = render_ring; - - if (!I915_NEED_GFX_HWS(dev)) { - dev_priv->render_ring.status_page.page_addr - = dev_priv->status_page_dmah->vaddr; - memset(dev_priv->render_ring.status_page.page_addr, - 0, PAGE_SIZE); - } - if (HAS_PIPE_CONTROL(dev)) { ret = i915_gem_init_pipe_control(dev); if (ret) return ret; } - ret = intel_init_ring_buffer(dev, &dev_priv->render_ring); + ret = intel_init_render_ring_buffer(dev); if (ret) goto cleanup_pipe_control; if (HAS_BSD(dev)) { - dev_priv->bsd_ring = bsd_ring; - ret = intel_init_ring_buffer(dev, &dev_priv->bsd_ring); + ret = intel_init_bsd_ring_buffer(dev); if (ret) goto cleanup_render_ring; } @@ -4573,11 +4701,8 @@ i915_gem_entervt_ioctl(struct drm_device *dev, void *data, return ret; } - spin_lock(&dev_priv->mm.active_list_lock); BUG_ON(!list_empty(&dev_priv->render_ring.active_list)); BUG_ON(HAS_BSD(dev) && !list_empty(&dev_priv->bsd_ring.active_list)); - spin_unlock(&dev_priv->mm.active_list_lock); - BUG_ON(!list_empty(&dev_priv->mm.flushing_list)); BUG_ON(!list_empty(&dev_priv->mm.inactive_list)); BUG_ON(!list_empty(&dev_priv->render_ring.request_list)); @@ -4629,10 +4754,10 @@ i915_gem_load(struct drm_device *dev) int i; drm_i915_private_t *dev_priv = dev->dev_private; - spin_lock_init(&dev_priv->mm.active_list_lock); INIT_LIST_HEAD(&dev_priv->mm.flushing_list); INIT_LIST_HEAD(&dev_priv->mm.gpu_write_list); INIT_LIST_HEAD(&dev_priv->mm.inactive_list); + INIT_LIST_HEAD(&dev_priv->mm.pinned_list); INIT_LIST_HEAD(&dev_priv->mm.fence_list); INIT_LIST_HEAD(&dev_priv->mm.deferred_free_list); INIT_LIST_HEAD(&dev_priv->render_ring.active_list); @@ -4645,6 +4770,7 @@ i915_gem_load(struct drm_device *dev) INIT_LIST_HEAD(&dev_priv->fence_regs[i].lru_list); INIT_DELAYED_WORK(&dev_priv->mm.retire_work, i915_gem_retire_work_handler); + init_completion(&dev_priv->error_completion); spin_lock(&shrink_list_lock); list_add(&dev_priv->mm.shrink_list, &shrink_list); spin_unlock(&shrink_list_lock); @@ -4663,21 +4789,30 @@ i915_gem_load(struct drm_device *dev) if (!drm_core_check_feature(dev, DRIVER_MODESET)) dev_priv->fence_reg_start = 3; - if (IS_I965G(dev) || IS_I945G(dev) || IS_I945GM(dev) || IS_G33(dev)) + if (INTEL_INFO(dev)->gen >= 4 || IS_I945G(dev) || IS_I945GM(dev) || IS_G33(dev)) dev_priv->num_fence_regs = 16; else dev_priv->num_fence_regs = 8; /* Initialize fence registers to zero */ - if (IS_I965G(dev)) { + switch (INTEL_INFO(dev)->gen) { + case 6: + for (i = 0; i < 16; i++) + I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 + (i * 8), 0); + break; + case 5: + case 4: for (i = 0; i < 16; i++) I915_WRITE64(FENCE_REG_965_0 + (i * 8), 0); - } else { - for (i = 0; i < 8; i++) - I915_WRITE(FENCE_REG_830_0 + (i * 4), 0); + break; + case 3: if (IS_I945G(dev) || IS_I945GM(dev) || IS_G33(dev)) for (i = 0; i < 8; i++) I915_WRITE(FENCE_REG_945_8 + (i * 4), 0); + case 2: + for (i = 0; i < 8; i++) + I915_WRITE(FENCE_REG_830_0 + (i * 4), 0); + break; } i915_gem_detect_bit_6_swizzle(dev); init_waitqueue_head(&dev_priv->pending_flip_queue); @@ -4687,8 +4822,8 @@ i915_gem_load(struct drm_device *dev) * Create a physically contiguous memory object for this object * e.g. for cursor + overlay regs */ -int i915_gem_init_phys_object(struct drm_device *dev, - int id, int size, int align) +static int i915_gem_init_phys_object(struct drm_device *dev, + int id, int size, int align) { drm_i915_private_t *dev_priv = dev->dev_private; struct drm_i915_gem_phys_object *phys_obj; @@ -4720,7 +4855,7 @@ kfree_obj: return ret; } -void i915_gem_free_phys_object(struct drm_device *dev, int id) +static void i915_gem_free_phys_object(struct drm_device *dev, int id) { drm_i915_private_t *dev_priv = dev->dev_private; struct drm_i915_gem_phys_object *phys_obj; @@ -4865,18 +5000,25 @@ i915_gem_phys_pwrite(struct drm_device *dev, struct drm_gem_object *obj, return 0; } -void i915_gem_release(struct drm_device * dev, struct drm_file *file_priv) +void i915_gem_release(struct drm_device *dev, struct drm_file *file) { - struct drm_i915_file_private *i915_file_priv = file_priv->driver_priv; + struct drm_i915_file_private *file_priv = file->driver_priv; /* Clean up our request list when the client is going away, so that * later retire_requests won't dereference our soon-to-be-gone * file_priv. */ - mutex_lock(&dev->struct_mutex); - while (!list_empty(&i915_file_priv->mm.request_list)) - list_del_init(i915_file_priv->mm.request_list.next); - mutex_unlock(&dev->struct_mutex); + spin_lock(&file_priv->mm.lock); + while (!list_empty(&file_priv->mm.request_list)) { + struct drm_i915_gem_request *request; + + request = list_first_entry(&file_priv->mm.request_list, + struct drm_i915_gem_request, + client_list); + list_del(&request->client_list); + request->file_priv = NULL; + } + spin_unlock(&file_priv->mm.lock); } static int @@ -4885,12 +5027,10 @@ i915_gpu_is_active(struct drm_device *dev) drm_i915_private_t *dev_priv = dev->dev_private; int lists_empty; - spin_lock(&dev_priv->mm.active_list_lock); lists_empty = list_empty(&dev_priv->mm.flushing_list) && list_empty(&dev_priv->render_ring.active_list); if (HAS_BSD(dev)) lists_empty &= list_empty(&dev_priv->bsd_ring.active_list); - spin_unlock(&dev_priv->mm.active_list_lock); return !lists_empty; } |