drm/vbl rework: rework how the drm deals with vblank.

Other Authors: Michel Dänzer <michel@tungstengraphics.com>
mga: Ian Romanick <idr@us.ibm.com>
via: Thomas Hellstrom <thomas-at-tungstengraphics-dot-com>

This re-works the DRM internals to provide a better interface for drivers
to expose vblank on multiple crtcs.

It also includes work done by Michel on making i915 triple buffering and pageflipping work properly.

Signed-off-by: Dave Airlie <airlied@redhat.com>

20 files changed, 1372 insertions, 473 deletions

diff --git a/drivers/char/drm/drm.h b/drivers/char/drm/drm.h
index 3a05c6d5ebe..6874f31ca8c 100644
--- a/drivers/char/drm/drm.h
+++ b/drivers/char/drm/drm.h
@@ -471,6 +471,7 @@ struct drm_irq_busid {
 enum drm_vblank_seq_type {
 	_DRM_VBLANK_ABSOLUTE = 0x0,	/**< Wait for specific vblank sequence number */
 	_DRM_VBLANK_RELATIVE = 0x1,	/**< Wait for given number of vblanks */
+	_DRM_VBLANK_FLIP = 0x8000000,	/**< Scheduled buffer swap should flip */
 	_DRM_VBLANK_NEXTONMISS = 0x10000000,	/**< If missed, wait for next vblank */
 	_DRM_VBLANK_SECONDARY = 0x20000000,	/**< Secondary display controller */
 	_DRM_VBLANK_SIGNAL = 0x40000000	/**< Send signal instead of blocking */
@@ -503,6 +504,21 @@ union drm_wait_vblank {
 	struct drm_wait_vblank_reply reply;
 };
 
+enum drm_modeset_ctl_cmd {
+	_DRM_PRE_MODESET = 1,
+	_DRM_POST_MODESET = 2,
+};
+
+/**
+ * DRM_IOCTL_MODESET_CTL ioctl argument type
+ *
+ * \sa drmModesetCtl().
+ */
+struct drm_modeset_ctl {
+	unsigned long arg;
+	enum drm_modeset_ctl_cmd cmd;
+};
+
 /**
  * DRM_IOCTL_AGP_ENABLE ioctl argument type.
  *
@@ -587,6 +603,7 @@ struct drm_set_version {
 #define DRM_IOCTL_GET_CLIENT            DRM_IOWR(0x05, struct drm_client)
 #define DRM_IOCTL_GET_STATS             DRM_IOR( 0x06, struct drm_stats)
 #define DRM_IOCTL_SET_VERSION		DRM_IOWR(0x07, struct drm_set_version)
+#define DRM_IOCTL_MODESET_CTL           DRM_IOW(0x08, struct drm_modeset_ctl)
 
 #define DRM_IOCTL_SET_UNIQUE		DRM_IOW( 0x10, struct drm_unique)
 #define DRM_IOCTL_AUTH_MAGIC		DRM_IOW( 0x11, struct drm_auth)
diff --git a/drivers/char/drm/drmP.h b/drivers/char/drm/drmP.h
index 5446235094d..ecee3547a13 100644
--- a/drivers/char/drm/drmP.h
+++ b/drivers/char/drm/drmP.h
@@ -100,10 +100,8 @@ struct drm_device;
 #define DRIVER_HAVE_DMA    0x20
 #define DRIVER_HAVE_IRQ    0x40
 #define DRIVER_IRQ_SHARED  0x80
-#define DRIVER_IRQ_VBL     0x100
 #define DRIVER_DMA_QUEUE   0x200
 #define DRIVER_FB_DMA      0x400
-#define DRIVER_IRQ_VBL2    0x800
 
 /***********************************************************************/
 /** \name Begin the DRM... */
@@ -579,10 +577,52 @@ struct drm_driver {
 	int (*context_dtor) (struct drm_device *dev, int context);
 	int (*kernel_context_switch) (struct drm_device *dev, int old,
 				      int new);
-	void (*kernel_context_switch_unlock) (struct drm_device *dev);
-	int (*vblank_wait) (struct drm_device *dev, unsigned int *sequence);
-	int (*vblank_wait2) (struct drm_device *dev, unsigned int *sequence);
-	int (*dri_library_name) (struct drm_device *dev, char *buf);
+	void (*kernel_context_switch_unlock) (struct drm_device * dev);
+	/**
+	 * get_vblank_counter - get raw hardware vblank counter
+	 * @dev: DRM device
+	 * @crtc: counter to fetch
+	 *
+	 * Driver callback for fetching a raw hardware vblank counter
+	 * for @crtc.  If a device doesn't have a hardware counter, the
+	 * driver can simply return the value of drm_vblank_count and
+	 * make the enable_vblank() and disable_vblank() hooks into no-ops,
+	 * leaving interrupts enabled at all times.
+	 *
+	 * Wraparound handling and loss of events due to modesetting is dealt
+	 * with in the DRM core code.
+	 *
+	 * RETURNS
+	 * Raw vblank counter value.
+	 */
+	u32 (*get_vblank_counter) (struct drm_device *dev, int crtc);
+
+	/**
+	 * enable_vblank - enable vblank interrupt events
+	 * @dev: DRM device
+	 * @crtc: which irq to enable
+	 *
+	 * Enable vblank interrupts for @crtc.  If the device doesn't have
+	 * a hardware vblank counter, this routine should be a no-op, since
+	 * interrupts will have to stay on to keep the count accurate.
+	 *
+	 * RETURNS
+	 * Zero on success, appropriate errno if the given @crtc's vblank
+	 * interrupt cannot be enabled.
+	 */
+	int (*enable_vblank) (struct drm_device *dev, int crtc);
+
+	/**
+	 * disable_vblank - disable vblank interrupt events
+	 * @dev: DRM device
+	 * @crtc: which irq to enable
+	 *
+	 * Disable vblank interrupts for @crtc.  If the device doesn't have
+	 * a hardware vblank counter, this routine should be a no-op, since
+	 * interrupts will have to stay on to keep the count accurate.
+	 */
+	void (*disable_vblank) (struct drm_device *dev, int crtc);
+	int (*dri_library_name) (struct drm_device *dev, char * buf);
 
 	/**
 	 * Called by \c drm_device_is_agp.  Typically used to determine if a
@@ -601,7 +641,7 @@ struct drm_driver {
 
 	irqreturn_t(*irq_handler) (DRM_IRQ_ARGS);
 	void (*irq_preinstall) (struct drm_device *dev);
-	void (*irq_postinstall) (struct drm_device *dev);
+	int (*irq_postinstall) (struct drm_device *dev);
 	void (*irq_uninstall) (struct drm_device *dev);
 	void (*reclaim_buffers) (struct drm_device *dev,
 				 struct drm_file * file_priv);
@@ -730,13 +770,21 @@ struct drm_device {
 	/** \name VBLANK IRQ support */
 	/*@{ */
 
-	wait_queue_head_t vbl_queue;	/**< VBLANK wait queue */
-	atomic_t vbl_received;
-	atomic_t vbl_received2;		/**< number of secondary VBLANK interrupts */
+	wait_queue_head_t *vbl_queue;	/**< VBLANK wait queue */
+	atomic_t *_vblank_count;	/**< number of VBLANK interrupts (driver must alloc the right number of counters) */
 	spinlock_t vbl_lock;
-	struct list_head vbl_sigs;		/**< signal list to send on VBLANK */
-	struct list_head vbl_sigs2;	/**< signals to send on secondary VBLANK */
-	unsigned int vbl_pending;
+	struct list_head *vbl_sigs;		/**< signal list to send on VBLANK */
+	atomic_t vbl_signal_pending;	/* number of signals pending on all crtcs*/
+	atomic_t *vblank_refcount;	/* number of users of vblank interrupts per crtc */
+	u32 *last_vblank;		/* protected by dev->vbl_lock, used */
+					/* for wraparound handling */
+	u32 *vblank_offset;		/* used to track how many vblanks */
+	int *vblank_enabled;		/* so we don't call enable more than
+					   once per disable */
+	u32 *vblank_premodeset;		/*  were lost during modeset */
+	struct timer_list vblank_disable_timer;
+
+	unsigned long max_vblank_count; /**< size of vblank counter register */
 	spinlock_t tasklet_lock;	/**< For drm_locked_tasklet */
 	void (*locked_tasklet_func)(struct drm_device *dev);
 
@@ -756,6 +804,7 @@ struct drm_device {
 #ifdef __alpha__
 	struct pci_controller *hose;
 #endif
+	int num_crtcs;			/**< Number of CRTCs on this device */
 	struct drm_sg_mem *sg;	/**< Scatter gather memory */
 	void *dev_private;		/**< device private data */
 	struct drm_sigdata sigdata;	   /**< For block_all_signals */
@@ -990,11 +1039,19 @@ extern void drm_driver_irq_preinstall(struct drm_device *dev);
 extern void drm_driver_irq_postinstall(struct drm_device *dev);
 extern void drm_driver_irq_uninstall(struct drm_device *dev);
 
-extern int drm_wait_vblank(struct drm_device *dev, void *data,
-			   struct drm_file *file_priv);
-extern int drm_vblank_wait(struct drm_device *dev, unsigned int *vbl_seq);
-extern void drm_vbl_send_signals(struct drm_device *dev);
+extern int drm_vblank_init(struct drm_device *dev, int num_crtcs);
+extern int drm_wait_vblank(struct drm_device *dev, void *data, struct drm_file *filp);
+extern int drm_vblank_wait(struct drm_device * dev, unsigned int *vbl_seq);
 extern void drm_locked_tasklet(struct drm_device *dev, void(*func)(struct drm_device*));
+extern u32 drm_vblank_count(struct drm_device *dev, int crtc);
+extern void drm_update_vblank_count(struct drm_device *dev, int crtc);
+extern void drm_handle_vblank(struct drm_device *dev, int crtc);
+extern int drm_vblank_get(struct drm_device *dev, int crtc);
+extern void drm_vblank_put(struct drm_device *dev, int crtc);
+
+				/* Modesetting support */
+extern int drm_modeset_ctl(struct drm_device *dev, void *data,
+			   struct drm_file *file_priv);
 
 				/* AGP/GART support (drm_agpsupport.h) */
 extern struct drm_agp_head *drm_agp_init(struct drm_device *dev);
diff --git a/drivers/char/drm/drm_irq.c b/drivers/char/drm/drm_irq.c
index 089c015c01d..286f9d61e7d 100644
--- a/drivers/char/drm/drm_irq.c
+++ b/drivers/char/drm/drm_irq.c
@@ -71,6 +71,117 @@ int drm_irq_by_busid(struct drm_device *dev, void *data,
 	return 0;
 }
 
+static void vblank_disable_fn(unsigned long arg)
+{
+	struct drm_device *dev = (struct drm_device *)arg;
+	unsigned long irqflags;
+	int i;
+
+	for (i = 0; i < dev->num_crtcs; i++) {
+		spin_lock_irqsave(&dev->vbl_lock, irqflags);
+		if (atomic_read(&dev->vblank_refcount[i]) == 0 &&
+		    dev->vblank_enabled[i]) {
+			dev->driver->disable_vblank(dev, i);
+			dev->vblank_enabled[i] = 0;
+		}
+		spin_unlock_irqrestore(&dev->vbl_lock, irqflags);
+	}
+}
+
+static void drm_vblank_cleanup(struct drm_device *dev)
+{
+	/* Bail if the driver didn't call drm_vblank_init() */
+	if (dev->num_crtcs == 0)
+		return;
+
+	del_timer(&dev->vblank_disable_timer);
+
+	vblank_disable_fn((unsigned long)dev);
+
+	drm_free(dev->vbl_queue, sizeof(*dev->vbl_queue) * dev->num_crtcs,
+		 DRM_MEM_DRIVER);
+	drm_free(dev->vbl_sigs, sizeof(*dev->vbl_sigs) * dev->num_crtcs,
+		 DRM_MEM_DRIVER);
+	drm_free(dev->_vblank_count, sizeof(*dev->_vblank_count) *
+		 dev->num_crtcs, DRM_MEM_DRIVER);
+	drm_free(dev->vblank_refcount, sizeof(*dev->vblank_refcount) *
+		 dev->num_crtcs, DRM_MEM_DRIVER);
+	drm_free(dev->vblank_enabled, sizeof(*dev->vblank_enabled) *
+		 dev->num_crtcs, DRM_MEM_DRIVER);
+	drm_free(dev->last_vblank, sizeof(*dev->last_vblank) * dev->num_crtcs,
+		 DRM_MEM_DRIVER);
+	drm_free(dev->vblank_premodeset, sizeof(*dev->vblank_premodeset) *
+		 dev->num_crtcs, DRM_MEM_DRIVER);
+	drm_free(dev->vblank_offset, sizeof(*dev->vblank_offset) * dev->num_crtcs,
+		 DRM_MEM_DRIVER);
+
+	dev->num_crtcs = 0;
+}
+
+int drm_vblank_init(struct drm_device *dev, int num_crtcs)
+{
+	int i, ret = -ENOMEM;
+
+	setup_timer(&dev->vblank_disable_timer, vblank_disable_fn,
+		    (unsigned long)dev);
+	spin_lock_init(&dev->vbl_lock);
+	atomic_set(&dev->vbl_signal_pending, 0);
+	dev->num_crtcs = num_crtcs;
+
+	dev->vbl_queue = drm_alloc(sizeof(wait_queue_head_t) * num_crtcs,
+				   DRM_MEM_DRIVER);
+	if (!dev->vbl_queue)
+		goto err;
+
+	dev->vbl_sigs = drm_alloc(sizeof(struct list_head) * num_crtcs,
+				  DRM_MEM_DRIVER);
+	if (!dev->vbl_sigs)
+		goto err;
+
+	dev->_vblank_count = drm_alloc(sizeof(atomic_t) * num_crtcs,
+				      DRM_MEM_DRIVER);
+	if (!dev->_vblank_count)
+		goto err;
+
+	dev->vblank_refcount = drm_alloc(sizeof(atomic_t) * num_crtcs,
+					 DRM_MEM_DRIVER);
+	if (!dev->vblank_refcount)
+		goto err;
+
+	dev->vblank_enabled = drm_calloc(num_crtcs, sizeof(int),
+					 DRM_MEM_DRIVER);
+	if (!dev->vblank_enabled)
+		goto err;
+
+	dev->last_vblank = drm_calloc(num_crtcs, sizeof(u32), DRM_MEM_DRIVER);
+	if (!dev->last_vblank)
+		goto err;
+
+	dev->vblank_premodeset = drm_calloc(num_crtcs, sizeof(u32),
+					    DRM_MEM_DRIVER);
+	if (!dev->vblank_premodeset)
+		goto err;
+
+	dev->vblank_offset = drm_calloc(num_crtcs, sizeof(u32), DRM_MEM_DRIVER);
+	if (!dev->vblank_offset)
+		goto err;
+
+	/* Zero per-crtc vblank stuff */
+	for (i = 0; i < num_crtcs; i++) {
+		init_waitqueue_head(&dev->vbl_queue[i]);
+		INIT_LIST_HEAD(&dev->vbl_sigs[i]);
+		atomic_set(&dev->_vblank_count[i], 0);
+		atomic_set(&dev->vblank_refcount[i], 0);
+	}
+
+	return 0;
+
+err:
+	drm_vblank_cleanup(dev);
+	return ret;
+}
+EXPORT_SYMBOL(drm_vblank_init);
+
 /**
  * Install IRQ handler.
  *
@@ -109,17 +220,6 @@ static int drm_irq_install(struct drm_device * dev)
 
 	DRM_DEBUG("irq=%d\n", dev->irq);
 
-	if (drm_core_check_feature(dev, DRIVER_IRQ_VBL)) {
-		init_waitqueue_head(&dev->vbl_queue);
-
-		spin_lock_init(&dev->vbl_lock);
-
-		INIT_LIST_HEAD(&dev->vbl_sigs);
-		INIT_LIST_HEAD(&dev->vbl_sigs2);
-
-		dev->vbl_pending = 0;
-	}
-
 	/* Before installing handler */
 	dev->driver->irq_preinstall(dev);
 
@@ -137,9 +237,14 @@ static int drm_irq_install(struct drm_device * dev)
 	}
 
 	/* After installing handler */
-	dev->driver->irq_postinstall(dev);
+	ret = dev->driver->irq_postinstall(dev);
+	if (ret < 0) {
+		mutex_lock(&dev->struct_mutex);
+		dev->irq_enabled = 0;
+		mutex_unlock(&dev->struct_mutex);
+	}
 
-	return 0;
+	return ret;
 }
 
 /**
@@ -170,6 +275,8 @@ int drm_irq_uninstall(struct drm_device * dev)
 
 	free_irq(dev->irq, dev);
 
+	drm_vblank_cleanup(dev);
+
 	dev->locked_tasklet_func = NULL;
 
 	return 0;
@@ -214,6 +321,148 @@ int drm_control(struct drm_device *dev, void *data,
 }
 
 /**
+ * drm_vblank_count - retrieve "cooked" vblank counter value
+ * @dev: DRM device
+ * @crtc: which counter to retrieve
+ *
+ * Fetches the "cooked" vblank count value that represents the number of
+ * vblank events since the system was booted, including lost events due to
+ * modesetting activity.
+ */
+u32 drm_vblank_count(struct drm_device *dev, int crtc)
+{
+	return atomic_read(&dev->_vblank_count[crtc]) +
+		dev->vblank_offset[crtc];
+}
+EXPORT_SYMBOL(drm_vblank_count);
+
+/**
+ * drm_update_vblank_count - update the master vblank counter
+ * @dev: DRM device
+ * @crtc: counter to update
+ *
+ * Call back into the driver to update the appropriate vblank counter
+ * (specified by @crtc).  Deal with wraparound, if it occurred, and
+ * update the last read value so we can deal with wraparound on the next
+ * call if necessary.
+ */
+void drm_update_vblank_count(struct drm_device *dev, int crtc)
+{
+	unsigned long irqflags;
+	u32 cur_vblank, diff;
+
+	/*
+	 * Interrupts were disabled prior to this call, so deal with counter
+	 * wrap if needed.
+	 * NOTE!  It's possible we lost a full dev->max_vblank_count events
+	 * here if the register is small or we had vblank interrupts off for
+	 * a long time.
+	 */
+	cur_vblank = dev->driver->get_vblank_counter(dev, crtc);
+	spin_lock_irqsave(&dev->vbl_lock, irqflags);
+	if (cur_vblank < dev->last_vblank[crtc]) {
+		diff = dev->max_vblank_count -
+			dev->last_vblank[crtc];
+		diff += cur_vblank;
+	} else {
+		diff = cur_vblank - dev->last_vblank[crtc];
+	}
+	dev->last_vblank[crtc] = cur_vblank;
+	spin_unlock_irqrestore(&dev->vbl_lock, irqflags);
+
+	atomic_add(diff, &dev->_vblank_count[crtc]);
+}
+EXPORT_SYMBOL(drm_update_vblank_count);
+
+/**
+ * drm_vblank_get - get a reference count on vblank events
+ * @dev: DRM device
+ * @crtc: which CRTC to own
+ *
+ * Acquire a reference count on vblank events to avoid having them disabled
+ * while in use.  Note callers will probably want to update the master counter
+ * using drm_update_vblank_count() above before calling this routine so that
+ * wakeups occur on the right vblank event.
+ *
+ * RETURNS
+ * Zero on success, nonzero on failure.
+ */
+int drm_vblank_get(struct drm_device *dev, int crtc)
+{
+	unsigned long irqflags;
+	int ret = 0;
+
+	spin_lock_irqsave(&dev->vbl_lock, irqflags);
+	/* Going from 0->1 means we have to enable interrupts again */
+	if (atomic_add_return(1, &dev->vblank_refcount[crtc]) == 1 &&
+	    !dev->vblank_enabled[crtc]) {
+		ret = dev->driver->enable_vblank(dev, crtc);
+		if (ret)
+			atomic_dec(&dev->vblank_refcount[crtc]);
+		else
+			dev->vblank_enabled[crtc] = 1;
+	}
+	spin_unlock_irqrestore(&dev->vbl_lock, irqflags);
+
+	return ret;
+}
+EXPORT_SYMBOL(drm_vblank_get);
+
+/**
+ * drm_vblank_put - give up ownership of vblank events
+ * @dev: DRM device
+ * @crtc: which counter to give up
+ *
+ * Release ownership of a given vblank counter, turning off interrupts
+ * if possible.
+ */
+void drm_vblank_put(struct drm_device *dev, int crtc)
+{
+	/* Last user schedules interrupt disable */
+	if (atomic_dec_and_test(&dev->vblank_refcount[crtc]))
+	    mod_timer(&dev->vblank_disable_timer, jiffies + 5*DRM_HZ);
+}
+EXPORT_SYMBOL(drm_vblank_put);
+
+/**
+ * drm_modeset_ctl - handle vblank event counter changes across mode switch
+ * @DRM_IOCTL_ARGS: standard ioctl arguments
+ *
+ * Applications should call the %_DRM_PRE_MODESET and %_DRM_POST_MODESET
+ * ioctls around modesetting so that any lost vblank events are accounted for.
+ */
+int drm_modeset_ctl(struct drm_device *dev, void *data,
+		    struct drm_file *file_priv)
+{
+	struct drm_modeset_ctl *modeset = data;
+	int crtc, ret = 0;
+	u32 new;
+
+	crtc = modeset->arg;
+	if (crtc >= dev->num_crtcs) {
+		ret = -EINVAL;
+		goto out;
+	}
+
+	switch (modeset->cmd) {
+	case _DRM_PRE_MODESET:
+		dev->vblank_premodeset[crtc] =
+			dev->driver->get_vblank_counter(dev, crtc);
+		break;
+	case _DRM_POST_MODESET:
+		new = dev->driver->get_vblank_counter(dev, crtc);
+		dev->vblank_offset[crtc] = dev->vblank_premodeset[crtc] - new;
+		break;
+	default:
+		ret = -EINVAL;
+		break;
+	}
+
+out:
+	return ret;
+}
+
+/**
  * Wait for VBLANK.
  *
  * \param inode device inode.
@@ -232,12 +481,13 @@ int drm_control(struct drm_device *dev, void *data,
  *
  * If a signal is not requested, then calls vblank_wait().
  */
-int drm_wait_vblank(struct drm_device *dev, void *data, struct drm_file *file_priv)
+int drm_wait_vblank(struct drm_device *dev, void *data,
+		    struct drm_file *file_priv)
 {
 	union drm_wait_vblank *vblwait = data;
 	struct timeval now;
 	int ret = 0;
-	unsigned int flags, seq;
+	unsigned int flags, seq, crtc;
 
 	if ((!dev->irq) || (!dev->irq_enabled))
 		return -EINVAL;
@@ -251,13 +501,13 @@ int drm_wait_vblank(struct drm_device *dev, void *data, struct drm_file *file_pr
 	}
 
 	flags = vblwait->request.type & _DRM_VBLANK_FLAGS_MASK;
+	crtc = flags & _DRM_VBLANK_SECONDARY ? 1 : 0;
 
-	if (!drm_core_check_feature(dev, (flags & _DRM_VBLANK_SECONDARY) ?
-				    DRIVER_IRQ_VBL2 : DRIVER_IRQ_VBL))
+	if (crtc >= dev->num_crtcs)
 		return -EINVAL;
 
-	seq = atomic_read((flags & _DRM_VBLANK_SECONDARY) ? &dev->vbl_received2
-			  : &dev->vbl_received);
+	drm_update_vblank_count(dev, crtc);
+	seq = drm_vblank_count(dev, crtc);
 
 	switch (vblwait->request.type & _DRM_VBLANK_TYPES_MASK) {
 	case _DRM_VBLANK_RELATIVE:
@@ -276,8 +526,7 @@ int drm_wait_vblank(struct drm_device *dev, void *data, struct drm_file *file_pr
 
 	if (flags & _DRM_VBLANK_SIGNAL) {
 		unsigned long irqflags;
-		struct list_head *vbl_sigs = (flags & _DRM_VBLANK_SECONDARY)
-				      ? &dev->vbl_sigs2 : &dev->vbl_sigs;
+		struct list_head *vbl_sigs = &dev->vbl_sigs[crtc];
 		struct drm_vbl_sig *vbl_sig;
 
 		spin_lock_irqsave(&dev->vbl_lock, irqflags);
@@ -298,22 +547,26 @@ int drm_wait_vblank(struct drm_device *dev, void *data, struct drm_file *file_pr
 			}
 		}
 
-		if (dev->vbl_pending >= 100) {
+		if (atomic_read(&dev->vbl_signal_pending) >= 100) {
 			spin_unlock_irqrestore(&dev->vbl_lock, irqflags);
 			return -EBUSY;
 		}
 
-		dev->vbl_pending++;
-
 		spin_unlock_irqrestore(&dev->vbl_lock, irqflags);
 
-		if (!
-		    (vbl_sig =
-		     drm_alloc(sizeof(struct drm_vbl_sig), DRM_MEM_DRIVER))) {
+		vbl_sig = drm_calloc(1, sizeof(struct drm_vbl_sig),
+				     DRM_MEM_DRIVER);
+		if (!vbl_sig)
 			return -ENOMEM;
+
+		ret = drm_vblank_get(dev, crtc);
+		if (ret) {
+			drm_free(vbl_sig, sizeof(struct drm_vbl_sig),
+				 DRM_MEM_DRIVER);
+			return ret;
 		}
 
-		memset((void *)vbl_sig, 0, sizeof(*vbl_sig));
+		atomic_inc(&dev->vbl_signal_pending);
 
 		vbl_sig->sequence = vblwait->request.sequence;
 		vbl_sig->info.si_signo = vblwait->request.signal;
@@ -327,17 +580,20 @@ int drm_wait_vblank(struct drm_device *dev, void *data, struct drm_file *file_pr
 
 		vblwait->reply.sequence = seq;
 	} else {
-		if (flags & _DRM_VBLANK_SECONDARY) {
-			if (dev->driver->vblank_wait2)
-				ret = dev->driver->vblank_wait2(dev, &vblwait->request.sequence);
-		} else if (dev->driver->vblank_wait)
-			ret =
-			    dev->driver->vblank_wait(dev,
-						     &vblwait->request.sequence);
-
+		unsigned long cur_vblank;
+
+		ret = drm_vblank_get(dev, crtc);
+		if (ret)
+			return ret;
+		DRM_WAIT_ON(ret, dev->vbl_queue[crtc], 3 * DRM_HZ,
+			    (((cur_vblank = drm_vblank_count(dev, crtc))
+			      - vblwait->request.sequence) <= (1 << 23)));
+		drm_vblank_put(dev, crtc);
 		do_gettimeofday(&now);
+
 		vblwait->reply.tval_sec = now.tv_sec;
 		vblwait->reply.tval_usec = now.tv_usec;
+		vblwait->reply.sequence = cur_vblank;
 	}
 
       done:
@@ -348,44 +604,57 @@ int drm_wait_vblank(struct drm_device *dev, void *data, struct drm_file *file_pr
  * Send the VBLANK signals.
  *
  * \param dev DRM device.
+ * \param crtc CRTC where the vblank event occurred
  *
  * Sends a signal for each task in drm_device::vbl_sigs and empties the list.
  *
  * If a signal is not requested, then calls vblank_wait().
  */
-void drm_vbl_send_signals(struct drm_device * dev)
+static void drm_vbl_send_signals(struct drm_device * dev, int crtc)
 {
+	struct drm_vbl_sig *vbl_sig, *tmp;
+	struct list_head *vbl_sigs;
+	unsigned int vbl_seq;
 	unsigned long flags;
-	int i;
 
 	spin_lock_irqsave(&dev->vbl_lock, flags);
 
-	for (i = 0; i < 2; i++) {
-		struct drm_vbl_sig *vbl_sig, *tmp;
-		struct list_head *vbl_sigs = i ? &dev->vbl_sigs2 : &dev->vbl_sigs;
-		unsigned int vbl_seq = atomic_read(i ? &dev->vbl_received2 :
-						   &dev->vbl_received);
+	vbl_sigs = &dev->vbl_sigs[crtc];
+	vbl_seq = drm_vblank_count(dev, crtc);
 
-		list_for_each_entry_safe(vbl_sig, tmp, vbl_sigs, head) {
-			if ((vbl_seq - vbl_sig->sequence) <= (1 << 23)) {
-				vbl_sig->info.si_code = vbl_seq;
-				send_sig_info(vbl_sig->info.si_signo,
-					      &vbl_sig->info, vbl_sig->task);
+	list_for_each_entry_safe(vbl_sig, tmp, vbl_sigs, head) {
+	    if ((vbl_seq - vbl_sig->sequence) <= (1 << 23)) {
+		vbl_sig->info.si_code = vbl_seq;
+		send_sig_info(vbl_sig->info.si_signo,
+			      &vbl_sig->info, vbl_sig->task);
 
-				list_del(&vbl_sig->head);
+		list_del(&vbl_sig->head);
 
-				drm_free(vbl_sig, sizeof(*vbl_sig),
-					 DRM_MEM_DRIVER);
-
-				dev->vbl_pending--;
-			}
-		}
+		drm_free(vbl_sig, sizeof(*vbl_sig),
+			 DRM_MEM_DRIVER);
+		atomic_dec(&dev->vbl_signal_pending);
+		drm_vblank_put(dev, crtc);
+	    }
 	}
 
 	spin_unlock_irqrestore(&dev->vbl_lock, flags);
 }
 
-EXPORT_SYMBOL(drm_vbl_send_signals);
+/**
+ * drm_handle_vblank - handle a vblank event
+ * @dev: DRM device
+ * @crtc: where this event occurred
+ *
+ * Drivers should call this routine in their vblank interrupt handlers to
+ * update the vblank counter and send any signals that may be pending.
+ */
+void drm_handle_vblank(struct drm_device *dev, int crtc)
+{
+	drm_update_vblank_count(dev, crtc);
+	DRM_WAKEUP(&dev->vbl_queue[crtc]);
+	drm_vbl_send_signals(dev, crtc);
+}
+EXPORT_SYMBOL(drm_handle_vblank);
 
 /**
  * Tasklet wrapper function.
diff --git a/drivers/char/drm/i915_dma.c b/drivers/char/drm/i915_dma.c
index a043bb12301..ef7bf143a80 100644
--- a/drivers/char/drm/i915_dma.c
+++ b/drivers/char/drm/i915_dma.c
@@ -415,10 +415,13 @@ static void i915_emit_breadcrumb(struct drm_device *dev)
 	drm_i915_private_t *dev_priv = dev->dev_private;
 	RING_LOCALS;
 
-	dev_priv->sarea_priv->last_enqueue = ++dev_priv->counter;
+	if (++dev_priv->counter > BREADCRUMB_MASK) {
+		 dev_priv->counter = 1;
+		 DRM_DEBUG("Breadcrumb counter wrapped around\n");
+	}
 
-	if (dev_priv->counter > 0x7FFFFFFFUL)
-		dev_priv->sarea_priv->last_enqueue = dev_priv->counter = 1;
+	if (dev_priv->sarea_priv)
+		dev_priv->sarea_priv->last_enqueue = dev_priv->counter;
 
 	BEGIN_LP_RING(4);
 	OUT_RING(CMD_STORE_DWORD_IDX);
@@ -428,6 +431,26 @@ static void i915_emit_breadcrumb(struct drm_device *dev)
 	ADVANCE_LP_RING();
 }
 
+int i915_emit_mi_flush(struct drm_device *dev, uint32_t flush)
+{
+	drm_i915_private_t *dev_priv = dev->dev_private;
+	uint32_t flush_cmd = CMD_MI_FLUSH;
+	RING_LOCALS;
+
+	flush_cmd |= flush;
+
+	i915_kernel_lost_context(dev);
+
+	BEGIN_LP_RING(4);
+	OUT_RING(flush_cmd);
+	OUT_RING(0);
+	OUT_RING(0);
+	OUT_RING(0);
+	ADVANCE_LP_RING();
+
+	return 0;
+}
+
 static int i915_dispatch_cmdbuffer(struct drm_device * dev,
 				   drm_i915_cmdbuffer_t * cmd)
 {
@@ -511,52 +534,74 @@ static int i915_dispatch_batchbuffer(struct drm_device * dev,
 	return 0;
 }
 
-static int i915_dispatch_flip(struct drm_device * dev)
+static void i915_do_dispatch_flip(struct drm_device * dev, int plane, int sync)
 {
 	drm_i915_private_t *dev_priv = dev->dev_private;
+	u32 num_pages, current_page, next_page, dspbase;
+	int shift = 2 * plane, x, y;
 	RING_LOCALS;
 
-	DRM_DEBUG("%s: page=%d pfCurrentPage=%d\n",
-		  __FUNCTION__,
-		  dev_priv->current_page,
-		  dev_priv->sarea_priv->pf_current_page);
+	/* Calculate display base offset */
+	num_pages = dev_priv->sarea_priv->third_handle ? 3 : 2;
+	current_page = (dev_priv->sarea_priv->pf_current_page >> shift) & 0x3;
+	next_page = (current_page + 1) % num_pages;
 
-	i915_kernel_lost_context(dev);
-
-	BEGIN_LP_RING(2);
-	OUT_RING(INST_PARSER_CLIENT | INST_OP_FLUSH | INST_FLUSH_MAP_CACHE);
-	OUT_RING(0);
-	ADVANCE_LP_RING();
+	switch (next_page) {
+	default:
+	case 0:
+		dspbase = dev_priv->sarea_priv->front_offset;
+		break;
+	case 1:
+		dspbase = dev_priv->sarea_priv->back_offset;
+		break;
+	case 2:
+		dspbase = dev_priv->sarea_priv->third_offset;
+		break;
+	}
 
-	BEGIN_LP_RING(6);
-	OUT_RING(CMD_OP_DISPLAYBUFFER_INFO | ASYNC_FLIP);
-	OUT_RING(0);
-	if (dev_priv->current_page == 0) {
-		OUT_RING(dev_priv->back_offset);
-		dev_priv->current_page = 1;
+	if (plane == 0) {
+		x = dev_priv->sarea_priv->planeA_x;
+		y = dev_priv->sarea_priv->planeA_y;
 	} else {
-		OUT_RING(dev_priv->front_offset);
-		dev_priv->current_page = 0;
+		x = dev_priv->sarea_priv->planeB_x;
+		y = dev_priv->sarea_priv->planeB_y;
 	}
-	OUT_RING(0);
-	ADVANCE_LP_RING();
 
-	BEGIN_LP_RING(2);
-	OUT_RING(MI_WAIT_FOR_EVENT | MI_WAIT_FOR_PLANE_A_FLIP);
-	OUT_RING(0);
-	ADVANCE_LP_RING();
+	dspbase += (y * dev_priv->sarea_priv->pitch + x) * dev_priv->cpp;
 
-	dev_priv->sarea_priv->last_enqueue = dev_priv->counter++;
+	DRM_DEBUG("plane=%d current_page=%d dspbase=0x%x\n", plane, current_page,
+		  dspbase);
 
 	BEGIN_LP_RING(4);
-	OUT_RING(CMD_STORE_DWORD_IDX);
-	OUT_RING(20);
-	OUT_RING(dev_priv->counter);
-	OUT_RING(0);
+	OUT_RING(sync ? 0 :
+		 (MI_WAIT_FOR_EVENT | (plane ? MI_WAIT_FOR_PLANE_B_FLIP :
+				       MI_WAIT_FOR_PLANE_A_FLIP)));
+	OUT_RING(CMD_OP_DISPLAYBUFFER_INFO | (sync ? 0 : ASYNC_FLIP) |
+		 (plane ? DISPLAY_PLANE_B : DISPLAY_PLANE_A));
+	OUT_RING(dev_priv->sarea_priv->pitch * dev_priv->cpp);
+	OUT_RING(dspbase);
 	ADVANCE_LP_RING();
 
-	dev_priv->sarea_priv->pf_current_page = dev_priv->current_page;
-	return 0;
+	dev_priv->sarea_priv->pf_current_page &= ~(0x3 << shift);
+	dev_priv->sarea_priv->pf_current_page |= next_page << shift;
+}
+
+void i915_dispatch_flip(struct drm_device * dev, int planes, int sync)
+{
+	drm_i915_private_t *dev_priv = dev->dev_private;
+	int i;
+
+	DRM_DEBUG("planes=0x%x pfCurrentPage=%d\n",
+		  planes, dev_priv->sarea_priv->pf_current_page);
+
+	i915_emit_mi_flush(dev, MI_READ_FLUSH | MI_EXE_FLUSH);
+
+	for (i = 0; i < 2; i++)
+		if (planes & (1 << i))
+			i915_do_dispatch_flip(dev, i, sync);
+
+	i915_emit_breadcrumb(dev);
+
 }
 
 static int i915_quiescent(struct drm_device * dev)
@@ -579,7 +624,6 @@ static int i915_batchbuffer(struct drm_device *dev, void *data,
 			    struct drm_file *file_priv)
 {
 	drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
-	u32 *hw_status = dev_priv->hw_status_page;
 	drm_i915_sarea_t *sarea_priv = (drm_i915_sarea_t *)
 	    dev_priv->sarea_priv;
 	drm_i915_batchbuffer_t *batch = data;
@@ -602,7 +646,7 @@ static int i915_batchbuffer(struct drm_device *dev, void *data,
 
 	ret = i915_dispatch_batchbuffer(dev, batch);
 
-	sarea_priv->last_dispatch = (int)hw_status[5];
+	sarea_priv->last_dispatch = READ_BREADCRUMB(dev_priv);
 	return ret;
 }
 
@@ -610,7 +654,6 @@ static int i915_cmdbuffer(struct drm_device *dev, void *data,
 			  struct drm_file *file_priv)
 {
 	drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
-	u32 *hw_status = dev_priv->hw_status_page;
 	drm_i915_sarea_t *sarea_priv = (drm_i915_sarea_t *)
 	    dev_priv->sarea_priv;
 	drm_i915_cmdbuffer_t *cmdbuf = data;
@@ -635,18 +678,51 @@ static int i915_cmdbuffer(struct drm_device *dev, void *data,
 		return ret;
 	}
 
-	sarea_priv->last_dispatch = (int)hw_status[5];
+	sarea_priv->last_dispatch = READ_BREADCRUMB(dev_priv);
+	return 0;
+}
+
+static int i915_do_cleanup_pageflip(struct drm_device * dev)
+{
+	drm_i915_private_t *dev_priv = dev->dev_private;
+	int i, planes, num_pages = dev_priv->sarea_priv->third_handle ? 3 : 2;
+
+	DRM_DEBUG("\n");
+
+	for (i = 0, planes = 0; i < 2; i++)
+		if (dev_priv->sarea_priv->pf_current_page & (0x3 << (2 * i))) {
+			dev_priv->sarea_priv->pf_current_page =
+				(dev_priv->sarea_priv->pf_current_page &
+				 ~(0x3 << (2 * i))) | ((num_pages - 1) << (2 * i));
+
+			planes |= 1 << i;
+		}
+
+	if (planes)
+		i915_dispatch_flip(dev, planes, 0);
+
 	return 0;
 }
 
 static int i915_flip_bufs(struct drm_device *dev, void *data,
 			  struct drm_file *file_priv)
 {
-	DRM_DEBUG("%s\n", __FUNCTION__);
+	drm_i915_flip_t *param = data;
+
+	DRM_DEBUG("\n");
 
 	LOCK_TEST_WITH_RETURN(dev, file_priv);
 
-	r