ioat: split ioat_dma_probe into core/version-specific routines

Towards the removal of ioatdma_device.version split the initialization
path into distinct versions.  This conversion:
1/ moves version specific probe code to version specific routines
2/ removes the need for ioat_device
3/ turns off the ioat1 msi quirk if the device is reinitialized for intx

Signed-off-by: Maciej Sosnowski <maciej.sosnowski@intel.com>
Signed-off-by: Dan Williams <dan.j.williams@intel.com>

1 files changed, 154 insertions, 99 deletions

diff --git a/drivers/dma/ioat/dma.c b/drivers/dma/ioat/dma.c
index 462dae62719..b7508041c6d 100644
--- a/drivers/dma/ioat/dma.c
+++ b/drivers/dma/ioat/dma.c
@@ -121,52 +121,21 @@ static int ioat_dma_enumerate_channels(struct ioatdma_device *device)
 	int i;
 	struct ioat_dma_chan *ioat_chan;
 	struct device *dev = &device->pdev->dev;
+	struct dma_device *dma = &device->common;
 
-	/*
-	 * IOAT ver.3 workarounds
-	 */
-	if (device->version == IOAT_VER_3_0) {
-		u32 chan_err_mask;
-		u16 dev_id;
-		u32 dmauncerrsts;
-
-		/*
-		 * Write CHANERRMSK_INT with 3E07h to mask out the errors
-		 * that can cause stability issues for IOAT ver.3
-		 */
-		chan_err_mask = 0x3E07;
-		pci_write_config_dword(device->pdev,
-			IOAT_PCI_CHANERRMASK_INT_OFFSET,
-			chan_err_mask);
-
-		/*
-		 * Clear DMAUNCERRSTS Cfg-Reg Parity Error status bit
-		 * (workaround for spurious config parity error after restart)
-		 */
-		pci_read_config_word(device->pdev,
-			IOAT_PCI_DEVICE_ID_OFFSET,
-			&dev_id);
-		if (dev_id == PCI_DEVICE_ID_INTEL_IOAT_TBG0) {
-			dmauncerrsts = 0x10;
-			pci_write_config_dword(device->pdev,
-				IOAT_PCI_DMAUNCERRSTS_OFFSET,
-				dmauncerrsts);
-		}
-	}
-
-	device->common.chancnt = readb(device->reg_base + IOAT_CHANCNT_OFFSET);
+	INIT_LIST_HEAD(&dma->channels);
+	dma->chancnt = readb(device->reg_base + IOAT_CHANCNT_OFFSET);
 	xfercap_scale = readb(device->reg_base + IOAT_XFERCAP_OFFSET);
 	xfercap = (xfercap_scale == 0 ? -1 : (1UL << xfercap_scale));
 
 #ifdef  CONFIG_I7300_IDLE_IOAT_CHANNEL
-	if (i7300_idle_platform_probe(NULL, NULL, 1) == 0) {
-		device->common.chancnt--;
-	}
+	if (i7300_idle_platform_probe(NULL, NULL, 1) == 0)
+		dma->chancnt--;
 #endif
-	for (i = 0; i < device->common.chancnt; i++) {
+	for (i = 0; i < dma->chancnt; i++) {
 		ioat_chan = devm_kzalloc(dev, sizeof(*ioat_chan), GFP_KERNEL);
 		if (!ioat_chan) {
-			device->common.chancnt = i;
+			dma->chancnt = i;
 			break;
 		}
 
@@ -175,28 +144,20 @@ static int ioat_dma_enumerate_channels(struct ioatdma_device *device)
 		ioat_chan->xfercap = xfercap;
 		ioat_chan->desccount = 0;
 		INIT_DELAYED_WORK(&ioat_chan->work, ioat_dma_chan_reset_part2);
-		if (ioat_chan->device->version == IOAT_VER_2_0)
-			writel(IOAT_DCACTRL_CMPL_WRITE_ENABLE |
-			       IOAT_DMA_DCA_ANY_CPU,
-			       ioat_chan->reg_base + IOAT_DCACTRL_OFFSET);
-		else if (ioat_chan->device->version == IOAT_VER_3_0)
-			writel(IOAT_DMA_DCA_ANY_CPU,
-			       ioat_chan->reg_base + IOAT_DCACTRL_OFFSET);
 		spin_lock_init(&ioat_chan->cleanup_lock);
 		spin_lock_init(&ioat_chan->desc_lock);
 		INIT_LIST_HEAD(&ioat_chan->free_desc);
 		INIT_LIST_HEAD(&ioat_chan->used_desc);
 		/* This should be made common somewhere in dmaengine.c */
 		ioat_chan->common.device = &device->common;
-		list_add_tail(&ioat_chan->common.device_node,
-			      &device->common.channels);
+		list_add_tail(&ioat_chan->common.device_node, &dma->channels);
 		device->idx[i] = ioat_chan;
 		tasklet_init(&ioat_chan->cleanup_task,
 			     ioat_dma_cleanup_tasklet,
 			     (unsigned long) ioat_chan);
 		tasklet_disable(&ioat_chan->cleanup_task);
 	}
-	return device->common.chancnt;
+	return dma->chancnt;
 }
 
 /**
@@ -1504,15 +1465,6 @@ msi:
 		pci_disable_msi(pdev);
 		goto intx;
 	}
-	/*
-	 * CB 1.2 devices need a bit set in configuration space to enable MSI
-	 */
-	if (device->version == IOAT_VER_1_2) {
-		u32 dmactrl;
-		pci_read_config_dword(pdev, IOAT_PCI_DMACTRL_OFFSET, &dmactrl);
-		dmactrl |= IOAT_PCI_DMACTRL_MSI_EN;
-		pci_write_config_dword(pdev, IOAT_PCI_DMACTRL_OFFSET, dmactrl);
-	}
 	goto done;
 
 intx:
@@ -1522,6 +1474,8 @@ intx:
 		goto err_no_irq;
 
 done:
+	if (device->intr_quirk)
+		device->intr_quirk(device);
 	intrctrl |= IOAT_INTRCTRL_MASTER_INT_EN;
 	writeb(intrctrl, device->reg_base + IOAT_INTRCTRL_OFFSET);
 	return 0;
@@ -1539,21 +1493,12 @@ static void ioat_disable_interrupts(struct ioatdma_device *device)
 	writeb(0, device->reg_base + IOAT_INTRCTRL_OFFSET);
 }
 
-struct ioatdma_device *
-ioat_dma_probe(struct pci_dev *pdev, void __iomem *iobase)
+static int ioat_probe(struct ioatdma_device *device)
 {
-	int err;
+	int err = -ENODEV;
+	struct dma_device *dma = &device->common;
+	struct pci_dev *pdev = device->pdev;
 	struct device *dev = &pdev->dev;
-	struct ioatdma_device *device;
-	struct dma_device *dma;
-
-	device = devm_kzalloc(dev, sizeof(*device), GFP_KERNEL);
-	if (!device)
-		err = -ENOMEM;
-	device->pdev = pdev;
-	device->reg_base = iobase;
-	device->version = readb(device->reg_base + IOAT_VER_OFFSET);
-	dma = &device->common;
 
 	/* DMA coherent memory pool for DMA descriptor allocations */
 	device->dma_pool = pci_pool_create("dma_desc_pool", pdev,
@@ -1572,26 +1517,13 @@ ioat_dma_probe(struct pci_dev *pdev, void __iomem *iobase)
 		goto err_completion_pool;
 	}
 
-	INIT_LIST_HEAD(&dma->channels);
 	ioat_dma_enumerate_channels(device);
 
+	dma_cap_set(DMA_MEMCPY, dma->cap_mask);
 	dma->device_alloc_chan_resources = ioat_dma_alloc_chan_resources;
 	dma->device_free_chan_resources = ioat_dma_free_chan_resources;
-	dma->dev = &pdev->dev;
-
-	dma_cap_set(DMA_MEMCPY, dma->cap_mask);
 	dma->device_is_tx_complete = ioat_dma_is_complete;
-	switch (device->version) {
-	case IOAT_VER_1_2:
-		dma->device_prep_dma_memcpy = ioat1_dma_prep_memcpy;
-		dma->device_issue_pending = ioat1_dma_memcpy_issue_pending;
-		break;
-	case IOAT_VER_2_0:
-	case IOAT_VER_3_0:
-		dma->device_prep_dma_memcpy = ioat2_dma_prep_memcpy;
-		dma->device_issue_pending = ioat2_dma_memcpy_issue_pending;
-		break;
-	}
+	dma->dev = &pdev->dev;
 
 	dev_err(dev, "Intel(R) I/OAT DMA Engine found,"
 		" %d channels, device version 0x%02x, driver version %s\n",
@@ -1611,19 +1543,7 @@ ioat_dma_probe(struct pci_dev *pdev, void __iomem *iobase)
 	if (err)
 		goto err_self_test;
 
-	err = dma_async_device_register(dma);
-	if (err)
-		goto err_self_test;
-
-	ioat_set_tcp_copy_break(device);
-
-	if (device->version != IOAT_VER_3_0) {
-		INIT_DELAYED_WORK(&device->work, ioat_dma_chan_watchdog);
-		schedule_delayed_work(&device->work,
-				      WATCHDOG_DELAY);
-	}
-
-	return device;
+	return 0;
 
 err_self_test:
 	ioat_disable_interrupts(device);
@@ -1632,7 +1552,142 @@ err_setup_interrupts:
 err_completion_pool:
 	pci_pool_destroy(device->dma_pool);
 err_dma_pool:
-	return NULL;
+	return err;
+}
+
+static int ioat_register(struct ioatdma_device *device)
+{
+	int err = dma_async_device_register(&device->common);
+
+	if (err) {
+		ioat_disable_interrupts(device);
+		pci_pool_destroy(device->completion_pool);
+		pci_pool_destroy(device->dma_pool);
+	}
+
+	return err;
+}
+
+/* ioat1_intr_quirk - fix up dma ctrl register to enable / disable msi */
+static void ioat1_intr_quirk(struct ioatdma_device *device)
+{
+	struct pci_dev *pdev = device->pdev;
+	u32 dmactrl;
+
+	pci_read_config_dword(pdev, IOAT_PCI_DMACTRL_OFFSET, &dmactrl);
+	if (pdev->msi_enabled)
+		dmactrl |= IOAT_PCI_DMACTRL_MSI_EN;
+	else
+		dmactrl &= ~IOAT_PCI_DMACTRL_MSI_EN;
+	pci_write_config_dword(pdev, IOAT_PCI_DMACTRL_OFFSET, dmactrl);
+}
+
+int ioat1_dma_probe(struct ioatdma_device *device, int dca)
+{
+	struct pci_dev *pdev = device->pdev;
+	struct dma_device *dma;
+	int err;
+
+	device->intr_quirk = ioat1_intr_quirk;
+	dma = &device->common;
+	dma->device_prep_dma_memcpy = ioat1_dma_prep_memcpy;
+	dma->device_issue_pending = ioat1_dma_memcpy_issue_pending;
+
+	err = ioat_probe(device);
+	if (err)
+		return err;
+	ioat_set_tcp_copy_break(4096);
+	err = ioat_register(device);
+	if (err)
+		return err;
+	if (dca)
+		device->dca = ioat_dca_init(pdev, device->reg_base);
+
+	INIT_DELAYED_WORK(&device->work, ioat_dma_chan_watchdog);
+	schedule_delayed_work(&device->work, WATCHDOG_DELAY);
+
+	return err;
+}
+
+int ioat2_dma_probe(struct ioatdma_device *device, int dca)
+{
+	struct pci_dev *pdev = device->pdev;
+	struct dma_device *dma;
+	struct dma_chan *chan;
+	struct ioat_dma_chan *ioat_chan;
+	int err;
+
+	dma = &device->common;
+	dma->device_prep_dma_memcpy = ioat2_dma_prep_memcpy;
+	dma->device_issue_pending = ioat2_dma_memcpy_issue_pending;
+
+	err = ioat_probe(device);
+	if (err)
+		return err;
+	ioat_set_tcp_copy_break(2048);
+
+	list_for_each_entry(chan, &dma->channels, device_node) {
+		ioat_chan = to_ioat_chan(chan);
+		writel(IOAT_DCACTRL_CMPL_WRITE_ENABLE | IOAT_DMA_DCA_ANY_CPU,
+		       ioat_chan->reg_base + IOAT_DCACTRL_OFFSET);
+	}
+
+	err = ioat_register(device);
+	if (err)
+		return err;
+	if (dca)
+		device->dca = ioat2_dca_init(pdev, device->reg_base);
+
+	INIT_DELAYED_WORK(&device->work, ioat_dma_chan_watchdog);
+	schedule_delayed_work(&device->work, WATCHDOG_DELAY);
+
+	return err;
+}
+
+int ioat3_dma_probe(struct ioatdma_device *device, int dca)
+{
+	struct pci_dev *pdev = device->pdev;
+	struct dma_device *dma;
+	struct dma_chan *chan;
+	struct ioat_dma_chan *ioat_chan;
+	int err;
+	u16 dev_id;
+
+	dma = &device->common;
+	dma->device_prep_dma_memcpy = ioat2_dma_prep_memcpy;
+	dma->device_issue_pending = ioat2_dma_memcpy_issue_pending;
+
+	/* -= IOAT ver.3 workarounds =- */
+	/* Write CHANERRMSK_INT with 3E07h to mask out the errors
+	 * that can cause stability issues for IOAT ver.3
+	 */
+	pci_write_config_dword(pdev, IOAT_PCI_CHANERRMASK_INT_OFFSET, 0x3e07);
+
+	/* Clear DMAUNCERRSTS Cfg-Reg Parity Error status bit
+	 * (workaround for spurious config parity error after restart)
+	 */
+	pci_read_config_word(pdev, IOAT_PCI_DEVICE_ID_OFFSET, &dev_id);
+	if (dev_id == PCI_DEVICE_ID_INTEL_IOAT_TBG0)
+		pci_write_config_dword(pdev, IOAT_PCI_DMAUNCERRSTS_OFFSET, 0x10);
+
+	err = ioat_probe(device);
+	if (err)
+		return err;
+	ioat_set_tcp_copy_break(262144);
+
+	list_for_each_entry(chan, &dma->channels, device_node) {
+		ioat_chan = to_ioat_chan(chan);
+		writel(IOAT_DMA_DCA_ANY_CPU,
+		       ioat_chan->reg_base + IOAT_DCACTRL_OFFSET);
+	}
+
+	err = ioat_register(device);
+	if (err)
+		return err;
+	if (dca)
+		device->dca = ioat3_dca_init(pdev, device->reg_base);
+
+	return err;
 }
 
 void ioat_dma_remove(struct ioatdma_device *device)