aboutsummaryrefslogtreecommitdiff
path: root/drivers/iommu
diff options
context:
space:
mode:
Diffstat (limited to 'drivers/iommu')
-rw-r--r--drivers/iommu/Kconfig28
-rw-r--r--drivers/iommu/Makefile1
-rw-r--r--drivers/iommu/amd_iommu.c18
-rw-r--r--drivers/iommu/amd_iommu_init.c18
-rw-r--r--drivers/iommu/amd_iommu_types.h12
-rw-r--r--drivers/iommu/amd_iommu_v2.c196
-rw-r--r--drivers/iommu/arm-smmu.c210
-rw-r--r--drivers/iommu/dmar.c522
-rw-r--r--drivers/iommu/exynos-iommu.c1052
-rw-r--r--drivers/iommu/fsl_pamu.c11
-rw-r--r--drivers/iommu/fsl_pamu_domain.c18
-rw-r--r--drivers/iommu/intel-iommu.c1642
-rw-r--r--drivers/iommu/intel_irq_remapping.c108
-rw-r--r--drivers/iommu/iova.c64
-rw-r--r--drivers/iommu/ipmmu-vmsa.c1255
-rw-r--r--drivers/iommu/irq_remapping.c12
-rw-r--r--drivers/iommu/msm_iommu_dev.c38
-rw-r--r--drivers/iommu/omap-iommu-debug.c4
-rw-r--r--drivers/iommu/omap-iommu.c185
-rw-r--r--drivers/iommu/omap-iommu.h5
-rw-r--r--drivers/iommu/omap-iommu2.c3
-rw-r--r--drivers/iommu/omap-iopgtable.h3
-rw-r--r--drivers/iommu/shmobile-iommu.c2
-rw-r--r--drivers/iommu/shmobile-ipmmu.c20
24 files changed, 3771 insertions, 1656 deletions
diff --git a/drivers/iommu/Kconfig b/drivers/iommu/Kconfig
index 79bbc21c1d0..d260605e6d5 100644
--- a/drivers/iommu/Kconfig
+++ b/drivers/iommu/Kconfig
@@ -178,13 +178,13 @@ config TEGRA_IOMMU_SMMU
config EXYNOS_IOMMU
bool "Exynos IOMMU Support"
- depends on ARCH_EXYNOS && EXYNOS_DEV_SYSMMU
+ depends on ARCH_EXYNOS
select IOMMU_API
help
- Support for the IOMMU(System MMU) of Samsung Exynos application
- processor family. This enables H/W multimedia accellerators to see
- non-linear physical memory chunks as a linear memory in their
- address spaces
+ Support for the IOMMU (System MMU) of Samsung Exynos application
+ processor family. This enables H/W multimedia accelerators to see
+ non-linear physical memory chunks as linear memory in their
+ address space.
If unsure, say N here.
@@ -193,9 +193,9 @@ config EXYNOS_IOMMU_DEBUG
depends on EXYNOS_IOMMU
help
Select this to see the detailed log message that shows what
- happens in the IOMMU driver
+ happens in the IOMMU driver.
- Say N unless you need kernel log message for IOMMU debugging
+ Say N unless you need kernel log message for IOMMU debugging.
config SHMOBILE_IPMMU
bool
@@ -207,7 +207,7 @@ config SHMOBILE_IOMMU
bool "IOMMU for Renesas IPMMU/IPMMUI"
default n
depends on ARM
- depends on SH_MOBILE || COMPILE_TEST
+ depends on ARCH_SHMOBILE || COMPILE_TEST
select IOMMU_API
select ARM_DMA_USE_IOMMU
select SHMOBILE_IPMMU
@@ -272,6 +272,18 @@ config SHMOBILE_IOMMU_L1SIZE
default 256 if SHMOBILE_IOMMU_ADDRSIZE_64MB
default 128 if SHMOBILE_IOMMU_ADDRSIZE_32MB
+config IPMMU_VMSA
+ bool "Renesas VMSA-compatible IPMMU"
+ depends on ARM_LPAE
+ depends on ARCH_SHMOBILE || COMPILE_TEST
+ select IOMMU_API
+ select ARM_DMA_USE_IOMMU
+ help
+ Support for the Renesas VMSA-compatible IPMMU Renesas found in the
+ R-Mobile APE6 and R-Car H2/M2 SoCs.
+
+ If unsure, say N.
+
config SPAPR_TCE_IOMMU
bool "sPAPR TCE IOMMU Support"
depends on PPC_POWERNV || PPC_PSERIES
diff --git a/drivers/iommu/Makefile b/drivers/iommu/Makefile
index 5d58bf16e9e..8893bad048e 100644
--- a/drivers/iommu/Makefile
+++ b/drivers/iommu/Makefile
@@ -7,6 +7,7 @@ obj-$(CONFIG_AMD_IOMMU_V2) += amd_iommu_v2.o
obj-$(CONFIG_ARM_SMMU) += arm-smmu.o
obj-$(CONFIG_DMAR_TABLE) += dmar.o
obj-$(CONFIG_INTEL_IOMMU) += iova.o intel-iommu.o
+obj-$(CONFIG_IPMMU_VMSA) += ipmmu-vmsa.o
obj-$(CONFIG_IRQ_REMAP) += intel_irq_remapping.o irq_remapping.o
obj-$(CONFIG_OMAP_IOMMU) += omap-iommu.o
obj-$(CONFIG_OMAP_IOMMU) += omap-iommu2.o
diff --git a/drivers/iommu/amd_iommu.c b/drivers/iommu/amd_iommu.c
index faf0da4bb3a..4aec6a29e31 100644
--- a/drivers/iommu/amd_iommu.c
+++ b/drivers/iommu/amd_iommu.c
@@ -963,7 +963,7 @@ static void build_inv_iommu_pasid(struct iommu_cmd *cmd, u16 domid, int pasid,
address &= ~(0xfffULL);
- cmd->data[0] = pasid & PASID_MASK;
+ cmd->data[0] = pasid;
cmd->data[1] = domid;
cmd->data[2] = lower_32_bits(address);
cmd->data[3] = upper_32_bits(address);
@@ -982,10 +982,10 @@ static void build_inv_iotlb_pasid(struct iommu_cmd *cmd, u16 devid, int pasid,
address &= ~(0xfffULL);
cmd->data[0] = devid;
- cmd->data[0] |= (pasid & 0xff) << 16;
+ cmd->data[0] |= ((pasid >> 8) & 0xff) << 16;
cmd->data[0] |= (qdep & 0xff) << 24;
cmd->data[1] = devid;
- cmd->data[1] |= ((pasid >> 8) & 0xfff) << 16;
+ cmd->data[1] |= (pasid & 0xff) << 16;
cmd->data[2] = lower_32_bits(address);
cmd->data[2] |= CMD_INV_IOMMU_PAGES_GN_MASK;
cmd->data[3] = upper_32_bits(address);
@@ -1001,7 +1001,7 @@ static void build_complete_ppr(struct iommu_cmd *cmd, u16 devid, int pasid,
cmd->data[0] = devid;
if (gn) {
- cmd->data[1] = pasid & PASID_MASK;
+ cmd->data[1] = pasid;
cmd->data[2] = CMD_INV_IOMMU_PAGES_GN_MASK;
}
cmd->data[3] = tag & 0x1ff;
@@ -3499,8 +3499,6 @@ int __init amd_iommu_init_passthrough(void)
{
struct iommu_dev_data *dev_data;
struct pci_dev *dev = NULL;
- struct amd_iommu *iommu;
- u16 devid;
int ret;
ret = alloc_passthrough_domain();
@@ -3514,12 +3512,6 @@ int __init amd_iommu_init_passthrough(void)
dev_data = get_dev_data(&dev->dev);
dev_data->passthrough = true;
- devid = get_device_id(&dev->dev);
-
- iommu = amd_iommu_rlookup_table[devid];
- if (!iommu)
- continue;
-
attach_device(&dev->dev, pt_domain);
}
@@ -3999,7 +3991,7 @@ static struct irq_remap_table *get_irq_table(u16 devid, bool ioapic)
iommu_flush_dte(iommu, devid);
if (devid != alias) {
irq_lookup_table[alias] = table;
- set_dte_irq_entry(devid, table);
+ set_dte_irq_entry(alias, table);
iommu_flush_dte(iommu, alias);
}
diff --git a/drivers/iommu/amd_iommu_init.c b/drivers/iommu/amd_iommu_init.c
index 28b4bea7c10..0e08545d729 100644
--- a/drivers/iommu/amd_iommu_init.c
+++ b/drivers/iommu/amd_iommu_init.c
@@ -150,7 +150,7 @@ int amd_iommus_present;
bool amd_iommu_np_cache __read_mostly;
bool amd_iommu_iotlb_sup __read_mostly = true;
-u32 amd_iommu_max_pasids __read_mostly = ~0;
+u32 amd_iommu_max_pasid __read_mostly = ~0;
bool amd_iommu_v2_present __read_mostly;
bool amd_iommu_pc_present __read_mostly;
@@ -788,7 +788,7 @@ static void __init set_device_exclusion_range(u16 devid, struct ivmd_header *m)
* per device. But we can enable the exclusion range per
* device. This is done here
*/
- set_dev_entry_bit(m->devid, DEV_ENTRY_EX);
+ set_dev_entry_bit(devid, DEV_ENTRY_EX);
iommu->exclusion_start = m->range_start;
iommu->exclusion_length = m->range_length;
}
@@ -1231,14 +1231,16 @@ static int iommu_init_pci(struct amd_iommu *iommu)
if (iommu_feature(iommu, FEATURE_GT)) {
int glxval;
- u32 pasids;
- u64 shift;
+ u32 max_pasid;
+ u64 pasmax;
- shift = iommu->features & FEATURE_PASID_MASK;
- shift >>= FEATURE_PASID_SHIFT;
- pasids = (1 << shift);
+ pasmax = iommu->features & FEATURE_PASID_MASK;
+ pasmax >>= FEATURE_PASID_SHIFT;
+ max_pasid = (1 << (pasmax + 1)) - 1;
- amd_iommu_max_pasids = min(amd_iommu_max_pasids, pasids);
+ amd_iommu_max_pasid = min(amd_iommu_max_pasid, max_pasid);
+
+ BUG_ON(amd_iommu_max_pasid & ~PASID_MASK);
glxval = iommu->features & FEATURE_GLXVAL_MASK;
glxval >>= FEATURE_GLXVAL_SHIFT;
diff --git a/drivers/iommu/amd_iommu_types.h b/drivers/iommu/amd_iommu_types.h
index e400fbe411d..f1a5abf11ac 100644
--- a/drivers/iommu/amd_iommu_types.h
+++ b/drivers/iommu/amd_iommu_types.h
@@ -25,6 +25,7 @@
#include <linux/list.h>
#include <linux/spinlock.h>
#include <linux/pci.h>
+#include <linux/irqreturn.h>
/*
* Maximum number of IOMMUs supported
@@ -98,7 +99,12 @@
#define FEATURE_GLXVAL_SHIFT 14
#define FEATURE_GLXVAL_MASK (0x03ULL << FEATURE_GLXVAL_SHIFT)
-#define PASID_MASK 0x000fffff
+/* Note:
+ * The current driver only support 16-bit PASID.
+ * Currently, hardware only implement upto 16-bit PASID
+ * even though the spec says it could have upto 20 bits.
+ */
+#define PASID_MASK 0x0000ffff
/* MMIO status bits */
#define MMIO_STATUS_EVT_INT_MASK (1 << 1)
@@ -696,8 +702,8 @@ extern unsigned long *amd_iommu_pd_alloc_bitmap;
*/
extern u32 amd_iommu_unmap_flush;
-/* Smallest number of PASIDs supported by any IOMMU in the system */
-extern u32 amd_iommu_max_pasids;
+/* Smallest max PASID supported by any IOMMU in the system */
+extern u32 amd_iommu_max_pasid;
extern bool amd_iommu_v2_present;
diff --git a/drivers/iommu/amd_iommu_v2.c b/drivers/iommu/amd_iommu_v2.c
index 5208828792e..499b4366a98 100644
--- a/drivers/iommu/amd_iommu_v2.c
+++ b/drivers/iommu/amd_iommu_v2.c
@@ -45,17 +45,22 @@ struct pri_queue {
struct pasid_state {
struct list_head list; /* For global state-list */
atomic_t count; /* Reference count */
+ unsigned mmu_notifier_count; /* Counting nested mmu_notifier
+ calls */
struct task_struct *task; /* Task bound to this PASID */
struct mm_struct *mm; /* mm_struct for the faults */
struct mmu_notifier mn; /* mmu_otifier handle */
struct pri_queue pri[PRI_QUEUE_SIZE]; /* PRI tag states */
struct device_state *device_state; /* Link to our device_state */
int pasid; /* PASID index */
- spinlock_t lock; /* Protect pri_queues */
+ spinlock_t lock; /* Protect pri_queues and
+ mmu_notifer_count */
wait_queue_head_t wq; /* To wait for count == 0 */
};
struct device_state {
+ struct list_head list;
+ u16 devid;
atomic_t count;
struct pci_dev *pdev;
struct pasid_state **states;
@@ -81,13 +86,9 @@ struct fault {
u16 flags;
};
-static struct device_state **state_table;
+static LIST_HEAD(state_list);
static spinlock_t state_lock;
-/* List and lock for all pasid_states */
-static LIST_HEAD(pasid_state_list);
-static DEFINE_SPINLOCK(ps_lock);
-
static struct workqueue_struct *iommu_wq;
/*
@@ -99,7 +100,6 @@ static u64 *empty_page_table;
static void free_pasid_states(struct device_state *dev_state);
static void unbind_pasid(struct device_state *dev_state, int pasid);
-static int task_exit(struct notifier_block *nb, unsigned long e, void *data);
static u16 device_id(struct pci_dev *pdev)
{
@@ -111,13 +111,25 @@ static u16 device_id(struct pci_dev *pdev)
return devid;
}
+static struct device_state *__get_device_state(u16 devid)
+{
+ struct device_state *dev_state;
+
+ list_for_each_entry(dev_state, &state_list, list) {
+ if (dev_state->devid == devid)
+ return dev_state;
+ }
+
+ return NULL;
+}
+
static struct device_state *get_device_state(u16 devid)
{
struct device_state *dev_state;
unsigned long flags;
spin_lock_irqsave(&state_lock, flags);
- dev_state = state_table[devid];
+ dev_state = __get_device_state(devid);
if (dev_state != NULL)
atomic_inc(&dev_state->count);
spin_unlock_irqrestore(&state_lock, flags);
@@ -158,29 +170,6 @@ static void put_device_state_wait(struct device_state *dev_state)
free_device_state(dev_state);
}
-static struct notifier_block profile_nb = {
- .notifier_call = task_exit,
-};
-
-static void link_pasid_state(struct pasid_state *pasid_state)
-{
- spin_lock(&ps_lock);
- list_add_tail(&pasid_state->list, &pasid_state_list);
- spin_unlock(&ps_lock);
-}
-
-static void __unlink_pasid_state(struct pasid_state *pasid_state)
-{
- list_del(&pasid_state->list);
-}
-
-static void unlink_pasid_state(struct pasid_state *pasid_state)
-{
- spin_lock(&ps_lock);
- __unlink_pasid_state(pasid_state);
- spin_unlock(&ps_lock);
-}
-
/* Must be called under dev_state->lock */
static struct pasid_state **__get_pasid_state_ptr(struct device_state *dev_state,
int pasid, bool alloc)
@@ -337,7 +326,6 @@ static void unbind_pasid(struct device_state *dev_state, int pasid)
if (pasid_state == NULL)
return;
- unlink_pasid_state(pasid_state);
__unbind_pasid(pasid_state);
put_pasid_state_wait(pasid_state); /* Reference taken in this function */
}
@@ -379,7 +367,12 @@ static void free_pasid_states(struct device_state *dev_state)
continue;
put_pasid_state(pasid_state);
- unbind_pasid(dev_state, i);
+
+ /*
+ * This will call the mn_release function and
+ * unbind the PASID
+ */
+ mmu_notifier_unregister(&pasid_state->mn, pasid_state->mm);
}
if (dev_state->pasid_levels == 2)
@@ -439,12 +432,19 @@ static void mn_invalidate_range_start(struct mmu_notifier *mn,
{
struct pasid_state *pasid_state;
struct device_state *dev_state;
+ unsigned long flags;
pasid_state = mn_to_state(mn);
dev_state = pasid_state->device_state;
- amd_iommu_domain_set_gcr3(dev_state->domain, pasid_state->pasid,
- __pa(empty_page_table));
+ spin_lock_irqsave(&pasid_state->lock, flags);
+ if (pasid_state->mmu_notifier_count == 0) {
+ amd_iommu_domain_set_gcr3(dev_state->domain,
+ pasid_state->pasid,
+ __pa(empty_page_table));
+ }
+ pasid_state->mmu_notifier_count += 1;
+ spin_unlock_irqrestore(&pasid_state->lock, flags);
}
static void mn_invalidate_range_end(struct mmu_notifier *mn,
@@ -453,15 +453,39 @@ static void mn_invalidate_range_end(struct mmu_notifier *mn,
{
struct pasid_state *pasid_state;
struct device_state *dev_state;
+ unsigned long flags;
pasid_state = mn_to_state(mn);
dev_state = pasid_state->device_state;
- amd_iommu_domain_set_gcr3(dev_state->domain, pasid_state->pasid,
- __pa(pasid_state->mm->pgd));
+ spin_lock_irqsave(&pasid_state->lock, flags);
+ pasid_state->mmu_notifier_count -= 1;
+ if (pasid_state->mmu_notifier_count == 0) {
+ amd_iommu_domain_set_gcr3(dev_state->domain,
+ pasid_state->pasid,
+ __pa(pasid_state->mm->pgd));
+ }
+ spin_unlock_irqrestore(&pasid_state->lock, flags);
+}
+
+static void mn_release(struct mmu_notifier *mn, struct mm_struct *mm)
+{
+ struct pasid_state *pasid_state;
+ struct device_state *dev_state;
+
+ might_sleep();
+
+ pasid_state = mn_to_state(mn);
+ dev_state = pasid_state->device_state;
+
+ if (pasid_state->device_state->inv_ctx_cb)
+ dev_state->inv_ctx_cb(dev_state->pdev, pasid_state->pasid);
+
+ unbind_pasid(dev_state, pasid_state->pasid);
}
static struct mmu_notifier_ops iommu_mn = {
+ .release = mn_release,
.clear_flush_young = mn_clear_flush_young,
.change_pte = mn_change_pte,
.invalidate_page = mn_invalidate_page,
@@ -504,8 +528,10 @@ static void do_fault(struct work_struct *work)
write = !!(fault->flags & PPR_FAULT_WRITE);
+ down_read(&fault->state->mm->mmap_sem);
npages = get_user_pages(fault->state->task, fault->state->mm,
fault->address, 1, write, 0, &page, NULL);
+ up_read(&fault->state->mm->mmap_sem);
if (npages == 1) {
put_page(page);
@@ -604,53 +630,6 @@ static struct notifier_block ppr_nb = {
.notifier_call = ppr_notifier,
};
-static int task_exit(struct notifier_block *nb, unsigned long e, void *data)
-{
- struct pasid_state *pasid_state;
- struct task_struct *task;
-
- task = data;
-
- /*
- * Using this notifier is a hack - but there is no other choice
- * at the moment. What I really want is a sleeping notifier that
- * is called when an MM goes down. But such a notifier doesn't
- * exist yet. The notifier needs to sleep because it has to make
- * sure that the device does not use the PASID and the address
- * space anymore before it is destroyed. This includes waiting
- * for pending PRI requests to pass the workqueue. The
- * MMU-Notifiers would be a good fit, but they use RCU and so
- * they are not allowed to sleep. Lets see how we can solve this
- * in a more intelligent way in the future.
- */
-again:
- spin_lock(&ps_lock);
- list_for_each_entry(pasid_state, &pasid_state_list, list) {
- struct device_state *dev_state;
- int pasid;
-
- if (pasid_state->task != task)
- continue;
-
- /* Drop Lock and unbind */
- spin_unlock(&ps_lock);
-
- dev_state = pasid_state->device_state;
- pasid = pasid_state->pasid;
-
- if (pasid_state->device_state->inv_ctx_cb)
- dev_state->inv_ctx_cb(dev_state->pdev, pasid);
-
- unbind_pasid(dev_state, pasid);
-
- /* Task may be in the list multiple times */
- goto again;
- }
- spin_unlock(&ps_lock);
-
- return NOTIFY_OK;
-}
-
int amd_iommu_bind_pasid(struct pci_dev *pdev, int pasid,
struct task_struct *task)
{
@@ -703,8 +682,6 @@ int amd_iommu_bind_pasid(struct pci_dev *pdev, int pasid,
if (ret)
goto out_clear_state;
- link_pasid_state(pasid_state);
-
return 0;
out_clear_state:
@@ -725,6 +702,7 @@ EXPORT_SYMBOL(amd_iommu_bind_pasid);
void amd_iommu_unbind_pasid(struct pci_dev *pdev, int pasid)
{
+ struct pasid_state *pasid_state;
struct device_state *dev_state;
u16 devid;
@@ -741,7 +719,17 @@ void amd_iommu_unbind_pasid(struct pci_dev *pdev, int pasid)
if (pasid < 0 || pasid >= dev_state->max_pasids)
goto out;
- unbind_pasid(dev_state, pasid);
+ pasid_state = get_pasid_state(dev_state, pasid);
+ if (pasid_state == NULL)
+ goto out;
+ /*
+ * Drop reference taken here. We are safe because we still hold
+ * the reference taken in the amd_iommu_bind_pasid function.
+ */
+ put_pasid_state(pasid_state);
+
+ /* This will call the mn_release function and unbind the PASID */
+ mmu_notifier_unregister(&pasid_state->mn, pasid_state->mm);
out:
put_device_state(dev_state);
@@ -771,7 +759,8 @@ int amd_iommu_init_device(struct pci_dev *pdev, int pasids)
spin_lock_init(&dev_state->lock);
init_waitqueue_head(&dev_state->wq);
- dev_state->pdev = pdev;
+ dev_state->pdev = pdev;
+ dev_state->devid = devid;
tmp = pasids;
for (dev_state->pasid_levels = 0; (tmp - 1) & ~0x1ff; tmp >>= 9)
@@ -801,13 +790,13 @@ int amd_iommu_init_device(struct pci_dev *pdev, int pasids)
spin_lock_irqsave(&state_lock, flags);
- if (state_table[devid] != NULL) {
+ if (__get_device_state(devid) != NULL) {
spin_unlock_irqrestore(&state_lock, flags);
ret = -EBUSY;
goto out_free_domain;
}
- state_table[devid] = dev_state;
+ list_add_tail(&dev_state->list, &state_list);
spin_unlock_irqrestore(&state_lock, flags);
@@ -839,13 +828,13 @@ void amd_iommu_free_device(struct pci_dev *pdev)
spin_lock_irqsave(&state_lock, flags);
- dev_state = state_table[devid];
+ dev_state = __get_device_state(devid);
if (dev_state == NULL) {
spin_unlock_irqrestore(&state_lock, flags);
return;
}
- state_table[devid] = NULL;
+ list_del(&dev_state->list);
spin_unlock_irqrestore(&state_lock, flags);
@@ -872,7 +861,7 @@ int amd_iommu_set_invalid_ppr_cb(struct pci_dev *pdev,
spin_lock_irqsave(&state_lock, flags);
ret = -EINVAL;
- dev_state = state_table[devid];
+ dev_state = __get_device_state(devid);
if (dev_state == NULL)
goto out_unlock;
@@ -903,7 +892,7 @@ int amd_iommu_set_invalidate_ctx_cb(struct pci_dev *pdev,
spin_lock_irqsave(&state_lock, flags);
ret = -EINVAL;
- dev_state = state_table[devid];
+ dev_state = __get_device_state(devid);
if (dev_state == NULL)
goto out_unlock;
@@ -920,7 +909,6 @@ EXPORT_SYMBOL(amd_iommu_set_invalidate_ctx_cb);
static int __init amd_iommu_v2_init(void)
{
- size_t state_table_size;
int ret;
pr_info("AMD IOMMUv2 driver by Joerg Roedel <joerg.roedel@amd.com>\n");
@@ -936,16 +924,10 @@ static int __init amd_iommu_v2_init(void)
spin_lock_init(&state_lock);
- state_table_size = MAX_DEVICES * sizeof(struct device_state *);
- state_table = (void *)__get_free_pages(GFP_KERNEL | __GFP_ZERO,
- get_order(state_table_size));
- if (state_table == NULL)
- return -ENOMEM;
-
ret = -ENOMEM;
iommu_wq = create_workqueue("amd_iommu_v2");
if (iommu_wq == NULL)
- goto out_free;
+ goto out;
ret = -ENOMEM;
empty_page_table = (u64 *)get_zeroed_page(GFP_KERNEL);
@@ -953,29 +935,24 @@ static int __init amd_iommu_v2_init(void)
goto out_destroy_wq;
amd_iommu_register_ppr_notifier(&ppr_nb);
- profile_event_register(PROFILE_TASK_EXIT, &profile_nb);
return 0;
out_destroy_wq:
destroy_workqueue(iommu_wq);
-out_free:
- free_pages((unsigned long)state_table, get_order(state_table_size));
-
+out:
return ret;
}
static void __exit amd_iommu_v2_exit(void)
{
struct device_state *dev_state;
- size_t state_table_size;
int i;
if (!amd_iommu_v2_supported())
return;
- profile_event_unregister(PROFILE_TASK_EXIT, &profile_nb);
amd_iommu_unregister_ppr_notifier(&ppr_nb);
flush_workqueue(iommu_wq);
@@ -998,9 +975,6 @@ static void __exit amd_iommu_v2_exit(void)
destroy_workqueue(iommu_wq);
- state_table_size = MAX_DEVICES * sizeof(struct device_state *);
- free_pages((unsigned long)state_table, get_order(state_table_size));
-
free_page((unsigned long)empty_page_table);
}
diff --git a/drivers/iommu/arm-smmu.c b/drivers/iommu/arm-smmu.c
index 8911850c944..1599354e974 100644
--- a/drivers/iommu/arm-smmu.c
+++ b/drivers/iommu/arm-smmu.c
@@ -48,7 +48,7 @@
#include <asm/pgalloc.h>
/* Maximum number of stream IDs assigned to a single device */
-#define MAX_MASTER_STREAMIDS 8
+#define MAX_MASTER_STREAMIDS MAX_PHANDLE_ARGS
/* Maximum number of context banks per SMMU */
#define ARM_SMMU_MAX_CBS 128
@@ -60,6 +60,16 @@
#define ARM_SMMU_GR0(smmu) ((smmu)->base)
#define ARM_SMMU_GR1(smmu) ((smmu)->base + (smmu)->pagesize)
+/*
+ * SMMU global address space with conditional offset to access secure
+ * aliases of non-secure registers (e.g. nsCR0: 0x400, nsGFSR: 0x448,
+ * nsGFSYNR0: 0x450)
+ */
+#define ARM_SMMU_GR0_NS(smmu) \
+ ((smmu)->base + \
+ ((smmu->options & ARM_SMMU_OPT_SECURE_CFG_ACCESS) \
+ ? 0x400 : 0))
+
/* Page table bits */
#define ARM_SMMU_PTE_XN (((pteval_t)3) << 53)
#define ARM_SMMU_PTE_CONT (((pteval_t)1) << 52)
@@ -79,7 +89,6 @@
#define ARM_SMMU_PTE_CONT_SIZE (PAGE_SIZE * ARM_SMMU_PTE_CONT_ENTRIES)
#define ARM_SMMU_PTE_CONT_MASK (~(ARM_SMMU_PTE_CONT_SIZE - 1))
-#define ARM_SMMU_PTE_HWTABLE_SIZE (PTRS_PER_PTE * sizeof(pte_t))
/* Stage-1 PTE */
#define ARM_SMMU_PTE_AP_UNPRIV (((pteval_t)1) << 6)
@@ -191,6 +200,9 @@
#define ARM_SMMU_GR1_CBAR(n) (0x0 + ((n) << 2))
#define CBAR_VMID_SHIFT 0
#define CBAR_VMID_MASK 0xff
+#define CBAR_S1_BPSHCFG_SHIFT 8
+#define CBAR_S1_BPSHCFG_MASK 3
+#define CBAR_S1_BPSHCFG_NSH 3
#define CBAR_S1_MEMATTR_SHIFT 12
#define CBAR_S1_MEMATTR_MASK 0xf
#define CBAR_S1_MEMATTR_WB 0xf
@@ -349,6 +361,9 @@ struct arm_smmu_device {
#define ARM_SMMU_FEAT_TRANS_S2 (1 << 3)
#define ARM_SMMU_FEAT_TRANS_NESTED (1 << 4)
u32 features;
+
+#define ARM_SMMU_OPT_SECURE_CFG_ACCESS (1 << 0)
+ u32 options;
int version;
u32 num_context_banks;
@@ -393,12 +408,35 @@ struct arm_smmu_domain {
struct arm_smmu_cfg root_cfg;
phys_addr_t output_mask;
- struct mutex lock;
+ spinlock_t lock;
};
static DEFINE_SPINLOCK(arm_smmu_devices_lock);
static LIST_HEAD(arm_smmu_devices);
+struct arm_smmu_option_prop {
+ u32 opt;
+ const char *prop;
+};
+
+static struct arm_smmu_option_prop arm_smmu_options [] = {
+ { ARM_SMMU_OPT_SECURE_CFG_ACCESS, "calxeda,smmu-secure-config-access" },
+ { 0, NULL},
+};
+
+static void parse_driver_options(struct arm_smmu_device *smmu)
+{
+ int i = 0;
+ do {
+ if (of_property_read_bool(smmu->dev->of_node,
+ arm_smmu_options[i].prop)) {
+ smmu->options |= arm_smmu_options[i].opt;
+ dev_notice(smmu->dev, "option %s\n",
+ arm_smmu_options[i].prop);
+ }
+ } while (arm_smmu_options[++i].opt);
+}
+
static struct arm_smmu_master *find_smmu_master(struct arm_smmu_device *smmu,
struct device_node *dev_node)
{
@@ -612,16 +650,16 @@ static irqreturn_t arm_smmu_global_fault(int irq, void *dev)
{
u32 gfsr, gfsynr0, gfsynr1, gfsynr2;
struct arm_smmu_device *smmu = dev;
- void __iomem *gr0_base = ARM_SMMU_GR0(smmu);
+ void __iomem *gr0_base = ARM_SMMU_GR0_NS(smmu);
gfsr = readl_relaxed(gr0_base + ARM_SMMU_GR0_sGFSR);
- if (!gfsr)
- return IRQ_NONE;
-
gfsynr0 = readl_relaxed(gr0_base + ARM_SMMU_GR0_sGFSYNR0);
gfsynr1 = readl_relaxed(gr0_base + ARM_SMMU_GR0_sGFSYNR1);
gfsynr2 = readl_relaxed(gr0_base + ARM_SMMU_GR0_sGFSYNR2);
+ if (!gfsr)
+ return IRQ_NONE;
+
dev_err_ratelimited(smmu->dev,
"Unexpected global fault, this could be serious\n");
dev_err_ratelimited(smmu->dev,
@@ -632,6 +670,28 @@ static irqreturn_t arm_smmu_global_fault(int irq, void *dev)
return IRQ_HANDLED;
}
+static void arm_smmu_flush_pgtable(struct arm_smmu_device *smmu, void *addr,
+ size_t size)
+{
+ unsigned long offset = (unsigned long)addr & ~PAGE_MASK;
+
+
+ /* Ensure new page tables are visible to the hardware walker */
+ if (smmu->features & ARM_SMMU_FEAT_COHERENT_WALK) {
+ dsb(ishst);
+ } else {
+ /*
+ * If the SMMU can't walk tables in the CPU caches, treat them
+ * like non-coherent DMA since we need to flush the new entries
+ * all the way out to memory. There's no possibility of
+ * recursion here as the SMMU table walker will not be wired
+ * through another SMMU.
+ */
+ dma_map_page(smmu->dev, virt_to_page(addr), offset, size,
+ DMA_TO_DEVICE);
+ }
+}
+
static void arm_smmu_init_context_bank(struct arm_smmu_domain *smmu_domain)
{
u32 reg;
@@ -650,11 +710,16 @@ static void arm_smmu_init_context_bank(struct arm_smmu_domain *smmu_domain)
if (smmu->version == 1)
reg |= root_cfg->irptndx << CBAR_IRPTNDX_SHIFT;
- /* Use the weakest memory type, so it is overridden by the pte */
- if (stage1)
- reg |= (CBAR_S1_MEMATTR_WB << CBAR_S1_MEMATTR_SHIFT);
- else
+ /*
+ * Use the weakest shareability/memory types, so they are
+ * overridden by the ttbcr/pte.
+ */
+ if (stage1) {
+ reg |= (CBAR_S1_BPSHCFG_NSH << CBAR_S1_BPSHCFG_SHIFT) |
+ (CBAR_S1_MEMATTR_WB << CBAR_S1_MEMATTR_SHIFT);
+ } else {
reg |= ARM_SMMU_CB_VMID(root_cfg) << CBAR_VMID_SHIFT;
+ }
writel_relaxed(reg, gr1_base + ARM_SMMU_GR1_CBAR(root_cfg->cbndx));
if (smmu->version > 1) {
@@ -715,6 +780,8 @@ static void arm_smmu_init_context_bank(struct arm_smmu_domain *smmu_domain)
}
/* TTBR0 */
+ arm_smmu_flush_pgtable(smmu, root_cfg->pgd,
+ PTRS_PER_PGD * sizeof(pgd_t));
reg = __pa(root_cfg->pgd);
writel_relaxed(reg, cb_base + ARM_SMMU_CB_TTBR0_LO);
reg = (phys_addr_t)__pa(root_cfg->pgd) >> 32;
@@ -901,7 +968,7 @@ static int arm_smmu_domain_init(struct iommu_domain *domain)
goto out_free_domain;
smmu_domain->root_cfg.pgd = pgd;
- mutex_init(&smmu_domain->lock);
+ spin_lock_init(&smmu_domain->lock);
domain->priv = smmu_domain;
return 0;
@@ -959,9 +1026,8 @@ static void arm_smmu_free_pgtables(struct arm_smmu_domain *smmu_domain)
/*
* Recursively free the page tables for this domain. We don't
- * care about speculative TLB filling, because the TLB will be
- * nuked next time this context bank is re-allocated and no devices
- * currently map to these tables.
+ * care about speculative TLB filling because the tables should
+ * not be active in any context bank at this point (SCTLR.M is 0).
*/
pgd = pgd_base;
for (i = 0; i < PTRS_PER_PGD; ++i) {
@@ -1101,7 +1167,7 @@ static int arm_smmu_domain_add_master(struct arm_smmu_domain *smmu_domain,
for (i = 0; i < master->num_streamids; ++i) {
u32 idx, s2cr;
idx = master->smrs ? master->smrs[i].idx : master->streamids[i];
- s2cr = (S2CR_TYPE_TRANS << S2CR_TYPE_SHIFT) |
+ s2cr = S2CR_TYPE_TRANS |
(smmu_domain->root_cfg.cbndx << S2CR_CBNDX_SHIFT);
writel_relaxed(s2cr, gr0_base + ARM_SMMU_GR0_S2CR(idx));
}
@@ -1128,6 +1194,7 @@ static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev)
struct arm_smmu_domain *smmu_domain = domain->priv;
struct arm_smmu_device *device_smmu = dev->archdata.iommu;
struct arm_smmu_master *master;
+ unsigned long flags;
if (!device_smmu) {
dev_err(dev, "cannot attach to SMMU, is it on the same bus?\n");
@@ -1138,7 +1205,7 @@ static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev)
* Sanity check the domain. We don't currently support domains
* that cross between different SMMU chains.
*/
- mutex_lock(&smmu_domain->lock);
+ spin_lock_irqsave(&smmu_domain->lock, flags);
if (!smmu_domain->leaf_smmu) {
/* Now that we have a master, we can finalise the domain */
ret = arm_smmu_init_domain_context(domain, dev);
@@ -1153,7 +1220,7 @@ static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev)
dev_name(device_smmu->dev));
goto err_unlock;
}
- mutex_unlock(&smmu_domain->lock);
+ spin_unlock_irqrestore(&smmu_domain->lock, flags);
/* Looks ok, so add the device to the domain */
master = find_smmu_master(smmu_domain->leaf_smmu, dev->of_node);
@@ -1163,7 +1230,7 @@ static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev)
return arm_smmu_domain_add_master(smmu_domain, master);
err_unlock:
- mutex_unlock(&smmu_domain->lock);
+ spin_unlock_irqrestore(&smmu_domain->lock, flags);
return ret;
}
@@ -1177,23 +1244,6 @@ static void arm_smmu_detach_dev(struct iommu_domain *domain, struct device *dev)
arm_smmu_domain_remove_master(smmu_domain, master);
}
-static void arm_smmu_flush_pgtable(struct arm_smmu_device *smmu, void *addr,
- size_t size)
-{
- unsigned long offset = (unsigned long)addr & ~PAGE_MASK;
-
- /*
- * If the SMMU can't walk tables in the CPU caches, treat them
- * like non-coherent DMA since we need to flush the new entries
- * all the way out to memory. There's no possibility of recursion
- * here as the SMMU table walker will not be wired through another
- * SMMU.
- */
- if (!(smmu->features & ARM_SMMU_FEAT_COHERENT_WALK))
- dma_map_page(smmu->dev, virt_to_page(addr), offset, size,
- DMA_TO_DEVICE);
-}
-
static bool arm_smmu_pte_is_contiguous_range(unsigned long addr,
unsigned long end)
{
@@ -1203,19 +1253,18 @@ static bool arm_smmu_pte_is_contiguous_range(unsigned long addr,
static int arm_smmu_alloc_init_pte(struct arm_smmu_device *smmu, pmd_t *pmd,
unsigned long addr, unsigned long end,
- unsigned long pfn, int flags, int stage)
+ unsigned long pfn, int prot, int stage)
{
pte_t *pte, *start;
pteval_t pteval = ARM_SMMU_PTE_PAGE | ARM_SMMU_PTE_AF | ARM_SMMU_PTE_XN;
if (pmd_none(*pmd)) {
/* Allocate a new set of tables */
- pgtable_t table = alloc_page(PGALLOC_GFP);
+ pgtable_t table = alloc_page(GFP_ATOMIC|__GFP_ZERO);
if (!table)
return -ENOMEM;
- arm_smmu_flush_pgtable(smmu, page_address(table),
- ARM_SMMU_PTE_HWTABLE_SIZE);
+ arm_smmu_flush_pgtable(smmu, page_address(table), PAGE_SIZE);
if (!pgtable_page_ctor(table)) {
__free_page(table);
return -ENOMEM;
@@ -1226,28 +1275,28 @@ static int arm_smmu_alloc_init_pte(struct arm_smmu_device *smmu, pmd_t *pmd,
if (stage == 1) {
pteval |= ARM_SMMU_PTE_AP_UNPRIV | ARM_SMMU_PTE_nG;
- if (!(flags & IOMMU_WRITE) && (flags & IOMMU_READ))
+ if (!(prot & IOMMU_WRITE) && (prot & IOMMU_READ))
pteval |= ARM_SMMU_PTE_AP_RDONLY;
- if (flags & IOMMU_CACHE)
+ if (prot & IOMMU_CACHE)
pteval |= (MAIR_ATTR_IDX_CACHE <<
ARM_SMMU_PTE_ATTRINDX_SHIFT);
} else {
pteval |= ARM_SMMU_PTE_HAP_FAULT;
- if (flags & IOMMU_READ)
+ if (prot & IOMMU_READ)
pteval |= ARM_SMMU_PTE_HAP_READ;
- if (flags & IOMMU_WRITE)
+ if (prot & IOMMU_WRITE)
pteval |= ARM_SMMU_PTE_HAP_WRITE;
- if (flags & IOMMU_CACHE)
+ if (prot & IOMMU_CACHE)
pteval |= ARM_SMMU_PTE_MEMATTR_OIWB;
else
pteval |= ARM_SMMU_PTE_MEMATTR_NC;
}
/* If no access, create a faulting entry to avoid TLB fills */
- if (flags & IOMMU_EXEC)
+ if (prot & IOMMU_EXEC)
pteval &= ~ARM_SMMU_PTE_XN;
- else if (!(flags & (IOMMU_READ | IOMMU_WRITE)))
+ else if (!(prot & (IOMMU_READ | IOMMU_WRITE)))
pteval &= ~ARM_SMMU_PTE_PAGE;
pteval |= ARM_SMMU_PTE_SH_IS;
@@ -1309,7 +1358,7 @@ static int arm_smmu_alloc_init_pte(struct arm_smmu_device *smmu, pmd_t *pmd,
static int arm_smmu_alloc_init_pmd(struct arm_smmu_device *smmu, pud_t *pud,
unsigned long addr, unsigned long end,
- phys_addr_t phys, int flags, int stage)
+ phys_addr_t phys, int prot, int stage)
{
int ret;
pmd_t *pmd;
@@ -1317,19 +1366,23 @@ static int arm_smmu_alloc_init_pmd(struct arm_smmu_device *smmu, pud_t *pud,
#ifndef __PAGETABLE_PMD_FOLDED
if (pud_none(*pud)) {
- pmd = pmd_alloc_one(NULL, addr);
+ pmd = (pmd_t *)get_zeroed_page(GFP_ATOMIC);
if (!pmd)
return -ENOMEM;
+
+ arm_smmu_flush_pgtable(smmu, pmd, PAGE_SIZE);
+ pud_populate(NULL, pud, pmd);
+ arm_smmu_flush_pgtable(smmu, pud, sizeof(*pud));
+
+ pmd += pmd_index(addr);
} else
#endif
pmd = pmd_offset(pud, addr);
do {
next = pmd_addr_end(addr, end);
- ret = arm_smmu_alloc_init_pte(smmu, pmd, addr, end, pfn,
- flags, stage);
- pud_populate(NULL, pud, pmd);
- arm_smmu_flush_pgtable(smmu, pud, sizeof(*pud));
+ ret = arm_smmu_alloc_init_pte(smmu, pmd, addr, next, pfn,
+ prot, stage);
phys += next - addr;
} while (pmd++, addr = next, addr < end);
@@ -1338,7 +1391,7 @@ static int arm_smmu_alloc_init_pmd(struct arm_smmu_device *smmu, pud_t *pud,
static int arm_smmu_alloc_init_pud(struct arm_smmu_device *smmu, pgd_t *pgd,
unsigned long addr, unsigned long end,
- phys_addr_t phys, int flags, int stage)
+ phys_addr_t phys, int prot, int stage)
{
int ret = 0;
pud_t *pud;
@@ -1346,9 +1399,15 @@ static int arm_smmu_alloc_init_pud(struct arm_smmu_device *smmu, pgd_t *pgd,
#ifndef __PAGETABLE_PUD_FOLDED
if (pgd_none(*pgd)) {
- pud = pud_alloc_one(NULL, addr);
+ pud = (pud_t *)get_zeroed_page(GFP_ATOMIC);
if (!pud)
return -ENOMEM;
+
+ arm_smmu_flush_pgtable(smmu, pud, PAGE_SIZE);
+ pgd_populate(NULL, pgd, pud);
+ arm_smmu_flush_pgtable(smmu, pgd, sizeof(*pgd));
+
+ pud += pud_index(addr);
} else
#endif
pud = pud_offset(pgd, addr);
@@ -1356,9 +1415,7 @@ static int arm_smmu_alloc_init_pud(struct arm_smmu_device *smmu, pgd_t *pgd,
do {
next = pud_addr_end(addr, end);
ret = arm_smmu_alloc_init_pmd(smmu, pud, addr, next, phys,
- flags, stage);
- pgd_populate(NULL, pud, pgd);
- arm_smmu_flush_pgtable(smmu, pgd, sizeof(*pgd));
+ prot, stage);
phys += next - addr;
} while (pud++, addr = next, addr < end);
@@ -1367,7 +1424,7 @@ static int arm_smmu_alloc_init_pud(struct arm_smmu_device *smmu, pgd_t *pgd,
static int arm_smmu_handle_mapping(struct arm_smmu_domain *smmu_domain,
unsigned long iova, phys_addr_t paddr,
- size_t size, int flags)
+ size_t size, int prot)
{
int ret, stage;
unsigned long end;
@@ -1375,6 +1432,7 @@ static int arm_smmu_handle_mapping(struct arm_smmu_domain *smmu_domain,
struct arm_smmu_cfg *root_cfg = &smmu_domain->root_cfg;
pgd_t *pgd = root_cfg->pgd;
struct arm_smmu_device *smmu = root_cfg->smmu;
+ unsigned long flags;
if (root_cfg->cbar == CBAR_TYPE_S2_TRANS) {
stage = 2;
@@ -1397,14 +1455,14 @@ static int arm_smmu_handle_mapping(struct arm_smmu_domain *smmu_domain,
if (paddr & ~output_mask)
return -ERANGE;
- mutex_lock(&smmu_domain->lock);
+ spin_lock_irqsave(&smmu_domain->lock, flags);
pgd += pgd_index(iova);
end = iova + size;
do {
unsigned long next = pgd_addr_end(iova, end);
ret = arm_smmu_alloc_init_pud(smmu, pgd, iova, next, paddr,
- flags, stage);
+ prot, stage);
if (ret)
goto out_unlock;
@@ -1413,17 +1471,13 @@ static int arm_smmu_handle_mapping(struct arm_smmu_domain *smmu_domain,
} while (pgd++, iova != end);
out_unlock:
- mutex_unlock(&smmu_domain->lock);
-
- /* Ensure new page tables are visible to the hardware walker */
- if (smmu->features & ARM_SMMU_FEAT_COHERENT_WALK)
- dsb();
+ spin_unlock_irqrestore(&smmu_domain->lock, flags);
return ret;
}
static int arm_smmu_map(struct iommu_domain *domain, unsigned long iova,
- phys_addr_t paddr, size_t size, int flags)
+ phys_addr_t paddr, size_t size, int prot)
{
struct arm_smmu_domain *smmu_domain = domain->priv;
@@ -1434,7 +1488,7 @@ static int arm_smmu_map(struct iommu_domain *domain, unsigned long iova,
if ((phys_addr_t)iova & ~smmu_domain->output_mask)
return -ERANGE;
- return arm_smmu_handle_mapping(smmu_domain, iova, paddr, size, flags);
+ return arm_smmu_handle_mapping(smmu_domain, iova, paddr, size, prot);
}
static size_t arm_smmu_unmap(struct iommu_domain *domain, unsigned long iova,
@@ -1445,7 +1499,7 @@ static size_t arm_smmu_unmap(struct iommu_domain *domain, unsigned long iova,
ret = arm_smmu_handle_mapping(smmu_domain, iova, 0, size, 0);
arm_smmu_tlb_inv_context(&smmu_domain->root_cfg);
- return ret ? ret : size;
+ return ret ? 0 : size;
}
static phys_addr_t arm_smmu_iova_to_phys(struct iommu_domain *domain,
@@ -1578,9 +1632,9 @@ static void arm_smmu_device_reset(struct arm_smmu_device *smmu)
int i = 0;
u32 reg;
- /* Clear Global FSR */
- reg = readl_relaxed(gr0_base + ARM_SMMU_GR0_sGFSR);
- writel(reg, gr0_base + ARM_SMMU_GR0_sGFSR);
+ /* clear global FSR */
+ reg = readl_relaxed(ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sGFSR);
+ writel(reg, ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sGFSR);
/* Mark all SMRn as invalid and all S2CRn as bypass */
for (i = 0; i < smmu->num_mapping_groups; ++i) {
@@ -1600,7 +1654,7 @@ static void arm_smmu_device_reset(struct arm_smmu_device *smmu)
writel_relaxed(0, gr0_base + ARM_SMMU_GR0_TLBIALLH);
writel_relaxed(0, gr0_base + ARM_SMMU_GR0_TLBIALLNSNH);
- reg = readl_relaxed(gr0_base + ARM_SMMU_GR0_sCR0);
+ reg = readl_relaxed(ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sCR0);
/* Enable fault reporting */
reg |= (sCR0_GFRE | sCR0_GFIE | sCR0_GCFGFRE | sCR0_GCFGFIE);
@@ -1619,7 +1673,7 @@ static void arm_smmu_device_reset(struct arm_smmu_device *smmu)
/* Push the button */
arm_smmu_tlb_sync(smmu);
- writel_relaxed(reg, gr0_base + ARM_SMMU_GR0_sCR0);
+ writel(reg, ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sCR0);
}
static int arm_smmu_id_size_to_bits(int size)
@@ -1750,7 +1804,7 @@ static int arm_smmu_device_cfg_probe(struct arm_smmu_device *smmu)
* allocation (PTRS_PER_PGD).
*/
#ifdef CONFIG_64BIT
- smmu->s1_output_size = min(39UL, size);
+ smmu->s1_output_size = min((unsigned long)VA_BITS, size);
#else
smmu->s1_output_size = min(32UL, size);
#endif
@@ -1866,6 +1920,8 @@ static int arm_smmu_device_dt_probe(struct platform_device *pdev)
if (err)
goto out_put_parent;
+ parse_driver_options(smmu);
+
if (smmu->version > 1 &&
smmu->num_context_banks != smmu->num_context_irqs) {
dev_err(dev,
@@ -1950,7 +2006,7 @@ static int arm_smmu_device_remove(struct platform_device *pdev)
free_irq(smmu->irqs[i], smmu);
/* Turn the thing off */
- writel_relaxed(sCR0_CLIENTPD, ARM_SMMU_GR0(smmu) + ARM_SMMU_GR0_sCR0);
+ writel(sCR0_CLIENTPD,ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sCR0);
return 0;
}
@@ -1987,8 +2043,10 @@ static int __init arm_smmu_init(void)
if (!iommu_present(&platform_bus_type))
bus_set_iommu(&platform_bus_type, &arm_smmu_ops);
+#ifdef CONFIG_ARM_AMBA
if (!iommu_present(&amba_bustype))
bus_set_iommu(&amba_bustype, &arm_smmu_ops);
+#endif
return 0;
}
diff --git a/drivers/iommu/dmar.c b/drivers/iommu/dmar.c
index 15815654341..9a4f05e5b23 100644
--- a/drivers/iommu/dmar.c
+++ b/drivers/iommu/dmar.c
@@ -43,14 +43,24 @@
#include "irq_remapping.h"
-/* No locks are needed as DMA remapping hardware unit
- * list is constructed at boot time and hotplug of
- * these units are not supported by the architecture.
+/*
+ * Assumptions:
+ * 1) The hotplug framework guarentees that DMAR unit will be hot-added
+ * before IO devices managed by that unit.
+ * 2) The hotplug framework guarantees that DMAR unit will be hot-removed
+ * after IO devices managed by that unit.
+ * 3) Hotplug events are rare.
+ *
+ * Locking rules for DMA and interrupt remapping related global data structures:
+ * 1) Use dmar_global_lock in process context
+ * 2) Use RCU in interrupt context
*/
+DECLARE_RWSEM(dmar_global_lock);
LIST_HEAD(dmar_drhd_units);
struct acpi_table_header * __initdata dmar_tbl;
static acpi_size dmar_tbl_size;
+static int dmar_dev_scope_status = 1;
static int alloc_iommu(struct dmar_drhd_unit *drhd);
static void free_iommu(struct intel_iommu *iommu);
@@ -62,73 +72,20 @@ static void __init dmar_register_drhd_unit(struct dmar_drhd_unit *drhd)
* the very end.
*/
if (drhd->include_all)
- list_add_tail(&drhd->list, &dmar_drhd_units);
+ list_add_tail_rcu(&drhd->list, &dmar_drhd_units);
else
- list_add(&drhd->list, &dmar_drhd_units);
+ list_add_rcu(&drhd->list, &dmar_drhd_units);
}
-static int __init dmar_parse_one_dev_scope(struct acpi_dmar_device_scope *scope,
- struct pci_dev **dev, u16 segment)
-{
- struct pci_bus *bus;
- struct pci_dev *pdev = NULL;
- struct acpi_dmar_pci_path *path;
- int count;
-
- bus = pci_find_bus(segment, scope->bus);
- path = (struct acpi_dmar_pci_path *)(scope + 1);
- count = (scope->length - sizeof(struct acpi_dmar_device_scope))
- / sizeof(struct acpi_dmar_pci_path);
-
- while (count) {
- if (pdev)
- pci_dev_put(pdev);
- /*
- * Some BIOSes list non-exist devices in DMAR table, just
- * ignore it
- */
- if (!bus) {
- pr_warn("Device scope bus [%d] not found\n", scope->bus);
- break;
- }
- pdev = pci_get_slot(bus, PCI_DEVFN(path->device, path->function));
- if (!pdev) {
- /* warning will be printed below */
- break;
- }
- path ++;
- count --;
- bus = pdev->subordinate;
- }
- if (!pdev) {
- pr_warn("Device scope device [%04x:%02x:%02x.%02x] not found\n",
- segment, scope->bus, path->device, path->function);
- return 0;
- }
- if ((scope->entry_type == ACPI_DMAR_SCOPE_TYPE_ENDPOINT && \
- pdev->subordinate) || (scope->entry_type == \
- ACPI_DMAR_SCOPE_TYPE_BRIDGE && !pdev->subordinate)) {
- pci_dev_put(pdev);
- pr_warn("Device scope type does not match for %s\n",
- pci_name(pdev));
- return -EINVAL;
- }
- *dev = pdev;
- return 0;
-}
-
-int __init dmar_parse_dev_scope(void *start, void *end, int *cnt,
- struct pci_dev ***devices, u16 segment)
+void *dmar_alloc_dev_scope(void *start, void *end, int *cnt)
{
struct acpi_dmar_device_scope *scope;
- void * tmp = start;
- int index;
- int ret;
*cnt = 0;
while (start < end) {
scope = start;
- if (scope->entry_type == ACPI_DMAR_SCOPE_TYPE_ENDPOINT ||
+ if (scope->entry_type == ACPI_DMAR_SCOPE_TYPE_ACPI ||
+ scope->entry_type == ACPI_DMAR_SCOPE_TYPE_ENDPOINT ||
scope->entry_type == ACPI_DMAR_SCOPE_TYPE_BRIDGE)
(*cnt)++;
else if (scope->entry_type != ACPI_DMAR_SCOPE_TYPE_IOAPIC &&
@@ -138,43 +95,237 @@ int __init dmar_parse_dev_scope(void *start, void *end, int *cnt,
start += scope->length;
}
if (*cnt == 0)
- return 0;
+ return NULL;
- *devices = kcalloc(*cnt, sizeof(struct pci_dev *), GFP_KERNEL);
- if (!*devices)
- return -ENOMEM;
+ return kcalloc(*cnt, sizeof(struct dmar_dev_scope), GFP_KERNEL);
+}
- start = tmp;
- index = 0;
- while (start < end) {
+void dmar_free_dev_scope(struct dmar_dev_scope **devices, int *cnt)
+{
+ int i;
+ struct device *tmp_dev;
+
+ if (*devices && *cnt) {
+ for_each_active_dev_scope(*devices, *cnt, i, tmp_dev)
+ put_device(tmp_dev);
+ kfree(*devices);
+ }
+
+ *devices = NULL;
+ *cnt = 0;
+}
+
+/* Optimize out kzalloc()/kfree() for normal cases */
+static char dmar_pci_notify_info_buf[64];
+
+static struct dmar_pci_notify_info *
+dmar_alloc_pci_notify_info(struct pci_dev *dev, unsigned long event)
+{
+ int level = 0;
+ size_t size;
+ struct pci_dev *tmp;
+ struct dmar_pci_notify_info *info;
+
+ BUG_ON(dev->is_virtfn);
+
+ /* Only generate path[] for device addition event */
+ if (event == BUS_NOTIFY_ADD_DEVICE)
+ for (tmp = dev; tmp; tmp = tmp->bus->self)
+ level++;
+
+ size = sizeof(*info) + level * sizeof(struct acpi_dmar_pci_path);
+ if (size <= sizeof(dmar_pci_notify_info_buf)) {
+ info = (struct dmar_pci_notify_info *)dmar_pci_notify_info_buf;
+ } else {
+ info = kzalloc(size, GFP_KERNEL);
+ if (!info) {
+ pr_warn("Out of memory when allocating notify_info "
+ "for %s.\n", pci_name(dev));
+ if (dmar_dev_scope_status == 0)
+ dmar_dev_scope_status = -ENOMEM;
+ return NULL;
+ }
+ }
+
+ info->event = event;
+ info->dev = dev;
+ info->seg = pci_domain_nr(dev->bus);
+ info->level = level;
+ if (event == BUS_NOTIFY_ADD_DEVICE) {
+ for (tmp = dev; tmp; tmp = tmp->bus->self) {
+ level--;
+ info->path[level].device = PCI_SLOT(tmp->devfn);
+ info->path[level].function = PCI_FUNC(tmp->devfn);
+ if (pci_is_root_bus(tmp->bus))
+ info->bus = tmp->bus->number;
+ }
+ }
+
+ return info;
+}
+
+static inline void dmar_free_pci_notify_info(struct dmar_pci_notify_info *info)
+{
+ if ((void *)info != dmar_pci_notify_info_buf)
+ kfree(info);
+}
+
+static bool dmar_match_pci_path(struct dmar_pci_notify_info *info, int bus,
+ struct acpi_dmar_pci_path *path, int count)
+{
+ int i;
+
+ if (info->bus != bus)
+ return false;
+ if (info->level != count)
+ return false;
+
+ for (i = 0; i < count; i++) {
+ if (path[i].device != info->path[i].device ||
+ path[i].function != info->path[i].function)
+ return false;
+ }
+
+ return true;
+}
+
+/* Return: > 0 if match found, 0 if no match found, < 0 if error happens */
+int dmar_insert_dev_scope(struct dmar_pci_notify_info *info,
+ void *start, void*end, u16 segment,
+ struct dmar_dev_scope *devices,
+ int devices_cnt)
+{
+ int i, level;
+ struct device *tmp, *dev = &info->dev->dev;
+ struct acpi_dmar_device_scope *scope;
+ struct acpi_dmar_pci_path *path;
+
+ if (segment != info->seg)
+ return 0;
+
+ for (; start < end; start += scope->length) {
scope = start;
- if (scope->entry_type == ACPI_DMAR_SCOPE_TYPE_ENDPOINT ||
- scope->entry_type == ACPI_DMAR_SCOPE_TYPE_BRIDGE) {
- ret = dmar_parse_one_dev_scope(scope,
- &(*devices)[index], segment);
- if (ret) {
- dmar_free_dev_scope(devices, cnt);
- return ret;
- }
- index ++;
+ if (scope->entry_type != ACPI_DMAR_SCOPE_TYPE_ENDPOINT &&
+ scope->entry_type != ACPI_DMAR_SCOPE_TYPE_BRIDGE)
+ continue;
+
+ path = (struct acpi_dmar_pci_path *)(scope + 1);
+ level = (scope->length - sizeof(*scope)) / sizeof(*path);
+ if (!dmar_match_pci_path(info, scope->bus, path, level))
+ continue;
+
+ if ((scope->entry_type == ACPI_DMAR_SCOPE_TYPE_ENDPOINT) ^
+ (info->dev->hdr_type == PCI_HEADER_TYPE_NORMAL)) {
+ pr_warn("Device scope type does not match for %s\n",
+ pci_name(info->dev));
+ return -EINVAL;
}
- start += scope->length;
+
+ for_each_dev_scope(devices, devices_cnt, i, tmp)
+ if (tmp == NULL) {
+ devices[i].bus = info->dev->bus->number;
+ devices[i].devfn = info->dev->devfn;
+ rcu_assign_pointer(devices[i].dev,
+ get_device(dev));
+ return 1;
+ }
+ BUG_ON(i >= devices_cnt);
}
return 0;
}
-void dmar_free_dev_scope(struct pci_dev ***devices, int *cnt)
+int dmar_remove_dev_scope(struct dmar_pci_notify_info *info, u16 segment,
+ struct dmar_dev_scope *devices, int count)
{
- if (*devices && *cnt) {
- while (--*cnt >= 0)
- pci_dev_put((*devices)[*cnt]);
- kfree(*devices);
- *devices = NULL;
- *cnt = 0;
+ int index;
+ struct device *tmp;
+
+ if (info->seg != segment)
+ return 0;
+
+ for_each_active_dev_scope(devices, count, index, tmp)
+ if (tmp == &info->dev->dev) {
+ rcu_assign_pointer(devices[index].dev, NULL);
+ synchronize_rcu();
+ put_device(tmp);
+ return 1;
+ }
+
+ return 0;
+}
+
+static int dmar_pci_bus_add_dev(struct dmar_pci_notify_info *info)
+{
+ int ret = 0;
+ struct dmar_drhd_unit *dmaru;
+ struct acpi_dmar_hardware_unit *drhd;
+
+ for_each_drhd_unit(dmaru) {
+ if (dmaru->include_all)
+ continue;
+
+ drhd = container_of(dmaru->hdr,
+ struct acpi_dmar_hardware_unit, header);
+ ret = dmar_insert_dev_scope(info, (void *)(drhd + 1),
+ ((void *)drhd) + drhd->header.length,
+ dmaru->segment,
+ dmaru->devices, dmaru->devices_cnt);
+ if (ret != 0)
+ break;
}
+ if (ret >= 0)
+ ret = dmar_iommu_notify_scope_dev(info);
+ if (ret < 0 && dmar_dev_scope_status == 0)
+ dmar_dev_scope_status = ret;
+
+ return ret;
}
+static void dmar_pci_bus_del_dev(struct dmar_pci_notify_info *info)
+{
+ struct dmar_drhd_unit *dmaru;
+
+ for_each_drhd_unit(dmaru)
+ if (dmar_remove_dev_scope(info, dmaru->segment,
+ dmaru->devices, dmaru->devices_cnt))
+ break;
+ dmar_iommu_notify_scope_dev(info);
+}
+
+static int dmar_pci_bus_notifier(struct notifier_block *nb,
+ unsigned long action, void *data)
+{
+ struct pci_dev *pdev = to_pci_dev(data);
+ struct dmar_pci_notify_info *info;
+
+ /* Only care about add/remove events for physical functions */
+ if (pdev->is_virtfn)
+ return NOTIFY_DONE;
+ if (action != BUS_NOTIFY_ADD_DEVICE && action != BUS_NOTIFY_DEL_DEVICE)
+ return NOTIFY_DONE;
+
+ info = dmar_alloc_pci_notify_info(pdev, action);
+ if (!info)
+ return NOTIFY_DONE;
+
+ down_write(&dmar_global_lock);
+ if (action == BUS_NOTIFY_ADD_DEVICE)
+ dmar_pci_bus_add_dev(info);
+ else if (action == BUS_NOTIFY_DEL_DEVICE)
+ dmar_pci_bus_del_dev(info);
+ up_write(&dmar_global_lock);
+
+ dmar_free_pci_notify_info(info);
+
+ return NOTIFY_OK;
+}
+
+static struct notifier_block dmar_pci_bus_nb = {
+ .notifier_call = dmar_pci_bus_notifier,
+ .priority = INT_MIN,
+};
+
/**
* dmar_parse_one_drhd - parses exactly one DMA remapping hardware definition
* structure which uniquely represent one DMA remapping hardware unit
@@ -196,9 +347,18 @@ dmar_parse_one_drhd(struct acpi_dmar_header *header)
dmaru->reg_base_addr = drhd->address;
dmaru->segment = drhd->segment;
dmaru->include_all = drhd->flags & 0x1; /* BIT0: INCLUDE_ALL */
+ dmaru->devices = dmar_alloc_dev_scope((void *)(drhd + 1),
+ ((void *)drhd) + drhd->header.length,
+ &dmaru->devices_cnt);
+ if (dmaru->devices_cnt && dmaru->devices == NULL) {
+ kfree(dmaru);
+ return -ENOMEM;
+ }
ret = alloc_iommu(dmaru);
if (ret) {
+ dmar_free_dev_scope(&dmaru->devices,
+ &dmaru->devices_cnt);
kfree(dmaru);
return ret;
}
@@ -215,19 +375,24 @@ static void dmar_free_drhd(struct dmar_drhd_unit *dmaru)
kfree(dmaru);
}
-static int __init dmar_parse_dev(struct dmar_drhd_unit *dmaru)
+static int __init dmar_parse_one_andd(struct acpi_dmar_header *header)
{
- struct acpi_dmar_hardware_unit *drhd;
-
- drhd = (struct acpi_dmar_hardware_unit *) dmaru->hdr;
-
- if (dmaru->include_all)
- return 0;
+ struct acpi_dmar_andd *andd = (void *)header;
+
+ /* Check for NUL termination within the designated length */
+ if (strnlen(andd->object_name, header->length - 8) == header->length - 8) {
+ WARN_TAINT(1, TAINT_FIRMWARE_WORKAROUND,
+ "Your BIOS is broken; ANDD object name is not NUL-terminated\n"
+ "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
+ dmi_get_system_info(DMI_BIOS_VENDOR),
+ dmi_get_system_info(DMI_BIOS_VERSION),
+ dmi_get_system_info(DMI_PRODUCT_VERSION));
+ return -EINVAL;
+ }
+ pr_info("ANDD device: %x name: %s\n", andd->device_number,
+ andd->object_name);
- return dmar_parse_dev_scope((void *)(drhd + 1),
- ((void *)drhd) + drhd->header.length,
- &dmaru->devices_cnt, &dmaru->devices,
- drhd->segment);
+ return 0;
}
#ifdef CONFIG_ACPI_NUMA
@@ -293,6 +458,10 @@ dmar_table_print_dmar_entry(struct acpi_dmar_header *header)
(unsigned long long)rhsa->base_address,
rhsa->proximity_domain);
break;
+ case ACPI_DMAR_TYPE_ANDD:
+ /* We don't print this here because we need to sanity-check
+ it first. So print it in dmar_parse_one_andd() instead. */
+ break;
}
}
@@ -378,6 +547,9 @@ parse_dmar_table(void)
ret = dmar_parse_one_rhsa(entry_header);
#endif
break;
+ case ACPI_DMAR_TYPE_ANDD:
+ ret = dmar_parse_one_andd(entry_header);
+ break;
default:
pr_warn("Unknown DMAR structure type %d\n",
entry_header->type);
@@ -394,14 +566,15 @@ parse_dmar_table(void)
return ret;
}
-static int dmar_pci_device_match(struct pci_dev *devices[], int cnt,
- struct pci_dev *dev)
+static int dmar_pci_device_match(struct dmar_dev_scope devices[],
+ int cnt, struct pci_dev *dev)
{
int index;
+ struct device *tmp;
while (dev) {
- for (index = 0; index < cnt; index++)
- if (dev == devices[index])
+ for_each_active_dev_scope(devices, cnt, index, tmp)
+ if (dev_is_pci(tmp) && dev == to_pci_dev(tmp))
return 1;
/* Check our parent */
@@ -414,11 +587,12 @@ static int dmar_pci_device_match(struct pci_dev *devices[], int cnt,
struct dmar_drhd_unit *
dmar_find_matched_drhd_unit(struct pci_dev *dev)
{
- struct dmar_drhd_unit *dmaru = NULL;
+ struct dmar_drhd_unit *dmaru;
struct acpi_dmar_hardware_unit *drhd;
dev = pci_physfn(dev);
+ rcu_read_lock();
for_each_drhd_unit(dmaru) {
drhd = container_of(dmaru->hdr,
struct acpi_dmar_hardware_unit,
@@ -426,44 +600,128 @@ dmar_find_matched_drhd_unit(struct pci_dev *dev)
if (dmaru->include_all &&
drhd->segment == pci_domain_nr(dev->bus))
- return dmaru;
+ goto out;
if (dmar_pci_device_match(dmaru->devices,
dmaru->devices_cnt, dev))
- return dmaru;
+ goto out;
}
+ dmaru = NULL;
+out:
+ rcu_read_unlock();
- return NULL;
+ return dmaru;
}
-int __init dmar_dev_scope_init(void)
+static void __init dmar_acpi_insert_dev_scope(u8 device_number,
+ struct acpi_device *adev)
{
- static int dmar_dev_scope_initialized;
- struct dmar_drhd_unit *drhd;
- int ret = -ENODEV;
-
- if (dmar_dev_scope_initialized)
- return dmar_dev_scope_initialized;
+ struct dmar_drhd_unit *dmaru;
+ struct acpi_dmar_hardware_unit *drhd;
+ struct acpi_dmar_device_scope *scope;
+ struct device *tmp;
+ int i;
+ struct acpi_dmar_pci_path *path;
- if (list_empty(&dmar_drhd_units))
- goto fail;
+ for_each_drhd_unit(dmaru) {
+ drhd = container_of(dmaru->hdr,
+ struct acpi_dmar_hardware_unit,
+ header);
- list_for_each_entry(drhd, &dmar_drhd_units, list) {
- ret = dmar_parse_dev(drhd);
- if (ret)
- goto fail;
+ for (scope = (void *)(drhd + 1);
+ (unsigned long)scope < ((unsigned long)drhd) + drhd->header.length;
+ scope = ((void *)scope) + scope->length) {
+ if (scope->entry_type != ACPI_DMAR_SCOPE_TYPE_ACPI)
+ continue;
+ if (scope->enumeration_id != device_number)
+ continue;
+
+ path = (void *)(scope + 1);
+ pr_info("ACPI device \"%s\" under DMAR at %llx as %02x:%02x.%d\n",
+ dev_name(&adev->dev), dmaru->reg_base_addr,
+ scope->bus, path->device, path->function);
+ for_each_dev_scope(dmaru->devices, dmaru->devices_cnt, i, tmp)
+ if (tmp == NULL) {
+ dmaru->devices[i].bus = scope->bus;
+ dmaru->devices[i].devfn = PCI_DEVFN(path->device,
+ path->function);
+ rcu_assign_pointer(dmaru->devices[i].dev,
+ get_device(&adev->dev));
+ return;
+ }
+ BUG_ON(i >= dmaru->devices_cnt);
+ }
}
+ pr_warn("No IOMMU scope found for ANDD enumeration ID %d (%s)\n",
+ device_number, dev_name(&adev->dev));
+}
- ret = dmar_parse_rmrr_atsr_dev();
- if (ret)
- goto fail;
+static int __init dmar_acpi_dev_scope_init(void)
+{
+ struct acpi_dmar_andd *andd;
+
+ if (dmar_tbl == NULL)
+ return -ENODEV;
- dmar_dev_scope_initialized = 1;
+ for (andd = (void *)dmar_tbl + sizeof(struct acpi_table_dmar);
+ ((unsigned long)andd) < ((unsigned long)dmar_tbl) + dmar_tbl->length;
+ andd = ((void *)andd) + andd->header.length) {
+ if (andd->header.type == ACPI_DMAR_TYPE_ANDD) {
+ acpi_handle h;
+ struct acpi_device *adev;
+
+ if (!ACPI_SUCCESS(acpi_get_handle(ACPI_ROOT_OBJECT,
+ andd->object_name,
+ &h))) {
+ pr_err("Failed to find handle for ACPI object %s\n",
+ andd->object_name);
+ continue;
+ }
+ acpi_bus_get_device(h, &adev);
+ if (!adev) {
+ pr_err("Failed to get device for ACPI object %s\n",
+ andd->object_name);
+ continue;
+ }
+ dmar_acpi_insert_dev_scope(andd->device_number, adev);
+ }
+ }
return 0;
+}
-fail:
- dmar_dev_scope_initialized = ret;
- return ret;
+int __init dmar_dev_scope_init(void)
+{
+ struct pci_dev *dev = NULL;
+ struct dmar_pci_notify_info *info;
+
+ if (dmar_dev_scope_status != 1)
+ return dmar_dev_scope_status;
+
+ if (list_empty(&dmar_drhd_units)) {
+ dmar_dev_scope_status = -ENODEV;
+ } else {
+ dmar_dev_scope_status = 0;
+
+ dmar_acpi_dev_scope_init();
+
+ for_each_pci_dev(dev) {
+ if (dev->is_virtfn)
+ continue;
+
+ info = dmar_alloc_pci_notify_info(dev,
+ BUS_NOTIFY_ADD_DEVICE);
+ if (!info) {
+ return dmar_dev_scope_status;
+ } else {
+ dmar_pci_bus_add_dev(info);
+ dmar_free_pci_notify_info(info);
+ }
+ }
+
+ bus_register_notifier(&pci_bus_type, &dmar_pci_bus_nb);
+ }
+
+ return dmar_dev_scope_status;
}
@@ -557,6 +815,7 @@ int __init detect_intel_iommu(void)
{
int ret;
+ down_write(&dmar_global_lock);
ret = dmar_table_detect();
if (ret)
ret = check_zero_address();
@@ -574,6 +833,7 @@ int __init detect_intel_iommu(void)
}
early_acpi_os_unmap_memory((void __iomem *)dmar_tbl, dmar_tbl_size);
dmar_tbl = NULL;
+ up_write(&dmar_global_lock);
return ret ? 1 : -ENODEV;
}
@@ -696,6 +956,7 @@ static int alloc_iommu(struct dmar_drhd_unit *drhd)
}
iommu->agaw = agaw;
iommu->msagaw = msagaw;
+ iommu->segment = drhd->segment;
iommu->node = -1;
@@ -733,7 +994,7 @@ static void free_iommu(struct intel_iommu *iommu)
if (iommu->irq) {
free_irq(iommu->irq, iommu);
irq_set_handler_data(iommu->irq, NULL);
- destroy_irq(iommu->irq);
+ dmar_free_hwirq(iommu->irq);
}
if (iommu->qi) {
@@ -1289,8 +1550,8 @@ int dmar_set_interrupt(struct intel_iommu *iommu)
if (iommu->irq)
return 0;
- irq = create_irq();
- if (!irq) {
+ irq = dmar_alloc_hwirq();
+ if (irq <= 0) {
pr_err("IOMMU: no free vectors\n");
return -EINVAL;
}
@@ -1302,7 +1563,7 @@ int dmar_set_interrupt(struct intel_iommu *iommu)
if (ret) {
irq_set_handler_data(irq, NULL);
iommu->irq = 0;
- destroy_irq(irq);
+ dmar_free_hwirq(irq);
return ret;
}
@@ -1386,10 +1647,15 @@ static int __init dmar_free_unused_resources(void)
if (irq_remapping_enabled || intel_iommu_enabled)
return 0;
+ if (dmar_dev_scope_status != 1 && !list_empty(&dmar_drhd_units))
+ bus_unregister_notifier(&pci_bus_type, &dmar_pci_bus_nb);
+
+ down_write(&dmar_global_lock);
list_for_each_entry_safe(dmaru, dmaru_n, &dmar_drhd_units, list) {
list_del(&dmaru->list);
dmar_free_drhd(dmaru);
}
+ up_write(&dmar_global_lock);
return 0;
}
diff --git a/drivers/iommu/exynos-iommu.c b/drivers/iommu/exynos-iommu.c
index 074018979cd..99054d2c040 100644
--- a/drivers/iommu/exynos-iommu.c
+++ b/drivers/iommu/exynos-iommu.c
@@ -29,7 +29,8 @@
#include <asm/cacheflush.h>
#include <asm/pgtable.h>
-#include <mach/sysmmu.h>
+typedef u32 sysmmu_iova_t;
+typedef u32 sysmmu_pte_t;
/* We does not consider super section mapping (16MB) */
#define SECT_ORDER 20
@@ -44,28 +45,44 @@
#define LPAGE_MASK (~(LPAGE_SIZE - 1))
#define SPAGE_MASK (~(SPAGE_SIZE - 1))
-#define lv1ent_fault(sent) (((*(sent) & 3) == 0) || ((*(sent) & 3) == 3))
-#define lv1ent_page(sent) ((*(sent) & 3) == 1)
+#define lv1ent_fault(sent) ((*(sent) == ZERO_LV2LINK) || \
+ ((*(sent) & 3) == 0) || ((*(sent) & 3) == 3))
+#define lv1ent_zero(sent) (*(sent) == ZERO_LV2LINK)
+#define lv1ent_page_zero(sent) ((*(sent) & 3) == 1)
+#define lv1ent_page(sent) ((*(sent) != ZERO_LV2LINK) && \
+ ((*(sent) & 3) == 1))
#define lv1ent_section(sent) ((*(sent) & 3) == 2)
#define lv2ent_fault(pent) ((*(pent) & 3) == 0)
#define lv2ent_small(pent) ((*(pent) & 2) == 2)
#define lv2ent_large(pent) ((*(pent) & 3) == 1)
+static u32 sysmmu_page_offset(sysmmu_iova_t iova, u32 size)
+{
+ return iova & (size - 1);
+}
+
#define section_phys(sent) (*(sent) & SECT_MASK)
-#define section_offs(iova) ((iova) & 0xFFFFF)
+#define section_offs(iova) sysmmu_page_offset((iova), SECT_SIZE)
#define lpage_phys(pent) (*(pent) & LPAGE_MASK)
-#define lpage_offs(iova) ((iova) & 0xFFFF)
+#define lpage_offs(iova) sysmmu_page_offset((iova), LPAGE_SIZE)
#define spage_phys(pent) (*(pent) & SPAGE_MASK)
-#define spage_offs(iova) ((iova) & 0xFFF)
-
-#define lv1ent_offset(iova) ((iova) >> SECT_ORDER)
-#define lv2ent_offset(iova) (((iova) & 0xFF000) >> SPAGE_ORDER)
+#define spage_offs(iova) sysmmu_page_offset((iova), SPAGE_SIZE)
#define NUM_LV1ENTRIES 4096
-#define NUM_LV2ENTRIES 256
+#define NUM_LV2ENTRIES (SECT_SIZE / SPAGE_SIZE)
+
+static u32 lv1ent_offset(sysmmu_iova_t iova)
+{
+ return iova >> SECT_ORDER;
+}
+
+static u32 lv2ent_offset(sysmmu_iova_t iova)
+{
+ return (iova >> SPAGE_ORDER) & (NUM_LV2ENTRIES - 1);
+}
-#define LV2TABLE_SIZE (NUM_LV2ENTRIES * sizeof(long))
+#define LV2TABLE_SIZE (NUM_LV2ENTRIES * sizeof(sysmmu_pte_t))
#define SPAGES_PER_LPAGE (LPAGE_SIZE / SPAGE_SIZE)
@@ -80,6 +97,13 @@
#define CTRL_BLOCK 0x7
#define CTRL_DISABLE 0x0
+#define CFG_LRU 0x1
+#define CFG_QOS(n) ((n & 0xF) << 7)
+#define CFG_MASK 0x0150FFFF /* Selecting bit 0-15, 20, 22 and 24 */
+#define CFG_ACGEN (1 << 24) /* System MMU 3.3 only */
+#define CFG_SYSSEL (1 << 22) /* System MMU 3.2 only */
+#define CFG_FLPDCACHE (1 << 20) /* System MMU 3.2+ only */
+
#define REG_MMU_CTRL 0x000
#define REG_MMU_CFG 0x004
#define REG_MMU_STATUS 0x008
@@ -96,19 +120,32 @@
#define REG_MMU_VERSION 0x034
+#define MMU_MAJ_VER(val) ((val) >> 7)
+#define MMU_MIN_VER(val) ((val) & 0x7F)
+#define MMU_RAW_VER(reg) (((reg) >> 21) & ((1 << 11) - 1)) /* 11 bits */
+
+#define MAKE_MMU_VER(maj, min) ((((maj) & 0xF) << 7) | ((min) & 0x7F))
+
#define REG_PB0_SADDR 0x04C
#define REG_PB0_EADDR 0x050
#define REG_PB1_SADDR 0x054
#define REG_PB1_EADDR 0x058
-static unsigned long *section_entry(unsigned long *pgtable, unsigned long iova)
+#define has_sysmmu(dev) (dev->archdata.iommu != NULL)
+
+static struct kmem_cache *lv2table_kmem_cache;
+static sysmmu_pte_t *zero_lv2_table;
+#define ZERO_LV2LINK mk_lv1ent_page(virt_to_phys(zero_lv2_table))
+
+static sysmmu_pte_t *section_entry(sysmmu_pte_t *pgtable, sysmmu_iova_t iova)
{
return pgtable + lv1ent_offset(iova);
}
-static unsigned long *page_entry(unsigned long *sent, unsigned long iova)
+static sysmmu_pte_t *page_entry(sysmmu_pte_t *sent, sysmmu_iova_t iova)
{
- return (unsigned long *)__va(lv2table_base(sent)) + lv2ent_offset(iova);
+ return (sysmmu_pte_t *)phys_to_virt(
+ lv2table_base(sent)) + lv2ent_offset(iova);
}
enum exynos_sysmmu_inttype {
@@ -124,16 +161,6 @@ enum exynos_sysmmu_inttype {
SYSMMU_FAULTS_NUM
};
-/*
- * @itype: type of fault.
- * @pgtable_base: the physical address of page table base. This is 0 if @itype
- * is SYSMMU_BUSERROR.
- * @fault_addr: the device (virtual) address that the System MMU tried to
- * translated. This is 0 if @itype is SYSMMU_BUSERROR.
- */
-typedef int (*sysmmu_fault_handler_t)(enum exynos_sysmmu_inttype itype,
- unsigned long pgtable_base, unsigned long fault_addr);
-
static unsigned short fault_reg_offset[SYSMMU_FAULTS_NUM] = {
REG_PAGE_FAULT_ADDR,
REG_AR_FAULT_ADDR,
@@ -157,27 +184,34 @@ static char *sysmmu_fault_name[SYSMMU_FAULTS_NUM] = {
"UNKNOWN FAULT"
};
+/* attached to dev.archdata.iommu of the master device */
+struct exynos_iommu_owner {
+ struct list_head client; /* entry of exynos_iommu_domain.clients */
+ struct device *dev;
+ struct device *sysmmu;
+ struct iommu_domain *domain;
+ void *vmm_data; /* IO virtual memory manager's data */
+ spinlock_t lock; /* Lock to preserve consistency of System MMU */
+};
+
struct exynos_iommu_domain {
struct list_head clients; /* list of sysmmu_drvdata.node */
- unsigned long *pgtable; /* lv1 page table, 16KB */
+ sysmmu_pte_t *pgtable; /* lv1 page table, 16KB */
short *lv2entcnt; /* free lv2 entry counter for each section */
spinlock_t lock; /* lock for this structure */
spinlock_t pgtablelock; /* lock for modifying page table @ pgtable */
};
struct sysmmu_drvdata {
- struct list_head node; /* entry of exynos_iommu_domain.clients */
struct device *sysmmu; /* System MMU's device descriptor */
- struct device *dev; /* Owner of system MMU */
- char *dbgname;
- int nsfrs;
- void __iomem **sfrbases;
- struct clk *clk[2];
+ struct device *master; /* Owner of system MMU */
+ void __iomem *sfrbase;
+ struct clk *clk;
+ struct clk *clk_master;
int activations;
- rwlock_t lock;
+ spinlock_t lock;
struct iommu_domain *domain;
- sysmmu_fault_handler_t fault_handler;
- unsigned long pgtable;
+ phys_addr_t pgtable;
};
static bool set_sysmmu_active(struct sysmmu_drvdata *data)
@@ -204,6 +238,11 @@ static void sysmmu_unblock(void __iomem *sfrbase)
__raw_writel(CTRL_ENABLE, sfrbase + REG_MMU_CTRL);
}
+static unsigned int __raw_sysmmu_version(struct sysmmu_drvdata *data)
+{
+ return MMU_RAW_VER(__raw_readl(data->sfrbase + REG_MMU_VERSION));
+}
+
static bool sysmmu_block(void __iomem *sfrbase)
{
int i = 120;
@@ -226,434 +265,428 @@ static void __sysmmu_tlb_invalidate(void __iomem *sfrbase)
}
static void __sysmmu_tlb_invalidate_entry(void __iomem *sfrbase,
- unsigned long iova)
+ sysmmu_iova_t iova, unsigned int num_inv)
{
- __raw_writel((iova & SPAGE_MASK) | 1, sfrbase + REG_MMU_FLUSH_ENTRY);
+ unsigned int i;
+
+ for (i = 0; i < num_inv; i++) {
+ __raw_writel((iova & SPAGE_MASK) | 1,
+ sfrbase + REG_MMU_FLUSH_ENTRY);
+ iova += SPAGE_SIZE;
+ }
}
static void __sysmmu_set_ptbase(void __iomem *sfrbase,
- unsigned long pgd)
+ phys_addr_t pgd)
{
- __raw_writel(0x1, sfrbase + REG_MMU_CFG); /* 16KB LV1, LRU */
__raw_writel(pgd, sfrbase + REG_PT_BASE_ADDR);
__sysmmu_tlb_invalidate(sfrbase);
}
-static void __sysmmu_set_prefbuf(void __iomem *sfrbase, unsigned long base,
- unsigned long size, int idx)
-{
- __raw_writel(base, sfrbase + REG_PB0_SADDR + idx * 8);
- __raw_writel(size - 1 + base, sfrbase + REG_PB0_EADDR + idx * 8);
-}
-
-static void __set_fault_handler(struct sysmmu_drvdata *data,
- sysmmu_fault_handler_t handler)
-{
- unsigned long flags;
-
- write_lock_irqsave(&data->lock, flags);
- data->fault_handler = handler;
- write_unlock_irqrestore(&data->lock, flags);
-}
-
-void exynos_sysmmu_set_fault_handler(struct device *dev,
- sysmmu_fault_handler_t handler)
-{
- struct sysmmu_drvdata *data = dev_get_drvdata(dev->archdata.iommu);
-
- __set_fault_handler(data, handler);
-}
-
-static int default_fault_handler(enum exynos_sysmmu_inttype itype,
- unsigned long pgtable_base, unsigned long fault_addr)
+static void show_fault_information(const char *name,
+ enum exynos_sysmmu_inttype itype,
+ phys_addr_t pgtable_base, sysmmu_iova_t fault_addr)
{
- unsigned long *ent;
+ sysmmu_pte_t *ent;
if ((itype >= SYSMMU_FAULTS_NUM) || (itype < SYSMMU_PAGEFAULT))
itype = SYSMMU_FAULT_UNKNOWN;
- pr_err("%s occurred at 0x%lx(Page table base: 0x%lx)\n",
- sysmmu_fault_name[itype], fault_addr, pgtable_base);
+ pr_err("%s occurred at %#x by %s(Page table base: %pa)\n",
+ sysmmu_fault_name[itype], fault_addr, name, &pgtable_base);
- ent = section_entry(__va(pgtable_base), fault_addr);
- pr_err("\tLv1 entry: 0x%lx\n", *ent);
+ ent = section_entry(phys_to_virt(pgtable_base), fault_addr);
+ pr_err("\tLv1 entry: %#x\n", *ent);
if (lv1ent_page(ent)) {
ent = page_entry(ent, fault_addr);
- pr_err("\t Lv2 entry: 0x%lx\n", *ent);
+ pr_err("\t Lv2 entry: %#x\n", *ent);
}
-
- pr_err("Generating Kernel OOPS... because it is unrecoverable.\n");
-
- BUG();
-
- return 0;
}
static irqreturn_t exynos_sysmmu_irq(int irq, void *dev_id)
{
/* SYSMMU is in blocked when interrupt occurred. */
struct sysmmu_drvdata *data = dev_id;
- struct resource *irqres;
- struct platform_device *pdev;
enum exynos_sysmmu_inttype itype;
- unsigned long addr = -1;
-
- int i, ret = -ENOSYS;
-
- read_lock(&data->lock);
+ sysmmu_iova_t addr = -1;
+ int ret = -ENOSYS;
WARN_ON(!is_sysmmu_active(data));
- pdev = to_platform_device(data->sysmmu);
- for (i = 0; i < (pdev->num_resources / 2); i++) {
- irqres = platform_get_resource(pdev, IORESOURCE_IRQ, i);
- if (irqres && ((int)irqres->start == irq))
- break;
- }
+ spin_lock(&data->lock);
- if (i == pdev->num_resources) {
+ if (!IS_ERR(data->clk_master))
+ clk_enable(data->clk_master);
+
+ itype = (enum exynos_sysmmu_inttype)
+ __ffs(__raw_readl(data->sfrbase + REG_INT_STATUS));
+ if (WARN_ON(!((itype >= 0) && (itype < SYSMMU_FAULT_UNKNOWN))))
itype = SYSMMU_FAULT_UNKNOWN;
+ else
+ addr = __raw_readl(data->sfrbase + fault_reg_offset[itype]);
+
+ if (itype == SYSMMU_FAULT_UNKNOWN) {
+ pr_err("%s: Fault is not occurred by System MMU '%s'!\n",
+ __func__, dev_name(data->sysmmu));
+ pr_err("%s: Please check if IRQ is correctly configured.\n",
+ __func__);
+ BUG();
} else {
- itype = (enum exynos_sysmmu_inttype)
- __ffs(__raw_readl(data->sfrbases[i] + REG_INT_STATUS));
- if (WARN_ON(!((itype >= 0) && (itype < SYSMMU_FAULT_UNKNOWN))))
- itype = SYSMMU_FAULT_UNKNOWN;
- else
- addr = __raw_readl(
- data->sfrbases[i] + fault_reg_offset[itype]);
+ unsigned int base =
+ __raw_readl(data->sfrbase + REG_PT_BASE_ADDR);
+ show_fault_information(dev_name(data->sysmmu),
+ itype, base, addr);
+ if (data->domain)
+ ret = report_iommu_fault(data->domain,
+ data->master, addr, itype);
}
- if (data->domain)
- ret = report_iommu_fault(data->domain, data->dev,
- addr, itype);
+ /* fault is not recovered by fault handler */
+ BUG_ON(ret != 0);
- if ((ret == -ENOSYS) && data->fault_handler) {
- unsigned long base = data->pgtable;
- if (itype != SYSMMU_FAULT_UNKNOWN)
- base = __raw_readl(
- data->sfrbases[i] + REG_PT_BASE_ADDR);
- ret = data->fault_handler(itype, base, addr);
- }
+ __raw_writel(1 << itype, data->sfrbase + REG_INT_CLEAR);
- if (!ret && (itype != SYSMMU_FAULT_UNKNOWN))
- __raw_writel(1 << itype, data->sfrbases[i] + REG_INT_CLEAR);
- else
- dev_dbg(data->sysmmu, "(%s) %s is not handled.\n",
- data->dbgname, sysmmu_fault_name[itype]);
+ sysmmu_unblock(data->sfrbase);
- if (itype != SYSMMU_FAULT_UNKNOWN)
- sysmmu_unblock(data->sfrbases[i]);
+ if (!IS_ERR(data->clk_master))
+ clk_disable(data->clk_master);
- read_unlock(&data->lock);
+ spin_unlock(&data->lock);
return IRQ_HANDLED;
}
-static bool __exynos_sysmmu_disable(struct sysmmu_drvdata *data)
+static void __sysmmu_disable_nocount(struct sysmmu_drvdata *data)
{
+ if (!IS_ERR(data->clk_master))
+ clk_enable(data->clk_master);
+
+ __raw_writel(CTRL_DISABLE, data->sfrbase + REG_MMU_CTRL);
+ __raw_writel(0, data->sfrbase + REG_MMU_CFG);
+
+ clk_disable(data->clk);
+ if (!IS_ERR(data->clk_master))
+ clk_disable(data->clk_master);
+}
+
+static bool __sysmmu_disable(struct sysmmu_drvdata *data)
+{
+ bool disabled;
unsigned long flags;
- bool disabled = false;
- int i;
- write_lock_irqsave(&data->lock, flags);
+ spin_lock_irqsave(&data->lock, flags);
- if (!set_sysmmu_inactive(data))
- goto finish;
+ disabled = set_sysmmu_inactive(data);
- for (i = 0; i < data->nsfrs; i++)
- __raw_writel(CTRL_DISABLE, data->sfrbases[i] + REG_MMU_CTRL);
+ if (disabled) {
+ data->pgtable = 0;
+ data->domain = NULL;
- if (data->clk[1])
- clk_disable(data->clk[1]);
- if (data->clk[0])
- clk_disable(data->clk[0]);
+ __sysmmu_disable_nocount(data);
- disabled = true;
- data->pgtable = 0;
- data->domain = NULL;
-finish:
- write_unlock_irqrestore(&data->lock, flags);
+ dev_dbg(data->sysmmu, "Disabled\n");
+ } else {
+ dev_dbg(data->sysmmu, "%d times left to disable\n",
+ data->activations);
+ }
- if (disabled)
- dev_dbg(data->sysmmu, "(%s) Disabled\n", data->dbgname);
- else
- dev_dbg(data->sysmmu, "(%s) %d times left to be disabled\n",
- data->dbgname, data->activations);
+ spin_unlock_irqrestore(&data->lock, flags);
return disabled;
}
-/* __exynos_sysmmu_enable: Enables System MMU
- *
- * returns -error if an error occurred and System MMU is not enabled,
- * 0 if the System MMU has been just enabled and 1 if System MMU was already
- * enabled before.
- */
-static int __exynos_sysmmu_enable(struct sysmmu_drvdata *data,
- unsigned long pgtable, struct iommu_domain *domain)
+static void __sysmmu_init_config(struct sysmmu_drvdata *data)
{
- int i, ret = 0;
- unsigned long flags;
+ unsigned int cfg = CFG_LRU | CFG_QOS(15);
+ unsigned int ver;
+
+ ver = __raw_sysmmu_version(data);
+ if (MMU_MAJ_VER(ver) == 3) {
+ if (MMU_MIN_VER(ver) >= 2) {
+ cfg |= CFG_FLPDCACHE;
+ if (MMU_MIN_VER(ver) == 3) {
+ cfg |= CFG_ACGEN;
+ cfg &= ~CFG_LRU;
+ } else {
+ cfg |= CFG_SYSSEL;
+ }
+ }
+ }
- write_lock_irqsave(&data->lock, flags);
+ __raw_writel(cfg, data->sfrbase + REG_MMU_CFG);
+}
- if (!set_sysmmu_active(data)) {
- if (WARN_ON(pgtable != data->pgtable)) {
- ret = -EBUSY;
- set_sysmmu_inactive(data);
- } else {
- ret = 1;
- }
+static void __sysmmu_enable_nocount(struct sysmmu_drvdata *data)
+{
+ if (!IS_ERR(data->clk_master))
+ clk_enable(data->clk_master);
+ clk_enable(data->clk);
- dev_dbg(data->sysmmu, "(%s) Already enabled\n", data->dbgname);
- goto finish;
- }
+ __raw_writel(CTRL_BLOCK, data->sfrbase + REG_MMU_CTRL);
- if (data->clk[0])
- clk_enable(data->clk[0]);
- if (data->clk[1])
- clk_enable(data->clk[1]);
+ __sysmmu_init_config(data);
- data->pgtable = pgtable;
+ __sysmmu_set_ptbase(data->sfrbase, data->pgtable);
- for (i = 0; i < data->nsfrs; i++) {
- __sysmmu_set_ptbase(data->sfrbases[i], pgtable);
+ __raw_writel(CTRL_ENABLE, data->sfrbase + REG_MMU_CTRL);
- if ((readl(data->sfrbases[i] + REG_MMU_VERSION) >> 28) == 3) {
- /* System MMU version is 3.x */
- __raw_writel((1 << 12) | (2 << 28),
- data->sfrbases[i] + REG_MMU_CFG);
- __sysmmu_set_prefbuf(data->sfrbases[i], 0, -1, 0);
- __sysmmu_set_prefbuf(data->sfrbases[i], 0, -1, 1);
- }
+ if (!IS_ERR(data->clk_master))
+ clk_disable(data->clk_master);
+}
- __raw_writel(CTRL_ENABLE, data->sfrbases[i] + REG_MMU_CTRL);
+static int __sysmmu_enable(struct sysmmu_drvdata *data,
+ phys_addr_t pgtable, struct iommu_domain *domain)
+{
+ int ret = 0;
+ unsigned long flags;
+
+ spin_lock_irqsave(&data->lock, flags);
+ if (set_sysmmu_active(data)) {
+ data->pgtable = pgtable;
+ data->domain = domain;
+
+ __sysmmu_enable_nocount(data);
+
+ dev_dbg(data->sysmmu, "Enabled\n");
+ } else {
+ ret = (pgtable == data->pgtable) ? 1 : -EBUSY;
+
+ dev_dbg(data->sysmmu, "already enabled\n");
}
- data->domain = domain;
+ if (WARN_ON(ret < 0))
+ set_sysmmu_inactive(data); /* decrement count */
- dev_dbg(data->sysmmu, "(%s) Enabled\n", data->dbgname);
-finish:
- write_unlock_irqrestore(&data->lock, flags);
+ spin_unlock_irqrestore(&data->lock, flags);
return ret;
}
-int exynos_sysmmu_enable(struct device *dev, unsigned long pgtable)
+/* __exynos_sysmmu_enable: Enables System MMU
+ *
+ * returns -error if an error occurred and System MMU is not enabled,
+ * 0 if the System MMU has been just enabled and 1 if System MMU was already
+ * enabled before.
+ */
+static int __exynos_sysmmu_enable(struct device *dev, phys_addr_t pgtable,
+ struct iommu_domain *domain)
{
- struct sysmmu_drvdata *data = dev_get_drvdata(dev->archdata.iommu);
- int ret;
+ int ret = 0;
+ unsigned long flags;
+ struct exynos_iommu_owner *owner = dev->archdata.iommu;
+ struct sysmmu_drvdata *data;
- BUG_ON(!memblock_is_memory(pgtable));
+ BUG_ON(!has_sysmmu(dev));
- ret = pm_runtime_get_sync(data->sysmmu);
- if (ret < 0) {
- dev_dbg(data->sysmmu, "(%s) Failed to enable\n", data->dbgname);
- return ret;
- }
+ spin_lock_irqsave(&owner->lock, flags);
- ret = __exynos_sysmmu_enable(data, pgtable, NULL);
- if (WARN_ON(ret < 0)) {
- pm_runtime_put(data->sysmmu);
- dev_err(data->sysmmu,
- "(%s) Already enabled with page table %#lx\n",
- data->dbgname, data->pgtable);
- } else {
- data->dev = dev;
- }
+ data = dev_get_drvdata(owner->sysmmu);
+
+ ret = __sysmmu_enable(data, pgtable, domain);
+ if (ret >= 0)
+ data->master = dev;
+
+ spin_unlock_irqrestore(&owner->lock, flags);
return ret;
}
+int exynos_sysmmu_enable(struct device *dev, phys_addr_t pgtable)
+{
+ BUG_ON(!memblock_is_memory(pgtable));
+
+ return __exynos_sysmmu_enable(dev, pgtable, NULL);
+}
+
static bool exynos_sysmmu_disable(struct device *dev)
{
- struct sysmmu_drvdata *data = dev_get_drvdata(dev->archdata.iommu);
- bool disabled;
+ unsigned long flags;
+ bool disabled = true;
+ struct exynos_iommu_owner *owner = dev->archdata.iommu;
+ struct sysmmu_drvdata *data;
+
+ BUG_ON(!has_sysmmu(dev));
+
+ spin_lock_irqsave(&owner->lock, flags);
- disabled = __exynos_sysmmu_disable(data);
- pm_runtime_put(data->sysmmu);
+ data = dev_get_drvdata(owner->sysmmu);
+
+ disabled = __sysmmu_disable(data);
+ if (disabled)
+ data->master = NULL;
+
+ spin_unlock_irqrestore(&owner->lock, flags);
return disabled;
}
-static void sysmmu_tlb_invalidate_entry(struct device *dev, unsigned long iova)
+static void __sysmmu_tlb_invalidate_flpdcache(struct sysmmu_drvdata *data,
+ sysmmu_iova_t iova)
+{
+ if (__raw_sysmmu_version(data) == MAKE_MMU_VER(3, 3))
+ __raw_writel(iova | 0x1, data->sfrbase + REG_MMU_FLUSH_ENTRY);
+}
+
+static void sysmmu_tlb_invalidate_flpdcache(struct device *dev,
+ sysmmu_iova_t iova)
+{
+ unsigned long flags;
+ struct exynos_iommu_owner *owner = dev->archdata.iommu;
+ struct sysmmu_drvdata *data = dev_get_drvdata(owner->sysmmu);
+
+ if (!IS_ERR(data->clk_master))
+ clk_enable(data->clk_master);
+
+ spin_lock_irqsave(&data->lock, flags);
+ if (is_sysmmu_active(data))
+ __sysmmu_tlb_invalidate_flpdcache(data, iova);
+ spin_unlock_irqrestore(&data->lock, flags);
+
+ if (!IS_ERR(data->clk_master))
+ clk_disable(data->clk_master);
+}
+
+static void sysmmu_tlb_invalidate_entry(struct device *dev, sysmmu_iova_t iova,
+ size_t size)
{
+ struct exynos_iommu_owner *owner = dev->archdata.iommu;
unsigned long flags;
- struct sysmmu_drvdata *data = dev_get_drvdata(dev->archdata.iommu);
+ struct sysmmu_drvdata *data;
- read_lock_irqsave(&data->lock, flags);
+ data = dev_get_drvdata(owner->sysmmu);
+ spin_lock_irqsave(&data->lock, flags);
if (is_sysmmu_active(data)) {
- int i;
- for (i = 0; i < data->nsfrs; i++) {
- if (sysmmu_block(data->sfrbases[i])) {
- __sysmmu_tlb_invalidate_entry(
- data->sfrbases[i], iova);
- sysmmu_unblock(data->sfrbases[i]);
- }
+ unsigned int num_inv = 1;
+
+ if (!IS_ERR(data->clk_master))
+ clk_enable(data->clk_master);
+
+ /*
+ * L2TLB invalidation required
+ * 4KB page: 1 invalidation
+ * 64KB page: 16 invalidation
+ * 1MB page: 64 invalidation
+ * because it is set-associative TLB
+ * with 8-way and 64 sets.
+ * 1MB page can be cached in one of all sets.
+ * 64KB page can be one of 16 consecutive sets.
+ */
+ if (MMU_MAJ_VER(__raw_sysmmu_version(data)) == 2)
+ num_inv = min_t(unsigned int, size / PAGE_SIZE, 64);
+
+ if (sysmmu_block(data->sfrbase)) {
+ __sysmmu_tlb_invalidate_entry(
+ data->sfrbase, iova, num_inv);
+ sysmmu_unblock(data->sfrbase);
}
+ if (!IS_ERR(data->clk_master))
+ clk_disable(data->clk_master);
} else {
- dev_dbg(data->sysmmu,
- "(%s) Disabled. Skipping invalidating TLB.\n",
- data->dbgname);
+ dev_dbg(dev, "disabled. Skipping TLB invalidation @ %#x\n",
+ iova);
}
-
- read_unlock_irqrestore(&data->lock, flags);
+ spin_unlock_irqrestore(&data->lock, flags);
}
void exynos_sysmmu_tlb_invalidate(struct device *dev)
{
+ struct exynos_iommu_owner *owner = dev->archdata.iommu;
unsigned long flags;
- struct sysmmu_drvdata *data = dev_get_drvdata(dev->archdata.iommu);
+ struct sysmmu_drvdata *data;
- read_lock_irqsave(&data->lock, flags);
+ data = dev_get_drvdata(owner->sysmmu);
+ spin_lock_irqsave(&data->lock, flags);
if (is_sysmmu_active(data)) {
- int i;
- for (i = 0; i < data->nsfrs; i++) {
- if (sysmmu_block(data->sfrbases[i])) {
- __sysmmu_tlb_invalidate(data->sfrbases[i]);
- sysmmu_unblock(data->sfrbases[i]);
- }
+ if (!IS_ERR(data->clk_master))
+ clk_enable(data->clk_master);
+ if (sysmmu_block(data->sfrbase)) {
+ __sysmmu_tlb_invalidate(data->sfrbase);
+ sysmmu_unblock(data->sfrbase);
}
+ if (!IS_ERR(data->clk_master))
+ clk_disable(data->clk_master);
} else {
- dev_dbg(data->sysmmu,
- "(%s) Disabled. Skipping invalidating TLB.\n",
- data->dbgname);
+ dev_dbg(dev, "disabled. Skipping TLB invalidation\n");
}
-
- read_unlock_irqrestore(&data->lock, flags);
+ spin_unlock_irqrestore(&data->lock, flags);
}
-static int exynos_sysmmu_probe(struct platform_device *pdev)
+static int __init exynos_sysmmu_probe(struct platform_device *pdev)
{
- int i, ret;
- struct device *dev;
+ int irq, ret;
+ struct device *dev = &pdev->dev;
struct sysmmu_drvdata *data;
+ struct resource *res;
- dev = &pdev->dev;
+ data = devm_kzalloc(dev, sizeof(*data), GFP_KERNEL);
+ if (!data)
+ return -ENOMEM;
- data = kzalloc(sizeof(*data), GFP_KERNEL);
- if (!data) {
- dev_dbg(dev, "Not enough memory\n");
- ret = -ENOMEM;
- goto err_alloc;
- }
+ res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+ data->sfrbase = devm_ioremap_resource(dev, res);
+ if (IS_ERR(data->sfrbase))
+ return PTR_ERR(data->sfrbase);
- ret = dev_set_drvdata(dev, data);
- if (ret) {
- dev_dbg(dev, "Unabled to initialize driver data\n");
- goto err_init;
+ irq = platform_get_irq(pdev, 0);
+ if (irq <= 0) {
+ dev_err(dev, "Unable to find IRQ resource\n");
+ return irq;
}
- data->nsfrs = pdev->num_resources / 2;
- data->sfrbases = kmalloc(sizeof(*data->sfrbases) * data->nsfrs,
- GFP_KERNEL);
- if (data->sfrbases == NULL) {
- dev_dbg(dev, "Not enough memory\n");
- ret = -ENOMEM;
- goto err_init;
- }
-
- for (i = 0; i < data->nsfrs; i++) {
- struct resource *res;
- res = platform_get_resource(pdev, IORESOURCE_MEM, i);
- if (!res) {
- dev_dbg(dev, "Unable to find IOMEM region\n");
- ret = -ENOENT;
- goto err_res;
- }
-
- data->sfrbases[i] = ioremap(res->start, resource_size(res));
- if (!data->sfrbases[i]) {
- dev_dbg(dev, "Unable to map IOMEM @ PA:%#x\n",
- res->start);
- ret = -ENOENT;
- goto err_res;
- }
+ ret = devm_request_irq(dev, irq, exynos_sysmmu_irq, 0,
+ dev_name(dev), data);
+ if (ret) {
+ dev_err(dev, "Unabled to register handler of irq %d\n", irq);
+ return ret;
}
- for (i = 0; i < data->nsfrs; i++) {
- ret = platform_get_irq(pdev, i);
- if (ret <= 0) {
- dev_dbg(dev, "Unable to find IRQ resource\n");
- goto err_irq;
- }
-
- ret = request_irq(ret, exynos_sysmmu_irq, 0,
- dev_name(dev), data);
+ data->clk = devm_clk_get(dev, "sysmmu");
+ if (IS_ERR(data->clk)) {
+ dev_err(dev, "Failed to get clock!\n");
+ return PTR_ERR(data->clk);
+ } else {
+ ret = clk_prepare(data->clk);
if (ret) {
- dev_dbg(dev, "Unabled to register interrupt handler\n");
- goto err_irq;
+ dev_err(dev, "Failed to prepare clk\n");
+ return ret;
}
}
- if (dev_get_platdata(dev)) {
- char *deli, *beg;
- struct sysmmu_platform_data *platdata = dev_get_platdata(dev);
-
- beg = platdata->clockname;
-
- for (deli = beg; (*deli != '\0') && (*deli != ','); deli++)
- /* NOTHING */;
-
- if (*deli == '\0')
- deli = NULL;
- else
- *deli = '\0';
-
- data->clk[0] = clk_get(dev, beg);
- if (IS_ERR(data->clk[0])) {
- data->clk[0] = NULL;
- dev_dbg(dev, "No clock descriptor registered\n");
- }
-
- if (data->clk[0] && deli) {
- *deli = ',';
- data->clk[1] = clk_get(dev, deli + 1);
- if (IS_ERR(data->clk[1]))
- data->clk[1] = NULL;
+ data->clk_master = devm_clk_get(dev, "master");
+ if (!IS_ERR(data->clk_master)) {
+ ret = clk_prepare(data->clk_master);
+ if (ret) {
+ clk_unprepare(data->clk);
+ dev_err(dev, "Failed to prepare master's clk\n");
+ return ret;
}
-
- data->dbgname = platdata->dbgname;
}
data->sysmmu = dev;
- rwlock_init(&data->lock);
- INIT_LIST_HEAD(&data->node);
+ spin_lock_init(&data->lock);
- __set_fault_handler(data, &default_fault_handler);
+ platform_set_drvdata(pdev, data);
- if (dev->parent)
- pm_runtime_enable(dev);
+ pm_runtime_enable(dev);
- dev_dbg(dev, "(%s) Initialized\n", data->dbgname);
return 0;
-err_irq:
- while (i-- > 0) {
- int irq;
-
- irq = platform_get_irq(pdev, i);
- free_irq(irq, data);
- }
-err_res:
- while (data->nsfrs-- > 0)
- iounmap(data->sfrbases[data->nsfrs]);
- kfree(data->sfrbases);
-err_init:
- kfree(data);
-err_alloc:
- dev_err(dev, "Failed to initialize\n");
- return ret;
}
-static struct platform_driver exynos_sysmmu_driver = {
- .probe = exynos_sysmmu_probe,
- .driver = {
+static const struct of_device_id sysmmu_of_match[] __initconst = {
+ { .compatible = "samsung,exynos-sysmmu", },
+ { },
+};
+
+static struct platform_driver exynos_sysmmu_driver __refdata = {
+ .probe = exynos_sysmmu_probe,
+ .driver = {
.owner = THIS_MODULE,
.name = "exynos-sysmmu",
+ .of_match_table = sysmmu_of_match,
}
};
@@ -667,21 +700,32 @@ static inline void pgtable_flush(void *vastart, void *vaend)
static int exynos_iommu_domain_init(struct iommu_domain *domain)
{
struct exynos_iommu_domain *priv;
+ int i;
priv = kzalloc(sizeof(*priv), GFP_KERNEL);
if (!priv)
return -ENOMEM;
- priv->pgtable = (unsigned long *)__get_free_pages(
- GFP_KERNEL | __GFP_ZERO, 2);
+ priv->pgtable = (sysmmu_pte_t *)__get_free_pages(GFP_KERNEL, 2);
if (!priv->pgtable)
goto err_pgtable;
- priv->lv2entcnt = (short *)__get_free_pages(
- GFP_KERNEL | __GFP_ZERO, 1);
+ priv->lv2entcnt = (short *)__get_free_pages(GFP_KERNEL | __GFP_ZERO, 1);
if (!priv->lv2entcnt)
goto err_counter;
+ /* w/a of System MMU v3.3 to prevent caching 1MiB mapping */
+ for (i = 0; i < NUM_LV1ENTRIES; i += 8) {
+ priv->pgtable[i + 0] = ZERO_LV2LINK;
+ priv->pgtable[i + 1] = ZERO_LV2LINK;
+ priv->pgtable[i + 2] = ZERO_LV2LINK;
+ priv->pgtable[i + 3] = ZERO_LV2LINK;
+ priv->pgtable[i + 4] = ZERO_LV2LINK;
+ priv->pgtable[i + 5] = ZERO_LV2LINK;
+ priv->pgtable[i + 6] = ZERO_LV2LINK;
+ priv->pgtable[i + 7] = ZERO_LV2LINK;
+ }
+
pgtable_flush(priv->pgtable, priv->pgtable + NUM_LV1ENTRIES);
spin_lock_init(&priv->lock);
@@ -705,7 +749,7 @@ err_pgtable:
static void exynos_iommu_domain_destroy(struct iommu_domain *domain)
{
struct exynos_iommu_domain *priv = domain->priv;
- struct sysmmu_drvdata *data;
+ struct exynos_iommu_owner *owner;
unsigned long flags;
int i;
@@ -713,16 +757,20 @@ static void exynos_iommu_domain_destroy(struct iommu_domain *domain)
spin_lock_irqsave(&priv->lock, flags);
- list_for_each_entry(data, &priv->clients, node) {
- while (!exynos_sysmmu_disable(data->dev))
+ list_for_each_entry(owner, &priv->clients, client) {
+ while (!exynos_sysmmu_disable(owner->dev))
; /* until System MMU is actually disabled */
}
+ while (!list_empty(&priv->clients))
+ list_del_init(priv->clients.next);
+
spin_unlock_irqrestore(&priv->lock, flags);
for (i = 0; i < NUM_LV1ENTRIES; i++)
if (lv1ent_page(priv->pgtable + i))
- kfree(__va(lv2table_base(priv->pgtable + i)));
+ kmem_cache_free(lv2table_kmem_cache,
+ phys_to_virt(lv2table_base(priv->pgtable + i)));
free_pages((unsigned long)priv->pgtable, 2);
free_pages((unsigned long)priv->lv2entcnt, 1);
@@ -733,114 +781,134 @@ static void exynos_iommu_domain_destroy(struct iommu_domain *domain)
static int exynos_iommu_attach_device(struct iommu_domain *domain,
struct device *dev)
{
- struct sysmmu_drvdata *data = dev_get_drvdata(dev->archdata.iommu);
+ struct exynos_iommu_owner *owner = dev->archdata.iommu;
struct exynos_iommu_domain *priv = domain->priv;
+ phys_addr_t pagetable = virt_to_phys(priv->pgtable);
unsigned long flags;
int ret;
- ret = pm_runtime_get_sync(data->sysmmu);
- if (ret < 0)
- return ret;
-
- ret = 0;
-
spin_lock_irqsave(&priv->lock, flags);
- ret = __exynos_sysmmu_enable(data, __pa(priv->pgtable), domain);
-
+ ret = __exynos_sysmmu_enable(dev, pagetable, domain);
if (ret == 0) {
- /* 'data->node' must not be appeared in priv->clients */
- BUG_ON(!list_empty(&data->node));
- data->dev = dev;
- list_add_tail(&data->node, &priv->clients);
+ list_add_tail(&owner->client, &priv->clients);
+ owner->domain = domain;
}
spin_unlock_irqrestore(&priv->lock, flags);
if (ret < 0) {
- dev_err(dev, "%s: Failed to attach IOMMU with pgtable %#lx\n",
- __func__, __pa(priv->pgtable));
- pm_runtime_put(data->sysmmu);
- } else if (ret > 0) {
- dev_dbg(dev, "%s: IOMMU with pgtable 0x%lx already attached\n",
- __func__, __pa(priv->pgtable));
- } else {
- dev_dbg(dev, "%s: Attached new IOMMU with pgtable 0x%lx\n",
- __func__, __pa(priv->pgtable));
+ dev_err(dev, "%s: Failed to attach IOMMU with pgtable %pa\n",
+ __func__, &pagetable);
+ return ret;
}
+ dev_dbg(dev, "%s: Attached IOMMU with pgtable %pa %s\n",
+ __func__, &pagetable, (ret == 0) ? "" : ", again");
+
return ret;
}
static void exynos_iommu_detach_device(struct iommu_domain *domain,
struct device *dev)
{
- struct sysmmu_drvdata *data = dev_get_drvdata(dev->archdata.iommu);
+ struct exynos_iommu_owner *owner;
struct exynos_iommu_domain *priv = domain->priv;
- struct list_head *pos;
+ phys_addr_t pagetable = virt_to_phys(priv->pgtable);
unsigned long flags;
- bool found = false;
spin_lock_irqsave(&priv->lock, flags);
- list_for_each(pos, &priv->clients) {
- if (list_entry(pos, struct sysmmu_drvdata, node) == data) {
- found = true;
+ list_for_each_entry(owner, &priv->clients, client) {
+ if (owner == dev->archdata.iommu) {
+ if (exynos_sysmmu_disable(dev)) {
+ list_del_init(&owner->client);
+ owner->domain = NULL;
+ }
break;
}
}
- if (!found)
- goto finish;
-
- if (__exynos_sysmmu_disable(data)) {
- dev_dbg(dev, "%s: Detached IOMMU with pgtable %#lx\n",
- __func__, __pa(priv->pgtable));
- list_del_init(&data->node);
-
- } else {
- dev_dbg(dev, "%s: Detaching IOMMU with pgtable %#lx delayed",
- __func__, __pa(priv->pgtable));
- }
-
-finish:
spin_unlock_irqrestore(&priv->lock, flags);
- if (found)
- pm_runtime_put(data->sysmmu);
+ if (owner == dev->archdata.iommu)
+ dev_dbg(dev, "%s: Detached IOMMU with pgtable %pa\n",
+ __func__, &pagetable);
+ else
+ dev_err(dev, "%s: No IOMMU is attached\n", __func__);
}
-static unsigned long *alloc_lv2entry(unsigned long *sent, unsigned long iova,
- short *pgcounter)
+static sysmmu_pte_t *alloc_lv2entry(struct exynos_iommu_domain *priv,
+ sysmmu_pte_t *sent, sysmmu_iova_t iova, short *pgcounter)
{
+ if (lv1ent_section(sent)) {
+ WARN(1, "Trying mapping on %#08x mapped with 1MiB page", iova);
+ return ERR_PTR(-EADDRINUSE);
+ }
+
if (lv1ent_fault(sent)) {
- unsigned long *pent;
+ sysmmu_pte_t *pent;
+ bool need_flush_flpd_cache = lv1ent_zero(sent);
- pent = kzalloc(LV2TABLE_SIZE, GFP_ATOMIC);
- BUG_ON((unsigned long)pent & (LV2TABLE_SIZE - 1));
+ pent = kmem_cache_zalloc(lv2table_kmem_cache, GFP_ATOMIC);
+ BUG_ON((unsigned int)pent & (LV2TABLE_SIZE - 1));
if (!pent)
- return NULL;
+ return ERR_PTR(-ENOMEM);
- *sent = mk_lv1ent_page(__pa(pent));
+ *sent = mk_lv1ent_page(virt_to_phys(pent));
*pgcounter = NUM_LV2ENTRIES;
pgtable_flush(pent, pent + NUM_LV2ENTRIES);
pgtable_flush(sent, sent + 1);
+
+ /*
+ * If pretched SLPD is a fault SLPD in zero_l2_table, FLPD cache
+ * may caches the address of zero_l2_table. This function
+ * replaces the zero_l2_table with new L2 page table to write
+ * valid mappings.
+ * Accessing the valid area may cause page fault since FLPD
+ * cache may still caches zero_l2_table for the valid area
+ * instead of new L2 page table that have the mapping
+ * information of the valid area
+ * Thus any replacement of zero_l2_table with other valid L2
+ * page table must involve FLPD cache invalidation for System
+ * MMU v3.3.
+ * FLPD cache invalidation is performed with TLB invalidation
+ * by VPN without blocking. It is safe to invalidate TLB without
+ * blocking because the target address of TLB invalidation is
+ * not currently mapped.
+ */
+ if (need_flush_flpd_cache) {
+ struct exynos_iommu_owner *owner;
+
+ spin_lock(&priv->lock);
+ list_for_each_entry(owner, &priv->clients, client)
+ sysmmu_tlb_invalidate_flpdcache(
+ owner->dev, iova);
+ spin_unlock(&priv->lock);
+ }
}
return page_entry(sent, iova);
}
-static int lv1set_section(unsigned long *sent, phys_addr_t paddr, short *pgcnt)
+static int lv1set_section(struct exynos_iommu_domain *priv,
+ sysmmu_pte_t *sent, sysmmu_iova_t iova,
+ phys_addr_t paddr, short *pgcnt)
{
- if (lv1ent_section(sent))
+ if (lv1ent_section(sent)) {
+ WARN(1, "Trying mapping on 1MiB@%#08x that is mapped",
+ iova);
return -EADDRINUSE;
+ }
if (lv1ent_page(sent)) {
- if (*pgcnt != NUM_LV2ENTRIES)
+ if (*pgcnt != NUM_LV2ENTRIES) {
+ WARN(1, "Trying mapping on 1MiB@%#08x that is mapped",
+ iova);
return -EADDRINUSE;
+ }
- kfree(page_entry(sent, 0));
-
+ kmem_cache_free(lv2table_kmem_cache, page_entry(sent, 0));
*pgcnt = 0;
}
@@ -848,14 +916,26 @@ static int lv1set_section(unsigned long *sent, phys_addr_t paddr, short *pgcnt)
pgtable_flush(sent, sent + 1);
+ spin_lock(&priv->lock);
+ if (lv1ent_page_zero(sent)) {
+ struct exynos_iommu_owner *owner;
+ /*
+ * Flushing FLPD cache in System MMU v3.3 that may cache a FLPD
+ * entry by speculative prefetch of SLPD which has no mapping.
+ */
+ list_for_each_entry(owner, &priv->clients, client)
+ sysmmu_tlb_invalidate_flpdcache(owner->dev, iova);
+ }
+ spin_unlock(&priv->lock);
+
return 0;
}
-static int lv2set_page(unsigned long *pent, phys_addr_t paddr, size_t size,
+static int lv2set_page(sysmmu_pte_t *pent, phys_addr_t paddr, size_t size,
short *pgcnt)
{
if (size == SPAGE_SIZE) {
- if (!lv2ent_fault(pent))
+ if (WARN_ON(!lv2ent_fault(pent)))
return -EADDRINUSE;
*pent = mk_lv2ent_spage(paddr);
@@ -863,9 +943,11 @@ static int lv2set_page(unsigned long *pent, phys_addr_t paddr, size_t size,
*pgcnt -= 1;
} else { /* size == LPAGE_SIZE */
int i;
+
for (i = 0; i < SPAGES_PER_LPAGE; i++, pent++) {
- if (!lv2ent_fault(pent)) {
- memset(pent, 0, sizeof(*pent) * i);
+ if (WARN_ON(!lv2ent_fault(pent))) {
+ if (i > 0)
+ memset(pent - i, 0, sizeof(*pent) * i);
return -EADDRINUSE;
}
@@ -878,11 +960,38 @@ static int lv2set_page(unsigned long *pent, phys_addr_t paddr, size_t size,
return 0;
}
-static int exynos_iommu_map(struct iommu_domain *domain, unsigned long iova,
+/*
+ * *CAUTION* to the I/O virtual memory managers that support exynos-iommu:
+ *
+ * System MMU v3.x have an advanced logic to improve address translation
+ * performance with caching more page table entries by a page table walk.
+ * However, the logic has a bug that caching fault page table entries and System
+ * MMU reports page fault if the cached fault entry is hit even though the fault
+ * entry is updated to a valid entry after the entry is cached.
+ * To prevent caching fault page table entries which may be updated to valid
+ * entries later, the virtual memory manager should care about the w/a about the
+ * problem. The followings describe w/a.
+ *
+ * Any two consecutive I/O virtual address regions must have a hole of 128KiB
+ * in maximum to prevent misbehavior of System MMU 3.x. (w/a of h/w bug)
+ *
+ * Precisely, any start address of I/O virtual region must be aligned by
+ * the following sizes for System MMU v3.1 and v3.2.
+ * System MMU v3.1: 128KiB
+ * System MMU v3.2: 256KiB
+ *
+ * Because System MMU v3.3 caches page table entries more aggressively, it needs
+ * more w/a.
+ * - Any two consecutive I/O virtual regions must be have a hole of larger size
+ * than or equal size to 128KiB.
+ * - Start address of an I/O virtual region must be aligned by 128KiB.
+ */
+static int exynos_iommu_map(struct iommu_domain *domain, unsigned long l_iova,
phys_addr_t paddr, size_t size, int prot)
{
struct exynos_iommu_domain *priv = domain->priv;
- unsigned long *entry;
+ sysmmu_pte_t *entry;
+ sysmmu_iova_t iova = (sysmmu_iova_t)l_iova;
unsigned long flags;
int ret = -ENOMEM;
@@ -893,38 +1002,52 @@ static int exynos_iommu_map(struct iommu_domain *domain, unsigned long iova,
entry = section_entry(priv->pgtable, iova);
if (size == SECT_SIZE) {
- ret = lv1set_section(entry, paddr,
+ ret = lv1set_section(priv, entry, iova, paddr,
&priv->lv2entcnt[lv1ent_offset(iova)]);
} else {
- unsigned long *pent;
+ sysmmu_pte_t *pent;
- pent = alloc_lv2entry(entry, iova,
+ pent = alloc_lv2entry(priv, entry, iova,
&priv->lv2entcnt[lv1ent_offset(iova)]);
- if (!pent)
- ret = -ENOMEM;
+ if (IS_ERR(pent))
+ ret = PTR_ERR(pent);
else
ret = lv2set_page(pent, paddr, size,
&priv->lv2entcnt[lv1ent_offset(iova)]);
}
- if (ret) {
- pr_debug("%s: Failed to map iova 0x%lx/0x%x bytes\n",
- __func__, iova, size);
- }
+ if (ret)
+ pr_err("%s: Failed(%d) to map %#zx bytes @ %#x\n",
+ __func__, ret, size, iova);
spin_unlock_irqrestore(&priv->pgtablelock, flags);
return ret;
}
+static void exynos_iommu_tlb_invalidate_entry(struct exynos_iommu_domain *priv,
+ sysmmu_iova_t iova, size_t size)
+{
+ struct exynos_iommu_owner *owner;
+ unsigned long flags;
+
+ spin_lock_irqsave(&priv->lock, flags);
+
+ list_for_each_entry(owner, &priv->clients, client)
+ sysmmu_tlb_invalidate_entry(owner->dev, iova, size);
+
+ spin_unlock_irqrestore(&priv->lock, flags);
+}
+
static size_t exynos_iommu_unmap(struct iommu_domain *domain,
- unsigned long iova, size_t size)
+ unsigned long l_iova, size_t size)
{
struct exynos_iommu_domain *priv = domain->priv;
- struct sysmmu_drvdata *data;
+ sysmmu_iova_t iova = (sysmmu_iova_t)l_iova;
+ sysmmu_pte_t *ent;
+ size_t err_pgsize;
unsigned long flags;
- unsigned long *ent;
BUG_ON(priv->pgtable == NULL);
@@ -933,9 +1056,12 @@ static size_t exynos_iommu_unmap(struct iommu_domain *domain,
ent = section_entry(priv->pgtable, iova);
if (lv1ent_section(ent)) {
- BUG_ON(size < SECT_SIZE);
+ if (WARN_ON(size < SECT_SIZE)) {
+ err_pgsize = SECT_SIZE;
+ goto err;
+ }
- *ent = 0;
+ *ent = ZERO_LV2LINK; /* w/a for h/w bug in Sysmem MMU v3.3 */
pgtable_flush(ent, ent + 1);
size = SECT_SIZE;
goto done;
@@ -959,34 +1085,42 @@ static size_t exynos_iommu_unmap(struct iommu_domain *domain,
if (lv2ent_small(ent)) {
*ent = 0;
size = SPAGE_SIZE;
+ pgtable_flush(ent, ent + 1);
priv->lv2entcnt[lv1ent_offset(iova)] += 1;
goto done;
}
/* lv1ent_large(ent) == true here */
- BUG_ON(size < LPAGE_SIZE);
+ if (WARN_ON(size < LPAGE_SIZE)) {
+ err_pgsize = LPAGE_SIZE;
+ goto err;
+ }
memset(ent, 0, sizeof(*ent) * SPAGES_PER_LPAGE);
+ pgtable_flush(ent, ent + SPAGES_PER_LPAGE);
size = LPAGE_SIZE;
priv->lv2entcnt[lv1ent_offset(iova)] += SPAGES_PER_LPAGE;
done:
spin_unlock_irqrestore(&priv->pgtablelock, flags);
- spin_lock_irqsave(&priv->lock, flags);
- list_for_each_entry(data, &priv->clients, node)
- sysmmu_tlb_invalidate_entry(data->dev, iova);
- spin_unlock_irqrestore(&priv->lock, flags);
-
+ exynos_iommu_tlb_invalidate_entry(priv, iova, size);
return size;
+err:
+ spin_unlock_irqrestore(&priv->pgtablelock, flags);
+
+ pr_err("%s: Failed: size(%#zx) @ %#x is smaller than page size %#zx\n",
+ __func__, size, iova, err_pgsize);
+
+ return 0;
}
static phys_addr_t exynos_iommu_iova_to_phys(struct iommu_domain *domain,
dma_addr_t iova)
{
struct exynos_iommu_domain *priv = domain->priv;
- unsigned long *entry;
+ sysmmu_pte_t *entry;
unsigned long flags;
phys_addr_t phys = 0;
@@ -1010,14 +1144,42 @@ static phys_addr_t exynos_iommu_iova_to_phys(struct iommu_domain *domain,
return phys;
}
+static int exynos_iommu_add_device(struct device *dev)
+{
+ struct iommu_group *group;
+ int ret;
+
+ group = iommu_group_get(dev);
+
+ if (!group) {
+ group = iommu_group_alloc();
+ if (IS_ERR(group)) {
+ dev_err(dev, "Failed to allocate IOMMU group\n");
+ return PTR_ERR(group);
+ }
+ }
+
+ ret = iommu_group_add_device(group, dev);
+ iommu_group_put(group);
+
+ return ret;
+}
+
+static void exynos_iommu_remove_device(struct device *dev)
+{
+ iommu_group_remove_device(dev);
+}
+
static struct iommu_ops exynos_iommu_ops = {
- .domain_init = &exynos_iommu_domain_init,
- .domain_destroy = &exynos_iommu_domain_destroy,
- .attach_dev = &exynos_iommu_attach_device,
- .detach_dev = &exynos_iommu_detach_device,
- .map = &exynos_iommu_map,
- .unmap = &exynos_iommu_unmap,
- .iova_to_phys = &exynos_iommu_iova_to_phys,
+ .domain_init = exynos_iommu_domain_init,
+ .domain_destroy = exynos_iommu_domain_destroy,
+ .attach_dev = exynos_iommu_attach_device,
+ .detach_dev = exynos_iommu_detach_device,
+ .map = exynos_iommu_map,
+ .unmap = exynos_iommu_unmap,
+ .iova_to_phys = exynos_iommu_iova_to_phys,
+ .add_device = exynos_iommu_add_device,
+ .remove_device = exynos_iommu_remove_device,
.pgsize_bitmap = SECT_SIZE | LPAGE_SIZE | SPAGE_SIZE,
};
@@ -1025,11 +1187,41 @@ static int __init exynos_iommu_init(void)
{
int ret;
+ lv2table_kmem_cache = kmem_cache_create("exynos-iommu-lv2table",
+ LV2TABLE_SIZE, LV2TABLE_SIZE, 0, NULL);
+ if (!lv2table_kmem_cache) {
+ pr_err("%s: Failed to create kmem cache\n", __func__);
+ return -ENOMEM;
+ }
+
ret = platform_driver_register(&exynos_sysmmu_driver);
+ if (ret) {
+ pr_err("%s: Failed to register driver\n", __func__);
+ goto err_reg_driver;
+ }
- if (ret == 0)
- bus_set_iommu(&platform_bus_type, &exynos_iommu_ops);
+ zero_lv2_table = kmem_cache_zalloc(lv2table_kmem_cache, GFP_KERNEL);
+ if (zero_lv2_table == NULL) {
+ pr_err("%s: Failed to allocate zero level2 page table\n",
+ __func__);
+ ret = -ENOMEM;
+ goto err_zero_lv2;
+ }
+ ret = bus_set_iommu(&platform_bus_type, &exynos_iommu_ops);
+ if (ret) {
+ pr_err("%s: Failed to register exynos-iommu driver.\n",
+ __func__);
+ goto err_set_iommu;
+ }
+
+ return 0;
+err_set_iommu:
+ kmem_cache_free(lv2table_kmem_cache, zero_lv2_table);
+err_zero_lv2:
+ platform_driver_unregister(&exynos_sysmmu_driver);
+err_reg_driver:
+ kmem_cache_destroy(lv2table_kmem_cache);
return ret;
}
subsys_initcall(exynos_iommu_init);
diff --git a/drivers/iommu/fsl_pamu.c b/drivers/iommu/fsl_pamu.c
index cba0498eb01..bb446d742a2 100644
--- a/drivers/iommu/fsl_pamu.c
+++ b/drivers/iommu/fsl_pamu.c
@@ -170,10 +170,10 @@ int pamu_disable_liodn(int liodn)
static unsigned int map_addrspace_size_to_wse(phys_addr_t addrspace_size)
{
/* Bug if not a power of 2 */
- BUG_ON(!is_power_of_2(addrspace_size));
+ BUG_ON((addrspace_size & (addrspace_size - 1)));
/* window size is 2^(WSE+1) bytes */
- return __ffs(addrspace_size) - 1;
+ return fls64(addrspace_size) - 2;
}
/* Derive the PAACE window count encoding for the subwindow count */
@@ -351,7 +351,7 @@ int pamu_config_ppaace(int liodn, phys_addr_t win_addr, phys_addr_t win_size,
struct paace *ppaace;
unsigned long fspi;
- if (!is_power_of_2(win_size) || win_size < PAMU_PAGE_SIZE) {
+ if ((win_size & (win_size - 1)) || win_size < PAMU_PAGE_SIZE) {
pr_debug("window size too small or not a power of two %llx\n", win_size);
return -EINVAL;
}
@@ -464,7 +464,7 @@ int pamu_config_spaace(int liodn, u32 subwin_cnt, u32 subwin,
return -ENOENT;
}
- if (!is_power_of_2(subwin_size) || subwin_size < PAMU_PAGE_SIZE) {
+ if ((subwin_size & (subwin_size - 1)) || subwin_size < PAMU_PAGE_SIZE) {
pr_debug("subwindow size out of range, or not a power of 2\n");
return -EINVAL;
}
@@ -592,8 +592,7 @@ found_cpu_node:
/* advance to next node in cache hierarchy */
node = of_find_node_by_phandle(*prop);
if (!node) {
- pr_debug("Invalid node for cache hierarchy %s\n",
- node->full_name);
+ pr_debug("Invalid node for cache hierarchy\n");
return ~(u32)0;
}
}
diff --git a/drivers/iommu/fsl_pamu_domain.c b/drivers/iommu/fsl_pamu_domain.c
index 93072ba44b1..af47648301a 100644
--- a/drivers/iommu/fsl_pamu_domain.c
+++ b/drivers/iommu/fsl_pamu_domain.c
@@ -301,7 +301,7 @@ static int check_size(u64 size, dma_addr_t iova)
* Size must be a power of two and at least be equal
* to PAMU page size.
*/
- if (!is_power_of_2(size) || size < PAMU_PAGE_SIZE) {
+ if ((size & (size - 1)) || size < PAMU_PAGE_SIZE) {
pr_debug("%s: size too small or not a power of two\n", __func__);
return -EINVAL;
}
@@ -335,11 +335,6 @@ static struct fsl_dma_domain *iommu_alloc_dma_domain(void)
return domain;
}
-static inline struct device_domain_info *find_domain(struct device *dev)
-{
- return dev->archdata.iommu_domain;
-}
-
static void remove_device_ref(struct device_domain_info *info, u32 win_cnt)
{
unsigned long flags;
@@ -380,7 +375,7 @@ static void attach_device(struct fsl_dma_domain *dma_domain, int liodn, struct d
* Check here if the device is already attached to domain or not.
* If the device is already attached to a domain detach it.
*/
- old_domain_info = find_domain(dev);
+ old_domain_info = dev->archdata.iommu_domain;
if (old_domain_info && old_domain_info->domain != dma_domain) {
spin_unlock_irqrestore(&device_domain_lock, flags);
detach_device(dev, old_domain_info->domain);
@@ -399,7 +394,7 @@ static void attach_device(struct fsl_dma_domain *dma_domain, int liodn, struct d
* the info for the first LIODN as all
* LIODNs share the same domain
*/
- if (!old_domain_info)
+ if (!dev->archdata.iommu_domain)
dev->archdata.iommu_domain = info;
spin_unlock_irqrestore(&device_domain_lock, flags);
@@ -1042,12 +1037,15 @@ root_bus:
group = get_shared_pci_device_group(pdev);
}
+ if (!group)
+ group = ERR_PTR(-ENODEV);
+
return group;
}
static int fsl_pamu_add_device(struct device *dev)
{
- struct iommu_group *group = NULL;
+ struct iommu_group *group = ERR_PTR(-ENODEV);
struct pci_dev *pdev;
const u32 *prop;
int ret, len;
@@ -1070,7 +1068,7 @@ static int fsl_pamu_add_device(struct device *dev)
group = get_device_iommu_group(dev);
}
- if (!group || IS_ERR(group))
+ if (IS_ERR(group))
return PTR_ERR(group);
ret = iommu_group_add_device(group, dev);
diff --git a/drivers/iommu/intel-iommu.c b/drivers/iommu/intel-iommu.c
index a22c86c867f..51b6b77dc3e 100644
--- a/drivers/iommu/intel-iommu.c
+++ b/drivers/iommu/intel-iommu.c
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2006, Intel Corporation.
+ * Copyright © 2006-2014 Intel Corporation.
*
* This program is free software; you can redistribute it and/or modify it
* under the terms and conditions of the GNU General Public License,
@@ -10,15 +10,11 @@
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
* more details.
*
- * You should have received a copy of the GNU General Public License along with
- * this program; if not, write to the Free Software Foundation, Inc., 59 Temple
- * Place - Suite 330, Boston, MA 02111-1307 USA.
- *
- * Copyright (C) 2006-2008 Intel Corporation
- * Author: Ashok Raj <ashok.raj@intel.com>
- * Author: Shaohua Li <shaohua.li@intel.com>
- * Author: Anil S Keshavamurthy <anil.s.keshavamurthy@intel.com>
- * Author: Fenghua Yu <fenghua.yu@intel.com>
+ * Authors: David Woodhouse <dwmw2@infradead.org>,
+ * Ashok Raj <ashok.raj@intel.com>,
+ * Shaohua Li <shaohua.li@intel.com>,
+ * Anil S Keshavamurthy <anil.s.keshavamurthy@intel.com>,
+ * Fenghua Yu <fenghua.yu@intel.com>
*/
#include <linux/init.h>
@@ -33,6 +29,7 @@
#include <linux/dmar.h>
#include <linux/dma-mapping.h>
#include <linux/mempool.h>
+#include <linux/memory.h>
#include <linux/timer.h>
#include <linux/iova.h>
#include <linux/iommu.h>
@@ -42,6 +39,7 @@
#include <linux/dmi.h>
#include <linux/pci-ats.h>
#include <linux/memblock.h>
+#include <linux/dma-contiguous.h>
#include <asm/irq_remapping.h>
#include <asm/cacheflush.h>
#include <asm/iommu.h>
@@ -372,14 +370,36 @@ struct dmar_domain {
struct device_domain_info {
struct list_head link; /* link to domain siblings */
struct list_head global; /* link to global list */
- int segment; /* PCI domain */
u8 bus; /* PCI bus number */
u8 devfn; /* PCI devfn number */
- struct pci_dev *dev; /* it's NULL for PCIe-to-PCI bridge */
+ struct device *dev; /* it's NULL for PCIe-to-PCI bridge */
struct intel_iommu *iommu; /* IOMMU used by this device */
struct dmar_domain *domain; /* pointer to domain */
};
+struct dmar_rmrr_unit {
+ struct list_head list; /* list of rmrr units */
+ struct acpi_dmar_header *hdr; /* ACPI header */
+ u64 base_address; /* reserved base address*/
+ u64 end_address; /* reserved end address */
+ struct dmar_dev_scope *devices; /* target devices */
+ int devices_cnt; /* target device count */
+};
+
+struct dmar_atsr_unit {
+ struct list_head list; /* list of ATSR units */
+ struct acpi_dmar_header *hdr; /* ACPI header */
+ struct dmar_dev_scope *devices; /* target devices */
+ int devices_cnt; /* target device count */
+ u8 include_all:1; /* include all ports */
+};
+
+static LIST_HEAD(dmar_atsr_units);
+static LIST_HEAD(dmar_rmrr_units);
+
+#define for_each_rmrr_units(rmrr) \
+ list_for_each_entry(rmrr, &dmar_rmrr_units, list)
+
static void flush_unmaps_timeout(unsigned long data);
static DEFINE_TIMER(unmap_timer, flush_unmaps_timeout, 0, 0);
@@ -389,6 +409,7 @@ struct deferred_flush_tables {
int next;
struct iova *iova[HIGH_WATER_MARK];
struct dmar_domain *domain[HIGH_WATER_MARK];
+ struct page *freelist[HIGH_WATER_MARK];
};
static struct deferred_flush_tables *deferred_flush;
@@ -402,7 +423,12 @@ static LIST_HEAD(unmaps_to_do);
static int timer_on;
static long list_size;
+static void domain_exit(struct dmar_domain *domain);
static void domain_remove_dev_info(struct dmar_domain *domain);
+static void domain_remove_one_dev_info(struct dmar_domain *domain,
+ struct device *dev);
+static void iommu_detach_dependent_devices(struct intel_iommu *iommu,
+ struct device *dev);
#ifdef CONFIG_INTEL_IOMMU_DEFAULT_ON
int dmar_disabled = 0;
@@ -566,18 +592,31 @@ static struct intel_iommu *domain_get_iommu(struct dmar_domain *domain)
static void domain_update_iommu_coherency(struct dmar_domain *domain)
{
- int i;
-
- i = find_first_bit(domain->iommu_bmp, g_num_of_iommus);
+ struct dmar_drhd_unit *drhd;
+ struct intel_iommu *iommu;
+ int i, found = 0;
- domain->iommu_coherency = i < g_num_of_iommus ? 1 : 0;
+ domain->iommu_coherency = 1;
for_each_set_bit(i, domain->iommu_bmp, g_num_of_iommus) {
+ found = 1;
if (!ecap_coherent(g_iommus[i]->ecap)) {
domain->iommu_coherency = 0;
break;
}
}
+ if (found)
+ return;
+
+ /* No hardware attached; use lowest common denominator */
+ rcu_read_lock();
+ for_each_active_iommu(iommu, drhd) {
+ if (!ecap_coherent(iommu->ecap)) {
+ domain->iommu_coherency = 0;
+ break;
+ }
+ }
+ rcu_read_unlock();
}
static void domain_update_iommu_snooping(struct dmar_domain *domain)
@@ -606,12 +645,15 @@ static void domain_update_iommu_superpage(struct dmar_domain *domain)
}
/* set iommu_superpage to the smallest common denominator */
+ rcu_read_lock();
for_each_active_iommu(iommu, drhd) {
mask &= cap_super_page_val(iommu->cap);
if (!mask) {
break;
}
}
+ rcu_read_unlock();
+
domain->iommu_superpage = fls(mask);
}
@@ -623,32 +665,56 @@ static void domain_update_iommu_cap(struct dmar_domain *domain)
domain_update_iommu_superpage(domain);
}
-static struct intel_iommu *device_to_iommu(int segment, u8 bus, u8 devfn)
+static struct intel_iommu *device_to_iommu(struct device *dev, u8 *bus, u8 *devfn)
{
struct dmar_drhd_unit *drhd = NULL;
+ struct intel_iommu *iommu;
+ struct device *tmp;
+ struct pci_dev *ptmp, *pdev = NULL;
+ u16 segment;
int i;
- for_each_active_drhd_unit(drhd) {
- if (segment != drhd->segment)
+ if (dev_is_pci(dev)) {
+ pdev = to_pci_dev(dev);
+ segment = pci_domain_nr(pdev->bus);
+ } else if (ACPI_COMPANION(dev))
+ dev = &ACPI_COMPANION(dev)->dev;
+
+ rcu_read_lock();
+ for_each_active_iommu(iommu, drhd) {
+ if (pdev && segment != drhd->segment)
continue;
- for (i = 0; i < drhd->devices_cnt; i++) {
- if (drhd->devices[i] &&
- drhd->devices[i]->bus->number == bus &&
- drhd->devices[i]->devfn == devfn)
- return drhd->iommu;
- if (drhd->devices[i] &&
- drhd->devices[i]->subordinate &&
- drhd->devices[i]->subordinate->number <= bus &&
- drhd->devices[i]->subordinate->busn_res.end >= bus)
- return drhd->iommu;
+ for_each_active_dev_scope(drhd->devices,
+ drhd->devices_cnt, i, tmp) {
+ if (tmp == dev) {
+ *bus = drhd->devices[i].bus;
+ *devfn = drhd->devices[i].devfn;
+ goto out;
+ }
+
+ if (!pdev || !dev_is_pci(tmp))
+ continue;
+
+ ptmp = to_pci_dev(tmp);
+ if (ptmp->subordinate &&
+ ptmp->subordinate->number <= pdev->bus->number &&
+ ptmp->subordinate->busn_res.end >= pdev->bus->number)
+ goto got_pdev;
}
- if (drhd->include_all)
- return drhd->iommu;
+ if (pdev && drhd->include_all) {
+ got_pdev:
+ *bus = pdev->bus->number;
+ *devfn = pdev->devfn;
+ goto out;
+ }
}
+ iommu = NULL;
+ out:
+ rcu_read_unlock();
- return NULL;
+ return iommu;
}
static void domain_flush_cache(struct dmar_domain *domain,
@@ -748,7 +814,7 @@ out:
}
static struct dma_pte *pfn_to_dma_pte(struct dmar_domain *domain,
- unsigned long pfn, int target_level)
+ unsigned long pfn, int *target_level)
{
int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT;
struct dma_pte *parent, *pte = NULL;
@@ -763,14 +829,14 @@ static struct dma_pte *pfn_to_dma_pte(struct dmar_domain *domain,
parent = domain->pgd;
- while (level > 0) {
+ while (1) {
void *tmp_page;
offset = pfn_level_offset(pfn, level);
pte = &parent[offset];
- if (!target_level && (dma_pte_superpage(pte) || !dma_pte_present(pte)))
+ if (!*target_level && (dma_pte_superpage(pte) || !dma_pte_present(pte)))
break;
- if (level == target_level)
+ if (level == *target_level)
break;
if (!dma_pte_present(pte)) {
@@ -791,10 +857,16 @@ static struct dma_pte *pfn_to_dma_pte(struct dmar_domain *domain,
domain_flush_cache(domain, pte, sizeof(*pte));
}
}
+ if (level == 1)
+ break;
+
parent = phys_to_virt(dma_pte_addr(pte));
level--;
}
+ if (!*target_level)
+ *target_level = level;
+
return pte;
}
@@ -832,7 +904,7 @@ static struct dma_pte *dma_pfn_level_pte(struct dmar_domain *domain,
}
/* clear last level pte, a tlb flush should be followed */
-static int dma_pte_clear_range(struct dmar_domain *domain,
+static void dma_pte_clear_range(struct dmar_domain *domain,
unsigned long start_pfn,
unsigned long last_pfn)
{
@@ -862,8 +934,6 @@ static int dma_pte_clear_range(struct dmar_domain *domain,
(void *)pte - (void *)first_pte);
} while (start_pfn && start_pfn <= last_pfn);
-
- return min_t(int, (large_page - 1) * 9, MAX_AGAW_PFN_WIDTH);
}
static void dma_pte_free_level(struct dmar_domain *domain, int level,
@@ -921,6 +991,125 @@ static void dma_pte_free_pagetable(struct dmar_domain *domain,
}
}
+/* When a page at a given level is being unlinked from its parent, we don't
+ need to *modify* it at all. All we need to do is make a list of all the
+ pages which can be freed just as soon as we've flushed the IOTLB and we
+ know the hardware page-walk will no longer touch them.
+ The 'pte' argument is the *parent* PTE, pointing to the page that is to
+ be freed. */
+static struct page *dma_pte_list_pagetables(struct dmar_domain *domain,
+ int level, struct dma_pte *pte,
+ struct page *freelist)
+{
+ struct page *pg;
+
+ pg = pfn_to_page(dma_pte_addr(pte) >> PAGE_SHIFT);
+ pg->freelist = freelist;
+ freelist = pg;
+
+ if (level == 1)
+ return freelist;
+
+ pte = page_address(pg);
+ do {
+ if (dma_pte_present(pte) && !dma_pte_superpage(pte))
+ freelist = dma_pte_list_pagetables(domain, level - 1,
+ pte, freelist);
+ pte++;
+ } while (!first_pte_in_page(pte));
+
+ return freelist;
+}
+
+static struct page *dma_pte_clear_level(struct dmar_domain *domain, int level,
+ struct dma_pte *pte, unsigned long pfn,
+ unsigned long start_pfn,
+ unsigned long last_pfn,
+ struct page *freelist)
+{
+ struct dma_pte *first_pte = NULL, *last_pte = NULL;
+
+ pfn = max(start_pfn, pfn);
+ pte = &pte[pfn_level_offset(pfn, level)];
+
+ do {
+ unsigned long level_pfn;
+
+ if (!dma_pte_present(pte))
+ goto next;
+
+ level_pfn = pfn & level_mask(level);
+
+ /* If range covers entire pagetable, free it */
+ if (start_pfn <= level_pfn &&
+ last_pfn >= level_pfn + level_size(level) - 1) {
+ /* These suborbinate page tables are going away entirely. Don't
+ bother to clear them; we're just going to *free* them. */
+ if (level > 1 && !dma_pte_superpage(pte))
+ freelist = dma_pte_list_pagetables(domain, level - 1, pte, freelist);
+
+ dma_clear_pte(pte);
+ if (!first_pte)
+ first_pte = pte;
+ last_pte = pte;
+ } else if (level > 1) {
+ /* Recurse down into a level that isn't *entirely* obsolete */
+ freelist = dma_pte_clear_level(domain, level - 1,
+ phys_to_virt(dma_pte_addr(pte)),
+ level_pfn, start_pfn, last_pfn,
+ freelist);
+ }
+next:
+ pfn += level_size(level);
+ } while (!first_pte_in_page(++pte) && pfn <= last_pfn);
+
+ if (first_pte)
+ domain_flush_cache(domain, first_pte,
+ (void *)++last_pte - (void *)first_pte);
+
+ return freelist;
+}
+
+/* We can't just free the pages because the IOMMU may still be walking
+ the page tables, and may have cached the intermediate levels. The
+ pages can only be freed after the IOTLB flush has been done. */
+struct page *domain_unmap(struct dmar_domain *domain,
+ unsigned long start_pfn,
+ unsigned long last_pfn)
+{
+ int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT;
+ struct page *freelist = NULL;
+
+ BUG_ON(addr_width < BITS_PER_LONG && start_pfn >> addr_width);
+ BUG_ON(addr_width < BITS_PER_LONG && last_pfn >> addr_width);
+ BUG_ON(start_pfn > last_pfn);
+
+ /* we don't need lock here; nobody else touches the iova range */
+ freelist = dma_pte_clear_level(domain, agaw_to_level(domain->agaw),
+ domain->pgd, 0, start_pfn, last_pfn, NULL);
+
+ /* free pgd */
+ if (start_pfn == 0 && last_pfn == DOMAIN_MAX_PFN(domain->gaw)) {
+ struct page *pgd_page = virt_to_page(domain->pgd);
+ pgd_page->freelist = freelist;
+ freelist = pgd_page;
+
+ domain->pgd = NULL;
+ }
+
+ return freelist;
+}
+
+void dma_free_pagelist(struct page *freelist)
+{
+ struct page *pg;
+
+ while ((pg = freelist)) {
+ freelist = pg->freelist;
+ free_pgtable_page(page_address(pg));
+ }
+}
+
/* iommu handling */
static int iommu_alloc_root_entry(struct intel_iommu *iommu)
{
@@ -1030,7 +1219,7 @@ static void __iommu_flush_iotlb(struct intel_iommu *iommu, u16 did,
break;
case DMA_TLB_PSI_FLUSH:
val = DMA_TLB_PSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
- /* Note: always flush non-leaf currently */
+ /* IH bit is passed in as part of address */
val_iva = size_order | addr;
break;
default:
@@ -1069,13 +1258,14 @@ static void __iommu_flush_iotlb(struct intel_iommu *iommu, u16 did,
(unsigned long long)DMA_TLB_IAIG(val));
}
-static struct device_domain_info *iommu_support_dev_iotlb(
- struct dmar_domain *domain, int segment, u8 bus, u8 devfn)
+static struct device_domain_info *
+iommu_support_dev_iotlb (struct dmar_domain *domain, struct intel_iommu *iommu,
+ u8 bus, u8 devfn)
{
int found = 0;
unsigned long flags;
struct device_domain_info *info;
- struct intel_iommu *iommu = device_to_iommu(segment, bus, devfn);
+ struct pci_dev *pdev;
if (!ecap_dev_iotlb_support(iommu->ecap))
return NULL;
@@ -1091,34 +1281,35 @@ static struct device_domain_info *iommu_support_dev_iotlb(
}
spin_unlock_irqrestore(&device_domain_lock, flags);
- if (!found || !info->dev)
+ if (!found || !info->dev || !dev_is_pci(info->dev))
return NULL;
- if (!pci_find_ext_capability(info->dev, PCI_EXT_CAP_ID_ATS))
- return NULL;
+ pdev = to_pci_dev(info->dev);
- if (!dmar_find_matched_atsr_unit(info->dev))
+ if (!pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_ATS))
return NULL;
- info->iommu = iommu;
+ if (!dmar_find_matched_atsr_unit(pdev))
+ return NULL;
return info;
}
static void iommu_enable_dev_iotlb(struct device_domain_info *info)
{
- if (!info)
+ if (!info || !dev_is_pci(info->dev))
return;
- pci_enable_ats(info->dev, VTD_PAGE_SHIFT);
+ pci_enable_ats(to_pci_dev(info->dev), VTD_PAGE_SHIFT);
}
static void iommu_disable_dev_iotlb(struct device_domain_info *info)
{
- if (!info->dev || !pci_ats_enabled(info->dev))
+ if (!info->dev || !dev_is_pci(info->dev) ||
+ !pci_ats_enabled(to_pci_dev(info->dev)))
return;
- pci_disable_ats(info->dev);
+ pci_disable_ats(to_pci_dev(info->dev));
}
static void iommu_flush_dev_iotlb(struct dmar_domain *domain,
@@ -1130,24 +1321,31 @@ static void iommu_flush_dev_iotlb(struct dmar_domain *domain,
spin_lock_irqsave(&device_domain_lock, flags);
list_for_each_entry(info, &domain->devices, link) {
- if (!info->dev || !pci_ats_enabled(info->dev))
+ struct pci_dev *pdev;
+ if (!info->dev || !dev_is_pci(info->dev))
+ continue;
+
+ pdev = to_pci_dev(info->dev);
+ if (!pci_ats_enabled(pdev))
continue;
sid = info->bus << 8 | info->devfn;
- qdep = pci_ats_queue_depth(info->dev);
+ qdep = pci_ats_queue_depth(pdev);
qi_flush_dev_iotlb(info->iommu, sid, qdep, addr, mask);
}
spin_unlock_irqrestore(&device_domain_lock, flags);
}
static void iommu_flush_iotlb_psi(struct intel_iommu *iommu, u16 did,
- unsigned long pfn, unsigned int pages, int map)
+ unsigned long pfn, unsigned int pages, int ih, int map)
{
unsigned int mask = ilog2(__roundup_pow_of_two(pages));
uint64_t addr = (uint64_t)pfn << VTD_PAGE_SHIFT;
BUG_ON(pages == 0);
+ if (ih)
+ ih = 1 << 6;
/*
* Fallback to domain selective flush if no PSI support or the size is
* too big.
@@ -1158,7 +1356,7 @@ static void iommu_flush_iotlb_psi(struct intel_iommu *iommu, u16 did,
iommu->flush.flush_iotlb(iommu, did, 0, 0,
DMA_TLB_DSI_FLUSH);
else
- iommu->flush.flush_iotlb(iommu, did, addr, mask,
+ iommu->flush.flush_iotlb(iommu, did, addr | ih, mask,
DMA_TLB_PSI_FLUSH);
/*
@@ -1261,10 +1459,6 @@ static int iommu_init_domains(struct intel_iommu *iommu)
return 0;
}
-
-static void domain_exit(struct dmar_domain *domain);
-static void vm_domain_exit(struct dmar_domain *domain);
-
static void free_dmar_iommu(struct intel_iommu *iommu)
{
struct dmar_domain *domain;
@@ -1273,18 +1467,21 @@ static void free_dmar_iommu(struct intel_iommu *iommu)
if ((iommu->domains) && (iommu->domain_ids)) {
for_each_set_bit(i, iommu->domain_ids, cap_ndoms(iommu->cap)) {
+ /*
+ * Domain id 0 is reserved for invalid translation
+ * if hardware supports caching mode.
+ */
+ if (cap_caching_mode(iommu->cap) && i == 0)
+ continue;
+
domain = iommu->domains[i];
clear_bit(i, iommu->domain_ids);
spin_lock_irqsave(&domain->iommu_lock, flags);
count = --domain->iommu_count;
spin_unlock_irqrestore(&domain->iommu_lock, flags);
- if (count == 0) {
- if (domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE)
- vm_domain_exit(domain);
- else
- domain_exit(domain);
- }
+ if (count == 0)
+ domain_exit(domain);
}
}
@@ -1298,21 +1495,14 @@ static void free_dmar_iommu(struct intel_iommu *iommu)
g_iommus[iommu->seq_id] = NULL;
- /* if all iommus are freed, free g_iommus */
- for (i = 0; i < g_num_of_iommus; i++) {
- if (g_iommus[i])
- break;
- }
-
- if (i == g_num_of_iommus)
- kfree(g_iommus);
-
/* free context mapping */
free_context_table(iommu);
}
-static struct dmar_domain *alloc_domain(void)
+static struct dmar_domain *alloc_domain(bool vm)
{
+ /* domain id for virtual machine, it won't be set in context */
+ static atomic_t vm_domid = ATOMIC_INIT(0);
struct dmar_domain *domain;
domain = alloc_domain_mem();
@@ -1320,8 +1510,15 @@ static struct dmar_domain *alloc_domain(void)
return NULL;
domain->nid = -1;
+ domain->iommu_count = 0;
memset(domain->iommu_bmp, 0, sizeof(domain->iommu_bmp));
domain->flags = 0;
+ spin_lock_init(&domain->iommu_lock);
+ INIT_LIST_HEAD(&domain->devices);
+ if (vm) {
+ domain->id = atomic_inc_return(&vm_domid);
+ domain->flags = DOMAIN_FLAG_VIRTUAL_MACHINE;
+ }
return domain;
}
@@ -1345,6 +1542,7 @@ static int iommu_attach_domain(struct dmar_domain *domain,
}
domain->id = num;
+ domain->iommu_count++;
set_bit(num, iommu->domain_ids);
set_bit(iommu->seq_id, domain->iommu_bmp);
iommu->domains[num] = domain;
@@ -1358,22 +1556,16 @@ static void iommu_detach_domain(struct dmar_domain *domain,
{
unsigned long flags;
int num, ndomains;
- int found = 0;
spin_lock_irqsave(&iommu->lock, flags);
ndomains = cap_ndoms(iommu->cap);
for_each_set_bit(num, iommu->domain_ids, ndomains) {
if (iommu->domains[num] == domain) {
- found = 1;
+ clear_bit(num, iommu->domain_ids);
+ iommu->domains[num] = NULL;
break;
}
}
-
- if (found) {
- clear_bit(num, iommu->domain_ids);
- clear_bit(iommu->seq_id, domain->iommu_bmp);
- iommu->domains[num] = NULL;
- }
spin_unlock_irqrestore(&iommu->lock, flags);
}
@@ -1445,8 +1637,6 @@ static int domain_init(struct dmar_domain *domain, int guest_width)
unsigned long sagaw;
init_iova_domain(&domain->iovad, DMA_32BIT_PFN);
- spin_lock_init(&domain->iommu_lock);
-
domain_reserve_special_ranges(domain);
/* calculate AGAW */
@@ -1465,7 +1655,6 @@ static int domain_init(struct dmar_domain *domain, int guest_width)
return -ENODEV;
}
domain->agaw = agaw;
- INIT_LIST_HEAD(&domain->devices);
if (ecap_coherent(iommu->ecap))
domain->iommu_coherency = 1;
@@ -1477,8 +1666,11 @@ static int domain_init(struct dmar_domain *domain, int guest_width)
else
domain->iommu_snooping = 0;
- domain->iommu_superpage = fls(cap_super_page_val(iommu->cap));
- domain->iommu_count = 1;
+ if (intel_iommu_superpage)
+ domain->iommu_superpage = fls(cap_super_page_val(iommu->cap));
+ else
+ domain->iommu_superpage = 0;
+
domain->nid = iommu->node;
/* always allocate the top pgd */
@@ -1493,6 +1685,7 @@ static void domain_exit(struct dmar_domain *domain)
{
struct dmar_drhd_unit *drhd;
struct intel_iommu *iommu;
+ struct page *freelist = NULL;
/* Domain 0 is reserved, so dont process it */
if (!domain)
@@ -1502,29 +1695,33 @@ static void domain_exit(struct dmar_domain *domain)
if (!intel_iommu_strict)
flush_unmaps_timeout(0);
+ /* remove associated devices */
domain_remove_dev_info(domain);
+
/* destroy iovas */
put_iova_domain(&domain->iovad);
- /* clear ptes */
- dma_pte_clear_range(domain, 0, DOMAIN_MAX_PFN(domain->gaw));
-
- /* free page tables */
- dma_pte_free_pagetable(domain, 0, DOMAIN_MAX_PFN(domain->gaw));
+ freelist = domain_unmap(domain, 0, DOMAIN_MAX_PFN(domain->gaw));
+ /* clear attached or cached domains */
+ rcu_read_lock();
for_each_active_iommu(iommu, drhd)
- if (test_bit(iommu->seq_id, domain->iommu_bmp))
+ if (domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE ||
+ test_bit(iommu->seq_id, domain->iommu_bmp))
iommu_detach_domain(domain, iommu);
+ rcu_read_unlock();
+
+ dma_free_pagelist(freelist);
free_domain_mem(domain);
}
-static int domain_context_mapping_one(struct dmar_domain *domain, int segment,
- u8 bus, u8 devfn, int translation)
+static int domain_context_mapping_one(struct dmar_domain *domain,
+ struct intel_iommu *iommu,
+ u8 bus, u8 devfn, int translation)
{
struct context_entry *context;
unsigned long flags;
- struct intel_iommu *iommu;
struct dma_pte *pgd;
unsigned long num;
unsigned long ndomains;
@@ -1539,10 +1736,6 @@ static int domain_context_mapping_one(struct dmar_domain *domain, int segment,
BUG_ON(translation != CONTEXT_TT_PASS_THROUGH &&
translation != CONTEXT_TT_MULTI_LEVEL);
- iommu = device_to_iommu(segment, bus, devfn);
- if (!iommu)
- return -ENODEV;
-
context = device_to_context_entry(iommu, bus, devfn);
if (!context)
return -ENOMEM;
@@ -1600,7 +1793,7 @@ static int domain_context_mapping_one(struct dmar_domain *domain, int segment,
context_set_domain_id(context, id);
if (translation != CONTEXT_TT_PASS_THROUGH) {
- info = iommu_support_dev_iotlb(domain, segment, bus, devfn);
+ info = iommu_support_dev_iotlb(domain, iommu, bus, devfn);
translation = info ? CONTEXT_TT_DEV_IOTLB :
CONTEXT_TT_MULTI_LEVEL;
}
@@ -1650,27 +1843,32 @@ static int domain_context_mapping_one(struct dmar_domain *domain, int segment,
}
static int
-domain_context_mapping(struct dmar_domain *domain, struct pci_dev *pdev,
- int translation)
+domain_context_mapping(struct dmar_domain *domain, struct device *dev,
+ int translation)
{
int ret;
- struct pci_dev *tmp, *parent;
+ struct pci_dev *pdev, *tmp, *parent;
+ struct intel_iommu *iommu;
+ u8 bus, devfn;
+
+ iommu = device_to_iommu(dev, &bus, &devfn);
+ if (!iommu)
+ return -ENODEV;
- ret = domain_context_mapping_one(domain, pci_domain_nr(pdev->bus),
- pdev->bus->number, pdev->devfn,
+ ret = domain_context_mapping_one(domain, iommu, bus, devfn,
translation);
- if (ret)
+ if (ret || !dev_is_pci(dev))
return ret;
/* dependent device mapping */
+ pdev = to_pci_dev(dev);
tmp = pci_find_upstream_pcie_bridge(pdev);
if (!tmp)
return 0;
/* Secondary interface's bus number and devfn 0 */
parent = pdev->bus->self;
while (parent != tmp) {
- ret = domain_context_mapping_one(domain,
- pci_domain_nr(parent->bus),
+ ret = domain_context_mapping_one(domain, iommu,
parent->bus->number,
parent->devfn, translation);
if (ret)
@@ -1678,33 +1876,33 @@ domain_context_mapping(struct dmar_domain *domain, struct pci_dev *pdev,
parent = parent->bus->self;
}
if (pci_is_pcie(tmp)) /* this is a PCIe-to-PCI bridge */
- return domain_context_mapping_one(domain,
- pci_domain_nr(tmp->subordinate),
+ return domain_context_mapping_one(domain, iommu,
tmp->subordinate->number, 0,
translation);
else /* this is a legacy PCI bridge */
- return domain_context_mapping_one(domain,
- pci_domain_nr(tmp->bus),
+ return domain_context_mapping_one(domain, iommu,
tmp->bus->number,
tmp->devfn,
translation);
}
-static int domain_context_mapped(struct pci_dev *pdev)
+static int domain_context_mapped(struct device *dev)
{
int ret;
- struct pci_dev *tmp, *parent;
+ struct pci_dev *pdev, *tmp, *parent;
struct intel_iommu *iommu;
+ u8 bus, devfn;
- iommu = device_to_iommu(pci_domain_nr(pdev->bus), pdev->bus->number,
- pdev->devfn);
+ iommu = device_to_iommu(dev, &bus, &devfn);
if (!iommu)
return -ENODEV;
- ret = device_context_mapped(iommu, pdev->bus->number, pdev->devfn);
- if (!ret)
+ ret = device_context_mapped(iommu, bus, devfn);
+ if (!ret || !dev_is_pci(dev))
return ret;
+
/* dependent device mapping */
+ pdev = to_pci_dev(dev);
tmp = pci_find_upstream_pcie_bridge(pdev);
if (!tmp)
return ret;
@@ -1800,7 +1998,7 @@ static int __domain_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
if (!pte) {
largepage_lvl = hardware_largepage_caps(domain, iov_pfn, phys_pfn, sg_res);
- first_pte = pte = pfn_to_dma_pte(domain, iov_pfn, largepage_lvl);
+ first_pte = pte = pfn_to_dma_pte(domain, iov_pfn, &largepage_lvl);
if (!pte)
return -ENOMEM;
/* It is large page*/
@@ -1899,14 +2097,13 @@ static inline void unlink_domain_info(struct device_domain_info *info)
list_del(&info->link);
list_del(&info->global);
if (info->dev)
- info->dev->dev.archdata.iommu = NULL;
+ info->dev->archdata.iommu = NULL;
}
static void domain_remove_dev_info(struct dmar_domain *domain)
{
struct device_domain_info *info;
- unsigned long flags;
- struct intel_iommu *iommu;
+ unsigned long flags, flags2;
spin_lock_irqsave(&device_domain_lock, flags);
while (!list_empty(&domain->devices)) {
@@ -1916,10 +2113,23 @@ static void domain_remove_dev_info(struct dmar_domain *domain)
spin_unlock_irqrestore(&device_domain_lock, flags);
iommu_disable_dev_iotlb(info);
- iommu = device_to_iommu(info->segment, info->bus, info->devfn);
- iommu_detach_dev(iommu, info->bus, info->devfn);
- free_devinfo_mem(info);
+ iommu_detach_dev(info->iommu, info->bus, info->devfn);
+ if (domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE) {
+ iommu_detach_dependent_devices(info->iommu, info->dev);
+ /* clear this iommu in iommu_bmp, update iommu count
+ * and capabilities
+ */
+ spin_lock_irqsave(&domain->iommu_lock, flags2);
+ if (test_and_clear_bit(info->iommu->seq_id,
+ domain->iommu_bmp)) {
+ domain->iommu_count--;
+ domain_update_iommu_cap(domain);
+ }
+ spin_unlock_irqrestore(&domain->iommu_lock, flags2);
+ }
+
+ free_devinfo_mem(info);
spin_lock_irqsave(&device_domain_lock, flags);
}
spin_unlock_irqrestore(&device_domain_lock, flags);
@@ -1927,155 +2137,153 @@ static void domain_remove_dev_info(struct dmar_domain *domain)
/*
* find_domain
- * Note: we use struct pci_dev->dev.archdata.iommu stores the info
+ * Note: we use struct device->archdata.iommu stores the info
*/
-static struct dmar_domain *
-find_domain(struct pci_dev *pdev)
+static struct dmar_domain *find_domain(struct device *dev)
{
struct device_domain_info *info;
/* No lock here, assumes no domain exit in normal case */
- info = pdev->dev.archdata.iommu;
+ info = dev->archdata.iommu;
if (info)
return info->domain;
return NULL;
}
+static inline struct device_domain_info *
+dmar_search_domain_by_dev_info(int segment, int bus, int devfn)
+{
+ struct device_domain_info *info;
+
+ list_for_each_entry(info, &device_domain_list, global)
+ if (info->iommu->segment == segment && info->bus == bus &&
+ info->devfn == devfn)
+ return info;
+
+ return NULL;
+}
+
+static struct dmar_domain *dmar_insert_dev_info(struct intel_iommu *iommu,
+ int bus, int devfn,
+ struct device *dev,
+ struct dmar_domain *domain)
+{
+ struct dmar_domain *found = NULL;
+ struct device_domain_info *info;
+ unsigned long flags;
+
+ info = alloc_devinfo_mem();
+ if (!info)
+ return NULL;
+
+ info->bus = bus;
+ info->devfn = devfn;
+ info->dev = dev;
+ info->domain = domain;
+ info->iommu = iommu;
+ if (!dev)
+ domain->flags |= DOMAIN_FLAG_P2P_MULTIPLE_DEVICES;
+
+ spin_lock_irqsave(&device_domain_lock, flags);
+ if (dev)
+ found = find_domain(dev);
+ else {
+ struct device_domain_info *info2;
+ info2 = dmar_search_domain_by_dev_info(iommu->segment, bus, devfn);
+ if (info2)
+ found = info2->domain;
+ }
+ if (found) {
+ spin_unlock_irqrestore(&device_domain_lock, flags);
+ free_devinfo_mem(info);
+ /* Caller must free the original domain */
+ return found;
+ }
+
+ list_add(&info->link, &domain->devices);
+ list_add(&info->global, &device_domain_list);
+ if (dev)
+ dev->archdata.iommu = info;
+ spin_unlock_irqrestore(&device_domain_lock, flags);
+
+ return domain;
+}
+
/* domain is initialized */
-static struct dmar_domain *get_domain_for_dev(struct pci_dev *pdev, int gaw)
+static struct dmar_domain *get_domain_for_dev(struct device *dev, int gaw)
{
- struct dmar_domain *domain, *found = NULL;
- struct intel_iommu *iommu;
- struct dmar_drhd_unit *drhd;
- struct device_domain_info *info, *tmp;
- struct pci_dev *dev_tmp;
+ struct dmar_domain *domain, *free = NULL;
+ struct intel_iommu *iommu = NULL;
+ struct device_domain_info *info;
+ struct pci_dev *dev_tmp = NULL;
unsigned long flags;
- int bus = 0, devfn = 0;
- int segment;
- int ret;
+ u8 bus, devfn, bridge_bus, bridge_devfn;
- domain = find_domain(pdev);
+ domain = find_domain(dev);
if (domain)
return domain;
- segment = pci_domain_nr(pdev->bus);
+ if (dev_is_pci(dev)) {
+ struct pci_dev *pdev = to_pci_dev(dev);
+ u16 segment;
- dev_tmp = pci_find_upstream_pcie_bridge(pdev);
- if (dev_tmp) {
- if (pci_is_pcie(dev_tmp)) {
- bus = dev_tmp->subordinate->number;
- devfn = 0;
- } else {
- bus = dev_tmp->bus->number;
- devfn = dev_tmp->devfn;
- }
- spin_lock_irqsave(&device_domain_lock, flags);
- list_for_each_entry(info, &device_domain_list, global) {
- if (info->segment == segment &&
- info->bus == bus && info->devfn == devfn) {
- found = info->domain;
- break;
+ segment = pci_domain_nr(pdev->bus);
+ dev_tmp = pci_find_upstream_pcie_bridge(pdev);
+ if (dev_tmp) {
+ if (pci_is_pcie(dev_tmp)) {
+ bridge_bus = dev_tmp->subordinate->number;
+ bridge_devfn = 0;
+ } else {
+ bridge_bus = dev_tmp->bus->number;
+ bridge_devfn = dev_tmp->devfn;
}
- }
- spin_unlock_irqrestore(&device_domain_lock, flags);
- /* pcie-pci bridge already has a domain, uses it */
- if (found) {
- domain = found;
- goto found_domain;
+ spin_lock_irqsave(&device_domain_lock, flags);
+ info = dmar_search_domain_by_dev_info(segment,
+ bridge_bus,
+ bridge_devfn);
+ if (info) {
+ iommu = info->iommu;
+ domain = info->domain;
+ }
+ spin_unlock_irqrestore(&device_domain_lock, flags);
+ /* pcie-pci bridge already has a domain, uses it */
+ if (info)
+ goto found_domain;
}
}
- domain = alloc_domain();
- if (!domain)
+ iommu = device_to_iommu(dev, &bus, &devfn);
+ if (!iommu)
goto error;
- /* Allocate new domain for the device */
- drhd = dmar_find_matched_drhd_unit(pdev);
- if (!drhd) {
- printk(KERN_ERR "IOMMU: can't find DMAR for device %s\n",
- pci_name(pdev));
- free_domain_mem(domain);
- return NULL;
- }
- iommu = drhd->iommu;
-
- ret = iommu_attach_domain(domain, iommu);
- if (ret) {
+ /* Allocate and initialize new domain for the device */
+ domain = alloc_domain(false);
+ if (!domain)
+ goto error;
+ if (iommu_attach_domain(domain, iommu)) {
free_domain_mem(domain);
+ domain = NULL;
goto error;
}
-
- if (domain_init(domain, gaw)) {
- domain_exit(domain);
+ free = domain;
+ if (domain_init(domain, gaw))
goto error;
- }
/* register pcie-to-pci device */
if (dev_tmp) {
- info = alloc_devinfo_mem();
- if (!info) {
- domain_exit(domain);
+ domain = dmar_insert_dev_info(iommu, bridge_bus, bridge_devfn,
+ NULL, domain);
+ if (!domain)
goto error;
- }
- info->segment = segment;
- info->bus = bus;
- info->devfn = devfn;
- info->dev = NULL;
- info->domain = domain;
- /* This domain is shared by devices under p2p bridge */
- domain->flags |= DOMAIN_FLAG_P2P_MULTIPLE_DEVICES;
-
- /* pcie-to-pci bridge already has a domain, uses it */
- found = NULL;
- spin_lock_irqsave(&device_domain_lock, flags);
- list_for_each_entry(tmp, &device_domain_list, global) {
- if (tmp->segment == segment &&
- tmp->bus == bus && tmp->devfn == devfn) {
- found = tmp->domain;
- break;
- }
- }
- if (found) {
- spin_unlock_irqrestore(&device_domain_lock, flags);
- free_devinfo_mem(info);
- domain_exit(domain);
- domain = found;
- } else {
- list_add(&info->link, &domain->devices);
- list_add(&info->global, &device_domain_list);
- spin_unlock_irqrestore(&device_domain_lock, flags);
- }
}
found_domain:
- info = alloc_devinfo_mem();
- if (!info)
- goto error;
- info->segment = segment;
- info->bus = pdev->bus->number;
- info->devfn = pdev->devfn;
- info->dev = pdev;
- info->domain = domain;
- spin_lock_irqsave(&device_domain_lock, flags);
- /* somebody is fast */
- found = find_domain(pdev);
- if (found != NULL) {
- spin_unlock_irqrestore(&device_domain_lock, flags);
- if (found != domain) {
- domain_exit(domain);
- domain = found;
- }
- free_devinfo_mem(info);
- return domain;
- }
- list_add(&info->link, &domain->devices);
- list_add(&info->global, &device_domain_list);
- pdev->dev.archdata.iommu = info;
- spin_unlock_irqrestore(&device_domain_lock, flags);
- return domain;
+ domain = dmar_insert_dev_info(iommu, bus, devfn, dev, domain);
error:
- /* recheck it here, maybe others set it */
- return find_domain(pdev);
+ if (free != domain)
+ domain_exit(free);
+
+ return domain;
}
static int iommu_identity_mapping;
@@ -2109,14 +2317,14 @@ static int iommu_domain_identity_map(struct dmar_domain *domain,
DMA_PTE_READ|DMA_PTE_WRITE);
}
-static int iommu_prepare_identity_map(struct pci_dev *pdev,
+static int iommu_prepare_identity_map(struct device *dev,
unsigned long long start,
unsigned long long end)
{
struct dmar_domain *domain;
int ret;
- domain = get_domain_for_dev(pdev, DEFAULT_DOMAIN_ADDRESS_WIDTH);
+ domain = get_domain_for_dev(dev, DEFAULT_DOMAIN_ADDRESS_WIDTH);
if (!domain)
return -ENOMEM;
@@ -2126,13 +2334,13 @@ static int iommu_prepare_identity_map(struct pci_dev *pdev,
up to start with in si_domain */
if (domain == si_domain && hw_pass_through) {
printk("Ignoring identity map for HW passthrough device %s [0x%Lx - 0x%Lx]\n",
- pci_name(pdev), start, end);
+ dev_name(dev), start, end);
return 0;
}
printk(KERN_INFO
"IOMMU: Setting identity map for device %s [0x%Lx - 0x%Lx]\n",
- pci_name(pdev), start, end);
+ dev_name(dev), start, end);
if (end < start) {
WARN(1, "Your BIOS is broken; RMRR ends before it starts!\n"
@@ -2160,7 +2368,7 @@ static int iommu_prepare_identity_map(struct pci_dev *pdev,
goto error;
/* context entry init */
- ret = domain_context_mapping(domain, pdev, CONTEXT_TT_MULTI_LEVEL);
+ ret = domain_context_mapping(domain, dev, CONTEXT_TT_MULTI_LEVEL);
if (ret)
goto error;
@@ -2172,12 +2380,12 @@ static int iommu_prepare_identity_map(struct pci_dev *pdev,
}
static inline int iommu_prepare_rmrr_dev(struct dmar_rmrr_unit *rmrr,
- struct pci_dev *pdev)
+ struct device *dev)
{
- if (pdev->dev.archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO)
+ if (dev->archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO)
return 0;
- return iommu_prepare_identity_map(pdev, rmrr->base_address,
- rmrr->end_address);
+ return iommu_prepare_identity_map(dev, rmrr->base_address,
+ rmrr->end_address);
}
#ifdef CONFIG_INTEL_IOMMU_FLOPPY_WA
@@ -2191,7 +2399,7 @@ static inline void iommu_prepare_isa(void)
return;
printk(KERN_INFO "IOMMU: Prepare 0-16MiB unity mapping for LPC\n");
- ret = iommu_prepare_identity_map(pdev, 0, 16*1024*1024 - 1);
+ ret = iommu_prepare_identity_map(&pdev->dev, 0, 16*1024*1024 - 1);
if (ret)
printk(KERN_ERR "IOMMU: Failed to create 0-16MiB identity map; "
@@ -2213,10 +2421,12 @@ static int __init si_domain_init(int hw)
struct intel_iommu *iommu;
int nid, ret = 0;
- si_domain = alloc_domain();
+ si_domain = alloc_domain(false);
if (!si_domain)
return -EFAULT;
+ si_domain->flags = DOMAIN_FLAG_STATIC_IDENTITY;
+
for_each_active_iommu(iommu, drhd) {
ret = iommu_attach_domain(si_domain, iommu);
if (ret) {
@@ -2230,7 +2440,6 @@ static int __init si_domain_init(int hw)
return -EFAULT;
}
- si_domain->flags = DOMAIN_FLAG_STATIC_IDENTITY;
pr_debug("IOMMU: identity mapping domain is domain %d\n",
si_domain->id);
@@ -2252,16 +2461,14 @@ static int __init si_domain_init(int hw)
return 0;
}
-static void domain_remove_one_dev_info(struct dmar_domain *domain,
- struct pci_dev *pdev);
-static int identity_mapping(struct pci_dev *pdev)
+static int identity_mapping(struct device *dev)
{
struct device_domain_info *info;
if (likely(!iommu_identity_mapping))
return 0;
- info = pdev->dev.archdata.iommu;
+ info = dev->archdata.iommu;
if (info && info != DUMMY_DEVICE_DOMAIN_INFO)
return (info->domain == si_domain);
@@ -2269,111 +2476,112 @@ static int identity_mapping(struct pci_dev *pdev)
}
static int domain_add_dev_info(struct dmar_domain *domain,
- struct pci_dev *pdev,
- int translation)
+ struct device *dev, int translation)
{
- struct device_domain_info *info;
- unsigned long flags;
+ struct dmar_domain *ndomain;
+ struct intel_iommu *iommu;
+ u8 bus, devfn;
int ret;
- info = alloc_devinfo_mem();
- if (!info)
- return -ENOMEM;
-
- info->segment = pci_domain_nr(pdev->bus);
- info->bus = pdev->bus->number;
- info->devfn = pdev->devfn;
- info->dev = pdev;
- info->domain = domain;
+ iommu = device_to_iommu(dev, &bus, &devfn);
+ if (!iommu)
+ return -ENODEV;
- spin_lock_irqsave(&device_domain_lock, flags);
- list_add(&info->link, &domain->devices);
- list_add(&info->global, &device_domain_list);
- pdev->dev.archdata.iommu = info;
- spin_unlock_irqrestore(&device_domain_lock, flags);
+ ndomain = dmar_insert_dev_info(iommu, bus, devfn, dev, domain);
+ if (ndomain != domain)
+ return -EBUSY;
- ret = domain_context_mapping(domain, pdev, translation);
+ ret = domain_context_mapping(domain, dev, translation);
if (ret) {
- spin_lock_irqsave(&device_domain_lock, flags);
- unlink_domain_info(info);
- spin_unlock_irqrestore(&device_domain_lock, flags);
- free_devinfo_mem(info);
+ domain_remove_one_dev_info(domain, dev);
return ret;
}
return 0;
}
-static bool device_has_rmrr(struct pci_dev *dev)
+static bool device_has_rmrr(struct device *dev)
{
struct dmar_rmrr_unit *rmrr;
+ struct device *tmp;
int i;
+ rcu_read_lock();
for_each_rmrr_units(rmrr) {
- for (i = 0; i < rmrr->devices_cnt; i++) {
- /*
- * Return TRUE if this RMRR contains the device that
- * is passed in.
- */
- if (rmrr->devices[i] == dev)
+ /*
+ * Return TRUE if this RMRR contains the device that
+ * is passed in.
+ */
+ for_each_active_dev_scope(rmrr->devices,
+ rmrr->devices_cnt, i, tmp)
+ if (tmp == dev) {
+ rcu_read_unlock();
return true;
- }
+ }
}
+ rcu_read_unlock();
return false;
}
-static int iommu_should_identity_map(struct pci_dev *pdev, int startup)
+static int iommu_should_identity_map(struct device *dev, int startup)
{
- /*
- * We want to prevent any device associated with an RMRR from
- * getting placed into the SI Domain. This is done because
- * problems exist when devices are moved in and out of domains
- * and their respective RMRR info is lost. We exempt USB devices
- * from this process due to their usage of RMRRs that are known
- * to not be needed after BIOS hand-off to OS.
- */
- if (device_has_rmrr(pdev) &&
- (pdev->class >> 8) != PCI_CLASS_SERIAL_USB)
- return 0;
+ if (dev_is_pci(dev)) {
+ struct pci_dev *pdev = to_pci_dev(dev);
- if ((iommu_identity_mapping & IDENTMAP_AZALIA) && IS_AZALIA(pdev))
- return 1;
+ /*
+ * We want to prevent any device associated with an RMRR from
+ * getting placed into the SI Domain. This is done because
+ * problems exist when devices are moved in and out of domains
+ * and their respective RMRR info is lost. We exempt USB devices
+ * from this process due to their usage of RMRRs that are known
+ * to not be needed after BIOS hand-off to OS.
+ */
+ if (device_has_rmrr(dev) &&
+ (pdev->class >> 8) != PCI_CLASS_SERIAL_USB)
+ return 0;
- if ((iommu_identity_mapping & IDENTMAP_GFX) && IS_GFX_DEVICE(pdev))
- return 1;
+ if ((iommu_identity_mapping & IDENTMAP_AZALIA) && IS_AZALIA(pdev))
+ return 1;
- if (!(iommu_identity_mapping & IDENTMAP_ALL))
- return 0;
+ if ((iommu_identity_mapping & IDENTMAP_GFX) && IS_GFX_DEVICE(pdev))
+ return 1;
- /*
- * We want to start off with all devices in the 1:1 domain, and
- * take them out later if we find they can't access all of memory.
- *
- * However, we can't do this for PCI devices behind bridges,
- * because all PCI devices behind the same bridge will end up
- * with the same source-id on their transactions.
- *
- * Practically speaking, we can't change things around for these
- * devices at run-time, because we can't be sure there'll be no
- * DMA transactions in flight for any of their siblings.
- *
- * So PCI devices (unless they're on the root bus) as well as
- * their parent PCI-PCI or PCIe-PCI bridges must be left _out_ of
- * the 1:1 domain, just in _case_ one of their siblings turns out
- * not to be able to map all of memory.
- */
- if (!pci_is_pcie(pdev)) {
- if (!pci_is_root_bus(pdev->bus))
+ if (!(iommu_identity_mapping & IDENTMAP_ALL))
return 0;
- if (pdev->class >> 8 == PCI_CLASS_BRIDGE_PCI)
+
+ /*
+ * We want to start off with all devices in the 1:1 domain, and
+ * take them out later if we find they can't access all of memory.
+ *
+ * However, we can't do this for PCI devices behind bridges,
+ * because all PCI devices behind the same bridge will end up
+ * with the same source-id on their transactions.
+ *
+ * Practically speaking, we can't change things around for these
+ * devices at run-time, because we can't be sure there'll be no
+ * DMA transactions in flight for any of their siblings.
+ *
+ * So PCI devices (unless they're on the root bus) as well as
+ * their parent PCI-PCI or PCIe-PCI bridges must be left _out_ of
+ * the 1:1 domain, just in _case_ one of their siblings turns out
+ * not to be able to map all of memory.
+ */
+ if (!pci_is_pcie(pdev)) {
+ if (!pci_is_root_bus(pdev->bus))
+ return 0;
+ if (pdev->class >> 8 == PCI_CLASS_BRIDGE_PCI)
+ return 0;
+ } else if (pci_pcie_type(pdev) == PCI_EXP_TYPE_PCI_BRIDGE)
return 0;
- } else if (pci_pcie_type(pdev) == PCI_EXP_TYPE_PCI_BRIDGE)
- return 0;
+ } else {
+ if (device_has_rmrr(dev))
+ return 0;
+ }
- /*
+ /*
* At boot time, we don't yet know if devices will be 64-bit capable.
- * Assume that they will -- if they turn out not to be, then we can
+ * Assume that they will — if they turn out not to be, then we can
* take them out of the 1:1 domain later.
*/
if (!startup) {
@@ -2381,42 +2589,77 @@ static int iommu_should_identity_map(struct pci_dev *pdev, int startup)
* If the device's dma_mask is less than the system's memory
* size then this is not a candidate for identity mapping.
*/
- u64 dma_mask = pdev->dma_mask;
+ u64 dma_mask = *dev->dma_mask;
- if (pdev->dev.coherent_dma_mask &&
- pdev->dev.coherent_dma_mask < dma_mask)
- dma_mask = pdev->dev.coherent_dma_mask;
+ if (dev->coherent_dma_mask &&
+ dev->coherent_dma_mask < dma_mask)
+ dma_mask = dev->coherent_dma_mask;
- return dma_mask >= dma_get_required_mask(&pdev->dev);
+ return dma_mask >= dma_get_required_mask(dev);
}
return 1;
}
+static int __init dev_prepare_static_identity_mapping(struct device *dev, int hw)
+{
+ int ret;
+
+ if (!iommu_should_identity_map(dev, 1))
+ return 0;
+
+ ret = domain_add_dev_info(si_domain, dev,
+ hw ? CONTEXT_TT_PASS_THROUGH :
+ CONTEXT_TT_MULTI_LEVEL);
+ if (!ret)
+ pr_info("IOMMU: %s identity mapping for device %s\n",
+ hw ? "hardware" : "software", dev_name(dev));
+ else if (ret == -ENODEV)
+ /* device not associated with an iommu */
+ ret = 0;
+
+ return ret;
+}
+
+
static int __init iommu_prepare_static_identity_mapping(int hw)
{
struct pci_dev *pdev = NULL;
- int ret;
+ struct dmar_drhd_unit *drhd;
+ struct intel_iommu *iommu;
+ struct device *dev;
+ int i;
+ int ret = 0;
ret = si_domain_init(hw);
if (ret)
return -EFAULT;
for_each_pci_dev(pdev) {
- if (iommu_should_identity_map(pdev, 1)) {
- ret = domain_add_dev_info(si_domain, pdev,
- hw ? CONTEXT_TT_PASS_THROUGH :
- CONTEXT_TT_MULTI_LEVEL);
- if (ret) {
- /* device not associated with an iommu */
- if (ret == -ENODEV)
- continue;
- return ret;
+ ret = dev_prepare_static_identity_mapping(&pdev->dev, hw);
+ if (ret)
+ return ret;
+ }
+
+ for_each_active_iommu(iommu, drhd)
+ for_each_active_dev_scope(drhd->devices, drhd->devices_cnt, i, dev) {
+ struct acpi_device_physical_node *pn;
+ struct acpi_device *adev;
+
+ if (dev->bus != &acpi_bus_type)
+ continue;
+
+ adev= to_acpi_device(dev);
+ mutex_lock(&adev->physical_node_lock);
+ list_for_each_entry(pn, &adev->physical_node_list, node) {
+ ret = dev_prepare_static_identity_mapping(pn->dev, hw);
+ if (ret)
+ break;
}
- pr_info("IOMMU: %s identity mapping for device %s\n",
- hw ? "hardware" : "software", pci_name(pdev));
+ mutex_unlock(&adev->physical_node_lock);
+ if (ret)
+ return ret;
}
- }
return 0;
}
@@ -2425,7 +2668,7 @@ static int __init init_dmars(void)
{
struct dmar_drhd_unit *drhd;
struct dmar_rmrr_unit *rmrr;
- struct pci_dev *pdev;
+ struct device *dev;
struct intel_iommu *iommu;
int i, ret;
@@ -2461,7 +2704,7 @@ static int __init init_dmars(void)
sizeof(struct deferred_flush_tables), GFP_KERNEL);
if (!deferred_flush) {
ret = -ENOMEM;
- goto error;
+ goto free_g_iommus;
}
for_each_active_iommu(iommu, drhd) {
@@ -2469,7 +2712,7 @@ static int __init init_dmars(void)
ret = iommu_init_domains(iommu);
if (ret)
- goto error;
+ goto free_iommu;
/*
* TBD:
@@ -2479,7 +2722,7 @@ static int __init init_dmars(void)
ret = iommu_alloc_root_entry(iommu);
if (ret) {
printk(KERN_ERR "IOMMU: allocate root entry failed\n");
- goto error;
+ goto free_iommu;
}
if (!ecap_pass_through(iommu->ecap))
hw_pass_through = 0;
@@ -2548,7 +2791,7 @@ static int __init init_dmars(void)
ret = iommu_prepare_static_identity_mapping(hw_pass_through);
if (ret) {
printk(KERN_CRIT "Failed to setup IOMMU pass-through\n");
- goto error;
+ goto free_iommu;
}
}
/*
@@ -2567,15 +2810,10 @@ static int __init init_dmars(void)
*/
printk(KERN_INFO "IOMMU: Setting RMRR:\n");
for_each_rmrr_units(rmrr) {
- for (i = 0; i < rmrr->devices_cnt; i++) {
- pdev = rmrr->devices[i];
- /*
- * some BIOS lists non-exist devices in DMAR
- * table.
- */
- if (!pdev)
- continue;
- ret = iommu_prepare_rmrr_dev(rmrr, pdev);
+ /* some BIOS lists non-exist devices in DMAR table. */
+ for_each_active_dev_scope(rmrr->devices, rmrr->devices_cnt,
+ i, dev) {
+ ret = iommu_prepare_rmrr_dev(rmrr, dev);
if (ret)
printk(KERN_ERR
"IOMMU: mapping reserved region failed\n");
@@ -2606,7 +2844,7 @@ static int __init init_dmars(void)
ret = dmar_set_interrupt(iommu);
if (ret)
- goto error;
+ goto free_iommu;
iommu_set_root_entry(iommu);
@@ -2615,17 +2853,20 @@ static int __init init_dmars(void)
ret = iommu_enable_translation(iommu);
if (ret)
- goto error;
+ goto free_iommu;
iommu_disable_protect_mem_regions(iommu);
}
return 0;
-error:
+
+free_iommu:
for_each_active_iommu(iommu, drhd)
free_dmar_iommu(iommu);
kfree(deferred_flush);
+free_g_iommus:
kfree(g_iommus);
+error:
return ret;
}
@@ -2634,7 +2875,6 @@ static struct iova *intel_alloc_iova(struct device *dev,
struct dmar_domain *domain,
unsigned long nrpages, uint64_t dma_mask)
{
- struct pci_dev *pdev = to_pci_dev(dev);
struct iova *iova = NULL;
/* Restrict dma_mask to the width that the iommu can handle */
@@ -2654,34 +2894,31 @@ static struct iova *intel_alloc_iova(struct device *dev,
iova = alloc_iova(&domain->iovad, nrpages, IOVA_PFN(dma_mask), 1);
if (unlikely(!iova)) {
printk(KERN_ERR "Allocating %ld-page iova for %s failed",
- nrpages, pci_name(pdev));
+ nrpages, dev_name(dev));
return NULL;
}
return iova;
}
-static struct dmar_domain *__get_valid_domain_for_dev(struct pci_dev *pdev)
+static struct dmar_domain *__get_valid_domain_for_dev(struct device *dev)
{
struct dmar_domain *domain;
int ret;
- domain = get_domain_for_dev(pdev,
- DEFAULT_DOMAIN_ADDRESS_WIDTH);
+ domain = get_domain_for_dev(dev, DEFAULT_DOMAIN_ADDRESS_WIDTH);
if (!domain) {
- printk(KERN_ERR
- "Allocating domain for %s failed", pci_name(pdev));
+ printk(KERN_ERR "Allocating domain for %s failed",
+ dev_name(dev));
return NULL;
}
/* make sure context mapping is ok */
- if (unlikely(!domain_context_mapped(pdev))) {
- ret = domain_context_mapping(domain, pdev,
- CONTEXT_TT_MULTI_LEVEL);
+ if (unlikely(!domain_context_mapped(dev))) {
+ ret = domain_context_mapping(domain, dev, CONTEXT_TT_MULTI_LEVEL);
if (ret) {
- printk(KERN_ERR
- "Domain context map for %s failed",
- pci_name(pdev));
+ printk(KERN_ERR "Domain context map for %s failed",
+ dev_name(dev));
return NULL;
}
}
@@ -2689,51 +2926,46 @@ static struct dmar_domain *__get_valid_domain_for_dev(struct pci_dev *pdev)
return domain;
}
-static inline struct dmar_domain *get_valid_domain_for_dev(struct pci_dev *dev)
+static inline struct dmar_domain *get_valid_domain_for_dev(struct device *dev)
{
struct device_domain_info *info;
/* No lock here, assumes no domain exit in normal case */
- info = dev->dev.archdata.iommu;
+ info = dev->archdata.iommu;
if (likely(info))
return info->domain;
return __get_valid_domain_for_dev(dev);
}
-static int iommu_dummy(struct pci_dev *pdev)
+static int iommu_dummy(struct device *dev)
{
- return pdev->dev.archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO;
+ return dev->archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO;
}
-/* Check if the pdev needs to go through non-identity map and unmap process.*/
+/* Check if the dev needs to go through non-identity map and unmap process.*/
static int iommu_no_mapping(struct device *dev)
{
- struct pci_dev *pdev;
int found;
- if (unlikely(!dev_is_pci(dev)))
- return 1;
-
- pdev = to_pci_dev(dev);
- if (iommu_dummy(pdev))
+ if (iommu_dummy(dev))
return 1;
if (!iommu_identity_mapping)
return 0;
- found = identity_mapping(pdev);
+ found = identity_mapping(dev);
if (found) {
- if (iommu_should_identity_map(pdev, 0))
+ if (iommu_should_identity_map(dev, 0))
return 1;
else {
/*
* 32 bit DMA is removed from si_domain and fall back
* to non-identity mapping.
*/
- domain_remove_one_dev_info(si_domain, pdev);
+ domain_remove_one_dev_info(si_domain, dev);
printk(KERN_INFO "32bit %s uses non-identity mapping\n",
- pci_name(pdev));
+ dev_name(dev));
return 0;
}
} else {
@@ -2741,15 +2973,15 @@ static int iommu_no_mapping(struct device *dev)
* In case of a detached 64 bit DMA device from vm, the device
* is put into si_domain for identity mapping.
*/
- if (iommu_should_identity_map(pdev, 0)) {
+ if (iommu_should_identity_map(dev, 0)) {
int ret;
- ret = domain_add_dev_info(si_domain, pdev,
+ ret = domain_add_dev_info(si_domain, dev,
hw_pass_through ?
CONTEXT_TT_PASS_THROUGH :
CONTEXT_TT_MULTI_LEVEL);
if (!ret) {
printk(KERN_INFO "64bit %s uses identity mapping\n",
- pci_name(pdev));
+ dev_name(dev));
return 1;
}
}
@@ -2758,10 +2990,9 @@ static int iommu_no_mapping(struct device *dev)
return 0;
}
-static dma_addr_t __intel_map_single(struct device *hwdev, phys_addr_t paddr,
+static dma_addr_t __intel_map_single(struct device *dev, phys_addr_t paddr,
size_t size, int dir, u64 dma_mask)
{
- struct pci_dev *pdev = to_pci_dev(hwdev);
struct dmar_domain *domain;
phys_addr_t start_paddr;
struct iova *iova;
@@ -2772,17 +3003,17 @@ static dma_addr_t __intel_map_single(struct device *hwdev, phys_addr_t paddr,
BUG_ON(dir == DMA_NONE);
- if (iommu_no_mapping(hwdev))
+ if (iommu_no_mapping(dev))
return paddr;
- domain = get_valid_domain_for_dev(pdev);
+ domain = get_valid_domain_for_dev(dev);
if (!domain)
return 0;
iommu = domain_get_iommu(domain);
size = aligned_nrpages(paddr, size);
- iova = intel_alloc_iova(hwdev, domain, dma_to_mm_pfn(size), dma_mask);
+ iova = intel_alloc_iova(dev, domain, dma_to_mm_pfn(size), dma_mask);
if (!iova)
goto error;
@@ -2808,7 +3039,7 @@ static dma_addr_t __intel_map_single(struct device *hwdev, phys_addr_t paddr,
/* it's a non-present to present mapping. Only flush if caching mode */
if (cap_caching_mode(iommu->cap))
- iommu_flush_iotlb_psi(iommu, domain->id, mm_to_dma_pfn(iova->pfn_lo), size, 1);
+ iommu_flush_iotlb_psi(iommu, domain->id, mm_to_dma_pfn(iova->pfn_lo), size, 0, 1);
else
iommu_flush_write_buffer(iommu);
@@ -2820,7 +3051,7 @@ error:
if (iova)
__free_iova(&domain->iovad, iova);
printk(KERN_ERR"Device %s request: %zx@%llx dir %d --- failed\n",
- pci_name(pdev), size, (unsigned long long)paddr, dir);
+ dev_name(dev), size, (unsigned long long)paddr, dir);
return 0;
}
@@ -2830,7 +3061,7 @@ static dma_addr_t intel_map_page(struct device *dev, struct page *page,
struct dma_attrs *attrs)
{
return __intel_map_single(dev, page_to_phys(page) + offset, size,
- dir, to_pci_dev(dev)->dma_mask);
+ dir, *dev->dma_mask);
}
static void flush_unmaps(void)
@@ -2860,13 +3091,16 @@ static void flush_unmaps(void)
/* On real hardware multiple invalidations are expensive */
if (cap_caching_mode(iommu->cap))
iommu_flush_iotlb_psi(iommu, domain->id,
- iova->pfn_lo, iova->pfn_hi - iova->pfn_lo + 1, 0);
+ iova->pfn_lo, iova->pfn_hi - iova->pfn_lo + 1,
+ !deferred_flush[i].freelist[j], 0);
else {
mask = ilog2(mm_to_dma_pfn(iova->pfn_hi - iova->pfn_lo + 1));
iommu_flush_dev_iotlb(deferred_flush[i].domain[j],
(uint64_t)iova->pfn_lo << PAGE_SHIFT, mask);
}
__free_iova(&deferred_flush[i].domain[j]->iovad, iova);
+ if (deferred_flush[i].freelist[j])
+ dma_free_pagelist(deferred_flush[i].freelist[j]);
}
deferred_flush[i].next = 0;
}
@@ -2883,7 +3117,7 @@ static void flush_unmaps_timeout(unsigned long data)
spin_unlock_irqrestore(&async_umap_flush_lock, flags);
}
-static void add_unmap(struct dmar_domain *dom, struct iova *iova)
+static void add_unmap(struct dmar_domain *dom, struct iova *iova, struct page *freelist)
{
unsigned long flags;
int next, iommu_id;
@@ -2899,6 +3133,7 @@ static void add_unmap(struct dmar_domain *dom, struct iova *iova)
next = deferred_flush[iommu_id].next;
deferred_flush[iommu_id].domain[next] = dom;
deferred_flush[iommu_id].iova[next] = iova;
+ deferred_flush[iommu_id].freelist[next] = freelist;
deferred_flush[iommu_id].next++;
if (!timer_on) {
@@ -2913,16 +3148,16 @@ static void intel_unmap_page(struct device *dev, dma_addr_t dev_addr,
size_t size, enum dma_data_direction dir,
struct dma_attrs *attrs)
{
- struct pci_dev *pdev = to_pci_dev(dev);
struct dmar_domain *domain;
unsigned long start_pfn, last_pfn;
struct iova *iova;
struct intel_iommu *iommu;
+ struct page *freelist;
if (iommu_no_mapping(dev))
return;
- domain = find_domain(pdev);
+ domain = find_domain(dev);
BUG_ON(!domain);
iommu = domain_get_iommu(domain);
@@ -2936,21 +3171,18 @@ static void intel_unmap_page(struct device *dev, dma_addr_t dev_addr,
last_pfn = mm_to_dma_pfn(iova->pfn_hi + 1) - 1;
pr_debug("Device %s unmapping: pfn %lx-%lx\n",
- pci_name(pdev), start_pfn, last_pfn);
-
- /* clear the whole page */
- dma_pte_clear_range(domain, start_pfn, last_pfn);
+ dev_name(dev), start_pfn, last_pfn);
- /* free page tables */
- dma_pte_free_pagetable(domain, start_pfn, last_pfn);
+ freelist = domain_unmap(domain, start_pfn, last_pfn);
if (intel_iommu_strict) {
iommu_flush_iotlb_psi(iommu, domain->id, start_pfn,
- last_pfn - start_pfn + 1, 0);
+ last_pfn - start_pfn + 1, !freelist, 0);
/* free iova */
__free_iova(&domain->iovad, iova);
+ dma_free_pagelist(freelist);
} else {
- add_unmap(domain, iova);
+ add_unmap(domain, iova, freelist);
/*
* queue up the release of the unmap to save the 1/6th of the
* cpu used up by the iotlb flush operation...
@@ -2958,65 +3190,81 @@ static void intel_unmap_page(struct device *dev, dma_addr_t dev_addr,
}
}
-static void *intel_alloc_coherent(struct device *hwdev, size_t size,
+static void *intel_alloc_coherent(struct device *dev, size_t size,
dma_addr_t *dma_handle, gfp_t flags,
struct dma_attrs *attrs)
{
- void *vaddr;
+ struct page *page = NULL;
int order;
size = PAGE_ALIGN(size);
order = get_order(size);
- if (!iommu_no_mapping(hwdev))
+ if (!iommu_no_mapping(dev))
flags &= ~(GFP_DMA | GFP_DMA32);
- else if (hwdev->coherent_dma_mask < dma_get_required_mask(hwdev)) {
- if (hwdev->coherent_dma_mask < DMA_BIT_MASK(32))
+ else if (dev->coherent_dma_mask < dma_get_required_mask(dev)) {
+ if (dev->coherent_dma_mask < DMA_BIT_MASK(32))
flags |= GFP_DMA;
else
flags |= GFP_DMA32;
}
- vaddr = (void *)__get_free_pages(flags, order);
- if (!vaddr)
+ if (flags & __GFP_WAIT) {
+ unsigned int count = size >> PAGE_SHIFT;
+
+ page = dma_alloc_from_contiguous(dev, count, order);
+ if (page && iommu_no_mapping(dev) &&
+ page_to_phys(page) + size > dev->coherent_dma_mask) {
+ dma_release_from_contiguous(dev, page, count);
+ page = NULL;
+ }
+ }
+
+ if (!page)
+ page = alloc_pages(flags, order);
+ if (!page)
return NULL;
- memset(vaddr, 0, size);
+ memset(page_address(page), 0, size);
- *dma_handle = __intel_map_single(hwdev, virt_to_bus(vaddr), size,
+ *dma_handle = __intel_map_single(dev, page_to_phys(page), size,
DMA_BIDIRECTIONAL,
- hwdev->coherent_dma_mask);
+ dev->coherent_dma_mask);
if (*dma_handle)
- return vaddr;
- free_pages((unsigned long)vaddr, order);
+ return page_address(page);
+ if (!dma_release_from_contiguous(dev, page, size >> PAGE_SHIFT))
+ __free_pages(page, order);
+
return NULL;
}
-static void intel_free_coherent(struct device *hwdev, size_t size, void *vaddr,
+static void intel_free_coherent(struct device *dev, size_t size, void *vaddr,
dma_addr_t dma_handle, struct dma_attrs *attrs)
{
int order;
+ struct page *page = virt_to_page(vaddr);
size = PAGE_ALIGN(size);
order = get_order(size);
- intel_unmap_page(hwdev, dma_handle, size, DMA_BIDIRECTIONAL, NULL);
- free_pages((unsigned long)vaddr, order);
+ intel_unmap_page(dev, dma_handle, size, DMA_BIDIRECTIONAL, NULL);
+ if (!dma_release_from_contiguous(dev, page, size >> PAGE_SHIFT))
+ __free_pages(page, order);
}
-static void intel_unmap_sg(struct device *hwdev, struct scatterlist *sglist,
+static void intel_unmap_sg(struct device *dev, struct scatterlist *sglist,
int nelems, enum dma_data_direction dir,
struct dma_attrs *attrs)
{
- struct pci_dev *pdev = to_pci_dev(hwdev);
struct dmar_domain *domain;
unsigned long start_pfn, last_pfn;
struct iova *iova;
struct intel_iommu *iommu;
+ struct page *freelist;
- if (iommu_no_mapping(hwdev))
+ if (iommu_no_mapping(dev))
return;
- domain = find_domain(pdev);
+ domain = find_domain(dev);
BUG_ON(!domain);
iommu = domain_get_iommu(domain);
@@ -3029,19 +3277,16 @@ static void intel_unmap_sg(struct device *hwdev, struct scatterlist *sglist,
start_pfn = mm_to_dma_pfn(iova->pfn_lo);
last_pfn = mm_to_dma_pfn(iova->pfn_hi + 1) - 1;
- /* clear the whole page */
- dma_pte_clear_range(domain, start_pfn, last_pfn);
-
- /* free page tables */
- dma_pte_free_pagetable(domain, start_pfn, last_pfn);
+ freelist = domain_unmap(domain, start_pfn, last_pfn);
if (intel_iommu_strict) {
iommu_flush_iotlb_psi(iommu, domain->id, start_pfn,
- last_pfn - start_pfn + 1, 0);
+ last_pfn - start_pfn + 1, !freelist, 0);
/* free iova */
__free_iova(&domain->iovad, iova);
+ dma_free_pagelist(freelist);
} else {
- add_unmap(domain, iova);
+ add_unmap(domain, iova, freelist);
/*
* queue up the release of the unmap to save the 1/6th of the
* cpu used up by the iotlb flush operation...
@@ -3063,11 +3308,10 @@ static int intel_nontranslate_map_sg(struct device *hddev,
return nelems;
}
-static int intel_map_sg(struct device *hwdev, struct scatterlist *sglist, int nelems,
+static int intel_map_sg(struct device *dev, struct scatterlist *sglist, int nelems,
enum dma_data_direction dir, struct dma_attrs *attrs)
{
int i;
- struct pci_dev *pdev = to_pci_dev(hwdev);
struct dmar_domain *domain;
size_t size = 0;
int prot = 0;
@@ -3078,10 +3322,10 @@ static int intel_map_sg(struct device *hwdev, struct scatterlist *sglist, int ne
struct intel_iommu *iommu;
BUG_ON(dir == DMA_NONE);
- if (iommu_no_mapping(hwdev))
- return intel_nontranslate_map_sg(hwdev, sglist, nelems, dir);
+ if (iommu_no_mapping(dev))
+ return intel_nontranslate_map_sg(dev, sglist, nelems, dir);
- domain = get_valid_domain_for_dev(pdev);
+ domain = get_valid_domain_for_dev(dev);
if (!domain)
return 0;
@@ -3090,8 +3334,8 @@ static int intel_map_sg(struct device *hwdev, struct scatterlist *sglist, int ne
for_each_sg(sglist, sg, nelems, i)
size += aligned_nrpages(sg->offset, sg->length);
- iova = intel_alloc_iova(hwdev, domain, dma_to_mm_pfn(size),
- pdev->dma_mask);
+ iova = intel_alloc_iova(dev, domain, dma_to_mm_pfn(size),
+ *dev->dma_mask);
if (!iova) {
sglist->dma_length = 0;
return 0;
@@ -3124,7 +3368,7 @@ static int intel_map_sg(struct device *hwdev, struct scatterlist *sglist, int ne
/* it's a non-present to present mapping. Only flush if caching mode */
if (cap_caching_mode(iommu->cap))
- iommu_flush_iotlb_psi(iommu, domain->id, start_vpfn, size, 1);
+ iommu_flush_iotlb_psi(iommu, domain->id, start_vpfn, size, 0, 1);
else
iommu_flush_write_buffer(iommu);
@@ -3259,29 +3503,28 @@ DECLARE_PCI_FIXUP_ENABLE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_SNB, quir
static void __init init_no_remapping_devices(void)
{
struct dmar_drhd_unit *drhd;
+ struct device *dev;
+ int i;
for_each_drhd_unit(drhd) {
if (!drhd->include_all) {
- int i;
- for (i = 0; i < drhd->devices_cnt; i++)
- if (drhd->devices[i] != NULL)
- break;
- /* ignore DMAR unit if no pci devices exist */
+ for_each_active_dev_scope(drhd->devices,
+ drhd->devices_cnt, i, dev)
+ break;
+ /* ignore DMAR unit if no devices exist */
if (i == drhd->devices_cnt)
drhd->ignored = 1;
}
}
for_each_active_drhd_unit(drhd) {
- int i;
if (drhd->include_all)
continue;
- for (i = 0; i < drhd->devices_cnt; i++)
- if (drhd->devices[i] &&
- !IS_GFX_DEVICE(drhd->devices[i]))
+ for_each_active_dev_scope(drhd->devices,
+ drhd->devices_cnt, i, dev)
+ if (!dev_is_pci(dev) || !IS_GFX_DEVICE(to_pci_dev(dev)))
break;
-
if (i < drhd->devices_cnt)
continue;
@@ -3291,11 +3534,9 @@ static void __init init_no_remapping_devices(void)
intel_iommu_gfx_mapped = 1;
} else {
drhd->ignored = 1;
- for (i = 0; i < drhd->devices_cnt; i++) {
- if (!drhd->devices[i])
- continue;
- drhd->devices[i]->dev.archdata.iommu = DUMMY_DEVICE_DOMAIN_INFO;
- }
+ for_each_active_dev_scope(drhd->devices,
+ drhd->devices_cnt, i, dev)
+ dev->archdata.iommu = DUMMY_DEVICE_DOMAIN_INFO;
}
}
}
@@ -3438,13 +3679,6 @@ static void __init init_iommu_pm_ops(void)
static inline void init_iommu_pm_ops(void) {}
#endif /* CONFIG_PM */
-LIST_HEAD(dmar_rmrr_units);
-
-static void __init dmar_register_rmrr_unit(struct dmar_rmrr_unit *rmrr)
-{
- list_add(&rmrr->list, &dmar_rmrr_units);
-}
-
int __init dmar_parse_one_rmrr(struct acpi_dmar_header *header)
{
@@ -3459,25 +3693,19 @@ int __init dmar_parse_one_rmrr(struct acpi_dmar_header *header)
rmrr = (struct acpi_dmar_reserved_memory *)header;
rmrru->base_address = rmrr->base_address;
rmrru->end_address = rmrr->end_address;
+ rmrru->devices = dmar_alloc_dev_scope((void *)(rmrr + 1),
+ ((void *)rmrr) + rmrr->header.length,
+ &rmrru->devices_cnt);
+ if (rmrru->devices_cnt && rmrru->devices == NULL) {
+ kfree(rmrru);
+ return -ENOMEM;
+ }
- dmar_register_rmrr_unit(rmrru);
- return 0;
-}
-
-static int __init
-rmrr_parse_dev(struct dmar_rmrr_unit *rmrru)
-{
- struct acpi_dmar_reserved_memory *rmrr;
+ list_add(&rmrru->list, &dmar_rmrr_units);
- rmrr = (struct acpi_dmar_reserved_memory *) rmrru->hdr;
- return dmar_parse_dev_scope((void *)(rmrr + 1),
- ((void *)rmrr) + rmrr->header.length,
- &rmrru->devices_cnt, &rmrru->devices,
- rmrr->segment);
+ return 0;
}
-static LIST_HEAD(dmar_atsr_units);
-
int __init dmar_parse_one_atsr(struct acpi_dmar_header *hdr)
{
struct acpi_dmar_atsr *atsr;
@@ -3490,26 +3718,21 @@ int __init dmar_parse_one_atsr(struct acpi_dmar_header *hdr)
atsru->hdr = hdr;
atsru->include_all = atsr->flags & 0x1;
+ if (!atsru->include_all) {
+ atsru->devices = dmar_alloc_dev_scope((void *)(atsr + 1),
+ (void *)atsr + atsr->header.length,
+ &atsru->devices_cnt);
+ if (atsru->devices_cnt && atsru->devices == NULL) {
+ kfree(atsru);
+ return -ENOMEM;
+ }
+ }
- list_add(&atsru->list, &dmar_atsr_units);
+ list_add_rcu(&atsru->list, &dmar_atsr_units);
return 0;
}
-static int __init atsr_parse_dev(struct dmar_atsr_unit *atsru)
-{
- struct acpi_dmar_atsr *atsr;
-
- if (atsru->include_all)
- return 0;
-
- atsr = container_of(atsru->hdr, struct acpi_dmar_atsr, header);
- return dmar_parse_dev_scope((void *)(atsr + 1),
- (void *)atsr + atsr->header.length,
- &atsru->devices_cnt, &atsru->devices,
- atsr->segment);
-}
-
static void intel_iommu_free_atsr(struct dmar_atsr_unit *atsru)
{
dmar_free_dev_scope(&atsru->devices, &atsru->devices_cnt);
@@ -3535,62 +3758,94 @@ static void intel_iommu_free_dmars(void)
int dmar_find_matched_atsr_unit(struct pci_dev *dev)
{
- int i;
+ int i, ret = 1;
struct pci_bus *bus;
+ struct pci_dev *bridge = NULL;
+ struct device *tmp;
struct acpi_dmar_atsr *atsr;
struct dmar_atsr_unit *atsru;
dev = pci_physfn(dev);
-
- list_for_each_entry(atsru, &dmar_atsr_units, list) {
- atsr = container_of(atsru->hdr, struct acpi_dmar_atsr, header);
- if (atsr->segment == pci_domain_nr(dev->bus))
- goto found;
- }
-
- return 0;
-
-found:
for (bus = dev->bus; bus; bus = bus->parent) {
- struct pci_dev *bridge = bus->self;
-
+ bridge = bus->self;
if (!bridge || !pci_is_pcie(bridge) ||
pci_pcie_type(bridge) == PCI_EXP_TYPE_PCI_BRIDGE)
return 0;
-
- if (pci_pcie_type(bridge) == PCI_EXP_TYPE_ROOT_PORT) {
- for (i = 0; i < atsru->devices_cnt; i++)
- if (atsru->devices[i] == bridge)
- return 1;
+ if (pci_pcie_type(bridge) == PCI_EXP_TYPE_ROOT_PORT)
break;
- }
}
+ if (!bridge)
+ return 0;
- if (atsru->include_all)
- return 1;
+ rcu_read_lock();
+ list_for_each_entry_rcu(atsru, &dmar_atsr_units, list) {
+ atsr = container_of(atsru->hdr, struct acpi_dmar_atsr, header);
+ if (atsr->segment != pci_domain_nr(dev->bus))
+ continue;
- return 0;
+ for_each_dev_scope(atsru->devices, atsru->devices_cnt, i, tmp)
+ if (tmp == &bridge->dev)
+ goto out;
+
+ if (atsru->include_all)
+ goto out;
+ }
+ ret = 0;
+out:
+ rcu_read_unlock();
+
+ return ret;
}
-int __init dmar_parse_rmrr_atsr_dev(void)
+int dmar_iommu_notify_scope_dev(struct dmar_pci_notify_info *info)
{
- struct dmar_rmrr_unit *rmrr;
- struct dmar_atsr_unit *atsr;
int ret = 0;
+ struct dmar_rmrr_unit *rmrru;
+ struct dmar_atsr_unit *atsru;
+ struct acpi_dmar_atsr *atsr;
+ struct acpi_dmar_reserved_memory *rmrr;
- list_for_each_entry(rmrr, &dmar_rmrr_units, list) {
- ret = rmrr_parse_dev(rmrr);
- if (ret)
- return ret;
+ if (!intel_iommu_enabled && system_state != SYSTEM_BOOTING)
+ return 0;
+
+ list_for_each_entry(rmrru, &dmar_rmrr_units, list) {
+ rmrr = container_of(rmrru->hdr,
+ struct acpi_dmar_reserved_memory, header);
+ if (info->event == BUS_NOTIFY_ADD_DEVICE) {
+ ret = dmar_insert_dev_scope(info, (void *)(rmrr + 1),
+ ((void *)rmrr) + rmrr->header.length,
+ rmrr->segment, rmrru->devices,
+ rmrru->devices_cnt);
+ if(ret < 0)
+ return ret;
+ } else if (info->event == BUS_NOTIFY_DEL_DEVICE) {
+ dmar_remove_dev_scope(info, rmrr->segment,
+ rmrru->devices, rmrru->devices_cnt);
+ }
}
- list_for_each_entry(atsr, &dmar_atsr_units, list) {
- ret = atsr_parse_dev(atsr);
- if (ret)
- return ret;
+ list_for_each_entry(atsru, &dmar_atsr_units, list) {
+ if (atsru->include_all)
+ continue;
+
+ atsr = container_of(atsru->hdr, struct acpi_dmar_atsr, header);
+ if (info->event == BUS_NOTIFY_ADD_DEVICE) {
+ ret = dmar_insert_dev_scope(info, (void *)(atsr + 1),
+ (void *)atsr + atsr->header.length,
+ atsr->segment, atsru->devices,
+ atsru->devices_cnt);
+ if (ret > 0)
+ break;
+ else if(ret < 0)
+ return ret;
+ } else if (info->event == BUS_NOTIFY_DEL_DEVICE) {
+ if (dmar_remove_dev_scope(info, atsr->segment,
+ atsru->devices, atsru->devices_cnt))
+ break;
+ }
}
- return ret;
+ return 0;
}
/*
@@ -3603,24 +3858,26 @@ static int device_notifier(struct notifier_block *nb,
unsigned long action, void *data)
{
struct device *dev = data;
- struct pci_dev *pdev = to_pci_dev(dev);
struct dmar_domain *domain;
- if (iommu_no_mapping(dev))
+ if (iommu_dummy(dev))
return 0;
- domain = find_domain(pdev);
- if (!domain)
+ if (action != BUS_NOTIFY_UNBOUND_DRIVER &&
+ action != BUS_NOTIFY_DEL_DEVICE)
return 0;
- if (action == BUS_NOTIFY_UNBOUND_DRIVER && !iommu_pass_through) {
- domain_remove_one_dev_info(domain, pdev);
+ domain = find_domain(dev);
+ if (!domain)
+ return 0;
- if (!(domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE) &&
- !(domain->flags & DOMAIN_FLAG_STATIC_IDENTITY) &&
- list_empty(&domain->devices))
- domain_exit(domain);
- }
+ down_read(&dmar_global_lock);
+ domain_remove_one_dev_info(domain, dev);
+ if (!(domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE) &&
+ !(domain->flags & DOMAIN_FLAG_STATIC_IDENTITY) &&
+ list_empty(&domain->devices))
+ domain_exit(domain);
+ up_read(&dmar_global_lock);
return 0;
}
@@ -3629,6 +3886,75 @@ static struct notifier_block device_nb = {
.notifier_call = device_notifier,
};
+static int intel_iommu_memory_notifier(struct notifier_block *nb,
+ unsigned long val, void *v)
+{
+ struct memory_notify *mhp = v;
+ unsigned long long start, end;
+ unsigned long start_vpfn, last_vpfn;
+
+ switch (val) {
+ case MEM_GOING_ONLINE:
+ start = mhp->start_pfn << PAGE_SHIFT;
+ end = ((mhp->start_pfn + mhp->nr_pages) << PAGE_SHIFT) - 1;
+ if (iommu_domain_identity_map(si_domain, start, end)) {
+ pr_warn("dmar: failed to build identity map for [%llx-%llx]\n",
+ start, end);
+ return NOTIFY_BAD;
+ }
+ break;
+
+ case MEM_OFFLINE:
+ case MEM_CANCEL_ONLINE:
+ start_vpfn = mm_to_dma_pfn(mhp->start_pfn);
+ last_vpfn = mm_to_dma_pfn(mhp->start_pfn + mhp->nr_pages - 1);
+ while (start_vpfn <= last_vpfn) {
+ struct iova *iova;
+ struct dmar_drhd_unit *drhd;
+ struct intel_iommu *iommu;
+ struct page *freelist;
+
+ iova = find_iova(&si_domain->iovad, start_vpfn);
+ if (iova == NULL) {
+ pr_debug("dmar: failed get IOVA for PFN %lx\n",
+ start_vpfn);
+ break;
+ }
+
+ iova = split_and_remove_iova(&si_domain->iovad, iova,
+ start_vpfn, last_vpfn);
+ if (iova == NULL) {
+ pr_warn("dmar: failed to split IOVA PFN [%lx-%lx]\n",
+ start_vpfn, last_vpfn);
+ return NOTIFY_BAD;
+ }
+
+ freelist = domain_unmap(si_domain, iova->pfn_lo,
+ iova->pfn_hi);
+
+ rcu_read_lock();
+ for_each_active_iommu(iommu, drhd)
+ iommu_flush_iotlb_psi(iommu, si_domain->id,
+ iova->pfn_lo,
+ iova->pfn_hi - iova->pfn_lo + 1,
+ !freelist, 0);
+ rcu_read_unlock();
+ dma_free_pagelist(freelist);
+
+ start_vpfn = iova->pfn_hi + 1;
+ free_iova_mem(iova);
+ }
+ break;
+ }
+
+ return NOTIFY_OK;
+}
+
+static struct notifier_block intel_iommu_memory_nb = {
+ .notifier_call = intel_iommu_memory_notifier,
+ .priority = 0
+};
+
int __init intel_iommu_init(void)
{
int ret = -ENODEV;
@@ -3638,6 +3964,13 @@ int __init intel_iommu_init(void)
/* VT-d is required for a TXT/tboot launch, so enforce that */
force_on = tboot_force_iommu();
+ if (iommu_init_mempool()) {
+ if (force_on)
+ panic("tboot: Failed to initialize iommu memory\n");
+ return -ENOMEM;
+ }
+
+ down_write(&dmar_global_lock);
if (dmar_table_init()) {
if (force_on)
panic("tboot: Failed to initialize DMAR table\n");
@@ -3660,12 +3993,6 @@ int __init intel_iommu_init(void)
if (no_iommu || dmar_disabled)
goto out_free_dmar;
- if (iommu_init_mempool()) {
- if (force_on)
- panic("tboot: Failed to initialize iommu memory\n");
- goto out_free_dmar;
- }
-
if (list_empty(&dmar_rmrr_units))
printk(KERN_INFO "DMAR: No RMRR found\n");
@@ -3675,7 +4002,7 @@ int __init intel_iommu_init(void)
if (dmar_init_reserved_ranges()) {
if (force_on)
panic("tboot: Failed to reserve iommu ranges\n");
- goto out_free_mempool;
+ goto out_free_reserved_range;
}
init_no_remapping_devices();
@@ -3687,6 +4014,7 @@ int __init intel_iommu_init(void)
printk(KERN_ERR "IOMMU: dmar init failed\n");
goto out_free_reserved_range;
}
+ up_write(&dmar_global_lock);
printk(KERN_INFO
"PCI-DMA: Intel(R) Virtualization Technology for Directed I/O\n");
@@ -3699,8 +4027,9 @@ int __init intel_iommu_init(void)
init_iommu_pm_ops();
bus_set_iommu(&pci_bus_type, &intel_iommu_ops);
-
bus_register_notifier(&pci_bus_type, &device_nb);
+ if (si_domain && !hw_pass_through)
+ register_memory_notifier(&intel_iommu_memory_nb);
intel_iommu_enabled = 1;
@@ -3708,21 +4037,23 @@ int __init intel_iommu_init(void)
out_free_reserved_range:
put_iova_domain(&reserved_iova_list);
-out_free_mempool:
- iommu_exit_mempool();
out_free_dmar:
intel_iommu_free_dmars();
+ up_write(&dmar_global_lock);
+ iommu_exit_mempool();
return ret;
}
static void iommu_detach_dependent_devices(struct intel_iommu *iommu,
- struct pci_dev *pdev)
+ struct device *dev)
{
- struct pci_dev *tmp, *parent;
+ struct pci_dev *tmp, *parent, *pdev;
- if (!iommu || !pdev)
+ if (!iommu || !dev || !dev_is_pci(dev))
return;
+ pdev = to_pci_dev(dev);
+
/* dependent device detach */
tmp = pci_find_upstream_pcie_bridge(pdev);
/* Secondary interface's bus number and devfn 0 */
@@ -3743,29 +4074,28 @@ static void iommu_detach_dependent_devices(struct intel_iommu *iommu,
}
static void domain_remove_one_dev_info(struct dmar_domain *domain,
- struct pci_dev *pdev)
+ struct device *dev)
{
struct device_domain_info *info, *tmp;
struct intel_iommu *iommu;
unsigned long flags;
int found = 0;
+ u8 bus, devfn;
- iommu = device_to_iommu(pci_domain_nr(pdev->bus), pdev->bus->number,
- pdev->devfn);
+ iommu = device_to_iommu(dev, &bus, &devfn);
if (!iommu)
return;
spin_lock_irqsave(&device_domain_lock, flags);
list_for_each_entry_safe(info, tmp, &domain->devices, link) {
- if (info->segment == pci_domain_nr(pdev->bus) &&
- info->bus == pdev->bus->number &&
- info->devfn == pdev->devfn) {
+ if (info->iommu == iommu && info->bus == bus &&
+ info->devfn == devfn) {
unlink_domain_info(info);
spin_unlock_irqrestore(&device_domain_lock, flags);
iommu_disable_dev_iotlb(info);
iommu_detach_dev(iommu, info->bus, info->devfn);
- iommu_detach_dependent_devices(iommu, pdev);
+ iommu_detach_dependent_devices(iommu, dev);
free_devinfo_mem(info);
spin_lock_irqsave(&device_domain_lock, flags);
@@ -3780,8 +4110,7 @@ static void domain_remove_one_dev_info(struct dmar_domain *domain,
* owned by this domain, clear this iommu in iommu_bmp
* update iommu count and coherency
*/
- if (iommu == device_to_iommu(info->segment, info->bus,
- info->devfn))
+ if (info->iommu == iommu)
found = 1;
}
@@ -3805,67 +4134,11 @@ static void domain_remove_one_dev_info(struct dmar_domain *domain,
}
}
-static void vm_domain_remove_all_dev_info(struct dmar_domain *domain)
-{
- struct device_domain_info *info;
- struct intel_iommu *iommu;
- unsigned long flags1, flags2;
-
- spin_lock_irqsave(&device_domain_lock, flags1);
- while (!list_empty(&domain->devices)) {
- info = list_entry(domain->devices.next,
- struct device_domain_info, link);
- unlink_domain_info(info);
- spin_unlock_irqrestore(&device_domain_lock, flags1);
-
- iommu_disable_dev_iotlb(info);
- iommu = device_to_iommu(info->segment, info->bus, info->devfn);
- iommu_detach_dev(iommu, info->bus, info->devfn);
- iommu_detach_dependent_devices(iommu, info->dev);
-
- /* clear this iommu in iommu_bmp, update iommu count
- * and capabilities
- */
- spin_lock_irqsave(&domain->iommu_lock, flags2);
- if (test_and_clear_bit(iommu->seq_id,
- domain->iommu_bmp)) {
- domain->iommu_count--;
- domain_update_iommu_cap(domain);
- }
- spin_unlock_irqrestore(&domain->iommu_lock, flags2);
-
- free_devinfo_mem(info);
- spin_lock_irqsave(&device_domain_lock, flags1);
- }
- spin_unlock_irqrestore(&device_domain_lock, flags1);
-}
-
-/* domain id for virtual machine, it won't be set in context */
-static atomic_t vm_domid = ATOMIC_INIT(0);
-
-static struct dmar_domain *iommu_alloc_vm_domain(void)
-{
- struct dmar_domain *domain;
-
- domain = alloc_domain_mem();
- if (!domain)
- return NULL;
-
- domain->id = atomic_inc_return(&vm_domid);
- domain->nid = -1;
- memset(domain->iommu_bmp, 0, sizeof(domain->iommu_bmp));
- domain->flags = DOMAIN_FLAG_VIRTUAL_MACHINE;
-
- return domain;
-}
-
static int md_domain_init(struct dmar_domain *domain, int guest_width)
{
int adjust_width;
init_iova_domain(&domain->iovad, DMA_32BIT_PFN);
- spin_lock_init(&domain->iommu_lock);
-
domain_reserve_special_ranges(domain);
/* calculate AGAW */
@@ -3873,9 +4146,6 @@ static int md_domain_init(struct dmar_domain *domain, int guest_width)
adjust_width = guestwidth_to_adjustwidth(guest_width);
domain->agaw = width_to_agaw(adjust_width);
- INIT_LIST_HEAD(&domain->devices);
-
- domain->iommu_count = 0;
domain->iommu_coherency = 0;
domain->iommu_snooping = 0;
domain->iommu_superpage = 0;
@@ -3890,53 +4160,11 @@ static int md_domain_init(struct dmar_domain *domain, int guest_width)
return 0;
}
-static void iommu_free_vm_domain(struct dmar_domain *domain)
-{
- unsigned long flags;
- struct dmar_drhd_unit *drhd;
- struct intel_iommu *iommu;
- unsigned long i;
- unsigned long ndomains;
-
- for_each_active_iommu(iommu, drhd) {
- ndomains = cap_ndoms(iommu->cap);
- for_each_set_bit(i, iommu->domain_ids, ndomains) {
- if (iommu->domains[i] == domain) {
- spin_lock_irqsave(&iommu->lock, flags);
- clear_bit(i, iommu->domain_ids);
- iommu->domains[i] = NULL;
- spin_unlock_irqrestore(&iommu->lock, flags);
- break;
- }
- }
- }
-}
-
-static void vm_domain_exit(struct dmar_domain *domain)
-{
- /* Domain 0 is reserved, so dont process it */
- if (!domain)
- return;
-
- vm_domain_remove_all_dev_info(domain);
- /* destroy iovas */
- put_iova_domain(&domain->iovad);
-
- /* clear ptes */
- dma_pte_clear_range(domain, 0, DOMAIN_MAX_PFN(domain->gaw));
-
- /* free page tables */
- dma_pte_free_pagetable(domain, 0, DOMAIN_MAX_PFN(domain->gaw));
-
- iommu_free_vm_domain(domain);
- free_domain_mem(domain);
-}
-
static int intel_iommu_domain_init(struct iommu_domain *domain)
{
struct dmar_domain *dmar_domain;
- dmar_domain = iommu_alloc_vm_domain();
+ dmar_domain = alloc_domain(true);
if (!dmar_domain) {
printk(KERN_ERR
"intel_iommu_domain_init: dmar_domain == NULL\n");
@@ -3945,7 +4173,7 @@ static int intel_iommu_domain_init(struct iommu_domain *domain)
if (md_domain_init(dmar_domain, DEFAULT_DOMAIN_ADDRESS_WIDTH)) {
printk(KERN_ERR
"intel_iommu_domain_init() failed\n");
- vm_domain_exit(dmar_domain);
+ domain_exit(dmar_domain);
return -ENOMEM;
}
domain_update_iommu_cap(dmar_domain);
@@ -3963,33 +4191,32 @@ static void intel_iommu_domain_destroy(struct iommu_domain *domain)
struct dmar_domain *dmar_domain = domain->priv;
domain->priv = NULL;
- vm_domain_exit(dmar_domain);
+ domain_exit(dmar_domain);
}
static int intel_iommu_attach_device(struct iommu_domain *domain,
struct device *dev)
{
struct dmar_domain *dmar_domain = domain->priv;
- struct pci_dev *pdev = to_pci_dev(dev);
struct intel_iommu *iommu;
int addr_width;
+ u8 bus, devfn;
- /* normally pdev is not mapped */
- if (unlikely(domain_context_mapped(pdev))) {
+ /* normally dev is not mapped */
+ if (unlikely(domain_context_mapped(dev))) {
struct dmar_domain *old_domain;
- old_domain = find_domain(pdev);
+ old_domain = find_domain(dev);
if (old_domain) {
if (dmar_domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE ||
dmar_domain->flags & DOMAIN_FLAG_STATIC_IDENTITY)
- domain_remove_one_dev_info(old_domain, pdev);
+ domain_remove_one_dev_info(old_domain, dev);
else
domain_remove_dev_info(old_domain);
}
}
- iommu = device_to_iommu(pci_domain_nr(pdev->bus), pdev->bus->number,
- pdev->devfn);
+ iommu = device_to_iommu(dev, &bus, &devfn);
if (!iommu)
return -ENODEV;
@@ -4021,16 +4248,15 @@ static int intel_iommu_attach_device(struct iommu_domain *domain,
dmar_domain->agaw--;
}
- return domain_add_dev_info(dmar_domain, pdev, CONTEXT_TT_MULTI_LEVEL);
+ return domain_add_dev_info(dmar_domain, dev, CONTEXT_TT_MULTI_LEVEL);
}
static void intel_iommu_detach_device(struct iommu_domain *domain,
struct device *dev)
{
struct dmar_domain *dmar_domain = domain->priv;
- struct pci_dev *pdev = to_pci_dev(dev);
- domain_remove_one_dev_info(dmar_domain, pdev);
+ domain_remove_one_dev_info(dmar_domain, dev);
}
static int intel_iommu_map(struct iommu_domain *domain,
@@ -4072,18 +4298,51 @@ static int intel_iommu_map(struct iommu_domain *domain,
}
static size_t intel_iommu_unmap(struct iommu_domain *domain,
- unsigned long iova, size_t size)
+ unsigned long iova, size_t size)
{
struct dmar_domain *dmar_domain = domain->priv;
- int order;
+ struct page *freelist = NULL;
+ struct intel_iommu *iommu;
+ unsigned long start_pfn, last_pfn;
+ unsigned int npages;
+ int iommu_id, num, ndomains, level = 0;
+
+ /* Cope with horrid API which requires us to unmap more than the
+ size argument if it happens to be a large-page mapping. */
+ if (!pfn_to_dma_pte(dmar_domain, iova >> VTD_PAGE_SHIFT, &level))
+ BUG();
+
+ if (size < VTD_PAGE_SIZE << level_to_offset_bits(level))
+ size = VTD_PAGE_SIZE << level_to_offset_bits(level);
+
+ start_pfn = iova >> VTD_PAGE_SHIFT;
+ last_pfn = (iova + size - 1) >> VTD_PAGE_SHIFT;
+
+ freelist = domain_unmap(dmar_domain, start_pfn, last_pfn);
+
+ npages = last_pfn - start_pfn + 1;
+
+ for_each_set_bit(iommu_id, dmar_domain->iommu_bmp, g_num_of_iommus) {
+ iommu = g_iommus[iommu_id];
+
+ /*
+ * find bit position of dmar_domain
+ */
+ ndomains = cap_ndoms(iommu->cap);
+ for_each_set_bit(num, iommu->domain_ids, ndomains) {
+ if (iommu->domains[num] == dmar_domain)
+ iommu_flush_iotlb_psi(iommu, num, start_pfn,
+ npages, !freelist, 0);
+ }
+
+ }
- order = dma_pte_clear_range(dmar_domain, iova >> VTD_PAGE_SHIFT,
- (iova + size - 1) >> VTD_PAGE_SHIFT);
+ dma_free_pagelist(freelist);
if (dmar_domain->max_addr == iova + size)
dmar_domain->max_addr = iova;
- return PAGE_SIZE << order;
+ return size;
}
static phys_addr_t intel_iommu_iova_to_phys(struct iommu_domain *domain,
@@ -4091,9 +4350,10 @@ static phys_addr_t intel_iommu_iova_to_phys(struct iommu_domain *domain,
{
struct dmar_domain *dmar_domain = domain->priv;
struct dma_pte *pte;
+ int level = 0;
u64 phys = 0;
- pte = pfn_to_dma_pte(dmar_domain, iova >> VTD_PAGE_SHIFT, 0);
+ pte = pfn_to_dma_pte(dmar_domain, iova >> VTD_PAGE_SHIFT, &level);
if (pte)
phys = dma_pte_addr(pte);
@@ -4121,9 +4381,9 @@ static int intel_iommu_add_device(struct device *dev)
struct pci_dev *bridge, *dma_pdev = NULL;
struct iommu_group *group;
int ret;
+ u8 bus, devfn;
- if (!device_to_iommu(pci_domain_nr(pdev->bus),
- pdev->bus->number, pdev->devfn))
+ if (!device_to_iommu(dev, &bus, &devfn))
return -ENODEV;
bridge = pci_find_upstream_pcie_bridge(pdev);
diff --git a/drivers/iommu/intel_irq_remapping.c b/drivers/iommu/intel_irq_remapping.c
index ef5f65dbafe..9b174893f0f 100644
--- a/drivers/iommu/intel_irq_remapping.c
+++ b/drivers/iommu/intel_irq_remapping.c
@@ -38,6 +38,17 @@ static struct ioapic_scope ir_ioapic[MAX_IO_APICS];
static struct hpet_scope ir_hpet[MAX_HPET_TBS];
static int ir_ioapic_num, ir_hpet_num;
+/*
+ * Lock ordering:
+ * ->dmar_global_lock
+ * ->irq_2_ir_lock
+ * ->qi->q_lock
+ * ->iommu->register_lock
+ * Note:
+ * intel_irq_remap_ops.{supported,prepare,enable,disable,reenable} are called
+ * in single-threaded environment with interrupt disabled, so no need to tabke
+ * the dmar_global_lock.
+ */
static DEFINE_RAW_SPINLOCK(irq_2_ir_lock);
static int __init parse_ioapics_under_ir(void);
@@ -307,12 +318,14 @@ static int set_ioapic_sid(struct irte *irte, int apic)
if (!irte)
return -1;
+ down_read(&dmar_global_lock);
for (i = 0; i < MAX_IO_APICS; i++) {
if (ir_ioapic[i].id == apic) {
sid = (ir_ioapic[i].bus << 8) | ir_ioapic[i].devfn;
break;
}
}
+ up_read(&dmar_global_lock);
if (sid == 0) {
pr_warning("Failed to set source-id of IOAPIC (%d)\n", apic);
@@ -332,12 +345,14 @@ static int set_hpet_sid(struct irte *irte, u8 id)
if (!irte)
return -1;
+ down_read(&dmar_global_lock);
for (i = 0; i < MAX_HPET_TBS; i++) {
if (ir_hpet[i].id == id) {
sid = (ir_hpet[i].bus << 8) | ir_hpet[i].devfn;
break;
}
}
+ up_read(&dmar_global_lock);
if (sid == 0) {
pr_warning("Failed to set source-id of HPET block (%d)\n", id);
@@ -794,10 +809,16 @@ static int __init parse_ioapics_under_ir(void)
static int __init ir_dev_scope_init(void)
{
+ int ret;
+
if (!irq_remapping_enabled)
return 0;
- return dmar_dev_scope_init();
+ down_write(&dmar_global_lock);
+ ret = dmar_dev_scope_init();
+ up_write(&dmar_global_lock);
+
+ return ret;
}
rootfs_initcall(ir_dev_scope_init);
@@ -878,23 +899,27 @@ static int intel_setup_ioapic_entry(int irq,
struct io_apic_irq_attr *attr)
{
int ioapic_id = mpc_ioapic_id(attr->ioapic);
- struct intel_iommu *iommu = map_ioapic_to_ir(ioapic_id);
+ struct intel_iommu *iommu;
struct IR_IO_APIC_route_entry *entry;
struct irte irte;
int index;
+ down_read(&dmar_global_lock);
+ iommu = map_ioapic_to_ir(ioapic_id);
if (!iommu) {
pr_warn("No mapping iommu for ioapic %d\n", ioapic_id);
- return -ENODEV;
- }
-
- entry = (struct IR_IO_APIC_route_entry *)route_entry;
-
- index = alloc_irte(iommu, irq, 1);
- if (index < 0) {
- pr_warn("Failed to allocate IRTE for ioapic %d\n", ioapic_id);
- return -ENOMEM;
+ index = -ENODEV;
+ } else {
+ index = alloc_irte(iommu, irq, 1);
+ if (index < 0) {
+ pr_warn("Failed to allocate IRTE for ioapic %d\n",
+ ioapic_id);
+ index = -ENOMEM;
+ }
}
+ up_read(&dmar_global_lock);
+ if (index < 0)
+ return index;
prepare_irte(&irte, vector, destination);
@@ -913,6 +938,7 @@ static int intel_setup_ioapic_entry(int irq,
irte.avail, irte.vector, irte.dest_id,
irte.sid, irte.sq, irte.svt);
+ entry = (struct IR_IO_APIC_route_entry *)route_entry;
memset(entry, 0, sizeof(*entry));
entry->index2 = (index >> 15) & 0x1;
@@ -1043,20 +1069,23 @@ static int intel_msi_alloc_irq(struct pci_dev *dev, int irq, int nvec)
struct intel_iommu *iommu;
int index;
+ down_read(&dmar_global_lock);
iommu = map_dev_to_ir(dev);
if (!iommu) {
printk(KERN_ERR
"Unable to map PCI %s to iommu\n", pci_name(dev));
- return -ENOENT;
+ index = -ENOENT;
+ } else {
+ index = alloc_irte(iommu, irq, nvec);
+ if (index < 0) {
+ printk(KERN_ERR
+ "Unable to allocate %d IRTE for PCI %s\n",
+ nvec, pci_name(dev));
+ index = -ENOSPC;
+ }
}
+ up_read(&dmar_global_lock);
- index = alloc_irte(iommu, irq, nvec);
- if (index < 0) {
- printk(KERN_ERR
- "Unable to allocate %d IRTE for PCI %s\n", nvec,
- pci_name(dev));
- return -ENOSPC;
- }
return index;
}
@@ -1064,33 +1093,40 @@ static int intel_msi_setup_irq(struct pci_dev *pdev, unsigned int irq,
int index, int sub_handle)
{
struct intel_iommu *iommu;
+ int ret = -ENOENT;
+ down_read(&dmar_global_lock);
iommu = map_dev_to_ir(pdev);
- if (!iommu)
- return -ENOENT;
- /*
- * setup the mapping between the irq and the IRTE
- * base index, the sub_handle pointing to the
- * appropriate interrupt remap table entry.
- */
- set_irte_irq(irq, iommu, index, sub_handle);
+ if (iommu) {
+ /*
+ * setup the mapping between the irq and the IRTE
+ * base index, the sub_handle pointing to the
+ * appropriate interrupt remap table entry.
+ */
+ set_irte_irq(irq, iommu, index, sub_handle);
+ ret = 0;
+ }
+ up_read(&dmar_global_lock);
- return 0;
+ return ret;
}
static int intel_setup_hpet_msi(unsigned int irq, unsigned int id)
{
- struct intel_iommu *iommu = map_hpet_to_ir(id);
+ int ret = -1;
+ struct intel_iommu *iommu;
int index;
- if (!iommu)
- return -1;
-
- index = alloc_irte(iommu, irq, 1);
- if (index < 0)
- return -1;
+ down_read(&dmar_global_lock);
+ iommu = map_hpet_to_ir(id);
+ if (iommu) {
+ index = alloc_irte(iommu, irq, 1);
+ if (index >= 0)
+ ret = 0;
+ }
+ up_read(&dmar_global_lock);
- return 0;
+ return ret;
}
struct irq_remap_ops intel_irq_remap_ops = {
diff --git a/drivers/iommu/iova.c b/drivers/iommu/iova.c
index 67da6cff74e..f6b17e6af2f 100644
--- a/drivers/iommu/iova.c
+++ b/drivers/iommu/iova.c
@@ -342,19 +342,30 @@ __is_range_overlap(struct rb_node *node,
return 0;
}
+static inline struct iova *
+alloc_and_init_iova(unsigned long pfn_lo, unsigned long pfn_hi)
+{
+ struct iova *iova;
+
+ iova = alloc_iova_mem();
+ if (iova) {
+ iova->pfn_lo = pfn_lo;
+ iova->pfn_hi = pfn_hi;
+ }
+
+ return iova;
+}
+
static struct iova *
__insert_new_range(struct iova_domain *iovad,
unsigned long pfn_lo, unsigned long pfn_hi)
{
struct iova *iova;
- iova = alloc_iova_mem();
- if (!iova)
- return iova;
+ iova = alloc_and_init_iova(pfn_lo, pfn_hi);
+ if (iova)
+ iova_insert_rbtree(&iovad->rbroot, iova);
- iova->pfn_hi = pfn_hi;
- iova->pfn_lo = pfn_lo;
- iova_insert_rbtree(&iovad->rbroot, iova);
return iova;
}
@@ -433,3 +444,44 @@ copy_reserved_iova(struct iova_domain *from, struct iova_domain *to)
}
spin_unlock_irqrestore(&from->iova_rbtree_lock, flags);
}
+
+struct iova *
+split_and_remove_iova(struct iova_domain *iovad, struct iova *iova,
+ unsigned long pfn_lo, unsigned long pfn_hi)
+{
+ unsigned long flags;
+ struct iova *prev = NULL, *next = NULL;
+
+ spin_lock_irqsave(&iovad->iova_rbtree_lock, flags);
+ if (iova->pfn_lo < pfn_lo) {
+ prev = alloc_and_init_iova(iova->pfn_lo, pfn_lo - 1);
+ if (prev == NULL)
+ goto error;
+ }
+ if (iova->pfn_hi > pfn_hi) {
+ next = alloc_and_init_iova(pfn_hi + 1, iova->pfn_hi);
+ if (next == NULL)
+ goto error;
+ }
+
+ __cached_rbnode_delete_update(iovad, iova);
+ rb_erase(&iova->node, &iovad->rbroot);
+
+ if (prev) {
+ iova_insert_rbtree(&iovad->rbroot, prev);
+ iova->pfn_lo = pfn_lo;
+ }
+ if (next) {
+ iova_insert_rbtree(&iovad->rbroot, next);
+ iova->pfn_hi = pfn_hi;
+ }
+ spin_unlock_irqrestore(&iovad->iova_rbtree_lock, flags);
+
+ return iova;
+
+error:
+ spin_unlock_irqrestore(&iovad->iova_rbtree_lock, flags);
+ if (prev)
+ free_iova_mem(prev);
+ return NULL;
+}
diff --git a/drivers/iommu/ipmmu-vmsa.c b/drivers/iommu/ipmmu-vmsa.c
new file mode 100644
index 00000000000..53cde086e83
--- /dev/null
+++ b/drivers/iommu/ipmmu-vmsa.c
@@ -0,0 +1,1255 @@
+/*
+ * IPMMU VMSA
+ *
+ * Copyright (C) 2014 Renesas Electronics Corporation
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; version 2 of the License.
+ */
+
+#include <linux/delay.h>
+#include <linux/dma-mapping.h>
+#include <linux/err.h>
+#include <linux/export.h>
+#include <linux/interrupt.h>
+#include <linux/io.h>
+#include <linux/iommu.h>
+#include <linux/module.h>
+#include <linux/platform_data/ipmmu-vmsa.h>
+#include <linux/platform_device.h>
+#include <linux/sizes.h>
+#include <linux/slab.h>
+
+#include <asm/dma-iommu.h>
+#include <asm/pgalloc.h>
+
+struct ipmmu_vmsa_device {
+ struct device *dev;
+ void __iomem *base;
+ struct list_head list;
+
+ const struct ipmmu_vmsa_platform_data *pdata;
+ unsigned int num_utlbs;
+
+ struct dma_iommu_mapping *mapping;
+};
+
+struct ipmmu_vmsa_domain {
+ struct ipmmu_vmsa_device *mmu;
+ struct iommu_domain *io_domain;
+
+ unsigned int context_id;
+ spinlock_t lock; /* Protects mappings */
+ pgd_t *pgd;
+};
+
+struct ipmmu_vmsa_archdata {
+ struct ipmmu_vmsa_device *mmu;
+ unsigned int utlb;
+};
+
+static DEFINE_SPINLOCK(ipmmu_devices_lock);
+static LIST_HEAD(ipmmu_devices);
+
+#define TLB_LOOP_TIMEOUT 100 /* 100us */
+
+/* -----------------------------------------------------------------------------
+ * Registers Definition
+ */
+
+#define IM_CTX_SIZE 0x40
+
+#define IMCTR 0x0000
+#define IMCTR_TRE (1 << 17)
+#define IMCTR_AFE (1 << 16)
+#define IMCTR_RTSEL_MASK (3 << 4)
+#define IMCTR_RTSEL_SHIFT 4
+#define IMCTR_TREN (1 << 3)
+#define IMCTR_INTEN (1 << 2)
+#define IMCTR_FLUSH (1 << 1)
+#define IMCTR_MMUEN (1 << 0)
+
+#define IMCAAR 0x0004
+
+#define IMTTBCR 0x0008
+#define IMTTBCR_EAE (1 << 31)
+#define IMTTBCR_PMB (1 << 30)
+#define IMTTBCR_SH1_NON_SHAREABLE (0 << 28)
+#define IMTTBCR_SH1_OUTER_SHAREABLE (2 << 28)
+#define IMTTBCR_SH1_INNER_SHAREABLE (3 << 28)
+#define IMTTBCR_SH1_MASK (3 << 28)
+#define IMTTBCR_ORGN1_NC (0 << 26)
+#define IMTTBCR_ORGN1_WB_WA (1 << 26)
+#define IMTTBCR_ORGN1_WT (2 << 26)
+#define IMTTBCR_ORGN1_WB (3 << 26)
+#define IMTTBCR_ORGN1_MASK (3 << 26)
+#define IMTTBCR_IRGN1_NC (0 << 24)
+#define IMTTBCR_IRGN1_WB_WA (1 << 24)
+#define IMTTBCR_IRGN1_WT (2 << 24)
+#define IMTTBCR_IRGN1_WB (3 << 24)
+#define IMTTBCR_IRGN1_MASK (3 << 24)
+#define IMTTBCR_TSZ1_MASK (7 << 16)
+#define IMTTBCR_TSZ1_SHIFT 16
+#define IMTTBCR_SH0_NON_SHAREABLE (0 << 12)
+#define IMTTBCR_SH0_OUTER_SHAREABLE (2 << 12)
+#define IMTTBCR_SH0_INNER_SHAREABLE (3 << 12)
+#define IMTTBCR_SH0_MASK (3 << 12)
+#define IMTTBCR_ORGN0_NC (0 << 10)
+#define IMTTBCR_ORGN0_WB_WA (1 << 10)
+#define IMTTBCR_ORGN0_WT (2 << 10)
+#define IMTTBCR_ORGN0_WB (3 << 10)
+#define IMTTBCR_ORGN0_MASK (3 << 10)
+#define IMTTBCR_IRGN0_NC (0 << 8)
+#define IMTTBCR_IRGN0_WB_WA (1 << 8)
+#define IMTTBCR_IRGN0_WT (2 << 8)
+#define IMTTBCR_IRGN0_WB (3 << 8)
+#define IMTTBCR_IRGN0_MASK (3 << 8)
+#define IMTTBCR_SL0_LVL_2 (0 << 4)
+#define IMTTBCR_SL0_LVL_1 (1 << 4)
+#define IMTTBCR_TSZ0_MASK (7 << 0)
+#define IMTTBCR_TSZ0_SHIFT O
+
+#define IMBUSCR 0x000c
+#define IMBUSCR_DVM (1 << 2)
+#define IMBUSCR_BUSSEL_SYS (0 << 0)
+#define IMBUSCR_BUSSEL_CCI (1 << 0)
+#define IMBUSCR_BUSSEL_IMCAAR (2 << 0)
+#define IMBUSCR_BUSSEL_CCI_IMCAAR (3 << 0)
+#define IMBUSCR_BUSSEL_MASK (3 << 0)
+
+#define IMTTLBR0 0x0010
+#define IMTTUBR0 0x0014
+#define IMTTLBR1 0x0018
+#define IMTTUBR1 0x001c
+
+#define IMSTR 0x0020
+#define IMSTR_ERRLVL_MASK (3 << 12)
+#define IMSTR_ERRLVL_SHIFT 12
+#define IMSTR_ERRCODE_TLB_FORMAT (1 << 8)
+#define IMSTR_ERRCODE_ACCESS_PERM (4 << 8)
+#define IMSTR_ERRCODE_SECURE_ACCESS (5 << 8)
+#define IMSTR_ERRCODE_MASK (7 << 8)
+#define IMSTR_MHIT (1 << 4)
+#define IMSTR_ABORT (1 << 2)
+#define IMSTR_PF (1 << 1)
+#define IMSTR_TF (1 << 0)
+
+#define IMMAIR0 0x0028
+#define IMMAIR1 0x002c
+#define IMMAIR_ATTR_MASK 0xff
+#define IMMAIR_ATTR_DEVICE 0x04
+#define IMMAIR_ATTR_NC 0x44
+#define IMMAIR_ATTR_WBRWA 0xff
+#define IMMAIR_ATTR_SHIFT(n) ((n) << 3)
+#define IMMAIR_ATTR_IDX_NC 0
+#define IMMAIR_ATTR_IDX_WBRWA 1
+#define IMMAIR_ATTR_IDX_DEV 2
+
+#define IMEAR 0x0030
+
+#define IMPCTR 0x0200
+#define IMPSTR 0x0208
+#define IMPEAR 0x020c
+#define IMPMBA(n) (0x0280 + ((n) * 4))
+#define IMPMBD(n) (0x02c0 + ((n) * 4))
+
+#define IMUCTR(n) (0x0300 + ((n) * 16))
+#define IMUCTR_FIXADDEN (1 << 31)
+#define IMUCTR_FIXADD_MASK (0xff << 16)
+#define IMUCTR_FIXADD_SHIFT 16
+#define IMUCTR_TTSEL_MMU(n) ((n) << 4)
+#define IMUCTR_TTSEL_PMB (8 << 4)
+#define IMUCTR_TTSEL_MASK (15 << 4)
+#define IMUCTR_FLUSH (1 << 1)
+#define IMUCTR_MMUEN (1 << 0)
+
+#define IMUASID(n) (0x0308 + ((n) * 16))
+#define IMUASID_ASID8_MASK (0xff << 8)
+#define IMUASID_ASID8_SHIFT 8
+#define IMUASID_ASID0_MASK (0xff << 0)
+#define IMUASID_ASID0_SHIFT 0
+
+/* -----------------------------------------------------------------------------
+ * Page Table Bits
+ */
+
+/*
+ * VMSA states in section B3.6.3 "Control of Secure or Non-secure memory access,
+ * Long-descriptor format" that the NStable bit being set in a table descriptor
+ * will result in the NStable and NS bits of all child entries being ignored and
+ * considered as being set. The IPMMU seems not to comply with this, as it
+ * generates a secure access page fault if any of the NStable and NS bits isn't
+ * set when running in non-secure mode.
+ */
+#ifndef PMD_NSTABLE
+#define PMD_NSTABLE (_AT(pmdval_t, 1) << 63)
+#endif
+
+#define ARM_VMSA_PTE_XN (((pteval_t)3) << 53)
+#define ARM_VMSA_PTE_CONT (((pteval_t)1) << 52)
+#define ARM_VMSA_PTE_AF (((pteval_t)1) << 10)
+#define ARM_VMSA_PTE_SH_NS (((pteval_t)0) << 8)
+#define ARM_VMSA_PTE_SH_OS (((pteval_t)2) << 8)
+#define ARM_VMSA_PTE_SH_IS (((pteval_t)3) << 8)
+#define ARM_VMSA_PTE_SH_MASK (((pteval_t)3) << 8)
+#define ARM_VMSA_PTE_NS (((pteval_t)1) << 5)
+#define ARM_VMSA_PTE_PAGE (((pteval_t)3) << 0)
+
+/* Stage-1 PTE */
+#define ARM_VMSA_PTE_nG (((pteval_t)1) << 11)
+#define ARM_VMSA_PTE_AP_UNPRIV (((pteval_t)1) << 6)
+#define ARM_VMSA_PTE_AP_RDONLY (((pteval_t)2) << 6)
+#define ARM_VMSA_PTE_AP_MASK (((pteval_t)3) << 6)
+#define ARM_VMSA_PTE_ATTRINDX_MASK (((pteval_t)3) << 2)
+#define ARM_VMSA_PTE_ATTRINDX_SHIFT 2
+
+#define ARM_VMSA_PTE_ATTRS_MASK \
+ (ARM_VMSA_PTE_XN | ARM_VMSA_PTE_CONT | ARM_VMSA_PTE_nG | \
+ ARM_VMSA_PTE_AF | ARM_VMSA_PTE_SH_MASK | ARM_VMSA_PTE_AP_MASK | \
+ ARM_VMSA_PTE_NS | ARM_VMSA_PTE_ATTRINDX_MASK)
+
+#define ARM_VMSA_PTE_CONT_ENTRIES 16
+#define ARM_VMSA_PTE_CONT_SIZE (PAGE_SIZE * ARM_VMSA_PTE_CONT_ENTRIES)
+
+#define IPMMU_PTRS_PER_PTE 512
+#define IPMMU_PTRS_PER_PMD 512
+#define IPMMU_PTRS_PER_PGD 4
+
+/* -----------------------------------------------------------------------------
+ * Read/Write Access
+ */
+
+static u32 ipmmu_read(struct ipmmu_vmsa_device *mmu, unsigned int offset)
+{
+ return ioread32(mmu->base + offset);
+}
+
+static void ipmmu_write(struct ipmmu_vmsa_device *mmu, unsigned int offset,
+ u32 data)
+{
+ iowrite32(data, mmu->base + offset);
+}
+
+static u32 ipmmu_ctx_read(struct ipmmu_vmsa_domain *domain, unsigned int reg)
+{
+ return ipmmu_read(domain->mmu, domain->context_id * IM_CTX_SIZE + reg);
+}
+
+static void ipmmu_ctx_write(struct ipmmu_vmsa_domain *domain, unsigned int reg,
+ u32 data)
+{
+ ipmmu_write(domain->mmu, domain->context_id * IM_CTX_SIZE + reg, data);
+}
+
+/* -----------------------------------------------------------------------------
+ * TLB and microTLB Management
+ */
+
+/* Wait for any pending TLB invalidations to complete */
+static void ipmmu_tlb_sync(struct ipmmu_vmsa_domain *domain)
+{
+ unsigned int count = 0;
+
+ while (ipmmu_ctx_read(domain, IMCTR) & IMCTR_FLUSH) {
+ cpu_relax();
+ if (++count == TLB_LOOP_TIMEOUT) {
+ dev_err_ratelimited(domain->mmu->dev,
+ "TLB sync timed out -- MMU may be deadlocked\n");
+ return;
+ }
+ udelay(1);
+ }
+}
+
+static void ipmmu_tlb_invalidate(struct ipmmu_vmsa_domain *domain)
+{
+ u32 reg;
+
+ reg = ipmmu_ctx_read(domain, IMCTR);
+ reg |= IMCTR_FLUSH;
+ ipmmu_ctx_write(domain, IMCTR, reg);
+
+ ipmmu_tlb_sync(domain);
+}
+
+/*
+ * Enable MMU translation for the microTLB.
+ */
+static void ipmmu_utlb_enable(struct ipmmu_vmsa_domain *domain,
+ unsigned int utlb)
+{
+ struct ipmmu_vmsa_device *mmu = domain->mmu;
+
+ /*
+ * TODO: Reference-count the microTLB as several bus masters can be
+ * connected to the same microTLB.
+ */
+
+ /* TODO: What should we set the ASID to ? */
+ ipmmu_write(mmu, IMUASID(utlb), 0);
+ /* TODO: Do we need to flush the microTLB ? */
+ ipmmu_write(mmu, IMUCTR(utlb),
+ IMUCTR_TTSEL_MMU(domain->context_id) | IMUCTR_FLUSH |
+ IMUCTR_MMUEN);
+}
+
+/*
+ * Disable MMU translation for the microTLB.
+ */
+static void ipmmu_utlb_disable(struct ipmmu_vmsa_domain *domain,
+ unsigned int utlb)
+{
+ struct ipmmu_vmsa_device *mmu = domain->mmu;
+
+ ipmmu_write(mmu, IMUCTR(utlb), 0);
+}
+
+static void ipmmu_flush_pgtable(struct ipmmu_vmsa_device *mmu, void *addr,
+ size_t size)
+{
+ unsigned long offset = (unsigned long)addr & ~PAGE_MASK;
+
+ /*
+ * TODO: Add support for coherent walk through CCI with DVM and remove
+ * cache handling.
+ */
+ dma_map_page(mmu->dev, virt_to_page(addr), offset, size, DMA_TO_DEVICE);
+}
+
+/* -----------------------------------------------------------------------------
+ * Domain/Context Management
+ */
+
+static int ipmmu_domain_init_context(struct ipmmu_vmsa_domain *domain)
+{
+ phys_addr_t ttbr;
+ u32 reg;
+
+ /*
+ * TODO: When adding support for multiple contexts, find an unused
+ * context.
+ */
+ domain->context_id = 0;
+
+ /* TTBR0 */
+ ipmmu_flush_pgtable(domain->mmu, domain->pgd,
+ IPMMU_PTRS_PER_PGD * sizeof(*domain->pgd));
+ ttbr = __pa(domain->pgd);
+ ipmmu_ctx_write(domain, IMTTLBR0, ttbr);
+ ipmmu_ctx_write(domain, IMTTUBR0, ttbr >> 32);
+
+ /*
+ * TTBCR
+ * We use long descriptors with inner-shareable WBWA tables and allocate
+ * the whole 32-bit VA space to TTBR0.
+ */
+ ipmmu_ctx_write(domain, IMTTBCR, IMTTBCR_EAE |
+ IMTTBCR_SH0_INNER_SHAREABLE | IMTTBCR_ORGN0_WB_WA |
+ IMTTBCR_IRGN0_WB_WA | IMTTBCR_SL0_LVL_1);
+
+ /*
+ * MAIR0
+ * We need three attributes only, non-cacheable, write-back read/write
+ * allocate and device memory.
+ */
+ reg = (IMMAIR_ATTR_NC << IMMAIR_ATTR_SHIFT(IMMAIR_ATTR_IDX_NC))
+ | (IMMAIR_ATTR_WBRWA << IMMAIR_ATTR_SHIFT(IMMAIR_ATTR_IDX_WBRWA))
+ | (IMMAIR_ATTR_DEVICE << IMMAIR_ATTR_SHIFT(IMMAIR_ATTR_IDX_DEV));
+ ipmmu_ctx_write(domain, IMMAIR0, reg);
+
+ /* IMBUSCR */
+ ipmmu_ctx_write(domain, IMBUSCR,
+ ipmmu_ctx_read(domain, IMBUSCR) &
+ ~(IMBUSCR_DVM | IMBUSCR_BUSSEL_MASK));
+
+ /*
+ * IMSTR
+ * Clear all interrupt flags.
+ */
+ ipmmu_ctx_write(domain, IMSTR, ipmmu_ctx_read(domain, IMSTR));
+
+ /*
+ * IMCTR
+ * Enable the MMU and interrupt generation. The long-descriptor
+ * translation table format doesn't use TEX remapping. Don't enable AF
+ * software management as we have no use for it. Flush the TLB as
+ * required when modifying the context registers.
+ */
+ ipmmu_ctx_write(domain, IMCTR, IMCTR_INTEN | IMCTR_FLUSH | IMCTR_MMUEN);
+
+ return 0;
+}
+
+static void ipmmu_domain_destroy_context(struct ipmmu_vmsa_domain *domain)
+{
+ /*
+ * Disable the context. Flush the TLB as required when modifying the
+ * context registers.
+ *
+ * TODO: Is TLB flush really needed ?
+ */
+ ipmmu_ctx_write(domain, IMCTR, IMCTR_FLUSH);
+ ipmmu_tlb_sync(domain);
+}
+
+/* -----------------------------------------------------------------------------
+ * Fault Handling
+ */
+
+static irqreturn_t ipmmu_domain_irq(struct ipmmu_vmsa_domain *domain)
+{
+ const u32 err_mask = IMSTR_MHIT | IMSTR_ABORT | IMSTR_PF | IMSTR_TF;
+ struct ipmmu_vmsa_device *mmu = domain->mmu;
+ u32 status;
+ u32 iova;
+
+ status = ipmmu_ctx_read(domain, IMSTR);
+ if (!(status & err_mask))
+ return IRQ_NONE;
+
+ iova = ipmmu_ctx_read(domain, IMEAR);
+
+ /*
+ * Clear the error status flags. Unlike traditional interrupt flag
+ * registers that must be cleared by writing 1, this status register
+ * seems to require 0. The error address register must be read before,
+ * otherwise its value will be 0.
+ */
+ ipmmu_ctx_write(domain, IMSTR, 0);
+
+ /* Log fatal errors. */
+ if (status & IMSTR_MHIT)
+ dev_err_ratelimited(mmu->dev, "Multiple TLB hits @0x%08x\n",
+ iova);
+ if (status & IMSTR_ABORT)
+ dev_err_ratelimited(mmu->dev, "Page Table Walk Abort @0x%08x\n",
+ iova);
+
+ if (!(status & (IMSTR_PF | IMSTR_TF)))
+ return IRQ_NONE;
+
+ /*
+ * Try to handle page faults and translation faults.
+ *
+ * TODO: We need to look up the faulty device based on the I/O VA. Use
+ * the IOMMU device for now.
+ */
+ if (!report_iommu_fault(domain->io_domain, mmu->dev, iova, 0))
+ return IRQ_HANDLED;
+
+ dev_err_ratelimited(mmu->dev,
+ "Unhandled fault: status 0x%08x iova 0x%08x\n",
+ status, iova);
+
+ return IRQ_HANDLED;
+}
+
+static irqreturn_t ipmmu_irq(int irq, void *dev)
+{
+ struct ipmmu_vmsa_device *mmu = dev;
+ struct iommu_domain *io_domain;
+ struct ipmmu_vmsa_domain *domain;
+
+ if (!mmu->mapping)
+ return IRQ_NONE;
+
+ io_domain = mmu->mapping->domain;
+ domain = io_domain->priv;
+
+ return ipmmu_domain_irq(domain);
+}
+
+/* -----------------------------------------------------------------------------
+ * Page Table Management
+ */
+
+#define pud_pgtable(pud) pfn_to_page(__phys_to_pfn(pud_val(pud) & PHYS_MASK))
+
+static void ipmmu_free_ptes(pmd_t *pmd)
+{
+ pgtable_t table = pmd_pgtable(*pmd);
+ __free_page(table);
+}
+
+static void ipmmu_free_pmds(pud_t *pud)
+{
+ pmd_t *pmd = pmd_offset(pud, 0);
+ pgtable_t table;
+ unsigned int i;
+
+ for (i = 0; i < IPMMU_PTRS_PER_PMD; ++i) {
+ if (!pmd_table(*pmd))
+ continue;
+
+ ipmmu_free_ptes(pmd);
+ pmd++;
+ }
+
+ table = pud_pgtable(*pud);
+ __free_page(table);
+}
+
+static void ipmmu_free_pgtables(struct ipmmu_vmsa_domain *domain)
+{
+ pgd_t *pgd, *pgd_base = domain->pgd;
+ unsigned int i;
+
+ /*
+ * Recursively free the page tables for this domain. We don't care about
+ * speculative TLB filling, because the TLB will be nuked next time this
+ * context bank is re-allocated and no devices currently map to these
+ * tables.
+ */
+ pgd = pgd_base;
+ for (i = 0; i < IPMMU_PTRS_PER_PGD; ++i) {
+ if (pgd_none(*pgd))
+ continue;
+ ipmmu_free_pmds((pud_t *)pgd);
+ pgd++;
+ }
+
+ kfree(pgd_base);
+}
+
+/*
+ * We can't use the (pgd|pud|pmd|pte)_populate or the set_(pgd|pud|pmd|pte)
+ * functions as they would flush the CPU TLB.
+ */
+
+static pte_t *ipmmu_alloc_pte(struct ipmmu_vmsa_device *mmu, pmd_t *pmd,
+ unsigned long iova)
+{
+ pte_t *pte;
+
+ if (!pmd_none(*pmd))
+ return pte_offset_kernel(pmd, iova);
+
+ pte = (pte_t *)get_zeroed_page(GFP_ATOMIC);
+ if (!pte)
+ return NULL;
+
+ ipmmu_flush_pgtable(mmu, pte, PAGE_SIZE);
+ *pmd = __pmd(__pa(pte) | PMD_NSTABLE | PMD_TYPE_TABLE);
+ ipmmu_flush_pgtable(mmu, pmd, sizeof(*pmd));
+
+ return pte + pte_index(iova);
+}
+
+static pmd_t *ipmmu_alloc_pmd(struct ipmmu_vmsa_device *mmu, pgd_t *pgd,
+ unsigned long iova)
+{
+ pud_t *pud = (pud_t *)pgd;
+ pmd_t *pmd;
+
+ if (!pud_none(*pud))
+ return pmd_offset(pud, iova);
+
+ pmd = (pmd_t *)get_zeroed_page(GFP_ATOMIC);
+ if (!pmd)
+ return NULL;
+
+ ipmmu_flush_pgtable(mmu, pmd, PAGE_SIZE);
+ *pud = __pud(__pa(pmd) | PMD_NSTABLE | PMD_TYPE_TABLE);
+ ipmmu_flush_pgtable(mmu, pud, sizeof(*pud));
+
+ return pmd + pmd_index(iova);
+}
+
+static u64 ipmmu_page_prot(unsigned int prot, u64 type)
+{
+ u64 pgprot = ARM_VMSA_PTE_XN | ARM_VMSA_PTE_nG | ARM_VMSA_PTE_AF
+ | ARM_VMSA_PTE_SH_IS | ARM_VMSA_PTE_AP_UNPRIV
+ | ARM_VMSA_PTE_NS | type;
+
+ if (!(prot & IOMMU_WRITE) && (prot & IOMMU_READ))
+ pgprot |= ARM_VMSA_PTE_AP_RDONLY;
+
+ if (prot & IOMMU_CACHE)
+ pgprot |= IMMAIR_ATTR_IDX_WBRWA << ARM_VMSA_PTE_ATTRINDX_SHIFT;
+
+ if (prot & IOMMU_EXEC)
+ pgprot &= ~ARM_VMSA_PTE_XN;
+ else if (!(prot & (IOMMU_READ | IOMMU_WRITE)))
+ /* If no access create a faulting entry to avoid TLB fills. */
+ pgprot &= ~ARM_VMSA_PTE_PAGE;
+
+ return pgprot;
+}
+
+static int ipmmu_alloc_init_pte(struct ipmmu_vmsa_device *mmu, pmd_t *pmd,
+ unsigned long iova, unsigned long pfn,
+ size_t size, int prot)
+{
+ pteval_t pteval = ipmmu_page_prot(prot, ARM_VMSA_PTE_PAGE);
+ unsigned int num_ptes = 1;
+ pte_t *pte, *start;
+ unsigned int i;
+
+ pte = ipmmu_alloc_pte(mmu, pmd, iova);
+ if (!pte)
+ return -ENOMEM;
+
+ start = pte;
+
+ /*
+ * Install the page table entries. We can be called both for a single
+ * page or for a block of 16 physically contiguous pages. In the latter
+ * case set the PTE contiguous hint.
+ */
+ if (size == SZ_64K) {
+ pteval |= ARM_VMSA_PTE_CONT;
+ num_ptes = ARM_VMSA_PTE_CONT_ENTRIES;
+ }
+
+ for (i = num_ptes; i; --i)
+ *pte++ = pfn_pte(pfn++, __pgprot(pteval));
+
+ ipmmu_flush_pgtable(mmu, start, sizeof(*pte) * num_ptes);
+
+ return 0;
+}
+
+static int ipmmu_alloc_init_pmd(struct ipmmu_vmsa_device *mmu, pmd_t *pmd,
+ unsigned long iova, unsigned long pfn,
+ int prot)
+{
+ pmdval_t pmdval = ipmmu_page_prot(prot, PMD_TYPE_SECT);
+
+ *pmd = pfn_pmd(pfn, __pgprot(pmdval));
+ ipmmu_flush_pgtable(mmu, pmd, sizeof(*pmd));
+
+ return 0;
+}
+
+static int ipmmu_create_mapping(struct ipmmu_vmsa_domain *domain,
+ unsigned long iova, phys_addr_t paddr,
+ size_t size, int prot)
+{
+ struct ipmmu_vmsa_device *mmu = domain->mmu;
+ pgd_t *pgd = domain->pgd;
+ unsigned long flags;
+ unsigned long pfn;
+ pmd_t *pmd;
+ int ret;
+
+ if (!pgd)
+ return -EINVAL;
+
+ if (size & ~PAGE_MASK)
+ return -EINVAL;
+
+ if (paddr & ~((1ULL << 40) - 1))
+ return -ERANGE;
+
+ pfn = __phys_to_pfn(paddr);
+ pgd += pgd_index(iova);
+
+ /* Update the page tables. */
+ spin_lock_irqsave(&domain->lock, flags);
+
+ pmd = ipmmu_alloc_pmd(mmu, pgd, iova);
+ if (!pmd) {
+ ret = -ENOMEM;
+ goto done;
+ }
+
+ switch (size) {
+ case SZ_2M:
+ ret = ipmmu_alloc_init_pmd(mmu, pmd, iova, pfn, prot);
+ break;
+ case SZ_64K:
+ case SZ_4K:
+ ret = ipmmu_alloc_init_pte(mmu, pmd, iova, pfn, size, prot);
+ break;
+ default:
+ ret = -EINVAL;
+ break;
+ }
+
+done:
+ spin_unlock_irqrestore(&domain->lock, flags);
+
+ if (!ret)
+ ipmmu_tlb_invalidate(domain);
+
+ return ret;
+}
+
+static void ipmmu_clear_pud(struct ipmmu_vmsa_device *mmu, pud_t *pud)
+{
+ /* Free the page table. */
+ pgtable_t table = pud_pgtable(*pud);
+ __free_page(table);
+
+ /* Clear the PUD. */
+ *pud = __pud(0);
+ ipmmu_flush_pgtable(mmu, pud, sizeof(*pud));
+}
+
+static void ipmmu_clear_pmd(struct ipmmu_vmsa_device *mmu, pud_t *pud,
+ pmd_t *pmd)
+{
+ unsigned int i;
+
+ /* Free the page table. */
+ if (pmd_table(*pmd)) {
+ pgtable_t table = pmd_pgtable(*pmd);
+ __free_page(table);
+ }
+
+ /* Clear the PMD. */
+ *pmd = __pmd(0);
+ ipmmu_flush_pgtable(mmu, pmd, sizeof(*pmd));
+
+ /* Check whether the PUD is still needed. */
+ pmd = pmd_offset(pud, 0);
+ for (i = 0; i < IPMMU_PTRS_PER_PMD; ++i) {
+ if (!pmd_none(pmd[i]))
+ return;
+ }
+
+ /* Clear the parent PUD. */
+ ipmmu_clear_pud(mmu, pud);
+}
+
+static void ipmmu_clear_pte(struct ipmmu_vmsa_device *mmu, pud_t *pud,
+ pmd_t *pmd, pte_t *pte, unsigned int num_ptes)
+{
+ unsigned int i;
+
+ /* Clear the PTE. */
+ for (i = num_ptes; i; --i)
+ pte[i-1] = __pte(0);
+
+ ipmmu_flush_pgtable(mmu, pte, sizeof(*pte) * num_ptes);
+
+ /* Check whether the PMD is still needed. */
+ pte = pte_offset_kernel(pmd, 0);
+ for (i = 0; i < IPMMU_PTRS_PER_PTE; ++i) {
+ if (!pte_none(pte[i]))
+ return;
+ }
+
+ /* Clear the parent PMD. */
+ ipmmu_clear_pmd(mmu, pud, pmd);
+}
+
+static int ipmmu_split_pmd(struct ipmmu_vmsa_device *mmu, pmd_t *pmd)
+{
+ pte_t *pte, *start;
+ pteval_t pteval;
+ unsigned long pfn;
+ unsigned int i;
+
+ pte = (pte_t *)get_zeroed_page(GFP_ATOMIC);
+ if (!pte)
+ return -ENOMEM;
+
+ /* Copy the PMD attributes. */
+ pteval = (pmd_val(*pmd) & ARM_VMSA_PTE_ATTRS_MASK)
+ | ARM_VMSA_PTE_CONT | ARM_VMSA_PTE_PAGE;
+
+ pfn = pmd_pfn(*pmd);
+ start = pte;
+
+ for (i = IPMMU_PTRS_PER_PTE; i; --i)
+ *pte++ = pfn_pte(pfn++, __pgprot(pteval));
+
+ ipmmu_flush_pgtable(mmu, start, PAGE_SIZE);
+ *pmd = __pmd(__pa(start) | PMD_NSTABLE | PMD_TYPE_TABLE);
+ ipmmu_flush_pgtable(mmu, pmd, sizeof(*pmd));
+
+ return 0;
+}
+
+static void ipmmu_split_pte(struct ipmmu_vmsa_device *mmu, pte_t *pte)
+{
+ unsigned int i;
+
+ for (i = ARM_VMSA_PTE_CONT_ENTRIES; i; --i)
+ pte[i-1] = __pte(pte_val(*pte) & ~ARM_VMSA_PTE_CONT);
+
+ ipmmu_flush_pgtable(mmu, pte, sizeof(*pte) * ARM_VMSA_PTE_CONT_ENTRIES);
+}
+
+static int ipmmu_clear_mapping(struct ipmmu_vmsa_domain *domain,
+ unsigned long iova, size_t size)
+{
+ struct ipmmu_vmsa_device *mmu = domain->mmu;
+ unsigned long flags;
+ pgd_t *pgd = domain->pgd;
+ pud_t *pud;
+ pmd_t *pmd;
+ pte_t *pte;
+ int ret = 0;
+
+ if (!pgd)
+ return -EINVAL;
+
+ if (size & ~PAGE_MASK)
+ return -EINVAL;
+
+ pgd += pgd_index(iova);
+ pud = (pud_t *)pgd;
+
+ spin_lock_irqsave(&domain->lock, flags);
+
+ /* If there's no PUD or PMD we're done. */
+ if (pud_none(*pud))
+ goto done;
+
+ pmd = pmd_offset(pud, iova);
+ if (pmd_none(*pmd))
+ goto done;
+
+ /*
+ * When freeing a 2MB block just clear the PMD. In the unlikely case the
+ * block is mapped as individual pages this will free the corresponding
+ * PTE page table.
+ */
+ if (size == SZ_2M) {
+ ipmmu_clear_pmd(mmu, pud, pmd);
+ goto done;
+ }
+
+ /*
+ * If the PMD has been mapped as a section remap it as pages to allow
+ * freeing individual pages.
+ */
+ if (pmd_sect(*pmd))
+ ipmmu_split_pmd(mmu, pmd);
+
+ pte = pte_offset_kernel(pmd, iova);
+
+ /*
+ * When freeing a 64kB block just clear the PTE entries. We don't have
+ * to care about the contiguous hint of the surrounding entries.
+ */
+ if (size == SZ_64K) {
+ ipmmu_clear_pte(mmu, pud, pmd, pte, ARM_VMSA_PTE_CONT_ENTRIES);
+ goto done;
+ }
+
+ /*
+ * If the PTE has been mapped with the contiguous hint set remap it and
+ * its surrounding PTEs to allow unmapping a single page.
+ */
+ if (pte_val(*pte) & ARM_VMSA_PTE_CONT)
+ ipmmu_split_pte(mmu, pte);
+
+ /* Clear the PTE. */
+ ipmmu_clear_pte(mmu, pud, pmd, pte, 1);
+
+done:
+ spin_unlock_irqrestore(&domain->lock, flags);
+
+ if (ret)
+ ipmmu_tlb_invalidate(domain);
+
+ return 0;
+}
+
+/* -----------------------------------------------------------------------------
+ * IOMMU Operations
+ */
+
+static int ipmmu_domain_init(struct iommu_domain *io_domain)
+{
+ struct ipmmu_vmsa_domain *domain;
+
+ domain = kzalloc(sizeof(*domain), GFP_KERNEL);
+ if (!domain)
+ return -ENOMEM;
+
+ spin_lock_init(&domain->lock);
+
+ domain->pgd = kzalloc(IPMMU_PTRS_PER_PGD * sizeof(pgd_t), GFP_KERNEL);
+ if (!domain->pgd) {
+ kfree(domain);
+ return -ENOMEM;
+ }
+
+ io_domain->priv = domain;
+ domain->io_domain = io_domain;
+
+ return 0;
+}
+
+static void ipmmu_domain_destroy(struct iommu_domain *io_domain)
+{
+ struct ipmmu_vmsa_domain *domain = io_domain->priv;
+
+ /*
+ * Free the domain resources. We assume that all devices have already
+ * been detached.
+ */
+ ipmmu_domain_destroy_context(domain);
+ ipmmu_free_pgtables(domain);
+ kfree(domain);
+}
+
+static int ipmmu_attach_device(struct iommu_domain *io_domain,
+ struct device *dev)
+{
+ struct ipmmu_vmsa_archdata *archdata = dev->archdata.iommu;
+ struct ipmmu_vmsa_device *mmu = archdata->mmu;
+ struct ipmmu_vmsa_domain *domain = io_domain->priv;
+ unsigned long flags;
+ int ret = 0;
+
+ if (!mmu) {
+ dev_err(dev, "Cannot attach to IPMMU\n");
+ return -ENXIO;
+ }
+
+ spin_lock_irqsave(&domain->lock, flags);
+
+ if (!domain->mmu) {
+ /* The domain hasn't been used yet, initialize it. */
+ domain->mmu = mmu;
+ ret = ipmmu_domain_init_context(domain);
+ } else if (domain->mmu != mmu) {
+ /*
+ * Something is wrong, we can't attach two devices using
+ * different IOMMUs to the same domain.
+ */
+ dev_err(dev, "Can't attach IPMMU %s to domain on IPMMU %s\n",
+ dev_name(mmu->dev), dev_name(domain->mmu->dev));
+ ret = -EINVAL;
+ }
+
+ spin_unlock_irqrestore(&domain->lock, flags);
+
+ if (ret < 0)
+ return ret;
+
+ ipmmu_utlb_enable(domain, archdata->utlb);
+
+ return 0;
+}
+
+static void ipmmu_detach_device(struct iommu_domain *io_domain,
+ struct device *dev)
+{
+ struct ipmmu_vmsa_archdata *archdata = dev->archdata.iommu;
+ struct ipmmu_vmsa_domain *domain = io_domain->priv;
+
+ ipmmu_utlb_disable(domain, archdata->utlb);
+
+ /*
+ * TODO: Optimize by disabling the context when no device is attached.
+ */
+}
+
+static int ipmmu_map(struct iommu_domain *io_domain, unsigned long iova,
+ phys_addr_t paddr, size_t size, int prot)
+{
+ struct ipmmu_vmsa_domain *domain = io_domain->priv;
+
+ if (!domain)
+ return -ENODEV;
+
+ return ipmmu_create_mapping(domain, iova, paddr, size, prot);
+}
+
+static size_t ipmmu_unmap(struct iommu_domain *io_domain, unsigned long iova,
+ size_t size)
+{
+ struct ipmmu_vmsa_domain *domain = io_domain->priv;
+ int ret;
+
+ ret = ipmmu_clear_mapping(domain, iova, size);
+ return ret ? 0 : size;
+}
+
+static phys_addr_t ipmmu_iova_to_phys(struct iommu_domain *io_domain,
+ dma_addr_t iova)
+{
+ struct ipmmu_vmsa_domain *domain = io_domain->priv;
+ pgd_t pgd;
+ pud_t pud;
+ pmd_t pmd;
+ pte_t pte;
+
+ /* TODO: Is locking needed ? */
+
+ if (!domain->pgd)
+ return 0;
+
+ pgd = *(domain->pgd + pgd_index(iova));
+ if (pgd_none(pgd))
+ return 0;
+
+ pud = *pud_offset(&pgd, iova);
+ if (pud_none(pud))
+ return 0;
+
+ pmd = *pmd_offset(&pud, iova);
+ if (pmd_none(pmd))
+ return 0;
+
+ if (pmd_sect(pmd))
+ return __pfn_to_phys(pmd_pfn(pmd)) | (iova & ~PMD_MASK);
+
+ pte = *(pmd_page_vaddr(pmd) + pte_index(iova));
+ if (pte_none(pte))
+ return 0;
+
+ return __pfn_to_phys(pte_pfn(pte)) | (iova & ~PAGE_MASK);
+}
+
+static int ipmmu_find_utlb(struct ipmmu_vmsa_device *mmu, struct device *dev)
+{
+ const struct ipmmu_vmsa_master *master = mmu->pdata->masters;
+ const char *devname = dev_name(dev);
+ unsigned int i;
+
+ for (i = 0; i < mmu->pdata->num_masters; ++i, ++master) {
+ if (strcmp(master->name, devname) == 0)
+ return master->utlb;
+ }
+
+ return -1;
+}
+
+static int ipmmu_add_device(struct device *dev)
+{
+ struct ipmmu_vmsa_archdata *archdata;
+ struct ipmmu_vmsa_device *mmu;
+ struct iommu_group *group;
+ int utlb = -1;
+ int ret;
+
+ if (dev->archdata.iommu) {
+ dev_warn(dev, "IOMMU driver already assigned to device %s\n",
+ dev_name(dev));
+ return -EINVAL;
+ }
+
+ /* Find the master corresponding to the device. */
+ spin_lock(&ipmmu_devices_lock);
+
+ list_for_each_entry(mmu, &ipmmu_devices, list) {
+ utlb = ipmmu_find_utlb(mmu, dev);
+ if (utlb >= 0) {
+ /*
+ * TODO Take a reference to the MMU to protect
+ * against device removal.
+ */
+ break;
+ }
+ }
+
+ spin_unlock(&ipmmu_devices_lock);
+
+ if (utlb < 0)
+ return -ENODEV;
+
+ if (utlb >= mmu->num_utlbs)
+ return -EINVAL;
+
+ /* Create a device group and add the device to it. */
+ group = iommu_group_alloc();
+ if (IS_ERR(group)) {
+ dev_err(dev, "Failed to allocate IOMMU group\n");
+ return PTR_ERR(group);
+ }
+
+ ret = iommu_group_add_device(group, dev);
+ iommu_group_put(group);
+
+ if (ret < 0) {
+ dev_err(dev, "Failed to add device to IPMMU group\n");
+ return ret;
+ }
+
+ archdata = kzalloc(sizeof(*archdata), GFP_KERNEL);
+ if (!archdata) {
+ ret = -ENOMEM;
+ goto error;
+ }
+
+ archdata->mmu = mmu;
+ archdata->utlb = utlb;
+ dev->archdata.iommu = archdata;
+
+ /*
+ * Create the ARM mapping, used by the ARM DMA mapping core to allocate
+ * VAs. This will allocate a corresponding IOMMU domain.
+ *
+ * TODO:
+ * - Create one mapping per context (TLB).
+ * - Make the mapping size configurable ? We currently use a 2GB mapping
+ * at a 1GB offset to ensure that NULL VAs will fault.
+ */
+ if (!mmu->mapping) {
+ struct dma_iommu_mapping *mapping;
+
+ mapping = arm_iommu_create_mapping(&platform_bus_type,
+ SZ_1G, SZ_2G);
+ if (IS_ERR(mapping)) {
+ dev_err(mmu->dev, "failed to create ARM IOMMU mapping\n");
+ return PTR_ERR(mapping);
+ }
+
+ mmu->mapping = mapping;
+ }
+
+ /* Attach the ARM VA mapping to the device. */
+ ret = arm_iommu_attach_device(dev, mmu->mapping);
+ if (ret < 0) {
+ dev_err(dev, "Failed to attach device to VA mapping\n");
+ goto error;
+ }
+
+ return 0;
+
+error:
+ kfree(dev->archdata.iommu);
+ dev->archdata.iommu = NULL;
+ iommu_group_remove_device(dev);
+ return ret;
+}
+
+static void ipmmu_remove_device(struct device *dev)
+{
+ arm_iommu_detach_device(dev);
+ iommu_group_remove_device(dev);
+ kfree(dev->archdata.iommu);
+ dev->archdata.iommu = NULL;
+}
+
+static struct iommu_ops ipmmu_ops = {
+ .domain_init = ipmmu_domain_init,
+ .domain_destroy = ipmmu_domain_destroy,
+ .attach_dev = ipmmu_attach_device,
+ .detach_dev = ipmmu_detach_device,
+ .map = ipmmu_map,
+ .unmap = ipmmu_unmap,
+ .iova_to_phys = ipmmu_iova_to_phys,
+ .add_device = ipmmu_add_device,
+ .remove_device = ipmmu_remove_device,
+ .pgsize_bitmap = SZ_2M | SZ_64K | SZ_4K,
+};
+
+/* -----------------------------------------------------------------------------
+ * Probe/remove and init
+ */
+
+static void ipmmu_device_reset(struct ipmmu_vmsa_device *mmu)
+{
+ unsigned int i;
+
+ /* Disable all contexts. */
+ for (i = 0; i < 4; ++i)
+ ipmmu_write(mmu, i * IM_CTX_SIZE + IMCTR, 0);
+}
+
+static int ipmmu_probe(struct platform_device *pdev)
+{
+ struct ipmmu_vmsa_device *mmu;
+ struct resource *res;
+ int irq;
+ int ret;
+
+ if (!pdev->dev.platform_data) {
+ dev_err(&pdev->dev, "missing platform data\n");
+ return -EINVAL;
+ }
+
+ mmu = devm_kzalloc(&pdev->dev, sizeof(*mmu), GFP_KERNEL);
+ if (!mmu) {
+ dev_err(&pdev->dev, "cannot allocate device data\n");
+ return -ENOMEM;
+ }
+
+ mmu->dev = &pdev->dev;
+ mmu->pdata = pdev->dev.platform_data;
+ mmu->num_utlbs = 32;
+
+ /* Map I/O memory and request IRQ. */
+ res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+ mmu->base = devm_ioremap_resource(&pdev->dev, res);
+ if (IS_ERR(mmu->base))
+ return PTR_ERR(mmu->base);
+
+ irq = platform_get_irq(pdev, 0);
+ if (irq < 0) {
+ dev_err(&pdev->dev, "no IRQ found\n");
+ return irq;
+ }
+
+ ret = devm_request_irq(&pdev->dev, irq, ipmmu_irq, 0,
+ dev_name(&pdev->dev), mmu);
+ if (ret < 0) {
+ dev_err(&pdev->dev, "failed to request IRQ %d\n", irq);
+ return irq;
+ }
+
+ ipmmu_device_reset(mmu);
+
+ /*
+ * We can't create the ARM mapping here as it requires the bus to have
+ * an IOMMU, which only happens when bus_set_iommu() is called in
+ * ipmmu_init() after the probe function returns.
+ */
+
+ spin_lock(&ipmmu_devices_lock);
+ list_add(&mmu->list, &ipmmu_devices);
+ spin_unlock(&ipmmu_devices_lock);
+
+ platform_set_drvdata(pdev, mmu);
+
+ return 0;
+}
+
+static int ipmmu_remove(struct platform_device *pdev)
+{
+ struct ipmmu_vmsa_device *mmu = platform_get_drvdata(pdev);
+
+ spin_lock(&ipmmu_devices_lock);
+ list_del(&mmu->list);
+ spin_unlock(&ipmmu_devices_lock);
+
+ arm_iommu_release_mapping(mmu->mapping);
+
+ ipmmu_device_reset(mmu);
+
+ return 0;
+}
+
+static struct platform_driver ipmmu_driver = {
+ .driver = {
+ .owner = THIS_MODULE,
+ .name = "ipmmu-vmsa",
+ },
+ .probe = ipmmu_probe,
+ .remove = ipmmu_remove,
+};
+
+static int __init ipmmu_init(void)
+{
+ int ret;
+
+ ret = platform_driver_register(&ipmmu_driver);
+ if (ret < 0)
+ return ret;
+
+ if (!iommu_present(&platform_bus_type))
+ bus_set_iommu(&platform_bus_type, &ipmmu_ops);
+
+ return 0;
+}
+
+static void __exit ipmmu_exit(void)
+{
+ return platform_driver_unregister(&ipmmu_driver);
+}
+
+subsys_initcall(ipmmu_init);
+module_exit(ipmmu_exit);
+
+MODULE_DESCRIPTION("IOMMU API for Renesas VMSA-compatible IPMMU");
+MODULE_AUTHOR("Laurent Pinchart <laurent.pinchart@ideasonboard.com>");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/iommu/irq_remapping.c b/drivers/iommu/irq_remapping.c
index 228632c99ad..33c43952408 100644
--- a/drivers/iommu/irq_remapping.c
+++ b/drivers/iommu/irq_remapping.c
@@ -51,7 +51,7 @@ static void irq_remapping_disable_io_apic(void)
static int do_setup_msi_irqs(struct pci_dev *dev, int nvec)
{
- int node, ret, sub_handle, nvec_pow2, index = 0;
+ int ret, sub_handle, nvec_pow2, index = 0;
unsigned int irq;
struct msi_desc *msidesc;
@@ -61,8 +61,7 @@ static int do_setup_msi_irqs(struct pci_dev *dev, int nvec)
WARN_ON(msidesc->msi_attrib.multiple);
WARN_ON(msidesc->nvec_used);
- node = dev_to_node(&dev->dev);
- irq = __create_irqs(get_nr_irqs_gsi(), nvec, node);
+ irq = irq_alloc_hwirqs(nvec, dev_to_node(&dev->dev));
if (irq == 0)
return -ENOSPC;
@@ -89,7 +88,7 @@ static int do_setup_msi_irqs(struct pci_dev *dev, int nvec)
return 0;
error:
- destroy_irqs(irq, nvec);
+ irq_free_hwirqs(irq, nvec);
/*
* Restore altered MSI descriptor fields and prevent just destroyed
@@ -109,12 +108,11 @@ static int do_setup_msix_irqs(struct pci_dev *dev, int nvec)
unsigned int irq;
node = dev_to_node(&dev->dev);
- irq = get_nr_irqs_gsi();
sub_handle = 0;
list_for_each_entry(msidesc, &dev->msi_list, list) {
- irq = create_irq_nr(irq, node);
+ irq = irq_alloc_hwirq(node);
if (irq == 0)
return -1;
@@ -137,7 +135,7 @@ static int do_setup_msix_irqs(struct pci_dev *dev, int nvec)
return 0;
error:
- destroy_irq(irq);
+ irq_free_hwirq(irq);
return ret;
}
diff --git a/drivers/iommu/msm_iommu_dev.c b/drivers/iommu/msm_iommu_dev.c
index 08ba4972da9..61def7cb526 100644
--- a/drivers/iommu/msm_iommu_dev.c
+++ b/drivers/iommu/msm_iommu_dev.c
@@ -127,13 +127,12 @@ static void msm_iommu_reset(void __iomem *base, int ncb)
static int msm_iommu_probe(struct platform_device *pdev)
{
- struct resource *r, *r2;
+ struct resource *r;
struct clk *iommu_clk;
struct clk *iommu_pclk;
struct msm_iommu_drvdata *drvdata;
struct msm_iommu_dev *iommu_dev = pdev->dev.platform_data;
void __iomem *regs_base;
- resource_size_t len;
int ret, irq, par;
if (pdev->id == -1) {
@@ -178,35 +177,16 @@ static int msm_iommu_probe(struct platform_device *pdev)
iommu_clk = NULL;
r = platform_get_resource_byname(pdev, IORESOURCE_MEM, "physbase");
-
- if (!r) {
- ret = -ENODEV;
- goto fail_clk;
- }
-
- len = resource_size(r);
-
- r2 = request_mem_region(r->start, len, r->name);
- if (!r2) {
- pr_err("Could not request memory region: start=%p, len=%d\n",
- (void *) r->start, len);
- ret = -EBUSY;
+ regs_base = devm_ioremap_resource(&pdev->dev, r);
+ if (IS_ERR(regs_base)) {
+ ret = PTR_ERR(regs_base);
goto fail_clk;
}
- regs_base = ioremap(r2->start, len);
-
- if (!regs_base) {
- pr_err("Could not ioremap: start=%p, len=%d\n",
- (void *) r2->start, len);
- ret = -EBUSY;
- goto fail_mem;
- }
-
irq = platform_get_irq_byname(pdev, "secure_irq");
if (irq < 0) {
ret = -ENODEV;
- goto fail_io;
+ goto fail_clk;
}
msm_iommu_reset(regs_base, iommu_dev->ncb);
@@ -222,14 +202,14 @@ static int msm_iommu_probe(struct platform_device *pdev)
if (!par) {
pr_err("%s: Invalid PAR value detected\n", iommu_dev->name);
ret = -ENODEV;
- goto fail_io;
+ goto fail_clk;
}
ret = request_irq(irq, msm_iommu_fault_handler, 0,
"msm_iommu_secure_irpt_handler", drvdata);
if (ret) {
pr_err("Request IRQ %d failed with ret=%d\n", irq, ret);
- goto fail_io;
+ goto fail_clk;
}
@@ -250,10 +230,6 @@ static int msm_iommu_probe(struct platform_device *pdev)
clk_disable(iommu_pclk);
return 0;
-fail_io:
- iounmap(regs_base);
-fail_mem:
- release_mem_region(r->start, len);
fail_clk:
if (iommu_clk) {
clk_disable(iommu_clk);
diff --git a/drivers/iommu/omap-iommu-debug.c b/drivers/iommu/omap-iommu-debug.c
index d97fbe4fb9b..80fffba7f12 100644
--- a/drivers/iommu/omap-iommu-debug.c
+++ b/drivers/iommu/omap-iommu-debug.c
@@ -354,8 +354,8 @@ DEBUG_FOPS(mem);
return -ENOMEM; \
}
-#define DEBUG_ADD_FILE(name) __DEBUG_ADD_FILE(name, 600)
-#define DEBUG_ADD_FILE_RO(name) __DEBUG_ADD_FILE(name, 400)
+#define DEBUG_ADD_FILE(name) __DEBUG_ADD_FILE(name, 0600)
+#define DEBUG_ADD_FILE_RO(name) __DEBUG_ADD_FILE(name, 0400)
static int iommu_debug_register(struct device *dev, void *data)
{
diff --git a/drivers/iommu/omap-iommu.c b/drivers/iommu/omap-iommu.c
index bcd78a72063..895af06a667 100644
--- a/drivers/iommu/omap-iommu.c
+++ b/drivers/iommu/omap-iommu.c
@@ -23,6 +23,9 @@
#include <linux/spinlock.h>
#include <linux/io.h>
#include <linux/pm_runtime.h>
+#include <linux/of.h>
+#include <linux/of_iommu.h>
+#include <linux/of_irq.h>
#include <asm/cacheflush.h>
@@ -31,6 +34,9 @@
#include "omap-iopgtable.h"
#include "omap-iommu.h"
+#define to_iommu(dev) \
+ ((struct omap_iommu *)platform_get_drvdata(to_platform_device(dev)))
+
#define for_each_iotlb_cr(obj, n, __i, cr) \
for (__i = 0; \
(__i < (n)) && (cr = __iotlb_read_cr((obj), __i), true); \
@@ -146,13 +152,10 @@ static int iommu_enable(struct omap_iommu *obj)
struct platform_device *pdev = to_platform_device(obj->dev);
struct iommu_platform_data *pdata = pdev->dev.platform_data;
- if (!pdata)
- return -EINVAL;
-
if (!arch_iommu)
return -ENODEV;
- if (pdata->deassert_reset) {
+ if (pdata && pdata->deassert_reset) {
err = pdata->deassert_reset(pdev, pdata->reset_name);
if (err) {
dev_err(obj->dev, "deassert_reset failed: %d\n", err);
@@ -172,14 +175,11 @@ static void iommu_disable(struct omap_iommu *obj)
struct platform_device *pdev = to_platform_device(obj->dev);
struct iommu_platform_data *pdata = pdev->dev.platform_data;
- if (!pdata)
- return;
-
arch_iommu->disable(obj);
pm_runtime_put_sync(obj->dev);
- if (pdata->assert_reset)
+ if (pdata && pdata->assert_reset)
pdata->assert_reset(pdev, pdata->reset_name);
}
@@ -394,6 +394,7 @@ static void flush_iotlb_page(struct omap_iommu *obj, u32 da)
__func__, start, da, bytes);
iotlb_load_cr(obj, &cr);
iommu_write_reg(obj, 1, MMU_FLUSH_ENTRY);
+ break;
}
}
pm_runtime_put_sync(obj->dev);
@@ -523,7 +524,8 @@ static void flush_iopte_range(u32 *first, u32 *last)
static void iopte_free(u32 *iopte)
{
/* Note: freed iopte's must be clean ready for re-use */
- kmem_cache_free(iopte_cachep, iopte);
+ if (iopte)
+ kmem_cache_free(iopte_cachep, iopte);
}
static u32 *iopte_alloc(struct omap_iommu *obj, u32 *iopgd, u32 da)
@@ -863,7 +865,7 @@ static int device_match_by_alias(struct device *dev, void *data)
**/
static struct omap_iommu *omap_iommu_attach(const char *name, u32 *iopgd)
{
- int err = -ENOMEM;
+ int err;
struct device *dev;
struct omap_iommu *obj;
@@ -871,7 +873,7 @@ static struct omap_iommu *omap_iommu_attach(const char *name, u32 *iopgd)
(void *)name,
device_match_by_alias);
if (!dev)
- return NULL;
+ return ERR_PTR(-ENODEV);
obj = to_iommu(dev);
@@ -890,8 +892,10 @@ static struct omap_iommu *omap_iommu_attach(const char *name, u32 *iopgd)
goto err_enable;
flush_iotlb_all(obj);
- if (!try_module_get(obj->owner))
+ if (!try_module_get(obj->owner)) {
+ err = -ENODEV;
goto err_module;
+ }
spin_unlock(&obj->iommu_lock);
@@ -940,17 +944,41 @@ static int omap_iommu_probe(struct platform_device *pdev)
struct omap_iommu *obj;
struct resource *res;
struct iommu_platform_data *pdata = pdev->dev.platform_data;
+ struct device_node *of = pdev->dev.of_node;
- obj = kzalloc(sizeof(*obj) + MMU_REG_SIZE, GFP_KERNEL);
+ obj = devm_kzalloc(&pdev->dev, sizeof(*obj) + MMU_REG_SIZE, GFP_KERNEL);
if (!obj)
return -ENOMEM;
- obj->nr_tlb_entries = pdata->nr_tlb_entries;
- obj->name = pdata->name;
+ if (of) {
+ obj->name = dev_name(&pdev->dev);
+ obj->nr_tlb_entries = 32;
+ err = of_property_read_u32(of, "ti,#tlb-entries",
+ &obj->nr_tlb_entries);
+ if (err && err != -EINVAL)
+ return err;
+ if (obj->nr_tlb_entries != 32 && obj->nr_tlb_entries != 8)
+ return -EINVAL;
+ /*
+ * da_start and da_end are needed for omap-iovmm, so hardcode
+ * these values as used by OMAP3 ISP - the only user for
+ * omap-iovmm
+ */
+ obj->da_start = 0;
+ obj->da_end = 0xfffff000;
+ if (of_find_property(of, "ti,iommu-bus-err-back", NULL))
+ obj->has_bus_err_back = MMU_GP_REG_BUS_ERR_BACK_EN;
+ } else {
+ obj->nr_tlb_entries = pdata->nr_tlb_entries;
+ obj->name = pdata->name;
+ obj->da_start = pdata->da_start;
+ obj->da_end = pdata->da_end;
+ }
+ if (obj->da_end <= obj->da_start)
+ return -EINVAL;
+
obj->dev = &pdev->dev;
obj->ctx = (void *)obj + sizeof(*obj);
- obj->da_start = pdata->da_start;
- obj->da_end = pdata->da_end;
spin_lock_init(&obj->iommu_lock);
mutex_init(&obj->mmap_lock);
@@ -958,33 +986,18 @@ static int omap_iommu_probe(struct platform_device *pdev)
INIT_LIST_HEAD(&obj->mmap);
res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
- if (!res) {
- err = -ENODEV;
- goto err_mem;
- }
-
- res = request_mem_region(res->start, resource_size(res),
- dev_name(&pdev->dev));
- if (!res) {
- err = -EIO;
- goto err_mem;
- }
-
- obj->regbase = ioremap(res->start, resource_size(res));
- if (!obj->regbase) {
- err = -ENOMEM;
- goto err_ioremap;
- }
+ obj->regbase = devm_ioremap_resource(obj->dev, res);
+ if (IS_ERR(obj->regbase))
+ return PTR_ERR(obj->regbase);
irq = platform_get_irq(pdev, 0);
- if (irq < 0) {
- err = -ENODEV;
- goto err_irq;
- }
- err = request_irq(irq, iommu_fault_handler, IRQF_SHARED,
- dev_name(&pdev->dev), obj);
+ if (irq < 0)
+ return -ENODEV;
+
+ err = devm_request_irq(obj->dev, irq, iommu_fault_handler, IRQF_SHARED,
+ dev_name(obj->dev), obj);
if (err < 0)
- goto err_irq;
+ return err;
platform_set_drvdata(pdev, obj);
pm_runtime_irq_safe(obj->dev);
@@ -992,42 +1005,34 @@ static int omap_iommu_probe(struct platform_device *pdev)
dev_info(&pdev->dev, "%s registered\n", obj->name);
return 0;
-
-err_irq:
- iounmap(obj->regbase);
-err_ioremap:
- release_mem_region(res->start, resource_size(res));
-err_mem:
- kfree(obj);
- return err;
}
static int omap_iommu_remove(struct platform_device *pdev)
{
- int irq;
- struct resource *res;
struct omap_iommu *obj = platform_get_drvdata(pdev);
iopgtable_clear_entry_all(obj);
- irq = platform_get_irq(pdev, 0);
- free_irq(irq, obj);
- res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
- release_mem_region(res->start, resource_size(res));
- iounmap(obj->regbase);
-
pm_runtime_disable(obj->dev);
dev_info(&pdev->dev, "%s removed\n", obj->name);
- kfree(obj);
return 0;
}
+static struct of_device_id omap_iommu_of_match[] = {
+ { .compatible = "ti,omap2-iommu" },
+ { .compatible = "ti,omap4-iommu" },
+ { .compatible = "ti,dra7-iommu" },
+ {},
+};
+MODULE_DEVICE_TABLE(of, omap_iommu_of_match);
+
static struct platform_driver omap_iommu_driver = {
.probe = omap_iommu_probe,
.remove = omap_iommu_remove,
.driver = {
.name = "omap-iommu",
+ .of_match_table = of_match_ptr(omap_iommu_of_match),
},
};
@@ -1036,19 +1041,18 @@ static void iopte_cachep_ctor(void *iopte)
clean_dcache_area(iopte, IOPTE_TABLE_SIZE);
}
-static u32 iotlb_init_entry(struct iotlb_entry *e, u32 da, u32 pa,
- u32 flags)
+static u32 iotlb_init_entry(struct iotlb_entry *e, u32 da, u32 pa, int pgsz)
{
memset(e, 0, sizeof(*e));
e->da = da;
e->pa = pa;
- e->valid = 1;
+ e->valid = MMU_CAM_V;
/* FIXME: add OMAP1 support */
- e->pgsz = flags & MMU_CAM_PGSZ_MASK;
- e->endian = flags & MMU_RAM_ENDIAN_MASK;
- e->elsz = flags & MMU_RAM_ELSZ_MASK;
- e->mixed = flags & MMU_RAM_MIXED_MASK;
+ e->pgsz = pgsz;
+ e->endian = MMU_RAM_ENDIAN_LITTLE;
+ e->elsz = MMU_RAM_ELSZ_8;
+ e->mixed = 0;
return iopgsz_to_bytes(e->pgsz);
}
@@ -1061,9 +1065,8 @@ static int omap_iommu_map(struct iommu_domain *domain, unsigned long da,
struct device *dev = oiommu->dev;
struct iotlb_entry e;
int omap_pgsz;
- u32 ret, flags;
+ u32 ret;
- /* we only support mapping a single iommu page for now */
omap_pgsz = bytes_to_iopgsz(bytes);
if (omap_pgsz < 0) {
dev_err(dev, "invalid size to map: %d\n", bytes);
@@ -1072,9 +1075,7 @@ static int omap_iommu_map(struct iommu_domain *domain, unsigned long da,
dev_dbg(dev, "mapping da 0x%lx to pa 0x%x size 0x%x\n", da, pa, bytes);
- flags = omap_pgsz | prot;
-
- iotlb_init_entry(&e, da, pa, flags);
+ iotlb_init_entry(&e, da, pa, omap_pgsz);
ret = omap_iopgtable_store_entry(oiommu, &e);
if (ret)
@@ -1247,12 +1248,49 @@ static phys_addr_t omap_iommu_iova_to_phys(struct iommu_domain *domain,
return ret;
}
-static int omap_iommu_domain_has_cap(struct iommu_domain *domain,
- unsigned long cap)
+static int omap_iommu_add_device(struct device *dev)
{
+ struct omap_iommu_arch_data *arch_data;
+ struct device_node *np;
+
+ /*
+ * Allocate the archdata iommu structure for DT-based devices.
+ *
+ * TODO: Simplify this when removing non-DT support completely from the
+ * IOMMU users.
+ */
+ if (!dev->of_node)
+ return 0;
+
+ np = of_parse_phandle(dev->of_node, "iommus", 0);
+ if (!np)
+ return 0;
+
+ arch_data = kzalloc(sizeof(*arch_data), GFP_KERNEL);
+ if (!arch_data) {
+ of_node_put(np);
+ return -ENOMEM;
+ }
+
+ arch_data->name = kstrdup(dev_name(dev), GFP_KERNEL);
+ dev->archdata.iommu = arch_data;
+
+ of_node_put(np);
+
return 0;
}
+static void omap_iommu_remove_device(struct device *dev)
+{
+ struct omap_iommu_arch_data *arch_data = dev->archdata.iommu;
+
+ if (!dev->of_node || !arch_data)
+ return;
+
+ kfree(arch_data->name);
+ kfree(arch_data);
+}
+
static struct iommu_ops omap_iommu_ops = {
.domain_init = omap_iommu_domain_init,
.domain_destroy = omap_iommu_domain_destroy,
@@ -1261,7 +1299,8 @@ static struct iommu_ops omap_iommu_ops = {
.map = omap_iommu_map,
.unmap = omap_iommu_unmap,
.iova_to_phys = omap_iommu_iova_to_phys,
- .domain_has_cap = omap_iommu_domain_has_cap,
+ .add_device = omap_iommu_add_device,
+ .remove_device = omap_iommu_remove_device,
.pgsize_bitmap = OMAP_IOMMU_PGSIZES,
};
diff --git a/drivers/iommu/omap-iommu.h b/drivers/iommu/omap-iommu.h
index 12008420660..ea920c3e94f 100644
--- a/drivers/iommu/omap-iommu.h
+++ b/drivers/iommu/omap-iommu.h
@@ -52,6 +52,8 @@ struct omap_iommu {
void *ctx; /* iommu context: registres saved area */
u32 da_start;
u32 da_end;
+
+ int has_bus_err_back;
};
struct cr_regs {
@@ -130,6 +132,7 @@ static inline struct omap_iommu *dev_to_omap_iommu(struct device *dev)
#define MMU_READ_CAM 0x68
#define MMU_READ_RAM 0x6c
#define MMU_EMU_FAULT_AD 0x70
+#define MMU_GP_REG 0x88
#define MMU_REG_SIZE 256
@@ -163,6 +166,8 @@ static inline struct omap_iommu *dev_to_omap_iommu(struct device *dev)
#define MMU_RAM_MIXED_MASK (1 << MMU_RAM_MIXED_SHIFT)
#define MMU_RAM_MIXED MMU_RAM_MIXED_MASK
+#define MMU_GP_REG_BUS_ERR_BACK_EN 0x1
+
/*
* utilities for super page(16MB, 1MB, 64KB and 4KB)
*/
diff --git a/drivers/iommu/omap-iommu2.c b/drivers/iommu/omap-iommu2.c
index d745094a69d..5e1ea3b0bf1 100644
--- a/drivers/iommu/omap-iommu2.c
+++ b/drivers/iommu/omap-iommu2.c
@@ -98,6 +98,9 @@ static int omap2_iommu_enable(struct omap_iommu *obj)
iommu_write_reg(obj, pa, MMU_TTB);
+ if (obj->has_bus_err_back)
+ iommu_write_reg(obj, MMU_GP_REG_BUS_ERR_BACK_EN, MMU_GP_REG);
+
__iommu_set_twl(obj, true);
return 0;
diff --git a/drivers/iommu/omap-iopgtable.h b/drivers/iommu/omap-iopgtable.h
index b6f9a51746c..f891683e3f0 100644
--- a/drivers/iommu/omap-iopgtable.h
+++ b/drivers/iommu/omap-iopgtable.h
@@ -93,6 +93,3 @@ static inline phys_addr_t omap_iommu_translate(u32 d, u32 va, u32 mask)
/* to find an entry in the second-level page table. */
#define iopte_index(da) (((da) >> IOPTE_SHIFT) & (PTRS_PER_IOPTE - 1))
#define iopte_offset(iopgd, da) (iopgd_page_vaddr(iopgd) + iopte_index(da))
-
-#define to_iommu(dev) \
- (platform_get_drvdata(to_platform_device(dev)))
diff --git a/drivers/iommu/shmobile-iommu.c b/drivers/iommu/shmobile-iommu.c
index 7a3b928fad1..464acda0bbc 100644
--- a/drivers/iommu/shmobile-iommu.c
+++ b/drivers/iommu/shmobile-iommu.c
@@ -343,7 +343,7 @@ static int shmobile_iommu_add_device(struct device *dev)
mapping = archdata->iommu_mapping;
if (!mapping) {
mapping = arm_iommu_create_mapping(&platform_bus_type, 0,
- L1_LEN << 20, 0);
+ L1_LEN << 20);
if (IS_ERR(mapping))
return PTR_ERR(mapping);
archdata->iommu_mapping = mapping;
diff --git a/drivers/iommu/shmobile-ipmmu.c b/drivers/iommu/shmobile-ipmmu.c
index e3bc2e19b6d..bd97adecb1f 100644
--- a/drivers/iommu/shmobile-ipmmu.c
+++ b/drivers/iommu/shmobile-ipmmu.c
@@ -94,11 +94,6 @@ static int ipmmu_probe(struct platform_device *pdev)
struct resource *res;
struct shmobile_ipmmu_platform_data *pdata = pdev->dev.platform_data;
- res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
- if (!res) {
- dev_err(&pdev->dev, "cannot get platform resources\n");
- return -ENOENT;
- }
ipmmu = devm_kzalloc(&pdev->dev, sizeof(*ipmmu), GFP_KERNEL);
if (!ipmmu) {
dev_err(&pdev->dev, "cannot allocate device data\n");
@@ -106,19 +101,18 @@ static int ipmmu_probe(struct platform_device *pdev)
}
spin_lock_init(&ipmmu->flush_lock);
ipmmu->dev = &pdev->dev;
- ipmmu->ipmmu_base = devm_ioremap_nocache(&pdev->dev, res->start,
- resource_size(res));
- if (!ipmmu->ipmmu_base) {
- dev_err(&pdev->dev, "ioremap_nocache failed\n");
- return -ENOMEM;
- }
+
+ res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+ ipmmu->ipmmu_base = devm_ioremap_resource(&pdev->dev, res);
+ if (IS_ERR(ipmmu->ipmmu_base))
+ return PTR_ERR(ipmmu->ipmmu_base);
+
ipmmu->dev_names = pdata->dev_names;
ipmmu->num_dev_names = pdata->num_dev_names;
platform_set_drvdata(pdev, ipmmu);
ipmmu_reg_write(ipmmu, IMCTR1, 0x0); /* disable TLB */
ipmmu_reg_write(ipmmu, IMCTR2, 0x0); /* disable PMB */
- ipmmu_iommu_init(ipmmu);
- return 0;
+ return ipmmu_iommu_init(ipmmu);
}
static struct platform_driver ipmmu_driver = {
generated by cgit v1.2.3-70-g09d2 (git 2.46.0) at 2025-11-09 21:09:50 +0000