/*
* IOMMU implementation for Cell Broadband Processor Architecture
*
- * (C) Copyright IBM Corporation 2006
+ * (C) Copyright IBM Corporation 2006-2008
*
* Author: Jeremy Kerr <jk@ozlabs.org>
*
#include <linux/init.h>
#include <linux/interrupt.h>
#include <linux/notifier.h>
+#include <linux/of.h>
#include <linux/of_platform.h>
#include <asm/prom.h>
/* IOMMU sizing */
#define IO_SEGMENT_SHIFT 28
-#define IO_PAGENO_BITS (IO_SEGMENT_SHIFT - IOMMU_PAGE_SHIFT)
+#define IO_PAGENO_BITS(shift) (IO_SEGMENT_SHIFT - (shift))
/* The high bit needs to be set on every DMA address */
#define SPIDER_DMA_OFFSET 0x80000000ul
struct cbe_iommu *iommu;
unsigned long offset;
unsigned long size;
- unsigned long pte_offset;
unsigned int ioid;
struct iommu_table table;
};
(window->ioid & IOPTE_IOID_Mask);
#endif
- io_pte = (unsigned long *)tbl->it_base + (index - window->pte_offset);
+ io_pte = (unsigned long *)tbl->it_base + (index - tbl->it_offset);
for (i = 0; i < npages; i++, uaddr += IOMMU_PAGE_SIZE)
io_pte[i] = base_pte | (__pa(uaddr) & IOPTE_RPN_Mask);
| (window->ioid & IOPTE_IOID_Mask);
#endif
- io_pte = (unsigned long *)tbl->it_base + (index - window->pte_offset);
+ io_pte = (unsigned long *)tbl->it_base + (index - tbl->it_offset);
for (i = 0; i < npages; i++)
io_pte[i] = pte;
return -ENODEV;
}
-static void cell_iommu_setup_hardware(struct cbe_iommu *iommu, unsigned long size)
+static void cell_iommu_setup_stab(struct cbe_iommu *iommu,
+ unsigned long dbase, unsigned long dsize,
+ unsigned long fbase, unsigned long fsize)
{
struct page *page;
- int ret, i;
- unsigned long reg, segments, pages_per_segment, ptab_size, n_pte_pages;
- unsigned long xlate_base;
- unsigned int virq;
-
- if (cell_iommu_find_ioc(iommu->nid, &xlate_base))
- panic("%s: missing IOC register mappings for node %d\n",
- __FUNCTION__, iommu->nid);
-
- iommu->xlate_regs = ioremap(xlate_base, IOC_Reg_Size);
- iommu->cmd_regs = iommu->xlate_regs + IOC_IOCmd_Offset;
+ unsigned long segments, stab_size;
- segments = size >> IO_SEGMENT_SHIFT;
- pages_per_segment = 1ull << IO_PAGENO_BITS;
+ segments = max(dbase + dsize, fbase + fsize) >> IO_SEGMENT_SHIFT;
- pr_debug("%s: iommu[%d]: segments: %lu, pages per segment: %lu\n",
- __FUNCTION__, iommu->nid, segments, pages_per_segment);
+ pr_debug("%s: iommu[%d]: segments: %lu\n",
+ __FUNCTION__, iommu->nid, segments);
/* set up the segment table */
- page = alloc_pages_node(iommu->nid, GFP_KERNEL, 0);
+ stab_size = segments * sizeof(unsigned long);
+ page = alloc_pages_node(iommu->nid, GFP_KERNEL, get_order(stab_size));
BUG_ON(!page);
iommu->stab = page_address(page);
- clear_page(iommu->stab);
+ memset(iommu->stab, 0, stab_size);
+}
+
+static unsigned long *cell_iommu_alloc_ptab(struct cbe_iommu *iommu,
+ unsigned long base, unsigned long size, unsigned long gap_base,
+ unsigned long gap_size, unsigned long page_shift)
+{
+ struct page *page;
+ int i;
+ unsigned long reg, segments, pages_per_segment, ptab_size,
+ n_pte_pages, start_seg, *ptab;
+
+ start_seg = base >> IO_SEGMENT_SHIFT;
+ segments = size >> IO_SEGMENT_SHIFT;
+ pages_per_segment = 1ull << IO_PAGENO_BITS(page_shift);
+ /* PTEs for each segment must start on a 4K bounday */
+ pages_per_segment = max(pages_per_segment,
+ (1 << 12) / sizeof(unsigned long));
- /* ... and the page tables. Since these are contiguous, we can treat
- * the page tables as one array of ptes, like pSeries does.
- */
ptab_size = segments * pages_per_segment * sizeof(unsigned long);
pr_debug("%s: iommu[%d]: ptab_size: %lu, order: %d\n", __FUNCTION__,
iommu->nid, ptab_size, get_order(ptab_size));
page = alloc_pages_node(iommu->nid, GFP_KERNEL, get_order(ptab_size));
BUG_ON(!page);
- iommu->ptab = page_address(page);
- memset(iommu->ptab, 0, ptab_size);
-
- /* allocate a bogus page for the end of each mapping */
- page = alloc_pages_node(iommu->nid, GFP_KERNEL, 0);
- BUG_ON(!page);
- iommu->pad_page = page_address(page);
- clear_page(iommu->pad_page);
+ ptab = page_address(page);
+ memset(ptab, 0, ptab_size);
- /* number of pages needed for a page table */
- n_pte_pages = (pages_per_segment *
- sizeof(unsigned long)) >> IOMMU_PAGE_SHIFT;
+ /* number of 4K pages needed for a page table */
+ n_pte_pages = (pages_per_segment * sizeof(unsigned long)) >> 12;
pr_debug("%s: iommu[%d]: stab at %p, ptab at %p, n_pte_pages: %lu\n",
- __FUNCTION__, iommu->nid, iommu->stab, iommu->ptab,
+ __FUNCTION__, iommu->nid, iommu->stab, ptab,
n_pte_pages);
/* initialise the STEs */
reg = IOSTE_V | ((n_pte_pages - 1) << 5);
- if (IOMMU_PAGE_SIZE == 0x1000)
- reg |= IOSTE_PS_4K;
- else if (IOMMU_PAGE_SIZE == 0x10000)
- reg |= IOSTE_PS_64K;
- else {
- extern void __unknown_page_size_error(void);
- __unknown_page_size_error();
+ switch (page_shift) {
+ case 12: reg |= IOSTE_PS_4K; break;
+ case 16: reg |= IOSTE_PS_64K; break;
+ case 20: reg |= IOSTE_PS_1M; break;
+ case 24: reg |= IOSTE_PS_16M; break;
+ default: BUG();
}
+ gap_base = gap_base >> IO_SEGMENT_SHIFT;
+ gap_size = gap_size >> IO_SEGMENT_SHIFT;
+
pr_debug("Setting up IOMMU stab:\n");
- for (i = 0; i * (1ul << IO_SEGMENT_SHIFT) < size; i++) {
- iommu->stab[i] = reg |
- (__pa(iommu->ptab) + n_pte_pages * IOMMU_PAGE_SIZE * i);
+ for (i = start_seg; i < (start_seg + segments); i++) {
+ if (i >= gap_base && i < (gap_base + gap_size)) {
+ pr_debug("\toverlap at %d, skipping\n", i);
+ continue;
+ }
+ iommu->stab[i] = reg | (__pa(ptab) + (n_pte_pages << 12) *
+ (i - start_seg));
pr_debug("\t[%d] 0x%016lx\n", i, iommu->stab[i]);
}
+ return ptab;
+}
+
+static void cell_iommu_enable_hardware(struct cbe_iommu *iommu)
+{
+ int ret;
+ unsigned long reg, xlate_base;
+ unsigned int virq;
+
+ if (cell_iommu_find_ioc(iommu->nid, &xlate_base))
+ panic("%s: missing IOC register mappings for node %d\n",
+ __FUNCTION__, iommu->nid);
+
+ iommu->xlate_regs = ioremap(xlate_base, IOC_Reg_Size);
+ iommu->cmd_regs = iommu->xlate_regs + IOC_IOCmd_Offset;
+
/* ensure that the STEs have updated */
mb();
out_be64(iommu->cmd_regs + IOC_IOCmd_Cfg, reg);
}
+static void cell_iommu_setup_hardware(struct cbe_iommu *iommu,
+ unsigned long base, unsigned long size)
+{
+ cell_iommu_setup_stab(iommu, base, size, 0, 0);
+ iommu->ptab = cell_iommu_alloc_ptab(iommu, base, size, 0, 0,
+ IOMMU_PAGE_SHIFT);
+ cell_iommu_enable_hardware(iommu);
+}
+
#if 0/* Unused for now */
static struct iommu_window *find_window(struct cbe_iommu *iommu,
unsigned long offset, unsigned long size)
}
#endif
+static inline u32 cell_iommu_get_ioid(struct device_node *np)
+{
+ const u32 *ioid;
+
+ ioid = of_get_property(np, "ioid", NULL);
+ if (ioid == NULL) {
+ printk(KERN_WARNING "iommu: missing ioid for %s using 0\n",
+ np->full_name);
+ return 0;
+ }
+
+ return *ioid;
+}
+
static struct iommu_window * __init
cell_iommu_setup_window(struct cbe_iommu *iommu, struct device_node *np,
unsigned long offset, unsigned long size,
unsigned long pte_offset)
{
struct iommu_window *window;
- const unsigned int *ioid;
+ struct page *page;
+ u32 ioid;
- ioid = of_get_property(np, "ioid", NULL);
- if (ioid == NULL)
- printk(KERN_WARNING "iommu: missing ioid for %s using 0\n",
- np->full_name);
+ ioid = cell_iommu_get_ioid(np);
window = kmalloc_node(sizeof(*window), GFP_KERNEL, iommu->nid);
BUG_ON(window == NULL);
window->offset = offset;
window->size = size;
- window->ioid = ioid ? *ioid : 0;
+ window->ioid = ioid;
window->iommu = iommu;
- window->pte_offset = pte_offset;
window->table.it_blocksize = 16;
window->table.it_base = (unsigned long)iommu->ptab;
window->table.it_index = iommu->nid;
- window->table.it_offset = (offset >> IOMMU_PAGE_SHIFT) +
- window->pte_offset;
+ window->table.it_offset = (offset >> IOMMU_PAGE_SHIFT) + pte_offset;
window->table.it_size = size >> IOMMU_PAGE_SHIFT;
iommu_init_table(&window->table, iommu->nid);
* This code also assumes that we have a window that starts at 0,
* which is the case on all spider based blades.
*/
+ page = alloc_pages_node(iommu->nid, GFP_KERNEL, 0);
+ BUG_ON(!page);
+ iommu->pad_page = page_address(page);
+ clear_page(iommu->pad_page);
+
__set_bit(0, window->table.it_map);
tce_build_cell(&window->table, window->table.it_offset, 1,
(unsigned long)iommu->pad_page, DMA_TO_DEVICE);
static unsigned long cell_dma_direct_offset;
-static void cell_dma_dev_setup(struct device *dev)
+static unsigned long dma_iommu_fixed_base;
+struct dma_mapping_ops dma_iommu_fixed_ops;
+
+static void cell_dma_dev_setup_iommu(struct device *dev)
{
struct iommu_window *window;
struct cbe_iommu *iommu;
struct dev_archdata *archdata = &dev->archdata;
- if (get_pci_dma_ops() == &dma_direct_ops) {
- archdata->dma_data = (void *)cell_dma_direct_offset;
- return;
- }
-
/* Current implementation uses the first window available in that
* node's iommu. We -might- do something smarter later though it may
* never be necessary
archdata->dma_data = &window->table;
}
+static void cell_dma_dev_setup_fixed(struct device *dev);
+
+static void cell_dma_dev_setup(struct device *dev)
+{
+ struct dev_archdata *archdata = &dev->archdata;
+
+ /* Order is important here, these are not mutually exclusive */
+ if (get_dma_ops(dev) == &dma_iommu_fixed_ops)
+ cell_dma_dev_setup_fixed(dev);
+ else if (get_pci_dma_ops() == &dma_iommu_ops)
+ cell_dma_dev_setup_iommu(dev);
+ else if (get_pci_dma_ops() == &dma_direct_ops)
+ archdata->dma_data = (void *)cell_dma_direct_offset;
+ else
+ BUG();
+}
+
static void cell_pci_dma_dev_setup(struct pci_dev *dev)
{
cell_dma_dev_setup(&dev->dev);
return 0;
}
-static void __init cell_iommu_init_one(struct device_node *np, unsigned long offset)
+static struct cbe_iommu * __init cell_iommu_alloc(struct device_node *np)
{
struct cbe_iommu *iommu;
- unsigned long base, size;
int nid, i;
/* Get node ID */
if (nid < 0) {
printk(KERN_ERR "iommu: failed to get node for %s\n",
np->full_name);
- return;
+ return NULL;
}
pr_debug("iommu: setting up iommu for node %d (%s)\n",
nid, np->full_name);
if (cbe_nr_iommus >= NR_IOMMUS) {
printk(KERN_ERR "iommu: too many IOMMUs detected ! (%s)\n",
np->full_name);
- return;
+ return NULL;
}
/* Init base fields */
snprintf(iommu->name, sizeof(iommu->name), "iommu%d", i);
INIT_LIST_HEAD(&iommu->windows);
+ return iommu;
+}
+
+static void __init cell_iommu_init_one(struct device_node *np,
+ unsigned long offset)
+{
+ struct cbe_iommu *iommu;
+ unsigned long base, size;
+
+ iommu = cell_iommu_alloc(np);
+ if (!iommu)
+ return;
+
/* Obtain a window for it */
cell_iommu_get_window(np, &base, &size);
base, base + size - 1);
/* Initialize the hardware */
- cell_iommu_setup_hardware(iommu, size);
+ cell_iommu_setup_hardware(iommu, base, size);
/* Setup the iommu_table */
cell_iommu_setup_window(iommu, np, base, size,
return 0;
}
+/*
+ * Fixed IOMMU mapping support
+ *
+ * This code adds support for setting up a fixed IOMMU mapping on certain
+ * cell machines. For 64-bit devices this avoids the performance overhead of
+ * mapping and unmapping pages at runtime. 32-bit devices are unable to use
+ * the fixed mapping.
+ *
+ * The fixed mapping is established at boot, and maps all of physical memory
+ * 1:1 into device space at some offset. On machines with < 30 GB of memory
+ * we setup the fixed mapping immediately above the normal IOMMU window.
+ *
+ * For example a machine with 4GB of memory would end up with the normal
+ * IOMMU window from 0-2GB and the fixed mapping window from 2GB to 6GB. In
+ * this case a 64-bit device wishing to DMA to 1GB would be told to DMA to
+ * 3GB, plus any offset required by firmware. The firmware offset is encoded
+ * in the "dma-ranges" property.
+ *
+ * On machines with 30GB or more of memory, we are unable to place the fixed
+ * mapping above the normal IOMMU window as we would run out of address space.
+ * Instead we move the normal IOMMU window to coincide with the hash page
+ * table, this region does not need to be part of the fixed mapping as no
+ * device should ever be DMA'ing to it. We then setup the fixed mapping
+ * from 0 to 32GB.
+ */
+
+static u64 cell_iommu_get_fixed_address(struct device *dev)
+{
+ u64 cpu_addr, size, best_size, dev_addr = OF_BAD_ADDR;
+ struct device_node *np;
+ const u32 *ranges = NULL;
+ int i, len, best, naddr, nsize, pna, range_size;
+
+ np = of_node_get(dev->archdata.of_node);
+ while (1) {
+ naddr = of_n_addr_cells(np);
+ nsize = of_n_size_cells(np);
+ np = of_get_next_parent(np);
+ if (!np)
+ break;
+
+ ranges = of_get_property(np, "dma-ranges", &len);
+
+ /* Ignore empty ranges, they imply no translation required */
+ if (ranges && len > 0)
+ break;
+ }
+
+ if (!ranges) {
+ dev_dbg(dev, "iommu: no dma-ranges found\n");
+ goto out;
+ }
+
+ len /= sizeof(u32);
+
+ pna = of_n_addr_cells(np);
+ range_size = naddr + nsize + pna;
+
+ /* dma-ranges format:
+ * child addr : naddr cells
+ * parent addr : pna cells
+ * size : nsize cells
+ */
+ for (i = 0, best = -1, best_size = 0; i < len; i += range_size) {
+ cpu_addr = of_translate_dma_address(np, ranges + i + naddr);
+ size = of_read_number(ranges + i + naddr + pna, nsize);
+
+ if (cpu_addr == 0 && size > best_size) {
+ best = i;
+ best_size = size;
+ }
+ }
+
+ if (best >= 0) {
+ dev_addr = of_read_number(ranges + best, naddr);
+ } else
+ dev_dbg(dev, "iommu: no suitable range found!\n");
+
+out:
+ of_node_put(np);
+
+ return dev_addr;
+}
+
+static int dma_set_mask_and_switch(struct device *dev, u64 dma_mask)
+{
+ if (!dev->dma_mask || !dma_supported(dev, dma_mask))
+ return -EIO;
+
+ if (dma_mask == DMA_BIT_MASK(64) &&
+ cell_iommu_get_fixed_address(dev) != OF_BAD_ADDR)
+ {
+ dev_dbg(dev, "iommu: 64-bit OK, using fixed ops\n");
+ set_dma_ops(dev, &dma_iommu_fixed_ops);
+ } else {
+ dev_dbg(dev, "iommu: not 64-bit, using default ops\n");
+ set_dma_ops(dev, get_pci_dma_ops());
+ }
+
+ cell_dma_dev_setup(dev);
+
+ *dev->dma_mask = dma_mask;
+
+ return 0;
+}
+
+static void cell_dma_dev_setup_fixed(struct device *dev)
+{
+ struct dev_archdata *archdata = &dev->archdata;
+ u64 addr;
+
+ addr = cell_iommu_get_fixed_address(dev) + dma_iommu_fixed_base;
+ archdata->dma_data = (void *)addr;
+
+ dev_dbg(dev, "iommu: fixed addr = %lx\n", addr);
+}
+
+static void insert_16M_pte(unsigned long addr, unsigned long *ptab,
+ unsigned long base_pte)
+{
+ unsigned long segment, offset;
+
+ segment = addr >> IO_SEGMENT_SHIFT;
+ offset = (addr >> 24) - (segment << IO_PAGENO_BITS(24));
+ ptab = ptab + (segment * (1 << 12) / sizeof(unsigned long));
+
+ pr_debug("iommu: addr %lx ptab %p segment %lx offset %lx\n",
+ addr, ptab, segment, offset);
+
+ ptab[offset] = base_pte | (__pa(addr) & IOPTE_RPN_Mask);
+}
+
+static void cell_iommu_setup_fixed_ptab(struct cbe_iommu *iommu,
+ struct device_node *np, unsigned long dbase, unsigned long dsize,
+ unsigned long fbase, unsigned long fsize)
+{
+ unsigned long base_pte, uaddr, ioaddr, *ptab;
+
+ ptab = cell_iommu_alloc_ptab(iommu, fbase, fsize, dbase, dsize, 24);
+
+ dma_iommu_fixed_base = fbase;
+
+ pr_debug("iommu: mapping 0x%lx pages from 0x%lx\n", fsize, fbase);
+
+ base_pte = IOPTE_PP_W | IOPTE_PP_R | IOPTE_M | IOPTE_SO_RW
+ | (cell_iommu_get_ioid(np) & IOPTE_IOID_Mask);
+
+ for (uaddr = 0; uaddr < fsize; uaddr += (1 << 24)) {
+ /* Don't touch the dynamic region */
+ ioaddr = uaddr + fbase;
+ if (ioaddr >= dbase && ioaddr < (dbase + dsize)) {
+ pr_debug("iommu: fixed/dynamic overlap, skipping\n");
+ continue;
+ }
+
+ insert_16M_pte(uaddr, ptab, base_pte);
+ }
+
+ mb();
+}
+
+static int __init cell_iommu_fixed_mapping_init(void)
+{
+ unsigned long dbase, dsize, fbase, fsize, hbase, hend;
+ struct cbe_iommu *iommu;
+ struct device_node *np;
+
+ /* The fixed mapping is only supported on axon machines */
+ np = of_find_node_by_name(NULL, "axon");
+ if (!np) {
+ pr_debug("iommu: fixed mapping disabled, no axons found\n");
+ return -1;
+ }
+
+ /* We must have dma-ranges properties for fixed mapping to work */
+ for (np = NULL; (np = of_find_all_nodes(np));) {
+ if (of_find_property(np, "dma-ranges", NULL))
+ break;
+ }
+ of_node_put(np);
+
+ if (!np) {
+ pr_debug("iommu: no dma-ranges found, no fixed mapping\n");
+ return -1;
+ }
+
+ /* The default setup is to have the fixed mapping sit after the
+ * dynamic region, so find the top of the largest IOMMU window
+ * on any axon, then add the size of RAM and that's our max value.
+ * If that is > 32GB we have to do other shennanigans.
+ */
+ fbase = 0;
+ for_each_node_by_name(np, "axon") {
+ cell_iommu_get_window(np, &dbase, &dsize);
+ fbase = max(fbase, dbase + dsize);
+ }
+
+ fbase = _ALIGN_UP(fbase, 1 << IO_SEGMENT_SHIFT);
+ fsize = lmb_phys_mem_size();
+
+ if ((fbase + fsize) <= 0x800000000)
+ hbase = 0; /* use the device tree window */
+ else {
+ /* If we're over 32 GB we need to cheat. We can't map all of
+ * RAM with the fixed mapping, and also fit the dynamic
+ * region. So try to place the dynamic region where the hash
+ * table sits, drivers never need to DMA to it, we don't
+ * need a fixed mapping for that area.
+ */
+ if (!htab_address) {
+ pr_debug("iommu: htab is NULL, on LPAR? Huh?\n");
+ return -1;
+ }
+ hbase = __pa(htab_address);
+ hend = hbase + htab_size_bytes;
+
+ /* The window must start and end on a segment boundary */
+ if ((hbase != _ALIGN_UP(hbase, 1 << IO_SEGMENT_SHIFT)) ||
+ (hend != _ALIGN_UP(hend, 1 << IO_SEGMENT_SHIFT))) {
+ pr_debug("iommu: hash window not segment aligned\n");
+ return -1;
+ }
+
+ /* Check the hash window fits inside the real DMA window */
+ for_each_node_by_name(np, "axon") {
+ cell_iommu_get_window(np, &dbase, &dsize);
+
+ if (hbase < dbase || (hend > (dbase + dsize))) {
+ pr_debug("iommu: hash window doesn't fit in"
+ "real DMA window\n");
+ return -1;
+ }
+ }
+
+ fbase = 0;
+ }
+
+ /* Setup the dynamic regions */
+ for_each_node_by_name(np, "axon") {
+ iommu = cell_iommu_alloc(np);
+ BUG_ON(!iommu);
+
+ if (hbase == 0)
+ cell_iommu_get_window(np, &dbase, &dsize);
+ else {
+ dbase = hbase;
+ dsize = htab_size_bytes;
+ }
+
+ printk(KERN_DEBUG "iommu: node %d, dynamic window 0x%lx-0x%lx "
+ "fixed window 0x%lx-0x%lx\n", iommu->nid, dbase,
+ dbase + dsize, fbase, fbase + fsize);
+
+ cell_iommu_setup_stab(iommu, dbase, dsize, fbase, fsize);
+ iommu->ptab = cell_iommu_alloc_ptab(iommu, dbase, dsize, 0, 0,
+ IOMMU_PAGE_SHIFT);
+ cell_iommu_setup_fixed_ptab(iommu, np, dbase, dsize,
+ fbase, fsize);
+ cell_iommu_enable_hardware(iommu);
+ cell_iommu_setup_window(iommu, np, dbase, dsize, 0);
+ }
+
+ dma_iommu_fixed_ops = dma_direct_ops;
+ dma_iommu_fixed_ops.set_dma_mask = dma_set_mask_and_switch;
+
+ dma_iommu_ops.set_dma_mask = dma_set_mask_and_switch;
+ set_pci_dma_ops(&dma_iommu_ops);
+
+ return 0;
+}
+
+static int iommu_fixed_disabled;
+
+static int __init setup_iommu_fixed(char *str)
+{
+ if (strcmp(str, "off") == 0)
+ iommu_fixed_disabled = 1;
+
+ return 1;
+}
+__setup("iommu_fixed=", setup_iommu_fixed);
+
static int __init cell_iommu_init(void)
{
struct device_node *np;
ppc_md.tce_build = tce_build_cell;
ppc_md.tce_free = tce_free_cell;
+ if (!iommu_fixed_disabled && cell_iommu_fixed_mapping_init() == 0)
+ goto bail;
+
/* Create an iommu for each /axon node. */
for_each_node_by_name(np, "axon") {
if (np->parent == NULL || np->parent->parent != NULL)