1 #include <linux/dma-mapping.h>
2 #include <linux/dmar.h>
3 #include <linux/bootmem.h>
9 #include <asm/calgary.h>
11 int forbid_dac __read_mostly;
12 EXPORT_SYMBOL(forbid_dac);
14 const struct dma_mapping_ops *dma_ops;
15 EXPORT_SYMBOL(dma_ops);
17 static int iommu_sac_force __read_mostly;
19 #ifdef CONFIG_IOMMU_DEBUG
20 int panic_on_overflow __read_mostly = 1;
21 int force_iommu __read_mostly = 1;
23 int panic_on_overflow __read_mostly = 0;
24 int force_iommu __read_mostly = 0;
27 int iommu_merge __read_mostly = 0;
29 int no_iommu __read_mostly;
30 /* Set this to 1 if there is a HW IOMMU in the system */
31 int iommu_detected __read_mostly = 0;
33 /* This tells the BIO block layer to assume merging. Default to off
34 because we cannot guarantee merging later. */
35 int iommu_bio_merge __read_mostly = 0;
36 EXPORT_SYMBOL(iommu_bio_merge);
38 dma_addr_t bad_dma_address __read_mostly = 0;
39 EXPORT_SYMBOL(bad_dma_address);
41 /* Dummy device used for NULL arguments (normally ISA). Better would
42 be probably a smaller DMA mask, but this is bug-to-bug compatible
44 struct device fallback_dev = {
45 .bus_id = "fallback device",
46 .coherent_dma_mask = DMA_32BIT_MASK,
47 .dma_mask = &fallback_dev.coherent_dma_mask,
50 int dma_set_mask(struct device *dev, u64 mask)
52 if (!dev->dma_mask || !dma_supported(dev, mask))
55 *dev->dma_mask = mask;
59 EXPORT_SYMBOL(dma_set_mask);
62 static __initdata void *dma32_bootmem_ptr;
63 static unsigned long dma32_bootmem_size __initdata = (128ULL<<20);
65 static int __init parse_dma32_size_opt(char *p)
69 dma32_bootmem_size = memparse(p, &p);
72 early_param("dma32_size", parse_dma32_size_opt);
74 void __init dma32_reserve_bootmem(void)
76 unsigned long size, align;
77 if (end_pfn <= MAX_DMA32_PFN)
81 * check aperture_64.c allocate_aperture() for reason about
85 size = round_up(dma32_bootmem_size, align);
86 dma32_bootmem_ptr = __alloc_bootmem_nopanic(size, align,
88 if (dma32_bootmem_ptr)
89 dma32_bootmem_size = size;
91 dma32_bootmem_size = 0;
93 static void __init dma32_free_bootmem(void)
97 if (end_pfn <= MAX_DMA32_PFN)
100 if (!dma32_bootmem_ptr)
103 for_each_online_node(node)
104 free_bootmem_node(NODE_DATA(node), __pa(dma32_bootmem_ptr),
107 dma32_bootmem_ptr = NULL;
108 dma32_bootmem_size = 0;
111 void __init pci_iommu_alloc(void)
113 /* free the range so iommu could get some range less than 4G */
114 dma32_free_bootmem();
116 * The order of these functions is important for
117 * fall-back/fail-over reasons
119 #ifdef CONFIG_GART_IOMMU
120 gart_iommu_hole_init();
123 #ifdef CONFIG_CALGARY_IOMMU
127 detect_intel_iommu();
129 #ifdef CONFIG_SWIOTLB
136 * See <Documentation/x86_64/boot-options.txt> for the iommu kernel parameter
139 static __init int iommu_setup(char *p)
147 if (!strncmp(p, "off", 3))
149 /* gart_parse_options has more force support */
150 if (!strncmp(p, "force", 5))
152 if (!strncmp(p, "noforce", 7)) {
157 if (!strncmp(p, "biomerge", 8)) {
158 iommu_bio_merge = 4096;
162 if (!strncmp(p, "panic", 5))
163 panic_on_overflow = 1;
164 if (!strncmp(p, "nopanic", 7))
165 panic_on_overflow = 0;
166 if (!strncmp(p, "merge", 5)) {
170 if (!strncmp(p, "nomerge", 7))
172 if (!strncmp(p, "forcesac", 8))
174 if (!strncmp(p, "allowdac", 8))
176 if (!strncmp(p, "nodac", 5))
178 if (!strncmp(p, "usedac", 6)) {
182 #ifdef CONFIG_SWIOTLB
183 if (!strncmp(p, "soft", 4))
187 #ifdef CONFIG_GART_IOMMU
188 gart_parse_options(p);
191 #ifdef CONFIG_CALGARY_IOMMU
192 if (!strncmp(p, "calgary", 7))
194 #endif /* CONFIG_CALGARY_IOMMU */
196 p += strcspn(p, ",");
202 early_param("iommu", iommu_setup);
205 int dma_declare_coherent_memory(struct device *dev, dma_addr_t bus_addr,
206 dma_addr_t device_addr, size_t size, int flags)
208 void __iomem *mem_base = NULL;
209 int pages = size >> PAGE_SHIFT;
210 int bitmap_size = BITS_TO_LONGS(pages) * sizeof(long);
212 if ((flags & (DMA_MEMORY_MAP | DMA_MEMORY_IO)) == 0)
219 /* FIXME: this routine just ignores DMA_MEMORY_INCLUDES_CHILDREN */
221 mem_base = ioremap(bus_addr, size);
225 dev->dma_mem = kzalloc(sizeof(struct dma_coherent_mem), GFP_KERNEL);
228 dev->dma_mem->bitmap = kzalloc(bitmap_size, GFP_KERNEL);
229 if (!dev->dma_mem->bitmap)
232 dev->dma_mem->virt_base = mem_base;
233 dev->dma_mem->device_base = device_addr;
234 dev->dma_mem->size = pages;
235 dev->dma_mem->flags = flags;
237 if (flags & DMA_MEMORY_MAP)
238 return DMA_MEMORY_MAP;
240 return DMA_MEMORY_IO;
249 EXPORT_SYMBOL(dma_declare_coherent_memory);
251 void dma_release_declared_memory(struct device *dev)
253 struct dma_coherent_mem *mem = dev->dma_mem;
258 iounmap(mem->virt_base);
262 EXPORT_SYMBOL(dma_release_declared_memory);
264 void *dma_mark_declared_memory_occupied(struct device *dev,
265 dma_addr_t device_addr, size_t size)
267 struct dma_coherent_mem *mem = dev->dma_mem;
269 int pages = (size + (device_addr & ~PAGE_MASK) + PAGE_SIZE - 1);
271 pages >>= PAGE_SHIFT;
274 return ERR_PTR(-EINVAL);
276 pos = (device_addr - mem->device_base) >> PAGE_SHIFT;
277 err = bitmap_allocate_region(mem->bitmap, pos, get_order(pages));
280 return mem->virt_base + (pos << PAGE_SHIFT);
282 EXPORT_SYMBOL(dma_mark_declared_memory_occupied);
284 static int dma_alloc_from_coherent_mem(struct device *dev, ssize_t size,
285 dma_addr_t *dma_handle, void **ret)
287 struct dma_coherent_mem *mem = dev ? dev->dma_mem : NULL;
288 int order = get_order(size);
291 int page = bitmap_find_free_region(mem->bitmap, mem->size,
294 *dma_handle = mem->device_base + (page << PAGE_SHIFT);
295 *ret = mem->virt_base + (page << PAGE_SHIFT);
296 memset(*ret, 0, size);
298 if (mem->flags & DMA_MEMORY_EXCLUSIVE)
301 return (mem != NULL);
304 static int dma_release_coherent(struct device *dev, int order, void *vaddr)
306 struct dma_coherent_mem *mem = dev ? dev->dma_mem : NULL;
308 if (mem && vaddr >= mem->virt_base && vaddr <
309 (mem->virt_base + (mem->size << PAGE_SHIFT))) {
310 int page = (vaddr - mem->virt_base) >> PAGE_SHIFT;
312 bitmap_release_region(mem->bitmap, page, order);
318 #define dma_alloc_from_coherent_mem(dev, size, handle, ret) (0)
319 #define dma_release_coherent(dev, order, vaddr) (0)
320 #endif /* CONFIG_X86_32 */
322 int dma_supported(struct device *dev, u64 mask)
325 if (mask > 0xffffffff && forbid_dac > 0) {
326 printk(KERN_INFO "PCI: Disallowing DAC for device %s\n",
332 if (dma_ops->dma_supported)
333 return dma_ops->dma_supported(dev, mask);
335 /* Copied from i386. Doesn't make much sense, because it will
336 only work for pci_alloc_coherent.
337 The caller just has to use GFP_DMA in this case. */
338 if (mask < DMA_24BIT_MASK)
341 /* Tell the device to use SAC when IOMMU force is on. This
342 allows the driver to use cheaper accesses in some cases.
344 Problem with this is that if we overflow the IOMMU area and
345 return DAC as fallback address the device may not handle it
348 As a special case some controllers have a 39bit address
349 mode that is as efficient as 32bit (aic79xx). Don't force
350 SAC for these. Assume all masks <= 40 bits are of this
351 type. Normally this doesn't make any difference, but gives
352 more gentle handling of IOMMU overflow. */
353 if (iommu_sac_force && (mask >= DMA_40BIT_MASK)) {
354 printk(KERN_INFO "%s: Force SAC with mask %Lx\n",
361 EXPORT_SYMBOL(dma_supported);
363 /* Allocate DMA memory on node near device */
364 noinline struct page *
365 dma_alloc_pages(struct device *dev, gfp_t gfp, unsigned order)
369 node = dev_to_node(dev);
371 return alloc_pages_node(node, gfp, order);
375 * Allocate memory for a coherent mapping.
378 dma_alloc_coherent(struct device *dev, size_t size, dma_addr_t *dma_handle,
383 unsigned long dma_mask = 0;
386 /* ignore region specifiers */
387 gfp &= ~(__GFP_DMA | __GFP_HIGHMEM | __GFP_DMA32);
389 if (dma_alloc_from_coherent_mem(dev, size, dma_handle, &memory))
394 dma_mask = dev->coherent_dma_mask;
396 dma_mask = DMA_32BIT_MASK;
398 /* Device not DMA able */
399 if (dev->dma_mask == NULL)
402 /* Don't invoke OOM killer */
403 gfp |= __GFP_NORETRY;
406 /* Why <=? Even when the mask is smaller than 4GB it is often
407 larger than 16MB and in this case we have a chance of
408 finding fitting memory in the next higher zone first. If
409 not retry with true GFP_DMA. -AK */
410 if (dma_mask <= DMA_32BIT_MASK)
415 page = dma_alloc_pages(dev, gfp, get_order(size));
421 bus = page_to_phys(page);
422 memory = page_address(page);
423 high = (bus + size) >= dma_mask;
425 if (force_iommu && !(gfp & GFP_DMA))
428 free_pages((unsigned long)memory,
431 /* Don't use the 16MB ZONE_DMA unless absolutely
432 needed. It's better to use remapping first. */
433 if (dma_mask < DMA_32BIT_MASK && !(gfp & GFP_DMA)) {
434 gfp = (gfp & ~GFP_DMA32) | GFP_DMA;
438 /* Let low level make its own zone decisions */
439 gfp &= ~(GFP_DMA32|GFP_DMA);
441 if (dma_ops->alloc_coherent)
442 return dma_ops->alloc_coherent(dev, size,
447 memset(memory, 0, size);
454 if (dma_ops->alloc_coherent) {
455 free_pages((unsigned long)memory, get_order(size));
456 gfp &= ~(GFP_DMA|GFP_DMA32);
457 return dma_ops->alloc_coherent(dev, size, dma_handle, gfp);
460 if (dma_ops->map_simple) {
461 *dma_handle = dma_ops->map_simple(dev, virt_to_phys(memory),
463 PCI_DMA_BIDIRECTIONAL);
464 if (*dma_handle != bad_dma_address)
468 if (panic_on_overflow)
469 panic("dma_alloc_coherent: IOMMU overflow by %lu bytes\n",
470 (unsigned long)size);
471 free_pages((unsigned long)memory, get_order(size));
474 EXPORT_SYMBOL(dma_alloc_coherent);
477 * Unmap coherent memory.
478 * The caller must ensure that the device has finished accessing the mapping.
480 void dma_free_coherent(struct device *dev, size_t size,
481 void *vaddr, dma_addr_t bus)
483 int order = get_order(size);
484 WARN_ON(irqs_disabled()); /* for portability */
485 if (dma_release_coherent(dev, order, vaddr))
487 if (dma_ops->unmap_single)
488 dma_ops->unmap_single(dev, bus, size, 0);
489 free_pages((unsigned long)vaddr, order);
491 EXPORT_SYMBOL(dma_free_coherent);
493 static int __init pci_iommu_init(void)
495 #ifdef CONFIG_CALGARY_IOMMU
496 calgary_iommu_init();
501 #ifdef CONFIG_GART_IOMMU
509 void pci_iommu_shutdown(void)
511 gart_iommu_shutdown();
513 /* Must execute after PCI subsystem */
514 fs_initcall(pci_iommu_init);
517 /* Many VIA bridges seem to corrupt data for DAC. Disable it here */
519 static __devinit void via_no_dac(struct pci_dev *dev)
521 if ((dev->class >> 8) == PCI_CLASS_BRIDGE_PCI && forbid_dac == 0) {
522 printk(KERN_INFO "PCI: VIA PCI bridge detected."
527 DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_VIA, PCI_ANY_ID, via_no_dac);