2 * Re-map IO memory to kernel address space so that we can access it.
3 * This is needed for high PCI addresses that aren't mapped in the
4 * 640k-1MB IO memory area on PC's
6 * (C) Copyright 1995 1996 Linus Torvalds
9 #include <linux/bootmem.h>
10 #include <linux/init.h>
12 #include <linux/module.h>
13 #include <linux/slab.h>
14 #include <linux/vmalloc.h>
16 #include <asm/cacheflush.h>
18 #include <asm/fixmap.h>
19 #include <asm/pgtable.h>
20 #include <asm/tlbflush.h>
24 unsigned long __phys_addr(unsigned long x)
26 if (x >= __START_KERNEL_map)
27 return x - __START_KERNEL_map + phys_base;
28 return x - PAGE_OFFSET;
30 EXPORT_SYMBOL(__phys_addr);
34 int page_is_ram(unsigned long pagenr)
36 unsigned long addr, end;
39 for (i = 0; i < e820.nr_map; i++) {
43 if (e820.map[i].type != E820_RAM)
45 addr = (e820.map[i].addr + PAGE_SIZE-1) >> PAGE_SHIFT;
46 end = (e820.map[i].addr + e820.map[i].size) >> PAGE_SHIFT;
49 * Sanity check: Some BIOSen report areas as RAM that
50 * are not. Notably the 640->1Mb area, which is the
53 if (addr >= (BIOS_BEGIN >> PAGE_SHIFT) &&
54 end < (BIOS_END >> PAGE_SHIFT))
57 if ((pagenr >= addr) && (pagenr < end))
64 * Fix up the linear direct mapping of the kernel to avoid cache attribute
67 static int ioremap_change_attr(unsigned long phys_addr, unsigned long size,
70 unsigned long npages, vaddr, last_addr = phys_addr + size - 1;
73 /* No change for pages after the last mapping */
74 if (last_addr >= (max_pfn_mapped << PAGE_SHIFT))
77 npages = (size + PAGE_SIZE - 1) >> PAGE_SHIFT;
78 vaddr = (unsigned long) __va(phys_addr);
81 * If there is no identity map for this address,
82 * change_page_attr_addr is unnecessary
84 if (!lookup_address(vaddr, &level))
88 * Must use an address here and not struct page because the
89 * phys addr can be a in hole between nodes and not have a
92 err = change_page_attr_addr(vaddr, npages, prot);
101 * Remap an arbitrary physical address space into the kernel virtual
102 * address space. Needed when the kernel wants to access high addresses
105 * NOTE! We need to allow non-page-aligned mappings too: we will obviously
106 * have to convert them into an offset in a page-aligned mapping, but the
107 * caller shouldn't need to know that small detail.
109 void __iomem *__ioremap(unsigned long phys_addr, unsigned long size,
113 struct vm_struct *area;
114 unsigned long offset, last_addr;
117 /* Don't allow wraparound or zero size */
118 last_addr = phys_addr + size - 1;
119 if (!size || last_addr < phys_addr)
123 * Don't remap the low PCI/ISA area, it's always mapped..
125 if (phys_addr >= ISA_START_ADDRESS && last_addr < ISA_END_ADDRESS)
126 return (__force void __iomem *)phys_to_virt(phys_addr);
129 * Don't allow anybody to remap normal RAM that we're using..
131 for (offset = phys_addr >> PAGE_SHIFT; offset < max_pfn_mapped &&
132 (offset << PAGE_SHIFT) < last_addr; offset++) {
133 if (page_is_ram(offset))
137 pgprot = MAKE_GLOBAL(__PAGE_KERNEL | flags);
140 * Mappings have to be page-aligned
142 offset = phys_addr & ~PAGE_MASK;
143 phys_addr &= PAGE_MASK;
144 size = PAGE_ALIGN(last_addr+1) - phys_addr;
149 area = get_vm_area(size, VM_IOREMAP);
152 area->phys_addr = phys_addr;
153 addr = (void __iomem *) area->addr;
154 if (ioremap_page_range((unsigned long)addr, (unsigned long)addr + size,
155 phys_addr, pgprot)) {
156 remove_vm_area((void *)(PAGE_MASK & (unsigned long) addr));
160 if (ioremap_change_attr(phys_addr, size, pgprot) < 0) {
165 return (void __iomem *) (offset + (char __iomem *)addr);
167 EXPORT_SYMBOL(__ioremap);
170 * ioremap_nocache - map bus memory into CPU space
171 * @offset: bus address of the memory
172 * @size: size of the resource to map
174 * ioremap_nocache performs a platform specific sequence of operations to
175 * make bus memory CPU accessible via the readb/readw/readl/writeb/
176 * writew/writel functions and the other mmio helpers. The returned
177 * address is not guaranteed to be usable directly as a virtual
180 * This version of ioremap ensures that the memory is marked uncachable
181 * on the CPU as well as honouring existing caching rules from things like
182 * the PCI bus. Note that there are other caches and buffers on many
183 * busses. In particular driver authors should read up on PCI writes
185 * It's useful if some control registers are in such an area and
186 * write combining or read caching is not desirable:
188 * Must be freed with iounmap.
190 void __iomem *ioremap_nocache(unsigned long phys_addr, unsigned long size)
192 return __ioremap(phys_addr, size, _PAGE_PCD | _PAGE_PWT);
194 EXPORT_SYMBOL(ioremap_nocache);
197 * iounmap - Free a IO remapping
198 * @addr: virtual address from ioremap_*
200 * Caller must ensure there is only one unmapping for the same pointer.
202 void iounmap(volatile void __iomem *addr)
204 struct vm_struct *p, *o;
206 if ((void __force *)addr <= high_memory)
210 * __ioremap special-cases the PCI/ISA range by not instantiating a
211 * vm_area and by simply returning an address into the kernel mapping
212 * of ISA space. So handle that here.
214 if (addr >= phys_to_virt(ISA_START_ADDRESS) &&
215 addr < phys_to_virt(ISA_END_ADDRESS))
218 addr = (volatile void __iomem *)
219 (PAGE_MASK & (unsigned long __force)addr);
221 /* Use the vm area unlocked, assuming the caller
222 ensures there isn't another iounmap for the same address
223 in parallel. Reuse of the virtual address is prevented by
224 leaving it in the global lists until we're done with it.
225 cpa takes care of the direct mappings. */
226 read_lock(&vmlist_lock);
227 for (p = vmlist; p; p = p->next) {
231 read_unlock(&vmlist_lock);
234 printk(KERN_ERR "iounmap: bad address %p\n", addr);
239 /* Reset the direct mapping. Can block */
240 ioremap_change_attr(p->phys_addr, p->size, PAGE_KERNEL);
242 /* Finally remove it */
243 o = remove_vm_area((void *)addr);
244 BUG_ON(p != o || o == NULL);
247 EXPORT_SYMBOL(iounmap);
251 int __initdata early_ioremap_debug;
253 static int __init early_ioremap_debug_setup(char *str)
255 early_ioremap_debug = 1;
259 early_param("early_ioremap_debug", early_ioremap_debug_setup);
261 static __initdata int after_paging_init;
262 static __initdata unsigned long bm_pte[1024]
263 __attribute__((aligned(PAGE_SIZE)));
265 static inline unsigned long * __init early_ioremap_pgd(unsigned long addr)
267 return (unsigned long *)swapper_pg_dir + ((addr >> 22) & 1023);
270 static inline unsigned long * __init early_ioremap_pte(unsigned long addr)
272 return bm_pte + ((addr >> PAGE_SHIFT) & 1023);
275 void __init early_ioremap_init(void)
279 if (early_ioremap_debug)
280 printk(KERN_DEBUG "early_ioremap_init()\n");
282 pgd = early_ioremap_pgd(fix_to_virt(FIX_BTMAP_BEGIN));
283 *pgd = __pa(bm_pte) | _PAGE_TABLE;
284 memset(bm_pte, 0, sizeof(bm_pte));
286 * The boot-ioremap range spans multiple pgds, for which
287 * we are not prepared:
289 if (pgd != early_ioremap_pgd(fix_to_virt(FIX_BTMAP_END))) {
291 printk(KERN_WARNING "pgd %p != %p\n",
292 pgd, early_ioremap_pgd(fix_to_virt(FIX_BTMAP_END)));
293 printk(KERN_WARNING "fix_to_virt(FIX_BTMAP_BEGIN): %08lx\n",
294 fix_to_virt(FIX_BTMAP_BEGIN));
295 printk(KERN_WARNING "fix_to_virt(FIX_BTMAP_END): %08lx\n",
296 fix_to_virt(FIX_BTMAP_END));
298 printk(KERN_WARNING "FIX_BTMAP_END: %d\n", FIX_BTMAP_END);
299 printk(KERN_WARNING "FIX_BTMAP_BEGIN: %d\n",
304 void __init early_ioremap_clear(void)
308 if (early_ioremap_debug)
309 printk(KERN_DEBUG "early_ioremap_clear()\n");
311 pgd = early_ioremap_pgd(fix_to_virt(FIX_BTMAP_BEGIN));
316 void __init early_ioremap_reset(void)
318 enum fixed_addresses idx;
319 unsigned long *pte, phys, addr;
321 after_paging_init = 1;
322 for (idx = FIX_BTMAP_BEGIN; idx >= FIX_BTMAP_END; idx--) {
323 addr = fix_to_virt(idx);
324 pte = early_ioremap_pte(addr);
325 if (!*pte & _PAGE_PRESENT) {
326 phys = *pte & PAGE_MASK;
327 set_fixmap(idx, phys);
332 static void __init __early_set_fixmap(enum fixed_addresses idx,
333 unsigned long phys, pgprot_t flags)
335 unsigned long *pte, addr = __fix_to_virt(idx);
337 if (idx >= __end_of_fixed_addresses) {
341 pte = early_ioremap_pte(addr);
342 if (pgprot_val(flags))
343 *pte = (phys & PAGE_MASK) | pgprot_val(flags);
346 __flush_tlb_one(addr);
349 static inline void __init early_set_fixmap(enum fixed_addresses idx,
352 if (after_paging_init)
353 set_fixmap(idx, phys);
355 __early_set_fixmap(idx, phys, PAGE_KERNEL);
358 static inline void __init early_clear_fixmap(enum fixed_addresses idx)
360 if (after_paging_init)
363 __early_set_fixmap(idx, 0, __pgprot(0));
367 int __initdata early_ioremap_nested;
369 static int __init check_early_ioremap_leak(void)
371 if (!early_ioremap_nested)
375 "Debug warning: early ioremap leak of %d areas detected.\n",
376 early_ioremap_nested);
378 "please boot with early_ioremap_debug and report the dmesg.\n");
383 late_initcall(check_early_ioremap_leak);
385 void __init *early_ioremap(unsigned long phys_addr, unsigned long size)
387 unsigned long offset, last_addr;
388 unsigned int nrpages, nesting;
389 enum fixed_addresses idx0, idx;
391 WARN_ON(system_state != SYSTEM_BOOTING);
393 nesting = early_ioremap_nested;
394 if (early_ioremap_debug) {
395 printk(KERN_DEBUG "early_ioremap(%08lx, %08lx) [%d] => ",
396 phys_addr, size, nesting);
400 /* Don't allow wraparound or zero size */
401 last_addr = phys_addr + size - 1;
402 if (!size || last_addr < phys_addr) {
407 if (nesting >= FIX_BTMAPS_NESTING) {
411 early_ioremap_nested++;
413 * Mappings have to be page-aligned
415 offset = phys_addr & ~PAGE_MASK;
416 phys_addr &= PAGE_MASK;
417 size = PAGE_ALIGN(last_addr) - phys_addr;
420 * Mappings have to fit in the FIX_BTMAP area.
422 nrpages = size >> PAGE_SHIFT;
423 if (nrpages > NR_FIX_BTMAPS) {
431 idx0 = FIX_BTMAP_BEGIN - NR_FIX_BTMAPS*nesting;
433 while (nrpages > 0) {
434 early_set_fixmap(idx, phys_addr);
435 phys_addr += PAGE_SIZE;
439 if (early_ioremap_debug)
440 printk(KERN_CONT "%08lx + %08lx\n", offset, fix_to_virt(idx0));
442 return (void *) (offset + fix_to_virt(idx0));
445 void __init early_iounmap(void *addr, unsigned long size)
447 unsigned long virt_addr;
448 unsigned long offset;
449 unsigned int nrpages;
450 enum fixed_addresses idx;
451 unsigned int nesting;
453 nesting = --early_ioremap_nested;
454 WARN_ON(nesting < 0);
456 if (early_ioremap_debug) {
457 printk(KERN_DEBUG "early_iounmap(%p, %08lx) [%d]\n", addr,
462 virt_addr = (unsigned long)addr;
463 if (virt_addr < fix_to_virt(FIX_BTMAP_BEGIN)) {
467 offset = virt_addr & ~PAGE_MASK;
468 nrpages = PAGE_ALIGN(offset + size - 1) >> PAGE_SHIFT;
470 idx = FIX_BTMAP_BEGIN - NR_FIX_BTMAPS*nesting;
471 while (nrpages > 0) {
472 early_clear_fixmap(idx);
478 void __this_fixmap_does_not_exist(void)
483 #endif /* CONFIG_X86_32 */