drivers/pci/intel-iommu.c

   1 /*
   2  * Copyright (c) 2006, Intel Corporation.
   3  *
   4  * This program is free software; you can redistribute it and/or modify it
   5  * under the terms and conditions of the GNU General Public License,
   6  * version 2, as published by the Free Software Foundation.
   7  *
   8  * This program is distributed in the hope it will be useful, but WITHOUT
   9  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  10  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  11  * more details.
  12  *
  13  * You should have received a copy of the GNU General Public License along with
  14  * this program; if not, write to the Free Software Foundation, Inc., 59 Temple
  15  * Place - Suite 330, Boston, MA 02111-1307 USA.
  16  *
  17  * Copyright (C) Ashok Raj <ashok.raj@intel.com>
  18  * Copyright (C) Shaohua Li <shaohua.li@intel.com>
  19  * Copyright (C) Anil S Keshavamurthy <anil.s.keshavamurthy@intel.com>
  20  */
  21
  22 #include <linux/init.h>
  23 #include <linux/bitmap.h>
  24 #include <linux/slab.h>
  25 #include <linux/irq.h>
  26 #include <linux/interrupt.h>
  27 #include <linux/sysdev.h>
  28 #include <linux/spinlock.h>
  29 #include <linux/pci.h>
  30 #include <linux/dmar.h>
  31 #include <linux/dma-mapping.h>
  32 #include <linux/mempool.h>
  33 #include "iova.h"
  34 #include "intel-iommu.h"
  35 #include <asm/proto.h> /* force_iommu in this header in x86-64*/
  36 #include <asm/cacheflush.h>
  37 #include <asm/iommu.h>
  38 #include "pci.h"
  39
  40 #define IS_GFX_DEVICE(pdev) ((pdev->class >> 16) == PCI_BASE_CLASS_DISPLAY)
  41 #define IS_ISA_DEVICE(pdev) ((pdev->class >> 8) == PCI_CLASS_BRIDGE_ISA)
  42
  43 #define IOAPIC_RANGE_START      (0xfee00000)
  44 #define IOAPIC_RANGE_END        (0xfeefffff)
  45 #define IOVA_START_ADDR         (0x1000)
  46
  47 #define DEFAULT_DOMAIN_ADDRESS_WIDTH 48
  48
  49 #define DMAR_OPERATION_TIMEOUT (HZ*60) /* 1m */
  50
  51 #define DOMAIN_MAX_ADDR(gaw) ((((u64)1) << gaw) - 1)
  52
  53 static void domain_remove_dev_info(struct dmar_domain *domain);
  54
  55 static int dmar_disabled;
  56 static int __initdata dmar_map_gfx = 1;
  57
  58 #define DUMMY_DEVICE_DOMAIN_INFO ((struct device_domain_info *)(-1))
  59 static DEFINE_SPINLOCK(device_domain_lock);
  60 static LIST_HEAD(device_domain_list);
  61
  62 static int __init intel_iommu_setup(char *str)
  63 {
  64         if (!str)
  65                 return -EINVAL;
  66         while (*str) {
  67                 if (!strncmp(str, "off", 3)) {
  68                         dmar_disabled = 1;
  69                         printk(KERN_INFO"Intel-IOMMU: disabled\n");
  70                 } else if (!strncmp(str, "igfx_off", 8)) {
  71                         dmar_map_gfx = 0;
  72                         printk(KERN_INFO
  73                                 "Intel-IOMMU: disable GFX device mapping\n");
  74                 }
  75
  76                 str += strcspn(str, ",");
  77                 while (*str == ',')
  78                         str++;
  79         }
  80         return 0;
  81 }
  82 __setup("intel_iommu=", intel_iommu_setup);
  83
  84 static struct kmem_cache *iommu_domain_cache;
  85 static struct kmem_cache *iommu_devinfo_cache;
  86 static struct kmem_cache *iommu_iova_cache;
  87
  88 static inline void *iommu_kmem_cache_alloc(struct kmem_cache *cachep)
  89 {
  90         unsigned int flags;
  91         void *vaddr;
  92
  93         /* trying to avoid low memory issues */
  94         flags = current->flags & PF_MEMALLOC;
  95         current->flags |= PF_MEMALLOC;
  96         vaddr = kmem_cache_alloc(cachep, GFP_ATOMIC);
  97         current->flags &= (~PF_MEMALLOC | flags);
  98         return vaddr;
  99 }
 100
 101
 102 static inline void *alloc_pgtable_page(void)
 103 {
 104         unsigned int flags;
 105         void *vaddr;
 106
 107         /* trying to avoid low memory issues */
 108         flags = current->flags & PF_MEMALLOC;
 109         current->flags |= PF_MEMALLOC;
 110         vaddr = (void *)get_zeroed_page(GFP_ATOMIC);
 111         current->flags &= (~PF_MEMALLOC | flags);
 112         return vaddr;
 113 }
 114
 115 static inline void free_pgtable_page(void *vaddr)
 116 {
 117         free_page((unsigned long)vaddr);
 118 }
 119
 120 static inline void *alloc_domain_mem(void)
 121 {
 122         return iommu_kmem_cache_alloc(iommu_domain_cache);
 123 }
 124
 125 static inline void free_domain_mem(void *vaddr)
 126 {
 127         kmem_cache_free(iommu_domain_cache, vaddr);
 128 }
 129
 130 static inline void * alloc_devinfo_mem(void)
 131 {
 132         return iommu_kmem_cache_alloc(iommu_devinfo_cache);
 133 }
 134
 135 static inline void free_devinfo_mem(void *vaddr)
 136 {
 137         kmem_cache_free(iommu_devinfo_cache, vaddr);
 138 }
 139
 140 struct iova *alloc_iova_mem(void)
 141 {
 142         return iommu_kmem_cache_alloc(iommu_iova_cache);
 143 }
 144
 145 void free_iova_mem(struct iova *iova)
 146 {
 147         kmem_cache_free(iommu_iova_cache, iova);
 148 }
 149
 150 static inline void __iommu_flush_cache(
 151         struct intel_iommu *iommu, void *addr, int size)
 152 {
 153         if (!ecap_coherent(iommu->ecap))
 154                 clflush_cache_range(addr, size);
 155 }
 156
 157 /* Gets context entry for a given bus and devfn */
 158 static struct context_entry * device_to_context_entry(struct intel_iommu *iommu,
 159                 u8 bus, u8 devfn)
 160 {
 161         struct root_entry *root;
 162         struct context_entry *context;
 163         unsigned long phy_addr;
 164         unsigned long flags;
 165
 166         spin_lock_irqsave(&iommu->lock, flags);
 167         root = &iommu->root_entry[bus];
 168         context = get_context_addr_from_root(root);
 169         if (!context) {
 170                 context = (struct context_entry *)alloc_pgtable_page();
 171                 if (!context) {
 172                         spin_unlock_irqrestore(&iommu->lock, flags);
 173                         return NULL;
 174                 }
 175                 __iommu_flush_cache(iommu, (void *)context, PAGE_SIZE_4K);
 176                 phy_addr = virt_to_phys((void *)context);
 177                 set_root_value(root, phy_addr);
 178                 set_root_present(root);
 179                 __iommu_flush_cache(iommu, root, sizeof(*root));
 180         }
 181         spin_unlock_irqrestore(&iommu->lock, flags);
 182         return &context[devfn];
 183 }
 184
 185 static int device_context_mapped(struct intel_iommu *iommu, u8 bus, u8 devfn)
 186 {
 187         struct root_entry *root;
 188         struct context_entry *context;
 189         int ret;
 190         unsigned long flags;
 191
 192         spin_lock_irqsave(&iommu->lock, flags);
 193         root = &iommu->root_entry[bus];
 194         context = get_context_addr_from_root(root);
 195         if (!context) {
 196                 ret = 0;
 197                 goto out;
 198         }
 199         ret = context_present(context[devfn]);
 200 out:
 201         spin_unlock_irqrestore(&iommu->lock, flags);
 202         return ret;
 203 }
 204
 205 static void clear_context_table(struct intel_iommu *iommu, u8 bus, u8 devfn)
 206 {
 207         struct root_entry *root;
 208         struct context_entry *context;
 209         unsigned long flags;
 210
 211         spin_lock_irqsave(&iommu->lock, flags);
 212         root = &iommu->root_entry[bus];
 213         context = get_context_addr_from_root(root);
 214         if (context) {
 215                 context_clear_entry(context[devfn]);
 216                 __iommu_flush_cache(iommu, &context[devfn], \
 217                         sizeof(*context));
 218         }
 219         spin_unlock_irqrestore(&iommu->lock, flags);
 220 }
 221
 222 static void free_context_table(struct intel_iommu *iommu)
 223 {
 224         struct root_entry *root;
 225         int i;
 226         unsigned long flags;
 227         struct context_entry *context;
 228
 229         spin_lock_irqsave(&iommu->lock, flags);
 230         if (!iommu->root_entry) {
 231                 goto out;
 232         }
 233         for (i = 0; i < ROOT_ENTRY_NR; i++) {
 234                 root = &iommu->root_entry[i];
 235                 context = get_context_addr_from_root(root);
 236                 if (context)
 237                         free_pgtable_page(context);
 238         }
 239         free_pgtable_page(iommu->root_entry);
 240         iommu->root_entry = NULL;
 241 out:
 242         spin_unlock_irqrestore(&iommu->lock, flags);
 243 }
 244
 245 /* page table handling */
 246 #define LEVEL_STRIDE            (9)
 247 #define LEVEL_MASK              (((u64)1 << LEVEL_STRIDE) - 1)
 248
 249 static inline int agaw_to_level(int agaw)
 250 {
 251         return agaw + 2;
 252 }
 253
 254 static inline int agaw_to_width(int agaw)
 255 {
 256         return 30 + agaw * LEVEL_STRIDE;
 257
 258 }
 259
 260 static inline int width_to_agaw(int width)
 261 {
 262         return (width - 30) / LEVEL_STRIDE;
 263 }
 264
 265 static inline unsigned int level_to_offset_bits(int level)
 266 {
 267         return (12 + (level - 1) * LEVEL_STRIDE);
 268 }
 269
 270 static inline int address_level_offset(u64 addr, int level)
 271 {
 272         return ((addr >> level_to_offset_bits(level)) & LEVEL_MASK);
 273 }
 274
 275 static inline u64 level_mask(int level)
 276 {
 277         return ((u64)-1 << level_to_offset_bits(level));
 278 }
 279
 280 static inline u64 level_size(int level)
 281 {
 282         return ((u64)1 << level_to_offset_bits(level));
 283 }
 284
 285 static inline u64 align_to_level(u64 addr, int level)
 286 {
 287         return ((addr + level_size(level) - 1) & level_mask(level));
 288 }
 289
 290 static struct dma_pte * addr_to_dma_pte(struct dmar_domain *domain, u64 addr)
 291 {
 292         int addr_width = agaw_to_width(domain->agaw);
 293         struct dma_pte *parent, *pte = NULL;
 294         int level = agaw_to_level(domain->agaw);
 295         int offset;
 296         unsigned long flags;
 297
 298         BUG_ON(!domain->pgd);
 299
 300         addr &= (((u64)1) << addr_width) - 1;
 301         parent = domain->pgd;
 302
 303         spin_lock_irqsave(&domain->mapping_lock, flags);
 304         while (level > 0) {
 305                 void *tmp_page;
 306
 307                 offset = address_level_offset(addr, level);
 308                 pte = &parent[offset];
 309                 if (level == 1)
 310                         break;
 311
 312                 if (!dma_pte_present(*pte)) {
 313                         tmp_page = alloc_pgtable_page();
 314
 315                         if (!tmp_page) {
 316                                 spin_unlock_irqrestore(&domain->mapping_lock,
 317                                         flags);
 318                                 return NULL;
 319                         }
 320                         __iommu_flush_cache(domain->iommu, tmp_page,
 321                                         PAGE_SIZE_4K);
 322                         dma_set_pte_addr(*pte, virt_to_phys(tmp_page));
 323                         /*
 324                          * high level table always sets r/w, last level page
 325                          * table control read/write
 326                          */
 327                         dma_set_pte_readable(*pte);
 328                         dma_set_pte_writable(*pte);
 329                         __iommu_flush_cache(domain->iommu, pte, sizeof(*pte));
 330                 }
 331                 parent = phys_to_virt(dma_pte_addr(*pte));
 332                 level--;
 333         }
 334
 335         spin_unlock_irqrestore(&domain->mapping_lock, flags);
 336         return pte;
 337 }
 338
 339 /* return address's pte at specific level */
 340 static struct dma_pte *dma_addr_level_pte(struct dmar_domain *domain, u64 addr,
 341                 int level)
 342 {
 343         struct dma_pte *parent, *pte = NULL;
 344         int total = agaw_to_level(domain->agaw);
 345         int offset;
 346
 347         parent = domain->pgd;
 348         while (level <= total) {
 349                 offset = address_level_offset(addr, total);
 350                 pte = &parent[offset];
 351                 if (level == total)
 352                         return pte;
 353
 354                 if (!dma_pte_present(*pte))
 355                         break;
 356                 parent = phys_to_virt(dma_pte_addr(*pte));
 357                 total--;
 358         }
 359         return NULL;
 360 }
 361
 362 /* clear one page's page table */
 363 static void dma_pte_clear_one(struct dmar_domain *domain, u64 addr)
 364 {
 365         struct dma_pte *pte = NULL;
 366
 367         /* get last level pte */
 368         pte = dma_addr_level_pte(domain, addr, 1);
 369
 370         if (pte) {
 371                 dma_clear_pte(*pte);
 372                 __iommu_flush_cache(domain->iommu, pte, sizeof(*pte));
 373         }
 374 }
 375
 376 /* clear last level pte, a tlb flush should be followed */
 377 static void dma_pte_clear_range(struct dmar_domain *domain, u64 start, u64 end)
 378 {
 379         int addr_width = agaw_to_width(domain->agaw);
 380
 381         start &= (((u64)1) << addr_width) - 1;
 382         end &= (((u64)1) << addr_width) - 1;
 383         /* in case it's partial page */
 384         start = PAGE_ALIGN_4K(start);
 385         end &= PAGE_MASK_4K;
 386
 387         /* we don't need lock here, nobody else touches the iova range */
 388         while (start < end) {
 389                 dma_pte_clear_one(domain, start);
 390                 start += PAGE_SIZE_4K;
 391         }
 392 }
 393
 394 /* free page table pages. last level pte should already be cleared */
 395 static void dma_pte_free_pagetable(struct dmar_domain *domain,
 396         u64 start, u64 end)
 397 {
 398         int addr_width = agaw_to_width(domain->agaw);
 399         struct dma_pte *pte;
 400         int total = agaw_to_level(domain->agaw);
 401         int level;
 402         u64 tmp;
 403
 404         start &= (((u64)1) << addr_width) - 1;
 405         end &= (((u64)1) << addr_width) - 1;
 406
 407         /* we don't need lock here, nobody else touches the iova range */
 408         level = 2;
 409         while (level <= total) {
 410                 tmp = align_to_level(start, level);
 411                 if (tmp >= end || (tmp + level_size(level) > end))
 412                         return;
 413
 414                 while (tmp < end) {
 415                         pte = dma_addr_level_pte(domain, tmp, level);
 416                         if (pte) {
 417                                 free_pgtable_page(
 418                                         phys_to_virt(dma_pte_addr(*pte)));
 419                                 dma_clear_pte(*pte);
 420                                 __iommu_flush_cache(domain->iommu,
 421                                                 pte, sizeof(*pte));
 422                         }
 423                         tmp += level_size(level);
 424                 }
 425                 level++;
 426         }
 427         /* free pgd */
 428         if (start == 0 && end >= ((((u64)1) << addr_width) - 1)) {
 429                 free_pgtable_page(domain->pgd);
 430                 domain->pgd = NULL;
 431         }
 432 }
 433
 434 /* iommu handling */
 435 static int iommu_alloc_root_entry(struct intel_iommu *iommu)
 436 {
 437         struct root_entry *root;
 438         unsigned long flags;
 439
 440         root = (struct root_entry *)alloc_pgtable_page();
 441         if (!root)
 442                 return -ENOMEM;
 443
 444         __iommu_flush_cache(iommu, root, PAGE_SIZE_4K);
 445
 446         spin_lock_irqsave(&iommu->lock, flags);
 447         iommu->root_entry = root;
 448         spin_unlock_irqrestore(&iommu->lock, flags);
 449
 450         return 0;
 451 }
 452
 453 #define IOMMU_WAIT_OP(iommu, offset, op, cond, sts) \
 454 {\
 455         unsigned long start_time = jiffies;\
 456         while (1) {\
 457                 sts = op (iommu->reg + offset);\
 458                 if (cond)\
 459                         break;\
 460                 if (time_after(jiffies, start_time + DMAR_OPERATION_TIMEOUT))\
 461                         panic("DMAR hardware is malfunctioning\n");\
 462                 cpu_relax();\
 463         }\
 464 }
 465
 466 static void iommu_set_root_entry(struct intel_iommu *iommu)
 467 {
 468         void *addr;
 469         u32 cmd, sts;
 470         unsigned long flag;
 471
 472         addr = iommu->root_entry;
 473
 474         spin_lock_irqsave(&iommu->register_lock, flag);
 475         dmar_writeq(iommu->reg + DMAR_RTADDR_REG, virt_to_phys(addr));
 476
 477         cmd = iommu->gcmd | DMA_GCMD_SRTP;
 478         writel(cmd, iommu->reg + DMAR_GCMD_REG);
 479
 480         /* Make sure hardware complete it */
 481         IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
 482                 readl, (sts & DMA_GSTS_RTPS), sts);
 483
 484         spin_unlock_irqrestore(&iommu->register_lock, flag);
 485 }
 486
 487 static void iommu_flush_write_buffer(struct intel_iommu *iommu)
 488 {
 489         u32 val;
 490         unsigned long flag;
 491
 492         if (!cap_rwbf(iommu->cap))
 493                 return;
 494         val = iommu->gcmd | DMA_GCMD_WBF;
 495
 496         spin_lock_irqsave(&iommu->register_lock, flag);
 497         writel(val, iommu->reg + DMAR_GCMD_REG);
 498
 499         /* Make sure hardware complete it */
 500         IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
 501                         readl, (!(val & DMA_GSTS_WBFS)), val);
 502
 503         spin_unlock_irqrestore(&iommu->register_lock, flag);
 504 }
 505
 506 /* return value determine if we need a write buffer flush */
 507 static int __iommu_flush_context(struct intel_iommu *iommu,
 508         u16 did, u16 source_id, u8 function_mask, u64 type,
 509         int non_present_entry_flush)
 510 {
 511         u64 val = 0;
 512         unsigned long flag;
 513
 514         /*
 515          * In the non-present entry flush case, if hardware doesn't cache
 516          * non-present entry we do nothing and if hardware cache non-present
 517          * entry, we flush entries of domain 0 (the domain id is used to cache
 518          * any non-present entries)
 519          */
 520         if (non_present_entry_flush) {
 521                 if (!cap_caching_mode(iommu->cap))
 522                         return 1;
 523                 else
 524                         did = 0;
 525         }
 526
 527         switch (type) {
 528         case DMA_CCMD_GLOBAL_INVL:
 529                 val = DMA_CCMD_GLOBAL_INVL;
 530                 break;
 531         case DMA_CCMD_DOMAIN_INVL:
 532                 val = DMA_CCMD_DOMAIN_INVL|DMA_CCMD_DID(did);
 533                 break;
 534         case DMA_CCMD_DEVICE_INVL:
 535                 val = DMA_CCMD_DEVICE_INVL|DMA_CCMD_DID(did)
 536                         | DMA_CCMD_SID(source_id) | DMA_CCMD_FM(function_mask);
 537                 break;
 538         default:
 539                 BUG();
 540         }
 541         val |= DMA_CCMD_ICC;
 542
 543         spin_lock_irqsave(&iommu->register_lock, flag);
 544         dmar_writeq(iommu->reg + DMAR_CCMD_REG, val);
 545
 546         /* Make sure hardware complete it */
 547         IOMMU_WAIT_OP(iommu, DMAR_CCMD_REG,
 548                 dmar_readq, (!(val & DMA_CCMD_ICC)), val);
 549
 550         spin_unlock_irqrestore(&iommu->register_lock, flag);
 551
 552         /* flush context entry will implictly flush write buffer */
 553         return 0;
 554 }
 555
 556 static int inline iommu_flush_context_global(struct intel_iommu *iommu,
 557         int non_present_entry_flush)
 558 {
 559         return __iommu_flush_context(iommu, 0, 0, 0, DMA_CCMD_GLOBAL_INVL,
 560                 non_present_entry_flush);
 561 }
 562
 563 static int inline iommu_flush_context_domain(struct intel_iommu *iommu, u16 did,
 564         int non_present_entry_flush)
 565 {
 566         return __iommu_flush_context(iommu, did, 0, 0, DMA_CCMD_DOMAIN_INVL,
 567                 non_present_entry_flush);
 568 }
 569
 570 static int inline iommu_flush_context_device(struct intel_iommu *iommu,
 571         u16 did, u16 source_id, u8 function_mask, int non_present_entry_flush)
 572 {
 573         return __iommu_flush_context(iommu, did, source_id, function_mask,
 574                 DMA_CCMD_DEVICE_INVL, non_present_entry_flush);
 575 }
 576
 577 /* return value determine if we need a write buffer flush */
 578 static int __iommu_flush_iotlb(struct intel_iommu *iommu, u16 did,
 579         u64 addr, unsigned int size_order, u64 type,
 580         int non_present_entry_flush)
 581 {
 582         int tlb_offset = ecap_iotlb_offset(iommu->ecap);
 583         u64 val = 0, val_iva = 0;
 584         unsigned long flag;
 585
 586         /*
 587          * In the non-present entry flush case, if hardware doesn't cache
 588          * non-present entry we do nothing and if hardware cache non-present
 589          * entry, we flush entries of domain 0 (the domain id is used to cache
 590          * any non-present entries)
 591          */
 592         if (non_present_entry_flush) {
 593                 if (!cap_caching_mode(iommu->cap))
 594                         return 1;
 595                 else
 596                         did = 0;
 597         }
 598
 599         switch (type) {
 600         case DMA_TLB_GLOBAL_FLUSH:
 601                 /* global flush doesn't need set IVA_REG */
 602                 val = DMA_TLB_GLOBAL_FLUSH|DMA_TLB_IVT;
 603                 break;
 604         case DMA_TLB_DSI_FLUSH:
 605                 val = DMA_TLB_DSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
 606                 break;
 607         case DMA_TLB_PSI_FLUSH:
 608                 val = DMA_TLB_PSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
 609                 /* Note: always flush non-leaf currently */
 610                 val_iva = size_order | addr;
 611                 break;
 612         default:
 613                 BUG();
 614         }
 615         /* Note: set drain read/write */
 616 #if 0
 617         /*
 618          * This is probably to be super secure.. Looks like we can
 619          * ignore it without any impact.
 620          */
 621         if (cap_read_drain(iommu->cap))
 622                 val |= DMA_TLB_READ_DRAIN;
 623 #endif
 624         if (cap_write_drain(iommu->cap))
 625                 val |= DMA_TLB_WRITE_DRAIN;
 626
 627         spin_lock_irqsave(&iommu->register_lock, flag);
 628         /* Note: Only uses first TLB reg currently */
 629         if (val_iva)
 630                 dmar_writeq(iommu->reg + tlb_offset, val_iva);
 631         dmar_writeq(iommu->reg + tlb_offset + 8, val);
 632
 633         /* Make sure hardware complete it */
 634         IOMMU_WAIT_OP(iommu, tlb_offset + 8,
 635                 dmar_readq, (!(val & DMA_TLB_IVT)), val);
 636
 637         spin_unlock_irqrestore(&iommu->register_lock, flag);
 638
 639         /* check IOTLB invalidation granularity */
 640         if (DMA_TLB_IAIG(val) == 0)
 641                 printk(KERN_ERR"IOMMU: flush IOTLB failed\n");
 642         if (DMA_TLB_IAIG(val) != DMA_TLB_IIRG(type))
 643                 pr_debug("IOMMU: tlb flush request %Lx, actual %Lx\n",
 644                         DMA_TLB_IIRG(type), DMA_TLB_IAIG(val));
 645         /* flush context entry will implictly flush write buffer */
 646         return 0;
 647 }
 648
 649 static int inline iommu_flush_iotlb_global(struct intel_iommu *iommu,
 650         int non_present_entry_flush)
 651 {
 652         return __iommu_flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH,
 653                 non_present_entry_flush);
 654 }
 655
 656 static int inline iommu_flush_iotlb_dsi(struct intel_iommu *iommu, u16 did,
 657         int non_present_entry_flush)
 658 {
 659         return __iommu_flush_iotlb(iommu, did, 0, 0, DMA_TLB_DSI_FLUSH,
 660                 non_present_entry_flush);
 661 }
 662
 663 static int iommu_get_alignment(u64 base, unsigned int size)
 664 {
 665         int t = 0;
 666         u64 end;
 667
 668         end = base + size - 1;
 669         while (base != end) {
 670                 t++;
 671                 base >>= 1;
 672                 end >>= 1;
 673         }
 674         return t;
 675 }
 676
 677 static int iommu_flush_iotlb_psi(struct intel_iommu *iommu, u16 did,
 678         u64 addr, unsigned int pages, int non_present_entry_flush)
 679 {
 680         unsigned int align;
 681
 682         BUG_ON(addr & (~PAGE_MASK_4K));
 683         BUG_ON(pages == 0);
 684
 685         /* Fallback to domain selective flush if no PSI support */
 686         if (!cap_pgsel_inv(iommu->cap))
 687                 return iommu_flush_iotlb_dsi(iommu, did,
 688                         non_present_entry_flush);
 689
 690         /*
 691          * PSI requires page size to be 2 ^ x, and the base address is naturally
 692          * aligned to the size
 693          */
 694         align = iommu_get_alignment(addr >> PAGE_SHIFT_4K, pages);
 695         /* Fallback to domain selective flush if size is too big */
 696         if (align > cap_max_amask_val(iommu->cap))
 697                 return iommu_flush_iotlb_dsi(iommu, did,
 698                         non_present_entry_flush);
 699
 700         addr >>= PAGE_SHIFT_4K + align;
 701         addr <<= PAGE_SHIFT_4K + align;
 702
 703         return __iommu_flush_iotlb(iommu, did, addr, align,
 704                 DMA_TLB_PSI_FLUSH, non_present_entry_flush);
 705 }
 706
 707 static int iommu_enable_translation(struct intel_iommu *iommu)
 708 {
 709         u32 sts;
 710         unsigned long flags;
 711
 712         spin_lock_irqsave(&iommu->register_lock, flags);
 713         writel(iommu->gcmd|DMA_GCMD_TE, iommu->reg + DMAR_GCMD_REG);
 714
 715         /* Make sure hardware complete it */
 716         IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
 717                 readl, (sts & DMA_GSTS_TES), sts);
 718
 719         iommu->gcmd |= DMA_GCMD_TE;
 720         spin_unlock_irqrestore(&iommu->register_lock, flags);
 721         return 0;
 722 }
 723
 724 static int iommu_disable_translation(struct intel_iommu *iommu)
 725 {
 726         u32 sts;
 727         unsigned long flag;
 728
 729         spin_lock_irqsave(&iommu->register_lock, flag);
 730         iommu->gcmd &= ~DMA_GCMD_TE;
 731         writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
 732
 733         /* Make sure hardware complete it */
 734         IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
 735                 readl, (!(sts & DMA_GSTS_TES)), sts);
 736
 737         spin_unlock_irqrestore(&iommu->register_lock, flag);
 738         return 0;
 739 }
 740
 741 static int iommu_init_domains(struct intel_iommu *iommu)
 742 {
 743         unsigned long ndomains;
 744         unsigned long nlongs;
 745
 746         ndomains = cap_ndoms(iommu->cap);
 747         pr_debug("Number of Domains supportd <%ld>\n", ndomains);
 748         nlongs = BITS_TO_LONGS(ndomains);
 749
 750         /* TBD: there might be 64K domains,
 751          * consider other allocation for future chip
 752          */
 753         iommu->domain_ids = kcalloc(nlongs, sizeof(unsigned long), GFP_KERNEL);
 754         if (!iommu->domain_ids) {
 755                 printk(KERN_ERR "Allocating domain id array failed\n");
 756                 return -ENOMEM;
 757         }
 758         iommu->domains = kcalloc(ndomains, sizeof(struct dmar_domain *),
 759                         GFP_KERNEL);
 760         if (!iommu->domains) {
 761                 printk(KERN_ERR "Allocating domain array failed\n");
 762                 kfree(iommu->domain_ids);
 763                 return -ENOMEM;
 764         }
 765
 766         /*
 767          * if Caching mode is set, then invalid translations are tagged
 768          * with domainid 0. Hence we need to pre-allocate it.
 769          */
 770         if (cap_caching_mode(iommu->cap))
 771                 set_bit(0, iommu->domain_ids);
 772         return 0;
 773 }
 774
 775 static struct intel_iommu *alloc_iommu(struct dmar_drhd_unit *drhd)
 776 {
 777         struct intel_iommu *iommu;
 778         int ret;
 779         int map_size;
 780         u32 ver;
 781
 782         iommu = kzalloc(sizeof(*iommu), GFP_KERNEL);
 783         if (!iommu)
 784                 return NULL;
 785         iommu->reg = ioremap(drhd->reg_base_addr, PAGE_SIZE_4K);
 786         if (!iommu->reg) {
 787                 printk(KERN_ERR "IOMMU: can't map the region\n");
 788                 goto error;
 789         }
 790         iommu->cap = dmar_readq(iommu->reg + DMAR_CAP_REG);
 791         iommu->ecap = dmar_readq(iommu->reg + DMAR_ECAP_REG);
 792
 793         /* the registers might be more than one page */
 794         map_size = max_t(int, ecap_max_iotlb_offset(iommu->ecap),
 795                 cap_max_fault_reg_offset(iommu->cap));
 796         map_size = PAGE_ALIGN_4K(map_size);
 797         if (map_size > PAGE_SIZE_4K) {
 798                 iounmap(iommu->reg);
 799                 iommu->reg = ioremap(drhd->reg_base_addr, map_size);
 800                 if (!iommu->reg) {
 801                         printk(KERN_ERR "IOMMU: can't map the region\n");
 802                         goto error;
 803                 }
 804         }
 805
 806         ver = readl(iommu->reg + DMAR_VER_REG);
 807         pr_debug("IOMMU %llx: ver %d:%d cap %llx ecap %llx\n",
 808                 drhd->reg_base_addr, DMAR_VER_MAJOR(ver), DMAR_VER_MINOR(ver),
 809                 iommu->cap, iommu->ecap);
 810         ret = iommu_init_domains(iommu);
 811         if (ret)
 812                 goto error_unmap;
 813         spin_lock_init(&iommu->lock);
 814         spin_lock_init(&iommu->register_lock);
 815
 816         drhd->iommu = iommu;
 817         return iommu;
 818 error_unmap:
 819         iounmap(iommu->reg);
 820         iommu->reg = 0;
 821 error:
 822         kfree(iommu);
 823         return NULL;
 824 }
 825
 826 static void domain_exit(struct dmar_domain *domain);
 827 static void free_iommu(struct intel_iommu *iommu)
 828 {
 829         struct dmar_domain *domain;
 830         int i;
 831
 832         if (!iommu)
 833                 return;
 834
 835         i = find_first_bit(iommu->domain_ids, cap_ndoms(iommu->cap));
 836         for (; i < cap_ndoms(iommu->cap); ) {
 837                 domain = iommu->domains[i];
 838                 clear_bit(i, iommu->domain_ids);
 839                 domain_exit(domain);
 840                 i = find_next_bit(iommu->domain_ids,
 841                         cap_ndoms(iommu->cap), i+1);
 842         }
 843
 844         if (iommu->gcmd & DMA_GCMD_TE)
 845                 iommu_disable_translation(iommu);
 846
 847         if (iommu->irq) {
 848                 set_irq_data(iommu->irq, NULL);
 849                 /* This will mask the irq */
 850                 free_irq(iommu->irq, iommu);
 851                 destroy_irq(iommu->irq);
 852         }
 853
 854         kfree(iommu->domains);
 855         kfree(iommu->domain_ids);
 856
 857         /* free context mapping */
 858         free_context_table(iommu);
 859
 860         if (iommu->reg)
 861                 iounmap(iommu->reg);
 862         kfree(iommu);
 863 }
 864
 865 static struct dmar_domain * iommu_alloc_domain(struct intel_iommu *iommu)
 866 {
 867         unsigned long num;
 868         unsigned long ndomains;
 869         struct dmar_domain *domain;
 870         unsigned long flags;
 871
 872         domain = alloc_domain_mem();
 873         if (!domain)
 874                 return NULL;
 875
 876         ndomains = cap_ndoms(iommu->cap);
 877
 878         spin_lock_irqsave(&iommu->lock, flags);
 879         num = find_first_zero_bit(iommu->domain_ids, ndomains);
 880         if (num >= ndomains) {
 881                 spin_unlock_irqrestore(&iommu->lock, flags);
 882                 free_domain_mem(domain);
 883                 printk(KERN_ERR "IOMMU: no free domain ids\n");
 884                 return NULL;
 885         }
 886
 887         set_bit(num, iommu->domain_ids);
 888         domain->id = num;
 889         domain->iommu = iommu;
 890         iommu->domains[num] = domain;
 891         spin_unlock_irqrestore(&iommu->lock, flags);
 892
 893         return domain;
 894 }
 895
 896 static void iommu_free_domain(struct dmar_domain *domain)
 897 {
 898         unsigned long flags;
 899
 900         spin_lock_irqsave(&domain->iommu->lock, flags);
 901         clear_bit(domain->id, domain->iommu->domain_ids);
 902         spin_unlock_irqrestore(&domain->iommu->lock, flags);
 903 }
 904
 905 static struct iova_domain reserved_iova_list;
 906
 907 static void dmar_init_reserved_ranges(void)
 908 {
 909         struct pci_dev *pdev = NULL;
 910         struct iova *iova;
 911         int i;
 912         u64 addr, size;
 913
 914         init_iova_domain(&reserved_iova_list);
 915
 916         /* IOAPIC ranges shouldn't be accessed by DMA */
 917         iova = reserve_iova(&reserved_iova_list, IOVA_PFN(IOAPIC_RANGE_START),
 918                 IOVA_PFN(IOAPIC_RANGE_END));
 919         if (!iova)
 920                 printk(KERN_ERR "Reserve IOAPIC range failed\n");
 921
 922         /* Reserve all PCI MMIO to avoid peer-to-peer access */
 923         for_each_pci_dev(pdev) {
 924                 struct resource *r;
 925
 926                 for (i = 0; i < PCI_NUM_RESOURCES; i++) {
 927                         r = &pdev->resource[i];
 928                         if (!r->flags || !(r->flags & IORESOURCE_MEM))
 929                                 continue;
 930                         addr = r->start;
 931                         addr &= PAGE_MASK_4K;
 932                         size = r->end - addr;
 933                         size = PAGE_ALIGN_4K(size);
 934                         iova = reserve_iova(&reserved_iova_list, IOVA_PFN(addr),
 935                                 IOVA_PFN(size + addr) - 1);
 936                         if (!iova)
 937                                 printk(KERN_ERR "Reserve iova failed\n");
 938                 }
 939         }
 940
 941 }
 942
 943 static void domain_reserve_special_ranges(struct dmar_domain *domain)
 944 {
 945         copy_reserved_iova(&reserved_iova_list, &domain->iovad);
 946 }
 947
 948 static inline int guestwidth_to_adjustwidth(int gaw)
 949 {
 950         int agaw;
 951         int r = (gaw - 12) % 9;
 952
 953         if (r == 0)
 954                 agaw = gaw;
 955         else
 956                 agaw = gaw + 9 - r;
 957         if (agaw > 64)
 958                 agaw = 64;
 959         return agaw;
 960 }
 961
 962 static int domain_init(struct dmar_domain *domain, int guest_width)
 963 {
 964         struct intel_iommu *iommu;
 965         int adjust_width, agaw;
 966         unsigned long sagaw;
 967
 968         init_iova_domain(&domain->iovad);
 969         spin_lock_init(&domain->mapping_lock);
 970
 971         domain_reserve_special_ranges(domain);
 972
 973         /* calculate AGAW */
 974         iommu = domain->iommu;
 975         if (guest_width > cap_mgaw(iommu->cap))
 976                 guest_width = cap_mgaw(iommu->cap);
 977         domain->gaw = guest_width;
 978         adjust_width = guestwidth_to_adjustwidth(guest_width);
 979         agaw = width_to_agaw(adjust_width);
 980         sagaw = cap_sagaw(iommu->cap);
 981         if (!test_bit(agaw, &sagaw)) {
 982                 /* hardware doesn't support it, choose a bigger one */
 983                 pr_debug("IOMMU: hardware doesn't support agaw %d\n", agaw);
 984                 agaw = find_next_bit(&sagaw, 5, agaw);
 985                 if (agaw >= 5)
 986                         return -ENODEV;
 987         }
 988         domain->agaw = agaw;
 989         INIT_LIST_HEAD(&domain->devices);
 990
 991         /* always allocate the top pgd */
 992         domain->pgd = (struct dma_pte *)alloc_pgtable_page();
 993         if (!domain->pgd)
 994                 return -ENOMEM;
 995         __iommu_flush_cache(iommu, domain->pgd, PAGE_SIZE_4K);
 996         return 0;
 997 }
 998
 999 static void domain_exit(struct dmar_domain *domain)
1000 {
1001         u64 end;
1002
1003         /* Domain 0 is reserved, so dont process it */
1004         if (!domain)
1005                 return;
1006
1007         domain_remove_dev_info(domain);
1008         /* destroy iovas */
1009         put_iova_domain(&domain->iovad);
1010         end = DOMAIN_MAX_ADDR(domain->gaw);
1011         end = end & (~PAGE_MASK_4K);
1012
1013         /* clear ptes */
1014         dma_pte_clear_range(domain, 0, end);
1015
1016         /* free page tables */
1017         dma_pte_free_pagetable(domain, 0, end);
1018
1019         iommu_free_domain(domain);
1020         free_domain_mem(domain);
1021 }
1022
1023 static int domain_context_mapping_one(struct dmar_domain *domain,
1024                 u8 bus, u8 devfn)
1025 {
1026         struct context_entry *context;
1027         struct intel_iommu *iommu = domain->iommu;
1028         unsigned long flags;
1029
1030         pr_debug("Set context mapping for %02x:%02x.%d\n",
1031                 bus, PCI_SLOT(devfn), PCI_FUNC(devfn));
1032         BUG_ON(!domain->pgd);
1033         context = device_to_context_entry(iommu, bus, devfn);
1034         if (!context)
1035                 return -ENOMEM;
1036         spin_lock_irqsave(&iommu->lock, flags);
1037         if (context_present(*context)) {
1038                 spin_unlock_irqrestore(&iommu->lock, flags);
1039                 return 0;
1040         }
1041
1042         context_set_domain_id(*context, domain->id);
1043         context_set_address_width(*context, domain->agaw);
1044         context_set_address_root(*context, virt_to_phys(domain->pgd));
1045         context_set_translation_type(*context, CONTEXT_TT_MULTI_LEVEL);
1046         context_set_fault_enable(*context);
1047         context_set_present(*context);
1048         __iommu_flush_cache(iommu, context, sizeof(*context));
1049
1050         /* it's a non-present to present mapping */
1051         if (iommu_flush_context_device(iommu, domain->id,
1052                         (((u16)bus) << 8) | devfn, DMA_CCMD_MASK_NOBIT, 1))
1053                 iommu_flush_write_buffer(iommu);
1054         else
1055                 iommu_flush_iotlb_dsi(iommu, 0, 0);
1056         spin_unlock_irqrestore(&iommu->lock, flags);
1057         return 0;
1058 }
1059
1060 static int
1061 domain_context_mapping(struct dmar_domain *domain, struct pci_dev *pdev)
1062 {
1063         int ret;
1064         struct pci_dev *tmp, *parent;
1065
1066         ret = domain_context_mapping_one(domain, pdev->bus->number,
1067                 pdev->devfn);
1068         if (ret)
1069                 return ret;
1070
1071         /* dependent device mapping */
1072         tmp = pci_find_upstream_pcie_bridge(pdev);
1073         if (!tmp)
1074                 return 0;
1075         /* Secondary interface's bus number and devfn 0 */
1076         parent = pdev->bus->self;
1077         while (parent != tmp) {
1078                 ret = domain_context_mapping_one(domain, parent->bus->number,
1079                         parent->devfn);
1080                 if (ret)
1081                         return ret;
1082                 parent = parent->bus->self;
1083         }
1084         if (tmp->is_pcie) /* this is a PCIE-to-PCI bridge */
1085                 return domain_context_mapping_one(domain,
1086                         tmp->subordinate->number, 0);
1087         else /* this is a legacy PCI bridge */
1088                 return domain_context_mapping_one(domain,
1089                         tmp->bus->number, tmp->devfn);
1090 }
1091
1092 static int domain_context_mapped(struct dmar_domain *domain,
1093         struct pci_dev *pdev)
1094 {
1095         int ret;
1096         struct pci_dev *tmp, *parent;
1097
1098         ret = device_context_mapped(domain->iommu,
1099                 pdev->bus->number, pdev->devfn);
1100         if (!ret)
1101                 return ret;
1102         /* dependent device mapping */
1103         tmp = pci_find_upstream_pcie_bridge(pdev);
1104         if (!tmp)
1105                 return ret;
1106         /* Secondary interface's bus number and devfn 0 */
1107         parent = pdev->bus->self;
1108         while (parent != tmp) {
1109                 ret = device_context_mapped(domain->iommu, parent->bus->number,
1110                         parent->devfn);
1111                 if (!ret)
1112                         return ret;
1113                 parent = parent->bus->self;
1114         }
1115         if (tmp->is_pcie)
1116                 return device_context_mapped(domain->iommu,
1117                         tmp->subordinate->number, 0);
1118         else
1119                 return device_context_mapped(domain->iommu,
1120                         tmp->bus->number, tmp->devfn);
1121 }
1122
1123 static int
1124 domain_page_mapping(struct dmar_domain *domain, dma_addr_t iova,
1125                         u64 hpa, size_t size, int prot)
1126 {
1127         u64 start_pfn, end_pfn;
1128         struct dma_pte *pte;
1129         int index;
1130
1131         if ((prot & (DMA_PTE_READ|DMA_PTE_WRITE)) == 0)
1132                 return -EINVAL;
1133         iova &= PAGE_MASK_4K;
1134         start_pfn = ((u64)hpa) >> PAGE_SHIFT_4K;
1135         end_pfn = (PAGE_ALIGN_4K(((u64)hpa) + size)) >> PAGE_SHIFT_4K;
1136         index = 0;
1137         while (start_pfn < end_pfn) {
1138                 pte = addr_to_dma_pte(domain, iova + PAGE_SIZE_4K * index);
1139                 if (!pte)
1140                         return -ENOMEM;
1141                 /* We don't need lock here, nobody else
1142                  * touches the iova range
1143                  */
1144                 BUG_ON(dma_pte_addr(*pte));
1145                 dma_set_pte_addr(*pte, start_pfn << PAGE_SHIFT_4K);
1146                 dma_set_pte_prot(*pte, prot);
1147                 __iommu_flush_cache(domain->iommu, pte, sizeof(*pte));
1148                 start_pfn++;
1149                 index++;
1150         }
1151         return 0;
1152 }
1153
1154 static void detach_domain_for_dev(struct dmar_domain *domain, u8 bus, u8 devfn)
1155 {
1156         clear_context_table(domain->iommu, bus, devfn);
1157         iommu_flush_context_global(domain->iommu, 0);
1158         iommu_flush_iotlb_global(domain->iommu, 0);
1159 }
1160
1161 static void domain_remove_dev_info(struct dmar_domain *domain)
1162 {
1163         struct device_domain_info *info;
1164         unsigned long flags;
1165
1166         spin_lock_irqsave(&device_domain_lock, flags);
1167         while (!list_empty(&domain->devices)) {
1168                 info = list_entry(domain->devices.next,
1169                         struct device_domain_info, link);
1170                 list_del(&info->link);
1171                 list_del(&info->global);
1172                 if (info->dev)
1173                         info->dev->sysdata = NULL;
1174                 spin_unlock_irqrestore(&device_domain_lock, flags);
1175
1176                 detach_domain_for_dev(info->domain, info->bus, info->devfn);
1177                 free_devinfo_mem(info);
1178
1179                 spin_lock_irqsave(&device_domain_lock, flags);
1180         }
1181         spin_unlock_irqrestore(&device_domain_lock, flags);
1182 }
1183
1184 /*
1185  * find_domain
1186  * Note: we use struct pci_dev->sysdata stores the info
1187  */
1188 struct dmar_domain *
1189 find_domain(struct pci_dev *pdev)
1190 {
1191         struct device_domain_info *info;
1192
1193         /* No lock here, assumes no domain exit in normal case */
1194         info = pdev->sysdata;
1195         if (info)
1196                 return info->domain;
1197         return NULL;
1198 }
1199
1200 static int dmar_pci_device_match(struct pci_dev *devices[], int cnt,
1201      struct pci_dev *dev)
1202 {
1203         int index;
1204
1205         while (dev) {
1206                 for (index = 0; index < cnt; index ++)
1207                         if (dev == devices[index])
1208                                 return 1;
1209
1210                 /* Check our parent */
1211                 dev = dev->bus->self;
1212         }
1213
1214         return 0;
1215 }
1216
1217 static struct dmar_drhd_unit *
1218 dmar_find_matched_drhd_unit(struct pci_dev *dev)
1219 {
1220         struct dmar_drhd_unit *drhd = NULL;
1221
1222         list_for_each_entry(drhd, &dmar_drhd_units, list) {
1223                 if (drhd->include_all || dmar_pci_device_match(drhd->devices,
1224                                                 drhd->devices_cnt, dev))
1225                         return drhd;
1226         }
1227
1228         return NULL;
1229 }
1230
1231 /* domain is initialized */
1232 static struct dmar_domain *get_domain_for_dev(struct pci_dev *pdev, int gaw)
1233 {
1234         struct dmar_domain *domain, *found = NULL;
1235         struct intel_iommu *iommu;
1236         struct dmar_drhd_unit *drhd;
1237         struct device_domain_info *info, *tmp;
1238         struct pci_dev *dev_tmp;
1239         unsigned long flags;
1240         int bus = 0, devfn = 0;
1241
1242         domain = find_domain(pdev);
1243         if (domain)
1244                 return domain;
1245
1246         dev_tmp = pci_find_upstream_pcie_bridge(pdev);
1247         if (dev_tmp) {
1248                 if (dev_tmp->is_pcie) {
1249                         bus = dev_tmp->subordinate->number;
1250                         devfn = 0;
1251                 } else {
1252                         bus = dev_tmp->bus->number;
1253                         devfn = dev_tmp->devfn;
1254                 }
1255                 spin_lock_irqsave(&device_domain_lock, flags);
1256                 list_for_each_entry(info, &device_domain_list, global) {
1257                         if (info->bus == bus && info->devfn == devfn) {
1258                                 found = info->domain;
1259                                 break;
1260                         }
1261                 }
1262                 spin_unlock_irqrestore(&device_domain_lock, flags);
1263                 /* pcie-pci bridge already has a domain, uses it */
1264                 if (found) {
1265                         domain = found;
1266                         goto found_domain;
1267                 }
1268         }
1269
1270         /* Allocate new domain for the device */
1271         drhd = dmar_find_matched_drhd_unit(pdev);
1272         if (!drhd) {
1273                 printk(KERN_ERR "IOMMU: can't find DMAR for device %s\n",
1274                         pci_name(pdev));
1275                 return NULL;
1276         }
1277         iommu = drhd->iommu;
1278
1279         domain = iommu_alloc_domain(iommu);
1280         if (!domain)
1281                 goto error;
1282
1283         if (domain_init(domain, gaw)) {
1284                 domain_exit(domain);
1285                 goto error;
1286         }
1287
1288         /* register pcie-to-pci device */
1289         if (dev_tmp) {
1290                 info = alloc_devinfo_mem();
1291                 if (!info) {
1292                         domain_exit(domain);
1293                         goto error;
1294                 }
1295                 info->bus = bus;
1296                 info->devfn = devfn;
1297                 info->dev = NULL;
1298                 info->domain = domain;
1299                 /* This domain is shared by devices under p2p bridge */
1300                 domain->flags |= DOMAIN_FLAG_MULTIPLE_DEVICES;
1301
1302                 /* pcie-to-pci bridge already has a domain, uses it */
1303                 found = NULL;
1304                 spin_lock_irqsave(&device_domain_lock, flags);
1305                 list_for_each_entry(tmp, &device_domain_list, global) {
1306                         if (tmp->bus == bus && tmp->devfn == devfn) {
1307                                 found = tmp->domain;
1308                                 break;
1309                         }
1310                 }
1311                 if (found) {
1312                         free_devinfo_mem(info);
1313                         domain_exit(domain);
1314                         domain = found;
1315                 } else {
1316                         list_add(&info->link, &domain->devices);
1317                         list_add(&info->global, &device_domain_list);
1318                 }
1319                 spin_unlock_irqrestore(&device_domain_lock, flags);
1320         }
1321
1322 found_domain:
1323         info = alloc_devinfo_mem();
1324         if (!info)
1325                 goto error;
1326         info->bus = pdev->bus->number;
1327         info->devfn = pdev->devfn;
1328         info->dev = pdev;
1329         info->domain = domain;
1330         spin_lock_irqsave(&device_domain_lock, flags);
1331         /* somebody is fast */
1332         found = find_domain(pdev);
1333         if (found != NULL) {
1334                 spin_unlock_irqrestore(&device_domain_lock, flags);
1335                 if (found != domain) {
1336                         domain_exit(domain);
1337                         domain = found;
1338                 }
1339                 free_devinfo_mem(info);
1340                 return domain;
1341         }
1342         list_add(&info->link, &domain->devices);
1343         list_add(&info->global, &device_domain_list);
1344         pdev->sysdata = info;
1345         spin_unlock_irqrestore(&device_domain_lock, flags);
1346         return domain;
1347 error:
1348         /* recheck it here, maybe others set it */
1349         return find_domain(pdev);
1350 }
1351
1352 static int iommu_prepare_identity_map(struct pci_dev *pdev, u64 start, u64 end)
1353 {
1354         struct dmar_domain *domain;
1355         unsigned long size;
1356         u64 base;
1357         int ret;
1358
1359         printk(KERN_INFO
1360                 "IOMMU: Setting identity map for device %s [0x%Lx - 0x%Lx]\n",
1361                 pci_name(pdev), start, end);
1362         /* page table init */
1363         domain = get_domain_for_dev(pdev, DEFAULT_DOMAIN_ADDRESS_WIDTH);
1364         if (!domain)
1365                 return -ENOMEM;
1366
1367         /* The address might not be aligned */
1368         base = start & PAGE_MASK_4K;
1369         size = end - base;
1370         size = PAGE_ALIGN_4K(size);
1371         if (!reserve_iova(&domain->iovad, IOVA_PFN(base),
1372                         IOVA_PFN(base + size) - 1)) {
1373                 printk(KERN_ERR "IOMMU: reserve iova failed\n");
1374                 ret = -ENOMEM;
1375                 goto error;
1376         }
1377
1378         pr_debug("Mapping reserved region %lx@%llx for %s\n",
1379                 size, base, pci_name(pdev));
1380         /*
1381          * RMRR range might have overlap with physical memory range,
1382          * clear it first
1383          */
1384         dma_pte_clear_range(domain, base, base + size);
1385
1386         ret = domain_page_mapping(domain, base, base, size,
1387                 DMA_PTE_READ|DMA_PTE_WRITE);
1388         if (ret)
1389                 goto error;
1390
1391         /* context entry init */
1392         ret = domain_context_mapping(domain, pdev);
1393         if (!ret)
1394                 return 0;
1395 error:
1396         domain_exit(domain);
1397         return ret;
1398
1399 }
1400
1401 static inline int iommu_prepare_rmrr_dev(struct dmar_rmrr_unit *rmrr,
1402         struct pci_dev *pdev)
1403 {
1404         if (pdev->sysdata == DUMMY_DEVICE_DOMAIN_INFO)
1405                 return 0;
1406         return iommu_prepare_identity_map(pdev, rmrr->base_address,
1407                 rmrr->end_address + 1);
1408 }
1409
1410 int __init init_dmars(void)
1411 {
1412         struct dmar_drhd_unit *drhd;
1413         struct dmar_rmrr_unit *rmrr;
1414         struct pci_dev *pdev;
1415         struct intel_iommu *iommu;
1416         int ret, unit = 0;
1417
1418         /*
1419          * for each drhd
1420          *    allocate root
1421          *    initialize and program root entry to not present
1422          * endfor
1423          */
1424         for_each_drhd_unit(drhd) {
1425                 if (drhd->ignored)
1426                         continue;
1427                 iommu = alloc_iommu(drhd);
1428                 if (!iommu) {
1429                         ret = -ENOMEM;
1430                         goto error;
1431                 }
1432
1433                 /*
1434                  * TBD:
1435                  * we could share the same root & context tables
1436                  * amoung all IOMMU's. Need to Split it later.
1437                  */
1438                 ret = iommu_alloc_root_entry(iommu);
1439                 if (ret) {
1440                         printk(KERN_ERR "IOMMU: allocate root entry failed\n");
1441                         goto error;
1442                 }
1443         }
1444
1445         /*
1446          * For each rmrr
1447          *   for each dev attached to rmrr
1448          *   do
1449          *     locate drhd for dev, alloc domain for dev
1450          *     allocate free domain
1451          *     allocate page table entries for rmrr
1452          *     if context not allocated for bus
1453          *           allocate and init context
1454          *           set present in root table for this bus
1455          *     init context with domain, translation etc
1456          *    endfor
1457          * endfor
1458          */
1459         for_each_rmrr_units(rmrr) {
1460                 int i;
1461                 for (i = 0; i < rmrr->devices_cnt; i++) {
1462                         pdev = rmrr->devices[i];
1463                         /* some BIOS lists non-exist devices in DMAR table */
1464                         if (!pdev)
1465                                 continue;
1466                         ret = iommu_prepare_rmrr_dev(rmrr, pdev);
1467                         if (ret)
1468                                 printk(KERN_ERR
1469                                  "IOMMU: mapping reserved region failed\n");
1470                 }
1471         }
1472
1473         /*
1474          * for each drhd
1475          *   enable fault log
1476          *   global invalidate context cache
1477          *   global invalidate iotlb
1478          *   enable translation
1479          */
1480         for_each_drhd_unit(drhd) {
1481                 if (drhd->ignored)
1482                         continue;
1483                 iommu = drhd->iommu;
1484                 sprintf (iommu->name, "dmar%d", unit++);
1485
1486                 iommu_flush_write_buffer(iommu);
1487
1488                 iommu_set_root_entry(iommu);
1489
1490                 iommu_flush_context_global(iommu, 0);
1491                 iommu_flush_iotlb_global(iommu, 0);
1492
1493                 ret = iommu_enable_translation(iommu);
1494                 if (ret)
1495                         goto error;
1496         }
1497
1498         return 0;
1499 error:
1500         for_each_drhd_unit(drhd) {
1501                 if (drhd->ignored)
1502                         continue;
1503                 iommu = drhd->iommu;
1504                 free_iommu(iommu);
1505         }
1506         return ret;
1507 }
1508
1509 static inline u64 aligned_size(u64 host_addr, size_t size)
1510 {
1511         u64 addr;
1512         addr = (host_addr & (~PAGE_MASK_4K)) + size;
1513         return PAGE_ALIGN_4K(addr);
1514 }
1515
1516 struct iova *
1517 iommu_alloc_iova(struct dmar_domain *domain, void *host_addr, size_t size,
1518                 u64 start, u64 end)
1519 {
1520         u64 start_addr;
1521         struct iova *piova;
1522
1523         /* Make sure it's in range */
1524         if ((start > DOMAIN_MAX_ADDR(domain->gaw)) || end < start)
1525                 return NULL;
1526
1527         end = min_t(u64, DOMAIN_MAX_ADDR(domain->gaw), end);
1528         start_addr = PAGE_ALIGN_4K(start);
1529         size = aligned_size((u64)host_addr, size);
1530         if (!size || (start_addr + size > end))
1531                 return NULL;
1532
1533         piova = alloc_iova(&domain->iovad,
1534                         size >> PAGE_SHIFT_4K, IOVA_PFN(end));
1535
1536         return piova;
1537 }
1538
1539 static dma_addr_t __intel_map_single(struct device *dev, void *addr,
1540         size_t size, int dir, u64 *flush_addr, unsigned int *flush_size)
1541 {
1542         struct dmar_domain *domain;
1543         struct pci_dev *pdev = to_pci_dev(dev);
1544         int ret;
1545         int prot = 0;
1546         struct iova *iova = NULL;
1547         u64 start_addr;
1548
1549         addr = (void *)virt_to_phys(addr);
1550
1551         domain = get_domain_for_dev(pdev,
1552                         DEFAULT_DOMAIN_ADDRESS_WIDTH);
1553         if (!domain) {
1554                 printk(KERN_ERR
1555                         "Allocating domain for %s failed", pci_name(pdev));
1556                 return 0;
1557         }
1558
1559         start_addr = IOVA_START_ADDR;
1560
1561         if (pdev->dma_mask <= DMA_32BIT_MASK) {
1562                 iova = iommu_alloc_iova(domain, addr, size, start_addr,
1563                         pdev->dma_mask);
1564         } else  {
1565                 /*
1566                  * First try to allocate an io virtual address in
1567                  * DMA_32BIT_MASK and if that fails then try allocating
1568                  * from higer range
1569                  */
1570                 iova = iommu_alloc_iova(domain, addr, size, start_addr,
1571                         DMA_32BIT_MASK);
1572                 if (!iova)
1573                         iova = iommu_alloc_iova(domain, addr, size, start_addr,
1574                         pdev->dma_mask);
1575         }
1576
1577         if (!iova) {
1578                 printk(KERN_ERR"Allocating iova for %s failed", pci_name(pdev));
1579                 return 0;
1580         }
1581
1582         /* make sure context mapping is ok */
1583         if (unlikely(!domain_context_mapped(domain, pdev))) {
1584                 ret = domain_context_mapping(domain, pdev);
1585                 if (ret)
1586                         goto error;
1587         }
1588
1589         /*
1590          * Check if DMAR supports zero-length reads on write only
1591          * mappings..
1592          */
1593         if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL || \
1594                         !cap_zlr(domain->iommu->cap))
1595                 prot |= DMA_PTE_READ;
1596         if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
1597                 prot |= DMA_PTE_WRITE;
1598         /*
1599          * addr - (addr + size) might be partial page, we should map the whole
1600          * page.  Note: if two part of one page are separately mapped, we
1601          * might have two guest_addr mapping to the same host addr, but this
1602          * is not a big problem
1603          */
1604         ret = domain_page_mapping(domain, iova->pfn_lo << PAGE_SHIFT_4K,
1605                 ((u64)addr) & PAGE_MASK_4K,
1606                 (iova->pfn_hi - iova->pfn_lo + 1) << PAGE_SHIFT_4K, prot);
1607         if (ret)
1608                 goto error;
1609
1610         pr_debug("Device %s request: %lx@%llx mapping: %lx@%llx, dir %d\n",
1611                 pci_name(pdev), size, (u64)addr,
1612                 (iova->pfn_hi - iova->pfn_lo + 1) << PAGE_SHIFT_4K,
1613                 (u64)(iova->pfn_lo << PAGE_SHIFT_4K), dir);
1614
1615         *flush_addr = iova->pfn_lo << PAGE_SHIFT_4K;
1616         *flush_size = (iova->pfn_hi - iova->pfn_lo + 1) << PAGE_SHIFT_4K;
1617         return (iova->pfn_lo << PAGE_SHIFT_4K) + ((u64)addr & (~PAGE_MASK_4K));
1618 error:
1619         __free_iova(&domain->iovad, iova);
1620         printk(KERN_ERR"Device %s request: %lx@%llx dir %d --- failed\n",
1621                 pci_name(pdev), size, (u64)addr, dir);
1622         return 0;
1623 }
1624
1625 static dma_addr_t intel_map_single(struct device *hwdev, void *addr,
1626         size_t size, int dir)
1627 {
1628         struct pci_dev *pdev = to_pci_dev(hwdev);
1629         dma_addr_t ret;
1630         struct dmar_domain *domain;
1631         u64 flush_addr;
1632         unsigned int flush_size;
1633
1634         BUG_ON(dir == DMA_NONE);
1635         if (pdev->sysdata == DUMMY_DEVICE_DOMAIN_INFO)
1636                 return virt_to_bus(addr);
1637
1638         ret = __intel_map_single(hwdev, addr, size,
1639                         dir, &flush_addr, &flush_size);
1640         if (ret) {
1641                 domain = find_domain(pdev);
1642                 /* it's a non-present to present mapping */
1643                 if (iommu_flush_iotlb_psi(domain->iommu, domain->id,
1644                                 flush_addr, flush_size >> PAGE_SHIFT_4K, 1))
1645                         iommu_flush_write_buffer(domain->iommu);
1646         }
1647         return ret;
1648 }
1649
1650 static void __intel_unmap_single(struct device *dev, dma_addr_t dev_addr,
1651         size_t size, int dir, u64 *flush_addr, unsigned int *flush_size)
1652 {
1653         struct dmar_domain *domain;
1654         struct pci_dev *pdev = to_pci_dev(dev);
1655         struct iova *iova;
1656
1657         domain = find_domain(pdev);
1658         BUG_ON(!domain);
1659
1660         iova = find_iova(&domain->iovad, IOVA_PFN(dev_addr));
1661         if (!iova) {
1662                 *flush_size = 0;
1663                 return;
1664         }
1665         pr_debug("Device %s unmapping: %lx@%llx\n",
1666                 pci_name(pdev),
1667                 (iova->pfn_hi - iova->pfn_lo + 1) << PAGE_SHIFT_4K,
1668                 (u64)(iova->pfn_lo << PAGE_SHIFT_4K));
1669
1670         *flush_addr = iova->pfn_lo << PAGE_SHIFT_4K;
1671         *flush_size = (iova->pfn_hi - iova->pfn_lo + 1) << PAGE_SHIFT_4K;
1672         /*  clear the whole page, not just dev_addr - (dev_addr + size) */
1673         dma_pte_clear_range(domain, *flush_addr, *flush_addr + *flush_size);
1674         /* free page tables */
1675         dma_pte_free_pagetable(domain, *flush_addr, *flush_addr + *flush_size);
1676         /* free iova */
1677         __free_iova(&domain->iovad, iova);
1678 }
1679
1680 static void intel_unmap_single(struct device *dev, dma_addr_t dev_addr,
1681         size_t size, int dir)
1682 {
1683         struct pci_dev *pdev = to_pci_dev(dev);
1684         struct dmar_domain *domain;
1685         u64 flush_addr;
1686         unsigned int flush_size;
1687
1688         if (pdev->sysdata == DUMMY_DEVICE_DOMAIN_INFO)
1689                 return;
1690
1691         domain = find_domain(pdev);
1692         __intel_unmap_single(dev, dev_addr, size,
1693                 dir, &flush_addr, &flush_size);
1694         if (flush_size == 0)
1695                 return;
1696         if (iommu_flush_iotlb_psi(domain->iommu, domain->id, flush_addr,
1697                         flush_size >> PAGE_SHIFT_4K, 0))
1698                 iommu_flush_write_buffer(domain->iommu);
1699 }
1700
1701 static void * intel_alloc_coherent(struct device *hwdev, size_t size,
1702                        dma_addr_t *dma_handle, gfp_t flags)
1703 {
1704         void *vaddr;
1705         int order;
1706
1707         size = PAGE_ALIGN_4K(size);
1708         order = get_order(size);
1709         flags &= ~(GFP_DMA | GFP_DMA32);
1710
1711         vaddr = (void *)__get_free_pages(flags, order);
1712         if (!vaddr)
1713                 return NULL;
1714         memset(vaddr, 0, size);
1715
1716         *dma_handle = intel_map_single(hwdev, vaddr, size, DMA_BIDIRECTIONAL);
1717         if (*dma_handle)
1718                 return vaddr;
1719         free_pages((unsigned long)vaddr, order);
1720         return NULL;
1721 }
1722
1723 static void intel_free_coherent(struct device *hwdev, size_t size,
1724         void *vaddr, dma_addr_t dma_handle)
1725 {
1726         int order;
1727
1728         size = PAGE_ALIGN_4K(size);
1729         order = get_order(size);
1730
1731         intel_unmap_single(hwdev, dma_handle, size, DMA_BIDIRECTIONAL);
1732         free_pages((unsigned long)vaddr, order);
1733 }
1734
1735 static void intel_unmap_sg(struct device *hwdev, struct scatterlist *sg,
1736         int nelems, int dir)
1737 {
1738         int i;
1739         struct pci_dev *pdev = to_pci_dev(hwdev);
1740         struct dmar_domain *domain;
1741         u64 flush_addr;
1742         unsigned int flush_size;
1743
1744         if (pdev->sysdata == DUMMY_DEVICE_DOMAIN_INFO)
1745                 return;
1746
1747         domain = find_domain(pdev);
1748         for (i = 0; i < nelems; i++, sg++)
1749                 __intel_unmap_single(hwdev, sg->dma_address,
1750                         sg->dma_length, dir, &flush_addr, &flush_size);
1751
1752         if (iommu_flush_iotlb_dsi(domain->iommu, domain->id, 0))
1753                 iommu_flush_write_buffer(domain->iommu);
1754 }
1755
1756 #define SG_ENT_VIRT_ADDRESS(sg) (page_address((sg)->page) + (sg)->offset)
1757 static int intel_nontranslate_map_sg(struct device *hddev,
1758         struct scatterlist *sg, int nelems, int dir)
1759 {
1760         int i;
1761
1762         for (i = 0; i < nelems; i++) {
1763                 struct scatterlist *s = &sg[i];
1764                 BUG_ON(!s->page);
1765                 s->dma_address = virt_to_bus(SG_ENT_VIRT_ADDRESS(s));
1766                 s->dma_length = s->length;
1767         }
1768         return nelems;
1769 }
1770
1771 static int intel_map_sg(struct device *hwdev, struct scatterlist *sg,
1772         int nelems, int dir)
1773 {
1774         void *addr;
1775         int i;
1776         dma_addr_t dma_handle;
1777         struct pci_dev *pdev = to_pci_dev(hwdev);
1778         struct dmar_domain *domain;
1779         u64 flush_addr;
1780         unsigned int flush_size;
1781
1782         BUG_ON(dir == DMA_NONE);
1783         if (pdev->sysdata == DUMMY_DEVICE_DOMAIN_INFO)
1784                 return intel_nontranslate_map_sg(hwdev, sg, nelems, dir);
1785
1786         for (i = 0; i < nelems; i++, sg++) {
1787                 addr = SG_ENT_VIRT_ADDRESS(sg);
1788                 dma_handle = __intel_map_single(hwdev, addr,
1789                                 sg->length, dir, &flush_addr, &flush_size);
1790                 if (!dma_handle) {
1791                         intel_unmap_sg(hwdev, sg - i, i, dir);
1792                         sg[0].dma_length = 0;
1793                         return 0;
1794                 }
1795                 sg->dma_address = dma_handle;
1796                 sg->dma_length = sg->length;
1797         }
1798
1799         domain = find_domain(pdev);
1800
1801         /* it's a non-present to present mapping */
1802         if (iommu_flush_iotlb_dsi(domain->iommu, domain->id, 1))
1803                 iommu_flush_write_buffer(domain->iommu);
1804         return nelems;
1805 }
1806
1807 static struct dma_mapping_ops intel_dma_ops = {
1808         .alloc_coherent = intel_alloc_coherent,
1809         .free_coherent = intel_free_coherent,
1810         .map_single = intel_map_single,
1811         .unmap_single = intel_unmap_single,
1812         .map_sg = intel_map_sg,
1813         .unmap_sg = intel_unmap_sg,
1814 };
1815
1816 static inline int iommu_domain_cache_init(void)
1817 {
1818         int ret = 0;
1819
1820         iommu_domain_cache = kmem_cache_create("iommu_domain",
1821                                          sizeof(struct dmar_domain),
1822                                          0,
1823                                          SLAB_HWCACHE_ALIGN,
1824
1825                                          NULL);
1826         if (!iommu_domain_cache) {
1827                 printk(KERN_ERR "Couldn't create iommu_domain cache\n");
1828                 ret = -ENOMEM;
1829         }
1830
1831         return ret;
1832 }
1833
1834 static inline int iommu_devinfo_cache_init(void)
1835 {
1836         int ret = 0;
1837
1838         iommu_devinfo_cache = kmem_cache_create("iommu_devinfo",
1839                                          sizeof(struct device_domain_info),
1840                                          0,
1841                                          SLAB_HWCACHE_ALIGN,
1842
1843                                          NULL);
1844         if (!iommu_devinfo_cache) {
1845                 printk(KERN_ERR "Couldn't create devinfo cache\n");
1846                 ret = -ENOMEM;
1847         }
1848
1849         return ret;
1850 }
1851
1852 static inline int iommu_iova_cache_init(void)
1853 {
1854         int ret = 0;
1855
1856         iommu_iova_cache = kmem_cache_create("iommu_iova",
1857                                          sizeof(struct iova),
1858                                          0,
1859                                          SLAB_HWCACHE_ALIGN,
1860
1861                                          NULL);
1862         if (!iommu_iova_cache) {
1863                 printk(KERN_ERR "Couldn't create iova cache\n");
1864                 ret = -ENOMEM;
1865         }
1866
1867         return ret;
1868 }
1869
1870 static int __init iommu_init_mempool(void)
1871 {
1872         int ret;
1873         ret = iommu_iova_cache_init();
1874         if (ret)
1875                 return ret;
1876
1877         ret = iommu_domain_cache_init();
1878         if (ret)
1879                 goto domain_error;
1880
1881         ret = iommu_devinfo_cache_init();
1882         if (!ret)
1883                 return ret;
1884
1885         kmem_cache_destroy(iommu_domain_cache);
1886 domain_error:
1887         kmem_cache_destroy(iommu_iova_cache);
1888
1889         return -ENOMEM;
1890 }
1891
1892 static void __init iommu_exit_mempool(void)
1893 {
1894         kmem_cache_destroy(iommu_devinfo_cache);
1895         kmem_cache_destroy(iommu_domain_cache);
1896         kmem_cache_destroy(iommu_iova_cache);
1897
1898 }
1899
1900 void __init detect_intel_iommu(void)
1901 {
1902         if (swiotlb || no_iommu || iommu_detected || dmar_disabled)
1903                 return;
1904         if (early_dmar_detect()) {
1905                 iommu_detected = 1;
1906         }
1907 }
1908
1909 static void __init init_no_remapping_devices(void)
1910 {
1911         struct dmar_drhd_unit *drhd;
1912
1913         for_each_drhd_unit(drhd) {
1914                 if (!drhd->include_all) {
1915                         int i;
1916                         for (i = 0; i < drhd->devices_cnt; i++)
1917                                 if (drhd->devices[i] != NULL)
1918                                         break;
1919                         /* ignore DMAR unit if no pci devices exist */
1920                         if (i == drhd->devices_cnt)
1921                                 drhd->ignored = 1;
1922                 }
1923         }
1924
1925         if (dmar_map_gfx)
1926                 return;
1927
1928         for_each_drhd_unit(drhd) {
1929                 int i;
1930                 if (drhd->ignored || drhd->include_all)
1931                         continue;
1932
1933                 for (i = 0; i < drhd->devices_cnt; i++)
1934                         if (drhd->devices[i] &&
1935                                 !IS_GFX_DEVICE(drhd->devices[i]))
1936                                 break;
1937
1938                 if (i < drhd->devices_cnt)
1939                         continue;
1940
1941                 /* bypass IOMMU if it is just for gfx devices */
1942                 drhd->ignored = 1;
1943                 for (i = 0; i < drhd->devices_cnt; i++) {
1944                         if (!drhd->devices[i])
1945                                 continue;
1946                         drhd->devices[i]->sysdata = DUMMY_DEVICE_DOMAIN_INFO;
1947                 }
1948         }
1949 }
1950
1951 int __init intel_iommu_init(void)
1952 {
1953         int ret = 0;
1954
1955         if (no_iommu || swiotlb || dmar_disabled)
1956                 return -ENODEV;
1957
1958         if (dmar_table_init())
1959                 return  -ENODEV;
1960
1961         iommu_init_mempool();
1962         dmar_init_reserved_ranges();
1963
1964         init_no_remapping_devices();
1965
1966         ret = init_dmars();
1967         if (ret) {
1968                 printk(KERN_ERR "IOMMU: dmar init failed\n");
1969                 put_iova_domain(&reserved_iova_list);
1970                 iommu_exit_mempool();
1971                 return ret;
1972         }
1973         printk(KERN_INFO
1974         "PCI-DMA: Intel(R) Virtualization Technology for Directed I/O\n");
1975
1976         force_iommu = 1;
1977         dma_ops = &intel_dma_ops;
1978         return 0;
1979 }