drivers/pci/intel-iommu.c

   1 /*
   2  * Copyright (c) 2006, Intel Corporation.
   3  *
   4  * This program is free software; you can redistribute it and/or modify it
   5  * under the terms and conditions of the GNU General Public License,
   6  * version 2, as published by the Free Software Foundation.
   7  *
   8  * This program is distributed in the hope it will be useful, but WITHOUT
   9  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  10  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  11  * more details.
  12  *
  13  * You should have received a copy of the GNU General Public License along with
  14  * this program; if not, write to the Free Software Foundation, Inc., 59 Temple
  15  * Place - Suite 330, Boston, MA 02111-1307 USA.
  16  *
  17  * Copyright (C) 2006-2008 Intel Corporation
  18  * Author: Ashok Raj <ashok.raj@intel.com>
  19  * Author: Shaohua Li <shaohua.li@intel.com>
  20  * Author: Anil S Keshavamurthy <anil.s.keshavamurthy@intel.com>
  21  * Author: Fenghua Yu <fenghua.yu@intel.com>
  22  */
  23
  24 #include <linux/init.h>
  25 #include <linux/bitmap.h>
  26 #include <linux/debugfs.h>
  27 #include <linux/slab.h>
  28 #include <linux/irq.h>
  29 #include <linux/interrupt.h>
  30 #include <linux/spinlock.h>
  31 #include <linux/pci.h>
  32 #include <linux/dmar.h>
  33 #include <linux/dma-mapping.h>
  34 #include <linux/mempool.h>
  35 #include <linux/timer.h>
  36 #include <linux/iova.h>
  37 #include <linux/intel-iommu.h>
  38 #include <asm/cacheflush.h>
  39 #include <asm/iommu.h>
  40 #include "pci.h"
  41
  42 #define ROOT_SIZE               VTD_PAGE_SIZE
  43 #define CONTEXT_SIZE            VTD_PAGE_SIZE
  44
  45 #define IS_GFX_DEVICE(pdev) ((pdev->class >> 16) == PCI_BASE_CLASS_DISPLAY)
  46 #define IS_ISA_DEVICE(pdev) ((pdev->class >> 8) == PCI_CLASS_BRIDGE_ISA)
  47
  48 #define IOAPIC_RANGE_START      (0xfee00000)
  49 #define IOAPIC_RANGE_END        (0xfeefffff)
  50 #define IOVA_START_ADDR         (0x1000)
  51
  52 #define DEFAULT_DOMAIN_ADDRESS_WIDTH 48
  53
  54 #define DOMAIN_MAX_ADDR(gaw) ((((u64)1) << gaw) - 1)
  55
  56 #define IOVA_PFN(addr)          ((addr) >> PAGE_SHIFT)
  57 #define DMA_32BIT_PFN           IOVA_PFN(DMA_32BIT_MASK)
  58 #define DMA_64BIT_PFN           IOVA_PFN(DMA_64BIT_MASK)
  59
  60 /*
  61  * 0: Present
  62  * 1-11: Reserved
  63  * 12-63: Context Ptr (12 - (haw-1))
  64  * 64-127: Reserved
  65  */
  66 struct root_entry {
  67         u64     val;
  68         u64     rsvd1;
  69 };
  70 #define ROOT_ENTRY_NR (VTD_PAGE_SIZE/sizeof(struct root_entry))
  71 static inline bool root_present(struct root_entry *root)
  72 {
  73         return (root->val & 1);
  74 }
  75 static inline void set_root_present(struct root_entry *root)
  76 {
  77         root->val |= 1;
  78 }
  79 static inline void set_root_value(struct root_entry *root, unsigned long value)
  80 {
  81         root->val |= value & VTD_PAGE_MASK;
  82 }
  83
  84 static inline struct context_entry *
  85 get_context_addr_from_root(struct root_entry *root)
  86 {
  87         return (struct context_entry *)
  88                 (root_present(root)?phys_to_virt(
  89                 root->val & VTD_PAGE_MASK) :
  90                 NULL);
  91 }
  92
  93 /*
  94  * low 64 bits:
  95  * 0: present
  96  * 1: fault processing disable
  97  * 2-3: translation type
  98  * 12-63: address space root
  99  * high 64 bits:
 100  * 0-2: address width
 101  * 3-6: aval
 102  * 8-23: domain id
 103  */
 104 struct context_entry {
 105         u64 lo;
 106         u64 hi;
 107 };
 108
 109 static inline bool context_present(struct context_entry *context)
 110 {
 111         return (context->lo & 1);
 112 }
 113 static inline void context_set_present(struct context_entry *context)
 114 {
 115         context->lo |= 1;
 116 }
 117
 118 static inline void context_set_fault_enable(struct context_entry *context)
 119 {
 120         context->lo &= (((u64)-1) << 2) | 1;
 121 }
 122
 123 #define CONTEXT_TT_MULTI_LEVEL 0
 124
 125 static inline void context_set_translation_type(struct context_entry *context,
 126                                                 unsigned long value)
 127 {
 128         context->lo &= (((u64)-1) << 4) | 3;
 129         context->lo |= (value & 3) << 2;
 130 }
 131
 132 static inline void context_set_address_root(struct context_entry *context,
 133                                             unsigned long value)
 134 {
 135         context->lo |= value & VTD_PAGE_MASK;
 136 }
 137
 138 static inline void context_set_address_width(struct context_entry *context,
 139                                              unsigned long value)
 140 {
 141         context->hi |= value & 7;
 142 }
 143
 144 static inline void context_set_domain_id(struct context_entry *context,
 145                                          unsigned long value)
 146 {
 147         context->hi |= (value & ((1 << 16) - 1)) << 8;
 148 }
 149
 150 static inline void context_clear_entry(struct context_entry *context)
 151 {
 152         context->lo = 0;
 153         context->hi = 0;
 154 }
 155
 156 /*
 157  * 0: readable
 158  * 1: writable
 159  * 2-6: reserved
 160  * 7: super page
 161  * 8-11: available
 162  * 12-63: Host physcial address
 163  */
 164 struct dma_pte {
 165         u64 val;
 166 };
 167
 168 static inline void dma_clear_pte(struct dma_pte *pte)
 169 {
 170         pte->val = 0;
 171 }
 172
 173 static inline void dma_set_pte_readable(struct dma_pte *pte)
 174 {
 175         pte->val |= DMA_PTE_READ;
 176 }
 177
 178 static inline void dma_set_pte_writable(struct dma_pte *pte)
 179 {
 180         pte->val |= DMA_PTE_WRITE;
 181 }
 182
 183 static inline void dma_set_pte_prot(struct dma_pte *pte, unsigned long prot)
 184 {
 185         pte->val = (pte->val & ~3) | (prot & 3);
 186 }
 187
 188 static inline u64 dma_pte_addr(struct dma_pte *pte)
 189 {
 190         return (pte->val & VTD_PAGE_MASK);
 191 }
 192
 193 static inline void dma_set_pte_addr(struct dma_pte *pte, u64 addr)
 194 {
 195         pte->val |= (addr & VTD_PAGE_MASK);
 196 }
 197
 198 static inline bool dma_pte_present(struct dma_pte *pte)
 199 {
 200         return (pte->val & 3) != 0;
 201 }
 202
 203 struct dmar_domain {
 204         int     id;                     /* domain id */
 205         struct intel_iommu *iommu;      /* back pointer to owning iommu */
 206
 207         struct list_head devices;       /* all devices' list */
 208         struct iova_domain iovad;       /* iova's that belong to this domain */
 209
 210         struct dma_pte  *pgd;           /* virtual address */
 211         spinlock_t      mapping_lock;   /* page table lock */
 212         int             gaw;            /* max guest address width */
 213
 214         /* adjusted guest address width, 0 is level 2 30-bit */
 215         int             agaw;
 216
 217 #define DOMAIN_FLAG_MULTIPLE_DEVICES 1
 218         int             flags;
 219 };
 220
 221 /* PCI domain-device relationship */
 222 struct device_domain_info {
 223         struct list_head link;  /* link to domain siblings */
 224         struct list_head global; /* link to global list */
 225         u8 bus;                 /* PCI bus numer */
 226         u8 devfn;               /* PCI devfn number */
 227         struct pci_dev *dev; /* it's NULL for PCIE-to-PCI bridge */
 228         struct dmar_domain *domain; /* pointer to domain */
 229 };
 230
 231 static void flush_unmaps_timeout(unsigned long data);
 232
 233 DEFINE_TIMER(unmap_timer,  flush_unmaps_timeout, 0, 0);
 234
 235 #define HIGH_WATER_MARK 250
 236 struct deferred_flush_tables {
 237         int next;
 238         struct iova *iova[HIGH_WATER_MARK];
 239         struct dmar_domain *domain[HIGH_WATER_MARK];
 240 };
 241
 242 static struct deferred_flush_tables *deferred_flush;
 243
 244 /* bitmap for indexing intel_iommus */
 245 static int g_num_of_iommus;
 246
 247 static DEFINE_SPINLOCK(async_umap_flush_lock);
 248 static LIST_HEAD(unmaps_to_do);
 249
 250 static int timer_on;
 251 static long list_size;
 252
 253 static void domain_remove_dev_info(struct dmar_domain *domain);
 254
 255 int dmar_disabled;
 256 static int __initdata dmar_map_gfx = 1;
 257 static int dmar_forcedac;
 258 static int intel_iommu_strict;
 259
 260 #define DUMMY_DEVICE_DOMAIN_INFO ((struct device_domain_info *)(-1))
 261 static DEFINE_SPINLOCK(device_domain_lock);
 262 static LIST_HEAD(device_domain_list);
 263
 264 static int __init intel_iommu_setup(char *str)
 265 {
 266         if (!str)
 267                 return -EINVAL;
 268         while (*str) {
 269                 if (!strncmp(str, "off", 3)) {
 270                         dmar_disabled = 1;
 271                         printk(KERN_INFO"Intel-IOMMU: disabled\n");
 272                 } else if (!strncmp(str, "igfx_off", 8)) {
 273                         dmar_map_gfx = 0;
 274                         printk(KERN_INFO
 275                                 "Intel-IOMMU: disable GFX device mapping\n");
 276                 } else if (!strncmp(str, "forcedac", 8)) {
 277                         printk(KERN_INFO
 278                                 "Intel-IOMMU: Forcing DAC for PCI devices\n");
 279                         dmar_forcedac = 1;
 280                 } else if (!strncmp(str, "strict", 6)) {
 281                         printk(KERN_INFO
 282                                 "Intel-IOMMU: disable batched IOTLB flush\n");
 283                         intel_iommu_strict = 1;
 284                 }
 285
 286                 str += strcspn(str, ",");
 287                 while (*str == ',')
 288                         str++;
 289         }
 290         return 0;
 291 }
 292 __setup("intel_iommu=", intel_iommu_setup);
 293
 294 static struct kmem_cache *iommu_domain_cache;
 295 static struct kmem_cache *iommu_devinfo_cache;
 296 static struct kmem_cache *iommu_iova_cache;
 297
 298 static inline void *iommu_kmem_cache_alloc(struct kmem_cache *cachep)
 299 {
 300         unsigned int flags;
 301         void *vaddr;
 302
 303         /* trying to avoid low memory issues */
 304         flags = current->flags & PF_MEMALLOC;
 305         current->flags |= PF_MEMALLOC;
 306         vaddr = kmem_cache_alloc(cachep, GFP_ATOMIC);
 307         current->flags &= (~PF_MEMALLOC | flags);
 308         return vaddr;
 309 }
 310
 311
 312 static inline void *alloc_pgtable_page(void)
 313 {
 314         unsigned int flags;
 315         void *vaddr;
 316
 317         /* trying to avoid low memory issues */
 318         flags = current->flags & PF_MEMALLOC;
 319         current->flags |= PF_MEMALLOC;
 320         vaddr = (void *)get_zeroed_page(GFP_ATOMIC);
 321         current->flags &= (~PF_MEMALLOC | flags);
 322         return vaddr;
 323 }
 324
 325 static inline void free_pgtable_page(void *vaddr)
 326 {
 327         free_page((unsigned long)vaddr);
 328 }
 329
 330 static inline void *alloc_domain_mem(void)
 331 {
 332         return iommu_kmem_cache_alloc(iommu_domain_cache);
 333 }
 334
 335 static void free_domain_mem(void *vaddr)
 336 {
 337         kmem_cache_free(iommu_domain_cache, vaddr);
 338 }
 339
 340 static inline void * alloc_devinfo_mem(void)
 341 {
 342         return iommu_kmem_cache_alloc(iommu_devinfo_cache);
 343 }
 344
 345 static inline void free_devinfo_mem(void *vaddr)
 346 {
 347         kmem_cache_free(iommu_devinfo_cache, vaddr);
 348 }
 349
 350 struct iova *alloc_iova_mem(void)
 351 {
 352         return iommu_kmem_cache_alloc(iommu_iova_cache);
 353 }
 354
 355 void free_iova_mem(struct iova *iova)
 356 {
 357         kmem_cache_free(iommu_iova_cache, iova);
 358 }
 359
 360 /* Gets context entry for a given bus and devfn */
 361 static struct context_entry * device_to_context_entry(struct intel_iommu *iommu,
 362                 u8 bus, u8 devfn)
 363 {
 364         struct root_entry *root;
 365         struct context_entry *context;
 366         unsigned long phy_addr;
 367         unsigned long flags;
 368
 369         spin_lock_irqsave(&iommu->lock, flags);
 370         root = &iommu->root_entry[bus];
 371         context = get_context_addr_from_root(root);
 372         if (!context) {
 373                 context = (struct context_entry *)alloc_pgtable_page();
 374                 if (!context) {
 375                         spin_unlock_irqrestore(&iommu->lock, flags);
 376                         return NULL;
 377                 }
 378                 __iommu_flush_cache(iommu, (void *)context, CONTEXT_SIZE);
 379                 phy_addr = virt_to_phys((void *)context);
 380                 set_root_value(root, phy_addr);
 381                 set_root_present(root);
 382                 __iommu_flush_cache(iommu, root, sizeof(*root));
 383         }
 384         spin_unlock_irqrestore(&iommu->lock, flags);
 385         return &context[devfn];
 386 }
 387
 388 static int device_context_mapped(struct intel_iommu *iommu, u8 bus, u8 devfn)
 389 {
 390         struct root_entry *root;
 391         struct context_entry *context;
 392         int ret;
 393         unsigned long flags;
 394
 395         spin_lock_irqsave(&iommu->lock, flags);
 396         root = &iommu->root_entry[bus];
 397         context = get_context_addr_from_root(root);
 398         if (!context) {
 399                 ret = 0;
 400                 goto out;
 401         }
 402         ret = context_present(&context[devfn]);
 403 out:
 404         spin_unlock_irqrestore(&iommu->lock, flags);
 405         return ret;
 406 }
 407
 408 static void clear_context_table(struct intel_iommu *iommu, u8 bus, u8 devfn)
 409 {
 410         struct root_entry *root;
 411         struct context_entry *context;
 412         unsigned long flags;
 413
 414         spin_lock_irqsave(&iommu->lock, flags);
 415         root = &iommu->root_entry[bus];
 416         context = get_context_addr_from_root(root);
 417         if (context) {
 418                 context_clear_entry(&context[devfn]);
 419                 __iommu_flush_cache(iommu, &context[devfn], \
 420                         sizeof(*context));
 421         }
 422         spin_unlock_irqrestore(&iommu->lock, flags);
 423 }
 424
 425 static void free_context_table(struct intel_iommu *iommu)
 426 {
 427         struct root_entry *root;
 428         int i;
 429         unsigned long flags;
 430         struct context_entry *context;
 431
 432         spin_lock_irqsave(&iommu->lock, flags);
 433         if (!iommu->root_entry) {
 434                 goto out;
 435         }
 436         for (i = 0; i < ROOT_ENTRY_NR; i++) {
 437                 root = &iommu->root_entry[i];
 438                 context = get_context_addr_from_root(root);
 439                 if (context)
 440                         free_pgtable_page(context);
 441         }
 442         free_pgtable_page(iommu->root_entry);
 443         iommu->root_entry = NULL;
 444 out:
 445         spin_unlock_irqrestore(&iommu->lock, flags);
 446 }
 447
 448 /* page table handling */
 449 #define LEVEL_STRIDE            (9)
 450 #define LEVEL_MASK              (((u64)1 << LEVEL_STRIDE) - 1)
 451
 452 static inline int agaw_to_level(int agaw)
 453 {
 454         return agaw + 2;
 455 }
 456
 457 static inline int agaw_to_width(int agaw)
 458 {
 459         return 30 + agaw * LEVEL_STRIDE;
 460
 461 }
 462
 463 static inline int width_to_agaw(int width)
 464 {
 465         return (width - 30) / LEVEL_STRIDE;
 466 }
 467
 468 static inline unsigned int level_to_offset_bits(int level)
 469 {
 470         return (12 + (level - 1) * LEVEL_STRIDE);
 471 }
 472
 473 static inline int address_level_offset(u64 addr, int level)
 474 {
 475         return ((addr >> level_to_offset_bits(level)) & LEVEL_MASK);
 476 }
 477
 478 static inline u64 level_mask(int level)
 479 {
 480         return ((u64)-1 << level_to_offset_bits(level));
 481 }
 482
 483 static inline u64 level_size(int level)
 484 {
 485         return ((u64)1 << level_to_offset_bits(level));
 486 }
 487
 488 static inline u64 align_to_level(u64 addr, int level)
 489 {
 490         return ((addr + level_size(level) - 1) & level_mask(level));
 491 }
 492
 493 static struct dma_pte * addr_to_dma_pte(struct dmar_domain *domain, u64 addr)
 494 {
 495         int addr_width = agaw_to_width(domain->agaw);
 496         struct dma_pte *parent, *pte = NULL;
 497         int level = agaw_to_level(domain->agaw);
 498         int offset;
 499         unsigned long flags;
 500
 501         BUG_ON(!domain->pgd);
 502
 503         addr &= (((u64)1) << addr_width) - 1;
 504         parent = domain->pgd;
 505
 506         spin_lock_irqsave(&domain->mapping_lock, flags);
 507         while (level > 0) {
 508                 void *tmp_page;
 509
 510                 offset = address_level_offset(addr, level);
 511                 pte = &parent[offset];
 512                 if (level == 1)
 513                         break;
 514
 515                 if (!dma_pte_present(pte)) {
 516                         tmp_page = alloc_pgtable_page();
 517
 518                         if (!tmp_page) {
 519                                 spin_unlock_irqrestore(&domain->mapping_lock,
 520                                         flags);
 521                                 return NULL;
 522                         }
 523                         __iommu_flush_cache(domain->iommu, tmp_page,
 524                                         PAGE_SIZE);
 525                         dma_set_pte_addr(pte, virt_to_phys(tmp_page));
 526                         /*
 527                          * high level table always sets r/w, last level page
 528                          * table control read/write
 529                          */
 530                         dma_set_pte_readable(pte);
 531                         dma_set_pte_writable(pte);
 532                         __iommu_flush_cache(domain->iommu, pte, sizeof(*pte));
 533                 }
 534                 parent = phys_to_virt(dma_pte_addr(pte));
 535                 level--;
 536         }
 537
 538         spin_unlock_irqrestore(&domain->mapping_lock, flags);
 539         return pte;
 540 }
 541
 542 /* return address's pte at specific level */
 543 static struct dma_pte *dma_addr_level_pte(struct dmar_domain *domain, u64 addr,
 544                 int level)
 545 {
 546         struct dma_pte *parent, *pte = NULL;
 547         int total = agaw_to_level(domain->agaw);
 548         int offset;
 549
 550         parent = domain->pgd;
 551         while (level <= total) {
 552                 offset = address_level_offset(addr, total);
 553                 pte = &parent[offset];
 554                 if (level == total)
 555                         return pte;
 556
 557                 if (!dma_pte_present(pte))
 558                         break;
 559                 parent = phys_to_virt(dma_pte_addr(pte));
 560                 total--;
 561         }
 562         return NULL;
 563 }
 564
 565 /* clear one page's page table */
 566 static void dma_pte_clear_one(struct dmar_domain *domain, u64 addr)
 567 {
 568         struct dma_pte *pte = NULL;
 569
 570         /* get last level pte */
 571         pte = dma_addr_level_pte(domain, addr, 1);
 572
 573         if (pte) {
 574                 dma_clear_pte(pte);
 575                 __iommu_flush_cache(domain->iommu, pte, sizeof(*pte));
 576         }
 577 }
 578
 579 /* clear last level pte, a tlb flush should be followed */
 580 static void dma_pte_clear_range(struct dmar_domain *domain, u64 start, u64 end)
 581 {
 582         int addr_width = agaw_to_width(domain->agaw);
 583
 584         start &= (((u64)1) << addr_width) - 1;
 585         end &= (((u64)1) << addr_width) - 1;
 586         /* in case it's partial page */
 587         start = PAGE_ALIGN(start);
 588         end &= PAGE_MASK;
 589
 590         /* we don't need lock here, nobody else touches the iova range */
 591         while (start < end) {
 592                 dma_pte_clear_one(domain, start);
 593                 start += VTD_PAGE_SIZE;
 594         }
 595 }
 596
 597 /* free page table pages. last level pte should already be cleared */
 598 static void dma_pte_free_pagetable(struct dmar_domain *domain,
 599         u64 start, u64 end)
 600 {
 601         int addr_width = agaw_to_width(domain->agaw);
 602         struct dma_pte *pte;
 603         int total = agaw_to_level(domain->agaw);
 604         int level;
 605         u64 tmp;
 606
 607         start &= (((u64)1) << addr_width) - 1;
 608         end &= (((u64)1) << addr_width) - 1;
 609
 610         /* we don't need lock here, nobody else touches the iova range */
 611         level = 2;
 612         while (level <= total) {
 613                 tmp = align_to_level(start, level);
 614                 if (tmp >= end || (tmp + level_size(level) > end))
 615                         return;
 616
 617                 while (tmp < end) {
 618                         pte = dma_addr_level_pte(domain, tmp, level);
 619                         if (pte) {
 620                                 free_pgtable_page(
 621                                         phys_to_virt(dma_pte_addr(pte)));
 622                                 dma_clear_pte(pte);
 623                                 __iommu_flush_cache(domain->iommu,
 624                                                 pte, sizeof(*pte));
 625                         }
 626                         tmp += level_size(level);
 627                 }
 628                 level++;
 629         }
 630         /* free pgd */
 631         if (start == 0 && end >= ((((u64)1) << addr_width) - 1)) {
 632                 free_pgtable_page(domain->pgd);
 633                 domain->pgd = NULL;
 634         }
 635 }
 636
 637 /* iommu handling */
 638 static int iommu_alloc_root_entry(struct intel_iommu *iommu)
 639 {
 640         struct root_entry *root;
 641         unsigned long flags;
 642
 643         root = (struct root_entry *)alloc_pgtable_page();
 644         if (!root)
 645                 return -ENOMEM;
 646
 647         __iommu_flush_cache(iommu, root, ROOT_SIZE);
 648
 649         spin_lock_irqsave(&iommu->lock, flags);
 650         iommu->root_entry = root;
 651         spin_unlock_irqrestore(&iommu->lock, flags);
 652
 653         return 0;
 654 }
 655
 656 static void iommu_set_root_entry(struct intel_iommu *iommu)
 657 {
 658         void *addr;
 659         u32 cmd, sts;
 660         unsigned long flag;
 661
 662         addr = iommu->root_entry;
 663
 664         spin_lock_irqsave(&iommu->register_lock, flag);
 665         dmar_writeq(iommu->reg + DMAR_RTADDR_REG, virt_to_phys(addr));
 666
 667         cmd = iommu->gcmd | DMA_GCMD_SRTP;
 668         writel(cmd, iommu->reg + DMAR_GCMD_REG);
 669
 670         /* Make sure hardware complete it */
 671         IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
 672                 readl, (sts & DMA_GSTS_RTPS), sts);
 673
 674         spin_unlock_irqrestore(&iommu->register_lock, flag);
 675 }
 676
 677 static void iommu_flush_write_buffer(struct intel_iommu *iommu)
 678 {
 679         u32 val;
 680         unsigned long flag;
 681
 682         if (!cap_rwbf(iommu->cap))
 683                 return;
 684         val = iommu->gcmd | DMA_GCMD_WBF;
 685
 686         spin_lock_irqsave(&iommu->register_lock, flag);
 687         writel(val, iommu->reg + DMAR_GCMD_REG);
 688
 689         /* Make sure hardware complete it */
 690         IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
 691                         readl, (!(val & DMA_GSTS_WBFS)), val);
 692
 693         spin_unlock_irqrestore(&iommu->register_lock, flag);
 694 }
 695
 696 /* return value determine if we need a write buffer flush */
 697 static int __iommu_flush_context(struct intel_iommu *iommu,
 698         u16 did, u16 source_id, u8 function_mask, u64 type,
 699         int non_present_entry_flush)
 700 {
 701         u64 val = 0;
 702         unsigned long flag;
 703
 704         /*
 705          * In the non-present entry flush case, if hardware doesn't cache
 706          * non-present entry we do nothing and if hardware cache non-present
 707          * entry, we flush entries of domain 0 (the domain id is used to cache
 708          * any non-present entries)
 709          */
 710         if (non_present_entry_flush) {
 711                 if (!cap_caching_mode(iommu->cap))
 712                         return 1;
 713                 else
 714                         did = 0;
 715         }
 716
 717         switch (type) {
 718         case DMA_CCMD_GLOBAL_INVL:
 719                 val = DMA_CCMD_GLOBAL_INVL;
 720                 break;
 721         case DMA_CCMD_DOMAIN_INVL:
 722                 val = DMA_CCMD_DOMAIN_INVL|DMA_CCMD_DID(did);
 723                 break;
 724         case DMA_CCMD_DEVICE_INVL:
 725                 val = DMA_CCMD_DEVICE_INVL|DMA_CCMD_DID(did)
 726                         | DMA_CCMD_SID(source_id) | DMA_CCMD_FM(function_mask);
 727                 break;
 728         default:
 729                 BUG();
 730         }
 731         val |= DMA_CCMD_ICC;
 732
 733         spin_lock_irqsave(&iommu->register_lock, flag);
 734         dmar_writeq(iommu->reg + DMAR_CCMD_REG, val);
 735
 736         /* Make sure hardware complete it */
 737         IOMMU_WAIT_OP(iommu, DMAR_CCMD_REG,
 738                 dmar_readq, (!(val & DMA_CCMD_ICC)), val);
 739
 740         spin_unlock_irqrestore(&iommu->register_lock, flag);
 741
 742         /* flush context entry will implicitly flush write buffer */
 743         return 0;
 744 }
 745
 746 /* return value determine if we need a write buffer flush */
 747 static int __iommu_flush_iotlb(struct intel_iommu *iommu, u16 did,
 748         u64 addr, unsigned int size_order, u64 type,
 749         int non_present_entry_flush)
 750 {
 751         int tlb_offset = ecap_iotlb_offset(iommu->ecap);
 752         u64 val = 0, val_iva = 0;
 753         unsigned long flag;
 754
 755         /*
 756          * In the non-present entry flush case, if hardware doesn't cache
 757          * non-present entry we do nothing and if hardware cache non-present
 758          * entry, we flush entries of domain 0 (the domain id is used to cache
 759          * any non-present entries)
 760          */
 761         if (non_present_entry_flush) {
 762                 if (!cap_caching_mode(iommu->cap))
 763                         return 1;
 764                 else
 765                         did = 0;
 766         }
 767
 768         switch (type) {
 769         case DMA_TLB_GLOBAL_FLUSH:
 770                 /* global flush doesn't need set IVA_REG */
 771                 val = DMA_TLB_GLOBAL_FLUSH|DMA_TLB_IVT;
 772                 break;
 773         case DMA_TLB_DSI_FLUSH:
 774                 val = DMA_TLB_DSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
 775                 break;
 776         case DMA_TLB_PSI_FLUSH:
 777                 val = DMA_TLB_PSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
 778                 /* Note: always flush non-leaf currently */
 779                 val_iva = size_order | addr;
 780                 break;
 781         default:
 782                 BUG();
 783         }
 784         /* Note: set drain read/write */
 785 #if 0
 786         /*
 787          * This is probably to be super secure.. Looks like we can
 788          * ignore it without any impact.
 789          */
 790         if (cap_read_drain(iommu->cap))
 791                 val |= DMA_TLB_READ_DRAIN;
 792 #endif
 793         if (cap_write_drain(iommu->cap))
 794                 val |= DMA_TLB_WRITE_DRAIN;
 795
 796         spin_lock_irqsave(&iommu->register_lock, flag);
 797         /* Note: Only uses first TLB reg currently */
 798         if (val_iva)
 799                 dmar_writeq(iommu->reg + tlb_offset, val_iva);
 800         dmar_writeq(iommu->reg + tlb_offset + 8, val);
 801
 802         /* Make sure hardware complete it */
 803         IOMMU_WAIT_OP(iommu, tlb_offset + 8,
 804                 dmar_readq, (!(val & DMA_TLB_IVT)), val);
 805
 806         spin_unlock_irqrestore(&iommu->register_lock, flag);
 807
 808         /* check IOTLB invalidation granularity */
 809         if (DMA_TLB_IAIG(val) == 0)
 810                 printk(KERN_ERR"IOMMU: flush IOTLB failed\n");
 811         if (DMA_TLB_IAIG(val) != DMA_TLB_IIRG(type))
 812                 pr_debug("IOMMU: tlb flush request %Lx, actual %Lx\n",
 813                         (unsigned long long)DMA_TLB_IIRG(type),
 814                         (unsigned long long)DMA_TLB_IAIG(val));
 815         /* flush iotlb entry will implicitly flush write buffer */
 816         return 0;
 817 }
 818
 819 static int iommu_flush_iotlb_psi(struct intel_iommu *iommu, u16 did,
 820         u64 addr, unsigned int pages, int non_present_entry_flush)
 821 {
 822         unsigned int mask;
 823
 824         BUG_ON(addr & (~VTD_PAGE_MASK));
 825         BUG_ON(pages == 0);
 826
 827         /* Fallback to domain selective flush if no PSI support */
 828         if (!cap_pgsel_inv(iommu->cap))
 829                 return iommu->flush.flush_iotlb(iommu, did, 0, 0,
 830                                                 DMA_TLB_DSI_FLUSH,
 831                                                 non_present_entry_flush);
 832
 833         /*
 834          * PSI requires page size to be 2 ^ x, and the base address is naturally
 835          * aligned to the size
 836          */
 837         mask = ilog2(__roundup_pow_of_two(pages));
 838         /* Fallback to domain selective flush if size is too big */
 839         if (mask > cap_max_amask_val(iommu->cap))
 840                 return iommu->flush.flush_iotlb(iommu, did, 0, 0,
 841                         DMA_TLB_DSI_FLUSH, non_present_entry_flush);
 842
 843         return iommu->flush.flush_iotlb(iommu, did, addr, mask,
 844                                         DMA_TLB_PSI_FLUSH,
 845                                         non_present_entry_flush);
 846 }
 847
 848 static void iommu_disable_protect_mem_regions(struct intel_iommu *iommu)
 849 {
 850         u32 pmen;
 851         unsigned long flags;
 852
 853         spin_lock_irqsave(&iommu->register_lock, flags);
 854         pmen = readl(iommu->reg + DMAR_PMEN_REG);
 855         pmen &= ~DMA_PMEN_EPM;
 856         writel(pmen, iommu->reg + DMAR_PMEN_REG);
 857
 858         /* wait for the protected region status bit to clear */
 859         IOMMU_WAIT_OP(iommu, DMAR_PMEN_REG,
 860                 readl, !(pmen & DMA_PMEN_PRS), pmen);
 861
 862         spin_unlock_irqrestore(&iommu->register_lock, flags);
 863 }
 864
 865 static int iommu_enable_translation(struct intel_iommu *iommu)
 866 {
 867         u32 sts;
 868         unsigned long flags;
 869
 870         spin_lock_irqsave(&iommu->register_lock, flags);
 871         writel(iommu->gcmd|DMA_GCMD_TE, iommu->reg + DMAR_GCMD_REG);
 872
 873         /* Make sure hardware complete it */
 874         IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
 875                 readl, (sts & DMA_GSTS_TES), sts);
 876
 877         iommu->gcmd |= DMA_GCMD_TE;
 878         spin_unlock_irqrestore(&iommu->register_lock, flags);
 879         return 0;
 880 }
 881
 882 static int iommu_disable_translation(struct intel_iommu *iommu)
 883 {
 884         u32 sts;
 885         unsigned long flag;
 886
 887         spin_lock_irqsave(&iommu->register_lock, flag);
 888         iommu->gcmd &= ~DMA_GCMD_TE;
 889         writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
 890
 891         /* Make sure hardware complete it */
 892         IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
 893                 readl, (!(sts & DMA_GSTS_TES)), sts);
 894
 895         spin_unlock_irqrestore(&iommu->register_lock, flag);
 896         return 0;
 897 }
 898
 899 /* iommu interrupt handling. Most stuff are MSI-like. */
 900
 901 static const char *fault_reason_strings[] =
 902 {
 903         "Software",
 904         "Present bit in root entry is clear",
 905         "Present bit in context entry is clear",
 906         "Invalid context entry",
 907         "Access beyond MGAW",
 908         "PTE Write access is not set",
 909         "PTE Read access is not set",
 910         "Next page table ptr is invalid",
 911         "Root table address invalid",
 912         "Context table ptr is invalid",
 913         "non-zero reserved fields in RTP",
 914         "non-zero reserved fields in CTP",
 915         "non-zero reserved fields in PTE",
 916 };
 917 #define MAX_FAULT_REASON_IDX    (ARRAY_SIZE(fault_reason_strings) - 1)
 918
 919 const char *dmar_get_fault_reason(u8 fault_reason)
 920 {
 921         if (fault_reason > MAX_FAULT_REASON_IDX)
 922                 return "Unknown";
 923         else
 924                 return fault_reason_strings[fault_reason];
 925 }
 926
 927 void dmar_msi_unmask(unsigned int irq)
 928 {
 929         struct intel_iommu *iommu = get_irq_data(irq);
 930         unsigned long flag;
 931
 932         /* unmask it */
 933         spin_lock_irqsave(&iommu->register_lock, flag);
 934         writel(0, iommu->reg + DMAR_FECTL_REG);
 935         /* Read a reg to force flush the post write */
 936         readl(iommu->reg + DMAR_FECTL_REG);
 937         spin_unlock_irqrestore(&iommu->register_lock, flag);
 938 }
 939
 940 void dmar_msi_mask(unsigned int irq)
 941 {
 942         unsigned long flag;
 943         struct intel_iommu *iommu = get_irq_data(irq);
 944
 945         /* mask it */
 946         spin_lock_irqsave(&iommu->register_lock, flag);
 947         writel(DMA_FECTL_IM, iommu->reg + DMAR_FECTL_REG);
 948         /* Read a reg to force flush the post write */
 949         readl(iommu->reg + DMAR_FECTL_REG);
 950         spin_unlock_irqrestore(&iommu->register_lock, flag);
 951 }
 952
 953 void dmar_msi_write(int irq, struct msi_msg *msg)
 954 {
 955         struct intel_iommu *iommu = get_irq_data(irq);
 956         unsigned long flag;
 957
 958         spin_lock_irqsave(&iommu->register_lock, flag);
 959         writel(msg->data, iommu->reg + DMAR_FEDATA_REG);
 960         writel(msg->address_lo, iommu->reg + DMAR_FEADDR_REG);
 961         writel(msg->address_hi, iommu->reg + DMAR_FEUADDR_REG);
 962         spin_unlock_irqrestore(&iommu->register_lock, flag);
 963 }
 964
 965 void dmar_msi_read(int irq, struct msi_msg *msg)
 966 {
 967         struct intel_iommu *iommu = get_irq_data(irq);
 968         unsigned long flag;
 969
 970         spin_lock_irqsave(&iommu->register_lock, flag);
 971         msg->data = readl(iommu->reg + DMAR_FEDATA_REG);
 972         msg->address_lo = readl(iommu->reg + DMAR_FEADDR_REG);
 973         msg->address_hi = readl(iommu->reg + DMAR_FEUADDR_REG);
 974         spin_unlock_irqrestore(&iommu->register_lock, flag);
 975 }
 976
 977 static int iommu_page_fault_do_one(struct intel_iommu *iommu, int type,
 978                 u8 fault_reason, u16 source_id, unsigned long long addr)
 979 {
 980         const char *reason;
 981
 982         reason = dmar_get_fault_reason(fault_reason);
 983
 984         printk(KERN_ERR
 985                 "DMAR:[%s] Request device [%02x:%02x.%d] "
 986                 "fault addr %llx \n"
 987                 "DMAR:[fault reason %02d] %s\n",
 988                 (type ? "DMA Read" : "DMA Write"),
 989                 (source_id >> 8), PCI_SLOT(source_id & 0xFF),
 990                 PCI_FUNC(source_id & 0xFF), addr, fault_reason, reason);
 991         return 0;
 992 }
 993
 994 #define PRIMARY_FAULT_REG_LEN (16)
 995 static irqreturn_t iommu_page_fault(int irq, void *dev_id)
 996 {
 997         struct intel_iommu *iommu = dev_id;
 998         int reg, fault_index;
 999         u32 fault_status;
1000         unsigned long flag;
1001
1002         spin_lock_irqsave(&iommu->register_lock, flag);
1003         fault_status = readl(iommu->reg + DMAR_FSTS_REG);
1004
1005         /* TBD: ignore advanced fault log currently */
1006         if (!(fault_status & DMA_FSTS_PPF))
1007                 goto clear_overflow;
1008
1009         fault_index = dma_fsts_fault_record_index(fault_status);
1010         reg = cap_fault_reg_offset(iommu->cap);
1011         while (1) {
1012                 u8 fault_reason;
1013                 u16 source_id;
1014                 u64 guest_addr;
1015                 int type;
1016                 u32 data;
1017
1018                 /* highest 32 bits */
1019                 data = readl(iommu->reg + reg +
1020                                 fault_index * PRIMARY_FAULT_REG_LEN + 12);
1021                 if (!(data & DMA_FRCD_F))
1022                         break;
1023
1024                 fault_reason = dma_frcd_fault_reason(data);
1025                 type = dma_frcd_type(data);
1026
1027                 data = readl(iommu->reg + reg +
1028                                 fault_index * PRIMARY_FAULT_REG_LEN + 8);
1029                 source_id = dma_frcd_source_id(data);
1030
1031                 guest_addr = dmar_readq(iommu->reg + reg +
1032                                 fault_index * PRIMARY_FAULT_REG_LEN);
1033                 guest_addr = dma_frcd_page_addr(guest_addr);
1034                 /* clear the fault */
1035                 writel(DMA_FRCD_F, iommu->reg + reg +
1036                         fault_index * PRIMARY_FAULT_REG_LEN + 12);
1037
1038                 spin_unlock_irqrestore(&iommu->register_lock, flag);
1039
1040                 iommu_page_fault_do_one(iommu, type, fault_reason,
1041                                 source_id, guest_addr);
1042
1043                 fault_index++;
1044                 if (fault_index > cap_num_fault_regs(iommu->cap))
1045                         fault_index = 0;
1046                 spin_lock_irqsave(&iommu->register_lock, flag);
1047         }
1048 clear_overflow:
1049         /* clear primary fault overflow */
1050         fault_status = readl(iommu->reg + DMAR_FSTS_REG);
1051         if (fault_status & DMA_FSTS_PFO)
1052                 writel(DMA_FSTS_PFO, iommu->reg + DMAR_FSTS_REG);
1053
1054         spin_unlock_irqrestore(&iommu->register_lock, flag);
1055         return IRQ_HANDLED;
1056 }
1057
1058 int dmar_set_interrupt(struct intel_iommu *iommu)
1059 {
1060         int irq, ret;
1061
1062         irq = create_irq();
1063         if (!irq) {
1064                 printk(KERN_ERR "IOMMU: no free vectors\n");
1065                 return -EINVAL;
1066         }
1067
1068         set_irq_data(irq, iommu);
1069         iommu->irq = irq;
1070
1071         ret = arch_setup_dmar_msi(irq);
1072         if (ret) {
1073                 set_irq_data(irq, NULL);
1074                 iommu->irq = 0;
1075                 destroy_irq(irq);
1076                 return 0;
1077         }
1078
1079         /* Force fault register is cleared */
1080         iommu_page_fault(irq, iommu);
1081
1082         ret = request_irq(irq, iommu_page_fault, 0, iommu->name, iommu);
1083         if (ret)
1084                 printk(KERN_ERR "IOMMU: can't request irq\n");
1085         return ret;
1086 }
1087
1088 static int iommu_init_domains(struct intel_iommu *iommu)
1089 {
1090         unsigned long ndomains;
1091         unsigned long nlongs;
1092
1093         ndomains = cap_ndoms(iommu->cap);
1094         pr_debug("Number of Domains supportd <%ld>\n", ndomains);
1095         nlongs = BITS_TO_LONGS(ndomains);
1096
1097         /* TBD: there might be 64K domains,
1098          * consider other allocation for future chip
1099          */
1100         iommu->domain_ids = kcalloc(nlongs, sizeof(unsigned long), GFP_KERNEL);
1101         if (!iommu->domain_ids) {
1102                 printk(KERN_ERR "Allocating domain id array failed\n");
1103                 return -ENOMEM;
1104         }
1105         iommu->domains = kcalloc(ndomains, sizeof(struct dmar_domain *),
1106                         GFP_KERNEL);
1107         if (!iommu->domains) {
1108                 printk(KERN_ERR "Allocating domain array failed\n");
1109                 kfree(iommu->domain_ids);
1110                 return -ENOMEM;
1111         }
1112
1113         spin_lock_init(&iommu->lock);
1114
1115         /*
1116          * if Caching mode is set, then invalid translations are tagged
1117          * with domainid 0. Hence we need to pre-allocate it.
1118          */
1119         if (cap_caching_mode(iommu->cap))
1120                 set_bit(0, iommu->domain_ids);
1121         return 0;
1122 }
1123
1124
1125 static void domain_exit(struct dmar_domain *domain);
1126
1127 void free_dmar_iommu(struct intel_iommu *iommu)
1128 {
1129         struct dmar_domain *domain;
1130         int i;
1131
1132         i = find_first_bit(iommu->domain_ids, cap_ndoms(iommu->cap));
1133         for (; i < cap_ndoms(iommu->cap); ) {
1134                 domain = iommu->domains[i];
1135                 clear_bit(i, iommu->domain_ids);
1136                 domain_exit(domain);
1137                 i = find_next_bit(iommu->domain_ids,
1138                         cap_ndoms(iommu->cap), i+1);
1139         }
1140
1141         if (iommu->gcmd & DMA_GCMD_TE)
1142                 iommu_disable_translation(iommu);
1143
1144         if (iommu->irq) {
1145                 set_irq_data(iommu->irq, NULL);
1146                 /* This will mask the irq */
1147                 free_irq(iommu->irq, iommu);
1148                 destroy_irq(iommu->irq);
1149         }
1150
1151         kfree(iommu->domains);
1152         kfree(iommu->domain_ids);
1153
1154         /* free context mapping */
1155         free_context_table(iommu);
1156 }
1157
1158 static struct dmar_domain * iommu_alloc_domain(struct intel_iommu *iommu)
1159 {
1160         unsigned long num;
1161         unsigned long ndomains;
1162         struct dmar_domain *domain;
1163         unsigned long flags;
1164
1165         domain = alloc_domain_mem();
1166         if (!domain)
1167                 return NULL;
1168
1169         ndomains = cap_ndoms(iommu->cap);
1170
1171         spin_lock_irqsave(&iommu->lock, flags);
1172         num = find_first_zero_bit(iommu->domain_ids, ndomains);
1173         if (num >= ndomains) {
1174                 spin_unlock_irqrestore(&iommu->lock, flags);
1175                 free_domain_mem(domain);
1176                 printk(KERN_ERR "IOMMU: no free domain ids\n");
1177                 return NULL;
1178         }
1179
1180         set_bit(num, iommu->domain_ids);
1181         domain->id = num;
1182         domain->iommu = iommu;
1183         domain->flags = 0;
1184         iommu->domains[num] = domain;
1185         spin_unlock_irqrestore(&iommu->lock, flags);
1186
1187         return domain;
1188 }
1189
1190 static void iommu_free_domain(struct dmar_domain *domain)
1191 {
1192         unsigned long flags;
1193
1194         spin_lock_irqsave(&domain->iommu->lock, flags);
1195         clear_bit(domain->id, domain->iommu->domain_ids);
1196         spin_unlock_irqrestore(&domain->iommu->lock, flags);
1197 }
1198
1199 static struct iova_domain reserved_iova_list;
1200 static struct lock_class_key reserved_alloc_key;
1201 static struct lock_class_key reserved_rbtree_key;
1202
1203 static void dmar_init_reserved_ranges(void)
1204 {
1205         struct pci_dev *pdev = NULL;
1206         struct iova *iova;
1207         int i;
1208         u64 addr, size;
1209
1210         init_iova_domain(&reserved_iova_list, DMA_32BIT_PFN);
1211
1212         lockdep_set_class(&reserved_iova_list.iova_alloc_lock,
1213                 &reserved_alloc_key);
1214         lockdep_set_class(&reserved_iova_list.iova_rbtree_lock,
1215                 &reserved_rbtree_key);
1216
1217         /* IOAPIC ranges shouldn't be accessed by DMA */
1218         iova = reserve_iova(&reserved_iova_list, IOVA_PFN(IOAPIC_RANGE_START),
1219                 IOVA_PFN(IOAPIC_RANGE_END));
1220         if (!iova)
1221                 printk(KERN_ERR "Reserve IOAPIC range failed\n");
1222
1223         /* Reserve all PCI MMIO to avoid peer-to-peer access */
1224         for_each_pci_dev(pdev) {
1225                 struct resource *r;
1226
1227                 for (i = 0; i < PCI_NUM_RESOURCES; i++) {
1228                         r = &pdev->resource[i];
1229                         if (!r->flags || !(r->flags & IORESOURCE_MEM))
1230                                 continue;
1231                         addr = r->start;
1232                         addr &= PAGE_MASK;
1233                         size = r->end - addr;
1234                         size = PAGE_ALIGN(size);
1235                         iova = reserve_iova(&reserved_iova_list, IOVA_PFN(addr),
1236                                 IOVA_PFN(size + addr) - 1);
1237                         if (!iova)
1238                                 printk(KERN_ERR "Reserve iova failed\n");
1239                 }
1240         }
1241
1242 }
1243
1244 static void domain_reserve_special_ranges(struct dmar_domain *domain)
1245 {
1246         copy_reserved_iova(&reserved_iova_list, &domain->iovad);
1247 }
1248
1249 static inline int guestwidth_to_adjustwidth(int gaw)
1250 {
1251         int agaw;
1252         int r = (gaw - 12) % 9;
1253
1254         if (r == 0)
1255                 agaw = gaw;
1256         else
1257                 agaw = gaw + 9 - r;
1258         if (agaw > 64)
1259                 agaw = 64;
1260         return agaw;
1261 }
1262
1263 static int domain_init(struct dmar_domain *domain, int guest_width)
1264 {
1265         struct intel_iommu *iommu;
1266         int adjust_width, agaw;
1267         unsigned long sagaw;
1268
1269         init_iova_domain(&domain->iovad, DMA_32BIT_PFN);
1270         spin_lock_init(&domain->mapping_lock);
1271
1272         domain_reserve_special_ranges(domain);
1273
1274         /* calculate AGAW */
1275         iommu = domain->iommu;
1276         if (guest_width > cap_mgaw(iommu->cap))
1277                 guest_width = cap_mgaw(iommu->cap);
1278         domain->gaw = guest_width;
1279         adjust_width = guestwidth_to_adjustwidth(guest_width);
1280         agaw = width_to_agaw(adjust_width);
1281         sagaw = cap_sagaw(iommu->cap);
1282         if (!test_bit(agaw, &sagaw)) {
1283                 /* hardware doesn't support it, choose a bigger one */
1284                 pr_debug("IOMMU: hardware doesn't support agaw %d\n", agaw);
1285                 agaw = find_next_bit(&sagaw, 5, agaw);
1286                 if (agaw >= 5)
1287                         return -ENODEV;
1288         }
1289         domain->agaw = agaw;
1290         INIT_LIST_HEAD(&domain->devices);
1291
1292         /* always allocate the top pgd */
1293         domain->pgd = (struct dma_pte *)alloc_pgtable_page();
1294         if (!domain->pgd)
1295                 return -ENOMEM;
1296         __iommu_flush_cache(iommu, domain->pgd, PAGE_SIZE);
1297         return 0;
1298 }
1299
1300 static void domain_exit(struct dmar_domain *domain)
1301 {
1302         u64 end;
1303
1304         /* Domain 0 is reserved, so dont process it */
1305         if (!domain)
1306                 return;
1307
1308         domain_remove_dev_info(domain);
1309         /* destroy iovas */
1310         put_iova_domain(&domain->iovad);
1311         end = DOMAIN_MAX_ADDR(domain->gaw);
1312         end = end & (~PAGE_MASK);
1313
1314         /* clear ptes */
1315         dma_pte_clear_range(domain, 0, end);
1316
1317         /* free page tables */
1318         dma_pte_free_pagetable(domain, 0, end);
1319
1320         iommu_free_domain(domain);
1321         free_domain_mem(domain);
1322 }
1323
1324 static int domain_context_mapping_one(struct dmar_domain *domain,
1325                 u8 bus, u8 devfn)
1326 {
1327         struct context_entry *context;
1328         struct intel_iommu *iommu = domain->iommu;
1329         unsigned long flags;
1330
1331         pr_debug("Set context mapping for %02x:%02x.%d\n",
1332                 bus, PCI_SLOT(devfn), PCI_FUNC(devfn));
1333         BUG_ON(!domain->pgd);
1334         context = device_to_context_entry(iommu, bus, devfn);
1335         if (!context)
1336                 return -ENOMEM;
1337         spin_lock_irqsave(&iommu->lock, flags);
1338         if (context_present(context)) {
1339                 spin_unlock_irqrestore(&iommu->lock, flags);
1340                 return 0;
1341         }
1342
1343         context_set_domain_id(context, domain->id);
1344         context_set_address_width(context, domain->agaw);
1345         context_set_address_root(context, virt_to_phys(domain->pgd));
1346         context_set_translation_type(context, CONTEXT_TT_MULTI_LEVEL);
1347         context_set_fault_enable(context);
1348         context_set_present(context);
1349         __iommu_flush_cache(iommu, context, sizeof(*context));
1350
1351         /* it's a non-present to present mapping */
1352         if (iommu->flush.flush_context(iommu, domain->id,
1353                 (((u16)bus) << 8) | devfn, DMA_CCMD_MASK_NOBIT,
1354                 DMA_CCMD_DEVICE_INVL, 1))
1355                 iommu_flush_write_buffer(iommu);
1356         else
1357                 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_DSI_FLUSH, 0);
1358
1359         spin_unlock_irqrestore(&iommu->lock, flags);
1360         return 0;
1361 }
1362
1363 static int
1364 domain_context_mapping(struct dmar_domain *domain, struct pci_dev *pdev)
1365 {
1366         int ret;
1367         struct pci_dev *tmp, *parent;
1368
1369         ret = domain_context_mapping_one(domain, pdev->bus->number,
1370                 pdev->devfn);
1371         if (ret)
1372                 return ret;
1373
1374         /* dependent device mapping */
1375         tmp = pci_find_upstream_pcie_bridge(pdev);
1376         if (!tmp)
1377                 return 0;
1378         /* Secondary interface's bus number and devfn 0 */
1379         parent = pdev->bus->self;
1380         while (parent != tmp) {
1381                 ret = domain_context_mapping_one(domain, parent->bus->number,
1382                         parent->devfn);
1383                 if (ret)
1384                         return ret;
1385                 parent = parent->bus->self;
1386         }
1387         if (tmp->is_pcie) /* this is a PCIE-to-PCI bridge */
1388                 return domain_context_mapping_one(domain,
1389                         tmp->subordinate->number, 0);
1390         else /* this is a legacy PCI bridge */
1391                 return domain_context_mapping_one(domain,
1392                         tmp->bus->number, tmp->devfn);
1393 }
1394
1395 static int domain_context_mapped(struct dmar_domain *domain,
1396         struct pci_dev *pdev)
1397 {
1398         int ret;
1399         struct pci_dev *tmp, *parent;
1400
1401         ret = device_context_mapped(domain->iommu,
1402                 pdev->bus->number, pdev->devfn);
1403         if (!ret)
1404                 return ret;
1405         /* dependent device mapping */
1406         tmp = pci_find_upstream_pcie_bridge(pdev);
1407         if (!tmp)
1408                 return ret;
1409         /* Secondary interface's bus number and devfn 0 */
1410         parent = pdev->bus->self;
1411         while (parent != tmp) {
1412                 ret = device_context_mapped(domain->iommu, parent->bus->number,
1413                         parent->devfn);
1414                 if (!ret)
1415                         return ret;
1416                 parent = parent->bus->self;
1417         }
1418         if (tmp->is_pcie)
1419                 return device_context_mapped(domain->iommu,
1420                         tmp->subordinate->number, 0);
1421         else
1422                 return device_context_mapped(domain->iommu,
1423                         tmp->bus->number, tmp->devfn);
1424 }
1425
1426 static int
1427 domain_page_mapping(struct dmar_domain *domain, dma_addr_t iova,
1428                         u64 hpa, size_t size, int prot)
1429 {
1430         u64 start_pfn, end_pfn;
1431         struct dma_pte *pte;
1432         int index;
1433         int addr_width = agaw_to_width(domain->agaw);
1434
1435         hpa &= (((u64)1) << addr_width) - 1;
1436
1437         if ((prot & (DMA_PTE_READ|DMA_PTE_WRITE)) == 0)
1438                 return -EINVAL;
1439         iova &= PAGE_MASK;
1440         start_pfn = ((u64)hpa) >> VTD_PAGE_SHIFT;
1441         end_pfn = (VTD_PAGE_ALIGN(((u64)hpa) + size)) >> VTD_PAGE_SHIFT;
1442         index = 0;
1443         while (start_pfn < end_pfn) {
1444                 pte = addr_to_dma_pte(domain, iova + VTD_PAGE_SIZE * index);
1445                 if (!pte)
1446                         return -ENOMEM;
1447                 /* We don't need lock here, nobody else
1448                  * touches the iova range
1449                  */
1450                 BUG_ON(dma_pte_addr(pte));
1451                 dma_set_pte_addr(pte, start_pfn << VTD_PAGE_SHIFT);
1452                 dma_set_pte_prot(pte, prot);
1453                 __iommu_flush_cache(domain->iommu, pte, sizeof(*pte));
1454                 start_pfn++;
1455                 index++;
1456         }
1457         return 0;
1458 }
1459
1460 static void detach_domain_for_dev(struct dmar_domain *domain, u8 bus, u8 devfn)
1461 {
1462         clear_context_table(domain->iommu, bus, devfn);
1463         domain->iommu->flush.flush_context(domain->iommu, 0, 0, 0,
1464                                            DMA_CCMD_GLOBAL_INVL, 0);
1465         domain->iommu->flush.flush_iotlb(domain->iommu, 0, 0, 0,
1466                                          DMA_TLB_GLOBAL_FLUSH, 0);
1467 }
1468
1469 static void domain_remove_dev_info(struct dmar_domain *domain)
1470 {
1471         struct device_domain_info *info;
1472         unsigned long flags;
1473
1474         spin_lock_irqsave(&device_domain_lock, flags);
1475         while (!list_empty(&domain->devices)) {
1476                 info = list_entry(domain->devices.next,
1477                         struct device_domain_info, link);
1478                 list_del(&info->link);
1479                 list_del(&info->global);
1480                 if (info->dev)
1481                         info->dev->dev.archdata.iommu = NULL;
1482                 spin_unlock_irqrestore(&device_domain_lock, flags);
1483
1484                 detach_domain_for_dev(info->domain, info->bus, info->devfn);
1485                 free_devinfo_mem(info);
1486
1487                 spin_lock_irqsave(&device_domain_lock, flags);
1488         }
1489         spin_unlock_irqrestore(&device_domain_lock, flags);
1490 }
1491
1492 /*
1493  * find_domain
1494  * Note: we use struct pci_dev->dev.archdata.iommu stores the info
1495  */
1496 static struct dmar_domain *
1497 find_domain(struct pci_dev *pdev)
1498 {
1499         struct device_domain_info *info;
1500
1501         /* No lock here, assumes no domain exit in normal case */
1502         info = pdev->dev.archdata.iommu;
1503         if (info)
1504                 return info->domain;
1505         return NULL;
1506 }
1507
1508 /* domain is initialized */
1509 static struct dmar_domain *get_domain_for_dev(struct pci_dev *pdev, int gaw)
1510 {
1511         struct dmar_domain *domain, *found = NULL;
1512         struct intel_iommu *iommu;
1513         struct dmar_drhd_unit *drhd;
1514         struct device_domain_info *info, *tmp;
1515         struct pci_dev *dev_tmp;
1516         unsigned long flags;
1517         int bus = 0, devfn = 0;
1518
1519         domain = find_domain(pdev);
1520         if (domain)
1521                 return domain;
1522
1523         dev_tmp = pci_find_upstream_pcie_bridge(pdev);
1524         if (dev_tmp) {
1525                 if (dev_tmp->is_pcie) {
1526                         bus = dev_tmp->subordinate->number;
1527                         devfn = 0;
1528                 } else {
1529                         bus = dev_tmp->bus->number;
1530                         devfn = dev_tmp->devfn;
1531                 }
1532                 spin_lock_irqsave(&device_domain_lock, flags);
1533                 list_for_each_entry(info, &device_domain_list, global) {
1534                         if (info->bus == bus && info->devfn == devfn) {
1535                                 found = info->domain;
1536                                 break;
1537                         }
1538                 }
1539                 spin_unlock_irqrestore(&device_domain_lock, flags);
1540                 /* pcie-pci bridge already has a domain, uses it */
1541                 if (found) {
1542                         domain = found;
1543                         goto found_domain;
1544                 }
1545         }
1546
1547         /* Allocate new domain for the device */
1548         drhd = dmar_find_matched_drhd_unit(pdev);
1549         if (!drhd) {
1550                 printk(KERN_ERR "IOMMU: can't find DMAR for device %s\n",
1551                         pci_name(pdev));
1552                 return NULL;
1553         }
1554         iommu = drhd->iommu;
1555
1556         domain = iommu_alloc_domain(iommu);
1557         if (!domain)
1558                 goto error;
1559
1560         if (domain_init(domain, gaw)) {
1561                 domain_exit(domain);
1562                 goto error;
1563         }
1564
1565         /* register pcie-to-pci device */
1566         if (dev_tmp) {
1567                 info = alloc_devinfo_mem();
1568                 if (!info) {
1569                         domain_exit(domain);
1570                         goto error;
1571                 }
1572                 info->bus = bus;
1573                 info->devfn = devfn;
1574                 info->dev = NULL;
1575                 info->domain = domain;
1576                 /* This domain is shared by devices under p2p bridge */
1577                 domain->flags |= DOMAIN_FLAG_MULTIPLE_DEVICES;
1578
1579                 /* pcie-to-pci bridge already has a domain, uses it */
1580                 found = NULL;
1581                 spin_lock_irqsave(&device_domain_lock, flags);
1582                 list_for_each_entry(tmp, &device_domain_list, global) {
1583                         if (tmp->bus == bus && tmp->devfn == devfn) {
1584                                 found = tmp->domain;
1585                                 break;
1586                         }
1587                 }
1588                 if (found) {
1589                         free_devinfo_mem(info);
1590                         domain_exit(domain);
1591                         domain = found;
1592                 } else {
1593                         list_add(&info->link, &domain->devices);
1594                         list_add(&info->global, &device_domain_list);
1595                 }
1596                 spin_unlock_irqrestore(&device_domain_lock, flags);
1597         }
1598
1599 found_domain:
1600         info = alloc_devinfo_mem();
1601         if (!info)
1602                 goto error;
1603         info->bus = pdev->bus->number;
1604         info->devfn = pdev->devfn;
1605         info->dev = pdev;
1606         info->domain = domain;
1607         spin_lock_irqsave(&device_domain_lock, flags);
1608         /* somebody is fast */
1609         found = find_domain(pdev);
1610         if (found != NULL) {
1611                 spin_unlock_irqrestore(&device_domain_lock, flags);
1612                 if (found != domain) {
1613                         domain_exit(domain);
1614                         domain = found;
1615                 }
1616                 free_devinfo_mem(info);
1617                 return domain;
1618         }
1619         list_add(&info->link, &domain->devices);
1620         list_add(&info->global, &device_domain_list);
1621         pdev->dev.archdata.iommu = info;
1622         spin_unlock_irqrestore(&device_domain_lock, flags);
1623         return domain;
1624 error:
1625         /* recheck it here, maybe others set it */
1626         return find_domain(pdev);
1627 }
1628
1629 static int iommu_prepare_identity_map(struct pci_dev *pdev,
1630                                       unsigned long long start,
1631                                       unsigned long long end)
1632 {
1633         struct dmar_domain *domain;
1634         unsigned long size;
1635         unsigned long long base;
1636         int ret;
1637
1638         printk(KERN_INFO
1639                 "IOMMU: Setting identity map for device %s [0x%Lx - 0x%Lx]\n",
1640                 pci_name(pdev), start, end);
1641         /* page table init */
1642         domain = get_domain_for_dev(pdev, DEFAULT_DOMAIN_ADDRESS_WIDTH);
1643         if (!domain)
1644                 return -ENOMEM;
1645
1646         /* The address might not be aligned */
1647         base = start & PAGE_MASK;
1648         size = end - base;
1649         size = PAGE_ALIGN(size);
1650         if (!reserve_iova(&domain->iovad, IOVA_PFN(base),
1651                         IOVA_PFN(base + size) - 1)) {
1652                 printk(KERN_ERR "IOMMU: reserve iova failed\n");
1653                 ret = -ENOMEM;
1654                 goto error;
1655         }
1656
1657         pr_debug("Mapping reserved region %lx@%llx for %s\n",
1658                 size, base, pci_name(pdev));
1659         /*
1660          * RMRR range might have overlap with physical memory range,
1661          * clear it first
1662          */
1663         dma_pte_clear_range(domain, base, base + size);
1664
1665         ret = domain_page_mapping(domain, base, base, size,
1666                 DMA_PTE_READ|DMA_PTE_WRITE);
1667         if (ret)
1668                 goto error;
1669
1670         /* context entry init */
1671         ret = domain_context_mapping(domain, pdev);
1672         if (!ret)
1673                 return 0;
1674 error:
1675         domain_exit(domain);
1676         return ret;
1677
1678 }
1679
1680 static inline int iommu_prepare_rmrr_dev(struct dmar_rmrr_unit *rmrr,
1681         struct pci_dev *pdev)
1682 {
1683         if (pdev->dev.archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO)
1684                 return 0;
1685         return iommu_prepare_identity_map(pdev, rmrr->base_address,
1686                 rmrr->end_address + 1);
1687 }
1688
1689 #ifdef CONFIG_DMAR_GFX_WA
1690 struct iommu_prepare_data {
1691         struct pci_dev *pdev;
1692         int ret;
1693 };
1694
1695 static int __init iommu_prepare_work_fn(unsigned long start_pfn,
1696                                          unsigned long end_pfn, void *datax)
1697 {
1698         struct iommu_prepare_data *data;
1699
1700         data = (struct iommu_prepare_data *)datax;
1701
1702         data->ret = iommu_prepare_identity_map(data->pdev,
1703                                 start_pfn<<PAGE_SHIFT, end_pfn<<PAGE_SHIFT);
1704         return data->ret;
1705
1706 }
1707
1708 static int __init iommu_prepare_with_active_regions(struct pci_dev *pdev)
1709 {
1710         int nid;
1711         struct iommu_prepare_data data;
1712
1713         data.pdev = pdev;
1714         data.ret = 0;
1715
1716         for_each_online_node(nid) {
1717                 work_with_active_regions(nid, iommu_prepare_work_fn, &data);
1718                 if (data.ret)
1719                         return data.ret;
1720         }
1721         return data.ret;
1722 }
1723
1724 static void __init iommu_prepare_gfx_mapping(void)
1725 {
1726         struct pci_dev *pdev = NULL;
1727         int ret;
1728
1729         for_each_pci_dev(pdev) {
1730                 if (pdev->dev.archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO ||
1731                                 !IS_GFX_DEVICE(pdev))
1732                         continue;
1733                 printk(KERN_INFO "IOMMU: gfx device %s 1-1 mapping\n",
1734                         pci_name(pdev));
1735                 ret = iommu_prepare_with_active_regions(pdev);
1736                 if (ret)
1737                         printk(KERN_ERR "IOMMU: mapping reserved region failed\n");
1738         }
1739 }
1740 #else /* !CONFIG_DMAR_GFX_WA */
1741 static inline void iommu_prepare_gfx_mapping(void)
1742 {
1743         return;
1744 }
1745 #endif
1746
1747 #ifdef CONFIG_DMAR_FLOPPY_WA
1748 static inline void iommu_prepare_isa(void)
1749 {
1750         struct pci_dev *pdev;
1751         int ret;
1752
1753         pdev = pci_get_class(PCI_CLASS_BRIDGE_ISA << 8, NULL);
1754         if (!pdev)
1755                 return;
1756
1757         printk(KERN_INFO "IOMMU: Prepare 0-16M unity mapping for LPC\n");
1758         ret = iommu_prepare_identity_map(pdev, 0, 16*1024*1024);
1759
1760         if (ret)
1761                 printk("IOMMU: Failed to create 0-64M identity map, "
1762                         "floppy might not work\n");
1763
1764 }
1765 #else
1766 static inline void iommu_prepare_isa(void)
1767 {
1768         return;
1769 }
1770 #endif /* !CONFIG_DMAR_FLPY_WA */
1771
1772 static int __init init_dmars(void)
1773 {
1774         struct dmar_drhd_unit *drhd;
1775         struct dmar_rmrr_unit *rmrr;
1776         struct pci_dev *pdev;
1777         struct intel_iommu *iommu;
1778         int i, ret, unit = 0;
1779
1780         /*
1781          * for each drhd
1782          *    allocate root
1783          *    initialize and program root entry to not present
1784          * endfor
1785          */
1786         for_each_drhd_unit(drhd) {
1787                 g_num_of_iommus++;
1788                 /*
1789                  * lock not needed as this is only incremented in the single
1790                  * threaded kernel __init code path all other access are read
1791                  * only
1792                  */
1793         }
1794
1795         deferred_flush = kzalloc(g_num_of_iommus *
1796                 sizeof(struct deferred_flush_tables), GFP_KERNEL);
1797         if (!deferred_flush) {
1798                 ret = -ENOMEM;
1799                 goto error;
1800         }
1801
1802         for_each_drhd_unit(drhd) {
1803                 if (drhd->ignored)
1804                         continue;
1805
1806                 iommu = drhd->iommu;
1807
1808                 ret = iommu_init_domains(iommu);
1809                 if (ret)
1810                         goto error;
1811
1812                 /*
1813                  * TBD:
1814                  * we could share the same root & context tables
1815                  * amoung all IOMMU's. Need to Split it later.
1816                  */
1817                 ret = iommu_alloc_root_entry(iommu);
1818                 if (ret) {
1819                         printk(KERN_ERR "IOMMU: allocate root entry failed\n");
1820                         goto error;
1821                 }
1822         }
1823
1824         for_each_drhd_unit(drhd) {
1825                 if (drhd->ignored)
1826                         continue;
1827
1828                 iommu = drhd->iommu;
1829                 if (dmar_enable_qi(iommu)) {
1830                         /*
1831                          * Queued Invalidate not enabled, use Register Based
1832                          * Invalidate
1833                          */
1834                         iommu->flush.flush_context = __iommu_flush_context;
1835                         iommu->flush.flush_iotlb = __iommu_flush_iotlb;
1836                         printk(KERN_INFO "IOMMU 0x%Lx: using Register based "
1837                                "invalidation\n",
1838                                (unsigned long long)drhd->reg_base_addr);
1839                 } else {
1840                         iommu->flush.flush_context = qi_flush_context;
1841                         iommu->flush.flush_iotlb = qi_flush_iotlb;
1842                         printk(KERN_INFO "IOMMU 0x%Lx: using Queued "
1843                                "invalidation\n",
1844                                (unsigned long long)drhd->reg_base_addr);
1845                 }
1846         }
1847
1848         /*
1849          * For each rmrr
1850          *   for each dev attached to rmrr
1851          *   do
1852          *     locate drhd for dev, alloc domain for dev
1853          *     allocate free domain
1854          *     allocate page table entries for rmrr
1855          *     if context not allocated for bus
1856          *           allocate and init context
1857          *           set present in root table for this bus
1858          *     init context with domain, translation etc
1859          *    endfor
1860          * endfor
1861          */
1862         for_each_rmrr_units(rmrr) {
1863                 for (i = 0; i < rmrr->devices_cnt; i++) {
1864                         pdev = rmrr->devices[i];
1865                         /* some BIOS lists non-exist devices in DMAR table */
1866                         if (!pdev)
1867                                 continue;
1868                         ret = iommu_prepare_rmrr_dev(rmrr, pdev);
1869                         if (ret)
1870                                 printk(KERN_ERR
1871                                  "IOMMU: mapping reserved region failed\n");
1872                 }
1873         }
1874
1875         iommu_prepare_gfx_mapping();
1876
1877         iommu_prepare_isa();
1878
1879         /*
1880          * for each drhd
1881          *   enable fault log
1882          *   global invalidate context cache
1883          *   global invalidate iotlb
1884          *   enable translation
1885          */
1886         for_each_drhd_unit(drhd) {
1887                 if (drhd->ignored)
1888                         continue;
1889                 iommu = drhd->iommu;
1890                 sprintf (iommu->name, "dmar%d", unit++);
1891
1892                 iommu_flush_write_buffer(iommu);
1893
1894                 ret = dmar_set_interrupt(iommu);
1895                 if (ret)
1896                         goto error;
1897
1898                 iommu_set_root_entry(iommu);
1899
1900                 iommu->flush.flush_context(iommu, 0, 0, 0, DMA_CCMD_GLOBAL_INVL,
1901                                            0);
1902                 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH,
1903                                          0);
1904                 iommu_disable_protect_mem_regions(iommu);
1905
1906                 ret = iommu_enable_translation(iommu);
1907                 if (ret)
1908                         goto error;
1909         }
1910
1911         return 0;
1912 error:
1913         for_each_drhd_unit(drhd) {
1914                 if (drhd->ignored)
1915                         continue;
1916                 iommu = drhd->iommu;
1917                 free_iommu(iommu);
1918         }
1919         return ret;
1920 }
1921
1922 static inline u64 aligned_size(u64 host_addr, size_t size)
1923 {
1924         u64 addr;
1925         addr = (host_addr & (~PAGE_MASK)) + size;
1926         return PAGE_ALIGN(addr);
1927 }
1928
1929 struct iova *
1930 iommu_alloc_iova(struct dmar_domain *domain, size_t size, u64 end)
1931 {
1932         struct iova *piova;
1933
1934         /* Make sure it's in range */
1935         end = min_t(u64, DOMAIN_MAX_ADDR(domain->gaw), end);
1936         if (!size || (IOVA_START_ADDR + size > end))
1937                 return NULL;
1938
1939         piova = alloc_iova(&domain->iovad,
1940                         size >> PAGE_SHIFT, IOVA_PFN(end), 1);
1941         return piova;
1942 }
1943
1944 static struct iova *
1945 __intel_alloc_iova(struct device *dev, struct dmar_domain *domain,
1946                    size_t size, u64 dma_mask)
1947 {
1948         struct pci_dev *pdev = to_pci_dev(dev);
1949         struct iova *iova = NULL;
1950
1951         if (dma_mask <= DMA_32BIT_MASK || dmar_forcedac)
1952                 iova = iommu_alloc_iova(domain, size, dma_mask);
1953         else {
1954                 /*
1955                  * First try to allocate an io virtual address in
1956                  * DMA_32BIT_MASK and if that fails then try allocating
1957                  * from higher range
1958                  */
1959                 iova = iommu_alloc_iova(domain, size, DMA_32BIT_MASK);
1960                 if (!iova)
1961                         iova = iommu_alloc_iova(domain, size, dma_mask);
1962         }
1963
1964         if (!iova) {
1965                 printk(KERN_ERR"Allocating iova for %s failed", pci_name(pdev));
1966                 return NULL;
1967         }
1968
1969         return iova;
1970 }
1971
1972 static struct dmar_domain *
1973 get_valid_domain_for_dev(struct pci_dev *pdev)
1974 {
1975         struct dmar_domain *domain;
1976         int ret;
1977
1978         domain = get_domain_for_dev(pdev,
1979                         DEFAULT_DOMAIN_ADDRESS_WIDTH);
1980         if (!domain) {
1981                 printk(KERN_ERR
1982                         "Allocating domain for %s failed", pci_name(pdev));
1983                 return NULL;
1984         }
1985
1986         /* make sure context mapping is ok */
1987         if (unlikely(!domain_context_mapped(domain, pdev))) {
1988                 ret = domain_context_mapping(domain, pdev);
1989                 if (ret) {
1990                         printk(KERN_ERR
1991                                 "Domain context map for %s failed",
1992                                 pci_name(pdev));
1993                         return NULL;
1994                 }
1995         }
1996
1997         return domain;
1998 }
1999
2000 static dma_addr_t __intel_map_single(struct device *hwdev, phys_addr_t paddr,
2001                                      size_t size, int dir, u64 dma_mask)
2002 {
2003         struct pci_dev *pdev = to_pci_dev(hwdev);
2004         struct dmar_domain *domain;
2005         phys_addr_t start_paddr;
2006         struct iova *iova;
2007         int prot = 0;
2008         int ret;
2009
2010         BUG_ON(dir == DMA_NONE);
2011         if (pdev->dev.archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO)
2012                 return paddr;
2013
2014         domain = get_valid_domain_for_dev(pdev);
2015         if (!domain)
2016                 return 0;
2017
2018         size = aligned_size((u64)paddr, size);
2019
2020         iova = __intel_alloc_iova(hwdev, domain, size, pdev->dma_mask);
2021         if (!iova)
2022                 goto error;
2023
2024         start_paddr = (phys_addr_t)iova->pfn_lo << PAGE_SHIFT;
2025
2026         /*
2027          * Check if DMAR supports zero-length reads on write only
2028          * mappings..
2029          */
2030         if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL || \
2031                         !cap_zlr(domain->iommu->cap))
2032                 prot |= DMA_PTE_READ;
2033         if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
2034                 prot |= DMA_PTE_WRITE;
2035         /*
2036          * paddr - (paddr + size) might be partial page, we should map the whole
2037          * page.  Note: if two part of one page are separately mapped, we
2038          * might have two guest_addr mapping to the same host paddr, but this
2039          * is not a big problem
2040          */
2041         ret = domain_page_mapping(domain, start_paddr,
2042                 ((u64)paddr) & PAGE_MASK, size, prot);
2043         if (ret)
2044                 goto error;
2045
2046         /* it's a non-present to present mapping */
2047         ret = iommu_flush_iotlb_psi(domain->iommu, domain->id,
2048                         start_paddr, size >> VTD_PAGE_SHIFT, 1);
2049         if (ret)
2050                 iommu_flush_write_buffer(domain->iommu);
2051
2052         return start_paddr + ((u64)paddr & (~PAGE_MASK));
2053
2054 error:
2055         if (iova)
2056                 __free_iova(&domain->iovad, iova);
2057         printk(KERN_ERR"Device %s request: %lx@%llx dir %d --- failed\n",
2058                 pci_name(pdev), size, (unsigned long long)paddr, dir);
2059         return 0;
2060 }
2061
2062 dma_addr_t intel_map_single(struct device *hwdev, phys_addr_t paddr,
2063                             size_t size, int dir)
2064 {
2065         return __intel_map_single(hwdev, paddr, size, dir,
2066                                   to_pci_dev(hwdev)->dma_mask);
2067 }
2068
2069 static void flush_unmaps(void)
2070 {
2071         int i, j;
2072
2073         timer_on = 0;
2074
2075         /* just flush them all */
2076         for (i = 0; i < g_num_of_iommus; i++) {
2077                 if (deferred_flush[i].next) {
2078                         struct intel_iommu *iommu =
2079                                 deferred_flush[i].domain[0]->iommu;
2080
2081                         iommu->flush.flush_iotlb(iommu, 0, 0, 0,
2082                                                  DMA_TLB_GLOBAL_FLUSH, 0);
2083                         for (j = 0; j < deferred_flush[i].next; j++) {
2084                                 __free_iova(&deferred_flush[i].domain[j]->iovad,
2085                                                 deferred_flush[i].iova[j]);
2086                         }
2087                         deferred_flush[i].next = 0;
2088                 }
2089         }
2090
2091         list_size = 0;
2092 }
2093
2094 static void flush_unmaps_timeout(unsigned long data)
2095 {
2096         unsigned long flags;
2097
2098         spin_lock_irqsave(&async_umap_flush_lock, flags);
2099         flush_unmaps();
2100         spin_unlock_irqrestore(&async_umap_flush_lock, flags);
2101 }
2102
2103 static void add_unmap(struct dmar_domain *dom, struct iova *iova)
2104 {
2105         unsigned long flags;
2106         int next, iommu_id;
2107
2108         spin_lock_irqsave(&async_umap_flush_lock, flags);
2109         if (list_size == HIGH_WATER_MARK)
2110                 flush_unmaps();
2111
2112         iommu_id = dom->iommu->seq_id;
2113
2114         next = deferred_flush[iommu_id].next;
2115         deferred_flush[iommu_id].domain[next] = dom;
2116         deferred_flush[iommu_id].iova[next] = iova;
2117         deferred_flush[iommu_id].next++;
2118
2119         if (!timer_on) {
2120                 mod_timer(&unmap_timer, jiffies + msecs_to_jiffies(10));
2121                 timer_on = 1;
2122         }
2123         list_size++;
2124         spin_unlock_irqrestore(&async_umap_flush_lock, flags);
2125 }
2126
2127 void intel_unmap_single(struct device *dev, dma_addr_t dev_addr, size_t size,
2128                         int dir)
2129 {
2130         struct pci_dev *pdev = to_pci_dev(dev);
2131         struct dmar_domain *domain;
2132         unsigned long start_addr;
2133         struct iova *iova;
2134
2135         if (pdev->dev.archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO)
2136                 return;
2137         domain = find_domain(pdev);
2138         BUG_ON(!domain);
2139
2140         iova = find_iova(&domain->iovad, IOVA_PFN(dev_addr));
2141         if (!iova)
2142                 return;
2143
2144         start_addr = iova->pfn_lo << PAGE_SHIFT;
2145         size = aligned_size((u64)dev_addr, size);
2146
2147         pr_debug("Device %s unmapping: %lx@%llx\n",
2148                 pci_name(pdev), size, (unsigned long long)start_addr);
2149
2150         /*  clear the whole page */
2151         dma_pte_clear_range(domain, start_addr, start_addr + size);
2152         /* free page tables */
2153         dma_pte_free_pagetable(domain, start_addr, start_addr + size);
2154         if (intel_iommu_strict) {
2155                 if (iommu_flush_iotlb_psi(domain->iommu,
2156                         domain->id, start_addr, size >> VTD_PAGE_SHIFT, 0))
2157                         iommu_flush_write_buffer(domain->iommu);
2158                 /* free iova */
2159                 __free_iova(&domain->iovad, iova);
2160         } else {
2161                 add_unmap(domain, iova);
2162                 /*
2163                  * queue up the release of the unmap to save the 1/6th of the
2164                  * cpu used up by the iotlb flush operation...
2165                  */
2166         }
2167 }
2168
2169 void *intel_alloc_coherent(struct device *hwdev, size_t size,
2170                            dma_addr_t *dma_handle, gfp_t flags)
2171 {
2172         void *vaddr;
2173         int order;
2174
2175         size = PAGE_ALIGN(size);
2176         order = get_order(size);
2177         flags &= ~(GFP_DMA | GFP_DMA32);
2178
2179         vaddr = (void *)__get_free_pages(flags, order);
2180         if (!vaddr)
2181                 return NULL;
2182         memset(vaddr, 0, size);
2183
2184         *dma_handle = __intel_map_single(hwdev, virt_to_bus(vaddr), size,
2185                                          DMA_BIDIRECTIONAL,
2186                                          hwdev->coherent_dma_mask);
2187         if (*dma_handle)
2188                 return vaddr;
2189         free_pages((unsigned long)vaddr, order);
2190         return NULL;
2191 }
2192
2193 void intel_free_coherent(struct device *hwdev, size_t size, void *vaddr,
2194                          dma_addr_t dma_handle)
2195 {
2196         int order;
2197
2198         size = PAGE_ALIGN(size);
2199         order = get_order(size);
2200
2201         intel_unmap_single(hwdev, dma_handle, size, DMA_BIDIRECTIONAL);
2202         free_pages((unsigned long)vaddr, order);
2203 }
2204
2205 #define SG_ENT_VIRT_ADDRESS(sg) (sg_virt((sg)))
2206
2207 void intel_unmap_sg(struct device *hwdev, struct scatterlist *sglist,
2208                     int nelems, int dir)
2209 {
2210         int i;
2211         struct pci_dev *pdev = to_pci_dev(hwdev);
2212         struct dmar_domain *domain;
2213         unsigned long start_addr;
2214         struct iova *iova;
2215         size_t size = 0;
2216         void *addr;
2217         struct scatterlist *sg;
2218
2219         if (pdev->dev.archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO)
2220                 return;
2221
2222         domain = find_domain(pdev);
2223
2224         iova = find_iova(&domain->iovad, IOVA_PFN(sglist[0].dma_address));
2225         if (!iova)
2226                 return;
2227         for_each_sg(sglist, sg, nelems, i) {
2228                 addr = SG_ENT_VIRT_ADDRESS(sg);
2229                 size += aligned_size((u64)addr, sg->length);
2230         }
2231
2232         start_addr = iova->pfn_lo << PAGE_SHIFT;
2233
2234         /*  clear the whole page */
2235         dma_pte_clear_range(domain, start_addr, start_addr + size);
2236         /* free page tables */
2237         dma_pte_free_pagetable(domain, start_addr, start_addr + size);
2238
2239         if (iommu_flush_iotlb_psi(domain->iommu, domain->id, start_addr,
2240                         size >> VTD_PAGE_SHIFT, 0))
2241                 iommu_flush_write_buffer(domain->iommu);
2242
2243         /* free iova */
2244         __free_iova(&domain->iovad, iova);
2245 }
2246
2247 static int intel_nontranslate_map_sg(struct device *hddev,
2248         struct scatterlist *sglist, int nelems, int dir)
2249 {
2250         int i;
2251         struct scatterlist *sg;
2252
2253         for_each_sg(sglist, sg, nelems, i) {
2254                 BUG_ON(!sg_page(sg));
2255                 sg->dma_address = virt_to_bus(SG_ENT_VIRT_ADDRESS(sg));
2256                 sg->dma_length = sg->length;
2257         }
2258         return nelems;
2259 }
2260
2261 int intel_map_sg(struct device *hwdev, struct scatterlist *sglist, int nelems,
2262                  int dir)
2263 {
2264         void *addr;
2265         int i;
2266         struct pci_dev *pdev = to_pci_dev(hwdev);
2267         struct dmar_domain *domain;
2268         size_t size = 0;
2269         int prot = 0;
2270         size_t offset = 0;
2271         struct iova *iova = NULL;
2272         int ret;
2273         struct scatterlist *sg;
2274         unsigned long start_addr;
2275
2276         BUG_ON(dir == DMA_NONE);
2277         if (pdev->dev.archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO)
2278                 return intel_nontranslate_map_sg(hwdev, sglist, nelems, dir);
2279
2280         domain = get_valid_domain_for_dev(pdev);
2281         if (!domain)
2282                 return 0;
2283
2284         for_each_sg(sglist, sg, nelems, i) {
2285                 addr = SG_ENT_VIRT_ADDRESS(sg);
2286                 addr = (void *)virt_to_phys(addr);
2287                 size += aligned_size((u64)addr, sg->length);
2288         }
2289
2290         iova = __intel_alloc_iova(hwdev, domain, size, pdev->dma_mask);
2291         if (!iova) {
2292                 sglist->dma_length = 0;
2293                 return 0;
2294         }
2295
2296         /*
2297          * Check if DMAR supports zero-length reads on write only
2298          * mappings..
2299          */
2300         if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL || \
2301                         !cap_zlr(domain->iommu->cap))
2302                 prot |= DMA_PTE_READ;
2303         if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
2304                 prot |= DMA_PTE_WRITE;
2305
2306         start_addr = iova->pfn_lo << PAGE_SHIFT;
2307         offset = 0;
2308         for_each_sg(sglist, sg, nelems, i) {
2309                 addr = SG_ENT_VIRT_ADDRESS(sg);
2310                 addr = (void *)virt_to_phys(addr);
2311                 size = aligned_size((u64)addr, sg->length);
2312                 ret = domain_page_mapping(domain, start_addr + offset,
2313                         ((u64)addr) & PAGE_MASK,
2314                         size, prot);
2315                 if (ret) {
2316                         /*  clear the page */
2317                         dma_pte_clear_range(domain, start_addr,
2318                                   start_addr + offset);
2319                         /* free page tables */
2320                         dma_pte_free_pagetable(domain, start_addr,
2321                                   start_addr + offset);
2322                         /* free iova */
2323                         __free_iova(&domain->iovad, iova);
2324                         return 0;
2325                 }
2326                 sg->dma_address = start_addr + offset +
2327                                 ((u64)addr & (~PAGE_MASK));
2328                 sg->dma_length = sg->length;
2329                 offset += size;
2330         }
2331
2332         /* it's a non-present to present mapping */
2333         if (iommu_flush_iotlb_psi(domain->iommu, domain->id,
2334                         start_addr, offset >> VTD_PAGE_SHIFT, 1))
2335                 iommu_flush_write_buffer(domain->iommu);
2336         return nelems;
2337 }
2338
2339 static struct dma_mapping_ops intel_dma_ops = {
2340         .alloc_coherent = intel_alloc_coherent,
2341         .free_coherent = intel_free_coherent,
2342         .map_single = intel_map_single,
2343         .unmap_single = intel_unmap_single,
2344         .map_sg = intel_map_sg,
2345         .unmap_sg = intel_unmap_sg,
2346 };
2347
2348 static inline int iommu_domain_cache_init(void)
2349 {
2350         int ret = 0;
2351
2352         iommu_domain_cache = kmem_cache_create("iommu_domain",
2353                                          sizeof(struct dmar_domain),
2354                                          0,
2355                                          SLAB_HWCACHE_ALIGN,
2356
2357                                          NULL);
2358         if (!iommu_domain_cache) {
2359                 printk(KERN_ERR "Couldn't create iommu_domain cache\n");
2360                 ret = -ENOMEM;
2361         }
2362
2363         return ret;
2364 }
2365
2366 static inline int iommu_devinfo_cache_init(void)
2367 {
2368         int ret = 0;
2369
2370         iommu_devinfo_cache = kmem_cache_create("iommu_devinfo",
2371                                          sizeof(struct device_domain_info),
2372                                          0,
2373                                          SLAB_HWCACHE_ALIGN,
2374                                          NULL);
2375         if (!iommu_devinfo_cache) {
2376                 printk(KERN_ERR "Couldn't create devinfo cache\n");
2377                 ret = -ENOMEM;
2378         }
2379
2380         return ret;
2381 }
2382
2383 static inline int iommu_iova_cache_init(void)
2384 {
2385         int ret = 0;
2386
2387         iommu_iova_cache = kmem_cache_create("iommu_iova",
2388                                          sizeof(struct iova),
2389                                          0,
2390                                          SLAB_HWCACHE_ALIGN,
2391                                          NULL);
2392         if (!iommu_iova_cache) {
2393                 printk(KERN_ERR "Couldn't create iova cache\n");
2394                 ret = -ENOMEM;
2395         }
2396
2397         return ret;
2398 }
2399
2400 static int __init iommu_init_mempool(void)
2401 {
2402         int ret;
2403         ret = iommu_iova_cache_init();
2404         if (ret)
2405                 return ret;
2406
2407         ret = iommu_domain_cache_init();
2408         if (ret)
2409                 goto domain_error;
2410
2411         ret = iommu_devinfo_cache_init();
2412         if (!ret)
2413                 return ret;
2414
2415         kmem_cache_destroy(iommu_domain_cache);
2416 domain_error:
2417         kmem_cache_destroy(iommu_iova_cache);
2418
2419         return -ENOMEM;
2420 }
2421
2422 static void __init iommu_exit_mempool(void)
2423 {
2424         kmem_cache_destroy(iommu_devinfo_cache);
2425         kmem_cache_destroy(iommu_domain_cache);
2426         kmem_cache_destroy(iommu_iova_cache);
2427
2428 }
2429
2430 static void __init init_no_remapping_devices(void)
2431 {
2432         struct dmar_drhd_unit *drhd;
2433
2434         for_each_drhd_unit(drhd) {
2435                 if (!drhd->include_all) {
2436                         int i;
2437                         for (i = 0; i < drhd->devices_cnt; i++)
2438                                 if (drhd->devices[i] != NULL)
2439                                         break;
2440                         /* ignore DMAR unit if no pci devices exist */
2441                         if (i == drhd->devices_cnt)
2442                                 drhd->ignored = 1;
2443                 }
2444         }
2445
2446         if (dmar_map_gfx)
2447                 return;
2448
2449         for_each_drhd_unit(drhd) {
2450                 int i;
2451                 if (drhd->ignored || drhd->include_all)
2452                         continue;
2453
2454                 for (i = 0; i < drhd->devices_cnt; i++)
2455                         if (drhd->devices[i] &&
2456                                 !IS_GFX_DEVICE(drhd->devices[i]))
2457                                 break;
2458
2459                 if (i < drhd->devices_cnt)
2460                         continue;
2461
2462                 /* bypass IOMMU if it is just for gfx devices */
2463                 drhd->ignored = 1;
2464                 for (i = 0; i < drhd->devices_cnt; i++) {
2465                         if (!drhd->devices[i])
2466                                 continue;
2467                         drhd->devices[i]->dev.archdata.iommu = DUMMY_DEVICE_DOMAIN_INFO;
2468                 }
2469         }
2470 }
2471
2472 int __init intel_iommu_init(void)
2473 {
2474         int ret = 0;
2475
2476         if (dmar_table_init())
2477                 return  -ENODEV;
2478
2479         if (dmar_dev_scope_init())
2480                 return  -ENODEV;
2481
2482         /*
2483          * Check the need for DMA-remapping initialization now.
2484          * Above initialization will also be used by Interrupt-remapping.
2485          */
2486         if (no_iommu || swiotlb || dmar_disabled)
2487                 return -ENODEV;
2488
2489         iommu_init_mempool();
2490         dmar_init_reserved_ranges();
2491
2492         init_no_remapping_devices();
2493
2494         ret = init_dmars();
2495         if (ret) {
2496                 printk(KERN_ERR "IOMMU: dmar init failed\n");
2497                 put_iova_domain(&reserved_iova_list);
2498                 iommu_exit_mempool();
2499                 return ret;
2500         }
2501         printk(KERN_INFO
2502         "PCI-DMA: Intel(R) Virtualization Technology for Directed I/O\n");
2503
2504         init_timer(&unmap_timer);
2505         force_iommu = 1;
2506         dma_ops = &intel_dma_ops;
2507         return 0;
2508 }
2509
2510 void intel_iommu_domain_exit(struct dmar_domain *domain)
2511 {
2512         u64 end;
2513
2514         /* Domain 0 is reserved, so dont process it */
2515         if (!domain)
2516                 return;
2517
2518         end = DOMAIN_MAX_ADDR(domain->gaw);
2519         end = end & (~VTD_PAGE_MASK);
2520
2521         /* clear ptes */
2522         dma_pte_clear_range(domain, 0, end);
2523
2524         /* free page tables */
2525         dma_pte_free_pagetable(domain, 0, end);
2526
2527         iommu_free_domain(domain);
2528         free_domain_mem(domain);
2529 }
2530 EXPORT_SYMBOL_GPL(intel_iommu_domain_exit);
2531
2532 struct dmar_domain *intel_iommu_domain_alloc(struct pci_dev *pdev)
2533 {
2534         struct dmar_drhd_unit *drhd;
2535         struct dmar_domain *domain;
2536         struct intel_iommu *iommu;
2537
2538         drhd = dmar_find_matched_drhd_unit(pdev);
2539         if (!drhd) {
2540                 printk(KERN_ERR "intel_iommu_domain_alloc: drhd == NULL\n");
2541                 return NULL;
2542         }
2543
2544         iommu = drhd->iommu;
2545         if (!iommu) {
2546                 printk(KERN_ERR
2547                         "intel_iommu_domain_alloc: iommu == NULL\n");
2548                 return NULL;
2549         }
2550         domain = iommu_alloc_domain(iommu);
2551         if (!domain) {
2552                 printk(KERN_ERR
2553                         "intel_iommu_domain_alloc: domain == NULL\n");
2554                 return NULL;
2555         }
2556         if (domain_init(domain, DEFAULT_DOMAIN_ADDRESS_WIDTH)) {
2557                 printk(KERN_ERR
2558                         "intel_iommu_domain_alloc: domain_init() failed\n");
2559                 intel_iommu_domain_exit(domain);
2560                 return NULL;
2561         }
2562         return domain;
2563 }
2564 EXPORT_SYMBOL_GPL(intel_iommu_domain_alloc);
2565
2566 int intel_iommu_context_mapping(
2567         struct dmar_domain *domain, struct pci_dev *pdev)
2568 {
2569         int rc;
2570         rc = domain_context_mapping(domain, pdev);
2571         return rc;
2572 }
2573 EXPORT_SYMBOL_GPL(intel_iommu_context_mapping);
2574
2575 int intel_iommu_page_mapping(
2576         struct dmar_domain *domain, dma_addr_t iova,
2577         u64 hpa, size_t size, int prot)
2578 {
2579         int rc;
2580         rc = domain_page_mapping(domain, iova, hpa, size, prot);
2581         return rc;
2582 }
2583 EXPORT_SYMBOL_GPL(intel_iommu_page_mapping);
2584
2585 void intel_iommu_detach_dev(struct dmar_domain *domain, u8 bus, u8 devfn)
2586 {
2587         detach_domain_for_dev(domain, bus, devfn);
2588 }
2589 EXPORT_SYMBOL_GPL(intel_iommu_detach_dev);
2590
2591 struct dmar_domain *
2592 intel_iommu_find_domain(struct pci_dev *pdev)
2593 {
2594         return find_domain(pdev);
2595 }
2596 EXPORT_SYMBOL_GPL(intel_iommu_find_domain);
2597
2598 int intel_iommu_found(void)
2599 {
2600         return g_num_of_iommus;
2601 }
2602 EXPORT_SYMBOL_GPL(intel_iommu_found);
2603
2604 u64 intel_iommu_iova_to_pfn(struct dmar_domain *domain, u64 iova)
2605 {
2606         struct dma_pte *pte;
2607         u64 pfn;
2608
2609         pfn = 0;
2610         pte = addr_to_dma_pte(domain, iova);
2611
2612         if (pte)
2613                 pfn = dma_pte_addr(pte);
2614
2615         return pfn >> VTD_PAGE_SHIFT;
2616 }
2617 EXPORT_SYMBOL_GPL(intel_iommu_iova_to_pfn);