arch/x86/kvm/x86.c

   1 /*
   2  * Kernel-based Virtual Machine driver for Linux
   3  *
   4  * derived from drivers/kvm/kvm_main.c
   5  *
   6  * Copyright (C) 2006 Qumranet, Inc.
   7  *
   8  * Authors:
   9  *   Avi Kivity   <avi@qumranet.com>
  10  *   Yaniv Kamay  <yaniv@qumranet.com>
  11  *
  12  * This work is licensed under the terms of the GNU GPL, version 2.  See
  13  * the COPYING file in the top-level directory.
  14  *
  15  */
  16
  17 #include <linux/kvm_host.h>
  18 #include "irq.h"
  19 #include "mmu.h"
  20 #include "i8254.h"
  21 #include "tss.h"
  22 #include "kvm_cache_regs.h"
  23
  24 #include <linux/clocksource.h>
  25 #include <linux/kvm.h>
  26 #include <linux/fs.h>
  27 #include <linux/vmalloc.h>
  28 #include <linux/module.h>
  29 #include <linux/mman.h>
  30 #include <linux/highmem.h>
  31
  32 #include <asm/uaccess.h>
  33 #include <asm/msr.h>
  34 #include <asm/desc.h>
  35
  36 #define MAX_IO_MSRS 256
  37 #define CR0_RESERVED_BITS                                               \
  38         (~(unsigned long)(X86_CR0_PE | X86_CR0_MP | X86_CR0_EM | X86_CR0_TS \
  39                           | X86_CR0_ET | X86_CR0_NE | X86_CR0_WP | X86_CR0_AM \
  40                           | X86_CR0_NW | X86_CR0_CD | X86_CR0_PG))
  41 #define CR4_RESERVED_BITS                                               \
  42         (~(unsigned long)(X86_CR4_VME | X86_CR4_PVI | X86_CR4_TSD | X86_CR4_DE\
  43                           | X86_CR4_PSE | X86_CR4_PAE | X86_CR4_MCE     \
  44                           | X86_CR4_PGE | X86_CR4_PCE | X86_CR4_OSFXSR  \
  45                           | X86_CR4_OSXMMEXCPT | X86_CR4_VMXE))
  46
  47 #define CR8_RESERVED_BITS (~(unsigned long)X86_CR8_TPR)
  48 /* EFER defaults:
  49  * - enable syscall per default because its emulated by KVM
  50  * - enable LME and LMA per default on 64 bit KVM
  51  */
  52 #ifdef CONFIG_X86_64
  53 static u64 __read_mostly efer_reserved_bits = 0xfffffffffffffafeULL;
  54 #else
  55 static u64 __read_mostly efer_reserved_bits = 0xfffffffffffffffeULL;
  56 #endif
  57
  58 #define VM_STAT(x) offsetof(struct kvm, stat.x), KVM_STAT_VM
  59 #define VCPU_STAT(x) offsetof(struct kvm_vcpu, stat.x), KVM_STAT_VCPU
  60
  61 static int kvm_dev_ioctl_get_supported_cpuid(struct kvm_cpuid2 *cpuid,
  62                                     struct kvm_cpuid_entry2 __user *entries);
  63
  64 struct kvm_x86_ops *kvm_x86_ops;
  65 EXPORT_SYMBOL_GPL(kvm_x86_ops);
  66
  67 struct kvm_stats_debugfs_item debugfs_entries[] = {
  68         { "pf_fixed", VCPU_STAT(pf_fixed) },
  69         { "pf_guest", VCPU_STAT(pf_guest) },
  70         { "tlb_flush", VCPU_STAT(tlb_flush) },
  71         { "invlpg", VCPU_STAT(invlpg) },
  72         { "exits", VCPU_STAT(exits) },
  73         { "io_exits", VCPU_STAT(io_exits) },
  74         { "mmio_exits", VCPU_STAT(mmio_exits) },
  75         { "signal_exits", VCPU_STAT(signal_exits) },
  76         { "irq_window", VCPU_STAT(irq_window_exits) },
  77         { "nmi_window", VCPU_STAT(nmi_window_exits) },
  78         { "halt_exits", VCPU_STAT(halt_exits) },
  79         { "halt_wakeup", VCPU_STAT(halt_wakeup) },
  80         { "hypercalls", VCPU_STAT(hypercalls) },
  81         { "request_irq", VCPU_STAT(request_irq_exits) },
  82         { "irq_exits", VCPU_STAT(irq_exits) },
  83         { "host_state_reload", VCPU_STAT(host_state_reload) },
  84         { "efer_reload", VCPU_STAT(efer_reload) },
  85         { "fpu_reload", VCPU_STAT(fpu_reload) },
  86         { "insn_emulation", VCPU_STAT(insn_emulation) },
  87         { "insn_emulation_fail", VCPU_STAT(insn_emulation_fail) },
  88         { "mmu_shadow_zapped", VM_STAT(mmu_shadow_zapped) },
  89         { "mmu_pte_write", VM_STAT(mmu_pte_write) },
  90         { "mmu_pte_updated", VM_STAT(mmu_pte_updated) },
  91         { "mmu_pde_zapped", VM_STAT(mmu_pde_zapped) },
  92         { "mmu_flooded", VM_STAT(mmu_flooded) },
  93         { "mmu_recycled", VM_STAT(mmu_recycled) },
  94         { "mmu_cache_miss", VM_STAT(mmu_cache_miss) },
  95         { "remote_tlb_flush", VM_STAT(remote_tlb_flush) },
  96         { "largepages", VM_STAT(lpages) },
  97         { NULL }
  98 };
  99
 100
 101 unsigned long segment_base(u16 selector)
 102 {
 103         struct descriptor_table gdt;
 104         struct desc_struct *d;
 105         unsigned long table_base;
 106         unsigned long v;
 107
 108         if (selector == 0)
 109                 return 0;
 110
 111         asm("sgdt %0" : "=m"(gdt));
 112         table_base = gdt.base;
 113
 114         if (selector & 4) {           /* from ldt */
 115                 u16 ldt_selector;
 116
 117                 asm("sldt %0" : "=g"(ldt_selector));
 118                 table_base = segment_base(ldt_selector);
 119         }
 120         d = (struct desc_struct *)(table_base + (selector & ~7));
 121         v = d->base0 | ((unsigned long)d->base1 << 16) |
 122                 ((unsigned long)d->base2 << 24);
 123 #ifdef CONFIG_X86_64
 124         if (d->s == 0 && (d->type == 2 || d->type == 9 || d->type == 11))
 125                 v |= ((unsigned long)((struct ldttss_desc64 *)d)->base3) << 32;
 126 #endif
 127         return v;
 128 }
 129 EXPORT_SYMBOL_GPL(segment_base);
 130
 131 u64 kvm_get_apic_base(struct kvm_vcpu *vcpu)
 132 {
 133         if (irqchip_in_kernel(vcpu->kvm))
 134                 return vcpu->arch.apic_base;
 135         else
 136                 return vcpu->arch.apic_base;
 137 }
 138 EXPORT_SYMBOL_GPL(kvm_get_apic_base);
 139
 140 void kvm_set_apic_base(struct kvm_vcpu *vcpu, u64 data)
 141 {
 142         /* TODO: reserve bits check */
 143         if (irqchip_in_kernel(vcpu->kvm))
 144                 kvm_lapic_set_base(vcpu, data);
 145         else
 146                 vcpu->arch.apic_base = data;
 147 }
 148 EXPORT_SYMBOL_GPL(kvm_set_apic_base);
 149
 150 void kvm_queue_exception(struct kvm_vcpu *vcpu, unsigned nr)
 151 {
 152         WARN_ON(vcpu->arch.exception.pending);
 153         vcpu->arch.exception.pending = true;
 154         vcpu->arch.exception.has_error_code = false;
 155         vcpu->arch.exception.nr = nr;
 156 }
 157 EXPORT_SYMBOL_GPL(kvm_queue_exception);
 158
 159 void kvm_inject_page_fault(struct kvm_vcpu *vcpu, unsigned long addr,
 160                            u32 error_code)
 161 {
 162         ++vcpu->stat.pf_guest;
 163         if (vcpu->arch.exception.pending) {
 164                 if (vcpu->arch.exception.nr == PF_VECTOR) {
 165                         printk(KERN_DEBUG "kvm: inject_page_fault:"
 166                                         " double fault 0x%lx\n", addr);
 167                         vcpu->arch.exception.nr = DF_VECTOR;
 168                         vcpu->arch.exception.error_code = 0;
 169                 } else if (vcpu->arch.exception.nr == DF_VECTOR) {
 170                         /* triple fault -> shutdown */
 171                         set_bit(KVM_REQ_TRIPLE_FAULT, &vcpu->requests);
 172                 }
 173                 return;
 174         }
 175         vcpu->arch.cr2 = addr;
 176         kvm_queue_exception_e(vcpu, PF_VECTOR, error_code);
 177 }
 178
 179 void kvm_inject_nmi(struct kvm_vcpu *vcpu)
 180 {
 181         vcpu->arch.nmi_pending = 1;
 182 }
 183 EXPORT_SYMBOL_GPL(kvm_inject_nmi);
 184
 185 void kvm_queue_exception_e(struct kvm_vcpu *vcpu, unsigned nr, u32 error_code)
 186 {
 187         WARN_ON(vcpu->arch.exception.pending);
 188         vcpu->arch.exception.pending = true;
 189         vcpu->arch.exception.has_error_code = true;
 190         vcpu->arch.exception.nr = nr;
 191         vcpu->arch.exception.error_code = error_code;
 192 }
 193 EXPORT_SYMBOL_GPL(kvm_queue_exception_e);
 194
 195 static void __queue_exception(struct kvm_vcpu *vcpu)
 196 {
 197         kvm_x86_ops->queue_exception(vcpu, vcpu->arch.exception.nr,
 198                                      vcpu->arch.exception.has_error_code,
 199                                      vcpu->arch.exception.error_code);
 200 }
 201
 202 /*
 203  * Load the pae pdptrs.  Return true is they are all valid.
 204  */
 205 int load_pdptrs(struct kvm_vcpu *vcpu, unsigned long cr3)
 206 {
 207         gfn_t pdpt_gfn = cr3 >> PAGE_SHIFT;
 208         unsigned offset = ((cr3 & (PAGE_SIZE-1)) >> 5) << 2;
 209         int i;
 210         int ret;
 211         u64 pdpte[ARRAY_SIZE(vcpu->arch.pdptrs)];
 212
 213         ret = kvm_read_guest_page(vcpu->kvm, pdpt_gfn, pdpte,
 214                                   offset * sizeof(u64), sizeof(pdpte));
 215         if (ret < 0) {
 216                 ret = 0;
 217                 goto out;
 218         }
 219         for (i = 0; i < ARRAY_SIZE(pdpte); ++i) {
 220                 if ((pdpte[i] & 1) && (pdpte[i] & 0xfffffff0000001e6ull)) {
 221                         ret = 0;
 222                         goto out;
 223                 }
 224         }
 225         ret = 1;
 226
 227         memcpy(vcpu->arch.pdptrs, pdpte, sizeof(vcpu->arch.pdptrs));
 228 out:
 229
 230         return ret;
 231 }
 232 EXPORT_SYMBOL_GPL(load_pdptrs);
 233
 234 static bool pdptrs_changed(struct kvm_vcpu *vcpu)
 235 {
 236         u64 pdpte[ARRAY_SIZE(vcpu->arch.pdptrs)];
 237         bool changed = true;
 238         int r;
 239
 240         if (is_long_mode(vcpu) || !is_pae(vcpu))
 241                 return false;
 242
 243         r = kvm_read_guest(vcpu->kvm, vcpu->arch.cr3 & ~31u, pdpte, sizeof(pdpte));
 244         if (r < 0)
 245                 goto out;
 246         changed = memcmp(pdpte, vcpu->arch.pdptrs, sizeof(pdpte)) != 0;
 247 out:
 248
 249         return changed;
 250 }
 251
 252 void kvm_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0)
 253 {
 254         if (cr0 & CR0_RESERVED_BITS) {
 255                 printk(KERN_DEBUG "set_cr0: 0x%lx #GP, reserved bits 0x%lx\n",
 256                        cr0, vcpu->arch.cr0);
 257                 kvm_inject_gp(vcpu, 0);
 258                 return;
 259         }
 260
 261         if ((cr0 & X86_CR0_NW) && !(cr0 & X86_CR0_CD)) {
 262                 printk(KERN_DEBUG "set_cr0: #GP, CD == 0 && NW == 1\n");
 263                 kvm_inject_gp(vcpu, 0);
 264                 return;
 265         }
 266
 267         if ((cr0 & X86_CR0_PG) && !(cr0 & X86_CR0_PE)) {
 268                 printk(KERN_DEBUG "set_cr0: #GP, set PG flag "
 269                        "and a clear PE flag\n");
 270                 kvm_inject_gp(vcpu, 0);
 271                 return;
 272         }
 273
 274         if (!is_paging(vcpu) && (cr0 & X86_CR0_PG)) {
 275 #ifdef CONFIG_X86_64
 276                 if ((vcpu->arch.shadow_efer & EFER_LME)) {
 277                         int cs_db, cs_l;
 278
 279                         if (!is_pae(vcpu)) {
 280                                 printk(KERN_DEBUG "set_cr0: #GP, start paging "
 281                                        "in long mode while PAE is disabled\n");
 282                                 kvm_inject_gp(vcpu, 0);
 283                                 return;
 284                         }
 285                         kvm_x86_ops->get_cs_db_l_bits(vcpu, &cs_db, &cs_l);
 286                         if (cs_l) {
 287                                 printk(KERN_DEBUG "set_cr0: #GP, start paging "
 288                                        "in long mode while CS.L == 1\n");
 289                                 kvm_inject_gp(vcpu, 0);
 290                                 return;
 291
 292                         }
 293                 } else
 294 #endif
 295                 if (is_pae(vcpu) && !load_pdptrs(vcpu, vcpu->arch.cr3)) {
 296                         printk(KERN_DEBUG "set_cr0: #GP, pdptrs "
 297                                "reserved bits\n");
 298                         kvm_inject_gp(vcpu, 0);
 299                         return;
 300                 }
 301
 302         }
 303
 304         kvm_x86_ops->set_cr0(vcpu, cr0);
 305         vcpu->arch.cr0 = cr0;
 306
 307         kvm_mmu_reset_context(vcpu);
 308         return;
 309 }
 310 EXPORT_SYMBOL_GPL(kvm_set_cr0);
 311
 312 void kvm_lmsw(struct kvm_vcpu *vcpu, unsigned long msw)
 313 {
 314         kvm_set_cr0(vcpu, (vcpu->arch.cr0 & ~0x0ful) | (msw & 0x0f));
 315         KVMTRACE_1D(LMSW, vcpu,
 316                     (u32)((vcpu->arch.cr0 & ~0x0ful) | (msw & 0x0f)),
 317                     handler);
 318 }
 319 EXPORT_SYMBOL_GPL(kvm_lmsw);
 320
 321 void kvm_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4)
 322 {
 323         if (cr4 & CR4_RESERVED_BITS) {
 324                 printk(KERN_DEBUG "set_cr4: #GP, reserved bits\n");
 325                 kvm_inject_gp(vcpu, 0);
 326                 return;
 327         }
 328
 329         if (is_long_mode(vcpu)) {
 330                 if (!(cr4 & X86_CR4_PAE)) {
 331                         printk(KERN_DEBUG "set_cr4: #GP, clearing PAE while "
 332                                "in long mode\n");
 333                         kvm_inject_gp(vcpu, 0);
 334                         return;
 335                 }
 336         } else if (is_paging(vcpu) && !is_pae(vcpu) && (cr4 & X86_CR4_PAE)
 337                    && !load_pdptrs(vcpu, vcpu->arch.cr3)) {
 338                 printk(KERN_DEBUG "set_cr4: #GP, pdptrs reserved bits\n");
 339                 kvm_inject_gp(vcpu, 0);
 340                 return;
 341         }
 342
 343         if (cr4 & X86_CR4_VMXE) {
 344                 printk(KERN_DEBUG "set_cr4: #GP, setting VMXE\n");
 345                 kvm_inject_gp(vcpu, 0);
 346                 return;
 347         }
 348         kvm_x86_ops->set_cr4(vcpu, cr4);
 349         vcpu->arch.cr4 = cr4;
 350         kvm_mmu_reset_context(vcpu);
 351 }
 352 EXPORT_SYMBOL_GPL(kvm_set_cr4);
 353
 354 void kvm_set_cr3(struct kvm_vcpu *vcpu, unsigned long cr3)
 355 {
 356         if (cr3 == vcpu->arch.cr3 && !pdptrs_changed(vcpu)) {
 357                 kvm_mmu_flush_tlb(vcpu);
 358                 return;
 359         }
 360
 361         if (is_long_mode(vcpu)) {
 362                 if (cr3 & CR3_L_MODE_RESERVED_BITS) {
 363                         printk(KERN_DEBUG "set_cr3: #GP, reserved bits\n");
 364                         kvm_inject_gp(vcpu, 0);
 365                         return;
 366                 }
 367         } else {
 368                 if (is_pae(vcpu)) {
 369                         if (cr3 & CR3_PAE_RESERVED_BITS) {
 370                                 printk(KERN_DEBUG
 371                                        "set_cr3: #GP, reserved bits\n");
 372                                 kvm_inject_gp(vcpu, 0);
 373                                 return;
 374                         }
 375                         if (is_paging(vcpu) && !load_pdptrs(vcpu, cr3)) {
 376                                 printk(KERN_DEBUG "set_cr3: #GP, pdptrs "
 377                                        "reserved bits\n");
 378                                 kvm_inject_gp(vcpu, 0);
 379                                 return;
 380                         }
 381                 }
 382                 /*
 383                  * We don't check reserved bits in nonpae mode, because
 384                  * this isn't enforced, and VMware depends on this.
 385                  */
 386         }
 387
 388         /*
 389          * Does the new cr3 value map to physical memory? (Note, we
 390          * catch an invalid cr3 even in real-mode, because it would
 391          * cause trouble later on when we turn on paging anyway.)
 392          *
 393          * A real CPU would silently accept an invalid cr3 and would
 394          * attempt to use it - with largely undefined (and often hard
 395          * to debug) behavior on the guest side.
 396          */
 397         if (unlikely(!gfn_to_memslot(vcpu->kvm, cr3 >> PAGE_SHIFT)))
 398                 kvm_inject_gp(vcpu, 0);
 399         else {
 400                 vcpu->arch.cr3 = cr3;
 401                 vcpu->arch.mmu.new_cr3(vcpu);
 402         }
 403 }
 404 EXPORT_SYMBOL_GPL(kvm_set_cr3);
 405
 406 void kvm_set_cr8(struct kvm_vcpu *vcpu, unsigned long cr8)
 407 {
 408         if (cr8 & CR8_RESERVED_BITS) {
 409                 printk(KERN_DEBUG "set_cr8: #GP, reserved bits 0x%lx\n", cr8);
 410                 kvm_inject_gp(vcpu, 0);
 411                 return;
 412         }
 413         if (irqchip_in_kernel(vcpu->kvm))
 414                 kvm_lapic_set_tpr(vcpu, cr8);
 415         else
 416                 vcpu->arch.cr8 = cr8;
 417 }
 418 EXPORT_SYMBOL_GPL(kvm_set_cr8);
 419
 420 unsigned long kvm_get_cr8(struct kvm_vcpu *vcpu)
 421 {
 422         if (irqchip_in_kernel(vcpu->kvm))
 423                 return kvm_lapic_get_cr8(vcpu);
 424         else
 425                 return vcpu->arch.cr8;
 426 }
 427 EXPORT_SYMBOL_GPL(kvm_get_cr8);
 428
 429 /*
 430  * List of msr numbers which we expose to userspace through KVM_GET_MSRS
 431  * and KVM_SET_MSRS, and KVM_GET_MSR_INDEX_LIST.
 432  *
 433  * This list is modified at module load time to reflect the
 434  * capabilities of the host cpu.
 435  */
 436 static u32 msrs_to_save[] = {
 437         MSR_IA32_SYSENTER_CS, MSR_IA32_SYSENTER_ESP, MSR_IA32_SYSENTER_EIP,
 438         MSR_K6_STAR,
 439 #ifdef CONFIG_X86_64
 440         MSR_CSTAR, MSR_KERNEL_GS_BASE, MSR_SYSCALL_MASK, MSR_LSTAR,
 441 #endif
 442         MSR_IA32_TIME_STAMP_COUNTER, MSR_KVM_SYSTEM_TIME, MSR_KVM_WALL_CLOCK,
 443         MSR_IA32_PERF_STATUS,
 444 };
 445
 446 static unsigned num_msrs_to_save;
 447
 448 static u32 emulated_msrs[] = {
 449         MSR_IA32_MISC_ENABLE,
 450 };
 451
 452 static void set_efer(struct kvm_vcpu *vcpu, u64 efer)
 453 {
 454         if (efer & efer_reserved_bits) {
 455                 printk(KERN_DEBUG "set_efer: 0x%llx #GP, reserved bits\n",
 456                        efer);
 457                 kvm_inject_gp(vcpu, 0);
 458                 return;
 459         }
 460
 461         if (is_paging(vcpu)
 462             && (vcpu->arch.shadow_efer & EFER_LME) != (efer & EFER_LME)) {
 463                 printk(KERN_DEBUG "set_efer: #GP, change LME while paging\n");
 464                 kvm_inject_gp(vcpu, 0);
 465                 return;
 466         }
 467
 468         kvm_x86_ops->set_efer(vcpu, efer);
 469
 470         efer &= ~EFER_LMA;
 471         efer |= vcpu->arch.shadow_efer & EFER_LMA;
 472
 473         vcpu->arch.shadow_efer = efer;
 474 }
 475
 476 void kvm_enable_efer_bits(u64 mask)
 477 {
 478        efer_reserved_bits &= ~mask;
 479 }
 480 EXPORT_SYMBOL_GPL(kvm_enable_efer_bits);
 481
 482
 483 /*
 484  * Writes msr value into into the appropriate "register".
 485  * Returns 0 on success, non-0 otherwise.
 486  * Assumes vcpu_load() was already called.
 487  */
 488 int kvm_set_msr(struct kvm_vcpu *vcpu, u32 msr_index, u64 data)
 489 {
 490         return kvm_x86_ops->set_msr(vcpu, msr_index, data);
 491 }
 492
 493 /*
 494  * Adapt set_msr() to msr_io()'s calling convention
 495  */
 496 static int do_set_msr(struct kvm_vcpu *vcpu, unsigned index, u64 *data)
 497 {
 498         return kvm_set_msr(vcpu, index, *data);
 499 }
 500
 501 static void kvm_write_wall_clock(struct kvm *kvm, gpa_t wall_clock)
 502 {
 503         static int version;
 504         struct pvclock_wall_clock wc;
 505         struct timespec now, sys, boot;
 506
 507         if (!wall_clock)
 508                 return;
 509
 510         version++;
 511
 512         kvm_write_guest(kvm, wall_clock, &version, sizeof(version));
 513
 514         /*
 515          * The guest calculates current wall clock time by adding
 516          * system time (updated by kvm_write_guest_time below) to the
 517          * wall clock specified here.  guest system time equals host
 518          * system time for us, thus we must fill in host boot time here.
 519          */
 520         now = current_kernel_time();
 521         ktime_get_ts(&sys);
 522         boot = ns_to_timespec(timespec_to_ns(&now) - timespec_to_ns(&sys));
 523
 524         wc.sec = boot.tv_sec;
 525         wc.nsec = boot.tv_nsec;
 526         wc.version = version;
 527
 528         kvm_write_guest(kvm, wall_clock, &wc, sizeof(wc));
 529
 530         version++;
 531         kvm_write_guest(kvm, wall_clock, &version, sizeof(version));
 532 }
 533
 534 static uint32_t div_frac(uint32_t dividend, uint32_t divisor)
 535 {
 536         uint32_t quotient, remainder;
 537
 538         /* Don't try to replace with do_div(), this one calculates
 539          * "(dividend << 32) / divisor" */
 540         __asm__ ( "divl %4"
 541                   : "=a" (quotient), "=d" (remainder)
 542                   : "0" (0), "1" (dividend), "r" (divisor) );
 543         return quotient;
 544 }
 545
 546 static void kvm_set_time_scale(uint32_t tsc_khz, struct pvclock_vcpu_time_info *hv_clock)
 547 {
 548         uint64_t nsecs = 1000000000LL;
 549         int32_t  shift = 0;
 550         uint64_t tps64;
 551         uint32_t tps32;
 552
 553         tps64 = tsc_khz * 1000LL;
 554         while (tps64 > nsecs*2) {
 555                 tps64 >>= 1;
 556                 shift--;
 557         }
 558
 559         tps32 = (uint32_t)tps64;
 560         while (tps32 <= (uint32_t)nsecs) {
 561                 tps32 <<= 1;
 562                 shift++;
 563         }
 564
 565         hv_clock->tsc_shift = shift;
 566         hv_clock->tsc_to_system_mul = div_frac(nsecs, tps32);
 567
 568         pr_debug("%s: tsc_khz %u, tsc_shift %d, tsc_mul %u\n",
 569                  __FUNCTION__, tsc_khz, hv_clock->tsc_shift,
 570                  hv_clock->tsc_to_system_mul);
 571 }
 572
 573 static void kvm_write_guest_time(struct kvm_vcpu *v)
 574 {
 575         struct timespec ts;
 576         unsigned long flags;
 577         struct kvm_vcpu_arch *vcpu = &v->arch;
 578         void *shared_kaddr;
 579
 580         if ((!vcpu->time_page))
 581                 return;
 582
 583         if (unlikely(vcpu->hv_clock_tsc_khz != tsc_khz)) {
 584                 kvm_set_time_scale(tsc_khz, &vcpu->hv_clock);
 585                 vcpu->hv_clock_tsc_khz = tsc_khz;
 586         }
 587
 588         /* Keep irq disabled to prevent changes to the clock */
 589         local_irq_save(flags);
 590         kvm_get_msr(v, MSR_IA32_TIME_STAMP_COUNTER,
 591                           &vcpu->hv_clock.tsc_timestamp);
 592         ktime_get_ts(&ts);
 593         local_irq_restore(flags);
 594
 595         /* With all the info we got, fill in the values */
 596
 597         vcpu->hv_clock.system_time = ts.tv_nsec +
 598                                      (NSEC_PER_SEC * (u64)ts.tv_sec);
 599         /*
 600          * The interface expects us to write an even number signaling that the
 601          * update is finished. Since the guest won't see the intermediate
 602          * state, we just increase by 2 at the end.
 603          */
 604         vcpu->hv_clock.version += 2;
 605
 606         shared_kaddr = kmap_atomic(vcpu->time_page, KM_USER0);
 607
 608         memcpy(shared_kaddr + vcpu->time_offset, &vcpu->hv_clock,
 609                sizeof(vcpu->hv_clock));
 610
 611         kunmap_atomic(shared_kaddr, KM_USER0);
 612
 613         mark_page_dirty(v->kvm, vcpu->time >> PAGE_SHIFT);
 614 }
 615
 616 static bool msr_mtrr_valid(unsigned msr)
 617 {
 618         switch (msr) {
 619         case 0x200 ... 0x200 + 2 * KVM_NR_VAR_MTRR - 1:
 620         case MSR_MTRRfix64K_00000:
 621         case MSR_MTRRfix16K_80000:
 622         case MSR_MTRRfix16K_A0000:
 623         case MSR_MTRRfix4K_C0000:
 624         case MSR_MTRRfix4K_C8000:
 625         case MSR_MTRRfix4K_D0000:
 626         case MSR_MTRRfix4K_D8000:
 627         case MSR_MTRRfix4K_E0000:
 628         case MSR_MTRRfix4K_E8000:
 629         case MSR_MTRRfix4K_F0000:
 630         case MSR_MTRRfix4K_F8000:
 631         case MSR_MTRRdefType:
 632         case MSR_IA32_CR_PAT:
 633                 return true;
 634         case 0x2f8:
 635                 return true;
 636         }
 637         return false;
 638 }
 639
 640 static int set_msr_mtrr(struct kvm_vcpu *vcpu, u32 msr, u64 data)
 641 {
 642         if (!msr_mtrr_valid(msr))
 643                 return 1;
 644
 645         vcpu->arch.mtrr[msr - 0x200] = data;
 646         return 0;
 647 }
 648
 649 int kvm_set_msr_common(struct kvm_vcpu *vcpu, u32 msr, u64 data)
 650 {
 651         switch (msr) {
 652         case MSR_EFER:
 653                 set_efer(vcpu, data);
 654                 break;
 655         case MSR_IA32_MC0_STATUS:
 656                 pr_unimpl(vcpu, "%s: MSR_IA32_MC0_STATUS 0x%llx, nop\n",
 657                        __func__, data);
 658                 break;
 659         case MSR_IA32_MCG_STATUS:
 660                 pr_unimpl(vcpu, "%s: MSR_IA32_MCG_STATUS 0x%llx, nop\n",
 661                         __func__, data);
 662                 break;
 663         case MSR_IA32_MCG_CTL:
 664                 pr_unimpl(vcpu, "%s: MSR_IA32_MCG_CTL 0x%llx, nop\n",
 665                         __func__, data);
 666                 break;
 667         case MSR_IA32_UCODE_REV:
 668         case MSR_IA32_UCODE_WRITE:
 669                 break;
 670         case 0x200 ... 0x2ff:
 671                 return set_msr_mtrr(vcpu, msr, data);
 672         case MSR_IA32_APICBASE:
 673                 kvm_set_apic_base(vcpu, data);
 674                 break;
 675         case MSR_IA32_MISC_ENABLE:
 676                 vcpu->arch.ia32_misc_enable_msr = data;
 677                 break;
 678         case MSR_KVM_WALL_CLOCK:
 679                 vcpu->kvm->arch.wall_clock = data;
 680                 kvm_write_wall_clock(vcpu->kvm, data);
 681                 break;
 682         case MSR_KVM_SYSTEM_TIME: {
 683                 if (vcpu->arch.time_page) {
 684                         kvm_release_page_dirty(vcpu->arch.time_page);
 685                         vcpu->arch.time_page = NULL;
 686                 }
 687
 688                 vcpu->arch.time = data;
 689
 690                 /* we verify if the enable bit is set... */
 691                 if (!(data & 1))
 692                         break;
 693
 694                 /* ...but clean it before doing the actual write */
 695                 vcpu->arch.time_offset = data & ~(PAGE_MASK | 1);
 696
 697                 down_read(&current->mm->mmap_sem);
 698                 vcpu->arch.time_page =
 699                                 gfn_to_page(vcpu->kvm, data >> PAGE_SHIFT);
 700                 up_read(&current->mm->mmap_sem);
 701
 702                 if (is_error_page(vcpu->arch.time_page)) {
 703                         kvm_release_page_clean(vcpu->arch.time_page);
 704                         vcpu->arch.time_page = NULL;
 705                 }
 706
 707                 kvm_write_guest_time(vcpu);
 708                 break;
 709         }
 710         default:
 711                 pr_unimpl(vcpu, "unhandled wrmsr: 0x%x data %llx\n", msr, data);
 712                 return 1;
 713         }
 714         return 0;
 715 }
 716 EXPORT_SYMBOL_GPL(kvm_set_msr_common);
 717
 718
 719 /*
 720  * Reads an msr value (of 'msr_index') into 'pdata'.
 721  * Returns 0 on success, non-0 otherwise.
 722  * Assumes vcpu_load() was already called.
 723  */
 724 int kvm_get_msr(struct kvm_vcpu *vcpu, u32 msr_index, u64 *pdata)
 725 {
 726         return kvm_x86_ops->get_msr(vcpu, msr_index, pdata);
 727 }
 728
 729 static int get_msr_mtrr(struct kvm_vcpu *vcpu, u32 msr, u64 *pdata)
 730 {
 731         if (!msr_mtrr_valid(msr))
 732                 return 1;
 733
 734         *pdata = vcpu->arch.mtrr[msr - 0x200];
 735         return 0;
 736 }
 737
 738 int kvm_get_msr_common(struct kvm_vcpu *vcpu, u32 msr, u64 *pdata)
 739 {
 740         u64 data;
 741
 742         switch (msr) {
 743         case 0xc0010010: /* SYSCFG */
 744         case 0xc0010015: /* HWCR */
 745         case MSR_IA32_PLATFORM_ID:
 746         case MSR_IA32_P5_MC_ADDR:
 747         case MSR_IA32_P5_MC_TYPE:
 748         case MSR_IA32_MC0_CTL:
 749         case MSR_IA32_MCG_STATUS:
 750         case MSR_IA32_MCG_CAP:
 751         case MSR_IA32_MCG_CTL:
 752         case MSR_IA32_MC0_MISC:
 753         case MSR_IA32_MC0_MISC+4:
 754         case MSR_IA32_MC0_MISC+8:
 755         case MSR_IA32_MC0_MISC+12:
 756         case MSR_IA32_MC0_MISC+16:
 757         case MSR_IA32_UCODE_REV:
 758         case MSR_IA32_EBL_CR_POWERON:
 759                 data = 0;
 760                 break;
 761         case MSR_MTRRcap:
 762                 data = 0x500 | KVM_NR_VAR_MTRR;
 763                 break;
 764         case 0x200 ... 0x2ff:
 765                 return get_msr_mtrr(vcpu, msr, pdata);
 766         case 0xcd: /* fsb frequency */
 767                 data = 3;
 768                 break;
 769         case MSR_IA32_APICBASE:
 770                 data = kvm_get_apic_base(vcpu);
 771                 break;
 772         case MSR_IA32_MISC_ENABLE:
 773                 data = vcpu->arch.ia32_misc_enable_msr;
 774                 break;
 775         case MSR_IA32_PERF_STATUS:
 776                 /* TSC increment by tick */
 777                 data = 1000ULL;
 778                 /* CPU multiplier */
 779                 data |= (((uint64_t)4ULL) << 40);
 780                 break;
 781         case MSR_EFER:
 782                 data = vcpu->arch.shadow_efer;
 783                 break;
 784         case MSR_KVM_WALL_CLOCK:
 785                 data = vcpu->kvm->arch.wall_clock;
 786                 break;
 787         case MSR_KVM_SYSTEM_TIME:
 788                 data = vcpu->arch.time;
 789                 break;
 790         default:
 791                 pr_unimpl(vcpu, "unhandled rdmsr: 0x%x\n", msr);
 792                 return 1;
 793         }
 794         *pdata = data;
 795         return 0;
 796 }
 797 EXPORT_SYMBOL_GPL(kvm_get_msr_common);
 798
 799 /*
 800  * Read or write a bunch of msrs. All parameters are kernel addresses.
 801  *
 802  * @return number of msrs set successfully.
 803  */
 804 static int __msr_io(struct kvm_vcpu *vcpu, struct kvm_msrs *msrs,
 805                     struct kvm_msr_entry *entries,
 806                     int (*do_msr)(struct kvm_vcpu *vcpu,
 807                                   unsigned index, u64 *data))
 808 {
 809         int i;
 810
 811         vcpu_load(vcpu);
 812
 813         down_read(&vcpu->kvm->slots_lock);
 814         for (i = 0; i < msrs->nmsrs; ++i)
 815                 if (do_msr(vcpu, entries[i].index, &entries[i].data))
 816                         break;
 817         up_read(&vcpu->kvm->slots_lock);
 818
 819         vcpu_put(vcpu);
 820
 821         return i;
 822 }
 823
 824 /*
 825  * Read or write a bunch of msrs. Parameters are user addresses.
 826  *
 827  * @return number of msrs set successfully.
 828  */
 829 static int msr_io(struct kvm_vcpu *vcpu, struct kvm_msrs __user *user_msrs,
 830                   int (*do_msr)(struct kvm_vcpu *vcpu,
 831                                 unsigned index, u64 *data),
 832                   int writeback)
 833 {
 834         struct kvm_msrs msrs;
 835         struct kvm_msr_entry *entries;
 836         int r, n;
 837         unsigned size;
 838
 839         r = -EFAULT;
 840         if (copy_from_user(&msrs, user_msrs, sizeof msrs))
 841                 goto out;
 842
 843         r = -E2BIG;
 844         if (msrs.nmsrs >= MAX_IO_MSRS)
 845                 goto out;
 846
 847         r = -ENOMEM;
 848         size = sizeof(struct kvm_msr_entry) * msrs.nmsrs;
 849         entries = vmalloc(size);
 850         if (!entries)
 851                 goto out;
 852
 853         r = -EFAULT;
 854         if (copy_from_user(entries, user_msrs->entries, size))
 855                 goto out_free;
 856
 857         r = n = __msr_io(vcpu, &msrs, entries, do_msr);
 858         if (r < 0)
 859                 goto out_free;
 860
 861         r = -EFAULT;
 862         if (writeback && copy_to_user(user_msrs->entries, entries, size))
 863                 goto out_free;
 864
 865         r = n;
 866
 867 out_free:
 868         vfree(entries);
 869 out:
 870         return r;
 871 }
 872
 873 int kvm_dev_ioctl_check_extension(long ext)
 874 {
 875         int r;
 876
 877         switch (ext) {
 878         case KVM_CAP_IRQCHIP:
 879         case KVM_CAP_HLT:
 880         case KVM_CAP_MMU_SHADOW_CACHE_CONTROL:
 881         case KVM_CAP_USER_MEMORY:
 882         case KVM_CAP_SET_TSS_ADDR:
 883         case KVM_CAP_EXT_CPUID:
 884         case KVM_CAP_CLOCKSOURCE:
 885         case KVM_CAP_PIT:
 886         case KVM_CAP_NOP_IO_DELAY:
 887         case KVM_CAP_MP_STATE:
 888         case KVM_CAP_SYNC_MMU:
 889                 r = 1;
 890                 break;
 891         case KVM_CAP_COALESCED_MMIO:
 892                 r = KVM_COALESCED_MMIO_PAGE_OFFSET;
 893                 break;
 894         case KVM_CAP_VAPIC:
 895                 r = !kvm_x86_ops->cpu_has_accelerated_tpr();
 896                 break;
 897         case KVM_CAP_NR_VCPUS:
 898                 r = KVM_MAX_VCPUS;
 899                 break;
 900         case KVM_CAP_NR_MEMSLOTS:
 901                 r = KVM_MEMORY_SLOTS;
 902                 break;
 903         case KVM_CAP_PV_MMU:
 904                 r = !tdp_enabled;
 905                 break;
 906         default:
 907                 r = 0;
 908                 break;
 909         }
 910         return r;
 911
 912 }
 913
 914 long kvm_arch_dev_ioctl(struct file *filp,
 915                         unsigned int ioctl, unsigned long arg)
 916 {
 917         void __user *argp = (void __user *)arg;
 918         long r;
 919
 920         switch (ioctl) {
 921         case KVM_GET_MSR_INDEX_LIST: {
 922                 struct kvm_msr_list __user *user_msr_list = argp;
 923                 struct kvm_msr_list msr_list;
 924                 unsigned n;
 925
 926                 r = -EFAULT;
 927                 if (copy_from_user(&msr_list, user_msr_list, sizeof msr_list))
 928                         goto out;
 929                 n = msr_list.nmsrs;
 930                 msr_list.nmsrs = num_msrs_to_save + ARRAY_SIZE(emulated_msrs);
 931                 if (copy_to_user(user_msr_list, &msr_list, sizeof msr_list))
 932                         goto out;
 933                 r = -E2BIG;
 934                 if (n < num_msrs_to_save)
 935                         goto out;
 936                 r = -EFAULT;
 937                 if (copy_to_user(user_msr_list->indices, &msrs_to_save,
 938                                  num_msrs_to_save * sizeof(u32)))
 939                         goto out;
 940                 if (copy_to_user(user_msr_list->indices
 941                                  + num_msrs_to_save * sizeof(u32),
 942                                  &emulated_msrs,
 943                                  ARRAY_SIZE(emulated_msrs) * sizeof(u32)))
 944                         goto out;
 945                 r = 0;
 946                 break;
 947         }
 948         case KVM_GET_SUPPORTED_CPUID: {
 949                 struct kvm_cpuid2 __user *cpuid_arg = argp;
 950                 struct kvm_cpuid2 cpuid;
 951
 952                 r = -EFAULT;
 953                 if (copy_from_user(&cpuid, cpuid_arg, sizeof cpuid))
 954                         goto out;
 955                 r = kvm_dev_ioctl_get_supported_cpuid(&cpuid,
 956                         cpuid_arg->entries);
 957                 if (r)
 958                         goto out;
 959
 960                 r = -EFAULT;
 961                 if (copy_to_user(cpuid_arg, &cpuid, sizeof cpuid))
 962                         goto out;
 963                 r = 0;
 964                 break;
 965         }
 966         default:
 967                 r = -EINVAL;
 968         }
 969 out:
 970         return r;
 971 }
 972
 973 void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
 974 {
 975         kvm_x86_ops->vcpu_load(vcpu, cpu);
 976         kvm_write_guest_time(vcpu);
 977 }
 978
 979 void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu)
 980 {
 981         kvm_x86_ops->vcpu_put(vcpu);
 982         kvm_put_guest_fpu(vcpu);
 983 }
 984
 985 static int is_efer_nx(void)
 986 {
 987         u64 efer;
 988
 989         rdmsrl(MSR_EFER, efer);
 990         return efer & EFER_NX;
 991 }
 992
 993 static void cpuid_fix_nx_cap(struct kvm_vcpu *vcpu)
 994 {
 995         int i;
 996         struct kvm_cpuid_entry2 *e, *entry;
 997
 998         entry = NULL;
 999         for (i = 0; i < vcpu->arch.cpuid_nent; ++i) {
1000                 e = &vcpu->arch.cpuid_entries[i];
1001                 if (e->function == 0x80000001) {
1002                         entry = e;
1003                         break;
1004                 }
1005         }
1006         if (entry && (entry->edx & (1 << 20)) && !is_efer_nx()) {
1007                 entry->edx &= ~(1 << 20);
1008                 printk(KERN_INFO "kvm: guest NX capability removed\n");
1009         }
1010 }
1011
1012 /* when an old userspace process fills a new kernel module */
1013 static int kvm_vcpu_ioctl_set_cpuid(struct kvm_vcpu *vcpu,
1014                                     struct kvm_cpuid *cpuid,
1015                                     struct kvm_cpuid_entry __user *entries)
1016 {
1017         int r, i;
1018         struct kvm_cpuid_entry *cpuid_entries;
1019
1020         r = -E2BIG;
1021         if (cpuid->nent > KVM_MAX_CPUID_ENTRIES)
1022                 goto out;
1023         r = -ENOMEM;
1024         cpuid_entries = vmalloc(sizeof(struct kvm_cpuid_entry) * cpuid->nent);
1025         if (!cpuid_entries)
1026                 goto out;
1027         r = -EFAULT;
1028         if (copy_from_user(cpuid_entries, entries,
1029                            cpuid->nent * sizeof(struct kvm_cpuid_entry)))
1030                 goto out_free;
1031         for (i = 0; i < cpuid->nent; i++) {
1032                 vcpu->arch.cpuid_entries[i].function = cpuid_entries[i].function;
1033                 vcpu->arch.cpuid_entries[i].eax = cpuid_entries[i].eax;
1034                 vcpu->arch.cpuid_entries[i].ebx = cpuid_entries[i].ebx;
1035                 vcpu->arch.cpuid_entries[i].ecx = cpuid_entries[i].ecx;
1036                 vcpu->arch.cpuid_entries[i].edx = cpuid_entries[i].edx;
1037                 vcpu->arch.cpuid_entries[i].index = 0;
1038                 vcpu->arch.cpuid_entries[i].flags = 0;
1039                 vcpu->arch.cpuid_entries[i].padding[0] = 0;
1040                 vcpu->arch.cpuid_entries[i].padding[1] = 0;
1041                 vcpu->arch.cpuid_entries[i].padding[2] = 0;
1042         }
1043         vcpu->arch.cpuid_nent = cpuid->nent;
1044         cpuid_fix_nx_cap(vcpu);
1045         r = 0;
1046
1047 out_free:
1048         vfree(cpuid_entries);
1049 out:
1050         return r;
1051 }
1052
1053 static int kvm_vcpu_ioctl_set_cpuid2(struct kvm_vcpu *vcpu,
1054                                     struct kvm_cpuid2 *cpuid,
1055                                     struct kvm_cpuid_entry2 __user *entries)
1056 {
1057         int r;
1058
1059         r = -E2BIG;
1060         if (cpuid->nent > KVM_MAX_CPUID_ENTRIES)
1061                 goto out;
1062         r = -EFAULT;
1063         if (copy_from_user(&vcpu->arch.cpuid_entries, entries,
1064                            cpuid->nent * sizeof(struct kvm_cpuid_entry2)))
1065                 goto out;
1066         vcpu->arch.cpuid_nent = cpuid->nent;
1067         return 0;
1068
1069 out:
1070         return r;
1071 }
1072
1073 static int kvm_vcpu_ioctl_get_cpuid2(struct kvm_vcpu *vcpu,
1074                                     struct kvm_cpuid2 *cpuid,
1075                                     struct kvm_cpuid_entry2 __user *entries)
1076 {
1077         int r;
1078
1079         r = -E2BIG;
1080         if (cpuid->nent < vcpu->arch.cpuid_nent)
1081                 goto out;
1082         r = -EFAULT;
1083         if (copy_to_user(entries, &vcpu->arch.cpuid_entries,
1084                            vcpu->arch.cpuid_nent * sizeof(struct kvm_cpuid_entry2)))
1085                 goto out;
1086         return 0;
1087
1088 out:
1089         cpuid->nent = vcpu->arch.cpuid_nent;
1090         return r;
1091 }
1092
1093 static inline u32 bit(int bitno)
1094 {
1095         return 1 << (bitno & 31);
1096 }
1097
1098 static void do_cpuid_1_ent(struct kvm_cpuid_entry2 *entry, u32 function,
1099                           u32 index)
1100 {
1101         entry->function = function;
1102         entry->index = index;
1103         cpuid_count(entry->function, entry->index,
1104                 &entry->eax, &entry->ebx, &entry->ecx, &entry->edx);
1105         entry->flags = 0;
1106 }
1107
1108 static void do_cpuid_ent(struct kvm_cpuid_entry2 *entry, u32 function,
1109                          u32 index, int *nent, int maxnent)
1110 {
1111         const u32 kvm_supported_word0_x86_features = bit(X86_FEATURE_FPU) |
1112                 bit(X86_FEATURE_VME) | bit(X86_FEATURE_DE) |
1113                 bit(X86_FEATURE_PSE) | bit(X86_FEATURE_TSC) |
1114                 bit(X86_FEATURE_MSR) | bit(X86_FEATURE_PAE) |
1115                 bit(X86_FEATURE_CX8) | bit(X86_FEATURE_APIC) |
1116                 bit(X86_FEATURE_SEP) | bit(X86_FEATURE_PGE) |
1117                 bit(X86_FEATURE_CMOV) | bit(X86_FEATURE_PSE36) |
1118                 bit(X86_FEATURE_CLFLSH) | bit(X86_FEATURE_MMX) |
1119                 bit(X86_FEATURE_FXSR) | bit(X86_FEATURE_XMM) |
1120                 bit(X86_FEATURE_XMM2) | bit(X86_FEATURE_SELFSNOOP);
1121         const u32 kvm_supported_word1_x86_features = bit(X86_FEATURE_FPU) |
1122                 bit(X86_FEATURE_VME) | bit(X86_FEATURE_DE) |
1123                 bit(X86_FEATURE_PSE) | bit(X86_FEATURE_TSC) |
1124                 bit(X86_FEATURE_MSR) | bit(X86_FEATURE_PAE) |
1125                 bit(X86_FEATURE_CX8) | bit(X86_FEATURE_APIC) |
1126                 bit(X86_FEATURE_PGE) |
1127                 bit(X86_FEATURE_CMOV) | bit(X86_FEATURE_PSE36) |
1128                 bit(X86_FEATURE_MMX) | bit(X86_FEATURE_FXSR) |
1129                 bit(X86_FEATURE_SYSCALL) |
1130                 (bit(X86_FEATURE_NX) && is_efer_nx()) |
1131 #ifdef CONFIG_X86_64
1132                 bit(X86_FEATURE_LM) |
1133 #endif
1134                 bit(X86_FEATURE_MMXEXT) |
1135                 bit(X86_FEATURE_3DNOWEXT) |
1136                 bit(X86_FEATURE_3DNOW);
1137         const u32 kvm_supported_word3_x86_features =
1138                 bit(X86_FEATURE_XMM3) | bit(X86_FEATURE_CX16);
1139         const u32 kvm_supported_word6_x86_features =
1140                 bit(X86_FEATURE_LAHF_LM) | bit(X86_FEATURE_CMP_LEGACY);
1141
1142         /* all func 2 cpuid_count() should be called on the same cpu */
1143         get_cpu();
1144         do_cpuid_1_ent(entry, function, index);
1145         ++*nent;
1146
1147         switch (function) {
1148         case 0:
1149                 entry->eax = min(entry->eax, (u32)0xb);
1150                 break;
1151         case 1:
1152                 entry->edx &= kvm_supported_word0_x86_features;
1153                 entry->ecx &= kvm_supported_word3_x86_features;
1154                 break;
1155         /* function 2 entries are STATEFUL. That is, repeated cpuid commands
1156          * may return different values. This forces us to get_cpu() before
1157          * issuing the first command, and also to emulate this annoying behavior
1158          * in kvm_emulate_cpuid() using KVM_CPUID_FLAG_STATE_READ_NEXT */
1159         case 2: {
1160                 int t, times = entry->eax & 0xff;
1161
1162                 entry->flags |= KVM_CPUID_FLAG_STATEFUL_FUNC;
1163                 for (t = 1; t < times && *nent < maxnent; ++t) {
1164                         do_cpuid_1_ent(&entry[t], function, 0);
1165                         entry[t].flags |= KVM_CPUID_FLAG_STATEFUL_FUNC;
1166                         ++*nent;
1167                 }
1168                 break;
1169         }
1170         /* function 4 and 0xb have additional index. */
1171         case 4: {
1172                 int i, cache_type;
1173
1174                 entry->flags |= KVM_CPUID_FLAG_SIGNIFCANT_INDEX;
1175                 /* read more entries until cache_type is zero */
1176                 for (i = 1; *nent < maxnent; ++i) {
1177                         cache_type = entry[i - 1].eax & 0x1f;
1178                         if (!cache_type)
1179                                 break;
1180                         do_cpuid_1_ent(&entry[i], function, i);
1181                         entry[i].flags |=
1182                                KVM_CPUID_FLAG_SIGNIFCANT_INDEX;
1183                         ++*nent;
1184                 }
1185                 break;
1186         }
1187         case 0xb: {
1188                 int i, level_type;
1189
1190                 entry->flags |= KVM_CPUID_FLAG_SIGNIFCANT_INDEX;
1191                 /* read more entries until level_type is zero */
1192                 for (i = 1; *nent < maxnent; ++i) {
1193                         level_type = entry[i - 1].ecx & 0xff;
1194                         if (!level_type)
1195                                 break;
1196                         do_cpuid_1_ent(&entry[i], function, i);
1197                         entry[i].flags |=
1198                                KVM_CPUID_FLAG_SIGNIFCANT_INDEX;
1199                         ++*nent;
1200                 }
1201                 break;
1202         }
1203         case 0x80000000:
1204                 entry->eax = min(entry->eax, 0x8000001a);
1205                 break;
1206         case 0x80000001:
1207                 entry->edx &= kvm_supported_word1_x86_features;
1208                 entry->ecx &= kvm_supported_word6_x86_features;
1209                 break;
1210         }
1211         put_cpu();
1212 }
1213
1214 static int kvm_dev_ioctl_get_supported_cpuid(struct kvm_cpuid2 *cpuid,
1215                                     struct kvm_cpuid_entry2 __user *entries)
1216 {
1217         struct kvm_cpuid_entry2 *cpuid_entries;
1218         int limit, nent = 0, r = -E2BIG;
1219         u32 func;
1220
1221         if (cpuid->nent < 1)
1222                 goto out;
1223         r = -ENOMEM;
1224         cpuid_entries = vmalloc(sizeof(struct kvm_cpuid_entry2) * cpuid->nent);
1225         if (!cpuid_entries)
1226                 goto out;
1227
1228         do_cpuid_ent(&cpuid_entries[0], 0, 0, &nent, cpuid->nent);
1229         limit = cpuid_entries[0].eax;
1230         for (func = 1; func <= limit && nent < cpuid->nent; ++func)
1231                 do_cpuid_ent(&cpuid_entries[nent], func, 0,
1232                                 &nent, cpuid->nent);
1233         r = -E2BIG;
1234         if (nent >= cpuid->nent)
1235                 goto out_free;
1236
1237         do_cpuid_ent(&cpuid_entries[nent], 0x80000000, 0, &nent, cpuid->nent);
1238         limit = cpuid_entries[nent - 1].eax;
1239         for (func = 0x80000001; func <= limit && nent < cpuid->nent; ++func)
1240                 do_cpuid_ent(&cpuid_entries[nent], func, 0,
1241                                &nent, cpuid->nent);
1242         r = -EFAULT;
1243         if (copy_to_user(entries, cpuid_entries,
1244                         nent * sizeof(struct kvm_cpuid_entry2)))
1245                 goto out_free;
1246         cpuid->nent = nent;
1247         r = 0;
1248
1249 out_free:
1250         vfree(cpuid_entries);
1251 out:
1252         return r;
1253 }
1254
1255 static int kvm_vcpu_ioctl_get_lapic(struct kvm_vcpu *vcpu,
1256                                     struct kvm_lapic_state *s)
1257 {
1258         vcpu_load(vcpu);
1259         memcpy(s->regs, vcpu->arch.apic->regs, sizeof *s);
1260         vcpu_put(vcpu);
1261
1262         return 0;
1263 }
1264
1265 static int kvm_vcpu_ioctl_set_lapic(struct kvm_vcpu *vcpu,
1266                                     struct kvm_lapic_state *s)
1267 {
1268         vcpu_load(vcpu);
1269         memcpy(vcpu->arch.apic->regs, s->regs, sizeof *s);
1270         kvm_apic_post_state_restore(vcpu);
1271         vcpu_put(vcpu);
1272
1273         return 0;
1274 }
1275
1276 static int kvm_vcpu_ioctl_interrupt(struct kvm_vcpu *vcpu,
1277                                     struct kvm_interrupt *irq)
1278 {
1279         if (irq->irq < 0 || irq->irq >= 256)
1280                 return -EINVAL;
1281         if (irqchip_in_kernel(vcpu->kvm))
1282                 return -ENXIO;
1283         vcpu_load(vcpu);
1284
1285         set_bit(irq->irq, vcpu->arch.irq_pending);
1286         set_bit(irq->irq / BITS_PER_LONG, &vcpu->arch.irq_summary);
1287
1288         vcpu_put(vcpu);
1289
1290         return 0;
1291 }
1292
1293 static int vcpu_ioctl_tpr_access_reporting(struct kvm_vcpu *vcpu,
1294                                            struct kvm_tpr_access_ctl *tac)
1295 {
1296         if (tac->flags)
1297                 return -EINVAL;
1298         vcpu->arch.tpr_access_reporting = !!tac->enabled;
1299         return 0;
1300 }
1301
1302 long kvm_arch_vcpu_ioctl(struct file *filp,
1303                          unsigned int ioctl, unsigned long arg)
1304 {
1305         struct kvm_vcpu *vcpu = filp->private_data;
1306         void __user *argp = (void __user *)arg;
1307         int r;
1308
1309         switch (ioctl) {
1310         case KVM_GET_LAPIC: {
1311                 struct kvm_lapic_state lapic;
1312
1313                 memset(&lapic, 0, sizeof lapic);
1314                 r = kvm_vcpu_ioctl_get_lapic(vcpu, &lapic);
1315                 if (r)
1316                         goto out;
1317                 r = -EFAULT;
1318                 if (copy_to_user(argp, &lapic, sizeof lapic))
1319                         goto out;
1320                 r = 0;
1321                 break;
1322         }
1323         case KVM_SET_LAPIC: {
1324                 struct kvm_lapic_state lapic;
1325
1326                 r = -EFAULT;
1327                 if (copy_from_user(&lapic, argp, sizeof lapic))
1328                         goto out;
1329                 r = kvm_vcpu_ioctl_set_lapic(vcpu, &lapic);;
1330                 if (r)
1331                         goto out;
1332                 r = 0;
1333                 break;
1334         }
1335         case KVM_INTERRUPT: {
1336                 struct kvm_interrupt irq;
1337
1338                 r = -EFAULT;
1339                 if (copy_from_user(&irq, argp, sizeof irq))
1340                         goto out;
1341                 r = kvm_vcpu_ioctl_interrupt(vcpu, &irq);
1342                 if (r)
1343                         goto out;
1344                 r = 0;
1345                 break;
1346         }
1347         case KVM_SET_CPUID: {
1348                 struct kvm_cpuid __user *cpuid_arg = argp;
1349                 struct kvm_cpuid cpuid;
1350
1351                 r = -EFAULT;
1352                 if (copy_from_user(&cpuid, cpuid_arg, sizeof cpuid))
1353                         goto out;
1354                 r = kvm_vcpu_ioctl_set_cpuid(vcpu, &cpuid, cpuid_arg->entries);
1355                 if (r)
1356                         goto out;
1357                 break;
1358         }
1359         case KVM_SET_CPUID2: {
1360                 struct kvm_cpuid2 __user *cpuid_arg = argp;
1361                 struct kvm_cpuid2 cpuid;
1362
1363                 r = -EFAULT;
1364                 if (copy_from_user(&cpuid, cpuid_arg, sizeof cpuid))
1365                         goto out;
1366                 r = kvm_vcpu_ioctl_set_cpuid2(vcpu, &cpuid,
1367                                 cpuid_arg->entries);
1368                 if (r)
1369                         goto out;
1370                 break;
1371         }
1372         case KVM_GET_CPUID2: {
1373                 struct kvm_cpuid2 __user *cpuid_arg = argp;
1374                 struct kvm_cpuid2 cpuid;
1375
1376                 r = -EFAULT;
1377                 if (copy_from_user(&cpuid, cpuid_arg, sizeof cpuid))
1378                         goto out;
1379                 r = kvm_vcpu_ioctl_get_cpuid2(vcpu, &cpuid,
1380                                 cpuid_arg->entries);
1381                 if (r)
1382                         goto out;
1383                 r = -EFAULT;
1384                 if (copy_to_user(cpuid_arg, &cpuid, sizeof cpuid))
1385                         goto out;
1386                 r = 0;
1387                 break;
1388         }
1389         case KVM_GET_MSRS:
1390                 r = msr_io(vcpu, argp, kvm_get_msr, 1);
1391                 break;
1392         case KVM_SET_MSRS:
1393                 r = msr_io(vcpu, argp, do_set_msr, 0);
1394                 break;
1395         case KVM_TPR_ACCESS_REPORTING: {
1396                 struct kvm_tpr_access_ctl tac;
1397
1398                 r = -EFAULT;
1399                 if (copy_from_user(&tac, argp, sizeof tac))
1400                         goto out;
1401                 r = vcpu_ioctl_tpr_access_reporting(vcpu, &tac);
1402                 if (r)
1403                         goto out;
1404                 r = -EFAULT;
1405                 if (copy_to_user(argp, &tac, sizeof tac))
1406                         goto out;
1407                 r = 0;
1408                 break;
1409         };
1410         case KVM_SET_VAPIC_ADDR: {
1411                 struct kvm_vapic_addr va;
1412
1413                 r = -EINVAL;
1414                 if (!irqchip_in_kernel(vcpu->kvm))
1415                         goto out;
1416                 r = -EFAULT;
1417                 if (copy_from_user(&va, argp, sizeof va))
1418                         goto out;
1419                 r = 0;
1420                 kvm_lapic_set_vapic_addr(vcpu, va.vapic_addr);
1421                 break;
1422         }
1423         default:
1424                 r = -EINVAL;
1425         }
1426 out:
1427         return r;
1428 }
1429
1430 static int kvm_vm_ioctl_set_tss_addr(struct kvm *kvm, unsigned long addr)
1431 {
1432         int ret;
1433
1434         if (addr > (unsigned int)(-3 * PAGE_SIZE))
1435                 return -1;
1436         ret = kvm_x86_ops->set_tss_addr(kvm, addr);
1437         return ret;
1438 }
1439
1440 static int kvm_vm_ioctl_set_nr_mmu_pages(struct kvm *kvm,
1441                                           u32 kvm_nr_mmu_pages)
1442 {
1443         if (kvm_nr_mmu_pages < KVM_MIN_ALLOC_MMU_PAGES)
1444                 return -EINVAL;
1445
1446         down_write(&kvm->slots_lock);
1447
1448         kvm_mmu_change_mmu_pages(kvm, kvm_nr_mmu_pages);
1449         kvm->arch.n_requested_mmu_pages = kvm_nr_mmu_pages;
1450
1451         up_write(&kvm->slots_lock);
1452         return 0;
1453 }
1454
1455 static int kvm_vm_ioctl_get_nr_mmu_pages(struct kvm *kvm)
1456 {
1457         return kvm->arch.n_alloc_mmu_pages;
1458 }
1459
1460 gfn_t unalias_gfn(struct kvm *kvm, gfn_t gfn)
1461 {
1462         int i;
1463         struct kvm_mem_alias *alias;
1464
1465         for (i = 0; i < kvm->arch.naliases; ++i) {
1466                 alias = &kvm->arch.aliases[i];
1467                 if (gfn >= alias->base_gfn
1468                     && gfn < alias->base_gfn + alias->npages)
1469                         return alias->target_gfn + gfn - alias->base_gfn;
1470         }
1471         return gfn;
1472 }
1473
1474 /*
1475  * Set a new alias region.  Aliases map a portion of physical memory into
1476  * another portion.  This is useful for memory windows, for example the PC
1477  * VGA region.
1478  */
1479 static int kvm_vm_ioctl_set_memory_alias(struct kvm *kvm,
1480                                          struct kvm_memory_alias *alias)
1481 {
1482         int r, n;
1483         struct kvm_mem_alias *p;
1484
1485         r = -EINVAL;
1486         /* General sanity checks */
1487         if (alias->memory_size & (PAGE_SIZE - 1))
1488                 goto out;
1489         if (alias->guest_phys_addr & (PAGE_SIZE - 1))
1490                 goto out;
1491         if (alias->slot >= KVM_ALIAS_SLOTS)
1492                 goto out;
1493         if (alias->guest_phys_addr + alias->memory_size
1494             < alias->guest_phys_addr)
1495                 goto out;
1496         if (alias->target_phys_addr + alias->memory_size
1497             < alias->target_phys_addr)
1498                 goto out;
1499
1500         down_write(&kvm->slots_lock);
1501         spin_lock(&kvm->mmu_lock);
1502
1503         p = &kvm->arch.aliases[alias->slot];
1504         p->base_gfn = alias->guest_phys_addr >> PAGE_SHIFT;
1505         p->npages = alias->memory_size >> PAGE_SHIFT;
1506         p->target_gfn = alias->target_phys_addr >> PAGE_SHIFT;
1507
1508         for (n = KVM_ALIAS_SLOTS; n > 0; --n)
1509                 if (kvm->arch.aliases[n - 1].npages)
1510                         break;
1511         kvm->arch.naliases = n;
1512
1513         spin_unlock(&kvm->mmu_lock);
1514         kvm_mmu_zap_all(kvm);
1515
1516         up_write(&kvm->slots_lock);
1517
1518         return 0;
1519
1520 out:
1521         return r;
1522 }
1523
1524 static int kvm_vm_ioctl_get_irqchip(struct kvm *kvm, struct kvm_irqchip *chip)
1525 {
1526         int r;
1527
1528         r = 0;
1529         switch (chip->chip_id) {
1530         case KVM_IRQCHIP_PIC_MASTER:
1531                 memcpy(&chip->chip.pic,
1532                         &pic_irqchip(kvm)->pics[0],
1533                         sizeof(struct kvm_pic_state));
1534                 break;
1535         case KVM_IRQCHIP_PIC_SLAVE:
1536                 memcpy(&chip->chip.pic,
1537                         &pic_irqchip(kvm)->pics[1],
1538                         sizeof(struct kvm_pic_state));
1539                 break;
1540         case KVM_IRQCHIP_IOAPIC:
1541                 memcpy(&chip->chip.ioapic,
1542                         ioapic_irqchip(kvm),
1543                         sizeof(struct kvm_ioapic_state));
1544                 break;
1545         default:
1546                 r = -EINVAL;
1547                 break;
1548         }
1549         return r;
1550 }
1551
1552 static int kvm_vm_ioctl_set_irqchip(struct kvm *kvm, struct kvm_irqchip *chip)
1553 {
1554         int r;
1555
1556         r = 0;
1557         switch (chip->chip_id) {
1558         case KVM_IRQCHIP_PIC_MASTER:
1559                 memcpy(&pic_irqchip(kvm)->pics[0],
1560                         &chip->chip.pic,
1561                         sizeof(struct kvm_pic_state));
1562                 break;
1563         case KVM_IRQCHIP_PIC_SLAVE:
1564                 memcpy(&pic_irqchip(kvm)->pics[1],
1565                         &chip->chip.pic,
1566                         sizeof(struct kvm_pic_state));
1567                 break;
1568         case KVM_IRQCHIP_IOAPIC:
1569                 memcpy(ioapic_irqchip(kvm),
1570                         &chip->chip.ioapic,
1571                         sizeof(struct kvm_ioapic_state));
1572                 break;
1573         default:
1574                 r = -EINVAL;
1575                 break;
1576         }
1577         kvm_pic_update_irq(pic_irqchip(kvm));
1578         return r;
1579 }
1580
1581 static int kvm_vm_ioctl_get_pit(struct kvm *kvm, struct kvm_pit_state *ps)
1582 {
1583         int r = 0;
1584
1585         memcpy(ps, &kvm->arch.vpit->pit_state, sizeof(struct kvm_pit_state));
1586         return r;
1587 }
1588
1589 static int kvm_vm_ioctl_set_pit(struct kvm *kvm, struct kvm_pit_state *ps)
1590 {
1591         int r = 0;
1592
1593         memcpy(&kvm->arch.vpit->pit_state, ps, sizeof(struct kvm_pit_state));
1594         kvm_pit_load_count(kvm, 0, ps->channels[0].count);
1595         return r;
1596 }
1597
1598 /*
1599  * Get (and clear) the dirty memory log for a memory slot.
1600  */
1601 int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm,
1602                                       struct kvm_dirty_log *log)
1603 {
1604         int r;
1605         int n;
1606         struct kvm_memory_slot *memslot;
1607         int is_dirty = 0;
1608
1609         down_write(&kvm->slots_lock);
1610
1611         r = kvm_get_dirty_log(kvm, log, &is_dirty);
1612         if (r)
1613                 goto out;
1614
1615         /* If nothing is dirty, don't bother messing with page tables. */
1616         if (is_dirty) {
1617                 kvm_mmu_slot_remove_write_access(kvm, log->slot);
1618                 kvm_flush_remote_tlbs(kvm);
1619                 memslot = &kvm->memslots[log->slot];
1620                 n = ALIGN(memslot->npages, BITS_PER_LONG) / 8;
1621                 memset(memslot->dirty_bitmap, 0, n);
1622         }
1623         r = 0;
1624 out:
1625         up_write(&kvm->slots_lock);
1626         return r;
1627 }
1628
1629 long kvm_arch_vm_ioctl(struct file *filp,
1630                        unsigned int ioctl, unsigned long arg)
1631 {
1632         struct kvm *kvm = filp->private_data;
1633         void __user *argp = (void __user *)arg;
1634         int r = -EINVAL;
1635
1636         switch (ioctl) {
1637         case KVM_SET_TSS_ADDR:
1638                 r = kvm_vm_ioctl_set_tss_addr(kvm, arg);
1639                 if (r < 0)
1640                         goto out;
1641                 break;
1642         case KVM_SET_MEMORY_REGION: {
1643                 struct kvm_memory_region kvm_mem;
1644                 struct kvm_userspace_memory_region kvm_userspace_mem;
1645
1646                 r = -EFAULT;
1647                 if (copy_from_user(&kvm_mem, argp, sizeof kvm_mem))
1648                         goto out;
1649                 kvm_userspace_mem.slot = kvm_mem.slot;
1650                 kvm_userspace_mem.flags = kvm_mem.flags;
1651                 kvm_userspace_mem.guest_phys_addr = kvm_mem.guest_phys_addr;
1652                 kvm_userspace_mem.memory_size = kvm_mem.memory_size;
1653                 r = kvm_vm_ioctl_set_memory_region(kvm, &kvm_userspace_mem, 0);
1654                 if (r)
1655                         goto out;
1656                 break;
1657         }
1658         case KVM_SET_NR_MMU_PAGES:
1659                 r = kvm_vm_ioctl_set_nr_mmu_pages(kvm, arg);
1660                 if (r)
1661                         goto out;
1662                 break;
1663         case KVM_GET_NR_MMU_PAGES:
1664                 r = kvm_vm_ioctl_get_nr_mmu_pages(kvm);
1665                 break;
1666         case KVM_SET_MEMORY_ALIAS: {
1667                 struct kvm_memory_alias alias;
1668
1669                 r = -EFAULT;
1670                 if (copy_from_user(&alias, argp, sizeof alias))
1671                         goto out;
1672                 r = kvm_vm_ioctl_set_memory_alias(kvm, &alias);
1673                 if (r)
1674                         goto out;
1675                 break;
1676         }
1677         case KVM_CREATE_IRQCHIP:
1678                 r = -ENOMEM;
1679                 kvm->arch.vpic = kvm_create_pic(kvm);
1680                 if (kvm->arch.vpic) {
1681                         r = kvm_ioapic_init(kvm);
1682                         if (r) {
1683                                 kfree(kvm->arch.vpic);
1684                                 kvm->arch.vpic = NULL;
1685                                 goto out;
1686                         }
1687                 } else
1688                         goto out;
1689                 break;
1690         case KVM_CREATE_PIT:
1691                 r = -ENOMEM;
1692                 kvm->arch.vpit = kvm_create_pit(kvm);
1693                 if (kvm->arch.vpit)
1694                         r = 0;
1695                 break;
1696         case KVM_IRQ_LINE: {
1697                 struct kvm_irq_level irq_event;
1698
1699                 r = -EFAULT;
1700                 if (copy_from_user(&irq_event, argp, sizeof irq_event))
1701                         goto out;
1702                 if (irqchip_in_kernel(kvm)) {
1703                         mutex_lock(&kvm->lock);
1704                         if (irq_event.irq < 16)
1705                                 kvm_pic_set_irq(pic_irqchip(kvm),
1706                                         irq_event.irq,
1707                                         irq_event.level);
1708                         kvm_ioapic_set_irq(kvm->arch.vioapic,
1709                                         irq_event.irq,
1710                                         irq_event.level);
1711                         mutex_unlock(&kvm->lock);
1712                         r = 0;
1713                 }
1714                 break;
1715         }
1716         case KVM_GET_IRQCHIP: {
1717                 /* 0: PIC master, 1: PIC slave, 2: IOAPIC */
1718                 struct kvm_irqchip chip;
1719
1720                 r = -EFAULT;
1721                 if (copy_from_user(&chip, argp, sizeof chip))
1722                         goto out;
1723                 r = -ENXIO;
1724                 if (!irqchip_in_kernel(kvm))
1725                         goto out;
1726                 r = kvm_vm_ioctl_get_irqchip(kvm, &chip);
1727                 if (r)
1728                         goto out;
1729                 r = -EFAULT;
1730                 if (copy_to_user(argp, &chip, sizeof chip))
1731                         goto out;
1732                 r = 0;
1733                 break;
1734         }
1735         case KVM_SET_IRQCHIP: {
1736                 /* 0: PIC master, 1: PIC slave, 2: IOAPIC */
1737                 struct kvm_irqchip chip;
1738
1739                 r = -EFAULT;
1740                 if (copy_from_user(&chip, argp, sizeof chip))
1741                         goto out;
1742                 r = -ENXIO;
1743                 if (!irqchip_in_kernel(kvm))
1744                         goto out;
1745                 r = kvm_vm_ioctl_set_irqchip(kvm, &chip);
1746                 if (r)
1747                         goto out;
1748                 r = 0;
1749                 break;
1750         }
1751         case KVM_GET_PIT: {
1752                 struct kvm_pit_state ps;
1753                 r = -EFAULT;
1754                 if (copy_from_user(&ps, argp, sizeof ps))
1755                         goto out;
1756                 r = -ENXIO;
1757                 if (!kvm->arch.vpit)
1758                         goto out;
1759                 r = kvm_vm_ioctl_get_pit(kvm, &ps);
1760                 if (r)
1761                         goto out;
1762                 r = -EFAULT;
1763                 if (copy_to_user(argp, &ps, sizeof ps))
1764                         goto out;
1765                 r = 0;
1766                 break;
1767         }
1768         case KVM_SET_PIT: {
1769                 struct kvm_pit_state ps;
1770                 r = -EFAULT;
1771                 if (copy_from_user(&ps, argp, sizeof ps))
1772                         goto out;
1773                 r = -ENXIO;
1774                 if (!kvm->arch.vpit)
1775                         goto out;
1776                 r = kvm_vm_ioctl_set_pit(kvm, &ps);
1777                 if (r)
1778                         goto out;
1779                 r = 0;
1780                 break;
1781         }
1782         default:
1783                 ;
1784         }
1785 out:
1786         return r;
1787 }
1788
1789 static void kvm_init_msr_list(void)
1790 {
1791         u32 dummy[2];
1792         unsigned i, j;
1793
1794         for (i = j = 0; i < ARRAY_SIZE(msrs_to_save); i++) {
1795                 if (rdmsr_safe(msrs_to_save[i], &dummy[0], &dummy[1]) < 0)
1796                         continue;
1797                 if (j < i)
1798                         msrs_to_save[j] = msrs_to_save[i];
1799                 j++;
1800         }
1801         num_msrs_to_save = j;
1802 }
1803
1804 /*
1805  * Only apic need an MMIO device hook, so shortcut now..
1806  */
1807 static struct kvm_io_device *vcpu_find_pervcpu_dev(struct kvm_vcpu *vcpu,
1808                                                 gpa_t addr, int len,
1809                                                 int is_write)
1810 {
1811         struct kvm_io_device *dev;
1812
1813         if (vcpu->arch.apic) {
1814                 dev = &vcpu->arch.apic->dev;
1815                 if (dev->in_range(dev, addr, len, is_write))
1816                         return dev;
1817         }
1818         return NULL;
1819 }
1820
1821
1822 static struct kvm_io_device *vcpu_find_mmio_dev(struct kvm_vcpu *vcpu,
1823                                                 gpa_t addr, int len,
1824                                                 int is_write)
1825 {
1826         struct kvm_io_device *dev;
1827
1828         dev = vcpu_find_pervcpu_dev(vcpu, addr, len, is_write);
1829         if (dev == NULL)
1830                 dev = kvm_io_bus_find_dev(&vcpu->kvm->mmio_bus, addr, len,
1831                                           is_write);
1832         return dev;
1833 }
1834
1835 int emulator_read_std(unsigned long addr,
1836                              void *val,
1837                              unsigned int bytes,
1838                              struct kvm_vcpu *vcpu)
1839 {
1840         void *data = val;
1841         int r = X86EMUL_CONTINUE;
1842
1843         while (bytes) {
1844                 gpa_t gpa = vcpu->arch.mmu.gva_to_gpa(vcpu, addr);
1845                 unsigned offset = addr & (PAGE_SIZE-1);
1846                 unsigned tocopy = min(bytes, (unsigned)PAGE_SIZE - offset);
1847                 int ret;
1848
1849                 if (gpa == UNMAPPED_GVA) {
1850                         r = X86EMUL_PROPAGATE_FAULT;
1851                         goto out;
1852                 }
1853                 ret = kvm_read_guest(vcpu->kvm, gpa, data, tocopy);
1854                 if (ret < 0) {
1855                         r = X86EMUL_UNHANDLEABLE;
1856                         goto out;
1857                 }
1858
1859                 bytes -= tocopy;
1860                 data += tocopy;
1861                 addr += tocopy;
1862         }
1863 out:
1864         return r;
1865 }
1866 EXPORT_SYMBOL_GPL(emulator_read_std);
1867
1868 static int emulator_read_emulated(unsigned long addr,
1869                                   void *val,
1870                                   unsigned int bytes,
1871                                   struct kvm_vcpu *vcpu)
1872 {
1873         struct kvm_io_device *mmio_dev;
1874         gpa_t                 gpa;
1875
1876         if (vcpu->mmio_read_completed) {
1877                 memcpy(val, vcpu->mmio_data, bytes);
1878                 vcpu->mmio_read_completed = 0;
1879                 return X86EMUL_CONTINUE;
1880         }
1881
1882         gpa = vcpu->arch.mmu.gva_to_gpa(vcpu, addr);
1883
1884         /* For APIC access vmexit */
1885         if ((gpa & PAGE_MASK) == APIC_DEFAULT_PHYS_BASE)
1886                 goto mmio;
1887
1888         if (emulator_read_std(addr, val, bytes, vcpu)
1889                         == X86EMUL_CONTINUE)
1890                 return X86EMUL_CONTINUE;
1891         if (gpa == UNMAPPED_GVA)
1892                 return X86EMUL_PROPAGATE_FAULT;
1893
1894 mmio:
1895         /*
1896          * Is this MMIO handled locally?
1897          */
1898         mutex_lock(&vcpu->kvm->lock);
1899         mmio_dev = vcpu_find_mmio_dev(vcpu, gpa, bytes, 0);
1900         if (mmio_dev) {
1901                 kvm_iodevice_read(mmio_dev, gpa, bytes, val);
1902                 mutex_unlock(&vcpu->kvm->lock);
1903                 return X86EMUL_CONTINUE;
1904         }
1905         mutex_unlock(&vcpu->kvm->lock);
1906
1907         vcpu->mmio_needed = 1;
1908         vcpu->mmio_phys_addr = gpa;
1909         vcpu->mmio_size = bytes;
1910         vcpu->mmio_is_write = 0;
1911
1912         return X86EMUL_UNHANDLEABLE;
1913 }
1914
1915 int emulator_write_phys(struct kvm_vcpu *vcpu, gpa_t gpa,
1916                           const void *val, int bytes)
1917 {
1918         int ret;
1919
1920         ret = kvm_write_guest(vcpu->kvm, gpa, val, bytes);
1921         if (ret < 0)
1922                 return 0;
1923         kvm_mmu_pte_write(vcpu, gpa, val, bytes);
1924         return 1;
1925 }
1926
1927 static int emulator_write_emulated_onepage(unsigned long addr,
1928                                            const void *val,
1929                                            unsigned int bytes,
1930                                            struct kvm_vcpu *vcpu)
1931 {
1932         struct kvm_io_device *mmio_dev;
1933         gpa_t                 gpa;
1934
1935         gpa = vcpu->arch.mmu.gva_to_gpa(vcpu, addr);
1936
1937         if (gpa == UNMAPPED_GVA) {
1938                 kvm_inject_page_fault(vcpu, addr, 2);
1939                 return X86EMUL_PROPAGATE_FAULT;
1940         }
1941
1942         /* For APIC access vmexit */
1943         if ((gpa & PAGE_MASK) == APIC_DEFAULT_PHYS_BASE)
1944                 goto mmio;
1945
1946         if (emulator_write_phys(vcpu, gpa, val, bytes))
1947                 return X86EMUL_CONTINUE;
1948
1949 mmio:
1950         /*
1951          * Is this MMIO handled locally?
1952          */
1953         mutex_lock(&vcpu->kvm->lock);
1954         mmio_dev = vcpu_find_mmio_dev(vcpu, gpa, bytes, 1);
1955         if (mmio_dev) {
1956                 kvm_iodevice_write(mmio_dev, gpa, bytes, val);
1957                 mutex_unlock(&vcpu->kvm->lock);
1958                 return X86EMUL_CONTINUE;
1959         }
1960         mutex_unlock(&vcpu->kvm->lock);
1961
1962         vcpu->mmio_needed = 1;
1963         vcpu->mmio_phys_addr = gpa;
1964         vcpu->mmio_size = bytes;
1965         vcpu->mmio_is_write = 1;
1966         memcpy(vcpu->mmio_data, val, bytes);
1967
1968         return X86EMUL_CONTINUE;
1969 }
1970
1971 int emulator_write_emulated(unsigned long addr,
1972                                    const void *val,
1973                                    unsigned int bytes,
1974                                    struct kvm_vcpu *vcpu)
1975 {
1976         /* Crossing a page boundary? */
1977         if (((addr + bytes - 1) ^ addr) & PAGE_MASK) {
1978                 int rc, now;
1979
1980                 now = -addr & ~PAGE_MASK;
1981                 rc = emulator_write_emulated_onepage(addr, val, now, vcpu);
1982                 if (rc != X86EMUL_CONTINUE)
1983                         return rc;
1984                 addr += now;
1985                 val += now;
1986                 bytes -= now;
1987         }
1988         return emulator_write_emulated_onepage(addr, val, bytes, vcpu);
1989 }
1990 EXPORT_SYMBOL_GPL(emulator_write_emulated);
1991
1992 static int emulator_cmpxchg_emulated(unsigned long addr,
1993                                      const void *old,
1994                                      const void *new,
1995                                      unsigned int bytes,
1996                                      struct kvm_vcpu *vcpu)
1997 {
1998         static int reported;
1999
2000         if (!reported) {
2001                 reported = 1;
2002                 printk(KERN_WARNING "kvm: emulating exchange as write\n");
2003         }
2004 #ifndef CONFIG_X86_64
2005         /* guests cmpxchg8b have to be emulated atomically */
2006         if (bytes == 8) {
2007                 gpa_t gpa;
2008                 struct page *page;
2009                 char *kaddr;
2010                 u64 val;
2011
2012                 gpa = vcpu->arch.mmu.gva_to_gpa(vcpu, addr);
2013
2014                 if (gpa == UNMAPPED_GVA ||
2015                    (gpa & PAGE_MASK) == APIC_DEFAULT_PHYS_BASE)
2016                         goto emul_write;
2017
2018                 if (((gpa + bytes - 1) & PAGE_MASK) != (gpa & PAGE_MASK))
2019                         goto emul_write;
2020
2021                 val = *(u64 *)new;
2022
2023                 down_read(&current->mm->mmap_sem);
2024                 page = gfn_to_page(vcpu->kvm, gpa >> PAGE_SHIFT);
2025                 up_read(&current->mm->mmap_sem);
2026
2027                 kaddr = kmap_atomic(page, KM_USER0);
2028                 set_64bit((u64 *)(kaddr + offset_in_page(gpa)), val);
2029                 kunmap_atomic(kaddr, KM_USER0);
2030                 kvm_release_page_dirty(page);
2031         }
2032 emul_write:
2033 #endif
2034
2035         return emulator_write_emulated(addr, new, bytes, vcpu);
2036 }
2037
2038 static unsigned long get_segment_base(struct kvm_vcpu *vcpu, int seg)
2039 {
2040         return kvm_x86_ops->get_segment_base(vcpu, seg);
2041 }
2042
2043 int emulate_invlpg(struct kvm_vcpu *vcpu, gva_t address)
2044 {
2045         return X86EMUL_CONTINUE;
2046 }
2047
2048 int emulate_clts(struct kvm_vcpu *vcpu)
2049 {
2050         KVMTRACE_0D(CLTS, vcpu, handler);
2051         kvm_x86_ops->set_cr0(vcpu, vcpu->arch.cr0 & ~X86_CR0_TS);
2052         return X86EMUL_CONTINUE;
2053 }
2054
2055 int emulator_get_dr(struct x86_emulate_ctxt *ctxt, int dr, unsigned long *dest)
2056 {
2057         struct kvm_vcpu *vcpu = ctxt->vcpu;
2058
2059         switch (dr) {
2060         case 0 ... 3:
2061                 *dest = kvm_x86_ops->get_dr(vcpu, dr);
2062                 return X86EMUL_CONTINUE;
2063         default:
2064                 pr_unimpl(vcpu, "%s: unexpected dr %u\n", __func__, dr);
2065                 return X86EMUL_UNHANDLEABLE;
2066         }
2067 }
2068
2069 int emulator_set_dr(struct x86_emulate_ctxt *ctxt, int dr, unsigned long value)
2070 {
2071         unsigned long mask = (ctxt->mode == X86EMUL_MODE_PROT64) ? ~0ULL : ~0U;
2072         int exception;
2073
2074         kvm_x86_ops->set_dr(ctxt->vcpu, dr, value & mask, &exception);
2075         if (exception) {
2076                 /* FIXME: better handling */
2077                 return X86EMUL_UNHANDLEABLE;
2078         }
2079         return X86EMUL_CONTINUE;
2080 }
2081
2082 void kvm_report_emulation_failure(struct kvm_vcpu *vcpu, const char *context)
2083 {
2084         u8 opcodes[4];
2085         unsigned long rip = kvm_rip_read(vcpu);
2086         unsigned long rip_linear;
2087
2088         if (!printk_ratelimit())
2089                 return;
2090
2091         rip_linear = rip + get_segment_base(vcpu, VCPU_SREG_CS);
2092
2093         emulator_read_std(rip_linear, (void *)opcodes, 4, vcpu);
2094
2095         printk(KERN_ERR "emulation failed (%s) rip %lx %02x %02x %02x %02x\n",
2096                context, rip, opcodes[0], opcodes[1], opcodes[2], opcodes[3]);
2097 }
2098 EXPORT_SYMBOL_GPL(kvm_report_emulation_failure);
2099
2100 static struct x86_emulate_ops emulate_ops = {
2101         .read_std            = emulator_read_std,
2102         .read_emulated       = emulator_read_emulated,
2103         .write_emulated      = emulator_write_emulated,
2104         .cmpxchg_emulated    = emulator_cmpxchg_emulated,
2105 };
2106
2107 static void cache_all_regs(struct kvm_vcpu *vcpu)
2108 {
2109         kvm_register_read(vcpu, VCPU_REGS_RAX);
2110         kvm_register_read(vcpu, VCPU_REGS_RSP);
2111         kvm_register_read(vcpu, VCPU_REGS_RIP);
2112         vcpu->arch.regs_dirty = ~0;
2113 }
2114
2115 int emulate_instruction(struct kvm_vcpu *vcpu,
2116                         struct kvm_run *run,
2117                         unsigned long cr2,
2118                         u16 error_code,
2119                         int emulation_type)
2120 {
2121         int r;
2122         struct decode_cache *c;
2123
2124         vcpu->arch.mmio_fault_cr2 = cr2;
2125         /*
2126          * TODO: fix x86_emulate.c to use guest_read/write_register
2127          * instead of direct ->regs accesses, can save hundred cycles
2128          * on Intel for instructions that don't read/change RSP, for
2129          * for example.
2130          */
2131         cache_all_regs(vcpu);
2132
2133         vcpu->mmio_is_write = 0;
2134         vcpu->arch.pio.string = 0;
2135
2136         if (!(emulation_type & EMULTYPE_NO_DECODE)) {
2137                 int cs_db, cs_l;
2138                 kvm_x86_ops->get_cs_db_l_bits(vcpu, &cs_db, &cs_l);
2139
2140                 vcpu->arch.emulate_ctxt.vcpu = vcpu;
2141                 vcpu->arch.emulate_ctxt.eflags = kvm_x86_ops->get_rflags(vcpu);
2142                 vcpu->arch.emulate_ctxt.mode =
2143                         (vcpu->arch.emulate_ctxt.eflags & X86_EFLAGS_VM)
2144                         ? X86EMUL_MODE_REAL : cs_l
2145                         ? X86EMUL_MODE_PROT64 : cs_db
2146                         ? X86EMUL_MODE_PROT32 : X86EMUL_MODE_PROT16;
2147
2148                 r = x86_decode_insn(&vcpu->arch.emulate_ctxt, &emulate_ops);
2149
2150                 /* Reject the instructions other than VMCALL/VMMCALL when
2151                  * try to emulate invalid opcode */
2152                 c = &vcpu->arch.emulate_ctxt.decode;
2153                 if ((emulation_type & EMULTYPE_TRAP_UD) &&
2154                     (!(c->twobyte && c->b == 0x01 &&
2155                       (c->modrm_reg == 0 || c->modrm_reg == 3) &&
2156                        c->modrm_mod == 3 && c->modrm_rm == 1)))
2157                         return EMULATE_FAIL;
2158
2159                 ++vcpu->stat.insn_emulation;
2160                 if (r)  {
2161                         ++vcpu->stat.insn_emulation_fail;
2162                         if (kvm_mmu_unprotect_page_virt(vcpu, cr2))
2163                                 return EMULATE_DONE;
2164                         return EMULATE_FAIL;
2165                 }
2166         }
2167
2168         r = x86_emulate_insn(&vcpu->arch.emulate_ctxt, &emulate_ops);
2169
2170         if (vcpu->arch.pio.string)
2171                 return EMULATE_DO_MMIO;
2172
2173         if ((r || vcpu->mmio_is_write) && run) {
2174                 run->exit_reason = KVM_EXIT_MMIO;
2175                 run->mmio.phys_addr = vcpu->mmio_phys_addr;
2176                 memcpy(run->mmio.data, vcpu->mmio_data, 8);
2177                 run->mmio.len = vcpu->mmio_size;
2178                 run->mmio.is_write = vcpu->mmio_is_write;
2179         }
2180
2181         if (r) {
2182                 if (kvm_mmu_unprotect_page_virt(vcpu, cr2))
2183                         return EMULATE_DONE;
2184                 if (!vcpu->mmio_needed) {
2185                         kvm_report_emulation_failure(vcpu, "mmio");
2186                         return EMULATE_FAIL;
2187                 }
2188                 return EMULATE_DO_MMIO;
2189         }
2190
2191         kvm_x86_ops->set_rflags(vcpu, vcpu->arch.emulate_ctxt.eflags);
2192
2193         if (vcpu->mmio_is_write) {
2194                 vcpu->mmio_needed = 0;
2195                 return EMULATE_DO_MMIO;
2196         }
2197
2198         return EMULATE_DONE;
2199 }
2200 EXPORT_SYMBOL_GPL(emulate_instruction);
2201
2202 static void free_pio_guest_pages(struct kvm_vcpu *vcpu)
2203 {
2204         int i;
2205
2206         for (i = 0; i < ARRAY_SIZE(vcpu->arch.pio.guest_pages); ++i)
2207                 if (vcpu->arch.pio.guest_pages[i]) {
2208                         kvm_release_page_dirty(vcpu->arch.pio.guest_pages[i]);
2209                         vcpu->arch.pio.guest_pages[i] = NULL;
2210                 }
2211 }
2212
2213 static int pio_copy_data(struct kvm_vcpu *vcpu)
2214 {
2215         void *p = vcpu->arch.pio_data;
2216         void *q;
2217         unsigned bytes;
2218         int nr_pages = vcpu->arch.pio.guest_pages[1] ? 2 : 1;
2219
2220         q = vmap(vcpu->arch.pio.guest_pages, nr_pages, VM_READ|VM_WRITE,
2221                  PAGE_KERNEL);
2222         if (!q) {
2223                 free_pio_guest_pages(vcpu);
2224                 return -ENOMEM;
2225         }
2226         q += vcpu->arch.pio.guest_page_offset;
2227         bytes = vcpu->arch.pio.size * vcpu->arch.pio.cur_count;
2228         if (vcpu->arch.pio.in)
2229                 memcpy(q, p, bytes);
2230         else
2231                 memcpy(p, q, bytes);
2232         q -= vcpu->arch.pio.guest_page_offset;
2233         vunmap(q);
2234         free_pio_guest_pages(vcpu);
2235         return 0;
2236 }
2237
2238 int complete_pio(struct kvm_vcpu *vcpu)
2239 {
2240         struct kvm_pio_request *io = &vcpu->arch.pio;
2241         long delta;
2242         int r;
2243         unsigned long val;
2244
2245         if (!io->string) {
2246                 if (io->in) {
2247                         val = kvm_register_read(vcpu, VCPU_REGS_RAX);
2248                         memcpy(&val, vcpu->arch.pio_data, io->size);
2249                         kvm_register_write(vcpu, VCPU_REGS_RAX, val);
2250                 }
2251         } else {
2252                 if (io->in) {
2253                         r = pio_copy_data(vcpu);
2254                         if (r)
2255                                 return r;
2256                 }
2257
2258                 delta = 1;
2259                 if (io->rep) {
2260                         delta *= io->cur_count;
2261                         /*
2262                          * The size of the register should really depend on
2263                          * current address size.
2264                          */
2265                         val = kvm_register_read(vcpu, VCPU_REGS_RCX);
2266                         val -= delta;
2267                         kvm_register_write(vcpu, VCPU_REGS_RCX, val);
2268                 }
2269                 if (io->down)
2270                         delta = -delta;
2271                 delta *= io->size;
2272                 if (io->in) {
2273                         val = kvm_register_read(vcpu, VCPU_REGS_RDI);
2274                         val += delta;
2275                         kvm_register_write(vcpu, VCPU_REGS_RDI, val);
2276                 } else {
2277                         val = kvm_register_read(vcpu, VCPU_REGS_RSI);
2278                         val += delta;
2279                         kvm_register_write(vcpu, VCPU_REGS_RSI, val);
2280                 }
2281         }
2282
2283         io->count -= io->cur_count;
2284         io->cur_count = 0;
2285
2286         return 0;
2287 }
2288
2289 static void kernel_pio(struct kvm_io_device *pio_dev,
2290                        struct kvm_vcpu *vcpu,
2291                        void *pd)
2292 {
2293         /* TODO: String I/O for in kernel device */
2294
2295         mutex_lock(&vcpu->kvm->lock);
2296         if (vcpu->arch.pio.in)
2297                 kvm_iodevice_read(pio_dev, vcpu->arch.pio.port,
2298                                   vcpu->arch.pio.size,
2299                                   pd);
2300         else
2301                 kvm_iodevice_write(pio_dev, vcpu->arch.pio.port,
2302                                    vcpu->arch.pio.size,
2303                                    pd);
2304         mutex_unlock(&vcpu->kvm->lock);
2305 }
2306
2307 static void pio_string_write(struct kvm_io_device *pio_dev,
2308                              struct kvm_vcpu *vcpu)
2309 {
2310         struct kvm_pio_request *io = &vcpu->arch.pio;
2311         void *pd = vcpu->arch.pio_data;
2312         int i;
2313
2314         mutex_lock(&vcpu->kvm->lock);
2315         for (i = 0; i < io->cur_count; i++) {
2316                 kvm_iodevice_write(pio_dev, io->port,
2317                                    io->size,
2318                                    pd);
2319                 pd += io->size;
2320         }
2321         mutex_unlock(&vcpu->kvm->lock);
2322 }
2323
2324 static struct kvm_io_device *vcpu_find_pio_dev(struct kvm_vcpu *vcpu,
2325                                                gpa_t addr, int len,
2326                                                int is_write)
2327 {
2328         return kvm_io_bus_find_dev(&vcpu->kvm->pio_bus, addr, len, is_write);
2329 }
2330
2331 int kvm_emulate_pio(struct kvm_vcpu *vcpu, struct kvm_run *run, int in,
2332                   int size, unsigned port)
2333 {
2334         struct kvm_io_device *pio_dev;
2335         unsigned long val;
2336
2337         vcpu->run->exit_reason = KVM_EXIT_IO;
2338         vcpu->run->io.direction = in ? KVM_EXIT_IO_IN : KVM_EXIT_IO_OUT;
2339         vcpu->run->io.size = vcpu->arch.pio.size = size;
2340         vcpu->run->io.data_offset = KVM_PIO_PAGE_OFFSET * PAGE_SIZE;
2341         vcpu->run->io.count = vcpu->arch.pio.count = vcpu->arch.pio.cur_count = 1;
2342         vcpu->run->io.port = vcpu->arch.pio.port = port;
2343         vcpu->arch.pio.in = in;
2344         vcpu->arch.pio.string = 0;
2345         vcpu->arch.pio.down = 0;
2346         vcpu->arch.pio.guest_page_offset = 0;
2347         vcpu->arch.pio.rep = 0;
2348
2349         if (vcpu->run->io.direction == KVM_EXIT_IO_IN)
2350                 KVMTRACE_2D(IO_READ, vcpu, vcpu->run->io.port, (u32)size,
2351                             handler);
2352         else
2353                 KVMTRACE_2D(IO_WRITE, vcpu, vcpu->run->io.port, (u32)size,
2354                             handler);
2355
2356         val = kvm_register_read(vcpu, VCPU_REGS_RAX);
2357         memcpy(vcpu->arch.pio_data, &val, 4);
2358
2359         kvm_x86_ops->skip_emulated_instruction(vcpu);
2360
2361         pio_dev = vcpu_find_pio_dev(vcpu, port, size, !in);
2362         if (pio_dev) {
2363                 kernel_pio(pio_dev, vcpu, vcpu->arch.pio_data);
2364                 complete_pio(vcpu);
2365                 return 1;
2366         }
2367         return 0;
2368 }
2369 EXPORT_SYMBOL_GPL(kvm_emulate_pio);
2370
2371 int kvm_emulate_pio_string(struct kvm_vcpu *vcpu, struct kvm_run *run, int in,
2372                   int size, unsigned long count, int down,
2373                   gva_t address, int rep, unsigned port)
2374 {
2375         unsigned now, in_page;
2376         int i, ret = 0;
2377         int nr_pages = 1;
2378         struct page *page;
2379         struct kvm_io_device *pio_dev;
2380
2381         vcpu->run->exit_reason = KVM_EXIT_IO;
2382         vcpu->run->io.direction = in ? KVM_EXIT_IO_IN : KVM_EXIT_IO_OUT;
2383         vcpu->run->io.size = vcpu->arch.pio.size = size;
2384         vcpu->run->io.data_offset = KVM_PIO_PAGE_OFFSET * PAGE_SIZE;
2385         vcpu->run->io.count = vcpu->arch.pio.count = vcpu->arch.pio.cur_count = count;
2386         vcpu->run->io.port = vcpu->arch.pio.port = port;
2387         vcpu->arch.pio.in = in;
2388         vcpu->arch.pio.string = 1;
2389         vcpu->arch.pio.down = down;
2390         vcpu->arch.pio.guest_page_offset = offset_in_page(address);
2391         vcpu->arch.pio.rep = rep;
2392
2393         if (vcpu->run->io.direction == KVM_EXIT_IO_IN)
2394                 KVMTRACE_2D(IO_READ, vcpu, vcpu->run->io.port, (u32)size,
2395                             handler);
2396         else
2397                 KVMTRACE_2D(IO_WRITE, vcpu, vcpu->run->io.port, (u32)size,
2398                             handler);
2399
2400         if (!count) {
2401                 kvm_x86_ops->skip_emulated_instruction(vcpu);
2402                 return 1;
2403         }
2404
2405         if (!down)
2406                 in_page = PAGE_SIZE - offset_in_page(address);
2407         else
2408                 in_page = offset_in_page(address) + size;
2409         now = min(count, (unsigned long)in_page / size);
2410         if (!now) {
2411                 /*
2412                  * String I/O straddles page boundary.  Pin two guest pages
2413                  * so that we satisfy atomicity constraints.  Do just one
2414                  * transaction to avoid complexity.
2415                  */
2416                 nr_pages = 2;
2417                 now = 1;
2418         }
2419         if (down) {
2420                 /*
2421                  * String I/O in reverse.  Yuck.  Kill the guest, fix later.
2422                  */
2423                 pr_unimpl(vcpu, "guest string pio down\n");
2424                 kvm_inject_gp(vcpu, 0);
2425                 return 1;
2426         }
2427         vcpu->run->io.count = now;
2428         vcpu->arch.pio.cur_count = now;
2429
2430         if (vcpu->arch.pio.cur_count == vcpu->arch.pio.count)
2431                 kvm_x86_ops->skip_emulated_instruction(vcpu);
2432
2433         for (i = 0; i < nr_pages; ++i) {
2434                 page = gva_to_page(vcpu, address + i * PAGE_SIZE);
2435                 vcpu->arch.pio.guest_pages[i] = page;
2436                 if (!page) {
2437                         kvm_inject_gp(vcpu, 0);
2438                         free_pio_guest_pages(vcpu);
2439                         return 1;
2440                 }
2441         }
2442
2443         pio_dev = vcpu_find_pio_dev(vcpu, port,
2444                                     vcpu->arch.pio.cur_count,
2445                                     !vcpu->arch.pio.in);
2446         if (!vcpu->arch.pio.in) {
2447                 /* string PIO write */
2448                 ret = pio_copy_data(vcpu);
2449                 if (ret >= 0 && pio_dev) {
2450                         pio_string_write(pio_dev, vcpu);
2451                         complete_pio(vcpu);
2452                         if (vcpu->arch.pio.count == 0)
2453                                 ret = 1;
2454                 }
2455         } else if (pio_dev)
2456                 pr_unimpl(vcpu, "no string pio read support yet, "
2457                        "port %x size %d count %ld\n",
2458                         port, size, count);
2459
2460         return ret;
2461 }
2462 EXPORT_SYMBOL_GPL(kvm_emulate_pio_string);
2463
2464 int kvm_arch_init(void *opaque)
2465 {
2466         int r;
2467         struct kvm_x86_ops *ops = (struct kvm_x86_ops *)opaque;
2468
2469         if (kvm_x86_ops) {
2470                 printk(KERN_ERR "kvm: already loaded the other module\n");
2471                 r = -EEXIST;
2472                 goto out;
2473         }
2474
2475         if (!ops->cpu_has_kvm_support()) {
2476                 printk(KERN_ERR "kvm: no hardware support\n");
2477                 r = -EOPNOTSUPP;
2478                 goto out;
2479         }
2480         if (ops->disabled_by_bios()) {
2481                 printk(KERN_ERR "kvm: disabled by bios\n");
2482                 r = -EOPNOTSUPP;
2483                 goto out;
2484         }
2485
2486         r = kvm_mmu_module_init();
2487         if (r)
2488                 goto out;
2489
2490         kvm_init_msr_list();
2491
2492         kvm_x86_ops = ops;
2493         kvm_mmu_set_nonpresent_ptes(0ull, 0ull);
2494         kvm_mmu_set_base_ptes(PT_PRESENT_MASK);
2495         kvm_mmu_set_mask_ptes(PT_USER_MASK, PT_ACCESSED_MASK,
2496                         PT_DIRTY_MASK, PT64_NX_MASK, 0);
2497         return 0;
2498
2499 out:
2500         return r;
2501 }
2502
2503 void kvm_arch_exit(void)
2504 {
2505         kvm_x86_ops = NULL;
2506         kvm_mmu_module_exit();
2507 }
2508
2509 int kvm_emulate_halt(struct kvm_vcpu *vcpu)
2510 {
2511         ++vcpu->stat.halt_exits;
2512         KVMTRACE_0D(HLT, vcpu, handler);
2513         if (irqchip_in_kernel(vcpu->kvm)) {
2514                 vcpu->arch.mp_state = KVM_MP_STATE_HALTED;
2515                 up_read(&vcpu->kvm->slots_lock);
2516                 kvm_vcpu_block(vcpu);
2517                 down_read(&vcpu->kvm->slots_lock);
2518                 if (vcpu->arch.mp_state != KVM_MP_STATE_RUNNABLE)
2519                         return -EINTR;
2520                 return 1;
2521         } else {
2522                 vcpu->run->exit_reason = KVM_EXIT_HLT;
2523                 return 0;
2524         }
2525 }
2526 EXPORT_SYMBOL_GPL(kvm_emulate_halt);
2527
2528 static inline gpa_t hc_gpa(struct kvm_vcpu *vcpu, unsigned long a0,
2529                            unsigned long a1)
2530 {
2531         if (is_long_mode(vcpu))
2532                 return a0;
2533         else
2534                 return a0 | ((gpa_t)a1 << 32);
2535 }
2536
2537 int kvm_emulate_hypercall(struct kvm_vcpu *vcpu)
2538 {
2539         unsigned long nr, a0, a1, a2, a3, ret;
2540         int r = 1;
2541
2542         nr = kvm_register_read(vcpu, VCPU_REGS_RAX);
2543         a0 = kvm_register_read(vcpu, VCPU_REGS_RBX);
2544         a1 = kvm_register_read(vcpu, VCPU_REGS_RCX);
2545         a2 = kvm_register_read(vcpu, VCPU_REGS_RDX);
2546         a3 = kvm_register_read(vcpu, VCPU_REGS_RSI);
2547
2548         KVMTRACE_1D(VMMCALL, vcpu, (u32)nr, handler);
2549
2550         if (!is_long_mode(vcpu)) {
2551                 nr &= 0xFFFFFFFF;
2552                 a0 &= 0xFFFFFFFF;
2553                 a1 &= 0xFFFFFFFF;
2554                 a2 &= 0xFFFFFFFF;
2555                 a3 &= 0xFFFFFFFF;
2556         }
2557
2558         switch (nr) {
2559         case KVM_HC_VAPIC_POLL_IRQ:
2560                 ret = 0;
2561                 break;
2562         case KVM_HC_MMU_OP:
2563                 r = kvm_pv_mmu_op(vcpu, a0, hc_gpa(vcpu, a1, a2), &ret);
2564                 break;
2565         default:
2566                 ret = -KVM_ENOSYS;
2567                 break;
2568         }
2569         kvm_register_write(vcpu, VCPU_REGS_RAX, ret);
2570         ++vcpu->stat.hypercalls;
2571         return r;
2572 }
2573 EXPORT_SYMBOL_GPL(kvm_emulate_hypercall);
2574
2575 int kvm_fix_hypercall(struct kvm_vcpu *vcpu)
2576 {
2577         char instruction[3];
2578         int ret = 0;
2579         unsigned long rip = kvm_rip_read(vcpu);
2580
2581
2582         /*
2583          * Blow out the MMU to ensure that no other VCPU has an active mapping
2584          * to ensure that the updated hypercall appears atomically across all
2585          * VCPUs.
2586          */
2587         kvm_mmu_zap_all(vcpu->kvm);
2588
2589         kvm_x86_ops->patch_hypercall(vcpu, instruction);
2590         if (emulator_write_emulated(rip, instruction, 3, vcpu)
2591             != X86EMUL_CONTINUE)
2592                 ret = -EFAULT;
2593
2594         return ret;
2595 }
2596
2597 static u64 mk_cr_64(u64 curr_cr, u32 new_val)
2598 {
2599         return (curr_cr & ~((1ULL << 32) - 1)) | new_val;
2600 }
2601
2602 void realmode_lgdt(struct kvm_vcpu *vcpu, u16 limit, unsigned long base)
2603 {
2604         struct descriptor_table dt = { limit, base };
2605
2606         kvm_x86_ops->set_gdt(vcpu, &dt);
2607 }
2608
2609 void realmode_lidt(struct kvm_vcpu *vcpu, u16 limit, unsigned long base)
2610 {
2611         struct descriptor_table dt = { limit, base };
2612
2613         kvm_x86_ops->set_idt(vcpu, &dt);
2614 }
2615
2616 void realmode_lmsw(struct kvm_vcpu *vcpu, unsigned long msw,
2617                    unsigned long *rflags)
2618 {
2619         kvm_lmsw(vcpu, msw);
2620         *rflags = kvm_x86_ops->get_rflags(vcpu);
2621 }
2622
2623 unsigned long realmode_get_cr(struct kvm_vcpu *vcpu, int cr)
2624 {
2625         unsigned long value;
2626
2627         kvm_x86_ops->decache_cr4_guest_bits(vcpu);
2628         switch (cr) {
2629         case 0:
2630                 value = vcpu->arch.cr0;
2631                 break;
2632         case 2:
2633                 value = vcpu->arch.cr2;
2634                 break;
2635         case 3:
2636                 value = vcpu->arch.cr3;
2637                 break;
2638         case 4:
2639                 value = vcpu->arch.cr4;
2640                 break;
2641         case 8:
2642                 value = kvm_get_cr8(vcpu);
2643                 break;
2644         default:
2645                 vcpu_printf(vcpu, "%s: unexpected cr %u\n", __func__, cr);
2646                 return 0;
2647         }
2648         KVMTRACE_3D(CR_READ, vcpu, (u32)cr, (u32)value,
2649                     (u32)((u64)value >> 32), handler);
2650
2651         return value;
2652 }
2653
2654 void realmode_set_cr(struct kvm_vcpu *vcpu, int cr, unsigned long val,
2655                      unsigned long *rflags)
2656 {
2657         KVMTRACE_3D(CR_WRITE, vcpu, (u32)cr, (u32)val,
2658                     (u32)((u64)val >> 32), handler);
2659
2660         switch (cr) {
2661         case 0:
2662                 kvm_set_cr0(vcpu, mk_cr_64(vcpu->arch.cr0, val));
2663                 *rflags = kvm_x86_ops->get_rflags(vcpu);
2664                 break;
2665         case 2:
2666                 vcpu->arch.cr2 = val;
2667                 break;
2668         case 3:
2669                 kvm_set_cr3(vcpu, val);
2670                 break;
2671         case 4:
2672                 kvm_set_cr4(vcpu, mk_cr_64(vcpu->arch.cr4, val));
2673                 break;
2674         case 8:
2675                 kvm_set_cr8(vcpu, val & 0xfUL);
2676                 break;
2677         default:
2678                 vcpu_printf(vcpu, "%s: unexpected cr %u\n", __func__, cr);
2679         }
2680 }
2681
2682 static int move_to_next_stateful_cpuid_entry(struct kvm_vcpu *vcpu, int i)
2683 {
2684         struct kvm_cpuid_entry2 *e = &vcpu->arch.cpuid_entries[i];
2685         int j, nent = vcpu->arch.cpuid_nent;
2686
2687         e->flags &= ~KVM_CPUID_FLAG_STATE_READ_NEXT;
2688         /* when no next entry is found, the current entry[i] is reselected */
2689         for (j = i + 1; j == i; j = (j + 1) % nent) {
2690                 struct kvm_cpuid_entry2 *ej = &vcpu->arch.cpuid_entries[j];
2691                 if (ej->function == e->function) {
2692                         ej->flags |= KVM_CPUID_FLAG_STATE_READ_NEXT;
2693                         return j;
2694                 }
2695         }
2696         return 0; /* silence gcc, even though control never reaches here */
2697 }
2698
2699 /* find an entry with matching function, matching index (if needed), and that
2700  * should be read next (if it's stateful) */
2701 static int is_matching_cpuid_entry(struct kvm_cpuid_entry2 *e,
2702         u32 function, u32 index)
2703 {
2704         if (e->function != function)
2705                 return 0;
2706         if ((e->flags & KVM_CPUID_FLAG_SIGNIFCANT_INDEX) && e->index != index)
2707                 return 0;
2708         if ((e->flags & KVM_CPUID_FLAG_STATEFUL_FUNC) &&
2709                 !(e->flags & KVM_CPUID_FLAG_STATE_READ_NEXT))
2710                 return 0;
2711         return 1;
2712 }
2713
2714 void kvm_emulate_cpuid(struct kvm_vcpu *vcpu)
2715 {
2716         int i;
2717         u32 function, index;
2718         struct kvm_cpuid_entry2 *e, *best;
2719
2720         function = kvm_register_read(vcpu, VCPU_REGS_RAX);
2721         index = kvm_register_read(vcpu, VCPU_REGS_RCX);
2722         kvm_register_write(vcpu, VCPU_REGS_RAX, 0);
2723         kvm_register_write(vcpu, VCPU_REGS_RBX, 0);
2724         kvm_register_write(vcpu, VCPU_REGS_RCX, 0);
2725         kvm_register_write(vcpu, VCPU_REGS_RDX, 0);
2726         best = NULL;
2727         for (i = 0; i < vcpu->arch.cpuid_nent; ++i) {
2728                 e = &vcpu->arch.cpuid_entries[i];
2729                 if (is_matching_cpuid_entry(e, function, index)) {
2730                         if (e->flags & KVM_CPUID_FLAG_STATEFUL_FUNC)
2731                                 move_to_next_stateful_cpuid_entry(vcpu, i);
2732                         best = e;
2733                         break;
2734                 }
2735                 /*
2736                  * Both basic or both extended?
2737                  */
2738                 if (((e->function ^ function) & 0x80000000) == 0)
2739                         if (!best || e->function > best->function)
2740                                 best = e;
2741         }
2742         if (best) {
2743                 kvm_register_write(vcpu, VCPU_REGS_RAX, best->eax);
2744                 kvm_register_write(vcpu, VCPU_REGS_RBX, best->ebx);
2745                 kvm_register_write(vcpu, VCPU_REGS_RCX, best->ecx);
2746                 kvm_register_write(vcpu, VCPU_REGS_RDX, best->edx);
2747         }
2748         kvm_x86_ops->skip_emulated_instruction(vcpu);
2749         KVMTRACE_5D(CPUID, vcpu, function,
2750                     (u32)kvm_register_read(vcpu, VCPU_REGS_RAX),
2751                     (u32)kvm_register_read(vcpu, VCPU_REGS_RBX),
2752                     (u32)kvm_register_read(vcpu, VCPU_REGS_RCX),
2753                     (u32)kvm_register_read(vcpu, VCPU_REGS_RDX), handler);
2754 }
2755 EXPORT_SYMBOL_GPL(kvm_emulate_cpuid);
2756
2757 /*
2758  * Check if userspace requested an interrupt window, and that the
2759  * interrupt window is open.
2760  *
2761  * No need to exit to userspace if we already have an interrupt queued.
2762  */
2763 static int dm_request_for_irq_injection(struct kvm_vcpu *vcpu,
2764                                           struct kvm_run *kvm_run)
2765 {
2766         return (!vcpu->arch.irq_summary &&
2767                 kvm_run->request_interrupt_window &&
2768                 vcpu->arch.interrupt_window_open &&
2769                 (kvm_x86_ops->get_rflags(vcpu) & X86_EFLAGS_IF));
2770 }
2771
2772 static void post_kvm_run_save(struct kvm_vcpu *vcpu,
2773                               struct kvm_run *kvm_run)
2774 {
2775         kvm_run->if_flag = (kvm_x86_ops->get_rflags(vcpu) & X86_EFLAGS_IF) != 0;
2776         kvm_run->cr8 = kvm_get_cr8(vcpu);
2777         kvm_run->apic_base = kvm_get_apic_base(vcpu);
2778         if (irqchip_in_kernel(vcpu->kvm))
2779                 kvm_run->ready_for_interrupt_injection = 1;
2780         else
2781                 kvm_run->ready_for_interrupt_injection =
2782                                         (vcpu->arch.interrupt_window_open &&
2783                                          vcpu->arch.irq_summary == 0);
2784 }
2785
2786 static void vapic_enter(struct kvm_vcpu *vcpu)
2787 {
2788         struct kvm_lapic *apic = vcpu->arch.apic;
2789         struct page *page;
2790
2791         if (!apic || !apic->vapic_addr)
2792                 return;
2793
2794         down_read(&current->mm->mmap_sem);
2795         page = gfn_to_page(vcpu->kvm, apic->vapic_addr >> PAGE_SHIFT);
2796         up_read(&current->mm->mmap_sem);
2797
2798         vcpu->arch.apic->vapic_page = page;
2799 }
2800
2801 static void vapic_exit(struct kvm_vcpu *vcpu)
2802 {
2803         struct kvm_lapic *apic = vcpu->arch.apic;
2804
2805         if (!apic || !apic->vapic_addr)
2806                 return;
2807
2808         down_read(&vcpu->kvm->slots_lock);
2809         kvm_release_page_dirty(apic->vapic_page);
2810         mark_page_dirty(vcpu->kvm, apic->vapic_addr >> PAGE_SHIFT);
2811         up_read(&vcpu->kvm->slots_lock);
2812 }
2813
2814 static int __vcpu_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
2815 {
2816         int r;
2817
2818         if (unlikely(vcpu->arch.mp_state == KVM_MP_STATE_SIPI_RECEIVED)) {
2819                 pr_debug("vcpu %d received sipi with vector # %x\n",
2820                        vcpu->vcpu_id, vcpu->arch.sipi_vector);
2821                 kvm_lapic_reset(vcpu);
2822                 r = kvm_x86_ops->vcpu_reset(vcpu);
2823                 if (r)
2824                         return r;
2825                 vcpu->arch.mp_state = KVM_MP_STATE_RUNNABLE;
2826         }
2827
2828         down_read(&vcpu->kvm->slots_lock);
2829         vapic_enter(vcpu);
2830
2831 preempted:
2832         if (vcpu->guest_debug.enabled)
2833                 kvm_x86_ops->guest_debug_pre(vcpu);
2834
2835 again:
2836         if (vcpu->requests)
2837                 if (test_and_clear_bit(KVM_REQ_MMU_RELOAD, &vcpu->requests))
2838                         kvm_mmu_unload(vcpu);
2839
2840         r = kvm_mmu_reload(vcpu);
2841         if (unlikely(r))
2842                 goto out;
2843
2844         if (vcpu->requests) {
2845                 if (test_and_clear_bit(KVM_REQ_MIGRATE_TIMER, &vcpu->requests))
2846                         __kvm_migrate_timers(vcpu);
2847                 if (test_and_clear_bit(KVM_REQ_TLB_FLUSH, &vcpu->requests))
2848                         kvm_x86_ops->tlb_flush(vcpu);
2849                 if (test_and_clear_bit(KVM_REQ_REPORT_TPR_ACCESS,
2850                                        &vcpu->requests)) {
2851                         kvm_run->exit_reason = KVM_EXIT_TPR_ACCESS;
2852                         r = 0;
2853                         goto out;
2854                 }
2855                 if (test_and_clear_bit(KVM_REQ_TRIPLE_FAULT, &vcpu->requests)) {
2856                         kvm_run->exit_reason = KVM_EXIT_SHUTDOWN;
2857                         r = 0;
2858                         goto out;
2859                 }
2860         }
2861
2862         clear_bit(KVM_REQ_PENDING_TIMER, &vcpu->requests);
2863         kvm_inject_pending_timer_irqs(vcpu);
2864
2865         preempt_disable();
2866
2867         kvm_x86_ops->prepare_guest_switch(vcpu);
2868         kvm_load_guest_fpu(vcpu);
2869
2870         local_irq_disable();
2871
2872         if (vcpu->requests || need_resched()) {
2873                 local_irq_enable();
2874                 preempt_enable();
2875                 r = 1;
2876                 goto out;
2877         }
2878
2879         if (signal_pending(current)) {
2880                 local_irq_enable();
2881                 preempt_enable();
2882                 r = -EINTR;
2883                 kvm_run->exit_reason = KVM_EXIT_INTR;
2884                 ++vcpu->stat.signal_exits;
2885                 goto out;
2886         }
2887
2888         vcpu->guest_mode = 1;
2889         /*
2890          * Make sure that guest_mode assignment won't happen after
2891          * testing the pending IRQ vector bitmap.
2892          */
2893         smp_wmb();
2894
2895         if (vcpu->arch.exception.pending)
2896                 __queue_exception(vcpu);
2897         else if (irqchip_in_kernel(vcpu->kvm))
2898                 kvm_x86_ops->inject_pending_irq(vcpu);
2899         else
2900                 kvm_x86_ops->inject_pending_vectors(vcpu, kvm_run);
2901
2902         kvm_lapic_sync_to_vapic(vcpu);
2903
2904         up_read(&vcpu->kvm->slots_lock);
2905
2906         kvm_guest_enter();
2907
2908
2909         KVMTRACE_0D(VMENTRY, vcpu, entryexit);
2910         kvm_x86_ops->run(vcpu, kvm_run);
2911
2912         vcpu->guest_mode = 0;
2913         local_irq_enable();
2914
2915         ++vcpu->stat.exits;
2916
2917         /*
2918          * We must have an instruction between local_irq_enable() and
2919          * kvm_guest_exit(), so the timer interrupt isn't delayed by
2920          * the interrupt shadow.  The stat.exits increment will do nicely.
2921          * But we need to prevent reordering, hence this barrier():
2922          */
2923         barrier();
2924
2925         kvm_guest_exit();
2926
2927         preempt_enable();
2928
2929         down_read(&vcpu->kvm->slots_lock);
2930
2931         /*
2932          * Profile KVM exit RIPs:
2933          */
2934         if (unlikely(prof_on == KVM_PROFILING)) {
2935                 unsigned long rip = kvm_rip_read(vcpu);
2936                 profile_hit(KVM_PROFILING, (void *)rip);
2937         }
2938
2939         if (vcpu->arch.exception.pending && kvm_x86_ops->exception_injected(vcpu))
2940                 vcpu->arch.exception.pending = false;
2941
2942         kvm_lapic_sync_from_vapic(vcpu);
2943
2944         r = kvm_x86_ops->handle_exit(kvm_run, vcpu);
2945
2946         if (r > 0) {
2947                 if (dm_request_for_irq_injection(vcpu, kvm_run)) {
2948                         r = -EINTR;
2949                         kvm_run->exit_reason = KVM_EXIT_INTR;
2950                         ++vcpu->stat.request_irq_exits;
2951                         goto out;
2952                 }
2953                 if (!need_resched())
2954                         goto again;
2955         }
2956
2957 out:
2958         up_read(&vcpu->kvm->slots_lock);
2959         if (r > 0) {
2960                 kvm_resched(vcpu);
2961                 down_read(&vcpu->kvm->slots_lock);
2962                 goto preempted;
2963         }
2964
2965         post_kvm_run_save(vcpu, kvm_run);
2966
2967         vapic_exit(vcpu);
2968
2969         return r;
2970 }
2971
2972 int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
2973 {
2974         int r;
2975         sigset_t sigsaved;
2976
2977         vcpu_load(vcpu);
2978
2979         if (vcpu->sigset_active)
2980                 sigprocmask(SIG_SETMASK, &vcpu->sigset, &sigsaved);
2981
2982         if (unlikely(vcpu->arch.mp_state == KVM_MP_STATE_UNINITIALIZED)) {
2983                 kvm_vcpu_block(vcpu);
2984                 r = -EAGAIN;
2985                 goto out;
2986         }
2987
2988         /* re-sync apic's tpr */
2989         if (!irqchip_in_kernel(vcpu->kvm))
2990                 kvm_set_cr8(vcpu, kvm_run->cr8);
2991
2992         if (vcpu->arch.pio.cur_count) {
2993                 r = complete_pio(vcpu);
2994                 if (r)
2995                         goto out;
2996         }
2997 #if CONFIG_HAS_IOMEM
2998         if (vcpu->mmio_needed) {
2999                 memcpy(vcpu->mmio_data, kvm_run->mmio.data, 8);
3000                 vcpu->mmio_read_completed = 1;
3001                 vcpu->mmio_needed = 0;
3002
3003                 down_read(&vcpu->kvm->slots_lock);
3004                 r = emulate_instruction(vcpu, kvm_run,
3005                                         vcpu->arch.mmio_fault_cr2, 0,
3006                                         EMULTYPE_NO_DECODE);
3007                 up_read(&vcpu->kvm->slots_lock);
3008                 if (r == EMULATE_DO_MMIO) {
3009                         /*
3010                          * Read-modify-write.  Back to userspace.
3011                          */
3012                         r = 0;
3013                         goto out;
3014                 }
3015         }
3016 #endif
3017         if (kvm_run->exit_reason == KVM_EXIT_HYPERCALL)
3018                 kvm_register_write(vcpu, VCPU_REGS_RAX,
3019                                      kvm_run->hypercall.ret);
3020
3021         r = __vcpu_run(vcpu, kvm_run);
3022
3023 out:
3024         if (vcpu->sigset_active)
3025                 sigprocmask(SIG_SETMASK, &sigsaved, NULL);
3026
3027         vcpu_put(vcpu);
3028         return r;
3029 }
3030
3031 int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
3032 {
3033         vcpu_load(vcpu);
3034
3035         regs->rax = kvm_register_read(vcpu, VCPU_REGS_RAX);
3036         regs->rbx = kvm_register_read(vcpu, VCPU_REGS_RBX);
3037         regs->rcx = kvm_register_read(vcpu, VCPU_REGS_RCX);
3038         regs->rdx = kvm_register_read(vcpu, VCPU_REGS_RDX);
3039         regs->rsi = kvm_register_read(vcpu, VCPU_REGS_RSI);
3040         regs->rdi = kvm_register_read(vcpu, VCPU_REGS_RDI);
3041         regs->rsp = kvm_register_read(vcpu, VCPU_REGS_RSP);
3042         regs->rbp = kvm_register_read(vcpu, VCPU_REGS_RBP);
3043 #ifdef CONFIG_X86_64
3044         regs->r8 = kvm_register_read(vcpu, VCPU_REGS_R8);
3045         regs->r9 = kvm_register_read(vcpu, VCPU_REGS_R9);
3046         regs->r10 = kvm_register_read(vcpu, VCPU_REGS_R10);
3047         regs->r11 = kvm_register_read(vcpu, VCPU_REGS_R11);
3048         regs->r12 = kvm_register_read(vcpu, VCPU_REGS_R12);
3049         regs->r13 = kvm_register_read(vcpu, VCPU_REGS_R13);
3050         regs->r14 = kvm_register_read(vcpu, VCPU_REGS_R14);
3051         regs->r15 = kvm_register_read(vcpu, VCPU_REGS_R15);
3052 #endif
3053
3054         regs->rip = kvm_rip_read(vcpu);
3055         regs->rflags = kvm_x86_ops->get_rflags(vcpu);
3056
3057         /*
3058          * Don't leak debug flags in case they were set for guest debugging
3059          */
3060         if (vcpu->guest_debug.enabled && vcpu->guest_debug.singlestep)
3061                 regs->rflags &= ~(X86_EFLAGS_TF | X86_EFLAGS_RF);
3062
3063         vcpu_put(vcpu);
3064
3065         return 0;
3066 }
3067
3068 int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
3069 {
3070         vcpu_load(vcpu);
3071
3072         kvm_register_write(vcpu, VCPU_REGS_RAX, regs->rax);
3073         kvm_register_write(vcpu, VCPU_REGS_RBX, regs->rbx);
3074         kvm_register_write(vcpu, VCPU_REGS_RCX, regs->rcx);
3075         kvm_register_write(vcpu, VCPU_REGS_RDX, regs->rdx);
3076         kvm_register_write(vcpu, VCPU_REGS_RSI, regs->rsi);
3077         kvm_register_write(vcpu, VCPU_REGS_RDI, regs->rdi);
3078         kvm_register_write(vcpu, VCPU_REGS_RSP, regs->rsp);
3079         kvm_register_write(vcpu, VCPU_REGS_RBP, regs->rbp);
3080 #ifdef CONFIG_X86_64
3081         kvm_register_write(vcpu, VCPU_REGS_R8, regs->r8);
3082         kvm_register_write(vcpu, VCPU_REGS_R9, regs->r9);
3083         kvm_register_write(vcpu, VCPU_REGS_R10, regs->r10);
3084         kvm_register_write(vcpu, VCPU_REGS_R11, regs->r11);
3085         kvm_register_write(vcpu, VCPU_REGS_R12, regs->r12);
3086         kvm_register_write(vcpu, VCPU_REGS_R13, regs->r13);
3087         kvm_register_write(vcpu, VCPU_REGS_R14, regs->r14);
3088         kvm_register_write(vcpu, VCPU_REGS_R15, regs->r15);
3089
3090 #endif
3091
3092         kvm_rip_write(vcpu, regs->rip);
3093         kvm_x86_ops->set_rflags(vcpu, regs->rflags);
3094
3095
3096         vcpu->arch.exception.pending = false;
3097
3098         vcpu_put(vcpu);
3099
3100         return 0;
3101 }
3102
3103 void kvm_get_segment(struct kvm_vcpu *vcpu,
3104                      struct kvm_segment *var, int seg)
3105 {
3106         kvm_x86_ops->get_segment(vcpu, var, seg);
3107 }
3108
3109 void kvm_get_cs_db_l_bits(struct kvm_vcpu *vcpu, int *db, int *l)
3110 {
3111         struct kvm_segment cs;
3112
3113         kvm_get_segment(vcpu, &cs, VCPU_SREG_CS);
3114         *db = cs.db;
3115         *l = cs.l;
3116 }
3117 EXPORT_SYMBOL_GPL(kvm_get_cs_db_l_bits);
3118
3119 int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu,
3120                                   struct kvm_sregs *sregs)
3121 {
3122         struct descriptor_table dt;
3123         int pending_vec;
3124
3125         vcpu_load(vcpu);
3126
3127         kvm_get_segment(vcpu, &sregs->cs, VCPU_SREG_CS);
3128         kvm_get_segment(vcpu, &sregs->ds, VCPU_SREG_DS);
3129         kvm_get_segment(vcpu, &sregs->es, VCPU_SREG_ES);
3130         kvm_get_segment(vcpu, &sregs->fs, VCPU_SREG_FS);
3131         kvm_get_segment(vcpu, &sregs->gs, VCPU_SREG_GS);
3132         kvm_get_segment(vcpu, &sregs->ss, VCPU_SREG_SS);
3133
3134         kvm_get_segment(vcpu, &sregs->tr, VCPU_SREG_TR);
3135         kvm_get_segment(vcpu, &sregs->ldt, VCPU_SREG_LDTR);
3136
3137         kvm_x86_ops->get_idt(vcpu, &dt);
3138         sregs->idt.limit = dt.limit;
3139         sregs->idt.base = dt.base;
3140         kvm_x86_ops->get_gdt(vcpu, &dt);
3141         sregs->gdt.limit = dt.limit;
3142         sregs->gdt.base = dt.base;
3143
3144         kvm_x86_ops->decache_cr4_guest_bits(vcpu);
3145         sregs->cr0 = vcpu->arch.cr0;
3146         sregs->cr2 = vcpu->arch.cr2;
3147         sregs->cr3 = vcpu->arch.cr3;
3148         sregs->cr4 = vcpu->arch.cr4;
3149         sregs->cr8 = kvm_get_cr8(vcpu);
3150         sregs->efer = vcpu->arch.shadow_efer;
3151         sregs->apic_base = kvm_get_apic_base(vcpu);
3152
3153         if (irqchip_in_kernel(vcpu->kvm)) {
3154                 memset(sregs->interrupt_bitmap, 0,
3155                        sizeof sregs->interrupt_bitmap);
3156                 pending_vec = kvm_x86_ops->get_irq(vcpu);
3157                 if (pending_vec >= 0)
3158                         set_bit(pending_vec,
3159                                 (unsigned long *)sregs->interrupt_bitmap);
3160         } else
3161                 memcpy(sregs->interrupt_bitmap, vcpu->arch.irq_pending,
3162                        sizeof sregs->interrupt_bitmap);
3163
3164         vcpu_put(vcpu);
3165
3166         return 0;
3167 }
3168
3169 int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu,
3170                                     struct kvm_mp_state *mp_state)
3171 {
3172         vcpu_load(vcpu);
3173         mp_state->mp_state = vcpu->arch.mp_state;
3174         vcpu_put(vcpu);
3175         return 0;
3176 }
3177
3178 int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu,
3179                                     struct kvm_mp_state *mp_state)
3180 {
3181         vcpu_load(vcpu);
3182         vcpu->arch.mp_state = mp_state->mp_state;
3183         vcpu_put(vcpu);
3184         return 0;
3185 }
3186
3187 static void kvm_set_segment(struct kvm_vcpu *vcpu,
3188                         struct kvm_segment *var, int seg)
3189 {
3190         kvm_x86_ops->set_segment(vcpu, var, seg);
3191 }
3192
3193 static void seg_desct_to_kvm_desct(struct desc_struct *seg_desc, u16 selector,
3194                                    struct kvm_segment *kvm_desct)
3195 {
3196         kvm_desct->base = seg_desc->base0;
3197         kvm_desct->base |= seg_desc->base1 << 16;
3198         kvm_desct->base |= seg_desc->base2 << 24;
3199         kvm_desct->limit = seg_desc->limit0;
3200         kvm_desct->limit |= seg_desc->limit << 16;
3201         if (seg_desc->g) {
3202                 kvm_desct->limit <<= 12;
3203                 kvm_desct->limit |= 0xfff;
3204         }
3205         kvm_desct->selector = selector;
3206         kvm_desct->type = seg_desc->type;
3207         kvm_desct->present = seg_desc->p;
3208         kvm_desct->dpl = seg_desc->dpl;
3209         kvm_desct->db = seg_desc->d;
3210         kvm_desct->s = seg_desc->s;
3211         kvm_desct->l = seg_desc->l;
3212         kvm_desct->g = seg_desc->g;
3213         kvm_desct->avl = seg_desc->avl;
3214         if (!selector)
3215                 kvm_desct->unusable = 1;
3216         else
3217                 kvm_desct->unusable = 0;
3218         kvm_desct->padding = 0;
3219 }
3220
3221 static void get_segment_descritptor_dtable(struct kvm_vcpu *vcpu,
3222                                            u16 selector,
3223                                            struct descriptor_table *dtable)
3224 {
3225         if (selector & 1 << 2) {
3226                 struct kvm_segment kvm_seg;
3227
3228                 kvm_get_segment(vcpu, &kvm_seg, VCPU_SREG_LDTR);
3229
3230                 if (kvm_seg.unusable)
3231                         dtable->limit = 0;
3232                 else
3233                         dtable->limit = kvm_seg.limit;
3234                 dtable->base = kvm_seg.base;
3235         }
3236         else
3237                 kvm_x86_ops->get_gdt(vcpu, dtable);
3238 }
3239
3240 /* allowed just for 8 bytes segments */
3241 static int load_guest_segment_descriptor(struct kvm_vcpu *vcpu, u16 selector,
3242                                          struct desc_struct *seg_desc)
3243 {
3244         gpa_t gpa;
3245         struct descriptor_table dtable;
3246         u16 index = selector >> 3;
3247
3248         get_segment_descritptor_dtable(vcpu, selector, &dtable);
3249
3250         if (dtable.limit < index * 8 + 7) {
3251                 kvm_queue_exception_e(vcpu, GP_VECTOR, selector & 0xfffc);
3252                 return 1;
3253         }
3254         gpa = vcpu->arch.mmu.gva_to_gpa(vcpu, dtable.base);
3255         gpa += index * 8;
3256         return kvm_read_guest(vcpu->kvm, gpa, seg_desc, 8);
3257 }
3258
3259 /* allowed just for 8 bytes segments */
3260 static int save_guest_segment_descriptor(struct kvm_vcpu *vcpu, u16 selector,
3261                                          struct desc_struct *seg_desc)
3262 {
3263         gpa_t gpa;
3264         struct descriptor_table dtable;
3265         u16 index = selector >> 3;
3266
3267         get_segment_descritptor_dtable(vcpu, selector, &dtable);
3268
3269         if (dtable.limit < index * 8 + 7)
3270                 return 1;
3271         gpa = vcpu->arch.mmu.gva_to_gpa(vcpu, dtable.base);
3272         gpa += index * 8;
3273         return kvm_write_guest(vcpu->kvm, gpa, seg_desc, 8);
3274 }
3275
3276 static u32 get_tss_base_addr(struct kvm_vcpu *vcpu,
3277                              struct desc_struct *seg_desc)
3278 {
3279         u32 base_addr;
3280
3281         base_addr = seg_desc->base0;
3282         base_addr |= (seg_desc->base1 << 16);
3283         base_addr |= (seg_desc->base2 << 24);
3284
3285         return vcpu->arch.mmu.gva_to_gpa(vcpu, base_addr);
3286 }
3287
3288 static u16 get_segment_selector(struct kvm_vcpu *vcpu, int seg)
3289 {
3290         struct kvm_segment kvm_seg;
3291
3292         kvm_get_segment(vcpu, &kvm_seg, seg);
3293         return kvm_seg.selector;
3294 }
3295
3296 static int load_segment_descriptor_to_kvm_desct(struct kvm_vcpu *vcpu,
3297                                                 u16 selector,
3298                                                 struct kvm_segment *kvm_seg)
3299 {
3300         struct desc_struct seg_desc;
3301
3302         if (load_guest_segment_descriptor(vcpu, selector, &seg_desc))
3303                 return 1;
3304         seg_desct_to_kvm_desct(&seg_desc, selector, kvm_seg);
3305         return 0;
3306 }
3307
3308 int kvm_load_segment_descriptor(struct kvm_vcpu *vcpu, u16 selector,
3309                                 int type_bits, int seg)
3310 {
3311         struct kvm_segment kvm_seg;
3312
3313         if (load_segment_descriptor_to_kvm_desct(vcpu, selector, &kvm_seg))
3314                 return 1;
3315         kvm_seg.type |= type_bits;
3316
3317         if (seg != VCPU_SREG_SS && seg != VCPU_SREG_CS &&
3318             seg != VCPU_SREG_LDTR)
3319                 if (!kvm_seg.s)
3320                         kvm_seg.unusable = 1;
3321
3322         kvm_set_segment(vcpu, &kvm_seg, seg);
3323         return 0;
3324 }
3325
3326 static void save_state_to_tss32(struct kvm_vcpu *vcpu,
3327                                 struct tss_segment_32 *tss)
3328 {
3329         tss->cr3 = vcpu->arch.cr3;
3330         tss->eip = kvm_rip_read(vcpu);
3331         tss->eflags = kvm_x86_ops->get_rflags(vcpu);
3332         tss->eax = kvm_register_read(vcpu, VCPU_REGS_RAX);
3333         tss->ecx = kvm_register_read(vcpu, VCPU_REGS_RCX);
3334         tss->edx = kvm_register_read(vcpu, VCPU_REGS_RDX);
3335         tss->ebx = kvm_register_read(vcpu, VCPU_REGS_RBX);
3336         tss->esp = kvm_register_read(vcpu, VCPU_REGS_RSP);
3337         tss->ebp = kvm_register_read(vcpu, VCPU_REGS_RBP);
3338         tss->esi = kvm_register_read(vcpu, VCPU_REGS_RSI);
3339         tss->edi = kvm_register_read(vcpu, VCPU_REGS_RDI);
3340         tss->es = get_segment_selector(vcpu, VCPU_SREG_ES);
3341         tss->cs = get_segment_selector(vcpu, VCPU_SREG_CS);
3342         tss->ss = get_segment_selector(vcpu, VCPU_SREG_SS);
3343         tss->ds = get_segment_selector(vcpu, VCPU_SREG_DS);
3344         tss->fs = get_segment_selector(vcpu, VCPU_SREG_FS);
3345         tss->gs = get_segment_selector(vcpu, VCPU_SREG_GS);
3346         tss->ldt_selector = get_segment_selector(vcpu, VCPU_SREG_LDTR);
3347         tss->prev_task_link = get_segment_selector(vcpu, VCPU_SREG_TR);
3348 }
3349
3350 static int load_state_from_tss32(struct kvm_vcpu *vcpu,
3351                                   struct tss_segment_32 *tss)
3352 {
3353         kvm_set_cr3(vcpu, tss->cr3);
3354
3355         kvm_rip_write(vcpu, tss->eip);
3356         kvm_x86_ops->set_rflags(vcpu, tss->eflags | 2);
3357
3358         kvm_register_write(vcpu, VCPU_REGS_RAX, tss->eax);
3359         kvm_register_write(vcpu, VCPU_REGS_RCX, tss->ecx);
3360         kvm_register_write(vcpu, VCPU_REGS_RDX, tss->edx);
3361         kvm_register_write(vcpu, VCPU_REGS_RBX, tss->ebx);
3362         kvm_register_write(vcpu, VCPU_REGS_RSP, tss->esp);
3363         kvm_register_write(vcpu, VCPU_REGS_RBP, tss->ebp);
3364         kvm_register_write(vcpu, VCPU_REGS_RSI, tss->esi);
3365         kvm_register_write(vcpu, VCPU_REGS_RDI, tss->edi);
3366
3367         if (kvm_load_segment_descriptor(vcpu, tss->ldt_selector, 0, VCPU_SREG_LDTR))
3368                 return 1;
3369
3370         if (kvm_load_segment_descriptor(vcpu, tss->es, 1, VCPU_SREG_ES))
3371                 return 1;
3372
3373         if (kvm_load_segment_descriptor(vcpu, tss->cs, 9, VCPU_SREG_CS))
3374                 return 1;
3375
3376         if (kvm_load_segment_descriptor(vcpu, tss->ss, 1, VCPU_SREG_SS))
3377                 return 1;
3378
3379         if (kvm_load_segment_descriptor(vcpu, tss->ds, 1, VCPU_SREG_DS))
3380                 return 1;
3381
3382         if (kvm_load_segment_descriptor(vcpu, tss->fs, 1, VCPU_SREG_FS))
3383                 return 1;
3384
3385         if (kvm_load_segment_descriptor(vcpu, tss->gs, 1, VCPU_SREG_GS))
3386                 return 1;
3387         return 0;
3388 }
3389
3390 static void save_state_to_tss16(struct kvm_vcpu *vcpu,
3391                                 struct tss_segment_16 *tss)
3392 {
3393         tss->ip = kvm_rip_read(vcpu);
3394         tss->flag = kvm_x86_ops->get_rflags(vcpu);
3395         tss->ax = kvm_register_read(vcpu, VCPU_REGS_RAX);
3396         tss->cx = kvm_register_read(vcpu, VCPU_REGS_RCX);
3397         tss->dx = kvm_register_read(vcpu, VCPU_REGS_RDX);
3398         tss->bx = kvm_register_read(vcpu, VCPU_REGS_RBX);
3399         tss->sp = kvm_register_read(vcpu, VCPU_REGS_RSP);
3400         tss->bp = kvm_register_read(vcpu, VCPU_REGS_RBP);
3401         tss->si = kvm_register_read(vcpu, VCPU_REGS_RSI);
3402         tss->di = kvm_register_read(vcpu, VCPU_REGS_RDI);
3403
3404         tss->es = get_segment_selector(vcpu, VCPU_SREG_ES);
3405         tss->cs = get_segment_selector(vcpu, VCPU_SREG_CS);
3406         tss->ss = get_segment_selector(vcpu, VCPU_SREG_SS);
3407         tss->ds = get_segment_selector(vcpu, VCPU_SREG_DS);
3408         tss->ldt = get_segment_selector(vcpu, VCPU_SREG_LDTR);
3409         tss->prev_task_link = get_segment_selector(vcpu, VCPU_SREG_TR);
3410 }
3411
3412 static int load_state_from_tss16(struct kvm_vcpu *vcpu,
3413                                  struct tss_segment_16 *tss)
3414 {
3415         kvm_rip_write(vcpu, tss->ip);
3416         kvm_x86_ops->set_rflags(vcpu, tss->flag | 2);
3417         kvm_register_write(vcpu, VCPU_REGS_RAX, tss->ax);
3418         kvm_register_write(vcpu, VCPU_REGS_RCX, tss->cx);
3419         kvm_register_write(vcpu, VCPU_REGS_RDX, tss->dx);
3420         kvm_register_write(vcpu, VCPU_REGS_RBX, tss->bx);
3421         kvm_register_write(vcpu, VCPU_REGS_RSP, tss->sp);
3422         kvm_register_write(vcpu, VCPU_REGS_RBP, tss->bp);
3423         kvm_register_write(vcpu, VCPU_REGS_RSI, tss->si);
3424         kvm_register_write(vcpu, VCPU_REGS_RDI, tss->di);
3425
3426         if (kvm_load_segment_descriptor(vcpu, tss->ldt, 0, VCPU_SREG_LDTR))
3427                 return 1;
3428
3429         if (kvm_load_segment_descriptor(vcpu, tss->es, 1, VCPU_SREG_ES))
3430                 return 1;
3431
3432         if (kvm_load_segment_descriptor(vcpu, tss->cs, 9, VCPU_SREG_CS))
3433                 return 1;
3434
3435         if (kvm_load_segment_descriptor(vcpu, tss->ss, 1, VCPU_SREG_SS))
3436                 return 1;
3437
3438         if (kvm_load_segment_descriptor(vcpu, tss->ds, 1, VCPU_SREG_DS))
3439                 return 1;
3440         return 0;
3441 }
3442
3443 static int kvm_task_switch_16(struct kvm_vcpu *vcpu, u16 tss_selector,
3444                        u32 old_tss_base,
3445                        struct desc_struct *nseg_desc)
3446 {
3447         struct tss_segment_16 tss_segment_16;
3448         int ret = 0;
3449
3450         if (kvm_read_guest(vcpu->kvm, old_tss_base, &tss_segment_16,
3451                            sizeof tss_segment_16))
3452                 goto out;
3453
3454         save_state_to_tss16(vcpu, &tss_segment_16);
3455
3456         if (kvm_write_guest(vcpu->kvm, old_tss_base, &tss_segment_16,
3457                             sizeof tss_segment_16))
3458                 goto out;
3459
3460         if (kvm_read_guest(vcpu->kvm, get_tss_base_addr(vcpu, nseg_desc),
3461                            &tss_segment_16, sizeof tss_segment_16))
3462                 goto out;
3463
3464         if (load_state_from_tss16(vcpu, &tss_segment_16))
3465                 goto out;
3466
3467         ret = 1;
3468 out:
3469         return ret;
3470 }
3471
3472 static int kvm_task_switch_32(struct kvm_vcpu *vcpu, u16 tss_selector,
3473                        u32 old_tss_base,
3474                        struct desc_struct *nseg_desc)
3475 {
3476         struct tss_segment_32 tss_segment_32;
3477         int ret = 0;
3478
3479         if (kvm_read_guest(vcpu->kvm, old_tss_base, &tss_segment_32,
3480                            sizeof tss_segment_32))
3481                 goto out;
3482
3483         save_state_to_tss32(vcpu, &tss_segment_32);
3484
3485         if (kvm_write_guest(vcpu->kvm, old_tss_base, &tss_segment_32,
3486                             sizeof tss_segment_32))
3487                 goto out;
3488
3489         if (kvm_read_guest(vcpu->kvm, get_tss_base_addr(vcpu, nseg_desc),
3490                            &tss_segment_32, sizeof tss_segment_32))
3491                 goto out;
3492
3493         if (load_state_from_tss32(vcpu, &tss_segment_32))
3494                 goto out;
3495
3496         ret = 1;
3497 out:
3498         return ret;
3499 }
3500
3501 int kvm_task_switch(struct kvm_vcpu *vcpu, u16 tss_selector, int reason)
3502 {
3503         struct kvm_segment tr_seg;
3504         struct desc_struct cseg_desc;
3505         struct desc_struct nseg_desc;
3506         int ret = 0;
3507         u32 old_tss_base = get_segment_base(vcpu, VCPU_SREG_TR);
3508         u16 old_tss_sel = get_segment_selector(vcpu, VCPU_SREG_TR);
3509
3510         old_tss_base = vcpu->arch.mmu.gva_to_gpa(vcpu, old_tss_base);
3511
3512         /* FIXME: Handle errors. Failure to read either TSS or their
3513          * descriptors should generate a pagefault.
3514          */
3515         if (load_guest_segment_descriptor(vcpu, tss_selector, &nseg_desc))
3516                 goto out;
3517
3518         if (load_guest_segment_descriptor(vcpu, old_tss_sel, &cseg_desc))
3519                 goto out;
3520
3521         if (reason != TASK_SWITCH_IRET) {
3522                 int cpl;
3523
3524                 cpl = kvm_x86_ops->get_cpl(vcpu);
3525                 if ((tss_selector & 3) > nseg_desc.dpl || cpl > nseg_desc.dpl) {
3526                         kvm_queue_exception_e(vcpu, GP_VECTOR, 0);
3527                         return 1;
3528                 }
3529         }
3530
3531         if (!nseg_desc.p || (nseg_desc.limit0 | nseg_desc.limit << 16) < 0x67) {
3532                 kvm_queue_exception_e(vcpu, TS_VECTOR, tss_selector & 0xfffc);
3533                 return 1;
3534         }
3535
3536         if (reason == TASK_SWITCH_IRET || reason == TASK_SWITCH_JMP) {
3537                 cseg_desc.type &= ~(1 << 1); //clear the B flag
3538                 save_guest_segment_descriptor(vcpu, old_tss_sel, &cseg_desc);
3539         }
3540
3541         if (reason == TASK_SWITCH_IRET) {
3542                 u32 eflags = kvm_x86_ops->get_rflags(vcpu);
3543                 kvm_x86_ops->set_rflags(vcpu, eflags & ~X86_EFLAGS_NT);
3544         }
3545
3546         kvm_x86_ops->skip_emulated_instruction(vcpu);
3547
3548         if (nseg_desc.type & 8)
3549                 ret = kvm_task_switch_32(vcpu, tss_selector, old_tss_base,
3550                                          &nseg_desc);
3551         else
3552                 ret = kvm_task_switch_16(vcpu, tss_selector, old_tss_base,
3553                                          &nseg_desc);
3554
3555         if (reason == TASK_SWITCH_CALL || reason == TASK_SWITCH_GATE) {
3556                 u32 eflags = kvm_x86_ops->get_rflags(vcpu);
3557                 kvm_x86_ops->set_rflags(vcpu, eflags | X86_EFLAGS_NT);
3558         }
3559
3560         if (reason != TASK_SWITCH_IRET) {
3561                 nseg_desc.type |= (1 << 1);
3562                 save_guest_segment_descriptor(vcpu, tss_selector,
3563                                               &nseg_desc);
3564         }
3565
3566         kvm_x86_ops->set_cr0(vcpu, vcpu->arch.cr0 | X86_CR0_TS);
3567         seg_desct_to_kvm_desct(&nseg_desc, tss_selector, &tr_seg);
3568         tr_seg.type = 11;
3569         kvm_set_segment(vcpu, &tr_seg, VCPU_SREG_TR);
3570 out:
3571         return ret;
3572 }
3573 EXPORT_SYMBOL_GPL(kvm_task_switch);
3574
3575 int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu,
3576                                   struct kvm_sregs *sregs)
3577 {
3578         int mmu_reset_needed = 0;
3579         int i, pending_vec, max_bits;
3580         struct descriptor_table dt;
3581
3582         vcpu_load(vcpu);
3583
3584         dt.limit = sregs->idt.limit;
3585         dt.base = sregs->idt.base;
3586         kvm_x86_ops->set_idt(vcpu, &dt);
3587         dt.limit = sregs->gdt.limit;
3588         dt.base = sregs->gdt.base;
3589         kvm_x86_ops->set_gdt(vcpu, &dt);
3590
3591         vcpu->arch.cr2 = sregs->cr2;
3592         mmu_reset_needed |= vcpu->arch.cr3 != sregs->cr3;
3593         vcpu->arch.cr3 = sregs->cr3;
3594
3595         kvm_set_cr8(vcpu, sregs->cr8);
3596
3597         mmu_reset_needed |= vcpu->arch.shadow_efer != sregs->efer;
3598         kvm_x86_ops->set_efer(vcpu, sregs->efer);
3599         kvm_set_apic_base(vcpu, sregs->apic_base);
3600
3601         kvm_x86_ops->decache_cr4_guest_bits(vcpu);
3602
3603         mmu_reset_needed |= vcpu->arch.cr0 != sregs->cr0;
3604         kvm_x86_ops->set_cr0(vcpu, sregs->cr0);
3605         vcpu->arch.cr0 = sregs->cr0;
3606
3607         mmu_reset_needed |= vcpu->arch.cr4 != sregs->cr4;
3608         kvm_x86_ops->set_cr4(vcpu, sregs->cr4);
3609         if (!is_long_mode(vcpu) && is_pae(vcpu))
3610                 load_pdptrs(vcpu, vcpu->arch.cr3);
3611
3612         if (mmu_reset_needed)
3613                 kvm_mmu_reset_context(vcpu);
3614
3615         if (!irqchip_in_kernel(vcpu->kvm)) {
3616                 memcpy(vcpu->arch.irq_pending, sregs->interrupt_bitmap,
3617                        sizeof vcpu->arch.irq_pending);
3618                 vcpu->arch.irq_summary = 0;
3619                 for (i = 0; i < ARRAY_SIZE(vcpu->arch.irq_pending); ++i)
3620                         if (vcpu->arch.irq_pending[i])
3621                                 __set_bit(i, &vcpu->arch.irq_summary);
3622         } else {
3623                 max_bits = (sizeof sregs->interrupt_bitmap) << 3;
3624                 pending_vec = find_first_bit(
3625                         (const unsigned long *)sregs->interrupt_bitmap,
3626                         max_bits);
3627                 /* Only pending external irq is handled here */
3628                 if (pending_vec < max_bits) {
3629                         kvm_x86_ops->set_irq(vcpu, pending_vec);
3630                         pr_debug("Set back pending irq %d\n",
3631                                  pending_vec);
3632                 }
3633         }
3634
3635         kvm_set_segment(vcpu, &sregs->cs, VCPU_SREG_CS);
3636         kvm_set_segment(vcpu, &sregs->ds, VCPU_SREG_DS);
3637         kvm_set_segment(vcpu, &sregs->es, VCPU_SREG_ES);
3638         kvm_set_segment(vcpu, &sregs->fs, VCPU_SREG_FS);
3639         kvm_set_segment(vcpu, &sregs->gs, VCPU_SREG_GS);
3640         kvm_set_segment(vcpu, &sregs->ss, VCPU_SREG_SS);
3641
3642         kvm_set_segment(vcpu, &sregs->tr, VCPU_SREG_TR);
3643         kvm_set_segment(vcpu, &sregs->ldt, VCPU_SREG_LDTR);
3644
3645         vcpu_put(vcpu);
3646
3647         return 0;
3648 }
3649
3650 int kvm_arch_vcpu_ioctl_debug_guest(struct kvm_vcpu *vcpu,
3651                                     struct kvm_debug_guest *dbg)
3652 {
3653         int r;
3654
3655         vcpu_load(vcpu);
3656
3657         r = kvm_x86_ops->set_guest_debug(vcpu, dbg);
3658
3659         vcpu_put(vcpu);
3660
3661         return r;
3662 }
3663
3664 /*
3665  * fxsave fpu state.  Taken from x86_64/processor.h.  To be killed when
3666  * we have asm/x86/processor.h
3667  */
3668 struct fxsave {
3669         u16     cwd;
3670         u16     swd;
3671         u16     twd;
3672         u16     fop;
3673         u64     rip;
3674         u64     rdp;
3675         u32     mxcsr;
3676         u32     mxcsr_mask;
3677         u32     st_space[32];   /* 8*16 bytes for each FP-reg = 128 bytes */
3678 #ifdef CONFIG_X86_64
3679         u32     xmm_space[64];  /* 16*16 bytes for each XMM-reg = 256 bytes */
3680 #else
3681         u32     xmm_space[32];  /* 8*16 bytes for each XMM-reg = 128 bytes */
3682 #endif
3683 };
3684
3685 /*
3686  * Translate a guest virtual address to a guest physical address.
3687  */
3688 int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu *vcpu,
3689                                     struct kvm_translation *tr)
3690 {
3691         unsigned long vaddr = tr->linear_address;
3692         gpa_t gpa;
3693
3694         vcpu_load(vcpu);
3695         down_read(&vcpu->kvm->slots_lock);
3696         gpa = vcpu->arch.mmu.gva_to_gpa(vcpu, vaddr);
3697         up_read(&vcpu->kvm->slots_lock);
3698         tr->physical_address = gpa;
3699         tr->valid = gpa != UNMAPPED_GVA;
3700         tr->writeable = 1;
3701         tr->usermode = 0;
3702         vcpu_put(vcpu);
3703
3704         return 0;
3705 }
3706
3707 int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
3708 {
3709         struct fxsave *fxsave = (struct fxsave *)&vcpu->arch.guest_fx_image;
3710
3711         vcpu_load(vcpu);
3712
3713         memcpy(fpu->fpr, fxsave->st_space, 128);
3714         fpu->fcw = fxsave->cwd;
3715         fpu->fsw = fxsave->swd;
3716         fpu->ftwx = fxsave->twd;
3717         fpu->last_opcode = fxsave->fop;
3718         fpu->last_ip = fxsave->rip;
3719         fpu->last_dp = fxsave->rdp;
3720         memcpy(fpu->xmm, fxsave->xmm_space, sizeof fxsave->xmm_space);
3721
3722         vcpu_put(vcpu);
3723
3724         return 0;
3725 }
3726
3727 int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
3728 {
3729         struct fxsave *fxsave = (struct fxsave *)&vcpu->arch.guest_fx_image;
3730
3731         vcpu_load(vcpu);
3732
3733         memcpy(fxsave->st_space, fpu->fpr, 128);
3734         fxsave->cwd = fpu->fcw;
3735         fxsave->swd = fpu->fsw;
3736         fxsave->twd = fpu->ftwx;
3737         fxsave->fop = fpu->last_opcode;
3738         fxsave->rip = fpu->last_ip;
3739         fxsave->rdp = fpu->last_dp;
3740         memcpy(fxsave->xmm_space, fpu->xmm, sizeof fxsave->xmm_space);
3741
3742         vcpu_put(vcpu);
3743
3744         return 0;
3745 }
3746
3747 void fx_init(struct kvm_vcpu *vcpu)
3748 {
3749         unsigned after_mxcsr_mask;
3750
3751         /*
3752          * Touch the fpu the first time in non atomic context as if
3753          * this is the first fpu instruction the exception handler
3754          * will fire before the instruction returns and it'll have to
3755          * allocate ram with GFP_KERNEL.
3756          */
3757         if (!used_math())
3758                 kvm_fx_save(&vcpu->arch.host_fx_image);
3759
3760         /* Initialize guest FPU by resetting ours and saving into guest's */
3761         preempt_disable();
3762         kvm_fx_save(&vcpu->arch.host_fx_image);
3763         kvm_fx_finit();
3764         kvm_fx_save(&vcpu->arch.guest_fx_image);
3765         kvm_fx_restore(&vcpu->arch.host_fx_image);
3766         preempt_enable();
3767
3768         vcpu->arch.cr0 |= X86_CR0_ET;
3769         after_mxcsr_mask = offsetof(struct i387_fxsave_struct, st_space);
3770         vcpu->arch.guest_fx_image.mxcsr = 0x1f80;
3771         memset((void *)&vcpu->arch.guest_fx_image + after_mxcsr_mask,
3772                0, sizeof(struct i387_fxsave_struct) - after_mxcsr_mask);
3773 }
3774 EXPORT_SYMBOL_GPL(fx_init);
3775
3776 void kvm_load_guest_fpu(struct kvm_vcpu *vcpu)
3777 {
3778         if (!vcpu->fpu_active || vcpu->guest_fpu_loaded)
3779                 return;
3780
3781         vcpu->guest_fpu_loaded = 1;
3782         kvm_fx_save(&vcpu->arch.host_fx_image);
3783         kvm_fx_restore(&vcpu->arch.guest_fx_image);
3784 }
3785 EXPORT_SYMBOL_GPL(kvm_load_guest_fpu);
3786
3787 void kvm_put_guest_fpu(struct kvm_vcpu *vcpu)
3788 {
3789         if (!vcpu->guest_fpu_loaded)
3790                 return;
3791
3792         vcpu->guest_fpu_loaded = 0;
3793         kvm_fx_save(&vcpu->arch.guest_fx_image);
3794         kvm_fx_restore(&vcpu->arch.host_fx_image);
3795         ++vcpu->stat.fpu_reload;
3796 }
3797 EXPORT_SYMBOL_GPL(kvm_put_guest_fpu);
3798
3799 void kvm_arch_vcpu_free(struct kvm_vcpu *vcpu)
3800 {
3801         kvm_x86_ops->vcpu_free(vcpu);
3802 }
3803
3804 struct kvm_vcpu *kvm_arch_vcpu_create(struct kvm *kvm,
3805                                                 unsigned int id)
3806 {
3807         return kvm_x86_ops->vcpu_create(kvm, id);
3808 }
3809
3810 int kvm_arch_vcpu_setup(struct kvm_vcpu *vcpu)
3811 {
3812         int r;
3813
3814         /* We do fxsave: this must be aligned. */
3815         BUG_ON((unsigned long)&vcpu->arch.host_fx_image & 0xF);
3816
3817         vcpu_load(vcpu);
3818         r = kvm_arch_vcpu_reset(vcpu);
3819         if (r == 0)
3820                 r = kvm_mmu_setup(vcpu);
3821         vcpu_put(vcpu);
3822         if (r < 0)
3823                 goto free_vcpu;
3824
3825         return 0;
3826 free_vcpu:
3827         kvm_x86_ops->vcpu_free(vcpu);
3828         return r;
3829 }
3830
3831 void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu)
3832 {
3833         vcpu_load(vcpu);
3834         kvm_mmu_unload(vcpu);
3835         vcpu_put(vcpu);
3836
3837         kvm_x86_ops->vcpu_free(vcpu);
3838 }
3839
3840 int kvm_arch_vcpu_reset(struct kvm_vcpu *vcpu)
3841 {
3842         return kvm_x86_ops->vcpu_reset(vcpu);
3843 }
3844
3845 void kvm_arch_hardware_enable(void *garbage)
3846 {
3847         kvm_x86_ops->hardware_enable(garbage);
3848 }
3849
3850 void kvm_arch_hardware_disable(void *garbage)
3851 {
3852         kvm_x86_ops->hardware_disable(garbage);
3853 }
3854
3855 int kvm_arch_hardware_setup(void)
3856 {
3857         return kvm_x86_ops->hardware_setup();
3858 }
3859
3860 void kvm_arch_hardware_unsetup(void)
3861 {
3862         kvm_x86_ops->hardware_unsetup();
3863 }
3864
3865 void kvm_arch_check_processor_compat(void *rtn)
3866 {
3867         kvm_x86_ops->check_processor_compatibility(rtn);
3868 }
3869
3870 int kvm_arch_vcpu_init(struct kvm_vcpu *vcpu)
3871 {
3872         struct page *page;
3873         struct kvm *kvm;
3874         int r;
3875
3876         BUG_ON(vcpu->kvm == NULL);
3877         kvm = vcpu->kvm;
3878
3879         vcpu->arch.mmu.root_hpa = INVALID_PAGE;
3880         if (!irqchip_in_kernel(kvm) || vcpu->vcpu_id == 0)
3881                 vcpu->arch.mp_state = KVM_MP_STATE_RUNNABLE;
3882         else
3883                 vcpu->arch.mp_state = KVM_MP_STATE_UNINITIALIZED;
3884
3885         page = alloc_page(GFP_KERNEL | __GFP_ZERO);
3886         if (!page) {
3887                 r = -ENOMEM;
3888                 goto fail;
3889         }
3890         vcpu->arch.pio_data = page_address(page);
3891
3892         r = kvm_mmu_create(vcpu);
3893         if (r < 0)
3894                 goto fail_free_pio_data;
3895
3896         if (irqchip_in_kernel(kvm)) {
3897                 r = kvm_create_lapic(vcpu);
3898                 if (r < 0)
3899                         goto fail_mmu_destroy;
3900         }
3901
3902         return 0;
3903
3904 fail_mmu_destroy:
3905         kvm_mmu_destroy(vcpu);
3906 fail_free_pio_data:
3907         free_page((unsigned long)vcpu->arch.pio_data);
3908 fail:
3909         return r;
3910 }
3911
3912 void kvm_arch_vcpu_uninit(struct kvm_vcpu *vcpu)
3913 {
3914         kvm_free_lapic(vcpu);
3915         down_read(&vcpu->kvm->slots_lock);
3916         kvm_mmu_destroy(vcpu);
3917         up_read(&vcpu->kvm->slots_lock);
3918         free_page((unsigned long)vcpu->arch.pio_data);
3919 }
3920
3921 struct  kvm *kvm_arch_create_vm(void)
3922 {
3923         struct kvm *kvm = kzalloc(sizeof(struct kvm), GFP_KERNEL);
3924
3925         if (!kvm)
3926                 return ERR_PTR(-ENOMEM);
3927
3928         INIT_LIST_HEAD(&kvm->arch.active_mmu_pages);
3929
3930         return kvm;
3931 }
3932
3933 static void kvm_unload_vcpu_mmu(struct kvm_vcpu *vcpu)
3934 {
3935         vcpu_load(vcpu);
3936         kvm_mmu_unload(vcpu);
3937         vcpu_put(vcpu);
3938 }
3939
3940 static void kvm_free_vcpus(struct kvm *kvm)
3941 {
3942         unsigned int i;
3943
3944         /*
3945          * Unpin any mmu pages first.
3946          */
3947         for (i = 0; i < KVM_MAX_VCPUS; ++i)
3948                 if (kvm->vcpus[i])
3949                         kvm_unload_vcpu_mmu(kvm->vcpus[i]);
3950         for (i = 0; i < KVM_MAX_VCPUS; ++i) {
3951                 if (kvm->vcpus[i]) {
3952                         kvm_arch_vcpu_free(kvm->vcpus[i]);
3953                         kvm->vcpus[i] = NULL;
3954                 }
3955         }
3956
3957 }
3958
3959 void kvm_arch_destroy_vm(struct kvm *kvm)
3960 {
3961         kvm_free_pit(kvm);
3962         kfree(kvm->arch.vpic);
3963         kfree(kvm->arch.vioapic);
3964         kvm_free_vcpus(kvm);
3965         kvm_free_physmem(kvm);
3966         if (kvm->arch.apic_access_page)
3967                 put_page(kvm->arch.apic_access_page);
3968         if (kvm->arch.ept_identity_pagetable)
3969                 put_page(kvm->arch.ept_identity_pagetable);
3970         kfree(kvm);
3971 }
3972
3973 int kvm_arch_set_memory_region(struct kvm *kvm,
3974                                 struct kvm_userspace_memory_region *mem,
3975                                 struct kvm_memory_slot old,
3976                                 int user_alloc)
3977 {
3978         int npages = mem->memory_size >> PAGE_SHIFT;
3979         struct kvm_memory_slot *memslot = &kvm->memslots[mem->slot];
3980
3981         /*To keep backward compatibility with older userspace,
3982          *x86 needs to hanlde !user_alloc case.
3983          */
3984         if (!user_alloc) {
3985                 if (npages && !old.rmap) {
3986                         unsigned long userspace_addr;
3987
3988                         down_write(&current->mm->mmap_sem);
3989                         userspace_addr = do_mmap(NULL, 0,
3990                                                  npages * PAGE_SIZE,
3991                                                  PROT_READ | PROT_WRITE,
3992                                                  MAP_SHARED | MAP_ANONYMOUS,
3993                                                  0);
3994                         up_write(&current->mm->mmap_sem);
3995
3996                         if (IS_ERR((void *)userspace_addr))
3997                                 return PTR_ERR((void *)userspace_addr);
3998
3999                         /* set userspace_addr atomically for kvm_hva_to_rmapp */
4000                         spin_lock(&kvm->mmu_lock);
4001                         memslot->userspace_addr = userspace_addr;
4002                         spin_unlock(&kvm->mmu_lock);
4003                 } else {
4004                         if (!old.user_alloc && old.rmap) {
4005                                 int ret;
4006
4007                                 down_write(&current->mm->mmap_sem);
4008                                 ret = do_munmap(current->mm, old.userspace_addr,
4009                                                 old.npages * PAGE_SIZE);
4010                                 up_write(&current->mm->mmap_sem);
4011                                 if (ret < 0)
4012                                         printk(KERN_WARNING
4013                                        "kvm_vm_ioctl_set_memory_region: "
4014                                        "failed to munmap memory\n");
4015                         }
4016                 }
4017         }
4018
4019         if (!kvm->arch.n_requested_mmu_pages) {
4020                 unsigned int nr_mmu_pages = kvm_mmu_calculate_mmu_pages(kvm);
4021                 kvm_mmu_change_mmu_pages(kvm, nr_mmu_pages);
4022         }
4023
4024         kvm_mmu_slot_remove_write_access(kvm, mem->slot);
4025         kvm_flush_remote_tlbs(kvm);
4026
4027         return 0;
4028 }
4029
4030 void kvm_arch_flush_shadow(struct kvm *kvm)
4031 {
4032         kvm_mmu_zap_all(kvm);
4033 }
4034
4035 int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu)
4036 {
4037         return vcpu->arch.mp_state == KVM_MP_STATE_RUNNABLE
4038                || vcpu->arch.mp_state == KVM_MP_STATE_SIPI_RECEIVED;
4039 }
4040
4041 static void vcpu_kick_intr(void *info)
4042 {
4043 #ifdef DEBUG
4044         struct kvm_vcpu *vcpu = (struct kvm_vcpu *)info;
4045         printk(KERN_DEBUG "vcpu_kick_intr %p \n", vcpu);
4046 #endif
4047 }
4048
4049 void kvm_vcpu_kick(struct kvm_vcpu *vcpu)
4050 {
4051         int ipi_pcpu = vcpu->cpu;
4052         int cpu = get_cpu();
4053
4054         if (waitqueue_active(&vcpu->wq)) {
4055                 wake_up_interruptible(&vcpu->wq);
4056                 ++vcpu->stat.halt_wakeup;
4057         }
4058         /*
4059          * We may be called synchronously with irqs disabled in guest mode,
4060          * So need not to call smp_call_function_single() in that case.
4061          */
4062         if (vcpu->guest_mode && vcpu->cpu != cpu)
4063                 smp_call_function_single(ipi_pcpu, vcpu_kick_intr, vcpu, 0);
4064         put_cpu();
4065 }