drivers/kvm/vmx.c

   1 /*
   2  * Kernel-based Virtual Machine driver for Linux
   3  *
   4  * This module enables machines with Intel VT-x extensions to run virtual
   5  * machines without emulation or binary translation.
   6  *
   7  * Copyright (C) 2006 Qumranet, Inc.
   8  *
   9  * Authors:
  10  *   Avi Kivity   <avi@qumranet.com>
  11  *   Yaniv Kamay  <yaniv@qumranet.com>
  12  *
  13  * This work is licensed under the terms of the GNU GPL, version 2.  See
  14  * the COPYING file in the top-level directory.
  15  *
  16  */
  17
  18 #include "kvm.h"
  19 #include "x86_emulate.h"
  20 #include "irq.h"
  21 #include "vmx.h"
  22 #include "segment_descriptor.h"
  23
  24 #include <linux/module.h>
  25 #include <linux/kernel.h>
  26 #include <linux/mm.h>
  27 #include <linux/highmem.h>
  28 #include <linux/profile.h>
  29 #include <linux/sched.h>
  30
  31 #include <asm/io.h>
  32 #include <asm/desc.h>
  33
  34 MODULE_AUTHOR("Qumranet");
  35 MODULE_LICENSE("GPL");
  36
  37 struct vmcs {
  38         u32 revision_id;
  39         u32 abort;
  40         char data[0];
  41 };
  42
  43 struct vcpu_vmx {
  44         struct kvm_vcpu       vcpu;
  45         int                   launched;
  46         u8                    fail;
  47         struct kvm_msr_entry *guest_msrs;
  48         struct kvm_msr_entry *host_msrs;
  49         int                   nmsrs;
  50         int                   save_nmsrs;
  51         int                   msr_offset_efer;
  52 #ifdef CONFIG_X86_64
  53         int                   msr_offset_kernel_gs_base;
  54 #endif
  55         struct vmcs          *vmcs;
  56         struct {
  57                 int           loaded;
  58                 u16           fs_sel, gs_sel, ldt_sel;
  59                 int           gs_ldt_reload_needed;
  60                 int           fs_reload_needed;
  61         }host_state;
  62
  63 };
  64
  65 static inline struct vcpu_vmx *to_vmx(struct kvm_vcpu *vcpu)
  66 {
  67         return container_of(vcpu, struct vcpu_vmx, vcpu);
  68 }
  69
  70 static int init_rmode_tss(struct kvm *kvm);
  71
  72 static DEFINE_PER_CPU(struct vmcs *, vmxarea);
  73 static DEFINE_PER_CPU(struct vmcs *, current_vmcs);
  74
  75 static struct page *vmx_io_bitmap_a;
  76 static struct page *vmx_io_bitmap_b;
  77
  78 #define EFER_SAVE_RESTORE_BITS ((u64)EFER_SCE)
  79
  80 static struct vmcs_config {
  81         int size;
  82         int order;
  83         u32 revision_id;
  84         u32 pin_based_exec_ctrl;
  85         u32 cpu_based_exec_ctrl;
  86         u32 vmexit_ctrl;
  87         u32 vmentry_ctrl;
  88 } vmcs_config;
  89
  90 #define VMX_SEGMENT_FIELD(seg)                                  \
  91         [VCPU_SREG_##seg] = {                                   \
  92                 .selector = GUEST_##seg##_SELECTOR,             \
  93                 .base = GUEST_##seg##_BASE,                     \
  94                 .limit = GUEST_##seg##_LIMIT,                   \
  95                 .ar_bytes = GUEST_##seg##_AR_BYTES,             \
  96         }
  97
  98 static struct kvm_vmx_segment_field {
  99         unsigned selector;
 100         unsigned base;
 101         unsigned limit;
 102         unsigned ar_bytes;
 103 } kvm_vmx_segment_fields[] = {
 104         VMX_SEGMENT_FIELD(CS),
 105         VMX_SEGMENT_FIELD(DS),
 106         VMX_SEGMENT_FIELD(ES),
 107         VMX_SEGMENT_FIELD(FS),
 108         VMX_SEGMENT_FIELD(GS),
 109         VMX_SEGMENT_FIELD(SS),
 110         VMX_SEGMENT_FIELD(TR),
 111         VMX_SEGMENT_FIELD(LDTR),
 112 };
 113
 114 /*
 115  * Keep MSR_K6_STAR at the end, as setup_msrs() will try to optimize it
 116  * away by decrementing the array size.
 117  */
 118 static const u32 vmx_msr_index[] = {
 119 #ifdef CONFIG_X86_64
 120         MSR_SYSCALL_MASK, MSR_LSTAR, MSR_CSTAR, MSR_KERNEL_GS_BASE,
 121 #endif
 122         MSR_EFER, MSR_K6_STAR,
 123 };
 124 #define NR_VMX_MSR ARRAY_SIZE(vmx_msr_index)
 125
 126 static void load_msrs(struct kvm_msr_entry *e, int n)
 127 {
 128         int i;
 129
 130         for (i = 0; i < n; ++i)
 131                 wrmsrl(e[i].index, e[i].data);
 132 }
 133
 134 static void save_msrs(struct kvm_msr_entry *e, int n)
 135 {
 136         int i;
 137
 138         for (i = 0; i < n; ++i)
 139                 rdmsrl(e[i].index, e[i].data);
 140 }
 141
 142 static inline u64 msr_efer_save_restore_bits(struct kvm_msr_entry msr)
 143 {
 144         return (u64)msr.data & EFER_SAVE_RESTORE_BITS;
 145 }
 146
 147 static inline int msr_efer_need_save_restore(struct vcpu_vmx *vmx)
 148 {
 149         int efer_offset = vmx->msr_offset_efer;
 150         return msr_efer_save_restore_bits(vmx->host_msrs[efer_offset]) !=
 151                 msr_efer_save_restore_bits(vmx->guest_msrs[efer_offset]);
 152 }
 153
 154 static inline int is_page_fault(u32 intr_info)
 155 {
 156         return (intr_info & (INTR_INFO_INTR_TYPE_MASK | INTR_INFO_VECTOR_MASK |
 157                              INTR_INFO_VALID_MASK)) ==
 158                 (INTR_TYPE_EXCEPTION | PF_VECTOR | INTR_INFO_VALID_MASK);
 159 }
 160
 161 static inline int is_no_device(u32 intr_info)
 162 {
 163         return (intr_info & (INTR_INFO_INTR_TYPE_MASK | INTR_INFO_VECTOR_MASK |
 164                              INTR_INFO_VALID_MASK)) ==
 165                 (INTR_TYPE_EXCEPTION | NM_VECTOR | INTR_INFO_VALID_MASK);
 166 }
 167
 168 static inline int is_external_interrupt(u32 intr_info)
 169 {
 170         return (intr_info & (INTR_INFO_INTR_TYPE_MASK | INTR_INFO_VALID_MASK))
 171                 == (INTR_TYPE_EXT_INTR | INTR_INFO_VALID_MASK);
 172 }
 173
 174 static inline int cpu_has_vmx_tpr_shadow(void)
 175 {
 176         return (vmcs_config.cpu_based_exec_ctrl & CPU_BASED_TPR_SHADOW);
 177 }
 178
 179 static inline int vm_need_tpr_shadow(struct kvm *kvm)
 180 {
 181         return ((cpu_has_vmx_tpr_shadow()) && (irqchip_in_kernel(kvm)));
 182 }
 183
 184 static int __find_msr_index(struct vcpu_vmx *vmx, u32 msr)
 185 {
 186         int i;
 187
 188         for (i = 0; i < vmx->nmsrs; ++i)
 189                 if (vmx->guest_msrs[i].index == msr)
 190                         return i;
 191         return -1;
 192 }
 193
 194 static struct kvm_msr_entry *find_msr_entry(struct vcpu_vmx *vmx, u32 msr)
 195 {
 196         int i;
 197
 198         i = __find_msr_index(vmx, msr);
 199         if (i >= 0)
 200                 return &vmx->guest_msrs[i];
 201         return NULL;
 202 }
 203
 204 static void vmcs_clear(struct vmcs *vmcs)
 205 {
 206         u64 phys_addr = __pa(vmcs);
 207         u8 error;
 208
 209         asm volatile (ASM_VMX_VMCLEAR_RAX "; setna %0"
 210                       : "=g"(error) : "a"(&phys_addr), "m"(phys_addr)
 211                       : "cc", "memory");
 212         if (error)
 213                 printk(KERN_ERR "kvm: vmclear fail: %p/%llx\n",
 214                        vmcs, phys_addr);
 215 }
 216
 217 static void __vcpu_clear(void *arg)
 218 {
 219         struct vcpu_vmx *vmx = arg;
 220         int cpu = raw_smp_processor_id();
 221
 222         if (vmx->vcpu.cpu == cpu)
 223                 vmcs_clear(vmx->vmcs);
 224         if (per_cpu(current_vmcs, cpu) == vmx->vmcs)
 225                 per_cpu(current_vmcs, cpu) = NULL;
 226         rdtscll(vmx->vcpu.host_tsc);
 227 }
 228
 229 static void vcpu_clear(struct vcpu_vmx *vmx)
 230 {
 231         if (vmx->vcpu.cpu != raw_smp_processor_id() && vmx->vcpu.cpu != -1)
 232                 smp_call_function_single(vmx->vcpu.cpu, __vcpu_clear,
 233                                          vmx, 0, 1);
 234         else
 235                 __vcpu_clear(vmx);
 236         vmx->launched = 0;
 237 }
 238
 239 static unsigned long vmcs_readl(unsigned long field)
 240 {
 241         unsigned long value;
 242
 243         asm volatile (ASM_VMX_VMREAD_RDX_RAX
 244                       : "=a"(value) : "d"(field) : "cc");
 245         return value;
 246 }
 247
 248 static u16 vmcs_read16(unsigned long field)
 249 {
 250         return vmcs_readl(field);
 251 }
 252
 253 static u32 vmcs_read32(unsigned long field)
 254 {
 255         return vmcs_readl(field);
 256 }
 257
 258 static u64 vmcs_read64(unsigned long field)
 259 {
 260 #ifdef CONFIG_X86_64
 261         return vmcs_readl(field);
 262 #else
 263         return vmcs_readl(field) | ((u64)vmcs_readl(field+1) << 32);
 264 #endif
 265 }
 266
 267 static noinline void vmwrite_error(unsigned long field, unsigned long value)
 268 {
 269         printk(KERN_ERR "vmwrite error: reg %lx value %lx (err %d)\n",
 270                field, value, vmcs_read32(VM_INSTRUCTION_ERROR));
 271         dump_stack();
 272 }
 273
 274 static void vmcs_writel(unsigned long field, unsigned long value)
 275 {
 276         u8 error;
 277
 278         asm volatile (ASM_VMX_VMWRITE_RAX_RDX "; setna %0"
 279                        : "=q"(error) : "a"(value), "d"(field) : "cc" );
 280         if (unlikely(error))
 281                 vmwrite_error(field, value);
 282 }
 283
 284 static void vmcs_write16(unsigned long field, u16 value)
 285 {
 286         vmcs_writel(field, value);
 287 }
 288
 289 static void vmcs_write32(unsigned long field, u32 value)
 290 {
 291         vmcs_writel(field, value);
 292 }
 293
 294 static void vmcs_write64(unsigned long field, u64 value)
 295 {
 296 #ifdef CONFIG_X86_64
 297         vmcs_writel(field, value);
 298 #else
 299         vmcs_writel(field, value);
 300         asm volatile ("");
 301         vmcs_writel(field+1, value >> 32);
 302 #endif
 303 }
 304
 305 static void vmcs_clear_bits(unsigned long field, u32 mask)
 306 {
 307         vmcs_writel(field, vmcs_readl(field) & ~mask);
 308 }
 309
 310 static void vmcs_set_bits(unsigned long field, u32 mask)
 311 {
 312         vmcs_writel(field, vmcs_readl(field) | mask);
 313 }
 314
 315 static void update_exception_bitmap(struct kvm_vcpu *vcpu)
 316 {
 317         u32 eb;
 318
 319         eb = 1u << PF_VECTOR;
 320         if (!vcpu->fpu_active)
 321                 eb |= 1u << NM_VECTOR;
 322         if (vcpu->guest_debug.enabled)
 323                 eb |= 1u << 1;
 324         if (vcpu->rmode.active)
 325                 eb = ~0;
 326         vmcs_write32(EXCEPTION_BITMAP, eb);
 327 }
 328
 329 static void reload_tss(void)
 330 {
 331 #ifndef CONFIG_X86_64
 332
 333         /*
 334          * VT restores TR but not its size.  Useless.
 335          */
 336         struct descriptor_table gdt;
 337         struct segment_descriptor *descs;
 338
 339         get_gdt(&gdt);
 340         descs = (void *)gdt.base;
 341         descs[GDT_ENTRY_TSS].type = 9; /* available TSS */
 342         load_TR_desc();
 343 #endif
 344 }
 345
 346 static void load_transition_efer(struct vcpu_vmx *vmx)
 347 {
 348         u64 trans_efer;
 349         int efer_offset = vmx->msr_offset_efer;
 350
 351         trans_efer = vmx->host_msrs[efer_offset].data;
 352         trans_efer &= ~EFER_SAVE_RESTORE_BITS;
 353         trans_efer |= msr_efer_save_restore_bits(vmx->guest_msrs[efer_offset]);
 354         wrmsrl(MSR_EFER, trans_efer);
 355         vmx->vcpu.stat.efer_reload++;
 356 }
 357
 358 static void vmx_save_host_state(struct vcpu_vmx *vmx)
 359 {
 360         if (vmx->host_state.loaded)
 361                 return;
 362
 363         vmx->host_state.loaded = 1;
 364         /*
 365          * Set host fs and gs selectors.  Unfortunately, 22.2.3 does not
 366          * allow segment selectors with cpl > 0 or ti == 1.
 367          */
 368         vmx->host_state.ldt_sel = read_ldt();
 369         vmx->host_state.gs_ldt_reload_needed = vmx->host_state.ldt_sel;
 370         vmx->host_state.fs_sel = read_fs();
 371         if (!(vmx->host_state.fs_sel & 7)) {
 372                 vmcs_write16(HOST_FS_SELECTOR, vmx->host_state.fs_sel);
 373                 vmx->host_state.fs_reload_needed = 0;
 374         } else {
 375                 vmcs_write16(HOST_FS_SELECTOR, 0);
 376                 vmx->host_state.fs_reload_needed = 1;
 377         }
 378         vmx->host_state.gs_sel = read_gs();
 379         if (!(vmx->host_state.gs_sel & 7))
 380                 vmcs_write16(HOST_GS_SELECTOR, vmx->host_state.gs_sel);
 381         else {
 382                 vmcs_write16(HOST_GS_SELECTOR, 0);
 383                 vmx->host_state.gs_ldt_reload_needed = 1;
 384         }
 385
 386 #ifdef CONFIG_X86_64
 387         vmcs_writel(HOST_FS_BASE, read_msr(MSR_FS_BASE));
 388         vmcs_writel(HOST_GS_BASE, read_msr(MSR_GS_BASE));
 389 #else
 390         vmcs_writel(HOST_FS_BASE, segment_base(vmx->host_state.fs_sel));
 391         vmcs_writel(HOST_GS_BASE, segment_base(vmx->host_state.gs_sel));
 392 #endif
 393
 394 #ifdef CONFIG_X86_64
 395         if (is_long_mode(&vmx->vcpu)) {
 396                 save_msrs(vmx->host_msrs +
 397                           vmx->msr_offset_kernel_gs_base, 1);
 398         }
 399 #endif
 400         load_msrs(vmx->guest_msrs, vmx->save_nmsrs);
 401         if (msr_efer_need_save_restore(vmx))
 402                 load_transition_efer(vmx);
 403 }
 404
 405 static void vmx_load_host_state(struct vcpu_vmx *vmx)
 406 {
 407         unsigned long flags;
 408
 409         if (!vmx->host_state.loaded)
 410                 return;
 411
 412         vmx->host_state.loaded = 0;
 413         if (vmx->host_state.fs_reload_needed)
 414                 load_fs(vmx->host_state.fs_sel);
 415         if (vmx->host_state.gs_ldt_reload_needed) {
 416                 load_ldt(vmx->host_state.ldt_sel);
 417                 /*
 418                  * If we have to reload gs, we must take care to
 419                  * preserve our gs base.
 420                  */
 421                 local_irq_save(flags);
 422                 load_gs(vmx->host_state.gs_sel);
 423 #ifdef CONFIG_X86_64
 424                 wrmsrl(MSR_GS_BASE, vmcs_readl(HOST_GS_BASE));
 425 #endif
 426                 local_irq_restore(flags);
 427         }
 428         reload_tss();
 429         save_msrs(vmx->guest_msrs, vmx->save_nmsrs);
 430         load_msrs(vmx->host_msrs, vmx->save_nmsrs);
 431         if (msr_efer_need_save_restore(vmx))
 432                 load_msrs(vmx->host_msrs + vmx->msr_offset_efer, 1);
 433 }
 434
 435 /*
 436  * Switches to specified vcpu, until a matching vcpu_put(), but assumes
 437  * vcpu mutex is already taken.
 438  */
 439 static void vmx_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
 440 {
 441         struct vcpu_vmx *vmx = to_vmx(vcpu);
 442         u64 phys_addr = __pa(vmx->vmcs);
 443         u64 tsc_this, delta;
 444
 445         if (vcpu->cpu != cpu) {
 446                 vcpu_clear(vmx);
 447                 kvm_migrate_apic_timer(vcpu);
 448         }
 449
 450         if (per_cpu(current_vmcs, cpu) != vmx->vmcs) {
 451                 u8 error;
 452
 453                 per_cpu(current_vmcs, cpu) = vmx->vmcs;
 454                 asm volatile (ASM_VMX_VMPTRLD_RAX "; setna %0"
 455                               : "=g"(error) : "a"(&phys_addr), "m"(phys_addr)
 456                               : "cc");
 457                 if (error)
 458                         printk(KERN_ERR "kvm: vmptrld %p/%llx fail\n",
 459                                vmx->vmcs, phys_addr);
 460         }
 461
 462         if (vcpu->cpu != cpu) {
 463                 struct descriptor_table dt;
 464                 unsigned long sysenter_esp;
 465
 466                 vcpu->cpu = cpu;
 467                 /*
 468                  * Linux uses per-cpu TSS and GDT, so set these when switching
 469                  * processors.
 470                  */
 471                 vmcs_writel(HOST_TR_BASE, read_tr_base()); /* 22.2.4 */
 472                 get_gdt(&dt);
 473                 vmcs_writel(HOST_GDTR_BASE, dt.base);   /* 22.2.4 */
 474
 475                 rdmsrl(MSR_IA32_SYSENTER_ESP, sysenter_esp);
 476                 vmcs_writel(HOST_IA32_SYSENTER_ESP, sysenter_esp); /* 22.2.3 */
 477
 478                 /*
 479                  * Make sure the time stamp counter is monotonous.
 480                  */
 481                 rdtscll(tsc_this);
 482                 delta = vcpu->host_tsc - tsc_this;
 483                 vmcs_write64(TSC_OFFSET, vmcs_read64(TSC_OFFSET) + delta);
 484         }
 485 }
 486
 487 static void vmx_vcpu_put(struct kvm_vcpu *vcpu)
 488 {
 489         vmx_load_host_state(to_vmx(vcpu));
 490         kvm_put_guest_fpu(vcpu);
 491 }
 492
 493 static void vmx_fpu_activate(struct kvm_vcpu *vcpu)
 494 {
 495         if (vcpu->fpu_active)
 496                 return;
 497         vcpu->fpu_active = 1;
 498         vmcs_clear_bits(GUEST_CR0, X86_CR0_TS);
 499         if (vcpu->cr0 & X86_CR0_TS)
 500                 vmcs_set_bits(GUEST_CR0, X86_CR0_TS);
 501         update_exception_bitmap(vcpu);
 502 }
 503
 504 static void vmx_fpu_deactivate(struct kvm_vcpu *vcpu)
 505 {
 506         if (!vcpu->fpu_active)
 507                 return;
 508         vcpu->fpu_active = 0;
 509         vmcs_set_bits(GUEST_CR0, X86_CR0_TS);
 510         update_exception_bitmap(vcpu);
 511 }
 512
 513 static void vmx_vcpu_decache(struct kvm_vcpu *vcpu)
 514 {
 515         vcpu_clear(to_vmx(vcpu));
 516 }
 517
 518 static unsigned long vmx_get_rflags(struct kvm_vcpu *vcpu)
 519 {
 520         return vmcs_readl(GUEST_RFLAGS);
 521 }
 522
 523 static void vmx_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags)
 524 {
 525         vmcs_writel(GUEST_RFLAGS, rflags);
 526 }
 527
 528 static void skip_emulated_instruction(struct kvm_vcpu *vcpu)
 529 {
 530         unsigned long rip;
 531         u32 interruptibility;
 532
 533         rip = vmcs_readl(GUEST_RIP);
 534         rip += vmcs_read32(VM_EXIT_INSTRUCTION_LEN);
 535         vmcs_writel(GUEST_RIP, rip);
 536
 537         /*
 538          * We emulated an instruction, so temporary interrupt blocking
 539          * should be removed, if set.
 540          */
 541         interruptibility = vmcs_read32(GUEST_INTERRUPTIBILITY_INFO);
 542         if (interruptibility & 3)
 543                 vmcs_write32(GUEST_INTERRUPTIBILITY_INFO,
 544                              interruptibility & ~3);
 545         vcpu->interrupt_window_open = 1;
 546 }
 547
 548 static void vmx_inject_gp(struct kvm_vcpu *vcpu, unsigned error_code)
 549 {
 550         printk(KERN_DEBUG "inject_general_protection: rip 0x%lx\n",
 551                vmcs_readl(GUEST_RIP));
 552         vmcs_write32(VM_ENTRY_EXCEPTION_ERROR_CODE, error_code);
 553         vmcs_write32(VM_ENTRY_INTR_INFO_FIELD,
 554                      GP_VECTOR |
 555                      INTR_TYPE_EXCEPTION |
 556                      INTR_INFO_DELIEVER_CODE_MASK |
 557                      INTR_INFO_VALID_MASK);
 558 }
 559
 560 /*
 561  * Swap MSR entry in host/guest MSR entry array.
 562  */
 563 #ifdef CONFIG_X86_64
 564 static void move_msr_up(struct vcpu_vmx *vmx, int from, int to)
 565 {
 566         struct kvm_msr_entry tmp;
 567
 568         tmp = vmx->guest_msrs[to];
 569         vmx->guest_msrs[to] = vmx->guest_msrs[from];
 570         vmx->guest_msrs[from] = tmp;
 571         tmp = vmx->host_msrs[to];
 572         vmx->host_msrs[to] = vmx->host_msrs[from];
 573         vmx->host_msrs[from] = tmp;
 574 }
 575 #endif
 576
 577 /*
 578  * Set up the vmcs to automatically save and restore system
 579  * msrs.  Don't touch the 64-bit msrs if the guest is in legacy
 580  * mode, as fiddling with msrs is very expensive.
 581  */
 582 static void setup_msrs(struct vcpu_vmx *vmx)
 583 {
 584         int save_nmsrs;
 585
 586         save_nmsrs = 0;
 587 #ifdef CONFIG_X86_64
 588         if (is_long_mode(&vmx->vcpu)) {
 589                 int index;
 590
 591                 index = __find_msr_index(vmx, MSR_SYSCALL_MASK);
 592                 if (index >= 0)
 593                         move_msr_up(vmx, index, save_nmsrs++);
 594                 index = __find_msr_index(vmx, MSR_LSTAR);
 595                 if (index >= 0)
 596                         move_msr_up(vmx, index, save_nmsrs++);
 597                 index = __find_msr_index(vmx, MSR_CSTAR);
 598                 if (index >= 0)
 599                         move_msr_up(vmx, index, save_nmsrs++);
 600                 index = __find_msr_index(vmx, MSR_KERNEL_GS_BASE);
 601                 if (index >= 0)
 602                         move_msr_up(vmx, index, save_nmsrs++);
 603                 /*
 604                  * MSR_K6_STAR is only needed on long mode guests, and only
 605                  * if efer.sce is enabled.
 606                  */
 607                 index = __find_msr_index(vmx, MSR_K6_STAR);
 608                 if ((index >= 0) && (vmx->vcpu.shadow_efer & EFER_SCE))
 609                         move_msr_up(vmx, index, save_nmsrs++);
 610         }
 611 #endif
 612         vmx->save_nmsrs = save_nmsrs;
 613
 614 #ifdef CONFIG_X86_64
 615         vmx->msr_offset_kernel_gs_base =
 616                 __find_msr_index(vmx, MSR_KERNEL_GS_BASE);
 617 #endif
 618         vmx->msr_offset_efer = __find_msr_index(vmx, MSR_EFER);
 619 }
 620
 621 /*
 622  * reads and returns guest's timestamp counter "register"
 623  * guest_tsc = host_tsc + tsc_offset    -- 21.3
 624  */
 625 static u64 guest_read_tsc(void)
 626 {
 627         u64 host_tsc, tsc_offset;
 628
 629         rdtscll(host_tsc);
 630         tsc_offset = vmcs_read64(TSC_OFFSET);
 631         return host_tsc + tsc_offset;
 632 }
 633
 634 /*
 635  * writes 'guest_tsc' into guest's timestamp counter "register"
 636  * guest_tsc = host_tsc + tsc_offset ==> tsc_offset = guest_tsc - host_tsc
 637  */
 638 static void guest_write_tsc(u64 guest_tsc)
 639 {
 640         u64 host_tsc;
 641
 642         rdtscll(host_tsc);
 643         vmcs_write64(TSC_OFFSET, guest_tsc - host_tsc);
 644 }
 645
 646 /*
 647  * Reads an msr value (of 'msr_index') into 'pdata'.
 648  * Returns 0 on success, non-0 otherwise.
 649  * Assumes vcpu_load() was already called.
 650  */
 651 static int vmx_get_msr(struct kvm_vcpu *vcpu, u32 msr_index, u64 *pdata)
 652 {
 653         u64 data;
 654         struct kvm_msr_entry *msr;
 655
 656         if (!pdata) {
 657                 printk(KERN_ERR "BUG: get_msr called with NULL pdata\n");
 658                 return -EINVAL;
 659         }
 660
 661         switch (msr_index) {
 662 #ifdef CONFIG_X86_64
 663         case MSR_FS_BASE:
 664                 data = vmcs_readl(GUEST_FS_BASE);
 665                 break;
 666         case MSR_GS_BASE:
 667                 data = vmcs_readl(GUEST_GS_BASE);
 668                 break;
 669         case MSR_EFER:
 670                 return kvm_get_msr_common(vcpu, msr_index, pdata);
 671 #endif
 672         case MSR_IA32_TIME_STAMP_COUNTER:
 673                 data = guest_read_tsc();
 674                 break;
 675         case MSR_IA32_SYSENTER_CS:
 676                 data = vmcs_read32(GUEST_SYSENTER_CS);
 677                 break;
 678         case MSR_IA32_SYSENTER_EIP:
 679                 data = vmcs_readl(GUEST_SYSENTER_EIP);
 680                 break;
 681         case MSR_IA32_SYSENTER_ESP:
 682                 data = vmcs_readl(GUEST_SYSENTER_ESP);
 683                 break;
 684         default:
 685                 msr = find_msr_entry(to_vmx(vcpu), msr_index);
 686                 if (msr) {
 687                         data = msr->data;
 688                         break;
 689                 }
 690                 return kvm_get_msr_common(vcpu, msr_index, pdata);
 691         }
 692
 693         *pdata = data;
 694         return 0;
 695 }
 696
 697 /*
 698  * Writes msr value into into the appropriate "register".
 699  * Returns 0 on success, non-0 otherwise.
 700  * Assumes vcpu_load() was already called.
 701  */
 702 static int vmx_set_msr(struct kvm_vcpu *vcpu, u32 msr_index, u64 data)
 703 {
 704         struct vcpu_vmx *vmx = to_vmx(vcpu);
 705         struct kvm_msr_entry *msr;
 706         int ret = 0;
 707
 708         switch (msr_index) {
 709 #ifdef CONFIG_X86_64
 710         case MSR_EFER:
 711                 ret = kvm_set_msr_common(vcpu, msr_index, data);
 712                 if (vmx->host_state.loaded)
 713                         load_transition_efer(vmx);
 714                 break;
 715         case MSR_FS_BASE:
 716                 vmcs_writel(GUEST_FS_BASE, data);
 717                 break;
 718         case MSR_GS_BASE:
 719                 vmcs_writel(GUEST_GS_BASE, data);
 720                 break;
 721 #endif
 722         case MSR_IA32_SYSENTER_CS:
 723                 vmcs_write32(GUEST_SYSENTER_CS, data);
 724                 break;
 725         case MSR_IA32_SYSENTER_EIP:
 726                 vmcs_writel(GUEST_SYSENTER_EIP, data);
 727                 break;
 728         case MSR_IA32_SYSENTER_ESP:
 729                 vmcs_writel(GUEST_SYSENTER_ESP, data);
 730                 break;
 731         case MSR_IA32_TIME_STAMP_COUNTER:
 732                 guest_write_tsc(data);
 733                 break;
 734         default:
 735                 msr = find_msr_entry(vmx, msr_index);
 736                 if (msr) {
 737                         msr->data = data;
 738                         if (vmx->host_state.loaded)
 739                                 load_msrs(vmx->guest_msrs, vmx->save_nmsrs);
 740                         break;
 741                 }
 742                 ret = kvm_set_msr_common(vcpu, msr_index, data);
 743         }
 744
 745         return ret;
 746 }
 747
 748 /*
 749  * Sync the rsp and rip registers into the vcpu structure.  This allows
 750  * registers to be accessed by indexing vcpu->regs.
 751  */
 752 static void vcpu_load_rsp_rip(struct kvm_vcpu *vcpu)
 753 {
 754         vcpu->regs[VCPU_REGS_RSP] = vmcs_readl(GUEST_RSP);
 755         vcpu->rip = vmcs_readl(GUEST_RIP);
 756 }
 757
 758 /*
 759  * Syncs rsp and rip back into the vmcs.  Should be called after possible
 760  * modification.
 761  */
 762 static void vcpu_put_rsp_rip(struct kvm_vcpu *vcpu)
 763 {
 764         vmcs_writel(GUEST_RSP, vcpu->regs[VCPU_REGS_RSP]);
 765         vmcs_writel(GUEST_RIP, vcpu->rip);
 766 }
 767
 768 static int set_guest_debug(struct kvm_vcpu *vcpu, struct kvm_debug_guest *dbg)
 769 {
 770         unsigned long dr7 = 0x400;
 771         int old_singlestep;
 772
 773         old_singlestep = vcpu->guest_debug.singlestep;
 774
 775         vcpu->guest_debug.enabled = dbg->enabled;
 776         if (vcpu->guest_debug.enabled) {
 777                 int i;
 778
 779                 dr7 |= 0x200;  /* exact */
 780                 for (i = 0; i < 4; ++i) {
 781                         if (!dbg->breakpoints[i].enabled)
 782                                 continue;
 783                         vcpu->guest_debug.bp[i] = dbg->breakpoints[i].address;
 784                         dr7 |= 2 << (i*2);    /* global enable */
 785                         dr7 |= 0 << (i*4+16); /* execution breakpoint */
 786                 }
 787
 788                 vcpu->guest_debug.singlestep = dbg->singlestep;
 789         } else
 790                 vcpu->guest_debug.singlestep = 0;
 791
 792         if (old_singlestep && !vcpu->guest_debug.singlestep) {
 793                 unsigned long flags;
 794
 795                 flags = vmcs_readl(GUEST_RFLAGS);
 796                 flags &= ~(X86_EFLAGS_TF | X86_EFLAGS_RF);
 797                 vmcs_writel(GUEST_RFLAGS, flags);
 798         }
 799
 800         update_exception_bitmap(vcpu);
 801         vmcs_writel(GUEST_DR7, dr7);
 802
 803         return 0;
 804 }
 805
 806 static int vmx_get_irq(struct kvm_vcpu *vcpu)
 807 {
 808         u32 idtv_info_field;
 809
 810         idtv_info_field = vmcs_read32(IDT_VECTORING_INFO_FIELD);
 811         if (idtv_info_field & INTR_INFO_VALID_MASK) {
 812                 if (is_external_interrupt(idtv_info_field))
 813                         return idtv_info_field & VECTORING_INFO_VECTOR_MASK;
 814                 else
 815                         printk("pending exception: not handled yet\n");
 816         }
 817         return -1;
 818 }
 819
 820 static __init int cpu_has_kvm_support(void)
 821 {
 822         unsigned long ecx = cpuid_ecx(1);
 823         return test_bit(5, &ecx); /* CPUID.1:ECX.VMX[bit 5] -> VT */
 824 }
 825
 826 static __init int vmx_disabled_by_bios(void)
 827 {
 828         u64 msr;
 829
 830         rdmsrl(MSR_IA32_FEATURE_CONTROL, msr);
 831         return (msr & (MSR_IA32_FEATURE_CONTROL_LOCKED |
 832                        MSR_IA32_FEATURE_CONTROL_VMXON_ENABLED))
 833             == MSR_IA32_FEATURE_CONTROL_LOCKED;
 834         /* locked but not enabled */
 835 }
 836
 837 static void hardware_enable(void *garbage)
 838 {
 839         int cpu = raw_smp_processor_id();
 840         u64 phys_addr = __pa(per_cpu(vmxarea, cpu));
 841         u64 old;
 842
 843         rdmsrl(MSR_IA32_FEATURE_CONTROL, old);
 844         if ((old & (MSR_IA32_FEATURE_CONTROL_LOCKED |
 845                     MSR_IA32_FEATURE_CONTROL_VMXON_ENABLED))
 846             != (MSR_IA32_FEATURE_CONTROL_LOCKED |
 847                 MSR_IA32_FEATURE_CONTROL_VMXON_ENABLED))
 848                 /* enable and lock */
 849                 wrmsrl(MSR_IA32_FEATURE_CONTROL, old |
 850                        MSR_IA32_FEATURE_CONTROL_LOCKED |
 851                        MSR_IA32_FEATURE_CONTROL_VMXON_ENABLED);
 852         write_cr4(read_cr4() | X86_CR4_VMXE); /* FIXME: not cpu hotplug safe */
 853         asm volatile (ASM_VMX_VMXON_RAX : : "a"(&phys_addr), "m"(phys_addr)
 854                       : "memory", "cc");
 855 }
 856
 857 static void hardware_disable(void *garbage)
 858 {
 859         asm volatile (ASM_VMX_VMXOFF : : : "cc");
 860 }
 861
 862 static __init int adjust_vmx_controls(u32 ctl_min, u32 ctl_opt,
 863                                       u32 msr, u32* result)
 864 {
 865         u32 vmx_msr_low, vmx_msr_high;
 866         u32 ctl = ctl_min | ctl_opt;
 867
 868         rdmsr(msr, vmx_msr_low, vmx_msr_high);
 869
 870         ctl &= vmx_msr_high; /* bit == 0 in high word ==> must be zero */
 871         ctl |= vmx_msr_low;  /* bit == 1 in low word  ==> must be one  */
 872
 873         /* Ensure minimum (required) set of control bits are supported. */
 874         if (ctl_min & ~ctl)
 875                 return -EIO;
 876
 877         *result = ctl;
 878         return 0;
 879 }
 880
 881 static __init int setup_vmcs_config(struct vmcs_config *vmcs_conf)
 882 {
 883         u32 vmx_msr_low, vmx_msr_high;
 884         u32 min, opt;
 885         u32 _pin_based_exec_control = 0;
 886         u32 _cpu_based_exec_control = 0;
 887         u32 _vmexit_control = 0;
 888         u32 _vmentry_control = 0;
 889
 890         min = PIN_BASED_EXT_INTR_MASK | PIN_BASED_NMI_EXITING;
 891         opt = 0;
 892         if (adjust_vmx_controls(min, opt, MSR_IA32_VMX_PINBASED_CTLS,
 893                                 &_pin_based_exec_control) < 0)
 894                 return -EIO;
 895
 896         min = CPU_BASED_HLT_EXITING |
 897 #ifdef CONFIG_X86_64
 898               CPU_BASED_CR8_LOAD_EXITING |
 899               CPU_BASED_CR8_STORE_EXITING |
 900 #endif
 901               CPU_BASED_USE_IO_BITMAPS |
 902               CPU_BASED_MOV_DR_EXITING |
 903               CPU_BASED_USE_TSC_OFFSETING;
 904 #ifdef CONFIG_X86_64
 905         opt = CPU_BASED_TPR_SHADOW;
 906 #else
 907         opt = 0;
 908 #endif
 909         if (adjust_vmx_controls(min, opt, MSR_IA32_VMX_PROCBASED_CTLS,
 910                                 &_cpu_based_exec_control) < 0)
 911                 return -EIO;
 912 #ifdef CONFIG_X86_64
 913         if ((_cpu_based_exec_control & CPU_BASED_TPR_SHADOW))
 914                 _cpu_based_exec_control &= ~CPU_BASED_CR8_LOAD_EXITING &
 915                                            ~CPU_BASED_CR8_STORE_EXITING;
 916 #endif
 917
 918         min = 0;
 919 #ifdef CONFIG_X86_64
 920         min |= VM_EXIT_HOST_ADDR_SPACE_SIZE;
 921 #endif
 922         opt = 0;
 923         if (adjust_vmx_controls(min, opt, MSR_IA32_VMX_EXIT_CTLS,
 924                                 &_vmexit_control) < 0)
 925                 return -EIO;
 926
 927         min = opt = 0;
 928         if (adjust_vmx_controls(min, opt, MSR_IA32_VMX_ENTRY_CTLS,
 929                                 &_vmentry_control) < 0)
 930                 return -EIO;
 931
 932         rdmsr(MSR_IA32_VMX_BASIC, vmx_msr_low, vmx_msr_high);
 933
 934         /* IA-32 SDM Vol 3B: VMCS size is never greater than 4kB. */
 935         if ((vmx_msr_high & 0x1fff) > PAGE_SIZE)
 936                 return -EIO;
 937
 938 #ifdef CONFIG_X86_64
 939         /* IA-32 SDM Vol 3B: 64-bit CPUs always have VMX_BASIC_MSR[48]==0. */
 940         if (vmx_msr_high & (1u<<16))
 941                 return -EIO;
 942 #endif
 943
 944         /* Require Write-Back (WB) memory type for VMCS accesses. */
 945         if (((vmx_msr_high >> 18) & 15) != 6)
 946                 return -EIO;
 947
 948         vmcs_conf->size = vmx_msr_high & 0x1fff;
 949         vmcs_conf->order = get_order(vmcs_config.size);
 950         vmcs_conf->revision_id = vmx_msr_low;
 951
 952         vmcs_conf->pin_based_exec_ctrl = _pin_based_exec_control;
 953         vmcs_conf->cpu_based_exec_ctrl = _cpu_based_exec_control;
 954         vmcs_conf->vmexit_ctrl         = _vmexit_control;
 955         vmcs_conf->vmentry_ctrl        = _vmentry_control;
 956
 957         return 0;
 958 }
 959
 960 static struct vmcs *alloc_vmcs_cpu(int cpu)
 961 {
 962         int node = cpu_to_node(cpu);
 963         struct page *pages;
 964         struct vmcs *vmcs;
 965
 966         pages = alloc_pages_node(node, GFP_KERNEL, vmcs_config.order);
 967         if (!pages)
 968                 return NULL;
 969         vmcs = page_address(pages);
 970         memset(vmcs, 0, vmcs_config.size);
 971         vmcs->revision_id = vmcs_config.revision_id; /* vmcs revision id */
 972         return vmcs;
 973 }
 974
 975 static struct vmcs *alloc_vmcs(void)
 976 {
 977         return alloc_vmcs_cpu(raw_smp_processor_id());
 978 }
 979
 980 static void free_vmcs(struct vmcs *vmcs)
 981 {
 982         free_pages((unsigned long)vmcs, vmcs_config.order);
 983 }
 984
 985 static void free_kvm_area(void)
 986 {
 987         int cpu;
 988
 989         for_each_online_cpu(cpu)
 990                 free_vmcs(per_cpu(vmxarea, cpu));
 991 }
 992
 993 static __init int alloc_kvm_area(void)
 994 {
 995         int cpu;
 996
 997         for_each_online_cpu(cpu) {
 998                 struct vmcs *vmcs;
 999
1000                 vmcs = alloc_vmcs_cpu(cpu);
1001                 if (!vmcs) {
1002                         free_kvm_area();
1003                         return -ENOMEM;
1004                 }
1005
1006                 per_cpu(vmxarea, cpu) = vmcs;
1007         }
1008         return 0;
1009 }
1010
1011 static __init int hardware_setup(void)
1012 {
1013         if (setup_vmcs_config(&vmcs_config) < 0)
1014                 return -EIO;
1015         return alloc_kvm_area();
1016 }
1017
1018 static __exit void hardware_unsetup(void)
1019 {
1020         free_kvm_area();
1021 }
1022
1023 static void fix_pmode_dataseg(int seg, struct kvm_save_segment *save)
1024 {
1025         struct kvm_vmx_segment_field *sf = &kvm_vmx_segment_fields[seg];
1026
1027         if (vmcs_readl(sf->base) == save->base && (save->base & AR_S_MASK)) {
1028                 vmcs_write16(sf->selector, save->selector);
1029                 vmcs_writel(sf->base, save->base);
1030                 vmcs_write32(sf->limit, save->limit);
1031                 vmcs_write32(sf->ar_bytes, save->ar);
1032         } else {
1033                 u32 dpl = (vmcs_read16(sf->selector) & SELECTOR_RPL_MASK)
1034                         << AR_DPL_SHIFT;
1035                 vmcs_write32(sf->ar_bytes, 0x93 | dpl);
1036         }
1037 }
1038
1039 static void enter_pmode(struct kvm_vcpu *vcpu)
1040 {
1041         unsigned long flags;
1042
1043         vcpu->rmode.active = 0;
1044
1045         vmcs_writel(GUEST_TR_BASE, vcpu->rmode.tr.base);
1046         vmcs_write32(GUEST_TR_LIMIT, vcpu->rmode.tr.limit);
1047         vmcs_write32(GUEST_TR_AR_BYTES, vcpu->rmode.tr.ar);
1048
1049         flags = vmcs_readl(GUEST_RFLAGS);
1050         flags &= ~(IOPL_MASK | X86_EFLAGS_VM);
1051         flags |= (vcpu->rmode.save_iopl << IOPL_SHIFT);
1052         vmcs_writel(GUEST_RFLAGS, flags);
1053
1054         vmcs_writel(GUEST_CR4, (vmcs_readl(GUEST_CR4) & ~X86_CR4_VME) |
1055                         (vmcs_readl(CR4_READ_SHADOW) & X86_CR4_VME));
1056
1057         update_exception_bitmap(vcpu);
1058
1059         fix_pmode_dataseg(VCPU_SREG_ES, &vcpu->rmode.es);
1060         fix_pmode_dataseg(VCPU_SREG_DS, &vcpu->rmode.ds);
1061         fix_pmode_dataseg(VCPU_SREG_GS, &vcpu->rmode.gs);
1062         fix_pmode_dataseg(VCPU_SREG_FS, &vcpu->rmode.fs);
1063
1064         vmcs_write16(GUEST_SS_SELECTOR, 0);
1065         vmcs_write32(GUEST_SS_AR_BYTES, 0x93);
1066
1067         vmcs_write16(GUEST_CS_SELECTOR,
1068                      vmcs_read16(GUEST_CS_SELECTOR) & ~SELECTOR_RPL_MASK);
1069         vmcs_write32(GUEST_CS_AR_BYTES, 0x9b);
1070 }
1071
1072 static gva_t rmode_tss_base(struct kvm* kvm)
1073 {
1074         gfn_t base_gfn = kvm->memslots[0].base_gfn + kvm->memslots[0].npages - 3;
1075         return base_gfn << PAGE_SHIFT;
1076 }
1077
1078 static void fix_rmode_seg(int seg, struct kvm_save_segment *save)
1079 {
1080         struct kvm_vmx_segment_field *sf = &kvm_vmx_segment_fields[seg];
1081
1082         save->selector = vmcs_read16(sf->selector);
1083         save->base = vmcs_readl(sf->base);
1084         save->limit = vmcs_read32(sf->limit);
1085         save->ar = vmcs_read32(sf->ar_bytes);
1086         vmcs_write16(sf->selector, vmcs_readl(sf->base) >> 4);
1087         vmcs_write32(sf->limit, 0xffff);
1088         vmcs_write32(sf->ar_bytes, 0xf3);
1089 }
1090
1091 static void enter_rmode(struct kvm_vcpu *vcpu)
1092 {
1093         unsigned long flags;
1094
1095         vcpu->rmode.active = 1;
1096
1097         vcpu->rmode.tr.base = vmcs_readl(GUEST_TR_BASE);
1098         vmcs_writel(GUEST_TR_BASE, rmode_tss_base(vcpu->kvm));
1099
1100         vcpu->rmode.tr.limit = vmcs_read32(GUEST_TR_LIMIT);
1101         vmcs_write32(GUEST_TR_LIMIT, RMODE_TSS_SIZE - 1);
1102
1103         vcpu->rmode.tr.ar = vmcs_read32(GUEST_TR_AR_BYTES);
1104         vmcs_write32(GUEST_TR_AR_BYTES, 0x008b);
1105
1106         flags = vmcs_readl(GUEST_RFLAGS);
1107         vcpu->rmode.save_iopl = (flags & IOPL_MASK) >> IOPL_SHIFT;
1108
1109         flags |= IOPL_MASK | X86_EFLAGS_VM;
1110
1111         vmcs_writel(GUEST_RFLAGS, flags);
1112         vmcs_writel(GUEST_CR4, vmcs_readl(GUEST_CR4) | X86_CR4_VME);
1113         update_exception_bitmap(vcpu);
1114
1115         vmcs_write16(GUEST_SS_SELECTOR, vmcs_readl(GUEST_SS_BASE) >> 4);
1116         vmcs_write32(GUEST_SS_LIMIT, 0xffff);
1117         vmcs_write32(GUEST_SS_AR_BYTES, 0xf3);
1118
1119         vmcs_write32(GUEST_CS_AR_BYTES, 0xf3);
1120         vmcs_write32(GUEST_CS_LIMIT, 0xffff);
1121         if (vmcs_readl(GUEST_CS_BASE) == 0xffff0000)
1122                 vmcs_writel(GUEST_CS_BASE, 0xf0000);
1123         vmcs_write16(GUEST_CS_SELECTOR, vmcs_readl(GUEST_CS_BASE) >> 4);
1124
1125         fix_rmode_seg(VCPU_SREG_ES, &vcpu->rmode.es);
1126         fix_rmode_seg(VCPU_SREG_DS, &vcpu->rmode.ds);
1127         fix_rmode_seg(VCPU_SREG_GS, &vcpu->rmode.gs);
1128         fix_rmode_seg(VCPU_SREG_FS, &vcpu->rmode.fs);
1129
1130         init_rmode_tss(vcpu->kvm);
1131 }
1132
1133 #ifdef CONFIG_X86_64
1134
1135 static void enter_lmode(struct kvm_vcpu *vcpu)
1136 {
1137         u32 guest_tr_ar;
1138
1139         guest_tr_ar = vmcs_read32(GUEST_TR_AR_BYTES);
1140         if ((guest_tr_ar & AR_TYPE_MASK) != AR_TYPE_BUSY_64_TSS) {
1141                 printk(KERN_DEBUG "%s: tss fixup for long mode. \n",
1142                        __FUNCTION__);
1143                 vmcs_write32(GUEST_TR_AR_BYTES,
1144                              (guest_tr_ar & ~AR_TYPE_MASK)
1145                              | AR_TYPE_BUSY_64_TSS);
1146         }
1147
1148         vcpu->shadow_efer |= EFER_LMA;
1149
1150         find_msr_entry(to_vmx(vcpu), MSR_EFER)->data |= EFER_LMA | EFER_LME;
1151         vmcs_write32(VM_ENTRY_CONTROLS,
1152                      vmcs_read32(VM_ENTRY_CONTROLS)
1153                      | VM_ENTRY_IA32E_MODE);
1154 }
1155
1156 static void exit_lmode(struct kvm_vcpu *vcpu)
1157 {
1158         vcpu->shadow_efer &= ~EFER_LMA;
1159
1160         vmcs_write32(VM_ENTRY_CONTROLS,
1161                      vmcs_read32(VM_ENTRY_CONTROLS)
1162                      & ~VM_ENTRY_IA32E_MODE);
1163 }
1164
1165 #endif
1166
1167 static void vmx_decache_cr4_guest_bits(struct kvm_vcpu *vcpu)
1168 {
1169         vcpu->cr4 &= KVM_GUEST_CR4_MASK;
1170         vcpu->cr4 |= vmcs_readl(GUEST_CR4) & ~KVM_GUEST_CR4_MASK;
1171 }
1172
1173 static void vmx_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0)
1174 {
1175         vmx_fpu_deactivate(vcpu);
1176
1177         if (vcpu->rmode.active && (cr0 & X86_CR0_PE))
1178                 enter_pmode(vcpu);
1179
1180         if (!vcpu->rmode.active && !(cr0 & X86_CR0_PE))
1181                 enter_rmode(vcpu);
1182
1183 #ifdef CONFIG_X86_64
1184         if (vcpu->shadow_efer & EFER_LME) {
1185                 if (!is_paging(vcpu) && (cr0 & X86_CR0_PG))
1186                         enter_lmode(vcpu);
1187                 if (is_paging(vcpu) && !(cr0 & X86_CR0_PG))
1188                         exit_lmode(vcpu);
1189         }
1190 #endif
1191
1192         vmcs_writel(CR0_READ_SHADOW, cr0);
1193         vmcs_writel(GUEST_CR0,
1194                     (cr0 & ~KVM_GUEST_CR0_MASK) | KVM_VM_CR0_ALWAYS_ON);
1195         vcpu->cr0 = cr0;
1196
1197         if (!(cr0 & X86_CR0_TS) || !(cr0 & X86_CR0_PE))
1198                 vmx_fpu_activate(vcpu);
1199 }
1200
1201 static void vmx_set_cr3(struct kvm_vcpu *vcpu, unsigned long cr3)
1202 {
1203         vmcs_writel(GUEST_CR3, cr3);
1204         if (vcpu->cr0 & X86_CR0_PE)
1205                 vmx_fpu_deactivate(vcpu);
1206 }
1207
1208 static void vmx_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4)
1209 {
1210         vmcs_writel(CR4_READ_SHADOW, cr4);
1211         vmcs_writel(GUEST_CR4, cr4 | (vcpu->rmode.active ?
1212                     KVM_RMODE_VM_CR4_ALWAYS_ON : KVM_PMODE_VM_CR4_ALWAYS_ON));
1213         vcpu->cr4 = cr4;
1214 }
1215
1216 #ifdef CONFIG_X86_64
1217
1218 static void vmx_set_efer(struct kvm_vcpu *vcpu, u64 efer)
1219 {
1220         struct vcpu_vmx *vmx = to_vmx(vcpu);
1221         struct kvm_msr_entry *msr = find_msr_entry(vmx, MSR_EFER);
1222
1223         vcpu->shadow_efer = efer;
1224         if (efer & EFER_LMA) {
1225                 vmcs_write32(VM_ENTRY_CONTROLS,
1226                                      vmcs_read32(VM_ENTRY_CONTROLS) |
1227                                      VM_ENTRY_IA32E_MODE);
1228                 msr->data = efer;
1229
1230         } else {
1231                 vmcs_write32(VM_ENTRY_CONTROLS,
1232                                      vmcs_read32(VM_ENTRY_CONTROLS) &
1233                                      ~VM_ENTRY_IA32E_MODE);
1234
1235                 msr->data = efer & ~EFER_LME;
1236         }
1237         setup_msrs(vmx);
1238 }
1239
1240 #endif
1241
1242 static u64 vmx_get_segment_base(struct kvm_vcpu *vcpu, int seg)
1243 {
1244         struct kvm_vmx_segment_field *sf = &kvm_vmx_segment_fields[seg];
1245
1246         return vmcs_readl(sf->base);
1247 }
1248
1249 static void vmx_get_segment(struct kvm_vcpu *vcpu,
1250                             struct kvm_segment *var, int seg)
1251 {
1252         struct kvm_vmx_segment_field *sf = &kvm_vmx_segment_fields[seg];
1253         u32 ar;
1254
1255         var->base = vmcs_readl(sf->base);
1256         var->limit = vmcs_read32(sf->limit);
1257         var->selector = vmcs_read16(sf->selector);
1258         ar = vmcs_read32(sf->ar_bytes);
1259         if (ar & AR_UNUSABLE_MASK)
1260                 ar = 0;
1261         var->type = ar & 15;
1262         var->s = (ar >> 4) & 1;
1263         var->dpl = (ar >> 5) & 3;
1264         var->present = (ar >> 7) & 1;
1265         var->avl = (ar >> 12) & 1;
1266         var->l = (ar >> 13) & 1;
1267         var->db = (ar >> 14) & 1;
1268         var->g = (ar >> 15) & 1;
1269         var->unusable = (ar >> 16) & 1;
1270 }
1271
1272 static u32 vmx_segment_access_rights(struct kvm_segment *var)
1273 {
1274         u32 ar;
1275
1276         if (var->unusable)
1277                 ar = 1 << 16;
1278         else {
1279                 ar = var->type & 15;
1280                 ar |= (var->s & 1) << 4;
1281                 ar |= (var->dpl & 3) << 5;
1282                 ar |= (var->present & 1) << 7;
1283                 ar |= (var->avl & 1) << 12;
1284                 ar |= (var->l & 1) << 13;
1285                 ar |= (var->db & 1) << 14;
1286                 ar |= (var->g & 1) << 15;
1287         }
1288         if (ar == 0) /* a 0 value means unusable */
1289                 ar = AR_UNUSABLE_MASK;
1290
1291         return ar;
1292 }
1293
1294 static void vmx_set_segment(struct kvm_vcpu *vcpu,
1295                             struct kvm_segment *var, int seg)
1296 {
1297         struct kvm_vmx_segment_field *sf = &kvm_vmx_segment_fields[seg];
1298         u32 ar;
1299
1300         if (vcpu->rmode.active && seg == VCPU_SREG_TR) {
1301                 vcpu->rmode.tr.selector = var->selector;
1302                 vcpu->rmode.tr.base = var->base;
1303                 vcpu->rmode.tr.limit = var->limit;
1304                 vcpu->rmode.tr.ar = vmx_segment_access_rights(var);
1305                 return;
1306         }
1307         vmcs_writel(sf->base, var->base);
1308         vmcs_write32(sf->limit, var->limit);
1309         vmcs_write16(sf->selector, var->selector);
1310         if (vcpu->rmode.active && var->s) {
1311                 /*
1312                  * Hack real-mode segments into vm86 compatibility.
1313                  */
1314                 if (var->base == 0xffff0000 && var->selector == 0xf000)
1315                         vmcs_writel(sf->base, 0xf0000);
1316                 ar = 0xf3;
1317         } else
1318                 ar = vmx_segment_access_rights(var);
1319         vmcs_write32(sf->ar_bytes, ar);
1320 }
1321
1322 static void vmx_get_cs_db_l_bits(struct kvm_vcpu *vcpu, int *db, int *l)
1323 {
1324         u32 ar = vmcs_read32(GUEST_CS_AR_BYTES);
1325
1326         *db = (ar >> 14) & 1;
1327         *l = (ar >> 13) & 1;
1328 }
1329
1330 static void vmx_get_idt(struct kvm_vcpu *vcpu, struct descriptor_table *dt)
1331 {
1332         dt->limit = vmcs_read32(GUEST_IDTR_LIMIT);
1333         dt->base = vmcs_readl(GUEST_IDTR_BASE);
1334 }
1335
1336 static void vmx_set_idt(struct kvm_vcpu *vcpu, struct descriptor_table *dt)
1337 {
1338         vmcs_write32(GUEST_IDTR_LIMIT, dt->limit);
1339         vmcs_writel(GUEST_IDTR_BASE, dt->base);
1340 }
1341
1342 static void vmx_get_gdt(struct kvm_vcpu *vcpu, struct descriptor_table *dt)
1343 {
1344         dt->limit = vmcs_read32(GUEST_GDTR_LIMIT);
1345         dt->base = vmcs_readl(GUEST_GDTR_BASE);
1346 }
1347
1348 static void vmx_set_gdt(struct kvm_vcpu *vcpu, struct descriptor_table *dt)
1349 {
1350         vmcs_write32(GUEST_GDTR_LIMIT, dt->limit);
1351         vmcs_writel(GUEST_GDTR_BASE, dt->base);
1352 }
1353
1354 static int init_rmode_tss(struct kvm* kvm)
1355 {
1356         struct page *p1, *p2, *p3;
1357         gfn_t fn = rmode_tss_base(kvm) >> PAGE_SHIFT;
1358         char *page;
1359
1360         p1 = gfn_to_page(kvm, fn++);
1361         p2 = gfn_to_page(kvm, fn++);
1362         p3 = gfn_to_page(kvm, fn);
1363
1364         if (!p1 || !p2 || !p3) {
1365                 kvm_printf(kvm,"%s: gfn_to_page failed\n", __FUNCTION__);
1366                 return 0;
1367         }
1368
1369         page = kmap_atomic(p1, KM_USER0);
1370         clear_page(page);
1371         *(u16*)(page + 0x66) = TSS_BASE_SIZE + TSS_REDIRECTION_SIZE;
1372         kunmap_atomic(page, KM_USER0);
1373
1374         page = kmap_atomic(p2, KM_USER0);
1375         clear_page(page);
1376         kunmap_atomic(page, KM_USER0);
1377
1378         page = kmap_atomic(p3, KM_USER0);
1379         clear_page(page);
1380         *(page + RMODE_TSS_SIZE - 2 * PAGE_SIZE - 1) = ~0;
1381         kunmap_atomic(page, KM_USER0);
1382
1383         return 1;
1384 }
1385
1386 static void seg_setup(int seg)
1387 {
1388         struct kvm_vmx_segment_field *sf = &kvm_vmx_segment_fields[seg];
1389
1390         vmcs_write16(sf->selector, 0);
1391         vmcs_writel(sf->base, 0);
1392         vmcs_write32(sf->limit, 0xffff);
1393         vmcs_write32(sf->ar_bytes, 0x93);
1394 }
1395
1396 /*
1397  * Sets up the vmcs for emulated real mode.
1398  */
1399 static int vmx_vcpu_setup(struct vcpu_vmx *vmx)
1400 {
1401         u32 host_sysenter_cs;
1402         u32 junk;
1403         unsigned long a;
1404         struct descriptor_table dt;
1405         int i;
1406         int ret = 0;
1407         unsigned long kvm_vmx_return;
1408         u64 msr;
1409         u32 exec_control;
1410
1411         if (!init_rmode_tss(vmx->vcpu.kvm)) {
1412                 ret = -ENOMEM;
1413                 goto out;
1414         }
1415
1416         vmx->vcpu.rmode.active = 0;
1417
1418         vmx->vcpu.regs[VCPU_REGS_RDX] = get_rdx_init_val();
1419         set_cr8(&vmx->vcpu, 0);
1420         msr = 0xfee00000 | MSR_IA32_APICBASE_ENABLE;
1421         if (vmx->vcpu.vcpu_id == 0)
1422                 msr |= MSR_IA32_APICBASE_BSP;
1423         kvm_set_apic_base(&vmx->vcpu, msr);
1424
1425         fx_init(&vmx->vcpu);
1426
1427         /*
1428          * GUEST_CS_BASE should really be 0xffff0000, but VT vm86 mode
1429          * insists on having GUEST_CS_BASE == GUEST_CS_SELECTOR << 4.  Sigh.
1430          */
1431         if (vmx->vcpu.vcpu_id == 0) {
1432                 vmcs_write16(GUEST_CS_SELECTOR, 0xf000);
1433                 vmcs_writel(GUEST_CS_BASE, 0x000f0000);
1434         } else {
1435                 vmcs_write16(GUEST_CS_SELECTOR, vmx->vcpu.sipi_vector << 8);
1436                 vmcs_writel(GUEST_CS_BASE, vmx->vcpu.sipi_vector << 12);
1437         }
1438         vmcs_write32(GUEST_CS_LIMIT, 0xffff);
1439         vmcs_write32(GUEST_CS_AR_BYTES, 0x9b);
1440
1441         seg_setup(VCPU_SREG_DS);
1442         seg_setup(VCPU_SREG_ES);
1443         seg_setup(VCPU_SREG_FS);
1444         seg_setup(VCPU_SREG_GS);
1445         seg_setup(VCPU_SREG_SS);
1446
1447         vmcs_write16(GUEST_TR_SELECTOR, 0);
1448         vmcs_writel(GUEST_TR_BASE, 0);
1449         vmcs_write32(GUEST_TR_LIMIT, 0xffff);
1450         vmcs_write32(GUEST_TR_AR_BYTES, 0x008b);
1451
1452         vmcs_write16(GUEST_LDTR_SELECTOR, 0);
1453         vmcs_writel(GUEST_LDTR_BASE, 0);
1454         vmcs_write32(GUEST_LDTR_LIMIT, 0xffff);
1455         vmcs_write32(GUEST_LDTR_AR_BYTES, 0x00082);
1456
1457         vmcs_write32(GUEST_SYSENTER_CS, 0);
1458         vmcs_writel(GUEST_SYSENTER_ESP, 0);
1459         vmcs_writel(GUEST_SYSENTER_EIP, 0);
1460
1461         vmcs_writel(GUEST_RFLAGS, 0x02);
1462         if (vmx->vcpu.vcpu_id == 0)
1463                 vmcs_writel(GUEST_RIP, 0xfff0);
1464         else
1465                 vmcs_writel(GUEST_RIP, 0);
1466         vmcs_writel(GUEST_RSP, 0);
1467
1468         //todo: dr0 = dr1 = dr2 = dr3 = 0; dr6 = 0xffff0ff0
1469         vmcs_writel(GUEST_DR7, 0x400);
1470
1471         vmcs_writel(GUEST_GDTR_BASE, 0);
1472         vmcs_write32(GUEST_GDTR_LIMIT, 0xffff);
1473
1474         vmcs_writel(GUEST_IDTR_BASE, 0);
1475         vmcs_write32(GUEST_IDTR_LIMIT, 0xffff);
1476
1477         vmcs_write32(GUEST_ACTIVITY_STATE, 0);
1478         vmcs_write32(GUEST_INTERRUPTIBILITY_INFO, 0);
1479         vmcs_write32(GUEST_PENDING_DBG_EXCEPTIONS, 0);
1480
1481         /* I/O */
1482         vmcs_write64(IO_BITMAP_A, page_to_phys(vmx_io_bitmap_a));
1483         vmcs_write64(IO_BITMAP_B, page_to_phys(vmx_io_bitmap_b));
1484
1485         guest_write_tsc(0);
1486
1487         vmcs_write64(VMCS_LINK_POINTER, -1ull); /* 22.3.1.5 */
1488
1489         /* Special registers */
1490         vmcs_write64(GUEST_IA32_DEBUGCTL, 0);
1491
1492         /* Control */
1493         vmcs_write32(PIN_BASED_VM_EXEC_CONTROL,
1494                 vmcs_config.pin_based_exec_ctrl);
1495
1496         exec_control = vmcs_config.cpu_based_exec_ctrl;
1497         if (!vm_need_tpr_shadow(vmx->vcpu.kvm)) {
1498                 exec_control &= ~CPU_BASED_TPR_SHADOW;
1499 #ifdef CONFIG_X86_64
1500                 exec_control |= CPU_BASED_CR8_STORE_EXITING |
1501                                 CPU_BASED_CR8_LOAD_EXITING;
1502 #endif
1503         }
1504         vmcs_write32(CPU_BASED_VM_EXEC_CONTROL, exec_control);
1505
1506         vmcs_write32(PAGE_FAULT_ERROR_CODE_MASK, 0);
1507         vmcs_write32(PAGE_FAULT_ERROR_CODE_MATCH, 0);
1508         vmcs_write32(CR3_TARGET_COUNT, 0);           /* 22.2.1 */
1509
1510         vmcs_writel(HOST_CR0, read_cr0());  /* 22.2.3 */
1511         vmcs_writel(HOST_CR4, read_cr4());  /* 22.2.3, 22.2.5 */
1512         vmcs_writel(HOST_CR3, read_cr3());  /* 22.2.3  FIXME: shadow tables */
1513
1514         vmcs_write16(HOST_CS_SELECTOR, __KERNEL_CS);  /* 22.2.4 */
1515         vmcs_write16(HOST_DS_SELECTOR, __KERNEL_DS);  /* 22.2.4 */
1516         vmcs_write16(HOST_ES_SELECTOR, __KERNEL_DS);  /* 22.2.4 */
1517         vmcs_write16(HOST_FS_SELECTOR, read_fs());    /* 22.2.4 */
1518         vmcs_write16(HOST_GS_SELECTOR, read_gs());    /* 22.2.4 */
1519         vmcs_write16(HOST_SS_SELECTOR, __KERNEL_DS);  /* 22.2.4 */
1520 #ifdef CONFIG_X86_64
1521         rdmsrl(MSR_FS_BASE, a);
1522         vmcs_writel(HOST_FS_BASE, a); /* 22.2.4 */
1523         rdmsrl(MSR_GS_BASE, a);
1524         vmcs_writel(HOST_GS_BASE, a); /* 22.2.4 */
1525 #else
1526         vmcs_writel(HOST_FS_BASE, 0); /* 22.2.4 */
1527         vmcs_writel(HOST_GS_BASE, 0); /* 22.2.4 */
1528 #endif
1529
1530         vmcs_write16(HOST_TR_SELECTOR, GDT_ENTRY_TSS*8);  /* 22.2.4 */
1531
1532         get_idt(&dt);
1533         vmcs_writel(HOST_IDTR_BASE, dt.base);   /* 22.2.4 */
1534
1535         asm ("mov $.Lkvm_vmx_return, %0" : "=r"(kvm_vmx_return));
1536         vmcs_writel(HOST_RIP, kvm_vmx_return); /* 22.2.5 */
1537         vmcs_write32(VM_EXIT_MSR_STORE_COUNT, 0);
1538         vmcs_write32(VM_EXIT_MSR_LOAD_COUNT, 0);
1539         vmcs_write32(VM_ENTRY_MSR_LOAD_COUNT, 0);
1540
1541         rdmsr(MSR_IA32_SYSENTER_CS, host_sysenter_cs, junk);
1542         vmcs_write32(HOST_IA32_SYSENTER_CS, host_sysenter_cs);
1543         rdmsrl(MSR_IA32_SYSENTER_ESP, a);
1544         vmcs_writel(HOST_IA32_SYSENTER_ESP, a);   /* 22.2.3 */
1545         rdmsrl(MSR_IA32_SYSENTER_EIP, a);
1546         vmcs_writel(HOST_IA32_SYSENTER_EIP, a);   /* 22.2.3 */
1547
1548         for (i = 0; i < NR_VMX_MSR; ++i) {
1549                 u32 index = vmx_msr_index[i];
1550                 u32 data_low, data_high;
1551                 u64 data;
1552                 int j = vmx->nmsrs;
1553
1554                 if (rdmsr_safe(index, &data_low, &data_high) < 0)
1555                         continue;
1556                 if (wrmsr_safe(index, data_low, data_high) < 0)
1557                         continue;
1558                 data = data_low | ((u64)data_high << 32);
1559                 vmx->host_msrs[j].index = index;
1560                 vmx->host_msrs[j].reserved = 0;
1561                 vmx->host_msrs[j].data = data;
1562                 vmx->guest_msrs[j] = vmx->host_msrs[j];
1563                 ++vmx->nmsrs;
1564         }
1565
1566         setup_msrs(vmx);
1567
1568         vmcs_write32(VM_EXIT_CONTROLS, vmcs_config.vmexit_ctrl);
1569
1570         /* 22.2.1, 20.8.1 */
1571         vmcs_write32(VM_ENTRY_CONTROLS, vmcs_config.vmentry_ctrl);
1572
1573         vmcs_write32(VM_ENTRY_INTR_INFO_FIELD, 0);  /* 22.2.1 */
1574
1575 #ifdef CONFIG_X86_64
1576         vmcs_write64(VIRTUAL_APIC_PAGE_ADDR, 0);
1577         if (vm_need_tpr_shadow(vmx->vcpu.kvm))
1578                 vmcs_write64(VIRTUAL_APIC_PAGE_ADDR,
1579                              page_to_phys(vmx->vcpu.apic->regs_page));
1580         vmcs_write32(TPR_THRESHOLD, 0);
1581 #endif
1582
1583         vmcs_writel(CR0_GUEST_HOST_MASK, ~0UL);
1584         vmcs_writel(CR4_GUEST_HOST_MASK, KVM_GUEST_CR4_MASK);
1585
1586         vmx->vcpu.cr0 = 0x60000010;
1587         vmx_set_cr0(&vmx->vcpu, vmx->vcpu.cr0); // enter rmode
1588         vmx_set_cr4(&vmx->vcpu, 0);
1589 #ifdef CONFIG_X86_64
1590         vmx_set_efer(&vmx->vcpu, 0);
1591 #endif
1592         vmx_fpu_activate(&vmx->vcpu);
1593         update_exception_bitmap(&vmx->vcpu);
1594
1595         return 0;
1596
1597 out:
1598         return ret;
1599 }
1600
1601 static void inject_rmode_irq(struct kvm_vcpu *vcpu, int irq)
1602 {
1603         u16 ent[2];
1604         u16 cs;
1605         u16 ip;
1606         unsigned long flags;
1607         unsigned long ss_base = vmcs_readl(GUEST_SS_BASE);
1608         u16 sp =  vmcs_readl(GUEST_RSP);
1609         u32 ss_limit = vmcs_read32(GUEST_SS_LIMIT);
1610
1611         if (sp > ss_limit || sp < 6 ) {
1612                 vcpu_printf(vcpu, "%s: #SS, rsp 0x%lx ss 0x%lx limit 0x%x\n",
1613                             __FUNCTION__,
1614                             vmcs_readl(GUEST_RSP),
1615                             vmcs_readl(GUEST_SS_BASE),
1616                             vmcs_read32(GUEST_SS_LIMIT));
1617                 return;
1618         }
1619
1620         if (emulator_read_std(irq * sizeof(ent), &ent, sizeof(ent), vcpu) !=
1621                                                         X86EMUL_CONTINUE) {
1622                 vcpu_printf(vcpu, "%s: read guest err\n", __FUNCTION__);
1623                 return;
1624         }
1625
1626         flags =  vmcs_readl(GUEST_RFLAGS);
1627         cs =  vmcs_readl(GUEST_CS_BASE) >> 4;
1628         ip =  vmcs_readl(GUEST_RIP);
1629
1630
1631         if (emulator_write_emulated(ss_base + sp - 2, &flags, 2, vcpu) != X86EMUL_CONTINUE ||
1632             emulator_write_emulated(ss_base + sp - 4, &cs, 2, vcpu) != X86EMUL_CONTINUE ||
1633             emulator_write_emulated(ss_base + sp - 6, &ip, 2, vcpu) != X86EMUL_CONTINUE) {
1634                 vcpu_printf(vcpu, "%s: write guest err\n", __FUNCTION__);
1635                 return;
1636         }
1637
1638         vmcs_writel(GUEST_RFLAGS, flags &
1639                     ~( X86_EFLAGS_IF | X86_EFLAGS_AC | X86_EFLAGS_TF));
1640         vmcs_write16(GUEST_CS_SELECTOR, ent[1]) ;
1641         vmcs_writel(GUEST_CS_BASE, ent[1] << 4);
1642         vmcs_writel(GUEST_RIP, ent[0]);
1643         vmcs_writel(GUEST_RSP, (vmcs_readl(GUEST_RSP) & ~0xffff) | (sp - 6));
1644 }
1645
1646 static void vmx_inject_irq(struct kvm_vcpu *vcpu, int irq)
1647 {
1648         if (vcpu->rmode.active) {
1649                 inject_rmode_irq(vcpu, irq);
1650                 return;
1651         }
1652         vmcs_write32(VM_ENTRY_INTR_INFO_FIELD,
1653                         irq | INTR_TYPE_EXT_INTR | INTR_INFO_VALID_MASK);
1654 }
1655
1656 static void kvm_do_inject_irq(struct kvm_vcpu *vcpu)
1657 {
1658         int word_index = __ffs(vcpu->irq_summary);
1659         int bit_index = __ffs(vcpu->irq_pending[word_index]);
1660         int irq = word_index * BITS_PER_LONG + bit_index;
1661
1662         clear_bit(bit_index, &vcpu->irq_pending[word_index]);
1663         if (!vcpu->irq_pending[word_index])
1664                 clear_bit(word_index, &vcpu->irq_summary);
1665         vmx_inject_irq(vcpu, irq);
1666 }
1667
1668
1669 static void do_interrupt_requests(struct kvm_vcpu *vcpu,
1670                                        struct kvm_run *kvm_run)
1671 {
1672         u32 cpu_based_vm_exec_control;
1673
1674         vcpu->interrupt_window_open =
1675                 ((vmcs_readl(GUEST_RFLAGS) & X86_EFLAGS_IF) &&
1676                  (vmcs_read32(GUEST_INTERRUPTIBILITY_INFO) & 3) == 0);
1677
1678         if (vcpu->interrupt_window_open &&
1679             vcpu->irq_summary &&
1680             !(vmcs_read32(VM_ENTRY_INTR_INFO_FIELD) & INTR_INFO_VALID_MASK))
1681                 /*
1682                  * If interrupts enabled, and not blocked by sti or mov ss. Good.
1683                  */
1684                 kvm_do_inject_irq(vcpu);
1685
1686         cpu_based_vm_exec_control = vmcs_read32(CPU_BASED_VM_EXEC_CONTROL);
1687         if (!vcpu->interrupt_window_open &&
1688             (vcpu->irq_summary || kvm_run->request_interrupt_window))
1689                 /*
1690                  * Interrupts blocked.  Wait for unblock.
1691                  */
1692                 cpu_based_vm_exec_control |= CPU_BASED_VIRTUAL_INTR_PENDING;
1693         else
1694                 cpu_based_vm_exec_control &= ~CPU_BASED_VIRTUAL_INTR_PENDING;
1695         vmcs_write32(CPU_BASED_VM_EXEC_CONTROL, cpu_based_vm_exec_control);
1696 }
1697
1698 static void kvm_guest_debug_pre(struct kvm_vcpu *vcpu)
1699 {
1700         struct kvm_guest_debug *dbg = &vcpu->guest_debug;
1701
1702         set_debugreg(dbg->bp[0], 0);
1703         set_debugreg(dbg->bp[1], 1);
1704         set_debugreg(dbg->bp[2], 2);
1705         set_debugreg(dbg->bp[3], 3);
1706
1707         if (dbg->singlestep) {
1708                 unsigned long flags;
1709
1710                 flags = vmcs_readl(GUEST_RFLAGS);
1711                 flags |= X86_EFLAGS_TF | X86_EFLAGS_RF;
1712                 vmcs_writel(GUEST_RFLAGS, flags);
1713         }
1714 }
1715
1716 static int handle_rmode_exception(struct kvm_vcpu *vcpu,
1717                                   int vec, u32 err_code)
1718 {
1719         if (!vcpu->rmode.active)
1720                 return 0;
1721
1722         /*
1723          * Instruction with address size override prefix opcode 0x67
1724          * Cause the #SS fault with 0 error code in VM86 mode.
1725          */
1726         if (((vec == GP_VECTOR) || (vec == SS_VECTOR)) && err_code == 0)
1727                 if (emulate_instruction(vcpu, NULL, 0, 0) == EMULATE_DONE)
1728                         return 1;
1729         return 0;
1730 }
1731
1732 static int handle_exception(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
1733 {
1734         u32 intr_info, error_code;
1735         unsigned long cr2, rip;
1736         u32 vect_info;
1737         enum emulation_result er;
1738         int r;
1739
1740         vect_info = vmcs_read32(IDT_VECTORING_INFO_FIELD);
1741         intr_info = vmcs_read32(VM_EXIT_INTR_INFO);
1742
1743         if ((vect_info & VECTORING_INFO_VALID_MASK) &&
1744                                                 !is_page_fault(intr_info)) {
1745                 printk(KERN_ERR "%s: unexpected, vectoring info 0x%x "
1746                        "intr info 0x%x\n", __FUNCTION__, vect_info, intr_info);
1747         }
1748
1749         if (!irqchip_in_kernel(vcpu->kvm) && is_external_interrupt(vect_info)) {
1750                 int irq = vect_info & VECTORING_INFO_VECTOR_MASK;
1751                 set_bit(irq, vcpu->irq_pending);
1752                 set_bit(irq / BITS_PER_LONG, &vcpu->irq_summary);
1753         }
1754
1755         if ((intr_info & INTR_INFO_INTR_TYPE_MASK) == 0x200) { /* nmi */
1756                 asm ("int $2");
1757                 return 1;
1758         }
1759
1760         if (is_no_device(intr_info)) {
1761                 vmx_fpu_activate(vcpu);
1762                 return 1;
1763         }
1764
1765         error_code = 0;
1766         rip = vmcs_readl(GUEST_RIP);
1767         if (intr_info & INTR_INFO_DELIEVER_CODE_MASK)
1768                 error_code = vmcs_read32(VM_EXIT_INTR_ERROR_CODE);
1769         if (is_page_fault(intr_info)) {
1770                 cr2 = vmcs_readl(EXIT_QUALIFICATION);
1771
1772                 mutex_lock(&vcpu->kvm->lock);
1773                 r = kvm_mmu_page_fault(vcpu, cr2, error_code);
1774                 if (r < 0) {
1775                         mutex_unlock(&vcpu->kvm->lock);
1776                         return r;
1777                 }
1778                 if (!r) {
1779                         mutex_unlock(&vcpu->kvm->lock);
1780                         return 1;
1781                 }
1782
1783                 er = emulate_instruction(vcpu, kvm_run, cr2, error_code);
1784                 mutex_unlock(&vcpu->kvm->lock);
1785
1786                 switch (er) {
1787                 case EMULATE_DONE:
1788                         return 1;
1789                 case EMULATE_DO_MMIO:
1790                         ++vcpu->stat.mmio_exits;
1791                         return 0;
1792                  case EMULATE_FAIL:
1793                         vcpu_printf(vcpu, "%s: emulate fail\n", __FUNCTION__);
1794                         break;
1795                 default:
1796                         BUG();
1797                 }
1798         }
1799
1800         if (vcpu->rmode.active &&
1801             handle_rmode_exception(vcpu, intr_info & INTR_INFO_VECTOR_MASK,
1802                                                                 error_code)) {
1803                 if (vcpu->halt_request) {
1804                         vcpu->halt_request = 0;
1805                         return kvm_emulate_halt(vcpu);
1806                 }
1807                 return 1;
1808         }
1809
1810         if ((intr_info & (INTR_INFO_INTR_TYPE_MASK | INTR_INFO_VECTOR_MASK)) == (INTR_TYPE_EXCEPTION | 1)) {
1811                 kvm_run->exit_reason = KVM_EXIT_DEBUG;
1812                 return 0;
1813         }
1814         kvm_run->exit_reason = KVM_EXIT_EXCEPTION;
1815         kvm_run->ex.exception = intr_info & INTR_INFO_VECTOR_MASK;
1816         kvm_run->ex.error_code = error_code;
1817         return 0;
1818 }
1819
1820 static int handle_external_interrupt(struct kvm_vcpu *vcpu,
1821                                      struct kvm_run *kvm_run)
1822 {
1823         ++vcpu->stat.irq_exits;
1824         return 1;
1825 }
1826
1827 static int handle_triple_fault(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
1828 {
1829         kvm_run->exit_reason = KVM_EXIT_SHUTDOWN;
1830         return 0;
1831 }
1832
1833 static int handle_io(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
1834 {
1835         u64 exit_qualification;
1836         int size, down, in, string, rep;
1837         unsigned port;
1838
1839         ++vcpu->stat.io_exits;
1840         exit_qualification = vmcs_read64(EXIT_QUALIFICATION);
1841         string = (exit_qualification & 16) != 0;
1842
1843         if (string) {
1844                 if (emulate_instruction(vcpu, kvm_run, 0, 0) == EMULATE_DO_MMIO)
1845                         return 0;
1846                 return 1;
1847         }
1848
1849         size = (exit_qualification & 7) + 1;
1850         in = (exit_qualification & 8) != 0;
1851         down = (vmcs_readl(GUEST_RFLAGS) & X86_EFLAGS_DF) != 0;
1852         rep = (exit_qualification & 32) != 0;
1853         port = exit_qualification >> 16;
1854
1855         return kvm_emulate_pio(vcpu, kvm_run, in, size, port);
1856 }
1857
1858 static void
1859 vmx_patch_hypercall(struct kvm_vcpu *vcpu, unsigned char *hypercall)
1860 {
1861         /*
1862          * Patch in the VMCALL instruction:
1863          */
1864         hypercall[0] = 0x0f;
1865         hypercall[1] = 0x01;
1866         hypercall[2] = 0xc1;
1867         hypercall[3] = 0xc3;
1868 }
1869
1870 static int handle_cr(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
1871 {
1872         u64 exit_qualification;
1873         int cr;
1874         int reg;
1875
1876         exit_qualification = vmcs_read64(EXIT_QUALIFICATION);
1877         cr = exit_qualification & 15;
1878         reg = (exit_qualification >> 8) & 15;
1879         switch ((exit_qualification >> 4) & 3) {
1880         case 0: /* mov to cr */
1881                 switch (cr) {
1882                 case 0:
1883                         vcpu_load_rsp_rip(vcpu);
1884                         set_cr0(vcpu, vcpu->regs[reg]);
1885                         skip_emulated_instruction(vcpu);
1886                         return 1;
1887                 case 3:
1888                         vcpu_load_rsp_rip(vcpu);
1889                         set_cr3(vcpu, vcpu->regs[reg]);
1890                         skip_emulated_instruction(vcpu);
1891                         return 1;
1892                 case 4:
1893                         vcpu_load_rsp_rip(vcpu);
1894                         set_cr4(vcpu, vcpu->regs[reg]);
1895                         skip_emulated_instruction(vcpu);
1896                         return 1;
1897                 case 8:
1898                         vcpu_load_rsp_rip(vcpu);
1899                         set_cr8(vcpu, vcpu->regs[reg]);
1900                         skip_emulated_instruction(vcpu);
1901                         kvm_run->exit_reason = KVM_EXIT_SET_TPR;
1902                         return 0;
1903                 };
1904                 break;
1905         case 2: /* clts */
1906                 vcpu_load_rsp_rip(vcpu);
1907                 vmx_fpu_deactivate(vcpu);
1908                 vcpu->cr0 &= ~X86_CR0_TS;
1909                 vmcs_writel(CR0_READ_SHADOW, vcpu->cr0);
1910                 vmx_fpu_activate(vcpu);
1911                 skip_emulated_instruction(vcpu);
1912                 return 1;
1913         case 1: /*mov from cr*/
1914                 switch (cr) {
1915                 case 3:
1916                         vcpu_load_rsp_rip(vcpu);
1917                         vcpu->regs[reg] = vcpu->cr3;
1918                         vcpu_put_rsp_rip(vcpu);
1919                         skip_emulated_instruction(vcpu);
1920                         return 1;
1921                 case 8:
1922                         vcpu_load_rsp_rip(vcpu);
1923                         vcpu->regs[reg] = get_cr8(vcpu);
1924                         vcpu_put_rsp_rip(vcpu);
1925                         skip_emulated_instruction(vcpu);
1926                         return 1;
1927                 }
1928                 break;
1929         case 3: /* lmsw */
1930                 lmsw(vcpu, (exit_qualification >> LMSW_SOURCE_DATA_SHIFT) & 0x0f);
1931
1932                 skip_emulated_instruction(vcpu);
1933                 return 1;
1934         default:
1935                 break;
1936         }
1937         kvm_run->exit_reason = 0;
1938         pr_unimpl(vcpu, "unhandled control register: op %d cr %d\n",
1939                (int)(exit_qualification >> 4) & 3, cr);
1940         return 0;
1941 }
1942
1943 static int handle_dr(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
1944 {
1945         u64 exit_qualification;
1946         unsigned long val;
1947         int dr, reg;
1948
1949         /*
1950          * FIXME: this code assumes the host is debugging the guest.
1951          *        need to deal with guest debugging itself too.
1952          */
1953         exit_qualification = vmcs_read64(EXIT_QUALIFICATION);
1954         dr = exit_qualification & 7;
1955         reg = (exit_qualification >> 8) & 15;
1956         vcpu_load_rsp_rip(vcpu);
1957         if (exit_qualification & 16) {
1958                 /* mov from dr */
1959                 switch (dr) {
1960                 case 6:
1961                         val = 0xffff0ff0;
1962                         break;
1963                 case 7:
1964                         val = 0x400;
1965                         break;
1966                 default:
1967                         val = 0;
1968                 }
1969                 vcpu->regs[reg] = val;
1970         } else {
1971                 /* mov to dr */
1972         }
1973         vcpu_put_rsp_rip(vcpu);
1974         skip_emulated_instruction(vcpu);
1975         return 1;
1976 }
1977
1978 static int handle_cpuid(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
1979 {
1980         kvm_emulate_cpuid(vcpu);
1981         return 1;
1982 }
1983
1984 static int handle_rdmsr(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
1985 {
1986         u32 ecx = vcpu->regs[VCPU_REGS_RCX];
1987         u64 data;
1988
1989         if (vmx_get_msr(vcpu, ecx, &data)) {
1990                 vmx_inject_gp(vcpu, 0);
1991                 return 1;
1992         }
1993
1994         /* FIXME: handling of bits 32:63 of rax, rdx */
1995         vcpu->regs[VCPU_REGS_RAX] = data & -1u;
1996         vcpu->regs[VCPU_REGS_RDX] = (data >> 32) & -1u;
1997         skip_emulated_instruction(vcpu);
1998         return 1;
1999 }
2000
2001 static int handle_wrmsr(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
2002 {
2003         u32 ecx = vcpu->regs[VCPU_REGS_RCX];
2004         u64 data = (vcpu->regs[VCPU_REGS_RAX] & -1u)
2005                 | ((u64)(vcpu->regs[VCPU_REGS_RDX] & -1u) << 32);
2006
2007         if (vmx_set_msr(vcpu, ecx, data) != 0) {
2008                 vmx_inject_gp(vcpu, 0);
2009                 return 1;
2010         }
2011
2012         skip_emulated_instruction(vcpu);
2013         return 1;
2014 }
2015
2016 static int handle_tpr_below_threshold(struct kvm_vcpu *vcpu,
2017                                       struct kvm_run *kvm_run)
2018 {
2019         return 1;
2020 }
2021
2022 static void post_kvm_run_save(struct kvm_vcpu *vcpu,
2023                               struct kvm_run *kvm_run)
2024 {
2025         kvm_run->if_flag = (vmcs_readl(GUEST_RFLAGS) & X86_EFLAGS_IF) != 0;
2026         kvm_run->cr8 = get_cr8(vcpu);
2027         kvm_run->apic_base = kvm_get_apic_base(vcpu);
2028         if (irqchip_in_kernel(vcpu->kvm))
2029                 kvm_run->ready_for_interrupt_injection = 1;
2030         else
2031                 kvm_run->ready_for_interrupt_injection =
2032                                         (vcpu->interrupt_window_open &&
2033                                          vcpu->irq_summary == 0);
2034 }
2035
2036 static int handle_interrupt_window(struct kvm_vcpu *vcpu,
2037                                    struct kvm_run *kvm_run)
2038 {
2039         u32 cpu_based_vm_exec_control;
2040
2041         /* clear pending irq */
2042         cpu_based_vm_exec_control = vmcs_read32(CPU_BASED_VM_EXEC_CONTROL);
2043         cpu_based_vm_exec_control &= ~CPU_BASED_VIRTUAL_INTR_PENDING;
2044         vmcs_write32(CPU_BASED_VM_EXEC_CONTROL, cpu_based_vm_exec_control);
2045         /*
2046          * If the user space waits to inject interrupts, exit as soon as
2047          * possible
2048          */
2049         if (kvm_run->request_interrupt_window &&
2050             !vcpu->irq_summary) {
2051                 kvm_run->exit_reason = KVM_EXIT_IRQ_WINDOW_OPEN;
2052                 ++vcpu->stat.irq_window_exits;
2053                 return 0;
2054         }
2055         return 1;
2056 }
2057
2058 static int handle_halt(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
2059 {
2060         skip_emulated_instruction(vcpu);
2061         return kvm_emulate_halt(vcpu);
2062 }
2063
2064 static int handle_vmcall(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
2065 {
2066         skip_emulated_instruction(vcpu);
2067         return kvm_hypercall(vcpu, kvm_run);
2068 }
2069
2070 /*
2071  * The exit handlers return 1 if the exit was handled fully and guest execution
2072  * may resume.  Otherwise they set the kvm_run parameter to indicate what needs
2073  * to be done to userspace and return 0.
2074  */
2075 static int (*kvm_vmx_exit_handlers[])(struct kvm_vcpu *vcpu,
2076                                       struct kvm_run *kvm_run) = {
2077         [EXIT_REASON_EXCEPTION_NMI]           = handle_exception,
2078         [EXIT_REASON_EXTERNAL_INTERRUPT]      = handle_external_interrupt,
2079         [EXIT_REASON_TRIPLE_FAULT]            = handle_triple_fault,
2080         [EXIT_REASON_IO_INSTRUCTION]          = handle_io,
2081         [EXIT_REASON_CR_ACCESS]               = handle_cr,
2082         [EXIT_REASON_DR_ACCESS]               = handle_dr,
2083         [EXIT_REASON_CPUID]                   = handle_cpuid,
2084         [EXIT_REASON_MSR_READ]                = handle_rdmsr,
2085         [EXIT_REASON_MSR_WRITE]               = handle_wrmsr,
2086         [EXIT_REASON_PENDING_INTERRUPT]       = handle_interrupt_window,
2087         [EXIT_REASON_HLT]                     = handle_halt,
2088         [EXIT_REASON_VMCALL]                  = handle_vmcall,
2089         [EXIT_REASON_TPR_BELOW_THRESHOLD]     = handle_tpr_below_threshold
2090 };
2091
2092 static const int kvm_vmx_max_exit_handlers =
2093         ARRAY_SIZE(kvm_vmx_exit_handlers);
2094
2095 /*
2096  * The guest has exited.  See if we can fix it or if we need userspace
2097  * assistance.
2098  */
2099 static int kvm_handle_exit(struct kvm_run *kvm_run, struct kvm_vcpu *vcpu)
2100 {
2101         u32 vectoring_info = vmcs_read32(IDT_VECTORING_INFO_FIELD);
2102         u32 exit_reason = vmcs_read32(VM_EXIT_REASON);
2103         struct vcpu_vmx *vmx = to_vmx(vcpu);
2104
2105         if (unlikely(vmx->fail)) {
2106                 kvm_run->exit_reason = KVM_EXIT_FAIL_ENTRY;
2107                 kvm_run->fail_entry.hardware_entry_failure_reason
2108                         = vmcs_read32(VM_INSTRUCTION_ERROR);
2109                 return 0;
2110         }
2111
2112         if ( (vectoring_info & VECTORING_INFO_VALID_MASK) &&
2113                                 exit_reason != EXIT_REASON_EXCEPTION_NMI )
2114                 printk(KERN_WARNING "%s: unexpected, valid vectoring info and "
2115                        "exit reason is 0x%x\n", __FUNCTION__, exit_reason);
2116         if (exit_reason < kvm_vmx_max_exit_handlers
2117             && kvm_vmx_exit_handlers[exit_reason])
2118                 return kvm_vmx_exit_handlers[exit_reason](vcpu, kvm_run);
2119         else {
2120                 kvm_run->exit_reason = KVM_EXIT_UNKNOWN;
2121                 kvm_run->hw.hardware_exit_reason = exit_reason;
2122         }
2123         return 0;
2124 }
2125
2126 /*
2127  * Check if userspace requested an interrupt window, and that the
2128  * interrupt window is open.
2129  *
2130  * No need to exit to userspace if we already have an interrupt queued.
2131  */
2132 static int dm_request_for_irq_injection(struct kvm_vcpu *vcpu,
2133                                           struct kvm_run *kvm_run)
2134 {
2135         return (!vcpu->irq_summary &&
2136                 kvm_run->request_interrupt_window &&
2137                 vcpu->interrupt_window_open &&
2138                 (vmcs_readl(GUEST_RFLAGS) & X86_EFLAGS_IF));
2139 }
2140
2141 static void vmx_flush_tlb(struct kvm_vcpu *vcpu)
2142 {
2143 }
2144
2145 static void update_tpr_threshold(struct kvm_vcpu *vcpu)
2146 {
2147         int max_irr, tpr;
2148
2149         if (!vm_need_tpr_shadow(vcpu->kvm))
2150                 return;
2151
2152         if (!kvm_lapic_enabled(vcpu) ||
2153             ((max_irr = kvm_lapic_find_highest_irr(vcpu)) == -1)) {
2154                 vmcs_write32(TPR_THRESHOLD, 0);
2155                 return;
2156         }
2157
2158         tpr = (kvm_lapic_get_cr8(vcpu) & 0x0f) << 4;
2159         vmcs_write32(TPR_THRESHOLD, (max_irr > tpr) ? tpr >> 4 : max_irr >> 4);
2160 }
2161
2162 static void enable_irq_window(struct kvm_vcpu *vcpu)
2163 {
2164         u32 cpu_based_vm_exec_control;
2165
2166         cpu_based_vm_exec_control = vmcs_read32(CPU_BASED_VM_EXEC_CONTROL);
2167         cpu_based_vm_exec_control |= CPU_BASED_VIRTUAL_INTR_PENDING;
2168         vmcs_write32(CPU_BASED_VM_EXEC_CONTROL, cpu_based_vm_exec_control);
2169 }
2170
2171 static void vmx_intr_assist(struct kvm_vcpu *vcpu)
2172 {
2173         u32 idtv_info_field, intr_info_field;
2174         int has_ext_irq, interrupt_window_open;
2175         int vector;
2176
2177         kvm_inject_pending_timer_irqs(vcpu);
2178         update_tpr_threshold(vcpu);
2179
2180         has_ext_irq = kvm_cpu_has_interrupt(vcpu);
2181         intr_info_field = vmcs_read32(VM_ENTRY_INTR_INFO_FIELD);
2182         idtv_info_field = vmcs_read32(IDT_VECTORING_INFO_FIELD);
2183         if (intr_info_field & INTR_INFO_VALID_MASK) {
2184                 if (idtv_info_field & INTR_INFO_VALID_MASK) {
2185                         /* TODO: fault when IDT_Vectoring */
2186                         printk(KERN_ERR "Fault when IDT_Vectoring\n");
2187                 }
2188                 if (has_ext_irq)
2189                         enable_irq_window(vcpu);
2190                 return;
2191         }
2192         if (unlikely(idtv_info_field & INTR_INFO_VALID_MASK)) {
2193                 vmcs_write32(VM_ENTRY_INTR_INFO_FIELD, idtv_info_field);
2194                 vmcs_write32(VM_ENTRY_INSTRUCTION_LEN,
2195                                 vmcs_read32(VM_EXIT_INSTRUCTION_LEN));
2196
2197                 if (unlikely(idtv_info_field & INTR_INFO_DELIEVER_CODE_MASK))
2198                         vmcs_write32(VM_ENTRY_EXCEPTION_ERROR_CODE,
2199                                 vmcs_read32(IDT_VECTORING_ERROR_CODE));
2200                 if (unlikely(has_ext_irq))
2201                         enable_irq_window(vcpu);
2202                 return;
2203         }
2204         if (!has_ext_irq)
2205                 return;
2206         interrupt_window_open =
2207                 ((vmcs_readl(GUEST_RFLAGS) & X86_EFLAGS_IF) &&
2208                  (vmcs_read32(GUEST_INTERRUPTIBILITY_INFO) & 3) == 0);
2209         if (interrupt_window_open) {
2210                 vector = kvm_cpu_get_interrupt(vcpu);
2211                 vmx_inject_irq(vcpu, vector);
2212                 kvm_timer_intr_post(vcpu, vector);
2213         } else
2214                 enable_irq_window(vcpu);
2215 }
2216
2217 static int vmx_vcpu_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
2218 {
2219         struct vcpu_vmx *vmx = to_vmx(vcpu);
2220         int r;
2221
2222         if (unlikely(vcpu->mp_state == VCPU_MP_STATE_SIPI_RECEIVED)) {
2223                 printk("vcpu %d received sipi with vector # %x\n",
2224                        vcpu->vcpu_id, vcpu->sipi_vector);
2225                 kvm_lapic_reset(vcpu);
2226                 vmx_vcpu_setup(vmx);
2227                 vcpu->mp_state = VCPU_MP_STATE_RUNNABLE;
2228         }
2229
2230 preempted:
2231         if (vcpu->guest_debug.enabled)
2232                 kvm_guest_debug_pre(vcpu);
2233
2234 again:
2235         r = kvm_mmu_reload(vcpu);
2236         if (unlikely(r))
2237                 goto out;
2238
2239         preempt_disable();
2240
2241         vmx_save_host_state(vmx);
2242         kvm_load_guest_fpu(vcpu);
2243
2244         /*
2245          * Loading guest fpu may have cleared host cr0.ts
2246          */
2247         vmcs_writel(HOST_CR0, read_cr0());
2248
2249         local_irq_disable();
2250
2251         if (signal_pending(current)) {
2252                 local_irq_enable();
2253                 preempt_enable();
2254                 r = -EINTR;
2255                 kvm_run->exit_reason = KVM_EXIT_INTR;
2256                 ++vcpu->stat.signal_exits;
2257                 goto out;
2258         }
2259
2260         if (irqchip_in_kernel(vcpu->kvm))
2261                 vmx_intr_assist(vcpu);
2262         else if (!vcpu->mmio_read_completed)
2263                 do_interrupt_requests(vcpu, kvm_run);
2264
2265         vcpu->guest_mode = 1;
2266         if (vcpu->requests)
2267                 if (test_and_clear_bit(KVM_TLB_FLUSH, &vcpu->requests))
2268                     vmx_flush_tlb(vcpu);
2269
2270         asm (
2271                 /* Store host registers */
2272 #ifdef CONFIG_X86_64
2273                 "push %%rax; push %%rbx; push %%rdx;"
2274                 "push %%rsi; push %%rdi; push %%rbp;"
2275                 "push %%r8;  push %%r9;  push %%r10; push %%r11;"
2276                 "push %%r12; push %%r13; push %%r14; push %%r15;"
2277                 "push %%rcx \n\t"
2278                 ASM_VMX_VMWRITE_RSP_RDX "\n\t"
2279 #else
2280                 "pusha; push %%ecx \n\t"
2281                 ASM_VMX_VMWRITE_RSP_RDX "\n\t"
2282 #endif
2283                 /* Check if vmlaunch of vmresume is needed */
2284                 "cmp $0, %1 \n\t"
2285                 /* Load guest registers.  Don't clobber flags. */
2286 #ifdef CONFIG_X86_64
2287                 "mov %c[cr2](%3), %%rax \n\t"
2288                 "mov %%rax, %%cr2 \n\t"
2289                 "mov %c[rax](%3), %%rax \n\t"
2290                 "mov %c[rbx](%3), %%rbx \n\t"
2291                 "mov %c[rdx](%3), %%rdx \n\t"
2292                 "mov %c[rsi](%3), %%rsi \n\t"
2293                 "mov %c[rdi](%3), %%rdi \n\t"
2294                 "mov %c[rbp](%3), %%rbp \n\t"
2295                 "mov %c[r8](%3),  %%r8  \n\t"
2296                 "mov %c[r9](%3),  %%r9  \n\t"
2297                 "mov %c[r10](%3), %%r10 \n\t"
2298                 "mov %c[r11](%3), %%r11 \n\t"
2299                 "mov %c[r12](%3), %%r12 \n\t"
2300                 "mov %c[r13](%3), %%r13 \n\t"
2301                 "mov %c[r14](%3), %%r14 \n\t"
2302                 "mov %c[r15](%3), %%r15 \n\t"
2303                 "mov %c[rcx](%3), %%rcx \n\t" /* kills %3 (rcx) */
2304 #else
2305                 "mov %c[cr2](%3), %%eax \n\t"
2306                 "mov %%eax,   %%cr2 \n\t"
2307                 "mov %c[rax](%3), %%eax \n\t"
2308                 "mov %c[rbx](%3), %%ebx \n\t"
2309                 "mov %c[rdx](%3), %%edx \n\t"
2310                 "mov %c[rsi](%3), %%esi \n\t"
2311                 "mov %c[rdi](%3), %%edi \n\t"
2312                 "mov %c[rbp](%3), %%ebp \n\t"
2313                 "mov %c[rcx](%3), %%ecx \n\t" /* kills %3 (ecx) */
2314 #endif
2315                 /* Enter guest mode */
2316                 "jne .Llaunched \n\t"
2317                 ASM_VMX_VMLAUNCH "\n\t"
2318                 "jmp .Lkvm_vmx_return \n\t"
2319                 ".Llaunched: " ASM_VMX_VMRESUME "\n\t"
2320                 ".Lkvm_vmx_return: "
2321                 /* Save guest registers, load host registers, keep flags */
2322 #ifdef CONFIG_X86_64
2323                 "xchg %3,     (%%rsp) \n\t"
2324                 "mov %%rax, %c[rax](%3) \n\t"
2325                 "mov %%rbx, %c[rbx](%3) \n\t"
2326                 "pushq (%%rsp); popq %c[rcx](%3) \n\t"
2327                 "mov %%rdx, %c[rdx](%3) \n\t"
2328                 "mov %%rsi, %c[rsi](%3) \n\t"
2329                 "mov %%rdi, %c[rdi](%3) \n\t"
2330                 "mov %%rbp, %c[rbp](%3) \n\t"
2331                 "mov %%r8,  %c[r8](%3) \n\t"
2332                 "mov %%r9,  %c[r9](%3) \n\t"
2333                 "mov %%r10, %c[r10](%3) \n\t"
2334                 "mov %%r11, %c[r11](%3) \n\t"
2335                 "mov %%r12, %c[r12](%3) \n\t"
2336                 "mov %%r13, %c[r13](%3) \n\t"
2337                 "mov %%r14, %c[r14](%3) \n\t"
2338                 "mov %%r15, %c[r15](%3) \n\t"
2339                 "mov %%cr2, %%rax   \n\t"
2340                 "mov %%rax, %c[cr2](%3) \n\t"
2341                 "mov (%%rsp), %3 \n\t"
2342
2343                 "pop  %%rcx; pop  %%r15; pop  %%r14; pop  %%r13; pop  %%r12;"
2344                 "pop  %%r11; pop  %%r10; pop  %%r9;  pop  %%r8;"
2345                 "pop  %%rbp; pop  %%rdi; pop  %%rsi;"
2346                 "pop  %%rdx; pop  %%rbx; pop  %%rax \n\t"
2347 #else
2348                 "xchg %3, (%%esp) \n\t"
2349                 "mov %%eax, %c[rax](%3) \n\t"
2350                 "mov %%ebx, %c[rbx](%3) \n\t"
2351                 "pushl (%%esp); popl %c[rcx](%3) \n\t"
2352                 "mov %%edx, %c[rdx](%3) \n\t"
2353                 "mov %%esi, %c[rsi](%3) \n\t"
2354                 "mov %%edi, %c[rdi](%3) \n\t"
2355                 "mov %%ebp, %c[rbp](%3) \n\t"
2356                 "mov %%cr2, %%eax  \n\t"
2357                 "mov %%eax, %c[cr2](%3) \n\t"
2358                 "mov (%%esp), %3 \n\t"
2359
2360                 "pop %%ecx; popa \n\t"
2361 #endif
2362                 "setbe %0 \n\t"
2363               : "=q" (vmx->fail)
2364               : "r"(vmx->launched), "d"((unsigned long)HOST_RSP),
2365                 "c"(vcpu),
2366                 [rax]"i"(offsetof(struct kvm_vcpu, regs[VCPU_REGS_RAX])),
2367                 [rbx]"i"(offsetof(struct kvm_vcpu, regs[VCPU_REGS_RBX])),
2368                 [rcx]"i"(offsetof(struct kvm_vcpu, regs[VCPU_REGS_RCX])),
2369                 [rdx]"i"(offsetof(struct kvm_vcpu, regs[VCPU_REGS_RDX])),
2370                 [rsi]"i"(offsetof(struct kvm_vcpu, regs[VCPU_REGS_RSI])),
2371                 [rdi]"i"(offsetof(struct kvm_vcpu, regs[VCPU_REGS_RDI])),
2372                 [rbp]"i"(offsetof(struct kvm_vcpu, regs[VCPU_REGS_RBP])),
2373 #ifdef CONFIG_X86_64
2374                 [r8 ]"i"(offsetof(struct kvm_vcpu, regs[VCPU_REGS_R8 ])),
2375                 [r9 ]"i"(offsetof(struct kvm_vcpu, regs[VCPU_REGS_R9 ])),
2376                 [r10]"i"(offsetof(struct kvm_vcpu, regs[VCPU_REGS_R10])),
2377                 [r11]"i"(offsetof(struct kvm_vcpu, regs[VCPU_REGS_R11])),
2378                 [r12]"i"(offsetof(struct kvm_vcpu, regs[VCPU_REGS_R12])),
2379                 [r13]"i"(offsetof(struct kvm_vcpu, regs[VCPU_REGS_R13])),
2380                 [r14]"i"(offsetof(struct kvm_vcpu, regs[VCPU_REGS_R14])),
2381                 [r15]"i"(offsetof(struct kvm_vcpu, regs[VCPU_REGS_R15])),
2382 #endif
2383                 [cr2]"i"(offsetof(struct kvm_vcpu, cr2))
2384               : "cc", "memory" );
2385
2386         vcpu->guest_mode = 0;
2387         local_irq_enable();
2388
2389         ++vcpu->stat.exits;
2390
2391         vcpu->interrupt_window_open = (vmcs_read32(GUEST_INTERRUPTIBILITY_INFO) & 3) == 0;
2392
2393         asm ("mov %0, %%ds; mov %0, %%es" : : "r"(__USER_DS));
2394         vmx->launched = 1;
2395
2396         preempt_enable();
2397
2398         /*
2399          * Profile KVM exit RIPs:
2400          */
2401         if (unlikely(prof_on == KVM_PROFILING))
2402                 profile_hit(KVM_PROFILING, (void *)vmcs_readl(GUEST_RIP));
2403
2404         r = kvm_handle_exit(kvm_run, vcpu);
2405         if (r > 0) {
2406                 if (dm_request_for_irq_injection(vcpu, kvm_run)) {
2407                         r = -EINTR;
2408                         kvm_run->exit_reason = KVM_EXIT_INTR;
2409                         ++vcpu->stat.request_irq_exits;
2410                         goto out;
2411                 }
2412                 if (!need_resched()) {
2413                         ++vcpu->stat.light_exits;
2414                         goto again;
2415                 }
2416         }
2417
2418 out:
2419         if (r > 0) {
2420                 kvm_resched(vcpu);
2421                 goto preempted;
2422         }
2423
2424         post_kvm_run_save(vcpu, kvm_run);
2425         return r;
2426 }
2427
2428 static void vmx_inject_page_fault(struct kvm_vcpu *vcpu,
2429                                   unsigned long addr,
2430                                   u32 err_code)
2431 {
2432         u32 vect_info = vmcs_read32(IDT_VECTORING_INFO_FIELD);
2433
2434         ++vcpu->stat.pf_guest;
2435
2436         if (is_page_fault(vect_info)) {
2437                 printk(KERN_DEBUG "inject_page_fault: "
2438                        "double fault 0x%lx @ 0x%lx\n",
2439                        addr, vmcs_readl(GUEST_RIP));
2440                 vmcs_write32(VM_ENTRY_EXCEPTION_ERROR_CODE, 0);
2441                 vmcs_write32(VM_ENTRY_INTR_INFO_FIELD,
2442                              DF_VECTOR |
2443                              INTR_TYPE_EXCEPTION |
2444                              INTR_INFO_DELIEVER_CODE_MASK |
2445                              INTR_INFO_VALID_MASK);
2446                 return;
2447         }
2448         vcpu->cr2 = addr;
2449         vmcs_write32(VM_ENTRY_EXCEPTION_ERROR_CODE, err_code);
2450         vmcs_write32(VM_ENTRY_INTR_INFO_FIELD,
2451                      PF_VECTOR |
2452                      INTR_TYPE_EXCEPTION |
2453                      INTR_INFO_DELIEVER_CODE_MASK |
2454                      INTR_INFO_VALID_MASK);
2455
2456 }
2457
2458 static void vmx_free_vmcs(struct kvm_vcpu *vcpu)
2459 {
2460         struct vcpu_vmx *vmx = to_vmx(vcpu);
2461
2462         if (vmx->vmcs) {
2463                 on_each_cpu(__vcpu_clear, vmx, 0, 1);
2464                 free_vmcs(vmx->vmcs);
2465                 vmx->vmcs = NULL;
2466         }
2467 }
2468
2469 static void vmx_free_vcpu(struct kvm_vcpu *vcpu)
2470 {
2471         struct vcpu_vmx *vmx = to_vmx(vcpu);
2472
2473         vmx_free_vmcs(vcpu);
2474         kfree(vmx->host_msrs);
2475         kfree(vmx->guest_msrs);
2476         kvm_vcpu_uninit(vcpu);
2477         kmem_cache_free(kvm_vcpu_cache, vmx);
2478 }
2479
2480 static struct kvm_vcpu *vmx_create_vcpu(struct kvm *kvm, unsigned int id)
2481 {
2482         int err;
2483         struct vcpu_vmx *vmx = kmem_cache_zalloc(kvm_vcpu_cache, GFP_KERNEL);
2484         int cpu;
2485
2486         if (!vmx)
2487                 return ERR_PTR(-ENOMEM);
2488
2489         err = kvm_vcpu_init(&vmx->vcpu, kvm, id);
2490         if (err)
2491                 goto free_vcpu;
2492
2493         if (irqchip_in_kernel(kvm)) {
2494                 err = kvm_create_lapic(&vmx->vcpu);
2495                 if (err < 0)
2496                         goto free_vcpu;
2497         }
2498
2499         vmx->guest_msrs = kmalloc(PAGE_SIZE, GFP_KERNEL);
2500         if (!vmx->guest_msrs) {
2501                 err = -ENOMEM;
2502                 goto uninit_vcpu;
2503         }
2504
2505         vmx->host_msrs = kmalloc(PAGE_SIZE, GFP_KERNEL);
2506         if (!vmx->host_msrs)
2507                 goto free_guest_msrs;
2508
2509         vmx->vmcs = alloc_vmcs();
2510         if (!vmx->vmcs)
2511                 goto free_msrs;
2512
2513         vmcs_clear(vmx->vmcs);
2514
2515         cpu = get_cpu();
2516         vmx_vcpu_load(&vmx->vcpu, cpu);
2517         err = vmx_vcpu_setup(vmx);
2518         vmx_vcpu_put(&vmx->vcpu);
2519         put_cpu();
2520         if (err)
2521                 goto free_vmcs;
2522
2523         return &vmx->vcpu;
2524
2525 free_vmcs:
2526         free_vmcs(vmx->vmcs);
2527 free_msrs:
2528         kfree(vmx->host_msrs);
2529 free_guest_msrs:
2530         kfree(vmx->guest_msrs);
2531 uninit_vcpu:
2532         kvm_vcpu_uninit(&vmx->vcpu);
2533 free_vcpu:
2534         kmem_cache_free(kvm_vcpu_cache, vmx);
2535         return ERR_PTR(err);
2536 }
2537
2538 static void __init vmx_check_processor_compat(void *rtn)
2539 {
2540         struct vmcs_config vmcs_conf;
2541
2542         *(int *)rtn = 0;
2543         if (setup_vmcs_config(&vmcs_conf) < 0)
2544                 *(int *)rtn = -EIO;
2545         if (memcmp(&vmcs_config, &vmcs_conf, sizeof(struct vmcs_config)) != 0) {
2546                 printk(KERN_ERR "kvm: CPU %d feature inconsistency!\n",
2547                                 smp_processor_id());
2548                 *(int *)rtn = -EIO;
2549         }
2550 }
2551
2552 static struct kvm_x86_ops vmx_x86_ops = {
2553         .cpu_has_kvm_support = cpu_has_kvm_support,
2554         .disabled_by_bios = vmx_disabled_by_bios,
2555         .hardware_setup = hardware_setup,
2556         .hardware_unsetup = hardware_unsetup,
2557         .check_processor_compatibility = vmx_check_processor_compat,
2558         .hardware_enable = hardware_enable,
2559         .hardware_disable = hardware_disable,
2560
2561         .vcpu_create = vmx_create_vcpu,
2562         .vcpu_free = vmx_free_vcpu,
2563
2564         .vcpu_load = vmx_vcpu_load,
2565         .vcpu_put = vmx_vcpu_put,
2566         .vcpu_decache = vmx_vcpu_decache,
2567
2568         .set_guest_debug = set_guest_debug,
2569         .get_msr = vmx_get_msr,
2570         .set_msr = vmx_set_msr,
2571         .get_segment_base = vmx_get_segment_base,
2572         .get_segment = vmx_get_segment,
2573         .set_segment = vmx_set_segment,
2574         .get_cs_db_l_bits = vmx_get_cs_db_l_bits,
2575         .decache_cr4_guest_bits = vmx_decache_cr4_guest_bits,
2576         .set_cr0 = vmx_set_cr0,
2577         .set_cr3 = vmx_set_cr3,
2578         .set_cr4 = vmx_set_cr4,
2579 #ifdef CONFIG_X86_64
2580         .set_efer = vmx_set_efer,
2581 #endif
2582         .get_idt = vmx_get_idt,
2583         .set_idt = vmx_set_idt,
2584         .get_gdt = vmx_get_gdt,
2585         .set_gdt = vmx_set_gdt,
2586         .cache_regs = vcpu_load_rsp_rip,
2587         .decache_regs = vcpu_put_rsp_rip,
2588         .get_rflags = vmx_get_rflags,
2589         .set_rflags = vmx_set_rflags,
2590
2591         .tlb_flush = vmx_flush_tlb,
2592         .inject_page_fault = vmx_inject_page_fault,
2593
2594         .inject_gp = vmx_inject_gp,
2595
2596         .run = vmx_vcpu_run,
2597         .skip_emulated_instruction = skip_emulated_instruction,
2598         .patch_hypercall = vmx_patch_hypercall,
2599         .get_irq = vmx_get_irq,
2600         .set_irq = vmx_inject_irq,
2601 };
2602
2603 static int __init vmx_init(void)
2604 {
2605         void *iova;
2606         int r;
2607
2608         vmx_io_bitmap_a = alloc_page(GFP_KERNEL | __GFP_HIGHMEM);
2609         if (!vmx_io_bitmap_a)
2610                 return -ENOMEM;
2611
2612         vmx_io_bitmap_b = alloc_page(GFP_KERNEL | __GFP_HIGHMEM);
2613         if (!vmx_io_bitmap_b) {
2614                 r = -ENOMEM;
2615                 goto out;
2616         }
2617
2618         /*
2619          * Allow direct access to the PC debug port (it is often used for I/O
2620          * delays, but the vmexits simply slow things down).
2621          */
2622         iova = kmap(vmx_io_bitmap_a);
2623         memset(iova, 0xff, PAGE_SIZE);
2624         clear_bit(0x80, iova);
2625         kunmap(vmx_io_bitmap_a);
2626
2627         iova = kmap(vmx_io_bitmap_b);
2628         memset(iova, 0xff, PAGE_SIZE);
2629         kunmap(vmx_io_bitmap_b);
2630
2631         r = kvm_init_x86(&vmx_x86_ops, sizeof(struct vcpu_vmx), THIS_MODULE);
2632         if (r)
2633                 goto out1;
2634
2635         return 0;
2636
2637 out1:
2638         __free_page(vmx_io_bitmap_b);
2639 out:
2640         __free_page(vmx_io_bitmap_a);
2641         return r;
2642 }
2643
2644 static void __exit vmx_exit(void)
2645 {
2646         __free_page(vmx_io_bitmap_b);
2647         __free_page(vmx_io_bitmap_a);
2648
2649         kvm_exit_x86();
2650 }
2651
2652 module_init(vmx_init)
2653 module_exit(vmx_exit)