drivers/kvm/kvm_main.c

   1 /*
   2  * Kernel-based Virtual Machine driver for Linux
   3  *
   4  * This module enables machines with Intel VT-x extensions to run virtual
   5  * machines without emulation or binary translation.
   6  *
   7  * Copyright (C) 2006 Qumranet, Inc.
   8  *
   9  * Authors:
  10  *   Avi Kivity   <avi@qumranet.com>
  11  *   Yaniv Kamay  <yaniv@qumranet.com>
  12  *
  13  * This work is licensed under the terms of the GNU GPL, version 2.  See
  14  * the COPYING file in the top-level directory.
  15  *
  16  */
  17
  18 #include "kvm.h"
  19
  20 #include <linux/kvm.h>
  21 #include <linux/module.h>
  22 #include <linux/errno.h>
  23 #include <asm/processor.h>
  24 #include <linux/percpu.h>
  25 #include <linux/gfp.h>
  26 #include <asm/msr.h>
  27 #include <linux/mm.h>
  28 #include <linux/miscdevice.h>
  29 #include <linux/vmalloc.h>
  30 #include <asm/uaccess.h>
  31 #include <linux/reboot.h>
  32 #include <asm/io.h>
  33 #include <linux/debugfs.h>
  34 #include <linux/highmem.h>
  35 #include <linux/file.h>
  36 #include <asm/desc.h>
  37 #include <linux/sysdev.h>
  38 #include <linux/cpu.h>
  39 #include <linux/file.h>
  40 #include <linux/fs.h>
  41 #include <linux/mount.h>
  42
  43 #include "x86_emulate.h"
  44 #include "segment_descriptor.h"
  45
  46 MODULE_AUTHOR("Qumranet");
  47 MODULE_LICENSE("GPL");
  48
  49 static DEFINE_SPINLOCK(kvm_lock);
  50 static LIST_HEAD(vm_list);
  51
  52 struct kvm_arch_ops *kvm_arch_ops;
  53 struct kvm_stat kvm_stat;
  54 EXPORT_SYMBOL_GPL(kvm_stat);
  55
  56 static struct kvm_stats_debugfs_item {
  57         const char *name;
  58         u32 *data;
  59         struct dentry *dentry;
  60 } debugfs_entries[] = {
  61         { "pf_fixed", &kvm_stat.pf_fixed },
  62         { "pf_guest", &kvm_stat.pf_guest },
  63         { "tlb_flush", &kvm_stat.tlb_flush },
  64         { "invlpg", &kvm_stat.invlpg },
  65         { "exits", &kvm_stat.exits },
  66         { "io_exits", &kvm_stat.io_exits },
  67         { "mmio_exits", &kvm_stat.mmio_exits },
  68         { "signal_exits", &kvm_stat.signal_exits },
  69         { "irq_window", &kvm_stat.irq_window_exits },
  70         { "halt_exits", &kvm_stat.halt_exits },
  71         { "request_irq", &kvm_stat.request_irq_exits },
  72         { "irq_exits", &kvm_stat.irq_exits },
  73         { NULL, NULL }
  74 };
  75
  76 static struct dentry *debugfs_dir;
  77
  78 #define KVMFS_MAGIC 0x19700426
  79 struct vfsmount *kvmfs_mnt;
  80
  81 #define MAX_IO_MSRS 256
  82
  83 #define CR0_RESEVED_BITS 0xffffffff1ffaffc0ULL
  84 #define LMSW_GUEST_MASK 0x0eULL
  85 #define CR4_RESEVED_BITS (~((1ULL << 11) - 1))
  86 #define CR8_RESEVED_BITS (~0x0fULL)
  87 #define EFER_RESERVED_BITS 0xfffffffffffff2fe
  88
  89 #ifdef CONFIG_X86_64
  90 // LDT or TSS descriptor in the GDT. 16 bytes.
  91 struct segment_descriptor_64 {
  92         struct segment_descriptor s;
  93         u32 base_higher;
  94         u32 pad_zero;
  95 };
  96
  97 #endif
  98
  99 static struct inode *kvmfs_inode(struct file_operations *fops)
 100 {
 101         int error = -ENOMEM;
 102         struct inode *inode = new_inode(kvmfs_mnt->mnt_sb);
 103
 104         if (!inode)
 105                 goto eexit_1;
 106
 107         inode->i_fop = fops;
 108
 109         /*
 110          * Mark the inode dirty from the very beginning,
 111          * that way it will never be moved to the dirty
 112          * list because mark_inode_dirty() will think
 113          * that it already _is_ on the dirty list.
 114          */
 115         inode->i_state = I_DIRTY;
 116         inode->i_mode = S_IRUSR | S_IWUSR;
 117         inode->i_uid = current->fsuid;
 118         inode->i_gid = current->fsgid;
 119         inode->i_atime = inode->i_mtime = inode->i_ctime = CURRENT_TIME;
 120         return inode;
 121
 122 eexit_1:
 123         return ERR_PTR(error);
 124 }
 125
 126 static struct file *kvmfs_file(struct inode *inode, void *private_data)
 127 {
 128         struct file *file = get_empty_filp();
 129
 130         if (!file)
 131                 return ERR_PTR(-ENFILE);
 132
 133         file->f_path.mnt = mntget(kvmfs_mnt);
 134         file->f_path.dentry = d_alloc_anon(inode);
 135         if (!file->f_path.dentry)
 136                 return ERR_PTR(-ENOMEM);
 137         file->f_mapping = inode->i_mapping;
 138
 139         file->f_pos = 0;
 140         file->f_flags = O_RDWR;
 141         file->f_op = inode->i_fop;
 142         file->f_mode = FMODE_READ | FMODE_WRITE;
 143         file->f_version = 0;
 144         file->private_data = private_data;
 145         return file;
 146 }
 147
 148 unsigned long segment_base(u16 selector)
 149 {
 150         struct descriptor_table gdt;
 151         struct segment_descriptor *d;
 152         unsigned long table_base;
 153         typedef unsigned long ul;
 154         unsigned long v;
 155
 156         if (selector == 0)
 157                 return 0;
 158
 159         asm ("sgdt %0" : "=m"(gdt));
 160         table_base = gdt.base;
 161
 162         if (selector & 4) {           /* from ldt */
 163                 u16 ldt_selector;
 164
 165                 asm ("sldt %0" : "=g"(ldt_selector));
 166                 table_base = segment_base(ldt_selector);
 167         }
 168         d = (struct segment_descriptor *)(table_base + (selector & ~7));
 169         v = d->base_low | ((ul)d->base_mid << 16) | ((ul)d->base_high << 24);
 170 #ifdef CONFIG_X86_64
 171         if (d->system == 0
 172             && (d->type == 2 || d->type == 9 || d->type == 11))
 173                 v |= ((ul)((struct segment_descriptor_64 *)d)->base_higher) << 32;
 174 #endif
 175         return v;
 176 }
 177 EXPORT_SYMBOL_GPL(segment_base);
 178
 179 static inline int valid_vcpu(int n)
 180 {
 181         return likely(n >= 0 && n < KVM_MAX_VCPUS);
 182 }
 183
 184 int kvm_read_guest(struct kvm_vcpu *vcpu, gva_t addr, unsigned long size,
 185                    void *dest)
 186 {
 187         unsigned char *host_buf = dest;
 188         unsigned long req_size = size;
 189
 190         while (size) {
 191                 hpa_t paddr;
 192                 unsigned now;
 193                 unsigned offset;
 194                 hva_t guest_buf;
 195
 196                 paddr = gva_to_hpa(vcpu, addr);
 197
 198                 if (is_error_hpa(paddr))
 199                         break;
 200
 201                 guest_buf = (hva_t)kmap_atomic(
 202                                         pfn_to_page(paddr >> PAGE_SHIFT),
 203                                         KM_USER0);
 204                 offset = addr & ~PAGE_MASK;
 205                 guest_buf |= offset;
 206                 now = min(size, PAGE_SIZE - offset);
 207                 memcpy(host_buf, (void*)guest_buf, now);
 208                 host_buf += now;
 209                 addr += now;
 210                 size -= now;
 211                 kunmap_atomic((void *)(guest_buf & PAGE_MASK), KM_USER0);
 212         }
 213         return req_size - size;
 214 }
 215 EXPORT_SYMBOL_GPL(kvm_read_guest);
 216
 217 int kvm_write_guest(struct kvm_vcpu *vcpu, gva_t addr, unsigned long size,
 218                     void *data)
 219 {
 220         unsigned char *host_buf = data;
 221         unsigned long req_size = size;
 222
 223         while (size) {
 224                 hpa_t paddr;
 225                 unsigned now;
 226                 unsigned offset;
 227                 hva_t guest_buf;
 228
 229                 paddr = gva_to_hpa(vcpu, addr);
 230
 231                 if (is_error_hpa(paddr))
 232                         break;
 233
 234                 guest_buf = (hva_t)kmap_atomic(
 235                                 pfn_to_page(paddr >> PAGE_SHIFT), KM_USER0);
 236                 offset = addr & ~PAGE_MASK;
 237                 guest_buf |= offset;
 238                 now = min(size, PAGE_SIZE - offset);
 239                 memcpy((void*)guest_buf, host_buf, now);
 240                 host_buf += now;
 241                 addr += now;
 242                 size -= now;
 243                 kunmap_atomic((void *)(guest_buf & PAGE_MASK), KM_USER0);
 244         }
 245         return req_size - size;
 246 }
 247 EXPORT_SYMBOL_GPL(kvm_write_guest);
 248
 249 static int vcpu_slot(struct kvm_vcpu *vcpu)
 250 {
 251         return vcpu - vcpu->kvm->vcpus;
 252 }
 253
 254 /*
 255  * Switches to specified vcpu, until a matching vcpu_put()
 256  */
 257 static struct kvm_vcpu *vcpu_load(struct kvm *kvm, int vcpu_slot)
 258 {
 259         struct kvm_vcpu *vcpu = &kvm->vcpus[vcpu_slot];
 260
 261         mutex_lock(&vcpu->mutex);
 262         if (unlikely(!vcpu->vmcs)) {
 263                 mutex_unlock(&vcpu->mutex);
 264                 return NULL;
 265         }
 266         return kvm_arch_ops->vcpu_load(vcpu);
 267 }
 268
 269 static void vcpu_put(struct kvm_vcpu *vcpu)
 270 {
 271         kvm_arch_ops->vcpu_put(vcpu);
 272         mutex_unlock(&vcpu->mutex);
 273 }
 274
 275 static struct kvm *kvm_create_vm(void)
 276 {
 277         struct kvm *kvm = kzalloc(sizeof(struct kvm), GFP_KERNEL);
 278         int i;
 279
 280         if (!kvm)
 281                 return ERR_PTR(-ENOMEM);
 282
 283         spin_lock_init(&kvm->lock);
 284         INIT_LIST_HEAD(&kvm->active_mmu_pages);
 285         for (i = 0; i < KVM_MAX_VCPUS; ++i) {
 286                 struct kvm_vcpu *vcpu = &kvm->vcpus[i];
 287
 288                 mutex_init(&vcpu->mutex);
 289                 vcpu->cpu = -1;
 290                 vcpu->kvm = kvm;
 291                 vcpu->mmu.root_hpa = INVALID_PAGE;
 292                 INIT_LIST_HEAD(&vcpu->free_pages);
 293                 spin_lock(&kvm_lock);
 294                 list_add(&kvm->vm_list, &vm_list);
 295                 spin_unlock(&kvm_lock);
 296         }
 297         return kvm;
 298 }
 299
 300 static int kvm_dev_open(struct inode *inode, struct file *filp)
 301 {
 302         return 0;
 303 }
 304
 305 /*
 306  * Free any memory in @free but not in @dont.
 307  */
 308 static void kvm_free_physmem_slot(struct kvm_memory_slot *free,
 309                                   struct kvm_memory_slot *dont)
 310 {
 311         int i;
 312
 313         if (!dont || free->phys_mem != dont->phys_mem)
 314                 if (free->phys_mem) {
 315                         for (i = 0; i < free->npages; ++i)
 316                                 if (free->phys_mem[i])
 317                                         __free_page(free->phys_mem[i]);
 318                         vfree(free->phys_mem);
 319                 }
 320
 321         if (!dont || free->dirty_bitmap != dont->dirty_bitmap)
 322                 vfree(free->dirty_bitmap);
 323
 324         free->phys_mem = NULL;
 325         free->npages = 0;
 326         free->dirty_bitmap = NULL;
 327 }
 328
 329 static void kvm_free_physmem(struct kvm *kvm)
 330 {
 331         int i;
 332
 333         for (i = 0; i < kvm->nmemslots; ++i)
 334                 kvm_free_physmem_slot(&kvm->memslots[i], NULL);
 335 }
 336
 337 static void kvm_free_vcpu(struct kvm_vcpu *vcpu)
 338 {
 339         if (!vcpu_load(vcpu->kvm, vcpu_slot(vcpu)))
 340                 return;
 341
 342         kvm_mmu_destroy(vcpu);
 343         vcpu_put(vcpu);
 344         kvm_arch_ops->vcpu_free(vcpu);
 345 }
 346
 347 static void kvm_free_vcpus(struct kvm *kvm)
 348 {
 349         unsigned int i;
 350
 351         for (i = 0; i < KVM_MAX_VCPUS; ++i)
 352                 kvm_free_vcpu(&kvm->vcpus[i]);
 353 }
 354
 355 static int kvm_dev_release(struct inode *inode, struct file *filp)
 356 {
 357         return 0;
 358 }
 359
 360 static void kvm_destroy_vm(struct kvm *kvm)
 361 {
 362         spin_lock(&kvm_lock);
 363         list_del(&kvm->vm_list);
 364         spin_unlock(&kvm_lock);
 365         kvm_free_vcpus(kvm);
 366         kvm_free_physmem(kvm);
 367         kfree(kvm);
 368 }
 369
 370 static int kvm_vm_release(struct inode *inode, struct file *filp)
 371 {
 372         struct kvm *kvm = filp->private_data;
 373
 374         kvm_destroy_vm(kvm);
 375         return 0;
 376 }
 377
 378 static void inject_gp(struct kvm_vcpu *vcpu)
 379 {
 380         kvm_arch_ops->inject_gp(vcpu, 0);
 381 }
 382
 383 /*
 384  * Load the pae pdptrs.  Return true is they are all valid.
 385  */
 386 static int load_pdptrs(struct kvm_vcpu *vcpu, unsigned long cr3)
 387 {
 388         gfn_t pdpt_gfn = cr3 >> PAGE_SHIFT;
 389         unsigned offset = ((cr3 & (PAGE_SIZE-1)) >> 5) << 2;
 390         int i;
 391         u64 pdpte;
 392         u64 *pdpt;
 393         int ret;
 394         struct kvm_memory_slot *memslot;
 395
 396         spin_lock(&vcpu->kvm->lock);
 397         memslot = gfn_to_memslot(vcpu->kvm, pdpt_gfn);
 398         /* FIXME: !memslot - emulate? 0xff? */
 399         pdpt = kmap_atomic(gfn_to_page(memslot, pdpt_gfn), KM_USER0);
 400
 401         ret = 1;
 402         for (i = 0; i < 4; ++i) {
 403                 pdpte = pdpt[offset + i];
 404                 if ((pdpte & 1) && (pdpte & 0xfffffff0000001e6ull)) {
 405                         ret = 0;
 406                         goto out;
 407                 }
 408         }
 409
 410         for (i = 0; i < 4; ++i)
 411                 vcpu->pdptrs[i] = pdpt[offset + i];
 412
 413 out:
 414         kunmap_atomic(pdpt, KM_USER0);
 415         spin_unlock(&vcpu->kvm->lock);
 416
 417         return ret;
 418 }
 419
 420 void set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0)
 421 {
 422         if (cr0 & CR0_RESEVED_BITS) {
 423                 printk(KERN_DEBUG "set_cr0: 0x%lx #GP, reserved bits 0x%lx\n",
 424                        cr0, vcpu->cr0);
 425                 inject_gp(vcpu);
 426                 return;
 427         }
 428
 429         if ((cr0 & CR0_NW_MASK) && !(cr0 & CR0_CD_MASK)) {
 430                 printk(KERN_DEBUG "set_cr0: #GP, CD == 0 && NW == 1\n");
 431                 inject_gp(vcpu);
 432                 return;
 433         }
 434
 435         if ((cr0 & CR0_PG_MASK) && !(cr0 & CR0_PE_MASK)) {
 436                 printk(KERN_DEBUG "set_cr0: #GP, set PG flag "
 437                        "and a clear PE flag\n");
 438                 inject_gp(vcpu);
 439                 return;
 440         }
 441
 442         if (!is_paging(vcpu) && (cr0 & CR0_PG_MASK)) {
 443 #ifdef CONFIG_X86_64
 444                 if ((vcpu->shadow_efer & EFER_LME)) {
 445                         int cs_db, cs_l;
 446
 447                         if (!is_pae(vcpu)) {
 448                                 printk(KERN_DEBUG "set_cr0: #GP, start paging "
 449                                        "in long mode while PAE is disabled\n");
 450                                 inject_gp(vcpu);
 451                                 return;
 452                         }
 453                         kvm_arch_ops->get_cs_db_l_bits(vcpu, &cs_db, &cs_l);
 454                         if (cs_l) {
 455                                 printk(KERN_DEBUG "set_cr0: #GP, start paging "
 456                                        "in long mode while CS.L == 1\n");
 457                                 inject_gp(vcpu);
 458                                 return;
 459
 460                         }
 461                 } else
 462 #endif
 463                 if (is_pae(vcpu) && !load_pdptrs(vcpu, vcpu->cr3)) {
 464                         printk(KERN_DEBUG "set_cr0: #GP, pdptrs "
 465                                "reserved bits\n");
 466                         inject_gp(vcpu);
 467                         return;
 468                 }
 469
 470         }
 471
 472         kvm_arch_ops->set_cr0(vcpu, cr0);
 473         vcpu->cr0 = cr0;
 474
 475         spin_lock(&vcpu->kvm->lock);
 476         kvm_mmu_reset_context(vcpu);
 477         spin_unlock(&vcpu->kvm->lock);
 478         return;
 479 }
 480 EXPORT_SYMBOL_GPL(set_cr0);
 481
 482 void lmsw(struct kvm_vcpu *vcpu, unsigned long msw)
 483 {
 484         kvm_arch_ops->decache_cr0_cr4_guest_bits(vcpu);
 485         set_cr0(vcpu, (vcpu->cr0 & ~0x0ful) | (msw & 0x0f));
 486 }
 487 EXPORT_SYMBOL_GPL(lmsw);
 488
 489 void set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4)
 490 {
 491         if (cr4 & CR4_RESEVED_BITS) {
 492                 printk(KERN_DEBUG "set_cr4: #GP, reserved bits\n");
 493                 inject_gp(vcpu);
 494                 return;
 495         }
 496
 497         if (is_long_mode(vcpu)) {
 498                 if (!(cr4 & CR4_PAE_MASK)) {
 499                         printk(KERN_DEBUG "set_cr4: #GP, clearing PAE while "
 500                                "in long mode\n");
 501                         inject_gp(vcpu);
 502                         return;
 503                 }
 504         } else if (is_paging(vcpu) && !is_pae(vcpu) && (cr4 & CR4_PAE_MASK)
 505                    && !load_pdptrs(vcpu, vcpu->cr3)) {
 506                 printk(KERN_DEBUG "set_cr4: #GP, pdptrs reserved bits\n");
 507                 inject_gp(vcpu);
 508         }
 509
 510         if (cr4 & CR4_VMXE_MASK) {
 511                 printk(KERN_DEBUG "set_cr4: #GP, setting VMXE\n");
 512                 inject_gp(vcpu);
 513                 return;
 514         }
 515         kvm_arch_ops->set_cr4(vcpu, cr4);
 516         spin_lock(&vcpu->kvm->lock);
 517         kvm_mmu_reset_context(vcpu);
 518         spin_unlock(&vcpu->kvm->lock);
 519 }
 520 EXPORT_SYMBOL_GPL(set_cr4);
 521
 522 void set_cr3(struct kvm_vcpu *vcpu, unsigned long cr3)
 523 {
 524         if (is_long_mode(vcpu)) {
 525                 if (cr3 & CR3_L_MODE_RESEVED_BITS) {
 526                         printk(KERN_DEBUG "set_cr3: #GP, reserved bits\n");
 527                         inject_gp(vcpu);
 528                         return;
 529                 }
 530         } else {
 531                 if (cr3 & CR3_RESEVED_BITS) {
 532                         printk(KERN_DEBUG "set_cr3: #GP, reserved bits\n");
 533                         inject_gp(vcpu);
 534                         return;
 535                 }
 536                 if (is_paging(vcpu) && is_pae(vcpu) &&
 537                     !load_pdptrs(vcpu, cr3)) {
 538                         printk(KERN_DEBUG "set_cr3: #GP, pdptrs "
 539                                "reserved bits\n");
 540                         inject_gp(vcpu);
 541                         return;
 542                 }
 543         }
 544
 545         vcpu->cr3 = cr3;
 546         spin_lock(&vcpu->kvm->lock);
 547         /*
 548          * Does the new cr3 value map to physical memory? (Note, we
 549          * catch an invalid cr3 even in real-mode, because it would
 550          * cause trouble later on when we turn on paging anyway.)
 551          *
 552          * A real CPU would silently accept an invalid cr3 and would
 553          * attempt to use it - with largely undefined (and often hard
 554          * to debug) behavior on the guest side.
 555          */
 556         if (unlikely(!gfn_to_memslot(vcpu->kvm, cr3 >> PAGE_SHIFT)))
 557                 inject_gp(vcpu);
 558         else
 559                 vcpu->mmu.new_cr3(vcpu);
 560         spin_unlock(&vcpu->kvm->lock);
 561 }
 562 EXPORT_SYMBOL_GPL(set_cr3);
 563
 564 void set_cr8(struct kvm_vcpu *vcpu, unsigned long cr8)
 565 {
 566         if ( cr8 & CR8_RESEVED_BITS) {
 567                 printk(KERN_DEBUG "set_cr8: #GP, reserved bits 0x%lx\n", cr8);
 568                 inject_gp(vcpu);
 569                 return;
 570         }
 571         vcpu->cr8 = cr8;
 572 }
 573 EXPORT_SYMBOL_GPL(set_cr8);
 574
 575 void fx_init(struct kvm_vcpu *vcpu)
 576 {
 577         struct __attribute__ ((__packed__)) fx_image_s {
 578                 u16 control; //fcw
 579                 u16 status; //fsw
 580                 u16 tag; // ftw
 581                 u16 opcode; //fop
 582                 u64 ip; // fpu ip
 583                 u64 operand;// fpu dp
 584                 u32 mxcsr;
 585                 u32 mxcsr_mask;
 586
 587         } *fx_image;
 588
 589         fx_save(vcpu->host_fx_image);
 590         fpu_init();
 591         fx_save(vcpu->guest_fx_image);
 592         fx_restore(vcpu->host_fx_image);
 593
 594         fx_image = (struct fx_image_s *)vcpu->guest_fx_image;
 595         fx_image->mxcsr = 0x1f80;
 596         memset(vcpu->guest_fx_image + sizeof(struct fx_image_s),
 597                0, FX_IMAGE_SIZE - sizeof(struct fx_image_s));
 598 }
 599 EXPORT_SYMBOL_GPL(fx_init);
 600
 601 /*
 602  * Creates some virtual cpus.  Good luck creating more than one.
 603  */
 604 static int kvm_vm_ioctl_create_vcpu(struct kvm *kvm, int n)
 605 {
 606         int r;
 607         struct kvm_vcpu *vcpu;
 608
 609         r = -EINVAL;
 610         if (!valid_vcpu(n))
 611                 goto out;
 612
 613         vcpu = &kvm->vcpus[n];
 614
 615         mutex_lock(&vcpu->mutex);
 616
 617         if (vcpu->vmcs) {
 618                 mutex_unlock(&vcpu->mutex);
 619                 return -EEXIST;
 620         }
 621
 622         vcpu->host_fx_image = (char*)ALIGN((hva_t)vcpu->fx_buf,
 623                                            FX_IMAGE_ALIGN);
 624         vcpu->guest_fx_image = vcpu->host_fx_image + FX_IMAGE_SIZE;
 625
 626         r = kvm_arch_ops->vcpu_create(vcpu);
 627         if (r < 0)
 628                 goto out_free_vcpus;
 629
 630         r = kvm_mmu_create(vcpu);
 631         if (r < 0)
 632                 goto out_free_vcpus;
 633
 634         kvm_arch_ops->vcpu_load(vcpu);
 635         r = kvm_mmu_setup(vcpu);
 636         if (r >= 0)
 637                 r = kvm_arch_ops->vcpu_setup(vcpu);
 638         vcpu_put(vcpu);
 639
 640         if (r < 0)
 641                 goto out_free_vcpus;
 642
 643         return 0;
 644
 645 out_free_vcpus:
 646         kvm_free_vcpu(vcpu);
 647         mutex_unlock(&vcpu->mutex);
 648 out:
 649         return r;
 650 }
 651
 652 /*
 653  * Allocate some memory and give it an address in the guest physical address
 654  * space.
 655  *
 656  * Discontiguous memory is allowed, mostly for framebuffers.
 657  */
 658 static int kvm_vm_ioctl_set_memory_region(struct kvm *kvm,
 659                                           struct kvm_memory_region *mem)
 660 {
 661         int r;
 662         gfn_t base_gfn;
 663         unsigned long npages;
 664         unsigned long i;
 665         struct kvm_memory_slot *memslot;
 666         struct kvm_memory_slot old, new;
 667         int memory_config_version;
 668
 669         r = -EINVAL;
 670         /* General sanity checks */
 671         if (mem->memory_size & (PAGE_SIZE - 1))
 672                 goto out;
 673         if (mem->guest_phys_addr & (PAGE_SIZE - 1))
 674                 goto out;
 675         if (mem->slot >= KVM_MEMORY_SLOTS)
 676                 goto out;
 677         if (mem->guest_phys_addr + mem->memory_size < mem->guest_phys_addr)
 678                 goto out;
 679
 680         memslot = &kvm->memslots[mem->slot];
 681         base_gfn = mem->guest_phys_addr >> PAGE_SHIFT;
 682         npages = mem->memory_size >> PAGE_SHIFT;
 683
 684         if (!npages)
 685                 mem->flags &= ~KVM_MEM_LOG_DIRTY_PAGES;
 686
 687 raced:
 688         spin_lock(&kvm->lock);
 689
 690         memory_config_version = kvm->memory_config_version;
 691         new = old = *memslot;
 692
 693         new.base_gfn = base_gfn;
 694         new.npages = npages;
 695         new.flags = mem->flags;
 696
 697         /* Disallow changing a memory slot's size. */
 698         r = -EINVAL;
 699         if (npages && old.npages && npages != old.npages)
 700                 goto out_unlock;
 701
 702         /* Check for overlaps */
 703         r = -EEXIST;
 704         for (i = 0; i < KVM_MEMORY_SLOTS; ++i) {
 705                 struct kvm_memory_slot *s = &kvm->memslots[i];
 706
 707                 if (s == memslot)
 708                         continue;
 709                 if (!((base_gfn + npages <= s->base_gfn) ||
 710                       (base_gfn >= s->base_gfn + s->npages)))
 711                         goto out_unlock;
 712         }
 713         /*
 714          * Do memory allocations outside lock.  memory_config_version will
 715          * detect any races.
 716          */
 717         spin_unlock(&kvm->lock);
 718
 719         /* Deallocate if slot is being removed */
 720         if (!npages)
 721                 new.phys_mem = NULL;
 722
 723         /* Free page dirty bitmap if unneeded */
 724         if (!(new.flags & KVM_MEM_LOG_DIRTY_PAGES))
 725                 new.dirty_bitmap = NULL;
 726
 727         r = -ENOMEM;
 728
 729         /* Allocate if a slot is being created */
 730         if (npages && !new.phys_mem) {
 731                 new.phys_mem = vmalloc(npages * sizeof(struct page *));
 732
 733                 if (!new.phys_mem)
 734                         goto out_free;
 735
 736                 memset(new.phys_mem, 0, npages * sizeof(struct page *));
 737                 for (i = 0; i < npages; ++i) {
 738                         new.phys_mem[i] = alloc_page(GFP_HIGHUSER
 739                                                      | __GFP_ZERO);
 740                         if (!new.phys_mem[i])
 741                                 goto out_free;
 742                         set_page_private(new.phys_mem[i],0);
 743                 }
 744         }
 745
 746         /* Allocate page dirty bitmap if needed */
 747         if ((new.flags & KVM_MEM_LOG_DIRTY_PAGES) && !new.dirty_bitmap) {
 748                 unsigned dirty_bytes = ALIGN(npages, BITS_PER_LONG) / 8;
 749
 750                 new.dirty_bitmap = vmalloc(dirty_bytes);
 751                 if (!new.dirty_bitmap)
 752                         goto out_free;
 753                 memset(new.dirty_bitmap, 0, dirty_bytes);
 754         }
 755
 756         spin_lock(&kvm->lock);
 757
 758         if (memory_config_version != kvm->memory_config_version) {
 759                 spin_unlock(&kvm->lock);
 760                 kvm_free_physmem_slot(&new, &old);
 761                 goto raced;
 762         }
 763
 764         r = -EAGAIN;
 765         if (kvm->busy)
 766                 goto out_unlock;
 767
 768         if (mem->slot >= kvm->nmemslots)
 769                 kvm->nmemslots = mem->slot + 1;
 770
 771         *memslot = new;
 772         ++kvm->memory_config_version;
 773
 774         spin_unlock(&kvm->lock);
 775
 776         for (i = 0; i < KVM_MAX_VCPUS; ++i) {
 777                 struct kvm_vcpu *vcpu;
 778
 779                 vcpu = vcpu_load(kvm, i);
 780                 if (!vcpu)
 781                         continue;
 782                 kvm_mmu_reset_context(vcpu);
 783                 vcpu_put(vcpu);
 784         }
 785
 786         kvm_free_physmem_slot(&old, &new);
 787         return 0;
 788
 789 out_unlock:
 790         spin_unlock(&kvm->lock);
 791 out_free:
 792         kvm_free_physmem_slot(&new, &old);
 793 out:
 794         return r;
 795 }
 796
 797 static void do_remove_write_access(struct kvm_vcpu *vcpu, int slot)
 798 {
 799         spin_lock(&vcpu->kvm->lock);
 800         kvm_mmu_slot_remove_write_access(vcpu, slot);
 801         spin_unlock(&vcpu->kvm->lock);
 802 }
 803
 804 /*
 805  * Get (and clear) the dirty memory log for a memory slot.
 806  */
 807 static int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm,
 808                                       struct kvm_dirty_log *log)
 809 {
 810         struct kvm_memory_slot *memslot;
 811         int r, i;
 812         int n;
 813         int cleared;
 814         unsigned long any = 0;
 815
 816         spin_lock(&kvm->lock);
 817
 818         /*
 819          * Prevent changes to guest memory configuration even while the lock
 820          * is not taken.
 821          */
 822         ++kvm->busy;
 823         spin_unlock(&kvm->lock);
 824         r = -EINVAL;
 825         if (log->slot >= KVM_MEMORY_SLOTS)
 826                 goto out;
 827
 828         memslot = &kvm->memslots[log->slot];
 829         r = -ENOENT;
 830         if (!memslot->dirty_bitmap)
 831                 goto out;
 832
 833         n = ALIGN(memslot->npages, 8) / 8;
 834
 835         for (i = 0; !any && i < n; ++i)
 836                 any = memslot->dirty_bitmap[i];
 837
 838         r = -EFAULT;
 839         if (copy_to_user(log->dirty_bitmap, memslot->dirty_bitmap, n))
 840                 goto out;
 841
 842         if (any) {
 843                 cleared = 0;
 844                 for (i = 0; i < KVM_MAX_VCPUS; ++i) {
 845                         struct kvm_vcpu *vcpu = vcpu_load(kvm, i);
 846
 847                         if (!vcpu)
 848                                 continue;
 849                         if (!cleared) {
 850                                 do_remove_write_access(vcpu, log->slot);
 851                                 memset(memslot->dirty_bitmap, 0, n);
 852                                 cleared = 1;
 853                         }
 854                         kvm_arch_ops->tlb_flush(vcpu);
 855                         vcpu_put(vcpu);
 856                 }
 857         }
 858
 859         r = 0;
 860
 861 out:
 862         spin_lock(&kvm->lock);
 863         --kvm->busy;
 864         spin_unlock(&kvm->lock);
 865         return r;
 866 }
 867
 868 struct kvm_memory_slot *gfn_to_memslot(struct kvm *kvm, gfn_t gfn)
 869 {
 870         int i;
 871
 872         for (i = 0; i < kvm->nmemslots; ++i) {
 873                 struct kvm_memory_slot *memslot = &kvm->memslots[i];
 874
 875                 if (gfn >= memslot->base_gfn
 876                     && gfn < memslot->base_gfn + memslot->npages)
 877                         return memslot;
 878         }
 879         return NULL;
 880 }
 881 EXPORT_SYMBOL_GPL(gfn_to_memslot);
 882
 883 void mark_page_dirty(struct kvm *kvm, gfn_t gfn)
 884 {
 885         int i;
 886         struct kvm_memory_slot *memslot = NULL;
 887         unsigned long rel_gfn;
 888
 889         for (i = 0; i < kvm->nmemslots; ++i) {
 890                 memslot = &kvm->memslots[i];
 891
 892                 if (gfn >= memslot->base_gfn
 893                     && gfn < memslot->base_gfn + memslot->npages) {
 894
 895                         if (!memslot || !memslot->dirty_bitmap)
 896                                 return;
 897
 898                         rel_gfn = gfn - memslot->base_gfn;
 899
 900                         /* avoid RMW */
 901                         if (!test_bit(rel_gfn, memslot->dirty_bitmap))
 902                                 set_bit(rel_gfn, memslot->dirty_bitmap);
 903                         return;
 904                 }
 905         }
 906 }
 907
 908 static int emulator_read_std(unsigned long addr,
 909                              unsigned long *val,
 910                              unsigned int bytes,
 911                              struct x86_emulate_ctxt *ctxt)
 912 {
 913         struct kvm_vcpu *vcpu = ctxt->vcpu;
 914         void *data = val;
 915
 916         while (bytes) {
 917                 gpa_t gpa = vcpu->mmu.gva_to_gpa(vcpu, addr);
 918                 unsigned offset = addr & (PAGE_SIZE-1);
 919                 unsigned tocopy = min(bytes, (unsigned)PAGE_SIZE - offset);
 920                 unsigned long pfn;
 921                 struct kvm_memory_slot *memslot;
 922                 void *page;
 923
 924                 if (gpa == UNMAPPED_GVA)
 925                         return X86EMUL_PROPAGATE_FAULT;
 926                 pfn = gpa >> PAGE_SHIFT;
 927                 memslot = gfn_to_memslot(vcpu->kvm, pfn);
 928                 if (!memslot)
 929                         return X86EMUL_UNHANDLEABLE;
 930                 page = kmap_atomic(gfn_to_page(memslot, pfn), KM_USER0);
 931
 932                 memcpy(data, page + offset, tocopy);
 933
 934                 kunmap_atomic(page, KM_USER0);
 935
 936                 bytes -= tocopy;
 937                 data += tocopy;
 938                 addr += tocopy;
 939         }
 940
 941         return X86EMUL_CONTINUE;
 942 }
 943
 944 static int emulator_write_std(unsigned long addr,
 945                               unsigned long val,
 946                               unsigned int bytes,
 947                               struct x86_emulate_ctxt *ctxt)
 948 {
 949         printk(KERN_ERR "emulator_write_std: addr %lx n %d\n",
 950                addr, bytes);
 951         return X86EMUL_UNHANDLEABLE;
 952 }
 953
 954 static int emulator_read_emulated(unsigned long addr,
 955                                   unsigned long *val,
 956                                   unsigned int bytes,
 957                                   struct x86_emulate_ctxt *ctxt)
 958 {
 959         struct kvm_vcpu *vcpu = ctxt->vcpu;
 960
 961         if (vcpu->mmio_read_completed) {
 962                 memcpy(val, vcpu->mmio_data, bytes);
 963                 vcpu->mmio_read_completed = 0;
 964                 return X86EMUL_CONTINUE;
 965         } else if (emulator_read_std(addr, val, bytes, ctxt)
 966                    == X86EMUL_CONTINUE)
 967                 return X86EMUL_CONTINUE;
 968         else {
 969                 gpa_t gpa = vcpu->mmu.gva_to_gpa(vcpu, addr);
 970
 971                 if (gpa == UNMAPPED_GVA)
 972                         return X86EMUL_PROPAGATE_FAULT;
 973                 vcpu->mmio_needed = 1;
 974                 vcpu->mmio_phys_addr = gpa;
 975                 vcpu->mmio_size = bytes;
 976                 vcpu->mmio_is_write = 0;
 977
 978                 return X86EMUL_UNHANDLEABLE;
 979         }
 980 }
 981
 982 static int emulator_write_phys(struct kvm_vcpu *vcpu, gpa_t gpa,
 983                                unsigned long val, int bytes)
 984 {
 985         struct kvm_memory_slot *m;
 986         struct page *page;
 987         void *virt;
 988
 989         if (((gpa + bytes - 1) >> PAGE_SHIFT) != (gpa >> PAGE_SHIFT))
 990                 return 0;
 991         m = gfn_to_memslot(vcpu->kvm, gpa >> PAGE_SHIFT);
 992         if (!m)
 993                 return 0;
 994         page = gfn_to_page(m, gpa >> PAGE_SHIFT);
 995         kvm_mmu_pre_write(vcpu, gpa, bytes);
 996         virt = kmap_atomic(page, KM_USER0);
 997         memcpy(virt + offset_in_page(gpa), &val, bytes);
 998         kunmap_atomic(virt, KM_USER0);
 999         kvm_mmu_post_write(vcpu, gpa, bytes);
1000         return 1;
1001 }
1002
1003 static int emulator_write_emulated(unsigned long addr,
1004                                    unsigned long val,
1005                                    unsigned int bytes,
1006                                    struct x86_emulate_ctxt *ctxt)
1007 {
1008         struct kvm_vcpu *vcpu = ctxt->vcpu;
1009         gpa_t gpa = vcpu->mmu.gva_to_gpa(vcpu, addr);
1010
1011         if (gpa == UNMAPPED_GVA)
1012                 return X86EMUL_PROPAGATE_FAULT;
1013
1014         if (emulator_write_phys(vcpu, gpa, val, bytes))
1015                 return X86EMUL_CONTINUE;
1016
1017         vcpu->mmio_needed = 1;
1018         vcpu->mmio_phys_addr = gpa;
1019         vcpu->mmio_size = bytes;
1020         vcpu->mmio_is_write = 1;
1021         memcpy(vcpu->mmio_data, &val, bytes);
1022
1023         return X86EMUL_CONTINUE;
1024 }
1025
1026 static int emulator_cmpxchg_emulated(unsigned long addr,
1027                                      unsigned long old,
1028                                      unsigned long new,
1029                                      unsigned int bytes,
1030                                      struct x86_emulate_ctxt *ctxt)
1031 {
1032         static int reported;
1033
1034         if (!reported) {
1035                 reported = 1;
1036                 printk(KERN_WARNING "kvm: emulating exchange as write\n");
1037         }
1038         return emulator_write_emulated(addr, new, bytes, ctxt);
1039 }
1040
1041 #ifdef CONFIG_X86_32
1042
1043 static int emulator_cmpxchg8b_emulated(unsigned long addr,
1044                                        unsigned long old_lo,
1045                                        unsigned long old_hi,
1046                                        unsigned long new_lo,
1047                                        unsigned long new_hi,
1048                                        struct x86_emulate_ctxt *ctxt)
1049 {
1050         static int reported;
1051         int r;
1052
1053         if (!reported) {
1054                 reported = 1;
1055                 printk(KERN_WARNING "kvm: emulating exchange8b as write\n");
1056         }
1057         r = emulator_write_emulated(addr, new_lo, 4, ctxt);
1058         if (r != X86EMUL_CONTINUE)
1059                 return r;
1060         return emulator_write_emulated(addr+4, new_hi, 4, ctxt);
1061 }
1062
1063 #endif
1064
1065 static unsigned long get_segment_base(struct kvm_vcpu *vcpu, int seg)
1066 {
1067         return kvm_arch_ops->get_segment_base(vcpu, seg);
1068 }
1069
1070 int emulate_invlpg(struct kvm_vcpu *vcpu, gva_t address)
1071 {
1072         return X86EMUL_CONTINUE;
1073 }
1074
1075 int emulate_clts(struct kvm_vcpu *vcpu)
1076 {
1077         unsigned long cr0;
1078
1079         kvm_arch_ops->decache_cr0_cr4_guest_bits(vcpu);
1080         cr0 = vcpu->cr0 & ~CR0_TS_MASK;
1081         kvm_arch_ops->set_cr0(vcpu, cr0);
1082         return X86EMUL_CONTINUE;
1083 }
1084
1085 int emulator_get_dr(struct x86_emulate_ctxt* ctxt, int dr, unsigned long *dest)
1086 {
1087         struct kvm_vcpu *vcpu = ctxt->vcpu;
1088
1089         switch (dr) {
1090         case 0 ... 3:
1091                 *dest = kvm_arch_ops->get_dr(vcpu, dr);
1092                 return X86EMUL_CONTINUE;
1093         default:
1094                 printk(KERN_DEBUG "%s: unexpected dr %u\n",
1095                        __FUNCTION__, dr);
1096                 return X86EMUL_UNHANDLEABLE;
1097         }
1098 }
1099
1100 int emulator_set_dr(struct x86_emulate_ctxt *ctxt, int dr, unsigned long value)
1101 {
1102         unsigned long mask = (ctxt->mode == X86EMUL_MODE_PROT64) ? ~0ULL : ~0U;
1103         int exception;
1104
1105         kvm_arch_ops->set_dr(ctxt->vcpu, dr, value & mask, &exception);
1106         if (exception) {
1107                 /* FIXME: better handling */
1108                 return X86EMUL_UNHANDLEABLE;
1109         }
1110         return X86EMUL_CONTINUE;
1111 }
1112
1113 static void report_emulation_failure(struct x86_emulate_ctxt *ctxt)
1114 {
1115         static int reported;
1116         u8 opcodes[4];
1117         unsigned long rip = ctxt->vcpu->rip;
1118         unsigned long rip_linear;
1119
1120         rip_linear = rip + get_segment_base(ctxt->vcpu, VCPU_SREG_CS);
1121
1122         if (reported)
1123                 return;
1124
1125         emulator_read_std(rip_linear, (void *)opcodes, 4, ctxt);
1126
1127         printk(KERN_ERR "emulation failed but !mmio_needed?"
1128                " rip %lx %02x %02x %02x %02x\n",
1129                rip, opcodes[0], opcodes[1], opcodes[2], opcodes[3]);
1130         reported = 1;
1131 }
1132
1133 struct x86_emulate_ops emulate_ops = {
1134         .read_std            = emulator_read_std,
1135         .write_std           = emulator_write_std,
1136         .read_emulated       = emulator_read_emulated,
1137         .write_emulated      = emulator_write_emulated,
1138         .cmpxchg_emulated    = emulator_cmpxchg_emulated,
1139 #ifdef CONFIG_X86_32
1140         .cmpxchg8b_emulated  = emulator_cmpxchg8b_emulated,
1141 #endif
1142 };
1143
1144 int emulate_instruction(struct kvm_vcpu *vcpu,
1145                         struct kvm_run *run,
1146                         unsigned long cr2,
1147                         u16 error_code)
1148 {
1149         struct x86_emulate_ctxt emulate_ctxt;
1150         int r;
1151         int cs_db, cs_l;
1152
1153         kvm_arch_ops->cache_regs(vcpu);
1154
1155         kvm_arch_ops->get_cs_db_l_bits(vcpu, &cs_db, &cs_l);
1156
1157         emulate_ctxt.vcpu = vcpu;
1158         emulate_ctxt.eflags = kvm_arch_ops->get_rflags(vcpu);
1159         emulate_ctxt.cr2 = cr2;
1160         emulate_ctxt.mode = (emulate_ctxt.eflags & X86_EFLAGS_VM)
1161                 ? X86EMUL_MODE_REAL : cs_l
1162                 ? X86EMUL_MODE_PROT64 : cs_db
1163                 ? X86EMUL_MODE_PROT32 : X86EMUL_MODE_PROT16;
1164
1165         if (emulate_ctxt.mode == X86EMUL_MODE_PROT64) {
1166                 emulate_ctxt.cs_base = 0;
1167                 emulate_ctxt.ds_base = 0;
1168                 emulate_ctxt.es_base = 0;
1169                 emulate_ctxt.ss_base = 0;
1170         } else {
1171                 emulate_ctxt.cs_base = get_segment_base(vcpu, VCPU_SREG_CS);
1172                 emulate_ctxt.ds_base = get_segment_base(vcpu, VCPU_SREG_DS);
1173                 emulate_ctxt.es_base = get_segment_base(vcpu, VCPU_SREG_ES);
1174                 emulate_ctxt.ss_base = get_segment_base(vcpu, VCPU_SREG_SS);
1175         }
1176
1177         emulate_ctxt.gs_base = get_segment_base(vcpu, VCPU_SREG_GS);
1178         emulate_ctxt.fs_base = get_segment_base(vcpu, VCPU_SREG_FS);
1179
1180         vcpu->mmio_is_write = 0;
1181         r = x86_emulate_memop(&emulate_ctxt, &emulate_ops);
1182
1183         if ((r || vcpu->mmio_is_write) && run) {
1184                 run->mmio.phys_addr = vcpu->mmio_phys_addr;
1185                 memcpy(run->mmio.data, vcpu->mmio_data, 8);
1186                 run->mmio.len = vcpu->mmio_size;
1187                 run->mmio.is_write = vcpu->mmio_is_write;
1188         }
1189
1190         if (r) {
1191                 if (kvm_mmu_unprotect_page_virt(vcpu, cr2))
1192                         return EMULATE_DONE;
1193                 if (!vcpu->mmio_needed) {
1194                         report_emulation_failure(&emulate_ctxt);
1195                         return EMULATE_FAIL;
1196                 }
1197                 return EMULATE_DO_MMIO;
1198         }
1199
1200         kvm_arch_ops->decache_regs(vcpu);
1201         kvm_arch_ops->set_rflags(vcpu, emulate_ctxt.eflags);
1202
1203         if (vcpu->mmio_is_write)
1204                 return EMULATE_DO_MMIO;
1205
1206         return EMULATE_DONE;
1207 }
1208 EXPORT_SYMBOL_GPL(emulate_instruction);
1209
1210 int kvm_hypercall(struct kvm_vcpu *vcpu, struct kvm_run *run)
1211 {
1212         unsigned long nr, a0, a1, a2, a3, a4, a5, ret;
1213
1214         kvm_arch_ops->decache_regs(vcpu);
1215         ret = -KVM_EINVAL;
1216 #ifdef CONFIG_X86_64
1217         if (is_long_mode(vcpu)) {
1218                 nr = vcpu->regs[VCPU_REGS_RAX];
1219                 a0 = vcpu->regs[VCPU_REGS_RDI];
1220                 a1 = vcpu->regs[VCPU_REGS_RSI];
1221                 a2 = vcpu->regs[VCPU_REGS_RDX];
1222                 a3 = vcpu->regs[VCPU_REGS_RCX];
1223                 a4 = vcpu->regs[VCPU_REGS_R8];
1224                 a5 = vcpu->regs[VCPU_REGS_R9];
1225         } else
1226 #endif
1227         {
1228                 nr = vcpu->regs[VCPU_REGS_RBX] & -1u;
1229                 a0 = vcpu->regs[VCPU_REGS_RAX] & -1u;
1230                 a1 = vcpu->regs[VCPU_REGS_RCX] & -1u;
1231                 a2 = vcpu->regs[VCPU_REGS_RDX] & -1u;
1232                 a3 = vcpu->regs[VCPU_REGS_RSI] & -1u;
1233                 a4 = vcpu->regs[VCPU_REGS_RDI] & -1u;
1234                 a5 = vcpu->regs[VCPU_REGS_RBP] & -1u;
1235         }
1236         switch (nr) {
1237         default:
1238                 ;
1239         }
1240         vcpu->regs[VCPU_REGS_RAX] = ret;
1241         kvm_arch_ops->cache_regs(vcpu);
1242         return 1;
1243 }
1244 EXPORT_SYMBOL_GPL(kvm_hypercall);
1245
1246 static u64 mk_cr_64(u64 curr_cr, u32 new_val)
1247 {
1248         return (curr_cr & ~((1ULL << 32) - 1)) | new_val;
1249 }
1250
1251 void realmode_lgdt(struct kvm_vcpu *vcpu, u16 limit, unsigned long base)
1252 {
1253         struct descriptor_table dt = { limit, base };
1254
1255         kvm_arch_ops->set_gdt(vcpu, &dt);
1256 }
1257
1258 void realmode_lidt(struct kvm_vcpu *vcpu, u16 limit, unsigned long base)
1259 {
1260         struct descriptor_table dt = { limit, base };
1261
1262         kvm_arch_ops->set_idt(vcpu, &dt);
1263 }
1264
1265 void realmode_lmsw(struct kvm_vcpu *vcpu, unsigned long msw,
1266                    unsigned long *rflags)
1267 {
1268         lmsw(vcpu, msw);
1269         *rflags = kvm_arch_ops->get_rflags(vcpu);
1270 }
1271
1272 unsigned long realmode_get_cr(struct kvm_vcpu *vcpu, int cr)
1273 {
1274         kvm_arch_ops->decache_cr0_cr4_guest_bits(vcpu);
1275         switch (cr) {
1276         case 0:
1277                 return vcpu->cr0;
1278         case 2:
1279                 return vcpu->cr2;
1280         case 3:
1281                 return vcpu->cr3;
1282         case 4:
1283                 return vcpu->cr4;
1284         default:
1285                 vcpu_printf(vcpu, "%s: unexpected cr %u\n", __FUNCTION__, cr);
1286                 return 0;
1287         }
1288 }
1289
1290 void realmode_set_cr(struct kvm_vcpu *vcpu, int cr, unsigned long val,
1291                      unsigned long *rflags)
1292 {
1293         switch (cr) {
1294         case 0:
1295                 set_cr0(vcpu, mk_cr_64(vcpu->cr0, val));
1296                 *rflags = kvm_arch_ops->get_rflags(vcpu);
1297                 break;
1298         case 2:
1299                 vcpu->cr2 = val;
1300                 break;
1301         case 3:
1302                 set_cr3(vcpu, val);
1303                 break;
1304         case 4:
1305                 set_cr4(vcpu, mk_cr_64(vcpu->cr4, val));
1306                 break;
1307         default:
1308                 vcpu_printf(vcpu, "%s: unexpected cr %u\n", __FUNCTION__, cr);
1309         }
1310 }
1311
1312 /*
1313  * Register the para guest with the host:
1314  */
1315 static int vcpu_register_para(struct kvm_vcpu *vcpu, gpa_t para_state_gpa)
1316 {
1317         struct kvm_vcpu_para_state *para_state;
1318         hpa_t para_state_hpa, hypercall_hpa;
1319         struct page *para_state_page;
1320         unsigned char *hypercall;
1321         gpa_t hypercall_gpa;
1322
1323         printk(KERN_DEBUG "kvm: guest trying to enter paravirtual mode\n");
1324         printk(KERN_DEBUG ".... para_state_gpa: %08Lx\n", para_state_gpa);
1325
1326         /*
1327          * Needs to be page aligned:
1328          */
1329         if (para_state_gpa != PAGE_ALIGN(para_state_gpa))
1330                 goto err_gp;
1331
1332         para_state_hpa = gpa_to_hpa(vcpu, para_state_gpa);
1333         printk(KERN_DEBUG ".... para_state_hpa: %08Lx\n", para_state_hpa);
1334         if (is_error_hpa(para_state_hpa))
1335                 goto err_gp;
1336
1337         para_state_page = pfn_to_page(para_state_hpa >> PAGE_SHIFT);
1338         para_state = kmap_atomic(para_state_page, KM_USER0);
1339
1340         printk(KERN_DEBUG "....  guest version: %d\n", para_state->guest_version);
1341         printk(KERN_DEBUG "....           size: %d\n", para_state->size);
1342
1343         para_state->host_version = KVM_PARA_API_VERSION;
1344         /*
1345          * We cannot support guests that try to register themselves
1346          * with a newer API version than the host supports:
1347          */
1348         if (para_state->guest_version > KVM_PARA_API_VERSION) {
1349                 para_state->ret = -KVM_EINVAL;
1350                 goto err_kunmap_skip;
1351         }
1352
1353         hypercall_gpa = para_state->hypercall_gpa;
1354         hypercall_hpa = gpa_to_hpa(vcpu, hypercall_gpa);
1355         printk(KERN_DEBUG ".... hypercall_hpa: %08Lx\n", hypercall_hpa);
1356         if (is_error_hpa(hypercall_hpa)) {
1357                 para_state->ret = -KVM_EINVAL;
1358                 goto err_kunmap_skip;
1359         }
1360
1361         printk(KERN_DEBUG "kvm: para guest successfully registered.\n");
1362         vcpu->para_state_page = para_state_page;
1363         vcpu->para_state_gpa = para_state_gpa;
1364         vcpu->hypercall_gpa = hypercall_gpa;
1365
1366         hypercall = kmap_atomic(pfn_to_page(hypercall_hpa >> PAGE_SHIFT),
1367                                 KM_USER1) + (hypercall_hpa & ~PAGE_MASK);
1368         kvm_arch_ops->patch_hypercall(vcpu, hypercall);
1369         kunmap_atomic(hypercall, KM_USER1);
1370
1371         para_state->ret = 0;
1372 err_kunmap_skip:
1373         kunmap_atomic(para_state, KM_USER0);
1374         return 0;
1375 err_gp:
1376         return 1;
1377 }
1378
1379 int kvm_get_msr_common(struct kvm_vcpu *vcpu, u32 msr, u64 *pdata)
1380 {
1381         u64 data;
1382
1383         switch (msr) {
1384         case 0xc0010010: /* SYSCFG */
1385         case 0xc0010015: /* HWCR */
1386         case MSR_IA32_PLATFORM_ID:
1387         case MSR_IA32_P5_MC_ADDR:
1388         case MSR_IA32_P5_MC_TYPE:
1389         case MSR_IA32_MC0_CTL:
1390         case MSR_IA32_MCG_STATUS:
1391         case MSR_IA32_MCG_CAP:
1392         case MSR_IA32_MC0_MISC:
1393         case MSR_IA32_MC0_MISC+4:
1394         case MSR_IA32_MC0_MISC+8:
1395         case MSR_IA32_MC0_MISC+12:
1396         case MSR_IA32_MC0_MISC+16:
1397         case MSR_IA32_UCODE_REV:
1398         case MSR_IA32_PERF_STATUS:
1399                 /* MTRR registers */
1400         case 0xfe:
1401         case 0x200 ... 0x2ff:
1402                 data = 0;
1403                 break;
1404         case 0xcd: /* fsb frequency */
1405                 data = 3;
1406                 break;
1407         case MSR_IA32_APICBASE:
1408                 data = vcpu->apic_base;
1409                 break;
1410         case MSR_IA32_MISC_ENABLE:
1411                 data = vcpu->ia32_misc_enable_msr;
1412                 break;
1413 #ifdef CONFIG_X86_64
1414         case MSR_EFER:
1415                 data = vcpu->shadow_efer;
1416                 break;
1417 #endif
1418         default:
1419                 printk(KERN_ERR "kvm: unhandled rdmsr: 0x%x\n", msr);
1420                 return 1;
1421         }
1422         *pdata = data;
1423         return 0;
1424 }
1425 EXPORT_SYMBOL_GPL(kvm_get_msr_common);
1426
1427 /*
1428  * Reads an msr value (of 'msr_index') into 'pdata'.
1429  * Returns 0 on success, non-0 otherwise.
1430  * Assumes vcpu_load() was already called.
1431  */
1432 static int get_msr(struct kvm_vcpu *vcpu, u32 msr_index, u64 *pdata)
1433 {
1434         return kvm_arch_ops->get_msr(vcpu, msr_index, pdata);
1435 }
1436
1437 #ifdef CONFIG_X86_64
1438
1439 static void set_efer(struct kvm_vcpu *vcpu, u64 efer)
1440 {
1441         if (efer & EFER_RESERVED_BITS) {
1442                 printk(KERN_DEBUG "set_efer: 0x%llx #GP, reserved bits\n",
1443                        efer);
1444                 inject_gp(vcpu);
1445                 return;
1446         }
1447
1448         if (is_paging(vcpu)
1449             && (vcpu->shadow_efer & EFER_LME) != (efer & EFER_LME)) {
1450                 printk(KERN_DEBUG "set_efer: #GP, change LME while paging\n");
1451                 inject_gp(vcpu);
1452                 return;
1453         }
1454
1455         kvm_arch_ops->set_efer(vcpu, efer);
1456
1457         efer &= ~EFER_LMA;
1458         efer |= vcpu->shadow_efer & EFER_LMA;
1459
1460         vcpu->shadow_efer = efer;
1461 }
1462
1463 #endif
1464
1465 int kvm_set_msr_common(struct kvm_vcpu *vcpu, u32 msr, u64 data)
1466 {
1467         switch (msr) {
1468 #ifdef CONFIG_X86_64
1469         case MSR_EFER:
1470                 set_efer(vcpu, data);
1471                 break;
1472 #endif
1473         case MSR_IA32_MC0_STATUS:
1474                 printk(KERN_WARNING "%s: MSR_IA32_MC0_STATUS 0x%llx, nop\n",
1475                        __FUNCTION__, data);
1476                 break;
1477         case MSR_IA32_UCODE_REV:
1478         case MSR_IA32_UCODE_WRITE:
1479         case 0x200 ... 0x2ff: /* MTRRs */
1480                 break;
1481         case MSR_IA32_APICBASE:
1482                 vcpu->apic_base = data;
1483                 break;
1484         case MSR_IA32_MISC_ENABLE:
1485                 vcpu->ia32_misc_enable_msr = data;
1486                 break;
1487         /*
1488          * This is the 'probe whether the host is KVM' logic:
1489          */
1490         case MSR_KVM_API_MAGIC:
1491                 return vcpu_register_para(vcpu, data);
1492
1493         default:
1494                 printk(KERN_ERR "kvm: unhandled wrmsr: 0x%x\n", msr);
1495                 return 1;
1496         }
1497         return 0;
1498 }
1499 EXPORT_SYMBOL_GPL(kvm_set_msr_common);
1500
1501 /*
1502  * Writes msr value into into the appropriate "register".
1503  * Returns 0 on success, non-0 otherwise.
1504  * Assumes vcpu_load() was already called.
1505  */
1506 static int set_msr(struct kvm_vcpu *vcpu, u32 msr_index, u64 data)
1507 {
1508         return kvm_arch_ops->set_msr(vcpu, msr_index, data);
1509 }
1510
1511 void kvm_resched(struct kvm_vcpu *vcpu)
1512 {
1513         vcpu_put(vcpu);
1514         cond_resched();
1515         /* Cannot fail -  no vcpu unplug yet. */
1516         vcpu_load(vcpu->kvm, vcpu_slot(vcpu));
1517 }
1518 EXPORT_SYMBOL_GPL(kvm_resched);
1519
1520 void load_msrs(struct vmx_msr_entry *e, int n)
1521 {
1522         int i;
1523
1524         for (i = 0; i < n; ++i)
1525                 wrmsrl(e[i].index, e[i].data);
1526 }
1527 EXPORT_SYMBOL_GPL(load_msrs);
1528
1529 void save_msrs(struct vmx_msr_entry *e, int n)
1530 {
1531         int i;
1532
1533         for (i = 0; i < n; ++i)
1534                 rdmsrl(e[i].index, e[i].data);
1535 }
1536 EXPORT_SYMBOL_GPL(save_msrs);
1537
1538 static int kvm_vm_ioctl_run(struct kvm *kvm, struct kvm_run *kvm_run)
1539 {
1540         struct kvm_vcpu *vcpu;
1541         int r;
1542
1543         if (!valid_vcpu(kvm_run->vcpu))
1544                 return -EINVAL;
1545
1546         vcpu = vcpu_load(kvm, kvm_run->vcpu);
1547         if (!vcpu)
1548                 return -ENOENT;
1549
1550         /* re-sync apic's tpr */
1551         vcpu->cr8 = kvm_run->cr8;
1552
1553         if (kvm_run->emulated) {
1554                 kvm_arch_ops->skip_emulated_instruction(vcpu);
1555                 kvm_run->emulated = 0;
1556         }
1557
1558         if (kvm_run->mmio_completed) {
1559                 memcpy(vcpu->mmio_data, kvm_run->mmio.data, 8);
1560                 vcpu->mmio_read_completed = 1;
1561         }
1562
1563         vcpu->mmio_needed = 0;
1564
1565         r = kvm_arch_ops->run(vcpu, kvm_run);
1566
1567         vcpu_put(vcpu);
1568         return r;
1569 }
1570
1571 static int kvm_vm_ioctl_get_regs(struct kvm *kvm, struct kvm_regs *regs)
1572 {
1573         struct kvm_vcpu *vcpu;
1574
1575         if (!valid_vcpu(regs->vcpu))
1576                 return -EINVAL;
1577
1578         vcpu = vcpu_load(kvm, regs->vcpu);
1579         if (!vcpu)
1580                 return -ENOENT;
1581
1582         kvm_arch_ops->cache_regs(vcpu);
1583
1584         regs->rax = vcpu->regs[VCPU_REGS_RAX];
1585         regs->rbx = vcpu->regs[VCPU_REGS_RBX];
1586         regs->rcx = vcpu->regs[VCPU_REGS_RCX];
1587         regs->rdx = vcpu->regs[VCPU_REGS_RDX];
1588         regs->rsi = vcpu->regs[VCPU_REGS_RSI];
1589         regs->rdi = vcpu->regs[VCPU_REGS_RDI];
1590         regs->rsp = vcpu->regs[VCPU_REGS_RSP];
1591         regs->rbp = vcpu->regs[VCPU_REGS_RBP];
1592 #ifdef CONFIG_X86_64
1593         regs->r8 = vcpu->regs[VCPU_REGS_R8];
1594         regs->r9 = vcpu->regs[VCPU_REGS_R9];
1595         regs->r10 = vcpu->regs[VCPU_REGS_R10];
1596         regs->r11 = vcpu->regs[VCPU_REGS_R11];
1597         regs->r12 = vcpu->regs[VCPU_REGS_R12];
1598         regs->r13 = vcpu->regs[VCPU_REGS_R13];
1599         regs->r14 = vcpu->regs[VCPU_REGS_R14];
1600         regs->r15 = vcpu->regs[VCPU_REGS_R15];
1601 #endif
1602
1603         regs->rip = vcpu->rip;
1604         regs->rflags = kvm_arch_ops->get_rflags(vcpu);
1605
1606         /*
1607          * Don't leak debug flags in case they were set for guest debugging
1608          */
1609         if (vcpu->guest_debug.enabled && vcpu->guest_debug.singlestep)
1610                 regs->rflags &= ~(X86_EFLAGS_TF | X86_EFLAGS_RF);
1611
1612         vcpu_put(vcpu);
1613
1614         return 0;
1615 }
1616
1617 static int kvm_vm_ioctl_set_regs(struct kvm *kvm, struct kvm_regs *regs)
1618 {
1619         struct kvm_vcpu *vcpu;
1620
1621         if (!valid_vcpu(regs->vcpu))
1622                 return -EINVAL;
1623
1624         vcpu = vcpu_load(kvm, regs->vcpu);
1625         if (!vcpu)
1626                 return -ENOENT;
1627
1628         vcpu->regs[VCPU_REGS_RAX] = regs->rax;
1629         vcpu->regs[VCPU_REGS_RBX] = regs->rbx;
1630         vcpu->regs[VCPU_REGS_RCX] = regs->rcx;
1631         vcpu->regs[VCPU_REGS_RDX] = regs->rdx;
1632         vcpu->regs[VCPU_REGS_RSI] = regs->rsi;
1633         vcpu->regs[VCPU_REGS_RDI] = regs->rdi;
1634         vcpu->regs[VCPU_REGS_RSP] = regs->rsp;
1635         vcpu->regs[VCPU_REGS_RBP] = regs->rbp;
1636 #ifdef CONFIG_X86_64
1637         vcpu->regs[VCPU_REGS_R8] = regs->r8;
1638         vcpu->regs[VCPU_REGS_R9] = regs->r9;
1639         vcpu->regs[VCPU_REGS_R10] = regs->r10;
1640         vcpu->regs[VCPU_REGS_R11] = regs->r11;
1641         vcpu->regs[VCPU_REGS_R12] = regs->r12;
1642         vcpu->regs[VCPU_REGS_R13] = regs->r13;
1643         vcpu->regs[VCPU_REGS_R14] = regs->r14;
1644         vcpu->regs[VCPU_REGS_R15] = regs->r15;
1645 #endif
1646
1647         vcpu->rip = regs->rip;
1648         kvm_arch_ops->set_rflags(vcpu, regs->rflags);
1649
1650         kvm_arch_ops->decache_regs(vcpu);
1651
1652         vcpu_put(vcpu);
1653
1654         return 0;
1655 }
1656
1657 static void get_segment(struct kvm_vcpu *vcpu,
1658                         struct kvm_segment *var, int seg)
1659 {
1660         return kvm_arch_ops->get_segment(vcpu, var, seg);
1661 }
1662
1663 static int kvm_vm_ioctl_get_sregs(struct kvm *kvm, struct kvm_sregs *sregs)
1664 {
1665         struct kvm_vcpu *vcpu;
1666         struct descriptor_table dt;
1667
1668         if (!valid_vcpu(sregs->vcpu))
1669                 return -EINVAL;
1670         vcpu = vcpu_load(kvm, sregs->vcpu);
1671         if (!vcpu)
1672                 return -ENOENT;
1673
1674         get_segment(vcpu, &sregs->cs, VCPU_SREG_CS);
1675         get_segment(vcpu, &sregs->ds, VCPU_SREG_DS);
1676         get_segment(vcpu, &sregs->es, VCPU_SREG_ES);
1677         get_segment(vcpu, &sregs->fs, VCPU_SREG_FS);
1678         get_segment(vcpu, &sregs->gs, VCPU_SREG_GS);
1679         get_segment(vcpu, &sregs->ss, VCPU_SREG_SS);
1680
1681         get_segment(vcpu, &sregs->tr, VCPU_SREG_TR);
1682         get_segment(vcpu, &sregs->ldt, VCPU_SREG_LDTR);
1683
1684         kvm_arch_ops->get_idt(vcpu, &dt);
1685         sregs->idt.limit = dt.limit;
1686         sregs->idt.base = dt.base;
1687         kvm_arch_ops->get_gdt(vcpu, &dt);
1688         sregs->gdt.limit = dt.limit;
1689         sregs->gdt.base = dt.base;
1690
1691         kvm_arch_ops->decache_cr0_cr4_guest_bits(vcpu);
1692         sregs->cr0 = vcpu->cr0;
1693         sregs->cr2 = vcpu->cr2;
1694         sregs->cr3 = vcpu->cr3;
1695         sregs->cr4 = vcpu->cr4;
1696         sregs->cr8 = vcpu->cr8;
1697         sregs->efer = vcpu->shadow_efer;
1698         sregs->apic_base = vcpu->apic_base;
1699
1700         memcpy(sregs->interrupt_bitmap, vcpu->irq_pending,
1701                sizeof sregs->interrupt_bitmap);
1702
1703         vcpu_put(vcpu);
1704
1705         return 0;
1706 }
1707
1708 static void set_segment(struct kvm_vcpu *vcpu,
1709                         struct kvm_segment *var, int seg)
1710 {
1711         return kvm_arch_ops->set_segment(vcpu, var, seg);
1712 }
1713
1714 static int kvm_vm_ioctl_set_sregs(struct kvm *kvm, struct kvm_sregs *sregs)
1715 {
1716         struct kvm_vcpu *vcpu;
1717         int mmu_reset_needed = 0;
1718         int i;
1719         struct descriptor_table dt;
1720
1721         if (!valid_vcpu(sregs->vcpu))
1722                 return -EINVAL;
1723         vcpu = vcpu_load(kvm, sregs->vcpu);
1724         if (!vcpu)
1725                 return -ENOENT;
1726
1727         set_segment(vcpu, &sregs->cs, VCPU_SREG_CS);
1728         set_segment(vcpu, &sregs->ds, VCPU_SREG_DS);
1729         set_segment(vcpu, &sregs->es, VCPU_SREG_ES);
1730         set_segment(vcpu, &sregs->fs, VCPU_SREG_FS);
1731         set_segment(vcpu, &sregs->gs, VCPU_SREG_GS);
1732         set_segment(vcpu, &sregs->ss, VCPU_SREG_SS);
1733
1734         set_segment(vcpu, &sregs->tr, VCPU_SREG_TR);
1735         set_segment(vcpu, &sregs->ldt, VCPU_SREG_LDTR);
1736
1737         dt.limit = sregs->idt.limit;
1738         dt.base = sregs->idt.base;
1739         kvm_arch_ops->set_idt(vcpu, &dt);
1740         dt.limit = sregs->gdt.limit;
1741         dt.base = sregs->gdt.base;
1742         kvm_arch_ops->set_gdt(vcpu, &dt);
1743
1744         vcpu->cr2 = sregs->cr2;
1745         mmu_reset_needed |= vcpu->cr3 != sregs->cr3;
1746         vcpu->cr3 = sregs->cr3;
1747
1748         vcpu->cr8 = sregs->cr8;
1749
1750         mmu_reset_needed |= vcpu->shadow_efer != sregs->efer;
1751 #ifdef CONFIG_X86_64
1752         kvm_arch_ops->set_efer(vcpu, sregs->efer);
1753 #endif
1754         vcpu->apic_base = sregs->apic_base;
1755
1756         kvm_arch_ops->decache_cr0_cr4_guest_bits(vcpu);
1757
1758         mmu_reset_needed |= vcpu->cr0 != sregs->cr0;
1759         kvm_arch_ops->set_cr0_no_modeswitch(vcpu, sregs->cr0);
1760
1761         mmu_reset_needed |= vcpu->cr4 != sregs->cr4;
1762         kvm_arch_ops->set_cr4(vcpu, sregs->cr4);
1763         if (!is_long_mode(vcpu) && is_pae(vcpu))
1764                 load_pdptrs(vcpu, vcpu->cr3);
1765
1766         if (mmu_reset_needed)
1767                 kvm_mmu_reset_context(vcpu);
1768
1769         memcpy(vcpu->irq_pending, sregs->interrupt_bitmap,
1770                sizeof vcpu->irq_pending);
1771         vcpu->irq_summary = 0;
1772         for (i = 0; i < NR_IRQ_WORDS; ++i)
1773                 if (vcpu->irq_pending[i])
1774                         __set_bit(i, &vcpu->irq_summary);
1775
1776         vcpu_put(vcpu);
1777
1778         return 0;
1779 }
1780
1781 /*
1782  * List of msr numbers which we expose to userspace through KVM_GET_MSRS
1783  * and KVM_SET_MSRS, and KVM_GET_MSR_INDEX_LIST.
1784  *
1785  * This list is modified at module load time to reflect the
1786  * capabilities of the host cpu.
1787  */
1788 static u32 msrs_to_save[] = {
1789         MSR_IA32_SYSENTER_CS, MSR_IA32_SYSENTER_ESP, MSR_IA32_SYSENTER_EIP,
1790         MSR_K6_STAR,
1791 #ifdef CONFIG_X86_64
1792         MSR_CSTAR, MSR_KERNEL_GS_BASE, MSR_SYSCALL_MASK, MSR_LSTAR,
1793 #endif
1794         MSR_IA32_TIME_STAMP_COUNTER,
1795 };
1796
1797 static unsigned num_msrs_to_save;
1798
1799 static u32 emulated_msrs[] = {
1800         MSR_IA32_MISC_ENABLE,
1801 };
1802
1803 static __init void kvm_init_msr_list(void)
1804 {
1805         u32 dummy[2];
1806         unsigned i, j;
1807
1808         for (i = j = 0; i < ARRAY_SIZE(msrs_to_save); i++) {
1809                 if (rdmsr_safe(msrs_to_save[i], &dummy[0], &dummy[1]) < 0)
1810                         continue;
1811                 if (j < i)
1812                         msrs_to_save[j] = msrs_to_save[i];
1813                 j++;
1814         }
1815         num_msrs_to_save = j;
1816 }
1817
1818 /*
1819  * Adapt set_msr() to msr_io()'s calling convention
1820  */
1821 static int do_set_msr(struct kvm_vcpu *vcpu, unsigned index, u64 *data)
1822 {
1823         return set_msr(vcpu, index, *data);
1824 }
1825
1826 /*
1827  * Read or write a bunch of msrs. All parameters are kernel addresses.
1828  *
1829  * @return number of msrs set successfully.
1830  */
1831 static int __msr_io(struct kvm *kvm, struct kvm_msrs *msrs,
1832                     struct kvm_msr_entry *entries,
1833                     int (*do_msr)(struct kvm_vcpu *vcpu,
1834                                   unsigned index, u64 *data))
1835 {
1836         struct kvm_vcpu *vcpu;
1837         int i;
1838
1839         if (!valid_vcpu(msrs->vcpu))
1840                 return -EINVAL;
1841
1842         vcpu = vcpu_load(kvm, msrs->vcpu);
1843         if (!vcpu)
1844                 return -ENOENT;
1845
1846         for (i = 0; i < msrs->nmsrs; ++i)
1847                 if (do_msr(vcpu, entries[i].index, &entries[i].data))
1848                         break;
1849
1850         vcpu_put(vcpu);
1851
1852         return i;
1853 }
1854
1855 /*
1856  * Read or write a bunch of msrs. Parameters are user addresses.
1857  *
1858  * @return number of msrs set successfully.
1859  */
1860 static int msr_io(struct kvm *kvm, struct kvm_msrs __user *user_msrs,
1861                   int (*do_msr)(struct kvm_vcpu *vcpu,
1862                                 unsigned index, u64 *data),
1863                   int writeback)
1864 {
1865         struct kvm_msrs msrs;
1866         struct kvm_msr_entry *entries;
1867         int r, n;
1868         unsigned size;
1869
1870         r = -EFAULT;
1871         if (copy_from_user(&msrs, user_msrs, sizeof msrs))
1872                 goto out;
1873
1874         r = -E2BIG;
1875         if (msrs.nmsrs >= MAX_IO_MSRS)
1876                 goto out;
1877
1878         r = -ENOMEM;
1879         size = sizeof(struct kvm_msr_entry) * msrs.nmsrs;
1880         entries = vmalloc(size);
1881         if (!entries)
1882                 goto out;
1883
1884         r = -EFAULT;
1885         if (copy_from_user(entries, user_msrs->entries, size))
1886                 goto out_free;
1887
1888         r = n = __msr_io(kvm, &msrs, entries, do_msr);
1889         if (r < 0)
1890                 goto out_free;
1891
1892         r = -EFAULT;
1893         if (writeback && copy_to_user(user_msrs->entries, entries, size))
1894                 goto out_free;
1895
1896         r = n;
1897
1898 out_free:
1899         vfree(entries);
1900 out:
1901         return r;
1902 }
1903
1904 /*
1905  * Translate a guest virtual address to a guest physical address.
1906  */
1907 static int kvm_vm_ioctl_translate(struct kvm *kvm, struct kvm_translation *tr)
1908 {
1909         unsigned long vaddr = tr->linear_address;
1910         struct kvm_vcpu *vcpu;
1911         gpa_t gpa;
1912
1913         vcpu = vcpu_load(kvm, tr->vcpu);
1914         if (!vcpu)
1915                 return -ENOENT;
1916         spin_lock(&kvm->lock);
1917         gpa = vcpu->mmu.gva_to_gpa(vcpu, vaddr);
1918         tr->physical_address = gpa;
1919         tr->valid = gpa != UNMAPPED_GVA;
1920         tr->writeable = 1;
1921         tr->usermode = 0;
1922         spin_unlock(&kvm->lock);
1923         vcpu_put(vcpu);
1924
1925         return 0;
1926 }
1927
1928 static int kvm_vm_ioctl_interrupt(struct kvm *kvm, struct kvm_interrupt *irq)
1929 {
1930         struct kvm_vcpu *vcpu;
1931
1932         if (!valid_vcpu(irq->vcpu))
1933                 return -EINVAL;
1934         if (irq->irq < 0 || irq->irq >= 256)
1935                 return -EINVAL;
1936         vcpu = vcpu_load(kvm, irq->vcpu);
1937         if (!vcpu)
1938                 return -ENOENT;
1939
1940         set_bit(irq->irq, vcpu->irq_pending);
1941         set_bit(irq->irq / BITS_PER_LONG, &vcpu->irq_summary);
1942
1943         vcpu_put(vcpu);
1944
1945         return 0;
1946 }
1947
1948 static int kvm_vm_ioctl_debug_guest(struct kvm *kvm,
1949                                      struct kvm_debug_guest *dbg)
1950 {
1951         struct kvm_vcpu *vcpu;
1952         int r;
1953
1954         if (!valid_vcpu(dbg->vcpu))
1955                 return -EINVAL;
1956         vcpu = vcpu_load(kvm, dbg->vcpu);
1957         if (!vcpu)
1958                 return -ENOENT;
1959
1960         r = kvm_arch_ops->set_guest_debug(vcpu, dbg);
1961
1962         vcpu_put(vcpu);
1963
1964         return r;
1965 }
1966
1967 static long kvm_vm_ioctl(struct file *filp,
1968                          unsigned int ioctl, unsigned long arg)
1969 {
1970         struct kvm *kvm = filp->private_data;
1971         void __user *argp = (void __user *)arg;
1972         int r = -EINVAL;
1973
1974         switch (ioctl) {
1975         case KVM_CREATE_VCPU:
1976                 r = kvm_vm_ioctl_create_vcpu(kvm, arg);
1977                 if (r)
1978                         goto out;
1979                 break;
1980         case KVM_RUN: {
1981                 struct kvm_run kvm_run;
1982
1983                 r = -EFAULT;
1984                 if (copy_from_user(&kvm_run, argp, sizeof kvm_run))
1985                         goto out;
1986                 r = kvm_vm_ioctl_run(kvm, &kvm_run);
1987                 if (r < 0 &&  r != -EINTR)
1988                         goto out;
1989                 if (copy_to_user(argp, &kvm_run, sizeof kvm_run)) {
1990                         r = -EFAULT;
1991                         goto out;
1992                 }
1993                 break;
1994         }
1995         case KVM_GET_REGS: {
1996                 struct kvm_regs kvm_regs;
1997
1998                 r = -EFAULT;
1999                 if (copy_from_user(&kvm_regs, argp, sizeof kvm_regs))
2000                         goto out;
2001                 r = kvm_vm_ioctl_get_regs(kvm, &kvm_regs);
2002                 if (r)
2003                         goto out;
2004                 r = -EFAULT;
2005                 if (copy_to_user(argp, &kvm_regs, sizeof kvm_regs))
2006                         goto out;
2007                 r = 0;
2008                 break;
2009         }
2010         case KVM_SET_REGS: {
2011                 struct kvm_regs kvm_regs;
2012
2013                 r = -EFAULT;
2014                 if (copy_from_user(&kvm_regs, argp, sizeof kvm_regs))
2015                         goto out;
2016                 r = kvm_vm_ioctl_set_regs(kvm, &kvm_regs);
2017                 if (r)
2018                         goto out;
2019                 r = 0;
2020                 break;
2021         }
2022         case KVM_GET_SREGS: {
2023                 struct kvm_sregs kvm_sregs;
2024
2025                 r = -EFAULT;
2026                 if (copy_from_user(&kvm_sregs, argp, sizeof kvm_sregs))
2027                         goto out;
2028                 r = kvm_vm_ioctl_get_sregs(kvm, &kvm_sregs);
2029                 if (r)
2030                         goto out;
2031                 r = -EFAULT;
2032                 if (copy_to_user(argp, &kvm_sregs, sizeof kvm_sregs))
2033                         goto out;
2034                 r = 0;
2035                 break;
2036         }
2037         case KVM_SET_SREGS: {
2038                 struct kvm_sregs kvm_sregs;
2039
2040                 r = -EFAULT;
2041                 if (copy_from_user(&kvm_sregs, argp, sizeof kvm_sregs))
2042                         goto out;
2043                 r = kvm_vm_ioctl_set_sregs(kvm, &kvm_sregs);
2044                 if (r)
2045                         goto out;
2046                 r = 0;
2047                 break;
2048         }
2049         case KVM_TRANSLATE: {
2050                 struct kvm_translation tr;
2051
2052                 r = -EFAULT;
2053                 if (copy_from_user(&tr, argp, sizeof tr))
2054                         goto out;
2055                 r = kvm_vm_ioctl_translate(kvm, &tr);
2056                 if (r)
2057                         goto out;
2058                 r = -EFAULT;
2059                 if (copy_to_user(argp, &tr, sizeof tr))
2060                         goto out;
2061                 r = 0;
2062                 break;
2063         }
2064         case KVM_INTERRUPT: {
2065                 struct kvm_interrupt irq;
2066
2067                 r = -EFAULT;
2068                 if (copy_from_user(&irq, argp, sizeof irq))
2069                         goto out;
2070                 r = kvm_vm_ioctl_interrupt(kvm, &irq);
2071                 if (r)
2072                         goto out;
2073                 r = 0;
2074                 break;
2075         }
2076         case KVM_DEBUG_GUEST: {
2077                 struct kvm_debug_guest dbg;
2078
2079                 r = -EFAULT;
2080                 if (copy_from_user(&dbg, argp, sizeof dbg))
2081                         goto out;
2082                 r = kvm_vm_ioctl_debug_guest(kvm, &dbg);
2083                 if (r)
2084                         goto out;
2085                 r = 0;
2086                 break;
2087         }
2088         case KVM_SET_MEMORY_REGION: {
2089                 struct kvm_memory_region kvm_mem;
2090
2091                 r = -EFAULT;
2092                 if (copy_from_user(&kvm_mem, argp, sizeof kvm_mem))
2093                         goto out;
2094                 r = kvm_vm_ioctl_set_memory_region(kvm, &kvm_mem);
2095                 if (r)
2096                         goto out;
2097                 break;
2098         }
2099         case KVM_GET_DIRTY_LOG: {
2100                 struct kvm_dirty_log log;
2101
2102                 r = -EFAULT;
2103                 if (copy_from_user(&log, argp, sizeof log))
2104                         goto out;
2105                 r = kvm_vm_ioctl_get_dirty_log(kvm, &log);
2106                 if (r)
2107                         goto out;
2108                 break;
2109         }
2110         case KVM_GET_MSRS:
2111                 r = msr_io(kvm, argp, get_msr, 1);
2112                 break;
2113         case KVM_SET_MSRS:
2114                 r = msr_io(kvm, argp, do_set_msr, 0);
2115                 break;
2116         default:
2117                 ;
2118         }
2119 out:
2120         return r;
2121 }
2122
2123 static struct page *kvm_vm_nopage(struct vm_area_struct *vma,
2124                                   unsigned long address,
2125                                   int *type)
2126 {
2127         struct kvm *kvm = vma->vm_file->private_data;
2128         unsigned long pgoff;
2129         struct kvm_memory_slot *slot;
2130         struct page *page;
2131
2132         *type = VM_FAULT_MINOR;
2133         pgoff = ((address - vma->vm_start) >> PAGE_SHIFT) + vma->vm_pgoff;
2134         slot = gfn_to_memslot(kvm, pgoff);
2135         if (!slot)
2136                 return NOPAGE_SIGBUS;
2137         page = gfn_to_page(slot, pgoff);
2138         if (!page)
2139                 return NOPAGE_SIGBUS;
2140         get_page(page);
2141         return page;
2142 }
2143
2144 static struct vm_operations_struct kvm_vm_vm_ops = {
2145         .nopage = kvm_vm_nopage,
2146 };
2147
2148 static int kvm_vm_mmap(struct file *file, struct vm_area_struct *vma)
2149 {
2150         vma->vm_ops = &kvm_vm_vm_ops;
2151         return 0;
2152 }
2153
2154 static struct file_operations kvm_vm_fops = {
2155         .release        = kvm_vm_release,
2156         .unlocked_ioctl = kvm_vm_ioctl,
2157         .compat_ioctl   = kvm_vm_ioctl,
2158         .mmap           = kvm_vm_mmap,
2159 };
2160
2161 static int kvm_dev_ioctl_create_vm(void)
2162 {
2163         int fd, r;
2164         struct inode *inode;
2165         struct file *file;
2166         struct kvm *kvm;
2167
2168         inode = kvmfs_inode(&kvm_vm_fops);
2169         if (IS_ERR(inode)) {
2170                 r = PTR_ERR(inode);
2171                 goto out1;
2172         }
2173
2174         kvm = kvm_create_vm();
2175         if (IS_ERR(kvm)) {
2176                 r = PTR_ERR(kvm);
2177                 goto out2;
2178         }
2179
2180         file = kvmfs_file(inode, kvm);
2181         if (IS_ERR(file)) {
2182                 r = PTR_ERR(file);
2183                 goto out3;
2184         }
2185
2186         r = get_unused_fd();
2187         if (r < 0)
2188                 goto out4;
2189         fd = r;
2190         fd_install(fd, file);
2191
2192         return fd;
2193
2194 out4:
2195         fput(file);
2196 out3:
2197         kvm_destroy_vm(kvm);
2198 out2:
2199         iput(inode);
2200 out1:
2201         return r;
2202 }
2203
2204 static long kvm_dev_ioctl(struct file *filp,
2205                           unsigned int ioctl, unsigned long arg)
2206 {
2207         void __user *argp = (void __user *)arg;
2208         int r = -EINVAL;
2209
2210         switch (ioctl) {
2211         case KVM_GET_API_VERSION:
2212                 r = KVM_API_VERSION;
2213                 break;
2214         case KVM_CREATE_VM:
2215                 r = kvm_dev_ioctl_create_vm();
2216                 break;
2217         case KVM_GET_MSR_INDEX_LIST: {
2218                 struct kvm_msr_list __user *user_msr_list = argp;
2219                 struct kvm_msr_list msr_list;
2220                 unsigned n;
2221
2222                 r = -EFAULT;
2223                 if (copy_from_user(&msr_list, user_msr_list, sizeof msr_list))
2224                         goto out;
2225                 n = msr_list.nmsrs;
2226                 msr_list.nmsrs = num_msrs_to_save + ARRAY_SIZE(emulated_msrs);
2227                 if (copy_to_user(user_msr_list, &msr_list, sizeof msr_list))
2228                         goto out;
2229                 r = -E2BIG;
2230                 if (n < num_msrs_to_save)
2231                         goto out;
2232                 r = -EFAULT;
2233                 if (copy_to_user(user_msr_list->indices, &msrs_to_save,
2234                                  num_msrs_to_save * sizeof(u32)))
2235                         goto out;
2236                 if (copy_to_user(user_msr_list->indices
2237                                  + num_msrs_to_save * sizeof(u32),
2238                                  &emulated_msrs,
2239                                  ARRAY_SIZE(emulated_msrs) * sizeof(u32)))
2240                         goto out;
2241                 r = 0;
2242                 break;
2243         }
2244         default:
2245                 ;
2246         }
2247 out:
2248         return r;
2249 }
2250
2251 static struct file_operations kvm_chardev_ops = {
2252         .open           = kvm_dev_open,
2253         .release        = kvm_dev_release,
2254         .unlocked_ioctl = kvm_dev_ioctl,
2255         .compat_ioctl   = kvm_dev_ioctl,
2256 };
2257
2258 static struct miscdevice kvm_dev = {
2259         MISC_DYNAMIC_MINOR,
2260         "kvm",
2261         &kvm_chardev_ops,
2262 };
2263
2264 static int kvm_reboot(struct notifier_block *notifier, unsigned long val,
2265                        void *v)
2266 {
2267         if (val == SYS_RESTART) {
2268                 /*
2269                  * Some (well, at least mine) BIOSes hang on reboot if
2270                  * in vmx root mode.
2271                  */
2272                 printk(KERN_INFO "kvm: exiting hardware virtualization\n");
2273                 on_each_cpu(kvm_arch_ops->hardware_disable, NULL, 0, 1);
2274         }
2275         return NOTIFY_OK;
2276 }
2277
2278 static struct notifier_block kvm_reboot_notifier = {
2279         .notifier_call = kvm_reboot,
2280         .priority = 0,
2281 };
2282
2283 /*
2284  * Make sure that a cpu that is being hot-unplugged does not have any vcpus
2285  * cached on it.
2286  */
2287 static void decache_vcpus_on_cpu(int cpu)
2288 {
2289         struct kvm *vm;
2290         struct kvm_vcpu *vcpu;
2291         int i;
2292
2293         spin_lock(&kvm_lock);
2294         list_for_each_entry(vm, &vm_list, vm_list)
2295                 for (i = 0; i < KVM_MAX_VCPUS; ++i) {
2296                         vcpu = &vm->vcpus[i];
2297                         /*
2298                          * If the vcpu is locked, then it is running on some
2299                          * other cpu and therefore it is not cached on the
2300                          * cpu in question.
2301                          *
2302                          * If it's not locked, check the last cpu it executed
2303                          * on.
2304                          */
2305                         if (mutex_trylock(&vcpu->mutex)) {
2306                                 if (vcpu->cpu == cpu) {
2307                                         kvm_arch_ops->vcpu_decache(vcpu);
2308                                         vcpu->cpu = -1;
2309                                 }
2310                                 mutex_unlock(&vcpu->mutex);
2311                         }
2312                 }
2313         spin_unlock(&kvm_lock);
2314 }
2315
2316 static int kvm_cpu_hotplug(struct notifier_block *notifier, unsigned long val,
2317                            void *v)
2318 {
2319         int cpu = (long)v;
2320
2321         switch (val) {
2322         case CPU_DOWN_PREPARE:
2323         case CPU_UP_CANCELED:
2324                 printk(KERN_INFO "kvm: disabling virtualization on CPU%d\n",
2325                        cpu);
2326                 decache_vcpus_on_cpu(cpu);
2327                 smp_call_function_single(cpu, kvm_arch_ops->hardware_disable,
2328                                          NULL, 0, 1);
2329                 break;
2330         case CPU_ONLINE:
2331                 printk(KERN_INFO "kvm: enabling virtualization on CPU%d\n",
2332                        cpu);
2333                 smp_call_function_single(cpu, kvm_arch_ops->hardware_enable,
2334                                          NULL, 0, 1);
2335                 break;
2336         }
2337         return NOTIFY_OK;
2338 }
2339
2340 static struct notifier_block kvm_cpu_notifier = {
2341         .notifier_call = kvm_cpu_hotplug,
2342         .priority = 20, /* must be > scheduler priority */
2343 };
2344
2345 static __init void kvm_init_debug(void)
2346 {
2347         struct kvm_stats_debugfs_item *p;
2348
2349         debugfs_dir = debugfs_create_dir("kvm", NULL);
2350         for (p = debugfs_entries; p->name; ++p)
2351                 p->dentry = debugfs_create_u32(p->name, 0444, debugfs_dir,
2352                                                p->data);
2353 }
2354
2355 static void kvm_exit_debug(void)
2356 {
2357         struct kvm_stats_debugfs_item *p;
2358
2359         for (p = debugfs_entries; p->name; ++p)
2360                 debugfs_remove(p->dentry);
2361         debugfs_remove(debugfs_dir);
2362 }
2363
2364 static int kvm_suspend(struct sys_device *dev, pm_message_t state)
2365 {
2366         decache_vcpus_on_cpu(raw_smp_processor_id());
2367         on_each_cpu(kvm_arch_ops->hardware_disable, NULL, 0, 1);
2368         return 0;
2369 }
2370
2371 static int kvm_resume(struct sys_device *dev)
2372 {
2373         on_each_cpu(kvm_arch_ops->hardware_enable, NULL, 0, 1);
2374         return 0;
2375 }
2376
2377 static struct sysdev_class kvm_sysdev_class = {
2378         set_kset_name("kvm"),
2379         .suspend = kvm_suspend,
2380         .resume = kvm_resume,
2381 };
2382
2383 static struct sys_device kvm_sysdev = {
2384         .id = 0,
2385         .cls = &kvm_sysdev_class,
2386 };
2387
2388 hpa_t bad_page_address;
2389
2390 static int kvmfs_get_sb(struct file_system_type *fs_type, int flags,
2391                         const char *dev_name, void *data, struct vfsmount *mnt)
2392 {
2393         return get_sb_pseudo(fs_type, "kvm:", NULL, KVMFS_MAGIC, mnt);
2394 }
2395
2396 static struct file_system_type kvm_fs_type = {
2397         .name           = "kvmfs",
2398         .get_sb         = kvmfs_get_sb,
2399         .kill_sb        = kill_anon_super,
2400 };
2401
2402 int kvm_init_arch(struct kvm_arch_ops *ops, struct module *module)
2403 {
2404         int r;
2405
2406         if (kvm_arch_ops) {
2407                 printk(KERN_ERR "kvm: already loaded the other module\n");
2408                 return -EEXIST;
2409         }
2410
2411         if (!ops->cpu_has_kvm_support()) {
2412                 printk(KERN_ERR "kvm: no hardware support\n");
2413                 return -EOPNOTSUPP;
2414         }
2415         if (ops->disabled_by_bios()) {
2416                 printk(KERN_ERR "kvm: disabled by bios\n");
2417                 return -EOPNOTSUPP;
2418         }
2419
2420         kvm_arch_ops = ops;
2421
2422         r = kvm_arch_ops->hardware_setup();
2423         if (r < 0)
2424             return r;
2425
2426         on_each_cpu(kvm_arch_ops->hardware_enable, NULL, 0, 1);
2427         r = register_cpu_notifier(&kvm_cpu_notifier);
2428         if (r)
2429                 goto out_free_1;
2430         register_reboot_notifier(&kvm_reboot_notifier);
2431
2432         r = sysdev_class_register(&kvm_sysdev_class);
2433         if (r)
2434                 goto out_free_2;
2435
2436         r = sysdev_register(&kvm_sysdev);
2437         if (r)
2438                 goto out_free_3;
2439
2440         kvm_chardev_ops.owner = module;
2441
2442         r = misc_register(&kvm_dev);
2443         if (r) {
2444                 printk (KERN_ERR "kvm: misc device register failed\n");
2445                 goto out_free;
2446         }
2447
2448         return r;
2449
2450 out_free:
2451         sysdev_unregister(&kvm_sysdev);
2452 out_free_3:
2453         sysdev_class_unregister(&kvm_sysdev_class);
2454 out_free_2:
2455         unregister_reboot_notifier(&kvm_reboot_notifier);
2456         unregister_cpu_notifier(&kvm_cpu_notifier);
2457 out_free_1:
2458         on_each_cpu(kvm_arch_ops->hardware_disable, NULL, 0, 1);
2459         kvm_arch_ops->hardware_unsetup();
2460         return r;
2461 }
2462
2463 void kvm_exit_arch(void)
2464 {
2465         misc_deregister(&kvm_dev);
2466         sysdev_unregister(&kvm_sysdev);
2467         sysdev_class_unregister(&kvm_sysdev_class);
2468         unregister_reboot_notifier(&kvm_reboot_notifier);
2469         unregister_cpu_notifier(&kvm_cpu_notifier);
2470         on_each_cpu(kvm_arch_ops->hardware_disable, NULL, 0, 1);
2471         kvm_arch_ops->hardware_unsetup();
2472         kvm_arch_ops = NULL;
2473 }
2474
2475 static __init int kvm_init(void)
2476 {
2477         static struct page *bad_page;
2478         int r;
2479
2480         r = register_filesystem(&kvm_fs_type);
2481         if (r)
2482                 goto out3;
2483
2484         kvmfs_mnt = kern_mount(&kvm_fs_type);
2485         r = PTR_ERR(kvmfs_mnt);
2486         if (IS_ERR(kvmfs_mnt))
2487                 goto out2;
2488         kvm_init_debug();
2489
2490         kvm_init_msr_list();
2491
2492         if ((bad_page = alloc_page(GFP_KERNEL)) == NULL) {
2493                 r = -ENOMEM;
2494                 goto out;
2495         }
2496
2497         bad_page_address = page_to_pfn(bad_page) << PAGE_SHIFT;
2498         memset(__va(bad_page_address), 0, PAGE_SIZE);
2499
2500         return r;
2501
2502 out:
2503         kvm_exit_debug();
2504         mntput(kvmfs_mnt);
2505 out2:
2506         unregister_filesystem(&kvm_fs_type);
2507 out3:
2508         return r;
2509 }
2510
2511 static __exit void kvm_exit(void)
2512 {
2513         kvm_exit_debug();
2514         __free_page(pfn_to_page(bad_page_address >> PAGE_SHIFT));
2515         mntput(kvmfs_mnt);
2516         unregister_filesystem(&kvm_fs_type);
2517 }
2518
2519 module_init(kvm_init)
2520 module_exit(kvm_exit)
2521
2522 EXPORT_SYMBOL_GPL(kvm_init_arch);
2523 EXPORT_SYMBOL_GPL(kvm_exit_arch);