X-Git-Url: http://pilppa.org/gitweb/gitweb.cgi?a=blobdiff_plain;f=drivers%2Fkvm%2Fpaging_tmpl.h;h=211fef83be5da898432d312c73ff34eff887ce0c;hb=41074d07c78b086b18fc2af590caef05dbcca568;hp=a9e687b5c1e932277eb5ced9650e80a80626c6e8;hpb=c4fcc2724628c6548748ec80a90b548fc300e81f;p=linux-2.6-omap-h63xx.git diff --git a/drivers/kvm/paging_tmpl.h b/drivers/kvm/paging_tmpl.h index a9e687b5c1e..211fef83be5 100644 --- a/drivers/kvm/paging_tmpl.h +++ b/drivers/kvm/paging_tmpl.h @@ -34,7 +34,9 @@ #define PT_LEVEL_BITS PT64_LEVEL_BITS #ifdef CONFIG_X86_64 #define PT_MAX_FULL_LEVELS 4 + #define CMPXCHG cmpxchg #else + #define CMPXCHG cmpxchg64 #define PT_MAX_FULL_LEVELS 2 #endif #elif PTTYPE == 32 @@ -48,10 +50,14 @@ #define PT_LEVEL_MASK(level) PT32_LEVEL_MASK(level) #define PT_LEVEL_BITS PT32_LEVEL_BITS #define PT_MAX_FULL_LEVELS 2 + #define CMPXCHG cmpxchg #else #error Invalid PTTYPE value #endif +#define gpte_to_gfn FNAME(gpte_to_gfn) +#define gpte_to_gfn_pde FNAME(gpte_to_gfn_pde) + /* * The guest_walker structure emulates the behavior of the hardware page * table walker. @@ -59,30 +65,53 @@ struct guest_walker { int level; gfn_t table_gfn[PT_MAX_FULL_LEVELS]; - pt_element_t *table; pt_element_t pte; - pt_element_t *ptep; - struct page *page; - int index; - pt_element_t inherited_ar; + unsigned pt_access; + unsigned pte_access; gfn_t gfn; u32 error_code; }; -static void FNAME(update_dirty_bit)(struct kvm_vcpu *vcpu, - int write_fault, - pt_element_t *ptep, - gfn_t table_gfn) +static gfn_t gpte_to_gfn(pt_element_t gpte) { - gpa_t pte_gpa; + return (gpte & PT_BASE_ADDR_MASK) >> PAGE_SHIFT; +} - if (write_fault && !is_dirty_pte(*ptep)) { - mark_page_dirty(vcpu->kvm, table_gfn); - *ptep |= PT_DIRTY_MASK; - pte_gpa = ((gpa_t)table_gfn << PAGE_SHIFT); - pte_gpa += offset_in_page(ptep); - kvm_mmu_pte_write(vcpu, pte_gpa, (u8 *)ptep, sizeof(*ptep)); - } +static gfn_t gpte_to_gfn_pde(pt_element_t gpte) +{ + return (gpte & PT_DIR_BASE_ADDR_MASK) >> PAGE_SHIFT; +} + +static bool FNAME(cmpxchg_gpte)(struct kvm *kvm, + gfn_t table_gfn, unsigned index, + pt_element_t orig_pte, pt_element_t new_pte) +{ + pt_element_t ret; + pt_element_t *table; + struct page *page; + + page = gfn_to_page(kvm, table_gfn); + table = kmap_atomic(page, KM_USER0); + + ret = CMPXCHG(&table[index], orig_pte, new_pte); + + kunmap_atomic(table, KM_USER0); + + kvm_release_page_dirty(page); + + return (ret != orig_pte); +} + +static unsigned FNAME(gpte_access)(struct kvm_vcpu *vcpu, pt_element_t gpte) +{ + unsigned access; + + access = (gpte & (PT_WRITABLE_MASK | PT_USER_MASK)) | ACC_EXEC_MASK; +#if PTTYPE == 64 + if (is_nx(vcpu)) + access &= ~(gpte >> PT64_NX_SHIFT); +#endif + return access; } /* @@ -92,107 +121,101 @@ static int FNAME(walk_addr)(struct guest_walker *walker, struct kvm_vcpu *vcpu, gva_t addr, int write_fault, int user_fault, int fetch_fault) { - hpa_t hpa; - struct kvm_memory_slot *slot; - pt_element_t *ptep; - pt_element_t root; + pt_element_t pte; gfn_t table_gfn; + unsigned index, pt_access, pte_access; + gpa_t pte_gpa; pgprintk("%s: addr %lx\n", __FUNCTION__, addr); +walk: walker->level = vcpu->mmu.root_level; - walker->table = NULL; - walker->page = NULL; - walker->ptep = NULL; - root = vcpu->cr3; + pte = vcpu->cr3; #if PTTYPE == 64 if (!is_long_mode(vcpu)) { - walker->ptep = &vcpu->pdptrs[(addr >> 30) & 3]; - root = *walker->ptep; - walker->pte = root; - if (!(root & PT_PRESENT_MASK)) + pte = vcpu->pdptrs[(addr >> 30) & 3]; + if (!is_present_pte(pte)) goto not_present; --walker->level; } #endif - table_gfn = (root & PT64_BASE_ADDR_MASK) >> PAGE_SHIFT; - walker->table_gfn[walker->level - 1] = table_gfn; - pgprintk("%s: table_gfn[%d] %lx\n", __FUNCTION__, - walker->level - 1, table_gfn); - slot = gfn_to_memslot(vcpu->kvm, table_gfn); - hpa = safe_gpa_to_hpa(vcpu->kvm, root & PT64_BASE_ADDR_MASK); - walker->page = pfn_to_page(hpa >> PAGE_SHIFT); - walker->table = kmap_atomic(walker->page, KM_USER0); - ASSERT((!is_long_mode(vcpu) && is_pae(vcpu)) || (vcpu->cr3 & CR3_NONPAE_RESERVED_BITS) == 0); - walker->inherited_ar = PT_USER_MASK | PT_WRITABLE_MASK; + pt_access = ACC_ALL; for (;;) { - int index = PT_INDEX(addr, walker->level); - hpa_t paddr; + index = PT_INDEX(addr, walker->level); - ptep = &walker->table[index]; - walker->index = index; - ASSERT(((unsigned long)walker->table & PAGE_MASK) == - ((unsigned long)ptep & PAGE_MASK)); + table_gfn = gpte_to_gfn(pte); + pte_gpa = gfn_to_gpa(table_gfn); + pte_gpa += index * sizeof(pt_element_t); + walker->table_gfn[walker->level - 1] = table_gfn; + pgprintk("%s: table_gfn[%d] %lx\n", __FUNCTION__, + walker->level - 1, table_gfn); + + kvm_read_guest(vcpu->kvm, pte_gpa, &pte, sizeof(pte)); - if (!is_present_pte(*ptep)) + if (!is_present_pte(pte)) goto not_present; - if (write_fault && !is_writeble_pte(*ptep)) + if (write_fault && !is_writeble_pte(pte)) if (user_fault || is_write_protection(vcpu)) goto access_error; - if (user_fault && !(*ptep & PT_USER_MASK)) + if (user_fault && !(pte & PT_USER_MASK)) goto access_error; #if PTTYPE == 64 - if (fetch_fault && is_nx(vcpu) && (*ptep & PT64_NX_MASK)) + if (fetch_fault && is_nx(vcpu) && (pte & PT64_NX_MASK)) goto access_error; #endif - if (!(*ptep & PT_ACCESSED_MASK)) { + if (!(pte & PT_ACCESSED_MASK)) { mark_page_dirty(vcpu->kvm, table_gfn); - *ptep |= PT_ACCESSED_MASK; + if (FNAME(cmpxchg_gpte)(vcpu->kvm, table_gfn, + index, pte, pte|PT_ACCESSED_MASK)) + goto walk; + pte |= PT_ACCESSED_MASK; } + pte_access = pt_access & FNAME(gpte_access)(vcpu, pte); + if (walker->level == PT_PAGE_TABLE_LEVEL) { - walker->gfn = (*ptep & PT_BASE_ADDR_MASK) - >> PAGE_SHIFT; - FNAME(update_dirty_bit)(vcpu, write_fault, ptep, - table_gfn); + walker->gfn = gpte_to_gfn(pte); break; } if (walker->level == PT_DIRECTORY_LEVEL - && (*ptep & PT_PAGE_SIZE_MASK) + && (pte & PT_PAGE_SIZE_MASK) && (PTTYPE == 64 || is_pse(vcpu))) { - walker->gfn = (*ptep & PT_DIR_BASE_ADDR_MASK) - >> PAGE_SHIFT; + walker->gfn = gpte_to_gfn_pde(pte); walker->gfn += PT_INDEX(addr, PT_PAGE_TABLE_LEVEL); - FNAME(update_dirty_bit)(vcpu, write_fault, ptep, - table_gfn); + if (PTTYPE == 32 && is_cpuid_PSE36()) + walker->gfn += pse36_gfn_delta(pte); break; } - walker->inherited_ar &= walker->table[index]; - table_gfn = (*ptep & PT_BASE_ADDR_MASK) >> PAGE_SHIFT; - kunmap_atomic(walker->table, KM_USER0); - paddr = safe_gpa_to_hpa(vcpu->kvm, table_gfn << PAGE_SHIFT); - walker->page = pfn_to_page(paddr >> PAGE_SHIFT); - walker->table = kmap_atomic(walker->page, KM_USER0); + pt_access = pte_access; --walker->level; - walker->table_gfn[walker->level - 1] = table_gfn; - pgprintk("%s: table_gfn[%d] %lx\n", __FUNCTION__, - walker->level - 1, table_gfn); } - walker->pte = *ptep; - if (walker->page) - walker->ptep = NULL; - if (walker->table) - kunmap_atomic(walker->table, KM_USER0); - pgprintk("%s: pte %llx\n", __FUNCTION__, (u64)*ptep); + + if (write_fault && !is_dirty_pte(pte)) { + bool ret; + + mark_page_dirty(vcpu->kvm, table_gfn); + ret = FNAME(cmpxchg_gpte)(vcpu->kvm, table_gfn, index, pte, + pte|PT_DIRTY_MASK); + if (ret) + goto walk; + pte |= PT_DIRTY_MASK; + kvm_mmu_pte_write(vcpu, pte_gpa, (u8 *)&pte, sizeof(pte)); + } + + walker->pte = pte; + walker->pt_access = pt_access; + walker->pte_access = pte_access; + pgprintk("%s: pte %llx pte_access %x pt_access %x\n", + __FUNCTION__, (u64)pte, pt_access, pte_access); return 1; not_present: @@ -209,30 +232,24 @@ err: walker->error_code |= PFERR_USER_MASK; if (fetch_fault) walker->error_code |= PFERR_FETCH_MASK; - if (walker->table) - kunmap_atomic(walker->table, KM_USER0); return 0; } -static void FNAME(set_pte_common)(struct kvm_vcpu *vcpu, - u64 *shadow_pte, - gpa_t gaddr, - pt_element_t gpte, - u64 access_bits, - int user_fault, - int write_fault, - int *ptwrite, - struct guest_walker *walker, - gfn_t gfn) +static void FNAME(set_pte)(struct kvm_vcpu *vcpu, pt_element_t gpte, + u64 *shadow_pte, unsigned pt_access, + unsigned pte_access, + int user_fault, int write_fault, + int *ptwrite, struct guest_walker *walker, + gfn_t gfn) { - hpa_t paddr; int dirty = gpte & PT_DIRTY_MASK; u64 spte; int was_rmapped = is_rmap_pte(*shadow_pte); + struct page *page; - pgprintk("%s: spte %llx gpte %llx access %llx write_fault %d" + pgprintk("%s: spte %llx gpte %llx access %x write_fault %d" " user_fault %d gfn %lx\n", - __FUNCTION__, *shadow_pte, (u64)gpte, access_bits, + __FUNCTION__, *shadow_pte, (u64)gpte, pt_access, write_fault, user_fault, gfn); /* @@ -243,23 +260,26 @@ static void FNAME(set_pte_common)(struct kvm_vcpu *vcpu, spte = PT_PRESENT_MASK | PT_DIRTY_MASK; spte |= gpte & PT64_NX_MASK; if (!dirty) - access_bits &= ~PT_WRITABLE_MASK; + pte_access &= ~ACC_WRITE_MASK; + if (!(pte_access & ACC_EXEC_MASK)) + spte |= PT64_NX_MASK; - paddr = gpa_to_hpa(vcpu->kvm, gaddr & PT64_BASE_ADDR_MASK); + page = gfn_to_page(vcpu->kvm, gfn); spte |= PT_PRESENT_MASK; - if (access_bits & PT_USER_MASK) + if (pte_access & ACC_USER_MASK) spte |= PT_USER_MASK; - if (is_error_hpa(paddr)) { + if (is_error_page(page)) { set_shadow_pte(shadow_pte, shadow_trap_nonpresent_pte | PT_SHADOW_IO_MARK); + kvm_release_page_clean(page); return; } - spte |= paddr; + spte |= page_to_phys(page); - if ((access_bits & PT_WRITABLE_MASK) + if ((pte_access & ACC_WRITE_MASK) || (write_fault && !is_write_protection(vcpu) && !user_fault)) { struct kvm_mmu_page *shadow; @@ -273,7 +293,7 @@ static void FNAME(set_pte_common)(struct kvm_vcpu *vcpu, if (shadow) { pgprintk("%s: found shadow page for %lx, marking ro\n", __FUNCTION__, gfn); - access_bits &= ~PT_WRITABLE_MASK; + pte_access &= ~ACC_WRITE_MASK; if (is_writeble_pte(spte)) { spte &= ~PT_WRITABLE_MASK; kvm_x86_ops->tlb_flush(vcpu); @@ -285,35 +305,29 @@ static void FNAME(set_pte_common)(struct kvm_vcpu *vcpu, unshadowed: - if (access_bits & PT_WRITABLE_MASK) - mark_page_dirty(vcpu->kvm, gaddr >> PAGE_SHIFT); + if (pte_access & ACC_WRITE_MASK) + mark_page_dirty(vcpu->kvm, gfn); pgprintk("%s: setting spte %llx\n", __FUNCTION__, spte); set_shadow_pte(shadow_pte, spte); - page_header_update_slot(vcpu->kvm, shadow_pte, gaddr); - if (!was_rmapped) - rmap_add(vcpu, shadow_pte, (gaddr & PT64_BASE_ADDR_MASK) - >> PAGE_SHIFT); + page_header_update_slot(vcpu->kvm, shadow_pte, gfn); + if (!was_rmapped) { + rmap_add(vcpu, shadow_pte, gfn); + if (!is_rmap_pte(*shadow_pte)) + kvm_release_page_clean(page); + } + else + kvm_release_page_clean(page); if (!ptwrite || !*ptwrite) vcpu->last_pte_updated = shadow_pte; } -static void FNAME(set_pte)(struct kvm_vcpu *vcpu, pt_element_t gpte, - u64 *shadow_pte, u64 access_bits, - int user_fault, int write_fault, int *ptwrite, - struct guest_walker *walker, gfn_t gfn) -{ - access_bits &= gpte; - FNAME(set_pte_common)(vcpu, shadow_pte, gpte & PT_BASE_ADDR_MASK, - gpte, access_bits, user_fault, write_fault, - ptwrite, walker, gfn); -} - static void FNAME(update_pte)(struct kvm_vcpu *vcpu, struct kvm_mmu_page *page, u64 *spte, const void *pte, int bytes, int offset_in_pte) { pt_element_t gpte; + unsigned pte_access; gpte = *(const pt_element_t *)pte; if (~gpte & (PT_PRESENT_MASK | PT_ACCESSED_MASK)) { @@ -324,26 +338,9 @@ static void FNAME(update_pte)(struct kvm_vcpu *vcpu, struct kvm_mmu_page *page, if (bytes < sizeof(pt_element_t)) return; pgprintk("%s: gpte %llx spte %p\n", __FUNCTION__, (u64)gpte, spte); - FNAME(set_pte)(vcpu, gpte, spte, PT_USER_MASK | PT_WRITABLE_MASK, 0, - 0, NULL, NULL, - (gpte & PT_BASE_ADDR_MASK) >> PAGE_SHIFT); -} - -static void FNAME(set_pde)(struct kvm_vcpu *vcpu, pt_element_t gpde, - u64 *shadow_pte, u64 access_bits, - int user_fault, int write_fault, int *ptwrite, - struct guest_walker *walker, gfn_t gfn) -{ - gpa_t gaddr; - - access_bits &= gpde; - gaddr = (gpa_t)gfn << PAGE_SHIFT; - if (PTTYPE == 32 && is_cpuid_PSE36()) - gaddr |= (gpde & PT32_DIR_PSE36_MASK) << - (32 - PT32_DIR_PSE36_SHIFT); - FNAME(set_pte_common)(vcpu, shadow_pte, gaddr, - gpde, access_bits, user_fault, write_fault, - ptwrite, walker, gfn); + pte_access = page->role.access & FNAME(gpte_access)(vcpu, gpte); + FNAME(set_pte)(vcpu, gpte, spte, page->role.access, pte_access, + 0, 0, NULL, NULL, gpte_to_gfn(gpte)); } /* @@ -356,7 +353,7 @@ static u64 *FNAME(fetch)(struct kvm_vcpu *vcpu, gva_t addr, hpa_t shadow_addr; int level; u64 *shadow_ent; - u64 *prev_shadow_ent = NULL; + unsigned access = walker->pt_access; if (!is_present_pte(walker->pte)) return NULL; @@ -375,14 +372,12 @@ static u64 *FNAME(fetch)(struct kvm_vcpu *vcpu, gva_t addr, u64 shadow_pte; int metaphysical; gfn_t table_gfn; - unsigned hugepage_access = 0; shadow_ent = ((u64 *)__va(shadow_addr)) + index; if (is_shadow_present_pte(*shadow_ent)) { if (level == PT_PAGE_TABLE_LEVEL) break; shadow_addr = *shadow_ent & PT64_BASE_ADDR_MASK; - prev_shadow_ent = shadow_ent; continue; } @@ -392,39 +387,27 @@ static u64 *FNAME(fetch)(struct kvm_vcpu *vcpu, gva_t addr, if (level - 1 == PT_PAGE_TABLE_LEVEL && walker->level == PT_DIRECTORY_LEVEL) { metaphysical = 1; - hugepage_access = walker->pte; - hugepage_access &= PT_USER_MASK | PT_WRITABLE_MASK; if (!is_dirty_pte(walker->pte)) - hugepage_access &= ~PT_WRITABLE_MASK; - hugepage_access >>= PT_WRITABLE_SHIFT; - if (walker->pte & PT64_NX_MASK) - hugepage_access |= (1 << 2); - table_gfn = (walker->pte & PT_BASE_ADDR_MASK) - >> PAGE_SHIFT; + access &= ~ACC_WRITE_MASK; + table_gfn = gpte_to_gfn(walker->pte); } else { metaphysical = 0; table_gfn = walker->table_gfn[level - 2]; } shadow_page = kvm_mmu_get_page(vcpu, table_gfn, addr, level-1, - metaphysical, hugepage_access, + metaphysical, access, shadow_ent); shadow_addr = __pa(shadow_page->spt); shadow_pte = shadow_addr | PT_PRESENT_MASK | PT_ACCESSED_MASK | PT_WRITABLE_MASK | PT_USER_MASK; *shadow_ent = shadow_pte; - prev_shadow_ent = shadow_ent; } - if (walker->level == PT_DIRECTORY_LEVEL) { - FNAME(set_pde)(vcpu, walker->pte, shadow_ent, - walker->inherited_ar, user_fault, write_fault, - ptwrite, walker, walker->gfn); - } else { - ASSERT(walker->level == PT_PAGE_TABLE_LEVEL); - FNAME(set_pte)(vcpu, walker->pte, shadow_ent, - walker->inherited_ar, user_fault, write_fault, - ptwrite, walker, walker->gfn); - } + FNAME(set_pte)(vcpu, walker->pte, shadow_ent, + access, walker->pte_access & access, + user_fault, write_fault, + ptwrite, walker, walker->gfn); + return shadow_ent; } @@ -505,7 +488,7 @@ static gpa_t FNAME(gva_to_gpa)(struct kvm_vcpu *vcpu, gva_t vaddr) r = FNAME(walk_addr)(&walker, vcpu, vaddr, 0, 0, 0); if (r) { - gpa = (gpa_t)walker.gfn << PAGE_SHIFT; + gpa = gfn_to_gpa(walker.gfn); gpa |= vaddr & ~PAGE_MASK; } @@ -515,21 +498,27 @@ static gpa_t FNAME(gva_to_gpa)(struct kvm_vcpu *vcpu, gva_t vaddr) static void FNAME(prefetch_page)(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp) { - int i; + int i, offset = 0; pt_element_t *gpt; + struct page *page; - if (sp->role.metaphysical || PTTYPE == 32) { + if (sp->role.metaphysical + || (PTTYPE == 32 && sp->role.level > PT_PAGE_TABLE_LEVEL)) { nonpaging_prefetch_page(vcpu, sp); return; } - gpt = kmap_atomic(gfn_to_page(vcpu->kvm, sp->gfn), KM_USER0); + if (PTTYPE == 32) + offset = sp->role.quadrant << PT64_LEVEL_BITS; + page = gfn_to_page(vcpu->kvm, sp->gfn); + gpt = kmap_atomic(page, KM_USER0); for (i = 0; i < PT64_ENT_PER_PAGE; ++i) - if (is_present_pte(gpt[i])) + if (is_present_pte(gpt[offset + i])) sp->spt[i] = shadow_trap_nonpresent_pte; else sp->spt[i] = shadow_notrap_nonpresent_pte; kunmap_atomic(gpt, KM_USER0); + kvm_release_page_clean(page); } #undef pt_element_t @@ -542,3 +531,6 @@ static void FNAME(prefetch_page)(struct kvm_vcpu *vcpu, #undef PT_DIR_BASE_ADDR_MASK #undef PT_LEVEL_BITS #undef PT_MAX_FULL_LEVELS +#undef gpte_to_gfn +#undef gpte_to_gfn_pde +#undef CMPXCHG