Merge branch 'master' of master.kernel.org:/pub/scm/linux/kernel/git/davem/net-2.6

[linux-2.6-omap-h63xx.git] / arch / powerpc / mm / hash_native_64.c

diff --git a/arch/powerpc/mm/hash_native_64.c b/arch/powerpc/mm/hash_native_64.c
index 7b7fe2d7b9dc7c1e882984ab2b8ba17461b70617..6ba9b47e55afbe746734761c672761dcea07e9f0 100644 (file)
--- a/arch/powerpc/mm/hash_native_64.c
+++ b/arch/powerpc/mm/hash_native_64.c
@@ -104,7 +104,7 @@ static inline void tlbie(unsigned long va, int psize, int local)
                spin_unlock(&native_tlbie_lock);
 }
 
-static inline void native_lock_hpte(hpte_t *hptep)
+static inline void native_lock_hpte(struct hash_pte *hptep)
 {
        unsigned long *word = &hptep->v;
 
@@ -116,7 +116,7 @@ static inline void native_lock_hpte(hpte_t *hptep)
        }
 }
 
-static inline void native_unlock_hpte(hpte_t *hptep)
+static inline void native_unlock_hpte(struct hash_pte *hptep)
 {
        unsigned long *word = &hptep->v;
 
@@ -128,7 +128,7 @@ static long native_hpte_insert(unsigned long hpte_group, unsigned long va,
                        unsigned long pa, unsigned long rflags,
                        unsigned long vflags, int psize)
 {
-       hpte_t *hptep = htab_address + hpte_group;
+       struct hash_pte *hptep = htab_address + hpte_group;
        unsigned long hpte_v, hpte_r;
        int i;
 
@@ -163,7 +163,7 @@ static long native_hpte_insert(unsigned long hpte_group, unsigned long va,
 
        hptep->r = hpte_r;
        /* Guarantee the second dword is visible before the valid bit */
-       __asm__ __volatile__ ("eieio" : : : "memory");
+       eieio();
        /*
         * Now set the first dword including the valid bit
         * NOTE: this also unlocks the hpte
@@ -177,7 +177,7 @@ static long native_hpte_insert(unsigned long hpte_group, unsigned long va,
 
 static long native_hpte_remove(unsigned long hpte_group)
 {
-       hpte_t *hptep;
+       struct hash_pte *hptep;
        int i;
        int slot_offset;
        unsigned long hpte_v;
@@ -217,7 +217,7 @@ static long native_hpte_remove(unsigned long hpte_group)
 static long native_hpte_updatepp(unsigned long slot, unsigned long newpp,
                                 unsigned long va, int psize, int local)
 {
-       hpte_t *hptep = htab_address + slot;
+       struct hash_pte *hptep = htab_address + slot;
        unsigned long hpte_v, want_v;
        int ret = 0;
 
@@ -233,15 +233,14 @@ static long native_hpte_updatepp(unsigned long slot, unsigned long newpp,
        /* Even if we miss, we need to invalidate the TLB */
        if (!HPTE_V_COMPARE(hpte_v, want_v) || !(hpte_v & HPTE_V_VALID)) {
                DBG_LOW(" -> miss\n");
-               native_unlock_hpte(hptep);
                ret = -1;
        } else {
                DBG_LOW(" -> hit\n");
                /* Update the HPTE */
                hptep->r = (hptep->r & ~(HPTE_R_PP | HPTE_R_N)) |
                        (newpp & (HPTE_R_PP | HPTE_R_N | HPTE_R_C));
-               native_unlock_hpte(hptep);
        }
+       native_unlock_hpte(hptep);
 
        /* Ensure it is out of the tlb too. */
        tlbie(va, psize, local);
@@ -251,7 +250,7 @@ static long native_hpte_updatepp(unsigned long slot, unsigned long newpp,
 
 static long native_hpte_find(unsigned long va, int psize)
 {
-       hpte_t *hptep;
+       struct hash_pte *hptep;
        unsigned long hash;
        unsigned long i, j;
        long slot;
@@ -294,7 +293,7 @@ static void native_hpte_updateboltedpp(unsigned long newpp, unsigned long ea,
 {
        unsigned long vsid, va;
        long slot;
-       hpte_t *hptep;
+       struct hash_pte *hptep;
 
        vsid = get_kernel_vsid(ea);
        va = (vsid << 28) | (ea & 0x0fffffff);
@@ -315,7 +314,7 @@ static void native_hpte_updateboltedpp(unsigned long newpp, unsigned long ea,
 static void native_hpte_invalidate(unsigned long slot, unsigned long va,
                                   int psize, int local)
 {
-       hpte_t *hptep = htab_address + slot;
+       struct hash_pte *hptep = htab_address + slot;
        unsigned long hpte_v;
        unsigned long want_v;
        unsigned long flags;
@@ -345,13 +344,13 @@ static void native_hpte_invalidate(unsigned long slot, unsigned long va,
 #define LP_BITS                8
 #define LP_MASK(i)     ((0xFF >> (i)) << LP_SHIFT)
 
-static void hpte_decode(hpte_t *hpte, unsigned long slot,
+static void hpte_decode(struct hash_pte *hpte, unsigned long slot,
                        int *psize, unsigned long *va)
 {
        unsigned long hpte_r = hpte->r;
        unsigned long hpte_v = hpte->v;
        unsigned long avpn;
-       int i, size, shift, penc, avpnm_bits;
+       int i, size, shift, penc;
 
        if (!(hpte_v & HPTE_V_LARGE))
                size = MMU_PAGE_4K;
@@ -376,31 +375,28 @@ static void hpte_decode(hpte_t *hpte, unsigned long slot,
                }
        }
 
-       /*
-        * FIXME, the code below works for 16M, 64K, and 4K pages as these
-        * fall under the p<=23 rules for calculating the virtual address.
-        * In the case of 16M pages, an extra bit is stolen from the AVPN
-        * field to achieve the requisite 24 bits.
-        *
-        * Does not work for 16G pages or 1 TB segments.
-        */
+       /* This works for all page sizes, and for 256M and 1T segments */
        shift = mmu_psize_defs[size].shift;
-       if (mmu_psize_defs[size].avpnm)
-               avpnm_bits = __ilog2_u64(mmu_psize_defs[size].avpnm) + 1;
-       else
-               avpnm_bits = 0;
-       if (shift - avpnm_bits <= 23) {
-               avpn = HPTE_V_AVPN_VAL(hpte_v) << 23;
+       avpn = (HPTE_V_AVPN_VAL(hpte_v) & ~mmu_psize_defs[size].avpnm) << 23;
 
-               if (shift < 23) {
-                       unsigned long vpi, pteg;
+       if (shift < 23) {
+               unsigned long vpi, vsid, pteg;
 
-                       pteg = slot / HPTES_PER_GROUP;
-                       if (hpte_v & HPTE_V_SECONDARY)
-                               pteg = ~pteg;
+               pteg = slot / HPTES_PER_GROUP;
+               if (hpte_v & HPTE_V_SECONDARY)
+                       pteg = ~pteg;
+               switch (hpte_v >> HPTE_V_SSIZE_SHIFT) {
+               case MMU_SEGSIZE_256M:
                        vpi = ((avpn >> 28) ^ pteg) & htab_hash_mask;
-                       avpn |= (vpi << mmu_psize_defs[size].shift);
+                       break;
+               case MMU_SEGSIZE_1T:
+                       vsid = avpn >> 40;
+                       vpi = (vsid ^ (vsid << 25) ^ pteg) & htab_hash_mask;
+                       break;
+               default:
+                       avpn = vpi = size = 0;
                }
+               avpn |= (vpi << mmu_psize_defs[size].shift);
        }
 
        *va = avpn;
@@ -418,7 +414,7 @@ static void hpte_decode(hpte_t *hpte, unsigned long slot,
 static void native_hpte_clear(void)
 {
        unsigned long slot, slots, flags;
-       hpte_t *hptep = htab_address;
+       struct hash_pte *hptep = htab_address;
        unsigned long hpte_v, va;
        unsigned long pteg_count;
        int psize;
@@ -465,7 +461,7 @@ static void native_hpte_clear(void)
 static void native_flush_hash_range(unsigned long number, int local)
 {
        unsigned long va, hash, index, hidx, shift, slot;
-       hpte_t *hptep;
+       struct hash_pte *hptep;
        unsigned long hpte_v;
        unsigned long want_v;
        unsigned long flags;