x86: revert "defer cr3 reload when doing pud_clear()"

[linux-2.6-omap-h63xx.git] / include / asm-x86 / system.h

diff --git a/include/asm-x86/system.h b/include/asm-x86/system.h
index 6c7d1fda499549245d371a37d8832d2e1fac7ad3..e9c15c97dfe2404b40d2921d546abf9fa8603b24 100644 (file)
--- a/include/asm-x86/system.h
+++ b/include/asm-x86/system.h
@@ -5,6 +5,7 @@
 #include <asm/segment.h>
 #include <asm/cpufeature.h>
 #include <asm/cmpxchg.h>
+#include <asm/nops.h>
 
 #include <linux/kernel.h>
 #include <linux/irqflags.h>
@@ -129,10 +130,7 @@ extern void load_gs_index(unsigned);
                "movl %k1, %%" #seg "\n\t"      \
                "jmp 2b\n"                      \
                ".previous\n"                   \
-               ".section __ex_table,\"a\"\n\t" \
-               _ASM_ALIGN "\n\t"               \
-               _ASM_PTR " 1b,3b\n"             \
-               ".previous"                     \
+               _ASM_EXTABLE(1b,3b)             \
                : :"r" (value), "r" (0))
 
 
@@ -213,12 +211,10 @@ static inline unsigned long native_read_cr4_safe(void)
        /* This could fault if %cr4 does not exist. In x86_64, a cr4 always
         * exists, so it will never fail. */
 #ifdef CONFIG_X86_32
-       asm volatile("1: mov %%cr4, %0          \n"
-               "2:                             \n"
-               ".section __ex_table,\"a\"      \n"
-               ".long 1b,2b                    \n"
-               ".previous                      \n"
-               : "=r" (val), "=m" (__force_order) : "0" (0));
+       asm volatile("1: mov %%cr4, %0\n"
+                    "2:\n"
+                    _ASM_EXTABLE(1b,2b)
+                    : "=r" (val), "=m" (__force_order) : "0" (0));
 #else
        val = native_read_cr4();
 #endif
@@ -230,6 +226,20 @@ static inline void native_write_cr4(unsigned long val)
        asm volatile("mov %0,%%cr4": :"r" (val), "m" (__force_order));
 }
 
+#ifdef CONFIG_X86_64
+static inline unsigned long native_read_cr8(void)
+{
+       unsigned long cr8;
+       asm volatile("movq %%cr8,%0" : "=r" (cr8));
+       return cr8;
+}
+
+static inline void native_write_cr8(unsigned long val)
+{
+       asm volatile("movq %0,%%cr8" :: "r" (val) : "memory");
+}
+#endif
+
 static inline void native_wbinvd(void)
 {
        asm volatile("wbinvd": : :"memory");
@@ -247,21 +257,9 @@ static inline void native_wbinvd(void)
 #define read_cr4_safe()        (native_read_cr4_safe())
 #define write_cr4(x)   (native_write_cr4(x))
 #define wbinvd()       (native_wbinvd())
-
 #ifdef CONFIG_X86_64
-
-static inline unsigned long read_cr8(void)
-{
-       unsigned long cr8;
-       asm volatile("movq %%cr8,%0" : "=r" (cr8));
-       return cr8;
-}
-
-static inline void write_cr8(unsigned long val)
-{
-       asm volatile("movq %0,%%cr8" :: "r" (val) : "memory");
-}
-
+#define read_cr8()     (native_read_cr8())
+#define write_cr8(x)   (native_write_cr8(x))
 #endif
 
 /* Clear the 'TS' bit */
@@ -273,9 +271,9 @@ static inline void write_cr8(unsigned long val)
 
 #endif /* __KERNEL__ */
 
-static inline void clflush(void *__p)
+static inline void clflush(volatile void *__p)
 {
-       asm volatile("clflush %0" : "+m" (*(char __force *)__p));
+       asm volatile("clflush %0" : "+m" (*(volatile char __force *)__p));
 }
 
 #define nop() __asm__ __volatile__ ("nop")
@@ -395,5 +393,17 @@ void default_idle(void);
 #define set_mb(var, value) do { var = value; barrier(); } while (0)
 #endif
 
+/*
+ * Stop RDTSC speculation. This is needed when you need to use RDTSC
+ * (or get_cycles or vread that possibly accesses the TSC) in a defined
+ * code region.
+ *
+ * (Could use an alternative three way for this if there was one.)
+ */
+static inline void rdtsc_barrier(void)
+{
+       alternative(ASM_NOP3, "mfence", X86_FEATURE_MFENCE_RDTSC);
+       alternative(ASM_NOP3, "lfence", X86_FEATURE_LFENCE_RDTSC);
+}
 
 #endif