Merge git://git.kernel.org/pub/scm/linux/kernel/git/x86/linux-2.6-x86

[linux-2.6-omap-h63xx.git] / include / asm-mips / spinlock.h

diff --git a/include/asm-mips/spinlock.h b/include/asm-mips/spinlock.h
index fc3217fc1118deabd9c4c40cb3442cdf1573c2ab..bb897016c491a24e8369d461a50a59a892457154 100644 (file)
--- a/include/asm-mips/spinlock.h
+++ b/include/asm-mips/spinlock.h
@@ -3,7 +3,7 @@
  * License.  See the file "COPYING" in the main directory of this archive
  * for more details.
  *
- * Copyright (C) 1999, 2000, 06 by Ralf Baechle
+ * Copyright (C) 1999, 2000, 06 Ralf Baechle (ralf@linux-mips.org)
  * Copyright (C) 1999, 2000 Silicon Graphics, Inc.
  */
 #ifndef _ASM_SPINLOCK_H
@@ -49,18 +49,25 @@ static inline void __raw_spin_lock(raw_spinlock_t *lock)
                __asm__ __volatile__(
                "       .set    noreorder       # __raw_spin_lock       \n"
                "1:     ll      %1, %2                                  \n"
-               "       bnez    %1, 1b                                  \n"
+               "       bnez    %1, 2f                                  \n"
                "        li     %1, 1                                   \n"
                "       sc      %1, %0                                  \n"
-               "       beqz    %1, 1b                                  \n"
+               "       beqz    %1, 2f                                  \n"
                "        nop                                            \n"
+               "       .subsection 2                                   \n"
+               "2:     ll      %1, %2                                  \n"
+               "       bnez    %1, 2b                                  \n"
+               "        li     %1, 1                                   \n"
+               "       b       1b                                      \n"
+               "        nop                                            \n"
+               "       .previous                                       \n"
                "       .set    reorder                                 \n"
                : "=m" (lock->lock), "=&r" (tmp)
                : "m" (lock->lock)
                : "memory");
        }
 
-       smp_mb();
+       smp_llsc_mb();
 }
 
 static inline void __raw_spin_unlock(raw_spinlock_t *lock)
@@ -99,15 +106,19 @@ static inline unsigned int __raw_spin_trylock(raw_spinlock_t *lock)
                "1:     ll      %0, %3                                  \n"
                "       ori     %2, %0, 1                               \n"
                "       sc      %2, %1                                  \n"
-               "       beqz    %2, 1b                                  \n"
+               "       beqz    %2, 2f                                  \n"
                "        andi   %2, %0, 1                               \n"
+               "       .subsection 2                                   \n"
+               "2:     b       1b                                      \n"
+               "        nop                                            \n"
+               "       .previous                                       \n"
                "       .set    reorder"
                : "=&r" (temp), "=m" (lock->lock), "=&r" (res)
                : "m" (lock->lock)
                : "memory");
        }
 
-       smp_mb();
+       smp_llsc_mb();
 
        return res == 0;
 }
@@ -154,18 +165,25 @@ static inline void __raw_read_lock(raw_rwlock_t *rw)
                __asm__ __volatile__(
                "       .set    noreorder       # __raw_read_lock       \n"
                "1:     ll      %1, %2                                  \n"
-               "       bltz    %1, 1b                                  \n"
+               "       bltz    %1, 2f                                  \n"
                "        addu   %1, 1                                   \n"
                "       sc      %1, %0                                  \n"
                "       beqz    %1, 1b                                  \n"
                "        nop                                            \n"
+               "       .subsection 2                                   \n"
+               "2:     ll      %1, %2                                  \n"
+               "       bltz    %1, 2b                                  \n"
+               "        addu   %1, 1                                   \n"
+               "       b       1b                                      \n"
+               "        nop                                            \n"
+               "       .previous                                       \n"
                "       .set    reorder                                 \n"
                : "=m" (rw->lock), "=&r" (tmp)
                : "m" (rw->lock)
                : "memory");
        }
 
-       smp_mb();
+       smp_llsc_mb();
 }
 
 /* Note the use of sub, not subu which will make the kernel die with an
@@ -175,7 +193,7 @@ static inline void __raw_read_unlock(raw_rwlock_t *rw)
 {
        unsigned int tmp;
 
-       smp_mb();
+       smp_llsc_mb();
 
        if (R10000_LLSC_WAR) {
                __asm__ __volatile__(
@@ -192,8 +210,12 @@ static inline void __raw_read_unlock(raw_rwlock_t *rw)
                "1:     ll      %1, %2                                  \n"
                "       sub     %1, 1                                   \n"
                "       sc      %1, %0                                  \n"
-               "       beqz    %1, 1b                                  \n"
+               "       beqz    %1, 2f                                  \n"
                "        nop                                            \n"
+               "       .subsection 2                                   \n"
+               "2:     b       1b                                      \n"
+               "        nop                                            \n"
+               "       .previous                                       \n"
                "       .set    reorder                                 \n"
                : "=m" (rw->lock), "=&r" (tmp)
                : "m" (rw->lock)
@@ -222,18 +244,25 @@ static inline void __raw_write_lock(raw_rwlock_t *rw)
                __asm__ __volatile__(
                "       .set    noreorder       # __raw_write_lock      \n"
                "1:     ll      %1, %2                                  \n"
-               "       bnez    %1, 1b                                  \n"
+               "       bnez    %1, 2f                                  \n"
                "        lui    %1, 0x8000                              \n"
                "       sc      %1, %0                                  \n"
-               "       beqz    %1, 1b                                  \n"
+               "       beqz    %1, 2f                                  \n"
                "        nop                                            \n"
+               "       .subsection 2                                   \n"
+               "2:     ll      %1, %2                                  \n"
+               "       bnez    %1, 2b                                  \n"
+               "        lui    %1, 0x8000                              \n"
+               "       b       1b                                      \n"
+               "        nop                                            \n"
+               "       .previous                                       \n"
                "       .set    reorder                                 \n"
                : "=m" (rw->lock), "=&r" (tmp)
                : "m" (rw->lock)
                : "memory");
        }
 
-       smp_mb();
+       smp_llsc_mb();
 }
 
 static inline void __raw_write_unlock(raw_rwlock_t *rw)
@@ -258,13 +287,13 @@ static inline int __raw_read_trylock(raw_rwlock_t *rw)
                "       .set    noreorder       # __raw_read_trylock    \n"
                "       li      %2, 0                                   \n"
                "1:     ll      %1, %3                                  \n"
-               "       bnez    %1, 2f                                  \n"
+               "       bltz    %1, 2f                                  \n"
                "        addu   %1, 1                                   \n"
                "       sc      %1, %0                                  \n"
                "       .set    reorder                                 \n"
                "       beqzl   %1, 1b                                  \n"
                "        nop                                            \n"
-               __WEAK_ORDERING_MB
+               __WEAK_LLSC_MB
                "       li      %2, 1                                   \n"
                "2:                                                     \n"
                : "=m" (rw->lock), "=&r" (tmp), "=&r" (ret)
@@ -275,13 +304,13 @@ static inline int __raw_read_trylock(raw_rwlock_t *rw)
                "       .set    noreorder       # __raw_read_trylock    \n"
                "       li      %2, 0                                   \n"
                "1:     ll      %1, %3                                  \n"
-               "       bnez    %1, 2f                                  \n"
+               "       bltz    %1, 2f                                  \n"
                "        addu   %1, 1                                   \n"
                "       sc      %1, %0                                  \n"
                "       beqz    %1, 1b                                  \n"
                "        nop                                            \n"
                "       .set    reorder                                 \n"
-               __WEAK_ORDERING_MB
+               __WEAK_LLSC_MB
                "       li      %2, 1                                   \n"
                "2:                                                     \n"
                : "=m" (rw->lock), "=&r" (tmp), "=&r" (ret)
@@ -307,7 +336,7 @@ static inline int __raw_write_trylock(raw_rwlock_t *rw)
                "       sc      %1, %0                                  \n"
                "       beqzl   %1, 1b                                  \n"
                "        nop                                            \n"
-               __WEAK_ORDERING_MB
+               __WEAK_LLSC_MB
                "       li      %2, 1                                   \n"
                "       .set    reorder                                 \n"
                "2:                                                     \n"
@@ -322,12 +351,15 @@ static inline int __raw_write_trylock(raw_rwlock_t *rw)
                "       bnez    %1, 2f                                  \n"
                "       lui     %1, 0x8000                              \n"
                "       sc      %1, %0                                  \n"
-               "       beqz    %1, 1b                                  \n"
-               "        nop                                            \n"
-               __WEAK_ORDERING_MB
-               "       li      %2, 1                                   \n"
-               "       .set    reorder                                 \n"
+               "       beqz    %1, 3f                                  \n"
+               "        li     %2, 1                                   \n"
                "2:                                                     \n"
+               __WEAK_LLSC_MB
+               "       .subsection 2                                   \n"
+               "3:     b       1b                                      \n"
+               "        li     %2, 0                                   \n"
+               "       .previous                                       \n"
+               "       .set    reorder                                 \n"
                : "=m" (rw->lock), "=&r" (tmp), "=&r" (ret)
                : "m" (rw->lock)
                : "memory");