Merge branch 'upstream' of git://ftp.linux-mips.org/pub/scm/upstream-linus

[linux-2.6-omap-h63xx.git] / arch / sparc64 / lib / bitops.S

diff --git a/arch/sparc64/lib/bitops.S b/arch/sparc64/lib/bitops.S
index 886dcd2b376a0e9d1359b35b04da1124db7746b8..6b015a6eefb50017e7d22b2d1edc0637408a1fef 100644 (file)
--- a/arch/sparc64/lib/bitops.S
+++ b/arch/sparc64/lib/bitops.S
@@ -1,29 +1,35 @@
-/* $Id: bitops.S,v 1.3 2001/11/18 00:12:56 davem Exp $
- * bitops.S: Sparc64 atomic bit operations.
+/* bitops.S: Sparc64 atomic bit operations.
  *
- * Copyright (C) 2000 David S. Miller (davem@redhat.com)
+ * Copyright (C) 2000, 2007 David S. Miller (davem@davemloft.net)
  */
 
-#include <linux/config.h>
 #include <asm/asi.h>
+#include <asm/backoff.h>
+
+       .text
 
        /* On SMP we need to use memory barriers to ensure
         * correct memory operation ordering, nop these out
         * for uniprocessor.
         */
+
 #ifdef CONFIG_SMP
 #define BITOP_PRE_BARRIER      membar #StoreLoad | #LoadLoad
-#define BITOP_POST_BARRIER     membar #StoreLoad | #StoreStore
+#define BITOP_POST_BARRIER     \
+       ba,pt   %xcc, 80b;      \
+       membar #StoreLoad | #StoreStore
+
+80:    retl
+        nop
 #else
-#define BITOP_PRE_BARRIER      nop
-#define BITOP_POST_BARRIER     nop
+#define BITOP_PRE_BARRIER
+#define BITOP_POST_BARRIER
 #endif
 
-       .text
-
        .globl  test_and_set_bit
        .type   test_and_set_bit,#function
 test_and_set_bit:      /* %o0=nr, %o1=addr */
+       BACKOFF_SETUP(%o3)
        BITOP_PRE_BARRIER
        srlx    %o0, 6, %g1
        mov     1, %o2
@@ -35,17 +41,20 @@ test_and_set_bit:   /* %o0=nr, %o1=addr */
        or      %g7, %o2, %g1
        casx    [%o1], %g7, %g1
        cmp     %g7, %g1
-       bne,pn  %xcc, 1b
+       bne,pn  %xcc, 2f
         and    %g7, %o2, %g2
-       BITOP_POST_BARRIER
        clr     %o0
+       movrne  %g2, 1, %o0
+       BITOP_POST_BARRIER
        retl
-        movrne %g2, 1, %o0
+        nop
+2:     BACKOFF_SPIN(%o3, %o4, 1b)
        .size   test_and_set_bit, .-test_and_set_bit
 
        .globl  test_and_clear_bit
        .type   test_and_clear_bit,#function
 test_and_clear_bit:    /* %o0=nr, %o1=addr */
+       BACKOFF_SETUP(%o3)
        BITOP_PRE_BARRIER
        srlx    %o0, 6, %g1
        mov     1, %o2
@@ -57,17 +66,20 @@ test_and_clear_bit: /* %o0=nr, %o1=addr */
        andn    %g7, %o2, %g1
        casx    [%o1], %g7, %g1
        cmp     %g7, %g1
-       bne,pn  %xcc, 1b
+       bne,pn  %xcc, 2f
         and    %g7, %o2, %g2
-       BITOP_POST_BARRIER
        clr     %o0
+       movrne  %g2, 1, %o0
+       BITOP_POST_BARRIER
        retl
-        movrne %g2, 1, %o0
+        nop
+2:     BACKOFF_SPIN(%o3, %o4, 1b)
        .size   test_and_clear_bit, .-test_and_clear_bit
 
        .globl  test_and_change_bit
        .type   test_and_change_bit,#function
 test_and_change_bit:   /* %o0=nr, %o1=addr */
+       BACKOFF_SETUP(%o3)
        BITOP_PRE_BARRIER
        srlx    %o0, 6, %g1
        mov     1, %o2
@@ -79,17 +91,20 @@ test_and_change_bit:        /* %o0=nr, %o1=addr */
        xor     %g7, %o2, %g1
        casx    [%o1], %g7, %g1
        cmp     %g7, %g1
-       bne,pn  %xcc, 1b
+       bne,pn  %xcc, 2f
         and    %g7, %o2, %g2
-       BITOP_POST_BARRIER
        clr     %o0
+       movrne  %g2, 1, %o0
+       BITOP_POST_BARRIER
        retl
-        movrne %g2, 1, %o0
+        nop
+2:     BACKOFF_SPIN(%o3, %o4, 1b)
        .size   test_and_change_bit, .-test_and_change_bit
 
        .globl  set_bit
        .type   set_bit,#function
 set_bit:               /* %o0=nr, %o1=addr */
+       BACKOFF_SETUP(%o3)
        srlx    %o0, 6, %g1
        mov     1, %o2
        sllx    %g1, 3, %g3
@@ -100,15 +115,17 @@ set_bit:          /* %o0=nr, %o1=addr */
        or      %g7, %o2, %g1
        casx    [%o1], %g7, %g1
        cmp     %g7, %g1
-       bne,pn  %xcc, 1b
+       bne,pn  %xcc, 2f
         nop
        retl
         nop
+2:     BACKOFF_SPIN(%o3, %o4, 1b)
        .size   set_bit, .-set_bit
 
        .globl  clear_bit
        .type   clear_bit,#function
 clear_bit:             /* %o0=nr, %o1=addr */
+       BACKOFF_SETUP(%o3)
        srlx    %o0, 6, %g1
        mov     1, %o2
        sllx    %g1, 3, %g3
@@ -119,15 +136,17 @@ clear_bit:                /* %o0=nr, %o1=addr */
        andn    %g7, %o2, %g1
        casx    [%o1], %g7, %g1
        cmp     %g7, %g1
-       bne,pn  %xcc, 1b
+       bne,pn  %xcc, 2f
         nop
        retl
         nop
+2:     BACKOFF_SPIN(%o3, %o4, 1b)
        .size   clear_bit, .-clear_bit
 
        .globl  change_bit
        .type   change_bit,#function
 change_bit:            /* %o0=nr, %o1=addr */
+       BACKOFF_SETUP(%o3)
        srlx    %o0, 6, %g1
        mov     1, %o2
        sllx    %g1, 3, %g3
@@ -138,8 +157,9 @@ change_bit:         /* %o0=nr, %o1=addr */
        xor     %g7, %o2, %g1
        casx    [%o1], %g7, %g1
        cmp     %g7, %g1
-       bne,pn  %xcc, 1b
+       bne,pn  %xcc, 2f
         nop
        retl
         nop
+2:     BACKOFF_SPIN(%o3, %o4, 1b)
        .size   change_bit, .-change_bit