1 #ifndef _ASM_X86_BITOPS_H
2 #define _ASM_X86_BITOPS_H
5 * Copyright 1992, Linus Torvalds.
8 #ifndef _LINUX_BITOPS_H
9 #error only <linux/bitops.h> can be included directly
12 #include <linux/compiler.h>
13 #include <asm/alternative.h>
16 * These have to be done with inline assembly: that way the bit-setting
17 * is guaranteed to be atomic. All bit operations return 0 if the bit
18 * was cleared before the operation and != 0 if it was not.
20 * bit 0 is the LSB of addr; bit 32 is the LSB of (addr+1).
23 #if __GNUC__ < 4 || (__GNUC__ == 4 && __GNUC_MINOR__ < 1)
24 /* Technically wrong, but this avoids compilation errors on some gcc
26 #define BITOP_ADDR(x) "=m" (*(volatile long *) (x))
28 #define BITOP_ADDR(x) "+m" (*(volatile long *) (x))
31 #define ADDR BITOP_ADDR(addr)
34 * We do the locked ops that don't return the old value as
35 * a mask operation on a byte.
37 #define IS_IMMEDIATE(nr) \
38 (__builtin_constant_p(nr))
39 #define CONST_MASK_ADDR BITOP_ADDR(addr + (nr>>3))
40 #define CONST_MASK (1 << (nr & 7))
43 * set_bit - Atomically set a bit in memory
45 * @addr: the address to start counting from
47 * This function is atomic and may not be reordered. See __set_bit()
48 * if you do not require the atomic guarantees.
50 * Note: there are no guarantees that this function will not be reordered
51 * on non x86 architectures, so if you are writing portable code,
52 * make sure not to rely on its reordering guarantees.
54 * Note that @nr may be almost arbitrarily large; this function is not
55 * restricted to acting on a single-word quantity.
57 static inline void set_bit(unsigned int nr, volatile unsigned long *addr)
60 asm volatile(LOCK_PREFIX "orb %1,%0" : CONST_MASK_ADDR : "i" (CONST_MASK) : "memory");
62 asm volatile(LOCK_PREFIX "bts %1,%0" : ADDR : "Ir" (nr) : "memory");
67 * __set_bit - Set a bit in memory
69 * @addr: the address to start counting from
71 * Unlike set_bit(), this function is non-atomic and may be reordered.
72 * If it's called on the same region of memory simultaneously, the effect
73 * may be that only one operation succeeds.
75 static inline void __set_bit(int nr, volatile unsigned long *addr)
77 asm volatile("bts %1,%0" : ADDR : "Ir" (nr) : "memory");
81 * clear_bit - Clears a bit in memory
83 * @addr: Address to start counting from
85 * clear_bit() is atomic and may not be reordered. However, it does
86 * not contain a memory barrier, so if it is used for locking purposes,
87 * you should call smp_mb__before_clear_bit() and/or smp_mb__after_clear_bit()
88 * in order to ensure changes are visible on other processors.
90 static inline void clear_bit(int nr, volatile unsigned long *addr)
93 asm volatile(LOCK_PREFIX "andb %1,%0" : CONST_MASK_ADDR : "i" (~CONST_MASK));
95 asm volatile(LOCK_PREFIX "btr %1,%0" : ADDR : "Ir" (nr));
99 * clear_bit_unlock - Clears a bit in memory
101 * @addr: Address to start counting from
103 * clear_bit() is atomic and implies release semantics before the memory
104 * operation. It can be used for an unlock.
106 static inline void clear_bit_unlock(unsigned nr, volatile unsigned long *addr)
112 static inline void __clear_bit(int nr, volatile unsigned long *addr)
114 asm volatile("btr %1,%0" : ADDR : "Ir" (nr));
118 * __clear_bit_unlock - Clears a bit in memory
120 * @addr: Address to start counting from
122 * __clear_bit() is non-atomic and implies release semantics before the memory
123 * operation. It can be used for an unlock if no other CPUs can concurrently
124 * modify other bits in the word.
126 * No memory barrier is required here, because x86 cannot reorder stores past
127 * older loads. Same principle as spin_unlock.
129 static inline void __clear_bit_unlock(unsigned nr, volatile unsigned long *addr)
132 __clear_bit(nr, addr);
135 #define smp_mb__before_clear_bit() barrier()
136 #define smp_mb__after_clear_bit() barrier()
139 * __change_bit - Toggle a bit in memory
140 * @nr: the bit to change
141 * @addr: the address to start counting from
143 * Unlike change_bit(), this function is non-atomic and may be reordered.
144 * If it's called on the same region of memory simultaneously, the effect
145 * may be that only one operation succeeds.
147 static inline void __change_bit(int nr, volatile unsigned long *addr)
149 asm volatile("btc %1,%0" : ADDR : "Ir" (nr));
153 * change_bit - Toggle a bit in memory
155 * @addr: Address to start counting from
157 * change_bit() is atomic and may not be reordered.
158 * Note that @nr may be almost arbitrarily large; this function is not
159 * restricted to acting on a single-word quantity.
161 static inline void change_bit(int nr, volatile unsigned long *addr)
163 asm volatile(LOCK_PREFIX "btc %1,%0" : ADDR : "Ir" (nr));
167 * test_and_set_bit - Set a bit and return its old value
169 * @addr: Address to count from
171 * This operation is atomic and cannot be reordered.
172 * It also implies a memory barrier.
174 static inline int test_and_set_bit(int nr, volatile unsigned long *addr)
178 asm volatile(LOCK_PREFIX "bts %2,%1\n\t"
179 "sbb %0,%0" : "=r" (oldbit), ADDR : "Ir" (nr) : "memory");
185 * test_and_set_bit_lock - Set a bit and return its old value for lock
187 * @addr: Address to count from
189 * This is the same as test_and_set_bit on x86.
191 static inline int test_and_set_bit_lock(int nr, volatile unsigned long *addr)
193 return test_and_set_bit(nr, addr);
197 * __test_and_set_bit - Set a bit and return its old value
199 * @addr: Address to count from
201 * This operation is non-atomic and can be reordered.
202 * If two examples of this operation race, one can appear to succeed
203 * but actually fail. You must protect multiple accesses with a lock.
205 static inline int __test_and_set_bit(int nr, volatile unsigned long *addr)
211 : "=r" (oldbit), ADDR
217 * test_and_clear_bit - Clear a bit and return its old value
219 * @addr: Address to count from
221 * This operation is atomic and cannot be reordered.
222 * It also implies a memory barrier.
224 static inline int test_and_clear_bit(int nr, volatile unsigned long *addr)
228 asm volatile(LOCK_PREFIX "btr %2,%1\n\t"
230 : "=r" (oldbit), ADDR : "Ir" (nr) : "memory");
236 * __test_and_clear_bit - Clear a bit and return its old value
238 * @addr: Address to count from
240 * This operation is non-atomic and can be reordered.
241 * If two examples of this operation race, one can appear to succeed
242 * but actually fail. You must protect multiple accesses with a lock.
244 static inline int __test_and_clear_bit(int nr, volatile unsigned long *addr)
248 asm volatile("btr %2,%1\n\t"
250 : "=r" (oldbit), ADDR
255 /* WARNING: non atomic and it can be reordered! */
256 static inline int __test_and_change_bit(int nr, volatile unsigned long *addr)
260 asm volatile("btc %2,%1\n\t"
262 : "=r" (oldbit), ADDR
263 : "Ir" (nr) : "memory");
269 * test_and_change_bit - Change a bit and return its old value
271 * @addr: Address to count from
273 * This operation is atomic and cannot be reordered.
274 * It also implies a memory barrier.
276 static inline int test_and_change_bit(int nr, volatile unsigned long *addr)
280 asm volatile(LOCK_PREFIX "btc %2,%1\n\t"
282 : "=r" (oldbit), ADDR : "Ir" (nr) : "memory");
287 static inline int constant_test_bit(int nr, const volatile unsigned long *addr)
289 return ((1UL << (nr % BITS_PER_LONG)) &
290 (((unsigned long *)addr)[nr / BITS_PER_LONG])) != 0;
293 static inline int variable_test_bit(int nr, volatile const unsigned long *addr)
297 asm volatile("bt %2,%1\n\t"
300 : "m" (*(unsigned long *)addr), "Ir" (nr));
305 #if 0 /* Fool kernel-doc since it doesn't do macros yet */
307 * test_bit - Determine whether a bit is set
308 * @nr: bit number to test
309 * @addr: Address to start counting from
311 static int test_bit(int nr, const volatile unsigned long *addr);
314 #define test_bit(nr, addr) \
315 (__builtin_constant_p((nr)) \
316 ? constant_test_bit((nr), (addr)) \
317 : variable_test_bit((nr), (addr)))
320 * __ffs - find first set bit in word
321 * @word: The word to search
323 * Undefined if no bit exists, so code should check against 0 first.
325 static inline unsigned long __ffs(unsigned long word)
334 * ffz - find first zero bit in word
335 * @word: The word to search
337 * Undefined if no zero exists, so code should check against ~0UL first.
339 static inline unsigned long ffz(unsigned long word)
348 * __fls: find last set bit in word
349 * @word: The word to search
351 * Undefined if no zero exists, so code should check against ~0UL first.
353 static inline unsigned long __fls(unsigned long word)
363 * ffs - find first set bit in word
364 * @x: the word to search
366 * This is defined the same way as the libc and compiler builtin ffs
367 * routines, therefore differs in spirit from the other bitops.
369 * ffs(value) returns 0 if value is 0 or the position of the first
370 * set bit if value is nonzero. The first (least significant) bit
373 static inline int ffs(int x)
376 #ifdef CONFIG_X86_CMOV
379 : "=r" (r) : "rm" (x), "r" (-1));
384 "1:" : "=r" (r) : "rm" (x));
390 * fls - find last set bit in word
391 * @x: the word to search
393 * This is defined in a similar way as the libc and compiler builtin
394 * ffs, but returns the position of the most significant set bit.
396 * fls(value) returns 0 if value is 0 or the position of the last
397 * set bit if value is nonzero. The last (most significant) bit is
400 static inline int fls(int x)
403 #ifdef CONFIG_X86_CMOV
406 : "=&r" (r) : "rm" (x), "rm" (-1));
411 "1:" : "=r" (r) : "rm" (x));
415 #endif /* __KERNEL__ */
419 static inline void set_bit_string(unsigned long *bitmap,
420 unsigned long i, int len)
422 unsigned long end = i + len;
424 __set_bit(i, bitmap);
431 #include <asm-generic/bitops/sched.h>
433 #define ARCH_HAS_FAST_MULTIPLIER 1
435 #include <asm-generic/bitops/hweight.h>
437 #endif /* __KERNEL__ */
439 #include <asm-generic/bitops/fls64.h>
443 #include <asm-generic/bitops/ext2-non-atomic.h>
445 #define ext2_set_bit_atomic(lock, nr, addr) \
446 test_and_set_bit((nr), (unsigned long *)(addr))
447 #define ext2_clear_bit_atomic(lock, nr, addr) \
448 test_and_clear_bit((nr), (unsigned long *)(addr))
450 #include <asm-generic/bitops/minix.h>
452 #endif /* __KERNEL__ */
453 #endif /* _ASM_X86_BITOPS_H */