9248e52fec
Since the non-atomic arch_*() bitops use plain accesses, they are implicitly instrumnted by the compiler, and we work around this in the instrumented wrappers to avoid double instrumentation. It's simpler to avoid the wrappers entirely, and use the preprocessor to alias the arch_*() bitops to their regular versions, removing the need for checks in the instrumented wrappers. Suggested-by: Marco Elver <elver@google.com> Signed-off-by: Mark Rutland <mark.rutland@arm.com> Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org> Reviewed-by: Marco Elver <elver@google.com> Link: https://lore.kernel.org/r/20210721155813.17082-1-mark.rutland@arm.com
123 lines
3.4 KiB
C
123 lines
3.4 KiB
C
/* SPDX-License-Identifier: GPL-2.0 */
|
|
#ifndef _ASM_GENERIC_BITOPS_NON_ATOMIC_H_
|
|
#define _ASM_GENERIC_BITOPS_NON_ATOMIC_H_
|
|
|
|
#include <asm/types.h>
|
|
|
|
/**
|
|
* arch___set_bit - Set a bit in memory
|
|
* @nr: the bit to set
|
|
* @addr: the address to start counting from
|
|
*
|
|
* Unlike set_bit(), this function is non-atomic and may be reordered.
|
|
* If it's called on the same region of memory simultaneously, the effect
|
|
* may be that only one operation succeeds.
|
|
*/
|
|
static __always_inline void
|
|
arch___set_bit(int nr, volatile unsigned long *addr)
|
|
{
|
|
unsigned long mask = BIT_MASK(nr);
|
|
unsigned long *p = ((unsigned long *)addr) + BIT_WORD(nr);
|
|
|
|
*p |= mask;
|
|
}
|
|
#define __set_bit arch___set_bit
|
|
|
|
static __always_inline void
|
|
arch___clear_bit(int nr, volatile unsigned long *addr)
|
|
{
|
|
unsigned long mask = BIT_MASK(nr);
|
|
unsigned long *p = ((unsigned long *)addr) + BIT_WORD(nr);
|
|
|
|
*p &= ~mask;
|
|
}
|
|
#define __clear_bit arch___clear_bit
|
|
|
|
/**
|
|
* arch___change_bit - Toggle a bit in memory
|
|
* @nr: the bit to change
|
|
* @addr: the address to start counting from
|
|
*
|
|
* Unlike change_bit(), this function is non-atomic and may be reordered.
|
|
* If it's called on the same region of memory simultaneously, the effect
|
|
* may be that only one operation succeeds.
|
|
*/
|
|
static __always_inline
|
|
void arch___change_bit(int nr, volatile unsigned long *addr)
|
|
{
|
|
unsigned long mask = BIT_MASK(nr);
|
|
unsigned long *p = ((unsigned long *)addr) + BIT_WORD(nr);
|
|
|
|
*p ^= mask;
|
|
}
|
|
#define __change_bit arch___change_bit
|
|
|
|
/**
|
|
* arch___test_and_set_bit - Set a bit and return its old value
|
|
* @nr: Bit to set
|
|
* @addr: Address to count from
|
|
*
|
|
* This operation is non-atomic and can be reordered.
|
|
* If two examples of this operation race, one can appear to succeed
|
|
* but actually fail. You must protect multiple accesses with a lock.
|
|
*/
|
|
static __always_inline int
|
|
arch___test_and_set_bit(int nr, volatile unsigned long *addr)
|
|
{
|
|
unsigned long mask = BIT_MASK(nr);
|
|
unsigned long *p = ((unsigned long *)addr) + BIT_WORD(nr);
|
|
unsigned long old = *p;
|
|
|
|
*p = old | mask;
|
|
return (old & mask) != 0;
|
|
}
|
|
#define __test_and_set_bit arch___test_and_set_bit
|
|
|
|
/**
|
|
* arch___test_and_clear_bit - Clear a bit and return its old value
|
|
* @nr: Bit to clear
|
|
* @addr: Address to count from
|
|
*
|
|
* This operation is non-atomic and can be reordered.
|
|
* If two examples of this operation race, one can appear to succeed
|
|
* but actually fail. You must protect multiple accesses with a lock.
|
|
*/
|
|
static __always_inline int
|
|
arch___test_and_clear_bit(int nr, volatile unsigned long *addr)
|
|
{
|
|
unsigned long mask = BIT_MASK(nr);
|
|
unsigned long *p = ((unsigned long *)addr) + BIT_WORD(nr);
|
|
unsigned long old = *p;
|
|
|
|
*p = old & ~mask;
|
|
return (old & mask) != 0;
|
|
}
|
|
#define __test_and_clear_bit arch___test_and_clear_bit
|
|
|
|
/* WARNING: non atomic and it can be reordered! */
|
|
static __always_inline int
|
|
arch___test_and_change_bit(int nr, volatile unsigned long *addr)
|
|
{
|
|
unsigned long mask = BIT_MASK(nr);
|
|
unsigned long *p = ((unsigned long *)addr) + BIT_WORD(nr);
|
|
unsigned long old = *p;
|
|
|
|
*p = old ^ mask;
|
|
return (old & mask) != 0;
|
|
}
|
|
#define __test_and_change_bit arch___test_and_change_bit
|
|
|
|
/**
|
|
* arch_test_bit - Determine whether a bit is set
|
|
* @nr: bit number to test
|
|
* @addr: Address to start counting from
|
|
*/
|
|
static __always_inline int
|
|
arch_test_bit(int nr, const volatile unsigned long *addr)
|
|
{
|
|
return 1UL & (addr[BIT_WORD(nr)] >> (nr & (BITS_PER_LONG-1)));
|
|
}
|
|
#define test_bit arch_test_bit
|
|
|
|
#endif /* _ASM_GENERIC_BITOPS_NON_ATOMIC_H_ */
|