746479cdcb
Just like all other architectures we should use out-of-line find bit operations, since the inline variant bloat the size of the kernel image. And also like all other architecures we should only supply optimized variants of the __ffs, ffs, etc. primitives. Therefore this patch removes the inlined s390 find bit functions and uses the generic out-of-line variants instead. The optimization of the primitives follows with the next patch. With this patch also the functions find_first_bit_left() and find_next_bit_left() have been reimplemented, since logically, they are nothing else but a find_first_bit()/find_next_bit() implementation that use an inverted __fls() instead of __ffs(). Also the restriction that these functions only work on machines which support the "flogr" instruction is gone now. This reduces the size of the kernel image (defconfig, -march=z9-109) by 144,482 bytes. Alone the size of the function build_sched_domains() gets reduced from 7 KB to 3,5 KB. We also git rid of unused functions like find_first_bit_le()... Signed-off-by: Heiko Carstens <heiko.carstens@de.ibm.com> Signed-off-by: Martin Schwidefsky <schwidefsky@de.ibm.com>
324 lines
8.2 KiB
C
324 lines
8.2 KiB
C
/*
|
|
* Copyright IBM Corp. 1999,2013
|
|
*
|
|
* Author(s): Martin Schwidefsky <schwidefsky@de.ibm.com>,
|
|
*
|
|
* The description below was taken in large parts from the powerpc
|
|
* bitops header file:
|
|
* Within a word, bits are numbered LSB first. Lot's of places make
|
|
* this assumption by directly testing bits with (val & (1<<nr)).
|
|
* This can cause confusion for large (> 1 word) bitmaps on a
|
|
* big-endian system because, unlike little endian, the number of each
|
|
* bit depends on the word size.
|
|
*
|
|
* The bitop functions are defined to work on unsigned longs, so for an
|
|
* s390x system the bits end up numbered:
|
|
* |63..............0|127............64|191...........128|255...........196|
|
|
* and on s390:
|
|
* |31.....0|63....31|95....64|127...96|159..128|191..160|223..192|255..224|
|
|
*
|
|
* There are a few little-endian macros used mostly for filesystem
|
|
* bitmaps, these work on similar bit arrays layouts, but
|
|
* byte-oriented:
|
|
* |7...0|15...8|23...16|31...24|39...32|47...40|55...48|63...56|
|
|
*
|
|
* The main difference is that bit 3-5 (64b) or 3-4 (32b) in the bit
|
|
* number field needs to be reversed compared to the big-endian bit
|
|
* fields. This can be achieved by XOR with 0x38 (64b) or 0x18 (32b).
|
|
*
|
|
* We also have special functions which work with an MSB0 encoding:
|
|
* on an s390x system the bits are numbered:
|
|
* |0..............63|64............127|128...........191|192...........255|
|
|
* and on s390:
|
|
* |0.....31|31....63|64....95|96...127|128..159|160..191|192..223|224..255|
|
|
*
|
|
* The main difference is that bit 0-63 (64b) or 0-31 (32b) in the bit
|
|
* number field needs to be reversed compared to the LSB0 encoded bit
|
|
* fields. This can be achieved by XOR with 0x3f (64b) or 0x1f (32b).
|
|
*
|
|
*/
|
|
|
|
#ifndef _S390_BITOPS_H
|
|
#define _S390_BITOPS_H
|
|
|
|
#ifndef _LINUX_BITOPS_H
|
|
#error only <linux/bitops.h> can be included directly
|
|
#endif
|
|
|
|
#include <linux/typecheck.h>
|
|
#include <linux/compiler.h>
|
|
|
|
#ifndef CONFIG_64BIT
|
|
|
|
#define __BITOPS_OR "or"
|
|
#define __BITOPS_AND "nr"
|
|
#define __BITOPS_XOR "xr"
|
|
|
|
#define __BITOPS_LOOP(__addr, __val, __op_string) \
|
|
({ \
|
|
unsigned long __old, __new; \
|
|
\
|
|
typecheck(unsigned long *, (__addr)); \
|
|
asm volatile( \
|
|
" l %0,%2\n" \
|
|
"0: lr %1,%0\n" \
|
|
__op_string " %1,%3\n" \
|
|
" cs %0,%1,%2\n" \
|
|
" jl 0b" \
|
|
: "=&d" (__old), "=&d" (__new), "+Q" (*(__addr))\
|
|
: "d" (__val) \
|
|
: "cc"); \
|
|
__old; \
|
|
})
|
|
|
|
#else /* CONFIG_64BIT */
|
|
|
|
#ifdef CONFIG_HAVE_MARCH_Z196_FEATURES
|
|
|
|
#define __BITOPS_OR "laog"
|
|
#define __BITOPS_AND "lang"
|
|
#define __BITOPS_XOR "laxg"
|
|
|
|
#define __BITOPS_LOOP(__addr, __val, __op_string) \
|
|
({ \
|
|
unsigned long __old; \
|
|
\
|
|
typecheck(unsigned long *, (__addr)); \
|
|
asm volatile( \
|
|
__op_string " %0,%2,%1\n" \
|
|
: "=d" (__old), "+Q" (*(__addr)) \
|
|
: "d" (__val) \
|
|
: "cc"); \
|
|
__old; \
|
|
})
|
|
|
|
#else /* CONFIG_HAVE_MARCH_Z196_FEATURES */
|
|
|
|
#define __BITOPS_OR "ogr"
|
|
#define __BITOPS_AND "ngr"
|
|
#define __BITOPS_XOR "xgr"
|
|
|
|
#define __BITOPS_LOOP(__addr, __val, __op_string) \
|
|
({ \
|
|
unsigned long __old, __new; \
|
|
\
|
|
typecheck(unsigned long *, (__addr)); \
|
|
asm volatile( \
|
|
" lg %0,%2\n" \
|
|
"0: lgr %1,%0\n" \
|
|
__op_string " %1,%3\n" \
|
|
" csg %0,%1,%2\n" \
|
|
" jl 0b" \
|
|
: "=&d" (__old), "=&d" (__new), "+Q" (*(__addr))\
|
|
: "d" (__val) \
|
|
: "cc"); \
|
|
__old; \
|
|
})
|
|
|
|
#endif /* CONFIG_HAVE_MARCH_Z196_FEATURES */
|
|
|
|
#endif /* CONFIG_64BIT */
|
|
|
|
#define __BITOPS_WORDS(bits) (((bits) + BITS_PER_LONG - 1) / BITS_PER_LONG)
|
|
|
|
static inline unsigned long *
|
|
__bitops_word(unsigned long nr, volatile unsigned long *ptr)
|
|
{
|
|
unsigned long addr;
|
|
|
|
addr = (unsigned long)ptr + ((nr ^ (nr & (BITS_PER_LONG - 1))) >> 3);
|
|
return (unsigned long *)addr;
|
|
}
|
|
|
|
static inline unsigned char *
|
|
__bitops_byte(unsigned long nr, volatile unsigned long *ptr)
|
|
{
|
|
return ((unsigned char *)ptr) + ((nr ^ (BITS_PER_LONG - 8)) >> 3);
|
|
}
|
|
|
|
static inline void set_bit(unsigned long nr, volatile unsigned long *ptr)
|
|
{
|
|
unsigned long *addr = __bitops_word(nr, ptr);
|
|
unsigned long mask;
|
|
|
|
#ifdef CONFIG_HAVE_MARCH_ZEC12_FEATURES
|
|
if (__builtin_constant_p(nr)) {
|
|
unsigned char *caddr = __bitops_byte(nr, ptr);
|
|
|
|
asm volatile(
|
|
"oi %0,%b1\n"
|
|
: "+Q" (*caddr)
|
|
: "i" (1 << (nr & 7))
|
|
: "cc");
|
|
return;
|
|
}
|
|
#endif
|
|
mask = 1UL << (nr & (BITS_PER_LONG - 1));
|
|
__BITOPS_LOOP(addr, mask, __BITOPS_OR);
|
|
}
|
|
|
|
static inline void clear_bit(unsigned long nr, volatile unsigned long *ptr)
|
|
{
|
|
unsigned long *addr = __bitops_word(nr, ptr);
|
|
unsigned long mask;
|
|
|
|
#ifdef CONFIG_HAVE_MARCH_ZEC12_FEATURES
|
|
if (__builtin_constant_p(nr)) {
|
|
unsigned char *caddr = __bitops_byte(nr, ptr);
|
|
|
|
asm volatile(
|
|
"ni %0,%b1\n"
|
|
: "+Q" (*caddr)
|
|
: "i" (~(1 << (nr & 7)))
|
|
: "cc");
|
|
return;
|
|
}
|
|
#endif
|
|
mask = ~(1UL << (nr & (BITS_PER_LONG - 1)));
|
|
__BITOPS_LOOP(addr, mask, __BITOPS_AND);
|
|
}
|
|
|
|
static inline void change_bit(unsigned long nr, volatile unsigned long *ptr)
|
|
{
|
|
unsigned long *addr = __bitops_word(nr, ptr);
|
|
unsigned long mask;
|
|
|
|
#ifdef CONFIG_HAVE_MARCH_ZEC12_FEATURES
|
|
if (__builtin_constant_p(nr)) {
|
|
unsigned char *caddr = __bitops_byte(nr, ptr);
|
|
|
|
asm volatile(
|
|
"xi %0,%b1\n"
|
|
: "+Q" (*caddr)
|
|
: "i" (1 << (nr & 7))
|
|
: "cc");
|
|
return;
|
|
}
|
|
#endif
|
|
mask = 1UL << (nr & (BITS_PER_LONG - 1));
|
|
__BITOPS_LOOP(addr, mask, __BITOPS_XOR);
|
|
}
|
|
|
|
static inline int
|
|
test_and_set_bit(unsigned long nr, volatile unsigned long *ptr)
|
|
{
|
|
unsigned long *addr = __bitops_word(nr, ptr);
|
|
unsigned long old, mask;
|
|
|
|
mask = 1UL << (nr & (BITS_PER_LONG - 1));
|
|
old = __BITOPS_LOOP(addr, mask, __BITOPS_OR);
|
|
barrier();
|
|
return (old & mask) != 0;
|
|
}
|
|
|
|
static inline int
|
|
test_and_clear_bit(unsigned long nr, volatile unsigned long *ptr)
|
|
{
|
|
unsigned long *addr = __bitops_word(nr, ptr);
|
|
unsigned long old, mask;
|
|
|
|
mask = ~(1UL << (nr & (BITS_PER_LONG - 1)));
|
|
old = __BITOPS_LOOP(addr, mask, __BITOPS_AND);
|
|
barrier();
|
|
return (old & ~mask) != 0;
|
|
}
|
|
|
|
static inline int
|
|
test_and_change_bit(unsigned long nr, volatile unsigned long *ptr)
|
|
{
|
|
unsigned long *addr = __bitops_word(nr, ptr);
|
|
unsigned long old, mask;
|
|
|
|
mask = 1UL << (nr & (BITS_PER_LONG - 1));
|
|
old = __BITOPS_LOOP(addr, mask, __BITOPS_XOR);
|
|
barrier();
|
|
return (old & mask) != 0;
|
|
}
|
|
|
|
static inline void __set_bit(unsigned long nr, volatile unsigned long *ptr)
|
|
{
|
|
unsigned char *addr = __bitops_byte(nr, ptr);
|
|
|
|
*addr |= 1 << (nr & 7);
|
|
}
|
|
|
|
static inline void
|
|
__clear_bit(unsigned long nr, volatile unsigned long *ptr)
|
|
{
|
|
unsigned char *addr = __bitops_byte(nr, ptr);
|
|
|
|
*addr &= ~(1 << (nr & 7));
|
|
}
|
|
|
|
static inline void __change_bit(unsigned long nr, volatile unsigned long *ptr)
|
|
{
|
|
unsigned char *addr = __bitops_byte(nr, ptr);
|
|
|
|
*addr ^= 1 << (nr & 7);
|
|
}
|
|
|
|
static inline int
|
|
__test_and_set_bit(unsigned long nr, volatile unsigned long *ptr)
|
|
{
|
|
unsigned char *addr = __bitops_byte(nr, ptr);
|
|
unsigned char ch;
|
|
|
|
ch = *addr;
|
|
*addr |= 1 << (nr & 7);
|
|
return (ch >> (nr & 7)) & 1;
|
|
}
|
|
|
|
static inline int
|
|
__test_and_clear_bit(unsigned long nr, volatile unsigned long *ptr)
|
|
{
|
|
unsigned char *addr = __bitops_byte(nr, ptr);
|
|
unsigned char ch;
|
|
|
|
ch = *addr;
|
|
*addr &= ~(1 << (nr & 7));
|
|
return (ch >> (nr & 7)) & 1;
|
|
}
|
|
|
|
static inline int
|
|
__test_and_change_bit(unsigned long nr, volatile unsigned long *ptr)
|
|
{
|
|
unsigned char *addr = __bitops_byte(nr, ptr);
|
|
unsigned char ch;
|
|
|
|
ch = *addr;
|
|
*addr ^= 1 << (nr & 7);
|
|
return (ch >> (nr & 7)) & 1;
|
|
}
|
|
|
|
static inline int test_bit(unsigned long nr, const volatile unsigned long *ptr)
|
|
{
|
|
const volatile unsigned char *addr;
|
|
|
|
addr = ((const volatile unsigned char *)ptr);
|
|
addr += (nr ^ (BITS_PER_LONG - 8)) >> 3;
|
|
return (*addr >> (nr & 7)) & 1;
|
|
}
|
|
|
|
/*
|
|
* ATTENTION:
|
|
* find_first_bit_left() and find_next_bit_left() use MSB0 encoding.
|
|
*/
|
|
unsigned long find_first_bit_left(const unsigned long *addr, unsigned long size);
|
|
unsigned long find_next_bit_left(const unsigned long *addr, unsigned long size,
|
|
unsigned long offset);
|
|
|
|
#include <asm-generic/bitops/__ffs.h>
|
|
#include <asm-generic/bitops/ffs.h>
|
|
#include <asm-generic/bitops/__fls.h>
|
|
#include <asm-generic/bitops/fls.h>
|
|
#include <asm-generic/bitops/fls64.h>
|
|
#include <asm-generic/bitops/ffz.h>
|
|
#include <asm-generic/bitops/find.h>
|
|
#include <asm-generic/bitops/hweight.h>
|
|
#include <asm-generic/bitops/lock.h>
|
|
#include <asm-generic/bitops/sched.h>
|
|
#include <asm-generic/bitops/le.h>
|
|
#include <asm-generic/bitops/ext2-atomic-setbit.h>
|
|
|
|
#endif /* _S390_BITOPS_H */
|