linux/lib/lockref.c
Uros Bizjak 3378323bbb locking/lockref: Use try_cmpxchg64 in CMPXCHG_LOOP macro
Use try_cmpxchg64 instead of cmpxchg64 in CMPXCHG_LOOP macro.
x86 CMPXCHG instruction returns success in ZF flag, so this
change saves a compare after cmpxchg (and related move instruction
in front of cmpxchg). The main loop of lockref_get improves from:

  13:	48 89 c1             	mov    %rax,%rcx
  16:	48 c1 f9 20          	sar    $0x20,%rcx
  1a:	83 c1 01             	add    $0x1,%ecx
  1d:	48 89 ce             	mov    %rcx,%rsi
  20:	89 c1                	mov    %eax,%ecx
  22:	48 89 d0             	mov    %rdx,%rax
  25:	48 c1 e6 20          	shl    $0x20,%rsi
  29:	48 09 f1             	or     %rsi,%rcx
  2c:	f0 48 0f b1 4d 00    	lock cmpxchg %rcx,0x0(%rbp)
  32:	48 39 d0             	cmp    %rdx,%rax
  35:	75 17                	jne    4e <lockref_get+0x4e>

to:

  13:	48 89 ca             	mov    %rcx,%rdx
  16:	48 c1 fa 20          	sar    $0x20,%rdx
  1a:	83 c2 01             	add    $0x1,%edx
  1d:	48 89 d6             	mov    %rdx,%rsi
  20:	89 ca                	mov    %ecx,%edx
  22:	48 c1 e6 20          	shl    $0x20,%rsi
  26:	48 09 f2             	or     %rsi,%rdx
  29:	f0 48 0f b1 55 00    	lock cmpxchg %rdx,0x0(%rbp)
  2f:	75 02                	jne    33 <lockref_get+0x33>

[ Michael Ellerman and Mark Rutland confirm that code generation on
  powerpc and arm64 respectively is also ok, even though they do not
  have a native arch_try_cmpxchg() implementation, and rely on the
  default fallback case    - Linus ]

Signed-off-by: Uros Bizjak <ubizjak@gmail.com>
Tested-by: Michael Ellerman <mpe@ellerman.id.au>
Tested-by: Mark Rutland <mark.rutland@arm.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Waiman.Long@hp.com
Cc: paulmck@linux.vnet.ibm.com
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2022-05-26 09:52:53 -07:00

220 lines
4.4 KiB
C

// SPDX-License-Identifier: GPL-2.0
#include <linux/export.h>
#include <linux/lockref.h>
#if USE_CMPXCHG_LOCKREF
/*
* Note that the "cmpxchg()" reloads the "old" value for the
* failure case.
*/
#define CMPXCHG_LOOP(CODE, SUCCESS) do { \
int retry = 100; \
struct lockref old; \
BUILD_BUG_ON(sizeof(old) != 8); \
old.lock_count = READ_ONCE(lockref->lock_count); \
while (likely(arch_spin_value_unlocked(old.lock.rlock.raw_lock))) { \
struct lockref new = old; \
CODE \
if (likely(try_cmpxchg64_relaxed(&lockref->lock_count, \
&old.lock_count, \
new.lock_count))) { \
SUCCESS; \
} \
if (!--retry) \
break; \
cpu_relax(); \
} \
} while (0)
#else
#define CMPXCHG_LOOP(CODE, SUCCESS) do { } while (0)
#endif
/**
* lockref_get - Increments reference count unconditionally
* @lockref: pointer to lockref structure
*
* This operation is only valid if you already hold a reference
* to the object, so you know the count cannot be zero.
*/
void lockref_get(struct lockref *lockref)
{
CMPXCHG_LOOP(
new.count++;
,
return;
);
spin_lock(&lockref->lock);
lockref->count++;
spin_unlock(&lockref->lock);
}
EXPORT_SYMBOL(lockref_get);
/**
* lockref_get_not_zero - Increments count unless the count is 0 or dead
* @lockref: pointer to lockref structure
* Return: 1 if count updated successfully or 0 if count was zero
*/
int lockref_get_not_zero(struct lockref *lockref)
{
int retval;
CMPXCHG_LOOP(
new.count++;
if (old.count <= 0)
return 0;
,
return 1;
);
spin_lock(&lockref->lock);
retval = 0;
if (lockref->count > 0) {
lockref->count++;
retval = 1;
}
spin_unlock(&lockref->lock);
return retval;
}
EXPORT_SYMBOL(lockref_get_not_zero);
/**
* lockref_put_not_zero - Decrements count unless count <= 1 before decrement
* @lockref: pointer to lockref structure
* Return: 1 if count updated successfully or 0 if count would become zero
*/
int lockref_put_not_zero(struct lockref *lockref)
{
int retval;
CMPXCHG_LOOP(
new.count--;
if (old.count <= 1)
return 0;
,
return 1;
);
spin_lock(&lockref->lock);
retval = 0;
if (lockref->count > 1) {
lockref->count--;
retval = 1;
}
spin_unlock(&lockref->lock);
return retval;
}
EXPORT_SYMBOL(lockref_put_not_zero);
/**
* lockref_get_or_lock - Increments count unless the count is 0 or dead
* @lockref: pointer to lockref structure
* Return: 1 if count updated successfully or 0 if count was zero
* and we got the lock instead.
*/
int lockref_get_or_lock(struct lockref *lockref)
{
CMPXCHG_LOOP(
new.count++;
if (old.count <= 0)
break;
,
return 1;
);
spin_lock(&lockref->lock);
if (lockref->count <= 0)
return 0;
lockref->count++;
spin_unlock(&lockref->lock);
return 1;
}
EXPORT_SYMBOL(lockref_get_or_lock);
/**
* lockref_put_return - Decrement reference count if possible
* @lockref: pointer to lockref structure
*
* Decrement the reference count and return the new value.
* If the lockref was dead or locked, return an error.
*/
int lockref_put_return(struct lockref *lockref)
{
CMPXCHG_LOOP(
new.count--;
if (old.count <= 0)
return -1;
,
return new.count;
);
return -1;
}
EXPORT_SYMBOL(lockref_put_return);
/**
* lockref_put_or_lock - decrements count unless count <= 1 before decrement
* @lockref: pointer to lockref structure
* Return: 1 if count updated successfully or 0 if count <= 1 and lock taken
*/
int lockref_put_or_lock(struct lockref *lockref)
{
CMPXCHG_LOOP(
new.count--;
if (old.count <= 1)
break;
,
return 1;
);
spin_lock(&lockref->lock);
if (lockref->count <= 1)
return 0;
lockref->count--;
spin_unlock(&lockref->lock);
return 1;
}
EXPORT_SYMBOL(lockref_put_or_lock);
/**
* lockref_mark_dead - mark lockref dead
* @lockref: pointer to lockref structure
*/
void lockref_mark_dead(struct lockref *lockref)
{
assert_spin_locked(&lockref->lock);
lockref->count = -128;
}
EXPORT_SYMBOL(lockref_mark_dead);
/**
* lockref_get_not_dead - Increments count unless the ref is dead
* @lockref: pointer to lockref structure
* Return: 1 if count updated successfully or 0 if lockref was dead
*/
int lockref_get_not_dead(struct lockref *lockref)
{
int retval;
CMPXCHG_LOOP(
new.count++;
if (old.count < 0)
return 0;
,
return 1;
);
spin_lock(&lockref->lock);
retval = 0;
if (lockref->count >= 0) {
lockref->count++;
retval = 1;
}
spin_unlock(&lockref->lock);
return retval;
}
EXPORT_SYMBOL(lockref_get_not_dead);