mirror of
https://github.com/torvalds/linux.git
synced 2024-11-10 14:11:52 +00:00
sched/wait: Split out the wait_bit*() APIs from <linux/wait.h> into <linux/wait_bit.h>
The wait_bit*() types and APIs are mixed into wait.h, but they are a pretty orthogonal extension of wait-queues. Furthermore, only about 50 kernel files use these APIs, while over 1000 use the regular wait-queue functionality. So clean up the main wait.h by moving the wait-bit functionality out of it, into a separate .h and .c file: include/linux/wait_bit.h for types and APIs kernel/sched/wait_bit.c for the implementation Update all header dependencies. This reduces the size of wait.h rather significantly, by about 30%. Cc: Linus Torvalds <torvalds@linux-foundation.org> Cc: Peter Zijlstra <peterz@infradead.org> Cc: Thomas Gleixner <tglx@linutronix.de> Cc: linux-kernel@vger.kernel.org Signed-off-by: Ingo Molnar <mingo@kernel.org>
This commit is contained in:
Ingo Molnar
parent
4b1c480bfa
commit
5dd43ce2f6
@ -18,7 +18,7 @@
|
||||
|
||||
#include <linux/fscache-cache.h>
|
||||
#include <linux/timer.h>
|
||||
#include <linux/wait.h>
|
||||
#include <linux/wait_bit.h>
|
||||
#include <linux/cred.h>
|
||||
#include <linux/workqueue.h>
|
||||
#include <linux/security.h>
|
||||
|
||||
@ -24,6 +24,7 @@
|
||||
#include <linux/pagemap.h>
|
||||
#include <linux/freezer.h>
|
||||
#include <linux/sched/signal.h>
|
||||
#include <linux/wait_bit.h>
|
||||
|
||||
#include <asm/div64.h>
|
||||
#include "cifsfs.h"
|
||||
|
||||
@ -7,6 +7,7 @@
|
||||
#include <linux/security.h>
|
||||
#include <linux/crc32.h>
|
||||
#include <linux/nfs_page.h>
|
||||
#include <linux/wait_bit.h>
|
||||
|
||||
#define NFS_MS_MASK (MS_RDONLY|MS_NOSUID|MS_NODEV|MS_NOEXEC|MS_SYNCHRONOUS)
|
||||
|
||||
|
||||
@ -2,7 +2,7 @@
|
||||
#define _LINUX_FS_H
|
||||
|
||||
#include <linux/linkage.h>
|
||||
#include <linux/wait.h>
|
||||
#include <linux/wait_bit.h>
|
||||
#include <linux/kdev_t.h>
|
||||
#include <linux/dcache.h>
|
||||
#include <linux/path.h>
|
||||
|
||||
@ -13,7 +13,7 @@
|
||||
#include <linux/ktime.h>
|
||||
#include <linux/sunrpc/types.h>
|
||||
#include <linux/spinlock.h>
|
||||
#include <linux/wait.h>
|
||||
#include <linux/wait_bit.h>
|
||||
#include <linux/workqueue.h>
|
||||
#include <linux/sunrpc/xdr.h>
|
||||
|
||||
|
||||
@ -29,18 +29,6 @@ struct wait_queue_entry {
|
||||
struct list_head task_list;
|
||||
};
|
||||
|
||||
struct wait_bit_key {
|
||||
void *flags;
|
||||
int bit_nr;
|
||||
#define WAIT_ATOMIC_T_BIT_NR -1
|
||||
unsigned long timeout;
|
||||
};
|
||||
|
||||
struct wait_bit_queue_entry {
|
||||
struct wait_bit_key key;
|
||||
struct wait_queue_entry wq_entry;
|
||||
};
|
||||
|
||||
struct wait_queue_head {
|
||||
spinlock_t lock;
|
||||
struct list_head task_list;
|
||||
@ -68,12 +56,6 @@ struct task_struct;
|
||||
#define DECLARE_WAIT_QUEUE_HEAD(name) \
|
||||
struct wait_queue_head name = __WAIT_QUEUE_HEAD_INITIALIZER(name)
|
||||
|
||||
#define __WAIT_BIT_KEY_INITIALIZER(word, bit) \
|
||||
{ .flags = word, .bit_nr = bit, }
|
||||
|
||||
#define __WAIT_ATOMIC_T_KEY_INITIALIZER(p) \
|
||||
{ .flags = p, .bit_nr = WAIT_ATOMIC_T_BIT_NR, }
|
||||
|
||||
extern void __init_waitqueue_head(struct wait_queue_head *wq_head, const char *name, struct lock_class_key *);
|
||||
|
||||
#define init_waitqueue_head(wq_head) \
|
||||
@ -200,22 +182,11 @@ __remove_wait_queue(struct wait_queue_head *wq_head, struct wait_queue_entry *wq
|
||||
list_del(&wq_entry->task_list);
|
||||
}
|
||||
|
||||
typedef int wait_bit_action_f(struct wait_bit_key *key, int mode);
|
||||
void __wake_up(struct wait_queue_head *wq_head, unsigned int mode, int nr, void *key);
|
||||
void __wake_up_locked_key(struct wait_queue_head *wq_head, unsigned int mode, void *key);
|
||||
void __wake_up_sync_key(struct wait_queue_head *wq_head, unsigned int mode, int nr, void *key);
|
||||
void __wake_up_locked(struct wait_queue_head *wq_head, unsigned int mode, int nr);
|
||||
void __wake_up_sync(struct wait_queue_head *wq_head, unsigned int mode, int nr);
|
||||
void __wake_up_bit(struct wait_queue_head *wq_head, void *word, int bit);
|
||||
int __wait_on_bit(struct wait_queue_head *wq_head, struct wait_bit_queue_entry *wbq_entry, wait_bit_action_f *action, unsigned int mode);
|
||||
int __wait_on_bit_lock(struct wait_queue_head *wq_head, struct wait_bit_queue_entry *wbq_entry, wait_bit_action_f *action, unsigned int mode);
|
||||
void wake_up_bit(void *word, int bit);
|
||||
void wake_up_atomic_t(atomic_t *p);
|
||||
int out_of_line_wait_on_bit(void *word, int, wait_bit_action_f *action, unsigned int mode);
|
||||
int out_of_line_wait_on_bit_timeout(void *word, int, wait_bit_action_f *action, unsigned int mode, unsigned long timeout);
|
||||
int out_of_line_wait_on_bit_lock(void *word, int, wait_bit_action_f *action, unsigned int mode);
|
||||
int out_of_line_wait_on_atomic_t(atomic_t *p, int (*)(atomic_t *), unsigned int mode);
|
||||
struct wait_queue_head *bit_waitqueue(void *word, int bit);
|
||||
|
||||
#define wake_up(x) __wake_up(x, TASK_NORMAL, 1, NULL)
|
||||
#define wake_up_nr(x, nr) __wake_up(x, TASK_NORMAL, nr, NULL)
|
||||
@ -976,7 +947,6 @@ void finish_wait(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_en
|
||||
long wait_woken(struct wait_queue_entry *wq_entry, unsigned mode, long timeout);
|
||||
int woken_wake_function(struct wait_queue_entry *wq_entry, unsigned mode, int sync, void *key);
|
||||
int autoremove_wake_function(struct wait_queue_entry *wq_entry, unsigned mode, int sync, void *key);
|
||||
int wake_bit_function(struct wait_queue_entry *wq_entry, unsigned mode, int sync, void *key);
|
||||
|
||||
#define DEFINE_WAIT_FUNC(name, function) \
|
||||
struct wait_queue_entry name = { \
|
||||
@ -987,17 +957,6 @@ int wake_bit_function(struct wait_queue_entry *wq_entry, unsigned mode, int sync
|
||||
|
||||
#define DEFINE_WAIT(name) DEFINE_WAIT_FUNC(name, autoremove_wake_function)
|
||||
|
||||
#define DEFINE_WAIT_BIT(name, word, bit) \
|
||||
struct wait_bit_queue_entry name = { \
|
||||
.key = __WAIT_BIT_KEY_INITIALIZER(word, bit), \
|
||||
.wq_entry = { \
|
||||
.private = current, \
|
||||
.func = wake_bit_function, \
|
||||
.task_list = \
|
||||
LIST_HEAD_INIT((name).wq_entry.task_list), \
|
||||
}, \
|
||||
}
|
||||
|
||||
#define init_wait(wait) \
|
||||
do { \
|
||||
(wait)->private = current; \
|
||||
@ -1006,213 +965,4 @@ int wake_bit_function(struct wait_queue_entry *wq_entry, unsigned mode, int sync
|
||||
(wait)->flags = 0; \
|
||||
} while (0)
|
||||
|
||||
|
||||
extern int bit_wait(struct wait_bit_key *key, int bit);
|
||||
extern int bit_wait_io(struct wait_bit_key *key, int bit);
|
||||
extern int bit_wait_timeout(struct wait_bit_key *key, int bit);
|
||||
extern int bit_wait_io_timeout(struct wait_bit_key *key, int bit);
|
||||
|
||||
/**
|
||||
* wait_on_bit - wait for a bit to be cleared
|
||||
* @word: the word being waited on, a kernel virtual address
|
||||
* @bit: the bit of the word being waited on
|
||||
* @mode: the task state to sleep in
|
||||
*
|
||||
* There is a standard hashed waitqueue table for generic use. This
|
||||
* is the part of the hashtable's accessor API that waits on a bit.
|
||||
* For instance, if one were to have waiters on a bitflag, one would
|
||||
* call wait_on_bit() in threads waiting for the bit to clear.
|
||||
* One uses wait_on_bit() where one is waiting for the bit to clear,
|
||||
* but has no intention of setting it.
|
||||
* Returned value will be zero if the bit was cleared, or non-zero
|
||||
* if the process received a signal and the mode permitted wakeup
|
||||
* on that signal.
|
||||
*/
|
||||
static inline int
|
||||
wait_on_bit(unsigned long *word, int bit, unsigned mode)
|
||||
{
|
||||
might_sleep();
|
||||
if (!test_bit(bit, word))
|
||||
return 0;
|
||||
return out_of_line_wait_on_bit(word, bit,
|
||||
bit_wait,
|
||||
mode);
|
||||
}
|
||||
|
||||
/**
|
||||
* wait_on_bit_io - wait for a bit to be cleared
|
||||
* @word: the word being waited on, a kernel virtual address
|
||||
* @bit: the bit of the word being waited on
|
||||
* @mode: the task state to sleep in
|
||||
*
|
||||
* Use the standard hashed waitqueue table to wait for a bit
|
||||
* to be cleared. This is similar to wait_on_bit(), but calls
|
||||
* io_schedule() instead of schedule() for the actual waiting.
|
||||
*
|
||||
* Returned value will be zero if the bit was cleared, or non-zero
|
||||
* if the process received a signal and the mode permitted wakeup
|
||||
* on that signal.
|
||||
*/
|
||||
static inline int
|
||||
wait_on_bit_io(unsigned long *word, int bit, unsigned mode)
|
||||
{
|
||||
might_sleep();
|
||||
if (!test_bit(bit, word))
|
||||
return 0;
|
||||
return out_of_line_wait_on_bit(word, bit,
|
||||
bit_wait_io,
|
||||
mode);
|
||||
}
|
||||
|
||||
/**
|
||||
* wait_on_bit_timeout - wait for a bit to be cleared or a timeout elapses
|
||||
* @word: the word being waited on, a kernel virtual address
|
||||
* @bit: the bit of the word being waited on
|
||||
* @mode: the task state to sleep in
|
||||
* @timeout: timeout, in jiffies
|
||||
*
|
||||
* Use the standard hashed waitqueue table to wait for a bit
|
||||
* to be cleared. This is similar to wait_on_bit(), except also takes a
|
||||
* timeout parameter.
|
||||
*
|
||||
* Returned value will be zero if the bit was cleared before the
|
||||
* @timeout elapsed, or non-zero if the @timeout elapsed or process
|
||||
* received a signal and the mode permitted wakeup on that signal.
|
||||
*/
|
||||
static inline int
|
||||
wait_on_bit_timeout(unsigned long *word, int bit, unsigned mode,
|
||||
unsigned long timeout)
|
||||
{
|
||||
might_sleep();
|
||||
if (!test_bit(bit, word))
|
||||
return 0;
|
||||
return out_of_line_wait_on_bit_timeout(word, bit,
|
||||
bit_wait_timeout,
|
||||
mode, timeout);
|
||||
}
|
||||
|
||||
/**
|
||||
* wait_on_bit_action - wait for a bit to be cleared
|
||||
* @word: the word being waited on, a kernel virtual address
|
||||
* @bit: the bit of the word being waited on
|
||||
* @action: the function used to sleep, which may take special actions
|
||||
* @mode: the task state to sleep in
|
||||
*
|
||||
* Use the standard hashed waitqueue table to wait for a bit
|
||||
* to be cleared, and allow the waiting action to be specified.
|
||||
* This is like wait_on_bit() but allows fine control of how the waiting
|
||||
* is done.
|
||||
*
|
||||
* Returned value will be zero if the bit was cleared, or non-zero
|
||||
* if the process received a signal and the mode permitted wakeup
|
||||
* on that signal.
|
||||
*/
|
||||
static inline int
|
||||
wait_on_bit_action(unsigned long *word, int bit, wait_bit_action_f *action,
|
||||
unsigned mode)
|
||||
{
|
||||
might_sleep();
|
||||
if (!test_bit(bit, word))
|
||||
return 0;
|
||||
return out_of_line_wait_on_bit(word, bit, action, mode);
|
||||
}
|
||||
|
||||
/**
|
||||
* wait_on_bit_lock - wait for a bit to be cleared, when wanting to set it
|
||||
* @word: the word being waited on, a kernel virtual address
|
||||
* @bit: the bit of the word being waited on
|
||||
* @mode: the task state to sleep in
|
||||
*
|
||||
* There is a standard hashed waitqueue table for generic use. This
|
||||
* is the part of the hashtable's accessor API that waits on a bit
|
||||
* when one intends to set it, for instance, trying to lock bitflags.
|
||||
* For instance, if one were to have waiters trying to set bitflag
|
||||
* and waiting for it to clear before setting it, one would call
|
||||
* wait_on_bit() in threads waiting to be able to set the bit.
|
||||
* One uses wait_on_bit_lock() where one is waiting for the bit to
|
||||
* clear with the intention of setting it, and when done, clearing it.
|
||||
*
|
||||
* Returns zero if the bit was (eventually) found to be clear and was
|
||||
* set. Returns non-zero if a signal was delivered to the process and
|
||||
* the @mode allows that signal to wake the process.
|
||||
*/
|
||||
static inline int
|
||||
wait_on_bit_lock(unsigned long *word, int bit, unsigned mode)
|
||||
{
|
||||
might_sleep();
|
||||
if (!test_and_set_bit(bit, word))
|
||||
return 0;
|
||||
return out_of_line_wait_on_bit_lock(word, bit, bit_wait, mode);
|
||||
}
|
||||
|
||||
/**
|
||||
* wait_on_bit_lock_io - wait for a bit to be cleared, when wanting to set it
|
||||
* @word: the word being waited on, a kernel virtual address
|
||||
* @bit: the bit of the word being waited on
|
||||
* @mode: the task state to sleep in
|
||||
*
|
||||
* Use the standard hashed waitqueue table to wait for a bit
|
||||
* to be cleared and then to atomically set it. This is similar
|
||||
* to wait_on_bit(), but calls io_schedule() instead of schedule()
|
||||
* for the actual waiting.
|
||||
*
|
||||
* Returns zero if the bit was (eventually) found to be clear and was
|
||||
* set. Returns non-zero if a signal was delivered to the process and
|
||||
* the @mode allows that signal to wake the process.
|
||||
*/
|
||||
static inline int
|
||||
wait_on_bit_lock_io(unsigned long *word, int bit, unsigned mode)
|
||||
{
|
||||
might_sleep();
|
||||
if (!test_and_set_bit(bit, word))
|
||||
return 0;
|
||||
return out_of_line_wait_on_bit_lock(word, bit, bit_wait_io, mode);
|
||||
}
|
||||
|
||||
/**
|
||||
* wait_on_bit_lock_action - wait for a bit to be cleared, when wanting to set it
|
||||
* @word: the word being waited on, a kernel virtual address
|
||||
* @bit: the bit of the word being waited on
|
||||
* @action: the function used to sleep, which may take special actions
|
||||
* @mode: the task state to sleep in
|
||||
*
|
||||
* Use the standard hashed waitqueue table to wait for a bit
|
||||
* to be cleared and then to set it, and allow the waiting action
|
||||
* to be specified.
|
||||
* This is like wait_on_bit() but allows fine control of how the waiting
|
||||
* is done.
|
||||
*
|
||||
* Returns zero if the bit was (eventually) found to be clear and was
|
||||
* set. Returns non-zero if a signal was delivered to the process and
|
||||
* the @mode allows that signal to wake the process.
|
||||
*/
|
||||
static inline int
|
||||
wait_on_bit_lock_action(unsigned long *word, int bit, wait_bit_action_f *action,
|
||||
unsigned mode)
|
||||
{
|
||||
might_sleep();
|
||||
if (!test_and_set_bit(bit, word))
|
||||
return 0;
|
||||
return out_of_line_wait_on_bit_lock(word, bit, action, mode);
|
||||
}
|
||||
|
||||
/**
|
||||
* wait_on_atomic_t - Wait for an atomic_t to become 0
|
||||
* @val: The atomic value being waited on, a kernel virtual address
|
||||
* @action: the function used to sleep, which may take special actions
|
||||
* @mode: the task state to sleep in
|
||||
*
|
||||
* Wait for an atomic_t to become 0. We abuse the bit-wait waitqueue table for
|
||||
* the purpose of getting a waitqueue, but we set the key to a bit number
|
||||
* outside of the target 'word'.
|
||||
*/
|
||||
static inline
|
||||
int wait_on_atomic_t(atomic_t *val, int (*action)(atomic_t *), unsigned mode)
|
||||
{
|
||||
might_sleep();
|
||||
if (atomic_read(val) == 0)
|
||||
return 0;
|
||||
return out_of_line_wait_on_atomic_t(val, action, mode);
|
||||
}
|
||||
|
||||
#endif /* _LINUX_WAIT_H */
|
||||
|
||||
260
include/linux/wait_bit.h
Normal file
260
include/linux/wait_bit.h
Normal file
@ -0,0 +1,260 @@
|
||||
#ifndef _LINUX_WAIT_BIT_H
|
||||
#define _LINUX_WAIT_BIT_H
|
||||
|
||||
/*
|
||||
* Linux wait-bit related types and methods:
|
||||
*/
|
||||
#include <linux/wait.h>
|
||||
|
||||
struct wait_bit_key {
|
||||
void *flags;
|
||||
int bit_nr;
|
||||
#define WAIT_ATOMIC_T_BIT_NR -1
|
||||
unsigned long timeout;
|
||||
};
|
||||
|
||||
struct wait_bit_queue_entry {
|
||||
struct wait_bit_key key;
|
||||
struct wait_queue_entry wq_entry;
|
||||
};
|
||||
|
||||
#define __WAIT_BIT_KEY_INITIALIZER(word, bit) \
|
||||
{ .flags = word, .bit_nr = bit, }
|
||||
|
||||
#define __WAIT_ATOMIC_T_KEY_INITIALIZER(p) \
|
||||
{ .flags = p, .bit_nr = WAIT_ATOMIC_T_BIT_NR, }
|
||||
|
||||
typedef int wait_bit_action_f(struct wait_bit_key *key, int mode);
|
||||
void __wake_up_bit(struct wait_queue_head *wq_head, void *word, int bit);
|
||||
int __wait_on_bit(struct wait_queue_head *wq_head, struct wait_bit_queue_entry *wbq_entry, wait_bit_action_f *action, unsigned int mode);
|
||||
int __wait_on_bit_lock(struct wait_queue_head *wq_head, struct wait_bit_queue_entry *wbq_entry, wait_bit_action_f *action, unsigned int mode);
|
||||
void wake_up_bit(void *word, int bit);
|
||||
void wake_up_atomic_t(atomic_t *p);
|
||||
int out_of_line_wait_on_bit(void *word, int, wait_bit_action_f *action, unsigned int mode);
|
||||
int out_of_line_wait_on_bit_timeout(void *word, int, wait_bit_action_f *action, unsigned int mode, unsigned long timeout);
|
||||
int out_of_line_wait_on_bit_lock(void *word, int, wait_bit_action_f *action, unsigned int mode);
|
||||
int out_of_line_wait_on_atomic_t(atomic_t *p, int (*)(atomic_t *), unsigned int mode);
|
||||
struct wait_queue_head *bit_waitqueue(void *word, int bit);
|
||||
|
||||
int wake_bit_function(struct wait_queue_entry *wq_entry, unsigned mode, int sync, void *key);
|
||||
|
||||
#define DEFINE_WAIT_BIT(name, word, bit) \
|
||||
struct wait_bit_queue_entry name = { \
|
||||
.key = __WAIT_BIT_KEY_INITIALIZER(word, bit), \
|
||||
.wq_entry = { \
|
||||
.private = current, \
|
||||
.func = wake_bit_function, \
|
||||
.task_list = \
|
||||
LIST_HEAD_INIT((name).wq_entry.task_list), \
|
||||
}, \
|
||||
}
|
||||
|
||||
extern int bit_wait(struct wait_bit_key *key, int bit);
|
||||
extern int bit_wait_io(struct wait_bit_key *key, int bit);
|
||||
extern int bit_wait_timeout(struct wait_bit_key *key, int bit);
|
||||
extern int bit_wait_io_timeout(struct wait_bit_key *key, int bit);
|
||||
|
||||
/**
|
||||
* wait_on_bit - wait for a bit to be cleared
|
||||
* @word: the word being waited on, a kernel virtual address
|
||||
* @bit: the bit of the word being waited on
|
||||
* @mode: the task state to sleep in
|
||||
*
|
||||
* There is a standard hashed waitqueue table for generic use. This
|
||||
* is the part of the hashtable's accessor API that waits on a bit.
|
||||
* For instance, if one were to have waiters on a bitflag, one would
|
||||
* call wait_on_bit() in threads waiting for the bit to clear.
|
||||
* One uses wait_on_bit() where one is waiting for the bit to clear,
|
||||
* but has no intention of setting it.
|
||||
* Returned value will be zero if the bit was cleared, or non-zero
|
||||
* if the process received a signal and the mode permitted wakeup
|
||||
* on that signal.
|
||||
*/
|
||||
static inline int
|
||||
wait_on_bit(unsigned long *word, int bit, unsigned mode)
|
||||
{
|
||||
might_sleep();
|
||||
if (!test_bit(bit, word))
|
||||
return 0;
|
||||
return out_of_line_wait_on_bit(word, bit,
|
||||
bit_wait,
|
||||
mode);
|
||||
}
|
||||
|
||||
/**
|
||||
* wait_on_bit_io - wait for a bit to be cleared
|
||||
* @word: the word being waited on, a kernel virtual address
|
||||
* @bit: the bit of the word being waited on
|
||||
* @mode: the task state to sleep in
|
||||
*
|
||||
* Use the standard hashed waitqueue table to wait for a bit
|
||||
* to be cleared. This is similar to wait_on_bit(), but calls
|
||||
* io_schedule() instead of schedule() for the actual waiting.
|
||||
*
|
||||
* Returned value will be zero if the bit was cleared, or non-zero
|
||||
* if the process received a signal and the mode permitted wakeup
|
||||
* on that signal.
|
||||
*/
|
||||
static inline int
|
||||
wait_on_bit_io(unsigned long *word, int bit, unsigned mode)
|
||||
{
|
||||
might_sleep();
|
||||
if (!test_bit(bit, word))
|
||||
return 0;
|
||||
return out_of_line_wait_on_bit(word, bit,
|
||||
bit_wait_io,
|
||||
mode);
|
||||
}
|
||||
|
||||
/**
|
||||
* wait_on_bit_timeout - wait for a bit to be cleared or a timeout elapses
|
||||
* @word: the word being waited on, a kernel virtual address
|
||||
* @bit: the bit of the word being waited on
|
||||
* @mode: the task state to sleep in
|
||||
* @timeout: timeout, in jiffies
|
||||
*
|
||||
* Use the standard hashed waitqueue table to wait for a bit
|
||||
* to be cleared. This is similar to wait_on_bit(), except also takes a
|
||||
* timeout parameter.
|
||||
*
|
||||
* Returned value will be zero if the bit was cleared before the
|
||||
* @timeout elapsed, or non-zero if the @timeout elapsed or process
|
||||
* received a signal and the mode permitted wakeup on that signal.
|
||||
*/
|
||||
static inline int
|
||||
wait_on_bit_timeout(unsigned long *word, int bit, unsigned mode,
|
||||
unsigned long timeout)
|
||||
{
|
||||
might_sleep();
|
||||
if (!test_bit(bit, word))
|
||||
return 0;
|
||||
return out_of_line_wait_on_bit_timeout(word, bit,
|
||||
bit_wait_timeout,
|
||||
mode, timeout);
|
||||
}
|
||||
|
||||
/**
|
||||
* wait_on_bit_action - wait for a bit to be cleared
|
||||
* @word: the word being waited on, a kernel virtual address
|
||||
* @bit: the bit of the word being waited on
|
||||
* @action: the function used to sleep, which may take special actions
|
||||
* @mode: the task state to sleep in
|
||||
*
|
||||
* Use the standard hashed waitqueue table to wait for a bit
|
||||
* to be cleared, and allow the waiting action to be specified.
|
||||
* This is like wait_on_bit() but allows fine control of how the waiting
|
||||
* is done.
|
||||
*
|
||||
* Returned value will be zero if the bit was cleared, or non-zero
|
||||
* if the process received a signal and the mode permitted wakeup
|
||||
* on that signal.
|
||||
*/
|
||||
static inline int
|
||||
wait_on_bit_action(unsigned long *word, int bit, wait_bit_action_f *action,
|
||||
unsigned mode)
|
||||
{
|
||||
might_sleep();
|
||||
if (!test_bit(bit, word))
|
||||
return 0;
|
||||
return out_of_line_wait_on_bit(word, bit, action, mode);
|
||||
}
|
||||
|
||||
/**
|
||||
* wait_on_bit_lock - wait for a bit to be cleared, when wanting to set it
|
||||
* @word: the word being waited on, a kernel virtual address
|
||||
* @bit: the bit of the word being waited on
|
||||
* @mode: the task state to sleep in
|
||||
*
|
||||
* There is a standard hashed waitqueue table for generic use. This
|
||||
* is the part of the hashtable's accessor API that waits on a bit
|
||||
* when one intends to set it, for instance, trying to lock bitflags.
|
||||
* For instance, if one were to have waiters trying to set bitflag
|
||||
* and waiting for it to clear before setting it, one would call
|
||||
* wait_on_bit() in threads waiting to be able to set the bit.
|
||||
* One uses wait_on_bit_lock() where one is waiting for the bit to
|
||||
* clear with the intention of setting it, and when done, clearing it.
|
||||
*
|
||||
* Returns zero if the bit was (eventually) found to be clear and was
|
||||
* set. Returns non-zero if a signal was delivered to the process and
|
||||
* the @mode allows that signal to wake the process.
|
||||
*/
|
||||
static inline int
|
||||
wait_on_bit_lock(unsigned long *word, int bit, unsigned mode)
|
||||
{
|
||||
might_sleep();
|
||||
if (!test_and_set_bit(bit, word))
|
||||
return 0;
|
||||
return out_of_line_wait_on_bit_lock(word, bit, bit_wait, mode);
|
||||
}
|
||||
|
||||
/**
|
||||
* wait_on_bit_lock_io - wait for a bit to be cleared, when wanting to set it
|
||||
* @word: the word being waited on, a kernel virtual address
|
||||
* @bit: the bit of the word being waited on
|
||||
* @mode: the task state to sleep in
|
||||
*
|
||||
* Use the standard hashed waitqueue table to wait for a bit
|
||||
* to be cleared and then to atomically set it. This is similar
|
||||
* to wait_on_bit(), but calls io_schedule() instead of schedule()
|
||||
* for the actual waiting.
|
||||
*
|
||||
* Returns zero if the bit was (eventually) found to be clear and was
|
||||
* set. Returns non-zero if a signal was delivered to the process and
|
||||
* the @mode allows that signal to wake the process.
|
||||
*/
|
||||
static inline int
|
||||
wait_on_bit_lock_io(unsigned long *word, int bit, unsigned mode)
|
||||
{
|
||||
might_sleep();
|
||||
if (!test_and_set_bit(bit, word))
|
||||
return 0;
|
||||
return out_of_line_wait_on_bit_lock(word, bit, bit_wait_io, mode);
|
||||
}
|
||||
|
||||
/**
|
||||
* wait_on_bit_lock_action - wait for a bit to be cleared, when wanting to set it
|
||||
* @word: the word being waited on, a kernel virtual address
|
||||
* @bit: the bit of the word being waited on
|
||||
* @action: the function used to sleep, which may take special actions
|
||||
* @mode: the task state to sleep in
|
||||
*
|
||||
* Use the standard hashed waitqueue table to wait for a bit
|
||||
* to be cleared and then to set it, and allow the waiting action
|
||||
* to be specified.
|
||||
* This is like wait_on_bit() but allows fine control of how the waiting
|
||||
* is done.
|
||||
*
|
||||
* Returns zero if the bit was (eventually) found to be clear and was
|
||||
* set. Returns non-zero if a signal was delivered to the process and
|
||||
* the @mode allows that signal to wake the process.
|
||||
*/
|
||||
static inline int
|
||||
wait_on_bit_lock_action(unsigned long *word, int bit, wait_bit_action_f *action,
|
||||
unsigned mode)
|
||||
{
|
||||
might_sleep();
|
||||
if (!test_and_set_bit(bit, word))
|
||||
return 0;
|
||||
return out_of_line_wait_on_bit_lock(word, bit, action, mode);
|
||||
}
|
||||
|
||||
/**
|
||||
* wait_on_atomic_t - Wait for an atomic_t to become 0
|
||||
* @val: The atomic value being waited on, a kernel virtual address
|
||||
* @action: the function used to sleep, which may take special actions
|
||||
* @mode: the task state to sleep in
|
||||
*
|
||||
* Wait for an atomic_t to become 0. We abuse the bit-wait waitqueue table for
|
||||
* the purpose of getting a waitqueue, but we set the key to a bit number
|
||||
* outside of the target 'word'.
|
||||
*/
|
||||
static inline
|
||||
int wait_on_atomic_t(atomic_t *val, int (*action)(atomic_t *), unsigned mode)
|
||||
{
|
||||
might_sleep();
|
||||
if (atomic_read(val) == 0)
|
||||
return 0;
|
||||
return out_of_line_wait_on_atomic_t(val, action, mode);
|
||||
}
|
||||
|
||||
#endif /* _LINUX_WAIT_BIT_H */
|
||||
@ -17,7 +17,7 @@ endif
|
||||
|
||||
obj-y += core.o loadavg.o clock.o cputime.o
|
||||
obj-y += idle_task.o fair.o rt.o deadline.o stop_task.o
|
||||
obj-y += wait.o swait.o completion.o idle.o
|
||||
obj-y += wait.o wait_bit.o swait.o completion.o idle.o
|
||||
obj-$(CONFIG_SMP) += cpupri.o cpudeadline.o topology.o
|
||||
obj-$(CONFIG_SCHED_AUTOGROUP) += autogroup.o
|
||||
obj-$(CONFIG_SCHEDSTATS) += stats.o
|
||||
|
||||
@ -390,260 +390,3 @@ int woken_wake_function(struct wait_queue_entry *wq_entry, unsigned mode, int sy
|
||||
return default_wake_function(wq_entry, mode, sync, key);
|
||||
}
|
||||
EXPORT_SYMBOL(woken_wake_function);
|
||||
|
||||
int wake_bit_function(struct wait_queue_entry *wq_entry, unsigned mode, int sync, void *arg)
|
||||
{
|
||||
struct wait_bit_key *key = arg;
|
||||
struct wait_bit_queue_entry *wait_bit = container_of(wq_entry, struct wait_bit_queue_entry, wq_entry);
|
||||
|
||||
if (wait_bit->key.flags != key->flags ||
|
||||
wait_bit->key.bit_nr != key->bit_nr ||
|
||||
test_bit(key->bit_nr, key->flags))
|
||||
return 0;
|
||||
else
|
||||
return autoremove_wake_function(wq_entry, mode, sync, key);
|
||||
}
|
||||
EXPORT_SYMBOL(wake_bit_function);
|
||||
|
||||
/*
|
||||
* To allow interruptible waiting and asynchronous (i.e. nonblocking)
|
||||
* waiting, the actions of __wait_on_bit() and __wait_on_bit_lock() are
|
||||
* permitted return codes. Nonzero return codes halt waiting and return.
|
||||
*/
|
||||
int __sched
|
||||
__wait_on_bit(struct wait_queue_head *wq_head, struct wait_bit_queue_entry *wbq_entry,
|
||||
wait_bit_action_f *action, unsigned mode)
|
||||
{
|
||||
int ret = 0;
|
||||
|
||||
do {
|
||||
prepare_to_wait(wq_head, &wbq_entry->wq_entry, mode);
|
||||
if (test_bit(wbq_entry->key.bit_nr, wbq_entry->key.flags))
|
||||
ret = (*action)(&wbq_entry->key, mode);
|
||||
} while (test_bit(wbq_entry->key.bit_nr, wbq_entry->key.flags) && !ret);
|
||||
finish_wait(wq_head, &wbq_entry->wq_entry);
|
||||
return ret;
|
||||
}
|
||||
EXPORT_SYMBOL(__wait_on_bit);
|
||||
|
||||
int __sched out_of_line_wait_on_bit(void *word, int bit,
|
||||
wait_bit_action_f *action, unsigned mode)
|
||||
{
|
||||
struct wait_queue_head *wq_head = bit_waitqueue(word, bit);
|
||||
DEFINE_WAIT_BIT(wq_entry, word, bit);
|
||||
|
||||
return __wait_on_bit(wq_head, &wq_entry, action, mode);
|
||||
}
|
||||
EXPORT_SYMBOL(out_of_line_wait_on_bit);
|
||||
|
||||
int __sched out_of_line_wait_on_bit_timeout(
|
||||
void *word, int bit, wait_bit_action_f *action,
|
||||
unsigned mode, unsigned long timeout)
|
||||
{
|
||||
struct wait_queue_head *wq_head = bit_waitqueue(word, bit);
|
||||
DEFINE_WAIT_BIT(wq_entry, word, bit);
|
||||
|
||||
wq_entry.key.timeout = jiffies + timeout;
|
||||
return __wait_on_bit(wq_head, &wq_entry, action, mode);
|
||||
}
|
||||
EXPORT_SYMBOL_GPL(out_of_line_wait_on_bit_timeout);
|
||||
|
||||
int __sched
|
||||
__wait_on_bit_lock(struct wait_queue_head *wq_head, struct wait_bit_queue_entry *wbq_entry,
|
||||
wait_bit_action_f *action, unsigned mode)
|
||||
{
|
||||
int ret = 0;
|
||||
|
||||
for (;;) {
|
||||
prepare_to_wait_exclusive(wq_head, &wbq_entry->wq_entry, mode);
|
||||
if (test_bit(wbq_entry->key.bit_nr, wbq_entry->key.flags)) {
|
||||
ret = action(&wbq_entry->key, mode);
|
||||
/*
|
||||
* See the comment in prepare_to_wait_event().
|
||||
* finish_wait() does not necessarily takes wwq_head->lock,
|
||||
* but test_and_set_bit() implies mb() which pairs with
|
||||
* smp_mb__after_atomic() before wake_up_page().
|
||||
*/
|
||||
if (ret)
|
||||
finish_wait(wq_head, &wbq_entry->wq_entry);
|
||||
}
|
||||
if (!test_and_set_bit(wbq_entry->key.bit_nr, wbq_entry->key.flags)) {
|
||||
if (!ret)
|
||||
finish_wait(wq_head, &wbq_entry->wq_entry);
|
||||
return 0;
|
||||
} else if (ret) {
|
||||
return ret;
|
||||
}
|
||||
}
|
||||
}
|
||||
EXPORT_SYMBOL(__wait_on_bit_lock);
|
||||
|
||||
int __sched out_of_line_wait_on_bit_lock(void *word, int bit,
|
||||
wait_bit_action_f *action, unsigned mode)
|
||||
{
|
||||
struct wait_queue_head *wq_head = bit_waitqueue(word, bit);
|
||||
DEFINE_WAIT_BIT(wq_entry, word, bit);
|
||||
|
||||
return __wait_on_bit_lock(wq_head, &wq_entry, action, mode);
|
||||
}
|
||||
EXPORT_SYMBOL(out_of_line_wait_on_bit_lock);
|
||||
|
||||
void __wake_up_bit(struct wait_queue_head *wq_head, void *word, int bit)
|
||||
{
|
||||
struct wait_bit_key key = __WAIT_BIT_KEY_INITIALIZER(word, bit);
|
||||
if (waitqueue_active(wq_head))
|
||||
__wake_up(wq_head, TASK_NORMAL, 1, &key);
|
||||
}
|
||||
EXPORT_SYMBOL(__wake_up_bit);
|
||||
|
||||
/**
|
||||
* wake_up_bit - wake up a waiter on a bit
|
||||
* @word: the word being waited on, a kernel virtual address
|
||||
* @bit: the bit of the word being waited on
|
||||
*
|
||||
* There is a standard hashed waitqueue table for generic use. This
|
||||
* is the part of the hashtable's accessor API that wakes up waiters
|
||||
* on a bit. For instance, if one were to have waiters on a bitflag,
|
||||
* one would call wake_up_bit() after clearing the bit.
|
||||
*
|
||||
* In order for this to function properly, as it uses waitqueue_active()
|
||||
* internally, some kind of memory barrier must be done prior to calling
|
||||
* this. Typically, this will be smp_mb__after_atomic(), but in some
|
||||
* cases where bitflags are manipulated non-atomically under a lock, one
|
||||
* may need to use a less regular barrier, such fs/inode.c's smp_mb(),
|
||||
* because spin_unlock() does not guarantee a memory barrier.
|
||||
*/
|
||||
void wake_up_bit(void *word, int bit)
|
||||
{
|
||||
__wake_up_bit(bit_waitqueue(word, bit), word, bit);
|
||||
}
|
||||
EXPORT_SYMBOL(wake_up_bit);
|
||||
|
||||
/*
|
||||
* Manipulate the atomic_t address to produce a better bit waitqueue table hash
|
||||
* index (we're keying off bit -1, but that would produce a horrible hash
|
||||
* value).
|
||||
*/
|
||||
static inline wait_queue_head_t *atomic_t_waitqueue(atomic_t *p)
|
||||
{
|
||||
if (BITS_PER_LONG == 64) {
|
||||
unsigned long q = (unsigned long)p;
|
||||
return bit_waitqueue((void *)(q & ~1), q & 1);
|
||||
}
|
||||
return bit_waitqueue(p, 0);
|
||||
}
|
||||
|
||||
static int wake_atomic_t_function(struct wait_queue_entry *wq_entry, unsigned mode, int sync,
|
||||
void *arg)
|
||||
{
|
||||
struct wait_bit_key *key = arg;
|
||||
struct wait_bit_queue_entry *wait_bit = container_of(wq_entry, struct wait_bit_queue_entry, wq_entry);
|
||||
atomic_t *val = key->flags;
|
||||
|
||||
if (wait_bit->key.flags != key->flags ||
|
||||
wait_bit->key.bit_nr != key->bit_nr ||
|
||||
atomic_read(val) != 0)
|
||||
return 0;
|
||||
return autoremove_wake_function(wq_entry, mode, sync, key);
|
||||
}
|
||||
|
||||
/*
|
||||
* To allow interruptible waiting and asynchronous (i.e. nonblocking) waiting,
|
||||
* the actions of __wait_on_atomic_t() are permitted return codes. Nonzero
|
||||
* return codes halt waiting and return.
|
||||
*/
|
||||
static __sched
|
||||
int __wait_on_atomic_t(struct wait_queue_head *wq_head, struct wait_bit_queue_entry *wbq_entry,
|
||||
int (*action)(atomic_t *), unsigned mode)
|
||||
{
|
||||
atomic_t *val;
|
||||
int ret = 0;
|
||||
|
||||
do {
|
||||
prepare_to_wait(wq_head, &wbq_entry->wq_entry, mode);
|
||||
val = wbq_entry->key.flags;
|
||||
if (atomic_read(val) == 0)
|
||||
break;
|
||||
ret = (*action)(val);
|
||||
} while (!ret && atomic_read(val) != 0);
|
||||
finish_wait(wq_head, &wbq_entry->wq_entry);
|
||||
return ret;
|
||||
}
|
||||
|
||||
#define DEFINE_WAIT_ATOMIC_T(name, p) \
|
||||
struct wait_bit_queue_entry name = { \
|
||||
.key = __WAIT_ATOMIC_T_KEY_INITIALIZER(p), \
|
||||
.wq_entry = { \
|
||||
.private = current, \
|
||||
.func = wake_atomic_t_function, \
|
||||
.task_list = \
|
||||
LIST_HEAD_INIT((name).wq_entry.task_list), \
|
||||
}, \
|
||||
}
|
||||
|
||||
__sched int out_of_line_wait_on_atomic_t(atomic_t *p, int (*action)(atomic_t *),
|
||||
unsigned mode)
|
||||
{
|
||||
struct wait_queue_head *wq_head = atomic_t_waitqueue(p);
|
||||
DEFINE_WAIT_ATOMIC_T(wq_entry, p);
|
||||
|
||||
return __wait_on_atomic_t(wq_head, &wq_entry, action, mode);
|
||||
}
|
||||
EXPORT_SYMBOL(out_of_line_wait_on_atomic_t);
|
||||
|
||||
/**
|
||||
* wake_up_atomic_t - Wake up a waiter on a atomic_t
|
||||
* @p: The atomic_t being waited on, a kernel virtual address
|
||||
*
|
||||
* Wake up anyone waiting for the atomic_t to go to zero.
|
||||
*
|
||||
* Abuse the bit-waker function and its waitqueue hash table set (the atomic_t
|
||||
* check is done by the waiter's wake function, not the by the waker itself).
|
||||
*/
|
||||
void wake_up_atomic_t(atomic_t *p)
|
||||
{
|
||||
__wake_up_bit(atomic_t_waitqueue(p), p, WAIT_ATOMIC_T_BIT_NR);
|
||||
}
|
||||
EXPORT_SYMBOL(wake_up_atomic_t);
|
||||
|
||||
__sched int bit_wait(struct wait_bit_key *word, int mode)
|
||||
{
|
||||
schedule();
|
||||
if (signal_pending_state(mode, current))
|
||||
return -EINTR;
|
||||
return 0;
|
||||
}
|
||||
EXPORT_SYMBOL(bit_wait);
|
||||
|
||||
__sched int bit_wait_io(struct wait_bit_key *word, int mode)
|
||||
{
|
||||
io_schedule();
|
||||
if (signal_pending_state(mode, current))
|
||||
return -EINTR;
|
||||
return 0;
|
||||
}
|
||||
EXPORT_SYMBOL(bit_wait_io);
|
||||
|
||||
__sched int bit_wait_timeout(struct wait_bit_key *word, int mode)
|
||||
{
|
||||
unsigned long now = READ_ONCE(jiffies);
|
||||
if (time_after_eq(now, word->timeout))
|
||||
return -EAGAIN;
|
||||
schedule_timeout(word->timeout - now);
|
||||
if (signal_pending_state(mode, current))
|
||||
return -EINTR;
|
||||
return 0;
|
||||
}
|
||||
EXPORT_SYMBOL_GPL(bit_wait_timeout);
|
||||
|
||||
__sched int bit_wait_io_timeout(struct wait_bit_key *word, int mode)
|
||||
{
|
||||
unsigned long now = READ_ONCE(jiffies);
|
||||
if (time_after_eq(now, word->timeout))
|
||||
return -EAGAIN;
|
||||
io_schedule_timeout(word->timeout - now);
|
||||
if (signal_pending_state(mode, current))
|
||||
return -EINTR;
|
||||
return 0;
|
||||
}
|
||||
EXPORT_SYMBOL_GPL(bit_wait_io_timeout);
|
||||
|
||||
263
kernel/sched/wait_bit.c
Normal file
263
kernel/sched/wait_bit.c
Normal file
@ -0,0 +1,263 @@
|
||||
/*
|
||||
* The implementation of the wait_bit*() and related waiting APIs:
|
||||
*/
|
||||
#include <linux/wait_bit.h>
|
||||
#include <linux/sched/signal.h>
|
||||
#include <linux/sched/debug.h>
|
||||
|
||||
int wake_bit_function(struct wait_queue_entry *wq_entry, unsigned mode, int sync, void *arg)
|
||||
{
|
||||
struct wait_bit_key *key = arg;
|
||||
struct wait_bit_queue_entry *wait_bit = container_of(wq_entry, struct wait_bit_queue_entry, wq_entry);
|
||||
|
||||
if (wait_bit->key.flags != key->flags ||
|
||||
wait_bit->key.bit_nr != key->bit_nr ||
|
||||
test_bit(key->bit_nr, key->flags))
|
||||
return 0;
|
||||
else
|
||||
return autoremove_wake_function(wq_entry, mode, sync, key);
|
||||
}
|
||||
EXPORT_SYMBOL(wake_bit_function);
|
||||
|
||||
/*
|
||||
* To allow interruptible waiting and asynchronous (i.e. nonblocking)
|
||||
* waiting, the actions of __wait_on_bit() and __wait_on_bit_lock() are
|
||||
* permitted return codes. Nonzero return codes halt waiting and return.
|
||||
*/
|
||||
int __sched
|
||||
__wait_on_bit(struct wait_queue_head *wq_head, struct wait_bit_queue_entry *wbq_entry,
|
||||
wait_bit_action_f *action, unsigned mode)
|
||||
{
|
||||
int ret = 0;
|
||||
|
||||
do {
|
||||
prepare_to_wait(wq_head, &wbq_entry->wq_entry, mode);
|
||||
if (test_bit(wbq_entry->key.bit_nr, wbq_entry->key.flags))
|
||||
ret = (*action)(&wbq_entry->key, mode);
|
||||
} while (test_bit(wbq_entry->key.bit_nr, wbq_entry->key.flags) && !ret);
|
||||
finish_wait(wq_head, &wbq_entry->wq_entry);
|
||||
return ret;
|
||||
}
|
||||
EXPORT_SYMBOL(__wait_on_bit);
|
||||
|
||||
int __sched out_of_line_wait_on_bit(void *word, int bit,
|
||||
wait_bit_action_f *action, unsigned mode)
|
||||
{
|
||||
struct wait_queue_head *wq_head = bit_waitqueue(word, bit);
|
||||
DEFINE_WAIT_BIT(wq_entry, word, bit);
|
||||
|
||||
return __wait_on_bit(wq_head, &wq_entry, action, mode);
|
||||
}
|
||||
EXPORT_SYMBOL(out_of_line_wait_on_bit);
|
||||
|
||||
int __sched out_of_line_wait_on_bit_timeout(
|
||||
void *word, int bit, wait_bit_action_f *action,
|
||||
unsigned mode, unsigned long timeout)
|
||||
{
|
||||
struct wait_queue_head *wq_head = bit_waitqueue(word, bit);
|
||||
DEFINE_WAIT_BIT(wq_entry, word, bit);
|
||||
|
||||
wq_entry.key.timeout = jiffies + timeout;
|
||||
return __wait_on_bit(wq_head, &wq_entry, action, mode);
|
||||
}
|
||||
EXPORT_SYMBOL_GPL(out_of_line_wait_on_bit_timeout);
|
||||
|
||||
int __sched
|
||||
__wait_on_bit_lock(struct wait_queue_head *wq_head, struct wait_bit_queue_entry *wbq_entry,
|
||||
wait_bit_action_f *action, unsigned mode)
|
||||
{
|
||||
int ret = 0;
|
||||
|
||||
for (;;) {
|
||||
prepare_to_wait_exclusive(wq_head, &wbq_entry->wq_entry, mode);
|
||||
if (test_bit(wbq_entry->key.bit_nr, wbq_entry->key.flags)) {
|
||||
ret = action(&wbq_entry->key, mode);
|
||||
/*
|
||||
* See the comment in prepare_to_wait_event().
|
||||
* finish_wait() does not necessarily takes wwq_head->lock,
|
||||
* but test_and_set_bit() implies mb() which pairs with
|
||||
* smp_mb__after_atomic() before wake_up_page().
|
||||
*/
|
||||
if (ret)
|
||||
finish_wait(wq_head, &wbq_entry->wq_entry);
|
||||
}
|
||||
if (!test_and_set_bit(wbq_entry->key.bit_nr, wbq_entry->key.flags)) {
|
||||
if (!ret)
|
||||
finish_wait(wq_head, &wbq_entry->wq_entry);
|
||||
return 0;
|
||||
} else if (ret) {
|
||||
return ret;
|
||||
}
|
||||
}
|
||||
}
|
||||
EXPORT_SYMBOL(__wait_on_bit_lock);
|
||||
|
||||
int __sched out_of_line_wait_on_bit_lock(void *word, int bit,
|
||||
wait_bit_action_f *action, unsigned mode)
|
||||
{
|
||||
struct wait_queue_head *wq_head = bit_waitqueue(word, bit);
|
||||
DEFINE_WAIT_BIT(wq_entry, word, bit);
|
||||
|
||||
return __wait_on_bit_lock(wq_head, &wq_entry, action, mode);
|
||||
}
|
||||
EXPORT_SYMBOL(out_of_line_wait_on_bit_lock);
|
||||
|
||||
void __wake_up_bit(struct wait_queue_head *wq_head, void *word, int bit)
|
||||
{
|
||||
struct wait_bit_key key = __WAIT_BIT_KEY_INITIALIZER(word, bit);
|
||||
if (waitqueue_active(wq_head))
|
||||
__wake_up(wq_head, TASK_NORMAL, 1, &key);
|
||||
}
|
||||
EXPORT_SYMBOL(__wake_up_bit);
|
||||
|
||||
/**
|
||||
* wake_up_bit - wake up a waiter on a bit
|
||||
* @word: the word being waited on, a kernel virtual address
|
||||
* @bit: the bit of the word being waited on
|
||||
*
|
||||
* There is a standard hashed waitqueue table for generic use. This
|
||||
* is the part of the hashtable's accessor API that wakes up waiters
|
||||
* on a bit. For instance, if one were to have waiters on a bitflag,
|
||||
* one would call wake_up_bit() after clearing the bit.
|
||||
*
|
||||
* In order for this to function properly, as it uses waitqueue_active()
|
||||
* internally, some kind of memory barrier must be done prior to calling
|
||||
* this. Typically, this will be smp_mb__after_atomic(), but in some
|
||||
* cases where bitflags are manipulated non-atomically under a lock, one
|
||||
* may need to use a less regular barrier, such fs/inode.c's smp_mb(),
|
||||
* because spin_unlock() does not guarantee a memory barrier.
|
||||
*/
|
||||
void wake_up_bit(void *word, int bit)
|
||||
{
|
||||
__wake_up_bit(bit_waitqueue(word, bit), word, bit);
|
||||
}
|
||||
EXPORT_SYMBOL(wake_up_bit);
|
||||
|
||||
/*
|
||||
* Manipulate the atomic_t address to produce a better bit waitqueue table hash
|
||||
* index (we're keying off bit -1, but that would produce a horrible hash
|
||||
* value).
|
||||
*/
|
||||
static inline wait_queue_head_t *atomic_t_waitqueue(atomic_t *p)
|
||||
{
|
||||
if (BITS_PER_LONG == 64) {
|
||||
unsigned long q = (unsigned long)p;
|
||||
return bit_waitqueue((void *)(q & ~1), q & 1);
|
||||
}
|
||||
return bit_waitqueue(p, 0);
|
||||
}
|
||||
|
||||
static int wake_atomic_t_function(struct wait_queue_entry *wq_entry, unsigned mode, int sync,
|
||||
void *arg)
|
||||
{
|
||||
struct wait_bit_key *key = arg;
|
||||
struct wait_bit_queue_entry *wait_bit = container_of(wq_entry, struct wait_bit_queue_entry, wq_entry);
|
||||
atomic_t *val = key->flags;
|
||||
|
||||
if (wait_bit->key.flags != key->flags ||
|
||||
wait_bit->key.bit_nr != key->bit_nr ||
|
||||
atomic_read(val) != 0)
|
||||
return 0;
|
||||
return autoremove_wake_function(wq_entry, mode, sync, key);
|
||||
}
|
||||
|
||||
/*
|
||||
* To allow interruptible waiting and asynchronous (i.e. nonblocking) waiting,
|
||||
* the actions of __wait_on_atomic_t() are permitted return codes. Nonzero
|
||||
* return codes halt waiting and return.
|
||||
*/
|
||||
static __sched
|
||||
int __wait_on_atomic_t(struct wait_queue_head *wq_head, struct wait_bit_queue_entry *wbq_entry,
|
||||
int (*action)(atomic_t *), unsigned mode)
|
||||
{
|
||||
atomic_t *val;
|
||||
int ret = 0;
|
||||
|
||||
do {
|
||||
prepare_to_wait(wq_head, &wbq_entry->wq_entry, mode);
|
||||
val = wbq_entry->key.flags;
|
||||
if (atomic_read(val) == 0)
|
||||
break;
|
||||
ret = (*action)(val);
|
||||
} while (!ret && atomic_read(val) != 0);
|
||||
finish_wait(wq_head, &wbq_entry->wq_entry);
|
||||
return ret;
|
||||
}
|
||||
|
||||
#define DEFINE_WAIT_ATOMIC_T(name, p) \
|
||||
struct wait_bit_queue_entry name = { \
|
||||
.key = __WAIT_ATOMIC_T_KEY_INITIALIZER(p), \
|
||||
.wq_entry = { \
|
||||
.private = current, \
|
||||
.func = wake_atomic_t_function, \
|
||||
.task_list = \
|
||||
LIST_HEAD_INIT((name).wq_entry.task_list), \
|
||||
}, \
|
||||
}
|
||||
|
||||
__sched int out_of_line_wait_on_atomic_t(atomic_t *p, int (*action)(atomic_t *),
|
||||
unsigned mode)
|
||||
{
|
||||
struct wait_queue_head *wq_head = atomic_t_waitqueue(p);
|
||||
DEFINE_WAIT_ATOMIC_T(wq_entry, p);
|
||||
|
||||
return __wait_on_atomic_t(wq_head, &wq_entry, action, mode);
|
||||
}
|
||||
EXPORT_SYMBOL(out_of_line_wait_on_atomic_t);
|
||||
|
||||
/**
|
||||
* wake_up_atomic_t - Wake up a waiter on a atomic_t
|
||||
* @p: The atomic_t being waited on, a kernel virtual address
|
||||
*
|
||||
* Wake up anyone waiting for the atomic_t to go to zero.
|
||||
*
|
||||
* Abuse the bit-waker function and its waitqueue hash table set (the atomic_t
|
||||
* check is done by the waiter's wake function, not the by the waker itself).
|
||||
*/
|
||||
void wake_up_atomic_t(atomic_t *p)
|
||||
{
|
||||
__wake_up_bit(atomic_t_waitqueue(p), p, WAIT_ATOMIC_T_BIT_NR);
|
||||
}
|
||||
EXPORT_SYMBOL(wake_up_atomic_t);
|
||||
|
||||
__sched int bit_wait(struct wait_bit_key *word, int mode)
|
||||
{
|
||||
schedule();
|
||||
if (signal_pending_state(mode, current))
|
||||
return -EINTR;
|
||||
return 0;
|
||||
}
|
||||
EXPORT_SYMBOL(bit_wait);
|
||||
|
||||
__sched int bit_wait_io(struct wait_bit_key *word, int mode)
|
||||
{
|
||||
io_schedule();
|
||||
if (signal_pending_state(mode, current))
|
||||
return -EINTR;
|
||||
return 0;
|
||||
}
|
||||
EXPORT_SYMBOL(bit_wait_io);
|
||||
|
||||
__sched int bit_wait_timeout(struct wait_bit_key *word, int mode)
|
||||
{
|
||||
unsigned long now = READ_ONCE(jiffies);
|
||||
if (time_after_eq(now, word->timeout))
|
||||
return -EAGAIN;
|
||||
schedule_timeout(word->timeout - now);
|
||||
if (signal_pending_state(mode, current))
|
||||
return -EINTR;
|
||||
return 0;
|
||||
}
|
||||
EXPORT_SYMBOL_GPL(bit_wait_timeout);
|
||||
|
||||
__sched int bit_wait_io_timeout(struct wait_bit_key *word, int mode)
|
||||
{
|
||||
unsigned long now = READ_ONCE(jiffies);
|
||||
if (time_after_eq(now, word->timeout))
|
||||
return -EAGAIN;
|
||||
io_schedule_timeout(word->timeout - now);
|
||||
if (signal_pending_state(mode, current))
|
||||
return -EINTR;
|
||||
return 0;
|
||||
}
|
||||
EXPORT_SYMBOL_GPL(bit_wait_io_timeout);
|
||||
@ -13,6 +13,7 @@
|
||||
#define _INTERNAL_H
|
||||
|
||||
#include <linux/sched.h>
|
||||
#include <linux/wait_bit.h>
|
||||
#include <linux/cred.h>
|
||||
#include <linux/key-type.h>
|
||||
#include <linux/task_work.h>
|
||||
|
||||
Loading…
Reference in New Issue
Block a user