leandrof/linux
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

io_uring: improve sqpoll event/state handling

Browse Source 
As sqd->state changes rarely, don't check every event one by one but
look them all at once. Add a helper function. Also don't go into event
waiting sleeping with STOP flag set.

Signed-off-by: Pavel Begunkov <asml.silence@gmail.com>
Link: https://lore.kernel.org/r/645025f95c7eeec97f88ff497785f4f1d6f3966f.1621201931.git.asml.silence@gmail.com
Signed-off-by: Jens Axboe <axboe@kernel.dk>
This commit is contained in:
Pavel Begunkov
2021-05-16 22:58:00 +01:00
committed by Jens Axboe
parent 009c9aa5be
commit e4b6d902a9
1 changed files with 28 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files

45
fs/io_uring.c
View File

@@ -6757,6 +6757,11 @@ static int io_submit_sqes(struct io_ring_ctx *ctx, unsigned int nr)
return submitted; return submitted;
} }
static inline bool io_sqd_events_pending(struct io_sq_data *sqd)
{
return READ_ONCE(sqd->state);
}
static inline void io_ring_set_wakeup_flag(struct io_ring_ctx *ctx) static inline void io_ring_set_wakeup_flag(struct io_ring_ctx *ctx)
{ {
/* Tell userspace we may need a wakeup call */ /* Tell userspace we may need a wakeup call */
@@ -6815,6 +6820,24 @@ static void io_sqd_update_thread_idle(struct io_sq_data *sqd)
sqd->sq_thread_idle = sq_thread_idle; sqd->sq_thread_idle = sq_thread_idle;
} }
static bool io_sqd_handle_event(struct io_sq_data *sqd)
{
bool did_sig = false;
struct ksignal ksig;
if (test_bit(IO_SQ_THREAD_SHOULD_PARK, &sqd->state) ||
signal_pending(current)) {
mutex_unlock(&sqd->lock);
if (signal_pending(current))
did_sig = get_signal(&ksig);
cond_resched();
mutex_lock(&sqd->lock);
}
io_run_task_work();
io_run_task_work_head(&sqd->park_task_work);
return did_sig || test_bit(IO_SQ_THREAD_SHOULD_STOP, &sqd->state);
}
static int io_sq_thread(void *data) static int io_sq_thread(void *data)
{ {
struct io_sq_data *sqd = data; struct io_sq_data *sqd = data;
@@ -6836,29 +6859,17 @@ static int io_sq_thread(void *data)
/* a user may had exited before the thread started */ /* a user may had exited before the thread started */
io_run_task_work_head(&sqd->park_task_work); io_run_task_work_head(&sqd->park_task_work);
while (!test_bit(IO_SQ_THREAD_SHOULD_STOP, &sqd->state)) { while (1) {
int ret; int ret;
bool cap_entries, sqt_spin, needs_sched; bool cap_entries, sqt_spin, needs_sched;
if (test_bit(IO_SQ_THREAD_SHOULD_PARK, &sqd->state) || if (io_sqd_events_pending(sqd) || signal_pending(current)) {
signal_pending(current)) { if (io_sqd_handle_event(sqd))
bool did_sig = false;
mutex_unlock(&sqd->lock);
if (signal_pending(current)) {
struct ksignal ksig;
did_sig = get_signal(&ksig);
}
cond_resched();
mutex_lock(&sqd->lock);
io_run_task_work();
io_run_task_work_head(&sqd->park_task_work);
if (did_sig)
break; break;
timeout = jiffies + sqd->sq_thread_idle; timeout = jiffies + sqd->sq_thread_idle;
continue; continue;
} }
sqt_spin = false; sqt_spin = false;
cap_entries = !list_is_singular(&sqd->ctx_list); cap_entries = !list_is_singular(&sqd->ctx_list);
list_for_each_entry(ctx, &sqd->ctx_list, sqd_list) { list_for_each_entry(ctx, &sqd->ctx_list, sqd_list) {
@@ -6882,7 +6893,7 @@ static int io_sq_thread(void *data)
} }
prepare_to_wait(&sqd->wait, &wait, TASK_INTERRUPTIBLE); prepare_to_wait(&sqd->wait, &wait, TASK_INTERRUPTIBLE);
if (!test_bit(IO_SQ_THREAD_SHOULD_PARK, &sqd->state)) { if (!io_sqd_events_pending(sqd)) {
list_for_each_entry(ctx, &sqd->ctx_list, sqd_list) list_for_each_entry(ctx, &sqd->ctx_list, sqd_list)
io_ring_set_wakeup_flag(ctx); io_ring_set_wakeup_flag(ctx);