Minki/linux
1
0
Fork 1
mirror of https://github.com/torvalds/linux.git synced 2024-12-26 12:52:30 +00:00
Code Issues Packages Projects Releases Wiki Activity

io_uring: unlock if __io_run_local_work locked inside

Browse Source 
It is possible for tw to lock the ring, and this was not propogated out to
io_run_local_work. This can cause an unlock to be missed.

Instead pass a pointer to locked into __io_run_local_work.

Fixes: 8ac5d85a89 ("io_uring: add local task_work run helper that is entered locked")
Signed-off-by: Dylan Yudaken <dylany@meta.com>
Link: https://lore.kernel.org/r/20221027144429.3971400-3-dylany@meta.com
[axboe: WARN_ON() -> WARN_ON_ONCE() and add a minor comment]
Signed-off-by: Jens Axboe <axboe@kernel.dk>
This commit is contained in:
Dylan Yudaken 2022-10-27 07:44:29 -07:00 committed by Jens Axboe
parent 8de11cdc96
commit b3026767e1
2 changed files with 15 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
io_uring/io_uring.c
View File

@ -1173,7 +1173,7 @@ static void __cold io_move_task_work_from_local(struct io_ring_ctx *ctx)
} }
} }
int __io_run_local_work(struct io_ring_ctx *ctx, bool locked) int __io_run_local_work(struct io_ring_ctx *ctx, bool *locked)
{ {
struct llist_node *node; struct llist_node *node;
struct llist_node fake; struct llist_node fake;
@ -1192,7 +1192,7 @@ again:
struct io_kiocb *req = container_of(node, struct io_kiocb, struct io_kiocb *req = container_of(node, struct io_kiocb,
io_task_work.node); io_task_work.node);
prefetch(container_of(next, struct io_kiocb, io_task_work.node)); prefetch(container_of(next, struct io_kiocb, io_task_work.node));
req->io_task_work.func(req, &locked); req->io_task_work.func(req, locked);
ret++; ret++;
node = next; node = next;
} }
@ -1208,7 +1208,7 @@ again:
goto again; goto again;
} }
if (locked) if (*locked)
io_submit_flush_completions(ctx); io_submit_flush_completions(ctx);
trace_io_uring_local_work_run(ctx, ret, loops); trace_io_uring_local_work_run(ctx, ret, loops);
return ret; return ret;
@ -1225,7 +1225,7 @@ int io_run_local_work(struct io_ring_ctx *ctx)
__set_current_state(TASK_RUNNING); __set_current_state(TASK_RUNNING);
locked = mutex_trylock(&ctx->uring_lock); locked = mutex_trylock(&ctx->uring_lock);
ret = __io_run_local_work(ctx, locked); ret = __io_run_local_work(ctx, &locked);
if (locked) if (locked)
mutex_unlock(&ctx->uring_lock); mutex_unlock(&ctx->uring_lock);

13
io_uring/io_uring.h
View File

@ -27,7 +27,7 @@ enum {
struct io_uring_cqe *__io_get_cqe(struct io_ring_ctx *ctx, bool overflow); struct io_uring_cqe *__io_get_cqe(struct io_ring_ctx *ctx, bool overflow);
bool io_req_cqe_overflow(struct io_kiocb *req); bool io_req_cqe_overflow(struct io_kiocb *req);
int io_run_task_work_sig(struct io_ring_ctx *ctx); int io_run_task_work_sig(struct io_ring_ctx *ctx);
int __io_run_local_work(struct io_ring_ctx *ctx, bool locked); int __io_run_local_work(struct io_ring_ctx *ctx, bool *locked);
int io_run_local_work(struct io_ring_ctx *ctx); int io_run_local_work(struct io_ring_ctx *ctx);
void io_req_complete_failed(struct io_kiocb *req, s32 res); void io_req_complete_failed(struct io_kiocb *req, s32 res);
void __io_req_complete(struct io_kiocb *req, unsigned issue_flags); void __io_req_complete(struct io_kiocb *req, unsigned issue_flags);
@ -277,9 +277,18 @@ static inline int io_run_task_work_ctx(struct io_ring_ctx *ctx)
static inline int io_run_local_work_locked(struct io_ring_ctx *ctx) static inline int io_run_local_work_locked(struct io_ring_ctx *ctx)
{ {
bool locked;
int ret;
if (llist_empty(&ctx->work_llist)) if (llist_empty(&ctx->work_llist))
return 0; return 0;
return __io_run_local_work(ctx, true);
locked = true;
ret = __io_run_local_work(ctx, &locked);
/* shouldn't happen! */
if (WARN_ON_ONCE(!locked))
mutex_lock(&ctx->uring_lock);
return ret;
} }
static inline void io_tw_lock(struct io_ring_ctx *ctx, bool *locked) static inline void io_tw_lock(struct io_ring_ctx *ctx, bool *locked)