leandrof/linux
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

io_uring: hook all linked requests via link_list

Browse Source 
Links are created by chaining requests through req->list with an
exception that head uses req->link_list. (e.g. link_list->list->list)
Because of that, io_req_link_next() needs complex splicing to advance.

Link them all through list_list. Also, it seems to be simpler and more
consistent IMHO.

Signed-off-by: Pavel Begunkov <asml.silence@gmail.com>
Signed-off-by: Jens Axboe <axboe@kernel.dk>
This commit is contained in:
Pavel Begunkov 2019-12-05 16:16:35 +03:00 committed by Jens Axboe
parent 2e6e1fde32
commit 4493233edc
1 changed files with 20 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

42
fs/io_uring.c
View File

@ -916,7 +916,6 @@ static bool io_link_cancel_timeout(struct io_kiocb *req)
static void io_req_link_next(struct io_kiocb *req, struct io_kiocb **nxtptr) static void io_req_link_next(struct io_kiocb *req, struct io_kiocb **nxtptr)
{ {
struct io_ring_ctx *ctx = req->ctx; struct io_ring_ctx *ctx = req->ctx;
struct io_kiocb *nxt;
bool wake_ev = false; bool wake_ev = false;
/* Already got next link */ /* Already got next link */
@ -928,24 +927,21 @@ static void io_req_link_next(struct io_kiocb *req, struct io_kiocb **nxtptr)
* potentially happen if the chain is messed up, check to be on the * potentially happen if the chain is messed up, check to be on the
* safe side. * safe side.
*/ */
nxt = list_first_entry_or_null(&req->link_list, struct io_kiocb, list); while (!list_empty(&req->link_list)) {
while (nxt) { struct io_kiocb *nxt = list_first_entry(&req->link_list,
list_del_init(&nxt->list); struct io_kiocb, link_list);
if ((req->flags & REQ_F_LINK_TIMEOUT) && if (unlikely((req->flags & REQ_F_LINK_TIMEOUT) &&
(nxt->flags & REQ_F_TIMEOUT)) { (nxt->flags & REQ_F_TIMEOUT))) {
list_del_init(&nxt->link_list);
wake_ev |= io_link_cancel_timeout(nxt); wake_ev |= io_link_cancel_timeout(nxt);
nxt = list_first_entry_or_null(&req->link_list,
struct io_kiocb, list);
req->flags &= ~REQ_F_LINK_TIMEOUT; req->flags &= ~REQ_F_LINK_TIMEOUT;
continue; continue;
} }
if (!list_empty(&req->link_list)) {
INIT_LIST_HEAD(&nxt->link_list);
list_splice(&req->link_list, &nxt->link_list);
nxt->flags |= REQ_F_LINK;
}
list_del_init(&req->link_list);
if (!list_empty(&nxt->link_list))
nxt->flags |= REQ_F_LINK;
*nxtptr = nxt; *nxtptr = nxt;
break; break;
} }
@ -961,15 +957,15 @@ static void io_req_link_next(struct io_kiocb *req, struct io_kiocb **nxtptr)
static void io_fail_links(struct io_kiocb *req) static void io_fail_links(struct io_kiocb *req)
{ {
struct io_ring_ctx *ctx = req->ctx; struct io_ring_ctx *ctx = req->ctx;
struct io_kiocb *link;
unsigned long flags; unsigned long flags;
spin_lock_irqsave(&ctx->completion_lock, flags); spin_lock_irqsave(&ctx->completion_lock, flags);
while (!list_empty(&req->link_list)) { while (!list_empty(&req->link_list)) {
link = list_first_entry(&req->link_list, struct io_kiocb, list); struct io_kiocb *link = list_first_entry(&req->link_list,
list_del_init(&link->list); struct io_kiocb, link_list);
list_del_init(&link->link_list);
trace_io_uring_fail_link(req, link); trace_io_uring_fail_link(req, link);
if ((req->flags & REQ_F_LINK_TIMEOUT) && if ((req->flags & REQ_F_LINK_TIMEOUT) &&
@ -3170,10 +3166,11 @@ static enum hrtimer_restart io_link_timeout_fn(struct hrtimer *timer)
* We don't expect the list to be empty, that will only happen if we * We don't expect the list to be empty, that will only happen if we
* race with the completion of the linked work. * race with the completion of the linked work.
*/ */
if (!list_empty(&req->list)) { if (!list_empty(&req->link_list)) {
prev = list_entry(req->list.prev, struct io_kiocb, link_list); prev = list_entry(req->link_list.prev, struct io_kiocb,
link_list);
if (refcount_inc_not_zero(&prev->refs)) { if (refcount_inc_not_zero(&prev->refs)) {
list_del_init(&req->list); list_del_init(&req->link_list);
prev->flags &= ~REQ_F_LINK_TIMEOUT; prev->flags &= ~REQ_F_LINK_TIMEOUT;
} else } else
prev = NULL; prev = NULL;
@ -3203,7 +3200,7 @@ static void io_queue_linked_timeout(struct io_kiocb *req)
* we got a chance to setup the timer * we got a chance to setup the timer
*/ */
spin_lock_irq(&ctx->completion_lock); spin_lock_irq(&ctx->completion_lock);
if (!list_empty(&req->list)) { if (!list_empty(&req->link_list)) {
struct io_timeout_data *data = &req->io->timeout; struct io_timeout_data *data = &req->io->timeout;
data->timer.function = io_link_timeout_fn; data->timer.function = io_link_timeout_fn;
@ -3223,7 +3220,8 @@ static struct io_kiocb *io_prep_linked_timeout(struct io_kiocb *req)
if (!(req->flags & REQ_F_LINK)) if (!(req->flags & REQ_F_LINK))
return NULL; return NULL;
nxt = list_first_entry_or_null(&req->link_list, struct io_kiocb, list); nxt = list_first_entry_or_null(&req->link_list, struct io_kiocb,
link_list);
if (!nxt || nxt->sqe->opcode != IORING_OP_LINK_TIMEOUT) if (!nxt || nxt->sqe->opcode != IORING_OP_LINK_TIMEOUT)
return NULL; return NULL;
@ -3364,7 +3362,7 @@ err_req:
goto err_req; goto err_req;
} }
trace_io_uring_link(ctx, req, prev); trace_io_uring_link(ctx, req, prev);
list_add_tail(&req->list, &prev->link_list); list_add_tail(&req->link_list, &prev->link_list);
} else if (req->sqe->flags & IOSQE_IO_LINK) { } else if (req->sqe->flags & IOSQE_IO_LINK) {
req->flags |= REQ_F_LINK; req->flags |= REQ_F_LINK;