leandrof/linux
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

io_uring: simplify io_rsrc_data refcounting

Browse Source 
We don't take many references of struct io_rsrc_data, only one per each
io_rsrc_node, so using percpu refs is overkill. Use atomic ref instead,
which is much simpler.

Signed-off-by: Pavel Begunkov <asml.silence@gmail.com>
Link: https://lore.kernel.org/r/1551d90f7c9b183cf2f0d7b5e5b923430acb03fa.1618101759.git.asml.silence@gmail.com
Signed-off-by: Jens Axboe <axboe@kernel.dk>
This commit is contained in:
Pavel Begunkov
2021-04-11 01:46:34 +01:00
committed by Jens Axboe
parent a1ff1e3f0e
commit 3e9424989b
1 changed files with 16 additions and 32 deletions
Download Patch File Download Diff File Expand all files Collapse all files

48
fs/io_uring.c
View File

@@ -240,7 +240,7 @@ struct io_rsrc_data {
struct io_ring_ctx *ctx; struct io_ring_ctx *ctx;
rsrc_put_fn *do_put; rsrc_put_fn *do_put;
struct percpu_ref refs; atomic_t refs;
struct completion done; struct completion done;
bool quiesce; bool quiesce;
}; };
@@ -7077,13 +7077,6 @@ static void __io_sqe_files_unregister(struct io_ring_ctx *ctx)
#endif #endif
} }
static void io_rsrc_data_ref_zero(struct percpu_ref *ref)
{
struct io_rsrc_data *data = container_of(ref, struct io_rsrc_data, refs);
complete(&data->done);
}
static inline void io_rsrc_ref_lock(struct io_ring_ctx *ctx) static inline void io_rsrc_ref_lock(struct io_ring_ctx *ctx)
{ {
spin_lock_bh(&ctx->rsrc_ref_lock); spin_lock_bh(&ctx->rsrc_ref_lock);
@@ -7114,7 +7107,7 @@ static void io_rsrc_node_switch(struct io_ring_ctx *ctx,
list_add_tail(&rsrc_node->node, &ctx->rsrc_ref_list); list_add_tail(&rsrc_node->node, &ctx->rsrc_ref_list);
io_rsrc_ref_unlock(ctx); io_rsrc_ref_unlock(ctx);
percpu_ref_get(&data_to_kill->refs); atomic_inc(&data_to_kill->refs);
percpu_ref_kill(&rsrc_node->refs); percpu_ref_kill(&rsrc_node->refs);
ctx->rsrc_node = NULL; ctx->rsrc_node = NULL;
} }
@@ -7148,14 +7141,17 @@ static int io_rsrc_ref_quiesce(struct io_rsrc_data *data, struct io_ring_ctx *ct
break; break;
io_rsrc_node_switch(ctx, data); io_rsrc_node_switch(ctx, data);
percpu_ref_kill(&data->refs); /* kill initial ref, already quiesced if zero */
if (atomic_dec_and_test(&data->refs))
break;
flush_delayed_work(&ctx->rsrc_put_work); flush_delayed_work(&ctx->rsrc_put_work);
ret = wait_for_completion_interruptible(&data->done); ret = wait_for_completion_interruptible(&data->done);
if (!ret) if (!ret)
break; break;
percpu_ref_resurrect(&data->refs); atomic_inc(&data->refs);
/* wait for all works potentially completing data->done */
flush_delayed_work(&ctx->rsrc_put_work);
reinit_completion(&data->done); reinit_completion(&data->done);
mutex_unlock(&ctx->uring_lock); mutex_unlock(&ctx->uring_lock);
@@ -7176,23 +7172,13 @@ static struct io_rsrc_data *io_rsrc_data_alloc(struct io_ring_ctx *ctx,
if (!data) if (!data)
return NULL; return NULL;
if (percpu_ref_init(&data->refs, io_rsrc_data_ref_zero, atomic_set(&data->refs, 1);
PERCPU_REF_ALLOW_REINIT, GFP_KERNEL)) {
kfree(data);
return NULL;
}
data->ctx = ctx; data->ctx = ctx;
data->do_put = do_put; data->do_put = do_put;
init_completion(&data->done); init_completion(&data->done);
return data; return data;
} }
static void io_rsrc_data_free(struct io_rsrc_data *data)
{
percpu_ref_exit(&data->refs);
kfree(data);
}
static int io_sqe_files_unregister(struct io_ring_ctx *ctx) static int io_sqe_files_unregister(struct io_ring_ctx *ctx)
{ {
struct io_rsrc_data *data = ctx->file_data; struct io_rsrc_data *data = ctx->file_data;
@@ -7206,7 +7192,7 @@ static int io_sqe_files_unregister(struct io_ring_ctx *ctx)
__io_sqe_files_unregister(ctx); __io_sqe_files_unregister(ctx);
io_free_file_tables(data, ctx->nr_user_files); io_free_file_tables(data, ctx->nr_user_files);
io_rsrc_data_free(data); kfree(data);
ctx->file_data = NULL; ctx->file_data = NULL;
ctx->nr_user_files = 0; ctx->nr_user_files = 0;
return 0; return 0;
@@ -7539,7 +7525,8 @@ static void __io_rsrc_put_work(struct io_rsrc_node *ref_node)
} }
io_rsrc_node_destroy(ref_node); io_rsrc_node_destroy(ref_node);
percpu_ref_put(&rsrc_data->refs); if (atomic_dec_and_test(&rsrc_data->refs))
complete(&rsrc_data->done);
} }
static void io_rsrc_put_work(struct work_struct *work) static void io_rsrc_put_work(struct work_struct *work)
@@ -7563,10 +7550,8 @@ static void io_rsrc_put_work(struct work_struct *work)
static void io_rsrc_node_ref_zero(struct percpu_ref *ref) static void io_rsrc_node_ref_zero(struct percpu_ref *ref)
{ {
struct io_rsrc_node *node = container_of(ref, struct io_rsrc_node, refs); struct io_rsrc_node *node = container_of(ref, struct io_rsrc_node, refs);
struct io_rsrc_data *data = node->rsrc_data; struct io_ring_ctx *ctx = node->rsrc_data->ctx;
struct io_ring_ctx *ctx = data->ctx;
bool first_add = false; bool first_add = false;
int delay;
io_rsrc_ref_lock(ctx); io_rsrc_ref_lock(ctx);
node->done = true; node->done = true;
@@ -7582,9 +7567,8 @@ static void io_rsrc_node_ref_zero(struct percpu_ref *ref)
} }
io_rsrc_ref_unlock(ctx); io_rsrc_ref_unlock(ctx);
delay = percpu_ref_is_dying(&data->refs) ? 0 : HZ; if (first_add)
if (first_add || !delay) mod_delayed_work(system_wq, &ctx->rsrc_put_work, HZ);
mod_delayed_work(system_wq, &ctx->rsrc_put_work, delay);
} }
static struct io_rsrc_node *io_rsrc_node_alloc(struct io_ring_ctx *ctx) static struct io_rsrc_node *io_rsrc_node_alloc(struct io_ring_ctx *ctx)
@@ -7679,7 +7663,7 @@ out_fput:
io_free_file_tables(file_data, nr_args); io_free_file_tables(file_data, nr_args);
ctx->nr_user_files = 0; ctx->nr_user_files = 0;
out_free: out_free:
io_rsrc_data_free(ctx->file_data); kfree(ctx->file_data);
ctx->file_data = NULL; ctx->file_data = NULL;
return ret; return ret;
} }