Minki/linux
1
0
Fork 1
mirror of https://github.com/torvalds/linux.git synced 2024-12-25 20:32:22 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'for-4.4/io-poll' of git://git.kernel.dk/linux-block

Browse Source 
Pull block IO poll support from Jens Axboe:
 "Various groups have been doing experimentation around IO polling for
  (really) fast devices.  The code has been reviewed and has been
  sitting on the side for a few releases, but this is now good enough
  for coordinated benchmarking and further experimentation.

  Currently O_DIRECT sync read/write are supported.  A framework is in
  the works that allows scalable stats tracking so we can auto-tune
  this.  And we'll add libaio support as well soon.  Fow now, it's an
  opt-in feature for test purposes"

* 'for-4.4/io-poll' of git://git.kernel.dk/linux-block:
  direct-io: be sure to assign dio->bio_bdev for both paths
  directio: add block polling support
  NVMe: add blk polling support
  block: add block polling support
  blk-mq: return tag/queue combo in the make_request_fn handlers
  block: change ->make_request_fn() and users to return a queue cookie
This commit is contained in:
Linus Torvalds 2015-11-10 17:23:49 -08:00
commit 3419b45039
33 changed files with 287 additions and 91 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
arch/m68k/emu/nfblock.c
View File

@ -59,7 +59,7 @@ struct nfhd_device {
struct gendisk *disk;
};
static void nfhd_make_request(struct request_queue *queue, struct bio *bio)
static blk_qc_t nfhd_make_request(struct request_queue *queue, struct bio *bio)
{
struct nfhd_device *dev = queue->queuedata;
struct bio_vec bvec;
@ -77,6 +77,7 @@ static void nfhd_make_request(struct request_queue *queue, struct bio *bio)
sec += len;
}
bio_endio(bio);
return BLK_QC_T_NONE;
}
static int nfhd_getgeo(struct block_device *bdev, struct hd_geometry *geo)

5
arch/powerpc/sysdev/axonram.c
View File

@ -103,7 +103,7 @@ axon_ram_irq_handler(int irq, void *dev)
* axon_ram_make_request - make_request() method for block device
* @queue, @bio: see blk_queue_make_request()
*/
static void
static blk_qc_t
axon_ram_make_request(struct request_queue *queue, struct bio *bio)
{
struct axon_ram_bank *bank = bio->bi_bdev->bd_disk->private_data;
@ -120,7 +120,7 @@ axon_ram_make_request(struct request_queue *queue, struct bio *bio)
bio_for_each_segment(vec, bio, iter) {
if (unlikely(phys_mem + vec.bv_len > phys_end)) {
bio_io_error(bio);
return;
return BLK_QC_T_NONE;
}
user_mem = page_address(vec.bv_page) + vec.bv_offset;
@ -133,6 +133,7 @@ axon_ram_make_request(struct request_queue *queue, struct bio *bio)
transfered += vec.bv_len;
}
bio_endio(bio);
return BLK_QC_T_NONE;
}
/**

3
arch/xtensa/platforms/iss/simdisk.c
View File

@ -101,7 +101,7 @@ static void simdisk_transfer(struct simdisk *dev, unsigned long sector,
spin_unlock(&dev->lock);
}
static void simdisk_make_request(struct request_queue *q, struct bio *bio)
static blk_qc_t simdisk_make_request(struct request_queue *q, struct bio *bio)
{
struct simdisk *dev = q->queuedata;
struct bio_vec bvec;
@ -119,6 +119,7 @@ static void simdisk_make_request(struct request_queue *q, struct bio *bio)
}
bio_endio(bio);
return BLK_QC_T_NONE;
}
static int simdisk_open(struct block_device *bdev, fmode_t mode)

67
block/blk-core.c
View File

@ -809,7 +809,7 @@ blk_init_queue_node(request_fn_proc *rfn, spinlock_t *lock, int node_id)
}
EXPORT_SYMBOL(blk_init_queue_node);
static void blk_queue_bio(struct request_queue *q, struct bio *bio);
static blk_qc_t blk_queue_bio(struct request_queue *q, struct bio *bio);
struct request_queue *
blk_init_allocated_queue(struct request_queue *q, request_fn_proc *rfn,
@ -1678,7 +1678,7 @@ void init_request_from_bio(struct request *req, struct bio *bio)
blk_rq_bio_prep(req->q, req, bio);
}
static void blk_queue_bio(struct request_queue *q, struct bio *bio)
static blk_qc_t blk_queue_bio(struct request_queue *q, struct bio *bio)
{
const bool sync = !!(bio->bi_rw & REQ_SYNC);
struct blk_plug *plug;
@ -1698,7 +1698,7 @@ static void blk_queue_bio(struct request_queue *q, struct bio *bio)
if (bio_integrity_enabled(bio) && bio_integrity_prep(bio)) {
bio->bi_error = -EIO;
bio_endio(bio);
return;
return BLK_QC_T_NONE;
}
if (bio->bi_rw & (REQ_FLUSH | REQ_FUA)) {
@ -1713,7 +1713,7 @@ static void blk_queue_bio(struct request_queue *q, struct bio *bio)
*/
if (!blk_queue_nomerges(q)) {
if (blk_attempt_plug_merge(q, bio, &request_count, NULL))
return;
return BLK_QC_T_NONE;
} else
request_count = blk_plug_queued_count(q);
@ -1791,6 +1791,8 @@ get_rq:
out_unlock:
spin_unlock_irq(q->queue_lock);
}
return BLK_QC_T_NONE;
}
/*
@ -1996,12 +1998,13 @@ end_io:
* a lower device by calling into generic_make_request recursively, which
* means the bio should NOT be touched after the call to ->make_request_fn.
*/
void generic_make_request(struct bio *bio)
blk_qc_t generic_make_request(struct bio *bio)
{
struct bio_list bio_list_on_stack;
blk_qc_t ret = BLK_QC_T_NONE;
if (!generic_make_request_checks(bio))
return;
goto out;
/*
* We only want one ->make_request_fn to be active at a time, else
@ -2015,7 +2018,7 @@ void generic_make_request(struct bio *bio)
*/
if (current->bio_list) {
bio_list_add(current->bio_list, bio);
return;
goto out;
}
/* following loop may be a bit non-obvious, and so deserves some
@ -2040,7 +2043,7 @@ void generic_make_request(struct bio *bio)
if (likely(blk_queue_enter(q, __GFP_DIRECT_RECLAIM) == 0)) {
q->make_request_fn(q, bio);
ret = q->make_request_fn(q, bio);
blk_queue_exit(q);
@ -2053,6 +2056,9 @@ void generic_make_request(struct bio *bio)
}
} while (bio);
current->bio_list = NULL; /* deactivate */
out:
return ret;
}
EXPORT_SYMBOL(generic_make_request);
@ -2066,7 +2072,7 @@ EXPORT_SYMBOL(generic_make_request);
* interfaces; @bio must be presetup and ready for I/O.
*
*/
void submit_bio(int rw, struct bio *bio)
blk_qc_t submit_bio(int rw, struct bio *bio)
{
bio->bi_rw |= rw;
@ -2100,7 +2106,7 @@ void submit_bio(int rw, struct bio *bio)
}
}
generic_make_request(bio);
return generic_make_request(bio);
}
EXPORT_SYMBOL(submit_bio);
@ -3306,6 +3312,47 @@ void blk_finish_plug(struct blk_plug *plug)
}
EXPORT_SYMBOL(blk_finish_plug);
bool blk_poll(struct request_queue *q, blk_qc_t cookie)
{
struct blk_plug *plug;
long state;
if (!q->mq_ops || !q->mq_ops->poll || !blk_qc_t_valid(cookie) ||
!test_bit(QUEUE_FLAG_POLL, &q->queue_flags))
return false;
plug = current->plug;
if (plug)
blk_flush_plug_list(plug, false);
state = current->state;
while (!need_resched()) {
unsigned int queue_num = blk_qc_t_to_queue_num(cookie);
struct blk_mq_hw_ctx *hctx = q->queue_hw_ctx[queue_num];
int ret;
hctx->poll_invoked++;
ret = q->mq_ops->poll(hctx, blk_qc_t_to_tag(cookie));
if (ret > 0) {
hctx->poll_success++;
set_current_state(TASK_RUNNING);
return true;
}
if (signal_pending_state(state, current))
set_current_state(TASK_RUNNING);
if (current->state == TASK_RUNNING)
return true;
if (ret < 0)
break;
cpu_relax();
}
return false;
}
#ifdef CONFIG_PM
/**
* blk_pm_runtime_init - Block layer runtime PM initialization routine

10
block/blk-mq-sysfs.c
View File

@ -174,6 +174,11 @@ static ssize_t blk_mq_sysfs_rq_list_show(struct blk_mq_ctx *ctx, char *page)
return ret;
}
static ssize_t blk_mq_hw_sysfs_poll_show(struct blk_mq_hw_ctx *hctx, char *page)
{
return sprintf(page, "invoked=%lu, success=%lu\n", hctx->poll_invoked, hctx->poll_success);
}
static ssize_t blk_mq_hw_sysfs_queued_show(struct blk_mq_hw_ctx *hctx,
char *page)
{
@ -295,6 +300,10 @@ static struct blk_mq_hw_ctx_sysfs_entry blk_mq_hw_sysfs_cpus = {
.attr = {.name = "cpu_list", .mode = S_IRUGO },
.show = blk_mq_hw_sysfs_cpus_show,
};
static struct blk_mq_hw_ctx_sysfs_entry blk_mq_hw_sysfs_poll = {
.attr = {.name = "io_poll", .mode = S_IRUGO },
.show = blk_mq_hw_sysfs_poll_show,
};
static struct attribute *default_hw_ctx_attrs[] = {
&blk_mq_hw_sysfs_queued.attr,
@ -304,6 +313,7 @@ static struct attribute *default_hw_ctx_attrs[] = {
&blk_mq_hw_sysfs_tags.attr,
&blk_mq_hw_sysfs_cpus.attr,
&blk_mq_hw_sysfs_active.attr,
&blk_mq_hw_sysfs_poll.attr,
NULL,
};

61
block/blk-mq.c
View File

@ -1198,7 +1198,7 @@ static struct request *blk_mq_map_request(struct request_queue *q,
return rq;
}
static int blk_mq_direct_issue_request(struct request *rq)
static int blk_mq_direct_issue_request(struct request *rq, blk_qc_t *cookie)
{
int ret;
struct request_queue *q = rq->q;
@ -1209,6 +1209,7 @@ static int blk_mq_direct_issue_request(struct request *rq)
.list = NULL,
.last = 1
};
blk_qc_t new_cookie = blk_tag_to_qc_t(rq->tag, hctx->queue_num);
/*
* For OK queue, we are done. For error, kill it. Any other
@ -1216,18 +1217,21 @@ static int blk_mq_direct_issue_request(struct request *rq)
* would have done
*/
ret = q->mq_ops->queue_rq(hctx, &bd);
if (ret == BLK_MQ_RQ_QUEUE_OK)
if (ret == BLK_MQ_RQ_QUEUE_OK) {
*cookie = new_cookie;
return 0;
else {
__blk_mq_requeue_request(rq);
if (ret == BLK_MQ_RQ_QUEUE_ERROR) {
rq->errors = -EIO;
blk_mq_end_request(rq, rq->errors);
return 0;
}
return -1;
}
__blk_mq_requeue_request(rq);
if (ret == BLK_MQ_RQ_QUEUE_ERROR) {
*cookie = BLK_QC_T_NONE;
rq->errors = -EIO;
blk_mq_end_request(rq, rq->errors);
return 0;
}
return -1;
}
/*
@ -1235,7 +1239,7 @@ static int blk_mq_direct_issue_request(struct request *rq)
* but will attempt to bypass the hctx queueing if we can go straight to
* hardware for SYNC IO.
*/
static void blk_mq_make_request(struct request_queue *q, struct bio *bio)
static blk_qc_t blk_mq_make_request(struct request_queue *q, struct bio *bio)
{
const int is_sync = rw_is_sync(bio->bi_rw);
const int is_flush_fua = bio->bi_rw & (REQ_FLUSH | REQ_FUA);
@ -1244,12 +1248,13 @@ static void blk_mq_make_request(struct request_queue *q, struct bio *bio)
unsigned int request_count = 0;
struct blk_plug *plug;
struct request *same_queue_rq = NULL;
blk_qc_t cookie;
blk_queue_bounce(q, &bio);
if (bio_integrity_enabled(bio) && bio_integrity_prep(bio)) {
bio_io_error(bio);
return;
return BLK_QC_T_NONE;
}
blk_queue_split(q, &bio, q->bio_split);
@ -1257,13 +1262,15 @@ static void blk_mq_make_request(struct request_queue *q, struct bio *bio)
if (!is_flush_fua && !blk_queue_nomerges(q)) {
if (blk_attempt_plug_merge(q, bio, &request_count,
&same_queue_rq))
return;
return BLK_QC_T_NONE;
} else
request_count = blk_plug_queued_count(q);
rq = blk_mq_map_request(q, bio, &data);
if (unlikely(!rq))
return;
return BLK_QC_T_NONE;
cookie = blk_tag_to_qc_t(rq->tag, data.hctx->queue_num);
if (unlikely(is_flush_fua)) {
blk_mq_bio_to_request(rq, bio);
@ -1302,11 +1309,11 @@ static void blk_mq_make_request(struct request_queue *q, struct bio *bio)
old_rq = rq;
blk_mq_put_ctx(data.ctx);
if (!old_rq)
return;
if (!blk_mq_direct_issue_request(old_rq))
return;
goto done;
if (!blk_mq_direct_issue_request(old_rq, &cookie))
goto done;
blk_mq_insert_request(old_rq, false, true, true);
return;
goto done;
}
if (!blk_mq_merge_queue_io(data.hctx, data.ctx, rq, bio)) {
@ -1320,13 +1327,15 @@ run_queue:
blk_mq_run_hw_queue(data.hctx, !is_sync || is_flush_fua);
}
blk_mq_put_ctx(data.ctx);
done:
return cookie;
}
/*
* Single hardware queue variant. This will attempt to use any per-process
* plug for merging and IO deferral.
*/
static void blk_sq_make_request(struct request_queue *q, struct bio *bio)
static blk_qc_t blk_sq_make_request(struct request_queue *q, struct bio *bio)
{
const int is_sync = rw_is_sync(bio->bi_rw);
const int is_flush_fua = bio->bi_rw & (REQ_FLUSH | REQ_FUA);
@ -1334,23 +1343,26 @@ static void blk_sq_make_request(struct request_queue *q, struct bio *bio)
unsigned int request_count = 0;
struct blk_map_ctx data;
struct request *rq;
blk_qc_t cookie;
blk_queue_bounce(q, &bio);
if (bio_integrity_enabled(bio) && bio_integrity_prep(bio)) {
bio_io_error(bio);
return;
return BLK_QC_T_NONE;
}
blk_queue_split(q, &bio, q->bio_split);
if (!is_flush_fua && !blk_queue_nomerges(q) &&
blk_attempt_plug_merge(q, bio, &request_count, NULL))
return;
return BLK_QC_T_NONE;
rq = blk_mq_map_request(q, bio, &data);
if (unlikely(!rq))
return;
return BLK_QC_T_NONE;
cookie = blk_tag_to_qc_t(rq->tag, data.hctx->queue_num);
if (unlikely(is_flush_fua)) {
blk_mq_bio_to_request(rq, bio);
@ -1374,7 +1386,7 @@ static void blk_sq_make_request(struct request_queue *q, struct bio *bio)
}
list_add_tail(&rq->queuelist, &plug->mq_list);
blk_mq_put_ctx(data.ctx);
return;
return cookie;
}
if (!blk_mq_merge_queue_io(data.hctx, data.ctx, rq, bio)) {
@ -1389,6 +1401,7 @@ run_queue:
}
blk_mq_put_ctx(data.ctx);
return cookie;
}
/*

35
block/blk-sysfs.c
View File

@ -317,6 +317,34 @@ queue_rq_affinity_store(struct request_queue *q, const char *page, size_t count)
return ret;
}
static ssize_t queue_poll_show(struct request_queue *q, char *page)
{
return queue_var_show(test_bit(QUEUE_FLAG_POLL, &q->queue_flags), page);
}
static ssize_t queue_poll_store(struct request_queue *q, const char *page,
size_t count)
{
unsigned long poll_on;
ssize_t ret;
if (!q->mq_ops || !q->mq_ops->poll)
return -EINVAL;
ret = queue_var_store(&poll_on, page, count);
if (ret < 0)
return ret;
spin_lock_irq(q->queue_lock);
if (poll_on)
queue_flag_set(QUEUE_FLAG_POLL, q);
else
queue_flag_clear(QUEUE_FLAG_POLL, q);
spin_unlock_irq(q->queue_lock);
return ret;
}
static struct queue_sysfs_entry queue_requests_entry = {
.attr = {.name = "nr_requests", .mode = S_IRUGO | S_IWUSR },
.show = queue_requests_show,
@ -442,6 +470,12 @@ static struct queue_sysfs_entry queue_random_entry = {
.store = queue_store_random,
};
static struct queue_sysfs_entry queue_poll_entry = {
.attr = {.name = "io_poll", .mode = S_IRUGO | S_IWUSR },
.show = queue_poll_show,
.store = queue_poll_store,
};
static struct attribute *default_attrs[] = {
&queue_requests_entry.attr,
&queue_ra_entry.attr,
@ -466,6 +500,7 @@ static struct attribute *default_attrs[] = {
&queue_rq_affinity_entry.attr,
&queue_iostats_entry.attr,
&queue_random_entry.attr,
&queue_poll_entry.attr,
NULL,
};

5
drivers/block/brd.c
View File

@ -323,7 +323,7 @@ out:
return err;
}
static void brd_make_request(struct request_queue *q, struct bio *bio)
static blk_qc_t brd_make_request(struct request_queue *q, struct bio *bio)
{
struct block_device *bdev = bio->bi_bdev;
struct brd_device *brd = bdev->bd_disk->private_data;
@ -358,9 +358,10 @@ static void brd_make_request(struct request_queue *q, struct bio *bio)
out:
bio_endio(bio);
return;
return BLK_QC_T_NONE;
io_error:
bio_io_error(bio);
return BLK_QC_T_NONE;
}
static int brd_rw_page(struct block_device *bdev, sector_t sector,

2
drivers/block/drbd/drbd_int.h
View File

@ -1448,7 +1448,7 @@ extern int proc_details;
/* drbd_req */
extern void do_submit(struct work_struct *ws);
extern void __drbd_make_request(struct drbd_device *, struct bio *, unsigned long);
extern void drbd_make_request(struct request_queue *q, struct bio *bio);
extern blk_qc_t drbd_make_request(struct request_queue *q, struct bio *bio);
extern int drbd_read_remote(struct drbd_device *device, struct drbd_request *req);
extern int is_valid_ar_handle(struct drbd_request *, sector_t);

3
drivers/block/drbd/drbd_req.c
View File

@ -1494,7 +1494,7 @@ void do_submit(struct work_struct *ws)
}
}
void drbd_make_request(struct request_queue *q, struct bio *bio)
blk_qc_t drbd_make_request(struct request_queue *q, struct bio *bio)
{
struct drbd_device *device = (struct drbd_device *) q->queuedata;
unsigned long start_jif;
@ -1510,6 +1510,7 @@ void drbd_make_request(struct request_queue *q, struct bio *bio)
inc_ap_bio(device);
__drbd_make_request(device, bio, start_jif);
return BLK_QC_T_NONE;
}
void request_timer_fn(unsigned long data)

3
drivers/block/null_blk.c
View File

@ -321,7 +321,7 @@ static struct nullb_queue *nullb_to_queue(struct nullb *nullb)
return &nullb->queues[index];
}
static void null_queue_bio(struct request_queue *q, struct bio *bio)
static blk_qc_t null_queue_bio(struct request_queue *q, struct bio *bio)
{
struct nullb *nullb = q->queuedata;
struct nullb_queue *nq = nullb_to_queue(nullb);
@ -331,6 +331,7 @@ static void null_queue_bio(struct request_queue *q, struct bio *bio)
cmd->bio = bio;
null_handle_cmd(cmd);
return BLK_QC_T_NONE;
}
static int null_rq_prep_fn(struct request_queue *q, struct request *req)

9
drivers/block/pktcdvd.c
View File

@ -2441,7 +2441,7 @@ static void pkt_make_request_write(struct request_queue *q, struct bio *bio)
}
}
static void pkt_make_request(struct request_queue *q, struct bio *bio)
static blk_qc_t pkt_make_request(struct request_queue *q, struct bio *bio)
{
struct pktcdvd_device *pd;
char b[BDEVNAME_SIZE];
@ -2467,7 +2467,7 @@ static void pkt_make_request(struct request_queue *q, struct bio *bio)
*/
if (bio_data_dir(bio) == READ) {
pkt_make_request_read(pd, bio);
return;
return BLK_QC_T_NONE;
}
if (!test_bit(PACKET_WRITABLE, &pd->flags)) {
@ -2499,13 +2499,12 @@ static void pkt_make_request(struct request_queue *q, struct bio *bio)
pkt_make_request_write(q, split);
} while (split != bio);
return;
return BLK_QC_T_NONE;
end_io:
bio_io_error(bio);
return BLK_QC_T_NONE;
}
static void pkt_init_queue(struct pktcdvd_device *pd)
{
struct request_queue *q = pd->disk->queue;

6
drivers/block/ps3vram.c
View File

@ -598,7 +598,7 @@ out:
return next;
}
static void ps3vram_make_request(struct request_queue *q, struct bio *bio)
static blk_qc_t ps3vram_make_request(struct request_queue *q, struct bio *bio)
{
struct ps3_system_bus_device *dev = q->queuedata;
struct ps3vram_priv *priv = ps3_system_bus_get_drvdata(dev);
@ -614,11 +614,13 @@ static void ps3vram_make_request(struct request_queue *q, struct bio *bio)
spin_unlock_irq(&priv->lock);
if (busy)
return;
return BLK_QC_T_NONE;
do {
bio = ps3vram_do_bio(dev, bio);
} while (bio);
return BLK_QC_T_NONE;
}
static int ps3vram_probe(struct ps3_system_bus_device *dev)

5
drivers/block/rsxx/dev.c
View File

@ -145,7 +145,7 @@ static void bio_dma_done_cb(struct rsxx_cardinfo *card,
}
}
static void rsxx_make_request(struct request_queue *q, struct bio *bio)
static blk_qc_t rsxx_make_request(struct request_queue *q, struct bio *bio)
{
struct rsxx_cardinfo *card = q->queuedata;
struct rsxx_bio_meta *bio_meta;
@ -199,7 +199,7 @@ static void rsxx_make_request(struct request_queue *q, struct bio *bio)
if (st)
goto queue_err;
return;
return BLK_QC_T_NONE;
queue_err:
kmem_cache_free(bio_meta_pool, bio_meta);
@ -207,6 +207,7 @@ req_err:
if (st)
bio->bi_error = st;
bio_endio(bio);
return BLK_QC_T_NONE;
}
/*----------------- Device Setup -------------------*/

4
drivers/block/umem.c
View File

@ -524,7 +524,7 @@ static int mm_check_plugged(struct cardinfo *card)
return !!blk_check_plugged(mm_unplug, card, sizeof(struct blk_plug_cb));
}
static void mm_make_request(struct request_queue *q, struct bio *bio)
static blk_qc_t mm_make_request(struct request_queue *q, struct bio *bio)
{
struct cardinfo *card = q->queuedata;
pr_debug("mm_make_request %llu %u\n",
@ -541,7 +541,7 @@ static void mm_make_request(struct request_queue *q, struct bio *bio)
activate(card);
spin_unlock_irq(&card->lock);
return;
return BLK_QC_T_NONE;
}
static irqreturn_t mm_interrupt(int irq, void *__card)

5
drivers/block/zram/zram_drv.c
View File

@ -894,7 +894,7 @@ out:
/*
* Handler function for all zram I/O requests.
*/
static void zram_make_request(struct request_queue *queue, struct bio *bio)
static blk_qc_t zram_make_request(struct request_queue *queue, struct bio *bio)
{
struct zram *zram = queue->queuedata;
@ -911,11 +911,12 @@ static void zram_make_request(struct request_queue *queue, struct bio *bio)
__zram_make_request(zram, bio);
zram_meta_put(zram);
return;
return BLK_QC_T_NONE;
put_zram:
zram_meta_put(zram);
error:
bio_io_error(bio);
return BLK_QC_T_NONE;
}
static void zram_slot_free_notify(struct block_device *bdev,

9
drivers/lightnvm/rrpc.c
View File

@ -803,7 +803,7 @@ static int rrpc_submit_io(struct rrpc *rrpc, struct bio *bio,
return NVM_IO_OK;
}
static void rrpc_make_rq(struct request_queue *q, struct bio *bio)
static blk_qc_t rrpc_make_rq(struct request_queue *q, struct bio *bio)
{
struct rrpc *rrpc = q->queuedata;
struct nvm_rq *rqd;
@ -811,21 +811,21 @@ static void rrpc_make_rq(struct request_queue *q, struct bio *bio)
if (bio->bi_rw & REQ_DISCARD) {
rrpc_discard(rrpc, bio);
return;
return BLK_QC_T_NONE;
}
rqd = mempool_alloc(rrpc->rq_pool, GFP_KERNEL);
if (!rqd) {
pr_err_ratelimited("rrpc: not able to queue bio.");
bio_io_error(bio);
return;
return BLK_QC_T_NONE;
}
memset(rqd, 0, sizeof(struct nvm_rq));
err = rrpc_submit_io(rrpc, bio, rqd, NVM_IOTYPE_NONE);
switch (err) {
case NVM_IO_OK:
return;
return BLK_QC_T_NONE;
case NVM_IO_ERR:
bio_io_error(bio);
break;
@ -841,6 +841,7 @@ static void rrpc_make_rq(struct request_queue *q, struct bio *bio)
}
mempool_free(rqd, rrpc->rq_pool);
return BLK_QC_T_NONE;
}
static void rrpc_requeue(struct work_struct *work)

11
drivers/md/bcache/request.c
View File

@ -958,7 +958,8 @@ static void cached_dev_nodata(struct closure *cl)
/* Cached devices - read & write stuff */
static void cached_dev_make_request(struct request_queue *q, struct bio *bio)
static blk_qc_t cached_dev_make_request(struct request_queue *q,
struct bio *bio)
{
struct search *s;
struct bcache_device *d = bio->bi_bdev->bd_disk->private_data;
@ -997,6 +998,8 @@ static void cached_dev_make_request(struct request_queue *q, struct bio *bio)
else
generic_make_request(bio);
}
return BLK_QC_T_NONE;
}
static int cached_dev_ioctl(struct bcache_device *d, fmode_t mode,
@ -1070,7 +1073,8 @@ static void flash_dev_nodata(struct closure *cl)
continue_at(cl, search_free, NULL);
}
static void flash_dev_make_request(struct request_queue *q, struct bio *bio)
static blk_qc_t flash_dev_make_request(struct request_queue *q,
struct bio *bio)
{
struct search *s;
struct closure *cl;
@ -1093,7 +1097,7 @@ static void flash_dev_make_request(struct request_queue *q, struct bio *bio)
continue_at_nobarrier(&s->cl,
flash_dev_nodata,
bcache_wq);
return;
return BLK_QC_T_NONE;
} else if (rw) {
bch_keybuf_check_overlapping(&s->iop.c->moving_gc_keys,
&KEY(d->id, bio->bi_iter.bi_sector, 0),
@ -1109,6 +1113,7 @@ static void flash_dev_make_request(struct request_queue *q, struct bio *bio)
}
continue_at(cl, search_free, NULL);
return BLK_QC_T_NONE;
}
static int flash_dev_ioctl(struct bcache_device *d, fmode_t mode,

6
drivers/md/dm.c
View File

@ -1755,7 +1755,7 @@ static void __split_and_process_bio(struct mapped_device *md,
* The request function that just remaps the bio built up by
* dm_merge_bvec.
*/
static void dm_make_request(struct request_queue *q, struct bio *bio)
static blk_qc_t dm_make_request(struct request_queue *q, struct bio *bio)
{
int rw = bio_data_dir(bio);
struct mapped_device *md = q->queuedata;
@ -1774,12 +1774,12 @@ static void dm_make_request(struct request_queue *q, struct bio *bio)
queue_io(md, bio);
else
bio_io_error(bio);
return;
return BLK_QC_T_NONE;
}
__split_and_process_bio(md, map, bio);
dm_put_live_table(md, srcu_idx);
return;
return BLK_QC_T_NONE;
}
int dm_request_based(struct mapped_device *md)

8
drivers/md/md.c
View File

@ -250,7 +250,7 @@ static DEFINE_SPINLOCK(all_mddevs_lock);
* call has finished, the bio has been linked into some internal structure
* and so is visible to ->quiesce(), so we don't need the refcount any more.
*/
static void md_make_request(struct request_queue *q, struct bio *bio)
static blk_qc_t md_make_request(struct request_queue *q, struct bio *bio)
{
const int rw = bio_data_dir(bio);
struct mddev *mddev = q->queuedata;
@ -262,13 +262,13 @@ static void md_make_request(struct request_queue *q, struct bio *bio)
if (mddev == NULL || mddev->pers == NULL
|| !mddev->ready) {
bio_io_error(bio);
return;
return BLK_QC_T_NONE;
}
if (mddev->ro == 1 && unlikely(rw == WRITE)) {
if (bio_sectors(bio) != 0)
bio->bi_error = -EROFS;
bio_endio(bio);
return;
return BLK_QC_T_NONE;
}
smp_rmb(); /* Ensure implications of 'active' are visible */
rcu_read_lock();
@ -302,6 +302,8 @@ static void md_make_request(struct request_queue *q, struct bio *bio)
if (atomic_dec_and_test(&mddev->active_io) && mddev->suspended)
wake_up(&mddev->sb_wait);
return BLK_QC_T_NONE;
}
/* mddev_suspend makes sure no new requests are submitted

3
drivers/nvdimm/blk.c
View File

@ -161,7 +161,7 @@ static int nd_blk_do_bvec(struct nd_blk_device *blk_dev,
return err;
}
static void nd_blk_make_request(struct request_queue *q, struct bio *bio)
static blk_qc_t nd_blk_make_request(struct request_queue *q, struct bio *bio)
{
struct block_device *bdev = bio->bi_bdev;
struct gendisk *disk = bdev->bd_disk;
@ -208,6 +208,7 @@ static void nd_blk_make_request(struct request_queue *q, struct bio *bio)
out:
bio_endio(bio);
return BLK_QC_T_NONE;
}
static int nd_blk_rw_bytes(struct nd_namespace_common *ndns,

3
drivers/nvdimm/btt.c
View File

@ -1150,7 +1150,7 @@ static int btt_do_bvec(struct btt *btt, struct bio_integrity_payload *bip,
return ret;
}
static void btt_make_request(struct request_queue *q, struct bio *bio)
static blk_qc_t btt_make_request(struct request_queue *q, struct bio *bio)
{
struct bio_integrity_payload *bip = bio_integrity(bio);
struct btt *btt = q->queuedata;
@ -1198,6 +1198,7 @@ static void btt_make_request(struct request_queue *q, struct bio *bio)
out:
bio_endio(bio);
return BLK_QC_T_NONE;
}
static int btt_rw_page(struct block_device *bdev, sector_t sector,

3
drivers/nvdimm/pmem.c
View File

@ -64,7 +64,7 @@ static void pmem_do_bvec(struct pmem_device *pmem, struct page *page,
kunmap_atomic(mem);
}
static void pmem_make_request(struct request_queue *q, struct bio *bio)
static blk_qc_t pmem_make_request(struct request_queue *q, struct bio *bio)
{
bool do_acct;
unsigned long start;
@ -84,6 +84,7 @@ static void pmem_make_request(struct request_queue *q, struct bio *bio)
wmb_pmem();
bio_endio(bio);
return BLK_QC_T_NONE;
}
static int pmem_rw_page(struct block_device *bdev, sector_t sector,

32
drivers/nvme/host/pci.c
View File

@ -90,7 +90,7 @@ static struct class *nvme_class;
static int __nvme_reset(struct nvme_dev *dev);
static int nvme_reset(struct nvme_dev *dev);
static int nvme_process_cq(struct nvme_queue *nvmeq);
static void nvme_process_cq(struct nvme_queue *nvmeq);
static void nvme_dead_ctrl(struct nvme_dev *dev);
struct async_cmd_info {
@ -935,7 +935,7 @@ static int nvme_queue_rq(struct blk_mq_hw_ctx *hctx,
return BLK_MQ_RQ_QUEUE_BUSY;
}
static int nvme_process_cq(struct nvme_queue *nvmeq)
static void __nvme_process_cq(struct nvme_queue *nvmeq, unsigned int *tag)
{
u16 head, phase;
@ -953,6 +953,8 @@ static int nvme_process_cq(struct nvme_queue *nvmeq)
head = 0;
phase = !phase;
}
if (tag && *tag == cqe.command_id)
*tag = -1;
ctx = nvme_finish_cmd(nvmeq, cqe.command_id, &fn);
fn(nvmeq, ctx, &cqe);
}
@ -964,14 +966,18 @@ static int nvme_process_cq(struct nvme_queue *nvmeq)
* a big problem.
*/
if (head == nvmeq->cq_head && phase == nvmeq->cq_phase)
return 0;
return;
writel(head, nvmeq->q_db + nvmeq->dev->db_stride);
nvmeq->cq_head = head;
nvmeq->cq_phase = phase;
nvmeq->cqe_seen = 1;
return 1;
}
static void nvme_process_cq(struct nvme_queue *nvmeq)
{
__nvme_process_cq(nvmeq, NULL);
}
static irqreturn_t nvme_irq(int irq, void *data)
@ -995,6 +1001,23 @@ static irqreturn_t nvme_irq_check(int irq, void *data)
return IRQ_WAKE_THREAD;
}
static int nvme_poll(struct blk_mq_hw_ctx *hctx, unsigned int tag)
{
struct nvme_queue *nvmeq = hctx->driver_data;
if ((le16_to_cpu(nvmeq->cqes[nvmeq->cq_head].status) & 1) ==
nvmeq->cq_phase) {
spin_lock_irq(&nvmeq->q_lock);
__nvme_process_cq(nvmeq, &tag);
spin_unlock_irq(&nvmeq->q_lock);
if (tag == -1)
return 1;
}
return 0;
}
/*
* Returns 0 on success. If the result is negative, it's a Linux error code;
* if the result is positive, it's an NVM Express status code
@ -1656,6 +1679,7 @@ static struct blk_mq_ops nvme_mq_ops = {
.init_hctx = nvme_init_hctx,
.init_request = nvme_init_request,
.timeout = nvme_timeout,
.poll = nvme_poll,
};
static void nvme_dev_remove_admin(struct nvme_dev *dev)

8
drivers/s390/block/dcssblk.c
View File

@ -27,7 +27,8 @@
static int dcssblk_open(struct block_device *bdev, fmode_t mode);
static void dcssblk_release(struct gendisk *disk, fmode_t mode);
static void dcssblk_make_request(struct request_queue *q, struct bio *bio);
static blk_qc_t dcssblk_make_request(struct request_queue *q,
struct bio *bio);
static long dcssblk_direct_access(struct block_device *bdev, sector_t secnum,
void __pmem **kaddr, unsigned long *pfn);
@ -815,7 +816,7 @@ dcssblk_release(struct gendisk *disk, fmode_t mode)
up_write(&dcssblk_devices_sem);
}
static void
static blk_qc_t
dcssblk_make_request(struct request_queue *q, struct bio *bio)
{
struct dcssblk_dev_info *dev_info;
@ -874,9 +875,10 @@ dcssblk_make_request(struct request_queue *q, struct bio *bio)
bytes_done += bvec.bv_len;
}
bio_endio(bio);
return;
return BLK_QC_T_NONE;
fail:
bio_io_error(bio);
return BLK_QC_T_NONE;
}
static long

5
drivers/s390/block/xpram.c
View File

@ -181,7 +181,7 @@ static unsigned long xpram_highest_page_index(void)
/*
* Block device make request function.
*/
static void xpram_make_request(struct request_queue *q, struct bio *bio)
static blk_qc_t xpram_make_request(struct request_queue *q, struct bio *bio)
{
xpram_device_t *xdev = bio->bi_bdev->bd_disk->private_data;
struct bio_vec bvec;
@ -223,9 +223,10 @@ static void xpram_make_request(struct request_queue *q, struct bio *bio)
}
}
bio_endio(bio);
return;
return BLK_QC_T_NONE;
fail:
bio_io_error(bio);
return BLK_QC_T_NONE;
}
static int xpram_getgeo(struct block_device *bdev, struct hd_geometry *geo)

5
drivers/staging/lustre/lustre/llite/lloop.c
View File

@ -333,7 +333,7 @@ static unsigned int loop_get_bio(struct lloop_device *lo, struct bio **req)
return count;
}
static void loop_make_request(struct request_queue *q, struct bio *old_bio)
static blk_qc_t loop_make_request(struct request_queue *q, struct bio *old_bio)
{
struct lloop_device *lo = q->queuedata;
int rw = bio_rw(old_bio);
@ -364,9 +364,10 @@ static void loop_make_request(struct request_queue *q, struct bio *old_bio)
goto err;
}
loop_add_bio(lo, old_bio);
return;
return BLK_QC_T_NONE;
err:
bio_io_error(old_bio);
return BLK_QC_T_NONE;
}
static inline void loop_handle_bio(struct lloop_device *lo, struct bio *bio)

14
fs/direct-io.c
View File

@ -109,6 +109,8 @@ struct dio_submit {
struct dio {
int flags; /* doesn't change */
int rw;
blk_qc_t bio_cookie;
struct block_device *bio_bdev;
struct inode *inode;
loff_t i_size; /* i_size when submitted */
dio_iodone_t *end_io; /* IO completion function */
@ -397,11 +399,14 @@ static inline void dio_bio_submit(struct dio *dio, struct dio_submit *sdio)
if (dio->is_async && dio->rw == READ && dio->should_dirty)
bio_set_pages_dirty(bio);
if (sdio->submit_io)
dio->bio_bdev = bio->bi_bdev;
if (sdio->submit_io) {
sdio->submit_io(dio->rw, bio, dio->inode,
sdio->logical_offset_in_bio);
else
submit_bio(dio->rw, bio);
dio->bio_cookie = BLK_QC_T_NONE;
} else
dio->bio_cookie = submit_bio(dio->rw, bio);
sdio->bio = NULL;
sdio->boundary = 0;
@ -440,7 +445,8 @@ static struct bio *dio_await_one(struct dio *dio)
__set_current_state(TASK_UNINTERRUPTIBLE);
dio->waiter = current;
spin_unlock_irqrestore(&dio->bio_lock, flags);
io_schedule();
if (!blk_poll(bdev_get_queue(dio->bio_bdev), dio->bio_cookie))
io_schedule();
/* wake up sets us TASK_RUNNING */
spin_lock_irqsave(&dio->bio_lock, flags);
dio->waiter = NULL;

10
include/linux/blk-mq.h
View File

@ -59,6 +59,9 @@ struct blk_mq_hw_ctx {
struct blk_mq_cpu_notifier cpu_notifier;
struct kobject kobj;
unsigned long poll_invoked;
unsigned long poll_success;
};
struct blk_mq_tag_set {
@ -97,6 +100,8 @@ typedef void (exit_request_fn)(void *, struct request *, unsigned int,
typedef void (busy_iter_fn)(struct blk_mq_hw_ctx *, struct request *, void *,
bool);
typedef void (busy_tag_iter_fn)(struct request *, void *, bool);
typedef int (poll_fn)(struct blk_mq_hw_ctx *, unsigned int);
struct blk_mq_ops {
/*
@ -114,6 +119,11 @@ struct blk_mq_ops {
*/
timeout_fn *timeout;
/*
* Called to poll for completion of a specific tag.
*/
poll_fn *poll;
softirq_done_fn *complete;
/*

24
include/linux/blk_types.h
View File

@ -244,4 +244,28 @@ enum rq_flag_bits {
#define REQ_MQ_INFLIGHT (1ULL << __REQ_MQ_INFLIGHT)
#define REQ_NO_TIMEOUT (1ULL << __REQ_NO_TIMEOUT)
typedef unsigned int blk_qc_t;
#define BLK_QC_T_NONE -1U
#define BLK_QC_T_SHIFT 16
static inline bool blk_qc_t_valid(blk_qc_t cookie)
{
return cookie != BLK_QC_T_NONE;
}
static inline blk_qc_t blk_tag_to_qc_t(unsigned int tag, unsigned int queue_num)
{
return tag | (queue_num << BLK_QC_T_SHIFT);
}
static inline unsigned int blk_qc_t_to_queue_num(blk_qc_t cookie)
{
return cookie >> BLK_QC_T_SHIFT;
}
static inline unsigned int blk_qc_t_to_tag(blk_qc_t cookie)
{
return cookie & 0xffff;
}
#endif /* __LINUX_BLK_TYPES_H */

7
include/linux/blkdev.h
View File

@ -209,7 +209,7 @@ static inline unsigned short req_get_ioprio(struct request *req)
struct blk_queue_ctx;
typedef void (request_fn_proc) (struct request_queue *q);
typedef void (make_request_fn) (struct request_queue *q, struct bio *bio);
typedef blk_qc_t (make_request_fn) (struct request_queue *q, struct bio *bio);
typedef int (prep_rq_fn) (struct request_queue *, struct request *);
typedef void (unprep_rq_fn) (struct request_queue *, struct request *);
@ -487,6 +487,7 @@ struct request_queue {
#define QUEUE_FLAG_DEAD 19 /* queue tear-down finished */
#define QUEUE_FLAG_INIT_DONE 20 /* queue is initialized */
#define QUEUE_FLAG_NO_SG_MERGE 21 /* don't attempt to merge SG segments*/
#define QUEUE_FLAG_POLL 22 /* IO polling enabled if set */
#define QUEUE_FLAG_DEFAULT ((1 << QUEUE_FLAG_IO_STAT) | \
(1 << QUEUE_FLAG_STACKABLE) | \
@ -761,7 +762,7 @@ static inline void rq_flush_dcache_pages(struct request *rq)
extern int blk_register_queue(struct gendisk *disk);
extern void blk_unregister_queue(struct gendisk *disk);
extern void generic_make_request(struct bio *bio);
extern blk_qc_t generic_make_request(struct bio *bio);
extern void blk_rq_init(struct request_queue *q, struct request *rq);
extern void blk_put_request(struct request *);
extern void __blk_put_request(struct request_queue *, struct request *);
@ -814,6 +815,8 @@ extern int blk_execute_rq(struct request_queue *, struct gendisk *,
extern void blk_execute_rq_nowait(struct request_queue *, struct gendisk *,
struct request *, int, rq_end_io_fn *);
bool blk_poll(struct request_queue *q, blk_qc_t cookie);
static inline struct request_queue *bdev_get_queue(struct block_device *bdev)
{
return bdev->bd_disk->queue; /* this is never NULL */

2
include/linux/fs.h
View File

@ -2613,7 +2613,7 @@ static inline void remove_inode_hash(struct inode *inode)
extern void inode_sb_list_add(struct inode *inode);
#ifdef CONFIG_BLOCK
extern void submit_bio(int, struct bio *);
extern blk_qc_t submit_bio(int, struct bio *);
extern int bdev_read_only(struct block_device *);
#endif
extern int set_blocksize(struct block_device *, int);

2
include/linux/lightnvm.h
View File

@ -426,7 +426,7 @@ static inline struct ppa_addr block_to_ppa(struct nvm_dev *dev,
return ppa;
}
typedef void (nvm_tgt_make_rq_fn)(struct request_queue *, struct bio *);
typedef blk_qc_t (nvm_tgt_make_rq_fn)(struct request_queue *, struct bio *);
typedef sector_t (nvm_tgt_capacity_fn)(void *);
typedef int (nvm_tgt_end_io_fn)(struct nvm_rq *, int);
typedef void *(nvm_tgt_init_fn)(struct nvm_dev *, struct gendisk *, int, int);