304419d8a7
The mmc_queue_req is a per-request state container the MMC core uses to carry bounce buffers, pointers to asynchronous requests and so on. Currently allocated as a static array of objects, then as a request comes in, a mmc_queue_req is assigned to it, and used during the lifetime of the request. This is backwards compared to how other block layer drivers work: they usally let the block core provide a per-request struct that get allocated right beind the struct request, and which can be obtained using the blk_mq_rq_to_pdu() helper. (The _mq_ infix in this function name is misleading: it is used by both the old and the MQ block layer.) The per-request struct gets allocated to the size stored in the queue variable .cmd_size initialized using the .init_rq_fn() and cleaned up using .exit_rq_fn(). The block layer code makes the MMC core rely on this mechanism to allocate the per-request mmc_queue_req state container. Doing this make a lot of complicated queue handling go away. We only need to keep the .qnct that keeps count of how many request are currently being processed by the MMC layer. The MQ block layer will replace also this once we transition to it. Doing this refactoring is necessary to move the ioctl() operations into custom block layer requests tagged with REQ_OP_DRV_[IN|OUT] instead of the custom code using the BigMMCHostLock that we have today: those require that per-request data be obtainable easily from a request after creating a custom request with e.g.: struct request *rq = blk_get_request(q, REQ_OP_DRV_IN, __GFP_RECLAIM); struct mmc_queue_req *mq_rq = req_to_mq_rq(rq); And this is not possible with the current construction, as the request is not immediately assigned the per-request state container, but instead it gets assigned when the request finally enters the MMC queue, which is way too late for custom requests. Signed-off-by: Linus Walleij <linus.walleij@linaro.org> [Ulf: Folded in the fix to drop a call to blk_cleanup_queue()] Signed-off-by: Ulf Hansson <ulf.hansson@linaro.org> Tested-by: Heiner Kallweit <hkallweit1@gmail.com>
425 lines
9.8 KiB
C
425 lines
9.8 KiB
C
/*
|
|
* Copyright (C) 2003 Russell King, All Rights Reserved.
|
|
* Copyright 2006-2007 Pierre Ossman
|
|
*
|
|
* This program is free software; you can redistribute it and/or modify
|
|
* it under the terms of the GNU General Public License version 2 as
|
|
* published by the Free Software Foundation.
|
|
*
|
|
*/
|
|
#include <linux/slab.h>
|
|
#include <linux/module.h>
|
|
#include <linux/blkdev.h>
|
|
#include <linux/freezer.h>
|
|
#include <linux/kthread.h>
|
|
#include <linux/scatterlist.h>
|
|
#include <linux/dma-mapping.h>
|
|
|
|
#include <linux/mmc/card.h>
|
|
#include <linux/mmc/host.h>
|
|
|
|
#include "queue.h"
|
|
#include "block.h"
|
|
#include "core.h"
|
|
#include "card.h"
|
|
|
|
#define MMC_QUEUE_BOUNCESZ 65536
|
|
|
|
/*
|
|
* Prepare a MMC request. This just filters out odd stuff.
|
|
*/
|
|
static int mmc_prep_request(struct request_queue *q, struct request *req)
|
|
{
|
|
struct mmc_queue *mq = q->queuedata;
|
|
|
|
if (mq && (mmc_card_removed(mq->card) || mmc_access_rpmb(mq)))
|
|
return BLKPREP_KILL;
|
|
|
|
req->rq_flags |= RQF_DONTPREP;
|
|
|
|
return BLKPREP_OK;
|
|
}
|
|
|
|
static int mmc_queue_thread(void *d)
|
|
{
|
|
struct mmc_queue *mq = d;
|
|
struct request_queue *q = mq->queue;
|
|
struct mmc_context_info *cntx = &mq->card->host->context_info;
|
|
|
|
current->flags |= PF_MEMALLOC;
|
|
|
|
down(&mq->thread_sem);
|
|
do {
|
|
struct request *req;
|
|
|
|
spin_lock_irq(q->queue_lock);
|
|
set_current_state(TASK_INTERRUPTIBLE);
|
|
req = blk_fetch_request(q);
|
|
mq->asleep = false;
|
|
cntx->is_waiting_last_req = false;
|
|
cntx->is_new_req = false;
|
|
if (!req) {
|
|
/*
|
|
* Dispatch queue is empty so set flags for
|
|
* mmc_request_fn() to wake us up.
|
|
*/
|
|
if (mq->qcnt)
|
|
cntx->is_waiting_last_req = true;
|
|
else
|
|
mq->asleep = true;
|
|
}
|
|
spin_unlock_irq(q->queue_lock);
|
|
|
|
if (req || mq->qcnt) {
|
|
set_current_state(TASK_RUNNING);
|
|
mmc_blk_issue_rq(mq, req);
|
|
cond_resched();
|
|
} else {
|
|
if (kthread_should_stop()) {
|
|
set_current_state(TASK_RUNNING);
|
|
break;
|
|
}
|
|
up(&mq->thread_sem);
|
|
schedule();
|
|
down(&mq->thread_sem);
|
|
}
|
|
} while (1);
|
|
up(&mq->thread_sem);
|
|
|
|
return 0;
|
|
}
|
|
|
|
/*
|
|
* Generic MMC request handler. This is called for any queue on a
|
|
* particular host. When the host is not busy, we look for a request
|
|
* on any queue on this host, and attempt to issue it. This may
|
|
* not be the queue we were asked to process.
|
|
*/
|
|
static void mmc_request_fn(struct request_queue *q)
|
|
{
|
|
struct mmc_queue *mq = q->queuedata;
|
|
struct request *req;
|
|
struct mmc_context_info *cntx;
|
|
|
|
if (!mq) {
|
|
while ((req = blk_fetch_request(q)) != NULL) {
|
|
req->rq_flags |= RQF_QUIET;
|
|
__blk_end_request_all(req, -EIO);
|
|
}
|
|
return;
|
|
}
|
|
|
|
cntx = &mq->card->host->context_info;
|
|
|
|
if (cntx->is_waiting_last_req) {
|
|
cntx->is_new_req = true;
|
|
wake_up_interruptible(&cntx->wait);
|
|
}
|
|
|
|
if (mq->asleep)
|
|
wake_up_process(mq->thread);
|
|
}
|
|
|
|
static struct scatterlist *mmc_alloc_sg(int sg_len, gfp_t gfp)
|
|
{
|
|
struct scatterlist *sg;
|
|
|
|
sg = kmalloc_array(sg_len, sizeof(*sg), gfp);
|
|
if (sg)
|
|
sg_init_table(sg, sg_len);
|
|
|
|
return sg;
|
|
}
|
|
|
|
static void mmc_queue_setup_discard(struct request_queue *q,
|
|
struct mmc_card *card)
|
|
{
|
|
unsigned max_discard;
|
|
|
|
max_discard = mmc_calc_max_discard(card);
|
|
if (!max_discard)
|
|
return;
|
|
|
|
queue_flag_set_unlocked(QUEUE_FLAG_DISCARD, q);
|
|
blk_queue_max_discard_sectors(q, max_discard);
|
|
q->limits.discard_granularity = card->pref_erase << 9;
|
|
/* granularity must not be greater than max. discard */
|
|
if (card->pref_erase > max_discard)
|
|
q->limits.discard_granularity = 0;
|
|
if (mmc_can_secure_erase_trim(card))
|
|
queue_flag_set_unlocked(QUEUE_FLAG_SECERASE, q);
|
|
}
|
|
|
|
static unsigned int mmc_queue_calc_bouncesz(struct mmc_host *host)
|
|
{
|
|
unsigned int bouncesz = MMC_QUEUE_BOUNCESZ;
|
|
|
|
if (host->max_segs != 1 || (host->caps & MMC_CAP_NO_BOUNCE_BUFF))
|
|
return 0;
|
|
|
|
if (bouncesz > host->max_req_size)
|
|
bouncesz = host->max_req_size;
|
|
if (bouncesz > host->max_seg_size)
|
|
bouncesz = host->max_seg_size;
|
|
if (bouncesz > host->max_blk_count * 512)
|
|
bouncesz = host->max_blk_count * 512;
|
|
|
|
if (bouncesz <= 512)
|
|
return 0;
|
|
|
|
return bouncesz;
|
|
}
|
|
|
|
/**
|
|
* mmc_init_request() - initialize the MMC-specific per-request data
|
|
* @q: the request queue
|
|
* @req: the request
|
|
* @gfp: memory allocation policy
|
|
*/
|
|
static int mmc_init_request(struct request_queue *q, struct request *req,
|
|
gfp_t gfp)
|
|
{
|
|
struct mmc_queue_req *mq_rq = req_to_mmc_queue_req(req);
|
|
struct mmc_queue *mq = q->queuedata;
|
|
struct mmc_card *card = mq->card;
|
|
struct mmc_host *host = card->host;
|
|
|
|
mq_rq->req = req;
|
|
|
|
if (card->bouncesz) {
|
|
mq_rq->bounce_buf = kmalloc(card->bouncesz, gfp);
|
|
if (!mq_rq->bounce_buf)
|
|
return -ENOMEM;
|
|
if (card->bouncesz > 512) {
|
|
mq_rq->sg = mmc_alloc_sg(1, gfp);
|
|
if (!mq_rq->sg)
|
|
return -ENOMEM;
|
|
mq_rq->bounce_sg = mmc_alloc_sg(card->bouncesz / 512,
|
|
gfp);
|
|
if (!mq_rq->bounce_sg)
|
|
return -ENOMEM;
|
|
}
|
|
} else {
|
|
mq_rq->bounce_buf = NULL;
|
|
mq_rq->bounce_sg = NULL;
|
|
mq_rq->sg = mmc_alloc_sg(host->max_segs, gfp);
|
|
if (!mq_rq->sg)
|
|
return -ENOMEM;
|
|
}
|
|
|
|
return 0;
|
|
}
|
|
|
|
static void mmc_exit_request(struct request_queue *q, struct request *req)
|
|
{
|
|
struct mmc_queue_req *mq_rq = req_to_mmc_queue_req(req);
|
|
|
|
/* It is OK to kfree(NULL) so this will be smooth */
|
|
kfree(mq_rq->bounce_sg);
|
|
mq_rq->bounce_sg = NULL;
|
|
|
|
kfree(mq_rq->bounce_buf);
|
|
mq_rq->bounce_buf = NULL;
|
|
|
|
kfree(mq_rq->sg);
|
|
mq_rq->sg = NULL;
|
|
|
|
mq_rq->req = NULL;
|
|
}
|
|
|
|
/**
|
|
* mmc_init_queue - initialise a queue structure.
|
|
* @mq: mmc queue
|
|
* @card: mmc card to attach this queue
|
|
* @lock: queue lock
|
|
* @subname: partition subname
|
|
*
|
|
* Initialise a MMC card request queue.
|
|
*/
|
|
int mmc_init_queue(struct mmc_queue *mq, struct mmc_card *card,
|
|
spinlock_t *lock, const char *subname)
|
|
{
|
|
struct mmc_host *host = card->host;
|
|
u64 limit = BLK_BOUNCE_HIGH;
|
|
int ret = -ENOMEM;
|
|
|
|
if (mmc_dev(host)->dma_mask && *mmc_dev(host)->dma_mask)
|
|
limit = (u64)dma_max_pfn(mmc_dev(host)) << PAGE_SHIFT;
|
|
|
|
mq->card = card;
|
|
mq->queue = blk_alloc_queue(GFP_KERNEL);
|
|
if (!mq->queue)
|
|
return -ENOMEM;
|
|
mq->queue->queue_lock = lock;
|
|
mq->queue->request_fn = mmc_request_fn;
|
|
mq->queue->init_rq_fn = mmc_init_request;
|
|
mq->queue->exit_rq_fn = mmc_exit_request;
|
|
mq->queue->cmd_size = sizeof(struct mmc_queue_req);
|
|
mq->queue->queuedata = mq;
|
|
mq->qcnt = 0;
|
|
ret = blk_init_allocated_queue(mq->queue);
|
|
if (ret) {
|
|
blk_cleanup_queue(mq->queue);
|
|
return ret;
|
|
}
|
|
|
|
blk_queue_prep_rq(mq->queue, mmc_prep_request);
|
|
queue_flag_set_unlocked(QUEUE_FLAG_NONROT, mq->queue);
|
|
queue_flag_clear_unlocked(QUEUE_FLAG_ADD_RANDOM, mq->queue);
|
|
if (mmc_can_erase(card))
|
|
mmc_queue_setup_discard(mq->queue, card);
|
|
|
|
card->bouncesz = mmc_queue_calc_bouncesz(host);
|
|
if (card->bouncesz) {
|
|
blk_queue_bounce_limit(mq->queue, BLK_BOUNCE_ANY);
|
|
blk_queue_max_hw_sectors(mq->queue, card->bouncesz / 512);
|
|
blk_queue_max_segments(mq->queue, card->bouncesz / 512);
|
|
blk_queue_max_segment_size(mq->queue, card->bouncesz);
|
|
} else {
|
|
blk_queue_bounce_limit(mq->queue, limit);
|
|
blk_queue_max_hw_sectors(mq->queue,
|
|
min(host->max_blk_count, host->max_req_size / 512));
|
|
blk_queue_max_segments(mq->queue, host->max_segs);
|
|
blk_queue_max_segment_size(mq->queue, host->max_seg_size);
|
|
}
|
|
|
|
sema_init(&mq->thread_sem, 1);
|
|
|
|
mq->thread = kthread_run(mmc_queue_thread, mq, "mmcqd/%d%s",
|
|
host->index, subname ? subname : "");
|
|
|
|
if (IS_ERR(mq->thread)) {
|
|
ret = PTR_ERR(mq->thread);
|
|
goto cleanup_queue;
|
|
}
|
|
|
|
return 0;
|
|
|
|
cleanup_queue:
|
|
blk_cleanup_queue(mq->queue);
|
|
return ret;
|
|
}
|
|
|
|
void mmc_cleanup_queue(struct mmc_queue *mq)
|
|
{
|
|
struct request_queue *q = mq->queue;
|
|
unsigned long flags;
|
|
|
|
/* Make sure the queue isn't suspended, as that will deadlock */
|
|
mmc_queue_resume(mq);
|
|
|
|
/* Then terminate our worker thread */
|
|
kthread_stop(mq->thread);
|
|
|
|
/* Empty the queue */
|
|
spin_lock_irqsave(q->queue_lock, flags);
|
|
q->queuedata = NULL;
|
|
blk_start_queue(q);
|
|
spin_unlock_irqrestore(q->queue_lock, flags);
|
|
|
|
mq->card = NULL;
|
|
}
|
|
EXPORT_SYMBOL(mmc_cleanup_queue);
|
|
|
|
/**
|
|
* mmc_queue_suspend - suspend a MMC request queue
|
|
* @mq: MMC queue to suspend
|
|
*
|
|
* Stop the block request queue, and wait for our thread to
|
|
* complete any outstanding requests. This ensures that we
|
|
* won't suspend while a request is being processed.
|
|
*/
|
|
void mmc_queue_suspend(struct mmc_queue *mq)
|
|
{
|
|
struct request_queue *q = mq->queue;
|
|
unsigned long flags;
|
|
|
|
if (!mq->suspended) {
|
|
mq->suspended |= true;
|
|
|
|
spin_lock_irqsave(q->queue_lock, flags);
|
|
blk_stop_queue(q);
|
|
spin_unlock_irqrestore(q->queue_lock, flags);
|
|
|
|
down(&mq->thread_sem);
|
|
}
|
|
}
|
|
|
|
/**
|
|
* mmc_queue_resume - resume a previously suspended MMC request queue
|
|
* @mq: MMC queue to resume
|
|
*/
|
|
void mmc_queue_resume(struct mmc_queue *mq)
|
|
{
|
|
struct request_queue *q = mq->queue;
|
|
unsigned long flags;
|
|
|
|
if (mq->suspended) {
|
|
mq->suspended = false;
|
|
|
|
up(&mq->thread_sem);
|
|
|
|
spin_lock_irqsave(q->queue_lock, flags);
|
|
blk_start_queue(q);
|
|
spin_unlock_irqrestore(q->queue_lock, flags);
|
|
}
|
|
}
|
|
|
|
/*
|
|
* Prepare the sg list(s) to be handed of to the host driver
|
|
*/
|
|
unsigned int mmc_queue_map_sg(struct mmc_queue *mq, struct mmc_queue_req *mqrq)
|
|
{
|
|
unsigned int sg_len;
|
|
size_t buflen;
|
|
struct scatterlist *sg;
|
|
int i;
|
|
|
|
if (!mqrq->bounce_buf)
|
|
return blk_rq_map_sg(mq->queue, mqrq->req, mqrq->sg);
|
|
|
|
sg_len = blk_rq_map_sg(mq->queue, mqrq->req, mqrq->bounce_sg);
|
|
|
|
mqrq->bounce_sg_len = sg_len;
|
|
|
|
buflen = 0;
|
|
for_each_sg(mqrq->bounce_sg, sg, sg_len, i)
|
|
buflen += sg->length;
|
|
|
|
sg_init_one(mqrq->sg, mqrq->bounce_buf, buflen);
|
|
|
|
return 1;
|
|
}
|
|
|
|
/*
|
|
* If writing, bounce the data to the buffer before the request
|
|
* is sent to the host driver
|
|
*/
|
|
void mmc_queue_bounce_pre(struct mmc_queue_req *mqrq)
|
|
{
|
|
if (!mqrq->bounce_buf)
|
|
return;
|
|
|
|
if (rq_data_dir(mqrq->req) != WRITE)
|
|
return;
|
|
|
|
sg_copy_to_buffer(mqrq->bounce_sg, mqrq->bounce_sg_len,
|
|
mqrq->bounce_buf, mqrq->sg[0].length);
|
|
}
|
|
|
|
/*
|
|
* If reading, bounce the data from the buffer after the request
|
|
* has been handled by the host driver
|
|
*/
|
|
void mmc_queue_bounce_post(struct mmc_queue_req *mqrq)
|
|
{
|
|
if (!mqrq->bounce_buf)
|
|
return;
|
|
|
|
if (rq_data_dir(mqrq->req) != READ)
|
|
return;
|
|
|
|
sg_copy_from_buffer(mqrq->bounce_sg, mqrq->bounce_sg_len,
|
|
mqrq->bounce_buf, mqrq->sg[0].length);
|
|
}
|