linux/drivers/mtd/nand/ecc.c
Rob Herring c2fc6b6947 mtd: Explicitly include correct DT includes
The DT of_device.h and of_platform.h date back to the separate
of_platform_bus_type before it was merged into the regular platform bus.
As part of that merge prepping Arm DT support 13 years ago, they
"temporarily" include each other. They also include platform_device.h
and of.h. As a result, there's a pretty much random mix of those include
files used throughout the tree. In order to detangle these headers and
replace the implicit includes with struct declarations, users need to
explicitly include the correct includes.

Signed-off-by: Rob Herring <robh@kernel.org>
Reviewed-by: Linus Walleij <linus.walleij@linaro.org>
Acked-by: Heiko Stuebner <heiko@sntech.de>
Signed-off-by: Miquel Raynal <miquel.raynal@bootlin.com>
Link: https://lore.kernel.org/linux-mtd/20230714174751.4060439-1-robh@kernel.org
2023-07-27 17:03:41 +02:00

736 lines
20 KiB
C

// SPDX-License-Identifier: GPL-2.0+
/*
* Generic Error-Correcting Code (ECC) engine
*
* Copyright (C) 2019 Macronix
* Author:
* Miquèl RAYNAL <miquel.raynal@bootlin.com>
*
*
* This file describes the abstraction of any NAND ECC engine. It has been
* designed to fit most cases, including parallel NANDs and SPI-NANDs.
*
* There are three main situations where instantiating this ECC engine makes
* sense:
* - external: The ECC engine is outside the NAND pipeline, typically this
* is a software ECC engine, or an hardware engine that is
* outside the NAND controller pipeline.
* - pipelined: The ECC engine is inside the NAND pipeline, ie. on the
* controller's side. This is the case of most of the raw NAND
* controllers. In the pipeline case, the ECC bytes are
* generated/data corrected on the fly when a page is
* written/read.
* - ondie: The ECC engine is inside the NAND pipeline, on the chip's side.
* Some NAND chips can correct themselves the data.
*
* Besides the initial setup and final cleanups, the interfaces are rather
* simple:
* - prepare: Prepare an I/O request. Enable/disable the ECC engine based on
* the I/O request type. In case of software correction or external
* engine, this step may involve to derive the ECC bytes and place
* them in the OOB area before a write.
* - finish: Finish an I/O request. Correct the data in case of a read
* request and report the number of corrected bits/uncorrectable
* errors. Most likely empty for write operations, unless you have
* hardware specific stuff to do, like shutting down the engine to
* save power.
*
* The I/O request should be enclosed in a prepare()/finish() pair of calls
* and will behave differently depending on the requested I/O type:
* - raw: Correction disabled
* - ecc: Correction enabled
*
* The request direction is impacting the logic as well:
* - read: Load data from the NAND chip
* - write: Store data in the NAND chip
*
* Mixing all this combinations together gives the following behavior.
* Those are just examples, drivers are free to add custom steps in their
* prepare/finish hook.
*
* [external ECC engine]
* - external + prepare + raw + read: do nothing
* - external + finish + raw + read: do nothing
* - external + prepare + raw + write: do nothing
* - external + finish + raw + write: do nothing
* - external + prepare + ecc + read: do nothing
* - external + finish + ecc + read: calculate expected ECC bytes, extract
* ECC bytes from OOB buffer, correct
* and report any bitflip/error
* - external + prepare + ecc + write: calculate ECC bytes and store them at
* the right place in the OOB buffer based
* on the OOB layout
* - external + finish + ecc + write: do nothing
*
* [pipelined ECC engine]
* - pipelined + prepare + raw + read: disable the controller's ECC engine if
* activated
* - pipelined + finish + raw + read: do nothing
* - pipelined + prepare + raw + write: disable the controller's ECC engine if
* activated
* - pipelined + finish + raw + write: do nothing
* - pipelined + prepare + ecc + read: enable the controller's ECC engine if
* deactivated
* - pipelined + finish + ecc + read: check the status, report any
* error/bitflip
* - pipelined + prepare + ecc + write: enable the controller's ECC engine if
* deactivated
* - pipelined + finish + ecc + write: do nothing
*
* [ondie ECC engine]
* - ondie + prepare + raw + read: send commands to disable the on-chip ECC
* engine if activated
* - ondie + finish + raw + read: do nothing
* - ondie + prepare + raw + write: send commands to disable the on-chip ECC
* engine if activated
* - ondie + finish + raw + write: do nothing
* - ondie + prepare + ecc + read: send commands to enable the on-chip ECC
* engine if deactivated
* - ondie + finish + ecc + read: send commands to check the status, report
* any error/bitflip
* - ondie + prepare + ecc + write: send commands to enable the on-chip ECC
* engine if deactivated
* - ondie + finish + ecc + write: do nothing
*/
#include <linux/module.h>
#include <linux/mtd/nand.h>
#include <linux/platform_device.h>
#include <linux/slab.h>
#include <linux/of.h>
#include <linux/of_platform.h>
static LIST_HEAD(on_host_hw_engines);
static DEFINE_MUTEX(on_host_hw_engines_mutex);
/**
* nand_ecc_init_ctx - Init the ECC engine context
* @nand: the NAND device
*
* On success, the caller is responsible of calling @nand_ecc_cleanup_ctx().
*/
int nand_ecc_init_ctx(struct nand_device *nand)
{
if (!nand->ecc.engine || !nand->ecc.engine->ops->init_ctx)
return 0;
return nand->ecc.engine->ops->init_ctx(nand);
}
EXPORT_SYMBOL(nand_ecc_init_ctx);
/**
* nand_ecc_cleanup_ctx - Cleanup the ECC engine context
* @nand: the NAND device
*/
void nand_ecc_cleanup_ctx(struct nand_device *nand)
{
if (nand->ecc.engine && nand->ecc.engine->ops->cleanup_ctx)
nand->ecc.engine->ops->cleanup_ctx(nand);
}
EXPORT_SYMBOL(nand_ecc_cleanup_ctx);
/**
* nand_ecc_prepare_io_req - Prepare an I/O request
* @nand: the NAND device
* @req: the I/O request
*/
int nand_ecc_prepare_io_req(struct nand_device *nand,
struct nand_page_io_req *req)
{
if (!nand->ecc.engine || !nand->ecc.engine->ops->prepare_io_req)
return 0;
return nand->ecc.engine->ops->prepare_io_req(nand, req);
}
EXPORT_SYMBOL(nand_ecc_prepare_io_req);
/**
* nand_ecc_finish_io_req - Finish an I/O request
* @nand: the NAND device
* @req: the I/O request
*/
int nand_ecc_finish_io_req(struct nand_device *nand,
struct nand_page_io_req *req)
{
if (!nand->ecc.engine || !nand->ecc.engine->ops->finish_io_req)
return 0;
return nand->ecc.engine->ops->finish_io_req(nand, req);
}
EXPORT_SYMBOL(nand_ecc_finish_io_req);
/* Define default OOB placement schemes for large and small page devices */
static int nand_ooblayout_ecc_sp(struct mtd_info *mtd, int section,
struct mtd_oob_region *oobregion)
{
struct nand_device *nand = mtd_to_nanddev(mtd);
unsigned int total_ecc_bytes = nand->ecc.ctx.total;
if (section > 1)
return -ERANGE;
if (!section) {
oobregion->offset = 0;
if (mtd->oobsize == 16)
oobregion->length = 4;
else
oobregion->length = 3;
} else {
if (mtd->oobsize == 8)
return -ERANGE;
oobregion->offset = 6;
oobregion->length = total_ecc_bytes - 4;
}
return 0;
}
static int nand_ooblayout_free_sp(struct mtd_info *mtd, int section,
struct mtd_oob_region *oobregion)
{
if (section > 1)
return -ERANGE;
if (mtd->oobsize == 16) {
if (section)
return -ERANGE;
oobregion->length = 8;
oobregion->offset = 8;
} else {
oobregion->length = 2;
if (!section)
oobregion->offset = 3;
else
oobregion->offset = 6;
}
return 0;
}
static const struct mtd_ooblayout_ops nand_ooblayout_sp_ops = {
.ecc = nand_ooblayout_ecc_sp,
.free = nand_ooblayout_free_sp,
};
const struct mtd_ooblayout_ops *nand_get_small_page_ooblayout(void)
{
return &nand_ooblayout_sp_ops;
}
EXPORT_SYMBOL_GPL(nand_get_small_page_ooblayout);
static int nand_ooblayout_ecc_lp(struct mtd_info *mtd, int section,
struct mtd_oob_region *oobregion)
{
struct nand_device *nand = mtd_to_nanddev(mtd);
unsigned int total_ecc_bytes = nand->ecc.ctx.total;
if (section || !total_ecc_bytes)
return -ERANGE;
oobregion->length = total_ecc_bytes;
oobregion->offset = mtd->oobsize - oobregion->length;
return 0;
}
static int nand_ooblayout_free_lp(struct mtd_info *mtd, int section,
struct mtd_oob_region *oobregion)
{
struct nand_device *nand = mtd_to_nanddev(mtd);
unsigned int total_ecc_bytes = nand->ecc.ctx.total;
if (section)
return -ERANGE;
oobregion->length = mtd->oobsize - total_ecc_bytes - 2;
oobregion->offset = 2;
return 0;
}
static const struct mtd_ooblayout_ops nand_ooblayout_lp_ops = {
.ecc = nand_ooblayout_ecc_lp,
.free = nand_ooblayout_free_lp,
};
const struct mtd_ooblayout_ops *nand_get_large_page_ooblayout(void)
{
return &nand_ooblayout_lp_ops;
}
EXPORT_SYMBOL_GPL(nand_get_large_page_ooblayout);
/*
* Support the old "large page" layout used for 1-bit Hamming ECC where ECC
* are placed at a fixed offset.
*/
static int nand_ooblayout_ecc_lp_hamming(struct mtd_info *mtd, int section,
struct mtd_oob_region *oobregion)
{
struct nand_device *nand = mtd_to_nanddev(mtd);
unsigned int total_ecc_bytes = nand->ecc.ctx.total;
if (section)
return -ERANGE;
switch (mtd->oobsize) {
case 64:
oobregion->offset = 40;
break;
case 128:
oobregion->offset = 80;
break;
default:
return -EINVAL;
}
oobregion->length = total_ecc_bytes;
if (oobregion->offset + oobregion->length > mtd->oobsize)
return -ERANGE;
return 0;
}
static int nand_ooblayout_free_lp_hamming(struct mtd_info *mtd, int section,
struct mtd_oob_region *oobregion)
{
struct nand_device *nand = mtd_to_nanddev(mtd);
unsigned int total_ecc_bytes = nand->ecc.ctx.total;
int ecc_offset = 0;
if (section < 0 || section > 1)
return -ERANGE;
switch (mtd->oobsize) {
case 64:
ecc_offset = 40;
break;
case 128:
ecc_offset = 80;
break;
default:
return -EINVAL;
}
if (section == 0) {
oobregion->offset = 2;
oobregion->length = ecc_offset - 2;
} else {
oobregion->offset = ecc_offset + total_ecc_bytes;
oobregion->length = mtd->oobsize - oobregion->offset;
}
return 0;
}
static const struct mtd_ooblayout_ops nand_ooblayout_lp_hamming_ops = {
.ecc = nand_ooblayout_ecc_lp_hamming,
.free = nand_ooblayout_free_lp_hamming,
};
const struct mtd_ooblayout_ops *nand_get_large_page_hamming_ooblayout(void)
{
return &nand_ooblayout_lp_hamming_ops;
}
EXPORT_SYMBOL_GPL(nand_get_large_page_hamming_ooblayout);
static enum nand_ecc_engine_type
of_get_nand_ecc_engine_type(struct device_node *np)
{
struct device_node *eng_np;
if (of_property_read_bool(np, "nand-no-ecc-engine"))
return NAND_ECC_ENGINE_TYPE_NONE;
if (of_property_read_bool(np, "nand-use-soft-ecc-engine"))
return NAND_ECC_ENGINE_TYPE_SOFT;
eng_np = of_parse_phandle(np, "nand-ecc-engine", 0);
of_node_put(eng_np);
if (eng_np) {
if (eng_np == np)
return NAND_ECC_ENGINE_TYPE_ON_DIE;
else
return NAND_ECC_ENGINE_TYPE_ON_HOST;
}
return NAND_ECC_ENGINE_TYPE_INVALID;
}
static const char * const nand_ecc_placement[] = {
[NAND_ECC_PLACEMENT_OOB] = "oob",
[NAND_ECC_PLACEMENT_INTERLEAVED] = "interleaved",
};
static enum nand_ecc_placement of_get_nand_ecc_placement(struct device_node *np)
{
enum nand_ecc_placement placement;
const char *pm;
int err;
err = of_property_read_string(np, "nand-ecc-placement", &pm);
if (!err) {
for (placement = NAND_ECC_PLACEMENT_OOB;
placement < ARRAY_SIZE(nand_ecc_placement); placement++) {
if (!strcasecmp(pm, nand_ecc_placement[placement]))
return placement;
}
}
return NAND_ECC_PLACEMENT_UNKNOWN;
}
static const char * const nand_ecc_algos[] = {
[NAND_ECC_ALGO_HAMMING] = "hamming",
[NAND_ECC_ALGO_BCH] = "bch",
[NAND_ECC_ALGO_RS] = "rs",
};
static enum nand_ecc_algo of_get_nand_ecc_algo(struct device_node *np)
{
enum nand_ecc_algo ecc_algo;
const char *pm;
int err;
err = of_property_read_string(np, "nand-ecc-algo", &pm);
if (!err) {
for (ecc_algo = NAND_ECC_ALGO_HAMMING;
ecc_algo < ARRAY_SIZE(nand_ecc_algos);
ecc_algo++) {
if (!strcasecmp(pm, nand_ecc_algos[ecc_algo]))
return ecc_algo;
}
}
return NAND_ECC_ALGO_UNKNOWN;
}
static int of_get_nand_ecc_step_size(struct device_node *np)
{
int ret;
u32 val;
ret = of_property_read_u32(np, "nand-ecc-step-size", &val);
return ret ? ret : val;
}
static int of_get_nand_ecc_strength(struct device_node *np)
{
int ret;
u32 val;
ret = of_property_read_u32(np, "nand-ecc-strength", &val);
return ret ? ret : val;
}
void of_get_nand_ecc_user_config(struct nand_device *nand)
{
struct device_node *dn = nanddev_get_of_node(nand);
int strength, size;
nand->ecc.user_conf.engine_type = of_get_nand_ecc_engine_type(dn);
nand->ecc.user_conf.algo = of_get_nand_ecc_algo(dn);
nand->ecc.user_conf.placement = of_get_nand_ecc_placement(dn);
strength = of_get_nand_ecc_strength(dn);
if (strength >= 0)
nand->ecc.user_conf.strength = strength;
size = of_get_nand_ecc_step_size(dn);
if (size >= 0)
nand->ecc.user_conf.step_size = size;
if (of_property_read_bool(dn, "nand-ecc-maximize"))
nand->ecc.user_conf.flags |= NAND_ECC_MAXIMIZE_STRENGTH;
}
EXPORT_SYMBOL(of_get_nand_ecc_user_config);
/**
* nand_ecc_is_strong_enough - Check if the chip configuration meets the
* datasheet requirements.
*
* @nand: Device to check
*
* If our configuration corrects A bits per B bytes and the minimum
* required correction level is X bits per Y bytes, then we must ensure
* both of the following are true:
*
* (1) A / B >= X / Y
* (2) A >= X
*
* Requirement (1) ensures we can correct for the required bitflip density.
* Requirement (2) ensures we can correct even when all bitflips are clumped
* in the same sector.
*/
bool nand_ecc_is_strong_enough(struct nand_device *nand)
{
const struct nand_ecc_props *reqs = nanddev_get_ecc_requirements(nand);
const struct nand_ecc_props *conf = nanddev_get_ecc_conf(nand);
struct mtd_info *mtd = nanddev_to_mtd(nand);
int corr, ds_corr;
if (conf->step_size == 0 || reqs->step_size == 0)
/* Not enough information */
return true;
/*
* We get the number of corrected bits per page to compare
* the correction density.
*/
corr = (mtd->writesize * conf->strength) / conf->step_size;
ds_corr = (mtd->writesize * reqs->strength) / reqs->step_size;
return corr >= ds_corr && conf->strength >= reqs->strength;
}
EXPORT_SYMBOL(nand_ecc_is_strong_enough);
/* ECC engine driver internal helpers */
int nand_ecc_init_req_tweaking(struct nand_ecc_req_tweak_ctx *ctx,
struct nand_device *nand)
{
unsigned int total_buffer_size;
ctx->nand = nand;
/* Let the user decide the exact length of each buffer */
if (!ctx->page_buffer_size)
ctx->page_buffer_size = nanddev_page_size(nand);
if (!ctx->oob_buffer_size)
ctx->oob_buffer_size = nanddev_per_page_oobsize(nand);
total_buffer_size = ctx->page_buffer_size + ctx->oob_buffer_size;
ctx->spare_databuf = kzalloc(total_buffer_size, GFP_KERNEL);
if (!ctx->spare_databuf)
return -ENOMEM;
ctx->spare_oobbuf = ctx->spare_databuf + ctx->page_buffer_size;
return 0;
}
EXPORT_SYMBOL_GPL(nand_ecc_init_req_tweaking);
void nand_ecc_cleanup_req_tweaking(struct nand_ecc_req_tweak_ctx *ctx)
{
kfree(ctx->spare_databuf);
}
EXPORT_SYMBOL_GPL(nand_ecc_cleanup_req_tweaking);
/*
* Ensure data and OOB area is fully read/written otherwise the correction might
* not work as expected.
*/
void nand_ecc_tweak_req(struct nand_ecc_req_tweak_ctx *ctx,
struct nand_page_io_req *req)
{
struct nand_device *nand = ctx->nand;
struct nand_page_io_req *orig, *tweak;
/* Save the original request */
ctx->orig_req = *req;
ctx->bounce_data = false;
ctx->bounce_oob = false;
orig = &ctx->orig_req;
tweak = req;
/* Ensure the request covers the entire page */
if (orig->datalen < nanddev_page_size(nand)) {
ctx->bounce_data = true;
tweak->dataoffs = 0;
tweak->datalen = nanddev_page_size(nand);
tweak->databuf.in = ctx->spare_databuf;
memset(tweak->databuf.in, 0xFF, ctx->page_buffer_size);
}
if (orig->ooblen < nanddev_per_page_oobsize(nand)) {
ctx->bounce_oob = true;
tweak->ooboffs = 0;
tweak->ooblen = nanddev_per_page_oobsize(nand);
tweak->oobbuf.in = ctx->spare_oobbuf;
memset(tweak->oobbuf.in, 0xFF, ctx->oob_buffer_size);
}
/* Copy the data that must be writen in the bounce buffers, if needed */
if (orig->type == NAND_PAGE_WRITE) {
if (ctx->bounce_data)
memcpy((void *)tweak->databuf.out + orig->dataoffs,
orig->databuf.out, orig->datalen);
if (ctx->bounce_oob)
memcpy((void *)tweak->oobbuf.out + orig->ooboffs,
orig->oobbuf.out, orig->ooblen);
}
}
EXPORT_SYMBOL_GPL(nand_ecc_tweak_req);
void nand_ecc_restore_req(struct nand_ecc_req_tweak_ctx *ctx,
struct nand_page_io_req *req)
{
struct nand_page_io_req *orig, *tweak;
orig = &ctx->orig_req;
tweak = req;
/* Restore the data read from the bounce buffers, if needed */
if (orig->type == NAND_PAGE_READ) {
if (ctx->bounce_data)
memcpy(orig->databuf.in,
tweak->databuf.in + orig->dataoffs,
orig->datalen);
if (ctx->bounce_oob)
memcpy(orig->oobbuf.in,
tweak->oobbuf.in + orig->ooboffs,
orig->ooblen);
}
/* Ensure the original request is restored */
*req = *orig;
}
EXPORT_SYMBOL_GPL(nand_ecc_restore_req);
struct nand_ecc_engine *nand_ecc_get_sw_engine(struct nand_device *nand)
{
unsigned int algo = nand->ecc.user_conf.algo;
if (algo == NAND_ECC_ALGO_UNKNOWN)
algo = nand->ecc.defaults.algo;
switch (algo) {
case NAND_ECC_ALGO_HAMMING:
return nand_ecc_sw_hamming_get_engine();
case NAND_ECC_ALGO_BCH:
return nand_ecc_sw_bch_get_engine();
default:
break;
}
return NULL;
}
EXPORT_SYMBOL(nand_ecc_get_sw_engine);
struct nand_ecc_engine *nand_ecc_get_on_die_hw_engine(struct nand_device *nand)
{
return nand->ecc.ondie_engine;
}
EXPORT_SYMBOL(nand_ecc_get_on_die_hw_engine);
int nand_ecc_register_on_host_hw_engine(struct nand_ecc_engine *engine)
{
struct nand_ecc_engine *item;
if (!engine)
return -EINVAL;
/* Prevent multiple registrations of one engine */
list_for_each_entry(item, &on_host_hw_engines, node)
if (item == engine)
return 0;
mutex_lock(&on_host_hw_engines_mutex);
list_add_tail(&engine->node, &on_host_hw_engines);
mutex_unlock(&on_host_hw_engines_mutex);
return 0;
}
EXPORT_SYMBOL(nand_ecc_register_on_host_hw_engine);
int nand_ecc_unregister_on_host_hw_engine(struct nand_ecc_engine *engine)
{
if (!engine)
return -EINVAL;
mutex_lock(&on_host_hw_engines_mutex);
list_del(&engine->node);
mutex_unlock(&on_host_hw_engines_mutex);
return 0;
}
EXPORT_SYMBOL(nand_ecc_unregister_on_host_hw_engine);
static struct nand_ecc_engine *nand_ecc_match_on_host_hw_engine(struct device *dev)
{
struct nand_ecc_engine *item;
list_for_each_entry(item, &on_host_hw_engines, node)
if (item->dev == dev)
return item;
return NULL;
}
struct nand_ecc_engine *nand_ecc_get_on_host_hw_engine(struct nand_device *nand)
{
struct nand_ecc_engine *engine = NULL;
struct device *dev = &nand->mtd.dev;
struct platform_device *pdev;
struct device_node *np;
if (list_empty(&on_host_hw_engines))
return NULL;
/* Check for an explicit nand-ecc-engine property */
np = of_parse_phandle(dev->of_node, "nand-ecc-engine", 0);
if (np) {
pdev = of_find_device_by_node(np);
if (!pdev)
return ERR_PTR(-EPROBE_DEFER);
engine = nand_ecc_match_on_host_hw_engine(&pdev->dev);
platform_device_put(pdev);
of_node_put(np);
if (!engine)
return ERR_PTR(-EPROBE_DEFER);
}
if (engine)
get_device(engine->dev);
return engine;
}
EXPORT_SYMBOL(nand_ecc_get_on_host_hw_engine);
void nand_ecc_put_on_host_hw_engine(struct nand_device *nand)
{
put_device(nand->ecc.engine->dev);
}
EXPORT_SYMBOL(nand_ecc_put_on_host_hw_engine);
/*
* In the case of a pipelined engine, the device registering the ECC
* engine is not necessarily the ECC engine itself but may be a host controller.
* It is then useful to provide a helper to retrieve the right device object
* which actually represents the ECC engine.
*/
struct device *nand_ecc_get_engine_dev(struct device *host)
{
struct platform_device *ecc_pdev;
struct device_node *np;
/*
* If the device node contains this property, it means we need to follow
* it in order to get the right ECC engine device we are looking for.
*/
np = of_parse_phandle(host->of_node, "nand-ecc-engine", 0);
if (!np)
return host;
ecc_pdev = of_find_device_by_node(np);
if (!ecc_pdev) {
of_node_put(np);
return NULL;
}
platform_device_put(ecc_pdev);
of_node_put(np);
return &ecc_pdev->dev;
}
MODULE_LICENSE("GPL");
MODULE_AUTHOR("Miquel Raynal <miquel.raynal@bootlin.com>");
MODULE_DESCRIPTION("Generic ECC engine");