linux/crypto/api.c

715 lines
16 KiB
C
Raw Normal View History

// SPDX-License-Identifier: GPL-2.0-or-later
/*
* Scatterlist Cryptographic API.
*
* Copyright (c) 2002 James Morris <jmorris@intercode.com.au>
* Copyright (c) 2002 David S. Miller (davem@redhat.com)
* Copyright (c) 2005 Herbert Xu <herbert@gondor.apana.org.au>
*
* Portions derived from Cryptoapi, by Alexander Kjeldaas <astor@fast.no>
* and Nettle, by Niels Möller.
*/
#include <linux/err.h>
#include <linux/errno.h>
#include <linux/jump_label.h>
#include <linux/kernel.h>
#include <linux/kmod.h>
#include <linux/module.h>
#include <linux/param.h>
#include <linux/sched/signal.h>
#include <linux/slab.h>
#include <linux/string.h>
#include <linux/completion.h>
#include "internal.h"
LIST_HEAD(crypto_alg_list);
EXPORT_SYMBOL_GPL(crypto_alg_list);
DECLARE_RWSEM(crypto_alg_sem);
EXPORT_SYMBOL_GPL(crypto_alg_sem);
BLOCKING_NOTIFIER_HEAD(crypto_chain);
EXPORT_SYMBOL_GPL(crypto_chain);
#if IS_BUILTIN(CONFIG_CRYPTO_ALGAPI) && \
!IS_ENABLED(CONFIG_CRYPTO_MANAGER_DISABLE_TESTS)
DEFINE_STATIC_KEY_FALSE(__crypto_boot_test_finished);
#endif
static struct crypto_alg *crypto_larval_wait(struct crypto_alg *alg);
static struct crypto_alg *crypto_alg_lookup(const char *name, u32 type,
u32 mask);
struct crypto_alg *crypto_mod_get(struct crypto_alg *alg)
{
return try_module_get(alg->cra_module) ? crypto_alg_get(alg) : NULL;
}
EXPORT_SYMBOL_GPL(crypto_mod_get);
void crypto_mod_put(struct crypto_alg *alg)
{
struct module *module = alg->cra_module;
crypto_alg_put(alg);
module_put(module);
}
EXPORT_SYMBOL_GPL(crypto_mod_put);
static struct crypto_alg *__crypto_alg_lookup(const char *name, u32 type,
u32 mask)
{
struct crypto_alg *q, *alg = NULL;
int best = -2;
list_for_each_entry(q, &crypto_alg_list, cra_list) {
int exact, fuzzy;
if (crypto_is_moribund(q))
continue;
if ((q->cra_flags ^ type) & mask)
continue;
exact = !strcmp(q->cra_driver_name, name);
fuzzy = !strcmp(q->cra_name, name);
if (!exact && !(fuzzy && q->cra_priority > best))
continue;
if (unlikely(!crypto_mod_get(q)))
continue;
best = q->cra_priority;
if (alg)
crypto_mod_put(alg);
alg = q;
if (exact)
break;
}
return alg;
}
static void crypto_larval_destroy(struct crypto_alg *alg)
{
struct crypto_larval *larval = (void *)alg;
BUG_ON(!crypto_is_larval(alg));
if (!IS_ERR_OR_NULL(larval->adult))
crypto_mod_put(larval->adult);
kfree(larval);
}
struct crypto_larval *crypto_larval_alloc(const char *name, u32 type, u32 mask)
{
struct crypto_larval *larval;
larval = kzalloc(sizeof(*larval), GFP_KERNEL);
if (!larval)
return ERR_PTR(-ENOMEM);
crypto: api - Fix generic algorithm self-test races On Fri, Aug 30, 2024 at 10:51:54AM -0700, Eric Biggers wrote: > > Given below in defconfig form, use 'make olddefconfig' to apply. The failures > are nondeterministic and sometimes there are different ones, for example: > > [ 0.358017] alg: skcipher: failed to allocate transform for cbc(twofish-generic): -2 > [ 0.358365] alg: self-tests for cbc(twofish) using cbc(twofish-generic) failed (rc=-2) > [ 0.358535] alg: skcipher: failed to allocate transform for cbc(camellia-generic): -2 > [ 0.358918] alg: self-tests for cbc(camellia) using cbc(camellia-generic) failed (rc=-2) > [ 0.371533] alg: skcipher: failed to allocate transform for xts(ecb(aes-generic)): -2 > [ 0.371922] alg: self-tests for xts(aes) using xts(ecb(aes-generic)) failed (rc=-2) > > Modules are not enabled, maybe that matters (I haven't checked yet). Yes I think that was the key. This triggers a massive self-test run which executes in parallel and reveals a few race conditions in the system. I think it boils down to the following scenario: Base algorithm X-generic, X-optimised Template Y Optimised algorithm Y-X-optimised Everything gets registered, and then the self-tests are started. When Y-X-optimised gets tested, it requests the creation of the generic Y(X-generic). Which then itself undergoes testing. The race is that after Y(X-generic) gets registered, but just before it gets tested, X-optimised finally finishes self-testing which then causes all spawns of X-generic to be destroyed. So by the time the self-test for Y(X-generic) comes along, it can no longer find the algorithm. This error then bubbles up all the way up to the self-test of Y-X-optimised which then fails. Note that there is some complexity that I've omitted here because when the generic self-test fails to find Y(X-generic) it actually triggers the construction of it again which then fails for various other reasons (these are not important because the construction should *not* be triggered at this point). So in a way the error is expected, and we should probably remove the pr_err for the case where ENOENT is returned for the algorithm that we're currently testing. The solution is two-fold. First when an algorithm undergoes self-testing it should not trigger its construction. Secondly if an instance larval fails to materialise due to it being destroyed by a more optimised algorithm coming along, it should obviously retry the construction. Remove the check in __crypto_alg_lookup that stops a larval from matching new requests based on differences in the mask. It is better to block new requests even if it is wrong and then simply retry the lookup. If this ends up being the wrong larval it will sort iself out during the retry. Reduce the CRYPTO_ALG_TYPE_MASK bits in type during larval creation as otherwise LSKCIPHER algorithms may not match SKCIPHER larvals. Also block the instance creation during self-testing in the function crypto_larval_lookup by checking for CRYPTO_ALG_TESTED in the mask field. Finally change the return value when crypto_alg_lookup fails in crypto_larval_wait to EAGAIN to redo the lookup. Fixes: 37da5d0ffa7b ("crypto: api - Do not wait for tests during registration") Reported-by: Eric Biggers <ebiggers@kernel.org> Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au>
2024-09-01 08:05:40 +00:00
type &= ~CRYPTO_ALG_TYPE_MASK | (mask ?: CRYPTO_ALG_TYPE_MASK);
larval->mask = mask;
larval->alg.cra_flags = CRYPTO_ALG_LARVAL | type;
larval->alg.cra_priority = -1;
larval->alg.cra_destroy = crypto_larval_destroy;
strscpy(larval->alg.cra_name, name, CRYPTO_MAX_ALG_NAME);
init_completion(&larval->completion);
return larval;
}
EXPORT_SYMBOL_GPL(crypto_larval_alloc);
static struct crypto_alg *crypto_larval_add(const char *name, u32 type,
u32 mask)
{
struct crypto_alg *alg;
struct crypto_larval *larval;
larval = crypto_larval_alloc(name, type, mask);
if (IS_ERR(larval))
return ERR_CAST(larval);
refcount_set(&larval->alg.cra_refcnt, 2);
down_write(&crypto_alg_sem);
alg = __crypto_alg_lookup(name, type, mask);
if (!alg) {
alg = &larval->alg;
list_add(&alg->cra_list, &crypto_alg_list);
}
up_write(&crypto_alg_sem);
if (alg != &larval->alg) {
kfree(larval);
if (crypto_is_larval(alg))
alg = crypto_larval_wait(alg);
}
return alg;
}
static void crypto_larval_kill(struct crypto_larval *larval)
{
bool unlinked;
down_write(&crypto_alg_sem);
unlinked = list_empty(&larval->alg.cra_list);
if (!unlinked)
list_del_init(&larval->alg.cra_list);
up_write(&crypto_alg_sem);
if (unlinked)
return;
complete_all(&larval->completion);
crypto_alg_put(&larval->alg);
}
void crypto_schedule_test(struct crypto_larval *larval)
{
int err;
err = crypto_probing_notify(CRYPTO_MSG_ALG_REGISTER, larval->adult);
WARN_ON_ONCE(err != NOTIFY_STOP);
}
EXPORT_SYMBOL_GPL(crypto_schedule_test);
static void crypto_start_test(struct crypto_larval *larval)
{
if (!crypto_is_test_larval(larval))
return;
if (larval->test_started)
return;
down_write(&crypto_alg_sem);
if (larval->test_started) {
up_write(&crypto_alg_sem);
return;
}
larval->test_started = true;
up_write(&crypto_alg_sem);
crypto_schedule_test(larval);
}
static struct crypto_alg *crypto_larval_wait(struct crypto_alg *alg)
{
struct crypto_larval *larval;
long time_left;
again:
larval = container_of(alg, struct crypto_larval, alg);
if (!crypto_boot_test_finished())
crypto_start_test(larval);
time_left = wait_for_completion_killable_timeout(
&larval->completion, 60 * HZ);
alg = larval->adult;
if (time_left < 0)
alg = ERR_PTR(-EINTR);
else if (!time_left) {
if (crypto_is_test_larval(larval))
crypto_larval_kill(larval);
alg = ERR_PTR(-ETIMEDOUT);
} else if (!alg) {
u32 type;
u32 mask;
alg = &larval->alg;
type = alg->cra_flags & ~(CRYPTO_ALG_LARVAL | CRYPTO_ALG_DEAD);
mask = larval->mask;
alg = crypto_alg_lookup(alg->cra_name, type, mask) ?:
crypto: api - Fix generic algorithm self-test races On Fri, Aug 30, 2024 at 10:51:54AM -0700, Eric Biggers wrote: > > Given below in defconfig form, use 'make olddefconfig' to apply. The failures > are nondeterministic and sometimes there are different ones, for example: > > [ 0.358017] alg: skcipher: failed to allocate transform for cbc(twofish-generic): -2 > [ 0.358365] alg: self-tests for cbc(twofish) using cbc(twofish-generic) failed (rc=-2) > [ 0.358535] alg: skcipher: failed to allocate transform for cbc(camellia-generic): -2 > [ 0.358918] alg: self-tests for cbc(camellia) using cbc(camellia-generic) failed (rc=-2) > [ 0.371533] alg: skcipher: failed to allocate transform for xts(ecb(aes-generic)): -2 > [ 0.371922] alg: self-tests for xts(aes) using xts(ecb(aes-generic)) failed (rc=-2) > > Modules are not enabled, maybe that matters (I haven't checked yet). Yes I think that was the key. This triggers a massive self-test run which executes in parallel and reveals a few race conditions in the system. I think it boils down to the following scenario: Base algorithm X-generic, X-optimised Template Y Optimised algorithm Y-X-optimised Everything gets registered, and then the self-tests are started. When Y-X-optimised gets tested, it requests the creation of the generic Y(X-generic). Which then itself undergoes testing. The race is that after Y(X-generic) gets registered, but just before it gets tested, X-optimised finally finishes self-testing which then causes all spawns of X-generic to be destroyed. So by the time the self-test for Y(X-generic) comes along, it can no longer find the algorithm. This error then bubbles up all the way up to the self-test of Y-X-optimised which then fails. Note that there is some complexity that I've omitted here because when the generic self-test fails to find Y(X-generic) it actually triggers the construction of it again which then fails for various other reasons (these are not important because the construction should *not* be triggered at this point). So in a way the error is expected, and we should probably remove the pr_err for the case where ENOENT is returned for the algorithm that we're currently testing. The solution is two-fold. First when an algorithm undergoes self-testing it should not trigger its construction. Secondly if an instance larval fails to materialise due to it being destroyed by a more optimised algorithm coming along, it should obviously retry the construction. Remove the check in __crypto_alg_lookup that stops a larval from matching new requests based on differences in the mask. It is better to block new requests even if it is wrong and then simply retry the lookup. If this ends up being the wrong larval it will sort iself out during the retry. Reduce the CRYPTO_ALG_TYPE_MASK bits in type during larval creation as otherwise LSKCIPHER algorithms may not match SKCIPHER larvals. Also block the instance creation during self-testing in the function crypto_larval_lookup by checking for CRYPTO_ALG_TESTED in the mask field. Finally change the return value when crypto_alg_lookup fails in crypto_larval_wait to EAGAIN to redo the lookup. Fixes: 37da5d0ffa7b ("crypto: api - Do not wait for tests during registration") Reported-by: Eric Biggers <ebiggers@kernel.org> Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au>
2024-09-01 08:05:40 +00:00
ERR_PTR(-EAGAIN);
} else if (IS_ERR(alg))
;
else if (crypto_is_test_larval(larval) &&
!(alg->cra_flags & CRYPTO_ALG_TESTED))
alg = ERR_PTR(-EAGAIN);
crypto: api - allow algs only in specific constructions in FIPS mode Currently we do not distinguish between algorithms that fail on the self-test vs. those which are disabled in FIPS mode (not allowed). Both are marked as having failed the self-test. Recently the need arose to allow the usage of certain algorithms only as arguments to specific template instantiations in FIPS mode. For example, standalone "dh" must be blocked, but e.g. "ffdhe2048(dh)" is allowed. Other potential use cases include "cbcmac(aes)", which must only be used with ccm(), or "ghash", which must be used only for gcm(). This patch allows this scenario by adding a new flag FIPS_INTERNAL to indicate those algorithms that are not FIPS-allowed. They can then be used as template arguments only, i.e. when looked up via crypto_grab_spawn() to be more specific. The FIPS_INTERNAL bit gets propagated upwards recursively into the surrounding template instances, until the construction eventually matches an explicit testmgr entry with ->fips_allowed being set, if any. The behaviour to skip !->fips_allowed self-test executions in FIPS mode will be retained. Note that this effectively means that FIPS_INTERNAL algorithms are handled very similarly to the INTERNAL ones in this regard. It is expected that the FIPS_INTERNAL algorithms will receive sufficient testing when the larger constructions they're a part of, if any, get exercised by testmgr. Note that as a side-effect of this patch algorithms which are not FIPS-allowed will now return ENOENT instead of ELIBBAD. Hopefully this is not an issue as some people were relying on this already. Link: https://lore.kernel.org/r/YeEVSaMEVJb3cQkq@gondor.apana.org.au Originally-by: Herbert Xu <herbert@gondor.apana.org.au> Signed-off-by: Nicolai Stange <nstange@suse.de> Reviewed-by: Hannes Reinecke <hare@suse.de> Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au>
2022-02-21 12:10:58 +00:00
else if (alg->cra_flags & CRYPTO_ALG_FIPS_INTERNAL)
alg = ERR_PTR(-EAGAIN);
else if (!crypto_mod_get(alg))
alg = ERR_PTR(-EAGAIN);
crypto_mod_put(&larval->alg);
if (!IS_ERR(alg) && crypto_is_larval(alg))
goto again;
return alg;
}
static struct crypto_alg *crypto_alg_lookup(const char *name, u32 type,
u32 mask)
{
crypto: api - allow algs only in specific constructions in FIPS mode Currently we do not distinguish between algorithms that fail on the self-test vs. those which are disabled in FIPS mode (not allowed). Both are marked as having failed the self-test. Recently the need arose to allow the usage of certain algorithms only as arguments to specific template instantiations in FIPS mode. For example, standalone "dh" must be blocked, but e.g. "ffdhe2048(dh)" is allowed. Other potential use cases include "cbcmac(aes)", which must only be used with ccm(), or "ghash", which must be used only for gcm(). This patch allows this scenario by adding a new flag FIPS_INTERNAL to indicate those algorithms that are not FIPS-allowed. They can then be used as template arguments only, i.e. when looked up via crypto_grab_spawn() to be more specific. The FIPS_INTERNAL bit gets propagated upwards recursively into the surrounding template instances, until the construction eventually matches an explicit testmgr entry with ->fips_allowed being set, if any. The behaviour to skip !->fips_allowed self-test executions in FIPS mode will be retained. Note that this effectively means that FIPS_INTERNAL algorithms are handled very similarly to the INTERNAL ones in this regard. It is expected that the FIPS_INTERNAL algorithms will receive sufficient testing when the larger constructions they're a part of, if any, get exercised by testmgr. Note that as a side-effect of this patch algorithms which are not FIPS-allowed will now return ENOENT instead of ELIBBAD. Hopefully this is not an issue as some people were relying on this already. Link: https://lore.kernel.org/r/YeEVSaMEVJb3cQkq@gondor.apana.org.au Originally-by: Herbert Xu <herbert@gondor.apana.org.au> Signed-off-by: Nicolai Stange <nstange@suse.de> Reviewed-by: Hannes Reinecke <hare@suse.de> Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au>
2022-02-21 12:10:58 +00:00
const u32 fips = CRYPTO_ALG_FIPS_INTERNAL;
struct crypto_alg *alg;
u32 test = 0;
if (!((type | mask) & CRYPTO_ALG_TESTED))
test |= CRYPTO_ALG_TESTED;
down_read(&crypto_alg_sem);
crypto: api - allow algs only in specific constructions in FIPS mode Currently we do not distinguish between algorithms that fail on the self-test vs. those which are disabled in FIPS mode (not allowed). Both are marked as having failed the self-test. Recently the need arose to allow the usage of certain algorithms only as arguments to specific template instantiations in FIPS mode. For example, standalone "dh" must be blocked, but e.g. "ffdhe2048(dh)" is allowed. Other potential use cases include "cbcmac(aes)", which must only be used with ccm(), or "ghash", which must be used only for gcm(). This patch allows this scenario by adding a new flag FIPS_INTERNAL to indicate those algorithms that are not FIPS-allowed. They can then be used as template arguments only, i.e. when looked up via crypto_grab_spawn() to be more specific. The FIPS_INTERNAL bit gets propagated upwards recursively into the surrounding template instances, until the construction eventually matches an explicit testmgr entry with ->fips_allowed being set, if any. The behaviour to skip !->fips_allowed self-test executions in FIPS mode will be retained. Note that this effectively means that FIPS_INTERNAL algorithms are handled very similarly to the INTERNAL ones in this regard. It is expected that the FIPS_INTERNAL algorithms will receive sufficient testing when the larger constructions they're a part of, if any, get exercised by testmgr. Note that as a side-effect of this patch algorithms which are not FIPS-allowed will now return ENOENT instead of ELIBBAD. Hopefully this is not an issue as some people were relying on this already. Link: https://lore.kernel.org/r/YeEVSaMEVJb3cQkq@gondor.apana.org.au Originally-by: Herbert Xu <herbert@gondor.apana.org.au> Signed-off-by: Nicolai Stange <nstange@suse.de> Reviewed-by: Hannes Reinecke <hare@suse.de> Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au>
2022-02-21 12:10:58 +00:00
alg = __crypto_alg_lookup(name, (type | test) & ~fips,
(mask | test) & ~fips);
if (alg) {
if (((type | mask) ^ fips) & fips)
mask |= fips;
mask &= fips;
if (!crypto_is_larval(alg) &&
((type ^ alg->cra_flags) & mask)) {
/* Algorithm is disallowed in FIPS mode. */
crypto_mod_put(alg);
alg = ERR_PTR(-ENOENT);
}
} else if (test) {
alg = __crypto_alg_lookup(name, type, mask);
if (alg && !crypto_is_larval(alg)) {
/* Test failed */
crypto_mod_put(alg);
alg = ERR_PTR(-ELIBBAD);
}
}
up_read(&crypto_alg_sem);
return alg;
}
static struct crypto_alg *crypto_larval_lookup(const char *name, u32 type,
u32 mask)
{
struct crypto_alg *alg;
if (!name)
return ERR_PTR(-ENOENT);
type &= ~(CRYPTO_ALG_LARVAL | CRYPTO_ALG_DEAD);
mask &= ~(CRYPTO_ALG_LARVAL | CRYPTO_ALG_DEAD);
alg = crypto_alg_lookup(name, type, mask);
if (!alg && !(mask & CRYPTO_NOLOAD)) {
request_module("crypto-%s", name);
if (!((type ^ CRYPTO_ALG_NEED_FALLBACK) & mask &
CRYPTO_ALG_NEED_FALLBACK))
request_module("crypto-%s-all", name);
alg = crypto_alg_lookup(name, type, mask);
}
if (!IS_ERR_OR_NULL(alg) && crypto_is_larval(alg))
alg = crypto_larval_wait(alg);
crypto: api - Fix generic algorithm self-test races On Fri, Aug 30, 2024 at 10:51:54AM -0700, Eric Biggers wrote: > > Given below in defconfig form, use 'make olddefconfig' to apply. The failures > are nondeterministic and sometimes there are different ones, for example: > > [ 0.358017] alg: skcipher: failed to allocate transform for cbc(twofish-generic): -2 > [ 0.358365] alg: self-tests for cbc(twofish) using cbc(twofish-generic) failed (rc=-2) > [ 0.358535] alg: skcipher: failed to allocate transform for cbc(camellia-generic): -2 > [ 0.358918] alg: self-tests for cbc(camellia) using cbc(camellia-generic) failed (rc=-2) > [ 0.371533] alg: skcipher: failed to allocate transform for xts(ecb(aes-generic)): -2 > [ 0.371922] alg: self-tests for xts(aes) using xts(ecb(aes-generic)) failed (rc=-2) > > Modules are not enabled, maybe that matters (I haven't checked yet). Yes I think that was the key. This triggers a massive self-test run which executes in parallel and reveals a few race conditions in the system. I think it boils down to the following scenario: Base algorithm X-generic, X-optimised Template Y Optimised algorithm Y-X-optimised Everything gets registered, and then the self-tests are started. When Y-X-optimised gets tested, it requests the creation of the generic Y(X-generic). Which then itself undergoes testing. The race is that after Y(X-generic) gets registered, but just before it gets tested, X-optimised finally finishes self-testing which then causes all spawns of X-generic to be destroyed. So by the time the self-test for Y(X-generic) comes along, it can no longer find the algorithm. This error then bubbles up all the way up to the self-test of Y-X-optimised which then fails. Note that there is some complexity that I've omitted here because when the generic self-test fails to find Y(X-generic) it actually triggers the construction of it again which then fails for various other reasons (these are not important because the construction should *not* be triggered at this point). So in a way the error is expected, and we should probably remove the pr_err for the case where ENOENT is returned for the algorithm that we're currently testing. The solution is two-fold. First when an algorithm undergoes self-testing it should not trigger its construction. Secondly if an instance larval fails to materialise due to it being destroyed by a more optimised algorithm coming along, it should obviously retry the construction. Remove the check in __crypto_alg_lookup that stops a larval from matching new requests based on differences in the mask. It is better to block new requests even if it is wrong and then simply retry the lookup. If this ends up being the wrong larval it will sort iself out during the retry. Reduce the CRYPTO_ALG_TYPE_MASK bits in type during larval creation as otherwise LSKCIPHER algorithms may not match SKCIPHER larvals. Also block the instance creation during self-testing in the function crypto_larval_lookup by checking for CRYPTO_ALG_TESTED in the mask field. Finally change the return value when crypto_alg_lookup fails in crypto_larval_wait to EAGAIN to redo the lookup. Fixes: 37da5d0ffa7b ("crypto: api - Do not wait for tests during registration") Reported-by: Eric Biggers <ebiggers@kernel.org> Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au>
2024-09-01 08:05:40 +00:00
else if (alg)
;
else if (!(mask & CRYPTO_ALG_TESTED))
alg = crypto_larval_add(name, type, mask);
crypto: api - Fix generic algorithm self-test races On Fri, Aug 30, 2024 at 10:51:54AM -0700, Eric Biggers wrote: > > Given below in defconfig form, use 'make olddefconfig' to apply. The failures > are nondeterministic and sometimes there are different ones, for example: > > [ 0.358017] alg: skcipher: failed to allocate transform for cbc(twofish-generic): -2 > [ 0.358365] alg: self-tests for cbc(twofish) using cbc(twofish-generic) failed (rc=-2) > [ 0.358535] alg: skcipher: failed to allocate transform for cbc(camellia-generic): -2 > [ 0.358918] alg: self-tests for cbc(camellia) using cbc(camellia-generic) failed (rc=-2) > [ 0.371533] alg: skcipher: failed to allocate transform for xts(ecb(aes-generic)): -2 > [ 0.371922] alg: self-tests for xts(aes) using xts(ecb(aes-generic)) failed (rc=-2) > > Modules are not enabled, maybe that matters (I haven't checked yet). Yes I think that was the key. This triggers a massive self-test run which executes in parallel and reveals a few race conditions in the system. I think it boils down to the following scenario: Base algorithm X-generic, X-optimised Template Y Optimised algorithm Y-X-optimised Everything gets registered, and then the self-tests are started. When Y-X-optimised gets tested, it requests the creation of the generic Y(X-generic). Which then itself undergoes testing. The race is that after Y(X-generic) gets registered, but just before it gets tested, X-optimised finally finishes self-testing which then causes all spawns of X-generic to be destroyed. So by the time the self-test for Y(X-generic) comes along, it can no longer find the algorithm. This error then bubbles up all the way up to the self-test of Y-X-optimised which then fails. Note that there is some complexity that I've omitted here because when the generic self-test fails to find Y(X-generic) it actually triggers the construction of it again which then fails for various other reasons (these are not important because the construction should *not* be triggered at this point). So in a way the error is expected, and we should probably remove the pr_err for the case where ENOENT is returned for the algorithm that we're currently testing. The solution is two-fold. First when an algorithm undergoes self-testing it should not trigger its construction. Secondly if an instance larval fails to materialise due to it being destroyed by a more optimised algorithm coming along, it should obviously retry the construction. Remove the check in __crypto_alg_lookup that stops a larval from matching new requests based on differences in the mask. It is better to block new requests even if it is wrong and then simply retry the lookup. If this ends up being the wrong larval it will sort iself out during the retry. Reduce the CRYPTO_ALG_TYPE_MASK bits in type during larval creation as otherwise LSKCIPHER algorithms may not match SKCIPHER larvals. Also block the instance creation during self-testing in the function crypto_larval_lookup by checking for CRYPTO_ALG_TESTED in the mask field. Finally change the return value when crypto_alg_lookup fails in crypto_larval_wait to EAGAIN to redo the lookup. Fixes: 37da5d0ffa7b ("crypto: api - Do not wait for tests during registration") Reported-by: Eric Biggers <ebiggers@kernel.org> Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au>
2024-09-01 08:05:40 +00:00
else
alg = ERR_PTR(-ENOENT);
return alg;
}
int crypto_probing_notify(unsigned long val, void *v)
{
int ok;
ok = blocking_notifier_call_chain(&crypto_chain, val, v);
if (ok == NOTIFY_DONE) {
request_module("cryptomgr");
ok = blocking_notifier_call_chain(&crypto_chain, val, v);
}
return ok;
}
EXPORT_SYMBOL_GPL(crypto_probing_notify);
struct crypto_alg *crypto_alg_mod_lookup(const char *name, u32 type, u32 mask)
{
struct crypto_alg *alg;
struct crypto_alg *larval;
int ok;
/*
* If the internal flag is set for a cipher, require a caller to
* invoke the cipher with the internal flag to use that cipher.
* Also, if a caller wants to allocate a cipher that may or may
* not be an internal cipher, use type | CRYPTO_ALG_INTERNAL and
* !(mask & CRYPTO_ALG_INTERNAL).
*/
if (!((type | mask) & CRYPTO_ALG_INTERNAL))
mask |= CRYPTO_ALG_INTERNAL;
larval = crypto_larval_lookup(name, type, mask);
if (IS_ERR(larval) || !crypto_is_larval(larval))
return larval;
ok = crypto_probing_notify(CRYPTO_MSG_ALG_REQUEST, larval);
if (ok == NOTIFY_STOP)
alg = crypto_larval_wait(larval);
else {
crypto_mod_put(larval);
alg = ERR_PTR(-ENOENT);
}
crypto_larval_kill(container_of(larval, struct crypto_larval, alg));
return alg;
}
EXPORT_SYMBOL_GPL(crypto_alg_mod_lookup);
static void crypto_exit_ops(struct crypto_tfm *tfm)
{
const struct crypto_type *type = tfm->__crt_alg->cra_type;
if (type && tfm->exit)
tfm->exit(tfm);
}
static unsigned int crypto_ctxsize(struct crypto_alg *alg, u32 type, u32 mask)
{
const struct crypto_type *type_obj = alg->cra_type;
unsigned int len;
len = alg->cra_alignmask & ~(crypto_tfm_ctx_alignment() - 1);
if (type_obj)
return len + type_obj->ctxsize(alg, type, mask);
switch (alg->cra_flags & CRYPTO_ALG_TYPE_MASK) {
default:
BUG();
case CRYPTO_ALG_TYPE_CIPHER:
len += crypto_cipher_ctxsize(alg);
break;
case CRYPTO_ALG_TYPE_COMPRESS:
len += crypto_compress_ctxsize(alg);
break;
}
return len;
}
void crypto_shoot_alg(struct crypto_alg *alg)
{
down_write(&crypto_alg_sem);
alg->cra_flags |= CRYPTO_ALG_DYING;
up_write(&crypto_alg_sem);
}
EXPORT_SYMBOL_GPL(crypto_shoot_alg);
struct crypto_tfm *__crypto_alloc_tfmgfp(struct crypto_alg *alg, u32 type,
u32 mask, gfp_t gfp)
{
struct crypto_tfm *tfm;
unsigned int tfm_size;
int err = -ENOMEM;
tfm_size = sizeof(*tfm) + crypto_ctxsize(alg, type, mask);
tfm = kzalloc(tfm_size, gfp);
if (tfm == NULL)
goto out_err;
tfm->__crt_alg = alg;
refcount_set(&tfm->refcnt, 1);
if (!tfm->exit && alg->cra_init && (err = alg->cra_init(tfm)))
goto cra_init_failed;
goto out;
cra_init_failed:
crypto_exit_ops(tfm);
if (err == -EAGAIN)
crypto_shoot_alg(alg);
kfree(tfm);
out_err:
tfm = ERR_PTR(err);
out:
return tfm;
}
EXPORT_SYMBOL_GPL(__crypto_alloc_tfmgfp);
struct crypto_tfm *__crypto_alloc_tfm(struct crypto_alg *alg, u32 type,
u32 mask)
{
return __crypto_alloc_tfmgfp(alg, type, mask, GFP_KERNEL);
}
EXPORT_SYMBOL_GPL(__crypto_alloc_tfm);
/*
* crypto_alloc_base - Locate algorithm and allocate transform
* @alg_name: Name of algorithm
* @type: Type of algorithm
* @mask: Mask for type comparison
*
* This function should not be used by new algorithm types.
* Please use crypto_alloc_tfm instead.
*
* crypto_alloc_base() will first attempt to locate an already loaded
* algorithm. If that fails and the kernel supports dynamically loadable
* modules, it will then attempt to load a module of the same name or
* alias. If that fails it will send a query to any loaded crypto manager
* to construct an algorithm on the fly. A refcount is grabbed on the
* algorithm which is then associated with the new transform.
*
* The returned transform is of a non-determinate type. Most people
* should use one of the more specific allocation functions such as
* crypto_alloc_skcipher().
*
* In case of error the return value is an error pointer.
*/
struct crypto_tfm *crypto_alloc_base(const char *alg_name, u32 type, u32 mask)
{
struct crypto_tfm *tfm;
int err;
for (;;) {
struct crypto_alg *alg;
alg = crypto_alg_mod_lookup(alg_name, type, mask);
if (IS_ERR(alg)) {
err = PTR_ERR(alg);
goto err;
}
tfm = __crypto_alloc_tfm(alg, type, mask);
if (!IS_ERR(tfm))
return tfm;
crypto_mod_put(alg);
err = PTR_ERR(tfm);
err:
if (err != -EAGAIN)
break;
if (fatal_signal_pending(current)) {
err = -EINTR;
break;
}
}
return ERR_PTR(err);
}
EXPORT_SYMBOL_GPL(crypto_alloc_base);
static void *crypto_alloc_tfmmem(struct crypto_alg *alg,
const struct crypto_type *frontend, int node,
gfp_t gfp)
{
struct crypto_tfm *tfm;
unsigned int tfmsize;
unsigned int total;
char *mem;
tfmsize = frontend->tfmsize;
total = tfmsize + sizeof(*tfm) + frontend->extsize(alg);
mem = kzalloc_node(total, gfp, node);
if (mem == NULL)
return ERR_PTR(-ENOMEM);
tfm = (struct crypto_tfm *)(mem + tfmsize);
tfm->__crt_alg = alg;
tfm->node = node;
refcount_set(&tfm->refcnt, 1);
return mem;
}
void *crypto_create_tfm_node(struct crypto_alg *alg,
const struct crypto_type *frontend,
int node)
{
struct crypto_tfm *tfm;
char *mem;
int err;
mem = crypto_alloc_tfmmem(alg, frontend, node, GFP_KERNEL);
if (IS_ERR(mem))
goto out;
tfm = (struct crypto_tfm *)(mem + frontend->tfmsize);
err = frontend->init_tfm(tfm);
if (err)
goto out_free_tfm;
if (!tfm->exit && alg->cra_init && (err = alg->cra_init(tfm)))
goto cra_init_failed;
goto out;
cra_init_failed:
crypto_exit_ops(tfm);
out_free_tfm:
if (err == -EAGAIN)
crypto_shoot_alg(alg);
kfree(mem);
mem = ERR_PTR(err);
out:
return mem;
}
EXPORT_SYMBOL_GPL(crypto_create_tfm_node);
void *crypto_clone_tfm(const struct crypto_type *frontend,
struct crypto_tfm *otfm)
{
struct crypto_alg *alg = otfm->__crt_alg;
struct crypto_tfm *tfm;
char *mem;
mem = ERR_PTR(-ESTALE);
if (unlikely(!crypto_mod_get(alg)))
goto out;
mem = crypto_alloc_tfmmem(alg, frontend, otfm->node, GFP_ATOMIC);
if (IS_ERR(mem)) {
crypto_mod_put(alg);
goto out;
}
tfm = (struct crypto_tfm *)(mem + frontend->tfmsize);
tfm->crt_flags = otfm->crt_flags;
tfm->exit = otfm->exit;
out:
return mem;
}
EXPORT_SYMBOL_GPL(crypto_clone_tfm);
struct crypto_alg *crypto_find_alg(const char *alg_name,
const struct crypto_type *frontend,
u32 type, u32 mask)
{
if (frontend) {
type &= frontend->maskclear;
mask &= frontend->maskclear;
type |= frontend->type;
mask |= frontend->maskset;
}
return crypto_alg_mod_lookup(alg_name, type, mask);
}
EXPORT_SYMBOL_GPL(crypto_find_alg);
/*
* crypto_alloc_tfm_node - Locate algorithm and allocate transform
* @alg_name: Name of algorithm
* @frontend: Frontend algorithm type
* @type: Type of algorithm
* @mask: Mask for type comparison
* @node: NUMA node in which users desire to put requests, if node is
* NUMA_NO_NODE, it means users have no special requirement.
*
* crypto_alloc_tfm() will first attempt to locate an already loaded
* algorithm. If that fails and the kernel supports dynamically loadable
* modules, it will then attempt to load a module of the same name or
* alias. If that fails it will send a query to any loaded crypto manager
* to construct an algorithm on the fly. A refcount is grabbed on the
* algorithm which is then associated with the new transform.
*
* The returned transform is of a non-determinate type. Most people
* should use one of the more specific allocation functions such as
* crypto_alloc_skcipher().
*
* In case of error the return value is an error pointer.
*/
void *crypto_alloc_tfm_node(const char *alg_name,
const struct crypto_type *frontend, u32 type, u32 mask,
int node)
{
void *tfm;
int err;
for (;;) {
struct crypto_alg *alg;
alg = crypto_find_alg(alg_name, frontend, type, mask);
if (IS_ERR(alg)) {
err = PTR_ERR(alg);
goto err;
}
tfm = crypto_create_tfm_node(alg, frontend, node);
if (!IS_ERR(tfm))
return tfm;
crypto_mod_put(alg);
err = PTR_ERR(tfm);
err:
if (err != -EAGAIN)
break;
if (fatal_signal_pending(current)) {
err = -EINTR;
break;
}
}
return ERR_PTR(err);
}
EXPORT_SYMBOL_GPL(crypto_alloc_tfm_node);
/*
* crypto_destroy_tfm - Free crypto transform
* @mem: Start of tfm slab
* @tfm: Transform to free
*
* This function frees up the transform and any associated resources,
* then drops the refcount on the associated algorithm.
*/
void crypto_destroy_tfm(void *mem, struct crypto_tfm *tfm)
{
struct crypto_alg *alg;
if (IS_ERR_OR_NULL(mem))
return;
if (!refcount_dec_and_test(&tfm->refcnt))
return;
alg = tfm->__crt_alg;
if (!tfm->exit && alg->cra_exit)
alg->cra_exit(tfm);
crypto_exit_ops(tfm);
crypto_mod_put(alg);
mm, treewide: rename kzfree() to kfree_sensitive() As said by Linus: A symmetric naming is only helpful if it implies symmetries in use. Otherwise it's actively misleading. In "kzalloc()", the z is meaningful and an important part of what the caller wants. In "kzfree()", the z is actively detrimental, because maybe in the future we really _might_ want to use that "memfill(0xdeadbeef)" or something. The "zero" part of the interface isn't even _relevant_. The main reason that kzfree() exists is to clear sensitive information that should not be leaked to other future users of the same memory objects. Rename kzfree() to kfree_sensitive() to follow the example of the recently added kvfree_sensitive() and make the intention of the API more explicit. In addition, memzero_explicit() is used to clear the memory to make sure that it won't get optimized away by the compiler. The renaming is done by using the command sequence: git grep -w --name-only kzfree |\ xargs sed -i 's/kzfree/kfree_sensitive/' followed by some editing of the kfree_sensitive() kerneldoc and adding a kzfree backward compatibility macro in slab.h. [akpm@linux-foundation.org: fs/crypto/inline_crypt.c needs linux/slab.h] [akpm@linux-foundation.org: fix fs/crypto/inline_crypt.c some more] Suggested-by: Joe Perches <joe@perches.com> Signed-off-by: Waiman Long <longman@redhat.com> Signed-off-by: Andrew Morton <akpm@linux-foundation.org> Acked-by: David Howells <dhowells@redhat.com> Acked-by: Michal Hocko <mhocko@suse.com> Acked-by: Johannes Weiner <hannes@cmpxchg.org> Cc: Jarkko Sakkinen <jarkko.sakkinen@linux.intel.com> Cc: James Morris <jmorris@namei.org> Cc: "Serge E. Hallyn" <serge@hallyn.com> Cc: Joe Perches <joe@perches.com> Cc: Matthew Wilcox <willy@infradead.org> Cc: David Rientjes <rientjes@google.com> Cc: Dan Carpenter <dan.carpenter@oracle.com> Cc: "Jason A . Donenfeld" <Jason@zx2c4.com> Link: http://lkml.kernel.org/r/20200616154311.12314-3-longman@redhat.com Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2020-08-07 06:18:13 +00:00
kfree_sensitive(mem);
}
EXPORT_SYMBOL_GPL(crypto_destroy_tfm);
int crypto_has_alg(const char *name, u32 type, u32 mask)
{
int ret = 0;
struct crypto_alg *alg = crypto_alg_mod_lookup(name, type, mask);
if (!IS_ERR(alg)) {
crypto_mod_put(alg);
ret = 1;
}
return ret;
}
EXPORT_SYMBOL_GPL(crypto_has_alg);
void crypto_req_done(void *data, int err)
{
struct crypto_wait *wait = data;
if (err == -EINPROGRESS)
return;
wait->err = err;
complete(&wait->completion);
}
EXPORT_SYMBOL_GPL(crypto_req_done);
MODULE_DESCRIPTION("Cryptographic core API");
MODULE_LICENSE("GPL");