2019-07-22 16:26:24 +00:00
|
|
|
// SPDX-License-Identifier: GPL-2.0
|
|
|
|
/*
|
|
|
|
* fs/ext4/verity.c: fs-verity support for ext4
|
|
|
|
*
|
|
|
|
* Copyright 2019 Google LLC
|
|
|
|
*/
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Implementation of fsverity_operations for ext4.
|
|
|
|
*
|
|
|
|
* ext4 stores the verity metadata (Merkle tree and fsverity_descriptor) past
|
|
|
|
* the end of the file, starting at the first 64K boundary beyond i_size. This
|
|
|
|
* approach works because (a) verity files are readonly, and (b) pages fully
|
|
|
|
* beyond i_size aren't visible to userspace but can be read/written internally
|
|
|
|
* by ext4 with only some relatively small changes to ext4. This approach
|
|
|
|
* avoids having to depend on the EA_INODE feature and on rearchitecturing
|
|
|
|
* ext4's xattr support to support paging multi-gigabyte xattrs into memory, and
|
|
|
|
* to support encrypting xattrs. Note that the verity metadata *must* be
|
|
|
|
* encrypted when the file is, since it contains hashes of the plaintext data.
|
|
|
|
*
|
|
|
|
* Using a 64K boundary rather than a 4K one keeps things ready for
|
|
|
|
* architectures with 64K pages, and it doesn't necessarily waste space on-disk
|
|
|
|
* since there can be a hole between i_size and the start of the Merkle tree.
|
|
|
|
*/
|
|
|
|
|
|
|
|
#include <linux/quotaops.h>
|
|
|
|
|
|
|
|
#include "ext4.h"
|
|
|
|
#include "ext4_extents.h"
|
|
|
|
#include "ext4_jbd2.h"
|
|
|
|
|
|
|
|
static inline loff_t ext4_verity_metadata_pos(const struct inode *inode)
|
|
|
|
{
|
|
|
|
return round_up(inode->i_size, 65536);
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Read some verity metadata from the inode. __vfs_read() can't be used because
|
|
|
|
* we need to read beyond i_size.
|
|
|
|
*/
|
|
|
|
static int pagecache_read(struct inode *inode, void *buf, size_t count,
|
|
|
|
loff_t pos)
|
|
|
|
{
|
|
|
|
while (count) {
|
|
|
|
size_t n = min_t(size_t, count,
|
|
|
|
PAGE_SIZE - offset_in_page(pos));
|
|
|
|
struct page *page;
|
|
|
|
void *addr;
|
|
|
|
|
|
|
|
page = read_mapping_page(inode->i_mapping, pos >> PAGE_SHIFT,
|
|
|
|
NULL);
|
|
|
|
if (IS_ERR(page))
|
|
|
|
return PTR_ERR(page);
|
|
|
|
|
|
|
|
addr = kmap_atomic(page);
|
|
|
|
memcpy(buf, addr + offset_in_page(pos), n);
|
|
|
|
kunmap_atomic(addr);
|
|
|
|
|
|
|
|
put_page(page);
|
|
|
|
|
|
|
|
buf += n;
|
|
|
|
pos += n;
|
|
|
|
count -= n;
|
|
|
|
}
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Write some verity metadata to the inode for FS_IOC_ENABLE_VERITY.
|
|
|
|
* kernel_write() can't be used because the file descriptor is readonly.
|
|
|
|
*/
|
|
|
|
static int pagecache_write(struct inode *inode, const void *buf, size_t count,
|
|
|
|
loff_t pos)
|
|
|
|
{
|
|
|
|
if (pos + count > inode->i_sb->s_maxbytes)
|
|
|
|
return -EFBIG;
|
|
|
|
|
|
|
|
while (count) {
|
|
|
|
size_t n = min_t(size_t, count,
|
|
|
|
PAGE_SIZE - offset_in_page(pos));
|
|
|
|
struct page *page;
|
|
|
|
void *fsdata;
|
|
|
|
void *addr;
|
|
|
|
int res;
|
|
|
|
|
|
|
|
res = pagecache_write_begin(NULL, inode->i_mapping, pos, n, 0,
|
|
|
|
&page, &fsdata);
|
|
|
|
if (res)
|
|
|
|
return res;
|
|
|
|
|
|
|
|
addr = kmap_atomic(page);
|
|
|
|
memcpy(addr + offset_in_page(pos), buf, n);
|
|
|
|
kunmap_atomic(addr);
|
|
|
|
|
|
|
|
res = pagecache_write_end(NULL, inode->i_mapping, pos, n, n,
|
|
|
|
page, fsdata);
|
|
|
|
if (res < 0)
|
|
|
|
return res;
|
|
|
|
if (res != n)
|
|
|
|
return -EIO;
|
|
|
|
|
|
|
|
buf += n;
|
|
|
|
pos += n;
|
|
|
|
count -= n;
|
|
|
|
}
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
static int ext4_begin_enable_verity(struct file *filp)
|
|
|
|
{
|
|
|
|
struct inode *inode = file_inode(filp);
|
|
|
|
const int credits = 2; /* superblock and inode for ext4_orphan_add() */
|
|
|
|
handle_t *handle;
|
|
|
|
int err;
|
|
|
|
|
2020-05-28 15:00:02 +00:00
|
|
|
if (IS_DAX(inode) || ext4_test_inode_flag(inode, EXT4_INODE_DAX))
|
2020-05-28 14:59:56 +00:00
|
|
|
return -EINVAL;
|
|
|
|
|
2019-07-22 16:26:24 +00:00
|
|
|
if (ext4_verity_in_progress(inode))
|
|
|
|
return -EBUSY;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Since the file was opened readonly, we have to initialize the jbd
|
|
|
|
* inode and quotas here and not rely on ->open() doing it. This must
|
|
|
|
* be done before evicting the inline data.
|
|
|
|
*/
|
|
|
|
|
|
|
|
err = ext4_inode_attach_jinode(inode);
|
|
|
|
if (err)
|
|
|
|
return err;
|
|
|
|
|
|
|
|
err = dquot_initialize(inode);
|
|
|
|
if (err)
|
|
|
|
return err;
|
|
|
|
|
|
|
|
err = ext4_convert_inline_data(inode);
|
|
|
|
if (err)
|
|
|
|
return err;
|
|
|
|
|
|
|
|
if (!ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS)) {
|
|
|
|
ext4_warning_inode(inode,
|
|
|
|
"verity is only allowed on extent-based files");
|
|
|
|
return -EOPNOTSUPP;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* ext4 uses the last allocated block to find the verity descriptor, so
|
|
|
|
* we must remove any other blocks past EOF which might confuse things.
|
|
|
|
*/
|
|
|
|
err = ext4_truncate(inode);
|
|
|
|
if (err)
|
|
|
|
return err;
|
|
|
|
|
|
|
|
handle = ext4_journal_start(inode, EXT4_HT_INODE, credits);
|
|
|
|
if (IS_ERR(handle))
|
|
|
|
return PTR_ERR(handle);
|
|
|
|
|
|
|
|
err = ext4_orphan_add(handle, inode);
|
|
|
|
if (err == 0)
|
|
|
|
ext4_set_inode_state(inode, EXT4_STATE_VERITY_IN_PROGRESS);
|
|
|
|
|
|
|
|
ext4_journal_stop(handle);
|
|
|
|
return err;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* ext4 stores the verity descriptor beginning on the next filesystem block
|
|
|
|
* boundary after the Merkle tree. Then, the descriptor size is stored in the
|
|
|
|
* last 4 bytes of the last allocated filesystem block --- which is either the
|
|
|
|
* block in which the descriptor ends, or the next block after that if there
|
|
|
|
* weren't at least 4 bytes remaining.
|
|
|
|
*
|
|
|
|
* We can't simply store the descriptor in an xattr because it *must* be
|
|
|
|
* encrypted when ext4 encryption is used, but ext4 encryption doesn't encrypt
|
|
|
|
* xattrs. Also, if the descriptor includes a large signature blob it may be
|
|
|
|
* too large to store in an xattr without the EA_INODE feature.
|
|
|
|
*/
|
|
|
|
static int ext4_write_verity_descriptor(struct inode *inode, const void *desc,
|
|
|
|
size_t desc_size, u64 merkle_tree_size)
|
|
|
|
{
|
|
|
|
const u64 desc_pos = round_up(ext4_verity_metadata_pos(inode) +
|
|
|
|
merkle_tree_size, i_blocksize(inode));
|
|
|
|
const u64 desc_end = desc_pos + desc_size;
|
|
|
|
const __le32 desc_size_disk = cpu_to_le32(desc_size);
|
|
|
|
const u64 desc_size_pos = round_up(desc_end + sizeof(desc_size_disk),
|
|
|
|
i_blocksize(inode)) -
|
|
|
|
sizeof(desc_size_disk);
|
|
|
|
int err;
|
|
|
|
|
|
|
|
err = pagecache_write(inode, desc, desc_size, desc_pos);
|
|
|
|
if (err)
|
|
|
|
return err;
|
|
|
|
|
|
|
|
return pagecache_write(inode, &desc_size_disk, sizeof(desc_size_disk),
|
|
|
|
desc_size_pos);
|
|
|
|
}
|
|
|
|
|
|
|
|
static int ext4_end_enable_verity(struct file *filp, const void *desc,
|
|
|
|
size_t desc_size, u64 merkle_tree_size)
|
|
|
|
{
|
|
|
|
struct inode *inode = file_inode(filp);
|
|
|
|
const int credits = 2; /* superblock and inode for ext4_orphan_del() */
|
|
|
|
handle_t *handle;
|
|
|
|
int err = 0;
|
|
|
|
int err2;
|
|
|
|
|
|
|
|
if (desc != NULL) {
|
|
|
|
/* Succeeded; write the verity descriptor. */
|
|
|
|
err = ext4_write_verity_descriptor(inode, desc, desc_size,
|
|
|
|
merkle_tree_size);
|
|
|
|
|
|
|
|
/* Write all pages before clearing VERITY_IN_PROGRESS. */
|
|
|
|
if (!err)
|
|
|
|
err = filemap_write_and_wait(inode->i_mapping);
|
|
|
|
}
|
|
|
|
|
|
|
|
/* If we failed, truncate anything we wrote past i_size. */
|
|
|
|
if (desc == NULL || err)
|
|
|
|
ext4_truncate(inode);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* We must always clean up by clearing EXT4_STATE_VERITY_IN_PROGRESS and
|
|
|
|
* deleting the inode from the orphan list, even if something failed.
|
|
|
|
* If everything succeeded, we'll also set the verity bit in the same
|
|
|
|
* transaction.
|
|
|
|
*/
|
|
|
|
|
|
|
|
ext4_clear_inode_state(inode, EXT4_STATE_VERITY_IN_PROGRESS);
|
|
|
|
|
|
|
|
handle = ext4_journal_start(inode, EXT4_HT_INODE, credits);
|
|
|
|
if (IS_ERR(handle)) {
|
|
|
|
ext4_orphan_del(NULL, inode);
|
|
|
|
return PTR_ERR(handle);
|
|
|
|
}
|
|
|
|
|
|
|
|
err2 = ext4_orphan_del(handle, inode);
|
|
|
|
if (err2)
|
|
|
|
goto out_stop;
|
|
|
|
|
|
|
|
if (desc != NULL && !err) {
|
|
|
|
struct ext4_iloc iloc;
|
|
|
|
|
|
|
|
err = ext4_reserve_inode_write(handle, inode, &iloc);
|
|
|
|
if (err)
|
|
|
|
goto out_stop;
|
|
|
|
ext4_set_inode_flag(inode, EXT4_INODE_VERITY);
|
2020-05-28 14:59:59 +00:00
|
|
|
ext4_set_inode_flags(inode, false);
|
2019-07-22 16:26:24 +00:00
|
|
|
err = ext4_mark_iloc_dirty(handle, inode, &iloc);
|
|
|
|
}
|
|
|
|
out_stop:
|
|
|
|
ext4_journal_stop(handle);
|
|
|
|
return err ?: err2;
|
|
|
|
}
|
|
|
|
|
|
|
|
static int ext4_get_verity_descriptor_location(struct inode *inode,
|
|
|
|
size_t *desc_size_ret,
|
|
|
|
u64 *desc_pos_ret)
|
|
|
|
{
|
|
|
|
struct ext4_ext_path *path;
|
|
|
|
struct ext4_extent *last_extent;
|
|
|
|
u32 end_lblk;
|
|
|
|
u64 desc_size_pos;
|
|
|
|
__le32 desc_size_disk;
|
|
|
|
u32 desc_size;
|
|
|
|
u64 desc_pos;
|
|
|
|
int err;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Descriptor size is in last 4 bytes of last allocated block.
|
|
|
|
* See ext4_write_verity_descriptor().
|
|
|
|
*/
|
|
|
|
|
|
|
|
if (!ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS)) {
|
|
|
|
EXT4_ERROR_INODE(inode, "verity file doesn't use extents");
|
|
|
|
return -EFSCORRUPTED;
|
|
|
|
}
|
|
|
|
|
|
|
|
path = ext4_find_extent(inode, EXT_MAX_BLOCKS - 1, NULL, 0);
|
|
|
|
if (IS_ERR(path))
|
|
|
|
return PTR_ERR(path);
|
|
|
|
|
|
|
|
last_extent = path[path->p_depth].p_ext;
|
|
|
|
if (!last_extent) {
|
|
|
|
EXT4_ERROR_INODE(inode, "verity file has no extents");
|
|
|
|
ext4_ext_drop_refs(path);
|
|
|
|
kfree(path);
|
|
|
|
return -EFSCORRUPTED;
|
|
|
|
}
|
|
|
|
|
|
|
|
end_lblk = le32_to_cpu(last_extent->ee_block) +
|
|
|
|
ext4_ext_get_actual_len(last_extent);
|
|
|
|
desc_size_pos = (u64)end_lblk << inode->i_blkbits;
|
|
|
|
ext4_ext_drop_refs(path);
|
|
|
|
kfree(path);
|
|
|
|
|
|
|
|
if (desc_size_pos < sizeof(desc_size_disk))
|
|
|
|
goto bad;
|
|
|
|
desc_size_pos -= sizeof(desc_size_disk);
|
|
|
|
|
|
|
|
err = pagecache_read(inode, &desc_size_disk, sizeof(desc_size_disk),
|
|
|
|
desc_size_pos);
|
|
|
|
if (err)
|
|
|
|
return err;
|
|
|
|
desc_size = le32_to_cpu(desc_size_disk);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* The descriptor is stored just before the desc_size_disk, but starting
|
|
|
|
* on a filesystem block boundary.
|
|
|
|
*/
|
|
|
|
|
|
|
|
if (desc_size > INT_MAX || desc_size > desc_size_pos)
|
|
|
|
goto bad;
|
|
|
|
|
|
|
|
desc_pos = round_down(desc_size_pos - desc_size, i_blocksize(inode));
|
|
|
|
if (desc_pos < ext4_verity_metadata_pos(inode))
|
|
|
|
goto bad;
|
|
|
|
|
|
|
|
*desc_size_ret = desc_size;
|
|
|
|
*desc_pos_ret = desc_pos;
|
|
|
|
return 0;
|
|
|
|
|
|
|
|
bad:
|
|
|
|
EXT4_ERROR_INODE(inode, "verity file corrupted; can't find descriptor");
|
|
|
|
return -EFSCORRUPTED;
|
|
|
|
}
|
|
|
|
|
|
|
|
static int ext4_get_verity_descriptor(struct inode *inode, void *buf,
|
|
|
|
size_t buf_size)
|
|
|
|
{
|
|
|
|
size_t desc_size = 0;
|
|
|
|
u64 desc_pos = 0;
|
|
|
|
int err;
|
|
|
|
|
|
|
|
err = ext4_get_verity_descriptor_location(inode, &desc_size, &desc_pos);
|
|
|
|
if (err)
|
|
|
|
return err;
|
|
|
|
|
|
|
|
if (buf_size) {
|
|
|
|
if (desc_size > buf_size)
|
|
|
|
return -ERANGE;
|
|
|
|
err = pagecache_read(inode, buf, desc_size, desc_pos);
|
|
|
|
if (err)
|
|
|
|
return err;
|
|
|
|
}
|
|
|
|
return desc_size;
|
|
|
|
}
|
|
|
|
|
|
|
|
static struct page *ext4_read_merkle_tree_page(struct inode *inode,
|
fs-verity: implement readahead of Merkle tree pages
When fs-verity verifies data pages, currently it reads each Merkle tree
page synchronously using read_mapping_page().
Therefore, when the Merkle tree pages aren't already cached, fs-verity
causes an extra 4 KiB I/O request for every 512 KiB of data (assuming
that the Merkle tree uses SHA-256 and 4 KiB blocks). This results in
more I/O requests and performance loss than is strictly necessary.
Therefore, implement readahead of the Merkle tree pages.
For simplicity, we take advantage of the fact that the kernel already
does readahead of the file's *data*, just like it does for any other
file. Due to this, we don't really need a separate readahead state
(struct file_ra_state) just for the Merkle tree, but rather we just need
to piggy-back on the existing data readahead requests.
We also only really need to bother with the first level of the Merkle
tree, since the usual fan-out factor is 128, so normally over 99% of
Merkle tree I/O requests are for the first level.
Therefore, make fsverity_verify_bio() enable readahead of the first
Merkle tree level, for up to 1/4 the number of pages in the bio, when it
sees that the REQ_RAHEAD flag is set on the bio. The readahead size is
then passed down to ->read_merkle_tree_page() for the filesystem to
(optionally) implement if it sees that the requested page is uncached.
While we're at it, also make build_merkle_tree_level() set the Merkle
tree readahead size, since it's easy to do there.
However, for now don't set the readahead size in fsverity_verify_page(),
since currently it's only used to verify holes on ext4 and f2fs, and it
would need parameters added to know how much to read ahead.
This patch significantly improves fs-verity sequential read performance.
Some quick benchmarks with 'cat'-ing a 250MB file after dropping caches:
On an ARM64 phone (using sha256-ce):
Before: 217 MB/s
After: 263 MB/s
(compare to sha256sum of non-verity file: 357 MB/s)
In an x86_64 VM (using sha256-avx2):
Before: 173 MB/s
After: 215 MB/s
(compare to sha256sum of non-verity file: 223 MB/s)
Link: https://lore.kernel.org/r/20200106205533.137005-1-ebiggers@kernel.org
Reviewed-by: Theodore Ts'o <tytso@mit.edu>
Signed-off-by: Eric Biggers <ebiggers@google.com>
2020-01-06 20:55:33 +00:00
|
|
|
pgoff_t index,
|
|
|
|
unsigned long num_ra_pages)
|
2019-07-22 16:26:24 +00:00
|
|
|
{
|
2020-10-16 03:06:14 +00:00
|
|
|
DEFINE_READAHEAD(ractl, NULL, inode->i_mapping, index);
|
fs-verity: implement readahead of Merkle tree pages
When fs-verity verifies data pages, currently it reads each Merkle tree
page synchronously using read_mapping_page().
Therefore, when the Merkle tree pages aren't already cached, fs-verity
causes an extra 4 KiB I/O request for every 512 KiB of data (assuming
that the Merkle tree uses SHA-256 and 4 KiB blocks). This results in
more I/O requests and performance loss than is strictly necessary.
Therefore, implement readahead of the Merkle tree pages.
For simplicity, we take advantage of the fact that the kernel already
does readahead of the file's *data*, just like it does for any other
file. Due to this, we don't really need a separate readahead state
(struct file_ra_state) just for the Merkle tree, but rather we just need
to piggy-back on the existing data readahead requests.
We also only really need to bother with the first level of the Merkle
tree, since the usual fan-out factor is 128, so normally over 99% of
Merkle tree I/O requests are for the first level.
Therefore, make fsverity_verify_bio() enable readahead of the first
Merkle tree level, for up to 1/4 the number of pages in the bio, when it
sees that the REQ_RAHEAD flag is set on the bio. The readahead size is
then passed down to ->read_merkle_tree_page() for the filesystem to
(optionally) implement if it sees that the requested page is uncached.
While we're at it, also make build_merkle_tree_level() set the Merkle
tree readahead size, since it's easy to do there.
However, for now don't set the readahead size in fsverity_verify_page(),
since currently it's only used to verify holes on ext4 and f2fs, and it
would need parameters added to know how much to read ahead.
This patch significantly improves fs-verity sequential read performance.
Some quick benchmarks with 'cat'-ing a 250MB file after dropping caches:
On an ARM64 phone (using sha256-ce):
Before: 217 MB/s
After: 263 MB/s
(compare to sha256sum of non-verity file: 357 MB/s)
In an x86_64 VM (using sha256-avx2):
Before: 173 MB/s
After: 215 MB/s
(compare to sha256sum of non-verity file: 223 MB/s)
Link: https://lore.kernel.org/r/20200106205533.137005-1-ebiggers@kernel.org
Reviewed-by: Theodore Ts'o <tytso@mit.edu>
Signed-off-by: Eric Biggers <ebiggers@google.com>
2020-01-06 20:55:33 +00:00
|
|
|
struct page *page;
|
|
|
|
|
2019-07-22 16:26:24 +00:00
|
|
|
index += ext4_verity_metadata_pos(inode) >> PAGE_SHIFT;
|
|
|
|
|
fs-verity: implement readahead of Merkle tree pages
When fs-verity verifies data pages, currently it reads each Merkle tree
page synchronously using read_mapping_page().
Therefore, when the Merkle tree pages aren't already cached, fs-verity
causes an extra 4 KiB I/O request for every 512 KiB of data (assuming
that the Merkle tree uses SHA-256 and 4 KiB blocks). This results in
more I/O requests and performance loss than is strictly necessary.
Therefore, implement readahead of the Merkle tree pages.
For simplicity, we take advantage of the fact that the kernel already
does readahead of the file's *data*, just like it does for any other
file. Due to this, we don't really need a separate readahead state
(struct file_ra_state) just for the Merkle tree, but rather we just need
to piggy-back on the existing data readahead requests.
We also only really need to bother with the first level of the Merkle
tree, since the usual fan-out factor is 128, so normally over 99% of
Merkle tree I/O requests are for the first level.
Therefore, make fsverity_verify_bio() enable readahead of the first
Merkle tree level, for up to 1/4 the number of pages in the bio, when it
sees that the REQ_RAHEAD flag is set on the bio. The readahead size is
then passed down to ->read_merkle_tree_page() for the filesystem to
(optionally) implement if it sees that the requested page is uncached.
While we're at it, also make build_merkle_tree_level() set the Merkle
tree readahead size, since it's easy to do there.
However, for now don't set the readahead size in fsverity_verify_page(),
since currently it's only used to verify holes on ext4 and f2fs, and it
would need parameters added to know how much to read ahead.
This patch significantly improves fs-verity sequential read performance.
Some quick benchmarks with 'cat'-ing a 250MB file after dropping caches:
On an ARM64 phone (using sha256-ce):
Before: 217 MB/s
After: 263 MB/s
(compare to sha256sum of non-verity file: 357 MB/s)
In an x86_64 VM (using sha256-avx2):
Before: 173 MB/s
After: 215 MB/s
(compare to sha256sum of non-verity file: 223 MB/s)
Link: https://lore.kernel.org/r/20200106205533.137005-1-ebiggers@kernel.org
Reviewed-by: Theodore Ts'o <tytso@mit.edu>
Signed-off-by: Eric Biggers <ebiggers@google.com>
2020-01-06 20:55:33 +00:00
|
|
|
page = find_get_page_flags(inode->i_mapping, index, FGP_ACCESSED);
|
|
|
|
if (!page || !PageUptodate(page)) {
|
|
|
|
if (page)
|
|
|
|
put_page(page);
|
|
|
|
else if (num_ra_pages > 1)
|
2020-10-16 03:06:14 +00:00
|
|
|
page_cache_ra_unbounded(&ractl, num_ra_pages, 0);
|
fs-verity: implement readahead of Merkle tree pages
When fs-verity verifies data pages, currently it reads each Merkle tree
page synchronously using read_mapping_page().
Therefore, when the Merkle tree pages aren't already cached, fs-verity
causes an extra 4 KiB I/O request for every 512 KiB of data (assuming
that the Merkle tree uses SHA-256 and 4 KiB blocks). This results in
more I/O requests and performance loss than is strictly necessary.
Therefore, implement readahead of the Merkle tree pages.
For simplicity, we take advantage of the fact that the kernel already
does readahead of the file's *data*, just like it does for any other
file. Due to this, we don't really need a separate readahead state
(struct file_ra_state) just for the Merkle tree, but rather we just need
to piggy-back on the existing data readahead requests.
We also only really need to bother with the first level of the Merkle
tree, since the usual fan-out factor is 128, so normally over 99% of
Merkle tree I/O requests are for the first level.
Therefore, make fsverity_verify_bio() enable readahead of the first
Merkle tree level, for up to 1/4 the number of pages in the bio, when it
sees that the REQ_RAHEAD flag is set on the bio. The readahead size is
then passed down to ->read_merkle_tree_page() for the filesystem to
(optionally) implement if it sees that the requested page is uncached.
While we're at it, also make build_merkle_tree_level() set the Merkle
tree readahead size, since it's easy to do there.
However, for now don't set the readahead size in fsverity_verify_page(),
since currently it's only used to verify holes on ext4 and f2fs, and it
would need parameters added to know how much to read ahead.
This patch significantly improves fs-verity sequential read performance.
Some quick benchmarks with 'cat'-ing a 250MB file after dropping caches:
On an ARM64 phone (using sha256-ce):
Before: 217 MB/s
After: 263 MB/s
(compare to sha256sum of non-verity file: 357 MB/s)
In an x86_64 VM (using sha256-avx2):
Before: 173 MB/s
After: 215 MB/s
(compare to sha256sum of non-verity file: 223 MB/s)
Link: https://lore.kernel.org/r/20200106205533.137005-1-ebiggers@kernel.org
Reviewed-by: Theodore Ts'o <tytso@mit.edu>
Signed-off-by: Eric Biggers <ebiggers@google.com>
2020-01-06 20:55:33 +00:00
|
|
|
page = read_mapping_page(inode->i_mapping, index, NULL);
|
|
|
|
}
|
|
|
|
return page;
|
2019-07-22 16:26:24 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
static int ext4_write_merkle_tree_block(struct inode *inode, const void *buf,
|
|
|
|
u64 index, int log_blocksize)
|
|
|
|
{
|
|
|
|
loff_t pos = ext4_verity_metadata_pos(inode) + (index << log_blocksize);
|
|
|
|
|
|
|
|
return pagecache_write(inode, buf, 1 << log_blocksize, pos);
|
|
|
|
}
|
|
|
|
|
|
|
|
const struct fsverity_operations ext4_verityops = {
|
|
|
|
.begin_enable_verity = ext4_begin_enable_verity,
|
|
|
|
.end_enable_verity = ext4_end_enable_verity,
|
|
|
|
.get_verity_descriptor = ext4_get_verity_descriptor,
|
|
|
|
.read_merkle_tree_page = ext4_read_merkle_tree_page,
|
|
|
|
.write_merkle_tree_block = ext4_write_merkle_tree_block,
|
|
|
|
};
|