2018-06-06 02:42:14 +00:00
|
|
|
// SPDX-License-Identifier: GPL-2.0
|
2005-04-16 22:20:36 +00:00
|
|
|
/*
|
2005-11-02 03:58:39 +00:00
|
|
|
* Copyright (c) 2000-2005 Silicon Graphics, Inc.
|
2018-07-12 05:26:06 +00:00
|
|
|
* Copyright (c) 2016-2018 Christoph Hellwig.
|
2005-11-02 03:58:39 +00:00
|
|
|
* All Rights Reserved.
|
2005-04-16 22:20:36 +00:00
|
|
|
*/
|
|
|
|
|
#include "xfs.h"
|
2013-10-22 23:36:05 +00:00
|
|
|
#include "xfs_shared.h"
|
2013-10-22 23:50:10 +00:00
|
|
|
#include "xfs_format.h"
|
|
|
|
|
#include "xfs_log_format.h"
|
|
|
|
|
#include "xfs_trans_resv.h"
|
2005-04-16 22:20:36 +00:00
|
|
|
#include "xfs_mount.h"
|
|
|
|
|
#include "xfs_inode.h"
|
2013-10-22 23:50:10 +00:00
|
|
|
#include "xfs_trans.h"
|
2005-04-16 22:20:36 +00:00
|
|
|
#include "xfs_iomap.h"
|
2009-12-14 23:14:59 +00:00
|
|
|
#include "xfs_trace.h"
|
2010-03-05 02:00:42 +00:00
|
|
|
#include "xfs_bmap.h"
|
2013-08-12 10:49:42 +00:00
|
|
|
#include "xfs_bmap_util.h"
|
2016-10-03 16:11:34 +00:00
|
|
|
#include "xfs_reflink.h"
|
2022-11-29 01:24:35 +00:00
|
|
|
#include "xfs_errortag.h"
|
|
|
|
|
#include "xfs_error.h"
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2016-02-15 06:21:19 +00:00
|
|
|
struct xfs_writepage_ctx {
|
2019-10-17 20:12:15 +00:00
|
|
|
struct iomap_writepage_ctx ctx;
|
xfs: validate writeback mapping using data fork seq counter
The writeback code caches the current extent mapping across multiple
xfs_do_writepage() calls to avoid repeated lookups for sequential
pages backed by the same extent. This is known to be slightly racy
with extent fork changes in certain difficult to reproduce
scenarios. The cached extent is trimmed to within EOF to help avoid
the most common vector for this problem via speculative
preallocation management, but this is a band-aid that does not
address the fundamental problem.
Now that we have an xfs_ifork sequence counter mechanism used to
facilitate COW writeback, we can use the same mechanism to validate
consistency between the data fork and cached writeback mappings. On
its face, this is somewhat of a big hammer approach because any
change to the data fork invalidates any mapping currently cached by
a writeback in progress regardless of whether the data fork change
overlaps with the range under writeback. In practice, however, the
impact of this approach is minimal in most cases.
First, data fork changes (delayed allocations) caused by sustained
sequential buffered writes are amortized across speculative
preallocations. This means that a cached mapping won't be
invalidated by each buffered write of a common file copy workload,
but rather only on less frequent allocation events. Second, the
extent tree is always entirely in-core so an additional lookup of a
usable extent mostly costs a shared ilock cycle and in-memory tree
lookup. This means that a cached mapping reval is relatively cheap
compared to the I/O itself. Third, spurious invalidations don't
impact ioend construction. This means that even if the same extent
is revalidated multiple times across multiple writepage instances,
we still construct and submit the same size ioend (and bio) if the
blocks are physically contiguous.
Update struct xfs_writepage_ctx with a new field to hold the
sequence number of the data fork associated with the currently
cached mapping. Check the wpc seqno against the data fork when the
mapping is validated and reestablish the mapping whenever the fork
has changed since the mapping was cached. This ensures that
writeback always uses a valid extent mapping and thus prevents lost
writebacks and stale delalloc block problems.
Signed-off-by: Brian Foster <bfoster@redhat.com>
Reviewed-by: Allison Henderson <allison.henderson@oracle.com>
Reviewed-by: Christoph Hellwig <hch@lst.de>
Reviewed-by: Darrick J. Wong <darrick.wong@oracle.com>
Signed-off-by: Darrick J. Wong <darrick.wong@oracle.com>
2019-02-01 17:14:23 +00:00
|
|
|
unsigned int data_seq;
|
2018-07-17 23:51:52 +00:00
|
|
|
unsigned int cow_seq;
|
2016-02-15 06:21:19 +00:00
|
|
|
};
|
|
|
|
|
|
2019-10-17 20:12:15 +00:00
|
|
|
static inline struct xfs_writepage_ctx *
|
|
|
|
|
XFS_WPC(struct iomap_writepage_ctx *ctx)
|
|
|
|
|
{
|
|
|
|
|
return container_of(ctx, struct xfs_writepage_ctx, ctx);
|
|
|
|
|
}
|
|
|
|
|
|
2011-08-23 08:28:11 +00:00
|
|
|
/*
|
|
|
|
|
* Fast and loose check if this write could update the on-disk inode size.
|
|
|
|
|
*/
|
2019-10-17 20:12:15 +00:00
|
|
|
static inline bool xfs_ioend_is_append(struct iomap_ioend *ioend)
|
2011-08-23 08:28:11 +00:00
|
|
|
{
|
|
|
|
|
return ioend->io_offset + ioend->io_size >
|
2021-03-29 18:11:40 +00:00
|
|
|
XFS_I(ioend->io_inode)->i_disk_size;
|
2011-08-23 08:28:11 +00:00
|
|
|
}
|
|
|
|
|
|
[XFS] Fix to prevent the notorious 'NULL files' problem after a crash.
The problem that has been addressed is that of synchronising updates of
the file size with writes that extend a file. Without the fix the update
of a file's size, as a result of a write beyond eof, is independent of
when the cached data is flushed to disk. Often the file size update would
be written to the filesystem log before the data is flushed to disk. When
a system crashes between these two events and the filesystem log is
replayed on mount the file's size will be set but since the contents never
made it to disk the file is full of holes. If some of the cached data was
flushed to disk then it may just be a section of the file at the end that
has holes.
There are existing fixes to help alleviate this problem, particularly in
the case where a file has been truncated, that force cached data to be
flushed to disk when the file is closed. If the system crashes while the
file(s) are still open then this flushing will never occur.
The fix that we have implemented is to introduce a second file size,
called the in-memory file size, that represents the current file size as
viewed by the user. The existing file size, called the on-disk file size,
is the one that get's written to the filesystem log and we only update it
when it is safe to do so. When we write to a file beyond eof we only
update the in- memory file size in the write operation. Later when the I/O
operation, that flushes the cached data to disk completes, an I/O
completion routine will update the on-disk file size. The on-disk file
size will be updated to the maximum offset of the I/O or to the value of
the in-memory file size if the I/O includes eof.
SGI-PV: 958522
SGI-Modid: xfs-linux-melb:xfs-kern:28322a
Signed-off-by: Lachlan McIlroy <lachlan@sgi.com>
Signed-off-by: David Chinner <dgc@sgi.com>
Signed-off-by: Tim Shimmin <tes@sgi.com>
2007-05-08 03:49:46 +00:00
|
|
|
/*
|
2011-12-18 20:00:12 +00:00
|
|
|
* Update on-disk file size now that data has been written to disk.
|
[XFS] Fix to prevent the notorious 'NULL files' problem after a crash.
The problem that has been addressed is that of synchronising updates of
the file size with writes that extend a file. Without the fix the update
of a file's size, as a result of a write beyond eof, is independent of
when the cached data is flushed to disk. Often the file size update would
be written to the filesystem log before the data is flushed to disk. When
a system crashes between these two events and the filesystem log is
replayed on mount the file's size will be set but since the contents never
made it to disk the file is full of holes. If some of the cached data was
flushed to disk then it may just be a section of the file at the end that
has holes.
There are existing fixes to help alleviate this problem, particularly in
the case where a file has been truncated, that force cached data to be
flushed to disk when the file is closed. If the system crashes while the
file(s) are still open then this flushing will never occur.
The fix that we have implemented is to introduce a second file size,
called the in-memory file size, that represents the current file size as
viewed by the user. The existing file size, called the on-disk file size,
is the one that get's written to the filesystem log and we only update it
when it is safe to do so. When we write to a file beyond eof we only
update the in- memory file size in the write operation. Later when the I/O
operation, that flushes the cached data to disk completes, an I/O
completion routine will update the on-disk file size. The on-disk file
size will be updated to the maximum offset of the I/O or to the value of
the in-memory file size if the I/O includes eof.
SGI-PV: 958522
SGI-Modid: xfs-linux-melb:xfs-kern:28322a
Signed-off-by: Lachlan McIlroy <lachlan@sgi.com>
Signed-off-by: David Chinner <dgc@sgi.com>
Signed-off-by: Tim Shimmin <tes@sgi.com>
2007-05-08 03:49:46 +00:00
|
|
|
*/
|
2021-04-09 17:27:56 +00:00
|
|
|
int
|
|
|
|
|
xfs_setfilesize(
|
2015-02-01 23:02:09 +00:00
|
|
|
struct xfs_inode *ip,
|
|
|
|
|
xfs_off_t offset,
|
|
|
|
|
size_t size)
|
[XFS] Fix to prevent the notorious 'NULL files' problem after a crash.
The problem that has been addressed is that of synchronising updates of
the file size with writes that extend a file. Without the fix the update
of a file's size, as a result of a write beyond eof, is independent of
when the cached data is flushed to disk. Often the file size update would
be written to the filesystem log before the data is flushed to disk. When
a system crashes between these two events and the filesystem log is
replayed on mount the file's size will be set but since the contents never
made it to disk the file is full of holes. If some of the cached data was
flushed to disk then it may just be a section of the file at the end that
has holes.
There are existing fixes to help alleviate this problem, particularly in
the case where a file has been truncated, that force cached data to be
flushed to disk when the file is closed. If the system crashes while the
file(s) are still open then this flushing will never occur.
The fix that we have implemented is to introduce a second file size,
called the in-memory file size, that represents the current file size as
viewed by the user. The existing file size, called the on-disk file size,
is the one that get's written to the filesystem log and we only update it
when it is safe to do so. When we write to a file beyond eof we only
update the in- memory file size in the write operation. Later when the I/O
operation, that flushes the cached data to disk completes, an I/O
completion routine will update the on-disk file size. The on-disk file
size will be updated to the maximum offset of the I/O or to the value of
the in-memory file size if the I/O includes eof.
SGI-PV: 958522
SGI-Modid: xfs-linux-melb:xfs-kern:28322a
Signed-off-by: Lachlan McIlroy <lachlan@sgi.com>
Signed-off-by: David Chinner <dgc@sgi.com>
Signed-off-by: Tim Shimmin <tes@sgi.com>
2007-05-08 03:49:46 +00:00
|
|
|
{
|
2021-04-09 17:27:56 +00:00
|
|
|
struct xfs_mount *mp = ip->i_mount;
|
|
|
|
|
struct xfs_trans *tp;
|
[XFS] Fix to prevent the notorious 'NULL files' problem after a crash.
The problem that has been addressed is that of synchronising updates of
the file size with writes that extend a file. Without the fix the update
of a file's size, as a result of a write beyond eof, is independent of
when the cached data is flushed to disk. Often the file size update would
be written to the filesystem log before the data is flushed to disk. When
a system crashes between these two events and the filesystem log is
replayed on mount the file's size will be set but since the contents never
made it to disk the file is full of holes. If some of the cached data was
flushed to disk then it may just be a section of the file at the end that
has holes.
There are existing fixes to help alleviate this problem, particularly in
the case where a file has been truncated, that force cached data to be
flushed to disk when the file is closed. If the system crashes while the
file(s) are still open then this flushing will never occur.
The fix that we have implemented is to introduce a second file size,
called the in-memory file size, that represents the current file size as
viewed by the user. The existing file size, called the on-disk file size,
is the one that get's written to the filesystem log and we only update it
when it is safe to do so. When we write to a file beyond eof we only
update the in- memory file size in the write operation. Later when the I/O
operation, that flushes the cached data to disk completes, an I/O
completion routine will update the on-disk file size. The on-disk file
size will be updated to the maximum offset of the I/O or to the value of
the in-memory file size if the I/O includes eof.
SGI-PV: 958522
SGI-Modid: xfs-linux-melb:xfs-kern:28322a
Signed-off-by: Lachlan McIlroy <lachlan@sgi.com>
Signed-off-by: David Chinner <dgc@sgi.com>
Signed-off-by: Tim Shimmin <tes@sgi.com>
2007-05-08 03:49:46 +00:00
|
|
|
xfs_fsize_t isize;
|
2021-04-09 17:27:56 +00:00
|
|
|
int error;
|
|
|
|
|
|
|
|
|
|
error = xfs_trans_alloc(mp, &M_RES(mp)->tr_fsyncts, 0, 0, 0, &tp);
|
|
|
|
|
if (error)
|
|
|
|
|
return error;
|
[XFS] Fix to prevent the notorious 'NULL files' problem after a crash.
The problem that has been addressed is that of synchronising updates of
the file size with writes that extend a file. Without the fix the update
of a file's size, as a result of a write beyond eof, is independent of
when the cached data is flushed to disk. Often the file size update would
be written to the filesystem log before the data is flushed to disk. When
a system crashes between these two events and the filesystem log is
replayed on mount the file's size will be set but since the contents never
made it to disk the file is full of holes. If some of the cached data was
flushed to disk then it may just be a section of the file at the end that
has holes.
There are existing fixes to help alleviate this problem, particularly in
the case where a file has been truncated, that force cached data to be
flushed to disk when the file is closed. If the system crashes while the
file(s) are still open then this flushing will never occur.
The fix that we have implemented is to introduce a second file size,
called the in-memory file size, that represents the current file size as
viewed by the user. The existing file size, called the on-disk file size,
is the one that get's written to the filesystem log and we only update it
when it is safe to do so. When we write to a file beyond eof we only
update the in- memory file size in the write operation. Later when the I/O
operation, that flushes the cached data to disk completes, an I/O
completion routine will update the on-disk file size. The on-disk file
size will be updated to the maximum offset of the I/O or to the value of
the in-memory file size if the I/O includes eof.
SGI-PV: 958522
SGI-Modid: xfs-linux-melb:xfs-kern:28322a
Signed-off-by: Lachlan McIlroy <lachlan@sgi.com>
Signed-off-by: David Chinner <dgc@sgi.com>
Signed-off-by: Tim Shimmin <tes@sgi.com>
2007-05-08 03:49:46 +00:00
|
|
|
|
2012-02-29 09:53:48 +00:00
|
|
|
xfs_ilock(ip, XFS_ILOCK_EXCL);
|
2015-02-01 23:02:09 +00:00
|
|
|
isize = xfs_new_eof(ip, offset + size);
|
2012-03-13 08:41:05 +00:00
|
|
|
if (!isize) {
|
|
|
|
|
xfs_iunlock(ip, XFS_ILOCK_EXCL);
|
2015-06-04 03:47:56 +00:00
|
|
|
xfs_trans_cancel(tp);
|
2012-03-13 08:41:05 +00:00
|
|
|
return 0;
|
[XFS] Fix to prevent the notorious 'NULL files' problem after a crash.
The problem that has been addressed is that of synchronising updates of
the file size with writes that extend a file. Without the fix the update
of a file's size, as a result of a write beyond eof, is independent of
when the cached data is flushed to disk. Often the file size update would
be written to the filesystem log before the data is flushed to disk. When
a system crashes between these two events and the filesystem log is
replayed on mount the file's size will be set but since the contents never
made it to disk the file is full of holes. If some of the cached data was
flushed to disk then it may just be a section of the file at the end that
has holes.
There are existing fixes to help alleviate this problem, particularly in
the case where a file has been truncated, that force cached data to be
flushed to disk when the file is closed. If the system crashes while the
file(s) are still open then this flushing will never occur.
The fix that we have implemented is to introduce a second file size,
called the in-memory file size, that represents the current file size as
viewed by the user. The existing file size, called the on-disk file size,
is the one that get's written to the filesystem log and we only update it
when it is safe to do so. When we write to a file beyond eof we only
update the in- memory file size in the write operation. Later when the I/O
operation, that flushes the cached data to disk completes, an I/O
completion routine will update the on-disk file size. The on-disk file
size will be updated to the maximum offset of the I/O or to the value of
the in-memory file size if the I/O includes eof.
SGI-PV: 958522
SGI-Modid: xfs-linux-melb:xfs-kern:28322a
Signed-off-by: Lachlan McIlroy <lachlan@sgi.com>
Signed-off-by: David Chinner <dgc@sgi.com>
Signed-off-by: Tim Shimmin <tes@sgi.com>
2007-05-08 03:49:46 +00:00
|
|
|
}
|
|
|
|
|
|
2015-02-01 23:02:09 +00:00
|
|
|
trace_xfs_setfilesize(ip, offset, size);
|
2012-03-13 08:41:05 +00:00
|
|
|
|
2021-03-29 18:11:40 +00:00
|
|
|
ip->i_disk_size = isize;
|
2012-03-13 08:41:05 +00:00
|
|
|
xfs_trans_ijoin(tp, ip, XFS_ILOCK_EXCL);
|
|
|
|
|
xfs_trans_log_inode(tp, ip, XFS_ILOG_CORE);
|
|
|
|
|
|
2015-06-04 03:48:08 +00:00
|
|
|
return xfs_trans_commit(tp);
|
2010-02-17 05:36:29 +00:00
|
|
|
}
|
|
|
|
|
|
2005-09-02 06:58:49 +00:00
|
|
|
/*
|
2009-10-30 09:11:47 +00:00
|
|
|
* IO write completion.
|
2006-01-11 04:40:13 +00:00
|
|
|
*/
|
|
|
|
|
STATIC void
|
xfs: implement per-inode writeback completion queues
When scheduling writeback of dirty file data in the page cache, XFS uses
IO completion workqueue items to ensure that filesystem metadata only
updates after the write completes successfully. This is essential for
converting unwritten extents to real extents at the right time and
performing COW remappings.
Unfortunately, XFS queues each IO completion work item to an unbounded
workqueue, which means that the kernel can spawn dozens of threads to
try to handle the items quickly. These threads need to take the ILOCK
to update file metadata, which results in heavy ILOCK contention if a
large number of the work items target a single file, which is
inefficient.
Worse yet, the writeback completion threads get stuck waiting for the
ILOCK while holding transaction reservations, which can use up all
available log reservation space. When that happens, metadata updates to
other parts of the filesystem grind to a halt, even if the filesystem
could otherwise have handled it.
Even worse, if one of the things grinding to a halt happens to be a
thread in the middle of a defer-ops finish holding the same ILOCK and
trying to obtain more log reservation having exhausted the permanent
reservation, we now have an ABBA deadlock - writeback completion has a
transaction reserved and wants the ILOCK, and someone else has the ILOCK
and wants a transaction reservation.
Therefore, we create a per-inode writeback io completion queue + work
item. When writeback finishes, it can add the ioend to the per-inode
queue and let the single worker item process that queue. This
dramatically cuts down on the number of kworkers and ILOCK contention in
the system, and seems to have eliminated an occasional deadlock I was
seeing while running generic/476.
Testing with a program that simulates a heavy random-write workload to a
single file demonstrates that the number of kworkers drops from
approximately 120 threads per file to 1, without dramatically changing
write bandwidth or pagecache access latency.
Note that we leave the xfs-conv workqueue's max_active alone because we
still want to be able to run ioend processing for as many inodes as the
system can handle.
Signed-off-by: Darrick J. Wong <darrick.wong@oracle.com>
Reviewed-by: Brian Foster <bfoster@redhat.com>
2019-04-15 20:13:20 +00:00
|
|
|
xfs_end_ioend(
|
2019-10-17 20:12:15 +00:00
|
|
|
struct iomap_ioend *ioend)
|
2005-09-02 06:58:49 +00:00
|
|
|
{
|
2016-04-05 22:34:30 +00:00
|
|
|
struct xfs_inode *ip = XFS_I(ioend->io_inode);
|
xfs: punch out data fork delalloc blocks on COW writeback failure
If writeback I/O to a COW extent fails, the COW fork blocks are
punched out and the data fork blocks left alone. It is possible for
COW fork blocks to overlap non-shared data fork blocks (due to
cowextsz hint prealloc), however, and writeback unconditionally maps
to the COW fork whenever blocks exist at the corresponding offset of
the page undergoing writeback. This means it's quite possible for a
COW fork extent to overlap delalloc data fork blocks, writeback to
convert and map to the COW fork blocks, writeback to fail, and
finally for ioend completion to cancel the COW fork blocks and leave
stale data fork delalloc blocks around in the inode. The blocks are
effectively stale because writeback failure also discards dirty page
state.
If this occurs, it is likely to trigger assert failures, free space
accounting corruption and failures in unrelated file operations. For
example, a subsequent reflink attempt of the affected file to a new
target file will trip over the stale delalloc in the source file and
fail. Several of these issues are occasionally reproduced by
generic/648, but are reproducible on demand with the right sequence
of operations and timely I/O error injection.
To fix this problem, update the ioend failure path to also punch out
underlying data fork delalloc blocks on I/O error. This is analogous
to the writeback submission failure path in xfs_discard_page() where
we might fail to map data fork delalloc blocks and consistent with
the successful COW writeback completion path, which is responsible
for unmapping from the data fork and remapping in COW fork blocks.
Fixes: 787eb485509f ("xfs: fix and streamline error handling in xfs_end_io")
Signed-off-by: Brian Foster <bfoster@redhat.com>
Reviewed-by: Darrick J. Wong <djwong@kernel.org>
Signed-off-by: Darrick J. Wong <djwong@kernel.org>
2021-10-21 21:11:55 +00:00
|
|
|
struct xfs_mount *mp = ip->i_mount;
|
2017-03-02 23:02:51 +00:00
|
|
|
xfs_off_t offset = ioend->io_offset;
|
|
|
|
|
size_t size = ioend->io_size;
|
2019-06-29 02:31:38 +00:00
|
|
|
unsigned int nofs_flag;
|
2017-06-03 07:38:06 +00:00
|
|
|
int error;
|
[XFS] Fix to prevent the notorious 'NULL files' problem after a crash.
The problem that has been addressed is that of synchronising updates of
the file size with writes that extend a file. Without the fix the update
of a file's size, as a result of a write beyond eof, is independent of
when the cached data is flushed to disk. Often the file size update would
be written to the filesystem log before the data is flushed to disk. When
a system crashes between these two events and the filesystem log is
replayed on mount the file's size will be set but since the contents never
made it to disk the file is full of holes. If some of the cached data was
flushed to disk then it may just be a section of the file at the end that
has holes.
There are existing fixes to help alleviate this problem, particularly in
the case where a file has been truncated, that force cached data to be
flushed to disk when the file is closed. If the system crashes while the
file(s) are still open then this flushing will never occur.
The fix that we have implemented is to introduce a second file size,
called the in-memory file size, that represents the current file size as
viewed by the user. The existing file size, called the on-disk file size,
is the one that get's written to the filesystem log and we only update it
when it is safe to do so. When we write to a file beyond eof we only
update the in- memory file size in the write operation. Later when the I/O
operation, that flushes the cached data to disk completes, an I/O
completion routine will update the on-disk file size. The on-disk file
size will be updated to the maximum offset of the I/O or to the value of
the in-memory file size if the I/O includes eof.
SGI-PV: 958522
SGI-Modid: xfs-linux-melb:xfs-kern:28322a
Signed-off-by: Lachlan McIlroy <lachlan@sgi.com>
Signed-off-by: David Chinner <dgc@sgi.com>
Signed-off-by: Tim Shimmin <tes@sgi.com>
2007-05-08 03:49:46 +00:00
|
|
|
|
2019-06-29 02:31:38 +00:00
|
|
|
/*
|
|
|
|
|
* We can allocate memory here while doing writeback on behalf of
|
|
|
|
|
* memory reclaim. To avoid memory allocation deadlocks set the
|
|
|
|
|
* task-wide nofs context for the following operations.
|
|
|
|
|
*/
|
|
|
|
|
nofs_flag = memalloc_nofs_save();
|
|
|
|
|
|
2016-02-08 04:00:02 +00:00
|
|
|
/*
|
2021-03-23 23:59:30 +00:00
|
|
|
* Just clean up the in-memory structures if the fs has been shut down.
|
2016-02-08 04:00:02 +00:00
|
|
|
*/
|
xfs: punch out data fork delalloc blocks on COW writeback failure
If writeback I/O to a COW extent fails, the COW fork blocks are
punched out and the data fork blocks left alone. It is possible for
COW fork blocks to overlap non-shared data fork blocks (due to
cowextsz hint prealloc), however, and writeback unconditionally maps
to the COW fork whenever blocks exist at the corresponding offset of
the page undergoing writeback. This means it's quite possible for a
COW fork extent to overlap delalloc data fork blocks, writeback to
convert and map to the COW fork blocks, writeback to fail, and
finally for ioend completion to cancel the COW fork blocks and leave
stale data fork delalloc blocks around in the inode. The blocks are
effectively stale because writeback failure also discards dirty page
state.
If this occurs, it is likely to trigger assert failures, free space
accounting corruption and failures in unrelated file operations. For
example, a subsequent reflink attempt of the affected file to a new
target file will trip over the stale delalloc in the source file and
fail. Several of these issues are occasionally reproduced by
generic/648, but are reproducible on demand with the right sequence
of operations and timely I/O error injection.
To fix this problem, update the ioend failure path to also punch out
underlying data fork delalloc blocks on I/O error. This is analogous
to the writeback submission failure path in xfs_discard_page() where
we might fail to map data fork delalloc blocks and consistent with
the successful COW writeback completion path, which is responsible
for unmapping from the data fork and remapping in COW fork blocks.
Fixes: 787eb485509f ("xfs: fix and streamline error handling in xfs_end_io")
Signed-off-by: Brian Foster <bfoster@redhat.com>
Reviewed-by: Darrick J. Wong <djwong@kernel.org>
Signed-off-by: Darrick J. Wong <djwong@kernel.org>
2021-10-21 21:11:55 +00:00
|
|
|
if (xfs_is_shutdown(mp)) {
|
2016-04-05 22:34:30 +00:00
|
|
|
error = -EIO;
|
2017-03-02 23:02:51 +00:00
|
|
|
goto done;
|
|
|
|
|
}
|
2011-08-24 05:59:25 +00:00
|
|
|
|
2016-10-03 16:11:35 +00:00
|
|
|
/*
|
xfs: punch out data fork delalloc blocks on COW writeback failure
If writeback I/O to a COW extent fails, the COW fork blocks are
punched out and the data fork blocks left alone. It is possible for
COW fork blocks to overlap non-shared data fork blocks (due to
cowextsz hint prealloc), however, and writeback unconditionally maps
to the COW fork whenever blocks exist at the corresponding offset of
the page undergoing writeback. This means it's quite possible for a
COW fork extent to overlap delalloc data fork blocks, writeback to
convert and map to the COW fork blocks, writeback to fail, and
finally for ioend completion to cancel the COW fork blocks and leave
stale data fork delalloc blocks around in the inode. The blocks are
effectively stale because writeback failure also discards dirty page
state.
If this occurs, it is likely to trigger assert failures, free space
accounting corruption and failures in unrelated file operations. For
example, a subsequent reflink attempt of the affected file to a new
target file will trip over the stale delalloc in the source file and
fail. Several of these issues are occasionally reproduced by
generic/648, but are reproducible on demand with the right sequence
of operations and timely I/O error injection.
To fix this problem, update the ioend failure path to also punch out
underlying data fork delalloc blocks on I/O error. This is analogous
to the writeback submission failure path in xfs_discard_page() where
we might fail to map data fork delalloc blocks and consistent with
the successful COW writeback completion path, which is responsible
for unmapping from the data fork and remapping in COW fork blocks.
Fixes: 787eb485509f ("xfs: fix and streamline error handling in xfs_end_io")
Signed-off-by: Brian Foster <bfoster@redhat.com>
Reviewed-by: Darrick J. Wong <djwong@kernel.org>
Signed-off-by: Darrick J. Wong <djwong@kernel.org>
2021-10-21 21:11:55 +00:00
|
|
|
* Clean up all COW blocks and underlying data fork delalloc blocks on
|
|
|
|
|
* I/O error. The delalloc punch is required because this ioend was
|
|
|
|
|
* mapped to blocks in the COW fork and the associated pages are no
|
|
|
|
|
* longer dirty. If we don't remove delalloc blocks here, they become
|
|
|
|
|
* stale and can corrupt free space accounting on unmount.
|
2016-10-03 16:11:35 +00:00
|
|
|
*/
|
2023-12-07 07:27:05 +00:00
|
|
|
error = blk_status_to_errno(ioend->io_bio.bi_status);
|
2017-03-02 23:02:51 +00:00
|
|
|
if (unlikely(error)) {
|
xfs: punch out data fork delalloc blocks on COW writeback failure
If writeback I/O to a COW extent fails, the COW fork blocks are
punched out and the data fork blocks left alone. It is possible for
COW fork blocks to overlap non-shared data fork blocks (due to
cowextsz hint prealloc), however, and writeback unconditionally maps
to the COW fork whenever blocks exist at the corresponding offset of
the page undergoing writeback. This means it's quite possible for a
COW fork extent to overlap delalloc data fork blocks, writeback to
convert and map to the COW fork blocks, writeback to fail, and
finally for ioend completion to cancel the COW fork blocks and leave
stale data fork delalloc blocks around in the inode. The blocks are
effectively stale because writeback failure also discards dirty page
state.
If this occurs, it is likely to trigger assert failures, free space
accounting corruption and failures in unrelated file operations. For
example, a subsequent reflink attempt of the affected file to a new
target file will trip over the stale delalloc in the source file and
fail. Several of these issues are occasionally reproduced by
generic/648, but are reproducible on demand with the right sequence
of operations and timely I/O error injection.
To fix this problem, update the ioend failure path to also punch out
underlying data fork delalloc blocks on I/O error. This is analogous
to the writeback submission failure path in xfs_discard_page() where
we might fail to map data fork delalloc blocks and consistent with
the successful COW writeback completion path, which is responsible
for unmapping from the data fork and remapping in COW fork blocks.
Fixes: 787eb485509f ("xfs: fix and streamline error handling in xfs_end_io")
Signed-off-by: Brian Foster <bfoster@redhat.com>
Reviewed-by: Darrick J. Wong <djwong@kernel.org>
Signed-off-by: Darrick J. Wong <djwong@kernel.org>
2021-10-21 21:11:55 +00:00
|
|
|
if (ioend->io_flags & IOMAP_F_SHARED) {
|
2017-03-02 23:02:51 +00:00
|
|
|
xfs_reflink_cancel_cow_range(ip, offset, size, true);
|
2022-11-28 22:09:17 +00:00
|
|
|
xfs_bmap_punch_delalloc_range(ip, offset,
|
|
|
|
|
offset + size);
|
xfs: punch out data fork delalloc blocks on COW writeback failure
If writeback I/O to a COW extent fails, the COW fork blocks are
punched out and the data fork blocks left alone. It is possible for
COW fork blocks to overlap non-shared data fork blocks (due to
cowextsz hint prealloc), however, and writeback unconditionally maps
to the COW fork whenever blocks exist at the corresponding offset of
the page undergoing writeback. This means it's quite possible for a
COW fork extent to overlap delalloc data fork blocks, writeback to
convert and map to the COW fork blocks, writeback to fail, and
finally for ioend completion to cancel the COW fork blocks and leave
stale data fork delalloc blocks around in the inode. The blocks are
effectively stale because writeback failure also discards dirty page
state.
If this occurs, it is likely to trigger assert failures, free space
accounting corruption and failures in unrelated file operations. For
example, a subsequent reflink attempt of the affected file to a new
target file will trip over the stale delalloc in the source file and
fail. Several of these issues are occasionally reproduced by
generic/648, but are reproducible on demand with the right sequence
of operations and timely I/O error injection.
To fix this problem, update the ioend failure path to also punch out
underlying data fork delalloc blocks on I/O error. This is analogous
to the writeback submission failure path in xfs_discard_page() where
we might fail to map data fork delalloc blocks and consistent with
the successful COW writeback completion path, which is responsible
for unmapping from the data fork and remapping in COW fork blocks.
Fixes: 787eb485509f ("xfs: fix and streamline error handling in xfs_end_io")
Signed-off-by: Brian Foster <bfoster@redhat.com>
Reviewed-by: Darrick J. Wong <djwong@kernel.org>
Signed-off-by: Darrick J. Wong <djwong@kernel.org>
2021-10-21 21:11:55 +00:00
|
|
|
}
|
2017-03-02 23:02:51 +00:00
|
|
|
goto done;
|
2016-10-03 16:11:35 +00:00
|
|
|
}
|
|
|
|
|
|
2009-10-30 09:11:47 +00:00
|
|
|
/*
|
2019-02-15 16:02:46 +00:00
|
|
|
* Success: commit the COW or unwritten blocks if needed.
|
2009-10-30 09:11:47 +00:00
|
|
|
*/
|
2019-10-17 20:12:10 +00:00
|
|
|
if (ioend->io_flags & IOMAP_F_SHARED)
|
2017-03-02 23:02:51 +00:00
|
|
|
error = xfs_reflink_end_cow(ip, offset, size);
|
2019-10-17 20:12:06 +00:00
|
|
|
else if (ioend->io_type == IOMAP_UNWRITTEN)
|
2017-09-21 18:26:18 +00:00
|
|
|
error = xfs_iomap_write_unwritten(ip, offset, size, false);
|
[XFS] Fix to prevent the notorious 'NULL files' problem after a crash.
The problem that has been addressed is that of synchronising updates of
the file size with writes that extend a file. Without the fix the update
of a file's size, as a result of a write beyond eof, is independent of
when the cached data is flushed to disk. Often the file size update would
be written to the filesystem log before the data is flushed to disk. When
a system crashes between these two events and the filesystem log is
replayed on mount the file's size will be set but since the contents never
made it to disk the file is full of holes. If some of the cached data was
flushed to disk then it may just be a section of the file at the end that
has holes.
There are existing fixes to help alleviate this problem, particularly in
the case where a file has been truncated, that force cached data to be
flushed to disk when the file is closed. If the system crashes while the
file(s) are still open then this flushing will never occur.
The fix that we have implemented is to introduce a second file size,
called the in-memory file size, that represents the current file size as
viewed by the user. The existing file size, called the on-disk file size,
is the one that get's written to the filesystem log and we only update it
when it is safe to do so. When we write to a file beyond eof we only
update the in- memory file size in the write operation. Later when the I/O
operation, that flushes the cached data to disk completes, an I/O
completion routine will update the on-disk file size. The on-disk file
size will be updated to the maximum offset of the I/O or to the value of
the in-memory file size if the I/O includes eof.
SGI-PV: 958522
SGI-Modid: xfs-linux-melb:xfs-kern:28322a
Signed-off-by: Lachlan McIlroy <lachlan@sgi.com>
Signed-off-by: David Chinner <dgc@sgi.com>
Signed-off-by: Tim Shimmin <tes@sgi.com>
2007-05-08 03:49:46 +00:00
|
|
|
|
2021-04-09 17:27:43 +00:00
|
|
|
if (!error && xfs_ioend_is_append(ioend))
|
|
|
|
|
error = xfs_setfilesize(ip, ioend->io_offset, ioend->io_size);
|
2011-08-24 05:59:25 +00:00
|
|
|
done:
|
2019-10-17 20:12:15 +00:00
|
|
|
iomap_finish_ioends(ioend, error);
|
2019-06-29 02:31:38 +00:00
|
|
|
memalloc_nofs_restore(nofs_flag);
|
2019-04-15 20:13:21 +00:00
|
|
|
}
|
|
|
|
|
|
xfs, iomap: limit individual ioend chain lengths in writeback
Trond Myklebust reported soft lockups in XFS IO completion such as
this:
watchdog: BUG: soft lockup - CPU#12 stuck for 23s! [kworker/12:1:3106]
CPU: 12 PID: 3106 Comm: kworker/12:1 Not tainted 4.18.0-305.10.2.el8_4.x86_64 #1
Workqueue: xfs-conv/md127 xfs_end_io [xfs]
RIP: 0010:_raw_spin_unlock_irqrestore+0x11/0x20
Call Trace:
wake_up_page_bit+0x8a/0x110
iomap_finish_ioend+0xd7/0x1c0
iomap_finish_ioends+0x7f/0xb0
xfs_end_ioend+0x6b/0x100 [xfs]
xfs_end_io+0xb9/0xe0 [xfs]
process_one_work+0x1a7/0x360
worker_thread+0x1fa/0x390
kthread+0x116/0x130
ret_from_fork+0x35/0x40
Ioends are processed as an atomic completion unit when all the
chained bios in the ioend have completed their IO. Logically
contiguous ioends can also be merged and completed as a single,
larger unit. Both of these things can be problematic as both the
bio chains per ioend and the size of the merged ioends processed as
a single completion are both unbound.
If we have a large sequential dirty region in the page cache,
write_cache_pages() will keep feeding us sequential pages and we
will keep mapping them into ioends and bios until we get a dirty
page at a non-sequential file offset. These large sequential runs
can will result in bio and ioend chaining to optimise the io
patterns. The pages iunder writeback are pinned within these chains
until the submission chaining is broken, allowing the entire chain
to be completed. This can result in huge chains being processed
in IO completion context.
We get deep bio chaining if we have large contiguous physical
extents. We will keep adding pages to the current bio until it is
full, then we'll chain a new bio to keep adding pages for writeback.
Hence we can build bio chains that map millions of pages and tens of
gigabytes of RAM if the page cache contains big enough contiguous
dirty file regions. This long bio chain pins those pages until the
final bio in the chain completes and the ioend can iterate all the
chained bios and complete them.
OTOH, if we have a physically fragmented file, we end up submitting
one ioend per physical fragment that each have a small bio or bio
chain attached to them. We do not chain these at IO submission time,
but instead we chain them at completion time based on file
offset via iomap_ioend_try_merge(). Hence we can end up with unbound
ioend chains being built via completion merging.
XFS can then do COW remapping or unwritten extent conversion on that
merged chain, which involves walking an extent fragment at a time
and running a transaction to modify the physical extent information.
IOWs, we merge all the discontiguous ioends together into a
contiguous file range, only to then process them individually as
discontiguous extents.
This extent manipulation is computationally expensive and can run in
a tight loop, so merging logically contiguous but physically
discontigous ioends gains us nothing except for hiding the fact the
fact we broke the ioends up into individual physical extents at
submission and then need to loop over those individual physical
extents at completion.
Hence we need to have mechanisms to limit ioend sizes and
to break up completion processing of large merged ioend chains:
1. bio chains per ioend need to be bound in length. Pure overwrites
go straight to iomap_finish_ioend() in softirq context with the
exact bio chain attached to the ioend by submission. Hence the only
way to prevent long holdoffs here is to bound ioend submission
sizes because we can't reschedule in softirq context.
2. iomap_finish_ioends() has to handle unbound merged ioend chains
correctly. This relies on any one call to iomap_finish_ioend() being
bound in runtime so that cond_resched() can be issued regularly as
the long ioend chain is processed. i.e. this relies on mechanism #1
to limit individual ioend sizes to work correctly.
3. filesystems have to loop over the merged ioends to process
physical extent manipulations. This means they can loop internally,
and so we break merging at physical extent boundaries so the
filesystem can easily insert reschedule points between individual
extent manipulations.
Signed-off-by: Dave Chinner <dchinner@redhat.com>
Reported-and-tested-by: Trond Myklebust <trondmy@hammerspace.com>
Reviewed-by: Darrick J. Wong <djwong@kernel.org>
Signed-off-by: Darrick J. Wong <djwong@kernel.org>
2022-01-26 17:19:20 +00:00
|
|
|
/*
|
|
|
|
|
* Finish all pending IO completions that require transactional modifications.
|
|
|
|
|
*
|
|
|
|
|
* We try to merge physical and logically contiguous ioends before completion to
|
|
|
|
|
* minimise the number of transactions we need to perform during IO completion.
|
|
|
|
|
* Both unwritten extent conversion and COW remapping need to iterate and modify
|
|
|
|
|
* one physical extent at a time, so we gain nothing by merging physically
|
|
|
|
|
* discontiguous extents here.
|
|
|
|
|
*
|
|
|
|
|
* The ioend chain length that we can be processing here is largely unbound in
|
|
|
|
|
* length and we may have to perform significant amounts of work on each ioend
|
|
|
|
|
* to complete it. Hence we have to be careful about holding the CPU for too
|
|
|
|
|
* long in this loop.
|
|
|
|
|
*/
|
xfs: implement per-inode writeback completion queues
When scheduling writeback of dirty file data in the page cache, XFS uses
IO completion workqueue items to ensure that filesystem metadata only
updates after the write completes successfully. This is essential for
converting unwritten extents to real extents at the right time and
performing COW remappings.
Unfortunately, XFS queues each IO completion work item to an unbounded
workqueue, which means that the kernel can spawn dozens of threads to
try to handle the items quickly. These threads need to take the ILOCK
to update file metadata, which results in heavy ILOCK contention if a
large number of the work items target a single file, which is
inefficient.
Worse yet, the writeback completion threads get stuck waiting for the
ILOCK while holding transaction reservations, which can use up all
available log reservation space. When that happens, metadata updates to
other parts of the filesystem grind to a halt, even if the filesystem
could otherwise have handled it.
Even worse, if one of the things grinding to a halt happens to be a
thread in the middle of a defer-ops finish holding the same ILOCK and
trying to obtain more log reservation having exhausted the permanent
reservation, we now have an ABBA deadlock - writeback completion has a
transaction reserved and wants the ILOCK, and someone else has the ILOCK
and wants a transaction reservation.
Therefore, we create a per-inode writeback io completion queue + work
item. When writeback finishes, it can add the ioend to the per-inode
queue and let the single worker item process that queue. This
dramatically cuts down on the number of kworkers and ILOCK contention in
the system, and seems to have eliminated an occasional deadlock I was
seeing while running generic/476.
Testing with a program that simulates a heavy random-write workload to a
single file demonstrates that the number of kworkers drops from
approximately 120 threads per file to 1, without dramatically changing
write bandwidth or pagecache access latency.
Note that we leave the xfs-conv workqueue's max_active alone because we
still want to be able to run ioend processing for as many inodes as the
system can handle.
Signed-off-by: Darrick J. Wong <darrick.wong@oracle.com>
Reviewed-by: Brian Foster <bfoster@redhat.com>
2019-04-15 20:13:20 +00:00
|
|
|
void
|
|
|
|
|
xfs_end_io(
|
|
|
|
|
struct work_struct *work)
|
|
|
|
|
{
|
2019-10-17 20:12:07 +00:00
|
|
|
struct xfs_inode *ip =
|
|
|
|
|
container_of(work, struct xfs_inode, i_ioend_work);
|
2019-10-17 20:12:15 +00:00
|
|
|
struct iomap_ioend *ioend;
|
2019-10-17 20:12:07 +00:00
|
|
|
struct list_head tmp;
|
xfs: implement per-inode writeback completion queues
When scheduling writeback of dirty file data in the page cache, XFS uses
IO completion workqueue items to ensure that filesystem metadata only
updates after the write completes successfully. This is essential for
converting unwritten extents to real extents at the right time and
performing COW remappings.
Unfortunately, XFS queues each IO completion work item to an unbounded
workqueue, which means that the kernel can spawn dozens of threads to
try to handle the items quickly. These threads need to take the ILOCK
to update file metadata, which results in heavy ILOCK contention if a
large number of the work items target a single file, which is
inefficient.
Worse yet, the writeback completion threads get stuck waiting for the
ILOCK while holding transaction reservations, which can use up all
available log reservation space. When that happens, metadata updates to
other parts of the filesystem grind to a halt, even if the filesystem
could otherwise have handled it.
Even worse, if one of the things grinding to a halt happens to be a
thread in the middle of a defer-ops finish holding the same ILOCK and
trying to obtain more log reservation having exhausted the permanent
reservation, we now have an ABBA deadlock - writeback completion has a
transaction reserved and wants the ILOCK, and someone else has the ILOCK
and wants a transaction reservation.
Therefore, we create a per-inode writeback io completion queue + work
item. When writeback finishes, it can add the ioend to the per-inode
queue and let the single worker item process that queue. This
dramatically cuts down on the number of kworkers and ILOCK contention in
the system, and seems to have eliminated an occasional deadlock I was
seeing while running generic/476.
Testing with a program that simulates a heavy random-write workload to a
single file demonstrates that the number of kworkers drops from
approximately 120 threads per file to 1, without dramatically changing
write bandwidth or pagecache access latency.
Note that we leave the xfs-conv workqueue's max_active alone because we
still want to be able to run ioend processing for as many inodes as the
system can handle.
Signed-off-by: Darrick J. Wong <darrick.wong@oracle.com>
Reviewed-by: Brian Foster <bfoster@redhat.com>
2019-04-15 20:13:20 +00:00
|
|
|
unsigned long flags;
|
|
|
|
|
|
|
|
|
|
spin_lock_irqsave(&ip->i_ioend_lock, flags);
|
2019-10-17 20:12:07 +00:00
|
|
|
list_replace_init(&ip->i_ioend_list, &tmp);
|
xfs: implement per-inode writeback completion queues
When scheduling writeback of dirty file data in the page cache, XFS uses
IO completion workqueue items to ensure that filesystem metadata only
updates after the write completes successfully. This is essential for
converting unwritten extents to real extents at the right time and
performing COW remappings.
Unfortunately, XFS queues each IO completion work item to an unbounded
workqueue, which means that the kernel can spawn dozens of threads to
try to handle the items quickly. These threads need to take the ILOCK
to update file metadata, which results in heavy ILOCK contention if a
large number of the work items target a single file, which is
inefficient.
Worse yet, the writeback completion threads get stuck waiting for the
ILOCK while holding transaction reservations, which can use up all
available log reservation space. When that happens, metadata updates to
other parts of the filesystem grind to a halt, even if the filesystem
could otherwise have handled it.
Even worse, if one of the things grinding to a halt happens to be a
thread in the middle of a defer-ops finish holding the same ILOCK and
trying to obtain more log reservation having exhausted the permanent
reservation, we now have an ABBA deadlock - writeback completion has a
transaction reserved and wants the ILOCK, and someone else has the ILOCK
and wants a transaction reservation.
Therefore, we create a per-inode writeback io completion queue + work
item. When writeback finishes, it can add the ioend to the per-inode
queue and let the single worker item process that queue. This
dramatically cuts down on the number of kworkers and ILOCK contention in
the system, and seems to have eliminated an occasional deadlock I was
seeing while running generic/476.
Testing with a program that simulates a heavy random-write workload to a
single file demonstrates that the number of kworkers drops from
approximately 120 threads per file to 1, without dramatically changing
write bandwidth or pagecache access latency.
Note that we leave the xfs-conv workqueue's max_active alone because we
still want to be able to run ioend processing for as many inodes as the
system can handle.
Signed-off-by: Darrick J. Wong <darrick.wong@oracle.com>
Reviewed-by: Brian Foster <bfoster@redhat.com>
2019-04-15 20:13:20 +00:00
|
|
|
spin_unlock_irqrestore(&ip->i_ioend_lock, flags);
|
|
|
|
|
|
2019-10-17 20:12:15 +00:00
|
|
|
iomap_sort_ioends(&tmp);
|
|
|
|
|
while ((ioend = list_first_entry_or_null(&tmp, struct iomap_ioend,
|
2019-10-17 20:12:07 +00:00
|
|
|
io_list))) {
|
xfs: implement per-inode writeback completion queues
When scheduling writeback of dirty file data in the page cache, XFS uses
IO completion workqueue items to ensure that filesystem metadata only
updates after the write completes successfully. This is essential for
converting unwritten extents to real extents at the right time and
performing COW remappings.
Unfortunately, XFS queues each IO completion work item to an unbounded
workqueue, which means that the kernel can spawn dozens of threads to
try to handle the items quickly. These threads need to take the ILOCK
to update file metadata, which results in heavy ILOCK contention if a
large number of the work items target a single file, which is
inefficient.
Worse yet, the writeback completion threads get stuck waiting for the
ILOCK while holding transaction reservations, which can use up all
available log reservation space. When that happens, metadata updates to
other parts of the filesystem grind to a halt, even if the filesystem
could otherwise have handled it.
Even worse, if one of the things grinding to a halt happens to be a
thread in the middle of a defer-ops finish holding the same ILOCK and
trying to obtain more log reservation having exhausted the permanent
reservation, we now have an ABBA deadlock - writeback completion has a
transaction reserved and wants the ILOCK, and someone else has the ILOCK
and wants a transaction reservation.
Therefore, we create a per-inode writeback io completion queue + work
item. When writeback finishes, it can add the ioend to the per-inode
queue and let the single worker item process that queue. This
dramatically cuts down on the number of kworkers and ILOCK contention in
the system, and seems to have eliminated an occasional deadlock I was
seeing while running generic/476.
Testing with a program that simulates a heavy random-write workload to a
single file demonstrates that the number of kworkers drops from
approximately 120 threads per file to 1, without dramatically changing
write bandwidth or pagecache access latency.
Note that we leave the xfs-conv workqueue's max_active alone because we
still want to be able to run ioend processing for as many inodes as the
system can handle.
Signed-off-by: Darrick J. Wong <darrick.wong@oracle.com>
Reviewed-by: Brian Foster <bfoster@redhat.com>
2019-04-15 20:13:20 +00:00
|
|
|
list_del_init(&ioend->io_list);
|
2021-05-04 15:54:29 +00:00
|
|
|
iomap_ioend_try_merge(ioend, &tmp);
|
xfs: implement per-inode writeback completion queues
When scheduling writeback of dirty file data in the page cache, XFS uses
IO completion workqueue items to ensure that filesystem metadata only
updates after the write completes successfully. This is essential for
converting unwritten extents to real extents at the right time and
performing COW remappings.
Unfortunately, XFS queues each IO completion work item to an unbounded
workqueue, which means that the kernel can spawn dozens of threads to
try to handle the items quickly. These threads need to take the ILOCK
to update file metadata, which results in heavy ILOCK contention if a
large number of the work items target a single file, which is
inefficient.
Worse yet, the writeback completion threads get stuck waiting for the
ILOCK while holding transaction reservations, which can use up all
available log reservation space. When that happens, metadata updates to
other parts of the filesystem grind to a halt, even if the filesystem
could otherwise have handled it.
Even worse, if one of the things grinding to a halt happens to be a
thread in the middle of a defer-ops finish holding the same ILOCK and
trying to obtain more log reservation having exhausted the permanent
reservation, we now have an ABBA deadlock - writeback completion has a
transaction reserved and wants the ILOCK, and someone else has the ILOCK
and wants a transaction reservation.
Therefore, we create a per-inode writeback io completion queue + work
item. When writeback finishes, it can add the ioend to the per-inode
queue and let the single worker item process that queue. This
dramatically cuts down on the number of kworkers and ILOCK contention in
the system, and seems to have eliminated an occasional deadlock I was
seeing while running generic/476.
Testing with a program that simulates a heavy random-write workload to a
single file demonstrates that the number of kworkers drops from
approximately 120 threads per file to 1, without dramatically changing
write bandwidth or pagecache access latency.
Note that we leave the xfs-conv workqueue's max_active alone because we
still want to be able to run ioend processing for as many inodes as the
system can handle.
Signed-off-by: Darrick J. Wong <darrick.wong@oracle.com>
Reviewed-by: Brian Foster <bfoster@redhat.com>
2019-04-15 20:13:20 +00:00
|
|
|
xfs_end_ioend(ioend);
|
xfs, iomap: limit individual ioend chain lengths in writeback
Trond Myklebust reported soft lockups in XFS IO completion such as
this:
watchdog: BUG: soft lockup - CPU#12 stuck for 23s! [kworker/12:1:3106]
CPU: 12 PID: 3106 Comm: kworker/12:1 Not tainted 4.18.0-305.10.2.el8_4.x86_64 #1
Workqueue: xfs-conv/md127 xfs_end_io [xfs]
RIP: 0010:_raw_spin_unlock_irqrestore+0x11/0x20
Call Trace:
wake_up_page_bit+0x8a/0x110
iomap_finish_ioend+0xd7/0x1c0
iomap_finish_ioends+0x7f/0xb0
xfs_end_ioend+0x6b/0x100 [xfs]
xfs_end_io+0xb9/0xe0 [xfs]
process_one_work+0x1a7/0x360
worker_thread+0x1fa/0x390
kthread+0x116/0x130
ret_from_fork+0x35/0x40
Ioends are processed as an atomic completion unit when all the
chained bios in the ioend have completed their IO. Logically
contiguous ioends can also be merged and completed as a single,
larger unit. Both of these things can be problematic as both the
bio chains per ioend and the size of the merged ioends processed as
a single completion are both unbound.
If we have a large sequential dirty region in the page cache,
write_cache_pages() will keep feeding us sequential pages and we
will keep mapping them into ioends and bios until we get a dirty
page at a non-sequential file offset. These large sequential runs
can will result in bio and ioend chaining to optimise the io
patterns. The pages iunder writeback are pinned within these chains
until the submission chaining is broken, allowing the entire chain
to be completed. This can result in huge chains being processed
in IO completion context.
We get deep bio chaining if we have large contiguous physical
extents. We will keep adding pages to the current bio until it is
full, then we'll chain a new bio to keep adding pages for writeback.
Hence we can build bio chains that map millions of pages and tens of
gigabytes of RAM if the page cache contains big enough contiguous
dirty file regions. This long bio chain pins those pages until the
final bio in the chain completes and the ioend can iterate all the
chained bios and complete them.
OTOH, if we have a physically fragmented file, we end up submitting
one ioend per physical fragment that each have a small bio or bio
chain attached to them. We do not chain these at IO submission time,
but instead we chain them at completion time based on file
offset via iomap_ioend_try_merge(). Hence we can end up with unbound
ioend chains being built via completion merging.
XFS can then do COW remapping or unwritten extent conversion on that
merged chain, which involves walking an extent fragment at a time
and running a transaction to modify the physical extent information.
IOWs, we merge all the discontiguous ioends together into a
contiguous file range, only to then process them individually as
discontiguous extents.
This extent manipulation is computationally expensive and can run in
a tight loop, so merging logically contiguous but physically
discontigous ioends gains us nothing except for hiding the fact the
fact we broke the ioends up into individual physical extents at
submission and then need to loop over those individual physical
extents at completion.
Hence we need to have mechanisms to limit ioend sizes and
to break up completion processing of large merged ioend chains:
1. bio chains per ioend need to be bound in length. Pure overwrites
go straight to iomap_finish_ioend() in softirq context with the
exact bio chain attached to the ioend by submission. Hence the only
way to prevent long holdoffs here is to bound ioend submission
sizes because we can't reschedule in softirq context.
2. iomap_finish_ioends() has to handle unbound merged ioend chains
correctly. This relies on any one call to iomap_finish_ioend() being
bound in runtime so that cond_resched() can be issued regularly as
the long ioend chain is processed. i.e. this relies on mechanism #1
to limit individual ioend sizes to work correctly.
3. filesystems have to loop over the merged ioends to process
physical extent manipulations. This means they can loop internally,
and so we break merging at physical extent boundaries so the
filesystem can easily insert reschedule points between individual
extent manipulations.
Signed-off-by: Dave Chinner <dchinner@redhat.com>
Reported-and-tested-by: Trond Myklebust <trondmy@hammerspace.com>
Reviewed-by: Darrick J. Wong <djwong@kernel.org>
Signed-off-by: Darrick J. Wong <djwong@kernel.org>
2022-01-26 17:19:20 +00:00
|
|
|
cond_resched();
|
xfs: implement per-inode writeback completion queues
When scheduling writeback of dirty file data in the page cache, XFS uses
IO completion workqueue items to ensure that filesystem metadata only
updates after the write completes successfully. This is essential for
converting unwritten extents to real extents at the right time and
performing COW remappings.
Unfortunately, XFS queues each IO completion work item to an unbounded
workqueue, which means that the kernel can spawn dozens of threads to
try to handle the items quickly. These threads need to take the ILOCK
to update file metadata, which results in heavy ILOCK contention if a
large number of the work items target a single file, which is
inefficient.
Worse yet, the writeback completion threads get stuck waiting for the
ILOCK while holding transaction reservations, which can use up all
available log reservation space. When that happens, metadata updates to
other parts of the filesystem grind to a halt, even if the filesystem
could otherwise have handled it.
Even worse, if one of the things grinding to a halt happens to be a
thread in the middle of a defer-ops finish holding the same ILOCK and
trying to obtain more log reservation having exhausted the permanent
reservation, we now have an ABBA deadlock - writeback completion has a
transaction reserved and wants the ILOCK, and someone else has the ILOCK
and wants a transaction reservation.
Therefore, we create a per-inode writeback io completion queue + work
item. When writeback finishes, it can add the ioend to the per-inode
queue and let the single worker item process that queue. This
dramatically cuts down on the number of kworkers and ILOCK contention in
the system, and seems to have eliminated an occasional deadlock I was
seeing while running generic/476.
Testing with a program that simulates a heavy random-write workload to a
single file demonstrates that the number of kworkers drops from
approximately 120 threads per file to 1, without dramatically changing
write bandwidth or pagecache access latency.
Note that we leave the xfs-conv workqueue's max_active alone because we
still want to be able to run ioend processing for as many inodes as the
system can handle.
Signed-off-by: Darrick J. Wong <darrick.wong@oracle.com>
Reviewed-by: Brian Foster <bfoster@redhat.com>
2019-04-15 20:13:20 +00:00
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2016-04-05 22:34:30 +00:00
|
|
|
STATIC void
|
|
|
|
|
xfs_end_bio(
|
|
|
|
|
struct bio *bio)
|
2005-09-02 06:58:49 +00:00
|
|
|
{
|
2023-12-07 07:27:05 +00:00
|
|
|
struct iomap_ioend *ioend = iomap_ioend_from_bio(bio);
|
xfs: implement per-inode writeback completion queues
When scheduling writeback of dirty file data in the page cache, XFS uses
IO completion workqueue items to ensure that filesystem metadata only
updates after the write completes successfully. This is essential for
converting unwritten extents to real extents at the right time and
performing COW remappings.
Unfortunately, XFS queues each IO completion work item to an unbounded
workqueue, which means that the kernel can spawn dozens of threads to
try to handle the items quickly. These threads need to take the ILOCK
to update file metadata, which results in heavy ILOCK contention if a
large number of the work items target a single file, which is
inefficient.
Worse yet, the writeback completion threads get stuck waiting for the
ILOCK while holding transaction reservations, which can use up all
available log reservation space. When that happens, metadata updates to
other parts of the filesystem grind to a halt, even if the filesystem
could otherwise have handled it.
Even worse, if one of the things grinding to a halt happens to be a
thread in the middle of a defer-ops finish holding the same ILOCK and
trying to obtain more log reservation having exhausted the permanent
reservation, we now have an ABBA deadlock - writeback completion has a
transaction reserved and wants the ILOCK, and someone else has the ILOCK
and wants a transaction reservation.
Therefore, we create a per-inode writeback io completion queue + work
item. When writeback finishes, it can add the ioend to the per-inode
queue and let the single worker item process that queue. This
dramatically cuts down on the number of kworkers and ILOCK contention in
the system, and seems to have eliminated an occasional deadlock I was
seeing while running generic/476.
Testing with a program that simulates a heavy random-write workload to a
single file demonstrates that the number of kworkers drops from
approximately 120 threads per file to 1, without dramatically changing
write bandwidth or pagecache access latency.
Note that we leave the xfs-conv workqueue's max_active alone because we
still want to be able to run ioend processing for as many inodes as the
system can handle.
Signed-off-by: Darrick J. Wong <darrick.wong@oracle.com>
Reviewed-by: Brian Foster <bfoster@redhat.com>
2019-04-15 20:13:20 +00:00
|
|
|
struct xfs_inode *ip = XFS_I(ioend->io_inode);
|
|
|
|
|
unsigned long flags;
|
2005-09-02 06:58:49 +00:00
|
|
|
|
2019-10-17 20:12:15 +00:00
|
|
|
spin_lock_irqsave(&ip->i_ioend_lock, flags);
|
|
|
|
|
if (list_empty(&ip->i_ioend_list))
|
|
|
|
|
WARN_ON_ONCE(!queue_work(ip->i_mount->m_unwritten_workqueue,
|
|
|
|
|
&ip->i_ioend_work));
|
|
|
|
|
list_add_tail(&ioend->io_list, &ip->i_ioend_list);
|
|
|
|
|
spin_unlock_irqrestore(&ip->i_ioend_lock, flags);
|
2005-09-02 06:58:49 +00:00
|
|
|
}
|
|
|
|
|
|
xfs: validate writeback mapping using data fork seq counter
The writeback code caches the current extent mapping across multiple
xfs_do_writepage() calls to avoid repeated lookups for sequential
pages backed by the same extent. This is known to be slightly racy
with extent fork changes in certain difficult to reproduce
scenarios. The cached extent is trimmed to within EOF to help avoid
the most common vector for this problem via speculative
preallocation management, but this is a band-aid that does not
address the fundamental problem.
Now that we have an xfs_ifork sequence counter mechanism used to
facilitate COW writeback, we can use the same mechanism to validate
consistency between the data fork and cached writeback mappings. On
its face, this is somewhat of a big hammer approach because any
change to the data fork invalidates any mapping currently cached by
a writeback in progress regardless of whether the data fork change
overlaps with the range under writeback. In practice, however, the
impact of this approach is minimal in most cases.
First, data fork changes (delayed allocations) caused by sustained
sequential buffered writes are amortized across speculative
preallocations. This means that a cached mapping won't be
invalidated by each buffered write of a common file copy workload,
but rather only on less frequent allocation events. Second, the
extent tree is always entirely in-core so an additional lookup of a
usable extent mostly costs a shared ilock cycle and in-memory tree
lookup. This means that a cached mapping reval is relatively cheap
compared to the I/O itself. Third, spurious invalidations don't
impact ioend construction. This means that even if the same extent
is revalidated multiple times across multiple writepage instances,
we still construct and submit the same size ioend (and bio) if the
blocks are physically contiguous.
Update struct xfs_writepage_ctx with a new field to hold the
sequence number of the data fork associated with the currently
cached mapping. Check the wpc seqno against the data fork when the
mapping is validated and reestablish the mapping whenever the fork
has changed since the mapping was cached. This ensures that
writeback always uses a valid extent mapping and thus prevents lost
writebacks and stale delalloc block problems.
Signed-off-by: Brian Foster <bfoster@redhat.com>
Reviewed-by: Allison Henderson <allison.henderson@oracle.com>
Reviewed-by: Christoph Hellwig <hch@lst.de>
Reviewed-by: Darrick J. Wong <darrick.wong@oracle.com>
Signed-off-by: Darrick J. Wong <darrick.wong@oracle.com>
2019-02-01 17:14:23 +00:00
|
|
|
/*
|
|
|
|
|
* Fast revalidation of the cached writeback mapping. Return true if the current
|
|
|
|
|
* mapping is valid, false otherwise.
|
|
|
|
|
*/
|
|
|
|
|
static bool
|
|
|
|
|
xfs_imap_valid(
|
2019-10-17 20:12:15 +00:00
|
|
|
struct iomap_writepage_ctx *wpc,
|
xfs: validate writeback mapping using data fork seq counter
The writeback code caches the current extent mapping across multiple
xfs_do_writepage() calls to avoid repeated lookups for sequential
pages backed by the same extent. This is known to be slightly racy
with extent fork changes in certain difficult to reproduce
scenarios. The cached extent is trimmed to within EOF to help avoid
the most common vector for this problem via speculative
preallocation management, but this is a band-aid that does not
address the fundamental problem.
Now that we have an xfs_ifork sequence counter mechanism used to
facilitate COW writeback, we can use the same mechanism to validate
consistency between the data fork and cached writeback mappings. On
its face, this is somewhat of a big hammer approach because any
change to the data fork invalidates any mapping currently cached by
a writeback in progress regardless of whether the data fork change
overlaps with the range under writeback. In practice, however, the
impact of this approach is minimal in most cases.
First, data fork changes (delayed allocations) caused by sustained
sequential buffered writes are amortized across speculative
preallocations. This means that a cached mapping won't be
invalidated by each buffered write of a common file copy workload,
but rather only on less frequent allocation events. Second, the
extent tree is always entirely in-core so an additional lookup of a
usable extent mostly costs a shared ilock cycle and in-memory tree
lookup. This means that a cached mapping reval is relatively cheap
compared to the I/O itself. Third, spurious invalidations don't
impact ioend construction. This means that even if the same extent
is revalidated multiple times across multiple writepage instances,
we still construct and submit the same size ioend (and bio) if the
blocks are physically contiguous.
Update struct xfs_writepage_ctx with a new field to hold the
sequence number of the data fork associated with the currently
cached mapping. Check the wpc seqno against the data fork when the
mapping is validated and reestablish the mapping whenever the fork
has changed since the mapping was cached. This ensures that
writeback always uses a valid extent mapping and thus prevents lost
writebacks and stale delalloc block problems.
Signed-off-by: Brian Foster <bfoster@redhat.com>
Reviewed-by: Allison Henderson <allison.henderson@oracle.com>
Reviewed-by: Christoph Hellwig <hch@lst.de>
Reviewed-by: Darrick J. Wong <darrick.wong@oracle.com>
Signed-off-by: Darrick J. Wong <darrick.wong@oracle.com>
2019-02-01 17:14:23 +00:00
|
|
|
struct xfs_inode *ip,
|
2019-10-17 20:12:06 +00:00
|
|
|
loff_t offset)
|
xfs: validate writeback mapping using data fork seq counter
The writeback code caches the current extent mapping across multiple
xfs_do_writepage() calls to avoid repeated lookups for sequential
pages backed by the same extent. This is known to be slightly racy
with extent fork changes in certain difficult to reproduce
scenarios. The cached extent is trimmed to within EOF to help avoid
the most common vector for this problem via speculative
preallocation management, but this is a band-aid that does not
address the fundamental problem.
Now that we have an xfs_ifork sequence counter mechanism used to
facilitate COW writeback, we can use the same mechanism to validate
consistency between the data fork and cached writeback mappings. On
its face, this is somewhat of a big hammer approach because any
change to the data fork invalidates any mapping currently cached by
a writeback in progress regardless of whether the data fork change
overlaps with the range under writeback. In practice, however, the
impact of this approach is minimal in most cases.
First, data fork changes (delayed allocations) caused by sustained
sequential buffered writes are amortized across speculative
preallocations. This means that a cached mapping won't be
invalidated by each buffered write of a common file copy workload,
but rather only on less frequent allocation events. Second, the
extent tree is always entirely in-core so an additional lookup of a
usable extent mostly costs a shared ilock cycle and in-memory tree
lookup. This means that a cached mapping reval is relatively cheap
compared to the I/O itself. Third, spurious invalidations don't
impact ioend construction. This means that even if the same extent
is revalidated multiple times across multiple writepage instances,
we still construct and submit the same size ioend (and bio) if the
blocks are physically contiguous.
Update struct xfs_writepage_ctx with a new field to hold the
sequence number of the data fork associated with the currently
cached mapping. Check the wpc seqno against the data fork when the
mapping is validated and reestablish the mapping whenever the fork
has changed since the mapping was cached. This ensures that
writeback always uses a valid extent mapping and thus prevents lost
writebacks and stale delalloc block problems.
Signed-off-by: Brian Foster <bfoster@redhat.com>
Reviewed-by: Allison Henderson <allison.henderson@oracle.com>
Reviewed-by: Christoph Hellwig <hch@lst.de>
Reviewed-by: Darrick J. Wong <darrick.wong@oracle.com>
Signed-off-by: Darrick J. Wong <darrick.wong@oracle.com>
2019-02-01 17:14:23 +00:00
|
|
|
{
|
2019-10-17 20:12:06 +00:00
|
|
|
if (offset < wpc->iomap.offset ||
|
|
|
|
|
offset >= wpc->iomap.offset + wpc->iomap.length)
|
xfs: validate writeback mapping using data fork seq counter
The writeback code caches the current extent mapping across multiple
xfs_do_writepage() calls to avoid repeated lookups for sequential
pages backed by the same extent. This is known to be slightly racy
with extent fork changes in certain difficult to reproduce
scenarios. The cached extent is trimmed to within EOF to help avoid
the most common vector for this problem via speculative
preallocation management, but this is a band-aid that does not
address the fundamental problem.
Now that we have an xfs_ifork sequence counter mechanism used to
facilitate COW writeback, we can use the same mechanism to validate
consistency between the data fork and cached writeback mappings. On
its face, this is somewhat of a big hammer approach because any
change to the data fork invalidates any mapping currently cached by
a writeback in progress regardless of whether the data fork change
overlaps with the range under writeback. In practice, however, the
impact of this approach is minimal in most cases.
First, data fork changes (delayed allocations) caused by sustained
sequential buffered writes are amortized across speculative
preallocations. This means that a cached mapping won't be
invalidated by each buffered write of a common file copy workload,
but rather only on less frequent allocation events. Second, the
extent tree is always entirely in-core so an additional lookup of a
usable extent mostly costs a shared ilock cycle and in-memory tree
lookup. This means that a cached mapping reval is relatively cheap
compared to the I/O itself. Third, spurious invalidations don't
impact ioend construction. This means that even if the same extent
is revalidated multiple times across multiple writepage instances,
we still construct and submit the same size ioend (and bio) if the
blocks are physically contiguous.
Update struct xfs_writepage_ctx with a new field to hold the
sequence number of the data fork associated with the currently
cached mapping. Check the wpc seqno against the data fork when the
mapping is validated and reestablish the mapping whenever the fork
has changed since the mapping was cached. This ensures that
writeback always uses a valid extent mapping and thus prevents lost
writebacks and stale delalloc block problems.
Signed-off-by: Brian Foster <bfoster@redhat.com>
Reviewed-by: Allison Henderson <allison.henderson@oracle.com>
Reviewed-by: Christoph Hellwig <hch@lst.de>
Reviewed-by: Darrick J. Wong <darrick.wong@oracle.com>
Signed-off-by: Darrick J. Wong <darrick.wong@oracle.com>
2019-02-01 17:14:23 +00:00
|
|
|
return false;
|
|
|
|
|
/*
|
|
|
|
|
* If this is a COW mapping, it is sufficient to check that the mapping
|
|
|
|
|
* covers the offset. Be careful to check this first because the caller
|
|
|
|
|
* can revalidate a COW mapping without updating the data seqno.
|
|
|
|
|
*/
|
2019-10-17 20:12:10 +00:00
|
|
|
if (wpc->iomap.flags & IOMAP_F_SHARED)
|
xfs: validate writeback mapping using data fork seq counter
The writeback code caches the current extent mapping across multiple
xfs_do_writepage() calls to avoid repeated lookups for sequential
pages backed by the same extent. This is known to be slightly racy
with extent fork changes in certain difficult to reproduce
scenarios. The cached extent is trimmed to within EOF to help avoid
the most common vector for this problem via speculative
preallocation management, but this is a band-aid that does not
address the fundamental problem.
Now that we have an xfs_ifork sequence counter mechanism used to
facilitate COW writeback, we can use the same mechanism to validate
consistency between the data fork and cached writeback mappings. On
its face, this is somewhat of a big hammer approach because any
change to the data fork invalidates any mapping currently cached by
a writeback in progress regardless of whether the data fork change
overlaps with the range under writeback. In practice, however, the
impact of this approach is minimal in most cases.
First, data fork changes (delayed allocations) caused by sustained
sequential buffered writes are amortized across speculative
preallocations. This means that a cached mapping won't be
invalidated by each buffered write of a common file copy workload,
but rather only on less frequent allocation events. Second, the
extent tree is always entirely in-core so an additional lookup of a
usable extent mostly costs a shared ilock cycle and in-memory tree
lookup. This means that a cached mapping reval is relatively cheap
compared to the I/O itself. Third, spurious invalidations don't
impact ioend construction. This means that even if the same extent
is revalidated multiple times across multiple writepage instances,
we still construct and submit the same size ioend (and bio) if the
blocks are physically contiguous.
Update struct xfs_writepage_ctx with a new field to hold the
sequence number of the data fork associated with the currently
cached mapping. Check the wpc seqno against the data fork when the
mapping is validated and reestablish the mapping whenever the fork
has changed since the mapping was cached. This ensures that
writeback always uses a valid extent mapping and thus prevents lost
writebacks and stale delalloc block problems.
Signed-off-by: Brian Foster <bfoster@redhat.com>
Reviewed-by: Allison Henderson <allison.henderson@oracle.com>
Reviewed-by: Christoph Hellwig <hch@lst.de>
Reviewed-by: Darrick J. Wong <darrick.wong@oracle.com>
Signed-off-by: Darrick J. Wong <darrick.wong@oracle.com>
2019-02-01 17:14:23 +00:00
|
|
|
return true;
|
|
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* This is not a COW mapping. Check the sequence number of the data fork
|
|
|
|
|
* because concurrent changes could have invalidated the extent. Check
|
|
|
|
|
* the COW fork because concurrent changes since the last time we
|
|
|
|
|
* checked (and found nothing at this offset) could have added
|
|
|
|
|
* overlapping blocks.
|
|
|
|
|
*/
|
2022-11-29 01:24:35 +00:00
|
|
|
if (XFS_WPC(wpc)->data_seq != READ_ONCE(ip->i_df.if_seq)) {
|
|
|
|
|
trace_xfs_wb_data_iomap_invalid(ip, &wpc->iomap,
|
|
|
|
|
XFS_WPC(wpc)->data_seq, XFS_DATA_FORK);
|
xfs: validate writeback mapping using data fork seq counter
The writeback code caches the current extent mapping across multiple
xfs_do_writepage() calls to avoid repeated lookups for sequential
pages backed by the same extent. This is known to be slightly racy
with extent fork changes in certain difficult to reproduce
scenarios. The cached extent is trimmed to within EOF to help avoid
the most common vector for this problem via speculative
preallocation management, but this is a band-aid that does not
address the fundamental problem.
Now that we have an xfs_ifork sequence counter mechanism used to
facilitate COW writeback, we can use the same mechanism to validate
consistency between the data fork and cached writeback mappings. On
its face, this is somewhat of a big hammer approach because any
change to the data fork invalidates any mapping currently cached by
a writeback in progress regardless of whether the data fork change
overlaps with the range under writeback. In practice, however, the
impact of this approach is minimal in most cases.
First, data fork changes (delayed allocations) caused by sustained
sequential buffered writes are amortized across speculative
preallocations. This means that a cached mapping won't be
invalidated by each buffered write of a common file copy workload,
but rather only on less frequent allocation events. Second, the
extent tree is always entirely in-core so an additional lookup of a
usable extent mostly costs a shared ilock cycle and in-memory tree
lookup. This means that a cached mapping reval is relatively cheap
compared to the I/O itself. Third, spurious invalidations don't
impact ioend construction. This means that even if the same extent
is revalidated multiple times across multiple writepage instances,
we still construct and submit the same size ioend (and bio) if the
blocks are physically contiguous.
Update struct xfs_writepage_ctx with a new field to hold the
sequence number of the data fork associated with the currently
cached mapping. Check the wpc seqno against the data fork when the
mapping is validated and reestablish the mapping whenever the fork
has changed since the mapping was cached. This ensures that
writeback always uses a valid extent mapping and thus prevents lost
writebacks and stale delalloc block problems.
Signed-off-by: Brian Foster <bfoster@redhat.com>
Reviewed-by: Allison Henderson <allison.henderson@oracle.com>
Reviewed-by: Christoph Hellwig <hch@lst.de>
Reviewed-by: Darrick J. Wong <darrick.wong@oracle.com>
Signed-off-by: Darrick J. Wong <darrick.wong@oracle.com>
2019-02-01 17:14:23 +00:00
|
|
|
return false;
|
2022-11-29 01:24:35 +00:00
|
|
|
}
|
xfs: validate writeback mapping using data fork seq counter
The writeback code caches the current extent mapping across multiple
xfs_do_writepage() calls to avoid repeated lookups for sequential
pages backed by the same extent. This is known to be slightly racy
with extent fork changes in certain difficult to reproduce
scenarios. The cached extent is trimmed to within EOF to help avoid
the most common vector for this problem via speculative
preallocation management, but this is a band-aid that does not
address the fundamental problem.
Now that we have an xfs_ifork sequence counter mechanism used to
facilitate COW writeback, we can use the same mechanism to validate
consistency between the data fork and cached writeback mappings. On
its face, this is somewhat of a big hammer approach because any
change to the data fork invalidates any mapping currently cached by
a writeback in progress regardless of whether the data fork change
overlaps with the range under writeback. In practice, however, the
impact of this approach is minimal in most cases.
First, data fork changes (delayed allocations) caused by sustained
sequential buffered writes are amortized across speculative
preallocations. This means that a cached mapping won't be
invalidated by each buffered write of a common file copy workload,
but rather only on less frequent allocation events. Second, the
extent tree is always entirely in-core so an additional lookup of a
usable extent mostly costs a shared ilock cycle and in-memory tree
lookup. This means that a cached mapping reval is relatively cheap
compared to the I/O itself. Third, spurious invalidations don't
impact ioend construction. This means that even if the same extent
is revalidated multiple times across multiple writepage instances,
we still construct and submit the same size ioend (and bio) if the
blocks are physically contiguous.
Update struct xfs_writepage_ctx with a new field to hold the
sequence number of the data fork associated with the currently
cached mapping. Check the wpc seqno against the data fork when the
mapping is validated and reestablish the mapping whenever the fork
has changed since the mapping was cached. This ensures that
writeback always uses a valid extent mapping and thus prevents lost
writebacks and stale delalloc block problems.
Signed-off-by: Brian Foster <bfoster@redhat.com>
Reviewed-by: Allison Henderson <allison.henderson@oracle.com>
Reviewed-by: Christoph Hellwig <hch@lst.de>
Reviewed-by: Darrick J. Wong <darrick.wong@oracle.com>
Signed-off-by: Darrick J. Wong <darrick.wong@oracle.com>
2019-02-01 17:14:23 +00:00
|
|
|
if (xfs_inode_has_cow_data(ip) &&
|
2022-11-29 01:24:35 +00:00
|
|
|
XFS_WPC(wpc)->cow_seq != READ_ONCE(ip->i_cowfp->if_seq)) {
|
|
|
|
|
trace_xfs_wb_cow_iomap_invalid(ip, &wpc->iomap,
|
|
|
|
|
XFS_WPC(wpc)->cow_seq, XFS_COW_FORK);
|
xfs: validate writeback mapping using data fork seq counter
The writeback code caches the current extent mapping across multiple
xfs_do_writepage() calls to avoid repeated lookups for sequential
pages backed by the same extent. This is known to be slightly racy
with extent fork changes in certain difficult to reproduce
scenarios. The cached extent is trimmed to within EOF to help avoid
the most common vector for this problem via speculative
preallocation management, but this is a band-aid that does not
address the fundamental problem.
Now that we have an xfs_ifork sequence counter mechanism used to
facilitate COW writeback, we can use the same mechanism to validate
consistency between the data fork and cached writeback mappings. On
its face, this is somewhat of a big hammer approach because any
change to the data fork invalidates any mapping currently cached by
a writeback in progress regardless of whether the data fork change
overlaps with the range under writeback. In practice, however, the
impact of this approach is minimal in most cases.
First, data fork changes (delayed allocations) caused by sustained
sequential buffered writes are amortized across speculative
preallocations. This means that a cached mapping won't be
invalidated by each buffered write of a common file copy workload,
but rather only on less frequent allocation events. Second, the
extent tree is always entirely in-core so an additional lookup of a
usable extent mostly costs a shared ilock cycle and in-memory tree
lookup. This means that a cached mapping reval is relatively cheap
compared to the I/O itself. Third, spurious invalidations don't
impact ioend construction. This means that even if the same extent
is revalidated multiple times across multiple writepage instances,
we still construct and submit the same size ioend (and bio) if the
blocks are physically contiguous.
Update struct xfs_writepage_ctx with a new field to hold the
sequence number of the data fork associated with the currently
cached mapping. Check the wpc seqno against the data fork when the
mapping is validated and reestablish the mapping whenever the fork
has changed since the mapping was cached. This ensures that
writeback always uses a valid extent mapping and thus prevents lost
writebacks and stale delalloc block problems.
Signed-off-by: Brian Foster <bfoster@redhat.com>
Reviewed-by: Allison Henderson <allison.henderson@oracle.com>
Reviewed-by: Christoph Hellwig <hch@lst.de>
Reviewed-by: Darrick J. Wong <darrick.wong@oracle.com>
Signed-off-by: Darrick J. Wong <darrick.wong@oracle.com>
2019-02-01 17:14:23 +00:00
|
|
|
return false;
|
2022-11-29 01:24:35 +00:00
|
|
|
}
|
xfs: validate writeback mapping using data fork seq counter
The writeback code caches the current extent mapping across multiple
xfs_do_writepage() calls to avoid repeated lookups for sequential
pages backed by the same extent. This is known to be slightly racy
with extent fork changes in certain difficult to reproduce
scenarios. The cached extent is trimmed to within EOF to help avoid
the most common vector for this problem via speculative
preallocation management, but this is a band-aid that does not
address the fundamental problem.
Now that we have an xfs_ifork sequence counter mechanism used to
facilitate COW writeback, we can use the same mechanism to validate
consistency between the data fork and cached writeback mappings. On
its face, this is somewhat of a big hammer approach because any
change to the data fork invalidates any mapping currently cached by
a writeback in progress regardless of whether the data fork change
overlaps with the range under writeback. In practice, however, the
impact of this approach is minimal in most cases.
First, data fork changes (delayed allocations) caused by sustained
sequential buffered writes are amortized across speculative
preallocations. This means that a cached mapping won't be
invalidated by each buffered write of a common file copy workload,
but rather only on less frequent allocation events. Second, the
extent tree is always entirely in-core so an additional lookup of a
usable extent mostly costs a shared ilock cycle and in-memory tree
lookup. This means that a cached mapping reval is relatively cheap
compared to the I/O itself. Third, spurious invalidations don't
impact ioend construction. This means that even if the same extent
is revalidated multiple times across multiple writepage instances,
we still construct and submit the same size ioend (and bio) if the
blocks are physically contiguous.
Update struct xfs_writepage_ctx with a new field to hold the
sequence number of the data fork associated with the currently
cached mapping. Check the wpc seqno against the data fork when the
mapping is validated and reestablish the mapping whenever the fork
has changed since the mapping was cached. This ensures that
writeback always uses a valid extent mapping and thus prevents lost
writebacks and stale delalloc block problems.
Signed-off-by: Brian Foster <bfoster@redhat.com>
Reviewed-by: Allison Henderson <allison.henderson@oracle.com>
Reviewed-by: Christoph Hellwig <hch@lst.de>
Reviewed-by: Darrick J. Wong <darrick.wong@oracle.com>
Signed-off-by: Darrick J. Wong <darrick.wong@oracle.com>
2019-02-01 17:14:23 +00:00
|
|
|
return true;
|
|
|
|
|
}
|
|
|
|
|
|
2019-10-17 20:12:15 +00:00
|
|
|
static int
|
2005-04-16 22:20:36 +00:00
|
|
|
xfs_map_blocks(
|
2019-10-17 20:12:15 +00:00
|
|
|
struct iomap_writepage_ctx *wpc,
|
2005-04-16 22:20:36 +00:00
|
|
|
struct inode *inode,
|
2023-12-07 07:27:10 +00:00
|
|
|
loff_t offset,
|
|
|
|
|
unsigned int len)
|
2005-04-16 22:20:36 +00:00
|
|
|
{
|
2010-12-10 08:42:20 +00:00
|
|
|
struct xfs_inode *ip = XFS_I(inode);
|
|
|
|
|
struct xfs_mount *mp = ip->i_mount;
|
2017-02-27 22:28:32 +00:00
|
|
|
ssize_t count = i_blocksize(inode);
|
2019-02-15 16:02:47 +00:00
|
|
|
xfs_fileoff_t offset_fsb = XFS_B_TO_FSBT(mp, offset);
|
|
|
|
|
xfs_fileoff_t end_fsb = XFS_B_TO_FSB(mp, offset + count);
|
xfs: fix missing CoW blocks writeback conversion retry
In commit 7588cbeec6df, we tried to fix a race stemming from the lack of
coordination between higher level code that wants to allocate and remap
CoW fork extents into the data fork. Christoph cites as examples the
always_cow mode, and a directio write completion racing with writeback.
According to the comments before the goto retry, we want to restart the
lookup to catch the extent in the data fork, but we don't actually reset
whichfork or cow_fsb, which means the second try executes using stale
information. Up until now I think we've gotten lucky that either
there's something left in the CoW fork to cause cow_fsb to be reset, or
either data/cow fork sequence numbers have advanced enough to force a
fresh lookup from the data fork. However, if we reach the retry with an
empty stable CoW fork and a stable data fork, neither of those things
happens. The retry foolishly re-calls xfs_convert_blocks on the CoW
fork which fails again. This time, we toss the write.
I've recently been working on extending reflink to the realtime device.
When the realtime extent size is larger than a single block, we have to
force the page cache to CoW the entire rt extent if a write (or
fallocate) are not aligned with the rt extent size. The strategy I've
chosen to deal with this is derived from Dave's blocksize > pagesize
series: dirtying around the write range, and ensuring that writeback
always starts mapping on an rt extent boundary. This has brought this
race front and center, since generic/522 blows up immediately.
However, I'm pretty sure this is a bug outright, independent of that.
Fixes: 7588cbeec6df ("xfs: retry COW fork delalloc conversion when no extent was found")
Signed-off-by: Darrick J. Wong <darrick.wong@oracle.com>
Reviewed-by: Christoph Hellwig <hch@lst.de>
2020-11-03 01:14:06 +00:00
|
|
|
xfs_fileoff_t cow_fsb;
|
|
|
|
|
int whichfork;
|
2018-07-12 05:25:59 +00:00
|
|
|
struct xfs_bmbt_irec imap;
|
2018-07-12 05:26:01 +00:00
|
|
|
struct xfs_iext_cursor icur;
|
2019-02-15 16:02:50 +00:00
|
|
|
int retries = 0;
|
2010-12-10 08:42:20 +00:00
|
|
|
int error = 0;
|
2024-04-25 13:13:29 +00:00
|
|
|
unsigned int *seq;
|
2010-12-10 08:42:20 +00:00
|
|
|
|
2021-08-19 01:46:53 +00:00
|
|
|
if (xfs_is_shutdown(mp))
|
xfs: validate writeback mapping using data fork seq counter
The writeback code caches the current extent mapping across multiple
xfs_do_writepage() calls to avoid repeated lookups for sequential
pages backed by the same extent. This is known to be slightly racy
with extent fork changes in certain difficult to reproduce
scenarios. The cached extent is trimmed to within EOF to help avoid
the most common vector for this problem via speculative
preallocation management, but this is a band-aid that does not
address the fundamental problem.
Now that we have an xfs_ifork sequence counter mechanism used to
facilitate COW writeback, we can use the same mechanism to validate
consistency between the data fork and cached writeback mappings. On
its face, this is somewhat of a big hammer approach because any
change to the data fork invalidates any mapping currently cached by
a writeback in progress regardless of whether the data fork change
overlaps with the range under writeback. In practice, however, the
impact of this approach is minimal in most cases.
First, data fork changes (delayed allocations) caused by sustained
sequential buffered writes are amortized across speculative
preallocations. This means that a cached mapping won't be
invalidated by each buffered write of a common file copy workload,
but rather only on less frequent allocation events. Second, the
extent tree is always entirely in-core so an additional lookup of a
usable extent mostly costs a shared ilock cycle and in-memory tree
lookup. This means that a cached mapping reval is relatively cheap
compared to the I/O itself. Third, spurious invalidations don't
impact ioend construction. This means that even if the same extent
is revalidated multiple times across multiple writepage instances,
we still construct and submit the same size ioend (and bio) if the
blocks are physically contiguous.
Update struct xfs_writepage_ctx with a new field to hold the
sequence number of the data fork associated with the currently
cached mapping. Check the wpc seqno against the data fork when the
mapping is validated and reestablish the mapping whenever the fork
has changed since the mapping was cached. This ensures that
writeback always uses a valid extent mapping and thus prevents lost
writebacks and stale delalloc block problems.
Signed-off-by: Brian Foster <bfoster@redhat.com>
Reviewed-by: Allison Henderson <allison.henderson@oracle.com>
Reviewed-by: Christoph Hellwig <hch@lst.de>
Reviewed-by: Darrick J. Wong <darrick.wong@oracle.com>
Signed-off-by: Darrick J. Wong <darrick.wong@oracle.com>
2019-02-01 17:14:23 +00:00
|
|
|
return -EIO;
|
|
|
|
|
|
2022-11-29 01:24:35 +00:00
|
|
|
XFS_ERRORTAG_DELAY(mp, XFS_ERRTAG_WB_DELAY_MS);
|
|
|
|
|
|
2018-07-12 05:26:02 +00:00
|
|
|
/*
|
|
|
|
|
* COW fork blocks can overlap data fork blocks even if the blocks
|
|
|
|
|
* aren't shared. COW I/O always takes precedent, so we must always
|
|
|
|
|
* check for overlap on reflink inodes unless the mapping is already a
|
2018-07-17 23:51:52 +00:00
|
|
|
* COW one, or the COW fork hasn't changed from the last time we looked
|
|
|
|
|
* at it.
|
|
|
|
|
*
|
|
|
|
|
* It's safe to check the COW fork if_seq here without the ILOCK because
|
|
|
|
|
* we've indirectly protected against concurrent updates: writeback has
|
|
|
|
|
* the page locked, which prevents concurrent invalidations by reflink
|
|
|
|
|
* and directio and prevents concurrent buffered writes to the same
|
|
|
|
|
* page. Changes to if_seq always happen under i_lock, which protects
|
|
|
|
|
* against concurrent updates and provides a memory barrier on the way
|
|
|
|
|
* out that ensures that we always see the current value.
|
2018-07-12 05:26:02 +00:00
|
|
|
*/
|
2019-10-17 20:12:06 +00:00
|
|
|
if (xfs_imap_valid(wpc, ip, offset))
|
2018-07-12 05:26:02 +00:00
|
|
|
return 0;
|
|
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* If we don't have a valid map, now it's time to get a new one for this
|
|
|
|
|
* offset. This will convert delayed allocations (including COW ones)
|
|
|
|
|
* into real extents. If we return without a valid map, it means we
|
|
|
|
|
* landed in a hole and we skip the block.
|
|
|
|
|
*/
|
2019-02-15 16:02:50 +00:00
|
|
|
retry:
|
xfs: fix missing CoW blocks writeback conversion retry
In commit 7588cbeec6df, we tried to fix a race stemming from the lack of
coordination between higher level code that wants to allocate and remap
CoW fork extents into the data fork. Christoph cites as examples the
always_cow mode, and a directio write completion racing with writeback.
According to the comments before the goto retry, we want to restart the
lookup to catch the extent in the data fork, but we don't actually reset
whichfork or cow_fsb, which means the second try executes using stale
information. Up until now I think we've gotten lucky that either
there's something left in the CoW fork to cause cow_fsb to be reset, or
either data/cow fork sequence numbers have advanced enough to force a
fresh lookup from the data fork. However, if we reach the retry with an
empty stable CoW fork and a stable data fork, neither of those things
happens. The retry foolishly re-calls xfs_convert_blocks on the CoW
fork which fails again. This time, we toss the write.
I've recently been working on extending reflink to the realtime device.
When the realtime extent size is larger than a single block, we have to
force the page cache to CoW the entire rt extent if a write (or
fallocate) are not aligned with the rt extent size. The strategy I've
chosen to deal with this is derived from Dave's blocksize > pagesize
series: dirtying around the write range, and ensuring that writeback
always starts mapping on an rt extent boundary. This has brought this
race front and center, since generic/522 blows up immediately.
However, I'm pretty sure this is a bug outright, independent of that.
Fixes: 7588cbeec6df ("xfs: retry COW fork delalloc conversion when no extent was found")
Signed-off-by: Darrick J. Wong <darrick.wong@oracle.com>
Reviewed-by: Christoph Hellwig <hch@lst.de>
2020-11-03 01:14:06 +00:00
|
|
|
cow_fsb = NULLFILEOFF;
|
|
|
|
|
whichfork = XFS_DATA_FORK;
|
2016-02-15 06:20:50 +00:00
|
|
|
xfs_ilock(ip, XFS_ILOCK_SHARED);
|
2021-04-13 18:15:12 +00:00
|
|
|
ASSERT(!xfs_need_iread_extents(&ip->i_df));
|
2018-07-12 05:26:01 +00:00
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* Check if this is offset is covered by a COW extents, and if yes use
|
|
|
|
|
* it directly instead of looking up anything in the data fork.
|
|
|
|
|
*/
|
2018-07-17 23:51:51 +00:00
|
|
|
if (xfs_inode_has_cow_data(ip) &&
|
2018-07-17 23:51:52 +00:00
|
|
|
xfs_iext_lookup_extent(ip, ip->i_cowfp, offset_fsb, &icur, &imap))
|
|
|
|
|
cow_fsb = imap.br_startoff;
|
|
|
|
|
if (cow_fsb != NULLFILEOFF && cow_fsb <= offset_fsb) {
|
2019-10-17 20:12:15 +00:00
|
|
|
XFS_WPC(wpc)->cow_seq = READ_ONCE(ip->i_cowfp->if_seq);
|
2018-07-12 05:25:59 +00:00
|
|
|
xfs_iunlock(ip, XFS_ILOCK_SHARED);
|
2019-02-15 16:02:46 +00:00
|
|
|
|
2019-10-17 20:12:10 +00:00
|
|
|
whichfork = XFS_COW_FORK;
|
2018-07-12 05:25:59 +00:00
|
|
|
goto allocate_blocks;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/*
|
xfs: validate writeback mapping using data fork seq counter
The writeback code caches the current extent mapping across multiple
xfs_do_writepage() calls to avoid repeated lookups for sequential
pages backed by the same extent. This is known to be slightly racy
with extent fork changes in certain difficult to reproduce
scenarios. The cached extent is trimmed to within EOF to help avoid
the most common vector for this problem via speculative
preallocation management, but this is a band-aid that does not
address the fundamental problem.
Now that we have an xfs_ifork sequence counter mechanism used to
facilitate COW writeback, we can use the same mechanism to validate
consistency between the data fork and cached writeback mappings. On
its face, this is somewhat of a big hammer approach because any
change to the data fork invalidates any mapping currently cached by
a writeback in progress regardless of whether the data fork change
overlaps with the range under writeback. In practice, however, the
impact of this approach is minimal in most cases.
First, data fork changes (delayed allocations) caused by sustained
sequential buffered writes are amortized across speculative
preallocations. This means that a cached mapping won't be
invalidated by each buffered write of a common file copy workload,
but rather only on less frequent allocation events. Second, the
extent tree is always entirely in-core so an additional lookup of a
usable extent mostly costs a shared ilock cycle and in-memory tree
lookup. This means that a cached mapping reval is relatively cheap
compared to the I/O itself. Third, spurious invalidations don't
impact ioend construction. This means that even if the same extent
is revalidated multiple times across multiple writepage instances,
we still construct and submit the same size ioend (and bio) if the
blocks are physically contiguous.
Update struct xfs_writepage_ctx with a new field to hold the
sequence number of the data fork associated with the currently
cached mapping. Check the wpc seqno against the data fork when the
mapping is validated and reestablish the mapping whenever the fork
has changed since the mapping was cached. This ensures that
writeback always uses a valid extent mapping and thus prevents lost
writebacks and stale delalloc block problems.
Signed-off-by: Brian Foster <bfoster@redhat.com>
Reviewed-by: Allison Henderson <allison.henderson@oracle.com>
Reviewed-by: Christoph Hellwig <hch@lst.de>
Reviewed-by: Darrick J. Wong <darrick.wong@oracle.com>
Signed-off-by: Darrick J. Wong <darrick.wong@oracle.com>
2019-02-01 17:14:23 +00:00
|
|
|
* No COW extent overlap. Revalidate now that we may have updated
|
|
|
|
|
* ->cow_seq. If the data mapping is still valid, we're done.
|
2018-07-12 05:25:59 +00:00
|
|
|
*/
|
2019-10-17 20:12:06 +00:00
|
|
|
if (xfs_imap_valid(wpc, ip, offset)) {
|
2018-07-12 05:25:59 +00:00
|
|
|
xfs_iunlock(ip, XFS_ILOCK_SHARED);
|
|
|
|
|
return 0;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* If we don't have a valid map, now it's time to get a new one for this
|
|
|
|
|
* offset. This will convert delayed allocations (including COW ones)
|
|
|
|
|
* into real extents.
|
|
|
|
|
*/
|
2018-07-12 05:26:02 +00:00
|
|
|
if (!xfs_iext_lookup_extent(ip, &ip->i_df, offset_fsb, &icur, &imap))
|
|
|
|
|
imap.br_startoff = end_fsb; /* fake a hole past EOF */
|
2019-10-17 20:12:15 +00:00
|
|
|
XFS_WPC(wpc)->data_seq = READ_ONCE(ip->i_df.if_seq);
|
2010-12-10 08:42:21 +00:00
|
|
|
xfs_iunlock(ip, XFS_ILOCK_SHARED);
|
2010-12-10 08:42:20 +00:00
|
|
|
|
2019-02-18 17:38:47 +00:00
|
|
|
/* landed in a hole or beyond EOF? */
|
2018-07-12 05:26:02 +00:00
|
|
|
if (imap.br_startoff > offset_fsb) {
|
|
|
|
|
imap.br_blockcount = imap.br_startoff - offset_fsb;
|
2018-07-12 05:25:59 +00:00
|
|
|
imap.br_startoff = offset_fsb;
|
|
|
|
|
imap.br_startblock = HOLESTARTBLOCK;
|
2019-02-15 16:02:46 +00:00
|
|
|
imap.br_state = XFS_EXT_NORM;
|
2010-12-10 08:42:21 +00:00
|
|
|
}
|
xfs: make xfs_writepage_map extent map centric
xfs_writepage_map() iterates over the bufferheads on a page to decide
what sort of IO to do and what actions to take. However, when it comes
to reflink and deciding when it needs to execute a COW operation, we no
longer look at the bufferhead state but instead we ignore than and look
up internal state held in the COW fork extent list.
This means xfs_writepage_map() is somewhat confused. It does stuff, then
ignores it, then tries to handle the impedence mismatch by shovelling the
results inside the existing mapping code. It works, but it's a bit of a
mess and it makes it hard to fix the cached map bug that the writepage
code currently has.
To unify the two different mechanisms, we first have to choose a direction.
That's already been set - we're de-emphasising bufferheads so they are no
longer a control structure as we need to do taht to allow for eventual
removal. Hence we need to move away from looking at bufferhead state to
determine what operations we need to perform.
We can't completely get rid of bufferheads yet - they do contain some
state that is absolutely necessary, such as whether that part of the page
contains valid data or not (buffer_uptodate()). Other state in the
bufferhead is redundant:
BH_dirty - the page is dirty, so we can ignore this and just
write it
BH_delay - we have delalloc extent info in the DATA fork extent
tree
BH_unwritten - same as BH_delay
BH_mapped - indicates we've already used it once for IO and it is
mapped to a disk address. Needs to be ignored for COW
blocks.
The BH_mapped flag is an interesting case - it's supposed to indicate that
it's already mapped to disk and so we can just use it "as is". In theory,
we don't even have to do an extent lookup to find where to write it too,
but we have to do that anyway to determine we are actually writing over a
valid extent. Hence it's not even serving the purpose of avoiding a an
extent lookup during writeback, and so we can pretty much ignore it.
Especially as we have to ignore it for COW operations...
Therefore, use the extent map as the source of information to tell us
what actions we need to take and what sort of IO we should perform. The
first step is to have xfs_map_blocks() set the io type according to what
it looks up. This means it can easily handle both normal overwrite and
COW cases. The only thing we also need to add is the ability to return
hole mappings.
We need to return and cache hole mappings now for the case of multiple
blocks per page. We no longer use the BH_mapped to indicate a block over
a hole, so we have to get that info from xfs_map_blocks(). We cache it so
that holes that span two pages don't need separate lookups. This allows us
to avoid ever doing write IO over a hole, too.
Now that we have xfs_map_blocks() returning both a cached map and the type
of IO we need to perform, we can rewrite xfs_writepage_map() to drop all
the bufferhead control. It's also much simplified because it doesn't need
to explicitly handle COW operations. Instead of iterating bufferheads, it
iterates blocks within the page and then looks up what per-block state is
required from the appropriate bufferhead. It then validates the cached
map, and if it's not valid, we get a new map. If we don't get a valid map
or it's over a hole, we skip the block.
At this point, we have to remap the bufferhead via xfs_map_at_offset().
As previously noted, we had to do this even if the buffer was already
mapped as the mapping would be stale for XFS_IO_DELALLOC, XFS_IO_UNWRITTEN
and XFS_IO_COW IO types. With xfs_map_blocks() now controlling the type,
even XFS_IO_OVERWRITE types need remapping, as converted-but-not-yet-
written delalloc extents beyond EOF can be reported at XFS_IO_OVERWRITE.
Bufferheads that span such regions still need their BH_Delay flags cleared
and their block numbers calculated, so we now unconditionally map each
bufferhead before submission.
But wait! There's more - remember the old "treat unwritten extents as
holes on read" hack? Yeah, that means we can have a dirty page with
unmapped, unwritten bufferheads that contain data! What makes these so
special is that the unwritten "hole" bufferheads do not have a valid block
device pointer, so if we attempt to write them xfs_add_to_ioend() blows
up. So we make xfs_map_at_offset() do the "realtime or data device"
lookup from the inode and ignore what was or wasn't put into the
bufferhead when the buffer was instantiated.
The astute reader will have realised by now that this code treats
unwritten extents in multiple-blocks-per-page situations differently.
If we get any combination of unwritten blocks on a dirty page that contain
valid data in the page, we're going to convert them to real extents. This
can actually be a win, because it means that pages with interleaving
unwritten and written blocks will get converted to a single written extent
with zeros replacing the interspersed unwritten blocks. This is actually
good for reducing extent list and conversion overhead, and it means we
issue a contiguous IO instead of lots of little ones. The downside is
that we use up a little extra IO bandwidth. Neither of these seem like a
bad thing given that spinning disks are seek sensitive, and SSDs/pmem have
bandwidth to burn and the lower Io latency/CPU overhead of fewer, larger
IOs will result in better performance on them...
As a result of all this, the only state we actually care about from the
bufferhead is a single flag - BH_Uptodate. We still use the bufferhead to
pass some information to the bio via xfs_add_to_ioend(), but that is
trivial to separate and pass explicitly. This means we really only need
1 bit of state per block per page from the buffered write path in the
writeback path. Everything else we do with the bufferhead is purely to
make the buffered IO front end continue to work correctly. i.e we've
pretty much marginalised bufferheads in the writeback path completely.
Signed-off-By: Dave Chinner <dchinner@redhat.com>
[hch: forward port, refactor and split off bits into other commits]
Signed-off-by: Christoph Hellwig <hch@lst.de>
Reviewed-by: Brian Foster <bfoster@redhat.com>
Reviewed-by: Darrick J. Wong <darrick.wong@oracle.com>
Signed-off-by: Darrick J. Wong <darrick.wong@oracle.com>
2018-07-12 05:26:00 +00:00
|
|
|
|
2019-02-18 17:38:47 +00:00
|
|
|
/*
|
|
|
|
|
* Truncate to the next COW extent if there is one. This is the only
|
|
|
|
|
* opportunity to do this because we can skip COW fork lookups for the
|
|
|
|
|
* subsequent blocks in the mapping; however, the requirement to treat
|
|
|
|
|
* the COW range separately remains.
|
|
|
|
|
*/
|
|
|
|
|
if (cow_fsb != NULLFILEOFF &&
|
|
|
|
|
cow_fsb < imap.br_startoff + imap.br_blockcount)
|
|
|
|
|
imap.br_blockcount = cow_fsb - imap.br_startoff;
|
|
|
|
|
|
|
|
|
|
/* got a delalloc extent? */
|
|
|
|
|
if (imap.br_startblock != HOLESTARTBLOCK &&
|
|
|
|
|
isnullstartblock(imap.br_startblock))
|
|
|
|
|
goto allocate_blocks;
|
|
|
|
|
|
2022-11-28 22:09:17 +00:00
|
|
|
xfs_bmbt_to_iomap(ip, &wpc->iomap, &imap, 0, 0, XFS_WPC(wpc)->data_seq);
|
2019-10-17 20:12:10 +00:00
|
|
|
trace_xfs_map_blocks_found(ip, offset, count, whichfork, &imap);
|
2018-07-12 05:25:59 +00:00
|
|
|
return 0;
|
|
|
|
|
allocate_blocks:
|
2024-04-25 13:13:29 +00:00
|
|
|
/*
|
|
|
|
|
* Convert a dellalloc extent to a real one. The current page is held
|
|
|
|
|
* locked so nothing could have removed the block backing offset_fsb,
|
|
|
|
|
* although it could have moved from the COW to the data fork by another
|
|
|
|
|
* thread.
|
|
|
|
|
*/
|
|
|
|
|
if (whichfork == XFS_COW_FORK)
|
|
|
|
|
seq = &XFS_WPC(wpc)->cow_seq;
|
|
|
|
|
else
|
|
|
|
|
seq = &XFS_WPC(wpc)->data_seq;
|
|
|
|
|
|
|
|
|
|
error = xfs_bmapi_convert_delalloc(ip, whichfork, offset,
|
|
|
|
|
&wpc->iomap, seq);
|
2019-02-15 16:02:50 +00:00
|
|
|
if (error) {
|
|
|
|
|
/*
|
|
|
|
|
* If we failed to find the extent in the COW fork we might have
|
|
|
|
|
* raced with a COW to data fork conversion or truncate.
|
|
|
|
|
* Restart the lookup to catch the extent in the data fork for
|
|
|
|
|
* the former case, but prevent additional retries to avoid
|
|
|
|
|
* looping forever for the latter case.
|
|
|
|
|
*/
|
2019-10-17 20:12:10 +00:00
|
|
|
if (error == -EAGAIN && whichfork == XFS_COW_FORK && !retries++)
|
2019-02-15 16:02:50 +00:00
|
|
|
goto retry;
|
|
|
|
|
ASSERT(error != -EAGAIN);
|
2018-07-12 05:25:59 +00:00
|
|
|
return error;
|
2019-02-15 16:02:50 +00:00
|
|
|
}
|
2019-02-15 16:02:49 +00:00
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* Due to merging the return real extent might be larger than the
|
|
|
|
|
* original delalloc one. Trim the return extent to the next COW
|
|
|
|
|
* boundary again to force a re-lookup.
|
|
|
|
|
*/
|
2019-10-17 20:12:10 +00:00
|
|
|
if (whichfork != XFS_COW_FORK && cow_fsb != NULLFILEOFF) {
|
2019-10-17 20:12:06 +00:00
|
|
|
loff_t cow_offset = XFS_FSB_TO_B(mp, cow_fsb);
|
|
|
|
|
|
|
|
|
|
if (cow_offset < wpc->iomap.offset + wpc->iomap.length)
|
|
|
|
|
wpc->iomap.length = cow_offset - wpc->iomap.offset;
|
|
|
|
|
}
|
2019-02-15 16:02:49 +00:00
|
|
|
|
2019-10-17 20:12:06 +00:00
|
|
|
ASSERT(wpc->iomap.offset <= offset);
|
|
|
|
|
ASSERT(wpc->iomap.offset + wpc->iomap.length > offset);
|
2019-10-17 20:12:10 +00:00
|
|
|
trace_xfs_map_blocks_alloc(ip, offset, count, whichfork, &imap);
|
2010-12-10 08:42:21 +00:00
|
|
|
return 0;
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
|
|
|
|
|
2019-10-17 20:12:15 +00:00
|
|
|
static int
|
|
|
|
|
xfs_prepare_ioend(
|
|
|
|
|
struct iomap_ioend *ioend,
|
2016-02-15 06:23:12 +00:00
|
|
|
int status)
|
2006-01-11 04:40:13 +00:00
|
|
|
{
|
2019-06-29 02:31:38 +00:00
|
|
|
unsigned int nofs_flag;
|
|
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* We can allocate memory here while doing writeback on behalf of
|
|
|
|
|
* memory reclaim. To avoid memory allocation deadlocks set the
|
|
|
|
|
* task-wide nofs context for the following operations.
|
|
|
|
|
*/
|
|
|
|
|
nofs_flag = memalloc_nofs_save();
|
|
|
|
|
|
2017-02-02 23:14:02 +00:00
|
|
|
/* Convert CoW extents to regular */
|
2019-10-17 20:12:10 +00:00
|
|
|
if (!status && (ioend->io_flags & IOMAP_F_SHARED)) {
|
2017-02-02 23:14:02 +00:00
|
|
|
status = xfs_reflink_convert_cow(XFS_I(ioend->io_inode),
|
|
|
|
|
ioend->io_offset, ioend->io_size);
|
|
|
|
|
}
|
|
|
|
|
|
2019-06-29 02:31:38 +00:00
|
|
|
memalloc_nofs_restore(nofs_flag);
|
|
|
|
|
|
2021-04-09 17:27:55 +00:00
|
|
|
/* send ioends that might require a transaction to the completion wq */
|
|
|
|
|
if (xfs_ioend_is_append(ioend) || ioend->io_type == IOMAP_UNWRITTEN ||
|
|
|
|
|
(ioend->io_flags & IOMAP_F_SHARED))
|
2023-12-07 07:27:05 +00:00
|
|
|
ioend->io_bio.bi_end_io = xfs_end_bio;
|
2019-10-17 20:12:15 +00:00
|
|
|
return status;
|
2006-01-11 04:40:13 +00:00
|
|
|
}
|
|
|
|
|
|
2010-03-05 02:00:42 +00:00
|
|
|
/*
|
2023-03-05 23:13:23 +00:00
|
|
|
* If the folio has delalloc blocks on it, the caller is asking us to punch them
|
|
|
|
|
* out. If we don't, we can leave a stale delalloc mapping covered by a clean
|
|
|
|
|
* page that needs to be dirtied again before the delalloc mapping can be
|
|
|
|
|
* converted. This stale delalloc mapping can trip up a later direct I/O read
|
|
|
|
|
* operation on the same region.
|
2010-03-05 02:00:42 +00:00
|
|
|
*
|
2023-03-05 23:13:23 +00:00
|
|
|
* We prevent this by truncating away the delalloc regions on the folio. Because
|
2018-07-12 05:26:05 +00:00
|
|
|
* they are delalloc, we can do this without needing a transaction. Indeed - if
|
|
|
|
|
* we get ENOSPC errors, we have to be able to do this truncation without a
|
2023-03-05 23:13:23 +00:00
|
|
|
* transaction as there is no space left for block reservation (typically why
|
|
|
|
|
* we see a ENOSPC in writeback).
|
2010-03-05 02:00:42 +00:00
|
|
|
*/
|
2019-10-17 20:12:15 +00:00
|
|
|
static void
|
2021-07-30 13:56:05 +00:00
|
|
|
xfs_discard_folio(
|
|
|
|
|
struct folio *folio,
|
|
|
|
|
loff_t pos)
|
2010-03-05 02:00:42 +00:00
|
|
|
{
|
2022-11-28 22:09:17 +00:00
|
|
|
struct xfs_inode *ip = XFS_I(folio->mapping->host);
|
2018-07-12 05:25:57 +00:00
|
|
|
struct xfs_mount *mp = ip->i_mount;
|
2010-03-05 02:00:42 +00:00
|
|
|
|
2021-08-19 01:46:53 +00:00
|
|
|
if (xfs_is_shutdown(mp))
|
iomap: don't invalidate folios after writeback errors
XFS has the unique behavior (as compared to the other Linux filesystems)
that on writeback errors it will completely invalidate the affected
folio and force the page cache to reread the contents from disk. All
other filesystems leave the page mapped and up to date.
This is a rude awakening for user programs, since (in the case where
write fails but reread doesn't) file contents will appear to revert to
old disk contents with no notification other than an EIO on fsync. This
might have been annoying back in the days when iomap dealt with one page
at a time, but with multipage folios, we can now throw away *megabytes*
worth of data for a single write error.
On *most* Linux filesystems, a program can respond to an EIO on write by
redirtying the entire file and scheduling it for writeback. This isn't
foolproof, since the page that failed writeback is no longer dirty and
could be evicted, but programs that want to recover properly *also*
have to detect XFS and regenerate every write they've made to the file.
When running xfs/314 on arm64, I noticed a UAF when xfs_discard_folio
invalidates multipage folios that could be undergoing writeback. If,
say, we have a 256K folio caching a mix of written and unwritten
extents, it's possible that we could start writeback of the first (say)
64K of the folio and then hit a writeback error on the next 64K. We
then free the iop attached to the folio, which is really bad because
writeback completion on the first 64k will trip over the "blocks per
folio > 1 && !iop" assertion.
This can't be fixed by only invalidating the folio if writeback fails at
the start of the folio, since the folio is marked !uptodate, which trips
other assertions elsewhere. Get rid of the whole behavior entirely.
Signed-off-by: Darrick J. Wong <djwong@kernel.org>
Reviewed-by: Matthew Wilcox (Oracle) <willy@infradead.org>
Reviewed-by: Jeff Layton <jlayton@kernel.org>
Reviewed-by: Christoph Hellwig <hch@lst.de>
2022-05-16 22:27:38 +00:00
|
|
|
return;
|
2010-03-15 02:36:35 +00:00
|
|
|
|
2020-02-21 15:34:48 +00:00
|
|
|
xfs_alert_ratelimited(mp,
|
2021-07-30 13:56:05 +00:00
|
|
|
"page discard on page "PTR_FMT", inode 0x%llx, pos %llu.",
|
|
|
|
|
folio, ip->i_ino, pos);
|
2010-03-05 02:00:42 +00:00
|
|
|
|
2023-03-05 23:13:23 +00:00
|
|
|
/*
|
2023-08-16 10:37:14 +00:00
|
|
|
* The end of the punch range is always the offset of the first
|
2023-03-05 23:13:23 +00:00
|
|
|
* byte of the next folio. Hence the end offset is only dependent on the
|
|
|
|
|
* folio itself and not the start offset that is passed in.
|
|
|
|
|
*/
|
2024-04-30 12:58:22 +00:00
|
|
|
xfs_bmap_punch_delalloc_range(ip, pos,
|
2023-03-05 23:13:23 +00:00
|
|
|
folio_pos(folio) + folio_size(folio));
|
2010-03-05 02:00:42 +00:00
|
|
|
}
|
|
|
|
|
|
2019-10-17 20:12:15 +00:00
|
|
|
static const struct iomap_writeback_ops xfs_writeback_ops = {
|
|
|
|
|
.map_blocks = xfs_map_blocks,
|
|
|
|
|
.prepare_ioend = xfs_prepare_ioend,
|
2021-07-30 13:56:05 +00:00
|
|
|
.discard_folio = xfs_discard_folio,
|
2019-10-17 20:12:15 +00:00
|
|
|
};
|
2006-03-14 02:26:27 +00:00
|
|
|
|
2006-06-09 05:27:16 +00:00
|
|
|
STATIC int
|
|
|
|
|
xfs_vm_writepages(
|
|
|
|
|
struct address_space *mapping,
|
|
|
|
|
struct writeback_control *wbc)
|
|
|
|
|
{
|
2019-02-15 16:02:46 +00:00
|
|
|
struct xfs_writepage_ctx wpc = { };
|
2016-02-15 06:21:19 +00:00
|
|
|
|
2007-08-29 01:44:37 +00:00
|
|
|
xfs_iflags_clear(XFS_I(mapping->host), XFS_ITRUNCATED);
|
2019-10-17 20:12:15 +00:00
|
|
|
return iomap_writepages(mapping, wbc, &wpc.ctx, &xfs_writeback_ops);
|
2006-06-09 05:27:16 +00:00
|
|
|
}
|
|
|
|
|
|
2018-03-07 23:26:44 +00:00
|
|
|
STATIC int
|
|
|
|
|
xfs_dax_writepages(
|
|
|
|
|
struct address_space *mapping,
|
|
|
|
|
struct writeback_control *wbc)
|
|
|
|
|
{
|
2019-10-25 05:25:38 +00:00
|
|
|
struct xfs_inode *ip = XFS_I(mapping->host);
|
|
|
|
|
|
|
|
|
|
xfs_iflags_clear(ip, XFS_ITRUNCATED);
|
2018-03-07 23:26:44 +00:00
|
|
|
return dax_writeback_mapping_range(mapping,
|
2020-01-03 18:33:07 +00:00
|
|
|
xfs_inode_buftarg(ip)->bt_daxdev, wbc);
|
2018-03-07 23:26:44 +00:00
|
|
|
}
|
|
|
|
|
|
2005-04-16 22:20:36 +00:00
|
|
|
STATIC sector_t
|
2006-03-14 02:54:26 +00:00
|
|
|
xfs_vm_bmap(
|
2005-04-16 22:20:36 +00:00
|
|
|
struct address_space *mapping,
|
|
|
|
|
sector_t block)
|
|
|
|
|
{
|
2018-06-01 16:03:09 +00:00
|
|
|
struct xfs_inode *ip = XFS_I(mapping->host);
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2018-06-01 16:03:09 +00:00
|
|
|
trace_xfs_vm_bmap(ip);
|
2016-10-03 16:11:36 +00:00
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* The swap code (ab-)uses ->bmap to get a block mapping and then
|
2018-02-28 08:39:48 +00:00
|
|
|
* bypasses the file system for actual I/O. We really can't allow
|
2016-10-03 16:11:36 +00:00
|
|
|
* that on reflinks inodes, so we have to skip out here. And yes,
|
2017-06-22 03:27:35 +00:00
|
|
|
* 0 is the magic code for a bmap error.
|
|
|
|
|
*
|
|
|
|
|
* Since we don't pass back blockdev info, we can't return bmap
|
|
|
|
|
* information for rt files either.
|
2016-10-03 16:11:36 +00:00
|
|
|
*/
|
2019-02-18 17:38:49 +00:00
|
|
|
if (xfs_is_cow_inode(ip) || XFS_IS_REALTIME_INODE(ip))
|
2016-10-03 16:11:36 +00:00
|
|
|
return 0;
|
2019-10-19 16:09:45 +00:00
|
|
|
return iomap_bmap(mapping, block, &xfs_read_iomap_ops);
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
STATIC int
|
2022-04-29 12:54:32 +00:00
|
|
|
xfs_vm_read_folio(
|
2005-04-16 22:20:36 +00:00
|
|
|
struct file *unused,
|
2022-04-29 12:54:32 +00:00
|
|
|
struct folio *folio)
|
2005-04-16 22:20:36 +00:00
|
|
|
{
|
2022-04-29 12:54:32 +00:00
|
|
|
return iomap_read_folio(folio, &xfs_read_iomap_ops);
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
|
|
|
|
|
2020-06-02 04:47:34 +00:00
|
|
|
STATIC void
|
|
|
|
|
xfs_vm_readahead(
|
|
|
|
|
struct readahead_control *rac)
|
2005-04-16 22:20:36 +00:00
|
|
|
{
|
2020-06-02 04:47:34 +00:00
|
|
|
iomap_readahead(rac, &xfs_read_iomap_ops);
|
xfs: don't dirty buffers beyond EOF
generic/263 is failing fsx at this point with a page spanning
EOF that cannot be invalidated. The operations are:
1190 mapwrite 0x52c00 thru 0x5e569 (0xb96a bytes)
1191 mapread 0x5c000 thru 0x5d636 (0x1637 bytes)
1192 write 0x5b600 thru 0x771ff (0x1bc00 bytes)
where 1190 extents EOF from 0x54000 to 0x5e569. When the direct IO
write attempts to invalidate the cached page over this range, it
fails with -EBUSY and so any attempt to do page invalidation fails.
The real question is this: Why can't that page be invalidated after
it has been written to disk and cleaned?
Well, there's data on the first two buffers in the page (1k block
size, 4k page), but the third buffer on the page (i.e. beyond EOF)
is failing drop_buffers because it's bh->b_state == 0x3, which is
BH_Uptodate | BH_Dirty. IOWs, there's dirty buffers beyond EOF. Say
what?
OK, set_buffer_dirty() is called on all buffers from
__set_page_buffers_dirty(), regardless of whether the buffer is
beyond EOF or not, which means that when we get to ->writepage,
we have buffers marked dirty beyond EOF that we need to clean.
So, we need to implement our own .set_page_dirty method that
doesn't dirty buffers beyond EOF.
This is messy because the buffer code is not meant to be shared
and it has interesting locking issues on the buffer dirty bits.
So just copy and paste it and then modify it to suit what we need.
Note: the solutions the other filesystems and generic block code use
of marking the buffers clean in ->writepage does not work for XFS.
It still leaves dirty buffers beyond EOF and invalidations still
fail. Hence rather than play whack-a-mole, this patch simply
prevents those buffers from being dirtied in the first place.
cc: <stable@kernel.org>
Signed-off-by: Dave Chinner <dchinner@redhat.com>
Reviewed-by: Brian Foster <bfoster@redhat.com>
Signed-off-by: Dave Chinner <david@fromorbit.com>
2014-09-02 02:12:51 +00:00
|
|
|
}
|
|
|
|
|
|
2018-05-10 15:38:15 +00:00
|
|
|
static int
|
|
|
|
|
xfs_iomap_swapfile_activate(
|
|
|
|
|
struct swap_info_struct *sis,
|
|
|
|
|
struct file *swap_file,
|
|
|
|
|
sector_t *span)
|
|
|
|
|
{
|
2019-10-25 05:25:38 +00:00
|
|
|
sis->bdev = xfs_inode_buftarg(XFS_I(file_inode(swap_file)))->bt_bdev;
|
2019-10-19 16:09:45 +00:00
|
|
|
return iomap_swapfile_activate(sis, swap_file, span,
|
|
|
|
|
&xfs_read_iomap_ops);
|
2018-05-10 15:38:15 +00:00
|
|
|
}
|
|
|
|
|
|
2006-06-28 11:26:44 +00:00
|
|
|
const struct address_space_operations xfs_address_space_operations = {
|
2022-04-29 12:54:32 +00:00
|
|
|
.read_folio = xfs_vm_read_folio,
|
2020-06-02 04:47:34 +00:00
|
|
|
.readahead = xfs_vm_readahead,
|
2006-06-09 05:27:16 +00:00
|
|
|
.writepages = xfs_vm_writepages,
|
2023-07-10 21:12:43 +00:00
|
|
|
.dirty_folio = iomap_dirty_folio,
|
2022-05-01 03:01:08 +00:00
|
|
|
.release_folio = iomap_release_folio,
|
2022-02-09 20:21:33 +00:00
|
|
|
.invalidate_folio = iomap_invalidate_folio,
|
2006-03-14 02:54:26 +00:00
|
|
|
.bmap = xfs_vm_bmap,
|
2022-06-06 16:55:08 +00:00
|
|
|
.migrate_folio = filemap_migrate_folio,
|
2018-07-12 05:26:05 +00:00
|
|
|
.is_partially_uptodate = iomap_is_partially_uptodate,
|
2023-11-17 16:14:47 +00:00
|
|
|
.error_remove_folio = generic_error_remove_folio,
|
2018-05-10 15:38:15 +00:00
|
|
|
.swap_activate = xfs_iomap_swapfile_activate,
|
2005-04-16 22:20:36 +00:00
|
|
|
};
|
2018-03-07 23:26:44 +00:00
|
|
|
|
|
|
|
|
const struct address_space_operations xfs_dax_aops = {
|
|
|
|
|
.writepages = xfs_dax_writepages,
|
2022-02-09 20:22:13 +00:00
|
|
|
.dirty_folio = noop_dirty_folio,
|
2018-05-10 15:38:15 +00:00
|
|
|
.swap_activate = xfs_iomap_swapfile_activate,
|
2018-03-07 23:26:44 +00:00
|
|
|
};
|
