mirror of
https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git
synced 2026-08-09 06:14:34 +02:00
Fengnan ChangandChristian Brauner
bd6a1379a4
When iomap_iter() finishes its iteration (returns <= 0), it is no longer necessary to memset the entire iomap and srcmap structures. In high-IOPS scenarios (like 4k randread NVMe polling with io_uring), where the majority of I/Os complete in a single extent map, this wasted memory write bandwidth, as the caller will just discard the iterator. Use this command to test: taskset -c 30 ./t/io_uring -p1 -d512 -b4096 -s32 -c32 -F1 -B1 -R1 -X1 -n1 -P1 /mnt/testfile IOPS improve about 5% on ext4 and XFS. However, we MUST still call iomap_iter_reset_iomap() to release the folio_batch if IOMAP_F_FOLIO_BATCH is set, otherwise we leak page references. Therefore, split the cleanup logic: always release the folio_batch, but skip the memset() when ret <= 0. Signed-off-by: Fengnan Chang <changfengnan@bytedance.com> Link: https://patch.msgid.link/20260420061630.62077-1-changfengnan@bytedance.com Reviewed-by: Brian Foster <bfoster@redhat.com> Reviewed-by: Christoph Hellwig <hch@lst.de> Signed-off-by: Christian Brauner <brauner@kernel.org>
117 lines
3.3 KiB
C
117 lines
3.3 KiB
C
// SPDX-License-Identifier: GPL-2.0
|
|
/*
|
|
* Copyright (C) 2010 Red Hat, Inc.
|
|
* Copyright (c) 2016-2021 Christoph Hellwig.
|
|
*/
|
|
#include <linux/iomap.h>
|
|
#include "trace.h"
|
|
|
|
static inline void iomap_iter_clean_fbatch(struct iomap_iter *iter)
|
|
{
|
|
if (iter->iomap.flags & IOMAP_F_FOLIO_BATCH) {
|
|
folio_batch_release(iter->fbatch);
|
|
folio_batch_reinit(iter->fbatch);
|
|
iter->iomap.flags &= ~IOMAP_F_FOLIO_BATCH;
|
|
}
|
|
}
|
|
|
|
/* Advance the current iterator position and decrement the remaining length */
|
|
int iomap_iter_advance(struct iomap_iter *iter, u64 count)
|
|
{
|
|
if (WARN_ON_ONCE(count > iomap_length(iter)))
|
|
return -EIO;
|
|
iter->pos += count;
|
|
iter->len -= count;
|
|
return 0;
|
|
}
|
|
|
|
static inline void iomap_iter_done(struct iomap_iter *iter)
|
|
{
|
|
WARN_ON_ONCE(iter->iomap.offset > iter->pos);
|
|
WARN_ON_ONCE(iter->iomap.length == 0);
|
|
WARN_ON_ONCE(iter->iomap.offset + iter->iomap.length <= iter->pos);
|
|
WARN_ON_ONCE(iter->iomap.flags & IOMAP_F_STALE);
|
|
|
|
iter->iter_start_pos = iter->pos;
|
|
|
|
trace_iomap_iter_dstmap(iter->inode, &iter->iomap);
|
|
if (iter->srcmap.type != IOMAP_HOLE)
|
|
trace_iomap_iter_srcmap(iter->inode, &iter->srcmap);
|
|
}
|
|
|
|
/**
|
|
* iomap_iter - iterate over a ranges in a file
|
|
* @iter: iteration structue
|
|
* @ops: iomap ops provided by the file system
|
|
*
|
|
* Iterate over filesystem-provided space mappings for the provided file range.
|
|
*
|
|
* This function handles cleanup of resources acquired for iteration when the
|
|
* filesystem indicates there are no more space mappings, which means that this
|
|
* function must be called in a loop that continues as long it returns a
|
|
* positive value. If 0 or a negative value is returned, the caller must not
|
|
* return to the loop body. Within a loop body, there are two ways to break out
|
|
* of the loop body: leave @iter.status unchanged, or set it to a negative
|
|
* errno.
|
|
*/
|
|
int iomap_iter(struct iomap_iter *iter, const struct iomap_ops *ops)
|
|
{
|
|
bool stale = iter->iomap.flags & IOMAP_F_STALE;
|
|
ssize_t advanced;
|
|
u64 olen;
|
|
int ret;
|
|
|
|
trace_iomap_iter(iter, ops, _RET_IP_);
|
|
|
|
if (!iter->iomap.length)
|
|
goto begin;
|
|
|
|
/*
|
|
* Calculate how far the iter was advanced and the original length bytes
|
|
* for ->iomap_end().
|
|
*/
|
|
advanced = iter->pos - iter->iter_start_pos;
|
|
olen = iter->len + advanced;
|
|
|
|
if (ops->iomap_end) {
|
|
ret = ops->iomap_end(iter->inode, iter->iter_start_pos,
|
|
iomap_length_trim(iter, iter->iter_start_pos,
|
|
olen),
|
|
advanced, iter->flags, &iter->iomap);
|
|
if (ret < 0 && !advanced)
|
|
return ret;
|
|
}
|
|
|
|
/* detect old return semantics where this would advance */
|
|
if (WARN_ON_ONCE(iter->status > 0))
|
|
iter->status = -EIO;
|
|
|
|
/*
|
|
* Use iter->len to determine whether to continue onto the next mapping.
|
|
* Explicitly terminate on error status or if the current iter has not
|
|
* advanced at all (i.e. no work was done for some reason) unless the
|
|
* mapping has been marked stale and needs to be reprocessed.
|
|
*/
|
|
if (iter->status < 0)
|
|
ret = iter->status;
|
|
else if (iter->len == 0 || (!advanced && !stale))
|
|
ret = 0;
|
|
else
|
|
ret = 1;
|
|
iomap_iter_clean_fbatch(iter);
|
|
iter->status = 0;
|
|
if (ret <= 0)
|
|
return ret;
|
|
|
|
memset(&iter->iomap, 0, sizeof(iter->iomap));
|
|
memset(&iter->srcmap, 0, sizeof(iter->srcmap));
|
|
|
|
begin:
|
|
ret = ops->iomap_begin(iter->inode, iter->pos, iter->len, iter->flags,
|
|
&iter->iomap, &iter->srcmap);
|
|
if (ret < 0)
|
|
return ret;
|
|
iomap_iter_done(iter);
|
|
return 1;
|
|
}
|