diff options
Diffstat (limited to 'lib/iov_iter.c')
| -rw-r--r-- | lib/iov_iter.c | 2610 |
1 files changed, 1277 insertions, 1333 deletions
diff --git a/lib/iov_iter.c b/lib/iov_iter.c index f99c41d4eb54..cf2eb2b2f983 100644 --- a/lib/iov_iter.c +++ b/lib/iov_iter.c @@ -1,690 +1,224 @@ // SPDX-License-Identifier: GPL-2.0-only #include <linux/export.h> #include <linux/bvec.h> +#include <linux/fault-inject-usercopy.h> #include <linux/uio.h> #include <linux/pagemap.h> +#include <linux/highmem.h> #include <linux/slab.h> #include <linux/vmalloc.h> #include <linux/splice.h> -#include <net/checksum.h> +#include <linux/compat.h> #include <linux/scatterlist.h> +#include <linux/instrumented.h> +#include <linux/iov_iter.h> -#define PIPE_PARANOIA /* for now */ - -#define iterate_iovec(i, n, __v, __p, skip, STEP) { \ - size_t left; \ - size_t wanted = n; \ - __p = i->iov; \ - __v.iov_len = min(n, __p->iov_len - skip); \ - if (likely(__v.iov_len)) { \ - __v.iov_base = __p->iov_base + skip; \ - left = (STEP); \ - __v.iov_len -= left; \ - skip += __v.iov_len; \ - n -= __v.iov_len; \ - } else { \ - left = 0; \ - } \ - while (unlikely(!left && n)) { \ - __p++; \ - __v.iov_len = min(n, __p->iov_len); \ - if (unlikely(!__v.iov_len)) \ - continue; \ - __v.iov_base = __p->iov_base; \ - left = (STEP); \ - __v.iov_len -= left; \ - skip = __v.iov_len; \ - n -= __v.iov_len; \ - } \ - n = wanted - n; \ -} - -#define iterate_kvec(i, n, __v, __p, skip, STEP) { \ - size_t wanted = n; \ - __p = i->kvec; \ - __v.iov_len = min(n, __p->iov_len - skip); \ - if (likely(__v.iov_len)) { \ - __v.iov_base = __p->iov_base + skip; \ - (void)(STEP); \ - skip += __v.iov_len; \ - n -= __v.iov_len; \ - } \ - while (unlikely(n)) { \ - __p++; \ - __v.iov_len = min(n, __p->iov_len); \ - if (unlikely(!__v.iov_len)) \ - continue; \ - __v.iov_base = __p->iov_base; \ - (void)(STEP); \ - skip = __v.iov_len; \ - n -= __v.iov_len; \ - } \ - n = wanted; \ -} - -#define iterate_bvec(i, n, __v, __bi, skip, STEP) { \ - struct bvec_iter __start; \ - __start.bi_size = n; \ - __start.bi_bvec_done = skip; \ - __start.bi_idx = 0; \ - for_each_bvec(__v, i->bvec, __bi, __start) { \ - if (!__v.bv_len) \ - continue; \ - (void)(STEP); \ - } \ -} - -#define iterate_all_kinds(i, n, v, I, B, K) { \ - if (likely(n)) { \ - size_t skip = i->iov_offset; \ - if (unlikely(i->type & ITER_BVEC)) { \ - struct bio_vec v; \ - struct bvec_iter __bi; \ - iterate_bvec(i, n, v, __bi, skip, (B)) \ - } else if (unlikely(i->type & ITER_KVEC)) { \ - const struct kvec *kvec; \ - struct kvec v; \ - iterate_kvec(i, n, v, kvec, skip, (K)) \ - } else if (unlikely(i->type & ITER_DISCARD)) { \ - } else { \ - const struct iovec *iov; \ - struct iovec v; \ - iterate_iovec(i, n, v, iov, skip, (I)) \ - } \ - } \ -} - -#define iterate_and_advance(i, n, v, I, B, K) { \ - if (unlikely(i->count < n)) \ - n = i->count; \ - if (i->count) { \ - size_t skip = i->iov_offset; \ - if (unlikely(i->type & ITER_BVEC)) { \ - const struct bio_vec *bvec = i->bvec; \ - struct bio_vec v; \ - struct bvec_iter __bi; \ - iterate_bvec(i, n, v, __bi, skip, (B)) \ - i->bvec = __bvec_iter_bvec(i->bvec, __bi); \ - i->nr_segs -= i->bvec - bvec; \ - skip = __bi.bi_bvec_done; \ - } else if (unlikely(i->type & ITER_KVEC)) { \ - const struct kvec *kvec; \ - struct kvec v; \ - iterate_kvec(i, n, v, kvec, skip, (K)) \ - if (skip == kvec->iov_len) { \ - kvec++; \ - skip = 0; \ - } \ - i->nr_segs -= kvec - i->kvec; \ - i->kvec = kvec; \ - } else if (unlikely(i->type & ITER_DISCARD)) { \ - skip += n; \ - } else { \ - const struct iovec *iov; \ - struct iovec v; \ - iterate_iovec(i, n, v, iov, skip, (I)) \ - if (skip == iov->iov_len) { \ - iov++; \ - skip = 0; \ - } \ - i->nr_segs -= iov - i->iov; \ - i->iov = iov; \ - } \ - i->count -= n; \ - i->iov_offset = skip; \ - } \ -} - -static int copyout(void __user *to, const void *from, size_t n) -{ - if (access_ok(to, n)) { - kasan_check_read(from, n); - n = raw_copy_to_user(to, from, n); - } - return n; -} - -static int copyin(void *to, const void __user *from, size_t n) +static __always_inline +size_t copy_to_user_iter(void __user *iter_to, size_t progress, + size_t len, void *from, void *priv2) { - if (access_ok(from, n)) { - kasan_check_write(to, n); - n = raw_copy_from_user(to, from, n); + if (should_fail_usercopy()) + return len; + if (access_ok(iter_to, len)) { + from += progress; + instrument_copy_to_user(iter_to, from, len); + len = raw_copy_to_user(iter_to, from, len); } - return n; + return len; } -static size_t copy_page_to_iter_iovec(struct page *page, size_t offset, size_t bytes, - struct iov_iter *i) +static __always_inline +size_t copy_to_user_iter_nofault(void __user *iter_to, size_t progress, + size_t len, void *from, void *priv2) { - size_t skip, copy, left, wanted; - const struct iovec *iov; - char __user *buf; - void *kaddr, *from; + ssize_t res; - if (unlikely(bytes > i->count)) - bytes = i->count; - - if (unlikely(!bytes)) - return 0; - - might_fault(); - wanted = bytes; - iov = i->iov; - skip = i->iov_offset; - buf = iov->iov_base + skip; - copy = min(bytes, iov->iov_len - skip); - - if (IS_ENABLED(CONFIG_HIGHMEM) && !fault_in_pages_writeable(buf, copy)) { - kaddr = kmap_atomic(page); - from = kaddr + offset; - - /* first chunk, usually the only one */ - left = copyout(buf, from, copy); - copy -= left; - skip += copy; - from += copy; - bytes -= copy; - - while (unlikely(!left && bytes)) { - iov++; - buf = iov->iov_base; - copy = min(bytes, iov->iov_len); - left = copyout(buf, from, copy); - copy -= left; - skip = copy; - from += copy; - bytes -= copy; - } - if (likely(!bytes)) { - kunmap_atomic(kaddr); - goto done; - } - offset = from - kaddr; - buf += copy; - kunmap_atomic(kaddr); - copy = min(bytes, iov->iov_len - skip); - } - /* Too bad - revert to non-atomic kmap */ - - kaddr = kmap(page); - from = kaddr + offset; - left = copyout(buf, from, copy); - copy -= left; - skip += copy; - from += copy; - bytes -= copy; - while (unlikely(!left && bytes)) { - iov++; - buf = iov->iov_base; - copy = min(bytes, iov->iov_len); - left = copyout(buf, from, copy); - copy -= left; - skip = copy; - from += copy; - bytes -= copy; - } - kunmap(page); + if (should_fail_usercopy()) + return len; -done: - if (skip == iov->iov_len) { - iov++; - skip = 0; - } - i->count -= wanted - bytes; - i->nr_segs -= iov - i->iov; - i->iov = iov; - i->iov_offset = skip; - return wanted - bytes; + from += progress; + res = copy_to_user_nofault(iter_to, from, len); + return res < 0 ? len : res; } -static size_t copy_page_from_iter_iovec(struct page *page, size_t offset, size_t bytes, - struct iov_iter *i) +static __always_inline +size_t copy_from_user_iter(void __user *iter_from, size_t progress, + size_t len, void *to, void *priv2) { - size_t skip, copy, left, wanted; - const struct iovec *iov; - char __user *buf; - void *kaddr, *to; - - if (unlikely(bytes > i->count)) - bytes = i->count; - - if (unlikely(!bytes)) - return 0; - - might_fault(); - wanted = bytes; - iov = i->iov; - skip = i->iov_offset; - buf = iov->iov_base + skip; - copy = min(bytes, iov->iov_len - skip); - - if (IS_ENABLED(CONFIG_HIGHMEM) && !fault_in_pages_readable(buf, copy)) { - kaddr = kmap_atomic(page); - to = kaddr + offset; - - /* first chunk, usually the only one */ - left = copyin(to, buf, copy); - copy -= left; - skip += copy; - to += copy; - bytes -= copy; - - while (unlikely(!left && bytes)) { - iov++; - buf = iov->iov_base; - copy = min(bytes, iov->iov_len); - left = copyin(to, buf, copy); - copy -= left; - skip = copy; - to += copy; - bytes -= copy; - } - if (likely(!bytes)) { - kunmap_atomic(kaddr); - goto done; - } - offset = to - kaddr; - buf += copy; - kunmap_atomic(kaddr); - copy = min(bytes, iov->iov_len - skip); + size_t res = len; + + if (should_fail_usercopy()) + return len; + if (access_ok(iter_from, len)) { + to += progress; + instrument_copy_from_user_before(to, iter_from, len); + res = raw_copy_from_user(to, iter_from, len); + instrument_copy_from_user_after(to, iter_from, len, res); } - /* Too bad - revert to non-atomic kmap */ - - kaddr = kmap(page); - to = kaddr + offset; - left = copyin(to, buf, copy); - copy -= left; - skip += copy; - to += copy; - bytes -= copy; - while (unlikely(!left && bytes)) { - iov++; - buf = iov->iov_base; - copy = min(bytes, iov->iov_len); - left = copyin(to, buf, copy); - copy -= left; - skip = copy; - to += copy; - bytes -= copy; - } - kunmap(page); - -done: - if (skip == iov->iov_len) { - iov++; - skip = 0; - } - i->count -= wanted - bytes; - i->nr_segs -= iov - i->iov; - i->iov = iov; - i->iov_offset = skip; - return wanted - bytes; -} - -#ifdef PIPE_PARANOIA -static bool sanity(const struct iov_iter *i) -{ - struct pipe_inode_info *pipe = i->pipe; - int idx = i->idx; - int next = pipe->curbuf + pipe->nrbufs; - if (i->iov_offset) { - struct pipe_buffer *p; - if (unlikely(!pipe->nrbufs)) - goto Bad; // pipe must be non-empty - if (unlikely(idx != ((next - 1) & (pipe->buffers - 1)))) - goto Bad; // must be at the last buffer... - - p = &pipe->bufs[idx]; - if (unlikely(p->offset + p->len != i->iov_offset)) - goto Bad; // ... at the end of segment - } else { - if (idx != (next & (pipe->buffers - 1))) - goto Bad; // must be right after the last buffer - } - return true; -Bad: - printk(KERN_ERR "idx = %d, offset = %zd\n", i->idx, i->iov_offset); - printk(KERN_ERR "curbuf = %d, nrbufs = %d, buffers = %d\n", - pipe->curbuf, pipe->nrbufs, pipe->buffers); - for (idx = 0; idx < pipe->buffers; idx++) - printk(KERN_ERR "[%p %p %d %d]\n", - pipe->bufs[idx].ops, - pipe->bufs[idx].page, - pipe->bufs[idx].offset, - pipe->bufs[idx].len); - WARN_ON(1); - return false; -} -#else -#define sanity(i) true -#endif + return res; +} -static inline int next_idx(int idx, struct pipe_inode_info *pipe) +static __always_inline +size_t memcpy_to_iter(void *iter_to, size_t progress, + size_t len, void *from, void *priv2) { - return (idx + 1) & (pipe->buffers - 1); + memcpy(iter_to, from + progress, len); + return 0; } -static size_t copy_page_to_iter_pipe(struct page *page, size_t offset, size_t bytes, - struct iov_iter *i) +static __always_inline +size_t memcpy_from_iter(void *iter_from, size_t progress, + size_t len, void *to, void *priv2) { - struct pipe_inode_info *pipe = i->pipe; - struct pipe_buffer *buf; - size_t off; - int idx; - - if (unlikely(bytes > i->count)) - bytes = i->count; - - if (unlikely(!bytes)) - return 0; - - if (!sanity(i)) - return 0; + memcpy(to + progress, iter_from, len); + return 0; +} - off = i->iov_offset; - idx = i->idx; - buf = &pipe->bufs[idx]; - if (off) { - if (offset == off && buf->page == page) { - /* merge with the last one */ - buf->len += bytes; - i->iov_offset += bytes; - goto out; +/* + * fault_in_iov_iter_readable - fault in iov iterator for reading + * @i: iterator + * @size: maximum length + * + * Fault in one or more iovecs of the given iov_iter, to a maximum length of + * @size. For each iovec, fault in each page that constitutes the iovec. + * + * Returns the number of bytes not faulted in (like copy_to_user() and + * copy_from_user()). + * + * Always returns 0 for non-userspace iterators. + */ +size_t fault_in_iov_iter_readable(const struct iov_iter *i, size_t size) +{ + if (iter_is_ubuf(i)) { + size_t n = min(size, iov_iter_count(i)); + n -= fault_in_readable(i->ubuf + i->iov_offset, n); + return size - n; + } else if (iter_is_iovec(i)) { + size_t count = min(size, iov_iter_count(i)); + const struct iovec *p; + size_t skip; + + size -= count; + for (p = iter_iov(i), skip = i->iov_offset; count; p++, skip = 0) { + size_t len = min(count, p->iov_len - skip); + size_t ret; + + if (unlikely(!len)) + continue; + ret = fault_in_readable(p->iov_base + skip, len); + count -= len - ret; + if (ret) + break; } - idx = next_idx(idx, pipe); - buf = &pipe->bufs[idx]; + return count + size; } - if (idx == pipe->curbuf && pipe->nrbufs) - return 0; - pipe->nrbufs++; - buf->ops = &page_cache_pipe_buf_ops; - get_page(buf->page = page); - buf->offset = offset; - buf->len = bytes; - i->iov_offset = offset + bytes; - i->idx = idx; -out: - i->count -= bytes; - return bytes; + return 0; } +EXPORT_SYMBOL(fault_in_iov_iter_readable); /* - * Fault in one or more iovecs of the given iov_iter, to a maximum length of - * bytes. For each iovec, fault in each page that constitutes the iovec. + * fault_in_iov_iter_writeable - fault in iov iterator for writing + * @i: iterator + * @size: maximum length * - * Return 0 on success, or non-zero if the memory could not be accessed (i.e. - * because it is an invalid address). + * Faults in the iterator using get_user_pages(), i.e., without triggering + * hardware page faults. This is primarily useful when we already know that + * some or all of the pages in @i aren't in memory. + * + * Returns the number of bytes not faulted in, like copy_to_user() and + * copy_from_user(). + * + * Always returns 0 for non-user-space iterators. */ -int iov_iter_fault_in_readable(struct iov_iter *i, size_t bytes) +size_t fault_in_iov_iter_writeable(const struct iov_iter *i, size_t size) { - size_t skip = i->iov_offset; - const struct iovec *iov; - int err; - struct iovec v; - - if (!(i->type & (ITER_BVEC|ITER_KVEC))) { - iterate_iovec(i, bytes, v, iov, skip, ({ - err = fault_in_pages_readable(v.iov_base, v.iov_len); - if (unlikely(err)) - return err; - 0;})) + if (iter_is_ubuf(i)) { + size_t n = min(size, iov_iter_count(i)); + n -= fault_in_safe_writeable(i->ubuf + i->iov_offset, n); + return size - n; + } else if (iter_is_iovec(i)) { + size_t count = min(size, iov_iter_count(i)); + const struct iovec *p; + size_t skip; + + size -= count; + for (p = iter_iov(i), skip = i->iov_offset; count; p++, skip = 0) { + size_t len = min(count, p->iov_len - skip); + size_t ret; + + if (unlikely(!len)) + continue; + ret = fault_in_safe_writeable(p->iov_base + skip, len); + count -= len - ret; + if (ret) + break; + } + return count + size; } return 0; } -EXPORT_SYMBOL(iov_iter_fault_in_readable); +EXPORT_SYMBOL(fault_in_iov_iter_writeable); void iov_iter_init(struct iov_iter *i, unsigned int direction, const struct iovec *iov, unsigned long nr_segs, size_t count) { WARN_ON(direction & ~(READ | WRITE)); - direction &= READ | WRITE; - - /* It will get better. Eventually... */ - if (uaccess_kernel()) { - i->type = ITER_KVEC | direction; - i->kvec = (struct kvec *)iov; - } else { - i->type = ITER_IOVEC | direction; - i->iov = iov; - } - i->nr_segs = nr_segs; - i->iov_offset = 0; - i->count = count; + *i = (struct iov_iter) { + .iter_type = ITER_IOVEC, + .nofault = false, + .data_source = direction, + .__iov = iov, + .nr_segs = nr_segs, + .iov_offset = 0, + .count = count + }; } EXPORT_SYMBOL(iov_iter_init); -static void memcpy_from_page(char *to, struct page *page, size_t offset, size_t len) -{ - char *from = kmap_atomic(page); - memcpy(to, from + offset, len); - kunmap_atomic(from); -} - -static void memcpy_to_page(struct page *page, size_t offset, const char *from, size_t len) -{ - char *to = kmap_atomic(page); - memcpy(to + offset, from, len); - kunmap_atomic(to); -} - -static void memzero_page(struct page *page, size_t offset, size_t len) -{ - char *addr = kmap_atomic(page); - memset(addr + offset, 0, len); - kunmap_atomic(addr); -} - -static inline bool allocated(struct pipe_buffer *buf) -{ - return buf->ops == &default_pipe_buf_ops; -} - -static inline void data_start(const struct iov_iter *i, int *idxp, size_t *offp) -{ - size_t off = i->iov_offset; - int idx = i->idx; - if (off && (!allocated(&i->pipe->bufs[idx]) || off == PAGE_SIZE)) { - idx = next_idx(idx, i->pipe); - off = 0; - } - *idxp = idx; - *offp = off; -} - -static size_t push_pipe(struct iov_iter *i, size_t size, - int *idxp, size_t *offp) -{ - struct pipe_inode_info *pipe = i->pipe; - size_t off; - int idx; - ssize_t left; - - if (unlikely(size > i->count)) - size = i->count; - if (unlikely(!size)) - return 0; - - left = size; - data_start(i, &idx, &off); - *idxp = idx; - *offp = off; - if (off) { - left -= PAGE_SIZE - off; - if (left <= 0) { - pipe->bufs[idx].len += size; - return size; - } - pipe->bufs[idx].len = PAGE_SIZE; - idx = next_idx(idx, pipe); - } - while (idx != pipe->curbuf || !pipe->nrbufs) { - struct page *page = alloc_page(GFP_USER); - if (!page) - break; - pipe->nrbufs++; - pipe->bufs[idx].ops = &default_pipe_buf_ops; - pipe->bufs[idx].page = page; - pipe->bufs[idx].offset = 0; - if (left <= PAGE_SIZE) { - pipe->bufs[idx].len = left; - return size; - } - pipe->bufs[idx].len = PAGE_SIZE; - left -= PAGE_SIZE; - idx = next_idx(idx, pipe); - } - return size - left; -} - -static size_t copy_pipe_to_iter(const void *addr, size_t bytes, - struct iov_iter *i) -{ - struct pipe_inode_info *pipe = i->pipe; - size_t n, off; - int idx; - - if (!sanity(i)) - return 0; - - bytes = n = push_pipe(i, bytes, &idx, &off); - if (unlikely(!n)) - return 0; - for ( ; n; idx = next_idx(idx, pipe), off = 0) { - size_t chunk = min_t(size_t, n, PAGE_SIZE - off); - memcpy_to_page(pipe->bufs[idx].page, off, addr, chunk); - i->idx = idx; - i->iov_offset = off + chunk; - n -= chunk; - addr += chunk; - } - i->count -= bytes; - return bytes; -} - -static __wsum csum_and_memcpy(void *to, const void *from, size_t len, - __wsum sum, size_t off) -{ - __wsum next = csum_partial_copy_nocheck(from, to, len, 0); - return csum_block_add(sum, next, off); -} - -static size_t csum_and_copy_to_pipe_iter(const void *addr, size_t bytes, - __wsum *csum, struct iov_iter *i) -{ - struct pipe_inode_info *pipe = i->pipe; - size_t n, r; - size_t off = 0; - __wsum sum = *csum; - int idx; - - if (!sanity(i)) - return 0; - - bytes = n = push_pipe(i, bytes, &idx, &r); - if (unlikely(!n)) - return 0; - for ( ; n; idx = next_idx(idx, pipe), r = 0) { - size_t chunk = min_t(size_t, n, PAGE_SIZE - r); - char *p = kmap_atomic(pipe->bufs[idx].page); - sum = csum_and_memcpy(p + r, addr, chunk, sum, off); - kunmap_atomic(p); - i->idx = idx; - i->iov_offset = r + chunk; - n -= chunk; - off += chunk; - addr += chunk; - } - i->count -= bytes; - *csum = sum; - return bytes; -} - size_t _copy_to_iter(const void *addr, size_t bytes, struct iov_iter *i) { - const char *from = addr; - if (unlikely(iov_iter_is_pipe(i))) - return copy_pipe_to_iter(addr, bytes, i); - if (iter_is_iovec(i)) + if (WARN_ON_ONCE(i->data_source)) + return 0; + if (user_backed_iter(i)) might_fault(); - iterate_and_advance(i, bytes, v, - copyout(v.iov_base, (from += v.iov_len) - v.iov_len, v.iov_len), - memcpy_to_page(v.bv_page, v.bv_offset, - (from += v.bv_len) - v.bv_len, v.bv_len), - memcpy(v.iov_base, (from += v.iov_len) - v.iov_len, v.iov_len) - ) - - return bytes; + return iterate_and_advance(i, bytes, (void *)addr, + copy_to_user_iter, memcpy_to_iter); } EXPORT_SYMBOL(_copy_to_iter); -#ifdef CONFIG_ARCH_HAS_UACCESS_MCSAFE -static int copyout_mcsafe(void __user *to, const void *from, size_t n) +#ifdef CONFIG_ARCH_HAS_COPY_MC +static __always_inline +size_t copy_to_user_iter_mc(void __user *iter_to, size_t progress, + size_t len, void *from, void *priv2) { - if (access_ok(to, n)) { - kasan_check_read(from, n); - n = copy_to_user_mcsafe((__force void *) to, from, n); + if (access_ok(iter_to, len)) { + from += progress; + instrument_copy_to_user(iter_to, from, len); + len = copy_mc_to_user(iter_to, from, len); } - return n; + return len; } -static unsigned long memcpy_mcsafe_to_page(struct page *page, size_t offset, - const char *from, size_t len) +static __always_inline +size_t memcpy_to_iter_mc(void *iter_to, size_t progress, + size_t len, void *from, void *priv2) { - unsigned long ret; - char *to; - - to = kmap_atomic(page); - ret = memcpy_mcsafe(to + offset, from, len); - kunmap_atomic(to); - - return ret; -} - -static size_t copy_pipe_to_iter_mcsafe(const void *addr, size_t bytes, - struct iov_iter *i) -{ - struct pipe_inode_info *pipe = i->pipe; - size_t n, off, xfer = 0; - int idx; - - if (!sanity(i)) - return 0; - - bytes = n = push_pipe(i, bytes, &idx, &off); - if (unlikely(!n)) - return 0; - for ( ; n; idx = next_idx(idx, pipe), off = 0) { - size_t chunk = min_t(size_t, n, PAGE_SIZE - off); - unsigned long rem; - - rem = memcpy_mcsafe_to_page(pipe->bufs[idx].page, off, addr, - chunk); - i->idx = idx; - i->iov_offset = off + chunk - rem; - xfer += chunk - rem; - if (rem) - break; - n -= chunk; - addr += chunk; - } - i->count -= xfer; - return xfer; + return copy_mc_to_kernel(iter_to, from + progress, len); } /** - * _copy_to_iter_mcsafe - copy to user with source-read error exception handling + * _copy_mc_to_iter - copy to iter with source memory error exception handling * @addr: source kernel address * @bytes: total transfer length - * @iter: destination iterator + * @i: destination iterator * - * The pmem driver arranges for filesystem-dax to use this facility via - * dax_copy_to_iter() for protecting read/write to persistent memory. - * Unless / until an architecture can guarantee identical performance - * between _copy_to_iter_mcsafe() and _copy_to_iter() it would be a - * performance regression to switch more users to the mcsafe version. + * The pmem driver deploys this for the dax operation + * (dax_copy_to_iter()) for dax reads (bypass page-cache and the + * block-layer). Upon #MC read(2) aborts and returns EIO or the bytes + * successfully copied. * - * Otherwise, the main differences between this and typical _copy_to_iter(). + * The main differences between this and typical _copy_to_iter(). * * * Typical tail/residue handling after a fault retries the copy * byte-by-byte until the fault happens again. Re-triggering machine @@ -692,120 +226,80 @@ static size_t copy_pipe_to_iter_mcsafe(const void *addr, size_t bytes, * alignment and poison alignment assumptions to avoid re-triggering * hardware exceptions. * - * * ITER_KVEC, ITER_PIPE, and ITER_BVEC can return short copies. - * Compare to copy_to_iter() where only ITER_IOVEC attempts might return - * a short copy. + * * ITER_KVEC and ITER_BVEC can return short copies. Compare to + * copy_to_iter() where only ITER_IOVEC attempts might return a short copy. * - * See MCSAFE_TEST for self-test. + * Return: number of bytes copied (may be %0) */ -size_t _copy_to_iter_mcsafe(const void *addr, size_t bytes, struct iov_iter *i) +size_t _copy_mc_to_iter(const void *addr, size_t bytes, struct iov_iter *i) { - const char *from = addr; - unsigned long rem, curr_addr, s_addr = (unsigned long) addr; - - if (unlikely(iov_iter_is_pipe(i))) - return copy_pipe_to_iter_mcsafe(addr, bytes, i); - if (iter_is_iovec(i)) + if (WARN_ON_ONCE(i->data_source)) + return 0; + if (user_backed_iter(i)) might_fault(); - iterate_and_advance(i, bytes, v, - copyout_mcsafe(v.iov_base, (from += v.iov_len) - v.iov_len, v.iov_len), - ({ - rem = memcpy_mcsafe_to_page(v.bv_page, v.bv_offset, - (from += v.bv_len) - v.bv_len, v.bv_len); - if (rem) { - curr_addr = (unsigned long) from; - bytes = curr_addr - s_addr - rem; - return bytes; - } - }), - ({ - rem = memcpy_mcsafe(v.iov_base, (from += v.iov_len) - v.iov_len, - v.iov_len); - if (rem) { - curr_addr = (unsigned long) from; - bytes = curr_addr - s_addr - rem; - return bytes; - } - }) - ) + return iterate_and_advance(i, bytes, (void *)addr, + copy_to_user_iter_mc, memcpy_to_iter_mc); +} +EXPORT_SYMBOL_GPL(_copy_mc_to_iter); +#endif /* CONFIG_ARCH_HAS_COPY_MC */ - return bytes; +static __always_inline +size_t __copy_from_iter(void *addr, size_t bytes, struct iov_iter *i) +{ + return iterate_and_advance(i, bytes, addr, + copy_from_user_iter, memcpy_from_iter); } -EXPORT_SYMBOL_GPL(_copy_to_iter_mcsafe); -#endif /* CONFIG_ARCH_HAS_UACCESS_MCSAFE */ size_t _copy_from_iter(void *addr, size_t bytes, struct iov_iter *i) { - char *to = addr; - if (unlikely(iov_iter_is_pipe(i))) { - WARN_ON(1); + if (WARN_ON_ONCE(!i->data_source)) return 0; - } - if (iter_is_iovec(i)) - might_fault(); - iterate_and_advance(i, bytes, v, - copyin((to += v.iov_len) - v.iov_len, v.iov_base, v.iov_len), - memcpy_from_page((to += v.bv_len) - v.bv_len, v.bv_page, - v.bv_offset, v.bv_len), - memcpy((to += v.iov_len) - v.iov_len, v.iov_base, v.iov_len) - ) - return bytes; + if (user_backed_iter(i)) + might_fault(); + return __copy_from_iter(addr, bytes, i); } EXPORT_SYMBOL(_copy_from_iter); -bool _copy_from_iter_full(void *addr, size_t bytes, struct iov_iter *i) +static __always_inline +size_t copy_from_user_iter_nocache(void __user *iter_from, size_t progress, + size_t len, void *to, void *priv2) { - char *to = addr; - if (unlikely(iov_iter_is_pipe(i))) { - WARN_ON(1); - return false; - } - if (unlikely(i->count < bytes)) - return false; - - if (iter_is_iovec(i)) - might_fault(); - iterate_all_kinds(i, bytes, v, ({ - if (copyin((to += v.iov_len) - v.iov_len, - v.iov_base, v.iov_len)) - return false; - 0;}), - memcpy_from_page((to += v.bv_len) - v.bv_len, v.bv_page, - v.bv_offset, v.bv_len), - memcpy((to += v.iov_len) - v.iov_len, v.iov_base, v.iov_len) - ) - - iov_iter_advance(i, bytes); - return true; + return __copy_from_user_inatomic_nocache(to + progress, iter_from, len); } -EXPORT_SYMBOL(_copy_from_iter_full); size_t _copy_from_iter_nocache(void *addr, size_t bytes, struct iov_iter *i) { - char *to = addr; - if (unlikely(iov_iter_is_pipe(i))) { - WARN_ON(1); + if (WARN_ON_ONCE(!i->data_source)) return 0; - } - iterate_and_advance(i, bytes, v, - __copy_from_user_inatomic_nocache((to += v.iov_len) - v.iov_len, - v.iov_base, v.iov_len), - memcpy_from_page((to += v.bv_len) - v.bv_len, v.bv_page, - v.bv_offset, v.bv_len), - memcpy((to += v.iov_len) - v.iov_len, v.iov_base, v.iov_len) - ) - return bytes; + return iterate_and_advance(i, bytes, addr, + copy_from_user_iter_nocache, + memcpy_from_iter); } EXPORT_SYMBOL(_copy_from_iter_nocache); #ifdef CONFIG_ARCH_HAS_UACCESS_FLUSHCACHE +static __always_inline +size_t copy_from_user_iter_flushcache(void __user *iter_from, size_t progress, + size_t len, void *to, void *priv2) +{ + return __copy_from_user_flushcache(to + progress, iter_from, len); +} + +static __always_inline +size_t memcpy_from_iter_flushcache(void *iter_from, size_t progress, + size_t len, void *to, void *priv2) +{ + memcpy_flushcache(to + progress, iter_from, len); + return 0; +} + /** * _copy_from_iter_flushcache - write destination through cpu cache * @addr: destination kernel address * @bytes: total transfer length - * @iter: source iterator + * @i: source iterator * * The pmem driver arranges for filesystem-dax to use this facility via * dax_copy_from_iter() for ensuring that writes to persistent memory @@ -814,52 +308,21 @@ EXPORT_SYMBOL(_copy_from_iter_nocache); * all iterator types. The _copy_from_iter_nocache() only attempts to * bypass the cache for the ITER_IOVEC case, and on some archs may use * instructions that strand dirty-data in the cache. + * + * Return: number of bytes copied (may be %0) */ size_t _copy_from_iter_flushcache(void *addr, size_t bytes, struct iov_iter *i) { - char *to = addr; - if (unlikely(iov_iter_is_pipe(i))) { - WARN_ON(1); + if (WARN_ON_ONCE(!i->data_source)) return 0; - } - iterate_and_advance(i, bytes, v, - __copy_from_user_flushcache((to += v.iov_len) - v.iov_len, - v.iov_base, v.iov_len), - memcpy_page_flushcache((to += v.bv_len) - v.bv_len, v.bv_page, - v.bv_offset, v.bv_len), - memcpy_flushcache((to += v.iov_len) - v.iov_len, v.iov_base, - v.iov_len) - ) - return bytes; + return iterate_and_advance(i, bytes, addr, + copy_from_user_iter_flushcache, + memcpy_from_iter_flushcache); } EXPORT_SYMBOL_GPL(_copy_from_iter_flushcache); #endif -bool _copy_from_iter_full_nocache(void *addr, size_t bytes, struct iov_iter *i) -{ - char *to = addr; - if (unlikely(iov_iter_is_pipe(i))) { - WARN_ON(1); - return false; - } - if (unlikely(i->count < bytes)) - return false; - iterate_all_kinds(i, bytes, v, ({ - if (__copy_from_user_inatomic_nocache((to += v.iov_len) - v.iov_len, - v.iov_base, v.iov_len)) - return false; - 0;}), - memcpy_from_page((to += v.bv_len) - v.bv_len, v.bv_page, - v.bv_offset, v.bv_len), - memcpy((to += v.iov_len) - v.iov_len, v.iov_base, v.iov_len) - ) - - iov_iter_advance(i, bytes); - return true; -} -EXPORT_SYMBOL(_copy_from_iter_full_nocache); - static inline bool page_copy_sane(struct page *page, size_t offset, size_t n) { struct page *head; @@ -878,169 +341,207 @@ static inline bool page_copy_sane(struct page *page, size_t offset, size_t n) head = compound_head(page); v += (page - head) << PAGE_SHIFT; - if (likely(n <= v && v <= (PAGE_SIZE << compound_order(head)))) - return true; - WARN_ON(1); - return false; + if (WARN_ON(n > v || v > page_size(head))) + return false; + return true; } size_t copy_page_to_iter(struct page *page, size_t offset, size_t bytes, struct iov_iter *i) { - if (unlikely(!page_copy_sane(page, offset, bytes))) + size_t res = 0; + if (!page_copy_sane(page, offset, bytes)) return 0; - if (i->type & (ITER_BVEC|ITER_KVEC)) { - void *kaddr = kmap_atomic(page); - size_t wanted = copy_to_iter(kaddr + offset, bytes, i); - kunmap_atomic(kaddr); - return wanted; - } else if (unlikely(iov_iter_is_discard(i))) - return bytes; - else if (likely(!iov_iter_is_pipe(i))) - return copy_page_to_iter_iovec(page, offset, bytes, i); - else - return copy_page_to_iter_pipe(page, offset, bytes, i); + if (WARN_ON_ONCE(i->data_source)) + return 0; + page += offset / PAGE_SIZE; // first subpage + offset %= PAGE_SIZE; + while (1) { + void *kaddr = kmap_local_page(page); + size_t n = min(bytes, (size_t)PAGE_SIZE - offset); + n = _copy_to_iter(kaddr + offset, n, i); + kunmap_local(kaddr); + res += n; + bytes -= n; + if (!bytes || !n) + break; + offset += n; + if (offset == PAGE_SIZE) { + page++; + offset = 0; + } + } + return res; } EXPORT_SYMBOL(copy_page_to_iter); -size_t copy_page_from_iter(struct page *page, size_t offset, size_t bytes, - struct iov_iter *i) +size_t copy_page_to_iter_nofault(struct page *page, unsigned offset, size_t bytes, + struct iov_iter *i) { - if (unlikely(!page_copy_sane(page, offset, bytes))) + size_t res = 0; + + if (!page_copy_sane(page, offset, bytes)) return 0; - if (unlikely(iov_iter_is_pipe(i) || iov_iter_is_discard(i))) { - WARN_ON(1); + if (WARN_ON_ONCE(i->data_source)) return 0; + page += offset / PAGE_SIZE; // first subpage + offset %= PAGE_SIZE; + while (1) { + void *kaddr = kmap_local_page(page); + size_t n = min(bytes, (size_t)PAGE_SIZE - offset); + + n = iterate_and_advance(i, n, kaddr + offset, + copy_to_user_iter_nofault, + memcpy_to_iter); + kunmap_local(kaddr); + res += n; + bytes -= n; + if (!bytes || !n) + break; + offset += n; + if (offset == PAGE_SIZE) { + page++; + offset = 0; + } } - if (i->type & (ITER_BVEC|ITER_KVEC)) { - void *kaddr = kmap_atomic(page); - size_t wanted = _copy_from_iter(kaddr + offset, bytes, i); - kunmap_atomic(kaddr); - return wanted; - } else - return copy_page_from_iter_iovec(page, offset, bytes, i); + return res; } -EXPORT_SYMBOL(copy_page_from_iter); +EXPORT_SYMBOL(copy_page_to_iter_nofault); -static size_t pipe_zero(size_t bytes, struct iov_iter *i) +size_t copy_page_from_iter(struct page *page, size_t offset, size_t bytes, + struct iov_iter *i) { - struct pipe_inode_info *pipe = i->pipe; - size_t n, off; - int idx; - - if (!sanity(i)) + size_t res = 0; + if (!page_copy_sane(page, offset, bytes)) return 0; + page += offset / PAGE_SIZE; // first subpage + offset %= PAGE_SIZE; + while (1) { + void *kaddr = kmap_local_page(page); + size_t n = min(bytes, (size_t)PAGE_SIZE - offset); + n = _copy_from_iter(kaddr + offset, n, i); + kunmap_local(kaddr); + res += n; + bytes -= n; + if (!bytes || !n) + break; + offset += n; + if (offset == PAGE_SIZE) { + page++; + offset = 0; + } + } + return res; +} +EXPORT_SYMBOL(copy_page_from_iter); - bytes = n = push_pipe(i, bytes, &idx, &off); - if (unlikely(!n)) - return 0; +static __always_inline +size_t zero_to_user_iter(void __user *iter_to, size_t progress, + size_t len, void *priv, void *priv2) +{ + return clear_user(iter_to, len); +} - for ( ; n; idx = next_idx(idx, pipe), off = 0) { - size_t chunk = min_t(size_t, n, PAGE_SIZE - off); - memzero_page(pipe->bufs[idx].page, off, chunk); - i->idx = idx; - i->iov_offset = off + chunk; - n -= chunk; - } - i->count -= bytes; - return bytes; +static __always_inline +size_t zero_to_iter(void *iter_to, size_t progress, + size_t len, void *priv, void *priv2) +{ + memset(iter_to, 0, len); + return 0; } size_t iov_iter_zero(size_t bytes, struct iov_iter *i) { - if (unlikely(iov_iter_is_pipe(i))) - return pipe_zero(bytes, i); - iterate_and_advance(i, bytes, v, - clear_user(v.iov_base, v.iov_len), - memzero_page(v.bv_page, v.bv_offset, v.bv_len), - memset(v.iov_base, 0, v.iov_len) - ) - - return bytes; + return iterate_and_advance(i, bytes, NULL, + zero_to_user_iter, zero_to_iter); } EXPORT_SYMBOL(iov_iter_zero); -size_t iov_iter_copy_from_user_atomic(struct page *page, - struct iov_iter *i, unsigned long offset, size_t bytes) +size_t copy_page_from_iter_atomic(struct page *page, size_t offset, + size_t bytes, struct iov_iter *i) { - char *kaddr = kmap_atomic(page), *p = kaddr + offset; - if (unlikely(!page_copy_sane(page, offset, bytes))) { - kunmap_atomic(kaddr); + size_t n, copied = 0; + + if (!page_copy_sane(page, offset, bytes)) return 0; - } - if (unlikely(iov_iter_is_pipe(i) || iov_iter_is_discard(i))) { - kunmap_atomic(kaddr); - WARN_ON(1); + if (WARN_ON_ONCE(!i->data_source)) return 0; - } - iterate_all_kinds(i, bytes, v, - copyin((p += v.iov_len) - v.iov_len, v.iov_base, v.iov_len), - memcpy_from_page((p += v.bv_len) - v.bv_len, v.bv_page, - v.bv_offset, v.bv_len), - memcpy((p += v.iov_len) - v.iov_len, v.iov_base, v.iov_len) - ) - kunmap_atomic(kaddr); - return bytes; -} -EXPORT_SYMBOL(iov_iter_copy_from_user_atomic); - -static inline void pipe_truncate(struct iov_iter *i) -{ - struct pipe_inode_info *pipe = i->pipe; - if (pipe->nrbufs) { - size_t off = i->iov_offset; - int idx = i->idx; - int nrbufs = (idx - pipe->curbuf) & (pipe->buffers - 1); - if (off) { - pipe->bufs[idx].len = off - pipe->bufs[idx].offset; - idx = next_idx(idx, pipe); - nrbufs++; - } - while (pipe->nrbufs > nrbufs) { - pipe_buf_release(pipe, &pipe->bufs[idx]); - idx = next_idx(idx, pipe); - pipe->nrbufs--; + + do { + char *p; + + n = bytes - copied; + if (PageHighMem(page)) { + page += offset / PAGE_SIZE; + offset %= PAGE_SIZE; + n = min_t(size_t, n, PAGE_SIZE - offset); } - } + + p = kmap_atomic(page) + offset; + n = __copy_from_iter(p, n, i); + kunmap_atomic(p); + copied += n; + offset += n; + } while (PageHighMem(page) && copied != bytes && n > 0); + + return copied; } +EXPORT_SYMBOL(copy_page_from_iter_atomic); -static void pipe_advance(struct iov_iter *i, size_t size) +static void iov_iter_bvec_advance(struct iov_iter *i, size_t size) { - struct pipe_inode_info *pipe = i->pipe; - if (unlikely(i->count < size)) - size = i->count; - if (size) { - struct pipe_buffer *buf; - size_t off = i->iov_offset, left = size; - int idx = i->idx; - if (off) /* make it relative to the beginning of buffer */ - left += off - pipe->bufs[idx].offset; - while (1) { - buf = &pipe->bufs[idx]; - if (left <= buf->len) - break; - left -= buf->len; - idx = next_idx(idx, pipe); - } - i->idx = idx; - i->iov_offset = buf->offset + left; - } + const struct bio_vec *bvec, *end; + + if (!i->count) + return; i->count -= size; - /* ... and discard everything past that point */ - pipe_truncate(i); + + size += i->iov_offset; + + for (bvec = i->bvec, end = bvec + i->nr_segs; bvec < end; bvec++) { + if (likely(size < bvec->bv_len)) + break; + size -= bvec->bv_len; + } + i->iov_offset = size; + i->nr_segs -= bvec - i->bvec; + i->bvec = bvec; } -void iov_iter_advance(struct iov_iter *i, size_t size) +static void iov_iter_iovec_advance(struct iov_iter *i, size_t size) { - if (unlikely(iov_iter_is_pipe(i))) { - pipe_advance(i, size); + const struct iovec *iov, *end; + + if (!i->count) return; + i->count -= size; + + size += i->iov_offset; // from beginning of current segment + for (iov = iter_iov(i), end = iov + i->nr_segs; iov < end; iov++) { + if (likely(size < iov->iov_len)) + break; + size -= iov->iov_len; } - if (unlikely(iov_iter_is_discard(i))) { + i->iov_offset = size; + i->nr_segs -= iov - iter_iov(i); + i->__iov = iov; +} + +void iov_iter_advance(struct iov_iter *i, size_t size) +{ + if (unlikely(i->count < size)) + size = i->count; + if (likely(iter_is_ubuf(i)) || unlikely(iov_iter_is_xarray(i))) { + i->iov_offset += size; + i->count -= size; + } else if (likely(iter_is_iovec(i) || iov_iter_is_kvec(i))) { + /* iovec and kvec have identical layouts */ + iov_iter_iovec_advance(i, size); + } else if (iov_iter_is_bvec(i)) { + iov_iter_bvec_advance(i, size); + } else if (iov_iter_is_discard(i)) { i->count -= size; - return; } - iterate_and_advance(i, size, v, 0, 0, 0) } EXPORT_SYMBOL(iov_iter_advance); @@ -1051,30 +552,6 @@ void iov_iter_revert(struct iov_iter *i, size_t unroll) if (WARN_ON(unroll > MAX_RW_COUNT)) return; i->count += unroll; - if (unlikely(iov_iter_is_pipe(i))) { - struct pipe_inode_info *pipe = i->pipe; - int idx = i->idx; - size_t off = i->iov_offset; - while (1) { - size_t n = off - pipe->bufs[idx].offset; - if (unroll < n) { - off -= unroll; - break; - } - unroll -= n; - if (!unroll && idx == i->start_idx) { - off = 0; - break; - } - if (!idx--) - idx = pipe->buffers - 1; - off = pipe->bufs[idx].offset + pipe->bufs[idx].len; - } - i->iov_offset = off; - i->idx = idx; - pipe_truncate(i); - return; - } if (unlikely(iov_iter_is_discard(i))) return; if (unroll <= i->iov_offset) { @@ -1082,7 +559,12 @@ void iov_iter_revert(struct iov_iter *i, size_t unroll) return; } unroll -= i->iov_offset; - if (iov_iter_is_bvec(i)) { + if (iov_iter_is_xarray(i) || iter_is_ubuf(i)) { + BUG(); /* We should never go beyond the start of the specified + * range since we might then be straying into pages that + * aren't pinned. + */ + } else if (iov_iter_is_bvec(i)) { const struct bio_vec *bvec = i->bvec; while (1) { size_t n = (--bvec)->bv_len; @@ -1095,12 +577,12 @@ void iov_iter_revert(struct iov_iter *i, size_t unroll) unroll -= n; } } else { /* same logics for iovec and kvec */ - const struct iovec *iov = i->iov; + const struct iovec *iov = iter_iov(i); while (1) { size_t n = (--iov)->iov_len; i->nr_segs++; if (unroll <= n) { - i->iov = iov; + i->__iov = iov; i->iov_offset = n - unroll; return; } @@ -1115,16 +597,13 @@ EXPORT_SYMBOL(iov_iter_revert); */ size_t iov_iter_single_seg_count(const struct iov_iter *i) { - if (unlikely(iov_iter_is_pipe(i))) - return i->count; // it is a silly place, anyway - if (i->nr_segs == 1) - return i->count; - if (unlikely(iov_iter_is_discard(i))) - return i->count; - else if (iov_iter_is_bvec(i)) - return min(i->count, i->bvec->bv_len - i->iov_offset); - else - return min(i->count, i->iov->iov_len - i->iov_offset); + if (i->nr_segs > 1) { + if (likely(iter_is_iovec(i) || iov_iter_is_kvec(i))) + return min(i->count, iter_iov(i)->iov_len - i->iov_offset); + if (iov_iter_is_bvec(i)) + return min(i->count, i->bvec->bv_len - i->iov_offset); + } + return i->count; } EXPORT_SYMBOL(iov_iter_single_seg_count); @@ -1133,11 +612,14 @@ void iov_iter_kvec(struct iov_iter *i, unsigned int direction, size_t count) { WARN_ON(direction & ~(READ | WRITE)); - i->type = ITER_KVEC | (direction & (READ | WRITE)); - i->kvec = kvec; - i->nr_segs = nr_segs; - i->iov_offset = 0; - i->count = count; + *i = (struct iov_iter){ + .iter_type = ITER_KVEC, + .data_source = direction, + .kvec = kvec, + .nr_segs = nr_segs, + .iov_offset = 0, + .count = count + }; } EXPORT_SYMBOL(iov_iter_kvec); @@ -1146,28 +628,44 @@ void iov_iter_bvec(struct iov_iter *i, unsigned int direction, size_t count) { WARN_ON(direction & ~(READ | WRITE)); - i->type = ITER_BVEC | (direction & (READ | WRITE)); - i->bvec = bvec; - i->nr_segs = nr_segs; - i->iov_offset = 0; - i->count = count; + *i = (struct iov_iter){ + .iter_type = ITER_BVEC, + .data_source = direction, + .bvec = bvec, + .nr_segs = nr_segs, + .iov_offset = 0, + .count = count + }; } EXPORT_SYMBOL(iov_iter_bvec); -void iov_iter_pipe(struct iov_iter *i, unsigned int direction, - struct pipe_inode_info *pipe, - size_t count) +/** + * iov_iter_xarray - Initialise an I/O iterator to use the pages in an xarray + * @i: The iterator to initialise. + * @direction: The direction of the transfer. + * @xarray: The xarray to access. + * @start: The start file position. + * @count: The size of the I/O buffer in bytes. + * + * Set up an I/O iterator to either draw data out of the pages attached to an + * inode or to inject data into those pages. The pages *must* be prevented + * from evaporation, either by taking a ref on them or locking them by the + * caller. + */ +void iov_iter_xarray(struct iov_iter *i, unsigned int direction, + struct xarray *xarray, loff_t start, size_t count) { - BUG_ON(direction != READ); - WARN_ON(pipe->nrbufs == pipe->buffers); - i->type = ITER_PIPE | READ; - i->pipe = pipe; - i->idx = (pipe->curbuf + pipe->nrbufs) & (pipe->buffers - 1); - i->iov_offset = 0; - i->count = count; - i->start_idx = i->idx; -} -EXPORT_SYMBOL(iov_iter_pipe); + BUG_ON(direction & ~1); + *i = (struct iov_iter) { + .iter_type = ITER_XARRAY, + .data_source = direction, + .xarray = xarray, + .xarray_start = start, + .count = count, + .iov_offset = 0 + }; +} +EXPORT_SYMBOL(iov_iter_xarray); /** * iov_iter_discard - Initialise an I/O iterator that discards data @@ -1181,449 +679,652 @@ EXPORT_SYMBOL(iov_iter_pipe); void iov_iter_discard(struct iov_iter *i, unsigned int direction, size_t count) { BUG_ON(direction != READ); - i->type = ITER_DISCARD | READ; - i->count = count; - i->iov_offset = 0; + *i = (struct iov_iter){ + .iter_type = ITER_DISCARD, + .data_source = false, + .count = count, + .iov_offset = 0 + }; } EXPORT_SYMBOL(iov_iter_discard); -unsigned long iov_iter_alignment(const struct iov_iter *i) +static bool iov_iter_aligned_iovec(const struct iov_iter *i, unsigned addr_mask, + unsigned len_mask) { - unsigned long res = 0; size_t size = i->count; + size_t skip = i->iov_offset; + unsigned k; + + for (k = 0; k < i->nr_segs; k++, skip = 0) { + const struct iovec *iov = iter_iov(i) + k; + size_t len = iov->iov_len - skip; - if (unlikely(iov_iter_is_pipe(i))) { - if (size && i->iov_offset && allocated(&i->pipe->bufs[i->idx])) - return size | i->iov_offset; - return size; + if (len > size) + len = size; + if (len & len_mask) + return false; + if ((unsigned long)(iov->iov_base + skip) & addr_mask) + return false; + + size -= len; + if (!size) + break; } - iterate_all_kinds(i, size, v, - (res |= (unsigned long)v.iov_base | v.iov_len, 0), - res |= v.bv_offset | v.bv_len, - res |= (unsigned long)v.iov_base | v.iov_len - ) - return res; + return true; } -EXPORT_SYMBOL(iov_iter_alignment); -unsigned long iov_iter_gap_alignment(const struct iov_iter *i) +static bool iov_iter_aligned_bvec(const struct iov_iter *i, unsigned addr_mask, + unsigned len_mask) { - unsigned long res = 0; size_t size = i->count; + unsigned skip = i->iov_offset; + unsigned k; - if (unlikely(iov_iter_is_pipe(i) || iov_iter_is_discard(i))) { - WARN_ON(1); - return ~0U; - } + for (k = 0; k < i->nr_segs; k++, skip = 0) { + size_t len = i->bvec[k].bv_len - skip; - iterate_all_kinds(i, size, v, - (res |= (!res ? 0 : (unsigned long)v.iov_base) | - (size != v.iov_len ? size : 0), 0), - (res |= (!res ? 0 : (unsigned long)v.bv_offset) | - (size != v.bv_len ? size : 0)), - (res |= (!res ? 0 : (unsigned long)v.iov_base) | - (size != v.iov_len ? size : 0)) - ); - return res; + if (len > size) + len = size; + if (len & len_mask) + return false; + if ((unsigned long)(i->bvec[k].bv_offset + skip) & addr_mask) + return false; + + size -= len; + if (!size) + break; + } + return true; } -EXPORT_SYMBOL(iov_iter_gap_alignment); -static inline ssize_t __pipe_get_pages(struct iov_iter *i, - size_t maxsize, - struct page **pages, - int idx, - size_t *start) +/** + * iov_iter_is_aligned() - Check if the addresses and lengths of each segments + * are aligned to the parameters. + * + * @i: &struct iov_iter to restore + * @addr_mask: bit mask to check against the iov element's addresses + * @len_mask: bit mask to check against the iov element's lengths + * + * Return: false if any addresses or lengths intersect with the provided masks + */ +bool iov_iter_is_aligned(const struct iov_iter *i, unsigned addr_mask, + unsigned len_mask) { - struct pipe_inode_info *pipe = i->pipe; - ssize_t n = push_pipe(i, maxsize, &idx, start); - if (!n) - return -EFAULT; + if (likely(iter_is_ubuf(i))) { + if (i->count & len_mask) + return false; + if ((unsigned long)(i->ubuf + i->iov_offset) & addr_mask) + return false; + return true; + } - maxsize = n; - n += *start; - while (n > 0) { - get_page(*pages++ = pipe->bufs[idx].page); - idx = next_idx(idx, pipe); - n -= PAGE_SIZE; + if (likely(iter_is_iovec(i) || iov_iter_is_kvec(i))) + return iov_iter_aligned_iovec(i, addr_mask, len_mask); + + if (iov_iter_is_bvec(i)) + return iov_iter_aligned_bvec(i, addr_mask, len_mask); + + if (iov_iter_is_xarray(i)) { + if (i->count & len_mask) + return false; + if ((i->xarray_start + i->iov_offset) & addr_mask) + return false; } - return maxsize; + return true; +} +EXPORT_SYMBOL_GPL(iov_iter_is_aligned); + +static unsigned long iov_iter_alignment_iovec(const struct iov_iter *i) +{ + unsigned long res = 0; + size_t size = i->count; + size_t skip = i->iov_offset; + unsigned k; + + for (k = 0; k < i->nr_segs; k++, skip = 0) { + const struct iovec *iov = iter_iov(i) + k; + size_t len = iov->iov_len - skip; + if (len) { + res |= (unsigned long)iov->iov_base + skip; + if (len > size) + len = size; + res |= len; + size -= len; + if (!size) + break; + } + } + return res; } -static ssize_t pipe_get_pages(struct iov_iter *i, - struct page **pages, size_t maxsize, unsigned maxpages, - size_t *start) +static unsigned long iov_iter_alignment_bvec(const struct iov_iter *i) { - unsigned npages; - size_t capacity; - int idx; + unsigned res = 0; + size_t size = i->count; + unsigned skip = i->iov_offset; + unsigned k; + + for (k = 0; k < i->nr_segs; k++, skip = 0) { + size_t len = i->bvec[k].bv_len - skip; + res |= (unsigned long)i->bvec[k].bv_offset + skip; + if (len > size) + len = size; + res |= len; + size -= len; + if (!size) + break; + } + return res; +} - if (!maxsize) +unsigned long iov_iter_alignment(const struct iov_iter *i) +{ + if (likely(iter_is_ubuf(i))) { + size_t size = i->count; + if (size) + return ((unsigned long)i->ubuf + i->iov_offset) | size; return 0; + } - if (!sanity(i)) - return -EFAULT; + /* iovec and kvec have identical layouts */ + if (likely(iter_is_iovec(i) || iov_iter_is_kvec(i))) + return iov_iter_alignment_iovec(i); + + if (iov_iter_is_bvec(i)) + return iov_iter_alignment_bvec(i); - data_start(i, &idx, start); - /* some of this one + all after this one */ - npages = ((i->pipe->curbuf - idx - 1) & (i->pipe->buffers - 1)) + 1; - capacity = min(npages,maxpages) * PAGE_SIZE - *start; + if (iov_iter_is_xarray(i)) + return (i->xarray_start + i->iov_offset) | i->count; - return __pipe_get_pages(i, min(maxsize, capacity), pages, idx, start); + return 0; } +EXPORT_SYMBOL(iov_iter_alignment); -ssize_t iov_iter_get_pages(struct iov_iter *i, - struct page **pages, size_t maxsize, unsigned maxpages, - size_t *start) +unsigned long iov_iter_gap_alignment(const struct iov_iter *i) { - if (maxsize > i->count) - maxsize = i->count; + unsigned long res = 0; + unsigned long v = 0; + size_t size = i->count; + unsigned k; - if (unlikely(iov_iter_is_pipe(i))) - return pipe_get_pages(i, pages, maxsize, maxpages, start); - if (unlikely(iov_iter_is_discard(i))) - return -EFAULT; + if (iter_is_ubuf(i)) + return 0; - iterate_all_kinds(i, maxsize, v, ({ - unsigned long addr = (unsigned long)v.iov_base; - size_t len = v.iov_len + (*start = addr & (PAGE_SIZE - 1)); - int n; - int res; + if (WARN_ON(!iter_is_iovec(i))) + return ~0U; - if (len > maxpages * PAGE_SIZE) - len = maxpages * PAGE_SIZE; - addr &= ~(PAGE_SIZE - 1); - n = DIV_ROUND_UP(len, PAGE_SIZE); - res = get_user_pages_fast(addr, n, - iov_iter_rw(i) != WRITE ? FOLL_WRITE : 0, - pages); - if (unlikely(res < 0)) - return res; - return (res == n ? len : res * PAGE_SIZE) - *start; - 0;}),({ - /* can't be more than PAGE_SIZE */ - *start = v.bv_offset; - get_page(*pages = v.bv_page); - return v.bv_len; - }),({ - return -EFAULT; - }) - ) - return 0; + for (k = 0; k < i->nr_segs; k++) { + const struct iovec *iov = iter_iov(i) + k; + if (iov->iov_len) { + unsigned long base = (unsigned long)iov->iov_base; + if (v) // if not the first one + res |= base | v; // this start | previous end + v = base + iov->iov_len; + if (size <= iov->iov_len) + break; + size -= iov->iov_len; + } + } + return res; } -EXPORT_SYMBOL(iov_iter_get_pages); +EXPORT_SYMBOL(iov_iter_gap_alignment); -static struct page **get_pages_array(size_t n) +static int want_pages_array(struct page ***res, size_t size, + size_t start, unsigned int maxpages) { - return kvmalloc_array(n, sizeof(struct page *), GFP_KERNEL); + unsigned int count = DIV_ROUND_UP(size + start, PAGE_SIZE); + + if (count > maxpages) + count = maxpages; + WARN_ON(!count); // caller should've prevented that + if (!*res) { + *res = kvmalloc_array(count, sizeof(struct page *), GFP_KERNEL); + if (!*res) + return 0; + } + return count; } -static ssize_t pipe_get_pages_alloc(struct iov_iter *i, - struct page ***pages, size_t maxsize, - size_t *start) +static ssize_t iter_xarray_populate_pages(struct page **pages, struct xarray *xa, + pgoff_t index, unsigned int nr_pages) { - struct page **p; - ssize_t n; - int idx; - int npages; + XA_STATE(xas, xa, index); + struct page *page; + unsigned int ret = 0; + + rcu_read_lock(); + for (page = xas_load(&xas); page; page = xas_next(&xas)) { + if (xas_retry(&xas, page)) + continue; + + /* Has the page moved or been split? */ + if (unlikely(page != xas_reload(&xas))) { + xas_reset(&xas); + continue; + } - if (!maxsize) - return 0; + pages[ret] = find_subpage(page, xas.xa_index); + get_page(pages[ret]); + if (++ret == nr_pages) + break; + } + rcu_read_unlock(); + return ret; +} - if (!sanity(i)) - return -EFAULT; +static ssize_t iter_xarray_get_pages(struct iov_iter *i, + struct page ***pages, size_t maxsize, + unsigned maxpages, size_t *_start_offset) +{ + unsigned nr, offset, count; + pgoff_t index; + loff_t pos; - data_start(i, &idx, start); - /* some of this one + all after this one */ - npages = ((i->pipe->curbuf - idx - 1) & (i->pipe->buffers - 1)) + 1; - n = npages * PAGE_SIZE - *start; - if (maxsize > n) - maxsize = n; - else - npages = DIV_ROUND_UP(maxsize + *start, PAGE_SIZE); - p = get_pages_array(npages); - if (!p) + pos = i->xarray_start + i->iov_offset; + index = pos >> PAGE_SHIFT; + offset = pos & ~PAGE_MASK; + *_start_offset = offset; + + count = want_pages_array(pages, maxsize, offset, maxpages); + if (!count) return -ENOMEM; - n = __pipe_get_pages(i, maxsize, p, idx, start); - if (n > 0) - *pages = p; - else - kvfree(p); - return n; + nr = iter_xarray_populate_pages(*pages, i->xarray, index, count); + if (nr == 0) + return 0; + + maxsize = min_t(size_t, nr * PAGE_SIZE - offset, maxsize); + i->iov_offset += maxsize; + i->count -= maxsize; + return maxsize; } -ssize_t iov_iter_get_pages_alloc(struct iov_iter *i, +/* must be done on non-empty ITER_UBUF or ITER_IOVEC one */ +static unsigned long first_iovec_segment(const struct iov_iter *i, size_t *size) +{ + size_t skip; + long k; + + if (iter_is_ubuf(i)) + return (unsigned long)i->ubuf + i->iov_offset; + + for (k = 0, skip = i->iov_offset; k < i->nr_segs; k++, skip = 0) { + const struct iovec *iov = iter_iov(i) + k; + size_t len = iov->iov_len - skip; + + if (unlikely(!len)) + continue; + if (*size > len) + *size = len; + return (unsigned long)iov->iov_base + skip; + } + BUG(); // if it had been empty, we wouldn't get called +} + +/* must be done on non-empty ITER_BVEC one */ +static struct page *first_bvec_segment(const struct iov_iter *i, + size_t *size, size_t *start) +{ + struct page *page; + size_t skip = i->iov_offset, len; + + len = i->bvec->bv_len - skip; + if (*size > len) + *size = len; + skip += i->bvec->bv_offset; + page = i->bvec->bv_page + skip / PAGE_SIZE; + *start = skip % PAGE_SIZE; + return page; +} + +static ssize_t __iov_iter_get_pages_alloc(struct iov_iter *i, struct page ***pages, size_t maxsize, - size_t *start) + unsigned int maxpages, size_t *start) { - struct page **p; + unsigned int n, gup_flags = 0; if (maxsize > i->count) maxsize = i->count; + if (!maxsize) + return 0; + if (maxsize > MAX_RW_COUNT) + maxsize = MAX_RW_COUNT; - if (unlikely(iov_iter_is_pipe(i))) - return pipe_get_pages_alloc(i, pages, maxsize, start); - if (unlikely(iov_iter_is_discard(i))) - return -EFAULT; - - iterate_all_kinds(i, maxsize, v, ({ - unsigned long addr = (unsigned long)v.iov_base; - size_t len = v.iov_len + (*start = addr & (PAGE_SIZE - 1)); - int n; + if (likely(user_backed_iter(i))) { + unsigned long addr; int res; - addr &= ~(PAGE_SIZE - 1); - n = DIV_ROUND_UP(len, PAGE_SIZE); - p = get_pages_array(n); - if (!p) + if (iov_iter_rw(i) != WRITE) + gup_flags |= FOLL_WRITE; + if (i->nofault) + gup_flags |= FOLL_NOFAULT; + + addr = first_iovec_segment(i, &maxsize); + *start = addr % PAGE_SIZE; + addr &= PAGE_MASK; + n = want_pages_array(pages, maxsize, *start, maxpages); + if (!n) return -ENOMEM; - res = get_user_pages_fast(addr, n, - iov_iter_rw(i) != WRITE ? FOLL_WRITE : 0, p); - if (unlikely(res < 0)) { - kvfree(p); + res = get_user_pages_fast(addr, n, gup_flags, *pages); + if (unlikely(res <= 0)) return res; - } - *pages = p; - return (res == n ? len : res * PAGE_SIZE) - *start; - 0;}),({ - /* can't be more than PAGE_SIZE */ - *start = v.bv_offset; - *pages = p = get_pages_array(1); - if (!p) + maxsize = min_t(size_t, maxsize, res * PAGE_SIZE - *start); + iov_iter_advance(i, maxsize); + return maxsize; + } + if (iov_iter_is_bvec(i)) { + struct page **p; + struct page *page; + + page = first_bvec_segment(i, &maxsize, start); + n = want_pages_array(pages, maxsize, *start, maxpages); + if (!n) return -ENOMEM; - get_page(*p = v.bv_page); - return v.bv_len; - }),({ - return -EFAULT; - }) - ) - return 0; + p = *pages; + for (int k = 0; k < n; k++) + get_page(p[k] = page + k); + maxsize = min_t(size_t, maxsize, n * PAGE_SIZE - *start); + i->count -= maxsize; + i->iov_offset += maxsize; + if (i->iov_offset == i->bvec->bv_len) { + i->iov_offset = 0; + i->bvec++; + i->nr_segs--; + } + return maxsize; + } + if (iov_iter_is_xarray(i)) + return iter_xarray_get_pages(i, pages, maxsize, maxpages, start); + return -EFAULT; } -EXPORT_SYMBOL(iov_iter_get_pages_alloc); -size_t csum_and_copy_from_iter(void *addr, size_t bytes, __wsum *csum, - struct iov_iter *i) +ssize_t iov_iter_get_pages2(struct iov_iter *i, struct page **pages, + size_t maxsize, unsigned maxpages, size_t *start) { - char *to = addr; - __wsum sum, next; - size_t off = 0; - sum = *csum; - if (unlikely(iov_iter_is_pipe(i) || iov_iter_is_discard(i))) { - WARN_ON(1); + if (!maxpages) return 0; - } - iterate_and_advance(i, bytes, v, ({ - int err = 0; - next = csum_and_copy_from_user(v.iov_base, - (to += v.iov_len) - v.iov_len, - v.iov_len, 0, &err); - if (!err) { - sum = csum_block_add(sum, next, off); - off += v.iov_len; - } - err ? v.iov_len : 0; - }), ({ - char *p = kmap_atomic(v.bv_page); - sum = csum_and_memcpy((to += v.bv_len) - v.bv_len, - p + v.bv_offset, v.bv_len, - sum, off); - kunmap_atomic(p); - off += v.bv_len; - }),({ - sum = csum_and_memcpy((to += v.iov_len) - v.iov_len, - v.iov_base, v.iov_len, - sum, off); - off += v.iov_len; - }) - ) - *csum = sum; - return bytes; -} -EXPORT_SYMBOL(csum_and_copy_from_iter); - -bool csum_and_copy_from_iter_full(void *addr, size_t bytes, __wsum *csum, - struct iov_iter *i) -{ - char *to = addr; - __wsum sum, next; - size_t off = 0; - sum = *csum; - if (unlikely(iov_iter_is_pipe(i) || iov_iter_is_discard(i))) { - WARN_ON(1); - return false; - } - if (unlikely(i->count < bytes)) - return false; - iterate_all_kinds(i, bytes, v, ({ - int err = 0; - next = csum_and_copy_from_user(v.iov_base, - (to += v.iov_len) - v.iov_len, - v.iov_len, 0, &err); - if (err) - return false; - sum = csum_block_add(sum, next, off); - off += v.iov_len; - 0; - }), ({ - char *p = kmap_atomic(v.bv_page); - sum = csum_and_memcpy((to += v.bv_len) - v.bv_len, - p + v.bv_offset, v.bv_len, - sum, off); - kunmap_atomic(p); - off += v.bv_len; - }),({ - sum = csum_and_memcpy((to += v.iov_len) - v.iov_len, - v.iov_base, v.iov_len, - sum, off); - off += v.iov_len; - }) - ) - *csum = sum; - iov_iter_advance(i, bytes); - return true; + BUG_ON(!pages); + + return __iov_iter_get_pages_alloc(i, &pages, maxsize, maxpages, start); } -EXPORT_SYMBOL(csum_and_copy_from_iter_full); +EXPORT_SYMBOL(iov_iter_get_pages2); -size_t csum_and_copy_to_iter(const void *addr, size_t bytes, void *csump, - struct iov_iter *i) +ssize_t iov_iter_get_pages_alloc2(struct iov_iter *i, + struct page ***pages, size_t maxsize, size_t *start) { - const char *from = addr; - __wsum *csum = csump; - __wsum sum, next; - size_t off = 0; + ssize_t len; - if (unlikely(iov_iter_is_pipe(i))) - return csum_and_copy_to_pipe_iter(addr, bytes, csum, i); + *pages = NULL; - sum = *csum; - if (unlikely(iov_iter_is_discard(i))) { - WARN_ON(1); /* for now */ - return 0; + len = __iov_iter_get_pages_alloc(i, pages, maxsize, ~0U, start); + if (len <= 0) { + kvfree(*pages); + *pages = NULL; } - iterate_and_advance(i, bytes, v, ({ - int err = 0; - next = csum_and_copy_to_user((from += v.iov_len) - v.iov_len, - v.iov_base, - v.iov_len, 0, &err); - if (!err) { - sum = csum_block_add(sum, next, off); - off += v.iov_len; - } - err ? v.iov_len : 0; - }), ({ - char *p = kmap_atomic(v.bv_page); - sum = csum_and_memcpy(p + v.bv_offset, - (from += v.bv_len) - v.bv_len, - v.bv_len, sum, off); - kunmap_atomic(p); - off += v.bv_len; - }),({ - sum = csum_and_memcpy(v.iov_base, - (from += v.iov_len) - v.iov_len, - v.iov_len, sum, off); - off += v.iov_len; - }) - ) - *csum = sum; - return bytes; -} -EXPORT_SYMBOL(csum_and_copy_to_iter); - -size_t hash_and_copy_to_iter(const void *addr, size_t bytes, void *hashp, - struct iov_iter *i) -{ -#ifdef CONFIG_CRYPTO - struct ahash_request *hash = hashp; - struct scatterlist sg; - size_t copied; - - copied = copy_to_iter(addr, bytes, i); - sg_init_one(&sg, addr, copied); - ahash_request_set_crypt(hash, &sg, NULL, copied); - crypto_ahash_update(hash); - return copied; -#else - return 0; -#endif + return len; } -EXPORT_SYMBOL(hash_and_copy_to_iter); +EXPORT_SYMBOL(iov_iter_get_pages_alloc2); -int iov_iter_npages(const struct iov_iter *i, int maxpages) +static int iov_npages(const struct iov_iter *i, int maxpages) { - size_t size = i->count; + size_t skip = i->iov_offset, size = i->count; + const struct iovec *p; int npages = 0; - if (!size) - return 0; - if (unlikely(iov_iter_is_discard(i))) - return 0; + for (p = iter_iov(i); size; skip = 0, p++) { + unsigned offs = offset_in_page(p->iov_base + skip); + size_t len = min(p->iov_len - skip, size); - if (unlikely(iov_iter_is_pipe(i))) { - struct pipe_inode_info *pipe = i->pipe; - size_t off; - int idx; + if (len) { + size -= len; + npages += DIV_ROUND_UP(offs + len, PAGE_SIZE); + if (unlikely(npages > maxpages)) + return maxpages; + } + } + return npages; +} - if (!sanity(i)) - return 0; +static int bvec_npages(const struct iov_iter *i, int maxpages) +{ + size_t skip = i->iov_offset, size = i->count; + const struct bio_vec *p; + int npages = 0; - data_start(i, &idx, &off); - /* some of this one + all after this one */ - npages = ((pipe->curbuf - idx - 1) & (pipe->buffers - 1)) + 1; - if (npages >= maxpages) - return maxpages; - } else iterate_all_kinds(i, size, v, ({ - unsigned long p = (unsigned long)v.iov_base; - npages += DIV_ROUND_UP(p + v.iov_len, PAGE_SIZE) - - p / PAGE_SIZE; - if (npages >= maxpages) - return maxpages; - 0;}),({ - npages++; - if (npages >= maxpages) - return maxpages; - }),({ - unsigned long p = (unsigned long)v.iov_base; - npages += DIV_ROUND_UP(p + v.iov_len, PAGE_SIZE) - - p / PAGE_SIZE; - if (npages >= maxpages) + for (p = i->bvec; size; skip = 0, p++) { + unsigned offs = (p->bv_offset + skip) % PAGE_SIZE; + size_t len = min(p->bv_len - skip, size); + + size -= len; + npages += DIV_ROUND_UP(offs + len, PAGE_SIZE); + if (unlikely(npages > maxpages)) return maxpages; - }) - ) + } return npages; } + +int iov_iter_npages(const struct iov_iter *i, int maxpages) +{ + if (unlikely(!i->count)) + return 0; + if (likely(iter_is_ubuf(i))) { + unsigned offs = offset_in_page(i->ubuf + i->iov_offset); + int npages = DIV_ROUND_UP(offs + i->count, PAGE_SIZE); + return min(npages, maxpages); + } + /* iovec and kvec have identical layouts */ + if (likely(iter_is_iovec(i) || iov_iter_is_kvec(i))) + return iov_npages(i, maxpages); + if (iov_iter_is_bvec(i)) + return bvec_npages(i, maxpages); + if (iov_iter_is_xarray(i)) { + unsigned offset = (i->xarray_start + i->iov_offset) % PAGE_SIZE; + int npages = DIV_ROUND_UP(offset + i->count, PAGE_SIZE); + return min(npages, maxpages); + } + return 0; +} EXPORT_SYMBOL(iov_iter_npages); const void *dup_iter(struct iov_iter *new, struct iov_iter *old, gfp_t flags) { *new = *old; - if (unlikely(iov_iter_is_pipe(new))) { - WARN_ON(1); - return NULL; - } - if (unlikely(iov_iter_is_discard(new))) - return NULL; if (iov_iter_is_bvec(new)) return new->bvec = kmemdup(new->bvec, new->nr_segs * sizeof(struct bio_vec), flags); - else + else if (iov_iter_is_kvec(new) || iter_is_iovec(new)) /* iovec and kvec have identical layout */ - return new->iov = kmemdup(new->iov, + return new->__iov = kmemdup(new->__iov, new->nr_segs * sizeof(struct iovec), flags); + return NULL; } EXPORT_SYMBOL(dup_iter); +static __noclone int copy_compat_iovec_from_user(struct iovec *iov, + const struct iovec __user *uvec, unsigned long nr_segs) +{ + const struct compat_iovec __user *uiov = + (const struct compat_iovec __user *)uvec; + int ret = -EFAULT, i; + + if (!user_access_begin(uiov, nr_segs * sizeof(*uiov))) + return -EFAULT; + + for (i = 0; i < nr_segs; i++) { + compat_uptr_t buf; + compat_ssize_t len; + + unsafe_get_user(len, &uiov[i].iov_len, uaccess_end); + unsafe_get_user(buf, &uiov[i].iov_base, uaccess_end); + + /* check for compat_size_t not fitting in compat_ssize_t .. */ + if (len < 0) { + ret = -EINVAL; + goto uaccess_end; + } + iov[i].iov_base = compat_ptr(buf); + iov[i].iov_len = len; + } + + ret = 0; +uaccess_end: + user_access_end(); + return ret; +} + +static __noclone int copy_iovec_from_user(struct iovec *iov, + const struct iovec __user *uiov, unsigned long nr_segs) +{ + int ret = -EFAULT; + + if (!user_access_begin(uiov, nr_segs * sizeof(*uiov))) + return -EFAULT; + + do { + void __user *buf; + ssize_t len; + + unsafe_get_user(len, &uiov->iov_len, uaccess_end); + unsafe_get_user(buf, &uiov->iov_base, uaccess_end); + + /* check for size_t not fitting in ssize_t .. */ + if (unlikely(len < 0)) { + ret = -EINVAL; + goto uaccess_end; + } + iov->iov_base = buf; + iov->iov_len = len; + + uiov++; iov++; + } while (--nr_segs); + + ret = 0; +uaccess_end: + user_access_end(); + return ret; +} + +struct iovec *iovec_from_user(const struct iovec __user *uvec, + unsigned long nr_segs, unsigned long fast_segs, + struct iovec *fast_iov, bool compat) +{ + struct iovec *iov = fast_iov; + int ret; + + /* + * SuS says "The readv() function *may* fail if the iovcnt argument was + * less than or equal to 0, or greater than {IOV_MAX}. Linux has + * traditionally returned zero for zero segments, so... + */ + if (nr_segs == 0) + return iov; + if (nr_segs > UIO_MAXIOV) + return ERR_PTR(-EINVAL); + if (nr_segs > fast_segs) { + iov = kmalloc_array(nr_segs, sizeof(struct iovec), GFP_KERNEL); + if (!iov) + return ERR_PTR(-ENOMEM); + } + + if (unlikely(compat)) + ret = copy_compat_iovec_from_user(iov, uvec, nr_segs); + else + ret = copy_iovec_from_user(iov, uvec, nr_segs); + if (ret) { + if (iov != fast_iov) + kfree(iov); + return ERR_PTR(ret); + } + + return iov; +} + +/* + * Single segment iovec supplied by the user, import it as ITER_UBUF. + */ +static ssize_t __import_iovec_ubuf(int type, const struct iovec __user *uvec, + struct iovec **iovp, struct iov_iter *i, + bool compat) +{ + struct iovec *iov = *iovp; + ssize_t ret; + + if (compat) + ret = copy_compat_iovec_from_user(iov, uvec, 1); + else + ret = copy_iovec_from_user(iov, uvec, 1); + if (unlikely(ret)) + return ret; + + ret = import_ubuf(type, iov->iov_base, iov->iov_len, i); + if (unlikely(ret)) + return ret; + *iovp = NULL; + return i->count; +} + +ssize_t __import_iovec(int type, const struct iovec __user *uvec, + unsigned nr_segs, unsigned fast_segs, struct iovec **iovp, + struct iov_iter *i, bool compat) +{ + ssize_t total_len = 0; + unsigned long seg; + struct iovec *iov; + + if (nr_segs == 1) + return __import_iovec_ubuf(type, uvec, iovp, i, compat); + + iov = iovec_from_user(uvec, nr_segs, fast_segs, *iovp, compat); + if (IS_ERR(iov)) { + *iovp = NULL; + return PTR_ERR(iov); + } + + /* + * According to the Single Unix Specification we should return EINVAL if + * an element length is < 0 when cast to ssize_t or if the total length + * would overflow the ssize_t return value of the system call. + * + * Linux caps all read/write calls to MAX_RW_COUNT, and avoids the + * overflow case. + */ + for (seg = 0; seg < nr_segs; seg++) { + ssize_t len = (ssize_t)iov[seg].iov_len; + + if (!access_ok(iov[seg].iov_base, len)) { + if (iov != *iovp) + kfree(iov); + *iovp = NULL; + return -EFAULT; + } + + if (len > MAX_RW_COUNT - total_len) { + len = MAX_RW_COUNT - total_len; + iov[seg].iov_len = len; + } + total_len += len; + } + + iov_iter_init(i, type, iov, nr_segs, total_len); + if (iov == *iovp) + *iovp = NULL; + else + *iovp = iov; + return total_len; +} + /** * import_iovec() - Copy an array of &struct iovec from userspace * into the kernel, check that it is valid, and initialize a new * &struct iov_iter iterator to access it. * * @type: One of %READ or %WRITE. - * @uvector: Pointer to the userspace array. + * @uvec: Pointer to the userspace array. * @nr_segs: Number of elements in userspace array. * @fast_segs: Number of elements in @iov. - * @iov: (input and output parameter) Pointer to pointer to (usually small + * @iovp: (input and output parameter) Pointer to pointer to (usually small * on-stack) kernel array. * @i: Pointer to iterator that will be initialized on success. * @@ -1634,84 +1335,327 @@ EXPORT_SYMBOL(dup_iter); * on-stack array was used or not (and regardless of whether this function * returns an error or not). * - * Return: 0 on success or negative error code on error. + * Return: Negative error code on error, bytes imported on success */ -int import_iovec(int type, const struct iovec __user * uvector, +ssize_t import_iovec(int type, const struct iovec __user *uvec, unsigned nr_segs, unsigned fast_segs, - struct iovec **iov, struct iov_iter *i) -{ - ssize_t n; - struct iovec *p; - n = rw_copy_check_uvector(type, uvector, nr_segs, fast_segs, - *iov, &p); - if (n < 0) { - if (p != *iov) - kfree(p); - *iov = NULL; - return n; - } - iov_iter_init(i, type, p, nr_segs, n); - *iov = p == *iov ? NULL : p; - return 0; + struct iovec **iovp, struct iov_iter *i) +{ + return __import_iovec(type, uvec, nr_segs, fast_segs, iovp, i, + in_compat_syscall()); } EXPORT_SYMBOL(import_iovec); -#ifdef CONFIG_COMPAT -#include <linux/compat.h> - -int compat_import_iovec(int type, const struct compat_iovec __user * uvector, - unsigned nr_segs, unsigned fast_segs, - struct iovec **iov, struct iov_iter *i) -{ - ssize_t n; - struct iovec *p; - n = compat_rw_copy_check_uvector(type, uvector, nr_segs, fast_segs, - *iov, &p); - if (n < 0) { - if (p != *iov) - kfree(p); - *iov = NULL; - return n; - } - iov_iter_init(i, type, p, nr_segs, n); - *iov = p == *iov ? NULL : p; - return 0; -} -#endif - -int import_single_range(int rw, void __user *buf, size_t len, - struct iovec *iov, struct iov_iter *i) +int import_ubuf(int rw, void __user *buf, size_t len, struct iov_iter *i) { if (len > MAX_RW_COUNT) len = MAX_RW_COUNT; if (unlikely(!access_ok(buf, len))) return -EFAULT; - iov->iov_base = buf; - iov->iov_len = len; - iov_iter_init(i, rw, iov, 1, len); + iov_iter_ubuf(i, rw, buf, len); return 0; } -EXPORT_SYMBOL(import_single_range); +EXPORT_SYMBOL_GPL(import_ubuf); -int iov_iter_for_each_range(struct iov_iter *i, size_t bytes, - int (*f)(struct kvec *vec, void *context), - void *context) +/** + * iov_iter_restore() - Restore a &struct iov_iter to the same state as when + * iov_iter_save_state() was called. + * + * @i: &struct iov_iter to restore + * @state: state to restore from + * + * Used after iov_iter_save_state() to bring restore @i, if operations may + * have advanced it. + * + * Note: only works on ITER_IOVEC, ITER_BVEC, and ITER_KVEC + */ +void iov_iter_restore(struct iov_iter *i, struct iov_iter_state *state) { - struct kvec w; - int err = -EINVAL; - if (!bytes) + if (WARN_ON_ONCE(!iov_iter_is_bvec(i) && !iter_is_iovec(i) && + !iter_is_ubuf(i)) && !iov_iter_is_kvec(i)) + return; + i->iov_offset = state->iov_offset; + i->count = state->count; + if (iter_is_ubuf(i)) + return; + /* + * For the *vec iters, nr_segs + iov is constant - if we increment + * the vec, then we also decrement the nr_segs count. Hence we don't + * need to track both of these, just one is enough and we can deduct + * the other from that. ITER_KVEC and ITER_IOVEC are the same struct + * size, so we can just increment the iov pointer as they are unionzed. + * ITER_BVEC _may_ be the same size on some archs, but on others it is + * not. Be safe and handle it separately. + */ + BUILD_BUG_ON(sizeof(struct iovec) != sizeof(struct kvec)); + if (iov_iter_is_bvec(i)) + i->bvec -= state->nr_segs - i->nr_segs; + else + i->__iov -= state->nr_segs - i->nr_segs; + i->nr_segs = state->nr_segs; +} + +/* + * Extract a list of contiguous pages from an ITER_XARRAY iterator. This does not + * get references on the pages, nor does it get a pin on them. + */ +static ssize_t iov_iter_extract_xarray_pages(struct iov_iter *i, + struct page ***pages, size_t maxsize, + unsigned int maxpages, + iov_iter_extraction_t extraction_flags, + size_t *offset0) +{ + struct page *page, **p; + unsigned int nr = 0, offset; + loff_t pos = i->xarray_start + i->iov_offset; + pgoff_t index = pos >> PAGE_SHIFT; + XA_STATE(xas, i->xarray, index); + + offset = pos & ~PAGE_MASK; + *offset0 = offset; + + maxpages = want_pages_array(pages, maxsize, offset, maxpages); + if (!maxpages) + return -ENOMEM; + p = *pages; + + rcu_read_lock(); + for (page = xas_load(&xas); page; page = xas_next(&xas)) { + if (xas_retry(&xas, page)) + continue; + + /* Has the page moved or been split? */ + if (unlikely(page != xas_reload(&xas))) { + xas_reset(&xas); + continue; + } + + p[nr++] = find_subpage(page, xas.xa_index); + if (nr == maxpages) + break; + } + rcu_read_unlock(); + + maxsize = min_t(size_t, nr * PAGE_SIZE - offset, maxsize); + iov_iter_advance(i, maxsize); + return maxsize; +} + +/* + * Extract a list of contiguous pages from an ITER_BVEC iterator. This does + * not get references on the pages, nor does it get a pin on them. + */ +static ssize_t iov_iter_extract_bvec_pages(struct iov_iter *i, + struct page ***pages, size_t maxsize, + unsigned int maxpages, + iov_iter_extraction_t extraction_flags, + size_t *offset0) +{ + struct page **p, *page; + size_t skip = i->iov_offset, offset, size; + int k; + + for (;;) { + if (i->nr_segs == 0) + return 0; + size = min(maxsize, i->bvec->bv_len - skip); + if (size) + break; + i->iov_offset = 0; + i->nr_segs--; + i->bvec++; + skip = 0; + } + + skip += i->bvec->bv_offset; + page = i->bvec->bv_page + skip / PAGE_SIZE; + offset = skip % PAGE_SIZE; + *offset0 = offset; + + maxpages = want_pages_array(pages, size, offset, maxpages); + if (!maxpages) + return -ENOMEM; + p = *pages; + for (k = 0; k < maxpages; k++) + p[k] = page + k; + + size = min_t(size_t, size, maxpages * PAGE_SIZE - offset); + iov_iter_advance(i, size); + return size; +} + +/* + * Extract a list of virtually contiguous pages from an ITER_KVEC iterator. + * This does not get references on the pages, nor does it get a pin on them. + */ +static ssize_t iov_iter_extract_kvec_pages(struct iov_iter *i, + struct page ***pages, size_t maxsize, + unsigned int maxpages, + iov_iter_extraction_t extraction_flags, + size_t *offset0) +{ + struct page **p, *page; + const void *kaddr; + size_t skip = i->iov_offset, offset, len, size; + int k; + + for (;;) { + if (i->nr_segs == 0) + return 0; + size = min(maxsize, i->kvec->iov_len - skip); + if (size) + break; + i->iov_offset = 0; + i->nr_segs--; + i->kvec++; + skip = 0; + } + + kaddr = i->kvec->iov_base + skip; + offset = (unsigned long)kaddr & ~PAGE_MASK; + *offset0 = offset; + + maxpages = want_pages_array(pages, size, offset, maxpages); + if (!maxpages) + return -ENOMEM; + p = *pages; + + kaddr -= offset; + len = offset + size; + for (k = 0; k < maxpages; k++) { + size_t seg = min_t(size_t, len, PAGE_SIZE); + + if (is_vmalloc_or_module_addr(kaddr)) + page = vmalloc_to_page(kaddr); + else + page = virt_to_page(kaddr); + + p[k] = page; + len -= seg; + kaddr += PAGE_SIZE; + } + + size = min_t(size_t, size, maxpages * PAGE_SIZE - offset); + iov_iter_advance(i, size); + return size; +} + +/* + * Extract a list of contiguous pages from a user iterator and get a pin on + * each of them. This should only be used if the iterator is user-backed + * (IOBUF/UBUF). + * + * It does not get refs on the pages, but the pages must be unpinned by the + * caller once the transfer is complete. + * + * This is safe to be used where background IO/DMA *is* going to be modifying + * the buffer; using a pin rather than a ref makes forces fork() to give the + * child a copy of the page. + */ +static ssize_t iov_iter_extract_user_pages(struct iov_iter *i, + struct page ***pages, + size_t maxsize, + unsigned int maxpages, + iov_iter_extraction_t extraction_flags, + size_t *offset0) +{ + unsigned long addr; + unsigned int gup_flags = 0; + size_t offset; + int res; + + if (i->data_source == ITER_DEST) + gup_flags |= FOLL_WRITE; + if (extraction_flags & ITER_ALLOW_P2PDMA) + gup_flags |= FOLL_PCI_P2PDMA; + if (i->nofault) + gup_flags |= FOLL_NOFAULT; + + addr = first_iovec_segment(i, &maxsize); + *offset0 = offset = addr % PAGE_SIZE; + addr &= PAGE_MASK; + maxpages = want_pages_array(pages, maxsize, offset, maxpages); + if (!maxpages) + return -ENOMEM; + res = pin_user_pages_fast(addr, maxpages, gup_flags, *pages); + if (unlikely(res <= 0)) + return res; + maxsize = min_t(size_t, maxsize, res * PAGE_SIZE - offset); + iov_iter_advance(i, maxsize); + return maxsize; +} + +/** + * iov_iter_extract_pages - Extract a list of contiguous pages from an iterator + * @i: The iterator to extract from + * @pages: Where to return the list of pages + * @maxsize: The maximum amount of iterator to extract + * @maxpages: The maximum size of the list of pages + * @extraction_flags: Flags to qualify request + * @offset0: Where to return the starting offset into (*@pages)[0] + * + * Extract a list of contiguous pages from the current point of the iterator, + * advancing the iterator. The maximum number of pages and the maximum amount + * of page contents can be set. + * + * If *@pages is NULL, a page list will be allocated to the required size and + * *@pages will be set to its base. If *@pages is not NULL, it will be assumed + * that the caller allocated a page list at least @maxpages in size and this + * will be filled in. + * + * @extraction_flags can have ITER_ALLOW_P2PDMA set to request peer-to-peer DMA + * be allowed on the pages extracted. + * + * The iov_iter_extract_will_pin() function can be used to query how cleanup + * should be performed. + * + * Extra refs or pins on the pages may be obtained as follows: + * + * (*) If the iterator is user-backed (ITER_IOVEC/ITER_UBUF), pins will be + * added to the pages, but refs will not be taken. + * iov_iter_extract_will_pin() will return true. + * + * (*) If the iterator is ITER_KVEC, ITER_BVEC or ITER_XARRAY, the pages are + * merely listed; no extra refs or pins are obtained. + * iov_iter_extract_will_pin() will return 0. + * + * Note also: + * + * (*) Use with ITER_DISCARD is not supported as that has no content. + * + * On success, the function sets *@pages to the new pagelist, if allocated, and + * sets *offset0 to the offset into the first page. + * + * It may also return -ENOMEM and -EFAULT. + */ +ssize_t iov_iter_extract_pages(struct iov_iter *i, + struct page ***pages, + size_t maxsize, + unsigned int maxpages, + iov_iter_extraction_t extraction_flags, + size_t *offset0) +{ + maxsize = min_t(size_t, min_t(size_t, maxsize, i->count), MAX_RW_COUNT); + if (!maxsize) return 0; - iterate_all_kinds(i, bytes, v, -EINVAL, ({ - w.iov_base = kmap(v.bv_page) + v.bv_offset; - w.iov_len = v.bv_len; - err = f(&w, context); - kunmap(v.bv_page); - err;}), ({ - w = v; - err = f(&w, context);}) - ) - return err; -} -EXPORT_SYMBOL(iov_iter_for_each_range); + if (likely(user_backed_iter(i))) + return iov_iter_extract_user_pages(i, pages, maxsize, + maxpages, extraction_flags, + offset0); + if (iov_iter_is_kvec(i)) + return iov_iter_extract_kvec_pages(i, pages, maxsize, + maxpages, extraction_flags, + offset0); + if (iov_iter_is_bvec(i)) + return iov_iter_extract_bvec_pages(i, pages, maxsize, + maxpages, extraction_flags, + offset0); + if (iov_iter_is_xarray(i)) + return iov_iter_extract_xarray_pages(i, pages, maxsize, + maxpages, extraction_flags, + offset0); + return -EFAULT; +} +EXPORT_SYMBOL_GPL(iov_iter_extract_pages); |