diff options
| -rw-r--r-- | mm/filemap.c | 36 | ||||
| -rw-r--r-- | mm/filemap.h | 100 | ||||
| -rw-r--r-- | mm/filemap_xip.c | 17 |
3 files changed, 67 insertions, 86 deletions
diff --git a/mm/filemap.c b/mm/filemap.c index fb4c1c0792e..c59d5b3cd99 100644 --- a/mm/filemap.c +++ b/mm/filemap.c @@ -1823,12 +1823,7 @@ generic_file_buffered_write(struct kiocb *iocb, const struct iovec *iov, /* * handle partial DIO write. Adjust cur_iov if needed. */ - if (likely(nr_segs == 1)) - buf = iov->iov_base + written; - else { - filemap_set_next_iovec(&cur_iov, &iov_offset, written); - buf = cur_iov->iov_base + iov_offset; - } + filemap_set_next_iovec(&cur_iov, nr_segs, &iov_offset, written); do { struct page *page; @@ -1838,6 +1833,7 @@ generic_file_buffered_write(struct kiocb *iocb, const struct iovec *iov, size_t bytes; /* Bytes to write to page */ size_t copied; /* Bytes copied from user */ + buf = cur_iov->iov_base + iov_offset; offset = (pos & (PAGE_CACHE_SIZE - 1)); index = pos >> PAGE_CACHE_SHIFT; bytes = PAGE_CACHE_SIZE - offset; @@ -1869,13 +1865,10 @@ generic_file_buffered_write(struct kiocb *iocb, const struct iovec *iov, if (unlikely(status)) goto fs_write_aop_error; - if (likely(nr_segs == 1)) - copied = filemap_copy_from_user(page, offset, - buf, bytes); - else - copied = filemap_copy_from_user_iovec(page, offset, - cur_iov, iov_offset, bytes); + copied = filemap_copy_from_user(page, offset, + cur_iov, nr_segs, iov_offset, bytes); flush_dcache_page(page); + status = a_ops->commit_write(file, page, offset, offset+bytes); if (unlikely(status < 0 || status == AOP_TRUNCATED_PAGE)) goto fs_write_aop_error; @@ -1886,20 +1879,11 @@ generic_file_buffered_write(struct kiocb *iocb, const struct iovec *iov, if (unlikely(status > 0)) /* filesystem did partial write */ copied = status; - if (likely(copied > 0)) { - written += copied; - count -= copied; - pos += copied; - buf += copied; - if (unlikely(nr_segs > 1)) { - filemap_set_next_iovec(&cur_iov, - &iov_offset, copied); - if (count) - buf = cur_iov->iov_base + iov_offset; - } else { - iov_offset += copied; - } - } + written += copied; + count -= copied; + pos += copied; + filemap_set_next_iovec(&cur_iov, nr_segs, &iov_offset, copied); + unlock_page(page); mark_page_accessed(page); page_cache_release(page); diff --git a/mm/filemap.h b/mm/filemap.h index a1e10a232e9..b500d936cec 100644 --- a/mm/filemap.h +++ b/mm/filemap.h @@ -22,82 +22,82 @@ __filemap_copy_from_user_iovec_inatomic(char *vaddr, /* * Copy as much as we can into the page and return the number of bytes which - * were sucessfully copied. If a fault is encountered then clear the page - * out to (offset+bytes) and return the number of bytes which were copied. - * - * NOTE: For this to work reliably we really want copy_from_user_inatomic_nocache - * to *NOT* zero any tail of the buffer that it failed to copy. If it does, - * and if the following non-atomic copy succeeds, then there is a small window - * where the target page contains neither the data before the write, nor the - * data after the write (it contains zero). A read at this time will see - * data that is inconsistent with any ordering of the read and the write. - * (This has been detected in practice). + * were sucessfully copied. If a fault is encountered then return the number of + * bytes which were copied. */ static inline size_t -filemap_copy_from_user(struct page *page, unsigned long offset, - const char __user *buf, unsigned bytes) +filemap_copy_from_user_atomic(struct page *page, unsigned long offset, + const struct iovec *iov, unsigned long nr_segs, + size_t base, size_t bytes) { char *kaddr; - int left; + size_t copied; kaddr = kmap_atomic(page, KM_USER0); - left = __copy_from_user_inatomic_nocache(kaddr + offset, buf, bytes); + if (likely(nr_segs == 1)) { + int left; + char __user *buf = iov->iov_base + base; + left = __copy_from_user_inatomic_nocache(kaddr + offset, + buf, bytes); + copied = bytes - left; + } else { + copied = __filemap_copy_from_user_iovec_inatomic(kaddr + offset, + iov, base, bytes); + } kunmap_atomic(kaddr, KM_USER0); - if (left != 0) { - /* Do it the slow way */ - kaddr = kmap(page); - left = __copy_from_user_nocache(kaddr + offset, buf, bytes); - kunmap(page); - } - return bytes - left; + return copied; } /* - * This has the same sideeffects and return value as filemap_copy_from_user(). - * The difference is that on a fault we need to memset the remainder of the - * page (out to offset+bytes), to emulate filemap_copy_from_user()'s - * single-segment behaviour. + * This has the same sideeffects and return value as + * filemap_copy_from_user_atomic(). + * The difference is that it attempts to resolve faults. */ static inline size_t -filemap_copy_from_user_iovec(struct page *page, unsigned long offset, - const struct iovec *iov, size_t base, size_t bytes) +filemap_copy_from_user(struct page *page, unsigned long offset, + const struct iovec *iov, unsigned long nr_segs, + size_t base, size_t bytes) { char *kaddr; size_t copied; - kaddr = kmap_atomic(page, KM_USER0); - copied = __filemap_copy_from_user_iovec_inatomic(kaddr + offset, iov, - base, bytes); - kunmap_atomic(kaddr, KM_USER0); - if (copied != bytes) { - kaddr = kmap(page); - copied = __filemap_copy_from_user_iovec_inatomic(kaddr + offset, iov, - base, bytes); - if (bytes - copied) - memset(kaddr + offset + copied, 0, bytes - copied); - kunmap(page); + kaddr = kmap(page); + if (likely(nr_segs == 1)) { + int left; + char __user *buf = iov->iov_base + base; + left = __copy_from_user_nocache(kaddr + offset, buf, bytes); + copied = bytes - left; + } else { + copied = __filemap_copy_from_user_iovec_inatomic(kaddr + offset, + iov, base, bytes); } + kunmap(page); return copied; } static inline void -filemap_set_next_iovec(const struct iovec **iovp, size_t *basep, size_t bytes) +filemap_set_next_iovec(const struct iovec **iovp, unsigned long nr_segs, + size_t *basep, size_t bytes) { - const struct iovec *iov = *iovp; - size_t base = *basep; + if (likely(nr_segs == 1)) { + *basep += bytes; + } else { + const struct iovec *iov = *iovp; + size_t base = *basep; - while (bytes) { - int copy = min(bytes, iov->iov_len - base); + while (bytes) { + int copy = min(bytes, iov->iov_len - base); - bytes -= copy; - base += copy; - if (iov->iov_len == base) { - iov++; - base = 0; + bytes -= copy; + base += copy; + if (iov->iov_len == base) { + iov++; + base = 0; + } } + *iovp = iov; + *basep = base; } - *iovp = iov; - *basep = base; } #endif diff --git a/mm/filemap_xip.c b/mm/filemap_xip.c index 53ee6a29963..32132f3cd64 100644 --- a/mm/filemap_xip.c +++ b/mm/filemap_xip.c @@ -15,7 +15,6 @@ #include <linux/rmap.h> #include <linux/sched.h> #include <asm/tlbflush.h> -#include "filemap.h" /* * We do use our own empty page to avoid interference with other users @@ -288,6 +287,7 @@ __xip_file_write(struct file *filp, const char __user *buf, unsigned long index; unsigned long offset; size_t copied; + char *kaddr; offset = (pos & (PAGE_CACHE_SIZE -1)); /* Within page */ index = pos >> PAGE_CACHE_SHIFT; @@ -295,14 +295,6 @@ __xip_file_write(struct file *filp, const char __user *buf, if (bytes > count) bytes = count; - /* - * Bring in the user page that we will copy from _first_. - * Otherwise there's a nasty deadlock on copying from the - * same page as we're writing to, without it being marked - * up-to-date. - */ - fault_in_pages_readable(buf, bytes); - page = a_ops->get_xip_page(mapping, index*(PAGE_SIZE/512), 0); if (IS_ERR(page) && (PTR_ERR(page) == -ENODATA)) { @@ -319,8 +311,13 @@ __xip_file_write(struct file *filp, const char __user *buf, break; } - copied = filemap_copy_from_user(page, offset, buf, bytes); + fault_in_pages_readable(buf, bytes); + kaddr = kmap_atomic(page, KM_USER0); + copied = bytes - + __copy_from_user_inatomic_nocache(kaddr, buf, bytes); + kunmap_atomic(kaddr, KM_USER0); flush_dcache_page(page); + if (likely(copied > 0)) { status = copied; |