whiterose

linux unikernel
Log | Files | Refs | README | LICENSE | git clone https://git.ne02ptzero.me/git/whiterose

read_write.c (50793B)


      1 // SPDX-License-Identifier: GPL-2.0
      2 /*
      3  *  linux/fs/read_write.c
      4  *
      5  *  Copyright (C) 1991, 1992  Linus Torvalds
      6  */
      7 
      8 #include <linux/slab.h>
      9 #include <linux/stat.h>
     10 #include <linux/sched/xacct.h>
     11 #include <linux/fcntl.h>
     12 #include <linux/file.h>
     13 #include <linux/uio.h>
     14 #include <linux/fsnotify.h>
     15 #include <linux/security.h>
     16 #include <linux/export.h>
     17 #include <linux/syscalls.h>
     18 #include <linux/pagemap.h>
     19 #include <linux/splice.h>
     20 #include <linux/compat.h>
     21 #include <linux/mount.h>
     22 #include <linux/fs.h>
     23 #include "internal.h"
     24 
     25 #include <linux/uaccess.h>
     26 #include <asm/unistd.h>
     27 
     28 const struct file_operations generic_ro_fops = {
     29 	.llseek		= generic_file_llseek,
     30 	.read_iter	= generic_file_read_iter,
     31 	.mmap		= generic_file_readonly_mmap,
     32 	.splice_read	= generic_file_splice_read,
     33 };
     34 
     35 EXPORT_SYMBOL(generic_ro_fops);
     36 
     37 static inline bool unsigned_offsets(struct file *file)
     38 {
     39 	return file->f_mode & FMODE_UNSIGNED_OFFSET;
     40 }
     41 
     42 /**
     43  * vfs_setpos - update the file offset for lseek
     44  * @file:	file structure in question
     45  * @offset:	file offset to seek to
     46  * @maxsize:	maximum file size
     47  *
     48  * This is a low-level filesystem helper for updating the file offset to
     49  * the value specified by @offset if the given offset is valid and it is
     50  * not equal to the current file offset.
     51  *
     52  * Return the specified offset on success and -EINVAL on invalid offset.
     53  */
     54 loff_t vfs_setpos(struct file *file, loff_t offset, loff_t maxsize)
     55 {
     56 	if (offset < 0 && !unsigned_offsets(file))
     57 		return -EINVAL;
     58 	if (offset > maxsize)
     59 		return -EINVAL;
     60 
     61 	if (offset != file->f_pos) {
     62 		file->f_pos = offset;
     63 		file->f_version = 0;
     64 	}
     65 	return offset;
     66 }
     67 EXPORT_SYMBOL(vfs_setpos);
     68 
     69 /**
     70  * generic_file_llseek_size - generic llseek implementation for regular files
     71  * @file:	file structure to seek on
     72  * @offset:	file offset to seek to
     73  * @whence:	type of seek
     74  * @size:	max size of this file in file system
     75  * @eof:	offset used for SEEK_END position
     76  *
     77  * This is a variant of generic_file_llseek that allows passing in a custom
     78  * maximum file size and a custom EOF position, for e.g. hashed directories
     79  *
     80  * Synchronization:
     81  * SEEK_SET and SEEK_END are unsynchronized (but atomic on 64bit platforms)
     82  * SEEK_CUR is synchronized against other SEEK_CURs, but not read/writes.
     83  * read/writes behave like SEEK_SET against seeks.
     84  */
     85 loff_t
     86 generic_file_llseek_size(struct file *file, loff_t offset, int whence,
     87 		loff_t maxsize, loff_t eof)
     88 {
     89 	switch (whence) {
     90 	case SEEK_END:
     91 		offset += eof;
     92 		break;
     93 	case SEEK_CUR:
     94 		/*
     95 		 * Here we special-case the lseek(fd, 0, SEEK_CUR)
     96 		 * position-querying operation.  Avoid rewriting the "same"
     97 		 * f_pos value back to the file because a concurrent read(),
     98 		 * write() or lseek() might have altered it
     99 		 */
    100 		if (offset == 0)
    101 			return file->f_pos;
    102 		/*
    103 		 * f_lock protects against read/modify/write race with other
    104 		 * SEEK_CURs. Note that parallel writes and reads behave
    105 		 * like SEEK_SET.
    106 		 */
    107 		spin_lock(&file->f_lock);
    108 		offset = vfs_setpos(file, file->f_pos + offset, maxsize);
    109 		spin_unlock(&file->f_lock);
    110 		return offset;
    111 	case SEEK_DATA:
    112 		/*
    113 		 * In the generic case the entire file is data, so as long as
    114 		 * offset isn't at the end of the file then the offset is data.
    115 		 */
    116 		if ((unsigned long long)offset >= eof)
    117 			return -ENXIO;
    118 		break;
    119 	case SEEK_HOLE:
    120 		/*
    121 		 * There is a virtual hole at the end of the file, so as long as
    122 		 * offset isn't i_size or larger, return i_size.
    123 		 */
    124 		if ((unsigned long long)offset >= eof)
    125 			return -ENXIO;
    126 		offset = eof;
    127 		break;
    128 	}
    129 
    130 	return vfs_setpos(file, offset, maxsize);
    131 }
    132 EXPORT_SYMBOL(generic_file_llseek_size);
    133 
    134 /**
    135  * generic_file_llseek - generic llseek implementation for regular files
    136  * @file:	file structure to seek on
    137  * @offset:	file offset to seek to
    138  * @whence:	type of seek
    139  *
    140  * This is a generic implemenation of ->llseek useable for all normal local
    141  * filesystems.  It just updates the file offset to the value specified by
    142  * @offset and @whence.
    143  */
    144 loff_t generic_file_llseek(struct file *file, loff_t offset, int whence)
    145 {
    146 	struct inode *inode = file->f_mapping->host;
    147 
    148 	return generic_file_llseek_size(file, offset, whence,
    149 					inode->i_sb->s_maxbytes,
    150 					i_size_read(inode));
    151 }
    152 EXPORT_SYMBOL(generic_file_llseek);
    153 
    154 /**
    155  * fixed_size_llseek - llseek implementation for fixed-sized devices
    156  * @file:	file structure to seek on
    157  * @offset:	file offset to seek to
    158  * @whence:	type of seek
    159  * @size:	size of the file
    160  *
    161  */
    162 loff_t fixed_size_llseek(struct file *file, loff_t offset, int whence, loff_t size)
    163 {
    164 	switch (whence) {
    165 	case SEEK_SET: case SEEK_CUR: case SEEK_END:
    166 		return generic_file_llseek_size(file, offset, whence,
    167 						size, size);
    168 	default:
    169 		return -EINVAL;
    170 	}
    171 }
    172 EXPORT_SYMBOL(fixed_size_llseek);
    173 
    174 /**
    175  * no_seek_end_llseek - llseek implementation for fixed-sized devices
    176  * @file:	file structure to seek on
    177  * @offset:	file offset to seek to
    178  * @whence:	type of seek
    179  *
    180  */
    181 loff_t no_seek_end_llseek(struct file *file, loff_t offset, int whence)
    182 {
    183 	switch (whence) {
    184 	case SEEK_SET: case SEEK_CUR:
    185 		return generic_file_llseek_size(file, offset, whence,
    186 						OFFSET_MAX, 0);
    187 	default:
    188 		return -EINVAL;
    189 	}
    190 }
    191 EXPORT_SYMBOL(no_seek_end_llseek);
    192 
    193 /**
    194  * no_seek_end_llseek_size - llseek implementation for fixed-sized devices
    195  * @file:	file structure to seek on
    196  * @offset:	file offset to seek to
    197  * @whence:	type of seek
    198  * @size:	maximal offset allowed
    199  *
    200  */
    201 loff_t no_seek_end_llseek_size(struct file *file, loff_t offset, int whence, loff_t size)
    202 {
    203 	switch (whence) {
    204 	case SEEK_SET: case SEEK_CUR:
    205 		return generic_file_llseek_size(file, offset, whence,
    206 						size, 0);
    207 	default:
    208 		return -EINVAL;
    209 	}
    210 }
    211 EXPORT_SYMBOL(no_seek_end_llseek_size);
    212 
    213 /**
    214  * noop_llseek - No Operation Performed llseek implementation
    215  * @file:	file structure to seek on
    216  * @offset:	file offset to seek to
    217  * @whence:	type of seek
    218  *
    219  * This is an implementation of ->llseek useable for the rare special case when
    220  * userspace expects the seek to succeed but the (device) file is actually not
    221  * able to perform the seek. In this case you use noop_llseek() instead of
    222  * falling back to the default implementation of ->llseek.
    223  */
    224 loff_t noop_llseek(struct file *file, loff_t offset, int whence)
    225 {
    226 	return file->f_pos;
    227 }
    228 EXPORT_SYMBOL(noop_llseek);
    229 
    230 loff_t no_llseek(struct file *file, loff_t offset, int whence)
    231 {
    232 	return -ESPIPE;
    233 }
    234 EXPORT_SYMBOL(no_llseek);
    235 
    236 loff_t default_llseek(struct file *file, loff_t offset, int whence)
    237 {
    238 	struct inode *inode = file_inode(file);
    239 	loff_t retval;
    240 
    241 	inode_lock(inode);
    242 	switch (whence) {
    243 		case SEEK_END:
    244 			offset += i_size_read(inode);
    245 			break;
    246 		case SEEK_CUR:
    247 			if (offset == 0) {
    248 				retval = file->f_pos;
    249 				goto out;
    250 			}
    251 			offset += file->f_pos;
    252 			break;
    253 		case SEEK_DATA:
    254 			/*
    255 			 * In the generic case the entire file is data, so as
    256 			 * long as offset isn't at the end of the file then the
    257 			 * offset is data.
    258 			 */
    259 			if (offset >= inode->i_size) {
    260 				retval = -ENXIO;
    261 				goto out;
    262 			}
    263 			break;
    264 		case SEEK_HOLE:
    265 			/*
    266 			 * There is a virtual hole at the end of the file, so
    267 			 * as long as offset isn't i_size or larger, return
    268 			 * i_size.
    269 			 */
    270 			if (offset >= inode->i_size) {
    271 				retval = -ENXIO;
    272 				goto out;
    273 			}
    274 			offset = inode->i_size;
    275 			break;
    276 	}
    277 	retval = -EINVAL;
    278 	if (offset >= 0 || unsigned_offsets(file)) {
    279 		if (offset != file->f_pos) {
    280 			file->f_pos = offset;
    281 			file->f_version = 0;
    282 		}
    283 		retval = offset;
    284 	}
    285 out:
    286 	inode_unlock(inode);
    287 	return retval;
    288 }
    289 EXPORT_SYMBOL(default_llseek);
    290 
    291 loff_t vfs_llseek(struct file *file, loff_t offset, int whence)
    292 {
    293 	loff_t (*fn)(struct file *, loff_t, int);
    294 
    295 	fn = no_llseek;
    296 	if (file->f_mode & FMODE_LSEEK) {
    297 		if (file->f_op->llseek)
    298 			fn = file->f_op->llseek;
    299 	}
    300 	return fn(file, offset, whence);
    301 }
    302 EXPORT_SYMBOL(vfs_llseek);
    303 
    304 off_t ksys_lseek(unsigned int fd, off_t offset, unsigned int whence)
    305 {
    306 	off_t retval;
    307 	struct fd f = fdget_pos(fd);
    308 	if (!f.file)
    309 		return -EBADF;
    310 
    311 	retval = -EINVAL;
    312 	if (whence <= SEEK_MAX) {
    313 		loff_t res = vfs_llseek(f.file, offset, whence);
    314 		retval = res;
    315 		if (res != (loff_t)retval)
    316 			retval = -EOVERFLOW;	/* LFS: should only happen on 32 bit platforms */
    317 	}
    318 	fdput_pos(f);
    319 	return retval;
    320 }
    321 
    322 SYSCALL_DEFINE3(lseek, unsigned int, fd, off_t, offset, unsigned int, whence)
    323 {
    324 	return ksys_lseek(fd, offset, whence);
    325 }
    326 
    327 #ifdef CONFIG_COMPAT
    328 COMPAT_SYSCALL_DEFINE3(lseek, unsigned int, fd, compat_off_t, offset, unsigned int, whence)
    329 {
    330 	return ksys_lseek(fd, offset, whence);
    331 }
    332 #endif
    333 
    334 #if !defined(CONFIG_64BIT) || defined(CONFIG_COMPAT)
    335 SYSCALL_DEFINE5(llseek, unsigned int, fd, unsigned long, offset_high,
    336 		unsigned long, offset_low, loff_t __user *, result,
    337 		unsigned int, whence)
    338 {
    339 	int retval;
    340 	struct fd f = fdget_pos(fd);
    341 	loff_t offset;
    342 
    343 	if (!f.file)
    344 		return -EBADF;
    345 
    346 	retval = -EINVAL;
    347 	if (whence > SEEK_MAX)
    348 		goto out_putf;
    349 
    350 	offset = vfs_llseek(f.file, ((loff_t) offset_high << 32) | offset_low,
    351 			whence);
    352 
    353 	retval = (int)offset;
    354 	if (offset >= 0) {
    355 		retval = -EFAULT;
    356 		if (!copy_to_user(result, &offset, sizeof(offset)))
    357 			retval = 0;
    358 	}
    359 out_putf:
    360 	fdput_pos(f);
    361 	return retval;
    362 }
    363 #endif
    364 
    365 int rw_verify_area(int read_write, struct file *file, const loff_t *ppos, size_t count)
    366 {
    367 	struct inode *inode;
    368 	loff_t pos;
    369 	int retval = -EINVAL;
    370 
    371 	inode = file_inode(file);
    372 	if (unlikely((ssize_t) count < 0))
    373 		return retval;
    374 	pos = *ppos;
    375 	if (unlikely(pos < 0)) {
    376 		if (!unsigned_offsets(file))
    377 			return retval;
    378 		if (count >= -pos) /* both values are in 0..LLONG_MAX */
    379 			return -EOVERFLOW;
    380 	} else if (unlikely((loff_t) (pos + count) < 0)) {
    381 		if (!unsigned_offsets(file))
    382 			return retval;
    383 	}
    384 
    385 	if (unlikely(inode->i_flctx && mandatory_lock(inode))) {
    386 		retval = locks_mandatory_area(inode, file, pos, pos + count - 1,
    387 				read_write == READ ? F_RDLCK : F_WRLCK);
    388 		if (retval < 0)
    389 			return retval;
    390 	}
    391 	return security_file_permission(file,
    392 				read_write == READ ? MAY_READ : MAY_WRITE);
    393 }
    394 
    395 static ssize_t new_sync_read(struct file *filp, char __user *buf, size_t len, loff_t *ppos)
    396 {
    397 	struct iovec iov = { .iov_base = buf, .iov_len = len };
    398 	struct kiocb kiocb;
    399 	struct iov_iter iter;
    400 	ssize_t ret;
    401 
    402 	init_sync_kiocb(&kiocb, filp);
    403 	kiocb.ki_pos = *ppos;
    404 	iov_iter_init(&iter, READ, &iov, 1, len);
    405 
    406 	ret = call_read_iter(filp, &kiocb, &iter);
    407 	BUG_ON(ret == -EIOCBQUEUED);
    408 	*ppos = kiocb.ki_pos;
    409 	return ret;
    410 }
    411 
    412 ssize_t __vfs_read(struct file *file, char __user *buf, size_t count,
    413 		   loff_t *pos)
    414 {
    415 	if (file->f_op->read)
    416 		return file->f_op->read(file, buf, count, pos);
    417 	else if (file->f_op->read_iter)
    418 		return new_sync_read(file, buf, count, pos);
    419 	else
    420 		return -EINVAL;
    421 }
    422 
    423 ssize_t kernel_read(struct file *file, void *buf, size_t count, loff_t *pos)
    424 {
    425 	mm_segment_t old_fs;
    426 	ssize_t result;
    427 
    428 	old_fs = get_fs();
    429 	set_fs(KERNEL_DS);
    430 	/* The cast to a user pointer is valid due to the set_fs() */
    431 	result = vfs_read(file, (void __user *)buf, count, pos);
    432 	set_fs(old_fs);
    433 	return result;
    434 }
    435 EXPORT_SYMBOL(kernel_read);
    436 
    437 ssize_t vfs_read(struct file *file, char __user *buf, size_t count, loff_t *pos)
    438 {
    439 	ssize_t ret;
    440 
    441 	if (!(file->f_mode & FMODE_READ))
    442 		return -EBADF;
    443 	if (!(file->f_mode & FMODE_CAN_READ))
    444 		return -EINVAL;
    445 	if (unlikely(!access_ok(buf, count)))
    446 		return -EFAULT;
    447 
    448 	ret = rw_verify_area(READ, file, pos, count);
    449 	if (!ret) {
    450 		if (count > MAX_RW_COUNT)
    451 			count =  MAX_RW_COUNT;
    452 		ret = __vfs_read(file, buf, count, pos);
    453 		if (ret > 0) {
    454 			fsnotify_access(file);
    455 			add_rchar(current, ret);
    456 		}
    457 		inc_syscr(current);
    458 	}
    459 
    460 	return ret;
    461 }
    462 
    463 static ssize_t new_sync_write(struct file *filp, const char __user *buf, size_t len, loff_t *ppos)
    464 {
    465 	struct iovec iov = { .iov_base = (void __user *)buf, .iov_len = len };
    466 	struct kiocb kiocb;
    467 	struct iov_iter iter;
    468 	ssize_t ret;
    469 
    470 	init_sync_kiocb(&kiocb, filp);
    471 	kiocb.ki_pos = *ppos;
    472 	iov_iter_init(&iter, WRITE, &iov, 1, len);
    473 
    474 	ret = call_write_iter(filp, &kiocb, &iter);
    475 	BUG_ON(ret == -EIOCBQUEUED);
    476 	if (ret > 0)
    477 		*ppos = kiocb.ki_pos;
    478 	return ret;
    479 }
    480 
    481 static ssize_t __vfs_write(struct file *file, const char __user *p,
    482 			   size_t count, loff_t *pos)
    483 {
    484 	if (file->f_op->write)
    485 		return file->f_op->write(file, p, count, pos);
    486 	else if (file->f_op->write_iter)
    487 		return new_sync_write(file, p, count, pos);
    488 	else
    489 		return -EINVAL;
    490 }
    491 
    492 ssize_t __kernel_write(struct file *file, const void *buf, size_t count, loff_t *pos)
    493 {
    494 	mm_segment_t old_fs;
    495 	const char __user *p;
    496 	ssize_t ret;
    497 
    498 	if (!(file->f_mode & FMODE_CAN_WRITE))
    499 		return -EINVAL;
    500 
    501 	old_fs = get_fs();
    502 	set_fs(KERNEL_DS);
    503 	p = (__force const char __user *)buf;
    504 	if (count > MAX_RW_COUNT)
    505 		count =  MAX_RW_COUNT;
    506 	ret = __vfs_write(file, p, count, pos);
    507 	set_fs(old_fs);
    508 	if (ret > 0) {
    509 		fsnotify_modify(file);
    510 		add_wchar(current, ret);
    511 	}
    512 	inc_syscw(current);
    513 	return ret;
    514 }
    515 EXPORT_SYMBOL(__kernel_write);
    516 
    517 ssize_t kernel_write(struct file *file, const void *buf, size_t count,
    518 			    loff_t *pos)
    519 {
    520 	mm_segment_t old_fs;
    521 	ssize_t res;
    522 
    523 	old_fs = get_fs();
    524 	set_fs(KERNEL_DS);
    525 	/* The cast to a user pointer is valid due to the set_fs() */
    526 	res = vfs_write(file, (__force const char __user *)buf, count, pos);
    527 	set_fs(old_fs);
    528 
    529 	return res;
    530 }
    531 EXPORT_SYMBOL(kernel_write);
    532 
    533 ssize_t vfs_write(struct file *file, const char __user *buf, size_t count, loff_t *pos)
    534 {
    535 	ssize_t ret;
    536 
    537 	if (!(file->f_mode & FMODE_WRITE))
    538 		return -EBADF;
    539 	if (!(file->f_mode & FMODE_CAN_WRITE))
    540 		return -EINVAL;
    541 	if (unlikely(!access_ok(buf, count)))
    542 		return -EFAULT;
    543 
    544 	ret = rw_verify_area(WRITE, file, pos, count);
    545 	if (!ret) {
    546 		if (count > MAX_RW_COUNT)
    547 			count =  MAX_RW_COUNT;
    548 		file_start_write(file);
    549 		ret = __vfs_write(file, buf, count, pos);
    550 		if (ret > 0) {
    551 			fsnotify_modify(file);
    552 			add_wchar(current, ret);
    553 		}
    554 		inc_syscw(current);
    555 		file_end_write(file);
    556 	}
    557 
    558 	return ret;
    559 }
    560 
    561 static inline loff_t file_pos_read(struct file *file)
    562 {
    563 	return file->f_mode & FMODE_STREAM ? 0 : file->f_pos;
    564 }
    565 
    566 static inline void file_pos_write(struct file *file, loff_t pos)
    567 {
    568 	if ((file->f_mode & FMODE_STREAM) == 0)
    569 		file->f_pos = pos;
    570 }
    571 
    572 ssize_t ksys_read(unsigned int fd, char __user *buf, size_t count)
    573 {
    574 	struct fd f = fdget_pos(fd);
    575 	ssize_t ret = -EBADF;
    576 
    577 	if (f.file) {
    578 		loff_t pos = file_pos_read(f.file);
    579 		ret = vfs_read(f.file, buf, count, &pos);
    580 		if (ret >= 0)
    581 			file_pos_write(f.file, pos);
    582 		fdput_pos(f);
    583 	}
    584 	return ret;
    585 }
    586 
    587 SYSCALL_DEFINE3(read, unsigned int, fd, char __user *, buf, size_t, count)
    588 {
    589 	return ksys_read(fd, buf, count);
    590 }
    591 
    592 ssize_t ksys_write(unsigned int fd, const char __user *buf, size_t count)
    593 {
    594 	struct fd f = fdget_pos(fd);
    595 	ssize_t ret = -EBADF;
    596 
    597 	if (f.file) {
    598 		loff_t pos = file_pos_read(f.file);
    599 		ret = vfs_write(f.file, buf, count, &pos);
    600 		if (ret >= 0)
    601 			file_pos_write(f.file, pos);
    602 		fdput_pos(f);
    603 	}
    604 
    605 	return ret;
    606 }
    607 
    608 SYSCALL_DEFINE3(write, unsigned int, fd, const char __user *, buf,
    609 		size_t, count)
    610 {
    611 	return ksys_write(fd, buf, count);
    612 }
    613 
    614 ssize_t ksys_pread64(unsigned int fd, char __user *buf, size_t count,
    615 		     loff_t pos)
    616 {
    617 	struct fd f;
    618 	ssize_t ret = -EBADF;
    619 
    620 	if (pos < 0)
    621 		return -EINVAL;
    622 
    623 	f = fdget(fd);
    624 	if (f.file) {
    625 		ret = -ESPIPE;
    626 		if (f.file->f_mode & FMODE_PREAD)
    627 			ret = vfs_read(f.file, buf, count, &pos);
    628 		fdput(f);
    629 	}
    630 
    631 	return ret;
    632 }
    633 
    634 SYSCALL_DEFINE4(pread64, unsigned int, fd, char __user *, buf,
    635 			size_t, count, loff_t, pos)
    636 {
    637 	return ksys_pread64(fd, buf, count, pos);
    638 }
    639 
    640 ssize_t ksys_pwrite64(unsigned int fd, const char __user *buf,
    641 		      size_t count, loff_t pos)
    642 {
    643 	struct fd f;
    644 	ssize_t ret = -EBADF;
    645 
    646 	if (pos < 0)
    647 		return -EINVAL;
    648 
    649 	f = fdget(fd);
    650 	if (f.file) {
    651 		ret = -ESPIPE;
    652 		if (f.file->f_mode & FMODE_PWRITE)  
    653 			ret = vfs_write(f.file, buf, count, &pos);
    654 		fdput(f);
    655 	}
    656 
    657 	return ret;
    658 }
    659 
    660 SYSCALL_DEFINE4(pwrite64, unsigned int, fd, const char __user *, buf,
    661 			 size_t, count, loff_t, pos)
    662 {
    663 	return ksys_pwrite64(fd, buf, count, pos);
    664 }
    665 
    666 static ssize_t do_iter_readv_writev(struct file *filp, struct iov_iter *iter,
    667 		loff_t *ppos, int type, rwf_t flags)
    668 {
    669 	struct kiocb kiocb;
    670 	ssize_t ret;
    671 
    672 	init_sync_kiocb(&kiocb, filp);
    673 	ret = kiocb_set_rw_flags(&kiocb, flags);
    674 	if (ret)
    675 		return ret;
    676 	kiocb.ki_pos = *ppos;
    677 
    678 	if (type == READ)
    679 		ret = call_read_iter(filp, &kiocb, iter);
    680 	else
    681 		ret = call_write_iter(filp, &kiocb, iter);
    682 	BUG_ON(ret == -EIOCBQUEUED);
    683 	*ppos = kiocb.ki_pos;
    684 	return ret;
    685 }
    686 
    687 /* Do it by hand, with file-ops */
    688 static ssize_t do_loop_readv_writev(struct file *filp, struct iov_iter *iter,
    689 		loff_t *ppos, int type, rwf_t flags)
    690 {
    691 	ssize_t ret = 0;
    692 
    693 	if (flags & ~RWF_HIPRI)
    694 		return -EOPNOTSUPP;
    695 
    696 	while (iov_iter_count(iter)) {
    697 		struct iovec iovec = iov_iter_iovec(iter);
    698 		ssize_t nr;
    699 
    700 		if (type == READ) {
    701 			nr = filp->f_op->read(filp, iovec.iov_base,
    702 					      iovec.iov_len, ppos);
    703 		} else {
    704 			nr = filp->f_op->write(filp, iovec.iov_base,
    705 					       iovec.iov_len, ppos);
    706 		}
    707 
    708 		if (nr < 0) {
    709 			if (!ret)
    710 				ret = nr;
    711 			break;
    712 		}
    713 		ret += nr;
    714 		if (nr != iovec.iov_len)
    715 			break;
    716 		iov_iter_advance(iter, nr);
    717 	}
    718 
    719 	return ret;
    720 }
    721 
    722 /**
    723  * rw_copy_check_uvector() - Copy an array of &struct iovec from userspace
    724  *     into the kernel and check that it is valid.
    725  *
    726  * @type: One of %CHECK_IOVEC_ONLY, %READ, or %WRITE.
    727  * @uvector: Pointer to the userspace array.
    728  * @nr_segs: Number of elements in userspace array.
    729  * @fast_segs: Number of elements in @fast_pointer.
    730  * @fast_pointer: Pointer to (usually small on-stack) kernel array.
    731  * @ret_pointer: (output parameter) Pointer to a variable that will point to
    732  *     either @fast_pointer, a newly allocated kernel array, or NULL,
    733  *     depending on which array was used.
    734  *
    735  * This function copies an array of &struct iovec of @nr_segs from
    736  * userspace into the kernel and checks that each element is valid (e.g.
    737  * it does not point to a kernel address or cause overflow by being too
    738  * large, etc.).
    739  *
    740  * As an optimization, the caller may provide a pointer to a small
    741  * on-stack array in @fast_pointer, typically %UIO_FASTIOV elements long
    742  * (the size of this array, or 0 if unused, should be given in @fast_segs).
    743  *
    744  * @ret_pointer will always point to the array that was used, so the
    745  * caller must take care not to call kfree() on it e.g. in case the
    746  * @fast_pointer array was used and it was allocated on the stack.
    747  *
    748  * Return: The total number of bytes covered by the iovec array on success
    749  *   or a negative error code on error.
    750  */
    751 ssize_t rw_copy_check_uvector(int type, const struct iovec __user * uvector,
    752 			      unsigned long nr_segs, unsigned long fast_segs,
    753 			      struct iovec *fast_pointer,
    754 			      struct iovec **ret_pointer)
    755 {
    756 	unsigned long seg;
    757 	ssize_t ret;
    758 	struct iovec *iov = fast_pointer;
    759 
    760 	/*
    761 	 * SuS says "The readv() function *may* fail if the iovcnt argument
    762 	 * was less than or equal to 0, or greater than {IOV_MAX}.  Linux has
    763 	 * traditionally returned zero for zero segments, so...
    764 	 */
    765 	if (nr_segs == 0) {
    766 		ret = 0;
    767 		goto out;
    768 	}
    769 
    770 	/*
    771 	 * First get the "struct iovec" from user memory and
    772 	 * verify all the pointers
    773 	 */
    774 	if (nr_segs > UIO_MAXIOV) {
    775 		ret = -EINVAL;
    776 		goto out;
    777 	}
    778 	if (nr_segs > fast_segs) {
    779 		iov = kmalloc_array(nr_segs, sizeof(struct iovec), GFP_KERNEL);
    780 		if (iov == NULL) {
    781 			ret = -ENOMEM;
    782 			goto out;
    783 		}
    784 	}
    785 	if (copy_from_user(iov, uvector, nr_segs*sizeof(*uvector))) {
    786 		ret = -EFAULT;
    787 		goto out;
    788 	}
    789 
    790 	/*
    791 	 * According to the Single Unix Specification we should return EINVAL
    792 	 * if an element length is < 0 when cast to ssize_t or if the
    793 	 * total length would overflow the ssize_t return value of the
    794 	 * system call.
    795 	 *
    796 	 * Linux caps all read/write calls to MAX_RW_COUNT, and avoids the
    797 	 * overflow case.
    798 	 */
    799 	ret = 0;
    800 	for (seg = 0; seg < nr_segs; seg++) {
    801 		void __user *buf = iov[seg].iov_base;
    802 		ssize_t len = (ssize_t)iov[seg].iov_len;
    803 
    804 		/* see if we we're about to use an invalid len or if
    805 		 * it's about to overflow ssize_t */
    806 		if (len < 0) {
    807 			ret = -EINVAL;
    808 			goto out;
    809 		}
    810 		if (type >= 0
    811 		    && unlikely(!access_ok(buf, len))) {
    812 			ret = -EFAULT;
    813 			goto out;
    814 		}
    815 		if (len > MAX_RW_COUNT - ret) {
    816 			len = MAX_RW_COUNT - ret;
    817 			iov[seg].iov_len = len;
    818 		}
    819 		ret += len;
    820 	}
    821 out:
    822 	*ret_pointer = iov;
    823 	return ret;
    824 }
    825 
    826 #ifdef CONFIG_COMPAT
    827 ssize_t compat_rw_copy_check_uvector(int type,
    828 		const struct compat_iovec __user *uvector, unsigned long nr_segs,
    829 		unsigned long fast_segs, struct iovec *fast_pointer,
    830 		struct iovec **ret_pointer)
    831 {
    832 	compat_ssize_t tot_len;
    833 	struct iovec *iov = *ret_pointer = fast_pointer;
    834 	ssize_t ret = 0;
    835 	int seg;
    836 
    837 	/*
    838 	 * SuS says "The readv() function *may* fail if the iovcnt argument
    839 	 * was less than or equal to 0, or greater than {IOV_MAX}.  Linux has
    840 	 * traditionally returned zero for zero segments, so...
    841 	 */
    842 	if (nr_segs == 0)
    843 		goto out;
    844 
    845 	ret = -EINVAL;
    846 	if (nr_segs > UIO_MAXIOV)
    847 		goto out;
    848 	if (nr_segs > fast_segs) {
    849 		ret = -ENOMEM;
    850 		iov = kmalloc_array(nr_segs, sizeof(struct iovec), GFP_KERNEL);
    851 		if (iov == NULL)
    852 			goto out;
    853 	}
    854 	*ret_pointer = iov;
    855 
    856 	ret = -EFAULT;
    857 	if (!access_ok(uvector, nr_segs*sizeof(*uvector)))
    858 		goto out;
    859 
    860 	/*
    861 	 * Single unix specification:
    862 	 * We should -EINVAL if an element length is not >= 0 and fitting an
    863 	 * ssize_t.
    864 	 *
    865 	 * In Linux, the total length is limited to MAX_RW_COUNT, there is
    866 	 * no overflow possibility.
    867 	 */
    868 	tot_len = 0;
    869 	ret = -EINVAL;
    870 	for (seg = 0; seg < nr_segs; seg++) {
    871 		compat_uptr_t buf;
    872 		compat_ssize_t len;
    873 
    874 		if (__get_user(len, &uvector->iov_len) ||
    875 		   __get_user(buf, &uvector->iov_base)) {
    876 			ret = -EFAULT;
    877 			goto out;
    878 		}
    879 		if (len < 0)	/* size_t not fitting in compat_ssize_t .. */
    880 			goto out;
    881 		if (type >= 0 &&
    882 		    !access_ok(compat_ptr(buf), len)) {
    883 			ret = -EFAULT;
    884 			goto out;
    885 		}
    886 		if (len > MAX_RW_COUNT - tot_len)
    887 			len = MAX_RW_COUNT - tot_len;
    888 		tot_len += len;
    889 		iov->iov_base = compat_ptr(buf);
    890 		iov->iov_len = (compat_size_t) len;
    891 		uvector++;
    892 		iov++;
    893 	}
    894 	ret = tot_len;
    895 
    896 out:
    897 	return ret;
    898 }
    899 #endif
    900 
    901 static ssize_t do_iter_read(struct file *file, struct iov_iter *iter,
    902 		loff_t *pos, rwf_t flags)
    903 {
    904 	size_t tot_len;
    905 	ssize_t ret = 0;
    906 
    907 	if (!(file->f_mode & FMODE_READ))
    908 		return -EBADF;
    909 	if (!(file->f_mode & FMODE_CAN_READ))
    910 		return -EINVAL;
    911 
    912 	tot_len = iov_iter_count(iter);
    913 	if (!tot_len)
    914 		goto out;
    915 	ret = rw_verify_area(READ, file, pos, tot_len);
    916 	if (ret < 0)
    917 		return ret;
    918 
    919 	if (file->f_op->read_iter)
    920 		ret = do_iter_readv_writev(file, iter, pos, READ, flags);
    921 	else
    922 		ret = do_loop_readv_writev(file, iter, pos, READ, flags);
    923 out:
    924 	if (ret >= 0)
    925 		fsnotify_access(file);
    926 	return ret;
    927 }
    928 
    929 ssize_t vfs_iter_read(struct file *file, struct iov_iter *iter, loff_t *ppos,
    930 		rwf_t flags)
    931 {
    932 	if (!file->f_op->read_iter)
    933 		return -EINVAL;
    934 	return do_iter_read(file, iter, ppos, flags);
    935 }
    936 EXPORT_SYMBOL(vfs_iter_read);
    937 
    938 static ssize_t do_iter_write(struct file *file, struct iov_iter *iter,
    939 		loff_t *pos, rwf_t flags)
    940 {
    941 	size_t tot_len;
    942 	ssize_t ret = 0;
    943 
    944 	if (!(file->f_mode & FMODE_WRITE))
    945 		return -EBADF;
    946 	if (!(file->f_mode & FMODE_CAN_WRITE))
    947 		return -EINVAL;
    948 
    949 	tot_len = iov_iter_count(iter);
    950 	if (!tot_len)
    951 		return 0;
    952 	ret = rw_verify_area(WRITE, file, pos, tot_len);
    953 	if (ret < 0)
    954 		return ret;
    955 
    956 	if (file->f_op->write_iter)
    957 		ret = do_iter_readv_writev(file, iter, pos, WRITE, flags);
    958 	else
    959 		ret = do_loop_readv_writev(file, iter, pos, WRITE, flags);
    960 	if (ret > 0)
    961 		fsnotify_modify(file);
    962 	return ret;
    963 }
    964 
    965 ssize_t vfs_iter_write(struct file *file, struct iov_iter *iter, loff_t *ppos,
    966 		rwf_t flags)
    967 {
    968 	if (!file->f_op->write_iter)
    969 		return -EINVAL;
    970 	return do_iter_write(file, iter, ppos, flags);
    971 }
    972 EXPORT_SYMBOL(vfs_iter_write);
    973 
    974 ssize_t vfs_readv(struct file *file, const struct iovec __user *vec,
    975 		  unsigned long vlen, loff_t *pos, rwf_t flags)
    976 {
    977 	struct iovec iovstack[UIO_FASTIOV];
    978 	struct iovec *iov = iovstack;
    979 	struct iov_iter iter;
    980 	ssize_t ret;
    981 
    982 	ret = import_iovec(READ, vec, vlen, ARRAY_SIZE(iovstack), &iov, &iter);
    983 	if (ret >= 0) {
    984 		ret = do_iter_read(file, &iter, pos, flags);
    985 		kfree(iov);
    986 	}
    987 
    988 	return ret;
    989 }
    990 
    991 static ssize_t vfs_writev(struct file *file, const struct iovec __user *vec,
    992 		   unsigned long vlen, loff_t *pos, rwf_t flags)
    993 {
    994 	struct iovec iovstack[UIO_FASTIOV];
    995 	struct iovec *iov = iovstack;
    996 	struct iov_iter iter;
    997 	ssize_t ret;
    998 
    999 	ret = import_iovec(WRITE, vec, vlen, ARRAY_SIZE(iovstack), &iov, &iter);
   1000 	if (ret >= 0) {
   1001 		file_start_write(file);
   1002 		ret = do_iter_write(file, &iter, pos, flags);
   1003 		file_end_write(file);
   1004 		kfree(iov);
   1005 	}
   1006 	return ret;
   1007 }
   1008 
   1009 __static_ukl ssize_t do_readv(unsigned long fd, const struct iovec __user *vec,
   1010 			unsigned long vlen, rwf_t flags)
   1011 {
   1012 	struct fd f = fdget_pos(fd);
   1013 	ssize_t ret = -EBADF;
   1014 
   1015 	if (f.file) {
   1016 		loff_t pos = file_pos_read(f.file);
   1017 		ret = vfs_readv(f.file, vec, vlen, &pos, flags);
   1018 		if (ret >= 0)
   1019 			file_pos_write(f.file, pos);
   1020 		fdput_pos(f);
   1021 	}
   1022 
   1023 	if (ret > 0)
   1024 		add_rchar(current, ret);
   1025 	inc_syscr(current);
   1026 	return ret;
   1027 }
   1028 
   1029 __static_ukl ssize_t do_writev(unsigned long fd, const struct iovec __user *vec,
   1030 			 unsigned long vlen, rwf_t flags)
   1031 {
   1032 	struct fd f = fdget_pos(fd);
   1033 	ssize_t ret = -EBADF;
   1034 
   1035 	if (f.file) {
   1036 		loff_t pos = file_pos_read(f.file);
   1037 		ret = vfs_writev(f.file, vec, vlen, &pos, flags);
   1038 		if (ret >= 0)
   1039 			file_pos_write(f.file, pos);
   1040 		fdput_pos(f);
   1041 	}
   1042 
   1043 	if (ret > 0)
   1044 		add_wchar(current, ret);
   1045 	inc_syscw(current);
   1046 	return ret;
   1047 }
   1048 
   1049 static inline loff_t pos_from_hilo(unsigned long high, unsigned long low)
   1050 {
   1051 #define HALF_LONG_BITS (BITS_PER_LONG / 2)
   1052 	return (((loff_t)high << HALF_LONG_BITS) << HALF_LONG_BITS) | low;
   1053 }
   1054 
   1055 static ssize_t do_preadv(unsigned long fd, const struct iovec __user *vec,
   1056 			 unsigned long vlen, loff_t pos, rwf_t flags)
   1057 {
   1058 	struct fd f;
   1059 	ssize_t ret = -EBADF;
   1060 
   1061 	if (pos < 0)
   1062 		return -EINVAL;
   1063 
   1064 	f = fdget(fd);
   1065 	if (f.file) {
   1066 		ret = -ESPIPE;
   1067 		if (f.file->f_mode & FMODE_PREAD)
   1068 			ret = vfs_readv(f.file, vec, vlen, &pos, flags);
   1069 		fdput(f);
   1070 	}
   1071 
   1072 	if (ret > 0)
   1073 		add_rchar(current, ret);
   1074 	inc_syscr(current);
   1075 	return ret;
   1076 }
   1077 
   1078 static ssize_t do_pwritev(unsigned long fd, const struct iovec __user *vec,
   1079 			  unsigned long vlen, loff_t pos, rwf_t flags)
   1080 {
   1081 	struct fd f;
   1082 	ssize_t ret = -EBADF;
   1083 
   1084 	if (pos < 0)
   1085 		return -EINVAL;
   1086 
   1087 	f = fdget(fd);
   1088 	if (f.file) {
   1089 		ret = -ESPIPE;
   1090 		if (f.file->f_mode & FMODE_PWRITE)
   1091 			ret = vfs_writev(f.file, vec, vlen, &pos, flags);
   1092 		fdput(f);
   1093 	}
   1094 
   1095 	if (ret > 0)
   1096 		add_wchar(current, ret);
   1097 	inc_syscw(current);
   1098 	return ret;
   1099 }
   1100 
   1101 SYSCALL_DEFINE3(readv, unsigned long, fd, const struct iovec __user *, vec,
   1102 		unsigned long, vlen)
   1103 {
   1104 	return do_readv(fd, vec, vlen, 0);
   1105 }
   1106 
   1107 SYSCALL_DEFINE3(writev, unsigned long, fd, const struct iovec __user *, vec,
   1108 		unsigned long, vlen)
   1109 {
   1110 	return do_writev(fd, vec, vlen, 0);
   1111 }
   1112 
   1113 SYSCALL_DEFINE5(preadv, unsigned long, fd, const struct iovec __user *, vec,
   1114 		unsigned long, vlen, unsigned long, pos_l, unsigned long, pos_h)
   1115 {
   1116 	loff_t pos = pos_from_hilo(pos_h, pos_l);
   1117 
   1118 	return do_preadv(fd, vec, vlen, pos, 0);
   1119 }
   1120 
   1121 SYSCALL_DEFINE6(preadv2, unsigned long, fd, const struct iovec __user *, vec,
   1122 		unsigned long, vlen, unsigned long, pos_l, unsigned long, pos_h,
   1123 		rwf_t, flags)
   1124 {
   1125 	loff_t pos = pos_from_hilo(pos_h, pos_l);
   1126 
   1127 	if (pos == -1)
   1128 		return do_readv(fd, vec, vlen, flags);
   1129 
   1130 	return do_preadv(fd, vec, vlen, pos, flags);
   1131 }
   1132 
   1133 SYSCALL_DEFINE5(pwritev, unsigned long, fd, const struct iovec __user *, vec,
   1134 		unsigned long, vlen, unsigned long, pos_l, unsigned long, pos_h)
   1135 {
   1136 	loff_t pos = pos_from_hilo(pos_h, pos_l);
   1137 
   1138 	return do_pwritev(fd, vec, vlen, pos, 0);
   1139 }
   1140 
   1141 SYSCALL_DEFINE6(pwritev2, unsigned long, fd, const struct iovec __user *, vec,
   1142 		unsigned long, vlen, unsigned long, pos_l, unsigned long, pos_h,
   1143 		rwf_t, flags)
   1144 {
   1145 	loff_t pos = pos_from_hilo(pos_h, pos_l);
   1146 
   1147 	if (pos == -1)
   1148 		return do_writev(fd, vec, vlen, flags);
   1149 
   1150 	return do_pwritev(fd, vec, vlen, pos, flags);
   1151 }
   1152 
   1153 #ifdef CONFIG_COMPAT
   1154 static size_t compat_readv(struct file *file,
   1155 			   const struct compat_iovec __user *vec,
   1156 			   unsigned long vlen, loff_t *pos, rwf_t flags)
   1157 {
   1158 	struct iovec iovstack[UIO_FASTIOV];
   1159 	struct iovec *iov = iovstack;
   1160 	struct iov_iter iter;
   1161 	ssize_t ret;
   1162 
   1163 	ret = compat_import_iovec(READ, vec, vlen, UIO_FASTIOV, &iov, &iter);
   1164 	if (ret >= 0) {
   1165 		ret = do_iter_read(file, &iter, pos, flags);
   1166 		kfree(iov);
   1167 	}
   1168 	if (ret > 0)
   1169 		add_rchar(current, ret);
   1170 	inc_syscr(current);
   1171 	return ret;
   1172 }
   1173 
   1174 static size_t do_compat_readv(compat_ulong_t fd,
   1175 				 const struct compat_iovec __user *vec,
   1176 				 compat_ulong_t vlen, rwf_t flags)
   1177 {
   1178 	struct fd f = fdget_pos(fd);
   1179 	ssize_t ret;
   1180 	loff_t pos;
   1181 
   1182 	if (!f.file)
   1183 		return -EBADF;
   1184 	pos = f.file->f_pos;
   1185 	ret = compat_readv(f.file, vec, vlen, &pos, flags);
   1186 	if (ret >= 0)
   1187 		f.file->f_pos = pos;
   1188 	fdput_pos(f);
   1189 	return ret;
   1190 
   1191 }
   1192 
   1193 COMPAT_SYSCALL_DEFINE3(readv, compat_ulong_t, fd,
   1194 		const struct compat_iovec __user *,vec,
   1195 		compat_ulong_t, vlen)
   1196 {
   1197 	return do_compat_readv(fd, vec, vlen, 0);
   1198 }
   1199 
   1200 static long do_compat_preadv64(unsigned long fd,
   1201 				  const struct compat_iovec __user *vec,
   1202 				  unsigned long vlen, loff_t pos, rwf_t flags)
   1203 {
   1204 	struct fd f;
   1205 	ssize_t ret;
   1206 
   1207 	if (pos < 0)
   1208 		return -EINVAL;
   1209 	f = fdget(fd);
   1210 	if (!f.file)
   1211 		return -EBADF;
   1212 	ret = -ESPIPE;
   1213 	if (f.file->f_mode & FMODE_PREAD)
   1214 		ret = compat_readv(f.file, vec, vlen, &pos, flags);
   1215 	fdput(f);
   1216 	return ret;
   1217 }
   1218 
   1219 #ifdef __ARCH_WANT_COMPAT_SYS_PREADV64
   1220 COMPAT_SYSCALL_DEFINE4(preadv64, unsigned long, fd,
   1221 		const struct compat_iovec __user *,vec,
   1222 		unsigned long, vlen, loff_t, pos)
   1223 {
   1224 	return do_compat_preadv64(fd, vec, vlen, pos, 0);
   1225 }
   1226 #endif
   1227 
   1228 COMPAT_SYSCALL_DEFINE5(preadv, compat_ulong_t, fd,
   1229 		const struct compat_iovec __user *,vec,
   1230 		compat_ulong_t, vlen, u32, pos_low, u32, pos_high)
   1231 {
   1232 	loff_t pos = ((loff_t)pos_high << 32) | pos_low;
   1233 
   1234 	return do_compat_preadv64(fd, vec, vlen, pos, 0);
   1235 }
   1236 
   1237 #ifdef __ARCH_WANT_COMPAT_SYS_PREADV64V2
   1238 COMPAT_SYSCALL_DEFINE5(preadv64v2, unsigned long, fd,
   1239 		const struct compat_iovec __user *,vec,
   1240 		unsigned long, vlen, loff_t, pos, rwf_t, flags)
   1241 {
   1242 	if (pos == -1)
   1243 		return do_compat_readv(fd, vec, vlen, flags);
   1244 
   1245 	return do_compat_preadv64(fd, vec, vlen, pos, flags);
   1246 }
   1247 #endif
   1248 
   1249 COMPAT_SYSCALL_DEFINE6(preadv2, compat_ulong_t, fd,
   1250 		const struct compat_iovec __user *,vec,
   1251 		compat_ulong_t, vlen, u32, pos_low, u32, pos_high,
   1252 		rwf_t, flags)
   1253 {
   1254 	loff_t pos = ((loff_t)pos_high << 32) | pos_low;
   1255 
   1256 	if (pos == -1)
   1257 		return do_compat_readv(fd, vec, vlen, flags);
   1258 
   1259 	return do_compat_preadv64(fd, vec, vlen, pos, flags);
   1260 }
   1261 
   1262 static size_t compat_writev(struct file *file,
   1263 			    const struct compat_iovec __user *vec,
   1264 			    unsigned long vlen, loff_t *pos, rwf_t flags)
   1265 {
   1266 	struct iovec iovstack[UIO_FASTIOV];
   1267 	struct iovec *iov = iovstack;
   1268 	struct iov_iter iter;
   1269 	ssize_t ret;
   1270 
   1271 	ret = compat_import_iovec(WRITE, vec, vlen, UIO_FASTIOV, &iov, &iter);
   1272 	if (ret >= 0) {
   1273 		file_start_write(file);
   1274 		ret = do_iter_write(file, &iter, pos, flags);
   1275 		file_end_write(file);
   1276 		kfree(iov);
   1277 	}
   1278 	if (ret > 0)
   1279 		add_wchar(current, ret);
   1280 	inc_syscw(current);
   1281 	return ret;
   1282 }
   1283 
   1284 static size_t do_compat_writev(compat_ulong_t fd,
   1285 				  const struct compat_iovec __user* vec,
   1286 				  compat_ulong_t vlen, rwf_t flags)
   1287 {
   1288 	struct fd f = fdget_pos(fd);
   1289 	ssize_t ret;
   1290 	loff_t pos;
   1291 
   1292 	if (!f.file)
   1293 		return -EBADF;
   1294 	pos = f.file->f_pos;
   1295 	ret = compat_writev(f.file, vec, vlen, &pos, flags);
   1296 	if (ret >= 0)
   1297 		f.file->f_pos = pos;
   1298 	fdput_pos(f);
   1299 	return ret;
   1300 }
   1301 
   1302 COMPAT_SYSCALL_DEFINE3(writev, compat_ulong_t, fd,
   1303 		const struct compat_iovec __user *, vec,
   1304 		compat_ulong_t, vlen)
   1305 {
   1306 	return do_compat_writev(fd, vec, vlen, 0);
   1307 }
   1308 
   1309 static long do_compat_pwritev64(unsigned long fd,
   1310 				   const struct compat_iovec __user *vec,
   1311 				   unsigned long vlen, loff_t pos, rwf_t flags)
   1312 {
   1313 	struct fd f;
   1314 	ssize_t ret;
   1315 
   1316 	if (pos < 0)
   1317 		return -EINVAL;
   1318 	f = fdget(fd);
   1319 	if (!f.file)
   1320 		return -EBADF;
   1321 	ret = -ESPIPE;
   1322 	if (f.file->f_mode & FMODE_PWRITE)
   1323 		ret = compat_writev(f.file, vec, vlen, &pos, flags);
   1324 	fdput(f);
   1325 	return ret;
   1326 }
   1327 
   1328 #ifdef __ARCH_WANT_COMPAT_SYS_PWRITEV64
   1329 COMPAT_SYSCALL_DEFINE4(pwritev64, unsigned long, fd,
   1330 		const struct compat_iovec __user *,vec,
   1331 		unsigned long, vlen, loff_t, pos)
   1332 {
   1333 	return do_compat_pwritev64(fd, vec, vlen, pos, 0);
   1334 }
   1335 #endif
   1336 
   1337 COMPAT_SYSCALL_DEFINE5(pwritev, compat_ulong_t, fd,
   1338 		const struct compat_iovec __user *,vec,
   1339 		compat_ulong_t, vlen, u32, pos_low, u32, pos_high)
   1340 {
   1341 	loff_t pos = ((loff_t)pos_high << 32) | pos_low;
   1342 
   1343 	return do_compat_pwritev64(fd, vec, vlen, pos, 0);
   1344 }
   1345 
   1346 #ifdef __ARCH_WANT_COMPAT_SYS_PWRITEV64V2
   1347 COMPAT_SYSCALL_DEFINE5(pwritev64v2, unsigned long, fd,
   1348 		const struct compat_iovec __user *,vec,
   1349 		unsigned long, vlen, loff_t, pos, rwf_t, flags)
   1350 {
   1351 	if (pos == -1)
   1352 		return do_compat_writev(fd, vec, vlen, flags);
   1353 
   1354 	return do_compat_pwritev64(fd, vec, vlen, pos, flags);
   1355 }
   1356 #endif
   1357 
   1358 COMPAT_SYSCALL_DEFINE6(pwritev2, compat_ulong_t, fd,
   1359 		const struct compat_iovec __user *,vec,
   1360 		compat_ulong_t, vlen, u32, pos_low, u32, pos_high, rwf_t, flags)
   1361 {
   1362 	loff_t pos = ((loff_t)pos_high << 32) | pos_low;
   1363 
   1364 	if (pos == -1)
   1365 		return do_compat_writev(fd, vec, vlen, flags);
   1366 
   1367 	return do_compat_pwritev64(fd, vec, vlen, pos, flags);
   1368 }
   1369 
   1370 #endif
   1371 
   1372 static ssize_t do_sendfile(int out_fd, int in_fd, loff_t *ppos,
   1373 		  	   size_t count, loff_t max)
   1374 {
   1375 	struct fd in, out;
   1376 	struct inode *in_inode, *out_inode;
   1377 	loff_t pos;
   1378 	loff_t out_pos;
   1379 	ssize_t retval;
   1380 	int fl;
   1381 
   1382 	/*
   1383 	 * Get input file, and verify that it is ok..
   1384 	 */
   1385 	retval = -EBADF;
   1386 	in = fdget(in_fd);
   1387 	if (!in.file)
   1388 		goto out;
   1389 	if (!(in.file->f_mode & FMODE_READ))
   1390 		goto fput_in;
   1391 	retval = -ESPIPE;
   1392 	if (!ppos) {
   1393 		pos = in.file->f_pos;
   1394 	} else {
   1395 		pos = *ppos;
   1396 		if (!(in.file->f_mode & FMODE_PREAD))
   1397 			goto fput_in;
   1398 	}
   1399 	retval = rw_verify_area(READ, in.file, &pos, count);
   1400 	if (retval < 0)
   1401 		goto fput_in;
   1402 	if (count > MAX_RW_COUNT)
   1403 		count =  MAX_RW_COUNT;
   1404 
   1405 	/*
   1406 	 * Get output file, and verify that it is ok..
   1407 	 */
   1408 	retval = -EBADF;
   1409 	out = fdget(out_fd);
   1410 	if (!out.file)
   1411 		goto fput_in;
   1412 	if (!(out.file->f_mode & FMODE_WRITE))
   1413 		goto fput_out;
   1414 	in_inode = file_inode(in.file);
   1415 	out_inode = file_inode(out.file);
   1416 	out_pos = out.file->f_pos;
   1417 	retval = rw_verify_area(WRITE, out.file, &out_pos, count);
   1418 	if (retval < 0)
   1419 		goto fput_out;
   1420 
   1421 	if (!max)
   1422 		max = min(in_inode->i_sb->s_maxbytes, out_inode->i_sb->s_maxbytes);
   1423 
   1424 	if (unlikely(pos + count > max)) {
   1425 		retval = -EOVERFLOW;
   1426 		if (pos >= max)
   1427 			goto fput_out;
   1428 		count = max - pos;
   1429 	}
   1430 
   1431 	fl = 0;
   1432 #if 0
   1433 	/*
   1434 	 * We need to debate whether we can enable this or not. The
   1435 	 * man page documents EAGAIN return for the output at least,
   1436 	 * and the application is arguably buggy if it doesn't expect
   1437 	 * EAGAIN on a non-blocking file descriptor.
   1438 	 */
   1439 	if (in.file->f_flags & O_NONBLOCK)
   1440 		fl = SPLICE_F_NONBLOCK;
   1441 #endif
   1442 	file_start_write(out.file);
   1443 	retval = do_splice_direct(in.file, &pos, out.file, &out_pos, count, fl);
   1444 	file_end_write(out.file);
   1445 
   1446 	if (retval > 0) {
   1447 		add_rchar(current, retval);
   1448 		add_wchar(current, retval);
   1449 		fsnotify_access(in.file);
   1450 		fsnotify_modify(out.file);
   1451 		out.file->f_pos = out_pos;
   1452 		if (ppos)
   1453 			*ppos = pos;
   1454 		else
   1455 			in.file->f_pos = pos;
   1456 	}
   1457 
   1458 	inc_syscr(current);
   1459 	inc_syscw(current);
   1460 	if (pos > max)
   1461 		retval = -EOVERFLOW;
   1462 
   1463 fput_out:
   1464 	fdput(out);
   1465 fput_in:
   1466 	fdput(in);
   1467 out:
   1468 	return retval;
   1469 }
   1470 
   1471 SYSCALL_DEFINE4(sendfile, int, out_fd, int, in_fd, off_t __user *, offset, size_t, count)
   1472 {
   1473 	loff_t pos;
   1474 	off_t off;
   1475 	ssize_t ret;
   1476 
   1477 	if (offset) {
   1478 		if (unlikely(get_user(off, offset)))
   1479 			return -EFAULT;
   1480 		pos = off;
   1481 		ret = do_sendfile(out_fd, in_fd, &pos, count, MAX_NON_LFS);
   1482 		if (unlikely(put_user(pos, offset)))
   1483 			return -EFAULT;
   1484 		return ret;
   1485 	}
   1486 
   1487 	return do_sendfile(out_fd, in_fd, NULL, count, 0);
   1488 }
   1489 
   1490 SYSCALL_DEFINE4(sendfile64, int, out_fd, int, in_fd, loff_t __user *, offset, size_t, count)
   1491 {
   1492 	loff_t pos;
   1493 	ssize_t ret;
   1494 
   1495 	if (offset) {
   1496 		if (unlikely(copy_from_user(&pos, offset, sizeof(loff_t))))
   1497 			return -EFAULT;
   1498 		ret = do_sendfile(out_fd, in_fd, &pos, count, 0);
   1499 		if (unlikely(put_user(pos, offset)))
   1500 			return -EFAULT;
   1501 		return ret;
   1502 	}
   1503 
   1504 	return do_sendfile(out_fd, in_fd, NULL, count, 0);
   1505 }
   1506 
   1507 #ifdef CONFIG_COMPAT
   1508 COMPAT_SYSCALL_DEFINE4(sendfile, int, out_fd, int, in_fd,
   1509 		compat_off_t __user *, offset, compat_size_t, count)
   1510 {
   1511 	loff_t pos;
   1512 	off_t off;
   1513 	ssize_t ret;
   1514 
   1515 	if (offset) {
   1516 		if (unlikely(get_user(off, offset)))
   1517 			return -EFAULT;
   1518 		pos = off;
   1519 		ret = do_sendfile(out_fd, in_fd, &pos, count, MAX_NON_LFS);
   1520 		if (unlikely(put_user(pos, offset)))
   1521 			return -EFAULT;
   1522 		return ret;
   1523 	}
   1524 
   1525 	return do_sendfile(out_fd, in_fd, NULL, count, 0);
   1526 }
   1527 
   1528 COMPAT_SYSCALL_DEFINE4(sendfile64, int, out_fd, int, in_fd,
   1529 		compat_loff_t __user *, offset, compat_size_t, count)
   1530 {
   1531 	loff_t pos;
   1532 	ssize_t ret;
   1533 
   1534 	if (offset) {
   1535 		if (unlikely(copy_from_user(&pos, offset, sizeof(loff_t))))
   1536 			return -EFAULT;
   1537 		ret = do_sendfile(out_fd, in_fd, &pos, count, 0);
   1538 		if (unlikely(put_user(pos, offset)))
   1539 			return -EFAULT;
   1540 		return ret;
   1541 	}
   1542 
   1543 	return do_sendfile(out_fd, in_fd, NULL, count, 0);
   1544 }
   1545 #endif
   1546 
   1547 /*
   1548  * copy_file_range() differs from regular file read and write in that it
   1549  * specifically allows return partial success.  When it does so is up to
   1550  * the copy_file_range method.
   1551  */
   1552 ssize_t vfs_copy_file_range(struct file *file_in, loff_t pos_in,
   1553 			    struct file *file_out, loff_t pos_out,
   1554 			    size_t len, unsigned int flags)
   1555 {
   1556 	struct inode *inode_in = file_inode(file_in);
   1557 	struct inode *inode_out = file_inode(file_out);
   1558 	ssize_t ret;
   1559 
   1560 	if (flags != 0)
   1561 		return -EINVAL;
   1562 
   1563 	if (S_ISDIR(inode_in->i_mode) || S_ISDIR(inode_out->i_mode))
   1564 		return -EISDIR;
   1565 	if (!S_ISREG(inode_in->i_mode) || !S_ISREG(inode_out->i_mode))
   1566 		return -EINVAL;
   1567 
   1568 	ret = rw_verify_area(READ, file_in, &pos_in, len);
   1569 	if (unlikely(ret))
   1570 		return ret;
   1571 
   1572 	ret = rw_verify_area(WRITE, file_out, &pos_out, len);
   1573 	if (unlikely(ret))
   1574 		return ret;
   1575 
   1576 	if (!(file_in->f_mode & FMODE_READ) ||
   1577 	    !(file_out->f_mode & FMODE_WRITE) ||
   1578 	    (file_out->f_flags & O_APPEND))
   1579 		return -EBADF;
   1580 
   1581 	/* this could be relaxed once a method supports cross-fs copies */
   1582 	if (inode_in->i_sb != inode_out->i_sb)
   1583 		return -EXDEV;
   1584 
   1585 	if (len == 0)
   1586 		return 0;
   1587 
   1588 	file_start_write(file_out);
   1589 
   1590 	/*
   1591 	 * Try cloning first, this is supported by more file systems, and
   1592 	 * more efficient if both clone and copy are supported (e.g. NFS).
   1593 	 */
   1594 	if (file_in->f_op->remap_file_range) {
   1595 		loff_t cloned;
   1596 
   1597 		cloned = file_in->f_op->remap_file_range(file_in, pos_in,
   1598 				file_out, pos_out,
   1599 				min_t(loff_t, MAX_RW_COUNT, len),
   1600 				REMAP_FILE_CAN_SHORTEN);
   1601 		if (cloned > 0) {
   1602 			ret = cloned;
   1603 			goto done;
   1604 		}
   1605 	}
   1606 
   1607 	if (file_out->f_op->copy_file_range) {
   1608 		ret = file_out->f_op->copy_file_range(file_in, pos_in, file_out,
   1609 						      pos_out, len, flags);
   1610 		if (ret != -EOPNOTSUPP)
   1611 			goto done;
   1612 	}
   1613 
   1614 	ret = do_splice_direct(file_in, &pos_in, file_out, &pos_out,
   1615 			len > MAX_RW_COUNT ? MAX_RW_COUNT : len, 0);
   1616 
   1617 done:
   1618 	if (ret > 0) {
   1619 		fsnotify_access(file_in);
   1620 		add_rchar(current, ret);
   1621 		fsnotify_modify(file_out);
   1622 		add_wchar(current, ret);
   1623 	}
   1624 
   1625 	inc_syscr(current);
   1626 	inc_syscw(current);
   1627 
   1628 	file_end_write(file_out);
   1629 
   1630 	return ret;
   1631 }
   1632 EXPORT_SYMBOL(vfs_copy_file_range);
   1633 
   1634 SYSCALL_DEFINE6(copy_file_range, int, fd_in, loff_t __user *, off_in,
   1635 		int, fd_out, loff_t __user *, off_out,
   1636 		size_t, len, unsigned int, flags)
   1637 {
   1638 	loff_t pos_in;
   1639 	loff_t pos_out;
   1640 	struct fd f_in;
   1641 	struct fd f_out;
   1642 	ssize_t ret = -EBADF;
   1643 
   1644 	f_in = fdget(fd_in);
   1645 	if (!f_in.file)
   1646 		goto out2;
   1647 
   1648 	f_out = fdget(fd_out);
   1649 	if (!f_out.file)
   1650 		goto out1;
   1651 
   1652 	ret = -EFAULT;
   1653 	if (off_in) {
   1654 		if (copy_from_user(&pos_in, off_in, sizeof(loff_t)))
   1655 			goto out;
   1656 	} else {
   1657 		pos_in = f_in.file->f_pos;
   1658 	}
   1659 
   1660 	if (off_out) {
   1661 		if (copy_from_user(&pos_out, off_out, sizeof(loff_t)))
   1662 			goto out;
   1663 	} else {
   1664 		pos_out = f_out.file->f_pos;
   1665 	}
   1666 
   1667 	ret = vfs_copy_file_range(f_in.file, pos_in, f_out.file, pos_out, len,
   1668 				  flags);
   1669 	if (ret > 0) {
   1670 		pos_in += ret;
   1671 		pos_out += ret;
   1672 
   1673 		if (off_in) {
   1674 			if (copy_to_user(off_in, &pos_in, sizeof(loff_t)))
   1675 				ret = -EFAULT;
   1676 		} else {
   1677 			f_in.file->f_pos = pos_in;
   1678 		}
   1679 
   1680 		if (off_out) {
   1681 			if (copy_to_user(off_out, &pos_out, sizeof(loff_t)))
   1682 				ret = -EFAULT;
   1683 		} else {
   1684 			f_out.file->f_pos = pos_out;
   1685 		}
   1686 	}
   1687 
   1688 out:
   1689 	fdput(f_out);
   1690 out1:
   1691 	fdput(f_in);
   1692 out2:
   1693 	return ret;
   1694 }
   1695 
   1696 static int remap_verify_area(struct file *file, loff_t pos, loff_t len,
   1697 			     bool write)
   1698 {
   1699 	struct inode *inode = file_inode(file);
   1700 
   1701 	if (unlikely(pos < 0 || len < 0))
   1702 		return -EINVAL;
   1703 
   1704 	 if (unlikely((loff_t) (pos + len) < 0))
   1705 		return -EINVAL;
   1706 
   1707 	if (unlikely(inode->i_flctx && mandatory_lock(inode))) {
   1708 		loff_t end = len ? pos + len - 1 : OFFSET_MAX;
   1709 		int retval;
   1710 
   1711 		retval = locks_mandatory_area(inode, file, pos, end,
   1712 				write ? F_WRLCK : F_RDLCK);
   1713 		if (retval < 0)
   1714 			return retval;
   1715 	}
   1716 
   1717 	return security_file_permission(file, write ? MAY_WRITE : MAY_READ);
   1718 }
   1719 /*
   1720  * Ensure that we don't remap a partial EOF block in the middle of something
   1721  * else.  Assume that the offsets have already been checked for block
   1722  * alignment.
   1723  *
   1724  * For deduplication we always scale down to the previous block because we
   1725  * can't meaningfully compare post-EOF contents.
   1726  *
   1727  * For clone we only link a partial EOF block above the destination file's EOF.
   1728  *
   1729  * Shorten the request if possible.
   1730  */
   1731 static int generic_remap_check_len(struct inode *inode_in,
   1732 				   struct inode *inode_out,
   1733 				   loff_t pos_out,
   1734 				   loff_t *len,
   1735 				   unsigned int remap_flags)
   1736 {
   1737 	u64 blkmask = i_blocksize(inode_in) - 1;
   1738 	loff_t new_len = *len;
   1739 
   1740 	if ((*len & blkmask) == 0)
   1741 		return 0;
   1742 
   1743 	if ((remap_flags & REMAP_FILE_DEDUP) ||
   1744 	    pos_out + *len < i_size_read(inode_out))
   1745 		new_len &= ~blkmask;
   1746 
   1747 	if (new_len == *len)
   1748 		return 0;
   1749 
   1750 	if (remap_flags & REMAP_FILE_CAN_SHORTEN) {
   1751 		*len = new_len;
   1752 		return 0;
   1753 	}
   1754 
   1755 	return (remap_flags & REMAP_FILE_DEDUP) ? -EBADE : -EINVAL;
   1756 }
   1757 
   1758 /*
   1759  * Read a page's worth of file data into the page cache.  Return the page
   1760  * locked.
   1761  */
   1762 static struct page *vfs_dedupe_get_page(struct inode *inode, loff_t offset)
   1763 {
   1764 	struct page *page;
   1765 
   1766 	page = read_mapping_page(inode->i_mapping, offset >> PAGE_SHIFT, NULL);
   1767 	if (IS_ERR(page))
   1768 		return page;
   1769 	if (!PageUptodate(page)) {
   1770 		put_page(page);
   1771 		return ERR_PTR(-EIO);
   1772 	}
   1773 	lock_page(page);
   1774 	return page;
   1775 }
   1776 
   1777 /*
   1778  * Compare extents of two files to see if they are the same.
   1779  * Caller must have locked both inodes to prevent write races.
   1780  */
   1781 static int vfs_dedupe_file_range_compare(struct inode *src, loff_t srcoff,
   1782 					 struct inode *dest, loff_t destoff,
   1783 					 loff_t len, bool *is_same)
   1784 {
   1785 	loff_t src_poff;
   1786 	loff_t dest_poff;
   1787 	void *src_addr;
   1788 	void *dest_addr;
   1789 	struct page *src_page;
   1790 	struct page *dest_page;
   1791 	loff_t cmp_len;
   1792 	bool same;
   1793 	int error;
   1794 
   1795 	error = -EINVAL;
   1796 	same = true;
   1797 	while (len) {
   1798 		src_poff = srcoff & (PAGE_SIZE - 1);
   1799 		dest_poff = destoff & (PAGE_SIZE - 1);
   1800 		cmp_len = min(PAGE_SIZE - src_poff,
   1801 			      PAGE_SIZE - dest_poff);
   1802 		cmp_len = min(cmp_len, len);
   1803 		if (cmp_len <= 0)
   1804 			goto out_error;
   1805 
   1806 		src_page = vfs_dedupe_get_page(src, srcoff);
   1807 		if (IS_ERR(src_page)) {
   1808 			error = PTR_ERR(src_page);
   1809 			goto out_error;
   1810 		}
   1811 		dest_page = vfs_dedupe_get_page(dest, destoff);
   1812 		if (IS_ERR(dest_page)) {
   1813 			error = PTR_ERR(dest_page);
   1814 			unlock_page(src_page);
   1815 			put_page(src_page);
   1816 			goto out_error;
   1817 		}
   1818 		src_addr = kmap_atomic(src_page);
   1819 		dest_addr = kmap_atomic(dest_page);
   1820 
   1821 		flush_dcache_page(src_page);
   1822 		flush_dcache_page(dest_page);
   1823 
   1824 		if (memcmp(src_addr + src_poff, dest_addr + dest_poff, cmp_len))
   1825 			same = false;
   1826 
   1827 		kunmap_atomic(dest_addr);
   1828 		kunmap_atomic(src_addr);
   1829 		unlock_page(dest_page);
   1830 		unlock_page(src_page);
   1831 		put_page(dest_page);
   1832 		put_page(src_page);
   1833 
   1834 		if (!same)
   1835 			break;
   1836 
   1837 		srcoff += cmp_len;
   1838 		destoff += cmp_len;
   1839 		len -= cmp_len;
   1840 	}
   1841 
   1842 	*is_same = same;
   1843 	return 0;
   1844 
   1845 out_error:
   1846 	return error;
   1847 }
   1848 
   1849 /*
   1850  * Check that the two inodes are eligible for cloning, the ranges make
   1851  * sense, and then flush all dirty data.  Caller must ensure that the
   1852  * inodes have been locked against any other modifications.
   1853  *
   1854  * If there's an error, then the usual negative error code is returned.
   1855  * Otherwise returns 0 with *len set to the request length.
   1856  */
   1857 int generic_remap_file_range_prep(struct file *file_in, loff_t pos_in,
   1858 				  struct file *file_out, loff_t pos_out,
   1859 				  loff_t *len, unsigned int remap_flags)
   1860 {
   1861 	struct inode *inode_in = file_inode(file_in);
   1862 	struct inode *inode_out = file_inode(file_out);
   1863 	bool same_inode = (inode_in == inode_out);
   1864 	int ret;
   1865 
   1866 	/* Don't touch certain kinds of inodes */
   1867 	if (IS_IMMUTABLE(inode_out))
   1868 		return -EPERM;
   1869 
   1870 	if (IS_SWAPFILE(inode_in) || IS_SWAPFILE(inode_out))
   1871 		return -ETXTBSY;
   1872 
   1873 	/* Don't reflink dirs, pipes, sockets... */
   1874 	if (S_ISDIR(inode_in->i_mode) || S_ISDIR(inode_out->i_mode))
   1875 		return -EISDIR;
   1876 	if (!S_ISREG(inode_in->i_mode) || !S_ISREG(inode_out->i_mode))
   1877 		return -EINVAL;
   1878 
   1879 	/* Zero length dedupe exits immediately; reflink goes to EOF. */
   1880 	if (*len == 0) {
   1881 		loff_t isize = i_size_read(inode_in);
   1882 
   1883 		if ((remap_flags & REMAP_FILE_DEDUP) || pos_in == isize)
   1884 			return 0;
   1885 		if (pos_in > isize)
   1886 			return -EINVAL;
   1887 		*len = isize - pos_in;
   1888 		if (*len == 0)
   1889 			return 0;
   1890 	}
   1891 
   1892 	/* Check that we don't violate system file offset limits. */
   1893 	ret = generic_remap_checks(file_in, pos_in, file_out, pos_out, len,
   1894 			remap_flags);
   1895 	if (ret)
   1896 		return ret;
   1897 
   1898 	/* Wait for the completion of any pending IOs on both files */
   1899 	inode_dio_wait(inode_in);
   1900 	if (!same_inode)
   1901 		inode_dio_wait(inode_out);
   1902 
   1903 	ret = filemap_write_and_wait_range(inode_in->i_mapping,
   1904 			pos_in, pos_in + *len - 1);
   1905 	if (ret)
   1906 		return ret;
   1907 
   1908 	ret = filemap_write_and_wait_range(inode_out->i_mapping,
   1909 			pos_out, pos_out + *len - 1);
   1910 	if (ret)
   1911 		return ret;
   1912 
   1913 	/*
   1914 	 * Check that the extents are the same.
   1915 	 */
   1916 	if (remap_flags & REMAP_FILE_DEDUP) {
   1917 		bool		is_same = false;
   1918 
   1919 		ret = vfs_dedupe_file_range_compare(inode_in, pos_in,
   1920 				inode_out, pos_out, *len, &is_same);
   1921 		if (ret)
   1922 			return ret;
   1923 		if (!is_same)
   1924 			return -EBADE;
   1925 	}
   1926 
   1927 	ret = generic_remap_check_len(inode_in, inode_out, pos_out, len,
   1928 			remap_flags);
   1929 	if (ret)
   1930 		return ret;
   1931 
   1932 	/* If can't alter the file contents, we're done. */
   1933 	if (!(remap_flags & REMAP_FILE_DEDUP)) {
   1934 		/* Update the timestamps, since we can alter file contents. */
   1935 		if (!(file_out->f_mode & FMODE_NOCMTIME)) {
   1936 			ret = file_update_time(file_out);
   1937 			if (ret)
   1938 				return ret;
   1939 		}
   1940 
   1941 		/*
   1942 		 * Clear the security bits if the process is not being run by
   1943 		 * root.  This keeps people from modifying setuid and setgid
   1944 		 * binaries.
   1945 		 */
   1946 		ret = file_remove_privs(file_out);
   1947 		if (ret)
   1948 			return ret;
   1949 	}
   1950 
   1951 	return 0;
   1952 }
   1953 EXPORT_SYMBOL(generic_remap_file_range_prep);
   1954 
   1955 loff_t do_clone_file_range(struct file *file_in, loff_t pos_in,
   1956 			   struct file *file_out, loff_t pos_out,
   1957 			   loff_t len, unsigned int remap_flags)
   1958 {
   1959 	struct inode *inode_in = file_inode(file_in);
   1960 	struct inode *inode_out = file_inode(file_out);
   1961 	loff_t ret;
   1962 
   1963 	WARN_ON_ONCE(remap_flags & REMAP_FILE_DEDUP);
   1964 
   1965 	if (S_ISDIR(inode_in->i_mode) || S_ISDIR(inode_out->i_mode))
   1966 		return -EISDIR;
   1967 	if (!S_ISREG(inode_in->i_mode) || !S_ISREG(inode_out->i_mode))
   1968 		return -EINVAL;
   1969 
   1970 	/*
   1971 	 * FICLONE/FICLONERANGE ioctls enforce that src and dest files are on
   1972 	 * the same mount. Practically, they only need to be on the same file
   1973 	 * system.
   1974 	 */
   1975 	if (inode_in->i_sb != inode_out->i_sb)
   1976 		return -EXDEV;
   1977 
   1978 	if (!(file_in->f_mode & FMODE_READ) ||
   1979 	    !(file_out->f_mode & FMODE_WRITE) ||
   1980 	    (file_out->f_flags & O_APPEND))
   1981 		return -EBADF;
   1982 
   1983 	if (!file_in->f_op->remap_file_range)
   1984 		return -EOPNOTSUPP;
   1985 
   1986 	ret = remap_verify_area(file_in, pos_in, len, false);
   1987 	if (ret)
   1988 		return ret;
   1989 
   1990 	ret = remap_verify_area(file_out, pos_out, len, true);
   1991 	if (ret)
   1992 		return ret;
   1993 
   1994 	ret = file_in->f_op->remap_file_range(file_in, pos_in,
   1995 			file_out, pos_out, len, remap_flags);
   1996 	if (ret < 0)
   1997 		return ret;
   1998 
   1999 	fsnotify_access(file_in);
   2000 	fsnotify_modify(file_out);
   2001 	return ret;
   2002 }
   2003 EXPORT_SYMBOL(do_clone_file_range);
   2004 
   2005 loff_t vfs_clone_file_range(struct file *file_in, loff_t pos_in,
   2006 			    struct file *file_out, loff_t pos_out,
   2007 			    loff_t len, unsigned int remap_flags)
   2008 {
   2009 	loff_t ret;
   2010 
   2011 	file_start_write(file_out);
   2012 	ret = do_clone_file_range(file_in, pos_in, file_out, pos_out, len,
   2013 				  remap_flags);
   2014 	file_end_write(file_out);
   2015 
   2016 	return ret;
   2017 }
   2018 EXPORT_SYMBOL(vfs_clone_file_range);
   2019 
   2020 /* Check whether we are allowed to dedupe the destination file */
   2021 static bool allow_file_dedupe(struct file *file)
   2022 {
   2023 	if (capable(CAP_SYS_ADMIN))
   2024 		return true;
   2025 	if (file->f_mode & FMODE_WRITE)
   2026 		return true;
   2027 	if (uid_eq(current_fsuid(), file_inode(file)->i_uid))
   2028 		return true;
   2029 	if (!inode_permission(file_inode(file), MAY_WRITE))
   2030 		return true;
   2031 	return false;
   2032 }
   2033 
   2034 loff_t vfs_dedupe_file_range_one(struct file *src_file, loff_t src_pos,
   2035 				 struct file *dst_file, loff_t dst_pos,
   2036 				 loff_t len, unsigned int remap_flags)
   2037 {
   2038 	loff_t ret;
   2039 
   2040 	WARN_ON_ONCE(remap_flags & ~(REMAP_FILE_DEDUP |
   2041 				     REMAP_FILE_CAN_SHORTEN));
   2042 
   2043 	ret = mnt_want_write_file(dst_file);
   2044 	if (ret)
   2045 		return ret;
   2046 
   2047 	ret = remap_verify_area(dst_file, dst_pos, len, true);
   2048 	if (ret < 0)
   2049 		goto out_drop_write;
   2050 
   2051 	ret = -EPERM;
   2052 	if (!allow_file_dedupe(dst_file))
   2053 		goto out_drop_write;
   2054 
   2055 	ret = -EXDEV;
   2056 	if (src_file->f_path.mnt != dst_file->f_path.mnt)
   2057 		goto out_drop_write;
   2058 
   2059 	ret = -EISDIR;
   2060 	if (S_ISDIR(file_inode(dst_file)->i_mode))
   2061 		goto out_drop_write;
   2062 
   2063 	ret = -EINVAL;
   2064 	if (!dst_file->f_op->remap_file_range)
   2065 		goto out_drop_write;
   2066 
   2067 	if (len == 0) {
   2068 		ret = 0;
   2069 		goto out_drop_write;
   2070 	}
   2071 
   2072 	ret = dst_file->f_op->remap_file_range(src_file, src_pos, dst_file,
   2073 			dst_pos, len, remap_flags | REMAP_FILE_DEDUP);
   2074 out_drop_write:
   2075 	mnt_drop_write_file(dst_file);
   2076 
   2077 	return ret;
   2078 }
   2079 EXPORT_SYMBOL(vfs_dedupe_file_range_one);
   2080 
   2081 int vfs_dedupe_file_range(struct file *file, struct file_dedupe_range *same)
   2082 {
   2083 	struct file_dedupe_range_info *info;
   2084 	struct inode *src = file_inode(file);
   2085 	u64 off;
   2086 	u64 len;
   2087 	int i;
   2088 	int ret;
   2089 	u16 count = same->dest_count;
   2090 	loff_t deduped;
   2091 
   2092 	if (!(file->f_mode & FMODE_READ))
   2093 		return -EINVAL;
   2094 
   2095 	if (same->reserved1 || same->reserved2)
   2096 		return -EINVAL;
   2097 
   2098 	off = same->src_offset;
   2099 	len = same->src_length;
   2100 
   2101 	if (S_ISDIR(src->i_mode))
   2102 		return -EISDIR;
   2103 
   2104 	if (!S_ISREG(src->i_mode))
   2105 		return -EINVAL;
   2106 
   2107 	if (!file->f_op->remap_file_range)
   2108 		return -EOPNOTSUPP;
   2109 
   2110 	ret = remap_verify_area(file, off, len, false);
   2111 	if (ret < 0)
   2112 		return ret;
   2113 	ret = 0;
   2114 
   2115 	if (off + len > i_size_read(src))
   2116 		return -EINVAL;
   2117 
   2118 	/* Arbitrary 1G limit on a single dedupe request, can be raised. */
   2119 	len = min_t(u64, len, 1 << 30);
   2120 
   2121 	/* pre-format output fields to sane values */
   2122 	for (i = 0; i < count; i++) {
   2123 		same->info[i].bytes_deduped = 0ULL;
   2124 		same->info[i].status = FILE_DEDUPE_RANGE_SAME;
   2125 	}
   2126 
   2127 	for (i = 0, info = same->info; i < count; i++, info++) {
   2128 		struct fd dst_fd = fdget(info->dest_fd);
   2129 		struct file *dst_file = dst_fd.file;
   2130 
   2131 		if (!dst_file) {
   2132 			info->status = -EBADF;
   2133 			goto next_loop;
   2134 		}
   2135 
   2136 		if (info->reserved) {
   2137 			info->status = -EINVAL;
   2138 			goto next_fdput;
   2139 		}
   2140 
   2141 		deduped = vfs_dedupe_file_range_one(file, off, dst_file,
   2142 						    info->dest_offset, len,
   2143 						    REMAP_FILE_CAN_SHORTEN);
   2144 		if (deduped == -EBADE)
   2145 			info->status = FILE_DEDUPE_RANGE_DIFFERS;
   2146 		else if (deduped < 0)
   2147 			info->status = deduped;
   2148 		else
   2149 			info->bytes_deduped = len;
   2150 
   2151 next_fdput:
   2152 		fdput(dst_fd);
   2153 next_loop:
   2154 		if (fatal_signal_pending(current))
   2155 			break;
   2156 	}
   2157 	return ret;
   2158 }
   2159 EXPORT_SYMBOL(vfs_dedupe_file_range);