Attachment #859551 for bug #1200450

View | Details | Raw Unified | Return to bug 1200450
Collapse All | Expand All




	/* In the fs-verity case, f2fs_end_enable_verity() does the truncate */
	if (to > i_size && !f2fs_verity_in_progress(inode)) {
		down_write(&F2FS_I(inode)->i_gc_rwsem[WRITE]);
		down_write(&F2FS_I(inode)->i_mmap_sem);

		truncate_pagecache(inode, i_size);
		f2fs_truncate_blocks(inode, i_size, true);

		up_write(&F2FS_I(inode)->i_mmap_sem);
		up_write(&F2FS_I(inode)->i_gc_rwsem[WRITE]);
	}
}

	int ret = 0;

	down_write(&F2FS_I(inode)->i_gc_rwsem[WRITE]);
	down_write(&F2FS_I(inode)->i_mmap_sem);

	set_inode_flag(inode, FI_ALIGNED_WRITE);


	clear_inode_flag(inode, FI_DO_DEFRAG);
	clear_inode_flag(inode, FI_ALIGNED_WRITE);

	up_write(&F2FS_I(inode)->i_mmap_sem);
	up_write(&F2FS_I(inode)->i_gc_rwsem[WRITE]);

	return ret;

Lines 755-760 struct f2fs_inode_info { Link Here

(-)a/fs/f2fs/f2fs.h (+1 lines)
755		755
756	/* avoid racing between foreground op and gc */	756	/* avoid racing between foreground op and gc */
757	struct rw_semaphore i_gc_rwsem[2];	757	struct rw_semaphore i_gc_rwsem[2];
		758	struct rw_semaphore i_mmap_sem;
758	struct rw_semaphore i_xattr_sem; /* avoid racing between reading and changing EAs */	759	struct rw_semaphore i_xattr_sem; /* avoid racing between reading and changing EAs */
759		760
760	int i_extra_isize; /* size of extra space located in i_addr */	761	int i_extra_isize; /* size of extra space located in i_addr */

Lines 38-44 static vm_fault_t f2fs_filemap_fault(struct vm_fault *vmf) Link Here

(-)a/fs/f2fs/file.c (-30 / +32 lines)
38	struct inode *inode = file_inode(vmf->vma->vm_file);	38	struct inode *inode = file_inode(vmf->vma->vm_file);
39	vm_fault_t ret;	39	vm_fault_t ret;
40		40
		41	down_read(&F2FS_I(inode)->i_mmap_sem);
41	ret = filemap_fault(vmf);	42	ret = filemap_fault(vmf);
		43	up_read(&F2FS_I(inode)->i_mmap_sem);
		44
42	if (!ret)	45	if (!ret)
43	f2fs_update_iostat(F2FS_I_SB(inode), APP_MAPPED_READ_IO,	46	f2fs_update_iostat(F2FS_I_SB(inode), APP_MAPPED_READ_IO,
44	F2FS_BLKSIZE);	47	F2FS_BLKSIZE);
Lines 98-104 static vm_fault_t f2fs_vm_page_mkwrite(struct vm_fault *vmf) Link Here
98	f2fs_bug_on(sbi, f2fs_has_inline_data(inode));	101	f2fs_bug_on(sbi, f2fs_has_inline_data(inode));
99		102
100	file_update_time(vmf->vma->vm_file);	103	file_update_time(vmf->vma->vm_file);
101	filemap_invalidate_lock_shared(inode->i_mapping);	104	down_read(&F2FS_I(inode)->i_mmap_sem);
102	lock_page(page);	105	lock_page(page);
103	if (unlikely(page->mapping != inode->i_mapping \|\|	106	if (unlikely(page->mapping != inode->i_mapping \|\|
104	page_offset(page) > i_size_read(inode) \|\|	107	page_offset(page) > i_size_read(inode) \|\|
Lines 156-162 static vm_fault_t f2fs_vm_page_mkwrite(struct vm_fault *vmf) Link Here
156		159
157	trace_f2fs_vm_page_mkwrite(page, DATA);	160	trace_f2fs_vm_page_mkwrite(page, DATA);
158	out_sem:	161	out_sem:
159	filemap_invalidate_unlock_shared(inode->i_mapping);	162	up_read(&F2FS_I(inode)->i_mmap_sem);
160		163
161	sb_end_pagefault(inode->i_sb);	164	sb_end_pagefault(inode->i_sb);
162	err:	165	err:
Lines 936-942 int f2fs_setattr(struct user_namespace mnt_userns, struct dentry dentry, Link Here
936	}	939	}
937		940
938	down_write(&F2FS_I(inode)->i_gc_rwsem[WRITE]);	941	down_write(&F2FS_I(inode)->i_gc_rwsem[WRITE]);
939	filemap_invalidate_lock(inode->i_mapping);	942	down_write(&F2FS_I(inode)->i_mmap_sem);
940		943
941	truncate_setsize(inode, attr->ia_size);	944	truncate_setsize(inode, attr->ia_size);
942		945
Lines 946-952 int f2fs_setattr(struct user_namespace mnt_userns, struct dentry dentry, Link Here
946	* do not trim all blocks after i_size if target size is	949	* do not trim all blocks after i_size if target size is
947	* larger than i_size.	950	* larger than i_size.
948	*/	951	*/
949	filemap_invalidate_unlock(inode->i_mapping);	952	up_write(&F2FS_I(inode)->i_mmap_sem);
950	up_write(&F2FS_I(inode)->i_gc_rwsem[WRITE]);	953	up_write(&F2FS_I(inode)->i_gc_rwsem[WRITE]);
951	if (err)	954	if (err)
952	return err;	955	return err;
Lines 1090-1096 static int punch_hole(struct inode *inode, loff_t offset, loff_t len) Link Here
1090	blk_end = (loff_t)pg_end << PAGE_SHIFT;	1093	blk_end = (loff_t)pg_end << PAGE_SHIFT;
1091		1094
1092	down_write(&F2FS_I(inode)->i_gc_rwsem[WRITE]);	1095	down_write(&F2FS_I(inode)->i_gc_rwsem[WRITE]);
1093	filemap_invalidate_lock(mapping);	1096	down_write(&F2FS_I(inode)->i_mmap_sem);
1094		1097
1095	truncate_pagecache_range(inode, blk_start, blk_end - 1);	1098	truncate_pagecache_range(inode, blk_start, blk_end - 1);
1096		1099
Lines 1098-1104 static int punch_hole(struct inode *inode, loff_t offset, loff_t len) Link Here
1098	ret = f2fs_truncate_hole(inode, pg_start, pg_end);	1101	ret = f2fs_truncate_hole(inode, pg_start, pg_end);
1099	f2fs_unlock_op(sbi);	1102	f2fs_unlock_op(sbi);
1100		1103
1101	filemap_invalidate_unlock(mapping);	1104	up_write(&F2FS_I(inode)->i_mmap_sem);
1102	up_write(&F2FS_I(inode)->i_gc_rwsem[WRITE]);	1105	up_write(&F2FS_I(inode)->i_gc_rwsem[WRITE]);
1103	}	1106	}
1104	}	1107	}
Lines 1333-1339 static int f2fs_do_collapse(struct inode *inode, loff_t offset, loff_t len) Link Here
1333		1336
1334	/* avoid gc operation during block exchange */	1337	/* avoid gc operation during block exchange */
1335	down_write(&F2FS_I(inode)->i_gc_rwsem[WRITE]);	1338	down_write(&F2FS_I(inode)->i_gc_rwsem[WRITE]);
1336	filemap_invalidate_lock(inode->i_mapping);	1339	down_write(&F2FS_I(inode)->i_mmap_sem);
1337		1340
1338	f2fs_lock_op(sbi);	1341	f2fs_lock_op(sbi);
1339	f2fs_drop_extent_tree(inode);	1342	f2fs_drop_extent_tree(inode);
Lines 1341-1347 static int f2fs_do_collapse(struct inode *inode, loff_t offset, loff_t len) Link Here
1341	ret = __exchange_data_block(inode, inode, end, start, nrpages - end, true);	1344	ret = __exchange_data_block(inode, inode, end, start, nrpages - end, true);
1342	f2fs_unlock_op(sbi);	1345	f2fs_unlock_op(sbi);
1343		1346
1344	filemap_invalidate_unlock(inode->i_mapping);	1347	up_write(&F2FS_I(inode)->i_mmap_sem);
1345	up_write(&F2FS_I(inode)->i_gc_rwsem[WRITE]);	1348	up_write(&F2FS_I(inode)->i_gc_rwsem[WRITE]);
1346	return ret;	1349	return ret;
1347	}	1350	}
Lines 1372-1384 static int f2fs_collapse_range(struct inode *inode, loff_t offset, loff_t len) Link Here
1372	return ret;	1375	return ret;
1373		1376
1374	/* write out all moved pages, if possible */	1377	/* write out all moved pages, if possible */
1375	filemap_invalidate_lock(inode->i_mapping);	1378	down_write(&F2FS_I(inode)->i_mmap_sem);
1376	filemap_write_and_wait_range(inode->i_mapping, offset, LLONG_MAX);	1379	filemap_write_and_wait_range(inode->i_mapping, offset, LLONG_MAX);
1377	truncate_pagecache(inode, offset);	1380	truncate_pagecache(inode, offset);
1378		1381
1379	new_size = i_size_read(inode) - len;	1382	new_size = i_size_read(inode) - len;
1380	ret = f2fs_truncate_blocks(inode, new_size, true);	1383	ret = f2fs_truncate_blocks(inode, new_size, true);
1381	filemap_invalidate_unlock(inode->i_mapping);	1384	up_write(&F2FS_I(inode)->i_mmap_sem);
1382	if (!ret)	1385	if (!ret)
1383	f2fs_i_size_write(inode, new_size);	1386	f2fs_i_size_write(inode, new_size);
1384	return ret;	1387	return ret;
Lines 1478-1484 static int f2fs_zero_range(struct inode *inode, loff_t offset, loff_t len, Link Here
1478	pgoff_t end;	1481	pgoff_t end;
1479		1482
1480	down_write(&F2FS_I(inode)->i_gc_rwsem[WRITE]);	1483	down_write(&F2FS_I(inode)->i_gc_rwsem[WRITE]);
1481	filemap_invalidate_lock(mapping);	1484	down_write(&F2FS_I(inode)->i_mmap_sem);
1482		1485
1483	truncate_pagecache_range(inode,	1486	truncate_pagecache_range(inode,
1484	(loff_t)index << PAGE_SHIFT,	1487	(loff_t)index << PAGE_SHIFT,
Lines 1490-1496 static int f2fs_zero_range(struct inode *inode, loff_t offset, loff_t len, Link Here
1490	ret = f2fs_get_dnode_of_data(&dn, index, ALLOC_NODE);	1493	ret = f2fs_get_dnode_of_data(&dn, index, ALLOC_NODE);
1491	if (ret) {	1494	if (ret) {
1492	f2fs_unlock_op(sbi);	1495	f2fs_unlock_op(sbi);
1493	filemap_invalidate_unlock(mapping);	1496	up_write(&F2FS_I(inode)->i_mmap_sem);
1494	up_write(&F2FS_I(inode)->i_gc_rwsem[WRITE]);	1497	up_write(&F2FS_I(inode)->i_gc_rwsem[WRITE]);
1495	goto out;	1498	goto out;
1496	}	1499	}
Lines 1502-1508 static int f2fs_zero_range(struct inode *inode, loff_t offset, loff_t len, Link Here
1502	f2fs_put_dnode(&dn);	1505	f2fs_put_dnode(&dn);
1503		1506
1504	f2fs_unlock_op(sbi);	1507	f2fs_unlock_op(sbi);
1505	filemap_invalidate_unlock(mapping);	1508	up_write(&F2FS_I(inode)->i_mmap_sem);
1506	up_write(&F2FS_I(inode)->i_gc_rwsem[WRITE]);	1509	up_write(&F2FS_I(inode)->i_gc_rwsem[WRITE]);
1507		1510
1508	f2fs_balance_fs(sbi, dn.node_changed);	1511	f2fs_balance_fs(sbi, dn.node_changed);
Lines 1537-1543 static int f2fs_zero_range(struct inode *inode, loff_t offset, loff_t len, Link Here
1537	static int f2fs_insert_range(struct inode *inode, loff_t offset, loff_t len)	1540	static int f2fs_insert_range(struct inode *inode, loff_t offset, loff_t len)
1538	{	1541	{
1539	struct f2fs_sb_info *sbi = F2FS_I_SB(inode);	1542	struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
1540	struct address_space *mapping = inode->i_mapping;
1541	pgoff_t nr, pg_start, pg_end, delta, idx;	1543	pgoff_t nr, pg_start, pg_end, delta, idx;
1542	loff_t new_size;	1544	loff_t new_size;
1543	int ret = 0;	1545	int ret = 0;
Lines 1560-1573 static int f2fs_insert_range(struct inode *inode, loff_t offset, loff_t len) Link Here
1560		1562
1561	f2fs_balance_fs(sbi, true);	1563	f2fs_balance_fs(sbi, true);
1562		1564
1563	filemap_invalidate_lock(mapping);	1565	down_write(&F2FS_I(inode)->i_mmap_sem);
1564	ret = f2fs_truncate_blocks(inode, i_size_read(inode), true);	1566	ret = f2fs_truncate_blocks(inode, i_size_read(inode), true);
1565	filemap_invalidate_unlock(mapping);	1567	up_write(&F2FS_I(inode)->i_mmap_sem);
1566	if (ret)	1568	if (ret)
1567	return ret;	1569	return ret;
1568		1570
1569	/* write out all dirty pages from offset */	1571	/* write out all dirty pages from offset */
1570	ret = filemap_write_and_wait_range(mapping, offset, LLONG_MAX);	1572	ret = filemap_write_and_wait_range(inode->i_mapping, offset, LLONG_MAX);
1571	if (ret)	1573	if (ret)
1572	return ret;	1574	return ret;
1573		1575
Lines 1578-1584 static int f2fs_insert_range(struct inode *inode, loff_t offset, loff_t len) Link Here
1578		1580
1579	/* avoid gc operation during block exchange */	1581	/* avoid gc operation during block exchange */
1580	down_write(&F2FS_I(inode)->i_gc_rwsem[WRITE]);	1582	down_write(&F2FS_I(inode)->i_gc_rwsem[WRITE]);
1581	filemap_invalidate_lock(mapping);	1583	down_write(&F2FS_I(inode)->i_mmap_sem);
1582	truncate_pagecache(inode, offset);	1584	truncate_pagecache(inode, offset);
1583		1585
1584	while (!ret && idx > pg_start) {	1586	while (!ret && idx > pg_start) {
Lines 1594-1607 static int f2fs_insert_range(struct inode *inode, loff_t offset, loff_t len) Link Here
1594	idx + delta, nr, false);	1596	idx + delta, nr, false);
1595	f2fs_unlock_op(sbi);	1597	f2fs_unlock_op(sbi);
1596	}	1598	}
1597	filemap_invalidate_unlock(mapping);	1599	up_write(&F2FS_I(inode)->i_mmap_sem);
1598	up_write(&F2FS_I(inode)->i_gc_rwsem[WRITE]);	1600	up_write(&F2FS_I(inode)->i_gc_rwsem[WRITE]);
1599		1601
1600	/* write out all moved pages, if possible */	1602	/* write out all moved pages, if possible */
1601	filemap_invalidate_lock(mapping);	1603	down_write(&F2FS_I(inode)->i_mmap_sem);
1602	filemap_write_and_wait_range(mapping, offset, LLONG_MAX);	1604	filemap_write_and_wait_range(inode->i_mapping, offset, LLONG_MAX);
1603	truncate_pagecache(inode, offset);	1605	truncate_pagecache(inode, offset);
1604	filemap_invalidate_unlock(mapping);	1606	up_write(&F2FS_I(inode)->i_mmap_sem);
1605		1607
1606	if (!ret)	1608	if (!ret)
1607	f2fs_i_size_write(inode, new_size);	1609	f2fs_i_size_write(inode, new_size);
Lines 3435-3441 static int f2fs_release_compress_blocks(struct file *filp, unsigned long arg) Link Here
3435	goto out;	3437	goto out;
3436		3438
3437	down_write(&F2FS_I(inode)->i_gc_rwsem[WRITE]);	3439	down_write(&F2FS_I(inode)->i_gc_rwsem[WRITE]);
3438	filemap_invalidate_lock(inode->i_mapping);	3440	down_write(&F2FS_I(inode)->i_mmap_sem);
3439		3441
3440	last_idx = DIV_ROUND_UP(i_size_read(inode), PAGE_SIZE);	3442	last_idx = DIV_ROUND_UP(i_size_read(inode), PAGE_SIZE);
3441		3443
Lines 3471-3477 static int f2fs_release_compress_blocks(struct file *filp, unsigned long arg) Link Here
3471	}	3473	}
3472		3474
3473	up_write(&F2FS_I(inode)->i_gc_rwsem[WRITE]);	3475	up_write(&F2FS_I(inode)->i_gc_rwsem[WRITE]);
3474	filemap_invalidate_unlock(inode->i_mapping);	3476	up_write(&F2FS_I(inode)->i_mmap_sem);
3475	out:	3477	out:
3476	inode_unlock(inode);	3478	inode_unlock(inode);
3477		3479
Lines 3588-3594 static int f2fs_reserve_compress_blocks(struct file *filp, unsigned long arg) Link Here
3588	}	3590	}
3589		3591
3590	down_write(&F2FS_I(inode)->i_gc_rwsem[WRITE]);	3592	down_write(&F2FS_I(inode)->i_gc_rwsem[WRITE]);
3591	filemap_invalidate_lock(inode->i_mapping);	3593	down_write(&F2FS_I(inode)->i_mmap_sem);
3592		3594
3593	last_idx = DIV_ROUND_UP(i_size_read(inode), PAGE_SIZE);	3595	last_idx = DIV_ROUND_UP(i_size_read(inode), PAGE_SIZE);
3594		3596
Lines 3624-3630 static int f2fs_reserve_compress_blocks(struct file *filp, unsigned long arg) Link Here
3624	}	3626	}
3625		3627
3626	up_write(&F2FS_I(inode)->i_gc_rwsem[WRITE]);	3628	up_write(&F2FS_I(inode)->i_gc_rwsem[WRITE]);
3627	filemap_invalidate_unlock(inode->i_mapping);	3629	up_write(&F2FS_I(inode)->i_mmap_sem);
3628		3630
3629	if (ret >= 0) {	3631	if (ret >= 0) {
3630	clear_inode_flag(inode, FI_COMPRESS_RELEASED);	3632	clear_inode_flag(inode, FI_COMPRESS_RELEASED);
Lines 3743-3749 static int f2fs_sec_trim_file(struct file *filp, unsigned long arg) Link Here
3743	goto err;	3745	goto err;
3744		3746
3745	down_write(&F2FS_I(inode)->i_gc_rwsem[WRITE]);	3747	down_write(&F2FS_I(inode)->i_gc_rwsem[WRITE]);
3746	filemap_invalidate_lock(mapping);	3748	down_write(&F2FS_I(inode)->i_mmap_sem);
3747		3749
3748	ret = filemap_write_and_wait_range(mapping, range.start,	3750	ret = filemap_write_and_wait_range(mapping, range.start,
3749	to_end ? LLONG_MAX : end_addr - 1);	3751	to_end ? LLONG_MAX : end_addr - 1);
Lines 3830-3836 static int f2fs_sec_trim_file(struct file *filp, unsigned long arg) Link Here
3830	ret = f2fs_secure_erase(prev_bdev, inode, prev_index,	3832	ret = f2fs_secure_erase(prev_bdev, inode, prev_index,
3831	prev_block, len, range.flags);	3833	prev_block, len, range.flags);
3832	out:	3834	out:
3833	filemap_invalidate_unlock(mapping);	3835	up_write(&F2FS_I(inode)->i_mmap_sem);
3834	up_write(&F2FS_I(inode)->i_gc_rwsem[WRITE]);	3836	up_write(&F2FS_I(inode)->i_gc_rwsem[WRITE]);
3835	err:	3837	err:
3836	inode_unlock(inode);	3838	inode_unlock(inode);
Lines 4308-4316 static ssize_t f2fs_file_write_iter(struct kiocb iocb, struct iov_iter from) Link Here
4308	/* if we couldn't write data, we should deallocate blocks. */	4310	/* if we couldn't write data, we should deallocate blocks. */
4309	if (preallocated && i_size_read(inode) < target_size) {	4311	if (preallocated && i_size_read(inode) < target_size) {
4310	down_write(&F2FS_I(inode)->i_gc_rwsem[WRITE]);	4312	down_write(&F2FS_I(inode)->i_gc_rwsem[WRITE]);
4311	filemap_invalidate_lock(inode->i_mapping);	4313	down_write(&F2FS_I(inode)->i_mmap_sem);
4312	f2fs_truncate(inode);	4314	f2fs_truncate(inode);
4313	filemap_invalidate_unlock(inode->i_mapping);	4315	up_write(&F2FS_I(inode)->i_mmap_sem);
4314	up_write(&F2FS_I(inode)->i_gc_rwsem[WRITE]);	4316	up_write(&F2FS_I(inode)->i_gc_rwsem[WRITE]);
4315	}	4317	}
4316		4318

Lines 1289-1294 static struct inode f2fs_alloc_inode(struct super_block sb) Link Here

(-)a/fs/f2fs/super.c (+1 lines)
1289	mutex_init(&fi->inmem_lock);	1289	mutex_init(&fi->inmem_lock);
1290	init_rwsem(&fi->i_gc_rwsem[READ]);	1290	init_rwsem(&fi->i_gc_rwsem[READ]);
1291	init_rwsem(&fi->i_gc_rwsem[WRITE]);	1291	init_rwsem(&fi->i_gc_rwsem[WRITE]);
		1292	init_rwsem(&fi->i_mmap_sem);
1292	init_rwsem(&fi->i_xattr_sem);	1293	init_rwsem(&fi->i_xattr_sem);
1293		1294
1294	/* Will be used by directory only */	1295	/* Will be used by directory only */

Return to bug 1200450

Lines 3207-3218 static void f2fs_write_failed(struct address_space *mapping, loff_t to) Link Here

(-)a/fs/f2fs/data.c (-4 / +4 lines)
3207	/* In the fs-verity case, f2fs_end_enable_verity() does the truncate */	3207	/* In the fs-verity case, f2fs_end_enable_verity() does the truncate */
3208	if (to > i_size && !f2fs_verity_in_progress(inode)) {	3208	if (to > i_size && !f2fs_verity_in_progress(inode)) {
3209	down_write(&F2FS_I(inode)->i_gc_rwsem[WRITE]);	3209	down_write(&F2FS_I(inode)->i_gc_rwsem[WRITE]);
3210	filemap_invalidate_lock(mapping);	3210	down_write(&F2FS_I(inode)->i_mmap_sem);
3211		3211
3212	truncate_pagecache(inode, i_size);	3212	truncate_pagecache(inode, i_size);
3213	f2fs_truncate_blocks(inode, i_size, true);	3213	f2fs_truncate_blocks(inode, i_size, true);
3214		3214
3215	filemap_invalidate_unlock(mapping);	3215	up_write(&F2FS_I(inode)->i_mmap_sem);
3216	up_write(&F2FS_I(inode)->i_gc_rwsem[WRITE]);	3216	up_write(&F2FS_I(inode)->i_gc_rwsem[WRITE]);
3217	}	3217	}
3218	}	3218	}
Lines 3872-3878 static int f2fs_migrate_blocks(struct inode *inode, block_t start_blk, Link Here
3872	int ret = 0;	3872	int ret = 0;
3873		3873
3874	down_write(&F2FS_I(inode)->i_gc_rwsem[WRITE]);	3874	down_write(&F2FS_I(inode)->i_gc_rwsem[WRITE]);
3875	filemap_invalidate_lock(inode->i_mapping);	3875	down_write(&F2FS_I(inode)->i_mmap_sem);
3876		3876
3877	set_inode_flag(inode, FI_ALIGNED_WRITE);	3877	set_inode_flag(inode, FI_ALIGNED_WRITE);
3878		3878
Lines 3914-3920 static int f2fs_migrate_blocks(struct inode *inode, block_t start_blk, Link Here
3914	clear_inode_flag(inode, FI_DO_DEFRAG);	3914	clear_inode_flag(inode, FI_DO_DEFRAG);
3915	clear_inode_flag(inode, FI_ALIGNED_WRITE);	3915	clear_inode_flag(inode, FI_ALIGNED_WRITE);
3916		3916
3917	filemap_invalidate_unlock(inode->i_mapping);	3917	up_write(&F2FS_I(inode)->i_mmap_sem);
3918	up_write(&F2FS_I(inode)->i_gc_rwsem[WRITE]);	3918	up_write(&F2FS_I(inode)->i_gc_rwsem[WRITE]);
3919		3919
3920	return ret;	3920	return ret;