kernel/fs: Handle O_RDONLY | O_TRUNC in the VFS rather than filesystems.

The POSIX specification says that the behavior of specifying O_TRUNC
with O_RDONLY is "undefined", but the Linux manpages ominously state
"On many systems the file is actually truncated." I tested this,
and indeed on Linux the file is actually truncated.

This doesn't seem like a very sensible behavior, so in this commit
it's changed to return B_NOT_ALLOWED (EPERM) if those flags are
specified together. The FAT driver already did this, but most other
filesystem drivers just checked write access permissions and
truncated the file anyway; so this is indeed a behavioral change.

Change-Id: If2e76782743ee91d934dc7e0c2f306f37b159a0f
Reviewed-on: https://review.haiku-os.org/c/haiku/+/8625
Reviewed-by: waddlesplash <[email protected]>
Tested-by: Commit checker robot <[email protected]>
Reviewed-by: Axel Dörfler <[email protected]>

src/add-ons/kernel/file_systems/bfs/Attribute.cpp

@@ -69,8 +69,7 @@ Attribute::CheckAccess(const char* name, int openMode)
 		|| !strcmp(name, "size")*/)
 		RETURN_ERROR(B_NOT_ALLOWED);
 
-	return fInode->CheckPermissions(open_mode_to_access(openMode)
+	return fInode->CheckPermissions(open_mode_to_access(openMode));
-		| (openMode & O_TRUNC ? W_OK : 0));
 }
 
 

src/add-ons/kernel/file_systems/bfs/Inode.cpp

@@ -2669,8 +2669,7 @@ Inode::Create(Transaction& transaction, Inode* parent, const char* name,
 				return B_NOT_A_DIRECTORY;
 
 			// we want to open the file, so we should have the rights to do so
-			if (inode->CheckPermissions(open_mode_to_access(openMode)
+			if (inode->CheckPermissions(open_mode_to_access(openMode)) != B_OK)
-					| ((openMode & O_TRUNC) != 0 ? W_OK : 0)) != B_OK)
 				return B_NOT_ALLOWED;
 
 			if ((openMode & O_TRUNC) != 0) {

src/add-ons/kernel/file_systems/bfs/kernel_interface.cpp

@@ -1368,8 +1368,7 @@ bfs_open(fs_volume* _volume, fs_vnode* _node, int openMode, void** _cookie)
 	if ((openMode & O_DIRECTORY) != 0 && !inode->IsDirectory())
 		return B_NOT_A_DIRECTORY;
 
-	status_t status = inode->CheckPermissions(open_mode_to_access(openMode)
+	status_t status = inode->CheckPermissions(open_mode_to_access(openMode));
-		| ((openMode & O_TRUNC) != 0 ? W_OK : 0));
 	if (status != B_OK)
 		RETURN_ERROR(status);
 

src/add-ons/kernel/file_systems/btrfs/Attribute.cpp

@@ -49,8 +49,7 @@ Attribute::~Attribute()
 status_t
 Attribute::CheckAccess(const char* name, int openMode)
 {
-	return fInode->CheckPermissions(open_mode_to_access(openMode)
+	return fInode->CheckPermissions(open_mode_to_access(openMode));
-		| (openMode & O_TRUNC ? W_OK : 0));
 }
 
 

src/add-ons/kernel/file_systems/btrfs/kernel_interface.cpp

@@ -541,8 +541,7 @@ btrfs_open(fs_volume* /*_volume*/, fs_vnode* _node, int openMode,
 	if (inode->IsDirectory() && (openMode & O_RWMASK) != 0)
 		return B_IS_A_DIRECTORY;
 
-	status_t status =  inode->CheckPermissions(open_mode_to_access(openMode)
+	status_t status =  inode->CheckPermissions(open_mode_to_access(openMode));
-		| (openMode & O_TRUNC ? W_OK : 0));
 	if (status != B_OK)
 		return status;
 

src/add-ons/kernel/file_systems/exfat/kernel_interface.cpp

@@ -462,8 +462,7 @@ exfat_open(fs_volume* /*_volume*/, fs_vnode* _node, int openMode,
 	if (inode->IsDirectory() && (openMode & O_RWMASK) != 0)
 		return B_IS_A_DIRECTORY;
 
-	status_t status =  inode->CheckPermissions(open_mode_to_access(openMode)
+	status_t status =  inode->CheckPermissions(open_mode_to_access(openMode));
-		| (openMode & O_TRUNC ? W_OK : 0));
 	if (status != B_OK)
 		return status;
 

src/add-ons/kernel/file_systems/ext2/Attribute.cpp

@@ -64,8 +64,7 @@ Attribute::InitCheck()
 status_t
 Attribute::CheckAccess(const char* name, int openMode)
 {
-	return fInode->CheckPermissions(open_mode_to_access(openMode)
+	return fInode->CheckPermissions(open_mode_to_access(openMode));
-		| (openMode & O_TRUNC ? W_OK : 0));
 }
 
 

src/add-ons/kernel/file_systems/ext2/Inode.cpp

@@ -546,8 +546,7 @@ Inode::Create(Transaction& transaction, Inode* parent, const char* name,
 			if ((openMode & O_DIRECTORY) != 0 && !inode->IsDirectory())
 				return B_NOT_A_DIRECTORY;
 
-			if (inode->CheckPermissions(open_mode_to_access(openMode)
+			if (inode->CheckPermissions(open_mode_to_access(openMode)) != B_OK)
-					| ((openMode & O_TRUNC) != 0 ? W_OK : 0)) != B_OK)
 				return B_NOT_ALLOWED;
 
 			if ((openMode & O_TRUNC) != 0) {

src/add-ons/kernel/file_systems/ext2/kernel_interface.cpp

@@ -1148,8 +1148,7 @@ ext2_open(fs_volume* _volume, fs_vnode* _node, int openMode, void** _cookie)
 	if (inode->IsDirectory() && (openMode & O_RWMASK) != 0)
 		return B_IS_A_DIRECTORY;
 
-	status_t status =  inode->CheckPermissions(open_mode_to_access(openMode)
+	status_t status =  inode->CheckPermissions(open_mode_to_access(openMode));
-		| (openMode & O_TRUNC ? W_OK : 0));
 	if (status != B_OK)
 		return status;
 

src/add-ons/kernel/file_systems/fat/kernel_interface.cpp

@@ -2177,9 +2177,6 @@ dosfs_open(fs_volume* volume, fs_vnode* vnode, int openMode, void** _cookie)
 	if ((bsdVolume->mnt_flag & MNT_RDONLY) != 0 || (fatNode->de_Attributes & ATTR_READONLY) != 0)
 		openMode = (openMode & ~O_RWMASK) | O_RDONLY;
 
-	if ((openMode & O_TRUNC) != 0 && (openMode & O_RWMASK) == O_RDONLY)
-		return B_NOT_ALLOWED;
-
 	status_t status = _dosfs_access(bsdVolume, bsdNode, open_mode_to_access(openMode));
 	if (status != B_OK)
 		RETURN_ERROR(status);

src/add-ons/kernel/file_systems/xfs/kernel_interface.cpp

@@ -274,8 +274,7 @@ xfs_open(fs_volume * /*_volume*/, fs_vnode *_node, int openMode,
 	if (inode->IsDirectory() && (openMode & O_RWMASK) != 0)
 		return B_IS_A_DIRECTORY;
 
-	status_t status =  inode->CheckPermissions(open_mode_to_access(openMode)
+	status_t status =  inode->CheckPermissions(open_mode_to_access(openMode));
-		| (openMode & O_TRUNC ? W_OK : 0));
 	if (status != B_OK)
 		return status;
 
@@ -556,8 +555,7 @@ xfs_open_attr(fs_volume *_volume, fs_vnode *_node, const char *name,
 
 	Inode* inode = (Inode*)_node->private_node;
 
-	int accessMode = open_mode_to_access(openMode) | (openMode & O_TRUNC ? W_OK : 0);
+	status = inode->CheckPermissions(open_mode_to_access(openMode));
-	status = inode->CheckPermissions(accessMode);
 	if (status < B_OK)
 		return status;
 

src/system/kernel/fs/vfs.cpp

@@ -2833,12 +2833,14 @@ get_new_fd(struct fd_ops* ops, struct fs_mount* mount, struct vnode* vnode,
 
 	// If the vnode is locked, we don't allow creating a new file/directory
 	// file_descriptor for it
-	if (vnode && vnode->mandatory_locked_by != NULL
+	if (vnode != NULL && vnode->mandatory_locked_by != NULL
-		&& (ops == &sFileOps || ops == &sDirectoryOps))
+			&& (ops == &sFileOps || ops == &sDirectoryOps))
 		return B_BUSY;
 
 	if ((openMode & O_RDWR) != 0 && (openMode & O_WRONLY) != 0)
 		return B_BAD_VALUE;
+	if ((openMode & O_RWMASK) == O_RDONLY && (openMode & O_TRUNC) != 0)
+		return B_NOT_ALLOWED;
 
 	descriptor = alloc_fd();
 	if (!descriptor)