btrfs-progs: mkfs/rootdir: factor out subvol validation code into a helper

Currently we do the validation of subvolume parameters inside
mkfs/main.c, but considering all things like structure rootdir_subvol
are all inside rootdir.[ch], it's better to move the validation part
into rootdir.[ch].

Extract the validation part into a helper,
btrfs_mkfs_validate_subvols(), into rootdir.[ch].

Furthermore since we're here, also slightly enhance the duplicated
subvolume check, so that the runtime is halved.

Signed-off-by: Qu Wenruo <wqu@suse.com>

Author: adam900710

mkfs/main.c

@@ -1527,50 +1527,9 @@ int BOX_MAIN(mkfs)(int argc, char **argv)
 		}
 	}
 
-	list_for_each_entry(rds, &subvols, list) {
-		char path[PATH_MAX];
-		struct rootdir_subvol *rds2;
-
-		if (path_cat_out(path, source_dir, rds->dir)) {
-			error("path invalid: %s", path);
-			ret = 1;
-			goto error;
-		}
-
-		if (!realpath(path, rds->full_path)) {
-			error("could not get canonical path: %s", rds->dir);
-			ret = 1;
-			goto error;
-		}
-
-		if (!path_exists(rds->full_path)) {
-			error("subvolume path does not exist: %s", rds->dir);
-			ret = 1;
-			goto error;
-		}
-
-		if (!path_is_dir(rds->full_path)) {
-			error("subvolume is not a directory: %s", rds->dir);
-			ret = 1;
-			goto error;
-		}
-
-		if (!path_is_in_dir(source_dir, rds->full_path)) {
-			error("subvolume %s is not a child of %s", rds->dir, source_dir);
-			ret = 1;
-			goto error;
-		}
-
-		for (rds2 = list_first_entry(&subvols, struct rootdir_subvol, list);
-		     rds2 != rds;
-		     rds2 = list_next_entry(rds2, list)) {
-			if (strcmp(rds2->full_path, rds->full_path) == 0) {
-				error("subvolume specified more than once: %s", rds->dir);
-				ret = 1;
-				goto error;
-			}
-		}
-	}
+	ret = btrfs_mkfs_validate_subvols(source_dir, &subvols);
+	if (ret < 0)
+		goto error;
 
 	list_for_each_entry(rif, &inode_flags_list, list) {
 		char path[PATH_MAX];

mkfs/rootdir.c

@@ -1077,6 +1077,52 @@ static ssize_t zstd_compress_inline_extent(char *buf, u64 size, char **comp_buf)
 }
 #endif
 
+int btrfs_mkfs_validate_subvols(const char *source_dir, struct list_head *subvols)
+{
+	struct rootdir_subvol *rds;
+
+	list_for_each_entry(rds, subvols, list) {
+		char path[PATH_MAX];
+		struct rootdir_subvol *rds2;
+		int ret;
+
+		ret = path_cat_out(path, source_dir, rds->dir);
+		if (ret < 0) {
+			errno = -ret;
+			error("path invalid '%s': %m", path);
+			return ret;
+		}
+		if (!realpath(path, rds->full_path)) {
+			ret = -errno;
+			error("could not get canonical path of '%s': %m", rds->dir);
+			return ret;
+		}
+		ret = path_exists(rds->full_path);
+		if (ret < 0) {
+			error("subvolume path does not exist: %s", rds->dir);
+			return ret;
+		}
+		ret = path_is_dir(rds->full_path);
+		if (ret < 0) {
+			error("subvolume is not a directory: %s", rds->dir);
+			return ret;
+		}
+		list_for_each_entry(rds2, subvols, list) {
+			/*
+			 * Only compare entries before us, So we won't compare
+			 * the same pair twice.
+			 */
+			if (rds2 == rds)
+				break;
+			if (strcmp(rds2->full_path, rds->full_path) == 0) {
+				error("subvolume specified more than once: %s", rds->dir);
+				return -EINVAL;
+			}
+		}
+	}
+	return 0;
+}
+
 static int add_file_items(struct btrfs_trans_handle *trans,
 			  struct btrfs_root *root,
 			  struct btrfs_inode_item *btrfs_inode, u64 objectid,

mkfs/rootdir.h

@@ -39,6 +39,7 @@ struct btrfs_root;
 
 struct rootdir_subvol {
 	struct list_head list;
+	/* The path inside the source_dir. */
 	char dir[PATH_MAX];
 	char full_path[PATH_MAX];
 	bool is_default;
@@ -59,6 +60,7 @@ struct rootdir_inode_flags_entry {
 	bool nodatasum;
 };
 
+int btrfs_mkfs_validate_subvols(const char *source_dir, struct list_head *subvols);
 int btrfs_mkfs_fill_dir(struct btrfs_trans_handle *trans, const char *source_dir,
 			struct btrfs_root *root, struct list_head *subvols,
 			struct list_head *inode_flags_list,