Move type based filtering to libutil dir_tree_iterator_t

Limited unit testing for the flags is added, particularly the
abillity to recurse into sub-directories, but not report the
parents as individual entries.

Signed-off-by: David Oberhollenzer <david.oberhollenzer@sigma-star.at>

7 files changed, 262 insertions, 74 deletions

diff --git a/bin/gensquashfs/src/fstree_from_dir.c b/bin/gensquashfs/src/fstree_from_dir.c
index 9241428..4c6a828 100644
--- a/bin/gensquashfs/src/fstree_from_dir.c
+++ b/bin/gensquashfs/src/fstree_from_dir.c
@@ -12,32 +12,6 @@
 #include <string.h>
 #include <errno.h>
 
-static bool should_skip(const dir_iterator_t *dir, const dir_entry_t *ent,
-			unsigned int flags)
-{
-	if ((flags & DIR_SCAN_ONE_FILESYSTEM) && ent->dev != dir->dev)
-		return true;
-
-	switch (ent->mode & S_IFMT) {
-	case S_IFSOCK:
-		return (flags & DIR_SCAN_NO_SOCK) != 0;
-	case S_IFLNK:
-		return (flags & DIR_SCAN_NO_SLINK) != 0;
-	case S_IFREG:
-		return (flags & DIR_SCAN_NO_FILE) != 0;
-	case S_IFBLK:
-		return (flags & DIR_SCAN_NO_BLK) != 0;
-	case S_IFCHR:
-		return (flags & DIR_SCAN_NO_CHR) != 0;
-	case S_IFIFO:
-		return (flags & DIR_SCAN_NO_FIFO) != 0;
-	default:
-		break;
-	}
-
-	return false;
-}
-
 static sqfs_u32 clamp_timestamp(sqfs_s64 ts)
 {
 	if (ts < 0)
@@ -82,11 +56,6 @@ static int scan_dir(fstree_t *fs, tree_node_t *root, dir_iterator_t *dir,
 			return -1;
 		}
 
-		if (should_skip(dir, ent, flags)) {
-			free(ent);
-			continue;
-		}
-
 		if (S_ISLNK(ent->mode)) {
 			ret = dir->read_link(dir, &extra);
 			if (ret) {
@@ -96,9 +65,6 @@ static int scan_dir(fstree_t *fs, tree_node_t *root, dir_iterator_t *dir,
 			}
 		}
 
-		if (!(flags & DIR_SCAN_KEEP_TIME))
-			ent->mtime = fs->defaults.mtime;
-
 		if (S_ISDIR(ent->mode) && (flags & DIR_SCAN_NO_DIR)) {
 			n = fstree_get_node_by_path(fs, root, ent->name,
 						    false, false);
@@ -140,11 +106,7 @@ static int scan_dir(fstree_t *fs, tree_node_t *root, dir_iterator_t *dir,
 			if (S_ISDIR(n->mode))
 				dir_tree_iterator_skip(dir);
 			discard_node(n->parent, n);
-			continue;
 		}
-
-		if ((flags & DIR_SCAN_NO_RECURSION) && S_ISDIR(n->mode))
-			dir_tree_iterator_skip(dir);
 	}
 
 	return 0;
@@ -156,6 +118,7 @@ int fstree_from_subdir(fstree_t *fs, tree_node_t *root,
 		       unsigned int flags)
 {
 	dir_iterator_t *dir;
+	dir_tree_cfg_t cfg;
 	size_t plen, slen;
 	char *temp = NULL;
 	int ret;
@@ -186,7 +149,11 @@ int fstree_from_subdir(fstree_t *fs, tree_node_t *root,
 		path = temp;
 	}
 
-	dir = dir_tree_iterator_create(path);
+	memset(&cfg, 0, sizeof(cfg));
+	cfg.flags = flags & ~(DIR_SCAN_NO_DIR);
+	cfg.def_mtime = fs->defaults.mtime;
+
+	dir = dir_tree_iterator_create(path, &cfg);
 	free(temp);
 	if (dir == NULL)
 		return -1;
diff --git a/bin/gensquashfs/src/mkfs.h b/bin/gensquashfs/src/mkfs.h
index 28bcfde..a58365d 100644
--- a/bin/gensquashfs/src/mkfs.h
+++ b/bin/gensquashfs/src/mkfs.h
@@ -42,22 +42,6 @@
 #include <errno.h>
 #include <ctype.h>
 
-enum {
-	DIR_SCAN_KEEP_TIME = 0x01,
-
-	DIR_SCAN_ONE_FILESYSTEM = 0x02,
-
-	DIR_SCAN_NO_RECURSION = 0x04,
-
-	DIR_SCAN_NO_SOCK = 0x0008,
-	DIR_SCAN_NO_SLINK = 0x0010,
-	DIR_SCAN_NO_FILE = 0x0020,
-	DIR_SCAN_NO_BLK = 0x0040,
-	DIR_SCAN_NO_DIR = 0x0080,
-	DIR_SCAN_NO_CHR = 0x0100,
-	DIR_SCAN_NO_FIFO = 0x0200,
-};
-
 /*
   Optionally used by fstree_from_dir and fstree_from_subdir to
   execute custom actions for each discovered node.
diff --git a/include/util/dir_tree_iterator.h b/include/util/dir_tree_iterator.h
index 953459f..3c624c9 100644
--- a/include/util/dir_tree_iterator.h
+++ b/include/util/dir_tree_iterator.h
@@ -9,7 +9,27 @@
 
 #include "util/dir_iterator.h"
 
-dir_iterator_t *dir_tree_iterator_create(const char *path);
+enum {
+	DIR_SCAN_NO_SOCK = 0x0001,
+	DIR_SCAN_NO_SLINK = 0x0002,
+	DIR_SCAN_NO_FILE = 0x0004,
+	DIR_SCAN_NO_BLK = 0x0008,
+	DIR_SCAN_NO_DIR = 0x0010,
+	DIR_SCAN_NO_CHR = 0x0020,
+	DIR_SCAN_NO_FIFO = 0x0040,
+
+	DIR_SCAN_KEEP_TIME = 0x0100,
+	DIR_SCAN_ONE_FILESYSTEM = 0x0200,
+	DIR_SCAN_NO_RECURSION = 0x0400,
+};
+
+typedef struct {
+	sqfs_u32 flags;
+	sqfs_s64 def_mtime;
+} dir_tree_cfg_t;
+
+dir_iterator_t *dir_tree_iterator_create(const char *path,
+					 const dir_tree_cfg_t *cfg);
 
 void dir_tree_iterator_skip(dir_iterator_t *it);
 
diff --git a/lib/util/Makemodule.am b/lib/util/Makemodule.am
index 26056ea..37eac93 100644
--- a/lib/util/Makemodule.am
+++ b/lib/util/Makemodule.am
@@ -91,11 +91,16 @@ test_dir_tree_iterator_LDADD = libutil.a libcompat.a
 test_dir_tree_iterator_CPPFLAGS = $(AM_CPPFLAGS)
 test_dir_tree_iterator_CPPFLAGS += -DTESTPATH=$(top_srcdir)/lib/util/test/testdir
 
+test_dir_tree_iterator2_SOURCES = lib/util/test/dir_tree_iterator2.c
+test_dir_tree_iterator2_LDADD = libutil.a libcompat.a
+test_dir_tree_iterator2_CPPFLAGS = $(AM_CPPFLAGS)
+test_dir_tree_iterator2_CPPFLAGS += -DTESTPATH=$(top_srcdir)/lib/util/test/testdir
+
 LIBUTIL_TESTS = \
 	test_str_table test_rbtree test_xxhash test_threadpool test_ismemzero \
 	test_canonicalize_name test_filename_sane test_filename_sane_w32 \
 	test_sdate_epoch test_hex_decode test_base64_decode test_dir_iterator \
-	test_dir_tree_iterator
+	test_dir_tree_iterator test_dir_tree_iterator2
 
 check_PROGRAMS += $(LIBUTIL_TESTS)
 TESTS += $(LIBUTIL_TESTS)
diff --git a/lib/util/src/dir_tree_iterator.c b/lib/util/src/dir_tree_iterator.c
index 88b6840..ccc63ac 100644
--- a/lib/util/src/dir_tree_iterator.c
+++ b/lib/util/src/dir_tree_iterator.c
@@ -21,6 +21,7 @@ typedef struct dir_stack_t {
 typedef struct {
 	dir_iterator_t base;
 
+	dir_tree_cfg_t cfg;
 	int state;
 	dir_stack_t *top;
 } dir_tree_iterator_t;
@@ -50,6 +51,36 @@ static int push(dir_tree_iterator_t *it, const char *name, dir_iterator_t *dir)
 	return 0;
 }
 
+static bool should_skip(const dir_tree_iterator_t *dir, const dir_entry_t *ent)
+{
+	if (!strcmp(ent->name, ".") || !strcmp(ent->name, ".."))
+		return true;
+
+	if ((dir->cfg.flags & DIR_SCAN_ONE_FILESYSTEM)) {
+		if (ent->dev != ((const dir_iterator_t *)dir)->dev)
+			return true;
+	}
+
+	switch (ent->mode & S_IFMT) {
+	case S_IFSOCK:
+		return (dir->cfg.flags & DIR_SCAN_NO_SOCK) != 0;
+	case S_IFLNK:
+		return (dir->cfg.flags & DIR_SCAN_NO_SLINK) != 0;
+	case S_IFREG:
+		return (dir->cfg.flags & DIR_SCAN_NO_FILE) != 0;
+	case S_IFBLK:
+		return (dir->cfg.flags & DIR_SCAN_NO_BLK) != 0;
+	case S_IFCHR:
+		return (dir->cfg.flags & DIR_SCAN_NO_CHR) != 0;
+	case S_IFIFO:
+		return (dir->cfg.flags & DIR_SCAN_NO_FIFO) != 0;
+	default:
+		break;
+	}
+
+	return false;
+}
+
 /*****************************************************************************/
 
 static void destroy(sqfs_object_t *obj)
@@ -65,13 +96,17 @@ static void destroy(sqfs_object_t *obj)
 static int next(dir_iterator_t *base, dir_entry_t **out)
 {
 	dir_tree_iterator_t *it = (dir_tree_iterator_t *)base;
-	dir_iterator_t *sub = NULL;
-	dir_entry_t *ent = NULL;
-	dir_stack_t *sit = NULL;
-	size_t plen = 0;
+	dir_iterator_t *sub;
+	dir_entry_t *ent;
+	dir_stack_t *sit;
+	size_t plen;
 	int ret;
-
+retry:
 	*out = NULL;
+	sub = NULL;
+	ent = NULL;
+	sit = NULL;
+	plen = 0;
 
 	if (it->state != 0)
 		return it->state;
@@ -91,7 +126,7 @@ static int next(dir_iterator_t *base, dir_entry_t **out)
 			continue;
 		}
 
-		if (!strcmp(ent->name, ".") || !strcmp(ent->name, "..")) {
+		if (should_skip(it, ent)) {
 			free(ent);
 			ent = NULL;
 			continue;
@@ -130,15 +165,26 @@ static int next(dir_iterator_t *base, dir_entry_t **out)
 		}
 	}
 
+	if (!(it->cfg.flags & DIR_SCAN_KEEP_TIME))
+		ent->mtime = it->cfg.def_mtime;
+
 	if (S_ISDIR(ent->mode)) {
-		ret = it->top->dir->open_subdir(it->top->dir, &sub);
-		if (ret != 0)
-			goto fail;
+		if (!(it->cfg.flags & DIR_SCAN_NO_RECURSION)) {
+			ret = it->top->dir->open_subdir(it->top->dir, &sub);
+			if (ret != 0)
+				goto fail;
+
+			ret = push(it, ent->name + plen, sub);
+			sqfs_drop(sub);
+			if (ret != 0)
+				goto fail;
+		}
 
-		ret = push(it, ent->name + plen, sub);
-		sqfs_drop(sub);
-		if (ret != 0)
-			goto fail;
+		if (it->cfg.flags & DIR_SCAN_NO_DIR) {
+			free(ent);
+			ent = NULL;
+			goto retry;
+		}
 	}
 
 	*out = ent;
@@ -173,7 +219,8 @@ static int open_subdir(dir_iterator_t *base, dir_iterator_t **out)
 	return it->top->dir->open_subdir(it->top->dir, out);
 }
 
-dir_iterator_t *dir_tree_iterator_create(const char *path)
+dir_iterator_t *dir_tree_iterator_create(const char *path,
+					 const dir_tree_cfg_t *cfg)
 {
 	dir_tree_iterator_t *it = calloc(1, sizeof(*it));
 	dir_iterator_t *dir;
@@ -184,6 +231,8 @@ dir_iterator_t *dir_tree_iterator_create(const char *path)
 		return NULL;
 	}
 
+	it->cfg = *cfg;
+
 	dir = dir_iterator_create(path);
 	if (dir == NULL)
 		goto fail;
diff --git a/lib/util/test/dir_tree_iterator.c b/lib/util/test/dir_tree_iterator.c
index e47efed..7c1aa88 100644
--- a/lib/util/test/dir_tree_iterator.c
+++ b/lib/util/test/dir_tree_iterator.c
@@ -23,11 +23,14 @@ int main(int argc, char **argv)
 {
 	dir_entry_t *ent[17];
 	dir_iterator_t *dir;
+	dir_tree_cfg_t cfg;
 	size_t i;
 	int ret;
 	(void)argc; (void)argv;
 
-	dir = dir_tree_iterator_create(TEST_PATH);
+	memset(&cfg, 0, sizeof(cfg));
+
+	dir = dir_tree_iterator_create(TEST_PATH, &cfg);
 	TEST_NOT_NULL(dir);
 
 	for (i = 0; i < 16; ++i) {
@@ -88,7 +91,7 @@ int main(int argc, char **argv)
 	/* retry with skipping */
 	printf("**********\n");
 
-	dir = dir_tree_iterator_create(TEST_PATH);
+	dir = dir_tree_iterator_create(TEST_PATH, &cfg);
 	TEST_NOT_NULL(dir);
 
 	for (i = 0; i < 13; ++i) {
@@ -146,7 +149,7 @@ int main(int argc, char **argv)
 	/* retry with skipping */
 	printf("**********\n");
 
-	dir = dir_tree_iterator_create(TEST_PATH);
+	dir = dir_tree_iterator_create(TEST_PATH, &cfg);
 	TEST_NOT_NULL(dir);
 
 	for (i = 0; i < 9; ++i) {
diff --git a/lib/util/test/dir_tree_iterator2.c b/lib/util/test/dir_tree_iterator2.c
new file mode 100644
index 0000000..4b2b6f1
--- /dev/null
+++ b/lib/util/test/dir_tree_iterator2.c
@@ -0,0 +1,160 @@
+/* SPDX-License-Identifier: GPL-3.0-or-later */
+/*
+ * dir_tree_iterator2.c
+ *
+ * Copyright (C) 2023 David Oberhollenzer <goliath@infraroot.at>
+ */
+#include "config.h"
+
+#include "util/dir_tree_iterator.h"
+#include "sqfs/error.h"
+#include "util/test.h"
+#include "compat.h"
+
+static int compare_entries(const void *a, const void *b)
+{
+	const dir_entry_t *const *lhs = a;
+	const dir_entry_t *const *rhs = b;
+
+	return strcmp((*lhs)->name, (*rhs)->name);
+}
+
+int main(int argc, char **argv)
+{
+	dir_entry_t *ent[17];
+	dir_iterator_t *dir;
+	dir_tree_cfg_t cfg;
+	size_t i;
+	int ret;
+	(void)argc; (void)argv;
+
+	/********** without files **********/
+	memset(&cfg, 0, sizeof(cfg));
+	cfg.flags |= DIR_SCAN_NO_FILE;
+
+	dir = dir_tree_iterator_create(TEST_PATH, &cfg);
+	TEST_NOT_NULL(dir);
+
+	for (i = 0; i < 4; ++i) {
+		ret = dir->next(dir, &ent[i]);
+		TEST_NOT_NULL(ent[i]);
+		TEST_EQUAL_I(ret, 0);
+		printf("READ %s\n", ent[i]->name);
+	}
+
+	ret = dir->next(dir, &ent[4]);
+	TEST_NULL(ent[4]);
+	TEST_ASSERT(ret > 0);
+
+	dir = sqfs_drop(dir);
+
+	qsort(ent, 4, sizeof(ent[0]), compare_entries);
+
+	printf("After sort:\n");
+	for (i = 0; i < 4; ++i)
+		printf("%s\n", ent[i]->name);
+
+	TEST_STR_EQUAL(ent[0]->name, "dira");
+	TEST_ASSERT(S_ISDIR(ent[0]->mode));
+	TEST_STR_EQUAL(ent[1]->name, "dirb");
+	TEST_ASSERT(S_ISDIR(ent[1]->mode));
+	TEST_STR_EQUAL(ent[2]->name, "dirb/dirx");
+	TEST_ASSERT(S_ISDIR(ent[2]->mode));
+	TEST_STR_EQUAL(ent[3]->name, "dirc");
+	TEST_ASSERT(S_ISDIR(ent[3]->mode));
+
+	for (i = 0; i < 4; ++i)
+		free(ent[i]);
+
+	/********** recursive but without dirs **********/
+	memset(&cfg, 0, sizeof(cfg));
+	cfg.flags |= DIR_SCAN_NO_DIR;
+
+	dir = dir_tree_iterator_create(TEST_PATH, &cfg);
+	TEST_NOT_NULL(dir);
+
+	for (i = 0; i < 12; ++i) {
+		ret = dir->next(dir, &ent[i]);
+		TEST_NOT_NULL(ent[i]);
+		TEST_EQUAL_I(ret, 0);
+		printf("READ %s\n", ent[i]->name);
+	}
+
+	ret = dir->next(dir, &ent[12]);
+	TEST_NULL(ent[12]);
+	TEST_ASSERT(ret > 0);
+
+	dir = sqfs_drop(dir);
+
+	qsort(ent, 12, sizeof(ent[0]), compare_entries);
+
+	printf("After sort:\n");
+	for (i = 0; i < 12; ++i)
+		printf("%s\n", ent[i]->name);
+
+	TEST_STR_EQUAL(ent[0]->name, "dira/file_a0");
+	TEST_ASSERT(S_ISREG(ent[0]->mode));
+	TEST_STR_EQUAL(ent[1]->name, "dira/file_a1");
+	TEST_ASSERT(S_ISREG(ent[1]->mode));
+	TEST_STR_EQUAL(ent[2]->name, "dira/file_a2");
+	TEST_ASSERT(S_ISREG(ent[2]->mode));
+	TEST_STR_EQUAL(ent[3]->name, "dirb/dirx/file_x0");
+	TEST_ASSERT(S_ISREG(ent[3]->mode));
+	TEST_STR_EQUAL(ent[4]->name, "dirb/dirx/file_x1");
+	TEST_ASSERT(S_ISREG(ent[4]->mode));
+	TEST_STR_EQUAL(ent[5]->name, "dirb/dirx/file_x2");
+	TEST_ASSERT(S_ISREG(ent[5]->mode));
+	TEST_STR_EQUAL(ent[6]->name, "dirb/file_b0");
+	TEST_ASSERT(S_ISREG(ent[6]->mode));
+	TEST_STR_EQUAL(ent[7]->name, "dirb/file_b1");
+	TEST_ASSERT(S_ISREG(ent[7]->mode));
+	TEST_STR_EQUAL(ent[8]->name, "dirb/file_b2");
+	TEST_ASSERT(S_ISREG(ent[8]->mode));
+	TEST_STR_EQUAL(ent[9]->name, "dirc/file_c0");
+	TEST_ASSERT(S_ISREG(ent[9]->mode));
+	TEST_STR_EQUAL(ent[10]->name, "dirc/file_c1");
+	TEST_ASSERT(S_ISREG(ent[10]->mode));
+	TEST_STR_EQUAL(ent[11]->name, "dirc/file_c2");
+	TEST_ASSERT(S_ISREG(ent[11]->mode));
+
+	for (i = 0; i < 12; ++i)
+		free(ent[i]);
+
+	/********** non-recursive **********/
+	memset(&cfg, 0, sizeof(cfg));
+	cfg.flags |= DIR_SCAN_NO_RECURSION;
+
+	dir = dir_tree_iterator_create(TEST_PATH, &cfg);
+	TEST_NOT_NULL(dir);
+
+	for (i = 0; i < 3; ++i) {
+		ret = dir->next(dir, &ent[i]);
+		TEST_NOT_NULL(ent[i]);
+		TEST_EQUAL_I(ret, 0);
+		printf("READ %s\n", ent[i]->name);
+	}
+
+	ret = dir->next(dir, &ent[3]);
+	TEST_NULL(ent[3]);
+	TEST_ASSERT(ret > 0);
+
+	dir = sqfs_drop(dir);
+
+	qsort(ent, 3, sizeof(ent[0]), compare_entries);
+
+	printf("After sort:\n");
+	for (i = 0; i < 3; ++i)
+		printf("%s\n", ent[i]->name);
+
+	TEST_STR_EQUAL(ent[0]->name, "dira");
+	TEST_ASSERT(S_ISDIR(ent[0]->mode));
+	TEST_STR_EQUAL(ent[1]->name, "dirb");
+	TEST_ASSERT(S_ISDIR(ent[1]->mode));
+	TEST_STR_EQUAL(ent[2]->name, "dirc");
+	TEST_ASSERT(S_ISDIR(ent[2]->mode));
+
+	for (i = 0; i < 3; ++i)
+		free(ent[i]);
+
+	return EXIT_SUCCESS;
+}