libtar: add a dir_iterator_t implementation for tar files

The existing istream_t wrapper is mered into this one as well, we
can open the files via the iterators open_file_ro function. Unit
tests and tar2sqfs are modified accordingly.

Signed-off-by: David Oberhollenzer <david.oberhollenzer@sigma-star.at>

19 files changed, 848 insertions, 669 deletions

diff --git a/bin/gensquashfs/test/fstree_glob1.c b/bin/gensquashfs/test/fstree_glob1.c
index 8df1d0f..f7a61e0 100644
--- a/bin/gensquashfs/test/fstree_glob1.c
+++ b/bin/gensquashfs/test/fstree_glob1.c
@@ -76,7 +76,7 @@ static void check_hierarchy(tree_node_t *root, bool subdir, bool recursive)
 
 	n = n->next;
 	TEST_NOT_NULL(n);
-	TEST_STR_EQUAL(n->name, "istream");
+	TEST_STR_EQUAL(n->name, "iterator");
 	TEST_ASSERT(S_ISDIR(n->mode));
 	TEST_ASSERT(n->parent == parentdir);
 	TEST_NULL(n->data.children);
diff --git a/bin/tar2sqfs/src/process_tarball.c b/bin/tar2sqfs/src/process_tarball.c
index bcc60db..2399dfa 100644
--- a/bin/tar2sqfs/src/process_tarball.c
+++ b/bin/tar2sqfs/src/process_tarball.c
@@ -6,26 +6,25 @@
  */
 #include "tar2sqfs.h"
 
-static int write_file(istream_t *input_file, sqfs_writer_t *sqfs,
-		      const tar_header_decoded_t *hdr, tree_node_t *n)
+static int write_file(sqfs_writer_t *sqfs, dir_iterator_t *it,
+		      const dir_entry_t *ent, tree_node_t *n)
 {
 	int flags = 0, ret = 0;
 	ostream_t *out;
 	istream_t *in;
 
-	if (no_tail_pack && hdr->actual_size > cfg.block_size)
+	if (no_tail_pack && ent->size > cfg.block_size)
 		flags |= SQFS_BLK_DONT_FRAGMENT;
 
-	out = data_writer_ostream_create(hdr->name, sqfs->data,
+	out = data_writer_ostream_create(ent->name, sqfs->data,
 					 &(n->data.file.inode), flags);
-
 	if (out == NULL)
 		return -1;
 
-	in = tar_record_istream_create(input_file, hdr);
-	if (in == NULL) {
+	ret = it->open_file_ro(it, &in);
+	if (ret != 0) {
 		sqfs_drop(out);
-		return -1;
+		return ret;
 	}
 
 	do {
@@ -38,19 +37,25 @@ static int write_file(istream_t *input_file, sqfs_writer_t *sqfs,
 	return ret;
 }
 
-static int copy_xattr(sqfs_writer_t *sqfs, tree_node_t *node,
-		      const tar_header_decoded_t *hdr)
+static int copy_xattr(sqfs_writer_t *sqfs, const char *filename,
+		      tree_node_t *node, dir_iterator_t *it)
 {
-	dir_entry_xattr_t *xattr;
+	dir_entry_xattr_t *xattr, *list;
 	int ret;
 
-	ret = sqfs_xattr_writer_begin(sqfs->xwr, 0);
+	ret = it->read_xattr(it, &list);
 	if (ret) {
-		sqfs_perror(hdr->name, "beginning xattr block", ret);
+		sqfs_perror(filename, "reading xattrs", ret);
 		return -1;
 	}
 
-	for (xattr = hdr->xattr; xattr != NULL; xattr = xattr->next) {
+	ret = sqfs_xattr_writer_begin(sqfs->xwr, 0);
+	if (ret) {
+		sqfs_perror(filename, "beginning xattr block", ret);
+		goto fail;
+	}
+
+	for (xattr = list; xattr != NULL; xattr = xattr->next) {
 		if (sqfs_get_xattr_prefix_id(xattr->key) < 0) {
 			fprintf(stderr, "%s: squashfs does not "
 				"support xattr prefix of %s\n",
@@ -58,101 +63,98 @@ static int copy_xattr(sqfs_writer_t *sqfs, tree_node_t *node,
 				xattr->key);
 
 			if (dont_skip)
-				return -1;
+				goto fail;
 			continue;
 		}
 
 		ret = sqfs_xattr_writer_add(sqfs->xwr, xattr->key, xattr->value,
 					    xattr->value_len);
 		if (ret) {
-			sqfs_perror(hdr->name, "storing xattr key-value pair",
+			sqfs_perror(filename, "storing xattr key-value pair",
 				    ret);
-			return -1;
+			goto fail;
 		}
 	}
 
 	ret = sqfs_xattr_writer_end(sqfs->xwr, &node->xattr_idx);
 	if (ret) {
-		sqfs_perror(hdr->name, "completing xattr block", ret);
-		return -1;
+		sqfs_perror(filename, "completing xattr block", ret);
+		goto fail;
 	}
 
+	dir_entry_xattr_list_free(list);
 	return 0;
+fail:
+	dir_entry_xattr_list_free(list);
+	return -1;
 }
 
-static int create_node_and_repack_data(istream_t *input_file,
-				       sqfs_writer_t *sqfs,
-				       tar_header_decoded_t *hdr)
+static int create_node_and_repack_data(sqfs_writer_t *sqfs, dir_iterator_t *it,
+				       const dir_entry_t *ent, const char *link)
 {
 	tree_node_t *node;
 	struct stat sb;
 
-	if (hdr->is_hard_link) {
-		node = fstree_add_hard_link(&sqfs->fs, hdr->name,
-					    hdr->link_target);
+	if (ent->flags & DIR_ENTRY_FLAG_HARD_LINK) {
+		node = fstree_add_hard_link(&sqfs->fs, ent->name, link);
 		if (node == NULL)
 			goto fail_errno;
 
-		if (!cfg.quiet) {
-			printf("Hard link %s -> %s\n", hdr->name,
-			       hdr->link_target);
-		}
+		if (!cfg.quiet)
+			printf("Hard link %s -> %s\n", ent->name, link);
 		return 0;
 	}
 
-	if (!keep_time) {
-		hdr->mtime = sqfs->fs.defaults.mtime;
-	}
-
 	memset(&sb, 0, sizeof(sb));
-	sb.st_mode = hdr->mode;
-	sb.st_uid = hdr->uid;
-	sb.st_gid = hdr->gid;
-	sb.st_rdev = hdr->devno;
-	sb.st_size = hdr->actual_size;
-	sb.st_mtime = hdr->mtime;
-
-	node = fstree_add_generic(&sqfs->fs, hdr->name,
-				  &sb, hdr->link_target);
+	sb.st_mode = ent->mode;
+	sb.st_uid = ent->uid;
+	sb.st_gid = ent->gid;
+	sb.st_rdev = ent->rdev;
+	sb.st_mtime = keep_time ? ent->mtime : sqfs->fs.defaults.mtime;
+
+	node = fstree_add_generic(&sqfs->fs, ent->name, &sb, link);
 	if (node == NULL)
 		goto fail_errno;
 
 	if (!cfg.quiet)
-		printf("Packing %s\n", hdr->name);
+		printf("Packing %s\n", ent->name);
 
 	if (!cfg.no_xattr) {
-		if (copy_xattr(sqfs, node, hdr))
+		if (copy_xattr(sqfs, ent->name, node, it))
 			return -1;
 	}
 
-	if (S_ISREG(hdr->mode)) {
-		if (write_file(input_file, sqfs, hdr, node))
+	if (S_ISREG(ent->mode)) {
+		int ret = write_file(sqfs, it, ent, node);
+		if (ret != 0) {
+			sqfs_perror(ent->name, "packing data", ret);
 			return -1;
+		}
 	}
 
 	return 0;
 fail_errno:
-	perror(hdr->name);
+	perror(ent->name);
 	return -1;
 }
 
-static int set_root_attribs(sqfs_writer_t *sqfs,
-			    const tar_header_decoded_t *hdr)
+static int set_root_attribs(sqfs_writer_t *sqfs, dir_iterator_t *it,
+			    const dir_entry_t *ent)
 {
-	if (hdr->is_hard_link || !S_ISDIR(hdr->mode)) {
-		fprintf(stderr, "'%s' is not a directory!\n", hdr->name);
+	if ((ent->flags & DIR_ENTRY_FLAG_HARD_LINK) || !S_ISDIR(ent->mode)) {
+		fprintf(stderr, "'%s' is not a directory!\n", ent->name);
 		return -1;
 	}
 
-	sqfs->fs.root->uid = hdr->uid;
-	sqfs->fs.root->gid = hdr->gid;
-	sqfs->fs.root->mode = hdr->mode;
+	sqfs->fs.root->uid = ent->uid;
+	sqfs->fs.root->gid = ent->gid;
+	sqfs->fs.root->mode = ent->mode;
 
 	if (keep_time)
-		sqfs->fs.root->mod_time = hdr->mtime;
+		sqfs->fs.root->mod_time = ent->mtime;
 
 	if (!cfg.no_xattr) {
-		if (copy_xattr(sqfs, sqfs->fs.root, hdr))
+		if (copy_xattr(sqfs, "/", sqfs->fs.root, it))
 			return -1;
 	}
 
@@ -161,141 +163,94 @@ static int set_root_attribs(sqfs_writer_t *sqfs,
 
 int process_tarball(istream_t *input_file, sqfs_writer_t *sqfs)
 {
-	bool skip, is_root, is_prefixed;
-	tar_header_decoded_t hdr;
-	sqfs_u64 offset, count;
-	sparse_map_t *m;
-	size_t rootlen;
-	char *target;
-	int ret;
+	dir_iterator_t *it = tar_open_stream(input_file);
+	size_t rootlen = root_becomes == NULL ? 0 : strlen(root_becomes);
 
-	rootlen = root_becomes == NULL ? 0 : strlen(root_becomes);
+	if (it == NULL) {
+		fputs("Creating tar stream: out-of-memory\n", stderr);
+		return -1;
+	}
 
 	for (;;) {
-		ret = read_header(input_file, &hdr);
+		bool skip = false, is_root = false, is_prefixed = true;
+		dir_entry_t *ent = NULL;
+		char *link = NULL;
+		int ret;
+
+		ret = it->next(it, &ent);
 		if (ret > 0)
 			break;
 		if (ret < 0)
-			return -1;
+			goto fail;
 
-		if (hdr.mtime < 0)
-			hdr.mtime = 0;
+		if (ent->mtime < 0)
+			ent->mtime = 0;
 
-		if ((sqfs_u64)hdr.mtime > 0x0FFFFFFFFUL)
-			hdr.mtime = 0x0FFFFFFFFUL;
+		if ((sqfs_u64)ent->mtime > 0x0FFFFFFFFUL)
+			ent->mtime = 0x0FFFFFFFFUL;
 
-		skip = false;
-		is_root = false;
-		is_prefixed = true;
+		if (S_ISLNK(ent->mode)) {
+			ret = it->read_link(it, &link);
+			if (ret != 0) {
+				sqfs_perror(ent->name, "read link", ret);
+				free(ent);
+				goto fail;
+			}
+		}
 
-		if (hdr.name == NULL || canonicalize_name(hdr.name) != 0) {
-			fprintf(stderr, "skipping '%s' (invalid name)\n",
-				hdr.name);
-			skip = true;
-		} else if (root_becomes != NULL) {
-			if (strncmp(hdr.name, root_becomes, rootlen) == 0) {
-				if (hdr.name[rootlen] == '\0') {
+		if (root_becomes != NULL) {
+			if (strncmp(ent->name, root_becomes, rootlen) == 0) {
+				if (ent->name[rootlen] == '\0') {
 					is_root = true;
-				} else if (hdr.name[rootlen] != '/') {
+				} else if (ent->name[rootlen] != '/') {
 					is_prefixed = false;
 				}
 			} else {
 				is_prefixed = false;
 			}
 
-			if (is_prefixed && !is_root) {
-				memmove(hdr.name, hdr.name + rootlen + 1,
-					strlen(hdr.name + rootlen + 1) + 1);
+			if (!is_prefixed) {
+				free(ent);
+				free(link);
+				continue;
 			}
 
-			if (is_prefixed && hdr.name[0] == '\0') {
-				fputs("skipping entry with empty name\n",
-				      stderr);
-				skip = true;
+			if (!is_root) {
+				memmove(ent->name, ent->name + rootlen + 1,
+					strlen(ent->name + rootlen + 1) + 1);
 			}
 
-			if (hdr.link_target != NULL &&
-			    (hdr.is_hard_link || !no_symlink_retarget)) {
-				target = strdup(hdr.link_target);
-				if (target == NULL) {
-					fprintf(stderr, "packing '%s': %s\n",
-						hdr.name, strerror(errno));
-					goto fail;
+			if (link != NULL &&
+			    ((ent->flags & DIR_ENTRY_FLAG_HARD_LINK) ||
+			     !no_symlink_retarget)) {
+				if (canonicalize_name(link) == 0 &&
+				    !strncmp(link, root_becomes, rootlen) &&
+				    link[rootlen] == '/') {
+					memmove(link, link + rootlen,
+						strlen(link + rootlen) + 1);
 				}
-
-				if (canonicalize_name(target) == 0 &&
-				    !strncmp(target, root_becomes, rootlen) &&
-				    target[rootlen] == '/') {
-					memmove(hdr.link_target,
-						target + rootlen,
-						strlen(target + rootlen) + 1);
-				}
-
-				free(target);
 			}
-		} else if (hdr.name[0] == '\0') {
+		} else if (ent->name[0] == '\0') {
 			is_root = true;
 		}
 
-		if (!is_prefixed) {
-			if (skip_entry(input_file, hdr.record_size))
-				goto fail;
-			clear_header(&hdr);
-			continue;
-		}
-
 		if (is_root) {
-			if (set_root_attribs(sqfs, &hdr))
-				goto fail;
-			clear_header(&hdr);
-			continue;
-		}
-
-		if (!skip && hdr.unknown_record) {
-			fprintf(stderr, "%s: unknown entry type\n", hdr.name);
-			skip = true;
+			ret = set_root_attribs(sqfs, it, ent);
+		} else if (skip) {
+			ret = dont_skip ? -1 : 0;
+		} else {
+			ret = create_node_and_repack_data(sqfs, it, ent, link);
 		}
 
-		if (!skip && hdr.sparse != NULL) {
-			offset = hdr.sparse->offset;
-			count = 0;
-
-			for (m = hdr.sparse; m != NULL; m = m->next) {
-				if (m->offset < offset) {
-					skip = true;
-					break;
-				}
-				offset = m->offset + m->count;
-				count += m->count;
-			}
-
-			if (count != hdr.record_size)
-				skip = true;
-
-			if (skip) {
-				fprintf(stderr, "%s: broken sparse "
-					"file layout\n", hdr.name);
-			}
-		}
-
-		if (skip) {
-			if (dont_skip)
-				goto fail;
-			if (skip_entry(input_file, hdr.record_size))
-				goto fail;
-
-			clear_header(&hdr);
-			continue;
-		}
-
-		if (create_node_and_repack_data(input_file, sqfs, &hdr))
+		free(ent);
+		free(link);
+		if (ret)
 			goto fail;
-
-		clear_header(&hdr);
 	}
 
+	sqfs_drop(it);
 	return 0;
 fail:
-	clear_header(&hdr);
+	sqfs_drop(it);
 	return -1;
 }
diff --git a/include/io/dir_iterator.h b/include/io/dir_iterator.h
index 377f07e..20fcdff 100644
--- a/include/io/dir_iterator.h
+++ b/include/io/dir_iterator.h
@@ -24,6 +24,11 @@ typedef enum {
  */
 typedef struct {
 	/**
+	 * @brief Total size of file entries
+	 */
+	sqfs_u64 size;
+
+	/**
 	 * @brief Unix time stamp when the entry was last modified.
 	 *
 	 * If necessary, the OS native time stamp is converted to Unix time.
@@ -49,14 +54,14 @@ typedef struct {
 	 *
 	 * On Windows and other non-Unix OSes, this always reports user 0.
 	 */
-	sqfs_u32 uid;
+	sqfs_u64 uid;
 
 	/**
 	 * @brief ID of the group that owns the entry.
 	 *
 	 * On Windows and other non-Unix OSes, this always reports group 0.
 	 */
-	sqfs_u32 gid;
+	sqfs_u64 gid;
 
 	/**
 	 * @brief Unix style permissions and entry type.
diff --git a/include/io/xattr.h b/include/io/xattr.h
index cf35bca..d9f3e65 100644
--- a/include/io/xattr.h
+++ b/include/io/xattr.h
@@ -24,6 +24,8 @@ extern "C" {
 dir_entry_xattr_t *dir_entry_xattr_create(const char *key, const sqfs_u8 *value,
 					  size_t value_len);
 
+dir_entry_xattr_t *dir_entry_xattr_list_copy(const dir_entry_xattr_t *list);
+
 void dir_entry_xattr_list_free(dir_entry_xattr_t *list);
 
 #ifdef __cplusplus
diff --git a/include/tar/tar.h b/include/tar/tar.h
index 21ded7d..f77b27c 100644
--- a/include/tar/tar.h
+++ b/include/tar/tar.h
@@ -59,14 +59,11 @@ int write_hard_link(ostream_t *fp, const struct stat *sb, const char *name,
 		    const char *target, unsigned int counter);
 
 /* round up to block size and skip the entire entry */
-int skip_entry(istream_t *fp, sqfs_u64 size);
-
 int read_header(istream_t *fp, tar_header_decoded_t *out);
 
 void clear_header(tar_header_decoded_t *hdr);
 
-istream_t *tar_record_istream_create(istream_t *parent,
-				     const tar_header_decoded_t *hdr);
+dir_iterator_t *tar_open_stream(istream_t *stream);
 
 /*
   Write zero bytes to an output file to padd it to the tar record size.
diff --git a/lib/io/src/unix/dir_iterator.c b/lib/io/src/unix/dir_iterator.c
index fb7edae..7154ec9 100644
--- a/lib/io/src/unix/dir_iterator.c
+++ b/lib/io/src/unix/dir_iterator.c
@@ -118,6 +118,9 @@ static int dir_next(dir_iterator_t *base, dir_entry_t **out)
 	decoded->gid = it->sb.st_gid;
 	decoded->mode = it->sb.st_mode;
 
+	if (S_ISREG(it->sb.st_mode))
+		decoded->size = it->sb.st_size;
+
 	if (decoded->dev != it->device)
 		decoded->flags |= DIR_ENTRY_FLAG_MOUNT_POINT;
 
diff --git a/lib/io/src/win32/dir_iterator.c b/lib/io/src/win32/dir_iterator.c
index c9defc7..f504b65 100644
--- a/lib/io/src/win32/dir_iterator.c
+++ b/lib/io/src/win32/dir_iterator.c
@@ -91,6 +91,9 @@ static int dir_iterator_next(dir_iterator_t *it, dir_entry_t **out)
 		ent->mode = S_IFDIR | 0755;
 	} else {
 		ent->mode = S_IFREG | 0644;
+		ent->size = w32->ent.nFileSizeHigh;
+		ent->size <<= 32UL;
+		ent->size |= w32->ent.nFileSizeLow;
 	}
 
 	ent->mtime = w32time_to_unix(&(w32->ent.ftLastWriteTime));
diff --git a/lib/io/src/xattr.c b/lib/io/src/xattr.c
index dd9a338..85b7e53 100644
--- a/lib/io/src/xattr.c
+++ b/lib/io/src/xattr.c
@@ -28,6 +28,31 @@ dir_entry_xattr_t *dir_entry_xattr_create(const char *key, const sqfs_u8 *value,
 	return out;
 }
 
+dir_entry_xattr_t *dir_entry_xattr_list_copy(const dir_entry_xattr_t *list)
+{
+	dir_entry_xattr_t *new, *copy = NULL, *copy_last = NULL;
+	const dir_entry_xattr_t *it;
+
+	for (it = list; it != NULL; it = it->next) {
+		new = dir_entry_xattr_create(it->key, it->value,
+					     it->value_len);
+		if (new == NULL) {
+			dir_entry_xattr_list_free(copy);
+			return NULL;
+		}
+
+		if (copy_last == NULL) {
+			copy = new;
+			copy_last = new;
+		} else {
+			copy_last->next = new;
+			copy_last = new;
+		}
+	}
+
+	return copy;
+}
+
 void dir_entry_xattr_list_free(dir_entry_xattr_t *list)
 {
 	dir_entry_xattr_t *old;
diff --git a/lib/tar/Makemodule.am b/lib/tar/Makemodule.am
index 07eb497..7e58cc6 100644
--- a/lib/tar/Makemodule.am
+++ b/lib/tar/Makemodule.am
@@ -3,7 +3,8 @@ libtar_a_SOURCES = lib/tar/src/read_header.c lib/tar/src/write_header.c \
 	lib/tar/src/read_sparse_map_old.c lib/tar/src/internal.h \
 	lib/tar/src/padd_file.c lib/tar/src/record_to_memory.c \
 	lib/tar/src/pax_header.c lib/tar/src/read_sparse_map_new.c \
-	lib/tar/src/istream.c include/tar/tar.h include/tar/format.h
+	lib/tar/src/iterator.c \
+	include/tar/tar.h include/tar/format.h
 
 noinst_LIBRARIES += libtar.a
 
@@ -162,19 +163,19 @@ test_tar_xattr_schily_bin_LDADD = libtar.a libio.a libutil.a libcompat.a
 test_tar_xattr_schily_bin_CPPFLAGS = $(AM_CPPFLAGS) -DTESTPATH=$(TARDATADIR)
 test_tar_xattr_schily_bin_CPPFLAGS += -DTESTFILE=xattr/xattr-schily-binary.tar
 
-test_tar_istream_SOURCES = lib/tar/test/tar_istream.c
-test_tar_istream_LDADD = libtar.a libio.a libutil.a libcompat.a
-test_tar_istream_CPPFLAGS = $(AM_CPPFLAGS) -DTESTPATH=$(TARDATADIR)
-test_tar_istream_CPPFLAGS += -DTESTFILE=format-acceptance/gnu.tar
+test_tar_iterator_SOURCES = lib/tar/test/tar_iterator.c
+test_tar_iterator_LDADD = libtar.a libio.a libutil.a libcompat.a
+test_tar_iterator_CPPFLAGS = $(AM_CPPFLAGS) -DTESTPATH=$(TARDATADIR)
+test_tar_iterator_CPPFLAGS += -DTESTFILE=format-acceptance/gnu.tar
 
-test_tar_istream2_SOURCES = lib/tar/test/tar_istream2.c
-test_tar_istream2_LDADD = libtar.a libio.a libutil.a libcompat.a
-test_tar_istream2_CPPFLAGS = $(AM_CPPFLAGS) -DTESTPATH=$(TARDATADIR)
+test_tar_iterator2_SOURCES = lib/tar/test/tar_iterator2.c
+test_tar_iterator2_LDADD = libtar.a libio.a libutil.a libcompat.a
+test_tar_iterator2_CPPFLAGS = $(AM_CPPFLAGS) -DTESTPATH=$(TARDATADIR)
+test_tar_iterator2_CPPFLAGS += -DTESTFILE=iterator/sparse.tar
 
-test_tar_istream3_SOURCES = lib/tar/test/tar_istream3.c
-test_tar_istream3_LDADD = libtar.a libio.a libutil.a libcompat.a
-test_tar_istream3_CPPFLAGS = $(AM_CPPFLAGS) -DTESTPATH=$(TARDATADIR)
-test_tar_istream3_CPPFLAGS += -DTESTFILE=istream/sparse.tar
+test_tar_iterator3_SOURCES = lib/tar/test/tar_iterator3.c
+test_tar_iterator3_LDADD = libtar.a libio.a libutil.a libcompat.a
+test_tar_iterator3_CPPFLAGS = $(AM_CPPFLAGS) -DTESTPATH=$(TARDATADIR)
 
 tar_fuzz_SOURCES = lib/tar/test/tar_fuzz.c
 tar_fuzz_LDADD = libtar.a libio.a libutil.a libcompat.a
@@ -195,7 +196,7 @@ LIBTAR_TESTS = \
 	test_tar_sparse_gnu2 test_tar_sparse_gnu3 \
 	test_tar_xattr_bsd test_tar_xattr_schily test_tar_xattr_schily_bin \
 	test_tar_target_filled \
-	test_tar_istream test_tar_istream2 test_tar_istream3 \
+	test_tar_iterator test_tar_iterator2 test_tar_iterator3 \
 	test_tar_write_simple
 
 check_PROGRAMS += $(LIBTAR_TESTS)
diff --git a/lib/tar/src/istream.c b/lib/tar/src/istream.c
deleted file mode 100644
index 80519b1..0000000
--- a/lib/tar/src/istream.c
+++ /dev/null
@@ -1,240 +0,0 @@
-/* SPDX-License-Identifier: GPL-3.0-or-later */
-/*
- * istream.c
- *
- * Copyright (C) 2019 David Oberhollenzer <goliath@infraroot.at>
- */
-#include "internal.h"
-
-#include <string.h>
-#include <stdlib.h>
-
-typedef struct {
-	sqfs_u64 offset;
-	sqfs_u64 count;
-} sparse_ent_t;
-
-typedef struct {
-	istream_t base;
-
-	istream_t *parent;
-
-	char *filename;
-
-	sparse_ent_t *sparse;
-	size_t num_sparse;
-
-	sqfs_u64 record_size;
-	sqfs_u64 file_size;
-	sqfs_u64 offset;
-
-	size_t padding;
-	size_t last_chunk;
-	bool last_sparse;
-
-	sqfs_u8 buffer[4096];
-} tar_istream_t;
-
-static bool is_sparse_region(tar_istream_t *tar, sqfs_u64 *count)
-{
-	size_t i;
-
-	*count = tar->file_size - tar->offset;
-	if (tar->num_sparse == 0)
-		return false;
-
-	for (i = 0; i < tar->num_sparse; ++i) {
-		if (tar->offset >= tar->sparse[i].offset) {
-			sqfs_u64 diff = tar->offset - tar->sparse[i].offset;
-
-			if (diff < tar->sparse[i].count) {
-				*count = tar->sparse[i].count - diff;
-				return false;
-			}
-		}
-	}
-
-	for (i = 0; i < tar->num_sparse; ++i) {
-		if (tar->offset < tar->sparse[i].offset) {
-			sqfs_u64 diff = tar->sparse[i].offset - tar->offset;
-
-			if (diff < *count)
-				*count = diff;
-		}
-	}
-
-	return true;
-}
-
-static int precache(istream_t *strm)
-{
-	tar_istream_t *tar = (tar_istream_t *)strm;
-	sqfs_u64 diff, avail;
-
-	tar->offset += tar->last_chunk;
-
-	if (!tar->last_sparse) {
-		tar->parent->buffer_offset += tar->last_chunk;
-		tar->record_size -= tar->last_chunk;
-	}
-
-	if (tar->offset >= tar->file_size) {
-		strm->eof = true;
-		strm->buffer_used = 0;
-		strm->buffer = tar->buffer;
-		if (tar->record_size > 0)
-			goto fail_rec_sz;
-		if (istream_skip(tar->parent, tar->padding))
-			goto fail;
-		tar->padding = 0;
-		return 0;
-	}
-
-	if (is_sparse_region(tar, &diff)) {
-		if (diff > sizeof(tar->buffer))
-			diff = sizeof(tar->buffer);
-
-		strm->buffer = tar->buffer;
-		strm->buffer_used = diff;
-		tar->last_chunk = diff;
-		tar->last_sparse = true;
-
-		memset(tar->buffer, 0, diff);
-	} else {
-		if (diff > tar->record_size)
-			goto fail_rec_sz;
-
-		avail = tar->parent->buffer_used - tar->parent->buffer_offset;
-
-		if ((diff > avail) &&
-		    ((tar->parent->buffer_offset > 0) || avail == 0)) {
-			if (istream_precache(tar->parent))
-				goto fail;
-
-			if (tar->parent->buffer_used == 0 && tar->parent->eof)
-				goto fail_eof;
-
-			avail = tar->parent->buffer_used;
-		}
-
-		if (diff > avail)
-			diff = avail;
-
-		strm->buffer = tar->parent->buffer + tar->parent->buffer_offset;
-		strm->buffer_used = diff;
-		tar->last_chunk = diff;
-		tar->last_sparse = false;
-	}
-
-	return 0;
-fail_rec_sz:
-	fprintf(stderr,
-		"%s: missmatch in tar record size vs file size for `%s`.\n",
-		istream_get_filename(tar->parent), istream_get_filename(strm));
-	goto fail;
-fail_eof:
-	fprintf(stderr, "%s: unexpected end-of-file while reading `%s`\n",
-		istream_get_filename(tar->parent), istream_get_filename(strm));
-	goto fail;
-fail:
-	tar->record_size = 0;
-	tar->padding = 0;
-	return -1;
-}
-
-static const char *get_filename(istream_t *strm)
-{
-	return ((tar_istream_t *)strm)->filename;
-}
-
-static void tar_istream_destroy(sqfs_object_t *obj)
-{
-	tar_istream_t *strm = (tar_istream_t *)obj;
-
-	if (strm->record_size > 0)
-		istream_skip(strm->parent, strm->record_size);
-
-	if (strm->padding > 0)
-		istream_skip(strm->parent, strm->padding);
-
-	sqfs_drop(strm->parent);
-	free(strm->sparse);
-	free(strm->filename);
-	free(strm);
-}
-
-istream_t *tar_record_istream_create(istream_t *parent,
-				     const tar_header_decoded_t *hdr)
-{
-	tar_istream_t *strm;
-	sparse_map_t *it;
-	sqfs_u64 diff;
-	size_t idx;
-
-	strm = calloc(1, sizeof(*strm));
-	if (strm == NULL)
-		goto fail_oom;
-
-	sqfs_object_init(strm, tar_istream_destroy, NULL);
-
-	strm->filename = strdup(hdr->name);
-	if (strm->filename == NULL)
-		goto fail_oom;
-
-	strm->num_sparse = 0;
-	for (it = hdr->sparse; it != NULL; it = it->next)
-		strm->num_sparse += 1;
-
-	if (strm->num_sparse > 0) {
-		strm->sparse = alloc_array(sizeof(strm->sparse[0]),
-					   strm->num_sparse);
-		if (strm->sparse == NULL)
-			goto fail_oom;
-
-		idx = 0;
-		it = hdr->sparse;
-		while (it != NULL && idx < strm->num_sparse) {
-			strm->sparse[idx].offset = it->offset;
-			strm->sparse[idx].count = it->count;
-			++idx;
-			it = it->next;
-		}
-	}
-
-	for (idx = 1; idx < strm->num_sparse; ++idx) {
-		if (strm->sparse[idx].offset <= strm->sparse[idx - 1].offset)
-			goto fail_sparse;
-
-		diff = strm->sparse[idx].offset - strm->sparse[idx - 1].offset;
-
-		if (diff < strm->sparse[idx - 1].count)
-			goto fail_sparse;
-	}
-
-	strm->padding = hdr->record_size % 512;
-	if (strm->padding > 0)
-		strm->padding = 512 - strm->padding;
-
-	strm->record_size = hdr->record_size;
-	strm->file_size = hdr->actual_size;
-	strm->parent = sqfs_grab(parent);
-
-	((istream_t *)strm)->precache = precache;
-	((istream_t *)strm)->get_filename = get_filename;
-	((istream_t *)strm)->buffer = strm->buffer;
-	((istream_t *)strm)->eof = false;
-	return (istream_t *)strm;
-fail_sparse:
-	fprintf(stderr, "%s: sparse map is not ordered or overlapping!\n",
-		hdr->name);
-	goto fail;
-fail_oom:
-	fputs("tar istream create: out-of-memory\n", stderr);
-	goto fail;
-fail:
-	if (strm != NULL) {
-		free(strm->filename);
-		free(strm);
-	}
-	return NULL;
-}
diff --git a/lib/tar/src/iterator.c b/lib/tar/src/iterator.c
new file mode 100644
index 0000000..9a6b9f7
--- /dev/null
+++ b/lib/tar/src/iterator.c
@@ -0,0 +1,339 @@
+/* SPDX-License-Identifier: GPL-3.0-or-later */
+/*
+ * iterator.c
+ *
+ * Copyright (C) 2023 David Oberhollenzer <goliath@infraroot.at>
+ */
+#include "tar/tar.h"
+#include "sqfs/error.h"
+#include "util/util.h"
+
+#include <stdlib.h>
+#include <string.h>
+#include <stdbool.h>
+
+typedef struct {
+	dir_iterator_t base;
+	tar_header_decoded_t current;
+	istream_t *stream;
+	int state;
+
+	/* File I/O wrapper related */
+	bool locked;
+
+	sqfs_u64 record_size;
+	sqfs_u64 file_size;
+	sqfs_u64 offset;
+
+	size_t padding;
+	size_t last_chunk;
+	bool last_sparse;
+} tar_iterator_t;
+
+typedef struct {
+	istream_t base;
+
+	tar_iterator_t *parent;
+
+	sqfs_u8 buffer[4096];
+} tar_istream_t;
+
+static bool is_sparse_region(const tar_iterator_t *tar, sqfs_u64 *count)
+{
+	const sparse_map_t *it;
+
+	*count = tar->file_size - tar->offset;
+	if (tar->current.sparse == NULL)
+		return false;
+
+	for (it = tar->current.sparse; it != NULL; it = it->next) {
+		if (tar->offset >= it->offset) {
+			sqfs_u64 diff = tar->offset - it->offset;
+
+			if (diff < it->count) {
+				*count = it->count - diff;
+				return false;
+			}
+		}
+	}
+
+	for (it = tar->current.sparse; it != NULL; it = it->next) {
+		if (tar->offset < it->offset) {
+			sqfs_u64 diff = it->offset - tar->offset;
+
+			if (diff < *count)
+				*count = diff;
+		}
+	}
+
+	return true;
+}
+
+static int data_available(tar_iterator_t *tar, sqfs_u64 want, sqfs_u64 *out)
+{
+	sqfs_u64 avail = tar->stream->buffer_used - tar->stream->buffer_offset;
+
+	if ((want > avail) &&
+	    ((tar->stream->buffer_offset > 0) || avail == 0)) {
+		if (istream_precache(tar->stream)) {
+			tar->state = SQFS_ERROR_IO;
+			return -1;
+		}
+
+		if (tar->stream->buffer_used == 0 && tar->stream->eof) {
+			tar->state = SQFS_ERROR_CORRUPTED;
+			return -1;
+		}
+
+		avail = tar->stream->buffer_used;
+	}
+
+	*out = avail <= want ? avail : want;
+	return 0;
+}
+
+/*****************************************************************************/
+
+static const char *strm_get_filename(istream_t *strm)
+{
+	return ((tar_istream_t *)strm)->parent->current.name;
+}
+
+static int strm_precache(istream_t *strm)
+{
+	tar_istream_t *tar = (tar_istream_t *)strm;
+	sqfs_u64 diff;
+
+	tar->parent->offset += tar->parent->last_chunk;
+
+	if (!tar->parent->last_sparse) {
+		tar->parent->stream->buffer_offset += tar->parent->last_chunk;
+		tar->parent->record_size -= tar->parent->last_chunk;
+	}
+
+	if (tar->parent->offset >= tar->parent->file_size)
+		goto out_eof;
+
+	if (is_sparse_region(tar->parent, &diff)) {
+		if (diff > sizeof(tar->buffer))
+			diff = sizeof(tar->buffer);
+
+		strm->buffer = tar->buffer;
+		strm->buffer_used = diff;
+		tar->parent->last_chunk = diff;
+		tar->parent->last_sparse = true;
+
+		memset(tar->buffer, 0, diff);
+	} else {
+		if (data_available(tar->parent, diff, &diff))
+			goto out_eof;
+
+		strm->buffer = tar->parent->stream->buffer +
+			tar->parent->stream->buffer_offset;
+		strm->buffer_used = diff;
+		tar->parent->last_chunk = diff;
+		tar->parent->last_sparse = false;
+	}
+
+	return 0;
+out_eof:
+	strm->eof = true;
+	strm->buffer_used = 0;
+	strm->buffer = tar->buffer;
+	tar->parent->locked = false;
+	return tar->parent->state < 0 ? -1 : 0;
+}
+
+static void strm_destroy(sqfs_object_t *obj)
+{
+	tar_istream_t *tar = (tar_istream_t *)obj;
+
+	tar->parent->locked = false;
+	sqfs_drop(tar->parent);
+	free(tar);
+}
+
+/*****************************************************************************/
+
+static int it_next(dir_iterator_t *it, dir_entry_t **out)
+{
+	tar_iterator_t *tar = (tar_iterator_t *)it;
+	dir_entry_t *ent;
+	int ret;
+
+	*out = NULL;
+	if (tar->locked)
+		return SQFS_ERROR_SEQUENCE;
+
+	if (tar->state != 0)
+		return tar->state;
+retry:
+	if (tar->record_size > 0) {
+		ret = istream_skip(tar->stream, tar->record_size);
+		if (ret)
+			goto fail;
+	}
+
+	if (tar->padding > 0) {
+		ret = istream_skip(tar->stream, tar->padding);
+		if (ret)
+			goto fail;
+	}
+
+	clear_header(&(tar->current));
+	ret = read_header(tar->stream, &(tar->current));
+	if (ret != 0)
+		goto fail;
+
+	tar->offset = 0;
+	tar->last_chunk = 0;
+	tar->last_sparse = false;
+	tar->record_size = tar->current.record_size;
+	tar->file_size = tar->current.actual_size;
+	tar->padding = tar->current.record_size % 512;
+	if (tar->padding > 0)
+		tar->padding = 512 - tar->padding;
+
+	if (tar->current.unknown_record)
+		goto retry;
+
+	if (canonicalize_name(tar->current.name) != 0) {
+		tar->state = SQFS_ERROR_CORRUPTED;
+		return tar->state;
+	}
+
+	ent = calloc(1, sizeof(*ent) + strlen(tar->current.name) + 1);
+	if (ent == NULL) {
+		tar->state = SQFS_ERROR_ALLOC;
+		return tar->state;
+	}
+
+	ent->mtime = tar->current.mtime;
+	ent->rdev = tar->current.devno;
+	ent->uid = tar->current.uid;
+	ent->gid = tar->current.gid;
+	ent->mode = tar->current.mode;
+	strcpy(ent->name, tar->current.name);
+
+	if (tar->current.is_hard_link) {
+		ent->mode = (S_IFLNK | 0777);
+		ent->flags |= DIR_ENTRY_FLAG_HARD_LINK;
+	}
+
+	if (S_ISREG(ent->mode))
+		ent->size = tar->current.actual_size;
+
+	*out = ent;
+	return 0;
+fail:
+	tar->state = ret < 0 ? SQFS_ERROR_IO : 1;
+	return tar->state;
+}
+
+static int it_read_link(dir_iterator_t *it, char **out)
+{
+	tar_iterator_t *tar = (tar_iterator_t *)it;
+
+	*out = NULL;
+	if (tar->locked)
+		return SQFS_ERROR_SEQUENCE;
+
+	if (tar->state != 0 || tar->current.link_target == NULL)
+		return tar->state < 0 ? tar->state : SQFS_ERROR_NO_ENTRY;
+
+	*out = strdup(tar->current.link_target);
+	return (*out == NULL) ? SQFS_ERROR_ALLOC : 0;
+}
+
+static int it_open_subdir(dir_iterator_t *it, dir_iterator_t **out)
+{
+	(void)it;
+	*out = NULL;
+	return SQFS_ERROR_UNSUPPORTED;
+}
+
+static void it_ignore_subdir(dir_iterator_t *it)
+{
+	(void)it;
+	/* TODO: skip list */
+}
+
+static int it_open_file_ro(dir_iterator_t *it, istream_t **out)
+{
+	tar_iterator_t *tar = (tar_iterator_t *)it;
+	tar_istream_t *strm;
+
+	*out = NULL;
+	if (tar->locked)
+		return SQFS_ERROR_SEQUENCE;
+
+	if (tar->state != 0)
+		return tar->state < 0 ? tar->state : SQFS_ERROR_NO_ENTRY;
+
+	if (!S_ISREG(tar->current.mode))
+		return SQFS_ERROR_NOT_FILE;
+
+	strm = calloc(1, sizeof(*strm));
+	if (strm == NULL)
+		return SQFS_ERROR_ALLOC;
+
+	sqfs_object_init(strm, strm_destroy, NULL);
+	strm->parent = sqfs_grab(tar);
+
+	((istream_t *)strm)->precache = strm_precache;
+	((istream_t *)strm)->get_filename = strm_get_filename;
+	((istream_t *)strm)->buffer = strm->buffer;
+
+	tar->locked = true;
+	*out = (istream_t *)strm;
+	return 0;
+}
+
+static int it_read_xattr(dir_iterator_t *it, dir_entry_xattr_t **out)
+{
+	tar_iterator_t *tar = (tar_iterator_t *)it;
+
+	*out = NULL;
+	if (tar->locked)
+		return SQFS_ERROR_SEQUENCE;
+
+	if (tar->state != 0)
+		return tar->state < 0 ? tar->state : SQFS_ERROR_NO_ENTRY;
+
+	if (tar->current.xattr != NULL) {
+		*out = dir_entry_xattr_list_copy(tar->current.xattr);
+		if (*out == NULL)
+			return SQFS_ERROR_ALLOC;
+	}
+
+	return 0;
+}
+
+static void it_destroy(sqfs_object_t *obj)
+{
+	tar_iterator_t *tar = (tar_iterator_t *)obj;
+
+	clear_header(&(tar->current));
+	sqfs_drop(tar->stream);
+	free(tar);
+}
+
+dir_iterator_t *tar_open_stream(istream_t *stream)
+{
+	tar_iterator_t *tar = calloc(1, sizeof(*tar));
+	dir_iterator_t *it = (dir_iterator_t *)tar;
+
+	if (tar == NULL)
+		return NULL;
+
+	sqfs_object_init(it, it_destroy, NULL);
+	tar->stream = sqfs_grab(stream);
+	it->next = it_next;
+	it->read_link = it_read_link;
+	it->open_subdir = it_open_subdir;
+	it->ignore_subdir = it_ignore_subdir;
+	it->open_file_ro = it_open_file_ro;
+	it->read_xattr = it_read_xattr;
+
+	return it;
+}
diff --git a/lib/tar/src/read_header.c b/lib/tar/src/read_header.c
index 8d3145b..751c5dc 100644
--- a/lib/tar/src/read_header.c
+++ b/lib/tar/src/read_header.c
@@ -224,7 +224,9 @@ int read_header(istream_t *fp, tar_header_decoded_t *out)
 		case TAR_TYPE_PAX_GLOBAL:
 			if (read_number(hdr.size, sizeof(hdr.size), &pax_size))
 				goto fail;
-			skip_entry(fp, pax_size);
+			if (pax_size % 512)
+				pax_size += 512 - (pax_size % 512);
+			istream_skip(fp, pax_size);
 			continue;
 		case TAR_TYPE_PAX:
 			clear_header(out);
@@ -291,10 +293,3 @@ fail:
 	clear_header(out);
 	return -1;
 }
-
-int skip_entry(istream_t *fp, sqfs_u64 size)
-{
-	size_t tail = size % 512;
-
-	return istream_skip(fp, tail ? (size + 512 - tail) : size);
-}
diff --git a/lib/tar/test/data/CREDITS b/lib/tar/test/data/CREDITS
index 1e7fb32..4c20256 100644
--- a/lib/tar/test/data/CREDITS
+++ b/lib/tar/test/data/CREDITS
@@ -33,9 +33,9 @@ The following addtional files have been added:
      Contributed in GitHub issue #64. A tar ball that contains a hard link
      where the 100 byte target field is completely filled without containing
      a null-terminator.
- - istream/sparse.tar
+ - iterator/sparse.tar
      Derived from sparse/gnu.tar and contains some test data for testing the
-     tar istream implementation.
+     tar iterator implementation.
  - write/simple.tar
      Created using the tar writer to verify that it works as intended. Used
      for testing the tar writer to check if it still produces identical
diff --git a/lib/tar/test/data/istream/sparse.tar b/lib/tar/test/data/iterator/sparse.tar
index 7f4700d..7f4700d 100644
--- a/lib/tar/test/data/istream/sparse.tar
+++ b/lib/tar/test/data/iterator/sparse.tar
diff --git a/lib/tar/test/tar_istream.c b/lib/tar/test/tar_istream.c
deleted file mode 100644
index cc31282..0000000
--- a/lib/tar/test/tar_istream.c
+++ /dev/null
@@ -1,73 +0,0 @@
-/* SPDX-License-Identifier: GPL-3.0-or-later */
-/*
- * tar_istream.c
- *
- * Copyright (C) 2019 David Oberhollenzer <goliath@infraroot.at>
- */
-#include "config.h"
-#include "io/file.h"
-#include "tar/tar.h"
-#include "util/test.h"
-
-#ifndef TESTUID
-#define TESTUID 1000
-#endif
-
-#ifndef TESTGID
-#define TESTGID TESTUID
-#endif
-
-#ifndef TESTFNAME
-#define TESTFNAME input.txt
-#endif
-
-#ifndef TESTTS
-#define TESTTS 1542905892
-#endif
-
-static const char *fname = STRVALUE(TESTFNAME);
-
-int main(int argc, char **argv)
-{
-	tar_header_decoded_t hdr;
-	char buffer[100];
-	sqfs_s64 ts;
-	istream_t *fp;
-	istream_t *ti;
-	sqfs_s32 ret;
-	(void)argc; (void)argv;
-
-	fp = istream_open_file(STRVALUE(TESTPATH) "/" STRVALUE(TESTFILE));
-	TEST_NOT_NULL(fp);
-	TEST_EQUAL_UI(((sqfs_object_t *)fp)->refcount, 1);
-	TEST_ASSERT(read_header(fp, &hdr) == 0);
-	TEST_EQUAL_UI(((sqfs_object_t *)fp)->refcount, 1);
-	TEST_EQUAL_UI(hdr.mode, S_IFREG | 0644);
-	TEST_EQUAL_UI(hdr.uid, TESTUID);
-	TEST_EQUAL_UI(hdr.gid, TESTGID);
-	TEST_EQUAL_UI(hdr.actual_size, 5);
-
-	ts = TESTTS;
-	TEST_EQUAL_UI(hdr.mtime, ts);
-	TEST_STR_EQUAL(hdr.name, fname);
-	TEST_ASSERT(!hdr.unknown_record);
-
-	ti = tar_record_istream_create(fp, &hdr);
-	TEST_NOT_NULL(ti);
-	TEST_EQUAL_UI(((sqfs_object_t *)fp)->refcount, 2);
-	TEST_EQUAL_UI(((sqfs_object_t *)ti)->refcount, 1);
-
-	ret = istream_read(ti, buffer, sizeof(buffer));
-	TEST_EQUAL_I(ret, 5);
-	buffer[5] = '\0';
-	TEST_STR_EQUAL(buffer, "test\n");
-
-	ret = istream_read(ti, buffer, sizeof(buffer));
-	TEST_EQUAL_I(ret, 0);
-
-	clear_header(&hdr);
-	sqfs_drop(ti);
-	TEST_EQUAL_UI(((sqfs_object_t *)fp)->refcount, 1);
-	sqfs_drop(fp);
-	return EXIT_SUCCESS;
-}
diff --git a/lib/tar/test/tar_istream2.c b/lib/tar/test/tar_istream2.c
deleted file mode 100644
index e5cdca8..0000000
--- a/lib/tar/test/tar_istream2.c
+++ /dev/null
@@ -1,151 +0,0 @@
-/* SPDX-License-Identifier: GPL-3.0-or-later */
-/*
- * tar_istream2.c
- *
- * Copyright (C) 2019 David Oberhollenzer <goliath@infraroot.at>
- */
-#include "config.h"
-#include "io/file.h"
-#include "tar/tar.h"
-#include "util/test.h"
-
-int main(int argc, char **argv)
-{
-	tar_header_decoded_t hdr;
-	char buffer[100];
-	istream_t *fp;
-	istream_t *ti;
-	(void)argc; (void)argv;
-
-	TEST_ASSERT(chdir(TEST_PATH) == 0);
-
-	fp = istream_open_file("format-acceptance/link_filled.tar");
-	TEST_NOT_NULL(fp);
-	TEST_EQUAL_UI(((sqfs_object_t *)fp)->refcount, 1);
-
-	/* "deep" directory hierarchy containg 2 files */
-	TEST_EQUAL_UI(((sqfs_object_t *)fp)->refcount, 1);
-	TEST_ASSERT(read_header(fp, &hdr) == 0);
-	TEST_EQUAL_UI(((sqfs_object_t *)fp)->refcount, 1);
-	TEST_EQUAL_UI(hdr.mode, S_IFDIR | 0777);
-	TEST_STR_EQUAL(hdr.name, "20_characters_here01/");
-	clear_header(&hdr);
-
-	TEST_EQUAL_UI(((sqfs_object_t *)fp)->refcount, 1);
-	TEST_ASSERT(read_header(fp, &hdr) == 0);
-	TEST_EQUAL_UI(((sqfs_object_t *)fp)->refcount, 1);
-	TEST_EQUAL_UI(hdr.mode, S_IFDIR | 0777);
-	TEST_STR_EQUAL(hdr.name, "20_characters_here01/20_characters_here02/");
-	clear_header(&hdr);
-
-	TEST_EQUAL_UI(((sqfs_object_t *)fp)->refcount, 1);
-	TEST_ASSERT(read_header(fp, &hdr) == 0);
-	TEST_EQUAL_UI(((sqfs_object_t *)fp)->refcount, 1);
-	TEST_EQUAL_UI(hdr.mode, S_IFDIR | 0777);
-	TEST_STR_EQUAL(hdr.name, "20_characters_here01/20_characters_here02/"
-		       "20_characters_here03/");
-	clear_header(&hdr);
-
-	TEST_EQUAL_UI(((sqfs_object_t *)fp)->refcount, 1);
-	TEST_ASSERT(read_header(fp, &hdr) == 0);
-	TEST_EQUAL_UI(((sqfs_object_t *)fp)->refcount, 1);
-	TEST_EQUAL_UI(hdr.mode, S_IFDIR | 0777);
-	TEST_STR_EQUAL(hdr.name, "20_characters_here01/20_characters_here02/"
-		       "20_characters_here03/20_characters_here04/");
-	clear_header(&hdr);
-
-	TEST_EQUAL_UI(((sqfs_object_t *)fp)->refcount, 1);
-	TEST_ASSERT(read_header(fp, &hdr) == 0);
-	TEST_EQUAL_UI(((sqfs_object_t *)fp)->refcount, 1);
-	TEST_EQUAL_UI(hdr.mode, S_IFREG | 0777);
-	TEST_STR_EQUAL(hdr.name, "20_characters_here01/20_characters_here02/"
-		       "20_characters_here03/20_characters_here04/"
-		       "errored_file_tst");
-	TEST_EQUAL_UI(hdr.actual_size, 5);
-
-	ti = tar_record_istream_create(fp, &hdr);
-	TEST_NOT_NULL(ti);
-	TEST_EQUAL_UI(((sqfs_object_t *)fp)->refcount, 2);
-	TEST_EQUAL_UI(((sqfs_object_t *)ti)->refcount, 1);
-	clear_header(&hdr);
-
-	TEST_ASSERT(istream_read(ti, buffer, sizeof(buffer)) == 5);
-	buffer[5] = '\0';
-	TEST_STR_EQUAL(buffer, "test\n");
-	TEST_ASSERT(istream_read(ti, buffer, sizeof(buffer)) == 0);
-
-	ti = sqfs_drop(ti);
-	TEST_EQUAL_UI(((sqfs_object_t *)fp)->refcount, 1);
-
-	TEST_EQUAL_UI(((sqfs_object_t *)fp)->refcount, 1);
-	TEST_ASSERT(read_header(fp, &hdr) == 0);
-	TEST_EQUAL_UI(((sqfs_object_t *)fp)->refcount, 1);
-	TEST_EQUAL_UI(hdr.mode, S_IFREG | 0777);
-	TEST_STR_EQUAL(hdr.name, "20_characters_here01/20_characters_here02/"
-		       "20_characters_here03/20_characters_here04/"
-		       "some_test_file");
-	TEST_EQUAL_UI(hdr.actual_size, 5);
-
-	ti = tar_record_istream_create(fp, &hdr);
-	TEST_NOT_NULL(ti);
-	TEST_EQUAL_UI(((sqfs_object_t *)fp)->refcount, 2);
-	TEST_EQUAL_UI(((sqfs_object_t *)ti)->refcount, 1);
-	clear_header(&hdr);
-
-	TEST_ASSERT(istream_read(ti, buffer, sizeof(buffer)) == 5);
-	buffer[5] = '\0';
-	TEST_STR_EQUAL(buffer, "test\n");
-	TEST_ASSERT(istream_read(ti, buffer, sizeof(buffer)) == 0);
-
-	ti = sqfs_drop(ti);
-	TEST_EQUAL_UI(((sqfs_object_t *)fp)->refcount, 1);
-
-	/* "deep" directory hierarchy containg a hard link */
-	TEST_EQUAL_UI(((sqfs_object_t *)fp)->refcount, 1);
-	TEST_ASSERT(read_header(fp, &hdr) == 0);
-	TEST_EQUAL_UI(((sqfs_object_t *)fp)->refcount, 1);
-	TEST_EQUAL_UI(hdr.mode, S_IFDIR | 0777);
-	TEST_STR_EQUAL(hdr.name, "20CharsForLnkTest001/");
-	clear_header(&hdr);
-
-	TEST_EQUAL_UI(((sqfs_object_t *)fp)->refcount, 1);
-	TEST_ASSERT(read_header(fp, &hdr) == 0);
-	TEST_EQUAL_UI(((sqfs_object_t *)fp)->refcount, 1);
-	TEST_EQUAL_UI(hdr.mode, S_IFDIR | 0777);
-	TEST_STR_EQUAL(hdr.name, "20CharsForLnkTest001/20CharsForLnkTest002/");
-	clear_header(&hdr);
-
-	TEST_EQUAL_UI(((sqfs_object_t *)fp)->refcount, 1);
-	TEST_ASSERT(read_header(fp, &hdr) == 0);
-	TEST_EQUAL_UI(((sqfs_object_t *)fp)->refcount, 1);
-	TEST_EQUAL_UI(hdr.mode, S_IFDIR | 0777);
-	TEST_STR_EQUAL(hdr.name, "20CharsForLnkTest001/20CharsForLnkTest002/"
-		       "20CharsForLnkTest003/");
-	clear_header(&hdr);
-
-	TEST_EQUAL_UI(((sqfs_object_t *)fp)->refcount, 1);
-	TEST_ASSERT(read_header(fp, &hdr) == 0);
-	TEST_EQUAL_UI(((sqfs_object_t *)fp)->refcount, 1);
-	TEST_EQUAL_UI(hdr.mode, S_IFDIR | 0777);
-	TEST_STR_EQUAL(hdr.name, "20CharsForLnkTest001/20CharsForLnkTest002/"
-		       "20CharsForLnkTest003/20CharsForLnkTest004/");
-	clear_header(&hdr);
-
-	TEST_EQUAL_UI(((sqfs_object_t *)fp)->refcount, 1);
-	TEST_ASSERT(read_header(fp, &hdr) == 0);
-	TEST_EQUAL_UI(((sqfs_object_t *)fp)->refcount, 1);
-	TEST_STR_EQUAL(hdr.name, "20CharsForLnkTest001/20CharsForLnkTest002/"
-		       "20CharsForLnkTest003/20CharsForLnkTest004/"
-		       "01234567890123456789");
-	TEST_ASSERT(hdr.is_hard_link);
-
-	TEST_STR_EQUAL(hdr.link_target, "20_characters_here01/"
-		       "20_characters_here02/20_characters_here03/"
-		       "20_characters_here04/errored_file_tst");
-	clear_header(&hdr);
-
-	/* end of file */
-	TEST_ASSERT(read_header(fp, &hdr) > 0);
-	sqfs_drop(fp);
-	return EXIT_SUCCESS;
-}
diff --git a/lib/tar/test/tar_iterator.c b/lib/tar/test/tar_iterator.c
new file mode 100644
index 0000000..21bd3e6
--- /dev/null
+++ b/lib/tar/test/tar_iterator.c
@@ -0,0 +1,139 @@
+/* SPDX-License-Identifier: GPL-3.0-or-later */
+/*
+ * tar_iterator.c
+ *
+ * Copyright (C) 2019 David Oberhollenzer <goliath@infraroot.at>
+ */
+#include "config.h"
+#include "io/file.h"
+#include "tar/tar.h"
+#include "util/test.h"
+#include "sqfs/error.h"
+
+#ifndef TESTUID
+#define TESTUID 1000
+#endif
+
+#ifndef TESTGID
+#define TESTGID TESTUID
+#endif
+
+#ifndef TESTFNAME
+#define TESTFNAME input.txt
+#endif
+
+#ifndef TESTTS
+#define TESTTS 1542905892
+#endif
+
+static const char *fname = STRVALUE(TESTFNAME);
+
+int main(int argc, char **argv)
+{
+	istream_t *fp, *ti, *ti2;
+	dir_iterator_t *it;
+	dir_entry_t *ent;
+	char buffer[100];
+	sqfs_s32 ret;
+	sqfs_s64 ts;
+	(void)argc; (void)argv;
+
+	/* Open the file, create an iterator */
+	fp = istream_open_file(STRVALUE(TESTPATH) "/" STRVALUE(TESTFILE));
+	TEST_NOT_NULL(fp);
+	TEST_EQUAL_UI(((sqfs_object_t *)fp)->refcount, 1);
+	it = tar_open_stream(fp);
+	TEST_NOT_NULL(it);
+	TEST_EQUAL_UI(((sqfs_object_t *)fp)->refcount, 2);
+	TEST_EQUAL_UI(((sqfs_object_t *)it)->refcount, 1);
+
+	/* read entry */
+	ret = it->next(it, &ent);
+	TEST_EQUAL_I(ret, 0);
+	TEST_NOT_NULL(ent);
+	TEST_EQUAL_UI(((sqfs_object_t *)fp)->refcount, 2);
+	TEST_EQUAL_UI(((sqfs_object_t *)it)->refcount, 1);
+
+	ts = TESTTS;
+	TEST_EQUAL_UI(ent->mode, S_IFREG | 0644);
+	TEST_EQUAL_UI(ent->uid, TESTUID);
+	TEST_EQUAL_UI(ent->gid, TESTGID);
+	TEST_EQUAL_UI(ent->mtime, ts);
+	TEST_STR_EQUAL(ent->name, fname);
+	free(ent);
+	ent = NULL;
+
+	/* Open file stream */
+	ret = it->open_file_ro(it, &ti);
+	TEST_EQUAL_I(ret, 0);
+	TEST_NOT_NULL(ti);
+	TEST_EQUAL_UI(((sqfs_object_t *)fp)->refcount, 2);
+	TEST_EQUAL_UI(((sqfs_object_t *)it)->refcount, 2);
+	TEST_EQUAL_UI(((sqfs_object_t *)ti)->refcount, 1);
+
+	/* test that the iterator is now "locked" */
+	ret = it->open_file_ro(it, &ti2);
+	TEST_EQUAL_I(ret, SQFS_ERROR_SEQUENCE);
+	TEST_NULL(ti2);
+	ret = it->next(it, &ent);
+	TEST_EQUAL_I(ret, SQFS_ERROR_SEQUENCE);
+	TEST_NULL(ent);
+	TEST_EQUAL_UI(((sqfs_object_t *)fp)->refcount, 2);
+	TEST_EQUAL_UI(((sqfs_object_t *)it)->refcount, 2);
+	TEST_EQUAL_UI(((sqfs_object_t *)ti)->refcount, 1);
+
+	/* read the data from the stream */
+	ret = istream_read(ti, buffer, sizeof(buffer));
+	TEST_EQUAL_I(ret, 5);
+	buffer[5] = '\0';
+	TEST_STR_EQUAL(buffer, "test\n");
+
+	ret = istream_read(ti, buffer, sizeof(buffer));
+	TEST_EQUAL_I(ret, 0);
+
+	sqfs_drop(ti);
+	TEST_EQUAL_UI(((sqfs_object_t *)fp)->refcount, 2);
+	TEST_EQUAL_UI(((sqfs_object_t *)it)->refcount, 1);
+
+	/* read past EOF on the iterator */
+	ret = it->next(it, &ent);
+	TEST_ASSERT(ret > 0);
+	TEST_NULL(ent);
+
+	/* cleanup */
+	sqfs_drop(it);
+	TEST_EQUAL_UI(((sqfs_object_t *)fp)->refcount, 1);
+	sqfs_drop(fp);
+
+	/* re-open the tar iterator */
+	fp = istream_open_file(STRVALUE(TESTPATH) "/" STRVALUE(TESTFILE));
+	TEST_NOT_NULL(fp);
+	it = tar_open_stream(fp);
+	TEST_NOT_NULL(it);
+
+	/* read entry */
+	ret = it->next(it, &ent);
+	TEST_EQUAL_I(ret, 0);
+	TEST_NOT_NULL(ent);
+
+	ts = TESTTS;
+	TEST_EQUAL_UI(ent->mode, S_IFREG | 0644);
+	TEST_EQUAL_UI(ent->uid, TESTUID);
+	TEST_EQUAL_UI(ent->gid, TESTGID);
+	TEST_EQUAL_UI(ent->mtime, ts);
+	TEST_STR_EQUAL(ent->name, fname);
+	free(ent);
+	ent = NULL;
+
+	/* read next entry, mus treturn EOF */
+	ret = it->next(it, &ent);
+	TEST_ASSERT(ret > 0);
+	TEST_NULL(ent);
+
+	/* cleanup */
+	sqfs_drop(it);
+	TEST_EQUAL_UI(((sqfs_object_t *)fp)->refcount, 1);
+	sqfs_drop(fp);
+
+	return EXIT_SUCCESS;
+}
diff --git a/lib/tar/test/tar_istream3.c b/lib/tar/test/tar_iterator2.c
index d287081..7f1f3bf 100644
--- a/lib/tar/test/tar_istream3.c
+++ b/lib/tar/test/tar_iterator2.c
@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: GPL-3.0-or-later */
 /*
- * tar_istream3.c
+ * tar_iterator2.c
  *
  * Copyright (C) 2019 David Oberhollenzer <goliath@infraroot.at>
  */
@@ -44,27 +44,34 @@ static int byte_from_offset(uint64_t offset)
 int main(int argc, char **argv)
 {
 	unsigned char buffer[941];
-	tar_header_decoded_t hdr;
+	dir_iterator_t *it;
 	istream_t *fp, *ti;
+	dir_entry_t *ent;
 	uint64_t offset;
 	sqfs_s32 i, ret;
 	(void)argc; (void)argv;
 
 	fp = istream_open_file(STRVALUE(TESTPATH) "/" STRVALUE(TESTFILE));
 	TEST_NOT_NULL(fp);
-	TEST_ASSERT(read_header(fp, &hdr) == 0);
-	TEST_EQUAL_UI(hdr.mode, S_IFREG | 0644);
-	TEST_EQUAL_UI(hdr.uid, 01750);
-	TEST_EQUAL_UI(hdr.gid, 01750);
-	TEST_EQUAL_UI(hdr.actual_size, 2097152);
-	TEST_EQUAL_UI(hdr.record_size, 32768);
-	TEST_STR_EQUAL(hdr.name, "input.bin");
-	TEST_ASSERT(!hdr.unknown_record);
-
-	ti = tar_record_istream_create(fp, &hdr);
-	TEST_EQUAL_UI(((sqfs_object_t *)fp)->refcount, 2);
-	TEST_EQUAL_UI(((sqfs_object_t *)ti)->refcount, 1);
-	clear_header(&hdr);
+	it = tar_open_stream(fp);
+	TEST_NOT_NULL(it);
+	sqfs_drop(fp);
+
+	ret = it->next(it, &ent);
+	TEST_EQUAL_I(ret, 0);
+	TEST_NOT_NULL(ent);
+
+	TEST_EQUAL_UI(ent->mode, S_IFREG | 0644);
+	TEST_EQUAL_UI(ent->uid, 01750);
+	TEST_EQUAL_UI(ent->gid, 01750);
+	TEST_EQUAL_UI(ent->size, 2097152);
+	//TEST_EQUAL_UI(ent->on_disk_size, 32768);
+	TEST_STR_EQUAL(ent->name, "input.bin");
+	free(ent);
+
+	ret = it->open_file_ro(it, &ti);
+	TEST_EQUAL_I(ret, 0);
+	TEST_NOT_NULL(ti);
 
 	offset = 0;
 
@@ -93,8 +100,7 @@ int main(int argc, char **argv)
 
 	TEST_EQUAL_UI(offset, 2097152);
 
+	sqfs_drop(it);
 	sqfs_drop(ti);
-	TEST_EQUAL_UI(((sqfs_object_t *)fp)->refcount, 1);
-	sqfs_drop(fp);
 	return EXIT_SUCCESS;
 }
diff --git a/lib/tar/test/tar_iterator3.c b/lib/tar/test/tar_iterator3.c
new file mode 100644
index 0000000..b2ad6f9
--- /dev/null
+++ b/lib/tar/test/tar_iterator3.c
@@ -0,0 +1,173 @@
+
+/* SPDX-License-Identifier: GPL-3.0-or-later */
+/*
+ * tar_iterator3.c
+ *
+ * Copyright (C) 2019 David Oberhollenzer <goliath@infraroot.at>
+ */
+#include "config.h"
+#include "io/file.h"
+#include "tar/tar.h"
+#include "util/test.h"
+#include "sqfs/error.h"
+
+int main(int argc, char **argv)
+{
+	dir_iterator_t *it;
+	istream_t *fp, *ti;
+	char buffer[100];
+	dir_entry_t *ent;
+	char *link;
+	int ret;
+	(void)argc; (void)argv;
+
+	TEST_ASSERT(chdir(TEST_PATH) == 0);
+
+	fp = istream_open_file("format-acceptance/link_filled.tar");
+	TEST_NOT_NULL(fp);
+	TEST_EQUAL_UI(((sqfs_object_t *)fp)->refcount, 1);
+	it = tar_open_stream(fp);
+	TEST_NOT_NULL(it);
+	TEST_EQUAL_UI(((sqfs_object_t *)fp)->refcount, 2);
+	TEST_EQUAL_UI(((sqfs_object_t *)it)->refcount, 1);
+	sqfs_drop(fp);
+
+	/* "deep" directory hierarchy containg 2 files */
+	ret = it->next(it, &ent);
+	TEST_EQUAL_I(ret, 0);
+	TEST_NOT_NULL(ent);
+	TEST_EQUAL_UI(ent->mode, S_IFDIR | 0777);
+	TEST_STR_EQUAL(ent->name, "20_characters_here01");
+	free(ent);
+
+	ret = it->read_link(it, &link);
+	TEST_EQUAL_I(ret, SQFS_ERROR_NO_ENTRY);
+	TEST_NULL(link);
+	ret = it->open_file_ro(it, &ti);
+	TEST_EQUAL_I(ret, SQFS_ERROR_NOT_FILE);
+	TEST_NULL(ti);
+
+	ret = it->next(it, &ent);
+	TEST_EQUAL_I(ret, 0);
+	TEST_NOT_NULL(ent);
+	TEST_EQUAL_UI(ent->mode, S_IFDIR | 0777);
+	TEST_STR_EQUAL(ent->name, "20_characters_here01/20_characters_here02");
+	free(ent);
+
+	ret = it->next(it, &ent);
+	TEST_EQUAL_I(ret, 0);
+	TEST_NOT_NULL(ent);
+	TEST_EQUAL_UI(ent->mode, S_IFDIR | 0777);
+	TEST_STR_EQUAL(ent->name, "20_characters_here01/20_characters_here02/"
+		       "20_characters_here03");
+	free(ent);
+
+	ret = it->next(it, &ent);
+	TEST_EQUAL_I(ret, 0);
+	TEST_NOT_NULL(ent);
+	TEST_EQUAL_UI(ent->mode, S_IFDIR | 0777);
+	TEST_STR_EQUAL(ent->name, "20_characters_here01/20_characters_here02/"
+		       "20_characters_here03/20_characters_here04");
+	free(ent);
+
+	ret = it->next(it, &ent);
+	TEST_EQUAL_I(ret, 0);
+	TEST_NOT_NULL(ent);
+	TEST_EQUAL_UI(ent->mode, S_IFREG | 0777);
+	TEST_STR_EQUAL(ent->name, "20_characters_here01/20_characters_here02/"
+		       "20_characters_here03/20_characters_here04/"
+		       "errored_file_tst");
+	free(ent);
+
+	ret = it->read_link(it, &link);
+	TEST_EQUAL_I(ret, SQFS_ERROR_NO_ENTRY);
+	TEST_NULL(link);
+	ret = it->open_file_ro(it, &ti);
+	TEST_EQUAL_I(ret, 0);
+	TEST_NOT_NULL(ti);
+
+	TEST_ASSERT(istream_read(ti, buffer, sizeof(buffer)) == 5);
+	buffer[5] = '\0';
+	TEST_STR_EQUAL(buffer, "test\n");
+	TEST_ASSERT(istream_read(ti, buffer, sizeof(buffer)) == 0);
+	ti = sqfs_drop(ti);
+
+	ret = it->next(it, &ent);
+	TEST_EQUAL_I(ret, 0);
+	TEST_NOT_NULL(ent);
+	TEST_EQUAL_UI(ent->mode, S_IFREG | 0777);
+	TEST_STR_EQUAL(ent->name, "20_characters_here01/20_characters_here02/"
+		       "20_characters_here03/20_characters_here04/"
+		       "some_test_file");
+	free(ent);
+
+	ret = it->open_file_ro(it, &ti);
+	TEST_EQUAL_I(ret, 0);
+	TEST_NOT_NULL(ti);
+
+	TEST_ASSERT(istream_read(ti, buffer, sizeof(buffer)) == 5);
+	buffer[5] = '\0';
+	TEST_STR_EQUAL(buffer, "test\n");
+	TEST_ASSERT(istream_read(ti, buffer, sizeof(buffer)) == 0);
+	ti = sqfs_drop(ti);
+
+	/* "deep" directory hierarchy containg a hard link */
+	ret = it->next(it, &ent);
+	TEST_EQUAL_I(ret, 0);
+	TEST_NOT_NULL(ent);
+	TEST_EQUAL_UI(ent->mode, S_IFDIR | 0777);
+	TEST_STR_EQUAL(ent->name, "20CharsForLnkTest001");
+	free(ent);
+
+	ret = it->next(it, &ent);
+	TEST_EQUAL_I(ret, 0);
+	TEST_NOT_NULL(ent);
+	TEST_EQUAL_UI(ent->mode, S_IFDIR | 0777);
+	TEST_STR_EQUAL(ent->name, "20CharsForLnkTest001/20CharsForLnkTest002");
+	free(ent);
+
+	ret = it->next(it, &ent);
+	TEST_EQUAL_I(ret, 0);
+	TEST_NOT_NULL(ent);
+	TEST_EQUAL_UI(ent->mode, S_IFDIR | 0777);
+	TEST_STR_EQUAL(ent->name, "20CharsForLnkTest001/20CharsForLnkTest002/"
+		       "20CharsForLnkTest003");
+	free(ent);
+
+	ret = it->next(it, &ent);
+	TEST_EQUAL_I(ret, 0);
+	TEST_NOT_NULL(ent);
+	TEST_EQUAL_UI(ent->mode, S_IFDIR | 0777);
+	TEST_STR_EQUAL(ent->name, "20CharsForLnkTest001/20CharsForLnkTest002/"
+		       "20CharsForLnkTest003/20CharsForLnkTest004");
+	free(ent);
+
+	ret = it->next(it, &ent);
+	TEST_EQUAL_I(ret, 0);
+	TEST_NOT_NULL(ent);
+	TEST_STR_EQUAL(ent->name, "20CharsForLnkTest001/20CharsForLnkTest002/"
+		       "20CharsForLnkTest003/20CharsForLnkTest004/"
+		       "01234567890123456789");
+	TEST_EQUAL_UI(ent->mode, S_IFLNK | 0777);
+	TEST_ASSERT((ent->flags & DIR_ENTRY_FLAG_HARD_LINK) != 0);
+	free(ent);
+
+	ret = it->read_link(it, &link);
+	TEST_EQUAL_I(ret, 0);
+	TEST_NOT_NULL(link);
+	ret = it->open_file_ro(it, &ti);
+	TEST_EQUAL_I(ret, SQFS_ERROR_NOT_FILE);
+	TEST_NULL(ti);
+
+	TEST_STR_EQUAL(link, "20_characters_here01/"
+		       "20_characters_here02/20_characters_here03/"
+		       "20_characters_here04/errored_file_tst");
+	free(link);
+
+	/* end of file */
+	ret = it->next(it, &ent);
+	TEST_ASSERT(ret > 0);
+	TEST_NULL(ent);
+	sqfs_drop(it);
+	return EXIT_SUCCESS;
+}