Remove condensed sparse file handling from libsquashfs

This only exists for tar2sqfs. Move the sparse file map to libtar
and add the ability to do this into the stind sqfs_file_t abstraction,
so it acts like a normal file but internally stitches the data
together from the sparse implementation.

Signed-off-by: David Oberhollenzer <david.oberhollenzer@sigma-star.at>

17 files changed, 101 insertions, 178 deletions

diff --git a/include/data_writer.h b/include/data_writer.h
index e29881b..cdf7a44 100644
--- a/include/data_writer.h
+++ b/include/data_writer.h
@@ -88,20 +88,6 @@ int data_writer_sync(data_writer_t *data);
 int write_data_from_file(data_writer_t *data, sqfs_inode_generic_t *inode,
 			 sqfs_file_t *file, int flags);
 
-/*
-  Does the same as write_data_from_fd but the input file is the condensed
-  representation of a sparse file. The layout must be in order and
-  non-overlapping.
-
-  The flags argument is a combination of DW_* flags. After completion the
-  data writer collects the 'fi' in an internal list it uses for deduplication.
-
-  Returns 0 on success, prints errors to stderr.
- */
-int write_data_from_file_condensed(data_writer_t *data, sqfs_file_t *file,
-				   sqfs_inode_generic_t *inode,
-				   const sqfs_sparse_map_t *map, int flags);
-
 data_writer_stats_t *data_writer_get_stats(data_writer_t *data);
 
 #endif /* DATA_WRITER_H */
diff --git a/include/highlevel.h b/include/highlevel.h
index ed0f4bf..bfc2d91 100644
--- a/include/highlevel.h
+++ b/include/highlevel.h
@@ -23,6 +23,7 @@
 #include "sqfs/data_reader.h"
 #include "data_writer.h"
 #include "fstree.h"
+#include "tar.h"
 
 #include <sys/stat.h>
 #include <stdint.h>
@@ -83,6 +84,6 @@ int sqfs_data_reader_dump(sqfs_data_reader_t *data,
 			  const sqfs_inode_generic_t *inode,
 			  int outfd, size_t block_size, bool allow_sparse);
 
-sqfs_file_t *sqfs_get_stdin_file(uint64_t size);
+sqfs_file_t *sqfs_get_stdin_file(const sparse_map_t *map, uint64_t size);
 
 #endif /* HIGHLEVEL_H */
diff --git a/include/sqfs/io.h b/include/sqfs/io.h
index d7a923c..5c1bfd5 100644
--- a/include/sqfs/io.h
+++ b/include/sqfs/io.h
@@ -120,20 +120,6 @@ struct sqfs_file_t {
 	int (*truncate)(sqfs_file_t *file, uint64_t size);
 };
 
-/**
- * @struct sqfs_sparse_map_t
- *
- * @brief Describes the layout of a sparse file.
- *
- * This structure is part of a linked list that indicates where the actual
- * data is located in a sparse file.
- */
-struct sqfs_sparse_map_t {
-	sqfs_sparse_map_t *next;
-	uint64_t offset;
-	uint64_t count;
-};
-
 #ifdef __cplusplus
 extern "C" {
 #endif
@@ -175,36 +161,6 @@ SQFS_API int sqfs_file_create_block(sqfs_file_t *file, uint64_t offset,
 				    size_t size, sqfs_inode_generic_t *inode,
 				    uint32_t flags, sqfs_block_t **out);
 
-/**
- * @brief Read a chunk from a condensed version of a sparse file and turn it
- *        into a block that can be fed to a block processor.
- *
- * @member sqfs_file_t
- *
- * This function works on condensed sparse files, i.e. a sparse file that had
- * its holdes removed. The given mapping describes the original data region
- * that are actually packed next to each other. The function emulates the
- * orignal sparse file by zero-initializing the block data, then figuring
- * out which regions overlap the block, working out their physical location and
- * stitching the block together.
- *
- * @param file A pointer to a file implementation.
- * @param offset A byte offset into the file.
- * @param size The number of bytes to read, starting at the given offset.
- * @param inode The inode pointer to set for the block.
- * @param flags The flags to store in the newly created block.
- * @param map Describes the data regions of the original sparse file.
- * @param out Returns a pointer to a block on success.
- *
- * @return Zero on success, an @ref E_SQFS_ERROR identifier on failure.
- */
-SQFS_API int sqfs_file_create_block_dense(sqfs_file_t *file, uint64_t offset,
-					  size_t size,
-					  sqfs_inode_generic_t *inode,
-					  uint32_t flags,
-					  const sqfs_sparse_map_t *map,
-					  sqfs_block_t **out);
-
 #ifdef __cplusplus
 }
 #endif
diff --git a/include/sqfs/predef.h b/include/sqfs/predef.h
index 8a554dd..ebe99e1 100644
--- a/include/sqfs/predef.h
+++ b/include/sqfs/predef.h
@@ -69,7 +69,6 @@ typedef struct sqfs_meta_reader_t sqfs_meta_reader_t;
 typedef struct sqfs_meta_writer_t sqfs_meta_writer_t;
 typedef struct sqfs_xattr_reader_t sqfs_xattr_reader_t;
 typedef struct sqfs_file_t sqfs_file_t;
-typedef struct sqfs_sparse_map_t sqfs_sparse_map_t;
 typedef struct sqfs_tree_node_t sqfs_tree_node_t;
 typedef struct sqfs_data_reader_t sqfs_data_reader_t;
 typedef struct sqfs_block_hooks_t sqfs_block_hooks_t;
diff --git a/include/tar.h b/include/tar.h
index 38f6289..45457da 100644
--- a/include/tar.h
+++ b/include/tar.h
@@ -8,13 +8,17 @@
 #define TAR_H
 
 #include "config.h"
+#include "util.h"
 
 #include <sys/stat.h>
 #include <stdbool.h>
 #include <stdint.h>
 
-#include "sqfs/io.h"
-#include "util.h"
+typedef struct sparse_map_t {
+	struct sparse_map_t *next;
+	uint64_t offset;
+	uint64_t count;
+} sparse_map_t;
 
 typedef struct {
 	char name[100];
@@ -75,7 +79,7 @@ typedef struct {
 	struct stat sb;
 	char *name;
 	char *link_target;
-	sqfs_sparse_map_t *sparse;
+	sparse_map_t *sparse;
 	uint64_t actual_size;
 	uint64_t record_size;
 	bool unknown_record;
diff --git a/lib/sqfs/io.c b/lib/sqfs/io.c
index f4ffda2..0021a30 100644
--- a/lib/sqfs/io.c
+++ b/lib/sqfs/io.c
@@ -37,69 +37,3 @@ int sqfs_file_create_block(sqfs_file_t *file, uint64_t offset,
 	*out = blk;
 	return 0;
 }
-
-int sqfs_file_create_block_dense(sqfs_file_t *file, uint64_t offset,
-				 size_t size, sqfs_inode_generic_t *inode,
-				 uint32_t flags, const sqfs_sparse_map_t *map,
-				 sqfs_block_t **out)
-{
-	sqfs_block_t *blk = alloc_flex(sizeof(*blk), 1, size);
-	size_t dst_start, diff, count;
-	const sqfs_sparse_map_t *it;
-	uint64_t poffset, src_start;
-	int err;
-
-	if (blk == NULL)
-		return SQFS_ERROR_ALLOC;
-
-	poffset = 0;
-
-	for (it = map; it != NULL; it = it->next) {
-		if (it->offset + it->count <= offset) {
-			poffset += it->count;
-			continue;
-		}
-
-		if (it->offset >= offset + size) {
-			poffset += it->count;
-			continue;
-		}
-
-		count = size;
-
-		if (offset + count >= it->offset + it->count)
-			count = it->offset + it->count - offset;
-
-		if (it->offset < offset) {
-			diff = offset - it->offset;
-
-			src_start = poffset + diff;
-			dst_start = 0;
-			count -= diff;
-		} else if (it->offset > offset) {
-			diff = it->offset - offset;
-
-			src_start = poffset;
-			dst_start = diff;
-		} else {
-			src_start = poffset;
-			dst_start = 0;
-		}
-
-		err = file->read_at(file, src_start,
-				    blk->data + dst_start, count);
-		if (err) {
-			free(blk);
-			return err;
-		}
-
-		poffset += it->count;
-	}
-
-	blk->inode = inode;
-	blk->size = size;
-	blk->flags = flags;
-
-	*out = blk;
-	return 0;
-}
diff --git a/lib/sqfshelper/data_writer.c b/lib/sqfshelper/data_writer.c
index 96cd9ee..a365e6d 100644
--- a/lib/sqfshelper/data_writer.c
+++ b/lib/sqfshelper/data_writer.c
@@ -98,9 +98,8 @@ static int add_sentinel_block(data_writer_t *data, sqfs_inode_generic_t *inode,
 	return sqfs_data_writer_enqueue(data->proc, blk);
 }
 
-int write_data_from_file_condensed(data_writer_t *data, sqfs_file_t *file,
-				   sqfs_inode_generic_t *inode,
-				   const sqfs_sparse_map_t *map, int flags)
+int write_data_from_file(data_writer_t *data, sqfs_inode_generic_t *inode,
+			 sqfs_file_t *file, int flags)
 {
 	uint32_t blk_flags = SQFS_BLK_FIRST_BLOCK;
 	uint64_t filesz, offset;
@@ -123,14 +122,8 @@ int write_data_from_file_condensed(data_writer_t *data, sqfs_file_t *file,
 			diff = filesz - offset;
 		}
 
-		if (map == NULL) {
-			ret = sqfs_file_create_block(file, offset, diff, inode,
-						     blk_flags, &blk);
-		} else {
-			ret = sqfs_file_create_block_dense(file, offset, diff,
-							   inode, blk_flags,
-							   map, &blk);
-		}
+		ret = sqfs_file_create_block(file, offset, diff, inode,
+					     blk_flags, &blk);
 
 		if (ret)
 			return -1;
@@ -189,12 +182,6 @@ int write_data_from_file_condensed(data_writer_t *data, sqfs_file_t *file,
 	return 0;
 }
 
-int write_data_from_file(data_writer_t *data, sqfs_inode_generic_t *inode,
-			 sqfs_file_t *file, int flags)
-{
-	return write_data_from_file_condensed(data, file, inode, NULL, flags);
-}
-
 data_writer_t *data_writer_create(sqfs_super_t *super, sqfs_compressor_t *cmp,
 				  sqfs_file_t *file, size_t devblksize,
 				  unsigned int num_jobs, size_t max_backlog)
diff --git a/lib/sqfshelper/io_stdin.c b/lib/sqfshelper/io_stdin.c
index 6cb45d4..8568f5e 100644
--- a/lib/sqfshelper/io_stdin.c
+++ b/lib/sqfshelper/io_stdin.c
@@ -11,12 +11,14 @@
 
 #include <stdlib.h>
 #include <unistd.h>
+#include <string.h>
 #include <errno.h>
 
 
 typedef struct {
 	sqfs_file_t base;
 
+	const sparse_map_t *map;
 	uint64_t offset;
 	uint64_t size;
 } sqfs_file_stdin_t;
@@ -78,6 +80,60 @@ static int stdin_read_at(sqfs_file_t *base, uint64_t offset,
 	return 0;
 }
 
+static int stdin_read_condensed(sqfs_file_t *base, uint64_t offset,
+				void *buffer, size_t size)
+{
+	sqfs_file_stdin_t *file = (sqfs_file_stdin_t *)base;
+	uint64_t poffset = 0, src_start;
+	size_t dst_start, diff, count;
+	const sparse_map_t *it;
+	int err;
+
+	memset(buffer, 0, size);
+
+	for (it = file->map; it != NULL; it = it->next) {
+		if (it->offset + it->count <= offset) {
+			poffset += it->count;
+			continue;
+		}
+
+		if (it->offset >= offset + size) {
+			poffset += it->count;
+			continue;
+		}
+
+		count = size;
+
+		if (offset + count >= it->offset + it->count)
+			count = it->offset + it->count - offset;
+
+		if (it->offset < offset) {
+			diff = offset - it->offset;
+
+			src_start = poffset + diff;
+			dst_start = 0;
+			count -= diff;
+		} else if (it->offset > offset) {
+			diff = it->offset - offset;
+
+			src_start = poffset;
+			dst_start = diff;
+		} else {
+			src_start = poffset;
+			dst_start = 0;
+		}
+
+		err = stdin_read_at(base, src_start,
+				    (char *)buffer + dst_start, count);
+		if (err)
+			return err;
+
+		poffset += it->count;
+	}
+
+	return 0;
+}
+
 static int stdin_write_at(sqfs_file_t *base, uint64_t offset,
 			  const void *buffer, size_t size)
 {
@@ -96,7 +152,7 @@ static int stdin_truncate(sqfs_file_t *base, uint64_t size)
 	return SQFS_ERROR_IO;
 }
 
-sqfs_file_t *sqfs_get_stdin_file(uint64_t size)
+sqfs_file_t *sqfs_get_stdin_file(const sparse_map_t *map, uint64_t size)
 {
 	sqfs_file_stdin_t *file = calloc(1, sizeof(*file));
 	sqfs_file_t *base = (sqfs_file_t *)file;
@@ -105,10 +161,17 @@ sqfs_file_t *sqfs_get_stdin_file(uint64_t size)
 		return NULL;
 
 	file->size = size;
+	file->map = map;
+
 	base->destroy = stdin_destroy;
-	base->read_at = stdin_read_at;
 	base->write_at = stdin_write_at;
 	base->get_size = stdin_get_size;
 	base->truncate = stdin_truncate;
+
+	if (map == NULL) {
+		base->read_at = stdin_read_at;
+	} else {
+		base->read_at = stdin_read_condensed;
+	}
 	return base;
 }
diff --git a/lib/tar/cleanup.c b/lib/tar/cleanup.c
index 3c1d0e9..2f814bf 100644
--- a/lib/tar/cleanup.c
+++ b/lib/tar/cleanup.c
@@ -8,9 +8,9 @@
 
 #include "internal.h"
 
-void free_sparse_list(sqfs_sparse_map_t *sparse)
+void free_sparse_list(sparse_map_t *sparse)
 {
-	sqfs_sparse_map_t *old;
+	sparse_map_t *old;
 
 	while (sparse != NULL) {
 		old = sparse;
diff --git a/lib/tar/internal.h b/lib/tar/internal.h
index bc9a3a8..d95ef52 100644
--- a/lib/tar/internal.h
+++ b/lib/tar/internal.h
@@ -55,11 +55,11 @@ void update_checksum(tar_header_t *hdr);
 
 bool is_checksum_valid(const tar_header_t *hdr);
 
-sqfs_sparse_map_t *read_sparse_map(const char *line);
+sparse_map_t *read_sparse_map(const char *line);
 
-sqfs_sparse_map_t *read_gnu_old_sparse(int fd, tar_header_t *hdr);
+sparse_map_t *read_gnu_old_sparse(int fd, tar_header_t *hdr);
 
-void free_sparse_list(sqfs_sparse_map_t *sparse);
+void free_sparse_list(sparse_map_t *sparse);
 
 void free_xattr_list(tar_xattr_t *list);
 
diff --git a/lib/tar/read_header.c b/lib/tar/read_header.c
index a874972..63de699 100644
--- a/lib/tar/read_header.c
+++ b/lib/tar/read_header.c
@@ -77,7 +77,7 @@ static tar_xattr_t *mkxattr(const char *key, size_t keylen,
 static int read_pax_header(int fd, uint64_t entsize, unsigned int *set_by_pax,
 			   tar_header_decoded_t *out)
 {
-	sqfs_sparse_map_t *sparse_last = NULL, *sparse;
+	sparse_map_t *sparse_last = NULL, *sparse;
 	uint64_t field, offset = 0, num_bytes = 0;
 	char *buffer, *line, *key, *ptr, *value;
 	tar_xattr_t *xattr;
diff --git a/lib/tar/read_sparse_map.c b/lib/tar/read_sparse_map.c
index 981a808..0779b96 100644
--- a/lib/tar/read_sparse_map.c
+++ b/lib/tar/read_sparse_map.c
@@ -8,9 +8,9 @@
 
 #include "internal.h"
 
-sqfs_sparse_map_t *read_sparse_map(const char *line)
+sparse_map_t *read_sparse_map(const char *line)
 {
-	sqfs_sparse_map_t *last = NULL, *list = NULL, *ent = NULL;
+	sparse_map_t *last = NULL, *list = NULL, *ent = NULL;
 
 	do {
 		ent = calloc(1, sizeof(*ent));
diff --git a/lib/tar/read_sparse_map_old.c b/lib/tar/read_sparse_map_old.c
index 7fb4cb3..5891bdb 100644
--- a/lib/tar/read_sparse_map_old.c
+++ b/lib/tar/read_sparse_map_old.c
@@ -8,9 +8,9 @@
 
 #include "internal.h"
 
-sqfs_sparse_map_t *read_gnu_old_sparse(int fd, tar_header_t *hdr)
+sparse_map_t *read_gnu_old_sparse(int fd, tar_header_t *hdr)
 {
-	sqfs_sparse_map_t *list = NULL, *end = NULL, *node;
+	sparse_map_t *list = NULL, *end = NULL, *node;
 	gnu_sparse_t sph;
 	uint64_t off, sz;
 	int i;
diff --git a/tar/tar2sqfs.c b/tar/tar2sqfs.c
index e62985a..82720a8 100644
--- a/tar/tar2sqfs.c
+++ b/tar/tar2sqfs.c
@@ -225,7 +225,7 @@ fail_arg:
 static int write_file(tar_header_decoded_t *hdr, file_info_t *fi,
 		      data_writer_t *data, uint64_t filesize)
 {
-	const sqfs_sparse_map_t *it;
+	const sparse_map_t *it;
 	sqfs_inode_generic_t *inode;
 	size_t max_blk_count;
 	sqfs_file_t *file;
@@ -253,25 +253,17 @@ static int write_file(tar_header_decoded_t *hdr, file_info_t *fi,
 		for (sum = 0, it = hdr->sparse; it != NULL; it = it->next)
 			sum += it->count;
 
-		file = sqfs_get_stdin_file(sum);
+		file = sqfs_get_stdin_file(hdr->sparse, sum);
 		if (file == NULL) {
 			perror("packing files");
 			return -1;
 		}
-
-		ret = write_data_from_file_condensed(data, file, inode,
-						     hdr->sparse, 0);
-		file->destroy(file);
-		if (ret)
+	} else {
+		file = sqfs_get_stdin_file(NULL, filesize);
+		if (file == NULL) {
+			perror("packing files");
 			return -1;
-
-		return skip_padding(STDIN_FILENO, hdr->record_size);
-	}
-
-	file = sqfs_get_stdin_file(filesize);
-	if (file == NULL) {
-		perror("packing files");
-		return -1;
+		}
 	}
 
 	ret = write_data_from_file(data, inode, file, 0);
@@ -280,7 +272,8 @@ static int write_file(tar_header_decoded_t *hdr, file_info_t *fi,
 	if (ret)
 		return -1;
 
-	return skip_padding(STDIN_FILENO, filesize);
+	return skip_padding(STDIN_FILENO, hdr->sparse == NULL ?
+			    filesize : hdr->record_size);
 }
 
 static int copy_xattr(fstree_t *fs, tree_node_t *node,
@@ -344,7 +337,7 @@ static int process_tar_ball(fstree_t *fs, data_writer_t *data)
 {
 	tar_header_decoded_t hdr;
 	uint64_t offset, count;
-	sqfs_sparse_map_t *m;
+	sparse_map_t *m;
 	bool skip;
 	int ret;
 
diff --git a/tests/tar_sparse_gnu.c b/tests/tar_sparse_gnu.c
index e0a0200..7cd6132 100644
--- a/tests/tar_sparse_gnu.c
+++ b/tests/tar_sparse_gnu.c
@@ -36,7 +36,7 @@ static int open_read(const char *path)
 int main(void)
 {
 	tar_header_decoded_t hdr;
-	sqfs_sparse_map_t *sparse;
+	sparse_map_t *sparse;
 	int fd;
 
 	assert(chdir(TEST_PATH) == 0);
diff --git a/tests/tar_sparse_gnu1.c b/tests/tar_sparse_gnu1.c
index 0d8ab6b..366b0f4 100644
--- a/tests/tar_sparse_gnu1.c
+++ b/tests/tar_sparse_gnu1.c
@@ -36,7 +36,7 @@ static int open_read(const char *path)
 int main(void)
 {
 	tar_header_decoded_t hdr;
-	sqfs_sparse_map_t *sparse;
+	sparse_map_t *sparse;
 	int fd;
 
 	assert(chdir(TEST_PATH) == 0);
diff --git a/tests/tar_sparse_gnu2.c b/tests/tar_sparse_gnu2.c
index 568c402..cfc7eb3 100644
--- a/tests/tar_sparse_gnu2.c
+++ b/tests/tar_sparse_gnu2.c
@@ -36,7 +36,7 @@ static int open_read(const char *path)
 int main(void)
 {
 	tar_header_decoded_t hdr;
-	sqfs_sparse_map_t *sparse;
+	sparse_map_t *sparse;
 	int fd;
 
 	assert(chdir(TEST_PATH) == 0);