libio: remove ostream_append_sparse and fallback implementation

Instead of a separate append-sparse function, simply accept NULL
as an input for append. For both Unix and Win32, a fallback needs
to be implemented. For XFRM, we can just memset the input buffer
to zero, same for the libsquashfs data writer.

Signed-off-by: David Oberhollenzer <david.oberhollenzer@sigma-star.at>

13 files changed, 167 insertions, 200 deletions

diff --git a/include/io/file.h b/include/io/file.h
index f3723b2..44fd224 100644
--- a/include/io/file.h
+++ b/include/io/file.h
@@ -18,6 +18,11 @@ typedef HANDLE os_file_t;
 typedef int os_file_t;
 #endif
 
+enum {
+	OSTREAM_OPEN_OVERWRITE = 0x01,
+	OSTREAM_OPEN_SPARSE = 0x02,
+};
+
 #ifdef __cplusplus
 extern "C" {
 #endif
diff --git a/include/io/ostream.h b/include/io/ostream.h
index 8fc8cef..7a57e48 100644
--- a/include/io/ostream.h
+++ b/include/io/ostream.h
@@ -23,15 +23,14 @@ typedef struct ostream_t {
 	 * @brief Append a block of data to an output stream.
 	 *
 	 * @param strm A pointer to an output stream.
-	 * @param data A pointer to the data block to append.
+	 * @param data A pointer to the data block to append. If NULL,
+	 *             synthesize a chunk of zero bytes.
 	 * @param size The number of bytes to append.
 	 *
 	 * @return Zero on success, -1 on failure.
 	 */
 	int (*append)(struct ostream_t *strm, const void *data, size_t size);
 
-	int (*append_sparse)(struct ostream_t *strm, size_t size);
-
 	/**
 	 * @brief Process all pending, buffered data and flush it to disk.
 	 *
@@ -56,33 +55,4 @@ typedef struct ostream_t {
 	const char *(*get_filename)(struct ostream_t *strm);
 } ostream_t;
 
-enum {
-	OSTREAM_OPEN_OVERWRITE = 0x01,
-	OSTREAM_OPEN_SPARSE = 0x02,
-};
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-/**
- * @brief Append a number of zero bytes to an output stream.
- *
- * @memberof ostream_t
- *
- * If the unerlying implementation supports sparse files, this function can be
- * used to create a "hole". If the implementation does not support it, a
- * fallback is used that just appends a block of zeros manualy.
- *
- * @param strm A pointer to an output stream.
- * @param size The number of zero bytes to append.
- *
- * @return Zero on success, -1 on failure.
- */
-SQFS_INTERNAL int ostream_append_sparse(ostream_t *strm, size_t size);
-
-#ifdef __cplusplus
-}
-#endif
-
 #endif /* IO_OSTREAM_H */
diff --git a/lib/common/src/data_reader_dump.c b/lib/common/src/data_reader_dump.c
index 29043f8..8f38bd2 100644
--- a/lib/common/src/data_reader_dump.c
+++ b/lib/common/src/data_reader_dump.c
@@ -26,7 +26,7 @@ int sqfs_data_reader_dump(const char *name, sqfs_data_reader_t *data,
 		diff = (filesz < block_size) ? filesz : block_size;
 
 		if (SQFS_IS_SPARSE_BLOCK(inode->extra[i])) {
-			if (ostream_append_sparse(fp, diff))
+			if (fp->append(fp, NULL, diff))
 				return -1;
 		} else {
 			err = sqfs_data_reader_get_block(data, inode, i,
diff --git a/lib/io/Makemodule.am b/lib/io/Makemodule.am
index 5ec1f7e..badaa99 100644
--- a/lib/io/Makemodule.am
+++ b/lib/io/Makemodule.am
@@ -1,7 +1,6 @@
 libio_a_SOURCES = include/io/istream.h include/io/ostream.h include/io/xfrm.h \
 	include/io/file.h include/io/std.h include/io/dir_entry.h \
-	include/io/dir_iterator.h include/io/mem.h \
-	lib/io/src/internal.h lib/io/src/ostream.c \
+	include/io/dir_iterator.h include/io/mem.h lib/io/src/internal.h \
 	lib/io/src/istream.c lib/io/src/get_line.c lib/io/src/xfrm/ostream.c \
 	lib/io/src/xfrm/istream.c lib/io/src/dir_tree_iterator.c \
 	lib/io/src/dir_entry.c lib/io/src/mem.c
@@ -19,7 +18,7 @@ endif
 
 noinst_LIBRARIES += libio.a
 
-LIBIO_TESTS = test_istream_mem test_get_line test_sparse_fb test_istream_read \
+LIBIO_TESTS = test_istream_mem test_get_line test_istream_read \
 	test_istream_skip test_stream_splice test_dir_iterator \
 	test_dir_tree_iterator test_dir_tree_iterator2 test_dir_tree_iterator3
 
@@ -31,9 +30,6 @@ test_get_line_SOURCES = lib/io/test/get_line.c
 test_get_line_LDADD = libio.a libcompat.a
 test_get_line_CPPFLAGS = $(AM_CPPFLAGS)
 
-test_sparse_fb_SOURCES = lib/io/test/sparse_fb.c
-test_sparse_fb_LDADD = libio.a libutil.a libcompat.a
-
 test_istream_read_SOURCES = lib/io/test/istream_read.c
 test_istream_read_LDADD = libio.a libutil.a libcompat.a
 
diff --git a/lib/io/src/ostream.c b/lib/io/src/ostream.c
deleted file mode 100644
index 35f98d4..0000000
--- a/lib/io/src/ostream.c
+++ /dev/null
@@ -1,35 +0,0 @@
-/* SPDX-License-Identifier: GPL-3.0-or-later */
-/*
- * ostream.c
- *
- * Copyright (C) 2019 David Oberhollenzer <goliath@infraroot.at>
- */
-#include "internal.h"
-
-
-static int append_sparse_fallback(ostream_t *strm, size_t size)
-{
-	char buffer[512];
-	size_t diff;
-
-	memset(buffer, 0, sizeof(buffer));
-
-	while (size > 0) {
-		diff = size < sizeof(buffer) ? size : sizeof(buffer);
-
-		if (strm->append(strm, buffer, diff))
-			return -1;
-
-		size -= diff;
-	}
-
-	return 0;
-}
-
-int ostream_append_sparse(ostream_t *strm, size_t size)
-{
-	if (strm->append_sparse == NULL)
-		return append_sparse_fallback(strm, size);
-
-	return strm->append_sparse(strm, size);
-}
diff --git a/lib/io/src/unix/ostream.c b/lib/io/src/unix/ostream.c
index 294a15e..702a354 100644
--- a/lib/io/src/unix/ostream.c
+++ b/lib/io/src/unix/ostream.c
@@ -9,81 +9,115 @@
 typedef struct {
 	ostream_t base;
 	char *path;
+	int flags;
 	int fd;
 
 	off_t sparse_count;
 	off_t size;
 } file_ostream_t;
 
-static int file_append(ostream_t *strm, const void *data, size_t size)
+static int write_all(file_ostream_t *file, const sqfs_u8 *data, size_t size)
 {
-	file_ostream_t *file = (file_ostream_t *)strm;
-	ssize_t ret;
-
-	if (size == 0)
-		return 0;
-
-	if (file->sparse_count > 0) {
-		if (lseek(file->fd, file->sparse_count, SEEK_CUR) == (off_t)-1)
-			goto fail_errno;
-
-		file->sparse_count = 0;
-	}
-
 	while (size > 0) {
-		ret = write(file->fd, data, size);
+		ssize_t ret = write(file->fd, data, size);
 
 		if (ret == 0) {
-			fprintf(stderr, "%s: truncated data write.\n",
-				file->path);
+			fprintf(stderr, "%s: truncated write.\n", file->path);
 			return -1;
 		}
 
 		if (ret < 0) {
 			if (errno == EINTR)
 				continue;
-			goto fail_errno;
+			perror(file->path);
+			return -1;
 		}
 
 		file->size += ret;
 		size -= ret;
-		data = (const char *)data + ret;
+		data += ret;
+	}
+
+	return 0;
+}
+
+static int realize_sparse(file_ostream_t *file)
+{
+	unsigned char *buffer;
+	size_t diff, bufsz;
+
+	if (file->sparse_count == 0)
+		return 0;
+
+	if (file->flags & OSTREAM_OPEN_SPARSE) {
+		if (lseek(file->fd, file->sparse_count, SEEK_CUR) == (off_t)-1)
+			goto fail;
+
+		if (ftruncate(file->fd, file->size) != 0)
+			goto fail;
+
+		file->sparse_count = 0;
+	} else {
+		bufsz = file->sparse_count > 1024 ? 1024 : file->sparse_count;
+		buffer = calloc(1, bufsz);
+		if (buffer == NULL)
+			goto fail;
+
+		while (file->sparse_count > 0) {
+			diff = file->sparse_count > (off_t)bufsz ?
+				bufsz : (size_t)file->sparse_count;
+
+			if (write_all(file, buffer, diff)) {
+				free(buffer);
+				return -1;
+			}
+
+			file->sparse_count -= diff;
+		}
+
+		free(buffer);
 	}
 
 	return 0;
-fail_errno:
+fail:
 	perror(file->path);
 	return -1;
 }
 
-static int file_append_sparse(ostream_t *strm, size_t size)
+static int file_append(ostream_t *strm, const void *data, size_t size)
 {
 	file_ostream_t *file = (file_ostream_t *)strm;
 
-	file->sparse_count += size;
-	file->size += size;
-	return 0;
+	if (size == 0)
+		return 0;
+
+	if (data == NULL) {
+		file->sparse_count += size;
+		file->size += size;
+		return 0;
+	}
+
+	if (realize_sparse(file))
+		return -1;
+
+	return write_all(file, data, size);
 }
 
 static int file_flush(ostream_t *strm)
 {
 	file_ostream_t *file = (file_ostream_t *)strm;
 
-	if (file->sparse_count > 0) {
-		if (ftruncate(file->fd, file->size) != 0)
-			goto fail;
-	}
+	if (realize_sparse(file))
+		return -1;
 
 	if (fsync(file->fd) != 0) {
 		if (errno == EINVAL)
 			return 0;
-		goto fail;
+		perror(file->path);
+		return -1;
 	}
 
 	return 0;
-fail:
-	perror(file->path);
-	return -1;
 }
 
 static void file_destroy(sqfs_object_t *obj)
@@ -128,9 +162,7 @@ ostream_t *ostream_open_handle(const char *path, int fd, int flags)
 
 	close(fd);
 
-	if (flags & OSTREAM_OPEN_SPARSE)
-		strm->append_sparse = file_append_sparse;
-
+	file->flags = flags;
 	strm->append = file_append;
 	strm->flush = file_flush;
 	strm->get_filename = file_get_filename;
diff --git a/lib/io/src/win32/ostream.c b/lib/io/src/win32/ostream.c
index d18130f..9b488d8 100644
--- a/lib/io/src/win32/ostream.c
+++ b/lib/io/src/win32/ostream.c
@@ -11,13 +11,14 @@
 
 typedef struct {
 	ostream_t base;
+	sqfs_u64 sparse_count;
 	char *path;
 	HANDLE hnd;
+	int flags;
 } file_ostream_t;
 
-static int file_append(ostream_t *strm, const void *data, size_t size)
+static int write_data(file_ostream_t *file, const void *data, size_t size)
 {
-	file_ostream_t *file = (file_ostream_t *)strm;
 	DWORD diff;
 
 	while (size > 0) {
@@ -33,18 +34,48 @@ static int file_append(ostream_t *strm, const void *data, size_t size)
 	return 0;
 }
 
-static int file_append_sparse(ostream_t *strm, size_t size)
+static int realize_sparse(file_ostream_t *file)
 {
-	file_ostream_t *file = (file_ostream_t *)strm;
+	size_t bufsz, diff;
 	LARGE_INTEGER pos;
+	void *buffer;
+
+	if (file->sparse_count == 0)
+		return 0;
 
-	pos.QuadPart = size;
+	if (file->flags & OSTREAM_OPEN_SPARSE) {
+		pos.QuadPart = file->sparse_count;
 
-	if (!SetFilePointerEx(file->hnd, pos, NULL, FILE_CURRENT))
-		goto fail;
+		if (!SetFilePointerEx(file->hnd, pos, NULL, FILE_CURRENT))
+			goto fail;
 
-	if (!SetEndOfFile(file->hnd))
-		goto fail;
+		if (!SetEndOfFile(file->hnd))
+			goto fail;
+
+		file->sparse_count = 0;
+	} else {
+		bufsz = file->sparse_count > 1024 ? 1024 : file->sparse_count;
+		buffer = calloc(1, bufsz);
+
+		if (buffer == NULL) {
+			fputs("out-of-memory\n", stderr);
+			return -1;
+		}
+
+		while (file->sparse_count > 0) {
+			diff = file->sparse_count > bufsz ?
+				bufsz : file->sparse_count;
+
+			if (write_data(file, buffer, diff)) {
+				free(buffer);
+				return -1;
+			}
+
+			file->sparse_count -= diff;
+		}
+
+		free(buffer);
+	}
 
 	return 0;
 fail:
@@ -52,10 +83,31 @@ fail:
 	return -1;
 }
 
+static int file_append(ostream_t *strm, const void *data, size_t size)
+{
+	file_ostream_t *file = (file_ostream_t *)strm;
+
+	if (size == 0)
+		return 0;
+
+	if (data == NULL) {
+		file->sparse_count += size;
+		return 0;
+	}
+
+	if (realize_sparse(file))
+		return -1;
+
+	return write_data(file, data, size);
+}
+
 static int file_flush(ostream_t *strm)
 {
 	file_ostream_t *file = (file_ostream_t *)strm;
 
+	if (realize_sparse(file))
+		return -1;
+
 	if (!FlushFileBuffers(file->hnd)) {
 		w32_perror(file->path);
 		return -1;
@@ -109,9 +161,7 @@ ostream_t *ostream_open_handle(const char *path, HANDLE hnd, int flags)
 
 	CloseHandle(hnd);
 
-	if (flags & OSTREAM_OPEN_SPARSE)
-		strm->append_sparse = file_append_sparse;
-
+	file->flags = flags;
 	strm->append = file_append;
 	strm->flush = file_flush;
 	strm->get_filename = file_get_filename;
diff --git a/lib/io/src/xfrm/ostream.c b/lib/io/src/xfrm/ostream.c
index 79ed49a..e55e38c 100644
--- a/lib/io/src/xfrm/ostream.c
+++ b/lib/io/src/xfrm/ostream.c
@@ -82,10 +82,14 @@ static int xfrm_append(ostream_t *strm, const void *data, size_t size)
 		if (diff > size)
 			diff = size;
 
-		memcpy(xfrm->inbuf + xfrm->inbuf_used, data, diff);
+		if (data == NULL) {
+			memset(xfrm->inbuf + xfrm->inbuf_used, 0, diff);
+		} else {
+			memcpy(xfrm->inbuf + xfrm->inbuf_used, data, diff);
+			data = (const char *)data + diff;
+		}
 
 		xfrm->inbuf_used += diff;
-		data = (const char *)data + diff;
 		size -= diff;
 	}
 
diff --git a/lib/io/test/sparse_fb.c b/lib/io/test/sparse_fb.c
deleted file mode 100644
index 5ee6004..0000000
--- a/lib/io/test/sparse_fb.c
+++ /dev/null
@@ -1,66 +0,0 @@
-/* SPDX-License-Identifier: GPL-3.0-or-later */
-/*
- * get_line.c
- *
- * Copyright (C) 2019 David Oberhollenzer <goliath@infraroot.at>
- */
-#include "config.h"
-
-#include "io/ostream.h"
-#include "util/test.h"
-#include "util/util.h"
-
-static ostream_t dummy;
-static size_t total = 0;
-
-static int dummy_append(ostream_t *strm, const void *data, size_t size)
-{
-	bool bret;
-
-	TEST_ASSERT(strm == &dummy);
-	TEST_NOT_NULL(data);
-	TEST_ASSERT(size > 0);
-
-	bret = is_memory_zero(data, size);
-	TEST_ASSERT(bret);
-
-	bret = SZ_ADD_OV(total, size, &total);
-	TEST_ASSERT(!bret);
-	return 0;
-}
-
-static int dummy_flush(ostream_t *strm)
-{
-	TEST_ASSERT(strm == &dummy);
-	return 0;
-}
-
-static ostream_t dummy = {
-	{
-		1,
-		NULL,
-		NULL,
-	},
-	dummy_append,
-	NULL,
-	dummy_flush,
-	NULL,
-};
-
-int main(int argc, char **argv)
-{
-	size_t ref;
-	int ret;
-	(void)argc; (void)argv;
-
-	ref = 131072 + 1337;
-
-	ret = ostream_append_sparse(&dummy, ref);
-	TEST_EQUAL_I(ret, 0);
-
-	ret = dummy.flush(&dummy);
-	TEST_EQUAL_I(ret, 0);
-
-	TEST_EQUAL_UI(ref, total);
-	return EXIT_SUCCESS;
-}
diff --git a/lib/io/test/xfrm.c b/lib/io/test/xfrm.c
index c2e450c..184db9e 100644
--- a/lib/io/test/xfrm.c
+++ b/lib/io/test/xfrm.c
@@ -374,7 +374,6 @@ static int mem_flush(ostream_t *strm);
 static ostream_t mem_ostream = {
 	{ 1, NULL, NULL, },
 	mem_append,
-	NULL,
 	mem_flush,
 	NULL,
 };
@@ -382,15 +381,18 @@ static ostream_t mem_ostream = {
 static int mem_append(ostream_t *strm, const void *data, size_t size)
 {
 	TEST_ASSERT(strm == &mem_ostream);
-	TEST_NOT_NULL(data);
 	TEST_ASSERT(size > 0);
 
 	TEST_ASSERT(mo_written <= sizeof(mo_buffer));
 	TEST_ASSERT(size <= (sizeof(mo_buffer) - mo_written));
 
-	memcpy(mo_buffer + mo_written, data, size);
-	mo_written += size;
+	if (data == NULL) {
+		memset(mo_buffer + mo_written, 0, size);
+	} else {
+		memcpy(mo_buffer + mo_written, data, size);
+	}
 
+	mo_written += size;
 	return 0;
 }
 
diff --git a/lib/sqfs/src/block_processor/frontend.c b/lib/sqfs/src/block_processor/frontend.c
index e8a4207..de89913 100644
--- a/lib/sqfs/src/block_processor/frontend.c
+++ b/lib/sqfs/src/block_processor/frontend.c
@@ -114,6 +114,7 @@ int sqfs_block_processor_append(sqfs_block_processor_t *proc, const void *data,
 	sqfs_block_t *new;
 	sqfs_u64 filesize;
 	size_t diff;
+	void *dst;
 	int err;
 
 	if (!proc->begin_called)
@@ -152,12 +153,17 @@ int sqfs_block_processor_append(sqfs_block_processor_t *proc, const void *data,
 		if (diff > size)
 			diff = size;
 
-		memcpy(proc->blk_current->data + proc->blk_current->size,
-		       data, diff);
+		dst = proc->blk_current->data + proc->blk_current->size;
+
+		if (data == NULL) {
+			memset(dst, 0, diff);
+		} else {
+			memcpy(dst, data, diff);
+			data = (const char *)data + diff;
+		}
 
 		size -= diff;
 		proc->blk_current->size += diff;
-		data = (const char *)data + diff;
 
 		proc->stats.input_bytes_read += diff;
 	}
diff --git a/lib/tar/src/padd_file.c b/lib/tar/src/padd_file.c
index 053ff1e..0f6ee0a 100644
--- a/lib/tar/src/padd_file.c
+++ b/lib/tar/src/padd_file.c
@@ -15,5 +15,5 @@ int padd_file(ostream_t *fp, sqfs_u64 size)
 	if (padd_sz == 0)
 		return 0;
 
-	return ostream_append_sparse(fp, TAR_RECORD_SIZE - padd_sz);
+	return fp->append(fp, NULL, TAR_RECORD_SIZE - padd_sz);
 }
diff --git a/lib/tar/test/tar_write_simple.c b/lib/tar/test/tar_write_simple.c
index 2302307..b34c9c5 100644
--- a/lib/tar/test/tar_write_simple.c
+++ b/lib/tar/test/tar_write_simple.c
@@ -21,7 +21,6 @@ static ostream_t mem_stream = {
 	{ 1, NULL, NULL },
 	buffer_append,
 	NULL,
-	NULL,
 	buffer_get_filename,
 };
 
@@ -33,12 +32,16 @@ static sqfs_u8 rd_buffer[1024 * 10];
 static int buffer_append(ostream_t *strm, const void *data, size_t size)
 {
 	TEST_ASSERT(strm == &mem_stream);
-	TEST_NOT_NULL(data);
 	TEST_ASSERT(wr_offset < sizeof(wr_buffer));
 	TEST_ASSERT(size > 0);
 	TEST_ASSERT((sizeof(wr_buffer) - wr_offset) >= size);
 
-	memcpy(wr_buffer + wr_offset, data, size);
+	if (data == NULL) {
+		memset(wr_buffer + wr_offset, 0, size);
+	} else {
+		memcpy(wr_buffer + wr_offset, data, size);
+	}
+
 	wr_offset += size;
 	return 0;
 }