Remove remnants of the old data writer

 - Move the statstics hooks to the rest of the statistics code
 - Used the new data writer directly in gensquashfs & tar2sqfs
 - Demote what is left to a helper function for processing an
   input file and submitting it to the new data writer

Signed-off-by: David Oberhollenzer <david.oberhollenzer@sigma-star.at>

8 files changed, 142 insertions, 253 deletions

diff --git a/include/data_writer.h b/include/data_writer.h
deleted file mode 100644
index cdf7a44..0000000
--- a/include/data_writer.h
+++ /dev/null
@@ -1,93 +0,0 @@
-/* SPDX-License-Identifier: GPL-3.0-or-later */
-/*
- * data_writer.h
- *
- * Copyright (C) 2019 David Oberhollenzer <goliath@infraroot.at>
- */
-#ifndef DATA_WRITER_H
-#define DATA_WRITER_H
-
-#include "config.h"
-
-#include "sqfs/compress.h"
-#include "sqfs/data.h"
-#include "fstree.h"
-#include "util.h"
-
-typedef struct data_writer_t data_writer_t;
-
-typedef struct {
-	size_t file_count;
-	size_t blocks_written;
-	size_t frag_blocks_written;
-	size_t duplicate_blocks;
-	size_t sparse_blocks;
-	size_t frag_count;
-	size_t frag_dup;
-	uint64_t bytes_written;
-	uint64_t bytes_read;
-} data_writer_stats_t;
-
-enum {
-	/* Don't generate fragments, always write the last block to disk as a
-	   block, even if it is incomplete. */
-	DW_DONT_FRAGMENT = 0x01,
-
-	/* Intentionally write all blocks uncompressed. */
-	DW_DONT_COMPRESS = 0x02,
-
-	/* Make sure the first block of a file is alligned to
-	   device block size */
-	DW_ALLIGN_DEVBLK = 0x04,
-};
-
-/*
-  Create a data writer. The pointer to the super block is kept internally and
-  used to automatically update various counters when writing data.
-
-  Returns NULL on failure and prints errors to stderr.
- */
-data_writer_t *data_writer_create(sqfs_super_t *super, sqfs_compressor_t *cmp,
-				  sqfs_file_t *file, size_t devblksize,
-				  unsigned int num_jobs, size_t max_backlog);
-
-void data_writer_destroy(data_writer_t *data);
-
-/*
-  Write the finalfragment table to the underlying file.
-
-  Returns 0 on success, prints errors to stderr.
-*/
-int data_writer_write_fragment_table(data_writer_t *data);
-
-/*
-  Wait for everything to be written to disk. This also forces a currently
-  pending fragment block to be compressed and wrtten.
-
-  Returns 0 on success, prints errors to stderr.
-*/
-int data_writer_sync(data_writer_t *data);
-
-/*
-  Read data from the given file descriptor, partition it into blocks and
-  write them out (possibly compressed) to the underlying file. If the size
-  is not a multiple of the block size, the last bit is kept in an internal
-  fragment buffer which is written out if full.
-
-  The file_info_t object is updated accordingly and used to determine the
-  number of bytes to write and the input file name to report errors.
-
-  Blocks or fragments that are all zero bytes automatically detected,
-  not written out and the sparse file accounting updated accordingly.
-
-  The flags argument is a combination of DW_* flags. After completion the
-  data writer collects the 'fi' in an internal list it uses for deduplication.
-
-  Returns 0 on success, prints errors to stderr.
-*/
-int write_data_from_file(data_writer_t *data, sqfs_inode_generic_t *inode,
-			 sqfs_file_t *file, int flags);
-
-data_writer_stats_t *data_writer_get_stats(data_writer_t *data);
-
-#endif /* DATA_WRITER_H */
diff --git a/include/highlevel.h b/include/highlevel.h
index bfc2d91..5bc4a96 100644
--- a/include/highlevel.h
+++ b/include/highlevel.h
@@ -15,20 +15,47 @@
 #include "sqfs/data.h"
 #include "sqfs/table.h"
 #include "sqfs/meta_writer.h"
+#include "sqfs/data_reader.h"
+#include "sqfs/data_writer.h"
 #include "sqfs/dir_writer.h"
 #include "sqfs/dir_reader.h"
+#include "sqfs/block.h"
 #include "sqfs/xattr.h"
 #include "sqfs/dir.h"
 #include "sqfs/io.h"
-#include "sqfs/data_reader.h"
-#include "data_writer.h"
 #include "fstree.h"
+#include "util.h"
 #include "tar.h"
 
 #include <sys/stat.h>
 #include <stdint.h>
 #include <stddef.h>
 
+typedef struct {
+	size_t file_count;
+	size_t blocks_written;
+	size_t frag_blocks_written;
+	size_t duplicate_blocks;
+	size_t sparse_blocks;
+	size_t frag_count;
+	size_t frag_dup;
+	uint64_t bytes_written;
+	uint64_t bytes_read;
+} data_writer_stats_t;
+
+enum {
+	/* Don't generate fragments, always write the last block to disk as a
+	   block, even if it is incomplete. */
+	DW_DONT_FRAGMENT = 0x01,
+
+	/* Intentionally write all blocks uncompressed. */
+	DW_DONT_COMPRESS = 0x02,
+
+	/* Make sure the first block of a file is alligned to
+	   device block size */
+	DW_ALLIGN_DEVBLK = 0x04,
+};
+
 /*
   High level helper function to serialize an entire file system tree to
   a squashfs inode table and directory table.
@@ -86,4 +113,9 @@ int sqfs_data_reader_dump(sqfs_data_reader_t *data,
 
 sqfs_file_t *sqfs_get_stdin_file(const sparse_map_t *map, uint64_t size);
 
+void register_stat_hooks(sqfs_data_writer_t *data, data_writer_stats_t *stats);
+
+int write_data_from_file(sqfs_data_writer_t *data, sqfs_inode_generic_t *inode,
+			 sqfs_file_t *file, size_t block_size, int flags);
+
 #endif /* HIGHLEVEL_H */
diff --git a/lib/sqfshelper/Makemodule.am b/lib/sqfshelper/Makemodule.am
index 8a8deaa..896605a 100644
--- a/lib/sqfshelper/Makemodule.am
+++ b/lib/sqfshelper/Makemodule.am
@@ -6,7 +6,7 @@ libsqfshelper_a_SOURCES += lib/sqfshelper/print_version.c
 libsqfshelper_a_SOURCES += lib/sqfshelper/inode_stat.c
 libsqfshelper_a_SOURCES += lib/sqfshelper/data_reader_dump.c
 libsqfshelper_a_SOURCES += lib/sqfshelper/compress.c lib/sqfshelper/comp_opt.c
-libsqfshelper_a_SOURCES += include/data_writer.h lib/sqfshelper/data_writer.c
+libsqfshelper_a_SOURCES += lib/sqfshelper/data_writer.c
 libsqfshelper_a_SOURCES += lib/sqfshelper/write_xattr.c include/highlevel.h
 libsqfshelper_a_SOURCES += lib/sqfshelper/get_path.c lib/sqfshelper/io_stdin.c
 
diff --git a/lib/sqfshelper/data_writer.c b/lib/sqfshelper/data_writer.c
index a365e6d..5c01cb8 100644
--- a/lib/sqfshelper/data_writer.c
+++ b/lib/sqfshelper/data_writer.c
@@ -6,10 +6,6 @@
  */
 #include "config.h"
 
-#include "sqfs/data_writer.h"
-#include "sqfs/block.h"
-
-#include "data_writer.h"
 #include "highlevel.h"
 #include "util.h"
 
@@ -17,72 +13,14 @@
 #include <string.h>
 #include <unistd.h>
 #include <stdio.h>
-#include <errno.h>
-#include <zlib.h>
-
-struct data_writer_t {
-	sqfs_data_writer_t *proc;
-	sqfs_compressor_t *cmp;
-	sqfs_super_t *super;
-
-	data_writer_stats_t stats;
-};
-
-
-static void post_block_write(void *user, const sqfs_block_t *block,
-			     sqfs_file_t *file)
-{
-	data_writer_t *data = user;
-	(void)file;
-
-	if (block->flags & SQFS_BLK_FRAGMENT_BLOCK) {
-		data->stats.frag_blocks_written += 1;
-	} else {
-		data->stats.blocks_written += 1;
-	}
-
-	data->stats.bytes_written += block->size;
-}
-
-static void pre_fragment_store(void *user, sqfs_block_t *block)
-{
-	data_writer_t *data = user;
-	(void)block;
-
-	data->stats.frag_count += 1;
-}
-
-static void notify_blocks_erased(void *user, size_t count, uint64_t bytes)
-{
-	data_writer_t *data = user;
-	(void)bytes;
-
-	data->stats.bytes_written -= bytes;
-	data->stats.blocks_written -= count;
-	data->stats.duplicate_blocks += count;
-}
-
-static void notify_fragment_discard(void *user, const sqfs_block_t *block)
-{
-	data_writer_t *data = user;
-	(void)block;
-
-	data->stats.frag_dup += 1;
-}
-
-static const sqfs_block_hooks_t hooks = {
-	.post_block_write = post_block_write,
-	.pre_fragment_store = pre_fragment_store,
-	.notify_blocks_erased = notify_blocks_erased,
-	.notify_fragment_discard = notify_fragment_discard,
-};
 
 static bool is_zero_block(unsigned char *ptr, size_t size)
 {
 	return ptr[0] == 0 && memcmp(ptr, ptr + 1, size - 1) == 0;
 }
 
-static int add_sentinel_block(data_writer_t *data, sqfs_inode_generic_t *inode,
+static int add_sentinel_block(sqfs_data_writer_t *data,
+			      sqfs_inode_generic_t *inode,
 			      uint32_t flags)
 {
 	sqfs_block_t *blk = calloc(1, sizeof(*blk));
@@ -95,11 +33,11 @@ static int add_sentinel_block(data_writer_t *data, sqfs_inode_generic_t *inode,
 	blk->inode = inode;
 	blk->flags = flags;
 
-	return sqfs_data_writer_enqueue(data->proc, blk);
+	return sqfs_data_writer_enqueue(data, blk);
 }
 
-int write_data_from_file(data_writer_t *data, sqfs_inode_generic_t *inode,
-			 sqfs_file_t *file, int flags)
+int write_data_from_file(sqfs_data_writer_t *data, sqfs_inode_generic_t *inode,
+			 sqfs_file_t *file, size_t block_size, int flags)
 {
 	uint32_t blk_flags = SQFS_BLK_FIRST_BLOCK;
 	uint64_t filesz, offset;
@@ -116,8 +54,8 @@ int write_data_from_file(data_writer_t *data, sqfs_inode_generic_t *inode,
 	sqfs_inode_get_file_size(inode, &filesz);
 
 	for (offset = 0; offset < filesz; offset += diff) {
-		if (filesz - offset > (uint64_t)data->super->block_size) {
-			diff = data->super->block_size;
+		if (filesz - offset > (uint64_t)block_size) {
+			diff = block_size;
 		} else {
 			diff = filesz - offset;
 		}
@@ -131,8 +69,6 @@ int write_data_from_file(data_writer_t *data, sqfs_inode_generic_t *inode,
 		blk->index = i++;
 
 		if (is_zero_block(blk->data, blk->size)) {
-			data->stats.sparse_blocks += 1;
-
 			sqfs_inode_make_extended(inode);
 			inode->data.file_ext.sparse += blk->size;
 			inode->num_file_blocks += 1;
@@ -142,8 +78,7 @@ int write_data_from_file(data_writer_t *data, sqfs_inode_generic_t *inode,
 			continue;
 		}
 
-		if (diff < data->super->block_size &&
-		    !(flags & DW_DONT_FRAGMENT)) {
+		if (diff < block_size && !(flags & DW_DONT_FRAGMENT)) {
 			if (!(blk_flags & (SQFS_BLK_FIRST_BLOCK |
 					   SQFS_BLK_LAST_BLOCK))) {
 				blk_flags |= SQFS_BLK_LAST_BLOCK;
@@ -157,10 +92,10 @@ int write_data_from_file(data_writer_t *data, sqfs_inode_generic_t *inode,
 
 			blk->flags |= SQFS_BLK_IS_FRAGMENT;
 
-			if (sqfs_data_writer_enqueue(data->proc, blk))
+			if (sqfs_data_writer_enqueue(data, blk))
 				return -1;
 		} else {
-			if (sqfs_data_writer_enqueue(data->proc, blk))
+			if (sqfs_data_writer_enqueue(data, blk))
 				return -1;
 
 			blk_flags &= ~SQFS_BLK_FIRST_BLOCK;
@@ -174,62 +109,5 @@ int write_data_from_file(data_writer_t *data, sqfs_inode_generic_t *inode,
 		if (add_sentinel_block(data, inode, blk_flags))
 			return -1;
 	}
-
-	sqfs_inode_make_basic(inode);
-
-	data->stats.bytes_read += filesz;
-	data->stats.file_count += 1;
-	return 0;
-}
-
-data_writer_t *data_writer_create(sqfs_super_t *super, sqfs_compressor_t *cmp,
-				  sqfs_file_t *file, size_t devblksize,
-				  unsigned int num_jobs, size_t max_backlog)
-{
-	data_writer_t *data = calloc(1, sizeof(*data));
-
-	if (data == NULL) {
-		perror("creating data writer");
-		return NULL;
-	}
-
-	data->proc = sqfs_data_writer_create(super->block_size, cmp,
-					     num_jobs, max_backlog,
-					     devblksize, file);
-	if (data->proc == NULL) {
-		perror("creating data block processor");
-		free(data);
-		return NULL;
-	}
-
-	sqfs_data_writer_set_hooks(data->proc, data, &hooks);
-
-	data->cmp = cmp;
-	data->super = super;
-	return data;
-}
-
-void data_writer_destroy(data_writer_t *data)
-{
-	sqfs_data_writer_destroy(data->proc);
-	free(data);
-}
-
-int data_writer_write_fragment_table(data_writer_t *data)
-{
-	if (sqfs_data_writer_write_fragment_table(data->proc, data->super)) {
-		fputs("error storing fragment table\n", stderr);
-		return -1;
-	}
 	return 0;
 }
-
-int data_writer_sync(data_writer_t *data)
-{
-	return sqfs_data_writer_finish(data->proc);
-}
-
-data_writer_stats_t *data_writer_get_stats(data_writer_t *data)
-{
-	return &data->stats;
-}
diff --git a/lib/sqfshelper/statistics.c b/lib/sqfshelper/statistics.c
index 87efc30..26c121f 100644
--- a/lib/sqfshelper/statistics.c
+++ b/lib/sqfshelper/statistics.c
@@ -7,8 +7,65 @@
 #include "config.h"
 #include "highlevel.h"
 
+#include "sqfs/block.h"
+
 #include <stdio.h>
 
+static void post_block_write(void *user, const sqfs_block_t *block,
+			     sqfs_file_t *file)
+{
+	data_writer_stats_t *stats = user;
+	(void)file;
+
+	if (block->size == 0)
+		return;
+
+	if (block->flags & SQFS_BLK_FRAGMENT_BLOCK) {
+		stats->frag_blocks_written += 1;
+	} else {
+		stats->blocks_written += 1;
+	}
+
+	stats->bytes_written += block->size;
+}
+
+static void pre_fragment_store(void *user, sqfs_block_t *block)
+{
+	data_writer_stats_t *stats = user;
+	(void)block;
+
+	stats->frag_count += 1;
+}
+
+static void notify_blocks_erased(void *user, size_t count, uint64_t bytes)
+{
+	data_writer_stats_t *stats = user;
+
+	stats->bytes_written -= bytes;
+	stats->blocks_written -= count;
+	stats->duplicate_blocks += count;
+}
+
+static void notify_fragment_discard(void *user, const sqfs_block_t *block)
+{
+	data_writer_stats_t *stats = user;
+	(void)block;
+
+	stats->frag_dup += 1;
+}
+
+static const sqfs_block_hooks_t hooks = {
+	.post_block_write = post_block_write,
+	.pre_fragment_store = pre_fragment_store,
+	.notify_blocks_erased = notify_blocks_erased,
+	.notify_fragment_discard = notify_fragment_discard,
+};
+
+void register_stat_hooks(sqfs_data_writer_t *data, data_writer_stats_t *stats)
+{
+	sqfs_data_writer_set_hooks(data, stats, &hooks);
+}
+
 void sqfs_print_statistics(sqfs_super_t *super, data_writer_stats_t *stats)
 {
 	size_t ratio;
diff --git a/mkfs/mkfs.c b/mkfs/mkfs.c
index acb3202..71e1863 100644
--- a/mkfs/mkfs.c
+++ b/mkfs/mkfs.c
@@ -28,7 +28,8 @@ static int restore_working_dir(options_t *opt)
 	return 0;
 }
 
-static int pack_files(data_writer_t *data, fstree_t *fs, options_t *opt)
+static int pack_files(sqfs_data_writer_t *data, fstree_t *fs,
+		      data_writer_stats_t *stats, options_t *opt)
 {
 	sqfs_inode_generic_t *inode;
 	size_t max_blk_count;
@@ -72,14 +73,18 @@ static int pack_files(data_writer_t *data, fstree_t *fs, options_t *opt)
 
 		fi->user_ptr = inode;
 
-		ret = write_data_from_file(data, inode, file, 0);
+		ret = write_data_from_file(data, inode, file,
+					   fs->block_size, 0);
 		file->destroy(file);
 
 		if (ret)
 			return -1;
+
+		stats->file_count += 1;
+		stats->bytes_read += filesize;
 	}
 
-	if (data_writer_sync(data))
+	if (sqfs_data_writer_finish(data))
 		return -1;
 
 	return restore_working_dir(opt);
@@ -118,10 +123,11 @@ int main(int argc, char **argv)
 {
 	int status = EXIT_FAILURE, ret;
 	sqfs_compressor_config_t cfg;
+	data_writer_stats_t stats;
+	sqfs_data_writer_t *data;
 	sqfs_compressor_t *cmp;
 	sqfs_id_table_t *idtbl;
 	sqfs_file_t *outfile;
-	data_writer_t *data;
 	sqfs_super_t super;
 	options_t opt;
 	fstree_t fs;
@@ -188,18 +194,21 @@ int main(int argc, char **argv)
 	if (ret > 0)
 		super.flags |= SQFS_FLAG_COMPRESSOR_OPTIONS;
 
-	data = data_writer_create(&super, cmp, outfile,
-				  opt.devblksz, opt.num_jobs, opt.max_backlog);
+	data = sqfs_data_writer_create(super.block_size, cmp, opt.num_jobs,
+				       opt.max_backlog, opt.devblksz, outfile);
 	if (data == NULL)
 		goto out_cmp;
 
-	if (pack_files(data, &fs, &opt))
+	memset(&stats, 0, sizeof(stats));
+	register_stat_hooks(data, &stats);
+
+	if (pack_files(data, &fs, &stats, &opt))
 		goto out_data;
 
 	if (sqfs_serialize_fstree(outfile, &super, &fs, cmp, idtbl))
 		goto out_data;
 
-	if (data_writer_write_fragment_table(data))
+	if (sqfs_data_writer_write_fragment_table(data, &super))
 		goto out_data;
 
 	if (opt.exportable) {
@@ -222,11 +231,11 @@ int main(int argc, char **argv)
 		goto out_data;
 
 	if (!opt.quiet)
-		sqfs_print_statistics(&super, data_writer_get_stats(data));
+		sqfs_print_statistics(&super, &stats);
 
 	status = EXIT_SUCCESS;
 out_data:
-	data_writer_destroy(data);
+	sqfs_data_writer_destroy(data);
 out_cmp:
 	cmp->destroy(cmp);
 out_outfile:
diff --git a/mkfs/mkfs.h b/mkfs/mkfs.h
index 9d34d02..bbe5572 100644
--- a/mkfs/mkfs.h
+++ b/mkfs/mkfs.h
@@ -15,7 +15,6 @@
 #include "sqfs/data.h"
 #include "sqfs/io.h"
 
-#include "data_writer.h"
 #include "highlevel.h"
 #include "fstree.h"
 #include "util.h"
diff --git a/tar/tar2sqfs.c b/tar/tar2sqfs.c
index 82720a8..4770810 100644
--- a/tar/tar2sqfs.c
+++ b/tar/tar2sqfs.c
@@ -12,7 +12,6 @@
 #include "sqfs/data.h"
 #include "sqfs/io.h"
 
-#include "data_writer.h"
 #include "highlevel.h"
 #include "fstree.h"
 #include "util.h"
@@ -111,6 +110,8 @@ static bool dont_skip = false;
 static bool no_xattr = false;
 static bool exportable = false;
 static bool keep_time = false;
+static data_writer_stats_t stats;
+static sqfs_super_t super;
 
 static void process_args(int argc, char **argv)
 {
@@ -223,7 +224,7 @@ fail_arg:
 }
 
 static int write_file(tar_header_decoded_t *hdr, file_info_t *fi,
-		      data_writer_t *data, uint64_t filesize)
+		      sqfs_data_writer_t *data, uint64_t filesize)
 {
 	const sparse_map_t *it;
 	sqfs_inode_generic_t *inode;
@@ -266,9 +267,12 @@ static int write_file(tar_header_decoded_t *hdr, file_info_t *fi,
 		}
 	}
 
-	ret = write_data_from_file(data, inode, file, 0);
+	ret = write_data_from_file(data, inode, file, super.block_size, 0);
 	file->destroy(file);
 
+	stats.bytes_read += filesize;
+	stats.file_count += 1;
+
 	if (ret)
 		return -1;
 
@@ -302,7 +306,7 @@ static int copy_xattr(fstree_t *fs, tree_node_t *node,
 }
 
 static int create_node_and_repack_data(tar_header_decoded_t *hdr, fstree_t *fs,
-				       data_writer_t *data)
+				       sqfs_data_writer_t *data)
 {
 	tree_node_t *node;
 
@@ -333,7 +337,7 @@ fail_errno:
 	return -1;
 }
 
-static int process_tar_ball(fstree_t *fs, data_writer_t *data)
+static int process_tar_ball(fstree_t *fs, sqfs_data_writer_t *data)
 {
 	tar_header_decoded_t hdr;
 	uint64_t offset, count;
@@ -407,11 +411,10 @@ int main(int argc, char **argv)
 {
 	sqfs_compressor_config_t cfg;
 	int status = EXIT_SUCCESS;
+	sqfs_data_writer_t *data;
 	sqfs_compressor_t *cmp;
 	sqfs_id_table_t *idtbl;
 	sqfs_file_t *outfile;
-	data_writer_t *data;
-	sqfs_super_t super;
 	fstree_t fs;
 	int ret;
 
@@ -450,10 +453,14 @@ int main(int argc, char **argv)
 	if (ret > 0)
 		super.flags |= SQFS_FLAG_COMPRESSOR_OPTIONS;
 
-	data = data_writer_create(&super, cmp, outfile, devblksize,
-				  num_jobs, max_backlog);
-	if (data == NULL)
+	data = sqfs_data_writer_create(super.block_size, cmp, num_jobs,
+				       max_backlog, devblksize, outfile);
+	if (data == NULL) {
+		perror("creating data block processor");
 		goto out_cmp;
+	}
+
+	register_stat_hooks(data, &stats);
 
 	idtbl = sqfs_id_table_create();
 	if (idtbl == NULL)
@@ -462,7 +469,7 @@ int main(int argc, char **argv)
 	if (process_tar_ball(&fs, data))
 		goto out;
 
-	if (data_writer_sync(data))
+	if (sqfs_data_writer_finish(data))
 		goto out;
 
 	tree_node_sort_recursive(fs.root);
@@ -476,7 +483,7 @@ int main(int argc, char **argv)
 	if (sqfs_serialize_fstree(outfile, &super, &fs, cmp, idtbl))
 		goto out;
 
-	if (data_writer_write_fragment_table(data))
+	if (sqfs_data_writer_write_fragment_table(data, &super))
 		goto out;
 
 	if (exportable) {
@@ -499,13 +506,13 @@ int main(int argc, char **argv)
 		goto out;
 
 	if (!quiet)
-		sqfs_print_statistics(&super, data_writer_get_stats(data));
+		sqfs_print_statistics(&super, &stats);
 
 	status = EXIT_SUCCESS;
 out:
 	sqfs_id_table_destroy(idtbl);
 out_data:
-	data_writer_destroy(data);
+	sqfs_data_writer_destroy(data);
 out_cmp:
 	cmp->destroy(cmp);
 out_fs: