Remove fstree file flags

As a side effect, this requires the data writer to keep track of
statistics.

Signed-off-by: David Oberhollenzer <david.oberhollenzer@sigma-star.at>

12 files changed, 89 insertions, 110 deletions

diff --git a/include/data_writer.h b/include/data_writer.h
index 9e82d11..5987cd6 100644
--- a/include/data_writer.h
+++ b/include/data_writer.h
@@ -16,6 +16,17 @@
 
 typedef struct data_writer_t data_writer_t;
 
+typedef struct {
+	size_t file_count;
+	size_t blocks_written;
+	size_t frag_blocks_written;
+	size_t duplicate_blocks;
+	size_t sparse_blocks;
+	size_t frag_count;
+	size_t frag_dup;
+	uint64_t bytes_read;
+} data_writer_stats_t;
+
 enum {
 	/* Don't generate fragments, always write the last block to disk as a
 	   block, even if it is incomplete. */
@@ -89,4 +100,6 @@ int write_data_from_fd(data_writer_t *data, file_info_t *fi, int infd,
 int write_data_from_fd_condensed(data_writer_t *data, file_info_t *fi,
 				 int infd, sparse_map_t *map, int flags);
 
+data_writer_stats_t *data_writer_get_stats(data_writer_t *data);
+
 #endif /* DATA_WRITER_H */
diff --git a/include/fstree.h b/include/fstree.h
index 8d0b952..4c02ce5 100644
--- a/include/fstree.h
+++ b/include/fstree.h
@@ -28,14 +28,6 @@ typedef struct fstree_t fstree_t;
 typedef struct tree_xattr_t tree_xattr_t;
 
 enum {
-	FILE_FLAG_HAS_FRAGMENT = 0x01,
-
-	FILE_FLAG_FRAGMENT_IS_DUPLICATE = 0x02,
-
-	FILE_FLAG_BLOCKS_ARE_DUPLICATE = 0x04,
-};
-
-enum {
 	DIR_SCAN_KEEP_TIME = 0x01,
 
 	DIR_SCAN_ONE_FILESYSTEM = 0x02,
@@ -99,9 +91,6 @@ struct file_info_t {
 	/* Byte offset into the fragment block. */
 	uint32_t fragment_offset;
 
-	/* combination of FILE_FLAG_* flags */
-	uint32_t flags;
-
 	/* Stores data about each full data block. */
 	/* Bit (1 << 24) is set if the block is stored uncompressed. */
 	uint32_t block_size[];
diff --git a/include/highlevel.h b/include/highlevel.h
index 4c560f7..8173d6f 100644
--- a/include/highlevel.h
+++ b/include/highlevel.h
@@ -19,6 +19,7 @@
 #include "sqfs/dir.h"
 #include "sqfs/io.h"
 #include "data_reader.h"
+#include "data_writer.h"
 #include "fstree.h"
 
 #include <stdint.h>
@@ -63,8 +64,7 @@ int sqfs_serialize_fstree(sqfs_file_t *file, sqfs_super_t *super, fstree_t *fs,
  */
 tree_node_t *tree_node_from_inode(sqfs_inode_generic_t *inode,
 				  const sqfs_id_table_t *idtbl,
-				  const char *name,
-				  size_t block_size);
+				  const char *name);
 
 /*
   Restore a file system tree from a squashfs image. The given flags are a
@@ -92,7 +92,7 @@ int write_export_table(sqfs_file_t *file, fstree_t *fs, sqfs_super_t *super,
 		       sqfs_compressor_t *cmp);
 
 /* Print out fancy statistics for squashfs packing tools */
-void sqfs_print_statistics(fstree_t *fs, sqfs_super_t *super);
+void sqfs_print_statistics(sqfs_super_t *super, data_writer_stats_t *stats);
 
 /* Open a squashfs file, extract all the information we may need and
    construct datastructures we need to access its contents.
diff --git a/lib/fstree/optimize_unpack_order.c b/lib/fstree/optimize_unpack_order.c
index ad036e8..51576bd 100644
--- a/lib/fstree/optimize_unpack_order.c
+++ b/lib/fstree/optimize_unpack_order.c
@@ -7,6 +7,15 @@
 #include "config.h"
 #include "fstree.h"
 
+static bool has_fragment(const fstree_t *fs, const file_info_t *file)
+{
+	if (file->size % fs->block_size == 0)
+		return false;
+
+	return file->fragment_offset < fs->block_size &&
+		(file->fragment != 0xFFFFFFFF);
+}
+
 static int compare_files(const fstree_t *fs, const file_info_t *lhs,
 			 const file_info_t *rhs)
 {
@@ -20,8 +29,8 @@ static int compare_files(const fstree_t *fs, const file_info_t *lhs,
 	/* Files with fragments come first, ordered by ID.
 	   In case of tie, files without data blocks come first,
 	   and the others are ordered by start block. */
-	if (lhs->flags & FILE_FLAG_HAS_FRAGMENT) {
-		if (!(rhs->flags & FILE_FLAG_HAS_FRAGMENT))
+	if (has_fragment(fs, lhs)) {
+		if (!(has_fragment(fs, rhs)))
 			return -1;
 
 		if (lhs->fragment < rhs->fragment)
@@ -36,7 +45,7 @@ static int compare_files(const fstree_t *fs, const file_info_t *lhs,
 		goto order_by_start;
 	}
 
-	if (rhs->flags & FILE_FLAG_HAS_FRAGMENT)
+	if (has_fragment(fs, rhs))
 		return 1;
 
 	/* order the rest by start block */
diff --git a/lib/sqfshelper/data_reader.c b/lib/sqfshelper/data_reader.c
index 6906933..42a351f 100644
--- a/lib/sqfshelper/data_reader.c
+++ b/lib/sqfshelper/data_reader.c
@@ -181,7 +181,8 @@ int data_reader_dump_file(data_reader_t *data, file_info_t *fi, int outfd,
 	off_t off = fi->startblock;
 	size_t i, diff;
 
-	if (fragsz != 0 && !(fi->flags & FILE_FLAG_HAS_FRAGMENT)) {
+	if (fragsz != 0 && (fi->fragment_offset >= data->block_size ||
+			    fi->fragment == 0xFFFFFFFF)) {
 		fragsz = 0;
 		++count;
 	}
@@ -249,7 +250,8 @@ ssize_t data_reader_read(data_reader_t *data, file_info_t *fi,
 	fragsz = fi->size % data->block_size;
 	count = fi->size / data->block_size;
 
-	if (fragsz != 0 && !(fi->flags & FILE_FLAG_HAS_FRAGMENT)) {
+	if (fragsz != 0 && (fi->fragment_offset >= data->block_size ||
+			    fi->fragment == 0xFFFFFFFF)) {
 		fragsz = 0;
 		++count;
 	}
diff --git a/lib/sqfshelper/data_writer.c b/lib/sqfshelper/data_writer.c
index 7d84762..43d5851 100644
--- a/lib/sqfshelper/data_writer.c
+++ b/lib/sqfshelper/data_writer.c
@@ -58,6 +58,8 @@ struct data_writer_t {
 	size_t frag_list_num;
 	size_t frag_list_max;
 	frag_info_t *frag_list;
+
+	data_writer_stats_t stats;
 };
 
 enum {
@@ -146,12 +148,16 @@ static int block_callback(void *user, sqfs_block_t *blk)
 
 			data->super->flags &= ~SQFS_FLAG_NO_FRAGMENTS;
 			data->super->flags |= SQFS_FLAG_ALWAYS_FRAGMENTS;
+
+			data->stats.frag_blocks_written += 1;
 		} else {
 			fi->block_size[blk->index] = htole32(out);
 
 			if (store_block_location(data, offset, out,
 						 blk->checksum))
 				return -1;
+
+			data->stats.blocks_written += 1;
 		}
 
 		if (data->file->write_at(data->file, offset,
@@ -170,9 +176,10 @@ static int block_callback(void *user, sqfs_block_t *blk)
 		fi->startblock = data->blocks[start].offset;
 
 		if (start + count < data->file_start) {
-			fi->flags |= FILE_FLAG_BLOCKS_ARE_DUPLICATE;
 			data->num_blocks = data->file_start;
 
+			data->stats.duplicate_blocks += count;
+
 			if (data->file->truncate(data->file, data->start)) {
 				perror("truncating squashfs image after "
 				       "file deduplication");
@@ -227,8 +234,9 @@ static int handle_fragment(data_writer_t *data, sqfs_block_t *frag)
 		if (data->frag_list[i].signature == signature) {
 			fi->fragment_offset = data->frag_list[i].offset;
 			fi->fragment = data->frag_list[i].index;
-			fi->flags |= FILE_FLAG_FRAGMENT_IS_DUPLICATE;
 			free(frag);
+
+			data->stats.frag_dup += 1;
 			return 0;
 		}
 	}
@@ -282,6 +290,8 @@ static int handle_fragment(data_writer_t *data, sqfs_block_t *frag)
 
 	data->frag_block->size += frag->size;
 	free(frag);
+
+	data->stats.frag_count += 1;
 	return 0;
 fail:
 	free(frag);
@@ -360,6 +370,8 @@ int write_data_from_fd(data_writer_t *data, file_info_t *fi,
 		blk->index = i++;
 
 		if (is_zero_block(blk->data, blk->size)) {
+			data->stats.sparse_blocks += 1;
+
 			fi->block_size[blk->index] = 0;
 			free(blk);
 			continue;
@@ -367,8 +379,6 @@ int write_data_from_fd(data_writer_t *data, file_info_t *fi,
 
 		if (diff < data->super->block_size &&
 		    !(flags & DW_DONT_FRAGMENT)) {
-			fi->flags |= FILE_FLAG_HAS_FRAGMENT;
-
 			if (!(blk_flags & (BLK_FIRST_BLOCK | BLK_LAST_BLOCK))) {
 				blk_flags |= BLK_LAST_BLOCK;
 
@@ -395,6 +405,8 @@ int write_data_from_fd(data_writer_t *data, file_info_t *fi,
 			return -1;
 	}
 
+	data->stats.bytes_read += fi->size;
+	data->stats.file_count += 1;
 	return 0;
 }
 
@@ -494,6 +506,8 @@ int write_data_from_fd_condensed(data_writer_t *data, file_info_t *fi,
 		}
 
 		if (is_zero_block(blk->data, blk->size)) {
+			data->stats.sparse_blocks += 1;
+
 			fi->block_size[blk->index] = 0;
 			free(blk);
 			continue;
@@ -501,8 +515,6 @@ int write_data_from_fd_condensed(data_writer_t *data, file_info_t *fi,
 
 		if (diff < data->super->block_size &&
 		    !(flags & DW_DONT_FRAGMENT)) {
-			fi->flags |= FILE_FLAG_HAS_FRAGMENT;
-
 			if (!(blk_flags & (BLK_FIRST_BLOCK | BLK_LAST_BLOCK))) {
 				blk_flags |= BLK_LAST_BLOCK;
 
@@ -529,6 +541,8 @@ int write_data_from_fd_condensed(data_writer_t *data, file_info_t *fi,
 			return -1;
 	}
 
+	data->stats.bytes_read += fi->size;
+	data->stats.file_count += 1;
 	return 0;
 }
 
@@ -623,3 +637,8 @@ int data_writer_sync(data_writer_t *data)
 
 	return sqfs_block_processor_finish(data->proc);
 }
+
+data_writer_stats_t *data_writer_get_stats(data_writer_t *data)
+{
+	return &data->stats;
+}
diff --git a/lib/sqfshelper/deserialize_fstree.c b/lib/sqfshelper/deserialize_fstree.c
index 6c536c3..ef000d4 100644
--- a/lib/sqfshelper/deserialize_fstree.c
+++ b/lib/sqfshelper/deserialize_fstree.c
@@ -155,8 +155,7 @@ static int fill_dir(sqfs_meta_reader_t *ir, sqfs_meta_reader_t *dr,
 			}
 
 			n = tree_node_from_inode(inode, idtbl,
-						 (char *)ent->name,
-						 fs->block_size);
+						 (char *)ent->name);
 
 			if (n == NULL) {
 				free(ent);
@@ -283,7 +282,7 @@ int deserialize_fstree(fstree_t *out, sqfs_super_t *super,
 	out->defaults.st_mode = 0755;
 	out->defaults.st_mtime = super->modification_time;
 
-	out->root = tree_node_from_inode(root, idtbl, "", out->block_size);
+	out->root = tree_node_from_inode(root, idtbl, "");
 
 	if (out->root == NULL) {
 		free(root);
diff --git a/lib/sqfshelper/statistics.c b/lib/sqfshelper/statistics.c
index 0fe325b..fda4c3f 100644
--- a/lib/sqfshelper/statistics.c
+++ b/lib/sqfshelper/statistics.c
@@ -9,73 +9,25 @@
 
 #include <stdio.h>
 
-void sqfs_print_statistics(fstree_t *fs, sqfs_super_t *super)
+void sqfs_print_statistics(sqfs_super_t *super, data_writer_stats_t *stats)
 {
-	size_t blocks_written = 0, duplicate_blocks = 0, sparse_blocks = 0;
-	size_t ratio, file_count = 0, file_dup_count = 0;
-	size_t frag_count = 0, frag_dup = 0;
-	size_t i, num_blocks, sparse;
-	uint64_t output_bytes = 0;
-	uint64_t input_bytes = 0;
-	file_info_t *fi;
-	bool is_dupe;
+	uint64_t bytes_written = super->inode_table_start - sizeof(*super);
+	size_t ratio;
 
-	for (fi = fs->files; fi != NULL; fi = fi->next) {
-		num_blocks = fi->size / fs->block_size;
-		is_dupe = true;
-
-		if ((fi->size % fs->block_size) &&
-		    !(fi->flags & FILE_FLAG_HAS_FRAGMENT)) {
-			++num_blocks;
-		}
-
-		for (sparse = 0, i = 0; i < num_blocks; ++i) {
-			if (fi->block_size[i] == 0)
-				sparse += 1;
-		}
-
-		if (num_blocks > sparse) {
-			if (fi->flags & FILE_FLAG_BLOCKS_ARE_DUPLICATE) {
-				duplicate_blocks += num_blocks - sparse;
-			} else {
-				blocks_written += num_blocks - sparse;
-				is_dupe = false;
-			}
-		}
-
-		if (fi->flags & FILE_FLAG_HAS_FRAGMENT) {
-			if (fi->flags & FILE_FLAG_FRAGMENT_IS_DUPLICATE) {
-				frag_dup += 1;
-			} else {
-				frag_count += 1;
-				is_dupe = false;
-			}
-		}
-
-		if (is_dupe)
-			file_dup_count += 1;
-
-		sparse_blocks += sparse;
-		file_count += 1;
-		input_bytes += fi->size;
-	}
-
-	if (input_bytes > 0) {
-		output_bytes = super->inode_table_start - sizeof(*super);
-		ratio = (100 * output_bytes) / input_bytes;
+	if (bytes_written > 0) {
+		ratio = (100 * bytes_written) / stats->bytes_read;
 	} else {
 		ratio = 100;
 	}
 
 	fputs("---------------------------------------------------\n", stdout);
-	printf("Input files processed: %zu\n", file_count);
-	printf("Files that were complete duplicates: %zu\n", file_dup_count);
-	printf("Data blocks actually written: %zu\n", blocks_written);
-	printf("Fragment blocks written: %u\n", super->fragment_entry_count);
-	printf("Duplicate data blocks omitted: %zu\n", duplicate_blocks);
-	printf("Sparse blocks omitted: %zu\n", sparse_blocks);
-	printf("Fragments actually written: %zu\n", frag_count);
-	printf("Duplicated fragments omitted: %zu\n", frag_dup);
+	printf("Input files processed: %zu\n", stats->file_count);
+	printf("Data blocks actually written: %zu\n", stats->blocks_written);
+	printf("Fragment blocks written: %zu\n", stats->frag_blocks_written);
+	printf("Duplicate data blocks omitted: %zu\n", stats->duplicate_blocks);
+	printf("Sparse blocks omitted: %zu\n", stats->sparse_blocks);
+	printf("Fragments actually written: %zu\n", stats->frag_count);
+	printf("Duplicated fragments omitted: %zu\n", stats->frag_dup);
 	printf("Total number of inodes: %u\n", super->inode_count);
 	printf("Number of unique group/user IDs: %u\n", super->id_count);
 	printf("Data compression ratio: %zu%%\n", ratio);
diff --git a/lib/sqfshelper/tree_node_from_inode.c b/lib/sqfshelper/tree_node_from_inode.c
index fee191b..f5b643c 100644
--- a/lib/sqfshelper/tree_node_from_inode.c
+++ b/lib/sqfshelper/tree_node_from_inode.c
@@ -38,18 +38,10 @@ static size_t compute_size(sqfs_inode_generic_t *inode, const char *name)
 	return size;
 }
 
-static void copy_block_sizes(sqfs_inode_generic_t *inode, tree_node_t *out,
-			     size_t block_size)
+static void copy_block_sizes(sqfs_inode_generic_t *inode, tree_node_t *out)
 {
 	size_t i;
 
-	if ((out->data.file->size % block_size) != 0) {
-		if (out->data.file->fragment != 0xFFFFFFFF &&
-		    out->data.file->fragment_offset != 0xFFFFFFFF) {
-			out->data.file->flags |= FILE_FLAG_HAS_FRAGMENT;
-		}
-	}
-
 	out->name += inode->num_file_blocks *
 		sizeof(out->data.file->block_size[0]);
 
@@ -59,8 +51,7 @@ static void copy_block_sizes(sqfs_inode_generic_t *inode, tree_node_t *out,
 
 tree_node_t *tree_node_from_inode(sqfs_inode_generic_t *inode,
 				  const sqfs_id_table_t *idtbl,
-				  const char *name,
-				  size_t block_size)
+				  const char *name)
 {
 	tree_node_t *out;
 
@@ -112,7 +103,7 @@ tree_node_t *tree_node_from_inode(sqfs_inode_generic_t *inode,
 		out->data.file->fragment_offset =
 			inode->data.file.fragment_offset;
 
-		copy_block_sizes(inode, out, block_size);
+		copy_block_sizes(inode, out);
 		break;
 	case SQFS_INODE_EXT_FILE:
 		out->data.file = (file_info_t *)out->payload;
@@ -125,7 +116,7 @@ tree_node_t *tree_node_from_inode(sqfs_inode_generic_t *inode,
 		out->data.file->fragment_offset =
 			inode->data.file_ext.fragment_offset;
 
-		copy_block_sizes(inode, out, block_size);
+		copy_block_sizes(inode, out);
 		break;
 	case SQFS_INODE_SLINK:
 	case SQFS_INODE_EXT_SLINK:
diff --git a/lib/sqfshelper/tree_node_to_inode.c b/lib/sqfshelper/tree_node_to_inode.c
index cc76a8d..2fdc21b 100644
--- a/lib/sqfshelper/tree_node_to_inode.c
+++ b/lib/sqfshelper/tree_node_to_inode.c
@@ -81,6 +81,15 @@ static int get_type(tree_node_t *node)
 	assert(0);
 }
 
+static bool has_fragment(const fstree_t *fs, const file_info_t *file)
+{
+	if (file->size % fs->block_size == 0)
+		return false;
+
+	return file->fragment_offset < fs->block_size &&
+		(file->fragment != 0xFFFFFFFF);
+}
+
 sqfs_inode_generic_t *tree_node_to_inode(fstree_t *fs, sqfs_id_table_t *idtbl,
 					 tree_node_t *node)
 {
@@ -97,10 +106,8 @@ sqfs_inode_generic_t *tree_node_to_inode(fstree_t *fs, sqfs_id_table_t *idtbl,
 
 		block_count = fi->size / fs->block_size;
 
-		if ((fi->size % fs->block_size) != 0 &&
-		    !(fi->flags & FILE_FLAG_HAS_FRAGMENT)) {
+		if ((fi->size % fs->block_size) != 0 && !has_fragment(fs, fi))
 			++block_count;
-		}
 
 		extra = block_count * sizeof(uint32_t);
 	}
@@ -176,8 +183,7 @@ sqfs_inode_generic_t *tree_node_to_inode(fstree_t *fs, sqfs_id_table_t *idtbl,
 		inode->data.file.fragment_offset = 0xFFFFFFFF;
 		inode->data.file.file_size = fi->size;
 
-		if ((fi->size % fs->block_size) != 0 &&
-		    (fi->flags & FILE_FLAG_HAS_FRAGMENT)) {
+		if (has_fragment(fs, fi)) {
 			inode->data.file.fragment_index = fi->fragment;
 			inode->data.file.fragment_offset = fi->fragment_offset;
 		}
@@ -191,8 +197,7 @@ sqfs_inode_generic_t *tree_node_to_inode(fstree_t *fs, sqfs_id_table_t *idtbl,
 		inode->data.file_ext.fragment_offset = 0xFFFFFFFF;
 		inode->data.file_ext.xattr_idx = xattr;
 
-		if ((fi->size % fs->block_size) != 0 &&
-		    (fi->flags & FILE_FLAG_HAS_FRAGMENT)) {
+		if (has_fragment(fs, fi)) {
 			inode->data.file_ext.fragment_idx = fi->fragment;
 			inode->data.file_ext.fragment_offset =
 				fi->fragment_offset;
diff --git a/mkfs/mkfs.c b/mkfs/mkfs.c
index 0c4e9a5..a046cbd 100644
--- a/mkfs/mkfs.c
+++ b/mkfs/mkfs.c
@@ -196,7 +196,7 @@ int main(int argc, char **argv)
 
 	if (!opt.quiet) {
 		fstree_gen_file_list(&fs);
-		sqfs_print_statistics(&fs, &super);
+		sqfs_print_statistics(&super, data_writer_get_stats(data));
 	}
 
 	status = EXIT_SUCCESS;
diff --git a/tar/tar2sqfs.c b/tar/tar2sqfs.c
index 5f53547..13276b8 100644
--- a/tar/tar2sqfs.c
+++ b/tar/tar2sqfs.c
@@ -466,7 +466,7 @@ int main(int argc, char **argv)
 
 	if (!quiet) {
 		fstree_gen_file_list(&fs);
-		sqfs_print_statistics(&fs, &super);
+		sqfs_print_statistics(&super, data_writer_get_stats(data));
 	}
 
 	status = EXIT_SUCCESS;