Add support for packing sparse files

This commit adds support for packing sparse files into squashfs images
as follows:

 - In the data writer: simply detect zero blocks and write a zero to the
   block size field and don't emit any data. Record the number of bytes
   saved this way. For fragments, set the fragment offset to invalid.
 - In the inode writer: write out the number of bytes saved for sparse
   files. If there should be a fragment but there is none, append a block
   count of 0.
 - In the data reader: if the block size is 0, read nothing from disk and
   emit an empty block. Do the same if the fragment is missing.
 - In the inode reader: restore the number of bytes saved for sparse files.

The sparse files can be packed and unpacked, but the unpacking will not
create sparse files for now.

Signed-off-by: David Oberhollenzer <david.oberhollenzer@sigma-star.at>

4 files changed, 53 insertions, 14 deletions

diff --git a/lib/sqfs/data_reader.c b/lib/sqfs/data_reader.c
index 22413f2..421d914 100644
--- a/lib/sqfs/data_reader.c
+++ b/lib/sqfs/data_reader.c
@@ -5,6 +5,7 @@
 
 #include <stdlib.h>
 #include <unistd.h>
+#include <string.h>
 #include <stdio.h>
 
 struct data_reader_t {
@@ -81,12 +82,18 @@ int data_reader_dump_file(data_reader_t *data, file_info_t *fi, int outfd)
 			if (bs > data->block_size)
 				goto fail_bs;
 
-			ret = read_retry(data->sqfsfd, data->buffer, bs);
-			if (ret < 0)
-				goto fail_rd;
+			if (bs == 0) {
+				bs = data->block_size;
+				memset(data->buffer, 0, bs);
+				compressed = false;
+			} else {
+				ret = read_retry(data->sqfsfd, data->buffer, bs);
+				if (ret < 0)
+					goto fail_rd;
 
-			if ((size_t)ret < bs)
-				goto fail_trunc;
+				if ((size_t)ret < bs)
+					goto fail_trunc;
+			}
 
 			if (compressed) {
 				ret = data->cmp->do_block(data->cmp,
@@ -114,13 +121,18 @@ int data_reader_dump_file(data_reader_t *data, file_info_t *fi, int outfd)
 	fragsz = fi->size % data->block_size;
 
 	if (fragsz > 0) {
-		if (data->frag == NULL)
-			goto fail_frag;
-
-		if (frag_reader_read(data->frag, fi->fragment,
-				     fi->fragment_offset, data->buffer,
-				     fragsz)) {
-			return -1;
+		if (fi->fragment == 0xFFFFFFFF ||
+		    fi->fragment_offset == 0xFFFFFFFF) {
+			memset(data->buffer, 0, fragsz);
+		} else {
+			if (data->frag == NULL)
+				goto fail_frag;
+
+			if (frag_reader_read(data->frag, fi->fragment,
+					     fi->fragment_offset, data->buffer,
+					     fragsz)) {
+				return -1;
+			}
 		}
 
 		ret = write_retry(outfd, data->buffer, fragsz);
diff --git a/lib/sqfs/data_writer.c b/lib/sqfs/data_writer.c
index 753e5d4..70fe884 100644
--- a/lib/sqfs/data_writer.c
+++ b/lib/sqfs/data_writer.c
@@ -78,6 +78,11 @@ static int grow_fragment_table(data_writer_t *data)
 	return 0;
 }
 
+static bool is_zero_block(unsigned char *ptr, size_t size)
+{
+	return ptr[0] == 0 && memcmp(ptr, ptr + 1, size - 1) == 0;
+}
+
 int data_writer_flush_fragments(data_writer_t *data)
 {
 	uint64_t offset;
@@ -115,6 +120,7 @@ int write_data_from_fd(data_writer_t *data, file_info_t *fi, int infd)
 	size_t diff;
 
 	fi->startblock = data->super->bytes_used;
+	fi->sparse = 0;
 
 	while (count != 0) {
 		diff = count > (uint64_t)data->super->block_size ?
@@ -126,6 +132,18 @@ int write_data_from_fd(data_writer_t *data, file_info_t *fi, int infd)
 		if ((size_t)ret < diff)
 			goto fail_trunc;
 
+		if (is_zero_block(data->block, diff)) {
+			if (diff < data->super->block_size) {
+				fi->fragment_offset = 0xFFFFFFFF;
+				fi->fragment = 0xFFFFFFFF;
+			} else {
+				fi->blocksizes[blk_idx++] = 0;
+			}
+			fi->sparse += diff;
+			count -= diff;
+			continue;
+		}
+
 		if (diff < data->super->block_size) {
 			if (data->frag_offset + diff > data->super->block_size) {
 				if (data_writer_flush_fragments(data))
diff --git a/lib/sqfs/tree_node_from_inode.c b/lib/sqfs/tree_node_from_inode.c
index f9a6599..9c96701 100644
--- a/lib/sqfs/tree_node_from_inode.c
+++ b/lib/sqfs/tree_node_from_inode.c
@@ -113,6 +113,7 @@ tree_node_t *tree_node_from_inode(sqfs_inode_generic_t *inode,
 		out->name += sizeof(file_info_t);
 
 		out->data.file->size = inode->data.file_ext.file_size;
+		out->data.file->sparse = inode->data.file_ext.sparse;
 		out->data.file->startblock = inode->data.file_ext.blocks_start;
 		out->data.file->fragment = inode->data.file_ext.fragment_idx;
 		out->data.file->fragment_offset =
diff --git a/lib/sqfs/write_inode.c b/lib/sqfs/write_inode.c
index fd14913..82663e7 100644
--- a/lib/sqfs/write_inode.c
+++ b/lib/sqfs/write_inode.c
@@ -86,7 +86,7 @@ int meta_writer_write_inode(fstree_t *fs, id_table_t *idtbl, meta_writer_t *im,
 		fi = node->data.file;
 
 		if (fi->startblock > 0xFFFFFFFFUL || fi->size > 0xFFFFFFFFUL ||
-		    hard_link_count(node) > 1) {
+		    hard_link_count(node) > 1 || fi->sparse > 0) {
 			type = SQFS_INODE_EXT_FILE;
 		}
 	}
@@ -185,7 +185,7 @@ int meta_writer_write_inode(fstree_t *fs, id_table_t *idtbl, meta_writer_t *im,
 		sqfs_inode_file_ext_t ext = {
 			.blocks_start = htole64(fi->startblock),
 			.file_size = htole64(fi->size),
-			.sparse = htole64(0),
+			.sparse = htole64(fi->sparse),
 			.nlink = htole32(hard_link_count(node)),
 			.fragment_idx = htole32(0xFFFFFFFF),
 			.fragment_offset = htole32(0xFFFFFFFF),
@@ -298,5 +298,13 @@ out_file_blocks:
 		if (meta_writer_append(im, &bs, sizeof(bs)))
 			return -1;
 	}
+
+	if ((fi->size % fs->block_size) != 0 &&
+	    (fi->fragment == 0xFFFFFFFF || fi->fragment_offset == 0xFFFFFFFF)) {
+		bs = htole32(0);
+
+		if (meta_writer_append(im, &bs, sizeof(bs)))
+			return -1;
+	}
 	return 0;
 }