Split the block writing/deduplication away from the block processor

This commit moves the entire block writing and deduplication of data
blocks over to a different data type named "block writer".

For simplicity, the interfaces of the block processor are left as is
and are turned into warppers. Likewise, most of the code in the block
writer is just verbatim from the block processor, to be cleaned up in
subsequent commits.

Signed-off-by: David Oberhollenzer <david.oberhollenzer@sigma-star.at>

5 files changed, 258 insertions, 175 deletions

diff --git a/lib/sqfs/Makemodule.am b/lib/sqfs/Makemodule.am
index cde947d..644ced2 100644
--- a/lib/sqfs/Makemodule.am
+++ b/lib/sqfs/Makemodule.am
@@ -8,7 +8,7 @@ LIBSQFS_HEARDS = include/sqfs/meta_writer.h \
 		include/sqfs/dir_writer.h include/sqfs/io.h \
 		include/sqfs/data_reader.h include/sqfs/block.h \
 		include/sqfs/xattr_reader.h include/sqfs/xattr_writer.h \
-		include/sqfs/frag_table.h
+		include/sqfs/frag_table.h include/sqfs/block_writer.h
 
 libsquashfs_la_SOURCES = $(LIBSQFS_HEARDS) lib/sqfs/id_table.c lib/sqfs/super.c
 libsquashfs_la_SOURCES += lib/sqfs/readdir.c lib/sqfs/xattr.c
@@ -28,6 +28,7 @@ libsquashfs_la_SOURCES += lib/sqfs/block_processor/fileapi.c
 libsquashfs_la_SOURCES += lib/sqfs/str_table.c lib/sqfs/str_table.h
 libsquashfs_la_SOURCES += lib/sqfs/alloc.c lib/sqfs/util.h
 libsquashfs_la_SOURCES += lib/sqfs/frag_table.c include/sqfs/frag_table.h
+libsquashfs_la_SOURCES += lib/sqfs/block_writer.c include/sqfs/block_writer.h
 libsquashfs_la_CPPFLAGS = $(AM_CPPFLAGS)
 libsquashfs_la_LDFLAGS = $(AM_LDFLAGS)
 libsquashfs_la_CFLAGS = $(AM_CFLAGS) $(PTHREAD_CFLAGS) $(ZLIB_CFLAGS)
diff --git a/lib/sqfs/block_processor/block.c b/lib/sqfs/block_processor/block.c
index 49892be..e88456a 100644
--- a/lib/sqfs/block_processor/block.c
+++ b/lib/sqfs/block_processor/block.c
@@ -9,164 +9,33 @@
 
 #include <string.h>
 
-static int store_block_location(sqfs_block_processor_t *proc, sqfs_u64 offset,
-				sqfs_u32 size, sqfs_u32 chksum)
-{
-	size_t new_sz;
-	void *new;
-
-	if (proc->num_blocks == proc->max_blocks) {
-		new_sz = proc->max_blocks * 2;
-		new = realloc(proc->blocks, sizeof(proc->blocks[0]) * new_sz);
-
-		if (new == NULL)
-			return SQFS_ERROR_ALLOC;
-
-		proc->blocks = new;
-		proc->max_blocks = new_sz;
-	}
-
-	proc->blocks[proc->num_blocks].offset = offset;
-	proc->blocks[proc->num_blocks].hash = MK_BLK_HASH(chksum, size);
-	proc->num_blocks += 1;
-	return 0;
-}
-
-static size_t deduplicate_blocks(sqfs_block_processor_t *proc, size_t count)
-{
-	size_t i, j;
-
-	for (i = 0; i < proc->file_start; ++i) {
-		for (j = 0; j < count; ++j) {
-			if (proc->blocks[i + j].hash !=
-			    proc->blocks[proc->file_start + j].hash)
-				break;
-		}
-
-		if (j == count)
-			break;
-	}
-
-	return i;
-}
-
-static int align_file(sqfs_block_processor_t *proc, sqfs_block_t *blk)
-{
-	sqfs_u32 chksum;
-	void *padding;
-	sqfs_u64 size;
-	size_t diff;
-	int ret;
-
-	if (!(blk->flags & SQFS_BLK_ALIGN))
-		return 0;
-
-	size = proc->file->get_size(proc->file);
-	diff = size % proc->devblksz;
-	if (diff == 0)
-		return 0;
-
-	padding = calloc(1, diff);
-	if (padding == 0)
-		return SQFS_ERROR_ALLOC;
-
-	if (proc->hooks != NULL && proc->hooks->prepare_padding != NULL)
-		proc->hooks->prepare_padding(proc->user_ptr, padding, diff);
-
-	chksum = crc32(0, padding, diff);
-
-	ret = proc->file->write_at(proc->file, size, padding, diff);
-	free(padding);
-	if (ret)
-		return ret;
-
-	return store_block_location(proc, size, diff | (1 << 24), chksum);
-}
-
 int process_completed_block(sqfs_block_processor_t *proc, sqfs_block_t *blk)
 {
-	sqfs_u64 offset, bytes;
-	size_t start, count;
-	sqfs_u32 out;
+	sqfs_u64 location;
+	sqfs_u32 size;
 	int err;
 
-	if (proc->hooks != NULL && proc->hooks->pre_block_write != NULL) {
-		proc->hooks->pre_block_write(proc->user_ptr, blk, proc->file);
-	}
-
-	if (blk->flags & SQFS_BLK_FIRST_BLOCK) {
-		proc->start = proc->file->get_size(proc->file);
-		proc->file_start = proc->num_blocks;
+	size = blk->size;
+	if (!(blk->flags & SQFS_BLK_IS_COMPRESSED))
+		size |= 1 << 24;
 
-		err = align_file(proc, blk);
-		if (err)
-			return err;
-	}
+	err = sqfs_block_writer_write(proc->wr, blk, &location);
+	if (err)
+		return err;
 
 	if (blk->size != 0) {
-		out = blk->size;
-		if (!(blk->flags & SQFS_BLK_IS_COMPRESSED))
-			out |= 1 << 24;
-
-		offset = proc->file->get_size(proc->file);
-
 		if (blk->flags & SQFS_BLK_FRAGMENT_BLOCK) {
 			err = sqfs_frag_table_set(proc->frag_tbl, blk->index,
-						  offset, out);
+						  location, size);
 			if (err)
 				return err;
 		} else {
-			blk->inode->extra[blk->index] = out;
+			blk->inode->extra[blk->index] = size;
 		}
-
-		err = store_block_location(proc, offset, out, blk->checksum);
-		if (err)
-			return err;
-
-		err = proc->file->write_at(proc->file, offset,
-					   blk->data, blk->size);
-		if (err)
-			return err;
-	}
-
-	if (proc->hooks != NULL && proc->hooks->post_block_write != NULL) {
-		proc->hooks->post_block_write(proc->user_ptr, blk, proc->file);
 	}
 
-	if (blk->flags & SQFS_BLK_LAST_BLOCK) {
-		err = align_file(proc, blk);
-		if (err)
-			return err;
-
-		count = proc->num_blocks - proc->file_start;
-		start = deduplicate_blocks(proc, count);
-		offset = proc->blocks[start].offset;
-
-		sqfs_inode_set_file_block_start(blk->inode, offset);
-
-		if (start >= proc->file_start)
-			return 0;
-
-		offset = start + count;
-		if (offset >= proc->file_start) {
-			count = proc->num_blocks - offset;
-			proc->num_blocks = offset;
-		} else {
-			proc->num_blocks = proc->file_start;
-		}
-
-		if (proc->hooks != NULL &&
-		    proc->hooks->notify_blocks_erased != NULL) {
-			bytes = proc->file->get_size(proc->file) - proc->start;
-
-			proc->hooks->notify_blocks_erased(proc->user_ptr,
-							  count, bytes);
-		}
-
-		err = proc->file->truncate(proc->file, proc->start);
-		if (err)
-			return err;
-	}
+	if (blk->flags & SQFS_BLK_LAST_BLOCK)
+		sqfs_inode_set_file_block_start(blk->inode, location);
 
 	return 0;
 }
diff --git a/lib/sqfs/block_processor/common.c b/lib/sqfs/block_processor/common.c
index c6375dd..806a595 100644
--- a/lib/sqfs/block_processor/common.c
+++ b/lib/sqfs/block_processor/common.c
@@ -25,19 +25,16 @@ int block_processor_init(sqfs_block_processor_t *proc, size_t max_block_size,
 	proc->max_block_size = max_block_size;
 	proc->num_workers = num_workers;
 	proc->max_backlog = max_backlog;
-	proc->devblksz = devblksz;
 	proc->cmp = cmp;
 	proc->file = file;
-	proc->max_blocks = INIT_BLOCK_COUNT;
 
 	proc->frag_tbl = sqfs_frag_table_create(0);
 	if (proc->frag_tbl == NULL)
 		return -1;
 
-	proc->blocks = alloc_array(sizeof(proc->blocks[0]), proc->max_blocks);
-	if (proc->blocks == NULL)
+	proc->wr = sqfs_block_writer_create(file, devblksz, 0);
+	if (proc->wr == NULL)
 		return -1;
-
 	return 0;
 }
 
@@ -45,11 +42,12 @@ void block_processor_cleanup(sqfs_block_processor_t *proc)
 {
 	if (proc->frag_tbl != NULL)
 		sqfs_frag_table_destroy(proc->frag_tbl);
+	if (proc->wr != NULL)
+		sqfs_block_writer_destroy(proc->wr);
 	free_blk_list(proc->queue);
 	free_blk_list(proc->done);
 	free(proc->blk_current);
 	free(proc->frag_block);
-	free(proc->blocks);
 	free(proc);
 }
 
@@ -63,10 +61,8 @@ int sqfs_block_processor_write_fragment_table(sqfs_block_processor_t *proc,
 int sqfs_block_processor_set_hooks(sqfs_block_processor_t *proc, void *user_ptr,
 				   const sqfs_block_hooks_t *hooks)
 {
-	if (hooks->size != sizeof(*hooks))
-		return SQFS_ERROR_UNSUPPORTED;
-
 	proc->hooks = hooks;
 	proc->user_ptr = user_ptr;
-	return 0;
+
+	return sqfs_block_writer_set_hooks(proc->wr, user_ptr, hooks);
 }
diff --git a/lib/sqfs/block_processor/internal.h b/lib/sqfs/block_processor/internal.h
index 40871b9..16f0edb 100644
--- a/lib/sqfs/block_processor/internal.h
+++ b/lib/sqfs/block_processor/internal.h
@@ -10,6 +10,7 @@
 #include "config.h"
 
 #include "sqfs/block_processor.h"
+#include "sqfs/block_writer.h"
 #include "sqfs/frag_table.h"
 #include "sqfs/compressor.h"
 #include "sqfs/inode.h"
@@ -31,19 +32,6 @@
 #include <windows.h>
 #endif
 
-
-#define MK_BLK_HASH(chksum, size) \
-	(((sqfs_u64)(size) << 32) | (sqfs_u64)(chksum))
-
-#define INIT_BLOCK_COUNT (128)
-
-
-typedef struct {
-	sqfs_u64 offset;
-	sqfs_u64 hash;
-} blk_info_t;
-
-
 typedef struct compress_worker_t compress_worker_t;
 
 struct sqfs_block_processor_t {
@@ -73,24 +61,18 @@ struct sqfs_block_processor_t {
 	unsigned int num_workers;
 	size_t max_backlog;
 
-	size_t devblksz;
 	sqfs_file_t *file;
 
 	sqfs_frag_table_t *frag_tbl;
 
-	sqfs_u64 start;
-
-	size_t file_start;
-	size_t num_blocks;
-	size_t max_blocks;
-	blk_info_t *blocks;
 	sqfs_compressor_t *cmp;
 
 	sqfs_block_t *frag_block;
+	bool notify_threads;
 
+	sqfs_block_writer_t *wr;
 	const sqfs_block_hooks_t *hooks;
 	void *user_ptr;
-	bool notify_threads;
 
 	/* file API */
 	sqfs_inode_generic_t *inode;
diff --git a/lib/sqfs/block_writer.c b/lib/sqfs/block_writer.c
new file mode 100644
index 0000000..c955532
--- /dev/null
+++ b/lib/sqfs/block_writer.c
@@ -0,0 +1,235 @@
+/* SPDX-License-Identifier: LGPL-3.0-or-later */
+/*
+ * block_writer.c
+ *
+ * Copyright (C) 2019 David Oberhollenzer <goliath@infraroot.at>
+ */
+#define SQFS_BUILDING_DLL
+#include "config.h"
+
+#include "sqfs/block_writer.h"
+#include "sqfs/error.h"
+#include "sqfs/block.h"
+#include "sqfs/io.h"
+#include "util.h"
+
+#include <stdlib.h>
+
+#define MK_BLK_HASH(chksum, size) \
+	(((sqfs_u64)(size) << 32) | (sqfs_u64)(chksum))
+
+#define INIT_BLOCK_COUNT (128)
+
+typedef struct {
+	sqfs_u64 offset;
+	sqfs_u64 hash;
+} blk_info_t;
+
+struct sqfs_block_writer_t {
+	sqfs_file_t *file;
+
+	size_t num_blocks;
+	size_t max_blocks;
+	blk_info_t *blocks;
+	size_t devblksz;
+
+	const sqfs_block_hooks_t *hooks;
+	void *user_ptr;
+
+	sqfs_u64 start;
+	size_t file_start;
+};
+
+static int store_block_location(sqfs_block_writer_t *wr, sqfs_u64 offset,
+				sqfs_u32 size, sqfs_u32 chksum)
+{
+	size_t new_sz;
+	void *new;
+
+	if (wr->num_blocks == wr->max_blocks) {
+		new_sz = wr->max_blocks * 2;
+		new = realloc(wr->blocks, sizeof(wr->blocks[0]) * new_sz);
+
+		if (new == NULL)
+			return SQFS_ERROR_ALLOC;
+
+		wr->blocks = new;
+		wr->max_blocks = new_sz;
+	}
+
+	wr->blocks[wr->num_blocks].offset = offset;
+	wr->blocks[wr->num_blocks].hash = MK_BLK_HASH(chksum, size);
+	wr->num_blocks += 1;
+	return 0;
+}
+
+static size_t deduplicate_blocks(sqfs_block_writer_t *wr, size_t count)
+{
+	size_t i, j;
+
+	for (i = 0; i < wr->file_start; ++i) {
+		for (j = 0; j < count; ++j) {
+			if (wr->blocks[i + j].hash == 0)
+				break;
+
+			if (wr->blocks[i + j].hash !=
+			    wr->blocks[wr->file_start + j].hash)
+				break;
+		}
+
+		if (j == count)
+			break;
+	}
+
+	return i;
+}
+
+static int align_file(sqfs_block_writer_t *wr, sqfs_block_t *blk)
+{
+	void *padding;
+	sqfs_u64 size;
+	size_t diff;
+	int ret;
+
+	if (!(blk->flags & SQFS_BLK_ALIGN))
+		return 0;
+
+	size = wr->file->get_size(wr->file);
+	diff = size % wr->devblksz;
+	if (diff == 0)
+		return 0;
+
+	padding = calloc(1, diff);
+	if (padding == 0)
+		return SQFS_ERROR_ALLOC;
+
+	if (wr->hooks != NULL && wr->hooks->prepare_padding != NULL)
+		wr->hooks->prepare_padding(wr->user_ptr, padding, diff);
+
+	ret = wr->file->write_at(wr->file, size, padding, diff);
+	free(padding);
+	if (ret)
+		return ret;
+
+	return store_block_location(wr, size, 0, 0);
+}
+
+sqfs_block_writer_t *sqfs_block_writer_create(sqfs_file_t *file,
+					      size_t devblksz, sqfs_u32 flags)
+{
+	sqfs_block_writer_t *wr;
+
+	if (flags != 0)
+		return NULL;
+
+	wr = calloc(1, sizeof(*wr));
+	if (wr == NULL)
+		return NULL;
+
+	wr->file = file;
+	wr->devblksz = devblksz;
+	wr->max_blocks = INIT_BLOCK_COUNT;
+
+	wr->blocks = alloc_array(sizeof(wr->blocks[0]), wr->max_blocks);
+	if (wr->blocks == NULL) {
+		free(wr);
+		return NULL;
+	}
+
+	return wr;
+}
+
+int sqfs_block_writer_set_hooks(sqfs_block_writer_t *wr, void *user_ptr,
+				const sqfs_block_hooks_t *hooks)
+{
+	if (hooks->size != sizeof(*hooks))
+		return SQFS_ERROR_UNSUPPORTED;
+
+	wr->hooks = hooks;
+	wr->user_ptr = user_ptr;
+	return 0;
+}
+
+void sqfs_block_writer_destroy(sqfs_block_writer_t *wr)
+{
+	free(wr->blocks);
+	free(wr);
+}
+
+int sqfs_block_writer_write(sqfs_block_writer_t *wr, sqfs_block_t *block,
+			    sqfs_u64 *location)
+{
+	sqfs_u64 offset, bytes;
+	size_t start, count;
+	sqfs_u32 out;
+	int err;
+
+	if (wr->hooks != NULL && wr->hooks->pre_block_write != NULL)
+		wr->hooks->pre_block_write(wr->user_ptr, block, wr->file);
+
+	if (block->flags & SQFS_BLK_FIRST_BLOCK) {
+		wr->start = wr->file->get_size(wr->file);
+		wr->file_start = wr->num_blocks;
+
+		err = align_file(wr, block);
+		if (err)
+			return err;
+	}
+
+	if (block->size != 0) {
+		out = block->size;
+		if (!(block->flags & SQFS_BLK_IS_COMPRESSED))
+			out |= 1 << 24;
+
+		offset = wr->file->get_size(wr->file);
+		*location = offset;
+
+		err = store_block_location(wr, offset, out, block->checksum);
+		if (err)
+			return err;
+
+		err = wr->file->write_at(wr->file, offset,
+					 block->data, block->size);
+		if (err)
+			return err;
+	}
+
+	if (wr->hooks != NULL && wr->hooks->post_block_write != NULL)
+		wr->hooks->post_block_write(wr->user_ptr, block, wr->file);
+
+	if (block->flags & SQFS_BLK_LAST_BLOCK) {
+		err = align_file(wr, block);
+		if (err)
+			return err;
+
+		count = wr->num_blocks - wr->file_start;
+		start = deduplicate_blocks(wr, count);
+		offset = wr->blocks[start].offset;
+
+		*location = offset;
+		if (start >= wr->file_start)
+			return 0;
+
+		offset = start + count;
+		if (offset >= wr->file_start) {
+			count = wr->num_blocks - offset;
+			wr->num_blocks = offset;
+		} else {
+			wr->num_blocks = wr->file_start;
+		}
+
+		if (wr->hooks != NULL &&
+		    wr->hooks->notify_blocks_erased != NULL) {
+			bytes = wr->file->get_size(wr->file) - wr->start;
+
+			wr->hooks->notify_blocks_erased(wr->user_ptr,
+							count, bytes);
+		}
+
+		err = wr->file->truncate(wr->file, wr->start);
+		if (err)
+			return err;
+	}
+
+	return 0;
+}