Cleanup: libcommon: try to split up some of the larger files

The sqsf_writer is moved to a separate header, as well as the compressor
related stuff. The statistics function is moved into the writer code, as
this is the only place that actually uses it. The writer code itself is
split up into a hand full of file in their own subdirectory.

Signed-off-by: David Oberhollenzer <david.oberhollenzer@sigma-star.at>

4 files changed, 601 insertions, 0 deletions

diff --git a/lib/common/writer/cleanup.c b/lib/common/writer/cleanup.c
new file mode 100644
index 0000000..16e846b
--- /dev/null
+++ b/lib/common/writer/cleanup.c
@@ -0,0 +1,40 @@
+/* SPDX-License-Identifier: GPL-3.0-or-later */
+/*
+ * cleanup.c
+ *
+ * Copyright (C) 2019 David Oberhollenzer <goliath@infraroot.at>
+ */
+#include "simple_writer.h"
+
+#include <stdlib.h>
+
+void sqfs_writer_cleanup(sqfs_writer_t *sqfs, int status)
+{
+	if (sqfs->xwr != NULL)
+		sqfs_destroy(sqfs->xwr);
+
+	sqfs_destroy(sqfs->dirwr);
+	sqfs_destroy(sqfs->dm);
+	sqfs_destroy(sqfs->im);
+	sqfs_destroy(sqfs->idtbl);
+	sqfs_destroy(sqfs->data);
+	sqfs_destroy(sqfs->blkwr);
+	sqfs_destroy(sqfs->fragtbl);
+	sqfs_destroy(sqfs->cmp);
+	fstree_cleanup(&sqfs->fs);
+	sqfs_destroy(sqfs->outfile);
+
+	if (status != EXIT_SUCCESS) {
+#if defined(_WIN32) || defined(__WINDOWS__)
+		WCHAR *path = path_to_windows(sqfs->filename);
+
+		if (path != NULL)
+			DeleteFileW(path);
+
+		free(path);
+#else
+		unlink(sqfs->filename);
+#endif
+	}
+}
+
diff --git a/lib/common/writer/finish.c b/lib/common/writer/finish.c
new file mode 100644
index 0000000..e005628
--- /dev/null
+++ b/lib/common/writer/finish.c
@@ -0,0 +1,176 @@
+/* SPDX-License-Identifier: GPL-3.0-or-later */
+/*
+ * finish.c
+ *
+ * Copyright (C) 2019 David Oberhollenzer <goliath@infraroot.at>
+ */
+#include "simple_writer.h"
+#include "common.h"
+
+#include <stdlib.h>
+
+static void print_statistics(const sqfs_super_t *super,
+			     const sqfs_block_processor_t *blk,
+			     const sqfs_block_writer_t *wr)
+{
+	const sqfs_block_processor_stats_t *proc_stats;
+	sqfs_u64 bytes_written, blocks_written;
+	char read_sz[32], written_sz[32];
+	size_t ratio;
+
+	proc_stats = sqfs_block_processor_get_stats(blk);
+	blocks_written = wr->get_block_count(wr);
+
+	bytes_written = super->inode_table_start - sizeof(*super);
+
+	if (proc_stats->input_bytes_read > 0) {
+		ratio = (100 * bytes_written) / proc_stats->input_bytes_read;
+	} else {
+		ratio = 100;
+	}
+
+	print_size(proc_stats->input_bytes_read, read_sz, false);
+	print_size(bytes_written, written_sz, false);
+
+	fputs("---------------------------------------------------\n", stdout);
+	printf("Data bytes read: %s\n", read_sz);
+	printf("Data bytes written: %s\n", written_sz);
+	printf("Data compression ratio: " PRI_SZ "%%\n", ratio);
+	fputc('\n', stdout);
+
+	printf("Data blocks written: " PRI_U64 "\n", blocks_written);
+	printf("Out of which where fragment blocks: " PRI_U64 "\n",
+	       proc_stats->frag_block_count);
+
+	printf("Duplicate blocks omitted: " PRI_U64 "\n",
+	       proc_stats->data_block_count + proc_stats->frag_block_count -
+	       blocks_written);
+
+	printf("Sparse blocks omitted: " PRI_U64 "\n",
+	       proc_stats->sparse_block_count);
+	fputc('\n', stdout);
+
+	printf("Fragments actually written: " PRI_U64 "\n",
+	       proc_stats->actual_frag_count);
+	printf("Duplicated fragments omitted: " PRI_U64 "\n",
+	       proc_stats->total_frag_count - proc_stats->actual_frag_count);
+	printf("Total number of inodes: %u\n", super->inode_count);
+	printf("Number of unique group/user IDs: %u\n", super->id_count);
+	fputc('\n', stdout);
+}
+
+static int padd_sqfs(sqfs_file_t *file, sqfs_u64 size, size_t blocksize)
+{
+	size_t padd_sz = size % blocksize;
+	int status = -1;
+	sqfs_u8 *buffer;
+
+	if (padd_sz == 0)
+		return 0;
+
+	padd_sz = blocksize - padd_sz;
+
+	buffer = calloc(1, padd_sz);
+	if (buffer == NULL)
+		goto fail_errno;
+
+	if (file->write_at(file, file->get_size(file),
+			   buffer, padd_sz)) {
+		goto fail_errno;
+	}
+
+	status = 0;
+out:
+	free(buffer);
+	return status;
+fail_errno:
+	perror("padding output file to block size");
+	goto out;
+}
+
+int sqfs_writer_finish(sqfs_writer_t *sqfs, const sqfs_writer_cfg_t *cfg)
+{
+	int ret;
+
+	if (!cfg->quiet)
+		fputs("Waiting for remaining data blocks...\n", stdout);
+
+	ret = sqfs_block_processor_finish(sqfs->data);
+	if (ret) {
+		sqfs_perror(cfg->filename, "finishing data blocks", ret);
+		return -1;
+	}
+
+	if (!cfg->quiet)
+		fputs("Writing inodes and directories...\n", stdout);
+
+	sqfs->super.inode_count = sqfs->fs.unique_inode_count;
+
+	if (sqfs_serialize_fstree(cfg->filename, sqfs))
+		return -1;
+
+	if (!cfg->quiet)
+		fputs("Writing fragment table...\n", stdout);
+
+	ret = sqfs_frag_table_write(sqfs->fragtbl, sqfs->outfile,
+				    &sqfs->super, sqfs->cmp);
+	if (ret) {
+		sqfs_perror(cfg->filename, "writing fragment table", ret);
+		return -1;
+	}
+
+	if (cfg->exportable) {
+		if (!cfg->quiet)
+			fputs("Writing export table...\n", stdout);
+
+
+		ret = sqfs_dir_writer_write_export_table(sqfs->dirwr,
+						sqfs->outfile, sqfs->cmp,
+						sqfs->fs.root->inode_num,
+						sqfs->fs.root->inode_ref,
+						&sqfs->super);
+		if (ret)
+			return -1;
+	}
+
+	if (!cfg->quiet)
+		fputs("Writing ID table...\n", stdout);
+
+	ret = sqfs_id_table_write(sqfs->idtbl, sqfs->outfile,
+				  &sqfs->super, sqfs->cmp);
+	if (ret) {
+		sqfs_perror(cfg->filename, "writing ID table", ret);
+		return -1;
+	}
+
+	if (!cfg->no_xattr) {
+		if (!cfg->quiet)
+			fputs("Writing extended attributes...\n", stdout);
+
+		ret = sqfs_xattr_writer_flush(sqfs->xwr, sqfs->outfile,
+					      &sqfs->super, sqfs->cmp);
+		if (ret) {
+			sqfs_perror(cfg->filename,
+				    "writing extended attributes", ret);
+			return -1;
+		}
+	}
+
+	sqfs->super.bytes_used = sqfs->outfile->get_size(sqfs->outfile);
+
+	ret = sqfs_super_write(&sqfs->super, sqfs->outfile);
+	if (ret) {
+		sqfs_perror(cfg->filename, "updating super block", ret);
+		return -1;
+	}
+
+	if (padd_sqfs(sqfs->outfile, sqfs->super.bytes_used,
+		      cfg->devblksize)) {
+		return -1;
+	}
+
+	if (!cfg->quiet)
+		print_statistics(&sqfs->super, sqfs->data, sqfs->blkwr);
+
+	return 0;
+}
diff --git a/lib/common/writer/init.c b/lib/common/writer/init.c
new file mode 100644
index 0000000..d606d26
--- /dev/null
+++ b/lib/common/writer/init.c
@@ -0,0 +1,186 @@
+/* SPDX-License-Identifier: GPL-3.0-or-later */
+/*
+ * init.c
+ *
+ * Copyright (C) 2019 David Oberhollenzer <goliath@infraroot.at>
+ */
+#include "simple_writer.h"
+#include "compress_cli.h"
+#include "common.h"
+
+#include <string.h>
+#include <stdlib.h>
+
+#ifdef HAVE_SYS_SYSINFO_H
+#include <sys/sysinfo.h>
+
+static size_t os_get_num_jobs(void)
+{
+	int nprocs;
+
+	nprocs = get_nprocs_conf();
+	return nprocs < 1 ? 1 : nprocs;
+}
+#else
+static size_t os_get_num_jobs(void)
+{
+	return 1;
+}
+#endif
+
+void sqfs_writer_cfg_init(sqfs_writer_cfg_t *cfg)
+{
+	memset(cfg, 0, sizeof(*cfg));
+
+	cfg->num_jobs = os_get_num_jobs();
+	cfg->block_size = SQFS_DEFAULT_BLOCK_SIZE;
+	cfg->devblksize = SQFS_DEVBLK_SIZE;
+	cfg->comp_id = compressor_get_default();
+}
+
+int sqfs_writer_init(sqfs_writer_t *sqfs, const sqfs_writer_cfg_t *wrcfg)
+{
+	sqfs_compressor_config_t cfg;
+	int ret, flags;
+
+	sqfs->filename = wrcfg->filename;
+
+	if (compressor_cfg_init_options(&cfg, wrcfg->comp_id,
+					wrcfg->block_size,
+					wrcfg->comp_extra)) {
+		return -1;
+	}
+
+	sqfs->outfile = sqfs_open_file(wrcfg->filename, wrcfg->outmode);
+	if (sqfs->outfile == NULL) {
+		perror(wrcfg->filename);
+		return -1;
+	}
+
+	if (fstree_init(&sqfs->fs, wrcfg->fs_defaults))
+		goto fail_file;
+
+	ret = sqfs_compressor_create(&cfg, &sqfs->cmp);
+
+#ifdef WITH_LZO
+	if (cfg.id == SQFS_COMP_LZO) {
+		if (sqfs->cmp != NULL)
+			sqfs_destroy(sqfs->cmp);
+
+		ret = lzo_compressor_create(&cfg, &sqfs->cmp);
+	}
+#endif
+
+	if (ret != 0) {
+		sqfs_perror(wrcfg->filename, "creating compressor", ret);
+		goto fail_fs;
+	}
+
+	ret = sqfs_super_init(&sqfs->super, wrcfg->block_size,
+			      sqfs->fs.defaults.st_mtime, wrcfg->comp_id);
+	if (ret) {
+		sqfs_perror(wrcfg->filename, "initializing super block", ret);
+		goto fail_cmp;
+	}
+
+	ret = sqfs_super_write(&sqfs->super, sqfs->outfile);
+	if (ret) {
+		sqfs_perror(wrcfg->filename, "writing super block", ret);
+		goto fail_cmp;
+	}
+
+	ret = sqfs->cmp->write_options(sqfs->cmp, sqfs->outfile);
+	if (ret < 0) {
+		sqfs_perror(wrcfg->filename, "writing compressor options", ret);
+		goto fail_cmp;
+	}
+
+	if (ret > 0)
+		sqfs->super.flags |= SQFS_FLAG_COMPRESSOR_OPTIONS;
+
+	sqfs->blkwr = sqfs_block_writer_create(sqfs->outfile,
+					       wrcfg->devblksize, 0);
+	if (sqfs->blkwr == NULL) {
+		perror("creating block writer");
+		goto fail_cmp;
+	}
+
+	sqfs->fragtbl = sqfs_frag_table_create(0);
+	if (sqfs->fragtbl == NULL) {
+		perror("creating fragment table");
+		goto fail_blkwr;
+	}
+
+	sqfs->data = sqfs_block_processor_create(sqfs->super.block_size,
+						 sqfs->cmp, wrcfg->num_jobs,
+						 wrcfg->max_backlog,
+						 sqfs->blkwr, sqfs->fragtbl);
+	if (sqfs->data == NULL) {
+		perror("creating data block processor");
+		goto fail_fragtbl;
+	}
+
+	sqfs->idtbl = sqfs_id_table_create(0);
+	if (sqfs->idtbl == NULL) {
+		sqfs_perror(wrcfg->filename, "creating ID table",
+			    SQFS_ERROR_ALLOC);
+		goto fail_data;
+	}
+
+	if (!wrcfg->no_xattr) {
+		sqfs->xwr = sqfs_xattr_writer_create(0);
+
+		if (sqfs->xwr == NULL) {
+			sqfs_perror(wrcfg->filename, "creating xattr writer",
+				    SQFS_ERROR_ALLOC);
+			goto fail_id;
+		}
+	}
+
+	sqfs->im = sqfs_meta_writer_create(sqfs->outfile, sqfs->cmp, 0);
+	if (sqfs->im == NULL) {
+		fputs("Error creating inode meta data writer.\n", stderr);
+		goto fail_xwr;
+	}
+
+	sqfs->dm = sqfs_meta_writer_create(sqfs->outfile, sqfs->cmp,
+					   SQFS_META_WRITER_KEEP_IN_MEMORY);
+	if (sqfs->dm == NULL) {
+		fputs("Error creating directory meta data writer.\n", stderr);
+		goto fail_im;
+	}
+
+	flags = 0;
+	if (wrcfg->exportable)
+		flags |= SQFS_DIR_WRITER_CREATE_EXPORT_TABLE;
+
+	sqfs->dirwr = sqfs_dir_writer_create(sqfs->dm, flags);
+	if (sqfs->dirwr == NULL) {
+		fputs("Error creating directory table writer.\n", stderr);
+		goto fail_dm;
+	}
+
+	return 0;
+fail_dm:
+	sqfs_destroy(sqfs->dm);
+fail_im:
+	sqfs_destroy(sqfs->im);
+fail_xwr:
+	if (sqfs->xwr != NULL)
+		sqfs_destroy(sqfs->xwr);
+fail_id:
+	sqfs_destroy(sqfs->idtbl);
+fail_data:
+	sqfs_destroy(sqfs->data);
+fail_fragtbl:
+	sqfs_destroy(sqfs->fragtbl);
+fail_blkwr:
+	sqfs_destroy(sqfs->blkwr);
+fail_cmp:
+	sqfs_destroy(sqfs->cmp);
+fail_fs:
+	fstree_cleanup(&sqfs->fs);
+fail_file:
+	sqfs_destroy(sqfs->outfile);
+	return -1;
+}
diff --git a/lib/common/writer/serialize_fstree.c b/lib/common/writer/serialize_fstree.c
new file mode 100644
index 0000000..c69f0ea
--- /dev/null
+++ b/lib/common/writer/serialize_fstree.c
@@ -0,0 +1,199 @@
+/* SPDX-License-Identifier: GPL-3.0-or-later */
+/*
+ * serialize_fstree.c
+ *
+ * Copyright (C) 2019 David Oberhollenzer <goliath@infraroot.at>
+ */
+#include "common.h"
+
+#include <assert.h>
+#include <stdlib.h>
+#include <string.h>
+#include <stdio.h>
+
+static sqfs_inode_generic_t *tree_node_to_inode(tree_node_t *node)
+{
+	sqfs_inode_generic_t *inode;
+	size_t extra = 0;
+
+	if (S_ISLNK(node->mode))
+		extra = strlen(node->data.target);
+
+	inode = calloc(1, sizeof(*inode) + extra);
+	if (inode == NULL) {
+		perror("creating inode");
+		return NULL;
+	}
+
+	switch (node->mode & S_IFMT) {
+	case S_IFSOCK:
+		inode->base.type = SQFS_INODE_SOCKET;
+		inode->data.ipc.nlink = node->link_count;
+		break;
+	case S_IFIFO:
+		inode->base.type = SQFS_INODE_FIFO;
+		inode->data.ipc.nlink = node->link_count;
+		break;
+	case S_IFLNK:
+		inode->base.type = SQFS_INODE_SLINK;
+		inode->data.slink.nlink = node->link_count;
+		inode->data.slink.target_size = extra;
+		memcpy(inode->extra, node->data.target, extra);
+		break;
+	case S_IFBLK:
+		inode->base.type = SQFS_INODE_BDEV;
+		inode->data.dev.nlink = node->link_count;
+		inode->data.dev.devno = node->data.devno;
+		break;
+	case S_IFCHR:
+		inode->base.type = SQFS_INODE_CDEV;
+		inode->data.dev.nlink = node->link_count;
+		inode->data.dev.devno = node->data.devno;
+		break;
+	default:
+		assert(0);
+	}
+
+	return inode;
+}
+
+static sqfs_inode_generic_t *write_dir_entries(const char *filename,
+					       sqfs_dir_writer_t *dirw,
+					       tree_node_t *node)
+{
+	sqfs_u32 xattr, parent_inode;
+	sqfs_inode_generic_t *inode;
+	tree_node_t *it, *tgt;
+	int ret;
+
+	ret = sqfs_dir_writer_begin(dirw, 0);
+	if (ret)
+		goto fail;
+
+	for (it = node->data.dir.children; it != NULL; it = it->next) {
+		if (it->mode == FSTREE_MODE_HARD_LINK_RESOLVED) {
+			tgt = it->data.target_node;
+		} else {
+			tgt = it;
+		}
+
+		ret = sqfs_dir_writer_add_entry(dirw, it->name, tgt->inode_num,
+						tgt->inode_ref, tgt->mode);
+		if (ret)
+			goto fail;
+	}
+
+	ret = sqfs_dir_writer_end(dirw);
+	if (ret)
+		goto fail;
+
+	xattr = node->xattr_idx;
+	parent_inode = (node->parent == NULL) ? 0 : node->parent->inode_num;
+
+	inode = sqfs_dir_writer_create_inode(dirw, 0, xattr, parent_inode);
+	if (inode == NULL) {
+		ret = SQFS_ERROR_ALLOC;
+		goto fail;
+	}
+
+	if (inode->base.type == SQFS_INODE_DIR) {
+		inode->data.dir.nlink = node->link_count;
+	} else {
+		inode->data.dir_ext.nlink = node->link_count;
+	}
+
+	return inode;
+fail:
+	sqfs_perror(filename, "recoding directory entries", ret);
+	return NULL;
+}
+
+static int serialize_tree_node(const char *filename, sqfs_writer_t *wr,
+			       tree_node_t *n)
+{
+	sqfs_inode_generic_t *inode;
+	sqfs_u32 offset;
+	sqfs_u64 block;
+	int ret;
+
+	if (S_ISDIR(n->mode)) {
+		inode = write_dir_entries(filename, wr->dirwr, n);
+		ret = SQFS_ERROR_INTERNAL;
+	} else if (S_ISREG(n->mode)) {
+		inode = n->data.file.user_ptr;
+		n->data.file.user_ptr = NULL;
+		ret = SQFS_ERROR_INTERNAL;
+
+		if (inode->base.type == SQFS_INODE_FILE && n->link_count > 1) {
+			sqfs_inode_make_extended(inode);
+			inode->data.file_ext.nlink = n->link_count;
+		} else {
+			inode->data.file_ext.nlink = n->link_count;
+		}
+	} else {
+		inode = tree_node_to_inode(n);
+		ret = SQFS_ERROR_ALLOC;
+	}
+
+	if (inode == NULL)
+		return ret;
+
+	inode->base.mode = n->mode;
+	inode->base.mod_time = n->mod_time;
+	inode->base.inode_number = n->inode_num;
+
+	sqfs_inode_set_xattr_index(inode, n->xattr_idx);
+
+	ret = sqfs_id_table_id_to_index(wr->idtbl, n->uid,
+					&inode->base.uid_idx);
+	if (ret)
+		goto out;
+
+	ret = sqfs_id_table_id_to_index(wr->idtbl, n->gid,
+					&inode->base.gid_idx);
+	if (ret)
+		goto out;
+
+	sqfs_meta_writer_get_position(wr->im, &block, &offset);
+	n->inode_ref = (block << 16) | offset;
+
+	ret = sqfs_meta_writer_write_inode(wr->im, inode);
+out:
+	free(inode);
+	return ret;
+}
+
+int sqfs_serialize_fstree(const char *filename, sqfs_writer_t *wr)
+{
+	size_t i;
+	int ret;
+
+	wr->super.inode_table_start = wr->outfile->get_size(wr->outfile);
+
+	for (i = 0; i < wr->fs.unique_inode_count; ++i) {
+		ret = serialize_tree_node(filename, wr, wr->fs.inodes[i]);
+		if (ret)
+			goto out;
+	}
+
+	ret = sqfs_meta_writer_flush(wr->im);
+	if (ret)
+		goto out;
+
+	ret = sqfs_meta_writer_flush(wr->dm);
+	if (ret)
+		goto out;
+
+	wr->super.root_inode_ref = wr->fs.root->inode_ref;
+	wr->super.directory_table_start = wr->outfile->get_size(wr->outfile);
+
+	ret = sqfs_meta_write_write_to_file(wr->dm);
+	if (ret)
+		goto out;
+
+	ret = 0;
+out:
+	if (ret)
+		sqfs_perror(filename, "storing filesystem tree", ret);
+	return ret;
+}