4 files changed, 601 insertions, 0 deletions

diff --git a/lib/common/writer/cleanup.c b/lib/common/writer/cleanup.c
new file mode 100644
index 0000000..16e846b
--- /dev/null
+++ b/lib/common/writer/cleanup.c
@@ -0,0 +1,40 @@
+/* SPDX-License-Identifier: GPL-3.0-or-later */
+/*
+ * cleanup.c
+ *
+ * Copyright (C) 2019 David Oberhollenzer <goliath@infraroot.at>
+ */
+#include "simple_writer.h"
+
+#include <stdlib.h>
+
+void sqfs_writer_cleanup(sqfs_writer_t *sqfs, int status)
+{
+	if (sqfs->xwr != NULL)
+		sqfs_destroy(sqfs->xwr);
+
+	sqfs_destroy(sqfs->dirwr);
+	sqfs_destroy(sqfs->dm);
+	sqfs_destroy(sqfs->im);
+	sqfs_destroy(sqfs->idtbl);
+	sqfs_destroy(sqfs->data);
+	sqfs_destroy(sqfs->blkwr);
+	sqfs_destroy(sqfs->fragtbl);
+	sqfs_destroy(sqfs->cmp);
+	fstree_cleanup(&sqfs->fs);
+	sqfs_destroy(sqfs->outfile);
+
+	if (status != EXIT_SUCCESS) {
+#if defined(_WIN32) || defined(__WINDOWS__)
+		WCHAR *path = path_to_windows(sqfs->filename);
+
+		if (path != NULL)
+			DeleteFileW(path);
+
+		free(path);
+#else
+		unlink(sqfs->filename);
+#endif
+	}
+}
+
diff --git a/lib/common/writer/finish.c b/lib/common/writer/finish.c
new file mode 100644
index 0000000..e005628
--- /dev/null
+++ b/lib/common/writer/finish.c
@@ -0,0 +1,176 @@
+/* SPDX-License-Identifier: GPL-3.0-or-later */
+/*
+ * finish.c
+ *
+ * Copyright (C) 2019 David Oberhollenzer <goliath@infraroot.at>
+ */
+#include "simple_writer.h"
+#include "common.h"
+
+#include <stdlib.h>
+
+static void print_statistics(const sqfs_super_t *super,
+			     const sqfs_block_processor_t *blk,
+			     const sqfs_block_writer_t *wr)
+{
+	const sqfs_block_processor_stats_t *proc_stats;
+	sqfs_u64 bytes_written, blocks_written;
+	char read_sz[32], written_sz[32];
+	size_t ratio;
+
+	proc_stats = sqfs_block_processor_get_stats(blk);
+	blocks_written = wr->get_block_count(wr);
+
+	bytes_written = super->inode_table_start - sizeof(*super);
+
+	if (proc_stats->input_bytes_read > 0) {
+		ratio = (100 * bytes_written) / proc_stats->input_bytes_read;
+	} else {
+		ratio = 100;
+	}
+
+	print_size(proc_stats->input_bytes_read, read_sz, false);
+	print_size(bytes_written, written_sz, false);
+
+	fputs("---------------------------------------------------\n", stdout);
+	printf("Data bytes read: %s\n", read_sz);
+	printf("Data bytes written: %s\n", written_sz);
+	printf("Data compression ratio: " PRI_SZ "%%\n", ratio);
+	fputc('\n', stdout);
+
+	printf("Data blocks written: " PRI_U64 "\n", blocks_written);
+	printf("Out of which where fragment blocks: " PRI_U64 "\n",
+	       proc_stats->frag_block_count);
+
+	printf("Duplicate blocks omitted: " PRI_U64 "\n",
+	       proc_stats->data_block_count + proc_stats->frag_block_count -
+	       blocks_written);
+
+	printf("Sparse blocks omitted: " PRI_U64 "\n",
+	       proc_stats->sparse_block_count);
+	fputc('\n', stdout);
+
+	printf("Fragments actually written: " PRI_U64 "\n",
+	       proc_stats->actual_frag_count);
+	printf("Duplicated fragments omitted: " PRI_U64 "\n",
+	       proc_stats->total_frag_count - proc_stats->actual_frag_count);
+	printf("Total number of inodes: %u\n", super->inode_count);
+	printf("Number of unique group/user IDs: %u\n", super->id_count);
+	fputc('\n', stdout);
+}
+
+static int padd_sqfs(sqfs_file_t *file, sqfs_u64 size, size_t blocksize)
+{
+	size_t padd_sz = size % blocksize;
+	int status = -1;
+	sqfs_u8 *buffer;
+
+	if (padd_sz == 0)
+		return 0;
+
+	padd_sz = blocksize - padd_sz;
+
+	buffer = calloc(1, padd_sz);
+	if (buffer == NULL)
+		goto fail_errno;
+
+	if (file->write_at(file, file->get_size(file),
+			   buffer, padd_sz)) {
+		goto fail_errno;
+	}
+
+	status = 0;
+out:
+	free(buffer);
+	return status;
+fail_errno:
+	perror("padding output file to block size");
+	goto out;
+}
+
+int sqfs_writer_finish(sqfs_writer_t *sqfs, const sqfs_writer_cfg_t *cfg)
+{
+	int ret;
+
+	if (!cfg->quiet)
+		fputs("Waiting for remaining data blocks...\n", stdout);
+
+	ret = sqfs_block_processor_finish(sqfs->data);
+	if (ret) {
+		sqfs_perror(cfg->filename, "finishing data blocks", ret);
+		return -1;
+	}
+
+	if (!cfg->quiet)
+		fputs("Writing inodes and directories...\n", stdout);
+
+	sqfs->super.inode_count = sqfs->fs.unique_inode_count;
+
+	if (sqfs_serialize_fstree(cfg->filename, sqfs))
+		return -1;
+
+	if (!cfg->quiet)
+		fputs("Writing fragment table...\n", stdout);
+
+	ret = sqfs_frag_table_write(sqfs->fragtbl, sqfs->outfile,
+				    &sqfs->super, sqfs->cmp);
+	if (ret) {
+		sqfs_perror(cfg->filename, "writing fragment table", ret);
+		return -1;
+	}
+
+	if (cfg->exportable) {
+		if (!cfg->quiet)
+			fputs("Writing export table...\n", stdout);
+
+
+		ret = sqfs_dir_writer_write_export_table(sqfs->dirwr,
+						sqfs->outfile, sqfs->cmp,
+						sqfs->fs.root->inode_num,
+						sqfs->fs.root->inode_ref,
+						&sqfs->super);
+		if (ret)
+			return -1;
+	}
+
+	if (!cfg->quiet)
+		fputs("Writing ID table...\n", stdout);
+
+	ret = sqfs_id_table_write(sqfs->idtbl, sqfs->outfile,
+				  &sqfs->super, sqfs->cmp);
+	if (ret) {
+		sqfs_perror(cfg->filename, "writing ID table", ret);
+		return -1;
+	}
+
+	if (!cfg->no_xattr) {
+		if (!cfg->quiet)
+			fputs("Writing extended attributes...\n", stdout);
+
+		ret = sqfs_xattr_writer_flush(sqfs->xwr, sqfs->outfile,
+					      &sqfs->super, sqfs->cmp);
+		if (ret) {
+			sqfs_perror(cfg->filename,
+				    "writing extended attributes", ret);
+			return -1;
+		}
+	}
+
+	sqfs->super.bytes_used = sqfs->outfile->get_size(sqfs->outfile);
+
+	ret = sqfs_super_write(&sqfs->super, sqfs->outfile);
+	if (ret) {
+		sqfs_perror(cfg->filename, "updating super block", ret);
+		return -1;
+	}
+
+	if (padd_sqfs(sqfs->outfile, sqfs->super.bytes_used,
+		      cfg->devblksize)) {
+		return -1;
+	}
+
+	if (!cfg->quiet)
+		print_statistics(&sqfs->super, sqfs->data, sqfs->blkwr);
+
+	return 0;
+}
diff --git a/lib/common/writer/init.c b/lib/common/writer/init.c
new file mode 100644
index 0000000..d606d26
--- /dev/null
+++ b/lib/common/writer/init.c
@@ -0,0 +1,186 @@
+/* SPDX-License-Identifier: GPL-3.0-or-later */
+/*
+ * init.c
+ *
+ * Copyright (C) 2019 David Oberhollenzer <goliath@infraroot.at>
+ */
+#include "simple_writer.h"
+#include "compress_cli.h"
+#include "common.h"
+
+#include <string.h>
+#include <stdlib.h>
+
+#ifdef HAVE_SYS_SYSINFO_H
+#include <sys/sysinfo.h>
+
+static size_t os_get_num_jobs(void)
+{
+	int nprocs;
+
+	nprocs = get_nprocs_conf();
+	return nprocs < 1 ? 1 : nprocs;
+}
+#else
+static size_t os_get_num_jobs(void)
+{
+	return 1;
+}
+#endif
+
+void sqfs_writer_cfg_init(sqfs_writer_cfg_t *cfg)
+{
+	memset(cfg, 0, sizeof(*cfg));
+
+	cfg->num_jobs = os_get_num_jobs();
+	cfg->block_size = SQFS_DEFAULT_BLOCK_SIZE;
+	cfg->devblksize = SQFS_DEVBLK_SIZE;
+	cfg->comp_id = compressor_get_default();
+}
+
+int sqfs_writer_init(sqfs_writer_t *sqfs, const sqfs_writer_cfg_t *wrcfg)
+{
+	sqfs_compressor_config_t cfg;
+	int ret, flags;
+
+	sqfs->filename = wrcfg->filename;
+
+	if (compressor_cfg_init_options(&cfg, wrcfg->comp_id,
+					wrcfg->block_size,
+					wrcfg->comp_extra)) {
+		return -1;
+	}
+
+	sqfs->outfile = sqfs_open_file(wrcfg->filename, wrcfg->outmode);
+	if (sqfs->outfile == NULL) {
+		perror(wrcfg->filename);
+		return -1;
+	}
+
+	if (fstree_init(&sqfs->fs, wrcfg->fs_defaults))
+		goto fail_file;
+
+	ret = sqfs_compressor_create(&cfg, &sqfs->cmp);
+
+#ifdef WITH_LZO
+	if (cfg.id == SQFS_COMP_LZO) {
+		if (sqfs->cmp != NULL)
+			sqfs_destroy(sqfs->cmp);
+
+		ret = lzo_compressor_create(&cfg, &sqfs->cmp);
+	}
+#endif
+
+	if (ret != 0) {
+		sqfs_perror(wrcfg->filename, "creating compressor", ret);
+		goto fail_fs;
+	}
+
+	ret = sqfs_super_init(&sqfs->super, wrcfg->block_size,
+			      sqfs->fs.defaults.st_mtime, wrcfg->comp_id);
+	if (ret) {
+		sqfs_perror(wrcfg->filename, "initializing super block", ret);
+		goto fail_cmp;
+	}
+
+	ret = sqfs_super_write(&sqfs->super, sqfs->outfile);
+	if (ret) {
+		sqfs_perror(wrcfg->filename, "writing super block", ret);
+		goto fail_cmp;
+	}
+
+	ret = sqfs->cmp->write_options(sqfs->cmp, sqfs->outfile);
+	if (ret < 0) {
+		sqfs_perror(wrcfg->filename, "writing compressor options", ret);
+		goto fail_cmp;
+	}
+
+	if (ret > 0)
+		sqfs->super.flags |= SQFS_FLAG_COMPRESSOR_OPTIONS;
+
+	sqfs->blkwr = sqfs_block_writer_create(sqfs->outfile,
+					       wrcfg->devblksize, 0);
+	if (sqfs->blkwr == NULL) {
+		perror("creating block writer");
+		goto fail_cmp;
+	}
+
+	sqfs->fragtbl = sqfs_frag_table_create(0);
+	if (sqfs->fragtbl == NULL) {
+		perror("creating fragment table");
+		goto fail_blkwr;
+	}
+
+	sqfs->data = sqfs_block_processor_create(sqfs->super.block_size,
+						 sqfs->cmp, wrcfg->num_jobs,
+						 wrcfg->max_backlog,
+						 sqfs->blkwr, sqfs->fragtbl);
+	if (sqfs->data == NULL) {
+		perror("creating data block processor");
+		goto fail_fragtbl;
+	}
+
+	sqfs->idtbl = sqfs_id_table_create(0);
+	if (sqfs->idtbl == NULL) {
+		sqfs_perror(wrcfg->filename, "creating ID table",
+			    SQFS_ERROR_ALLOC);
+		goto fail_data;
+	}
+
+	if (!wrcfg->no_xattr) {
+		sqfs->xwr = sqfs_xattr_writer_create(0);
+
+		if (sqfs->xwr == NULL) {
+			sqfs_perror(wrcfg->filename, "creating xattr writer",
+				    SQFS_ERROR_ALLOC);
+			goto fail_id;
+		}
+	}
+
+	sqfs->im = sqfs_meta_writer_create(sqfs->outfile, sqfs->cmp, 0);
+	if (sqfs->im == NULL) {
+		fputs("Error creating inode meta data writer.\n", stderr);
+		goto fail_xwr;
+	}
+
+	sqfs->dm = sqfs_meta_writer_create(sqfs->outfile, sqfs->cmp,
+					   SQFS_META_WRITER_KEEP_IN_MEMORY);
+	if (sqfs->dm == NULL) {
+		fputs("Error creating directory meta data writer.\n", stderr);
+		goto fail_im;
+	}
+
+	flags = 0;
+	if (wrcfg->exportable)
+		flags |= SQFS_DIR_WRITER_CREATE_EXPORT_TABLE;
+
+	sqfs->dirwr = sqfs_dir_writer_create(sqfs->dm, flags);
+	if (sqfs->dirwr == NULL) {
+		fputs("Error creating directory table writer.\n", stderr);
+		goto fail_dm;
+	}
+
+	return 0;
+fail_dm:
+	sqfs_destroy(sqfs->dm);
+fail_im:
+	sqfs_destroy(sqfs->im);
+fail_xwr:
+	if (sqfs->xwr != NULL)
+		sqfs_destroy(sqfs->xwr);
+fail_id:
+	sqfs_destroy(sqfs->idtbl);
+fail_data:
+	sqfs_destroy(sqfs->data);
+fail_fragtbl:
+	sqfs_destroy(sqfs->fragtbl);
+fail_blkwr:
+	sqfs_destroy(sqfs->blkwr);
+fail_cmp:
+	sqfs_destroy(sqfs->cmp);
+fail_fs:
+	fstree_cleanup(&sqfs->fs);
+fail_file:
+	sqfs_destroy(sqfs->outfile);
+	return -1;
+}
diff --git a/lib/common/writer/serialize_fstree.c b/lib/common/writer/serialize_fstree.c
new file mode 100644
index 0000000..c69f0ea
--- /dev/null
+++ b/lib/common/writer/serialize_fstree.c
@@ -0,0 +1,199 @@
+/* SPDX-License-Identifier: GPL-3.0-or-later */
+/*
+ * serialize_fstree.c
+ *
+ * Copyright (C) 2019 David Oberhollenzer <goliath@infraroot.at>
+ */
+#include "common.h"
+
+#include <assert.h>
+#include <stdlib.h>
+#include <string.h>
+#include <stdio.h>
+
+static sqfs_inode_generic_t *tree_node_to_inode(tree_node_t *node)
+{
+	sqfs_inode_generic_t *inode;
+	size_t extra = 0;
+
+	if (S_ISLNK(node->mode))
+		extra = strlen(node->data.target);
+
+	inode = calloc(1, sizeof(*inode) + extra);
+	if (inode == NULL) {
+		perror("creating inode");
+		return NULL;
+	}
+
+	switch (node->mode & S_IFMT) {
+	case S_IFSOCK:
+		inode->base.type = SQFS_INODE_SOCKET;
+		inode->data.ipc.nlink = node->link_count;
+		break;
+	case S_IFIFO:
+		inode->base.type = SQFS_INODE_FIFO;
+		inode->data.ipc.nlink = node->link_count;
+		break;
+	case S_IFLNK:
+		inode->base.type = SQFS_INODE_SLINK;
+		inode->data.slink.nlink = node->link_count;
+		inode->data.slink.target_size = extra;
+		memcpy(inode->extra, node->data.target, extra);
+		break;
+	case S_IFBLK:
+		inode->base.type = SQFS_INODE_BDEV;
+		inode->data.dev.nlink = node->link_count;
+		inode->data.dev.devno = node->data.devno;
+		break;
+	case S_IFCHR:
+		inode->base.type = SQFS_INODE_CDEV;
+		inode->data.dev.nlink = node->link_count;
+		inode->data.dev.devno = node->data.devno;
+		break;
+	default:
+		assert(0);
+	}
+
+	return inode;
+}
+
+static sqfs_inode_generic_t *write_dir_entries(const char *filename,
+					       sqfs_dir_writer_t *dirw,
+					       tree_node_t *node)
+{
+	sqfs_u32 xattr, parent_inode;
+	sqfs_inode_generic_t *inode;
+	tree_node_t *it, *tgt;
+	int ret;
+
+	ret = sqfs_dir_writer_begin(dirw, 0);
+	if (ret)
+		goto fail;
+
+	for (it = node->data.dir.children; it != NULL; it = it->next) {
+		if (it->mode == FSTREE_MODE_HARD_LINK_RESOLVED) {
+			tgt = it->data.target_node;
+		} else {
+			tgt = it;
+		}
+
+		ret = sqfs_dir_writer_add_entry(dirw, it->name, tgt->inode_num,
+						tgt->inode_ref, tgt->mode);
+		if (ret)
+			goto fail;
+	}
+
+	ret = sqfs_dir_writer_end(dirw);
+	if (ret)
+		goto fail;
+
+	xattr = node->xattr_idx;
+	parent_inode = (node->parent == NULL) ? 0 : node->parent->inode_num;
+
+	inode = sqfs_dir_writer_create_inode(dirw, 0, xattr, parent_inode);
+	if (inode == NULL) {
+		ret = SQFS_ERROR_ALLOC;
+		goto fail;
+	}
+
+	if (inode->base.type == SQFS_INODE_DIR) {
+		inode->data.dir.nlink = node->link_count;
+	} else {
+		inode->data.dir_ext.nlink = node->link_count;
+	}
+
+	return inode;
+fail:
+	sqfs_perror(filename, "recoding directory entries", ret);
+	return NULL;
+}
+
+static int serialize_tree_node(const char *filename, sqfs_writer_t *wr,
+			       tree_node_t *n)
+{
+	sqfs_inode_generic_t *inode;
+	sqfs_u32 offset;
+	sqfs_u64 block;
+	int ret;
+
+	if (S_ISDIR(n->mode)) {
+		inode = write_dir_entries(filename, wr->dirwr, n);
+		ret = SQFS_ERROR_INTERNAL;
+	} else if (S_ISREG(n->mode)) {
+		inode = n->data.file.user_ptr;
+		n->data.file.user_ptr = NULL;
+		ret = SQFS_ERROR_INTERNAL;
+
+		if (inode->base.type == SQFS_INODE_FILE && n->link_count > 1) {
+			sqfs_inode_make_extended(inode);
+			inode->data.file_ext.nlink = n->link_count;
+		} else {
+			inode->data.file_ext.nlink = n->link_count;
+		}
+	} else {
+		inode = tree_node_to_inode(n);
+		ret = SQFS_ERROR_ALLOC;
+	}
+
+	if (inode == NULL)
+		return ret;
+
+	inode->base.mode = n->mode;
+	inode->base.mod_time = n->mod_time;
+	inode->base.inode_number = n->inode_num;
+
+	sqfs_inode_set_xattr_index(inode, n->xattr_idx);
+
+	ret = sqfs_id_table_id_to_index(wr->idtbl, n->uid,
+					&inode->base.uid_idx);
+	if (ret)
+		goto out;
+
+	ret = sqfs_id_table_id_to_index(wr->idtbl, n->gid,
+					&inode->base.gid_idx);
+	if (ret)
+		goto out;
+
+	sqfs_meta_writer_get_position(wr->im, &block, &offset);
+	n->inode_ref = (block << 16) | offset;
+
+	ret = sqfs_meta_writer_write_inode(wr->im, inode);
+out:
+	free(inode);
+	return ret;
+}
+
+int sqfs_serialize_fstree(const char *filename, sqfs_writer_t *wr)
+{
+	size_t i;
+	int ret;
+
+	wr->super.inode_table_start = wr->outfile->get_size(wr->outfile);
+
+	for (i = 0; i < wr->fs.unique_inode_count; ++i) {
+		ret = serialize_tree_node(filename, wr, wr->fs.inodes[i]);
+		if (ret)
+			goto out;
+	}
+
+	ret = sqfs_meta_writer_flush(wr->im);
+	if (ret)
+		goto out;
+
+	ret = sqfs_meta_writer_flush(wr->dm);
+	if (ret)
+		goto out;
+
+	wr->super.root_inode_ref = wr->fs.root->inode_ref;
+	wr->super.directory_table_start = wr->outfile->get_size(wr->outfile);
+
+	ret = sqfs_meta_write_write_to_file(wr->dm);
+	if (ret)
+		goto out;
+
+	ret = 0;
+out:
+	if (ret)
+		sqfs_perror(filename, "storing filesystem tree", ret);
+	return ret;
+}