summaryrefslogtreecommitdiff
path: root/lib/common/writer
diff options
context:
space:
mode:
Diffstat (limited to 'lib/common/writer')
-rw-r--r--lib/common/writer/cleanup.c40
-rw-r--r--lib/common/writer/finish.c176
-rw-r--r--lib/common/writer/init.c186
-rw-r--r--lib/common/writer/serialize_fstree.c199
4 files changed, 601 insertions, 0 deletions
diff --git a/lib/common/writer/cleanup.c b/lib/common/writer/cleanup.c
new file mode 100644
index 0000000..16e846b
--- /dev/null
+++ b/lib/common/writer/cleanup.c
@@ -0,0 +1,40 @@
+/* SPDX-License-Identifier: GPL-3.0-or-later */
+/*
+ * cleanup.c
+ *
+ * Copyright (C) 2019 David Oberhollenzer <goliath@infraroot.at>
+ */
+#include "simple_writer.h"
+
+#include <stdlib.h>
+
+void sqfs_writer_cleanup(sqfs_writer_t *sqfs, int status)
+{
+ if (sqfs->xwr != NULL)
+ sqfs_destroy(sqfs->xwr);
+
+ sqfs_destroy(sqfs->dirwr);
+ sqfs_destroy(sqfs->dm);
+ sqfs_destroy(sqfs->im);
+ sqfs_destroy(sqfs->idtbl);
+ sqfs_destroy(sqfs->data);
+ sqfs_destroy(sqfs->blkwr);
+ sqfs_destroy(sqfs->fragtbl);
+ sqfs_destroy(sqfs->cmp);
+ fstree_cleanup(&sqfs->fs);
+ sqfs_destroy(sqfs->outfile);
+
+ if (status != EXIT_SUCCESS) {
+#if defined(_WIN32) || defined(__WINDOWS__)
+ WCHAR *path = path_to_windows(sqfs->filename);
+
+ if (path != NULL)
+ DeleteFileW(path);
+
+ free(path);
+#else
+ unlink(sqfs->filename);
+#endif
+ }
+}
+
diff --git a/lib/common/writer/finish.c b/lib/common/writer/finish.c
new file mode 100644
index 0000000..e005628
--- /dev/null
+++ b/lib/common/writer/finish.c
@@ -0,0 +1,176 @@
+/* SPDX-License-Identifier: GPL-3.0-or-later */
+/*
+ * finish.c
+ *
+ * Copyright (C) 2019 David Oberhollenzer <goliath@infraroot.at>
+ */
+#include "simple_writer.h"
+#include "common.h"
+
+#include <stdlib.h>
+
+static void print_statistics(const sqfs_super_t *super,
+ const sqfs_block_processor_t *blk,
+ const sqfs_block_writer_t *wr)
+{
+ const sqfs_block_processor_stats_t *proc_stats;
+ sqfs_u64 bytes_written, blocks_written;
+ char read_sz[32], written_sz[32];
+ size_t ratio;
+
+ proc_stats = sqfs_block_processor_get_stats(blk);
+ blocks_written = wr->get_block_count(wr);
+
+ bytes_written = super->inode_table_start - sizeof(*super);
+
+ if (proc_stats->input_bytes_read > 0) {
+ ratio = (100 * bytes_written) / proc_stats->input_bytes_read;
+ } else {
+ ratio = 100;
+ }
+
+ print_size(proc_stats->input_bytes_read, read_sz, false);
+ print_size(bytes_written, written_sz, false);
+
+ fputs("---------------------------------------------------\n", stdout);
+ printf("Data bytes read: %s\n", read_sz);
+ printf("Data bytes written: %s\n", written_sz);
+ printf("Data compression ratio: " PRI_SZ "%%\n", ratio);
+ fputc('\n', stdout);
+
+ printf("Data blocks written: " PRI_U64 "\n", blocks_written);
+ printf("Out of which where fragment blocks: " PRI_U64 "\n",
+ proc_stats->frag_block_count);
+
+ printf("Duplicate blocks omitted: " PRI_U64 "\n",
+ proc_stats->data_block_count + proc_stats->frag_block_count -
+ blocks_written);
+
+ printf("Sparse blocks omitted: " PRI_U64 "\n",
+ proc_stats->sparse_block_count);
+ fputc('\n', stdout);
+
+ printf("Fragments actually written: " PRI_U64 "\n",
+ proc_stats->actual_frag_count);
+ printf("Duplicated fragments omitted: " PRI_U64 "\n",
+ proc_stats->total_frag_count - proc_stats->actual_frag_count);
+ printf("Total number of inodes: %u\n", super->inode_count);
+ printf("Number of unique group/user IDs: %u\n", super->id_count);
+ fputc('\n', stdout);
+}
+
+static int padd_sqfs(sqfs_file_t *file, sqfs_u64 size, size_t blocksize)
+{
+ size_t padd_sz = size % blocksize;
+ int status = -1;
+ sqfs_u8 *buffer;
+
+ if (padd_sz == 0)
+ return 0;
+
+ padd_sz = blocksize - padd_sz;
+
+ buffer = calloc(1, padd_sz);
+ if (buffer == NULL)
+ goto fail_errno;
+
+ if (file->write_at(file, file->get_size(file),
+ buffer, padd_sz)) {
+ goto fail_errno;
+ }
+
+ status = 0;
+out:
+ free(buffer);
+ return status;
+fail_errno:
+ perror("padding output file to block size");
+ goto out;
+}
+
+int sqfs_writer_finish(sqfs_writer_t *sqfs, const sqfs_writer_cfg_t *cfg)
+{
+ int ret;
+
+ if (!cfg->quiet)
+ fputs("Waiting for remaining data blocks...\n", stdout);
+
+ ret = sqfs_block_processor_finish(sqfs->data);
+ if (ret) {
+ sqfs_perror(cfg->filename, "finishing data blocks", ret);
+ return -1;
+ }
+
+ if (!cfg->quiet)
+ fputs("Writing inodes and directories...\n", stdout);
+
+ sqfs->super.inode_count = sqfs->fs.unique_inode_count;
+
+ if (sqfs_serialize_fstree(cfg->filename, sqfs))
+ return -1;
+
+ if (!cfg->quiet)
+ fputs("Writing fragment table...\n", stdout);
+
+ ret = sqfs_frag_table_write(sqfs->fragtbl, sqfs->outfile,
+ &sqfs->super, sqfs->cmp);
+ if (ret) {
+ sqfs_perror(cfg->filename, "writing fragment table", ret);
+ return -1;
+ }
+
+ if (cfg->exportable) {
+ if (!cfg->quiet)
+ fputs("Writing export table...\n", stdout);
+
+
+ ret = sqfs_dir_writer_write_export_table(sqfs->dirwr,
+ sqfs->outfile, sqfs->cmp,
+ sqfs->fs.root->inode_num,
+ sqfs->fs.root->inode_ref,
+ &sqfs->super);
+ if (ret)
+ return -1;
+ }
+
+ if (!cfg->quiet)
+ fputs("Writing ID table...\n", stdout);
+
+ ret = sqfs_id_table_write(sqfs->idtbl, sqfs->outfile,
+ &sqfs->super, sqfs->cmp);
+ if (ret) {
+ sqfs_perror(cfg->filename, "writing ID table", ret);
+ return -1;
+ }
+
+ if (!cfg->no_xattr) {
+ if (!cfg->quiet)
+ fputs("Writing extended attributes...\n", stdout);
+
+ ret = sqfs_xattr_writer_flush(sqfs->xwr, sqfs->outfile,
+ &sqfs->super, sqfs->cmp);
+ if (ret) {
+ sqfs_perror(cfg->filename,
+ "writing extended attributes", ret);
+ return -1;
+ }
+ }
+
+ sqfs->super.bytes_used = sqfs->outfile->get_size(sqfs->outfile);
+
+ ret = sqfs_super_write(&sqfs->super, sqfs->outfile);
+ if (ret) {
+ sqfs_perror(cfg->filename, "updating super block", ret);
+ return -1;
+ }
+
+ if (padd_sqfs(sqfs->outfile, sqfs->super.bytes_used,
+ cfg->devblksize)) {
+ return -1;
+ }
+
+ if (!cfg->quiet)
+ print_statistics(&sqfs->super, sqfs->data, sqfs->blkwr);
+
+ return 0;
+}
diff --git a/lib/common/writer/init.c b/lib/common/writer/init.c
new file mode 100644
index 0000000..d606d26
--- /dev/null
+++ b/lib/common/writer/init.c
@@ -0,0 +1,186 @@
+/* SPDX-License-Identifier: GPL-3.0-or-later */
+/*
+ * init.c
+ *
+ * Copyright (C) 2019 David Oberhollenzer <goliath@infraroot.at>
+ */
+#include "simple_writer.h"
+#include "compress_cli.h"
+#include "common.h"
+
+#include <string.h>
+#include <stdlib.h>
+
+#ifdef HAVE_SYS_SYSINFO_H
+#include <sys/sysinfo.h>
+
+static size_t os_get_num_jobs(void)
+{
+ int nprocs;
+
+ nprocs = get_nprocs_conf();
+ return nprocs < 1 ? 1 : nprocs;
+}
+#else
+static size_t os_get_num_jobs(void)
+{
+ return 1;
+}
+#endif
+
+void sqfs_writer_cfg_init(sqfs_writer_cfg_t *cfg)
+{
+ memset(cfg, 0, sizeof(*cfg));
+
+ cfg->num_jobs = os_get_num_jobs();
+ cfg->block_size = SQFS_DEFAULT_BLOCK_SIZE;
+ cfg->devblksize = SQFS_DEVBLK_SIZE;
+ cfg->comp_id = compressor_get_default();
+}
+
+int sqfs_writer_init(sqfs_writer_t *sqfs, const sqfs_writer_cfg_t *wrcfg)
+{
+ sqfs_compressor_config_t cfg;
+ int ret, flags;
+
+ sqfs->filename = wrcfg->filename;
+
+ if (compressor_cfg_init_options(&cfg, wrcfg->comp_id,
+ wrcfg->block_size,
+ wrcfg->comp_extra)) {
+ return -1;
+ }
+
+ sqfs->outfile = sqfs_open_file(wrcfg->filename, wrcfg->outmode);
+ if (sqfs->outfile == NULL) {
+ perror(wrcfg->filename);
+ return -1;
+ }
+
+ if (fstree_init(&sqfs->fs, wrcfg->fs_defaults))
+ goto fail_file;
+
+ ret = sqfs_compressor_create(&cfg, &sqfs->cmp);
+
+#ifdef WITH_LZO
+ if (cfg.id == SQFS_COMP_LZO) {
+ if (sqfs->cmp != NULL)
+ sqfs_destroy(sqfs->cmp);
+
+ ret = lzo_compressor_create(&cfg, &sqfs->cmp);
+ }
+#endif
+
+ if (ret != 0) {
+ sqfs_perror(wrcfg->filename, "creating compressor", ret);
+ goto fail_fs;
+ }
+
+ ret = sqfs_super_init(&sqfs->super, wrcfg->block_size,
+ sqfs->fs.defaults.st_mtime, wrcfg->comp_id);
+ if (ret) {
+ sqfs_perror(wrcfg->filename, "initializing super block", ret);
+ goto fail_cmp;
+ }
+
+ ret = sqfs_super_write(&sqfs->super, sqfs->outfile);
+ if (ret) {
+ sqfs_perror(wrcfg->filename, "writing super block", ret);
+ goto fail_cmp;
+ }
+
+ ret = sqfs->cmp->write_options(sqfs->cmp, sqfs->outfile);
+ if (ret < 0) {
+ sqfs_perror(wrcfg->filename, "writing compressor options", ret);
+ goto fail_cmp;
+ }
+
+ if (ret > 0)
+ sqfs->super.flags |= SQFS_FLAG_COMPRESSOR_OPTIONS;
+
+ sqfs->blkwr = sqfs_block_writer_create(sqfs->outfile,
+ wrcfg->devblksize, 0);
+ if (sqfs->blkwr == NULL) {
+ perror("creating block writer");
+ goto fail_cmp;
+ }
+
+ sqfs->fragtbl = sqfs_frag_table_create(0);
+ if (sqfs->fragtbl == NULL) {
+ perror("creating fragment table");
+ goto fail_blkwr;
+ }
+
+ sqfs->data = sqfs_block_processor_create(sqfs->super.block_size,
+ sqfs->cmp, wrcfg->num_jobs,
+ wrcfg->max_backlog,
+ sqfs->blkwr, sqfs->fragtbl);
+ if (sqfs->data == NULL) {
+ perror("creating data block processor");
+ goto fail_fragtbl;
+ }
+
+ sqfs->idtbl = sqfs_id_table_create(0);
+ if (sqfs->idtbl == NULL) {
+ sqfs_perror(wrcfg->filename, "creating ID table",
+ SQFS_ERROR_ALLOC);
+ goto fail_data;
+ }
+
+ if (!wrcfg->no_xattr) {
+ sqfs->xwr = sqfs_xattr_writer_create(0);
+
+ if (sqfs->xwr == NULL) {
+ sqfs_perror(wrcfg->filename, "creating xattr writer",
+ SQFS_ERROR_ALLOC);
+ goto fail_id;
+ }
+ }
+
+ sqfs->im = sqfs_meta_writer_create(sqfs->outfile, sqfs->cmp, 0);
+ if (sqfs->im == NULL) {
+ fputs("Error creating inode meta data writer.\n", stderr);
+ goto fail_xwr;
+ }
+
+ sqfs->dm = sqfs_meta_writer_create(sqfs->outfile, sqfs->cmp,
+ SQFS_META_WRITER_KEEP_IN_MEMORY);
+ if (sqfs->dm == NULL) {
+ fputs("Error creating directory meta data writer.\n", stderr);
+ goto fail_im;
+ }
+
+ flags = 0;
+ if (wrcfg->exportable)
+ flags |= SQFS_DIR_WRITER_CREATE_EXPORT_TABLE;
+
+ sqfs->dirwr = sqfs_dir_writer_create(sqfs->dm, flags);
+ if (sqfs->dirwr == NULL) {
+ fputs("Error creating directory table writer.\n", stderr);
+ goto fail_dm;
+ }
+
+ return 0;
+fail_dm:
+ sqfs_destroy(sqfs->dm);
+fail_im:
+ sqfs_destroy(sqfs->im);
+fail_xwr:
+ if (sqfs->xwr != NULL)
+ sqfs_destroy(sqfs->xwr);
+fail_id:
+ sqfs_destroy(sqfs->idtbl);
+fail_data:
+ sqfs_destroy(sqfs->data);
+fail_fragtbl:
+ sqfs_destroy(sqfs->fragtbl);
+fail_blkwr:
+ sqfs_destroy(sqfs->blkwr);
+fail_cmp:
+ sqfs_destroy(sqfs->cmp);
+fail_fs:
+ fstree_cleanup(&sqfs->fs);
+fail_file:
+ sqfs_destroy(sqfs->outfile);
+ return -1;
+}
diff --git a/lib/common/writer/serialize_fstree.c b/lib/common/writer/serialize_fstree.c
new file mode 100644
index 0000000..c69f0ea
--- /dev/null
+++ b/lib/common/writer/serialize_fstree.c
@@ -0,0 +1,199 @@
+/* SPDX-License-Identifier: GPL-3.0-or-later */
+/*
+ * serialize_fstree.c
+ *
+ * Copyright (C) 2019 David Oberhollenzer <goliath@infraroot.at>
+ */
+#include "common.h"
+
+#include <assert.h>
+#include <stdlib.h>
+#include <string.h>
+#include <stdio.h>
+
+static sqfs_inode_generic_t *tree_node_to_inode(tree_node_t *node)
+{
+ sqfs_inode_generic_t *inode;
+ size_t extra = 0;
+
+ if (S_ISLNK(node->mode))
+ extra = strlen(node->data.target);
+
+ inode = calloc(1, sizeof(*inode) + extra);
+ if (inode == NULL) {
+ perror("creating inode");
+ return NULL;
+ }
+
+ switch (node->mode & S_IFMT) {
+ case S_IFSOCK:
+ inode->base.type = SQFS_INODE_SOCKET;
+ inode->data.ipc.nlink = node->link_count;
+ break;
+ case S_IFIFO:
+ inode->base.type = SQFS_INODE_FIFO;
+ inode->data.ipc.nlink = node->link_count;
+ break;
+ case S_IFLNK:
+ inode->base.type = SQFS_INODE_SLINK;
+ inode->data.slink.nlink = node->link_count;
+ inode->data.slink.target_size = extra;
+ memcpy(inode->extra, node->data.target, extra);
+ break;
+ case S_IFBLK:
+ inode->base.type = SQFS_INODE_BDEV;
+ inode->data.dev.nlink = node->link_count;
+ inode->data.dev.devno = node->data.devno;
+ break;
+ case S_IFCHR:
+ inode->base.type = SQFS_INODE_CDEV;
+ inode->data.dev.nlink = node->link_count;
+ inode->data.dev.devno = node->data.devno;
+ break;
+ default:
+ assert(0);
+ }
+
+ return inode;
+}
+
+static sqfs_inode_generic_t *write_dir_entries(const char *filename,
+ sqfs_dir_writer_t *dirw,
+ tree_node_t *node)
+{
+ sqfs_u32 xattr, parent_inode;
+ sqfs_inode_generic_t *inode;
+ tree_node_t *it, *tgt;
+ int ret;
+
+ ret = sqfs_dir_writer_begin(dirw, 0);
+ if (ret)
+ goto fail;
+
+ for (it = node->data.dir.children; it != NULL; it = it->next) {
+ if (it->mode == FSTREE_MODE_HARD_LINK_RESOLVED) {
+ tgt = it->data.target_node;
+ } else {
+ tgt = it;
+ }
+
+ ret = sqfs_dir_writer_add_entry(dirw, it->name, tgt->inode_num,
+ tgt->inode_ref, tgt->mode);
+ if (ret)
+ goto fail;
+ }
+
+ ret = sqfs_dir_writer_end(dirw);
+ if (ret)
+ goto fail;
+
+ xattr = node->xattr_idx;
+ parent_inode = (node->parent == NULL) ? 0 : node->parent->inode_num;
+
+ inode = sqfs_dir_writer_create_inode(dirw, 0, xattr, parent_inode);
+ if (inode == NULL) {
+ ret = SQFS_ERROR_ALLOC;
+ goto fail;
+ }
+
+ if (inode->base.type == SQFS_INODE_DIR) {
+ inode->data.dir.nlink = node->link_count;
+ } else {
+ inode->data.dir_ext.nlink = node->link_count;
+ }
+
+ return inode;
+fail:
+ sqfs_perror(filename, "recoding directory entries", ret);
+ return NULL;
+}
+
+static int serialize_tree_node(const char *filename, sqfs_writer_t *wr,
+ tree_node_t *n)
+{
+ sqfs_inode_generic_t *inode;
+ sqfs_u32 offset;
+ sqfs_u64 block;
+ int ret;
+
+ if (S_ISDIR(n->mode)) {
+ inode = write_dir_entries(filename, wr->dirwr, n);
+ ret = SQFS_ERROR_INTERNAL;
+ } else if (S_ISREG(n->mode)) {
+ inode = n->data.file.user_ptr;
+ n->data.file.user_ptr = NULL;
+ ret = SQFS_ERROR_INTERNAL;
+
+ if (inode->base.type == SQFS_INODE_FILE && n->link_count > 1) {
+ sqfs_inode_make_extended(inode);
+ inode->data.file_ext.nlink = n->link_count;
+ } else {
+ inode->data.file_ext.nlink = n->link_count;
+ }
+ } else {
+ inode = tree_node_to_inode(n);
+ ret = SQFS_ERROR_ALLOC;
+ }
+
+ if (inode == NULL)
+ return ret;
+
+ inode->base.mode = n->mode;
+ inode->base.mod_time = n->mod_time;
+ inode->base.inode_number = n->inode_num;
+
+ sqfs_inode_set_xattr_index(inode, n->xattr_idx);
+
+ ret = sqfs_id_table_id_to_index(wr->idtbl, n->uid,
+ &inode->base.uid_idx);
+ if (ret)
+ goto out;
+
+ ret = sqfs_id_table_id_to_index(wr->idtbl, n->gid,
+ &inode->base.gid_idx);
+ if (ret)
+ goto out;
+
+ sqfs_meta_writer_get_position(wr->im, &block, &offset);
+ n->inode_ref = (block << 16) | offset;
+
+ ret = sqfs_meta_writer_write_inode(wr->im, inode);
+out:
+ free(inode);
+ return ret;
+}
+
+int sqfs_serialize_fstree(const char *filename, sqfs_writer_t *wr)
+{
+ size_t i;
+ int ret;
+
+ wr->super.inode_table_start = wr->outfile->get_size(wr->outfile);
+
+ for (i = 0; i < wr->fs.unique_inode_count; ++i) {
+ ret = serialize_tree_node(filename, wr, wr->fs.inodes[i]);
+ if (ret)
+ goto out;
+ }
+
+ ret = sqfs_meta_writer_flush(wr->im);
+ if (ret)
+ goto out;
+
+ ret = sqfs_meta_writer_flush(wr->dm);
+ if (ret)
+ goto out;
+
+ wr->super.root_inode_ref = wr->fs.root->inode_ref;
+ wr->super.directory_table_start = wr->outfile->get_size(wr->outfile);
+
+ ret = sqfs_meta_write_write_to_file(wr->dm);
+ if (ret)
+ goto out;
+
+ ret = 0;
+out:
+ if (ret)
+ sqfs_perror(filename, "storing filesystem tree", ret);
+ return ret;
+}
generated by cgit v1.2.3 (git 2.39.1) at 2024-11-08 04:57:04 +0000