From 411f659fe0140bacbd56f8503cda269816d4a887 Mon Sep 17 00:00:00 2001 From: David Oberhollenzer Date: Sat, 7 Nov 2020 16:04:40 +0100 Subject: Cleanup: libcommon: try to split up some of the larger files The sqsf_writer is moved to a separate header, as well as the compressor related stuff. The statistics function is moved into the writer code, as this is the only place that actually uses it. The writer code itself is split up into a hand full of file in their own subdirectory. Signed-off-by: David Oberhollenzer --- lib/common/Makemodule.am | 11 +- lib/common/comp_lzo.c | 2 +- lib/common/comp_opt.c | 2 + lib/common/compress.c | 5 +- lib/common/serialize_fstree.c | 199 --------------------- lib/common/statistics.c | 59 ------- lib/common/writer.c | 329 ----------------------------------- lib/common/writer/cleanup.c | 40 +++++ lib/common/writer/finish.c | 176 +++++++++++++++++++ lib/common/writer/init.c | 186 ++++++++++++++++++++ lib/common/writer/serialize_fstree.c | 199 +++++++++++++++++++++ 11 files changed, 615 insertions(+), 593 deletions(-) delete mode 100644 lib/common/serialize_fstree.c delete mode 100644 lib/common/statistics.c delete mode 100644 lib/common/writer.c create mode 100644 lib/common/writer/cleanup.c create mode 100644 lib/common/writer/finish.c create mode 100644 lib/common/writer/init.c create mode 100644 lib/common/writer/serialize_fstree.c (limited to 'lib') diff --git a/lib/common/Makemodule.am b/lib/common/Makemodule.am index e191e29..dd0a0d0 100644 --- a/lib/common/Makemodule.am +++ b/lib/common/Makemodule.am @@ -1,12 +1,15 @@ -libcommon_a_SOURCES = lib/common/serialize_fstree.c lib/common/statistics.c -libcommon_a_SOURCES += lib/common/inode_stat.c lib/common/hardlink.c +libcommon_a_SOURCES = lib/common/inode_stat.c lib/common/hardlink.c libcommon_a_SOURCES += lib/common/print_version.c lib/common/data_reader_dump.c libcommon_a_SOURCES += lib/common/compress.c lib/common/comp_opt.c libcommon_a_SOURCES += lib/common/data_writer.c include/common.h libcommon_a_SOURCES += lib/common/get_path.c lib/common/data_writer_ostream.c -libcommon_a_SOURCES += lib/common/writer.c lib/common/perror.c +libcommon_a_SOURCES += lib/common/perror.c libcommon_a_SOURCES += lib/common/mkdir_p.c lib/common/parse_size.c -libcommon_a_SOURCES += lib/common/print_size.c +libcommon_a_SOURCES += lib/common/print_size.c include/simple_writer.h +libcommon_a_SOURCES += include/compress_cli.h +libcommon_a_SOURCES += lib/common/writer/init.c lib/common/writer/cleanup.c +libcommon_a_SOURCES += lib/common/writer/serialize_fstree.c +libcommon_a_SOURCES += lib/common/writer/finish.c libcommon_a_CFLAGS = $(AM_CFLAGS) $(LZO_CFLAGS) if WITH_LZO diff --git a/lib/common/comp_lzo.c b/lib/common/comp_lzo.c index 3452647..21dc7b2 100644 --- a/lib/common/comp_lzo.c +++ b/lib/common/comp_lzo.c @@ -5,7 +5,7 @@ * Copyright (C) 2019 David Oberhollenzer */ #include "config.h" -#include "common.h" +#include "compress_cli.h" #include #include diff --git a/lib/common/comp_opt.c b/lib/common/comp_opt.c index f77ac20..a0d4cf7 100644 --- a/lib/common/comp_opt.c +++ b/lib/common/comp_opt.c @@ -4,11 +4,13 @@ * * Copyright (C) 2019 David Oberhollenzer */ +#include "config.h" #include "common.h" #include #include #include +#include #include typedef struct { diff --git a/lib/common/compress.c b/lib/common/compress.c index 9a66095..b11efbd 100644 --- a/lib/common/compress.c +++ b/lib/common/compress.c @@ -4,8 +4,11 @@ * * Copyright (C) 2019 David Oberhollenzer */ -#include "common.h" +#include "config.h" +#include "compress_cli.h" + #include +#include static int cmp_ids[] = { SQFS_COMP_XZ, diff --git a/lib/common/serialize_fstree.c b/lib/common/serialize_fstree.c deleted file mode 100644 index c69f0ea..0000000 --- a/lib/common/serialize_fstree.c +++ /dev/null @@ -1,199 +0,0 @@ -/* SPDX-License-Identifier: GPL-3.0-or-later */ -/* - * serialize_fstree.c - * - * Copyright (C) 2019 David Oberhollenzer - */ -#include "common.h" - -#include -#include -#include -#include - -static sqfs_inode_generic_t *tree_node_to_inode(tree_node_t *node) -{ - sqfs_inode_generic_t *inode; - size_t extra = 0; - - if (S_ISLNK(node->mode)) - extra = strlen(node->data.target); - - inode = calloc(1, sizeof(*inode) + extra); - if (inode == NULL) { - perror("creating inode"); - return NULL; - } - - switch (node->mode & S_IFMT) { - case S_IFSOCK: - inode->base.type = SQFS_INODE_SOCKET; - inode->data.ipc.nlink = node->link_count; - break; - case S_IFIFO: - inode->base.type = SQFS_INODE_FIFO; - inode->data.ipc.nlink = node->link_count; - break; - case S_IFLNK: - inode->base.type = SQFS_INODE_SLINK; - inode->data.slink.nlink = node->link_count; - inode->data.slink.target_size = extra; - memcpy(inode->extra, node->data.target, extra); - break; - case S_IFBLK: - inode->base.type = SQFS_INODE_BDEV; - inode->data.dev.nlink = node->link_count; - inode->data.dev.devno = node->data.devno; - break; - case S_IFCHR: - inode->base.type = SQFS_INODE_CDEV; - inode->data.dev.nlink = node->link_count; - inode->data.dev.devno = node->data.devno; - break; - default: - assert(0); - } - - return inode; -} - -static sqfs_inode_generic_t *write_dir_entries(const char *filename, - sqfs_dir_writer_t *dirw, - tree_node_t *node) -{ - sqfs_u32 xattr, parent_inode; - sqfs_inode_generic_t *inode; - tree_node_t *it, *tgt; - int ret; - - ret = sqfs_dir_writer_begin(dirw, 0); - if (ret) - goto fail; - - for (it = node->data.dir.children; it != NULL; it = it->next) { - if (it->mode == FSTREE_MODE_HARD_LINK_RESOLVED) { - tgt = it->data.target_node; - } else { - tgt = it; - } - - ret = sqfs_dir_writer_add_entry(dirw, it->name, tgt->inode_num, - tgt->inode_ref, tgt->mode); - if (ret) - goto fail; - } - - ret = sqfs_dir_writer_end(dirw); - if (ret) - goto fail; - - xattr = node->xattr_idx; - parent_inode = (node->parent == NULL) ? 0 : node->parent->inode_num; - - inode = sqfs_dir_writer_create_inode(dirw, 0, xattr, parent_inode); - if (inode == NULL) { - ret = SQFS_ERROR_ALLOC; - goto fail; - } - - if (inode->base.type == SQFS_INODE_DIR) { - inode->data.dir.nlink = node->link_count; - } else { - inode->data.dir_ext.nlink = node->link_count; - } - - return inode; -fail: - sqfs_perror(filename, "recoding directory entries", ret); - return NULL; -} - -static int serialize_tree_node(const char *filename, sqfs_writer_t *wr, - tree_node_t *n) -{ - sqfs_inode_generic_t *inode; - sqfs_u32 offset; - sqfs_u64 block; - int ret; - - if (S_ISDIR(n->mode)) { - inode = write_dir_entries(filename, wr->dirwr, n); - ret = SQFS_ERROR_INTERNAL; - } else if (S_ISREG(n->mode)) { - inode = n->data.file.user_ptr; - n->data.file.user_ptr = NULL; - ret = SQFS_ERROR_INTERNAL; - - if (inode->base.type == SQFS_INODE_FILE && n->link_count > 1) { - sqfs_inode_make_extended(inode); - inode->data.file_ext.nlink = n->link_count; - } else { - inode->data.file_ext.nlink = n->link_count; - } - } else { - inode = tree_node_to_inode(n); - ret = SQFS_ERROR_ALLOC; - } - - if (inode == NULL) - return ret; - - inode->base.mode = n->mode; - inode->base.mod_time = n->mod_time; - inode->base.inode_number = n->inode_num; - - sqfs_inode_set_xattr_index(inode, n->xattr_idx); - - ret = sqfs_id_table_id_to_index(wr->idtbl, n->uid, - &inode->base.uid_idx); - if (ret) - goto out; - - ret = sqfs_id_table_id_to_index(wr->idtbl, n->gid, - &inode->base.gid_idx); - if (ret) - goto out; - - sqfs_meta_writer_get_position(wr->im, &block, &offset); - n->inode_ref = (block << 16) | offset; - - ret = sqfs_meta_writer_write_inode(wr->im, inode); -out: - free(inode); - return ret; -} - -int sqfs_serialize_fstree(const char *filename, sqfs_writer_t *wr) -{ - size_t i; - int ret; - - wr->super.inode_table_start = wr->outfile->get_size(wr->outfile); - - for (i = 0; i < wr->fs.unique_inode_count; ++i) { - ret = serialize_tree_node(filename, wr, wr->fs.inodes[i]); - if (ret) - goto out; - } - - ret = sqfs_meta_writer_flush(wr->im); - if (ret) - goto out; - - ret = sqfs_meta_writer_flush(wr->dm); - if (ret) - goto out; - - wr->super.root_inode_ref = wr->fs.root->inode_ref; - wr->super.directory_table_start = wr->outfile->get_size(wr->outfile); - - ret = sqfs_meta_write_write_to_file(wr->dm); - if (ret) - goto out; - - ret = 0; -out: - if (ret) - sqfs_perror(filename, "storing filesystem tree", ret); - return ret; -} diff --git a/lib/common/statistics.c b/lib/common/statistics.c deleted file mode 100644 index 82e545d..0000000 --- a/lib/common/statistics.c +++ /dev/null @@ -1,59 +0,0 @@ -/* SPDX-License-Identifier: GPL-3.0-or-later */ -/* - * statistics.c - * - * Copyright (C) 2019 David Oberhollenzer - */ -#include "common.h" - -#include - -void sqfs_print_statistics(const sqfs_super_t *super, - const sqfs_block_processor_t *blk, - const sqfs_block_writer_t *wr) -{ - const sqfs_block_processor_stats_t *proc_stats; - sqfs_u64 bytes_written, blocks_written; - char read_sz[32], written_sz[32]; - size_t ratio; - - proc_stats = sqfs_block_processor_get_stats(blk); - blocks_written = wr->get_block_count(wr); - - bytes_written = super->inode_table_start - sizeof(*super); - - if (proc_stats->input_bytes_read > 0) { - ratio = (100 * bytes_written) / proc_stats->input_bytes_read; - } else { - ratio = 100; - } - - print_size(proc_stats->input_bytes_read, read_sz, false); - print_size(bytes_written, written_sz, false); - - fputs("---------------------------------------------------\n", stdout); - printf("Data bytes read: %s\n", read_sz); - printf("Data bytes written: %s\n", written_sz); - printf("Data compression ratio: " PRI_SZ "%%\n", ratio); - fputc('\n', stdout); - - printf("Data blocks written: " PRI_U64 "\n", blocks_written); - printf("Out of which where fragment blocks: " PRI_U64 "\n", - proc_stats->frag_block_count); - - printf("Duplicate blocks omitted: " PRI_U64 "\n", - proc_stats->data_block_count + proc_stats->frag_block_count - - blocks_written); - - printf("Sparse blocks omitted: " PRI_U64 "\n", - proc_stats->sparse_block_count); - fputc('\n', stdout); - - printf("Fragments actually written: " PRI_U64 "\n", - proc_stats->actual_frag_count); - printf("Duplicated fragments omitted: " PRI_U64 "\n", - proc_stats->total_frag_count - proc_stats->actual_frag_count); - printf("Total number of inodes: %u\n", super->inode_count); - printf("Number of unique group/user IDs: %u\n", super->id_count); - fputc('\n', stdout); -} diff --git a/lib/common/writer.c b/lib/common/writer.c deleted file mode 100644 index 9032a99..0000000 --- a/lib/common/writer.c +++ /dev/null @@ -1,329 +0,0 @@ -/* SPDX-License-Identifier: GPL-3.0-or-later */ -/* - * writer.c - * - * Copyright (C) 2019 David Oberhollenzer - */ -#include "common.h" - -#include -#include - -#ifdef HAVE_SYS_SYSINFO_H -#include - -static size_t os_get_num_jobs(void) -{ - int nprocs; - - nprocs = get_nprocs_conf(); - return nprocs < 1 ? 1 : nprocs; -} -#else -static size_t os_get_num_jobs(void) -{ - return 1; -} -#endif - -static int padd_sqfs(sqfs_file_t *file, sqfs_u64 size, size_t blocksize) -{ - size_t padd_sz = size % blocksize; - int status = -1; - sqfs_u8 *buffer; - - if (padd_sz == 0) - return 0; - - padd_sz = blocksize - padd_sz; - - buffer = calloc(1, padd_sz); - if (buffer == NULL) - goto fail_errno; - - if (file->write_at(file, file->get_size(file), - buffer, padd_sz)) { - goto fail_errno; - } - - status = 0; -out: - free(buffer); - return status; -fail_errno: - perror("padding output file to block size"); - goto out; -} - -void sqfs_writer_cfg_init(sqfs_writer_cfg_t *cfg) -{ - memset(cfg, 0, sizeof(*cfg)); - - cfg->num_jobs = os_get_num_jobs(); - cfg->block_size = SQFS_DEFAULT_BLOCK_SIZE; - cfg->devblksize = SQFS_DEVBLK_SIZE; - cfg->comp_id = compressor_get_default(); -} - -int sqfs_writer_init(sqfs_writer_t *sqfs, const sqfs_writer_cfg_t *wrcfg) -{ - sqfs_compressor_config_t cfg; - int ret, flags; - - sqfs->filename = wrcfg->filename; - - if (compressor_cfg_init_options(&cfg, wrcfg->comp_id, - wrcfg->block_size, - wrcfg->comp_extra)) { - return -1; - } - - sqfs->outfile = sqfs_open_file(wrcfg->filename, wrcfg->outmode); - if (sqfs->outfile == NULL) { - perror(wrcfg->filename); - return -1; - } - - if (fstree_init(&sqfs->fs, wrcfg->fs_defaults)) - goto fail_file; - - ret = sqfs_compressor_create(&cfg, &sqfs->cmp); - -#ifdef WITH_LZO - if (cfg.id == SQFS_COMP_LZO) { - if (sqfs->cmp != NULL) - sqfs_destroy(sqfs->cmp); - - ret = lzo_compressor_create(&cfg, &sqfs->cmp); - } -#endif - - if (ret != 0) { - sqfs_perror(wrcfg->filename, "creating compressor", ret); - goto fail_fs; - } - - ret = sqfs_super_init(&sqfs->super, wrcfg->block_size, - sqfs->fs.defaults.st_mtime, wrcfg->comp_id); - if (ret) { - sqfs_perror(wrcfg->filename, "initializing super block", ret); - goto fail_cmp; - } - - ret = sqfs_super_write(&sqfs->super, sqfs->outfile); - if (ret) { - sqfs_perror(wrcfg->filename, "writing super block", ret); - goto fail_cmp; - } - - ret = sqfs->cmp->write_options(sqfs->cmp, sqfs->outfile); - if (ret < 0) { - sqfs_perror(wrcfg->filename, "writing compressor options", ret); - goto fail_cmp; - } - - if (ret > 0) - sqfs->super.flags |= SQFS_FLAG_COMPRESSOR_OPTIONS; - - sqfs->blkwr = sqfs_block_writer_create(sqfs->outfile, - wrcfg->devblksize, 0); - if (sqfs->blkwr == NULL) { - perror("creating block writer"); - goto fail_cmp; - } - - sqfs->fragtbl = sqfs_frag_table_create(0); - if (sqfs->fragtbl == NULL) { - perror("creating fragment table"); - goto fail_blkwr; - } - - sqfs->data = sqfs_block_processor_create(sqfs->super.block_size, - sqfs->cmp, wrcfg->num_jobs, - wrcfg->max_backlog, - sqfs->blkwr, sqfs->fragtbl); - if (sqfs->data == NULL) { - perror("creating data block processor"); - goto fail_fragtbl; - } - - sqfs->idtbl = sqfs_id_table_create(0); - if (sqfs->idtbl == NULL) { - sqfs_perror(wrcfg->filename, "creating ID table", - SQFS_ERROR_ALLOC); - goto fail_data; - } - - if (!wrcfg->no_xattr) { - sqfs->xwr = sqfs_xattr_writer_create(0); - - if (sqfs->xwr == NULL) { - sqfs_perror(wrcfg->filename, "creating xattr writer", - SQFS_ERROR_ALLOC); - goto fail_id; - } - } - - sqfs->im = sqfs_meta_writer_create(sqfs->outfile, sqfs->cmp, 0); - if (sqfs->im == NULL) { - fputs("Error creating inode meta data writer.\n", stderr); - goto fail_xwr; - } - - sqfs->dm = sqfs_meta_writer_create(sqfs->outfile, sqfs->cmp, - SQFS_META_WRITER_KEEP_IN_MEMORY); - if (sqfs->dm == NULL) { - fputs("Error creating directory meta data writer.\n", stderr); - goto fail_im; - } - - flags = 0; - if (wrcfg->exportable) - flags |= SQFS_DIR_WRITER_CREATE_EXPORT_TABLE; - - sqfs->dirwr = sqfs_dir_writer_create(sqfs->dm, flags); - if (sqfs->dirwr == NULL) { - fputs("Error creating directory table writer.\n", stderr); - goto fail_dm; - } - - return 0; -fail_dm: - sqfs_destroy(sqfs->dm); -fail_im: - sqfs_destroy(sqfs->im); -fail_xwr: - if (sqfs->xwr != NULL) - sqfs_destroy(sqfs->xwr); -fail_id: - sqfs_destroy(sqfs->idtbl); -fail_data: - sqfs_destroy(sqfs->data); -fail_fragtbl: - sqfs_destroy(sqfs->fragtbl); -fail_blkwr: - sqfs_destroy(sqfs->blkwr); -fail_cmp: - sqfs_destroy(sqfs->cmp); -fail_fs: - fstree_cleanup(&sqfs->fs); -fail_file: - sqfs_destroy(sqfs->outfile); - return -1; -} - -int sqfs_writer_finish(sqfs_writer_t *sqfs, const sqfs_writer_cfg_t *cfg) -{ - int ret; - - if (!cfg->quiet) - fputs("Waiting for remaining data blocks...\n", stdout); - - ret = sqfs_block_processor_finish(sqfs->data); - if (ret) { - sqfs_perror(cfg->filename, "finishing data blocks", ret); - return -1; - } - - if (!cfg->quiet) - fputs("Writing inodes and directories...\n", stdout); - - sqfs->super.inode_count = sqfs->fs.unique_inode_count; - - if (sqfs_serialize_fstree(cfg->filename, sqfs)) - return -1; - - if (!cfg->quiet) - fputs("Writing fragment table...\n", stdout); - - ret = sqfs_frag_table_write(sqfs->fragtbl, sqfs->outfile, - &sqfs->super, sqfs->cmp); - if (ret) { - sqfs_perror(cfg->filename, "writing fragment table", ret); - return -1; - } - - if (cfg->exportable) { - if (!cfg->quiet) - fputs("Writing export table...\n", stdout); - - - ret = sqfs_dir_writer_write_export_table(sqfs->dirwr, - sqfs->outfile, sqfs->cmp, - sqfs->fs.root->inode_num, - sqfs->fs.root->inode_ref, - &sqfs->super); - if (ret) - return -1; - } - - if (!cfg->quiet) - fputs("Writing ID table...\n", stdout); - - ret = sqfs_id_table_write(sqfs->idtbl, sqfs->outfile, - &sqfs->super, sqfs->cmp); - if (ret) { - sqfs_perror(cfg->filename, "writing ID table", ret); - return -1; - } - - if (!cfg->no_xattr) { - if (!cfg->quiet) - fputs("Writing extended attributes...\n", stdout); - - ret = sqfs_xattr_writer_flush(sqfs->xwr, sqfs->outfile, - &sqfs->super, sqfs->cmp); - if (ret) { - sqfs_perror(cfg->filename, "writing extended attributes", ret); - return -1; - } - } - - sqfs->super.bytes_used = sqfs->outfile->get_size(sqfs->outfile); - - ret = sqfs_super_write(&sqfs->super, sqfs->outfile); - if (ret) { - sqfs_perror(cfg->filename, "updating super block", ret); - return -1; - } - - if (padd_sqfs(sqfs->outfile, sqfs->super.bytes_used, - cfg->devblksize)) { - return -1; - } - - if (!cfg->quiet) - sqfs_print_statistics(&sqfs->super, sqfs->data, sqfs->blkwr); - - return 0; -} - -void sqfs_writer_cleanup(sqfs_writer_t *sqfs, int status) -{ - if (sqfs->xwr != NULL) - sqfs_destroy(sqfs->xwr); - - sqfs_destroy(sqfs->dirwr); - sqfs_destroy(sqfs->dm); - sqfs_destroy(sqfs->im); - sqfs_destroy(sqfs->idtbl); - sqfs_destroy(sqfs->data); - sqfs_destroy(sqfs->blkwr); - sqfs_destroy(sqfs->fragtbl); - sqfs_destroy(sqfs->cmp); - fstree_cleanup(&sqfs->fs); - sqfs_destroy(sqfs->outfile); - - if (status != EXIT_SUCCESS) { -#if defined(_WIN32) || defined(__WINDOWS__) - WCHAR *path = path_to_windows(sqfs->filename); - - if (path != NULL) - DeleteFileW(path); - - free(path); -#else - unlink(sqfs->filename); -#endif - } -} diff --git a/lib/common/writer/cleanup.c b/lib/common/writer/cleanup.c new file mode 100644 index 0000000..16e846b --- /dev/null +++ b/lib/common/writer/cleanup.c @@ -0,0 +1,40 @@ +/* SPDX-License-Identifier: GPL-3.0-or-later */ +/* + * cleanup.c + * + * Copyright (C) 2019 David Oberhollenzer + */ +#include "simple_writer.h" + +#include + +void sqfs_writer_cleanup(sqfs_writer_t *sqfs, int status) +{ + if (sqfs->xwr != NULL) + sqfs_destroy(sqfs->xwr); + + sqfs_destroy(sqfs->dirwr); + sqfs_destroy(sqfs->dm); + sqfs_destroy(sqfs->im); + sqfs_destroy(sqfs->idtbl); + sqfs_destroy(sqfs->data); + sqfs_destroy(sqfs->blkwr); + sqfs_destroy(sqfs->fragtbl); + sqfs_destroy(sqfs->cmp); + fstree_cleanup(&sqfs->fs); + sqfs_destroy(sqfs->outfile); + + if (status != EXIT_SUCCESS) { +#if defined(_WIN32) || defined(__WINDOWS__) + WCHAR *path = path_to_windows(sqfs->filename); + + if (path != NULL) + DeleteFileW(path); + + free(path); +#else + unlink(sqfs->filename); +#endif + } +} + diff --git a/lib/common/writer/finish.c b/lib/common/writer/finish.c new file mode 100644 index 0000000..e005628 --- /dev/null +++ b/lib/common/writer/finish.c @@ -0,0 +1,176 @@ +/* SPDX-License-Identifier: GPL-3.0-or-later */ +/* + * finish.c + * + * Copyright (C) 2019 David Oberhollenzer + */ +#include "simple_writer.h" +#include "common.h" + +#include + +static void print_statistics(const sqfs_super_t *super, + const sqfs_block_processor_t *blk, + const sqfs_block_writer_t *wr) +{ + const sqfs_block_processor_stats_t *proc_stats; + sqfs_u64 bytes_written, blocks_written; + char read_sz[32], written_sz[32]; + size_t ratio; + + proc_stats = sqfs_block_processor_get_stats(blk); + blocks_written = wr->get_block_count(wr); + + bytes_written = super->inode_table_start - sizeof(*super); + + if (proc_stats->input_bytes_read > 0) { + ratio = (100 * bytes_written) / proc_stats->input_bytes_read; + } else { + ratio = 100; + } + + print_size(proc_stats->input_bytes_read, read_sz, false); + print_size(bytes_written, written_sz, false); + + fputs("---------------------------------------------------\n", stdout); + printf("Data bytes read: %s\n", read_sz); + printf("Data bytes written: %s\n", written_sz); + printf("Data compression ratio: " PRI_SZ "%%\n", ratio); + fputc('\n', stdout); + + printf("Data blocks written: " PRI_U64 "\n", blocks_written); + printf("Out of which where fragment blocks: " PRI_U64 "\n", + proc_stats->frag_block_count); + + printf("Duplicate blocks omitted: " PRI_U64 "\n", + proc_stats->data_block_count + proc_stats->frag_block_count - + blocks_written); + + printf("Sparse blocks omitted: " PRI_U64 "\n", + proc_stats->sparse_block_count); + fputc('\n', stdout); + + printf("Fragments actually written: " PRI_U64 "\n", + proc_stats->actual_frag_count); + printf("Duplicated fragments omitted: " PRI_U64 "\n", + proc_stats->total_frag_count - proc_stats->actual_frag_count); + printf("Total number of inodes: %u\n", super->inode_count); + printf("Number of unique group/user IDs: %u\n", super->id_count); + fputc('\n', stdout); +} + +static int padd_sqfs(sqfs_file_t *file, sqfs_u64 size, size_t blocksize) +{ + size_t padd_sz = size % blocksize; + int status = -1; + sqfs_u8 *buffer; + + if (padd_sz == 0) + return 0; + + padd_sz = blocksize - padd_sz; + + buffer = calloc(1, padd_sz); + if (buffer == NULL) + goto fail_errno; + + if (file->write_at(file, file->get_size(file), + buffer, padd_sz)) { + goto fail_errno; + } + + status = 0; +out: + free(buffer); + return status; +fail_errno: + perror("padding output file to block size"); + goto out; +} + +int sqfs_writer_finish(sqfs_writer_t *sqfs, const sqfs_writer_cfg_t *cfg) +{ + int ret; + + if (!cfg->quiet) + fputs("Waiting for remaining data blocks...\n", stdout); + + ret = sqfs_block_processor_finish(sqfs->data); + if (ret) { + sqfs_perror(cfg->filename, "finishing data blocks", ret); + return -1; + } + + if (!cfg->quiet) + fputs("Writing inodes and directories...\n", stdout); + + sqfs->super.inode_count = sqfs->fs.unique_inode_count; + + if (sqfs_serialize_fstree(cfg->filename, sqfs)) + return -1; + + if (!cfg->quiet) + fputs("Writing fragment table...\n", stdout); + + ret = sqfs_frag_table_write(sqfs->fragtbl, sqfs->outfile, + &sqfs->super, sqfs->cmp); + if (ret) { + sqfs_perror(cfg->filename, "writing fragment table", ret); + return -1; + } + + if (cfg->exportable) { + if (!cfg->quiet) + fputs("Writing export table...\n", stdout); + + + ret = sqfs_dir_writer_write_export_table(sqfs->dirwr, + sqfs->outfile, sqfs->cmp, + sqfs->fs.root->inode_num, + sqfs->fs.root->inode_ref, + &sqfs->super); + if (ret) + return -1; + } + + if (!cfg->quiet) + fputs("Writing ID table...\n", stdout); + + ret = sqfs_id_table_write(sqfs->idtbl, sqfs->outfile, + &sqfs->super, sqfs->cmp); + if (ret) { + sqfs_perror(cfg->filename, "writing ID table", ret); + return -1; + } + + if (!cfg->no_xattr) { + if (!cfg->quiet) + fputs("Writing extended attributes...\n", stdout); + + ret = sqfs_xattr_writer_flush(sqfs->xwr, sqfs->outfile, + &sqfs->super, sqfs->cmp); + if (ret) { + sqfs_perror(cfg->filename, + "writing extended attributes", ret); + return -1; + } + } + + sqfs->super.bytes_used = sqfs->outfile->get_size(sqfs->outfile); + + ret = sqfs_super_write(&sqfs->super, sqfs->outfile); + if (ret) { + sqfs_perror(cfg->filename, "updating super block", ret); + return -1; + } + + if (padd_sqfs(sqfs->outfile, sqfs->super.bytes_used, + cfg->devblksize)) { + return -1; + } + + if (!cfg->quiet) + print_statistics(&sqfs->super, sqfs->data, sqfs->blkwr); + + return 0; +} diff --git a/lib/common/writer/init.c b/lib/common/writer/init.c new file mode 100644 index 0000000..d606d26 --- /dev/null +++ b/lib/common/writer/init.c @@ -0,0 +1,186 @@ +/* SPDX-License-Identifier: GPL-3.0-or-later */ +/* + * init.c + * + * Copyright (C) 2019 David Oberhollenzer + */ +#include "simple_writer.h" +#include "compress_cli.h" +#include "common.h" + +#include +#include + +#ifdef HAVE_SYS_SYSINFO_H +#include + +static size_t os_get_num_jobs(void) +{ + int nprocs; + + nprocs = get_nprocs_conf(); + return nprocs < 1 ? 1 : nprocs; +} +#else +static size_t os_get_num_jobs(void) +{ + return 1; +} +#endif + +void sqfs_writer_cfg_init(sqfs_writer_cfg_t *cfg) +{ + memset(cfg, 0, sizeof(*cfg)); + + cfg->num_jobs = os_get_num_jobs(); + cfg->block_size = SQFS_DEFAULT_BLOCK_SIZE; + cfg->devblksize = SQFS_DEVBLK_SIZE; + cfg->comp_id = compressor_get_default(); +} + +int sqfs_writer_init(sqfs_writer_t *sqfs, const sqfs_writer_cfg_t *wrcfg) +{ + sqfs_compressor_config_t cfg; + int ret, flags; + + sqfs->filename = wrcfg->filename; + + if (compressor_cfg_init_options(&cfg, wrcfg->comp_id, + wrcfg->block_size, + wrcfg->comp_extra)) { + return -1; + } + + sqfs->outfile = sqfs_open_file(wrcfg->filename, wrcfg->outmode); + if (sqfs->outfile == NULL) { + perror(wrcfg->filename); + return -1; + } + + if (fstree_init(&sqfs->fs, wrcfg->fs_defaults)) + goto fail_file; + + ret = sqfs_compressor_create(&cfg, &sqfs->cmp); + +#ifdef WITH_LZO + if (cfg.id == SQFS_COMP_LZO) { + if (sqfs->cmp != NULL) + sqfs_destroy(sqfs->cmp); + + ret = lzo_compressor_create(&cfg, &sqfs->cmp); + } +#endif + + if (ret != 0) { + sqfs_perror(wrcfg->filename, "creating compressor", ret); + goto fail_fs; + } + + ret = sqfs_super_init(&sqfs->super, wrcfg->block_size, + sqfs->fs.defaults.st_mtime, wrcfg->comp_id); + if (ret) { + sqfs_perror(wrcfg->filename, "initializing super block", ret); + goto fail_cmp; + } + + ret = sqfs_super_write(&sqfs->super, sqfs->outfile); + if (ret) { + sqfs_perror(wrcfg->filename, "writing super block", ret); + goto fail_cmp; + } + + ret = sqfs->cmp->write_options(sqfs->cmp, sqfs->outfile); + if (ret < 0) { + sqfs_perror(wrcfg->filename, "writing compressor options", ret); + goto fail_cmp; + } + + if (ret > 0) + sqfs->super.flags |= SQFS_FLAG_COMPRESSOR_OPTIONS; + + sqfs->blkwr = sqfs_block_writer_create(sqfs->outfile, + wrcfg->devblksize, 0); + if (sqfs->blkwr == NULL) { + perror("creating block writer"); + goto fail_cmp; + } + + sqfs->fragtbl = sqfs_frag_table_create(0); + if (sqfs->fragtbl == NULL) { + perror("creating fragment table"); + goto fail_blkwr; + } + + sqfs->data = sqfs_block_processor_create(sqfs->super.block_size, + sqfs->cmp, wrcfg->num_jobs, + wrcfg->max_backlog, + sqfs->blkwr, sqfs->fragtbl); + if (sqfs->data == NULL) { + perror("creating data block processor"); + goto fail_fragtbl; + } + + sqfs->idtbl = sqfs_id_table_create(0); + if (sqfs->idtbl == NULL) { + sqfs_perror(wrcfg->filename, "creating ID table", + SQFS_ERROR_ALLOC); + goto fail_data; + } + + if (!wrcfg->no_xattr) { + sqfs->xwr = sqfs_xattr_writer_create(0); + + if (sqfs->xwr == NULL) { + sqfs_perror(wrcfg->filename, "creating xattr writer", + SQFS_ERROR_ALLOC); + goto fail_id; + } + } + + sqfs->im = sqfs_meta_writer_create(sqfs->outfile, sqfs->cmp, 0); + if (sqfs->im == NULL) { + fputs("Error creating inode meta data writer.\n", stderr); + goto fail_xwr; + } + + sqfs->dm = sqfs_meta_writer_create(sqfs->outfile, sqfs->cmp, + SQFS_META_WRITER_KEEP_IN_MEMORY); + if (sqfs->dm == NULL) { + fputs("Error creating directory meta data writer.\n", stderr); + goto fail_im; + } + + flags = 0; + if (wrcfg->exportable) + flags |= SQFS_DIR_WRITER_CREATE_EXPORT_TABLE; + + sqfs->dirwr = sqfs_dir_writer_create(sqfs->dm, flags); + if (sqfs->dirwr == NULL) { + fputs("Error creating directory table writer.\n", stderr); + goto fail_dm; + } + + return 0; +fail_dm: + sqfs_destroy(sqfs->dm); +fail_im: + sqfs_destroy(sqfs->im); +fail_xwr: + if (sqfs->xwr != NULL) + sqfs_destroy(sqfs->xwr); +fail_id: + sqfs_destroy(sqfs->idtbl); +fail_data: + sqfs_destroy(sqfs->data); +fail_fragtbl: + sqfs_destroy(sqfs->fragtbl); +fail_blkwr: + sqfs_destroy(sqfs->blkwr); +fail_cmp: + sqfs_destroy(sqfs->cmp); +fail_fs: + fstree_cleanup(&sqfs->fs); +fail_file: + sqfs_destroy(sqfs->outfile); + return -1; +} diff --git a/lib/common/writer/serialize_fstree.c b/lib/common/writer/serialize_fstree.c new file mode 100644 index 0000000..c69f0ea --- /dev/null +++ b/lib/common/writer/serialize_fstree.c @@ -0,0 +1,199 @@ +/* SPDX-License-Identifier: GPL-3.0-or-later */ +/* + * serialize_fstree.c + * + * Copyright (C) 2019 David Oberhollenzer + */ +#include "common.h" + +#include +#include +#include +#include + +static sqfs_inode_generic_t *tree_node_to_inode(tree_node_t *node) +{ + sqfs_inode_generic_t *inode; + size_t extra = 0; + + if (S_ISLNK(node->mode)) + extra = strlen(node->data.target); + + inode = calloc(1, sizeof(*inode) + extra); + if (inode == NULL) { + perror("creating inode"); + return NULL; + } + + switch (node->mode & S_IFMT) { + case S_IFSOCK: + inode->base.type = SQFS_INODE_SOCKET; + inode->data.ipc.nlink = node->link_count; + break; + case S_IFIFO: + inode->base.type = SQFS_INODE_FIFO; + inode->data.ipc.nlink = node->link_count; + break; + case S_IFLNK: + inode->base.type = SQFS_INODE_SLINK; + inode->data.slink.nlink = node->link_count; + inode->data.slink.target_size = extra; + memcpy(inode->extra, node->data.target, extra); + break; + case S_IFBLK: + inode->base.type = SQFS_INODE_BDEV; + inode->data.dev.nlink = node->link_count; + inode->data.dev.devno = node->data.devno; + break; + case S_IFCHR: + inode->base.type = SQFS_INODE_CDEV; + inode->data.dev.nlink = node->link_count; + inode->data.dev.devno = node->data.devno; + break; + default: + assert(0); + } + + return inode; +} + +static sqfs_inode_generic_t *write_dir_entries(const char *filename, + sqfs_dir_writer_t *dirw, + tree_node_t *node) +{ + sqfs_u32 xattr, parent_inode; + sqfs_inode_generic_t *inode; + tree_node_t *it, *tgt; + int ret; + + ret = sqfs_dir_writer_begin(dirw, 0); + if (ret) + goto fail; + + for (it = node->data.dir.children; it != NULL; it = it->next) { + if (it->mode == FSTREE_MODE_HARD_LINK_RESOLVED) { + tgt = it->data.target_node; + } else { + tgt = it; + } + + ret = sqfs_dir_writer_add_entry(dirw, it->name, tgt->inode_num, + tgt->inode_ref, tgt->mode); + if (ret) + goto fail; + } + + ret = sqfs_dir_writer_end(dirw); + if (ret) + goto fail; + + xattr = node->xattr_idx; + parent_inode = (node->parent == NULL) ? 0 : node->parent->inode_num; + + inode = sqfs_dir_writer_create_inode(dirw, 0, xattr, parent_inode); + if (inode == NULL) { + ret = SQFS_ERROR_ALLOC; + goto fail; + } + + if (inode->base.type == SQFS_INODE_DIR) { + inode->data.dir.nlink = node->link_count; + } else { + inode->data.dir_ext.nlink = node->link_count; + } + + return inode; +fail: + sqfs_perror(filename, "recoding directory entries", ret); + return NULL; +} + +static int serialize_tree_node(const char *filename, sqfs_writer_t *wr, + tree_node_t *n) +{ + sqfs_inode_generic_t *inode; + sqfs_u32 offset; + sqfs_u64 block; + int ret; + + if (S_ISDIR(n->mode)) { + inode = write_dir_entries(filename, wr->dirwr, n); + ret = SQFS_ERROR_INTERNAL; + } else if (S_ISREG(n->mode)) { + inode = n->data.file.user_ptr; + n->data.file.user_ptr = NULL; + ret = SQFS_ERROR_INTERNAL; + + if (inode->base.type == SQFS_INODE_FILE && n->link_count > 1) { + sqfs_inode_make_extended(inode); + inode->data.file_ext.nlink = n->link_count; + } else { + inode->data.file_ext.nlink = n->link_count; + } + } else { + inode = tree_node_to_inode(n); + ret = SQFS_ERROR_ALLOC; + } + + if (inode == NULL) + return ret; + + inode->base.mode = n->mode; + inode->base.mod_time = n->mod_time; + inode->base.inode_number = n->inode_num; + + sqfs_inode_set_xattr_index(inode, n->xattr_idx); + + ret = sqfs_id_table_id_to_index(wr->idtbl, n->uid, + &inode->base.uid_idx); + if (ret) + goto out; + + ret = sqfs_id_table_id_to_index(wr->idtbl, n->gid, + &inode->base.gid_idx); + if (ret) + goto out; + + sqfs_meta_writer_get_position(wr->im, &block, &offset); + n->inode_ref = (block << 16) | offset; + + ret = sqfs_meta_writer_write_inode(wr->im, inode); +out: + free(inode); + return ret; +} + +int sqfs_serialize_fstree(const char *filename, sqfs_writer_t *wr) +{ + size_t i; + int ret; + + wr->super.inode_table_start = wr->outfile->get_size(wr->outfile); + + for (i = 0; i < wr->fs.unique_inode_count; ++i) { + ret = serialize_tree_node(filename, wr, wr->fs.inodes[i]); + if (ret) + goto out; + } + + ret = sqfs_meta_writer_flush(wr->im); + if (ret) + goto out; + + ret = sqfs_meta_writer_flush(wr->dm); + if (ret) + goto out; + + wr->super.root_inode_ref = wr->fs.root->inode_ref; + wr->super.directory_table_start = wr->outfile->get_size(wr->outfile); + + ret = sqfs_meta_write_write_to_file(wr->dm); + if (ret) + goto out; + + ret = 0; +out: + if (ret) + sqfs_perror(filename, "storing filesystem tree", ret); + return ret; +} -- cgit v1.2.3