From 411f659fe0140bacbd56f8503cda269816d4a887 Mon Sep 17 00:00:00 2001 From: David Oberhollenzer Date: Sat, 7 Nov 2020 16:04:40 +0100 Subject: Cleanup: libcommon: try to split up some of the larger files The sqsf_writer is moved to a separate header, as well as the compressor related stuff. The statistics function is moved into the writer code, as this is the only place that actually uses it. The writer code itself is split up into a hand full of file in their own subdirectory. Signed-off-by: David Oberhollenzer --- include/common.h | 90 +--------- include/compress_cli.h | 42 +++++ include/simple_writer.h | 87 +++++++++ lib/common/Makemodule.am | 11 +- lib/common/comp_lzo.c | 2 +- lib/common/comp_opt.c | 2 + lib/common/compress.c | 5 +- lib/common/serialize_fstree.c | 199 --------------------- lib/common/statistics.c | 59 ------- lib/common/writer.c | 329 ----------------------------------- lib/common/writer/cleanup.c | 40 +++++ lib/common/writer/finish.c | 176 +++++++++++++++++++ lib/common/writer/init.c | 186 ++++++++++++++++++++ lib/common/writer/serialize_fstree.c | 199 +++++++++++++++++++++ 14 files changed, 746 insertions(+), 681 deletions(-) create mode 100644 include/compress_cli.h create mode 100644 include/simple_writer.h delete mode 100644 lib/common/serialize_fstree.c delete mode 100644 lib/common/statistics.c delete mode 100644 lib/common/writer.c create mode 100644 lib/common/writer/cleanup.c create mode 100644 lib/common/writer/finish.c create mode 100644 lib/common/writer/init.c create mode 100644 lib/common/writer/serialize_fstree.c diff --git a/include/common.h b/include/common.h index ea27edc..b0c7abb 100644 --- a/include/common.h +++ b/include/common.h @@ -9,25 +9,17 @@ #include "config.h" -#include "sqfs/xattr_writer.h" #include "sqfs/xattr_reader.h" -#include "sqfs/compressor.h" -#include "sqfs/id_table.h" #include "sqfs/inode.h" #include "sqfs/table.h" -#include "sqfs/error.h" -#include "sqfs/meta_writer.h" #include "sqfs/data_reader.h" -#include "sqfs/block_processor.h" -#include "sqfs/block_writer.h" -#include "sqfs/frag_table.h" -#include "sqfs/dir_writer.h" #include "sqfs/dir_reader.h" #include "sqfs/block.h" #include "sqfs/xattr.h" #include "sqfs/dir.h" -#include "sqfs/io.h" +#include "simple_writer.h" +#include "compress_cli.h" #include "fstream.h" #include "compat.h" #include "fstree.h" @@ -35,39 +27,6 @@ #include -typedef struct { - const char *filename; - sqfs_block_writer_t *blkwr; - sqfs_frag_table_t *fragtbl; - sqfs_block_processor_t *data; - sqfs_dir_writer_t *dirwr; - sqfs_meta_writer_t *dm; - sqfs_meta_writer_t *im; - sqfs_compressor_t *cmp; - sqfs_id_table_t *idtbl; - sqfs_file_t *outfile; - sqfs_super_t super; - fstree_t fs; - sqfs_xattr_writer_t *xwr; -} sqfs_writer_t; - -typedef struct { - const char *filename; - char *fs_defaults; - char *comp_extra; - size_t block_size; - size_t devblksize; - size_t max_backlog; - size_t num_jobs; - - int outmode; - SQFS_COMPRESSOR comp_id; - - bool exportable; - bool no_xattr; - bool quiet; -} sqfs_writer_cfg_t; - typedef struct sqfs_hard_link_t { struct sqfs_hard_link_t *next; sqfs_u32 inode_number; @@ -77,35 +36,6 @@ typedef struct sqfs_hard_link_t { #define container_of(ptr, type, member) \ ((type *)((char *)ptr - offsetof(type, member))) -/* - High level helper function to serialize an entire file system tree to - a squashfs inode table and directory table. - - The data is written to the given file descriptor and the super block is - update accordingly (inode and directory table start and total size). - - The function internally creates two meta data writers and uses - meta_writer_write_inode to serialize the inode table of the fstree. - - Returns 0 on success. Prints error messages to stderr on failure. - */ -int sqfs_serialize_fstree(const char *filename, sqfs_writer_t *wr); - -/* Print out fancy statistics for squashfs packing tools */ -void sqfs_print_statistics(const sqfs_super_t *super, - const sqfs_block_processor_t *blk, - const sqfs_block_writer_t *wr); - -void compressor_print_available(void); - -SQFS_COMPRESSOR compressor_get_default(void); - -int compressor_cfg_init_options(sqfs_compressor_config_t *cfg, - SQFS_COMPRESSOR id, - size_t block_size, char *options); - -void compressor_print_help(SQFS_COMPRESSOR id); - int inode_stat(const sqfs_tree_node_t *node, struct stat *sb); char *sqfs_tree_node_get_path(const sqfs_tree_node_t *node); @@ -118,14 +48,6 @@ int write_data_from_file(const char *filename, sqfs_block_processor_t *data, sqfs_inode_generic_t **inode, sqfs_file_t *file, int flags); -void sqfs_writer_cfg_init(sqfs_writer_cfg_t *cfg); - -int sqfs_writer_init(sqfs_writer_t *sqfs, const sqfs_writer_cfg_t *wrcfg); - -int sqfs_writer_finish(sqfs_writer_t *sqfs, const sqfs_writer_cfg_t *cfg); - -void sqfs_writer_cleanup(sqfs_writer_t *sqfs, int status); - void sqfs_perror(const char *file, const char *action, int error_code); int sqfs_tree_find_hard_links(const sqfs_tree_node_t *root, @@ -142,14 +64,6 @@ int mkdir_p(const char *path); /* A common implementation of the '--version' command line flag. */ void print_version(const char *progname); -/* - Create an liblzo2 based LZO compressor. - - XXX: This must be in libcommon instead of libsquashfs for legal reasons. - */ -int lzo_compressor_create(const sqfs_compressor_config_t *cfg, - sqfs_compressor_t **out); - /* Parse a number optionally followed by a KMG suffix (case insensitive). Prints an error message to stderr and returns -1 on failure, 0 on success. diff --git a/include/compress_cli.h b/include/compress_cli.h new file mode 100644 index 0000000..1ac7972 --- /dev/null +++ b/include/compress_cli.h @@ -0,0 +1,42 @@ +/* SPDX-License-Identifier: GPL-3.0-or-later */ +/* + * compress_cli.h + * + * Copyright (C) 2019 David Oberhollenzer + */ +#ifndef COMPRESS_CLI_H +#define COMPRESS_CLI_H + +#include "sqfs/compressor.h" +#include "sqfs/super.h" +#include "sqfs/block.h" +#include "sqfs/error.h" +#include "sqfs/io.h" + +#ifdef __cplusplus +extern "C" { +#endif + +void compressor_print_available(void); + +SQFS_COMPRESSOR compressor_get_default(void); + +int compressor_cfg_init_options(sqfs_compressor_config_t *cfg, + SQFS_COMPRESSOR id, + size_t block_size, char *options); + +void compressor_print_help(SQFS_COMPRESSOR id); + +/* + Create an liblzo2 based LZO compressor. + + XXX: This must be in libcommon instead of libsquashfs for legal reasons. + */ +int lzo_compressor_create(const sqfs_compressor_config_t *cfg, + sqfs_compressor_t **out); + +#ifdef __cplusplus +} +#endif + +#endif /* COMPRESS_CLI_H */ diff --git a/include/simple_writer.h b/include/simple_writer.h new file mode 100644 index 0000000..1a3302b --- /dev/null +++ b/include/simple_writer.h @@ -0,0 +1,87 @@ +/* SPDX-License-Identifier: GPL-3.0-or-later */ +/* + * simple_writer.h + * + * Copyright (C) 2019 David Oberhollenzer + */ +#ifndef SIMPLE_WRITER_H +#define SIMPLE_WRITER_H + +#include "config.h" + +#include "sqfs/block_processor.h" +#include "sqfs/block_writer.h" +#include "sqfs/xattr_writer.h" +#include "sqfs/meta_writer.h" +#include "sqfs/frag_table.h" +#include "sqfs/dir_writer.h" +#include "sqfs/compressor.h" +#include "sqfs/id_table.h" +#include "sqfs/error.h" +#include "sqfs/io.h" + +#include "fstree.h" + +typedef struct { + const char *filename; + sqfs_block_writer_t *blkwr; + sqfs_frag_table_t *fragtbl; + sqfs_block_processor_t *data; + sqfs_dir_writer_t *dirwr; + sqfs_meta_writer_t *dm; + sqfs_meta_writer_t *im; + sqfs_compressor_t *cmp; + sqfs_id_table_t *idtbl; + sqfs_file_t *outfile; + sqfs_super_t super; + fstree_t fs; + sqfs_xattr_writer_t *xwr; +} sqfs_writer_t; + +typedef struct { + const char *filename; + char *fs_defaults; + char *comp_extra; + size_t block_size; + size_t devblksize; + size_t max_backlog; + size_t num_jobs; + + int outmode; + SQFS_COMPRESSOR comp_id; + + bool exportable; + bool no_xattr; + bool quiet; +} sqfs_writer_cfg_t; + +#ifdef __cplusplus +extern "C" { +#endif + +void sqfs_writer_cfg_init(sqfs_writer_cfg_t *cfg); + +int sqfs_writer_init(sqfs_writer_t *sqfs, const sqfs_writer_cfg_t *wrcfg); + +int sqfs_writer_finish(sqfs_writer_t *sqfs, const sqfs_writer_cfg_t *cfg); + +void sqfs_writer_cleanup(sqfs_writer_t *sqfs, int status); + +/* + High level helper function to serialize an entire file system tree to + a squashfs inode table and directory table. The super block is update + accordingly. + + The function internally creates two meta data writers and uses + meta_writer_write_inode to serialize the inode table of the fstree. + + Returns 0 on success. Prints error messages to stderr on failure. + The filename is used to prefix error messages. + */ +int sqfs_serialize_fstree(const char *filename, sqfs_writer_t *wr); + +#ifdef __cplusplus +} +#endif + +#endif /* SIMPLE_WRITER_H */ diff --git a/lib/common/Makemodule.am b/lib/common/Makemodule.am index e191e29..dd0a0d0 100644 --- a/lib/common/Makemodule.am +++ b/lib/common/Makemodule.am @@ -1,12 +1,15 @@ -libcommon_a_SOURCES = lib/common/serialize_fstree.c lib/common/statistics.c -libcommon_a_SOURCES += lib/common/inode_stat.c lib/common/hardlink.c +libcommon_a_SOURCES = lib/common/inode_stat.c lib/common/hardlink.c libcommon_a_SOURCES += lib/common/print_version.c lib/common/data_reader_dump.c libcommon_a_SOURCES += lib/common/compress.c lib/common/comp_opt.c libcommon_a_SOURCES += lib/common/data_writer.c include/common.h libcommon_a_SOURCES += lib/common/get_path.c lib/common/data_writer_ostream.c -libcommon_a_SOURCES += lib/common/writer.c lib/common/perror.c +libcommon_a_SOURCES += lib/common/perror.c libcommon_a_SOURCES += lib/common/mkdir_p.c lib/common/parse_size.c -libcommon_a_SOURCES += lib/common/print_size.c +libcommon_a_SOURCES += lib/common/print_size.c include/simple_writer.h +libcommon_a_SOURCES += include/compress_cli.h +libcommon_a_SOURCES += lib/common/writer/init.c lib/common/writer/cleanup.c +libcommon_a_SOURCES += lib/common/writer/serialize_fstree.c +libcommon_a_SOURCES += lib/common/writer/finish.c libcommon_a_CFLAGS = $(AM_CFLAGS) $(LZO_CFLAGS) if WITH_LZO diff --git a/lib/common/comp_lzo.c b/lib/common/comp_lzo.c index 3452647..21dc7b2 100644 --- a/lib/common/comp_lzo.c +++ b/lib/common/comp_lzo.c @@ -5,7 +5,7 @@ * Copyright (C) 2019 David Oberhollenzer */ #include "config.h" -#include "common.h" +#include "compress_cli.h" #include #include diff --git a/lib/common/comp_opt.c b/lib/common/comp_opt.c index f77ac20..a0d4cf7 100644 --- a/lib/common/comp_opt.c +++ b/lib/common/comp_opt.c @@ -4,11 +4,13 @@ * * Copyright (C) 2019 David Oberhollenzer */ +#include "config.h" #include "common.h" #include #include #include +#include #include typedef struct { diff --git a/lib/common/compress.c b/lib/common/compress.c index 9a66095..b11efbd 100644 --- a/lib/common/compress.c +++ b/lib/common/compress.c @@ -4,8 +4,11 @@ * * Copyright (C) 2019 David Oberhollenzer */ -#include "common.h" +#include "config.h" +#include "compress_cli.h" + #include +#include static int cmp_ids[] = { SQFS_COMP_XZ, diff --git a/lib/common/serialize_fstree.c b/lib/common/serialize_fstree.c deleted file mode 100644 index c69f0ea..0000000 --- a/lib/common/serialize_fstree.c +++ /dev/null @@ -1,199 +0,0 @@ -/* SPDX-License-Identifier: GPL-3.0-or-later */ -/* - * serialize_fstree.c - * - * Copyright (C) 2019 David Oberhollenzer - */ -#include "common.h" - -#include -#include -#include -#include - -static sqfs_inode_generic_t *tree_node_to_inode(tree_node_t *node) -{ - sqfs_inode_generic_t *inode; - size_t extra = 0; - - if (S_ISLNK(node->mode)) - extra = strlen(node->data.target); - - inode = calloc(1, sizeof(*inode) + extra); - if (inode == NULL) { - perror("creating inode"); - return NULL; - } - - switch (node->mode & S_IFMT) { - case S_IFSOCK: - inode->base.type = SQFS_INODE_SOCKET; - inode->data.ipc.nlink = node->link_count; - break; - case S_IFIFO: - inode->base.type = SQFS_INODE_FIFO; - inode->data.ipc.nlink = node->link_count; - break; - case S_IFLNK: - inode->base.type = SQFS_INODE_SLINK; - inode->data.slink.nlink = node->link_count; - inode->data.slink.target_size = extra; - memcpy(inode->extra, node->data.target, extra); - break; - case S_IFBLK: - inode->base.type = SQFS_INODE_BDEV; - inode->data.dev.nlink = node->link_count; - inode->data.dev.devno = node->data.devno; - break; - case S_IFCHR: - inode->base.type = SQFS_INODE_CDEV; - inode->data.dev.nlink = node->link_count; - inode->data.dev.devno = node->data.devno; - break; - default: - assert(0); - } - - return inode; -} - -static sqfs_inode_generic_t *write_dir_entries(const char *filename, - sqfs_dir_writer_t *dirw, - tree_node_t *node) -{ - sqfs_u32 xattr, parent_inode; - sqfs_inode_generic_t *inode; - tree_node_t *it, *tgt; - int ret; - - ret = sqfs_dir_writer_begin(dirw, 0); - if (ret) - goto fail; - - for (it = node->data.dir.children; it != NULL; it = it->next) { - if (it->mode == FSTREE_MODE_HARD_LINK_RESOLVED) { - tgt = it->data.target_node; - } else { - tgt = it; - } - - ret = sqfs_dir_writer_add_entry(dirw, it->name, tgt->inode_num, - tgt->inode_ref, tgt->mode); - if (ret) - goto fail; - } - - ret = sqfs_dir_writer_end(dirw); - if (ret) - goto fail; - - xattr = node->xattr_idx; - parent_inode = (node->parent == NULL) ? 0 : node->parent->inode_num; - - inode = sqfs_dir_writer_create_inode(dirw, 0, xattr, parent_inode); - if (inode == NULL) { - ret = SQFS_ERROR_ALLOC; - goto fail; - } - - if (inode->base.type == SQFS_INODE_DIR) { - inode->data.dir.nlink = node->link_count; - } else { - inode->data.dir_ext.nlink = node->link_count; - } - - return inode; -fail: - sqfs_perror(filename, "recoding directory entries", ret); - return NULL; -} - -static int serialize_tree_node(const char *filename, sqfs_writer_t *wr, - tree_node_t *n) -{ - sqfs_inode_generic_t *inode; - sqfs_u32 offset; - sqfs_u64 block; - int ret; - - if (S_ISDIR(n->mode)) { - inode = write_dir_entries(filename, wr->dirwr, n); - ret = SQFS_ERROR_INTERNAL; - } else if (S_ISREG(n->mode)) { - inode = n->data.file.user_ptr; - n->data.file.user_ptr = NULL; - ret = SQFS_ERROR_INTERNAL; - - if (inode->base.type == SQFS_INODE_FILE && n->link_count > 1) { - sqfs_inode_make_extended(inode); - inode->data.file_ext.nlink = n->link_count; - } else { - inode->data.file_ext.nlink = n->link_count; - } - } else { - inode = tree_node_to_inode(n); - ret = SQFS_ERROR_ALLOC; - } - - if (inode == NULL) - return ret; - - inode->base.mode = n->mode; - inode->base.mod_time = n->mod_time; - inode->base.inode_number = n->inode_num; - - sqfs_inode_set_xattr_index(inode, n->xattr_idx); - - ret = sqfs_id_table_id_to_index(wr->idtbl, n->uid, - &inode->base.uid_idx); - if (ret) - goto out; - - ret = sqfs_id_table_id_to_index(wr->idtbl, n->gid, - &inode->base.gid_idx); - if (ret) - goto out; - - sqfs_meta_writer_get_position(wr->im, &block, &offset); - n->inode_ref = (block << 16) | offset; - - ret = sqfs_meta_writer_write_inode(wr->im, inode); -out: - free(inode); - return ret; -} - -int sqfs_serialize_fstree(const char *filename, sqfs_writer_t *wr) -{ - size_t i; - int ret; - - wr->super.inode_table_start = wr->outfile->get_size(wr->outfile); - - for (i = 0; i < wr->fs.unique_inode_count; ++i) { - ret = serialize_tree_node(filename, wr, wr->fs.inodes[i]); - if (ret) - goto out; - } - - ret = sqfs_meta_writer_flush(wr->im); - if (ret) - goto out; - - ret = sqfs_meta_writer_flush(wr->dm); - if (ret) - goto out; - - wr->super.root_inode_ref = wr->fs.root->inode_ref; - wr->super.directory_table_start = wr->outfile->get_size(wr->outfile); - - ret = sqfs_meta_write_write_to_file(wr->dm); - if (ret) - goto out; - - ret = 0; -out: - if (ret) - sqfs_perror(filename, "storing filesystem tree", ret); - return ret; -} diff --git a/lib/common/statistics.c b/lib/common/statistics.c deleted file mode 100644 index 82e545d..0000000 --- a/lib/common/statistics.c +++ /dev/null @@ -1,59 +0,0 @@ -/* SPDX-License-Identifier: GPL-3.0-or-later */ -/* - * statistics.c - * - * Copyright (C) 2019 David Oberhollenzer - */ -#include "common.h" - -#include - -void sqfs_print_statistics(const sqfs_super_t *super, - const sqfs_block_processor_t *blk, - const sqfs_block_writer_t *wr) -{ - const sqfs_block_processor_stats_t *proc_stats; - sqfs_u64 bytes_written, blocks_written; - char read_sz[32], written_sz[32]; - size_t ratio; - - proc_stats = sqfs_block_processor_get_stats(blk); - blocks_written = wr->get_block_count(wr); - - bytes_written = super->inode_table_start - sizeof(*super); - - if (proc_stats->input_bytes_read > 0) { - ratio = (100 * bytes_written) / proc_stats->input_bytes_read; - } else { - ratio = 100; - } - - print_size(proc_stats->input_bytes_read, read_sz, false); - print_size(bytes_written, written_sz, false); - - fputs("---------------------------------------------------\n", stdout); - printf("Data bytes read: %s\n", read_sz); - printf("Data bytes written: %s\n", written_sz); - printf("Data compression ratio: " PRI_SZ "%%\n", ratio); - fputc('\n', stdout); - - printf("Data blocks written: " PRI_U64 "\n", blocks_written); - printf("Out of which where fragment blocks: " PRI_U64 "\n", - proc_stats->frag_block_count); - - printf("Duplicate blocks omitted: " PRI_U64 "\n", - proc_stats->data_block_count + proc_stats->frag_block_count - - blocks_written); - - printf("Sparse blocks omitted: " PRI_U64 "\n", - proc_stats->sparse_block_count); - fputc('\n', stdout); - - printf("Fragments actually written: " PRI_U64 "\n", - proc_stats->actual_frag_count); - printf("Duplicated fragments omitted: " PRI_U64 "\n", - proc_stats->total_frag_count - proc_stats->actual_frag_count); - printf("Total number of inodes: %u\n", super->inode_count); - printf("Number of unique group/user IDs: %u\n", super->id_count); - fputc('\n', stdout); -} diff --git a/lib/common/writer.c b/lib/common/writer.c deleted file mode 100644 index 9032a99..0000000 --- a/lib/common/writer.c +++ /dev/null @@ -1,329 +0,0 @@ -/* SPDX-License-Identifier: GPL-3.0-or-later */ -/* - * writer.c - * - * Copyright (C) 2019 David Oberhollenzer - */ -#include "common.h" - -#include -#include - -#ifdef HAVE_SYS_SYSINFO_H -#include - -static size_t os_get_num_jobs(void) -{ - int nprocs; - - nprocs = get_nprocs_conf(); - return nprocs < 1 ? 1 : nprocs; -} -#else -static size_t os_get_num_jobs(void) -{ - return 1; -} -#endif - -static int padd_sqfs(sqfs_file_t *file, sqfs_u64 size, size_t blocksize) -{ - size_t padd_sz = size % blocksize; - int status = -1; - sqfs_u8 *buffer; - - if (padd_sz == 0) - return 0; - - padd_sz = blocksize - padd_sz; - - buffer = calloc(1, padd_sz); - if (buffer == NULL) - goto fail_errno; - - if (file->write_at(file, file->get_size(file), - buffer, padd_sz)) { - goto fail_errno; - } - - status = 0; -out: - free(buffer); - return status; -fail_errno: - perror("padding output file to block size"); - goto out; -} - -void sqfs_writer_cfg_init(sqfs_writer_cfg_t *cfg) -{ - memset(cfg, 0, sizeof(*cfg)); - - cfg->num_jobs = os_get_num_jobs(); - cfg->block_size = SQFS_DEFAULT_BLOCK_SIZE; - cfg->devblksize = SQFS_DEVBLK_SIZE; - cfg->comp_id = compressor_get_default(); -} - -int sqfs_writer_init(sqfs_writer_t *sqfs, const sqfs_writer_cfg_t *wrcfg) -{ - sqfs_compressor_config_t cfg; - int ret, flags; - - sqfs->filename = wrcfg->filename; - - if (compressor_cfg_init_options(&cfg, wrcfg->comp_id, - wrcfg->block_size, - wrcfg->comp_extra)) { - return -1; - } - - sqfs->outfile = sqfs_open_file(wrcfg->filename, wrcfg->outmode); - if (sqfs->outfile == NULL) { - perror(wrcfg->filename); - return -1; - } - - if (fstree_init(&sqfs->fs, wrcfg->fs_defaults)) - goto fail_file; - - ret = sqfs_compressor_create(&cfg, &sqfs->cmp); - -#ifdef WITH_LZO - if (cfg.id == SQFS_COMP_LZO) { - if (sqfs->cmp != NULL) - sqfs_destroy(sqfs->cmp); - - ret = lzo_compressor_create(&cfg, &sqfs->cmp); - } -#endif - - if (ret != 0) { - sqfs_perror(wrcfg->filename, "creating compressor", ret); - goto fail_fs; - } - - ret = sqfs_super_init(&sqfs->super, wrcfg->block_size, - sqfs->fs.defaults.st_mtime, wrcfg->comp_id); - if (ret) { - sqfs_perror(wrcfg->filename, "initializing super block", ret); - goto fail_cmp; - } - - ret = sqfs_super_write(&sqfs->super, sqfs->outfile); - if (ret) { - sqfs_perror(wrcfg->filename, "writing super block", ret); - goto fail_cmp; - } - - ret = sqfs->cmp->write_options(sqfs->cmp, sqfs->outfile); - if (ret < 0) { - sqfs_perror(wrcfg->filename, "writing compressor options", ret); - goto fail_cmp; - } - - if (ret > 0) - sqfs->super.flags |= SQFS_FLAG_COMPRESSOR_OPTIONS; - - sqfs->blkwr = sqfs_block_writer_create(sqfs->outfile, - wrcfg->devblksize, 0); - if (sqfs->blkwr == NULL) { - perror("creating block writer"); - goto fail_cmp; - } - - sqfs->fragtbl = sqfs_frag_table_create(0); - if (sqfs->fragtbl == NULL) { - perror("creating fragment table"); - goto fail_blkwr; - } - - sqfs->data = sqfs_block_processor_create(sqfs->super.block_size, - sqfs->cmp, wrcfg->num_jobs, - wrcfg->max_backlog, - sqfs->blkwr, sqfs->fragtbl); - if (sqfs->data == NULL) { - perror("creating data block processor"); - goto fail_fragtbl; - } - - sqfs->idtbl = sqfs_id_table_create(0); - if (sqfs->idtbl == NULL) { - sqfs_perror(wrcfg->filename, "creating ID table", - SQFS_ERROR_ALLOC); - goto fail_data; - } - - if (!wrcfg->no_xattr) { - sqfs->xwr = sqfs_xattr_writer_create(0); - - if (sqfs->xwr == NULL) { - sqfs_perror(wrcfg->filename, "creating xattr writer", - SQFS_ERROR_ALLOC); - goto fail_id; - } - } - - sqfs->im = sqfs_meta_writer_create(sqfs->outfile, sqfs->cmp, 0); - if (sqfs->im == NULL) { - fputs("Error creating inode meta data writer.\n", stderr); - goto fail_xwr; - } - - sqfs->dm = sqfs_meta_writer_create(sqfs->outfile, sqfs->cmp, - SQFS_META_WRITER_KEEP_IN_MEMORY); - if (sqfs->dm == NULL) { - fputs("Error creating directory meta data writer.\n", stderr); - goto fail_im; - } - - flags = 0; - if (wrcfg->exportable) - flags |= SQFS_DIR_WRITER_CREATE_EXPORT_TABLE; - - sqfs->dirwr = sqfs_dir_writer_create(sqfs->dm, flags); - if (sqfs->dirwr == NULL) { - fputs("Error creating directory table writer.\n", stderr); - goto fail_dm; - } - - return 0; -fail_dm: - sqfs_destroy(sqfs->dm); -fail_im: - sqfs_destroy(sqfs->im); -fail_xwr: - if (sqfs->xwr != NULL) - sqfs_destroy(sqfs->xwr); -fail_id: - sqfs_destroy(sqfs->idtbl); -fail_data: - sqfs_destroy(sqfs->data); -fail_fragtbl: - sqfs_destroy(sqfs->fragtbl); -fail_blkwr: - sqfs_destroy(sqfs->blkwr); -fail_cmp: - sqfs_destroy(sqfs->cmp); -fail_fs: - fstree_cleanup(&sqfs->fs); -fail_file: - sqfs_destroy(sqfs->outfile); - return -1; -} - -int sqfs_writer_finish(sqfs_writer_t *sqfs, const sqfs_writer_cfg_t *cfg) -{ - int ret; - - if (!cfg->quiet) - fputs("Waiting for remaining data blocks...\n", stdout); - - ret = sqfs_block_processor_finish(sqfs->data); - if (ret) { - sqfs_perror(cfg->filename, "finishing data blocks", ret); - return -1; - } - - if (!cfg->quiet) - fputs("Writing inodes and directories...\n", stdout); - - sqfs->super.inode_count = sqfs->fs.unique_inode_count; - - if (sqfs_serialize_fstree(cfg->filename, sqfs)) - return -1; - - if (!cfg->quiet) - fputs("Writing fragment table...\n", stdout); - - ret = sqfs_frag_table_write(sqfs->fragtbl, sqfs->outfile, - &sqfs->super, sqfs->cmp); - if (ret) { - sqfs_perror(cfg->filename, "writing fragment table", ret); - return -1; - } - - if (cfg->exportable) { - if (!cfg->quiet) - fputs("Writing export table...\n", stdout); - - - ret = sqfs_dir_writer_write_export_table(sqfs->dirwr, - sqfs->outfile, sqfs->cmp, - sqfs->fs.root->inode_num, - sqfs->fs.root->inode_ref, - &sqfs->super); - if (ret) - return -1; - } - - if (!cfg->quiet) - fputs("Writing ID table...\n", stdout); - - ret = sqfs_id_table_write(sqfs->idtbl, sqfs->outfile, - &sqfs->super, sqfs->cmp); - if (ret) { - sqfs_perror(cfg->filename, "writing ID table", ret); - return -1; - } - - if (!cfg->no_xattr) { - if (!cfg->quiet) - fputs("Writing extended attributes...\n", stdout); - - ret = sqfs_xattr_writer_flush(sqfs->xwr, sqfs->outfile, - &sqfs->super, sqfs->cmp); - if (ret) { - sqfs_perror(cfg->filename, "writing extended attributes", ret); - return -1; - } - } - - sqfs->super.bytes_used = sqfs->outfile->get_size(sqfs->outfile); - - ret = sqfs_super_write(&sqfs->super, sqfs->outfile); - if (ret) { - sqfs_perror(cfg->filename, "updating super block", ret); - return -1; - } - - if (padd_sqfs(sqfs->outfile, sqfs->super.bytes_used, - cfg->devblksize)) { - return -1; - } - - if (!cfg->quiet) - sqfs_print_statistics(&sqfs->super, sqfs->data, sqfs->blkwr); - - return 0; -} - -void sqfs_writer_cleanup(sqfs_writer_t *sqfs, int status) -{ - if (sqfs->xwr != NULL) - sqfs_destroy(sqfs->xwr); - - sqfs_destroy(sqfs->dirwr); - sqfs_destroy(sqfs->dm); - sqfs_destroy(sqfs->im); - sqfs_destroy(sqfs->idtbl); - sqfs_destroy(sqfs->data); - sqfs_destroy(sqfs->blkwr); - sqfs_destroy(sqfs->fragtbl); - sqfs_destroy(sqfs->cmp); - fstree_cleanup(&sqfs->fs); - sqfs_destroy(sqfs->outfile); - - if (status != EXIT_SUCCESS) { -#if defined(_WIN32) || defined(__WINDOWS__) - WCHAR *path = path_to_windows(sqfs->filename); - - if (path != NULL) - DeleteFileW(path); - - free(path); -#else - unlink(sqfs->filename); -#endif - } -} diff --git a/lib/common/writer/cleanup.c b/lib/common/writer/cleanup.c new file mode 100644 index 0000000..16e846b --- /dev/null +++ b/lib/common/writer/cleanup.c @@ -0,0 +1,40 @@ +/* SPDX-License-Identifier: GPL-3.0-or-later */ +/* + * cleanup.c + * + * Copyright (C) 2019 David Oberhollenzer + */ +#include "simple_writer.h" + +#include + +void sqfs_writer_cleanup(sqfs_writer_t *sqfs, int status) +{ + if (sqfs->xwr != NULL) + sqfs_destroy(sqfs->xwr); + + sqfs_destroy(sqfs->dirwr); + sqfs_destroy(sqfs->dm); + sqfs_destroy(sqfs->im); + sqfs_destroy(sqfs->idtbl); + sqfs_destroy(sqfs->data); + sqfs_destroy(sqfs->blkwr); + sqfs_destroy(sqfs->fragtbl); + sqfs_destroy(sqfs->cmp); + fstree_cleanup(&sqfs->fs); + sqfs_destroy(sqfs->outfile); + + if (status != EXIT_SUCCESS) { +#if defined(_WIN32) || defined(__WINDOWS__) + WCHAR *path = path_to_windows(sqfs->filename); + + if (path != NULL) + DeleteFileW(path); + + free(path); +#else + unlink(sqfs->filename); +#endif + } +} + diff --git a/lib/common/writer/finish.c b/lib/common/writer/finish.c new file mode 100644 index 0000000..e005628 --- /dev/null +++ b/lib/common/writer/finish.c @@ -0,0 +1,176 @@ +/* SPDX-License-Identifier: GPL-3.0-or-later */ +/* + * finish.c + * + * Copyright (C) 2019 David Oberhollenzer + */ +#include "simple_writer.h" +#include "common.h" + +#include + +static void print_statistics(const sqfs_super_t *super, + const sqfs_block_processor_t *blk, + const sqfs_block_writer_t *wr) +{ + const sqfs_block_processor_stats_t *proc_stats; + sqfs_u64 bytes_written, blocks_written; + char read_sz[32], written_sz[32]; + size_t ratio; + + proc_stats = sqfs_block_processor_get_stats(blk); + blocks_written = wr->get_block_count(wr); + + bytes_written = super->inode_table_start - sizeof(*super); + + if (proc_stats->input_bytes_read > 0) { + ratio = (100 * bytes_written) / proc_stats->input_bytes_read; + } else { + ratio = 100; + } + + print_size(proc_stats->input_bytes_read, read_sz, false); + print_size(bytes_written, written_sz, false); + + fputs("---------------------------------------------------\n", stdout); + printf("Data bytes read: %s\n", read_sz); + printf("Data bytes written: %s\n", written_sz); + printf("Data compression ratio: " PRI_SZ "%%\n", ratio); + fputc('\n', stdout); + + printf("Data blocks written: " PRI_U64 "\n", blocks_written); + printf("Out of which where fragment blocks: " PRI_U64 "\n", + proc_stats->frag_block_count); + + printf("Duplicate blocks omitted: " PRI_U64 "\n", + proc_stats->data_block_count + proc_stats->frag_block_count - + blocks_written); + + printf("Sparse blocks omitted: " PRI_U64 "\n", + proc_stats->sparse_block_count); + fputc('\n', stdout); + + printf("Fragments actually written: " PRI_U64 "\n", + proc_stats->actual_frag_count); + printf("Duplicated fragments omitted: " PRI_U64 "\n", + proc_stats->total_frag_count - proc_stats->actual_frag_count); + printf("Total number of inodes: %u\n", super->inode_count); + printf("Number of unique group/user IDs: %u\n", super->id_count); + fputc('\n', stdout); +} + +static int padd_sqfs(sqfs_file_t *file, sqfs_u64 size, size_t blocksize) +{ + size_t padd_sz = size % blocksize; + int status = -1; + sqfs_u8 *buffer; + + if (padd_sz == 0) + return 0; + + padd_sz = blocksize - padd_sz; + + buffer = calloc(1, padd_sz); + if (buffer == NULL) + goto fail_errno; + + if (file->write_at(file, file->get_size(file), + buffer, padd_sz)) { + goto fail_errno; + } + + status = 0; +out: + free(buffer); + return status; +fail_errno: + perror("padding output file to block size"); + goto out; +} + +int sqfs_writer_finish(sqfs_writer_t *sqfs, const sqfs_writer_cfg_t *cfg) +{ + int ret; + + if (!cfg->quiet) + fputs("Waiting for remaining data blocks...\n", stdout); + + ret = sqfs_block_processor_finish(sqfs->data); + if (ret) { + sqfs_perror(cfg->filename, "finishing data blocks", ret); + return -1; + } + + if (!cfg->quiet) + fputs("Writing inodes and directories...\n", stdout); + + sqfs->super.inode_count = sqfs->fs.unique_inode_count; + + if (sqfs_serialize_fstree(cfg->filename, sqfs)) + return -1; + + if (!cfg->quiet) + fputs("Writing fragment table...\n", stdout); + + ret = sqfs_frag_table_write(sqfs->fragtbl, sqfs->outfile, + &sqfs->super, sqfs->cmp); + if (ret) { + sqfs_perror(cfg->filename, "writing fragment table", ret); + return -1; + } + + if (cfg->exportable) { + if (!cfg->quiet) + fputs("Writing export table...\n", stdout); + + + ret = sqfs_dir_writer_write_export_table(sqfs->dirwr, + sqfs->outfile, sqfs->cmp, + sqfs->fs.root->inode_num, + sqfs->fs.root->inode_ref, + &sqfs->super); + if (ret) + return -1; + } + + if (!cfg->quiet) + fputs("Writing ID table...\n", stdout); + + ret = sqfs_id_table_write(sqfs->idtbl, sqfs->outfile, + &sqfs->super, sqfs->cmp); + if (ret) { + sqfs_perror(cfg->filename, "writing ID table", ret); + return -1; + } + + if (!cfg->no_xattr) { + if (!cfg->quiet) + fputs("Writing extended attributes...\n", stdout); + + ret = sqfs_xattr_writer_flush(sqfs->xwr, sqfs->outfile, + &sqfs->super, sqfs->cmp); + if (ret) { + sqfs_perror(cfg->filename, + "writing extended attributes", ret); + return -1; + } + } + + sqfs->super.bytes_used = sqfs->outfile->get_size(sqfs->outfile); + + ret = sqfs_super_write(&sqfs->super, sqfs->outfile); + if (ret) { + sqfs_perror(cfg->filename, "updating super block", ret); + return -1; + } + + if (padd_sqfs(sqfs->outfile, sqfs->super.bytes_used, + cfg->devblksize)) { + return -1; + } + + if (!cfg->quiet) + print_statistics(&sqfs->super, sqfs->data, sqfs->blkwr); + + return 0; +} diff --git a/lib/common/writer/init.c b/lib/common/writer/init.c new file mode 100644 index 0000000..d606d26 --- /dev/null +++ b/lib/common/writer/init.c @@ -0,0 +1,186 @@ +/* SPDX-License-Identifier: GPL-3.0-or-later */ +/* + * init.c + * + * Copyright (C) 2019 David Oberhollenzer + */ +#include "simple_writer.h" +#include "compress_cli.h" +#include "common.h" + +#include +#include + +#ifdef HAVE_SYS_SYSINFO_H +#include + +static size_t os_get_num_jobs(void) +{ + int nprocs; + + nprocs = get_nprocs_conf(); + return nprocs < 1 ? 1 : nprocs; +} +#else +static size_t os_get_num_jobs(void) +{ + return 1; +} +#endif + +void sqfs_writer_cfg_init(sqfs_writer_cfg_t *cfg) +{ + memset(cfg, 0, sizeof(*cfg)); + + cfg->num_jobs = os_get_num_jobs(); + cfg->block_size = SQFS_DEFAULT_BLOCK_SIZE; + cfg->devblksize = SQFS_DEVBLK_SIZE; + cfg->comp_id = compressor_get_default(); +} + +int sqfs_writer_init(sqfs_writer_t *sqfs, const sqfs_writer_cfg_t *wrcfg) +{ + sqfs_compressor_config_t cfg; + int ret, flags; + + sqfs->filename = wrcfg->filename; + + if (compressor_cfg_init_options(&cfg, wrcfg->comp_id, + wrcfg->block_size, + wrcfg->comp_extra)) { + return -1; + } + + sqfs->outfile = sqfs_open_file(wrcfg->filename, wrcfg->outmode); + if (sqfs->outfile == NULL) { + perror(wrcfg->filename); + return -1; + } + + if (fstree_init(&sqfs->fs, wrcfg->fs_defaults)) + goto fail_file; + + ret = sqfs_compressor_create(&cfg, &sqfs->cmp); + +#ifdef WITH_LZO + if (cfg.id == SQFS_COMP_LZO) { + if (sqfs->cmp != NULL) + sqfs_destroy(sqfs->cmp); + + ret = lzo_compressor_create(&cfg, &sqfs->cmp); + } +#endif + + if (ret != 0) { + sqfs_perror(wrcfg->filename, "creating compressor", ret); + goto fail_fs; + } + + ret = sqfs_super_init(&sqfs->super, wrcfg->block_size, + sqfs->fs.defaults.st_mtime, wrcfg->comp_id); + if (ret) { + sqfs_perror(wrcfg->filename, "initializing super block", ret); + goto fail_cmp; + } + + ret = sqfs_super_write(&sqfs->super, sqfs->outfile); + if (ret) { + sqfs_perror(wrcfg->filename, "writing super block", ret); + goto fail_cmp; + } + + ret = sqfs->cmp->write_options(sqfs->cmp, sqfs->outfile); + if (ret < 0) { + sqfs_perror(wrcfg->filename, "writing compressor options", ret); + goto fail_cmp; + } + + if (ret > 0) + sqfs->super.flags |= SQFS_FLAG_COMPRESSOR_OPTIONS; + + sqfs->blkwr = sqfs_block_writer_create(sqfs->outfile, + wrcfg->devblksize, 0); + if (sqfs->blkwr == NULL) { + perror("creating block writer"); + goto fail_cmp; + } + + sqfs->fragtbl = sqfs_frag_table_create(0); + if (sqfs->fragtbl == NULL) { + perror("creating fragment table"); + goto fail_blkwr; + } + + sqfs->data = sqfs_block_processor_create(sqfs->super.block_size, + sqfs->cmp, wrcfg->num_jobs, + wrcfg->max_backlog, + sqfs->blkwr, sqfs->fragtbl); + if (sqfs->data == NULL) { + perror("creating data block processor"); + goto fail_fragtbl; + } + + sqfs->idtbl = sqfs_id_table_create(0); + if (sqfs->idtbl == NULL) { + sqfs_perror(wrcfg->filename, "creating ID table", + SQFS_ERROR_ALLOC); + goto fail_data; + } + + if (!wrcfg->no_xattr) { + sqfs->xwr = sqfs_xattr_writer_create(0); + + if (sqfs->xwr == NULL) { + sqfs_perror(wrcfg->filename, "creating xattr writer", + SQFS_ERROR_ALLOC); + goto fail_id; + } + } + + sqfs->im = sqfs_meta_writer_create(sqfs->outfile, sqfs->cmp, 0); + if (sqfs->im == NULL) { + fputs("Error creating inode meta data writer.\n", stderr); + goto fail_xwr; + } + + sqfs->dm = sqfs_meta_writer_create(sqfs->outfile, sqfs->cmp, + SQFS_META_WRITER_KEEP_IN_MEMORY); + if (sqfs->dm == NULL) { + fputs("Error creating directory meta data writer.\n", stderr); + goto fail_im; + } + + flags = 0; + if (wrcfg->exportable) + flags |= SQFS_DIR_WRITER_CREATE_EXPORT_TABLE; + + sqfs->dirwr = sqfs_dir_writer_create(sqfs->dm, flags); + if (sqfs->dirwr == NULL) { + fputs("Error creating directory table writer.\n", stderr); + goto fail_dm; + } + + return 0; +fail_dm: + sqfs_destroy(sqfs->dm); +fail_im: + sqfs_destroy(sqfs->im); +fail_xwr: + if (sqfs->xwr != NULL) + sqfs_destroy(sqfs->xwr); +fail_id: + sqfs_destroy(sqfs->idtbl); +fail_data: + sqfs_destroy(sqfs->data); +fail_fragtbl: + sqfs_destroy(sqfs->fragtbl); +fail_blkwr: + sqfs_destroy(sqfs->blkwr); +fail_cmp: + sqfs_destroy(sqfs->cmp); +fail_fs: + fstree_cleanup(&sqfs->fs); +fail_file: + sqfs_destroy(sqfs->outfile); + return -1; +} diff --git a/lib/common/writer/serialize_fstree.c b/lib/common/writer/serialize_fstree.c new file mode 100644 index 0000000..c69f0ea --- /dev/null +++ b/lib/common/writer/serialize_fstree.c @@ -0,0 +1,199 @@ +/* SPDX-License-Identifier: GPL-3.0-or-later */ +/* + * serialize_fstree.c + * + * Copyright (C) 2019 David Oberhollenzer + */ +#include "common.h" + +#include +#include +#include +#include + +static sqfs_inode_generic_t *tree_node_to_inode(tree_node_t *node) +{ + sqfs_inode_generic_t *inode; + size_t extra = 0; + + if (S_ISLNK(node->mode)) + extra = strlen(node->data.target); + + inode = calloc(1, sizeof(*inode) + extra); + if (inode == NULL) { + perror("creating inode"); + return NULL; + } + + switch (node->mode & S_IFMT) { + case S_IFSOCK: + inode->base.type = SQFS_INODE_SOCKET; + inode->data.ipc.nlink = node->link_count; + break; + case S_IFIFO: + inode->base.type = SQFS_INODE_FIFO; + inode->data.ipc.nlink = node->link_count; + break; + case S_IFLNK: + inode->base.type = SQFS_INODE_SLINK; + inode->data.slink.nlink = node->link_count; + inode->data.slink.target_size = extra; + memcpy(inode->extra, node->data.target, extra); + break; + case S_IFBLK: + inode->base.type = SQFS_INODE_BDEV; + inode->data.dev.nlink = node->link_count; + inode->data.dev.devno = node->data.devno; + break; + case S_IFCHR: + inode->base.type = SQFS_INODE_CDEV; + inode->data.dev.nlink = node->link_count; + inode->data.dev.devno = node->data.devno; + break; + default: + assert(0); + } + + return inode; +} + +static sqfs_inode_generic_t *write_dir_entries(const char *filename, + sqfs_dir_writer_t *dirw, + tree_node_t *node) +{ + sqfs_u32 xattr, parent_inode; + sqfs_inode_generic_t *inode; + tree_node_t *it, *tgt; + int ret; + + ret = sqfs_dir_writer_begin(dirw, 0); + if (ret) + goto fail; + + for (it = node->data.dir.children; it != NULL; it = it->next) { + if (it->mode == FSTREE_MODE_HARD_LINK_RESOLVED) { + tgt = it->data.target_node; + } else { + tgt = it; + } + + ret = sqfs_dir_writer_add_entry(dirw, it->name, tgt->inode_num, + tgt->inode_ref, tgt->mode); + if (ret) + goto fail; + } + + ret = sqfs_dir_writer_end(dirw); + if (ret) + goto fail; + + xattr = node->xattr_idx; + parent_inode = (node->parent == NULL) ? 0 : node->parent->inode_num; + + inode = sqfs_dir_writer_create_inode(dirw, 0, xattr, parent_inode); + if (inode == NULL) { + ret = SQFS_ERROR_ALLOC; + goto fail; + } + + if (inode->base.type == SQFS_INODE_DIR) { + inode->data.dir.nlink = node->link_count; + } else { + inode->data.dir_ext.nlink = node->link_count; + } + + return inode; +fail: + sqfs_perror(filename, "recoding directory entries", ret); + return NULL; +} + +static int serialize_tree_node(const char *filename, sqfs_writer_t *wr, + tree_node_t *n) +{ + sqfs_inode_generic_t *inode; + sqfs_u32 offset; + sqfs_u64 block; + int ret; + + if (S_ISDIR(n->mode)) { + inode = write_dir_entries(filename, wr->dirwr, n); + ret = SQFS_ERROR_INTERNAL; + } else if (S_ISREG(n->mode)) { + inode = n->data.file.user_ptr; + n->data.file.user_ptr = NULL; + ret = SQFS_ERROR_INTERNAL; + + if (inode->base.type == SQFS_INODE_FILE && n->link_count > 1) { + sqfs_inode_make_extended(inode); + inode->data.file_ext.nlink = n->link_count; + } else { + inode->data.file_ext.nlink = n->link_count; + } + } else { + inode = tree_node_to_inode(n); + ret = SQFS_ERROR_ALLOC; + } + + if (inode == NULL) + return ret; + + inode->base.mode = n->mode; + inode->base.mod_time = n->mod_time; + inode->base.inode_number = n->inode_num; + + sqfs_inode_set_xattr_index(inode, n->xattr_idx); + + ret = sqfs_id_table_id_to_index(wr->idtbl, n->uid, + &inode->base.uid_idx); + if (ret) + goto out; + + ret = sqfs_id_table_id_to_index(wr->idtbl, n->gid, + &inode->base.gid_idx); + if (ret) + goto out; + + sqfs_meta_writer_get_position(wr->im, &block, &offset); + n->inode_ref = (block << 16) | offset; + + ret = sqfs_meta_writer_write_inode(wr->im, inode); +out: + free(inode); + return ret; +} + +int sqfs_serialize_fstree(const char *filename, sqfs_writer_t *wr) +{ + size_t i; + int ret; + + wr->super.inode_table_start = wr->outfile->get_size(wr->outfile); + + for (i = 0; i < wr->fs.unique_inode_count; ++i) { + ret = serialize_tree_node(filename, wr, wr->fs.inodes[i]); + if (ret) + goto out; + } + + ret = sqfs_meta_writer_flush(wr->im); + if (ret) + goto out; + + ret = sqfs_meta_writer_flush(wr->dm); + if (ret) + goto out; + + wr->super.root_inode_ref = wr->fs.root->inode_ref; + wr->super.directory_table_start = wr->outfile->get_size(wr->outfile); + + ret = sqfs_meta_write_write_to_file(wr->dm); + if (ret) + goto out; + + ret = 0; +out: + if (ret) + sqfs_perror(filename, "storing filesystem tree", ret); + return ret; +} -- cgit v1.2.3