From cdccc69c62579b0c13b35fad0728079652b8f3c9 Mon Sep 17 00:00:00 2001 From: David Oberhollenzer Date: Tue, 31 Jan 2023 11:21:30 +0100 Subject: Move library source into src sub-directory Signed-off-by: David Oberhollenzer --- lib/common/src/writer/cleanup.c | 39 ++++++ lib/common/src/writer/finish.c | 176 +++++++++++++++++++++++++ lib/common/src/writer/init.c | 218 +++++++++++++++++++++++++++++++ lib/common/src/writer/serialize_fstree.c | 202 ++++++++++++++++++++++++++++ 4 files changed, 635 insertions(+) create mode 100644 lib/common/src/writer/cleanup.c create mode 100644 lib/common/src/writer/finish.c create mode 100644 lib/common/src/writer/init.c create mode 100644 lib/common/src/writer/serialize_fstree.c (limited to 'lib/common/src/writer') diff --git a/lib/common/src/writer/cleanup.c b/lib/common/src/writer/cleanup.c new file mode 100644 index 0000000..a3fd039 --- /dev/null +++ b/lib/common/src/writer/cleanup.c @@ -0,0 +1,39 @@ +/* SPDX-License-Identifier: GPL-3.0-or-later */ +/* + * cleanup.c + * + * Copyright (C) 2019 David Oberhollenzer + */ +#include "simple_writer.h" + +#include + +void sqfs_writer_cleanup(sqfs_writer_t *sqfs, int status) +{ + sqfs_drop(sqfs->xwr); + sqfs_drop(sqfs->dirwr); + sqfs_drop(sqfs->dm); + sqfs_drop(sqfs->im); + sqfs_drop(sqfs->idtbl); + sqfs_drop(sqfs->data); + sqfs_drop(sqfs->blkwr); + sqfs_drop(sqfs->fragtbl); + sqfs_drop(sqfs->cmp); + sqfs_drop(sqfs->uncmp); + fstree_cleanup(&sqfs->fs); + sqfs_drop(sqfs->outfile); + + if (status != EXIT_SUCCESS) { +#if defined(_WIN32) || defined(__WINDOWS__) + WCHAR *path = path_to_windows(sqfs->filename); + + if (path != NULL) + DeleteFileW(path); + + free(path); +#else + unlink(sqfs->filename); +#endif + } +} + diff --git a/lib/common/src/writer/finish.c b/lib/common/src/writer/finish.c new file mode 100644 index 0000000..c539579 --- /dev/null +++ b/lib/common/src/writer/finish.c @@ -0,0 +1,176 @@ +/* SPDX-License-Identifier: GPL-3.0-or-later */ +/* + * finish.c + * + * Copyright (C) 2019 David Oberhollenzer + */ +#include "simple_writer.h" +#include "common.h" + +#include + +static void print_statistics(const sqfs_super_t *super, + const sqfs_block_processor_t *blk, + const sqfs_block_writer_t *wr) +{ + const sqfs_block_processor_stats_t *proc_stats; + sqfs_u64 bytes_written, blocks_written; + char read_sz[32], written_sz[32]; + size_t ratio; + + proc_stats = sqfs_block_processor_get_stats(blk); + blocks_written = wr->get_block_count(wr); + + bytes_written = super->inode_table_start - sizeof(*super); + + if (proc_stats->input_bytes_read > 0) { + ratio = (100 * bytes_written) / proc_stats->input_bytes_read; + } else { + ratio = 100; + } + + print_size(proc_stats->input_bytes_read, read_sz, false); + print_size(bytes_written, written_sz, false); + + fputs("---------------------------------------------------\n", stdout); + printf("Data bytes read: %s\n", read_sz); + printf("Data bytes written: %s\n", written_sz); + printf("Data compression ratio: " PRI_SZ "%%\n", ratio); + fputc('\n', stdout); + + printf("Data blocks written: " PRI_U64 "\n", blocks_written); + printf("Out of which were fragment blocks: " PRI_U64 "\n", + proc_stats->frag_block_count); + + printf("Duplicate blocks omitted: " PRI_U64 "\n", + proc_stats->data_block_count + proc_stats->frag_block_count - + blocks_written); + + printf("Sparse blocks omitted: " PRI_U64 "\n", + proc_stats->sparse_block_count); + fputc('\n', stdout); + + printf("Fragments actually written: " PRI_U64 "\n", + proc_stats->actual_frag_count); + printf("Duplicated fragments omitted: " PRI_U64 "\n", + proc_stats->total_frag_count - proc_stats->actual_frag_count); + printf("Total number of inodes: %u\n", super->inode_count); + printf("Number of unique group/user IDs: %u\n", super->id_count); + fputc('\n', stdout); +} + +static int padd_sqfs(sqfs_file_t *file, sqfs_u64 size, size_t blocksize) +{ + size_t padd_sz = size % blocksize; + int status = -1; + sqfs_u8 *buffer; + + if (padd_sz == 0) + return 0; + + padd_sz = blocksize - padd_sz; + + buffer = calloc(1, padd_sz); + if (buffer == NULL) + goto fail_errno; + + if (file->write_at(file, file->get_size(file), + buffer, padd_sz)) { + goto fail_errno; + } + + status = 0; +out: + free(buffer); + return status; +fail_errno: + perror("padding output file to block size"); + goto out; +} + +int sqfs_writer_finish(sqfs_writer_t *sqfs, const sqfs_writer_cfg_t *cfg) +{ + int ret; + + if (!cfg->quiet) + fputs("Waiting for remaining data blocks...\n", stdout); + + ret = sqfs_block_processor_finish(sqfs->data); + if (ret) { + sqfs_perror(cfg->filename, "finishing data blocks", ret); + return -1; + } + + if (!cfg->quiet) + fputs("Writing inodes and directories...\n", stdout); + + sqfs->super.inode_count = sqfs->fs.unique_inode_count; + + if (sqfs_serialize_fstree(cfg->filename, sqfs)) + return -1; + + if (!cfg->quiet) + fputs("Writing fragment table...\n", stdout); + + ret = sqfs_frag_table_write(sqfs->fragtbl, sqfs->outfile, + &sqfs->super, sqfs->cmp); + if (ret) { + sqfs_perror(cfg->filename, "writing fragment table", ret); + return -1; + } + + if (cfg->exportable) { + if (!cfg->quiet) + fputs("Writing export table...\n", stdout); + + + ret = sqfs_dir_writer_write_export_table(sqfs->dirwr, + sqfs->outfile, sqfs->cmp, + sqfs->fs.root->inode_num, + sqfs->fs.root->inode_ref, + &sqfs->super); + if (ret) + return -1; + } + + if (!cfg->quiet) + fputs("Writing ID table...\n", stdout); + + ret = sqfs_id_table_write(sqfs->idtbl, sqfs->outfile, + &sqfs->super, sqfs->cmp); + if (ret) { + sqfs_perror(cfg->filename, "writing ID table", ret); + return -1; + } + + if (!cfg->no_xattr) { + if (!cfg->quiet) + fputs("Writing extended attributes...\n", stdout); + + ret = sqfs_xattr_writer_flush(sqfs->xwr, sqfs->outfile, + &sqfs->super, sqfs->cmp); + if (ret) { + sqfs_perror(cfg->filename, + "writing extended attributes", ret); + return -1; + } + } + + sqfs->super.bytes_used = sqfs->outfile->get_size(sqfs->outfile); + + ret = sqfs_super_write(&sqfs->super, sqfs->outfile); + if (ret) { + sqfs_perror(cfg->filename, "updating super block", ret); + return -1; + } + + if (padd_sqfs(sqfs->outfile, sqfs->super.bytes_used, + cfg->devblksize)) { + return -1; + } + + if (!cfg->quiet) + print_statistics(&sqfs->super, sqfs->data, sqfs->blkwr); + + return 0; +} diff --git a/lib/common/src/writer/init.c b/lib/common/src/writer/init.c new file mode 100644 index 0000000..497fc6e --- /dev/null +++ b/lib/common/src/writer/init.c @@ -0,0 +1,218 @@ +/* SPDX-License-Identifier: GPL-3.0-or-later */ +/* + * init.c + * + * Copyright (C) 2019 David Oberhollenzer + */ +#include "simple_writer.h" +#include "compress_cli.h" +#include "common.h" + +#include +#include +#include + +#ifdef HAVE_SCHED_GETAFFINITY +#include + +static size_t os_get_num_jobs(void) +{ + cpu_set_t cpu_set; + CPU_ZERO(&cpu_set); + + if (sched_getaffinity(0, sizeof cpu_set, &cpu_set) == -1) + return 1; + else + return CPU_COUNT(&cpu_set); +} +#else +static size_t os_get_num_jobs(void) +{ + return 1; +} +#endif + +void sqfs_writer_cfg_init(sqfs_writer_cfg_t *cfg) +{ + memset(cfg, 0, sizeof(*cfg)); + + cfg->num_jobs = os_get_num_jobs(); + cfg->block_size = SQFS_DEFAULT_BLOCK_SIZE; + cfg->devblksize = SQFS_DEVBLK_SIZE; + cfg->comp_id = compressor_get_default(); +} + +int sqfs_writer_init(sqfs_writer_t *sqfs, const sqfs_writer_cfg_t *wrcfg) +{ + sqfs_block_processor_desc_t blkdesc; + sqfs_compressor_config_t cfg; + int ret, flags; + + sqfs->filename = wrcfg->filename; + + if (compressor_cfg_init_options(&cfg, wrcfg->comp_id, + wrcfg->block_size, + wrcfg->comp_extra)) { + return -1; + } + + sqfs->outfile = sqfs_open_file(wrcfg->filename, wrcfg->outmode); + if (sqfs->outfile == NULL) { + perror(wrcfg->filename); + return -1; + } + + if (fstree_init(&sqfs->fs, wrcfg->fs_defaults)) + goto fail_file; + + ret = sqfs_compressor_create(&cfg, &sqfs->cmp); + +#ifdef WITH_LZO + if (cfg.id == SQFS_COMP_LZO) { + if (sqfs->cmp != NULL) + sqfs_drop(sqfs->cmp); + + ret = lzo_compressor_create(&cfg, &sqfs->cmp); + } +#endif + + if (ret != 0) { + sqfs_perror(wrcfg->filename, "creating compressor", ret); + goto fail_fs; + } + + cfg.flags |= SQFS_COMP_FLAG_UNCOMPRESS; + ret = sqfs_compressor_create(&cfg, &sqfs->uncmp); + +#ifdef WITH_LZO + if (cfg.id == SQFS_COMP_LZO) { + if (ret == 0 && sqfs->uncmp != NULL) + sqfs_drop(sqfs->uncmp); + + ret = lzo_compressor_create(&cfg, &sqfs->uncmp); + } +#endif + + if (ret != 0) { + sqfs_perror(wrcfg->filename, "creating uncompressor", ret); + goto fail_cmp; + } + + ret = sqfs_super_init(&sqfs->super, wrcfg->block_size, + sqfs->fs.defaults.st_mtime, wrcfg->comp_id); + if (ret) { + sqfs_perror(wrcfg->filename, "initializing super block", ret); + goto fail_uncmp; + } + + ret = sqfs_super_write(&sqfs->super, sqfs->outfile); + if (ret) { + sqfs_perror(wrcfg->filename, "writing super block", ret); + goto fail_uncmp; + } + + ret = sqfs->cmp->write_options(sqfs->cmp, sqfs->outfile); + if (ret < 0) { + sqfs_perror(wrcfg->filename, "writing compressor options", ret); + goto fail_uncmp; + } + + if (ret > 0) + sqfs->super.flags |= SQFS_FLAG_COMPRESSOR_OPTIONS; + + sqfs->blkwr = sqfs_block_writer_create(sqfs->outfile, + wrcfg->devblksize, 0); + if (sqfs->blkwr == NULL) { + perror("creating block writer"); + goto fail_uncmp; + } + + sqfs->fragtbl = sqfs_frag_table_create(0); + if (sqfs->fragtbl == NULL) { + perror("creating fragment table"); + goto fail_blkwr; + } + + memset(&blkdesc, 0, sizeof(blkdesc)); + blkdesc.size = sizeof(blkdesc); + blkdesc.max_block_size = wrcfg->block_size; + blkdesc.num_workers = wrcfg->num_jobs; + blkdesc.max_backlog = wrcfg->max_backlog; + blkdesc.cmp = sqfs->cmp; + blkdesc.wr = sqfs->blkwr; + blkdesc.tbl = sqfs->fragtbl; + blkdesc.file = sqfs->outfile; + blkdesc.uncmp = sqfs->uncmp; + + ret = sqfs_block_processor_create_ex(&blkdesc, &sqfs->data); + if (ret != 0) { + sqfs_perror(wrcfg->filename, "creating data block processor", + ret); + goto fail_fragtbl; + } + + sqfs->idtbl = sqfs_id_table_create(0); + if (sqfs->idtbl == NULL) { + sqfs_perror(wrcfg->filename, "creating ID table", + SQFS_ERROR_ALLOC); + goto fail_data; + } + + if (!wrcfg->no_xattr) { + sqfs->xwr = sqfs_xattr_writer_create(0); + + if (sqfs->xwr == NULL) { + sqfs_perror(wrcfg->filename, "creating xattr writer", + SQFS_ERROR_ALLOC); + goto fail_id; + } + } + + sqfs->im = sqfs_meta_writer_create(sqfs->outfile, sqfs->cmp, 0); + if (sqfs->im == NULL) { + fputs("Error creating inode meta data writer.\n", stderr); + goto fail_xwr; + } + + sqfs->dm = sqfs_meta_writer_create(sqfs->outfile, sqfs->cmp, + SQFS_META_WRITER_KEEP_IN_MEMORY); + if (sqfs->dm == NULL) { + fputs("Error creating directory meta data writer.\n", stderr); + goto fail_im; + } + + flags = 0; + if (wrcfg->exportable) + flags |= SQFS_DIR_WRITER_CREATE_EXPORT_TABLE; + + sqfs->dirwr = sqfs_dir_writer_create(sqfs->dm, flags); + if (sqfs->dirwr == NULL) { + fputs("Error creating directory table writer.\n", stderr); + goto fail_dm; + } + + return 0; +fail_dm: + sqfs_drop(sqfs->dm); +fail_im: + sqfs_drop(sqfs->im); +fail_xwr: + sqfs_drop(sqfs->xwr); +fail_id: + sqfs_drop(sqfs->idtbl); +fail_data: + sqfs_drop(sqfs->data); +fail_fragtbl: + sqfs_drop(sqfs->fragtbl); +fail_blkwr: + sqfs_drop(sqfs->blkwr); +fail_uncmp: + sqfs_drop(sqfs->uncmp); +fail_cmp: + sqfs_drop(sqfs->cmp); +fail_fs: + fstree_cleanup(&sqfs->fs); +fail_file: + sqfs_drop(sqfs->outfile); + return -1; +} diff --git a/lib/common/src/writer/serialize_fstree.c b/lib/common/src/writer/serialize_fstree.c new file mode 100644 index 0000000..9776874 --- /dev/null +++ b/lib/common/src/writer/serialize_fstree.c @@ -0,0 +1,202 @@ +/* SPDX-License-Identifier: GPL-3.0-or-later */ +/* + * serialize_fstree.c + * + * Copyright (C) 2019 David Oberhollenzer + */ +#include "common.h" + +#include +#include +#include +#include + +static sqfs_inode_generic_t *tree_node_to_inode(tree_node_t *node) +{ + sqfs_inode_generic_t *inode; + size_t extra = 0; + + if (S_ISLNK(node->mode)) + extra = strlen(node->data.target); + + inode = calloc(1, sizeof(*inode) + extra); + if (inode == NULL) { + perror("creating inode"); + return NULL; + } + + switch (node->mode & S_IFMT) { + case S_IFSOCK: + inode->base.type = SQFS_INODE_SOCKET; + inode->data.ipc.nlink = node->link_count; + break; + case S_IFIFO: + inode->base.type = SQFS_INODE_FIFO; + inode->data.ipc.nlink = node->link_count; + break; + case S_IFLNK: + inode->base.type = SQFS_INODE_SLINK; + inode->data.slink.nlink = node->link_count; + inode->data.slink.target_size = extra; + memcpy(inode->extra, node->data.target, extra); + break; + case S_IFBLK: + inode->base.type = SQFS_INODE_BDEV; + inode->data.dev.nlink = node->link_count; + inode->data.dev.devno = node->data.devno; + break; + case S_IFCHR: + inode->base.type = SQFS_INODE_CDEV; + inode->data.dev.nlink = node->link_count; + inode->data.dev.devno = node->data.devno; + break; + default: + assert(0); + } + + return inode; +} + +static sqfs_inode_generic_t *write_dir_entries(const char *filename, + sqfs_dir_writer_t *dirw, + tree_node_t *node) +{ + sqfs_u32 xattr, parent_inode; + sqfs_inode_generic_t *inode; + tree_node_t *it, *tgt; + int ret; + + ret = sqfs_dir_writer_begin(dirw, 0); + if (ret) + goto fail; + + for (it = node->data.dir.children; it != NULL; it = it->next) { + if (it->mode == FSTREE_MODE_HARD_LINK_RESOLVED) { + tgt = it->data.target_node; + } else { + tgt = it; + } + + ret = sqfs_dir_writer_add_entry(dirw, it->name, tgt->inode_num, + tgt->inode_ref, tgt->mode); + if (ret) + goto fail; + } + + ret = sqfs_dir_writer_end(dirw); + if (ret) + goto fail; + + xattr = node->xattr_idx; + parent_inode = (node->parent == NULL) ? 0 : node->parent->inode_num; + + inode = sqfs_dir_writer_create_inode(dirw, 0, xattr, parent_inode); + if (inode == NULL) { + ret = SQFS_ERROR_ALLOC; + goto fail; + } + + if (inode->base.type == SQFS_INODE_DIR) { + inode->data.dir.nlink = node->link_count; + } else { + inode->data.dir_ext.nlink = node->link_count; + } + + return inode; +fail: + sqfs_perror(filename, "recoding directory entries", ret); + return NULL; +} + +static int serialize_tree_node(const char *filename, sqfs_writer_t *wr, + tree_node_t *n) +{ + sqfs_inode_generic_t *inode; + sqfs_u32 offset; + sqfs_u64 block; + int ret; + + if (S_ISDIR(n->mode)) { + inode = write_dir_entries(filename, wr->dirwr, n); + ret = SQFS_ERROR_INTERNAL; + } else if (S_ISREG(n->mode)) { + inode = n->data.file.inode; + n->data.file.inode = NULL; + ret = SQFS_ERROR_INTERNAL; + + if (inode->base.type == SQFS_INODE_FILE && n->link_count > 1) { + sqfs_inode_make_extended(inode); + inode->data.file_ext.nlink = n->link_count; + } else { + inode->data.file_ext.nlink = n->link_count; + } + } else { + inode = tree_node_to_inode(n); + ret = SQFS_ERROR_ALLOC; + } + + if (inode == NULL) + return ret; + + inode->base.mode = n->mode; + inode->base.mod_time = n->mod_time; + inode->base.inode_number = n->inode_num; + + sqfs_inode_set_xattr_index(inode, n->xattr_idx); + + if (n->xattr_idx == 0xFFFFFFFF && !S_ISDIR(n->mode)) + sqfs_inode_make_basic(inode); + + ret = sqfs_id_table_id_to_index(wr->idtbl, n->uid, + &inode->base.uid_idx); + if (ret) + goto out; + + ret = sqfs_id_table_id_to_index(wr->idtbl, n->gid, + &inode->base.gid_idx); + if (ret) + goto out; + + sqfs_meta_writer_get_position(wr->im, &block, &offset); + n->inode_ref = (block << 16) | offset; + + ret = sqfs_meta_writer_write_inode(wr->im, inode); +out: + free(inode); + return ret; +} + +int sqfs_serialize_fstree(const char *filename, sqfs_writer_t *wr) +{ + size_t i; + int ret; + + wr->super.inode_table_start = wr->outfile->get_size(wr->outfile); + + for (i = 0; i < wr->fs.unique_inode_count; ++i) { + ret = serialize_tree_node(filename, wr, wr->fs.inodes[i]); + if (ret) + goto out; + } + + ret = sqfs_meta_writer_flush(wr->im); + if (ret) + goto out; + + ret = sqfs_meta_writer_flush(wr->dm); + if (ret) + goto out; + + wr->super.root_inode_ref = wr->fs.root->inode_ref; + wr->super.directory_table_start = wr->outfile->get_size(wr->outfile); + + ret = sqfs_meta_write_write_to_file(wr->dm); + if (ret) + goto out; + + ret = 0; +out: + if (ret) + sqfs_perror(filename, "storing filesystem tree", ret); + return ret; +} -- cgit v1.2.3