From cdccc69c62579b0c13b35fad0728079652b8f3c9 Mon Sep 17 00:00:00 2001 From: David Oberhollenzer Date: Tue, 31 Jan 2023 11:21:30 +0100 Subject: Move library source into src sub-directory Signed-off-by: David Oberhollenzer --- lib/common/src/comp_lzo.c | 285 +++++++++++++++++++++++ lib/common/src/comp_opt.c | 378 +++++++++++++++++++++++++++++++ lib/common/src/compress.c | 88 +++++++ lib/common/src/data_reader_dump.c | 65 ++++++ lib/common/src/data_writer.c | 54 +++++ lib/common/src/data_writer_ostream.c | 91 ++++++++ lib/common/src/hardlink.c | 101 +++++++++ lib/common/src/parse_size.c | 79 +++++++ lib/common/src/perror.c | 79 +++++++ lib/common/src/print_size.c | 38 ++++ lib/common/src/print_version.c | 25 ++ lib/common/src/writer/cleanup.c | 39 ++++ lib/common/src/writer/finish.c | 176 ++++++++++++++ lib/common/src/writer/init.c | 218 ++++++++++++++++++ lib/common/src/writer/serialize_fstree.c | 202 +++++++++++++++++ 15 files changed, 1918 insertions(+) create mode 100644 lib/common/src/comp_lzo.c create mode 100644 lib/common/src/comp_opt.c create mode 100644 lib/common/src/compress.c create mode 100644 lib/common/src/data_reader_dump.c create mode 100644 lib/common/src/data_writer.c create mode 100644 lib/common/src/data_writer_ostream.c create mode 100644 lib/common/src/hardlink.c create mode 100644 lib/common/src/parse_size.c create mode 100644 lib/common/src/perror.c create mode 100644 lib/common/src/print_size.c create mode 100644 lib/common/src/print_version.c create mode 100644 lib/common/src/writer/cleanup.c create mode 100644 lib/common/src/writer/finish.c create mode 100644 lib/common/src/writer/init.c create mode 100644 lib/common/src/writer/serialize_fstree.c (limited to 'lib/common/src') diff --git a/lib/common/src/comp_lzo.c b/lib/common/src/comp_lzo.c new file mode 100644 index 0000000..2021d34 --- /dev/null +++ b/lib/common/src/comp_lzo.c @@ -0,0 +1,285 @@ +/* SPDX-License-Identifier: GPL-3.0-or-later */ +/* + * comp_lzo.c + * + * Copyright (C) 2019 David Oberhollenzer + */ +#include "config.h" +#include "compress_cli.h" +#include "compat.h" + +#include +#include +#include + +#include + +#define LZO_MAX_SIZE(size) (size + (size / 16) + 64 + 3) + +#define LZO_NUM_ALGS (sizeof(lzo_algs) / sizeof(lzo_algs[0])) + +typedef int (*lzo_cb_t)(const lzo_bytep src, lzo_uint src_len, lzo_bytep dst, + lzo_uintp dst_len, lzo_voidp wrkmem); + +static const struct { + lzo_cb_t compress; + size_t bufsize; +} lzo_algs[] = { + [SQFS_LZO1X_1] = { + .compress = lzo1x_1_compress, + .bufsize = LZO1X_1_MEM_COMPRESS, + }, + [SQFS_LZO1X_1_11] = { + .compress = lzo1x_1_11_compress, + .bufsize = LZO1X_1_11_MEM_COMPRESS, + }, + [SQFS_LZO1X_1_12] = { + .compress = lzo1x_1_12_compress, + .bufsize = LZO1X_1_12_MEM_COMPRESS, + }, + [SQFS_LZO1X_1_15] = { + .compress = lzo1x_1_15_compress, + .bufsize = LZO1X_1_15_MEM_COMPRESS, + }, + [SQFS_LZO1X_999] = { + .compress = lzo1x_999_compress, + .bufsize = LZO1X_999_MEM_COMPRESS, + }, +}; + +typedef struct { + sqfs_compressor_t base; + size_t block_size; + int algorithm; + int level; + + size_t buf_size; + size_t work_size; + + sqfs_u8 buffer[]; +} lzo_compressor_t; + +typedef struct { + sqfs_u32 algorithm; + sqfs_u32 level; +} lzo_options_t; + +static int lzo_write_options(sqfs_compressor_t *base, sqfs_file_t *file) +{ + lzo_compressor_t *lzo = (lzo_compressor_t *)base; + sqfs_u8 buffer[sizeof(lzo_options_t) + 2]; + lzo_options_t opt; + sqfs_u16 header; + int ret; + + if (lzo->algorithm == SQFS_LZO_DEFAULT_ALG && + lzo->level == SQFS_LZO_DEFAULT_LEVEL) { + return 0; + } + + opt.algorithm = htole32(lzo->algorithm); + + if (lzo->algorithm == SQFS_LZO1X_999) { + opt.level = htole32(lzo->level); + } else { + opt.level = 0; + } + + header = htole16(0x8000 | sizeof(opt)); + + memcpy(buffer, &header, sizeof(header)); + memcpy(buffer + 2, &opt, sizeof(opt)); + + ret = file->write_at(file, sizeof(sqfs_super_t), + buffer, sizeof(buffer)); + + return ret ? ret : (int)sizeof(buffer); +} + +static int lzo_read_options(sqfs_compressor_t *base, sqfs_file_t *file) +{ + lzo_compressor_t *lzo = (lzo_compressor_t *)base; + sqfs_u8 buffer[sizeof(lzo_options_t) + 2]; + lzo_options_t opt; + sqfs_u16 header; + int ret; + + ret = file->read_at(file, sizeof(sqfs_super_t), + buffer, sizeof(buffer)); + if (ret) + return ret; + + memcpy(&header, buffer, sizeof(header)); + if (le16toh(header) != (0x8000 | sizeof(opt))) + return SQFS_ERROR_CORRUPTED; + + memcpy(&opt, buffer + 2, sizeof(opt)); + lzo->algorithm = le32toh(opt.algorithm); + lzo->level = le32toh(opt.level); + + switch(lzo->algorithm) { + case SQFS_LZO1X_1: + case SQFS_LZO1X_1_11: + case SQFS_LZO1X_1_12: + case SQFS_LZO1X_1_15: + if (lzo->level != 0) + return SQFS_ERROR_UNSUPPORTED; + break; + case SQFS_LZO1X_999: + if (lzo->level < 1 || lzo->level > 9) + return SQFS_ERROR_UNSUPPORTED; + break; + default: + return SQFS_ERROR_UNSUPPORTED; + } + + return 0; +} + +static sqfs_s32 lzo_comp_block(sqfs_compressor_t *base, const sqfs_u8 *in, + sqfs_u32 size, sqfs_u8 *out, sqfs_u32 outsize) +{ + lzo_compressor_t *lzo = (lzo_compressor_t *)base; + void *scratch; + lzo_uint len; + int ret; + + if (size >= 0x7FFFFFFF) + return 0; + + scratch = lzo->buffer + lzo->work_size; + len = lzo->buf_size - lzo->work_size; + + if (lzo->algorithm == SQFS_LZO1X_999 && + lzo->level != SQFS_LZO_DEFAULT_LEVEL) { + ret = lzo1x_999_compress_level(in, size, scratch, &len, + lzo->buffer, NULL, 0, 0, + lzo->level); + } else { + ret = lzo_algs[lzo->algorithm].compress(in, size, scratch, + &len, lzo->buffer); + } + + if (ret != LZO_E_OK) + return SQFS_ERROR_COMPRESSOR; + + if (len < size && len <= outsize) { + memcpy(out, scratch, len); + return len; + } + + return 0; +} + +static sqfs_s32 lzo_uncomp_block(sqfs_compressor_t *base, const sqfs_u8 *in, + sqfs_u32 size, sqfs_u8 *out, sqfs_u32 outsize) +{ + lzo_compressor_t *lzo = (lzo_compressor_t *)base; + lzo_uint len = outsize; + int ret; + + if (outsize >= 0x7FFFFFFF) + return 0; + + ret = lzo1x_decompress_safe(in, size, out, &len, lzo->buffer); + + if (ret != LZO_E_OK) + return SQFS_ERROR_COMPRESSOR; + + return len; +} + +static void lzo_get_configuration(const sqfs_compressor_t *base, + sqfs_compressor_config_t *cfg) +{ + const lzo_compressor_t *lzo = (const lzo_compressor_t *)base; + + memset(cfg, 0, sizeof(*cfg)); + cfg->id = SQFS_COMP_LZO; + cfg->block_size = lzo->block_size; + + cfg->opt.lzo.algorithm = lzo->algorithm; + cfg->level = lzo->level; + + if (base->do_block == lzo_uncomp_block) + cfg->flags |= SQFS_COMP_FLAG_UNCOMPRESS; +} + +static sqfs_object_t *lzo_create_copy(const sqfs_object_t *cmp) +{ + const lzo_compressor_t *other = (const lzo_compressor_t *)cmp; + lzo_compressor_t *lzo; + + lzo = calloc(1, sizeof(*lzo) + other->buf_size); + if (lzo == NULL) + return NULL; + + memcpy(lzo, other, sizeof(*lzo)); + return (sqfs_object_t *)lzo; +} + +static void lzo_destroy(sqfs_object_t *base) +{ + free(base); +} + +int lzo_compressor_create(const sqfs_compressor_config_t *cfg, + sqfs_compressor_t **out) +{ + sqfs_compressor_t *base; + lzo_compressor_t *lzo; + size_t scratch_size; + + if (cfg->flags & ~SQFS_COMP_FLAG_GENERIC_ALL) + return SQFS_ERROR_UNSUPPORTED; + + if (cfg->opt.lzo.algorithm >= LZO_NUM_ALGS || + lzo_algs[cfg->opt.lzo.algorithm].compress == NULL) { + return SQFS_ERROR_UNSUPPORTED; + } + + if (cfg->opt.lzo.algorithm == SQFS_LZO1X_999) { + if (cfg->level > SQFS_LZO_MAX_LEVEL) + return SQFS_ERROR_UNSUPPORTED; + } else if (cfg->level != 0) { + return SQFS_ERROR_UNSUPPORTED; + } + + /* XXX: liblzo does not do bounds checking internally, + we need our own internal scratch buffer at worst case size... */ + if (cfg->flags & SQFS_COMP_FLAG_UNCOMPRESS) { + scratch_size = 0; + } else { + scratch_size = cfg->block_size; + if (scratch_size < SQFS_META_BLOCK_SIZE) + scratch_size = SQFS_META_BLOCK_SIZE; + + scratch_size = LZO_MAX_SIZE(scratch_size); + } + + /* ...in addition to the LZO work space buffer of course */ + scratch_size += lzo_algs[cfg->opt.lzo.algorithm].bufsize; + + lzo = calloc(1, sizeof(*lzo) + scratch_size); + base = (sqfs_compressor_t *)lzo; + + if (lzo == NULL) + return SQFS_ERROR_ALLOC; + + sqfs_object_init(lzo, lzo_destroy, lzo_create_copy); + + lzo->block_size = cfg->block_size; + lzo->algorithm = cfg->opt.lzo.algorithm; + lzo->level = cfg->level; + lzo->buf_size = scratch_size; + lzo->work_size = lzo_algs[cfg->opt.lzo.algorithm].bufsize; + + base->get_configuration = lzo_get_configuration; + base->do_block = (cfg->flags & SQFS_COMP_FLAG_UNCOMPRESS) ? + lzo_uncomp_block : lzo_comp_block; + base->write_options = lzo_write_options; + base->read_options = lzo_read_options; + + *out = base; + return 0; +} diff --git a/lib/common/src/comp_opt.c b/lib/common/src/comp_opt.c new file mode 100644 index 0000000..d605c0c --- /dev/null +++ b/lib/common/src/comp_opt.c @@ -0,0 +1,378 @@ +/* SPDX-License-Identifier: GPL-3.0-or-later */ +/* + * comp_opt.c + * + * Copyright (C) 2019 David Oberhollenzer + */ +#include "config.h" +#include "common.h" + +#include +#include +#include +#include +#include + +typedef struct { + const char *name; + sqfs_u16 flag; +} flag_t; + +static const flag_t gzip_flags[] = { + { "default", SQFS_COMP_FLAG_GZIP_DEFAULT }, + { "filtered", SQFS_COMP_FLAG_GZIP_FILTERED }, + { "huffman", SQFS_COMP_FLAG_GZIP_HUFFMAN }, + { "rle", SQFS_COMP_FLAG_GZIP_RLE }, + { "fixed", SQFS_COMP_FLAG_GZIP_FIXED }, +}; + +static const flag_t xz_flags[] = { + { "x86", SQFS_COMP_FLAG_XZ_X86 }, + { "powerpc", SQFS_COMP_FLAG_XZ_POWERPC }, + { "ia64", SQFS_COMP_FLAG_XZ_IA64 }, + { "arm", SQFS_COMP_FLAG_XZ_ARM }, + { "armthumb", SQFS_COMP_FLAG_XZ_ARMTHUMB }, + { "sparc", SQFS_COMP_FLAG_XZ_SPARC }, + { "extreme", SQFS_COMP_FLAG_XZ_EXTREME }, +}; + +static const flag_t lzma_flags[] = { + { "extreme", SQFS_COMP_FLAG_LZMA_EXTREME }, +}; + +static const flag_t lz4_flags[] = { + { "hc", SQFS_COMP_FLAG_LZ4_HC }, +}; + +static const struct { + const flag_t *flags; + size_t count; +} comp_flags[SQFS_COMP_MAX + 1] = { + [SQFS_COMP_GZIP] = { gzip_flags, sizeof(gzip_flags) / sizeof(flag_t) }, + [SQFS_COMP_XZ] = { xz_flags, sizeof(xz_flags) / sizeof(flag_t) }, + [SQFS_COMP_LZMA] = { lzma_flags, sizeof(lzma_flags) / sizeof(flag_t) }, + [SQFS_COMP_LZ4] = { lz4_flags, sizeof(lz4_flags) / sizeof(flag_t) }, +}; + +static const char *lzo_algs[] = { + [SQFS_LZO1X_1] = "lzo1x_1", + [SQFS_LZO1X_1_11] = "lzo1x_1_11", + [SQFS_LZO1X_1_12] = "lzo1x_1_12", + [SQFS_LZO1X_1_15] = "lzo1x_1_15", + [SQFS_LZO1X_999] = "lzo1x_999", +}; + +static int set_flag(sqfs_compressor_config_t *cfg, const char *name) +{ + const flag_t *flags = comp_flags[cfg->id].flags; + size_t i, num_flags = comp_flags[cfg->id].count; + + for (i = 0; i < num_flags; ++i) { + if (strcmp(flags[i].name, name) == 0) { + cfg->flags |= flags[i].flag; + return 0; + } + } + + return -1; +} + +static int find_lzo_alg(sqfs_compressor_config_t *cfg, const char *name) +{ + size_t i; + + for (i = 0; i < sizeof(lzo_algs) / sizeof(lzo_algs[0]); ++i) { + if (strcmp(lzo_algs[i], name) == 0) { + cfg->opt.lzo.algorithm = i; + return 0; + } + } + + return -1; +} + +enum { + OPT_WINDOW = 0, + OPT_LEVEL, + OPT_ALG, + OPT_DICT, + OPT_LC, + OPT_LP, + OPT_PB, + OPT_COUNT, +}; +static char *const token[] = { + [OPT_WINDOW] = (char *)"window", + [OPT_LEVEL] = (char *)"level", + [OPT_ALG] = (char *)"algorithm", + [OPT_DICT] = (char *)"dictsize", + [OPT_LC] = (char *)"lc", + [OPT_LP] = (char *)"lp", + [OPT_PB] = (char *)"pb", + NULL +}; + +static int opt_available[SQFS_COMP_MAX + 1] = { + [SQFS_COMP_GZIP] = (1 << OPT_WINDOW) | (1 << OPT_LEVEL), + [SQFS_COMP_XZ] = (1 << OPT_LEVEL) | (1 << OPT_DICT) | (1 << OPT_LC) | + (1 << OPT_LP) | (1 << OPT_PB), + [SQFS_COMP_LZMA] = (1 << OPT_LEVEL) | (1 << OPT_DICT) | (1 << OPT_LC) | + (1 << OPT_LP) | (1 << OPT_PB), + [SQFS_COMP_ZSTD] = (1 << OPT_LEVEL), + [SQFS_COMP_LZO] = (1 << OPT_LEVEL) | (1 << OPT_ALG), +}; + +static const struct { + int min; + int max; +} value_range[SQFS_COMP_MAX + 1][OPT_COUNT] = { + [SQFS_COMP_GZIP] = { + [OPT_LEVEL] = { SQFS_GZIP_MIN_LEVEL, SQFS_GZIP_MAX_LEVEL }, + [OPT_WINDOW] = { SQFS_GZIP_MIN_WINDOW, SQFS_GZIP_MAX_WINDOW }, + }, + [SQFS_COMP_XZ] = { + [OPT_LEVEL] = { SQFS_XZ_MIN_LEVEL, SQFS_XZ_MAX_LEVEL }, + [OPT_DICT] = { SQFS_XZ_MIN_DICT_SIZE, SQFS_XZ_MAX_DICT_SIZE }, + [OPT_LC] = { SQFS_XZ_MIN_LC, SQFS_XZ_MAX_LC }, + [OPT_LP] = { SQFS_XZ_MIN_LP, SQFS_XZ_MAX_LP }, + [OPT_PB] = { SQFS_XZ_MIN_PB, SQFS_XZ_MAX_PB }, + }, + [SQFS_COMP_LZMA] = { + [OPT_LEVEL] = { SQFS_LZMA_MIN_LEVEL, SQFS_LZMA_MAX_LEVEL }, + [OPT_DICT] = { SQFS_LZMA_MIN_DICT_SIZE, + SQFS_LZMA_MAX_DICT_SIZE }, + [OPT_LC] = { SQFS_LZMA_MIN_LC, SQFS_LZMA_MAX_LC }, + [OPT_LP] = { SQFS_LZMA_MIN_LP, SQFS_LZMA_MAX_LP }, + [OPT_PB] = { SQFS_LZMA_MIN_PB, SQFS_LZMA_MAX_PB }, + }, + [SQFS_COMP_ZSTD] = { + [OPT_LEVEL] = { SQFS_ZSTD_MIN_LEVEL, SQFS_ZSTD_MAX_LEVEL }, + }, + [SQFS_COMP_LZO] = { + [OPT_LEVEL] = { SQFS_LZO_MIN_LEVEL, SQFS_LZO_MAX_LEVEL }, + }, +}; + +int compressor_cfg_init_options(sqfs_compressor_config_t *cfg, + SQFS_COMPRESSOR id, + size_t block_size, char *options) +{ + char *subopts, *value; + int opt, ival; + size_t szval; + + if (sqfs_compressor_config_init(cfg, id, block_size, 0)) + return -1; + + if (options == NULL) + return 0; + + subopts = options; + + while (*subopts != '\0') { + opt = getsubopt(&subopts, token, &value); + + if (opt < 0) { + if (set_flag(cfg, value)) + goto fail_opt; + continue; + } + + if (!(opt_available[cfg->id] & (1 << opt))) + goto fail_opt; + + if (value == NULL) + goto fail_value; + + if (opt == OPT_ALG) { + if (find_lzo_alg(cfg, value)) + goto fail_lzo_alg; + continue; + } + + if (opt == OPT_DICT) { + if (parse_size("Parsing LZMA dictionary size", + &szval, value, cfg->block_size)) { + return -1; + } + ival = szval; + } else { + ival = strtol(value, NULL, 10); + } + + if (ival < value_range[cfg->id][opt].min) + goto fail_range; + if (ival > value_range[cfg->id][opt].max) + goto fail_range; + + switch (opt) { + case OPT_LEVEL: cfg->level = ival; break; + case OPT_LC: cfg->opt.xz.lc = ival; break; + case OPT_LP: cfg->opt.xz.lp = ival; break; + case OPT_PB: cfg->opt.xz.pb = ival; break; + case OPT_WINDOW: cfg->opt.gzip.window_size = ival; break; + case OPT_DICT: cfg->opt.xz.dict_size = ival; break; + default: + break; + } + } + + if (cfg->id == SQFS_COMP_XZ || cfg->id == SQFS_COMP_LZMA) { + if ((cfg->opt.xz.lp + cfg->opt.xz.lc) > 4) + goto fail_sum_lp_lc; + } + + return 0; +fail_sum_lp_lc: + fputs("Sum of XZ lc + lp must not exceed 4.\n", stderr); + return -1; +fail_lzo_alg: + fprintf(stderr, "Unknown lzo variant '%s'.\n", value); + return -1; +fail_range: + fprintf(stderr, "`%s` must be a number between %d and %d.\n", + token[opt], value_range[cfg->id][opt].min, + value_range[cfg->id][opt].max); + return -1; +fail_opt: + fprintf(stderr, "Unknown compressor option '%s'.\n", value); + return -1; +fail_value: + fprintf(stderr, "Missing value for compressor option '%s'.\n", + token[opt]); + return -1; +} + +typedef void (*compressor_help_fun_t)(void); + +static void gzip_print_help(void) +{ + size_t i; + + printf( +"Available options for gzip compressor:\n" +"\n" +" level= Compression level. Value from 1 to 9.\n" +" Defaults to %d.\n" +" window= Deflate compression window size. Value from 8 to 15.\n" +" Defaults to %d.\n" +"\n" +"In additon to the options, one or more strategies can be specified.\n" +"If multiple stratgies are provided, the one yielding the best compression\n" +"ratio will be used.\n" +"\n" +"The following strategies are available:\n", + SQFS_GZIP_DEFAULT_LEVEL, SQFS_GZIP_DEFAULT_WINDOW); + + for (i = 0; i < sizeof(gzip_flags) / sizeof(gzip_flags[0]); ++i) + printf("\t%s\n", gzip_flags[i].name); +} + +static void lz4_print_help(void) +{ + fputs("Available options for lz4 compressor:\n" + "\n" + " hc If present, use slower but better compressing\n" + " variant of lz4.\n" + "\n", + stdout); +} + +static void lzo_print_help(void) +{ + size_t i; + + fputs("Available options for lzo compressor:\n" + "\n" + " algorithm= Specify the variant of lzo to use.\n" + " Defaults to 'lzo1x_999'.\n" + " level= For lzo1x_999, the compression level.\n" + " Value from 1 to 9. Defaults to 8.\n" + " Ignored if algorithm is not lzo1x_999.\n" + "\n" + "Available algorithms:\n", + stdout); + + for (i = 0; i < sizeof(lzo_algs) / sizeof(lzo_algs[0]); ++i) + printf("\t%s\n", lzo_algs[i]); +} + +static void xz_lzma_print_help(void) +{ + size_t i; + + printf( +"Available options for LZMA and XZ (LZMA v2) compressors:\n" +"\n" +" dictsize= Dictionary size. Either a value in bytes or a\n" +" percentage of the block size. Defaults to 100%%.\n" +" The suffix '%%' indicates a percentage. 'K' and 'M'\n" +" can also be used for kibi and mebi bytes\n" +" respecitively.\n" +" level= Compression level. Value from %d to %d.\n" +" For XZ, defaults to %d, for LZMA defaults to %d.\n" +" lc= Number of literal context bits.\n" +" How many of the highest bits of the previous\n" +" uncompressed byte to take into account when\n" +" predicting the bits of the next byte.\n" +" Default is %d.\n" +" lp= Number of literal position bits.\n" +" Affects what kind of alignment in the uncompressed\n" +" data is assumed when encoding bytes.\n" +" Default is %d.\n" +" pb= Number of position bits.\n" +" This is the log2 of the assumed underlying alignment\n" +" of the input data, i.e. pb=0 means single byte\n" +" allignment, pb=1 means 16 bit, 2 means 32 bit.\n" +" Default is %d.\n" +" extreme If this flag is set, try to crunch the data extra hard\n" +" without increasing the decompressors memory\n" +" requirements." +"\n" +"If values are set, the sum of lc + lp must not exceed 4.\n" +"The maximum for pb is %d.\n" +"\n" +"In additon to the options, for the XZ compressor, one or more bcj filters\n" +"can be specified.\n" +"If multiple filters are provided, the one yielding the best compression\n" +"ratio will be used.\n" +"\n" +"The following filters are available:\n", + SQFS_XZ_MIN_LEVEL, SQFS_XZ_MAX_LEVEL, + SQFS_XZ_DEFAULT_LEVEL, SQFS_LZMA_DEFAULT_LEVEL, + SQFS_XZ_DEFAULT_LC, SQFS_XZ_DEFAULT_LP, SQFS_XZ_DEFAULT_PB, + SQFS_XZ_MAX_PB); + + for (i = 0; i < sizeof(xz_flags) / sizeof(xz_flags[0]); ++i) + printf("\t%s\n", xz_flags[i].name); +} + +static void zstd_print_help(void) +{ + printf("Available options for zstd compressor:\n" + "\n" + " level= Set compression level. Defaults to %d.\n" + " Maximum is %d.\n" + "\n", + SQFS_ZSTD_DEFAULT_LEVEL, SQFS_ZSTD_MAX_LEVEL); +} + +static const compressor_help_fun_t helpfuns[SQFS_COMP_MAX + 1] = { + [SQFS_COMP_GZIP] = gzip_print_help, + [SQFS_COMP_XZ] = xz_lzma_print_help, + [SQFS_COMP_LZMA] = xz_lzma_print_help, + [SQFS_COMP_LZO] = lzo_print_help, + [SQFS_COMP_LZ4] = lz4_print_help, + [SQFS_COMP_ZSTD] = zstd_print_help, +}; + +void compressor_print_help(SQFS_COMPRESSOR id) +{ + if (id < SQFS_COMP_MIN || id > SQFS_COMP_MAX) + return; + + if (helpfuns[id] == NULL) + return; + + helpfuns[id](); +} diff --git a/lib/common/src/compress.c b/lib/common/src/compress.c new file mode 100644 index 0000000..1e0ca06 --- /dev/null +++ b/lib/common/src/compress.c @@ -0,0 +1,88 @@ +/* SPDX-License-Identifier: GPL-3.0-or-later */ +/* + * compress.c + * + * Copyright (C) 2019 David Oberhollenzer + */ +#include "config.h" +#include "compress_cli.h" + +#include +#include + +static int cmp_ids[] = { + SQFS_COMP_XZ, + SQFS_COMP_ZSTD, + SQFS_COMP_GZIP, + SQFS_COMP_LZ4, + SQFS_COMP_LZO, +}; + +SQFS_COMPRESSOR compressor_get_default(void) +{ + sqfs_compressor_config_t cfg; + sqfs_compressor_t *temp; + size_t i; + int ret; + + for (i = 0; i < sizeof(cmp_ids) / sizeof(cmp_ids[0]); ++i) { + sqfs_compressor_config_init(&cfg, cmp_ids[i], + SQFS_DEFAULT_BLOCK_SIZE, 0); + + ret = sqfs_compressor_create(&cfg, &temp); + + if (ret == 0) { + sqfs_drop(temp); + return cmp_ids[i]; + } + } + +#ifdef WITH_LZO + return SQFS_COMP_LZO; +#else + assert(0); +#endif +} + +void compressor_print_available(void) +{ + sqfs_compressor_config_t cfg; + sqfs_compressor_t *temp; + bool have_compressor; + int i, ret, defcomp; + const char *name; + + defcomp = compressor_get_default(); + + fputs("Available SquashFS block compressors:\n", stdout); + + for (i = SQFS_COMP_MIN; i <= SQFS_COMP_MAX; ++i) { + sqfs_compressor_config_init(&cfg, i, + SQFS_DEFAULT_BLOCK_SIZE, 0); + + ret = sqfs_compressor_create(&cfg, &temp); + have_compressor = false; + + if (ret == 0) { + sqfs_drop(temp); + have_compressor = true; + } else { +#ifdef WITH_LZO + if (i == SQFS_COMP_LZO) + have_compressor = true; +#endif + } + + if (have_compressor) { + name = sqfs_compressor_name_from_id(i); + + if (defcomp == i) { + printf("\t%s (default)\n", name); + } else { + printf("\t%s\n", name); + } + } + } + + fputc('\n', stdout); +} diff --git a/lib/common/src/data_reader_dump.c b/lib/common/src/data_reader_dump.c new file mode 100644 index 0000000..7902c25 --- /dev/null +++ b/lib/common/src/data_reader_dump.c @@ -0,0 +1,65 @@ +/* SPDX-License-Identifier: GPL-3.0-or-later */ +/* + * data_reader_dump.c + * + * Copyright (C) 2019 David Oberhollenzer + */ +#include "common.h" + +#include +#include +#include +#include + +int sqfs_data_reader_dump(const char *name, sqfs_data_reader_t *data, + const sqfs_inode_generic_t *inode, + ostream_t *fp, size_t block_size) +{ + size_t i, diff, chunk_size; + sqfs_u64 filesz; + sqfs_u8 *chunk; + int err; + + sqfs_inode_get_file_size(inode, &filesz); + + for (i = 0; i < sqfs_inode_get_file_block_count(inode); ++i) { + diff = (filesz < block_size) ? filesz : block_size; + + if (SQFS_IS_SPARSE_BLOCK(inode->extra[i])) { + if (ostream_append_sparse(fp, diff)) + return -1; + } else { + err = sqfs_data_reader_get_block(data, inode, i, + &chunk_size, &chunk); + if (err) { + sqfs_perror(name, "reading data block", err); + return -1; + } + + err = ostream_append(fp, chunk, chunk_size); + free(chunk); + + if (err) + return -1; + } + + filesz -= diff; + } + + if (filesz > 0) { + err = sqfs_data_reader_get_fragment(data, inode, + &chunk_size, &chunk); + if (err) { + sqfs_perror(name, "reading fragment block", err); + return -1; + } + + err = ostream_append(fp, chunk, chunk_size); + free(chunk); + + if (err) + return -1; + } + + return 0; +} diff --git a/lib/common/src/data_writer.c b/lib/common/src/data_writer.c new file mode 100644 index 0000000..ceccaac --- /dev/null +++ b/lib/common/src/data_writer.c @@ -0,0 +1,54 @@ +/* SPDX-License-Identifier: GPL-3.0-or-later */ +/* + * data_writer.c + * + * Copyright (C) 2019 David Oberhollenzer + */ +#include "common.h" + +static sqfs_u8 buffer[4096]; + +int write_data_from_file(const char *filename, sqfs_block_processor_t *data, + sqfs_inode_generic_t **inode, sqfs_file_t *file, + int flags) +{ + sqfs_u64 filesz, offset; + size_t diff; + int ret; + + ret = sqfs_block_processor_begin_file(data, inode, NULL, flags); + if (ret) { + sqfs_perror(filename, "beginning file data blocks", ret); + return -1; + } + + filesz = file->get_size(file); + + for (offset = 0; offset < filesz; offset += diff) { + if (filesz - offset > sizeof(buffer)) { + diff = sizeof(buffer); + } else { + diff = filesz - offset; + } + + ret = file->read_at(file, offset, buffer, diff); + if (ret) { + sqfs_perror(filename, "reading file range", ret); + return -1; + } + + ret = sqfs_block_processor_append(data, buffer, diff); + if (ret) { + sqfs_perror(filename, "packing file data", ret); + return -1; + } + } + + ret = sqfs_block_processor_end_file(data); + if (ret) { + sqfs_perror(filename, "finishing file data", ret); + return -1; + } + + return 0; +} diff --git a/lib/common/src/data_writer_ostream.c b/lib/common/src/data_writer_ostream.c new file mode 100644 index 0000000..fbd0431 --- /dev/null +++ b/lib/common/src/data_writer_ostream.c @@ -0,0 +1,91 @@ +/* SPDX-License-Identifier: GPL-3.0-or-later */ +/* + * data_writer_ostream.c + * + * Copyright (C) 2019 David Oberhollenzer + */ +#include "config.h" +#include "common.h" + +#include + +typedef struct{ + ostream_t base; + + sqfs_block_processor_t *proc; + const char *filename; +} data_writer_ostream_t; + +static int stream_append(ostream_t *base, const void *data, size_t size) +{ + data_writer_ostream_t *strm = (data_writer_ostream_t *)base; + int ret; + + ret = sqfs_block_processor_append(strm->proc, data, size); + + if (ret != 0) { + sqfs_perror(strm->filename, NULL, ret); + return -1; + } + + return 0; +} + +static int stream_flush(ostream_t *base) +{ + data_writer_ostream_t *strm = (data_writer_ostream_t *)base; + int ret; + + ret = sqfs_block_processor_end_file(strm->proc); + + if (ret != 0) { + sqfs_perror(strm->filename, NULL, ret); + return -1; + } + + return 0; +} + +static const char *stream_get_filename(ostream_t *base) +{ + data_writer_ostream_t *strm = (data_writer_ostream_t *)base; + + return strm->filename; +} + +static void stream_destroy(sqfs_object_t *base) +{ + free(base); +} + +ostream_t *data_writer_ostream_create(const char *filename, + sqfs_block_processor_t *proc, + sqfs_inode_generic_t **inode, + int flags) +{ + data_writer_ostream_t *strm = calloc(1, sizeof(*strm)); + ostream_t *base = (ostream_t *)strm; + int ret; + + if (strm == NULL) { + perror(filename); + return NULL; + } + + sqfs_object_init(strm, stream_destroy, NULL); + + ret = sqfs_block_processor_begin_file(proc, inode, NULL, flags); + + if (ret != 0) { + sqfs_perror(filename, NULL, ret); + free(strm); + return NULL; + } + + strm->proc = proc; + strm->filename = filename; + base->append = stream_append; + base->flush = stream_flush; + base->get_filename = stream_get_filename; + return base; +} diff --git a/lib/common/src/hardlink.c b/lib/common/src/hardlink.c new file mode 100644 index 0000000..e43df33 --- /dev/null +++ b/lib/common/src/hardlink.c @@ -0,0 +1,101 @@ +/* SPDX-License-Identifier: GPL-3.0-or-later */ +/* + * hardlink.c + * + * Copyright (C) 2019 David Oberhollenzer + */ +#include "common.h" +#include "util/rbtree.h" +#include "util/util.h" + +#include +#include +#include + +static int map_nodes(rbtree_t *inumtree, sqfs_hard_link_t **out, + const sqfs_tree_node_t *n) +{ + const sqfs_tree_node_t *target; + sqfs_hard_link_t *lnk; + rbtree_node_t *tn; + sqfs_u32 idx; + int ret; + + /* XXX: refuse to generate hard links to directories */ + if (n->children != NULL) { + for (n = n->children; n != NULL; n = n->next) { + ret = map_nodes(inumtree, out, n); + if (ret != 0) + return ret; + } + return 0; + } + + if (!is_filename_sane((const char *)n->name, false)) + return SQFS_ERROR_CORRUPTED; + + idx = n->inode->base.inode_number; + tn = rbtree_lookup(inumtree, &idx); + + if (tn == NULL) + return rbtree_insert(inumtree, &idx, &n); + + target = *((const sqfs_tree_node_t **)rbtree_node_value(tn)); + + lnk = calloc(1, sizeof(*lnk)); + if (lnk == NULL) + return SQFS_ERROR_ALLOC; + + lnk->inode_number = idx; + ret = sqfs_tree_node_get_path(target, &lnk->target); + if (ret != 0) { + free(lnk); + return ret; + } + + if (canonicalize_name(lnk->target) != 0) { + sqfs_free(lnk->target); + free(lnk); + return SQFS_ERROR_CORRUPTED; + } + + lnk->next = (*out); + (*out) = lnk; + return 0; +} + +static int compare_inum(const void *ctx, const void *lhs, const void *rhs) +{ + sqfs_u32 l = *((const sqfs_u32 *)lhs), r = *((const sqfs_u32 *)rhs); + (void)ctx; + + return l < r ? -1 : (l > r ? 1 : 0); +} + +int sqfs_tree_find_hard_links(const sqfs_tree_node_t *root, + sqfs_hard_link_t **out) +{ + sqfs_hard_link_t *lnk = NULL; + rbtree_t inumtree; + int ret; + + ret = rbtree_init(&inumtree, sizeof(sqfs_u32), + sizeof(sqfs_tree_node_t *), + compare_inum); + if (ret != 0) + return ret; + + ret = map_nodes(&inumtree, out, root); + rbtree_cleanup(&inumtree); + + if (ret != 0) { + while ((*out) != NULL) { + lnk = (*out); + (*out) = lnk->next; + free(lnk->target); + free(lnk); + } + } + + return ret; +} diff --git a/lib/common/src/parse_size.c b/lib/common/src/parse_size.c new file mode 100644 index 0000000..3e79a19 --- /dev/null +++ b/lib/common/src/parse_size.c @@ -0,0 +1,79 @@ +/* SPDX-License-Identifier: GPL-3.0-or-later */ +/* + * parse_size.c + * + * Copyright (C) 2019 David Oberhollenzer + */ +#include "common.h" + +#include +#include + +int parse_size(const char *what, size_t *out, const char *str, + size_t reference) +{ + const char *in = str; + size_t acc = 0, x; + + if (!isdigit(*in)) + goto fail_nan; + + while (isdigit(*in)) { + x = *(in++) - '0'; + + if (SZ_MUL_OV(acc, 10, &acc)) + goto fail_ov; + + if (SZ_ADD_OV(acc, x, &acc)) + goto fail_ov; + } + + switch (*in) { + case 'k': + case 'K': + if (SZ_MUL_OV(acc, 1024, &acc)) + goto fail_ov; + ++in; + break; + case 'm': + case 'M': + if (SZ_MUL_OV(acc, 1048576, &acc)) + goto fail_ov; + ++in; + break; + case 'g': + case 'G': + if (SZ_MUL_OV(acc, 1073741824, &acc)) + goto fail_ov; + ++in; + break; + case '%': + if (reference == 0) + goto fail_suffix; + + if (SZ_MUL_OV(acc, reference, &acc)) + goto fail_ov; + + acc /= 100; + break; + case '\0': + break; + default: + goto fail_suffix; + } + + if (*in != '\0') + goto fail_suffix; + + *out = acc; + return 0; +fail_nan: + fprintf(stderr, "%s: '%s' is not a number.\n", what, str); + return -1; +fail_ov: + fprintf(stderr, "%s: numeric overflow parsing '%s'.\n", what, str); + return -1; +fail_suffix: + fprintf(stderr, "%s: unknown suffix in '%s'.\n", what, str); + return -1; +} diff --git a/lib/common/src/perror.c b/lib/common/src/perror.c new file mode 100644 index 0000000..53a8c16 --- /dev/null +++ b/lib/common/src/perror.c @@ -0,0 +1,79 @@ +/* SPDX-License-Identifier: GPL-3.0-or-later */ +/* + * print_version.c + * + * Copyright (C) 2019 David Oberhollenzer + */ +#include "common.h" + +#include + +void sqfs_perror(const char *file, const char *action, int error_code) +{ + const char *errstr; + + switch (error_code) { + case SQFS_ERROR_ALLOC: + errstr = "out of memory"; + break; + case SQFS_ERROR_IO: + errstr = "I/O error"; + break; + case SQFS_ERROR_COMPRESSOR: + errstr = "internal compressor error"; + break; + case SQFS_ERROR_INTERNAL: + errstr = "internal error"; + break; + case SQFS_ERROR_CORRUPTED: + errstr = "data corrupted"; + break; + case SQFS_ERROR_UNSUPPORTED: + errstr = "unknown or not supported"; + break; + case SQFS_ERROR_OVERFLOW: + errstr = "numeric overflow"; + break; + case SQFS_ERROR_OUT_OF_BOUNDS: + errstr = "location out of bounds"; + break; + case SFQS_ERROR_SUPER_MAGIC: + errstr = "wrong magic value in super block"; + break; + case SFQS_ERROR_SUPER_VERSION: + errstr = "wrong squashfs version in super block"; + break; + case SQFS_ERROR_SUPER_BLOCK_SIZE: + errstr = "invalid block size specified in super block"; + break; + case SQFS_ERROR_NOT_DIR: + errstr = "target is not a directory"; + break; + case SQFS_ERROR_NO_ENTRY: + errstr = "no such file or directory"; + break; + case SQFS_ERROR_LINK_LOOP: + errstr = "hard link loop detected"; + break; + case SQFS_ERROR_NOT_FILE: + errstr = "target is not a file"; + break; + case SQFS_ERROR_ARG_INVALID: + errstr = "invalid argument"; + break; + case SQFS_ERROR_SEQUENCE: + errstr = "illegal oder of operations"; + break; + default: + errstr = "libsquashfs returned an unknown error code"; + break; + } + + if (file != NULL) + fprintf(stderr, "%s: ", file); + + if (action != NULL) + fprintf(stderr, "%s: ", action); + + fprintf(stderr, "%s.\n", errstr); +} diff --git a/lib/common/src/print_size.c b/lib/common/src/print_size.c new file mode 100644 index 0000000..6e76805 --- /dev/null +++ b/lib/common/src/print_size.c @@ -0,0 +1,38 @@ +/* SPDX-License-Identifier: GPL-3.0-or-later */ +/* + * print_size.c + * + * Copyright (C) 2019 David Oberhollenzer + */ +#include "config.h" +#include "common.h" + +void print_size(sqfs_u64 size, char *buffer, bool round_to_int) +{ + static const char *fractions = "0112334456678899"; + static const char *suffices = "kMGTPEZY"; + unsigned int fraction; + int suffix = -1; + + while (size > 1024) { + ++suffix; + fraction = size % 1024; + size /= 1024; + } + + if (suffix >= 0) { + fraction /= 64; + + if (round_to_int) { + size = fraction >= 8 ? (size + 1) : size; + + sprintf(buffer, "%u%c", (unsigned int)size, + suffices[suffix]); + } else { + sprintf(buffer, "%u.%c%c", (unsigned int)size, + fractions[fraction], suffices[suffix]); + } + } else { + sprintf(buffer, "%u", (unsigned int)size); + } +} diff --git a/lib/common/src/print_version.c b/lib/common/src/print_version.c new file mode 100644 index 0000000..0c7fe5c --- /dev/null +++ b/lib/common/src/print_version.c @@ -0,0 +1,25 @@ +/* SPDX-License-Identifier: GPL-3.0-or-later */ +/* + * print_version.c + * + * Copyright (C) 2019 David Oberhollenzer + */ +#include "common.h" + +#include + +#define LICENSE_SHORT "GPLv3+" +#define LICENSE_LONG "GNU GPL version 3 or later" +#define LICENSE_URL "https://gnu.org/licenses/gpl.html" + +static const char *version_string = +"%s (%s) %s\n" +"Copyright (c) 2019 David Oberhollenzer et al\n" +"License " LICENSE_SHORT ": " LICENSE_LONG " <" LICENSE_URL ">.\n" +"This is free software: you are free to change and redistribute it.\n" +"There is NO WARRANTY, to the extent permitted by law.\n"; + +void print_version(const char *progname) +{ + printf(version_string, progname, PACKAGE_NAME, PACKAGE_VERSION); +} diff --git a/lib/common/src/writer/cleanup.c b/lib/common/src/writer/cleanup.c new file mode 100644 index 0000000..a3fd039 --- /dev/null +++ b/lib/common/src/writer/cleanup.c @@ -0,0 +1,39 @@ +/* SPDX-License-Identifier: GPL-3.0-or-later */ +/* + * cleanup.c + * + * Copyright (C) 2019 David Oberhollenzer + */ +#include "simple_writer.h" + +#include + +void sqfs_writer_cleanup(sqfs_writer_t *sqfs, int status) +{ + sqfs_drop(sqfs->xwr); + sqfs_drop(sqfs->dirwr); + sqfs_drop(sqfs->dm); + sqfs_drop(sqfs->im); + sqfs_drop(sqfs->idtbl); + sqfs_drop(sqfs->data); + sqfs_drop(sqfs->blkwr); + sqfs_drop(sqfs->fragtbl); + sqfs_drop(sqfs->cmp); + sqfs_drop(sqfs->uncmp); + fstree_cleanup(&sqfs->fs); + sqfs_drop(sqfs->outfile); + + if (status != EXIT_SUCCESS) { +#if defined(_WIN32) || defined(__WINDOWS__) + WCHAR *path = path_to_windows(sqfs->filename); + + if (path != NULL) + DeleteFileW(path); + + free(path); +#else + unlink(sqfs->filename); +#endif + } +} + diff --git a/lib/common/src/writer/finish.c b/lib/common/src/writer/finish.c new file mode 100644 index 0000000..c539579 --- /dev/null +++ b/lib/common/src/writer/finish.c @@ -0,0 +1,176 @@ +/* SPDX-License-Identifier: GPL-3.0-or-later */ +/* + * finish.c + * + * Copyright (C) 2019 David Oberhollenzer + */ +#include "simple_writer.h" +#include "common.h" + +#include + +static void print_statistics(const sqfs_super_t *super, + const sqfs_block_processor_t *blk, + const sqfs_block_writer_t *wr) +{ + const sqfs_block_processor_stats_t *proc_stats; + sqfs_u64 bytes_written, blocks_written; + char read_sz[32], written_sz[32]; + size_t ratio; + + proc_stats = sqfs_block_processor_get_stats(blk); + blocks_written = wr->get_block_count(wr); + + bytes_written = super->inode_table_start - sizeof(*super); + + if (proc_stats->input_bytes_read > 0) { + ratio = (100 * bytes_written) / proc_stats->input_bytes_read; + } else { + ratio = 100; + } + + print_size(proc_stats->input_bytes_read, read_sz, false); + print_size(bytes_written, written_sz, false); + + fputs("---------------------------------------------------\n", stdout); + printf("Data bytes read: %s\n", read_sz); + printf("Data bytes written: %s\n", written_sz); + printf("Data compression ratio: " PRI_SZ "%%\n", ratio); + fputc('\n', stdout); + + printf("Data blocks written: " PRI_U64 "\n", blocks_written); + printf("Out of which were fragment blocks: " PRI_U64 "\n", + proc_stats->frag_block_count); + + printf("Duplicate blocks omitted: " PRI_U64 "\n", + proc_stats->data_block_count + proc_stats->frag_block_count - + blocks_written); + + printf("Sparse blocks omitted: " PRI_U64 "\n", + proc_stats->sparse_block_count); + fputc('\n', stdout); + + printf("Fragments actually written: " PRI_U64 "\n", + proc_stats->actual_frag_count); + printf("Duplicated fragments omitted: " PRI_U64 "\n", + proc_stats->total_frag_count - proc_stats->actual_frag_count); + printf("Total number of inodes: %u\n", super->inode_count); + printf("Number of unique group/user IDs: %u\n", super->id_count); + fputc('\n', stdout); +} + +static int padd_sqfs(sqfs_file_t *file, sqfs_u64 size, size_t blocksize) +{ + size_t padd_sz = size % blocksize; + int status = -1; + sqfs_u8 *buffer; + + if (padd_sz == 0) + return 0; + + padd_sz = blocksize - padd_sz; + + buffer = calloc(1, padd_sz); + if (buffer == NULL) + goto fail_errno; + + if (file->write_at(file, file->get_size(file), + buffer, padd_sz)) { + goto fail_errno; + } + + status = 0; +out: + free(buffer); + return status; +fail_errno: + perror("padding output file to block size"); + goto out; +} + +int sqfs_writer_finish(sqfs_writer_t *sqfs, const sqfs_writer_cfg_t *cfg) +{ + int ret; + + if (!cfg->quiet) + fputs("Waiting for remaining data blocks...\n", stdout); + + ret = sqfs_block_processor_finish(sqfs->data); + if (ret) { + sqfs_perror(cfg->filename, "finishing data blocks", ret); + return -1; + } + + if (!cfg->quiet) + fputs("Writing inodes and directories...\n", stdout); + + sqfs->super.inode_count = sqfs->fs.unique_inode_count; + + if (sqfs_serialize_fstree(cfg->filename, sqfs)) + return -1; + + if (!cfg->quiet) + fputs("Writing fragment table...\n", stdout); + + ret = sqfs_frag_table_write(sqfs->fragtbl, sqfs->outfile, + &sqfs->super, sqfs->cmp); + if (ret) { + sqfs_perror(cfg->filename, "writing fragment table", ret); + return -1; + } + + if (cfg->exportable) { + if (!cfg->quiet) + fputs("Writing export table...\n", stdout); + + + ret = sqfs_dir_writer_write_export_table(sqfs->dirwr, + sqfs->outfile, sqfs->cmp, + sqfs->fs.root->inode_num, + sqfs->fs.root->inode_ref, + &sqfs->super); + if (ret) + return -1; + } + + if (!cfg->quiet) + fputs("Writing ID table...\n", stdout); + + ret = sqfs_id_table_write(sqfs->idtbl, sqfs->outfile, + &sqfs->super, sqfs->cmp); + if (ret) { + sqfs_perror(cfg->filename, "writing ID table", ret); + return -1; + } + + if (!cfg->no_xattr) { + if (!cfg->quiet) + fputs("Writing extended attributes...\n", stdout); + + ret = sqfs_xattr_writer_flush(sqfs->xwr, sqfs->outfile, + &sqfs->super, sqfs->cmp); + if (ret) { + sqfs_perror(cfg->filename, + "writing extended attributes", ret); + return -1; + } + } + + sqfs->super.bytes_used = sqfs->outfile->get_size(sqfs->outfile); + + ret = sqfs_super_write(&sqfs->super, sqfs->outfile); + if (ret) { + sqfs_perror(cfg->filename, "updating super block", ret); + return -1; + } + + if (padd_sqfs(sqfs->outfile, sqfs->super.bytes_used, + cfg->devblksize)) { + return -1; + } + + if (!cfg->quiet) + print_statistics(&sqfs->super, sqfs->data, sqfs->blkwr); + + return 0; +} diff --git a/lib/common/src/writer/init.c b/lib/common/src/writer/init.c new file mode 100644 index 0000000..497fc6e --- /dev/null +++ b/lib/common/src/writer/init.c @@ -0,0 +1,218 @@ +/* SPDX-License-Identifier: GPL-3.0-or-later */ +/* + * init.c + * + * Copyright (C) 2019 David Oberhollenzer + */ +#include "simple_writer.h" +#include "compress_cli.h" +#include "common.h" + +#include +#include +#include + +#ifdef HAVE_SCHED_GETAFFINITY +#include + +static size_t os_get_num_jobs(void) +{ + cpu_set_t cpu_set; + CPU_ZERO(&cpu_set); + + if (sched_getaffinity(0, sizeof cpu_set, &cpu_set) == -1) + return 1; + else + return CPU_COUNT(&cpu_set); +} +#else +static size_t os_get_num_jobs(void) +{ + return 1; +} +#endif + +void sqfs_writer_cfg_init(sqfs_writer_cfg_t *cfg) +{ + memset(cfg, 0, sizeof(*cfg)); + + cfg->num_jobs = os_get_num_jobs(); + cfg->block_size = SQFS_DEFAULT_BLOCK_SIZE; + cfg->devblksize = SQFS_DEVBLK_SIZE; + cfg->comp_id = compressor_get_default(); +} + +int sqfs_writer_init(sqfs_writer_t *sqfs, const sqfs_writer_cfg_t *wrcfg) +{ + sqfs_block_processor_desc_t blkdesc; + sqfs_compressor_config_t cfg; + int ret, flags; + + sqfs->filename = wrcfg->filename; + + if (compressor_cfg_init_options(&cfg, wrcfg->comp_id, + wrcfg->block_size, + wrcfg->comp_extra)) { + return -1; + } + + sqfs->outfile = sqfs_open_file(wrcfg->filename, wrcfg->outmode); + if (sqfs->outfile == NULL) { + perror(wrcfg->filename); + return -1; + } + + if (fstree_init(&sqfs->fs, wrcfg->fs_defaults)) + goto fail_file; + + ret = sqfs_compressor_create(&cfg, &sqfs->cmp); + +#ifdef WITH_LZO + if (cfg.id == SQFS_COMP_LZO) { + if (sqfs->cmp != NULL) + sqfs_drop(sqfs->cmp); + + ret = lzo_compressor_create(&cfg, &sqfs->cmp); + } +#endif + + if (ret != 0) { + sqfs_perror(wrcfg->filename, "creating compressor", ret); + goto fail_fs; + } + + cfg.flags |= SQFS_COMP_FLAG_UNCOMPRESS; + ret = sqfs_compressor_create(&cfg, &sqfs->uncmp); + +#ifdef WITH_LZO + if (cfg.id == SQFS_COMP_LZO) { + if (ret == 0 && sqfs->uncmp != NULL) + sqfs_drop(sqfs->uncmp); + + ret = lzo_compressor_create(&cfg, &sqfs->uncmp); + } +#endif + + if (ret != 0) { + sqfs_perror(wrcfg->filename, "creating uncompressor", ret); + goto fail_cmp; + } + + ret = sqfs_super_init(&sqfs->super, wrcfg->block_size, + sqfs->fs.defaults.st_mtime, wrcfg->comp_id); + if (ret) { + sqfs_perror(wrcfg->filename, "initializing super block", ret); + goto fail_uncmp; + } + + ret = sqfs_super_write(&sqfs->super, sqfs->outfile); + if (ret) { + sqfs_perror(wrcfg->filename, "writing super block", ret); + goto fail_uncmp; + } + + ret = sqfs->cmp->write_options(sqfs->cmp, sqfs->outfile); + if (ret < 0) { + sqfs_perror(wrcfg->filename, "writing compressor options", ret); + goto fail_uncmp; + } + + if (ret > 0) + sqfs->super.flags |= SQFS_FLAG_COMPRESSOR_OPTIONS; + + sqfs->blkwr = sqfs_block_writer_create(sqfs->outfile, + wrcfg->devblksize, 0); + if (sqfs->blkwr == NULL) { + perror("creating block writer"); + goto fail_uncmp; + } + + sqfs->fragtbl = sqfs_frag_table_create(0); + if (sqfs->fragtbl == NULL) { + perror("creating fragment table"); + goto fail_blkwr; + } + + memset(&blkdesc, 0, sizeof(blkdesc)); + blkdesc.size = sizeof(blkdesc); + blkdesc.max_block_size = wrcfg->block_size; + blkdesc.num_workers = wrcfg->num_jobs; + blkdesc.max_backlog = wrcfg->max_backlog; + blkdesc.cmp = sqfs->cmp; + blkdesc.wr = sqfs->blkwr; + blkdesc.tbl = sqfs->fragtbl; + blkdesc.file = sqfs->outfile; + blkdesc.uncmp = sqfs->uncmp; + + ret = sqfs_block_processor_create_ex(&blkdesc, &sqfs->data); + if (ret != 0) { + sqfs_perror(wrcfg->filename, "creating data block processor", + ret); + goto fail_fragtbl; + } + + sqfs->idtbl = sqfs_id_table_create(0); + if (sqfs->idtbl == NULL) { + sqfs_perror(wrcfg->filename, "creating ID table", + SQFS_ERROR_ALLOC); + goto fail_data; + } + + if (!wrcfg->no_xattr) { + sqfs->xwr = sqfs_xattr_writer_create(0); + + if (sqfs->xwr == NULL) { + sqfs_perror(wrcfg->filename, "creating xattr writer", + SQFS_ERROR_ALLOC); + goto fail_id; + } + } + + sqfs->im = sqfs_meta_writer_create(sqfs->outfile, sqfs->cmp, 0); + if (sqfs->im == NULL) { + fputs("Error creating inode meta data writer.\n", stderr); + goto fail_xwr; + } + + sqfs->dm = sqfs_meta_writer_create(sqfs->outfile, sqfs->cmp, + SQFS_META_WRITER_KEEP_IN_MEMORY); + if (sqfs->dm == NULL) { + fputs("Error creating directory meta data writer.\n", stderr); + goto fail_im; + } + + flags = 0; + if (wrcfg->exportable) + flags |= SQFS_DIR_WRITER_CREATE_EXPORT_TABLE; + + sqfs->dirwr = sqfs_dir_writer_create(sqfs->dm, flags); + if (sqfs->dirwr == NULL) { + fputs("Error creating directory table writer.\n", stderr); + goto fail_dm; + } + + return 0; +fail_dm: + sqfs_drop(sqfs->dm); +fail_im: + sqfs_drop(sqfs->im); +fail_xwr: + sqfs_drop(sqfs->xwr); +fail_id: + sqfs_drop(sqfs->idtbl); +fail_data: + sqfs_drop(sqfs->data); +fail_fragtbl: + sqfs_drop(sqfs->fragtbl); +fail_blkwr: + sqfs_drop(sqfs->blkwr); +fail_uncmp: + sqfs_drop(sqfs->uncmp); +fail_cmp: + sqfs_drop(sqfs->cmp); +fail_fs: + fstree_cleanup(&sqfs->fs); +fail_file: + sqfs_drop(sqfs->outfile); + return -1; +} diff --git a/lib/common/src/writer/serialize_fstree.c b/lib/common/src/writer/serialize_fstree.c new file mode 100644 index 0000000..9776874 --- /dev/null +++ b/lib/common/src/writer/serialize_fstree.c @@ -0,0 +1,202 @@ +/* SPDX-License-Identifier: GPL-3.0-or-later */ +/* + * serialize_fstree.c + * + * Copyright (C) 2019 David Oberhollenzer + */ +#include "common.h" + +#include +#include +#include +#include + +static sqfs_inode_generic_t *tree_node_to_inode(tree_node_t *node) +{ + sqfs_inode_generic_t *inode; + size_t extra = 0; + + if (S_ISLNK(node->mode)) + extra = strlen(node->data.target); + + inode = calloc(1, sizeof(*inode) + extra); + if (inode == NULL) { + perror("creating inode"); + return NULL; + } + + switch (node->mode & S_IFMT) { + case S_IFSOCK: + inode->base.type = SQFS_INODE_SOCKET; + inode->data.ipc.nlink = node->link_count; + break; + case S_IFIFO: + inode->base.type = SQFS_INODE_FIFO; + inode->data.ipc.nlink = node->link_count; + break; + case S_IFLNK: + inode->base.type = SQFS_INODE_SLINK; + inode->data.slink.nlink = node->link_count; + inode->data.slink.target_size = extra; + memcpy(inode->extra, node->data.target, extra); + break; + case S_IFBLK: + inode->base.type = SQFS_INODE_BDEV; + inode->data.dev.nlink = node->link_count; + inode->data.dev.devno = node->data.devno; + break; + case S_IFCHR: + inode->base.type = SQFS_INODE_CDEV; + inode->data.dev.nlink = node->link_count; + inode->data.dev.devno = node->data.devno; + break; + default: + assert(0); + } + + return inode; +} + +static sqfs_inode_generic_t *write_dir_entries(const char *filename, + sqfs_dir_writer_t *dirw, + tree_node_t *node) +{ + sqfs_u32 xattr, parent_inode; + sqfs_inode_generic_t *inode; + tree_node_t *it, *tgt; + int ret; + + ret = sqfs_dir_writer_begin(dirw, 0); + if (ret) + goto fail; + + for (it = node->data.dir.children; it != NULL; it = it->next) { + if (it->mode == FSTREE_MODE_HARD_LINK_RESOLVED) { + tgt = it->data.target_node; + } else { + tgt = it; + } + + ret = sqfs_dir_writer_add_entry(dirw, it->name, tgt->inode_num, + tgt->inode_ref, tgt->mode); + if (ret) + goto fail; + } + + ret = sqfs_dir_writer_end(dirw); + if (ret) + goto fail; + + xattr = node->xattr_idx; + parent_inode = (node->parent == NULL) ? 0 : node->parent->inode_num; + + inode = sqfs_dir_writer_create_inode(dirw, 0, xattr, parent_inode); + if (inode == NULL) { + ret = SQFS_ERROR_ALLOC; + goto fail; + } + + if (inode->base.type == SQFS_INODE_DIR) { + inode->data.dir.nlink = node->link_count; + } else { + inode->data.dir_ext.nlink = node->link_count; + } + + return inode; +fail: + sqfs_perror(filename, "recoding directory entries", ret); + return NULL; +} + +static int serialize_tree_node(const char *filename, sqfs_writer_t *wr, + tree_node_t *n) +{ + sqfs_inode_generic_t *inode; + sqfs_u32 offset; + sqfs_u64 block; + int ret; + + if (S_ISDIR(n->mode)) { + inode = write_dir_entries(filename, wr->dirwr, n); + ret = SQFS_ERROR_INTERNAL; + } else if (S_ISREG(n->mode)) { + inode = n->data.file.inode; + n->data.file.inode = NULL; + ret = SQFS_ERROR_INTERNAL; + + if (inode->base.type == SQFS_INODE_FILE && n->link_count > 1) { + sqfs_inode_make_extended(inode); + inode->data.file_ext.nlink = n->link_count; + } else { + inode->data.file_ext.nlink = n->link_count; + } + } else { + inode = tree_node_to_inode(n); + ret = SQFS_ERROR_ALLOC; + } + + if (inode == NULL) + return ret; + + inode->base.mode = n->mode; + inode->base.mod_time = n->mod_time; + inode->base.inode_number = n->inode_num; + + sqfs_inode_set_xattr_index(inode, n->xattr_idx); + + if (n->xattr_idx == 0xFFFFFFFF && !S_ISDIR(n->mode)) + sqfs_inode_make_basic(inode); + + ret = sqfs_id_table_id_to_index(wr->idtbl, n->uid, + &inode->base.uid_idx); + if (ret) + goto out; + + ret = sqfs_id_table_id_to_index(wr->idtbl, n->gid, + &inode->base.gid_idx); + if (ret) + goto out; + + sqfs_meta_writer_get_position(wr->im, &block, &offset); + n->inode_ref = (block << 16) | offset; + + ret = sqfs_meta_writer_write_inode(wr->im, inode); +out: + free(inode); + return ret; +} + +int sqfs_serialize_fstree(const char *filename, sqfs_writer_t *wr) +{ + size_t i; + int ret; + + wr->super.inode_table_start = wr->outfile->get_size(wr->outfile); + + for (i = 0; i < wr->fs.unique_inode_count; ++i) { + ret = serialize_tree_node(filename, wr, wr->fs.inodes[i]); + if (ret) + goto out; + } + + ret = sqfs_meta_writer_flush(wr->im); + if (ret) + goto out; + + ret = sqfs_meta_writer_flush(wr->dm); + if (ret) + goto out; + + wr->super.root_inode_ref = wr->fs.root->inode_ref; + wr->super.directory_table_start = wr->outfile->get_size(wr->outfile); + + ret = sqfs_meta_write_write_to_file(wr->dm); + if (ret) + goto out; + + ret = 0; +out: + if (ret) + sqfs_perror(filename, "storing filesystem tree", ret); + return ret; +} -- cgit v1.2.3