From 9b3d958fb7c37855a63ed75707281c61dc1d44c4 Mon Sep 17 00:00:00 2001 From: David Oberhollenzer Date: Sat, 31 Aug 2019 16:22:31 +0200 Subject: Merge libcompress.a into libsquashfs.a Signed-off-by: David Oberhollenzer --- lib/sqfs/comp/block_processor.c | 72 ++++++ lib/sqfs/comp/block_processor_parallel.c | 340 +++++++++++++++++++++++++ lib/sqfs/comp/compressor.c | 183 ++++++++++++++ lib/sqfs/comp/create_block.c | 37 +++ lib/sqfs/comp/gzip.c | 412 +++++++++++++++++++++++++++++++ lib/sqfs/comp/internal.h | 44 ++++ lib/sqfs/comp/lz4.c | 159 ++++++++++++ lib/sqfs/comp/lzo.c | 321 ++++++++++++++++++++++++ lib/sqfs/comp/process_block.c | 38 +++ lib/sqfs/comp/xz.c | 360 +++++++++++++++++++++++++++ lib/sqfs/comp/zstd.c | 188 ++++++++++++++ 11 files changed, 2154 insertions(+) create mode 100644 lib/sqfs/comp/block_processor.c create mode 100644 lib/sqfs/comp/block_processor_parallel.c create mode 100644 lib/sqfs/comp/compressor.c create mode 100644 lib/sqfs/comp/create_block.c create mode 100644 lib/sqfs/comp/gzip.c create mode 100644 lib/sqfs/comp/internal.h create mode 100644 lib/sqfs/comp/lz4.c create mode 100644 lib/sqfs/comp/lzo.c create mode 100644 lib/sqfs/comp/process_block.c create mode 100644 lib/sqfs/comp/xz.c create mode 100644 lib/sqfs/comp/zstd.c (limited to 'lib/sqfs') diff --git a/lib/sqfs/comp/block_processor.c b/lib/sqfs/comp/block_processor.c new file mode 100644 index 0000000..06dc384 --- /dev/null +++ b/lib/sqfs/comp/block_processor.c @@ -0,0 +1,72 @@ +/* SPDX-License-Identifier: GPL-3.0-or-later */ +/* + * block_processor.c + * + * Copyright (C) 2019 David Oberhollenzer + */ +#include "config.h" + +#include "block_processor.h" +#include "util.h" + +#include +#include +#include + +struct block_processor_t { + size_t max_block_size; + compressor_t *cmp; + block_cb cb; + void *user; + int status; + + uint8_t scratch[]; +}; + +block_processor_t *block_processor_create(size_t max_block_size, + compressor_t *cmp, + unsigned int num_workers, + void *user, + block_cb callback) +{ + block_processor_t *proc = alloc_flex(sizeof(*proc), 1, max_block_size); + (void)num_workers; + + if (proc == NULL) { + perror("Creating block processor"); + return NULL; + } + + proc->max_block_size = max_block_size; + proc->cmp = cmp; + proc->cb = callback; + proc->user = user; + return proc; +} + +void block_processor_destroy(block_processor_t *proc) +{ + free(proc); +} + +int block_processor_enqueue(block_processor_t *proc, block_t *block) +{ + if (process_block(block, proc->cmp, + proc->scratch, proc->max_block_size)) + goto fail; + + if (proc->cb(proc->user, block)) + goto fail; + + free(block); + return 0; +fail: + free(block); + proc->status = -1; + return -1; +} + +int block_processor_finish(block_processor_t *proc) +{ + return proc->status; +} diff --git a/lib/sqfs/comp/block_processor_parallel.c b/lib/sqfs/comp/block_processor_parallel.c new file mode 100644 index 0000000..b58ad5c --- /dev/null +++ b/lib/sqfs/comp/block_processor_parallel.c @@ -0,0 +1,340 @@ +/* SPDX-License-Identifier: GPL-3.0-or-later */ +/* + * block_processor.c + * + * Copyright (C) 2019 David Oberhollenzer + */ +#include "config.h" + +#include "block_processor.h" +#include "util.h" + +#include +#include +#include +#include + +#define MAX_BACKLOG_FACTOR (10) + +typedef struct { + block_processor_t *shared; + compressor_t *cmp; + pthread_t thread; + uint8_t scratch[]; +} compress_worker_t; + +struct block_processor_t { + pthread_mutex_t mtx; + pthread_cond_t queue_cond; + pthread_cond_t done_cond; + + /* needs rw access by worker and main thread */ + block_t *queue; + block_t *queue_last; + + block_t *done; + bool terminate; + size_t backlog; + + /* used by main thread only */ + uint32_t enqueue_id; + uint32_t dequeue_id; + + unsigned int num_workers; + block_cb cb; + void *user; + int status; + + /* used only by workers */ + size_t max_block_size; + + compress_worker_t *workers[]; +}; + +static void store_completed_block(block_processor_t *shared, block_t *blk) +{ + block_t *it = shared->done, *prev = NULL; + + while (it != NULL) { + if (it->sequence_number >= blk->sequence_number) + break; + prev = it; + it = it->next; + } + + if (prev == NULL) { + blk->next = shared->done; + shared->done = blk; + } else { + blk->next = prev->next; + prev->next = blk; + } +} + +static void *worker_proc(void *arg) +{ + compress_worker_t *worker = arg; + block_processor_t *shared = worker->shared; + block_t *blk = NULL; + + for (;;) { + pthread_mutex_lock(&shared->mtx); + if (blk != NULL) { + store_completed_block(shared, blk); + shared->backlog -= 1; + pthread_cond_broadcast(&shared->done_cond); + } + + while (shared->queue == NULL && !shared->terminate) { + pthread_cond_wait(&shared->queue_cond, + &shared->mtx); + } + + if (shared->terminate) { + pthread_mutex_unlock(&shared->mtx); + break; + } + + blk = shared->queue; + shared->queue = blk->next; + blk->next = NULL; + + if (shared->queue == NULL) + shared->queue_last = NULL; + pthread_mutex_unlock(&shared->mtx); + + if (process_block(blk, worker->cmp, worker->scratch, + shared->max_block_size)) { + blk->flags |= BLK_COMPRESS_ERROR; + } + } + return NULL; +} + +block_processor_t *block_processor_create(size_t max_block_size, + compressor_t *cmp, + unsigned int num_workers, + void *user, + block_cb callback) +{ + block_processor_t *proc; + unsigned int i; + int ret; + + if (num_workers < 1) + num_workers = 1; + + proc = alloc_flex(sizeof(*proc), + sizeof(proc->workers[0]), num_workers); + if (proc == NULL) { + perror("Creating block processor"); + return NULL; + } + + proc->max_block_size = max_block_size; + proc->cb = callback; + proc->user = user; + proc->num_workers = num_workers; + + if (pthread_mutex_init(&proc->mtx, NULL)) { + perror("Creating block processor mutex"); + goto fail_free; + } + + if (pthread_cond_init(&proc->queue_cond, NULL)) { + perror("Creating block processor conditional"); + goto fail_mtx; + } + + if (pthread_cond_init(&proc->done_cond, NULL)) { + perror("Creating block processor completion conditional"); + goto fail_cond; + } + + for (i = 0; i < num_workers; ++i) { + proc->workers[i] = alloc_flex(sizeof(compress_worker_t), + 1, max_block_size); + + if (proc->workers[i] == NULL) { + perror("Creating block worker data"); + goto fail_init; + } + + proc->workers[i]->shared = proc; + proc->workers[i]->cmp = cmp->create_copy(cmp); + + if (proc->workers[i]->cmp == NULL) + goto fail_init; + } + + for (i = 0; i < num_workers; ++i) { + ret = pthread_create(&proc->workers[i]->thread, NULL, + worker_proc, proc->workers[i]); + + if (ret != 0) { + perror("Creating block processor thread"); + goto fail_thread; + } + } + + return proc; +fail_thread: + pthread_mutex_lock(&proc->mtx); + proc->terminate = true; + pthread_cond_broadcast(&proc->queue_cond); + pthread_mutex_unlock(&proc->mtx); + + for (i = 0; i < num_workers; ++i) { + if (proc->workers[i]->thread > 0) { + pthread_join(proc->workers[i]->thread, NULL); + } + } +fail_init: + for (i = 0; i < num_workers; ++i) { + if (proc->workers[i] != NULL) { + if (proc->workers[i]->cmp != NULL) { + proc->workers[i]->cmp-> + destroy(proc->workers[i]->cmp); + } + + free(proc->workers[i]); + } + } + pthread_cond_destroy(&proc->done_cond); +fail_cond: + pthread_cond_destroy(&proc->queue_cond); +fail_mtx: + pthread_mutex_destroy(&proc->mtx); +fail_free: + free(proc); + return NULL; +} + +void block_processor_destroy(block_processor_t *proc) +{ + unsigned int i; + block_t *blk; + + pthread_mutex_lock(&proc->mtx); + proc->terminate = true; + pthread_cond_broadcast(&proc->queue_cond); + pthread_mutex_unlock(&proc->mtx); + + for (i = 0; i < proc->num_workers; ++i) { + pthread_join(proc->workers[i]->thread, NULL); + + proc->workers[i]->cmp->destroy(proc->workers[i]->cmp); + free(proc->workers[i]); + } + + pthread_cond_destroy(&proc->done_cond); + pthread_cond_destroy(&proc->queue_cond); + pthread_mutex_destroy(&proc->mtx); + + while (proc->queue != NULL) { + blk = proc->queue; + proc->queue = blk->next; + free(blk); + } + + while (proc->done != NULL) { + blk = proc->done; + proc->done = blk->next; + free(blk); + } + + free(proc); +} + +static int process_completed_blocks(block_processor_t *proc, block_t *queue) +{ + block_t *it; + + while (queue != NULL) { + it = queue; + queue = queue->next; + + if (it->flags & BLK_COMPRESS_ERROR) { + proc->status = -1; + } else { + if (proc->cb(proc->user, it)) + proc->status = -1; + } + + free(it); + } + + return proc->status; +} + +int block_processor_enqueue(block_processor_t *proc, block_t *block) +{ + block_t *queue = NULL, *it, *prev; + + block->sequence_number = proc->enqueue_id++; + block->next = NULL; + + pthread_mutex_lock(&proc->mtx); + if ((block->flags & BLK_DONT_COMPRESS) && + (block->flags & BLK_DONT_CHECKSUM)) { + store_completed_block(proc, block); + } else { + while (proc->backlog > proc->num_workers * MAX_BACKLOG_FACTOR) + pthread_cond_wait(&proc->done_cond, &proc->mtx); + + if (proc->queue_last == NULL) { + proc->queue = proc->queue_last = block; + } else { + proc->queue_last->next = block; + proc->queue_last = block; + } + + proc->backlog += 1; + } + + it = proc->done; + prev = NULL; + + while (it != NULL && it->sequence_number == proc->dequeue_id) { + prev = it; + it = it->next; + proc->dequeue_id += 1; + } + + if (prev != NULL) { + queue = proc->done; + prev->next = NULL; + proc->done = it; + } + + pthread_cond_broadcast(&proc->queue_cond); + pthread_mutex_unlock(&proc->mtx); + + return process_completed_blocks(proc, queue); +} + +int block_processor_finish(block_processor_t *proc) +{ + block_t *queue, *it; + + pthread_mutex_lock(&proc->mtx); + while (proc->backlog > 0) + pthread_cond_wait(&proc->done_cond, &proc->mtx); + + for (it = proc->done; it != NULL; it = it->next) { + if (it->sequence_number != proc->dequeue_id++) { + pthread_mutex_unlock(&proc->mtx); + goto bug_seqnum; + } + } + + queue = proc->done; + proc->done = NULL; + pthread_mutex_unlock(&proc->mtx); + + return process_completed_blocks(proc, queue); +bug_seqnum: + fputs("[BUG][parallel block processor] " + "gap in sequence numbers!\n", stderr); + return -1; +} diff --git a/lib/sqfs/comp/compressor.c b/lib/sqfs/comp/compressor.c new file mode 100644 index 0000000..e59d948 --- /dev/null +++ b/lib/sqfs/comp/compressor.c @@ -0,0 +1,183 @@ +/* SPDX-License-Identifier: GPL-3.0-or-later */ +/* + * compressor.c + * + * Copyright (C) 2019 David Oberhollenzer + */ +#include "config.h" + +#include +#include +#include + +#include "internal.h" +#include "util.h" + +typedef compressor_t *(*compressor_fun_t)(bool compress, size_t block_size, + char *options); + +typedef void (*compressor_help_fun_t)(void); + +static compressor_fun_t compressors[SQFS_COMP_MAX + 1] = { +#ifdef WITH_GZIP + [SQFS_COMP_GZIP] = create_gzip_compressor, +#endif +#ifdef WITH_XZ + [SQFS_COMP_XZ] = create_xz_compressor, +#endif +#ifdef WITH_LZO + [SQFS_COMP_LZO] = create_lzo_compressor, +#endif +#ifdef WITH_LZ4 + [SQFS_COMP_LZ4] = create_lz4_compressor, +#endif +#ifdef WITH_ZSTD + [SQFS_COMP_ZSTD] = create_zstd_compressor, +#endif +}; + +static const compressor_help_fun_t helpfuns[SQFS_COMP_MAX + 1] = { +#ifdef WITH_GZIP + [SQFS_COMP_GZIP] = compressor_gzip_print_help, +#endif +#ifdef WITH_XZ + [SQFS_COMP_XZ] = compressor_xz_print_help, +#endif +#ifdef WITH_LZO + [SQFS_COMP_LZO] = compressor_lzo_print_help, +#endif +#ifdef WITH_LZ4 + [SQFS_COMP_LZ4] = compressor_lz4_print_help, +#endif +#ifdef WITH_ZSTD + [SQFS_COMP_ZSTD] = compressor_zstd_print_help, +#endif +}; + +static const char *names[] = { + [SQFS_COMP_GZIP] = "gzip", + [SQFS_COMP_LZMA] = "lzma", + [SQFS_COMP_LZO] = "lzo", + [SQFS_COMP_XZ] = "xz", + [SQFS_COMP_LZ4] = "lz4", + [SQFS_COMP_ZSTD] = "zstd", +}; + +int generic_write_options(int fd, const void *data, size_t size) +{ + uint8_t buffer[size + 2]; + + *((uint16_t *)buffer) = htole16(0x8000 | size); + memcpy(buffer + 2, data, size); + + if (write_data("writing compressor options", + fd, buffer, sizeof(buffer))) { + return -1; + } + + return sizeof(buffer); +} + +int generic_read_options(int fd, void *data, size_t size) +{ + uint8_t buffer[size + 2]; + + if (read_data_at("reading compressor options", sizeof(sqfs_super_t), + fd, buffer, sizeof(buffer))) { + return -1; + } + + if (le16toh(*((uint16_t *)buffer)) != (0x8000 | size)) { + fputs("reading compressor options: invalid meta data header\n", + stderr); + return -1; + } + + memcpy(data, buffer + 2, size); + return 0; +} + +bool compressor_exists(E_SQFS_COMPRESSOR id) +{ + if (id < SQFS_COMP_MIN || id > SQFS_COMP_MAX) + return false; + + return (compressors[id] != NULL); +} + +compressor_t *compressor_create(E_SQFS_COMPRESSOR id, bool compress, + size_t block_size, char *options) +{ + if (id < SQFS_COMP_MIN || id > SQFS_COMP_MAX) + return NULL; + + if (compressors[id] == NULL) + return NULL; + + return compressors[id](compress, block_size, options); +} + +void compressor_print_help(E_SQFS_COMPRESSOR id) +{ + if (id < SQFS_COMP_MIN || id > SQFS_COMP_MAX) + return; + + if (compressors[id] == NULL) + return; + + helpfuns[id](); +} + +void compressor_print_available(void) +{ + size_t i; + + fputs("Available compressors:\n", stdout); + + for (i = 0; i < sizeof(names) / sizeof(names[0]); ++i) { + if (compressor_exists(i)) + printf("\t%s\n", names[i]); + } + + printf("\nDefault compressor: %s\n", names[compressor_get_default()]); +} + +const char *compressor_name_from_id(E_SQFS_COMPRESSOR id) +{ + if (id < 0 || (size_t)id >= sizeof(names) / sizeof(names[0])) + return NULL; + + return names[id]; +} + +int compressor_id_from_name(const char *name, E_SQFS_COMPRESSOR *out) +{ + size_t i; + + for (i = 0; i < sizeof(names) / sizeof(names[0]); ++i) { + if (names[i] != NULL && strcmp(names[i], name) == 0) { + *out = i; + return 0; + } + } + + return -1; +} + +E_SQFS_COMPRESSOR compressor_get_default(void) +{ +#if defined(WITH_XZ) + return SQFS_COMP_XZ; +#elif defined(WITH_ZSTD) + return SQFS_COMP_ZSTD; +#elif defined(WITH_GZIP) + return SQFS_COMP_GZIP; +#elif defined(WITH_LZO) + return SQFS_COMP_LZO; +#elif defined(WITH_LZ4) + return SQFS_COMP_LZ4; +#else + fputs("No compressor implementation available!\n", stderr); + exit(EXIT_FAILURE); +#endif +} diff --git a/lib/sqfs/comp/create_block.c b/lib/sqfs/comp/create_block.c new file mode 100644 index 0000000..90344bb --- /dev/null +++ b/lib/sqfs/comp/create_block.c @@ -0,0 +1,37 @@ +/* SPDX-License-Identifier: GPL-3.0-or-later */ +/* + * create_block.c + * + * Copyright (C) 2019 David Oberhollenzer + */ +#include "config.h" + +#include "block_processor.h" +#include "util.h" + +#include +#include +#include + +block_t *create_block(const char *filename, int fd, size_t size, + void *user, uint32_t flags) +{ + block_t *blk = alloc_flex(sizeof(*blk), 1, size); + + if (blk == NULL) { + perror(filename); + return NULL; + } + + if (fd >= 0) { + if (read_data(filename, fd, blk->data, size)) { + free(blk); + return NULL; + } + } + + blk->size = size; + blk->user = user; + blk->flags = flags; + return blk; +} diff --git a/lib/sqfs/comp/gzip.c b/lib/sqfs/comp/gzip.c new file mode 100644 index 0000000..1fdc051 --- /dev/null +++ b/lib/sqfs/comp/gzip.c @@ -0,0 +1,412 @@ +/* SPDX-License-Identifier: GPL-3.0-or-later */ +/* + * gzip.c + * + * Copyright (C) 2019 David Oberhollenzer + */ +#include "config.h" + +#include +#include +#include +#include +#include +#include + +#include "internal.h" + +#define GZIP_DEFAULT_LEVEL 9 +#define GZIP_DEFAULT_WINDOW 15 +#define GZIP_NUM_STRATEGIES (sizeof(strategies) / sizeof(strategies[0])) + +typedef enum { + GZIP_STRATEGY_DEFAULT = 0x01, + GZIP_STRATEGY_FILTERED = 0x02, + GZIP_STRATEGY_HUFFMAN = 0x04, + GZIP_STARTEGY_RLE = 0x08, + GZIP_STRATEGY_FIXED = 0x10, + + GZIP_ALL_STRATEGIES = 0x1F, +} GZIP_STRATEGIES; + +static const struct { + const char *name; + int flag; + int zlib; +} strategies[] = { + { "default", GZIP_STRATEGY_DEFAULT, Z_DEFAULT_STRATEGY }, + { "filtered", GZIP_STRATEGY_FILTERED, Z_FILTERED }, + { "huffman", GZIP_STRATEGY_HUFFMAN, Z_HUFFMAN_ONLY }, + { "rle", GZIP_STARTEGY_RLE, Z_RLE }, + { "fixed", GZIP_STRATEGY_FIXED, Z_FIXED }, +}; + +typedef struct { + uint32_t level; + uint16_t window; + uint16_t strategies; +} gzip_options_t; + +typedef struct { + compressor_t base; + + z_stream strm; + bool compress; + + size_t block_size; + gzip_options_t opt; +} gzip_compressor_t; + +static void gzip_destroy(compressor_t *base) +{ + gzip_compressor_t *gzip = (gzip_compressor_t *)base; + + if (gzip->compress) { + deflateEnd(&gzip->strm); + } else { + inflateEnd(&gzip->strm); + } + + free(gzip); +} + +static int gzip_write_options(compressor_t *base, int fd) +{ + gzip_compressor_t *gzip = (gzip_compressor_t *)base; + gzip_options_t opt; + + if (gzip->opt.level == GZIP_DEFAULT_LEVEL && + gzip->opt.window == GZIP_DEFAULT_WINDOW && + gzip->opt.strategies == 0) { + return 0; + } + + opt.level = htole32(gzip->opt.level); + opt.window = htole16(gzip->opt.window); + opt.strategies = htole16(gzip->opt.strategies); + + return generic_write_options(fd, &opt, sizeof(opt)); +} + +static int gzip_read_options(compressor_t *base, int fd) +{ + gzip_compressor_t *gzip = (gzip_compressor_t *)base; + gzip_options_t opt; + + if (generic_read_options(fd, &opt, sizeof(opt))) + return -1; + + gzip->opt.level = le32toh(opt.level); + gzip->opt.window = le16toh(opt.window); + gzip->opt.strategies = le16toh(opt.strategies); + + if (gzip->opt.level < 1 || gzip->opt.level > 9) { + fprintf(stderr, "Invalid gzip compression level '%d'.\n", + gzip->opt.level); + return -1; + } + + if (gzip->opt.window < 8 || gzip->opt.window > 15) { + fprintf(stderr, "Invalid gzip window size '%d'.\n", + gzip->opt.window); + return -1; + } + + if (gzip->opt.strategies & ~GZIP_ALL_STRATEGIES) { + fputs("Unknown gzip strategies selected.\n", stderr); + return -1; + } + + return 0; +} + +static int find_strategy(gzip_compressor_t *gzip, const uint8_t *in, + size_t size, uint8_t *out, size_t outsize) +{ + int ret, selected = Z_DEFAULT_STRATEGY; + size_t i, length, minlength = 0; + + for (i = 0; i < GZIP_NUM_STRATEGIES; ++i) { + if (!(strategies[i].flag & gzip->opt.strategies)) + continue; + + ret = deflateReset(&gzip->strm); + if (ret != Z_OK) { + fputs("resetting zlib stream failed\n", + stderr); + return -1; + } + + gzip->strm.next_in = (void *)in; + gzip->strm.avail_in = size; + gzip->strm.next_out = out; + gzip->strm.avail_out = outsize; + + ret = deflateParams(&gzip->strm, gzip->opt.level, + strategies[i].zlib); + if (ret != Z_OK) { + fputs("setting deflate parameters failed\n", + stderr); + return -1; + } + + ret = deflate(&gzip->strm, Z_FINISH); + + if (ret == Z_STREAM_END) { + length = gzip->strm.total_out; + + if (minlength == 0 || length < minlength) { + minlength = length; + selected = strategies[i].zlib; + } + } else if (ret != Z_OK && ret != Z_BUF_ERROR) { + fputs("gzip block processing failed\n", stderr); + return -1; + } + } + + return selected; +} + +static ssize_t gzip_do_block(compressor_t *base, const uint8_t *in, + size_t size, uint8_t *out, size_t outsize) +{ + gzip_compressor_t *gzip = (gzip_compressor_t *)base; + int ret, strategy = 0; + size_t written; + + if (gzip->compress && gzip->opt.strategies != 0) { + strategy = find_strategy(gzip, in, size, out, outsize); + if (strategy < 0) + return -1; + } + + if (gzip->compress) { + ret = deflateReset(&gzip->strm); + } else { + ret = inflateReset(&gzip->strm); + } + + if (ret != Z_OK) { + fputs("resetting zlib stream failed\n", stderr); + return -1; + } + + gzip->strm.next_in = (void *)in; + gzip->strm.avail_in = size; + gzip->strm.next_out = out; + gzip->strm.avail_out = outsize; + + if (gzip->compress && gzip->opt.strategies != 0) { + ret = deflateParams(&gzip->strm, gzip->opt.level, strategy); + if (ret != Z_OK) { + fputs("setting selcted deflate parameters failed\n", + stderr); + return -1; + } + } + + if (gzip->compress) { + ret = deflate(&gzip->strm, Z_FINISH); + } else { + ret = inflate(&gzip->strm, Z_FINISH); + } + + if (ret == Z_STREAM_END) { + written = gzip->strm.total_out; + + if (gzip->compress && written >= size) + return 0; + + return (ssize_t)written; + } + + if (ret != Z_OK && ret != Z_BUF_ERROR) { + fputs("gzip block processing failed\n", stderr); + return -1; + } + + return 0; +} + +static int process_options(char *options, int *level, int *window, int *flags) +{ + enum { + OPT_WINDOW = 0, + OPT_LEVEL, + }; + char *const token[] = { + [OPT_WINDOW] = (char *)"window", + [OPT_LEVEL] = (char *)"level", + NULL + }; + char *subopts, *value; + size_t i; + int opt; + + subopts = options; + + while (*subopts != '\0') { + opt = getsubopt(&subopts, token, &value); + + switch (opt) { + case OPT_WINDOW: + if (value == NULL) + goto fail_value; + + for (i = 0; isdigit(value[i]); ++i) + ; + + if (i < 1 || i > 3 || value[i] != '\0') + goto fail_window; + + *window = atoi(value); + + if (*window < 8 || *window > 15) + goto fail_window; + break; + case OPT_LEVEL: + if (value == NULL) + goto fail_value; + + for (i = 0; isdigit(value[i]); ++i) + ; + + if (i < 1 || i > 3 || value[i] != '\0') + goto fail_level; + + *level = atoi(value); + + if (*level < 1 || *level > 9) + goto fail_level; + break; + default: + for (i = 0; i < GZIP_NUM_STRATEGIES; ++i) { + if (strcmp(value, strategies[i].name) == 0) { + *flags |= strategies[i].flag; + break; + } + } + if (i == GZIP_NUM_STRATEGIES) + goto fail_opt; + break; + } + } + + return 0; +fail_window: + fputs("Window size must be a number between 8 and 15.\n", stderr); + return -1; +fail_level: + fputs("Compression level must be a number between 1 and 9.\n", stderr); + return -1; +fail_opt: + fprintf(stderr, "Unknown option '%s'.\n", value); + return -1; +fail_value: + fprintf(stderr, "Missing value for '%s'.\n", token[opt]); + return -1; +} + +static compressor_t *gzip_create_copy(compressor_t *cmp) +{ + gzip_compressor_t *gzip = malloc(sizeof(*gzip)); + int ret; + + if (gzip == NULL) { + perror("creating additional gzip compressor"); + return NULL; + } + + memcpy(gzip, cmp, sizeof(*gzip)); + memset(&gzip->strm, 0, sizeof(gzip->strm)); + + if (gzip->compress) { + ret = deflateInit2(&gzip->strm, gzip->opt.level, Z_DEFLATED, + gzip->opt.window, 8, Z_DEFAULT_STRATEGY); + } else { + ret = inflateInit(&gzip->strm); + } + + if (ret != Z_OK) { + fputs("internal error creating additional zlib stream\n", + stderr); + free(gzip); + return NULL; + } + + return (compressor_t *)gzip; +} + +compressor_t *create_gzip_compressor(bool compress, size_t block_size, + char *options) +{ + int window = GZIP_DEFAULT_WINDOW; + int level = GZIP_DEFAULT_LEVEL; + gzip_compressor_t *gzip; + compressor_t *base; + int flags = 0; + int ret; + + if (options != NULL) { + if (process_options(options, &level, &window, &flags)) + return NULL; + } + + gzip = calloc(1, sizeof(*gzip)); + base = (compressor_t *)gzip; + + if (gzip == NULL) { + perror("creating gzip compressor"); + return NULL; + } + + gzip->opt.level = level; + gzip->opt.window = window; + gzip->opt.strategies = flags; + gzip->compress = compress; + gzip->block_size = block_size; + base->do_block = gzip_do_block; + base->destroy = gzip_destroy; + base->write_options = gzip_write_options; + base->read_options = gzip_read_options; + base->create_copy = gzip_create_copy; + + if (compress) { + ret = deflateInit2(&gzip->strm, level, Z_DEFLATED, window, 8, + Z_DEFAULT_STRATEGY); + } else { + ret = inflateInit(&gzip->strm); + } + + if (ret != Z_OK) { + fputs("internal error creating zlib stream\n", stderr); + free(gzip); + return NULL; + } + + return base; +} + +void compressor_gzip_print_help(void) +{ + size_t i; + + printf( +"Available options for gzip compressor:\n" +"\n" +" level= Compression level. Value from 1 to 9.\n" +" Defaults to %d.\n" +" window= Deflate compression window size. Value from 8 to 15.\n" +" Defaults to %d.\n" +"\n" +"In additon to the options, one or more strategies can be specified.\n" +"If multiple stratgies are provided, the one yielding the best compression\n" +"ratio will be used.\n" +"\n" +"The following strategies are available:\n", + GZIP_DEFAULT_LEVEL, GZIP_DEFAULT_WINDOW); + + for (i = 0; i < GZIP_NUM_STRATEGIES; ++i) + printf("\t%s\n", strategies[i].name); +} diff --git a/lib/sqfs/comp/internal.h b/lib/sqfs/comp/internal.h new file mode 100644 index 0000000..986266d --- /dev/null +++ b/lib/sqfs/comp/internal.h @@ -0,0 +1,44 @@ +/* SPDX-License-Identifier: GPL-3.0-or-later */ +/* + * internal.h + * + * Copyright (C) 2019 David Oberhollenzer + */ +#ifndef INTERNAL_H +#define INTERNAL_H + +#include "config.h" + +#include "compress.h" +#include "util.h" + +int generic_write_options(int fd, const void *data, size_t size); + +int generic_read_options(int fd, void *data, size_t size); + +compressor_t *create_xz_compressor(bool compress, size_t block_size, + char *options); + +compressor_t *create_gzip_compressor(bool compress, size_t block_size, + char *options); + +compressor_t *create_lzo_compressor(bool compress, size_t block_size, + char *options); + +compressor_t *create_lz4_compressor(bool compress, size_t block_size, + char *options); + +compressor_t *create_zstd_compressor(bool compress, size_t block_size, + char *options); + +void compressor_xz_print_help(void); + +void compressor_gzip_print_help(void); + +void compressor_lzo_print_help(void); + +void compressor_lz4_print_help(void); + +void compressor_zstd_print_help(void); + +#endif /* INTERNAL_H */ diff --git a/lib/sqfs/comp/lz4.c b/lib/sqfs/comp/lz4.c new file mode 100644 index 0000000..abb6c5c --- /dev/null +++ b/lib/sqfs/comp/lz4.c @@ -0,0 +1,159 @@ +/* SPDX-License-Identifier: GPL-3.0-or-later */ +/* + * lz4.c + * + * Copyright (C) 2019 David Oberhollenzer + */ +#include "config.h" + +#include +#include +#include +#include + +#include +#include + +#include "internal.h" + +typedef struct { + compressor_t base; + bool high_compression; +} lz4_compressor_t; + +typedef struct { + uint32_t version; + uint32_t flags; +} lz4_options; + +#define LZ4LEGACY 1 +#define LZ4_FLAG_HC 0x01 + +static int lz4_write_options(compressor_t *base, int fd) +{ + lz4_compressor_t *lz4 = (lz4_compressor_t *)base; + lz4_options opt = { + .version = htole32(LZ4LEGACY), + .flags = htole32(lz4->high_compression ? LZ4_FLAG_HC : 0), + }; + + return generic_write_options(fd, &opt, sizeof(opt)); +} + +static int lz4_read_options(compressor_t *base, int fd) +{ + lz4_options opt; + (void)base; + + if (generic_read_options(fd, &opt, sizeof(opt))) + return -1; + + opt.version = le32toh(opt.version); + opt.flags = le32toh(opt.flags); + + if (opt.version != LZ4LEGACY) { + fprintf(stderr, "unsupported lz4 version '%d'\n", opt.version); + return -1; + } + + return 0; +} + +static ssize_t lz4_comp_block(compressor_t *base, const uint8_t *in, + size_t size, uint8_t *out, size_t outsize) +{ + lz4_compressor_t *lz4 = (lz4_compressor_t *)base; + int ret; + + if (lz4->high_compression) { + ret = LZ4_compress_HC((void *)in, (void *)out, + size, outsize, LZ4HC_CLEVEL_MAX); + } else { + ret = LZ4_compress_default((void *)in, (void *)out, + size, outsize); + } + + if (ret < 0) { + fputs("internal error in lz4 compressor\n", stderr); + return -1; + } + + return ret; +} + +static ssize_t lz4_uncomp_block(compressor_t *base, const uint8_t *in, + size_t size, uint8_t *out, size_t outsize) +{ + int ret; + (void)base; + + ret = LZ4_decompress_safe((void *)in, (void *)out, size, outsize); + + if (ret < 0) { + fputs("internal error in lz4 decompressor\n", stderr); + return -1; + } + + return ret; +} + +static compressor_t *lz4_create_copy(compressor_t *cmp) +{ + lz4_compressor_t *lz4 = malloc(sizeof(*lz4)); + + if (lz4 == NULL) { + perror("creating additional lz4 compressor"); + return NULL; + } + + memcpy(lz4, cmp, sizeof(*lz4)); + return (compressor_t *)lz4; +} + +static void lz4_destroy(compressor_t *base) +{ + free(base); +} + +compressor_t *create_lz4_compressor(bool compress, size_t block_size, + char *options) +{ + lz4_compressor_t *lz4 = calloc(1, sizeof(*lz4)); + compressor_t *base = (compressor_t *)lz4; + (void)block_size; + + if (lz4 == NULL) { + perror("creating lz4 compressor"); + return NULL; + } + + lz4->high_compression = false; + + if (options != NULL) { + if (strcmp(options, "hc") == 0) { + lz4->high_compression = true; + } else { + fputs("Unsupported extra options for lz4 " + "compressor.\n", stderr); + free(lz4); + return NULL; + } + } + + base->destroy = lz4_destroy; + base->do_block = compress ? lz4_comp_block : lz4_uncomp_block; + base->write_options = lz4_write_options; + base->read_options = lz4_read_options; + base->create_copy = lz4_create_copy; + return base; +} + +void compressor_lz4_print_help(void) +{ + fputs("Available options for lz4 compressor:\n" + "\n" + " hc If present, use slower but better compressing\n" + " variant of lz4.\n" + "\n", + stdout); +} diff --git a/lib/sqfs/comp/lzo.c b/lib/sqfs/comp/lzo.c new file mode 100644 index 0000000..09ef75c --- /dev/null +++ b/lib/sqfs/comp/lzo.c @@ -0,0 +1,321 @@ +/* SPDX-License-Identifier: GPL-3.0-or-later */ +/* + * lzo.c + * + * Copyright (C) 2019 David Oberhollenzer + */ +#include "config.h" + +#include +#include +#include +#include +#include + +#include + +#include "internal.h" + + +typedef enum { + LZO_ALGORITHM_LZO1X_1 = 0, + LZO_ALGORITHM_LZO1X_1_11 = 1, + LZO_ALGORITHM_LZO1X_1_12 = 2, + LZO_ALGORITHM_LZO1X_1_15 = 3, + LZO_ALGORITHM_LZO1X_999 = 4, +} LZO_ALGORITHM; + +#define LZO_DEFAULT_ALG LZO_ALGORITHM_LZO1X_999 +#define LZO_DEFAULT_LEVEL 8 +#define LZO_NUM_ALGS (sizeof(lzo_algs) / sizeof(lzo_algs[0])) + +typedef int (*lzo_cb_t)(const lzo_bytep src, lzo_uint src_len, lzo_bytep dst, + lzo_uintp dst_len, lzo_voidp wrkmem); + +static const struct { + const char *name; + lzo_cb_t compress; + size_t bufsize; +} lzo_algs[] = { + [LZO_ALGORITHM_LZO1X_1] = { + .name = "lzo1x_1", + .compress = lzo1x_1_compress, + .bufsize = LZO1X_1_MEM_COMPRESS, + }, + [LZO_ALGORITHM_LZO1X_1_11] = { + .name = "lzo1x_1_11", + .compress = lzo1x_1_11_compress, + .bufsize = LZO1X_1_11_MEM_COMPRESS, + }, + [LZO_ALGORITHM_LZO1X_1_12] = { + .name = "lzo1x_1_12", + .compress = lzo1x_1_12_compress, + .bufsize = LZO1X_1_12_MEM_COMPRESS, + }, + [LZO_ALGORITHM_LZO1X_1_15] = { + .name = "lzo1x_1_15", + .compress = lzo1x_1_15_compress, + .bufsize = LZO1X_1_15_MEM_COMPRESS, + }, + [LZO_ALGORITHM_LZO1X_999] = { + .name = "lzo1x_999", + .compress = lzo1x_999_compress, + .bufsize = LZO1X_999_MEM_COMPRESS, + }, +}; + +typedef struct { + compressor_t base; + int algorithm; + int level; + + uint8_t buffer[]; +} lzo_compressor_t; + +typedef struct { + uint32_t algorithm; + uint32_t level; +} lzo_options_t; + +static int lzo_write_options(compressor_t *base, int fd) +{ + lzo_compressor_t *lzo = (lzo_compressor_t *)base; + lzo_options_t opt; + + if (lzo->algorithm == LZO_DEFAULT_ALG && + lzo->level == LZO_DEFAULT_LEVEL) { + return 0; + } + + opt.algorithm = htole32(lzo->algorithm); + + if (lzo->algorithm == LZO_ALGORITHM_LZO1X_999) { + opt.level = htole32(lzo->level); + } else { + opt.level = 0; + } + + return generic_write_options(fd, &opt, sizeof(opt)); +} + +static int lzo_read_options(compressor_t *base, int fd) +{ + lzo_compressor_t *lzo = (lzo_compressor_t *)base; + lzo_options_t opt; + + if (generic_read_options(fd, &opt, sizeof(opt))) + return -1; + + lzo->algorithm = le32toh(opt.algorithm); + lzo->level = le32toh(opt.level); + + switch(lzo->algorithm) { + case LZO_ALGORITHM_LZO1X_1: + case LZO_ALGORITHM_LZO1X_1_11: + case LZO_ALGORITHM_LZO1X_1_12: + case LZO_ALGORITHM_LZO1X_1_15: + if (lzo->level != 0) + goto fail_level; + break; + case LZO_ALGORITHM_LZO1X_999: + if (lzo->level < 1 || lzo->level > 9) + goto fail_level; + break; + default: + fputs("Unsupported LZO variant specified.\n", stderr); + return -1; + } + + return 0; +fail_level: + fputs("Unsupported LZO compression level specified.\n", stderr); + return -1; +} + +static ssize_t lzo_comp_block(compressor_t *base, const uint8_t *in, + size_t size, uint8_t *out, size_t outsize) +{ + lzo_compressor_t *lzo = (lzo_compressor_t *)base; + lzo_uint len = outsize; + int ret; + + if (lzo->algorithm == LZO_ALGORITHM_LZO1X_999 && + lzo->level != LZO_DEFAULT_LEVEL) { + ret = lzo1x_999_compress_level(in, size, out, &len, + lzo->buffer, NULL, 0, 0, + lzo->level); + } else { + ret = lzo_algs[lzo->algorithm].compress(in, size, out, + &len, lzo->buffer); + } + + if (ret != LZO_E_OK) { + fputs("LZO compression failed.\n", stderr); + return -1; + } + + if (len < size) + return len; + + return 0; +} + +static ssize_t lzo_uncomp_block(compressor_t *base, const uint8_t *in, + size_t size, uint8_t *out, size_t outsize) +{ + lzo_compressor_t *lzo = (lzo_compressor_t *)base; + lzo_uint len = outsize; + int ret; + + ret = lzo1x_decompress_safe(in, size, out, &len, lzo->buffer); + + if (ret != LZO_E_OK) { + fputs("lzo decompress: input data is corrupted\n", stderr); + return -1; + } + + return len; +} + +static compressor_t *lzo_create_copy(compressor_t *cmp) +{ + lzo_compressor_t *other = (lzo_compressor_t *)cmp; + lzo_compressor_t *lzo; + + lzo = alloc_flex(sizeof(*lzo), 1, lzo_algs[other->algorithm].bufsize); + + if (lzo == NULL) { + perror("creating additional lzo compressor"); + return NULL; + } + + memcpy(lzo, other, sizeof(*lzo)); + return (compressor_t *)lzo; +} + +static void lzo_destroy(compressor_t *base) +{ + free(base); +} + +static int process_options(char *options, int *algorithm, int *level) +{ + enum { + OPT_ALG = 0, + OPT_LEVEL, + }; + char *const token[] = { + [OPT_ALG] = (char *)"algorithm", + [OPT_LEVEL] = (char *)"level", + NULL + }; + char *subopts, *value; + size_t i; + int opt; + + subopts = options; + + while (*subopts != '\0') { + opt = getsubopt(&subopts, token, &value); + + switch (opt) { + case OPT_ALG: + if (value == NULL) + goto fail_value; + + for (i = 0; i < LZO_NUM_ALGS; ++i) { + if (strcmp(lzo_algs[i].name, value) == 0) { + *algorithm = i; + break; + } + } + + if (i == LZO_NUM_ALGS) { + fprintf(stderr, "Unknown lzo variant '%s'.\n", + value); + return -1; + } + break; + case OPT_LEVEL: + if (value == NULL) + goto fail_value; + + for (i = 0; isdigit(value[i]); ++i) + ; + + if (i < 1 || i > 3 || value[i] != '\0') + goto fail_level; + + *level = atoi(value); + + if (*level < 1 || *level > 9) + goto fail_level; + break; + default: + goto fail_opt; + } + } + + return 0; +fail_level: + fputs("Compression level must be a number between 1 and 9.\n", stderr); + return -1; +fail_opt: + fprintf(stderr, "Unknown option '%s'.\n", value); + return -1; +fail_value: + fprintf(stderr, "Missing value for '%s'.\n", token[opt]); + return -1; +} + +compressor_t *create_lzo_compressor(bool compress, size_t block_size, + char *options) +{ + lzo_compressor_t *lzo; + compressor_t *base; + int level, alg; + (void)block_size; + + alg = LZO_DEFAULT_ALG; + level = LZO_DEFAULT_LEVEL; + + if (options != NULL && process_options(options, &alg, &level) != 0) + return NULL; + + lzo = alloc_flex(sizeof(*lzo), 1, lzo_algs[alg].bufsize); + base = (compressor_t *)lzo; + + if (lzo == NULL) { + perror("creating lzo compressor"); + return NULL; + } + + lzo->algorithm = alg; + lzo->level = level; + + base->destroy = lzo_destroy; + base->do_block = compress ? lzo_comp_block : lzo_uncomp_block; + base->write_options = lzo_write_options; + base->read_options = lzo_read_options; + base->create_copy = lzo_create_copy; + return base; +} + +void compressor_lzo_print_help(void) +{ + size_t i; + + fputs("Available options for lzo compressor:\n" + "\n" + " algorithm= Specify the variant of lzo to use.\n" + " Defaults to 'lzo1x_999'.\n" + " level= For lzo1x_999, the compression level.\n" + " Value from 1 to 9. Defaults to 8.\n" + " Ignored if algorithm is not lzo1x_999.\n" + "\n" + "Available algorithms:\n", + stdout); + + for (i = 0; i < LZO_NUM_ALGS; ++i) + printf("\t%s\n", lzo_algs[i].name); +} diff --git a/lib/sqfs/comp/process_block.c b/lib/sqfs/comp/process_block.c new file mode 100644 index 0000000..0fcbae0 --- /dev/null +++ b/lib/sqfs/comp/process_block.c @@ -0,0 +1,38 @@ +/* SPDX-License-Identifier: GPL-3.0-or-later */ +/* + * process_block.c + * + * Copyright (C) 2019 David Oberhollenzer + */ +#include "config.h" + +#include "block_processor.h" +#include "util.h" + +#include +#include + +int process_block(block_t *block, compressor_t *cmp, + uint8_t *scratch, size_t scratch_size) +{ + ssize_t ret; + + if (!(block->flags & BLK_DONT_CHECKSUM)) + block->checksum = crc32(0, block->data, block->size); + + if (!(block->flags & BLK_DONT_COMPRESS)) { + ret = cmp->do_block(cmp, block->data, block->size, + scratch, scratch_size); + + if (ret < 0) + return -1; + + if (ret > 0) { + memcpy(block->data, scratch, ret); + block->size = ret; + block->flags |= BLK_IS_COMPRESSED; + } + } + + return 0; +} diff --git a/lib/sqfs/comp/xz.c b/lib/sqfs/comp/xz.c new file mode 100644 index 0000000..d38aab6 --- /dev/null +++ b/lib/sqfs/comp/xz.c @@ -0,0 +1,360 @@ +/* SPDX-License-Identifier: GPL-3.0-or-later */ +/* + * xz.c + * + * Copyright (C) 2019 David Oberhollenzer + */ +#include "config.h" + +#include +#include +#include +#include +#include +#include + +#include "internal.h" + +typedef enum { + XZ_FILTER_X86 = 0x01, + XZ_FILTER_POWERPC = 0x02, + XZ_FILTER_IA64 = 0x04, + XZ_FILTER_ARM = 0x08, + XZ_FILTER_ARMTHUMB = 0x10, + XZ_FILTER_SPARC = 0x20, + + XZ_FILTER_ALL = 0x3F, +} XZ_FILTER_FLAG; + +typedef struct { + compressor_t base; + size_t block_size; + size_t dict_size; + int flags; +} xz_compressor_t; + +typedef struct { + uint32_t dict_size; + uint32_t flags; +} xz_options_t; + +static const struct { + const char *name; + lzma_vli filter; + int flag; +} xz_filters[] = { + { "x86", LZMA_FILTER_X86, XZ_FILTER_X86 }, + { "powerpc", LZMA_FILTER_POWERPC, XZ_FILTER_POWERPC }, + { "ia64", LZMA_FILTER_IA64, XZ_FILTER_IA64 }, + { "arm", LZMA_FILTER_ARM, XZ_FILTER_ARM }, + { "armthumb", LZMA_FILTER_ARMTHUMB, XZ_FILTER_ARMTHUMB }, + { "sparc", LZMA_FILTER_SPARC, XZ_FILTER_SPARC }, +}; + +#define XZ_NUM_FILTERS (sizeof(xz_filters) / sizeof(xz_filters[0])) + +static int xz_write_options(compressor_t *base, int fd) +{ + xz_compressor_t *xz = (xz_compressor_t *)base; + xz_options_t opt; + + if (xz->flags == 0 && xz->dict_size == xz->block_size) + return 0; + + opt.dict_size = htole32(xz->dict_size); + opt.flags = htole32(xz->flags); + + return generic_write_options(fd, &opt, sizeof(opt)); +} + +static int xz_read_options(compressor_t *base, int fd) +{ + xz_compressor_t *xz = (xz_compressor_t *)base; + xz_options_t opt; + uint32_t mask; + + if (generic_read_options(fd, &opt, sizeof(opt))) + return -1; + + opt.dict_size = le32toh(opt.dict_size); + opt.flags = le32toh(opt.flags); + + mask = opt.dict_size & (opt.dict_size - 1); + + if (mask != 0 && ((mask & (mask - 1)) != 0)) { + fputs("Invalid lzma dictionary size.\n", stderr); + return -1; + } + + if (opt.flags & ~XZ_FILTER_ALL) { + fputs("Unknown BCJ filter used.\n", stderr); + return -1; + } + + xz->flags = opt.flags; + xz->dict_size = opt.dict_size; + return 0; +} + +static ssize_t compress(xz_compressor_t *xz, lzma_vli filter, + const uint8_t *in, size_t size, + uint8_t *out, size_t outsize) +{ + lzma_filter filters[5]; + lzma_options_lzma opt; + size_t written = 0; + lzma_ret ret; + int i = 0; + + if (lzma_lzma_preset(&opt, LZMA_PRESET_DEFAULT)) { + fputs("error initializing xz options\n", stderr); + return -1; + } + + opt.dict_size = xz->dict_size; + + if (filter != LZMA_VLI_UNKNOWN) { + filters[i].id = filter; + filters[i].options = NULL; + ++i; + } + + filters[i].id = LZMA_FILTER_LZMA2; + filters[i].options = &opt; + ++i; + + filters[i].id = LZMA_VLI_UNKNOWN; + filters[i].options = NULL; + ++i; + + ret = lzma_stream_buffer_encode(filters, LZMA_CHECK_CRC32, NULL, + in, size, out, &written, outsize); + + if (ret == LZMA_OK) + return (written >= size) ? 0 : written; + + if (ret != LZMA_BUF_ERROR) { + fputs("xz block compress failed\n", stderr); + return -1; + } + + return 0; +} + +static ssize_t xz_comp_block(compressor_t *base, const uint8_t *in, + size_t size, uint8_t *out, size_t outsize) +{ + xz_compressor_t *xz = (xz_compressor_t *)base; + lzma_vli selected = LZMA_VLI_UNKNOWN; + size_t i, smallest; + ssize_t ret; + + ret = compress(xz, LZMA_VLI_UNKNOWN, in, size, out, outsize); + if (ret < 0 || xz->flags == 0) + return ret; + + smallest = ret; + + for (i = 0; i < XZ_NUM_FILTERS; ++i) { + if (!(xz->flags & xz_filters[i].flag)) + continue; + + ret = compress(xz, xz_filters[i].filter, in, size, out, outsize); + if (ret < 0) + return -1; + + if (ret > 0 && (smallest == 0 || (size_t)ret < smallest)) { + smallest = ret; + selected = xz_filters[i].filter; + } + } + + if (smallest == 0) + return 0; + + return compress(xz, selected, in, size, out, outsize); +} + +static ssize_t xz_uncomp_block(compressor_t *base, const uint8_t *in, + size_t size, uint8_t *out, size_t outsize) +{ + uint64_t memlimit = 32 * 1024 * 1024; + size_t dest_pos = 0; + size_t src_pos = 0; + lzma_ret ret; + (void)base; + + ret = lzma_stream_buffer_decode(&memlimit, 0, NULL, + in, &src_pos, size, + out, &dest_pos, outsize); + + if (ret == LZMA_OK && size == src_pos) + return (ssize_t)dest_pos; + + fputs("xz block extract failed\n", stderr); + return -1; +} + +static compressor_t *xz_create_copy(compressor_t *cmp) +{ + xz_compressor_t *xz = malloc(sizeof(*xz)); + + if (xz == NULL) { + perror("creating additional xz compressor"); + return NULL; + } + + memcpy(xz, cmp, sizeof(*xz)); + return (compressor_t *)xz; +} + +static void xz_destroy(compressor_t *base) +{ + free(base); +} + +static int process_options(char *options, size_t blocksize, + int *flags, uint64_t *dictsize) +{ + enum { + OPT_DICT = 0, + }; + char *const token[] = { + [OPT_DICT] = (char *)"dictsize", + NULL + }; + char *subopts, *value; + uint64_t mask; + size_t i; + int opt; + + subopts = options; + + while (*subopts != '\0') { + opt = getsubopt(&subopts, token, &value); + + switch (opt) { + case OPT_DICT: + if (value == NULL) + goto fail_value; + + for (i = 0; isdigit(value[i]); ++i) + ; + + if (i < 1 || i > 9) + goto fail_dict; + + *dictsize = atol(value); + + switch (value[i]) { + case '\0': + break; + case 'm': + case 'M': + *dictsize <<= 20; + break; + case 'k': + case 'K': + *dictsize <<= 10; + break; + case '%': + *dictsize = ((*dictsize) * blocksize) / 100; + break; + default: + goto fail_dict; + } + + if (*dictsize > 0x0FFFFFFFFUL) + goto fail_dict_ov; + + mask = *dictsize & (*dictsize - 1); + + if (mask != 0 && ((mask & (mask - 1)) != 0)) + goto fail_dict_pot; + break; + default: + for (i = 0; i < XZ_NUM_FILTERS; ++i) { + if (strcmp(value, xz_filters[i].name) == 0) { + *flags |= xz_filters[i].flag; + break; + } + } + if (i == XZ_NUM_FILTERS) + goto fail_opt; + break; + } + } + + return 0; +fail_dict_pot: + fputs("dictionary size must be either 2^n or 2^n + 2^(n-1)\n", stderr); + return -1; +fail_dict_ov: + fputs("dictionary size too large.\n", stderr); + return -1; +fail_dict: + fputs("dictionary size must be a number with the optional " + "suffix 'm','k' or '%'.\n", stderr); + return -1; +fail_opt: + fprintf(stderr, "Unknown option '%s'.\n", value); + return -1; +fail_value: + fprintf(stderr, "Missing value for '%s'.\n", token[opt]); + return -1; +} + +compressor_t *create_xz_compressor(bool compress, size_t block_size, + char *options) +{ + uint64_t dictsize = block_size; + xz_compressor_t *xz; + compressor_t *base; + int flags = 0; + + if (options != NULL) { + if (process_options(options, block_size, &flags, &dictsize)) + return NULL; + } + + xz = calloc(1, sizeof(*xz)); + base = (compressor_t *)xz; + if (xz == NULL) { + perror("creating xz compressor"); + return NULL; + } + + xz->flags = flags; + xz->dict_size = dictsize; + xz->block_size = block_size; + base->destroy = xz_destroy; + base->do_block = compress ? xz_comp_block : xz_uncomp_block; + base->write_options = xz_write_options; + base->read_options = xz_read_options; + base->create_copy = xz_create_copy; + return base; +} + +void compressor_xz_print_help(void) +{ + size_t i; + + fputs( +"Available options for xz compressor:\n" +"\n" +" dictsize= Dictionary size. Either a value in bytes or a\n" +" percentage of the block size. Defaults to 100%.\n" +" The suffix '%' indicates a percentage. 'K' and 'M'\n" +" can also be used for kibi and mebi bytes\n" +" respecitively.\n" +"\n" +"In additon to the options, one or more bcj filters can be specified.\n" +"If multiple filters are provided, the one yielding the best compression\n" +"ratio will be used.\n" +"\n" +"The following filters are available:\n", + stdout); + + for (i = 0; i < XZ_NUM_FILTERS; ++i) + printf("\t%s\n", xz_filters[i].name); +} diff --git a/lib/sqfs/comp/zstd.c b/lib/sqfs/comp/zstd.c new file mode 100644 index 0000000..e206338 --- /dev/null +++ b/lib/sqfs/comp/zstd.c @@ -0,0 +1,188 @@ +/* SPDX-License-Identifier: GPL-3.0-or-later */ +/* + * zstd.c + * + * Copyright (C) 2019 David Oberhollenzer + */ +#include "config.h" + +#include +#include +#include +#include +#include + +#include + +#include "internal.h" + +#define ZSTD_DEFAULT_COMPRESSION_LEVEL 15 + +typedef struct { + compressor_t base; + ZSTD_CCtx *zctx; + int level; +} zstd_compressor_t; + +typedef struct { + uint32_t level; +} zstd_options_t; + +static int zstd_write_options(compressor_t *base, int fd) +{ + zstd_compressor_t *zstd = (zstd_compressor_t *)base; + zstd_options_t opt; + (void)fd; + + if (zstd->level == ZSTD_DEFAULT_COMPRESSION_LEVEL) + return 0; + + opt.level = htole32(zstd->level); + return generic_write_options(fd, &opt, sizeof(opt)); +} + +static int zstd_read_options(compressor_t *base, int fd) +{ + zstd_options_t opt; + (void)base; + + if (generic_read_options(fd, &opt, sizeof(opt))) + return -1; + + opt.level = le32toh(opt.level); + return 0; +} + +static ssize_t zstd_comp_block(compressor_t *base, const uint8_t *in, + size_t size, uint8_t *out, size_t outsize) +{ + zstd_compressor_t *zstd = (zstd_compressor_t *)base; + size_t ret; + + ret = ZSTD_compressCCtx(zstd->zctx, out, outsize, in, size, + zstd->level); + + if (ZSTD_isError(ret)) { + fprintf(stderr, "internal error in ZSTD compressor: %s\n", + ZSTD_getErrorName(ret)); + return -1; + } + + return ret < size ? ret : 0; +} + +static ssize_t zstd_uncomp_block(compressor_t *base, const uint8_t *in, + size_t size, uint8_t *out, size_t outsize) +{ + size_t ret; + (void)base; + + ret = ZSTD_decompress(out, outsize, in, size); + + if (ZSTD_isError(ret)) { + fprintf(stderr, "error uncompressing ZSTD compressed data: %s", + ZSTD_getErrorName(ret)); + return -1; + } + + return ret; +} + +static compressor_t *zstd_create_copy(compressor_t *cmp) +{ + zstd_compressor_t *zstd = malloc(sizeof(*zstd)); + + if (zstd == NULL) { + perror("creating additional zstd compressor"); + return NULL; + } + + memcpy(zstd, cmp, sizeof(*zstd)); + + zstd->zctx = ZSTD_createCCtx(); + + if (zstd->zctx == NULL) { + fputs("error creating addtional zstd compression context\n", + stderr); + free(zstd); + return NULL; + } + + return (compressor_t *)zstd; +} + +static void zstd_destroy(compressor_t *base) +{ + zstd_compressor_t *zstd = (zstd_compressor_t *)base; + + ZSTD_freeCCtx(zstd->zctx); + free(zstd); +} + +compressor_t *create_zstd_compressor(bool compress, size_t block_size, + char *options) +{ + zstd_compressor_t *zstd = calloc(1, sizeof(*zstd)); + compressor_t *base = (compressor_t *)zstd; + size_t i; + (void)block_size; + + if (zstd == NULL) { + perror("creating zstd compressor"); + return NULL; + } + + zstd->level = ZSTD_DEFAULT_COMPRESSION_LEVEL; + + if (options != NULL) { + if (strncmp(options, "level=", 6) == 0) { + options += 6; + + for (i = 0; isdigit(options[i]); ++i) + ; + + if (i == 0 || options[i] != '\0' || i > 6) + goto fail_level; + + zstd->level = atoi(options); + + if (zstd->level < 1 || zstd->level > ZSTD_maxCLevel()) + goto fail_level; + } else { + goto fail_opt; + } + } + + zstd->zctx = ZSTD_createCCtx(); + if (zstd->zctx == NULL) { + fputs("error creating zstd compression context\n", stderr); + free(zstd); + return NULL; + } + + base->destroy = zstd_destroy; + base->do_block = compress ? zstd_comp_block : zstd_uncomp_block; + base->write_options = zstd_write_options; + base->read_options = zstd_read_options; + base->create_copy = zstd_create_copy; + return base; +fail_level: + fprintf(stderr, "zstd compression level must be a number in the range " + "1...%d\n", ZSTD_maxCLevel()); + free(zstd); + return NULL; +fail_opt: + fputs("Unsupported extra options for zstd compressor\n", stderr); + free(zstd); + return NULL; +} + +void compressor_zstd_print_help(void) +{ + printf("Available options for zstd compressor:\n" + "\n" + " level= Set compression level. Defaults to %d.\n" + " Maximum is %d.\n" + "\n", + ZSTD_DEFAULT_COMPRESSION_LEVEL, ZSTD_maxCLevel()); +} -- cgit v1.2.3