summaryrefslogtreecommitdiff
path: root/lib/sqfs/comp
diff options
context:
space:
mode:
Diffstat (limited to 'lib/sqfs/comp')
-rw-r--r--lib/sqfs/comp/block_processor.c72
-rw-r--r--lib/sqfs/comp/block_processor_parallel.c340
-rw-r--r--lib/sqfs/comp/compressor.c183
-rw-r--r--lib/sqfs/comp/create_block.c37
-rw-r--r--lib/sqfs/comp/gzip.c412
-rw-r--r--lib/sqfs/comp/internal.h44
-rw-r--r--lib/sqfs/comp/lz4.c159
-rw-r--r--lib/sqfs/comp/lzo.c321
-rw-r--r--lib/sqfs/comp/process_block.c38
-rw-r--r--lib/sqfs/comp/xz.c360
-rw-r--r--lib/sqfs/comp/zstd.c188
11 files changed, 2154 insertions, 0 deletions
diff --git a/lib/sqfs/comp/block_processor.c b/lib/sqfs/comp/block_processor.c
new file mode 100644
index 0000000..06dc384
--- /dev/null
+++ b/lib/sqfs/comp/block_processor.c
@@ -0,0 +1,72 @@
+/* SPDX-License-Identifier: GPL-3.0-or-later */
+/*
+ * block_processor.c
+ *
+ * Copyright (C) 2019 David Oberhollenzer <goliath@infraroot.at>
+ */
+#include "config.h"
+
+#include "block_processor.h"
+#include "util.h"
+
+#include <string.h>
+#include <stdlib.h>
+#include <stdio.h>
+
+struct block_processor_t {
+ size_t max_block_size;
+ compressor_t *cmp;
+ block_cb cb;
+ void *user;
+ int status;
+
+ uint8_t scratch[];
+};
+
+block_processor_t *block_processor_create(size_t max_block_size,
+ compressor_t *cmp,
+ unsigned int num_workers,
+ void *user,
+ block_cb callback)
+{
+ block_processor_t *proc = alloc_flex(sizeof(*proc), 1, max_block_size);
+ (void)num_workers;
+
+ if (proc == NULL) {
+ perror("Creating block processor");
+ return NULL;
+ }
+
+ proc->max_block_size = max_block_size;
+ proc->cmp = cmp;
+ proc->cb = callback;
+ proc->user = user;
+ return proc;
+}
+
+void block_processor_destroy(block_processor_t *proc)
+{
+ free(proc);
+}
+
+int block_processor_enqueue(block_processor_t *proc, block_t *block)
+{
+ if (process_block(block, proc->cmp,
+ proc->scratch, proc->max_block_size))
+ goto fail;
+
+ if (proc->cb(proc->user, block))
+ goto fail;
+
+ free(block);
+ return 0;
+fail:
+ free(block);
+ proc->status = -1;
+ return -1;
+}
+
+int block_processor_finish(block_processor_t *proc)
+{
+ return proc->status;
+}
diff --git a/lib/sqfs/comp/block_processor_parallel.c b/lib/sqfs/comp/block_processor_parallel.c
new file mode 100644
index 0000000..b58ad5c
--- /dev/null
+++ b/lib/sqfs/comp/block_processor_parallel.c
@@ -0,0 +1,340 @@
+/* SPDX-License-Identifier: GPL-3.0-or-later */
+/*
+ * block_processor.c
+ *
+ * Copyright (C) 2019 David Oberhollenzer <goliath@infraroot.at>
+ */
+#include "config.h"
+
+#include "block_processor.h"
+#include "util.h"
+
+#include <pthread.h>
+#include <string.h>
+#include <stdlib.h>
+#include <stdio.h>
+
+#define MAX_BACKLOG_FACTOR (10)
+
+typedef struct {
+ block_processor_t *shared;
+ compressor_t *cmp;
+ pthread_t thread;
+ uint8_t scratch[];
+} compress_worker_t;
+
+struct block_processor_t {
+ pthread_mutex_t mtx;
+ pthread_cond_t queue_cond;
+ pthread_cond_t done_cond;
+
+ /* needs rw access by worker and main thread */
+ block_t *queue;
+ block_t *queue_last;
+
+ block_t *done;
+ bool terminate;
+ size_t backlog;
+
+ /* used by main thread only */
+ uint32_t enqueue_id;
+ uint32_t dequeue_id;
+
+ unsigned int num_workers;
+ block_cb cb;
+ void *user;
+ int status;
+
+ /* used only by workers */
+ size_t max_block_size;
+
+ compress_worker_t *workers[];
+};
+
+static void store_completed_block(block_processor_t *shared, block_t *blk)
+{
+ block_t *it = shared->done, *prev = NULL;
+
+ while (it != NULL) {
+ if (it->sequence_number >= blk->sequence_number)
+ break;
+ prev = it;
+ it = it->next;
+ }
+
+ if (prev == NULL) {
+ blk->next = shared->done;
+ shared->done = blk;
+ } else {
+ blk->next = prev->next;
+ prev->next = blk;
+ }
+}
+
+static void *worker_proc(void *arg)
+{
+ compress_worker_t *worker = arg;
+ block_processor_t *shared = worker->shared;
+ block_t *blk = NULL;
+
+ for (;;) {
+ pthread_mutex_lock(&shared->mtx);
+ if (blk != NULL) {
+ store_completed_block(shared, blk);
+ shared->backlog -= 1;
+ pthread_cond_broadcast(&shared->done_cond);
+ }
+
+ while (shared->queue == NULL && !shared->terminate) {
+ pthread_cond_wait(&shared->queue_cond,
+ &shared->mtx);
+ }
+
+ if (shared->terminate) {
+ pthread_mutex_unlock(&shared->mtx);
+ break;
+ }
+
+ blk = shared->queue;
+ shared->queue = blk->next;
+ blk->next = NULL;
+
+ if (shared->queue == NULL)
+ shared->queue_last = NULL;
+ pthread_mutex_unlock(&shared->mtx);
+
+ if (process_block(blk, worker->cmp, worker->scratch,
+ shared->max_block_size)) {
+ blk->flags |= BLK_COMPRESS_ERROR;
+ }
+ }
+ return NULL;
+}
+
+block_processor_t *block_processor_create(size_t max_block_size,
+ compressor_t *cmp,
+ unsigned int num_workers,
+ void *user,
+ block_cb callback)
+{
+ block_processor_t *proc;
+ unsigned int i;
+ int ret;
+
+ if (num_workers < 1)
+ num_workers = 1;
+
+ proc = alloc_flex(sizeof(*proc),
+ sizeof(proc->workers[0]), num_workers);
+ if (proc == NULL) {
+ perror("Creating block processor");
+ return NULL;
+ }
+
+ proc->max_block_size = max_block_size;
+ proc->cb = callback;
+ proc->user = user;
+ proc->num_workers = num_workers;
+
+ if (pthread_mutex_init(&proc->mtx, NULL)) {
+ perror("Creating block processor mutex");
+ goto fail_free;
+ }
+
+ if (pthread_cond_init(&proc->queue_cond, NULL)) {
+ perror("Creating block processor conditional");
+ goto fail_mtx;
+ }
+
+ if (pthread_cond_init(&proc->done_cond, NULL)) {
+ perror("Creating block processor completion conditional");
+ goto fail_cond;
+ }
+
+ for (i = 0; i < num_workers; ++i) {
+ proc->workers[i] = alloc_flex(sizeof(compress_worker_t),
+ 1, max_block_size);
+
+ if (proc->workers[i] == NULL) {
+ perror("Creating block worker data");
+ goto fail_init;
+ }
+
+ proc->workers[i]->shared = proc;
+ proc->workers[i]->cmp = cmp->create_copy(cmp);
+
+ if (proc->workers[i]->cmp == NULL)
+ goto fail_init;
+ }
+
+ for (i = 0; i < num_workers; ++i) {
+ ret = pthread_create(&proc->workers[i]->thread, NULL,
+ worker_proc, proc->workers[i]);
+
+ if (ret != 0) {
+ perror("Creating block processor thread");
+ goto fail_thread;
+ }
+ }
+
+ return proc;
+fail_thread:
+ pthread_mutex_lock(&proc->mtx);
+ proc->terminate = true;
+ pthread_cond_broadcast(&proc->queue_cond);
+ pthread_mutex_unlock(&proc->mtx);
+
+ for (i = 0; i < num_workers; ++i) {
+ if (proc->workers[i]->thread > 0) {
+ pthread_join(proc->workers[i]->thread, NULL);
+ }
+ }
+fail_init:
+ for (i = 0; i < num_workers; ++i) {
+ if (proc->workers[i] != NULL) {
+ if (proc->workers[i]->cmp != NULL) {
+ proc->workers[i]->cmp->
+ destroy(proc->workers[i]->cmp);
+ }
+
+ free(proc->workers[i]);
+ }
+ }
+ pthread_cond_destroy(&proc->done_cond);
+fail_cond:
+ pthread_cond_destroy(&proc->queue_cond);
+fail_mtx:
+ pthread_mutex_destroy(&proc->mtx);
+fail_free:
+ free(proc);
+ return NULL;
+}
+
+void block_processor_destroy(block_processor_t *proc)
+{
+ unsigned int i;
+ block_t *blk;
+
+ pthread_mutex_lock(&proc->mtx);
+ proc->terminate = true;
+ pthread_cond_broadcast(&proc->queue_cond);
+ pthread_mutex_unlock(&proc->mtx);
+
+ for (i = 0; i < proc->num_workers; ++i) {
+ pthread_join(proc->workers[i]->thread, NULL);
+
+ proc->workers[i]->cmp->destroy(proc->workers[i]->cmp);
+ free(proc->workers[i]);
+ }
+
+ pthread_cond_destroy(&proc->done_cond);
+ pthread_cond_destroy(&proc->queue_cond);
+ pthread_mutex_destroy(&proc->mtx);
+
+ while (proc->queue != NULL) {
+ blk = proc->queue;
+ proc->queue = blk->next;
+ free(blk);
+ }
+
+ while (proc->done != NULL) {
+ blk = proc->done;
+ proc->done = blk->next;
+ free(blk);
+ }
+
+ free(proc);
+}
+
+static int process_completed_blocks(block_processor_t *proc, block_t *queue)
+{
+ block_t *it;
+
+ while (queue != NULL) {
+ it = queue;
+ queue = queue->next;
+
+ if (it->flags & BLK_COMPRESS_ERROR) {
+ proc->status = -1;
+ } else {
+ if (proc->cb(proc->user, it))
+ proc->status = -1;
+ }
+
+ free(it);
+ }
+
+ return proc->status;
+}
+
+int block_processor_enqueue(block_processor_t *proc, block_t *block)
+{
+ block_t *queue = NULL, *it, *prev;
+
+ block->sequence_number = proc->enqueue_id++;
+ block->next = NULL;
+
+ pthread_mutex_lock(&proc->mtx);
+ if ((block->flags & BLK_DONT_COMPRESS) &&
+ (block->flags & BLK_DONT_CHECKSUM)) {
+ store_completed_block(proc, block);
+ } else {
+ while (proc->backlog > proc->num_workers * MAX_BACKLOG_FACTOR)
+ pthread_cond_wait(&proc->done_cond, &proc->mtx);
+
+ if (proc->queue_last == NULL) {
+ proc->queue = proc->queue_last = block;
+ } else {
+ proc->queue_last->next = block;
+ proc->queue_last = block;
+ }
+
+ proc->backlog += 1;
+ }
+
+ it = proc->done;
+ prev = NULL;
+
+ while (it != NULL && it->sequence_number == proc->dequeue_id) {
+ prev = it;
+ it = it->next;
+ proc->dequeue_id += 1;
+ }
+
+ if (prev != NULL) {
+ queue = proc->done;
+ prev->next = NULL;
+ proc->done = it;
+ }
+
+ pthread_cond_broadcast(&proc->queue_cond);
+ pthread_mutex_unlock(&proc->mtx);
+
+ return process_completed_blocks(proc, queue);
+}
+
+int block_processor_finish(block_processor_t *proc)
+{
+ block_t *queue, *it;
+
+ pthread_mutex_lock(&proc->mtx);
+ while (proc->backlog > 0)
+ pthread_cond_wait(&proc->done_cond, &proc->mtx);
+
+ for (it = proc->done; it != NULL; it = it->next) {
+ if (it->sequence_number != proc->dequeue_id++) {
+ pthread_mutex_unlock(&proc->mtx);
+ goto bug_seqnum;
+ }
+ }
+
+ queue = proc->done;
+ proc->done = NULL;
+ pthread_mutex_unlock(&proc->mtx);
+
+ return process_completed_blocks(proc, queue);
+bug_seqnum:
+ fputs("[BUG][parallel block processor] "
+ "gap in sequence numbers!\n", stderr);
+ return -1;
+}
diff --git a/lib/sqfs/comp/compressor.c b/lib/sqfs/comp/compressor.c
new file mode 100644
index 0000000..e59d948
--- /dev/null
+++ b/lib/sqfs/comp/compressor.c
@@ -0,0 +1,183 @@
+/* SPDX-License-Identifier: GPL-3.0-or-later */
+/*
+ * compressor.c
+ *
+ * Copyright (C) 2019 David Oberhollenzer <goliath@infraroot.at>
+ */
+#include "config.h"
+
+#include <string.h>
+#include <stdlib.h>
+#include <stdio.h>
+
+#include "internal.h"
+#include "util.h"
+
+typedef compressor_t *(*compressor_fun_t)(bool compress, size_t block_size,
+ char *options);
+
+typedef void (*compressor_help_fun_t)(void);
+
+static compressor_fun_t compressors[SQFS_COMP_MAX + 1] = {
+#ifdef WITH_GZIP
+ [SQFS_COMP_GZIP] = create_gzip_compressor,
+#endif
+#ifdef WITH_XZ
+ [SQFS_COMP_XZ] = create_xz_compressor,
+#endif
+#ifdef WITH_LZO
+ [SQFS_COMP_LZO] = create_lzo_compressor,
+#endif
+#ifdef WITH_LZ4
+ [SQFS_COMP_LZ4] = create_lz4_compressor,
+#endif
+#ifdef WITH_ZSTD
+ [SQFS_COMP_ZSTD] = create_zstd_compressor,
+#endif
+};
+
+static const compressor_help_fun_t helpfuns[SQFS_COMP_MAX + 1] = {
+#ifdef WITH_GZIP
+ [SQFS_COMP_GZIP] = compressor_gzip_print_help,
+#endif
+#ifdef WITH_XZ
+ [SQFS_COMP_XZ] = compressor_xz_print_help,
+#endif
+#ifdef WITH_LZO
+ [SQFS_COMP_LZO] = compressor_lzo_print_help,
+#endif
+#ifdef WITH_LZ4
+ [SQFS_COMP_LZ4] = compressor_lz4_print_help,
+#endif
+#ifdef WITH_ZSTD
+ [SQFS_COMP_ZSTD] = compressor_zstd_print_help,
+#endif
+};
+
+static const char *names[] = {
+ [SQFS_COMP_GZIP] = "gzip",
+ [SQFS_COMP_LZMA] = "lzma",
+ [SQFS_COMP_LZO] = "lzo",
+ [SQFS_COMP_XZ] = "xz",
+ [SQFS_COMP_LZ4] = "lz4",
+ [SQFS_COMP_ZSTD] = "zstd",
+};
+
+int generic_write_options(int fd, const void *data, size_t size)
+{
+ uint8_t buffer[size + 2];
+
+ *((uint16_t *)buffer) = htole16(0x8000 | size);
+ memcpy(buffer + 2, data, size);
+
+ if (write_data("writing compressor options",
+ fd, buffer, sizeof(buffer))) {
+ return -1;
+ }
+
+ return sizeof(buffer);
+}
+
+int generic_read_options(int fd, void *data, size_t size)
+{
+ uint8_t buffer[size + 2];
+
+ if (read_data_at("reading compressor options", sizeof(sqfs_super_t),
+ fd, buffer, sizeof(buffer))) {
+ return -1;
+ }
+
+ if (le16toh(*((uint16_t *)buffer)) != (0x8000 | size)) {
+ fputs("reading compressor options: invalid meta data header\n",
+ stderr);
+ return -1;
+ }
+
+ memcpy(data, buffer + 2, size);
+ return 0;
+}
+
+bool compressor_exists(E_SQFS_COMPRESSOR id)
+{
+ if (id < SQFS_COMP_MIN || id > SQFS_COMP_MAX)
+ return false;
+
+ return (compressors[id] != NULL);
+}
+
+compressor_t *compressor_create(E_SQFS_COMPRESSOR id, bool compress,
+ size_t block_size, char *options)
+{
+ if (id < SQFS_COMP_MIN || id > SQFS_COMP_MAX)
+ return NULL;
+
+ if (compressors[id] == NULL)
+ return NULL;
+
+ return compressors[id](compress, block_size, options);
+}
+
+void compressor_print_help(E_SQFS_COMPRESSOR id)
+{
+ if (id < SQFS_COMP_MIN || id > SQFS_COMP_MAX)
+ return;
+
+ if (compressors[id] == NULL)
+ return;
+
+ helpfuns[id]();
+}
+
+void compressor_print_available(void)
+{
+ size_t i;
+
+ fputs("Available compressors:\n", stdout);
+
+ for (i = 0; i < sizeof(names) / sizeof(names[0]); ++i) {
+ if (compressor_exists(i))
+ printf("\t%s\n", names[i]);
+ }
+
+ printf("\nDefault compressor: %s\n", names[compressor_get_default()]);
+}
+
+const char *compressor_name_from_id(E_SQFS_COMPRESSOR id)
+{
+ if (id < 0 || (size_t)id >= sizeof(names) / sizeof(names[0]))
+ return NULL;
+
+ return names[id];
+}
+
+int compressor_id_from_name(const char *name, E_SQFS_COMPRESSOR *out)
+{
+ size_t i;
+
+ for (i = 0; i < sizeof(names) / sizeof(names[0]); ++i) {
+ if (names[i] != NULL && strcmp(names[i], name) == 0) {
+ *out = i;
+ return 0;
+ }
+ }
+
+ return -1;
+}
+
+E_SQFS_COMPRESSOR compressor_get_default(void)
+{
+#if defined(WITH_XZ)
+ return SQFS_COMP_XZ;
+#elif defined(WITH_ZSTD)
+ return SQFS_COMP_ZSTD;
+#elif defined(WITH_GZIP)
+ return SQFS_COMP_GZIP;
+#elif defined(WITH_LZO)
+ return SQFS_COMP_LZO;
+#elif defined(WITH_LZ4)
+ return SQFS_COMP_LZ4;
+#else
+ fputs("No compressor implementation available!\n", stderr);
+ exit(EXIT_FAILURE);
+#endif
+}
diff --git a/lib/sqfs/comp/create_block.c b/lib/sqfs/comp/create_block.c
new file mode 100644
index 0000000..90344bb
--- /dev/null
+++ b/lib/sqfs/comp/create_block.c
@@ -0,0 +1,37 @@
+/* SPDX-License-Identifier: GPL-3.0-or-later */
+/*
+ * create_block.c
+ *
+ * Copyright (C) 2019 David Oberhollenzer <goliath@infraroot.at>
+ */
+#include "config.h"
+
+#include "block_processor.h"
+#include "util.h"
+
+#include <string.h>
+#include <stdlib.h>
+#include <stdio.h>
+
+block_t *create_block(const char *filename, int fd, size_t size,
+ void *user, uint32_t flags)
+{
+ block_t *blk = alloc_flex(sizeof(*blk), 1, size);
+
+ if (blk == NULL) {
+ perror(filename);
+ return NULL;
+ }
+
+ if (fd >= 0) {
+ if (read_data(filename, fd, blk->data, size)) {
+ free(blk);
+ return NULL;
+ }
+ }
+
+ blk->size = size;
+ blk->user = user;
+ blk->flags = flags;
+ return blk;
+}
diff --git a/lib/sqfs/comp/gzip.c b/lib/sqfs/comp/gzip.c
new file mode 100644
index 0000000..1fdc051
--- /dev/null
+++ b/lib/sqfs/comp/gzip.c
@@ -0,0 +1,412 @@
+/* SPDX-License-Identifier: GPL-3.0-or-later */
+/*
+ * gzip.c
+ *
+ * Copyright (C) 2019 David Oberhollenzer <goliath@infraroot.at>
+ */
+#include "config.h"
+
+#include <stdbool.h>
+#include <stdlib.h>
+#include <string.h>
+#include <stdio.h>
+#include <ctype.h>
+#include <zlib.h>
+
+#include "internal.h"
+
+#define GZIP_DEFAULT_LEVEL 9
+#define GZIP_DEFAULT_WINDOW 15
+#define GZIP_NUM_STRATEGIES (sizeof(strategies) / sizeof(strategies[0]))
+
+typedef enum {
+ GZIP_STRATEGY_DEFAULT = 0x01,
+ GZIP_STRATEGY_FILTERED = 0x02,
+ GZIP_STRATEGY_HUFFMAN = 0x04,
+ GZIP_STARTEGY_RLE = 0x08,
+ GZIP_STRATEGY_FIXED = 0x10,
+
+ GZIP_ALL_STRATEGIES = 0x1F,
+} GZIP_STRATEGIES;
+
+static const struct {
+ const char *name;
+ int flag;
+ int zlib;
+} strategies[] = {
+ { "default", GZIP_STRATEGY_DEFAULT, Z_DEFAULT_STRATEGY },
+ { "filtered", GZIP_STRATEGY_FILTERED, Z_FILTERED },
+ { "huffman", GZIP_STRATEGY_HUFFMAN, Z_HUFFMAN_ONLY },
+ { "rle", GZIP_STARTEGY_RLE, Z_RLE },
+ { "fixed", GZIP_STRATEGY_FIXED, Z_FIXED },
+};
+
+typedef struct {
+ uint32_t level;
+ uint16_t window;
+ uint16_t strategies;
+} gzip_options_t;
+
+typedef struct {
+ compressor_t base;
+
+ z_stream strm;
+ bool compress;
+
+ size_t block_size;
+ gzip_options_t opt;
+} gzip_compressor_t;
+
+static void gzip_destroy(compressor_t *base)
+{
+ gzip_compressor_t *gzip = (gzip_compressor_t *)base;
+
+ if (gzip->compress) {
+ deflateEnd(&gzip->strm);
+ } else {
+ inflateEnd(&gzip->strm);
+ }
+
+ free(gzip);
+}
+
+static int gzip_write_options(compressor_t *base, int fd)
+{
+ gzip_compressor_t *gzip = (gzip_compressor_t *)base;
+ gzip_options_t opt;
+
+ if (gzip->opt.level == GZIP_DEFAULT_LEVEL &&
+ gzip->opt.window == GZIP_DEFAULT_WINDOW &&
+ gzip->opt.strategies == 0) {
+ return 0;
+ }
+
+ opt.level = htole32(gzip->opt.level);
+ opt.window = htole16(gzip->opt.window);
+ opt.strategies = htole16(gzip->opt.strategies);
+
+ return generic_write_options(fd, &opt, sizeof(opt));
+}
+
+static int gzip_read_options(compressor_t *base, int fd)
+{
+ gzip_compressor_t *gzip = (gzip_compressor_t *)base;
+ gzip_options_t opt;
+
+ if (generic_read_options(fd, &opt, sizeof(opt)))
+ return -1;
+
+ gzip->opt.level = le32toh(opt.level);
+ gzip->opt.window = le16toh(opt.window);
+ gzip->opt.strategies = le16toh(opt.strategies);
+
+ if (gzip->opt.level < 1 || gzip->opt.level > 9) {
+ fprintf(stderr, "Invalid gzip compression level '%d'.\n",
+ gzip->opt.level);
+ return -1;
+ }
+
+ if (gzip->opt.window < 8 || gzip->opt.window > 15) {
+ fprintf(stderr, "Invalid gzip window size '%d'.\n",
+ gzip->opt.window);
+ return -1;
+ }
+
+ if (gzip->opt.strategies & ~GZIP_ALL_STRATEGIES) {
+ fputs("Unknown gzip strategies selected.\n", stderr);
+ return -1;
+ }
+
+ return 0;
+}
+
+static int find_strategy(gzip_compressor_t *gzip, const uint8_t *in,
+ size_t size, uint8_t *out, size_t outsize)
+{
+ int ret, selected = Z_DEFAULT_STRATEGY;
+ size_t i, length, minlength = 0;
+
+ for (i = 0; i < GZIP_NUM_STRATEGIES; ++i) {
+ if (!(strategies[i].flag & gzip->opt.strategies))
+ continue;
+
+ ret = deflateReset(&gzip->strm);
+ if (ret != Z_OK) {
+ fputs("resetting zlib stream failed\n",
+ stderr);
+ return -1;
+ }
+
+ gzip->strm.next_in = (void *)in;
+ gzip->strm.avail_in = size;
+ gzip->strm.next_out = out;
+ gzip->strm.avail_out = outsize;
+
+ ret = deflateParams(&gzip->strm, gzip->opt.level,
+ strategies[i].zlib);
+ if (ret != Z_OK) {
+ fputs("setting deflate parameters failed\n",
+ stderr);
+ return -1;
+ }
+
+ ret = deflate(&gzip->strm, Z_FINISH);
+
+ if (ret == Z_STREAM_END) {
+ length = gzip->strm.total_out;
+
+ if (minlength == 0 || length < minlength) {
+ minlength = length;
+ selected = strategies[i].zlib;
+ }
+ } else if (ret != Z_OK && ret != Z_BUF_ERROR) {
+ fputs("gzip block processing failed\n", stderr);
+ return -1;
+ }
+ }
+
+ return selected;
+}
+
+static ssize_t gzip_do_block(compressor_t *base, const uint8_t *in,
+ size_t size, uint8_t *out, size_t outsize)
+{
+ gzip_compressor_t *gzip = (gzip_compressor_t *)base;
+ int ret, strategy = 0;
+ size_t written;
+
+ if (gzip->compress && gzip->opt.strategies != 0) {
+ strategy = find_strategy(gzip, in, size, out, outsize);
+ if (strategy < 0)
+ return -1;
+ }
+
+ if (gzip->compress) {
+ ret = deflateReset(&gzip->strm);
+ } else {
+ ret = inflateReset(&gzip->strm);
+ }
+
+ if (ret != Z_OK) {
+ fputs("resetting zlib stream failed\n", stderr);
+ return -1;
+ }
+
+ gzip->strm.next_in = (void *)in;
+ gzip->strm.avail_in = size;
+ gzip->strm.next_out = out;
+ gzip->strm.avail_out = outsize;
+
+ if (gzip->compress && gzip->opt.strategies != 0) {
+ ret = deflateParams(&gzip->strm, gzip->opt.level, strategy);
+ if (ret != Z_OK) {
+ fputs("setting selcted deflate parameters failed\n",
+ stderr);
+ return -1;
+ }
+ }
+
+ if (gzip->compress) {
+ ret = deflate(&gzip->strm, Z_FINISH);
+ } else {
+ ret = inflate(&gzip->strm, Z_FINISH);
+ }
+
+ if (ret == Z_STREAM_END) {
+ written = gzip->strm.total_out;
+
+ if (gzip->compress && written >= size)
+ return 0;
+
+ return (ssize_t)written;
+ }
+
+ if (ret != Z_OK && ret != Z_BUF_ERROR) {
+ fputs("gzip block processing failed\n", stderr);
+ return -1;
+ }
+
+ return 0;
+}
+
+static int process_options(char *options, int *level, int *window, int *flags)
+{
+ enum {
+ OPT_WINDOW = 0,
+ OPT_LEVEL,
+ };
+ char *const token[] = {
+ [OPT_WINDOW] = (char *)"window",
+ [OPT_LEVEL] = (char *)"level",
+ NULL
+ };
+ char *subopts, *value;
+ size_t i;
+ int opt;
+
+ subopts = options;
+
+ while (*subopts != '\0') {
+ opt = getsubopt(&subopts, token, &value);
+
+ switch (opt) {
+ case OPT_WINDOW:
+ if (value == NULL)
+ goto fail_value;
+
+ for (i = 0; isdigit(value[i]); ++i)
+ ;
+
+ if (i < 1 || i > 3 || value[i] != '\0')
+ goto fail_window;
+
+ *window = atoi(value);
+
+ if (*window < 8 || *window > 15)
+ goto fail_window;
+ break;
+ case OPT_LEVEL:
+ if (value == NULL)
+ goto fail_value;
+
+ for (i = 0; isdigit(value[i]); ++i)
+ ;
+
+ if (i < 1 || i > 3 || value[i] != '\0')
+ goto fail_level;
+
+ *level = atoi(value);
+
+ if (*level < 1 || *level > 9)
+ goto fail_level;
+ break;
+ default:
+ for (i = 0; i < GZIP_NUM_STRATEGIES; ++i) {
+ if (strcmp(value, strategies[i].name) == 0) {
+ *flags |= strategies[i].flag;
+ break;
+ }
+ }
+ if (i == GZIP_NUM_STRATEGIES)
+ goto fail_opt;
+ break;
+ }
+ }
+
+ return 0;
+fail_window:
+ fputs("Window size must be a number between 8 and 15.\n", stderr);
+ return -1;
+fail_level:
+ fputs("Compression level must be a number between 1 and 9.\n", stderr);
+ return -1;
+fail_opt:
+ fprintf(stderr, "Unknown option '%s'.\n", value);
+ return -1;
+fail_value:
+ fprintf(stderr, "Missing value for '%s'.\n", token[opt]);
+ return -1;
+}
+
+static compressor_t *gzip_create_copy(compressor_t *cmp)
+{
+ gzip_compressor_t *gzip = malloc(sizeof(*gzip));
+ int ret;
+
+ if (gzip == NULL) {
+ perror("creating additional gzip compressor");
+ return NULL;
+ }
+
+ memcpy(gzip, cmp, sizeof(*gzip));
+ memset(&gzip->strm, 0, sizeof(gzip->strm));
+
+ if (gzip->compress) {
+ ret = deflateInit2(&gzip->strm, gzip->opt.level, Z_DEFLATED,
+ gzip->opt.window, 8, Z_DEFAULT_STRATEGY);
+ } else {
+ ret = inflateInit(&gzip->strm);
+ }
+
+ if (ret != Z_OK) {
+ fputs("internal error creating additional zlib stream\n",
+ stderr);
+ free(gzip);
+ return NULL;
+ }
+
+ return (compressor_t *)gzip;
+}
+
+compressor_t *create_gzip_compressor(bool compress, size_t block_size,
+ char *options)
+{
+ int window = GZIP_DEFAULT_WINDOW;
+ int level = GZIP_DEFAULT_LEVEL;
+ gzip_compressor_t *gzip;
+ compressor_t *base;
+ int flags = 0;
+ int ret;
+
+ if (options != NULL) {
+ if (process_options(options, &level, &window, &flags))
+ return NULL;
+ }
+
+ gzip = calloc(1, sizeof(*gzip));
+ base = (compressor_t *)gzip;
+
+ if (gzip == NULL) {
+ perror("creating gzip compressor");
+ return NULL;
+ }
+
+ gzip->opt.level = level;
+ gzip->opt.window = window;
+ gzip->opt.strategies = flags;
+ gzip->compress = compress;
+ gzip->block_size = block_size;
+ base->do_block = gzip_do_block;
+ base->destroy = gzip_destroy;
+ base->write_options = gzip_write_options;
+ base->read_options = gzip_read_options;
+ base->create_copy = gzip_create_copy;
+
+ if (compress) {
+ ret = deflateInit2(&gzip->strm, level, Z_DEFLATED, window, 8,
+ Z_DEFAULT_STRATEGY);
+ } else {
+ ret = inflateInit(&gzip->strm);
+ }
+
+ if (ret != Z_OK) {
+ fputs("internal error creating zlib stream\n", stderr);
+ free(gzip);
+ return NULL;
+ }
+
+ return base;
+}
+
+void compressor_gzip_print_help(void)
+{
+ size_t i;
+
+ printf(
+"Available options for gzip compressor:\n"
+"\n"
+" level=<value> Compression level. Value from 1 to 9.\n"
+" Defaults to %d.\n"
+" window=<size> Deflate compression window size. Value from 8 to 15.\n"
+" Defaults to %d.\n"
+"\n"
+"In additon to the options, one or more strategies can be specified.\n"
+"If multiple stratgies are provided, the one yielding the best compression\n"
+"ratio will be used.\n"
+"\n"
+"The following strategies are available:\n",
+ GZIP_DEFAULT_LEVEL, GZIP_DEFAULT_WINDOW);
+
+ for (i = 0; i < GZIP_NUM_STRATEGIES; ++i)
+ printf("\t%s\n", strategies[i].name);
+}
diff --git a/lib/sqfs/comp/internal.h b/lib/sqfs/comp/internal.h
new file mode 100644
index 0000000..986266d
--- /dev/null
+++ b/lib/sqfs/comp/internal.h
@@ -0,0 +1,44 @@
+/* SPDX-License-Identifier: GPL-3.0-or-later */
+/*
+ * internal.h
+ *
+ * Copyright (C) 2019 David Oberhollenzer <goliath@infraroot.at>
+ */
+#ifndef INTERNAL_H
+#define INTERNAL_H
+
+#include "config.h"
+
+#include "compress.h"
+#include "util.h"
+
+int generic_write_options(int fd, const void *data, size_t size);
+
+int generic_read_options(int fd, void *data, size_t size);
+
+compressor_t *create_xz_compressor(bool compress, size_t block_size,
+ char *options);
+
+compressor_t *create_gzip_compressor(bool compress, size_t block_size,
+ char *options);
+
+compressor_t *create_lzo_compressor(bool compress, size_t block_size,
+ char *options);
+
+compressor_t *create_lz4_compressor(bool compress, size_t block_size,
+ char *options);
+
+compressor_t *create_zstd_compressor(bool compress, size_t block_size,
+ char *options);
+
+void compressor_xz_print_help(void);
+
+void compressor_gzip_print_help(void);
+
+void compressor_lzo_print_help(void);
+
+void compressor_lz4_print_help(void);
+
+void compressor_zstd_print_help(void);
+
+#endif /* INTERNAL_H */
diff --git a/lib/sqfs/comp/lz4.c b/lib/sqfs/comp/lz4.c
new file mode 100644
index 0000000..abb6c5c
--- /dev/null
+++ b/lib/sqfs/comp/lz4.c
@@ -0,0 +1,159 @@
+/* SPDX-License-Identifier: GPL-3.0-or-later */
+/*
+ * lz4.c
+ *
+ * Copyright (C) 2019 David Oberhollenzer <goliath@infraroot.at>
+ */
+#include "config.h"
+
+#include <stdbool.h>
+#include <stdlib.h>
+#include <string.h>
+#include <stdio.h>
+
+#include <lz4.h>
+#include <lz4hc.h>
+
+#include "internal.h"
+
+typedef struct {
+ compressor_t base;
+ bool high_compression;
+} lz4_compressor_t;
+
+typedef struct {
+ uint32_t version;
+ uint32_t flags;
+} lz4_options;
+
+#define LZ4LEGACY 1
+#define LZ4_FLAG_HC 0x01
+
+static int lz4_write_options(compressor_t *base, int fd)
+{
+ lz4_compressor_t *lz4 = (lz4_compressor_t *)base;
+ lz4_options opt = {
+ .version = htole32(LZ4LEGACY),
+ .flags = htole32(lz4->high_compression ? LZ4_FLAG_HC : 0),
+ };
+
+ return generic_write_options(fd, &opt, sizeof(opt));
+}
+
+static int lz4_read_options(compressor_t *base, int fd)
+{
+ lz4_options opt;
+ (void)base;
+
+ if (generic_read_options(fd, &opt, sizeof(opt)))
+ return -1;
+
+ opt.version = le32toh(opt.version);
+ opt.flags = le32toh(opt.flags);
+
+ if (opt.version != LZ4LEGACY) {
+ fprintf(stderr, "unsupported lz4 version '%d'\n", opt.version);
+ return -1;
+ }
+
+ return 0;
+}
+
+static ssize_t lz4_comp_block(compressor_t *base, const uint8_t *in,
+ size_t size, uint8_t *out, size_t outsize)
+{
+ lz4_compressor_t *lz4 = (lz4_compressor_t *)base;
+ int ret;
+
+ if (lz4->high_compression) {
+ ret = LZ4_compress_HC((void *)in, (void *)out,
+ size, outsize, LZ4HC_CLEVEL_MAX);
+ } else {
+ ret = LZ4_compress_default((void *)in, (void *)out,
+ size, outsize);
+ }
+
+ if (ret < 0) {
+ fputs("internal error in lz4 compressor\n", stderr);
+ return -1;
+ }
+
+ return ret;
+}
+
+static ssize_t lz4_uncomp_block(compressor_t *base, const uint8_t *in,
+ size_t size, uint8_t *out, size_t outsize)
+{
+ int ret;
+ (void)base;
+
+ ret = LZ4_decompress_safe((void *)in, (void *)out, size, outsize);
+
+ if (ret < 0) {
+ fputs("internal error in lz4 decompressor\n", stderr);
+ return -1;
+ }
+
+ return ret;
+}
+
+static compressor_t *lz4_create_copy(compressor_t *cmp)
+{
+ lz4_compressor_t *lz4 = malloc(sizeof(*lz4));
+
+ if (lz4 == NULL) {
+ perror("creating additional lz4 compressor");
+ return NULL;
+ }
+
+ memcpy(lz4, cmp, sizeof(*lz4));
+ return (compressor_t *)lz4;
+}
+
+static void lz4_destroy(compressor_t *base)
+{
+ free(base);
+}
+
+compressor_t *create_lz4_compressor(bool compress, size_t block_size,
+ char *options)
+{
+ lz4_compressor_t *lz4 = calloc(1, sizeof(*lz4));
+ compressor_t *base = (compressor_t *)lz4;
+ (void)block_size;
+
+ if (lz4 == NULL) {
+ perror("creating lz4 compressor");
+ return NULL;
+ }
+
+ lz4->high_compression = false;
+
+ if (options != NULL) {
+ if (strcmp(options, "hc") == 0) {
+ lz4->high_compression = true;
+ } else {
+ fputs("Unsupported extra options for lz4 "
+ "compressor.\n", stderr);
+ free(lz4);
+ return NULL;
+ }
+ }
+
+ base->destroy = lz4_destroy;
+ base->do_block = compress ? lz4_comp_block : lz4_uncomp_block;
+ base->write_options = lz4_write_options;
+ base->read_options = lz4_read_options;
+ base->create_copy = lz4_create_copy;
+ return base;
+}
+
+void compressor_lz4_print_help(void)
+{
+ fputs("Available options for lz4 compressor:\n"
+ "\n"
+ " hc If present, use slower but better compressing\n"
+ " variant of lz4.\n"
+ "\n",
+ stdout);
+}
diff --git a/lib/sqfs/comp/lzo.c b/lib/sqfs/comp/lzo.c
new file mode 100644
index 0000000..09ef75c
--- /dev/null
+++ b/lib/sqfs/comp/lzo.c
@@ -0,0 +1,321 @@
+/* SPDX-License-Identifier: GPL-3.0-or-later */
+/*
+ * lzo.c
+ *
+ * Copyright (C) 2019 David Oberhollenzer <goliath@infraroot.at>
+ */
+#include "config.h"
+
+#include <stdbool.h>
+#include <stdlib.h>
+#include <string.h>
+#include <stdio.h>
+#include <ctype.h>
+
+#include <lzo/lzo1x.h>
+
+#include "internal.h"
+
+
+typedef enum {
+ LZO_ALGORITHM_LZO1X_1 = 0,
+ LZO_ALGORITHM_LZO1X_1_11 = 1,
+ LZO_ALGORITHM_LZO1X_1_12 = 2,
+ LZO_ALGORITHM_LZO1X_1_15 = 3,
+ LZO_ALGORITHM_LZO1X_999 = 4,
+} LZO_ALGORITHM;
+
+#define LZO_DEFAULT_ALG LZO_ALGORITHM_LZO1X_999
+#define LZO_DEFAULT_LEVEL 8
+#define LZO_NUM_ALGS (sizeof(lzo_algs) / sizeof(lzo_algs[0]))
+
+typedef int (*lzo_cb_t)(const lzo_bytep src, lzo_uint src_len, lzo_bytep dst,
+ lzo_uintp dst_len, lzo_voidp wrkmem);
+
+static const struct {
+ const char *name;
+ lzo_cb_t compress;
+ size_t bufsize;
+} lzo_algs[] = {
+ [LZO_ALGORITHM_LZO1X_1] = {
+ .name = "lzo1x_1",
+ .compress = lzo1x_1_compress,
+ .bufsize = LZO1X_1_MEM_COMPRESS,
+ },
+ [LZO_ALGORITHM_LZO1X_1_11] = {
+ .name = "lzo1x_1_11",
+ .compress = lzo1x_1_11_compress,
+ .bufsize = LZO1X_1_11_MEM_COMPRESS,
+ },
+ [LZO_ALGORITHM_LZO1X_1_12] = {
+ .name = "lzo1x_1_12",
+ .compress = lzo1x_1_12_compress,
+ .bufsize = LZO1X_1_12_MEM_COMPRESS,
+ },
+ [LZO_ALGORITHM_LZO1X_1_15] = {
+ .name = "lzo1x_1_15",
+ .compress = lzo1x_1_15_compress,
+ .bufsize = LZO1X_1_15_MEM_COMPRESS,
+ },
+ [LZO_ALGORITHM_LZO1X_999] = {
+ .name = "lzo1x_999",
+ .compress = lzo1x_999_compress,
+ .bufsize = LZO1X_999_MEM_COMPRESS,
+ },
+};
+
+typedef struct {
+ compressor_t base;
+ int algorithm;
+ int level;
+
+ uint8_t buffer[];
+} lzo_compressor_t;
+
+typedef struct {
+ uint32_t algorithm;
+ uint32_t level;
+} lzo_options_t;
+
+static int lzo_write_options(compressor_t *base, int fd)
+{
+ lzo_compressor_t *lzo = (lzo_compressor_t *)base;
+ lzo_options_t opt;
+
+ if (lzo->algorithm == LZO_DEFAULT_ALG &&
+ lzo->level == LZO_DEFAULT_LEVEL) {
+ return 0;
+ }
+
+ opt.algorithm = htole32(lzo->algorithm);
+
+ if (lzo->algorithm == LZO_ALGORITHM_LZO1X_999) {
+ opt.level = htole32(lzo->level);
+ } else {
+ opt.level = 0;
+ }
+
+ return generic_write_options(fd, &opt, sizeof(opt));
+}
+
+static int lzo_read_options(compressor_t *base, int fd)
+{
+ lzo_compressor_t *lzo = (lzo_compressor_t *)base;
+ lzo_options_t opt;
+
+ if (generic_read_options(fd, &opt, sizeof(opt)))
+ return -1;
+
+ lzo->algorithm = le32toh(opt.algorithm);
+ lzo->level = le32toh(opt.level);
+
+ switch(lzo->algorithm) {
+ case LZO_ALGORITHM_LZO1X_1:
+ case LZO_ALGORITHM_LZO1X_1_11:
+ case LZO_ALGORITHM_LZO1X_1_12:
+ case LZO_ALGORITHM_LZO1X_1_15:
+ if (lzo->level != 0)
+ goto fail_level;
+ break;
+ case LZO_ALGORITHM_LZO1X_999:
+ if (lzo->level < 1 || lzo->level > 9)
+ goto fail_level;
+ break;
+ default:
+ fputs("Unsupported LZO variant specified.\n", stderr);
+ return -1;
+ }
+
+ return 0;
+fail_level:
+ fputs("Unsupported LZO compression level specified.\n", stderr);
+ return -1;
+}
+
+static ssize_t lzo_comp_block(compressor_t *base, const uint8_t *in,
+ size_t size, uint8_t *out, size_t outsize)
+{
+ lzo_compressor_t *lzo = (lzo_compressor_t *)base;
+ lzo_uint len = outsize;
+ int ret;
+
+ if (lzo->algorithm == LZO_ALGORITHM_LZO1X_999 &&
+ lzo->level != LZO_DEFAULT_LEVEL) {
+ ret = lzo1x_999_compress_level(in, size, out, &len,
+ lzo->buffer, NULL, 0, 0,
+ lzo->level);
+ } else {
+ ret = lzo_algs[lzo->algorithm].compress(in, size, out,
+ &len, lzo->buffer);
+ }
+
+ if (ret != LZO_E_OK) {
+ fputs("LZO compression failed.\n", stderr);
+ return -1;
+ }
+
+ if (len < size)
+ return len;
+
+ return 0;
+}
+
+static ssize_t lzo_uncomp_block(compressor_t *base, const uint8_t *in,
+ size_t size, uint8_t *out, size_t outsize)
+{
+ lzo_compressor_t *lzo = (lzo_compressor_t *)base;
+ lzo_uint len = outsize;
+ int ret;
+
+ ret = lzo1x_decompress_safe(in, size, out, &len, lzo->buffer);
+
+ if (ret != LZO_E_OK) {
+ fputs("lzo decompress: input data is corrupted\n", stderr);
+ return -1;
+ }
+
+ return len;
+}
+
+static compressor_t *lzo_create_copy(compressor_t *cmp)
+{
+ lzo_compressor_t *other = (lzo_compressor_t *)cmp;
+ lzo_compressor_t *lzo;
+
+ lzo = alloc_flex(sizeof(*lzo), 1, lzo_algs[other->algorithm].bufsize);
+
+ if (lzo == NULL) {
+ perror("creating additional lzo compressor");
+ return NULL;
+ }
+
+ memcpy(lzo, other, sizeof(*lzo));
+ return (compressor_t *)lzo;
+}
+
+static void lzo_destroy(compressor_t *base)
+{
+ free(base);
+}
+
+static int process_options(char *options, int *algorithm, int *level)
+{
+ enum {
+ OPT_ALG = 0,
+ OPT_LEVEL,
+ };
+ char *const token[] = {
+ [OPT_ALG] = (char *)"algorithm",
+ [OPT_LEVEL] = (char *)"level",
+ NULL
+ };
+ char *subopts, *value;
+ size_t i;
+ int opt;
+
+ subopts = options;
+
+ while (*subopts != '\0') {
+ opt = getsubopt(&subopts, token, &value);
+
+ switch (opt) {
+ case OPT_ALG:
+ if (value == NULL)
+ goto fail_value;
+
+ for (i = 0; i < LZO_NUM_ALGS; ++i) {
+ if (strcmp(lzo_algs[i].name, value) == 0) {
+ *algorithm = i;
+ break;
+ }
+ }
+
+ if (i == LZO_NUM_ALGS) {
+ fprintf(stderr, "Unknown lzo variant '%s'.\n",
+ value);
+ return -1;
+ }
+ break;
+ case OPT_LEVEL:
+ if (value == NULL)
+ goto fail_value;
+
+ for (i = 0; isdigit(value[i]); ++i)
+ ;
+
+ if (i < 1 || i > 3 || value[i] != '\0')
+ goto fail_level;
+
+ *level = atoi(value);
+
+ if (*level < 1 || *level > 9)
+ goto fail_level;
+ break;
+ default:
+ goto fail_opt;
+ }
+ }
+
+ return 0;
+fail_level:
+ fputs("Compression level must be a number between 1 and 9.\n", stderr);
+ return -1;
+fail_opt:
+ fprintf(stderr, "Unknown option '%s'.\n", value);
+ return -1;
+fail_value:
+ fprintf(stderr, "Missing value for '%s'.\n", token[opt]);
+ return -1;
+}
+
+compressor_t *create_lzo_compressor(bool compress, size_t block_size,
+ char *options)
+{
+ lzo_compressor_t *lzo;
+ compressor_t *base;
+ int level, alg;
+ (void)block_size;
+
+ alg = LZO_DEFAULT_ALG;
+ level = LZO_DEFAULT_LEVEL;
+
+ if (options != NULL && process_options(options, &alg, &level) != 0)
+ return NULL;
+
+ lzo = alloc_flex(sizeof(*lzo), 1, lzo_algs[alg].bufsize);
+ base = (compressor_t *)lzo;
+
+ if (lzo == NULL) {
+ perror("creating lzo compressor");
+ return NULL;
+ }
+
+ lzo->algorithm = alg;
+ lzo->level = level;
+
+ base->destroy = lzo_destroy;
+ base->do_block = compress ? lzo_comp_block : lzo_uncomp_block;
+ base->write_options = lzo_write_options;
+ base->read_options = lzo_read_options;
+ base->create_copy = lzo_create_copy;
+ return base;
+}
+
+void compressor_lzo_print_help(void)
+{
+ size_t i;
+
+ fputs("Available options for lzo compressor:\n"
+ "\n"
+ " algorithm=<name> Specify the variant of lzo to use.\n"
+ " Defaults to 'lzo1x_999'.\n"
+ " level=<value> For lzo1x_999, the compression level.\n"
+ " Value from 1 to 9. Defaults to 8.\n"
+ " Ignored if algorithm is not lzo1x_999.\n"
+ "\n"
+ "Available algorithms:\n",
+ stdout);
+
+ for (i = 0; i < LZO_NUM_ALGS; ++i)
+ printf("\t%s\n", lzo_algs[i].name);
+}
diff --git a/lib/sqfs/comp/process_block.c b/lib/sqfs/comp/process_block.c
new file mode 100644
index 0000000..0fcbae0
--- /dev/null
+++ b/lib/sqfs/comp/process_block.c
@@ -0,0 +1,38 @@
+/* SPDX-License-Identifier: GPL-3.0-or-later */
+/*
+ * process_block.c
+ *
+ * Copyright (C) 2019 David Oberhollenzer <goliath@infraroot.at>
+ */
+#include "config.h"
+
+#include "block_processor.h"
+#include "util.h"
+
+#include <string.h>
+#include <zlib.h>
+
+int process_block(block_t *block, compressor_t *cmp,
+ uint8_t *scratch, size_t scratch_size)
+{
+ ssize_t ret;
+
+ if (!(block->flags & BLK_DONT_CHECKSUM))
+ block->checksum = crc32(0, block->data, block->size);
+
+ if (!(block->flags & BLK_DONT_COMPRESS)) {
+ ret = cmp->do_block(cmp, block->data, block->size,
+ scratch, scratch_size);
+
+ if (ret < 0)
+ return -1;
+
+ if (ret > 0) {
+ memcpy(block->data, scratch, ret);
+ block->size = ret;
+ block->flags |= BLK_IS_COMPRESSED;
+ }
+ }
+
+ return 0;
+}
diff --git a/lib/sqfs/comp/xz.c b/lib/sqfs/comp/xz.c
new file mode 100644
index 0000000..d38aab6
--- /dev/null
+++ b/lib/sqfs/comp/xz.c
@@ -0,0 +1,360 @@
+/* SPDX-License-Identifier: GPL-3.0-or-later */
+/*
+ * xz.c
+ *
+ * Copyright (C) 2019 David Oberhollenzer <goliath@infraroot.at>
+ */
+#include "config.h"
+
+#include <stdbool.h>
+#include <stdlib.h>
+#include <string.h>
+#include <stdio.h>
+#include <ctype.h>
+#include <lzma.h>
+
+#include "internal.h"
+
+typedef enum {
+ XZ_FILTER_X86 = 0x01,
+ XZ_FILTER_POWERPC = 0x02,
+ XZ_FILTER_IA64 = 0x04,
+ XZ_FILTER_ARM = 0x08,
+ XZ_FILTER_ARMTHUMB = 0x10,
+ XZ_FILTER_SPARC = 0x20,
+
+ XZ_FILTER_ALL = 0x3F,
+} XZ_FILTER_FLAG;
+
+typedef struct {
+ compressor_t base;
+ size_t block_size;
+ size_t dict_size;
+ int flags;
+} xz_compressor_t;
+
+typedef struct {
+ uint32_t dict_size;
+ uint32_t flags;
+} xz_options_t;
+
+static const struct {
+ const char *name;
+ lzma_vli filter;
+ int flag;
+} xz_filters[] = {
+ { "x86", LZMA_FILTER_X86, XZ_FILTER_X86 },
+ { "powerpc", LZMA_FILTER_POWERPC, XZ_FILTER_POWERPC },
+ { "ia64", LZMA_FILTER_IA64, XZ_FILTER_IA64 },
+ { "arm", LZMA_FILTER_ARM, XZ_FILTER_ARM },
+ { "armthumb", LZMA_FILTER_ARMTHUMB, XZ_FILTER_ARMTHUMB },
+ { "sparc", LZMA_FILTER_SPARC, XZ_FILTER_SPARC },
+};
+
+#define XZ_NUM_FILTERS (sizeof(xz_filters) / sizeof(xz_filters[0]))
+
+static int xz_write_options(compressor_t *base, int fd)
+{
+ xz_compressor_t *xz = (xz_compressor_t *)base;
+ xz_options_t opt;
+
+ if (xz->flags == 0 && xz->dict_size == xz->block_size)
+ return 0;
+
+ opt.dict_size = htole32(xz->dict_size);
+ opt.flags = htole32(xz->flags);
+
+ return generic_write_options(fd, &opt, sizeof(opt));
+}
+
+static int xz_read_options(compressor_t *base, int fd)
+{
+ xz_compressor_t *xz = (xz_compressor_t *)base;
+ xz_options_t opt;
+ uint32_t mask;
+
+ if (generic_read_options(fd, &opt, sizeof(opt)))
+ return -1;
+
+ opt.dict_size = le32toh(opt.dict_size);
+ opt.flags = le32toh(opt.flags);
+
+ mask = opt.dict_size & (opt.dict_size - 1);
+
+ if (mask != 0 && ((mask & (mask - 1)) != 0)) {
+ fputs("Invalid lzma dictionary size.\n", stderr);
+ return -1;
+ }
+
+ if (opt.flags & ~XZ_FILTER_ALL) {
+ fputs("Unknown BCJ filter used.\n", stderr);
+ return -1;
+ }
+
+ xz->flags = opt.flags;
+ xz->dict_size = opt.dict_size;
+ return 0;
+}
+
+static ssize_t compress(xz_compressor_t *xz, lzma_vli filter,
+ const uint8_t *in, size_t size,
+ uint8_t *out, size_t outsize)
+{
+ lzma_filter filters[5];
+ lzma_options_lzma opt;
+ size_t written = 0;
+ lzma_ret ret;
+ int i = 0;
+
+ if (lzma_lzma_preset(&opt, LZMA_PRESET_DEFAULT)) {
+ fputs("error initializing xz options\n", stderr);
+ return -1;
+ }
+
+ opt.dict_size = xz->dict_size;
+
+ if (filter != LZMA_VLI_UNKNOWN) {
+ filters[i].id = filter;
+ filters[i].options = NULL;
+ ++i;
+ }
+
+ filters[i].id = LZMA_FILTER_LZMA2;
+ filters[i].options = &opt;
+ ++i;
+
+ filters[i].id = LZMA_VLI_UNKNOWN;
+ filters[i].options = NULL;
+ ++i;
+
+ ret = lzma_stream_buffer_encode(filters, LZMA_CHECK_CRC32, NULL,
+ in, size, out, &written, outsize);
+
+ if (ret == LZMA_OK)
+ return (written >= size) ? 0 : written;
+
+ if (ret != LZMA_BUF_ERROR) {
+ fputs("xz block compress failed\n", stderr);
+ return -1;
+ }
+
+ return 0;
+}
+
+static ssize_t xz_comp_block(compressor_t *base, const uint8_t *in,
+ size_t size, uint8_t *out, size_t outsize)
+{
+ xz_compressor_t *xz = (xz_compressor_t *)base;
+ lzma_vli selected = LZMA_VLI_UNKNOWN;
+ size_t i, smallest;
+ ssize_t ret;
+
+ ret = compress(xz, LZMA_VLI_UNKNOWN, in, size, out, outsize);
+ if (ret < 0 || xz->flags == 0)
+ return ret;
+
+ smallest = ret;
+
+ for (i = 0; i < XZ_NUM_FILTERS; ++i) {
+ if (!(xz->flags & xz_filters[i].flag))
+ continue;
+
+ ret = compress(xz, xz_filters[i].filter, in, size, out, outsize);
+ if (ret < 0)
+ return -1;
+
+ if (ret > 0 && (smallest == 0 || (size_t)ret < smallest)) {
+ smallest = ret;
+ selected = xz_filters[i].filter;
+ }
+ }
+
+ if (smallest == 0)
+ return 0;
+
+ return compress(xz, selected, in, size, out, outsize);
+}
+
+static ssize_t xz_uncomp_block(compressor_t *base, const uint8_t *in,
+ size_t size, uint8_t *out, size_t outsize)
+{
+ uint64_t memlimit = 32 * 1024 * 1024;
+ size_t dest_pos = 0;
+ size_t src_pos = 0;
+ lzma_ret ret;
+ (void)base;
+
+ ret = lzma_stream_buffer_decode(&memlimit, 0, NULL,
+ in, &src_pos, size,
+ out, &dest_pos, outsize);
+
+ if (ret == LZMA_OK && size == src_pos)
+ return (ssize_t)dest_pos;
+
+ fputs("xz block extract failed\n", stderr);
+ return -1;
+}
+
+static compressor_t *xz_create_copy(compressor_t *cmp)
+{
+ xz_compressor_t *xz = malloc(sizeof(*xz));
+
+ if (xz == NULL) {
+ perror("creating additional xz compressor");
+ return NULL;
+ }
+
+ memcpy(xz, cmp, sizeof(*xz));
+ return (compressor_t *)xz;
+}
+
+static void xz_destroy(compressor_t *base)
+{
+ free(base);
+}
+
+static int process_options(char *options, size_t blocksize,
+ int *flags, uint64_t *dictsize)
+{
+ enum {
+ OPT_DICT = 0,
+ };
+ char *const token[] = {
+ [OPT_DICT] = (char *)"dictsize",
+ NULL
+ };
+ char *subopts, *value;
+ uint64_t mask;
+ size_t i;
+ int opt;
+
+ subopts = options;
+
+ while (*subopts != '\0') {
+ opt = getsubopt(&subopts, token, &value);
+
+ switch (opt) {
+ case OPT_DICT:
+ if (value == NULL)
+ goto fail_value;
+
+ for (i = 0; isdigit(value[i]); ++i)
+ ;
+
+ if (i < 1 || i > 9)
+ goto fail_dict;
+
+ *dictsize = atol(value);
+
+ switch (value[i]) {
+ case '\0':
+ break;
+ case 'm':
+ case 'M':
+ *dictsize <<= 20;
+ break;
+ case 'k':
+ case 'K':
+ *dictsize <<= 10;
+ break;
+ case '%':
+ *dictsize = ((*dictsize) * blocksize) / 100;
+ break;
+ default:
+ goto fail_dict;
+ }
+
+ if (*dictsize > 0x0FFFFFFFFUL)
+ goto fail_dict_ov;
+
+ mask = *dictsize & (*dictsize - 1);
+
+ if (mask != 0 && ((mask & (mask - 1)) != 0))
+ goto fail_dict_pot;
+ break;
+ default:
+ for (i = 0; i < XZ_NUM_FILTERS; ++i) {
+ if (strcmp(value, xz_filters[i].name) == 0) {
+ *flags |= xz_filters[i].flag;
+ break;
+ }
+ }
+ if (i == XZ_NUM_FILTERS)
+ goto fail_opt;
+ break;
+ }
+ }
+
+ return 0;
+fail_dict_pot:
+ fputs("dictionary size must be either 2^n or 2^n + 2^(n-1)\n", stderr);
+ return -1;
+fail_dict_ov:
+ fputs("dictionary size too large.\n", stderr);
+ return -1;
+fail_dict:
+ fputs("dictionary size must be a number with the optional "
+ "suffix 'm','k' or '%'.\n", stderr);
+ return -1;
+fail_opt:
+ fprintf(stderr, "Unknown option '%s'.\n", value);
+ return -1;
+fail_value:
+ fprintf(stderr, "Missing value for '%s'.\n", token[opt]);
+ return -1;
+}
+
+compressor_t *create_xz_compressor(bool compress, size_t block_size,
+ char *options)
+{
+ uint64_t dictsize = block_size;
+ xz_compressor_t *xz;
+ compressor_t *base;
+ int flags = 0;
+
+ if (options != NULL) {
+ if (process_options(options, block_size, &flags, &dictsize))
+ return NULL;
+ }
+
+ xz = calloc(1, sizeof(*xz));
+ base = (compressor_t *)xz;
+ if (xz == NULL) {
+ perror("creating xz compressor");
+ return NULL;
+ }
+
+ xz->flags = flags;
+ xz->dict_size = dictsize;
+ xz->block_size = block_size;
+ base->destroy = xz_destroy;
+ base->do_block = compress ? xz_comp_block : xz_uncomp_block;
+ base->write_options = xz_write_options;
+ base->read_options = xz_read_options;
+ base->create_copy = xz_create_copy;
+ return base;
+}
+
+void compressor_xz_print_help(void)
+{
+ size_t i;
+
+ fputs(
+"Available options for xz compressor:\n"
+"\n"
+" dictsize=<value> Dictionary size. Either a value in bytes or a\n"
+" percentage of the block size. Defaults to 100%.\n"
+" The suffix '%' indicates a percentage. 'K' and 'M'\n"
+" can also be used for kibi and mebi bytes\n"
+" respecitively.\n"
+"\n"
+"In additon to the options, one or more bcj filters can be specified.\n"
+"If multiple filters are provided, the one yielding the best compression\n"
+"ratio will be used.\n"
+"\n"
+"The following filters are available:\n",
+ stdout);
+
+ for (i = 0; i < XZ_NUM_FILTERS; ++i)
+ printf("\t%s\n", xz_filters[i].name);
+}
diff --git a/lib/sqfs/comp/zstd.c b/lib/sqfs/comp/zstd.c
new file mode 100644
index 0000000..e206338
--- /dev/null
+++ b/lib/sqfs/comp/zstd.c
@@ -0,0 +1,188 @@
+/* SPDX-License-Identifier: GPL-3.0-or-later */
+/*
+ * zstd.c
+ *
+ * Copyright (C) 2019 David Oberhollenzer <goliath@infraroot.at>
+ */
+#include "config.h"
+
+#include <stdbool.h>
+#include <stdlib.h>
+#include <string.h>
+#include <stdio.h>
+#include <ctype.h>
+
+#include <zstd.h>
+
+#include "internal.h"
+
+#define ZSTD_DEFAULT_COMPRESSION_LEVEL 15
+
+typedef struct {
+ compressor_t base;
+ ZSTD_CCtx *zctx;
+ int level;
+} zstd_compressor_t;
+
+typedef struct {
+ uint32_t level;
+} zstd_options_t;
+
+static int zstd_write_options(compressor_t *base, int fd)
+{
+ zstd_compressor_t *zstd = (zstd_compressor_t *)base;
+ zstd_options_t opt;
+ (void)fd;
+
+ if (zstd->level == ZSTD_DEFAULT_COMPRESSION_LEVEL)
+ return 0;
+
+ opt.level = htole32(zstd->level);
+ return generic_write_options(fd, &opt, sizeof(opt));
+}
+
+static int zstd_read_options(compressor_t *base, int fd)
+{
+ zstd_options_t opt;
+ (void)base;
+
+ if (generic_read_options(fd, &opt, sizeof(opt)))
+ return -1;
+
+ opt.level = le32toh(opt.level);
+ return 0;
+}
+
+static ssize_t zstd_comp_block(compressor_t *base, const uint8_t *in,
+ size_t size, uint8_t *out, size_t outsize)
+{
+ zstd_compressor_t *zstd = (zstd_compressor_t *)base;
+ size_t ret;
+
+ ret = ZSTD_compressCCtx(zstd->zctx, out, outsize, in, size,
+ zstd->level);
+
+ if (ZSTD_isError(ret)) {
+ fprintf(stderr, "internal error in ZSTD compressor: %s\n",
+ ZSTD_getErrorName(ret));
+ return -1;
+ }
+
+ return ret < size ? ret : 0;
+}
+
+static ssize_t zstd_uncomp_block(compressor_t *base, const uint8_t *in,
+ size_t size, uint8_t *out, size_t outsize)
+{
+ size_t ret;
+ (void)base;
+
+ ret = ZSTD_decompress(out, outsize, in, size);
+
+ if (ZSTD_isError(ret)) {
+ fprintf(stderr, "error uncompressing ZSTD compressed data: %s",
+ ZSTD_getErrorName(ret));
+ return -1;
+ }
+
+ return ret;
+}
+
+static compressor_t *zstd_create_copy(compressor_t *cmp)
+{
+ zstd_compressor_t *zstd = malloc(sizeof(*zstd));
+
+ if (zstd == NULL) {
+ perror("creating additional zstd compressor");
+ return NULL;
+ }
+
+ memcpy(zstd, cmp, sizeof(*zstd));
+
+ zstd->zctx = ZSTD_createCCtx();
+
+ if (zstd->zctx == NULL) {
+ fputs("error creating addtional zstd compression context\n",
+ stderr);
+ free(zstd);
+ return NULL;
+ }
+
+ return (compressor_t *)zstd;
+}
+
+static void zstd_destroy(compressor_t *base)
+{
+ zstd_compressor_t *zstd = (zstd_compressor_t *)base;
+
+ ZSTD_freeCCtx(zstd->zctx);
+ free(zstd);
+}
+
+compressor_t *create_zstd_compressor(bool compress, size_t block_size,
+ char *options)
+{
+ zstd_compressor_t *zstd = calloc(1, sizeof(*zstd));
+ compressor_t *base = (compressor_t *)zstd;
+ size_t i;
+ (void)block_size;
+
+ if (zstd == NULL) {
+ perror("creating zstd compressor");
+ return NULL;
+ }
+
+ zstd->level = ZSTD_DEFAULT_COMPRESSION_LEVEL;
+
+ if (options != NULL) {
+ if (strncmp(options, "level=", 6) == 0) {
+ options += 6;
+
+ for (i = 0; isdigit(options[i]); ++i)
+ ;
+
+ if (i == 0 || options[i] != '\0' || i > 6)
+ goto fail_level;
+
+ zstd->level = atoi(options);
+
+ if (zstd->level < 1 || zstd->level > ZSTD_maxCLevel())
+ goto fail_level;
+ } else {
+ goto fail_opt;
+ }
+ }
+
+ zstd->zctx = ZSTD_createCCtx();
+ if (zstd->zctx == NULL) {
+ fputs("error creating zstd compression context\n", stderr);
+ free(zstd);
+ return NULL;
+ }
+
+ base->destroy = zstd_destroy;
+ base->do_block = compress ? zstd_comp_block : zstd_uncomp_block;
+ base->write_options = zstd_write_options;
+ base->read_options = zstd_read_options;
+ base->create_copy = zstd_create_copy;
+ return base;
+fail_level:
+ fprintf(stderr, "zstd compression level must be a number in the range "
+ "1...%d\n", ZSTD_maxCLevel());
+ free(zstd);
+ return NULL;
+fail_opt:
+ fputs("Unsupported extra options for zstd compressor\n", stderr);
+ free(zstd);
+ return NULL;
+}
+
+void compressor_zstd_print_help(void)
+{
+ printf("Available options for zstd compressor:\n"
+ "\n"
+ " level=<value> Set compression level. Defaults to %d.\n"
+ " Maximum is %d.\n"
+ "\n",
+ ZSTD_DEFAULT_COMPRESSION_LEVEL, ZSTD_maxCLevel());
+}
generated by cgit v1.2.3 (git 2.39.1) at 2024-11-01 09:34:41 +0000