From 9b3d958fb7c37855a63ed75707281c61dc1d44c4 Mon Sep 17 00:00:00 2001 From: David Oberhollenzer Date: Sat, 31 Aug 2019 16:22:31 +0200 Subject: Merge libcompress.a into libsquashfs.a Signed-off-by: David Oberhollenzer --- difftool/Makemodule.am | 2 +- lib/Makemodule.am | 50 ++-- lib/comp/block_processor.c | 72 ------ lib/comp/block_processor_parallel.c | 340 ------------------------- lib/comp/compressor.c | 183 -------------- lib/comp/create_block.c | 37 --- lib/comp/gzip.c | 412 ------------------------------- lib/comp/internal.h | 44 ---- lib/comp/lz4.c | 159 ------------ lib/comp/lzo.c | 321 ------------------------ lib/comp/process_block.c | 38 --- lib/comp/xz.c | 360 --------------------------- lib/comp/zstd.c | 188 -------------- lib/sqfs/comp/block_processor.c | 72 ++++++ lib/sqfs/comp/block_processor_parallel.c | 340 +++++++++++++++++++++++++ lib/sqfs/comp/compressor.c | 183 ++++++++++++++ lib/sqfs/comp/create_block.c | 37 +++ lib/sqfs/comp/gzip.c | 412 +++++++++++++++++++++++++++++++ lib/sqfs/comp/internal.h | 44 ++++ lib/sqfs/comp/lz4.c | 159 ++++++++++++ lib/sqfs/comp/lzo.c | 321 ++++++++++++++++++++++++ lib/sqfs/comp/process_block.c | 38 +++ lib/sqfs/comp/xz.c | 360 +++++++++++++++++++++++++++ lib/sqfs/comp/zstd.c | 188 ++++++++++++++ mkfs/Makemodule.am | 2 +- tar/Makemodule.am | 4 +- tests/Makemodule.am | 2 +- unpack/Makemodule.am | 2 +- 28 files changed, 2185 insertions(+), 2185 deletions(-) delete mode 100644 lib/comp/block_processor.c delete mode 100644 lib/comp/block_processor_parallel.c delete mode 100644 lib/comp/compressor.c delete mode 100644 lib/comp/create_block.c delete mode 100644 lib/comp/gzip.c delete mode 100644 lib/comp/internal.h delete mode 100644 lib/comp/lz4.c delete mode 100644 lib/comp/lzo.c delete mode 100644 lib/comp/process_block.c delete mode 100644 lib/comp/xz.c delete mode 100644 lib/comp/zstd.c create mode 100644 lib/sqfs/comp/block_processor.c create mode 100644 lib/sqfs/comp/block_processor_parallel.c create mode 100644 lib/sqfs/comp/compressor.c create mode 100644 lib/sqfs/comp/create_block.c create mode 100644 lib/sqfs/comp/gzip.c create mode 100644 lib/sqfs/comp/internal.h create mode 100644 lib/sqfs/comp/lz4.c create mode 100644 lib/sqfs/comp/lzo.c create mode 100644 lib/sqfs/comp/process_block.c create mode 100644 lib/sqfs/comp/xz.c create mode 100644 lib/sqfs/comp/zstd.c diff --git a/difftool/Makemodule.am b/difftool/Makemodule.am index 9b6e713..075dbf9 100644 --- a/difftool/Makemodule.am +++ b/difftool/Makemodule.am @@ -2,7 +2,7 @@ sqfsdiff_SOURCES = difftool/sqfsdiff.c difftool/sqfsdiff.h difftool/util.c sqfsdiff_SOURCES += difftool/compare_dir.c difftool/node_compare.c sqfsdiff_SOURCES += difftool/compare_files.c difftool/super.c sqfsdiff_SOURCES += difftool/extract.c difftool/options.c -sqfsdiff_LDADD = libsquashfs.a libfstree.a libcompress.a libutil.a +sqfsdiff_LDADD = libsquashfs.a libfstree.a libutil.a sqfsdiff_LDADD += $(XZ_LIBS) $(ZLIB_LIBS) $(LZO_LIBS) $(LZ4_LIBS) $(ZSTD_LIBS) bin_PROGRAMS += sqfsdiff diff --git a/lib/Makemodule.am b/lib/Makemodule.am index fb9d613..dee2b8d 100644 --- a/lib/Makemodule.am +++ b/lib/Makemodule.am @@ -17,12 +17,6 @@ libtar_a_SOURCES += include/tar.h libtar_a_CFLAGS = $(AM_CFLAGS) libtar_a_CPPFLAGS = $(AM_CPPFLAGS) -libcompress_a_SOURCES = lib/comp/compressor.c lib/comp/internal.h -libcompress_a_SOURCES += include/block_processor.h include/compress.h -libcompress_a_SOURCES += lib/comp/create_block.c lib/comp/process_block.c -libcompress_a_CFLAGS = $(AM_CFLAGS) -libcompress_a_CPPFLAGS = $(AM_CPPFLAGS) - libsquashfs_a_SOURCES = include/meta_writer.h include/squashfs.h libsquashfs_a_SOURCES += lib/sqfs/meta_writer.c lib/sqfs/super.c libsquashfs_a_SOURCES += lib/sqfs/id_table.c include/id_table.h @@ -41,6 +35,12 @@ libsquashfs_a_SOURCES += include/data_reader.h lib/sqfs/data_reader.c libsquashfs_a_SOURCES += lib/sqfs/write_export_table.c libsquashfs_a_SOURCES += lib/sqfs/read_table.c lib/sqfs/statistics.c libsquashfs_a_SOURCES += lib/sqfs/sqfs_reader.c +libsquashfs_a_SOURCES += lib/sqfs/comp/compressor.c lib/sqfs/comp/internal.h +libsquashfs_a_SOURCES += include/block_processor.h include/compress.h +libsquashfs_a_SOURCES += lib/sqfs/comp/create_block.c +libsquashfs_a_SOURCES += lib/sqfs/comp/process_block.c +libsquashfs_a_CFLAGS = $(AM_CFLAGS) +libsquashfs_a_CPPFLAGS = $(AM_CPPFLAGS) libutil_a_SOURCES = lib/util/canonicalize_name.c lib/util/write_data.c libutil_a_SOURCES += lib/util/read_data.c include/util.h @@ -53,48 +53,48 @@ libutil_a_CFLAGS = $(AM_CFLAGS) libutil_a_CPPFLAGS = $(AM_CPPFLAGS) if HAVE_PTHREAD -libcompress_a_SOURCES += lib/comp/block_processor_parallel.c -libcompress_a_CFLAGS += $(PTHREAD_CFLAGS) +libsquashfs_a_SOURCES += lib/sqfs/comp/block_processor_parallel.c +libsquashfs_a_CFLAGS += $(PTHREAD_CFLAGS) else -libcompress_a_SOURCES += lib/comp/block_processor.c +libsquashfs_a_SOURCES += lib/sqfs/comp/block_processor.c endif if WITH_GZIP -libcompress_a_SOURCES += lib/comp/gzip.c +libsquashfs_a_SOURCES += lib/sqfs/comp/gzip.c -libcompress_a_CFLAGS += $(ZLIB_CFLAGS) -libcompress_a_CPPFLAGS += -DWITH_GZIP +libsquashfs_a_CFLAGS += $(ZLIB_CFLAGS) +libsquashfs_a_CPPFLAGS += -DWITH_GZIP libutil_a_CFLAGS += $(ZLIB_CFLAGS) libutil_a_CPPFLAGS += -DWITH_GZIP endif if WITH_XZ -libcompress_a_SOURCES += lib/comp/xz.c +libsquashfs_a_SOURCES += lib/sqfs/comp/xz.c -libcompress_a_CFLAGS += $(XZ_CFLAGS) -libcompress_a_CPPFLAGS += -DWITH_XZ +libsquashfs_a_CFLAGS += $(XZ_CFLAGS) +libsquashfs_a_CPPFLAGS += -DWITH_XZ endif if WITH_LZO -libcompress_a_SOURCES += lib/comp/lzo.c +libsquashfs_a_SOURCES += lib/sqfs/comp/lzo.c -libcompress_a_CFLAGS += $(LZO_CFLAGS) -libcompress_a_CPPFLAGS += -DWITH_LZO +libsquashfs_a_CFLAGS += $(LZO_CFLAGS) +libsquashfs_a_CPPFLAGS += -DWITH_LZO endif if WITH_LZ4 -libcompress_a_SOURCES += lib/comp/lz4.c +libsquashfs_a_SOURCES += lib/sqfs/comp/lz4.c -libcompress_a_CFLAGS += $(LZ4_CFLAGS) -libcompress_a_CPPFLAGS += -DWITH_LZ4 +libsquashfs_a_CFLAGS += $(LZ4_CFLAGS) +libsquashfs_a_CPPFLAGS += -DWITH_LZ4 endif if WITH_ZSTD -libcompress_a_SOURCES += lib/comp/zstd.c +libsquashfs_a_SOURCES += lib/sqfs/comp/zstd.c -libcompress_a_CFLAGS += $(ZSTD_CFLAGS) -libcompress_a_CPPFLAGS += -DWITH_ZSTD +libsquashfs_a_CFLAGS += $(ZSTD_CFLAGS) +libsquashfs_a_CPPFLAGS += -DWITH_ZSTD endif if WITH_SELINUX @@ -104,4 +104,4 @@ libfstree_a_CFLAGS += $(LIBSELINUX_CFLAGS) libfstree_a_CPPFLAGS += -DWITH_SELINUX endif -noinst_LIBRARIES += libfstree.a libcompress.a libutil.a libsquashfs.a libtar.a +noinst_LIBRARIES += libfstree.a libutil.a libsquashfs.a libtar.a diff --git a/lib/comp/block_processor.c b/lib/comp/block_processor.c deleted file mode 100644 index 06dc384..0000000 --- a/lib/comp/block_processor.c +++ /dev/null @@ -1,72 +0,0 @@ -/* SPDX-License-Identifier: GPL-3.0-or-later */ -/* - * block_processor.c - * - * Copyright (C) 2019 David Oberhollenzer - */ -#include "config.h" - -#include "block_processor.h" -#include "util.h" - -#include -#include -#include - -struct block_processor_t { - size_t max_block_size; - compressor_t *cmp; - block_cb cb; - void *user; - int status; - - uint8_t scratch[]; -}; - -block_processor_t *block_processor_create(size_t max_block_size, - compressor_t *cmp, - unsigned int num_workers, - void *user, - block_cb callback) -{ - block_processor_t *proc = alloc_flex(sizeof(*proc), 1, max_block_size); - (void)num_workers; - - if (proc == NULL) { - perror("Creating block processor"); - return NULL; - } - - proc->max_block_size = max_block_size; - proc->cmp = cmp; - proc->cb = callback; - proc->user = user; - return proc; -} - -void block_processor_destroy(block_processor_t *proc) -{ - free(proc); -} - -int block_processor_enqueue(block_processor_t *proc, block_t *block) -{ - if (process_block(block, proc->cmp, - proc->scratch, proc->max_block_size)) - goto fail; - - if (proc->cb(proc->user, block)) - goto fail; - - free(block); - return 0; -fail: - free(block); - proc->status = -1; - return -1; -} - -int block_processor_finish(block_processor_t *proc) -{ - return proc->status; -} diff --git a/lib/comp/block_processor_parallel.c b/lib/comp/block_processor_parallel.c deleted file mode 100644 index b58ad5c..0000000 --- a/lib/comp/block_processor_parallel.c +++ /dev/null @@ -1,340 +0,0 @@ -/* SPDX-License-Identifier: GPL-3.0-or-later */ -/* - * block_processor.c - * - * Copyright (C) 2019 David Oberhollenzer - */ -#include "config.h" - -#include "block_processor.h" -#include "util.h" - -#include -#include -#include -#include - -#define MAX_BACKLOG_FACTOR (10) - -typedef struct { - block_processor_t *shared; - compressor_t *cmp; - pthread_t thread; - uint8_t scratch[]; -} compress_worker_t; - -struct block_processor_t { - pthread_mutex_t mtx; - pthread_cond_t queue_cond; - pthread_cond_t done_cond; - - /* needs rw access by worker and main thread */ - block_t *queue; - block_t *queue_last; - - block_t *done; - bool terminate; - size_t backlog; - - /* used by main thread only */ - uint32_t enqueue_id; - uint32_t dequeue_id; - - unsigned int num_workers; - block_cb cb; - void *user; - int status; - - /* used only by workers */ - size_t max_block_size; - - compress_worker_t *workers[]; -}; - -static void store_completed_block(block_processor_t *shared, block_t *blk) -{ - block_t *it = shared->done, *prev = NULL; - - while (it != NULL) { - if (it->sequence_number >= blk->sequence_number) - break; - prev = it; - it = it->next; - } - - if (prev == NULL) { - blk->next = shared->done; - shared->done = blk; - } else { - blk->next = prev->next; - prev->next = blk; - } -} - -static void *worker_proc(void *arg) -{ - compress_worker_t *worker = arg; - block_processor_t *shared = worker->shared; - block_t *blk = NULL; - - for (;;) { - pthread_mutex_lock(&shared->mtx); - if (blk != NULL) { - store_completed_block(shared, blk); - shared->backlog -= 1; - pthread_cond_broadcast(&shared->done_cond); - } - - while (shared->queue == NULL && !shared->terminate) { - pthread_cond_wait(&shared->queue_cond, - &shared->mtx); - } - - if (shared->terminate) { - pthread_mutex_unlock(&shared->mtx); - break; - } - - blk = shared->queue; - shared->queue = blk->next; - blk->next = NULL; - - if (shared->queue == NULL) - shared->queue_last = NULL; - pthread_mutex_unlock(&shared->mtx); - - if (process_block(blk, worker->cmp, worker->scratch, - shared->max_block_size)) { - blk->flags |= BLK_COMPRESS_ERROR; - } - } - return NULL; -} - -block_processor_t *block_processor_create(size_t max_block_size, - compressor_t *cmp, - unsigned int num_workers, - void *user, - block_cb callback) -{ - block_processor_t *proc; - unsigned int i; - int ret; - - if (num_workers < 1) - num_workers = 1; - - proc = alloc_flex(sizeof(*proc), - sizeof(proc->workers[0]), num_workers); - if (proc == NULL) { - perror("Creating block processor"); - return NULL; - } - - proc->max_block_size = max_block_size; - proc->cb = callback; - proc->user = user; - proc->num_workers = num_workers; - - if (pthread_mutex_init(&proc->mtx, NULL)) { - perror("Creating block processor mutex"); - goto fail_free; - } - - if (pthread_cond_init(&proc->queue_cond, NULL)) { - perror("Creating block processor conditional"); - goto fail_mtx; - } - - if (pthread_cond_init(&proc->done_cond, NULL)) { - perror("Creating block processor completion conditional"); - goto fail_cond; - } - - for (i = 0; i < num_workers; ++i) { - proc->workers[i] = alloc_flex(sizeof(compress_worker_t), - 1, max_block_size); - - if (proc->workers[i] == NULL) { - perror("Creating block worker data"); - goto fail_init; - } - - proc->workers[i]->shared = proc; - proc->workers[i]->cmp = cmp->create_copy(cmp); - - if (proc->workers[i]->cmp == NULL) - goto fail_init; - } - - for (i = 0; i < num_workers; ++i) { - ret = pthread_create(&proc->workers[i]->thread, NULL, - worker_proc, proc->workers[i]); - - if (ret != 0) { - perror("Creating block processor thread"); - goto fail_thread; - } - } - - return proc; -fail_thread: - pthread_mutex_lock(&proc->mtx); - proc->terminate = true; - pthread_cond_broadcast(&proc->queue_cond); - pthread_mutex_unlock(&proc->mtx); - - for (i = 0; i < num_workers; ++i) { - if (proc->workers[i]->thread > 0) { - pthread_join(proc->workers[i]->thread, NULL); - } - } -fail_init: - for (i = 0; i < num_workers; ++i) { - if (proc->workers[i] != NULL) { - if (proc->workers[i]->cmp != NULL) { - proc->workers[i]->cmp-> - destroy(proc->workers[i]->cmp); - } - - free(proc->workers[i]); - } - } - pthread_cond_destroy(&proc->done_cond); -fail_cond: - pthread_cond_destroy(&proc->queue_cond); -fail_mtx: - pthread_mutex_destroy(&proc->mtx); -fail_free: - free(proc); - return NULL; -} - -void block_processor_destroy(block_processor_t *proc) -{ - unsigned int i; - block_t *blk; - - pthread_mutex_lock(&proc->mtx); - proc->terminate = true; - pthread_cond_broadcast(&proc->queue_cond); - pthread_mutex_unlock(&proc->mtx); - - for (i = 0; i < proc->num_workers; ++i) { - pthread_join(proc->workers[i]->thread, NULL); - - proc->workers[i]->cmp->destroy(proc->workers[i]->cmp); - free(proc->workers[i]); - } - - pthread_cond_destroy(&proc->done_cond); - pthread_cond_destroy(&proc->queue_cond); - pthread_mutex_destroy(&proc->mtx); - - while (proc->queue != NULL) { - blk = proc->queue; - proc->queue = blk->next; - free(blk); - } - - while (proc->done != NULL) { - blk = proc->done; - proc->done = blk->next; - free(blk); - } - - free(proc); -} - -static int process_completed_blocks(block_processor_t *proc, block_t *queue) -{ - block_t *it; - - while (queue != NULL) { - it = queue; - queue = queue->next; - - if (it->flags & BLK_COMPRESS_ERROR) { - proc->status = -1; - } else { - if (proc->cb(proc->user, it)) - proc->status = -1; - } - - free(it); - } - - return proc->status; -} - -int block_processor_enqueue(block_processor_t *proc, block_t *block) -{ - block_t *queue = NULL, *it, *prev; - - block->sequence_number = proc->enqueue_id++; - block->next = NULL; - - pthread_mutex_lock(&proc->mtx); - if ((block->flags & BLK_DONT_COMPRESS) && - (block->flags & BLK_DONT_CHECKSUM)) { - store_completed_block(proc, block); - } else { - while (proc->backlog > proc->num_workers * MAX_BACKLOG_FACTOR) - pthread_cond_wait(&proc->done_cond, &proc->mtx); - - if (proc->queue_last == NULL) { - proc->queue = proc->queue_last = block; - } else { - proc->queue_last->next = block; - proc->queue_last = block; - } - - proc->backlog += 1; - } - - it = proc->done; - prev = NULL; - - while (it != NULL && it->sequence_number == proc->dequeue_id) { - prev = it; - it = it->next; - proc->dequeue_id += 1; - } - - if (prev != NULL) { - queue = proc->done; - prev->next = NULL; - proc->done = it; - } - - pthread_cond_broadcast(&proc->queue_cond); - pthread_mutex_unlock(&proc->mtx); - - return process_completed_blocks(proc, queue); -} - -int block_processor_finish(block_processor_t *proc) -{ - block_t *queue, *it; - - pthread_mutex_lock(&proc->mtx); - while (proc->backlog > 0) - pthread_cond_wait(&proc->done_cond, &proc->mtx); - - for (it = proc->done; it != NULL; it = it->next) { - if (it->sequence_number != proc->dequeue_id++) { - pthread_mutex_unlock(&proc->mtx); - goto bug_seqnum; - } - } - - queue = proc->done; - proc->done = NULL; - pthread_mutex_unlock(&proc->mtx); - - return process_completed_blocks(proc, queue); -bug_seqnum: - fputs("[BUG][parallel block processor] " - "gap in sequence numbers!\n", stderr); - return -1; -} diff --git a/lib/comp/compressor.c b/lib/comp/compressor.c deleted file mode 100644 index e59d948..0000000 --- a/lib/comp/compressor.c +++ /dev/null @@ -1,183 +0,0 @@ -/* SPDX-License-Identifier: GPL-3.0-or-later */ -/* - * compressor.c - * - * Copyright (C) 2019 David Oberhollenzer - */ -#include "config.h" - -#include -#include -#include - -#include "internal.h" -#include "util.h" - -typedef compressor_t *(*compressor_fun_t)(bool compress, size_t block_size, - char *options); - -typedef void (*compressor_help_fun_t)(void); - -static compressor_fun_t compressors[SQFS_COMP_MAX + 1] = { -#ifdef WITH_GZIP - [SQFS_COMP_GZIP] = create_gzip_compressor, -#endif -#ifdef WITH_XZ - [SQFS_COMP_XZ] = create_xz_compressor, -#endif -#ifdef WITH_LZO - [SQFS_COMP_LZO] = create_lzo_compressor, -#endif -#ifdef WITH_LZ4 - [SQFS_COMP_LZ4] = create_lz4_compressor, -#endif -#ifdef WITH_ZSTD - [SQFS_COMP_ZSTD] = create_zstd_compressor, -#endif -}; - -static const compressor_help_fun_t helpfuns[SQFS_COMP_MAX + 1] = { -#ifdef WITH_GZIP - [SQFS_COMP_GZIP] = compressor_gzip_print_help, -#endif -#ifdef WITH_XZ - [SQFS_COMP_XZ] = compressor_xz_print_help, -#endif -#ifdef WITH_LZO - [SQFS_COMP_LZO] = compressor_lzo_print_help, -#endif -#ifdef WITH_LZ4 - [SQFS_COMP_LZ4] = compressor_lz4_print_help, -#endif -#ifdef WITH_ZSTD - [SQFS_COMP_ZSTD] = compressor_zstd_print_help, -#endif -}; - -static const char *names[] = { - [SQFS_COMP_GZIP] = "gzip", - [SQFS_COMP_LZMA] = "lzma", - [SQFS_COMP_LZO] = "lzo", - [SQFS_COMP_XZ] = "xz", - [SQFS_COMP_LZ4] = "lz4", - [SQFS_COMP_ZSTD] = "zstd", -}; - -int generic_write_options(int fd, const void *data, size_t size) -{ - uint8_t buffer[size + 2]; - - *((uint16_t *)buffer) = htole16(0x8000 | size); - memcpy(buffer + 2, data, size); - - if (write_data("writing compressor options", - fd, buffer, sizeof(buffer))) { - return -1; - } - - return sizeof(buffer); -} - -int generic_read_options(int fd, void *data, size_t size) -{ - uint8_t buffer[size + 2]; - - if (read_data_at("reading compressor options", sizeof(sqfs_super_t), - fd, buffer, sizeof(buffer))) { - return -1; - } - - if (le16toh(*((uint16_t *)buffer)) != (0x8000 | size)) { - fputs("reading compressor options: invalid meta data header\n", - stderr); - return -1; - } - - memcpy(data, buffer + 2, size); - return 0; -} - -bool compressor_exists(E_SQFS_COMPRESSOR id) -{ - if (id < SQFS_COMP_MIN || id > SQFS_COMP_MAX) - return false; - - return (compressors[id] != NULL); -} - -compressor_t *compressor_create(E_SQFS_COMPRESSOR id, bool compress, - size_t block_size, char *options) -{ - if (id < SQFS_COMP_MIN || id > SQFS_COMP_MAX) - return NULL; - - if (compressors[id] == NULL) - return NULL; - - return compressors[id](compress, block_size, options); -} - -void compressor_print_help(E_SQFS_COMPRESSOR id) -{ - if (id < SQFS_COMP_MIN || id > SQFS_COMP_MAX) - return; - - if (compressors[id] == NULL) - return; - - helpfuns[id](); -} - -void compressor_print_available(void) -{ - size_t i; - - fputs("Available compressors:\n", stdout); - - for (i = 0; i < sizeof(names) / sizeof(names[0]); ++i) { - if (compressor_exists(i)) - printf("\t%s\n", names[i]); - } - - printf("\nDefault compressor: %s\n", names[compressor_get_default()]); -} - -const char *compressor_name_from_id(E_SQFS_COMPRESSOR id) -{ - if (id < 0 || (size_t)id >= sizeof(names) / sizeof(names[0])) - return NULL; - - return names[id]; -} - -int compressor_id_from_name(const char *name, E_SQFS_COMPRESSOR *out) -{ - size_t i; - - for (i = 0; i < sizeof(names) / sizeof(names[0]); ++i) { - if (names[i] != NULL && strcmp(names[i], name) == 0) { - *out = i; - return 0; - } - } - - return -1; -} - -E_SQFS_COMPRESSOR compressor_get_default(void) -{ -#if defined(WITH_XZ) - return SQFS_COMP_XZ; -#elif defined(WITH_ZSTD) - return SQFS_COMP_ZSTD; -#elif defined(WITH_GZIP) - return SQFS_COMP_GZIP; -#elif defined(WITH_LZO) - return SQFS_COMP_LZO; -#elif defined(WITH_LZ4) - return SQFS_COMP_LZ4; -#else - fputs("No compressor implementation available!\n", stderr); - exit(EXIT_FAILURE); -#endif -} diff --git a/lib/comp/create_block.c b/lib/comp/create_block.c deleted file mode 100644 index 90344bb..0000000 --- a/lib/comp/create_block.c +++ /dev/null @@ -1,37 +0,0 @@ -/* SPDX-License-Identifier: GPL-3.0-or-later */ -/* - * create_block.c - * - * Copyright (C) 2019 David Oberhollenzer - */ -#include "config.h" - -#include "block_processor.h" -#include "util.h" - -#include -#include -#include - -block_t *create_block(const char *filename, int fd, size_t size, - void *user, uint32_t flags) -{ - block_t *blk = alloc_flex(sizeof(*blk), 1, size); - - if (blk == NULL) { - perror(filename); - return NULL; - } - - if (fd >= 0) { - if (read_data(filename, fd, blk->data, size)) { - free(blk); - return NULL; - } - } - - blk->size = size; - blk->user = user; - blk->flags = flags; - return blk; -} diff --git a/lib/comp/gzip.c b/lib/comp/gzip.c deleted file mode 100644 index 1fdc051..0000000 --- a/lib/comp/gzip.c +++ /dev/null @@ -1,412 +0,0 @@ -/* SPDX-License-Identifier: GPL-3.0-or-later */ -/* - * gzip.c - * - * Copyright (C) 2019 David Oberhollenzer - */ -#include "config.h" - -#include -#include -#include -#include -#include -#include - -#include "internal.h" - -#define GZIP_DEFAULT_LEVEL 9 -#define GZIP_DEFAULT_WINDOW 15 -#define GZIP_NUM_STRATEGIES (sizeof(strategies) / sizeof(strategies[0])) - -typedef enum { - GZIP_STRATEGY_DEFAULT = 0x01, - GZIP_STRATEGY_FILTERED = 0x02, - GZIP_STRATEGY_HUFFMAN = 0x04, - GZIP_STARTEGY_RLE = 0x08, - GZIP_STRATEGY_FIXED = 0x10, - - GZIP_ALL_STRATEGIES = 0x1F, -} GZIP_STRATEGIES; - -static const struct { - const char *name; - int flag; - int zlib; -} strategies[] = { - { "default", GZIP_STRATEGY_DEFAULT, Z_DEFAULT_STRATEGY }, - { "filtered", GZIP_STRATEGY_FILTERED, Z_FILTERED }, - { "huffman", GZIP_STRATEGY_HUFFMAN, Z_HUFFMAN_ONLY }, - { "rle", GZIP_STARTEGY_RLE, Z_RLE }, - { "fixed", GZIP_STRATEGY_FIXED, Z_FIXED }, -}; - -typedef struct { - uint32_t level; - uint16_t window; - uint16_t strategies; -} gzip_options_t; - -typedef struct { - compressor_t base; - - z_stream strm; - bool compress; - - size_t block_size; - gzip_options_t opt; -} gzip_compressor_t; - -static void gzip_destroy(compressor_t *base) -{ - gzip_compressor_t *gzip = (gzip_compressor_t *)base; - - if (gzip->compress) { - deflateEnd(&gzip->strm); - } else { - inflateEnd(&gzip->strm); - } - - free(gzip); -} - -static int gzip_write_options(compressor_t *base, int fd) -{ - gzip_compressor_t *gzip = (gzip_compressor_t *)base; - gzip_options_t opt; - - if (gzip->opt.level == GZIP_DEFAULT_LEVEL && - gzip->opt.window == GZIP_DEFAULT_WINDOW && - gzip->opt.strategies == 0) { - return 0; - } - - opt.level = htole32(gzip->opt.level); - opt.window = htole16(gzip->opt.window); - opt.strategies = htole16(gzip->opt.strategies); - - return generic_write_options(fd, &opt, sizeof(opt)); -} - -static int gzip_read_options(compressor_t *base, int fd) -{ - gzip_compressor_t *gzip = (gzip_compressor_t *)base; - gzip_options_t opt; - - if (generic_read_options(fd, &opt, sizeof(opt))) - return -1; - - gzip->opt.level = le32toh(opt.level); - gzip->opt.window = le16toh(opt.window); - gzip->opt.strategies = le16toh(opt.strategies); - - if (gzip->opt.level < 1 || gzip->opt.level > 9) { - fprintf(stderr, "Invalid gzip compression level '%d'.\n", - gzip->opt.level); - return -1; - } - - if (gzip->opt.window < 8 || gzip->opt.window > 15) { - fprintf(stderr, "Invalid gzip window size '%d'.\n", - gzip->opt.window); - return -1; - } - - if (gzip->opt.strategies & ~GZIP_ALL_STRATEGIES) { - fputs("Unknown gzip strategies selected.\n", stderr); - return -1; - } - - return 0; -} - -static int find_strategy(gzip_compressor_t *gzip, const uint8_t *in, - size_t size, uint8_t *out, size_t outsize) -{ - int ret, selected = Z_DEFAULT_STRATEGY; - size_t i, length, minlength = 0; - - for (i = 0; i < GZIP_NUM_STRATEGIES; ++i) { - if (!(strategies[i].flag & gzip->opt.strategies)) - continue; - - ret = deflateReset(&gzip->strm); - if (ret != Z_OK) { - fputs("resetting zlib stream failed\n", - stderr); - return -1; - } - - gzip->strm.next_in = (void *)in; - gzip->strm.avail_in = size; - gzip->strm.next_out = out; - gzip->strm.avail_out = outsize; - - ret = deflateParams(&gzip->strm, gzip->opt.level, - strategies[i].zlib); - if (ret != Z_OK) { - fputs("setting deflate parameters failed\n", - stderr); - return -1; - } - - ret = deflate(&gzip->strm, Z_FINISH); - - if (ret == Z_STREAM_END) { - length = gzip->strm.total_out; - - if (minlength == 0 || length < minlength) { - minlength = length; - selected = strategies[i].zlib; - } - } else if (ret != Z_OK && ret != Z_BUF_ERROR) { - fputs("gzip block processing failed\n", stderr); - return -1; - } - } - - return selected; -} - -static ssize_t gzip_do_block(compressor_t *base, const uint8_t *in, - size_t size, uint8_t *out, size_t outsize) -{ - gzip_compressor_t *gzip = (gzip_compressor_t *)base; - int ret, strategy = 0; - size_t written; - - if (gzip->compress && gzip->opt.strategies != 0) { - strategy = find_strategy(gzip, in, size, out, outsize); - if (strategy < 0) - return -1; - } - - if (gzip->compress) { - ret = deflateReset(&gzip->strm); - } else { - ret = inflateReset(&gzip->strm); - } - - if (ret != Z_OK) { - fputs("resetting zlib stream failed\n", stderr); - return -1; - } - - gzip->strm.next_in = (void *)in; - gzip->strm.avail_in = size; - gzip->strm.next_out = out; - gzip->strm.avail_out = outsize; - - if (gzip->compress && gzip->opt.strategies != 0) { - ret = deflateParams(&gzip->strm, gzip->opt.level, strategy); - if (ret != Z_OK) { - fputs("setting selcted deflate parameters failed\n", - stderr); - return -1; - } - } - - if (gzip->compress) { - ret = deflate(&gzip->strm, Z_FINISH); - } else { - ret = inflate(&gzip->strm, Z_FINISH); - } - - if (ret == Z_STREAM_END) { - written = gzip->strm.total_out; - - if (gzip->compress && written >= size) - return 0; - - return (ssize_t)written; - } - - if (ret != Z_OK && ret != Z_BUF_ERROR) { - fputs("gzip block processing failed\n", stderr); - return -1; - } - - return 0; -} - -static int process_options(char *options, int *level, int *window, int *flags) -{ - enum { - OPT_WINDOW = 0, - OPT_LEVEL, - }; - char *const token[] = { - [OPT_WINDOW] = (char *)"window", - [OPT_LEVEL] = (char *)"level", - NULL - }; - char *subopts, *value; - size_t i; - int opt; - - subopts = options; - - while (*subopts != '\0') { - opt = getsubopt(&subopts, token, &value); - - switch (opt) { - case OPT_WINDOW: - if (value == NULL) - goto fail_value; - - for (i = 0; isdigit(value[i]); ++i) - ; - - if (i < 1 || i > 3 || value[i] != '\0') - goto fail_window; - - *window = atoi(value); - - if (*window < 8 || *window > 15) - goto fail_window; - break; - case OPT_LEVEL: - if (value == NULL) - goto fail_value; - - for (i = 0; isdigit(value[i]); ++i) - ; - - if (i < 1 || i > 3 || value[i] != '\0') - goto fail_level; - - *level = atoi(value); - - if (*level < 1 || *level > 9) - goto fail_level; - break; - default: - for (i = 0; i < GZIP_NUM_STRATEGIES; ++i) { - if (strcmp(value, strategies[i].name) == 0) { - *flags |= strategies[i].flag; - break; - } - } - if (i == GZIP_NUM_STRATEGIES) - goto fail_opt; - break; - } - } - - return 0; -fail_window: - fputs("Window size must be a number between 8 and 15.\n", stderr); - return -1; -fail_level: - fputs("Compression level must be a number between 1 and 9.\n", stderr); - return -1; -fail_opt: - fprintf(stderr, "Unknown option '%s'.\n", value); - return -1; -fail_value: - fprintf(stderr, "Missing value for '%s'.\n", token[opt]); - return -1; -} - -static compressor_t *gzip_create_copy(compressor_t *cmp) -{ - gzip_compressor_t *gzip = malloc(sizeof(*gzip)); - int ret; - - if (gzip == NULL) { - perror("creating additional gzip compressor"); - return NULL; - } - - memcpy(gzip, cmp, sizeof(*gzip)); - memset(&gzip->strm, 0, sizeof(gzip->strm)); - - if (gzip->compress) { - ret = deflateInit2(&gzip->strm, gzip->opt.level, Z_DEFLATED, - gzip->opt.window, 8, Z_DEFAULT_STRATEGY); - } else { - ret = inflateInit(&gzip->strm); - } - - if (ret != Z_OK) { - fputs("internal error creating additional zlib stream\n", - stderr); - free(gzip); - return NULL; - } - - return (compressor_t *)gzip; -} - -compressor_t *create_gzip_compressor(bool compress, size_t block_size, - char *options) -{ - int window = GZIP_DEFAULT_WINDOW; - int level = GZIP_DEFAULT_LEVEL; - gzip_compressor_t *gzip; - compressor_t *base; - int flags = 0; - int ret; - - if (options != NULL) { - if (process_options(options, &level, &window, &flags)) - return NULL; - } - - gzip = calloc(1, sizeof(*gzip)); - base = (compressor_t *)gzip; - - if (gzip == NULL) { - perror("creating gzip compressor"); - return NULL; - } - - gzip->opt.level = level; - gzip->opt.window = window; - gzip->opt.strategies = flags; - gzip->compress = compress; - gzip->block_size = block_size; - base->do_block = gzip_do_block; - base->destroy = gzip_destroy; - base->write_options = gzip_write_options; - base->read_options = gzip_read_options; - base->create_copy = gzip_create_copy; - - if (compress) { - ret = deflateInit2(&gzip->strm, level, Z_DEFLATED, window, 8, - Z_DEFAULT_STRATEGY); - } else { - ret = inflateInit(&gzip->strm); - } - - if (ret != Z_OK) { - fputs("internal error creating zlib stream\n", stderr); - free(gzip); - return NULL; - } - - return base; -} - -void compressor_gzip_print_help(void) -{ - size_t i; - - printf( -"Available options for gzip compressor:\n" -"\n" -" level= Compression level. Value from 1 to 9.\n" -" Defaults to %d.\n" -" window= Deflate compression window size. Value from 8 to 15.\n" -" Defaults to %d.\n" -"\n" -"In additon to the options, one or more strategies can be specified.\n" -"If multiple stratgies are provided, the one yielding the best compression\n" -"ratio will be used.\n" -"\n" -"The following strategies are available:\n", - GZIP_DEFAULT_LEVEL, GZIP_DEFAULT_WINDOW); - - for (i = 0; i < GZIP_NUM_STRATEGIES; ++i) - printf("\t%s\n", strategies[i].name); -} diff --git a/lib/comp/internal.h b/lib/comp/internal.h deleted file mode 100644 index 986266d..0000000 --- a/lib/comp/internal.h +++ /dev/null @@ -1,44 +0,0 @@ -/* SPDX-License-Identifier: GPL-3.0-or-later */ -/* - * internal.h - * - * Copyright (C) 2019 David Oberhollenzer - */ -#ifndef INTERNAL_H -#define INTERNAL_H - -#include "config.h" - -#include "compress.h" -#include "util.h" - -int generic_write_options(int fd, const void *data, size_t size); - -int generic_read_options(int fd, void *data, size_t size); - -compressor_t *create_xz_compressor(bool compress, size_t block_size, - char *options); - -compressor_t *create_gzip_compressor(bool compress, size_t block_size, - char *options); - -compressor_t *create_lzo_compressor(bool compress, size_t block_size, - char *options); - -compressor_t *create_lz4_compressor(bool compress, size_t block_size, - char *options); - -compressor_t *create_zstd_compressor(bool compress, size_t block_size, - char *options); - -void compressor_xz_print_help(void); - -void compressor_gzip_print_help(void); - -void compressor_lzo_print_help(void); - -void compressor_lz4_print_help(void); - -void compressor_zstd_print_help(void); - -#endif /* INTERNAL_H */ diff --git a/lib/comp/lz4.c b/lib/comp/lz4.c deleted file mode 100644 index abb6c5c..0000000 --- a/lib/comp/lz4.c +++ /dev/null @@ -1,159 +0,0 @@ -/* SPDX-License-Identifier: GPL-3.0-or-later */ -/* - * lz4.c - * - * Copyright (C) 2019 David Oberhollenzer - */ -#include "config.h" - -#include -#include -#include -#include - -#include -#include - -#include "internal.h" - -typedef struct { - compressor_t base; - bool high_compression; -} lz4_compressor_t; - -typedef struct { - uint32_t version; - uint32_t flags; -} lz4_options; - -#define LZ4LEGACY 1 -#define LZ4_FLAG_HC 0x01 - -static int lz4_write_options(compressor_t *base, int fd) -{ - lz4_compressor_t *lz4 = (lz4_compressor_t *)base; - lz4_options opt = { - .version = htole32(LZ4LEGACY), - .flags = htole32(lz4->high_compression ? LZ4_FLAG_HC : 0), - }; - - return generic_write_options(fd, &opt, sizeof(opt)); -} - -static int lz4_read_options(compressor_t *base, int fd) -{ - lz4_options opt; - (void)base; - - if (generic_read_options(fd, &opt, sizeof(opt))) - return -1; - - opt.version = le32toh(opt.version); - opt.flags = le32toh(opt.flags); - - if (opt.version != LZ4LEGACY) { - fprintf(stderr, "unsupported lz4 version '%d'\n", opt.version); - return -1; - } - - return 0; -} - -static ssize_t lz4_comp_block(compressor_t *base, const uint8_t *in, - size_t size, uint8_t *out, size_t outsize) -{ - lz4_compressor_t *lz4 = (lz4_compressor_t *)base; - int ret; - - if (lz4->high_compression) { - ret = LZ4_compress_HC((void *)in, (void *)out, - size, outsize, LZ4HC_CLEVEL_MAX); - } else { - ret = LZ4_compress_default((void *)in, (void *)out, - size, outsize); - } - - if (ret < 0) { - fputs("internal error in lz4 compressor\n", stderr); - return -1; - } - - return ret; -} - -static ssize_t lz4_uncomp_block(compressor_t *base, const uint8_t *in, - size_t size, uint8_t *out, size_t outsize) -{ - int ret; - (void)base; - - ret = LZ4_decompress_safe((void *)in, (void *)out, size, outsize); - - if (ret < 0) { - fputs("internal error in lz4 decompressor\n", stderr); - return -1; - } - - return ret; -} - -static compressor_t *lz4_create_copy(compressor_t *cmp) -{ - lz4_compressor_t *lz4 = malloc(sizeof(*lz4)); - - if (lz4 == NULL) { - perror("creating additional lz4 compressor"); - return NULL; - } - - memcpy(lz4, cmp, sizeof(*lz4)); - return (compressor_t *)lz4; -} - -static void lz4_destroy(compressor_t *base) -{ - free(base); -} - -compressor_t *create_lz4_compressor(bool compress, size_t block_size, - char *options) -{ - lz4_compressor_t *lz4 = calloc(1, sizeof(*lz4)); - compressor_t *base = (compressor_t *)lz4; - (void)block_size; - - if (lz4 == NULL) { - perror("creating lz4 compressor"); - return NULL; - } - - lz4->high_compression = false; - - if (options != NULL) { - if (strcmp(options, "hc") == 0) { - lz4->high_compression = true; - } else { - fputs("Unsupported extra options for lz4 " - "compressor.\n", stderr); - free(lz4); - return NULL; - } - } - - base->destroy = lz4_destroy; - base->do_block = compress ? lz4_comp_block : lz4_uncomp_block; - base->write_options = lz4_write_options; - base->read_options = lz4_read_options; - base->create_copy = lz4_create_copy; - return base; -} - -void compressor_lz4_print_help(void) -{ - fputs("Available options for lz4 compressor:\n" - "\n" - " hc If present, use slower but better compressing\n" - " variant of lz4.\n" - "\n", - stdout); -} diff --git a/lib/comp/lzo.c b/lib/comp/lzo.c deleted file mode 100644 index 09ef75c..0000000 --- a/lib/comp/lzo.c +++ /dev/null @@ -1,321 +0,0 @@ -/* SPDX-License-Identifier: GPL-3.0-or-later */ -/* - * lzo.c - * - * Copyright (C) 2019 David Oberhollenzer - */ -#include "config.h" - -#include -#include -#include -#include -#include - -#include - -#include "internal.h" - - -typedef enum { - LZO_ALGORITHM_LZO1X_1 = 0, - LZO_ALGORITHM_LZO1X_1_11 = 1, - LZO_ALGORITHM_LZO1X_1_12 = 2, - LZO_ALGORITHM_LZO1X_1_15 = 3, - LZO_ALGORITHM_LZO1X_999 = 4, -} LZO_ALGORITHM; - -#define LZO_DEFAULT_ALG LZO_ALGORITHM_LZO1X_999 -#define LZO_DEFAULT_LEVEL 8 -#define LZO_NUM_ALGS (sizeof(lzo_algs) / sizeof(lzo_algs[0])) - -typedef int (*lzo_cb_t)(const lzo_bytep src, lzo_uint src_len, lzo_bytep dst, - lzo_uintp dst_len, lzo_voidp wrkmem); - -static const struct { - const char *name; - lzo_cb_t compress; - size_t bufsize; -} lzo_algs[] = { - [LZO_ALGORITHM_LZO1X_1] = { - .name = "lzo1x_1", - .compress = lzo1x_1_compress, - .bufsize = LZO1X_1_MEM_COMPRESS, - }, - [LZO_ALGORITHM_LZO1X_1_11] = { - .name = "lzo1x_1_11", - .compress = lzo1x_1_11_compress, - .bufsize = LZO1X_1_11_MEM_COMPRESS, - }, - [LZO_ALGORITHM_LZO1X_1_12] = { - .name = "lzo1x_1_12", - .compress = lzo1x_1_12_compress, - .bufsize = LZO1X_1_12_MEM_COMPRESS, - }, - [LZO_ALGORITHM_LZO1X_1_15] = { - .name = "lzo1x_1_15", - .compress = lzo1x_1_15_compress, - .bufsize = LZO1X_1_15_MEM_COMPRESS, - }, - [LZO_ALGORITHM_LZO1X_999] = { - .name = "lzo1x_999", - .compress = lzo1x_999_compress, - .bufsize = LZO1X_999_MEM_COMPRESS, - }, -}; - -typedef struct { - compressor_t base; - int algorithm; - int level; - - uint8_t buffer[]; -} lzo_compressor_t; - -typedef struct { - uint32_t algorithm; - uint32_t level; -} lzo_options_t; - -static int lzo_write_options(compressor_t *base, int fd) -{ - lzo_compressor_t *lzo = (lzo_compressor_t *)base; - lzo_options_t opt; - - if (lzo->algorithm == LZO_DEFAULT_ALG && - lzo->level == LZO_DEFAULT_LEVEL) { - return 0; - } - - opt.algorithm = htole32(lzo->algorithm); - - if (lzo->algorithm == LZO_ALGORITHM_LZO1X_999) { - opt.level = htole32(lzo->level); - } else { - opt.level = 0; - } - - return generic_write_options(fd, &opt, sizeof(opt)); -} - -static int lzo_read_options(compressor_t *base, int fd) -{ - lzo_compressor_t *lzo = (lzo_compressor_t *)base; - lzo_options_t opt; - - if (generic_read_options(fd, &opt, sizeof(opt))) - return -1; - - lzo->algorithm = le32toh(opt.algorithm); - lzo->level = le32toh(opt.level); - - switch(lzo->algorithm) { - case LZO_ALGORITHM_LZO1X_1: - case LZO_ALGORITHM_LZO1X_1_11: - case LZO_ALGORITHM_LZO1X_1_12: - case LZO_ALGORITHM_LZO1X_1_15: - if (lzo->level != 0) - goto fail_level; - break; - case LZO_ALGORITHM_LZO1X_999: - if (lzo->level < 1 || lzo->level > 9) - goto fail_level; - break; - default: - fputs("Unsupported LZO variant specified.\n", stderr); - return -1; - } - - return 0; -fail_level: - fputs("Unsupported LZO compression level specified.\n", stderr); - return -1; -} - -static ssize_t lzo_comp_block(compressor_t *base, const uint8_t *in, - size_t size, uint8_t *out, size_t outsize) -{ - lzo_compressor_t *lzo = (lzo_compressor_t *)base; - lzo_uint len = outsize; - int ret; - - if (lzo->algorithm == LZO_ALGORITHM_LZO1X_999 && - lzo->level != LZO_DEFAULT_LEVEL) { - ret = lzo1x_999_compress_level(in, size, out, &len, - lzo->buffer, NULL, 0, 0, - lzo->level); - } else { - ret = lzo_algs[lzo->algorithm].compress(in, size, out, - &len, lzo->buffer); - } - - if (ret != LZO_E_OK) { - fputs("LZO compression failed.\n", stderr); - return -1; - } - - if (len < size) - return len; - - return 0; -} - -static ssize_t lzo_uncomp_block(compressor_t *base, const uint8_t *in, - size_t size, uint8_t *out, size_t outsize) -{ - lzo_compressor_t *lzo = (lzo_compressor_t *)base; - lzo_uint len = outsize; - int ret; - - ret = lzo1x_decompress_safe(in, size, out, &len, lzo->buffer); - - if (ret != LZO_E_OK) { - fputs("lzo decompress: input data is corrupted\n", stderr); - return -1; - } - - return len; -} - -static compressor_t *lzo_create_copy(compressor_t *cmp) -{ - lzo_compressor_t *other = (lzo_compressor_t *)cmp; - lzo_compressor_t *lzo; - - lzo = alloc_flex(sizeof(*lzo), 1, lzo_algs[other->algorithm].bufsize); - - if (lzo == NULL) { - perror("creating additional lzo compressor"); - return NULL; - } - - memcpy(lzo, other, sizeof(*lzo)); - return (compressor_t *)lzo; -} - -static void lzo_destroy(compressor_t *base) -{ - free(base); -} - -static int process_options(char *options, int *algorithm, int *level) -{ - enum { - OPT_ALG = 0, - OPT_LEVEL, - }; - char *const token[] = { - [OPT_ALG] = (char *)"algorithm", - [OPT_LEVEL] = (char *)"level", - NULL - }; - char *subopts, *value; - size_t i; - int opt; - - subopts = options; - - while (*subopts != '\0') { - opt = getsubopt(&subopts, token, &value); - - switch (opt) { - case OPT_ALG: - if (value == NULL) - goto fail_value; - - for (i = 0; i < LZO_NUM_ALGS; ++i) { - if (strcmp(lzo_algs[i].name, value) == 0) { - *algorithm = i; - break; - } - } - - if (i == LZO_NUM_ALGS) { - fprintf(stderr, "Unknown lzo variant '%s'.\n", - value); - return -1; - } - break; - case OPT_LEVEL: - if (value == NULL) - goto fail_value; - - for (i = 0; isdigit(value[i]); ++i) - ; - - if (i < 1 || i > 3 || value[i] != '\0') - goto fail_level; - - *level = atoi(value); - - if (*level < 1 || *level > 9) - goto fail_level; - break; - default: - goto fail_opt; - } - } - - return 0; -fail_level: - fputs("Compression level must be a number between 1 and 9.\n", stderr); - return -1; -fail_opt: - fprintf(stderr, "Unknown option '%s'.\n", value); - return -1; -fail_value: - fprintf(stderr, "Missing value for '%s'.\n", token[opt]); - return -1; -} - -compressor_t *create_lzo_compressor(bool compress, size_t block_size, - char *options) -{ - lzo_compressor_t *lzo; - compressor_t *base; - int level, alg; - (void)block_size; - - alg = LZO_DEFAULT_ALG; - level = LZO_DEFAULT_LEVEL; - - if (options != NULL && process_options(options, &alg, &level) != 0) - return NULL; - - lzo = alloc_flex(sizeof(*lzo), 1, lzo_algs[alg].bufsize); - base = (compressor_t *)lzo; - - if (lzo == NULL) { - perror("creating lzo compressor"); - return NULL; - } - - lzo->algorithm = alg; - lzo->level = level; - - base->destroy = lzo_destroy; - base->do_block = compress ? lzo_comp_block : lzo_uncomp_block; - base->write_options = lzo_write_options; - base->read_options = lzo_read_options; - base->create_copy = lzo_create_copy; - return base; -} - -void compressor_lzo_print_help(void) -{ - size_t i; - - fputs("Available options for lzo compressor:\n" - "\n" - " algorithm= Specify the variant of lzo to use.\n" - " Defaults to 'lzo1x_999'.\n" - " level= For lzo1x_999, the compression level.\n" - " Value from 1 to 9. Defaults to 8.\n" - " Ignored if algorithm is not lzo1x_999.\n" - "\n" - "Available algorithms:\n", - stdout); - - for (i = 0; i < LZO_NUM_ALGS; ++i) - printf("\t%s\n", lzo_algs[i].name); -} diff --git a/lib/comp/process_block.c b/lib/comp/process_block.c deleted file mode 100644 index 0fcbae0..0000000 --- a/lib/comp/process_block.c +++ /dev/null @@ -1,38 +0,0 @@ -/* SPDX-License-Identifier: GPL-3.0-or-later */ -/* - * process_block.c - * - * Copyright (C) 2019 David Oberhollenzer - */ -#include "config.h" - -#include "block_processor.h" -#include "util.h" - -#include -#include - -int process_block(block_t *block, compressor_t *cmp, - uint8_t *scratch, size_t scratch_size) -{ - ssize_t ret; - - if (!(block->flags & BLK_DONT_CHECKSUM)) - block->checksum = crc32(0, block->data, block->size); - - if (!(block->flags & BLK_DONT_COMPRESS)) { - ret = cmp->do_block(cmp, block->data, block->size, - scratch, scratch_size); - - if (ret < 0) - return -1; - - if (ret > 0) { - memcpy(block->data, scratch, ret); - block->size = ret; - block->flags |= BLK_IS_COMPRESSED; - } - } - - return 0; -} diff --git a/lib/comp/xz.c b/lib/comp/xz.c deleted file mode 100644 index d38aab6..0000000 --- a/lib/comp/xz.c +++ /dev/null @@ -1,360 +0,0 @@ -/* SPDX-License-Identifier: GPL-3.0-or-later */ -/* - * xz.c - * - * Copyright (C) 2019 David Oberhollenzer - */ -#include "config.h" - -#include -#include -#include -#include -#include -#include - -#include "internal.h" - -typedef enum { - XZ_FILTER_X86 = 0x01, - XZ_FILTER_POWERPC = 0x02, - XZ_FILTER_IA64 = 0x04, - XZ_FILTER_ARM = 0x08, - XZ_FILTER_ARMTHUMB = 0x10, - XZ_FILTER_SPARC = 0x20, - - XZ_FILTER_ALL = 0x3F, -} XZ_FILTER_FLAG; - -typedef struct { - compressor_t base; - size_t block_size; - size_t dict_size; - int flags; -} xz_compressor_t; - -typedef struct { - uint32_t dict_size; - uint32_t flags; -} xz_options_t; - -static const struct { - const char *name; - lzma_vli filter; - int flag; -} xz_filters[] = { - { "x86", LZMA_FILTER_X86, XZ_FILTER_X86 }, - { "powerpc", LZMA_FILTER_POWERPC, XZ_FILTER_POWERPC }, - { "ia64", LZMA_FILTER_IA64, XZ_FILTER_IA64 }, - { "arm", LZMA_FILTER_ARM, XZ_FILTER_ARM }, - { "armthumb", LZMA_FILTER_ARMTHUMB, XZ_FILTER_ARMTHUMB }, - { "sparc", LZMA_FILTER_SPARC, XZ_FILTER_SPARC }, -}; - -#define XZ_NUM_FILTERS (sizeof(xz_filters) / sizeof(xz_filters[0])) - -static int xz_write_options(compressor_t *base, int fd) -{ - xz_compressor_t *xz = (xz_compressor_t *)base; - xz_options_t opt; - - if (xz->flags == 0 && xz->dict_size == xz->block_size) - return 0; - - opt.dict_size = htole32(xz->dict_size); - opt.flags = htole32(xz->flags); - - return generic_write_options(fd, &opt, sizeof(opt)); -} - -static int xz_read_options(compressor_t *base, int fd) -{ - xz_compressor_t *xz = (xz_compressor_t *)base; - xz_options_t opt; - uint32_t mask; - - if (generic_read_options(fd, &opt, sizeof(opt))) - return -1; - - opt.dict_size = le32toh(opt.dict_size); - opt.flags = le32toh(opt.flags); - - mask = opt.dict_size & (opt.dict_size - 1); - - if (mask != 0 && ((mask & (mask - 1)) != 0)) { - fputs("Invalid lzma dictionary size.\n", stderr); - return -1; - } - - if (opt.flags & ~XZ_FILTER_ALL) { - fputs("Unknown BCJ filter used.\n", stderr); - return -1; - } - - xz->flags = opt.flags; - xz->dict_size = opt.dict_size; - return 0; -} - -static ssize_t compress(xz_compressor_t *xz, lzma_vli filter, - const uint8_t *in, size_t size, - uint8_t *out, size_t outsize) -{ - lzma_filter filters[5]; - lzma_options_lzma opt; - size_t written = 0; - lzma_ret ret; - int i = 0; - - if (lzma_lzma_preset(&opt, LZMA_PRESET_DEFAULT)) { - fputs("error initializing xz options\n", stderr); - return -1; - } - - opt.dict_size = xz->dict_size; - - if (filter != LZMA_VLI_UNKNOWN) { - filters[i].id = filter; - filters[i].options = NULL; - ++i; - } - - filters[i].id = LZMA_FILTER_LZMA2; - filters[i].options = &opt; - ++i; - - filters[i].id = LZMA_VLI_UNKNOWN; - filters[i].options = NULL; - ++i; - - ret = lzma_stream_buffer_encode(filters, LZMA_CHECK_CRC32, NULL, - in, size, out, &written, outsize); - - if (ret == LZMA_OK) - return (written >= size) ? 0 : written; - - if (ret != LZMA_BUF_ERROR) { - fputs("xz block compress failed\n", stderr); - return -1; - } - - return 0; -} - -static ssize_t xz_comp_block(compressor_t *base, const uint8_t *in, - size_t size, uint8_t *out, size_t outsize) -{ - xz_compressor_t *xz = (xz_compressor_t *)base; - lzma_vli selected = LZMA_VLI_UNKNOWN; - size_t i, smallest; - ssize_t ret; - - ret = compress(xz, LZMA_VLI_UNKNOWN, in, size, out, outsize); - if (ret < 0 || xz->flags == 0) - return ret; - - smallest = ret; - - for (i = 0; i < XZ_NUM_FILTERS; ++i) { - if (!(xz->flags & xz_filters[i].flag)) - continue; - - ret = compress(xz, xz_filters[i].filter, in, size, out, outsize); - if (ret < 0) - return -1; - - if (ret > 0 && (smallest == 0 || (size_t)ret < smallest)) { - smallest = ret; - selected = xz_filters[i].filter; - } - } - - if (smallest == 0) - return 0; - - return compress(xz, selected, in, size, out, outsize); -} - -static ssize_t xz_uncomp_block(compressor_t *base, const uint8_t *in, - size_t size, uint8_t *out, size_t outsize) -{ - uint64_t memlimit = 32 * 1024 * 1024; - size_t dest_pos = 0; - size_t src_pos = 0; - lzma_ret ret; - (void)base; - - ret = lzma_stream_buffer_decode(&memlimit, 0, NULL, - in, &src_pos, size, - out, &dest_pos, outsize); - - if (ret == LZMA_OK && size == src_pos) - return (ssize_t)dest_pos; - - fputs("xz block extract failed\n", stderr); - return -1; -} - -static compressor_t *xz_create_copy(compressor_t *cmp) -{ - xz_compressor_t *xz = malloc(sizeof(*xz)); - - if (xz == NULL) { - perror("creating additional xz compressor"); - return NULL; - } - - memcpy(xz, cmp, sizeof(*xz)); - return (compressor_t *)xz; -} - -static void xz_destroy(compressor_t *base) -{ - free(base); -} - -static int process_options(char *options, size_t blocksize, - int *flags, uint64_t *dictsize) -{ - enum { - OPT_DICT = 0, - }; - char *const token[] = { - [OPT_DICT] = (char *)"dictsize", - NULL - }; - char *subopts, *value; - uint64_t mask; - size_t i; - int opt; - - subopts = options; - - while (*subopts != '\0') { - opt = getsubopt(&subopts, token, &value); - - switch (opt) { - case OPT_DICT: - if (value == NULL) - goto fail_value; - - for (i = 0; isdigit(value[i]); ++i) - ; - - if (i < 1 || i > 9) - goto fail_dict; - - *dictsize = atol(value); - - switch (value[i]) { - case '\0': - break; - case 'm': - case 'M': - *dictsize <<= 20; - break; - case 'k': - case 'K': - *dictsize <<= 10; - break; - case '%': - *dictsize = ((*dictsize) * blocksize) / 100; - break; - default: - goto fail_dict; - } - - if (*dictsize > 0x0FFFFFFFFUL) - goto fail_dict_ov; - - mask = *dictsize & (*dictsize - 1); - - if (mask != 0 && ((mask & (mask - 1)) != 0)) - goto fail_dict_pot; - break; - default: - for (i = 0; i < XZ_NUM_FILTERS; ++i) { - if (strcmp(value, xz_filters[i].name) == 0) { - *flags |= xz_filters[i].flag; - break; - } - } - if (i == XZ_NUM_FILTERS) - goto fail_opt; - break; - } - } - - return 0; -fail_dict_pot: - fputs("dictionary size must be either 2^n or 2^n + 2^(n-1)\n", stderr); - return -1; -fail_dict_ov: - fputs("dictionary size too large.\n", stderr); - return -1; -fail_dict: - fputs("dictionary size must be a number with the optional " - "suffix 'm','k' or '%'.\n", stderr); - return -1; -fail_opt: - fprintf(stderr, "Unknown option '%s'.\n", value); - return -1; -fail_value: - fprintf(stderr, "Missing value for '%s'.\n", token[opt]); - return -1; -} - -compressor_t *create_xz_compressor(bool compress, size_t block_size, - char *options) -{ - uint64_t dictsize = block_size; - xz_compressor_t *xz; - compressor_t *base; - int flags = 0; - - if (options != NULL) { - if (process_options(options, block_size, &flags, &dictsize)) - return NULL; - } - - xz = calloc(1, sizeof(*xz)); - base = (compressor_t *)xz; - if (xz == NULL) { - perror("creating xz compressor"); - return NULL; - } - - xz->flags = flags; - xz->dict_size = dictsize; - xz->block_size = block_size; - base->destroy = xz_destroy; - base->do_block = compress ? xz_comp_block : xz_uncomp_block; - base->write_options = xz_write_options; - base->read_options = xz_read_options; - base->create_copy = xz_create_copy; - return base; -} - -void compressor_xz_print_help(void) -{ - size_t i; - - fputs( -"Available options for xz compressor:\n" -"\n" -" dictsize= Dictionary size. Either a value in bytes or a\n" -" percentage of the block size. Defaults to 100%.\n" -" The suffix '%' indicates a percentage. 'K' and 'M'\n" -" can also be used for kibi and mebi bytes\n" -" respecitively.\n" -"\n" -"In additon to the options, one or more bcj filters can be specified.\n" -"If multiple filters are provided, the one yielding the best compression\n" -"ratio will be used.\n" -"\n" -"The following filters are available:\n", - stdout); - - for (i = 0; i < XZ_NUM_FILTERS; ++i) - printf("\t%s\n", xz_filters[i].name); -} diff --git a/lib/comp/zstd.c b/lib/comp/zstd.c deleted file mode 100644 index e206338..0000000 --- a/lib/comp/zstd.c +++ /dev/null @@ -1,188 +0,0 @@ -/* SPDX-License-Identifier: GPL-3.0-or-later */ -/* - * zstd.c - * - * Copyright (C) 2019 David Oberhollenzer - */ -#include "config.h" - -#include -#include -#include -#include -#include - -#include - -#include "internal.h" - -#define ZSTD_DEFAULT_COMPRESSION_LEVEL 15 - -typedef struct { - compressor_t base; - ZSTD_CCtx *zctx; - int level; -} zstd_compressor_t; - -typedef struct { - uint32_t level; -} zstd_options_t; - -static int zstd_write_options(compressor_t *base, int fd) -{ - zstd_compressor_t *zstd = (zstd_compressor_t *)base; - zstd_options_t opt; - (void)fd; - - if (zstd->level == ZSTD_DEFAULT_COMPRESSION_LEVEL) - return 0; - - opt.level = htole32(zstd->level); - return generic_write_options(fd, &opt, sizeof(opt)); -} - -static int zstd_read_options(compressor_t *base, int fd) -{ - zstd_options_t opt; - (void)base; - - if (generic_read_options(fd, &opt, sizeof(opt))) - return -1; - - opt.level = le32toh(opt.level); - return 0; -} - -static ssize_t zstd_comp_block(compressor_t *base, const uint8_t *in, - size_t size, uint8_t *out, size_t outsize) -{ - zstd_compressor_t *zstd = (zstd_compressor_t *)base; - size_t ret; - - ret = ZSTD_compressCCtx(zstd->zctx, out, outsize, in, size, - zstd->level); - - if (ZSTD_isError(ret)) { - fprintf(stderr, "internal error in ZSTD compressor: %s\n", - ZSTD_getErrorName(ret)); - return -1; - } - - return ret < size ? ret : 0; -} - -static ssize_t zstd_uncomp_block(compressor_t *base, const uint8_t *in, - size_t size, uint8_t *out, size_t outsize) -{ - size_t ret; - (void)base; - - ret = ZSTD_decompress(out, outsize, in, size); - - if (ZSTD_isError(ret)) { - fprintf(stderr, "error uncompressing ZSTD compressed data: %s", - ZSTD_getErrorName(ret)); - return -1; - } - - return ret; -} - -static compressor_t *zstd_create_copy(compressor_t *cmp) -{ - zstd_compressor_t *zstd = malloc(sizeof(*zstd)); - - if (zstd == NULL) { - perror("creating additional zstd compressor"); - return NULL; - } - - memcpy(zstd, cmp, sizeof(*zstd)); - - zstd->zctx = ZSTD_createCCtx(); - - if (zstd->zctx == NULL) { - fputs("error creating addtional zstd compression context\n", - stderr); - free(zstd); - return NULL; - } - - return (compressor_t *)zstd; -} - -static void zstd_destroy(compressor_t *base) -{ - zstd_compressor_t *zstd = (zstd_compressor_t *)base; - - ZSTD_freeCCtx(zstd->zctx); - free(zstd); -} - -compressor_t *create_zstd_compressor(bool compress, size_t block_size, - char *options) -{ - zstd_compressor_t *zstd = calloc(1, sizeof(*zstd)); - compressor_t *base = (compressor_t *)zstd; - size_t i; - (void)block_size; - - if (zstd == NULL) { - perror("creating zstd compressor"); - return NULL; - } - - zstd->level = ZSTD_DEFAULT_COMPRESSION_LEVEL; - - if (options != NULL) { - if (strncmp(options, "level=", 6) == 0) { - options += 6; - - for (i = 0; isdigit(options[i]); ++i) - ; - - if (i == 0 || options[i] != '\0' || i > 6) - goto fail_level; - - zstd->level = atoi(options); - - if (zstd->level < 1 || zstd->level > ZSTD_maxCLevel()) - goto fail_level; - } else { - goto fail_opt; - } - } - - zstd->zctx = ZSTD_createCCtx(); - if (zstd->zctx == NULL) { - fputs("error creating zstd compression context\n", stderr); - free(zstd); - return NULL; - } - - base->destroy = zstd_destroy; - base->do_block = compress ? zstd_comp_block : zstd_uncomp_block; - base->write_options = zstd_write_options; - base->read_options = zstd_read_options; - base->create_copy = zstd_create_copy; - return base; -fail_level: - fprintf(stderr, "zstd compression level must be a number in the range " - "1...%d\n", ZSTD_maxCLevel()); - free(zstd); - return NULL; -fail_opt: - fputs("Unsupported extra options for zstd compressor\n", stderr); - free(zstd); - return NULL; -} - -void compressor_zstd_print_help(void) -{ - printf("Available options for zstd compressor:\n" - "\n" - " level= Set compression level. Defaults to %d.\n" - " Maximum is %d.\n" - "\n", - ZSTD_DEFAULT_COMPRESSION_LEVEL, ZSTD_maxCLevel()); -} diff --git a/lib/sqfs/comp/block_processor.c b/lib/sqfs/comp/block_processor.c new file mode 100644 index 0000000..06dc384 --- /dev/null +++ b/lib/sqfs/comp/block_processor.c @@ -0,0 +1,72 @@ +/* SPDX-License-Identifier: GPL-3.0-or-later */ +/* + * block_processor.c + * + * Copyright (C) 2019 David Oberhollenzer + */ +#include "config.h" + +#include "block_processor.h" +#include "util.h" + +#include +#include +#include + +struct block_processor_t { + size_t max_block_size; + compressor_t *cmp; + block_cb cb; + void *user; + int status; + + uint8_t scratch[]; +}; + +block_processor_t *block_processor_create(size_t max_block_size, + compressor_t *cmp, + unsigned int num_workers, + void *user, + block_cb callback) +{ + block_processor_t *proc = alloc_flex(sizeof(*proc), 1, max_block_size); + (void)num_workers; + + if (proc == NULL) { + perror("Creating block processor"); + return NULL; + } + + proc->max_block_size = max_block_size; + proc->cmp = cmp; + proc->cb = callback; + proc->user = user; + return proc; +} + +void block_processor_destroy(block_processor_t *proc) +{ + free(proc); +} + +int block_processor_enqueue(block_processor_t *proc, block_t *block) +{ + if (process_block(block, proc->cmp, + proc->scratch, proc->max_block_size)) + goto fail; + + if (proc->cb(proc->user, block)) + goto fail; + + free(block); + return 0; +fail: + free(block); + proc->status = -1; + return -1; +} + +int block_processor_finish(block_processor_t *proc) +{ + return proc->status; +} diff --git a/lib/sqfs/comp/block_processor_parallel.c b/lib/sqfs/comp/block_processor_parallel.c new file mode 100644 index 0000000..b58ad5c --- /dev/null +++ b/lib/sqfs/comp/block_processor_parallel.c @@ -0,0 +1,340 @@ +/* SPDX-License-Identifier: GPL-3.0-or-later */ +/* + * block_processor.c + * + * Copyright (C) 2019 David Oberhollenzer + */ +#include "config.h" + +#include "block_processor.h" +#include "util.h" + +#include +#include +#include +#include + +#define MAX_BACKLOG_FACTOR (10) + +typedef struct { + block_processor_t *shared; + compressor_t *cmp; + pthread_t thread; + uint8_t scratch[]; +} compress_worker_t; + +struct block_processor_t { + pthread_mutex_t mtx; + pthread_cond_t queue_cond; + pthread_cond_t done_cond; + + /* needs rw access by worker and main thread */ + block_t *queue; + block_t *queue_last; + + block_t *done; + bool terminate; + size_t backlog; + + /* used by main thread only */ + uint32_t enqueue_id; + uint32_t dequeue_id; + + unsigned int num_workers; + block_cb cb; + void *user; + int status; + + /* used only by workers */ + size_t max_block_size; + + compress_worker_t *workers[]; +}; + +static void store_completed_block(block_processor_t *shared, block_t *blk) +{ + block_t *it = shared->done, *prev = NULL; + + while (it != NULL) { + if (it->sequence_number >= blk->sequence_number) + break; + prev = it; + it = it->next; + } + + if (prev == NULL) { + blk->next = shared->done; + shared->done = blk; + } else { + blk->next = prev->next; + prev->next = blk; + } +} + +static void *worker_proc(void *arg) +{ + compress_worker_t *worker = arg; + block_processor_t *shared = worker->shared; + block_t *blk = NULL; + + for (;;) { + pthread_mutex_lock(&shared->mtx); + if (blk != NULL) { + store_completed_block(shared, blk); + shared->backlog -= 1; + pthread_cond_broadcast(&shared->done_cond); + } + + while (shared->queue == NULL && !shared->terminate) { + pthread_cond_wait(&shared->queue_cond, + &shared->mtx); + } + + if (shared->terminate) { + pthread_mutex_unlock(&shared->mtx); + break; + } + + blk = shared->queue; + shared->queue = blk->next; + blk->next = NULL; + + if (shared->queue == NULL) + shared->queue_last = NULL; + pthread_mutex_unlock(&shared->mtx); + + if (process_block(blk, worker->cmp, worker->scratch, + shared->max_block_size)) { + blk->flags |= BLK_COMPRESS_ERROR; + } + } + return NULL; +} + +block_processor_t *block_processor_create(size_t max_block_size, + compressor_t *cmp, + unsigned int num_workers, + void *user, + block_cb callback) +{ + block_processor_t *proc; + unsigned int i; + int ret; + + if (num_workers < 1) + num_workers = 1; + + proc = alloc_flex(sizeof(*proc), + sizeof(proc->workers[0]), num_workers); + if (proc == NULL) { + perror("Creating block processor"); + return NULL; + } + + proc->max_block_size = max_block_size; + proc->cb = callback; + proc->user = user; + proc->num_workers = num_workers; + + if (pthread_mutex_init(&proc->mtx, NULL)) { + perror("Creating block processor mutex"); + goto fail_free; + } + + if (pthread_cond_init(&proc->queue_cond, NULL)) { + perror("Creating block processor conditional"); + goto fail_mtx; + } + + if (pthread_cond_init(&proc->done_cond, NULL)) { + perror("Creating block processor completion conditional"); + goto fail_cond; + } + + for (i = 0; i < num_workers; ++i) { + proc->workers[i] = alloc_flex(sizeof(compress_worker_t), + 1, max_block_size); + + if (proc->workers[i] == NULL) { + perror("Creating block worker data"); + goto fail_init; + } + + proc->workers[i]->shared = proc; + proc->workers[i]->cmp = cmp->create_copy(cmp); + + if (proc->workers[i]->cmp == NULL) + goto fail_init; + } + + for (i = 0; i < num_workers; ++i) { + ret = pthread_create(&proc->workers[i]->thread, NULL, + worker_proc, proc->workers[i]); + + if (ret != 0) { + perror("Creating block processor thread"); + goto fail_thread; + } + } + + return proc; +fail_thread: + pthread_mutex_lock(&proc->mtx); + proc->terminate = true; + pthread_cond_broadcast(&proc->queue_cond); + pthread_mutex_unlock(&proc->mtx); + + for (i = 0; i < num_workers; ++i) { + if (proc->workers[i]->thread > 0) { + pthread_join(proc->workers[i]->thread, NULL); + } + } +fail_init: + for (i = 0; i < num_workers; ++i) { + if (proc->workers[i] != NULL) { + if (proc->workers[i]->cmp != NULL) { + proc->workers[i]->cmp-> + destroy(proc->workers[i]->cmp); + } + + free(proc->workers[i]); + } + } + pthread_cond_destroy(&proc->done_cond); +fail_cond: + pthread_cond_destroy(&proc->queue_cond); +fail_mtx: + pthread_mutex_destroy(&proc->mtx); +fail_free: + free(proc); + return NULL; +} + +void block_processor_destroy(block_processor_t *proc) +{ + unsigned int i; + block_t *blk; + + pthread_mutex_lock(&proc->mtx); + proc->terminate = true; + pthread_cond_broadcast(&proc->queue_cond); + pthread_mutex_unlock(&proc->mtx); + + for (i = 0; i < proc->num_workers; ++i) { + pthread_join(proc->workers[i]->thread, NULL); + + proc->workers[i]->cmp->destroy(proc->workers[i]->cmp); + free(proc->workers[i]); + } + + pthread_cond_destroy(&proc->done_cond); + pthread_cond_destroy(&proc->queue_cond); + pthread_mutex_destroy(&proc->mtx); + + while (proc->queue != NULL) { + blk = proc->queue; + proc->queue = blk->next; + free(blk); + } + + while (proc->done != NULL) { + blk = proc->done; + proc->done = blk->next; + free(blk); + } + + free(proc); +} + +static int process_completed_blocks(block_processor_t *proc, block_t *queue) +{ + block_t *it; + + while (queue != NULL) { + it = queue; + queue = queue->next; + + if (it->flags & BLK_COMPRESS_ERROR) { + proc->status = -1; + } else { + if (proc->cb(proc->user, it)) + proc->status = -1; + } + + free(it); + } + + return proc->status; +} + +int block_processor_enqueue(block_processor_t *proc, block_t *block) +{ + block_t *queue = NULL, *it, *prev; + + block->sequence_number = proc->enqueue_id++; + block->next = NULL; + + pthread_mutex_lock(&proc->mtx); + if ((block->flags & BLK_DONT_COMPRESS) && + (block->flags & BLK_DONT_CHECKSUM)) { + store_completed_block(proc, block); + } else { + while (proc->backlog > proc->num_workers * MAX_BACKLOG_FACTOR) + pthread_cond_wait(&proc->done_cond, &proc->mtx); + + if (proc->queue_last == NULL) { + proc->queue = proc->queue_last = block; + } else { + proc->queue_last->next = block; + proc->queue_last = block; + } + + proc->backlog += 1; + } + + it = proc->done; + prev = NULL; + + while (it != NULL && it->sequence_number == proc->dequeue_id) { + prev = it; + it = it->next; + proc->dequeue_id += 1; + } + + if (prev != NULL) { + queue = proc->done; + prev->next = NULL; + proc->done = it; + } + + pthread_cond_broadcast(&proc->queue_cond); + pthread_mutex_unlock(&proc->mtx); + + return process_completed_blocks(proc, queue); +} + +int block_processor_finish(block_processor_t *proc) +{ + block_t *queue, *it; + + pthread_mutex_lock(&proc->mtx); + while (proc->backlog > 0) + pthread_cond_wait(&proc->done_cond, &proc->mtx); + + for (it = proc->done; it != NULL; it = it->next) { + if (it->sequence_number != proc->dequeue_id++) { + pthread_mutex_unlock(&proc->mtx); + goto bug_seqnum; + } + } + + queue = proc->done; + proc->done = NULL; + pthread_mutex_unlock(&proc->mtx); + + return process_completed_blocks(proc, queue); +bug_seqnum: + fputs("[BUG][parallel block processor] " + "gap in sequence numbers!\n", stderr); + return -1; +} diff --git a/lib/sqfs/comp/compressor.c b/lib/sqfs/comp/compressor.c new file mode 100644 index 0000000..e59d948 --- /dev/null +++ b/lib/sqfs/comp/compressor.c @@ -0,0 +1,183 @@ +/* SPDX-License-Identifier: GPL-3.0-or-later */ +/* + * compressor.c + * + * Copyright (C) 2019 David Oberhollenzer + */ +#include "config.h" + +#include +#include +#include + +#include "internal.h" +#include "util.h" + +typedef compressor_t *(*compressor_fun_t)(bool compress, size_t block_size, + char *options); + +typedef void (*compressor_help_fun_t)(void); + +static compressor_fun_t compressors[SQFS_COMP_MAX + 1] = { +#ifdef WITH_GZIP + [SQFS_COMP_GZIP] = create_gzip_compressor, +#endif +#ifdef WITH_XZ + [SQFS_COMP_XZ] = create_xz_compressor, +#endif +#ifdef WITH_LZO + [SQFS_COMP_LZO] = create_lzo_compressor, +#endif +#ifdef WITH_LZ4 + [SQFS_COMP_LZ4] = create_lz4_compressor, +#endif +#ifdef WITH_ZSTD + [SQFS_COMP_ZSTD] = create_zstd_compressor, +#endif +}; + +static const compressor_help_fun_t helpfuns[SQFS_COMP_MAX + 1] = { +#ifdef WITH_GZIP + [SQFS_COMP_GZIP] = compressor_gzip_print_help, +#endif +#ifdef WITH_XZ + [SQFS_COMP_XZ] = compressor_xz_print_help, +#endif +#ifdef WITH_LZO + [SQFS_COMP_LZO] = compressor_lzo_print_help, +#endif +#ifdef WITH_LZ4 + [SQFS_COMP_LZ4] = compressor_lz4_print_help, +#endif +#ifdef WITH_ZSTD + [SQFS_COMP_ZSTD] = compressor_zstd_print_help, +#endif +}; + +static const char *names[] = { + [SQFS_COMP_GZIP] = "gzip", + [SQFS_COMP_LZMA] = "lzma", + [SQFS_COMP_LZO] = "lzo", + [SQFS_COMP_XZ] = "xz", + [SQFS_COMP_LZ4] = "lz4", + [SQFS_COMP_ZSTD] = "zstd", +}; + +int generic_write_options(int fd, const void *data, size_t size) +{ + uint8_t buffer[size + 2]; + + *((uint16_t *)buffer) = htole16(0x8000 | size); + memcpy(buffer + 2, data, size); + + if (write_data("writing compressor options", + fd, buffer, sizeof(buffer))) { + return -1; + } + + return sizeof(buffer); +} + +int generic_read_options(int fd, void *data, size_t size) +{ + uint8_t buffer[size + 2]; + + if (read_data_at("reading compressor options", sizeof(sqfs_super_t), + fd, buffer, sizeof(buffer))) { + return -1; + } + + if (le16toh(*((uint16_t *)buffer)) != (0x8000 | size)) { + fputs("reading compressor options: invalid meta data header\n", + stderr); + return -1; + } + + memcpy(data, buffer + 2, size); + return 0; +} + +bool compressor_exists(E_SQFS_COMPRESSOR id) +{ + if (id < SQFS_COMP_MIN || id > SQFS_COMP_MAX) + return false; + + return (compressors[id] != NULL); +} + +compressor_t *compressor_create(E_SQFS_COMPRESSOR id, bool compress, + size_t block_size, char *options) +{ + if (id < SQFS_COMP_MIN || id > SQFS_COMP_MAX) + return NULL; + + if (compressors[id] == NULL) + return NULL; + + return compressors[id](compress, block_size, options); +} + +void compressor_print_help(E_SQFS_COMPRESSOR id) +{ + if (id < SQFS_COMP_MIN || id > SQFS_COMP_MAX) + return; + + if (compressors[id] == NULL) + return; + + helpfuns[id](); +} + +void compressor_print_available(void) +{ + size_t i; + + fputs("Available compressors:\n", stdout); + + for (i = 0; i < sizeof(names) / sizeof(names[0]); ++i) { + if (compressor_exists(i)) + printf("\t%s\n", names[i]); + } + + printf("\nDefault compressor: %s\n", names[compressor_get_default()]); +} + +const char *compressor_name_from_id(E_SQFS_COMPRESSOR id) +{ + if (id < 0 || (size_t)id >= sizeof(names) / sizeof(names[0])) + return NULL; + + return names[id]; +} + +int compressor_id_from_name(const char *name, E_SQFS_COMPRESSOR *out) +{ + size_t i; + + for (i = 0; i < sizeof(names) / sizeof(names[0]); ++i) { + if (names[i] != NULL && strcmp(names[i], name) == 0) { + *out = i; + return 0; + } + } + + return -1; +} + +E_SQFS_COMPRESSOR compressor_get_default(void) +{ +#if defined(WITH_XZ) + return SQFS_COMP_XZ; +#elif defined(WITH_ZSTD) + return SQFS_COMP_ZSTD; +#elif defined(WITH_GZIP) + return SQFS_COMP_GZIP; +#elif defined(WITH_LZO) + return SQFS_COMP_LZO; +#elif defined(WITH_LZ4) + return SQFS_COMP_LZ4; +#else + fputs("No compressor implementation available!\n", stderr); + exit(EXIT_FAILURE); +#endif +} diff --git a/lib/sqfs/comp/create_block.c b/lib/sqfs/comp/create_block.c new file mode 100644 index 0000000..90344bb --- /dev/null +++ b/lib/sqfs/comp/create_block.c @@ -0,0 +1,37 @@ +/* SPDX-License-Identifier: GPL-3.0-or-later */ +/* + * create_block.c + * + * Copyright (C) 2019 David Oberhollenzer + */ +#include "config.h" + +#include "block_processor.h" +#include "util.h" + +#include +#include +#include + +block_t *create_block(const char *filename, int fd, size_t size, + void *user, uint32_t flags) +{ + block_t *blk = alloc_flex(sizeof(*blk), 1, size); + + if (blk == NULL) { + perror(filename); + return NULL; + } + + if (fd >= 0) { + if (read_data(filename, fd, blk->data, size)) { + free(blk); + return NULL; + } + } + + blk->size = size; + blk->user = user; + blk->flags = flags; + return blk; +} diff --git a/lib/sqfs/comp/gzip.c b/lib/sqfs/comp/gzip.c new file mode 100644 index 0000000..1fdc051 --- /dev/null +++ b/lib/sqfs/comp/gzip.c @@ -0,0 +1,412 @@ +/* SPDX-License-Identifier: GPL-3.0-or-later */ +/* + * gzip.c + * + * Copyright (C) 2019 David Oberhollenzer + */ +#include "config.h" + +#include +#include +#include +#include +#include +#include + +#include "internal.h" + +#define GZIP_DEFAULT_LEVEL 9 +#define GZIP_DEFAULT_WINDOW 15 +#define GZIP_NUM_STRATEGIES (sizeof(strategies) / sizeof(strategies[0])) + +typedef enum { + GZIP_STRATEGY_DEFAULT = 0x01, + GZIP_STRATEGY_FILTERED = 0x02, + GZIP_STRATEGY_HUFFMAN = 0x04, + GZIP_STARTEGY_RLE = 0x08, + GZIP_STRATEGY_FIXED = 0x10, + + GZIP_ALL_STRATEGIES = 0x1F, +} GZIP_STRATEGIES; + +static const struct { + const char *name; + int flag; + int zlib; +} strategies[] = { + { "default", GZIP_STRATEGY_DEFAULT, Z_DEFAULT_STRATEGY }, + { "filtered", GZIP_STRATEGY_FILTERED, Z_FILTERED }, + { "huffman", GZIP_STRATEGY_HUFFMAN, Z_HUFFMAN_ONLY }, + { "rle", GZIP_STARTEGY_RLE, Z_RLE }, + { "fixed", GZIP_STRATEGY_FIXED, Z_FIXED }, +}; + +typedef struct { + uint32_t level; + uint16_t window; + uint16_t strategies; +} gzip_options_t; + +typedef struct { + compressor_t base; + + z_stream strm; + bool compress; + + size_t block_size; + gzip_options_t opt; +} gzip_compressor_t; + +static void gzip_destroy(compressor_t *base) +{ + gzip_compressor_t *gzip = (gzip_compressor_t *)base; + + if (gzip->compress) { + deflateEnd(&gzip->strm); + } else { + inflateEnd(&gzip->strm); + } + + free(gzip); +} + +static int gzip_write_options(compressor_t *base, int fd) +{ + gzip_compressor_t *gzip = (gzip_compressor_t *)base; + gzip_options_t opt; + + if (gzip->opt.level == GZIP_DEFAULT_LEVEL && + gzip->opt.window == GZIP_DEFAULT_WINDOW && + gzip->opt.strategies == 0) { + return 0; + } + + opt.level = htole32(gzip->opt.level); + opt.window = htole16(gzip->opt.window); + opt.strategies = htole16(gzip->opt.strategies); + + return generic_write_options(fd, &opt, sizeof(opt)); +} + +static int gzip_read_options(compressor_t *base, int fd) +{ + gzip_compressor_t *gzip = (gzip_compressor_t *)base; + gzip_options_t opt; + + if (generic_read_options(fd, &opt, sizeof(opt))) + return -1; + + gzip->opt.level = le32toh(opt.level); + gzip->opt.window = le16toh(opt.window); + gzip->opt.strategies = le16toh(opt.strategies); + + if (gzip->opt.level < 1 || gzip->opt.level > 9) { + fprintf(stderr, "Invalid gzip compression level '%d'.\n", + gzip->opt.level); + return -1; + } + + if (gzip->opt.window < 8 || gzip->opt.window > 15) { + fprintf(stderr, "Invalid gzip window size '%d'.\n", + gzip->opt.window); + return -1; + } + + if (gzip->opt.strategies & ~GZIP_ALL_STRATEGIES) { + fputs("Unknown gzip strategies selected.\n", stderr); + return -1; + } + + return 0; +} + +static int find_strategy(gzip_compressor_t *gzip, const uint8_t *in, + size_t size, uint8_t *out, size_t outsize) +{ + int ret, selected = Z_DEFAULT_STRATEGY; + size_t i, length, minlength = 0; + + for (i = 0; i < GZIP_NUM_STRATEGIES; ++i) { + if (!(strategies[i].flag & gzip->opt.strategies)) + continue; + + ret = deflateReset(&gzip->strm); + if (ret != Z_OK) { + fputs("resetting zlib stream failed\n", + stderr); + return -1; + } + + gzip->strm.next_in = (void *)in; + gzip->strm.avail_in = size; + gzip->strm.next_out = out; + gzip->strm.avail_out = outsize; + + ret = deflateParams(&gzip->strm, gzip->opt.level, + strategies[i].zlib); + if (ret != Z_OK) { + fputs("setting deflate parameters failed\n", + stderr); + return -1; + } + + ret = deflate(&gzip->strm, Z_FINISH); + + if (ret == Z_STREAM_END) { + length = gzip->strm.total_out; + + if (minlength == 0 || length < minlength) { + minlength = length; + selected = strategies[i].zlib; + } + } else if (ret != Z_OK && ret != Z_BUF_ERROR) { + fputs("gzip block processing failed\n", stderr); + return -1; + } + } + + return selected; +} + +static ssize_t gzip_do_block(compressor_t *base, const uint8_t *in, + size_t size, uint8_t *out, size_t outsize) +{ + gzip_compressor_t *gzip = (gzip_compressor_t *)base; + int ret, strategy = 0; + size_t written; + + if (gzip->compress && gzip->opt.strategies != 0) { + strategy = find_strategy(gzip, in, size, out, outsize); + if (strategy < 0) + return -1; + } + + if (gzip->compress) { + ret = deflateReset(&gzip->strm); + } else { + ret = inflateReset(&gzip->strm); + } + + if (ret != Z_OK) { + fputs("resetting zlib stream failed\n", stderr); + return -1; + } + + gzip->strm.next_in = (void *)in; + gzip->strm.avail_in = size; + gzip->strm.next_out = out; + gzip->strm.avail_out = outsize; + + if (gzip->compress && gzip->opt.strategies != 0) { + ret = deflateParams(&gzip->strm, gzip->opt.level, strategy); + if (ret != Z_OK) { + fputs("setting selcted deflate parameters failed\n", + stderr); + return -1; + } + } + + if (gzip->compress) { + ret = deflate(&gzip->strm, Z_FINISH); + } else { + ret = inflate(&gzip->strm, Z_FINISH); + } + + if (ret == Z_STREAM_END) { + written = gzip->strm.total_out; + + if (gzip->compress && written >= size) + return 0; + + return (ssize_t)written; + } + + if (ret != Z_OK && ret != Z_BUF_ERROR) { + fputs("gzip block processing failed\n", stderr); + return -1; + } + + return 0; +} + +static int process_options(char *options, int *level, int *window, int *flags) +{ + enum { + OPT_WINDOW = 0, + OPT_LEVEL, + }; + char *const token[] = { + [OPT_WINDOW] = (char *)"window", + [OPT_LEVEL] = (char *)"level", + NULL + }; + char *subopts, *value; + size_t i; + int opt; + + subopts = options; + + while (*subopts != '\0') { + opt = getsubopt(&subopts, token, &value); + + switch (opt) { + case OPT_WINDOW: + if (value == NULL) + goto fail_value; + + for (i = 0; isdigit(value[i]); ++i) + ; + + if (i < 1 || i > 3 || value[i] != '\0') + goto fail_window; + + *window = atoi(value); + + if (*window < 8 || *window > 15) + goto fail_window; + break; + case OPT_LEVEL: + if (value == NULL) + goto fail_value; + + for (i = 0; isdigit(value[i]); ++i) + ; + + if (i < 1 || i > 3 || value[i] != '\0') + goto fail_level; + + *level = atoi(value); + + if (*level < 1 || *level > 9) + goto fail_level; + break; + default: + for (i = 0; i < GZIP_NUM_STRATEGIES; ++i) { + if (strcmp(value, strategies[i].name) == 0) { + *flags |= strategies[i].flag; + break; + } + } + if (i == GZIP_NUM_STRATEGIES) + goto fail_opt; + break; + } + } + + return 0; +fail_window: + fputs("Window size must be a number between 8 and 15.\n", stderr); + return -1; +fail_level: + fputs("Compression level must be a number between 1 and 9.\n", stderr); + return -1; +fail_opt: + fprintf(stderr, "Unknown option '%s'.\n", value); + return -1; +fail_value: + fprintf(stderr, "Missing value for '%s'.\n", token[opt]); + return -1; +} + +static compressor_t *gzip_create_copy(compressor_t *cmp) +{ + gzip_compressor_t *gzip = malloc(sizeof(*gzip)); + int ret; + + if (gzip == NULL) { + perror("creating additional gzip compressor"); + return NULL; + } + + memcpy(gzip, cmp, sizeof(*gzip)); + memset(&gzip->strm, 0, sizeof(gzip->strm)); + + if (gzip->compress) { + ret = deflateInit2(&gzip->strm, gzip->opt.level, Z_DEFLATED, + gzip->opt.window, 8, Z_DEFAULT_STRATEGY); + } else { + ret = inflateInit(&gzip->strm); + } + + if (ret != Z_OK) { + fputs("internal error creating additional zlib stream\n", + stderr); + free(gzip); + return NULL; + } + + return (compressor_t *)gzip; +} + +compressor_t *create_gzip_compressor(bool compress, size_t block_size, + char *options) +{ + int window = GZIP_DEFAULT_WINDOW; + int level = GZIP_DEFAULT_LEVEL; + gzip_compressor_t *gzip; + compressor_t *base; + int flags = 0; + int ret; + + if (options != NULL) { + if (process_options(options, &level, &window, &flags)) + return NULL; + } + + gzip = calloc(1, sizeof(*gzip)); + base = (compressor_t *)gzip; + + if (gzip == NULL) { + perror("creating gzip compressor"); + return NULL; + } + + gzip->opt.level = level; + gzip->opt.window = window; + gzip->opt.strategies = flags; + gzip->compress = compress; + gzip->block_size = block_size; + base->do_block = gzip_do_block; + base->destroy = gzip_destroy; + base->write_options = gzip_write_options; + base->read_options = gzip_read_options; + base->create_copy = gzip_create_copy; + + if (compress) { + ret = deflateInit2(&gzip->strm, level, Z_DEFLATED, window, 8, + Z_DEFAULT_STRATEGY); + } else { + ret = inflateInit(&gzip->strm); + } + + if (ret != Z_OK) { + fputs("internal error creating zlib stream\n", stderr); + free(gzip); + return NULL; + } + + return base; +} + +void compressor_gzip_print_help(void) +{ + size_t i; + + printf( +"Available options for gzip compressor:\n" +"\n" +" level= Compression level. Value from 1 to 9.\n" +" Defaults to %d.\n" +" window= Deflate compression window size. Value from 8 to 15.\n" +" Defaults to %d.\n" +"\n" +"In additon to the options, one or more strategies can be specified.\n" +"If multiple stratgies are provided, the one yielding the best compression\n" +"ratio will be used.\n" +"\n" +"The following strategies are available:\n", + GZIP_DEFAULT_LEVEL, GZIP_DEFAULT_WINDOW); + + for (i = 0; i < GZIP_NUM_STRATEGIES; ++i) + printf("\t%s\n", strategies[i].name); +} diff --git a/lib/sqfs/comp/internal.h b/lib/sqfs/comp/internal.h new file mode 100644 index 0000000..986266d --- /dev/null +++ b/lib/sqfs/comp/internal.h @@ -0,0 +1,44 @@ +/* SPDX-License-Identifier: GPL-3.0-or-later */ +/* + * internal.h + * + * Copyright (C) 2019 David Oberhollenzer + */ +#ifndef INTERNAL_H +#define INTERNAL_H + +#include "config.h" + +#include "compress.h" +#include "util.h" + +int generic_write_options(int fd, const void *data, size_t size); + +int generic_read_options(int fd, void *data, size_t size); + +compressor_t *create_xz_compressor(bool compress, size_t block_size, + char *options); + +compressor_t *create_gzip_compressor(bool compress, size_t block_size, + char *options); + +compressor_t *create_lzo_compressor(bool compress, size_t block_size, + char *options); + +compressor_t *create_lz4_compressor(bool compress, size_t block_size, + char *options); + +compressor_t *create_zstd_compressor(bool compress, size_t block_size, + char *options); + +void compressor_xz_print_help(void); + +void compressor_gzip_print_help(void); + +void compressor_lzo_print_help(void); + +void compressor_lz4_print_help(void); + +void compressor_zstd_print_help(void); + +#endif /* INTERNAL_H */ diff --git a/lib/sqfs/comp/lz4.c b/lib/sqfs/comp/lz4.c new file mode 100644 index 0000000..abb6c5c --- /dev/null +++ b/lib/sqfs/comp/lz4.c @@ -0,0 +1,159 @@ +/* SPDX-License-Identifier: GPL-3.0-or-later */ +/* + * lz4.c + * + * Copyright (C) 2019 David Oberhollenzer + */ +#include "config.h" + +#include +#include +#include +#include + +#include +#include + +#include "internal.h" + +typedef struct { + compressor_t base; + bool high_compression; +} lz4_compressor_t; + +typedef struct { + uint32_t version; + uint32_t flags; +} lz4_options; + +#define LZ4LEGACY 1 +#define LZ4_FLAG_HC 0x01 + +static int lz4_write_options(compressor_t *base, int fd) +{ + lz4_compressor_t *lz4 = (lz4_compressor_t *)base; + lz4_options opt = { + .version = htole32(LZ4LEGACY), + .flags = htole32(lz4->high_compression ? LZ4_FLAG_HC : 0), + }; + + return generic_write_options(fd, &opt, sizeof(opt)); +} + +static int lz4_read_options(compressor_t *base, int fd) +{ + lz4_options opt; + (void)base; + + if (generic_read_options(fd, &opt, sizeof(opt))) + return -1; + + opt.version = le32toh(opt.version); + opt.flags = le32toh(opt.flags); + + if (opt.version != LZ4LEGACY) { + fprintf(stderr, "unsupported lz4 version '%d'\n", opt.version); + return -1; + } + + return 0; +} + +static ssize_t lz4_comp_block(compressor_t *base, const uint8_t *in, + size_t size, uint8_t *out, size_t outsize) +{ + lz4_compressor_t *lz4 = (lz4_compressor_t *)base; + int ret; + + if (lz4->high_compression) { + ret = LZ4_compress_HC((void *)in, (void *)out, + size, outsize, LZ4HC_CLEVEL_MAX); + } else { + ret = LZ4_compress_default((void *)in, (void *)out, + size, outsize); + } + + if (ret < 0) { + fputs("internal error in lz4 compressor\n", stderr); + return -1; + } + + return ret; +} + +static ssize_t lz4_uncomp_block(compressor_t *base, const uint8_t *in, + size_t size, uint8_t *out, size_t outsize) +{ + int ret; + (void)base; + + ret = LZ4_decompress_safe((void *)in, (void *)out, size, outsize); + + if (ret < 0) { + fputs("internal error in lz4 decompressor\n", stderr); + return -1; + } + + return ret; +} + +static compressor_t *lz4_create_copy(compressor_t *cmp) +{ + lz4_compressor_t *lz4 = malloc(sizeof(*lz4)); + + if (lz4 == NULL) { + perror("creating additional lz4 compressor"); + return NULL; + } + + memcpy(lz4, cmp, sizeof(*lz4)); + return (compressor_t *)lz4; +} + +static void lz4_destroy(compressor_t *base) +{ + free(base); +} + +compressor_t *create_lz4_compressor(bool compress, size_t block_size, + char *options) +{ + lz4_compressor_t *lz4 = calloc(1, sizeof(*lz4)); + compressor_t *base = (compressor_t *)lz4; + (void)block_size; + + if (lz4 == NULL) { + perror("creating lz4 compressor"); + return NULL; + } + + lz4->high_compression = false; + + if (options != NULL) { + if (strcmp(options, "hc") == 0) { + lz4->high_compression = true; + } else { + fputs("Unsupported extra options for lz4 " + "compressor.\n", stderr); + free(lz4); + return NULL; + } + } + + base->destroy = lz4_destroy; + base->do_block = compress ? lz4_comp_block : lz4_uncomp_block; + base->write_options = lz4_write_options; + base->read_options = lz4_read_options; + base->create_copy = lz4_create_copy; + return base; +} + +void compressor_lz4_print_help(void) +{ + fputs("Available options for lz4 compressor:\n" + "\n" + " hc If present, use slower but better compressing\n" + " variant of lz4.\n" + "\n", + stdout); +} diff --git a/lib/sqfs/comp/lzo.c b/lib/sqfs/comp/lzo.c new file mode 100644 index 0000000..09ef75c --- /dev/null +++ b/lib/sqfs/comp/lzo.c @@ -0,0 +1,321 @@ +/* SPDX-License-Identifier: GPL-3.0-or-later */ +/* + * lzo.c + * + * Copyright (C) 2019 David Oberhollenzer + */ +#include "config.h" + +#include +#include +#include +#include +#include + +#include + +#include "internal.h" + + +typedef enum { + LZO_ALGORITHM_LZO1X_1 = 0, + LZO_ALGORITHM_LZO1X_1_11 = 1, + LZO_ALGORITHM_LZO1X_1_12 = 2, + LZO_ALGORITHM_LZO1X_1_15 = 3, + LZO_ALGORITHM_LZO1X_999 = 4, +} LZO_ALGORITHM; + +#define LZO_DEFAULT_ALG LZO_ALGORITHM_LZO1X_999 +#define LZO_DEFAULT_LEVEL 8 +#define LZO_NUM_ALGS (sizeof(lzo_algs) / sizeof(lzo_algs[0])) + +typedef int (*lzo_cb_t)(const lzo_bytep src, lzo_uint src_len, lzo_bytep dst, + lzo_uintp dst_len, lzo_voidp wrkmem); + +static const struct { + const char *name; + lzo_cb_t compress; + size_t bufsize; +} lzo_algs[] = { + [LZO_ALGORITHM_LZO1X_1] = { + .name = "lzo1x_1", + .compress = lzo1x_1_compress, + .bufsize = LZO1X_1_MEM_COMPRESS, + }, + [LZO_ALGORITHM_LZO1X_1_11] = { + .name = "lzo1x_1_11", + .compress = lzo1x_1_11_compress, + .bufsize = LZO1X_1_11_MEM_COMPRESS, + }, + [LZO_ALGORITHM_LZO1X_1_12] = { + .name = "lzo1x_1_12", + .compress = lzo1x_1_12_compress, + .bufsize = LZO1X_1_12_MEM_COMPRESS, + }, + [LZO_ALGORITHM_LZO1X_1_15] = { + .name = "lzo1x_1_15", + .compress = lzo1x_1_15_compress, + .bufsize = LZO1X_1_15_MEM_COMPRESS, + }, + [LZO_ALGORITHM_LZO1X_999] = { + .name = "lzo1x_999", + .compress = lzo1x_999_compress, + .bufsize = LZO1X_999_MEM_COMPRESS, + }, +}; + +typedef struct { + compressor_t base; + int algorithm; + int level; + + uint8_t buffer[]; +} lzo_compressor_t; + +typedef struct { + uint32_t algorithm; + uint32_t level; +} lzo_options_t; + +static int lzo_write_options(compressor_t *base, int fd) +{ + lzo_compressor_t *lzo = (lzo_compressor_t *)base; + lzo_options_t opt; + + if (lzo->algorithm == LZO_DEFAULT_ALG && + lzo->level == LZO_DEFAULT_LEVEL) { + return 0; + } + + opt.algorithm = htole32(lzo->algorithm); + + if (lzo->algorithm == LZO_ALGORITHM_LZO1X_999) { + opt.level = htole32(lzo->level); + } else { + opt.level = 0; + } + + return generic_write_options(fd, &opt, sizeof(opt)); +} + +static int lzo_read_options(compressor_t *base, int fd) +{ + lzo_compressor_t *lzo = (lzo_compressor_t *)base; + lzo_options_t opt; + + if (generic_read_options(fd, &opt, sizeof(opt))) + return -1; + + lzo->algorithm = le32toh(opt.algorithm); + lzo->level = le32toh(opt.level); + + switch(lzo->algorithm) { + case LZO_ALGORITHM_LZO1X_1: + case LZO_ALGORITHM_LZO1X_1_11: + case LZO_ALGORITHM_LZO1X_1_12: + case LZO_ALGORITHM_LZO1X_1_15: + if (lzo->level != 0) + goto fail_level; + break; + case LZO_ALGORITHM_LZO1X_999: + if (lzo->level < 1 || lzo->level > 9) + goto fail_level; + break; + default: + fputs("Unsupported LZO variant specified.\n", stderr); + return -1; + } + + return 0; +fail_level: + fputs("Unsupported LZO compression level specified.\n", stderr); + return -1; +} + +static ssize_t lzo_comp_block(compressor_t *base, const uint8_t *in, + size_t size, uint8_t *out, size_t outsize) +{ + lzo_compressor_t *lzo = (lzo_compressor_t *)base; + lzo_uint len = outsize; + int ret; + + if (lzo->algorithm == LZO_ALGORITHM_LZO1X_999 && + lzo->level != LZO_DEFAULT_LEVEL) { + ret = lzo1x_999_compress_level(in, size, out, &len, + lzo->buffer, NULL, 0, 0, + lzo->level); + } else { + ret = lzo_algs[lzo->algorithm].compress(in, size, out, + &len, lzo->buffer); + } + + if (ret != LZO_E_OK) { + fputs("LZO compression failed.\n", stderr); + return -1; + } + + if (len < size) + return len; + + return 0; +} + +static ssize_t lzo_uncomp_block(compressor_t *base, const uint8_t *in, + size_t size, uint8_t *out, size_t outsize) +{ + lzo_compressor_t *lzo = (lzo_compressor_t *)base; + lzo_uint len = outsize; + int ret; + + ret = lzo1x_decompress_safe(in, size, out, &len, lzo->buffer); + + if (ret != LZO_E_OK) { + fputs("lzo decompress: input data is corrupted\n", stderr); + return -1; + } + + return len; +} + +static compressor_t *lzo_create_copy(compressor_t *cmp) +{ + lzo_compressor_t *other = (lzo_compressor_t *)cmp; + lzo_compressor_t *lzo; + + lzo = alloc_flex(sizeof(*lzo), 1, lzo_algs[other->algorithm].bufsize); + + if (lzo == NULL) { + perror("creating additional lzo compressor"); + return NULL; + } + + memcpy(lzo, other, sizeof(*lzo)); + return (compressor_t *)lzo; +} + +static void lzo_destroy(compressor_t *base) +{ + free(base); +} + +static int process_options(char *options, int *algorithm, int *level) +{ + enum { + OPT_ALG = 0, + OPT_LEVEL, + }; + char *const token[] = { + [OPT_ALG] = (char *)"algorithm", + [OPT_LEVEL] = (char *)"level", + NULL + }; + char *subopts, *value; + size_t i; + int opt; + + subopts = options; + + while (*subopts != '\0') { + opt = getsubopt(&subopts, token, &value); + + switch (opt) { + case OPT_ALG: + if (value == NULL) + goto fail_value; + + for (i = 0; i < LZO_NUM_ALGS; ++i) { + if (strcmp(lzo_algs[i].name, value) == 0) { + *algorithm = i; + break; + } + } + + if (i == LZO_NUM_ALGS) { + fprintf(stderr, "Unknown lzo variant '%s'.\n", + value); + return -1; + } + break; + case OPT_LEVEL: + if (value == NULL) + goto fail_value; + + for (i = 0; isdigit(value[i]); ++i) + ; + + if (i < 1 || i > 3 || value[i] != '\0') + goto fail_level; + + *level = atoi(value); + + if (*level < 1 || *level > 9) + goto fail_level; + break; + default: + goto fail_opt; + } + } + + return 0; +fail_level: + fputs("Compression level must be a number between 1 and 9.\n", stderr); + return -1; +fail_opt: + fprintf(stderr, "Unknown option '%s'.\n", value); + return -1; +fail_value: + fprintf(stderr, "Missing value for '%s'.\n", token[opt]); + return -1; +} + +compressor_t *create_lzo_compressor(bool compress, size_t block_size, + char *options) +{ + lzo_compressor_t *lzo; + compressor_t *base; + int level, alg; + (void)block_size; + + alg = LZO_DEFAULT_ALG; + level = LZO_DEFAULT_LEVEL; + + if (options != NULL && process_options(options, &alg, &level) != 0) + return NULL; + + lzo = alloc_flex(sizeof(*lzo), 1, lzo_algs[alg].bufsize); + base = (compressor_t *)lzo; + + if (lzo == NULL) { + perror("creating lzo compressor"); + return NULL; + } + + lzo->algorithm = alg; + lzo->level = level; + + base->destroy = lzo_destroy; + base->do_block = compress ? lzo_comp_block : lzo_uncomp_block; + base->write_options = lzo_write_options; + base->read_options = lzo_read_options; + base->create_copy = lzo_create_copy; + return base; +} + +void compressor_lzo_print_help(void) +{ + size_t i; + + fputs("Available options for lzo compressor:\n" + "\n" + " algorithm= Specify the variant of lzo to use.\n" + " Defaults to 'lzo1x_999'.\n" + " level= For lzo1x_999, the compression level.\n" + " Value from 1 to 9. Defaults to 8.\n" + " Ignored if algorithm is not lzo1x_999.\n" + "\n" + "Available algorithms:\n", + stdout); + + for (i = 0; i < LZO_NUM_ALGS; ++i) + printf("\t%s\n", lzo_algs[i].name); +} diff --git a/lib/sqfs/comp/process_block.c b/lib/sqfs/comp/process_block.c new file mode 100644 index 0000000..0fcbae0 --- /dev/null +++ b/lib/sqfs/comp/process_block.c @@ -0,0 +1,38 @@ +/* SPDX-License-Identifier: GPL-3.0-or-later */ +/* + * process_block.c + * + * Copyright (C) 2019 David Oberhollenzer + */ +#include "config.h" + +#include "block_processor.h" +#include "util.h" + +#include +#include + +int process_block(block_t *block, compressor_t *cmp, + uint8_t *scratch, size_t scratch_size) +{ + ssize_t ret; + + if (!(block->flags & BLK_DONT_CHECKSUM)) + block->checksum = crc32(0, block->data, block->size); + + if (!(block->flags & BLK_DONT_COMPRESS)) { + ret = cmp->do_block(cmp, block->data, block->size, + scratch, scratch_size); + + if (ret < 0) + return -1; + + if (ret > 0) { + memcpy(block->data, scratch, ret); + block->size = ret; + block->flags |= BLK_IS_COMPRESSED; + } + } + + return 0; +} diff --git a/lib/sqfs/comp/xz.c b/lib/sqfs/comp/xz.c new file mode 100644 index 0000000..d38aab6 --- /dev/null +++ b/lib/sqfs/comp/xz.c @@ -0,0 +1,360 @@ +/* SPDX-License-Identifier: GPL-3.0-or-later */ +/* + * xz.c + * + * Copyright (C) 2019 David Oberhollenzer + */ +#include "config.h" + +#include +#include +#include +#include +#include +#include + +#include "internal.h" + +typedef enum { + XZ_FILTER_X86 = 0x01, + XZ_FILTER_POWERPC = 0x02, + XZ_FILTER_IA64 = 0x04, + XZ_FILTER_ARM = 0x08, + XZ_FILTER_ARMTHUMB = 0x10, + XZ_FILTER_SPARC = 0x20, + + XZ_FILTER_ALL = 0x3F, +} XZ_FILTER_FLAG; + +typedef struct { + compressor_t base; + size_t block_size; + size_t dict_size; + int flags; +} xz_compressor_t; + +typedef struct { + uint32_t dict_size; + uint32_t flags; +} xz_options_t; + +static const struct { + const char *name; + lzma_vli filter; + int flag; +} xz_filters[] = { + { "x86", LZMA_FILTER_X86, XZ_FILTER_X86 }, + { "powerpc", LZMA_FILTER_POWERPC, XZ_FILTER_POWERPC }, + { "ia64", LZMA_FILTER_IA64, XZ_FILTER_IA64 }, + { "arm", LZMA_FILTER_ARM, XZ_FILTER_ARM }, + { "armthumb", LZMA_FILTER_ARMTHUMB, XZ_FILTER_ARMTHUMB }, + { "sparc", LZMA_FILTER_SPARC, XZ_FILTER_SPARC }, +}; + +#define XZ_NUM_FILTERS (sizeof(xz_filters) / sizeof(xz_filters[0])) + +static int xz_write_options(compressor_t *base, int fd) +{ + xz_compressor_t *xz = (xz_compressor_t *)base; + xz_options_t opt; + + if (xz->flags == 0 && xz->dict_size == xz->block_size) + return 0; + + opt.dict_size = htole32(xz->dict_size); + opt.flags = htole32(xz->flags); + + return generic_write_options(fd, &opt, sizeof(opt)); +} + +static int xz_read_options(compressor_t *base, int fd) +{ + xz_compressor_t *xz = (xz_compressor_t *)base; + xz_options_t opt; + uint32_t mask; + + if (generic_read_options(fd, &opt, sizeof(opt))) + return -1; + + opt.dict_size = le32toh(opt.dict_size); + opt.flags = le32toh(opt.flags); + + mask = opt.dict_size & (opt.dict_size - 1); + + if (mask != 0 && ((mask & (mask - 1)) != 0)) { + fputs("Invalid lzma dictionary size.\n", stderr); + return -1; + } + + if (opt.flags & ~XZ_FILTER_ALL) { + fputs("Unknown BCJ filter used.\n", stderr); + return -1; + } + + xz->flags = opt.flags; + xz->dict_size = opt.dict_size; + return 0; +} + +static ssize_t compress(xz_compressor_t *xz, lzma_vli filter, + const uint8_t *in, size_t size, + uint8_t *out, size_t outsize) +{ + lzma_filter filters[5]; + lzma_options_lzma opt; + size_t written = 0; + lzma_ret ret; + int i = 0; + + if (lzma_lzma_preset(&opt, LZMA_PRESET_DEFAULT)) { + fputs("error initializing xz options\n", stderr); + return -1; + } + + opt.dict_size = xz->dict_size; + + if (filter != LZMA_VLI_UNKNOWN) { + filters[i].id = filter; + filters[i].options = NULL; + ++i; + } + + filters[i].id = LZMA_FILTER_LZMA2; + filters[i].options = &opt; + ++i; + + filters[i].id = LZMA_VLI_UNKNOWN; + filters[i].options = NULL; + ++i; + + ret = lzma_stream_buffer_encode(filters, LZMA_CHECK_CRC32, NULL, + in, size, out, &written, outsize); + + if (ret == LZMA_OK) + return (written >= size) ? 0 : written; + + if (ret != LZMA_BUF_ERROR) { + fputs("xz block compress failed\n", stderr); + return -1; + } + + return 0; +} + +static ssize_t xz_comp_block(compressor_t *base, const uint8_t *in, + size_t size, uint8_t *out, size_t outsize) +{ + xz_compressor_t *xz = (xz_compressor_t *)base; + lzma_vli selected = LZMA_VLI_UNKNOWN; + size_t i, smallest; + ssize_t ret; + + ret = compress(xz, LZMA_VLI_UNKNOWN, in, size, out, outsize); + if (ret < 0 || xz->flags == 0) + return ret; + + smallest = ret; + + for (i = 0; i < XZ_NUM_FILTERS; ++i) { + if (!(xz->flags & xz_filters[i].flag)) + continue; + + ret = compress(xz, xz_filters[i].filter, in, size, out, outsize); + if (ret < 0) + return -1; + + if (ret > 0 && (smallest == 0 || (size_t)ret < smallest)) { + smallest = ret; + selected = xz_filters[i].filter; + } + } + + if (smallest == 0) + return 0; + + return compress(xz, selected, in, size, out, outsize); +} + +static ssize_t xz_uncomp_block(compressor_t *base, const uint8_t *in, + size_t size, uint8_t *out, size_t outsize) +{ + uint64_t memlimit = 32 * 1024 * 1024; + size_t dest_pos = 0; + size_t src_pos = 0; + lzma_ret ret; + (void)base; + + ret = lzma_stream_buffer_decode(&memlimit, 0, NULL, + in, &src_pos, size, + out, &dest_pos, outsize); + + if (ret == LZMA_OK && size == src_pos) + return (ssize_t)dest_pos; + + fputs("xz block extract failed\n", stderr); + return -1; +} + +static compressor_t *xz_create_copy(compressor_t *cmp) +{ + xz_compressor_t *xz = malloc(sizeof(*xz)); + + if (xz == NULL) { + perror("creating additional xz compressor"); + return NULL; + } + + memcpy(xz, cmp, sizeof(*xz)); + return (compressor_t *)xz; +} + +static void xz_destroy(compressor_t *base) +{ + free(base); +} + +static int process_options(char *options, size_t blocksize, + int *flags, uint64_t *dictsize) +{ + enum { + OPT_DICT = 0, + }; + char *const token[] = { + [OPT_DICT] = (char *)"dictsize", + NULL + }; + char *subopts, *value; + uint64_t mask; + size_t i; + int opt; + + subopts = options; + + while (*subopts != '\0') { + opt = getsubopt(&subopts, token, &value); + + switch (opt) { + case OPT_DICT: + if (value == NULL) + goto fail_value; + + for (i = 0; isdigit(value[i]); ++i) + ; + + if (i < 1 || i > 9) + goto fail_dict; + + *dictsize = atol(value); + + switch (value[i]) { + case '\0': + break; + case 'm': + case 'M': + *dictsize <<= 20; + break; + case 'k': + case 'K': + *dictsize <<= 10; + break; + case '%': + *dictsize = ((*dictsize) * blocksize) / 100; + break; + default: + goto fail_dict; + } + + if (*dictsize > 0x0FFFFFFFFUL) + goto fail_dict_ov; + + mask = *dictsize & (*dictsize - 1); + + if (mask != 0 && ((mask & (mask - 1)) != 0)) + goto fail_dict_pot; + break; + default: + for (i = 0; i < XZ_NUM_FILTERS; ++i) { + if (strcmp(value, xz_filters[i].name) == 0) { + *flags |= xz_filters[i].flag; + break; + } + } + if (i == XZ_NUM_FILTERS) + goto fail_opt; + break; + } + } + + return 0; +fail_dict_pot: + fputs("dictionary size must be either 2^n or 2^n + 2^(n-1)\n", stderr); + return -1; +fail_dict_ov: + fputs("dictionary size too large.\n", stderr); + return -1; +fail_dict: + fputs("dictionary size must be a number with the optional " + "suffix 'm','k' or '%'.\n", stderr); + return -1; +fail_opt: + fprintf(stderr, "Unknown option '%s'.\n", value); + return -1; +fail_value: + fprintf(stderr, "Missing value for '%s'.\n", token[opt]); + return -1; +} + +compressor_t *create_xz_compressor(bool compress, size_t block_size, + char *options) +{ + uint64_t dictsize = block_size; + xz_compressor_t *xz; + compressor_t *base; + int flags = 0; + + if (options != NULL) { + if (process_options(options, block_size, &flags, &dictsize)) + return NULL; + } + + xz = calloc(1, sizeof(*xz)); + base = (compressor_t *)xz; + if (xz == NULL) { + perror("creating xz compressor"); + return NULL; + } + + xz->flags = flags; + xz->dict_size = dictsize; + xz->block_size = block_size; + base->destroy = xz_destroy; + base->do_block = compress ? xz_comp_block : xz_uncomp_block; + base->write_options = xz_write_options; + base->read_options = xz_read_options; + base->create_copy = xz_create_copy; + return base; +} + +void compressor_xz_print_help(void) +{ + size_t i; + + fputs( +"Available options for xz compressor:\n" +"\n" +" dictsize= Dictionary size. Either a value in bytes or a\n" +" percentage of the block size. Defaults to 100%.\n" +" The suffix '%' indicates a percentage. 'K' and 'M'\n" +" can also be used for kibi and mebi bytes\n" +" respecitively.\n" +"\n" +"In additon to the options, one or more bcj filters can be specified.\n" +"If multiple filters are provided, the one yielding the best compression\n" +"ratio will be used.\n" +"\n" +"The following filters are available:\n", + stdout); + + for (i = 0; i < XZ_NUM_FILTERS; ++i) + printf("\t%s\n", xz_filters[i].name); +} diff --git a/lib/sqfs/comp/zstd.c b/lib/sqfs/comp/zstd.c new file mode 100644 index 0000000..e206338 --- /dev/null +++ b/lib/sqfs/comp/zstd.c @@ -0,0 +1,188 @@ +/* SPDX-License-Identifier: GPL-3.0-or-later */ +/* + * zstd.c + * + * Copyright (C) 2019 David Oberhollenzer + */ +#include "config.h" + +#include +#include +#include +#include +#include + +#include + +#include "internal.h" + +#define ZSTD_DEFAULT_COMPRESSION_LEVEL 15 + +typedef struct { + compressor_t base; + ZSTD_CCtx *zctx; + int level; +} zstd_compressor_t; + +typedef struct { + uint32_t level; +} zstd_options_t; + +static int zstd_write_options(compressor_t *base, int fd) +{ + zstd_compressor_t *zstd = (zstd_compressor_t *)base; + zstd_options_t opt; + (void)fd; + + if (zstd->level == ZSTD_DEFAULT_COMPRESSION_LEVEL) + return 0; + + opt.level = htole32(zstd->level); + return generic_write_options(fd, &opt, sizeof(opt)); +} + +static int zstd_read_options(compressor_t *base, int fd) +{ + zstd_options_t opt; + (void)base; + + if (generic_read_options(fd, &opt, sizeof(opt))) + return -1; + + opt.level = le32toh(opt.level); + return 0; +} + +static ssize_t zstd_comp_block(compressor_t *base, const uint8_t *in, + size_t size, uint8_t *out, size_t outsize) +{ + zstd_compressor_t *zstd = (zstd_compressor_t *)base; + size_t ret; + + ret = ZSTD_compressCCtx(zstd->zctx, out, outsize, in, size, + zstd->level); + + if (ZSTD_isError(ret)) { + fprintf(stderr, "internal error in ZSTD compressor: %s\n", + ZSTD_getErrorName(ret)); + return -1; + } + + return ret < size ? ret : 0; +} + +static ssize_t zstd_uncomp_block(compressor_t *base, const uint8_t *in, + size_t size, uint8_t *out, size_t outsize) +{ + size_t ret; + (void)base; + + ret = ZSTD_decompress(out, outsize, in, size); + + if (ZSTD_isError(ret)) { + fprintf(stderr, "error uncompressing ZSTD compressed data: %s", + ZSTD_getErrorName(ret)); + return -1; + } + + return ret; +} + +static compressor_t *zstd_create_copy(compressor_t *cmp) +{ + zstd_compressor_t *zstd = malloc(sizeof(*zstd)); + + if (zstd == NULL) { + perror("creating additional zstd compressor"); + return NULL; + } + + memcpy(zstd, cmp, sizeof(*zstd)); + + zstd->zctx = ZSTD_createCCtx(); + + if (zstd->zctx == NULL) { + fputs("error creating addtional zstd compression context\n", + stderr); + free(zstd); + return NULL; + } + + return (compressor_t *)zstd; +} + +static void zstd_destroy(compressor_t *base) +{ + zstd_compressor_t *zstd = (zstd_compressor_t *)base; + + ZSTD_freeCCtx(zstd->zctx); + free(zstd); +} + +compressor_t *create_zstd_compressor(bool compress, size_t block_size, + char *options) +{ + zstd_compressor_t *zstd = calloc(1, sizeof(*zstd)); + compressor_t *base = (compressor_t *)zstd; + size_t i; + (void)block_size; + + if (zstd == NULL) { + perror("creating zstd compressor"); + return NULL; + } + + zstd->level = ZSTD_DEFAULT_COMPRESSION_LEVEL; + + if (options != NULL) { + if (strncmp(options, "level=", 6) == 0) { + options += 6; + + for (i = 0; isdigit(options[i]); ++i) + ; + + if (i == 0 || options[i] != '\0' || i > 6) + goto fail_level; + + zstd->level = atoi(options); + + if (zstd->level < 1 || zstd->level > ZSTD_maxCLevel()) + goto fail_level; + } else { + goto fail_opt; + } + } + + zstd->zctx = ZSTD_createCCtx(); + if (zstd->zctx == NULL) { + fputs("error creating zstd compression context\n", stderr); + free(zstd); + return NULL; + } + + base->destroy = zstd_destroy; + base->do_block = compress ? zstd_comp_block : zstd_uncomp_block; + base->write_options = zstd_write_options; + base->read_options = zstd_read_options; + base->create_copy = zstd_create_copy; + return base; +fail_level: + fprintf(stderr, "zstd compression level must be a number in the range " + "1...%d\n", ZSTD_maxCLevel()); + free(zstd); + return NULL; +fail_opt: + fputs("Unsupported extra options for zstd compressor\n", stderr); + free(zstd); + return NULL; +} + +void compressor_zstd_print_help(void) +{ + printf("Available options for zstd compressor:\n" + "\n" + " level= Set compression level. Defaults to %d.\n" + " Maximum is %d.\n" + "\n", + ZSTD_DEFAULT_COMPRESSION_LEVEL, ZSTD_maxCLevel()); +} diff --git a/mkfs/Makemodule.am b/mkfs/Makemodule.am index 1316c3d..9fbc806 100644 --- a/mkfs/Makemodule.am +++ b/mkfs/Makemodule.am @@ -1,5 +1,5 @@ gensquashfs_SOURCES = mkfs/mkfs.c mkfs/mkfs.h mkfs/options.c -gensquashfs_LDADD = libsquashfs.a libfstree.a libcompress.a libutil.a +gensquashfs_LDADD = libsquashfs.a libfstree.a libutil.a gensquashfs_CPPFLAGS = $(AM_CPPFLAGS) gensquashfs_CFLAGS = $(AM_CFLAGS) $(PTHREAD_CFLAGS) gensquashfs_LDADD += $(XZ_LIBS) $(ZLIB_LIBS) $(LZO_LIBS) $(LZ4_LIBS) diff --git a/tar/Makemodule.am b/tar/Makemodule.am index b8674c2..a9df4da 100644 --- a/tar/Makemodule.am +++ b/tar/Makemodule.am @@ -1,9 +1,9 @@ sqfs2tar_SOURCES = tar/sqfs2tar.c -sqfs2tar_LDADD = libsquashfs.a libtar.a libfstree.a libcompress.a libutil.a +sqfs2tar_LDADD = libsquashfs.a libtar.a libfstree.a libutil.a sqfs2tar_LDADD += $(XZ_LIBS) $(ZLIB_LIBS) $(LZO_LIBS) $(LZ4_LIBS) $(ZSTD_LIBS) tar2sqfs_SOURCES = tar/tar2sqfs.c -tar2sqfs_LDADD = libsquashfs.a libtar.a libfstree.a libcompress.a libutil.a +tar2sqfs_LDADD = libsquashfs.a libtar.a libfstree.a libutil.a tar2sqfs_CFLAGS = $(AM_CFLAGS) $(PTHREAD_CFLAGS) tar2sqfs_LDADD += $(XZ_LIBS) $(ZLIB_LIBS) $(LZO_LIBS) $(LZ4_LIBS) $(ZSTD_LIBS) tar2sqfs_LDADD += $(PTHREAD_LIBS) diff --git a/tests/Makemodule.am b/tests/Makemodule.am index c6ba1d9..4e12ff6 100644 --- a/tests/Makemodule.am +++ b/tests/Makemodule.am @@ -76,7 +76,7 @@ test_str_table_CPPFLAGS = $(AM_CPPFLAGS) -DTESTPATH=$(top_srcdir)/tests test_blk_proc_order_SOURCES = tests/blk_proc_order.c test_blk_proc_order_CPPFLAGS = $(AM_CPPFLAGS) test_blk_proc_order_CFLAGS = $(AM_CFLAGS) $(PTHREAD_CFLAGS) -test_blk_proc_order_LDADD = libcompress.a libutil.a $(ZLIB_LIBS) $(PTHREAD_LIBS) +test_blk_proc_order_LDADD = libsquashfs.a libutil.a $(ZLIB_LIBS) $(PTHREAD_LIBS) if HAVE_PTHREAD test_blk_proc_order_CPPFLAGS += -DHAVE_PTHREAD diff --git a/unpack/Makemodule.am b/unpack/Makemodule.am index 08a8b76..5681838 100644 --- a/unpack/Makemodule.am +++ b/unpack/Makemodule.am @@ -2,7 +2,7 @@ rdsquashfs_SOURCES = unpack/rdsquashfs.c unpack/rdsquashfs.h rdsquashfs_SOURCES += unpack/list_files.c unpack/options.c rdsquashfs_SOURCES += unpack/restore_fstree.c unpack/describe.c rdsquashfs_SOURCES += unpack/fill_files.c -rdsquashfs_LDADD = libsquashfs.a libfstree.a libcompress.a libutil.a +rdsquashfs_LDADD = libsquashfs.a libfstree.a libutil.a rdsquashfs_LDADD += $(XZ_LIBS) $(ZLIB_LIBS) $(LZO_LIBS) $(LZ4_LIBS) $(ZSTD_LIBS) bin_PROGRAMS += rdsquashfs -- cgit v1.2.3