diff options
| author | David Oberhollenzer <david.oberhollenzer@sigma-star.at> | 2022-12-13 09:15:19 +0100 |
|---|---|---|
| committer | David Oberhollenzer <david.oberhollenzer@sigma-star.at> | 2023-01-19 16:24:56 +0100 |
| commit | 551dd3879c288a2b6b6fbaca5c09c04fbe994ff4 (patch) | |
| tree | f3437139699edffd034168999854258f30c4023b | |
| parent | 722ecf27eaf83685dfc6e92adc9d66f0107da5ea (diff) | |
Split stream compression out of libio
Move it to a separate libxfrm library, where it can be independently
tested as well. The bulk of the new code is also mainly test cases
for the compressors.
Signed-off-by: David Oberhollenzer <david.oberhollenzer@sigma-star.at>
38 files changed, 1689 insertions, 1543 deletions
diff --git a/Makefile.am b/Makefile.am index 9c2f506..804bdae 100644 --- a/Makefile.am +++ b/Makefile.am @@ -27,6 +27,7 @@ TESTS = include lib/sqfs/Makemodule.am include lib/util/Makemodule.am +include lib/xfrm/Makemodule.am include lib/io/Makemodule.am if BUILD_TOOLS @@ -47,6 +48,7 @@ include tests/libutil/Makemodule.am include tests/libio/Makemodule.am include tests/libfstree/Makemodule.am include tests/libtar/Makemodule.am +include tests/libxfrm/Makemodule.am include tests/libsqfs/Makemodule.am include tests/gensquashfs/Makemodule.am include tests/rdsquashfs/Makemodule.am diff --git a/bin/sqfs2tar/Makemodule.am b/bin/sqfs2tar/Makemodule.am index 22d523e..05cee5b 100644 --- a/bin/sqfs2tar/Makemodule.am +++ b/bin/sqfs2tar/Makemodule.am @@ -3,7 +3,7 @@ sqfs2tar_SOURCES += bin/sqfs2tar/options.c bin/sqfs2tar/write_tree.c sqfs2tar_SOURCES += bin/sqfs2tar/xattr.c sqfs2tar_CFLAGS = $(AM_CFLAGS) $(PTHREAD_CFLAGS) sqfs2tar_LDADD = libcommon.a libutil.a libsquashfs.la libtar.a -sqfs2tar_LDADD += libio.a libcompat.a libfstree.a +sqfs2tar_LDADD += libio.a libxfrm.a libcompat.a libfstree.a sqfs2tar_LDADD += $(ZLIB_LIBS) $(XZ_LIBS) $(LZO_LIBS) $(ZSTD_LIBS) $(BZIP2_LIBS) sqfs2tar_LDADD += $(PTHREAD_LIBS) diff --git a/bin/sqfs2tar/options.c b/bin/sqfs2tar/options.c index 4f783e0..ba1588d 100644 --- a/bin/sqfs2tar/options.c +++ b/bin/sqfs2tar/options.c @@ -91,19 +91,12 @@ void process_args(int argc, char **argv) switch (i) { case 'c': - compressor = io_compressor_id_from_name(optarg); + compressor = xfrm_compressor_id_from_name(optarg); if (compressor <= 0) { fprintf(stderr, "unknown compressor '%s'.\n", optarg); goto fail; } - - if (!io_compressor_exists(compressor)) { - fprintf(stderr, - "%s compressor is not supported.\n", - optarg); - goto fail; - } break; case 'd': if (num_subdirs == max_subdirs) { @@ -163,11 +156,11 @@ void process_args(int argc, char **argv) case 'h': fputs(usagestr, stdout); - i = IO_COMPRESSOR_MIN; + i = XFRM_COMPRESSOR_MIN; - while (i <= IO_COMPRESSOR_MAX) { - name = io_compressor_name_from_id(i); - if (io_compressor_exists(i)) + while (i <= XFRM_COMPRESSOR_MAX) { + name = xfrm_compressor_name_from_id(i); + if (name != NULL) printf("\t%s\n", name); ++i; } diff --git a/bin/sqfs2tar/sqfs2tar.c b/bin/sqfs2tar/sqfs2tar.c index f8d3173..43f9e78 100644 --- a/bin/sqfs2tar/sqfs2tar.c +++ b/bin/sqfs2tar/sqfs2tar.c @@ -124,9 +124,15 @@ int main(int argc, char **argv) } if (compressor > 0) { - ostream_t *strm = ostream_compressor_create(out_file, - compressor); + xfrm_stream_t *xfrm = compressor_stream_create(compressor,NULL); + ostream_t *strm; + + if (xfrm == NULL) + goto out; + + strm = ostream_xfrm_create(out_file, xfrm); sqfs_drop(out_file); + sqfs_drop(xfrm); out_file = strm; if (out_file == NULL) diff --git a/bin/sqfs2tar/sqfs2tar.h b/bin/sqfs2tar/sqfs2tar.h index 71b491d..4bf5428 100644 --- a/bin/sqfs2tar/sqfs2tar.h +++ b/bin/sqfs2tar/sqfs2tar.h @@ -12,6 +12,7 @@ #include "util/util.h" #include "tar/tar.h" +#include "xfrm/compress.h" #include "io/xfrm.h" #include <getopt.h> diff --git a/bin/tar2sqfs/Makemodule.am b/bin/tar2sqfs/Makemodule.am index a9c503f..faa2948 100644 --- a/bin/tar2sqfs/Makemodule.am +++ b/bin/tar2sqfs/Makemodule.am @@ -1,7 +1,7 @@ tar2sqfs_SOURCES = bin/tar2sqfs/tar2sqfs.c bin/tar2sqfs/tar2sqfs.h tar2sqfs_SOURCES += bin/tar2sqfs/options.c bin/tar2sqfs/process_tarball.c tar2sqfs_CFLAGS = $(AM_CFLAGS) $(PTHREAD_CFLAGS) -tar2sqfs_LDADD = libcommon.a libsquashfs.la libtar.a libio.a +tar2sqfs_LDADD = libcommon.a libsquashfs.la libtar.a libio.a libxfrm.a tar2sqfs_LDADD += libfstree.a libcompat.a libfstree.a libutil.a $(LZO_LIBS) tar2sqfs_LDADD += $(ZLIB_LIBS) $(XZ_LIBS) $(ZSTD_LIBS) $(BZIP2_LIBS) tar2sqfs_LDADD += $(PTHREAD_LIBS) diff --git a/bin/tar2sqfs/options.c b/bin/tar2sqfs/options.c index 94e7036..f2185a6 100644 --- a/bin/tar2sqfs/options.c +++ b/bin/tar2sqfs/options.c @@ -94,15 +94,15 @@ char *root_becomes = NULL; static void input_compressor_print_available(void) { - int i = IO_COMPRESSOR_MIN; + int i = XFRM_COMPRESSOR_MIN; const char *name; fputs("\nSupported tar compression formats:\n", stdout); - while (i <= IO_COMPRESSOR_MAX) { - name = io_compressor_name_from_id(i); + while (i <= XFRM_COMPRESSOR_MAX) { + name = xfrm_compressor_name_from_id(i); - if (io_compressor_exists(i)) + if (name != NULL) printf("\t%s\n", name); ++i; diff --git a/bin/tar2sqfs/tar2sqfs.c b/bin/tar2sqfs/tar2sqfs.c index 572eb10..9257fed 100644 --- a/bin/tar2sqfs/tar2sqfs.c +++ b/bin/tar2sqfs/tar2sqfs.c @@ -32,12 +32,45 @@ static int tar_probe(const sqfs_u8 *data, size_t size) return 0; } +static istream_t *magic_autowrap(istream_t *strm) +{ + xfrm_stream_t *xfrm = NULL; + istream_t *wrapper = NULL; + const sqfs_u8 *data; + size_t avail; + int ret; + + ret = istream_precache(strm); + if (ret != 0) + goto out; + + data = strm->buffer + strm->buffer_offset; + avail = strm->buffer_used - strm->buffer_offset; + + ret = tar_probe(data, avail); + if (ret > 0) + return strm; + + ret = xfrm_compressor_id_from_magic(data, avail); + if (ret <= 0) + return strm; + + xfrm = decompressor_stream_create(ret); + if (xfrm == NULL) + goto out; + + wrapper = istream_xfrm_create(strm, xfrm); +out: + sqfs_drop(strm); + sqfs_drop(xfrm); + return wrapper; +} + int main(int argc, char **argv) { int status = EXIT_FAILURE; istream_t *input_file = NULL; sqfs_writer_t sqfs; - int ret; process_args(argc, argv); @@ -45,28 +78,9 @@ int main(int argc, char **argv) if (input_file == NULL) return EXIT_FAILURE; - ret = istream_detect_compressor(input_file, tar_probe); - if (ret < 0) - goto out_if; - - if (ret > 0) { - istream_t *strm; - - if (!io_compressor_exists(ret)) { - fprintf(stderr, - "%s: %s compression is not supported.\n", - istream_get_filename(input_file), - io_compressor_name_from_id(ret)); - goto out_if; - } - - strm = istream_compressor_create(input_file, ret); - sqfs_drop(input_file); - input_file = strm; - - if (input_file == NULL) - return EXIT_FAILURE; - } + input_file = magic_autowrap(input_file); + if (input_file == NULL) + return EXIT_FAILURE; memset(&sqfs, 0, sizeof(sqfs)); if (sqfs_writer_init(&sqfs, &cfg)) diff --git a/bin/tar2sqfs/tar2sqfs.h b/bin/tar2sqfs/tar2sqfs.h index 6e4d123..a21774b 100644 --- a/bin/tar2sqfs/tar2sqfs.h +++ b/bin/tar2sqfs/tar2sqfs.h @@ -14,6 +14,7 @@ #include "util/util.h" #include "tar/tar.h" #include "tar/format.h" +#include "xfrm/compress.h" #include "io/xfrm.h" #include <stdlib.h> diff --git a/include/io/xfrm.h b/include/io/xfrm.h index 22a42b6..3e601ea 100644 --- a/include/io/xfrm.h +++ b/include/io/xfrm.h @@ -9,117 +9,45 @@ #include "io/istream.h" #include "io/ostream.h" - -enum { - /** - * @brief Deflate compressor with gzip headers. - * - * This actually creates a gzip compatible file, including a - * gzip header and trailer. - */ - IO_COMPRESSOR_GZIP = 1, - - IO_COMPRESSOR_XZ = 2, - - IO_COMPRESSOR_ZSTD = 3, - - IO_COMPRESSOR_BZIP2 = 4, - - IO_COMPRESSOR_MIN = 1, - IO_COMPRESSOR_MAX = 4, -}; +#include "xfrm/stream.h" #ifdef __cplusplus extern "C" { #endif /** - * @brief Create an input stream that transparently uncompresses data. + * @brief Create an input stream that transparently decodes data. * * @memberof istream_t * * This function creates an input stream that wraps an underlying input stream - * that is compressed and transparently uncompresses the data when reading + * that is encoded/compressed and transparently decodes the data when reading * from it. * - * The new stream takes ownership of the wrapped stream and destroys it when - * the compressor stream is destroyed. If this function fails, the wrapped - * stream is also destroyed. - * * @param strm A pointer to another stream that should be wrapped. - * @param comp_id An identifier describing the compressor to use. + * @param xfrm The transformation stream to use. * * @return A pointer to an input stream on success, NULL on failure. */ -SQFS_INTERNAL istream_t *istream_compressor_create(istream_t *strm, - int comp_id); +SQFS_INTERNAL istream_t *istream_xfrm_create(istream_t *strm, + xfrm_stream_t *xfrm); /** - * @brief Create an output stream that transparently compresses data. + * @brief Create an output stream that transparently encodes data. * * @memberof ostream_t * - * This function creates an output stream that transparently compresses all - * data appended to it and writes the compressed data to an underlying, wrapped - * output stream. - * - * The new stream takes ownership of the wrapped stream and destroys it when - * the compressor stream is destroyed. If this function fails, the wrapped - * stream is also destroyed. + * This function creates an output stream that transparently encodes + * (e.g. compresses) all data appended to it and writes it to an + * underlying, wrapped output stream. * * @param strm A pointer to another stream that should be wrapped. - * @param comp_id An identifier describing the compressor to use. + * @param xfrm The transformation stream to use. * * @return A pointer to an output stream on success, NULL on failure. */ -SQFS_INTERNAL ostream_t *ostream_compressor_create(ostream_t *strm, - int comp_id); - -/** - * @brief Probe the buffered data in an istream to check if it is compressed. - * - * @memberof istream_t - * - * This function peeks into the internal buffer of an input stream to check - * for magic signatures of various compressors. - * - * @param strm A pointer to an input stream to check - * @param probe A callback used to check if raw/decoded data matches an - * expected format. Returns 0 if not, -1 on failure and +1 - * on success. - * - * @return A compressor ID on success, 0 if no match was found, -1 on failure. - */ -SQFS_INTERNAL int istream_detect_compressor(istream_t *strm, - int (*probe)(const sqfs_u8 *data, - size_t size)); - -/** - * @brief Resolve a compressor name to an ID. - * - * @param name A compressor name. - * - * @return A compressor ID on success, -1 on failure. - */ -SQFS_INTERNAL int io_compressor_id_from_name(const char *name); - -/** - * @brief Resolve a id to a compressor name. - * - * @param id A compressor ID. - * - * @return A compressor name on success, NULL on failure. - */ -SQFS_INTERNAL const char *io_compressor_name_from_id(int id); - -/** - * @brief Check if support for a given compressor has been built in. - * - * @param id A compressor ID. - * - * @return True if the compressor is supported, false if not. - */ -SQFS_INTERNAL bool io_compressor_exists(int id); +SQFS_INTERNAL ostream_t *ostream_xfrm_create(ostream_t *strm, + xfrm_stream_t *xfrm); #ifdef __cplusplus } diff --git a/include/xfrm/compress.h b/include/xfrm/compress.h new file mode 100644 index 0000000..b5db985 --- /dev/null +++ b/include/xfrm/compress.h @@ -0,0 +1,117 @@ +/* SPDX-License-Identifier: GPL-3.0-or-later */ +/* + * compress.h + * + * Copyright (C) 2021 David Oberhollenzer <goliath@infraroot.at> + */ +#ifndef XFRM_COMPRESS_H +#define XFRM_COMPRESS_H + +#include "xfrm/stream.h" + +typedef struct { + uint32_t flags; + uint32_t level; + + union { + struct { + uint8_t vli; + uint8_t pad0[15]; + } xz; + + struct { + uint16_t window_size; + uint16_t padd0[7]; + } gzip; + + struct { + uint8_t work_factor; + uint8_t padd0[15]; + } bzip2; + + uint64_t padd0[2]; + } opt; +} compressor_config_t; + +typedef enum { + COMP_FLAG_XZ_EXTREME = 0x0001, +} COMP_FLAG_XZ; + +typedef enum { + COMP_XZ_VLI_X86 = 1, + COMP_XZ_VLI_POWERPC = 2, + COMP_XZ_VLI_IA64 = 3, + COMP_XZ_VLI_ARM = 4, + COMP_XZ_VLI_ARMTHUMB = 5, + COMP_XZ_VLI_SPARC = 6, +} COMP_XZ_VLI; + +#define COMP_GZIP_MIN_LEVEL (1) +#define COMP_GZIP_MAX_LEVEL (9) +#define COMP_GZIP_DEFAULT_LEVEL (9) + +#define COMP_GZIP_MIN_WINDOW (8) +#define COMP_GZIP_MAX_WINDOW (15) +#define COMP_GZIP_DEFAULT_WINDOW (15) + +#define COMP_ZSTD_MIN_LEVEL (1) +#define COMP_ZSTD_MAX_LEVEL (22) +#define COMP_ZSTD_DEFAULT_LEVEL (15) + +#define COMP_BZIP2_MIN_LEVEL (1) +#define COMP_BZIP2_MAX_LEVEL (9) +#define COMP_BZIP2_DEFAULT_LEVEL (9) + +#define COMP_BZIP2_MIN_WORK_FACTOR (0) +#define COMP_BZIP2_MAX_WORK_FACTOR (250) +#define COMP_BZIP2_DEFAULT_WORK_FACTOR (30) + +#define COMP_XZ_MIN_LEVEL (0) +#define COMP_XZ_MAX_LEVEL (9) +#define COMP_XZ_DEFAULT_LEVEL (6) + +enum { + XFRM_COMPRESSOR_GZIP = 1, + XFRM_COMPRESSOR_XZ = 2, + XFRM_COMPRESSOR_ZSTD = 3, + XFRM_COMPRESSOR_BZIP2 = 4, + + XFRM_COMPRESSOR_MIN = 1, + XFRM_COMPRESSOR_MAX = 4, +}; + +#ifdef __cplusplus +extern "C" { +#endif + +xfrm_stream_t *compressor_stream_bzip2_create(const compressor_config_t *cfg); + +xfrm_stream_t *decompressor_stream_bzip2_create(void); + +xfrm_stream_t *compressor_stream_xz_create(const compressor_config_t *cfg); + +xfrm_stream_t *decompressor_stream_xz_create(void); + +xfrm_stream_t *compressor_stream_gzip_create(const compressor_config_t *cfg); + +xfrm_stream_t *decompressor_stream_gzip_create(void); + +xfrm_stream_t *compressor_stream_zstd_create(const compressor_config_t *cfg); + +xfrm_stream_t *decompressor_stream_zstd_create(void); + +int xfrm_compressor_id_from_name(const char *name); + +int xfrm_compressor_id_from_magic(const void *data, size_t count); + +const char *xfrm_compressor_name_from_id(int id); + +xfrm_stream_t *compressor_stream_create(int id, const compressor_config_t *cfg); + +xfrm_stream_t *decompressor_stream_create(int id); + +#ifdef __cplusplus +} +#endif + +#endif /* XFRM_COMPRESS_H */ diff --git a/include/xfrm/stream.h b/include/xfrm/stream.h new file mode 100644 index 0000000..01639cd --- /dev/null +++ b/include/xfrm/stream.h @@ -0,0 +1,39 @@ +/* SPDX-License-Identifier: GPL-3.0-or-later */ +/* + * stream.h + * + * Copyright (C) 2021 David Oberhollenzer <goliath@infraroot.at> + */ +#ifndef XFRM_STREAM_H +#define XFRM_STREAM_H + +#include "sqfs/predef.h" + +typedef enum { + XFRM_STREAM_FLUSH_NONE = 0, + XFRM_STREAM_FLUSH_SYNC, + XFRM_STREAM_FLUSH_FULL, + + XFRM_STREAM_FLUSH_COUNT, +} XFRM_STREAM_FLUSH; + +typedef enum { + XFRM_STREAM_ERROR = -1, + XFRM_STREAM_OK = 0, + XFRM_STREAM_END = 1, + XFRM_STREAM_BUFFER_FULL = 2, +} XFRM_STREAM_RESULT; + +typedef struct xfrm_stream_t xfrm_stream_t; + +struct xfrm_stream_t { + sqfs_object_t base; + + int (*process_data)(xfrm_stream_t *stream, + const void *in, sqfs_u32 in_size, + void *out, sqfs_u32 out_size, + sqfs_u32 *in_read, sqfs_u32 *out_written, + int flush_mode); +}; + +#endif /* XFRM_STREAM_H */ diff --git a/lib/io/Makemodule.am b/lib/io/Makemodule.am index 63ce958..918ce50 100644 --- a/lib/io/Makemodule.am +++ b/lib/io/Makemodule.am @@ -1,11 +1,8 @@ libio_a_SOURCES = lib/io/internal.h libio_a_SOURCES += include/io/istream.h lib/io/ostream.c lib/io/printf.c libio_a_SOURCES += include/io/ostream.h lib/io/istream.c lib/io/get_line.c -libio_a_SOURCES += include/io/xfrm.h lib/io/xfrm.c -libio_a_SOURCES += include/io/file.h include/io/std.h -libio_a_SOURCES += lib/io/compress/ostream_compressor.c -libio_a_SOURCES += lib/io/uncompress/istream_compressor.c -libio_a_SOURCES += lib/io/uncompress/autodetect.c +libio_a_SOURCES += include/io/xfrm.h include/io/file.h include/io/std.h +libio_a_SOURCES += lib/io/xfrm/ostream.c lib/io/xfrm/istream.c libio_a_CFLAGS = $(AM_CFLAGS) $(ZLIB_CFLAGS) $(XZ_CFLAGS) libio_a_CFLAGS += $(ZSTD_CFLAGS) $(BZIP2_CFLAGS) libio_a_CPPFLAGS = $(AM_CPPFLAGS) @@ -19,27 +16,4 @@ libio_a_SOURCES += lib/io/unix/ostream.c libio_a_SOURCES += lib/io/unix/istream.c endif -if WITH_XZ -libio_a_SOURCES += lib/io/compress/xz.c lib/io/uncompress/xz.c -libio_a_CPPFLAGS += -DWITH_XZ -endif - -if WITH_GZIP -libio_a_SOURCES += lib/io/compress/gzip.c -libio_a_SOURCES += lib/io/uncompress/gzip.c -libio_a_CPPFLAGS += -DWITH_GZIP -endif - -if WITH_ZSTD -libio_a_SOURCES += lib/io/compress/zstd.c -libio_a_SOURCES += lib/io/uncompress/zstd.c -libio_a_CPPFLAGS += -DWITH_ZSTD -endif - -if WITH_BZIP2 -libio_a_SOURCES += lib/io/compress/bzip2.c -libio_a_SOURCES += lib/io/uncompress/bzip2.c -libio_a_CPPFLAGS += -DWITH_BZIP2 -endif - noinst_LIBRARIES += libio.a diff --git a/lib/io/compress/bzip2.c b/lib/io/compress/bzip2.c deleted file mode 100644 index 7f0c09a..0000000 --- a/lib/io/compress/bzip2.c +++ /dev/null @@ -1,96 +0,0 @@ -/* SPDX-License-Identifier: GPL-3.0-or-later */ -/* - * bzip2.c - * - * Copyright (C) 2019 David Oberhollenzer <goliath@infraroot.at> - */ -#include "../internal.h" - -#include <bzlib.h> - -typedef struct { - ostream_comp_t base; - - bz_stream strm; -} ostream_bzip2_t; - -static int flush_inbuf(ostream_comp_t *base, bool finish) -{ - ostream_bzip2_t *bzip2 = (ostream_bzip2_t *)base; - size_t have; - int ret; - - bzip2->strm.next_in = (char *)base->inbuf; - - if (base->inbuf_used > sizeof(base->inbuf)) - base->inbuf_used = sizeof(base->inbuf); - - if ((sizeof(size_t) > sizeof(unsigned int)) && - (base->inbuf_used > (size_t)UINT_MAX)) { - bzip2->strm.avail_in = UINT_MAX; - } else { - bzip2->strm.avail_in = (unsigned int)base->inbuf_used; - } - - for (;;) { - bzip2->strm.next_out = (char *)base->outbuf; - bzip2->strm.avail_out = sizeof(base->outbuf); - - ret = BZ2_bzCompress(&bzip2->strm, finish ? BZ_FINISH : BZ_RUN); - - if (ret < 0 && ret != BZ_OUTBUFF_FULL) { - fprintf(stderr, "%s: internal error in bzip2 " - "compressor.\n", - base->wrapped->get_filename(base->wrapped)); - return -1; - } - - have = sizeof(base->outbuf) - bzip2->strm.avail_out; - - if (base->wrapped->append(base->wrapped, base->outbuf, have)) - return -1; - - if (ret == BZ_STREAM_END || ret == BZ_OUTBUFF_FULL || - bzip2->strm.avail_in == 0) { - break; - } - } - - if (bzip2->strm.avail_in > 0) { - memmove(base->inbuf, bzip2->strm.next_in, - bzip2->strm.avail_in); - } - - base->inbuf_used = bzip2->strm.avail_in; - return 0; -} - -static void cleanup(ostream_comp_t *base) -{ - ostream_bzip2_t *bzip2 = (ostream_bzip2_t *)base; - - BZ2_bzCompressEnd(&bzip2->strm); -} - -ostream_comp_t *ostream_bzip2_create(const char *filename) -{ - ostream_bzip2_t *bzip2 = calloc(1, sizeof(*bzip2)); - ostream_comp_t *base = (ostream_comp_t *)bzip2; - - if (bzip2 == NULL) { - fprintf(stderr, "%s: creating bzip2 compressor: %s.\n", - filename, strerror(errno)); - return NULL; - } - - if (BZ2_bzCompressInit(&bzip2->strm, 9, 0, 30) != BZ_OK) { - fprintf(stderr, "%s: error initializing bzip2 compressor.\n", - filename); - free(bzip2); - return NULL; - } - - base->flush_inbuf = flush_inbuf; - base->cleanup = cleanup; - return base; -} diff --git a/lib/io/compress/gzip.c b/lib/io/compress/gzip.c deleted file mode 100644 index b73a258..0000000 --- a/lib/io/compress/gzip.c +++ /dev/null @@ -1,92 +0,0 @@ -/* SPDX-License-Identifier: GPL-3.0-or-later */ -/* - * gzip.c - * - * Copyright (C) 2019 David Oberhollenzer <goliath@infraroot.at> - */ -#include "../internal.h" - -#include <zlib.h> - -typedef struct { - ostream_comp_t base; - - z_stream strm; -} ostream_gzip_t; - -static int flush_inbuf(ostream_comp_t *base, bool finish) -{ - ostream_gzip_t *gzip = (ostream_gzip_t *)base; - size_t have; - int ret; - - if (base->inbuf_used > sizeof(base->inbuf)) - base->inbuf_used = sizeof(base->inbuf); - - if (sizeof(size_t) > sizeof(uInt)) { - gzip->strm.avail_in = ~((uInt)0); - - if ((size_t)gzip->strm.avail_in > base->inbuf_used) - gzip->strm.avail_in = (uInt)base->inbuf_used; - } else { - gzip->strm.avail_in = (uInt)base->inbuf_used; - } - - gzip->strm.next_in = base->inbuf; - - do { - gzip->strm.avail_out = BUFSZ; - gzip->strm.next_out = base->outbuf; - - ret = deflate(&gzip->strm, finish ? Z_FINISH : Z_NO_FLUSH); - - if (ret == Z_STREAM_ERROR) { - fprintf(stderr, - "%s: internal error in gzip compressor.\n", - base->wrapped->get_filename(base->wrapped)); - return -1; - } - - have = BUFSZ - gzip->strm.avail_out; - - if (base->wrapped->append(base->wrapped, base->outbuf, have)) - return -1; - } while (gzip->strm.avail_out == 0); - - base->inbuf_used = 0; - return 0; -} - -static void cleanup(ostream_comp_t *base) -{ - ostream_gzip_t *gzip = (ostream_gzip_t *)base; - - deflateEnd(&gzip->strm); -} - -ostream_comp_t *ostream_gzip_create(const char *filename) -{ - ostream_gzip_t *gzip = calloc(1, sizeof(*gzip)); - ostream_comp_t *base = (ostream_comp_t *)gzip; - int ret; - - if (gzip == NULL) { - fprintf(stderr, "%s: creating gzip wrapper: %s.\n", - filename, strerror(errno)); - return NULL; - } - - ret = deflateInit2(&gzip->strm, 9, Z_DEFLATED, 16 + 15, 8, - Z_DEFAULT_STRATEGY); - if (ret != Z_OK) { - fprintf(stderr, - "%s: internal error creating gzip compressor.\n", - filename); - free(gzip); - return NULL; - } - - base->flush_inbuf = flush_inbuf; - base->cleanup = cleanup; - return base; -} diff --git a/lib/io/compress/ostream_compressor.c b/lib/io/compress/ostream_compressor.c deleted file mode 100644 index a5f16d5..0000000 --- a/lib/io/compress/ostream_compressor.c +++ /dev/null @@ -1,106 +0,0 @@ -/* SPDX-License-Identifier: GPL-3.0-or-later */ -/* - * ostream_compressor.c - * - * Copyright (C) 2019 David Oberhollenzer <goliath@infraroot.at> - */ -#include "../internal.h" - -static int comp_append(ostream_t *strm, const void *data, size_t size) -{ - ostream_comp_t *comp = (ostream_comp_t *)strm; - size_t diff; - - while (size > 0) { - if (comp->inbuf_used >= BUFSZ) { - if (comp->flush_inbuf(comp, false)) - return -1; - } - - diff = BUFSZ - comp->inbuf_used; - - if (diff > size) - diff = size; - - memcpy(comp->inbuf + comp->inbuf_used, data, diff); - - comp->inbuf_used += diff; - data = (const char *)data + diff; - size -= diff; - } - - return 0; -} - -static int comp_flush(ostream_t *strm) -{ - ostream_comp_t *comp = (ostream_comp_t *)strm; - - if (comp->inbuf_used > 0) { - if (comp->flush_inbuf(comp, true)) - return -1; - } - - return comp->wrapped->flush(comp->wrapped); -} - -static const char *comp_get_filename(ostream_t *strm) -{ - ostream_comp_t *comp = (ostream_comp_t *)strm; - - return comp->wrapped->get_filename(comp->wrapped); -} - -static void comp_destroy(sqfs_object_t *obj) -{ - ostream_comp_t *comp = (ostream_comp_t *)obj; - - comp->cleanup(comp); - sqfs_drop(comp->wrapped); - free(comp); -} - -ostream_t *ostream_compressor_create(ostream_t *strm, int comp_id) -{ - ostream_comp_t *comp = NULL; - ostream_t *base; - - switch (comp_id) { - case IO_COMPRESSOR_GZIP: -#ifdef WITH_GZIP - comp = ostream_gzip_create(strm->get_filename(strm)); -#endif - break; - case IO_COMPRESSOR_XZ: -#ifdef WITH_XZ - comp = ostream_xz_create(strm->get_filename(strm)); -#endif - break; - case IO_COMPRESSOR_ZSTD: -#if defined(WITH_ZSTD) && defined(HAVE_ZSTD_STREAM) - comp = ostream_zstd_create(strm->get_filename(strm)); -#endif - break; - case IO_COMPRESSOR_BZIP2: -#ifdef WITH_BZIP2 - comp = ostream_bzip2_create(strm->get_filename(strm)); -#endif - break; - default: - break; - } - - if (comp == NULL) - return NULL; - - sqfs_object_init(comp, comp_destroy, NULL); - - comp->wrapped = sqfs_grab(strm); - comp->inbuf_used = 0; - - base = (ostream_t *)comp; - base->append = comp_append; - base->flush = comp_flush; - base->get_filename = comp_get_filename; - return base; -} diff --git a/lib/io/compress/xz.c b/lib/io/compress/xz.c deleted file mode 100644 index 65bda0b..0000000 --- a/lib/io/compress/xz.c +++ /dev/null @@ -1,80 +0,0 @@ -/* SPDX-License-Identifier: GPL-3.0-or-later */ -/* - * xz.c - * - * Copyright (C) 2019 David Oberhollenzer <goliath@infraroot.at> - */ -#include "../internal.h" - -#include <lzma.h> - -typedef struct { - ostream_comp_t base; - - lzma_stream strm; -} ostream_xz_t; - -static int flush_inbuf(ostream_comp_t *base, bool finish) -{ - ostream_xz_t *xz = (ostream_xz_t *)base; - lzma_ret ret_xz; - size_t have; - - xz->strm.next_in = base->inbuf; - xz->strm.avail_in = base->inbuf_used; - - do { - xz->strm.next_out = base->outbuf; - xz->strm.avail_out = BUFSZ; - - ret_xz = lzma_code(&xz->strm, finish ? LZMA_FINISH : LZMA_RUN); - - if ((ret_xz != LZMA_OK) && (ret_xz != LZMA_STREAM_END)) { - fprintf(stderr, - "%s: internal error in XZ compressor.\n", - base->wrapped->get_filename(base->wrapped)); - return -1; - } - - have = BUFSZ - xz->strm.avail_out; - - if (base->wrapped->append(base->wrapped, base->outbuf, have)) - return -1; - } while (xz->strm.avail_out == 0); - - base->inbuf_used = 0; - return 0; -} - -static void cleanup(ostream_comp_t *base) -{ - ostream_xz_t *xz = (ostream_xz_t *)base; - - lzma_end(&xz->strm); -} - -ostream_comp_t *ostream_xz_create(const char *filename) -{ - ostream_xz_t *xz = calloc(1, sizeof(*xz)); - ostream_comp_t *base = (ostream_comp_t *)xz; - lzma_ret ret_xz; - - if (xz == NULL) { - fprintf(stderr, "%s: creating xz wrapper: %s.\n", - filename, strerror(errno)); - return NULL; - } - - ret_xz = lzma_easy_encoder(&xz->strm, LZMA_PRESET_DEFAULT, - LZMA_CHECK_CRC64); - if (ret_xz != LZMA_OK) { - fprintf(stderr, "%s: error initializing XZ compressor\n", - filename); - free(xz); - return NULL; - } - - base->flush_inbuf = flush_inbuf; - base->cleanup = cleanup; - return base; -} diff --git a/lib/io/compress/zstd.c b/lib/io/compress/zstd.c deleted file mode 100644 index c0b002e..0000000 --- a/lib/io/compress/zstd.c +++ /dev/null @@ -1,94 +0,0 @@ -/* SPDX-License-Identifier: GPL-3.0-or-later */ -/* - * zstd.c - * - * Copyright (C) 2019 David Oberhollenzer <goliath@infraroot.at> - */ -#include "../internal.h" - -#include <zstd.h> - -#ifdef HAVE_ZSTD_STREAM -typedef struct { - ostream_comp_t base; - - ZSTD_CStream *strm; -} ostream_zstd_t; - -static int flush_inbuf(ostream_comp_t *base, bool finish) -{ - ostream_zstd_t *zstd = (ostream_zstd_t *)base; - ZSTD_EndDirective op; - ZSTD_outBuffer out; - ZSTD_inBuffer in; - size_t ret; - - op = finish ? ZSTD_e_end : ZSTD_e_continue; - - do { - memset(&in, 0, sizeof(in)); - memset(&out, 0, sizeof(out)); - - in.src = base->inbuf; - in.size = base->inbuf_used; - - out.dst = base->outbuf; - out.size = BUFSZ; - - ret = ZSTD_compressStream2(zstd->strm, &out, &in, op); - - if (ZSTD_isError(ret)) { - fprintf(stderr, "%s: error in zstd compressor.\n", - base->wrapped->get_filename(base->wrapped)); - return -1; - } - - if (base->wrapped->append(base->wrapped, base->outbuf, - out.pos)) { - return -1; - } - - if (in.pos < in.size) { - base->inbuf_used = in.size - in.pos; - - memmove(base->inbuf, base->inbuf + in.pos, - base->inbuf_used); - } else { - base->inbuf_used = 0; - } - } while (finish && ret != 0); - - return 0; -} - -static void cleanup(ostream_comp_t *base) -{ - ostream_zstd_t *zstd = (ostream_zstd_t *)base; - - ZSTD_freeCStream(zstd->strm); -} - -ostream_comp_t *ostream_zstd_create(const char *filename) -{ - ostream_zstd_t *zstd = calloc(1, sizeof(*zstd)); - ostream_comp_t *base = (ostream_comp_t *)zstd; - - if (zstd == NULL) { - fprintf(stderr, "%s: creating zstd wrapper: %s.\n", - filename, strerror(errno)); - return NULL; - } - - zstd->strm = ZSTD_createCStream(); - if (zstd->strm == NULL) { - fprintf(stderr, "%s: error creating zstd decoder.\n", - filename); - free(zstd); - return NULL; - } - - base->flush_inbuf = flush_inbuf; - base->cleanup = cleanup; - return base; -} -#endif /* HAVE_ZSTD_STREAM */ diff --git a/lib/io/internal.h b/lib/io/internal.h index 4ac38f5..25a0196 100644 --- a/lib/io/internal.h +++ b/lib/io/internal.h @@ -14,6 +14,7 @@ #include "io/file.h" #include "io/xfrm.h" #include "io/std.h" +#include "xfrm/compress.h" #include <string.h> #include <stdlib.h> @@ -27,55 +28,4 @@ #define BUFSZ (262144) -typedef struct ostream_comp_t { - ostream_t base; - - ostream_t *wrapped; - - size_t inbuf_used; - - sqfs_u8 inbuf[BUFSZ]; - sqfs_u8 outbuf[BUFSZ]; - - int (*flush_inbuf)(struct ostream_comp_t *ostrm, bool finish); - - void (*cleanup)(struct ostream_comp_t *ostrm); -} ostream_comp_t; - -typedef struct istream_comp_t { - istream_t base; - - istream_t *wrapped; - - sqfs_u8 uncompressed[BUFSZ]; - - bool eof; - - void (*cleanup)(struct istream_comp_t *strm); -} istream_comp_t; - -#ifdef __cplusplus -extern "C" { -#endif - -SQFS_INTERNAL ostream_comp_t *ostream_gzip_create(const char *filename); - -SQFS_INTERNAL ostream_comp_t *ostream_xz_create(const char *filename); - -SQFS_INTERNAL ostream_comp_t *ostream_zstd_create(const char *filename); - -SQFS_INTERNAL ostream_comp_t *ostream_bzip2_create(const char *filename); - -SQFS_INTERNAL istream_comp_t *istream_gzip_create(const char *filename); - -SQFS_INTERNAL istream_comp_t *istream_xz_create(const char *filename); - -SQFS_INTERNAL istream_comp_t *istream_zstd_create(const char *filename); - -SQFS_INTERNAL istream_comp_t *istream_bzip2_create(const char *filename); - -#ifdef __cplusplus -} -#endif - #endif /* INTERNAL_H */ diff --git a/lib/io/uncompress/autodetect.c b/lib/io/uncompress/autodetect.c deleted file mode 100644 index dde33c8..0000000 --- a/lib/io/uncompress/autodetect.c +++ /dev/null @@ -1,55 +0,0 @@ -/* SPDX-License-Identifier: GPL-3.0-or-later */ -/* - * autodetect.c - * - * Copyright (C) 2019 David Oberhollenzer <goliath@infraroot.at> - */ -#include "../internal.h" - -static const struct { - int id; - const sqfs_u8 *value; - size_t len; -} magic[] = { - { IO_COMPRESSOR_GZIP, (const sqfs_u8 *)"\x1F\x8B\x08", 3 }, - { IO_COMPRESSOR_XZ, (const sqfs_u8 *)("\xFD" "7zXZ"), 6 }, - { IO_COMPRESSOR_ZSTD, (const sqfs_u8 *)"\x28\xB5\x2F\xFD", 4 }, - { IO_COMPRESSOR_BZIP2, (const sqfs_u8 *)"BZh", 3 }, -}; - -int istream_detect_compressor(istream_t *strm, - int (*probe)(const sqfs_u8 *data, size_t size)) -{ - size_t i; - int ret; - - ret = istream_precache(strm); - if (ret != 0) - return ret; - - if (probe != NULL) { - ret = probe(strm->buffer + strm->buffer_offset, - strm->buffer_used - strm->buffer_offset); - if (ret < 0) - return ret; - - /* XXX: this means the data is uncompressed. We do this check - first since it might be perfectly OK for the uncompressed - data to contain a magic number from the table. */ - if (ret > 0) - return 0; - } - - for (i = 0; i < sizeof(magic) / sizeof(magic[0]); ++i) { - if ((strm->buffer_used - strm->buffer_offset) < magic[i].len) - continue; - - ret = memcmp(strm->buffer + strm->buffer_offset, - magic[i].value, magic[i].len); - - if (ret == 0) - return magic[i].id; - } - - return 0; -} diff --git a/lib/io/uncompress/bzip2.c b/lib/io/uncompress/bzip2.c deleted file mode 100644 index 3b44383..0000000 --- a/lib/io/uncompress/bzip2.c +++ /dev/null @@ -1,118 +0,0 @@ -/* SPDX-License-Identifier: GPL-3.0-or-later */ -/* - * bzip2.c - * - * Copyright (C) 2019 David Oberhollenzer <goliath@infraroot.at> - */ -#include "../internal.h" - -#include <bzlib.h> - -typedef struct { - istream_comp_t base; - - bool initialized; - bz_stream strm; -} istream_bzip2_t; - -static int precache(istream_t *base) -{ - istream_bzip2_t *bzip2 = (istream_bzip2_t *)base; - istream_t *wrapped = ((istream_comp_t *)base)->wrapped; - size_t avail; - int ret; - - for (;;) { - if (!bzip2->initialized) { - if (BZ2_bzDecompressInit(&bzip2->strm, 0, 0) != BZ_OK) { - fprintf(stderr, "%s: error initializing " - "bzip2 decompressor.\n", - wrapped->get_filename(wrapped)); - return -1; - } - - bzip2->initialized = true; - } - - ret = istream_precache(wrapped); - if (ret != 0) - return ret; - - avail = wrapped->buffer_used; - if ((sizeof(size_t) > sizeof(unsigned int)) && - (avail > (size_t)UINT_MAX)) { - avail = UINT_MAX; - } - - bzip2->strm.next_in = (char *)wrapped->buffer; - bzip2->strm.avail_in = (unsigned int)avail; - - if (base->buffer_used > BUFSZ) - base->buffer_used = BUFSZ; - - avail = BUFSZ - base->buffer_used; - - if ((sizeof(size_t) > sizeof(unsigned int)) && - (avail > (size_t)UINT_MAX)) { - avail = UINT_MAX; - } - - bzip2->strm.next_out = (char *)base->buffer + base->buffer_used; - bzip2->strm.avail_out = (unsigned int)avail; - - if (bzip2->strm.avail_out < 1) - break; - - ret = BZ2_bzDecompress(&bzip2->strm); - - if (ret < 0) { - fprintf(stderr, "%s: internal error in bzip2 " - "decompressor.\n", - wrapped->get_filename(wrapped)); - return -1; - } - - base->buffer_used = BUFSZ - bzip2->strm.avail_out; - wrapped->buffer_offset = wrapped->buffer_used - - bzip2->strm.avail_in; - - if (ret == BZ_STREAM_END) { - if (istream_precache(wrapped)) - return -1; - - BZ2_bzDecompressEnd(&bzip2->strm); - bzip2->initialized = false; - - if (wrapped->buffer_used == 0) { - base->eof = true; - break; - } - } - } - - return 0; -} - -static void cleanup(istream_comp_t *base) -{ - istream_bzip2_t *bzip2 = (istream_bzip2_t *)base; - - if (bzip2->initialized) - BZ2_bzDecompressEnd(&bzip2->strm); -} - -istream_comp_t *istream_bzip2_create(const char *filename) -{ - istream_bzip2_t *bzip2 = calloc(1, sizeof(*bzip2)); - istream_comp_t *base = (istream_comp_t *)bzip2; - - if (bzip2 == NULL) { - fprintf(stderr, "%s: creating bzip2 compressor: %s.\n", - filename, strerror(errno)); - return NULL; - } - - ((istream_t *)base)->precache = precache; - base->cleanup = cleanup; - return base; -} diff --git a/lib/io/uncompress/gzip.c b/lib/io/uncompress/gzip.c deleted file mode 100644 index bce7f0a..0000000 --- a/lib/io/uncompress/gzip.c +++ /dev/null @@ -1,91 +0,0 @@ -/* SPDX-License-Identifier: GPL-3.0-or-later */ -/* - * gzip.c - * - * Copyright (C) 2019 David Oberhollenzer <goliath@infraroot.at> - */ -#include "../internal.h" - -#include <zlib.h> - -typedef struct { - istream_comp_t base; - - z_stream strm; -} istream_gzip_t; - -static int precache(istream_t *base) -{ - istream_t *wrapped = ((istream_comp_t *)base)->wrapped; - istream_gzip_t *gzip = (istream_gzip_t *)base; - int ret; - - for (;;) { - ret = istream_precache(wrapped); - if (ret != 0) - return ret; - - gzip->strm.avail_in = (uInt)wrapped->buffer_used; - gzip->strm.avail_out = (uInt)(BUFSZ - base->buffer_used); - - gzip->strm.next_in = wrapped->buffer; - gzip->strm.next_out = base->buffer + base->buffer_used; - - ret = inflate(&gzip->strm, Z_NO_FLUSH); - - wrapped->buffer_offset = wrapped->buffer_used - - gzip->strm.avail_in; - - base->buffer_used = BUFSZ - gzip->strm.avail_out; - - if (ret == Z_BUF_ERROR) - break; - - if (ret == Z_STREAM_END) { - base->eof = true; - break; - } - - if (ret != Z_OK) { - fprintf(stderr, - "%s: internal error in gzip decoder.\n", - wrapped->get_filename(wrapped)); - return -1; - } - } - - return 0; -} - -static void cleanup(istream_comp_t *base) -{ - istream_gzip_t *gzip = (istream_gzip_t *)base; - - inflateEnd(&gzip->strm); -} - -istream_comp_t *istream_gzip_create(const char *filename) -{ - istream_gzip_t *gzip = calloc(1, sizeof(*gzip)); - istream_comp_t *base = (istream_comp_t *)gzip; - int ret; - - if (gzip == NULL) { - fprintf(stderr, "%s: creating gzip decoder: %s.\n", - filename, strerror(errno)); - return NULL; - } - - ret = inflateInit2(&gzip->strm, 16 + 15); - if (ret != Z_OK) { - fprintf(stderr, - "%s: internal error creating gzip reader.\n", - filename); - free(gzip); - return NULL; - } - - ((istream_t *)base)->precache = precache; - base->cleanup = cleanup; - return base; -} diff --git a/lib/io/uncompress/istream_compressor.c b/lib/io/uncompress/istream_compressor.c deleted file mode 100644 index d4e1aea..0000000 --- a/lib/io/uncompress/istream_compressor.c +++ /dev/null @@ -1,67 +0,0 @@ -/* SPDX-License-Identifier: GPL-3.0-or-later */ -/* - * istream_compressor.c - * - * Copyright (C) 2019 David Oberhollenzer <goliath@infraroot.at> - */ -#include "../internal.h" - -static const char *comp_get_filename(istream_t *strm) -{ - istream_comp_t *comp = (istream_comp_t *)strm; - - return comp->wrapped->get_filename(comp->wrapped); -} - -static void comp_destroy(sqfs_object_t *obj) -{ - istream_comp_t *comp = (istream_comp_t *)obj; - - comp->cleanup(comp); - sqfs_drop(comp->wrapped); - free(comp); -} - -istream_t *istream_compressor_create(istream_t *strm, int comp_id) -{ - istream_comp_t *comp = NULL; - istream_t *base; - - switch (comp_id) { - case IO_COMPRESSOR_GZIP: -#ifdef WITH_GZIP - comp = istream_gzip_create(strm->get_filename(strm)); -#endif - break; - case IO_COMPRESSOR_XZ: -#ifdef WITH_XZ - comp = istream_xz_create(strm->get_filename(strm)); -#endif - break; - case IO_COMPRESSOR_ZSTD: -#if defined(WITH_ZSTD) && defined(HAVE_ZSTD_STREAM) - comp = istream_zstd_create(strm->get_filename(strm)); -#endif - break; - case IO_COMPRESSOR_BZIP2: -#ifdef WITH_BZIP2 - comp = istream_bzip2_create(strm->get_filename(strm)); -#endif - break; - default: - break; - } - - if (comp == NULL) - return NULL; - - sqfs_object_init(comp, comp_destroy, NULL); - - comp->wrapped = sqfs_grab(strm); - - base = (istream_t *)comp; - base->get_filename = comp_get_filename; - base->buffer = comp->uncompressed; - base->eof = false; - return base; -} diff --git a/lib/io/uncompress/xz.c b/lib/io/uncompress/xz.c deleted file mode 100644 index 0fd9ce6..0000000 --- a/lib/io/uncompress/xz.c +++ /dev/null @@ -1,96 +0,0 @@ -/* SPDX-License-Identifier: GPL-3.0-or-later */ -/* - * xz.c - * - * Copyright (C) 2019 David Oberhollenzer <goliath@infraroot.at> - */ -#include "../internal.h" - -#include <lzma.h> - -typedef struct { - istream_comp_t base; - - lzma_stream strm; -} istream_xz_t; - -static int precache(istream_t *base) -{ - istream_xz_t *xz = (istream_xz_t *)base; - istream_t *wrapped = ((istream_comp_t *)base)->wrapped; - lzma_action action; - lzma_ret ret_xz; - int ret; - - for (;;) { - ret = istream_precache(wrapped); - if (ret != 0) - return ret; - - action = wrapped->eof ? LZMA_FINISH : LZMA_RUN; - - xz->strm.avail_in = wrapped->buffer_used; - xz->strm.next_in = wrapped->buffer; - - xz->strm.avail_out = BUFSZ - base->buffer_used; - xz->strm.next_out = base->buffer + base->buffer_used; - - ret_xz = lzma_code(&xz->strm, action); - - base->buffer_used = BUFSZ - xz->strm.avail_out; - wrapped->buffer_offset = wrapped->buffer_used - - xz->strm.avail_in; - - if (ret_xz == LZMA_BUF_ERROR) - break; - - if (ret_xz == LZMA_STREAM_END) { - base->eof = true; - break; - } - - if (ret_xz != LZMA_OK) { - fprintf(stderr, - "%s: internal error in xz decoder.\n", - wrapped->get_filename(wrapped)); - return -1; - } - } - - return 0; -} - -static void cleanup(istream_comp_t *base) -{ - istream_xz_t *xz = (istream_xz_t *)base; - - lzma_end(&xz->strm); -} - -istream_comp_t *istream_xz_create(const char *filename) -{ - istream_xz_t *xz = calloc(1, sizeof(*xz)); - istream_comp_t *base = (istream_comp_t *)xz; - sqfs_u64 memlimit = 65 * 1024 * 1024; - lzma_ret ret_xz; - - if (xz == NULL) { - fprintf(stderr, "%s: creating xz decoder: %s.\n", - filename, strerror(errno)); - return NULL; - } - - ret_xz = lzma_stream_decoder(&xz->strm, memlimit, LZMA_CONCATENATED); - - if (ret_xz != LZMA_OK) { - fprintf(stderr, - "%s: error initializing xz decoder.\n", - filename); - free(xz); - return NULL; - } - - ((istream_t *)base)->precache = precache; - base->cleanup = cleanup; - return base; -} diff --git a/lib/io/uncompress/zstd.c b/lib/io/uncompress/zstd.c deleted file mode 100644 index fd22cbf..0000000 --- a/lib/io/uncompress/zstd.c +++ /dev/null @@ -1,81 +0,0 @@ -/* SPDX-License-Identifier: GPL-3.0-or-later */ -/* - * zstd.c - * - * Copyright (C) 2019 David Oberhollenzer <goliath@infraroot.at> - */ -#include "../internal.h" - -#include <zstd.h> - -#ifdef HAVE_ZSTD_STREAM -typedef struct { - istream_comp_t base; - - ZSTD_DStream* strm; -} istream_zstd_t; - -static int precache(istream_t *base) -{ - istream_zstd_t *zstd = (istream_zstd_t *)base; - istream_t *wrapped = ((istream_comp_t *)base)->wrapped; - ZSTD_outBuffer out; - ZSTD_inBuffer in; - size_t ret; - - if (istream_precache(wrapped)) - return -1; - - memset(&in, 0, sizeof(in)); - memset(&out, 0, sizeof(out)); - - in.src = wrapped->buffer; - in.size = wrapped->buffer_used; - - out.dst = ((istream_comp_t *)base)->uncompressed + base->buffer_used; - out.size = BUFSZ - base->buffer_used; - - ret = ZSTD_decompressStream(zstd->strm, &out, &in); - - if (ZSTD_isError(ret)) { - fprintf(stderr, "%s: error in zstd decoder.\n", - wrapped->get_filename(wrapped)); - return -1; - } - - wrapped->buffer_offset = in.pos; - base->buffer_used += out.pos; - return 0; -} - -static void cleanup(istream_comp_t *base) -{ - istream_zstd_t *zstd = (istream_zstd_t *)base; - - ZSTD_freeDStream(zstd->strm); -} - -istream_comp_t *istream_zstd_create(const char *filename) -{ - istream_zstd_t *zstd = calloc(1, sizeof(*zstd)); - istream_comp_t *base = (istream_comp_t *)zstd; - - if (zstd == NULL) { - fprintf(stderr, "%s: creating zstd decoder: %s.\n", - filename, strerror(errno)); - return NULL; - } - - zstd->strm = ZSTD_createDStream(); - if (zstd->strm == NULL) { - fprintf(stderr, "%s: error creating zstd decoder.\n", - filename); - free(zstd); - return NULL; - } - - ((istream_t *)base)->precache = precache; - base->cleanup = cleanup; - return base; -} -#endif /* HAVE_ZSTD_STREAM */ diff --git a/lib/io/xfrm.c b/lib/io/xfrm.c deleted file mode 100644 index 22fd953..0000000 --- a/lib/io/xfrm.c +++ /dev/null @@ -1,67 +0,0 @@ -/* SPDX-License-Identifier: GPL-3.0-or-later */ -/* - * compressor.c - * - * Copyright (C) 2019 David Oberhollenzer <goliath@infraroot.at> - */ -#include "internal.h" - -int io_compressor_id_from_name(const char *name) -{ - if (strcmp(name, "gzip") == 0) - return IO_COMPRESSOR_GZIP; - - if (strcmp(name, "xz") == 0) - return IO_COMPRESSOR_XZ; - - if (strcmp(name, "zstd") == 0) - return IO_COMPRESSOR_ZSTD; - - if (strcmp(name, "bzip2") == 0) - return IO_COMPRESSOR_BZIP2; - - return -1; -} - -const char *io_compressor_name_from_id(int id) -{ - if (id == IO_COMPRESSOR_GZIP) - return "gzip"; - - if (id == IO_COMPRESSOR_XZ) - return "xz"; - - if (id == IO_COMPRESSOR_ZSTD) - return "zstd"; - - if (id == IO_COMPRESSOR_BZIP2) - return "bzip2"; - - return NULL; -} - -bool io_compressor_exists(int id) -{ - switch (id) { -#ifdef WITH_GZIP - case IO_COMPRESSOR_GZIP: - return true; -#endif -#ifdef WITH_XZ - case IO_COMPRESSOR_XZ: - return true; -#endif -#if defined(WITH_ZSTD) && defined(HAVE_ZSTD_STREAM) - case IO_COMPRESSOR_ZSTD: - return true; -#endif -#ifdef WITH_BZIP2 - case IO_COMPRESSOR_BZIP2: - return true; -#endif - default: - break; - } - - return false; -} diff --git a/lib/io/xfrm/istream.c b/lib/io/xfrm/istream.c new file mode 100644 index 0000000..4a1ad77 --- /dev/null +++ b/lib/io/xfrm/istream.c @@ -0,0 +1,106 @@ +/* SPDX-License-Identifier: GPL-3.0-or-later */ +/* + * istream.c + * + * Copyright (C) 2019 David Oberhollenzer <goliath@infraroot.at> + */ +#include "../internal.h" + +typedef struct istream_xfrm_t { + istream_t base; + + istream_t *wrapped; + xfrm_stream_t *xfrm; + + sqfs_u8 uncompressed[BUFSZ]; +} istream_xfrm_t; + +static int xfrm_precache(istream_t *base) +{ + istream_xfrm_t *xfrm = (istream_xfrm_t *)base; + int ret; + + ret = istream_precache(xfrm->wrapped); + if (ret != 0) + return ret; + + for (;;) { + const sqfs_u32 in_sz = xfrm->wrapped->buffer_used; + const sqfs_u32 out_sz = sizeof(xfrm->uncompressed); + sqfs_u32 in_off = 0, out_off = base->buffer_used; + int mode = XFRM_STREAM_FLUSH_NONE; + + if (xfrm->wrapped->eof) + mode = XFRM_STREAM_FLUSH_FULL; + + ret = xfrm->xfrm->process_data(xfrm->xfrm, + xfrm->wrapped->buffer, in_sz, + base->buffer + out_off, + out_sz - out_off, + &in_off, &out_off, mode); + + if (ret == XFRM_STREAM_ERROR) { + fprintf(stderr, "%s: internal error in decompressor.\n", + base->get_filename(base)); + return -1; + } + + base->buffer_used = out_off; + xfrm->wrapped->buffer_offset = in_off; + + if (ret == XFRM_STREAM_BUFFER_FULL || out_off >= out_sz) + break; + + ret = istream_precache(xfrm->wrapped); + if (ret != 0) + return ret; + + if (xfrm->wrapped->eof && xfrm->wrapped->buffer_used == 0) { + if (base->buffer_used == 0) + base->eof = true; + break; + } + } + + return 0; +} + +static const char *xfrm_get_filename(istream_t *strm) +{ + istream_xfrm_t *xfrm = (istream_xfrm_t *)strm; + + return xfrm->wrapped->get_filename(xfrm->wrapped); +} + +static void xfrm_destroy(sqfs_object_t *obj) +{ + istream_xfrm_t *xfrm = (istream_xfrm_t *)obj; + + sqfs_drop(xfrm->xfrm); + sqfs_drop(xfrm->wrapped); + free(xfrm); +} + +istream_t *istream_xfrm_create(istream_t *strm, xfrm_stream_t *xfrm) +{ + istream_xfrm_t *stream = calloc(1, sizeof(*stream)); + istream_t *base = (istream_t *)stream; + + if (stream == NULL) + goto fail; + + sqfs_object_init(stream, xfrm_destroy, NULL); + + stream->wrapped = sqfs_grab(strm); + stream->xfrm = sqfs_grab(xfrm); + + base->precache = xfrm_precache; + base->get_filename = xfrm_get_filename; + base->buffer = stream->uncompressed; + base->eof = false; + return base; +fail: + fprintf(stderr, "%s: error initializing decompressor stream.\n", + strm->get_filename(strm)); + return NULL; +} diff --git a/lib/io/xfrm/ostream.c b/lib/io/xfrm/ostream.c new file mode 100644 index 0000000..bd94515 --- /dev/null +++ b/lib/io/xfrm/ostream.c @@ -0,0 +1,144 @@ +/* SPDX-License-Identifier: GPL-3.0-or-later */ +/* + * ostream.c + * + * Copyright (C) 2019 David Oberhollenzer <goliath@infraroot.at> + */ +#include "../internal.h" + +typedef struct ostream_xfrm_t { + ostream_t base; + + ostream_t *wrapped; + xfrm_stream_t *xfrm; + + size_t inbuf_used; + + sqfs_u8 inbuf[BUFSZ]; + sqfs_u8 outbuf[BUFSZ]; +} ostream_xfrm_t; + +static int flush_inbuf(ostream_xfrm_t *xfrm, bool finish) +{ + const sqfs_u32 avail_out = sizeof(xfrm->outbuf); + const sqfs_u32 avail_in = xfrm->inbuf_used; + const int mode = finish ? XFRM_STREAM_FLUSH_FULL : + XFRM_STREAM_FLUSH_NONE; + sqfs_u32 off_in = 0, off_out = 0; + int ret; + + while (finish || off_in < avail_in) { + ret = xfrm->xfrm->process_data(xfrm->xfrm, + xfrm->inbuf + off_in, + avail_in - off_in, + xfrm->outbuf + off_out, + avail_out - off_out, + &off_in, &off_out, mode); + + if (ret == XFRM_STREAM_ERROR) { + fprintf(stderr, + "%s: internal error in compressor.\n", + xfrm->wrapped->get_filename(xfrm->wrapped)); + return -1; + } + + if (ostream_append(xfrm->wrapped, xfrm->outbuf, off_out)) + return -1; + + off_out = 0; + + if (ret == XFRM_STREAM_END) + break; + } + + if (off_out > 0) { + if (ostream_append(xfrm->wrapped, xfrm->outbuf, off_out)) + return -1; + } + + if (off_in < avail_in) { + memmove(xfrm->inbuf, xfrm->inbuf + off_in, avail_in - off_in); + xfrm->inbuf_used -= off_in; + } else { + xfrm->inbuf_used = 0; + } + + return 0; +} + +static int xfrm_append(ostream_t *strm, const void *data, size_t size) +{ + ostream_xfrm_t *xfrm = (ostream_xfrm_t *)strm; + size_t diff; + + while (size > 0) { + if (xfrm->inbuf_used >= BUFSZ) { + if (flush_inbuf(xfrm, false)) + return -1; + } + + diff = BUFSZ - xfrm->inbuf_used; + + if (diff > size) + diff = size; + + memcpy(xfrm->inbuf + xfrm->inbuf_used, data, diff); + + xfrm->inbuf_used += diff; + data = (const char *)data + diff; + size -= diff; + } + + return 0; +} + +static int xfrm_flush(ostream_t *strm) +{ + ostream_xfrm_t *xfrm = (ostream_xfrm_t *)strm; + + if (xfrm->inbuf_used > 0) { + if (flush_inbuf(xfrm, true)) + return -1; + } + + return xfrm->wrapped->flush(xfrm->wrapped); +} + +static const char *xfrm_get_filename(ostream_t *strm) +{ + ostream_xfrm_t *xfrm = (ostream_xfrm_t *)strm; + + return xfrm->wrapped->get_filename(xfrm->wrapped); +} + +static void xfrm_destroy(sqfs_object_t *obj) +{ + ostream_xfrm_t *xfrm = (ostream_xfrm_t *)obj; + + sqfs_drop(xfrm->wrapped); + sqfs_drop(xfrm->xfrm); + free(xfrm); +} + +ostream_t *ostream_xfrm_create(ostream_t *strm, xfrm_stream_t *xfrm) +{ + ostream_xfrm_t *stream = calloc(1, sizeof(*stream)); + ostream_t *base = (ostream_t *)stream; + + if (stream == NULL) + goto fail; + + sqfs_object_init(stream, xfrm_destroy, NULL); + + stream->wrapped = sqfs_grab(strm); + stream->xfrm = sqfs_grab(xfrm); + stream->inbuf_used = 0; + base->append = xfrm_append; + base->flush = xfrm_flush; + base->get_filename = xfrm_get_filename; + return base; +fail: + fprintf(stderr, "%s: error initializing compressor.\n", + strm->get_filename(strm)); + return NULL; +} diff --git a/lib/xfrm/Makemodule.am b/lib/xfrm/Makemodule.am new file mode 100644 index 0000000..a344820 --- /dev/null +++ b/lib/xfrm/Makemodule.am @@ -0,0 +1,27 @@ +libxfrm_a_SOURCES = include/xfrm/stream.h include/xfrm/compress.h +libxfrm_a_SOURCES += lib/xfrm/compress.c +libxfrm_a_CFLAGS = $(AM_CFLAGS) + +if WITH_XZ +libxfrm_a_SOURCES += lib/xfrm/xz.c +libxfrm_a_CFLAGS += $(XZ_CFLAGS) -DWITH_XZ +endif + +if WITH_BZIP2 +libxfrm_a_SOURCES += lib/xfrm/bzip2.c +libxfrm_a_CFLAGS += $(BZIP2_CFLAGS) -DWITH_BZIP2 +endif + +if WITH_GZIP +libxfrm_a_SOURCES += lib/xfrm/gzip.c +libxfrm_a_CFLAGS += $(ZLIB_CFLAGS) -DWITH_GZIP +endif + +if WITH_ZSTD +if HAVE_ZSTD_STREAM +libxfrm_a_SOURCES += lib/xfrm/zstd.c +libxfrm_a_CFLAGS += $(ZSTD_CFLAGS) -DWITH_ZSTD +endif +endif + +noinst_LIBRARIES += libxfrm.a diff --git a/lib/xfrm/bzip2.c b/lib/xfrm/bzip2.c new file mode 100644 index 0000000..7e5807d --- /dev/null +++ b/lib/xfrm/bzip2.c @@ -0,0 +1,151 @@ +/* SPDX-License-Identifier: GPL-3.0-or-later */ +/* + * bzip2.c + * + * Copyright (C) 2021 David Oberhollenzer <goliath@infraroot.at> + */ +#include <stdlib.h> +#include <stdio.h> +#include <bzlib.h> + +#include "xfrm/stream.h" +#include "xfrm/compress.h" + +typedef struct { + xfrm_stream_t base; + + bz_stream strm; + bool compress; + bool initialized; + + int level; + int work_factor; +} xfrm_stream_bzip2_t; + +static const int bzlib_action[] = { + [XFRM_STREAM_FLUSH_NONE] = BZ_RUN, + [XFRM_STREAM_FLUSH_SYNC] = BZ_FLUSH, + [XFRM_STREAM_FLUSH_FULL] = BZ_FINISH, +}; + +static int process_data(xfrm_stream_t *stream, const void *in, sqfs_u32 in_size, + void *out, sqfs_u32 out_size, + sqfs_u32 *in_read, sqfs_u32 *out_written, + int flush_mode) +{ + xfrm_stream_bzip2_t *bzip2 = (xfrm_stream_bzip2_t *)stream; + sqfs_u32 diff; + int ret; + + if (!bzip2->initialized) { + if (bzip2->compress) { + ret = BZ2_bzCompressInit(&bzip2->strm, bzip2->level, 0, + bzip2->work_factor); + } else { + ret = BZ2_bzDecompressInit(&bzip2->strm, 0, 0); + } + + if (ret != BZ_OK) + return XFRM_STREAM_ERROR; + + bzip2->initialized = true; + } + + if (flush_mode < 0 || flush_mode >= XFRM_STREAM_FLUSH_COUNT) + flush_mode = XFRM_STREAM_FLUSH_NONE; + + while (in_size > 0 && out_size > 0) { + bzip2->strm.next_in = (char *)in; + bzip2->strm.avail_in = in_size; + + bzip2->strm.next_out = (char *)out; + bzip2->strm.avail_out = out_size; + + if (bzip2->compress) { + ret = BZ2_bzCompress(&bzip2->strm, + bzlib_action[flush_mode]); + } else { + ret = BZ2_bzDecompress(&bzip2->strm); + } + + if (ret == BZ_OUTBUFF_FULL) + return XFRM_STREAM_BUFFER_FULL; + + if (ret < 0) + return XFRM_STREAM_ERROR; + + diff = (in_size - bzip2->strm.avail_in); + in = (const char *)in + diff; + in_size -= diff; + *in_read += diff; + + diff = (out_size - bzip2->strm.avail_out); + out = (char *)out + diff; + out_size -= diff; + *out_written += diff; + + if (ret == BZ_STREAM_END) { + if (bzip2->compress) { + BZ2_bzCompressEnd(&bzip2->strm); + } else { + BZ2_bzDecompressEnd(&bzip2->strm); + } + + bzip2->initialized = false; + return XFRM_STREAM_END; + } + } + + return XFRM_STREAM_OK; +} + +static void destroy(sqfs_object_t *obj) +{ + xfrm_stream_bzip2_t *bzip2 = (xfrm_stream_bzip2_t *)obj; + + if (bzip2->initialized) { + if (bzip2->compress) { + BZ2_bzCompressEnd(&bzip2->strm); + } else { + BZ2_bzDecompressEnd(&bzip2->strm); + } + } + + free(bzip2); +} + +static xfrm_stream_t *stream_create(const compressor_config_t *cfg, + bool compress) +{ + xfrm_stream_bzip2_t *bzip2 = calloc(1, sizeof(*bzip2)); + xfrm_stream_t *xfrm = (xfrm_stream_t *)bzip2; + + if (bzip2 == NULL) { + perror("creating bzip2 stream compressor"); + return NULL; + } + + if (cfg == NULL) { + bzip2->level = COMP_BZIP2_DEFAULT_LEVEL; + bzip2->work_factor = COMP_BZIP2_DEFAULT_WORK_FACTOR; + } else { + bzip2->level = cfg->level; + bzip2->work_factor = cfg->opt.bzip2.work_factor; + } + + bzip2->initialized = false; + bzip2->compress = compress; + xfrm->process_data = process_data; + sqfs_object_init(bzip2, destroy, NULL); + return xfrm; +} + +xfrm_stream_t *compressor_stream_bzip2_create(const compressor_config_t *cfg) +{ + return stream_create(cfg, true); +} + +xfrm_stream_t *decompressor_stream_bzip2_create(void) +{ + return stream_create(NULL, false); +} diff --git a/lib/xfrm/compress.c b/lib/xfrm/compress.c new file mode 100644 index 0000000..fbd6987 --- /dev/null +++ b/lib/xfrm/compress.c @@ -0,0 +1,102 @@ +/* SPDX-License-Identifier: GPL-3.0-or-later */ +/* + * compress.c + * + * Copyright (C) 2019 David Oberhollenzer <goliath@infraroot.at> + */ +#include "xfrm/compress.h" +#include "config.h" + +#include <string.h> + +static const struct { + int id; + const char *name; + const sqfs_u8 *magic; + size_t count; + xfrm_stream_t *(*mk_comp_stream)(const compressor_config_t *); + xfrm_stream_t *(*mk_decomp_stream)(void); +} compressors[] = { +#ifdef WITH_GZIP + { XFRM_COMPRESSOR_GZIP, "gzip", (const sqfs_u8 *)"\x1F\x8B\x08", 3, + compressor_stream_gzip_create, decompressor_stream_gzip_create }, +#endif +#ifdef WITH_XZ + { XFRM_COMPRESSOR_XZ, "xz", (const sqfs_u8 *)("\xFD" "7zXZ"), 6, + compressor_stream_xz_create, decompressor_stream_xz_create }, +#endif +#if defined(WITH_ZSTD) && defined(HAVE_ZSTD_STREAM) + { XFRM_COMPRESSOR_ZSTD, "zstd", + (const sqfs_u8 *)"\x28\xB5\x2F\xFD", 4, + compressor_stream_zstd_create, decompressor_stream_zstd_create }, +#endif +#ifdef WITH_BZIP2 + { XFRM_COMPRESSOR_BZIP2, "bzip2", (const sqfs_u8 *)"BZh", 3, + compressor_stream_bzip2_create, decompressor_stream_bzip2_create }, +#endif +}; + +int xfrm_compressor_id_from_name(const char *name) +{ + size_t i; + + for (i = 0; i < sizeof(compressors) / sizeof(compressors[0]); ++i) { + if (strcmp(name, compressors[i].name) == 0) + return compressors[i].id; + } + + return -1; +} + +int xfrm_compressor_id_from_magic(const void *data, size_t count) +{ + size_t i; + int ret; + + for (i = 0; i < sizeof(compressors) / sizeof(compressors[0]); ++i) { + if (compressors[i].count > count) + continue; + + ret = memcmp(compressors[i].magic, data, compressors[i].count); + if (ret == 0) + return compressors[i].id; + } + + return -1; +} + +const char *xfrm_compressor_name_from_id(int id) +{ + size_t i; + + for (i = 0; i < sizeof(compressors) / sizeof(compressors[0]); ++i) { + if (compressors[i].id == id) + return compressors[i].name; + } + + return NULL; +} + +xfrm_stream_t *compressor_stream_create(int id, const compressor_config_t *cfg) +{ + size_t i; + + for (i = 0; i < sizeof(compressors) / sizeof(compressors[0]); ++i) { + if (compressors[i].id == id) + return compressors[i].mk_comp_stream(cfg); + } + + return NULL; +} + +xfrm_stream_t *decompressor_stream_create(int id) +{ + size_t i; + + for (i = 0; i < sizeof(compressors) / sizeof(compressors[0]); ++i) { + if (compressors[i].id == id) + return compressors[i].mk_decomp_stream(); + } + + return NULL; +} diff --git a/lib/xfrm/gzip.c b/lib/xfrm/gzip.c new file mode 100644 index 0000000..67224f7 --- /dev/null +++ b/lib/xfrm/gzip.c @@ -0,0 +1,144 @@ +/* SPDX-License-Identifier: GPL-3.0-or-later */ +/* + * gzip.c + * + * Copyright (C) 2021 David Oberhollenzer <goliath@infraroot.at> + */ +#include <stdlib.h> +#include <stdio.h> +#include <zlib.h> + +#include "xfrm/stream.h" +#include "xfrm/compress.h" + +typedef struct { + xfrm_stream_t base; + + z_stream strm; + bool compress; +} xfrm_stream_gzip_t; + +static const int zlib_action[] = { + [XFRM_STREAM_FLUSH_NONE] = Z_NO_FLUSH, + [XFRM_STREAM_FLUSH_SYNC] = Z_SYNC_FLUSH, + [XFRM_STREAM_FLUSH_FULL] = Z_FINISH, +}; + +static int process_data(xfrm_stream_t *stream, const void *in, + sqfs_u32 in_size, void *out, sqfs_u32 out_size, + sqfs_u32 *in_read, sqfs_u32 *out_written, + int flush_mode) +{ + xfrm_stream_gzip_t *gzip = (xfrm_stream_gzip_t *)stream; + sqfs_u32 diff; + int ret; + + if (flush_mode < 0 || flush_mode >= XFRM_STREAM_FLUSH_COUNT) + flush_mode = XFRM_STREAM_FLUSH_NONE; + + while (in_size > 0 && out_size > 0) { + gzip->strm.next_in = (void *)in; + gzip->strm.avail_in = in_size; + + gzip->strm.next_out = out; + gzip->strm.avail_out = out_size; + + if (gzip->compress) { + ret = deflate(&gzip->strm, zlib_action[flush_mode]); + } else { + ret = inflate(&gzip->strm, zlib_action[flush_mode]); + } + + if (ret == Z_STREAM_ERROR) + return XFRM_STREAM_ERROR; + + diff = in_size - gzip->strm.avail_in; + in = (const char *)in + diff; + in_size -= diff; + *in_read += diff; + + diff = out_size - gzip->strm.avail_out; + out = (char *)out + diff; + out_size -= diff; + *out_written += diff; + + if (ret == Z_STREAM_END) { + if (gzip->compress) { + ret = deflateReset(&gzip->strm); + } else { + ret = inflateReset(&gzip->strm); + } + + if (ret != Z_OK) + return XFRM_STREAM_ERROR; + + return XFRM_STREAM_END; + } + + if (ret == Z_BUF_ERROR) + return XFRM_STREAM_BUFFER_FULL; + } + + return XFRM_STREAM_OK; +} + +static void destroy(sqfs_object_t *obj) +{ + xfrm_stream_gzip_t *gzip = (xfrm_stream_gzip_t *)obj; + + if (gzip->compress) { + deflateEnd(&gzip->strm); + } else { + inflateEnd(&gzip->strm); + } + free(gzip); +} + +static xfrm_stream_t *create_stream(const compressor_config_t *cfg, + bool compress) +{ + xfrm_stream_gzip_t *gzip = calloc(1, sizeof(*gzip)); + xfrm_stream_t *xfrm = (xfrm_stream_t *)gzip; + int ret; + + if (gzip == NULL) { + perror("creating gzip stream compressor"); + return NULL; + } + + if (compress) { + int level = COMP_GZIP_DEFAULT_LEVEL; + int wnd = COMP_GZIP_DEFAULT_WINDOW; + + if (cfg != NULL) { + level = cfg->level; + wnd = cfg->opt.gzip.window_size; + } + + ret = deflateInit2(&gzip->strm, level, Z_DEFLATED, + wnd + 16, 8, Z_DEFAULT_STRATEGY); + } else { + ret = inflateInit2(&gzip->strm, 16 + 15); + } + + if (ret != Z_OK) { + fputs("internal error creating gzip compressor.\n", stderr); + free(gzip); + return NULL; + } + + gzip->compress = compress; + xfrm->process_data = process_data; + sqfs_object_init(xfrm, destroy, NULL); + return xfrm; +} + +xfrm_stream_t *compressor_stream_gzip_create(const compressor_config_t *cfg) +{ + return create_stream(cfg, true); +} + +xfrm_stream_t *decompressor_stream_gzip_create(void) +{ + return create_stream(NULL, false); +} diff --git a/lib/xfrm/xz.c b/lib/xfrm/xz.c new file mode 100644 index 0000000..5adb6f3 --- /dev/null +++ b/lib/xfrm/xz.c @@ -0,0 +1,195 @@ +/* SPDX-License-Identifier: GPL-3.0-or-later */ +/* + * xz.c + * + * Copyright (C) 2021 David Oberhollenzer <goliath@infraroot.at> + */ +#include <stdlib.h> +#include <stdio.h> +#include <lzma.h> + +#include "xfrm/stream.h" +#include "xfrm/compress.h" + +typedef struct { + xfrm_stream_t base; + + lzma_stream strm; + + sqfs_u64 memlimit; + lzma_filter filters[3]; + lzma_options_lzma opt; + lzma_vli vli_filter; + sqfs_u32 presets; + + bool compress; + bool initialized; +} xfrm_xz_t; + +static const lzma_action xzlib_action[] = { + [XFRM_STREAM_FLUSH_NONE] = LZMA_RUN, + [XFRM_STREAM_FLUSH_SYNC] = LZMA_FULL_FLUSH, + [XFRM_STREAM_FLUSH_FULL] = LZMA_FINISH, +}; + +static lzma_vli vli_filter_from_flags(int vli) +{ + switch (vli) { + case COMP_XZ_VLI_X86: + return LZMA_FILTER_X86; + case COMP_XZ_VLI_POWERPC: + return LZMA_FILTER_POWERPC; + case COMP_XZ_VLI_IA64: + return LZMA_FILTER_IA64; + case COMP_XZ_VLI_ARM: + return LZMA_FILTER_ARM; + case COMP_XZ_VLI_ARMTHUMB: + return LZMA_FILTER_ARMTHUMB; + case COMP_XZ_VLI_SPARC: + return LZMA_FILTER_SPARC; + default: + return LZMA_VLI_UNKNOWN; + } +} + +static int process_data(xfrm_stream_t *stream, const void *in, + sqfs_u32 in_size, void *out, sqfs_u32 out_size, + sqfs_u32 *in_read, sqfs_u32 *out_written, + int flush_mode) +{ + xfrm_xz_t *xz = (xfrm_xz_t *)stream; + lzma_ret ret_xz; + sqfs_u32 diff; + + if (!xz->initialized) { + if (xz->compress) { + ret_xz = lzma_stream_encoder(&xz->strm, xz->filters, + LZMA_CHECK_CRC32); + } else { + ret_xz = lzma_stream_decoder(&xz->strm, + xz->memlimit, 0); + } + + if (ret_xz != LZMA_OK) + return XFRM_STREAM_ERROR; + + xz->initialized = true; + } + + if (flush_mode < 0 || flush_mode >= XFRM_STREAM_FLUSH_COUNT) + flush_mode = XFRM_STREAM_FLUSH_NONE; + + while (in_size > 0 && out_size > 0) { + xz->strm.next_in = in; + xz->strm.avail_in = in_size; + + xz->strm.next_out = out; + xz->strm.avail_out = out_size; + + ret_xz = lzma_code(&xz->strm, xzlib_action[flush_mode]); + + if (ret_xz != LZMA_OK && ret_xz != LZMA_BUF_ERROR && + ret_xz != LZMA_STREAM_END) { + return XFRM_STREAM_ERROR; + } + + diff = in_size - xz->strm.avail_in; + in = (const char *)in + diff; + in_size -= diff; + *in_read += diff; + + diff = out_size - xz->strm.avail_out; + out = (char *)out + diff; + out_size -= diff; + *out_written += diff; + + if (ret_xz == LZMA_BUF_ERROR) + return XFRM_STREAM_BUFFER_FULL; + + if (ret_xz == LZMA_STREAM_END) { + lzma_end(&xz->strm); + xz->initialized = false; + return XFRM_STREAM_END; + } + } + + return XFRM_STREAM_OK; +} + +static void destroy(sqfs_object_t *obj) +{ + xfrm_xz_t *xz = (xfrm_xz_t *)obj; + + if (xz->initialized) + lzma_end(&xz->strm); + + free(xz); +} + +static xfrm_stream_t *create_stream(const compressor_config_t *cfg, + bool compress) +{ + xfrm_xz_t *xz = calloc(1, sizeof(*xz)); + xfrm_stream_t *xfrm = (xfrm_stream_t *)xz; + int i = 0; + + if (xz == NULL) { + perror("creating xz stream compressor"); + return NULL; + } + + xz->memlimit = 128 * 1024 * 1024; + xz->compress = compress; + xz->initialized = false; + + if (compress) { + if (cfg == NULL) { + xz->presets = COMP_XZ_DEFAULT_LEVEL; + } else { + xz->presets = cfg->level; + if (cfg->flags & COMP_FLAG_XZ_EXTREME) + xz->presets |= LZMA_PRESET_EXTREME; + } + + if (lzma_lzma_preset(&xz->opt, xz->presets)) + goto fail_init; + + if (cfg == NULL) { + xz->vli_filter = LZMA_VLI_UNKNOWN; + } else { + xz->vli_filter = vli_filter_from_flags(cfg->opt.xz.vli); + } + + if (xz->vli_filter != LZMA_VLI_UNKNOWN) { + xz->filters[i].id = xz->vli_filter; + xz->filters[i].options = NULL; + ++i; + } + + xz->filters[i].id = LZMA_FILTER_LZMA2; + xz->filters[i].options = &xz->opt; + ++i; + + xz->filters[i].id = LZMA_VLI_UNKNOWN; + xz->filters[i].options = NULL; + ++i; + } + + xfrm->process_data = process_data; + sqfs_object_init(xz, destroy, NULL); + return xfrm; +fail_init: + fputs("error initializing XZ compressor\n", stderr); + free(xz); + return NULL; +} + +xfrm_stream_t *compressor_stream_xz_create(const compressor_config_t *cfg) +{ + return create_stream(cfg, true); +} + +xfrm_stream_t *decompressor_stream_xz_create(void) +{ + return create_stream(NULL, false); +} diff --git a/lib/xfrm/zstd.c b/lib/xfrm/zstd.c new file mode 100644 index 0000000..70666c1 --- /dev/null +++ b/lib/xfrm/zstd.c @@ -0,0 +1,136 @@ +/* SPDX-License-Identifier: GPL-3.0-or-later */ +/* + * zstd.c + * + * Copyright (C) 2021 David Oberhollenzer <goliath@infraroot.at> + */ +#include <string.h> +#include <stdlib.h> +#include <stdio.h> +#include <zstd.h> + +#include "xfrm/stream.h" +#include "xfrm/compress.h" + +typedef struct { + xfrm_stream_t base; + + ZSTD_CStream *cstrm; + ZSTD_DStream *dstrm; + bool compress; +} xfrm_zstd_t; + +static const ZSTD_EndDirective zstd_action[] = { + [XFRM_STREAM_FLUSH_NONE] = ZSTD_e_continue, + [XFRM_STREAM_FLUSH_SYNC] = ZSTD_e_flush, + [XFRM_STREAM_FLUSH_FULL] = ZSTD_e_end, +}; + +static int process_data(xfrm_stream_t *stream, const void *in, + sqfs_u32 in_size, void *out, sqfs_u32 out_size, + sqfs_u32 *in_read, sqfs_u32 *out_written, + int flush_mode) +{ + xfrm_zstd_t *zstd = (xfrm_zstd_t *)stream; + ZSTD_outBuffer out_desc; + ZSTD_inBuffer in_desc; + size_t ret; + + if (flush_mode < 0 || flush_mode >= XFRM_STREAM_FLUSH_COUNT) + flush_mode = XFRM_STREAM_FLUSH_NONE; + + while (in_size > 0 && out_size > 0) { + memset(&in_desc, 0, sizeof(in_desc)); + in_desc.src = in; + in_desc.size = in_size; + + memset(&out_desc, 0, sizeof(out_desc)); + out_desc.dst = out; + out_desc.size = out_size; + + if (zstd->compress) { + ret = ZSTD_compressStream2(zstd->cstrm, &out_desc, + &in_desc, + zstd_action[flush_mode]); + } else { + ret = ZSTD_decompressStream(zstd->dstrm, &out_desc, + &in_desc); + } + + if (ZSTD_isError(ret)) + return XFRM_STREAM_ERROR; + + in = (const char *)in + in_desc.pos; + in_size -= in_desc.pos; + *in_read += in_desc.pos; + + out = (char *)out + out_desc.pos; + out_size -= out_desc.pos; + *out_written += out_desc.pos; + } + + if (flush_mode != XFRM_STREAM_FLUSH_NONE) { + if (in_size == 0) + return XFRM_STREAM_END; + } + + if (in_size > 0 && out_size == 0) + return XFRM_STREAM_BUFFER_FULL; + + return XFRM_STREAM_OK; +} + +static void destroy(sqfs_object_t *obj) +{ + xfrm_zstd_t *zstd = (xfrm_zstd_t *)obj; + + if (zstd->compress) { + ZSTD_freeCStream(zstd->cstrm); + } else { + ZSTD_freeDStream(zstd->dstrm); + } + + free(zstd); +} + +static xfrm_stream_t *stream_create(const compressor_config_t *cfg, + bool compress) +{ + xfrm_zstd_t *zstd = calloc(1, sizeof(*zstd)); + xfrm_stream_t *strm = (xfrm_stream_t *)zstd; + (void)cfg; + + if (zstd == NULL) { + perror("creating zstd stream compressor"); + return NULL; + } + + if (compress) { + zstd->cstrm = ZSTD_createCStream(); + if (zstd->cstrm == NULL) + goto fail_strm; + } else { + zstd->dstrm = ZSTD_createDStream(); + if (zstd->dstrm == NULL) + goto fail_strm; + } + + zstd->compress = compress; + strm->process_data = process_data; + sqfs_object_init(zstd, destroy, NULL); + return strm; +fail_strm: + fputs("error initializing zstd stream.\n", stderr); + free(zstd); + return NULL; +} + +xfrm_stream_t *compressor_stream_zstd_create(const compressor_config_t *cfg) +{ + return stream_create(cfg, true); +} + +xfrm_stream_t *decompressor_stream_zstd_create(void) +{ + return stream_create(NULL, false); +} diff --git a/tests/libio/Makemodule.am b/tests/libio/Makemodule.am index df2e9d3..3676d1a 100644 --- a/tests/libio/Makemodule.am +++ b/tests/libio/Makemodule.am @@ -3,66 +3,9 @@ test_get_line_LDADD = libio.a libcompat.a test_get_line_CPPFLAGS = $(AM_CPPFLAGS) test_get_line_CPPFLAGS += -DTESTFILE=$(top_srcdir)/tests/libio/get_line.txt -test_xfrm_bzip2_SOURCES = tests/libio/uncompress.c -test_xfrm_bzip2_LDADD = libio.a libcompat.a $(BZIP2_LIBS) $(ZLIB_LIBS) -test_xfrm_bzip2_LDADD += $(XZ_LIBS) $(ZSTD_LIBS) -test_xfrm_bzip2_CPPFLAGS = $(AM_CPPFLAGS) -DTEST_BZIP2=1 - -test_xfrm_bzip22_SOURCES = tests/libio/uncompress.c -test_xfrm_bzip22_LDADD = libio.a libcompat.a $(BZIP2_LIBS) $(ZLIB_LIBS) -test_xfrm_bzip22_LDADD += $(XZ_LIBS) $(ZSTD_LIBS) -test_xfrm_bzip22_CPPFLAGS = $(AM_CPPFLAGS) -DTEST_BZIP22=1 - -test_xfrm_xz_SOURCES = tests/libio/uncompress.c -test_xfrm_xz_LDADD = libio.a libcompat.a $(BZIP2_LIBS) $(ZLIB_LIBS) -test_xfrm_xz_LDADD += $(XZ_LIBS) $(ZSTD_LIBS) -test_xfrm_xz_CPPFLAGS = $(AM_CPPFLAGS) -DTEST_XZ=1 - -test_xfrm_xz2_SOURCES = tests/libio/uncompress.c -test_xfrm_xz2_LDADD = libio.a libcompat.a $(BZIP2_LIBS) $(ZLIB_LIBS) -test_xfrm_xz2_LDADD += $(XZ_LIBS) $(ZSTD_LIBS) -test_xfrm_xz2_CPPFLAGS = $(AM_CPPFLAGS) -DTEST_XZ2=1 - -test_xfrm_gzip_SOURCES = tests/libio/uncompress.c -test_xfrm_gzip_LDADD = libio.a libcompat.a $(BZIP2_LIBS) $(ZLIB_LIBS) -test_xfrm_gzip_LDADD += $(XZ_LIBS) $(ZSTD_LIBS) -test_xfrm_gzip_CPPFLAGS = $(AM_CPPFLAGS) -DTEST_GZIP=1 - -test_xfrm_zstd_SOURCES = tests/libio/uncompress.c -test_xfrm_zstd_LDADD = libio.a libcompat.a $(BZIP2_LIBS) $(ZLIB_LIBS) -test_xfrm_zstd_LDADD += $(XZ_LIBS) $(ZSTD_LIBS) -test_xfrm_zstd_CPPFLAGS = $(AM_CPPFLAGS) -DTEST_ZSTD=1 - -test_xfrm_zstd2_SOURCES = tests/libio/uncompress.c -test_xfrm_zstd2_LDADD = libio.a libcompat.a $(BZIP2_LIBS) $(ZLIB_LIBS) -test_xfrm_zstd2_LDADD += $(XZ_LIBS) $(ZSTD_LIBS) -test_xfrm_zstd2_CPPFLAGS = $(AM_CPPFLAGS) -DTEST_ZSTD2=1 - if BUILD_TOOLS check_PROGRAMS += test_get_line TESTS += test_get_line - -if WITH_BZIP2 -check_PROGRAMS += test_xfrm_bzip2 test_xfrm_bzip22 -TESTS += test_xfrm_bzip2 test_xfrm_bzip22 -endif - -if WITH_XZ -check_PROGRAMS += test_xfrm_xz test_xfrm_xz2 -TESTS += test_xfrm_xz test_xfrm_xz2 -endif - -if WITH_GZIP -check_PROGRAMS += test_xfrm_gzip -TESTS += test_xfrm_gzip -endif - -if WITH_ZSTD -if HAVE_ZSTD_STREAM -check_PROGRAMS += test_xfrm_zstd test_xfrm_zstd2 -TESTS += test_xfrm_zstd test_xfrm_zstd2 -endif -endif endif EXTRA_DIST += $(top_srcdir)/tests/libio/get_line.txt diff --git a/tests/libxfrm/Makemodule.am b/tests/libxfrm/Makemodule.am new file mode 100644 index 0000000..706406c --- /dev/null +++ b/tests/libxfrm/Makemodule.am @@ -0,0 +1,55 @@ +LIBXFRM_LIBS = $(ZLIB_LIBS) $(XZ_LIBS) $(BZIP2_LIBS) $(ZSTD_LIBS) +LIBXFRM_TESTS = + +if WITH_GZIP +test_unpack_gzip_SOURCES = tests/libxfrm/unpack.c +test_unpack_gzip_LDADD = libxfrm.a $(ZLIB_LIBS) +test_unpack_gzip_CPPFLAGS = $(AM_CPPFLAGS) -DDO_GZIP=1 + +test_pack_gzip_SOURCES = tests/libxfrm/pack.c +test_pack_gzip_LDADD = libxfrm.a $(LIBXFRM_LIBS) +test_pack_gzip_CPPFLAGS = $(AM_CPPFLAGS) -DDO_GZIP=1 + +LIBXFRM_TESTS += test_pack_gzip test_unpack_gzip +endif + +if WITH_XZ +test_unpack_xz_SOURCES = tests/libxfrm/unpack.c +test_unpack_xz_LDADD = libxfrm.a $(XZ_LIBS) +test_unpack_xz_CPPFLAGS = $(AM_CPPFLAGS) -DDO_XZ=1 + +test_pack_xz_SOURCES = tests/libxfrm/pack.c +test_pack_xz_LDADD = libxfrm.a $(LIBXFRM_LIBS) +test_pack_xz_CPPFLAGS = $(AM_CPPFLAGS) -DDO_XZ=1 + +LIBXFRM_TESTS += test_pack_xz test_unpack_xz +endif + +if WITH_BZIP2 +test_unpack_bzip2_SOURCES = tests/libxfrm/unpack.c +test_unpack_bzip2_LDADD = libxfrm.a $(BZIP2_LIBS) +test_unpack_bzip2_CPPFLAGS = $(AM_CPPFLAGS) -DDO_BZIP2=1 + +test_pack_bzip2_SOURCES = tests/libxfrm/pack.c +test_pack_bzip2_LDADD = libxfrm.a $(LIBXFRM_LIBS) +test_pack_bzip2_CPPFLAGS = $(AM_CPPFLAGS) -DDO_BZIP2=1 + +LIBXFRM_TESTS += test_unpack_bzip2 test_pack_bzip2 +endif + +if WITH_ZSTD +if HAVE_ZSTD_STREAM +test_pack_zstd_SOURCES = tests/libxfrm/unpack.c +test_pack_zstd_LDADD = libxfrm.a $(ZSTD_LIBS) +test_pack_zstd_CPPFLAGS = $(AM_CPPFLAGS) -DDO_ZSTD=1 + +test_unpack_zstd_SOURCES = tests/libxfrm/pack.c +test_unpack_zstd_LDADD = libxfrm.a $(LIBXFRM_LIBS) +test_unpack_zstd_CPPFLAGS = $(AM_CPPFLAGS) -DDO_ZSTD=1 + +LIBXFRM_TESTS += test_pack_zstd test_unpack_zstd +endif +endif + +check_PROGRAMS += $(LIBXFRM_TESTS) +TESTS += $(LIBXFRM_TESTS) diff --git a/tests/libxfrm/pack.c b/tests/libxfrm/pack.c new file mode 100644 index 0000000..c297005 --- /dev/null +++ b/tests/libxfrm/pack.c @@ -0,0 +1,165 @@ +/* SPDX-License-Identifier: GPL-3.0-or-later */ +/* + * compress.c + * + * Copyright (C) 2022 David Oberhollenzer <goliath@infraroot.at> + */ +#include "xfrm/compress.h" +#include "xfrm/stream.h" +#include "util/test.h" + +static const char text[] = +"Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod\n" +"tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam,\n" +"quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo\n" +"consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse\n" +"cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non\n" +"proident, sunt in culpa qui officia deserunt mollit anim id est laborum.\n"; + +static sqfs_u8 buffer_cmp[1024]; +static sqfs_u8 buffer_plain[1024]; +static sqfs_u8 ref_cmp[1024]; +static size_t ref_size; + +#if defined(DO_BZIP2) +#define mkdecompressor decompressor_stream_bzip2_create +#define mkcompressor compressor_stream_bzip2_create + +static sqfs_u8 magic[3] = "BZh"; +static int comp_id = XFRM_COMPRESSOR_BZIP2; +static const char *comp_name = "bzip2"; +#elif defined(DO_XZ) +#define mkdecompressor decompressor_stream_xz_create +#define mkcompressor compressor_stream_xz_create + +static sqfs_u8 magic[6] = "\xFD" "7zXZ"; +static int comp_id = XFRM_COMPRESSOR_XZ; +static const char *comp_name = "xz"; +#elif defined(DO_GZIP) +#define mkdecompressor decompressor_stream_gzip_create +#define mkcompressor compressor_stream_gzip_create + +static sqfs_u8 magic[3] = "\x1F\x8B\x08"; +static int comp_id = XFRM_COMPRESSOR_GZIP; +static const char *comp_name = "gzip"; +#elif defined(DO_ZSTD) +#define mkdecompressor decompressor_stream_zstd_create +#define mkcompressor compressor_stream_zstd_create + +static sqfs_u8 magic[4] = "\x28\xB5\x2F\xFD"; +static int comp_id = XFRM_COMPRESSOR_ZSTD; +static const char *comp_name = "zstd"; +#else +#error build configuration broken +#endif + +int main(int argc, char **argv) +{ + sqfs_u32 in_diff = 0, out_diff = 0; + xfrm_stream_t *xfrm; + const char *str; + size_t size; + int ret; + (void)argc; (void)argv; + + /* generic name/ID API */ + ret = xfrm_compressor_id_from_name(comp_name); + TEST_EQUAL_I(ret, comp_id); + + str = xfrm_compressor_name_from_id(ret); + TEST_STR_EQUAL(str, comp_name); + + /* compress the original text */ + xfrm = mkcompressor(NULL); + TEST_NOT_NULL(xfrm); + TEST_EQUAL_UI(((sqfs_object_t *)xfrm)->refcount, 1); + + ret = xfrm->process_data(xfrm, text, sizeof(text), + buffer_cmp, sizeof(buffer_cmp), + &in_diff, &out_diff, XFRM_STREAM_FLUSH_FULL); + TEST_EQUAL_I(ret, XFRM_STREAM_END); + + TEST_EQUAL_UI(in_diff, sizeof(text)); + TEST_ASSERT(out_diff > 0); + TEST_ASSERT(out_diff < sizeof(text)); + + sqfs_drop(xfrm); + + size = out_diff; + in_diff = out_diff = 0; + + memcpy(ref_cmp, buffer_cmp, size); + ref_size = size; + + /* check if it has the expected magic number */ + TEST_ASSERT(size >= sizeof(magic)); + ret = memcmp(buffer_cmp, magic, sizeof(magic)); + TEST_EQUAL_I(ret, 0); + + /* check if the auto detection correctly identifies it */ + ret = xfrm_compressor_id_from_magic(buffer_cmp, size); + TEST_EQUAL_I(ret, comp_id); + + ret = xfrm_compressor_id_from_magic(text, sizeof(text)); + TEST_EQUAL_I(ret, -1); + + /* unpack the compressed text and compare to the original */ + xfrm = mkdecompressor(); + TEST_NOT_NULL(xfrm); + TEST_EQUAL_UI(((sqfs_object_t *)xfrm)->refcount, 1); + + ret = xfrm->process_data(xfrm, buffer_cmp, size, + buffer_plain, sizeof(buffer_plain), + &in_diff, &out_diff, 0); + TEST_ASSERT(ret == XFRM_STREAM_END || ret == XFRM_STREAM_OK); + + TEST_EQUAL_UI(in_diff, size); + TEST_EQUAL_UI(out_diff, sizeof(text)); + ret = memcmp(buffer_plain, text, out_diff); + TEST_EQUAL_I(ret, 0); + + sqfs_drop(xfrm); + in_diff = out_diff = 0; + + /* retry packing but create the compressor using the ID */ + memset(buffer_cmp, 0, sizeof(buffer_cmp)); + memset(buffer_plain, 0, sizeof(buffer_plain)); + + xfrm = compressor_stream_create(comp_id, NULL); + TEST_NOT_NULL(xfrm); + TEST_EQUAL_UI(((sqfs_object_t *)xfrm)->refcount, 1); + + ret = xfrm->process_data(xfrm, text, sizeof(text), + buffer_cmp, sizeof(buffer_cmp), + &in_diff, &out_diff, XFRM_STREAM_FLUSH_FULL); + TEST_EQUAL_I(ret, XFRM_STREAM_END); + + TEST_EQUAL_UI(in_diff, sizeof(text)); + TEST_EQUAL_UI(out_diff, ref_size); + + sqfs_drop(xfrm); + size = out_diff; + in_diff = out_diff = 0; + + /* make sure we got an identical result */ + ret = memcmp(ref_cmp, buffer_cmp, size); + TEST_EQUAL_I(ret, 0); + + /* decompress it using ID */ + xfrm = decompressor_stream_create(comp_id); + TEST_NOT_NULL(xfrm); + TEST_EQUAL_UI(((sqfs_object_t *)xfrm)->refcount, 1); + + ret = xfrm->process_data(xfrm, buffer_cmp, size, + buffer_plain, sizeof(buffer_plain), + &in_diff, &out_diff, 0); + TEST_ASSERT(ret == XFRM_STREAM_END || ret == XFRM_STREAM_OK); + + TEST_EQUAL_UI(in_diff, size); + TEST_EQUAL_UI(out_diff, sizeof(text)); + ret = memcmp(buffer_plain, text, out_diff); + TEST_EQUAL_I(ret, 0); + + sqfs_drop(xfrm); + return EXIT_SUCCESS; +} diff --git a/tests/libio/uncompress.c b/tests/libxfrm/unpack.c index 4ff20b5..ca51f10 100644 --- a/tests/libio/uncompress.c +++ b/tests/libxfrm/unpack.c @@ -1,105 +1,15 @@ /* SPDX-License-Identifier: GPL-3.0-or-later */ /* - * uncompress.c + * unpack.c * - * Copyright (C) 2021 David Oberhollenzer <goliath@infraroot.at> + * Copyright (C) 2022 David Oberhollenzer <goliath@infraroot.at> */ -#include "io/istream.h" -#include "io/xfrm.h" +#include "xfrm/compress.h" +#include "xfrm/stream.h" #include "util/test.h" -static sqfs_u8 data_in[] = { -#if defined(TEST_BZIP2) - 0x42, 0x5a, 0x68, 0x39, 0x31, 0x41, 0x59, 0x26, - 0x53, 0x59, 0x05, 0x24, 0x28, 0x04, 0x00, 0x00, - 0x27, 0xd7, 0x80, 0x00, 0x10, 0x40, 0x05, 0x06, - 0x04, 0x02, 0x00, 0x3f, 0xe7, 0xff, 0x40, 0x30, - 0x01, 0x2d, 0x23, 0x62, 0x26, 0x05, 0x3d, 0x03, - 0x54, 0xfd, 0x53, 0x4c, 0x86, 0x9e, 0x90, 0x6a, - 0x9e, 0x9e, 0x85, 0x3c, 0xa0, 0x00, 0x00, 0x1a, - 0x9e, 0x41, 0x13, 0x13, 0x28, 0x69, 0x03, 0xd4, - 0x0f, 0x1c, 0x70, 0xd0, 0xb4, 0xe3, 0xe4, 0x75, - 0x4e, 0x8b, 0x67, 0x43, 0x7b, 0x38, 0x27, 0x77, - 0xe4, 0xc1, 0x98, 0x3a, 0x2d, 0x3a, 0xe4, 0x44, - 0x98, 0xdc, 0x49, 0x8b, 0x22, 0x48, 0xfc, 0xc8, - 0xe7, 0x57, 0x05, 0x3c, 0x5a, 0xee, 0x5a, 0x84, - 0xcd, 0x7c, 0x8f, 0x26, 0x6b, 0x6e, 0xf7, 0xb5, - 0x49, 0x1f, 0x79, 0x42, 0x5d, 0x09, 0x8c, 0xc6, - 0xde, 0x0c, 0x0d, 0xb1, 0x46, 0xb4, 0xee, 0xd9, - 0x8f, 0x33, 0x37, 0x04, 0xa9, 0x05, 0x49, 0xe3, - 0x04, 0x16, 0x62, 0x36, 0x3a, 0x01, 0xda, 0xd4, - 0xc8, 0x8a, 0x32, 0x02, 0x1f, 0x62, 0x4b, 0xa4, - 0x49, 0x59, 0xda, 0x50, 0x85, 0x69, 0x35, 0x21, - 0x10, 0xc6, 0x8a, 0x3c, 0x44, 0x95, 0xb0, 0xbc, - 0xc5, 0x6b, 0xea, 0xfb, 0x40, 0xbd, 0x14, 0x01, - 0x6a, 0xfa, 0xcd, 0x67, 0xd8, 0x2d, 0x93, 0x8b, - 0xda, 0x44, 0x1b, 0xe9, 0x5a, 0x87, 0x60, 0xb0, - 0xe0, 0x73, 0xd1, 0x01, 0x3a, 0x66, 0x05, 0xcc, - 0x34, 0xa0, 0x63, 0x8d, 0x35, 0x5e, 0xa0, 0x9f, - 0x05, 0x89, 0x15, 0x51, 0x48, 0x16, 0x0c, 0x61, - 0xf4, 0x30, 0xb8, 0x07, 0x29, 0xc0, 0xf5, 0x1a, - 0xe1, 0x0d, 0x6c, 0xfe, 0x91, 0xda, 0x13, 0x2f, - 0x8e, 0x5b, 0x1c, 0xfc, 0xb3, 0xb2, 0x30, 0x9d, - 0xf6, 0x09, 0x30, 0x55, 0x30, 0x67, 0xc2, 0x87, - 0xe9, 0x9a, 0xd4, 0x1d, 0x66, 0x11, 0x54, 0x89, - 0x21, 0xe1, 0x55, 0x84, 0xbf, 0xa6, 0x11, 0xa4, - 0xb8, 0x40, 0xed, 0x42, 0x20, 0xb9, 0xb7, 0x26, - 0x31, 0x14, 0x4f, 0x86, 0xdc, 0x50, 0x34, 0x38, - 0x8b, 0x57, 0x77, 0x21, 0xf6, 0x89, 0xbd, 0xc5, - 0x65, 0xc3, 0x23, 0x45, 0xec, 0x7f, 0x8b, 0xb9, - 0x22, 0x9c, 0x28, 0x48, 0x02, 0x92, 0x14, 0x02, - 0x00, -#elif defined(TEST_BZIP22) - 0x42, 0x5a, 0x68, 0x39, 0x31, 0x41, 0x59, 0x26, - 0x53, 0x59, 0x5d, 0x09, 0x24, 0x1d, 0x00, 0x00, - 0x13, 0xd7, 0x80, 0x00, 0x10, 0x40, 0x05, 0x00, - 0x04, 0x02, 0x00, 0x3e, 0xa7, 0xff, 0x40, 0x30, - 0x00, 0xac, 0x43, 0x54, 0xf5, 0x36, 0x4c, 0xa7, - 0xa8, 0xd3, 0x6a, 0x60, 0x81, 0x40, 0x00, 0xd0, - 0x32, 0x64, 0x0d, 0x53, 0xda, 0x02, 0x09, 0xa2, - 0x68, 0x34, 0xd1, 0x27, 0x4a, 0xdd, 0xf2, 0x0a, - 0x73, 0x43, 0xf9, 0xa2, 0x51, 0x85, 0x76, 0x45, - 0x9a, 0x68, 0x3a, 0xe7, 0x0d, 0xc0, 0x21, 0x4a, - 0xc4, 0xf9, 0xf7, 0x40, 0xc3, 0x10, 0xb2, 0x9b, - 0x58, 0x56, 0x71, 0x50, 0x2f, 0xa4, 0xc5, 0x61, - 0x19, 0xf6, 0x59, 0x06, 0x82, 0x03, 0x7f, 0xeb, - 0xd2, 0x61, 0x88, 0xcd, 0xe8, 0xf7, 0xe8, 0x87, - 0x59, 0x9d, 0xe1, 0xf8, 0x19, 0x6e, 0xad, 0x77, - 0xbf, 0x34, 0x17, 0x21, 0x6b, 0x91, 0xc9, 0x52, - 0xd0, 0x81, 0x1e, 0xb5, 0x0b, 0xee, 0x42, 0x84, - 0x80, 0xd5, 0xa1, 0x8a, 0x04, 0x18, 0x4d, 0xf3, - 0xda, 0x7e, 0x3c, 0x40, 0xa4, 0xdb, 0xe5, 0xf0, - 0x37, 0x40, 0x3a, 0x7d, 0xa7, 0x45, 0x21, 0xf2, - 0x5a, 0x7b, 0x59, 0x56, 0x16, 0xd5, 0xac, 0x9f, - 0x60, 0x85, 0x0e, 0xf5, 0x73, 0xd9, 0x47, 0xe2, - 0xee, 0x48, 0xa7, 0x0a, 0x12, 0x0b, 0xa1, 0x24, - 0x83, 0xa0, - 0x42, 0x5a, 0x68, 0x39, 0x31, 0x41, 0x59, 0x26, - 0x53, 0x59, 0x2c, 0x24, 0x39, 0xa0, 0x00, 0x00, - 0x1f, 0x55, 0x80, 0x00, 0x10, 0x40, 0x05, 0x06, - 0x00, 0x3f, 0xe7, 0xff, 0x40, 0x30, 0x00, 0xb5, - 0x91, 0x13, 0x4f, 0x54, 0x7a, 0x6a, 0x6d, 0x4d, - 0xa2, 0x68, 0x0c, 0x84, 0x53, 0xf5, 0x30, 0x89, - 0xa3, 0xd4, 0x0d, 0x0f, 0x49, 0xa0, 0xd4, 0xf4, - 0xd1, 0x53, 0xf4, 0x93, 0x69, 0x3c, 0x81, 0x1a, - 0x65, 0x53, 0x90, 0x51, 0x07, 0x2a, 0xad, 0x8f, - 0x63, 0xba, 0x25, 0xc2, 0x0c, 0x8b, 0xb9, 0x95, - 0x15, 0xd8, 0xda, 0x61, 0x5c, 0xa9, 0xe4, 0x0b, - 0x21, 0xc9, 0x97, 0x57, 0x01, 0x28, 0x9b, 0xfb, - 0x94, 0xb9, 0x48, 0xa3, 0x0a, 0xc6, 0x1c, 0x54, - 0x98, 0x9a, 0x39, 0xc3, 0x87, 0x90, 0x33, 0x58, - 0x2d, 0x3e, 0x16, 0xb1, 0xae, 0x26, 0x89, 0x75, - 0xf5, 0x77, 0xa5, 0x8e, 0x5b, 0x8c, 0x8a, 0x39, - 0xbd, 0x75, 0x21, 0x9d, 0x99, 0x18, 0x4a, 0x91, - 0xab, 0xbc, 0x08, 0x87, 0xa4, 0xf1, 0x81, 0xb5, - 0xb4, 0xb0, 0xfe, 0x6b, 0x9f, 0xbe, 0x19, 0x82, - 0xd1, 0x50, 0xe1, 0x5e, 0x13, 0xb5, 0xc6, 0x2c, - 0xa4, 0x82, 0xf2, 0x5c, 0xc3, 0x20, 0x41, 0x13, - 0x56, 0x63, 0x3d, 0xec, 0x71, 0x2a, 0xbf, 0x2c, - 0x60, 0x2f, 0x7a, 0x4d, 0xcb, 0x3f, 0x8b, 0xb9, - 0x22, 0x9c, 0x28, 0x48, 0x16, 0x12, 0x1c, 0xd0, - 0x00, -#elif defined(TEST_XZ) +static const sqfs_u8 blob_in[] = { +#if defined(DO_XZ) 0xfd, 0x37, 0x7a, 0x58, 0x5a, 0x00, 0x00, 0x00, 0xff, 0x12, 0xd9, 0x41, 0x02, 0x00, 0x21, 0x01, 0x1c, 0x00, 0x00, 0x00, 0x10, 0xcf, 0x58, 0xcc, @@ -148,71 +58,48 @@ static sqfs_u8 data_in[] = { 0xbe, 0x03, 0x00, 0x00, 0xda, 0x2c, 0x45, 0x49, 0xa8, 0x00, 0x0a, 0xfc, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x59, 0x5a -#elif defined(TEST_XZ2) - 0xfd, 0x37, 0x7a, 0x58, 0x5a, 0x00, 0x00, 0x04, - 0xe6, 0xd6, 0xb4, 0x46, 0x02, 0x00, 0x21, 0x01, - 0x16, 0x00, 0x00, 0x00, 0x74, 0x2f, 0xe5, 0xa3, - 0xe0, 0x00, 0xdc, 0x00, 0xb3, 0x5d, 0x00, 0x26, - 0x1b, 0xca, 0x46, 0x67, 0x5a, 0xf2, 0x77, 0xb8, - 0x7d, 0x86, 0xd8, 0x41, 0xdb, 0x05, 0x35, 0xcd, - 0x83, 0xa5, 0x7c, 0x12, 0xa5, 0x05, 0xdb, 0x90, - 0xbd, 0x2f, 0x14, 0xd3, 0x71, 0x72, 0x96, 0xa8, - 0x8a, 0x7d, 0x84, 0x56, 0x71, 0x8d, 0x6a, 0x22, - 0x98, 0xab, 0x9e, 0x3d, 0xc3, 0x55, 0xef, 0xcc, - 0xa5, 0xc3, 0xdd, 0x5b, 0x8e, 0xbf, 0x03, 0x81, - 0x21, 0x40, 0xd6, 0x26, 0x91, 0x02, 0x45, 0x4e, - 0x20, 0x91, 0xcf, 0x8c, 0x51, 0x22, 0x02, 0x70, - 0xba, 0x05, 0x6b, 0x83, 0xef, 0x3f, 0x8e, 0x09, - 0xef, 0x88, 0xf5, 0x37, 0x1b, 0x89, 0x8d, 0xff, - 0x1e, 0xee, 0xe8, 0xb0, 0xac, 0xf2, 0x6e, 0xd4, - 0x3e, 0x25, 0xaf, 0xa0, 0x6d, 0x2e, 0xc0, 0x7f, - 0xb5, 0xa0, 0xcb, 0x90, 0x1f, 0x08, 0x1a, 0xe2, - 0x90, 0x20, 0x19, 0x71, 0x0c, 0xe8, 0x3f, 0xe5, - 0x39, 0xeb, 0x9a, 0x62, 0x4f, 0x06, 0xda, 0x3c, - 0x32, 0x59, 0xcc, 0x83, 0xe3, 0x83, 0x0f, 0x38, - 0x7d, 0x43, 0x37, 0x6c, 0x0b, 0x05, 0x65, 0x98, - 0x25, 0xdb, 0xf2, 0xc0, 0x2d, 0x39, 0x36, 0x5d, - 0xd4, 0xb6, 0xc2, 0x79, 0x73, 0x3e, 0xc2, 0x6e, - 0x54, 0xec, 0x78, 0x2b, 0x5d, 0xf1, 0xd1, 0xb4, - 0xb3, 0xcd, 0xf3, 0x89, 0xf5, 0x80, 0x79, 0x46, - 0xc0, 0x00, 0x00, 0x00, 0xc4, 0xf5, 0x1d, 0x08, - 0xf0, 0x34, 0x3a, 0x59, 0x00, 0x01, 0xcf, 0x01, - 0xdd, 0x01, 0x00, 0x00, 0x7f, 0x5a, 0x77, 0xcb, - 0xb1, 0xc4, 0x67, 0xfb, 0x02, 0x00, 0x00, 0x00, - 0x00, 0x04, 0x59, 0x5a, - 0xfd, 0x37, 0x7a, 0x58, 0x5a, 0x00, 0x00, 0x04, - 0xe6, 0xd6, 0xb4, 0x46, 0x02, 0x00, 0x21, 0x01, - 0x16, 0x00, 0x00, 0x00, 0x74, 0x2f, 0xe5, 0xa3, - 0xe0, 0x00, 0xe0, 0x00, 0xb7, 0x5d, 0x00, 0x31, - 0x9b, 0xca, 0x19, 0xc5, 0x54, 0xec, 0xb6, 0x54, - 0xe7, 0xb1, 0x7d, 0xc4, 0x57, 0x9e, 0x6c, 0x89, - 0xad, 0x4a, 0x6d, 0x16, 0xd8, 0x3c, 0x05, 0x94, - 0x10, 0x16, 0x99, 0x38, 0x21, 0xa3, 0xb9, 0xc5, - 0x80, 0xff, 0xfc, 0xee, 0xd4, 0xd5, 0x3f, 0xdd, - 0x8c, 0xd7, 0x3d, 0x8f, 0x76, 0xec, 0x96, 0x9d, - 0x20, 0xac, 0xcb, 0x18, 0xf5, 0xb2, 0x9c, 0x12, - 0xf6, 0x7c, 0x33, 0xdc, 0x4f, 0x9a, 0xe5, 0x2d, - 0x63, 0x68, 0xa4, 0x2b, 0x1d, 0x0a, 0x1e, 0xf0, - 0xfe, 0x73, 0xf2, 0x5f, 0x7b, 0xb4, 0xea, 0x54, - 0xad, 0x27, 0xd1, 0xff, 0xb6, 0x50, 0x06, 0x7b, - 0x51, 0x3f, 0x25, 0x8a, 0xcf, 0x4c, 0x03, 0x3e, - 0xc3, 0xad, 0x47, 0x34, 0xcf, 0xba, 0x45, 0x79, - 0xd0, 0x7b, 0xf6, 0x66, 0x63, 0xc0, 0xc6, 0x69, - 0xa7, 0x51, 0x84, 0xa8, 0xa0, 0x0b, 0xbc, 0x6f, - 0x13, 0x89, 0xd6, 0x5e, 0xac, 0xca, 0x2f, 0xd2, - 0xe7, 0xe1, 0x1e, 0x78, 0x22, 0x3a, 0x59, 0x6c, - 0x9c, 0x8c, 0x65, 0xf1, 0x5b, 0xf4, 0xbf, 0xd5, - 0xdc, 0x05, 0xeb, 0x70, 0x10, 0xb8, 0x6c, 0xf2, - 0x13, 0x20, 0xb0, 0xdd, 0x3e, 0xb2, 0x92, 0x5b, - 0xa3, 0xf7, 0x94, 0xa1, 0xa1, 0x74, 0x36, 0x9a, - 0xf1, 0xd8, 0xc2, 0xf0, 0xc6, 0x29, 0x7e, 0x85, - 0x28, 0xf5, 0xf2, 0x21, 0x00, 0x00, 0x00, 0x00, - 0xc8, 0x80, 0x67, 0x40, 0xc3, 0xaa, 0x17, 0x57, - 0x00, 0x01, 0xd3, 0x01, 0xe1, 0x01, 0x00, 0x00, - 0x86, 0xdf, 0x9e, 0x05, 0xb1, 0xc4, 0x67, 0xfb, - 0x02, 0x00, 0x00, 0x00, 0x00, 0x04, 0x59, 0x5a -#elif defined(TEST_GZIP) - 0x1f, 0x8b, 0x08, 0x08, 0x82, 0xd4, 0x97, 0x60, +#elif defined(DO_BZIP2) + 0x42, 0x5a, 0x68, 0x39, 0x31, 0x41, 0x59, 0x26, + 0x53, 0x59, 0x05, 0x24, 0x28, 0x04, 0x00, 0x00, + 0x27, 0xd7, 0x80, 0x00, 0x10, 0x40, 0x05, 0x06, + 0x04, 0x02, 0x00, 0x3f, 0xe7, 0xff, 0x40, 0x30, + 0x01, 0x2d, 0x23, 0x62, 0x26, 0x05, 0x3d, 0x03, + 0x54, 0xfd, 0x53, 0x4c, 0x86, 0x9e, 0x90, 0x6a, + 0x9e, 0x9e, 0x85, 0x3c, 0xa0, 0x00, 0x00, 0x1a, + 0x9e, 0x41, 0x13, 0x13, 0x28, 0x69, 0x03, 0xd4, + 0x0f, 0x1c, 0x70, 0xd0, 0xb4, 0xe3, 0xe4, 0x75, + 0x4e, 0x8b, 0x67, 0x43, 0x7b, 0x38, 0x27, 0x77, + 0xe4, 0xc1, 0x98, 0x3a, 0x2d, 0x3a, 0xe4, 0x44, + 0x98, 0xdc, 0x49, 0x8b, 0x22, 0x48, 0xfc, 0xc8, + 0xe7, 0x57, 0x05, 0x3c, 0x5a, 0xee, 0x5a, 0x84, + 0xcd, 0x7c, 0x8f, 0x26, 0x6b, 0x6e, 0xf7, 0xb5, + 0x49, 0x1f, 0x79, 0x42, 0x5d, 0x09, 0x8c, 0xc6, + 0xde, 0x0c, 0x0d, 0xb1, 0x46, 0xb4, 0xee, 0xd9, + 0x8f, 0x33, 0x37, 0x04, 0xa9, 0x05, 0x49, 0xe3, + 0x04, 0x16, 0x62, 0x36, 0x3a, 0x01, 0xda, 0xd4, + 0xc8, 0x8a, 0x32, 0x02, 0x1f, 0x62, 0x4b, 0xa4, + 0x49, 0x59, 0xda, 0x50, 0x85, 0x69, 0x35, 0x21, + 0x10, 0xc6, 0x8a, 0x3c, 0x44, 0x95, 0xb0, 0xbc, + 0xc5, 0x6b, 0xea, 0xfb, 0x40, 0xbd, 0x14, 0x01, + 0x6a, 0xfa, 0xcd, 0x67, 0xd8, 0x2d, 0x93, 0x8b, + 0xda, 0x44, 0x1b, 0xe9, 0x5a, 0x87, 0x60, 0xb0, + 0xe0, 0x73, 0xd1, 0x01, 0x3a, 0x66, 0x05, 0xcc, + 0x34, 0xa0, 0x63, 0x8d, 0x35, 0x5e, 0xa0, 0x9f, + 0x05, 0x89, 0x15, 0x51, 0x48, 0x16, 0x0c, 0x61, + 0xf4, 0x30, 0xb8, 0x07, 0x29, 0xc0, 0xf5, 0x1a, + 0xe1, 0x0d, 0x6c, 0xfe, 0x91, 0xda, 0x13, 0x2f, + 0x8e, 0x5b, 0x1c, 0xfc, 0xb3, 0xb2, 0x30, 0x9d, + 0xf6, 0x09, 0x30, 0x55, 0x30, 0x67, 0xc2, 0x87, + 0xe9, 0x9a, 0xd4, 0x1d, 0x66, 0x11, 0x54, 0x89, + 0x21, 0xe1, 0x55, 0x84, 0xbf, 0xa6, 0x11, 0xa4, + 0xb8, 0x40, 0xed, 0x42, 0x20, 0xb9, 0xb7, 0x26, + 0x31, 0x14, 0x4f, 0x86, 0xdc, 0x50, 0x34, 0x38, + 0x8b, 0x57, 0x77, 0x21, 0xf6, 0x89, 0xbd, 0xc5, + 0x65, 0xc3, 0x23, 0x45, 0xec, 0x7f, 0x8b, 0xb9, + 0x22, 0x9c, 0x28, 0x48, 0x02, 0x92, 0x14, 0x02, + 0x00, +#elif defined(DO_GZIP) + 0x1f, 0x8b, 0x08, 0x08, 0xdb, 0xa1, 0x97, 0x63, 0x00, 0x03, 0x74, 0x65, 0x73, 0x74, 0x2e, 0x74, 0x78, 0x74, 0x00, 0x35, 0x90, 0xc1, 0x71, 0x43, 0x31, 0x08, 0x44, 0xef, 0xbf, 0x8a, 0x2d, 0x20, @@ -248,8 +135,8 @@ static sqfs_u8 data_in[] = { 0xec, 0x72, 0x51, 0x56, 0x42, 0x91, 0x25, 0x73, 0x77, 0x9b, 0xd5, 0x6d, 0x83, 0x36, 0x20, 0x4d, 0x1c, 0xeb, 0x8f, 0x6b, 0xb4, 0xf3, 0xf8, 0x05, - 0x6b, 0x8b, 0x8b, 0x20, 0xbe, 0x01, 0x00, 0x00 -#elif defined(TEST_ZSTD) + 0x6b, 0x8b, 0x8b, 0x20, 0xbe, 0x01, 0x00, 0x00, +#elif defined(DO_ZSTD) 0x28, 0xb5, 0x2f, 0xfd, 0x04, 0x88, 0xa5, 0x08, 0x00, 0x46, 0x97, 0x3a, 0x1a, 0x80, 0x37, 0xcd, 0x01, 0xc0, 0x8a, 0xec, 0xfe, 0x2d, 0xf2, 0xb9, @@ -286,8 +173,125 @@ static sqfs_u8 data_in[] = { 0xd0, 0x0c, 0x78, 0xbf, 0xdd, 0xc5, 0x24, 0x3e, 0xcb, 0x0a, 0x0a, 0x69, 0x40, 0xba, 0xb0, 0xc4, 0x2a, 0x9b, 0x1e, 0x0a, 0x51, 0xa6, 0x16, 0x98, - 0x76 -#elif defined(TEST_ZSTD2) + 0x76, +#endif +}; + +static const sqfs_u8 blob_in_concat[] = { +#if defined(DO_XZ) + 0xfd, 0x37, 0x7a, 0x58, 0x5a, 0x00, 0x00, 0x04, + 0xe6, 0xd6, 0xb4, 0x46, 0x02, 0x00, 0x21, 0x01, + 0x16, 0x00, 0x00, 0x00, 0x74, 0x2f, 0xe5, 0xa3, + 0xe0, 0x00, 0xdc, 0x00, 0xb3, 0x5d, 0x00, 0x26, + 0x1b, 0xca, 0x46, 0x67, 0x5a, 0xf2, 0x77, 0xb8, + 0x7d, 0x86, 0xd8, 0x41, 0xdb, 0x05, 0x35, 0xcd, + 0x83, 0xa5, 0x7c, 0x12, 0xa5, 0x05, 0xdb, 0x90, + 0xbd, 0x2f, 0x14, 0xd3, 0x71, 0x72, 0x96, 0xa8, + 0x8a, 0x7d, 0x84, 0x56, 0x71, 0x8d, 0x6a, 0x22, + 0x98, 0xab, 0x9e, 0x3d, 0xc3, 0x55, 0xef, 0xcc, + 0xa5, 0xc3, 0xdd, 0x5b, 0x8e, 0xbf, 0x03, 0x81, + 0x21, 0x40, 0xd6, 0x26, 0x91, 0x02, 0x45, 0x4e, + 0x20, 0x91, 0xcf, 0x8c, 0x51, 0x22, 0x02, 0x70, + 0xba, 0x05, 0x6b, 0x83, 0xef, 0x3f, 0x8e, 0x09, + 0xef, 0x88, 0xf5, 0x37, 0x1b, 0x89, 0x8d, 0xff, + 0x1e, 0xee, 0xe8, 0xb0, 0xac, 0xf2, 0x6e, 0xd4, + 0x3e, 0x25, 0xaf, 0xa0, 0x6d, 0x2e, 0xc0, 0x7f, + 0xb5, 0xa0, 0xcb, 0x90, 0x1f, 0x08, 0x1a, 0xe2, + 0x90, 0x20, 0x19, 0x71, 0x0c, 0xe8, 0x3f, 0xe5, + 0x39, 0xeb, 0x9a, 0x62, 0x4f, 0x06, 0xda, 0x3c, + 0x32, 0x59, 0xcc, 0x83, 0xe3, 0x83, 0x0f, 0x38, + 0x7d, 0x43, 0x37, 0x6c, 0x0b, 0x05, 0x65, 0x98, + 0x25, 0xdb, 0xf2, 0xc0, 0x2d, 0x39, 0x36, 0x5d, + 0xd4, 0xb6, 0xc2, 0x79, 0x73, 0x3e, 0xc2, 0x6e, + 0x54, 0xec, 0x78, 0x2b, 0x5d, 0xf1, 0xd1, 0xb4, + 0xb3, 0xcd, 0xf3, 0x89, 0xf5, 0x80, 0x79, 0x46, + 0xc0, 0x00, 0x00, 0x00, 0xc4, 0xf5, 0x1d, 0x08, + 0xf0, 0x34, 0x3a, 0x59, 0x00, 0x01, 0xcf, 0x01, + 0xdd, 0x01, 0x00, 0x00, 0x7f, 0x5a, 0x77, 0xcb, + 0xb1, 0xc4, 0x67, 0xfb, 0x02, 0x00, 0x00, 0x00, + 0x00, 0x04, 0x59, 0x5a, + 0xfd, 0x37, 0x7a, 0x58, 0x5a, 0x00, 0x00, 0x04, + 0xe6, 0xd6, 0xb4, 0x46, 0x02, 0x00, 0x21, 0x01, + 0x16, 0x00, 0x00, 0x00, 0x74, 0x2f, 0xe5, 0xa3, + 0xe0, 0x00, 0xe0, 0x00, 0xb7, 0x5d, 0x00, 0x31, + 0x9b, 0xca, 0x19, 0xc5, 0x54, 0xec, 0xb6, 0x54, + 0xe7, 0xb1, 0x7d, 0xc4, 0x57, 0x9e, 0x6c, 0x89, + 0xad, 0x4a, 0x6d, 0x16, 0xd8, 0x3c, 0x05, 0x94, + 0x10, 0x16, 0x99, 0x38, 0x21, 0xa3, 0xb9, 0xc5, + 0x80, 0xff, 0xfc, 0xee, 0xd4, 0xd5, 0x3f, 0xdd, + 0x8c, 0xd7, 0x3d, 0x8f, 0x76, 0xec, 0x96, 0x9d, + 0x20, 0xac, 0xcb, 0x18, 0xf5, 0xb2, 0x9c, 0x12, + 0xf6, 0x7c, 0x33, 0xdc, 0x4f, 0x9a, 0xe5, 0x2d, + 0x63, 0x68, 0xa4, 0x2b, 0x1d, 0x0a, 0x1e, 0xf0, + 0xfe, 0x73, 0xf2, 0x5f, 0x7b, 0xb4, 0xea, 0x54, + 0xad, 0x27, 0xd1, 0xff, 0xb6, 0x50, 0x06, 0x7b, + 0x51, 0x3f, 0x25, 0x8a, 0xcf, 0x4c, 0x03, 0x3e, + 0xc3, 0xad, 0x47, 0x34, 0xcf, 0xba, 0x45, 0x79, + 0xd0, 0x7b, 0xf6, 0x66, 0x63, 0xc0, 0xc6, 0x69, + 0xa7, 0x51, 0x84, 0xa8, 0xa0, 0x0b, 0xbc, 0x6f, + 0x13, 0x89, 0xd6, 0x5e, 0xac, 0xca, 0x2f, 0xd2, + 0xe7, 0xe1, 0x1e, 0x78, 0x22, 0x3a, 0x59, 0x6c, + 0x9c, 0x8c, 0x65, 0xf1, 0x5b, 0xf4, 0xbf, 0xd5, + 0xdc, 0x05, 0xeb, 0x70, 0x10, 0xb8, 0x6c, 0xf2, + 0x13, 0x20, 0xb0, 0xdd, 0x3e, 0xb2, 0x92, 0x5b, + 0xa3, 0xf7, 0x94, 0xa1, 0xa1, 0x74, 0x36, 0x9a, + 0xf1, 0xd8, 0xc2, 0xf0, 0xc6, 0x29, 0x7e, 0x85, + 0x28, 0xf5, 0xf2, 0x21, 0x00, 0x00, 0x00, 0x00, + 0xc8, 0x80, 0x67, 0x40, 0xc3, 0xaa, 0x17, 0x57, + 0x00, 0x01, 0xd3, 0x01, 0xe1, 0x01, 0x00, 0x00, + 0x86, 0xdf, 0x9e, 0x05, 0xb1, 0xc4, 0x67, 0xfb, + 0x02, 0x00, 0x00, 0x00, 0x00, 0x04, 0x59, 0x5a +#elif defined(DO_BZIP2) + 0x42, 0x5a, 0x68, 0x39, 0x31, 0x41, 0x59, 0x26, + 0x53, 0x59, 0x5d, 0x09, 0x24, 0x1d, 0x00, 0x00, + 0x13, 0xd7, 0x80, 0x00, 0x10, 0x40, 0x05, 0x00, + 0x04, 0x02, 0x00, 0x3e, 0xa7, 0xff, 0x40, 0x30, + 0x00, 0xac, 0x43, 0x54, 0xf5, 0x36, 0x4c, 0xa7, + 0xa8, 0xd3, 0x6a, 0x60, 0x81, 0x40, 0x00, 0xd0, + 0x32, 0x64, 0x0d, 0x53, 0xda, 0x02, 0x09, 0xa2, + 0x68, 0x34, 0xd1, 0x27, 0x4a, 0xdd, 0xf2, 0x0a, + 0x73, 0x43, 0xf9, 0xa2, 0x51, 0x85, 0x76, 0x45, + 0x9a, 0x68, 0x3a, 0xe7, 0x0d, 0xc0, 0x21, 0x4a, + 0xc4, 0xf9, 0xf7, 0x40, 0xc3, 0x10, 0xb2, 0x9b, + 0x58, 0x56, 0x71, 0x50, 0x2f, 0xa4, 0xc5, 0x61, + 0x19, 0xf6, 0x59, 0x06, 0x82, 0x03, 0x7f, 0xeb, + 0xd2, 0x61, 0x88, 0xcd, 0xe8, 0xf7, 0xe8, 0x87, + 0x59, 0x9d, 0xe1, 0xf8, 0x19, 0x6e, 0xad, 0x77, + 0xbf, 0x34, 0x17, 0x21, 0x6b, 0x91, 0xc9, 0x52, + 0xd0, 0x81, 0x1e, 0xb5, 0x0b, 0xee, 0x42, 0x84, + 0x80, 0xd5, 0xa1, 0x8a, 0x04, 0x18, 0x4d, 0xf3, + 0xda, 0x7e, 0x3c, 0x40, 0xa4, 0xdb, 0xe5, 0xf0, + 0x37, 0x40, 0x3a, 0x7d, 0xa7, 0x45, 0x21, 0xf2, + 0x5a, 0x7b, 0x59, 0x56, 0x16, 0xd5, 0xac, 0x9f, + 0x60, 0x85, 0x0e, 0xf5, 0x73, 0xd9, 0x47, 0xe2, + 0xee, 0x48, 0xa7, 0x0a, 0x12, 0x0b, 0xa1, 0x24, + 0x83, 0xa0, + 0x42, 0x5a, 0x68, 0x39, 0x31, 0x41, 0x59, 0x26, + 0x53, 0x59, 0x2c, 0x24, 0x39, 0xa0, 0x00, 0x00, + 0x1f, 0x55, 0x80, 0x00, 0x10, 0x40, 0x05, 0x06, + 0x00, 0x3f, 0xe7, 0xff, 0x40, 0x30, 0x00, 0xb5, + 0x91, 0x13, 0x4f, 0x54, 0x7a, 0x6a, 0x6d, 0x4d, + 0xa2, 0x68, 0x0c, 0x84, 0x53, 0xf5, 0x30, 0x89, + 0xa3, 0xd4, 0x0d, 0x0f, 0x49, 0xa0, 0xd4, 0xf4, + 0xd1, 0x53, 0xf4, 0x93, 0x69, 0x3c, 0x81, 0x1a, + 0x65, 0x53, 0x90, 0x51, 0x07, 0x2a, 0xad, 0x8f, + 0x63, 0xba, 0x25, 0xc2, 0x0c, 0x8b, 0xb9, 0x95, + 0x15, 0xd8, 0xda, 0x61, 0x5c, 0xa9, 0xe4, 0x0b, + 0x21, 0xc9, 0x97, 0x57, 0x01, 0x28, 0x9b, 0xfb, + 0x94, 0xb9, 0x48, 0xa3, 0x0a, 0xc6, 0x1c, 0x54, + 0x98, 0x9a, 0x39, 0xc3, 0x87, 0x90, 0x33, 0x58, + 0x2d, 0x3e, 0x16, 0xb1, 0xae, 0x26, 0x89, 0x75, + 0xf5, 0x77, 0xa5, 0x8e, 0x5b, 0x8c, 0x8a, 0x39, + 0xbd, 0x75, 0x21, 0x9d, 0x99, 0x18, 0x4a, 0x91, + 0xab, 0xbc, 0x08, 0x87, 0xa4, 0xf1, 0x81, 0xb5, + 0xb4, 0xb0, 0xfe, 0x6b, 0x9f, 0xbe, 0x19, 0x82, + 0xd1, 0x50, 0xe1, 0x5e, 0x13, 0xb5, 0xc6, 0x2c, + 0xa4, 0x82, 0xf2, 0x5c, 0xc3, 0x20, 0x41, 0x13, + 0x56, 0x63, 0x3d, 0xec, 0x71, 0x2a, 0xbf, 0x2c, + 0x60, 0x2f, 0x7a, 0x4d, 0xcb, 0x3f, 0x8b, 0xb9, + 0x22, 0x9c, 0x28, 0x48, 0x16, 0x12, 0x1c, 0xd0, + 0x00, +#elif defined(DO_ZSTD) 0x28, 0xb5, 0x2f, 0xfd, 0x04, 0x58, 0x75, 0x04, 0x00, 0xb2, 0x4c, 0x20, 0x17, 0xa0, 0x25, 0x69, 0x03, 0xf0, 0xb2, 0x37, 0xb1, 0x5e, 0xb9, 0x24, @@ -328,10 +332,31 @@ static sqfs_u8 data_in[] = { 0xf8, 0xb9, 0x59, 0xbe, 0x2e, 0xf9, 0xd4, 0x02, 0x98, 0x50, 0x5a, 0xc2, 0xcf, 0xe1, 0x08, 0x02, 0x00, 0x0f, 0x1e, 0x44, 0x40, 0x79, 0x50, 0x67, - 0x3d, 0xd3, 0x35, 0x8f + 0x3d, 0xd3, 0x35, 0x8f, +#elif defined(DO_GZIP) + 0, #endif }; +#if defined(DO_XZ) +static size_t in_stop = 244; +static size_t out_stop = 221; + +#define mkdecompressor decompressor_stream_xz_create +#elif defined(DO_BZIP2) +static size_t in_stop = 186; +static size_t out_stop = 221; + +#define mkdecompressor decompressor_stream_bzip2_create +#elif defined(DO_ZSTD) +static size_t in_stop = 319; +static size_t out_stop = 446; + +#define mkdecompressor decompressor_stream_zstd_create +#elif defined(DO_GZIP) +#define mkdecompressor decompressor_stream_gzip_create +#endif + static const char orig[] = "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod\n" "tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam,\n" @@ -340,93 +365,64 @@ static const char orig[] = "cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non\n" "proident, sunt in culpa qui officia deserunt mollit anim id est laborum.\n"; -#if defined(TEST_BZIP2) || defined(TEST_BZIP22) -#define COMP_NAME "bzip2" -#define COMP_ID IO_COMPRESSOR_BZIP2 -#elif defined(TEST_XZ) || defined(TEST_XZ2) -#define COMP_NAME "xz" -#define COMP_ID IO_COMPRESSOR_XZ -#elif defined(TEST_GZIP) -#define COMP_NAME "gzip" -#define COMP_ID IO_COMPRESSOR_GZIP -#elif defined(TEST_ZSTD) || defined(TEST_ZSTD2) -#define COMP_NAME "zstd" -#define COMP_ID IO_COMPRESSOR_ZSTD -#endif - -static void destroy_noop(sqfs_object_t *obj) -{ - (void)obj; -} - -static int precache_noop(istream_t *strm) -{ - (void)strm; - return 0; -} - -static const char *get_filename(istream_t *strm) -{ - (void)strm; - return "memstream"; -} - -static istream_t memstream = { - .base = { - .destroy = destroy_noop, - }, - - .buffer_used = sizeof(data_in) / sizeof(data_in[0]), - .buffer_offset = 0, - .eof = true, - .buffer = data_in, - - .precache = precache_noop, - .get_filename = get_filename, -}; - int main(int argc, char **argv) { - char buffer[2 * (sizeof(orig) / sizeof(orig[0]))]; - const char *name; - istream_t *xfrm; - size_t orig_sz; + sqfs_u32 in_diff = 0, out_diff = 0; + xfrm_stream_t *xfrm; + char buffer[1024]; int ret; (void)argc; (void)argv; - /* XXX: null terminator not included in the compressed blob */ - orig_sz = (sizeof(orig) / sizeof(orig[0])) - 1; - - /* generic API test */ - TEST_ASSERT(io_compressor_exists(COMP_ID)); + /* normal stream */ + xfrm = mkdecompressor(); + TEST_NOT_NULL(xfrm); + TEST_EQUAL_UI(((sqfs_object_t *)xfrm)->refcount, 1); - name = io_compressor_name_from_id(COMP_ID); - TEST_STR_EQUAL(name, COMP_NAME); + ret = xfrm->process_data(xfrm, blob_in, sizeof(blob_in), + buffer, sizeof(buffer), + &in_diff, &out_diff, XFRM_STREAM_FLUSH_FULL); + TEST_EQUAL_I(ret, XFRM_STREAM_END); - ret = io_compressor_id_from_name(name); - TEST_EQUAL_I(ret, COMP_ID); + TEST_EQUAL_UI(in_diff, sizeof(blob_in)); + TEST_EQUAL_UI(out_diff, sizeof(orig) - 1); + ret = memcmp(buffer, orig, out_diff); + TEST_EQUAL_I(ret, 0); - ret = istream_detect_compressor(&memstream, NULL); - TEST_EQUAL_I(ret, COMP_ID); + sqfs_drop(xfrm); - /* decoder test */ - xfrm = istream_compressor_create(&memstream, COMP_ID); + /* concatenated streams */ +#if !defined(DO_GZIP) + xfrm = mkdecompressor(); TEST_NOT_NULL(xfrm); + TEST_EQUAL_UI(((sqfs_object_t *)xfrm)->refcount, 1); - name = istream_get_filename(xfrm); - TEST_STR_EQUAL(name, "memstream"); + in_diff = 0; + out_diff = 0; - ret = istream_read(xfrm, buffer, sizeof(buffer)); - TEST_ASSERT(ret > 0); - TEST_EQUAL_UI((size_t)ret, orig_sz); + ret = xfrm->process_data(xfrm, blob_in_concat, sizeof(blob_in_concat), + buffer, sizeof(buffer), + &in_diff, &out_diff, XFRM_STREAM_FLUSH_FULL); + TEST_EQUAL_I(ret, XFRM_STREAM_END); - ret = memcmp(buffer, orig, ret); + TEST_EQUAL_UI(in_diff, in_stop); + TEST_EQUAL_UI(out_diff, out_stop); + ret = memcmp(buffer, orig, out_diff); TEST_EQUAL_I(ret, 0); - ret = istream_read(xfrm, buffer, sizeof(buffer)); + ret = xfrm->process_data(xfrm, blob_in_concat + in_diff, + sizeof(blob_in_concat) - in_diff, + buffer + out_diff, sizeof(buffer) - out_diff, + &in_diff, &out_diff, XFRM_STREAM_FLUSH_FULL); + TEST_EQUAL_I(ret, XFRM_STREAM_END); + + TEST_EQUAL_UI(in_diff, sizeof(blob_in_concat)); + TEST_EQUAL_UI(out_diff, sizeof(orig) - 1); + ret = memcmp(buffer, orig, out_diff); TEST_EQUAL_I(ret, 0); - /* cleanup */ sqfs_drop(xfrm); +#else + (void)blob_in_concat; +#endif return EXIT_SUCCESS; } |