From c9f5cfd3df2706da940a39d6d816ad084ba80fbd Mon Sep 17 00:00:00 2001 From: David Oberhollenzer Date: Mon, 14 Sep 2020 14:59:31 +0200 Subject: Implement istream decompression support Signed-off-by: David Oberhollenzer --- include/fstream.h | 40 +++++++++++++ lib/fstream/Makemodule.am | 5 +- lib/fstream/internal.h | 16 +++++ lib/fstream/uncompress/autodetect.c | 53 ++++++++++++++++ lib/fstream/uncompress/gzip.c | 91 ++++++++++++++++++++++++++++ lib/fstream/uncompress/istream_compressor.c | 59 ++++++++++++++++++ lib/fstream/uncompress/xz.c | 93 +++++++++++++++++++++++++++++ 7 files changed, 356 insertions(+), 1 deletion(-) create mode 100644 lib/fstream/uncompress/autodetect.c create mode 100644 lib/fstream/uncompress/gzip.c create mode 100644 lib/fstream/uncompress/istream_compressor.c create mode 100644 lib/fstream/uncompress/xz.c diff --git a/include/fstream.h b/include/fstream.h index cb992f8..c62e5ad 100644 --- a/include/fstream.h +++ b/include/fstream.h @@ -144,6 +144,46 @@ SQFS_INTERNAL istream_t *istream_open_stdin(void); SQFS_INTERNAL ostream_t *ostream_compressor_create(ostream_t *strm, int comp_id); +/** + * @brief Create an input stream that transparently uncompresses data. + * + * @memberof istream_t + * + * This function creates an input stream that wraps an underlying input stream + * that is compressed and transparently uncompresses the data when reading + * from it. + * + * The new stream takes ownership of the wrapped stream and destroys it when + * the compressor stream is destroyed. If this function fails, the wrapped + * stream is also destroyed. + * + * @param strm A pointer to another stream that should be wrapped. + * @param comp_id An identifier describing the compressor to use. + * + * @return A pointer to an input stream on success, NULL on failure. + */ +SQFS_INTERNAL istream_t *istream_compressor_create(istream_t *strm, + int comp_id); + +/** + * @brief Probe the buffered data in an istream to check if it is compressed. + * + * @memberof istream_t + * + * This function peeks into the internal buffer of an input stream to check + * for magic signatures of various compressors. + * + * @param strm A pointer to an input stream to check + * @param probe A callback used to check if raw/decoded data matches an + * expected format. Returns 0 if not, -1 on failure and +1 + * on success. + * + * @return A compressor ID on success, 0 if no match was found, -1 on failure. + */ +SQFS_INTERNAL int istream_detect_compressor(istream_t *strm, + int (*probe)(const sqfs_u8 *data, + size_t size)); + /** * @brief Append a block of data to an output stream. * diff --git a/lib/fstream/Makemodule.am b/lib/fstream/Makemodule.am index 9178647..a2e414e 100644 --- a/lib/fstream/Makemodule.am +++ b/lib/fstream/Makemodule.am @@ -4,6 +4,8 @@ libfstream_a_SOURCES += lib/fstream/ostream.c lib/fstream/printf.c libfstream_a_SOURCES += lib/fstream/istream.c libfstream_a_SOURCES += lib/fstream/compressor.c libfstream_a_SOURCES += lib/fstream/compress/ostream_compressor.c +libfstream_a_SOURCES += lib/fstream/uncompress/istream_compressor.c +libfstream_a_SOURCES += lib/fstream/uncompress/autodetect.c libfstream_a_CFLAGS = $(AM_CFLAGS) $(ZLIB_CFLAGS) $(XZ_CFLAGS) libfstream_a_CPPFLAGS = $(AM_CPPFLAGS) @@ -17,12 +19,13 @@ libfstream_a_SOURCES += lib/fstream/unix/istream.c endif if WITH_XZ -libfstream_a_SOURCES += lib/fstream/compress/xz.c +libfstream_a_SOURCES += lib/fstream/compress/xz.c lib/fstream/uncompress/xz.c libfstream_a_CPPFLAGS += -DWITH_XZ endif if WITH_GZIP libfstream_a_SOURCES += lib/fstream/compress/gzip.c +libfstream_a_SOURCES += lib/fstream/uncompress/gzip.c libfstream_a_CPPFLAGS += -DWITH_GZIP endif diff --git a/lib/fstream/internal.h b/lib/fstream/internal.h index ae6a29a..160a523 100644 --- a/lib/fstream/internal.h +++ b/lib/fstream/internal.h @@ -37,6 +37,18 @@ typedef struct ostream_comp_t { void (*cleanup)(struct ostream_comp_t *ostrm); } ostream_comp_t; +typedef struct istream_comp_t { + istream_t base; + + istream_t *wrapped; + + sqfs_u8 uncompressed[BUFSZ]; + + bool eof; + + void (*cleanup)(struct istream_comp_t *strm); +} istream_comp_t; + #ifdef __cplusplus extern "C" { #endif @@ -45,6 +57,10 @@ SQFS_INTERNAL ostream_comp_t *ostream_gzip_create(const char *filename); SQFS_INTERNAL ostream_comp_t *ostream_xz_create(const char *filename); +SQFS_INTERNAL istream_comp_t *istream_gzip_create(const char *filename); + +SQFS_INTERNAL istream_comp_t *istream_xz_create(const char *filename); + #ifdef __cplusplus } #endif diff --git a/lib/fstream/uncompress/autodetect.c b/lib/fstream/uncompress/autodetect.c new file mode 100644 index 0000000..4ffe078 --- /dev/null +++ b/lib/fstream/uncompress/autodetect.c @@ -0,0 +1,53 @@ +/* SPDX-License-Identifier: GPL-3.0-or-later */ +/* + * autodetect.c + * + * Copyright (C) 2019 David Oberhollenzer + */ +#include "../internal.h" + +static const struct { + int id; + const sqfs_u8 *value; + size_t len; +} magic[] = { + { FSTREAM_COMPRESSOR_GZIP, (const sqfs_u8 *)"\x1F\x8B\x08", 3 }, + { FSTREAM_COMPRESSOR_XZ, (const sqfs_u8 *)("\xFD" "7zXZ"), 6 }, +}; + +int istream_detect_compressor(istream_t *strm, + int (*probe)(const sqfs_u8 *data, size_t size)) +{ + size_t i; + int ret; + + ret = istream_precache(strm); + if (ret != 0) + return ret; + + if (probe != NULL) { + ret = probe(strm->buffer + strm->buffer_offset, + strm->buffer_used - strm->buffer_offset); + if (ret < 0) + return ret; + + /* XXX: this means the data is uncompressed. We do this check + first since it might be perfectly OK for the uncompressed + data to contain a magic number from the table. */ + if (ret > 0) + return 0; + } + + for (i = 0; i < sizeof(magic) / sizeof(magic[0]); ++i) { + if ((strm->buffer_used - strm->buffer_offset) < magic[i].len) + continue; + + ret = memcmp(strm->buffer + strm->buffer_offset, + magic[i].value, magic[i].len); + + if (ret == 0) + return magic[i].id; + } + + return 0; +} diff --git a/lib/fstream/uncompress/gzip.c b/lib/fstream/uncompress/gzip.c new file mode 100644 index 0000000..c2003db --- /dev/null +++ b/lib/fstream/uncompress/gzip.c @@ -0,0 +1,91 @@ +/* SPDX-License-Identifier: GPL-3.0-or-later */ +/* + * gzip.c + * + * Copyright (C) 2019 David Oberhollenzer + */ +#include "../internal.h" + +#include + +typedef struct { + istream_comp_t base; + + z_stream strm; +} istream_gzip_t; + +static int precache(istream_t *base) +{ + istream_t *wrapped = ((istream_comp_t *)base)->wrapped; + istream_gzip_t *gzip = (istream_gzip_t *)base; + int ret; + + for (;;) { + ret = istream_precache(wrapped); + if (ret != 0) + return ret; + + gzip->strm.avail_in = wrapped->buffer_used; + gzip->strm.next_in = wrapped->buffer; + + gzip->strm.avail_out = BUFSZ - base->buffer_used; + gzip->strm.next_out = base->buffer + base->buffer_used; + + ret = inflate(&gzip->strm, Z_NO_FLUSH); + + wrapped->buffer_offset = wrapped->buffer_used - + gzip->strm.avail_in; + + base->buffer_used = BUFSZ - gzip->strm.avail_out; + + if (ret == Z_BUF_ERROR) + break; + + if (ret == Z_STREAM_END) { + base->eof = true; + break; + } + + if (ret != Z_OK) { + fprintf(stderr, + "%s: internal error in gzip decoder.\n", + wrapped->get_filename(wrapped)); + return -1; + } + } + + return 0; +} + +static void cleanup(istream_comp_t *base) +{ + istream_gzip_t *gzip = (istream_gzip_t *)base; + + inflateEnd(&gzip->strm); +} + +istream_comp_t *istream_gzip_create(const char *filename) +{ + istream_gzip_t *gzip = calloc(1, sizeof(*gzip)); + istream_comp_t *base = (istream_comp_t *)gzip; + int ret; + + if (gzip == NULL) { + fprintf(stderr, "%s: creating gzip decoder: %s.\n", + filename, strerror(errno)); + return NULL; + } + + ret = inflateInit2(&gzip->strm, 16 + 15); + if (ret != Z_OK) { + fprintf(stderr, + "%s: internal error creating gzip reader.\n", + filename); + free(gzip); + return NULL; + } + + ((istream_t *)base)->precache = precache; + base->cleanup = cleanup; + return base; +} diff --git a/lib/fstream/uncompress/istream_compressor.c b/lib/fstream/uncompress/istream_compressor.c new file mode 100644 index 0000000..924f309 --- /dev/null +++ b/lib/fstream/uncompress/istream_compressor.c @@ -0,0 +1,59 @@ +/* SPDX-License-Identifier: GPL-3.0-or-later */ +/* + * istream_compressor.c + * + * Copyright (C) 2019 David Oberhollenzer + */ +#include "../internal.h" + +static const char *comp_get_filename(istream_t *strm) +{ + istream_comp_t *comp = (istream_comp_t *)strm; + + return comp->wrapped->get_filename(comp->wrapped); +} + +static void comp_destroy(sqfs_object_t *obj) +{ + istream_comp_t *comp = (istream_comp_t *)obj; + + comp->cleanup(comp); + sqfs_destroy(comp->wrapped); + free(comp); +} + +istream_t *istream_compressor_create(istream_t *strm, int comp_id) +{ + istream_comp_t *comp = NULL; + sqfs_object_t *obj; + istream_t *base; + + switch (comp_id) { + case FSTREAM_COMPRESSOR_GZIP: +#ifdef WITH_GZIP + comp = istream_gzip_create(strm->get_filename(strm)); +#endif + break; + case FSTREAM_COMPRESSOR_XZ: +#ifdef WITH_XZ + comp = istream_xz_create(strm->get_filename(strm)); +#endif + break; + default: + break; + } + + if (comp == NULL) + return NULL; + + comp->wrapped = strm; + + base = (istream_t *)comp; + base->get_filename = comp_get_filename; + base->buffer = comp->uncompressed; + base->eof = false; + + obj = (sqfs_object_t *)comp; + obj->destroy = comp_destroy; + return base; +} diff --git a/lib/fstream/uncompress/xz.c b/lib/fstream/uncompress/xz.c new file mode 100644 index 0000000..0e48468 --- /dev/null +++ b/lib/fstream/uncompress/xz.c @@ -0,0 +1,93 @@ +/* SPDX-License-Identifier: GPL-3.0-or-later */ +/* + * xz.c + * + * Copyright (C) 2019 David Oberhollenzer + */ +#include "../internal.h" + +#include + +typedef struct { + istream_comp_t base; + + lzma_stream strm; +} istream_xz_t; + +static int precache(istream_t *base) +{ + istream_xz_t *xz = (istream_xz_t *)base; + istream_t *wrapped = ((istream_comp_t *)base)->wrapped; + lzma_ret ret_xz; + int ret; + + for (;;) { + ret = istream_precache(wrapped); + if (ret != 0) + return ret; + + xz->strm.avail_in = wrapped->buffer_used; + xz->strm.next_in = wrapped->buffer; + + xz->strm.avail_out = BUFSZ - base->buffer_used; + xz->strm.next_out = base->buffer + base->buffer_used; + + ret_xz = lzma_code(&xz->strm, LZMA_RUN); + + base->buffer_used = BUFSZ - xz->strm.avail_out; + wrapped->buffer_offset = wrapped->buffer_used - + xz->strm.avail_in; + + if (ret_xz == LZMA_BUF_ERROR) + break; + + if (ret_xz == LZMA_STREAM_END) { + base->eof = true; + break; + } + + if (ret_xz != LZMA_OK) { + fprintf(stderr, + "%s: internal error in xz decoder.\n", + wrapped->get_filename(wrapped)); + return -1; + } + } + + return 0; +} + +static void cleanup(istream_comp_t *base) +{ + istream_xz_t *xz = (istream_xz_t *)base; + + lzma_end(&xz->strm); +} + +istream_comp_t *istream_xz_create(const char *filename) +{ + istream_xz_t *xz = calloc(1, sizeof(*xz)); + istream_comp_t *base = (istream_comp_t *)xz; + sqfs_u64 memlimit = 65 * 1024 * 1024; + lzma_ret ret_xz; + + if (xz == NULL) { + fprintf(stderr, "%s: creating xz decoder: %s.\n", + filename, strerror(errno)); + return NULL; + } + + ret_xz = lzma_stream_decoder(&xz->strm, memlimit, 0); + + if (ret_xz != LZMA_OK) { + fprintf(stderr, + "%s: error initializing xz decoder.\n", + filename); + free(xz); + return NULL; + } + + ((istream_t *)base)->precache = precache; + base->cleanup = cleanup; + return base; +} -- cgit v1.2.3