Implement istream decompression support

Signed-off-by: David Oberhollenzer <david.oberhollenzer@sigma-star.at>

7 files changed, 356 insertions, 1 deletions

diff --git a/include/fstream.h b/include/fstream.h
index cb992f8..c62e5ad 100644
--- a/include/fstream.h
+++ b/include/fstream.h
@@ -145,6 +145,46 @@ SQFS_INTERNAL ostream_t *ostream_compressor_create(ostream_t *strm,
 						   int comp_id);
 
 /**
+ * @brief Create an input stream that transparently uncompresses data.
+ *
+ * @memberof istream_t
+ *
+ * This function creates an input stream that wraps an underlying input stream
+ * that is compressed and transparently uncompresses the data when reading
+ * from it.
+ *
+ * The new stream takes ownership of the wrapped stream and destroys it when
+ * the compressor stream is destroyed. If this function fails, the wrapped
+ * stream is also destroyed.
+ *
+ * @param strm A pointer to another stream that should be wrapped.
+ * @param comp_id An identifier describing the compressor to use.
+ *
+ * @return A pointer to an input stream on success, NULL on failure.
+ */
+SQFS_INTERNAL istream_t *istream_compressor_create(istream_t *strm,
+						   int comp_id);
+
+/**
+ * @brief Probe the buffered data in an istream to check if it is compressed.
+ *
+ * @memberof istream_t
+ *
+ * This function peeks into the internal buffer of an input stream to check
+ * for magic signatures of various compressors.
+ *
+ * @param strm A pointer to an input stream to check
+ * @param probe A callback used to check if raw/decoded data matches an
+ *              expected format. Returns 0 if not, -1 on failure and +1
+ *              on success.
+ *
+ * @return A compressor ID on success, 0 if no match was found, -1 on failure.
+ */
+SQFS_INTERNAL int istream_detect_compressor(istream_t *strm,
+					    int (*probe)(const sqfs_u8 *data,
+							 size_t size));
+
+/**
  * @brief Append a block of data to an output stream.
  *
  * @memberof ostream_t
diff --git a/lib/fstream/Makemodule.am b/lib/fstream/Makemodule.am
index 9178647..a2e414e 100644
--- a/lib/fstream/Makemodule.am
+++ b/lib/fstream/Makemodule.am
@@ -4,6 +4,8 @@ libfstream_a_SOURCES += lib/fstream/ostream.c lib/fstream/printf.c
 libfstream_a_SOURCES += lib/fstream/istream.c
 libfstream_a_SOURCES += lib/fstream/compressor.c
 libfstream_a_SOURCES += lib/fstream/compress/ostream_compressor.c
+libfstream_a_SOURCES += lib/fstream/uncompress/istream_compressor.c
+libfstream_a_SOURCES += lib/fstream/uncompress/autodetect.c
 libfstream_a_CFLAGS = $(AM_CFLAGS) $(ZLIB_CFLAGS) $(XZ_CFLAGS)
 libfstream_a_CPPFLAGS = $(AM_CPPFLAGS)
 
@@ -17,12 +19,13 @@ libfstream_a_SOURCES += lib/fstream/unix/istream.c
 endif
 
 if WITH_XZ
-libfstream_a_SOURCES += lib/fstream/compress/xz.c
+libfstream_a_SOURCES += lib/fstream/compress/xz.c lib/fstream/uncompress/xz.c
 libfstream_a_CPPFLAGS += -DWITH_XZ
 endif
 
 if WITH_GZIP
 libfstream_a_SOURCES += lib/fstream/compress/gzip.c
+libfstream_a_SOURCES += lib/fstream/uncompress/gzip.c
 libfstream_a_CPPFLAGS += -DWITH_GZIP
 endif
 
diff --git a/lib/fstream/internal.h b/lib/fstream/internal.h
index ae6a29a..160a523 100644
--- a/lib/fstream/internal.h
+++ b/lib/fstream/internal.h
@@ -37,6 +37,18 @@ typedef struct ostream_comp_t {
 	void (*cleanup)(struct ostream_comp_t *ostrm);
 } ostream_comp_t;
 
+typedef struct istream_comp_t {
+	istream_t base;
+
+	istream_t *wrapped;
+
+	sqfs_u8 uncompressed[BUFSZ];
+
+	bool eof;
+
+	void (*cleanup)(struct istream_comp_t *strm);
+} istream_comp_t;
+
 #ifdef __cplusplus
 extern "C" {
 #endif
@@ -45,6 +57,10 @@ SQFS_INTERNAL ostream_comp_t *ostream_gzip_create(const char *filename);
 
 SQFS_INTERNAL ostream_comp_t *ostream_xz_create(const char *filename);
 
+SQFS_INTERNAL istream_comp_t *istream_gzip_create(const char *filename);
+
+SQFS_INTERNAL istream_comp_t *istream_xz_create(const char *filename);
+
 #ifdef __cplusplus
 }
 #endif
diff --git a/lib/fstream/uncompress/autodetect.c b/lib/fstream/uncompress/autodetect.c
new file mode 100644
index 0000000..4ffe078
--- /dev/null
+++ b/lib/fstream/uncompress/autodetect.c
@@ -0,0 +1,53 @@
+/* SPDX-License-Identifier: GPL-3.0-or-later */
+/*
+ * autodetect.c
+ *
+ * Copyright (C) 2019 David Oberhollenzer <goliath@infraroot.at>
+ */
+#include "../internal.h"
+
+static const struct {
+	int id;
+	const sqfs_u8 *value;
+	size_t len;
+} magic[] = {
+	{ FSTREAM_COMPRESSOR_GZIP, (const sqfs_u8 *)"\x1F\x8B\x08", 3 },
+	{ FSTREAM_COMPRESSOR_XZ, (const sqfs_u8 *)("\xFD" "7zXZ"), 6 },
+};
+
+int istream_detect_compressor(istream_t *strm,
+			      int (*probe)(const sqfs_u8 *data, size_t size))
+{
+	size_t i;
+	int ret;
+
+	ret = istream_precache(strm);
+	if (ret != 0)
+		return ret;
+
+	if (probe != NULL) {
+		ret = probe(strm->buffer + strm->buffer_offset,
+			    strm->buffer_used - strm->buffer_offset);
+		if (ret < 0)
+			return ret;
+
+		/* XXX: this means the data is uncompressed. We do this check
+		   first since it might be perfectly OK for the uncompressed
+		   data to contain a magic number from the table. */
+		if (ret > 0)
+			return 0;
+	}
+
+	for (i = 0; i < sizeof(magic) / sizeof(magic[0]); ++i) {
+		if ((strm->buffer_used - strm->buffer_offset) < magic[i].len)
+			continue;
+
+		ret = memcmp(strm->buffer + strm->buffer_offset,
+			     magic[i].value, magic[i].len);
+
+		if (ret == 0)
+			return magic[i].id;
+	}
+
+	return 0;
+}
diff --git a/lib/fstream/uncompress/gzip.c b/lib/fstream/uncompress/gzip.c
new file mode 100644
index 0000000..c2003db
--- /dev/null
+++ b/lib/fstream/uncompress/gzip.c
@@ -0,0 +1,91 @@
+/* SPDX-License-Identifier: GPL-3.0-or-later */
+/*
+ * gzip.c
+ *
+ * Copyright (C) 2019 David Oberhollenzer <goliath@infraroot.at>
+ */
+#include "../internal.h"
+
+#include <zlib.h>
+
+typedef struct {
+	istream_comp_t base;
+
+	z_stream strm;
+} istream_gzip_t;
+
+static int precache(istream_t *base)
+{
+	istream_t *wrapped = ((istream_comp_t *)base)->wrapped;
+	istream_gzip_t *gzip = (istream_gzip_t *)base;
+	int ret;
+
+	for (;;) {
+		ret = istream_precache(wrapped);
+		if (ret != 0)
+			return ret;
+
+		gzip->strm.avail_in = wrapped->buffer_used;
+		gzip->strm.next_in = wrapped->buffer;
+
+		gzip->strm.avail_out = BUFSZ - base->buffer_used;
+		gzip->strm.next_out = base->buffer + base->buffer_used;
+
+		ret = inflate(&gzip->strm, Z_NO_FLUSH);
+
+		wrapped->buffer_offset = wrapped->buffer_used -
+					 gzip->strm.avail_in;
+
+		base->buffer_used = BUFSZ - gzip->strm.avail_out;
+
+		if (ret == Z_BUF_ERROR)
+			break;
+
+		if (ret == Z_STREAM_END) {
+			base->eof = true;
+			break;
+		}
+
+		if (ret != Z_OK) {
+			fprintf(stderr,
+				"%s: internal error in gzip decoder.\n",
+				wrapped->get_filename(wrapped));
+			return -1;
+		}
+	}
+
+	return 0;
+}
+
+static void cleanup(istream_comp_t *base)
+{
+	istream_gzip_t *gzip = (istream_gzip_t *)base;
+
+	inflateEnd(&gzip->strm);
+}
+
+istream_comp_t *istream_gzip_create(const char *filename)
+{
+	istream_gzip_t *gzip = calloc(1, sizeof(*gzip));
+	istream_comp_t *base = (istream_comp_t *)gzip;
+	int ret;
+
+	if (gzip == NULL) {
+		fprintf(stderr, "%s: creating gzip decoder: %s.\n",
+			filename, strerror(errno));
+		return NULL;
+	}
+
+	ret = inflateInit2(&gzip->strm, 16 + 15);
+	if (ret != Z_OK) {
+		fprintf(stderr,
+			"%s: internal error creating gzip reader.\n",
+			filename);
+		free(gzip);
+		return NULL;
+	}
+
+	((istream_t *)base)->precache = precache;
+	base->cleanup = cleanup;
+	return base;
+}
diff --git a/lib/fstream/uncompress/istream_compressor.c b/lib/fstream/uncompress/istream_compressor.c
new file mode 100644
index 0000000..924f309
--- /dev/null
+++ b/lib/fstream/uncompress/istream_compressor.c
@@ -0,0 +1,59 @@
+/* SPDX-License-Identifier: GPL-3.0-or-later */
+/*
+ * istream_compressor.c
+ *
+ * Copyright (C) 2019 David Oberhollenzer <goliath@infraroot.at>
+ */
+#include "../internal.h"
+
+static const char *comp_get_filename(istream_t *strm)
+{
+	istream_comp_t *comp = (istream_comp_t *)strm;
+
+	return comp->wrapped->get_filename(comp->wrapped);
+}
+
+static void comp_destroy(sqfs_object_t *obj)
+{
+	istream_comp_t *comp = (istream_comp_t *)obj;
+
+	comp->cleanup(comp);
+	sqfs_destroy(comp->wrapped);
+	free(comp);
+}
+
+istream_t *istream_compressor_create(istream_t *strm, int comp_id)
+{
+	istream_comp_t *comp = NULL;
+	sqfs_object_t *obj;
+	istream_t *base;
+
+	switch (comp_id) {
+	case FSTREAM_COMPRESSOR_GZIP:
+#ifdef WITH_GZIP
+		comp = istream_gzip_create(strm->get_filename(strm));
+#endif
+		break;
+	case FSTREAM_COMPRESSOR_XZ:
+#ifdef WITH_XZ
+		comp = istream_xz_create(strm->get_filename(strm));
+#endif
+		break;
+	default:
+		break;
+	}
+
+	if (comp == NULL)
+		return NULL;
+
+	comp->wrapped = strm;
+
+	base = (istream_t *)comp;
+	base->get_filename = comp_get_filename;
+	base->buffer = comp->uncompressed;
+	base->eof = false;
+
+	obj = (sqfs_object_t *)comp;
+	obj->destroy = comp_destroy;
+	return base;
+}
diff --git a/lib/fstream/uncompress/xz.c b/lib/fstream/uncompress/xz.c
new file mode 100644
index 0000000..0e48468
--- /dev/null
+++ b/lib/fstream/uncompress/xz.c
@@ -0,0 +1,93 @@
+/* SPDX-License-Identifier: GPL-3.0-or-later */
+/*
+ * xz.c
+ *
+ * Copyright (C) 2019 David Oberhollenzer <goliath@infraroot.at>
+ */
+#include "../internal.h"
+
+#include <lzma.h>
+
+typedef struct {
+	istream_comp_t base;
+
+	lzma_stream strm;
+} istream_xz_t;
+
+static int precache(istream_t *base)
+{
+	istream_xz_t *xz = (istream_xz_t *)base;
+	istream_t *wrapped = ((istream_comp_t *)base)->wrapped;
+	lzma_ret ret_xz;
+	int ret;
+
+	for (;;) {
+		ret = istream_precache(wrapped);
+		if (ret != 0)
+			return ret;
+
+		xz->strm.avail_in = wrapped->buffer_used;
+		xz->strm.next_in = wrapped->buffer;
+
+		xz->strm.avail_out = BUFSZ - base->buffer_used;
+		xz->strm.next_out = base->buffer + base->buffer_used;
+
+		ret_xz = lzma_code(&xz->strm, LZMA_RUN);
+
+		base->buffer_used = BUFSZ - xz->strm.avail_out;
+		wrapped->buffer_offset = wrapped->buffer_used -
+					 xz->strm.avail_in;
+
+		if (ret_xz == LZMA_BUF_ERROR)
+			break;
+
+		if (ret_xz == LZMA_STREAM_END) {
+			base->eof = true;
+			break;
+		}
+
+		if (ret_xz != LZMA_OK) {
+			fprintf(stderr,
+				"%s: internal error in xz decoder.\n",
+				wrapped->get_filename(wrapped));
+			return -1;
+		}
+	}
+
+	return 0;
+}
+
+static void cleanup(istream_comp_t *base)
+{
+	istream_xz_t *xz = (istream_xz_t *)base;
+
+	lzma_end(&xz->strm);
+}
+
+istream_comp_t *istream_xz_create(const char *filename)
+{
+	istream_xz_t *xz = calloc(1, sizeof(*xz));
+	istream_comp_t *base = (istream_comp_t *)xz;
+	sqfs_u64 memlimit = 65 * 1024 * 1024;
+	lzma_ret ret_xz;
+
+	if (xz == NULL) {
+		fprintf(stderr, "%s: creating xz decoder: %s.\n",
+			filename, strerror(errno));
+		return NULL;
+	}
+
+	ret_xz = lzma_stream_decoder(&xz->strm, memlimit, 0);
+
+	if (ret_xz != LZMA_OK) {
+		fprintf(stderr,
+			"%s: error initializing xz decoder.\n",
+			filename);
+		free(xz);
+		return NULL;
+	}
+
+	((istream_t *)base)->precache = precache;
+	base->cleanup = cleanup;
+	return base;
+}