Implement ostream compressor wrapper

Signed-off-by: David Oberhollenzer <david.oberhollenzer@sigma-star.at>

7 files changed, 408 insertions, 0 deletions

diff --git a/include/fstream.h b/include/fstream.h
index cc562ca..cb992f8 100644
--- a/include/fstream.h
+++ b/include/fstream.h
@@ -55,6 +55,21 @@ enum {
 	OSTREAM_OPEN_SPARSE = 0x02,
 };
 
+enum {
+	/**
+	 * @brief Deflate compressor with gzip headers.
+	 *
+	 * This actually creates a gzip compatible file, including a
+	 * gzip header and trailer.
+	 */
+	FSTREAM_COMPRESSOR_GZIP = 1,
+
+	FSTREAM_COMPRESSOR_XZ = 2,
+
+	FSTREAM_COMPRESSOR_MIN = 1,
+	FSTREAM_COMPRESSOR_MAX = 2,
+};
+
 #ifdef __cplusplus
 extern "C" {
 #endif
@@ -109,6 +124,27 @@ SQFS_INTERNAL istream_t *istream_open_file(const char *path);
 SQFS_INTERNAL istream_t *istream_open_stdin(void);
 
 /**
+ * @brief Create an output stream that transparently compresses data.
+ *
+ * @memberof ostream_t
+ *
+ * This function creates an output stream that transparently compresses all
+ * data appended to it and writes the compressed data to an underlying, wrapped
+ * output stream.
+ *
+ * The new stream takes ownership of the wrapped stream and destroys it when
+ * the compressor stream is destroyed. If this function fails, the wrapped
+ * stream is also destroyed.
+ *
+ * @param strm A pointer to another stream that should be wrapped.
+ * @param comp_id An identifier describing the compressor to use.
+ *
+ * @return A pointer to an output stream on success, NULL on failure.
+ */
+SQFS_INTERNAL ostream_t *ostream_compressor_create(ostream_t *strm,
+						   int comp_id);
+
+/**
  * @brief Append a block of data to an output stream.
  *
  * @memberof ostream_t
@@ -245,6 +281,33 @@ SQFS_INTERNAL sqfs_s32 ostream_append_from_istream(ostream_t *out,
 						   istream_t *in,
 						   sqfs_u32 size);
 
+/**
+ * @brief Resolve a compressor name to an ID.
+ *
+ * @param name A compressor name.
+ *
+ * @return A compressor ID on success, -1 on failure.
+ */
+SQFS_INTERNAL int fstream_compressor_id_from_name(const char *name);
+
+/**
+ * @brief Resolve a id to a  compressor name.
+ *
+ * @param id A compressor ID.
+ *
+ * @return A compressor name on success, NULL on failure.
+ */
+SQFS_INTERNAL const char *fstream_compressor_name_from_id(int id);
+
+/**
+ * @brief Check if support for a given compressor has been built in.
+ *
+ * @param id A compressor ID.
+ *
+ * @return True if the compressor is supported, false if not.
+ */
+SQFS_INTERNAL bool fstream_compressor_exists(int id);
+
 #ifdef __cplusplus
 }
 #endif
diff --git a/lib/fstream/Makemodule.am b/lib/fstream/Makemodule.am
index 27e4701..9178647 100644
--- a/lib/fstream/Makemodule.am
+++ b/lib/fstream/Makemodule.am
@@ -2,6 +2,8 @@ libfstream_a_SOURCES = include/fstream.h
 libfstream_a_SOURCES += lib/fstream/internal.h
 libfstream_a_SOURCES += lib/fstream/ostream.c lib/fstream/printf.c
 libfstream_a_SOURCES += lib/fstream/istream.c
+libfstream_a_SOURCES += lib/fstream/compressor.c
+libfstream_a_SOURCES += lib/fstream/compress/ostream_compressor.c
 libfstream_a_CFLAGS = $(AM_CFLAGS) $(ZLIB_CFLAGS) $(XZ_CFLAGS)
 libfstream_a_CPPFLAGS = $(AM_CPPFLAGS)
 
@@ -14,4 +16,14 @@ libfstream_a_SOURCES += lib/fstream/unix/ostream.c
 libfstream_a_SOURCES += lib/fstream/unix/istream.c
 endif
 
+if WITH_XZ
+libfstream_a_SOURCES += lib/fstream/compress/xz.c
+libfstream_a_CPPFLAGS += -DWITH_XZ
+endif
+
+if WITH_GZIP
+libfstream_a_SOURCES += lib/fstream/compress/gzip.c
+libfstream_a_CPPFLAGS += -DWITH_GZIP
+endif
+
 noinst_LIBRARIES += libfstream.a
diff --git a/lib/fstream/compress/gzip.c b/lib/fstream/compress/gzip.c
new file mode 100644
index 0000000..f604b71
--- /dev/null
+++ b/lib/fstream/compress/gzip.c
@@ -0,0 +1,81 @@
+/* SPDX-License-Identifier: GPL-3.0-or-later */
+/*
+ * gzip.c
+ *
+ * Copyright (C) 2019 David Oberhollenzer <goliath@infraroot.at>
+ */
+#include "../internal.h"
+
+#include <zlib.h>
+
+typedef struct {
+	ostream_comp_t base;
+
+	z_stream strm;
+} ostream_gzip_t;
+
+static int flush_inbuf(ostream_comp_t *base, bool finish)
+{
+	ostream_gzip_t *gzip = (ostream_gzip_t *)base;
+	size_t have;
+	int ret;
+
+	gzip->strm.avail_in = base->inbuf_used;
+	gzip->strm.next_in = base->inbuf;
+
+	do {
+		gzip->strm.avail_out = BUFSZ;
+		gzip->strm.next_out = base->outbuf;
+
+		ret = deflate(&gzip->strm, finish ? Z_FINISH : Z_NO_FLUSH);
+
+		if (ret == Z_STREAM_ERROR) {
+			fprintf(stderr,
+				"%s: internal error in gzip compressor.\n",
+				base->wrapped->get_filename(base->wrapped));
+			return -1;
+		}
+
+		have = BUFSZ - gzip->strm.avail_out;
+
+		if (base->wrapped->append(base->wrapped, base->outbuf, have))
+			return -1;
+	} while (gzip->strm.avail_out == 0);
+
+	base->inbuf_used = 0;
+	return 0;
+}
+
+static void cleanup(ostream_comp_t *base)
+{
+	ostream_gzip_t *gzip = (ostream_gzip_t *)base;
+
+	deflateEnd(&gzip->strm);
+}
+
+ostream_comp_t *ostream_gzip_create(const char *filename)
+{
+	ostream_gzip_t *gzip = calloc(1, sizeof(*gzip));
+	ostream_comp_t *base = (ostream_comp_t *)gzip;
+	int ret;
+
+	if (gzip == NULL) {
+		fprintf(stderr, "%s: creating gzip wrapper: %s.\n",
+			filename, strerror(errno));
+		return NULL;
+	}
+
+	ret = deflateInit2(&gzip->strm, 9, Z_DEFLATED, 16 + 15, 8,
+			   Z_DEFAULT_STRATEGY);
+	if (ret != Z_OK) {
+		fprintf(stderr,
+			"%s: internal error creating gzip compressor.\n",
+			filename);
+		free(gzip);
+		return NULL;
+	}
+
+	base->flush_inbuf = flush_inbuf;
+	base->cleanup = cleanup;
+	return base;
+}
diff --git a/lib/fstream/compress/ostream_compressor.c b/lib/fstream/compress/ostream_compressor.c
new file mode 100644
index 0000000..5137f1d
--- /dev/null
+++ b/lib/fstream/compress/ostream_compressor.c
@@ -0,0 +1,98 @@
+/* SPDX-License-Identifier: GPL-3.0-or-later */
+/*
+ * ostream_compressor.c
+ *
+ * Copyright (C) 2019 David Oberhollenzer <goliath@infraroot.at>
+ */
+#include "../internal.h"
+
+static int comp_append(ostream_t *strm, const void *data, size_t size)
+{
+	ostream_comp_t *comp = (ostream_comp_t *)strm;
+	size_t diff;
+
+	while (size > 0) {
+		if (comp->inbuf_used >= BUFSZ) {
+			if (comp->flush_inbuf(comp, false))
+				return -1;
+		}
+
+		diff = BUFSZ - comp->inbuf_used;
+
+		if (diff > size)
+			diff = size;
+
+		memcpy(comp->inbuf + comp->inbuf_used, data, diff);
+
+		comp->inbuf_used += diff;
+		data = (const char *)data + diff;
+		size -= diff;
+	}
+
+	return 0;
+}
+
+static int comp_flush(ostream_t *strm)
+{
+	ostream_comp_t *comp = (ostream_comp_t *)strm;
+
+	if (comp->inbuf_used > 0) {
+		if (comp->flush_inbuf(comp, true))
+			return -1;
+	}
+
+	return comp->wrapped->flush(comp->wrapped);
+}
+
+static const char *comp_get_filename(ostream_t *strm)
+{
+	ostream_comp_t *comp = (ostream_comp_t *)strm;
+
+	return comp->wrapped->get_filename(comp->wrapped);
+}
+
+static void comp_destroy(sqfs_object_t *obj)
+{
+	ostream_comp_t *comp = (ostream_comp_t *)obj;
+
+	comp->cleanup(comp);
+	sqfs_destroy(comp->wrapped);
+	free(comp);
+}
+
+ostream_t *ostream_compressor_create(ostream_t *strm, int comp_id)
+{
+	ostream_comp_t *comp = NULL;
+	sqfs_object_t *obj;
+	ostream_t *base;
+
+	switch (comp_id) {
+	case FSTREAM_COMPRESSOR_GZIP:
+#ifdef WITH_GZIP
+		comp = ostream_gzip_create(strm->get_filename(strm));
+#endif
+		break;
+	case FSTREAM_COMPRESSOR_XZ:
+#ifdef WITH_XZ
+		comp = ostream_xz_create(strm->get_filename(strm));
+#endif
+		break;
+	default:
+		break;
+	}
+
+	if (comp == NULL)
+		return NULL;
+
+	comp->wrapped = strm;
+	comp->inbuf_used = 0;
+
+	base = (ostream_t *)comp;
+	base->append = comp_append;
+	base->flush = comp_flush;
+	base->get_filename = comp_get_filename;
+
+	obj = (sqfs_object_t *)comp;
+	obj->destroy = comp_destroy;
+	return base;
+}
diff --git a/lib/fstream/compress/xz.c b/lib/fstream/compress/xz.c
new file mode 100644
index 0000000..65bda0b
--- /dev/null
+++ b/lib/fstream/compress/xz.c
@@ -0,0 +1,80 @@
+/* SPDX-License-Identifier: GPL-3.0-or-later */
+/*
+ * xz.c
+ *
+ * Copyright (C) 2019 David Oberhollenzer <goliath@infraroot.at>
+ */
+#include "../internal.h"
+
+#include <lzma.h>
+
+typedef struct {
+	ostream_comp_t base;
+
+	lzma_stream strm;
+} ostream_xz_t;
+
+static int flush_inbuf(ostream_comp_t *base, bool finish)
+{
+	ostream_xz_t *xz = (ostream_xz_t *)base;
+	lzma_ret ret_xz;
+	size_t have;
+
+	xz->strm.next_in = base->inbuf;
+	xz->strm.avail_in = base->inbuf_used;
+
+	do {
+		xz->strm.next_out = base->outbuf;
+		xz->strm.avail_out = BUFSZ;
+
+		ret_xz = lzma_code(&xz->strm, finish ? LZMA_FINISH : LZMA_RUN);
+
+		if ((ret_xz != LZMA_OK) && (ret_xz != LZMA_STREAM_END)) {
+			fprintf(stderr,
+				"%s: internal error in XZ compressor.\n",
+				base->wrapped->get_filename(base->wrapped));
+			return -1;
+		}
+
+		have = BUFSZ - xz->strm.avail_out;
+
+		if (base->wrapped->append(base->wrapped, base->outbuf, have))
+			return -1;
+	} while (xz->strm.avail_out == 0);
+
+	base->inbuf_used = 0;
+	return 0;
+}
+
+static void cleanup(ostream_comp_t *base)
+{
+	ostream_xz_t *xz = (ostream_xz_t *)base;
+
+	lzma_end(&xz->strm);
+}
+
+ostream_comp_t *ostream_xz_create(const char *filename)
+{
+	ostream_xz_t *xz = calloc(1, sizeof(*xz));
+	ostream_comp_t *base = (ostream_comp_t *)xz;
+	lzma_ret ret_xz;
+
+	if (xz == NULL) {
+		fprintf(stderr, "%s: creating xz wrapper: %s.\n",
+			filename, strerror(errno));
+		return NULL;
+	}
+
+	ret_xz = lzma_easy_encoder(&xz->strm, LZMA_PRESET_DEFAULT,
+				   LZMA_CHECK_CRC64);
+	if (ret_xz != LZMA_OK) {
+		fprintf(stderr, "%s: error initializing XZ compressor\n",
+			filename);
+		free(xz);
+		return NULL;
+	}
+
+	base->flush_inbuf = flush_inbuf;
+	base->cleanup = cleanup;
+	return base;
+}
diff --git a/lib/fstream/compressor.c b/lib/fstream/compressor.c
new file mode 100644
index 0000000..b8f9c6b
--- /dev/null
+++ b/lib/fstream/compressor.c
@@ -0,0 +1,47 @@
+/* SPDX-License-Identifier: GPL-3.0-or-later */
+/*
+ * compressor.c
+ *
+ * Copyright (C) 2019 David Oberhollenzer <goliath@infraroot.at>
+ */
+#include "internal.h"
+
+int fstream_compressor_id_from_name(const char *name)
+{
+	if (strcmp(name, "gzip") == 0)
+		return FSTREAM_COMPRESSOR_GZIP;
+
+	if (strcmp(name, "xz") == 0)
+		return FSTREAM_COMPRESSOR_XZ;
+
+	return -1;
+}
+
+const char *fstream_compressor_name_from_id(int id)
+{
+	if (id == FSTREAM_COMPRESSOR_GZIP)
+		return "gzip";
+
+	if (id == FSTREAM_COMPRESSOR_XZ)
+		return "xz";
+
+	return NULL;
+}
+
+bool fstream_compressor_exists(int id)
+{
+	switch (id) {
+#ifdef WITH_GZIP
+	case FSTREAM_COMPRESSOR_GZIP:
+		return true;
+#endif
+#ifdef WITH_XZ
+	case FSTREAM_COMPRESSOR_XZ:
+		return true;
+#endif
+	default:
+		break;
+	}
+
+	return false;
+}
diff --git a/lib/fstream/internal.h b/lib/fstream/internal.h
index e8b0c07..ae6a29a 100644
--- a/lib/fstream/internal.h
+++ b/lib/fstream/internal.h
@@ -22,4 +22,31 @@
 
 #define BUFSZ (262144)
 
+typedef struct ostream_comp_t {
+	ostream_t base;
+
+	ostream_t *wrapped;
+
+	size_t inbuf_used;
+
+	sqfs_u8 inbuf[BUFSZ];
+	sqfs_u8 outbuf[BUFSZ];
+
+	int (*flush_inbuf)(struct ostream_comp_t *ostrm, bool finish);
+
+	void (*cleanup)(struct ostream_comp_t *ostrm);
+} ostream_comp_t;
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+SQFS_INTERNAL ostream_comp_t *ostream_gzip_create(const char *filename);
+
+SQFS_INTERNAL ostream_comp_t *ostream_xz_create(const char *filename);
+
+#ifdef __cplusplus
+}
+#endif
+
 #endif /* INTERNAL_H */