libfstream: guard against potential integer overflows

The differen compressor libraries use differnt integer types to tally
the buffer sizes. The libfstream library uses size_t, which may be
bigger than the actualy types, potentially causing an overflow if
trying to compress to much at once.

Signed-off-by: David Oberhollenzer <david.oberhollenzer@sigma-star.at>

2 files changed, 19 insertions, 2 deletions

diff --git a/lib/fstream/compress/bzip2.c b/lib/fstream/compress/bzip2.c
index 3ca425a..7f0c09a 100644
--- a/lib/fstream/compress/bzip2.c
+++ b/lib/fstream/compress/bzip2.c
@@ -21,7 +21,16 @@ static int flush_inbuf(ostream_comp_t *base, bool finish)
 	int ret;
 
 	bzip2->strm.next_in = (char *)base->inbuf;
-	bzip2->strm.avail_in = base->inbuf_used;
+
+	if (base->inbuf_used > sizeof(base->inbuf))
+		base->inbuf_used = sizeof(base->inbuf);
+
+	if ((sizeof(size_t) > sizeof(unsigned int)) &&
+	    (base->inbuf_used > (size_t)UINT_MAX)) {
+		bzip2->strm.avail_in = UINT_MAX;
+	} else {
+		bzip2->strm.avail_in = (unsigned int)base->inbuf_used;
+	}
 
 	for (;;) {
 		bzip2->strm.next_out = (char *)base->outbuf;
diff --git a/lib/fstream/compress/gzip.c b/lib/fstream/compress/gzip.c
index f604b71..e69f183 100644
--- a/lib/fstream/compress/gzip.c
+++ b/lib/fstream/compress/gzip.c
@@ -20,7 +20,15 @@ static int flush_inbuf(ostream_comp_t *base, bool finish)
 	size_t have;
 	int ret;
 
-	gzip->strm.avail_in = base->inbuf_used;
+	if (sizeof(size_t) > sizeof(uInt)) {
+		gzip->strm.avail_in = ~((uInt)0);
+
+		if ((size_t)gzip->strm.avail_in > base->inbuf_used)
+			gzip->strm.avail_in = (uInt)base->inbuf_used;
+	} else {
+		gzip->strm.avail_in = (uInt)base->inbuf_used;
+	}
+
 	gzip->strm.next_in = base->inbuf;
 
 	do {