libfstream: guard against potential integer overflows

The differen compressor libraries use differnt integer types to tally
the buffer sizes. The libfstream library uses size_t, which may be
bigger than the actualy types, potentially causing an overflow if
trying to compress to much at once.

Signed-off-by: David Oberhollenzer <david.oberhollenzer@sigma-star.at>

4 files changed, 56 insertions, 7 deletions

diff --git a/lib/fstream/compress/bzip2.c b/lib/fstream/compress/bzip2.c
index 3ca425a..7f0c09a 100644
--- a/lib/fstream/compress/bzip2.c
+++ b/lib/fstream/compress/bzip2.c
@@ -21,7 +21,16 @@ static int flush_inbuf(ostream_comp_t *base, bool finish)
 	int ret;
 
 	bzip2->strm.next_in = (char *)base->inbuf;
-	bzip2->strm.avail_in = base->inbuf_used;
+
+	if (base->inbuf_used > sizeof(base->inbuf))
+		base->inbuf_used = sizeof(base->inbuf);
+
+	if ((sizeof(size_t) > sizeof(unsigned int)) &&
+	    (base->inbuf_used > (size_t)UINT_MAX)) {
+		bzip2->strm.avail_in = UINT_MAX;
+	} else {
+		bzip2->strm.avail_in = (unsigned int)base->inbuf_used;
+	}
 
 	for (;;) {
 		bzip2->strm.next_out = (char *)base->outbuf;
diff --git a/lib/fstream/compress/gzip.c b/lib/fstream/compress/gzip.c
index f604b71..e69f183 100644
--- a/lib/fstream/compress/gzip.c
+++ b/lib/fstream/compress/gzip.c
@@ -20,7 +20,15 @@ static int flush_inbuf(ostream_comp_t *base, bool finish)
 	size_t have;
 	int ret;
 
-	gzip->strm.avail_in = base->inbuf_used;
+	if (sizeof(size_t) > sizeof(uInt)) {
+		gzip->strm.avail_in = ~((uInt)0);
+
+		if ((size_t)gzip->strm.avail_in > base->inbuf_used)
+			gzip->strm.avail_in = (uInt)base->inbuf_used;
+	} else {
+		gzip->strm.avail_in = (uInt)base->inbuf_used;
+	}
+
 	gzip->strm.next_in = base->inbuf;
 
 	do {
diff --git a/lib/fstream/uncompress/bzip2.c b/lib/fstream/uncompress/bzip2.c
index 429950a..3b44383 100644
--- a/lib/fstream/uncompress/bzip2.c
+++ b/lib/fstream/uncompress/bzip2.c
@@ -19,6 +19,7 @@ static int precache(istream_t *base)
 {
 	istream_bzip2_t *bzip2 = (istream_bzip2_t *)base;
 	istream_t *wrapped = ((istream_comp_t *)base)->wrapped;
+	size_t avail;
 	int ret;
 
 	for (;;) {
@@ -37,11 +38,27 @@ static int precache(istream_t *base)
 		if (ret != 0)
 			return ret;
 
+		avail = wrapped->buffer_used;
+		if ((sizeof(size_t) > sizeof(unsigned int)) &&
+		    (avail > (size_t)UINT_MAX)) {
+			avail = UINT_MAX;
+		}
+
 		bzip2->strm.next_in = (char *)wrapped->buffer;
-		bzip2->strm.avail_in = wrapped->buffer_used;
+		bzip2->strm.avail_in = (unsigned int)avail;
+
+		if (base->buffer_used > BUFSZ)
+			base->buffer_used = BUFSZ;
+
+		avail = BUFSZ - base->buffer_used;
+
+		if ((sizeof(size_t) > sizeof(unsigned int)) &&
+		    (avail > (size_t)UINT_MAX)) {
+			avail = UINT_MAX;
+		}
 
 		bzip2->strm.next_out = (char *)base->buffer + base->buffer_used;
-		bzip2->strm.avail_out = BUFSZ - base->buffer_used;
+		bzip2->strm.avail_out = (unsigned int)avail;
 
 		if (bzip2->strm.avail_out < 1)
 			break;
diff --git a/lib/fstream/uncompress/gzip.c b/lib/fstream/uncompress/gzip.c
index c2003db..1d6274c 100644
--- a/lib/fstream/uncompress/gzip.c
+++ b/lib/fstream/uncompress/gzip.c
@@ -18,6 +18,7 @@ static int precache(istream_t *base)
 {
 	istream_t *wrapped = ((istream_comp_t *)base)->wrapped;
 	istream_gzip_t *gzip = (istream_gzip_t *)base;
+	size_t avail_in, avail_out;
 	int ret;
 
 	for (;;) {
@@ -25,10 +26,24 @@ static int precache(istream_t *base)
 		if (ret != 0)
 			return ret;
 
-		gzip->strm.avail_in = wrapped->buffer_used;
-		gzip->strm.next_in = wrapped->buffer;
+		avail_in = wrapped->buffer_used;
+		avail_out = BUFSZ - base->buffer_used;
+
+		if (sizeof(size_t) > sizeof(uInt)) {
+			gzip->strm.avail_in = ~((uInt)0U);
+			gzip->strm.avail_out = ~((uInt)0U);
+
+			if ((size_t)gzip->strm.avail_in > avail_in)
+				gzip->strm.avail_in = (uInt)avail_in;
 
-		gzip->strm.avail_out = BUFSZ - base->buffer_used;
+			if ((size_t)gzip->strm.avail_out > avail_out)
+				gzip->strm.avail_out = (uInt)avail_out;
+		} else {
+			gzip->strm.avail_in = (uInt)avail_in;
+			gzip->strm.avail_out = (uInt)avail_out;
+		}
+
+		gzip->strm.next_in = wrapped->buffer;
 		gzip->strm.next_out = base->buffer + base->buffer_used;
 
 		ret = inflate(&gzip->strm, Z_NO_FLUSH);