libfstream: guard against potential integer overflows

The differen compressor libraries use differnt integer types to tally
the buffer sizes. The libfstream library uses size_t, which may be
bigger than the actualy types, potentially causing an overflow if
trying to compress to much at once.

Signed-off-by: David Oberhollenzer <david.oberhollenzer@sigma-star.at>

1 files changed, 18 insertions, 3 deletions

diff --git a/lib/fstream/uncompress/gzip.c b/lib/fstream/uncompress/gzip.c
index c2003db..1d6274c 100644
--- a/lib/fstream/uncompress/gzip.c
+++ b/lib/fstream/uncompress/gzip.c
@@ -18,6 +18,7 @@ static int precache(istream_t *base)
 {
 	istream_t *wrapped = ((istream_comp_t *)base)->wrapped;
 	istream_gzip_t *gzip = (istream_gzip_t *)base;
+	size_t avail_in, avail_out;
 	int ret;
 
 	for (;;) {
@@ -25,10 +26,24 @@ static int precache(istream_t *base)
 		if (ret != 0)
 			return ret;
 
-		gzip->strm.avail_in = wrapped->buffer_used;
-		gzip->strm.next_in = wrapped->buffer;
+		avail_in = wrapped->buffer_used;
+		avail_out = BUFSZ - base->buffer_used;
+
+		if (sizeof(size_t) > sizeof(uInt)) {
+			gzip->strm.avail_in = ~((uInt)0U);
+			gzip->strm.avail_out = ~((uInt)0U);
+
+			if ((size_t)gzip->strm.avail_in > avail_in)
+				gzip->strm.avail_in = (uInt)avail_in;
 
-		gzip->strm.avail_out = BUFSZ - base->buffer_used;
+			if ((size_t)gzip->strm.avail_out > avail_out)
+				gzip->strm.avail_out = (uInt)avail_out;
+		} else {
+			gzip->strm.avail_in = (uInt)avail_in;
+			gzip->strm.avail_out = (uInt)avail_out;
+		}
+
+		gzip->strm.next_in = wrapped->buffer;
 		gzip->strm.next_out = base->buffer + base->buffer_used;
 
 		ret = inflate(&gzip->strm, Z_NO_FLUSH);