libfstream: guard against potential integer overflows

The differen compressor libraries use differnt integer types to tally
the buffer sizes. The libfstream library uses size_t, which may be
bigger than the actualy types, potentially causing an overflow if
trying to compress to much at once.

Signed-off-by: David Oberhollenzer <david.oberhollenzer@sigma-star.at>

1 files changed, 10 insertions, 1 deletions

diff --git a/lib/fstream/compress/bzip2.c b/lib/fstream/compress/bzip2.c
index 3ca425a..7f0c09a 100644
--- a/lib/fstream/compress/bzip2.c
+++ b/lib/fstream/compress/bzip2.c
@@ -21,7 +21,16 @@ static int flush_inbuf(ostream_comp_t *base, bool finish)
 	int ret;
 
 	bzip2->strm.next_in = (char *)base->inbuf;
-	bzip2->strm.avail_in = base->inbuf_used;
+
+	if (base->inbuf_used > sizeof(base->inbuf))
+		base->inbuf_used = sizeof(base->inbuf);
+
+	if ((sizeof(size_t) > sizeof(unsigned int)) &&
+	    (base->inbuf_used > (size_t)UINT_MAX)) {
+		bzip2->strm.avail_in = UINT_MAX;
+	} else {
+		bzip2->strm.avail_in = (unsigned int)base->inbuf_used;
+	}
 
 	for (;;) {
 		bzip2->strm.next_out = (char *)base->outbuf;