From c7eae87870da755586446fecb4eb30063c1d641c Mon Sep 17 00:00:00 2001 From: David Oberhollenzer Date: Sun, 11 Jun 2023 17:59:21 +0200 Subject: libio: move istream buffer logic into interall callbacks Signed-off-by: David Oberhollenzer --- lib/io/src/mem.c | 58 ++++++++++++++++++++++-------------- lib/io/src/unix/istream.c | 72 +++++++++++++++++++++++++++++++++------------ lib/io/src/win32/istream.c | 64 ++++++++++++++++++++++++++++++---------- lib/io/src/xfrm/istream.c | 64 +++++++++++++++++++++++++++++----------- lib/io/test/istream_mem.c | 3 +- lib/io/test/istream_read.c | 1 - lib/io/test/istream_skip.c | 1 - lib/io/test/stream_splice.c | 1 - lib/io/test/xfrm.c | 2 -- 9 files changed, 188 insertions(+), 78 deletions(-) (limited to 'lib/io') diff --git a/lib/io/src/mem.c b/lib/io/src/mem.c index 39e0d6b..bbb186a 100644 --- a/lib/io/src/mem.c +++ b/lib/io/src/mem.c @@ -20,38 +20,52 @@ typedef struct { const void *data; size_t size; + size_t offset; + size_t visible; char *name; } mem_istream_t; -static int mem_in_precache(istream_t *strm) +static int mem_get_buffered_data(istream_t *strm, const sqfs_u8 **out, + size_t *size, size_t want) { mem_istream_t *mem = (mem_istream_t *)strm; - size_t diff; + size_t have = mem->size - mem->offset; - assert(strm->buffer >= mem->buffer); - assert(strm->buffer <= (mem->buffer + mem->bufsz)); - assert(strm->buffer_used <= mem->bufsz); - assert((size_t)(strm->buffer - mem->buffer) <= - (mem->bufsz - strm->buffer_used)); + if (have > mem->bufsz) + have = mem->bufsz; - if (strm->buffer_used > 0) - memmove(mem->buffer, strm->buffer, strm->buffer_used); + if (want > have) + want = have; - strm->buffer = mem->buffer; + if (mem->visible == 0 || mem->visible < want) { + memcpy(mem->buffer + mem->visible, + (const char *)mem->data + mem->offset + mem->visible, + have - mem->visible); + mem->visible = have; + } - diff = mem->bufsz - strm->buffer_used; - if (diff > mem->size) - diff = mem->size; + *out = mem->buffer; + *size = mem->visible; + return (mem->visible == 0) ? 1 : 0; +} - if (diff > 0) { - memcpy(mem->buffer + strm->buffer_used, mem->data, diff); - strm->buffer_used += diff; - mem->data = (const char *)mem->data + diff; - mem->size -= diff; - } +static void mem_advance_buffer(istream_t *strm, size_t count) +{ + mem_istream_t *mem = (mem_istream_t *)strm; - return 0; + assert(count <= mem->visible); + + if (count > 0 && count < mem->visible) + memmove(mem->buffer, mem->buffer + count, mem->visible - count); + + mem->offset += count; + mem->visible -= count; + + if (mem->visible < mem->bufsz) { + memset(mem->buffer + mem->visible, 0, + mem->bufsz - mem->visible); + } } static const char *mem_in_get_filename(istream_t *strm) @@ -88,8 +102,8 @@ istream_t *istream_memory_create(const char *name, size_t bufsz, mem->data = data; mem->size = size; mem->bufsz = bufsz; - strm->buffer = mem->buffer; - strm->precache = mem_in_precache; + strm->get_buffered_data = mem_get_buffered_data; + strm->advance_buffer = mem_advance_buffer; strm->get_filename = mem_in_get_filename; return strm; } diff --git a/lib/io/src/unix/istream.c b/lib/io/src/unix/istream.c index 8a2dc26..60abff1 100644 --- a/lib/io/src/unix/istream.c +++ b/lib/io/src/unix/istream.c @@ -12,30 +12,35 @@ typedef struct { int fd; bool eof; + size_t buffer_offset; + size_t buffer_used; sqfs_u8 buffer[BUFSZ]; } file_istream_t; -static int file_precache(istream_t *strm) +static int precache(istream_t *strm) { file_istream_t *file = (file_istream_t *)strm; - assert(strm->buffer >= file->buffer); - assert(strm->buffer <= (file->buffer + BUFSZ)); - assert(strm->buffer_used <= BUFSZ); - assert((size_t)(strm->buffer - file->buffer) <= - (BUFSZ - strm->buffer_used)); + if (file->eof) + return 0; - if (strm->buffer_used > 0) - memmove(file->buffer, strm->buffer, strm->buffer_used); + if (file->buffer_offset > 0 && + file->buffer_offset < file->buffer_used) { + memmove(file->buffer, file->buffer + file->buffer_offset, + file->buffer_used - file->buffer_offset); + } - strm->buffer = file->buffer; + file->buffer_used -= file->buffer_offset; + file->buffer_offset = 0; - while (!file->eof && strm->buffer_used < BUFSZ) { - ssize_t ret = read(file->fd, file->buffer + strm->buffer_used, - BUFSZ - strm->buffer_used); + while (file->buffer_used < BUFSZ) { + ssize_t ret = read(file->fd, file->buffer + file->buffer_used, + BUFSZ - file->buffer_used); - if (ret == 0) + if (ret == 0) { file->eof = true; + break; + } if (ret < 0) { if (errno == EINTR) @@ -45,12 +50,43 @@ static int file_precache(istream_t *strm) return -1; } - strm->buffer_used += ret; + file->buffer_used += ret; } return 0; } +static int file_get_buffered_data(istream_t *strm, const sqfs_u8 **out, + size_t *size, size_t want) +{ + file_istream_t *file = (file_istream_t *)strm; + + if (want > BUFSZ) + want = BUFSZ; + + if (file->buffer_used == 0 || + (file->buffer_used - file->buffer_offset) < want) { + int ret = precache(strm); + if (ret) + return ret; + } + + *out = file->buffer + file->buffer_offset; + *size = file->buffer_used - file->buffer_offset; + return (file->eof && *size == 0) ? 1 : 0; +} + +static void file_advance_buffer(istream_t *strm, size_t count) +{ + file_istream_t *file = (file_istream_t *)strm; + + assert(count <= file->buffer_used); + + file->buffer_offset += count; + + assert(file->buffer_offset <= file->buffer_used); +} + static const char *file_get_filename(istream_t *strm) { file_istream_t *file = (file_istream_t *)strm; @@ -93,8 +129,8 @@ istream_t *istream_open_file(const char *path) goto fail_path; } - strm->buffer = file->buffer; - strm->precache = file_precache; + strm->get_buffered_data = file_get_buffered_data; + strm->advance_buffer = file_advance_buffer; strm->get_filename = file_get_filename; return strm; fail_path: @@ -119,8 +155,8 @@ istream_t *istream_open_stdin(void) goto fail; file->fd = STDIN_FILENO; - strm->buffer = file->buffer; - strm->precache = file_precache; + strm->get_buffered_data = file_get_buffered_data; + strm->advance_buffer = file_advance_buffer; strm->get_filename = file_get_filename; return strm; fail: diff --git a/lib/io/src/win32/istream.c b/lib/io/src/win32/istream.c index c4d7458..9bc32a0 100644 --- a/lib/io/src/win32/istream.c +++ b/lib/io/src/win32/istream.c @@ -15,10 +15,12 @@ typedef struct { HANDLE hnd; bool eof; + size_t buffer_offset; + size_t buffer_used; sqfs_u8 buffer[BUFSZ]; } file_istream_t; -static int file_precache(istream_t *strm) +static int precache(istream_t *strm) { file_istream_t *file = (file_istream_t *)strm; DWORD diff, actual; @@ -27,22 +29,21 @@ static int file_precache(istream_t *strm) if (file->eof) return 0; - assert(strm->buffer >= file->buffer); - assert(strm->buffer <= (file->buffer + BUFSZ)); - assert(strm->buffer_used <= BUFSZ); - assert((size_t)(strm->buffer - file->buffer) <= - (BUFSZ - strm->buffer_used)); + if (file->buffer_offset > 0 && + file->buffer_offset < file->buffer_used) { + memmove(file->buffer, file->buffer + file->buffer_offset, + file->buffer_used - file->buffer_offset); + } - if (strm->buffer_used > 0) - memmove(file->buffer, strm->buffer, strm->buffer_used); + file->buffer_used -= file->buffer_offset; + file->buffer_offset = 0; - strm->buffer = file->buffer; hnd = file->path == NULL ? GetStdHandle(STD_INPUT_HANDLE) : file->hnd; - while (strm->buffer_used < sizeof(file->buffer)) { - diff = sizeof(file->buffer) - strm->buffer_used; + while (file->buffer_used < sizeof(file->buffer)) { + diff = sizeof(file->buffer) - file->buffer_used; - if (!ReadFile(hnd, file->buffer + strm->buffer_used, + if (!ReadFile(hnd, file->buffer + file->buffer_used, diff, &actual, NULL)) { DWORD error = GetLastError(); @@ -63,12 +64,43 @@ static int file_precache(istream_t *strm) break; } - strm->buffer_used += actual; + file->buffer_used += actual; } return 0; } +static int file_get_buffered_data(istream_t *strm, const sqfs_u8 **out, + size_t *size, size_t want) +{ + file_istream_t *file = (file_istream_t *)strm; + + if (want > BUFSZ) + want = BUFSZ; + + if (file->buffer_used == 0 || + (file->buffer_used - file->buffer_offset) < want) { + int ret = precache(strm); + if (ret) + return ret; + } + + *out = file->buffer + file->buffer_offset; + *size = file->buffer_used - file->buffer_offset; + return (file->eof && *size == 0) ? 1 : 0; +} + +static void file_advance_buffer(istream_t *strm, size_t count) +{ + file_istream_t *file = (file_istream_t *)strm; + + assert(count <= file->buffer_used); + + file->buffer_offset += count; + + assert(file->buffer_offset <= file->buffer_used); +} + static const char *file_get_filename(istream_t *strm) { file_istream_t *file = (file_istream_t *)strm; @@ -122,7 +154,8 @@ istream_t *istream_open_file(const char *path) free(wpath); strm->buffer = file->buffer; - strm->precache = file_precache; + strm->get_buffered_data = file_get_buffered_data; + strm->advance_buffer = file_advance_buffer; strm->get_filename = file_get_filename; return strm; fail_path: @@ -146,7 +179,8 @@ istream_t *istream_open_stdin(void) sqfs_object_init(file, file_destroy, NULL); strm->buffer = file->buffer; - strm->precache = file_precache; + strm->get_buffered_data = file_get_buffered_data; + strm->advance_buffer = file_advance_buffer; strm->get_filename = file_get_filename; return strm; } diff --git a/lib/io/src/xfrm/istream.c b/lib/io/src/xfrm/istream.c index bad3e22..c9bc41a 100644 --- a/lib/io/src/xfrm/istream.c +++ b/lib/io/src/xfrm/istream.c @@ -12,27 +12,28 @@ typedef struct istream_xfrm_t { istream_t *wrapped; xfrm_stream_t *xfrm; + size_t buffer_offset; + size_t buffer_used; sqfs_u8 uncompressed[BUFSZ]; } istream_xfrm_t; -static int xfrm_precache(istream_t *base) +static int precache(istream_t *base) { istream_xfrm_t *xfrm = (istream_xfrm_t *)base; int ret; - assert(base->buffer >= xfrm->uncompressed); - assert(base->buffer <= (xfrm->uncompressed + BUFSZ)); - assert(base->buffer_used <= BUFSZ); - assert((size_t)(base->buffer - xfrm->uncompressed) <= - (BUFSZ - base->buffer_used)); - - if (base->buffer_used > 0) - memmove(xfrm->uncompressed, base->buffer, base->buffer_used); + if (xfrm->buffer_offset > 0 && + xfrm->buffer_offset < xfrm->buffer_used) { + memmove(xfrm->uncompressed, + xfrm->uncompressed + xfrm->buffer_offset, + xfrm->buffer_used - xfrm->buffer_offset); + } - base->buffer = xfrm->uncompressed; + xfrm->buffer_used -= xfrm->buffer_offset; + xfrm->buffer_offset = 0; for (;;) { - sqfs_u32 in_off = 0, out_off = base->buffer_used; + sqfs_u32 in_off = 0, out_off = xfrm->buffer_used; int mode = XFRM_STREAM_FLUSH_NONE; const sqfs_u8 *ptr; size_t avail; @@ -41,10 +42,8 @@ static int xfrm_precache(istream_t *base) sizeof(xfrm->uncompressed)); if (ret < 0) return ret; - if (ret > 0) { + if (ret > 0) mode = XFRM_STREAM_FLUSH_FULL; - avail = 0; - } ret = xfrm->xfrm->process_data(xfrm->xfrm, ptr, avail, @@ -58,7 +57,7 @@ static int xfrm_precache(istream_t *base) return -1; } - base->buffer_used = out_off; + xfrm->buffer_used = out_off; istream_advance_buffer(xfrm->wrapped, in_off); if (ret == XFRM_STREAM_BUFFER_FULL || out_off >= BUFSZ) @@ -71,6 +70,37 @@ static int xfrm_precache(istream_t *base) return 0; } +static int xfrm_get_buffered_data(istream_t *strm, const sqfs_u8 **out, + size_t *size, size_t want) +{ + istream_xfrm_t *xfrm = (istream_xfrm_t *)strm; + + if (want > BUFSZ) + want = BUFSZ; + + if (xfrm->buffer_used == 0 || + (xfrm->buffer_used - xfrm->buffer_offset) < want) { + int ret = precache(strm); + if (ret) + return ret; + } + + *out = xfrm->uncompressed + xfrm->buffer_offset; + *size = xfrm->buffer_used - xfrm->buffer_offset; + return (*size == 0) ? 1 : 0; +} + +static void xfrm_advance_buffer(istream_t *strm, size_t count) +{ + istream_xfrm_t *xfrm = (istream_xfrm_t *)strm; + + assert(count <= xfrm->buffer_used); + + xfrm->buffer_offset += count; + + assert(xfrm->buffer_offset <= xfrm->buffer_used); +} + static const char *xfrm_get_filename(istream_t *strm) { istream_xfrm_t *xfrm = (istream_xfrm_t *)strm; @@ -100,9 +130,9 @@ istream_t *istream_xfrm_create(istream_t *strm, xfrm_stream_t *xfrm) stream->wrapped = sqfs_grab(strm); stream->xfrm = sqfs_grab(xfrm); - base->precache = xfrm_precache; + base->get_buffered_data = xfrm_get_buffered_data; + base->advance_buffer = xfrm_advance_buffer; base->get_filename = xfrm_get_filename; - base->buffer = stream->uncompressed; return base; fail: fprintf(stderr, "%s: error initializing decompressor stream.\n", diff --git a/lib/io/test/istream_mem.c b/lib/io/test/istream_mem.c index 84da9a9..a486899 100644 --- a/lib/io/test/istream_mem.c +++ b/lib/io/test/istream_mem.c @@ -63,8 +63,9 @@ int main(int argc, char **argv) TEST_EQUAL_UI(ptr[j], byte_at_offset(i + j)); } - istream_advance_buffer(in, eat_all ? size : (size / 2)); + diff = eat_all ? size : (size / 2); eat_all = !eat_all; + istream_advance_buffer(in, diff); } sqfs_drop(in); diff --git a/lib/io/test/istream_read.c b/lib/io/test/istream_read.c index 176832b..8ba4540 100644 --- a/lib/io/test/istream_read.c +++ b/lib/io/test/istream_read.c @@ -100,7 +100,6 @@ int main(int argc, char **argv) TEST_ASSERT(read_off <= end2); } - TEST_ASSERT(dummy->buffer_used == 0); sqfs_drop(dummy); return EXIT_SUCCESS; } diff --git a/lib/io/test/istream_skip.c b/lib/io/test/istream_skip.c index 15d7f5d..3a02c8c 100644 --- a/lib/io/test/istream_skip.c +++ b/lib/io/test/istream_skip.c @@ -89,7 +89,6 @@ int main(int argc, char **argv) TEST_ASSERT(read_off <= end2); } - TEST_ASSERT(dummy->buffer_used == 0); sqfs_drop(dummy); return EXIT_SUCCESS; } diff --git a/lib/io/test/stream_splice.c b/lib/io/test/stream_splice.c index 223c39d..754f9cb 100644 --- a/lib/io/test/stream_splice.c +++ b/lib/io/test/stream_splice.c @@ -91,7 +91,6 @@ int main(int argc, char **argv) TEST_EQUAL_UI(total, out_offset); } - TEST_ASSERT(in->buffer_used == 0); TEST_EQUAL_UI(total, end2); TEST_EQUAL_UI(out_offset, end2); sqfs_drop(in); diff --git a/lib/io/test/xfrm.c b/lib/io/test/xfrm.c index 95b1f72..323987c 100644 --- a/lib/io/test/xfrm.c +++ b/lib/io/test/xfrm.c @@ -439,8 +439,6 @@ static void run_unpack_test(const void *blob, size_t size) ret = istream_read(istream, &c, 1); TEST_EQUAL_I(ret, 0); - TEST_EQUAL_UI(mem_istream->buffer_used, 0); - sqfs_drop(istream); TEST_EQUAL_UI(((sqfs_object_t *)mem_istream)->refcount, 1); TEST_EQUAL_UI(((sqfs_object_t *)xfrm)->refcount, 1); -- cgit v1.2.3