From c7eae87870da755586446fecb4eb30063c1d641c Mon Sep 17 00:00:00 2001 From: David Oberhollenzer Date: Sun, 11 Jun 2023 17:59:21 +0200 Subject: libio: move istream buffer logic into interall callbacks Signed-off-by: David Oberhollenzer --- include/io/istream.h | 21 ++++--------- lib/io/src/mem.c | 58 ++++++++++++++++++++++-------------- lib/io/src/unix/istream.c | 72 +++++++++++++++++++++++++++++++++------------ lib/io/src/win32/istream.c | 64 ++++++++++++++++++++++++++++++---------- lib/io/src/xfrm/istream.c | 64 +++++++++++++++++++++++++++++----------- lib/io/test/istream_mem.c | 3 +- lib/io/test/istream_read.c | 1 - lib/io/test/istream_skip.c | 1 - lib/io/test/stream_splice.c | 1 - lib/io/test/xfrm.c | 2 -- lib/tar/src/iterator.c | 58 ++++++++++++++++++------------------ 11 files changed, 222 insertions(+), 123 deletions(-) diff --git a/include/io/istream.h b/include/io/istream.h index fca1440..8e86891 100644 --- a/include/io/istream.h +++ b/include/io/istream.h @@ -20,10 +20,10 @@ typedef struct istream_t { sqfs_object_t base; - size_t buffer_used; - const sqfs_u8 *buffer; + int (*get_buffered_data)(struct istream_t *strm, const sqfs_u8 **out, + size_t *size, size_t want); - int (*precache)(struct istream_t *strm); + void (*advance_buffer)(struct istream_t *strm, size_t count); const char *(*get_filename)(struct istream_t *strm); } istream_t; @@ -121,17 +121,7 @@ SQFS_INLINE const char *istream_get_filename(istream_t *strm) SQFS_INLINE int istream_get_buffered_data(istream_t *strm, const sqfs_u8 **out, size_t *size, size_t want) { - if (strm->buffer_used == 0 || strm->buffer_used < want) { - int ret = strm->precache(strm); - if (ret) - return ret; - if (strm->buffer_used == 0) - return 1; - } - - *out = strm->buffer; - *size = strm->buffer_used; - return 0; + return strm->get_buffered_data(strm, out, size, want); } /** @@ -148,8 +138,7 @@ SQFS_INLINE int istream_get_buffered_data(istream_t *strm, const sqfs_u8 **out, */ SQFS_INLINE void istream_advance_buffer(istream_t *strm, size_t count) { - strm->buffer += count; - strm->buffer_used -= count; + strm->advance_buffer(strm, count); } /** diff --git a/lib/io/src/mem.c b/lib/io/src/mem.c index 39e0d6b..bbb186a 100644 --- a/lib/io/src/mem.c +++ b/lib/io/src/mem.c @@ -20,38 +20,52 @@ typedef struct { const void *data; size_t size; + size_t offset; + size_t visible; char *name; } mem_istream_t; -static int mem_in_precache(istream_t *strm) +static int mem_get_buffered_data(istream_t *strm, const sqfs_u8 **out, + size_t *size, size_t want) { mem_istream_t *mem = (mem_istream_t *)strm; - size_t diff; + size_t have = mem->size - mem->offset; - assert(strm->buffer >= mem->buffer); - assert(strm->buffer <= (mem->buffer + mem->bufsz)); - assert(strm->buffer_used <= mem->bufsz); - assert((size_t)(strm->buffer - mem->buffer) <= - (mem->bufsz - strm->buffer_used)); + if (have > mem->bufsz) + have = mem->bufsz; - if (strm->buffer_used > 0) - memmove(mem->buffer, strm->buffer, strm->buffer_used); + if (want > have) + want = have; - strm->buffer = mem->buffer; + if (mem->visible == 0 || mem->visible < want) { + memcpy(mem->buffer + mem->visible, + (const char *)mem->data + mem->offset + mem->visible, + have - mem->visible); + mem->visible = have; + } - diff = mem->bufsz - strm->buffer_used; - if (diff > mem->size) - diff = mem->size; + *out = mem->buffer; + *size = mem->visible; + return (mem->visible == 0) ? 1 : 0; +} - if (diff > 0) { - memcpy(mem->buffer + strm->buffer_used, mem->data, diff); - strm->buffer_used += diff; - mem->data = (const char *)mem->data + diff; - mem->size -= diff; - } +static void mem_advance_buffer(istream_t *strm, size_t count) +{ + mem_istream_t *mem = (mem_istream_t *)strm; - return 0; + assert(count <= mem->visible); + + if (count > 0 && count < mem->visible) + memmove(mem->buffer, mem->buffer + count, mem->visible - count); + + mem->offset += count; + mem->visible -= count; + + if (mem->visible < mem->bufsz) { + memset(mem->buffer + mem->visible, 0, + mem->bufsz - mem->visible); + } } static const char *mem_in_get_filename(istream_t *strm) @@ -88,8 +102,8 @@ istream_t *istream_memory_create(const char *name, size_t bufsz, mem->data = data; mem->size = size; mem->bufsz = bufsz; - strm->buffer = mem->buffer; - strm->precache = mem_in_precache; + strm->get_buffered_data = mem_get_buffered_data; + strm->advance_buffer = mem_advance_buffer; strm->get_filename = mem_in_get_filename; return strm; } diff --git a/lib/io/src/unix/istream.c b/lib/io/src/unix/istream.c index 8a2dc26..60abff1 100644 --- a/lib/io/src/unix/istream.c +++ b/lib/io/src/unix/istream.c @@ -12,30 +12,35 @@ typedef struct { int fd; bool eof; + size_t buffer_offset; + size_t buffer_used; sqfs_u8 buffer[BUFSZ]; } file_istream_t; -static int file_precache(istream_t *strm) +static int precache(istream_t *strm) { file_istream_t *file = (file_istream_t *)strm; - assert(strm->buffer >= file->buffer); - assert(strm->buffer <= (file->buffer + BUFSZ)); - assert(strm->buffer_used <= BUFSZ); - assert((size_t)(strm->buffer - file->buffer) <= - (BUFSZ - strm->buffer_used)); + if (file->eof) + return 0; - if (strm->buffer_used > 0) - memmove(file->buffer, strm->buffer, strm->buffer_used); + if (file->buffer_offset > 0 && + file->buffer_offset < file->buffer_used) { + memmove(file->buffer, file->buffer + file->buffer_offset, + file->buffer_used - file->buffer_offset); + } - strm->buffer = file->buffer; + file->buffer_used -= file->buffer_offset; + file->buffer_offset = 0; - while (!file->eof && strm->buffer_used < BUFSZ) { - ssize_t ret = read(file->fd, file->buffer + strm->buffer_used, - BUFSZ - strm->buffer_used); + while (file->buffer_used < BUFSZ) { + ssize_t ret = read(file->fd, file->buffer + file->buffer_used, + BUFSZ - file->buffer_used); - if (ret == 0) + if (ret == 0) { file->eof = true; + break; + } if (ret < 0) { if (errno == EINTR) @@ -45,12 +50,43 @@ static int file_precache(istream_t *strm) return -1; } - strm->buffer_used += ret; + file->buffer_used += ret; } return 0; } +static int file_get_buffered_data(istream_t *strm, const sqfs_u8 **out, + size_t *size, size_t want) +{ + file_istream_t *file = (file_istream_t *)strm; + + if (want > BUFSZ) + want = BUFSZ; + + if (file->buffer_used == 0 || + (file->buffer_used - file->buffer_offset) < want) { + int ret = precache(strm); + if (ret) + return ret; + } + + *out = file->buffer + file->buffer_offset; + *size = file->buffer_used - file->buffer_offset; + return (file->eof && *size == 0) ? 1 : 0; +} + +static void file_advance_buffer(istream_t *strm, size_t count) +{ + file_istream_t *file = (file_istream_t *)strm; + + assert(count <= file->buffer_used); + + file->buffer_offset += count; + + assert(file->buffer_offset <= file->buffer_used); +} + static const char *file_get_filename(istream_t *strm) { file_istream_t *file = (file_istream_t *)strm; @@ -93,8 +129,8 @@ istream_t *istream_open_file(const char *path) goto fail_path; } - strm->buffer = file->buffer; - strm->precache = file_precache; + strm->get_buffered_data = file_get_buffered_data; + strm->advance_buffer = file_advance_buffer; strm->get_filename = file_get_filename; return strm; fail_path: @@ -119,8 +155,8 @@ istream_t *istream_open_stdin(void) goto fail; file->fd = STDIN_FILENO; - strm->buffer = file->buffer; - strm->precache = file_precache; + strm->get_buffered_data = file_get_buffered_data; + strm->advance_buffer = file_advance_buffer; strm->get_filename = file_get_filename; return strm; fail: diff --git a/lib/io/src/win32/istream.c b/lib/io/src/win32/istream.c index c4d7458..9bc32a0 100644 --- a/lib/io/src/win32/istream.c +++ b/lib/io/src/win32/istream.c @@ -15,10 +15,12 @@ typedef struct { HANDLE hnd; bool eof; + size_t buffer_offset; + size_t buffer_used; sqfs_u8 buffer[BUFSZ]; } file_istream_t; -static int file_precache(istream_t *strm) +static int precache(istream_t *strm) { file_istream_t *file = (file_istream_t *)strm; DWORD diff, actual; @@ -27,22 +29,21 @@ static int file_precache(istream_t *strm) if (file->eof) return 0; - assert(strm->buffer >= file->buffer); - assert(strm->buffer <= (file->buffer + BUFSZ)); - assert(strm->buffer_used <= BUFSZ); - assert((size_t)(strm->buffer - file->buffer) <= - (BUFSZ - strm->buffer_used)); + if (file->buffer_offset > 0 && + file->buffer_offset < file->buffer_used) { + memmove(file->buffer, file->buffer + file->buffer_offset, + file->buffer_used - file->buffer_offset); + } - if (strm->buffer_used > 0) - memmove(file->buffer, strm->buffer, strm->buffer_used); + file->buffer_used -= file->buffer_offset; + file->buffer_offset = 0; - strm->buffer = file->buffer; hnd = file->path == NULL ? GetStdHandle(STD_INPUT_HANDLE) : file->hnd; - while (strm->buffer_used < sizeof(file->buffer)) { - diff = sizeof(file->buffer) - strm->buffer_used; + while (file->buffer_used < sizeof(file->buffer)) { + diff = sizeof(file->buffer) - file->buffer_used; - if (!ReadFile(hnd, file->buffer + strm->buffer_used, + if (!ReadFile(hnd, file->buffer + file->buffer_used, diff, &actual, NULL)) { DWORD error = GetLastError(); @@ -63,12 +64,43 @@ static int file_precache(istream_t *strm) break; } - strm->buffer_used += actual; + file->buffer_used += actual; } return 0; } +static int file_get_buffered_data(istream_t *strm, const sqfs_u8 **out, + size_t *size, size_t want) +{ + file_istream_t *file = (file_istream_t *)strm; + + if (want > BUFSZ) + want = BUFSZ; + + if (file->buffer_used == 0 || + (file->buffer_used - file->buffer_offset) < want) { + int ret = precache(strm); + if (ret) + return ret; + } + + *out = file->buffer + file->buffer_offset; + *size = file->buffer_used - file->buffer_offset; + return (file->eof && *size == 0) ? 1 : 0; +} + +static void file_advance_buffer(istream_t *strm, size_t count) +{ + file_istream_t *file = (file_istream_t *)strm; + + assert(count <= file->buffer_used); + + file->buffer_offset += count; + + assert(file->buffer_offset <= file->buffer_used); +} + static const char *file_get_filename(istream_t *strm) { file_istream_t *file = (file_istream_t *)strm; @@ -122,7 +154,8 @@ istream_t *istream_open_file(const char *path) free(wpath); strm->buffer = file->buffer; - strm->precache = file_precache; + strm->get_buffered_data = file_get_buffered_data; + strm->advance_buffer = file_advance_buffer; strm->get_filename = file_get_filename; return strm; fail_path: @@ -146,7 +179,8 @@ istream_t *istream_open_stdin(void) sqfs_object_init(file, file_destroy, NULL); strm->buffer = file->buffer; - strm->precache = file_precache; + strm->get_buffered_data = file_get_buffered_data; + strm->advance_buffer = file_advance_buffer; strm->get_filename = file_get_filename; return strm; } diff --git a/lib/io/src/xfrm/istream.c b/lib/io/src/xfrm/istream.c index bad3e22..c9bc41a 100644 --- a/lib/io/src/xfrm/istream.c +++ b/lib/io/src/xfrm/istream.c @@ -12,27 +12,28 @@ typedef struct istream_xfrm_t { istream_t *wrapped; xfrm_stream_t *xfrm; + size_t buffer_offset; + size_t buffer_used; sqfs_u8 uncompressed[BUFSZ]; } istream_xfrm_t; -static int xfrm_precache(istream_t *base) +static int precache(istream_t *base) { istream_xfrm_t *xfrm = (istream_xfrm_t *)base; int ret; - assert(base->buffer >= xfrm->uncompressed); - assert(base->buffer <= (xfrm->uncompressed + BUFSZ)); - assert(base->buffer_used <= BUFSZ); - assert((size_t)(base->buffer - xfrm->uncompressed) <= - (BUFSZ - base->buffer_used)); - - if (base->buffer_used > 0) - memmove(xfrm->uncompressed, base->buffer, base->buffer_used); + if (xfrm->buffer_offset > 0 && + xfrm->buffer_offset < xfrm->buffer_used) { + memmove(xfrm->uncompressed, + xfrm->uncompressed + xfrm->buffer_offset, + xfrm->buffer_used - xfrm->buffer_offset); + } - base->buffer = xfrm->uncompressed; + xfrm->buffer_used -= xfrm->buffer_offset; + xfrm->buffer_offset = 0; for (;;) { - sqfs_u32 in_off = 0, out_off = base->buffer_used; + sqfs_u32 in_off = 0, out_off = xfrm->buffer_used; int mode = XFRM_STREAM_FLUSH_NONE; const sqfs_u8 *ptr; size_t avail; @@ -41,10 +42,8 @@ static int xfrm_precache(istream_t *base) sizeof(xfrm->uncompressed)); if (ret < 0) return ret; - if (ret > 0) { + if (ret > 0) mode = XFRM_STREAM_FLUSH_FULL; - avail = 0; - } ret = xfrm->xfrm->process_data(xfrm->xfrm, ptr, avail, @@ -58,7 +57,7 @@ static int xfrm_precache(istream_t *base) return -1; } - base->buffer_used = out_off; + xfrm->buffer_used = out_off; istream_advance_buffer(xfrm->wrapped, in_off); if (ret == XFRM_STREAM_BUFFER_FULL || out_off >= BUFSZ) @@ -71,6 +70,37 @@ static int xfrm_precache(istream_t *base) return 0; } +static int xfrm_get_buffered_data(istream_t *strm, const sqfs_u8 **out, + size_t *size, size_t want) +{ + istream_xfrm_t *xfrm = (istream_xfrm_t *)strm; + + if (want > BUFSZ) + want = BUFSZ; + + if (xfrm->buffer_used == 0 || + (xfrm->buffer_used - xfrm->buffer_offset) < want) { + int ret = precache(strm); + if (ret) + return ret; + } + + *out = xfrm->uncompressed + xfrm->buffer_offset; + *size = xfrm->buffer_used - xfrm->buffer_offset; + return (*size == 0) ? 1 : 0; +} + +static void xfrm_advance_buffer(istream_t *strm, size_t count) +{ + istream_xfrm_t *xfrm = (istream_xfrm_t *)strm; + + assert(count <= xfrm->buffer_used); + + xfrm->buffer_offset += count; + + assert(xfrm->buffer_offset <= xfrm->buffer_used); +} + static const char *xfrm_get_filename(istream_t *strm) { istream_xfrm_t *xfrm = (istream_xfrm_t *)strm; @@ -100,9 +130,9 @@ istream_t *istream_xfrm_create(istream_t *strm, xfrm_stream_t *xfrm) stream->wrapped = sqfs_grab(strm); stream->xfrm = sqfs_grab(xfrm); - base->precache = xfrm_precache; + base->get_buffered_data = xfrm_get_buffered_data; + base->advance_buffer = xfrm_advance_buffer; base->get_filename = xfrm_get_filename; - base->buffer = stream->uncompressed; return base; fail: fprintf(stderr, "%s: error initializing decompressor stream.\n", diff --git a/lib/io/test/istream_mem.c b/lib/io/test/istream_mem.c index 84da9a9..a486899 100644 --- a/lib/io/test/istream_mem.c +++ b/lib/io/test/istream_mem.c @@ -63,8 +63,9 @@ int main(int argc, char **argv) TEST_EQUAL_UI(ptr[j], byte_at_offset(i + j)); } - istream_advance_buffer(in, eat_all ? size : (size / 2)); + diff = eat_all ? size : (size / 2); eat_all = !eat_all; + istream_advance_buffer(in, diff); } sqfs_drop(in); diff --git a/lib/io/test/istream_read.c b/lib/io/test/istream_read.c index 176832b..8ba4540 100644 --- a/lib/io/test/istream_read.c +++ b/lib/io/test/istream_read.c @@ -100,7 +100,6 @@ int main(int argc, char **argv) TEST_ASSERT(read_off <= end2); } - TEST_ASSERT(dummy->buffer_used == 0); sqfs_drop(dummy); return EXIT_SUCCESS; } diff --git a/lib/io/test/istream_skip.c b/lib/io/test/istream_skip.c index 15d7f5d..3a02c8c 100644 --- a/lib/io/test/istream_skip.c +++ b/lib/io/test/istream_skip.c @@ -89,7 +89,6 @@ int main(int argc, char **argv) TEST_ASSERT(read_off <= end2); } - TEST_ASSERT(dummy->buffer_used == 0); sqfs_drop(dummy); return EXIT_SUCCESS; } diff --git a/lib/io/test/stream_splice.c b/lib/io/test/stream_splice.c index 223c39d..754f9cb 100644 --- a/lib/io/test/stream_splice.c +++ b/lib/io/test/stream_splice.c @@ -91,7 +91,6 @@ int main(int argc, char **argv) TEST_EQUAL_UI(total, out_offset); } - TEST_ASSERT(in->buffer_used == 0); TEST_EQUAL_UI(total, end2); TEST_EQUAL_UI(out_offset, end2); sqfs_drop(in); diff --git a/lib/io/test/xfrm.c b/lib/io/test/xfrm.c index 95b1f72..323987c 100644 --- a/lib/io/test/xfrm.c +++ b/lib/io/test/xfrm.c @@ -439,8 +439,6 @@ static void run_unpack_test(const void *blob, size_t size) ret = istream_read(istream, &c, 1); TEST_EQUAL_I(ret, 0); - TEST_EQUAL_UI(mem_istream->buffer_used, 0); - sqfs_drop(istream); TEST_EQUAL_UI(((sqfs_object_t *)mem_istream)->refcount, 1); TEST_EQUAL_UI(((sqfs_object_t *)xfrm)->refcount, 1); diff --git a/lib/tar/src/iterator.c b/lib/tar/src/iterator.c index df5c446..5d44c28 100644 --- a/lib/tar/src/iterator.c +++ b/lib/tar/src/iterator.c @@ -14,6 +14,7 @@ #include #include +#include #include typedef struct { @@ -30,7 +31,6 @@ typedef struct { sqfs_u64 offset; size_t padding; - size_t last_chunk; bool last_sparse; } tar_iterator_t; @@ -86,8 +86,6 @@ static void drop_parent(tar_istream_t *tar, int state) tar->parent->state = state; } - ((istream_t *)tar)->buffer_used = 0; - ((istream_t *)tar)->buffer = tar->buffer; tar->state = state; } @@ -96,60 +94,63 @@ static const char *strm_get_filename(istream_t *strm) return ((tar_istream_t *)strm)->parent->current.name; } -static int strm_precache(istream_t *strm) +static int strm_get_buffered_data(istream_t *strm, const sqfs_u8 **out, + size_t *size, size_t want) { tar_istream_t *tar = (tar_istream_t *)strm; sqfs_u64 diff; + int ret; if (tar->parent == NULL) return tar->state; - diff = (tar->parent->last_chunk - strm->buffer_used); - if (diff == 0 && strm->buffer_used > 0) - return tar->state; - - if (!tar->parent->last_sparse) { - istream_advance_buffer(tar->parent->stream, diff); - tar->parent->record_size -= diff; - } - - tar->parent->offset += diff; if (tar->parent->offset >= tar->parent->file_size) goto out_eof; tar->parent->last_sparse = is_sparse_region(tar->parent, &diff); if (diff == 0) goto out_eof; + if (diff > want) + diff = want; if (tar->parent->last_sparse) { - strm->buffer = tar->buffer; - strm->buffer_used = (diff <= sizeof(tar->buffer)) ? + *out = tar->buffer; + *size = (diff <= sizeof(tar->buffer)) ? diff : sizeof(tar->buffer); } else { - size_t avail; - int ret; - ret = istream_get_buffered_data(tar->parent->stream, - &strm->buffer, &avail, diff); + out, size, diff); if (ret > 0) goto fail_borked; if (ret < 0) goto fail_io; - - strm->buffer_used = (diff <= avail) ? diff : avail; + if (*size > diff) + *size = diff; } - tar->parent->last_chunk = strm->buffer_used; - return tar->state; + return 0; fail_io: - drop_parent(tar, SQFS_ERROR_IO); + drop_parent(tar, ret); return tar->state; fail_borked: drop_parent(tar, SQFS_ERROR_CORRUPTED); return tar->state; out_eof: drop_parent(tar, 0); - return tar->state; + tar->state = 1; + return 1; +} + +static void strm_advance_buffer(istream_t *strm, size_t count) +{ + tar_istream_t *tar = (tar_istream_t *)strm; + + if (!tar->parent->last_sparse) { + istream_advance_buffer(tar->parent->stream, count); + tar->parent->record_size -= count; + } + + tar->parent->offset += count; } static void strm_destroy(sqfs_object_t *obj) @@ -192,7 +193,6 @@ retry: goto fail; tar->offset = 0; - tar->last_chunk = 0; tar->last_sparse = false; tar->record_size = tar->current.record_size; tar->file_size = tar->current.actual_size; @@ -284,9 +284,9 @@ static int it_open_file_ro(dir_iterator_t *it, istream_t **out) sqfs_object_init(strm, strm_destroy, NULL); strm->parent = sqfs_grab(tar); - ((istream_t *)strm)->precache = strm_precache; + ((istream_t *)strm)->get_buffered_data = strm_get_buffered_data; + ((istream_t *)strm)->advance_buffer = strm_advance_buffer; ((istream_t *)strm)->get_filename = strm_get_filename; - ((istream_t *)strm)->buffer = strm->buffer; tar->locked = true; *out = (istream_t *)strm; -- cgit v1.2.3