From 061fbc12fe49ff49088a644def3227d3800cd8a7 Mon Sep 17 00:00:00 2001 From: David Oberhollenzer Date: Fri, 2 Jun 2023 17:42:30 +0200 Subject: libio: consolidate add-hoc memory istream_t implementations In several places, there are ad-hoc istream_t implementations that read from a memory buffer to test something else stacked on top. This commit consolidates those ad-hoc implmentations into a proper one in libio, and uses the chance to remove external files for some older tests that rely on file I/O instead. Signed-off-by: David Oberhollenzer --- lib/io/Makemodule.am | 6 +-- lib/io/src/mem.c | 85 +++++++++++++++++++++++++++++++++++++ lib/io/test/get_line.c | 37 ++++++++++------ lib/io/test/get_line.txt | 11 ----- lib/io/test/istream_read.c | 75 ++++++++++---------------------- lib/io/test/istream_skip.c | 73 ++++++++++---------------------- lib/io/test/stream_splice.c | 56 +++++++++--------------- lib/io/test/xfrm.c | 101 ++++++++------------------------------------ 8 files changed, 192 insertions(+), 252 deletions(-) create mode 100644 lib/io/src/mem.c delete mode 100644 lib/io/test/get_line.txt (limited to 'lib/io') diff --git a/lib/io/Makemodule.am b/lib/io/Makemodule.am index 5b5fc7f..7427b42 100644 --- a/lib/io/Makemodule.am +++ b/lib/io/Makemodule.am @@ -1,10 +1,10 @@ libio_a_SOURCES = include/io/istream.h include/io/ostream.h include/io/xfrm.h \ include/io/file.h include/io/std.h include/io/dir_entry.h \ - include/io/dir_iterator.h \ + include/io/dir_iterator.h include/io/mem.h \ lib/io/src/internal.h lib/io/src/ostream.c \ lib/io/src/istream.c lib/io/src/get_line.c lib/io/src/xfrm/ostream.c \ lib/io/src/xfrm/istream.c lib/io/src/dir_tree_iterator.c \ - lib/io/src/dir_entry.c + lib/io/src/dir_entry.c lib/io/src/mem.c libio_a_CFLAGS = $(AM_CFLAGS) $(ZLIB_CFLAGS) $(XZ_CFLAGS) libio_a_CFLAGS += $(ZSTD_CFLAGS) $(BZIP2_CFLAGS) @@ -26,7 +26,6 @@ LIBIO_TESTS = test_get_line test_sparse_fb test_istream_read \ test_get_line_SOURCES = lib/io/test/get_line.c test_get_line_LDADD = libio.a libcompat.a test_get_line_CPPFLAGS = $(AM_CPPFLAGS) -test_get_line_CPPFLAGS += -DTESTFILE=$(top_srcdir)/lib/io/test/get_line.txt test_sparse_fb_SOURCES = lib/io/test/sparse_fb.c test_sparse_fb_LDADD = libio.a libutil.a libcompat.a @@ -97,5 +96,4 @@ endif check_PROGRAMS += $(LIBIO_TESTS) TESTS += $(LIBIO_TESTS) -EXTRA_DIST += $(top_srcdir)/lib/io/test/get_line.txt EXTRA_DIST += $(top_srcdir)/lib/io/test/testdir diff --git a/lib/io/src/mem.c b/lib/io/src/mem.c new file mode 100644 index 0000000..4e9f9af --- /dev/null +++ b/lib/io/src/mem.c @@ -0,0 +1,85 @@ +/* SPDX-License-Identifier: GPL-3.0-or-later */ +/* + * mem.c + * + * Copyright (C) 2023 David Oberhollenzer + */ +#include "config.h" +#include "io/mem.h" +#include "compat.h" + +#include +#include + +typedef struct { + istream_t base; + + sqfs_u8 *buffer; + size_t bufsz; + + const void *data; + size_t size; + + char *name; +} mem_istream_t; + +static int mem_in_precache(istream_t *strm) +{ + mem_istream_t *mem = (mem_istream_t *)strm; + size_t diff = mem->bufsz - strm->buffer_used; + + if (diff > mem->size) + diff = mem->size; + + if (diff > 0) { + memcpy(mem->buffer + strm->buffer_used, mem->data, diff); + strm->buffer_used += diff; + mem->data = (const char *)mem->data + diff; + mem->size -= diff; + } + + if (mem->size == 0) + strm->eof = true; + + return 0; +} + +static const char *mem_in_get_filename(istream_t *strm) +{ + return ((mem_istream_t *)strm)->name; +} + +static void mem_in_destroy(sqfs_object_t *obj) +{ + free(((mem_istream_t *)obj)->buffer); + free(((mem_istream_t *)obj)->name); + free(obj); +} + +istream_t *istream_memory_create(const char *name, size_t bufsz, + const void *data, size_t size) +{ + mem_istream_t *mem = calloc(1, sizeof(*mem)); + istream_t *strm = (istream_t *)mem; + + if (mem == NULL) + return NULL; + + sqfs_object_init(mem, mem_in_destroy, NULL); + + mem->name = strdup(name); + if (mem->name == NULL) + return sqfs_drop(mem); + + mem->buffer = malloc(bufsz); + if (mem->buffer == NULL) + return sqfs_drop(mem); + + mem->data = data; + mem->size = size; + mem->bufsz = bufsz; + strm->buffer = mem->buffer; + strm->precache = mem_in_precache; + strm->get_filename = mem_in_get_filename; + return strm; +} diff --git a/lib/io/test/get_line.c b/lib/io/test/get_line.c index 2d0f9b7..bb06b3e 100644 --- a/lib/io/test/get_line.c +++ b/lib/io/test/get_line.c @@ -4,9 +4,7 @@ * * Copyright (C) 2019 David Oberhollenzer */ -#include "config.h" - -#include "io/file.h" +#include "io/mem.h" #include "util/test.h" typedef struct { @@ -14,7 +12,7 @@ typedef struct { const char *str; } line_t; -static void run_test_case(const line_t *lines, size_t count, +static void run_test_case(const char *raw, const line_t *lines, size_t count, int flags) { size_t i, line_num, old_line_num; @@ -22,7 +20,7 @@ static void run_test_case(const line_t *lines, size_t count, char *line; int ret; - fp = istream_open_file(STRVALUE(TESTFILE)); + fp = istream_memory_create("lines.txt", 512, raw, strlen(raw)); TEST_NOT_NULL(fp); line_num = 1; @@ -50,6 +48,19 @@ static void run_test_case(const line_t *lines, size_t count, sqfs_drop(fp); } +static const char *file = +"\r\n" +"The quick\r\n" +" \r\n" +" brown fox \r\n" +"\r\n" +"jumps over\r\n" +"the\r\n" +"lazy\r\n" +"\r\n" +"dog\r\n" +"\r\n"; + static const line_t lines_raw[] = { { 1, "" }, { 2, "The quick" }, @@ -147,18 +158,18 @@ int main(int argc, char **argv) { (void)argc; (void)argv; - run_test_case(lines_raw, 11, 0); - run_test_case(lines_ltrim, 11, ISTREAM_LINE_LTRIM); - run_test_case(lines_rtrim, 11, ISTREAM_LINE_RTRIM); - run_test_case(lines_trim, 11, + run_test_case(file, lines_raw, 11, 0); + run_test_case(file, lines_ltrim, 11, ISTREAM_LINE_LTRIM); + run_test_case(file, lines_rtrim, 11, ISTREAM_LINE_RTRIM); + run_test_case(file, lines_trim, 11, ISTREAM_LINE_LTRIM | ISTREAM_LINE_RTRIM); - run_test_case(lines_no_empty, 7, ISTREAM_LINE_SKIP_EMPTY); - run_test_case(lines_no_empty_ltrim, 6, + run_test_case(file, lines_no_empty, 7, ISTREAM_LINE_SKIP_EMPTY); + run_test_case(file, lines_no_empty_ltrim, 6, ISTREAM_LINE_SKIP_EMPTY | ISTREAM_LINE_LTRIM); - run_test_case(lines_no_empty_rtrim, 6, + run_test_case(file, lines_no_empty_rtrim, 6, ISTREAM_LINE_SKIP_EMPTY | ISTREAM_LINE_RTRIM); - run_test_case(lines_no_empty_trim, 6, + run_test_case(file, lines_no_empty_trim, 6, ISTREAM_LINE_SKIP_EMPTY | ISTREAM_LINE_LTRIM | ISTREAM_LINE_RTRIM); diff --git a/lib/io/test/get_line.txt b/lib/io/test/get_line.txt deleted file mode 100644 index a1994f0..0000000 --- a/lib/io/test/get_line.txt +++ /dev/null @@ -1,11 +0,0 @@ - -The quick - - brown fox - -jumps over -the -lazy - -dog - diff --git a/lib/io/test/istream_read.c b/lib/io/test/istream_read.c index 3e4c22f..87e52c5 100644 --- a/lib/io/test/istream_read.c +++ b/lib/io/test/istream_read.c @@ -8,72 +8,40 @@ #include "io/istream.h" #include "util/test.h" +#include "io/mem.h" static const sqfs_u64 end0 = 449; /* region 1: filled with 'A' */ static const sqfs_u64 end1 = 521; /* region 2: filled with 'B' */ static const sqfs_u64 end2 = 941; /* region 3: filled with 'C' */ -static sqfs_u8 buffer[103]; /* sliding window into the file */ -static sqfs_u64 offset = 0; /* byte offset into the "file" */ - -static int dummy_precache(istream_t *strm); -static const char *dummy_get_filename(istream_t *strm); - -static istream_t dummy = { - { - 1, - NULL, - NULL, - }, - 0, - 0, - false, - buffer, - dummy_precache, - dummy_get_filename, -}; - -static int dummy_precache(istream_t *strm) -{ - sqfs_u8 x; - - TEST_ASSERT(strm == &dummy); - - while (strm->buffer_used < sizeof(buffer)) { - if (offset < end0) { - x = 'A'; - } else if (offset < end1) { - x = 'B'; - } else if (offset < end2) { - x = 'C'; - } else { - strm->eof = true; - break; - } +static sqfs_u8 rd_buffer[941]; - strm->buffer[strm->buffer_used++] = x; - ++offset; - } - - return 0; +static sqfs_u8 byte_at_offset(sqfs_u64 off) +{ + if (off < end0) + return 'A'; + if (off < end1) + return 'B'; + return 'C'; } -static const char *dummy_get_filename(istream_t *strm) +static void init_rd_buffer(void) { - TEST_ASSERT(strm == &dummy); - return "dummy file"; + for (size_t i = 0; i < end2; ++i) + rd_buffer[i] = byte_at_offset(i); } int main(int argc, char **argv) { sqfs_u8 read_buffer[61]; sqfs_u64 read_off = 0; - const char *name; + istream_t *dummy; (void)argc; (void)argv; - name = istream_get_filename(&dummy); - TEST_NOT_NULL(name); - TEST_STR_EQUAL(name, "dummy file"); + init_rd_buffer(); + dummy = istream_memory_create("dummy file", 103, + rd_buffer, sizeof(rd_buffer)); + TEST_NOT_NULL(dummy); /* region 1 */ while (read_off < end0) { @@ -82,7 +50,7 @@ int main(int argc, char **argv) if (read_diff > sizeof(read_buffer)) read_diff = sizeof(read_buffer); - int ret = istream_read(&dummy, read_buffer, read_diff); + int ret = istream_read(dummy, read_buffer, read_diff); TEST_ASSERT(ret > 0); TEST_ASSERT((size_t)ret <= read_diff); @@ -100,7 +68,7 @@ int main(int argc, char **argv) if (read_diff > sizeof(read_buffer)) read_diff = sizeof(read_buffer); - int ret = istream_read(&dummy, read_buffer, read_diff); + int ret = istream_read(dummy, read_buffer, read_diff); TEST_ASSERT(ret > 0); TEST_ASSERT((size_t)ret <= read_diff); @@ -115,7 +83,7 @@ int main(int argc, char **argv) for (;;) { size_t read_diff = sizeof(read_buffer); - int ret = istream_read(&dummy, read_buffer, read_diff); + int ret = istream_read(dummy, read_buffer, read_diff); TEST_ASSERT(ret >= 0); TEST_ASSERT((size_t)ret <= read_diff); @@ -132,5 +100,8 @@ int main(int argc, char **argv) TEST_ASSERT(read_off <= end2); } + TEST_ASSERT(dummy->eof); + TEST_ASSERT(dummy->buffer_offset >= dummy->buffer_used); + sqfs_drop(dummy); return EXIT_SUCCESS; } diff --git a/lib/io/test/istream_skip.c b/lib/io/test/istream_skip.c index 3197b8b..40d4ea4 100644 --- a/lib/io/test/istream_skip.c +++ b/lib/io/test/istream_skip.c @@ -8,72 +8,40 @@ #include "io/istream.h" #include "util/test.h" +#include "io/mem.h" static const sqfs_u64 end0 = 449; /* region 1: filled with 'A' */ static const sqfs_u64 end1 = 521; /* region 2: filled with 'B' */ static const sqfs_u64 end2 = 941; /* region 3: filled with 'C' */ -static sqfs_u8 buffer[103]; /* sliding window into the file */ -static sqfs_u64 offset = 0; /* byte offset into the "file" */ +static sqfs_u8 rd_buffer[941]; -static int dummy_precache(istream_t *strm); -static const char *dummy_get_filename(istream_t *strm); - -static istream_t dummy = { - { - 1, - NULL, - NULL, - }, - 0, - 0, - false, - buffer, - dummy_precache, - dummy_get_filename, -}; - -static int dummy_precache(istream_t *strm) +static sqfs_u8 byte_at_offset(sqfs_u64 off) { - sqfs_u8 x; - - TEST_ASSERT(strm == &dummy); - - while (strm->buffer_used < sizeof(buffer)) { - if (offset < end0) { - x = 'A'; - } else if (offset < end1) { - x = 'B'; - } else if (offset < end2) { - x = 'C'; - } else { - strm->eof = true; - break; - } - - strm->buffer[strm->buffer_used++] = x; - ++offset; - } - - return 0; + if (off < end0) + return 'A'; + if (off < end1) + return 'B'; + return 'C'; } -static const char *dummy_get_filename(istream_t *strm) +static void init_rd_buffer(void) { - TEST_ASSERT(strm == &dummy); - return "dummy file"; + for (size_t i = 0; i < end2; ++i) + rd_buffer[i] = byte_at_offset(i); } int main(int argc, char **argv) { sqfs_u8 read_buffer[61]; sqfs_u64 read_off = 0; - const char *name; + istream_t *dummy; (void)argc; (void)argv; - name = istream_get_filename(&dummy); - TEST_NOT_NULL(name); - TEST_STR_EQUAL(name, "dummy file"); + init_rd_buffer(); + dummy = istream_memory_create("dummy file", 103, + rd_buffer, sizeof(rd_buffer)); + TEST_NOT_NULL(dummy); /* region 1 */ while (read_off < end0) { @@ -82,7 +50,7 @@ int main(int argc, char **argv) if (read_diff > sizeof(read_buffer)) read_diff = sizeof(read_buffer); - int ret = istream_read(&dummy, read_buffer, read_diff); + int ret = istream_read(dummy, read_buffer, read_diff); TEST_ASSERT(ret > 0); TEST_ASSERT((size_t)ret <= read_diff); @@ -95,7 +63,7 @@ int main(int argc, char **argv) /* region 2 */ { - int ret = istream_skip(&dummy, end2 - end1); + int ret = istream_skip(dummy, end2 - end1); TEST_EQUAL_I(ret, 0); read_off += (end2 - end1); } @@ -104,7 +72,7 @@ int main(int argc, char **argv) for (;;) { size_t read_diff = sizeof(read_buffer); - int ret = istream_read(&dummy, read_buffer, read_diff); + int ret = istream_read(dummy, read_buffer, read_diff); TEST_ASSERT(ret >= 0); TEST_ASSERT((size_t)ret <= read_diff); @@ -121,5 +89,8 @@ int main(int argc, char **argv) TEST_ASSERT(read_off <= end2); } + TEST_ASSERT(dummy->eof); + TEST_ASSERT(dummy->buffer_offset >= dummy->buffer_used); + sqfs_drop(dummy); return EXIT_SUCCESS; } diff --git a/lib/io/test/stream_splice.c b/lib/io/test/stream_splice.c index c5ca38e..5379b39 100644 --- a/lib/io/test/stream_splice.c +++ b/lib/io/test/stream_splice.c @@ -8,12 +8,15 @@ #include "io/istream.h" #include "io/ostream.h" +#include "io/mem.h" #include "util/test.h" static const sqfs_u64 end0 = 449; /* region 1: filled with 'A' */ static const sqfs_u64 end1 = 521; /* region 2: filled with 'B' */ static const sqfs_u64 end2 = 941; /* region 3: filled with 'C' */ +static sqfs_u8 rd_buffer[941]; + static sqfs_u8 byte_at_offset(sqfs_u64 off) { if (off < end0) @@ -23,6 +26,12 @@ static sqfs_u8 byte_at_offset(sqfs_u64 off) return 'C'; } +static void init_rd_buffer(void) +{ + for (size_t i = 0; i < end2; ++i) + rd_buffer[i] = byte_at_offset(i); +} + /*****************************************************************************/ static int out_append(ostream_t *strm, const void *data, size_t size); @@ -57,47 +66,20 @@ static int out_append(ostream_t *strm, const void *data, size_t size) /*****************************************************************************/ -static int in_precache(istream_t *strm); - -static sqfs_u8 in_buffer[109]; -static sqfs_u64 in_offset = 0; - -static istream_t in = { - { 1, NULL, NULL }, - 0, - 0, - false, - in_buffer, - in_precache, - NULL, -}; - -static int in_precache(istream_t *strm) -{ - TEST_ASSERT(strm == &in); - - while (strm->buffer_used < sizeof(in_buffer)) { - if (in_offset == end2) { - strm->eof = true; - break; - } - - in_buffer[strm->buffer_used++] = byte_at_offset(in_offset++); - } - - return 0; -} - -/*****************************************************************************/ - int main(int argc, char **argv) { sqfs_u64 total = 0; + istream_t *in; sqfs_s32 ret; (void)argc; (void)argv; + init_rd_buffer(); + in = istream_memory_create("memory_in", 109, + rd_buffer, sizeof(rd_buffer)); + TEST_NOT_NULL(in); + for (;;) { - ret = ostream_append_from_istream(&out, &in, 211); + ret = ostream_append_from_istream(&out, in, 211); TEST_ASSERT(ret >= 0); if (ret == 0) @@ -106,13 +88,13 @@ int main(int argc, char **argv) total += ret; TEST_ASSERT(total <= end2); TEST_ASSERT(out_offset <= end2); - TEST_ASSERT(in_offset <= end2); - TEST_ASSERT(in_offset >= out_offset); TEST_EQUAL_UI(total, out_offset); } + TEST_ASSERT(in->eof); + TEST_ASSERT(in->buffer_offset >= in->buffer_used); TEST_EQUAL_UI(total, end2); TEST_EQUAL_UI(out_offset, end2); - TEST_EQUAL_UI(in_offset, end2); + sqfs_drop(in); return EXIT_SUCCESS; } diff --git a/lib/io/test/xfrm.c b/lib/io/test/xfrm.c index de5c496..d06c558 100644 --- a/lib/io/test/xfrm.c +++ b/lib/io/test/xfrm.c @@ -8,6 +8,7 @@ #include "xfrm/stream.h" #include "util/test.h" #include "io/xfrm.h" +#include "io/mem.h" static const sqfs_u8 blob_in[] = { #if defined(DO_XZ) @@ -363,57 +364,6 @@ static const char orig[] = /*****************************************************************************/ -static const sqfs_u8 *rd_buffer; -static size_t rd_offset; -static size_t rd_size; -static sqfs_u8 mi_buffer[7]; - -static int mem_precache(istream_t *strm); -static const char *mem_get_filename(istream_t *strm); - -static istream_t mem_istream = { - { 1, NULL, NULL, }, - 0, - 0, - false, - mi_buffer, - mem_precache, - mem_get_filename, -}; - -static int mem_precache(istream_t *strm) -{ - size_t spc_in, spc_out, diff; - - TEST_ASSERT(strm == &mem_istream); - TEST_ASSERT(!strm->eof); - TEST_ASSERT(rd_offset < rd_size); - - spc_in = rd_size - rd_offset; - spc_out = sizeof(mi_buffer) - strm->buffer_used; - diff = spc_in < spc_out ? spc_in : spc_out; - - memcpy(mi_buffer + strm->buffer_used, rd_buffer + rd_offset, diff); - strm->buffer_used += diff; - rd_offset += diff; - - TEST_ASSERT(strm->buffer_used <= sizeof(mi_buffer)); - TEST_ASSERT(rd_offset <= rd_size); - - if (rd_offset == rd_size) - strm->eof = true; - - return 0; -} - -static const char *mem_get_filename(istream_t *strm) -{ - TEST_ASSERT(strm == &mem_istream); - return "memory stream"; -} - -/*****************************************************************************/ - static size_t mo_written = 0; static sqfs_u8 mo_buffer[1024]; static bool mo_flushed = false; @@ -454,25 +404,28 @@ static int mem_flush(ostream_t *strm) /*****************************************************************************/ -static void run_unpack_test(void) +static void run_unpack_test(const void *blob, size_t size) { + istream_t *istream, *mem_istream; xfrm_stream_t *xfrm; - istream_t *istream; sqfs_s32 ret; size_t i; char c; + mem_istream = istream_memory_create("memstream", 7, blob, size); + TEST_NOT_NULL(mem_istream); + xfrm = mkdecompressor(); TEST_NOT_NULL(xfrm); TEST_EQUAL_UI(((sqfs_object_t *)xfrm)->refcount, 1); - TEST_EQUAL_UI(((sqfs_object_t *)&mem_istream)->refcount, 1); + TEST_EQUAL_UI(((sqfs_object_t *)mem_istream)->refcount, 1); - istream = istream_xfrm_create(&mem_istream, xfrm); + istream = istream_xfrm_create(mem_istream, xfrm); TEST_NOT_NULL(istream); TEST_EQUAL_UI(((sqfs_object_t *)istream)->refcount, 1); TEST_EQUAL_UI(((sqfs_object_t *)xfrm)->refcount, 2); - TEST_EQUAL_UI(((sqfs_object_t *)&mem_istream)->refcount, 2); + TEST_EQUAL_UI(((sqfs_object_t *)mem_istream)->refcount, 2); for (i = 0; i < (sizeof(orig) - 1); ++i) { ret = istream_read(istream, &c, 1); @@ -486,14 +439,15 @@ static void run_unpack_test(void) ret = istream_read(istream, &c, 1); TEST_EQUAL_I(ret, 0); - TEST_EQUAL_UI(mem_istream.buffer_used, 0); - TEST_EQUAL_UI(mem_istream.buffer_offset, 0); - TEST_EQUAL_UI(rd_offset, rd_size); + TEST_EQUAL_UI(mem_istream->buffer_used, 0); + TEST_EQUAL_UI(mem_istream->buffer_offset, 0); + TEST_ASSERT(mem_istream->eof); sqfs_drop(istream); - TEST_EQUAL_UI(((sqfs_object_t *)&mem_istream)->refcount, 1); + TEST_EQUAL_UI(((sqfs_object_t *)mem_istream)->refcount, 1); TEST_EQUAL_UI(((sqfs_object_t *)xfrm)->refcount, 1); sqfs_drop(xfrm); + sqfs_drop(mem_istream); } static void run_pack_test(void) @@ -542,25 +496,11 @@ int main(int argc, char **argv) (void)argc; (void)argv; /* normal stream */ - rd_buffer = blob_in; - rd_offset = 0; - rd_size = sizeof(blob_in); - mem_istream.buffer_offset = 0; - mem_istream.buffer_used = 0; - mem_istream.eof = false; - - run_unpack_test(); + run_unpack_test(blob_in, sizeof(blob_in)); /* concatenated streams */ #if !defined(DO_GZIP) - rd_buffer = blob_in_concat; - rd_offset = 0; - rd_size = sizeof(blob_in_concat); - mem_istream.buffer_offset = 0; - mem_istream.buffer_used = 0; - mem_istream.eof = false; - - run_unpack_test(); + run_unpack_test(blob_in_concat, sizeof(blob_in_concat)); #else (void)blob_in_concat; #endif @@ -568,13 +508,6 @@ int main(int argc, char **argv) run_pack_test(); /* restore from compressed */ - rd_buffer = mo_buffer; - rd_offset = 0; - rd_size = mo_written; - mem_istream.buffer_offset = 0; - mem_istream.buffer_used = 0; - mem_istream.eof = false; - - run_unpack_test(); + run_unpack_test(mo_buffer, mo_written); return EXIT_SUCCESS; } -- cgit v1.2.3