diff options
Diffstat (limited to 'lib')
-rw-r--r-- | lib/tar/Makemodule.am | 19 | ||||
-rw-r--r-- | lib/tar/src/istream.c | 222 | ||||
-rw-r--r-- | lib/tar/test/data/CREDITS | 3 | ||||
-rw-r--r-- | lib/tar/test/data/istream/sparse.tar | bin | 0 -> 33792 bytes | |||
-rw-r--r-- | lib/tar/test/tar_istream.c | 73 | ||||
-rw-r--r-- | lib/tar/test/tar_istream2.c | 155 | ||||
-rw-r--r-- | lib/tar/test/tar_istream3.c | 100 |
7 files changed, 570 insertions, 2 deletions
diff --git a/lib/tar/Makemodule.am b/lib/tar/Makemodule.am index 896ce0f..9faeecf 100644 --- a/lib/tar/Makemodule.am +++ b/lib/tar/Makemodule.am @@ -3,7 +3,7 @@ libtar_a_SOURCES = lib/tar/src/read_header.c lib/tar/src/write_header.c \ lib/tar/src/read_sparse_map_old.c lib/tar/src/internal.h \ lib/tar/src/padd_file.c lib/tar/src/record_to_memory.c \ lib/tar/src/pax_header.c lib/tar/src/read_sparse_map_new.c \ - include/tar/tar.h include/tar/format.h + lib/tar/src/istream.c include/tar/tar.h include/tar/format.h noinst_LIBRARIES += libtar.a @@ -162,6 +162,20 @@ test_tar_xattr_schily_bin_LDADD = libtar.a libio.a libutil.a libcompat.a test_tar_xattr_schily_bin_CPPFLAGS = $(AM_CPPFLAGS) -DTESTPATH=$(TARDATADIR) test_tar_xattr_schily_bin_CPPFLAGS += -DTESTFILE=xattr/xattr-schily-binary.tar +test_tar_istream_SOURCES = lib/tar/test/tar_istream.c +test_tar_istream_LDADD = libtar.a libio.a libutil.a libcompat.a +test_tar_istream_CPPFLAGS = $(AM_CPPFLAGS) -DTESTPATH=$(TARDATADIR) +test_tar_istream_CPPFLAGS += -DTESTFILE=format-acceptance/gnu.tar + +test_tar_istream2_SOURCES = lib/tar/test/tar_istream2.c +test_tar_istream2_LDADD = libtar.a libio.a libutil.a libcompat.a +test_tar_istream2_CPPFLAGS = $(AM_CPPFLAGS) -DTESTPATH=$(TARDATADIR) + +test_tar_istream3_SOURCES = lib/tar/test/tar_istream3.c +test_tar_istream3_LDADD = libtar.a libio.a libutil.a libcompat.a +test_tar_istream3_CPPFLAGS = $(AM_CPPFLAGS) -DTESTPATH=$(TARDATADIR) +test_tar_istream3_CPPFLAGS += -DTESTFILE=istream/sparse.tar + tar_fuzz_SOURCES = lib/tar/test/tar_fuzz.c tar_fuzz_LDADD = libtar.a libio.a libutil.a libcompat.a @@ -175,7 +189,8 @@ LIBTAR_TESTS = \ test_tar_sparse_gnu test_tar_sparse_gnu0 test_tar_sparse_gnu1 \ test_tar_sparse_gnu2 test_tar_sparse_gnu3 \ test_tar_xattr_bsd test_tar_xattr_schily test_tar_xattr_schily_bin \ - test_tar_target_filled + test_tar_target_filled \ + test_tar_istream test_tar_istream2 test_tar_istream3 check_PROGRAMS += $(LIBTAR_TESTS) TESTS += $(LIBTAR_TESTS) diff --git a/lib/tar/src/istream.c b/lib/tar/src/istream.c new file mode 100644 index 0000000..30b8a34 --- /dev/null +++ b/lib/tar/src/istream.c @@ -0,0 +1,222 @@ +/* SPDX-License-Identifier: GPL-3.0-or-later */ +/* + * istream.c + * + * Copyright (C) 2019 David Oberhollenzer <goliath@infraroot.at> + */ +#include "internal.h" + +#include <string.h> +#include <stdlib.h> + +typedef struct { + sqfs_u64 offset; + sqfs_u64 count; +} sparse_ent_t; + +typedef struct { + istream_t base; + + istream_t *parent; + + char *filename; + + sparse_ent_t *sparse; + size_t num_sparse; + + sqfs_u64 record_size; + sqfs_u64 file_size; + sqfs_u64 offset; + + size_t last_chunk; + bool last_sparse; + + sqfs_u8 buffer[4096]; +} tar_istream_t; + +static bool is_sparse_region(tar_istream_t *tar, sqfs_u64 *count) +{ + size_t i; + + *count = tar->file_size - tar->offset; + if (tar->num_sparse == 0) + return false; + + for (i = 0; i < tar->num_sparse; ++i) { + if (tar->offset >= tar->sparse[i].offset) { + sqfs_u64 diff = tar->offset - tar->sparse[i].offset; + + if (diff < tar->sparse[i].count) { + *count = tar->sparse[i].count - diff; + return false; + } + } + } + + for (i = 0; i < tar->num_sparse; ++i) { + if (tar->offset < tar->sparse[i].offset) { + sqfs_u64 diff = tar->sparse[i].offset - tar->offset; + + if (diff < *count) + *count = diff; + } + } + + return true; +} + +static int precache(istream_t *strm) +{ + tar_istream_t *tar = (tar_istream_t *)strm; + sqfs_u64 diff, avail; + + tar->offset += tar->last_chunk; + + if (!tar->last_sparse) { + tar->parent->buffer_offset += tar->last_chunk; + tar->record_size -= tar->last_chunk; + } + + if (tar->offset >= tar->file_size) { + strm->eof = true; + strm->buffer_used = 0; + strm->buffer = tar->buffer; + if (tar->record_size > 0) + goto fail_rec_sz; + return 0; + } + + if (is_sparse_region(tar, &diff)) { + if (diff > sizeof(tar->buffer)) + diff = sizeof(tar->buffer); + + strm->buffer = tar->buffer; + strm->buffer_used = diff; + tar->last_chunk = diff; + tar->last_sparse = true; + + memset(tar->buffer, 0, diff); + } else { + if (diff > tar->record_size) + goto fail_rec_sz; + + avail = tar->parent->buffer_used - tar->parent->buffer_offset; + + if ((diff > avail) && + ((tar->parent->buffer_offset > 0) || avail == 0)) { + if (istream_precache(tar->parent)) + return -1; + + if (tar->parent->buffer_used == 0 && tar->parent->eof) + goto fail_eof; + + avail = tar->parent->buffer_used; + } + + if (diff > avail) + diff = avail; + + strm->buffer = tar->parent->buffer + tar->parent->buffer_offset; + strm->buffer_used = diff; + tar->last_chunk = diff; + tar->last_sparse = false; + } + + return 0; +fail_rec_sz: + fprintf(stderr, + "%s: missmatch in tar record size vs file size for `%s`.\n", + istream_get_filename(tar->parent), istream_get_filename(strm)); + return -1; +fail_eof: + fprintf(stderr, "%s: unexpected end-of-file while reading `%s`\n", + istream_get_filename(tar->parent), istream_get_filename(strm)); + return -1; +} + +static const char *get_filename(istream_t *strm) +{ + return ((tar_istream_t *)strm)->filename; +} + +static void tar_istream_destroy(sqfs_object_t *obj) +{ + tar_istream_t *strm = (tar_istream_t *)obj; + + sqfs_drop(strm->parent); + free(strm->sparse); + free(strm->filename); + free(strm); +} + +istream_t *tar_record_istream_create(istream_t *parent, + const tar_header_decoded_t *hdr) +{ + tar_istream_t *strm; + sparse_map_t *it; + sqfs_u64 diff; + size_t idx; + + strm = calloc(1, sizeof(*strm)); + if (strm == NULL) + goto fail_oom; + + sqfs_object_init(strm, tar_istream_destroy, NULL); + + strm->filename = strdup(hdr->name); + if (strm->filename == NULL) + goto fail_oom; + + strm->num_sparse = 0; + for (it = hdr->sparse; it != NULL; it = it->next) + strm->num_sparse += 1; + + if (strm->num_sparse > 0) { + strm->sparse = alloc_array(sizeof(strm->sparse[0]), + strm->num_sparse); + if (strm->sparse == NULL) + goto fail_oom; + + idx = 0; + it = hdr->sparse; + while (it != NULL && idx < strm->num_sparse) { + strm->sparse[idx].offset = it->offset; + strm->sparse[idx].count = it->count; + ++idx; + it = it->next; + } + } + + for (idx = 1; idx < strm->num_sparse; ++idx) { + if (strm->sparse[idx].offset <= strm->sparse[idx - 1].offset) + goto fail_sparse; + + diff = strm->sparse[idx].offset - strm->sparse[idx - 1].offset; + + if (diff < strm->sparse[idx - 1].count) + goto fail_sparse; + } + + strm->record_size = hdr->record_size; + strm->file_size = hdr->actual_size; + strm->parent = sqfs_grab(parent); + + ((istream_t *)strm)->precache = precache; + ((istream_t *)strm)->get_filename = get_filename; + ((istream_t *)strm)->buffer = strm->buffer; + ((istream_t *)strm)->eof = false; + return (istream_t *)strm; +fail_sparse: + fprintf(stderr, "%s: sparse map is not ordered or overlapping!\n", + hdr->name); + goto fail; +fail_oom: + fputs("tar istream create: out-of-memory\n", stderr); + goto fail; +fail: + if (strm != NULL) { + free(strm->filename); + free(strm); + } + return NULL; +} diff --git a/lib/tar/test/data/CREDITS b/lib/tar/test/data/CREDITS index 7a2738f..dab8951 100644 --- a/lib/tar/test/data/CREDITS +++ b/lib/tar/test/data/CREDITS @@ -33,3 +33,6 @@ The following addtional files have been added: Contributed in GitHub issue #64. A tar ball that contains a hard link where the 100 byte target field is completely filled without containing a null-terminator. + - istream/sparse.tar + Derived from sparse/gnu.tar and contains some test data for testing the + tar istream implementation. diff --git a/lib/tar/test/data/istream/sparse.tar b/lib/tar/test/data/istream/sparse.tar Binary files differnew file mode 100644 index 0000000..7f4700d --- /dev/null +++ b/lib/tar/test/data/istream/sparse.tar diff --git a/lib/tar/test/tar_istream.c b/lib/tar/test/tar_istream.c new file mode 100644 index 0000000..cc31282 --- /dev/null +++ b/lib/tar/test/tar_istream.c @@ -0,0 +1,73 @@ +/* SPDX-License-Identifier: GPL-3.0-or-later */ +/* + * tar_istream.c + * + * Copyright (C) 2019 David Oberhollenzer <goliath@infraroot.at> + */ +#include "config.h" +#include "io/file.h" +#include "tar/tar.h" +#include "util/test.h" + +#ifndef TESTUID +#define TESTUID 1000 +#endif + +#ifndef TESTGID +#define TESTGID TESTUID +#endif + +#ifndef TESTFNAME +#define TESTFNAME input.txt +#endif + +#ifndef TESTTS +#define TESTTS 1542905892 +#endif + +static const char *fname = STRVALUE(TESTFNAME); + +int main(int argc, char **argv) +{ + tar_header_decoded_t hdr; + char buffer[100]; + sqfs_s64 ts; + istream_t *fp; + istream_t *ti; + sqfs_s32 ret; + (void)argc; (void)argv; + + fp = istream_open_file(STRVALUE(TESTPATH) "/" STRVALUE(TESTFILE)); + TEST_NOT_NULL(fp); + TEST_EQUAL_UI(((sqfs_object_t *)fp)->refcount, 1); + TEST_ASSERT(read_header(fp, &hdr) == 0); + TEST_EQUAL_UI(((sqfs_object_t *)fp)->refcount, 1); + TEST_EQUAL_UI(hdr.mode, S_IFREG | 0644); + TEST_EQUAL_UI(hdr.uid, TESTUID); + TEST_EQUAL_UI(hdr.gid, TESTGID); + TEST_EQUAL_UI(hdr.actual_size, 5); + + ts = TESTTS; + TEST_EQUAL_UI(hdr.mtime, ts); + TEST_STR_EQUAL(hdr.name, fname); + TEST_ASSERT(!hdr.unknown_record); + + ti = tar_record_istream_create(fp, &hdr); + TEST_NOT_NULL(ti); + TEST_EQUAL_UI(((sqfs_object_t *)fp)->refcount, 2); + TEST_EQUAL_UI(((sqfs_object_t *)ti)->refcount, 1); + + ret = istream_read(ti, buffer, sizeof(buffer)); + TEST_EQUAL_I(ret, 5); + buffer[5] = '\0'; + TEST_STR_EQUAL(buffer, "test\n"); + + ret = istream_read(ti, buffer, sizeof(buffer)); + TEST_EQUAL_I(ret, 0); + + clear_header(&hdr); + sqfs_drop(ti); + TEST_EQUAL_UI(((sqfs_object_t *)fp)->refcount, 1); + sqfs_drop(fp); + return EXIT_SUCCESS; +} diff --git a/lib/tar/test/tar_istream2.c b/lib/tar/test/tar_istream2.c new file mode 100644 index 0000000..a3f27d5 --- /dev/null +++ b/lib/tar/test/tar_istream2.c @@ -0,0 +1,155 @@ +/* SPDX-License-Identifier: GPL-3.0-or-later */ +/* + * tar_istream2.c + * + * Copyright (C) 2019 David Oberhollenzer <goliath@infraroot.at> + */ +#include "config.h" +#include "io/file.h" +#include "tar/tar.h" +#include "util/test.h" + +int main(int argc, char **argv) +{ + tar_header_decoded_t hdr; + char buffer[100]; + istream_t *fp; + istream_t *ti; + (void)argc; (void)argv; + + TEST_ASSERT(chdir(TEST_PATH) == 0); + + fp = istream_open_file("format-acceptance/link_filled.tar"); + TEST_NOT_NULL(fp); + TEST_EQUAL_UI(((sqfs_object_t *)fp)->refcount, 1); + + /* "deep" directory hierarchy containg 2 files */ + TEST_EQUAL_UI(((sqfs_object_t *)fp)->refcount, 1); + TEST_ASSERT(read_header(fp, &hdr) == 0); + TEST_EQUAL_UI(((sqfs_object_t *)fp)->refcount, 1); + TEST_EQUAL_UI(hdr.mode, S_IFDIR | 0777); + TEST_STR_EQUAL(hdr.name, "20_characters_here01/"); + clear_header(&hdr); + + TEST_EQUAL_UI(((sqfs_object_t *)fp)->refcount, 1); + TEST_ASSERT(read_header(fp, &hdr) == 0); + TEST_EQUAL_UI(((sqfs_object_t *)fp)->refcount, 1); + TEST_EQUAL_UI(hdr.mode, S_IFDIR | 0777); + TEST_STR_EQUAL(hdr.name, "20_characters_here01/20_characters_here02/"); + clear_header(&hdr); + + TEST_EQUAL_UI(((sqfs_object_t *)fp)->refcount, 1); + TEST_ASSERT(read_header(fp, &hdr) == 0); + TEST_EQUAL_UI(((sqfs_object_t *)fp)->refcount, 1); + TEST_EQUAL_UI(hdr.mode, S_IFDIR | 0777); + TEST_STR_EQUAL(hdr.name, "20_characters_here01/20_characters_here02/" + "20_characters_here03/"); + clear_header(&hdr); + + TEST_EQUAL_UI(((sqfs_object_t *)fp)->refcount, 1); + TEST_ASSERT(read_header(fp, &hdr) == 0); + TEST_EQUAL_UI(((sqfs_object_t *)fp)->refcount, 1); + TEST_EQUAL_UI(hdr.mode, S_IFDIR | 0777); + TEST_STR_EQUAL(hdr.name, "20_characters_here01/20_characters_here02/" + "20_characters_here03/20_characters_here04/"); + clear_header(&hdr); + + TEST_EQUAL_UI(((sqfs_object_t *)fp)->refcount, 1); + TEST_ASSERT(read_header(fp, &hdr) == 0); + TEST_EQUAL_UI(((sqfs_object_t *)fp)->refcount, 1); + TEST_EQUAL_UI(hdr.mode, S_IFREG | 0777); + TEST_STR_EQUAL(hdr.name, "20_characters_here01/20_characters_here02/" + "20_characters_here03/20_characters_here04/" + "errored_file_tst"); + TEST_EQUAL_UI(hdr.actual_size, 5); + + ti = tar_record_istream_create(fp, &hdr); + TEST_NOT_NULL(ti); + TEST_EQUAL_UI(((sqfs_object_t *)fp)->refcount, 2); + TEST_EQUAL_UI(((sqfs_object_t *)ti)->refcount, 1); + clear_header(&hdr); + + TEST_ASSERT(istream_read(ti, buffer, sizeof(buffer)) == 5); + buffer[5] = '\0'; + TEST_STR_EQUAL(buffer, "test\n"); + TEST_ASSERT(istream_read(ti, buffer, sizeof(buffer)) == 0); + + ti = sqfs_drop(ti); + TEST_EQUAL_UI(((sqfs_object_t *)fp)->refcount, 1); + + TEST_ASSERT(skip_padding(fp, 5) == 0); + + TEST_EQUAL_UI(((sqfs_object_t *)fp)->refcount, 1); + TEST_ASSERT(read_header(fp, &hdr) == 0); + TEST_EQUAL_UI(((sqfs_object_t *)fp)->refcount, 1); + TEST_EQUAL_UI(hdr.mode, S_IFREG | 0777); + TEST_STR_EQUAL(hdr.name, "20_characters_here01/20_characters_here02/" + "20_characters_here03/20_characters_here04/" + "some_test_file"); + TEST_EQUAL_UI(hdr.actual_size, 5); + + ti = tar_record_istream_create(fp, &hdr); + TEST_NOT_NULL(ti); + TEST_EQUAL_UI(((sqfs_object_t *)fp)->refcount, 2); + TEST_EQUAL_UI(((sqfs_object_t *)ti)->refcount, 1); + clear_header(&hdr); + + TEST_ASSERT(istream_read(ti, buffer, sizeof(buffer)) == 5); + buffer[5] = '\0'; + TEST_STR_EQUAL(buffer, "test\n"); + TEST_ASSERT(istream_read(ti, buffer, sizeof(buffer)) == 0); + + ti = sqfs_drop(ti); + TEST_EQUAL_UI(((sqfs_object_t *)fp)->refcount, 1); + + TEST_ASSERT(skip_padding(fp, 5) == 0); + + /* "deep" directory hierarchy containg a hard link */ + TEST_EQUAL_UI(((sqfs_object_t *)fp)->refcount, 1); + TEST_ASSERT(read_header(fp, &hdr) == 0); + TEST_EQUAL_UI(((sqfs_object_t *)fp)->refcount, 1); + TEST_EQUAL_UI(hdr.mode, S_IFDIR | 0777); + TEST_STR_EQUAL(hdr.name, "20CharsForLnkTest001/"); + clear_header(&hdr); + + TEST_EQUAL_UI(((sqfs_object_t *)fp)->refcount, 1); + TEST_ASSERT(read_header(fp, &hdr) == 0); + TEST_EQUAL_UI(((sqfs_object_t *)fp)->refcount, 1); + TEST_EQUAL_UI(hdr.mode, S_IFDIR | 0777); + TEST_STR_EQUAL(hdr.name, "20CharsForLnkTest001/20CharsForLnkTest002/"); + clear_header(&hdr); + + TEST_EQUAL_UI(((sqfs_object_t *)fp)->refcount, 1); + TEST_ASSERT(read_header(fp, &hdr) == 0); + TEST_EQUAL_UI(((sqfs_object_t *)fp)->refcount, 1); + TEST_EQUAL_UI(hdr.mode, S_IFDIR | 0777); + TEST_STR_EQUAL(hdr.name, "20CharsForLnkTest001/20CharsForLnkTest002/" + "20CharsForLnkTest003/"); + clear_header(&hdr); + + TEST_EQUAL_UI(((sqfs_object_t *)fp)->refcount, 1); + TEST_ASSERT(read_header(fp, &hdr) == 0); + TEST_EQUAL_UI(((sqfs_object_t *)fp)->refcount, 1); + TEST_EQUAL_UI(hdr.mode, S_IFDIR | 0777); + TEST_STR_EQUAL(hdr.name, "20CharsForLnkTest001/20CharsForLnkTest002/" + "20CharsForLnkTest003/20CharsForLnkTest004/"); + clear_header(&hdr); + + TEST_EQUAL_UI(((sqfs_object_t *)fp)->refcount, 1); + TEST_ASSERT(read_header(fp, &hdr) == 0); + TEST_EQUAL_UI(((sqfs_object_t *)fp)->refcount, 1); + TEST_STR_EQUAL(hdr.name, "20CharsForLnkTest001/20CharsForLnkTest002/" + "20CharsForLnkTest003/20CharsForLnkTest004/" + "01234567890123456789"); + TEST_ASSERT(hdr.is_hard_link); + + TEST_STR_EQUAL(hdr.link_target, "20_characters_here01/" + "20_characters_here02/20_characters_here03/" + "20_characters_here04/errored_file_tst"); + clear_header(&hdr); + + /* end of file */ + TEST_ASSERT(read_header(fp, &hdr) > 0); + sqfs_drop(fp); + return EXIT_SUCCESS; +} diff --git a/lib/tar/test/tar_istream3.c b/lib/tar/test/tar_istream3.c new file mode 100644 index 0000000..d287081 --- /dev/null +++ b/lib/tar/test/tar_istream3.c @@ -0,0 +1,100 @@ +/* SPDX-License-Identifier: GPL-3.0-or-later */ +/* + * tar_istream3.c + * + * Copyright (C) 2019 David Oberhollenzer <goliath@infraroot.at> + */ +#include "config.h" +#include "io/file.h" +#include "tar/tar.h" +#include "util/test.h" + +static const struct { + uint64_t offset; + size_t size; + int fill; +} regions[] = { + { 0, 4096, 'A' }, + { 262144, 4096, 'B' }, + { 524288, 4096, 'C' }, + { 786432, 4096, 'D' }, + { 1048576, 4096, 'E' }, + { 1310720, 4096, 'F' }, + { 1572864, 4096, 'G' }, + { 1835008, 4096, 'H' }, +}; + +static int byte_from_offset(uint64_t offset) +{ + sqfs_u64 diff; + size_t i; + + for (i = 0; i < sizeof(regions) / sizeof(regions[0]); ++i) { + if (offset >= regions[i].offset) { + diff = (offset - regions[i].offset); + + if (diff < regions[i].size) + return regions[i].fill; + } + } + + return '\0'; +} + +int main(int argc, char **argv) +{ + unsigned char buffer[941]; + tar_header_decoded_t hdr; + istream_t *fp, *ti; + uint64_t offset; + sqfs_s32 i, ret; + (void)argc; (void)argv; + + fp = istream_open_file(STRVALUE(TESTPATH) "/" STRVALUE(TESTFILE)); + TEST_NOT_NULL(fp); + TEST_ASSERT(read_header(fp, &hdr) == 0); + TEST_EQUAL_UI(hdr.mode, S_IFREG | 0644); + TEST_EQUAL_UI(hdr.uid, 01750); + TEST_EQUAL_UI(hdr.gid, 01750); + TEST_EQUAL_UI(hdr.actual_size, 2097152); + TEST_EQUAL_UI(hdr.record_size, 32768); + TEST_STR_EQUAL(hdr.name, "input.bin"); + TEST_ASSERT(!hdr.unknown_record); + + ti = tar_record_istream_create(fp, &hdr); + TEST_EQUAL_UI(((sqfs_object_t *)fp)->refcount, 2); + TEST_EQUAL_UI(((sqfs_object_t *)ti)->refcount, 1); + clear_header(&hdr); + + offset = 0; + + for (;;) { + ret = istream_read(ti, buffer, sizeof(buffer)); + TEST_ASSERT(ret >= 0); + + if (ret == 0) + break; + + for (i = 0; i < ret; ++i) { + int ref_byte = byte_from_offset(offset + i); + + if (buffer[i] != ref_byte) { + fprintf(stderr, "Byte at offset %llu should " + "be 0x%02X, but is 0x%02X\n", + (unsigned long long)(offset + i), + (unsigned int)ref_byte, buffer[i]); + return EXIT_FAILURE; + } + } + + offset += ret; + TEST_ASSERT(offset <= 2097152); + } + + TEST_EQUAL_UI(offset, 2097152); + + sqfs_drop(ti); + TEST_EQUAL_UI(((sqfs_object_t *)fp)->refcount, 1); + sqfs_drop(fp); + return EXIT_SUCCESS; +} |