From eb9a4b9034453ae3093d678a6f3898303dc5a5a0 Mon Sep 17 00:00:00 2001 From: David Oberhollenzer Date: Mon, 12 Jun 2023 18:32:12 +0200 Subject: libio: remove ostream_append_sparse and fallback implementation Instead of a separate append-sparse function, simply accept NULL as an input for append. For both Unix and Win32, a fallback needs to be implemented. For XFRM, we can just memset the input buffer to zero, same for the libsquashfs data writer. Signed-off-by: David Oberhollenzer --- include/io/file.h | 5 ++ include/io/ostream.h | 34 +---------- lib/common/src/data_reader_dump.c | 2 +- lib/io/Makemodule.am | 8 +-- lib/io/src/ostream.c | 35 ----------- lib/io/src/unix/ostream.c | 102 +++++++++++++++++++++----------- lib/io/src/win32/ostream.c | 74 +++++++++++++++++++---- lib/io/src/xfrm/ostream.c | 8 ++- lib/io/test/sparse_fb.c | 66 --------------------- lib/io/test/xfrm.c | 10 ++-- lib/sqfs/src/block_processor/frontend.c | 12 +++- lib/tar/src/padd_file.c | 2 +- lib/tar/test/tar_write_simple.c | 9 ++- 13 files changed, 167 insertions(+), 200 deletions(-) delete mode 100644 lib/io/src/ostream.c delete mode 100644 lib/io/test/sparse_fb.c diff --git a/include/io/file.h b/include/io/file.h index f3723b2..44fd224 100644 --- a/include/io/file.h +++ b/include/io/file.h @@ -18,6 +18,11 @@ typedef HANDLE os_file_t; typedef int os_file_t; #endif +enum { + OSTREAM_OPEN_OVERWRITE = 0x01, + OSTREAM_OPEN_SPARSE = 0x02, +}; + #ifdef __cplusplus extern "C" { #endif diff --git a/include/io/ostream.h b/include/io/ostream.h index 8fc8cef..7a57e48 100644 --- a/include/io/ostream.h +++ b/include/io/ostream.h @@ -23,15 +23,14 @@ typedef struct ostream_t { * @brief Append a block of data to an output stream. * * @param strm A pointer to an output stream. - * @param data A pointer to the data block to append. + * @param data A pointer to the data block to append. If NULL, + * synthesize a chunk of zero bytes. * @param size The number of bytes to append. * * @return Zero on success, -1 on failure. */ int (*append)(struct ostream_t *strm, const void *data, size_t size); - int (*append_sparse)(struct ostream_t *strm, size_t size); - /** * @brief Process all pending, buffered data and flush it to disk. * @@ -56,33 +55,4 @@ typedef struct ostream_t { const char *(*get_filename)(struct ostream_t *strm); } ostream_t; -enum { - OSTREAM_OPEN_OVERWRITE = 0x01, - OSTREAM_OPEN_SPARSE = 0x02, -}; - -#ifdef __cplusplus -extern "C" { -#endif - -/** - * @brief Append a number of zero bytes to an output stream. - * - * @memberof ostream_t - * - * If the unerlying implementation supports sparse files, this function can be - * used to create a "hole". If the implementation does not support it, a - * fallback is used that just appends a block of zeros manualy. - * - * @param strm A pointer to an output stream. - * @param size The number of zero bytes to append. - * - * @return Zero on success, -1 on failure. - */ -SQFS_INTERNAL int ostream_append_sparse(ostream_t *strm, size_t size); - -#ifdef __cplusplus -} -#endif - #endif /* IO_OSTREAM_H */ diff --git a/lib/common/src/data_reader_dump.c b/lib/common/src/data_reader_dump.c index 29043f8..8f38bd2 100644 --- a/lib/common/src/data_reader_dump.c +++ b/lib/common/src/data_reader_dump.c @@ -26,7 +26,7 @@ int sqfs_data_reader_dump(const char *name, sqfs_data_reader_t *data, diff = (filesz < block_size) ? filesz : block_size; if (SQFS_IS_SPARSE_BLOCK(inode->extra[i])) { - if (ostream_append_sparse(fp, diff)) + if (fp->append(fp, NULL, diff)) return -1; } else { err = sqfs_data_reader_get_block(data, inode, i, diff --git a/lib/io/Makemodule.am b/lib/io/Makemodule.am index 5ec1f7e..badaa99 100644 --- a/lib/io/Makemodule.am +++ b/lib/io/Makemodule.am @@ -1,7 +1,6 @@ libio_a_SOURCES = include/io/istream.h include/io/ostream.h include/io/xfrm.h \ include/io/file.h include/io/std.h include/io/dir_entry.h \ - include/io/dir_iterator.h include/io/mem.h \ - lib/io/src/internal.h lib/io/src/ostream.c \ + include/io/dir_iterator.h include/io/mem.h lib/io/src/internal.h \ lib/io/src/istream.c lib/io/src/get_line.c lib/io/src/xfrm/ostream.c \ lib/io/src/xfrm/istream.c lib/io/src/dir_tree_iterator.c \ lib/io/src/dir_entry.c lib/io/src/mem.c @@ -19,7 +18,7 @@ endif noinst_LIBRARIES += libio.a -LIBIO_TESTS = test_istream_mem test_get_line test_sparse_fb test_istream_read \ +LIBIO_TESTS = test_istream_mem test_get_line test_istream_read \ test_istream_skip test_stream_splice test_dir_iterator \ test_dir_tree_iterator test_dir_tree_iterator2 test_dir_tree_iterator3 @@ -31,9 +30,6 @@ test_get_line_SOURCES = lib/io/test/get_line.c test_get_line_LDADD = libio.a libcompat.a test_get_line_CPPFLAGS = $(AM_CPPFLAGS) -test_sparse_fb_SOURCES = lib/io/test/sparse_fb.c -test_sparse_fb_LDADD = libio.a libutil.a libcompat.a - test_istream_read_SOURCES = lib/io/test/istream_read.c test_istream_read_LDADD = libio.a libutil.a libcompat.a diff --git a/lib/io/src/ostream.c b/lib/io/src/ostream.c deleted file mode 100644 index 35f98d4..0000000 --- a/lib/io/src/ostream.c +++ /dev/null @@ -1,35 +0,0 @@ -/* SPDX-License-Identifier: GPL-3.0-or-later */ -/* - * ostream.c - * - * Copyright (C) 2019 David Oberhollenzer - */ -#include "internal.h" - - -static int append_sparse_fallback(ostream_t *strm, size_t size) -{ - char buffer[512]; - size_t diff; - - memset(buffer, 0, sizeof(buffer)); - - while (size > 0) { - diff = size < sizeof(buffer) ? size : sizeof(buffer); - - if (strm->append(strm, buffer, diff)) - return -1; - - size -= diff; - } - - return 0; -} - -int ostream_append_sparse(ostream_t *strm, size_t size) -{ - if (strm->append_sparse == NULL) - return append_sparse_fallback(strm, size); - - return strm->append_sparse(strm, size); -} diff --git a/lib/io/src/unix/ostream.c b/lib/io/src/unix/ostream.c index 294a15e..702a354 100644 --- a/lib/io/src/unix/ostream.c +++ b/lib/io/src/unix/ostream.c @@ -9,81 +9,115 @@ typedef struct { ostream_t base; char *path; + int flags; int fd; off_t sparse_count; off_t size; } file_ostream_t; -static int file_append(ostream_t *strm, const void *data, size_t size) +static int write_all(file_ostream_t *file, const sqfs_u8 *data, size_t size) { - file_ostream_t *file = (file_ostream_t *)strm; - ssize_t ret; - - if (size == 0) - return 0; - - if (file->sparse_count > 0) { - if (lseek(file->fd, file->sparse_count, SEEK_CUR) == (off_t)-1) - goto fail_errno; - - file->sparse_count = 0; - } - while (size > 0) { - ret = write(file->fd, data, size); + ssize_t ret = write(file->fd, data, size); if (ret == 0) { - fprintf(stderr, "%s: truncated data write.\n", - file->path); + fprintf(stderr, "%s: truncated write.\n", file->path); return -1; } if (ret < 0) { if (errno == EINTR) continue; - goto fail_errno; + perror(file->path); + return -1; } file->size += ret; size -= ret; - data = (const char *)data + ret; + data += ret; + } + + return 0; +} + +static int realize_sparse(file_ostream_t *file) +{ + unsigned char *buffer; + size_t diff, bufsz; + + if (file->sparse_count == 0) + return 0; + + if (file->flags & OSTREAM_OPEN_SPARSE) { + if (lseek(file->fd, file->sparse_count, SEEK_CUR) == (off_t)-1) + goto fail; + + if (ftruncate(file->fd, file->size) != 0) + goto fail; + + file->sparse_count = 0; + } else { + bufsz = file->sparse_count > 1024 ? 1024 : file->sparse_count; + buffer = calloc(1, bufsz); + if (buffer == NULL) + goto fail; + + while (file->sparse_count > 0) { + diff = file->sparse_count > (off_t)bufsz ? + bufsz : (size_t)file->sparse_count; + + if (write_all(file, buffer, diff)) { + free(buffer); + return -1; + } + + file->sparse_count -= diff; + } + + free(buffer); } return 0; -fail_errno: +fail: perror(file->path); return -1; } -static int file_append_sparse(ostream_t *strm, size_t size) +static int file_append(ostream_t *strm, const void *data, size_t size) { file_ostream_t *file = (file_ostream_t *)strm; - file->sparse_count += size; - file->size += size; - return 0; + if (size == 0) + return 0; + + if (data == NULL) { + file->sparse_count += size; + file->size += size; + return 0; + } + + if (realize_sparse(file)) + return -1; + + return write_all(file, data, size); } static int file_flush(ostream_t *strm) { file_ostream_t *file = (file_ostream_t *)strm; - if (file->sparse_count > 0) { - if (ftruncate(file->fd, file->size) != 0) - goto fail; - } + if (realize_sparse(file)) + return -1; if (fsync(file->fd) != 0) { if (errno == EINVAL) return 0; - goto fail; + perror(file->path); + return -1; } return 0; -fail: - perror(file->path); - return -1; } static void file_destroy(sqfs_object_t *obj) @@ -128,9 +162,7 @@ ostream_t *ostream_open_handle(const char *path, int fd, int flags) close(fd); - if (flags & OSTREAM_OPEN_SPARSE) - strm->append_sparse = file_append_sparse; - + file->flags = flags; strm->append = file_append; strm->flush = file_flush; strm->get_filename = file_get_filename; diff --git a/lib/io/src/win32/ostream.c b/lib/io/src/win32/ostream.c index d18130f..9b488d8 100644 --- a/lib/io/src/win32/ostream.c +++ b/lib/io/src/win32/ostream.c @@ -11,13 +11,14 @@ typedef struct { ostream_t base; + sqfs_u64 sparse_count; char *path; HANDLE hnd; + int flags; } file_ostream_t; -static int file_append(ostream_t *strm, const void *data, size_t size) +static int write_data(file_ostream_t *file, const void *data, size_t size) { - file_ostream_t *file = (file_ostream_t *)strm; DWORD diff; while (size > 0) { @@ -33,18 +34,48 @@ static int file_append(ostream_t *strm, const void *data, size_t size) return 0; } -static int file_append_sparse(ostream_t *strm, size_t size) +static int realize_sparse(file_ostream_t *file) { - file_ostream_t *file = (file_ostream_t *)strm; + size_t bufsz, diff; LARGE_INTEGER pos; + void *buffer; + + if (file->sparse_count == 0) + return 0; - pos.QuadPart = size; + if (file->flags & OSTREAM_OPEN_SPARSE) { + pos.QuadPart = file->sparse_count; - if (!SetFilePointerEx(file->hnd, pos, NULL, FILE_CURRENT)) - goto fail; + if (!SetFilePointerEx(file->hnd, pos, NULL, FILE_CURRENT)) + goto fail; - if (!SetEndOfFile(file->hnd)) - goto fail; + if (!SetEndOfFile(file->hnd)) + goto fail; + + file->sparse_count = 0; + } else { + bufsz = file->sparse_count > 1024 ? 1024 : file->sparse_count; + buffer = calloc(1, bufsz); + + if (buffer == NULL) { + fputs("out-of-memory\n", stderr); + return -1; + } + + while (file->sparse_count > 0) { + diff = file->sparse_count > bufsz ? + bufsz : file->sparse_count; + + if (write_data(file, buffer, diff)) { + free(buffer); + return -1; + } + + file->sparse_count -= diff; + } + + free(buffer); + } return 0; fail: @@ -52,10 +83,31 @@ fail: return -1; } +static int file_append(ostream_t *strm, const void *data, size_t size) +{ + file_ostream_t *file = (file_ostream_t *)strm; + + if (size == 0) + return 0; + + if (data == NULL) { + file->sparse_count += size; + return 0; + } + + if (realize_sparse(file)) + return -1; + + return write_data(file, data, size); +} + static int file_flush(ostream_t *strm) { file_ostream_t *file = (file_ostream_t *)strm; + if (realize_sparse(file)) + return -1; + if (!FlushFileBuffers(file->hnd)) { w32_perror(file->path); return -1; @@ -109,9 +161,7 @@ ostream_t *ostream_open_handle(const char *path, HANDLE hnd, int flags) CloseHandle(hnd); - if (flags & OSTREAM_OPEN_SPARSE) - strm->append_sparse = file_append_sparse; - + file->flags = flags; strm->append = file_append; strm->flush = file_flush; strm->get_filename = file_get_filename; diff --git a/lib/io/src/xfrm/ostream.c b/lib/io/src/xfrm/ostream.c index 79ed49a..e55e38c 100644 --- a/lib/io/src/xfrm/ostream.c +++ b/lib/io/src/xfrm/ostream.c @@ -82,10 +82,14 @@ static int xfrm_append(ostream_t *strm, const void *data, size_t size) if (diff > size) diff = size; - memcpy(xfrm->inbuf + xfrm->inbuf_used, data, diff); + if (data == NULL) { + memset(xfrm->inbuf + xfrm->inbuf_used, 0, diff); + } else { + memcpy(xfrm->inbuf + xfrm->inbuf_used, data, diff); + data = (const char *)data + diff; + } xfrm->inbuf_used += diff; - data = (const char *)data + diff; size -= diff; } diff --git a/lib/io/test/sparse_fb.c b/lib/io/test/sparse_fb.c deleted file mode 100644 index 5ee6004..0000000 --- a/lib/io/test/sparse_fb.c +++ /dev/null @@ -1,66 +0,0 @@ -/* SPDX-License-Identifier: GPL-3.0-or-later */ -/* - * get_line.c - * - * Copyright (C) 2019 David Oberhollenzer - */ -#include "config.h" - -#include "io/ostream.h" -#include "util/test.h" -#include "util/util.h" - -static ostream_t dummy; -static size_t total = 0; - -static int dummy_append(ostream_t *strm, const void *data, size_t size) -{ - bool bret; - - TEST_ASSERT(strm == &dummy); - TEST_NOT_NULL(data); - TEST_ASSERT(size > 0); - - bret = is_memory_zero(data, size); - TEST_ASSERT(bret); - - bret = SZ_ADD_OV(total, size, &total); - TEST_ASSERT(!bret); - return 0; -} - -static int dummy_flush(ostream_t *strm) -{ - TEST_ASSERT(strm == &dummy); - return 0; -} - -static ostream_t dummy = { - { - 1, - NULL, - NULL, - }, - dummy_append, - NULL, - dummy_flush, - NULL, -}; - -int main(int argc, char **argv) -{ - size_t ref; - int ret; - (void)argc; (void)argv; - - ref = 131072 + 1337; - - ret = ostream_append_sparse(&dummy, ref); - TEST_EQUAL_I(ret, 0); - - ret = dummy.flush(&dummy); - TEST_EQUAL_I(ret, 0); - - TEST_EQUAL_UI(ref, total); - return EXIT_SUCCESS; -} diff --git a/lib/io/test/xfrm.c b/lib/io/test/xfrm.c index c2e450c..184db9e 100644 --- a/lib/io/test/xfrm.c +++ b/lib/io/test/xfrm.c @@ -374,7 +374,6 @@ static int mem_flush(ostream_t *strm); static ostream_t mem_ostream = { { 1, NULL, NULL, }, mem_append, - NULL, mem_flush, NULL, }; @@ -382,15 +381,18 @@ static ostream_t mem_ostream = { static int mem_append(ostream_t *strm, const void *data, size_t size) { TEST_ASSERT(strm == &mem_ostream); - TEST_NOT_NULL(data); TEST_ASSERT(size > 0); TEST_ASSERT(mo_written <= sizeof(mo_buffer)); TEST_ASSERT(size <= (sizeof(mo_buffer) - mo_written)); - memcpy(mo_buffer + mo_written, data, size); - mo_written += size; + if (data == NULL) { + memset(mo_buffer + mo_written, 0, size); + } else { + memcpy(mo_buffer + mo_written, data, size); + } + mo_written += size; return 0; } diff --git a/lib/sqfs/src/block_processor/frontend.c b/lib/sqfs/src/block_processor/frontend.c index e8a4207..de89913 100644 --- a/lib/sqfs/src/block_processor/frontend.c +++ b/lib/sqfs/src/block_processor/frontend.c @@ -114,6 +114,7 @@ int sqfs_block_processor_append(sqfs_block_processor_t *proc, const void *data, sqfs_block_t *new; sqfs_u64 filesize; size_t diff; + void *dst; int err; if (!proc->begin_called) @@ -152,12 +153,17 @@ int sqfs_block_processor_append(sqfs_block_processor_t *proc, const void *data, if (diff > size) diff = size; - memcpy(proc->blk_current->data + proc->blk_current->size, - data, diff); + dst = proc->blk_current->data + proc->blk_current->size; + + if (data == NULL) { + memset(dst, 0, diff); + } else { + memcpy(dst, data, diff); + data = (const char *)data + diff; + } size -= diff; proc->blk_current->size += diff; - data = (const char *)data + diff; proc->stats.input_bytes_read += diff; } diff --git a/lib/tar/src/padd_file.c b/lib/tar/src/padd_file.c index 053ff1e..0f6ee0a 100644 --- a/lib/tar/src/padd_file.c +++ b/lib/tar/src/padd_file.c @@ -15,5 +15,5 @@ int padd_file(ostream_t *fp, sqfs_u64 size) if (padd_sz == 0) return 0; - return ostream_append_sparse(fp, TAR_RECORD_SIZE - padd_sz); + return fp->append(fp, NULL, TAR_RECORD_SIZE - padd_sz); } diff --git a/lib/tar/test/tar_write_simple.c b/lib/tar/test/tar_write_simple.c index 2302307..b34c9c5 100644 --- a/lib/tar/test/tar_write_simple.c +++ b/lib/tar/test/tar_write_simple.c @@ -21,7 +21,6 @@ static ostream_t mem_stream = { { 1, NULL, NULL }, buffer_append, NULL, - NULL, buffer_get_filename, }; @@ -33,12 +32,16 @@ static sqfs_u8 rd_buffer[1024 * 10]; static int buffer_append(ostream_t *strm, const void *data, size_t size) { TEST_ASSERT(strm == &mem_stream); - TEST_NOT_NULL(data); TEST_ASSERT(wr_offset < sizeof(wr_buffer)); TEST_ASSERT(size > 0); TEST_ASSERT((sizeof(wr_buffer) - wr_offset) >= size); - memcpy(wr_buffer + wr_offset, data, size); + if (data == NULL) { + memset(wr_buffer + wr_offset, 0, size); + } else { + memcpy(wr_buffer + wr_offset, data, size); + } + wr_offset += size; return 0; } -- cgit v1.2.3