From 5a3b741b92b793be7221a481efca316aec208ebe Mon Sep 17 00:00:00 2001 From: David Oberhollenzer Date: Tue, 18 Jul 2023 21:54:50 +0200 Subject: Add a data reader based sqfs_istream_t implementation To the sqfs_data_reader_t is added, an sqfs_istream_t implementation that internally reads through the data reader. The uses of the data_reader_dump function are removed and the function is subsequently removed from libcommon. Signed-off-by: David Oberhollenzer --- bin/rdsquashfs/src/fill_files.c | 16 +++- bin/rdsquashfs/src/rdsquashfs.c | 17 +++- bin/sqfs2tar/src/write_tree.c | 19 +++- bin/sqfsdiff/src/extract.c | 16 +++- include/common.h | 4 - include/sqfs/data_reader.h | 21 ++++ lib/common/Makemodule.am | 2 +- lib/common/src/data_reader_dump.c | 67 ------------- lib/sqfs/src/data_reader.c | 197 ++++++++++++++++++++++++++++++++++++++ 9 files changed, 275 insertions(+), 84 deletions(-) delete mode 100644 lib/common/src/data_reader_dump.c diff --git a/bin/rdsquashfs/src/fill_files.c b/bin/rdsquashfs/src/fill_files.c index 7087883..b26041a 100644 --- a/bin/rdsquashfs/src/fill_files.c +++ b/bin/rdsquashfs/src/fill_files.c @@ -137,6 +137,7 @@ static int fill_files(sqfs_data_reader_t *data, int flags) { int ret, openflags; sqfs_ostream_t *fp; + sqfs_istream_t *in; size_t i; openflags = SQFS_FILE_OPEN_OVERWRITE; @@ -154,8 +155,19 @@ static int fill_files(sqfs_data_reader_t *data, int flags) if (!(flags & UNPACK_QUIET)) printf("unpacking %s\n", files[i].path); - ret = sqfs_data_reader_dump(files[i].path, data, files[i].inode, - fp, block_size); + ret = sqfs_data_reader_create_stream(data, files[i].inode, + files[i].path, &in); + if (ret) { + sqfs_perror(files[i].path, NULL, ret); + return -1; + } + + do { + ret = sqfs_istream_splice(in, fp, block_size); + } while (ret > 0); + + sqfs_drop(in); + if (ret == 0) ret = fp->flush(fp); diff --git a/bin/rdsquashfs/src/rdsquashfs.c b/bin/rdsquashfs/src/rdsquashfs.c index 77dc5ba..9d8f3ba 100644 --- a/bin/rdsquashfs/src/rdsquashfs.c +++ b/bin/rdsquashfs/src/rdsquashfs.c @@ -208,11 +208,13 @@ int main(int argc, char **argv) break; case OP_CAT: { sqfs_ostream_t *fp; + sqfs_istream_t *in; int ret; - if (!S_ISREG(n->inode->base.mode)) { - fprintf(stderr, "/%s: not a regular file\n", - opt.cmdpath); + ret = sqfs_data_reader_create_stream(data, n->inode, + opt.cmdpath, &in); + if (ret) { + sqfs_perror(opt.cmdpath, NULL, ret); goto out; } @@ -222,8 +224,13 @@ int main(int argc, char **argv) goto out; } - ret = sqfs_data_reader_dump(opt.cmdpath, data, n->inode, - fp, super.block_size); + do { + ret = sqfs_istream_splice(in, fp, super.block_size); + if (ret < 0) + sqfs_perror(opt.cmdpath, "splicing data", ret); + } while (ret > 0); + + sqfs_drop(in); sqfs_drop(fp); if (ret) goto out; diff --git a/bin/sqfs2tar/src/write_tree.c b/bin/sqfs2tar/src/write_tree.c index dc3ac72..ed82173 100644 --- a/bin/sqfs2tar/src/write_tree.c +++ b/bin/sqfs2tar/src/write_tree.c @@ -157,13 +157,26 @@ static int write_tree_dfs(const sqfs_tree_node_t *n) } if (S_ISREG(sb.st_mode)) { - if (sqfs_data_reader_dump(name, data, n->inode, out_file, - super.block_size)) { + sqfs_istream_t *in; + int ret; + + ret = sqfs_data_reader_create_stream(data, n->inode, name, &in); + if (ret) { + sqfs_perror(name, NULL, ret); sqfs_free(name); return -1; } - ret = padd_file(out_file, sb.st_size); + do { + ret = sqfs_istream_splice(in, out_file, + super.block_size); + } while (ret > 0); + + sqfs_drop(in); + + if (ret == 0) + ret = padd_file(out_file, sb.st_size); + if (ret) { sqfs_perror(name, NULL, ret); sqfs_free(name); diff --git a/bin/sqfsdiff/src/extract.c b/bin/sqfsdiff/src/extract.c index cbc4381..df74f0f 100644 --- a/bin/sqfsdiff/src/extract.c +++ b/bin/sqfsdiff/src/extract.c @@ -11,6 +11,7 @@ static int extract(sqfs_data_reader_t *data, const sqfs_inode_generic_t *inode, { char *ptr, *temp; sqfs_ostream_t *fp; + sqfs_istream_t *in; int ret; temp = alloca(strlen(prefix) + strlen(path) + 2); @@ -28,12 +29,23 @@ static int extract(sqfs_data_reader_t *data, const sqfs_inode_generic_t *inode, return -1; } - if (sqfs_data_reader_dump(path, data, inode, fp, block_size)) { + ret = sqfs_data_reader_create_stream(data, inode, path, &in); + if (ret) { sqfs_drop(fp); return -1; } - ret = fp->flush(fp); + do { + ret = sqfs_istream_splice(in, fp, block_size); + if (ret < 0) + sqfs_perror(path, "splicing data", ret); + } while (ret > 0); + + sqfs_drop(in); + + if (ret == 0) + ret = fp->flush(fp); + if (ret) { sqfs_perror(fp->get_filename(fp), NULL, ret); sqfs_drop(fp); diff --git a/include/common.h b/include/common.h index 8e24d03..707d46b 100644 --- a/include/common.h +++ b/include/common.h @@ -33,10 +33,6 @@ typedef struct sqfs_hard_link_t { char *target; } sqfs_hard_link_t; -int sqfs_data_reader_dump(const char *name, sqfs_data_reader_t *data, - const sqfs_inode_generic_t *inode, - sqfs_ostream_t *fp, size_t block_size); - int write_data_from_file(const char *filename, sqfs_block_processor_t *data, sqfs_inode_generic_t **inode, sqfs_file_t *file, int flags); diff --git a/include/sqfs/data_reader.h b/include/sqfs/data_reader.h index 29077b7..dabf5c8 100644 --- a/include/sqfs/data_reader.h +++ b/include/sqfs/data_reader.h @@ -145,6 +145,27 @@ SQFS_API sqfs_s32 sqfs_data_reader_read(sqfs_data_reader_t *data, sqfs_u64 offset, void *buffer, sqfs_u32 size); +/** + * @brief Create an @ref sqfs_istream_t implementation for a squashfs file + * + * @memberof sqfs_data_reader_t + * + * This function creates a simple file stream implementation that internally + * reads data from a file in a SquashFS image. The reader is grabbed, the inode + * and filename are copied internally and not needed after creation. + * + * @param data A pointer to a data reader object. + * @param inode A pointer to the inode describing the file. + * @param filename A file name that the stream should return when asked. + * @param out Returns a pointer to a stream on success, NULL on failure. + * + * @return Zero on success, a negative @ref SQFS_ERROR value on failure. + */ +SQFS_API int sqfs_data_reader_create_stream(sqfs_data_reader_t *data, + const sqfs_inode_generic_t *inode, + const char *filename, + sqfs_istream_t **out); + #ifdef __cplusplus } #endif diff --git a/lib/common/Makemodule.am b/lib/common/Makemodule.am index 61d87d4..e0cc551 100644 --- a/lib/common/Makemodule.am +++ b/lib/common/Makemodule.am @@ -1,7 +1,7 @@ libcommon_a_SOURCES = include/common.h include/simple_writer.h \ include/compress_cli.h \ lib/common/src/hardlink.c lib/common/src/print_version.c \ - lib/common/src/data_reader_dump.c lib/common/src/compress.c \ + lib/common/src/compress.c \ lib/common/src/comp_opt.c lib/common/src/data_writer.c \ lib/common/src/parse_size.c lib/common/src/print_size.c \ lib/common/src/writer/init.c lib/common/src/writer/cleanup.c \ diff --git a/lib/common/src/data_reader_dump.c b/lib/common/src/data_reader_dump.c deleted file mode 100644 index 920b2bd..0000000 --- a/lib/common/src/data_reader_dump.c +++ /dev/null @@ -1,67 +0,0 @@ -/* SPDX-License-Identifier: GPL-3.0-or-later */ -/* - * data_reader_dump.c - * - * Copyright (C) 2019 David Oberhollenzer - */ -#include "common.h" - -#include -#include -#include -#include - -int sqfs_data_reader_dump(const char *name, sqfs_data_reader_t *data, - const sqfs_inode_generic_t *inode, - sqfs_ostream_t *fp, size_t block_size) -{ - size_t i, diff, chunk_size; - sqfs_u64 filesz; - sqfs_u8 *chunk; - int err; - - sqfs_inode_get_file_size(inode, &filesz); - - for (i = 0; i < sqfs_inode_get_file_block_count(inode); ++i) { - diff = (filesz < block_size) ? filesz : block_size; - - if (SQFS_IS_SPARSE_BLOCK(inode->extra[i])) { - err = fp->append(fp, NULL, diff); - } else { - err = sqfs_data_reader_get_block(data, inode, i, - &chunk_size, &chunk); - if (err) { - sqfs_perror(name, "reading data block", err); - return -1; - } - - err = fp->append(fp, chunk, chunk_size); - free(chunk); - } - - if (err) - goto fail_io; - - filesz -= diff; - } - - if (filesz > 0) { - err = sqfs_data_reader_get_fragment(data, inode, - &chunk_size, &chunk); - if (err) { - sqfs_perror(name, "reading fragment block", err); - return -1; - } - - err = fp->append(fp, chunk, chunk_size); - free(chunk); - - if (err) - goto fail_io; - } - - return 0; -fail_io: - sqfs_perror(fp->get_filename(fp), "writing data block", err); - return -1; -} diff --git a/lib/sqfs/src/data_reader.c b/lib/sqfs/src/data_reader.c index 3f0cd74..e87259f 100644 --- a/lib/sqfs/src/data_reader.c +++ b/lib/sqfs/src/data_reader.c @@ -372,3 +372,200 @@ sqfs_s32 sqfs_data_reader_read(sqfs_data_reader_t *data, return total; } + +/*****************************************************************************/ + +typedef struct { + sqfs_istream_t base; + + sqfs_data_reader_t *rd; + const char *filename; + const sqfs_u32 *blocks; + + sqfs_u8 *buffer; + size_t buf_used; + size_t buf_off; + + sqfs_u64 filesz; + sqfs_u64 disk_offset; + + sqfs_u32 frag_idx; + sqfs_u32 frag_off; + + sqfs_u32 blk_idx; + sqfs_u32 blk_count; + + sqfs_u32 inodata[]; +} data_reader_istream_t; + +static int dr_stream_get_buffered_data(sqfs_istream_t *base, + const sqfs_u8 **out, + size_t *size, size_t want) +{ + data_reader_istream_t *stream = (data_reader_istream_t *)base; + sqfs_data_reader_t *rd = stream->rd; + int ret; + (void)want; + + if (stream->buf_off < stream->buf_used) { + *out = stream->buffer + stream->buf_off; + *size = stream->buf_used - stream->buf_off; + return 0; + } + + if (stream->filesz == 0) { + ret = 1; + goto fail; + } + + stream->buf_off = 0; + stream->buf_used = rd->block_size; + if (stream->filesz < (sqfs_u64)stream->buf_used) + stream->buf_used = stream->filesz; + + if (stream->blk_idx < stream->blk_count) { + sqfs_u32 blkword = stream->blocks[stream->blk_idx++]; + sqfs_u32 disksz = SQFS_ON_DISK_BLOCK_SIZE(blkword); + + if (disksz == 0) { + memset(stream->buffer, 0, stream->buf_used); + } else if (SQFS_IS_BLOCK_COMPRESSED(blkword)) { + ret = rd->file->read_at(rd->file, stream->disk_offset, + rd->scratch, disksz); + if (ret) + goto fail; + + ret = rd->cmp->do_block(rd->cmp, rd->scratch, disksz, + stream->buffer, + stream->buf_used); + if (ret <= 0) { + ret = ret < 0 ? ret : SQFS_ERROR_OVERFLOW; + goto fail; + } + + if ((size_t)ret < stream->buf_used) { + memset(stream->buffer + ret, 0, + stream->buf_used - ret); + } + } else { + ret = rd->file->read_at(rd->file, stream->disk_offset, + stream->buffer, disksz); + if (ret) + goto fail; + if (disksz < stream->buf_used) { + memset(stream->buffer + disksz, 0, + stream->buf_used - disksz); + } + } + + stream->disk_offset += disksz; + } else { + ret = precache_fragment_block(rd, stream->frag_idx); + if (ret) + return ret; + + if (rd->frag_blk_size < stream->frag_off || + (rd->frag_blk_size - stream->frag_off) < stream->buf_used) { + ret = SQFS_ERROR_CORRUPTED; + goto fail; + } + + memcpy(stream->buffer, rd->frag_block + stream->frag_off, + stream->buf_used); + } + + stream->filesz -= stream->buf_used; + *out = stream->buffer; + *size = stream->buf_used; + return 0; +fail: + free(stream->buffer); + stream->buffer = NULL; + stream->buf_used = 0; + stream->buf_off = 0; + stream->filesz = 0; + *out = NULL; + *size = 0; + return ret; +} + +static void dr_stream_advance_buffer(sqfs_istream_t *base, size_t count) +{ + data_reader_istream_t *stream = (data_reader_istream_t *)base; + size_t diff = stream->buf_used - stream->buf_off; + + stream->buf_off += (diff < count ? diff : count); +} + +static const char *dr_stream_get_filename(sqfs_istream_t *base) +{ + return ((data_reader_istream_t *)base)->filename; +} + +static void dr_stream_destroy(sqfs_object_t *obj) +{ + data_reader_istream_t *stream = (data_reader_istream_t *)obj; + + sqfs_drop(stream->rd); + free(stream->buffer); + free(stream); +} + +int sqfs_data_reader_create_stream(sqfs_data_reader_t *data, + const sqfs_inode_generic_t *inode, + const char *filename, sqfs_istream_t **out) +{ + data_reader_istream_t *stream; + size_t ino_sz, namelen, sz; + sqfs_u64 filesz; + char *nameptr; + int ret; + + *out = NULL; + + ret = sqfs_inode_get_file_size(inode, &filesz); + if (ret != 0) + return ret; + + ino_sz = inode->payload_bytes_used; + namelen = strlen(filename) + 1; + + if (SZ_ADD_OV(ino_sz, namelen, &sz)) + return SQFS_ERROR_ALLOC; + if (SZ_ADD_OV(sz, sizeof(*stream), &sz)) + return SQFS_ERROR_ALLOC; + + stream = calloc(1, sz); + if (stream == NULL) + return SQFS_ERROR_ALLOC; + + stream->buffer = malloc(data->block_size); + if (stream->buffer == NULL) { + free(stream); + return SQFS_ERROR_ALLOC; + } + + sqfs_object_init(stream, dr_stream_destroy, NULL); + + memcpy(stream->inodata, inode->extra, ino_sz); + stream->blocks = stream->inodata; + stream->blk_count = ino_sz / sizeof(stream->blocks[0]); + stream->filesz = filesz; + + nameptr = (char *)stream->inodata + ino_sz; + memcpy(nameptr, filename, namelen); + stream->filename = nameptr; + + sqfs_inode_get_file_block_start(inode, &stream->disk_offset); + sqfs_inode_get_frag_location(inode, &stream->frag_idx, + &stream->frag_off); + stream->rd = sqfs_grab(data); + + ((sqfs_istream_t *)stream)->advance_buffer = dr_stream_advance_buffer; + ((sqfs_istream_t *)stream)->get_filename = dr_stream_get_filename; + ((sqfs_istream_t *)stream)->get_buffered_data = + dr_stream_get_buffered_data; + + *out = (sqfs_istream_t *)stream; + return 0; +} -- cgit v1.2.3