From 3511b1fa7c6f71c579e161951e945904e552e1d9 Mon Sep 17 00:00:00 2001 From: David Oberhollenzer Date: Wed, 25 Sep 2019 17:47:19 +0200 Subject: Remove condensed sparse file handling from libsquashfs This only exists for tar2sqfs. Move the sparse file map to libtar and add the ability to do this into the stind sqfs_file_t abstraction, so it acts like a normal file but internally stitches the data together from the sparse implementation. Signed-off-by: David Oberhollenzer --- include/data_writer.h | 14 --------- include/highlevel.h | 3 +- include/sqfs/io.h | 44 ---------------------------- include/sqfs/predef.h | 1 - include/tar.h | 10 +++++-- lib/sqfs/io.c | 66 ------------------------------------------ lib/sqfshelper/data_writer.c | 21 +++----------- lib/sqfshelper/io_stdin.c | 67 +++++++++++++++++++++++++++++++++++++++++-- lib/tar/cleanup.c | 4 +-- lib/tar/internal.h | 6 ++-- lib/tar/read_header.c | 2 +- lib/tar/read_sparse_map.c | 4 +-- lib/tar/read_sparse_map_old.c | 4 +-- tar/tar2sqfs.c | 27 +++++++---------- tests/tar_sparse_gnu.c | 2 +- tests/tar_sparse_gnu1.c | 2 +- tests/tar_sparse_gnu2.c | 2 +- 17 files changed, 101 insertions(+), 178 deletions(-) diff --git a/include/data_writer.h b/include/data_writer.h index e29881b..cdf7a44 100644 --- a/include/data_writer.h +++ b/include/data_writer.h @@ -88,20 +88,6 @@ int data_writer_sync(data_writer_t *data); int write_data_from_file(data_writer_t *data, sqfs_inode_generic_t *inode, sqfs_file_t *file, int flags); -/* - Does the same as write_data_from_fd but the input file is the condensed - representation of a sparse file. The layout must be in order and - non-overlapping. - - The flags argument is a combination of DW_* flags. After completion the - data writer collects the 'fi' in an internal list it uses for deduplication. - - Returns 0 on success, prints errors to stderr. - */ -int write_data_from_file_condensed(data_writer_t *data, sqfs_file_t *file, - sqfs_inode_generic_t *inode, - const sqfs_sparse_map_t *map, int flags); - data_writer_stats_t *data_writer_get_stats(data_writer_t *data); #endif /* DATA_WRITER_H */ diff --git a/include/highlevel.h b/include/highlevel.h index ed0f4bf..bfc2d91 100644 --- a/include/highlevel.h +++ b/include/highlevel.h @@ -23,6 +23,7 @@ #include "sqfs/data_reader.h" #include "data_writer.h" #include "fstree.h" +#include "tar.h" #include #include @@ -83,6 +84,6 @@ int sqfs_data_reader_dump(sqfs_data_reader_t *data, const sqfs_inode_generic_t *inode, int outfd, size_t block_size, bool allow_sparse); -sqfs_file_t *sqfs_get_stdin_file(uint64_t size); +sqfs_file_t *sqfs_get_stdin_file(const sparse_map_t *map, uint64_t size); #endif /* HIGHLEVEL_H */ diff --git a/include/sqfs/io.h b/include/sqfs/io.h index d7a923c..5c1bfd5 100644 --- a/include/sqfs/io.h +++ b/include/sqfs/io.h @@ -120,20 +120,6 @@ struct sqfs_file_t { int (*truncate)(sqfs_file_t *file, uint64_t size); }; -/** - * @struct sqfs_sparse_map_t - * - * @brief Describes the layout of a sparse file. - * - * This structure is part of a linked list that indicates where the actual - * data is located in a sparse file. - */ -struct sqfs_sparse_map_t { - sqfs_sparse_map_t *next; - uint64_t offset; - uint64_t count; -}; - #ifdef __cplusplus extern "C" { #endif @@ -175,36 +161,6 @@ SQFS_API int sqfs_file_create_block(sqfs_file_t *file, uint64_t offset, size_t size, sqfs_inode_generic_t *inode, uint32_t flags, sqfs_block_t **out); -/** - * @brief Read a chunk from a condensed version of a sparse file and turn it - * into a block that can be fed to a block processor. - * - * @member sqfs_file_t - * - * This function works on condensed sparse files, i.e. a sparse file that had - * its holdes removed. The given mapping describes the original data region - * that are actually packed next to each other. The function emulates the - * orignal sparse file by zero-initializing the block data, then figuring - * out which regions overlap the block, working out their physical location and - * stitching the block together. - * - * @param file A pointer to a file implementation. - * @param offset A byte offset into the file. - * @param size The number of bytes to read, starting at the given offset. - * @param inode The inode pointer to set for the block. - * @param flags The flags to store in the newly created block. - * @param map Describes the data regions of the original sparse file. - * @param out Returns a pointer to a block on success. - * - * @return Zero on success, an @ref E_SQFS_ERROR identifier on failure. - */ -SQFS_API int sqfs_file_create_block_dense(sqfs_file_t *file, uint64_t offset, - size_t size, - sqfs_inode_generic_t *inode, - uint32_t flags, - const sqfs_sparse_map_t *map, - sqfs_block_t **out); - #ifdef __cplusplus } #endif diff --git a/include/sqfs/predef.h b/include/sqfs/predef.h index 8a554dd..ebe99e1 100644 --- a/include/sqfs/predef.h +++ b/include/sqfs/predef.h @@ -69,7 +69,6 @@ typedef struct sqfs_meta_reader_t sqfs_meta_reader_t; typedef struct sqfs_meta_writer_t sqfs_meta_writer_t; typedef struct sqfs_xattr_reader_t sqfs_xattr_reader_t; typedef struct sqfs_file_t sqfs_file_t; -typedef struct sqfs_sparse_map_t sqfs_sparse_map_t; typedef struct sqfs_tree_node_t sqfs_tree_node_t; typedef struct sqfs_data_reader_t sqfs_data_reader_t; typedef struct sqfs_block_hooks_t sqfs_block_hooks_t; diff --git a/include/tar.h b/include/tar.h index 38f6289..45457da 100644 --- a/include/tar.h +++ b/include/tar.h @@ -8,13 +8,17 @@ #define TAR_H #include "config.h" +#include "util.h" #include #include #include -#include "sqfs/io.h" -#include "util.h" +typedef struct sparse_map_t { + struct sparse_map_t *next; + uint64_t offset; + uint64_t count; +} sparse_map_t; typedef struct { char name[100]; @@ -75,7 +79,7 @@ typedef struct { struct stat sb; char *name; char *link_target; - sqfs_sparse_map_t *sparse; + sparse_map_t *sparse; uint64_t actual_size; uint64_t record_size; bool unknown_record; diff --git a/lib/sqfs/io.c b/lib/sqfs/io.c index f4ffda2..0021a30 100644 --- a/lib/sqfs/io.c +++ b/lib/sqfs/io.c @@ -37,69 +37,3 @@ int sqfs_file_create_block(sqfs_file_t *file, uint64_t offset, *out = blk; return 0; } - -int sqfs_file_create_block_dense(sqfs_file_t *file, uint64_t offset, - size_t size, sqfs_inode_generic_t *inode, - uint32_t flags, const sqfs_sparse_map_t *map, - sqfs_block_t **out) -{ - sqfs_block_t *blk = alloc_flex(sizeof(*blk), 1, size); - size_t dst_start, diff, count; - const sqfs_sparse_map_t *it; - uint64_t poffset, src_start; - int err; - - if (blk == NULL) - return SQFS_ERROR_ALLOC; - - poffset = 0; - - for (it = map; it != NULL; it = it->next) { - if (it->offset + it->count <= offset) { - poffset += it->count; - continue; - } - - if (it->offset >= offset + size) { - poffset += it->count; - continue; - } - - count = size; - - if (offset + count >= it->offset + it->count) - count = it->offset + it->count - offset; - - if (it->offset < offset) { - diff = offset - it->offset; - - src_start = poffset + diff; - dst_start = 0; - count -= diff; - } else if (it->offset > offset) { - diff = it->offset - offset; - - src_start = poffset; - dst_start = diff; - } else { - src_start = poffset; - dst_start = 0; - } - - err = file->read_at(file, src_start, - blk->data + dst_start, count); - if (err) { - free(blk); - return err; - } - - poffset += it->count; - } - - blk->inode = inode; - blk->size = size; - blk->flags = flags; - - *out = blk; - return 0; -} diff --git a/lib/sqfshelper/data_writer.c b/lib/sqfshelper/data_writer.c index 96cd9ee..a365e6d 100644 --- a/lib/sqfshelper/data_writer.c +++ b/lib/sqfshelper/data_writer.c @@ -98,9 +98,8 @@ static int add_sentinel_block(data_writer_t *data, sqfs_inode_generic_t *inode, return sqfs_data_writer_enqueue(data->proc, blk); } -int write_data_from_file_condensed(data_writer_t *data, sqfs_file_t *file, - sqfs_inode_generic_t *inode, - const sqfs_sparse_map_t *map, int flags) +int write_data_from_file(data_writer_t *data, sqfs_inode_generic_t *inode, + sqfs_file_t *file, int flags) { uint32_t blk_flags = SQFS_BLK_FIRST_BLOCK; uint64_t filesz, offset; @@ -123,14 +122,8 @@ int write_data_from_file_condensed(data_writer_t *data, sqfs_file_t *file, diff = filesz - offset; } - if (map == NULL) { - ret = sqfs_file_create_block(file, offset, diff, inode, - blk_flags, &blk); - } else { - ret = sqfs_file_create_block_dense(file, offset, diff, - inode, blk_flags, - map, &blk); - } + ret = sqfs_file_create_block(file, offset, diff, inode, + blk_flags, &blk); if (ret) return -1; @@ -189,12 +182,6 @@ int write_data_from_file_condensed(data_writer_t *data, sqfs_file_t *file, return 0; } -int write_data_from_file(data_writer_t *data, sqfs_inode_generic_t *inode, - sqfs_file_t *file, int flags) -{ - return write_data_from_file_condensed(data, file, inode, NULL, flags); -} - data_writer_t *data_writer_create(sqfs_super_t *super, sqfs_compressor_t *cmp, sqfs_file_t *file, size_t devblksize, unsigned int num_jobs, size_t max_backlog) diff --git a/lib/sqfshelper/io_stdin.c b/lib/sqfshelper/io_stdin.c index 6cb45d4..8568f5e 100644 --- a/lib/sqfshelper/io_stdin.c +++ b/lib/sqfshelper/io_stdin.c @@ -11,12 +11,14 @@ #include #include +#include #include typedef struct { sqfs_file_t base; + const sparse_map_t *map; uint64_t offset; uint64_t size; } sqfs_file_stdin_t; @@ -78,6 +80,60 @@ static int stdin_read_at(sqfs_file_t *base, uint64_t offset, return 0; } +static int stdin_read_condensed(sqfs_file_t *base, uint64_t offset, + void *buffer, size_t size) +{ + sqfs_file_stdin_t *file = (sqfs_file_stdin_t *)base; + uint64_t poffset = 0, src_start; + size_t dst_start, diff, count; + const sparse_map_t *it; + int err; + + memset(buffer, 0, size); + + for (it = file->map; it != NULL; it = it->next) { + if (it->offset + it->count <= offset) { + poffset += it->count; + continue; + } + + if (it->offset >= offset + size) { + poffset += it->count; + continue; + } + + count = size; + + if (offset + count >= it->offset + it->count) + count = it->offset + it->count - offset; + + if (it->offset < offset) { + diff = offset - it->offset; + + src_start = poffset + diff; + dst_start = 0; + count -= diff; + } else if (it->offset > offset) { + diff = it->offset - offset; + + src_start = poffset; + dst_start = diff; + } else { + src_start = poffset; + dst_start = 0; + } + + err = stdin_read_at(base, src_start, + (char *)buffer + dst_start, count); + if (err) + return err; + + poffset += it->count; + } + + return 0; +} + static int stdin_write_at(sqfs_file_t *base, uint64_t offset, const void *buffer, size_t size) { @@ -96,7 +152,7 @@ static int stdin_truncate(sqfs_file_t *base, uint64_t size) return SQFS_ERROR_IO; } -sqfs_file_t *sqfs_get_stdin_file(uint64_t size) +sqfs_file_t *sqfs_get_stdin_file(const sparse_map_t *map, uint64_t size) { sqfs_file_stdin_t *file = calloc(1, sizeof(*file)); sqfs_file_t *base = (sqfs_file_t *)file; @@ -105,10 +161,17 @@ sqfs_file_t *sqfs_get_stdin_file(uint64_t size) return NULL; file->size = size; + file->map = map; + base->destroy = stdin_destroy; - base->read_at = stdin_read_at; base->write_at = stdin_write_at; base->get_size = stdin_get_size; base->truncate = stdin_truncate; + + if (map == NULL) { + base->read_at = stdin_read_at; + } else { + base->read_at = stdin_read_condensed; + } return base; } diff --git a/lib/tar/cleanup.c b/lib/tar/cleanup.c index 3c1d0e9..2f814bf 100644 --- a/lib/tar/cleanup.c +++ b/lib/tar/cleanup.c @@ -8,9 +8,9 @@ #include "internal.h" -void free_sparse_list(sqfs_sparse_map_t *sparse) +void free_sparse_list(sparse_map_t *sparse) { - sqfs_sparse_map_t *old; + sparse_map_t *old; while (sparse != NULL) { old = sparse; diff --git a/lib/tar/internal.h b/lib/tar/internal.h index bc9a3a8..d95ef52 100644 --- a/lib/tar/internal.h +++ b/lib/tar/internal.h @@ -55,11 +55,11 @@ void update_checksum(tar_header_t *hdr); bool is_checksum_valid(const tar_header_t *hdr); -sqfs_sparse_map_t *read_sparse_map(const char *line); +sparse_map_t *read_sparse_map(const char *line); -sqfs_sparse_map_t *read_gnu_old_sparse(int fd, tar_header_t *hdr); +sparse_map_t *read_gnu_old_sparse(int fd, tar_header_t *hdr); -void free_sparse_list(sqfs_sparse_map_t *sparse); +void free_sparse_list(sparse_map_t *sparse); void free_xattr_list(tar_xattr_t *list); diff --git a/lib/tar/read_header.c b/lib/tar/read_header.c index a874972..63de699 100644 --- a/lib/tar/read_header.c +++ b/lib/tar/read_header.c @@ -77,7 +77,7 @@ static tar_xattr_t *mkxattr(const char *key, size_t keylen, static int read_pax_header(int fd, uint64_t entsize, unsigned int *set_by_pax, tar_header_decoded_t *out) { - sqfs_sparse_map_t *sparse_last = NULL, *sparse; + sparse_map_t *sparse_last = NULL, *sparse; uint64_t field, offset = 0, num_bytes = 0; char *buffer, *line, *key, *ptr, *value; tar_xattr_t *xattr; diff --git a/lib/tar/read_sparse_map.c b/lib/tar/read_sparse_map.c index 981a808..0779b96 100644 --- a/lib/tar/read_sparse_map.c +++ b/lib/tar/read_sparse_map.c @@ -8,9 +8,9 @@ #include "internal.h" -sqfs_sparse_map_t *read_sparse_map(const char *line) +sparse_map_t *read_sparse_map(const char *line) { - sqfs_sparse_map_t *last = NULL, *list = NULL, *ent = NULL; + sparse_map_t *last = NULL, *list = NULL, *ent = NULL; do { ent = calloc(1, sizeof(*ent)); diff --git a/lib/tar/read_sparse_map_old.c b/lib/tar/read_sparse_map_old.c index 7fb4cb3..5891bdb 100644 --- a/lib/tar/read_sparse_map_old.c +++ b/lib/tar/read_sparse_map_old.c @@ -8,9 +8,9 @@ #include "internal.h" -sqfs_sparse_map_t *read_gnu_old_sparse(int fd, tar_header_t *hdr) +sparse_map_t *read_gnu_old_sparse(int fd, tar_header_t *hdr) { - sqfs_sparse_map_t *list = NULL, *end = NULL, *node; + sparse_map_t *list = NULL, *end = NULL, *node; gnu_sparse_t sph; uint64_t off, sz; int i; diff --git a/tar/tar2sqfs.c b/tar/tar2sqfs.c index e62985a..82720a8 100644 --- a/tar/tar2sqfs.c +++ b/tar/tar2sqfs.c @@ -225,7 +225,7 @@ fail_arg: static int write_file(tar_header_decoded_t *hdr, file_info_t *fi, data_writer_t *data, uint64_t filesize) { - const sqfs_sparse_map_t *it; + const sparse_map_t *it; sqfs_inode_generic_t *inode; size_t max_blk_count; sqfs_file_t *file; @@ -253,25 +253,17 @@ static int write_file(tar_header_decoded_t *hdr, file_info_t *fi, for (sum = 0, it = hdr->sparse; it != NULL; it = it->next) sum += it->count; - file = sqfs_get_stdin_file(sum); + file = sqfs_get_stdin_file(hdr->sparse, sum); if (file == NULL) { perror("packing files"); return -1; } - - ret = write_data_from_file_condensed(data, file, inode, - hdr->sparse, 0); - file->destroy(file); - if (ret) + } else { + file = sqfs_get_stdin_file(NULL, filesize); + if (file == NULL) { + perror("packing files"); return -1; - - return skip_padding(STDIN_FILENO, hdr->record_size); - } - - file = sqfs_get_stdin_file(filesize); - if (file == NULL) { - perror("packing files"); - return -1; + } } ret = write_data_from_file(data, inode, file, 0); @@ -280,7 +272,8 @@ static int write_file(tar_header_decoded_t *hdr, file_info_t *fi, if (ret) return -1; - return skip_padding(STDIN_FILENO, filesize); + return skip_padding(STDIN_FILENO, hdr->sparse == NULL ? + filesize : hdr->record_size); } static int copy_xattr(fstree_t *fs, tree_node_t *node, @@ -344,7 +337,7 @@ static int process_tar_ball(fstree_t *fs, data_writer_t *data) { tar_header_decoded_t hdr; uint64_t offset, count; - sqfs_sparse_map_t *m; + sparse_map_t *m; bool skip; int ret; diff --git a/tests/tar_sparse_gnu.c b/tests/tar_sparse_gnu.c index e0a0200..7cd6132 100644 --- a/tests/tar_sparse_gnu.c +++ b/tests/tar_sparse_gnu.c @@ -36,7 +36,7 @@ static int open_read(const char *path) int main(void) { tar_header_decoded_t hdr; - sqfs_sparse_map_t *sparse; + sparse_map_t *sparse; int fd; assert(chdir(TEST_PATH) == 0); diff --git a/tests/tar_sparse_gnu1.c b/tests/tar_sparse_gnu1.c index 0d8ab6b..366b0f4 100644 --- a/tests/tar_sparse_gnu1.c +++ b/tests/tar_sparse_gnu1.c @@ -36,7 +36,7 @@ static int open_read(const char *path) int main(void) { tar_header_decoded_t hdr; - sqfs_sparse_map_t *sparse; + sparse_map_t *sparse; int fd; assert(chdir(TEST_PATH) == 0); diff --git a/tests/tar_sparse_gnu2.c b/tests/tar_sparse_gnu2.c index 568c402..cfc7eb3 100644 --- a/tests/tar_sparse_gnu2.c +++ b/tests/tar_sparse_gnu2.c @@ -36,7 +36,7 @@ static int open_read(const char *path) int main(void) { tar_header_decoded_t hdr; - sqfs_sparse_map_t *sparse; + sparse_map_t *sparse; int fd; assert(chdir(TEST_PATH) == 0); -- cgit v1.2.3