summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorDavid Oberhollenzer <david.oberhollenzer@sigma-star.at>2019-09-25 17:47:19 +0200
committerDavid Oberhollenzer <david.oberhollenzer@sigma-star.at>2019-09-25 17:47:19 +0200
commit3511b1fa7c6f71c579e161951e945904e552e1d9 (patch)
tree55fa94e5daef7bcc8e4b650f27d05af49fd1b02d
parent4d79f55f4a626a3cfd8bd18673aa29b48b16e137 (diff)
Remove condensed sparse file handling from libsquashfs
This only exists for tar2sqfs. Move the sparse file map to libtar and add the ability to do this into the stind sqfs_file_t abstraction, so it acts like a normal file but internally stitches the data together from the sparse implementation. Signed-off-by: David Oberhollenzer <david.oberhollenzer@sigma-star.at>
-rw-r--r--include/data_writer.h14
-rw-r--r--include/highlevel.h3
-rw-r--r--include/sqfs/io.h44
-rw-r--r--include/sqfs/predef.h1
-rw-r--r--include/tar.h10
-rw-r--r--lib/sqfs/io.c66
-rw-r--r--lib/sqfshelper/data_writer.c21
-rw-r--r--lib/sqfshelper/io_stdin.c67
-rw-r--r--lib/tar/cleanup.c4
-rw-r--r--lib/tar/internal.h6
-rw-r--r--lib/tar/read_header.c2
-rw-r--r--lib/tar/read_sparse_map.c4
-rw-r--r--lib/tar/read_sparse_map_old.c4
-rw-r--r--tar/tar2sqfs.c27
-rw-r--r--tests/tar_sparse_gnu.c2
-rw-r--r--tests/tar_sparse_gnu1.c2
-rw-r--r--tests/tar_sparse_gnu2.c2
17 files changed, 101 insertions, 178 deletions
diff --git a/include/data_writer.h b/include/data_writer.h
index e29881b..cdf7a44 100644
--- a/include/data_writer.h
+++ b/include/data_writer.h
@@ -88,20 +88,6 @@ int data_writer_sync(data_writer_t *data);
int write_data_from_file(data_writer_t *data, sqfs_inode_generic_t *inode,
sqfs_file_t *file, int flags);
-/*
- Does the same as write_data_from_fd but the input file is the condensed
- representation of a sparse file. The layout must be in order and
- non-overlapping.
-
- The flags argument is a combination of DW_* flags. After completion the
- data writer collects the 'fi' in an internal list it uses for deduplication.
-
- Returns 0 on success, prints errors to stderr.
- */
-int write_data_from_file_condensed(data_writer_t *data, sqfs_file_t *file,
- sqfs_inode_generic_t *inode,
- const sqfs_sparse_map_t *map, int flags);
-
data_writer_stats_t *data_writer_get_stats(data_writer_t *data);
#endif /* DATA_WRITER_H */
diff --git a/include/highlevel.h b/include/highlevel.h
index ed0f4bf..bfc2d91 100644
--- a/include/highlevel.h
+++ b/include/highlevel.h
@@ -23,6 +23,7 @@
#include "sqfs/data_reader.h"
#include "data_writer.h"
#include "fstree.h"
+#include "tar.h"
#include <sys/stat.h>
#include <stdint.h>
@@ -83,6 +84,6 @@ int sqfs_data_reader_dump(sqfs_data_reader_t *data,
const sqfs_inode_generic_t *inode,
int outfd, size_t block_size, bool allow_sparse);
-sqfs_file_t *sqfs_get_stdin_file(uint64_t size);
+sqfs_file_t *sqfs_get_stdin_file(const sparse_map_t *map, uint64_t size);
#endif /* HIGHLEVEL_H */
diff --git a/include/sqfs/io.h b/include/sqfs/io.h
index d7a923c..5c1bfd5 100644
--- a/include/sqfs/io.h
+++ b/include/sqfs/io.h
@@ -120,20 +120,6 @@ struct sqfs_file_t {
int (*truncate)(sqfs_file_t *file, uint64_t size);
};
-/**
- * @struct sqfs_sparse_map_t
- *
- * @brief Describes the layout of a sparse file.
- *
- * This structure is part of a linked list that indicates where the actual
- * data is located in a sparse file.
- */
-struct sqfs_sparse_map_t {
- sqfs_sparse_map_t *next;
- uint64_t offset;
- uint64_t count;
-};
-
#ifdef __cplusplus
extern "C" {
#endif
@@ -175,36 +161,6 @@ SQFS_API int sqfs_file_create_block(sqfs_file_t *file, uint64_t offset,
size_t size, sqfs_inode_generic_t *inode,
uint32_t flags, sqfs_block_t **out);
-/**
- * @brief Read a chunk from a condensed version of a sparse file and turn it
- * into a block that can be fed to a block processor.
- *
- * @member sqfs_file_t
- *
- * This function works on condensed sparse files, i.e. a sparse file that had
- * its holdes removed. The given mapping describes the original data region
- * that are actually packed next to each other. The function emulates the
- * orignal sparse file by zero-initializing the block data, then figuring
- * out which regions overlap the block, working out their physical location and
- * stitching the block together.
- *
- * @param file A pointer to a file implementation.
- * @param offset A byte offset into the file.
- * @param size The number of bytes to read, starting at the given offset.
- * @param inode The inode pointer to set for the block.
- * @param flags The flags to store in the newly created block.
- * @param map Describes the data regions of the original sparse file.
- * @param out Returns a pointer to a block on success.
- *
- * @return Zero on success, an @ref E_SQFS_ERROR identifier on failure.
- */
-SQFS_API int sqfs_file_create_block_dense(sqfs_file_t *file, uint64_t offset,
- size_t size,
- sqfs_inode_generic_t *inode,
- uint32_t flags,
- const sqfs_sparse_map_t *map,
- sqfs_block_t **out);
-
#ifdef __cplusplus
}
#endif
diff --git a/include/sqfs/predef.h b/include/sqfs/predef.h
index 8a554dd..ebe99e1 100644
--- a/include/sqfs/predef.h
+++ b/include/sqfs/predef.h
@@ -69,7 +69,6 @@ typedef struct sqfs_meta_reader_t sqfs_meta_reader_t;
typedef struct sqfs_meta_writer_t sqfs_meta_writer_t;
typedef struct sqfs_xattr_reader_t sqfs_xattr_reader_t;
typedef struct sqfs_file_t sqfs_file_t;
-typedef struct sqfs_sparse_map_t sqfs_sparse_map_t;
typedef struct sqfs_tree_node_t sqfs_tree_node_t;
typedef struct sqfs_data_reader_t sqfs_data_reader_t;
typedef struct sqfs_block_hooks_t sqfs_block_hooks_t;
diff --git a/include/tar.h b/include/tar.h
index 38f6289..45457da 100644
--- a/include/tar.h
+++ b/include/tar.h
@@ -8,13 +8,17 @@
#define TAR_H
#include "config.h"
+#include "util.h"
#include <sys/stat.h>
#include <stdbool.h>
#include <stdint.h>
-#include "sqfs/io.h"
-#include "util.h"
+typedef struct sparse_map_t {
+ struct sparse_map_t *next;
+ uint64_t offset;
+ uint64_t count;
+} sparse_map_t;
typedef struct {
char name[100];
@@ -75,7 +79,7 @@ typedef struct {
struct stat sb;
char *name;
char *link_target;
- sqfs_sparse_map_t *sparse;
+ sparse_map_t *sparse;
uint64_t actual_size;
uint64_t record_size;
bool unknown_record;
diff --git a/lib/sqfs/io.c b/lib/sqfs/io.c
index f4ffda2..0021a30 100644
--- a/lib/sqfs/io.c
+++ b/lib/sqfs/io.c
@@ -37,69 +37,3 @@ int sqfs_file_create_block(sqfs_file_t *file, uint64_t offset,
*out = blk;
return 0;
}
-
-int sqfs_file_create_block_dense(sqfs_file_t *file, uint64_t offset,
- size_t size, sqfs_inode_generic_t *inode,
- uint32_t flags, const sqfs_sparse_map_t *map,
- sqfs_block_t **out)
-{
- sqfs_block_t *blk = alloc_flex(sizeof(*blk), 1, size);
- size_t dst_start, diff, count;
- const sqfs_sparse_map_t *it;
- uint64_t poffset, src_start;
- int err;
-
- if (blk == NULL)
- return SQFS_ERROR_ALLOC;
-
- poffset = 0;
-
- for (it = map; it != NULL; it = it->next) {
- if (it->offset + it->count <= offset) {
- poffset += it->count;
- continue;
- }
-
- if (it->offset >= offset + size) {
- poffset += it->count;
- continue;
- }
-
- count = size;
-
- if (offset + count >= it->offset + it->count)
- count = it->offset + it->count - offset;
-
- if (it->offset < offset) {
- diff = offset - it->offset;
-
- src_start = poffset + diff;
- dst_start = 0;
- count -= diff;
- } else if (it->offset > offset) {
- diff = it->offset - offset;
-
- src_start = poffset;
- dst_start = diff;
- } else {
- src_start = poffset;
- dst_start = 0;
- }
-
- err = file->read_at(file, src_start,
- blk->data + dst_start, count);
- if (err) {
- free(blk);
- return err;
- }
-
- poffset += it->count;
- }
-
- blk->inode = inode;
- blk->size = size;
- blk->flags = flags;
-
- *out = blk;
- return 0;
-}
diff --git a/lib/sqfshelper/data_writer.c b/lib/sqfshelper/data_writer.c
index 96cd9ee..a365e6d 100644
--- a/lib/sqfshelper/data_writer.c
+++ b/lib/sqfshelper/data_writer.c
@@ -98,9 +98,8 @@ static int add_sentinel_block(data_writer_t *data, sqfs_inode_generic_t *inode,
return sqfs_data_writer_enqueue(data->proc, blk);
}
-int write_data_from_file_condensed(data_writer_t *data, sqfs_file_t *file,
- sqfs_inode_generic_t *inode,
- const sqfs_sparse_map_t *map, int flags)
+int write_data_from_file(data_writer_t *data, sqfs_inode_generic_t *inode,
+ sqfs_file_t *file, int flags)
{
uint32_t blk_flags = SQFS_BLK_FIRST_BLOCK;
uint64_t filesz, offset;
@@ -123,14 +122,8 @@ int write_data_from_file_condensed(data_writer_t *data, sqfs_file_t *file,
diff = filesz - offset;
}
- if (map == NULL) {
- ret = sqfs_file_create_block(file, offset, diff, inode,
- blk_flags, &blk);
- } else {
- ret = sqfs_file_create_block_dense(file, offset, diff,
- inode, blk_flags,
- map, &blk);
- }
+ ret = sqfs_file_create_block(file, offset, diff, inode,
+ blk_flags, &blk);
if (ret)
return -1;
@@ -189,12 +182,6 @@ int write_data_from_file_condensed(data_writer_t *data, sqfs_file_t *file,
return 0;
}
-int write_data_from_file(data_writer_t *data, sqfs_inode_generic_t *inode,
- sqfs_file_t *file, int flags)
-{
- return write_data_from_file_condensed(data, file, inode, NULL, flags);
-}
-
data_writer_t *data_writer_create(sqfs_super_t *super, sqfs_compressor_t *cmp,
sqfs_file_t *file, size_t devblksize,
unsigned int num_jobs, size_t max_backlog)
diff --git a/lib/sqfshelper/io_stdin.c b/lib/sqfshelper/io_stdin.c
index 6cb45d4..8568f5e 100644
--- a/lib/sqfshelper/io_stdin.c
+++ b/lib/sqfshelper/io_stdin.c
@@ -11,12 +11,14 @@
#include <stdlib.h>
#include <unistd.h>
+#include <string.h>
#include <errno.h>
typedef struct {
sqfs_file_t base;
+ const sparse_map_t *map;
uint64_t offset;
uint64_t size;
} sqfs_file_stdin_t;
@@ -78,6 +80,60 @@ static int stdin_read_at(sqfs_file_t *base, uint64_t offset,
return 0;
}
+static int stdin_read_condensed(sqfs_file_t *base, uint64_t offset,
+ void *buffer, size_t size)
+{
+ sqfs_file_stdin_t *file = (sqfs_file_stdin_t *)base;
+ uint64_t poffset = 0, src_start;
+ size_t dst_start, diff, count;
+ const sparse_map_t *it;
+ int err;
+
+ memset(buffer, 0, size);
+
+ for (it = file->map; it != NULL; it = it->next) {
+ if (it->offset + it->count <= offset) {
+ poffset += it->count;
+ continue;
+ }
+
+ if (it->offset >= offset + size) {
+ poffset += it->count;
+ continue;
+ }
+
+ count = size;
+
+ if (offset + count >= it->offset + it->count)
+ count = it->offset + it->count - offset;
+
+ if (it->offset < offset) {
+ diff = offset - it->offset;
+
+ src_start = poffset + diff;
+ dst_start = 0;
+ count -= diff;
+ } else if (it->offset > offset) {
+ diff = it->offset - offset;
+
+ src_start = poffset;
+ dst_start = diff;
+ } else {
+ src_start = poffset;
+ dst_start = 0;
+ }
+
+ err = stdin_read_at(base, src_start,
+ (char *)buffer + dst_start, count);
+ if (err)
+ return err;
+
+ poffset += it->count;
+ }
+
+ return 0;
+}
+
static int stdin_write_at(sqfs_file_t *base, uint64_t offset,
const void *buffer, size_t size)
{
@@ -96,7 +152,7 @@ static int stdin_truncate(sqfs_file_t *base, uint64_t size)
return SQFS_ERROR_IO;
}
-sqfs_file_t *sqfs_get_stdin_file(uint64_t size)
+sqfs_file_t *sqfs_get_stdin_file(const sparse_map_t *map, uint64_t size)
{
sqfs_file_stdin_t *file = calloc(1, sizeof(*file));
sqfs_file_t *base = (sqfs_file_t *)file;
@@ -105,10 +161,17 @@ sqfs_file_t *sqfs_get_stdin_file(uint64_t size)
return NULL;
file->size = size;
+ file->map = map;
+
base->destroy = stdin_destroy;
- base->read_at = stdin_read_at;
base->write_at = stdin_write_at;
base->get_size = stdin_get_size;
base->truncate = stdin_truncate;
+
+ if (map == NULL) {
+ base->read_at = stdin_read_at;
+ } else {
+ base->read_at = stdin_read_condensed;
+ }
return base;
}
diff --git a/lib/tar/cleanup.c b/lib/tar/cleanup.c
index 3c1d0e9..2f814bf 100644
--- a/lib/tar/cleanup.c
+++ b/lib/tar/cleanup.c
@@ -8,9 +8,9 @@
#include "internal.h"
-void free_sparse_list(sqfs_sparse_map_t *sparse)
+void free_sparse_list(sparse_map_t *sparse)
{
- sqfs_sparse_map_t *old;
+ sparse_map_t *old;
while (sparse != NULL) {
old = sparse;
diff --git a/lib/tar/internal.h b/lib/tar/internal.h
index bc9a3a8..d95ef52 100644
--- a/lib/tar/internal.h
+++ b/lib/tar/internal.h
@@ -55,11 +55,11 @@ void update_checksum(tar_header_t *hdr);
bool is_checksum_valid(const tar_header_t *hdr);
-sqfs_sparse_map_t *read_sparse_map(const char *line);
+sparse_map_t *read_sparse_map(const char *line);
-sqfs_sparse_map_t *read_gnu_old_sparse(int fd, tar_header_t *hdr);
+sparse_map_t *read_gnu_old_sparse(int fd, tar_header_t *hdr);
-void free_sparse_list(sqfs_sparse_map_t *sparse);
+void free_sparse_list(sparse_map_t *sparse);
void free_xattr_list(tar_xattr_t *list);
diff --git a/lib/tar/read_header.c b/lib/tar/read_header.c
index a874972..63de699 100644
--- a/lib/tar/read_header.c
+++ b/lib/tar/read_header.c
@@ -77,7 +77,7 @@ static tar_xattr_t *mkxattr(const char *key, size_t keylen,
static int read_pax_header(int fd, uint64_t entsize, unsigned int *set_by_pax,
tar_header_decoded_t *out)
{
- sqfs_sparse_map_t *sparse_last = NULL, *sparse;
+ sparse_map_t *sparse_last = NULL, *sparse;
uint64_t field, offset = 0, num_bytes = 0;
char *buffer, *line, *key, *ptr, *value;
tar_xattr_t *xattr;
diff --git a/lib/tar/read_sparse_map.c b/lib/tar/read_sparse_map.c
index 981a808..0779b96 100644
--- a/lib/tar/read_sparse_map.c
+++ b/lib/tar/read_sparse_map.c
@@ -8,9 +8,9 @@
#include "internal.h"
-sqfs_sparse_map_t *read_sparse_map(const char *line)
+sparse_map_t *read_sparse_map(const char *line)
{
- sqfs_sparse_map_t *last = NULL, *list = NULL, *ent = NULL;
+ sparse_map_t *last = NULL, *list = NULL, *ent = NULL;
do {
ent = calloc(1, sizeof(*ent));
diff --git a/lib/tar/read_sparse_map_old.c b/lib/tar/read_sparse_map_old.c
index 7fb4cb3..5891bdb 100644
--- a/lib/tar/read_sparse_map_old.c
+++ b/lib/tar/read_sparse_map_old.c
@@ -8,9 +8,9 @@
#include "internal.h"
-sqfs_sparse_map_t *read_gnu_old_sparse(int fd, tar_header_t *hdr)
+sparse_map_t *read_gnu_old_sparse(int fd, tar_header_t *hdr)
{
- sqfs_sparse_map_t *list = NULL, *end = NULL, *node;
+ sparse_map_t *list = NULL, *end = NULL, *node;
gnu_sparse_t sph;
uint64_t off, sz;
int i;
diff --git a/tar/tar2sqfs.c b/tar/tar2sqfs.c
index e62985a..82720a8 100644
--- a/tar/tar2sqfs.c
+++ b/tar/tar2sqfs.c
@@ -225,7 +225,7 @@ fail_arg:
static int write_file(tar_header_decoded_t *hdr, file_info_t *fi,
data_writer_t *data, uint64_t filesize)
{
- const sqfs_sparse_map_t *it;
+ const sparse_map_t *it;
sqfs_inode_generic_t *inode;
size_t max_blk_count;
sqfs_file_t *file;
@@ -253,25 +253,17 @@ static int write_file(tar_header_decoded_t *hdr, file_info_t *fi,
for (sum = 0, it = hdr->sparse; it != NULL; it = it->next)
sum += it->count;
- file = sqfs_get_stdin_file(sum);
+ file = sqfs_get_stdin_file(hdr->sparse, sum);
if (file == NULL) {
perror("packing files");
return -1;
}
-
- ret = write_data_from_file_condensed(data, file, inode,
- hdr->sparse, 0);
- file->destroy(file);
- if (ret)
+ } else {
+ file = sqfs_get_stdin_file(NULL, filesize);
+ if (file == NULL) {
+ perror("packing files");
return -1;
-
- return skip_padding(STDIN_FILENO, hdr->record_size);
- }
-
- file = sqfs_get_stdin_file(filesize);
- if (file == NULL) {
- perror("packing files");
- return -1;
+ }
}
ret = write_data_from_file(data, inode, file, 0);
@@ -280,7 +272,8 @@ static int write_file(tar_header_decoded_t *hdr, file_info_t *fi,
if (ret)
return -1;
- return skip_padding(STDIN_FILENO, filesize);
+ return skip_padding(STDIN_FILENO, hdr->sparse == NULL ?
+ filesize : hdr->record_size);
}
static int copy_xattr(fstree_t *fs, tree_node_t *node,
@@ -344,7 +337,7 @@ static int process_tar_ball(fstree_t *fs, data_writer_t *data)
{
tar_header_decoded_t hdr;
uint64_t offset, count;
- sqfs_sparse_map_t *m;
+ sparse_map_t *m;
bool skip;
int ret;
diff --git a/tests/tar_sparse_gnu.c b/tests/tar_sparse_gnu.c
index e0a0200..7cd6132 100644
--- a/tests/tar_sparse_gnu.c
+++ b/tests/tar_sparse_gnu.c
@@ -36,7 +36,7 @@ static int open_read(const char *path)
int main(void)
{
tar_header_decoded_t hdr;
- sqfs_sparse_map_t *sparse;
+ sparse_map_t *sparse;
int fd;
assert(chdir(TEST_PATH) == 0);
diff --git a/tests/tar_sparse_gnu1.c b/tests/tar_sparse_gnu1.c
index 0d8ab6b..366b0f4 100644
--- a/tests/tar_sparse_gnu1.c
+++ b/tests/tar_sparse_gnu1.c
@@ -36,7 +36,7 @@ static int open_read(const char *path)
int main(void)
{
tar_header_decoded_t hdr;
- sqfs_sparse_map_t *sparse;
+ sparse_map_t *sparse;
int fd;
assert(chdir(TEST_PATH) == 0);
diff --git a/tests/tar_sparse_gnu2.c b/tests/tar_sparse_gnu2.c
index 568c402..cfc7eb3 100644
--- a/tests/tar_sparse_gnu2.c
+++ b/tests/tar_sparse_gnu2.c
@@ -36,7 +36,7 @@ static int open_read(const char *path)
int main(void)
{
tar_header_decoded_t hdr;
- sqfs_sparse_map_t *sparse;
+ sparse_map_t *sparse;
int fd;
assert(chdir(TEST_PATH) == 0);