From 67800f0c2cd33bea2862a4b21132dc94e2d0fb73 Mon Sep 17 00:00:00 2001 From: David Oberhollenzer Date: Tue, 11 Jun 2019 02:24:58 +0200 Subject: Move data writer to libsqfs.a Signed-off-by: David Oberhollenzer --- include/data_writer.h | 49 ++++++++++++ lib/Makemodule.am | 2 + lib/sqfs/data_writer.c | 200 +++++++++++++++++++++++++++++++++++++++++++++++++ mkfs/block.c | 191 ---------------------------------------------- mkfs/mkfs.h | 14 +--- 5 files changed, 252 insertions(+), 204 deletions(-) create mode 100644 include/data_writer.h create mode 100644 lib/sqfs/data_writer.c diff --git a/include/data_writer.h b/include/data_writer.h new file mode 100644 index 0000000..cafe61e --- /dev/null +++ b/include/data_writer.h @@ -0,0 +1,49 @@ +/* SPDX-License-Identifier: GPL-3.0-or-later */ +#ifndef DATA_WRITER_H +#define DATA_WRITER_H + +#include "squashfs.h" +#include "compress.h" +#include "fstree.h" + +typedef struct data_writer_t data_writer_t; + +/* + Create a data writer. The pointer to the super block is kept internally and + used to automatically update various counters when writing data. + + Returns NULL on failure and prints errors to stderr. + */ +data_writer_t *data_writer_create(sqfs_super_t *super, compressor_t *cmp, + int outfd); + +void data_writer_destroy(data_writer_t *data); + +/* + Write the finalfragment table to the underlying file. + + Returns 0 on success, prints errors to stderr. +*/ +int data_writer_write_fragment_table(data_writer_t *data); + +/* + Compress and flush the current fragment buffer even if it is not full yet. + + Returns 0 on success, prints errors to stderr. +*/ +int data_writer_flush_fragments(data_writer_t *data); + +/* + Read data from the given file descriptor, partition it into blocks and + write them out (possibly compressed) to the underlying file. If the size + is not a multiple of the block size, the last bit is kept in an internal + fragment buffer which is written out if full. + + The file_info_t object is updated accordingly and used to determine the + number of bytes to write and the input file name to report errors. + + Returns 0 on success, prints errors to stderr. +*/ +int write_data_from_fd(data_writer_t *data, file_info_t *fi, int infd); + +#endif /* DATA_WRITER_H */ diff --git a/lib/Makemodule.am b/lib/Makemodule.am index 9496a9d..1b99a05 100644 --- a/lib/Makemodule.am +++ b/lib/Makemodule.am @@ -22,6 +22,8 @@ libsquashfs_a_SOURCES += lib/sqfs/write_dir.c lib/sqfs/write_inode.c libsquashfs_a_SOURCES += lib/sqfs/serialize_fstree.c libsquashfs_a_SOURCES += lib/sqfs/tree_node_from_inode.c libsquashfs_a_SOURCES += lib/sqfs/deserialize_fstree.c +libsquashfs_a_SOURCES += lib/sqfs/data_writer.c +libsquashfs_a_SOURCES += include/data_writer.h libsquashfs_a_SOURCES += include/frag_reader.h libutil_a_SOURCES = lib/util/canonicalize_name.c lib/util/write_retry.c diff --git a/lib/sqfs/data_writer.c b/lib/sqfs/data_writer.c new file mode 100644 index 0000000..a11e572 --- /dev/null +++ b/lib/sqfs/data_writer.c @@ -0,0 +1,200 @@ +/* SPDX-License-Identifier: GPL-3.0-or-later */ +#include "data_writer.h" +#include "highlevel.h" +#include "util.h" + +#include +#include +#include +#include + +struct data_writer_t { + void *block; + void *fragment; + void *scratch; + + sqfs_fragment_t *fragments; + size_t num_fragments; + size_t max_fragments; + size_t frag_offset; + + sqfs_super_t *super; + compressor_t *cmp; + int outfd; +}; + +static int write_compressed(data_writer_t *data, const void *in, size_t size, + uint32_t *outsize) +{ + ssize_t ret; + + ret = data->cmp->do_block(data->cmp, in, size, data->scratch, + data->super->block_size); + if (ret < 0) + return -1; + + if (ret > 0 && (size_t)ret < size) { + size = ret; + ret = write_retry(data->outfd, data->scratch, size); + *outsize = size; + } else { + ret = write_retry(data->outfd, in, size); + *outsize = size | (1 << 24); + } + + if (ret < 0) { + perror("writing to output file"); + return -1; + } + + if ((size_t)ret < size) { + fputs("write to output file truncated\n", stderr); + return -1; + } + + data->super->bytes_used += ret; + return 0; +} + +static int grow_fragment_table(data_writer_t *data) +{ + size_t newsz; + void *new; + + if (data->num_fragments == data->max_fragments) { + newsz = data->max_fragments ? data->max_fragments * 2 : 16; + new = realloc(data->fragments, + sizeof(data->fragments[0]) * newsz); + + if (new == NULL) { + perror("appending to fragment table"); + return -1; + } + + data->max_fragments = newsz; + data->fragments = new; + } + + return 0; +} + +int data_writer_flush_fragments(data_writer_t *data) +{ + uint64_t offset; + uint32_t out; + + if (grow_fragment_table(data)) + return -1; + + offset = data->super->bytes_used; + + if (write_compressed(data, data->fragment, data->frag_offset, &out)) + return -1; + + data->fragments[data->num_fragments].start_offset = htole64(offset); + data->fragments[data->num_fragments].pad0 = 0; + data->fragments[data->num_fragments].size = htole32(out); + + data->num_fragments += 1; + data->frag_offset = 0; + + data->super->flags &= ~SQFS_FLAG_NO_FRAGMENTS; + data->super->flags |= SQFS_FLAG_ALWAYS_FRAGMENTS; + return 0; +} + +int write_data_from_fd(data_writer_t *data, file_info_t *fi, int infd) +{ + uint64_t count = fi->size; + int blk_idx = 0; + uint32_t out; + ssize_t ret; + size_t diff; + + fi->startblock = data->super->bytes_used; + + while (count != 0) { + diff = count > (uint64_t)data->super->block_size ? + data->super->block_size : count; + + ret = read_retry(infd, data->block, diff); + if (ret < 0) + goto fail_read; + if ((size_t)ret < diff) + goto fail_trunc; + + if (diff < data->super->block_size) { + if (data->frag_offset + diff > data->super->block_size) { + if (data_writer_flush_fragments(data)) + return -1; + } + + fi->fragment_offset = data->frag_offset; + fi->fragment = data->num_fragments; + + memcpy((char *)data->fragment + data->frag_offset, + data->block, diff); + data->frag_offset += diff; + } else { + if (write_compressed(data, data->block, + data->super->block_size, &out)) { + return -1; + } + + fi->blocksizes[blk_idx++] = out; + } + + count -= diff; + } + + return 0; +fail_read: + fprintf(stderr, "read from %s: %s\n", fi->input_file, strerror(errno)); + return -1; +fail_trunc: + fprintf(stderr, "%s: truncated read\n", fi->input_file); + return -1; +} + +data_writer_t *data_writer_create(sqfs_super_t *super, compressor_t *cmp, + int outfd) +{ + data_writer_t *data; + + data = calloc(1, sizeof(*data) + super->block_size * 3); + if (data == NULL) { + perror("creating data writer"); + return NULL; + } + + data->block = (char *)data + sizeof(*data); + data->fragment = (char *)data->block + super->block_size; + data->scratch = (char *)data->fragment + super->block_size; + + data->super = super; + data->cmp = cmp; + data->outfd = outfd; + return data; +} + +void data_writer_destroy(data_writer_t *data) +{ + free(data->fragments); + free(data); +} + +int data_writer_write_fragment_table(data_writer_t *data) +{ + uint64_t start; + + data->super->fragment_entry_count = data->num_fragments; + + if (sqfs_write_table(data->outfd, data->super, data->fragments, + sizeof(data->fragments[0]), data->num_fragments, + &start, data->cmp)) { + return -1; + } + + data->super->fragment_table_start = start; + return 0; +} diff --git a/mkfs/block.c b/mkfs/block.c index 4bc9684..89bb689 100644 --- a/mkfs/block.c +++ b/mkfs/block.c @@ -2,197 +2,6 @@ #include "mkfs.h" #include "util.h" -struct data_writer_t { - void *block; - void *fragment; - void *scratch; - - sqfs_fragment_t *fragments; - size_t num_fragments; - size_t max_fragments; - size_t frag_offset; - - sqfs_super_t *super; - compressor_t *cmp; - int outfd; -}; - -static int write_compressed(data_writer_t *data, const void *in, size_t size, - uint32_t *outsize) -{ - ssize_t ret; - - ret = data->cmp->do_block(data->cmp, in, size, data->scratch, - data->super->block_size); - if (ret < 0) - return -1; - - if (ret > 0 && (size_t)ret < size) { - size = ret; - ret = write_retry(data->outfd, data->scratch, size); - *outsize = size; - } else { - ret = write_retry(data->outfd, in, size); - *outsize = size | (1 << 24); - } - - if (ret < 0) { - perror("writing to output file"); - return -1; - } - - if ((size_t)ret < size) { - fputs("write to output file truncated\n", stderr); - return -1; - } - - data->super->bytes_used += ret; - return 0; -} - -static int grow_fragment_table(data_writer_t *data) -{ - size_t newsz; - void *new; - - if (data->num_fragments == data->max_fragments) { - newsz = data->max_fragments ? data->max_fragments * 2 : 16; - new = realloc(data->fragments, - sizeof(data->fragments[0]) * newsz); - - if (new == NULL) { - perror("appending to fragment table"); - return -1; - } - - data->max_fragments = newsz; - data->fragments = new; - } - - return 0; -} - -int data_writer_flush_fragments(data_writer_t *data) -{ - uint64_t offset; - uint32_t out; - - if (grow_fragment_table(data)) - return -1; - - offset = data->super->bytes_used; - - if (write_compressed(data, data->fragment, data->frag_offset, &out)) - return -1; - - data->fragments[data->num_fragments].start_offset = htole64(offset); - data->fragments[data->num_fragments].pad0 = 0; - data->fragments[data->num_fragments].size = htole32(out); - - data->num_fragments += 1; - data->frag_offset = 0; - - data->super->flags &= ~SQFS_FLAG_NO_FRAGMENTS; - data->super->flags |= SQFS_FLAG_ALWAYS_FRAGMENTS; - return 0; -} - -int write_data_from_fd(data_writer_t *data, file_info_t *fi, int infd) -{ - uint64_t count = fi->size; - int blk_idx = 0; - uint32_t out; - ssize_t ret; - size_t diff; - - fi->startblock = data->super->bytes_used; - - while (count != 0) { - diff = count > (uint64_t)data->super->block_size ? - data->super->block_size : count; - - ret = read_retry(infd, data->block, diff); - if (ret < 0) - goto fail_read; - if ((size_t)ret < diff) - goto fail_trunc; - - if (diff < data->super->block_size) { - if (data->frag_offset + diff > data->super->block_size) { - if (data_writer_flush_fragments(data)) - return -1; - } - - fi->fragment_offset = data->frag_offset; - fi->fragment = data->num_fragments; - - memcpy((char *)data->fragment + data->frag_offset, - data->block, diff); - data->frag_offset += diff; - } else { - if (write_compressed(data, data->block, - data->super->block_size, &out)) { - return -1; - } - - fi->blocksizes[blk_idx++] = out; - } - - count -= diff; - } - - return 0; -fail_read: - fprintf(stderr, "read from %s: %s\n", fi->input_file, strerror(errno)); - return -1; -fail_trunc: - fprintf(stderr, "%s: truncated read\n", fi->input_file); - return -1; -} - -data_writer_t *data_writer_create(sqfs_super_t *super, compressor_t *cmp, - int outfd) -{ - data_writer_t *data; - - data = calloc(1, sizeof(*data) + super->block_size * 3); - if (data == NULL) { - perror("creating data writer"); - return NULL; - } - - data->block = (char *)data + sizeof(*data); - data->fragment = (char *)data->block + super->block_size; - data->scratch = (char *)data->fragment + super->block_size; - - data->super = super; - data->cmp = cmp; - data->outfd = outfd; - return data; -} - -void data_writer_destroy(data_writer_t *data) -{ - free(data->fragments); - free(data); -} - -int data_writer_write_fragment_table(data_writer_t *data) -{ - uint64_t start; - - data->super->fragment_entry_count = data->num_fragments; - - if (sqfs_write_table(data->outfd, data->super, data->fragments, - sizeof(data->fragments[0]), data->num_fragments, - &start, data->cmp)) { - return -1; - } - - data->super->fragment_table_start = start; - return 0; -} - static int process_file(data_writer_t *data, file_info_t *fi) { int ret, infd; diff --git a/mkfs/mkfs.h b/mkfs/mkfs.h index 63bfc10..53552fa 100644 --- a/mkfs/mkfs.h +++ b/mkfs/mkfs.h @@ -3,6 +3,7 @@ #define MKFS_H #include "meta_writer.h" +#include "data_writer.h" #include "highlevel.h" #include "squashfs.h" #include "compress.h" @@ -35,8 +36,6 @@ typedef struct { char *comp_extra; } options_t; -typedef struct data_writer_t data_writer_t; - typedef struct { int outfd; options_t opt; @@ -52,17 +51,6 @@ void process_command_line(options_t *opt, int argc, char **argv); int write_xattr(sqfs_info_t *info); -data_writer_t *data_writer_create(sqfs_super_t *super, compressor_t *cmp, - int outfd); - -void data_writer_destroy(data_writer_t *data); - -int data_writer_write_fragment_table(data_writer_t *data); - -int write_data_from_fd(data_writer_t *data, file_info_t *fi, int infd); - -int data_writer_flush_fragments(data_writer_t *data); - int write_data_to_image(data_writer_t *data, sqfs_info_t *info); #endif /* MKFS_H */ -- cgit v1.2.3