From 14a925f6da442ecade7df75eb46a6edb9a1499af Mon Sep 17 00:00:00 2001 From: David Oberhollenzer Date: Mon, 15 Jul 2019 14:44:44 +0200 Subject: Add flags to data writer to micro manage behaviour The added flags allow controlling the following on a per file level: - forcing a file to be written uncompressed - forcing a file to not have a fragment, i.e. the last truncated block actually being written as a block - padding a file to be alligned to device block size The flags are not yet exposed to anything user controllable (such as command line flags). Signed-off-by: David Oberhollenzer --- include/data_writer.h | 27 +++++++++++++++++++--- lib/sqfs/data_writer.c | 63 ++++++++++++++++++++++++++++++++++++++------------ mkfs/mkfs.c | 4 ++-- tar/tar2sqfs.c | 6 ++--- 4 files changed, 77 insertions(+), 23 deletions(-) diff --git a/include/data_writer.h b/include/data_writer.h index 2ace899..a52b37f 100644 --- a/include/data_writer.h +++ b/include/data_writer.h @@ -9,6 +9,22 @@ typedef struct data_writer_t data_writer_t; +enum { + /* Don't generate fragments, always write the last block to disk as a + block, even if it is incomplete. */ + DW_DONT_FRAGMENT = 0x01, + + /* Intentionally write all blocks uncompressed. This implies + DW_DONT_FRAGMENT since sharing a fragment block with other files + would otherwise require the entire fragment block to be + uncompressed. */ + DW_DONT_COMPRESS = 0x03, + + /* Make sure the first block of a file is alligned to + device block size */ + DW_ALLIGN_DEVBLK = 0x04, +}; + /* Create a data writer. The pointer to the super block is kept internally and used to automatically update various counters when writing data. @@ -16,7 +32,7 @@ typedef struct data_writer_t data_writer_t; Returns NULL on failure and prints errors to stderr. */ data_writer_t *data_writer_create(sqfs_super_t *super, compressor_t *cmp, - int outfd); + int outfd, size_t devblksize); void data_writer_destroy(data_writer_t *data); @@ -46,18 +62,23 @@ int data_writer_flush_fragments(data_writer_t *data); Blocks or fragments that are all zero bytes automatically detected, not written out and the sparse file accounting updated accordingly. + The flags argument is a combination of DW_* flags. + Returns 0 on success, prints errors to stderr. */ -int write_data_from_fd(data_writer_t *data, file_info_t *fi, int infd); +int write_data_from_fd(data_writer_t *data, file_info_t *fi, int infd, + int flags); /* Does the same as write_data_from_fd but the input file is the condensed representation of a sparse file. The layout must be in order and non-overlapping. + The flags argument is a combination of DW_* flags. + Returns 0 on success, prints errors to stderr. */ int write_data_from_fd_condensed(data_writer_t *data, file_info_t *fi, - int infd, sparse_map_t *map); + int infd, sparse_map_t *map, int flags); #endif /* DATA_WRITER_H */ diff --git a/lib/sqfs/data_writer.c b/lib/sqfs/data_writer.c index 557b85b..cbe2351 100644 --- a/lib/sqfs/data_writer.c +++ b/lib/sqfs/data_writer.c @@ -18,6 +18,8 @@ struct data_writer_t { size_t max_fragments; size_t frag_offset; + size_t devblksz; + int block_idx; sqfs_super_t *super; @@ -26,14 +28,16 @@ struct data_writer_t { }; static int write_compressed(data_writer_t *data, const void *in, size_t size, - uint32_t *outsize) + uint32_t *outsize, int flags) { - ssize_t ret; + ssize_t ret = 0; - ret = data->cmp->do_block(data->cmp, in, size, data->scratch, - data->super->block_size); - if (ret < 0) - return -1; + if (!(flags & DW_DONT_COMPRESS)) { + ret = data->cmp->do_block(data->cmp, in, size, data->scratch, + data->super->block_size); + if (ret < 0) + return -1; + } if (ret > 0 && (size_t)ret < size) { size = ret; @@ -85,6 +89,20 @@ static bool is_zero_block(unsigned char *ptr, size_t size) return ptr[0] == 0 && memcmp(ptr, ptr + 1, size - 1) == 0; } +static int allign_file(data_writer_t *data) +{ + size_t diff = data->super->bytes_used % data->devblksz; + + if (diff == 0) + return 0; + + if (padd_file(data->outfd, data->super->bytes_used, data->devblksz)) + return -1; + + data->super->bytes_used += data->devblksz - diff; + return 0; +} + int data_writer_flush_fragments(data_writer_t *data) { uint64_t offset; @@ -98,7 +116,7 @@ int data_writer_flush_fragments(data_writer_t *data) offset = data->super->bytes_used; - if (write_compressed(data, data->fragment, data->frag_offset, &out)) + if (write_compressed(data, data->fragment, data->frag_offset, &out, 0)) return -1; data->fragments[data->num_fragments].start_offset = htole64(offset); @@ -113,7 +131,8 @@ int data_writer_flush_fragments(data_writer_t *data) return 0; } -static int flush_data_block(data_writer_t *data, size_t size, file_info_t *fi) +static int flush_data_block(data_writer_t *data, size_t size, + file_info_t *fi, int flags) { uint32_t out; @@ -123,7 +142,7 @@ static int flush_data_block(data_writer_t *data, size_t size, file_info_t *fi) return 0; } - if (size < data->super->block_size) { + if (size < data->super->block_size && !(flags & DW_DONT_FRAGMENT)) { if (data->frag_offset + size > data->super->block_size) { if (data_writer_flush_fragments(data)) return -1; @@ -136,7 +155,7 @@ static int flush_data_block(data_writer_t *data, size_t size, file_info_t *fi) data->block, size); data->frag_offset += size; } else { - if (write_compressed(data, data->block, size, &out)) + if (write_compressed(data, data->block, size, &out, flags)) return -1; fi->blocksizes[data->block_idx++] = out; @@ -145,12 +164,16 @@ static int flush_data_block(data_writer_t *data, size_t size, file_info_t *fi) return 0; } -int write_data_from_fd(data_writer_t *data, file_info_t *fi, int infd) +int write_data_from_fd(data_writer_t *data, file_info_t *fi, + int infd, int flags) { uint64_t count; ssize_t ret; size_t diff; + if ((flags & DW_ALLIGN_DEVBLK) && allign_file(data) != 0) + return -1; + fi->startblock = data->super->bytes_used; fi->sparse = 0; data->block_idx = 0; @@ -165,10 +188,13 @@ int write_data_from_fd(data_writer_t *data, file_info_t *fi, int infd) if ((size_t)ret < diff) goto fail_trunc; - if (flush_data_block(data, diff, fi)) + if (flush_data_block(data, diff, fi, flags)) return -1; } + if ((flags & DW_ALLIGN_DEVBLK) && allign_file(data) != 0) + return -1; + return 0; fail_read: perror(fi->input_file); @@ -179,13 +205,16 @@ fail_trunc: } int write_data_from_fd_condensed(data_writer_t *data, file_info_t *fi, - int infd, sparse_map_t *map) + int infd, sparse_map_t *map, int flags) { size_t start, count, diff; sparse_map_t *m; uint64_t offset; ssize_t ret; + if ((flags & DW_ALLIGN_DEVBLK) && allign_file(data) != 0) + return -1; + fi->startblock = data->super->bytes_used; fi->sparse = 0; data->block_idx = 0; @@ -235,10 +264,13 @@ int write_data_from_fd_condensed(data_writer_t *data, file_info_t *fi, map = map->next; } - if (flush_data_block(data, diff, fi)) + if (flush_data_block(data, diff, fi, flags)) return -1; } + if ((flags & DW_ALLIGN_DEVBLK) && allign_file(data) != 0) + return -1; + return 0; fail_map_size: fprintf(stderr, "%s: sparse file map spans beyond file size\n", @@ -258,7 +290,7 @@ fail_trunc: } data_writer_t *data_writer_create(sqfs_super_t *super, compressor_t *cmp, - int outfd) + int outfd, size_t devblksize) { data_writer_t *data; @@ -275,6 +307,7 @@ data_writer_t *data_writer_create(sqfs_super_t *super, compressor_t *cmp, data->super = super; data->cmp = cmp; data->outfd = outfd; + data->devblksz = devblksize; return data; } diff --git a/mkfs/mkfs.c b/mkfs/mkfs.c index e72b669..f1c1f1b 100644 --- a/mkfs/mkfs.c +++ b/mkfs/mkfs.c @@ -18,7 +18,7 @@ static int process_file(data_writer_t *data, tree_node_t *n, bool quiet) return -1; } - ret = write_data_from_fd(data, n->data.file, infd); + ret = write_data_from_fd(data, n->data.file, infd, 0); close(infd); return ret; @@ -172,7 +172,7 @@ int main(int argc, char **argv) super.bytes_used += ret; } - data = data_writer_create(&super, cmp, outfd); + data = data_writer_create(&super, cmp, outfd, opt.devblksz); if (data == NULL) goto out_cmp; diff --git a/tar/tar2sqfs.c b/tar/tar2sqfs.c index 1b7d3aa..3417fd0 100644 --- a/tar/tar2sqfs.c +++ b/tar/tar2sqfs.c @@ -185,14 +185,14 @@ static int write_file(tar_header_decoded_t *hdr, file_info_t *fi, if (hdr->sparse != NULL) { ret = write_data_from_fd_condensed(data, fi, STDIN_FILENO, - hdr->sparse); + hdr->sparse, 0); if (ret) return -1; return skip_padding(STDIN_FILENO, hdr->record_size); } - if (write_data_from_fd(data, fi, STDIN_FILENO)) + if (write_data_from_fd(data, fi, STDIN_FILENO, 0)) return -1; return skip_padding(STDIN_FILENO, fi->size); @@ -361,7 +361,7 @@ int main(int argc, char **argv) super.bytes_used += ret; } - data = data_writer_create(&super, cmp, outfd); + data = data_writer_create(&super, cmp, outfd, devblksize); if (data == NULL) goto out_cmp; -- cgit v1.2.3