diff options
| -rw-r--r-- | include/data_writer.h | 93 | ||||
| -rw-r--r-- | include/highlevel.h | 36 | ||||
| -rw-r--r-- | lib/sqfshelper/Makemodule.am | 2 | ||||
| -rw-r--r-- | lib/sqfshelper/data_writer.c | 142 | ||||
| -rw-r--r-- | lib/sqfshelper/statistics.c | 57 | ||||
| -rw-r--r-- | mkfs/mkfs.c | 29 | ||||
| -rw-r--r-- | mkfs/mkfs.h | 1 | ||||
| -rw-r--r-- | tar/tar2sqfs.c | 35 |
8 files changed, 142 insertions, 253 deletions
diff --git a/include/data_writer.h b/include/data_writer.h deleted file mode 100644 index cdf7a44..0000000 --- a/include/data_writer.h +++ /dev/null @@ -1,93 +0,0 @@ -/* SPDX-License-Identifier: GPL-3.0-or-later */ -/* - * data_writer.h - * - * Copyright (C) 2019 David Oberhollenzer <goliath@infraroot.at> - */ -#ifndef DATA_WRITER_H -#define DATA_WRITER_H - -#include "config.h" - -#include "sqfs/compress.h" -#include "sqfs/data.h" -#include "fstree.h" -#include "util.h" - -typedef struct data_writer_t data_writer_t; - -typedef struct { - size_t file_count; - size_t blocks_written; - size_t frag_blocks_written; - size_t duplicate_blocks; - size_t sparse_blocks; - size_t frag_count; - size_t frag_dup; - uint64_t bytes_written; - uint64_t bytes_read; -} data_writer_stats_t; - -enum { - /* Don't generate fragments, always write the last block to disk as a - block, even if it is incomplete. */ - DW_DONT_FRAGMENT = 0x01, - - /* Intentionally write all blocks uncompressed. */ - DW_DONT_COMPRESS = 0x02, - - /* Make sure the first block of a file is alligned to - device block size */ - DW_ALLIGN_DEVBLK = 0x04, -}; - -/* - Create a data writer. The pointer to the super block is kept internally and - used to automatically update various counters when writing data. - - Returns NULL on failure and prints errors to stderr. - */ -data_writer_t *data_writer_create(sqfs_super_t *super, sqfs_compressor_t *cmp, - sqfs_file_t *file, size_t devblksize, - unsigned int num_jobs, size_t max_backlog); - -void data_writer_destroy(data_writer_t *data); - -/* - Write the finalfragment table to the underlying file. - - Returns 0 on success, prints errors to stderr. -*/ -int data_writer_write_fragment_table(data_writer_t *data); - -/* - Wait for everything to be written to disk. This also forces a currently - pending fragment block to be compressed and wrtten. - - Returns 0 on success, prints errors to stderr. -*/ -int data_writer_sync(data_writer_t *data); - -/* - Read data from the given file descriptor, partition it into blocks and - write them out (possibly compressed) to the underlying file. If the size - is not a multiple of the block size, the last bit is kept in an internal - fragment buffer which is written out if full. - - The file_info_t object is updated accordingly and used to determine the - number of bytes to write and the input file name to report errors. - - Blocks or fragments that are all zero bytes automatically detected, - not written out and the sparse file accounting updated accordingly. - - The flags argument is a combination of DW_* flags. After completion the - data writer collects the 'fi' in an internal list it uses for deduplication. - - Returns 0 on success, prints errors to stderr. -*/ -int write_data_from_file(data_writer_t *data, sqfs_inode_generic_t *inode, - sqfs_file_t *file, int flags); - -data_writer_stats_t *data_writer_get_stats(data_writer_t *data); - -#endif /* DATA_WRITER_H */ diff --git a/include/highlevel.h b/include/highlevel.h index bfc2d91..5bc4a96 100644 --- a/include/highlevel.h +++ b/include/highlevel.h @@ -15,20 +15,47 @@ #include "sqfs/data.h" #include "sqfs/table.h" #include "sqfs/meta_writer.h" +#include "sqfs/data_reader.h" +#include "sqfs/data_writer.h" #include "sqfs/dir_writer.h" #include "sqfs/dir_reader.h" +#include "sqfs/block.h" #include "sqfs/xattr.h" #include "sqfs/dir.h" #include "sqfs/io.h" -#include "sqfs/data_reader.h" -#include "data_writer.h" #include "fstree.h" +#include "util.h" #include "tar.h" #include <sys/stat.h> #include <stdint.h> #include <stddef.h> +typedef struct { + size_t file_count; + size_t blocks_written; + size_t frag_blocks_written; + size_t duplicate_blocks; + size_t sparse_blocks; + size_t frag_count; + size_t frag_dup; + uint64_t bytes_written; + uint64_t bytes_read; +} data_writer_stats_t; + +enum { + /* Don't generate fragments, always write the last block to disk as a + block, even if it is incomplete. */ + DW_DONT_FRAGMENT = 0x01, + + /* Intentionally write all blocks uncompressed. */ + DW_DONT_COMPRESS = 0x02, + + /* Make sure the first block of a file is alligned to + device block size */ + DW_ALLIGN_DEVBLK = 0x04, +}; + /* High level helper function to serialize an entire file system tree to a squashfs inode table and directory table. @@ -86,4 +113,9 @@ int sqfs_data_reader_dump(sqfs_data_reader_t *data, sqfs_file_t *sqfs_get_stdin_file(const sparse_map_t *map, uint64_t size); +void register_stat_hooks(sqfs_data_writer_t *data, data_writer_stats_t *stats); + +int write_data_from_file(sqfs_data_writer_t *data, sqfs_inode_generic_t *inode, + sqfs_file_t *file, size_t block_size, int flags); + #endif /* HIGHLEVEL_H */ diff --git a/lib/sqfshelper/Makemodule.am b/lib/sqfshelper/Makemodule.am index 8a8deaa..896605a 100644 --- a/lib/sqfshelper/Makemodule.am +++ b/lib/sqfshelper/Makemodule.am @@ -6,7 +6,7 @@ libsqfshelper_a_SOURCES += lib/sqfshelper/print_version.c libsqfshelper_a_SOURCES += lib/sqfshelper/inode_stat.c libsqfshelper_a_SOURCES += lib/sqfshelper/data_reader_dump.c libsqfshelper_a_SOURCES += lib/sqfshelper/compress.c lib/sqfshelper/comp_opt.c -libsqfshelper_a_SOURCES += include/data_writer.h lib/sqfshelper/data_writer.c +libsqfshelper_a_SOURCES += lib/sqfshelper/data_writer.c libsqfshelper_a_SOURCES += lib/sqfshelper/write_xattr.c include/highlevel.h libsqfshelper_a_SOURCES += lib/sqfshelper/get_path.c lib/sqfshelper/io_stdin.c diff --git a/lib/sqfshelper/data_writer.c b/lib/sqfshelper/data_writer.c index a365e6d..5c01cb8 100644 --- a/lib/sqfshelper/data_writer.c +++ b/lib/sqfshelper/data_writer.c @@ -6,10 +6,6 @@ */ #include "config.h" -#include "sqfs/data_writer.h" -#include "sqfs/block.h" - -#include "data_writer.h" #include "highlevel.h" #include "util.h" @@ -17,72 +13,14 @@ #include <string.h> #include <unistd.h> #include <stdio.h> -#include <errno.h> -#include <zlib.h> - -struct data_writer_t { - sqfs_data_writer_t *proc; - sqfs_compressor_t *cmp; - sqfs_super_t *super; - - data_writer_stats_t stats; -}; - - -static void post_block_write(void *user, const sqfs_block_t *block, - sqfs_file_t *file) -{ - data_writer_t *data = user; - (void)file; - - if (block->flags & SQFS_BLK_FRAGMENT_BLOCK) { - data->stats.frag_blocks_written += 1; - } else { - data->stats.blocks_written += 1; - } - - data->stats.bytes_written += block->size; -} - -static void pre_fragment_store(void *user, sqfs_block_t *block) -{ - data_writer_t *data = user; - (void)block; - - data->stats.frag_count += 1; -} - -static void notify_blocks_erased(void *user, size_t count, uint64_t bytes) -{ - data_writer_t *data = user; - (void)bytes; - - data->stats.bytes_written -= bytes; - data->stats.blocks_written -= count; - data->stats.duplicate_blocks += count; -} - -static void notify_fragment_discard(void *user, const sqfs_block_t *block) -{ - data_writer_t *data = user; - (void)block; - - data->stats.frag_dup += 1; -} - -static const sqfs_block_hooks_t hooks = { - .post_block_write = post_block_write, - .pre_fragment_store = pre_fragment_store, - .notify_blocks_erased = notify_blocks_erased, - .notify_fragment_discard = notify_fragment_discard, -}; static bool is_zero_block(unsigned char *ptr, size_t size) { return ptr[0] == 0 && memcmp(ptr, ptr + 1, size - 1) == 0; } -static int add_sentinel_block(data_writer_t *data, sqfs_inode_generic_t *inode, +static int add_sentinel_block(sqfs_data_writer_t *data, + sqfs_inode_generic_t *inode, uint32_t flags) { sqfs_block_t *blk = calloc(1, sizeof(*blk)); @@ -95,11 +33,11 @@ static int add_sentinel_block(data_writer_t *data, sqfs_inode_generic_t *inode, blk->inode = inode; blk->flags = flags; - return sqfs_data_writer_enqueue(data->proc, blk); + return sqfs_data_writer_enqueue(data, blk); } -int write_data_from_file(data_writer_t *data, sqfs_inode_generic_t *inode, - sqfs_file_t *file, int flags) +int write_data_from_file(sqfs_data_writer_t *data, sqfs_inode_generic_t *inode, + sqfs_file_t *file, size_t block_size, int flags) { uint32_t blk_flags = SQFS_BLK_FIRST_BLOCK; uint64_t filesz, offset; @@ -116,8 +54,8 @@ int write_data_from_file(data_writer_t *data, sqfs_inode_generic_t *inode, sqfs_inode_get_file_size(inode, &filesz); for (offset = 0; offset < filesz; offset += diff) { - if (filesz - offset > (uint64_t)data->super->block_size) { - diff = data->super->block_size; + if (filesz - offset > (uint64_t)block_size) { + diff = block_size; } else { diff = filesz - offset; } @@ -131,8 +69,6 @@ int write_data_from_file(data_writer_t *data, sqfs_inode_generic_t *inode, blk->index = i++; if (is_zero_block(blk->data, blk->size)) { - data->stats.sparse_blocks += 1; - sqfs_inode_make_extended(inode); inode->data.file_ext.sparse += blk->size; inode->num_file_blocks += 1; @@ -142,8 +78,7 @@ int write_data_from_file(data_writer_t *data, sqfs_inode_generic_t *inode, continue; } - if (diff < data->super->block_size && - !(flags & DW_DONT_FRAGMENT)) { + if (diff < block_size && !(flags & DW_DONT_FRAGMENT)) { if (!(blk_flags & (SQFS_BLK_FIRST_BLOCK | SQFS_BLK_LAST_BLOCK))) { blk_flags |= SQFS_BLK_LAST_BLOCK; @@ -157,10 +92,10 @@ int write_data_from_file(data_writer_t *data, sqfs_inode_generic_t *inode, blk->flags |= SQFS_BLK_IS_FRAGMENT; - if (sqfs_data_writer_enqueue(data->proc, blk)) + if (sqfs_data_writer_enqueue(data, blk)) return -1; } else { - if (sqfs_data_writer_enqueue(data->proc, blk)) + if (sqfs_data_writer_enqueue(data, blk)) return -1; blk_flags &= ~SQFS_BLK_FIRST_BLOCK; @@ -174,62 +109,5 @@ int write_data_from_file(data_writer_t *data, sqfs_inode_generic_t *inode, if (add_sentinel_block(data, inode, blk_flags)) return -1; } - - sqfs_inode_make_basic(inode); - - data->stats.bytes_read += filesz; - data->stats.file_count += 1; - return 0; -} - -data_writer_t *data_writer_create(sqfs_super_t *super, sqfs_compressor_t *cmp, - sqfs_file_t *file, size_t devblksize, - unsigned int num_jobs, size_t max_backlog) -{ - data_writer_t *data = calloc(1, sizeof(*data)); - - if (data == NULL) { - perror("creating data writer"); - return NULL; - } - - data->proc = sqfs_data_writer_create(super->block_size, cmp, - num_jobs, max_backlog, - devblksize, file); - if (data->proc == NULL) { - perror("creating data block processor"); - free(data); - return NULL; - } - - sqfs_data_writer_set_hooks(data->proc, data, &hooks); - - data->cmp = cmp; - data->super = super; - return data; -} - -void data_writer_destroy(data_writer_t *data) -{ - sqfs_data_writer_destroy(data->proc); - free(data); -} - -int data_writer_write_fragment_table(data_writer_t *data) -{ - if (sqfs_data_writer_write_fragment_table(data->proc, data->super)) { - fputs("error storing fragment table\n", stderr); - return -1; - } return 0; } - -int data_writer_sync(data_writer_t *data) -{ - return sqfs_data_writer_finish(data->proc); -} - -data_writer_stats_t *data_writer_get_stats(data_writer_t *data) -{ - return &data->stats; -} diff --git a/lib/sqfshelper/statistics.c b/lib/sqfshelper/statistics.c index 87efc30..26c121f 100644 --- a/lib/sqfshelper/statistics.c +++ b/lib/sqfshelper/statistics.c @@ -7,8 +7,65 @@ #include "config.h" #include "highlevel.h" +#include "sqfs/block.h" + #include <stdio.h> +static void post_block_write(void *user, const sqfs_block_t *block, + sqfs_file_t *file) +{ + data_writer_stats_t *stats = user; + (void)file; + + if (block->size == 0) + return; + + if (block->flags & SQFS_BLK_FRAGMENT_BLOCK) { + stats->frag_blocks_written += 1; + } else { + stats->blocks_written += 1; + } + + stats->bytes_written += block->size; +} + +static void pre_fragment_store(void *user, sqfs_block_t *block) +{ + data_writer_stats_t *stats = user; + (void)block; + + stats->frag_count += 1; +} + +static void notify_blocks_erased(void *user, size_t count, uint64_t bytes) +{ + data_writer_stats_t *stats = user; + + stats->bytes_written -= bytes; + stats->blocks_written -= count; + stats->duplicate_blocks += count; +} + +static void notify_fragment_discard(void *user, const sqfs_block_t *block) +{ + data_writer_stats_t *stats = user; + (void)block; + + stats->frag_dup += 1; +} + +static const sqfs_block_hooks_t hooks = { + .post_block_write = post_block_write, + .pre_fragment_store = pre_fragment_store, + .notify_blocks_erased = notify_blocks_erased, + .notify_fragment_discard = notify_fragment_discard, +}; + +void register_stat_hooks(sqfs_data_writer_t *data, data_writer_stats_t *stats) +{ + sqfs_data_writer_set_hooks(data, stats, &hooks); +} + void sqfs_print_statistics(sqfs_super_t *super, data_writer_stats_t *stats) { size_t ratio; diff --git a/mkfs/mkfs.c b/mkfs/mkfs.c index acb3202..71e1863 100644 --- a/mkfs/mkfs.c +++ b/mkfs/mkfs.c @@ -28,7 +28,8 @@ static int restore_working_dir(options_t *opt) return 0; } -static int pack_files(data_writer_t *data, fstree_t *fs, options_t *opt) +static int pack_files(sqfs_data_writer_t *data, fstree_t *fs, + data_writer_stats_t *stats, options_t *opt) { sqfs_inode_generic_t *inode; size_t max_blk_count; @@ -72,14 +73,18 @@ static int pack_files(data_writer_t *data, fstree_t *fs, options_t *opt) fi->user_ptr = inode; - ret = write_data_from_file(data, inode, file, 0); + ret = write_data_from_file(data, inode, file, + fs->block_size, 0); file->destroy(file); if (ret) return -1; + + stats->file_count += 1; + stats->bytes_read += filesize; } - if (data_writer_sync(data)) + if (sqfs_data_writer_finish(data)) return -1; return restore_working_dir(opt); @@ -118,10 +123,11 @@ int main(int argc, char **argv) { int status = EXIT_FAILURE, ret; sqfs_compressor_config_t cfg; + data_writer_stats_t stats; + sqfs_data_writer_t *data; sqfs_compressor_t *cmp; sqfs_id_table_t *idtbl; sqfs_file_t *outfile; - data_writer_t *data; sqfs_super_t super; options_t opt; fstree_t fs; @@ -188,18 +194,21 @@ int main(int argc, char **argv) if (ret > 0) super.flags |= SQFS_FLAG_COMPRESSOR_OPTIONS; - data = data_writer_create(&super, cmp, outfile, - opt.devblksz, opt.num_jobs, opt.max_backlog); + data = sqfs_data_writer_create(super.block_size, cmp, opt.num_jobs, + opt.max_backlog, opt.devblksz, outfile); if (data == NULL) goto out_cmp; - if (pack_files(data, &fs, &opt)) + memset(&stats, 0, sizeof(stats)); + register_stat_hooks(data, &stats); + + if (pack_files(data, &fs, &stats, &opt)) goto out_data; if (sqfs_serialize_fstree(outfile, &super, &fs, cmp, idtbl)) goto out_data; - if (data_writer_write_fragment_table(data)) + if (sqfs_data_writer_write_fragment_table(data, &super)) goto out_data; if (opt.exportable) { @@ -222,11 +231,11 @@ int main(int argc, char **argv) goto out_data; if (!opt.quiet) - sqfs_print_statistics(&super, data_writer_get_stats(data)); + sqfs_print_statistics(&super, &stats); status = EXIT_SUCCESS; out_data: - data_writer_destroy(data); + sqfs_data_writer_destroy(data); out_cmp: cmp->destroy(cmp); out_outfile: diff --git a/mkfs/mkfs.h b/mkfs/mkfs.h index 9d34d02..bbe5572 100644 --- a/mkfs/mkfs.h +++ b/mkfs/mkfs.h @@ -15,7 +15,6 @@ #include "sqfs/data.h" #include "sqfs/io.h" -#include "data_writer.h" #include "highlevel.h" #include "fstree.h" #include "util.h" diff --git a/tar/tar2sqfs.c b/tar/tar2sqfs.c index 82720a8..4770810 100644 --- a/tar/tar2sqfs.c +++ b/tar/tar2sqfs.c @@ -12,7 +12,6 @@ #include "sqfs/data.h" #include "sqfs/io.h" -#include "data_writer.h" #include "highlevel.h" #include "fstree.h" #include "util.h" @@ -111,6 +110,8 @@ static bool dont_skip = false; static bool no_xattr = false; static bool exportable = false; static bool keep_time = false; +static data_writer_stats_t stats; +static sqfs_super_t super; static void process_args(int argc, char **argv) { @@ -223,7 +224,7 @@ fail_arg: } static int write_file(tar_header_decoded_t *hdr, file_info_t *fi, - data_writer_t *data, uint64_t filesize) + sqfs_data_writer_t *data, uint64_t filesize) { const sparse_map_t *it; sqfs_inode_generic_t *inode; @@ -266,9 +267,12 @@ static int write_file(tar_header_decoded_t *hdr, file_info_t *fi, } } - ret = write_data_from_file(data, inode, file, 0); + ret = write_data_from_file(data, inode, file, super.block_size, 0); file->destroy(file); + stats.bytes_read += filesize; + stats.file_count += 1; + if (ret) return -1; @@ -302,7 +306,7 @@ static int copy_xattr(fstree_t *fs, tree_node_t *node, } static int create_node_and_repack_data(tar_header_decoded_t *hdr, fstree_t *fs, - data_writer_t *data) + sqfs_data_writer_t *data) { tree_node_t *node; @@ -333,7 +337,7 @@ fail_errno: return -1; } -static int process_tar_ball(fstree_t *fs, data_writer_t *data) +static int process_tar_ball(fstree_t *fs, sqfs_data_writer_t *data) { tar_header_decoded_t hdr; uint64_t offset, count; @@ -407,11 +411,10 @@ int main(int argc, char **argv) { sqfs_compressor_config_t cfg; int status = EXIT_SUCCESS; + sqfs_data_writer_t *data; sqfs_compressor_t *cmp; sqfs_id_table_t *idtbl; sqfs_file_t *outfile; - data_writer_t *data; - sqfs_super_t super; fstree_t fs; int ret; @@ -450,10 +453,14 @@ int main(int argc, char **argv) if (ret > 0) super.flags |= SQFS_FLAG_COMPRESSOR_OPTIONS; - data = data_writer_create(&super, cmp, outfile, devblksize, - num_jobs, max_backlog); - if (data == NULL) + data = sqfs_data_writer_create(super.block_size, cmp, num_jobs, + max_backlog, devblksize, outfile); + if (data == NULL) { + perror("creating data block processor"); goto out_cmp; + } + + register_stat_hooks(data, &stats); idtbl = sqfs_id_table_create(); if (idtbl == NULL) @@ -462,7 +469,7 @@ int main(int argc, char **argv) if (process_tar_ball(&fs, data)) goto out; - if (data_writer_sync(data)) + if (sqfs_data_writer_finish(data)) goto out; tree_node_sort_recursive(fs.root); @@ -476,7 +483,7 @@ int main(int argc, char **argv) if (sqfs_serialize_fstree(outfile, &super, &fs, cmp, idtbl)) goto out; - if (data_writer_write_fragment_table(data)) + if (sqfs_data_writer_write_fragment_table(data, &super)) goto out; if (exportable) { @@ -499,13 +506,13 @@ int main(int argc, char **argv) goto out; if (!quiet) - sqfs_print_statistics(&super, data_writer_get_stats(data)); + sqfs_print_statistics(&super, &stats); status = EXIT_SUCCESS; out: sqfs_id_table_destroy(idtbl); out_data: - data_writer_destroy(data); + sqfs_data_writer_destroy(data); out_cmp: cmp->destroy(cmp); out_fs: |