From 9d5b0c381a7961a14d2a94a6b31a4e25a2543eae Mon Sep 17 00:00:00 2001 From: David Oberhollenzer Date: Tue, 28 Jan 2020 21:23:21 +0100 Subject: Rename sqfs_data_writer_t back to sqfs_block_processor_t Signed-off-by: David Oberhollenzer --- include/common.h | 15 +- include/sqfs/block.h | 18 +- include/sqfs/block_processor.h | 312 +++++++++++++++++++++++ include/sqfs/data_writer.h | 310 ----------------------- include/sqfs/predef.h | 2 +- lib/common/data_writer.c | 8 +- lib/common/statistics.c | 15 +- lib/common/writer.c | 19 +- lib/sqfs/Makemodule.am | 19 +- lib/sqfs/block_processor/block.c | 203 +++++++++++++++ lib/sqfs/block_processor/common.c | 72 ++++++ lib/sqfs/block_processor/fileapi.c | 188 ++++++++++++++ lib/sqfs/block_processor/fragment.c | 81 ++++++ lib/sqfs/block_processor/internal.h | 141 +++++++++++ lib/sqfs/block_processor/serial.c | 88 +++++++ lib/sqfs/block_processor/winpthread.c | 450 ++++++++++++++++++++++++++++++++++ lib/sqfs/data_writer/block.c | 203 --------------- lib/sqfs/data_writer/common.c | 72 ------ lib/sqfs/data_writer/fileapi.c | 188 -------------- lib/sqfs/data_writer/fragment.c | 81 ------ lib/sqfs/data_writer/internal.h | 140 ----------- lib/sqfs/data_writer/serial.c | 88 ------- lib/sqfs/data_writer/winpthread.c | 450 ---------------------------------- mkfs/mkfs.c | 4 +- 24 files changed, 1587 insertions(+), 1580 deletions(-) create mode 100644 include/sqfs/block_processor.h delete mode 100644 include/sqfs/data_writer.h create mode 100644 lib/sqfs/block_processor/block.c create mode 100644 lib/sqfs/block_processor/common.c create mode 100644 lib/sqfs/block_processor/fileapi.c create mode 100644 lib/sqfs/block_processor/fragment.c create mode 100644 lib/sqfs/block_processor/internal.h create mode 100644 lib/sqfs/block_processor/serial.c create mode 100644 lib/sqfs/block_processor/winpthread.c delete mode 100644 lib/sqfs/data_writer/block.c delete mode 100644 lib/sqfs/data_writer/common.c delete mode 100644 lib/sqfs/data_writer/fileapi.c delete mode 100644 lib/sqfs/data_writer/fragment.c delete mode 100644 lib/sqfs/data_writer/internal.h delete mode 100644 lib/sqfs/data_writer/serial.c delete mode 100644 lib/sqfs/data_writer/winpthread.c diff --git a/include/common.h b/include/common.h index 386d8fd..f2431ab 100644 --- a/include/common.h +++ b/include/common.h @@ -18,7 +18,7 @@ #include "sqfs/error.h" #include "sqfs/meta_writer.h" #include "sqfs/data_reader.h" -#include "sqfs/data_writer.h" +#include "sqfs/block_processor.h" #include "sqfs/dir_writer.h" #include "sqfs/dir_reader.h" #include "sqfs/block.h" @@ -42,10 +42,10 @@ typedef struct { size_t frag_dup; sqfs_u64 bytes_written; sqfs_u64 bytes_read; -} data_writer_stats_t; +} block_processor_stats_t; typedef struct { - sqfs_data_writer_t *data; + sqfs_block_processor_t *data; sqfs_dir_writer_t *dirwr; sqfs_meta_writer_t *dm; sqfs_meta_writer_t *im; @@ -54,7 +54,7 @@ typedef struct { sqfs_file_t *outfile; sqfs_super_t super; fstree_t fs; - data_writer_stats_t stats; + block_processor_stats_t stats; sqfs_xattr_writer_t *xwr; } sqfs_writer_t; @@ -99,7 +99,7 @@ typedef struct sqfs_hard_link_t { int sqfs_serialize_fstree(const char *filename, sqfs_writer_t *wr); /* Print out fancy statistics for squashfs packing tools */ -void sqfs_print_statistics(sqfs_super_t *super, data_writer_stats_t *stats); +void sqfs_print_statistics(sqfs_super_t *super, block_processor_stats_t *stats); void compressor_print_available(void); @@ -122,9 +122,10 @@ int sqfs_data_reader_dump(const char *name, sqfs_data_reader_t *data, sqfs_file_t *sqfs_get_stdin_file(FILE *fp, const sparse_map_t *map, sqfs_u64 size); -void register_stat_hooks(sqfs_data_writer_t *data, data_writer_stats_t *stats); +void register_stat_hooks(sqfs_block_processor_t *data, + block_processor_stats_t *stats); -int write_data_from_file(const char *filename, sqfs_data_writer_t *data, +int write_data_from_file(const char *filename, sqfs_block_processor_t *data, sqfs_inode_generic_t *inode, sqfs_file_t *file, int flags); diff --git a/include/sqfs/block.h b/include/sqfs/block.h index b37cdd5..1098c96 100644 --- a/include/sqfs/block.h +++ b/include/sqfs/block.h @@ -68,14 +68,14 @@ typedef enum { * @brief Only calculate checksum, do NOT compress the data. * * If set, the blocks of a file will not be compressed by the - * @ref sqfs_data_writer_t. + * @ref sqfs_block_processor_t. */ SQFS_BLK_DONT_COMPRESS = 0x0001, /** * @brief Align the block on disk to device block size. * - * If set, the @ref sqfs_data_writer_t will add padding before the + * If set, the @ref sqfs_block_processor_t will add padding before the * first block of the affected file and after the last block. */ SQFS_BLK_ALIGN = 0x0002, @@ -83,38 +83,38 @@ typedef enum { /** * @brief Don't add the tail end of a file to a fragment block. * - * If set, the @ref sqfs_data_writer_t will always generate a final + * If set, the @ref sqfs_block_processor_t will always generate a final * block for a file, even if it is truncated. It will not add the * tail end to a fragment block. */ SQFS_BLK_DONT_FRAGMENT = 0x0004, /** - * @brief Set by the @ref sqfs_data_writer_t on the first + * @brief Set by the @ref sqfs_block_processor_t on the first * block of a file. */ SQFS_BLK_FIRST_BLOCK = 0x0800, /** - * @brief Set by the @ref sqfs_data_writer_t on the last + * @brief Set by the @ref sqfs_block_processor_t on the last * block of a file. */ SQFS_BLK_LAST_BLOCK = 0x1000, /** - * @brief Set by the @ref sqfs_data_writer_t to indicate that a block - * is a tail end of a file and the block. + * @brief Set by the @ref sqfs_block_processor_t to indicate that a + * block is a tail end of a file and the block. */ SQFS_BLK_IS_FRAGMENT = 0x2000, /** - * @brief Set by the @ref sqfs_data_writer_t on fragment blocks that + * @brief Set by the @ref sqfs_block_processor_t on fragment blocks that * it generates. */ SQFS_BLK_FRAGMENT_BLOCK = 0x4000, /** - * @brief Set by @ref sqfs_data_writer_t if the block was + * @brief Set by @ref sqfs_block_processor_t if the block was * actually compressed. */ SQFS_BLK_IS_COMPRESSED = 0x8000, diff --git a/include/sqfs/block_processor.h b/include/sqfs/block_processor.h new file mode 100644 index 0000000..c6dd734 --- /dev/null +++ b/include/sqfs/block_processor.h @@ -0,0 +1,312 @@ +/* SPDX-License-Identifier: LGPL-3.0-or-later */ +/* + * block_processor.h - This file is part of libsquashfs + * + * Copyright (C) 2019 David Oberhollenzer + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + */ +#ifndef SFQS_BLOCK_PROCESSOR_H +#define SFQS_BLOCK_PROCESSOR_H + +#include "sqfs/predef.h" + +/** + * @file block_processor.h + * + * @brief Contains declarations for the data block processor. + */ + +/** + * @struct sqfs_block_processor_t + * + * @brief Abstracts generating of file data and fragment blocks. + * + * This data structure provides a simple begin/append/end interface + * to generate file data blocks (see @ref sqfs_block_processor_begin_file, + * @ref sqfs_block_processor_append and @ref sqfs_block_processor_end + * respectively). + * + * Internally it takes care of partitioning data in the correct block sizes, + * adding tail-ens to fragment blocks, compressing the data, deduplicating data + * and finally writing it to disk. + */ + +/** + * @struct sqfs_block_hooks_t + * + * @brief A set of hooks for tapping into the data writer. + * + * This structure can be registered with an @ref sqfs_block_processor_t and + * contains function pointers that will be called during various stages + * when writing data to disk. + * + * The callbacks can not only be used for accounting but may also write extra + * data to the output file or make modifications to the blocks before they are + * writtien. + * + * The callbacks can be individually set to NULL to disable them. + */ +struct sqfs_block_hooks_t { + /** + * @brief Set this to the size of the struct. + * + * This is required for future expandabillity while maintaining ABI + * compatibillity. At the current time, the implementation of + * @ref sqfs_block_processor_set_hooks rejects any hook struct where + * this isn't the exact size. If new hooks are added in the future, + * the struct grows and the future implementation can tell by the size + * whether the application uses the new version or the old one. + */ + size_t size; + + /** + * @brief Gets called before writing a block to disk. + * + * If this is not NULL, it gets called before a block is written to + * disk. If the block has the @ref SQFS_BLK_ALIGN flag set, the + * function is called before padding the file. + * + * The function may modify the block itself or write data to the file. + * which is taken into account when padding the file. + * + * @param user A user pointer. + * @param block The block that is about to be written. + * @param file The file that the block will be written to. + */ + void (*pre_block_write)(void *user, sqfs_block_t *block, + sqfs_file_t *file); + + /** + * @brief Gets called after writing a block to disk. + * + * If this is not NULL, it gets called after a block is written to + * disk. If the block has the @ref SQFS_BLK_ALIGN flag set, the + * function is called before padding the file. + * + * Modifying the block is rather pointless, but the function may + * write data to the file which is taken into account when padding + * the file. + * + * @param user A user pointer. + * @param block The block that is about to be written. + * @param file The file that the block was written to. + */ + void (*post_block_write)(void *user, const sqfs_block_t *block, + sqfs_file_t *file); + + /** + * @brief Gets called before storing a fragment in a fragment block. + * + * The function can modify the block before it is stored. + * + * @param user A user pointer. + * @param block The data chunk that is about to be merged into the + * fragment block. + */ + void (*pre_fragment_store)(void *user, sqfs_block_t *block); + + /** + * @brief Gets called if block deduplication managed to get + * rid of the data blocks of a file. + * + * @param user A user pointer. + * @param count The number of blocks that have been erased. + * @param bytes The number of bytes that have been erased. Includes + * potential padding before and after the end. + */ + void (*notify_blocks_erased)(void *user, size_t count, + sqfs_u64 bytes); + + /** + * @brief Gets called before throwing away a fragment that turned out + * to be a duplicate. + * + * @param user A user pointer. + * @param block The data chunk that is about to be merged into the + * fragment block. + */ + void (*notify_fragment_discard)(void *user, const sqfs_block_t *block); + + /** + * @brief Gets called before writing a block of padding bytes to disk. + * + * @param user A user pointer. + * @param block The padding bytes that are about to be written. + * @param count The number of padding bytes in the block. + */ + void (*prepare_padding)(void *user, sqfs_u8 *block, size_t count); +}; + +#ifdef __cplusplus +extern "C" { +#endif + +/** + * @brief Create a data block writer. + * + * @memberof sqfs_block_processor_t + * + * @param max_block_size The maximum size of a data block. Required for the + * internal scratch buffer used for compressing data. + * @param cmp A pointer to a compressor. If multiple worker threads are used, + * the deep copy function of the compressor is used to create + * several instances that don't interfere with each other. + * @param num_workers The number of worker threads to create. + * @param max_backlog The maximum number of blocks currently in flight. When + * trying to add more, enqueueing blocks until the in-flight + * block count drops below the threshold. + * @param devblksz File can optionally be allgined to device block size. This + * specifies the desired alignment. + * @param file The output file to write the finished blocks to. + * + * @return A pointer to a data writer object on success, NULL on allocation + * failure or on failure to create and initialize the worker threads. + */ +SQFS_API +sqfs_block_processor_t *sqfs_block_processor_create(size_t max_block_size, + sqfs_compressor_t *cmp, + unsigned int num_workers, + size_t max_backlog, + size_t devblksz, + sqfs_file_t *file); + +/** + * @brief Destroy a data writer and free all memory used by it. + * + * @memberof sqfs_block_processor_t + * + * @param proc A pointer to a data writer object. + */ +SQFS_API void sqfs_block_processor_destroy(sqfs_block_processor_t *proc); + +/** + * @brief Start writing a file. + * + * @memberof sqfs_block_processor_t + * + * After calling this function, call @ref sqfs_block_processor_append + * repeatedly to add data to the file. Finally + * call @ref sqfs_block_processor_end_file when you + * are done. After writing all files, use @ref sqfs_block_processor_finish to + * wait until all blocks that are still in flight are done and written to disk. + * + * The specified inode pointer is kept internally and updated with the + * compressed block sizes and final destinations of the file and possible + * fragment. You need to make sure it has enough backing-store for all blocks + * to come. Furthermore, since there can still be blocks in-flight even after + * calling @ref sqfs_block_processor_end_file, the data in the inode may still + * change. The only point at which the data writer is guarnteed to not touch + * them anymore is after @ref sqfs_block_processor_finish has returned. + * + * @param proc A pointer to a data writer object. + * @param inode The regular file inode representing the file. The data writer + * internally updates it while writing blocks to disk. + * @param flags A combination of @ref E_SQFS_BLK_FLAGS that can be used to + * micro manage how the data is processed. + * + * @return Zero on success, an @ref E_SQFS_ERROR value on failure. + */ +SQFS_API int sqfs_block_processor_begin_file(sqfs_block_processor_t *proc, + sqfs_inode_generic_t *inode, + sqfs_u32 flags); + +/** + * @brief Append data to the current file. + * + * @memberof sqfs_block_processor_t + * + * Call this after @ref sqfs_block_processor_begin_file to add data to a file. + * + * @param proc A pointer to a data writer object. + * @param data A pointer to a buffer to read data from. + * @param size How many bytes should be copied out of the given + * buffer and written to disk. + * + * @return Zero on success, an @ref E_SQFS_ERROR value on failure. + */ +SQFS_API int sqfs_block_processor_append(sqfs_block_processor_t *proc, + const void *data, size_t size); + +/** + * @brief Stop writing the current file and flush everything that is + * buffered internally. + * + * @memberof sqfs_block_processor_t + * + * The counter part to @ref sqfs_block_processor_begin_file. + * + * Even after calling this, there might still be data blocks in-flight. + * Use @ref sqfs_block_processor_finish when you are done writing files to force + * the remaining blocks to be processed and written to disk. + * + * @param proc A pointer to a data writer object. + * + * @return Zero on success, an @ref E_SQFS_ERROR value on failure. + */ +SQFS_API int sqfs_block_processor_end_file(sqfs_block_processor_t *proc); + +/** + * @brief Wait for the in-flight data blocks to finish and finally flush the + * current fragment block. + * + * @memberof sqfs_block_processor_t + * + * @param proc A pointer to a block processor object. + * + * @return Zero on success, an @ref E_SQFS_ERROR value on failure. The failure + * return value can either be an error encountered during enqueueing, + * processing or writing to disk. + */ +SQFS_API int sqfs_block_processor_finish(sqfs_block_processor_t *proc); + +/** + * @brief Write the completed fragment table to disk. + * + * @memberof sqfs_block_processor_t + * + * Call this after producing the inode and directory table to generate + * the fragment table for the squashfs image. + * + * @param proc A pointer to a data writer object. + * @param super A pointer to a super block to write information about the + * fragment table to. + * + * @return Zero on success, an @ref E_SQFS_ERROR value on failure. + */ +SQFS_API +int sqfs_block_processor_write_fragment_table(sqfs_block_processor_t *proc, + sqfs_super_t *super); + +/** + * @brief Register a set of hooks to be invoked when writing blocks to disk. + * + * @memberof sqfs_block_processor_t + * + * @param proc A pointer to a data writer object. + * @param user_ptr A user pointer to pass to the callbacks. + * @param hooks A structure containing the hooks. + * + * @return Zero on success, @ref SQFS_ERROR_UNSUPPORTED if the size field of + * the hooks doesn't match any size knwon to the library. + */ +SQFS_API +int sqfs_block_processor_set_hooks(sqfs_block_processor_t *proc, void *user_ptr, + const sqfs_block_hooks_t *hooks); + +#ifdef __cplusplus +} +#endif + +#endif /* SFQS_BLOCK_PROCESSOR_H */ diff --git a/include/sqfs/data_writer.h b/include/sqfs/data_writer.h deleted file mode 100644 index 7b6ee0f..0000000 --- a/include/sqfs/data_writer.h +++ /dev/null @@ -1,310 +0,0 @@ -/* SPDX-License-Identifier: LGPL-3.0-or-later */ -/* - * data_writer.h - This file is part of libsquashfs - * - * Copyright (C) 2019 David Oberhollenzer - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program. If not, see . - */ -#ifndef SFQS_DATA_WRITER_H -#define SFQS_DATA_WRITER_H - -#include "sqfs/predef.h" - -/** - * @file data_writer.h - * - * @brief Contains declarations for the data block processor. - */ - -/** - * @struct sqfs_data_writer_t - * - * @brief Abstracts generating of file data and fragment blocks. - * - * This data structure provides a simple begin/append/end interface - * to generate file data blocks (see @ref sqfs_data_writer_begin_file, - * @ref sqfs_data_writer_append and @ref sqfs_data_writer_end respectively). - * - * Internally it takes care of partitioning data in the correct block sizes, - * adding tail-ens to fragment blocks, compressing the data, deduplicating data - * and finally writing it to disk. - */ - -/** - * @struct sqfs_block_hooks_t - * - * @brief A set of hooks for tapping into the data writer. - * - * This structure can be registered with an @ref sqfs_data_writer_t and - * contains function pointers that will be called during various stages - * when writing data to disk. - * - * The callbacks can not only be used for accounting but may also write extra - * data to the output file or make modifications to the blocks before they are - * writtien. - * - * The callbacks can be individually set to NULL to disable them. - */ -struct sqfs_block_hooks_t { - /** - * @brief Set this to the size of the struct. - * - * This is required for future expandabillity while maintaining ABI - * compatibillity. At the current time, the implementation of - * @ref sqfs_data_writer_set_hooks rejects any hook struct where this - * isn't the exact size. If new hooks are added in the future, the - * struct grows and the future implementation can tell by the size - * whether the application uses the new version or the old one. - */ - size_t size; - - /** - * @brief Gets called before writing a block to disk. - * - * If this is not NULL, it gets called before a block is written to - * disk. If the block has the @ref SQFS_BLK_ALIGN flag set, the - * function is called before padding the file. - * - * The function may modify the block itself or write data to the file. - * which is taken into account when padding the file. - * - * @param user A user pointer. - * @param block The block that is about to be written. - * @param file The file that the block will be written to. - */ - void (*pre_block_write)(void *user, sqfs_block_t *block, - sqfs_file_t *file); - - /** - * @brief Gets called after writing a block to disk. - * - * If this is not NULL, it gets called after a block is written to - * disk. If the block has the @ref SQFS_BLK_ALIGN flag set, the - * function is called before padding the file. - * - * Modifying the block is rather pointless, but the function may - * write data to the file which is taken into account when padding - * the file. - * - * @param user A user pointer. - * @param block The block that is about to be written. - * @param file The file that the block was written to. - */ - void (*post_block_write)(void *user, const sqfs_block_t *block, - sqfs_file_t *file); - - /** - * @brief Gets called before storing a fragment in a fragment block. - * - * The function can modify the block before it is stored. - * - * @param user A user pointer. - * @param block The data chunk that is about to be merged into the - * fragment block. - */ - void (*pre_fragment_store)(void *user, sqfs_block_t *block); - - /** - * @brief Gets called if block deduplication managed to get - * rid of the data blocks of a file. - * - * @param user A user pointer. - * @param count The number of blocks that have been erased. - * @param bytes The number of bytes that have been erased. Includes - * potential padding before and after the end. - */ - void (*notify_blocks_erased)(void *user, size_t count, - sqfs_u64 bytes); - - /** - * @brief Gets called before throwing away a fragment that turned out - * to be a duplicate. - * - * @param user A user pointer. - * @param block The data chunk that is about to be merged into the - * fragment block. - */ - void (*notify_fragment_discard)(void *user, const sqfs_block_t *block); - - /** - * @brief Gets called before writing a block of padding bytes to disk. - * - * @param user A user pointer. - * @param block The padding bytes that are about to be written. - * @param count The number of padding bytes in the block. - */ - void (*prepare_padding)(void *user, sqfs_u8 *block, size_t count); -}; - -#ifdef __cplusplus -extern "C" { -#endif - -/** - * @brief Create a data block writer. - * - * @memberof sqfs_data_writer_t - * - * @param max_block_size The maximum size of a data block. Required for the - * internal scratch buffer used for compressing data. - * @param cmp A pointer to a compressor. If multiple worker threads are used, - * the deep copy function of the compressor is used to create - * several instances that don't interfere with each other. - * @param num_workers The number of worker threads to create. - * @param max_backlog The maximum number of blocks currently in flight. When - * trying to add more, enqueueing blocks until the in-flight - * block count drops below the threshold. - * @param devblksz File can optionally be allgined to device block size. This - * specifies the desired alignment. - * @param file The output file to write the finished blocks to. - * - * @return A pointer to a data writer object on success, NULL on allocation - * failure or on failure to create and initialize the worker threads. - */ -SQFS_API -sqfs_data_writer_t *sqfs_data_writer_create(size_t max_block_size, - sqfs_compressor_t *cmp, - unsigned int num_workers, - size_t max_backlog, - size_t devblksz, - sqfs_file_t *file); - -/** - * @brief Destroy a data writer and free all memory used by it. - * - * @memberof sqfs_data_writer_t - * - * @param proc A pointer to a data writer object. - */ -SQFS_API void sqfs_data_writer_destroy(sqfs_data_writer_t *proc); - -/** - * @brief Start writing a file. - * - * @memberof sqfs_data_writer_t - * - * After calling this function, call @ref sqfs_data_writer_append repeatedly to - * add data to the file. Finally call @ref sqfs_data_writer_end_file when you - * are done. After writing all files, use @ref sqfs_data_writer_finish to wait - * until all blocks that are still in flight are done and written to disk. - * - * The specified inode pointer is kept internally and updated with the - * compressed block sizes and final destinations of the file and possible - * fragment. You need to make sure it has enough backing-store for all blocks - * to come. Furthermore, since there can still be blocks in-flight even after - * calling @ref sqfs_data_writer_end_file, the data in the inode may still - * change. The only point at which the data writer is guarnteed to not touch - * them anymore is after @ref sqfs_data_writer_finish has returned. - * - * @param proc A pointer to a data writer object. - * @param inode The regular file inode representing the file. The data writer - * internally updates it while writing blocks to disk. - * @param flags A combination of @ref E_SQFS_BLK_FLAGS that can be used to - * micro manage how the data is processed. - * - * @return Zero on success, an @ref E_SQFS_ERROR value on failure. - */ -SQFS_API int sqfs_data_writer_begin_file(sqfs_data_writer_t *proc, - sqfs_inode_generic_t *inode, - sqfs_u32 flags); - -/** - * @brief Append data to the current file. - * - * @memberof sqfs_data_writer_t - * - * Call this after @ref sqfs_data_writer_begin_file to add data to a file. - * - * @param proc A pointer to a data writer object. - * @param data A pointer to a buffer to read data from. - * @param size How many bytes should be copied out of the given - * buffer and written to disk. - * - * @return Zero on success, an @ref E_SQFS_ERROR value on failure. - */ -SQFS_API int sqfs_data_writer_append(sqfs_data_writer_t *proc, - const void *data, size_t size); - -/** - * @brief Stop writing the current file and flush everything that is - * buffered internally. - * - * @memberof sqfs_data_writer_t - * - * The counter part to @ref sqfs_data_writer_begin_file. - * - * Even after calling this, there might still be data blocks in-flight. - * Use @ref sqfs_data_writer_finish when you are done writing files to force - * the remaining blocks to be processed and written to disk. - * - * @param proc A pointer to a data writer object. - * - * @return Zero on success, an @ref E_SQFS_ERROR value on failure. - */ -SQFS_API int sqfs_data_writer_end_file(sqfs_data_writer_t *proc); - -/** - * @brief Wait for the in-flight data blocks to finish and finally flush the - * current fragment block. - * - * @memberof sqfs_data_writer_t - * - * @param proc A pointer to a block processor object. - * - * @return Zero on success, an @ref E_SQFS_ERROR value on failure. The failure - * return value can either be an error encountered during enqueueing, - * processing or writing to disk. - */ -SQFS_API int sqfs_data_writer_finish(sqfs_data_writer_t *proc); - -/** - * @brief Write the completed fragment table to disk. - * - * @memberof sqfs_data_writer_t - * - * Call this after producing the inode and directory table to generate - * the fragment table for the squashfs image. - * - * @param proc A pointer to a data writer object. - * @param super A pointer to a super block to write information about the - * fragment table to. - * - * @return Zero on success, an @ref E_SQFS_ERROR value on failure. - */ -SQFS_API -int sqfs_data_writer_write_fragment_table(sqfs_data_writer_t *proc, - sqfs_super_t *super); - -/** - * @brief Register a set of hooks to be invoked when writing blocks to disk. - * - * @memberof sqfs_data_writer_t - * - * @param proc A pointer to a data writer object. - * @param user_ptr A user pointer to pass to the callbacks. - * @param hooks A structure containing the hooks. - * - * @return Zero on success, @ref SQFS_ERROR_UNSUPPORTED if the size field of - * the hooks doesn't match any size knwon to the library. - */ -SQFS_API -int sqfs_data_writer_set_hooks(sqfs_data_writer_t *proc, void *user_ptr, - const sqfs_block_hooks_t *hooks); - -#ifdef __cplusplus -} -#endif - -#endif /* SFQS_DATA_WRITER_H */ diff --git a/include/sqfs/predef.h b/include/sqfs/predef.h index 234b015..924aecc 100644 --- a/include/sqfs/predef.h +++ b/include/sqfs/predef.h @@ -68,7 +68,7 @@ typedef int32_t sqfs_s32; typedef int64_t sqfs_s64; typedef struct sqfs_block_t sqfs_block_t; -typedef struct sqfs_data_writer_t sqfs_data_writer_t; +typedef struct sqfs_block_processor_t sqfs_block_processor_t; typedef struct sqfs_compressor_config_t sqfs_compressor_config_t; typedef struct sqfs_compressor_t sqfs_compressor_t; typedef struct sqfs_dir_writer_t sqfs_dir_writer_t; diff --git a/lib/common/data_writer.c b/lib/common/data_writer.c index c5beba8..8ebd868 100644 --- a/lib/common/data_writer.c +++ b/lib/common/data_writer.c @@ -8,7 +8,7 @@ static sqfs_u8 buffer[4096]; -int write_data_from_file(const char *filename, sqfs_data_writer_t *data, +int write_data_from_file(const char *filename, sqfs_block_processor_t *data, sqfs_inode_generic_t *inode, sqfs_file_t *file, int flags) { @@ -16,7 +16,7 @@ int write_data_from_file(const char *filename, sqfs_data_writer_t *data, size_t diff; int ret; - ret = sqfs_data_writer_begin_file(data, inode, flags); + ret = sqfs_block_processor_begin_file(data, inode, flags); if (ret) { sqfs_perror(filename, "beginning file data blocks", ret); return -1; @@ -37,14 +37,14 @@ int write_data_from_file(const char *filename, sqfs_data_writer_t *data, return -1; } - ret = sqfs_data_writer_append(data, buffer, diff); + ret = sqfs_block_processor_append(data, buffer, diff); if (ret) { sqfs_perror(filename, "packing file data", ret); return -1; } } - ret = sqfs_data_writer_end_file(data); + ret = sqfs_block_processor_end_file(data); if (ret) { sqfs_perror(filename, "finishing file data", ret); return -1; diff --git a/lib/common/statistics.c b/lib/common/statistics.c index 7bf8fca..b41cd22 100644 --- a/lib/common/statistics.c +++ b/lib/common/statistics.c @@ -11,7 +11,7 @@ static void post_block_write(void *user, const sqfs_block_t *block, sqfs_file_t *file) { - data_writer_stats_t *stats = user; + block_processor_stats_t *stats = user; (void)file; if (block->size == 0) @@ -28,7 +28,7 @@ static void post_block_write(void *user, const sqfs_block_t *block, static void pre_fragment_store(void *user, sqfs_block_t *block) { - data_writer_stats_t *stats = user; + block_processor_stats_t *stats = user; (void)block; stats->frag_count += 1; @@ -36,7 +36,7 @@ static void pre_fragment_store(void *user, sqfs_block_t *block) static void notify_blocks_erased(void *user, size_t count, sqfs_u64 bytes) { - data_writer_stats_t *stats = user; + block_processor_stats_t *stats = user; stats->bytes_written -= bytes; stats->blocks_written -= count; @@ -45,7 +45,7 @@ static void notify_blocks_erased(void *user, size_t count, sqfs_u64 bytes) static void notify_fragment_discard(void *user, const sqfs_block_t *block) { - data_writer_stats_t *stats = user; + block_processor_stats_t *stats = user; (void)block; stats->frag_dup += 1; @@ -59,12 +59,13 @@ static const sqfs_block_hooks_t hooks = { .notify_fragment_discard = notify_fragment_discard, }; -void register_stat_hooks(sqfs_data_writer_t *data, data_writer_stats_t *stats) +void register_stat_hooks(sqfs_block_processor_t *data, + block_processor_stats_t *stats) { - sqfs_data_writer_set_hooks(data, stats, &hooks); + sqfs_block_processor_set_hooks(data, stats, &hooks); } -void sqfs_print_statistics(sqfs_super_t *super, data_writer_stats_t *stats) +void sqfs_print_statistics(sqfs_super_t *super, block_processor_stats_t *stats) { size_t ratio; diff --git a/lib/common/writer.c b/lib/common/writer.c index b6adc59..ed3ebbd 100644 --- a/lib/common/writer.c +++ b/lib/common/writer.c @@ -123,11 +123,11 @@ int sqfs_writer_init(sqfs_writer_t *sqfs, const sqfs_writer_cfg_t *wrcfg) if (ret > 0) sqfs->super.flags |= SQFS_FLAG_COMPRESSOR_OPTIONS; - sqfs->data = sqfs_data_writer_create(sqfs->super.block_size, - sqfs->cmp, wrcfg->num_jobs, - wrcfg->max_backlog, - wrcfg->devblksize, - sqfs->outfile); + sqfs->data = sqfs_block_processor_create(sqfs->super.block_size, + sqfs->cmp, wrcfg->num_jobs, + wrcfg->max_backlog, + wrcfg->devblksize, + sqfs->outfile); if (sqfs->data == NULL) { perror("creating data block processor"); goto fail_cmp; @@ -187,7 +187,7 @@ fail_xwr: fail_id: sqfs_id_table_destroy(sqfs->idtbl); fail_data: - sqfs_data_writer_destroy(sqfs->data); + sqfs_block_processor_destroy(sqfs->data); fail_cmp: sqfs->cmp->destroy(sqfs->cmp); fail_fs: @@ -204,7 +204,7 @@ int sqfs_writer_finish(sqfs_writer_t *sqfs, const sqfs_writer_cfg_t *cfg) if (!cfg->quiet) fputs("Waiting for remaining data blocks...\n", stdout); - ret = sqfs_data_writer_finish(sqfs->data); + ret = sqfs_block_processor_finish(sqfs->data); if (ret) { sqfs_perror(cfg->filename, "finishing data blocks", ret); return -1; @@ -221,7 +221,8 @@ int sqfs_writer_finish(sqfs_writer_t *sqfs, const sqfs_writer_cfg_t *cfg) if (!cfg->quiet) fputs("Writing fragment table...\n", stdout); - ret = sqfs_data_writer_write_fragment_table(sqfs->data, &sqfs->super); + ret = sqfs_block_processor_write_fragment_table(sqfs->data, + &sqfs->super); if (ret) { sqfs_perror(cfg->filename, "writing fragment table", ret); return -1; @@ -291,7 +292,7 @@ void sqfs_writer_cleanup(sqfs_writer_t *sqfs) sqfs_meta_writer_destroy(sqfs->dm); sqfs_meta_writer_destroy(sqfs->im); sqfs_id_table_destroy(sqfs->idtbl); - sqfs_data_writer_destroy(sqfs->data); + sqfs_block_processor_destroy(sqfs->data); sqfs->cmp->destroy(sqfs->cmp); fstree_cleanup(&sqfs->fs); sqfs->outfile->destroy(sqfs->outfile); diff --git a/lib/sqfs/Makemodule.am b/lib/sqfs/Makemodule.am index 47b9ad7..cde947d 100644 --- a/lib/sqfs/Makemodule.am +++ b/lib/sqfs/Makemodule.am @@ -1,6 +1,6 @@ LIBSQFS_HEARDS = include/sqfs/meta_writer.h \ include/sqfs/meta_reader.h include/sqfs/id_table.h \ - include/sqfs/compressor.h include/sqfs/data_writer.h \ + include/sqfs/compressor.h include/sqfs/block_processor.h \ include/sqfs/super.h include/sqfs/inode.h \ include/sqfs/dir.h include/sqfs/xattr.h \ include/sqfs/table.h include/sqfs/predef.h \ @@ -19,11 +19,12 @@ libsquashfs_la_SOURCES += lib/sqfs/dir_writer.c lib/sqfs/xattr_reader.c libsquashfs_la_SOURCES += lib/sqfs/read_table.c lib/sqfs/comp/compressor.c libsquashfs_la_SOURCES += lib/sqfs/comp/internal.h lib/sqfs/xattr_writer.c libsquashfs_la_SOURCES += lib/sqfs/dir_reader.c lib/sqfs/read_tree.c -libsquashfs_la_SOURCES += lib/sqfs/inode.c lib/sqfs/data_writer/fragment.c -libsquashfs_la_SOURCES += lib/sqfs/write_super.c lib/sqfs/data_writer/block.c -libsquashfs_la_SOURCES += lib/sqfs/data_writer/internal.h lib/sqfs/data_reader.c -libsquashfs_la_SOURCES += lib/sqfs/data_writer/common.c -libsquashfs_la_SOURCES += lib/sqfs/data_writer/fileapi.c +libsquashfs_la_SOURCES += lib/sqfs/inode.c lib/sqfs/block_processor/fragment.c +libsquashfs_la_SOURCES += lib/sqfs/write_super.c lib/sqfs/data_reader.c +libsquashfs_la_SOURCES += lib/sqfs/block_processor/block.c +libsquashfs_la_SOURCES += lib/sqfs/block_processor/internal.h +libsquashfs_la_SOURCES += lib/sqfs/block_processor/common.c +libsquashfs_la_SOURCES += lib/sqfs/block_processor/fileapi.c libsquashfs_la_SOURCES += lib/sqfs/str_table.c lib/sqfs/str_table.h libsquashfs_la_SOURCES += lib/sqfs/alloc.c lib/sqfs/util.h libsquashfs_la_SOURCES += lib/sqfs/frag_table.c include/sqfs/frag_table.h @@ -45,13 +46,13 @@ libsquashfs_la_SOURCES += lib/sqfs/unix/io_file.c endif if HAVE_PTHREAD -libsquashfs_la_SOURCES += lib/sqfs/data_writer/winpthread.c +libsquashfs_la_SOURCES += lib/sqfs/block_processor/winpthread.c libsquashfs_la_CPPFLAGS += -DWITH_PTHREAD else if WINDOWS -libsquashfs_la_SOURCES += lib/sqfs/data_writer/winpthread.c +libsquashfs_la_SOURCES += lib/sqfs/block_processor/winpthread.c else -libsquashfs_la_SOURCES += lib/sqfs/data_writer/serial.c +libsquashfs_la_SOURCES += lib/sqfs/block_processor/serial.c endif endif diff --git a/lib/sqfs/block_processor/block.c b/lib/sqfs/block_processor/block.c new file mode 100644 index 0000000..49892be --- /dev/null +++ b/lib/sqfs/block_processor/block.c @@ -0,0 +1,203 @@ +/* SPDX-License-Identifier: LGPL-3.0-or-later */ +/* + * process_block.c + * + * Copyright (C) 2019 David Oberhollenzer + */ +#define SQFS_BUILDING_DLL +#include "internal.h" + +#include + +static int store_block_location(sqfs_block_processor_t *proc, sqfs_u64 offset, + sqfs_u32 size, sqfs_u32 chksum) +{ + size_t new_sz; + void *new; + + if (proc->num_blocks == proc->max_blocks) { + new_sz = proc->max_blocks * 2; + new = realloc(proc->blocks, sizeof(proc->blocks[0]) * new_sz); + + if (new == NULL) + return SQFS_ERROR_ALLOC; + + proc->blocks = new; + proc->max_blocks = new_sz; + } + + proc->blocks[proc->num_blocks].offset = offset; + proc->blocks[proc->num_blocks].hash = MK_BLK_HASH(chksum, size); + proc->num_blocks += 1; + return 0; +} + +static size_t deduplicate_blocks(sqfs_block_processor_t *proc, size_t count) +{ + size_t i, j; + + for (i = 0; i < proc->file_start; ++i) { + for (j = 0; j < count; ++j) { + if (proc->blocks[i + j].hash != + proc->blocks[proc->file_start + j].hash) + break; + } + + if (j == count) + break; + } + + return i; +} + +static int align_file(sqfs_block_processor_t *proc, sqfs_block_t *blk) +{ + sqfs_u32 chksum; + void *padding; + sqfs_u64 size; + size_t diff; + int ret; + + if (!(blk->flags & SQFS_BLK_ALIGN)) + return 0; + + size = proc->file->get_size(proc->file); + diff = size % proc->devblksz; + if (diff == 0) + return 0; + + padding = calloc(1, diff); + if (padding == 0) + return SQFS_ERROR_ALLOC; + + if (proc->hooks != NULL && proc->hooks->prepare_padding != NULL) + proc->hooks->prepare_padding(proc->user_ptr, padding, diff); + + chksum = crc32(0, padding, diff); + + ret = proc->file->write_at(proc->file, size, padding, diff); + free(padding); + if (ret) + return ret; + + return store_block_location(proc, size, diff | (1 << 24), chksum); +} + +int process_completed_block(sqfs_block_processor_t *proc, sqfs_block_t *blk) +{ + sqfs_u64 offset, bytes; + size_t start, count; + sqfs_u32 out; + int err; + + if (proc->hooks != NULL && proc->hooks->pre_block_write != NULL) { + proc->hooks->pre_block_write(proc->user_ptr, blk, proc->file); + } + + if (blk->flags & SQFS_BLK_FIRST_BLOCK) { + proc->start = proc->file->get_size(proc->file); + proc->file_start = proc->num_blocks; + + err = align_file(proc, blk); + if (err) + return err; + } + + if (blk->size != 0) { + out = blk->size; + if (!(blk->flags & SQFS_BLK_IS_COMPRESSED)) + out |= 1 << 24; + + offset = proc->file->get_size(proc->file); + + if (blk->flags & SQFS_BLK_FRAGMENT_BLOCK) { + err = sqfs_frag_table_set(proc->frag_tbl, blk->index, + offset, out); + if (err) + return err; + } else { + blk->inode->extra[blk->index] = out; + } + + err = store_block_location(proc, offset, out, blk->checksum); + if (err) + return err; + + err = proc->file->write_at(proc->file, offset, + blk->data, blk->size); + if (err) + return err; + } + + if (proc->hooks != NULL && proc->hooks->post_block_write != NULL) { + proc->hooks->post_block_write(proc->user_ptr, blk, proc->file); + } + + if (blk->flags & SQFS_BLK_LAST_BLOCK) { + err = align_file(proc, blk); + if (err) + return err; + + count = proc->num_blocks - proc->file_start; + start = deduplicate_blocks(proc, count); + offset = proc->blocks[start].offset; + + sqfs_inode_set_file_block_start(blk->inode, offset); + + if (start >= proc->file_start) + return 0; + + offset = start + count; + if (offset >= proc->file_start) { + count = proc->num_blocks - offset; + proc->num_blocks = offset; + } else { + proc->num_blocks = proc->file_start; + } + + if (proc->hooks != NULL && + proc->hooks->notify_blocks_erased != NULL) { + bytes = proc->file->get_size(proc->file) - proc->start; + + proc->hooks->notify_blocks_erased(proc->user_ptr, + count, bytes); + } + + err = proc->file->truncate(proc->file, proc->start); + if (err) + return err; + } + + return 0; +} + +int block_processor_do_block(sqfs_block_t *block, sqfs_compressor_t *cmp, + sqfs_u8 *scratch, size_t scratch_size) +{ + ssize_t ret; + + if (block->size == 0) { + block->checksum = 0; + return 0; + } + + block->checksum = crc32(0, block->data, block->size); + + if (block->flags & SQFS_BLK_IS_FRAGMENT) + return 0; + + if (!(block->flags & SQFS_BLK_DONT_COMPRESS)) { + ret = cmp->do_block(cmp, block->data, block->size, + scratch, scratch_size); + if (ret < 0) + return ret; + + if (ret > 0) { + memcpy(block->data, scratch, ret); + block->size = ret; + block->flags |= SQFS_BLK_IS_COMPRESSED; + } + } + + return 0; +} diff --git a/lib/sqfs/block_processor/common.c b/lib/sqfs/block_processor/common.c new file mode 100644 index 0000000..c6375dd --- /dev/null +++ b/lib/sqfs/block_processor/common.c @@ -0,0 +1,72 @@ +/* SPDX-License-Identifier: LGPL-3.0-or-later */ +/* + * common.c + * + * Copyright (C) 2019 David Oberhollenzer + */ +#define SQFS_BUILDING_DLL +#include "internal.h" + +void free_blk_list(sqfs_block_t *list) +{ + sqfs_block_t *it; + + while (list != NULL) { + it = list; + list = list->next; + free(it); + } +} + +int block_processor_init(sqfs_block_processor_t *proc, size_t max_block_size, + sqfs_compressor_t *cmp, unsigned int num_workers, + size_t max_backlog, size_t devblksz, sqfs_file_t *file) +{ + proc->max_block_size = max_block_size; + proc->num_workers = num_workers; + proc->max_backlog = max_backlog; + proc->devblksz = devblksz; + proc->cmp = cmp; + proc->file = file; + proc->max_blocks = INIT_BLOCK_COUNT; + + proc->frag_tbl = sqfs_frag_table_create(0); + if (proc->frag_tbl == NULL) + return -1; + + proc->blocks = alloc_array(sizeof(proc->blocks[0]), proc->max_blocks); + if (proc->blocks == NULL) + return -1; + + return 0; +} + +void block_processor_cleanup(sqfs_block_processor_t *proc) +{ + if (proc->frag_tbl != NULL) + sqfs_frag_table_destroy(proc->frag_tbl); + free_blk_list(proc->queue); + free_blk_list(proc->done); + free(proc->blk_current); + free(proc->frag_block); + free(proc->blocks); + free(proc); +} + +int sqfs_block_processor_write_fragment_table(sqfs_block_processor_t *proc, + sqfs_super_t *super) +{ + return sqfs_frag_table_write(proc->frag_tbl, proc->file, + super, proc->cmp); +} + +int sqfs_block_processor_set_hooks(sqfs_block_processor_t *proc, void *user_ptr, + const sqfs_block_hooks_t *hooks) +{ + if (hooks->size != sizeof(*hooks)) + return SQFS_ERROR_UNSUPPORTED; + + proc->hooks = hooks; + proc->user_ptr = user_ptr; + return 0; +} diff --git a/lib/sqfs/block_processor/fileapi.c b/lib/sqfs/block_processor/fileapi.c new file mode 100644 index 0000000..9e59c1d --- /dev/null +++ b/lib/sqfs/block_processor/fileapi.c @@ -0,0 +1,188 @@ +/* SPDX-License-Identifier: LGPL-3.0-or-later */ +/* + * fileapi.c + * + * Copyright (C) 2019 David Oberhollenzer + */ +#define SQFS_BUILDING_DLL +#include "internal.h" + +static bool is_zero_block(unsigned char *ptr, size_t size) +{ + return ptr[0] == 0 && memcmp(ptr, ptr + 1, size - 1) == 0; +} + +static int enqueue_block(sqfs_block_processor_t *proc, sqfs_block_t *block) +{ + int status; + + while (proc->backlog > proc->max_backlog) { + status = wait_completed(proc); + if (status) + return status; + } + + if (proc->backlog == proc->max_backlog) + proc->notify_threads = true; + + return append_to_work_queue(proc, block, proc->notify_threads); +} + +static int add_sentinel_block(sqfs_block_processor_t *proc) +{ + sqfs_block_t *blk = calloc(1, sizeof(*blk)); + + if (blk == NULL) + return test_and_set_status(proc, SQFS_ERROR_ALLOC); + + blk->inode = proc->inode; + blk->flags = proc->blk_flags | SQFS_BLK_LAST_BLOCK; + + return enqueue_block(proc, blk); +} + +int sqfs_block_processor_begin_file(sqfs_block_processor_t *proc, + sqfs_inode_generic_t *inode, sqfs_u32 flags) +{ + if (proc->inode != NULL) + return test_and_set_status(proc, SQFS_ERROR_INTERNAL); + + if (flags & ~SQFS_BLK_USER_SETTABLE_FLAGS) + return test_and_set_status(proc, SQFS_ERROR_UNSUPPORTED); + + proc->inode = inode; + proc->blk_flags = flags | SQFS_BLK_FIRST_BLOCK; + proc->blk_index = 0; + proc->blk_current = NULL; + return 0; +} + +static int flush_block(sqfs_block_processor_t *proc, sqfs_block_t *block) +{ + block->index = proc->blk_index++; + block->flags = proc->blk_flags; + block->inode = proc->inode; + + if (is_zero_block(block->data, block->size)) { + sqfs_inode_make_extended(proc->inode); + proc->inode->data.file_ext.sparse += block->size; + proc->inode->num_file_blocks += 1; + proc->inode->extra[block->index] = 0; + free(block); + return 0; + } + + if (block->size < proc->max_block_size && + !(block->flags & SQFS_BLK_DONT_FRAGMENT)) { + block->flags |= SQFS_BLK_IS_FRAGMENT; + } else { + proc->inode->num_file_blocks += 1; + proc->blk_flags &= ~SQFS_BLK_FIRST_BLOCK; + } + + return enqueue_block(proc, block); +} + +int sqfs_block_processor_append(sqfs_block_processor_t *proc, const void *data, + size_t size) +{ + size_t diff; + void *new; + int err; + + while (size > 0) { + if (proc->blk_current == NULL) { + new = alloc_flex(sizeof(*proc->blk_current), 1, + proc->max_block_size); + + if (new == NULL) + return test_and_set_status(proc, + SQFS_ERROR_ALLOC); + + proc->blk_current = new; + } + + diff = proc->max_block_size - proc->blk_current->size; + + if (diff == 0) { + err = flush_block(proc, proc->blk_current); + proc->blk_current = NULL; + if (err) + return err; + continue; + } + + if (diff > size) + diff = size; + + memcpy(proc->blk_current->data + proc->blk_current->size, + data, diff); + + size -= diff; + proc->blk_current->size += diff; + data = (const char *)data + diff; + } + + if (proc->blk_current != NULL && + proc->blk_current->size == proc->max_block_size) { + err = flush_block(proc, proc->blk_current); + proc->blk_current = NULL; + return err; + } + + return 0; +} + +int sqfs_block_processor_end_file(sqfs_block_processor_t *proc) +{ + int err; + + if (proc->inode == NULL) + return test_and_set_status(proc, SQFS_ERROR_INTERNAL); + + if (!(proc->blk_flags & SQFS_BLK_FIRST_BLOCK)) { + if (proc->blk_current != NULL && + (proc->blk_flags & SQFS_BLK_DONT_FRAGMENT)) { + proc->blk_flags |= SQFS_BLK_LAST_BLOCK; + } else { + err = add_sentinel_block(proc); + if (err) + return err; + } + } + + if (proc->blk_current != NULL) { + err = flush_block(proc, proc->blk_current); + proc->blk_current = NULL; + } + + proc->inode = NULL; + proc->blk_flags = 0; + proc->blk_index = 0; + return 0; +} + +int sqfs_block_processor_finish(sqfs_block_processor_t *proc) +{ + int status = 0; + + append_to_work_queue(proc, NULL, true); + + while (proc->backlog > 0) { + status = wait_completed(proc); + if (status) + return status; + } + + if (proc->frag_block != NULL) { + status = append_to_work_queue(proc, proc->frag_block, true); + proc->frag_block = NULL; + + if (status) + return status; + + status = wait_completed(proc); + } + + return status; +} diff --git a/lib/sqfs/block_processor/fragment.c b/lib/sqfs/block_processor/fragment.c new file mode 100644 index 0000000..3701b3c --- /dev/null +++ b/lib/sqfs/block_processor/fragment.c @@ -0,0 +1,81 @@ +/* SPDX-License-Identifier: LGPL-3.0-or-later */ +/* + * fragtbl.c + * + * Copyright (C) 2019 David Oberhollenzer + */ +#define SQFS_BUILDING_DLL +#include "internal.h" + +int process_completed_fragment(sqfs_block_processor_t *proc, sqfs_block_t *frag, + sqfs_block_t **blk_out) +{ + sqfs_u32 index, offset; + size_t size; + int err; + + err = sqfs_frag_table_find_tail_end(proc->frag_tbl, + frag->checksum, frag->size, + &index, &offset); + if (err == 0) + goto out_duplicate; + + if (proc->frag_block != NULL) { + size = proc->frag_block->size + frag->size; + + if (size > proc->max_block_size) { + *blk_out = proc->frag_block; + proc->frag_block = NULL; + } + } + + if (proc->frag_block == NULL) { + size = sizeof(sqfs_block_t) + proc->max_block_size; + + err= sqfs_frag_table_append(proc->frag_tbl, 0, 0, &index); + if (err) + goto fail; + + proc->frag_block = calloc(1, size); + if (proc->frag_block == NULL) { + err = SQFS_ERROR_ALLOC; + goto fail; + } + + proc->frag_block->index = index; + proc->frag_block->flags = SQFS_BLK_FRAGMENT_BLOCK; + } + + err = sqfs_frag_table_add_tail_end(proc->frag_tbl, + proc->frag_block->index, + proc->frag_block->size, + frag->size, frag->checksum); + if (err) + goto fail; + + sqfs_inode_set_frag_location(frag->inode, proc->frag_block->index, + proc->frag_block->size); + + if (proc->hooks != NULL && proc->hooks->pre_fragment_store != NULL) { + proc->hooks->pre_fragment_store(proc->user_ptr, frag); + } + + memcpy(proc->frag_block->data + proc->frag_block->size, + frag->data, frag->size); + + proc->frag_block->flags |= (frag->flags & SQFS_BLK_DONT_COMPRESS); + proc->frag_block->size += frag->size; + return 0; +fail: + free(*blk_out); + *blk_out = NULL; + return err; +out_duplicate: + sqfs_inode_set_frag_location(frag->inode, index, offset); + + if (proc->hooks != NULL && + proc->hooks->notify_fragment_discard != NULL) { + proc->hooks->notify_fragment_discard(proc->user_ptr, frag); + } + return 0; +} diff --git a/lib/sqfs/block_processor/internal.h b/lib/sqfs/block_processor/internal.h new file mode 100644 index 0000000..40871b9 --- /dev/null +++ b/lib/sqfs/block_processor/internal.h @@ -0,0 +1,141 @@ +/* SPDX-License-Identifier: LGPL-3.0-or-later */ +/* + * internal.h + * + * Copyright (C) 2019 David Oberhollenzer + */ +#ifndef INTERNAL_H +#define INTERNAL_H + +#include "config.h" + +#include "sqfs/block_processor.h" +#include "sqfs/frag_table.h" +#include "sqfs/compressor.h" +#include "sqfs/inode.h" +#include "sqfs/table.h" +#include "sqfs/error.h" +#include "sqfs/block.h" +#include "sqfs/io.h" +#include "../util.h" + +#include +#include +#include + +#ifdef WITH_PTHREAD +#include +#include +#elif defined(_WIN32) || defined(__WINDOWS__) +#define WIN32_LEAN_AND_MEAN +#include +#endif + + +#define MK_BLK_HASH(chksum, size) \ + (((sqfs_u64)(size) << 32) | (sqfs_u64)(chksum)) + +#define INIT_BLOCK_COUNT (128) + + +typedef struct { + sqfs_u64 offset; + sqfs_u64 hash; +} blk_info_t; + + +typedef struct compress_worker_t compress_worker_t; + +struct sqfs_block_processor_t { + /* synchronization primitives */ +#ifdef WITH_PTHREAD + pthread_mutex_t mtx; + pthread_cond_t queue_cond; + pthread_cond_t done_cond; +#elif defined(_WIN32) || defined(__WINDOWS__) + CRITICAL_SECTION mtx; + CONDITION_VARIABLE queue_cond; + CONDITION_VARIABLE done_cond; +#endif + + /* needs rw access by worker and main thread */ + sqfs_block_t *queue; + sqfs_block_t *queue_last; + + sqfs_block_t *done; + size_t backlog; + int status; + + /* used by main thread only */ + sqfs_u32 enqueue_id; + sqfs_u32 dequeue_id; + + unsigned int num_workers; + size_t max_backlog; + + size_t devblksz; + sqfs_file_t *file; + + sqfs_frag_table_t *frag_tbl; + + sqfs_u64 start; + + size_t file_start; + size_t num_blocks; + size_t max_blocks; + blk_info_t *blocks; + sqfs_compressor_t *cmp; + + sqfs_block_t *frag_block; + + const sqfs_block_hooks_t *hooks; + void *user_ptr; + bool notify_threads; + + /* file API */ + sqfs_inode_generic_t *inode; + sqfs_block_t *blk_current; + sqfs_u32 blk_flags; + size_t blk_index; + + /* used only by workers */ + size_t max_block_size; + +#if defined(WITH_PTHREAD) || defined(_WIN32) || defined(__WINDOWS__) + compress_worker_t *workers[]; +#else + sqfs_u8 scratch[]; +#endif +}; + +SQFS_INTERNAL int process_completed_block(sqfs_block_processor_t *proc, + sqfs_block_t *block); + +SQFS_INTERNAL +int process_completed_fragment(sqfs_block_processor_t *proc, sqfs_block_t *frag, + sqfs_block_t **blk_out); + +SQFS_INTERNAL void free_blk_list(sqfs_block_t *list); + +SQFS_INTERNAL +int block_processor_init(sqfs_block_processor_t *proc, size_t max_block_size, + sqfs_compressor_t *cmp, unsigned int num_workers, + size_t max_backlog, size_t devblksz, + sqfs_file_t *file); + +SQFS_INTERNAL void block_processor_cleanup(sqfs_block_processor_t *proc); + +SQFS_INTERNAL +int block_processor_do_block(sqfs_block_t *block, sqfs_compressor_t *cmp, + sqfs_u8 *scratch, size_t scratch_size); + +SQFS_INTERNAL +int test_and_set_status(sqfs_block_processor_t *proc, int status); + +SQFS_INTERNAL +int append_to_work_queue(sqfs_block_processor_t *proc, sqfs_block_t *block, + bool notify_threads); + +SQFS_INTERNAL int wait_completed(sqfs_block_processor_t *proc); + +#endif /* INTERNAL_H */ diff --git a/lib/sqfs/block_processor/serial.c b/lib/sqfs/block_processor/serial.c new file mode 100644 index 0000000..eedb19c --- /dev/null +++ b/lib/sqfs/block_processor/serial.c @@ -0,0 +1,88 @@ +/* SPDX-License-Identifier: LGPL-3.0-or-later */ +/* + * serial.c + * + * Copyright (C) 2019 David Oberhollenzer + */ +#define SQFS_BUILDING_DLL +#include "internal.h" + +sqfs_block_processor_t *sqfs_block_processor_create(size_t max_block_size, + sqfs_compressor_t *cmp, + unsigned int num_workers, + size_t max_backlog, + size_t devblksz, + sqfs_file_t *file) +{ + sqfs_block_processor_t *proc; + + proc = alloc_flex(sizeof(*proc), 1, max_block_size); + + if (proc == NULL) + return NULL; + + if (block_processor_init(proc, max_block_size, cmp, num_workers, + max_backlog, devblksz, file)) { + block_processor_cleanup(proc); + return NULL; + } + + return proc; +} + +void sqfs_block_processor_destroy(sqfs_block_processor_t *proc) +{ + block_processor_cleanup(proc); +} + +int test_and_set_status(sqfs_block_processor_t *proc, int status) +{ + if (proc->status == 0) + proc->status = status; + + return proc->status; +} + +int append_to_work_queue(sqfs_block_processor_t *proc, sqfs_block_t *block, + bool signal_threads) +{ + sqfs_block_t *fragblk = NULL; + (void)signal_threads; + + if (proc->status != 0 || block == NULL) { + free(block); + return proc->status; + } + + if (block->flags & SQFS_BLK_IS_FRAGMENT) { + block->checksum = crc32(0, block->data, block->size); + + proc->status = process_completed_fragment(proc, block, + &fragblk); + free(block); + + if (proc->status != 0) { + free(fragblk); + return proc->status; + } + + if (fragblk == NULL) + return 0; + + block = fragblk; + } + + proc->status = block_processor_do_block(block, proc->cmp, proc->scratch, + proc->max_block_size); + + if (proc->status == 0) + proc->status = process_completed_block(proc, block); + + free(block); + return proc->status; +} + +int wait_completed(sqfs_block_processor_t *proc) +{ + return proc->status; +} diff --git a/lib/sqfs/block_processor/winpthread.c b/lib/sqfs/block_processor/winpthread.c new file mode 100644 index 0000000..b16a17c --- /dev/null +++ b/lib/sqfs/block_processor/winpthread.c @@ -0,0 +1,450 @@ +/* SPDX-License-Identifier: LGPL-3.0-or-later */ +/* + * winpthread.c + * + * Copyright (C) 2019 David Oberhollenzer + */ +#define SQFS_BUILDING_DLL +#include "internal.h" + +#if defined(_WIN32) || defined(__WINDOWS__) +# define LOCK(mtx) EnterCriticalSection(mtx) +# define UNLOCK(mtx) LeaveCriticalSection(mtx) +# define AWAIT(cond, mtx) SleepConditionVariableCS(cond, mtx, INFINITE) +# define SIGNAL_ALL(cond) WakeAllConditionVariable(cond) +# define THREAD_EXIT_SUCCESS 0 +# define THREAD_TYPE DWORD WINAPI +# define THREAD_ARG LPVOID +# define THREAD_HANDLE HANDLE +#else +# define LOCK(mtx) pthread_mutex_lock(mtx) +# define UNLOCK(mtx) pthread_mutex_unlock(mtx) +# define AWAIT(cond, mtx) pthread_cond_wait(cond, mtx) +# define SIGNAL_ALL(cond) pthread_cond_broadcast(cond) +# define THREAD_EXIT_SUCCESS NULL +# define THREAD_TYPE void * +# define THREAD_ARG void * +# define THREAD_HANDLE pthread_t +#endif + +struct compress_worker_t { + sqfs_block_processor_t *shared; + sqfs_compressor_t *cmp; + THREAD_HANDLE thread; + sqfs_u8 scratch[]; +}; + +static THREAD_TYPE worker_proc(THREAD_ARG arg) +{ + compress_worker_t *worker = arg; + sqfs_block_processor_t *shared = worker->shared; + sqfs_block_t *it, *prev, *blk = NULL; + int status = 0; + + for (;;) { + LOCK(&shared->mtx); + if (blk != NULL) { + it = shared->done; + prev = NULL; + + while (it != NULL) { + if (it->sequence_number >= blk->sequence_number) + break; + prev = it; + it = it->next; + } + + if (prev == NULL) { + blk->next = shared->done; + shared->done = blk; + } else { + blk->next = prev->next; + prev->next = blk; + } + + if (status != 0 && shared->status == 0) + shared->status = status; + SIGNAL_ALL(&shared->done_cond); + } + + while (shared->queue == NULL && shared->status == 0) + AWAIT(&shared->queue_cond, &shared->mtx); + + if (shared->status == 0) { + blk = shared->queue; + shared->queue = blk->next; + blk->next = NULL; + + if (shared->queue == NULL) + shared->queue_last = NULL; + } else { + blk = NULL; + } + UNLOCK(&shared->mtx); + + if (blk == NULL) + break; + + status = block_processor_do_block(blk, worker->cmp, + worker->scratch, + shared->max_block_size); + } + + return THREAD_EXIT_SUCCESS; +} + +#if defined(_WIN32) || defined(__WINDOWS__) +sqfs_block_processor_t *sqfs_block_processor_create(size_t max_block_size, + sqfs_compressor_t *cmp, + unsigned int num_workers, + size_t max_backlog, + size_t devblksz, + sqfs_file_t *file) +{ + sqfs_block_processor_t *proc; + unsigned int i; + + if (num_workers < 1) + num_workers = 1; + + proc = alloc_flex(sizeof(*proc), + sizeof(proc->workers[0]), num_workers); + if (proc == NULL) + return NULL; + + InitializeCriticalSection(&proc->mtx); + InitializeConditionVariable(&proc->queue_cond); + InitializeConditionVariable(&proc->done_cond); + + if (block_processor_init(proc, max_block_size, cmp, num_workers, + max_backlog, devblksz, file)) { + goto fail; + } + + for (i = 0; i < num_workers; ++i) { + proc->workers[i] = alloc_flex(sizeof(compress_worker_t), + 1, max_block_size); + + if (proc->workers[i] == NULL) + goto fail; + + proc->workers[i]->shared = proc; + proc->workers[i]->cmp = cmp->create_copy(cmp); + + if (proc->workers[i]->cmp == NULL) + goto fail; + + proc->workers[i]->thread = CreateThread(NULL, 0, worker_proc, + proc->workers[i], 0, 0); + if (proc->workers[i]->thread == NULL) + goto fail; + } + + return proc; +fail: + sqfs_block_processor_destroy(proc); + return NULL; +} + +void sqfs_block_processor_destroy(sqfs_block_processor_t *proc) +{ + unsigned int i; + + EnterCriticalSection(&proc->mtx); + proc->status = -1; + WakeAllConditionVariable(&proc->queue_cond); + LeaveCriticalSection(&proc->mtx); + + for (i = 0; i < proc->num_workers; ++i) { + if (proc->workers[i] == NULL) + continue; + + if (proc->workers[i]->thread != NULL) { + WaitForSingleObject(proc->workers[i]->thread, INFINITE); + CloseHandle(proc->workers[i]->thread); + } + + if (proc->workers[i]->cmp != NULL) + proc->workers[i]->cmp->destroy(proc->workers[i]->cmp); + + free(proc->workers[i]); + } + + DeleteCriticalSection(&proc->mtx); + block_processor_cleanup(proc); +} +#else +sqfs_block_processor_t *sqfs_block_processor_create(size_t max_block_size, + sqfs_compressor_t *cmp, + unsigned int num_workers, + size_t max_backlog, + size_t devblksz, + sqfs_file_t *file) +{ + sqfs_block_processor_t *proc; + sigset_t set, oldset; + unsigned int i; + int ret; + + if (num_workers < 1) + num_workers = 1; + + proc = alloc_flex(sizeof(*proc), + sizeof(proc->workers[0]), num_workers); + if (proc == NULL) + return NULL; + + proc->mtx = (pthread_mutex_t)PTHREAD_MUTEX_INITIALIZER; + proc->queue_cond = (pthread_cond_t)PTHREAD_COND_INITIALIZER; + proc->done_cond = (pthread_cond_t)PTHREAD_COND_INITIALIZER; + + if (block_processor_init(proc, max_block_size, cmp, num_workers, + max_backlog, devblksz, file)) { + goto fail_init; + } + + for (i = 0; i < num_workers; ++i) { + proc->workers[i] = alloc_flex(sizeof(compress_worker_t), + 1, max_block_size); + + if (proc->workers[i] == NULL) + goto fail_init; + + proc->workers[i]->shared = proc; + proc->workers[i]->cmp = cmp->create_copy(cmp); + + if (proc->workers[i]->cmp == NULL) + goto fail_init; + } + + sigfillset(&set); + pthread_sigmask(SIG_SETMASK, &set, &oldset); + + for (i = 0; i < num_workers; ++i) { + ret = pthread_create(&proc->workers[i]->thread, NULL, + worker_proc, proc->workers[i]); + + if (ret != 0) + goto fail_thread; + } + + pthread_sigmask(SIG_SETMASK, &oldset, NULL); + + return proc; +fail_thread: + pthread_mutex_lock(&proc->mtx); + proc->status = -1; + pthread_cond_broadcast(&proc->queue_cond); + pthread_mutex_unlock(&proc->mtx); + + for (i = 0; i < num_workers; ++i) { + if (proc->workers[i]->thread > 0) { + pthread_join(proc->workers[i]->thread, NULL); + } + } + pthread_sigmask(SIG_SETMASK, &oldset, NULL); +fail_init: + for (i = 0; i < num_workers; ++i) { + if (proc->workers[i] != NULL) { + if (proc->workers[i]->cmp != NULL) { + proc->workers[i]->cmp-> + destroy(proc->workers[i]->cmp); + } + + free(proc->workers[i]); + } + } + pthread_cond_destroy(&proc->done_cond); + pthread_cond_destroy(&proc->queue_cond); + pthread_mutex_destroy(&proc->mtx); + block_processor_cleanup(proc); + return NULL; +} + +void sqfs_block_processor_destroy(sqfs_block_processor_t *proc) +{ + unsigned int i; + + pthread_mutex_lock(&proc->mtx); + proc->status = -1; + pthread_cond_broadcast(&proc->queue_cond); + pthread_mutex_unlock(&proc->mtx); + + for (i = 0; i < proc->num_workers; ++i) { + pthread_join(proc->workers[i]->thread, NULL); + + proc->workers[i]->cmp->destroy(proc->workers[i]->cmp); + free(proc->workers[i]); + } + + pthread_cond_destroy(&proc->done_cond); + pthread_cond_destroy(&proc->queue_cond); + pthread_mutex_destroy(&proc->mtx); + + block_processor_cleanup(proc); +} +#endif + +int append_to_work_queue(sqfs_block_processor_t *proc, sqfs_block_t *block, + bool signal_threads) +{ + int status; + + LOCK(&proc->mtx); + status = proc->status; + if (status != 0) + goto out; + + if (block != NULL) { + if (proc->queue_last == NULL) { + proc->queue = proc->queue_last = block; + } else { + proc->queue_last->next = block; + proc->queue_last = block; + } + + block->sequence_number = proc->enqueue_id++; + block->next = NULL; + proc->backlog += 1; + block = NULL; + } +out: + if (signal_threads) + SIGNAL_ALL(&proc->queue_cond); + + UNLOCK(&proc->mtx); + free(block); + return 0; +} + +static sqfs_block_t *try_dequeue(sqfs_block_processor_t *proc) +{ + sqfs_block_t *queue, *it, *prev; + + it = proc->done; + prev = NULL; + + while (it != NULL && it->sequence_number == proc->dequeue_id) { + prev = it; + it = it->next; + proc->dequeue_id += 1; + } + + if (prev == NULL) { + queue = NULL; + } else { + queue = proc->done; + prev->next = NULL; + proc->done = it; + } + + return queue; +} + +static sqfs_block_t *queue_merge(sqfs_block_t *lhs, sqfs_block_t *rhs) +{ + sqfs_block_t *it, *head = NULL, **next_ptr = &head; + + while (lhs != NULL && rhs != NULL) { + if (lhs->sequence_number <= rhs->sequence_number) { + it = lhs; + lhs = lhs->next; + } else { + it = rhs; + rhs = rhs->next; + } + + *next_ptr = it; + next_ptr = &it->next; + } + + it = (lhs != NULL ? lhs : rhs); + *next_ptr = it; + return head; +} + +static int process_done_queue(sqfs_block_processor_t *proc, sqfs_block_t *queue) +{ + sqfs_block_t *it, *block = NULL; + int status = 0; + + while (queue != NULL && status == 0) { + it = queue; + queue = it->next; + proc->backlog -= 1; + + if (it->flags & SQFS_BLK_IS_FRAGMENT) { + block = NULL; + status = process_completed_fragment(proc, it, &block); + + if (block != NULL && status == 0) { + LOCK(&proc->mtx); + proc->dequeue_id = it->sequence_number; + block->sequence_number = it->sequence_number; + + if (proc->queue == NULL) { + proc->queue = block; + proc->queue_last = block; + } else { + block->next = proc->queue; + proc->queue = block; + } + + proc->backlog += 1; + proc->done = queue_merge(queue, proc->done); + SIGNAL_ALL(&proc->queue_cond); + UNLOCK(&proc->mtx); + + queue = NULL; + } else { + free(block); + } + } else { + status = process_completed_block(proc, it); + } + + free(it); + } + + free_blk_list(queue); + return status; +} + +int test_and_set_status(sqfs_block_processor_t *proc, int status) +{ + LOCK(&proc->mtx); + if (proc->status == 0) { + proc->status = status; + } else { + status = proc->status; + } + SIGNAL_ALL(&proc->queue_cond); + UNLOCK(&proc->mtx); + return status; +} + +int wait_completed(sqfs_block_processor_t *proc) +{ + sqfs_block_t *queue; + int status; + + LOCK(&proc->mtx); + for (;;) { + queue = try_dequeue(proc); + status = proc->status; + + if (queue != NULL || status != 0) + break; + + AWAIT(&proc->done_cond, &proc->mtx); + } + UNLOCK(&proc->mtx); + + if (status != 0) { + free_blk_list(queue); + return status; + } + + status = process_done_queue(proc, queue); + return status ? test_and_set_status(proc, status) : status; +} diff --git a/lib/sqfs/data_writer/block.c b/lib/sqfs/data_writer/block.c deleted file mode 100644 index b45b1ff..0000000 --- a/lib/sqfs/data_writer/block.c +++ /dev/null @@ -1,203 +0,0 @@ -/* SPDX-License-Identifier: LGPL-3.0-or-later */ -/* - * process_block.c - * - * Copyright (C) 2019 David Oberhollenzer - */ -#define SQFS_BUILDING_DLL -#include "internal.h" - -#include - -static int store_block_location(sqfs_data_writer_t *proc, sqfs_u64 offset, - sqfs_u32 size, sqfs_u32 chksum) -{ - size_t new_sz; - void *new; - - if (proc->num_blocks == proc->max_blocks) { - new_sz = proc->max_blocks * 2; - new = realloc(proc->blocks, sizeof(proc->blocks[0]) * new_sz); - - if (new == NULL) - return SQFS_ERROR_ALLOC; - - proc->blocks = new; - proc->max_blocks = new_sz; - } - - proc->blocks[proc->num_blocks].offset = offset; - proc->blocks[proc->num_blocks].hash = MK_BLK_HASH(chksum, size); - proc->num_blocks += 1; - return 0; -} - -static size_t deduplicate_blocks(sqfs_data_writer_t *proc, size_t count) -{ - size_t i, j; - - for (i = 0; i < proc->file_start; ++i) { - for (j = 0; j < count; ++j) { - if (proc->blocks[i + j].hash != - proc->blocks[proc->file_start + j].hash) - break; - } - - if (j == count) - break; - } - - return i; -} - -static int align_file(sqfs_data_writer_t *proc, sqfs_block_t *blk) -{ - sqfs_u32 chksum; - void *padding; - sqfs_u64 size; - size_t diff; - int ret; - - if (!(blk->flags & SQFS_BLK_ALIGN)) - return 0; - - size = proc->file->get_size(proc->file); - diff = size % proc->devblksz; - if (diff == 0) - return 0; - - padding = calloc(1, diff); - if (padding == 0) - return SQFS_ERROR_ALLOC; - - if (proc->hooks != NULL && proc->hooks->prepare_padding != NULL) - proc->hooks->prepare_padding(proc->user_ptr, padding, diff); - - chksum = crc32(0, padding, diff); - - ret = proc->file->write_at(proc->file, size, padding, diff); - free(padding); - if (ret) - return ret; - - return store_block_location(proc, size, diff | (1 << 24), chksum); -} - -int process_completed_block(sqfs_data_writer_t *proc, sqfs_block_t *blk) -{ - sqfs_u64 offset, bytes; - size_t start, count; - sqfs_u32 out; - int err; - - if (proc->hooks != NULL && proc->hooks->pre_block_write != NULL) { - proc->hooks->pre_block_write(proc->user_ptr, blk, proc->file); - } - - if (blk->flags & SQFS_BLK_FIRST_BLOCK) { - proc->start = proc->file->get_size(proc->file); - proc->file_start = proc->num_blocks; - - err = align_file(proc, blk); - if (err) - return err; - } - - if (blk->size != 0) { - out = blk->size; - if (!(blk->flags & SQFS_BLK_IS_COMPRESSED)) - out |= 1 << 24; - - offset = proc->file->get_size(proc->file); - - if (blk->flags & SQFS_BLK_FRAGMENT_BLOCK) { - err = sqfs_frag_table_set(proc->frag_tbl, blk->index, - offset, out); - if (err) - return err; - } else { - blk->inode->extra[blk->index] = out; - } - - err = store_block_location(proc, offset, out, blk->checksum); - if (err) - return err; - - err = proc->file->write_at(proc->file, offset, - blk->data, blk->size); - if (err) - return err; - } - - if (proc->hooks != NULL && proc->hooks->post_block_write != NULL) { - proc->hooks->post_block_write(proc->user_ptr, blk, proc->file); - } - - if (blk->flags & SQFS_BLK_LAST_BLOCK) { - err = align_file(proc, blk); - if (err) - return err; - - count = proc->num_blocks - proc->file_start; - start = deduplicate_blocks(proc, count); - offset = proc->blocks[start].offset; - - sqfs_inode_set_file_block_start(blk->inode, offset); - - if (start >= proc->file_start) - return 0; - - offset = start + count; - if (offset >= proc->file_start) { - count = proc->num_blocks - offset; - proc->num_blocks = offset; - } else { - proc->num_blocks = proc->file_start; - } - - if (proc->hooks != NULL && - proc->hooks->notify_blocks_erased != NULL) { - bytes = proc->file->get_size(proc->file) - proc->start; - - proc->hooks->notify_blocks_erased(proc->user_ptr, - count, bytes); - } - - err = proc->file->truncate(proc->file, proc->start); - if (err) - return err; - } - - return 0; -} - -int data_writer_do_block(sqfs_block_t *block, sqfs_compressor_t *cmp, - sqfs_u8 *scratch, size_t scratch_size) -{ - ssize_t ret; - - if (block->size == 0) { - block->checksum = 0; - return 0; - } - - block->checksum = crc32(0, block->data, block->size); - - if (block->flags & SQFS_BLK_IS_FRAGMENT) - return 0; - - if (!(block->flags & SQFS_BLK_DONT_COMPRESS)) { - ret = cmp->do_block(cmp, block->data, block->size, - scratch, scratch_size); - if (ret < 0) - return ret; - - if (ret > 0) { - memcpy(block->data, scratch, ret); - block->size = ret; - block->flags |= SQFS_BLK_IS_COMPRESSED; - } - } - - return 0; -} diff --git a/lib/sqfs/data_writer/common.c b/lib/sqfs/data_writer/common.c deleted file mode 100644 index b207291..0000000 --- a/lib/sqfs/data_writer/common.c +++ /dev/null @@ -1,72 +0,0 @@ -/* SPDX-License-Identifier: LGPL-3.0-or-later */ -/* - * common.c - * - * Copyright (C) 2019 David Oberhollenzer - */ -#define SQFS_BUILDING_DLL -#include "internal.h" - -void free_blk_list(sqfs_block_t *list) -{ - sqfs_block_t *it; - - while (list != NULL) { - it = list; - list = list->next; - free(it); - } -} - -int data_writer_init(sqfs_data_writer_t *proc, size_t max_block_size, - sqfs_compressor_t *cmp, unsigned int num_workers, - size_t max_backlog, size_t devblksz, sqfs_file_t *file) -{ - proc->max_block_size = max_block_size; - proc->num_workers = num_workers; - proc->max_backlog = max_backlog; - proc->devblksz = devblksz; - proc->cmp = cmp; - proc->file = file; - proc->max_blocks = INIT_BLOCK_COUNT; - - proc->frag_tbl = sqfs_frag_table_create(0); - if (proc->frag_tbl == NULL) - return -1; - - proc->blocks = alloc_array(sizeof(proc->blocks[0]), proc->max_blocks); - if (proc->blocks == NULL) - return -1; - - return 0; -} - -void data_writer_cleanup(sqfs_data_writer_t *proc) -{ - if (proc->frag_tbl != NULL) - sqfs_frag_table_destroy(proc->frag_tbl); - free_blk_list(proc->queue); - free_blk_list(proc->done); - free(proc->blk_current); - free(proc->frag_block); - free(proc->blocks); - free(proc); -} - -int sqfs_data_writer_write_fragment_table(sqfs_data_writer_t *proc, - sqfs_super_t *super) -{ - return sqfs_frag_table_write(proc->frag_tbl, proc->file, - super, proc->cmp); -} - -int sqfs_data_writer_set_hooks(sqfs_data_writer_t *proc, void *user_ptr, - const sqfs_block_hooks_t *hooks) -{ - if (hooks->size != sizeof(*hooks)) - return SQFS_ERROR_UNSUPPORTED; - - proc->hooks = hooks; - proc->user_ptr = user_ptr; - return 0; -} diff --git a/lib/sqfs/data_writer/fileapi.c b/lib/sqfs/data_writer/fileapi.c deleted file mode 100644 index 89bac0b..0000000 --- a/lib/sqfs/data_writer/fileapi.c +++ /dev/null @@ -1,188 +0,0 @@ -/* SPDX-License-Identifier: LGPL-3.0-or-later */ -/* - * fileapi.c - * - * Copyright (C) 2019 David Oberhollenzer - */ -#define SQFS_BUILDING_DLL -#include "internal.h" - -static bool is_zero_block(unsigned char *ptr, size_t size) -{ - return ptr[0] == 0 && memcmp(ptr, ptr + 1, size - 1) == 0; -} - -static int enqueue_block(sqfs_data_writer_t *proc, sqfs_block_t *block) -{ - int status; - - while (proc->backlog > proc->max_backlog) { - status = wait_completed(proc); - if (status) - return status; - } - - if (proc->backlog == proc->max_backlog) - proc->notify_threads = true; - - return append_to_work_queue(proc, block, proc->notify_threads); -} - -static int add_sentinel_block(sqfs_data_writer_t *proc) -{ - sqfs_block_t *blk = calloc(1, sizeof(*blk)); - - if (blk == NULL) - return test_and_set_status(proc, SQFS_ERROR_ALLOC); - - blk->inode = proc->inode; - blk->flags = proc->blk_flags | SQFS_BLK_LAST_BLOCK; - - return enqueue_block(proc, blk); -} - -int sqfs_data_writer_begin_file(sqfs_data_writer_t *proc, - sqfs_inode_generic_t *inode, sqfs_u32 flags) -{ - if (proc->inode != NULL) - return test_and_set_status(proc, SQFS_ERROR_INTERNAL); - - if (flags & ~SQFS_BLK_USER_SETTABLE_FLAGS) - return test_and_set_status(proc, SQFS_ERROR_UNSUPPORTED); - - proc->inode = inode; - proc->blk_flags = flags | SQFS_BLK_FIRST_BLOCK; - proc->blk_index = 0; - proc->blk_current = NULL; - return 0; -} - -static int flush_block(sqfs_data_writer_t *proc, sqfs_block_t *block) -{ - block->index = proc->blk_index++; - block->flags = proc->blk_flags; - block->inode = proc->inode; - - if (is_zero_block(block->data, block->size)) { - sqfs_inode_make_extended(proc->inode); - proc->inode->data.file_ext.sparse += block->size; - proc->inode->num_file_blocks += 1; - proc->inode->extra[block->index] = 0; - free(block); - return 0; - } - - if (block->size < proc->max_block_size && - !(block->flags & SQFS_BLK_DONT_FRAGMENT)) { - block->flags |= SQFS_BLK_IS_FRAGMENT; - } else { - proc->inode->num_file_blocks += 1; - proc->blk_flags &= ~SQFS_BLK_FIRST_BLOCK; - } - - return enqueue_block(proc, block); -} - -int sqfs_data_writer_append(sqfs_data_writer_t *proc, const void *data, - size_t size) -{ - size_t diff; - void *new; - int err; - - while (size > 0) { - if (proc->blk_current == NULL) { - new = alloc_flex(sizeof(*proc->blk_current), 1, - proc->max_block_size); - - if (new == NULL) - return test_and_set_status(proc, - SQFS_ERROR_ALLOC); - - proc->blk_current = new; - } - - diff = proc->max_block_size - proc->blk_current->size; - - if (diff == 0) { - err = flush_block(proc, proc->blk_current); - proc->blk_current = NULL; - if (err) - return err; - continue; - } - - if (diff > size) - diff = size; - - memcpy(proc->blk_current->data + proc->blk_current->size, - data, diff); - - size -= diff; - proc->blk_current->size += diff; - data = (const char *)data + diff; - } - - if (proc->blk_current != NULL && - proc->blk_current->size == proc->max_block_size) { - err = flush_block(proc, proc->blk_current); - proc->blk_current = NULL; - return err; - } - - return 0; -} - -int sqfs_data_writer_end_file(sqfs_data_writer_t *proc) -{ - int err; - - if (proc->inode == NULL) - return test_and_set_status(proc, SQFS_ERROR_INTERNAL); - - if (!(proc->blk_flags & SQFS_BLK_FIRST_BLOCK)) { - if (proc->blk_current != NULL && - (proc->blk_flags & SQFS_BLK_DONT_FRAGMENT)) { - proc->blk_flags |= SQFS_BLK_LAST_BLOCK; - } else { - err = add_sentinel_block(proc); - if (err) - return err; - } - } - - if (proc->blk_current != NULL) { - err = flush_block(proc, proc->blk_current); - proc->blk_current = NULL; - } - - proc->inode = NULL; - proc->blk_flags = 0; - proc->blk_index = 0; - return 0; -} - -int sqfs_data_writer_finish(sqfs_data_writer_t *proc) -{ - int status = 0; - - append_to_work_queue(proc, NULL, true); - - while (proc->backlog > 0) { - status = wait_completed(proc); - if (status) - return status; - } - - if (proc->frag_block != NULL) { - status = append_to_work_queue(proc, proc->frag_block, true); - proc->frag_block = NULL; - - if (status) - return status; - - status = wait_completed(proc); - } - - return status; -} diff --git a/lib/sqfs/data_writer/fragment.c b/lib/sqfs/data_writer/fragment.c deleted file mode 100644 index 9862c89..0000000 --- a/lib/sqfs/data_writer/fragment.c +++ /dev/null @@ -1,81 +0,0 @@ -/* SPDX-License-Identifier: LGPL-3.0-or-later */ -/* - * fragtbl.c - * - * Copyright (C) 2019 David Oberhollenzer - */ -#define SQFS_BUILDING_DLL -#include "internal.h" - -int process_completed_fragment(sqfs_data_writer_t *proc, sqfs_block_t *frag, - sqfs_block_t **blk_out) -{ - sqfs_u32 index, offset; - size_t size; - int err; - - err = sqfs_frag_table_find_tail_end(proc->frag_tbl, - frag->checksum, frag->size, - &index, &offset); - if (err == 0) - goto out_duplicate; - - if (proc->frag_block != NULL) { - size = proc->frag_block->size + frag->size; - - if (size > proc->max_block_size) { - *blk_out = proc->frag_block; - proc->frag_block = NULL; - } - } - - if (proc->frag_block == NULL) { - size = sizeof(sqfs_block_t) + proc->max_block_size; - - err= sqfs_frag_table_append(proc->frag_tbl, 0, 0, &index); - if (err) - goto fail; - - proc->frag_block = calloc(1, size); - if (proc->frag_block == NULL) { - err = SQFS_ERROR_ALLOC; - goto fail; - } - - proc->frag_block->index = index; - proc->frag_block->flags = SQFS_BLK_FRAGMENT_BLOCK; - } - - err = sqfs_frag_table_add_tail_end(proc->frag_tbl, - proc->frag_block->index, - proc->frag_block->size, - frag->size, frag->checksum); - if (err) - goto fail; - - sqfs_inode_set_frag_location(frag->inode, proc->frag_block->index, - proc->frag_block->size); - - if (proc->hooks != NULL && proc->hooks->pre_fragment_store != NULL) { - proc->hooks->pre_fragment_store(proc->user_ptr, frag); - } - - memcpy(proc->frag_block->data + proc->frag_block->size, - frag->data, frag->size); - - proc->frag_block->flags |= (frag->flags & SQFS_BLK_DONT_COMPRESS); - proc->frag_block->size += frag->size; - return 0; -fail: - free(*blk_out); - *blk_out = NULL; - return err; -out_duplicate: - sqfs_inode_set_frag_location(frag->inode, index, offset); - - if (proc->hooks != NULL && - proc->hooks->notify_fragment_discard != NULL) { - proc->hooks->notify_fragment_discard(proc->user_ptr, frag); - } - return 0; -} diff --git a/lib/sqfs/data_writer/internal.h b/lib/sqfs/data_writer/internal.h deleted file mode 100644 index 8f59fb7..0000000 --- a/lib/sqfs/data_writer/internal.h +++ /dev/null @@ -1,140 +0,0 @@ -/* SPDX-License-Identifier: LGPL-3.0-or-later */ -/* - * internal.h - * - * Copyright (C) 2019 David Oberhollenzer - */ -#ifndef INTERNAL_H -#define INTERNAL_H - -#include "config.h" - -#include "sqfs/data_writer.h" -#include "sqfs/frag_table.h" -#include "sqfs/compressor.h" -#include "sqfs/inode.h" -#include "sqfs/table.h" -#include "sqfs/error.h" -#include "sqfs/block.h" -#include "sqfs/io.h" -#include "../util.h" - -#include -#include -#include - -#ifdef WITH_PTHREAD -#include -#include -#elif defined(_WIN32) || defined(__WINDOWS__) -#define WIN32_LEAN_AND_MEAN -#include -#endif - - -#define MK_BLK_HASH(chksum, size) \ - (((sqfs_u64)(size) << 32) | (sqfs_u64)(chksum)) - -#define INIT_BLOCK_COUNT (128) - - -typedef struct { - sqfs_u64 offset; - sqfs_u64 hash; -} blk_info_t; - - -typedef struct compress_worker_t compress_worker_t; - -struct sqfs_data_writer_t { - /* synchronization primitives */ -#ifdef WITH_PTHREAD - pthread_mutex_t mtx; - pthread_cond_t queue_cond; - pthread_cond_t done_cond; -#elif defined(_WIN32) || defined(__WINDOWS__) - CRITICAL_SECTION mtx; - CONDITION_VARIABLE queue_cond; - CONDITION_VARIABLE done_cond; -#endif - - /* needs rw access by worker and main thread */ - sqfs_block_t *queue; - sqfs_block_t *queue_last; - - sqfs_block_t *done; - size_t backlog; - int status; - - /* used by main thread only */ - sqfs_u32 enqueue_id; - sqfs_u32 dequeue_id; - - unsigned int num_workers; - size_t max_backlog; - - size_t devblksz; - sqfs_file_t *file; - - sqfs_frag_table_t *frag_tbl; - - sqfs_u64 start; - - size_t file_start; - size_t num_blocks; - size_t max_blocks; - blk_info_t *blocks; - sqfs_compressor_t *cmp; - - sqfs_block_t *frag_block; - - const sqfs_block_hooks_t *hooks; - void *user_ptr; - bool notify_threads; - - /* file API */ - sqfs_inode_generic_t *inode; - sqfs_block_t *blk_current; - sqfs_u32 blk_flags; - size_t blk_index; - - /* used only by workers */ - size_t max_block_size; - -#if defined(WITH_PTHREAD) || defined(_WIN32) || defined(__WINDOWS__) - compress_worker_t *workers[]; -#else - sqfs_u8 scratch[]; -#endif -}; - -SQFS_INTERNAL int process_completed_block(sqfs_data_writer_t *proc, - sqfs_block_t *block); - -SQFS_INTERNAL -int process_completed_fragment(sqfs_data_writer_t *proc, sqfs_block_t *frag, - sqfs_block_t **blk_out); - -SQFS_INTERNAL void free_blk_list(sqfs_block_t *list); - -SQFS_INTERNAL -int data_writer_init(sqfs_data_writer_t *proc, size_t max_block_size, - sqfs_compressor_t *cmp, unsigned int num_workers, - size_t max_backlog, size_t devblksz, sqfs_file_t *file); - -SQFS_INTERNAL void data_writer_cleanup(sqfs_data_writer_t *proc); - -SQFS_INTERNAL -int data_writer_do_block(sqfs_block_t *block, sqfs_compressor_t *cmp, - sqfs_u8 *scratch, size_t scratch_size); - -SQFS_INTERNAL -int test_and_set_status(sqfs_data_writer_t *proc, int status); - -SQFS_INTERNAL -int append_to_work_queue(sqfs_data_writer_t *proc, sqfs_block_t *block, - bool notify_threads); - -SQFS_INTERNAL int wait_completed(sqfs_data_writer_t *proc); - -#endif /* INTERNAL_H */ diff --git a/lib/sqfs/data_writer/serial.c b/lib/sqfs/data_writer/serial.c deleted file mode 100644 index 82f7836..0000000 --- a/lib/sqfs/data_writer/serial.c +++ /dev/null @@ -1,88 +0,0 @@ -/* SPDX-License-Identifier: LGPL-3.0-or-later */ -/* - * serial.c - * - * Copyright (C) 2019 David Oberhollenzer - */ -#define SQFS_BUILDING_DLL -#include "internal.h" - -sqfs_data_writer_t *sqfs_data_writer_create(size_t max_block_size, - sqfs_compressor_t *cmp, - unsigned int num_workers, - size_t max_backlog, - size_t devblksz, - sqfs_file_t *file) -{ - sqfs_data_writer_t *proc; - - proc = alloc_flex(sizeof(*proc), 1, max_block_size); - - if (proc == NULL) - return NULL; - - if (data_writer_init(proc, max_block_size, cmp, num_workers, - max_backlog, devblksz, file)) { - data_writer_cleanup(proc); - return NULL; - } - - return proc; -} - -void sqfs_data_writer_destroy(sqfs_data_writer_t *proc) -{ - data_writer_cleanup(proc); -} - -int test_and_set_status(sqfs_data_writer_t *proc, int status) -{ - if (proc->status == 0) - proc->status = status; - - return proc->status; -} - -int append_to_work_queue(sqfs_data_writer_t *proc, sqfs_block_t *block, - bool signal_threads) -{ - sqfs_block_t *fragblk = NULL; - (void)signal_threads; - - if (proc->status != 0 || block == NULL) { - free(block); - return proc->status; - } - - if (block->flags & SQFS_BLK_IS_FRAGMENT) { - block->checksum = crc32(0, block->data, block->size); - - proc->status = process_completed_fragment(proc, block, - &fragblk); - free(block); - - if (proc->status != 0) { - free(fragblk); - return proc->status; - } - - if (fragblk == NULL) - return 0; - - block = fragblk; - } - - proc->status = data_writer_do_block(block, proc->cmp, proc->scratch, - proc->max_block_size); - - if (proc->status == 0) - proc->status = process_completed_block(proc, block); - - free(block); - return proc->status; -} - -int wait_completed(sqfs_data_writer_t *proc) -{ - return proc->status; -} diff --git a/lib/sqfs/data_writer/winpthread.c b/lib/sqfs/data_writer/winpthread.c deleted file mode 100644 index e575859..0000000 --- a/lib/sqfs/data_writer/winpthread.c +++ /dev/null @@ -1,450 +0,0 @@ -/* SPDX-License-Identifier: LGPL-3.0-or-later */ -/* - * winpthread.c - * - * Copyright (C) 2019 David Oberhollenzer - */ -#define SQFS_BUILDING_DLL -#include "internal.h" - -#if defined(_WIN32) || defined(__WINDOWS__) -# define LOCK(mtx) EnterCriticalSection(mtx) -# define UNLOCK(mtx) LeaveCriticalSection(mtx) -# define AWAIT(cond, mtx) SleepConditionVariableCS(cond, mtx, INFINITE) -# define SIGNAL_ALL(cond) WakeAllConditionVariable(cond) -# define THREAD_EXIT_SUCCESS 0 -# define THREAD_TYPE DWORD WINAPI -# define THREAD_ARG LPVOID -# define THREAD_HANDLE HANDLE -#else -# define LOCK(mtx) pthread_mutex_lock(mtx) -# define UNLOCK(mtx) pthread_mutex_unlock(mtx) -# define AWAIT(cond, mtx) pthread_cond_wait(cond, mtx) -# define SIGNAL_ALL(cond) pthread_cond_broadcast(cond) -# define THREAD_EXIT_SUCCESS NULL -# define THREAD_TYPE void * -# define THREAD_ARG void * -# define THREAD_HANDLE pthread_t -#endif - -struct compress_worker_t { - sqfs_data_writer_t *shared; - sqfs_compressor_t *cmp; - THREAD_HANDLE thread; - sqfs_u8 scratch[]; -}; - -static THREAD_TYPE worker_proc(THREAD_ARG arg) -{ - compress_worker_t *worker = arg; - sqfs_data_writer_t *shared = worker->shared; - sqfs_block_t *it, *prev, *blk = NULL; - int status = 0; - - for (;;) { - LOCK(&shared->mtx); - if (blk != NULL) { - it = shared->done; - prev = NULL; - - while (it != NULL) { - if (it->sequence_number >= blk->sequence_number) - break; - prev = it; - it = it->next; - } - - if (prev == NULL) { - blk->next = shared->done; - shared->done = blk; - } else { - blk->next = prev->next; - prev->next = blk; - } - - if (status != 0 && shared->status == 0) - shared->status = status; - SIGNAL_ALL(&shared->done_cond); - } - - while (shared->queue == NULL && shared->status == 0) - AWAIT(&shared->queue_cond, &shared->mtx); - - if (shared->status == 0) { - blk = shared->queue; - shared->queue = blk->next; - blk->next = NULL; - - if (shared->queue == NULL) - shared->queue_last = NULL; - } else { - blk = NULL; - } - UNLOCK(&shared->mtx); - - if (blk == NULL) - break; - - status = data_writer_do_block(blk, worker->cmp, - worker->scratch, - shared->max_block_size); - } - - return THREAD_EXIT_SUCCESS; -} - -#if defined(_WIN32) || defined(__WINDOWS__) -sqfs_data_writer_t *sqfs_data_writer_create(size_t max_block_size, - sqfs_compressor_t *cmp, - unsigned int num_workers, - size_t max_backlog, - size_t devblksz, - sqfs_file_t *file) -{ - sqfs_data_writer_t *proc; - unsigned int i; - - if (num_workers < 1) - num_workers = 1; - - proc = alloc_flex(sizeof(*proc), - sizeof(proc->workers[0]), num_workers); - if (proc == NULL) - return NULL; - - InitializeCriticalSection(&proc->mtx); - InitializeConditionVariable(&proc->queue_cond); - InitializeConditionVariable(&proc->done_cond); - - if (data_writer_init(proc, max_block_size, cmp, num_workers, - max_backlog, devblksz, file)) { - goto fail; - } - - for (i = 0; i < num_workers; ++i) { - proc->workers[i] = alloc_flex(sizeof(compress_worker_t), - 1, max_block_size); - - if (proc->workers[i] == NULL) - goto fail; - - proc->workers[i]->shared = proc; - proc->workers[i]->cmp = cmp->create_copy(cmp); - - if (proc->workers[i]->cmp == NULL) - goto fail; - - proc->workers[i]->thread = CreateThread(NULL, 0, worker_proc, - proc->workers[i], 0, 0); - if (proc->workers[i]->thread == NULL) - goto fail; - } - - return proc; -fail: - sqfs_data_writer_destroy(proc); - return NULL; -} - -void sqfs_data_writer_destroy(sqfs_data_writer_t *proc) -{ - unsigned int i; - - EnterCriticalSection(&proc->mtx); - proc->status = -1; - WakeAllConditionVariable(&proc->queue_cond); - LeaveCriticalSection(&proc->mtx); - - for (i = 0; i < proc->num_workers; ++i) { - if (proc->workers[i] == NULL) - continue; - - if (proc->workers[i]->thread != NULL) { - WaitForSingleObject(proc->workers[i]->thread, INFINITE); - CloseHandle(proc->workers[i]->thread); - } - - if (proc->workers[i]->cmp != NULL) - proc->workers[i]->cmp->destroy(proc->workers[i]->cmp); - - free(proc->workers[i]); - } - - DeleteCriticalSection(&proc->mtx); - data_writer_cleanup(proc); -} -#else -sqfs_data_writer_t *sqfs_data_writer_create(size_t max_block_size, - sqfs_compressor_t *cmp, - unsigned int num_workers, - size_t max_backlog, - size_t devblksz, - sqfs_file_t *file) -{ - sqfs_data_writer_t *proc; - sigset_t set, oldset; - unsigned int i; - int ret; - - if (num_workers < 1) - num_workers = 1; - - proc = alloc_flex(sizeof(*proc), - sizeof(proc->workers[0]), num_workers); - if (proc == NULL) - return NULL; - - proc->mtx = (pthread_mutex_t)PTHREAD_MUTEX_INITIALIZER; - proc->queue_cond = (pthread_cond_t)PTHREAD_COND_INITIALIZER; - proc->done_cond = (pthread_cond_t)PTHREAD_COND_INITIALIZER; - - if (data_writer_init(proc, max_block_size, cmp, num_workers, - max_backlog, devblksz, file)) { - goto fail_init; - } - - for (i = 0; i < num_workers; ++i) { - proc->workers[i] = alloc_flex(sizeof(compress_worker_t), - 1, max_block_size); - - if (proc->workers[i] == NULL) - goto fail_init; - - proc->workers[i]->shared = proc; - proc->workers[i]->cmp = cmp->create_copy(cmp); - - if (proc->workers[i]->cmp == NULL) - goto fail_init; - } - - sigfillset(&set); - pthread_sigmask(SIG_SETMASK, &set, &oldset); - - for (i = 0; i < num_workers; ++i) { - ret = pthread_create(&proc->workers[i]->thread, NULL, - worker_proc, proc->workers[i]); - - if (ret != 0) - goto fail_thread; - } - - pthread_sigmask(SIG_SETMASK, &oldset, NULL); - - return proc; -fail_thread: - pthread_mutex_lock(&proc->mtx); - proc->status = -1; - pthread_cond_broadcast(&proc->queue_cond); - pthread_mutex_unlock(&proc->mtx); - - for (i = 0; i < num_workers; ++i) { - if (proc->workers[i]->thread > 0) { - pthread_join(proc->workers[i]->thread, NULL); - } - } - pthread_sigmask(SIG_SETMASK, &oldset, NULL); -fail_init: - for (i = 0; i < num_workers; ++i) { - if (proc->workers[i] != NULL) { - if (proc->workers[i]->cmp != NULL) { - proc->workers[i]->cmp-> - destroy(proc->workers[i]->cmp); - } - - free(proc->workers[i]); - } - } - pthread_cond_destroy(&proc->done_cond); - pthread_cond_destroy(&proc->queue_cond); - pthread_mutex_destroy(&proc->mtx); - data_writer_cleanup(proc); - return NULL; -} - -void sqfs_data_writer_destroy(sqfs_data_writer_t *proc) -{ - unsigned int i; - - pthread_mutex_lock(&proc->mtx); - proc->status = -1; - pthread_cond_broadcast(&proc->queue_cond); - pthread_mutex_unlock(&proc->mtx); - - for (i = 0; i < proc->num_workers; ++i) { - pthread_join(proc->workers[i]->thread, NULL); - - proc->workers[i]->cmp->destroy(proc->workers[i]->cmp); - free(proc->workers[i]); - } - - pthread_cond_destroy(&proc->done_cond); - pthread_cond_destroy(&proc->queue_cond); - pthread_mutex_destroy(&proc->mtx); - - data_writer_cleanup(proc); -} -#endif - -int append_to_work_queue(sqfs_data_writer_t *proc, sqfs_block_t *block, - bool signal_threads) -{ - int status; - - LOCK(&proc->mtx); - status = proc->status; - if (status != 0) - goto out; - - if (block != NULL) { - if (proc->queue_last == NULL) { - proc->queue = proc->queue_last = block; - } else { - proc->queue_last->next = block; - proc->queue_last = block; - } - - block->sequence_number = proc->enqueue_id++; - block->next = NULL; - proc->backlog += 1; - block = NULL; - } -out: - if (signal_threads) - SIGNAL_ALL(&proc->queue_cond); - - UNLOCK(&proc->mtx); - free(block); - return 0; -} - -static sqfs_block_t *try_dequeue(sqfs_data_writer_t *proc) -{ - sqfs_block_t *queue, *it, *prev; - - it = proc->done; - prev = NULL; - - while (it != NULL && it->sequence_number == proc->dequeue_id) { - prev = it; - it = it->next; - proc->dequeue_id += 1; - } - - if (prev == NULL) { - queue = NULL; - } else { - queue = proc->done; - prev->next = NULL; - proc->done = it; - } - - return queue; -} - -static sqfs_block_t *queue_merge(sqfs_block_t *lhs, sqfs_block_t *rhs) -{ - sqfs_block_t *it, *head = NULL, **next_ptr = &head; - - while (lhs != NULL && rhs != NULL) { - if (lhs->sequence_number <= rhs->sequence_number) { - it = lhs; - lhs = lhs->next; - } else { - it = rhs; - rhs = rhs->next; - } - - *next_ptr = it; - next_ptr = &it->next; - } - - it = (lhs != NULL ? lhs : rhs); - *next_ptr = it; - return head; -} - -static int process_done_queue(sqfs_data_writer_t *proc, sqfs_block_t *queue) -{ - sqfs_block_t *it, *block = NULL; - int status = 0; - - while (queue != NULL && status == 0) { - it = queue; - queue = it->next; - proc->backlog -= 1; - - if (it->flags & SQFS_BLK_IS_FRAGMENT) { - block = NULL; - status = process_completed_fragment(proc, it, &block); - - if (block != NULL && status == 0) { - LOCK(&proc->mtx); - proc->dequeue_id = it->sequence_number; - block->sequence_number = it->sequence_number; - - if (proc->queue == NULL) { - proc->queue = block; - proc->queue_last = block; - } else { - block->next = proc->queue; - proc->queue = block; - } - - proc->backlog += 1; - proc->done = queue_merge(queue, proc->done); - SIGNAL_ALL(&proc->queue_cond); - UNLOCK(&proc->mtx); - - queue = NULL; - } else { - free(block); - } - } else { - status = process_completed_block(proc, it); - } - - free(it); - } - - free_blk_list(queue); - return status; -} - -int test_and_set_status(sqfs_data_writer_t *proc, int status) -{ - LOCK(&proc->mtx); - if (proc->status == 0) { - proc->status = status; - } else { - status = proc->status; - } - SIGNAL_ALL(&proc->queue_cond); - UNLOCK(&proc->mtx); - return status; -} - -int wait_completed(sqfs_data_writer_t *proc) -{ - sqfs_block_t *queue; - int status; - - LOCK(&proc->mtx); - for (;;) { - queue = try_dequeue(proc); - status = proc->status; - - if (queue != NULL || status != 0) - break; - - AWAIT(&proc->done_cond, &proc->mtx); - } - UNLOCK(&proc->mtx); - - if (status != 0) { - free_blk_list(queue); - return status; - } - - status = process_done_queue(proc, queue); - return status ? test_and_set_status(proc, status) : status; -} diff --git a/mkfs/mkfs.c b/mkfs/mkfs.c index b5d3443..bccbd79 100644 --- a/mkfs/mkfs.c +++ b/mkfs/mkfs.c @@ -39,8 +39,8 @@ static int set_working_dir(options_t *opt) return 0; } -static int pack_files(sqfs_data_writer_t *data, fstree_t *fs, - data_writer_stats_t *stats, options_t *opt) +static int pack_files(sqfs_block_processor_t *data, fstree_t *fs, + block_processor_stats_t *stats, options_t *opt) { sqfs_inode_generic_t *inode; size_t max_blk_count; -- cgit v1.2.3