From 940c3e3333ba0063f536dfbecbb77d070dbcc87a Mon Sep 17 00:00:00 2001 From: David Oberhollenzer Date: Fri, 31 Jan 2020 17:15:04 +0100 Subject: Split the block writing/deduplication away from the block processor This commit moves the entire block writing and deduplication of data blocks over to a different data type named "block writer". For simplicity, the interfaces of the block processor are left as is and are turned into warppers. Likewise, most of the code in the block writer is just verbatim from the block processor, to be cleaned up in subsequent commits. Signed-off-by: David Oberhollenzer --- lib/sqfs/Makemodule.am | 3 +- lib/sqfs/block_processor/block.c | 155 ++---------------------- lib/sqfs/block_processor/common.c | 16 +-- lib/sqfs/block_processor/internal.h | 24 +--- lib/sqfs/block_writer.c | 235 ++++++++++++++++++++++++++++++++++++ 5 files changed, 258 insertions(+), 175 deletions(-) create mode 100644 lib/sqfs/block_writer.c (limited to 'lib/sqfs') diff --git a/lib/sqfs/Makemodule.am b/lib/sqfs/Makemodule.am index cde947d..644ced2 100644 --- a/lib/sqfs/Makemodule.am +++ b/lib/sqfs/Makemodule.am @@ -8,7 +8,7 @@ LIBSQFS_HEARDS = include/sqfs/meta_writer.h \ include/sqfs/dir_writer.h include/sqfs/io.h \ include/sqfs/data_reader.h include/sqfs/block.h \ include/sqfs/xattr_reader.h include/sqfs/xattr_writer.h \ - include/sqfs/frag_table.h + include/sqfs/frag_table.h include/sqfs/block_writer.h libsquashfs_la_SOURCES = $(LIBSQFS_HEARDS) lib/sqfs/id_table.c lib/sqfs/super.c libsquashfs_la_SOURCES += lib/sqfs/readdir.c lib/sqfs/xattr.c @@ -28,6 +28,7 @@ libsquashfs_la_SOURCES += lib/sqfs/block_processor/fileapi.c libsquashfs_la_SOURCES += lib/sqfs/str_table.c lib/sqfs/str_table.h libsquashfs_la_SOURCES += lib/sqfs/alloc.c lib/sqfs/util.h libsquashfs_la_SOURCES += lib/sqfs/frag_table.c include/sqfs/frag_table.h +libsquashfs_la_SOURCES += lib/sqfs/block_writer.c include/sqfs/block_writer.h libsquashfs_la_CPPFLAGS = $(AM_CPPFLAGS) libsquashfs_la_LDFLAGS = $(AM_LDFLAGS) libsquashfs_la_CFLAGS = $(AM_CFLAGS) $(PTHREAD_CFLAGS) $(ZLIB_CFLAGS) diff --git a/lib/sqfs/block_processor/block.c b/lib/sqfs/block_processor/block.c index 49892be..e88456a 100644 --- a/lib/sqfs/block_processor/block.c +++ b/lib/sqfs/block_processor/block.c @@ -9,164 +9,33 @@ #include -static int store_block_location(sqfs_block_processor_t *proc, sqfs_u64 offset, - sqfs_u32 size, sqfs_u32 chksum) -{ - size_t new_sz; - void *new; - - if (proc->num_blocks == proc->max_blocks) { - new_sz = proc->max_blocks * 2; - new = realloc(proc->blocks, sizeof(proc->blocks[0]) * new_sz); - - if (new == NULL) - return SQFS_ERROR_ALLOC; - - proc->blocks = new; - proc->max_blocks = new_sz; - } - - proc->blocks[proc->num_blocks].offset = offset; - proc->blocks[proc->num_blocks].hash = MK_BLK_HASH(chksum, size); - proc->num_blocks += 1; - return 0; -} - -static size_t deduplicate_blocks(sqfs_block_processor_t *proc, size_t count) -{ - size_t i, j; - - for (i = 0; i < proc->file_start; ++i) { - for (j = 0; j < count; ++j) { - if (proc->blocks[i + j].hash != - proc->blocks[proc->file_start + j].hash) - break; - } - - if (j == count) - break; - } - - return i; -} - -static int align_file(sqfs_block_processor_t *proc, sqfs_block_t *blk) -{ - sqfs_u32 chksum; - void *padding; - sqfs_u64 size; - size_t diff; - int ret; - - if (!(blk->flags & SQFS_BLK_ALIGN)) - return 0; - - size = proc->file->get_size(proc->file); - diff = size % proc->devblksz; - if (diff == 0) - return 0; - - padding = calloc(1, diff); - if (padding == 0) - return SQFS_ERROR_ALLOC; - - if (proc->hooks != NULL && proc->hooks->prepare_padding != NULL) - proc->hooks->prepare_padding(proc->user_ptr, padding, diff); - - chksum = crc32(0, padding, diff); - - ret = proc->file->write_at(proc->file, size, padding, diff); - free(padding); - if (ret) - return ret; - - return store_block_location(proc, size, diff | (1 << 24), chksum); -} - int process_completed_block(sqfs_block_processor_t *proc, sqfs_block_t *blk) { - sqfs_u64 offset, bytes; - size_t start, count; - sqfs_u32 out; + sqfs_u64 location; + sqfs_u32 size; int err; - if (proc->hooks != NULL && proc->hooks->pre_block_write != NULL) { - proc->hooks->pre_block_write(proc->user_ptr, blk, proc->file); - } - - if (blk->flags & SQFS_BLK_FIRST_BLOCK) { - proc->start = proc->file->get_size(proc->file); - proc->file_start = proc->num_blocks; + size = blk->size; + if (!(blk->flags & SQFS_BLK_IS_COMPRESSED)) + size |= 1 << 24; - err = align_file(proc, blk); - if (err) - return err; - } + err = sqfs_block_writer_write(proc->wr, blk, &location); + if (err) + return err; if (blk->size != 0) { - out = blk->size; - if (!(blk->flags & SQFS_BLK_IS_COMPRESSED)) - out |= 1 << 24; - - offset = proc->file->get_size(proc->file); - if (blk->flags & SQFS_BLK_FRAGMENT_BLOCK) { err = sqfs_frag_table_set(proc->frag_tbl, blk->index, - offset, out); + location, size); if (err) return err; } else { - blk->inode->extra[blk->index] = out; + blk->inode->extra[blk->index] = size; } - - err = store_block_location(proc, offset, out, blk->checksum); - if (err) - return err; - - err = proc->file->write_at(proc->file, offset, - blk->data, blk->size); - if (err) - return err; - } - - if (proc->hooks != NULL && proc->hooks->post_block_write != NULL) { - proc->hooks->post_block_write(proc->user_ptr, blk, proc->file); } - if (blk->flags & SQFS_BLK_LAST_BLOCK) { - err = align_file(proc, blk); - if (err) - return err; - - count = proc->num_blocks - proc->file_start; - start = deduplicate_blocks(proc, count); - offset = proc->blocks[start].offset; - - sqfs_inode_set_file_block_start(blk->inode, offset); - - if (start >= proc->file_start) - return 0; - - offset = start + count; - if (offset >= proc->file_start) { - count = proc->num_blocks - offset; - proc->num_blocks = offset; - } else { - proc->num_blocks = proc->file_start; - } - - if (proc->hooks != NULL && - proc->hooks->notify_blocks_erased != NULL) { - bytes = proc->file->get_size(proc->file) - proc->start; - - proc->hooks->notify_blocks_erased(proc->user_ptr, - count, bytes); - } - - err = proc->file->truncate(proc->file, proc->start); - if (err) - return err; - } + if (blk->flags & SQFS_BLK_LAST_BLOCK) + sqfs_inode_set_file_block_start(blk->inode, location); return 0; } diff --git a/lib/sqfs/block_processor/common.c b/lib/sqfs/block_processor/common.c index c6375dd..806a595 100644 --- a/lib/sqfs/block_processor/common.c +++ b/lib/sqfs/block_processor/common.c @@ -25,19 +25,16 @@ int block_processor_init(sqfs_block_processor_t *proc, size_t max_block_size, proc->max_block_size = max_block_size; proc->num_workers = num_workers; proc->max_backlog = max_backlog; - proc->devblksz = devblksz; proc->cmp = cmp; proc->file = file; - proc->max_blocks = INIT_BLOCK_COUNT; proc->frag_tbl = sqfs_frag_table_create(0); if (proc->frag_tbl == NULL) return -1; - proc->blocks = alloc_array(sizeof(proc->blocks[0]), proc->max_blocks); - if (proc->blocks == NULL) + proc->wr = sqfs_block_writer_create(file, devblksz, 0); + if (proc->wr == NULL) return -1; - return 0; } @@ -45,11 +42,12 @@ void block_processor_cleanup(sqfs_block_processor_t *proc) { if (proc->frag_tbl != NULL) sqfs_frag_table_destroy(proc->frag_tbl); + if (proc->wr != NULL) + sqfs_block_writer_destroy(proc->wr); free_blk_list(proc->queue); free_blk_list(proc->done); free(proc->blk_current); free(proc->frag_block); - free(proc->blocks); free(proc); } @@ -63,10 +61,8 @@ int sqfs_block_processor_write_fragment_table(sqfs_block_processor_t *proc, int sqfs_block_processor_set_hooks(sqfs_block_processor_t *proc, void *user_ptr, const sqfs_block_hooks_t *hooks) { - if (hooks->size != sizeof(*hooks)) - return SQFS_ERROR_UNSUPPORTED; - proc->hooks = hooks; proc->user_ptr = user_ptr; - return 0; + + return sqfs_block_writer_set_hooks(proc->wr, user_ptr, hooks); } diff --git a/lib/sqfs/block_processor/internal.h b/lib/sqfs/block_processor/internal.h index 40871b9..16f0edb 100644 --- a/lib/sqfs/block_processor/internal.h +++ b/lib/sqfs/block_processor/internal.h @@ -10,6 +10,7 @@ #include "config.h" #include "sqfs/block_processor.h" +#include "sqfs/block_writer.h" #include "sqfs/frag_table.h" #include "sqfs/compressor.h" #include "sqfs/inode.h" @@ -31,19 +32,6 @@ #include #endif - -#define MK_BLK_HASH(chksum, size) \ - (((sqfs_u64)(size) << 32) | (sqfs_u64)(chksum)) - -#define INIT_BLOCK_COUNT (128) - - -typedef struct { - sqfs_u64 offset; - sqfs_u64 hash; -} blk_info_t; - - typedef struct compress_worker_t compress_worker_t; struct sqfs_block_processor_t { @@ -73,24 +61,18 @@ struct sqfs_block_processor_t { unsigned int num_workers; size_t max_backlog; - size_t devblksz; sqfs_file_t *file; sqfs_frag_table_t *frag_tbl; - sqfs_u64 start; - - size_t file_start; - size_t num_blocks; - size_t max_blocks; - blk_info_t *blocks; sqfs_compressor_t *cmp; sqfs_block_t *frag_block; + bool notify_threads; + sqfs_block_writer_t *wr; const sqfs_block_hooks_t *hooks; void *user_ptr; - bool notify_threads; /* file API */ sqfs_inode_generic_t *inode; diff --git a/lib/sqfs/block_writer.c b/lib/sqfs/block_writer.c new file mode 100644 index 0000000..c955532 --- /dev/null +++ b/lib/sqfs/block_writer.c @@ -0,0 +1,235 @@ +/* SPDX-License-Identifier: LGPL-3.0-or-later */ +/* + * block_writer.c + * + * Copyright (C) 2019 David Oberhollenzer + */ +#define SQFS_BUILDING_DLL +#include "config.h" + +#include "sqfs/block_writer.h" +#include "sqfs/error.h" +#include "sqfs/block.h" +#include "sqfs/io.h" +#include "util.h" + +#include + +#define MK_BLK_HASH(chksum, size) \ + (((sqfs_u64)(size) << 32) | (sqfs_u64)(chksum)) + +#define INIT_BLOCK_COUNT (128) + +typedef struct { + sqfs_u64 offset; + sqfs_u64 hash; +} blk_info_t; + +struct sqfs_block_writer_t { + sqfs_file_t *file; + + size_t num_blocks; + size_t max_blocks; + blk_info_t *blocks; + size_t devblksz; + + const sqfs_block_hooks_t *hooks; + void *user_ptr; + + sqfs_u64 start; + size_t file_start; +}; + +static int store_block_location(sqfs_block_writer_t *wr, sqfs_u64 offset, + sqfs_u32 size, sqfs_u32 chksum) +{ + size_t new_sz; + void *new; + + if (wr->num_blocks == wr->max_blocks) { + new_sz = wr->max_blocks * 2; + new = realloc(wr->blocks, sizeof(wr->blocks[0]) * new_sz); + + if (new == NULL) + return SQFS_ERROR_ALLOC; + + wr->blocks = new; + wr->max_blocks = new_sz; + } + + wr->blocks[wr->num_blocks].offset = offset; + wr->blocks[wr->num_blocks].hash = MK_BLK_HASH(chksum, size); + wr->num_blocks += 1; + return 0; +} + +static size_t deduplicate_blocks(sqfs_block_writer_t *wr, size_t count) +{ + size_t i, j; + + for (i = 0; i < wr->file_start; ++i) { + for (j = 0; j < count; ++j) { + if (wr->blocks[i + j].hash == 0) + break; + + if (wr->blocks[i + j].hash != + wr->blocks[wr->file_start + j].hash) + break; + } + + if (j == count) + break; + } + + return i; +} + +static int align_file(sqfs_block_writer_t *wr, sqfs_block_t *blk) +{ + void *padding; + sqfs_u64 size; + size_t diff; + int ret; + + if (!(blk->flags & SQFS_BLK_ALIGN)) + return 0; + + size = wr->file->get_size(wr->file); + diff = size % wr->devblksz; + if (diff == 0) + return 0; + + padding = calloc(1, diff); + if (padding == 0) + return SQFS_ERROR_ALLOC; + + if (wr->hooks != NULL && wr->hooks->prepare_padding != NULL) + wr->hooks->prepare_padding(wr->user_ptr, padding, diff); + + ret = wr->file->write_at(wr->file, size, padding, diff); + free(padding); + if (ret) + return ret; + + return store_block_location(wr, size, 0, 0); +} + +sqfs_block_writer_t *sqfs_block_writer_create(sqfs_file_t *file, + size_t devblksz, sqfs_u32 flags) +{ + sqfs_block_writer_t *wr; + + if (flags != 0) + return NULL; + + wr = calloc(1, sizeof(*wr)); + if (wr == NULL) + return NULL; + + wr->file = file; + wr->devblksz = devblksz; + wr->max_blocks = INIT_BLOCK_COUNT; + + wr->blocks = alloc_array(sizeof(wr->blocks[0]), wr->max_blocks); + if (wr->blocks == NULL) { + free(wr); + return NULL; + } + + return wr; +} + +int sqfs_block_writer_set_hooks(sqfs_block_writer_t *wr, void *user_ptr, + const sqfs_block_hooks_t *hooks) +{ + if (hooks->size != sizeof(*hooks)) + return SQFS_ERROR_UNSUPPORTED; + + wr->hooks = hooks; + wr->user_ptr = user_ptr; + return 0; +} + +void sqfs_block_writer_destroy(sqfs_block_writer_t *wr) +{ + free(wr->blocks); + free(wr); +} + +int sqfs_block_writer_write(sqfs_block_writer_t *wr, sqfs_block_t *block, + sqfs_u64 *location) +{ + sqfs_u64 offset, bytes; + size_t start, count; + sqfs_u32 out; + int err; + + if (wr->hooks != NULL && wr->hooks->pre_block_write != NULL) + wr->hooks->pre_block_write(wr->user_ptr, block, wr->file); + + if (block->flags & SQFS_BLK_FIRST_BLOCK) { + wr->start = wr->file->get_size(wr->file); + wr->file_start = wr->num_blocks; + + err = align_file(wr, block); + if (err) + return err; + } + + if (block->size != 0) { + out = block->size; + if (!(block->flags & SQFS_BLK_IS_COMPRESSED)) + out |= 1 << 24; + + offset = wr->file->get_size(wr->file); + *location = offset; + + err = store_block_location(wr, offset, out, block->checksum); + if (err) + return err; + + err = wr->file->write_at(wr->file, offset, + block->data, block->size); + if (err) + return err; + } + + if (wr->hooks != NULL && wr->hooks->post_block_write != NULL) + wr->hooks->post_block_write(wr->user_ptr, block, wr->file); + + if (block->flags & SQFS_BLK_LAST_BLOCK) { + err = align_file(wr, block); + if (err) + return err; + + count = wr->num_blocks - wr->file_start; + start = deduplicate_blocks(wr, count); + offset = wr->blocks[start].offset; + + *location = offset; + if (start >= wr->file_start) + return 0; + + offset = start + count; + if (offset >= wr->file_start) { + count = wr->num_blocks - offset; + wr->num_blocks = offset; + } else { + wr->num_blocks = wr->file_start; + } + + if (wr->hooks != NULL && + wr->hooks->notify_blocks_erased != NULL) { + bytes = wr->file->get_size(wr->file) - wr->start; + + wr->hooks->notify_blocks_erased(wr->user_ptr, + count, bytes); + } + + err = wr->file->truncate(wr->file, wr->start); + if (err) + return err; + } + + return 0; +} -- cgit v1.2.3