From 9434bdfaedb733e1441d87fedde8056efeda7145 Mon Sep 17 00:00:00 2001 From: David Oberhollenzer Date: Wed, 25 Sep 2019 06:50:03 +0200 Subject: More refactoring of the block processor Basically just moving functions around and renaming things. Signed-off-by: David Oberhollenzer --- lib/sqfs/Makemodule.am | 5 +- lib/sqfs/blk_proc/block.c | 131 ++++++++++++++++++++++++++++++++++++++ lib/sqfs/blk_proc/deduplicate.c | 49 -------------- lib/sqfs/blk_proc/fragment.c | 128 +++++++++++++++++++++++++++++++++++++ lib/sqfs/blk_proc/fragtbl.c | 128 ------------------------------------- lib/sqfs/blk_proc/internal.h | 11 +--- lib/sqfs/blk_proc/process_block.c | 90 -------------------------- lib/sqfs/blk_proc/pthread.c | 2 +- lib/sqfs/blk_proc/serial.c | 3 +- 9 files changed, 266 insertions(+), 281 deletions(-) create mode 100644 lib/sqfs/blk_proc/block.c delete mode 100644 lib/sqfs/blk_proc/deduplicate.c create mode 100644 lib/sqfs/blk_proc/fragment.c delete mode 100644 lib/sqfs/blk_proc/fragtbl.c delete mode 100644 lib/sqfs/blk_proc/process_block.c (limited to 'lib/sqfs') diff --git a/lib/sqfs/Makemodule.am b/lib/sqfs/Makemodule.am index 7a02fa4..e8b9284 100644 --- a/lib/sqfs/Makemodule.am +++ b/lib/sqfs/Makemodule.am @@ -17,10 +17,9 @@ libsquashfs_la_SOURCES += lib/sqfs/dir_writer.c lib/sqfs/xattr_reader.c libsquashfs_la_SOURCES += lib/sqfs/read_table.c lib/sqfs/comp/compressor.c libsquashfs_la_SOURCES += lib/sqfs/io_stdin.c lib/sqfs/comp/internal.h libsquashfs_la_SOURCES += lib/sqfs/dir_reader.c lib/sqfs/read_tree.c -libsquashfs_la_SOURCES += lib/sqfs/inode.c lib/sqfs/blk_proc/fragtbl.c -libsquashfs_la_SOURCES += lib/sqfs/blk_proc/process_block.c lib/sqfs/io.c +libsquashfs_la_SOURCES += lib/sqfs/inode.c lib/sqfs/blk_proc/fragment.c +libsquashfs_la_SOURCES += lib/sqfs/blk_proc/block.c lib/sqfs/io.c libsquashfs_la_SOURCES += lib/sqfs/blk_proc/internal.h lib/sqfs/data_reader.c -libsquashfs_la_SOURCES += lib/sqfs/blk_proc/deduplicate.c libsquashfs_la_SOURCES += lib/sqfs/blk_proc/common.c libsquashfs_la_CPPFLAGS = $(AM_CPPFLAGS) libsquashfs_la_CFLAGS = $(AM_CFLAGS) $(PTHREAD_CFLAGS) $(ZLIB_CFLAGS) diff --git a/lib/sqfs/blk_proc/block.c b/lib/sqfs/blk_proc/block.c new file mode 100644 index 0000000..69a7119 --- /dev/null +++ b/lib/sqfs/blk_proc/block.c @@ -0,0 +1,131 @@ +/* SPDX-License-Identifier: LGPL-3.0-or-later */ +/* + * process_block.c + * + * Copyright (C) 2019 David Oberhollenzer + */ +#define SQFS_BUILDING_DLL +#include "internal.h" + +#include + +static int store_block_location(sqfs_block_processor_t *proc, uint64_t offset, + uint32_t size, uint32_t chksum) +{ + size_t new_sz; + void *new; + + if (proc->num_blocks == proc->max_blocks) { + new_sz = proc->max_blocks * 2; + new = realloc(proc->blocks, sizeof(proc->blocks[0]) * new_sz); + + if (new == NULL) + return SQFS_ERROR_ALLOC; + + proc->blocks = new; + proc->max_blocks = new_sz; + } + + proc->blocks[proc->num_blocks].offset = offset; + proc->blocks[proc->num_blocks].hash = MK_BLK_HASH(chksum, size); + proc->num_blocks += 1; + return 0; +} + +static size_t deduplicate_blocks(sqfs_block_processor_t *proc, size_t count) +{ + size_t i, j; + + for (i = 0; i < proc->file_start; ++i) { + for (j = 0; j < count; ++j) { + if (proc->blocks[i + j].hash != + proc->blocks[proc->file_start + j].hash) + break; + } + + if (j == count) + break; + } + + return i; +} + +static int allign_file(sqfs_block_processor_t *proc, sqfs_block_t *blk) +{ + if (!(blk->flags & SQFS_BLK_ALLIGN)) + return 0; + + return padd_sqfs(proc->file, proc->file->get_size(proc->file), + proc->devblksz); +} + +int process_completed_block(sqfs_block_processor_t *proc, sqfs_block_t *blk) +{ + size_t start, count; + uint64_t offset; + uint32_t out; + int err; + + if (blk->flags & SQFS_BLK_FIRST_BLOCK) { + proc->start = proc->file->get_size(proc->file); + proc->file_start = proc->num_blocks; + + err = allign_file(proc, blk); + if (err) + return err; + } + + if (blk->size != 0) { + out = blk->size; + if (!(blk->flags & SQFS_BLK_IS_COMPRESSED)) + out |= 1 << 24; + + offset = proc->file->get_size(proc->file); + + if (blk->flags & SQFS_BLK_FRAGMENT_BLOCK) { + offset = htole64(offset); + proc->fragments[blk->index].start_offset = offset; + proc->fragments[blk->index].pad0 = 0; + proc->fragments[blk->index].size = htole32(out); + } else { + blk->inode->block_sizes[blk->index] = out; + } + + err = store_block_location(proc, offset, out, blk->checksum); + if (err) + return err; + + err = proc->file->write_at(proc->file, offset, + blk->data, blk->size); + if (err) + return err; + } + + if (blk->flags & SQFS_BLK_LAST_BLOCK) { + err = allign_file(proc, blk); + if (err) + return err; + + count = proc->num_blocks - proc->file_start; + start = deduplicate_blocks(proc, count); + offset = proc->blocks[start].offset; + + sqfs_inode_set_file_block_start(blk->inode, offset); + + if (start < proc->file_start) { + offset = start + count; + + if (offset >= proc->file_start) { + proc->num_blocks = offset; + } else { + proc->num_blocks = proc->file_start; + } + + err = proc->file->truncate(proc->file, proc->start); + if (err) + return err; + } + } + + return 0; +} diff --git a/lib/sqfs/blk_proc/deduplicate.c b/lib/sqfs/blk_proc/deduplicate.c deleted file mode 100644 index 08a108e..0000000 --- a/lib/sqfs/blk_proc/deduplicate.c +++ /dev/null @@ -1,49 +0,0 @@ -/* SPDX-License-Identifier: LGPL-3.0-or-later */ -/* - * deduplicate.c - * - * Copyright (C) 2019 David Oberhollenzer - */ -#define SQFS_BUILDING_DLL -#include "internal.h" - -int store_block_location(sqfs_block_processor_t *proc, uint64_t offset, - uint32_t size, uint32_t chksum) -{ - size_t new_sz; - void *new; - - if (proc->num_blocks == proc->max_blocks) { - new_sz = proc->max_blocks * 2; - new = realloc(proc->blocks, sizeof(proc->blocks[0]) * new_sz); - - if (new == NULL) - return SQFS_ERROR_ALLOC; - - proc->blocks = new; - proc->max_blocks = new_sz; - } - - proc->blocks[proc->num_blocks].offset = offset; - proc->blocks[proc->num_blocks].hash = MK_BLK_HASH(chksum, size); - proc->num_blocks += 1; - return 0; -} - -size_t deduplicate_blocks(sqfs_block_processor_t *proc, size_t count) -{ - size_t i, j; - - for (i = 0; i < proc->file_start; ++i) { - for (j = 0; j < count; ++j) { - if (proc->blocks[i + j].hash != - proc->blocks[proc->file_start + j].hash) - break; - } - - if (j == count) - break; - } - - return i; -} diff --git a/lib/sqfs/blk_proc/fragment.c b/lib/sqfs/blk_proc/fragment.c new file mode 100644 index 0000000..5fa2349 --- /dev/null +++ b/lib/sqfs/blk_proc/fragment.c @@ -0,0 +1,128 @@ +/* SPDX-License-Identifier: LGPL-3.0-or-later */ +/* + * fragtbl.c + * + * Copyright (C) 2019 David Oberhollenzer + */ +#define SQFS_BUILDING_DLL +#include "internal.h" + +static int grow_fragment_table(sqfs_block_processor_t *proc) +{ + size_t newsz; + void *new; + + if (proc->num_fragments >= proc->max_fragments) { + newsz = proc->max_fragments ? proc->max_fragments * 2 : 16; + + new = realloc(proc->fragments, + sizeof(proc->fragments[0]) * newsz); + + if (new == NULL) + return SQFS_ERROR_ALLOC; + + proc->max_fragments = newsz; + proc->fragments = new; + } + + return 0; +} + +static int grow_deduplication_list(sqfs_block_processor_t *proc) +{ + size_t new_sz; + void *new; + + if (proc->frag_list_num == proc->frag_list_max) { + new_sz = proc->frag_list_max * 2; + new = realloc(proc->frag_list, + sizeof(proc->frag_list[0]) * new_sz); + + if (new == NULL) + return SQFS_ERROR_ALLOC; + + proc->frag_list = new; + proc->frag_list_max = new_sz; + } + + return 0; +} + +static int store_fragment(sqfs_block_processor_t *proc, sqfs_block_t *frag, + uint64_t hash) +{ + int err = grow_deduplication_list(proc); + + if (err) + return err; + + proc->frag_list[proc->frag_list_num].index = proc->frag_block->index; + proc->frag_list[proc->frag_list_num].offset = proc->frag_block->size; + proc->frag_list[proc->frag_list_num].hash = hash; + proc->frag_list_num += 1; + + sqfs_inode_set_frag_location(frag->inode, proc->frag_block->index, + proc->frag_block->size); + + memcpy(proc->frag_block->data + proc->frag_block->size, + frag->data, frag->size); + + proc->frag_block->flags |= (frag->flags & SQFS_BLK_DONT_COMPRESS); + proc->frag_block->size += frag->size; + return 0; +} + +int process_completed_fragment(sqfs_block_processor_t *proc, sqfs_block_t *frag, + sqfs_block_t **blk_out) +{ + uint64_t hash; + size_t i, size; + int err; + + hash = MK_BLK_HASH(frag->checksum, frag->size); + + for (i = 0; i < proc->frag_list_num; ++i) { + if (proc->frag_list[i].hash == hash) { + sqfs_inode_set_frag_location(frag->inode, + proc->frag_list[i].index, + proc->frag_list[i].offset); + return 0; + } + } + + if (proc->frag_block != NULL) { + size = proc->frag_block->size + frag->size; + + if (size > proc->max_block_size) { + *blk_out = proc->frag_block; + proc->frag_block = NULL; + } + } + + if (proc->frag_block == NULL) { + size = sizeof(sqfs_block_t) + proc->max_block_size; + + err = grow_fragment_table(proc); + if (err) + goto fail; + + proc->frag_block = calloc(1, size); + if (proc->frag_block == NULL) { + err = SQFS_ERROR_ALLOC; + goto fail; + } + + proc->frag_block->index = proc->num_fragments++; + proc->frag_block->flags = SQFS_BLK_FRAGMENT_BLOCK; + } + + err = store_fragment(proc, frag, hash); + if (err) + goto fail; + + return 0; +fail: + free(*blk_out); + *blk_out = NULL; + return err; +} diff --git a/lib/sqfs/blk_proc/fragtbl.c b/lib/sqfs/blk_proc/fragtbl.c deleted file mode 100644 index 39e8136..0000000 --- a/lib/sqfs/blk_proc/fragtbl.c +++ /dev/null @@ -1,128 +0,0 @@ -/* SPDX-License-Identifier: LGPL-3.0-or-later */ -/* - * fragtbl.c - * - * Copyright (C) 2019 David Oberhollenzer - */ -#define SQFS_BUILDING_DLL -#include "internal.h" - -static int grow_fragment_table(sqfs_block_processor_t *proc) -{ - size_t newsz; - void *new; - - if (proc->num_fragments >= proc->max_fragments) { - newsz = proc->max_fragments ? proc->max_fragments * 2 : 16; - - new = realloc(proc->fragments, - sizeof(proc->fragments[0]) * newsz); - - if (new == NULL) - return SQFS_ERROR_ALLOC; - - proc->max_fragments = newsz; - proc->fragments = new; - } - - return 0; -} - -static int grow_deduplication_list(sqfs_block_processor_t *proc) -{ - size_t new_sz; - void *new; - - if (proc->frag_list_num == proc->frag_list_max) { - new_sz = proc->frag_list_max * 2; - new = realloc(proc->frag_list, - sizeof(proc->frag_list[0]) * new_sz); - - if (new == NULL) - return SQFS_ERROR_ALLOC; - - proc->frag_list = new; - proc->frag_list_max = new_sz; - } - - return 0; -} - -static int store_fragment(sqfs_block_processor_t *proc, sqfs_block_t *frag, - uint64_t hash) -{ - int err = grow_deduplication_list(proc); - - if (err) - return err; - - proc->frag_list[proc->frag_list_num].index = proc->frag_block->index; - proc->frag_list[proc->frag_list_num].offset = proc->frag_block->size; - proc->frag_list[proc->frag_list_num].hash = hash; - proc->frag_list_num += 1; - - sqfs_inode_set_frag_location(frag->inode, proc->frag_block->index, - proc->frag_block->size); - - memcpy(proc->frag_block->data + proc->frag_block->size, - frag->data, frag->size); - - proc->frag_block->flags |= (frag->flags & SQFS_BLK_DONT_COMPRESS); - proc->frag_block->size += frag->size; - return 0; -} - -int handle_fragment(sqfs_block_processor_t *proc, sqfs_block_t *frag, - sqfs_block_t **blk_out) -{ - uint64_t hash; - size_t i, size; - int err; - - hash = MK_BLK_HASH(frag->checksum, frag->size); - - for (i = 0; i < proc->frag_list_num; ++i) { - if (proc->frag_list[i].hash == hash) { - sqfs_inode_set_frag_location(frag->inode, - proc->frag_list[i].index, - proc->frag_list[i].offset); - return 0; - } - } - - if (proc->frag_block != NULL) { - size = proc->frag_block->size + frag->size; - - if (size > proc->max_block_size) { - *blk_out = proc->frag_block; - proc->frag_block = NULL; - } - } - - if (proc->frag_block == NULL) { - size = sizeof(sqfs_block_t) + proc->max_block_size; - - err = grow_fragment_table(proc); - if (err) - goto fail; - - proc->frag_block = calloc(1, size); - if (proc->frag_block == NULL) { - err = SQFS_ERROR_ALLOC; - goto fail; - } - - proc->frag_block->index = proc->num_fragments++; - proc->frag_block->flags = SQFS_BLK_FRAGMENT_BLOCK; - } - - err = store_fragment(proc, frag, hash); - if (err) - goto fail; - - return 0; -fail: - free(*blk_out); - *blk_out = NULL; - return err; -} diff --git a/lib/sqfs/blk_proc/internal.h b/lib/sqfs/blk_proc/internal.h index b6a0342..1c403a9 100644 --- a/lib/sqfs/blk_proc/internal.h +++ b/lib/sqfs/blk_proc/internal.h @@ -111,15 +111,8 @@ SQFS_INTERNAL int process_completed_block(sqfs_block_processor_t *proc, sqfs_block_t *block); SQFS_INTERNAL -int handle_fragment(sqfs_block_processor_t *proc, sqfs_block_t *frag, - sqfs_block_t **blk_out); - -SQFS_INTERNAL size_t deduplicate_blocks(sqfs_block_processor_t *proc, - size_t count); - -SQFS_INTERNAL int store_block_location(sqfs_block_processor_t *proc, - uint64_t offset, uint32_t size, - uint32_t chksum); +int process_completed_fragment(sqfs_block_processor_t *proc, sqfs_block_t *frag, + sqfs_block_t **blk_out); SQFS_INTERNAL void free_blk_list(sqfs_block_t *list); diff --git a/lib/sqfs/blk_proc/process_block.c b/lib/sqfs/blk_proc/process_block.c deleted file mode 100644 index 2951406..0000000 --- a/lib/sqfs/blk_proc/process_block.c +++ /dev/null @@ -1,90 +0,0 @@ -/* SPDX-License-Identifier: LGPL-3.0-or-later */ -/* - * process_block.c - * - * Copyright (C) 2019 David Oberhollenzer - */ -#define SQFS_BUILDING_DLL -#include "internal.h" - -#include - -static int allign_file(sqfs_block_processor_t *proc, sqfs_block_t *blk) -{ - if (!(blk->flags & SQFS_BLK_ALLIGN)) - return 0; - - return padd_sqfs(proc->file, proc->file->get_size(proc->file), - proc->devblksz); -} - -int process_completed_block(sqfs_block_processor_t *proc, sqfs_block_t *blk) -{ - size_t start, count; - uint64_t offset; - uint32_t out; - int err; - - if (blk->flags & SQFS_BLK_FIRST_BLOCK) { - proc->start = proc->file->get_size(proc->file); - proc->file_start = proc->num_blocks; - - err = allign_file(proc, blk); - if (err) - return err; - } - - if (blk->size != 0) { - out = blk->size; - if (!(blk->flags & SQFS_BLK_IS_COMPRESSED)) - out |= 1 << 24; - - offset = proc->file->get_size(proc->file); - - if (blk->flags & SQFS_BLK_FRAGMENT_BLOCK) { - offset = htole64(offset); - proc->fragments[blk->index].start_offset = offset; - proc->fragments[blk->index].pad0 = 0; - proc->fragments[blk->index].size = htole32(out); - } else { - blk->inode->block_sizes[blk->index] = out; - } - - err = store_block_location(proc, offset, out, blk->checksum); - if (err) - return err; - - err = proc->file->write_at(proc->file, offset, - blk->data, blk->size); - if (err) - return err; - } - - if (blk->flags & SQFS_BLK_LAST_BLOCK) { - err = allign_file(proc, blk); - if (err) - return err; - - count = proc->num_blocks - proc->file_start; - start = deduplicate_blocks(proc, count); - offset = proc->blocks[start].offset; - - sqfs_inode_set_file_block_start(blk->inode, offset); - - if (start < proc->file_start) { - offset = start + count; - - if (offset >= proc->file_start) { - proc->num_blocks = offset; - } else { - proc->num_blocks = proc->file_start; - } - - err = proc->file->truncate(proc->file, proc->start); - if (err) - return err; - } - } - - return 0; -} diff --git a/lib/sqfs/blk_proc/pthread.c b/lib/sqfs/blk_proc/pthread.c index 57b61c1..7b95c7c 100644 --- a/lib/sqfs/blk_proc/pthread.c +++ b/lib/sqfs/blk_proc/pthread.c @@ -226,7 +226,7 @@ static int process_done_queue(sqfs_block_processor_t *proc, if (it->flags & SQFS_BLK_IS_FRAGMENT) { block = NULL; - status = handle_fragment(proc, it, &block); + status = process_completed_fragment(proc, it, &block); if (block != NULL && status == 0) { pthread_mutex_lock(&proc->mtx); diff --git a/lib/sqfs/blk_proc/serial.c b/lib/sqfs/blk_proc/serial.c index 67272a3..fcbf002 100644 --- a/lib/sqfs/blk_proc/serial.c +++ b/lib/sqfs/blk_proc/serial.c @@ -54,7 +54,8 @@ int sqfs_block_processor_enqueue(sqfs_block_processor_t *proc, if (block->flags & SQFS_BLK_IS_FRAGMENT) { block->checksum = crc32(0, block->data, block->size); - proc->status = handle_fragment(proc, block, &fragblk); + proc->status = process_completed_fragment(proc, block, + &fragblk); free(block); if (proc->status != 0) { -- cgit v1.2.3