diff options
Diffstat (limited to 'lib/sqfs/data_writer')
| -rw-r--r-- | lib/sqfs/data_writer/block.c | 203 | ||||
| -rw-r--r-- | lib/sqfs/data_writer/common.c | 72 | ||||
| -rw-r--r-- | lib/sqfs/data_writer/fileapi.c | 188 | ||||
| -rw-r--r-- | lib/sqfs/data_writer/fragment.c | 81 | ||||
| -rw-r--r-- | lib/sqfs/data_writer/internal.h | 140 | ||||
| -rw-r--r-- | lib/sqfs/data_writer/serial.c | 88 | ||||
| -rw-r--r-- | lib/sqfs/data_writer/winpthread.c | 450 |
7 files changed, 0 insertions, 1222 deletions
diff --git a/lib/sqfs/data_writer/block.c b/lib/sqfs/data_writer/block.c deleted file mode 100644 index b45b1ff..0000000 --- a/lib/sqfs/data_writer/block.c +++ /dev/null @@ -1,203 +0,0 @@ -/* SPDX-License-Identifier: LGPL-3.0-or-later */ -/* - * process_block.c - * - * Copyright (C) 2019 David Oberhollenzer <goliath@infraroot.at> - */ -#define SQFS_BUILDING_DLL -#include "internal.h" - -#include <string.h> - -static int store_block_location(sqfs_data_writer_t *proc, sqfs_u64 offset, - sqfs_u32 size, sqfs_u32 chksum) -{ - size_t new_sz; - void *new; - - if (proc->num_blocks == proc->max_blocks) { - new_sz = proc->max_blocks * 2; - new = realloc(proc->blocks, sizeof(proc->blocks[0]) * new_sz); - - if (new == NULL) - return SQFS_ERROR_ALLOC; - - proc->blocks = new; - proc->max_blocks = new_sz; - } - - proc->blocks[proc->num_blocks].offset = offset; - proc->blocks[proc->num_blocks].hash = MK_BLK_HASH(chksum, size); - proc->num_blocks += 1; - return 0; -} - -static size_t deduplicate_blocks(sqfs_data_writer_t *proc, size_t count) -{ - size_t i, j; - - for (i = 0; i < proc->file_start; ++i) { - for (j = 0; j < count; ++j) { - if (proc->blocks[i + j].hash != - proc->blocks[proc->file_start + j].hash) - break; - } - - if (j == count) - break; - } - - return i; -} - -static int align_file(sqfs_data_writer_t *proc, sqfs_block_t *blk) -{ - sqfs_u32 chksum; - void *padding; - sqfs_u64 size; - size_t diff; - int ret; - - if (!(blk->flags & SQFS_BLK_ALIGN)) - return 0; - - size = proc->file->get_size(proc->file); - diff = size % proc->devblksz; - if (diff == 0) - return 0; - - padding = calloc(1, diff); - if (padding == 0) - return SQFS_ERROR_ALLOC; - - if (proc->hooks != NULL && proc->hooks->prepare_padding != NULL) - proc->hooks->prepare_padding(proc->user_ptr, padding, diff); - - chksum = crc32(0, padding, diff); - - ret = proc->file->write_at(proc->file, size, padding, diff); - free(padding); - if (ret) - return ret; - - return store_block_location(proc, size, diff | (1 << 24), chksum); -} - -int process_completed_block(sqfs_data_writer_t *proc, sqfs_block_t *blk) -{ - sqfs_u64 offset, bytes; - size_t start, count; - sqfs_u32 out; - int err; - - if (proc->hooks != NULL && proc->hooks->pre_block_write != NULL) { - proc->hooks->pre_block_write(proc->user_ptr, blk, proc->file); - } - - if (blk->flags & SQFS_BLK_FIRST_BLOCK) { - proc->start = proc->file->get_size(proc->file); - proc->file_start = proc->num_blocks; - - err = align_file(proc, blk); - if (err) - return err; - } - - if (blk->size != 0) { - out = blk->size; - if (!(blk->flags & SQFS_BLK_IS_COMPRESSED)) - out |= 1 << 24; - - offset = proc->file->get_size(proc->file); - - if (blk->flags & SQFS_BLK_FRAGMENT_BLOCK) { - err = sqfs_frag_table_set(proc->frag_tbl, blk->index, - offset, out); - if (err) - return err; - } else { - blk->inode->extra[blk->index] = out; - } - - err = store_block_location(proc, offset, out, blk->checksum); - if (err) - return err; - - err = proc->file->write_at(proc->file, offset, - blk->data, blk->size); - if (err) - return err; - } - - if (proc->hooks != NULL && proc->hooks->post_block_write != NULL) { - proc->hooks->post_block_write(proc->user_ptr, blk, proc->file); - } - - if (blk->flags & SQFS_BLK_LAST_BLOCK) { - err = align_file(proc, blk); - if (err) - return err; - - count = proc->num_blocks - proc->file_start; - start = deduplicate_blocks(proc, count); - offset = proc->blocks[start].offset; - - sqfs_inode_set_file_block_start(blk->inode, offset); - - if (start >= proc->file_start) - return 0; - - offset = start + count; - if (offset >= proc->file_start) { - count = proc->num_blocks - offset; - proc->num_blocks = offset; - } else { - proc->num_blocks = proc->file_start; - } - - if (proc->hooks != NULL && - proc->hooks->notify_blocks_erased != NULL) { - bytes = proc->file->get_size(proc->file) - proc->start; - - proc->hooks->notify_blocks_erased(proc->user_ptr, - count, bytes); - } - - err = proc->file->truncate(proc->file, proc->start); - if (err) - return err; - } - - return 0; -} - -int data_writer_do_block(sqfs_block_t *block, sqfs_compressor_t *cmp, - sqfs_u8 *scratch, size_t scratch_size) -{ - ssize_t ret; - - if (block->size == 0) { - block->checksum = 0; - return 0; - } - - block->checksum = crc32(0, block->data, block->size); - - if (block->flags & SQFS_BLK_IS_FRAGMENT) - return 0; - - if (!(block->flags & SQFS_BLK_DONT_COMPRESS)) { - ret = cmp->do_block(cmp, block->data, block->size, - scratch, scratch_size); - if (ret < 0) - return ret; - - if (ret > 0) { - memcpy(block->data, scratch, ret); - block->size = ret; - block->flags |= SQFS_BLK_IS_COMPRESSED; - } - } - - return 0; -} diff --git a/lib/sqfs/data_writer/common.c b/lib/sqfs/data_writer/common.c deleted file mode 100644 index b207291..0000000 --- a/lib/sqfs/data_writer/common.c +++ /dev/null @@ -1,72 +0,0 @@ -/* SPDX-License-Identifier: LGPL-3.0-or-later */ -/* - * common.c - * - * Copyright (C) 2019 David Oberhollenzer <goliath@infraroot.at> - */ -#define SQFS_BUILDING_DLL -#include "internal.h" - -void free_blk_list(sqfs_block_t *list) -{ - sqfs_block_t *it; - - while (list != NULL) { - it = list; - list = list->next; - free(it); - } -} - -int data_writer_init(sqfs_data_writer_t *proc, size_t max_block_size, - sqfs_compressor_t *cmp, unsigned int num_workers, - size_t max_backlog, size_t devblksz, sqfs_file_t *file) -{ - proc->max_block_size = max_block_size; - proc->num_workers = num_workers; - proc->max_backlog = max_backlog; - proc->devblksz = devblksz; - proc->cmp = cmp; - proc->file = file; - proc->max_blocks = INIT_BLOCK_COUNT; - - proc->frag_tbl = sqfs_frag_table_create(0); - if (proc->frag_tbl == NULL) - return -1; - - proc->blocks = alloc_array(sizeof(proc->blocks[0]), proc->max_blocks); - if (proc->blocks == NULL) - return -1; - - return 0; -} - -void data_writer_cleanup(sqfs_data_writer_t *proc) -{ - if (proc->frag_tbl != NULL) - sqfs_frag_table_destroy(proc->frag_tbl); - free_blk_list(proc->queue); - free_blk_list(proc->done); - free(proc->blk_current); - free(proc->frag_block); - free(proc->blocks); - free(proc); -} - -int sqfs_data_writer_write_fragment_table(sqfs_data_writer_t *proc, - sqfs_super_t *super) -{ - return sqfs_frag_table_write(proc->frag_tbl, proc->file, - super, proc->cmp); -} - -int sqfs_data_writer_set_hooks(sqfs_data_writer_t *proc, void *user_ptr, - const sqfs_block_hooks_t *hooks) -{ - if (hooks->size != sizeof(*hooks)) - return SQFS_ERROR_UNSUPPORTED; - - proc->hooks = hooks; - proc->user_ptr = user_ptr; - return 0; -} diff --git a/lib/sqfs/data_writer/fileapi.c b/lib/sqfs/data_writer/fileapi.c deleted file mode 100644 index 89bac0b..0000000 --- a/lib/sqfs/data_writer/fileapi.c +++ /dev/null @@ -1,188 +0,0 @@ -/* SPDX-License-Identifier: LGPL-3.0-or-later */ -/* - * fileapi.c - * - * Copyright (C) 2019 David Oberhollenzer <goliath@infraroot.at> - */ -#define SQFS_BUILDING_DLL -#include "internal.h" - -static bool is_zero_block(unsigned char *ptr, size_t size) -{ - return ptr[0] == 0 && memcmp(ptr, ptr + 1, size - 1) == 0; -} - -static int enqueue_block(sqfs_data_writer_t *proc, sqfs_block_t *block) -{ - int status; - - while (proc->backlog > proc->max_backlog) { - status = wait_completed(proc); - if (status) - return status; - } - - if (proc->backlog == proc->max_backlog) - proc->notify_threads = true; - - return append_to_work_queue(proc, block, proc->notify_threads); -} - -static int add_sentinel_block(sqfs_data_writer_t *proc) -{ - sqfs_block_t *blk = calloc(1, sizeof(*blk)); - - if (blk == NULL) - return test_and_set_status(proc, SQFS_ERROR_ALLOC); - - blk->inode = proc->inode; - blk->flags = proc->blk_flags | SQFS_BLK_LAST_BLOCK; - - return enqueue_block(proc, blk); -} - -int sqfs_data_writer_begin_file(sqfs_data_writer_t *proc, - sqfs_inode_generic_t *inode, sqfs_u32 flags) -{ - if (proc->inode != NULL) - return test_and_set_status(proc, SQFS_ERROR_INTERNAL); - - if (flags & ~SQFS_BLK_USER_SETTABLE_FLAGS) - return test_and_set_status(proc, SQFS_ERROR_UNSUPPORTED); - - proc->inode = inode; - proc->blk_flags = flags | SQFS_BLK_FIRST_BLOCK; - proc->blk_index = 0; - proc->blk_current = NULL; - return 0; -} - -static int flush_block(sqfs_data_writer_t *proc, sqfs_block_t *block) -{ - block->index = proc->blk_index++; - block->flags = proc->blk_flags; - block->inode = proc->inode; - - if (is_zero_block(block->data, block->size)) { - sqfs_inode_make_extended(proc->inode); - proc->inode->data.file_ext.sparse += block->size; - proc->inode->num_file_blocks += 1; - proc->inode->extra[block->index] = 0; - free(block); - return 0; - } - - if (block->size < proc->max_block_size && - !(block->flags & SQFS_BLK_DONT_FRAGMENT)) { - block->flags |= SQFS_BLK_IS_FRAGMENT; - } else { - proc->inode->num_file_blocks += 1; - proc->blk_flags &= ~SQFS_BLK_FIRST_BLOCK; - } - - return enqueue_block(proc, block); -} - -int sqfs_data_writer_append(sqfs_data_writer_t *proc, const void *data, - size_t size) -{ - size_t diff; - void *new; - int err; - - while (size > 0) { - if (proc->blk_current == NULL) { - new = alloc_flex(sizeof(*proc->blk_current), 1, - proc->max_block_size); - - if (new == NULL) - return test_and_set_status(proc, - SQFS_ERROR_ALLOC); - - proc->blk_current = new; - } - - diff = proc->max_block_size - proc->blk_current->size; - - if (diff == 0) { - err = flush_block(proc, proc->blk_current); - proc->blk_current = NULL; - if (err) - return err; - continue; - } - - if (diff > size) - diff = size; - - memcpy(proc->blk_current->data + proc->blk_current->size, - data, diff); - - size -= diff; - proc->blk_current->size += diff; - data = (const char *)data + diff; - } - - if (proc->blk_current != NULL && - proc->blk_current->size == proc->max_block_size) { - err = flush_block(proc, proc->blk_current); - proc->blk_current = NULL; - return err; - } - - return 0; -} - -int sqfs_data_writer_end_file(sqfs_data_writer_t *proc) -{ - int err; - - if (proc->inode == NULL) - return test_and_set_status(proc, SQFS_ERROR_INTERNAL); - - if (!(proc->blk_flags & SQFS_BLK_FIRST_BLOCK)) { - if (proc->blk_current != NULL && - (proc->blk_flags & SQFS_BLK_DONT_FRAGMENT)) { - proc->blk_flags |= SQFS_BLK_LAST_BLOCK; - } else { - err = add_sentinel_block(proc); - if (err) - return err; - } - } - - if (proc->blk_current != NULL) { - err = flush_block(proc, proc->blk_current); - proc->blk_current = NULL; - } - - proc->inode = NULL; - proc->blk_flags = 0; - proc->blk_index = 0; - return 0; -} - -int sqfs_data_writer_finish(sqfs_data_writer_t *proc) -{ - int status = 0; - - append_to_work_queue(proc, NULL, true); - - while (proc->backlog > 0) { - status = wait_completed(proc); - if (status) - return status; - } - - if (proc->frag_block != NULL) { - status = append_to_work_queue(proc, proc->frag_block, true); - proc->frag_block = NULL; - - if (status) - return status; - - status = wait_completed(proc); - } - - return status; -} diff --git a/lib/sqfs/data_writer/fragment.c b/lib/sqfs/data_writer/fragment.c deleted file mode 100644 index 9862c89..0000000 --- a/lib/sqfs/data_writer/fragment.c +++ /dev/null @@ -1,81 +0,0 @@ -/* SPDX-License-Identifier: LGPL-3.0-or-later */ -/* - * fragtbl.c - * - * Copyright (C) 2019 David Oberhollenzer <goliath@infraroot.at> - */ -#define SQFS_BUILDING_DLL -#include "internal.h" - -int process_completed_fragment(sqfs_data_writer_t *proc, sqfs_block_t *frag, - sqfs_block_t **blk_out) -{ - sqfs_u32 index, offset; - size_t size; - int err; - - err = sqfs_frag_table_find_tail_end(proc->frag_tbl, - frag->checksum, frag->size, - &index, &offset); - if (err == 0) - goto out_duplicate; - - if (proc->frag_block != NULL) { - size = proc->frag_block->size + frag->size; - - if (size > proc->max_block_size) { - *blk_out = proc->frag_block; - proc->frag_block = NULL; - } - } - - if (proc->frag_block == NULL) { - size = sizeof(sqfs_block_t) + proc->max_block_size; - - err= sqfs_frag_table_append(proc->frag_tbl, 0, 0, &index); - if (err) - goto fail; - - proc->frag_block = calloc(1, size); - if (proc->frag_block == NULL) { - err = SQFS_ERROR_ALLOC; - goto fail; - } - - proc->frag_block->index = index; - proc->frag_block->flags = SQFS_BLK_FRAGMENT_BLOCK; - } - - err = sqfs_frag_table_add_tail_end(proc->frag_tbl, - proc->frag_block->index, - proc->frag_block->size, - frag->size, frag->checksum); - if (err) - goto fail; - - sqfs_inode_set_frag_location(frag->inode, proc->frag_block->index, - proc->frag_block->size); - - if (proc->hooks != NULL && proc->hooks->pre_fragment_store != NULL) { - proc->hooks->pre_fragment_store(proc->user_ptr, frag); - } - - memcpy(proc->frag_block->data + proc->frag_block->size, - frag->data, frag->size); - - proc->frag_block->flags |= (frag->flags & SQFS_BLK_DONT_COMPRESS); - proc->frag_block->size += frag->size; - return 0; -fail: - free(*blk_out); - *blk_out = NULL; - return err; -out_duplicate: - sqfs_inode_set_frag_location(frag->inode, index, offset); - - if (proc->hooks != NULL && - proc->hooks->notify_fragment_discard != NULL) { - proc->hooks->notify_fragment_discard(proc->user_ptr, frag); - } - return 0; -} diff --git a/lib/sqfs/data_writer/internal.h b/lib/sqfs/data_writer/internal.h deleted file mode 100644 index 8f59fb7..0000000 --- a/lib/sqfs/data_writer/internal.h +++ /dev/null @@ -1,140 +0,0 @@ -/* SPDX-License-Identifier: LGPL-3.0-or-later */ -/* - * internal.h - * - * Copyright (C) 2019 David Oberhollenzer <goliath@infraroot.at> - */ -#ifndef INTERNAL_H -#define INTERNAL_H - -#include "config.h" - -#include "sqfs/data_writer.h" -#include "sqfs/frag_table.h" -#include "sqfs/compressor.h" -#include "sqfs/inode.h" -#include "sqfs/table.h" -#include "sqfs/error.h" -#include "sqfs/block.h" -#include "sqfs/io.h" -#include "../util.h" - -#include <string.h> -#include <stdlib.h> -#include <zlib.h> - -#ifdef WITH_PTHREAD -#include <pthread.h> -#include <signal.h> -#elif defined(_WIN32) || defined(__WINDOWS__) -#define WIN32_LEAN_AND_MEAN -#include <windows.h> -#endif - - -#define MK_BLK_HASH(chksum, size) \ - (((sqfs_u64)(size) << 32) | (sqfs_u64)(chksum)) - -#define INIT_BLOCK_COUNT (128) - - -typedef struct { - sqfs_u64 offset; - sqfs_u64 hash; -} blk_info_t; - - -typedef struct compress_worker_t compress_worker_t; - -struct sqfs_data_writer_t { - /* synchronization primitives */ -#ifdef WITH_PTHREAD - pthread_mutex_t mtx; - pthread_cond_t queue_cond; - pthread_cond_t done_cond; -#elif defined(_WIN32) || defined(__WINDOWS__) - CRITICAL_SECTION mtx; - CONDITION_VARIABLE queue_cond; - CONDITION_VARIABLE done_cond; -#endif - - /* needs rw access by worker and main thread */ - sqfs_block_t *queue; - sqfs_block_t *queue_last; - - sqfs_block_t *done; - size_t backlog; - int status; - - /* used by main thread only */ - sqfs_u32 enqueue_id; - sqfs_u32 dequeue_id; - - unsigned int num_workers; - size_t max_backlog; - - size_t devblksz; - sqfs_file_t *file; - - sqfs_frag_table_t *frag_tbl; - - sqfs_u64 start; - - size_t file_start; - size_t num_blocks; - size_t max_blocks; - blk_info_t *blocks; - sqfs_compressor_t *cmp; - - sqfs_block_t *frag_block; - - const sqfs_block_hooks_t *hooks; - void *user_ptr; - bool notify_threads; - - /* file API */ - sqfs_inode_generic_t *inode; - sqfs_block_t *blk_current; - sqfs_u32 blk_flags; - size_t blk_index; - - /* used only by workers */ - size_t max_block_size; - -#if defined(WITH_PTHREAD) || defined(_WIN32) || defined(__WINDOWS__) - compress_worker_t *workers[]; -#else - sqfs_u8 scratch[]; -#endif -}; - -SQFS_INTERNAL int process_completed_block(sqfs_data_writer_t *proc, - sqfs_block_t *block); - -SQFS_INTERNAL -int process_completed_fragment(sqfs_data_writer_t *proc, sqfs_block_t *frag, - sqfs_block_t **blk_out); - -SQFS_INTERNAL void free_blk_list(sqfs_block_t *list); - -SQFS_INTERNAL -int data_writer_init(sqfs_data_writer_t *proc, size_t max_block_size, - sqfs_compressor_t *cmp, unsigned int num_workers, - size_t max_backlog, size_t devblksz, sqfs_file_t *file); - -SQFS_INTERNAL void data_writer_cleanup(sqfs_data_writer_t *proc); - -SQFS_INTERNAL -int data_writer_do_block(sqfs_block_t *block, sqfs_compressor_t *cmp, - sqfs_u8 *scratch, size_t scratch_size); - -SQFS_INTERNAL -int test_and_set_status(sqfs_data_writer_t *proc, int status); - -SQFS_INTERNAL -int append_to_work_queue(sqfs_data_writer_t *proc, sqfs_block_t *block, - bool notify_threads); - -SQFS_INTERNAL int wait_completed(sqfs_data_writer_t *proc); - -#endif /* INTERNAL_H */ diff --git a/lib/sqfs/data_writer/serial.c b/lib/sqfs/data_writer/serial.c deleted file mode 100644 index 82f7836..0000000 --- a/lib/sqfs/data_writer/serial.c +++ /dev/null @@ -1,88 +0,0 @@ -/* SPDX-License-Identifier: LGPL-3.0-or-later */ -/* - * serial.c - * - * Copyright (C) 2019 David Oberhollenzer <goliath@infraroot.at> - */ -#define SQFS_BUILDING_DLL -#include "internal.h" - -sqfs_data_writer_t *sqfs_data_writer_create(size_t max_block_size, - sqfs_compressor_t *cmp, - unsigned int num_workers, - size_t max_backlog, - size_t devblksz, - sqfs_file_t *file) -{ - sqfs_data_writer_t *proc; - - proc = alloc_flex(sizeof(*proc), 1, max_block_size); - - if (proc == NULL) - return NULL; - - if (data_writer_init(proc, max_block_size, cmp, num_workers, - max_backlog, devblksz, file)) { - data_writer_cleanup(proc); - return NULL; - } - - return proc; -} - -void sqfs_data_writer_destroy(sqfs_data_writer_t *proc) -{ - data_writer_cleanup(proc); -} - -int test_and_set_status(sqfs_data_writer_t *proc, int status) -{ - if (proc->status == 0) - proc->status = status; - - return proc->status; -} - -int append_to_work_queue(sqfs_data_writer_t *proc, sqfs_block_t *block, - bool signal_threads) -{ - sqfs_block_t *fragblk = NULL; - (void)signal_threads; - - if (proc->status != 0 || block == NULL) { - free(block); - return proc->status; - } - - if (block->flags & SQFS_BLK_IS_FRAGMENT) { - block->checksum = crc32(0, block->data, block->size); - - proc->status = process_completed_fragment(proc, block, - &fragblk); - free(block); - - if (proc->status != 0) { - free(fragblk); - return proc->status; - } - - if (fragblk == NULL) - return 0; - - block = fragblk; - } - - proc->status = data_writer_do_block(block, proc->cmp, proc->scratch, - proc->max_block_size); - - if (proc->status == 0) - proc->status = process_completed_block(proc, block); - - free(block); - return proc->status; -} - -int wait_completed(sqfs_data_writer_t *proc) -{ - return proc->status; -} diff --git a/lib/sqfs/data_writer/winpthread.c b/lib/sqfs/data_writer/winpthread.c deleted file mode 100644 index e575859..0000000 --- a/lib/sqfs/data_writer/winpthread.c +++ /dev/null @@ -1,450 +0,0 @@ -/* SPDX-License-Identifier: LGPL-3.0-or-later */ -/* - * winpthread.c - * - * Copyright (C) 2019 David Oberhollenzer <goliath@infraroot.at> - */ -#define SQFS_BUILDING_DLL -#include "internal.h" - -#if defined(_WIN32) || defined(__WINDOWS__) -# define LOCK(mtx) EnterCriticalSection(mtx) -# define UNLOCK(mtx) LeaveCriticalSection(mtx) -# define AWAIT(cond, mtx) SleepConditionVariableCS(cond, mtx, INFINITE) -# define SIGNAL_ALL(cond) WakeAllConditionVariable(cond) -# define THREAD_EXIT_SUCCESS 0 -# define THREAD_TYPE DWORD WINAPI -# define THREAD_ARG LPVOID -# define THREAD_HANDLE HANDLE -#else -# define LOCK(mtx) pthread_mutex_lock(mtx) -# define UNLOCK(mtx) pthread_mutex_unlock(mtx) -# define AWAIT(cond, mtx) pthread_cond_wait(cond, mtx) -# define SIGNAL_ALL(cond) pthread_cond_broadcast(cond) -# define THREAD_EXIT_SUCCESS NULL -# define THREAD_TYPE void * -# define THREAD_ARG void * -# define THREAD_HANDLE pthread_t -#endif - -struct compress_worker_t { - sqfs_data_writer_t *shared; - sqfs_compressor_t *cmp; - THREAD_HANDLE thread; - sqfs_u8 scratch[]; -}; - -static THREAD_TYPE worker_proc(THREAD_ARG arg) -{ - compress_worker_t *worker = arg; - sqfs_data_writer_t *shared = worker->shared; - sqfs_block_t *it, *prev, *blk = NULL; - int status = 0; - - for (;;) { - LOCK(&shared->mtx); - if (blk != NULL) { - it = shared->done; - prev = NULL; - - while (it != NULL) { - if (it->sequence_number >= blk->sequence_number) - break; - prev = it; - it = it->next; - } - - if (prev == NULL) { - blk->next = shared->done; - shared->done = blk; - } else { - blk->next = prev->next; - prev->next = blk; - } - - if (status != 0 && shared->status == 0) - shared->status = status; - SIGNAL_ALL(&shared->done_cond); - } - - while (shared->queue == NULL && shared->status == 0) - AWAIT(&shared->queue_cond, &shared->mtx); - - if (shared->status == 0) { - blk = shared->queue; - shared->queue = blk->next; - blk->next = NULL; - - if (shared->queue == NULL) - shared->queue_last = NULL; - } else { - blk = NULL; - } - UNLOCK(&shared->mtx); - - if (blk == NULL) - break; - - status = data_writer_do_block(blk, worker->cmp, - worker->scratch, - shared->max_block_size); - } - - return THREAD_EXIT_SUCCESS; -} - -#if defined(_WIN32) || defined(__WINDOWS__) -sqfs_data_writer_t *sqfs_data_writer_create(size_t max_block_size, - sqfs_compressor_t *cmp, - unsigned int num_workers, - size_t max_backlog, - size_t devblksz, - sqfs_file_t *file) -{ - sqfs_data_writer_t *proc; - unsigned int i; - - if (num_workers < 1) - num_workers = 1; - - proc = alloc_flex(sizeof(*proc), - sizeof(proc->workers[0]), num_workers); - if (proc == NULL) - return NULL; - - InitializeCriticalSection(&proc->mtx); - InitializeConditionVariable(&proc->queue_cond); - InitializeConditionVariable(&proc->done_cond); - - if (data_writer_init(proc, max_block_size, cmp, num_workers, - max_backlog, devblksz, file)) { - goto fail; - } - - for (i = 0; i < num_workers; ++i) { - proc->workers[i] = alloc_flex(sizeof(compress_worker_t), - 1, max_block_size); - - if (proc->workers[i] == NULL) - goto fail; - - proc->workers[i]->shared = proc; - proc->workers[i]->cmp = cmp->create_copy(cmp); - - if (proc->workers[i]->cmp == NULL) - goto fail; - - proc->workers[i]->thread = CreateThread(NULL, 0, worker_proc, - proc->workers[i], 0, 0); - if (proc->workers[i]->thread == NULL) - goto fail; - } - - return proc; -fail: - sqfs_data_writer_destroy(proc); - return NULL; -} - -void sqfs_data_writer_destroy(sqfs_data_writer_t *proc) -{ - unsigned int i; - - EnterCriticalSection(&proc->mtx); - proc->status = -1; - WakeAllConditionVariable(&proc->queue_cond); - LeaveCriticalSection(&proc->mtx); - - for (i = 0; i < proc->num_workers; ++i) { - if (proc->workers[i] == NULL) - continue; - - if (proc->workers[i]->thread != NULL) { - WaitForSingleObject(proc->workers[i]->thread, INFINITE); - CloseHandle(proc->workers[i]->thread); - } - - if (proc->workers[i]->cmp != NULL) - proc->workers[i]->cmp->destroy(proc->workers[i]->cmp); - - free(proc->workers[i]); - } - - DeleteCriticalSection(&proc->mtx); - data_writer_cleanup(proc); -} -#else -sqfs_data_writer_t *sqfs_data_writer_create(size_t max_block_size, - sqfs_compressor_t *cmp, - unsigned int num_workers, - size_t max_backlog, - size_t devblksz, - sqfs_file_t *file) -{ - sqfs_data_writer_t *proc; - sigset_t set, oldset; - unsigned int i; - int ret; - - if (num_workers < 1) - num_workers = 1; - - proc = alloc_flex(sizeof(*proc), - sizeof(proc->workers[0]), num_workers); - if (proc == NULL) - return NULL; - - proc->mtx = (pthread_mutex_t)PTHREAD_MUTEX_INITIALIZER; - proc->queue_cond = (pthread_cond_t)PTHREAD_COND_INITIALIZER; - proc->done_cond = (pthread_cond_t)PTHREAD_COND_INITIALIZER; - - if (data_writer_init(proc, max_block_size, cmp, num_workers, - max_backlog, devblksz, file)) { - goto fail_init; - } - - for (i = 0; i < num_workers; ++i) { - proc->workers[i] = alloc_flex(sizeof(compress_worker_t), - 1, max_block_size); - - if (proc->workers[i] == NULL) - goto fail_init; - - proc->workers[i]->shared = proc; - proc->workers[i]->cmp = cmp->create_copy(cmp); - - if (proc->workers[i]->cmp == NULL) - goto fail_init; - } - - sigfillset(&set); - pthread_sigmask(SIG_SETMASK, &set, &oldset); - - for (i = 0; i < num_workers; ++i) { - ret = pthread_create(&proc->workers[i]->thread, NULL, - worker_proc, proc->workers[i]); - - if (ret != 0) - goto fail_thread; - } - - pthread_sigmask(SIG_SETMASK, &oldset, NULL); - - return proc; -fail_thread: - pthread_mutex_lock(&proc->mtx); - proc->status = -1; - pthread_cond_broadcast(&proc->queue_cond); - pthread_mutex_unlock(&proc->mtx); - - for (i = 0; i < num_workers; ++i) { - if (proc->workers[i]->thread > 0) { - pthread_join(proc->workers[i]->thread, NULL); - } - } - pthread_sigmask(SIG_SETMASK, &oldset, NULL); -fail_init: - for (i = 0; i < num_workers; ++i) { - if (proc->workers[i] != NULL) { - if (proc->workers[i]->cmp != NULL) { - proc->workers[i]->cmp-> - destroy(proc->workers[i]->cmp); - } - - free(proc->workers[i]); - } - } - pthread_cond_destroy(&proc->done_cond); - pthread_cond_destroy(&proc->queue_cond); - pthread_mutex_destroy(&proc->mtx); - data_writer_cleanup(proc); - return NULL; -} - -void sqfs_data_writer_destroy(sqfs_data_writer_t *proc) -{ - unsigned int i; - - pthread_mutex_lock(&proc->mtx); - proc->status = -1; - pthread_cond_broadcast(&proc->queue_cond); - pthread_mutex_unlock(&proc->mtx); - - for (i = 0; i < proc->num_workers; ++i) { - pthread_join(proc->workers[i]->thread, NULL); - - proc->workers[i]->cmp->destroy(proc->workers[i]->cmp); - free(proc->workers[i]); - } - - pthread_cond_destroy(&proc->done_cond); - pthread_cond_destroy(&proc->queue_cond); - pthread_mutex_destroy(&proc->mtx); - - data_writer_cleanup(proc); -} -#endif - -int append_to_work_queue(sqfs_data_writer_t *proc, sqfs_block_t *block, - bool signal_threads) -{ - int status; - - LOCK(&proc->mtx); - status = proc->status; - if (status != 0) - goto out; - - if (block != NULL) { - if (proc->queue_last == NULL) { - proc->queue = proc->queue_last = block; - } else { - proc->queue_last->next = block; - proc->queue_last = block; - } - - block->sequence_number = proc->enqueue_id++; - block->next = NULL; - proc->backlog += 1; - block = NULL; - } -out: - if (signal_threads) - SIGNAL_ALL(&proc->queue_cond); - - UNLOCK(&proc->mtx); - free(block); - return 0; -} - -static sqfs_block_t *try_dequeue(sqfs_data_writer_t *proc) -{ - sqfs_block_t *queue, *it, *prev; - - it = proc->done; - prev = NULL; - - while (it != NULL && it->sequence_number == proc->dequeue_id) { - prev = it; - it = it->next; - proc->dequeue_id += 1; - } - - if (prev == NULL) { - queue = NULL; - } else { - queue = proc->done; - prev->next = NULL; - proc->done = it; - } - - return queue; -} - -static sqfs_block_t *queue_merge(sqfs_block_t *lhs, sqfs_block_t *rhs) -{ - sqfs_block_t *it, *head = NULL, **next_ptr = &head; - - while (lhs != NULL && rhs != NULL) { - if (lhs->sequence_number <= rhs->sequence_number) { - it = lhs; - lhs = lhs->next; - } else { - it = rhs; - rhs = rhs->next; - } - - *next_ptr = it; - next_ptr = &it->next; - } - - it = (lhs != NULL ? lhs : rhs); - *next_ptr = it; - return head; -} - -static int process_done_queue(sqfs_data_writer_t *proc, sqfs_block_t *queue) -{ - sqfs_block_t *it, *block = NULL; - int status = 0; - - while (queue != NULL && status == 0) { - it = queue; - queue = it->next; - proc->backlog -= 1; - - if (it->flags & SQFS_BLK_IS_FRAGMENT) { - block = NULL; - status = process_completed_fragment(proc, it, &block); - - if (block != NULL && status == 0) { - LOCK(&proc->mtx); - proc->dequeue_id = it->sequence_number; - block->sequence_number = it->sequence_number; - - if (proc->queue == NULL) { - proc->queue = block; - proc->queue_last = block; - } else { - block->next = proc->queue; - proc->queue = block; - } - - proc->backlog += 1; - proc->done = queue_merge(queue, proc->done); - SIGNAL_ALL(&proc->queue_cond); - UNLOCK(&proc->mtx); - - queue = NULL; - } else { - free(block); - } - } else { - status = process_completed_block(proc, it); - } - - free(it); - } - - free_blk_list(queue); - return status; -} - -int test_and_set_status(sqfs_data_writer_t *proc, int status) -{ - LOCK(&proc->mtx); - if (proc->status == 0) { - proc->status = status; - } else { - status = proc->status; - } - SIGNAL_ALL(&proc->queue_cond); - UNLOCK(&proc->mtx); - return status; -} - -int wait_completed(sqfs_data_writer_t *proc) -{ - sqfs_block_t *queue; - int status; - - LOCK(&proc->mtx); - for (;;) { - queue = try_dequeue(proc); - status = proc->status; - - if (queue != NULL || status != 0) - break; - - AWAIT(&proc->done_cond, &proc->mtx); - } - UNLOCK(&proc->mtx); - - if (status != 0) { - free_blk_list(queue); - return status; - } - - status = process_done_queue(proc, queue); - return status ? test_and_set_status(proc, status) : status; -} |