From b63e39ade1e376ad8f433a0d751eddf4826a5230 Mon Sep 17 00:00:00 2001 From: David Oberhollenzer Date: Fri, 21 Feb 2020 12:39:28 +0100 Subject: Cleanup: move utilities back out of libsquashfs This commit removes the allocation helpers and string table functions out of libsquashfs back into a "libutil.a". The problem of libsquashfs exporting stuff that it shouldn't is resolved by retaining the internal attributes and directly adding the source to libsquashfs instead of trying to somehow link against libutil.la. Signed-off-by: David Oberhollenzer --- Makefile.am | 1 + include/str_table.h | 51 +++++++++++ include/util.h | 31 +++++++ lib/sqfs/Makemodule.am | 5 +- lib/sqfs/alloc.c | 37 -------- lib/sqfs/block_processor/internal.h | 2 +- lib/sqfs/comp/internal.h | 2 +- lib/sqfs/str_table.c | 174 ------------------------------------ lib/sqfs/str_table.h | 51 ----------- lib/sqfs/util.h | 31 ------- lib/util/Makemodule.am | 6 ++ lib/util/alloc.c | 37 ++++++++ lib/util/str_table.c | 174 ++++++++++++++++++++++++++++++++++++ tests/Makemodule.am | 6 +- 14 files changed, 307 insertions(+), 301 deletions(-) create mode 100644 include/str_table.h create mode 100644 include/util.h delete mode 100644 lib/sqfs/alloc.c delete mode 100644 lib/sqfs/str_table.c delete mode 100644 lib/sqfs/str_table.h delete mode 100644 lib/sqfs/util.h create mode 100644 lib/util/Makemodule.am create mode 100644 lib/util/alloc.c create mode 100644 lib/util/str_table.c diff --git a/Makefile.am b/Makefile.am index c958de6..7d4f9ba 100644 --- a/Makefile.am +++ b/Makefile.am @@ -28,6 +28,7 @@ include lib/fstree/Makemodule.am include lib/common/Makemodule.am include lib/tar/Makemodule.am include lib/compat/Makemodule.am +include lib/util/Makemodule.am include tar/Makemodule.am include mkfs/Makemodule.am include unpack/Makemodule.am diff --git a/include/str_table.h b/include/str_table.h new file mode 100644 index 0000000..61f8aa5 --- /dev/null +++ b/include/str_table.h @@ -0,0 +1,51 @@ +/* SPDX-License-Identifier: LGPL-3.0-or-later */ +/* + * str_table.h + * + * Copyright (C) 2019 David Oberhollenzer + */ +#ifndef STR_TABLE_H +#define STR_TABLE_H + +#include "sqfs/predef.h" + +typedef struct str_bucket_t { + struct str_bucket_t *next; + char *str; + size_t index; + size_t refcount; +} str_bucket_t; + +/* Stores strings in a hash table and assigns an incremental, unique ID to + each string. Subsequent additions return the existing ID. The ID can be + used for (hopefully) constant time lookup of the original string. */ +typedef struct { + str_bucket_t **buckets; + size_t num_buckets; + + char **strings; + size_t num_strings; + size_t max_strings; +} str_table_t; + +/* `size` is the number of hash table buckets to use internally. */ +SQFS_INTERNAL int str_table_init(str_table_t *table, size_t size); + +SQFS_INTERNAL void str_table_cleanup(str_table_t *table); + +/* Resolve a string to an incremental, unique ID. */ +SQFS_INTERNAL +int str_table_get_index(str_table_t *table, const char *str, size_t *idx); + +/* Resolve a unique ID to the string it represents. + Returns NULL if the ID is unknown, i.e. out of bounds. */ +SQFS_INTERNAL +const char *str_table_get_string(str_table_t *table, size_t index); + +SQFS_INTERNAL void str_table_add_ref(str_table_t *table, size_t index); + +SQFS_INTERNAL void str_table_del_ref(str_table_t *table, size_t index); + +SQFS_INTERNAL size_t str_table_get_ref_count(str_table_t *table, size_t index); + +#endif /* STR_TABLE_H */ diff --git a/include/util.h b/include/util.h new file mode 100644 index 0000000..a379c0d --- /dev/null +++ b/include/util.h @@ -0,0 +1,31 @@ +/* SPDX-License-Identifier: LGPL-3.0-or-later */ +/* + * util.h + * + * Copyright (C) 2019 David Oberhollenzer + */ +#ifndef SQFS_UTIL_H +#define SQFS_UTIL_H + +#include "config.h" +#include "sqfs/predef.h" +#include "compat.h" + +#include + +/* + Helper for allocating data structures with flexible array members. + + 'base_size' is the size of the struct itself, 'item_size' the size of a + single array element and 'nmemb' the number of elements. + + Iternally checks for arithmetic overflows when allocating the combined thing. + */ +SQFS_INTERNAL +void *alloc_flex(size_t base_size, size_t item_size, size_t nmemb); + +/* Basically the same as calloc, but *ALWAYS* does overflow checking */ +SQFS_INTERNAL +void *alloc_array(size_t item_size, size_t nmemb); + +#endif /* SQFS_UTIL_H */ diff --git a/lib/sqfs/Makemodule.am b/lib/sqfs/Makemodule.am index aada6c6..591eda2 100644 --- a/lib/sqfs/Makemodule.am +++ b/lib/sqfs/Makemodule.am @@ -24,8 +24,6 @@ libsquashfs_la_SOURCES += lib/sqfs/write_super.c lib/sqfs/data_reader.c libsquashfs_la_SOURCES += lib/sqfs/block_processor/internal.h libsquashfs_la_SOURCES += lib/sqfs/block_processor/common.c libsquashfs_la_SOURCES += lib/sqfs/block_processor/xxhash.c -libsquashfs_la_SOURCES += lib/sqfs/str_table.c lib/sqfs/str_table.h -libsquashfs_la_SOURCES += lib/sqfs/alloc.c lib/sqfs/util.h libsquashfs_la_SOURCES += lib/sqfs/frag_table.c include/sqfs/frag_table.h libsquashfs_la_SOURCES += lib/sqfs/block_writer.c include/sqfs/block_writer.h libsquashfs_la_CPPFLAGS = $(AM_CPPFLAGS) @@ -36,6 +34,9 @@ libsquashfs_la_CFLAGS += $(ZSTD_CFLAGS) $(PTHREAD_CFLAGS) libsquashfs_la_LIBADD = $(XZ_LIBS) $(ZLIB_LIBS) $(LZ4_LIBS) libsquashfs_la_LIBADD += $(ZSTD_LIBS) $(PTHREAD_LIBS) +# directly "import" stuff from libutil +libsquashfs_la_SOURCES += lib/util/str_table.c lib/util/alloc.c + if WINDOWS libsquashfs_la_SOURCES += lib/sqfs/win32/io_file.c libsquashfs_la_CFLAGS += -DWINVER=0x0600 -D_WIN32_WINNT=0x0600 diff --git a/lib/sqfs/alloc.c b/lib/sqfs/alloc.c deleted file mode 100644 index e8305d8..0000000 --- a/lib/sqfs/alloc.c +++ /dev/null @@ -1,37 +0,0 @@ -/* SPDX-License-Identifier: LGPL-3.0-or-later */ -/* - * alloc.c - * - * Copyright (C) 2019 David Oberhollenzer - */ -#include "config.h" - -#include "util.h" - -#include -#include - -void *alloc_flex(size_t base_size, size_t item_size, size_t nmemb) -{ - size_t size; - - if (SZ_MUL_OV(nmemb, item_size, &size) || - SZ_ADD_OV(base_size, size, &size)) { - errno = EOVERFLOW; - return NULL; - } - - return calloc(1, size); -} - -void *alloc_array(size_t item_size, size_t nmemb) -{ - size_t size; - - if (SZ_MUL_OV(nmemb, item_size, &size)) { - errno = EOVERFLOW; - return NULL; - } - - return calloc(1, size); -} diff --git a/lib/sqfs/block_processor/internal.h b/lib/sqfs/block_processor/internal.h index fb9c4a0..03a8296 100644 --- a/lib/sqfs/block_processor/internal.h +++ b/lib/sqfs/block_processor/internal.h @@ -18,7 +18,7 @@ #include "sqfs/error.h" #include "sqfs/block.h" #include "sqfs/io.h" -#include "../util.h" +#include "util.h" #include #include diff --git a/lib/sqfs/comp/internal.h b/lib/sqfs/comp/internal.h index c687d41..7518f56 100644 --- a/lib/sqfs/comp/internal.h +++ b/lib/sqfs/comp/internal.h @@ -14,7 +14,7 @@ #include "sqfs/error.h" #include "sqfs/block.h" #include "sqfs/io.h" -#include "../util.h" +#include "util.h" SQFS_INTERNAL int sqfs_generic_write_options(sqfs_file_t *file, const void *data, diff --git a/lib/sqfs/str_table.c b/lib/sqfs/str_table.c deleted file mode 100644 index c0a364f..0000000 --- a/lib/sqfs/str_table.c +++ /dev/null @@ -1,174 +0,0 @@ -/* SPDX-License-Identifier: LGPL-3.0-or-later */ -/* - * str_table.c - * - * Copyright (C) 2019 David Oberhollenzer - */ -#include "config.h" - -#include -#include -#include - -#include "sqfs/error.h" -#include "str_table.h" -#include "util.h" - -/* R5 hash function (borrowed from reiserfs) */ -static sqfs_u32 strhash(const char *s) -{ - const signed char *str = (const signed char *)s; - sqfs_u32 a = 0; - - while (*str != '\0') { - a += *str << 4; - a += *str >> 4; - a *= 11; - str++; - } - - return a; -} - -static int strings_grow(str_table_t *table) -{ - size_t newsz; - void *new; - - if (table->num_strings < table->max_strings) - return 0; - - newsz = table->max_strings ? (table->max_strings * 2) : 16; - new = realloc(table->strings, sizeof(table->strings[0]) * newsz); - - if (new == NULL) - return SQFS_ERROR_ALLOC; - - table->strings = new; - table->max_strings = newsz; - return 0; -} - -int str_table_init(str_table_t *table, size_t size) -{ - memset(table, 0, sizeof(*table)); - - table->buckets = alloc_array(size, sizeof(table->buckets[0])); - table->num_buckets = size; - - if (table->buckets == NULL) - return SQFS_ERROR_ALLOC; - - return 0; -} - -void str_table_cleanup(str_table_t *table) -{ - str_bucket_t *bucket; - size_t i; - - for (i = 0; i < table->num_buckets; ++i) { - while (table->buckets[i] != NULL) { - bucket = table->buckets[i]; - table->buckets[i] = bucket->next; - - free(bucket->str); - free(bucket); - } - } - - free(table->buckets); - free(table->strings); - memset(table, 0, sizeof(*table)); -} - -int str_table_get_index(str_table_t *table, const char *str, size_t *idx) -{ - str_bucket_t *bucket; - sqfs_u32 hash; - size_t index; - int err; - - hash = strhash(str); - index = hash % table->num_buckets; - bucket = table->buckets[index]; - - while (bucket != NULL) { - if (strcmp(bucket->str, str) == 0) { - *idx = bucket->index; - return 0; - } - - bucket = bucket->next; - } - - err = strings_grow(table); - if (err) - return err; - - bucket = calloc(1, sizeof(*bucket)); - if (bucket == NULL) - goto fail_oom; - - bucket->str = strdup(str); - if (bucket->str == NULL) - goto fail_oom; - - bucket->index = table->num_strings; - table->strings[table->num_strings++] = bucket->str; - *idx = bucket->index; - - bucket->next = table->buckets[index]; - table->buckets[index] = bucket; - return 0; -fail_oom: - free(bucket); - return SQFS_ERROR_ALLOC; -} - -const char *str_table_get_string(str_table_t *table, size_t index) -{ - if (index >= table->num_strings) - return NULL; - - return table->strings[index]; -} - -static str_bucket_t *bucket_by_index(str_table_t *table, size_t index) -{ - str_bucket_t *bucket = NULL; - sqfs_u32 hash; - - if (index < table->num_strings) { - hash = strhash(table->strings[index]); - bucket = table->buckets[hash % table->num_buckets]; - - while (bucket != NULL && bucket->index != index) - bucket = bucket->next; - } - - return bucket; -} - -void str_table_add_ref(str_table_t *table, size_t index) -{ - str_bucket_t *bucket = bucket_by_index(table, index); - - if (bucket != NULL && bucket->refcount < ~((size_t)0)) - bucket->refcount += 1; -} - -void str_table_del_ref(str_table_t *table, size_t index) -{ - str_bucket_t *bucket = bucket_by_index(table, index); - - if (bucket != NULL && bucket->refcount > 0) - bucket->refcount -= 1; -} - -size_t str_table_get_ref_count(str_table_t *table, size_t index) -{ - str_bucket_t *bucket = bucket_by_index(table, index); - - return bucket != NULL ? bucket->refcount : 0; -} diff --git a/lib/sqfs/str_table.h b/lib/sqfs/str_table.h deleted file mode 100644 index 61f8aa5..0000000 --- a/lib/sqfs/str_table.h +++ /dev/null @@ -1,51 +0,0 @@ -/* SPDX-License-Identifier: LGPL-3.0-or-later */ -/* - * str_table.h - * - * Copyright (C) 2019 David Oberhollenzer - */ -#ifndef STR_TABLE_H -#define STR_TABLE_H - -#include "sqfs/predef.h" - -typedef struct str_bucket_t { - struct str_bucket_t *next; - char *str; - size_t index; - size_t refcount; -} str_bucket_t; - -/* Stores strings in a hash table and assigns an incremental, unique ID to - each string. Subsequent additions return the existing ID. The ID can be - used for (hopefully) constant time lookup of the original string. */ -typedef struct { - str_bucket_t **buckets; - size_t num_buckets; - - char **strings; - size_t num_strings; - size_t max_strings; -} str_table_t; - -/* `size` is the number of hash table buckets to use internally. */ -SQFS_INTERNAL int str_table_init(str_table_t *table, size_t size); - -SQFS_INTERNAL void str_table_cleanup(str_table_t *table); - -/* Resolve a string to an incremental, unique ID. */ -SQFS_INTERNAL -int str_table_get_index(str_table_t *table, const char *str, size_t *idx); - -/* Resolve a unique ID to the string it represents. - Returns NULL if the ID is unknown, i.e. out of bounds. */ -SQFS_INTERNAL -const char *str_table_get_string(str_table_t *table, size_t index); - -SQFS_INTERNAL void str_table_add_ref(str_table_t *table, size_t index); - -SQFS_INTERNAL void str_table_del_ref(str_table_t *table, size_t index); - -SQFS_INTERNAL size_t str_table_get_ref_count(str_table_t *table, size_t index); - -#endif /* STR_TABLE_H */ diff --git a/lib/sqfs/util.h b/lib/sqfs/util.h deleted file mode 100644 index a379c0d..0000000 --- a/lib/sqfs/util.h +++ /dev/null @@ -1,31 +0,0 @@ -/* SPDX-License-Identifier: LGPL-3.0-or-later */ -/* - * util.h - * - * Copyright (C) 2019 David Oberhollenzer - */ -#ifndef SQFS_UTIL_H -#define SQFS_UTIL_H - -#include "config.h" -#include "sqfs/predef.h" -#include "compat.h" - -#include - -/* - Helper for allocating data structures with flexible array members. - - 'base_size' is the size of the struct itself, 'item_size' the size of a - single array element and 'nmemb' the number of elements. - - Iternally checks for arithmetic overflows when allocating the combined thing. - */ -SQFS_INTERNAL -void *alloc_flex(size_t base_size, size_t item_size, size_t nmemb); - -/* Basically the same as calloc, but *ALWAYS* does overflow checking */ -SQFS_INTERNAL -void *alloc_array(size_t item_size, size_t nmemb); - -#endif /* SQFS_UTIL_H */ diff --git a/lib/util/Makemodule.am b/lib/util/Makemodule.am new file mode 100644 index 0000000..f5a5680 --- /dev/null +++ b/lib/util/Makemodule.am @@ -0,0 +1,6 @@ +libutil_a_SOURCES = include/util.h include/str_table.h +libutil_a_SOURCES += lib/util/str_table.c lib/util/alloc.c +libutil_a_CFLAGS = $(AM_CFLAGS) +libutil_a_CPPFLAGS = $(AM_CPPFLAGS) + +noinst_LIBRARIES += libutil.a diff --git a/lib/util/alloc.c b/lib/util/alloc.c new file mode 100644 index 0000000..e8305d8 --- /dev/null +++ b/lib/util/alloc.c @@ -0,0 +1,37 @@ +/* SPDX-License-Identifier: LGPL-3.0-or-later */ +/* + * alloc.c + * + * Copyright (C) 2019 David Oberhollenzer + */ +#include "config.h" + +#include "util.h" + +#include +#include + +void *alloc_flex(size_t base_size, size_t item_size, size_t nmemb) +{ + size_t size; + + if (SZ_MUL_OV(nmemb, item_size, &size) || + SZ_ADD_OV(base_size, size, &size)) { + errno = EOVERFLOW; + return NULL; + } + + return calloc(1, size); +} + +void *alloc_array(size_t item_size, size_t nmemb) +{ + size_t size; + + if (SZ_MUL_OV(nmemb, item_size, &size)) { + errno = EOVERFLOW; + return NULL; + } + + return calloc(1, size); +} diff --git a/lib/util/str_table.c b/lib/util/str_table.c new file mode 100644 index 0000000..c0a364f --- /dev/null +++ b/lib/util/str_table.c @@ -0,0 +1,174 @@ +/* SPDX-License-Identifier: LGPL-3.0-or-later */ +/* + * str_table.c + * + * Copyright (C) 2019 David Oberhollenzer + */ +#include "config.h" + +#include +#include +#include + +#include "sqfs/error.h" +#include "str_table.h" +#include "util.h" + +/* R5 hash function (borrowed from reiserfs) */ +static sqfs_u32 strhash(const char *s) +{ + const signed char *str = (const signed char *)s; + sqfs_u32 a = 0; + + while (*str != '\0') { + a += *str << 4; + a += *str >> 4; + a *= 11; + str++; + } + + return a; +} + +static int strings_grow(str_table_t *table) +{ + size_t newsz; + void *new; + + if (table->num_strings < table->max_strings) + return 0; + + newsz = table->max_strings ? (table->max_strings * 2) : 16; + new = realloc(table->strings, sizeof(table->strings[0]) * newsz); + + if (new == NULL) + return SQFS_ERROR_ALLOC; + + table->strings = new; + table->max_strings = newsz; + return 0; +} + +int str_table_init(str_table_t *table, size_t size) +{ + memset(table, 0, sizeof(*table)); + + table->buckets = alloc_array(size, sizeof(table->buckets[0])); + table->num_buckets = size; + + if (table->buckets == NULL) + return SQFS_ERROR_ALLOC; + + return 0; +} + +void str_table_cleanup(str_table_t *table) +{ + str_bucket_t *bucket; + size_t i; + + for (i = 0; i < table->num_buckets; ++i) { + while (table->buckets[i] != NULL) { + bucket = table->buckets[i]; + table->buckets[i] = bucket->next; + + free(bucket->str); + free(bucket); + } + } + + free(table->buckets); + free(table->strings); + memset(table, 0, sizeof(*table)); +} + +int str_table_get_index(str_table_t *table, const char *str, size_t *idx) +{ + str_bucket_t *bucket; + sqfs_u32 hash; + size_t index; + int err; + + hash = strhash(str); + index = hash % table->num_buckets; + bucket = table->buckets[index]; + + while (bucket != NULL) { + if (strcmp(bucket->str, str) == 0) { + *idx = bucket->index; + return 0; + } + + bucket = bucket->next; + } + + err = strings_grow(table); + if (err) + return err; + + bucket = calloc(1, sizeof(*bucket)); + if (bucket == NULL) + goto fail_oom; + + bucket->str = strdup(str); + if (bucket->str == NULL) + goto fail_oom; + + bucket->index = table->num_strings; + table->strings[table->num_strings++] = bucket->str; + *idx = bucket->index; + + bucket->next = table->buckets[index]; + table->buckets[index] = bucket; + return 0; +fail_oom: + free(bucket); + return SQFS_ERROR_ALLOC; +} + +const char *str_table_get_string(str_table_t *table, size_t index) +{ + if (index >= table->num_strings) + return NULL; + + return table->strings[index]; +} + +static str_bucket_t *bucket_by_index(str_table_t *table, size_t index) +{ + str_bucket_t *bucket = NULL; + sqfs_u32 hash; + + if (index < table->num_strings) { + hash = strhash(table->strings[index]); + bucket = table->buckets[hash % table->num_buckets]; + + while (bucket != NULL && bucket->index != index) + bucket = bucket->next; + } + + return bucket; +} + +void str_table_add_ref(str_table_t *table, size_t index) +{ + str_bucket_t *bucket = bucket_by_index(table, index); + + if (bucket != NULL && bucket->refcount < ~((size_t)0)) + bucket->refcount += 1; +} + +void str_table_del_ref(str_table_t *table, size_t index) +{ + str_bucket_t *bucket = bucket_by_index(table, index); + + if (bucket != NULL && bucket->refcount > 0) + bucket->refcount -= 1; +} + +size_t str_table_get_ref_count(str_table_t *table, size_t index) +{ + str_bucket_t *bucket = bucket_by_index(table, index); + + return bucket != NULL ? bucket->refcount : 0; +} diff --git a/tests/Makemodule.am b/tests/Makemodule.am index 0c3f6e5..83f8666 100644 --- a/tests/Makemodule.am +++ b/tests/Makemodule.am @@ -1,10 +1,8 @@ test_canonicalize_name_SOURCES = tests/canonicalize_name.c test_canonicalize_name_LDADD = libfstree.a -test_str_table_SOURCES = tests/str_table.c lib/sqfs/str_table.c -test_str_table_SOURCES += lib/sqfs/alloc.c lib/sqfs/util.h -test_str_table_SOURCES += lib/sqfs/str_table.h -test_str_table_LDADD = libcompat.a +test_str_table_SOURCES = tests/str_table.c +test_str_table_LDADD = libutil.a libcompat.a test_str_table_CPPFLAGS = $(AM_CPPFLAGS) -DTESTPATH=$(top_srcdir)/tests test_str_table_CPPFLAGS += -I$(top_srcdir)/lib/sqfs -- cgit v1.2.3