Add fstree independend directory writer to libsquashfs.so

This commit adds a directory writer to libsquashfs that wrapps a meta
data writer and provides a higher-level interface for writing directory
entries. Under the hood it enforces the rules that squashfs insists
upon.

Signed-off-by: David Oberhollenzer <david.oberhollenzer@sigma-star.at>

5 files changed, 325 insertions, 177 deletions

diff --git a/lib/Makemodule.am b/lib/Makemodule.am
index a5db913..1f709c1 100644
--- a/lib/Makemodule.am
+++ b/lib/Makemodule.am
@@ -24,7 +24,6 @@ libsqfshelper_a_SOURCES += lib/sqfshelper/serialize_fstree.c
 libsqfshelper_a_SOURCES += lib/sqfshelper/statistics.c
 libsqfshelper_a_SOURCES += lib/sqfshelper/tree_node_from_inode.c
 libsqfshelper_a_SOURCES += lib/sqfshelper/sqfs_reader.c
-libsqfshelper_a_SOURCES += lib/sqfshelper/write_dir.c
 libsqfshelper_a_SOURCES += lib/sqfshelper/write_inode.c
 libsqfshelper_a_SOURCES += lib/sqfshelper/write_export_table.c
 libsqfshelper_a_SOURCES += lib/sqfshelper/print_version.c
@@ -50,7 +49,7 @@ libsquashfs_la_SOURCES += include/sqfs/super.h include/sqfs/inode.h
 libsquashfs_la_SOURCES += include/sqfs/dir.h include/sqfs/xattr.h
 libsquashfs_la_SOURCES += include/sqfs/table.h
 libsquashfs_la_SOURCES += lib/sqfs/meta_writer.c lib/sqfs/super.c
-libsquashfs_la_SOURCES += lib/sqfs/id_table.c
+libsquashfs_la_SOURCES += lib/sqfs/id_table.c lib/sqfs/dir_writer.c
 libsquashfs_la_SOURCES += lib/sqfs/write_table.c include/highlevel.h
 libsquashfs_la_SOURCES += lib/sqfs/read_super.c lib/sqfs/meta_reader.c
 libsquashfs_la_SOURCES += lib/sqfs/read_inode.c
diff --git a/lib/sqfs/dir_writer.c b/lib/sqfs/dir_writer.c
new file mode 100644
index 0000000..3ef3bc7
--- /dev/null
+++ b/lib/sqfs/dir_writer.c
@@ -0,0 +1,287 @@
+/* SPDX-License-Identifier: GPL-3.0-or-later */
+/*
+ * dir_writer.c
+ *
+ * Copyright (C) 2019 David Oberhollenzer <goliath@infraroot.at>
+ */
+#include "config.h"
+
+#include "sqfs/inode.h"
+#include "sqfs/dir.h"
+#include "util.h"
+
+#include <sys/stat.h>
+#include <endian.h>
+#include <stdlib.h>
+#include <string.h>
+#include <assert.h>
+#include <stdio.h>
+
+typedef struct dir_entry_t {
+	struct dir_entry_t *next;
+	uint64_t inode_ref;
+	uint32_t inode_num;
+	uint16_t type;
+	size_t name_len;
+	char name[];
+} dir_entry_t;
+
+typedef struct index_ent_t {
+	struct index_ent_t *next;
+	dir_entry_t *ent;
+	uint64_t block;
+	uint32_t index;
+} index_ent_t;
+
+struct sqfs_dir_writer_t {
+	dir_entry_t *list;
+	dir_entry_t *list_end;
+
+	index_ent_t *idx;
+	index_ent_t *idx_end;
+
+	uint64_t dir_ref;
+	size_t dir_size;
+	size_t idx_size;
+	meta_writer_t *dm;
+};
+
+static int get_type(mode_t mode)
+{
+	switch (mode & S_IFMT) {
+	case S_IFSOCK: return SQFS_INODE_SOCKET;
+	case S_IFIFO:  return SQFS_INODE_FIFO;
+	case S_IFLNK:  return SQFS_INODE_SLINK;
+	case S_IFBLK:  return SQFS_INODE_BDEV;
+	case S_IFCHR:  return SQFS_INODE_CDEV;
+	case S_IFDIR:  return SQFS_INODE_DIR;
+	case S_IFREG:  return SQFS_INODE_FILE;
+	default:
+		assert(0);
+	}
+}
+
+static void writer_reset(sqfs_dir_writer_t *writer)
+{
+	dir_entry_t *ent;
+	index_ent_t *idx;
+
+	while (writer->idx != NULL) {
+		idx = writer->idx;
+		writer->idx = idx->next;
+		free(idx);
+	}
+
+	while (writer->list != NULL) {
+		ent = writer->list;
+		writer->list = ent->next;
+		free(ent);
+	}
+
+	writer->list_end = NULL;
+	writer->idx_end = NULL;
+	writer->dir_ref = 0;
+	writer->dir_size = 0;
+	writer->idx_size = 0;
+}
+
+sqfs_dir_writer_t *sqfs_dir_writer_create(meta_writer_t *dm)
+{
+	sqfs_dir_writer_t *writer = calloc(1, sizeof(*writer));
+
+	if (writer == NULL) {
+		perror("creating directory writer");
+		return NULL;
+	}
+
+	writer->dm = dm;
+	return writer;
+}
+
+void sqfs_dir_writer_destroy(sqfs_dir_writer_t *writer)
+{
+	writer_reset(writer);
+	free(writer);
+}
+
+int sqfs_dir_writer_begin(sqfs_dir_writer_t *writer)
+{
+	uint32_t offset;
+	uint64_t block;
+
+	writer_reset(writer);
+
+	meta_writer_get_position(writer->dm, &block, &offset);
+	writer->dir_ref = (block << 16) | offset;
+	return 0;
+}
+
+int sqfs_dir_writer_add_entry(sqfs_dir_writer_t *writer, const char *name,
+			      uint32_t inode_num, uint64_t inode_ref,
+			      mode_t mode)
+{
+	dir_entry_t *ent = alloc_flex(sizeof(*ent), 1, strlen(name));
+
+	if (ent == NULL) {
+		perror("creating directory entry");
+		return -1;
+	}
+
+	ent->inode_ref = inode_ref;
+	ent->inode_num = inode_num;
+	ent->type = get_type(mode);
+	ent->name_len = strlen(name);
+	memcpy(ent->name, name, ent->name_len);
+
+	if (writer->list_end == NULL) {
+		writer->list = writer->list_end = ent;
+	} else {
+		writer->list_end->next = ent;
+		writer->list_end = ent;
+	}
+
+	writer->dir_size += sizeof(ent) + ent->name_len;
+	return 0;
+}
+
+static size_t get_conseq_entry_count(uint32_t offset, dir_entry_t *head)
+{
+	size_t size, count = 0;
+	dir_entry_t *it;
+	int32_t diff;
+
+	size = (offset + sizeof(sqfs_dir_header_t)) % SQFS_META_BLOCK_SIZE;
+
+	for (it = head; it != NULL; it = it->next) {
+		if ((it->inode_ref >> 16) != (head->inode_ref >> 16))
+			break;
+
+		diff = it->inode_num - head->inode_num;
+
+		if (diff > 32767 || diff < -32767)
+			break;
+
+		size += sizeof(sqfs_dir_entry_t) + it->name_len;
+
+		if (count > 0 && size > SQFS_META_BLOCK_SIZE)
+			break;
+
+		count += 1;
+
+		if (count == SQFS_MAX_DIR_ENT)
+			break;
+	}
+
+	return count;
+}
+
+static int add_header(sqfs_dir_writer_t *writer, size_t count,
+		      dir_entry_t *ref, uint64_t block)
+{
+	sqfs_dir_header_t hdr;
+	index_ent_t *idx;
+
+	hdr.count = htole32(count - 1);
+	hdr.start_block = htole32(ref->inode_ref >> 16);
+	hdr.inode_number = htole32(ref->inode_num);
+
+	if (meta_writer_append(writer->dm, &hdr, sizeof(hdr)))
+		return -1;
+
+	idx = calloc(1, sizeof(*idx));
+	if (idx == NULL) {
+		perror("creating directory index entry");
+		return -1;
+	}
+
+	idx->ent = ref;
+	idx->block = block;
+	idx->index = writer->dir_size;
+
+	if (writer->idx_end == NULL) {
+		writer->idx = writer->idx_end = idx;
+	} else {
+		writer->idx_end->next = idx;
+		writer->idx_end = idx;
+	}
+
+	writer->dir_size += sizeof(hdr);
+	writer->idx_size += 1;
+	return 0;
+}
+
+int sqfs_dir_writer_end(sqfs_dir_writer_t *writer)
+{
+	dir_entry_t *it, *first;
+	sqfs_dir_entry_t ent;
+	uint16_t *diff_u16;
+	size_t i, count;
+	uint32_t offset;
+	uint64_t block;
+
+	for (it = writer->list; it != NULL; ) {
+		meta_writer_get_position(writer->dm, &block, &offset);
+		count = get_conseq_entry_count(offset, it);
+
+		if (add_header(writer, count, it, block))
+			return -1;
+
+		first = it;
+
+		for (i = 0; i < count; ++i) {
+			ent.offset = htole16(it->inode_ref & 0x0000FFFF);
+			ent.inode_diff = it->inode_num - first->inode_num;
+			ent.type = htole16(it->type);
+			ent.size = htole16(it->name_len - 1);
+
+			diff_u16 = (uint16_t *)&ent.inode_diff;
+			*diff_u16 = htole16(*diff_u16);
+
+			if (meta_writer_append(writer->dm, &ent, sizeof(ent)))
+				return -1;
+
+			if (meta_writer_append(writer->dm, it->name, it->name_len))
+				return -1;
+
+			it = it->next;
+		}
+	}
+
+	return 0;
+}
+
+size_t sqfs_dir_writer_get_size(sqfs_dir_writer_t *writer)
+{
+	return writer->dir_size;
+}
+
+uint64_t sqfs_dir_writer_get_dir_reference(sqfs_dir_writer_t *writer)
+{
+	return writer->dir_ref;
+}
+
+size_t sqfs_dir_writer_get_index_size(sqfs_dir_writer_t *writer)
+{
+	return writer->idx_size;
+}
+
+int sqfs_dir_writer_write_index(sqfs_dir_writer_t *writer,
+				meta_writer_t *im)
+{
+	sqfs_dir_index_t ent;
+	index_ent_t *idx;
+
+	for (idx = writer->idx; idx != NULL; idx = idx->next) {
+		ent.start_block = htole32(idx->block);
+		ent.index = htole32(idx->index);
+		ent.size = htole32(idx->ent->name_len - 1);
+
+		if (meta_writer_append(im, &ent, sizeof(ent)))
+			return -1;
+
+		if (meta_writer_append(im, idx->ent->name, idx->ent->name_len))
+			return -1;
+	}
+
+	return 0;
+}
diff --git a/lib/sqfshelper/serialize_fstree.c b/lib/sqfshelper/serialize_fstree.c
index e6eccea..53e2236 100644
--- a/lib/sqfshelper/serialize_fstree.c
+++ b/lib/sqfshelper/serialize_fstree.c
@@ -16,6 +16,7 @@
 int sqfs_serialize_fstree(int outfd, sqfs_super_t *super, fstree_t *fs,
 			  compressor_t *cmp, id_table_t *idtbl)
 {
+	sqfs_dir_writer_t *dirwr;
 	meta_writer_t *im, *dm;
 	uint32_t offset;
 	uint64_t block;
@@ -30,8 +31,12 @@ int sqfs_serialize_fstree(int outfd, sqfs_super_t *super, fstree_t *fs,
 	if (dm == NULL)
 		goto out_im;
 
+	dirwr = sqfs_dir_writer_create(dm);
+	if (dirwr == NULL)
+		goto out_dm;
+
 	for (i = 2; i < fs->inode_tbl_size; ++i) {
-		if (meta_writer_write_inode(fs, idtbl, im, dm,
+		if (meta_writer_write_inode(fs, idtbl, im, dirwr,
 					    fs->inode_table[i])) {
 			goto out;
 		}
@@ -58,6 +63,8 @@ int sqfs_serialize_fstree(int outfd, sqfs_super_t *super, fstree_t *fs,
 
 	ret = 0;
 out:
+	sqfs_dir_writer_destroy(dirwr);
+out_dm:
 	meta_writer_destroy(dm);
 out_im:
 	meta_writer_destroy(im);
diff --git a/lib/sqfshelper/write_dir.c b/lib/sqfshelper/write_dir.c
deleted file mode 100644
index b977bc4..0000000
--- a/lib/sqfshelper/write_dir.c
+++ /dev/null
@@ -1,144 +0,0 @@
-/* SPDX-License-Identifier: GPL-3.0-or-later */
-/*
- * write_dir.c
- *
- * Copyright (C) 2019 David Oberhollenzer <goliath@infraroot.at>
- */
-#include "config.h"
-
-#include "sqfs/inode.h"
-#include "sqfs/dir.h"
-#include "highlevel.h"
-#include "util.h"
-
-#include <sys/stat.h>
-#include <assert.h>
-#include <endian.h>
-#include <stdlib.h>
-#include <string.h>
-#include <stdio.h>
-
-static int get_type(mode_t mode)
-{
-	switch (mode & S_IFMT) {
-	case S_IFSOCK: return SQFS_INODE_SOCKET;
-	case S_IFIFO:  return SQFS_INODE_FIFO;
-	case S_IFLNK:  return SQFS_INODE_SLINK;
-	case S_IFBLK:  return SQFS_INODE_BDEV;
-	case S_IFCHR:  return SQFS_INODE_CDEV;
-	case S_IFDIR:  return SQFS_INODE_DIR;
-	case S_IFREG:  return SQFS_INODE_FILE;
-	default:
-		assert(0);
-	}
-}
-
-static int dir_index_grow(dir_index_t **index)
-{
-	size_t size = sizeof(dir_index_t) + sizeof(idx_ref_t) * 10;
-	void *new;
-
-	if (*index == NULL) {
-		new = calloc(1, size);
-	} else {
-		if ((*index)->num_nodes < (*index)->max_nodes)
-			return 0;
-
-		size += sizeof(idx_ref_t) * (*index)->num_nodes;
-		new = realloc(*index, size);
-	}
-
-	if (new == NULL) {
-		perror("creating directory index");
-		return -1;
-	}
-
-	*index = new;
-	(*index)->max_nodes += 10;
-	return 0;
-}
-
-int meta_writer_write_dir(meta_writer_t *dm, dir_info_t *dir,
-			  dir_index_t **index)
-{
-	size_t i, size, count;
-	sqfs_dir_header_t hdr;
-	sqfs_dir_entry_t ent;
-	tree_node_t *c, *d;
-	uint16_t *diff_u16;
-	uint32_t offset;
-	uint64_t block;
-	int32_t diff;
-
-	c = dir->children;
-	dir->size = 0;
-
-	meta_writer_get_position(dm, &dir->start_block, &dir->block_offset);
-
-	while (c != NULL) {
-		meta_writer_get_position(dm, &block, &offset);
-
-		count = 0;
-		size = (offset + sizeof(hdr)) % SQFS_META_BLOCK_SIZE;
-
-		for (d = c; d != NULL; d = d->next) {
-			if ((d->inode_ref >> 16) != (c->inode_ref >> 16))
-				break;
-
-			diff = d->inode_num - c->inode_num;
-
-			if (diff > 32767 || diff < -32767)
-				break;
-
-			size += sizeof(ent) + strlen(c->name);
-
-			if (count > 0 && size > SQFS_META_BLOCK_SIZE)
-				break;
-
-			count += 1;
-		}
-
-		if (count > SQFS_MAX_DIR_ENT)
-			count = SQFS_MAX_DIR_ENT;
-
-		if (dir_index_grow(index))
-			return -1;
-
-		meta_writer_get_position(dm, &block, &offset);
-
-		i = (*index)->num_nodes++;
-		(*index)->idx_nodes[i].node = c;
-		(*index)->idx_nodes[i].block = block;
-		(*index)->idx_nodes[i].index = dir->size;
-
-		hdr.count = htole32(count - 1);
-		hdr.start_block = htole32(c->inode_ref >> 16);
-		hdr.inode_number = htole32(c->inode_num);
-		dir->size += sizeof(hdr);
-
-		if (meta_writer_append(dm, &hdr, sizeof(hdr)))
-			return -1;
-
-		d = c;
-
-		for (i = 0; i < count; ++i) {
-			ent.inode_diff = c->inode_num - d->inode_num;
-
-			diff_u16 = (uint16_t *)&ent.inode_diff;
-			*diff_u16 = htole16(*diff_u16);
-
-			ent.offset = htole16(c->inode_ref & 0x0000FFFF);
-			ent.type = htole16(get_type(c->mode));
-			ent.size = htole16(strlen(c->name) - 1);
-			dir->size += sizeof(ent) + strlen(c->name);
-
-			if (meta_writer_append(dm, &ent, sizeof(ent)))
-				return -1;
-			if (meta_writer_append(dm, c->name, strlen(c->name)))
-				return -1;
-
-			c = c->next;
-		}
-	}
-	return 0;
-}
diff --git a/lib/sqfshelper/write_inode.c b/lib/sqfshelper/write_inode.c
index 1cd8e72..d9dfe53 100644
--- a/lib/sqfshelper/write_inode.c
+++ b/lib/sqfshelper/write_inode.c
@@ -103,32 +103,36 @@ static int write_file_blocks(fstree_t *fs, file_info_t *fi, meta_writer_t *im)
 	return 0;
 }
 
-static int write_dir_index(dir_index_t *diridx, meta_writer_t *im)
+static int write_dir_entries(sqfs_dir_writer_t *dirw, tree_node_t *node)
 {
-	sqfs_dir_index_t idx;
-	size_t i;
+	tree_node_t *it;
+	uint64_t ref;
+	int ret;
 
-	for (i = 0; i < diridx->num_nodes; ++i) {
-		idx.start_block = htole32(diridx->idx_nodes[i].block);
-		idx.index = htole32(diridx->idx_nodes[i].index);
-		idx.size = strlen(diridx->idx_nodes[i].node->name) - 1;
-		idx.size = htole32(idx.size);
-
-		if (meta_writer_append(im, &idx, sizeof(idx)))
-			return -1;
+	if (sqfs_dir_writer_begin(dirw))
+		return -1;
 
-		if (meta_writer_append(im, diridx->idx_nodes[i].node->name,
-				       le32toh(idx.size) + 1)) {
+	for (it = node->data.dir->children; it != NULL; it = it->next) {
+		ret = sqfs_dir_writer_add_entry(dirw, it->name, it->inode_num,
+						it->inode_ref, it->mode);
+		if (ret)
 			return -1;
-		}
 	}
+
+	if (sqfs_dir_writer_end(dirw))
+		return -1;
+
+	ref = sqfs_dir_writer_get_dir_reference(dirw);
+
+	node->data.dir->size = sqfs_dir_writer_get_size(dirw);
+	node->data.dir->start_block = ref >> 16;
+	node->data.dir->block_offset = ref & 0xFFFF;
 	return 0;
 }
 
 int meta_writer_write_inode(fstree_t *fs, id_table_t *idtbl, meta_writer_t *im,
-			    meta_writer_t *dm, tree_node_t *node)
+			    sqfs_dir_writer_t *dirw, tree_node_t *node)
 {
-	dir_index_t *diridx = NULL;
 	uint16_t uid_idx, gid_idx;
 	sqfs_inode_t base;
 	uint32_t offset;
@@ -144,7 +148,7 @@ int meta_writer_write_inode(fstree_t *fs, id_table_t *idtbl, meta_writer_t *im,
 	node->inode_ref = (block << 16) | offset;
 
 	if (S_ISDIR(node->mode)) {
-		if (meta_writer_write_dir(dm, node->data.dir, &diridx))
+		if (write_dir_entries(dirw, node))
 			return -1;
 	}
 
@@ -155,10 +159,8 @@ int meta_writer_write_inode(fstree_t *fs, id_table_t *idtbl, meta_writer_t *im,
 	base.mod_time = htole32(node->mod_time);
 	base.inode_number = htole32(node->inode_num);
 
-	if (meta_writer_append(im, &base, sizeof(base))) {
-		free(diridx);
+	if (meta_writer_append(im, &base, sizeof(base)))
 		return -1;
-	}
 
 	switch (le16toh(base.type)) {
 	case SQFS_INODE_FIFO:
@@ -292,6 +294,7 @@ int meta_writer_write_inode(fstree_t *fs, id_table_t *idtbl, meta_writer_t *im,
 		return meta_writer_append(im, &dir, sizeof(dir));
 	}
 	case SQFS_INODE_EXT_DIR: {
+		size_t idx_size;
 		sqfs_inode_dir_ext_t ext = {
 			.nlink = htole32(hard_link_count(node)),
 			.size = htole32(node->data.dir->size),
@@ -306,20 +309,16 @@ int meta_writer_write_inode(fstree_t *fs, id_table_t *idtbl, meta_writer_t *im,
 		if (node->xattr != NULL)
 			ext.xattr_idx = htole32(node->xattr->index);
 
-		if (diridx != NULL)
-			ext.inodex_count = htole32(diridx->num_nodes - 1);
+		idx_size = sqfs_dir_writer_get_index_size(dirw);
 
-		if (meta_writer_append(im, &ext, sizeof(ext))) {
-			free(diridx);
-			return -1;
-		}
+		if (idx_size > 0)
+			ext.inodex_count = htole32(idx_size - 1);
 
-		if (diridx != NULL && write_dir_index(diridx, im) != 0) {
-			free(diridx);
+		if (meta_writer_append(im, &ext, sizeof(ext)))
 			return -1;
-		}
 
-		free(diridx);
+		if (sqfs_dir_writer_write_index(dirw, im))
+			return -1;
 		break;
 	}
 	default: