Turn file inode management completely over to the block processor

If the block processor allocates and dynamically resizes inodes on
the fly, we can add data indefinitely without knowing the size of
the file ahead of time.

Signed-off-by: David Oberhollenzer <david.oberhollenzer@sigma-star.at>

7 files changed, 77 insertions, 87 deletions

diff --git a/include/common.h b/include/common.h
index 0fd349a..cf2d1b5 100644
--- a/include/common.h
+++ b/include/common.h
@@ -116,7 +116,7 @@ sqfs_file_t *sqfs_get_stdin_file(FILE *fp, const sparse_map_t *map,
 				 sqfs_u64 size);
 
 int write_data_from_file(const char *filename, sqfs_block_processor_t *data,
-			 sqfs_inode_generic_t *inode,
+			 sqfs_inode_generic_t **inode,
 			 sqfs_file_t *file, int flags);
 
 void sqfs_writer_cfg_init(sqfs_writer_cfg_t *cfg);
diff --git a/include/sqfs/block_processor.h b/include/sqfs/block_processor.h
index 6945633..2ad12c0 100644
--- a/include/sqfs/block_processor.h
+++ b/include/sqfs/block_processor.h
@@ -142,24 +142,27 @@ sqfs_block_processor_t *sqfs_block_processor_create(size_t max_block_size,
  * are done. After writing all files, use @ref sqfs_block_processor_finish to
  * wait until all blocks that are still in flight are done and written to disk.
  *
- * The specified inode pointer is kept internally and updated with the
- * compressed block sizes and final destinations of the file and possible
- * fragment. You need to make sure it has enough backing-store for all blocks
- * to come. Furthermore, since there can still be blocks in-flight even after
- * calling @ref sqfs_block_processor_end_file, the data in the inode may still
- * change. The only point at which the data writer is guarnteed to not touch
- * them anymore is after @ref sqfs_block_processor_finish has returned.
+ * The specified pointer to an inode pointer is kept internally and updated with
+ * the compressed block sizes and final destinations of the file and possible
+ * fragment. Since the inode may have to be grown to larger size, the actual
+ * inode pointer can change over time. Furthermore, since there can still be
+ * blocks in-flight even after calling @ref sqfs_block_processor_end_file, the
+ * data in the inode and the value of the pointer may still change. The only
+ * point at which the data writer is guaranteed to not touch them anymore is
+ * after @ref sqfs_block_processor_sync or @ref sqfs_block_processor_finish has
+ * returned.
  *
  * @param proc A pointer to a data writer object.
- * @param inode The regular file inode representing the file. The data writer
- *              internally updates it while writing blocks to disk.
+ * @param inode A pointer to a pointer to an inode. The block processor creates
+ *              a file inode and stores a pointer to it here and keeps updating
+ *              the inode as the file grows.
  * @param flags A combination of @ref E_SQFS_BLK_FLAGS that can be used to
  *              micro manage how the data is processed.
  *
  * @return Zero on success, an @ref E_SQFS_ERROR value on failure.
  */
 SQFS_API int sqfs_block_processor_begin_file(sqfs_block_processor_t *proc,
-					     sqfs_inode_generic_t *inode,
+					     sqfs_inode_generic_t **inode,
 					     sqfs_u32 flags);
 
 /**
diff --git a/lib/common/data_writer.c b/lib/common/data_writer.c
index 0b4dc07..bc5a43b 100644
--- a/lib/common/data_writer.c
+++ b/lib/common/data_writer.c
@@ -9,7 +9,7 @@
 static sqfs_u8 buffer[4096];
 
 int write_data_from_file(const char *filename, sqfs_block_processor_t *data,
-			 sqfs_inode_generic_t *inode, sqfs_file_t *file,
+			 sqfs_inode_generic_t **inode, sqfs_file_t *file,
 			 int flags)
 {
 	sqfs_u64 filesz, offset;
diff --git a/lib/sqfs/block_processor/common.c b/lib/sqfs/block_processor/common.c
index 51851ae..ab83b5d 100644
--- a/lib/sqfs/block_processor/common.c
+++ b/lib/sqfs/block_processor/common.c
@@ -7,17 +7,41 @@
 #define SQFS_BUILDING_DLL
 #include "internal.h"
 
-static void set_block_size(sqfs_inode_generic_t *inode,
-			   sqfs_u32 index, sqfs_u32 size)
+static int set_block_size(sqfs_inode_generic_t **inode,
+			  sqfs_u32 index, sqfs_u32 size)
 {
 	size_t min_size = (index + 1) * sizeof(sqfs_u32);
+	size_t avail = (*inode)->payload_bytes_available;
+	size_t newsz;
+	void *new;
 
-	inode->extra[index] = size;
+	if (avail < min_size) {
+		newsz = avail ? avail : (sizeof(sqfs_u32) * 4);
+		while (newsz < min_size)
+			newsz *= 2;
 
-	if (min_size >= inode->payload_bytes_used) {
-		inode->payload_bytes_used = min_size;
-		inode->payload_bytes_available = min_size;
+		if (SZ_ADD_OV(newsz, sizeof(**inode), &newsz))
+			return SQFS_ERROR_OVERFLOW;
+
+		if (sizeof(size_t) > sizeof(sqfs_u32)) {
+			if ((newsz - sizeof(**inode)) > 0x0FFFFFFFFUL)
+				return SQFS_ERROR_OVERFLOW;
+		}
+
+		new = realloc((*inode), newsz);
+		if (new == NULL)
+			return SQFS_ERROR_ALLOC;
+
+		(*inode) = new;
+		(*inode)->payload_bytes_available = newsz - sizeof(**inode);
 	}
+
+	(*inode)->extra[index] = size;
+
+	if (min_size >= (*inode)->payload_bytes_used)
+		(*inode)->payload_bytes_used = min_size;
+
+	return 0;
 }
 
 int process_completed_block(sqfs_block_processor_t *proc, sqfs_block_t *blk)
@@ -32,10 +56,12 @@ int process_completed_block(sqfs_block_processor_t *proc, sqfs_block_t *blk)
 		return err;
 
 	if (blk->flags & SQFS_BLK_IS_SPARSE) {
-		sqfs_inode_make_extended(blk->inode);
-		blk->inode->data.file_ext.sparse += blk->size;
+		sqfs_inode_make_extended(*(blk->inode));
+		(*(blk->inode))->data.file_ext.sparse += blk->size;
 
-		set_block_size(blk->inode, blk->index, 0);
+		err = set_block_size(blk->inode, blk->index, 0);
+		if (err)
+			return err;
 
 		proc->stats.sparse_block_count += 1;
 	} else if (blk->size != 0) {
@@ -49,13 +75,15 @@ int process_completed_block(sqfs_block_processor_t *proc, sqfs_block_t *blk)
 			if (err)
 				return err;
 		} else {
-			set_block_size(blk->inode, blk->index, size);
+			err = set_block_size(blk->inode, blk->index, size);
+			if (err)
+				return err;
 			proc->stats.data_block_count += 1;
 		}
 	}
 
 	if (blk->flags & SQFS_BLK_LAST_BLOCK)
-		sqfs_inode_set_file_block_start(blk->inode, location);
+		sqfs_inode_set_file_block_start(*(blk->inode), location);
 
 	return 0;
 }
@@ -104,9 +132,9 @@ int process_completed_fragment(sqfs_block_processor_t *proc, sqfs_block_t *frag,
 	int err;
 
 	if (frag->flags & SQFS_BLK_IS_SPARSE) {
-		sqfs_inode_make_extended(frag->inode);
+		sqfs_inode_make_extended(*(frag->inode));
 		set_block_size(frag->inode, frag->index, 0);
-		frag->inode->data.file_ext.sparse += frag->size;
+		(*(frag->inode))->data.file_ext.sparse += frag->size;
 
 		proc->stats.sparse_block_count += 1;
 		return 0;
@@ -117,7 +145,7 @@ int process_completed_fragment(sqfs_block_processor_t *proc, sqfs_block_t *frag,
 	err = sqfs_frag_table_find_tail_end(proc->frag_tbl, frag->checksum,
 					    frag->size, &index, &offset);
 	if (err == 0) {
-		sqfs_inode_set_frag_location(frag->inode, index, offset);
+		sqfs_inode_set_frag_location(*(frag->inode), index, offset);
 		return 0;
 	}
 
@@ -155,7 +183,7 @@ int process_completed_fragment(sqfs_block_processor_t *proc, sqfs_block_t *frag,
 	if (err)
 		goto fail;
 
-	sqfs_inode_set_frag_location(frag->inode, proc->frag_block->index,
+	sqfs_inode_set_frag_location(*(frag->inode), proc->frag_block->index,
 				     proc->frag_block->size);
 
 	memcpy(proc->frag_block->data + proc->frag_block->size,
@@ -202,7 +230,7 @@ static int flush_block(sqfs_block_processor_t *proc)
 }
 
 int sqfs_block_processor_begin_file(sqfs_block_processor_t *proc,
-				    sqfs_inode_generic_t *inode, sqfs_u32 flags)
+				    sqfs_inode_generic_t **inode, sqfs_u32 flags)
 {
 	if (proc->inode != NULL)
 		return SQFS_ERROR_SEQUENCE;
@@ -210,8 +238,12 @@ int sqfs_block_processor_begin_file(sqfs_block_processor_t *proc,
 	if (flags & ~SQFS_BLK_USER_SETTABLE_FLAGS)
 		return SQFS_ERROR_UNSUPPORTED;
 
-	sqfs_inode_set_file_size(inode, 0);
-	sqfs_inode_set_frag_location(inode, 0xFFFFFFFF, 0xFFFFFFFF);
+	(*inode) = calloc(1, sizeof(sqfs_inode_generic_t));
+	if ((*inode) == NULL)
+		return SQFS_ERROR_ALLOC;
+
+	(*inode)->base.type = SQFS_INODE_FILE;
+	sqfs_inode_set_frag_location(*inode, 0xFFFFFFFF, 0xFFFFFFFF);
 
 	proc->inode = inode;
 	proc->blk_flags = flags | SQFS_BLK_FIRST_BLOCK;
@@ -227,8 +259,8 @@ int sqfs_block_processor_append(sqfs_block_processor_t *proc, const void *data,
 	size_t diff;
 	int err;
 
-	sqfs_inode_get_file_size(proc->inode, &filesize);
-	sqfs_inode_set_file_size(proc->inode, filesize + size);
+	sqfs_inode_get_file_size(*(proc->inode), &filesize);
+	sqfs_inode_set_file_size(*(proc->inode), filesize + size);
 
 	while (size > 0) {
 		if (proc->blk_current == NULL) {
diff --git a/lib/sqfs/block_processor/internal.h b/lib/sqfs/block_processor/internal.h
index 03a8296..0e95551 100644
--- a/lib/sqfs/block_processor/internal.h
+++ b/lib/sqfs/block_processor/internal.h
@@ -25,7 +25,7 @@
 
 typedef struct sqfs_block_t {
 	struct sqfs_block_t *next;
-	sqfs_inode_generic_t *inode;
+	sqfs_inode_generic_t **inode;
 
 	sqfs_u32 proc_seq_num;
 	sqfs_u32 io_seq_num;
@@ -49,7 +49,7 @@ struct sqfs_block_processor_t {
 
 	sqfs_block_processor_stats_t stats;
 
-	sqfs_inode_generic_t *inode;
+	sqfs_inode_generic_t **inode;
 	sqfs_block_t *blk_current;
 	sqfs_u32 blk_flags;
 	sqfs_u32 blk_index;
diff --git a/mkfs/mkfs.c b/mkfs/mkfs.c
index 691c754..0ef65ca 100644
--- a/mkfs/mkfs.c
+++ b/mkfs/mkfs.c
@@ -42,15 +42,13 @@ static int set_working_dir(options_t *opt)
 static int pack_files(sqfs_block_processor_t *data, fstree_t *fs,
 		      options_t *opt)
 {
-	sqfs_inode_generic_t *inode;
-	size_t max_blk_count;
+	sqfs_inode_generic_t **inode_ptr;
 	sqfs_u64 filesize;
 	sqfs_file_t *file;
 	tree_node_t *node;
 	const char *path;
 	char *node_path;
 	file_info_t *fi;
-	size_t size;
 	int flags;
 	int ret;
 
@@ -86,38 +84,15 @@ static int pack_files(sqfs_block_processor_t *data, fstree_t *fs,
 			return -1;
 		}
 
-		filesize = file->get_size(file);
-
-		max_blk_count = filesize / opt->cfg.block_size;
-		if (filesize % opt->cfg.block_size)
-			++max_blk_count;
-
-		if (SZ_MUL_OV(sizeof(sqfs_u32), max_blk_count, &size) ||
-		    SZ_ADD_OV(sizeof(*inode), size, &size)) {
-			fputs("creating file inode: too many blocks\n",
-			      stderr);
-			sqfs_destroy(file);
-			free(node_path);
-			return -1;
-		}
-
-		inode = calloc(1, size);
-		if (inode == NULL) {
-			perror("creating file inode");
-			sqfs_destroy(file);
-			free(node_path);
-			return -1;
-		}
-
-		inode->base.type = SQFS_INODE_FILE;
-		fi->user_ptr = inode;
-
 		flags = 0;
+		filesize = file->get_size(file);
 
 		if (opt->no_tail_packing && filesize > opt->cfg.block_size)
 			flags |= SQFS_BLK_DONT_FRAGMENT;
 
-		ret = write_data_from_file(path, data, inode, file, flags);
+		inode_ptr = (sqfs_inode_generic_t **)&fi->user_ptr;
+
+		ret = write_data_from_file(path, data, inode_ptr, file, flags);
 		sqfs_destroy(file);
 		free(node_path);
 
diff --git a/tar/tar2sqfs.c b/tar/tar2sqfs.c
index 69d445e..5c18312 100644
--- a/tar/tar2sqfs.c
+++ b/tar/tar2sqfs.c
@@ -250,32 +250,10 @@ fail_arg:
 static int write_file(tar_header_decoded_t *hdr, file_info_t *fi,
 		      sqfs_u64 filesize)
 {
-	sqfs_inode_generic_t *inode;
-	size_t size, max_blk_count;
 	sqfs_file_t *file;
 	int flags;
 	int ret;
 
-	max_blk_count = filesize / cfg.block_size;
-	if (filesize % cfg.block_size)
-		++max_blk_count;
-
-	if (SZ_MUL_OV(sizeof(sqfs_u32), max_blk_count, &size) ||
-	    SZ_ADD_OV(sizeof(*inode), size, &size)) {
-		fputs("creating file inode: too many blocks\n",
-		      stderr);
-		return -1;
-	}
-
-	inode = calloc(1, size);
-	if (inode == NULL) {
-		perror("creating file inode");
-		return -1;
-	}
-
-	inode->base.type = SQFS_INODE_FILE;
-	fi->user_ptr = inode;
-
 	file = sqfs_get_stdin_file(input_file, hdr->sparse, filesize);
 	if (file == NULL) {
 		perror("packing files");
@@ -286,7 +264,9 @@ static int write_file(tar_header_decoded_t *hdr, file_info_t *fi,
 	if (no_tail_pack && filesize > cfg.block_size)
 		flags |= SQFS_BLK_DONT_FRAGMENT;
 
-	ret = write_data_from_file(hdr->name, sqfs.data, inode, file, 0);
+	ret = write_data_from_file(hdr->name, sqfs.data,
+				   (sqfs_inode_generic_t **)&fi->user_ptr,
+				   file, 0);
 	sqfs_destroy(file);
 
 	if (ret)