Ensure that tar2sqfs & sqfs2tar set stdin/out to binary mode

As usual, Windows has things different and is the platform where
the problem was actually discovered.

Signed-off-by: David Oberhollenzer <david.oberhollenzer@sigma-star.at>

4 files changed, 52 insertions, 15 deletions

diff --git a/include/common.h b/include/common.h
index b8fe6e4..684b047 100644
--- a/include/common.h
+++ b/include/common.h
@@ -120,7 +120,8 @@ int sqfs_data_reader_dump(const char *name, sqfs_data_reader_t *data,
 			  const sqfs_inode_generic_t *inode,
 			  FILE *fp, size_t block_size, bool allow_sparse);
 
-sqfs_file_t *sqfs_get_stdin_file(const sparse_map_t *map, sqfs_u64 size);
+sqfs_file_t *sqfs_get_stdin_file(FILE *fp, const sparse_map_t *map,
+				 sqfs_u64 size);
 
 void register_stat_hooks(sqfs_data_writer_t *data, data_writer_stats_t *stats);
 
diff --git a/lib/common/io_stdin.c b/lib/common/io_stdin.c
index dbdea5f..7abcd79 100644
--- a/lib/common/io_stdin.c
+++ b/lib/common/io_stdin.c
@@ -18,6 +18,7 @@ typedef struct {
 	const sparse_map_t *map;
 	sqfs_u64 offset;
 	sqfs_u64 size;
+	FILE *fp;
 } sqfs_file_stdinout_t;
 
 
@@ -58,19 +59,19 @@ static int stdin_read_at(sqfs_file_t *base, sqfs_u64 offset,
 		return SQFS_ERROR_OUT_OF_BOUNDS;
 
 	while (size > 0) {
-		if (ferror(stdin))
+		if (ferror(file->fp))
 			return SQFS_ERROR_IO;
 
-		if (feof(stdin))
+		if (feof(file->fp))
 			return SQFS_ERROR_OUT_OF_BOUNDS;
 
 		if (offset > file->offset) {
 			diff = file->offset - offset;
 			diff = diff > (sqfs_u64)temp_size ? temp_size : diff;
 
-			ret = fread(temp, 1, diff, stdin);
+			ret = fread(temp, 1, diff, file->fp);
 		} else {
-			ret = fread(buffer, 1, size, stdin);
+			ret = fread(buffer, 1, size, file->fp);
 		}
 
 		if (offset <= file->offset) {
@@ -146,7 +147,8 @@ static int stdin_write_at(sqfs_file_t *base, sqfs_u64 offset,
 	return SQFS_ERROR_IO;
 }
 
-sqfs_file_t *sqfs_get_stdin_file(const sparse_map_t *map, sqfs_u64 size)
+sqfs_file_t *sqfs_get_stdin_file(FILE *fp, const sparse_map_t *map,
+				 sqfs_u64 size)
 {
 	sqfs_file_stdinout_t *file = calloc(1, sizeof(*file));
 	sqfs_file_t *base = (sqfs_file_t *)file;
@@ -156,6 +158,7 @@ sqfs_file_t *sqfs_get_stdin_file(const sparse_map_t *map, sqfs_u64 size)
 
 	file->size = size;
 	file->map = map;
+	file->fp = fp;
 
 	base->destroy = stdinout_destroy;
 	base->write_at = stdin_write_at;
diff --git a/tar/sqfs2tar.c b/tar/sqfs2tar.c
index 8eac991..3182439 100644
--- a/tar/sqfs2tar.c
+++ b/tar/sqfs2tar.c
@@ -78,6 +78,8 @@ static sqfs_data_reader_t *data;
 static sqfs_file_t *file;
 static sqfs_super_t super;
 
+static FILE *out_file = NULL;
+
 static void process_args(int argc, char **argv)
 {
 	size_t idx, new_count;
@@ -174,7 +176,7 @@ static int terminate_archive(void)
 
 	memset(buffer, '\0', sizeof(buffer));
 
-	return write_retry("adding archive terminator", stdout,
+	return write_retry("adding archive terminator", out_file,
 			   buffer, sizeof(buffer));
 }
 
@@ -295,7 +297,7 @@ static int write_tree_dfs(const sqfs_tree_node_t *n)
 	}
 
 	target = S_ISLNK(sb.st_mode) ? n->inode->slink_target : NULL;
-	ret = write_tar_header(stdout, &sb, name, target, xattr,
+	ret = write_tar_header(out_file, &sb, name, target, xattr,
 			       record_counter++);
 
 	while (xattr != NULL) {
@@ -313,13 +315,13 @@ static int write_tree_dfs(const sqfs_tree_node_t *n)
 	}
 
 	if (S_ISREG(sb.st_mode)) {
-		if (sqfs_data_reader_dump(name, data, n->inode, stdout,
+		if (sqfs_data_reader_dump(name, data, n->inode, out_file,
 					  super.block_size, false)) {
 			free(name);
 			return -1;
 		}
 
-		if (padd_file(stdout, sb.st_size)) {
+		if (padd_file(out_file, sb.st_size)) {
 			free(name);
 			return -1;
 		}
@@ -395,6 +397,18 @@ int main(int argc, char **argv)
 
 	process_args(argc, argv);
 
+#ifdef _WIN32
+	_setmode(_fileno(stdout), _O_BINARY);
+	out_file = stdout;
+#else
+	out_file = freopen(NULL, "wb", stdout);
+#endif
+
+	if (out_file == NULL) {
+		perror("changing stdout to binary mode");
+		goto out_dirs;
+	}
+
 	file = sqfs_open_file(filename, SQFS_FILE_OPEN_READ_ONLY);
 	if (file == NULL) {
 		perror(filename);
@@ -518,6 +532,7 @@ int main(int argc, char **argv)
 		goto out;
 
 	status = EXIT_SUCCESS;
+	fflush(out_file);
 out:
 	if (root != NULL)
 		sqfs_dir_tree_destroy(root);
diff --git a/tar/tar2sqfs.c b/tar/tar2sqfs.c
index 94168fc..69ac5cf 100644
--- a/tar/tar2sqfs.c
+++ b/tar/tar2sqfs.c
@@ -13,6 +13,11 @@
 #include <unistd.h>
 #include <string.h>
 #include <stdio.h>
+#include <fcntl.h>
+
+#ifdef _WIN32
+#include <io.h>
+#endif
 
 static struct option long_opts[] = {
 	{ "compressor", required_argument, NULL, 'c' },
@@ -87,6 +92,7 @@ static bool dont_skip = false;
 static bool keep_time = true;
 static sqfs_writer_cfg_t cfg;
 static sqfs_writer_t sqfs;
+static FILE *input_file = NULL;
 
 static void process_args(int argc, char **argv)
 {
@@ -229,13 +235,13 @@ static int write_file(tar_header_decoded_t *hdr, file_info_t *fi,
 		for (sum = 0, it = hdr->sparse; it != NULL; it = it->next)
 			sum += it->count;
 
-		file = sqfs_get_stdin_file(hdr->sparse, sum);
+		file = sqfs_get_stdin_file(input_file, hdr->sparse, sum);
 		if (file == NULL) {
 			perror("packing files");
 			return -1;
 		}
 	} else {
-		file = sqfs_get_stdin_file(NULL, filesize);
+		file = sqfs_get_stdin_file(input_file, NULL, filesize);
 		if (file == NULL) {
 			perror("packing files");
 			return -1;
@@ -251,7 +257,7 @@ static int write_file(tar_header_decoded_t *hdr, file_info_t *fi,
 	if (ret)
 		return -1;
 
-	return skip_padding(stdin, hdr->sparse == NULL ?
+	return skip_padding(input_file, hdr->sparse == NULL ?
 			    filesize : hdr->record_size);
 }
 
@@ -337,7 +343,7 @@ static int process_tar_ball(void)
 	int ret;
 
 	for (;;) {
-		ret = read_header(stdin, &hdr);
+		ret = read_header(input_file, &hdr);
 		if (ret > 0)
 			break;
 		if (ret < 0)
@@ -395,7 +401,7 @@ static int process_tar_ball(void)
 		if (skip) {
 			if (dont_skip)
 				goto fail;
-			if (skip_entry(stdin, hdr.sb.st_size))
+			if (skip_entry(input_file, hdr.sb.st_size))
 				goto fail;
 
 			clear_header(&hdr);
@@ -420,6 +426,18 @@ int main(int argc, char **argv)
 
 	process_args(argc, argv);
 
+#ifdef _WIN32
+	_setmode(_fileno(stdin), _O_BINARY);
+	input_file = stdin;
+#else
+	input_file = freopen(NULL, "rb", stdin);
+#endif
+
+	if (input_file == NULL) {
+		perror("changing stdin to binary mode");
+		return EXIT_FAILURE;
+	}
+
 	if (sqfs_writer_init(&sqfs, &cfg))
 		return EXIT_FAILURE;