Windows: redirect standard I/O and convert text to UTF-16

Preprocessor magic is used to redirect putc/fputc/fputs/printf/fprintf
to custom implementations.

The custom implementations try to figure out if we are printing to the
console and, if so, convert the resulting strings to UTF-16 and print
them through ConsoleWriteW. If the output is redirected to a file or
a pipe, the original (presummed) UTF-8 is kept.

Simply setting the console output codepage to UTF-8 does not work,
because the standard I/O facilities of MSVCRT either does not support
unicode (in non-wchar mode), or has half-broken support through fputs,
which can still break up multi-byte sequences through its internal
buffering.

Likewise, changing the codepage and using ConsoleWriteA, or trying to
use fputws did not work in a test VM either.

This approach is the one that worked most consistently among the
ones tried, but also has problems. E.g. it breaks when setting the
codepage to UTF-8 manually (using `chcp 65001`).

Signed-off-by: David Oberhollenzer <david.oberhollenzer@sigma-star.at>

3 files changed, 140 insertions, 2 deletions

diff --git a/include/compat.h b/include/compat.h
index 59133c9..65872ec 100644
--- a/include/compat.h
+++ b/include/compat.h
@@ -8,9 +8,11 @@
 #define COMPAT_H
 
 #include "sqfs/predef.h"
+#include "fstream.h"
 #include "config.h"
 
 #include <limits.h>
+#include <stdio.h>
 
 #if defined(__GNUC__) && __GNUC__ >= 5
 #	define SZ_ADD_OV __builtin_add_overflow
@@ -228,9 +230,19 @@ int fnmatch(const char *, const char *, int);
 #endif
 
 #if defined(_WIN32) || defined(__WINDOWS__)
-#define main sqfs_tools_main
-
 extern int sqfs_tools_main(int argc, char **argv);
+
+int stfs_tools_fputc(int c, FILE *strm);
+int stfs_tools_fputs(const char *str, FILE *strm);
+int stfs_tools_printf(const char *fmt, ...) PRINTF_ATTRIB(1, 2);
+int stfs_tools_fprintf(FILE *strm, const char *fmt, ...) PRINTF_ATTRIB(2, 3);
+
+#define main sqfs_tools_main
+#define printf stfs_tools_printf
+#define fprintf stfs_tools_fprintf
+#define fputs stfs_tools_fputs
+#define fputc stfs_tools_fputc
+#define putc stfs_tools_fputc
 #endif
 
 #endif /* COMPAT_H */
diff --git a/lib/compat/Makemodule.am b/lib/compat/Makemodule.am
index f9dce2e..0426075 100644
--- a/lib/compat/Makemodule.am
+++ b/lib/compat/Makemodule.am
@@ -4,6 +4,7 @@ libcompat_a_SOURCES += lib/compat/chdir.c include/compat.h
 libcompat_a_SOURCES += lib/compat/path_to_windows.c
 libcompat_a_SOURCES += lib/compat/w32_perror.c
 libcompat_a_SOURCES += lib/compat/w32_wmain.c
+libcompat_a_SOURCES += lib/compat/w32_stdio.c
 libcompat_a_SOURCES += lib/compat/fnmatch.c
 libcompat_a_SOURCES += lib/compat/getopt.c
 libcompat_a_SOURCES += lib/compat/getopt_long.c
diff --git a/lib/compat/w32_stdio.c b/lib/compat/w32_stdio.c
new file mode 100644
index 0000000..3124899
--- /dev/null
+++ b/lib/compat/w32_stdio.c
@@ -0,0 +1,125 @@
+/* SPDX-License-Identifier: GPL-3.0-or-later */
+/*
+ * w32_stdio.c
+ *
+ * Copyright (C) 2021 David Oberhollenzer <goliath@infraroot.at>
+ */
+#include "config.h"
+#include "compat.h"
+
+#include <stdio.h>
+#include <stdlib.h>
+
+#ifdef _WIN32
+#define WIN32_LEAN_AND_MEAN
+#include <windows.h>
+#include <stdarg.h>
+
+#undef fputc
+#undef putc
+#undef fputs
+#undef fprintf
+#undef printf
+
+static HANDLE get_handle(FILE *strm)
+{
+	if (strm != stdout && strm != stderr)
+		return INVALID_HANDLE_VALUE;
+
+	return GetStdHandle(strm == stderr ?
+			    STD_ERROR_HANDLE : STD_OUTPUT_HANDLE);
+}
+
+static BOOL isatty(HANDLE hnd)
+{
+	if (hnd == INVALID_HANDLE_VALUE)
+		return FALSE;
+
+	return (GetFileType(hnd) == FILE_TYPE_CHAR);
+}
+
+int stfs_tools_fputs(const char *str, FILE *strm)
+{
+	DWORD length;
+	WCHAR *wstr;
+	HANDLE hnd;
+	int ret;
+
+	hnd = get_handle(strm);
+	if (!isatty(hnd))
+		return fputs(str, strm);
+
+	length = MultiByteToWideChar(CP_UTF8, 0, str, -1, NULL, 0);
+	if (length <= 0)
+		return EOF;
+
+	wstr = calloc(sizeof(wstr[0]), length);
+	if (wstr == NULL)
+		return EOF;
+
+	MultiByteToWideChar(CP_UTF8, 0, str, -1, wstr, length);
+
+	ret = WriteConsoleW(hnd, wstr, length, NULL, NULL) ? length : EOF;
+
+	free(wstr);
+	return ret;
+}
+
+int stfs_tools_fputc(int c, FILE *strm)
+{
+	char str[2];
+
+	str[0] = c;
+	str[1] = '\0';
+
+	return stfs_tools_fputs(str, strm);
+}
+
+static int sqfs_printf_common(FILE *out, const char *fmt, va_list ap)
+{
+	int ret, len;
+	char *str;
+
+	len = _vscprintf(fmt, ap);
+	if (len == -1)
+		return -1;
+
+	str = malloc((size_t)len + 1);
+	if (str == NULL)
+		return -1;
+
+	ret = vsprintf(str, fmt, ap);
+	if (ret == -1) {
+		free(str);
+		return -1;
+	}
+
+	if (stfs_tools_fputs(str, out) == EOF)
+		ret = -1;
+
+	free(str);
+	return ret;
+}
+
+int stfs_tools_printf(const char *fmt, ...)
+{
+	va_list ap;
+	int ret;
+
+	va_start(ap, fmt);
+	ret = sqfs_printf_common(stdout, fmt, ap);
+	va_end(ap);
+	return ret;
+}
+
+int stfs_tools_fprintf(FILE *strm, const char *fmt, ...)
+{
+	va_list ap;
+	int ret;
+
+	va_start(ap, fmt);
+	ret = sqfs_printf_common(strm, fmt, ap);
+	va_end(ap);
+	return ret;
+}
+#endif