Fix: consistently use the widechar file API on Windows

When opening files on windows, use the widechar versions and convert
from (assumed) UTF-8 to UTF-16 as needed.

Since the broken, code-page-random API may acutall be intended in some
use cases, leave that option in through an additional flag.

Signed-off-by: David Oberhollenzer <david.oberhollenzer@sigma-star.at>

3 files changed, 55 insertions, 7 deletions

diff --git a/lib/fstream/win32/istream.c b/lib/fstream/win32/istream.c
index 96c9f1c..36d7c42 100644
--- a/lib/fstream/win32/istream.c
+++ b/lib/fstream/win32/istream.c
@@ -69,19 +69,24 @@ istream_t *istream_open_file(const char *path)
 	file_istream_t *file = calloc(1, sizeof(*file));
 	sqfs_object_t *obj = (sqfs_object_t *)file;
 	istream_t *strm = (istream_t *)file;
+	WCHAR *wpath = NULL;
 
 	if (file == NULL) {
 		perror(path);
 		return NULL;
 	}
 
+	wpath = path_to_windows(path);
+	if (wpath == NULL)
+		goto fail_free;
+
 	file->path = strdup(path);
 	if (file->path == NULL) {
 		perror(path);
 		goto fail_free;
 	}
 
-	file->hnd = CreateFile(path, GENERIC_READ, FILE_SHARE_READ, NULL,
+	file->hnd = CreateFileW(wpath, GENERIC_READ, FILE_SHARE_READ, NULL,
 			       OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, NULL);
 
 	if (file->hnd == INVALID_HANDLE_VALUE) {
@@ -89,6 +94,8 @@ istream_t *istream_open_file(const char *path)
 		goto fail_path;
 	}
 
+	free(wpath);
+
 	strm->buffer = file->buffer;
 	strm->precache = file_precache;
 	strm->get_filename = file_get_filename;
@@ -97,6 +104,7 @@ istream_t *istream_open_file(const char *path)
 fail_path:
 	free(file->path);
 fail_free:
+	free(wpath);
 	free(file);
 	return NULL;
 }
diff --git a/lib/fstream/win32/ostream.c b/lib/fstream/win32/ostream.c
index 7b5bd3e..61dae37 100644
--- a/lib/fstream/win32/ostream.c
+++ b/lib/fstream/win32/ostream.c
@@ -128,12 +128,17 @@ ostream_t *ostream_open_file(const char *path, int flags)
 	sqfs_object_t *obj = (sqfs_object_t *)file;
 	ostream_t *strm = (ostream_t *)file;
 	int access_flags, creation_mode;
+	WCHAR *wpath = NULL;
 
 	if (file == NULL) {
 		perror(path);
 		return NULL;
 	}
 
+	wpath = path_to_windows(path);
+	if (wpath == NULL)
+		goto fail_free;
+
 	file->path = strdup(path);
 	if (file->path == NULL) {
 		perror(path);
@@ -148,14 +153,16 @@ ostream_t *ostream_open_file(const char *path, int flags)
 		creation_mode = CREATE_NEW;
 	}
 
-	file->hnd = CreateFile(path, access_flags, 0, NULL, creation_mode,
-			       FILE_ATTRIBUTE_NORMAL, NULL);
+	file->hnd = CreateFileW(wpath, access_flags, 0, NULL, creation_mode,
+				FILE_ATTRIBUTE_NORMAL, NULL);
 
 	if (file->hnd == INVALID_HANDLE_VALUE) {
 		w32_perror(path);
 		goto fail_path;
 	}
 
+	free(wpath);
+
 	if (flags & OSTREAM_OPEN_SPARSE)
 		strm->append_sparse = file_append_sparse;
 
@@ -168,6 +175,7 @@ fail_path:
 	free(file->path);
 fail_free:
 	free(file);
+	free(wpath);
 	return NULL;
 }
 
diff --git a/lib/sqfs/win32/io_file.c b/lib/sqfs/win32/io_file.c
index 7ff42b7..cfef296 100644
--- a/lib/sqfs/win32/io_file.c
+++ b/lib/sqfs/win32/io_file.c
@@ -155,14 +155,37 @@ sqfs_file_t *sqfs_open_file(const char *filename, sqfs_u32 flags)
 	sqfs_file_stdio_t *file;
 	LARGE_INTEGER size;
 	sqfs_file_t *base;
+	WCHAR *wpath = NULL;
+	DWORD length;
 
-	if (flags & ~SQFS_FILE_OPEN_ALL_FLAGS)
+	if (flags & ~SQFS_FILE_OPEN_ALL_FLAGS) {
+		SetLastError(ERROR_INVALID_PARAMETER);
 		return NULL;
+	}
+
+	if (!(flags & SQFS_FILE_OPEN_NO_CHARSET_XFRM)) {
+		length = MultiByteToWideChar(CP_UTF8, 0, filename, -1, NULL, 0);
+		if (length <= 0)
+			return NULL;
+
+		wpath = calloc(sizeof(wpath[0]), length + 1);
+		if (wpath == NULL) {
+			SetLastError(ERROR_NOT_ENOUGH_MEMORY);
+			return NULL;
+		}
+
+		MultiByteToWideChar(CP_UTF8, 0, filename, -1,
+				    wpath, length + 1);
+		wpath[length] = '\0';
+	}
 
 	file = calloc(1, sizeof(*file));
 	base = (sqfs_file_t *)file;
-	if (file == NULL)
+	if (file == NULL) {
+		free(wpath);
+		SetLastError(ERROR_NOT_ENOUGH_MEMORY);
 		return NULL;
+	}
 
 	if (flags & SQFS_FILE_OPEN_READ_ONLY) {
 		file->readonly = true;
@@ -181,8 +204,17 @@ sqfs_file_t *sqfs_open_file(const char *filename, sqfs_u32 flags)
 		}
 	}
 
-	file->fd = CreateFile(filename, access_flags, share_mode, NULL, creation_mode,
-			      FILE_ATTRIBUTE_NORMAL, NULL);
+	if (flags & SQFS_FILE_OPEN_NO_CHARSET_XFRM) {
+		file->fd = CreateFileA(filename, access_flags, share_mode, NULL,
+				       creation_mode, FILE_ATTRIBUTE_NORMAL,
+				       NULL);
+	} else {
+		file->fd = CreateFileW(wpath, access_flags, share_mode, NULL,
+				       creation_mode, FILE_ATTRIBUTE_NORMAL,
+				       NULL);
+	}
+
+	free(wpath);
 
 	if (file->fd == INVALID_HANDLE_VALUE) {
 		free(file);