Fix: consistently use the widechar file API on Windows

When opening files on windows, use the widechar versions and convert
from (assumed) UTF-8 to UTF-16 as needed.

Since the broken, code-page-random API may acutall be intended in some
use cases, leave that option in through an additional flag.

Signed-off-by: David Oberhollenzer <david.oberhollenzer@sigma-star.at>

4 files changed, 80 insertions, 9 deletions

diff --git a/include/sqfs/io.h b/include/sqfs/io.h
index 246ac70..4cdb574 100644
--- a/include/sqfs/io.h
+++ b/include/sqfs/io.h
@@ -54,7 +54,29 @@ typedef enum {
 	 */
 	SQFS_FILE_OPEN_OVERWRITE = 0x02,
 
-	SQFS_FILE_OPEN_ALL_FLAGS = 0x03,
+	/**
+	 * @brief If set, do not try to apply any character set transformations
+	 *        to the file path.
+	 *
+	 * This flag instructs the @ref sqfs_open_file API to pass the file
+	 * path to the OS dependent API as-is and not to attempt any character
+	 * set transformation. By default, if the underlying OS has a concept
+	 * of locale dependent file paths, the input path is UTF-8 and it is
+	 * transformed as needed, in order to retain a level of portabillity
+	 * and sanity.
+	 *
+	 * This flag currently only affects the Windows implementation. On
+	 * Unix-like systems, the path is always passed to the OS API as-is
+	 * and this flag has no effect.
+	 *
+	 * On Windows, the input path is converted from UTF-8 to UTF-16 and
+	 * then passed on to the wide-char based file API. If this flag is set,
+	 * the path is used as-is and passed on to the 8-bit codepage-whatever
+	 * API instead.
+	 */
+	SQFS_FILE_OPEN_NO_CHARSET_XFRM = 0x04,
+
+	SQFS_FILE_OPEN_ALL_FLAGS = 0x07,
 } SQFS_FILE_OPEN_FLAGS;
 
 /**
@@ -134,7 +156,8 @@ extern "C" {
  * API for file I/O.
  *
  * On Unix-like systems, if the open call fails, this function makes sure to
- * preserves the value in errno indicating the underlying problem.
+ * preserves the value in errno indicating the underlying problem. Similarly,
+ * on Windows, the implementation tries to preserve the GetLastError value.
  *
  * @param filename The name of the file to open.
  * @param flags A set of @ref SQFS_FILE_OPEN_FLAGS.
diff --git a/lib/fstream/win32/istream.c b/lib/fstream/win32/istream.c
index 96c9f1c..36d7c42 100644
--- a/lib/fstream/win32/istream.c
+++ b/lib/fstream/win32/istream.c
@@ -69,19 +69,24 @@ istream_t *istream_open_file(const char *path)
 	file_istream_t *file = calloc(1, sizeof(*file));
 	sqfs_object_t *obj = (sqfs_object_t *)file;
 	istream_t *strm = (istream_t *)file;
+	WCHAR *wpath = NULL;
 
 	if (file == NULL) {
 		perror(path);
 		return NULL;
 	}
 
+	wpath = path_to_windows(path);
+	if (wpath == NULL)
+		goto fail_free;
+
 	file->path = strdup(path);
 	if (file->path == NULL) {
 		perror(path);
 		goto fail_free;
 	}
 
-	file->hnd = CreateFile(path, GENERIC_READ, FILE_SHARE_READ, NULL,
+	file->hnd = CreateFileW(wpath, GENERIC_READ, FILE_SHARE_READ, NULL,
 			       OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, NULL);
 
 	if (file->hnd == INVALID_HANDLE_VALUE) {
@@ -89,6 +94,8 @@ istream_t *istream_open_file(const char *path)
 		goto fail_path;
 	}
 
+	free(wpath);
+
 	strm->buffer = file->buffer;
 	strm->precache = file_precache;
 	strm->get_filename = file_get_filename;
@@ -97,6 +104,7 @@ istream_t *istream_open_file(const char *path)
 fail_path:
 	free(file->path);
 fail_free:
+	free(wpath);
 	free(file);
 	return NULL;
 }
diff --git a/lib/fstream/win32/ostream.c b/lib/fstream/win32/ostream.c
index 7b5bd3e..61dae37 100644
--- a/lib/fstream/win32/ostream.c
+++ b/lib/fstream/win32/ostream.c
@@ -128,12 +128,17 @@ ostream_t *ostream_open_file(const char *path, int flags)
 	sqfs_object_t *obj = (sqfs_object_t *)file;
 	ostream_t *strm = (ostream_t *)file;
 	int access_flags, creation_mode;
+	WCHAR *wpath = NULL;
 
 	if (file == NULL) {
 		perror(path);
 		return NULL;
 	}
 
+	wpath = path_to_windows(path);
+	if (wpath == NULL)
+		goto fail_free;
+
 	file->path = strdup(path);
 	if (file->path == NULL) {
 		perror(path);
@@ -148,14 +153,16 @@ ostream_t *ostream_open_file(const char *path, int flags)
 		creation_mode = CREATE_NEW;
 	}
 
-	file->hnd = CreateFile(path, access_flags, 0, NULL, creation_mode,
-			       FILE_ATTRIBUTE_NORMAL, NULL);
+	file->hnd = CreateFileW(wpath, access_flags, 0, NULL, creation_mode,
+				FILE_ATTRIBUTE_NORMAL, NULL);
 
 	if (file->hnd == INVALID_HANDLE_VALUE) {
 		w32_perror(path);
 		goto fail_path;
 	}
 
+	free(wpath);
+
 	if (flags & OSTREAM_OPEN_SPARSE)
 		strm->append_sparse = file_append_sparse;
 
@@ -168,6 +175,7 @@ fail_path:
 	free(file->path);
 fail_free:
 	free(file);
+	free(wpath);
 	return NULL;
 }
 
diff --git a/lib/sqfs/win32/io_file.c b/lib/sqfs/win32/io_file.c
index 7ff42b7..cfef296 100644
--- a/lib/sqfs/win32/io_file.c
+++ b/lib/sqfs/win32/io_file.c
@@ -155,14 +155,37 @@ sqfs_file_t *sqfs_open_file(const char *filename, sqfs_u32 flags)
 	sqfs_file_stdio_t *file;
 	LARGE_INTEGER size;
 	sqfs_file_t *base;
+	WCHAR *wpath = NULL;
+	DWORD length;
 
-	if (flags & ~SQFS_FILE_OPEN_ALL_FLAGS)
+	if (flags & ~SQFS_FILE_OPEN_ALL_FLAGS) {
+		SetLastError(ERROR_INVALID_PARAMETER);
 		return NULL;
+	}
+
+	if (!(flags & SQFS_FILE_OPEN_NO_CHARSET_XFRM)) {
+		length = MultiByteToWideChar(CP_UTF8, 0, filename, -1, NULL, 0);
+		if (length <= 0)
+			return NULL;
+
+		wpath = calloc(sizeof(wpath[0]), length + 1);
+		if (wpath == NULL) {
+			SetLastError(ERROR_NOT_ENOUGH_MEMORY);
+			return NULL;
+		}
+
+		MultiByteToWideChar(CP_UTF8, 0, filename, -1,
+				    wpath, length + 1);
+		wpath[length] = '\0';
+	}
 
 	file = calloc(1, sizeof(*file));
 	base = (sqfs_file_t *)file;
-	if (file == NULL)
+	if (file == NULL) {
+		free(wpath);
+		SetLastError(ERROR_NOT_ENOUGH_MEMORY);
 		return NULL;
+	}
 
 	if (flags & SQFS_FILE_OPEN_READ_ONLY) {
 		file->readonly = true;
@@ -181,8 +204,17 @@ sqfs_file_t *sqfs_open_file(const char *filename, sqfs_u32 flags)
 		}
 	}
 
-	file->fd = CreateFile(filename, access_flags, share_mode, NULL, creation_mode,
-			      FILE_ATTRIBUTE_NORMAL, NULL);
+	if (flags & SQFS_FILE_OPEN_NO_CHARSET_XFRM) {
+		file->fd = CreateFileA(filename, access_flags, share_mode, NULL,
+				       creation_mode, FILE_ATTRIBUTE_NORMAL,
+				       NULL);
+	} else {
+		file->fd = CreateFileW(wpath, access_flags, share_mode, NULL,
+				       creation_mode, FILE_ATTRIBUTE_NORMAL,
+				       NULL);
+	}
+
+	free(wpath);
 
 	if (file->fd == INVALID_HANDLE_VALUE) {
 		free(file);