Fix: consistently use the widechar file API on Windows

When opening files on windows, use the widechar versions and convert
from (assumed) UTF-8 to UTF-16 as needed.

Since the broken, code-page-random API may acutall be intended in some
use cases, leave that option in through an additional flag.

Signed-off-by: David Oberhollenzer <david.oberhollenzer@sigma-star.at>

2 files changed, 61 insertions, 6 deletions

diff --git a/include/sqfs/io.h b/include/sqfs/io.h
index 246ac70..4cdb574 100644
--- a/include/sqfs/io.h
+++ b/include/sqfs/io.h
@@ -54,7 +54,29 @@ typedef enum {
 	 */
 	SQFS_FILE_OPEN_OVERWRITE = 0x02,
 
-	SQFS_FILE_OPEN_ALL_FLAGS = 0x03,
+	/**
+	 * @brief If set, do not try to apply any character set transformations
+	 *        to the file path.
+	 *
+	 * This flag instructs the @ref sqfs_open_file API to pass the file
+	 * path to the OS dependent API as-is and not to attempt any character
+	 * set transformation. By default, if the underlying OS has a concept
+	 * of locale dependent file paths, the input path is UTF-8 and it is
+	 * transformed as needed, in order to retain a level of portabillity
+	 * and sanity.
+	 *
+	 * This flag currently only affects the Windows implementation. On
+	 * Unix-like systems, the path is always passed to the OS API as-is
+	 * and this flag has no effect.
+	 *
+	 * On Windows, the input path is converted from UTF-8 to UTF-16 and
+	 * then passed on to the wide-char based file API. If this flag is set,
+	 * the path is used as-is and passed on to the 8-bit codepage-whatever
+	 * API instead.
+	 */
+	SQFS_FILE_OPEN_NO_CHARSET_XFRM = 0x04,
+
+	SQFS_FILE_OPEN_ALL_FLAGS = 0x07,
 } SQFS_FILE_OPEN_FLAGS;
 
 /**
@@ -134,7 +156,8 @@ extern "C" {
  * API for file I/O.
  *
  * On Unix-like systems, if the open call fails, this function makes sure to
- * preserves the value in errno indicating the underlying problem.
+ * preserves the value in errno indicating the underlying problem. Similarly,
+ * on Windows, the implementation tries to preserve the GetLastError value.
  *
  * @param filename The name of the file to open.
  * @param flags A set of @ref SQFS_FILE_OPEN_FLAGS.
diff --git a/lib/sqfs/win32/io_file.c b/lib/sqfs/win32/io_file.c
index 7ff42b7..cfef296 100644
--- a/lib/sqfs/win32/io_file.c
+++ b/lib/sqfs/win32/io_file.c
@@ -155,14 +155,37 @@ sqfs_file_t *sqfs_open_file(const char *filename, sqfs_u32 flags)
 	sqfs_file_stdio_t *file;
 	LARGE_INTEGER size;
 	sqfs_file_t *base;
+	WCHAR *wpath = NULL;
+	DWORD length;
 
-	if (flags & ~SQFS_FILE_OPEN_ALL_FLAGS)
+	if (flags & ~SQFS_FILE_OPEN_ALL_FLAGS) {
+		SetLastError(ERROR_INVALID_PARAMETER);
 		return NULL;
+	}
+
+	if (!(flags & SQFS_FILE_OPEN_NO_CHARSET_XFRM)) {
+		length = MultiByteToWideChar(CP_UTF8, 0, filename, -1, NULL, 0);
+		if (length <= 0)
+			return NULL;
+
+		wpath = calloc(sizeof(wpath[0]), length + 1);
+		if (wpath == NULL) {
+			SetLastError(ERROR_NOT_ENOUGH_MEMORY);
+			return NULL;
+		}
+
+		MultiByteToWideChar(CP_UTF8, 0, filename, -1,
+				    wpath, length + 1);
+		wpath[length] = '\0';
+	}
 
 	file = calloc(1, sizeof(*file));
 	base = (sqfs_file_t *)file;
-	if (file == NULL)
+	if (file == NULL) {
+		free(wpath);
+		SetLastError(ERROR_NOT_ENOUGH_MEMORY);
 		return NULL;
+	}
 
 	if (flags & SQFS_FILE_OPEN_READ_ONLY) {
 		file->readonly = true;
@@ -181,8 +204,17 @@ sqfs_file_t *sqfs_open_file(const char *filename, sqfs_u32 flags)
 		}
 	}
 
-	file->fd = CreateFile(filename, access_flags, share_mode, NULL, creation_mode,
-			      FILE_ATTRIBUTE_NORMAL, NULL);
+	if (flags & SQFS_FILE_OPEN_NO_CHARSET_XFRM) {
+		file->fd = CreateFileA(filename, access_flags, share_mode, NULL,
+				       creation_mode, FILE_ATTRIBUTE_NORMAL,
+				       NULL);
+	} else {
+		file->fd = CreateFileW(wpath, access_flags, share_mode, NULL,
+				       creation_mode, FILE_ATTRIBUTE_NORMAL,
+				       NULL);
+	}
+
+	free(wpath);
 
 	if (file->fd == INVALID_HANDLE_VALUE) {
 		free(file);