libsquashfs: Add sqfs_open_native_file function

Having a function to interpret the flags and open a native file handle
simplifies the istream/ostream/file code which shares that decoding
part, particularly on windows where the character set needs to be
transformed.

Signed-off-by: David Oberhollenzer <david.oberhollenzer@sigma-star.at>

2 files changed, 106 insertions, 89 deletions

diff --git a/lib/sqfs/src/unix/io_file.c b/lib/sqfs/src/unix/io_file.c
index 886f752..f9b602a 100644
--- a/lib/sqfs/src/unix/io_file.c
+++ b/lib/sqfs/src/unix/io_file.c
@@ -24,7 +24,7 @@ typedef struct {
 
 	bool readonly;
 	sqfs_u64 size;
-	int fd;
+	sqfs_file_handle_t fd;
 
 	char name[];
 } sqfs_file_stdio_t;
@@ -146,18 +146,40 @@ static const char *stdio_get_filename(sqfs_file_t *file)
 	return ((sqfs_file_stdio_t *)file)->name;
 }
 
+int sqfs_open_native_file(sqfs_file_handle_t *out, const char *filename,
+			  sqfs_u32 flags)
+{
+	int open_mode;
+
+	*out = -1;
+
+	if (flags & ~SQFS_FILE_OPEN_ALL_FLAGS)
+		return SQFS_ERROR_UNSUPPORTED;
+
+	if (flags & SQFS_FILE_OPEN_READ_ONLY) {
+		open_mode = O_RDONLY;
+	} else {
+		open_mode = O_CREAT | O_RDWR;
+
+		if (flags & SQFS_FILE_OPEN_OVERWRITE) {
+			open_mode |= O_TRUNC;
+		} else {
+			open_mode |= O_EXCL;
+		}
+	}
+
+	*out = open(filename, open_mode, 0644);
+
+	return (*out < 0) ? SQFS_ERROR_IO : 0;
+}
+
 sqfs_file_t *sqfs_open_file(const char *filename, sqfs_u32 flags)
 {
 	sqfs_file_stdio_t *file;
 	size_t size, namelen;
-	int open_mode, temp;
 	sqfs_file_t *base;
 	struct stat sb;
-
-	if (flags & ~SQFS_FILE_OPEN_ALL_FLAGS) {
-		errno = EINVAL;
-		return NULL;
-	}
+	int temp;
 
 	namelen = strlen(filename);
 	size = sizeof(*file) + 1;
@@ -175,37 +197,14 @@ sqfs_file_t *sqfs_open_file(const char *filename, sqfs_u32 flags)
 	sqfs_object_init(file, stdio_destroy, stdio_copy);
 	memcpy(file->name, filename, namelen);
 
-	if (flags & SQFS_FILE_OPEN_READ_ONLY) {
-		file->readonly = true;
-		open_mode = O_RDONLY;
-	} else {
-		file->readonly = false;
-		open_mode = O_CREAT | O_RDWR;
-
-		if (flags & SQFS_FILE_OPEN_OVERWRITE) {
-			open_mode |= O_TRUNC;
-		} else {
-			open_mode |= O_EXCL;
-		}
-	}
-
-	file->fd = open(filename, open_mode, 0644);
-	if (file->fd < 0) {
-		temp = errno;
-		free(file);
-		errno = temp;
-		return NULL;
-	}
+	if (sqfs_open_native_file(&file->fd, filename, flags))
+		goto fail;
 
-	if (fstat(file->fd, &sb)) {
-		temp = errno;
-		close(file->fd);
-		free(file);
-		errno = temp;
-		return NULL;
-	}
+	if (fstat(file->fd, &sb))
+		goto fail;
 
 	file->size = sb.st_size;
+	file->readonly = (flags & SQFS_FILE_OPEN_READ_ONLY) != 0;
 
 	base->read_at = stdio_read_at;
 	base->write_at = stdio_write_at;
@@ -213,4 +212,11 @@ sqfs_file_t *sqfs_open_file(const char *filename, sqfs_u32 flags)
 	base->truncate = stdio_truncate;
 	base->get_filename = stdio_get_filename;
 	return base;
+fail:
+	temp = errno;
+	if (file->fd >= 0)
+		close(file->fd);
+	free(file);
+	errno = temp;
+	return NULL;
 }
diff --git a/lib/sqfs/src/win32/io_file.c b/lib/sqfs/src/win32/io_file.c
index ec98900..dbd80d8 100644
--- a/lib/sqfs/src/win32/io_file.c
+++ b/lib/sqfs/src/win32/io_file.c
@@ -22,7 +22,7 @@ typedef struct {
 
 	bool readonly;
 	sqfs_u64 size;
-	HANDLE fd;
+	sqfs_file_handle_t fd;
 
 	char name[];
 } sqfs_file_stdio_t;
@@ -158,49 +158,84 @@ static const char *stdio_get_filename(sqfs_file_t *file)
 	return ((sqfs_file_stdio_t *)file)->name;
 }
 
-sqfs_file_t *sqfs_open_file(const char *filename, sqfs_u32 flags)
+int sqfs_open_native_file(sqfs_file_handle_t *out, const char *filename,
+			  sqfs_u32 flags)
 {
 	int access_flags, creation_mode, share_mode;
-	sqfs_file_stdio_t *file;
-	size_t namelen, total;
-	LARGE_INTEGER size;
-	sqfs_file_t *base;
 	WCHAR *wpath = NULL;
-	DWORD length;
+	DWORD length, err;
+
+	*out = INVALID_HANDLE_VALUE;
 
 	if (flags & ~SQFS_FILE_OPEN_ALL_FLAGS) {
 		SetLastError(ERROR_INVALID_PARAMETER);
-		return NULL;
+		return SQFS_ERROR_UNSUPPORTED;
 	}
 
-	namelen = strlen(filename);
-	total = sizeof(*file) + 1;
+	if (flags & SQFS_FILE_OPEN_READ_ONLY) {
+		access_flags = GENERIC_READ;
+		creation_mode = OPEN_EXISTING;
+		share_mode = FILE_SHARE_READ;
+	} else {
+		access_flags = GENERIC_READ | GENERIC_WRITE;
+		share_mode = 0;
 
-	if (SZ_ADD_OV(total, namelen, &total)) {
-		SetLastError(ERROR_NOT_ENOUGH_MEMORY);
-		return NULL;
+		if (flags & SQFS_FILE_OPEN_OVERWRITE) {
+			creation_mode = CREATE_ALWAYS;
+		} else {
+			creation_mode = CREATE_NEW;
+		}
 	}
 
-	if (!(flags & SQFS_FILE_OPEN_NO_CHARSET_XFRM)) {
+	if (flags & SQFS_FILE_OPEN_NO_CHARSET_XFRM) {
+		*out = CreateFileA(filename, access_flags, share_mode, NULL,
+				   creation_mode, FILE_ATTRIBUTE_NORMAL, NULL);
+	} else {
 		length = MultiByteToWideChar(CP_UTF8, 0, filename, -1, NULL, 0);
 		if (length <= 0)
-			return NULL;
+			return SQFS_ERROR_INTERNAL;
 
 		wpath = calloc(sizeof(wpath[0]), length + 1);
 		if (wpath == NULL) {
 			SetLastError(ERROR_NOT_ENOUGH_MEMORY);
-			return NULL;
+			return SQFS_ERROR_ALLOC;
 		}
 
 		MultiByteToWideChar(CP_UTF8, 0, filename, -1,
 				    wpath, length + 1);
 		wpath[length] = '\0';
+
+		*out = CreateFileW(wpath, access_flags, share_mode, NULL,
+				   creation_mode, FILE_ATTRIBUTE_NORMAL,
+				   NULL);
+
+		err = GetLastError();
+		free(wpath);
+		SetLastError(err);
+	}
+
+	return (*out == INVALID_HANDLE_VALUE) ? SQFS_ERROR_IO : 0;
+}
+
+sqfs_file_t *sqfs_open_file(const char *filename, sqfs_u32 flags)
+{
+	sqfs_file_stdio_t *file;
+	size_t namelen, total;
+	LARGE_INTEGER size;
+	sqfs_file_t *base;
+	DWORD err;
+
+	namelen = strlen(filename);
+	total = sizeof(*file) + 1;
+
+	if (SZ_ADD_OV(total, namelen, &total)) {
+		SetLastError(ERROR_NOT_ENOUGH_MEMORY);
+		return NULL;
 	}
 
 	file = calloc(1, total);
 	base = (sqfs_file_t *)file;
 	if (file == NULL) {
-		free(wpath);
 		SetLastError(ERROR_NOT_ENOUGH_MEMORY);
 		return NULL;
 	}
@@ -208,50 +243,26 @@ sqfs_file_t *sqfs_open_file(const char *filename, sqfs_u32 flags)
 	sqfs_object_init(file, stdio_destroy, stdio_copy);
 	memcpy(file->name, filename, namelen);
 
-	if (flags & SQFS_FILE_OPEN_READ_ONLY) {
-		file->readonly = true;
-		access_flags = GENERIC_READ;
-		creation_mode = OPEN_EXISTING;
-		share_mode = FILE_SHARE_READ;
-	} else {
-		file->readonly = false;
-		access_flags = GENERIC_READ | GENERIC_WRITE;
-		share_mode = 0;
-
-		if (flags & SQFS_FILE_OPEN_OVERWRITE) {
-			creation_mode = CREATE_ALWAYS;
-		} else {
-			creation_mode = CREATE_NEW;
-		}
-	}
-
-	if (flags & SQFS_FILE_OPEN_NO_CHARSET_XFRM) {
-		file->fd = CreateFileA(filename, access_flags, share_mode, NULL,
-				       creation_mode, FILE_ATTRIBUTE_NORMAL,
-				       NULL);
-	} else {
-		file->fd = CreateFileW(wpath, access_flags, share_mode, NULL,
-				       creation_mode, FILE_ATTRIBUTE_NORMAL,
-				       NULL);
-	}
+	if (sqfs_open_native_file(&file->fd, filename, flags))
+		goto fail;
 
-	free(wpath);
-
-	if (file->fd == INVALID_HANDLE_VALUE) {
-		free(file);
-		return NULL;
-	}
-
-	if (!GetFileSizeEx(file->fd, &size)) {
-		free(file);
-		return NULL;
-	}
+	if (!GetFileSizeEx(file->fd, &size))
+		goto fail;
 
 	file->size = size.QuadPart;
+	file->readonly = (flags & SQFS_FILE_OPEN_READ_ONLY) != 0;
+
 	base->read_at = stdio_read_at;
 	base->write_at = stdio_write_at;
 	base->get_size = stdio_get_size;
 	base->truncate = stdio_truncate;
 	base->get_filename = stdio_get_filename;
 	return base;
+fail:
+	err = GetLastError();
+	if (file->fd != INVALID_HANDLE_VALUE)
+		CloseHandle(file->fd);
+	free(file);
+	SetLastError(err);
+	return NULL;
 }