Cleanup libsquashfs invalid argument error codes

 - Add an explicit "you're holding it wrong" error code.
 - Consistently return error codes and not have some special places
   where -1 is returned.

Signed-off-by: David Oberhollenzer <david.oberhollenzer@sigma-star.at>

12 files changed, 25 insertions, 14 deletions

diff --git a/include/sqfs/compressor.h b/include/sqfs/compressor.h
index a53fbc1..4f22b0d 100644
--- a/include/sqfs/compressor.h
+++ b/include/sqfs/compressor.h
@@ -386,7 +386,8 @@ SQFS_API const char *sqfs_compressor_name_from_id(E_SQFS_COMPRESSOR id);
  * @param name The name of the compressor backend.
  * @param out Returns the coresponding @ref E_SQFS_COMPRESSOR identifier.
  *
- * @return Zero on success, -1 if the backend is unknown.
+ * @return Zero on success, @ref SQFS_ERROR_UNSUPPORTED if the backend
+ *         is unknown.
  */
 SQFS_API
 int sqfs_compressor_id_from_name(const char *name, E_SQFS_COMPRESSOR *out);
diff --git a/include/sqfs/error.h b/include/sqfs/error.h
index 9ff77d4..77e33ed 100644
--- a/include/sqfs/error.h
+++ b/include/sqfs/error.h
@@ -128,6 +128,11 @@ typedef enum {
 	 *        a regular file or a regular file inode.
 	 */
 	SQFS_ERROR_NOT_FILE = -15,
+
+	/**
+	 * @brief An invalid argument was passed to a library function.
+	 */
+	SQFS_ERROR_ARG_INVALID = -16,
 } E_SQFS_ERROR;
 
 #endif /* SQFS_ERROR_H */
diff --git a/include/sqfs/xattr.h b/include/sqfs/xattr.h
index 0f74c94..f2a87d3 100644
--- a/include/sqfs/xattr.h
+++ b/include/sqfs/xattr.h
@@ -173,7 +173,8 @@ SQFS_API const char *sqfs_get_xattr_prefix(E_SQFS_XATTR_TYPE id);
  * This function takes a key and finds the enumerator value that represents
  * its prefix. An error value is returned if the given prefix isn't supported.
  *
- * @return On success an @ref E_SQFS_XATTR_TYPE, -1 if it isn't supported.
+ * @return On success an @ref E_SQFS_XATTR_TYPE. If not supported, the
+ *         @ref SQFS_ERROR_UNSUPPORTED error code.
  */
 SQFS_API int sqfs_get_xattr_prefix_id(const char *key);
 
diff --git a/lib/common/perror.c b/lib/common/perror.c
index 9b9f041..a0a86aa 100644
--- a/lib/common/perror.c
+++ b/lib/common/perror.c
@@ -58,6 +58,9 @@ void sqfs_perror(const char *file, const char *action, int error_code)
 	case SQFS_ERROR_NOT_FILE:
 		errstr = "target is not a file";
 		break;
+	case SQFS_ERROR_ARG_INVALID:
+		errstr = "invalid argument";
+		break;
 	default:
 		errstr = "libsquashfs returned an unknown error code";
 		break;
diff --git a/lib/sqfs/comp/compressor.c b/lib/sqfs/comp/compressor.c
index c835f3f..0b8b5e9 100644
--- a/lib/sqfs/comp/compressor.c
+++ b/lib/sqfs/comp/compressor.c
@@ -141,7 +141,7 @@ int sqfs_compressor_id_from_name(const char *name, E_SQFS_COMPRESSOR *out)
 		}
 	}
 
-	return -1;
+	return SQFS_ERROR_UNSUPPORTED;
 }
 
 int sqfs_compressor_config_init(sqfs_compressor_config_t *cfg,
diff --git a/lib/sqfs/comp/gzip.c b/lib/sqfs/comp/gzip.c
index 3886e60..7183e5a 100644
--- a/lib/sqfs/comp/gzip.c
+++ b/lib/sqfs/comp/gzip.c
@@ -156,7 +156,7 @@ static sqfs_s32 gzip_do_block(sqfs_compressor_t *base, const sqfs_u8 *in,
 	size_t written;
 
 	if (size >= 0x7FFFFFFF)
-		return 0;
+		return SQFS_ERROR_ARG_INVALID;
 
 	if (gzip->compress && gzip->opt.strategies != 0) {
 		strategy = find_strategy(gzip, in, size, out, outsize);
diff --git a/lib/sqfs/comp/lz4.c b/lib/sqfs/comp/lz4.c
index dcee7d8..dd9b6ac 100644
--- a/lib/sqfs/comp/lz4.c
+++ b/lib/sqfs/comp/lz4.c
@@ -66,7 +66,7 @@ static sqfs_s32 lz4_comp_block(sqfs_compressor_t *base, const sqfs_u8 *in,
 	int ret;
 
 	if (size >= 0x7FFFFFFF)
-		return 0;
+		return SQFS_ERROR_ARG_INVALID;
 
 	if (lz4->high_compression) {
 		ret = LZ4_compress_HC((void *)in, (void *)out,
@@ -89,7 +89,7 @@ static sqfs_s32 lz4_uncomp_block(sqfs_compressor_t *base, const sqfs_u8 *in,
 	(void)base;
 
 	if (outsize >= 0x7FFFFFFF)
-		return 0;
+		return SQFS_ERROR_ARG_INVALID;
 
 	ret = LZ4_decompress_safe((void *)in, (void *)out, size, outsize);
 
diff --git a/lib/sqfs/comp/lzma.c b/lib/sqfs/comp/lzma.c
index 002376f..f7ab428 100644
--- a/lib/sqfs/comp/lzma.c
+++ b/lib/sqfs/comp/lzma.c
@@ -48,7 +48,7 @@ static sqfs_s32 lzma_comp_block(sqfs_compressor_t *base, const sqfs_u8 *in,
 	int ret;
 
 	if (outsize < LZMA_HEADER_SIZE || size >= 0x7FFFFFFF)
-		return 0;
+		return SQFS_ERROR_ARG_INVALID;
 
 	lzma_lzma_preset(&opt, LZMA_DEFAULT_LEVEL);
 	opt.dict_size = lzma->block_size;
@@ -93,7 +93,7 @@ static sqfs_s32 lzma_uncomp_block(sqfs_compressor_t *base, const sqfs_u8 *in,
 	(void)base;
 
 	if (size >= 0x7FFFFFFF)
-		return 0;
+		return SQFS_ERROR_ARG_INVALID;
 
 	if (size < sizeof(lzma_header))
 		return SQFS_ERROR_CORRUPTED;
diff --git a/lib/sqfs/comp/xz.c b/lib/sqfs/comp/xz.c
index fd0f70b..91d48da 100644
--- a/lib/sqfs/comp/xz.c
+++ b/lib/sqfs/comp/xz.c
@@ -144,7 +144,7 @@ static sqfs_s32 xz_comp_block(sqfs_compressor_t *base, const sqfs_u8 *in,
 	size_t i;
 
 	if (size >= 0x7FFFFFFF)
-		return 0;
+		return SQFS_ERROR_ARG_INVALID;
 
 	ret = compress(xz, LZMA_VLI_UNKNOWN, in, size, out, outsize);
 	if (ret < 0 || xz->flags == 0)
@@ -184,7 +184,7 @@ static sqfs_s32 xz_uncomp_block(sqfs_compressor_t *base, const sqfs_u8 *in,
 	(void)base;
 
 	if (outsize >= 0x7FFFFFFF)
-		return 0;
+		return SQFS_ERROR_ARG_INVALID;
 
 	ret = lzma_stream_buffer_decode(&memlimit, 0, NULL,
 					in, &src_pos, size,
diff --git a/lib/sqfs/comp/zstd.c b/lib/sqfs/comp/zstd.c
index b2e5d8b..3fac85f 100644
--- a/lib/sqfs/comp/zstd.c
+++ b/lib/sqfs/comp/zstd.c
@@ -58,7 +58,7 @@ static sqfs_s32 zstd_comp_block(sqfs_compressor_t *base, const sqfs_u8 *in,
 	size_t ret;
 
 	if (size >= 0x7FFFFFFF)
-		return 0;
+		return SQFS_ERROR_ARG_INVALID;
 
 	ret = ZSTD_compressCCtx(zstd->zctx, out, outsize, in, size,
 				zstd->level);
@@ -76,7 +76,7 @@ static sqfs_s32 zstd_uncomp_block(sqfs_compressor_t *base, const sqfs_u8 *in,
 	(void)base;
 
 	if (outsize >= 0x7FFFFFFF)
-		return 0;
+		return SQFS_ERROR_ARG_INVALID;
 
 	ret = ZSTD_decompress(out, outsize, in, size);
 
diff --git a/lib/sqfs/dir_writer.c b/lib/sqfs/dir_writer.c
index 8703b77..673c103 100644
--- a/lib/sqfs/dir_writer.c
+++ b/lib/sqfs/dir_writer.c
@@ -130,7 +130,7 @@ int sqfs_dir_writer_add_entry(sqfs_dir_writer_t *writer, const char *name,
 		return type;
 
 	if (name[0] == '\0')
-		return SQFS_ERROR_CORRUPTED;
+		return SQFS_ERROR_ARG_INVALID;
 
 	ent = alloc_flex(sizeof(*ent), 1, strlen(name));
 	if (ent == NULL)
diff --git a/lib/sqfs/xattr.c b/lib/sqfs/xattr.c
index 6ce6b9a..909a9a5 100644
--- a/lib/sqfs/xattr.c
+++ b/lib/sqfs/xattr.c
@@ -7,6 +7,7 @@
 #define SQFS_BUILDING_DLL
 #include "config.h"
 #include "sqfs/xattr.h"
+#include "sqfs/error.h"
 
 #include <string.h>
 
@@ -32,7 +33,7 @@ int sqfs_get_xattr_prefix_id(const char *key)
 		}
 	}
 
-	return -1;
+	return SQFS_ERROR_UNSUPPORTED;
 }
 
 const char *sqfs_get_xattr_prefix(E_SQFS_XATTR_TYPE id)