Expose more fine grained control values & flags on the XZ compressor

This patch allows external users to fiddle with the XZ compressors
compression strength, alignment and other values.

Signed-off-by: David Oberhollenzer <david.oberhollenzer@sigma-star.at>

4 files changed, 225 insertions, 17 deletions

diff --git a/include/sqfs/compressor.h b/include/sqfs/compressor.h
index 4190c67..af379e7 100644
--- a/include/sqfs/compressor.h
+++ b/include/sqfs/compressor.h
@@ -199,7 +199,50 @@ struct sqfs_compressor_config_t {
 			 */
 			sqfs_u32 dict_size;
 
-			sqfs_u32 padd0[3];
+			/**
+			 * @brief Compression level. Maximum 9, default is 6.
+			 */
+			sqfs_u8 level;
+
+			/**
+			 * @brief Number of literal context bits.
+			 *
+			 * How many of the highest bits of the previous
+			 * uncompressed byte to take into account when
+			 * predicting the bits of the next byte.
+			 *
+			 * The sum lc + lp must be at MOST 4. Default value of
+			 * lc is 3.
+			 */
+			sqfs_u8 lc;
+
+			/**
+			 * @brief Number of literal position bits.
+			 *
+			 * lp affects what kind of alignment in the uncompressed
+			 * data is assumed when encoding bytes.
+			 * See pb below for more information about alignment.
+			 *
+			 * The sum lc + lp must be at MOST 4. Default value of
+			 * lp is 0.
+			 */
+			sqfs_u8 lp;
+
+			/**
+			 * @brief Number of position bits.
+			 *
+			 * This is the log2 of the assumed underlying alignment
+			 * of the input data, i.e. pb=0 means single byte
+			 * allignment, pb=1 means 16 bit, 2 means 32 bit.
+			 *
+			 * When the alignment is known, setting pb may reduce
+			 * the file size.
+			 *
+			 * The default value is 2, i.e. 32 bit alignment.
+			 */
+			sqfs_u8 pb;
+
+			sqfs_u32 padd0[2];
 		} xz;
 
 		sqfs_u64 padd0[2];
@@ -247,7 +290,13 @@ typedef enum {
 	 * @brief For LZMA, set this to select the Sparc BCJ filter.
 	 */
 	SQFS_COMP_FLAG_XZ_SPARC = 0x0020,
-	SQFS_COMP_FLAG_XZ_ALL = 0x003F,
+
+	/**
+	 * @brief Tell the LZMA compressor to try the "extreme" option.
+	 */
+	SQFS_COMP_FLAG_XZ_EXTREME = 0x0100,
+
+	SQFS_COMP_FLAG_XZ_ALL = 0x013F,
 
 	/**
 	 * @brief For zlib deflate, set this to try the default strategy.
@@ -316,6 +365,22 @@ typedef enum {
 #define SQFS_GZIP_MIN_WINDOW (8)
 #define SQFS_GZIP_MAX_WINDOW (15)
 
+#define SQFS_XZ_MIN_LEVEL (0)
+#define SQFS_XZ_MAX_LEVEL (9)
+#define SQFS_XZ_DEFAULT_LEVEL (6)
+
+#define SQFS_XZ_MIN_LC 0
+#define SQFS_XZ_MAX_LC 4
+#define SQFS_XZ_DEFAULT_LC 3
+
+#define SQFS_XZ_MIN_LP 0
+#define SQFS_XZ_MAX_LP 4
+#define SQFS_XZ_DEFAULT_LP 0
+
+#define SQFS_XZ_MIN_PB 0
+#define SQFS_XZ_MAX_PB 4
+#define SQFS_XZ_DEFAULT_PB 2
+
 #ifdef __cplusplus
 extern "C" {
 #endif
diff --git a/lib/common/comp_opt.c b/lib/common/comp_opt.c
index 58a7a21..323fa4f 100644
--- a/lib/common/comp_opt.c
+++ b/lib/common/comp_opt.c
@@ -31,6 +31,7 @@ static const flag_t xz_flags[] = {
 	{ "arm", SQFS_COMP_FLAG_XZ_ARM },
 	{ "armthumb", SQFS_COMP_FLAG_XZ_ARMTHUMB },
 	{ "sparc", SQFS_COMP_FLAG_XZ_SPARC },
+	{ "extreme", SQFS_COMP_FLAG_XZ_EXTREME },
 };
 
 static const flag_t lz4_flags[] = {
@@ -79,12 +80,18 @@ enum {
 	OPT_LEVEL,
 	OPT_ALG,
 	OPT_DICT,
+	OPT_LC,
+	OPT_LP,
+	OPT_PB,
 };
 static char *const token[] = {
 	[OPT_WINDOW] = (char *)"window",
 	[OPT_LEVEL] = (char *)"level",
 	[OPT_ALG] = (char *)"algorithm",
 	[OPT_DICT] = (char *)"dictsize",
+	[OPT_LC] = (char *)"lc",
+	[OPT_LP] = (char *)"lp",
+	[OPT_PB] = (char *)"pb",
 	NULL
 };
 
@@ -119,6 +126,8 @@ int compressor_cfg_init_options(sqfs_compressor_config_t *cfg,
 		max_level = SQFS_ZSTD_MAX_LEVEL;
 		break;
 	case SQFS_COMP_XZ:
+		min_level = SQFS_XZ_MIN_LEVEL;
+		max_level = SQFS_XZ_MAX_LEVEL;
 		flags = xz_flags;
 		num_flags = sizeof(xz_flags) / sizeof(xz_flags[0]);
 		break;
@@ -180,6 +189,9 @@ int compressor_cfg_init_options(sqfs_compressor_config_t *cfg,
 			case SQFS_COMP_ZSTD:
 				cfg->opt.zstd.level = level;
 				break;
+			case SQFS_COMP_XZ:
+				cfg->opt.xz.level = level;
+				break;
 			default:
 				goto fail_opt;
 			}
@@ -208,6 +220,39 @@ int compressor_cfg_init_options(sqfs_compressor_config_t *cfg,
 
 			cfg->opt.xz.dict_size = dict_size;
 			break;
+		case OPT_LC:
+			if (cfg->id != SQFS_COMP_XZ)
+				goto fail_opt;
+
+			if (value == NULL)
+				goto fail_value;
+
+			cfg->opt.xz.lc = strtol(value, NULL, 10);
+			if (cfg->opt.xz.lc > SQFS_XZ_MAX_LC)
+				goto fail_lc;
+			break;
+		case OPT_LP:
+			if (cfg->id != SQFS_COMP_XZ)
+				goto fail_opt;
+
+			if (value == NULL)
+				goto fail_value;
+
+			cfg->opt.xz.lp = strtol(value, NULL, 10);
+			if (cfg->opt.xz.lp > SQFS_XZ_MAX_LP)
+				goto fail_lp;
+			break;
+		case OPT_PB:
+			if (cfg->id != SQFS_COMP_XZ)
+				goto fail_opt;
+
+			if (value == NULL)
+				goto fail_value;
+
+			cfg->opt.xz.pb = strtol(value, NULL, 10);
+			if (cfg->opt.xz.lp > SQFS_XZ_MAX_PB)
+				goto fail_pb;
+			break;
 		default:
 			if (set_flag(cfg, value, flags, num_flags))
 				goto fail_opt;
@@ -215,7 +260,26 @@ int compressor_cfg_init_options(sqfs_compressor_config_t *cfg,
 		}
 	}
 
+	if (cfg->id == SQFS_COMP_XZ && (cfg->opt.xz.lp + cfg->opt.xz.lc) > 4)
+		goto fail_sum_lp_lc;
+
 	return 0;
+fail_sum_lp_lc:
+	fputs("Sum of XZ lc + lp must not exceed 4.\n", stderr);
+	return -1;
+fail_lp:
+	fprintf(stderr,
+		"XZ literal position bits (lp) must be between %d and %d.\n",
+		SQFS_XZ_MIN_LP, SQFS_XZ_MAX_LP);
+	return -1;
+fail_lc:
+	fprintf(stderr, "XZ literal context (lc) must be between %d and %d.\n",
+		SQFS_XZ_MIN_LC, SQFS_XZ_MAX_LC);
+	return -1;
+fail_pb:
+	fprintf(stderr, "XZ position bits (bp) must be between %d and %d.\n",
+		SQFS_XZ_MIN_PB, SQFS_XZ_MAX_PB);
+	return -1;
 fail_lzo_alg:
 	fprintf(stderr, "Unknown lzo variant '%s'.\n", value);
 	return -1;
@@ -294,21 +358,42 @@ static void xz_print_help(void)
 {
 	size_t i;
 
-	fputs(
+	printf(
 "Available options for xz compressor:\n"
 "\n"
 "    dictsize=<value>  Dictionary size. Either a value in bytes or a\n"
-"                      percentage of the block size. Defaults to 100%.\n"
-"                      The suffix '%' indicates a percentage. 'K' and 'M'\n"
+"                      percentage of the block size. Defaults to 100%%.\n"
+"                      The suffix '%%' indicates a percentage. 'K' and 'M'\n"
 "                      can also be used for kibi and mebi bytes\n"
 "                      respecitively.\n"
+"    level=<value>     Compression level. Value from %d to %d.\n"
+"                      Defaults to %d.\n"
+"    lc=<value>        Number of literal context bits.\n"
+"                      How many of the highest bits of the previous\n"
+"                      uncompressed byte to take into account when\n"
+"                      predicting the bits of the next byte.\n"
+"                      Default is %d.\n"
+"    lp=<value>        Number of literal position bits.\n"
+"                      Affects what kind of alignment in the uncompressed\n"
+"                      data is assumed when encoding bytes.\n"
+"                      Default is %d.\n"
+"    pb=<value>        Number of position bits.\n"
+"                      This is the log2 of the assumed underlying alignment\n"
+"                      of the input data, i.e. pb=0 means single byte\n"
+"                      allignment, pb=1 means 16 bit, 2 means 32 bit.\n"
+"                      Default is %d.\n"
+"\n"
+"If values are set, the sum of lc + lp must not exceed 4.\n"
+"The maximum for pb is %d.\n"
 "\n"
 "In additon to the options, one or more bcj filters can be specified.\n"
 "If multiple filters are provided, the one yielding the best compression\n"
 "ratio will be used.\n"
 "\n"
 "The following filters are available:\n",
-	stdout);
+	SQFS_XZ_MIN_LEVEL, SQFS_XZ_MAX_LEVEL, SQFS_XZ_DEFAULT_LEVEL,
+	SQFS_XZ_DEFAULT_LC, SQFS_XZ_DEFAULT_LP, SQFS_XZ_DEFAULT_PB,
+	SQFS_XZ_MAX_PB);
 
 	for (i = 0; i < sizeof(xz_flags) / sizeof(xz_flags[0]); ++i)
 		printf("\t%s\n", xz_flags[i].name);
diff --git a/lib/sqfs/comp/compressor.c b/lib/sqfs/comp/compressor.c
index 946ee25..ea53339 100644
--- a/lib/sqfs/comp/compressor.c
+++ b/lib/sqfs/comp/compressor.c
@@ -165,6 +165,10 @@ int sqfs_compressor_config_init(sqfs_compressor_config_t *cfg,
 	case SQFS_COMP_XZ:
 		flag_mask |= SQFS_COMP_FLAG_XZ_ALL;
 		cfg->opt.xz.dict_size = block_size;
+		cfg->opt.xz.level = SQFS_XZ_DEFAULT_LEVEL;
+		cfg->opt.xz.lc = SQFS_XZ_DEFAULT_LC;
+		cfg->opt.xz.lp = SQFS_XZ_DEFAULT_LP;
+		cfg->opt.xz.pb = SQFS_XZ_DEFAULT_PB;
 		break;
 	case SQFS_COMP_LZMA:
 		break;
diff --git a/lib/sqfs/comp/xz.c b/lib/sqfs/comp/xz.c
index 32fb032..20e6445 100644
--- a/lib/sqfs/comp/xz.c
+++ b/lib/sqfs/comp/xz.c
@@ -18,6 +18,12 @@ typedef struct {
 	sqfs_compressor_t base;
 	size_t block_size;
 	size_t dict_size;
+
+	sqfs_u8 level;
+	sqfs_u8 lc;
+	sqfs_u8 lp;
+	sqfs_u8 pb;
+
 	int flags;
 } xz_compressor_t;
 
@@ -40,12 +46,16 @@ static int xz_write_options(sqfs_compressor_t *base, sqfs_file_t *file)
 {
 	xz_compressor_t *xz = (xz_compressor_t *)base;
 	xz_options_t opt;
+	sqfs_u32 flags;
 
 	if (xz->flags == 0 && xz->dict_size == xz->block_size)
 		return 0;
 
+	flags = xz->flags & SQFS_COMP_FLAG_XZ_ALL;
+	flags &= ~SQFS_COMP_FLAG_XZ_EXTREME;
+
 	opt.dict_size = htole32(xz->dict_size);
-	opt.flags = htole32(xz->flags);
+	opt.flags = htole32(flags);
 
 	return sqfs_generic_write_options(file, &opt, sizeof(opt));
 }
@@ -76,7 +86,8 @@ static int xz_read_options(sqfs_compressor_t *base, sqfs_file_t *file)
 
 static sqfs_s32 compress(xz_compressor_t *xz, lzma_vli filter,
 			 const sqfs_u8 *in, sqfs_u32 size,
-			 sqfs_u8 *out, sqfs_u32 outsize)
+			 sqfs_u8 *out, sqfs_u32 outsize,
+			 sqfs_u32 presets)
 {
 	lzma_filter filters[5];
 	lzma_options_lzma opt;
@@ -84,9 +95,12 @@ static sqfs_s32 compress(xz_compressor_t *xz, lzma_vli filter,
 	lzma_ret ret;
 	int i = 0;
 
-	if (lzma_lzma_preset(&opt, LZMA_PRESET_DEFAULT))
+	if (lzma_lzma_preset(&opt, presets))
 		return SQFS_ERROR_COMPRESSOR;
 
+	opt.lc = xz->lc;
+	opt.lp = xz->lp;
+	opt.pb = xz->pb;
 	opt.dict_size = xz->dict_size;
 
 	if (filter != LZMA_VLI_UNKNOWN) {
@@ -141,43 +155,66 @@ static sqfs_s32 xz_comp_block(sqfs_compressor_t *base, const sqfs_u8 *in,
 	xz_compressor_t *xz = (xz_compressor_t *)base;
 	lzma_vli filter, selected = LZMA_VLI_UNKNOWN;
 	sqfs_s32 ret, smallest;
+	bool extreme;
 	size_t i;
 
 	if (size >= 0x7FFFFFFF)
 		return SQFS_ERROR_ARG_INVALID;
 
-	ret = compress(xz, LZMA_VLI_UNKNOWN, in, size, out, outsize);
+	ret = compress(xz, LZMA_VLI_UNKNOWN, in, size, out,
+		       outsize, xz->level);
 	if (ret < 0 || xz->flags == 0)
 		return ret;
 
 	smallest = ret;
+	extreme = false;
+
+	if (xz->flags & SQFS_COMP_FLAG_XZ_EXTREME) {
+		ret = compress(xz, LZMA_VLI_UNKNOWN, in, size, out, outsize,
+			       xz->level | LZMA_PRESET_EXTREME);
+
+		if (ret > 0 && (smallest == 0 || ret < smallest)) {
+			smallest = ret;
+			extreme = true;
+		}
+	}
 
 	for (i = 1; i & SQFS_COMP_FLAG_XZ_ALL; i <<= 1) {
-		if ((xz->flags & i) == 0)
+		if ((i & SQFS_COMP_FLAG_XZ_EXTREME) || (xz->flags & i) == 0)
 			continue;
 
 		filter = flag_to_vli(i);
 
-		ret = compress(xz, filter, in, size, out, outsize);
-		if (ret < 0)
-			return ret;
-
+		ret = compress(xz, filter, in, size, out, outsize, xz->level);
 		if (ret > 0 && (smallest == 0 || ret < smallest)) {
 			smallest = ret;
 			selected = filter;
+			extreme = false;
+		}
+
+		if (xz->flags & SQFS_COMP_FLAG_XZ_EXTREME) {
+			ret = compress(xz, filter, in, size, out, outsize,
+				       xz->level | LZMA_PRESET_EXTREME);
+
+			if (ret > 0 && (smallest == 0 || ret < smallest)) {
+				smallest = ret;
+				selected = filter;
+				extreme = true;
+			}
 		}
 	}
 
 	if (smallest == 0)
 		return 0;
 
-	return compress(xz, selected, in, size, out, outsize);
+	return compress(xz, selected, in, size, out, outsize,
+			xz->level | (extreme ? LZMA_PRESET_EXTREME : 0));
 }
 
 static sqfs_s32 xz_uncomp_block(sqfs_compressor_t *base, const sqfs_u8 *in,
 				sqfs_u32 size, sqfs_u8 *out, sqfs_u32 outsize)
 {
-	sqfs_u64 memlimit = 32 * 1024 * 1024;
+	sqfs_u64 memlimit = 65 * 1024 * 1024;
 	size_t dest_pos = 0;
 	size_t src_pos = 0;
 	lzma_ret ret;
@@ -206,6 +243,10 @@ static void xz_get_configuration(const sqfs_compressor_t *base,
 	cfg->flags = xz->flags;
 	cfg->block_size = xz->block_size;
 	cfg->opt.xz.dict_size = xz->dict_size;
+	cfg->opt.xz.level = xz->level;
+	cfg->opt.xz.lc = xz->lc;
+	cfg->opt.xz.lp = xz->lp;
+	cfg->opt.xz.pb = xz->pb;
 
 	if (base->do_block == xz_uncomp_block)
 		cfg->flags |= SQFS_COMP_FLAG_UNCOMPRESS;
@@ -241,6 +282,15 @@ int xz_compressor_create(const sqfs_compressor_config_t *cfg,
 	if (!is_dict_size_valid(cfg->opt.xz.dict_size))
 		return SQFS_ERROR_UNSUPPORTED;
 
+	if (cfg->opt.xz.lc + cfg->opt.xz.lp > 4)
+		return SQFS_ERROR_UNSUPPORTED;
+
+	if (cfg->opt.xz.pb > SQFS_XZ_MAX_PB)
+		return SQFS_ERROR_UNSUPPORTED;
+
+	if (cfg->opt.xz.level > SQFS_XZ_MAX_LEVEL)
+		return SQFS_ERROR_UNSUPPORTED;
+
 	xz = calloc(1, sizeof(*xz));
 	base = (sqfs_compressor_t *)xz;
 	if (xz == NULL)
@@ -249,6 +299,10 @@ int xz_compressor_create(const sqfs_compressor_config_t *cfg,
 	xz->flags = cfg->flags;
 	xz->dict_size = cfg->opt.xz.dict_size;
 	xz->block_size = cfg->block_size;
+	xz->lc = cfg->opt.xz.lc;
+	xz->lp = cfg->opt.xz.lp;
+	xz->pb = cfg->opt.xz.pb;
+	xz->level = cfg->opt.xz.level;
 	base->get_configuration = xz_get_configuration;
 	base->do_block = (cfg->flags & SQFS_COMP_FLAG_UNCOMPRESS) ?
 		xz_uncomp_block : xz_comp_block;