From 9533dc84d7addf792db4f4702b6fc7d7ddd20e78 Mon Sep 17 00:00:00 2001 From: David Oberhollenzer Date: Sun, 1 Sep 2019 14:35:03 +0200 Subject: Install libsquashfs.so headers on the system in "sqfs" subdirectory Signed-off-by: David Oberhollenzer --- include/sqfs/block_processor.h | 111 +++++++++++++++++ include/sqfs/compress.h | 71 +++++++++++ include/sqfs/id_table.h | 48 ++++++++ include/sqfs/meta_reader.h | 52 ++++++++ include/sqfs/meta_writer.h | 87 +++++++++++++ include/sqfs/squashfs.h | 272 +++++++++++++++++++++++++++++++++++++++++ 6 files changed, 641 insertions(+) create mode 100644 include/sqfs/block_processor.h create mode 100644 include/sqfs/compress.h create mode 100644 include/sqfs/id_table.h create mode 100644 include/sqfs/meta_reader.h create mode 100644 include/sqfs/meta_writer.h create mode 100644 include/sqfs/squashfs.h (limited to 'include/sqfs') diff --git a/include/sqfs/block_processor.h b/include/sqfs/block_processor.h new file mode 100644 index 0000000..18d8c4a --- /dev/null +++ b/include/sqfs/block_processor.h @@ -0,0 +1,111 @@ +/* SPDX-License-Identifier: GPL-3.0-or-later */ +/* + * block_processor.h + * + * Copyright (C) 2019 David Oberhollenzer + */ +#ifndef BLOCK_PROCESSOR_H +#define BLOCK_PROCESSOR_H + +#include "config.h" +#include "sqfs/compress.h" + +enum { + /* only calculate checksum, do NOT compress the data */ + BLK_DONT_COMPRESS = 0x0001, + + /* set by compressor worker if the block was actually compressed */ + BLK_IS_COMPRESSED = 0x0002, + + /* do not calculate block checksum */ + BLK_DONT_CHECKSUM = 0x0004, + + /* set by compressor worker if compression failed */ + BLK_COMPRESS_ERROR = 0x0008, + + /* first user setable block flag */ + BLK_USER = 0x0080 +}; + +typedef struct block_t { + /* used internally, ignored and overwritten when enqueueing blocks */ + struct block_t *next; + uint32_t sequence_number; + + /* Size of the data area */ + uint32_t size; + + /* checksum of the input data */ + uint32_t checksum; + + /* user settable file block index */ + uint32_t index; + + /* user pointer associated with the block */ + void *user; + + /* user settable flag field */ + uint32_t flags; + + /* raw data to be processed */ + uint8_t data[]; +} block_t; + +typedef struct block_processor_t block_processor_t; + +/* + Gets called for each processed block. May be called from a different thread + than the one that calls enqueue, but only from one thread at a time. + Guaranteed to be called on blocks in the order that they are submitted + to enqueue. + + A non-zero return value is interpreted as fatal error. + */ +typedef int (*block_cb)(void *user, block_t *blk); + +block_processor_t *block_processor_create(size_t max_block_size, + compressor_t *cmp, + unsigned int num_workers, + void *user, + block_cb callback); + +void block_processor_destroy(block_processor_t *proc); + +/* + Add a block to be processed. Returns non-zero on error and prints a message + to stderr. + + The function takes over ownership of the submitted block. It is freed with + a after processing and calling the block callback. + + Even on failure, the workers may still be running and + block_processor_finish must be called before cleaning up. +*/ +int block_processor_enqueue(block_processor_t *proc, block_t *block); + +/* + Wait for the compressor workers to finish. Returns zero on success, non-zero + if an internal error occoured or one of the block callbacks returned a + non-zero value. + */ +int block_processor_finish(block_processor_t *proc); + +/* + Convenience function to create a block structure and optionally fill it with + content. + + filename is used for printing error messages. If fd is a valid file + descriptor (>= 0), the function attempts to populate the payload data + from the input file. + */ +block_t *create_block(const char *filename, int fd, size_t size, + void *user, uint32_t flags); + +/* + Convenience function to process a data block. Returns 0 on success, + prints to stderr on failure. + */ +int process_block(block_t *block, compressor_t *cmp, + uint8_t *scratch, size_t scratch_size); + +#endif /* BLOCK_PROCESSOR_H */ diff --git a/include/sqfs/compress.h b/include/sqfs/compress.h new file mode 100644 index 0000000..1dfba12 --- /dev/null +++ b/include/sqfs/compress.h @@ -0,0 +1,71 @@ +/* SPDX-License-Identifier: GPL-3.0-or-later */ +/* + * compress.h + * + * Copyright (C) 2019 David Oberhollenzer + */ +#ifndef COMPRESS_H +#define COMPRESS_H + +#include "config.h" + +#include +#include +#include +#include + +#include "squashfs.h" + +typedef struct compressor_t compressor_t; + +/* Encapsultes a compressor with a simple interface to compress or + uncompress/extract blocks of data. */ +struct compressor_t { + /* Write compressor options to the output file if necessary. + Returns the number of bytes written or -1 on failure. + Internally prints error messages to stderr. */ + int (*write_options)(compressor_t *cmp, int fd); + + /* Read compressor options to the input file. + Returns zero on success, -1 on failure. + Internally prints error messages to stderr. */ + int (*read_options)(compressor_t *cmp, int fd); + + /* + Compress or uncompress a chunk of data. + + Returns the number of bytes written to the output buffer, -1 on + failure or 0 if the output buffer was too small. + The compressor also returns 0 if the compressed result ends + up larger than the original input. + + Internally prints compressor specific error messages to stderr. + */ + ssize_t (*do_block)(compressor_t *cmp, const uint8_t *in, size_t size, + uint8_t *out, size_t outsize); + + /* create another compressor just like this one, i.e. + with the exact same settings */ + compressor_t *(*create_copy)(compressor_t *cmp); + + void (*destroy)(compressor_t *stream); +}; + +bool compressor_exists(E_SQFS_COMPRESSOR id); + +/* block_size is the configured block size for the SquashFS image. Needed + by some compressors to set internal defaults. */ +compressor_t *compressor_create(E_SQFS_COMPRESSOR id, bool compress, + size_t block_size, char *options); + +void compressor_print_help(E_SQFS_COMPRESSOR id); + +void compressor_print_available(void); + +E_SQFS_COMPRESSOR compressor_get_default(void); + +const char *compressor_name_from_id(E_SQFS_COMPRESSOR id); + +int compressor_id_from_name(const char *name, E_SQFS_COMPRESSOR *out); + +#endif /* COMPRESS_H */ diff --git a/include/sqfs/id_table.h b/include/sqfs/id_table.h new file mode 100644 index 0000000..dcc8a18 --- /dev/null +++ b/include/sqfs/id_table.h @@ -0,0 +1,48 @@ +/* SPDX-License-Identifier: GPL-3.0-or-later */ +/* + * id_table.h + * + * Copyright (C) 2019 David Oberhollenzer + */ +#ifndef ID_TABLE_H +#define ID_TABLE_H + +#include "config.h" + +#include +#include + +#include "compress.h" + +/* Encapsulates the ID table used by SquashFS */ +typedef struct { + /* Array of unique 32 bit IDs */ + uint32_t *ids; + + /* Number of 32 bit IDs stored in the array */ + size_t num_ids; + + /* Actual size of the array, i.e. maximum available */ + size_t max_ids; +} id_table_t; + +/* Returns 0 on success. Prints error message to stderr on failure. */ +int id_table_init(id_table_t *tbl); + +void id_table_cleanup(id_table_t *tbl); + +/* Resolve a 32 bit to a 16 bit table index. + Returns 0 on success. Internally prints errors to stderr. */ +int id_table_id_to_index(id_table_t *tbl, uint32_t id, uint16_t *out); + +/* Write an ID table to a SquashFS image. + Returns 0 on success. Internally prints error message to stderr. */ +int id_table_write(id_table_t *tbl, int outfd, sqfs_super_t *super, + compressor_t *cmp); + +/* Read an ID table from a SquashFS image. + Returns 0 on success. Internally prints error messages to stderr. */ +int id_table_read(id_table_t *tbl, int fd, sqfs_super_t *super, + compressor_t *cmp); + +#endif /* ID_TABLE_H */ diff --git a/include/sqfs/meta_reader.h b/include/sqfs/meta_reader.h new file mode 100644 index 0000000..d5628af --- /dev/null +++ b/include/sqfs/meta_reader.h @@ -0,0 +1,52 @@ +/* SPDX-License-Identifier: GPL-3.0-or-later */ +/* + * meta_reader.h + * + * Copyright (C) 2019 David Oberhollenzer + */ +#ifndef META_READER_H +#define META_READER_H + +#include "config.h" + +#include "compress.h" +#include "squashfs.h" + +typedef struct meta_reader_t meta_reader_t; + +/* Create a meta data reader using a given compressor to extract data. + Internally prints error message to stderr on failure. + + Start offset and limit can be specified to do bounds checking against + a subregion of the filesystem image. +*/ +meta_reader_t *meta_reader_create(int fd, compressor_t *cmp, + uint64_t start, uint64_t limit); + +void meta_reader_destroy(meta_reader_t *m); + +/* Returns 0 on success. Internally prints to stderr on failure */ +int meta_reader_seek(meta_reader_t *m, uint64_t block_start, + size_t offset); + +void meta_reader_get_position(meta_reader_t *m, uint64_t *block_start, + size_t *offset); + +/* Returns 0 on success. Internally prints to stderr on failure */ +int meta_reader_read(meta_reader_t *m, void *data, size_t size); + +/* Inode can be freed with a single free() call. + The function internally prints error message to stderr on failure. */ +sqfs_inode_generic_t *meta_reader_read_inode(meta_reader_t *ir, + sqfs_super_t *super, + uint64_t block_start, + size_t offset); + +/* Returns 0 on success. Internally prints to stderr on failure */ +int meta_reader_read_dir_header(meta_reader_t *m, sqfs_dir_header_t *hdr); + +/* Entry can be freed with a single free() call. + The function internally prints to stderr on failure */ +sqfs_dir_entry_t *meta_reader_read_dir_ent(meta_reader_t *m); + +#endif /* META_READER_H */ diff --git a/include/sqfs/meta_writer.h b/include/sqfs/meta_writer.h new file mode 100644 index 0000000..762cc38 --- /dev/null +++ b/include/sqfs/meta_writer.h @@ -0,0 +1,87 @@ +/* SPDX-License-Identifier: GPL-3.0-or-later */ +/* + * meta_writer.h + * + * Copyright (C) 2019 David Oberhollenzer + */ +#ifndef META_WRITER_H +#define META_WRITER_H + +#include "config.h" + +#include "compress.h" +#include "squashfs.h" +#include "id_table.h" +#include "fstree.h" + +typedef struct { + tree_node_t *node; + uint32_t block; + uint32_t index; +} idx_ref_t; + +typedef struct { + size_t num_nodes; + size_t max_nodes; + idx_ref_t idx_nodes[]; +} dir_index_t; + +typedef struct meta_writer_t meta_writer_t; + +/* Create a meta data reader using a given compressor to compress data. + Internally prints error message to stderr on failure. + If keep_in_mem is true, the blocks are collected in memory and must + be explicitly flushed to disk using meta_write_write_to_file. +*/ +meta_writer_t *meta_writer_create(int fd, compressor_t *cmp, bool keep_in_mem); + +void meta_writer_destroy(meta_writer_t *m); + +/* Compress and flush the currently unfinished block to disk. Returns 0 on + success, internally prints error message to stderr on failure */ +int meta_writer_flush(meta_writer_t *m); + +/* Returns 0 on success. Prints error message to stderr on failure. */ +int meta_writer_append(meta_writer_t *m, const void *data, size_t size); + +/* Query the current block start position and offset within the block */ +void meta_writer_get_position(const meta_writer_t *m, uint64_t *block_start, + uint32_t *offset); + +/* Reset all internal state, including the current block start position. */ +void meta_writer_reset(meta_writer_t *m); + +/* If created with keep_in_mem true, write the collected blocks to disk. + Does not flush the current block. Writes error messages to stderr and + returns non-zero on failure. */ +int meta_write_write_to_file(meta_writer_t *m); + +/* + High level helper function that writes squashfs directory entries to + a meta data writer. + + The dir_info_t structure is used to generate the listing and updated + accordingly (such as writing back the header position and total size). + A directory index is created on the fly and returned in *index. + A single free() call is sufficient. + + Returns 0 on success. Prints error messages to stderr on failure. + */ +int meta_writer_write_dir(meta_writer_t *dm, dir_info_t *dir, + dir_index_t **index); + +/* + High level helper function to serialize a tree_node_t to a squashfs inode + and write it to a meta data writer. + + The inode is written to `im`. If it is a directory node, the directory + contents are written to `dm` using meta_writer_write_dir. The given + id_table_t is used to store the uid and gid on the fly and write the + coresponding indices to the inode structure. + + Returns 0 on success. Prints error messages to stderr on failure. + */ +int meta_writer_write_inode(fstree_t *fs, id_table_t *idtbl, meta_writer_t *im, + meta_writer_t *dm, tree_node_t *node); + +#endif /* META_WRITER_H */ diff --git a/include/sqfs/squashfs.h b/include/sqfs/squashfs.h new file mode 100644 index 0000000..eb35fdd --- /dev/null +++ b/include/sqfs/squashfs.h @@ -0,0 +1,272 @@ +/* SPDX-License-Identifier: GPL-3.0-or-later */ +/* + * squashfs.h + * + * Copyright (C) 2019 David Oberhollenzer + */ +#ifndef SQUASHFS_H +#define SQUASHFS_H + +#include "config.h" + +#include +#include +#include + +#define SQFS_MAGIC 0x73717368 +#define SQFS_VERSION_MAJOR 4 +#define SQFS_VERSION_MINOR 0 +#define SQFS_META_BLOCK_SIZE 8192 +#define SQFS_DEFAULT_BLOCK_SIZE 131072 +#define SQFS_DEVBLK_SIZE 4096 +#define SQFS_MAX_DIR_ENT 256 + +#define SQFS_IS_BLOCK_COMPRESSED(size) (((size) & (1 << 24)) == 0) +#define SQFS_ON_DISK_BLOCK_SIZE(size) ((size) & ((1 << 24) - 1)) +#define SQFS_IS_SPARSE_BLOCK(size) (SQFS_ON_DISK_BLOCK_SIZE(size) == 0) + +typedef struct { + uint32_t magic; + uint32_t inode_count; + uint32_t modification_time; + uint32_t block_size; + uint32_t fragment_entry_count; + uint16_t compression_id; + uint16_t block_log; + uint16_t flags; + uint16_t id_count; + uint16_t version_major; + uint16_t version_minor; + uint64_t root_inode_ref; + uint64_t bytes_used; + uint64_t id_table_start; + uint64_t xattr_id_table_start; + uint64_t inode_table_start; + uint64_t directory_table_start; + uint64_t fragment_table_start; + uint64_t export_table_start; +} __attribute__((packed)) sqfs_super_t; + +typedef struct { + uint64_t start_offset; + uint32_t size; + uint32_t pad0; +} sqfs_fragment_t; + +typedef struct { + uint16_t type; + uint16_t mode; + uint16_t uid_idx; + uint16_t gid_idx; + uint32_t mod_time; + uint32_t inode_number; +} sqfs_inode_t; + +typedef struct { + uint32_t nlink; + uint32_t devno; +} sqfs_inode_dev_t; + +typedef struct { + uint32_t nlink; + uint32_t devno; + uint32_t xattr_idx; +} sqfs_inode_dev_ext_t; + +typedef struct { + uint32_t nlink; +} sqfs_inode_ipc_t; + +typedef struct { + uint32_t nlink; + uint32_t xattr_idx; +} sqfs_inode_ipc_ext_t; + +typedef struct { + uint32_t nlink; + uint32_t target_size; + /*uint8_t target[];*/ +} sqfs_inode_slink_t; + +typedef struct { + uint32_t nlink; + uint32_t target_size; + /*uint8_t target[];*/ + uint32_t xattr_idx; +} sqfs_inode_slink_ext_t; + +typedef struct { + uint32_t blocks_start; + uint32_t fragment_index; + uint32_t fragment_offset; + uint32_t file_size; + /*uint32_t block_sizes[];*/ +} sqfs_inode_file_t; + +typedef struct { + uint64_t blocks_start; + uint64_t file_size; + uint64_t sparse; + uint32_t nlink; + uint32_t fragment_idx; + uint32_t fragment_offset; + uint32_t xattr_idx; + /*uint32_t block_sizes[];*/ +} sqfs_inode_file_ext_t; + +typedef struct { + uint32_t start_block; + uint32_t nlink; + uint16_t size; + uint16_t offset; + uint32_t parent_inode; +} sqfs_inode_dir_t; + +typedef struct { + uint32_t nlink; + uint32_t size; + uint32_t start_block; + uint32_t parent_inode; + uint16_t inodex_count; + uint16_t offset; + uint32_t xattr_idx; +} sqfs_inode_dir_ext_t; + +typedef struct { + sqfs_inode_t base; + char *slink_target; + uint32_t *block_sizes; + + union { + sqfs_inode_dev_t dev; + sqfs_inode_dev_ext_t dev_ext; + sqfs_inode_ipc_t ipc; + sqfs_inode_ipc_ext_t ipc_ext; + sqfs_inode_slink_t slink; + sqfs_inode_slink_ext_t slink_ext; + sqfs_inode_file_t file; + sqfs_inode_file_ext_t file_ext; + sqfs_inode_dir_t dir; + sqfs_inode_dir_ext_t dir_ext; + } data; + + uint8_t extra[]; +} sqfs_inode_generic_t; + +typedef struct { + uint32_t count; + uint32_t start_block; + uint32_t inode_number; +} sqfs_dir_header_t; + +typedef struct { + uint16_t offset; + int16_t inode_diff; + uint16_t type; + uint16_t size; + uint8_t name[]; +} sqfs_dir_entry_t; + +typedef struct { + uint32_t index; + uint32_t start_block; + uint32_t size; + uint8_t name[]; +} sqfs_dir_index_t; + +typedef struct { + uint16_t type; + uint16_t size; + uint8_t key[]; +} sqfs_xattr_entry_t; + +typedef struct { + uint32_t size; + uint8_t value[]; +} sqfs_xattr_value_t; + +typedef struct { + uint64_t xattr; + uint32_t count; + uint32_t size; +} sqfs_xattr_id_t; + +typedef struct { + uint64_t xattr_table_start; + uint32_t xattr_ids; + uint32_t unused; +} sqfs_xattr_id_table_t; + + +typedef enum { + SQFS_COMP_GZIP = 1, + SQFS_COMP_LZMA = 2, + SQFS_COMP_LZO = 3, + SQFS_COMP_XZ = 4, + SQFS_COMP_LZ4 = 5, + SQFS_COMP_ZSTD = 6, + + SQFS_COMP_MIN = 1, + SQFS_COMP_MAX = 6, +} E_SQFS_COMPRESSOR; + +typedef enum { + SQFS_FLAG_UNCOMPRESSED_INODES = 0x0001, + SQFS_FLAG_UNCOMPRESSED_DATA = 0x0002, + SQFS_FLAG_UNCOMPRESSED_FRAGMENTS = 0x0008, + SQFS_FLAG_NO_FRAGMENTS = 0x0010, + SQFS_FLAG_ALWAYS_FRAGMENTS = 0x0020, + SQFS_FLAG_DUPLICATES = 0x0040, + SQFS_FLAG_EXPORTABLE = 0x0080, + SQFS_FLAG_UNCOMPRESSED_XATTRS = 0x0100, + SQFS_FLAG_NO_XATTRS = 0x0200, + SQFS_FLAG_COMPRESSOR_OPTIONS = 0x0400, + SQFS_FLAG_UNCOMPRESSED_IDS = 0x0800, +} E_SQFS_SUPER_FLAGS; + +typedef enum { + SQFS_INODE_DIR = 1, + SQFS_INODE_FILE = 2, + SQFS_INODE_SLINK = 3, + SQFS_INODE_BDEV = 4, + SQFS_INODE_CDEV = 5, + SQFS_INODE_FIFO = 6, + SQFS_INODE_SOCKET = 7, + SQFS_INODE_EXT_DIR = 8, + SQFS_INODE_EXT_FILE = 9, + SQFS_INODE_EXT_SLINK = 10, + SQFS_INODE_EXT_BDEV = 11, + SQFS_INODE_EXT_CDEV = 12, + SQFS_INODE_EXT_FIFO = 13, + SQFS_INODE_EXT_SOCKET = 14, +} E_SQFS_INODE_TYPE; + +typedef enum { + SQUASHFS_XATTR_USER = 0, + SQUASHFS_XATTR_TRUSTED = 1, + SQUASHFS_XATTR_SECURITY = 2, + + SQUASHFS_XATTR_FLAG_OOL = 0x100, + SQUASHFS_XATTR_PREFIX_MASK = 0xFF, +} E_SQFS_XATTR_TYPE; + +/* Returns 0 on success. Prints error messages to stderr on failure. */ +int sqfs_super_init(sqfs_super_t *super, size_t block_size, uint32_t mtime, + E_SQFS_COMPRESSOR compressor); + +/* Returns 0 on success. Prints error messages to stderr on failure. */ +int sqfs_super_write(sqfs_super_t *super, int fd); + +/* Returns 0 on success. Prints error messages to stderr on failure. */ +int sqfs_super_read(sqfs_super_t *super, int fd); + +/* Get id from xattr key prefix or -1 if not supported */ +int sqfs_get_xattr_prefix_id(const char *key); + +/* Get a prefix string from the ID or NULL if unknown */ +const char *sqfs_get_xattr_prefix(E_SQFS_XATTR_TYPE id); + +/* Check if a given xattr key can be encoded in squashfs at all. */ +bool sqfs_has_xattr(const char *key); + +#endif /* SQUASHFS_H */ -- cgit v1.2.3