From cce36f459ddb5698fd1a40061c466996482146eb Mon Sep 17 00:00:00 2001 From: David Oberhollenzer Date: Fri, 26 Jul 2019 23:07:41 +0200 Subject: Implement fragment deduplication in data writer The strategy is simple: - The data writer function that write data/fragment blocks get access to the list files. - When writing a fragment, we look for an already written file that has a fragment with the same size and checksum. - If we find one, we throw away the fragment and reuse the existing one. Signed-off-by: David Oberhollenzer --- include/data_writer.h | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) (limited to 'include/data_writer.h') diff --git a/include/data_writer.h b/include/data_writer.h index 7107868..f5ad572 100644 --- a/include/data_writer.h +++ b/include/data_writer.h @@ -66,10 +66,14 @@ int data_writer_flush_fragments(data_writer_t *data); The flags argument is a combination of DW_* flags. + If 'list' is not NULL, it is used for fragment and data block deduplication. + It is assumed that the list is processed in order and scanning stops as soon + as the current file info 'fi' is encountered in the list. + Returns 0 on success, prints errors to stderr. */ int write_data_from_fd(data_writer_t *data, file_info_t *fi, int infd, - int flags); + int flags, file_info_t *list); /* Does the same as write_data_from_fd but the input file is the condensed @@ -78,9 +82,14 @@ int write_data_from_fd(data_writer_t *data, file_info_t *fi, int infd, The flags argument is a combination of DW_* flags. + If 'list' is not NULL, it is used for fragment and data block deduplication. + It is assumed that the list is processed in order and scanning stops as soon + as the current file info 'fi' is encountered in the list. + Returns 0 on success, prints errors to stderr. */ int write_data_from_fd_condensed(data_writer_t *data, file_info_t *fi, - int infd, sparse_map_t *map, int flags); + int infd, sparse_map_t *map, int flags, + file_info_t *list); #endif /* DATA_WRITER_H */ -- cgit v1.2.3