Add lz4 support - dedup - deduplicating backup program
(HTM) git clone git://bitreich.org/dedup/ git://enlrupgkhuxnvlhsf6lc3fziv5h2hhfrinws65d7roiv6bfj7d652fid.onion/dedup/
(DIR) Log
(DIR) Files
(DIR) Refs
(DIR) Tags
(DIR) README
(DIR) LICENSE
---
(DIR) commit 9f54a917ef6c2f8f850684bed77fa708a062ca33
(DIR) parent 1ed8f937d82f094ae3e4db7511256dee7d5d471a
(HTM) Author: sin <sin@2f30.org>
Date: Sun, 5 May 2019 11:58:12 +0100
Add lz4 support
Diffstat:
M Makefile | 2 +-
M README | 1 +
M TODO | 1 -
M bcompress.c | 33 ++++++++++++++++++++++++++++++-
M bstorage.c | 6 ++++++
M dup-init.1 | 4 ++--
A test007 | 18 ++++++++++++++++++
7 files changed, 60 insertions(+), 5 deletions(-)
---
(DIR) diff --git a/Makefile b/Makefile
@@ -34,7 +34,7 @@ DPACKOBJ = $(COMMOBJ) dup-pack.o
DRMOBJ = $(COMMOBJ) dup-rm.o
DUNPACKOBJ = $(COMMOBJ) dup-unpack.o
-LDLIBS = -lsnappy -lsodium
+LDLIBS = -lsnappy -llz4 -lsodium
all: $(BIN)
(DIR) diff --git a/README b/README
@@ -58,6 +58,7 @@ Dependencies
============
- snappy
+ - liblz4
- libsodium
Contact
(DIR) diff --git a/TODO b/TODO
@@ -1,6 +1,5 @@
Use a ring buffer in the chunker (avoid memmove() call)
pledge/unveil support
Use flock() to avoid corruption
-Revert lz4 support
Ability to fetch key from dedicated fd so it never touches disk
Use pread/pwrite to avoid lseek() calls
(DIR) diff --git a/bcompress.c b/bcompress.c
@@ -12,12 +12,14 @@
#include <strings.h>
#include <unistd.h>
+#include <lz4.h>
#include <snappy-c.h>
#include "block.h"
#define CDNONETYPE 0x200
#define CDSNAPPYTYPE 0x201
+#define CDLZ4TYPE 0x202
#define CDSIZE (8 + 8)
extern int pack(unsigned char *, char *, ...);
@@ -98,6 +100,8 @@ bccreat(struct bctx *bctx, char *path, int mode, struct bparam *bpar)
type = CDNONETYPE;
} else if (strcasecmp(bpar->calgo, "snappy") == 0) {
type = CDSNAPPYTYPE;
+ } else if (strcasecmp(bpar->calgo, "lz4") == 0) {
+ type = CDLZ4TYPE;
} else {
bseterr("invalid compression type: %s", bpar->calgo);
return -1;
@@ -142,6 +146,8 @@ bcopen(struct bctx *bctx, char *path, int flags, int mode, struct bparam *bpar)
cctx->type = CDNONETYPE;
} else if (strcasecmp(bpar->calgo, "snappy") == 0) {
cctx->type = CDSNAPPYTYPE;
+ } else if (strcasecmp(bpar->calgo, "lz4") == 0) {
+ cctx->type = CDLZ4TYPE;
} else {
bops->close(bctx);
free(cctx);
@@ -166,6 +172,8 @@ bcput(struct bctx *bctx, void *buf, size_t n, unsigned char *md)
cn = n;
} else if (cctx->type == CDSNAPPYTYPE) {
cn = snappy_max_compressed_length(n);
+ } else if (cctx->type == CDLZ4TYPE) {
+ cn = LZ4_compressBound(n);
} else {
bseterr("invalid compression type: %d", cctx->type);
return -1;
@@ -187,6 +195,16 @@ bcput(struct bctx *bctx, void *buf, size_t n, unsigned char *md)
bseterr("snappy_compress: failed");
return -1;
}
+ } else if (cctx->type == CDLZ4TYPE) {
+ int ret;
+
+ ret = LZ4_compress_default(buf, &cbuf[CDSIZE], n, cn);
+ if (ret < 0) {
+ free(cbuf);
+ bseterr("LZ4_compress_default: failed");
+ return -1;
+ }
+ cn = ret;
} else {
free(cbuf);
bseterr("invalid compression type: %d", cctx->type);
@@ -218,7 +236,10 @@ bcget(struct bctx *bctx, unsigned char *md, void *buf, size_t *n)
/* Calculate maximum compressed block size */
size = *n;
- cn = snappy_max_compressed_length(size);
+ cn = snappy_max_compressed_length(*n);
+ if (cn > size)
+ size = cn;
+ cn = LZ4_compressBound(*n);
if (cn > size)
size = cn;
size += CDSIZE;
@@ -269,6 +290,16 @@ bcget(struct bctx *bctx, unsigned char *md, void *buf, size_t *n)
bseterr("snappy_uncompress: failed");
return -1;
}
+ } else if (cd.type == CDLZ4TYPE) {
+ int ret;
+
+ ret = LZ4_decompress_safe(&cbuf[CDSIZE], buf, cd.size, *n);
+ if (ret < 0) {
+ free(cbuf);
+ bseterr("LZ4_decompress_safe: failed");
+ return -1;
+ }
+ un = ret;
} else {
free(cbuf);
bseterr("invalid compression type: %d", cd.type);
(DIR) diff --git a/bstorage.c b/bstorage.c
@@ -45,6 +45,7 @@
#define CALGOMASK 0x7
#define CNONETYPE 0
#define CSNAPPYTYPE 1
+#define CLZ4TYPE 2
#define BHDRSIZE (NBHDRMAGIC + 8 + 8)
/* block descriptor constants */
@@ -339,6 +340,8 @@ bscreat(struct bctx *bctx, char *path, int mode, struct bparam *bpar)
bhdr->flags |= CNONETYPE << CALGOSHIFT;
} else if (strcasecmp(bpar->calgo, "snappy") == 0) {
bhdr->flags |= CSNAPPYTYPE << CALGOSHIFT;
+ } else if (strcasecmp(bpar->calgo, "lz4") == 0) {
+ bhdr->flags |= CLZ4TYPE << CALGOSHIFT;
} else {
free(sctx);
close(fd);
@@ -445,6 +448,9 @@ bsopen(struct bctx *bctx, char *path, int flags, int mode, struct bparam *bpar)
case CSNAPPYTYPE:
bpar->calgo = "snappy";
break;
+ case CLZ4TYPE:
+ bpar->calgo = "lz4";
+ break;
default:
free(sctx);
close(fd);
(DIR) diff --git a/dup-init.1 b/dup-init.1
@@ -1,4 +1,4 @@
-.Dd May 2, 2019
+.Dd May 5, 2019
.Dt DUP-INIT 1
.Os
.Sh NAME
@@ -27,7 +27,7 @@ By default none is used.
.It Fl Z Ar algo
The compressor algorithm used to compress the blocks
in the store.
-The supported compressor algorithms are none and snappy.
+The supported compressor algorithms are none and snappy and lz4.
By default snappy is used.
.It Fl k Ar keyfile
Path to encryption key.
(DIR) diff --git a/test007 b/test007
@@ -0,0 +1,18 @@
+#!/bin/sh
+set -ex
+
+SHA1SUM=${SHA1SUM:-sha1sum}
+
+repo=`mktemp -u`
+data=`mktemp`
+dd if=/dev/urandom of="$data" bs=1M count=64
+./dup-init -Z lz4 "$repo"
+./dup-pack -r "$repo" snap0 < "$data"
+./dup-pack -r "$repo" snap1 < "$data"
+du -sh "$repo"
+sum0=`$SHA1SUM < "$data" | awk '{print $1}'`
+sum1=`./dup-unpack -r "$repo" snap0 | $SHA1SUM | awk '{print $1}'`
+sum2=`./dup-unpack -r "$repo" snap1 | $SHA1SUM | awk '{print $1}'`
+[ "$sum0" = "$sum1" ]
+[ "$sum0" = "$sum2" ]
+rm -rf "$repo" "$data"