Change snapshot format - dedup - deduplicating backup program
(HTM) git clone git://bitreich.org/dedup/ git://enlrupgkhuxnvlhsf6lc3fziv5h2hhfrinws65d7roiv6bfj7d652fid.onion/dedup/
(DIR) Log
(DIR) Files
(DIR) Refs
(DIR) Tags
(DIR) README
(DIR) LICENSE
---
(DIR) commit 03aa3e43c5b81940bab9e6990d71bf036a188f27
(DIR) parent 14a97f465c31b9e0c707730ca8798dfd4140cb03
(HTM) Author: sin <sin@2f30.org>
Date: Tue, 7 May 2019 20:56:44 +0100
Change snapshot format
Include a header at the front so we can store the nonce etc.
Diffstat:
M snap.c | 136 +++++++++++++++++++++++++++++--
1 file changed, 128 insertions(+), 8 deletions(-)
---
(DIR) diff --git a/snap.c b/snap.c
@@ -2,6 +2,7 @@
#include <sys/types.h>
#include <sys/stat.h>
+#include <assert.h>
#include <errno.h>
#include <fcntl.h>
#include <limits.h>
@@ -12,13 +13,37 @@
#include <string.h>
#include <unistd.h>
+#include <sodium.h>
+
#include "config.h"
#include "misc.h"
#include "queue.h"
#include "snap.h"
+#define SHDRMAGIC "SNAPSNAPPYSNOOP"
+#define NSHDRMAGIC sizeof(SHDRMAGIC)
+#define VMIN 0
+#define VMAJ 1
+#define VMINMASK 0xff
+#define VMAJSHIFT 8
+#define VMAJMASK 0xff
+#define SHDRSIZE (NSHDRMAGIC + 24 + 24 + 8 + 8)
+
#define NERRBUF 128
+/* misc helpers */
+extern int pack(unsigned char *, char *, ...);
+extern int unpack(unsigned char *, char *, ...);
+
+/* Snapshot header structure */
+struct shdr {
+ char magic[NSHDRMAGIC]; /* magic number for file(1) */
+ unsigned char nonce[crypto_aead_xchacha20poly1305_ietf_NPUBBYTES];
+ unsigned char header[crypto_secretstream_xchacha20poly1305_HEADERBYTES];
+ uint64_t flags; /* version number */
+ uint64_t nbd; /* number of block hashes */
+};
+
struct mdnode {
unsigned char md[MDSIZE]; /* hash of block */
TAILQ_ENTRY(mdnode) e; /* mdhead link node */
@@ -29,10 +54,54 @@ struct sctx {
struct mdnode *mdnext; /* next hash to be returned via sget() */
int fd; /* underlying snapshot file descriptor */
int rdonly; /* when set to 1, the ssync() operation is a no-op */
+ struct shdr shdr;
};
static char errbuf[NERRBUF];
+/* Read snapshot header */
+static int
+unpackshdr(int fd, struct shdr *shdr)
+{
+ unsigned char buf[SHDRSIZE];
+ int n;
+
+ if (xread(fd, buf, sizeof(buf)) != sizeof(buf)) {
+ sseterr("failed to read snapshot header: %s", strerror(errno));
+ return -1;
+ }
+
+ n = unpack(buf, "'16'24qq",
+ shdr->magic,
+ shdr->nonce,
+ &shdr->flags,
+ &shdr->nbd);
+
+ assert(n == sizeof(buf));
+ return n;
+}
+
+/* Write snapshot header */
+static int
+packshdr(int fd, struct shdr *shdr)
+{
+ unsigned char buf[SHDRSIZE];
+ int n;
+
+ n = pack(buf, "'16'24qq",
+ shdr->magic,
+ shdr->nonce,
+ shdr->flags,
+ shdr->nbd);
+
+ assert(n == SHDRSIZE);
+ if (xwrite(fd, buf, n) != n) {
+ sseterr("failed to write snapshot header: %s", strerror(errno));
+ return -1;
+ }
+ return n;
+}
+
static int
loadmd(struct sctx *sctx)
{
@@ -55,15 +124,11 @@ loadmd(struct sctx *sctx)
static int
initmdhead(struct sctx *sctx)
{
- struct stat st;
- uint64_t i, n;
+ struct shdr *shdr;
+ uint64_t i;
- if (fstat(sctx->fd, &st) < 0) {
- sseterr("fstat: %s", strerror(errno));
- return -1;
- }
- n = st.st_size / MDSIZE;
- for (i = 0; i < n; i++) {
+ shdr = &sctx->shdr;
+ for (i = 0; i < shdr->nbd; i++) {
if (loadmd(sctx) == 0)
continue;
@@ -83,6 +148,7 @@ initmdhead(struct sctx *sctx)
int
screat(char *path, int mode, struct sctx **sctx)
{
+ struct shdr *shdr;
int fd;
if (path == NULL || sctx == NULL) {
@@ -90,6 +156,11 @@ screat(char *path, int mode, struct sctx **sctx)
return -1;
}
+ if (sodium_init() < 0) {
+ sseterr("sodium_init: failed");
+ return -1;
+ }
+
fd = open(path, O_RDWR | O_CREAT | O_EXCL, mode);
if (fd < 0) {
sseterr("open: %s", strerror(errno));
@@ -106,12 +177,25 @@ screat(char *path, int mode, struct sctx **sctx)
TAILQ_INIT(&(*sctx)->mdhead);
(*sctx)->mdnext = NULL;
(*sctx)->fd = fd;
+
+ shdr = &(*sctx)->shdr;
+ memcpy(shdr->magic, SHDRMAGIC, NSHDRMAGIC);
+ shdr->flags = (VMAJ << VMAJSHIFT) | VMIN;
+ shdr->nbd = 0;
+
+ if (packshdr(fd, shdr) < 0) {
+ free(*sctx);
+ close(fd);
+ return -1;
+ }
+
return 0;
}
int
sopen(char *path, int flags, int mode, struct sctx **sctx)
{
+ struct shdr *shdr;
int fd;
if (path == NULL || sctx == NULL) {
@@ -125,6 +209,11 @@ sopen(char *path, int flags, int mode, struct sctx **sctx)
return -1;
}
+ if (sodium_init() < 0) {
+ sseterr("sodium_init: failed");
+ return -1;
+ }
+
fd = open(path, O_RDONLY, mode);
if (fd < 0) {
sseterr("open: %s", strerror(errno));
@@ -143,6 +232,29 @@ sopen(char *path, int flags, int mode, struct sctx **sctx)
(*sctx)->fd = fd;
(*sctx)->rdonly = 1;
+ shdr = &(*sctx)->shdr;
+
+ if (unpackshdr(fd, shdr) < 0) {
+ free(sctx);
+ close(fd);
+ return -1;
+ }
+
+ if (memcmp(shdr->magic, SHDRMAGIC, NSHDRMAGIC) != 0) {
+ free(sctx);
+ close(fd);
+ sseterr("unknown snapshot header magic");
+ return -1;
+ }
+
+ /* If the major version is different, the format is incompatible */
+ if (((shdr->flags >> VMAJSHIFT) & VMAJMASK) != VMAJ) {
+ free(sctx);
+ close(fd);
+ sseterr("snapshot header version mismatch");
+ return -1;
+ }
+
if (initmdhead(*sctx) < 0) {
free(*sctx);
close(fd);
@@ -154,6 +266,7 @@ sopen(char *path, int flags, int mode, struct sctx **sctx)
int
sput(struct sctx *sctx, unsigned char *md)
{
+ struct shdr *shdr;
struct mdnode *mdnode;
if (sctx == NULL || md == NULL) {
@@ -166,6 +279,8 @@ sput(struct sctx *sctx, unsigned char *md)
sseterr("calloc: %s", strerror(errno));
return -1;
}
+ shdr = &sctx->shdr;
+ shdr->nbd++;
memcpy(mdnode->md, md, MDSIZE);
TAILQ_INSERT_TAIL(&sctx->mdhead, mdnode, e);
return 0;
@@ -208,6 +323,7 @@ srewind(struct sctx *sctx)
int
ssync(struct sctx *sctx)
{
+ struct shdr *shdr;
struct mdnode *mdnode;
if (sctx == NULL) {
@@ -222,6 +338,10 @@ ssync(struct sctx *sctx)
sseterr("lseek: %s", strerror(errno));
return -1;
}
+
+ shdr = &sctx->shdr;
+ if (packshdr(sctx->fd, shdr) < 0)
+ return -1;
TAILQ_FOREACH(mdnode, &sctx->mdhead, e) {
if (xwrite(sctx->fd, mdnode->md, MDSIZE) != MDSIZE) {
sseterr("failed to write block hash: %s",