diff --git a/configure b/configure index 2c7b62b..99b5358 100755 --- a/configure +++ b/configure @@ -4809,12 +4809,18 @@ if test "$have_zstd" = "yes"; then echo "zstd library: yes" + antispam_libs="$antispam_libs -lzstd" + defs="$defs -DHAVE_ZSTD" + + if test "$(which zstd)" = ""; then echo "Error: cannot find the zstd binary"; exit 1; fi + + zstd_magic=$(echo this is a test | zstd | hexdump -C | awk '/00000000/{print "ntohl(0x"$2$3$4$5 ")"}') + cat >>confdefs.h <<_ACEOF -#define HAVE_ZSTD 1 +#define ZSTD_DETECTED_MAGICNUMBER $zstd_magic _ACEOF - antispam_libs="$antispam_libs -lzstd" fi if test "$have_zip" = "yes"; then diff --git a/configure.in b/configure.in index 5eea65e..5c6f27c 100644 --- a/configure.in +++ b/configure.in @@ -441,8 +441,14 @@ if test "$have_zstd" = "yes"; then echo "zstd library: yes" - AC_DEFINE_UNQUOTED(HAVE_ZSTD, 1, [libzstd support]) antispam_libs="$antispam_libs -lzstd" + defs="$defs -DHAVE_ZSTD" + + if test "$(which zstd)" = ""; then echo "Error: cannot find the zstd binary"; exit 1; fi + + zstd_magic=$(echo this is a test | zstd | hexdump -C | awk '/00000000/{print "ntohl(0x"$2$3$4$5 ")"}') + + AC_DEFINE_UNQUOTED(ZSTD_DETECTED_MAGICNUMBER, $zstd_magic, [detected zstd magic number]) fi if test "$have_zip" = "yes"; then diff --git a/piler-config.h.in b/piler-config.h.in index a9915fc..72144ac 100644 --- a/piler-config.h.in +++ b/piler-config.h.in @@ -26,9 +26,8 @@ #undef HAVE_LIBWRAP -#undef HAVE_ZSTD - #undef HAVE_TWEAK_SENT_TIME #undef HAVE_SUPPORT_FOR_COMPAT_STORAGE_LAYOUT +#undef ZSTD_DETECTED_MAGICNUMBER diff --git a/src/archive.c b/src/archive.c index 3160b6a..6b23f30 100644 --- a/src/archive.c +++ b/src/archive.c @@ -20,6 +20,10 @@ #include #include +#ifdef HAVE_ZSTD + #include +#endif + void zerr(int ret){ @@ -131,12 +135,53 @@ } +int zstd_inf(unsigned char *in, int len, int mode, char **buffer, FILE *dest){ + size_t dSize; + unsigned long long const rSize = ZSTD_getDecompressedSize(in, len); + + if(rSize == 0) + return Z_DATA_ERROR; + + if(mode == WRITE_TO_STDOUT){ + void* const rBuff = malloc((size_t)rSize); + if(!rBuff) return Z_MEM_ERROR; + + dSize = ZSTD_decompress(rBuff, rSize, in, len); + + if(dSize != rSize){ + printf("error decoding buffer: %s\n", ZSTD_getErrorName(dSize)); + return Z_DATA_ERROR; + } + + fwrite(rBuff, 1, dSize, dest); + + free(rBuff); + } + else { + *buffer = malloc((size_t)rSize); + if(!*buffer) return Z_MEM_ERROR; + memset(*buffer, 0, rSize); + + dSize = ZSTD_decompress(*buffer, rSize, in, len); + if(dSize != rSize){ + printf("error decoding buffer: %s\n", ZSTD_getErrorName(dSize)); + return Z_DATA_ERROR; + } + } + + return Z_OK; +} + + + int retrieve_file_from_archive(char *filename, int mode, char **buffer, FILE *dest, struct __config *cfg){ int rc=0, n, olen, tlen, len, fd=-1; unsigned char *s=NULL, *addr=NULL, inbuf[REALLYBIGBUFSIZE]; struct stat st; EVP_CIPHER_CTX ctx; - +#ifdef HAVE_ZSTD + unsigned long magic; +#endif if(filename == NULL) return 1; @@ -188,11 +233,39 @@ tlen += olen; - rc = inf(s, tlen, mode, buffer, dest); + + if(st.st_size > 4){ + memcpy(&magic, addr, 4); + #ifdef HAVE_ZSTD + if(magic == ZSTD_DETECTED_MAGICNUMBER){ + rc = zstd_inf(s, tlen, mode, buffer, dest); + } + else { + #endif + rc = inf(s, tlen, mode, buffer, dest); + #ifdef HAVE_ZSTD + } + #endif + } } else { addr = mmap(NULL, st.st_size, PROT_READ, MAP_PRIVATE, fd, 0); - rc = inf(addr, st.st_size, mode, buffer, dest); + + if(st.st_size > 4){ + #ifdef HAVE_ZSTD + memcpy(&magic, addr, 4); + + if(magic == ZSTD_DETECTED_MAGICNUMBER){ + rc = zstd_inf(addr, st.st_size, mode, buffer, dest); + } + else { + #endif + rc = inf(addr, st.st_size, mode, buffer, dest); + #ifdef HAVE_ZSTD + } + #endif + } + munmap(addr, st.st_size); } diff --git a/src/store.c b/src/store.c index f06341d..d051ca2 100644 --- a/src/store.c +++ b/src/store.c @@ -97,22 +97,24 @@ if(dst == NULL){ munmap(addr, len); - syslog(LOG_PRIORITY, "%s: cannot malloc for zstd buffer", sdata->ttmpfile); + syslog(LOG_PRIORITY, "%s: ERROR: cannot malloc for zstd buffer", sdata->ttmpfile); return ret; } - size_t const cSize = ZSTD_compress(dst, destlen, addr, len, 1); + size_t const cSize = ZSTD_compress(dst, dstlen, addr, len, 1); if(ZSTD_isError(cSize)){ - syslog(LOG_PRIORITY, "%s: error zstd compressing: %s", sdata->ttmpfile, ZSTD_getErrorName(cSize)); + syslog(LOG_PRIORITY, "%s: ERROR: zstd compressing: %s", sdata->ttmpfile, ZSTD_getErrorName(cSize)); rc = ERR; } + + dstlen = cSize; #else dstlen = compressBound(len); dst = malloc(dstlen); if(dst == NULL){ munmap(addr, len); - syslog(LOG_PRIORITY, "%s: cannot malloc for z buffer", sdata->ttmpfile); + syslog(LOG_PRIORITY, "%s: ERROR: cannot malloc for z buffer", sdata->ttmpfile); return ret; }