diff --git a/configure b/configure index 74a295d..99b5358 100755 --- a/configure +++ b/configure @@ -3443,6 +3443,7 @@ have_tre="no" have_zip="no" have_zlib="no" +have_zstd="no" have_tcpwrappers="no" have_tweak_sent_time="no" @@ -4146,6 +4147,96 @@ ac_cv_lib_zip=ac_cv_lib_zip_main +for ac_header in zstd.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "zstd.h" "ac_cv_header_zstd_h" "$ac_includes_default" +if test "x$ac_cv_header_zstd_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_ZSTD_H 1 +_ACEOF + have_zstd=yes +else + have_zstd=no +fi + +done + +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for main in -lzstd" >&5 +$as_echo_n "checking for main in -lzstd... " >&6; } +if ${ac_cv_lib_zstd_main+:} false; then : + $as_echo_n "(cached) " >&6 +else + ac_check_lib_save_LIBS=$LIBS +LIBS="-lzstd $LIBS" +cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + + +int +main () +{ +return main (); + ; + return 0; +} +_ACEOF +if ac_fn_c_try_link "$LINENO"; then : + ac_cv_lib_zstd_main=yes +else + ac_cv_lib_zstd_main=no +fi +rm -f core conftest.err conftest.$ac_objext \ + conftest$ac_exeext conftest.$ac_ext +LIBS=$ac_check_lib_save_LIBS +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_zstd_main" >&5 +$as_echo "$ac_cv_lib_zstd_main" >&6; } +if test "x$ac_cv_lib_zstd_main" = xyes; then : + { $as_echo "$as_me:${as_lineno-$LINENO}: checking for ZSTD_compress in -lzstd" >&5 +$as_echo_n "checking for ZSTD_compress in -lzstd... " >&6; } +if ${ac_cv_lib_zstd_ZSTD_compress+:} false; then : + $as_echo_n "(cached) " >&6 +else + ac_check_lib_save_LIBS=$LIBS +LIBS="-lzstd $LIBS" +cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +/* Override any GCC internal prototype to avoid an error. + Use char because int might match the return type of a GCC + builtin and then its argument prototype would still apply. */ +#ifdef __cplusplus +extern "C" +#endif +char ZSTD_compress (); +int +main () +{ +return ZSTD_compress (); + ; + return 0; +} +_ACEOF +if ac_fn_c_try_link "$LINENO"; then : + ac_cv_lib_zstd_ZSTD_compress=yes +else + ac_cv_lib_zstd_ZSTD_compress=no +fi +rm -f core conftest.err conftest.$ac_objext \ + conftest$ac_exeext conftest.$ac_ext +LIBS=$ac_check_lib_save_LIBS +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_zstd_ZSTD_compress" >&5 +$as_echo "$ac_cv_lib_zstd_ZSTD_compress" >&6; } +if test "x$ac_cv_lib_zstd_ZSTD_compress" = xyes; then : + have_zstd=yes +else + echo "libzstd.so is not found"; have_zstd=no +fi + +fi +ac_cv_lib_z=ac_cv_lib_zstd_main + for ac_header in zlib.h do : @@ -4716,6 +4807,22 @@ antispam_libs="$antispam_libs -ltre" fi +if test "$have_zstd" = "yes"; then + echo "zstd library: yes" + antispam_libs="$antispam_libs -lzstd" + defs="$defs -DHAVE_ZSTD" + + if test "$(which zstd)" = ""; then echo "Error: cannot find the zstd binary"; exit 1; fi + + zstd_magic=$(echo this is a test | zstd | hexdump -C | awk '/00000000/{print "ntohl(0x"$2$3$4$5 ")"}') + + +cat >>confdefs.h <<_ACEOF +#define ZSTD_DETECTED_MAGICNUMBER $zstd_magic +_ACEOF + +fi + if test "$have_zip" = "yes"; then echo "zip library: yes" diff --git a/configure.in b/configure.in index 156f6db..5c6f27c 100644 --- a/configure.in +++ b/configure.in @@ -41,6 +41,7 @@ have_tre="no" have_zip="no" have_zlib="no" +have_zstd="no" have_tcpwrappers="no" have_tweak_sent_time="no" @@ -169,6 +170,10 @@ AC_CHECK_HEADERS(zip.h, have_zip=yes, echo "zip.h is not found") AC_CHECK_LIB([zip],[main],[AC_CHECK_LIB(zip, zip_open, have_zip=yes, echo "libzip.so is not found"; have_zip=no)],[],[])ac_cv_lib_zip=ac_cv_lib_zip_main +dnl check zstd library + +AC_CHECK_HEADERS(zstd.h, have_zstd=yes, have_zstd=no) +AC_CHECK_LIB([zstd],[main],[AC_CHECK_LIB(zstd, ZSTD_compress, have_zstd=yes, echo "libzstd.so is not found"; have_zstd=no)],[],[])ac_cv_lib_z=ac_cv_lib_zstd_main dnl zlib @@ -434,6 +439,18 @@ antispam_libs="$antispam_libs -ltre" fi +if test "$have_zstd" = "yes"; then + echo "zstd library: yes" + antispam_libs="$antispam_libs -lzstd" + defs="$defs -DHAVE_ZSTD" + + if test "$(which zstd)" = ""; then echo "Error: cannot find the zstd binary"; exit 1; fi + + zstd_magic=$(echo this is a test | zstd | hexdump -C | awk '/00000000/{print "ntohl(0x"$2$3$4$5 ")"}') + + AC_DEFINE_UNQUOTED(ZSTD_DETECTED_MAGICNUMBER, $zstd_magic, [detected zstd magic number]) +fi + if test "$have_zip" = "yes"; then echo "zip library: yes" AC_DEFINE_UNQUOTED(HAVE_ZIP, 1, [libzip support]) diff --git a/piler-config.h.in b/piler-config.h.in index 4053acc..72144ac 100644 --- a/piler-config.h.in +++ b/piler-config.h.in @@ -30,3 +30,4 @@ #undef HAVE_SUPPORT_FOR_COMPAT_STORAGE_LAYOUT +#undef ZSTD_DETECTED_MAGICNUMBER diff --git a/src/archive.c b/src/archive.c index 3160b6a..a1e9215 100644 --- a/src/archive.c +++ b/src/archive.c @@ -20,6 +20,10 @@ #include #include +#ifdef HAVE_ZSTD + #include +#endif + void zerr(int ret){ @@ -131,12 +135,54 @@ } +#ifdef HAVE_ZSTD +int zstd_inf(unsigned char *in, int len, int mode, char **buffer, FILE *dest){ + size_t dSize; + unsigned long long const rSize = ZSTD_getDecompressedSize(in, len); + + if(rSize == 0) + return Z_DATA_ERROR; + + if(mode == WRITE_TO_STDOUT){ + void* const rBuff = malloc((size_t)rSize); + if(!rBuff) return Z_MEM_ERROR; + + dSize = ZSTD_decompress(rBuff, rSize, in, len); + + if(dSize != rSize){ + printf("error decoding buffer: %s\n", ZSTD_getErrorName(dSize)); + return Z_DATA_ERROR; + } + + fwrite(rBuff, 1, dSize, dest); + + free(rBuff); + } + else { + *buffer = malloc((size_t)rSize); + if(!*buffer) return Z_MEM_ERROR; + memset(*buffer, 0, rSize); + + dSize = ZSTD_decompress(*buffer, rSize, in, len); + if(dSize != rSize){ + printf("error decoding buffer: %s\n", ZSTD_getErrorName(dSize)); + return Z_DATA_ERROR; + } + } + + return Z_OK; +} +#endif + + int retrieve_file_from_archive(char *filename, int mode, char **buffer, FILE *dest, struct __config *cfg){ int rc=0, n, olen, tlen, len, fd=-1; unsigned char *s=NULL, *addr=NULL, inbuf[REALLYBIGBUFSIZE]; struct stat st; EVP_CIPHER_CTX ctx; - +#ifdef HAVE_ZSTD + unsigned long magic; +#endif if(filename == NULL) return 1; @@ -188,11 +234,40 @@ tlen += olen; - rc = inf(s, tlen, mode, buffer, dest); + + if(tlen > 4){ + #ifdef HAVE_ZSTD + memcpy(&magic, addr, 4); + + if(magic == ZSTD_DETECTED_MAGICNUMBER){ + rc = zstd_inf(s, tlen, mode, buffer, dest); + } + else { + #endif + rc = inf(s, tlen, mode, buffer, dest); + #ifdef HAVE_ZSTD + } + #endif + } } else { addr = mmap(NULL, st.st_size, PROT_READ, MAP_PRIVATE, fd, 0); - rc = inf(addr, st.st_size, mode, buffer, dest); + + if(st.st_size > 4){ + #ifdef HAVE_ZSTD + memcpy(&magic, addr, 4); + + if(magic == ZSTD_DETECTED_MAGICNUMBER){ + rc = zstd_inf(addr, st.st_size, mode, buffer, dest); + } + else { + #endif + rc = inf(addr, st.st_size, mode, buffer, dest); + #ifdef HAVE_ZSTD + } + #endif + } + munmap(addr, st.st_size); } diff --git a/src/store.c b/src/store.c index 1230a45..d051ca2 100644 --- a/src/store.c +++ b/src/store.c @@ -13,8 +13,12 @@ #include #include #include +#ifdef HAVE_ZSTD + #include +#else + #include +#endif #include -#include #include #include #include @@ -43,8 +47,13 @@ int ret=0, rc, fd, n; char *addr, *p, *p0, *p1, *p2, s[SMALLBUFSIZE]; struct stat st; - Bytef *z=NULL; +#ifdef HAVE_ZSTD + void *dst; + size_t dstlen; +#else + Bytef *dst=NULL; uLongf dstlen; +#endif EVP_CIPHER_CTX ctx; unsigned char *outbuf=NULL; @@ -80,23 +89,45 @@ if(addr == MAP_FAILED) return ret; - dstlen = compressBound(len); + rc = OK; - z = malloc(dstlen); +#ifdef HAVE_ZSTD + dstlen = ZSTD_compressBound(len); + dst = malloc(dstlen); - if(z == NULL){ + if(dst == NULL){ munmap(addr, len); - syslog(LOG_PRIORITY, "%s: cannot malloc for z buffer", sdata->ttmpfile); + syslog(LOG_PRIORITY, "%s: ERROR: cannot malloc for zstd buffer", sdata->ttmpfile); return ret; } - rc = compress(z, &dstlen, (const Bytef *)addr, len); + size_t const cSize = ZSTD_compress(dst, dstlen, addr, len, 1); + if(ZSTD_isError(cSize)){ + syslog(LOG_PRIORITY, "%s: ERROR: zstd compressing: %s", sdata->ttmpfile, ZSTD_getErrorName(cSize)); + rc = ERR; + } + + dstlen = cSize; +#else + dstlen = compressBound(len); + dst = malloc(dstlen); + + if(dst == NULL){ + munmap(addr, len); + syslog(LOG_PRIORITY, "%s: ERROR: cannot malloc for z buffer", sdata->ttmpfile); + return ret; + } + + if(compress(dst, &dstlen, (const Bytef *)addr, len) != Z_OK) + rc = ERR; +#endif + gettimeofday(&tv2, &tz); sdata->__compress += tvdiff(tv2, tv1); munmap(addr, len); - if(rc != Z_OK) goto ENDE; + if(rc == ERR) goto ENDE; if(cfg->encrypt_messages == 1){ gettimeofday(&tv1, &tz); @@ -107,7 +138,7 @@ outbuf = malloc(dstlen + EVP_MAX_BLOCK_LENGTH); if(outbuf == NULL) goto ENDE; - if(!EVP_EncryptUpdate(&ctx, outbuf, &outlen, z, dstlen)) goto ENDE; + if(!EVP_EncryptUpdate(&ctx, outbuf, &outlen, dst, dstlen)) goto ENDE; if(!EVP_EncryptFinal_ex(&ctx, outbuf + outlen, &tmplen)) goto ENDE; outlen += tmplen; EVP_CIPHER_CTX_cleanup(&ctx); @@ -161,7 +192,7 @@ writelen = outlen; } else { - n = write(fd, z, dstlen); + n = write(fd, dst, dstlen); writelen = dstlen; } @@ -185,7 +216,7 @@ ENDE: if(outbuf) free(outbuf); - if(z) free(z); + if(dst) free(dst); return ret; }