diff --git a/etc/example.conf b/etc/example.conf index c002a15..853d8b5 100644 --- a/etc/example.conf +++ b/etc/example.conf @@ -34,6 +34,14 @@ ; which will cause problems. encrypt_messages=1 +; If the previous 3 letter store top directories, eg. /var/piler/store/00/5f4 +; are consolidated (1) to a single zip file or not (0). +; Eg. cd /var/piler/store/00 +; zip -r -0 5f3.zip 5f3 +; After the consolidation (ie. zipping the given directory) piler tries to +; read the given file from the appropriate zip file. +consolidated_store=0 + ; number of worker processes, ie. the number of simultaneous smtp connections to piler. ; This value should be the number of cpus + 1, ie. 2 for a single cpu host number_of_worker_processes=2 diff --git a/src/archive.c b/src/archive.c index 9984c26..c77403c 100644 --- a/src/archive.c +++ b/src/archive.c @@ -18,6 +18,7 @@ #include #include #include +#include #include @@ -130,34 +131,97 @@ } +unsigned char *extract_file_from_zip(char *zipfile, char *filename, zip_uint64_t *size){ + int i=0, errorp; + unsigned char *p=NULL; + struct zip *z; + struct zip_stat sb; + struct zip_file *zf; +#if LIBZIP_VERSION_MAJOR >= 1 && LIBZIP_VERSION_MINOR >= 1 + int zip_flags = ZIP_RDONLY; +#else + int zip_flags = 0; +#endif + + z = zip_open(zipfile, zip_flags, &errorp); + if(!z){ + syslog(LOG_INFO, "%s: error: corrupt zip file=%s, error code=%d", zipfile, filename, errorp); + return NULL; + } + + while(zip_stat_index(z, i, 0, &sb) == 0){ + if(sb.size > 0 && sb.comp_size > 0 && strncmp(sb.name, filename, strlen(filename)) == 0){ + *size = sb.comp_size; + zf = zip_fopen_index(z, i, 0); + if(zf){ + p = malloc(sb.comp_size); + if(p){ + if(zip_fread(zf, p, sb.comp_size) == -1){ + syslog(LOG_PRIORITY, "zip_fread(): Error reading %s from %s", filename, zipfile); + free(p); + p = NULL; + } + } + zip_fclose(zf); + } + else syslog(LOG_PRIORITY, "cannot extract '%s' from '%s'", filename, zipfile); + } + + i++; + } + + zip_close(z); + + return p; +} + + int retrieve_file_from_archive(char *filename, int mode, char **buffer, FILE *dest, struct config *cfg){ int rc=0, n, olen, tlen, len, fd=-1; - unsigned char *s=NULL, *addr=NULL, inbuf[REALLYBIGBUFSIZE]; + char *relfilename; + unsigned char *s=NULL, *addr=NULL, *zipbuf=NULL, inbuf[REALLYBIGBUFSIZE]; struct stat st; + zip_uint64_t zipped_size=0; #if OPENSSL_VERSION_NUMBER < 0x10100000L EVP_CIPHER_CTX ctx; #else EVP_CIPHER_CTX *ctx=NULL; #endif - if(filename == NULL) return 1; + relfilename = strchr(filename, ' '); + if(relfilename){ + // We'll read the given file from a zip file + // filename is decomposed to and + *relfilename = '\0'; + relfilename++; - fd = open(filename, O_RDONLY); - if(fd == -1){ - syslog(LOG_PRIORITY, "%s: cannot open()", filename); - return 1; + zipbuf = extract_file_from_zip(filename, relfilename, &zipped_size); + if(!zipped_size){ + syslog(LOG_PRIORITY, "%s not found in %s", relfilename, filename); + return 1; + } + + len = zipped_size+EVP_MAX_BLOCK_LENGTH; } + else { + // We have a distinct .m file or an attachment to read from + fd = open(filename, O_RDONLY); + if(fd == -1){ + syslog(LOG_PRIORITY, "%s: cannot open()", filename); + return 1; + } + if(fstat(fd, &st)){ + syslog(LOG_PRIORITY, "%s: cannot fstat()", filename); + close(fd); + return 1; + } - if(fstat(fd, &st)){ - syslog(LOG_PRIORITY, "%s: cannot fstat()", filename); - close(fd); - return 1; + len = st.st_size+EVP_MAX_BLOCK_LENGTH; } - if(cfg->encrypt_messages == 1){ #if OPENSSL_VERSION_NUMBER < 0x10100000L EVP_CIPHER_CTX_init(&ctx); @@ -170,8 +234,6 @@ EVP_DecryptInit_ex(ctx, EVP_bf_cbc(), NULL, cfg->key, cfg->iv); #endif - len = st.st_size+EVP_MAX_BLOCK_LENGTH; - s = malloc(len); if(!s){ @@ -181,27 +243,35 @@ tlen = 0; - while((n = read(fd, inbuf, sizeof(inbuf)))){ + if(!relfilename){ + while((n = read(fd, inbuf, sizeof(inbuf)))){ - #if OPENSSL_VERSION_NUMBER < 0x10100000L - if(!EVP_DecryptUpdate(&ctx, s+tlen, &olen, inbuf, n)){ - #else - if(!EVP_DecryptUpdate(ctx, s+tlen, &olen, inbuf, n)){ - #endif - syslog(LOG_PRIORITY, "%s: EVP_DecryptUpdate()", filename); + #if OPENSSL_VERSION_NUMBER < 0x10100000L + if(!EVP_DecryptUpdate(&ctx, s+tlen, &olen, inbuf, n)){ + #else + if(!EVP_DecryptUpdate(ctx, s+tlen, &olen, inbuf, n)){ + #endif + syslog(LOG_PRIORITY, "%s: EVP_DecryptUpdate()", filename); + goto CLEANUP; + } + + tlen += olen; + } + } else { + if(!EVP_DecryptUpdate(ctx, s+tlen, &olen, zipbuf, zipped_size)){ + printf("error in EVP_DecryptUpdate\n"); goto CLEANUP; } - tlen += olen; } - #if OPENSSL_VERSION_NUMBER < 0x10100000L if(EVP_DecryptFinal(&ctx, s + tlen, &olen) != 1){ #else if(EVP_DecryptFinal(ctx, s + tlen, &olen) != 1){ #endif syslog(LOG_PRIORITY, "%s: EVP_DecryptFinal()", filename); + printf("error in EVP_DecryptFinal\n"); goto CLEANUP; } @@ -222,6 +292,7 @@ CLEANUP: if(fd != -1) close(fd); if(s) free(s); + if(zipbuf) free(zipbuf); if(cfg->encrypt_messages == 1) #if OPENSSL_VERSION_NUMBER < 0x10100000L EVP_CIPHER_CTX_cleanup(&ctx); @@ -233,13 +304,56 @@ } +void assemble_filename(char *filename, int len, char *s, struct config *cfg){ + char zipfilename[SMALLBUFSIZE]; + struct stat st; + + // /var/piler/00/5f5/00/5f5_00.zip + snprintf(zipfilename, sizeof(zipfilename)-1, "%s/%02x/%c%c%c/%c%c/%c%c%c_%c%c.zip", cfg->queuedir, cfg->server_id, s[8], s[9], s[10], s[RND_STR_LEN-4], s[RND_STR_LEN-3], s[8], s[9], s[10], s[RND_STR_LEN-4], s[RND_STR_LEN-3]); + + if(cfg->consolidated_store == 1 && stat(zipfilename, &st) == 0){ + // If the zip file exists, then fix the filename to be ' ' + snprintf(filename, len-1, "%s %c%c/%s.m", zipfilename, s[RND_STR_LEN-2], s[RND_STR_LEN-1], s); + } + else { + snprintf(filename, len-1, "%s/%02x/%c%c%c/%c%c/%c%c/%s.m", + cfg->queuedir, cfg->server_id, s[8], s[9], s[10], s[RND_STR_LEN-4], s[RND_STR_LEN-3], s[RND_STR_LEN-2], s[RND_STR_LEN-1], s); + #ifdef HAVE_SUPPORT_FOR_COMPAT_STORAGE_LAYOUT + if(stat(filename, &st)){ + snprintf(filename, len-1, "%s/%02x/%c%c/%c%c/%c%c/%s.m", + cfg->queuedir, cfg->server_id, s[RND_STR_LEN-6], s[RND_STR_LEN-5], s[RND_STR_LEN-4], s[RND_STR_LEN-3], s[RND_STR_LEN-2], s[RND_STR_LEN-1], s); + } + #endif + } +} + + +void assemble_attachment_filename(char *filename, int len, char *s, int attachment_id, struct config *cfg){ + char zipfilename[SMALLBUFSIZE]; + struct stat st; + + snprintf(zipfilename, sizeof(zipfilename)-1, "%s/%02x/%c%c%c/%c%c/%c%c%c_%c%c.zip", cfg->queuedir, cfg->server_id, s[8], s[9], s[10], s[RND_STR_LEN-4], s[RND_STR_LEN-3], s[8], s[9], s[10], s[RND_STR_LEN-4], s[RND_STR_LEN-3]); + + if(cfg->consolidated_store == 1 && stat(zipfilename, &st) == 0){ + snprintf(filename, len-1, "%s %c%c/%s.a%d", zipfilename, s[RND_STR_LEN-2], s[RND_STR_LEN-1], s, attachment_id); + } + else { + snprintf(filename, len-1, "%s/%02x/%c%c%c/%c%c/%c%c/%s.a%d", cfg->queuedir, cfg->server_id, s[8], s[9], s[10], s[RND_STR_LEN-4], s[RND_STR_LEN-3], s[RND_STR_LEN-2], s[RND_STR_LEN-1], s, attachment_id); + + #ifdef HAVE_SUPPORT_FOR_COMPAT_STORAGE_LAYOUT + if(stat(filename, &st)){ + snprintf(filename, len-1, "%s/%02x/%c%c/%c%c/%c%c/%s.a%d", + cfg->queuedir, cfg->server_id, s[RND_STR_LEN-6], s[RND_STR_LEN-5], s[RND_STR_LEN-4], s[RND_STR_LEN-3], s[RND_STR_LEN-2], s[RND_STR_LEN-1], s, attachment_id); + } + #endif + } +} + + int retrieve_email_from_archive(struct session_data *sdata, FILE *dest, struct config *cfg){ int attachments; char *buffer=NULL, *saved_buffer, *p, filename[SMALLBUFSIZE]; struct ptr_array ptr_arr[MAX_ATTACHMENTS]; -#ifdef HAVE_SUPPORT_FOR_COMPAT_STORAGE_LAYOUT - struct stat st; -#endif if(strlen(sdata->ttmpfile) != RND_STR_LEN){ printf("invalid piler-id: %s\n", sdata->ttmpfile); @@ -253,12 +367,8 @@ return 1; } - snprintf(filename, sizeof(filename)-1, "%s/%02x/%c%c%c/%c%c/%c%c/%s.m", cfg->queuedir, cfg->server_id, *(sdata->ttmpfile+8), *(sdata->ttmpfile+9), *(sdata->ttmpfile+10), *(sdata->ttmpfile+RND_STR_LEN-4), *(sdata->ttmpfile+RND_STR_LEN-3), *(sdata->ttmpfile+RND_STR_LEN-2), *(sdata->ttmpfile+RND_STR_LEN-1), sdata->ttmpfile); -#ifdef HAVE_SUPPORT_FOR_COMPAT_STORAGE_LAYOUT - if(stat(filename, &st)){ - snprintf(filename, sizeof(filename)-1, "%s/%02x/%c%c/%c%c/%c%c/%s.m", cfg->queuedir, cfg->server_id, *(sdata->ttmpfile+RND_STR_LEN-6), *(sdata->ttmpfile+RND_STR_LEN-5), *(sdata->ttmpfile+RND_STR_LEN-4), *(sdata->ttmpfile+RND_STR_LEN-3), *(sdata->ttmpfile+RND_STR_LEN-2), *(sdata->ttmpfile+RND_STR_LEN-1), sdata->ttmpfile); - } -#endif + + assemble_filename(filename, sizeof(filename), sdata->ttmpfile, cfg); if(attachments == 0){ retrieve_file_from_archive(filename, WRITE_TO_STDOUT, &buffer, dest, cfg); @@ -280,14 +390,7 @@ buffer = p + strlen(pointer); if(strlen(ptr_arr[i].piler_id) == RND_STR_LEN){ - snprintf(filename, sizeof(filename)-1, "%s/%02x/%c%c%c/%c%c/%c%c/%s.a%d", cfg->queuedir, cfg->server_id, ptr_arr[i].piler_id[8], ptr_arr[i].piler_id[9], ptr_arr[i].piler_id[10], ptr_arr[i].piler_id[RND_STR_LEN-4], ptr_arr[i].piler_id[RND_STR_LEN-3], ptr_arr[i].piler_id[RND_STR_LEN-2], ptr_arr[i].piler_id[RND_STR_LEN-1], ptr_arr[i].piler_id, ptr_arr[i].attachment_id); - - #ifdef HAVE_SUPPORT_FOR_COMPAT_STORAGE_LAYOUT - if(stat(filename, &st)){ - snprintf(filename, sizeof(filename)-1, "%s/%02x/%c%c/%c%c/%c%c/%s.a%d", cfg->queuedir, cfg->server_id, ptr_arr[i].piler_id[RND_STR_LEN-6], ptr_arr[i].piler_id[RND_STR_LEN-5], ptr_arr[i].piler_id[RND_STR_LEN-4], ptr_arr[i].piler_id[RND_STR_LEN-3], ptr_arr[i].piler_id[RND_STR_LEN-2], ptr_arr[i].piler_id[RND_STR_LEN-1], ptr_arr[i].piler_id, ptr_arr[i].attachment_id); - } - #endif - + assemble_attachment_filename(filename, sizeof(filename), ptr_arr[i].piler_id, ptr_arr[i].attachment_id, cfg); retrieve_file_from_archive(filename, WRITE_TO_STDOUT, NULL, dest, cfg); } } diff --git a/src/cfg.c b/src/cfg.c index 0c40b70..8143abf 100644 --- a/src/cfg.c +++ b/src/cfg.c @@ -48,6 +48,7 @@ { "clamd_addr", "string", (void*) string_parser, offsetof(struct config, clamd_addr), "", MAXVAL-1}, { "clamd_port", "integer", (void*) int_parser, offsetof(struct config, clamd_port), "0", sizeof(int)}, { "clamd_socket", "string", (void*) string_parser, offsetof(struct config, clamd_socket), CLAMD_SOCKET, MAXVAL-1}, + { "consolidated_store", "integer", (void*) int_parser, offsetof(struct config, consolidated_store), "0", sizeof(int)}, { "debug", "integer", (void*) int_parser, offsetof(struct config, debug), "0", sizeof(int)}, { "default_retention_days", "integer", (void*) int_parser, offsetof(struct config, default_retention_days), "2557", sizeof(int)}, { "enable_chunking", "integer", (void*) int_parser, offsetof(struct config, enable_chunking), "0", sizeof(int)}, diff --git a/src/cfg.h b/src/cfg.h index 03ea7d1..4019b83 100644 --- a/src/cfg.h +++ b/src/cfg.h @@ -23,6 +23,8 @@ int clamd_port; char clamd_socket[MAXVAL]; + int consolidated_store; + int encrypt_messages; int enable_chunking; diff --git a/src/piler.h b/src/piler.h index e41e233..973c5ca 100644 --- a/src/piler.h +++ b/src/piler.h @@ -54,6 +54,7 @@ void update_counters(struct session_data *sdata, struct data *data, struct counters *counters, struct config *cfg); int retrieve_email_from_archive(struct session_data *sdata, FILE *dest, struct config *cfg); +void assemble_attachment_filename(char *filename, int len, char *s, int attachment_id, struct config *cfg); int file_from_archive_to_network(char *filename, int sd, int tls_enable, struct data *data, struct config *cfg); int get_folder_id(struct session_data *sdata, char *foldername, int parent_id); diff --git a/src/pileraget.c b/src/pileraget.c index 3b780cc..1ced2df 100644 --- a/src/pileraget.c +++ b/src/pileraget.c @@ -46,13 +46,8 @@ return 1; } - snprintf(filename, sizeof(filename)-1, "%s/%02x/%c%c%c/%c%c/%c%c/%s.a%d", cfg.queuedir, cfg.server_id, argv[1][8], argv[1][9], argv[1][10], argv[1][RND_STR_LEN-4], argv[1][RND_STR_LEN-3], argv[1][RND_STR_LEN-2], argv[1][RND_STR_LEN-1], argv[1], atoi(argv[2])); -#ifdef HAVE_SUPPORT_FOR_COMPAT_STORAGE_LAYOUT - if(stat(filename, &st)){ - snprintf(filename, sizeof(filename)-1, "%s/%02x/%c%c/%c%c/%c%c/%s.a%d", cfg.queuedir, cfg.server_id, argv[1][RND_STR_LEN-6], argv[1][RND_STR_LEN-5], argv[1][RND_STR_LEN-4], argv[1][RND_STR_LEN-3], argv[1][RND_STR_LEN-2], argv[1][RND_STR_LEN-1], argv[1], atoi(argv[2])); - } -#endif + assemble_attachment_filename(filename, sizeof(filename), argv[1], atoi(argv[2]), &cfg); retrieve_file_from_archive(filename, WRITE_TO_STDOUT, NULL, stdout, &cfg); return 0; diff --git a/util/Makefile.in b/util/Makefile.in index cf61f40..1a0b51c 100644 --- a/util/Makefile.in +++ b/util/Makefile.in @@ -32,6 +32,7 @@ install: $(INSTALL) -m 0755 $(srcdir)/automated-search.php $(DESTDIR)$(libexecdir)/piler + $(INSTALL) -m 0755 $(srcdir)/compact-store-dir.sh $(DESTDIR)$(libexecdir)/piler $(INSTALL) -m 0755 $(srcdir)/daily-report.php $(DESTDIR)$(libexecdir)/piler $(INSTALL) -m 0755 $(srcdir)/gmail-imap-import.php $(DESTDIR)$(libexecdir)/piler $(INSTALL) -m 0755 $(srcdir)/generate_stats.php $(DESTDIR)$(libexecdir)/piler diff --git a/util/compact-store-dir.sh b/util/compact-store-dir.sh new file mode 100755 index 0000000..e3e7e7c --- /dev/null +++ b/util/compact-store-dir.sh @@ -0,0 +1,30 @@ +#!/bin/bash + +set -o nounset +set -o errexit +set -o pipefail + +SCRIPT_PATH="$(readlink -f "$0")" +SCRIPT_NAME="${SCRIPT_PATH##*/}" + +SERVERID="00" + +usage() { + echo "usage: ${SCRIPT_NAME} " + exit 1 +} + + +[[ $# -gt 0 ]] || usage + +TOPDIR="$1" + +cd "/var/piler/store/${SERVERID}" + +for i in "$TOPDIR"/*; do + pushd "$i" > /dev/null + d="$(basename "$i")" + zip -r -0 -q -m "${TOPDIR}_${d}.zip" . + popd > /dev/null +done +