From a76189488310348b5303679f984f085cc6a72bd0 Mon Sep 17 00:00:00 2001 From: Art Cancro Date: Mon, 29 Oct 2007 16:02:58 +0000 Subject: [PATCH] Found a MUCH better way to store the UIDL's of previously seen remote POP3 messages. The first attempt involved another directory full of crap, lots of reading, writing, and parsing of yet another file format, and a bunch of extra code involving linked lists and string comparisons and after several days it still didn't work. This new version leverages the S_USETABLE database, uses about 50 lines of new code, and I got it working in about 20 minutes. Sometimes you've just got to take a couple of steps back and rethink things. --- citadel/modules/pop3client/serv_pop3client.c | 105 ++++++++++--------- citadel/serv_network.h | 5 - citadel/server.h | 10 ++ 3 files changed, 68 insertions(+), 52 deletions(-) diff --git a/citadel/modules/pop3client/serv_pop3client.c b/citadel/modules/pop3client/serv_pop3client.c index 8421d175d..9d6c5dd0d 100644 --- a/citadel/modules/pop3client/serv_pop3client.c +++ b/citadel/modules/pop3client/serv_pop3client.c @@ -36,6 +36,7 @@ #include "clientsocket.h" #include "msgbase.h" #include "internet_addressing.h" +#include "database.h" #include "citadel_dirs.h" struct pop3aggr { @@ -46,13 +47,9 @@ struct pop3aggr { char pop3pass[128]; }; -struct uidl { - struct uidl *next; - char uidl[64]; -}; - struct pop3aggr *palist = NULL; + void pop3_do_fetching(char *roomname, char *pop3host, char *pop3user, char *pop3pass, int delete_from_server) { int sock; @@ -66,8 +63,9 @@ void pop3_do_fetching(char *roomname, char *pop3host, char *pop3user, char *pop3 struct CtdlMessage *msg = NULL; long msgnum = 0; char this_uidl[64]; - struct uidl *new_uidl_map = NULL; - struct uidl *uptr; + char utmsgid[SIZ]; + struct cdbdata *cdbut; + struct UseTable ut; lprintf(CTDL_DEBUG, "POP3: %s %s %s \n", roomname, pop3host, pop3user); lprintf(CTDL_NOTICE, "Connecting to <%s>\n", pop3host); @@ -139,45 +137,64 @@ void pop3_do_fetching(char *roomname, char *pop3host, char *pop3user, char *pop3 if (sock_getln(sock, buf, sizeof buf) < 0) goto bail; lprintf(CTDL_DEBUG, ">%s\n", buf); if (strncasecmp(buf, "+OK", 3)) goto bail; - extract_token(this_uidl, buf, 3, ' ', sizeof this_uidl); + extract_token(this_uidl, buf, 2, ' ', sizeof this_uidl); - uptr = (struct uidl *) malloc(sizeof(struct uidl)); - if (uptr != NULL) { - safestrncpy(uptr->uidl, this_uidl, sizeof uptr->uidl); - uptr->next = new_uidl_map; - new_uidl_map = uptr; - } + snprintf(utmsgid, sizeof utmsgid, "pop3/%s/%s@%s", pop3user, this_uidl, pop3host); - /* Tell the server to fetch the message */ - snprintf(buf, sizeof buf, "RETR %d\r", msglist[i]); - lprintf(CTDL_DEBUG, "<%s\n", buf); - if (sock_puts(sock, buf) <0) goto bail; - if (sock_getln(sock, buf, sizeof buf) < 0) goto bail; - lprintf(CTDL_DEBUG, ">%s\n", buf); - if (strncasecmp(buf, "+OK", 3)) goto bail; + cdbut = cdb_fetch(CDB_USETABLE, utmsgid, strlen(utmsgid)); + if (cdbut != NULL) { + lprintf(CTDL_DEBUG, "%s has ALREADY BEEN SEEN\n", utmsgid); + /* message has already been seen */ + cdb_free(cdbut); - /* If we get to this point, the message is on its way. Read it. */ - body = CtdlReadMessageBody(".", config.c_maxmsglen, NULL, 1, sock); - if (body == NULL) goto bail; - - lprintf(CTDL_DEBUG, "Converting message...\n"); - msg = convert_internet_message(body); - body = NULL; /* yes, this should be dereferenced, NOT freed */ - - /* Do Something With It (tm) */ - msgnum = CtdlSubmitMsg(msg, NULL, roomname); - if (msgnum > 0L) { - /* Message has been committed to the store */ - - if (delete_from_server) { - snprintf(buf, sizeof buf, "DELE %d\r", msglist[i]); - lprintf(CTDL_DEBUG, "<%s\n", buf); - if (sock_puts(sock, buf) <0) goto bail; - if (sock_getln(sock, buf, sizeof buf) < 0) goto bail; - lprintf(CTDL_DEBUG, ">%s\n", buf); /* errors here are non-fatal */ + /* rewrite the record anyway, to update the timestamp */ + strcpy(ut.ut_msgid, utmsgid); + ut.ut_timestamp = time(NULL); + cdb_store(CDB_USETABLE, utmsgid, strlen(utmsgid), &ut, sizeof(struct UseTable) ); + } + + else { + /* message has not been seen -- fetch it! */ + lprintf(CTDL_DEBUG, "%s has NOT BEEN SEEN\n", utmsgid); + + /* Tell the server to fetch the message */ + snprintf(buf, sizeof buf, "RETR %d\r", msglist[i]); + lprintf(CTDL_DEBUG, "<%s\n", buf); + if (sock_puts(sock, buf) <0) goto bail; + if (sock_getln(sock, buf, sizeof buf) < 0) goto bail; + lprintf(CTDL_DEBUG, ">%s\n", buf); + if (strncasecmp(buf, "+OK", 3)) goto bail; + + /* If we get to this point, the message is on its way. Read it. */ + body = CtdlReadMessageBody(".", config.c_maxmsglen, NULL, 1, sock); + if (body == NULL) goto bail; + + lprintf(CTDL_DEBUG, "Converting message...\n"); + msg = convert_internet_message(body); + body = NULL; /* yes, this should be dereferenced, NOT freed */ + + /* Do Something With It (tm) */ + msgnum = CtdlSubmitMsg(msg, NULL, roomname); + if (msgnum > 0L) { + /* Message has been committed to the store */ + + if (delete_from_server) { + snprintf(buf, sizeof buf, "DELE %d\r", msglist[i]); + lprintf(CTDL_DEBUG, "<%s\n", buf); + if (sock_puts(sock, buf) <0) goto bail; + if (sock_getln(sock, buf, sizeof buf) < 0) goto bail; + lprintf(CTDL_DEBUG, ">%s\n", buf); /* errors here are non-fatal */ + } + + /* write the uidl to the use table so we don't fetch this message again */ + strcpy(ut.ut_msgid, utmsgid); + ut.ut_timestamp = time(NULL); + cdb_store(CDB_USETABLE, utmsgid, strlen(utmsgid), + &ut, sizeof(struct UseTable) ); } + CtdlFreeMessage(msg); + } - CtdlFreeMessage(msg); } /* Log out */ @@ -188,12 +205,6 @@ void pop3_do_fetching(char *roomname, char *pop3host, char *pop3user, char *pop3 lprintf(CTDL_DEBUG, ">%s\n", buf); bail: sock_close(sock); if (msglist) free(msglist); - - while (new_uidl_map != NULL) { - uptr = new_uidl_map->next; - free(new_uidl_map); - new_uidl_map = uptr; - } } diff --git a/citadel/serv_network.h b/citadel/serv_network.h index 92c2c83e4..59902277c 100644 --- a/citadel/serv_network.h +++ b/citadel/serv_network.h @@ -28,11 +28,6 @@ struct NetMap { }; -struct UseTable { - char ut_msgid[SIZ]; - time_t ut_timestamp; -}; - struct FilterList { struct FilterList *next; char fl_user[SIZ]; diff --git a/citadel/server.h b/citadel/server.h index 98f0755f3..49f2c73f4 100644 --- a/citadel/server.h +++ b/citadel/server.h @@ -520,6 +520,16 @@ struct ser_ret { }; +/* + * The S_USETABLE database is used in several modules now, so we define its format here. + */ +struct UseTable { + char ut_msgid[SIZ]; + time_t ut_timestamp; +}; + + + /* Preferred field order */ /* ********** Important fields */ /* *************** Semi-important fields */ -- 2.30.2