X-Git-Url: https://code.citadel.org/?a=blobdiff_plain;f=citadel%2Fmodules%2Fpop3client%2Fserv_pop3client.c;h=8a674c77f4df847f429c2ad479490c23ff795c43;hb=0387f48886a9395d89eaca01cd40ab751610426f;hp=3bc0984833e8491e80dc74118105f701ed0e2a43;hpb=e4a11960ed242bc53c90f6fde598bf301ece6f8a;p=citadel.git diff --git a/citadel/modules/pop3client/serv_pop3client.c b/citadel/modules/pop3client/serv_pop3client.c index 3bc098483..8a674c77f 100644 --- a/citadel/modules/pop3client/serv_pop3client.c +++ b/citadel/modules/pop3client/serv_pop3client.c @@ -1,28 +1,23 @@ /* - * $Id$ - * * Consolidate mail from remote POP3 accounts. * - * Copyright (c) 2007-2009 by the citadel.org team - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 3 of the License, or - * (at your option) any later version. + * Copyright (c) 2007-2020 by the citadel.org team * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. + * This program is open source software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License as published + * by the Free Software Foundation; either version 3 of the License, or + * (at your option) any later version. * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. */ #include #include #include +#include #if TIME_WITH_SYS_TIME # include @@ -41,6 +36,7 @@ #include #include #include +#include #include "citadel.h" #include "server.h" #include "citserver.h" @@ -53,271 +49,166 @@ #include "database.h" #include "citadel_dirs.h" -struct pop3aggr { - struct pop3aggr *next; - char roomname[ROOMNAMELEN]; - char pop3host[128]; - char pop3user[128]; - char pop3pass[128]; +struct p3cq { // module-local queue of pop3 client work that needs processing + struct p3cq *next; + char *room; + char *host; + char *user; + char *pass; int keep; - time_t interval; + long interval; }; -struct pop3aggr *palist = NULL; - +static int doing_pop3client = 0; +struct p3cq *p3cq = NULL; -void pop3_do_fetching(char *roomname, char *pop3host, char *pop3user, char *pop3pass, int keep) +/* + * Process one mailbox. + */ +void pop3client_one_mailbox(char *room, const char *host, const char *user, const char *pass, int keep, long interval) { - int sock; - char buf[SIZ]; - int msg_to_fetch = 0; - int *msglist = NULL; - int num_msgs = 0; - int alloc_msgs = 0; + syslog(LOG_DEBUG, "pop3client: room=<%s> host=<%s> user=<%s> keep=<%d> interval=<%ld>", room, host, user, keep, interval); + + char url[SIZ]; + CURL *curl; + CURLcode res = CURLE_OK; + StrBuf *Uidls = NULL; int i; - char *body = NULL; - struct CtdlMessage *msg = NULL; - long msgnum = 0; - char this_uidl[64]; - char utmsgid[SIZ]; - struct cdbdata *cdbut; - struct UseTable ut; - CitContext *CCC=CC; - - CtdlLogPrintf(CTDL_DEBUG, "POP3: %s %s %s \n", roomname, pop3host, pop3user); - CtdlLogPrintf(CTDL_NOTICE, "Connecting to <%s>\n", pop3host); - - if (CtdlThreadCheckStop()) + char cmd[1024]; + + curl = curl_easy_init(); + if (!curl) { return; - - sock = sock_connect(pop3host, "110", "tcp"); - if (sock < 0) { - CtdlLogPrintf(CTDL_ERR, "Could not connect: %s\n", strerror(errno)); + } + + Uidls = NewStrBuf(); + + curl_easy_setopt(curl, CURLOPT_USERNAME, user); + curl_easy_setopt(curl, CURLOPT_PASSWORD, pass); + curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 0L); + curl_easy_setopt(curl, CURLOPT_SSL_VERIFYHOST, 0L); + curl_easy_setopt(curl, CURLOPT_TIMEOUT, 15); + curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, CurlFillStrBuf_callback); // What to do with downloaded data + curl_easy_setopt(curl, CURLOPT_WRITEDATA, Uidls); // Give it our StrBuf to work with + curl_easy_setopt(curl, CURLOPT_CUSTOMREQUEST, "UIDL"); + + /* Try POP3S (SSL encrypted) first */ + snprintf(url, sizeof url, "pop3s://%s", host); + curl_easy_setopt(curl, CURLOPT_URL, url); + res = curl_easy_perform(curl); + if (res == CURLE_OK) { + } else { + syslog(LOG_DEBUG, "pop3client: POP3S connection failed: %s , trying POP3 next", curl_easy_strerror(res)); + snprintf(url, sizeof url, "pop3://%s", host); // try unencrypted next + curl_easy_setopt(curl, CURLOPT_URL, url); + FlushStrBuf(Uidls); + res = curl_easy_perform(curl); + } + + if (res != CURLE_OK) { + syslog(LOG_DEBUG, "pop3client: POP3 connection failed: %s", curl_easy_strerror(res)); + curl_easy_cleanup(curl); + FreeStrBuf(&Uidls); return; } - - if (CtdlThreadCheckStop()) - goto bail; - - CtdlLogPrintf(CTDL_DEBUG, "Connected!\n"); - CCC->sReadBuf = NewStrBuf(); - CCC->sMigrateBuf = NewStrBuf(); - CCC->sPos = NULL; - - /* Read the server greeting */ - if (sock_getln(&sock, buf, sizeof buf) < 0) goto bail; - CtdlLogPrintf(CTDL_DEBUG, ">%s\n", buf); - if (strncasecmp(buf, "+OK", 3)) goto bail; - - if (CtdlThreadCheckStop()) - goto bail; - - /* Identify ourselves. NOTE: we have to append a CR to each command. The LF will - * automatically be appended by sock_puts(). Believe it or not, leaving out the CR - * will cause problems if the server happens to be Exchange, which is so b0rken it - * actually barfs on LF-terminated newlines. - */ - snprintf(buf, sizeof buf, "USER %s\r", pop3user); - CtdlLogPrintf(CTDL_DEBUG, "<%s\n", buf); - if (sock_puts(&sock, buf) <0) goto bail; - if (sock_getln(&sock, buf, sizeof buf) < 0) goto bail; - CtdlLogPrintf(CTDL_DEBUG, ">%s\n", buf); - if (strncasecmp(buf, "+OK", 3)) goto bail; - - if (CtdlThreadCheckStop()) - goto bail; - - /* Password */ - snprintf(buf, sizeof buf, "PASS %s\r", pop3pass); - CtdlLogPrintf(CTDL_DEBUG, "\n"); - if (sock_puts(&sock, buf) <0) goto bail; - if (sock_getln(&sock, buf, sizeof buf) < 0) goto bail; - CtdlLogPrintf(CTDL_DEBUG, ">%s\n", buf); - if (strncasecmp(buf, "+OK", 3)) goto bail; - - if (CtdlThreadCheckStop()) - goto bail; - - /* Get the list of messages */ - snprintf(buf, sizeof buf, "LIST\r"); - CtdlLogPrintf(CTDL_DEBUG, "<%s\n", buf); - if (sock_puts(&sock, buf) <0) goto bail; - if (sock_getln(&sock, buf, sizeof buf) < 0) goto bail; - CtdlLogPrintf(CTDL_DEBUG, ">%s\n", buf); - if (strncasecmp(buf, "+OK", 3)) goto bail; - - if (CtdlThreadCheckStop()) - goto bail; - - do { - if (CtdlThreadCheckStop()) - goto bail; - - if (sock_getln(&sock, buf, sizeof buf) < 0) goto bail; - CtdlLogPrintf(CTDL_DEBUG, ">%s\n", buf); - msg_to_fetch = atoi(buf); - if (msg_to_fetch > 0) { - if (alloc_msgs == 0) { - alloc_msgs = 100; - msglist = malloc((alloc_msgs * (sizeof(int)))); - } - else if (num_msgs >= alloc_msgs) { - alloc_msgs = alloc_msgs * 2; - msglist = realloc(msglist, (alloc_msgs * sizeof(int))); - } - if (msglist == NULL) goto bail; - msglist[num_msgs++] = msg_to_fetch; - } - } while (buf[0] != '.'); - - if (num_msgs) for (i=0; i%s\n", buf); - if (strncasecmp(buf, "+OK", 3)) goto bail; - extract_token(this_uidl, buf, 2, ' ', sizeof this_uidl); - - snprintf(utmsgid, sizeof utmsgid, "pop3/%s/%s@%s", roomname, this_uidl, pop3host); - - if (CtdlThreadCheckStop()) - goto bail; - - cdbut = cdb_fetch(CDB_USETABLE, utmsgid, strlen(utmsgid)); - if (cdbut != NULL) { - /* message has already been seen */ - CtdlLogPrintf(CTDL_DEBUG, "%s has already been seen\n", utmsgid); - cdb_free(cdbut); - - /* rewrite the record anyway, to update the timestamp */ - strcpy(ut.ut_msgid, utmsgid); - ut.ut_timestamp = time(NULL); - cdb_store(CDB_USETABLE, utmsgid, strlen(utmsgid), &ut, sizeof(struct UseTable) ); - } - else { - /* Message has not been seen. Tell the server to fetch the message... */ - snprintf(buf, sizeof buf, "RETR %d\r", msglist[i]); - CtdlLogPrintf(CTDL_DEBUG, "<%s\n", buf); - if (sock_puts(&sock, buf) <0) goto bail; - if (sock_getln(&sock, buf, sizeof buf) < 0) goto bail; - CtdlLogPrintf(CTDL_DEBUG, ">%s\n", buf); - if (strncasecmp(buf, "+OK", 3)) goto bail; - - if (CtdlThreadCheckStop()) - goto bail; - /* If we get to this point, the message is on its way. Read it. */ - body = CtdlReadMessageBody(HKEY("."), config.c_maxmsglen, NULL, 1, &sock); - if (body == NULL) goto bail; - - CtdlLogPrintf(CTDL_DEBUG, "Converting message...\n"); - msg = convert_internet_message(body); - body = NULL; /* yes, this should be dereferenced, NOT freed */ - - /* Do Something With It (tm) */ - msgnum = CtdlSubmitMsg(msg, NULL, roomname, 0); - if (msgnum > 0L) { - /* Message has been committed to the store */ - - if (!keep) { - snprintf(buf, sizeof buf, "DELE %d\r", msglist[i]); - CtdlLogPrintf(CTDL_DEBUG, "<%s\n", buf); - if (sock_puts(&sock, buf) <0) goto bail; - if (sock_getln(&sock, buf, sizeof buf) < 0) goto bail; - CtdlLogPrintf(CTDL_DEBUG, ">%s\n", buf); /* errors here are non-fatal */ + // If we got this far, a connection was established, we know whether it's pop3s or pop3, and UIDL is supported. + // Now go through the UIDL list and look for messages. + + int num_msgs = num_tokens(ChrPtr(Uidls), '\n'); + syslog(LOG_DEBUG, "pop3client: there are %d messages", num_msgs); + for (i=0; i 2) { + if (oneuidl[strlen(oneuidl)-1] == '\r') { + oneuidl[strlen(oneuidl)-1] = 0; + } + int this_msg = atoi(oneuidl); + char *c = strchr(oneuidl, ' '); + if (c) strcpy(oneuidl, ++c); + + // Make up the Use Table record so we can check if we've already seen this message. + StrBuf *UT = NewStrBuf(); + StrBufPrintf(UT, "pop3/%s/%s:%s@%s", room, oneuidl, user, host); + int already_seen = CheckIfAlreadySeen(UT); + FreeStrBuf(&UT); + + // Only fetch the message if we haven't seen it before. + if (already_seen == 0) { + StrBuf *TheMsg = NewStrBuf(); + snprintf(cmd, sizeof cmd, "RETR %d", this_msg); + curl_easy_setopt(curl, CURLOPT_CUSTOMREQUEST, cmd); + curl_easy_setopt(curl, CURLOPT_WRITEDATA, TheMsg); + res = curl_easy_perform(curl); + if (res == CURLE_OK) { + struct CtdlMessage *msg = convert_internet_message_buf(&TheMsg); + CtdlSubmitMsg(msg, NULL, room); + CM_Free(msg); + } + else { + FreeStrBuf(&TheMsg); } - /* write the uidl to the use table so we don't fetch this message again */ - strcpy(ut.ut_msgid, utmsgid); - ut.ut_timestamp = time(NULL); - cdb_store(CDB_USETABLE, utmsgid, strlen(utmsgid), - &ut, sizeof(struct UseTable) ); + // Unless the configuration says to keep the message on the server, delete it. + if (keep == 0) { + snprintf(cmd, sizeof cmd, "DELE %d", this_msg); + curl_easy_setopt(curl, CURLOPT_CUSTOMREQUEST, cmd); + res = curl_easy_perform(curl); + } + } + else { + syslog(LOG_DEBUG, "pop3client: %s has already been retrieved", oneuidl); } - CtdlFreeMessage(msg); } } - /* Log out */ - snprintf(buf, sizeof buf, "QUIT\r"); - CtdlLogPrintf(CTDL_DEBUG, "<%s\n", buf); - if (sock_puts(&sock, buf) <0) goto bail; - if (sock_getln(&sock, buf, sizeof buf) < 0) goto bail; - CtdlLogPrintf(CTDL_DEBUG, ">%s\n", buf); -bail: - FreeStrBuf(&CCC->sReadBuf); - FreeStrBuf(&CCC->sMigrateBuf); - - if (sock != -1) - sock_close(sock); - if (msglist) free(msglist); + curl_easy_cleanup(curl); + FreeStrBuf(&Uidls); + return; } /* * Scan a room's netconfig to determine whether it requires POP3 aggregation */ -void pop3client_scan_room(struct ctdlroom *qrbuf, void *data) +void pop3client_scan_room(struct ctdlroom *qrbuf, void *data, OneRoomNetCfg *OneRNCFG) { - char filename[PATH_MAX]; - char buf[1024]; - char instr[32]; - FILE *fp; - struct pop3aggr *pptr; - - if (CtdlThreadCheckStop()) - return; - - assoc_file_name(filename, sizeof filename, qrbuf, ctdl_netcfg_dir); + const RoomNetCfgLine *pLine; + struct p3cq *pptr = NULL; - /* Only do net processing for rooms that have netconfigs */ - fp = fopen(filename, "r"); - if (fp == NULL) { - return; - } + if (server_shutting_down) return; - while (fgets(buf, sizeof buf, fp) != NULL) { - buf[strlen(buf)-1] = 0; - - extract_token(instr, buf, 0, '|', sizeof instr); - if (!strcasecmp(instr, "pop3client")) { - pptr = (struct pop3aggr *) malloc(sizeof(struct pop3aggr)); - if (pptr != NULL) { - safestrncpy(pptr->roomname, qrbuf->QRname, sizeof pptr->roomname); - extract_token(pptr->pop3host, buf, 1, '|', sizeof pptr->pop3host); - extract_token(pptr->pop3user, buf, 2, '|', sizeof pptr->pop3user); - extract_token(pptr->pop3pass, buf, 3, '|', sizeof pptr->pop3pass); - pptr->keep = extract_int(buf, 4); - pptr->interval = extract_long(buf, 5); - pptr->next = palist; - palist = pptr; - } - } + pLine = OneRNCFG->NetConfigs[pop3client]; + while (pLine != NULL) + { + pptr = malloc(sizeof(struct p3cq)); + pptr->next = p3cq; + p3cq = pptr; + pptr->room = strdup(qrbuf->QRname); + pptr->host = strdup(ChrPtr(pLine->Value[0])); + pptr->user = strdup(ChrPtr(pLine->Value[1])); + pptr->pass = strdup(ChrPtr(pLine->Value[2])); + pptr->keep = atoi(ChrPtr(pLine->Value[3])); + pptr->interval = atol(ChrPtr(pLine->Value[4])); + + pLine = pLine->next; } - - fclose(fp); - } void pop3client_scan(void) { static time_t last_run = 0L; - static int doing_pop3client = 0; - struct pop3aggr *pptr; time_t fastest_scan; - CitContext popclientCC; - - /* Give this thread its own private CitContext */ - CtdlFillSystemContext(&popclientCC, "popclient"); - citthread_setspecific(MyConKey, (void *)&popclientCC ); + struct p3cq *pptr = NULL; - if (config.c_pop3_fastest < config.c_pop3_fetch) - fastest_scan = config.c_pop3_fastest; - else - fastest_scan = config.c_pop3_fetch; + if (CtdlGetConfigLong("c_pop3_fastest") < CtdlGetConfigLong("c_pop3_fetch")) { + fastest_scan = CtdlGetConfigLong("c_pop3_fastest"); + } + else { + fastest_scan = CtdlGetConfigLong("c_pop3_fetch"); + } /* * Run POP3 aggregation no more frequently than once every n seconds @@ -327,31 +218,37 @@ void pop3client_scan(void) { } /* - * This is a simple concurrency check to make sure only one pop3client run - * is done at a time. We could do this with a mutex, but since we + * This is a simple concurrency check to make sure only one pop3client + * run is done at a time. We could do this with a mutex, but since we * don't really require extremely fine granularity here, we'll do it * with a static variable instead. */ if (doing_pop3client) return; doing_pop3client = 1; - CtdlLogPrintf(CTDL_DEBUG, "pop3client started\n"); - CtdlForEachRoom(pop3client_scan_room, NULL); + syslog(LOG_DEBUG, "pop3client: scan started"); + CtdlForEachNetCfgRoom(pop3client_scan_room, NULL); - while (palist != NULL && !CtdlThreadCheckStop()) { - if ((palist->interval && time(NULL) > (last_run + palist->interval)) - || (time(NULL) > last_run + config.c_pop3_fetch)) - pop3_do_fetching(palist->roomname, palist->pop3host, - palist->pop3user, palist->pop3pass, palist->keep); - pptr = palist; - palist = palist->next; + /* + * We have to queue and process in separate phases, otherwise we leave a cursor open + */ + syslog(LOG_DEBUG, "pop3client: processing started"); + while (p3cq != NULL) { + pptr = p3cq; + p3cq = p3cq->next; + + pop3client_one_mailbox(pptr->room, pptr->host, pptr->user, pptr->pass, pptr->keep, pptr->interval); + + free(pptr->room); + free(pptr->host); + free(pptr->user); + free(pptr->pass); free(pptr); } - CtdlLogPrintf(CTDL_DEBUG, "pop3client ended\n"); + syslog(LOG_DEBUG, "pop3client: ended"); last_run = time(NULL); doing_pop3client = 0; - CtdlClearSystemContext(); } @@ -359,9 +256,10 @@ CTDL_MODULE_INIT(pop3client) { if (!threading) { - CtdlRegisterSessionHook(pop3client_scan, EVT_TIMER); + CtdlREGISTERRoomCfgType(pop3client, ParseGeneric, 0, 5, SerializeGeneric, DeleteGenericCfgLine); + CtdlRegisterSessionHook(pop3client_scan, EVT_TIMER, PRIO_AGGR + 50); } - - /* return our Subversion id for the Log */ - return "$Id$"; + + /* return our module id for the log */ + return "pop3client"; }