2 * Consolidate mail from remote POP3 accounts.
4 * Copyright (c) 2007-2011 by the citadel.org team
6 * This program is open source software; you can redistribute it and/or
7 * modify it under the terms of the GNU General Public License as published
8 * by the Free Software Foundation; either version 3 of the License, or
9 * (at your option) any later version.
11 * This program is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 * GNU General Public License for more details.
16 * You should have received a copy of the GNU General Public License
17 * along with this program; if not, write to the Free Software
18 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
25 #if TIME_WITH_SYS_TIME
26 # include <sys/time.h>
30 # include <sys/time.h>
39 #include <sys/types.h>
41 #include <libcitadel.h>
44 #include "citserver.h"
47 #include "ctdl_module.h"
48 #include "clientsocket.h"
50 #include "internet_addressing.h"
52 #include "citadel_dirs.h"
53 #include "event_client.h"
56 #define POP3C_OK (strncasecmp(ChrPtr(RecvMsg->IO.IOBuf), "+OK", 3) == 0)
58 #define POP3C_DBG_SEND() \
60 "POP3 client[%ld]: > %s\n", \
61 RecvMsg->n, ChrPtr(RecvMsg->IO.SendBuf.Buf))
63 #define POP3C_DBG_READ() \
65 "POP3 client[%ld]: < %s\n", \
67 ChrPtr(RecvMsg->IO.IOBuf))
70 struct CitContext pop3_client_CC;
72 pthread_mutex_t POP3QueueMutex; /* locks the access to the following vars: */
73 HashList *POP3QueueRooms = NULL;
74 HashList *POP3FetchUrls = NULL;
76 typedef struct pop3aggr pop3aggr;
77 typedef eNextState(*Pop3ClientHandler)(pop3aggr* RecvMsg);
79 eNextState POP3_C_Shutdown(AsyncIO *IO);
80 eNextState POP3_C_Timeout(AsyncIO *IO);
81 eNextState POP3_C_ConnFail(AsyncIO *IO);
82 eNextState POP3_C_DNSFail(AsyncIO *IO);
83 eNextState POP3_C_DispatchReadDone(AsyncIO *IO);
84 eNextState POP3_C_DispatchWriteDone(AsyncIO *IO);
85 eNextState POP3_C_Terminate(AsyncIO *IO);
86 eReadState POP3_C_ReadServerStatus(AsyncIO *IO);
87 eNextState POP3_C_ReAttachToFetchMessages(AsyncIO *IO);
89 typedef struct __pop3_room_counter {
94 typedef enum ePOP3_C_States {
100 GetOneMessageIDState,
101 ReadMessageBodyFollowing,
109 typedef struct _FetchItem {
115 struct CtdlMessage *Msg;
118 void HfreeFetchItem(void *vItem)
120 FetchItem *Item = (FetchItem*) vItem;
121 FreeStrBuf(&Item->MsgUIDL);
122 FreeStrBuf(&Item->MsgUID);
131 DNSQueryParts HostLookup;
134 HashList *OtherQRnumbers;
139 StrBuf *RoomName; // TODO: fill me
142 ePOP3_C_States State;
143 HashList *MsgNumbers;
148 void DeletePOP3Aggregator(void *vptr)
150 pop3aggr *ptr = vptr;
151 DeleteHashPos(&ptr->Pos);
152 DeleteHash(&ptr->MsgNumbers);
153 // FreeStrBuf(&ptr->rooms);
154 FreeStrBuf(&ptr->pop3user);
155 FreeStrBuf(&ptr->pop3pass);
156 FreeStrBuf(&ptr->RoomName);
157 FreeURL(&ptr->IO.ConnectMe);
158 FreeStrBuf(&ptr->Url);
159 FreeStrBuf(&ptr->IO.IOBuf);
160 FreeStrBuf(&ptr->IO.SendBuf.Buf);
161 FreeStrBuf(&ptr->IO.RecvBuf.Buf);
162 DeleteAsyncMsg(&ptr->IO.ReadMsg);
163 ((struct CitContext*)ptr->IO.CitContext)->state = CON_IDLE;
164 ((struct CitContext*)ptr->IO.CitContext)->kill_me = 1;
165 FreeAsyncIOContents(&ptr->IO);
169 eNextState FinalizePOP3AggrRun(AsyncIO *IO)
172 pop3aggr *cptr = (pop3aggr *)IO->Data;
174 syslog(LOG_DEBUG, "Terminating Aggregator; bye.\n");
176 It = GetNewHashPos(POP3FetchUrls, 0);
177 pthread_mutex_lock(&POP3QueueMutex);
179 if (GetHashPosFromKey(POP3FetchUrls, SKEY(cptr->Url), It))
180 DeleteEntryFromHash(POP3FetchUrls, It);
182 pthread_mutex_unlock(&POP3QueueMutex);
187 eNextState FailAggregationRun(AsyncIO *IO)
192 eNextState POP3C_ReadGreeting(pop3aggr *RecvMsg)
195 /* Read the server greeting */
196 if (!POP3C_OK) return eTerminateConnection;
197 else return eSendReply;
200 eNextState POP3C_SendUser(pop3aggr *RecvMsg)
202 /* Identify ourselves. NOTE: we have to append a CR to each command.
203 * The LF will automatically be appended by sock_puts(). Believe it
204 * or not, leaving out the CR will cause problems if the server happens
205 * to be Exchange, which is so b0rken it actually barfs on
206 * LF-terminated newlines.
208 StrBufPrintf(RecvMsg->IO.SendBuf.Buf,
209 "USER %s\r\n", ChrPtr(RecvMsg->pop3user));
214 eNextState POP3C_GetUserState(pop3aggr *RecvMsg)
217 if (!POP3C_OK) return eTerminateConnection;
218 else return eSendReply;
221 eNextState POP3C_SendPassword(pop3aggr *RecvMsg)
224 StrBufPrintf(RecvMsg->IO.SendBuf.Buf,
225 "PASS %s\r\n", ChrPtr(RecvMsg->pop3pass));
226 syslog(LOG_DEBUG, "<PASS <password>\n");
227 // POP3C_DBG_SEND(); No, we won't write the passvoid to syslog...
231 eNextState POP3C_GetPassState(pop3aggr *RecvMsg)
234 if (!POP3C_OK) return eTerminateConnection;
235 else return eSendReply;
238 eNextState POP3C_SendListCommand(pop3aggr *RecvMsg)
240 /* Get the list of messages */
241 StrBufPlain(RecvMsg->IO.SendBuf.Buf, HKEY("LIST\r\n"));
246 eNextState POP3C_GetListCommandState(pop3aggr *RecvMsg)
249 if (!POP3C_OK) return eTerminateConnection;
250 RecvMsg->MsgNumbers = NewHash(1, NULL);
256 eNextState POP3C_GetListOneLine(pop3aggr *RecvMsg)
262 FetchItem *OneMsg = NULL;
265 if ((StrLength(RecvMsg->IO.IOBuf) == 1) &&
266 (ChrPtr(RecvMsg->IO.IOBuf)[0] == '.'))
268 if (GetCount(RecvMsg->MsgNumbers) == 0)
270 //// RecvMsg->Sate = ReadQuitState;
274 RecvMsg->Pos = GetNewHashPos(RecvMsg->MsgNumbers, 0);
279 OneMsg = (FetchItem*) malloc(sizeof(FetchItem));
280 memset(OneMsg, 0, sizeof(FetchItem));
281 OneMsg->MSGID = atol(ChrPtr(RecvMsg->IO.IOBuf));
283 pch = strchr(ChrPtr(RecvMsg->IO.IOBuf), ' ');
286 OneMsg->MSGSize = atol(pch + 1);
289 rc = TestValidateHash(RecvMsg->MsgNumbers);
291 syslog(LOG_DEBUG, "Hash Invalid: %d\n", rc);
294 Put(RecvMsg->MsgNumbers, LKEY(OneMsg->MSGID), OneMsg, HfreeFetchItem);
296 rc = TestValidateHash(RecvMsg->MsgNumbers);
298 syslog(LOG_DEBUG, "Hash Invalid: %d\n", rc);
300 //RecvMsg->State --; /* read next Line */
304 eNextState POP3_FetchNetworkUsetableEntry(AsyncIO *IO)
309 struct cdbdata *cdbut;
310 pop3aggr *RecvMsg = (pop3aggr *) IO->Data;
312 if(GetNextHashPos(RecvMsg->MsgNumbers,
319 if (server_shutting_down)
322 RecvMsg->CurrMsg = (FetchItem*) vData;
324 "CHECKING: whether %s has already been seen: ",
325 ChrPtr(RecvMsg->CurrMsg->MsgUID));
327 /* Find out if we've already seen this item */
328 safestrncpy(ut.ut_msgid,
329 ChrPtr(RecvMsg->CurrMsg->MsgUID),
330 sizeof(ut.ut_msgid));
331 ut.ut_timestamp = time(NULL);/// TODO: libev timestamp!
333 cdbut = cdb_fetch(CDB_USETABLE, SKEY(RecvMsg->CurrMsg->MsgUID));
335 /* Item has already been seen */
336 syslog(LOG_DEBUG, "YES\n");
339 /* rewrite the record anyway, to update the timestamp */
340 cdb_store(CDB_USETABLE,
341 SKEY(RecvMsg->CurrMsg->MsgUID),
342 &ut, sizeof(struct UseTable) );
343 RecvMsg->CurrMsg->NeedFetch = 0; ////TODO0;
347 syslog(LOG_DEBUG, "NO\n");
348 RecvMsg->CurrMsg->NeedFetch = 1;
350 return NextDBOperation(&RecvMsg->IO,
351 POP3_FetchNetworkUsetableEntry);
355 /* ok, now we know them all,
356 * continue with reading the actual messages. */
357 DeleteHashPos(&RecvMsg->Pos);
359 return QueueEventContext(IO, POP3_C_ReAttachToFetchMessages);
363 eNextState POP3C_GetOneMessagID(pop3aggr *RecvMsg)
371 rc = TestValidateHash(RecvMsg->MsgNumbers);
373 syslog(LOG_DEBUG, "Hash Invalid: %d\n", rc);
375 if(GetNextHashPos(RecvMsg->MsgNumbers,
380 RecvMsg->CurrMsg = (FetchItem*) vData;
381 /* Find out the UIDL of the message,
382 * to determine whether we've already downloaded it */
383 StrBufPrintf(RecvMsg->IO.SendBuf.Buf,
384 "UIDL %ld\r\n", RecvMsg->CurrMsg->MSGID);
390 DeleteHashPos(&RecvMsg->Pos);
391 /// done receiving uidls.. start looking them up now.
392 RecvMsg->Pos = GetNewHashPos(RecvMsg->MsgNumbers, 0);
393 return QueueDBOperation(&RecvMsg->IO,
394 POP3_FetchNetworkUsetableEntry);
396 return eReadMore; /* TODO */
399 eNextState POP3C_GetOneMessageIDState(pop3aggr *RecvMsg)
403 rc = TestValidateHash(RecvMsg->MsgNumbers);
405 syslog(LOG_DEBUG, "Hash Invalid: %d\n", rc);
409 if (!POP3C_OK) return eTerminateConnection;
410 RecvMsg->CurrMsg->MsgUIDL =
411 NewStrBufPlain(NULL, StrLength(RecvMsg->IO.IOBuf));
412 RecvMsg->CurrMsg->MsgUID =
413 NewStrBufPlain(NULL, StrLength(RecvMsg->IO.IOBuf) * 2);
415 StrBufExtract_token(RecvMsg->CurrMsg->MsgUIDL,
416 RecvMsg->IO.IOBuf, 2, ' ');
418 StrBufPrintf(RecvMsg->CurrMsg->MsgUID,
420 ChrPtr(RecvMsg->RoomName),
421 ChrPtr(RecvMsg->CurrMsg->MsgUIDL),
422 RecvMsg->IO.ConnectMe->User,
423 RecvMsg->IO.ConnectMe->Host);
429 eNextState POP3C_SendGetOneMsg(pop3aggr *RecvMsg)
435 RecvMsg->CurrMsg = NULL;
436 while (GetNextHashPos(RecvMsg->MsgNumbers,
440 (RecvMsg->CurrMsg = (FetchItem*) vData,
441 RecvMsg->CurrMsg->NeedFetch == 0))
444 if ((RecvMsg->CurrMsg != NULL ) && (RecvMsg->CurrMsg->NeedFetch == 1))
446 /* Message has not been seen.
447 * Tell the server to fetch the message... */
448 StrBufPrintf(RecvMsg->IO.SendBuf.Buf,
449 "RETR %ld\r\n", RecvMsg->CurrMsg->MSGID);
454 RecvMsg->State = ReadQuitState;
455 return POP3_C_DispatchWriteDone(&RecvMsg->IO);
460 eNextState POP3C_ReadMessageBodyFollowing(pop3aggr *RecvMsg)
463 if (!POP3C_OK) return eTerminateConnection;
464 RecvMsg->IO.ReadMsg = NewAsyncMsg(HKEY("."),
465 RecvMsg->CurrMsg->MSGSize,
474 eNextState POP3C_StoreMsgRead(AsyncIO *IO)
476 pop3aggr *RecvMsg = (pop3aggr *) IO->Data;
480 "MARKING: %s as seen: ",
481 ChrPtr(RecvMsg->CurrMsg->MsgUID));
483 safestrncpy(ut.ut_msgid,
484 ChrPtr(RecvMsg->CurrMsg->MsgUID),
485 sizeof(ut.ut_msgid));
486 ut.ut_timestamp = time(NULL); /* TODO: use libev time */
487 cdb_store(CDB_USETABLE,
488 ChrPtr(RecvMsg->CurrMsg->MsgUID),
489 StrLength(RecvMsg->CurrMsg->MsgUID),
491 sizeof(struct UseTable) );
493 return QueueEventContext(&RecvMsg->IO, POP3_C_ReAttachToFetchMessages);
495 eNextState POP3C_SaveMsg(AsyncIO *IO)
498 pop3aggr *RecvMsg = (pop3aggr *) IO->Data;
500 /* Do Something With It (tm) */
501 msgnum = CtdlSubmitMsg(RecvMsg->CurrMsg->Msg,
503 ChrPtr(RecvMsg->RoomName),
507 /* Message has been committed to the store
508 * write the uidl to the use table
509 * so we don't fetch this message again
512 CtdlFreeMessage(RecvMsg->CurrMsg->Msg);
514 return NextDBOperation(&RecvMsg->IO, POP3C_StoreMsgRead);
517 eNextState POP3C_ReadMessageBody(pop3aggr *RecvMsg)
519 syslog(LOG_DEBUG, "Converting message...\n");
520 RecvMsg->CurrMsg->Msg =
521 convert_internet_message_buf(&RecvMsg->IO.ReadMsg->MsgBuf);
523 return QueueDBOperation(&RecvMsg->IO, POP3C_SaveMsg);
526 eNextState POP3C_SendDelete(pop3aggr *RecvMsg)
528 if (!RecvMsg->keep) {
529 StrBufPrintf(RecvMsg->IO.SendBuf.Buf,
530 "DELE %ld\r\n", RecvMsg->CurrMsg->MSGID);
535 RecvMsg->State = ReadMessageBodyFollowing;
536 return POP3_C_DispatchWriteDone(&RecvMsg->IO);
539 eNextState POP3C_ReadDeleteState(pop3aggr *RecvMsg)
542 RecvMsg->State = GetOneMessageIDState;
546 eNextState POP3C_SendQuit(pop3aggr *RecvMsg)
549 StrBufPlain(RecvMsg->IO.SendBuf.Buf,
556 eNextState POP3C_ReadQuitState(pop3aggr *RecvMsg)
559 return eTerminateConnection;
562 const long POP3_C_ConnTimeout = 1000;
563 const long DefaultPOP3Port = 110;
565 Pop3ClientHandler POP3C_ReadHandlers[] = {
569 POP3C_GetListCommandState,
570 POP3C_GetListOneLine,
571 POP3C_GetOneMessageIDState,
572 POP3C_ReadMessageBodyFollowing,
573 POP3C_ReadMessageBody,
574 POP3C_ReadDeleteState,
578 const long POP3_C_SendTimeouts[POP3C_MaxRead] = {
588 const ConstStr POP3C_ReadErrors[POP3C_MaxRead] = {
589 {HKEY("Connection broken during ")},
590 {HKEY("Connection broken during ")},
591 {HKEY("Connection broken during ")},
592 {HKEY("Connection broken during ")},
593 {HKEY("Connection broken during ")},
594 {HKEY("Connection broken during ")},
595 {HKEY("Connection broken during ")},
596 {HKEY("Connection broken during ")}
599 Pop3ClientHandler POP3C_SendHandlers[] = {
600 NULL, /* we don't send a greeting */
603 POP3C_SendListCommand,
605 POP3C_GetOneMessagID,
612 const long POP3_C_ReadTimeouts[] = {
624 /*****************************************************************************/
625 /* POP3 CLIENT DISPATCHER */
626 /*****************************************************************************/
628 void POP3SetTimeout(eNextState NextTCPState, pop3aggr *pMsg)
630 double Timeout = 0.0;
632 syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
634 switch (NextTCPState) {
638 Timeout = POP3_C_SendTimeouts[pMsg->State];
640 if (pMsg->State == eDATABody) {
641 / * if we're sending a huge message, we need more time. * /
642 Timeout += StrLength(pMsg->msgtext) / 1024;
648 Timeout = POP3_C_ReadTimeouts[pMsg->State];
650 if (pMsg->State == eDATATerminateBody) {
652 * some mailservers take a nap before accepting the message
653 * content inspection and such.
655 Timeout += StrLength(pMsg->msgtext) / 1024;
666 case eTerminateConnection:
669 case eReadMore://// TODO
672 SetNextTimeout(&pMsg->IO, Timeout);
674 eNextState POP3_C_DispatchReadDone(AsyncIO *IO)
676 syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
677 pop3aggr *pMsg = IO->Data;
680 rc = POP3C_ReadHandlers[pMsg->State](pMsg);
683 POP3SetTimeout(rc, pMsg);
686 eNextState POP3_C_DispatchWriteDone(AsyncIO *IO)
688 syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
689 pop3aggr *pMsg = IO->Data;
692 rc = POP3C_SendHandlers[pMsg->State](pMsg);
693 POP3SetTimeout(rc, pMsg);
698 /*****************************************************************************/
699 /* POP3 CLIENT ERROR CATCHERS */
700 /*****************************************************************************/
701 eNextState POP3_C_Terminate(AsyncIO *IO)
703 /// pop3aggr *pMsg = (pop3aggr *)IO->Data;
705 syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
706 FinalizePOP3AggrRun(IO);
709 eNextState POP3_C_Timeout(AsyncIO *IO)
711 pop3aggr *pMsg = IO->Data;
713 syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
714 StrBufPlain(IO->ErrMsg, CKEY(POP3C_ReadErrors[pMsg->State]));
715 return FailAggregationRun(IO);
717 eNextState POP3_C_ConnFail(AsyncIO *IO)
719 pop3aggr *pMsg = (pop3aggr *)IO->Data;
721 syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
722 StrBufPlain(IO->ErrMsg, CKEY(POP3C_ReadErrors[pMsg->State]));
723 return FailAggregationRun(IO);
725 eNextState POP3_C_DNSFail(AsyncIO *IO)
727 pop3aggr *pMsg = (pop3aggr *)IO->Data;
729 syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
730 StrBufPlain(IO->ErrMsg, CKEY(POP3C_ReadErrors[pMsg->State]));
731 return FailAggregationRun(IO);
733 eNextState POP3_C_Shutdown(AsyncIO *IO)
735 syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
736 //// pop3aggr *pMsg = IO->Data;
738 ////pMsg->MyQEntry->Status = 3;
739 ///StrBufPlain(pMsg->MyQEntry->StatusMessage, HKEY("server shutdown during message retrieval."));
740 FinalizePOP3AggrRun(IO);
746 * @brief lineread Handler; understands when to read more POP3 lines,
747 * and when this is a one-lined reply.
749 eReadState POP3_C_ReadServerStatus(AsyncIO *IO)
751 eReadState Finished = eBufferNotEmpty;
753 switch (IO->NextState) {
758 case eTerminateConnection:
760 Finished = eReadFail;
767 Finished = StrBufChunkSipLine(IO->IOBuf, &IO->RecvBuf);
771 Finished = CtdlReadMessageBodyAsync(IO);
777 /*****************************************************************************
778 * So we connect our Server IP here. *
779 *****************************************************************************/
780 eNextState POP3_C_ReAttachToFetchMessages(AsyncIO *IO)
782 pop3aggr *cpptr = IO->Data;
784 syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
785 ////??? cpptr->State ++;
786 if (cpptr->Pos == NULL)
787 cpptr->Pos = GetNewHashPos(cpptr->MsgNumbers, 0);
789 POP3_C_DispatchWriteDone(IO);
790 ReAttachIO(IO, cpptr, 0);
791 IO->NextState = eReadMessage;
792 return IO->NextState;
795 eNextState pop3_connect_ip(AsyncIO *IO)
797 syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
799 return EvConnectSock(IO,
801 POP3_C_ReadTimeouts[0],
805 eNextState pop3_get_one_host_ip_done(AsyncIO *IO)
807 pop3aggr *cpptr = IO->Data;
808 struct hostent *hostent;
812 hostent = cpptr->HostLookup.VParsedDNSReply;
813 if ((cpptr->HostLookup.DNSStatus == ARES_SUCCESS) &&
814 (hostent != NULL) ) {
815 memset(&cpptr->IO.ConnectMe->Addr, 0, sizeof(struct in6_addr));
816 if (cpptr->IO.ConnectMe->IPv6) {
817 memcpy(&cpptr->IO.ConnectMe->Addr.sin6_addr.s6_addr,
818 &hostent->h_addr_list[0],
819 sizeof(struct in6_addr));
821 cpptr->IO.ConnectMe->Addr.sin6_family =
823 cpptr->IO.ConnectMe->Addr.sin6_port =
824 htons(DefaultPOP3Port);
827 struct sockaddr_in *addr =
828 (struct sockaddr_in*)
829 &cpptr->IO.ConnectMe->Addr;
831 memcpy(&addr->sin_addr.s_addr,
832 hostent->h_addr_list[0],
835 addr->sin_family = hostent->h_addrtype;
836 addr->sin_port = htons(DefaultPOP3Port);
838 return pop3_connect_ip(IO);
844 eNextState pop3_get_one_host_ip(AsyncIO *IO)
846 pop3aggr *cpptr = IO->Data;
848 * here we start with the lookup of one host. it might be...
849 * - the relay host *sigh*
850 * - the direct hostname if there was no mx record
856 syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
859 "POP3 client[%ld]: looking up %s-Record %s : %d ...\n",
861 (cpptr->IO.ConnectMe->IPv6)? "aaaa": "a",
862 cpptr->IO.ConnectMe->Host,
863 cpptr->IO.ConnectMe->Port);
865 QueueQuery((cpptr->IO.ConnectMe->IPv6)? ns_t_aaaa : ns_t_a,
866 cpptr->IO.ConnectMe->Host,
869 pop3_get_one_host_ip_done);
870 IO->NextState = eReadDNSReply;
871 return IO->NextState;
876 int pop3_do_fetching(pop3aggr *cpptr)
878 InitIOStruct(&cpptr->IO,
881 POP3_C_ReadServerStatus,
883 POP3_C_DispatchWriteDone,
884 POP3_C_DispatchReadDone,
890 safestrncpy(((CitContext *)cpptr->IO.CitContext)->cs_host,
892 sizeof(((CitContext *)cpptr->IO.CitContext)->cs_host));
894 if (cpptr->IO.ConnectMe->IsIP) {
895 QueueEventContext(&cpptr->IO,
899 QueueEventContext(&cpptr->IO,
900 pop3_get_one_host_ip);
906 * Scan a room's netconfig to determine whether it requires POP3 aggregation
908 void pop3client_scan_room(struct ctdlroom *qrbuf, void *data)
915 char filename[PATH_MAX];
919 const char *CfgPtr, *lPtr;
922 // pop3_room_counter *Count = NULL;
925 pthread_mutex_lock(&POP3QueueMutex);
926 if (GetHash(POP3QueueRooms, LKEY(qrbuf->QRnumber), &vptr))
929 "pop3client: [%ld] %s already in progress.\n",
932 pthread_mutex_unlock(&POP3QueueMutex);
934 pthread_mutex_unlock(&POP3QueueMutex);
936 if (server_shutting_down) return;
938 assoc_file_name(filename, sizeof filename, qrbuf, ctdl_netcfg_dir);
940 if (server_shutting_down)
943 /* Only do net processing for rooms that have netconfigs */
944 fd = open(filename, 0);
948 if (server_shutting_down)
950 if (fstat(fd, &statbuf) == -1) {
952 "ERROR: could not stat configfile '%s' - %s\n",
957 if (server_shutting_down)
959 CfgData = NewStrBufPlain(NULL, statbuf.st_size + 1);
960 if (StrBufReadBLOB(CfgData, &fd, 1, statbuf.st_size, &Err) < 0) {
962 FreeStrBuf(&CfgData);
963 syslog(LOG_DEBUG, "ERROR: reading config '%s' - %s<br>\n",
964 filename, strerror(errno));
968 if (server_shutting_down)
972 CfgType = NewStrBuf();
973 Line = NewStrBufPlain(NULL, StrLength(CfgData));
978 Done = StrBufSipLine(Line, CfgData, &CfgPtr) == 0;
979 if (StrLength(Line) > 0)
982 StrBufExtract_NextToken(CfgType, Line, &lPtr, '|');
983 if (!strcasecmp("pop3client", ChrPtr(CfgType)))
990 Count = malloc(sizeof(pop3_room_counter));
995 cptr = (pop3aggr *) malloc(sizeof(pop3aggr));
996 memset(cptr, 0, sizeof(pop3aggr));
997 ///TODO do we need this? cptr->roomlist_parts=1;
999 NewStrBufPlain(qrbuf->QRname, -1);
1001 NewStrBufPlain(NULL, StrLength(Line));
1003 NewStrBufPlain(NULL, StrLength(Line));
1004 cptr->Url = NewStrBuf();
1007 StrBufExtract_NextToken(Tmp, Line, &lPtr, '|');
1008 StrBufExtract_NextToken(cptr->pop3user,
1013 StrBufExtract_NextToken(cptr->pop3pass,
1018 cptr->keep = StrBufExtractNext_long(Line,
1022 cptr->interval = StrBufExtractNext_long(Line,
1026 StrBufPrintf(cptr->Url, "pop3://%s:%s@%s/%s",
1027 ChrPtr(cptr->pop3user),
1028 ChrPtr(cptr->pop3pass),
1030 ChrPtr(cptr->RoomName));
1032 ParseURL(&cptr->IO.ConnectMe, cptr->Url, 110);
1036 /* todo: we need to reunite the url to be shure. */
1038 pthread_mutex_lock(&POP3ueueMutex);
1039 GetHash(POP3FetchUrls, SKEY(ptr->Url), &vptr);
1040 use_this_cptr = (pop3aggr *)vptr;
1042 if (use_this_rncptr != NULL)
1044 /* mustn't attach to an active session */
1045 if (use_this_cptr->RefCount > 0)
1047 DeletePOP3Cfg(cptr);
1053 StrBufAppendBufPlain(
1054 use_this_cptr->rooms,
1057 if (use_this_cptr->roomlist_parts == 1)
1059 use_this_cptr->OtherQRnumbers
1060 = NewHash(1, lFlathash);
1062 QRnumber = (long*)malloc(sizeof(long));
1063 *QRnumber = qrbuf->QRnumber;
1064 Put(use_this_cptr->OtherQRnumbers,
1065 LKEY(qrbuf->QRnumber),
1069 use_this_cptr->roomlist_parts++;
1071 pthread_mutex_unlock(&POP3QueueMutex);
1074 pthread_mutex_unlock(&RSSQueueMutex);
1077 pthread_mutex_lock(&POP3QueueMutex);
1081 DeletePOP3Aggregator);
1083 pthread_mutex_unlock(&POP3QueueMutex);
1093 FreeStrBuf(&CfgType);
1094 FreeStrBuf(&CfgData);
1097 static int doing_pop3client = 0;
1099 void pop3client_scan(void) {
1100 static time_t last_run = 0L;
1101 time_t fastest_scan;
1108 become_session(&pop3_client_CC);
1110 if (config.c_pop3_fastest < config.c_pop3_fetch)
1111 fastest_scan = config.c_pop3_fastest;
1113 fastest_scan = config.c_pop3_fetch;
1116 * Run POP3 aggregation no more frequently than once every n seconds
1118 if ( (time(NULL) - last_run) < fastest_scan ) {
1123 * This is a simple concurrency check to make sure only one pop3client
1124 * run is done at a time. We could do this with a mutex, but since we
1125 * don't really require extremely fine granularity here, we'll do it
1126 * with a static variable instead.
1128 if (doing_pop3client) return;
1129 doing_pop3client = 1;
1131 syslog(LOG_DEBUG, "pop3client started");
1132 CtdlForEachRoom(pop3client_scan_room, NULL);
1134 pthread_mutex_lock(&POP3QueueMutex);
1135 it = GetNewHashPos(POP3FetchUrls, 0);
1136 while (!server_shutting_down &&
1137 GetNextHashPos(POP3FetchUrls, it, &len, &Key, &vrptr) &&
1139 cptr = (pop3aggr *)vrptr;
1140 if (cptr->RefCount == 0)
1141 if (!pop3_do_fetching(cptr))
1142 DeletePOP3Aggregator(cptr);////TODO
1145 if ((palist->interval && time(NULL) > (last_run + palist->interval))
1146 || (time(NULL) > last_run + config.c_pop3_fetch))
1147 pop3_do_fetching(palist->roomname, palist->pop3host,
1148 palist->pop3user, palist->pop3pass, palist->keep);
1150 palist = palist->next;
1155 pthread_mutex_unlock(&POP3QueueMutex);
1157 syslog(LOG_DEBUG, "pop3client ended");
1158 last_run = time(NULL);
1159 doing_pop3client = 0;
1163 void pop3_cleanup(void)
1165 /* citthread_mutex_destroy(&POP3QueueMutex); TODO */
1166 while (doing_pop3client != 0) ;
1167 DeleteHash(&POP3FetchUrls);
1168 DeleteHash(&POP3QueueRooms);
1171 CTDL_MODULE_INIT(pop3client)
1175 CtdlFillSystemContext(&pop3_client_CC, "POP3aggr");
1176 pthread_mutex_init(&POP3QueueMutex, NULL);
1177 POP3QueueRooms = NewHash(1, lFlathash);
1178 POP3FetchUrls = NewHash(1, NULL);
1179 CtdlRegisterSessionHook(pop3client_scan, EVT_TIMER);
1180 CtdlRegisterCleanupHook(pop3_cleanup);
1183 /* return our module id for the log */
1184 return "pop3client";