2 * Consolidate mail from remote POP3 accounts.
4 * Copyright (c) 2007-2011 by the citadel.org team
6 * This program is open source software; you can redistribute it and/or
7 * modify it under the terms of the GNU General Public License as published
8 * by the Free Software Foundation; either version 3 of the License, or
9 * (at your option) any later version.
11 * This program is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 * GNU General Public License for more details.
16 * You should have received a copy of the GNU General Public License
17 * along with this program; if not, write to the Free Software
18 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
25 #if TIME_WITH_SYS_TIME
26 # include <sys/time.h>
30 # include <sys/time.h>
39 #include <sys/types.h>
41 #include <libcitadel.h>
44 #include "citserver.h"
47 #include "ctdl_module.h"
48 #include "clientsocket.h"
50 #include "internet_addressing.h"
52 #include "citadel_dirs.h"
53 #include "event_client.h"
56 #define POP3C_OK (strncasecmp(ChrPtr(RecvMsg->IO.IOBuf), "+OK", 3) == 0)
58 int POP3ClientDebugEnabled = 0;
60 #define N ((pop3aggr*)IO->Data)->n
62 #define DBGLOG(LEVEL) if ((LEVEL != LOG_DEBUG) || (POP3ClientDebugEnabled != 0))
64 #define EVP3C_syslog(LEVEL, FORMAT, ...) \
65 DBGLOG(LEVEL) syslog(LEVEL, \
66 "%s[%ld]CC[%d][%ld]POP3: " FORMAT, \
67 IOSTR, IO->ID, CCID, N, __VA_ARGS__)
69 #define EVP3CM_syslog(LEVEL, FORMAT) \
70 DBGLOG(LEVEL) syslog(LEVEL, \
71 "%s[%ld]CC[%d][%ld]POP3: " FORMAT, \
72 IOSTR, IO->ID, CCID, N)
74 #define EVP3CQ_syslog(LEVEL, FORMAT, ...) \
75 DBGLOG(LEVEL) syslog(LEVEL, \
79 #define EVP3CQM_syslog(LEVEL, FORMAT) \
80 DBGLOG(LEVEL) syslog(LEVEL, \
84 #define EVP3CCS_syslog(LEVEL, FORMAT, ...) \
85 DBGLOG(LEVEL) syslog(LEVEL, "%s[%ld][%ld]POP3: " FORMAT, \
86 IOSTR, IO->ID, N, __VA_ARGS__)
88 #define EVP3CCSM_syslog(LEVEL, FORMAT) \
89 DBGLOG(LEVEL) syslog(LEVEL, "%s[%ld][%ld]POP3: " FORMAT, \
92 #define POP3C_DBG_SEND() \
93 EVP3C_syslog(LOG_DEBUG, \
94 "%s[%ld]CC[%d][%ld]POP3: > %s\n", \
95 IOSTR, IO->ID, CCID, N, \
96 ChrPtr(RecvMsg->IO.SendBuf.Buf))
98 #define POP3C_DBG_READ() \
99 EVP3C_syslog(LOG_DEBUG, \
100 "%s[%ld]CC[%d][%ld]POP3: < %s\n", \
101 IOSTR, IO->ID, CCID, N, \
102 ChrPtr(RecvMsg->IO.IOBuf))
105 struct CitContext pop3_client_CC;
107 pthread_mutex_t POP3QueueMutex; /* locks the access to the following vars: */
108 HashList *POP3QueueRooms = NULL;
109 HashList *POP3FetchUrls = NULL;
111 typedef struct pop3aggr pop3aggr;
112 typedef eNextState(*Pop3ClientHandler)(pop3aggr* RecvMsg);
114 eNextState POP3_C_Shutdown(AsyncIO *IO);
115 eNextState POP3_C_Timeout(AsyncIO *IO);
116 eNextState POP3_C_ConnFail(AsyncIO *IO);
117 eNextState POP3_C_DNSFail(AsyncIO *IO);
118 eNextState POP3_C_DispatchReadDone(AsyncIO *IO);
119 eNextState POP3_C_DispatchWriteDone(AsyncIO *IO);
120 eNextState POP3_C_Terminate(AsyncIO *IO);
121 eReadState POP3_C_ReadServerStatus(AsyncIO *IO);
122 eNextState POP3_C_ReAttachToFetchMessages(AsyncIO *IO);
124 typedef struct __pop3_room_counter {
129 typedef enum ePOP3_C_States {
135 GetOneMessageIDState,
136 ReadMessageBodyFollowing,
144 typedef struct _FetchItem {
150 struct CtdlMessage *Msg;
153 void HfreeFetchItem(void *vItem)
155 FetchItem *Item = (FetchItem*) vItem;
156 FreeStrBuf(&Item->MsgUIDL);
157 FreeStrBuf(&Item->MsgUID);
163 typedef enum _POP3State {
177 ConstStr POP3States[] = {
178 {HKEY("Aggregator created")},
179 {HKEY("Reading Greeting")},
180 {HKEY("Sending User")},
181 {HKEY("Sending Password")},
183 {HKEY("Fetching Usetable")},
184 {HKEY("Get MSG ID")},
185 {HKEY("Get Message")},
187 {HKEY("Delete Upstream")},
191 static void SetPOP3State(AsyncIO *IO, POP3State State)
193 CitContext* CCC = IO->CitContext;
195 memcpy(CCC->cs_clientname, POP3States[State].Key, POP3States[State].len + 1);
206 DNSQueryParts HostLookup;
209 HashList *OtherQRnumbers;
215 StrBuf *RoomName; // TODO: fill me
218 ePOP3_C_States State;
219 HashList *MsgNumbers;
224 void DeletePOP3Aggregator(void *vptr)
226 pop3aggr *ptr = vptr;
227 DeleteHashPos(&ptr->Pos);
228 DeleteHash(&ptr->MsgNumbers);
229 // FreeStrBuf(&ptr->rooms);
230 FreeStrBuf(&ptr->pop3user);
231 FreeStrBuf(&ptr->pop3pass);
232 FreeStrBuf(&ptr->Host);
233 FreeStrBuf(&ptr->RoomName);
234 FreeURL(&ptr->IO.ConnectMe);
235 FreeStrBuf(&ptr->Url);
236 FreeStrBuf(&ptr->IO.IOBuf);
237 FreeStrBuf(&ptr->IO.SendBuf.Buf);
238 FreeStrBuf(&ptr->IO.RecvBuf.Buf);
239 DeleteAsyncMsg(&ptr->IO.ReadMsg);
240 if (((struct CitContext*)ptr->IO.CitContext)) {
241 ((struct CitContext*)ptr->IO.CitContext)->state = CON_IDLE;
242 ((struct CitContext*)ptr->IO.CitContext)->kill_me = 1;
244 FreeAsyncIOContents(&ptr->IO);
248 eNextState FinalizePOP3AggrRun(AsyncIO *IO)
251 pop3aggr *cpptr = (pop3aggr *)IO->Data;
253 EVP3C_syslog(LOG_INFO,
254 "%s@%s: fetched %ld new of %d messages in %fs. bye.",
255 ChrPtr(cpptr->pop3user),
258 GetCount(cpptr->MsgNumbers),
259 IO->Now - cpptr->IOStart
262 It = GetNewHashPos(POP3FetchUrls, 0);
263 pthread_mutex_lock(&POP3QueueMutex);
265 if (GetHashPosFromKey(POP3FetchUrls, SKEY(cpptr->Url), It))
266 DeleteEntryFromHash(POP3FetchUrls, It);
268 pthread_mutex_unlock(&POP3QueueMutex);
273 eNextState FailAggregationRun(AsyncIO *IO)
278 eNextState POP3C_ReadGreeting(pop3aggr *RecvMsg)
280 AsyncIO *IO = &RecvMsg->IO;
281 SetPOP3State(IO, eGreeting);
283 /* Read the server greeting */
284 if (!POP3C_OK) return eTerminateConnection;
285 else return eSendReply;
288 eNextState POP3C_SendUser(pop3aggr *RecvMsg)
290 AsyncIO *IO = &RecvMsg->IO;
291 SetPOP3State(IO, eUser);
292 /* Identify ourselves. NOTE: we have to append a CR to each command.
293 * The LF will automatically be appended by sock_puts(). Believe it
294 * or not, leaving out the CR will cause problems if the server happens
295 * to be Exchange, which is so b0rken it actually barfs on
296 * LF-terminated newlines.
298 StrBufPrintf(RecvMsg->IO.SendBuf.Buf,
299 "USER %s\r\n", ChrPtr(RecvMsg->pop3user));
304 eNextState POP3C_GetUserState(pop3aggr *RecvMsg)
306 AsyncIO *IO = &RecvMsg->IO;
308 if (!POP3C_OK) return eTerminateConnection;
309 else return eSendReply;
312 eNextState POP3C_SendPassword(pop3aggr *RecvMsg)
314 AsyncIO *IO = &RecvMsg->IO;
315 SetPOP3State(IO, ePassword);
317 StrBufPrintf(RecvMsg->IO.SendBuf.Buf,
318 "PASS %s\r\n", ChrPtr(RecvMsg->pop3pass));
319 EVP3CM_syslog(LOG_DEBUG, "<PASS <password>\n");
320 // POP3C_DBG_SEND(); No, we won't write the passvoid to syslog...
324 eNextState POP3C_GetPassState(pop3aggr *RecvMsg)
326 AsyncIO *IO = &RecvMsg->IO;
328 if (!POP3C_OK) return eTerminateConnection;
329 else return eSendReply;
332 eNextState POP3C_SendListCommand(pop3aggr *RecvMsg)
334 AsyncIO *IO = &RecvMsg->IO;
335 SetPOP3State(IO, eListing);
337 /* Get the list of messages */
338 StrBufPlain(RecvMsg->IO.SendBuf.Buf, HKEY("LIST\r\n"));
343 eNextState POP3C_GetListCommandState(pop3aggr *RecvMsg)
345 AsyncIO *IO = &RecvMsg->IO;
347 if (!POP3C_OK) return eTerminateConnection;
348 RecvMsg->MsgNumbers = NewHash(1, NULL);
354 eNextState POP3C_GetListOneLine(pop3aggr *RecvMsg)
356 AsyncIO *IO = &RecvMsg->IO;
361 FetchItem *OneMsg = NULL;
364 if ((StrLength(RecvMsg->IO.IOBuf) == 1) &&
365 (ChrPtr(RecvMsg->IO.IOBuf)[0] == '.'))
367 if (GetCount(RecvMsg->MsgNumbers) == 0)
369 //// RecvMsg->Sate = ReadQuitState;
373 RecvMsg->Pos = GetNewHashPos(RecvMsg->MsgNumbers, 0);
380 * work around buggy pop3 servers which send
381 * empty lines in their listings.
383 if ((StrLength(RecvMsg->IO.IOBuf) == 0) ||
384 !isdigit(ChrPtr(RecvMsg->IO.IOBuf)[0]))
389 OneMsg = (FetchItem*) malloc(sizeof(FetchItem));
390 memset(OneMsg, 0, sizeof(FetchItem));
391 OneMsg->MSGID = atol(ChrPtr(RecvMsg->IO.IOBuf));
393 pch = strchr(ChrPtr(RecvMsg->IO.IOBuf), ' ');
396 OneMsg->MSGSize = atol(pch + 1);
399 rc = TestValidateHash(RecvMsg->MsgNumbers);
401 EVP3CCS_syslog(LOG_DEBUG, "Hash Invalid: %d\n", rc);
404 Put(RecvMsg->MsgNumbers, LKEY(OneMsg->MSGID), OneMsg, HfreeFetchItem);
406 rc = TestValidateHash(RecvMsg->MsgNumbers);
408 EVP3CCS_syslog(LOG_DEBUG, "Hash Invalid: %d\n", rc);
410 //RecvMsg->State --; /* read next Line */
414 eNextState POP3_FetchNetworkUsetableEntry(AsyncIO *IO)
419 pop3aggr *RecvMsg = (pop3aggr *) IO->Data;
421 SetPOP3State(IO, eUseTable);
423 if((RecvMsg->Pos != NULL) &&
424 GetNextHashPos(RecvMsg->MsgNumbers,
430 if (server_shutting_down)
433 if (CheckIfAlreadySeen("POP3 Item Seen",
434 RecvMsg->CurrMsg->MsgUID,
441 /* Item has already been seen */
442 RecvMsg->CurrMsg->NeedFetch = 0;
446 EVP3CCSM_syslog(LOG_DEBUG, "NO\n");
447 RecvMsg->CurrMsg->NeedFetch = 1;
449 return NextDBOperation(&RecvMsg->IO,
450 POP3_FetchNetworkUsetableEntry);
454 /* ok, now we know them all,
455 * continue with reading the actual messages. */
456 DeleteHashPos(&RecvMsg->Pos);
457 return DBQueueEventContext(IO, POP3_C_ReAttachToFetchMessages);
461 eNextState POP3C_GetOneMessagID(pop3aggr *RecvMsg)
463 AsyncIO *IO = &RecvMsg->IO;
468 SetPOP3State(IO, eGetMsgID);
471 rc = TestValidateHash(RecvMsg->MsgNumbers);
473 EVP3CCS_syslog(LOG_DEBUG, "Hash Invalid: %d\n", rc);
475 if((RecvMsg->Pos != NULL) &&
476 GetNextHashPos(RecvMsg->MsgNumbers,
481 RecvMsg->CurrMsg = (FetchItem*) vData;
482 /* Find out the UIDL of the message,
483 * to determine whether we've already downloaded it */
484 StrBufPrintf(RecvMsg->IO.SendBuf.Buf,
485 "UIDL %ld\r\n", RecvMsg->CurrMsg->MSGID);
491 DeleteHashPos(&RecvMsg->Pos);
492 /// done receiving uidls.. start looking them up now.
493 RecvMsg->Pos = GetNewHashPos(RecvMsg->MsgNumbers, 0);
494 return EventQueueDBOperation(&RecvMsg->IO,
495 POP3_FetchNetworkUsetableEntry);
497 return eReadMore; /* TODO */
500 eNextState POP3C_GetOneMessageIDState(pop3aggr *RecvMsg)
502 AsyncIO *IO = &RecvMsg->IO;
505 rc = TestValidateHash(RecvMsg->MsgNumbers);
507 EVP3CCS_syslog(LOG_DEBUG, "Hash Invalid: %d\n", rc);
511 if (!POP3C_OK) return eTerminateConnection;
512 RecvMsg->CurrMsg->MsgUIDL =
513 NewStrBufPlain(NULL, StrLength(RecvMsg->IO.IOBuf));
514 RecvMsg->CurrMsg->MsgUID =
515 NewStrBufPlain(NULL, StrLength(RecvMsg->IO.IOBuf) * 2);
517 StrBufExtract_token(RecvMsg->CurrMsg->MsgUIDL,
518 RecvMsg->IO.IOBuf, 2, ' ');
520 StrBufPrintf(RecvMsg->CurrMsg->MsgUID,
522 ChrPtr(RecvMsg->RoomName),
523 ChrPtr(RecvMsg->CurrMsg->MsgUIDL),
524 RecvMsg->IO.ConnectMe->User,
525 RecvMsg->IO.ConnectMe->Host);
531 eNextState POP3C_SendGetOneMsg(pop3aggr *RecvMsg)
533 AsyncIO *IO = &RecvMsg->IO;
538 SetPOP3State(IO, eGetMsg);
540 RecvMsg->CurrMsg = NULL;
541 while ((RecvMsg->Pos != NULL) &&
542 GetNextHashPos(RecvMsg->MsgNumbers,
546 (RecvMsg->CurrMsg = (FetchItem*) vData,
547 RecvMsg->CurrMsg->NeedFetch == 0))
550 if ((RecvMsg->CurrMsg != NULL ) && (RecvMsg->CurrMsg->NeedFetch == 1))
552 /* Message has not been seen.
553 * Tell the server to fetch the message... */
554 StrBufPrintf(RecvMsg->IO.SendBuf.Buf,
555 "RETR %ld\r\n", RecvMsg->CurrMsg->MSGID);
560 RecvMsg->State = ReadQuitState;
561 return POP3_C_DispatchWriteDone(&RecvMsg->IO);
566 eNextState POP3C_ReadMessageBodyFollowing(pop3aggr *RecvMsg)
568 AsyncIO *IO = &RecvMsg->IO;
570 if (!POP3C_OK) return eTerminateConnection;
571 RecvMsg->IO.ReadMsg = NewAsyncMsg(HKEY("."),
572 RecvMsg->CurrMsg->MSGSize,
581 eNextState POP3C_StoreMsgRead(AsyncIO *IO)
583 pop3aggr *RecvMsg = (pop3aggr *) IO->Data;
585 SetPOP3State(IO, eStoreMsg);
587 EVP3CCS_syslog(LOG_DEBUG,
588 "MARKING: %s as seen: ",
589 ChrPtr(RecvMsg->CurrMsg->MsgUID));
590 CheckIfAlreadySeen("POP3 Item Seen",
591 RecvMsg->CurrMsg->MsgUID,
597 return DBQueueEventContext(&RecvMsg->IO, POP3_C_ReAttachToFetchMessages);
599 eNextState POP3C_SaveMsg(AsyncIO *IO)
602 pop3aggr *RecvMsg = (pop3aggr *) IO->Data;
604 /* Do Something With It (tm) */
605 msgnum = CtdlSubmitMsg(RecvMsg->CurrMsg->Msg,
607 ChrPtr(RecvMsg->RoomName),
611 /* Message has been committed to the store
612 * write the uidl to the use table
613 * so we don't fetch this message again
616 CM_Free(RecvMsg->CurrMsg->Msg);
619 return NextDBOperation(&RecvMsg->IO, POP3C_StoreMsgRead);
622 eNextState POP3C_ReadMessageBody(pop3aggr *RecvMsg)
624 AsyncIO *IO = &RecvMsg->IO;
625 EVP3CM_syslog(LOG_DEBUG, "Converting message...");
626 RecvMsg->CurrMsg->Msg =
627 convert_internet_message_buf(&RecvMsg->IO.ReadMsg->MsgBuf);
628 return EventQueueDBOperation(&RecvMsg->IO, POP3C_SaveMsg);
631 eNextState POP3C_SendDelete(pop3aggr *RecvMsg)
633 AsyncIO *IO = &RecvMsg->IO;
635 SetPOP3State(IO, eDelete);
637 if (!RecvMsg->keep) {
638 StrBufPrintf(RecvMsg->IO.SendBuf.Buf,
639 "DELE %ld\r\n", RecvMsg->CurrMsg->MSGID);
644 RecvMsg->State = ReadMessageBodyFollowing;
645 return POP3_C_DispatchWriteDone(&RecvMsg->IO);
648 eNextState POP3C_ReadDeleteState(pop3aggr *RecvMsg)
650 AsyncIO *IO = &RecvMsg->IO;
652 RecvMsg->State = GetOneMessageIDState;
656 eNextState POP3C_SendQuit(pop3aggr *RecvMsg)
658 AsyncIO *IO = &RecvMsg->IO;
659 SetPOP3State(IO, eQuit);
662 StrBufPlain(RecvMsg->IO.SendBuf.Buf,
669 eNextState POP3C_ReadQuitState(pop3aggr *RecvMsg)
671 AsyncIO *IO = &RecvMsg->IO;
673 return eTerminateConnection;
676 const long POP3_C_ConnTimeout = 1000;
677 const long DefaultPOP3Port = 110;
679 Pop3ClientHandler POP3C_ReadHandlers[] = {
683 POP3C_GetListCommandState,
684 POP3C_GetListOneLine,
685 POP3C_GetOneMessageIDState,
686 POP3C_ReadMessageBodyFollowing,
687 POP3C_ReadMessageBody,
688 POP3C_ReadDeleteState,
692 const long POP3_C_SendTimeouts[POP3C_MaxRead] = {
702 const ConstStr POP3C_ReadErrors[POP3C_MaxRead] = {
703 {HKEY("Connection broken during ")},
704 {HKEY("Connection broken during ")},
705 {HKEY("Connection broken during ")},
706 {HKEY("Connection broken during ")},
707 {HKEY("Connection broken during ")},
708 {HKEY("Connection broken during ")},
709 {HKEY("Connection broken during ")},
710 {HKEY("Connection broken during ")}
713 Pop3ClientHandler POP3C_SendHandlers[] = {
714 NULL, /* we don't send a greeting */
717 POP3C_SendListCommand,
719 POP3C_GetOneMessagID,
726 const long POP3_C_ReadTimeouts[] = {
738 /*****************************************************************************/
739 /* POP3 CLIENT DISPATCHER */
740 /*****************************************************************************/
742 void POP3SetTimeout(eNextState NextTCPState, pop3aggr *pMsg)
744 AsyncIO *IO = &pMsg->IO;
745 double Timeout = 0.0;
747 EVP3C_syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
749 switch (NextTCPState) {
753 Timeout = POP3_C_SendTimeouts[pMsg->State];
755 if (pMsg->State == eDATABody) {
756 / * if we're sending a huge message, we need more time. * /
757 Timeout += StrLength(pMsg->msgtext) / 1024;
763 Timeout = POP3_C_ReadTimeouts[pMsg->State];
765 if (pMsg->State == eDATATerminateBody) {
767 * some mailservers take a nap before accepting the message
768 * content inspection and such.
770 Timeout += StrLength(pMsg->msgtext) / 1024;
781 case eTerminateConnection:
784 case eReadMore://// TODO
787 SetNextTimeout(&pMsg->IO, Timeout);
789 eNextState POP3_C_DispatchReadDone(AsyncIO *IO)
791 /* EVP3CCS_syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__); to noisy anyways. */
792 pop3aggr *pMsg = IO->Data;
795 rc = POP3C_ReadHandlers[pMsg->State](pMsg);
798 POP3SetTimeout(rc, pMsg);
801 eNextState POP3_C_DispatchWriteDone(AsyncIO *IO)
803 pop3aggr *pMsg = IO->Data;
806 /* EVP3CCS_syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__); to noisy anyways. */
807 rc = POP3C_SendHandlers[pMsg->State](pMsg);
808 POP3SetTimeout(rc, pMsg);
813 /*****************************************************************************/
814 /* POP3 CLIENT ERROR CATCHERS */
815 /*****************************************************************************/
816 eNextState POP3_C_Terminate(AsyncIO *IO)
818 /// pop3aggr *pMsg = (pop3aggr *)IO->Data;
820 EVP3CCS_syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
821 FinalizePOP3AggrRun(IO);
824 eNextState POP3_C_TerminateDB(AsyncIO *IO)
826 /// pop3aggr *pMsg = (pop3aggr *)IO->Data;
828 EVP3CCS_syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
829 FinalizePOP3AggrRun(IO);
832 eNextState POP3_C_Timeout(AsyncIO *IO)
834 pop3aggr *pMsg = IO->Data;
836 EVP3CCS_syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
837 StrBufPlain(IO->ErrMsg, CKEY(POP3C_ReadErrors[pMsg->State]));
838 return FailAggregationRun(IO);
840 eNextState POP3_C_ConnFail(AsyncIO *IO)
842 pop3aggr *pMsg = (pop3aggr *)IO->Data;
844 EVP3CCS_syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
845 StrBufPlain(IO->ErrMsg, CKEY(POP3C_ReadErrors[pMsg->State]));
846 return FailAggregationRun(IO);
848 eNextState POP3_C_DNSFail(AsyncIO *IO)
850 pop3aggr *pMsg = (pop3aggr *)IO->Data;
852 EVP3CCS_syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
853 StrBufPlain(IO->ErrMsg, CKEY(POP3C_ReadErrors[pMsg->State]));
854 return FailAggregationRun(IO);
856 eNextState POP3_C_Shutdown(AsyncIO *IO)
858 EVP3CCS_syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
859 //// pop3aggr *pMsg = IO->Data;
861 ////pMsg->MyQEntry->Status = 3;
862 ///StrBufPlain(pMsg->MyQEntry->StatusMessage, HKEY("server shutdown during message retrieval."));
863 FinalizePOP3AggrRun(IO);
869 * @brief lineread Handler; understands when to read more POP3 lines,
870 * and when this is a one-lined reply.
872 eReadState POP3_C_ReadServerStatus(AsyncIO *IO)
874 eReadState Finished = eBufferNotEmpty;
876 switch (IO->NextState) {
881 case eTerminateConnection:
883 Finished = eReadFail;
890 Finished = StrBufChunkSipLine(IO->IOBuf, &IO->RecvBuf);
894 Finished = CtdlReadMessageBodyAsync(IO);
900 /*****************************************************************************
901 * So we connect our Server IP here. *
902 *****************************************************************************/
903 eNextState POP3_C_ReAttachToFetchMessages(AsyncIO *IO)
905 pop3aggr *cpptr = IO->Data;
907 EVP3CCS_syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
908 ////??? cpptr->State ++;
909 if (cpptr->Pos == NULL)
910 cpptr->Pos = GetNewHashPos(cpptr->MsgNumbers, 0);
912 POP3_C_DispatchWriteDone(IO);
913 ReAttachIO(IO, cpptr, 0);
914 IO->NextState = eReadMessage;
915 return IO->NextState;
918 eNextState pop3_connect_ip(AsyncIO *IO)
920 pop3aggr *cpptr = IO->Data;
922 if (cpptr->IOStart == 0.0) /* whith or without DNS? */
923 cpptr->IOStart = IO->Now;
925 EVP3CCS_syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
927 return EvConnectSock(IO,
929 POP3_C_ReadTimeouts[0],
933 eNextState pop3_get_one_host_ip_done(AsyncIO *IO)
935 pop3aggr *cpptr = IO->Data;
936 struct hostent *hostent;
940 hostent = cpptr->HostLookup.VParsedDNSReply;
941 if ((cpptr->HostLookup.DNSStatus == ARES_SUCCESS) &&
942 (hostent != NULL) ) {
943 memset(&cpptr->IO.ConnectMe->Addr, 0, sizeof(struct in6_addr));
944 if (cpptr->IO.ConnectMe->IPv6) {
945 memcpy(&cpptr->IO.ConnectMe->Addr.sin6_addr.s6_addr,
946 &hostent->h_addr_list[0],
947 sizeof(struct in6_addr));
949 cpptr->IO.ConnectMe->Addr.sin6_family =
951 cpptr->IO.ConnectMe->Addr.sin6_port =
952 htons(DefaultPOP3Port);
955 struct sockaddr_in *addr =
956 (struct sockaddr_in*)
957 &cpptr->IO.ConnectMe->Addr;
959 memcpy(&addr->sin_addr.s_addr,
960 hostent->h_addr_list[0],
963 addr->sin_family = hostent->h_addrtype;
964 addr->sin_port = htons(DefaultPOP3Port);
966 return pop3_connect_ip(IO);
972 eNextState pop3_get_one_host_ip(AsyncIO *IO)
974 pop3aggr *cpptr = IO->Data;
976 cpptr->IOStart = IO->Now;
978 EVP3CCS_syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
980 EVP3CCS_syslog(LOG_DEBUG,
981 "POP3 client[%ld]: looking up %s-Record %s : %d ...\n",
983 (cpptr->IO.ConnectMe->IPv6)? "aaaa": "a",
984 cpptr->IO.ConnectMe->Host,
985 cpptr->IO.ConnectMe->Port);
987 QueueQuery((cpptr->IO.ConnectMe->IPv6)? ns_t_aaaa : ns_t_a,
988 cpptr->IO.ConnectMe->Host,
991 pop3_get_one_host_ip_done);
992 IO->NextState = eReadDNSReply;
993 return IO->NextState;
998 int pop3_do_fetching(pop3aggr *cpptr)
1000 AsyncIO *IO = &cpptr->IO;
1005 POP3_C_ReadServerStatus,
1007 POP3_C_DispatchWriteDone,
1008 POP3_C_DispatchReadDone,
1015 safestrncpy(((CitContext *)cpptr->IO.CitContext)->cs_host,
1017 sizeof(((CitContext *)cpptr->IO.CitContext)->cs_host));
1019 if (cpptr->IO.ConnectMe->IsIP) {
1020 QueueEventContext(&cpptr->IO,
1024 QueueEventContext(&cpptr->IO,
1025 pop3_get_one_host_ip);
1031 * Scan a room's netconfig to determine whether it requires POP3 aggregation
1033 void pop3client_scan_room(struct ctdlroom *qrbuf, void *data, OneRoomNetCfg *OneRNCFG)
1035 const RoomNetCfgLine *pLine;
1038 pthread_mutex_lock(&POP3QueueMutex);
1039 if (GetHash(POP3QueueRooms, LKEY(qrbuf->QRnumber), &vptr))
1041 pthread_mutex_unlock(&POP3QueueMutex);
1042 EVP3CQ_syslog(LOG_DEBUG,
1043 "pop3client: [%ld] %s already in progress.",
1048 pthread_mutex_unlock(&POP3QueueMutex);
1050 if (server_shutting_down) return;
1052 pLine = OneRNCFG->NetConfigs[pop3client];
1054 while (pLine != NULL)
1058 cptr = (pop3aggr *) malloc(sizeof(pop3aggr));
1059 memset(cptr, 0, sizeof(pop3aggr));
1060 ///TODO do we need this? cptr->roomlist_parts=1;
1061 cptr->RoomName = NewStrBufPlain(qrbuf->QRname, -1);
1062 cptr->pop3user = NewStrBufDup(pLine->Value[1]);
1063 cptr->pop3pass = NewStrBufDup(pLine->Value[2]);
1064 cptr->Url = NewStrBuf();
1065 cptr->Host = NewStrBufDup(pLine->Value[0]);
1067 cptr->keep = atol(ChrPtr(pLine->Value[3]));
1068 cptr->interval = atol(ChrPtr(pLine->Value[4]));
1070 StrBufAppendBufPlain(cptr->Url, HKEY("pop3://"), 0);
1071 StrBufUrlescUPAppend(cptr->Url, cptr->pop3user, NULL);
1072 StrBufAppendBufPlain(cptr->Url, HKEY(":"), 0);
1073 StrBufUrlescUPAppend(cptr->Url, cptr->pop3pass, NULL);
1074 StrBufAppendBufPlain(cptr->Url, HKEY("@"), 0);
1075 StrBufAppendBuf(cptr->Url, cptr->Host, 0);
1076 StrBufAppendBufPlain(cptr->Url, HKEY("/"), 0);
1077 StrBufUrlescAppend(cptr->Url, cptr->RoomName, NULL);
1079 ParseURL(&cptr->IO.ConnectMe, cptr->Url, 110);
1083 /* todo: we need to reunite the url to be shure. */
1085 pthread_mutex_lock(&POP3ueueMutex);
1086 GetHash(POP3FetchUrls, SKEY(ptr->Url), &vptr);
1087 use_this_cptr = (pop3aggr *)vptr;
1089 if (use_this_rncptr != NULL)
1091 /* mustn't attach to an active session */
1092 if (use_this_cptr->RefCount > 0)
1094 DeletePOP3Cfg(cptr);
1100 StrBufAppendBufPlain(
1101 use_this_cptr->rooms,
1104 if (use_this_cptr->roomlist_parts == 1)
1106 use_this_cptr->OtherQRnumbers
1107 = NewHash(1, lFlathash);
1109 QRnumber = (long*)malloc(sizeof(long));
1110 *QRnumber = qrbuf->QRnumber;
1111 Put(use_this_cptr->OtherQRnumbers,
1112 LKEY(qrbuf->QRnumber),
1116 use_this_cptr->roomlist_parts++;
1118 pthread_mutex_unlock(&POP3QueueMutex);
1121 pthread_mutex_unlock(&RSSQueueMutex);
1123 cptr->n = Pop3ClientID++;
1124 pthread_mutex_lock(&POP3QueueMutex);
1128 DeletePOP3Aggregator);
1130 pthread_mutex_unlock(&POP3QueueMutex);
1131 pLine = pLine->next;
1136 static int doing_pop3client = 0;
1138 void pop3client_scan(void) {
1139 static time_t last_run = 0L;
1140 time_t fastest_scan;
1147 become_session(&pop3_client_CC);
1149 if (config.c_pop3_fastest < config.c_pop3_fetch)
1150 fastest_scan = config.c_pop3_fastest;
1152 fastest_scan = config.c_pop3_fetch;
1155 * Run POP3 aggregation no more frequently than once every n seconds
1157 if ( (time(NULL) - last_run) < fastest_scan ) {
1162 * This is a simple concurrency check to make sure only one pop3client
1163 * run is done at a time. We could do this with a mutex, but since we
1164 * don't really require extremely fine granularity here, we'll do it
1165 * with a static variable instead.
1167 if (doing_pop3client) return;
1168 doing_pop3client = 1;
1170 EVP3CQM_syslog(LOG_DEBUG, "pop3client started");
1171 CtdlForEachNetCfgRoom(pop3client_scan_room, NULL, pop3client);
1173 pthread_mutex_lock(&POP3QueueMutex);
1174 it = GetNewHashPos(POP3FetchUrls, 0);
1175 while (!server_shutting_down &&
1176 GetNextHashPos(POP3FetchUrls, it, &len, &Key, &vrptr) &&
1178 cptr = (pop3aggr *)vrptr;
1179 if (cptr->RefCount == 0)
1180 if (!pop3_do_fetching(cptr))
1181 DeletePOP3Aggregator(cptr);////TODO
1184 if ((palist->interval && time(NULL) > (last_run + palist->interval))
1185 || (time(NULL) > last_run + config.c_pop3_fetch))
1186 pop3_do_fetching(palist->roomname, palist->pop3host,
1187 palist->pop3user, palist->pop3pass, palist->keep);
1189 palist = palist->next;
1194 pthread_mutex_unlock(&POP3QueueMutex);
1196 EVP3CQM_syslog(LOG_DEBUG, "pop3client ended");
1197 last_run = time(NULL);
1198 doing_pop3client = 0;
1202 void pop3_cleanup(void)
1204 /* citthread_mutex_destroy(&POP3QueueMutex); TODO */
1205 while (doing_pop3client != 0) ;
1206 DeleteHash(&POP3FetchUrls);
1207 DeleteHash(&POP3QueueRooms);
1212 void LogDebugEnablePOP3Client(const int n)
1214 POP3ClientDebugEnabled = n;
1217 CTDL_MODULE_INIT(pop3client)
1221 CtdlFillSystemContext(&pop3_client_CC, "POP3aggr");
1222 CtdlREGISTERRoomCfgType(pop3client, ParseGeneric, 0, 5, SerializeGeneric, DeleteGenericCfgLine);
1223 pthread_mutex_init(&POP3QueueMutex, NULL);
1224 POP3QueueRooms = NewHash(1, lFlathash);
1225 POP3FetchUrls = NewHash(1, NULL);
1226 CtdlRegisterSessionHook(pop3client_scan, EVT_TIMER, PRIO_AGGR + 50);
1227 CtdlRegisterEVCleanupHook(pop3_cleanup);
1228 CtdlRegisterDebugFlagHook(HKEY("pop3client"), LogDebugEnablePOP3Client, &POP3ClientDebugEnabled);
1231 /* return our module id for the log */
1232 return "pop3client";