2 * Consolidate mail from remote POP3 accounts.
4 * Copyright (c) 2007-2015 by the citadel.org team
6 * This program is open source software; you can redistribute it and/or
7 * modify it under the terms of the GNU General Public License as published
8 * by the Free Software Foundation; either version 3 of the License, or
9 * (at your option) any later version.
11 * This program is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 * GNU General Public License for more details.
20 #include <sysconfig.h>
22 #if TIME_WITH_SYS_TIME
23 # include <sys/time.h>
27 # include <sys/time.h>
36 #include <sys/types.h>
38 #include <libcitadel.h>
41 #include "citserver.h"
44 #include "ctdl_module.h"
45 #include "clientsocket.h"
47 #include "internet_addressing.h"
49 #include "citadel_dirs.h"
50 #include "event_client.h"
53 #define POP3C_OK (strncasecmp(ChrPtr(RecvMsg->IO.IOBuf), "+OK", 3) == 0)
55 int POP3ClientDebugEnabled = 0;
57 #define N ((pop3aggr*)IO->Data)->n
59 #define DBGLOG(LEVEL) if ((LEVEL != LOG_DEBUG) || (POP3ClientDebugEnabled != 0))
61 #define EVP3C_syslog(LEVEL, FORMAT, ...) \
62 DBGLOG(LEVEL) syslog(LEVEL, \
63 "%s[%ld]CC[%d][%ld]POP3: " FORMAT, \
64 IOSTR, IO->ID, CCID, N, __VA_ARGS__)
66 #define EVP3CM_syslog(LEVEL, FORMAT) \
67 DBGLOG(LEVEL) syslog(LEVEL, \
68 "%s[%ld]CC[%d][%ld]POP3: " FORMAT, \
69 IOSTR, IO->ID, CCID, N)
71 #define EVP3CQ_syslog(LEVEL, FORMAT, ...) \
72 DBGLOG(LEVEL) syslog(LEVEL, \
76 #define EVP3CQM_syslog(LEVEL, FORMAT) \
77 DBGLOG(LEVEL) syslog(LEVEL, \
81 #define EVP3CCS_syslog(LEVEL, FORMAT, ...) \
82 DBGLOG(LEVEL) syslog(LEVEL, "%s[%ld][%ld]POP3: " FORMAT, \
83 IOSTR, IO->ID, N, __VA_ARGS__)
85 #define EVP3CCSM_syslog(LEVEL, FORMAT) \
86 DBGLOG(LEVEL) syslog(LEVEL, "%s[%ld][%ld]POP3: " FORMAT, \
89 #define POP3C_DBG_SEND() \
90 EVP3C_syslog(LOG_DEBUG, \
91 "%s[%ld]CC[%d][%ld]POP3: > %s\n", \
92 IOSTR, IO->ID, CCID, N, \
93 ChrPtr(RecvMsg->IO.SendBuf.Buf))
95 #define POP3C_DBG_READ() \
96 EVP3C_syslog(LOG_DEBUG, \
97 "%s[%ld]CC[%d][%ld]POP3: < %s\n", \
98 IOSTR, IO->ID, CCID, N, \
99 ChrPtr(RecvMsg->IO.IOBuf))
102 struct CitContext pop3_client_CC;
104 pthread_mutex_t POP3QueueMutex; /* locks the access to the following vars: */
105 HashList *POP3QueueRooms = NULL;
106 HashList *POP3FetchUrls = NULL;
108 typedef struct pop3aggr pop3aggr;
109 typedef eNextState(*Pop3ClientHandler)(pop3aggr* RecvMsg);
111 eNextState POP3_C_Shutdown(AsyncIO *IO);
112 eNextState POP3_C_Timeout(AsyncIO *IO);
113 eNextState POP3_C_ConnFail(AsyncIO *IO);
114 eNextState POP3_C_DNSFail(AsyncIO *IO);
115 eNextState POP3_C_DispatchReadDone(AsyncIO *IO);
116 eNextState POP3_C_DispatchWriteDone(AsyncIO *IO);
117 eNextState POP3_C_Terminate(AsyncIO *IO);
118 eReadState POP3_C_ReadServerStatus(AsyncIO *IO);
119 eNextState POP3_C_ReAttachToFetchMessages(AsyncIO *IO);
121 typedef struct __pop3_room_counter {
126 typedef enum ePOP3_C_States {
132 GetOneMessageIDState,
133 ReadMessageBodyFollowing,
141 typedef struct _FetchItem {
147 struct CtdlMessage *Msg;
150 void HfreeFetchItem(void *vItem)
152 FetchItem *Item = (FetchItem*) vItem;
153 FreeStrBuf(&Item->MsgUIDL);
154 FreeStrBuf(&Item->MsgUID);
160 typedef enum _POP3State {
174 ConstStr POP3States[] = {
175 {HKEY("Aggregator created")},
176 {HKEY("Reading Greeting")},
177 {HKEY("Sending User")},
178 {HKEY("Sending Password")},
180 {HKEY("Fetching Usetable")},
181 {HKEY("Get MSG ID")},
182 {HKEY("Get Message")},
184 {HKEY("Delete Upstream")},
188 static void SetPOP3State(AsyncIO *IO, POP3State State)
190 CitContext* CCC = IO->CitContext;
192 memcpy(CCC->cs_clientname, POP3States[State].Key, POP3States[State].len + 1);
203 DNSQueryParts HostLookup;
206 HashList *OtherQRnumbers;
212 StrBuf *RoomName; // TODO: fill me
215 ePOP3_C_States State;
216 HashList *MsgNumbers;
221 void DeletePOP3Aggregator(void *vptr)
223 pop3aggr *ptr = vptr;
224 DeleteHashPos(&ptr->Pos);
225 DeleteHash(&ptr->MsgNumbers);
226 // FreeStrBuf(&ptr->rooms);
227 FreeStrBuf(&ptr->pop3user);
228 FreeStrBuf(&ptr->pop3pass);
229 FreeStrBuf(&ptr->Host);
230 FreeStrBuf(&ptr->RoomName);
231 FreeURL(&ptr->IO.ConnectMe);
232 FreeStrBuf(&ptr->Url);
233 FreeStrBuf(&ptr->IO.IOBuf);
234 FreeStrBuf(&ptr->IO.SendBuf.Buf);
235 FreeStrBuf(&ptr->IO.RecvBuf.Buf);
236 DeleteAsyncMsg(&ptr->IO.ReadMsg);
237 if (((struct CitContext*)ptr->IO.CitContext)) {
238 ((struct CitContext*)ptr->IO.CitContext)->state = CON_IDLE;
239 ((struct CitContext*)ptr->IO.CitContext)->kill_me = 1;
241 FreeAsyncIOContents(&ptr->IO);
245 eNextState FinalizePOP3AggrRun(AsyncIO *IO)
248 pop3aggr *cpptr = (pop3aggr *)IO->Data;
250 EVP3C_syslog(LOG_INFO,
251 "%s@%s: fetched %ld new of %d messages in %fs. bye.",
252 ChrPtr(cpptr->pop3user),
255 GetCount(cpptr->MsgNumbers),
256 IO->Now - cpptr->IOStart
259 It = GetNewHashPos(POP3FetchUrls, 0);
260 pthread_mutex_lock(&POP3QueueMutex);
262 if (GetHashPosFromKey(POP3FetchUrls, SKEY(cpptr->Url), It))
263 DeleteEntryFromHash(POP3FetchUrls, It);
265 pthread_mutex_unlock(&POP3QueueMutex);
270 eNextState FailAggregationRun(AsyncIO *IO)
275 eNextState POP3C_ReadGreeting(pop3aggr *RecvMsg)
277 AsyncIO *IO = &RecvMsg->IO;
278 SetPOP3State(IO, eGreeting);
280 /* Read the server greeting */
281 if (!POP3C_OK) return eTerminateConnection;
282 else return eSendReply;
285 eNextState POP3C_SendUser(pop3aggr *RecvMsg)
287 AsyncIO *IO = &RecvMsg->IO;
288 SetPOP3State(IO, eUser);
289 /* Identify ourselves. NOTE: we have to append a CR to each command.
290 * The LF will automatically be appended by sock_puts(). Believe it
291 * or not, leaving out the CR will cause problems if the server happens
292 * to be Exchange, which is so b0rken it actually barfs on
293 * LF-terminated newlines.
295 StrBufPrintf(RecvMsg->IO.SendBuf.Buf,
296 "USER %s\r\n", ChrPtr(RecvMsg->pop3user));
301 eNextState POP3C_GetUserState(pop3aggr *RecvMsg)
303 AsyncIO *IO = &RecvMsg->IO;
305 if (!POP3C_OK) return eTerminateConnection;
306 else return eSendReply;
309 eNextState POP3C_SendPassword(pop3aggr *RecvMsg)
311 AsyncIO *IO = &RecvMsg->IO;
312 SetPOP3State(IO, ePassword);
314 StrBufPrintf(RecvMsg->IO.SendBuf.Buf,
315 "PASS %s\r\n", ChrPtr(RecvMsg->pop3pass));
316 EVP3CM_syslog(LOG_DEBUG, "<PASS <password>\n");
317 // POP3C_DBG_SEND(); No, we won't write the passvoid to syslog...
321 eNextState POP3C_GetPassState(pop3aggr *RecvMsg)
323 AsyncIO *IO = &RecvMsg->IO;
325 if (!POP3C_OK) return eTerminateConnection;
326 else return eSendReply;
329 eNextState POP3C_SendListCommand(pop3aggr *RecvMsg)
331 AsyncIO *IO = &RecvMsg->IO;
332 SetPOP3State(IO, eListing);
334 /* Get the list of messages */
335 StrBufPlain(RecvMsg->IO.SendBuf.Buf, HKEY("LIST\r\n"));
340 eNextState POP3C_GetListCommandState(pop3aggr *RecvMsg)
342 AsyncIO *IO = &RecvMsg->IO;
344 if (!POP3C_OK) return eTerminateConnection;
345 RecvMsg->MsgNumbers = NewHash(1, NULL);
351 eNextState POP3C_GetListOneLine(pop3aggr *RecvMsg)
353 AsyncIO *IO = &RecvMsg->IO;
358 FetchItem *OneMsg = NULL;
361 if ((StrLength(RecvMsg->IO.IOBuf) == 1) &&
362 (ChrPtr(RecvMsg->IO.IOBuf)[0] == '.'))
364 if (GetCount(RecvMsg->MsgNumbers) == 0)
366 //// RecvMsg->Sate = ReadQuitState;
370 RecvMsg->Pos = GetNewHashPos(RecvMsg->MsgNumbers, 0);
377 * work around buggy pop3 servers which send
378 * empty lines in their listings.
380 if ((StrLength(RecvMsg->IO.IOBuf) == 0) ||
381 !isdigit(ChrPtr(RecvMsg->IO.IOBuf)[0]))
386 OneMsg = (FetchItem*) malloc(sizeof(FetchItem));
387 memset(OneMsg, 0, sizeof(FetchItem));
388 OneMsg->MSGID = atol(ChrPtr(RecvMsg->IO.IOBuf));
390 pch = strchr(ChrPtr(RecvMsg->IO.IOBuf), ' ');
393 OneMsg->MSGSize = atol(pch + 1);
396 rc = TestValidateHash(RecvMsg->MsgNumbers);
398 EVP3CCS_syslog(LOG_DEBUG, "Hash Invalid: %d\n", rc);
401 Put(RecvMsg->MsgNumbers, LKEY(OneMsg->MSGID), OneMsg, HfreeFetchItem);
403 rc = TestValidateHash(RecvMsg->MsgNumbers);
405 EVP3CCS_syslog(LOG_DEBUG, "Hash Invalid: %d\n", rc);
407 //RecvMsg->State --; /* read next Line */
411 eNextState POP3_FetchNetworkUsetableEntry(AsyncIO *IO)
416 pop3aggr *RecvMsg = (pop3aggr *) IO->Data;
418 SetPOP3State(IO, eUseTable);
420 if((RecvMsg->Pos != NULL) &&
421 GetNextHashPos(RecvMsg->MsgNumbers,
427 if (server_shutting_down)
430 if (CheckIfAlreadySeen("POP3 Item Seen",
431 RecvMsg->CurrMsg->MsgUID,
433 EvGetNow(IO) - USETABLE_ANTIEXPIRE,
438 /* Item has already been seen */
439 RecvMsg->CurrMsg->NeedFetch = 0;
443 EVP3CCSM_syslog(LOG_DEBUG, "NO\n");
444 RecvMsg->CurrMsg->NeedFetch = 1;
446 return NextDBOperation(&RecvMsg->IO,
447 POP3_FetchNetworkUsetableEntry);
451 /* ok, now we know them all,
452 * continue with reading the actual messages. */
453 DeleteHashPos(&RecvMsg->Pos);
454 return DBQueueEventContext(IO, POP3_C_ReAttachToFetchMessages);
458 eNextState POP3C_GetOneMessagID(pop3aggr *RecvMsg)
460 AsyncIO *IO = &RecvMsg->IO;
465 SetPOP3State(IO, eGetMsgID);
468 rc = TestValidateHash(RecvMsg->MsgNumbers);
470 EVP3CCS_syslog(LOG_DEBUG, "Hash Invalid: %d\n", rc);
472 if((RecvMsg->Pos != NULL) &&
473 GetNextHashPos(RecvMsg->MsgNumbers,
478 RecvMsg->CurrMsg = (FetchItem*) vData;
479 /* Find out the UIDL of the message,
480 * to determine whether we've already downloaded it */
481 StrBufPrintf(RecvMsg->IO.SendBuf.Buf,
482 "UIDL %ld\r\n", RecvMsg->CurrMsg->MSGID);
488 DeleteHashPos(&RecvMsg->Pos);
489 /// done receiving uidls.. start looking them up now.
490 RecvMsg->Pos = GetNewHashPos(RecvMsg->MsgNumbers, 0);
491 return EventQueueDBOperation(&RecvMsg->IO,
492 POP3_FetchNetworkUsetableEntry,
495 return eReadMore; /* TODO */
498 eNextState POP3C_GetOneMessageIDState(pop3aggr *RecvMsg)
500 AsyncIO *IO = &RecvMsg->IO;
503 rc = TestValidateHash(RecvMsg->MsgNumbers);
505 EVP3CCS_syslog(LOG_DEBUG, "Hash Invalid: %d\n", rc);
509 if (!POP3C_OK) return eTerminateConnection;
510 RecvMsg->CurrMsg->MsgUIDL =
511 NewStrBufPlain(NULL, StrLength(RecvMsg->IO.IOBuf));
512 RecvMsg->CurrMsg->MsgUID =
513 NewStrBufPlain(NULL, StrLength(RecvMsg->IO.IOBuf) * 2);
515 StrBufExtract_token(RecvMsg->CurrMsg->MsgUIDL,
516 RecvMsg->IO.IOBuf, 2, ' ');
518 StrBufPrintf(RecvMsg->CurrMsg->MsgUID,
520 ChrPtr(RecvMsg->RoomName),
521 ChrPtr(RecvMsg->CurrMsg->MsgUIDL),
522 RecvMsg->IO.ConnectMe->User,
523 RecvMsg->IO.ConnectMe->Host);
529 eNextState POP3C_SendGetOneMsg(pop3aggr *RecvMsg)
531 AsyncIO *IO = &RecvMsg->IO;
536 SetPOP3State(IO, eGetMsg);
538 RecvMsg->CurrMsg = NULL;
539 while ((RecvMsg->Pos != NULL) &&
540 GetNextHashPos(RecvMsg->MsgNumbers,
544 (RecvMsg->CurrMsg = (FetchItem*) vData,
545 RecvMsg->CurrMsg->NeedFetch == 0))
548 if ((RecvMsg->CurrMsg != NULL ) && (RecvMsg->CurrMsg->NeedFetch == 1))
550 /* Message has not been seen.
551 * Tell the server to fetch the message... */
552 StrBufPrintf(RecvMsg->IO.SendBuf.Buf,
553 "RETR %ld\r\n", RecvMsg->CurrMsg->MSGID);
558 RecvMsg->State = ReadQuitState;
559 return POP3_C_DispatchWriteDone(&RecvMsg->IO);
564 eNextState POP3C_ReadMessageBodyFollowing(pop3aggr *RecvMsg)
566 AsyncIO *IO = &RecvMsg->IO;
568 if (!POP3C_OK) return eTerminateConnection;
569 RecvMsg->IO.ReadMsg = NewAsyncMsg(HKEY("."),
570 RecvMsg->CurrMsg->MSGSize,
571 CtdlGetConfigLong("c_maxmsglen"),
579 eNextState POP3C_StoreMsgRead(AsyncIO *IO)
581 pop3aggr *RecvMsg = (pop3aggr *) IO->Data;
583 SetPOP3State(IO, eStoreMsg);
585 EVP3CCS_syslog(LOG_DEBUG,
586 "MARKING: %s as seen: ",
587 ChrPtr(RecvMsg->CurrMsg->MsgUID));
588 CheckIfAlreadySeen("POP3 Item Seen",
589 RecvMsg->CurrMsg->MsgUID,
591 EvGetNow(IO) - USETABLE_ANTIEXPIRE,
595 return DBQueueEventContext(&RecvMsg->IO, POP3_C_ReAttachToFetchMessages);
597 eNextState POP3C_SaveMsg(AsyncIO *IO)
600 pop3aggr *RecvMsg = (pop3aggr *) IO->Data;
602 /* Do Something With It (tm) */
603 msgnum = CtdlSubmitMsg(RecvMsg->CurrMsg->Msg,
605 ChrPtr(RecvMsg->RoomName),
609 /* Message has been committed to the store
610 * write the uidl to the use table
611 * so we don't fetch this message again
614 CM_Free(RecvMsg->CurrMsg->Msg);
617 return NextDBOperation(&RecvMsg->IO, POP3C_StoreMsgRead);
620 eNextState POP3C_ReadMessageBody(pop3aggr *RecvMsg)
622 AsyncIO *IO = &RecvMsg->IO;
623 EVP3CM_syslog(LOG_DEBUG, "Converting message...");
624 RecvMsg->CurrMsg->Msg =
625 convert_internet_message_buf(&RecvMsg->IO.ReadMsg->MsgBuf);
626 return EventQueueDBOperation(&RecvMsg->IO, POP3C_SaveMsg, 0);
629 eNextState POP3C_SendDelete(pop3aggr *RecvMsg)
631 AsyncIO *IO = &RecvMsg->IO;
633 SetPOP3State(IO, eDelete);
635 if (!RecvMsg->keep) {
636 StrBufPrintf(RecvMsg->IO.SendBuf.Buf,
637 "DELE %ld\r\n", RecvMsg->CurrMsg->MSGID);
642 RecvMsg->State = ReadMessageBodyFollowing;
643 return POP3_C_DispatchWriteDone(&RecvMsg->IO);
646 eNextState POP3C_ReadDeleteState(pop3aggr *RecvMsg)
648 AsyncIO *IO = &RecvMsg->IO;
650 RecvMsg->State = GetOneMessageIDState;
654 eNextState POP3C_SendQuit(pop3aggr *RecvMsg)
656 AsyncIO *IO = &RecvMsg->IO;
657 SetPOP3State(IO, eQuit);
660 StrBufPlain(RecvMsg->IO.SendBuf.Buf,
667 eNextState POP3C_ReadQuitState(pop3aggr *RecvMsg)
669 AsyncIO *IO = &RecvMsg->IO;
671 return eTerminateConnection;
674 const long POP3_C_ConnTimeout = 1000;
675 const long DefaultPOP3Port = 110;
677 Pop3ClientHandler POP3C_ReadHandlers[] = {
681 POP3C_GetListCommandState,
682 POP3C_GetListOneLine,
683 POP3C_GetOneMessageIDState,
684 POP3C_ReadMessageBodyFollowing,
685 POP3C_ReadMessageBody,
686 POP3C_ReadDeleteState,
690 const long POP3_C_SendTimeouts[POP3C_MaxRead] = {
700 const ConstStr POP3C_ReadErrors[POP3C_MaxRead] = {
701 {HKEY("Connection broken during ")},
702 {HKEY("Connection broken during ")},
703 {HKEY("Connection broken during ")},
704 {HKEY("Connection broken during ")},
705 {HKEY("Connection broken during ")},
706 {HKEY("Connection broken during ")},
707 {HKEY("Connection broken during ")},
708 {HKEY("Connection broken during ")}
711 Pop3ClientHandler POP3C_SendHandlers[] = {
712 NULL, /* we don't send a greeting */
715 POP3C_SendListCommand,
717 POP3C_GetOneMessagID,
724 const long POP3_C_ReadTimeouts[] = {
736 /*****************************************************************************/
737 /* POP3 CLIENT DISPATCHER */
738 /*****************************************************************************/
740 void POP3SetTimeout(eNextState NextTCPState, pop3aggr *pMsg)
742 AsyncIO *IO = &pMsg->IO;
743 double Timeout = 0.0;
745 EVP3C_syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
747 switch (NextTCPState) {
751 Timeout = POP3_C_SendTimeouts[pMsg->State];
753 if (pMsg->State == eDATABody) {
754 / * if we're sending a huge message, we need more time. * /
755 Timeout += StrLength(pMsg->msgtext) / 1024;
761 Timeout = POP3_C_ReadTimeouts[pMsg->State];
763 if (pMsg->State == eDATATerminateBody) {
765 * some mailservers take a nap before accepting the message
766 * content inspection and such.
768 Timeout += StrLength(pMsg->msgtext) / 1024;
779 case eTerminateConnection:
782 case eReadMore://// TODO
785 SetNextTimeout(&pMsg->IO, Timeout);
787 eNextState POP3_C_DispatchReadDone(AsyncIO *IO)
789 /* EVP3CCS_syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__); to noisy anyways. */
790 pop3aggr *pMsg = IO->Data;
793 rc = POP3C_ReadHandlers[pMsg->State](pMsg);
796 POP3SetTimeout(rc, pMsg);
799 eNextState POP3_C_DispatchWriteDone(AsyncIO *IO)
801 pop3aggr *pMsg = IO->Data;
804 /* EVP3CCS_syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__); to noisy anyways. */
805 rc = POP3C_SendHandlers[pMsg->State](pMsg);
806 POP3SetTimeout(rc, pMsg);
811 /*****************************************************************************/
812 /* POP3 CLIENT ERROR CATCHERS */
813 /*****************************************************************************/
814 eNextState POP3_C_Terminate(AsyncIO *IO)
816 /// pop3aggr *pMsg = (pop3aggr *)IO->Data;
818 EVP3CCS_syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
819 FinalizePOP3AggrRun(IO);
822 eNextState POP3_C_TerminateDB(AsyncIO *IO)
824 /// pop3aggr *pMsg = (pop3aggr *)IO->Data;
826 EVP3CCS_syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
827 FinalizePOP3AggrRun(IO);
830 eNextState POP3_C_Timeout(AsyncIO *IO)
832 pop3aggr *pMsg = IO->Data;
834 EVP3CCS_syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
835 StrBufPlain(IO->ErrMsg, CKEY(POP3C_ReadErrors[pMsg->State]));
836 return FailAggregationRun(IO);
838 eNextState POP3_C_ConnFail(AsyncIO *IO)
840 pop3aggr *pMsg = (pop3aggr *)IO->Data;
842 EVP3CCS_syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
843 StrBufPlain(IO->ErrMsg, CKEY(POP3C_ReadErrors[pMsg->State]));
844 return FailAggregationRun(IO);
846 eNextState POP3_C_DNSFail(AsyncIO *IO)
848 pop3aggr *pMsg = (pop3aggr *)IO->Data;
850 EVP3CCS_syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
851 StrBufPlain(IO->ErrMsg, CKEY(POP3C_ReadErrors[pMsg->State]));
852 return FailAggregationRun(IO);
854 eNextState POP3_C_Shutdown(AsyncIO *IO)
856 EVP3CCS_syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
857 //// pop3aggr *pMsg = IO->Data;
859 ////pMsg->MyQEntry->Status = 3;
860 ///StrBufPlain(pMsg->MyQEntry->StatusMessage, HKEY("server shutdown during message retrieval."));
861 FinalizePOP3AggrRun(IO);
867 * @brief lineread Handler; understands when to read more POP3 lines,
868 * and when this is a one-lined reply.
870 eReadState POP3_C_ReadServerStatus(AsyncIO *IO)
872 eReadState Finished = eBufferNotEmpty;
874 switch (IO->NextState) {
879 case eTerminateConnection:
881 Finished = eReadFail;
888 Finished = StrBufChunkSipLine(IO->IOBuf, &IO->RecvBuf);
892 Finished = CtdlReadMessageBodyAsync(IO);
898 /*****************************************************************************
899 * So we connect our Server IP here. *
900 *****************************************************************************/
901 eNextState POP3_C_ReAttachToFetchMessages(AsyncIO *IO)
903 pop3aggr *cpptr = IO->Data;
905 EVP3CCS_syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
906 ////??? cpptr->State ++;
907 if (cpptr->Pos == NULL)
908 cpptr->Pos = GetNewHashPos(cpptr->MsgNumbers, 0);
910 POP3_C_DispatchWriteDone(IO);
911 ReAttachIO(IO, cpptr, 0);
912 IO->NextState = eReadMessage;
913 return IO->NextState;
916 eNextState pop3_connect_ip(AsyncIO *IO)
918 pop3aggr *cpptr = IO->Data;
920 if (cpptr->IOStart == 0.0) /* whith or without DNS? */
921 cpptr->IOStart = IO->Now;
923 EVP3CCS_syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
925 return EvConnectSock(IO,
927 POP3_C_ReadTimeouts[0],
931 eNextState pop3_get_one_host_ip_done(AsyncIO *IO)
933 pop3aggr *cpptr = IO->Data;
934 struct hostent *hostent;
938 hostent = cpptr->HostLookup.VParsedDNSReply;
939 if ((cpptr->HostLookup.DNSStatus == ARES_SUCCESS) &&
940 (hostent != NULL) ) {
941 memset(&cpptr->IO.ConnectMe->Addr, 0, sizeof(struct in6_addr));
942 if (cpptr->IO.ConnectMe->IPv6) {
943 memcpy(&cpptr->IO.ConnectMe->Addr.sin6_addr.s6_addr,
944 &hostent->h_addr_list[0],
945 sizeof(struct in6_addr));
947 cpptr->IO.ConnectMe->Addr.sin6_family =
949 cpptr->IO.ConnectMe->Addr.sin6_port =
950 htons(DefaultPOP3Port);
953 struct sockaddr_in *addr =
954 (struct sockaddr_in*)
955 &cpptr->IO.ConnectMe->Addr;
957 memcpy(&addr->sin_addr.s_addr,
958 hostent->h_addr_list[0],
961 addr->sin_family = hostent->h_addrtype;
962 addr->sin_port = htons(DefaultPOP3Port);
964 return pop3_connect_ip(IO);
970 eNextState pop3_get_one_host_ip(AsyncIO *IO)
972 pop3aggr *cpptr = IO->Data;
974 cpptr->IOStart = IO->Now;
976 EVP3CCS_syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
978 EVP3CCS_syslog(LOG_DEBUG,
979 "POP3 client[%ld]: looking up %s-Record %s : %d ...\n",
981 (cpptr->IO.ConnectMe->IPv6)? "aaaa": "a",
982 cpptr->IO.ConnectMe->Host,
983 cpptr->IO.ConnectMe->Port);
985 QueueQuery((cpptr->IO.ConnectMe->IPv6)? ns_t_aaaa : ns_t_a,
986 cpptr->IO.ConnectMe->Host,
989 pop3_get_one_host_ip_done);
990 IO->NextState = eReadDNSReply;
991 return IO->NextState;
996 int pop3_do_fetching(pop3aggr *cpptr)
998 AsyncIO *IO = &cpptr->IO;
1003 POP3_C_ReadServerStatus,
1005 POP3_C_DispatchWriteDone,
1006 POP3_C_DispatchReadDone,
1013 safestrncpy(((CitContext *)cpptr->IO.CitContext)->cs_host,
1015 sizeof(((CitContext *)cpptr->IO.CitContext)->cs_host));
1017 if (cpptr->IO.ConnectMe->IsIP) {
1018 QueueEventContext(&cpptr->IO,
1022 QueueEventContext(&cpptr->IO,
1023 pop3_get_one_host_ip);
1029 * Scan a room's netconfig to determine whether it requires POP3 aggregation
1031 void pop3client_scan_room(struct ctdlroom *qrbuf, void *data, OneRoomNetCfg *OneRNCFG)
1033 const RoomNetCfgLine *pLine;
1036 pthread_mutex_lock(&POP3QueueMutex);
1037 if (GetHash(POP3QueueRooms, LKEY(qrbuf->QRnumber), &vptr))
1039 pthread_mutex_unlock(&POP3QueueMutex);
1040 EVP3CQ_syslog(LOG_DEBUG,
1041 "pop3client: [%ld] %s already in progress.",
1046 pthread_mutex_unlock(&POP3QueueMutex);
1048 if (server_shutting_down) return;
1050 pLine = OneRNCFG->NetConfigs[pop3client];
1052 while (pLine != NULL)
1056 cptr = (pop3aggr *) malloc(sizeof(pop3aggr));
1057 memset(cptr, 0, sizeof(pop3aggr));
1058 ///TODO do we need this? cptr->roomlist_parts=1;
1059 cptr->RoomName = NewStrBufPlain(qrbuf->QRname, -1);
1060 cptr->pop3user = NewStrBufDup(pLine->Value[1]);
1061 cptr->pop3pass = NewStrBufDup(pLine->Value[2]);
1062 cptr->Url = NewStrBuf();
1063 cptr->Host = NewStrBufDup(pLine->Value[0]);
1065 cptr->keep = atol(ChrPtr(pLine->Value[3]));
1066 cptr->interval = atol(ChrPtr(pLine->Value[4]));
1068 StrBufAppendBufPlain(cptr->Url, HKEY("pop3://"), 0);
1069 StrBufUrlescUPAppend(cptr->Url, cptr->pop3user, NULL);
1070 StrBufAppendBufPlain(cptr->Url, HKEY(":"), 0);
1071 StrBufUrlescUPAppend(cptr->Url, cptr->pop3pass, NULL);
1072 StrBufAppendBufPlain(cptr->Url, HKEY("@"), 0);
1073 StrBufAppendBuf(cptr->Url, cptr->Host, 0);
1074 StrBufAppendBufPlain(cptr->Url, HKEY("/"), 0);
1075 StrBufUrlescAppend(cptr->Url, cptr->RoomName, NULL);
1077 ParseURL(&cptr->IO.ConnectMe, cptr->Url, 110);
1081 /* todo: we need to reunite the url to be shure. */
1083 pthread_mutex_lock(&POP3ueueMutex);
1084 GetHash(POP3FetchUrls, SKEY(ptr->Url), &vptr);
1085 use_this_cptr = (pop3aggr *)vptr;
1087 if (use_this_rncptr != NULL)
1089 /* mustn't attach to an active session */
1090 if (use_this_cptr->RefCount > 0)
1092 DeletePOP3Cfg(cptr);
1098 StrBufAppendBufPlain(
1099 use_this_cptr->rooms,
1102 if (use_this_cptr->roomlist_parts == 1)
1104 use_this_cptr->OtherQRnumbers
1105 = NewHash(1, lFlathash);
1107 QRnumber = (long*)malloc(sizeof(long));
1108 *QRnumber = qrbuf->QRnumber;
1109 Put(use_this_cptr->OtherQRnumbers,
1110 LKEY(qrbuf->QRnumber),
1114 use_this_cptr->roomlist_parts++;
1116 pthread_mutex_unlock(&POP3QueueMutex);
1119 pthread_mutex_unlock(&RSSQueueMutex);
1121 cptr->n = Pop3ClientID++;
1122 pthread_mutex_lock(&POP3QueueMutex);
1126 DeletePOP3Aggregator);
1128 pthread_mutex_unlock(&POP3QueueMutex);
1129 pLine = pLine->next;
1134 static int doing_pop3client = 0;
1136 void pop3client_scan(void) {
1137 static time_t last_run = 0L;
1138 time_t fastest_scan;
1145 become_session(&pop3_client_CC);
1147 if (CtdlGetConfigLong("c_pop3_fastest") < CtdlGetConfigLong("c_pop3_fetch"))
1148 fastest_scan = CtdlGetConfigLong("c_pop3_fastest");
1150 fastest_scan = CtdlGetConfigLong("c_pop3_fetch");
1153 * Run POP3 aggregation no more frequently than once every n seconds
1155 if ( (time(NULL) - last_run) < fastest_scan ) {
1160 * This is a simple concurrency check to make sure only one pop3client
1161 * run is done at a time. We could do this with a mutex, but since we
1162 * don't really require extremely fine granularity here, we'll do it
1163 * with a static variable instead.
1165 if (doing_pop3client) return;
1166 doing_pop3client = 1;
1168 EVP3CQM_syslog(LOG_DEBUG, "pop3client started");
1169 CtdlForEachNetCfgRoom(pop3client_scan_room, NULL, pop3client);
1171 pthread_mutex_lock(&POP3QueueMutex);
1172 it = GetNewHashPos(POP3FetchUrls, 0);
1173 while (!server_shutting_down &&
1174 GetNextHashPos(POP3FetchUrls, it, &len, &Key, &vrptr) &&
1176 cptr = (pop3aggr *)vrptr;
1177 if (cptr->RefCount == 0)
1178 if (!pop3_do_fetching(cptr))
1179 DeletePOP3Aggregator(cptr);////TODO
1182 if ((palist->interval && time(NULL) > (last_run + palist->interval))
1183 || (time(NULL) > last_run + CtdlGetConfigLong("c_pop3_fetch")))
1184 pop3_do_fetching(palist->roomname, palist->pop3host,
1185 palist->pop3user, palist->pop3pass, palist->keep);
1187 palist = palist->next;
1192 pthread_mutex_unlock(&POP3QueueMutex);
1194 EVP3CQM_syslog(LOG_DEBUG, "pop3client ended");
1195 last_run = time(NULL);
1196 doing_pop3client = 0;
1200 void pop3_cleanup(void)
1202 /* citthread_mutex_destroy(&POP3QueueMutex); TODO */
1203 while (doing_pop3client != 0) ;
1204 DeleteHash(&POP3FetchUrls);
1205 DeleteHash(&POP3QueueRooms);
1210 void LogDebugEnablePOP3Client(const int n)
1212 POP3ClientDebugEnabled = n;
1215 CTDL_MODULE_INIT(pop3client)
1219 CtdlFillSystemContext(&pop3_client_CC, "POP3aggr");
1220 CtdlREGISTERRoomCfgType(pop3client, ParseGeneric, 0, 5, SerializeGeneric, DeleteGenericCfgLine);
1221 pthread_mutex_init(&POP3QueueMutex, NULL);
1222 POP3QueueRooms = NewHash(1, lFlathash);
1223 POP3FetchUrls = NewHash(1, NULL);
1224 CtdlRegisterSessionHook(pop3client_scan, EVT_TIMER, PRIO_AGGR + 50);
1225 CtdlRegisterEVCleanupHook(pop3_cleanup);
1226 CtdlRegisterDebugFlagHook(HKEY("pop3client"), LogDebugEnablePOP3Client, &POP3ClientDebugEnabled);
1229 /* return our module id for the log */
1230 return "pop3client";