2 * Consolidate mail from remote POP3 accounts.
4 * Copyright (c) 2007-2011 by the citadel.org team
6 * This program is open source software; you can redistribute it and/or
7 * modify it under the terms of the GNU General Public License as published
8 * by the Free Software Foundation; either version 3 of the License, or
9 * (at your option) any later version.
11 * This program is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 * GNU General Public License for more details.
16 * You should have received a copy of the GNU General Public License
17 * along with this program; if not, write to the Free Software
18 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
24 #include <sysconfig.h>
26 #if TIME_WITH_SYS_TIME
27 # include <sys/time.h>
31 # include <sys/time.h>
40 #include <sys/types.h>
42 #include <libcitadel.h>
45 #include "citserver.h"
48 #include "ctdl_module.h"
49 #include "clientsocket.h"
51 #include "internet_addressing.h"
53 #include "citadel_dirs.h"
54 #include "event_client.h"
57 #define POP3C_OK (strncasecmp(ChrPtr(RecvMsg->IO.IOBuf), "+OK", 3) == 0)
59 int POP3ClientDebugEnabled = 0;
61 #define N ((pop3aggr*)IO->Data)->n
63 #define DBGLOG(LEVEL) if ((LEVEL != LOG_DEBUG) || (POP3ClientDebugEnabled != 0))
65 #define EVP3C_syslog(LEVEL, FORMAT, ...) \
66 DBGLOG(LEVEL) syslog(LEVEL, \
67 "%s[%ld]CC[%d][%ld]POP3: " FORMAT, \
68 IOSTR, IO->ID, CCID, N, __VA_ARGS__)
70 #define EVP3CM_syslog(LEVEL, FORMAT) \
71 DBGLOG(LEVEL) syslog(LEVEL, \
72 "%s[%ld]CC[%d][%ld]POP3: " FORMAT, \
73 IOSTR, IO->ID, CCID, N)
75 #define EVP3CQ_syslog(LEVEL, FORMAT, ...) \
76 DBGLOG(LEVEL) syslog(LEVEL, \
80 #define EVP3CQM_syslog(LEVEL, FORMAT) \
81 DBGLOG(LEVEL) syslog(LEVEL, \
85 #define EVP3CCS_syslog(LEVEL, FORMAT, ...) \
86 DBGLOG(LEVEL) syslog(LEVEL, "%s[%ld][%ld]POP3: " FORMAT, \
87 IOSTR, IO->ID, N, __VA_ARGS__)
89 #define EVP3CCSM_syslog(LEVEL, FORMAT) \
90 DBGLOG(LEVEL) syslog(LEVEL, "%s[%ld][%ld]POP3: " FORMAT, \
93 #define POP3C_DBG_SEND() \
94 EVP3C_syslog(LOG_DEBUG, \
95 "%s[%ld]CC[%d][%ld]POP3: > %s\n", \
96 IOSTR, IO->ID, CCID, N, \
97 ChrPtr(RecvMsg->IO.SendBuf.Buf))
99 #define POP3C_DBG_READ() \
100 EVP3C_syslog(LOG_DEBUG, \
101 "%s[%ld]CC[%d][%ld]POP3: < %s\n", \
102 IOSTR, IO->ID, CCID, N, \
103 ChrPtr(RecvMsg->IO.IOBuf))
106 struct CitContext pop3_client_CC;
108 pthread_mutex_t POP3QueueMutex; /* locks the access to the following vars: */
109 HashList *POP3QueueRooms = NULL;
110 HashList *POP3FetchUrls = NULL;
112 typedef struct pop3aggr pop3aggr;
113 typedef eNextState(*Pop3ClientHandler)(pop3aggr* RecvMsg);
115 eNextState POP3_C_Shutdown(AsyncIO *IO);
116 eNextState POP3_C_Timeout(AsyncIO *IO);
117 eNextState POP3_C_ConnFail(AsyncIO *IO);
118 eNextState POP3_C_DNSFail(AsyncIO *IO);
119 eNextState POP3_C_DispatchReadDone(AsyncIO *IO);
120 eNextState POP3_C_DispatchWriteDone(AsyncIO *IO);
121 eNextState POP3_C_Terminate(AsyncIO *IO);
122 eReadState POP3_C_ReadServerStatus(AsyncIO *IO);
123 eNextState POP3_C_ReAttachToFetchMessages(AsyncIO *IO);
125 typedef struct __pop3_room_counter {
130 typedef enum ePOP3_C_States {
136 GetOneMessageIDState,
137 ReadMessageBodyFollowing,
145 typedef struct _FetchItem {
151 struct CtdlMessage *Msg;
154 void HfreeFetchItem(void *vItem)
156 FetchItem *Item = (FetchItem*) vItem;
157 FreeStrBuf(&Item->MsgUIDL);
158 FreeStrBuf(&Item->MsgUID);
164 typedef enum _POP3State {
178 ConstStr POP3States[] = {
179 {HKEY("Aggregator created")},
180 {HKEY("Reading Greeting")},
181 {HKEY("Sending User")},
182 {HKEY("Sending Password")},
184 {HKEY("Fetching Usetable")},
185 {HKEY("Get MSG ID")},
186 {HKEY("Get Message")},
188 {HKEY("Delete Upstream")},
192 static void SetPOP3State(AsyncIO *IO, POP3State State)
194 CitContext* CCC = IO->CitContext;
196 memcpy(CCC->cs_clientname, POP3States[State].Key, POP3States[State].len + 1);
207 DNSQueryParts HostLookup;
210 HashList *OtherQRnumbers;
216 StrBuf *RoomName; // TODO: fill me
219 ePOP3_C_States State;
220 HashList *MsgNumbers;
225 void DeletePOP3Aggregator(void *vptr)
227 pop3aggr *ptr = vptr;
228 DeleteHashPos(&ptr->Pos);
229 DeleteHash(&ptr->MsgNumbers);
230 // FreeStrBuf(&ptr->rooms);
231 FreeStrBuf(&ptr->pop3user);
232 FreeStrBuf(&ptr->pop3pass);
233 FreeStrBuf(&ptr->Host);
234 FreeStrBuf(&ptr->RoomName);
235 FreeURL(&ptr->IO.ConnectMe);
236 FreeStrBuf(&ptr->Url);
237 FreeStrBuf(&ptr->IO.IOBuf);
238 FreeStrBuf(&ptr->IO.SendBuf.Buf);
239 FreeStrBuf(&ptr->IO.RecvBuf.Buf);
240 DeleteAsyncMsg(&ptr->IO.ReadMsg);
241 if (((struct CitContext*)ptr->IO.CitContext)) {
242 ((struct CitContext*)ptr->IO.CitContext)->state = CON_IDLE;
243 ((struct CitContext*)ptr->IO.CitContext)->kill_me = 1;
245 FreeAsyncIOContents(&ptr->IO);
249 eNextState FinalizePOP3AggrRun(AsyncIO *IO)
252 pop3aggr *cpptr = (pop3aggr *)IO->Data;
254 EVP3C_syslog(LOG_INFO,
255 "%s@%s: fetched %ld new of %d messages in %fs. bye.",
256 ChrPtr(cpptr->pop3user),
259 GetCount(cpptr->MsgNumbers),
260 IO->Now - cpptr->IOStart
263 It = GetNewHashPos(POP3FetchUrls, 0);
264 pthread_mutex_lock(&POP3QueueMutex);
266 if (GetHashPosFromKey(POP3FetchUrls, SKEY(cpptr->Url), It))
267 DeleteEntryFromHash(POP3FetchUrls, It);
269 pthread_mutex_unlock(&POP3QueueMutex);
274 eNextState FailAggregationRun(AsyncIO *IO)
279 eNextState POP3C_ReadGreeting(pop3aggr *RecvMsg)
281 AsyncIO *IO = &RecvMsg->IO;
282 SetPOP3State(IO, eGreeting);
284 /* Read the server greeting */
285 if (!POP3C_OK) return eTerminateConnection;
286 else return eSendReply;
289 eNextState POP3C_SendUser(pop3aggr *RecvMsg)
291 AsyncIO *IO = &RecvMsg->IO;
292 SetPOP3State(IO, eUser);
293 /* Identify ourselves. NOTE: we have to append a CR to each command.
294 * The LF will automatically be appended by sock_puts(). Believe it
295 * or not, leaving out the CR will cause problems if the server happens
296 * to be Exchange, which is so b0rken it actually barfs on
297 * LF-terminated newlines.
299 StrBufPrintf(RecvMsg->IO.SendBuf.Buf,
300 "USER %s\r\n", ChrPtr(RecvMsg->pop3user));
305 eNextState POP3C_GetUserState(pop3aggr *RecvMsg)
307 AsyncIO *IO = &RecvMsg->IO;
309 if (!POP3C_OK) return eTerminateConnection;
310 else return eSendReply;
313 eNextState POP3C_SendPassword(pop3aggr *RecvMsg)
315 AsyncIO *IO = &RecvMsg->IO;
316 SetPOP3State(IO, ePassword);
318 StrBufPrintf(RecvMsg->IO.SendBuf.Buf,
319 "PASS %s\r\n", ChrPtr(RecvMsg->pop3pass));
320 EVP3CM_syslog(LOG_DEBUG, "<PASS <password>\n");
321 // POP3C_DBG_SEND(); No, we won't write the passvoid to syslog...
325 eNextState POP3C_GetPassState(pop3aggr *RecvMsg)
327 AsyncIO *IO = &RecvMsg->IO;
329 if (!POP3C_OK) return eTerminateConnection;
330 else return eSendReply;
333 eNextState POP3C_SendListCommand(pop3aggr *RecvMsg)
335 AsyncIO *IO = &RecvMsg->IO;
336 SetPOP3State(IO, eListing);
338 /* Get the list of messages */
339 StrBufPlain(RecvMsg->IO.SendBuf.Buf, HKEY("LIST\r\n"));
344 eNextState POP3C_GetListCommandState(pop3aggr *RecvMsg)
346 AsyncIO *IO = &RecvMsg->IO;
348 if (!POP3C_OK) return eTerminateConnection;
349 RecvMsg->MsgNumbers = NewHash(1, NULL);
355 eNextState POP3C_GetListOneLine(pop3aggr *RecvMsg)
357 AsyncIO *IO = &RecvMsg->IO;
362 FetchItem *OneMsg = NULL;
365 if ((StrLength(RecvMsg->IO.IOBuf) == 1) &&
366 (ChrPtr(RecvMsg->IO.IOBuf)[0] == '.'))
368 if (GetCount(RecvMsg->MsgNumbers) == 0)
370 //// RecvMsg->Sate = ReadQuitState;
374 RecvMsg->Pos = GetNewHashPos(RecvMsg->MsgNumbers, 0);
381 * work around buggy pop3 servers which send
382 * empty lines in their listings.
384 if ((StrLength(RecvMsg->IO.IOBuf) == 0) ||
385 !isdigit(ChrPtr(RecvMsg->IO.IOBuf)[0]))
390 OneMsg = (FetchItem*) malloc(sizeof(FetchItem));
391 memset(OneMsg, 0, sizeof(FetchItem));
392 OneMsg->MSGID = atol(ChrPtr(RecvMsg->IO.IOBuf));
394 pch = strchr(ChrPtr(RecvMsg->IO.IOBuf), ' ');
397 OneMsg->MSGSize = atol(pch + 1);
400 rc = TestValidateHash(RecvMsg->MsgNumbers);
402 EVP3CCS_syslog(LOG_DEBUG, "Hash Invalid: %d\n", rc);
405 Put(RecvMsg->MsgNumbers, LKEY(OneMsg->MSGID), OneMsg, HfreeFetchItem);
407 rc = TestValidateHash(RecvMsg->MsgNumbers);
409 EVP3CCS_syslog(LOG_DEBUG, "Hash Invalid: %d\n", rc);
411 //RecvMsg->State --; /* read next Line */
415 eNextState POP3_FetchNetworkUsetableEntry(AsyncIO *IO)
420 pop3aggr *RecvMsg = (pop3aggr *) IO->Data;
421 time_t seenstamp = 0;
423 SetPOP3State(IO, eUseTable);
425 if((RecvMsg->Pos != NULL) &&
426 GetNextHashPos(RecvMsg->MsgNumbers,
432 if (server_shutting_down)
435 RecvMsg->CurrMsg = (FetchItem*)vData;
437 seenstamp = CheckIfAlreadySeen("POP3 Item Seen",
438 RecvMsg->CurrMsg->MsgUID,
440 EvGetNow(IO) - USETABLE_ANTIEXPIRE,
445 /* Item has already been seen */
446 RecvMsg->CurrMsg->NeedFetch = 0;
450 EVP3CCSM_syslog(LOG_DEBUG, "NO\n");
451 RecvMsg->CurrMsg->NeedFetch = 1;
453 return NextDBOperation(&RecvMsg->IO,
454 POP3_FetchNetworkUsetableEntry);
458 /* ok, now we know them all,
459 * continue with reading the actual messages. */
460 DeleteHashPos(&RecvMsg->Pos);
461 return DBQueueEventContext(IO, POP3_C_ReAttachToFetchMessages);
465 eNextState POP3C_GetOneMessagID(pop3aggr *RecvMsg)
467 AsyncIO *IO = &RecvMsg->IO;
472 SetPOP3State(IO, eGetMsgID);
475 rc = TestValidateHash(RecvMsg->MsgNumbers);
477 EVP3CCS_syslog(LOG_DEBUG, "Hash Invalid: %d\n", rc);
479 if((RecvMsg->Pos != NULL) &&
480 GetNextHashPos(RecvMsg->MsgNumbers,
485 RecvMsg->CurrMsg = (FetchItem*) vData;
486 /* Find out the UIDL of the message,
487 * to determine whether we've already downloaded it */
488 StrBufPrintf(RecvMsg->IO.SendBuf.Buf,
489 "UIDL %ld\r\n", RecvMsg->CurrMsg->MSGID);
495 DeleteHashPos(&RecvMsg->Pos);
496 /// done receiving uidls.. start looking them up now.
497 RecvMsg->Pos = GetNewHashPos(RecvMsg->MsgNumbers, 0);
498 return EventQueueDBOperation(&RecvMsg->IO,
499 POP3_FetchNetworkUsetableEntry,
502 return eReadMore; /* TODO */
505 eNextState POP3C_GetOneMessageIDState(pop3aggr *RecvMsg)
507 AsyncIO *IO = &RecvMsg->IO;
510 rc = TestValidateHash(RecvMsg->MsgNumbers);
512 EVP3CCS_syslog(LOG_DEBUG, "Hash Invalid: %d\n", rc);
516 if (!POP3C_OK) return eTerminateConnection;
517 RecvMsg->CurrMsg->MsgUIDL =
518 NewStrBufPlain(NULL, StrLength(RecvMsg->IO.IOBuf));
519 RecvMsg->CurrMsg->MsgUID =
520 NewStrBufPlain(NULL, StrLength(RecvMsg->IO.IOBuf) * 2);
522 StrBufExtract_token(RecvMsg->CurrMsg->MsgUIDL,
523 RecvMsg->IO.IOBuf, 2, ' ');
525 StrBufPrintf(RecvMsg->CurrMsg->MsgUID,
527 ChrPtr(RecvMsg->RoomName),
528 ChrPtr(RecvMsg->CurrMsg->MsgUIDL),
529 RecvMsg->IO.ConnectMe->User,
530 RecvMsg->IO.ConnectMe->Host);
536 eNextState POP3C_SendGetOneMsg(pop3aggr *RecvMsg)
538 AsyncIO *IO = &RecvMsg->IO;
543 SetPOP3State(IO, eGetMsg);
545 EVP3CM_syslog(LOG_DEBUG, "fast forwarding to the next unknown message");
547 RecvMsg->CurrMsg = NULL;
548 while ((RecvMsg->Pos != NULL) &&
549 GetNextHashPos(RecvMsg->MsgNumbers,
553 (RecvMsg->CurrMsg = (FetchItem*) vData,
554 RecvMsg->CurrMsg->NeedFetch == 0))
557 if ((RecvMsg->CurrMsg != NULL ) && (RecvMsg->CurrMsg->NeedFetch == 1))
559 EVP3CM_syslog(LOG_DEBUG, "fetching next");
560 /* Message has not been seen.
561 * Tell the server to fetch the message... */
562 StrBufPrintf(RecvMsg->IO.SendBuf.Buf,
563 "RETR %ld\r\n", RecvMsg->CurrMsg->MSGID);
568 EVP3CM_syslog(LOG_DEBUG, "no more messages to fetch.");
569 RecvMsg->State = ReadQuitState;
570 return POP3_C_DispatchWriteDone(&RecvMsg->IO);
575 eNextState POP3C_ReadMessageBodyFollowing(pop3aggr *RecvMsg)
577 AsyncIO *IO = &RecvMsg->IO;
579 if (!POP3C_OK) return eTerminateConnection;
580 RecvMsg->IO.ReadMsg = NewAsyncMsg(HKEY("."),
581 RecvMsg->CurrMsg->MSGSize,
590 eNextState POP3C_StoreMsgRead(AsyncIO *IO)
592 pop3aggr *RecvMsg = (pop3aggr *) IO->Data;
594 SetPOP3State(IO, eStoreMsg);
596 EVP3CCS_syslog(LOG_DEBUG,
597 "MARKING: %s as seen: ",
598 ChrPtr(RecvMsg->CurrMsg->MsgUID));
599 CheckIfAlreadySeen("POP3 Item Seen",
600 RecvMsg->CurrMsg->MsgUID,
602 EvGetNow(IO) - USETABLE_ANTIEXPIRE,
606 return DBQueueEventContext(&RecvMsg->IO, POP3_C_ReAttachToFetchMessages);
608 eNextState POP3C_SaveMsg(AsyncIO *IO)
611 pop3aggr *RecvMsg = (pop3aggr *) IO->Data;
613 /* Do Something With It (tm) */
614 msgnum = CtdlSubmitMsg(RecvMsg->CurrMsg->Msg,
616 ChrPtr(RecvMsg->RoomName),
620 /* Message has been committed to the store
621 * write the uidl to the use table
622 * so we don't fetch this message again
625 CM_Free(RecvMsg->CurrMsg->Msg);
628 return NextDBOperation(&RecvMsg->IO, POP3C_StoreMsgRead);
631 eNextState POP3C_ReadMessageBody(pop3aggr *RecvMsg)
633 AsyncIO *IO = &RecvMsg->IO;
634 EVP3CM_syslog(LOG_DEBUG, "Converting message...");
635 RecvMsg->CurrMsg->Msg =
636 convert_internet_message_buf(&RecvMsg->IO.ReadMsg->MsgBuf);
637 return EventQueueDBOperation(&RecvMsg->IO, POP3C_SaveMsg, 0);
640 eNextState POP3C_SendDelete(pop3aggr *RecvMsg)
642 AsyncIO *IO = &RecvMsg->IO;
644 SetPOP3State(IO, eDelete);
646 if (!RecvMsg->keep) {
647 StrBufPrintf(RecvMsg->IO.SendBuf.Buf,
648 "DELE %ld\r\n", RecvMsg->CurrMsg->MSGID);
653 RecvMsg->State = ReadMessageBodyFollowing;
654 return POP3_C_DispatchWriteDone(&RecvMsg->IO);
657 eNextState POP3C_ReadDeleteState(pop3aggr *RecvMsg)
659 AsyncIO *IO = &RecvMsg->IO;
661 RecvMsg->State = GetOneMessageIDState;
662 return POP3_C_DispatchWriteDone(&RecvMsg->IO);
665 eNextState POP3C_SendQuit(pop3aggr *RecvMsg)
667 AsyncIO *IO = &RecvMsg->IO;
668 SetPOP3State(IO, eQuit);
671 StrBufPlain(RecvMsg->IO.SendBuf.Buf,
678 eNextState POP3C_ReadQuitState(pop3aggr *RecvMsg)
680 AsyncIO *IO = &RecvMsg->IO;
682 return eTerminateConnection;
685 const long POP3_C_ConnTimeout = 1000;
686 const long DefaultPOP3Port = 110;
688 Pop3ClientHandler POP3C_ReadHandlers[] = {
692 POP3C_GetListCommandState,
693 POP3C_GetListOneLine,
694 POP3C_GetOneMessageIDState,
695 POP3C_ReadMessageBodyFollowing,
696 POP3C_ReadMessageBody,
697 POP3C_ReadDeleteState,
701 const long POP3_C_SendTimeouts[POP3C_MaxRead] = {
711 const ConstStr POP3C_ReadErrors[POP3C_MaxRead] = {
712 {HKEY("Connection broken during ")},
713 {HKEY("Connection broken during ")},
714 {HKEY("Connection broken during ")},
715 {HKEY("Connection broken during ")},
716 {HKEY("Connection broken during ")},
717 {HKEY("Connection broken during ")},
718 {HKEY("Connection broken during ")},
719 {HKEY("Connection broken during ")}
722 Pop3ClientHandler POP3C_SendHandlers[] = {
723 NULL, /* we don't send a greeting */
726 POP3C_SendListCommand,
728 POP3C_GetOneMessagID,
735 const long POP3_C_ReadTimeouts[] = {
747 /*****************************************************************************/
748 /* POP3 CLIENT DISPATCHER */
749 /*****************************************************************************/
751 void POP3SetTimeout(eNextState NextTCPState, pop3aggr *pMsg)
753 AsyncIO *IO = &pMsg->IO;
754 double Timeout = 0.0;
756 EVP3C_syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
758 switch (NextTCPState) {
762 Timeout = POP3_C_SendTimeouts[pMsg->State];
764 if (pMsg->State == eDATABody) {
765 / * if we're sending a huge message, we need more time. * /
766 Timeout += StrLength(pMsg->msgtext) / 1024;
772 Timeout = POP3_C_ReadTimeouts[pMsg->State];
774 if (pMsg->State == eDATATerminateBody) {
776 * some mailservers take a nap before accepting the message
777 * content inspection and such.
779 Timeout += StrLength(pMsg->msgtext) / 1024;
790 case eTerminateConnection:
793 case eReadMore://// TODO
796 SetNextTimeout(&pMsg->IO, Timeout);
798 eNextState POP3_C_DispatchReadDone(AsyncIO *IO)
800 /* EVP3CCS_syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__); to noisy anyways. */
801 pop3aggr *pMsg = IO->Data;
804 rc = POP3C_ReadHandlers[pMsg->State](pMsg);
807 POP3SetTimeout(rc, pMsg);
810 eNextState POP3_C_DispatchWriteDone(AsyncIO *IO)
812 pop3aggr *pMsg = IO->Data;
815 /* EVP3CCS_syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__); to noisy anyways. */
816 rc = POP3C_SendHandlers[pMsg->State](pMsg);
817 POP3SetTimeout(rc, pMsg);
822 /*****************************************************************************/
823 /* POP3 CLIENT ERROR CATCHERS */
824 /*****************************************************************************/
825 eNextState POP3_C_Terminate(AsyncIO *IO)
827 /// pop3aggr *pMsg = (pop3aggr *)IO->Data;
829 EVP3CCS_syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
830 FinalizePOP3AggrRun(IO);
833 eNextState POP3_C_TerminateDB(AsyncIO *IO)
835 /// pop3aggr *pMsg = (pop3aggr *)IO->Data;
837 EVP3CCS_syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
838 FinalizePOP3AggrRun(IO);
841 eNextState POP3_C_Timeout(AsyncIO *IO)
843 pop3aggr *pMsg = IO->Data;
845 EVP3CCS_syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
846 StrBufPlain(IO->ErrMsg, CKEY(POP3C_ReadErrors[pMsg->State]));
847 return FailAggregationRun(IO);
849 eNextState POP3_C_ConnFail(AsyncIO *IO)
851 pop3aggr *pMsg = (pop3aggr *)IO->Data;
853 EVP3CCS_syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
854 StrBufPlain(IO->ErrMsg, CKEY(POP3C_ReadErrors[pMsg->State]));
855 return FailAggregationRun(IO);
857 eNextState POP3_C_DNSFail(AsyncIO *IO)
859 pop3aggr *pMsg = (pop3aggr *)IO->Data;
861 EVP3CCS_syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
862 StrBufPlain(IO->ErrMsg, CKEY(POP3C_ReadErrors[pMsg->State]));
863 return FailAggregationRun(IO);
865 eNextState POP3_C_Shutdown(AsyncIO *IO)
867 EVP3CCS_syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
868 //// pop3aggr *pMsg = IO->Data;
870 ////pMsg->MyQEntry->Status = 3;
871 ///StrBufPlain(pMsg->MyQEntry->StatusMessage, HKEY("server shutdown during message retrieval."));
872 FinalizePOP3AggrRun(IO);
878 * @brief lineread Handler; understands when to read more POP3 lines,
879 * and when this is a one-lined reply.
881 eReadState POP3_C_ReadServerStatus(AsyncIO *IO)
883 eReadState Finished = eBufferNotEmpty;
885 switch (IO->NextState) {
890 case eTerminateConnection:
892 Finished = eReadFail;
899 Finished = StrBufChunkSipLine(IO->IOBuf, &IO->RecvBuf);
903 Finished = CtdlReadMessageBodyAsync(IO);
909 /*****************************************************************************
910 * So we connect our Server IP here. *
911 *****************************************************************************/
912 eNextState POP3_C_ReAttachToFetchMessages(AsyncIO *IO)
914 pop3aggr *cpptr = IO->Data;
916 EVP3CCS_syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
917 ////??? cpptr->State ++;
918 if (cpptr->Pos == NULL)
919 cpptr->Pos = GetNewHashPos(cpptr->MsgNumbers, 0);
921 POP3_C_DispatchWriteDone(IO);
922 ReAttachIO(IO, cpptr, 0);
923 IO->NextState = eReadMessage;
924 return IO->NextState;
927 eNextState pop3_connect_ip(AsyncIO *IO)
929 pop3aggr *cpptr = IO->Data;
931 if (cpptr->IOStart == 0.0) /* whith or without DNS? */
932 cpptr->IOStart = IO->Now;
934 EVP3CCS_syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
936 return EvConnectSock(IO,
938 POP3_C_ReadTimeouts[0],
942 eNextState pop3_get_one_host_ip_done(AsyncIO *IO)
944 pop3aggr *cpptr = IO->Data;
945 struct hostent *hostent;
949 hostent = cpptr->HostLookup.VParsedDNSReply;
950 if ((cpptr->HostLookup.DNSStatus == ARES_SUCCESS) &&
951 (hostent != NULL) ) {
952 memset(&cpptr->IO.ConnectMe->Addr, 0, sizeof(struct in6_addr));
953 if (cpptr->IO.ConnectMe->IPv6) {
954 memcpy(&cpptr->IO.ConnectMe->Addr.sin6_addr.s6_addr,
955 &hostent->h_addr_list[0],
956 sizeof(struct in6_addr));
958 cpptr->IO.ConnectMe->Addr.sin6_family =
960 cpptr->IO.ConnectMe->Addr.sin6_port =
961 htons(DefaultPOP3Port);
964 struct sockaddr_in *addr =
965 (struct sockaddr_in*)
966 &cpptr->IO.ConnectMe->Addr;
968 memcpy(&addr->sin_addr.s_addr,
969 hostent->h_addr_list[0],
972 addr->sin_family = hostent->h_addrtype;
973 addr->sin_port = htons(DefaultPOP3Port);
975 return pop3_connect_ip(IO);
981 eNextState pop3_get_one_host_ip(AsyncIO *IO)
983 pop3aggr *cpptr = IO->Data;
985 cpptr->IOStart = IO->Now;
987 EVP3CCS_syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
989 EVP3CCS_syslog(LOG_DEBUG,
990 "POP3 client[%ld]: looking up %s-Record %s : %d ...\n",
992 (cpptr->IO.ConnectMe->IPv6)? "aaaa": "a",
993 cpptr->IO.ConnectMe->Host,
994 cpptr->IO.ConnectMe->Port);
996 QueueQuery((cpptr->IO.ConnectMe->IPv6)? ns_t_aaaa : ns_t_a,
997 cpptr->IO.ConnectMe->Host,
1000 pop3_get_one_host_ip_done);
1001 IO->NextState = eReadDNSReply;
1002 return IO->NextState;
1007 int pop3_do_fetching(pop3aggr *cpptr)
1009 AsyncIO *IO = &cpptr->IO;
1014 POP3_C_ReadServerStatus,
1016 POP3_C_DispatchWriteDone,
1017 POP3_C_DispatchReadDone,
1024 safestrncpy(((CitContext *)cpptr->IO.CitContext)->cs_host,
1026 sizeof(((CitContext *)cpptr->IO.CitContext)->cs_host));
1028 if (cpptr->IO.ConnectMe->IsIP) {
1029 QueueEventContext(&cpptr->IO,
1033 QueueEventContext(&cpptr->IO,
1034 pop3_get_one_host_ip);
1040 * Scan a room's netconfig to determine whether it requires POP3 aggregation
1042 void pop3client_scan_room(struct ctdlroom *qrbuf, void *data, OneRoomNetCfg *OneRNCFG)
1044 const RoomNetCfgLine *pLine;
1047 pthread_mutex_lock(&POP3QueueMutex);
1048 if (GetHash(POP3QueueRooms, LKEY(qrbuf->QRnumber), &vptr))
1050 pthread_mutex_unlock(&POP3QueueMutex);
1051 EVP3CQ_syslog(LOG_DEBUG,
1052 "pop3client: [%ld] %s already in progress.",
1057 pthread_mutex_unlock(&POP3QueueMutex);
1059 if (server_shutting_down) return;
1061 pLine = OneRNCFG->NetConfigs[pop3client];
1063 while (pLine != NULL)
1067 cptr = (pop3aggr *) malloc(sizeof(pop3aggr));
1068 memset(cptr, 0, sizeof(pop3aggr));
1069 ///TODO do we need this? cptr->roomlist_parts=1;
1070 cptr->RoomName = NewStrBufPlain(qrbuf->QRname, -1);
1071 cptr->pop3user = NewStrBufDup(pLine->Value[1]);
1072 cptr->pop3pass = NewStrBufDup(pLine->Value[2]);
1073 cptr->Url = NewStrBuf();
1074 cptr->Host = NewStrBufDup(pLine->Value[0]);
1076 cptr->keep = atol(ChrPtr(pLine->Value[3]));
1077 cptr->interval = atol(ChrPtr(pLine->Value[4]));
1079 StrBufAppendBufPlain(cptr->Url, HKEY("pop3://"), 0);
1080 StrBufUrlescUPAppend(cptr->Url, cptr->pop3user, NULL);
1081 StrBufAppendBufPlain(cptr->Url, HKEY(":"), 0);
1082 StrBufUrlescUPAppend(cptr->Url, cptr->pop3pass, NULL);
1083 StrBufAppendBufPlain(cptr->Url, HKEY("@"), 0);
1084 StrBufAppendBuf(cptr->Url, cptr->Host, 0);
1085 StrBufAppendBufPlain(cptr->Url, HKEY("/"), 0);
1086 StrBufUrlescAppend(cptr->Url, cptr->RoomName, NULL);
1088 ParseURL(&cptr->IO.ConnectMe, cptr->Url, 110);
1092 /* todo: we need to reunite the url to be shure. */
1094 pthread_mutex_lock(&POP3ueueMutex);
1095 GetHash(POP3FetchUrls, SKEY(ptr->Url), &vptr);
1096 use_this_cptr = (pop3aggr *)vptr;
1098 if (use_this_rncptr != NULL)
1100 /* mustn't attach to an active session */
1101 if (use_this_cptr->RefCount > 0)
1103 DeletePOP3Cfg(cptr);
1109 StrBufAppendBufPlain(
1110 use_this_cptr->rooms,
1113 if (use_this_cptr->roomlist_parts == 1)
1115 use_this_cptr->OtherQRnumbers
1116 = NewHash(1, lFlathash);
1118 QRnumber = (long*)malloc(sizeof(long));
1119 *QRnumber = qrbuf->QRnumber;
1120 Put(use_this_cptr->OtherQRnumbers,
1121 LKEY(qrbuf->QRnumber),
1125 use_this_cptr->roomlist_parts++;
1127 pthread_mutex_unlock(&POP3QueueMutex);
1130 pthread_mutex_unlock(&RSSQueueMutex);
1132 cptr->n = Pop3ClientID++;
1133 pthread_mutex_lock(&POP3QueueMutex);
1137 DeletePOP3Aggregator);
1139 pthread_mutex_unlock(&POP3QueueMutex);
1140 pLine = pLine->next;
1145 static int doing_pop3client = 0;
1147 void pop3client_scan(void) {
1148 static time_t last_run = 0L;
1149 time_t fastest_scan;
1156 become_session(&pop3_client_CC);
1158 if (config.c_pop3_fastest < config.c_pop3_fetch)
1159 fastest_scan = config.c_pop3_fastest;
1161 fastest_scan = config.c_pop3_fetch;
1164 * Run POP3 aggregation no more frequently than once every n seconds
1166 if ( (time(NULL) - last_run) < fastest_scan ) {
1171 * This is a simple concurrency check to make sure only one pop3client
1172 * run is done at a time. We could do this with a mutex, but since we
1173 * don't really require extremely fine granularity here, we'll do it
1174 * with a static variable instead.
1176 if (doing_pop3client) return;
1177 doing_pop3client = 1;
1179 EVP3CQM_syslog(LOG_DEBUG, "pop3client started");
1180 CtdlForEachNetCfgRoom(pop3client_scan_room, NULL, pop3client);
1182 pthread_mutex_lock(&POP3QueueMutex);
1183 it = GetNewHashPos(POP3FetchUrls, 0);
1184 while (!server_shutting_down &&
1185 GetNextHashPos(POP3FetchUrls, it, &len, &Key, &vrptr) &&
1187 cptr = (pop3aggr *)vrptr;
1188 if (cptr->RefCount == 0)
1189 if (!pop3_do_fetching(cptr))
1190 DeletePOP3Aggregator(cptr);////TODO
1193 if ((palist->interval && time(NULL) > (last_run + palist->interval))
1194 || (time(NULL) > last_run + config.c_pop3_fetch))
1195 pop3_do_fetching(palist->roomname, palist->pop3host,
1196 palist->pop3user, palist->pop3pass, palist->keep);
1198 palist = palist->next;
1203 pthread_mutex_unlock(&POP3QueueMutex);
1205 EVP3CQM_syslog(LOG_DEBUG, "pop3client ended");
1206 last_run = time(NULL);
1207 doing_pop3client = 0;
1211 void pop3_cleanup(void)
1213 /* citthread_mutex_destroy(&POP3QueueMutex); TODO */
1214 while (doing_pop3client != 0) ;
1215 DeleteHash(&POP3FetchUrls);
1216 DeleteHash(&POP3QueueRooms);
1221 void LogDebugEnablePOP3Client(const int n)
1223 POP3ClientDebugEnabled = n;
1226 CTDL_MODULE_INIT(pop3client)
1230 CtdlFillSystemContext(&pop3_client_CC, "POP3aggr");
1231 CtdlREGISTERRoomCfgType(pop3client, ParseGeneric, 0, 5, SerializeGeneric, DeleteGenericCfgLine);
1232 pthread_mutex_init(&POP3QueueMutex, NULL);
1233 POP3QueueRooms = NewHash(1, lFlathash);
1234 POP3FetchUrls = NewHash(1, NULL);
1235 CtdlRegisterSessionHook(pop3client_scan, EVT_TIMER, PRIO_AGGR + 50);
1236 CtdlRegisterEVCleanupHook(pop3_cleanup);
1237 CtdlRegisterDebugFlagHook(HKEY("pop3client"), LogDebugEnablePOP3Client, &POP3ClientDebugEnabled);
1240 /* return our module id for the log */
1241 return "pop3client";