2 * Consolidate mail from remote POP3 accounts.
4 * Copyright (c) 2007-2011 by the citadel.org team
6 * This program is open source software; you can redistribute it and/or
7 * modify it under the terms of the GNU General Public License as published
8 * by the Free Software Foundation; either version 3 of the License, or
9 * (at your option) any later version.
11 * This program is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 * GNU General Public License for more details.
16 * You should have received a copy of the GNU General Public License
17 * along with this program; if not, write to the Free Software
18 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
25 #if TIME_WITH_SYS_TIME
26 # include <sys/time.h>
30 # include <sys/time.h>
39 #include <sys/types.h>
41 #include <libcitadel.h>
44 #include "citserver.h"
47 #include "ctdl_module.h"
48 #include "clientsocket.h"
50 #include "internet_addressing.h"
52 #include "citadel_dirs.h"
53 #include "event_client.h"
56 #define POP3C_OK (strncasecmp(ChrPtr(RecvMsg->IO.IOBuf), "+OK", 3) == 0)
58 int POP3ClientDebugEnabled = 0;
60 #define N ((pop3aggr*)IO->Data)->n
62 #define DBGLOG(LEVEL) if ((LEVEL != LOG_DEBUG) || (POP3ClientDebugEnabled != 0))
64 #define EVP3C_syslog(LEVEL, FORMAT, ...) \
65 DBGLOG(LEVEL) syslog(LEVEL, \
66 "IO[%ld]CC[%d][%ld]POP3: " FORMAT, \
67 IO->ID, CCID, N, __VA_ARGS__)
69 #define EVP3CM_syslog(LEVEL, FORMAT) \
70 DBGLOG(LEVEL) syslog(LEVEL, \
71 "IO[%ld]CC[%d][%ld]POP3: " FORMAT, \
74 #define EVP3CQ_syslog(LEVEL, FORMAT, ...) \
75 DBGLOG(LEVEL) syslog(LEVEL, \
79 #define EVP3CQM_syslog(LEVEL, FORMAT) \
80 DBGLOG(LEVEL) syslog(LEVEL, \
84 #define EVP3CCS_syslog(LEVEL, FORMAT, ...) \
85 DBGLOG(LEVEL) syslog(LEVEL, "IO[%ld][%ld]POP3: " FORMAT, \
86 IO->ID, N, __VA_ARGS__)
88 #define EVP3CCSM_syslog(LEVEL, FORMAT) \
89 DBGLOG(LEVEL) syslog(LEVEL, "IO[%ld][%ld]POP3: " FORMAT, \
92 #define POP3C_DBG_SEND() \
93 EVP3C_syslog(LOG_DEBUG, \
94 "IO[%ld]CC[%d][%ld]POP3: > %s\n", \
96 ChrPtr(RecvMsg->IO.SendBuf.Buf))
98 #define POP3C_DBG_READ() \
99 EVP3C_syslog(LOG_DEBUG, \
100 "IO[%ld]CC[%d][%ld]POP3: < %s\n", \
102 ChrPtr(RecvMsg->IO.IOBuf))
105 struct CitContext pop3_client_CC;
107 pthread_mutex_t POP3QueueMutex; /* locks the access to the following vars: */
108 HashList *POP3QueueRooms = NULL;
109 HashList *POP3FetchUrls = NULL;
111 typedef struct pop3aggr pop3aggr;
112 typedef eNextState(*Pop3ClientHandler)(pop3aggr* RecvMsg);
114 eNextState POP3_C_Shutdown(AsyncIO *IO);
115 eNextState POP3_C_Timeout(AsyncIO *IO);
116 eNextState POP3_C_ConnFail(AsyncIO *IO);
117 eNextState POP3_C_DNSFail(AsyncIO *IO);
118 eNextState POP3_C_DispatchReadDone(AsyncIO *IO);
119 eNextState POP3_C_DispatchWriteDone(AsyncIO *IO);
120 eNextState POP3_C_Terminate(AsyncIO *IO);
121 eReadState POP3_C_ReadServerStatus(AsyncIO *IO);
122 eNextState POP3_C_ReAttachToFetchMessages(AsyncIO *IO);
124 typedef struct __pop3_room_counter {
129 typedef enum ePOP3_C_States {
135 GetOneMessageIDState,
136 ReadMessageBodyFollowing,
144 typedef struct _FetchItem {
150 struct CtdlMessage *Msg;
153 void HfreeFetchItem(void *vItem)
155 FetchItem *Item = (FetchItem*) vItem;
156 FreeStrBuf(&Item->MsgUIDL);
157 FreeStrBuf(&Item->MsgUID);
168 DNSQueryParts HostLookup;
171 HashList *OtherQRnumbers;
177 StrBuf *RoomName; // TODO: fill me
180 ePOP3_C_States State;
181 HashList *MsgNumbers;
186 void DeletePOP3Aggregator(void *vptr)
188 pop3aggr *ptr = vptr;
189 DeleteHashPos(&ptr->Pos);
190 DeleteHash(&ptr->MsgNumbers);
191 // FreeStrBuf(&ptr->rooms);
192 FreeStrBuf(&ptr->pop3user);
193 FreeStrBuf(&ptr->pop3pass);
194 FreeStrBuf(&ptr->Host);
195 FreeStrBuf(&ptr->RoomName);
196 FreeURL(&ptr->IO.ConnectMe);
197 FreeStrBuf(&ptr->Url);
198 FreeStrBuf(&ptr->IO.IOBuf);
199 FreeStrBuf(&ptr->IO.SendBuf.Buf);
200 FreeStrBuf(&ptr->IO.RecvBuf.Buf);
201 DeleteAsyncMsg(&ptr->IO.ReadMsg);
202 if (((struct CitContext*)ptr->IO.CitContext)) {
203 ((struct CitContext*)ptr->IO.CitContext)->state = CON_IDLE;
204 ((struct CitContext*)ptr->IO.CitContext)->kill_me = 1;
206 FreeAsyncIOContents(&ptr->IO);
210 eNextState FinalizePOP3AggrRun(AsyncIO *IO)
213 pop3aggr *cpptr = (pop3aggr *)IO->Data;
215 EVP3C_syslog(LOG_INFO,
216 "%s@%s: fetched %ld new of %d messages in %fs. bye.",
217 ChrPtr(cpptr->pop3user),
220 GetCount(cpptr->MsgNumbers),
221 IO->Now - cpptr->IOStart
224 It = GetNewHashPos(POP3FetchUrls, 0);
225 pthread_mutex_lock(&POP3QueueMutex);
227 if (GetHashPosFromKey(POP3FetchUrls, SKEY(cpptr->Url), It))
228 DeleteEntryFromHash(POP3FetchUrls, It);
230 pthread_mutex_unlock(&POP3QueueMutex);
235 eNextState FailAggregationRun(AsyncIO *IO)
240 eNextState POP3C_ReadGreeting(pop3aggr *RecvMsg)
242 AsyncIO *IO = &RecvMsg->IO;
244 /* Read the server greeting */
245 if (!POP3C_OK) return eTerminateConnection;
246 else return eSendReply;
249 eNextState POP3C_SendUser(pop3aggr *RecvMsg)
251 AsyncIO *IO = &RecvMsg->IO;
252 /* Identify ourselves. NOTE: we have to append a CR to each command.
253 * The LF will automatically be appended by sock_puts(). Believe it
254 * or not, leaving out the CR will cause problems if the server happens
255 * to be Exchange, which is so b0rken it actually barfs on
256 * LF-terminated newlines.
258 StrBufPrintf(RecvMsg->IO.SendBuf.Buf,
259 "USER %s\r\n", ChrPtr(RecvMsg->pop3user));
264 eNextState POP3C_GetUserState(pop3aggr *RecvMsg)
266 AsyncIO *IO = &RecvMsg->IO;
268 if (!POP3C_OK) return eTerminateConnection;
269 else return eSendReply;
272 eNextState POP3C_SendPassword(pop3aggr *RecvMsg)
274 AsyncIO *IO = &RecvMsg->IO;
276 StrBufPrintf(RecvMsg->IO.SendBuf.Buf,
277 "PASS %s\r\n", ChrPtr(RecvMsg->pop3pass));
278 EVP3CM_syslog(LOG_DEBUG, "<PASS <password>\n");
279 // POP3C_DBG_SEND(); No, we won't write the passvoid to syslog...
283 eNextState POP3C_GetPassState(pop3aggr *RecvMsg)
285 AsyncIO *IO = &RecvMsg->IO;
287 if (!POP3C_OK) return eTerminateConnection;
288 else return eSendReply;
291 eNextState POP3C_SendListCommand(pop3aggr *RecvMsg)
293 AsyncIO *IO = &RecvMsg->IO;
294 /* Get the list of messages */
295 StrBufPlain(RecvMsg->IO.SendBuf.Buf, HKEY("LIST\r\n"));
300 eNextState POP3C_GetListCommandState(pop3aggr *RecvMsg)
302 AsyncIO *IO = &RecvMsg->IO;
304 if (!POP3C_OK) return eTerminateConnection;
305 RecvMsg->MsgNumbers = NewHash(1, NULL);
311 eNextState POP3C_GetListOneLine(pop3aggr *RecvMsg)
313 AsyncIO *IO = &RecvMsg->IO;
318 FetchItem *OneMsg = NULL;
321 if ((StrLength(RecvMsg->IO.IOBuf) == 1) &&
322 (ChrPtr(RecvMsg->IO.IOBuf)[0] == '.'))
324 if (GetCount(RecvMsg->MsgNumbers) == 0)
326 //// RecvMsg->Sate = ReadQuitState;
330 RecvMsg->Pos = GetNewHashPos(RecvMsg->MsgNumbers, 0);
337 * work around buggy pop3 servers which send
338 * empty lines in their listings.
340 if ((StrLength(RecvMsg->IO.IOBuf) == 0) ||
341 !isdigit(ChrPtr(RecvMsg->IO.IOBuf)[0]))
346 OneMsg = (FetchItem*) malloc(sizeof(FetchItem));
347 memset(OneMsg, 0, sizeof(FetchItem));
348 OneMsg->MSGID = atol(ChrPtr(RecvMsg->IO.IOBuf));
350 pch = strchr(ChrPtr(RecvMsg->IO.IOBuf), ' ');
353 OneMsg->MSGSize = atol(pch + 1);
356 rc = TestValidateHash(RecvMsg->MsgNumbers);
358 EVP3CCS_syslog(LOG_DEBUG, "Hash Invalid: %d\n", rc);
361 Put(RecvMsg->MsgNumbers, LKEY(OneMsg->MSGID), OneMsg, HfreeFetchItem);
363 rc = TestValidateHash(RecvMsg->MsgNumbers);
365 EVP3CCS_syslog(LOG_DEBUG, "Hash Invalid: %d\n", rc);
367 //RecvMsg->State --; /* read next Line */
371 eNextState POP3_FetchNetworkUsetableEntry(AsyncIO *IO)
376 struct cdbdata *cdbut;
377 pop3aggr *RecvMsg = (pop3aggr *) IO->Data;
379 if((RecvMsg->Pos != NULL) &&
380 GetNextHashPos(RecvMsg->MsgNumbers,
387 if (server_shutting_down)
390 RecvMsg->CurrMsg = (FetchItem*) vData;
391 EVP3CCS_syslog(LOG_DEBUG,
392 "CHECKING: whether %s has already been seen: ",
393 ChrPtr(RecvMsg->CurrMsg->MsgUID));
395 /* Find out if we've already seen this item */
396 safestrncpy(ut.ut_msgid,
397 ChrPtr(RecvMsg->CurrMsg->MsgUID),
398 sizeof(ut.ut_msgid));
399 ut.ut_timestamp = time(NULL);/// TODO: libev timestamp!
401 cdbut = cdb_fetch(CDB_USETABLE, SKEY(RecvMsg->CurrMsg->MsgUID));
403 /* Item has already been seen */
404 EVP3CCSM_syslog(LOG_DEBUG, "YES\n");
407 /* rewrite the record anyway, to update the timestamp */
408 cdb_store(CDB_USETABLE,
409 SKEY(RecvMsg->CurrMsg->MsgUID),
410 &ut, sizeof(struct UseTable) );
411 RecvMsg->CurrMsg->NeedFetch = 0; ////TODO0;
415 EVP3CCSM_syslog(LOG_DEBUG, "NO\n");
416 RecvMsg->CurrMsg->NeedFetch = 1;
418 return NextDBOperation(&RecvMsg->IO,
419 POP3_FetchNetworkUsetableEntry);
423 /* ok, now we know them all,
424 * continue with reading the actual messages. */
425 DeleteHashPos(&RecvMsg->Pos);
427 return QueueEventContext(IO, POP3_C_ReAttachToFetchMessages);
431 eNextState POP3C_GetOneMessagID(pop3aggr *RecvMsg)
433 AsyncIO *IO = &RecvMsg->IO;
440 rc = TestValidateHash(RecvMsg->MsgNumbers);
442 EVP3CCS_syslog(LOG_DEBUG, "Hash Invalid: %d\n", rc);
444 if((RecvMsg->Pos != NULL) &&
445 GetNextHashPos(RecvMsg->MsgNumbers,
450 RecvMsg->CurrMsg = (FetchItem*) vData;
451 /* Find out the UIDL of the message,
452 * to determine whether we've already downloaded it */
453 StrBufPrintf(RecvMsg->IO.SendBuf.Buf,
454 "UIDL %ld\r\n", RecvMsg->CurrMsg->MSGID);
460 DeleteHashPos(&RecvMsg->Pos);
461 /// done receiving uidls.. start looking them up now.
462 RecvMsg->Pos = GetNewHashPos(RecvMsg->MsgNumbers, 0);
463 return QueueDBOperation(&RecvMsg->IO,
464 POP3_FetchNetworkUsetableEntry);
466 return eReadMore; /* TODO */
469 eNextState POP3C_GetOneMessageIDState(pop3aggr *RecvMsg)
471 AsyncIO *IO = &RecvMsg->IO;
474 rc = TestValidateHash(RecvMsg->MsgNumbers);
476 EVP3CCS_syslog(LOG_DEBUG, "Hash Invalid: %d\n", rc);
480 if (!POP3C_OK) return eTerminateConnection;
481 RecvMsg->CurrMsg->MsgUIDL =
482 NewStrBufPlain(NULL, StrLength(RecvMsg->IO.IOBuf));
483 RecvMsg->CurrMsg->MsgUID =
484 NewStrBufPlain(NULL, StrLength(RecvMsg->IO.IOBuf) * 2);
486 StrBufExtract_token(RecvMsg->CurrMsg->MsgUIDL,
487 RecvMsg->IO.IOBuf, 2, ' ');
489 StrBufPrintf(RecvMsg->CurrMsg->MsgUID,
491 ChrPtr(RecvMsg->RoomName),
492 ChrPtr(RecvMsg->CurrMsg->MsgUIDL),
493 RecvMsg->IO.ConnectMe->User,
494 RecvMsg->IO.ConnectMe->Host);
500 eNextState POP3C_SendGetOneMsg(pop3aggr *RecvMsg)
502 AsyncIO *IO = &RecvMsg->IO;
507 RecvMsg->CurrMsg = NULL;
508 while ((RecvMsg->Pos != NULL) &&
509 GetNextHashPos(RecvMsg->MsgNumbers,
513 (RecvMsg->CurrMsg = (FetchItem*) vData,
514 RecvMsg->CurrMsg->NeedFetch == 0))
517 if ((RecvMsg->CurrMsg != NULL ) && (RecvMsg->CurrMsg->NeedFetch == 1))
519 /* Message has not been seen.
520 * Tell the server to fetch the message... */
521 StrBufPrintf(RecvMsg->IO.SendBuf.Buf,
522 "RETR %ld\r\n", RecvMsg->CurrMsg->MSGID);
527 RecvMsg->State = ReadQuitState;
528 return POP3_C_DispatchWriteDone(&RecvMsg->IO);
533 eNextState POP3C_ReadMessageBodyFollowing(pop3aggr *RecvMsg)
535 AsyncIO *IO = &RecvMsg->IO;
537 if (!POP3C_OK) return eTerminateConnection;
538 RecvMsg->IO.ReadMsg = NewAsyncMsg(HKEY("."),
539 RecvMsg->CurrMsg->MSGSize,
548 eNextState POP3C_StoreMsgRead(AsyncIO *IO)
550 pop3aggr *RecvMsg = (pop3aggr *) IO->Data;
553 EVP3CCS_syslog(LOG_DEBUG,
554 "MARKING: %s as seen: ",
555 ChrPtr(RecvMsg->CurrMsg->MsgUID));
557 safestrncpy(ut.ut_msgid,
558 ChrPtr(RecvMsg->CurrMsg->MsgUID),
559 sizeof(ut.ut_msgid));
560 ut.ut_timestamp = time(NULL); /* TODO: use libev time */
561 cdb_store(CDB_USETABLE,
562 ChrPtr(RecvMsg->CurrMsg->MsgUID),
563 StrLength(RecvMsg->CurrMsg->MsgUID),
565 sizeof(struct UseTable) );
567 return QueueEventContext(&RecvMsg->IO, POP3_C_ReAttachToFetchMessages);
569 eNextState POP3C_SaveMsg(AsyncIO *IO)
572 pop3aggr *RecvMsg = (pop3aggr *) IO->Data;
574 /* Do Something With It (tm) */
575 msgnum = CtdlSubmitMsg(RecvMsg->CurrMsg->Msg,
577 ChrPtr(RecvMsg->RoomName),
581 /* Message has been committed to the store
582 * write the uidl to the use table
583 * so we don't fetch this message again
586 CtdlFreeMessage(RecvMsg->CurrMsg->Msg);
589 return NextDBOperation(&RecvMsg->IO, POP3C_StoreMsgRead);
592 eNextState POP3C_ReadMessageBody(pop3aggr *RecvMsg)
594 AsyncIO *IO = &RecvMsg->IO;
595 EVP3CM_syslog(LOG_DEBUG, "Converting message...");
596 RecvMsg->CurrMsg->Msg =
597 convert_internet_message_buf(&RecvMsg->IO.ReadMsg->MsgBuf);
598 StopClientWatchers(IO, 0);
599 return QueueDBOperation(&RecvMsg->IO, POP3C_SaveMsg);
602 eNextState POP3C_SendDelete(pop3aggr *RecvMsg)
604 AsyncIO *IO = &RecvMsg->IO;
605 if (!RecvMsg->keep) {
606 StrBufPrintf(RecvMsg->IO.SendBuf.Buf,
607 "DELE %ld\r\n", RecvMsg->CurrMsg->MSGID);
612 RecvMsg->State = ReadMessageBodyFollowing;
613 return POP3_C_DispatchWriteDone(&RecvMsg->IO);
616 eNextState POP3C_ReadDeleteState(pop3aggr *RecvMsg)
618 AsyncIO *IO = &RecvMsg->IO;
620 RecvMsg->State = GetOneMessageIDState;
624 eNextState POP3C_SendQuit(pop3aggr *RecvMsg)
626 AsyncIO *IO = &RecvMsg->IO;
628 StrBufPlain(RecvMsg->IO.SendBuf.Buf,
635 eNextState POP3C_ReadQuitState(pop3aggr *RecvMsg)
637 AsyncIO *IO = &RecvMsg->IO;
639 return eTerminateConnection;
642 const long POP3_C_ConnTimeout = 1000;
643 const long DefaultPOP3Port = 110;
645 Pop3ClientHandler POP3C_ReadHandlers[] = {
649 POP3C_GetListCommandState,
650 POP3C_GetListOneLine,
651 POP3C_GetOneMessageIDState,
652 POP3C_ReadMessageBodyFollowing,
653 POP3C_ReadMessageBody,
654 POP3C_ReadDeleteState,
658 const long POP3_C_SendTimeouts[POP3C_MaxRead] = {
668 const ConstStr POP3C_ReadErrors[POP3C_MaxRead] = {
669 {HKEY("Connection broken during ")},
670 {HKEY("Connection broken during ")},
671 {HKEY("Connection broken during ")},
672 {HKEY("Connection broken during ")},
673 {HKEY("Connection broken during ")},
674 {HKEY("Connection broken during ")},
675 {HKEY("Connection broken during ")},
676 {HKEY("Connection broken during ")}
679 Pop3ClientHandler POP3C_SendHandlers[] = {
680 NULL, /* we don't send a greeting */
683 POP3C_SendListCommand,
685 POP3C_GetOneMessagID,
692 const long POP3_C_ReadTimeouts[] = {
704 /*****************************************************************************/
705 /* POP3 CLIENT DISPATCHER */
706 /*****************************************************************************/
708 void POP3SetTimeout(eNextState NextTCPState, pop3aggr *pMsg)
710 AsyncIO *IO = &pMsg->IO;
711 double Timeout = 0.0;
713 EVP3C_syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
715 switch (NextTCPState) {
719 Timeout = POP3_C_SendTimeouts[pMsg->State];
721 if (pMsg->State == eDATABody) {
722 / * if we're sending a huge message, we need more time. * /
723 Timeout += StrLength(pMsg->msgtext) / 1024;
729 Timeout = POP3_C_ReadTimeouts[pMsg->State];
731 if (pMsg->State == eDATATerminateBody) {
733 * some mailservers take a nap before accepting the message
734 * content inspection and such.
736 Timeout += StrLength(pMsg->msgtext) / 1024;
747 case eTerminateConnection:
750 case eReadMore://// TODO
753 SetNextTimeout(&pMsg->IO, Timeout);
755 eNextState POP3_C_DispatchReadDone(AsyncIO *IO)
757 /* EVP3CCS_syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__); to noisy anyways. */
758 pop3aggr *pMsg = IO->Data;
761 rc = POP3C_ReadHandlers[pMsg->State](pMsg);
764 POP3SetTimeout(rc, pMsg);
767 eNextState POP3_C_DispatchWriteDone(AsyncIO *IO)
769 pop3aggr *pMsg = IO->Data;
772 /* EVP3CCS_syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__); to noisy anyways. */
773 rc = POP3C_SendHandlers[pMsg->State](pMsg);
774 POP3SetTimeout(rc, pMsg);
779 /*****************************************************************************/
780 /* POP3 CLIENT ERROR CATCHERS */
781 /*****************************************************************************/
782 eNextState POP3_C_Terminate(AsyncIO *IO)
784 /// pop3aggr *pMsg = (pop3aggr *)IO->Data;
786 EVP3CCS_syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
787 FinalizePOP3AggrRun(IO);
790 eNextState POP3_C_TerminateDB(AsyncIO *IO)
792 /// pop3aggr *pMsg = (pop3aggr *)IO->Data;
794 EVP3CCS_syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
795 FinalizePOP3AggrRun(IO);
798 eNextState POP3_C_Timeout(AsyncIO *IO)
800 pop3aggr *pMsg = IO->Data;
802 EVP3CCS_syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
803 StrBufPlain(IO->ErrMsg, CKEY(POP3C_ReadErrors[pMsg->State]));
804 return FailAggregationRun(IO);
806 eNextState POP3_C_ConnFail(AsyncIO *IO)
808 pop3aggr *pMsg = (pop3aggr *)IO->Data;
810 EVP3CCS_syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
811 StrBufPlain(IO->ErrMsg, CKEY(POP3C_ReadErrors[pMsg->State]));
812 return FailAggregationRun(IO);
814 eNextState POP3_C_DNSFail(AsyncIO *IO)
816 pop3aggr *pMsg = (pop3aggr *)IO->Data;
818 EVP3CCS_syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
819 StrBufPlain(IO->ErrMsg, CKEY(POP3C_ReadErrors[pMsg->State]));
820 return FailAggregationRun(IO);
822 eNextState POP3_C_Shutdown(AsyncIO *IO)
824 EVP3CCS_syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
825 //// pop3aggr *pMsg = IO->Data;
827 ////pMsg->MyQEntry->Status = 3;
828 ///StrBufPlain(pMsg->MyQEntry->StatusMessage, HKEY("server shutdown during message retrieval."));
829 FinalizePOP3AggrRun(IO);
835 * @brief lineread Handler; understands when to read more POP3 lines,
836 * and when this is a one-lined reply.
838 eReadState POP3_C_ReadServerStatus(AsyncIO *IO)
840 eReadState Finished = eBufferNotEmpty;
842 switch (IO->NextState) {
847 case eTerminateConnection:
849 Finished = eReadFail;
856 Finished = StrBufChunkSipLine(IO->IOBuf, &IO->RecvBuf);
860 Finished = CtdlReadMessageBodyAsync(IO);
866 /*****************************************************************************
867 * So we connect our Server IP here. *
868 *****************************************************************************/
869 eNextState POP3_C_ReAttachToFetchMessages(AsyncIO *IO)
871 pop3aggr *cpptr = IO->Data;
873 EVP3CCS_syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
874 ////??? cpptr->State ++;
875 if (cpptr->Pos == NULL)
876 cpptr->Pos = GetNewHashPos(cpptr->MsgNumbers, 0);
878 POP3_C_DispatchWriteDone(IO);
879 ReAttachIO(IO, cpptr, 0);
880 IO->NextState = eReadMessage;
881 return IO->NextState;
884 eNextState pop3_connect_ip(AsyncIO *IO)
886 pop3aggr *cpptr = IO->Data;
888 if (cpptr->IOStart == 0.0) /* whith or without DNS? */
889 cpptr->IOStart = IO->Now;
891 EVP3CCS_syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
893 return EvConnectSock(IO,
895 POP3_C_ReadTimeouts[0],
899 eNextState pop3_get_one_host_ip_done(AsyncIO *IO)
901 pop3aggr *cpptr = IO->Data;
902 struct hostent *hostent;
906 hostent = cpptr->HostLookup.VParsedDNSReply;
907 if ((cpptr->HostLookup.DNSStatus == ARES_SUCCESS) &&
908 (hostent != NULL) ) {
909 memset(&cpptr->IO.ConnectMe->Addr, 0, sizeof(struct in6_addr));
910 if (cpptr->IO.ConnectMe->IPv6) {
911 memcpy(&cpptr->IO.ConnectMe->Addr.sin6_addr.s6_addr,
912 &hostent->h_addr_list[0],
913 sizeof(struct in6_addr));
915 cpptr->IO.ConnectMe->Addr.sin6_family =
917 cpptr->IO.ConnectMe->Addr.sin6_port =
918 htons(DefaultPOP3Port);
921 struct sockaddr_in *addr =
922 (struct sockaddr_in*)
923 &cpptr->IO.ConnectMe->Addr;
925 memcpy(&addr->sin_addr.s_addr,
926 hostent->h_addr_list[0],
929 addr->sin_family = hostent->h_addrtype;
930 addr->sin_port = htons(DefaultPOP3Port);
932 return pop3_connect_ip(IO);
938 eNextState pop3_get_one_host_ip(AsyncIO *IO)
940 pop3aggr *cpptr = IO->Data;
942 cpptr->IOStart = IO->Now;
944 EVP3CCS_syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
946 EVP3CCS_syslog(LOG_DEBUG,
947 "POP3 client[%ld]: looking up %s-Record %s : %d ...\n",
949 (cpptr->IO.ConnectMe->IPv6)? "aaaa": "a",
950 cpptr->IO.ConnectMe->Host,
951 cpptr->IO.ConnectMe->Port);
953 QueueQuery((cpptr->IO.ConnectMe->IPv6)? ns_t_aaaa : ns_t_a,
954 cpptr->IO.ConnectMe->Host,
957 pop3_get_one_host_ip_done);
958 IO->NextState = eReadDNSReply;
959 return IO->NextState;
964 int pop3_do_fetching(pop3aggr *cpptr)
966 AsyncIO *IO = &cpptr->IO;
971 POP3_C_ReadServerStatus,
973 POP3_C_DispatchWriteDone,
974 POP3_C_DispatchReadDone,
981 safestrncpy(((CitContext *)cpptr->IO.CitContext)->cs_host,
983 sizeof(((CitContext *)cpptr->IO.CitContext)->cs_host));
985 if (cpptr->IO.ConnectMe->IsIP) {
986 QueueEventContext(&cpptr->IO,
990 QueueEventContext(&cpptr->IO,
991 pop3_get_one_host_ip);
997 * Scan a room's netconfig to determine whether it requires POP3 aggregation
999 void pop3client_scan_room(struct ctdlroom *qrbuf, void *data, OneRoomNetCfg *OneRNCFG)
1001 const RoomNetCfgLine *pLine;
1004 pthread_mutex_lock(&POP3QueueMutex);
1005 if (GetHash(POP3QueueRooms, LKEY(qrbuf->QRnumber), &vptr))
1007 pthread_mutex_unlock(&POP3QueueMutex);
1008 EVP3CQ_syslog(LOG_DEBUG,
1009 "pop3client: [%ld] %s already in progress.",
1014 pthread_mutex_unlock(&POP3QueueMutex);
1016 if (server_shutting_down) return;
1018 pLine = OneRNCFG->NetConfigs[pop3client];
1020 while (pLine != NULL)
1024 cptr = (pop3aggr *) malloc(sizeof(pop3aggr));
1025 memset(cptr, 0, sizeof(pop3aggr));
1026 ///TODO do we need this? cptr->roomlist_parts=1;
1027 cptr->RoomName = NewStrBufPlain(qrbuf->QRname, -1);
1028 cptr->pop3user = NewStrBufDup(pLine->Value[1]);
1029 cptr->pop3pass = NewStrBufDup(pLine->Value[2]);
1030 cptr->Url = NewStrBuf();
1031 cptr->Host = NewStrBufDup(pLine->Value[0]);
1033 cptr->keep = atol(ChrPtr(pLine->Value[3]));
1034 cptr->interval = atol(ChrPtr(pLine->Value[4]));
1036 StrBufAppendBufPlain(cptr->Url, HKEY("pop3://"), 0);
1037 StrBufUrlescUPAppend(cptr->Url, cptr->pop3user, NULL);
1038 StrBufAppendBufPlain(cptr->Url, HKEY(":"), 0);
1039 StrBufUrlescUPAppend(cptr->Url, cptr->pop3pass, NULL);
1040 StrBufAppendBufPlain(cptr->Url, HKEY("@"), 0);
1041 StrBufAppendBuf(cptr->Url, cptr->Host, 0);
1042 StrBufAppendBufPlain(cptr->Url, HKEY("/"), 0);
1043 StrBufUrlescAppend(cptr->Url, cptr->RoomName, NULL);
1045 ParseURL(&cptr->IO.ConnectMe, cptr->Url, 110);
1049 /* todo: we need to reunite the url to be shure. */
1051 pthread_mutex_lock(&POP3ueueMutex);
1052 GetHash(POP3FetchUrls, SKEY(ptr->Url), &vptr);
1053 use_this_cptr = (pop3aggr *)vptr;
1055 if (use_this_rncptr != NULL)
1057 /* mustn't attach to an active session */
1058 if (use_this_cptr->RefCount > 0)
1060 DeletePOP3Cfg(cptr);
1066 StrBufAppendBufPlain(
1067 use_this_cptr->rooms,
1070 if (use_this_cptr->roomlist_parts == 1)
1072 use_this_cptr->OtherQRnumbers
1073 = NewHash(1, lFlathash);
1075 QRnumber = (long*)malloc(sizeof(long));
1076 *QRnumber = qrbuf->QRnumber;
1077 Put(use_this_cptr->OtherQRnumbers,
1078 LKEY(qrbuf->QRnumber),
1082 use_this_cptr->roomlist_parts++;
1084 pthread_mutex_unlock(&POP3QueueMutex);
1087 pthread_mutex_unlock(&RSSQueueMutex);
1089 cptr->n = Pop3ClientID++;
1090 pthread_mutex_lock(&POP3QueueMutex);
1094 DeletePOP3Aggregator);
1096 pthread_mutex_unlock(&POP3QueueMutex);
1097 pLine = pLine->next;
1102 static int doing_pop3client = 0;
1104 void pop3client_scan(void) {
1105 static time_t last_run = 0L;
1106 time_t fastest_scan;
1113 become_session(&pop3_client_CC);
1115 if (config.c_pop3_fastest < config.c_pop3_fetch)
1116 fastest_scan = config.c_pop3_fastest;
1118 fastest_scan = config.c_pop3_fetch;
1121 * Run POP3 aggregation no more frequently than once every n seconds
1123 if ( (time(NULL) - last_run) < fastest_scan ) {
1128 * This is a simple concurrency check to make sure only one pop3client
1129 * run is done at a time. We could do this with a mutex, but since we
1130 * don't really require extremely fine granularity here, we'll do it
1131 * with a static variable instead.
1133 if (doing_pop3client) return;
1134 doing_pop3client = 1;
1136 EVP3CQM_syslog(LOG_DEBUG, "pop3client started");
1137 CtdlForEachNetCfgRoom(pop3client_scan_room, NULL, pop3client);
1139 pthread_mutex_lock(&POP3QueueMutex);
1140 it = GetNewHashPos(POP3FetchUrls, 0);
1141 while (!server_shutting_down &&
1142 GetNextHashPos(POP3FetchUrls, it, &len, &Key, &vrptr) &&
1144 cptr = (pop3aggr *)vrptr;
1145 if (cptr->RefCount == 0)
1146 if (!pop3_do_fetching(cptr))
1147 DeletePOP3Aggregator(cptr);////TODO
1150 if ((palist->interval && time(NULL) > (last_run + palist->interval))
1151 || (time(NULL) > last_run + config.c_pop3_fetch))
1152 pop3_do_fetching(palist->roomname, palist->pop3host,
1153 palist->pop3user, palist->pop3pass, palist->keep);
1155 palist = palist->next;
1160 pthread_mutex_unlock(&POP3QueueMutex);
1162 EVP3CQM_syslog(LOG_DEBUG, "pop3client ended");
1163 last_run = time(NULL);
1164 doing_pop3client = 0;
1168 void pop3_cleanup(void)
1170 /* citthread_mutex_destroy(&POP3QueueMutex); TODO */
1171 while (doing_pop3client != 0) ;
1172 DeleteHash(&POP3FetchUrls);
1173 DeleteHash(&POP3QueueRooms);
1178 void LogDebugEnablePOP3Client(const int n)
1180 POP3ClientDebugEnabled = n;
1183 CTDL_MODULE_INIT(pop3client)
1187 CtdlFillSystemContext(&pop3_client_CC, "POP3aggr");
1188 CtdlREGISTERRoomCfgType(pop3client, ParseGeneric, 0, 5, SerializeGeneric, DeleteGenericCfgLine);
1189 pthread_mutex_init(&POP3QueueMutex, NULL);
1190 POP3QueueRooms = NewHash(1, lFlathash);
1191 POP3FetchUrls = NewHash(1, NULL);
1192 CtdlRegisterSessionHook(pop3client_scan, EVT_TIMER, PRIO_AGGR + 50);
1193 CtdlRegisterEVCleanupHook(pop3_cleanup);
1194 CtdlRegisterDebugFlagHook(HKEY("pop3client"), LogDebugEnablePOP3Client, &POP3ClientDebugEnabled);
1197 /* return our module id for the log */
1198 return "pop3client";