2 * Consolidate mail from remote POP3 accounts.
4 * Copyright (c) 2007-2011 by the citadel.org team
6 * This program is open source software; you can redistribute it and/or
7 * modify it under the terms of the GNU General Public License as published
8 * by the Free Software Foundation; either version 3 of the License, or
9 * (at your option) any later version.
11 * This program is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 * GNU General Public License for more details.
16 * You should have received a copy of the GNU General Public License
17 * along with this program; if not, write to the Free Software
18 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
25 #if TIME_WITH_SYS_TIME
26 # include <sys/time.h>
30 # include <sys/time.h>
39 #include <sys/types.h>
41 #include <libcitadel.h>
44 #include "citserver.h"
47 #include "ctdl_module.h"
48 #include "clientsocket.h"
50 #include "internet_addressing.h"
52 #include "citadel_dirs.h"
53 #include "event_client.h"
56 #define POP3C_OK (strncasecmp(ChrPtr(RecvMsg->IO.IOBuf), "+OK", 3) == 0)
58 int POP3ClientDebugEnabled = 0;
60 #define N ((pop3aggr*)IO->Data)->n
62 #define DBGLOG(LEVEL) if ((LEVEL != LOG_DEBUG) || (POP3ClientDebugEnabled != 0))
64 #define EVP3C_syslog(LEVEL, FORMAT, ...) \
65 DBGLOG(LEVEL) syslog(LEVEL, \
66 "IO[%ld]CC[%d][%ld]" FORMAT, \
67 IO->ID, CCID, N, __VA_ARGS__)
69 #define EVP3CM_syslog(LEVEL, FORMAT) \
70 DBGLOG(LEVEL) syslog(LEVEL, \
71 "IO[%ld]CC[%d][%ld]" FORMAT, \
74 #define EVP3CQ_syslog(LEVEL, FORMAT, ...) \
75 DBGLOG(LEVEL) syslog(LEVEL, \
79 #define EVP3CQM_syslog(LEVEL, FORMAT) \
80 DBGLOG(LEVEL) syslog(LEVEL, \
84 #define EVP3CCS_syslog(LEVEL, FORMAT, ...) \
85 DBGLOG(LEVEL) syslog(LEVEL, "IO[%ld][%ld]" FORMAT, \
86 IO->ID, N, __VA_ARGS__)
88 #define EVP3CCSM_syslog(LEVEL, FORMAT) \
89 DBGLOG(LEVEL) syslog(LEVEL, "IO[%ld][%ld]" FORMAT, \
92 #define POP3C_DBG_SEND() \
93 EVP3C_syslog(LOG_DEBUG, \
95 ChrPtr(RecvMsg->IO.SendBuf.Buf))
97 #define POP3C_DBG_READ() \
98 EVP3C_syslog(LOG_DEBUG, \
100 ChrPtr(RecvMsg->IO.IOBuf))
103 struct CitContext pop3_client_CC;
105 pthread_mutex_t POP3QueueMutex; /* locks the access to the following vars: */
106 HashList *POP3QueueRooms = NULL;
107 HashList *POP3FetchUrls = NULL;
109 typedef struct pop3aggr pop3aggr;
110 typedef eNextState(*Pop3ClientHandler)(pop3aggr* RecvMsg);
112 eNextState POP3_C_Shutdown(AsyncIO *IO);
113 eNextState POP3_C_Timeout(AsyncIO *IO);
114 eNextState POP3_C_ConnFail(AsyncIO *IO);
115 eNextState POP3_C_DNSFail(AsyncIO *IO);
116 eNextState POP3_C_DispatchReadDone(AsyncIO *IO);
117 eNextState POP3_C_DispatchWriteDone(AsyncIO *IO);
118 eNextState POP3_C_Terminate(AsyncIO *IO);
119 eReadState POP3_C_ReadServerStatus(AsyncIO *IO);
120 eNextState POP3_C_ReAttachToFetchMessages(AsyncIO *IO);
122 typedef struct __pop3_room_counter {
127 typedef enum ePOP3_C_States {
133 GetOneMessageIDState,
134 ReadMessageBodyFollowing,
142 typedef struct _FetchItem {
148 struct CtdlMessage *Msg;
151 void HfreeFetchItem(void *vItem)
153 FetchItem *Item = (FetchItem*) vItem;
154 FreeStrBuf(&Item->MsgUIDL);
155 FreeStrBuf(&Item->MsgUID);
166 DNSQueryParts HostLookup;
169 HashList *OtherQRnumbers;
174 StrBuf *RoomName; // TODO: fill me
177 ePOP3_C_States State;
178 HashList *MsgNumbers;
183 void DeletePOP3Aggregator(void *vptr)
185 pop3aggr *ptr = vptr;
186 DeleteHashPos(&ptr->Pos);
187 DeleteHash(&ptr->MsgNumbers);
188 // FreeStrBuf(&ptr->rooms);
189 FreeStrBuf(&ptr->pop3user);
190 FreeStrBuf(&ptr->pop3pass);
191 FreeStrBuf(&ptr->RoomName);
192 FreeURL(&ptr->IO.ConnectMe);
193 FreeStrBuf(&ptr->Url);
194 FreeStrBuf(&ptr->IO.IOBuf);
195 FreeStrBuf(&ptr->IO.SendBuf.Buf);
196 FreeStrBuf(&ptr->IO.RecvBuf.Buf);
197 DeleteAsyncMsg(&ptr->IO.ReadMsg);
198 ((struct CitContext*)ptr->IO.CitContext)->state = CON_IDLE;
199 ((struct CitContext*)ptr->IO.CitContext)->kill_me = 1;
200 FreeAsyncIOContents(&ptr->IO);
204 eNextState FinalizePOP3AggrRun(AsyncIO *IO)
207 pop3aggr *cpptr = (pop3aggr *)IO->Data;
209 EVP3C_syslog(LOG_INFO,
210 "%s@%s: fetched %ld new of %d messages in %fs. bye.",
211 ChrPtr(cpptr->pop3user),
212 ChrPtr(cpptr->pop3pass),
214 GetCount(cpptr->MsgNumbers),
215 IO->Now - cpptr->IOStart
218 It = GetNewHashPos(POP3FetchUrls, 0);
219 pthread_mutex_lock(&POP3QueueMutex);
221 if (GetHashPosFromKey(POP3FetchUrls, SKEY(cpptr->Url), It))
222 DeleteEntryFromHash(POP3FetchUrls, It);
224 pthread_mutex_unlock(&POP3QueueMutex);
229 eNextState FailAggregationRun(AsyncIO *IO)
234 eNextState POP3C_ReadGreeting(pop3aggr *RecvMsg)
236 AsyncIO *IO = &RecvMsg->IO;
238 /* Read the server greeting */
239 if (!POP3C_OK) return eTerminateConnection;
240 else return eSendReply;
243 eNextState POP3C_SendUser(pop3aggr *RecvMsg)
245 AsyncIO *IO = &RecvMsg->IO;
246 /* Identify ourselves. NOTE: we have to append a CR to each command.
247 * The LF will automatically be appended by sock_puts(). Believe it
248 * or not, leaving out the CR will cause problems if the server happens
249 * to be Exchange, which is so b0rken it actually barfs on
250 * LF-terminated newlines.
252 StrBufPrintf(RecvMsg->IO.SendBuf.Buf,
253 "USER %s\r\n", ChrPtr(RecvMsg->pop3user));
258 eNextState POP3C_GetUserState(pop3aggr *RecvMsg)
260 AsyncIO *IO = &RecvMsg->IO;
262 if (!POP3C_OK) return eTerminateConnection;
263 else return eSendReply;
266 eNextState POP3C_SendPassword(pop3aggr *RecvMsg)
268 AsyncIO *IO = &RecvMsg->IO;
270 StrBufPrintf(RecvMsg->IO.SendBuf.Buf,
271 "PASS %s\r\n", ChrPtr(RecvMsg->pop3pass));
272 EVP3CM_syslog(LOG_DEBUG, "<PASS <password>\n");
273 // POP3C_DBG_SEND(); No, we won't write the passvoid to syslog...
277 eNextState POP3C_GetPassState(pop3aggr *RecvMsg)
279 AsyncIO *IO = &RecvMsg->IO;
281 if (!POP3C_OK) return eTerminateConnection;
282 else return eSendReply;
285 eNextState POP3C_SendListCommand(pop3aggr *RecvMsg)
287 AsyncIO *IO = &RecvMsg->IO;
288 /* Get the list of messages */
289 StrBufPlain(RecvMsg->IO.SendBuf.Buf, HKEY("LIST\r\n"));
294 eNextState POP3C_GetListCommandState(pop3aggr *RecvMsg)
296 AsyncIO *IO = &RecvMsg->IO;
298 if (!POP3C_OK) return eTerminateConnection;
299 RecvMsg->MsgNumbers = NewHash(1, NULL);
305 eNextState POP3C_GetListOneLine(pop3aggr *RecvMsg)
307 AsyncIO *IO = &RecvMsg->IO;
312 FetchItem *OneMsg = NULL;
315 if ((StrLength(RecvMsg->IO.IOBuf) == 1) &&
316 (ChrPtr(RecvMsg->IO.IOBuf)[0] == '.'))
318 if (GetCount(RecvMsg->MsgNumbers) == 0)
320 //// RecvMsg->Sate = ReadQuitState;
324 RecvMsg->Pos = GetNewHashPos(RecvMsg->MsgNumbers, 0);
329 OneMsg = (FetchItem*) malloc(sizeof(FetchItem));
330 memset(OneMsg, 0, sizeof(FetchItem));
331 OneMsg->MSGID = atol(ChrPtr(RecvMsg->IO.IOBuf));
333 pch = strchr(ChrPtr(RecvMsg->IO.IOBuf), ' ');
336 OneMsg->MSGSize = atol(pch + 1);
339 rc = TestValidateHash(RecvMsg->MsgNumbers);
341 EVP3CCS_syslog(LOG_DEBUG, "Hash Invalid: %d\n", rc);
344 Put(RecvMsg->MsgNumbers, LKEY(OneMsg->MSGID), OneMsg, HfreeFetchItem);
346 rc = TestValidateHash(RecvMsg->MsgNumbers);
348 EVP3CCS_syslog(LOG_DEBUG, "Hash Invalid: %d\n", rc);
350 //RecvMsg->State --; /* read next Line */
354 eNextState POP3_FetchNetworkUsetableEntry(AsyncIO *IO)
359 struct cdbdata *cdbut;
360 pop3aggr *RecvMsg = (pop3aggr *) IO->Data;
362 if(GetNextHashPos(RecvMsg->MsgNumbers,
369 if (server_shutting_down)
372 RecvMsg->CurrMsg = (FetchItem*) vData;
373 EVP3CCS_syslog(LOG_DEBUG,
374 "CHECKING: whether %s has already been seen: ",
375 ChrPtr(RecvMsg->CurrMsg->MsgUID));
377 /* Find out if we've already seen this item */
378 safestrncpy(ut.ut_msgid,
379 ChrPtr(RecvMsg->CurrMsg->MsgUID),
380 sizeof(ut.ut_msgid));
381 ut.ut_timestamp = time(NULL);/// TODO: libev timestamp!
383 cdbut = cdb_fetch(CDB_USETABLE, SKEY(RecvMsg->CurrMsg->MsgUID));
385 /* Item has already been seen */
386 EVP3CCSM_syslog(LOG_DEBUG, "YES\n");
389 /* rewrite the record anyway, to update the timestamp */
390 cdb_store(CDB_USETABLE,
391 SKEY(RecvMsg->CurrMsg->MsgUID),
392 &ut, sizeof(struct UseTable) );
393 RecvMsg->CurrMsg->NeedFetch = 0; ////TODO0;
397 EVP3CCSM_syslog(LOG_DEBUG, "NO\n");
398 RecvMsg->CurrMsg->NeedFetch = 1;
400 return NextDBOperation(&RecvMsg->IO,
401 POP3_FetchNetworkUsetableEntry);
405 /* ok, now we know them all,
406 * continue with reading the actual messages. */
407 DeleteHashPos(&RecvMsg->Pos);
409 return QueueEventContext(IO, POP3_C_ReAttachToFetchMessages);
413 eNextState POP3C_GetOneMessagID(pop3aggr *RecvMsg)
415 AsyncIO *IO = &RecvMsg->IO;
422 rc = TestValidateHash(RecvMsg->MsgNumbers);
424 EVP3CCS_syslog(LOG_DEBUG, "Hash Invalid: %d\n", rc);
426 if(GetNextHashPos(RecvMsg->MsgNumbers,
431 RecvMsg->CurrMsg = (FetchItem*) vData;
432 /* Find out the UIDL of the message,
433 * to determine whether we've already downloaded it */
434 StrBufPrintf(RecvMsg->IO.SendBuf.Buf,
435 "UIDL %ld\r\n", RecvMsg->CurrMsg->MSGID);
441 DeleteHashPos(&RecvMsg->Pos);
442 /// done receiving uidls.. start looking them up now.
443 RecvMsg->Pos = GetNewHashPos(RecvMsg->MsgNumbers, 0);
444 return QueueDBOperation(&RecvMsg->IO,
445 POP3_FetchNetworkUsetableEntry);
447 return eReadMore; /* TODO */
450 eNextState POP3C_GetOneMessageIDState(pop3aggr *RecvMsg)
452 AsyncIO *IO = &RecvMsg->IO;
455 rc = TestValidateHash(RecvMsg->MsgNumbers);
457 EVP3CCS_syslog(LOG_DEBUG, "Hash Invalid: %d\n", rc);
461 if (!POP3C_OK) return eTerminateConnection;
462 RecvMsg->CurrMsg->MsgUIDL =
463 NewStrBufPlain(NULL, StrLength(RecvMsg->IO.IOBuf));
464 RecvMsg->CurrMsg->MsgUID =
465 NewStrBufPlain(NULL, StrLength(RecvMsg->IO.IOBuf) * 2);
467 StrBufExtract_token(RecvMsg->CurrMsg->MsgUIDL,
468 RecvMsg->IO.IOBuf, 2, ' ');
470 StrBufPrintf(RecvMsg->CurrMsg->MsgUID,
472 ChrPtr(RecvMsg->RoomName),
473 ChrPtr(RecvMsg->CurrMsg->MsgUIDL),
474 RecvMsg->IO.ConnectMe->User,
475 RecvMsg->IO.ConnectMe->Host);
481 eNextState POP3C_SendGetOneMsg(pop3aggr *RecvMsg)
483 AsyncIO *IO = &RecvMsg->IO;
488 RecvMsg->CurrMsg = NULL;
489 while (GetNextHashPos(RecvMsg->MsgNumbers,
493 (RecvMsg->CurrMsg = (FetchItem*) vData,
494 RecvMsg->CurrMsg->NeedFetch == 0))
497 if ((RecvMsg->CurrMsg != NULL ) && (RecvMsg->CurrMsg->NeedFetch == 1))
499 /* Message has not been seen.
500 * Tell the server to fetch the message... */
501 StrBufPrintf(RecvMsg->IO.SendBuf.Buf,
502 "RETR %ld\r\n", RecvMsg->CurrMsg->MSGID);
507 RecvMsg->State = ReadQuitState;
508 return POP3_C_DispatchWriteDone(&RecvMsg->IO);
513 eNextState POP3C_ReadMessageBodyFollowing(pop3aggr *RecvMsg)
515 AsyncIO *IO = &RecvMsg->IO;
517 if (!POP3C_OK) return eTerminateConnection;
518 RecvMsg->IO.ReadMsg = NewAsyncMsg(HKEY("."),
519 RecvMsg->CurrMsg->MSGSize,
528 eNextState POP3C_StoreMsgRead(AsyncIO *IO)
530 pop3aggr *RecvMsg = (pop3aggr *) IO->Data;
533 EVP3CCS_syslog(LOG_DEBUG,
534 "MARKING: %s as seen: ",
535 ChrPtr(RecvMsg->CurrMsg->MsgUID));
537 safestrncpy(ut.ut_msgid,
538 ChrPtr(RecvMsg->CurrMsg->MsgUID),
539 sizeof(ut.ut_msgid));
540 ut.ut_timestamp = time(NULL); /* TODO: use libev time */
541 cdb_store(CDB_USETABLE,
542 ChrPtr(RecvMsg->CurrMsg->MsgUID),
543 StrLength(RecvMsg->CurrMsg->MsgUID),
545 sizeof(struct UseTable) );
547 return QueueEventContext(&RecvMsg->IO, POP3_C_ReAttachToFetchMessages);
549 eNextState POP3C_SaveMsg(AsyncIO *IO)
552 pop3aggr *RecvMsg = (pop3aggr *) IO->Data;
554 /* Do Something With It (tm) */
555 msgnum = CtdlSubmitMsg(RecvMsg->CurrMsg->Msg,
557 ChrPtr(RecvMsg->RoomName),
561 /* Message has been committed to the store
562 * write the uidl to the use table
563 * so we don't fetch this message again
566 CtdlFreeMessage(RecvMsg->CurrMsg->Msg);
569 return NextDBOperation(&RecvMsg->IO, POP3C_StoreMsgRead);
572 eNextState POP3C_ReadMessageBody(pop3aggr *RecvMsg)
574 AsyncIO *IO = &RecvMsg->IO;
575 EVP3CM_syslog(LOG_DEBUG, "Converting message...");
576 RecvMsg->CurrMsg->Msg =
577 convert_internet_message_buf(&RecvMsg->IO.ReadMsg->MsgBuf);
578 StopClientWatchers(IO, 0);
579 return QueueDBOperation(&RecvMsg->IO, POP3C_SaveMsg);
582 eNextState POP3C_SendDelete(pop3aggr *RecvMsg)
584 AsyncIO *IO = &RecvMsg->IO;
585 if (!RecvMsg->keep) {
586 StrBufPrintf(RecvMsg->IO.SendBuf.Buf,
587 "DELE %ld\r\n", RecvMsg->CurrMsg->MSGID);
592 RecvMsg->State = ReadMessageBodyFollowing;
593 return POP3_C_DispatchWriteDone(&RecvMsg->IO);
596 eNextState POP3C_ReadDeleteState(pop3aggr *RecvMsg)
598 AsyncIO *IO = &RecvMsg->IO;
600 RecvMsg->State = GetOneMessageIDState;
604 eNextState POP3C_SendQuit(pop3aggr *RecvMsg)
606 AsyncIO *IO = &RecvMsg->IO;
608 StrBufPlain(RecvMsg->IO.SendBuf.Buf,
615 eNextState POP3C_ReadQuitState(pop3aggr *RecvMsg)
617 AsyncIO *IO = &RecvMsg->IO;
619 return eTerminateConnection;
622 const long POP3_C_ConnTimeout = 1000;
623 const long DefaultPOP3Port = 110;
625 Pop3ClientHandler POP3C_ReadHandlers[] = {
629 POP3C_GetListCommandState,
630 POP3C_GetListOneLine,
631 POP3C_GetOneMessageIDState,
632 POP3C_ReadMessageBodyFollowing,
633 POP3C_ReadMessageBody,
634 POP3C_ReadDeleteState,
638 const long POP3_C_SendTimeouts[POP3C_MaxRead] = {
648 const ConstStr POP3C_ReadErrors[POP3C_MaxRead] = {
649 {HKEY("Connection broken during ")},
650 {HKEY("Connection broken during ")},
651 {HKEY("Connection broken during ")},
652 {HKEY("Connection broken during ")},
653 {HKEY("Connection broken during ")},
654 {HKEY("Connection broken during ")},
655 {HKEY("Connection broken during ")},
656 {HKEY("Connection broken during ")}
659 Pop3ClientHandler POP3C_SendHandlers[] = {
660 NULL, /* we don't send a greeting */
663 POP3C_SendListCommand,
665 POP3C_GetOneMessagID,
672 const long POP3_C_ReadTimeouts[] = {
684 /*****************************************************************************/
685 /* POP3 CLIENT DISPATCHER */
686 /*****************************************************************************/
688 void POP3SetTimeout(eNextState NextTCPState, pop3aggr *pMsg)
690 AsyncIO *IO = &pMsg->IO;
691 double Timeout = 0.0;
693 EVP3C_syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
695 switch (NextTCPState) {
699 Timeout = POP3_C_SendTimeouts[pMsg->State];
701 if (pMsg->State == eDATABody) {
702 / * if we're sending a huge message, we need more time. * /
703 Timeout += StrLength(pMsg->msgtext) / 1024;
709 Timeout = POP3_C_ReadTimeouts[pMsg->State];
711 if (pMsg->State == eDATATerminateBody) {
713 * some mailservers take a nap before accepting the message
714 * content inspection and such.
716 Timeout += StrLength(pMsg->msgtext) / 1024;
727 case eTerminateConnection:
730 case eReadMore://// TODO
733 SetNextTimeout(&pMsg->IO, Timeout);
735 eNextState POP3_C_DispatchReadDone(AsyncIO *IO)
737 /* EVP3CCS_syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__); to noisy anyways. */
738 pop3aggr *pMsg = IO->Data;
741 rc = POP3C_ReadHandlers[pMsg->State](pMsg);
744 POP3SetTimeout(rc, pMsg);
747 eNextState POP3_C_DispatchWriteDone(AsyncIO *IO)
749 pop3aggr *pMsg = IO->Data;
752 /* EVP3CCS_syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__); to noisy anyways. */
753 rc = POP3C_SendHandlers[pMsg->State](pMsg);
754 POP3SetTimeout(rc, pMsg);
759 /*****************************************************************************/
760 /* POP3 CLIENT ERROR CATCHERS */
761 /*****************************************************************************/
762 eNextState POP3_C_Terminate(AsyncIO *IO)
764 /// pop3aggr *pMsg = (pop3aggr *)IO->Data;
766 EVP3CCS_syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
767 FinalizePOP3AggrRun(IO);
770 eNextState POP3_C_TerminateDB(AsyncIO *IO)
772 /// pop3aggr *pMsg = (pop3aggr *)IO->Data;
774 EVP3CCS_syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
775 FinalizePOP3AggrRun(IO);
778 eNextState POP3_C_Timeout(AsyncIO *IO)
780 pop3aggr *pMsg = IO->Data;
782 EVP3CCS_syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
783 StrBufPlain(IO->ErrMsg, CKEY(POP3C_ReadErrors[pMsg->State]));
784 return FailAggregationRun(IO);
786 eNextState POP3_C_ConnFail(AsyncIO *IO)
788 pop3aggr *pMsg = (pop3aggr *)IO->Data;
790 EVP3CCS_syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
791 StrBufPlain(IO->ErrMsg, CKEY(POP3C_ReadErrors[pMsg->State]));
792 return FailAggregationRun(IO);
794 eNextState POP3_C_DNSFail(AsyncIO *IO)
796 pop3aggr *pMsg = (pop3aggr *)IO->Data;
798 EVP3CCS_syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
799 StrBufPlain(IO->ErrMsg, CKEY(POP3C_ReadErrors[pMsg->State]));
800 return FailAggregationRun(IO);
802 eNextState POP3_C_Shutdown(AsyncIO *IO)
804 EVP3CCS_syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
805 //// pop3aggr *pMsg = IO->Data;
807 ////pMsg->MyQEntry->Status = 3;
808 ///StrBufPlain(pMsg->MyQEntry->StatusMessage, HKEY("server shutdown during message retrieval."));
809 FinalizePOP3AggrRun(IO);
815 * @brief lineread Handler; understands when to read more POP3 lines,
816 * and when this is a one-lined reply.
818 eReadState POP3_C_ReadServerStatus(AsyncIO *IO)
820 eReadState Finished = eBufferNotEmpty;
822 switch (IO->NextState) {
827 case eTerminateConnection:
829 Finished = eReadFail;
836 Finished = StrBufChunkSipLine(IO->IOBuf, &IO->RecvBuf);
840 Finished = CtdlReadMessageBodyAsync(IO);
846 /*****************************************************************************
847 * So we connect our Server IP here. *
848 *****************************************************************************/
849 eNextState POP3_C_ReAttachToFetchMessages(AsyncIO *IO)
851 pop3aggr *cpptr = IO->Data;
853 EVP3CCS_syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
854 ////??? cpptr->State ++;
855 if (cpptr->Pos == NULL)
856 cpptr->Pos = GetNewHashPos(cpptr->MsgNumbers, 0);
858 POP3_C_DispatchWriteDone(IO);
859 ReAttachIO(IO, cpptr, 0);
860 IO->NextState = eReadMessage;
861 return IO->NextState;
864 eNextState pop3_connect_ip(AsyncIO *IO)
866 pop3aggr *cpptr = IO->Data;
868 if (cpptr->IOStart == 0.0) /* whith or without DNS? */
869 cpptr->IOStart = IO->Now;
871 EVP3CCS_syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
873 return EvConnectSock(IO,
875 POP3_C_ReadTimeouts[0],
879 eNextState pop3_get_one_host_ip_done(AsyncIO *IO)
881 pop3aggr *cpptr = IO->Data;
882 struct hostent *hostent;
886 hostent = cpptr->HostLookup.VParsedDNSReply;
887 if ((cpptr->HostLookup.DNSStatus == ARES_SUCCESS) &&
888 (hostent != NULL) ) {
889 memset(&cpptr->IO.ConnectMe->Addr, 0, sizeof(struct in6_addr));
890 if (cpptr->IO.ConnectMe->IPv6) {
891 memcpy(&cpptr->IO.ConnectMe->Addr.sin6_addr.s6_addr,
892 &hostent->h_addr_list[0],
893 sizeof(struct in6_addr));
895 cpptr->IO.ConnectMe->Addr.sin6_family =
897 cpptr->IO.ConnectMe->Addr.sin6_port =
898 htons(DefaultPOP3Port);
901 struct sockaddr_in *addr =
902 (struct sockaddr_in*)
903 &cpptr->IO.ConnectMe->Addr;
905 memcpy(&addr->sin_addr.s_addr,
906 hostent->h_addr_list[0],
909 addr->sin_family = hostent->h_addrtype;
910 addr->sin_port = htons(DefaultPOP3Port);
912 return pop3_connect_ip(IO);
918 eNextState pop3_get_one_host_ip(AsyncIO *IO)
920 pop3aggr *cpptr = IO->Data;
922 cpptr->IOStart = IO->Now;
924 EVP3CCS_syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
926 EVP3CCS_syslog(LOG_DEBUG,
927 "POP3 client[%ld]: looking up %s-Record %s : %d ...\n",
929 (cpptr->IO.ConnectMe->IPv6)? "aaaa": "a",
930 cpptr->IO.ConnectMe->Host,
931 cpptr->IO.ConnectMe->Port);
933 QueueQuery((cpptr->IO.ConnectMe->IPv6)? ns_t_aaaa : ns_t_a,
934 cpptr->IO.ConnectMe->Host,
937 pop3_get_one_host_ip_done);
938 IO->NextState = eReadDNSReply;
939 return IO->NextState;
944 int pop3_do_fetching(pop3aggr *cpptr)
946 AsyncIO *IO = &cpptr->IO;
951 POP3_C_ReadServerStatus,
953 POP3_C_DispatchWriteDone,
954 POP3_C_DispatchReadDone,
961 safestrncpy(((CitContext *)cpptr->IO.CitContext)->cs_host,
963 sizeof(((CitContext *)cpptr->IO.CitContext)->cs_host));
965 if (cpptr->IO.ConnectMe->IsIP) {
966 QueueEventContext(&cpptr->IO,
970 QueueEventContext(&cpptr->IO,
971 pop3_get_one_host_ip);
977 * Scan a room's netconfig to determine whether it requires POP3 aggregation
979 void pop3client_scan_room(struct ctdlroom *qrbuf, void *data)
986 char filename[PATH_MAX];
990 const char *CfgPtr, *lPtr;
993 // pop3_room_counter *Count = NULL;
996 pthread_mutex_lock(&POP3QueueMutex);
997 if (GetHash(POP3QueueRooms, LKEY(qrbuf->QRnumber), &vptr))
999 pthread_mutex_unlock(&POP3QueueMutex);
1000 EVP3CQ_syslog(LOG_DEBUG,
1001 "pop3client: [%ld] %s already in progress.",
1006 pthread_mutex_unlock(&POP3QueueMutex);
1008 if (server_shutting_down) return;
1010 assoc_file_name(filename, sizeof filename, qrbuf, ctdl_netcfg_dir);
1012 if (server_shutting_down)
1015 /* Only do net processing for rooms that have netconfigs */
1016 fd = open(filename, 0);
1020 if (server_shutting_down)
1022 if (fstat(fd, &statbuf) == -1) {
1023 EVP3CQ_syslog(LOG_INFO,
1024 "ERROR: could not stat configfile '%s' - %s",
1029 if (server_shutting_down)
1031 CfgData = NewStrBufPlain(NULL, statbuf.st_size + 1);
1032 if (StrBufReadBLOB(CfgData, &fd, 1, statbuf.st_size, &Err) < 0) {
1034 FreeStrBuf(&CfgData);
1035 EVP3CQ_syslog(LOG_INFO,
1036 "ERROR: reading config '%s' - %s",
1037 filename, strerror(errno));
1041 if (server_shutting_down)
1045 CfgType = NewStrBuf();
1046 Line = NewStrBufPlain(NULL, StrLength(CfgData));
1051 Done = StrBufSipLine(Line, CfgData, &CfgPtr) == 0;
1052 if (StrLength(Line) > 0)
1055 StrBufExtract_NextToken(CfgType, Line, &lPtr, '|');
1056 if (!strcasecmp("pop3client", ChrPtr(CfgType)))
1063 Count = malloc(sizeof(pop3_room_counter));
1068 cptr = (pop3aggr *) malloc(sizeof(pop3aggr));
1069 memset(cptr, 0, sizeof(pop3aggr));
1070 ///TODO do we need this? cptr->roomlist_parts=1;
1072 NewStrBufPlain(qrbuf->QRname, -1);
1074 NewStrBufPlain(NULL, StrLength(Line));
1076 NewStrBufPlain(NULL, StrLength(Line));
1077 cptr->Url = NewStrBuf();
1080 StrBufExtract_NextToken(Tmp, Line, &lPtr, '|');
1081 StrBufExtract_NextToken(cptr->pop3user,
1086 StrBufExtract_NextToken(cptr->pop3pass,
1091 cptr->keep = StrBufExtractNext_long(Line,
1095 cptr->interval = StrBufExtractNext_long(Line,
1099 StrBufAppendBufPlain(cptr->Url, HKEY("pop3://"), 0);
1100 StrBufUrlescUPAppend(cptr->Url, cptr->pop3user, NULL);
1101 StrBufAppendBufPlain(cptr->Url, HKEY(":"), 0);
1102 StrBufUrlescUPAppend(cptr->Url, cptr->pop3pass, NULL);
1103 StrBufAppendBufPlain(cptr->Url, HKEY("@"), 0);
1104 StrBufAppendBuf(cptr->Url, Tmp, 0);
1105 StrBufAppendBufPlain(cptr->Url, HKEY("/"), 0);
1106 StrBufUrlescAppend(cptr->Url, cptr->RoomName, NULL);
1109 ParseURL(&cptr->IO.ConnectMe, cptr->Url, 110);
1113 /* todo: we need to reunite the url to be shure. */
1115 pthread_mutex_lock(&POP3ueueMutex);
1116 GetHash(POP3FetchUrls, SKEY(ptr->Url), &vptr);
1117 use_this_cptr = (pop3aggr *)vptr;
1119 if (use_this_rncptr != NULL)
1121 /* mustn't attach to an active session */
1122 if (use_this_cptr->RefCount > 0)
1124 DeletePOP3Cfg(cptr);
1130 StrBufAppendBufPlain(
1131 use_this_cptr->rooms,
1134 if (use_this_cptr->roomlist_parts == 1)
1136 use_this_cptr->OtherQRnumbers
1137 = NewHash(1, lFlathash);
1139 QRnumber = (long*)malloc(sizeof(long));
1140 *QRnumber = qrbuf->QRnumber;
1141 Put(use_this_cptr->OtherQRnumbers,
1142 LKEY(qrbuf->QRnumber),
1146 use_this_cptr->roomlist_parts++;
1148 pthread_mutex_unlock(&POP3QueueMutex);
1151 pthread_mutex_unlock(&RSSQueueMutex);
1153 cptr->n = Pop3ClientID++;
1154 pthread_mutex_lock(&POP3QueueMutex);
1158 DeletePOP3Aggregator);
1160 pthread_mutex_unlock(&POP3QueueMutex);
1170 FreeStrBuf(&CfgType);
1171 FreeStrBuf(&CfgData);
1174 static int doing_pop3client = 0;
1176 void pop3client_scan(void) {
1177 static time_t last_run = 0L;
1178 time_t fastest_scan;
1185 become_session(&pop3_client_CC);
1187 if (config.c_pop3_fastest < config.c_pop3_fetch)
1188 fastest_scan = config.c_pop3_fastest;
1190 fastest_scan = config.c_pop3_fetch;
1193 * Run POP3 aggregation no more frequently than once every n seconds
1195 if ( (time(NULL) - last_run) < fastest_scan ) {
1200 * This is a simple concurrency check to make sure only one pop3client
1201 * run is done at a time. We could do this with a mutex, but since we
1202 * don't really require extremely fine granularity here, we'll do it
1203 * with a static variable instead.
1205 if (doing_pop3client) return;
1206 doing_pop3client = 1;
1208 EVP3CQM_syslog(LOG_DEBUG, "pop3client started");
1209 CtdlForEachRoom(pop3client_scan_room, NULL);
1211 pthread_mutex_lock(&POP3QueueMutex);
1212 it = GetNewHashPos(POP3FetchUrls, 0);
1213 while (!server_shutting_down &&
1214 GetNextHashPos(POP3FetchUrls, it, &len, &Key, &vrptr) &&
1216 cptr = (pop3aggr *)vrptr;
1217 if (cptr->RefCount == 0)
1218 if (!pop3_do_fetching(cptr))
1219 DeletePOP3Aggregator(cptr);////TODO
1222 if ((palist->interval && time(NULL) > (last_run + palist->interval))
1223 || (time(NULL) > last_run + config.c_pop3_fetch))
1224 pop3_do_fetching(palist->roomname, palist->pop3host,
1225 palist->pop3user, palist->pop3pass, palist->keep);
1227 palist = palist->next;
1232 pthread_mutex_unlock(&POP3QueueMutex);
1234 EVP3CQM_syslog(LOG_DEBUG, "pop3client ended");
1235 last_run = time(NULL);
1236 doing_pop3client = 0;
1240 void pop3_cleanup(void)
1242 /* citthread_mutex_destroy(&POP3QueueMutex); TODO */
1243 while (doing_pop3client != 0) ;
1244 DeleteHash(&POP3FetchUrls);
1245 DeleteHash(&POP3QueueRooms);
1250 void LogDebugEnablePOP3Client(const int n)
1252 POP3ClientDebugEnabled = n;
1255 CTDL_MODULE_INIT(pop3client)
1259 CtdlFillSystemContext(&pop3_client_CC, "POP3aggr");
1260 pthread_mutex_init(&POP3QueueMutex, NULL);
1261 POP3QueueRooms = NewHash(1, lFlathash);
1262 POP3FetchUrls = NewHash(1, NULL);
1263 CtdlRegisterSessionHook(pop3client_scan, EVT_TIMER);
1264 CtdlRegisterEVCleanupHook(pop3_cleanup);
1265 CtdlRegisterDebugFlagHook(HKEY("pop3client"), LogDebugEnablePOP3Client, &POP3ClientDebugEnabled);
1268 /* return our module id for the log */
1269 return "pop3client";