2 * Consolidate mail from remote POP3 accounts.
4 * Copyright (c) 2007-2011 by the citadel.org team
6 * This program is open source software; you can redistribute it and/or
7 * modify it under the terms of the GNU General Public License as published
8 * by the Free Software Foundation; either version 3 of the License, or
9 * (at your option) any later version.
11 * This program is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 * GNU General Public License for more details.
16 * You should have received a copy of the GNU General Public License
17 * along with this program; if not, write to the Free Software
18 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
25 #if TIME_WITH_SYS_TIME
26 # include <sys/time.h>
30 # include <sys/time.h>
39 #include <sys/types.h>
41 #include <libcitadel.h>
44 #include "citserver.h"
47 #include "ctdl_module.h"
48 #include "clientsocket.h"
50 #include "internet_addressing.h"
52 #include "citadel_dirs.h"
53 #include "event_client.h"
56 struct CitContext pop3_client_CC;
58 pthread_mutex_t POP3QueueMutex; /* locks the access to the following vars: */
59 HashList *POP3QueueRooms = NULL; /* rss_room_counter */
60 HashList *POP3FetchUrls = NULL; /* -> rss_aggregator; ->RefCount access to be locked too. */
62 typedef struct __pop3_room_counter {
67 typedef enum ePOP3_C_States {
74 ReadMessageBodyFollowing,
82 typedef struct _FetchItem {
88 struct CtdlMessage *Msg;
91 void HfreeFetchItem(void *vItem)
93 FetchItem *Item = (FetchItem*) vItem;
94 FreeStrBuf(&Item->MsgUIDL);
95 FreeStrBuf(&Item->MsgUID);
99 typedef struct __pop3aggr {
104 /// ParsedURL *Pop3Host;
105 DNSQueryParts HostLookup;
109 HashList *OtherQRnumbers;
112 /// StrBuf *pop3host; -> URL
115 StrBuf *RoomName; // TODO: fill me
118 ePOP3_C_States State;
119 HashList *MsgNumbers;
124 void DeletePOP3Aggregator(void *vptr)
126 pop3aggr *ptr = vptr;
127 DeleteHashPos(&ptr->Pos);
128 DeleteHash(&ptr->MsgNumbers);
129 // FreeStrBuf(&ptr->rooms);
130 FreeStrBuf(&ptr->pop3user);
131 FreeStrBuf(&ptr->pop3pass);
132 FreeStrBuf(&ptr->RoomName);
133 FreeURL(&ptr->IO.ConnectMe);
134 FreeStrBuf(&ptr->Url);
135 FreeStrBuf(&ptr->IO.IOBuf);
136 FreeStrBuf(&ptr->IO.SendBuf.Buf);
137 FreeStrBuf(&ptr->IO.RecvBuf.Buf);
138 DeleteAsyncMsg(&ptr->IO.ReadMsg);
139 ((struct CitContext*)ptr->IO.CitContext)->state = CON_IDLE;
140 ((struct CitContext*)ptr->IO.CitContext)->kill_me = 1;
141 FreeAsyncIOContents(&ptr->IO);
146 typedef eNextState(*Pop3ClientHandler)(pop3aggr* RecvMsg);
148 eNextState POP3_C_Shutdown(AsyncIO *IO);
149 eNextState POP3_C_Timeout(AsyncIO *IO);
150 eNextState POP3_C_ConnFail(AsyncIO *IO);
151 eNextState POP3_C_DNSFail(AsyncIO *IO);
152 eNextState POP3_C_DispatchReadDone(AsyncIO *IO);
153 eNextState POP3_C_DispatchWriteDone(AsyncIO *IO);
154 eNextState POP3_C_Terminate(AsyncIO *IO);
155 eReadState POP3_C_ReadServerStatus(AsyncIO *IO);
156 eNextState POP3_C_ReAttachToFetchMessages(AsyncIO *IO);
158 eNextState FinalizePOP3AggrRun(AsyncIO *IO)
161 pop3aggr *cptr = (pop3aggr *)IO->Data;
163 syslog(LOG_DEBUG, "Terminating Aggregator; bye.\n");
165 It = GetNewHashPos(POP3FetchUrls, 0);
166 pthread_mutex_lock(&POP3QueueMutex);
168 if (GetHashPosFromKey(POP3FetchUrls, SKEY(cptr->Url), It))
169 DeleteEntryFromHash(POP3FetchUrls, It);
171 pthread_mutex_unlock(&POP3QueueMutex);
176 eNextState FailAggregationRun(AsyncIO *IO)
182 #define POP3C_DBG_SEND() syslog(LOG_DEBUG, "POP3 client[%ld]: > %s\n", RecvMsg->n, ChrPtr(RecvMsg->IO.SendBuf.Buf))
183 #define POP3C_DBG_READ() syslog(LOG_DEBUG, "POP3 client[%ld]: < %s\n", RecvMsg->n, ChrPtr(RecvMsg->IO.IOBuf))
184 #define POP3C_OK (strncasecmp(ChrPtr(RecvMsg->IO.IOBuf), "+OK", 3) == 0)
186 eNextState POP3C_ReadGreeting(pop3aggr *RecvMsg)
189 /* Read the server greeting */
190 if (!POP3C_OK) return eTerminateConnection;
191 else return eSendReply;
194 eNextState POP3C_SendUser(pop3aggr *RecvMsg)
196 /* Identify ourselves. NOTE: we have to append a CR to each command. The LF will
197 * automatically be appended by sock_puts(). Believe it or not, leaving out the CR
198 * will cause problems if the server happens to be Exchange, which is so b0rken it
199 * actually barfs on LF-terminated newlines.
201 StrBufPrintf(RecvMsg->IO.SendBuf.Buf,
202 "USER %s\r\n", ChrPtr(RecvMsg->pop3user));
207 eNextState POP3C_GetUserState(pop3aggr *RecvMsg)
210 if (!POP3C_OK) return eTerminateConnection;
211 else return eSendReply;
214 eNextState POP3C_SendPassword(pop3aggr *RecvMsg)
217 StrBufPrintf(RecvMsg->IO.SendBuf.Buf,
218 "PASS %s\r\n", ChrPtr(RecvMsg->pop3pass));
219 syslog(LOG_DEBUG, "<PASS <password>\n");
224 eNextState POP3C_GetPassState(pop3aggr *RecvMsg)
227 if (!POP3C_OK) return eTerminateConnection;
228 else return eSendReply;
231 eNextState POP3C_SendListCommand(pop3aggr *RecvMsg)
233 /* Get the list of messages */
234 StrBufPlain(RecvMsg->IO.SendBuf.Buf, HKEY("LIST\r\n"));
239 eNextState POP3C_GetListCommandState(pop3aggr *RecvMsg)
242 if (!POP3C_OK) return eTerminateConnection;
243 RecvMsg->MsgNumbers = NewHash(1, NULL);
249 eNextState POP3C_GetListOneLine(pop3aggr *RecvMsg)
255 FetchItem *OneMsg = NULL;
258 if ((StrLength(RecvMsg->IO.IOBuf) == 1) &&
259 (ChrPtr(RecvMsg->IO.IOBuf)[0] == '.'))
261 if (GetCount(RecvMsg->MsgNumbers) == 0)
263 //// RecvMsg->Sate = ReadQuitState;
267 RecvMsg->Pos = GetNewHashPos(RecvMsg->MsgNumbers, 0);
272 OneMsg = (FetchItem*) malloc(sizeof(FetchItem));
273 memset(OneMsg, 0, sizeof(FetchItem));
274 OneMsg->MSGID = atol(ChrPtr(RecvMsg->IO.IOBuf));
276 pch = strchr(ChrPtr(RecvMsg->IO.IOBuf), ' ');
279 OneMsg->MSGSize = atol(pch + 1);
282 rc = TestValidateHash(RecvMsg->MsgNumbers);
284 syslog(LOG_DEBUG, "Hash Invalid: %d\n", rc);
287 Put(RecvMsg->MsgNumbers, LKEY(OneMsg->MSGID), OneMsg, HfreeFetchItem);
289 rc = TestValidateHash(RecvMsg->MsgNumbers);
291 syslog(LOG_DEBUG, "Hash Invalid: %d\n", rc);
293 //RecvMsg->State --; /* read next Line */
297 eNextState POP3_FetchNetworkUsetableEntry(AsyncIO *IO)
302 struct cdbdata *cdbut;
303 pop3aggr *RecvMsg = (pop3aggr *) IO->Data;
305 if(GetNextHashPos(RecvMsg->MsgNumbers, RecvMsg->Pos, &HKLen, &HKey, &vData))
308 if (server_shutting_down)
311 RecvMsg->CurrMsg = (FetchItem*) vData;
312 syslog(LOG_DEBUG, "CHECKING: whether %s has already been seen: ", ChrPtr(RecvMsg->CurrMsg->MsgUID));
313 /* Find out if we've already seen this item */
314 safestrncpy(ut.ut_msgid,
315 ChrPtr(RecvMsg->CurrMsg->MsgUID),
316 sizeof(ut.ut_msgid));
317 ut.ut_timestamp = time(NULL);/// TODO: libev timestamp!
319 cdbut = cdb_fetch(CDB_USETABLE, SKEY(RecvMsg->CurrMsg->MsgUID));
321 /* Item has already been seen */
322 syslog(LOG_DEBUG, "YES\n");
325 /* rewrite the record anyway, to update the timestamp */
326 cdb_store(CDB_USETABLE,
327 SKEY(RecvMsg->CurrMsg->MsgUID),
328 &ut, sizeof(struct UseTable) );
329 RecvMsg->CurrMsg->NeedFetch = 0; ////TODO0;
333 syslog(LOG_DEBUG, "NO\n");
334 RecvMsg->CurrMsg->NeedFetch = 1;
336 return NextDBOperation(&RecvMsg->IO, POP3_FetchNetworkUsetableEntry);
340 /* ok, now we know them all, continue with reading the actual messages. */
341 DeleteHashPos(&RecvMsg->Pos);
343 return QueueEventContext(IO, POP3_C_ReAttachToFetchMessages);
347 eNextState POP3C_GetOneMessagID(pop3aggr *RecvMsg)
355 rc = TestValidateHash(RecvMsg->MsgNumbers);
357 syslog(LOG_DEBUG, "Hash Invalid: %d\n", rc);
359 if(GetNextHashPos(RecvMsg->MsgNumbers, RecvMsg->Pos, &HKLen, &HKey, &vData))
361 RecvMsg->CurrMsg = (FetchItem*) vData;
362 /* Find out the UIDL of the message, to determine whether we've already downloaded it */
363 StrBufPrintf(RecvMsg->IO.SendBuf.Buf,
364 "UIDL %ld\r\n", RecvMsg->CurrMsg->MSGID);
370 DeleteHashPos(&RecvMsg->Pos);
371 /// done receiving uidls.. start looking them up now.
372 RecvMsg->Pos = GetNewHashPos(RecvMsg->MsgNumbers, 0);
373 return QueueDBOperation(&RecvMsg->IO, POP3_FetchNetworkUsetableEntry);
375 return eReadMore; /* TODO */
378 eNextState POP3C_GetOneMessageIDState(pop3aggr *RecvMsg)
382 rc = TestValidateHash(RecvMsg->MsgNumbers);
384 syslog(LOG_DEBUG, "Hash Invalid: %d\n", rc);
388 if (!POP3C_OK) return eTerminateConnection;
389 RecvMsg->CurrMsg->MsgUIDL = NewStrBufPlain(NULL, StrLength(RecvMsg->IO.IOBuf));
390 RecvMsg->CurrMsg->MsgUID = NewStrBufPlain(NULL, StrLength(RecvMsg->IO.IOBuf) * 2);
392 StrBufExtract_token(RecvMsg->CurrMsg->MsgUIDL, RecvMsg->IO.IOBuf, 2, ' ');
393 StrBufPrintf(RecvMsg->CurrMsg->MsgUID,
395 ChrPtr(RecvMsg->RoomName),
396 ChrPtr(RecvMsg->CurrMsg->MsgUIDL),
397 RecvMsg->IO.ConnectMe->User,
398 RecvMsg->IO.ConnectMe->Host);
404 eNextState POP3C_SendGetOneMsg(pop3aggr *RecvMsg)
410 RecvMsg->CurrMsg = NULL;
411 while (GetNextHashPos(RecvMsg->MsgNumbers, RecvMsg->Pos, &HKLen, &HKey, &vData) &&
412 (RecvMsg->CurrMsg = (FetchItem*) vData, RecvMsg->CurrMsg->NeedFetch == 0))
415 if ((RecvMsg->CurrMsg != NULL ) && (RecvMsg->CurrMsg->NeedFetch == 1))
417 /* Message has not been seen. Tell the server to fetch the message... */
418 StrBufPrintf(RecvMsg->IO.SendBuf.Buf,
419 "RETR %ld\r\n", RecvMsg->CurrMsg->MSGID);
424 RecvMsg->State = ReadQuitState;
425 return POP3_C_DispatchWriteDone(&RecvMsg->IO);
430 eNextState POP3C_ReadMessageBodyFollowing(pop3aggr *RecvMsg)
433 if (!POP3C_OK) return eTerminateConnection;
434 RecvMsg->IO.ReadMsg = NewAsyncMsg(HKEY("."),
435 RecvMsg->CurrMsg->MSGSize,
444 eNextState POP3C_StoreMsgRead(AsyncIO *IO)
446 pop3aggr *RecvMsg = (pop3aggr *) IO->Data;
449 syslog(LOG_DEBUG, "MARKING: %s as seen: ", ChrPtr(RecvMsg->CurrMsg->MsgUID));
451 safestrncpy(ut.ut_msgid,
452 ChrPtr(RecvMsg->CurrMsg->MsgUID),
453 sizeof(ut.ut_msgid));
454 ut.ut_timestamp = time(NULL); /* TODO: use libev time */
455 cdb_store(CDB_USETABLE,
456 ChrPtr(RecvMsg->CurrMsg->MsgUID),
457 StrLength(RecvMsg->CurrMsg->MsgUID),
459 sizeof(struct UseTable) );
461 return QueueEventContext(&RecvMsg->IO, POP3_C_ReAttachToFetchMessages);
463 eNextState POP3C_SaveMsg(AsyncIO *IO)
466 pop3aggr *RecvMsg = (pop3aggr *) IO->Data;
468 /* Do Something With It (tm) */
469 msgnum = CtdlSubmitMsg(RecvMsg->CurrMsg->Msg,
471 ChrPtr(RecvMsg->RoomName),
474 /* Message has been committed to the store */
475 /* write the uidl to the use table so we don't fetch this message again */
477 CtdlFreeMessage(RecvMsg->CurrMsg->Msg);
479 return NextDBOperation(&RecvMsg->IO, POP3C_StoreMsgRead);
482 eNextState POP3C_ReadMessageBody(pop3aggr *RecvMsg)
484 syslog(LOG_DEBUG, "Converting message...\n");
485 RecvMsg->CurrMsg->Msg = convert_internet_message_buf(&RecvMsg->IO.ReadMsg->MsgBuf);
487 return QueueDBOperation(&RecvMsg->IO, POP3C_SaveMsg);
490 eNextState POP3C_SendDelete(pop3aggr *RecvMsg)
492 if (!RecvMsg->keep) {
493 StrBufPrintf(RecvMsg->IO.SendBuf.Buf,
494 "DELE %ld\r\n", RecvMsg->CurrMsg->MSGID);
499 RecvMsg->State = ReadMessageBodyFollowing;
500 return POP3_C_DispatchWriteDone(&RecvMsg->IO);
503 eNextState POP3C_ReadDeleteState(pop3aggr *RecvMsg)
506 RecvMsg->State = GetOneMessageIDState;
510 eNextState POP3C_SendQuit(pop3aggr *RecvMsg)
513 StrBufPlain(RecvMsg->IO.SendBuf.Buf,
520 eNextState POP3C_ReadQuitState(pop3aggr *RecvMsg)
523 return eTerminateConnection;
526 const long POP3_C_ConnTimeout = 1000;
527 const long DefaultPOP3Port = 110;
529 Pop3ClientHandler POP3C_ReadHandlers[] = {
533 POP3C_GetListCommandState,
534 POP3C_GetListOneLine,
535 POP3C_GetOneMessageIDState,
536 POP3C_ReadMessageBodyFollowing,
537 POP3C_ReadMessageBody,
538 POP3C_ReadDeleteState,
542 const long POP3_C_SendTimeouts[POP3C_MaxRead] = {
552 const ConstStr POP3C_ReadErrors[POP3C_MaxRead] = {
553 {HKEY("Connection broken during ")},
554 {HKEY("Connection broken during ")},
555 {HKEY("Connection broken during ")},
556 {HKEY("Connection broken during ")},
557 {HKEY("Connection broken during ")},
558 {HKEY("Connection broken during ")},
559 {HKEY("Connection broken during ")},
560 {HKEY("Connection broken during ")}
563 Pop3ClientHandler POP3C_SendHandlers[] = {
564 NULL, /* we don't send a greeting */
567 POP3C_SendListCommand,
569 POP3C_GetOneMessagID,
576 const long POP3_C_ReadTimeouts[] = {
588 /*****************************************************************************/
589 /* POP3 CLIENT DISPATCHER */
590 /*****************************************************************************/
592 void POP3SetTimeout(eNextState NextTCPState, pop3aggr *pMsg)
594 double Timeout = 0.0;
596 syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
598 switch (NextTCPState) {
602 Timeout = POP3_C_SendTimeouts[pMsg->State];
604 if (pMsg->State == eDATABody) {
605 / * if we're sending a huge message, we need more time. * /
606 Timeout += StrLength(pMsg->msgtext) / 1024;
612 Timeout = POP3_C_ReadTimeouts[pMsg->State];
614 if (pMsg->State == eDATATerminateBody) {
616 * some mailservers take a nap before accepting the message
617 * content inspection and such.
619 Timeout += StrLength(pMsg->msgtext) / 1024;
630 case eTerminateConnection:
633 case eReadMore://// TODO
636 SetNextTimeout(&pMsg->IO, Timeout);
638 eNextState POP3_C_DispatchReadDone(AsyncIO *IO)
640 syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
641 pop3aggr *pMsg = IO->Data;
644 rc = POP3C_ReadHandlers[pMsg->State](pMsg);
647 POP3SetTimeout(rc, pMsg);
650 eNextState POP3_C_DispatchWriteDone(AsyncIO *IO)
652 syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
653 pop3aggr *pMsg = IO->Data;
656 rc = POP3C_SendHandlers[pMsg->State](pMsg);
657 POP3SetTimeout(rc, pMsg);
662 /*****************************************************************************/
663 /* POP3 CLIENT ERROR CATCHERS */
664 /*****************************************************************************/
665 eNextState POP3_C_Terminate(AsyncIO *IO)
667 /// pop3aggr *pMsg = (pop3aggr *)IO->Data;
669 syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
670 FinalizePOP3AggrRun(IO);
673 eNextState POP3_C_Timeout(AsyncIO *IO)
675 pop3aggr *pMsg = IO->Data;
677 syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
678 StrBufPlain(IO->ErrMsg, CKEY(POP3C_ReadErrors[pMsg->State]));
679 return FailAggregationRun(IO);
681 eNextState POP3_C_ConnFail(AsyncIO *IO)
683 pop3aggr *pMsg = (pop3aggr *)IO->Data;
685 syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
686 StrBufPlain(IO->ErrMsg, CKEY(POP3C_ReadErrors[pMsg->State]));
687 return FailAggregationRun(IO);
689 eNextState POP3_C_DNSFail(AsyncIO *IO)
691 pop3aggr *pMsg = (pop3aggr *)IO->Data;
693 syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
694 StrBufPlain(IO->ErrMsg, CKEY(POP3C_ReadErrors[pMsg->State]));
695 return FailAggregationRun(IO);
697 eNextState POP3_C_Shutdown(AsyncIO *IO)
699 syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
700 //// pop3aggr *pMsg = IO->Data;
702 ////pMsg->MyQEntry->Status = 3;
703 ///StrBufPlain(pMsg->MyQEntry->StatusMessage, HKEY("server shutdown during message retrieval."));
704 FinalizePOP3AggrRun(IO);
710 * @brief lineread Handler; understands when to read more POP3 lines, and when this is a one-lined reply.
712 eReadState POP3_C_ReadServerStatus(AsyncIO *IO)
714 eReadState Finished = eBufferNotEmpty;
716 switch (IO->NextState) {
721 case eTerminateConnection:
723 Finished = eReadFail;
730 Finished = StrBufChunkSipLine(IO->IOBuf, &IO->RecvBuf);
734 Finished = CtdlReadMessageBodyAsync(IO);
740 /*****************************************************************************
741 * So we connect our Server IP here. *
742 *****************************************************************************/
743 eNextState POP3_C_ReAttachToFetchMessages(AsyncIO *IO)
745 pop3aggr *cpptr = IO->Data;
747 syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
748 ////??? cpptr->State ++;
749 if (cpptr->Pos == NULL)
750 cpptr->Pos = GetNewHashPos(cpptr->MsgNumbers, 0);
752 POP3_C_DispatchWriteDone(IO);
753 ReAttachIO(IO, cpptr, 0);
754 IO->NextState = eReadMessage;
755 return IO->NextState;
758 eNextState pop3_connect_ip(AsyncIO *IO)
760 pop3aggr *cpptr = IO->Data;
762 syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
764 //// IO->ConnectMe = &cpptr->Pop3Host;
765 /* Bypass the ns lookup result like this: IO->Addr.sin_addr.s_addr = inet_addr("127.0.0.1"); */
767 /////// SetConnectStatus(IO);
769 return InitEventIO(IO, cpptr,
771 POP3_C_ReadTimeouts[0],
775 eNextState pop3_get_one_host_ip_done(AsyncIO *IO)
777 pop3aggr *cpptr = IO->Data;
778 struct hostent *hostent;
782 hostent = cpptr->HostLookup.VParsedDNSReply;
783 if ((cpptr->HostLookup.DNSStatus == ARES_SUCCESS) &&
784 (hostent != NULL) ) {
785 memset(&cpptr->IO.ConnectMe->Addr, 0, sizeof(struct in6_addr));
786 if (cpptr->IO.ConnectMe->IPv6) {
787 memcpy(&cpptr->IO.ConnectMe->Addr.sin6_addr.s6_addr,
788 &hostent->h_addr_list[0],
789 sizeof(struct in6_addr));
791 cpptr->IO.ConnectMe->Addr.sin6_family = hostent->h_addrtype;
792 cpptr->IO.ConnectMe->Addr.sin6_port = htons(DefaultPOP3Port);
795 struct sockaddr_in *addr = (struct sockaddr_in*) &cpptr->IO.ConnectMe->Addr;
796 /* Bypass the ns lookup result like this: IO->Addr.sin_addr.s_addr = inet_addr("127.0.0.1"); */
797 // addr->sin_addr.s_addr = htonl((uint32_t)&hostent->h_addr_list[0]);
798 memcpy(&addr->sin_addr.s_addr,
799 hostent->h_addr_list[0],
802 addr->sin_family = hostent->h_addrtype;
803 addr->sin_port = htons(DefaultPOP3Port);
806 return pop3_connect_ip(IO);
812 eNextState pop3_get_one_host_ip(AsyncIO *IO)
814 pop3aggr *cpptr = IO->Data;
816 * here we start with the lookup of one host. it might be...
817 * - the relay host *sigh*
818 * - the direct hostname if there was no mx record
824 syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
827 "POP3 client[%ld]: looking up %s-Record %s : %d ...\n",
829 (cpptr->IO.ConnectMe->IPv6)? "aaaa": "a",
830 cpptr->IO.ConnectMe->Host,
831 cpptr->IO.ConnectMe->Port);
833 QueueQuery((cpptr->IO.ConnectMe->IPv6)? ns_t_aaaa : ns_t_a,
834 cpptr->IO.ConnectMe->Host,
837 pop3_get_one_host_ip_done);
838 IO->NextState = eReadDNSReply;
839 return IO->NextState;
844 int pop3_do_fetching(pop3aggr *cpptr)
848 cpptr->IO.Data = cpptr;
850 cpptr->IO.SendDone = POP3_C_DispatchWriteDone;
851 cpptr->IO.ReadDone = POP3_C_DispatchReadDone;
852 cpptr->IO.Terminate = POP3_C_Terminate;
853 cpptr->IO.LineReader = POP3_C_ReadServerStatus;
854 cpptr->IO.ConnFail = POP3_C_ConnFail;
855 cpptr->IO.DNSFail = POP3_C_DNSFail;
856 cpptr->IO.Timeout = POP3_C_Timeout;
857 cpptr->IO.ShutdownAbort = POP3_C_Shutdown;
859 cpptr->IO.SendBuf.Buf = NewStrBufPlain(NULL, 1024);
860 cpptr->IO.RecvBuf.Buf = NewStrBufPlain(NULL, 1024);
861 cpptr->IO.IOBuf = NewStrBuf();
863 cpptr->IO.NextState = eReadMessage;
865 syslog(LOG_DEBUG, "POP3: %s %s %s <password>\n", roomname, pop3host, pop3user);
866 syslog(LOG_DEBUG, "Connecting to <%s>\n", pop3host);
869 SubC = CloneContext (&pop3_client_CC);
870 SubC->session_specific_data = (char*) cpptr;
871 cpptr->IO.CitContext = SubC;
872 safestrncpy(SubC->cs_host,
874 sizeof(SubC->cs_host));
876 if (cpptr->IO.ConnectMe->IsIP) {
877 QueueEventContext(&cpptr->IO,
880 else { /* uneducated admin has chosen to add DNS to the equation... */
881 QueueEventContext(&cpptr->IO,
882 pop3_get_one_host_ip);
888 * Scan a room's netconfig to determine whether it requires POP3 aggregation
890 void pop3client_scan_room(struct ctdlroom *qrbuf, void *data)
897 char filename[PATH_MAX];
901 const char *CfgPtr, *lPtr;
904 // pop3_room_counter *Count = NULL;
907 pthread_mutex_lock(&POP3QueueMutex);
908 if (GetHash(POP3QueueRooms, LKEY(qrbuf->QRnumber), &vptr))
911 "pop3client: [%ld] %s already in progress.\n",
914 pthread_mutex_unlock(&POP3QueueMutex);
916 pthread_mutex_unlock(&POP3QueueMutex);
918 if (server_shutting_down) return;
920 assoc_file_name(filename, sizeof filename, qrbuf, ctdl_netcfg_dir);
922 if (server_shutting_down)
925 /* Only do net processing for rooms that have netconfigs */
926 fd = open(filename, 0);
928 //syslog(LOG_DEBUG, "rssclient: %s no config.\n", qrbuf->QRname);
931 if (server_shutting_down)
933 if (fstat(fd, &statbuf) == -1) {
934 syslog(LOG_DEBUG, "ERROR: could not stat configfile '%s' - %s\n",
935 filename, strerror(errno));
938 if (server_shutting_down)
940 CfgData = NewStrBufPlain(NULL, statbuf.st_size + 1);
941 if (StrBufReadBLOB(CfgData, &fd, 1, statbuf.st_size, &Err) < 0) {
943 FreeStrBuf(&CfgData);
944 syslog(LOG_DEBUG, "ERROR: reading config '%s' - %s<br>\n",
945 filename, strerror(errno));
949 if (server_shutting_down)
953 CfgType = NewStrBuf();
954 Line = NewStrBufPlain(NULL, StrLength(CfgData));
959 Done = StrBufSipLine(Line, CfgData, &CfgPtr) == 0;
960 if (StrLength(Line) > 0)
963 StrBufExtract_NextToken(CfgType, Line, &lPtr, '|');
964 if (!strcasecmp("pop3client", ChrPtr(CfgType)))
971 Count = malloc(sizeof(pop3_room_counter));
976 cptr = (pop3aggr *) malloc(sizeof(pop3aggr));
977 memset(cptr, 0, sizeof(pop3aggr));
978 /// TODO do we need this? cptr->roomlist_parts = 1;
979 cptr->RoomName = NewStrBufPlain(qrbuf->QRname, -1);
980 cptr->pop3user = NewStrBufPlain(NULL, StrLength(Line));
981 cptr->pop3pass = NewStrBufPlain(NULL, StrLength(Line));
982 cptr->Url = NewStrBuf();
985 StrBufExtract_NextToken(Tmp, Line, &lPtr, '|');
986 StrBufExtract_NextToken(cptr->pop3user, Line, &lPtr, '|');
987 StrBufExtract_NextToken(cptr->pop3pass, Line, &lPtr, '|');
988 cptr->keep = StrBufExtractNext_long(Line, &lPtr, '|');
989 cptr->interval = StrBufExtractNext_long(Line, &lPtr, '|');
991 StrBufPrintf(cptr->Url, "pop3://%s:%s@%s/%s",
992 ChrPtr(cptr->pop3user),
993 ChrPtr(cptr->pop3pass),
995 ChrPtr(cptr->RoomName));
997 ParseURL(&cptr->IO.ConnectMe, cptr->Url, 110);
1001 /* todo: we need to reunite the url to be shure. */
1003 pthread_mutex_lock(&POP3ueueMutex);
1004 GetHash(POP3FetchUrls, SKEY(ptr->Url), &vptr);
1005 use_this_cptr = (pop3aggr *)vptr;
1007 if (use_this_rncptr != NULL)
1009 /* mustn't attach to an active session */
1010 if (use_this_cptr->RefCount > 0)
1012 DeletePOP3Cfg(cptr);
1018 StrBufAppendBufPlain(use_this_cptr->rooms,
1021 if (use_this_cptr->roomlist_parts == 1)
1023 use_this_cptr->OtherQRnumbers = NewHash(1, lFlathash);
1025 QRnumber = (long*)malloc(sizeof(long));
1026 *QRnumber = qrbuf->QRnumber;
1027 Put(use_this_cptr->OtherQRnumbers, LKEY(qrbuf->QRnumber), QRnumber, NULL);
1028 use_this_cptr->roomlist_parts++;
1030 pthread_mutex_unlock(&POP3QueueMutex);
1033 pthread_mutex_unlock(&RSSQueueMutex);
1036 pthread_mutex_lock(&POP3QueueMutex);
1037 Put(POP3FetchUrls, SKEY(cptr->Url), cptr, DeletePOP3Aggregator);
1038 pthread_mutex_unlock(&POP3QueueMutex);
1048 FreeStrBuf(&CfgType);
1049 FreeStrBuf(&CfgData);
1052 static int doing_pop3client = 0;
1054 void pop3client_scan(void) {
1055 static time_t last_run = 0L;
1056 /// struct pop3aggr *pptr;
1057 time_t fastest_scan;
1064 if (config.c_pop3_fastest < config.c_pop3_fetch)
1065 fastest_scan = config.c_pop3_fastest;
1067 fastest_scan = config.c_pop3_fetch;
1070 * Run POP3 aggregation no more frequently than once every n seconds
1072 if ( (time(NULL) - last_run) < fastest_scan ) {
1077 * This is a simple concurrency check to make sure only one pop3client run
1078 * is done at a time. We could do this with a mutex, but since we
1079 * don't really require extremely fine granularity here, we'll do it
1080 * with a static variable instead.
1082 if (doing_pop3client) return;
1083 doing_pop3client = 1;
1085 syslog(LOG_DEBUG, "pop3client started");
1086 CtdlForEachRoom(pop3client_scan_room, NULL);
1088 pthread_mutex_lock(&POP3QueueMutex);
1089 it = GetNewHashPos(POP3FetchUrls, 0);
1090 while (!server_shutting_down &&
1091 GetNextHashPos(POP3FetchUrls, it, &len, &Key, &vrptr) &&
1093 cptr = (pop3aggr *)vrptr;
1094 if (cptr->RefCount == 0)
1095 if (!pop3_do_fetching(cptr))
1096 DeletePOP3Aggregator(cptr);////TODO
1099 if ((palist->interval && time(NULL) > (last_run + palist->interval))
1100 || (time(NULL) > last_run + config.c_pop3_fetch))
1101 pop3_do_fetching(palist->roomname, palist->pop3host,
1102 palist->pop3user, palist->pop3pass, palist->keep);
1104 palist = palist->next;
1109 pthread_mutex_unlock(&POP3QueueMutex);
1111 syslog(LOG_DEBUG, "pop3client ended");
1112 last_run = time(NULL);
1113 doing_pop3client = 0;
1117 void pop3_cleanup(void)
1119 /* citthread_mutex_destroy(&POP3QueueMutex); TODO */
1120 while (doing_pop3client != 0) ;
1121 DeleteHash(&POP3FetchUrls);
1122 DeleteHash(&POP3QueueRooms);
1125 CTDL_MODULE_INIT(pop3client)
1129 CtdlFillSystemContext(&pop3_client_CC, "POP3aggr");
1130 pthread_mutex_init(&POP3QueueMutex, NULL);
1131 POP3QueueRooms = NewHash(1, lFlathash);
1132 POP3FetchUrls = NewHash(1, NULL);
1133 CtdlRegisterSessionHook(pop3client_scan, EVT_TIMER);
1134 CtdlRegisterCleanupHook(pop3_cleanup);
1137 /* return our module id for the log */
1138 return "pop3client";