81feca096c9410fc52ac54865480b007e765a2cd
[citadel.git] / citadel / modules / pop3client / serv_pop3client.c
1 /*
2  * Consolidate mail from remote POP3 accounts.
3  *
4  * Copyright (c) 2007-2011 by the citadel.org team
5  *
6  * This program is open source software; you can redistribute it and/or
7  * modify it under the terms of the GNU General Public License as published
8  * by the Free Software Foundation; either version 3 of the License, or
9  * (at your option) any later version.
10  *
11  * This program is distributed in the hope that it will be useful,
12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
14  * GNU General Public License for more details.
15  *
16  * You should have received a copy of the GNU General Public License
17  * along with this program; if not, write to the Free Software
18  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
19  */
20
21 #include <stdlib.h>
22 #include <unistd.h>
23 #include <stdio.h>
24
25 #if TIME_WITH_SYS_TIME
26 # include <sys/time.h>
27 # include <time.h>
28 #else
29 # if HAVE_SYS_TIME_H
30 #  include <sys/time.h>
31 # else
32 #  include <time.h>
33 # endif
34 #endif
35
36 #include <ctype.h>
37 #include <string.h>
38 #include <errno.h>
39 #include <sys/types.h>
40 #include <sys/stat.h>
41 #include <libcitadel.h>
42 #include "citadel.h"
43 #include "server.h"
44 #include "citserver.h"
45 #include "support.h"
46 #include "config.h"
47 #include "ctdl_module.h"
48 #include "clientsocket.h"
49 #include "msgbase.h"
50 #include "internet_addressing.h"
51 #include "database.h"
52 #include "citadel_dirs.h"
53 #include "event_client.h"
54
55
56 #define POP3C_OK (strncasecmp(ChrPtr(RecvMsg->IO.IOBuf), "+OK", 3) == 0)
57 int Pop3ClientID = 0;
58 int POP3ClientDebugEnabled = 0;
59
60 #define N ((pop3aggr*)IO->Data)->n
61
62 #define DBGLOG(LEVEL) if ((LEVEL != LOG_DEBUG) || (POP3ClientDebugEnabled != 0))
63
64 #define EVP3C_syslog(LEVEL, FORMAT, ...)                                \
65         DBGLOG(LEVEL) syslog(LEVEL,                                     \
66                              "IO[%ld]CC[%d][%ld]POP3: " FORMAT,         \
67                              IO->ID, CCID, N, __VA_ARGS__)
68
69 #define EVP3CM_syslog(LEVEL, FORMAT)                                    \
70         DBGLOG(LEVEL) syslog(LEVEL,                                     \
71                              "IO[%ld]CC[%d][%ld]POP3: " FORMAT,         \
72                              IO->ID, CCID, N)
73
74 #define EVP3CQ_syslog(LEVEL, FORMAT, ...)                               \
75         DBGLOG(LEVEL) syslog(LEVEL,                                     \
76                              "P3Q:" FORMAT,                             \
77                              __VA_ARGS__)
78
79 #define EVP3CQM_syslog(LEVEL, FORMAT)                                   \
80         DBGLOG(LEVEL) syslog(LEVEL,                                     \
81                              "P3Q" FORMAT                               \
82                 )
83
84 #define EVP3CCS_syslog(LEVEL, FORMAT, ...)                              \
85         DBGLOG(LEVEL) syslog(LEVEL, "IO[%ld][%ld]POP3: " FORMAT,        \
86                              IO->ID, N, __VA_ARGS__)
87
88 #define EVP3CCSM_syslog(LEVEL, FORMAT)                                  \
89         DBGLOG(LEVEL) syslog(LEVEL, "IO[%ld][%ld]POP3: " FORMAT,        \
90                              IO->ID, N)
91
92 #define POP3C_DBG_SEND()                                                \
93         EVP3C_syslog(LOG_DEBUG,                                         \
94                      "IO[%ld]CC[%d][%ld]POP3: > %s\n",                  \
95                      IO->ID, CCID, N,                                   \
96                      ChrPtr(RecvMsg->IO.SendBuf.Buf))
97
98 #define POP3C_DBG_READ()                                                \
99         EVP3C_syslog(LOG_DEBUG,                                         \
100                      "IO[%ld]CC[%d][%ld]POP3: < %s\n",                  \
101                      IO->ID, CCID, N,                                   \
102                      ChrPtr(RecvMsg->IO.IOBuf))
103
104
105 struct CitContext pop3_client_CC;
106
107 pthread_mutex_t POP3QueueMutex; /* locks the access to the following vars: */
108 HashList *POP3QueueRooms = NULL;
109 HashList *POP3FetchUrls = NULL;
110
111 typedef struct pop3aggr pop3aggr;
112 typedef eNextState(*Pop3ClientHandler)(pop3aggr* RecvMsg);
113
114 eNextState POP3_C_Shutdown(AsyncIO *IO);
115 eNextState POP3_C_Timeout(AsyncIO *IO);
116 eNextState POP3_C_ConnFail(AsyncIO *IO);
117 eNextState POP3_C_DNSFail(AsyncIO *IO);
118 eNextState POP3_C_DispatchReadDone(AsyncIO *IO);
119 eNextState POP3_C_DispatchWriteDone(AsyncIO *IO);
120 eNextState POP3_C_Terminate(AsyncIO *IO);
121 eReadState POP3_C_ReadServerStatus(AsyncIO *IO);
122 eNextState POP3_C_ReAttachToFetchMessages(AsyncIO *IO);
123
124 typedef struct __pop3_room_counter {
125         int count;
126         long QRnumber;
127 }pop3_room_counter;
128
129 typedef enum ePOP3_C_States {
130         ReadGreeting,
131         GetUserState,
132         GetPassState,
133         GetListCommandState,
134         GetListOneLine,
135         GetOneMessageIDState,
136         ReadMessageBodyFollowing,
137         ReadMessageBody,
138         GetDeleteState,
139         ReadQuitState,
140         POP3C_MaxRead
141 }ePOP3_C_States;
142
143
144 typedef struct _FetchItem {
145         long MSGID;
146         long MSGSize;
147         StrBuf *MsgUIDL;
148         StrBuf *MsgUID;
149         int NeedFetch;
150         struct CtdlMessage *Msg;
151 } FetchItem;
152
153 void HfreeFetchItem(void *vItem)
154 {
155         FetchItem *Item = (FetchItem*) vItem;
156         FreeStrBuf(&Item->MsgUIDL);
157         FreeStrBuf(&Item->MsgUID);
158         free(Item);
159 }
160
161 struct pop3aggr {
162         AsyncIO  IO;
163
164         long n;
165         double IOStart;
166         long count;
167         long RefCount;
168         DNSQueryParts HostLookup;
169
170         long             QRnumber;
171         HashList        *OtherQRnumbers;
172
173         StrBuf          *Url;
174         StrBuf *pop3user;
175         StrBuf *pop3pass;
176         StrBuf *Host;
177         StrBuf *RoomName; // TODO: fill me
178         int keep;
179         time_t interval;
180         ePOP3_C_States State;
181         HashList *MsgNumbers;
182         HashPos *Pos;
183         FetchItem *CurrMsg;
184 };
185
186 void DeletePOP3Aggregator(void *vptr)
187 {
188         pop3aggr *ptr = vptr;
189         DeleteHashPos(&ptr->Pos);
190         DeleteHash(&ptr->MsgNumbers);
191 //      FreeStrBuf(&ptr->rooms);
192         FreeStrBuf(&ptr->pop3user);
193         FreeStrBuf(&ptr->pop3pass);
194         FreeStrBuf(&ptr->Host);
195         FreeStrBuf(&ptr->RoomName);
196         FreeURL(&ptr->IO.ConnectMe);
197         FreeStrBuf(&ptr->Url);
198         FreeStrBuf(&ptr->IO.IOBuf);
199         FreeStrBuf(&ptr->IO.SendBuf.Buf);
200         FreeStrBuf(&ptr->IO.RecvBuf.Buf);
201         DeleteAsyncMsg(&ptr->IO.ReadMsg);
202         if (((struct CitContext*)ptr->IO.CitContext)) {
203                 ((struct CitContext*)ptr->IO.CitContext)->state = CON_IDLE;
204                 ((struct CitContext*)ptr->IO.CitContext)->kill_me = 1;
205         }
206         FreeAsyncIOContents(&ptr->IO);
207         free(ptr);
208 }
209
210 eNextState FinalizePOP3AggrRun(AsyncIO *IO)
211 {
212         HashPos  *It;
213         pop3aggr *cpptr = (pop3aggr *)IO->Data;
214
215         EVP3C_syslog(LOG_INFO,
216                      "%s@%s: fetched %ld new of %d messages in %fs. bye.",
217                      ChrPtr(cpptr->pop3user),
218                      ChrPtr(cpptr->Host),
219                      cpptr->count,
220                      GetCount(cpptr->MsgNumbers), 
221                      IO->Now - cpptr->IOStart 
222                 );
223
224         It = GetNewHashPos(POP3FetchUrls, 0);
225         pthread_mutex_lock(&POP3QueueMutex);
226         {
227                 if (GetHashPosFromKey(POP3FetchUrls, SKEY(cpptr->Url), It))
228                         DeleteEntryFromHash(POP3FetchUrls, It);
229         }
230         pthread_mutex_unlock(&POP3QueueMutex);
231         DeleteHashPos(&It);
232         return eAbort;
233 }
234
235 eNextState FailAggregationRun(AsyncIO *IO)
236 {
237         return eAbort;
238 }
239
240 eNextState POP3C_ReadGreeting(pop3aggr *RecvMsg)
241 {
242         AsyncIO *IO = &RecvMsg->IO;
243         POP3C_DBG_READ();
244         /* Read the server greeting */
245         if (!POP3C_OK) return eTerminateConnection;
246         else return eSendReply;
247 }
248
249 eNextState POP3C_SendUser(pop3aggr *RecvMsg)
250 {
251         AsyncIO *IO = &RecvMsg->IO;
252         /* Identify ourselves.  NOTE: we have to append a CR to each command.
253          *  The LF will automatically be appended by sock_puts().  Believe it
254          * or not, leaving out the CR will cause problems if the server happens
255          * to be Exchange, which is so b0rken it actually barfs on
256          * LF-terminated newlines.
257          */
258         StrBufPrintf(RecvMsg->IO.SendBuf.Buf,
259                      "USER %s\r\n", ChrPtr(RecvMsg->pop3user));
260         POP3C_DBG_SEND();
261         return eReadMessage;
262 }
263
264 eNextState POP3C_GetUserState(pop3aggr *RecvMsg)
265 {
266         AsyncIO *IO = &RecvMsg->IO;
267         POP3C_DBG_READ();
268         if (!POP3C_OK) return eTerminateConnection;
269         else return eSendReply;
270 }
271
272 eNextState POP3C_SendPassword(pop3aggr *RecvMsg)
273 {
274         AsyncIO *IO = &RecvMsg->IO;
275         /* Password */
276         StrBufPrintf(RecvMsg->IO.SendBuf.Buf,
277                      "PASS %s\r\n", ChrPtr(RecvMsg->pop3pass));
278         EVP3CM_syslog(LOG_DEBUG, "<PASS <password>\n");
279 //      POP3C_DBG_SEND(); No, we won't write the passvoid to syslog...
280         return eReadMessage;
281 }
282
283 eNextState POP3C_GetPassState(pop3aggr *RecvMsg)
284 {
285         AsyncIO *IO = &RecvMsg->IO;
286         POP3C_DBG_READ();
287         if (!POP3C_OK) return eTerminateConnection;
288         else return eSendReply;
289 }
290
291 eNextState POP3C_SendListCommand(pop3aggr *RecvMsg)
292 {
293         AsyncIO *IO = &RecvMsg->IO;
294         /* Get the list of messages */
295         StrBufPlain(RecvMsg->IO.SendBuf.Buf, HKEY("LIST\r\n"));
296         POP3C_DBG_SEND();
297         return eReadMessage;
298 }
299
300 eNextState POP3C_GetListCommandState(pop3aggr *RecvMsg)
301 {
302         AsyncIO *IO = &RecvMsg->IO;
303         POP3C_DBG_READ();
304         if (!POP3C_OK) return eTerminateConnection;
305         RecvMsg->MsgNumbers = NewHash(1, NULL);
306         RecvMsg->State++;
307         return eReadMore;
308 }
309
310
311 eNextState POP3C_GetListOneLine(pop3aggr *RecvMsg)
312 {
313         AsyncIO *IO = &RecvMsg->IO;
314 #if 0
315         int rc;
316 #endif
317         const char *pch;
318         FetchItem *OneMsg = NULL;
319         POP3C_DBG_READ();
320
321         if ((StrLength(RecvMsg->IO.IOBuf) == 1) &&
322             (ChrPtr(RecvMsg->IO.IOBuf)[0] == '.'))
323         {
324                 if (GetCount(RecvMsg->MsgNumbers) == 0)
325                 {
326                         ////    RecvMsg->Sate = ReadQuitState;
327                 }
328                 else
329                 {
330                         RecvMsg->Pos = GetNewHashPos(RecvMsg->MsgNumbers, 0);
331                 }
332                 return eSendReply;
333
334         }
335
336         /*
337          * work around buggy pop3 servers which send
338          * empty lines in their listings.
339         */
340         if ((StrLength(RecvMsg->IO.IOBuf) == 0) ||
341             !isdigit(ChrPtr(RecvMsg->IO.IOBuf)[0]))
342         {
343                 return eReadMore;
344         }
345
346         OneMsg = (FetchItem*) malloc(sizeof(FetchItem));
347         memset(OneMsg, 0, sizeof(FetchItem));
348         OneMsg->MSGID = atol(ChrPtr(RecvMsg->IO.IOBuf));
349
350         pch = strchr(ChrPtr(RecvMsg->IO.IOBuf), ' ');
351         if (pch != NULL)
352         {
353                 OneMsg->MSGSize = atol(pch + 1);
354         }
355 #if 0
356         rc = TestValidateHash(RecvMsg->MsgNumbers);
357         if (rc != 0)
358                 EVP3CCS_syslog(LOG_DEBUG, "Hash Invalid: %d\n", rc);
359 #endif
360
361         Put(RecvMsg->MsgNumbers, LKEY(OneMsg->MSGID), OneMsg, HfreeFetchItem);
362 #if 0
363         rc = TestValidateHash(RecvMsg->MsgNumbers);
364         if (rc != 0)
365                 EVP3CCS_syslog(LOG_DEBUG, "Hash Invalid: %d\n", rc);
366 #endif
367         //RecvMsg->State --; /* read next Line */
368         return eReadMore;
369 }
370
371 eNextState POP3_FetchNetworkUsetableEntry(AsyncIO *IO)
372 {
373         long HKLen;
374         const char *HKey;
375         void *vData;
376         struct cdbdata *cdbut;
377         pop3aggr *RecvMsg = (pop3aggr *) IO->Data;
378
379         if((RecvMsg->Pos != NULL) &&
380            GetNextHashPos(RecvMsg->MsgNumbers,
381                           RecvMsg->Pos,
382                           &HKLen,
383                           &HKey,
384                           &vData))
385         {
386                 struct UseTable ut;
387                 if (server_shutting_down)
388                         return eAbort;
389
390                 RecvMsg->CurrMsg = (FetchItem*) vData;
391                 EVP3CCS_syslog(LOG_DEBUG,
392                                "CHECKING: whether %s has already been seen: ",
393                                ChrPtr(RecvMsg->CurrMsg->MsgUID));
394
395                 /* Find out if we've already seen this item */
396                 safestrncpy(ut.ut_msgid,
397                             ChrPtr(RecvMsg->CurrMsg->MsgUID),
398                             sizeof(ut.ut_msgid));
399                 ut.ut_timestamp = time(NULL);/// TODO: libev timestamp!
400
401                 cdbut = cdb_fetch(CDB_USETABLE, SKEY(RecvMsg->CurrMsg->MsgUID));
402                 if (cdbut != NULL) {
403                         /* Item has already been seen */
404                         EVP3CCSM_syslog(LOG_DEBUG, "YES\n");
405                         cdb_free(cdbut);
406
407                         /* rewrite the record anyway, to update the timestamp */
408                         cdb_store(CDB_USETABLE,
409                                   SKEY(RecvMsg->CurrMsg->MsgUID),
410                                   &ut, sizeof(struct UseTable) );
411                         RecvMsg->CurrMsg->NeedFetch = 0; ////TODO0;
412                 }
413                 else
414                 {
415                         EVP3CCSM_syslog(LOG_DEBUG, "NO\n");
416                         RecvMsg->CurrMsg->NeedFetch = 1;
417                 }
418                 return NextDBOperation(&RecvMsg->IO,
419                                        POP3_FetchNetworkUsetableEntry);
420         }
421         else
422         {
423                 /* ok, now we know them all,
424                  * continue with reading the actual messages. */
425                 DeleteHashPos(&RecvMsg->Pos);
426                 StopDBWatchers(IO);
427                 return QueueEventContext(IO, POP3_C_ReAttachToFetchMessages);
428         }
429 }
430
431 eNextState POP3C_GetOneMessagID(pop3aggr *RecvMsg)
432 {
433         AsyncIO *IO = &RecvMsg->IO;
434         long HKLen;
435         const char *HKey;
436         void *vData;
437
438 #if 0
439         int rc;
440         rc = TestValidateHash(RecvMsg->MsgNumbers);
441         if (rc != 0)
442                 EVP3CCS_syslog(LOG_DEBUG, "Hash Invalid: %d\n", rc);
443 #endif
444         if((RecvMsg->Pos != NULL) &&
445            GetNextHashPos(RecvMsg->MsgNumbers,
446                           RecvMsg->Pos,
447                           &HKLen, &HKey,
448                           &vData))
449         {
450                 RecvMsg->CurrMsg = (FetchItem*) vData;
451                 /* Find out the UIDL of the message,
452                  * to determine whether we've already downloaded it */
453                 StrBufPrintf(RecvMsg->IO.SendBuf.Buf,
454                              "UIDL %ld\r\n", RecvMsg->CurrMsg->MSGID);
455                 POP3C_DBG_SEND();
456         }
457         else
458         {
459                 RecvMsg->State++;
460                 DeleteHashPos(&RecvMsg->Pos);
461                 /// done receiving uidls.. start looking them up now.
462                 RecvMsg->Pos = GetNewHashPos(RecvMsg->MsgNumbers, 0);
463                 return QueueDBOperation(&RecvMsg->IO,
464                                         POP3_FetchNetworkUsetableEntry);
465         }
466         return eReadMore; /* TODO */
467 }
468
469 eNextState POP3C_GetOneMessageIDState(pop3aggr *RecvMsg)
470 {
471         AsyncIO *IO = &RecvMsg->IO;
472 #if 0
473         int rc;
474         rc = TestValidateHash(RecvMsg->MsgNumbers);
475         if (rc != 0)
476                 EVP3CCS_syslog(LOG_DEBUG, "Hash Invalid: %d\n", rc);
477 #endif
478
479         POP3C_DBG_READ();
480         if (!POP3C_OK) return eTerminateConnection;
481         RecvMsg->CurrMsg->MsgUIDL =
482                 NewStrBufPlain(NULL, StrLength(RecvMsg->IO.IOBuf));
483         RecvMsg->CurrMsg->MsgUID =
484                 NewStrBufPlain(NULL, StrLength(RecvMsg->IO.IOBuf) * 2);
485
486         StrBufExtract_token(RecvMsg->CurrMsg->MsgUIDL,
487                             RecvMsg->IO.IOBuf, 2, ' ');
488
489         StrBufPrintf(RecvMsg->CurrMsg->MsgUID,
490                      "pop3/%s/%s:%s@%s",
491                      ChrPtr(RecvMsg->RoomName),
492                      ChrPtr(RecvMsg->CurrMsg->MsgUIDL),
493                      RecvMsg->IO.ConnectMe->User,
494                      RecvMsg->IO.ConnectMe->Host);
495         RecvMsg->State --;
496         return eSendReply;
497 }
498
499
500 eNextState POP3C_SendGetOneMsg(pop3aggr *RecvMsg)
501 {
502         AsyncIO *IO = &RecvMsg->IO;
503         long HKLen;
504         const char *HKey;
505         void *vData;
506
507         RecvMsg->CurrMsg = NULL;
508         while ((RecvMsg->Pos != NULL) && 
509                GetNextHashPos(RecvMsg->MsgNumbers,
510                               RecvMsg->Pos,
511                               &HKLen, &HKey,
512                               &vData) &&
513                (RecvMsg->CurrMsg = (FetchItem*) vData,
514                 RecvMsg->CurrMsg->NeedFetch == 0))
515         {}
516
517         if ((RecvMsg->CurrMsg != NULL ) && (RecvMsg->CurrMsg->NeedFetch == 1))
518         {
519                 /* Message has not been seen.
520                  * Tell the server to fetch the message... */
521                 StrBufPrintf(RecvMsg->IO.SendBuf.Buf,
522                              "RETR %ld\r\n", RecvMsg->CurrMsg->MSGID);
523                 POP3C_DBG_SEND();
524                 return eReadMessage;
525         }
526         else {
527                 RecvMsg->State = ReadQuitState;
528                 return POP3_C_DispatchWriteDone(&RecvMsg->IO);
529         }
530 }
531
532
533 eNextState POP3C_ReadMessageBodyFollowing(pop3aggr *RecvMsg)
534 {
535         AsyncIO *IO = &RecvMsg->IO;
536         POP3C_DBG_READ();
537         if (!POP3C_OK) return eTerminateConnection;
538         RecvMsg->IO.ReadMsg = NewAsyncMsg(HKEY("."),
539                                           RecvMsg->CurrMsg->MSGSize,
540                                           config.c_maxmsglen,
541                                           NULL, -1,
542                                           1);
543
544         return eReadPayload;
545 }
546
547
548 eNextState POP3C_StoreMsgRead(AsyncIO *IO)
549 {
550         pop3aggr *RecvMsg = (pop3aggr *) IO->Data;
551         struct UseTable ut;
552
553         EVP3CCS_syslog(LOG_DEBUG,
554                        "MARKING: %s as seen: ",
555                        ChrPtr(RecvMsg->CurrMsg->MsgUID));
556
557         safestrncpy(ut.ut_msgid,
558                     ChrPtr(RecvMsg->CurrMsg->MsgUID),
559                     sizeof(ut.ut_msgid));
560         ut.ut_timestamp = time(NULL); /* TODO: use libev time */
561         cdb_store(CDB_USETABLE,
562                   ChrPtr(RecvMsg->CurrMsg->MsgUID),
563                   StrLength(RecvMsg->CurrMsg->MsgUID),
564                   &ut,
565                   sizeof(struct UseTable) );
566         StopDBWatchers(IO);
567         return QueueEventContext(&RecvMsg->IO, POP3_C_ReAttachToFetchMessages);
568 }
569 eNextState POP3C_SaveMsg(AsyncIO *IO)
570 {
571         long msgnum;
572         pop3aggr *RecvMsg = (pop3aggr *) IO->Data;
573
574         /* Do Something With It (tm) */
575         msgnum = CtdlSubmitMsg(RecvMsg->CurrMsg->Msg,
576                                NULL,
577                                ChrPtr(RecvMsg->RoomName),
578                                0);
579         if (msgnum > 0L)
580         {
581                 /* Message has been committed to the store
582                  * write the uidl to the use table
583                  * so we don't fetch this message again
584                  */
585         }
586         CtdlFreeMessage(RecvMsg->CurrMsg->Msg);
587
588         RecvMsg->count ++;
589         return NextDBOperation(&RecvMsg->IO, POP3C_StoreMsgRead);
590 }
591
592 eNextState POP3C_ReadMessageBody(pop3aggr *RecvMsg)
593 {
594         AsyncIO *IO = &RecvMsg->IO;
595         EVP3CM_syslog(LOG_DEBUG, "Converting message...");
596         RecvMsg->CurrMsg->Msg =
597                 convert_internet_message_buf(&RecvMsg->IO.ReadMsg->MsgBuf);
598         StopClientWatchers(IO, 0);
599         return QueueDBOperation(&RecvMsg->IO, POP3C_SaveMsg);
600 }
601
602 eNextState POP3C_SendDelete(pop3aggr *RecvMsg)
603 {
604         AsyncIO *IO = &RecvMsg->IO;
605         if (!RecvMsg->keep) {
606                 StrBufPrintf(RecvMsg->IO.SendBuf.Buf,
607                              "DELE %ld\r\n", RecvMsg->CurrMsg->MSGID);
608                 POP3C_DBG_SEND();
609                 return eReadMessage;
610         }
611         else {
612                 RecvMsg->State = ReadMessageBodyFollowing;
613                 return POP3_C_DispatchWriteDone(&RecvMsg->IO);
614         }
615 }
616 eNextState POP3C_ReadDeleteState(pop3aggr *RecvMsg)
617 {
618         AsyncIO *IO = &RecvMsg->IO;
619         POP3C_DBG_READ();
620         RecvMsg->State = GetOneMessageIDState;
621         return eReadMessage;
622 }
623
624 eNextState POP3C_SendQuit(pop3aggr *RecvMsg)
625 {
626         AsyncIO *IO = &RecvMsg->IO;
627         /* Log out */
628         StrBufPlain(RecvMsg->IO.SendBuf.Buf,
629                     HKEY("QUIT\r\n3)"));
630         POP3C_DBG_SEND();
631         return eReadMessage;
632 }
633
634
635 eNextState POP3C_ReadQuitState(pop3aggr *RecvMsg)
636 {
637         AsyncIO *IO = &RecvMsg->IO;
638         POP3C_DBG_READ();
639         return eTerminateConnection;
640 }
641
642 const long POP3_C_ConnTimeout = 1000;
643 const long DefaultPOP3Port = 110;
644
645 Pop3ClientHandler POP3C_ReadHandlers[] = {
646         POP3C_ReadGreeting,
647         POP3C_GetUserState,
648         POP3C_GetPassState,
649         POP3C_GetListCommandState,
650         POP3C_GetListOneLine,
651         POP3C_GetOneMessageIDState,
652         POP3C_ReadMessageBodyFollowing,
653         POP3C_ReadMessageBody,
654         POP3C_ReadDeleteState,
655         POP3C_ReadQuitState,
656 };
657
658 const long POP3_C_SendTimeouts[POP3C_MaxRead] = {
659         100,
660         100,
661         100,
662         100,
663         100,
664         100,
665         100,
666         100
667 };
668 const ConstStr POP3C_ReadErrors[POP3C_MaxRead] = {
669         {HKEY("Connection broken during ")},
670         {HKEY("Connection broken during ")},
671         {HKEY("Connection broken during ")},
672         {HKEY("Connection broken during ")},
673         {HKEY("Connection broken during ")},
674         {HKEY("Connection broken during ")},
675         {HKEY("Connection broken during ")},
676         {HKEY("Connection broken during ")}
677 };
678
679 Pop3ClientHandler POP3C_SendHandlers[] = {
680         NULL, /* we don't send a greeting */
681         POP3C_SendUser,
682         POP3C_SendPassword,
683         POP3C_SendListCommand,
684         NULL,
685         POP3C_GetOneMessagID,
686         POP3C_SendGetOneMsg,
687         NULL,
688         POP3C_SendDelete,
689         POP3C_SendQuit
690 };
691
692 const long POP3_C_ReadTimeouts[] = {
693         100,
694         100,
695         100,
696         100,
697         100,
698         100,
699         100,
700         100,
701         100,
702         100
703 };
704 /*****************************************************************************/
705 /*                     POP3 CLIENT DISPATCHER                                */
706 /*****************************************************************************/
707
708 void POP3SetTimeout(eNextState NextTCPState, pop3aggr *pMsg)
709 {
710         AsyncIO *IO = &pMsg->IO;
711         double Timeout = 0.0;
712
713         EVP3C_syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
714
715         switch (NextTCPState) {
716         case eSendFile:
717         case eSendReply:
718         case eSendMore:
719                 Timeout = POP3_C_SendTimeouts[pMsg->State];
720 /*
721   if (pMsg->State == eDATABody) {
722   / * if we're sending a huge message, we need more time. * /
723   Timeout += StrLength(pMsg->msgtext) / 1024;
724   }
725 */
726                 break;
727         case eReadFile:
728         case eReadMessage:
729                 Timeout = POP3_C_ReadTimeouts[pMsg->State];
730 /*
731   if (pMsg->State == eDATATerminateBody) {
732   / *
733   * some mailservers take a nap before accepting the message
734   * content inspection and such.
735   * /
736   Timeout += StrLength(pMsg->msgtext) / 1024;
737   }
738 */
739                 break;
740         case eReadPayload:
741                 Timeout = 100000;
742                 /* TODO!!! */
743                 break;
744         case eSendDNSQuery:
745         case eReadDNSReply:
746         case eConnect:
747         case eTerminateConnection:
748         case eDBQuery:
749         case eAbort:
750         case eReadMore://// TODO
751                 return;
752         }
753         SetNextTimeout(&pMsg->IO, Timeout);
754 }
755 eNextState POP3_C_DispatchReadDone(AsyncIO *IO)
756 {
757 /*      EVP3CCS_syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__); to noisy anyways. */
758         pop3aggr *pMsg = IO->Data;
759         eNextState rc;
760
761         rc = POP3C_ReadHandlers[pMsg->State](pMsg);
762         if (rc != eReadMore)
763             pMsg->State++;
764         POP3SetTimeout(rc, pMsg);
765         return rc;
766 }
767 eNextState POP3_C_DispatchWriteDone(AsyncIO *IO)
768 {
769         pop3aggr *pMsg = IO->Data;
770         eNextState rc;
771
772 /*      EVP3CCS_syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__); to noisy anyways. */
773         rc = POP3C_SendHandlers[pMsg->State](pMsg);
774         POP3SetTimeout(rc, pMsg);
775         return rc;
776 }
777
778
779 /*****************************************************************************/
780 /*                     POP3 CLIENT ERROR CATCHERS                            */
781 /*****************************************************************************/
782 eNextState POP3_C_Terminate(AsyncIO *IO)
783 {
784 ///     pop3aggr *pMsg = (pop3aggr *)IO->Data;
785
786         EVP3CCS_syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
787         FinalizePOP3AggrRun(IO);
788         return eAbort;
789 }
790 eNextState POP3_C_TerminateDB(AsyncIO *IO)
791 {
792 ///     pop3aggr *pMsg = (pop3aggr *)IO->Data;
793
794         EVP3CCS_syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
795         FinalizePOP3AggrRun(IO);
796         return eAbort;
797 }
798 eNextState POP3_C_Timeout(AsyncIO *IO)
799 {
800         pop3aggr *pMsg = IO->Data;
801
802         EVP3CCS_syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
803         StrBufPlain(IO->ErrMsg, CKEY(POP3C_ReadErrors[pMsg->State]));
804         return FailAggregationRun(IO);
805 }
806 eNextState POP3_C_ConnFail(AsyncIO *IO)
807 {
808         pop3aggr *pMsg = (pop3aggr *)IO->Data;
809
810         EVP3CCS_syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
811         StrBufPlain(IO->ErrMsg, CKEY(POP3C_ReadErrors[pMsg->State]));
812         return FailAggregationRun(IO);
813 }
814 eNextState POP3_C_DNSFail(AsyncIO *IO)
815 {
816         pop3aggr *pMsg = (pop3aggr *)IO->Data;
817
818         EVP3CCS_syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
819         StrBufPlain(IO->ErrMsg, CKEY(POP3C_ReadErrors[pMsg->State]));
820         return FailAggregationRun(IO);
821 }
822 eNextState POP3_C_Shutdown(AsyncIO *IO)
823 {
824         EVP3CCS_syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
825 ////    pop3aggr *pMsg = IO->Data;
826
827 ////pMsg->MyQEntry->Status = 3;
828 ///StrBufPlain(pMsg->MyQEntry->StatusMessage, HKEY("server shutdown during message retrieval."));
829         FinalizePOP3AggrRun(IO);
830         return eAbort;
831 }
832
833
834 /**
835  * @brief lineread Handler; understands when to read more POP3 lines,
836  *   and when this is a one-lined reply.
837  */
838 eReadState POP3_C_ReadServerStatus(AsyncIO *IO)
839 {
840         eReadState Finished = eBufferNotEmpty;
841
842         switch (IO->NextState) {
843         case eSendDNSQuery:
844         case eReadDNSReply:
845         case eDBQuery:
846         case eConnect:
847         case eTerminateConnection:
848         case eAbort:
849                 Finished = eReadFail;
850                 break;
851         case eSendFile:
852         case eSendReply:
853         case eSendMore:
854         case eReadMore:
855         case eReadMessage:
856                 Finished = StrBufChunkSipLine(IO->IOBuf, &IO->RecvBuf);
857                 break;
858         case eReadFile:
859         case eReadPayload:
860                 Finished = CtdlReadMessageBodyAsync(IO);
861                 break;
862         }
863         return Finished;
864 }
865
866 /*****************************************************************************
867  * So we connect our Server IP here.                                         *
868  *****************************************************************************/
869 eNextState POP3_C_ReAttachToFetchMessages(AsyncIO *IO)
870 {
871         pop3aggr *cpptr = IO->Data;
872
873         EVP3CCS_syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
874 ////??? cpptr->State ++;
875         if (cpptr->Pos == NULL)
876                 cpptr->Pos = GetNewHashPos(cpptr->MsgNumbers, 0);
877
878         POP3_C_DispatchWriteDone(IO);
879         ReAttachIO(IO, cpptr, 0);
880         IO->NextState = eReadMessage;
881         return IO->NextState;
882 }
883
884 eNextState pop3_connect_ip(AsyncIO *IO)
885 {
886         pop3aggr *cpptr = IO->Data;
887
888         if (cpptr->IOStart == 0.0) /* whith or without DNS? */
889                 cpptr->IOStart = IO->Now;
890
891         EVP3CCS_syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
892
893         return EvConnectSock(IO,
894                              POP3_C_ConnTimeout,
895                              POP3_C_ReadTimeouts[0],
896                              1);
897 }
898
899 eNextState pop3_get_one_host_ip_done(AsyncIO *IO)
900 {
901         pop3aggr *cpptr = IO->Data;
902         struct hostent *hostent;
903
904         QueryCbDone(IO);
905
906         hostent = cpptr->HostLookup.VParsedDNSReply;
907         if ((cpptr->HostLookup.DNSStatus == ARES_SUCCESS) && 
908             (hostent != NULL) ) {
909                 memset(&cpptr->IO.ConnectMe->Addr, 0, sizeof(struct in6_addr));
910                 if (cpptr->IO.ConnectMe->IPv6) {
911                         memcpy(&cpptr->IO.ConnectMe->Addr.sin6_addr.s6_addr, 
912                                &hostent->h_addr_list[0],
913                                sizeof(struct in6_addr));
914
915                         cpptr->IO.ConnectMe->Addr.sin6_family =
916                                 hostent->h_addrtype;
917                         cpptr->IO.ConnectMe->Addr.sin6_port   =
918                                 htons(DefaultPOP3Port);
919                 }
920                 else {
921                         struct sockaddr_in *addr =
922                                 (struct sockaddr_in*)
923                                 &cpptr->IO.ConnectMe->Addr;
924
925                         memcpy(&addr->sin_addr.s_addr,
926                                hostent->h_addr_list[0],
927                                sizeof(uint32_t));
928
929                         addr->sin_family = hostent->h_addrtype;
930                         addr->sin_port   = htons(DefaultPOP3Port);
931                 }
932                 return pop3_connect_ip(IO);
933         }
934         else
935                 return eAbort;
936 }
937
938 eNextState pop3_get_one_host_ip(AsyncIO *IO)
939 {
940         pop3aggr *cpptr = IO->Data;
941
942         cpptr->IOStart = IO->Now;
943
944         EVP3CCS_syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
945
946         EVP3CCS_syslog(LOG_DEBUG, 
947                        "POP3 client[%ld]: looking up %s-Record %s : %d ...\n",
948                        cpptr->n,
949                        (cpptr->IO.ConnectMe->IPv6)? "aaaa": "a",
950                        cpptr->IO.ConnectMe->Host,
951                        cpptr->IO.ConnectMe->Port);
952
953         QueueQuery((cpptr->IO.ConnectMe->IPv6)? ns_t_aaaa : ns_t_a,
954                    cpptr->IO.ConnectMe->Host,
955                    &cpptr->IO,
956                    &cpptr->HostLookup,
957                    pop3_get_one_host_ip_done);
958         IO->NextState = eReadDNSReply;
959         return IO->NextState;
960 }
961
962
963
964 int pop3_do_fetching(pop3aggr *cpptr)
965 {
966         AsyncIO *IO = &cpptr->IO;
967
968         InitIOStruct(IO,
969                      cpptr,
970                      eReadMessage,
971                      POP3_C_ReadServerStatus,
972                      POP3_C_DNSFail,
973                      POP3_C_DispatchWriteDone,
974                      POP3_C_DispatchReadDone,
975                      POP3_C_Terminate,
976                      POP3_C_TerminateDB,
977                      POP3_C_ConnFail,
978                      POP3_C_Timeout,
979                      POP3_C_Shutdown);
980
981         safestrncpy(((CitContext *)cpptr->IO.CitContext)->cs_host,
982                     ChrPtr(cpptr->Url),
983                     sizeof(((CitContext *)cpptr->IO.CitContext)->cs_host));
984
985         if (cpptr->IO.ConnectMe->IsIP) {
986                 QueueEventContext(&cpptr->IO,
987                                   pop3_connect_ip);
988         }
989         else {
990                 QueueEventContext(&cpptr->IO,
991                                   pop3_get_one_host_ip);
992         }
993         return 1;
994 }
995
996 /*
997  * Scan a room's netconfig to determine whether it requires POP3 aggregation
998  */
999 void pop3client_scan_room(struct ctdlroom *qrbuf, void *data, OneRoomNetCfg *OneRNCFG)
1000 {
1001         const RoomNetCfgLine *pLine;
1002         void *vptr;
1003
1004         pthread_mutex_lock(&POP3QueueMutex);
1005         if (GetHash(POP3QueueRooms, LKEY(qrbuf->QRnumber), &vptr))
1006         {
1007                 pthread_mutex_unlock(&POP3QueueMutex);
1008                 EVP3CQ_syslog(LOG_DEBUG,
1009                               "pop3client: [%ld] %s already in progress.",
1010                               qrbuf->QRnumber,
1011                               qrbuf->QRname);
1012                 return;
1013         }
1014         pthread_mutex_unlock(&POP3QueueMutex);
1015
1016         if (server_shutting_down) return;
1017
1018         pLine = OneRNCFG->NetConfigs[pop3client];
1019
1020         while (pLine != NULL)
1021         {
1022                 pop3aggr *cptr;
1023
1024                 cptr = (pop3aggr *) malloc(sizeof(pop3aggr));
1025                 memset(cptr, 0, sizeof(pop3aggr));
1026                 ///TODO do we need this? cptr->roomlist_parts=1;
1027                 cptr->RoomName = NewStrBufPlain(qrbuf->QRname, -1);
1028                 cptr->pop3user = NewStrBufDup(pLine->Value[1]);
1029                 cptr->pop3pass = NewStrBufDup(pLine->Value[2]);
1030                 cptr->Url = NewStrBuf();
1031                 cptr->Host = NewStrBufDup(pLine->Value[0]);
1032
1033                 cptr->keep = atol(ChrPtr(pLine->Value[3]));
1034                 cptr->interval = atol(ChrPtr(pLine->Value[4]));
1035
1036                 StrBufAppendBufPlain(cptr->Url, HKEY("pop3://"), 0);
1037                 StrBufUrlescUPAppend(cptr->Url, cptr->pop3user, NULL);
1038                 StrBufAppendBufPlain(cptr->Url, HKEY(":"), 0);
1039                 StrBufUrlescUPAppend(cptr->Url, cptr->pop3pass, NULL);
1040                 StrBufAppendBufPlain(cptr->Url, HKEY("@"), 0);
1041                 StrBufAppendBuf(cptr->Url, cptr->Host, 0);
1042                 StrBufAppendBufPlain(cptr->Url, HKEY("/"), 0);
1043                 StrBufUrlescAppend(cptr->Url, cptr->RoomName, NULL);
1044
1045                 ParseURL(&cptr->IO.ConnectMe, cptr->Url, 110);
1046
1047
1048 #if 0
1049 /* todo: we need to reunite the url to be shure. */
1050
1051                 pthread_mutex_lock(&POP3ueueMutex);
1052                 GetHash(POP3FetchUrls, SKEY(ptr->Url), &vptr);
1053                 use_this_cptr = (pop3aggr *)vptr;
1054
1055                 if (use_this_rncptr != NULL)
1056                 {
1057                         /* mustn't attach to an active session */
1058                         if (use_this_cptr->RefCount > 0)
1059                         {
1060                                 DeletePOP3Cfg(cptr);
1061 ///                                             Count->count--;
1062                         }
1063                         else
1064                         {
1065                                 long *QRnumber;
1066                                 StrBufAppendBufPlain(
1067                                         use_this_cptr->rooms,
1068                                         qrbuf->QRname,
1069                                         -1, 0);
1070                                 if (use_this_cptr->roomlist_parts == 1)
1071                                 {
1072                                         use_this_cptr->OtherQRnumbers
1073                                                 = NewHash(1, lFlathash);
1074                                 }
1075                                 QRnumber = (long*)malloc(sizeof(long));
1076                                 *QRnumber = qrbuf->QRnumber;
1077                                 Put(use_this_cptr->OtherQRnumbers,
1078                                     LKEY(qrbuf->QRnumber),
1079                                     QRnumber,
1080                                     NULL);
1081
1082                                 use_this_cptr->roomlist_parts++;
1083                         }
1084                         pthread_mutex_unlock(&POP3QueueMutex);
1085                         continue;
1086                 }
1087                 pthread_mutex_unlock(&RSSQueueMutex);
1088 #endif
1089                 cptr->n = Pop3ClientID++;
1090                 pthread_mutex_lock(&POP3QueueMutex);
1091                 Put(POP3FetchUrls,
1092                     SKEY(cptr->Url),
1093                     cptr,
1094                     DeletePOP3Aggregator);
1095
1096                 pthread_mutex_unlock(&POP3QueueMutex);
1097                 pLine = pLine->next;
1098
1099         }
1100 }
1101
1102 static int doing_pop3client = 0;
1103
1104 void pop3client_scan(void) {
1105         static time_t last_run = 0L;
1106         time_t fastest_scan;
1107         HashPos *it;
1108         long len;
1109         const char *Key;
1110         void *vrptr;
1111         pop3aggr *cptr;
1112
1113         become_session(&pop3_client_CC);
1114
1115         if (config.c_pop3_fastest < config.c_pop3_fetch)
1116                 fastest_scan = config.c_pop3_fastest;
1117         else
1118                 fastest_scan = config.c_pop3_fetch;
1119
1120         /*
1121          * Run POP3 aggregation no more frequently than once every n seconds
1122          */
1123         if ( (time(NULL) - last_run) < fastest_scan ) {
1124                 return;
1125         }
1126
1127         /*
1128          * This is a simple concurrency check to make sure only one pop3client
1129          * run is done at a time.  We could do this with a mutex, but since we
1130          * don't really require extremely fine granularity here, we'll do it
1131          * with a static variable instead.
1132          */
1133         if (doing_pop3client) return;
1134         doing_pop3client = 1;
1135
1136         EVP3CQM_syslog(LOG_DEBUG, "pop3client started");
1137         CtdlForEachNetCfgRoom(pop3client_scan_room, NULL, pop3client);
1138
1139         pthread_mutex_lock(&POP3QueueMutex);
1140         it = GetNewHashPos(POP3FetchUrls, 0);
1141         while (!server_shutting_down &&
1142                GetNextHashPos(POP3FetchUrls, it, &len, &Key, &vrptr) &&
1143                (vrptr != NULL)) {
1144                 cptr = (pop3aggr *)vrptr;
1145                 if (cptr->RefCount == 0)
1146                         if (!pop3_do_fetching(cptr))
1147                                 DeletePOP3Aggregator(cptr);////TODO
1148
1149 /*
1150         if ((palist->interval && time(NULL) > (last_run + palist->interval))
1151                         || (time(NULL) > last_run + config.c_pop3_fetch))
1152                         pop3_do_fetching(palist->roomname, palist->pop3host,
1153                         palist->pop3user, palist->pop3pass, palist->keep);
1154                 pptr = palist;
1155                 palist = palist->next;
1156                 free(pptr);
1157 */
1158         }
1159         DeleteHashPos(&it);
1160         pthread_mutex_unlock(&POP3QueueMutex);
1161
1162         EVP3CQM_syslog(LOG_DEBUG, "pop3client ended");
1163         last_run = time(NULL);
1164         doing_pop3client = 0;
1165 }
1166
1167
1168 void pop3_cleanup(void)
1169 {
1170         /* citthread_mutex_destroy(&POP3QueueMutex); TODO */
1171         while (doing_pop3client != 0) ;
1172         DeleteHash(&POP3FetchUrls);
1173         DeleteHash(&POP3QueueRooms);
1174 }
1175
1176
1177
1178 void LogDebugEnablePOP3Client(const int n)
1179 {
1180         POP3ClientDebugEnabled = n;
1181 }
1182
1183 CTDL_MODULE_INIT(pop3client)
1184 {
1185         if (!threading)
1186         {
1187                 CtdlFillSystemContext(&pop3_client_CC, "POP3aggr");
1188                 CtdlREGISTERRoomCfgType(pop3client, ParseGeneric, 0, 5, SerializeGeneric, DeleteGenericCfgLine);
1189                 pthread_mutex_init(&POP3QueueMutex, NULL);
1190                 POP3QueueRooms = NewHash(1, lFlathash);
1191                 POP3FetchUrls = NewHash(1, NULL);
1192                 CtdlRegisterSessionHook(pop3client_scan, EVT_TIMER, PRIO_AGGR + 50);
1193                 CtdlRegisterEVCleanupHook(pop3_cleanup);
1194                 CtdlRegisterDebugFlagHook(HKEY("pop3client"), LogDebugEnablePOP3Client, &POP3ClientDebugEnabled);
1195         }
1196
1197         /* return our module id for the log */
1198         return "pop3client";
1199 }