fix pop3 Aggregator
[citadel.git] / citadel / modules / pop3client / serv_pop3client.c
1 /*
2  * Consolidate mail from remote POP3 accounts.
3  *
4  * Copyright (c) 2007-2011 by the citadel.org team
5  *
6  * This program is open source software; you can redistribute it and/or
7  * modify it under the terms of the GNU General Public License as published
8  * by the Free Software Foundation; either version 3 of the License, or
9  * (at your option) any later version.
10  *
11  * This program is distributed in the hope that it will be useful,
12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
14  * GNU General Public License for more details.
15  *
16  * You should have received a copy of the GNU General Public License
17  * along with this program; if not, write to the Free Software
18  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
19  */
20
21 #include <stdlib.h>
22 #include <unistd.h>
23 #include <stdio.h>
24 #include <sysconfig.h>
25
26 #if TIME_WITH_SYS_TIME
27 # include <sys/time.h>
28 # include <time.h>
29 #else
30 # if HAVE_SYS_TIME_H
31 #  include <sys/time.h>
32 # else
33 #  include <time.h>
34 # endif
35 #endif
36
37 #include <ctype.h>
38 #include <string.h>
39 #include <errno.h>
40 #include <sys/types.h>
41 #include <sys/stat.h>
42 #include <libcitadel.h>
43 #include "citadel.h"
44 #include "server.h"
45 #include "citserver.h"
46 #include "support.h"
47 #include "config.h"
48 #include "ctdl_module.h"
49 #include "clientsocket.h"
50 #include "msgbase.h"
51 #include "internet_addressing.h"
52 #include "database.h"
53 #include "citadel_dirs.h"
54 #include "event_client.h"
55
56
57 #define POP3C_OK (strncasecmp(ChrPtr(RecvMsg->IO.IOBuf), "+OK", 3) == 0)
58 int Pop3ClientID = 0;
59 int POP3ClientDebugEnabled = 0;
60
61 #define N ((pop3aggr*)IO->Data)->n
62
63 #define DBGLOG(LEVEL) if ((LEVEL != LOG_DEBUG) || (POP3ClientDebugEnabled != 0))
64
65 #define EVP3C_syslog(LEVEL, FORMAT, ...)                                \
66         DBGLOG(LEVEL) syslog(LEVEL,                                     \
67                              "%s[%ld]CC[%d][%ld]POP3: " FORMAT,         \
68                              IOSTR, IO->ID, CCID, N, __VA_ARGS__)
69
70 #define EVP3CM_syslog(LEVEL, FORMAT)                                    \
71         DBGLOG(LEVEL) syslog(LEVEL,                                     \
72                              "%s[%ld]CC[%d][%ld]POP3: " FORMAT,         \
73                              IOSTR, IO->ID, CCID, N)
74
75 #define EVP3CQ_syslog(LEVEL, FORMAT, ...)                               \
76         DBGLOG(LEVEL) syslog(LEVEL,                                     \
77                              "%s P3Q:" FORMAT,                          \
78                              IOSTR, __VA_ARGS__)
79
80 #define EVP3CQM_syslog(LEVEL, FORMAT)                                   \
81         DBGLOG(LEVEL) syslog(LEVEL,                                     \
82                              "%s P3Q" FORMAT,                           \
83                              IOSTR)
84
85 #define EVP3CCS_syslog(LEVEL, FORMAT, ...)                              \
86         DBGLOG(LEVEL) syslog(LEVEL, "%s[%ld][%ld]POP3: " FORMAT,        \
87                              IOSTR, IO->ID, N, __VA_ARGS__)
88
89 #define EVP3CCSM_syslog(LEVEL, FORMAT)                                  \
90         DBGLOG(LEVEL) syslog(LEVEL, "%s[%ld][%ld]POP3: " FORMAT,        \
91                              IOSTR, IO->ID, N)
92
93 #define POP3C_DBG_SEND()                                                \
94         EVP3C_syslog(LOG_DEBUG,                                         \
95                      "%s[%ld]CC[%d][%ld]POP3: > %s\n",                  \
96                      IOSTR, IO->ID, CCID, N,                            \
97                      ChrPtr(RecvMsg->IO.SendBuf.Buf))
98
99 #define POP3C_DBG_READ()                                                \
100         EVP3C_syslog(LOG_DEBUG,                                         \
101                      "%s[%ld]CC[%d][%ld]POP3: < %s\n",                  \
102                      IOSTR, IO->ID, CCID, N,                            \
103                      ChrPtr(RecvMsg->IO.IOBuf))
104
105
106 struct CitContext pop3_client_CC;
107
108 pthread_mutex_t POP3QueueMutex; /* locks the access to the following vars: */
109 HashList *POP3QueueRooms = NULL;
110 HashList *POP3FetchUrls = NULL;
111
112 typedef struct pop3aggr pop3aggr;
113 typedef eNextState(*Pop3ClientHandler)(pop3aggr* RecvMsg);
114
115 eNextState POP3_C_Shutdown(AsyncIO *IO);
116 eNextState POP3_C_Timeout(AsyncIO *IO);
117 eNextState POP3_C_ConnFail(AsyncIO *IO);
118 eNextState POP3_C_DNSFail(AsyncIO *IO);
119 eNextState POP3_C_DispatchReadDone(AsyncIO *IO);
120 eNextState POP3_C_DispatchWriteDone(AsyncIO *IO);
121 eNextState POP3_C_Terminate(AsyncIO *IO);
122 eReadState POP3_C_ReadServerStatus(AsyncIO *IO);
123 eNextState POP3_C_ReAttachToFetchMessages(AsyncIO *IO);
124
125 typedef struct __pop3_room_counter {
126         int count;
127         long QRnumber;
128 }pop3_room_counter;
129
130 typedef enum ePOP3_C_States {
131         ReadGreeting,
132         GetUserState,
133         GetPassState,
134         GetListCommandState,
135         GetListOneLine,
136         GetOneMessageIDState,
137         ReadMessageBodyFollowing,
138         ReadMessageBody,
139         GetDeleteState,
140         ReadQuitState,
141         POP3C_MaxRead
142 }ePOP3_C_States;
143
144
145 typedef struct _FetchItem {
146         long MSGID;
147         long MSGSize;
148         StrBuf *MsgUIDL;
149         StrBuf *MsgUID;
150         int NeedFetch;
151         struct CtdlMessage *Msg;
152 } FetchItem;
153
154 void HfreeFetchItem(void *vItem)
155 {
156         FetchItem *Item = (FetchItem*) vItem;
157         FreeStrBuf(&Item->MsgUIDL);
158         FreeStrBuf(&Item->MsgUID);
159         free(Item);
160 }
161
162
163
164 typedef enum _POP3State {
165         eCreated,
166         eGreeting,
167         eUser,
168         ePassword,
169         eListing,
170         eUseTable,
171         eGetMsgID,
172         eGetMsg,
173         eStoreMsg,
174         eDelete,
175         eQuit
176 } POP3State;
177
178 ConstStr POP3States[] = {
179         {HKEY("Aggregator created")},
180         {HKEY("Reading Greeting")},
181         {HKEY("Sending User")},
182         {HKEY("Sending Password")},
183         {HKEY("Listing")},
184         {HKEY("Fetching Usetable")},
185         {HKEY("Get MSG ID")},
186         {HKEY("Get Message")},
187         {HKEY("Store Msg")},
188         {HKEY("Delete Upstream")},
189         {HKEY("Quit")}
190 };
191
192 static void SetPOP3State(AsyncIO *IO, POP3State State)
193 {
194         CitContext* CCC = IO->CitContext;
195         if (CCC != NULL)
196                 memcpy(CCC->cs_clientname, POP3States[State].Key, POP3States[State].len + 1);
197 }
198
199
200 struct pop3aggr {
201         AsyncIO  IO;
202
203         long n;
204         double IOStart;
205         long count;
206         long RefCount;
207         DNSQueryParts HostLookup;
208
209         long             QRnumber;
210         HashList        *OtherQRnumbers;
211
212         StrBuf          *Url;
213         StrBuf *pop3user;
214         StrBuf *pop3pass;
215         StrBuf *Host;
216         StrBuf *RoomName; // TODO: fill me
217         int keep;
218         time_t interval;
219         ePOP3_C_States State;
220         HashList *MsgNumbers;
221         HashPos *Pos;
222         FetchItem *CurrMsg;
223 };
224
225 void DeletePOP3Aggregator(void *vptr)
226 {
227         pop3aggr *ptr = vptr;
228         DeleteHashPos(&ptr->Pos);
229         DeleteHash(&ptr->MsgNumbers);
230 //      FreeStrBuf(&ptr->rooms);
231         FreeStrBuf(&ptr->pop3user);
232         FreeStrBuf(&ptr->pop3pass);
233         FreeStrBuf(&ptr->Host);
234         FreeStrBuf(&ptr->RoomName);
235         FreeURL(&ptr->IO.ConnectMe);
236         FreeStrBuf(&ptr->Url);
237         FreeStrBuf(&ptr->IO.IOBuf);
238         FreeStrBuf(&ptr->IO.SendBuf.Buf);
239         FreeStrBuf(&ptr->IO.RecvBuf.Buf);
240         DeleteAsyncMsg(&ptr->IO.ReadMsg);
241         if (((struct CitContext*)ptr->IO.CitContext)) {
242                 ((struct CitContext*)ptr->IO.CitContext)->state = CON_IDLE;
243                 ((struct CitContext*)ptr->IO.CitContext)->kill_me = 1;
244         }
245         FreeAsyncIOContents(&ptr->IO);
246         free(ptr);
247 }
248
249 eNextState FinalizePOP3AggrRun(AsyncIO *IO)
250 {
251         HashPos  *It;
252         pop3aggr *cpptr = (pop3aggr *)IO->Data;
253
254         EVP3C_syslog(LOG_INFO,
255                      "%s@%s: fetched %ld new of %d messages in %fs. bye.",
256                      ChrPtr(cpptr->pop3user),
257                      ChrPtr(cpptr->Host),
258                      cpptr->count,
259                      GetCount(cpptr->MsgNumbers), 
260                      IO->Now - cpptr->IOStart 
261                 );
262
263         It = GetNewHashPos(POP3FetchUrls, 0);
264         pthread_mutex_lock(&POP3QueueMutex);
265         {
266                 if (GetHashPosFromKey(POP3FetchUrls, SKEY(cpptr->Url), It))
267                         DeleteEntryFromHash(POP3FetchUrls, It);
268         }
269         pthread_mutex_unlock(&POP3QueueMutex);
270         DeleteHashPos(&It);
271         return eAbort;
272 }
273
274 eNextState FailAggregationRun(AsyncIO *IO)
275 {
276         return eAbort;
277 }
278
279 eNextState POP3C_ReadGreeting(pop3aggr *RecvMsg)
280 {
281         AsyncIO *IO = &RecvMsg->IO;
282         SetPOP3State(IO, eGreeting);
283         POP3C_DBG_READ();
284         /* Read the server greeting */
285         if (!POP3C_OK) return eTerminateConnection;
286         else return eSendReply;
287 }
288
289 eNextState POP3C_SendUser(pop3aggr *RecvMsg)
290 {
291         AsyncIO *IO = &RecvMsg->IO;
292         SetPOP3State(IO, eUser);
293         /* Identify ourselves.  NOTE: we have to append a CR to each command.
294          *  The LF will automatically be appended by sock_puts().  Believe it
295          * or not, leaving out the CR will cause problems if the server happens
296          * to be Exchange, which is so b0rken it actually barfs on
297          * LF-terminated newlines.
298          */
299         StrBufPrintf(RecvMsg->IO.SendBuf.Buf,
300                      "USER %s\r\n", ChrPtr(RecvMsg->pop3user));
301         POP3C_DBG_SEND();
302         return eReadMessage;
303 }
304
305 eNextState POP3C_GetUserState(pop3aggr *RecvMsg)
306 {
307         AsyncIO *IO = &RecvMsg->IO;
308         POP3C_DBG_READ();
309         if (!POP3C_OK) return eTerminateConnection;
310         else return eSendReply;
311 }
312
313 eNextState POP3C_SendPassword(pop3aggr *RecvMsg)
314 {
315         AsyncIO *IO = &RecvMsg->IO;
316         SetPOP3State(IO, ePassword);
317         /* Password */
318         StrBufPrintf(RecvMsg->IO.SendBuf.Buf,
319                      "PASS %s\r\n", ChrPtr(RecvMsg->pop3pass));
320         EVP3CM_syslog(LOG_DEBUG, "<PASS <password>\n");
321 //      POP3C_DBG_SEND(); No, we won't write the passvoid to syslog...
322         return eReadMessage;
323 }
324
325 eNextState POP3C_GetPassState(pop3aggr *RecvMsg)
326 {
327         AsyncIO *IO = &RecvMsg->IO;
328         POP3C_DBG_READ();
329         if (!POP3C_OK) return eTerminateConnection;
330         else return eSendReply;
331 }
332
333 eNextState POP3C_SendListCommand(pop3aggr *RecvMsg)
334 {
335         AsyncIO *IO = &RecvMsg->IO;
336         SetPOP3State(IO, eListing);
337
338         /* Get the list of messages */
339         StrBufPlain(RecvMsg->IO.SendBuf.Buf, HKEY("LIST\r\n"));
340         POP3C_DBG_SEND();
341         return eReadMessage;
342 }
343
344 eNextState POP3C_GetListCommandState(pop3aggr *RecvMsg)
345 {
346         AsyncIO *IO = &RecvMsg->IO;
347         POP3C_DBG_READ();
348         if (!POP3C_OK) return eTerminateConnection;
349         RecvMsg->MsgNumbers = NewHash(1, NULL);
350         RecvMsg->State++;
351         return eReadMore;
352 }
353
354
355 eNextState POP3C_GetListOneLine(pop3aggr *RecvMsg)
356 {
357         AsyncIO *IO = &RecvMsg->IO;
358 #if 0
359         int rc;
360 #endif
361         const char *pch;
362         FetchItem *OneMsg = NULL;
363         POP3C_DBG_READ();
364
365         if ((StrLength(RecvMsg->IO.IOBuf) == 1) &&
366             (ChrPtr(RecvMsg->IO.IOBuf)[0] == '.'))
367         {
368                 if (GetCount(RecvMsg->MsgNumbers) == 0)
369                 {
370                         ////    RecvMsg->Sate = ReadQuitState;
371                 }
372                 else
373                 {
374                         RecvMsg->Pos = GetNewHashPos(RecvMsg->MsgNumbers, 0);
375                 }
376                 return eSendReply;
377
378         }
379
380         /*
381          * work around buggy pop3 servers which send
382          * empty lines in their listings.
383         */
384         if ((StrLength(RecvMsg->IO.IOBuf) == 0) ||
385             !isdigit(ChrPtr(RecvMsg->IO.IOBuf)[0]))
386         {
387                 return eReadMore;
388         }
389
390         OneMsg = (FetchItem*) malloc(sizeof(FetchItem));
391         memset(OneMsg, 0, sizeof(FetchItem));
392         OneMsg->MSGID = atol(ChrPtr(RecvMsg->IO.IOBuf));
393
394         pch = strchr(ChrPtr(RecvMsg->IO.IOBuf), ' ');
395         if (pch != NULL)
396         {
397                 OneMsg->MSGSize = atol(pch + 1);
398         }
399 #if 0
400         rc = TestValidateHash(RecvMsg->MsgNumbers);
401         if (rc != 0)
402                 EVP3CCS_syslog(LOG_DEBUG, "Hash Invalid: %d\n", rc);
403 #endif
404
405         Put(RecvMsg->MsgNumbers, LKEY(OneMsg->MSGID), OneMsg, HfreeFetchItem);
406 #if 0
407         rc = TestValidateHash(RecvMsg->MsgNumbers);
408         if (rc != 0)
409                 EVP3CCS_syslog(LOG_DEBUG, "Hash Invalid: %d\n", rc);
410 #endif
411         //RecvMsg->State --; /* read next Line */
412         return eReadMore;
413 }
414
415 eNextState POP3_FetchNetworkUsetableEntry(AsyncIO *IO)
416 {
417         long HKLen;
418         const char *HKey;
419         void *vData;
420         pop3aggr *RecvMsg = (pop3aggr *) IO->Data;
421         time_t seenstamp = 0;
422
423         SetPOP3State(IO, eUseTable);
424
425         if((RecvMsg->Pos != NULL) &&
426            GetNextHashPos(RecvMsg->MsgNumbers,
427                           RecvMsg->Pos,
428                           &HKLen,
429                           &HKey,
430                           &vData))
431         {
432                 if (server_shutting_down)
433                         return eAbort;
434
435                 RecvMsg->CurrMsg = (FetchItem*)vData;
436
437                 seenstamp = CheckIfAlreadySeen("POP3 Item Seen",
438                                                RecvMsg->CurrMsg->MsgUID,
439                                                EvGetNow(IO),
440                                                EvGetNow(IO) - USETABLE_ANTIEXPIRE,
441                                                eCheckUpdate,
442                                                IO->ID, CCID);
443                 if (seenstamp != 0)
444                 {
445                         /* Item has already been seen */
446                         RecvMsg->CurrMsg->NeedFetch = 0;
447                 }
448                 else
449                 {
450                         EVP3CCSM_syslog(LOG_DEBUG, "NO\n");
451                         RecvMsg->CurrMsg->NeedFetch = 1;
452                 }
453                 return NextDBOperation(&RecvMsg->IO,
454                                        POP3_FetchNetworkUsetableEntry);
455         }
456         else
457         {
458                 /* ok, now we know them all,
459                  * continue with reading the actual messages. */
460                 DeleteHashPos(&RecvMsg->Pos);
461                 return DBQueueEventContext(IO, POP3_C_ReAttachToFetchMessages);
462         }
463 }
464
465 eNextState POP3C_GetOneMessagID(pop3aggr *RecvMsg)
466 {
467         AsyncIO *IO = &RecvMsg->IO;
468         long HKLen;
469         const char *HKey;
470         void *vData;
471
472         SetPOP3State(IO, eGetMsgID);
473 #if 0
474         int rc;
475         rc = TestValidateHash(RecvMsg->MsgNumbers);
476         if (rc != 0)
477                 EVP3CCS_syslog(LOG_DEBUG, "Hash Invalid: %d\n", rc);
478 #endif
479         if((RecvMsg->Pos != NULL) &&
480            GetNextHashPos(RecvMsg->MsgNumbers,
481                           RecvMsg->Pos,
482                           &HKLen, &HKey,
483                           &vData))
484         {
485                 RecvMsg->CurrMsg = (FetchItem*) vData;
486                 /* Find out the UIDL of the message,
487                  * to determine whether we've already downloaded it */
488                 StrBufPrintf(RecvMsg->IO.SendBuf.Buf,
489                              "UIDL %ld\r\n", RecvMsg->CurrMsg->MSGID);
490                 POP3C_DBG_SEND();
491         }
492         else
493         {
494                 RecvMsg->State++;
495                 DeleteHashPos(&RecvMsg->Pos);
496                 /// done receiving uidls.. start looking them up now.
497                 RecvMsg->Pos = GetNewHashPos(RecvMsg->MsgNumbers, 0);
498                 return EventQueueDBOperation(&RecvMsg->IO,
499                                              POP3_FetchNetworkUsetableEntry,
500                                              0);
501         }
502         return eReadMore; /* TODO */
503 }
504
505 eNextState POP3C_GetOneMessageIDState(pop3aggr *RecvMsg)
506 {
507         AsyncIO *IO = &RecvMsg->IO;
508 #if 0
509         int rc;
510         rc = TestValidateHash(RecvMsg->MsgNumbers);
511         if (rc != 0)
512                 EVP3CCS_syslog(LOG_DEBUG, "Hash Invalid: %d\n", rc);
513 #endif
514
515         POP3C_DBG_READ();
516         if (!POP3C_OK) return eTerminateConnection;
517         RecvMsg->CurrMsg->MsgUIDL =
518                 NewStrBufPlain(NULL, StrLength(RecvMsg->IO.IOBuf));
519         RecvMsg->CurrMsg->MsgUID =
520                 NewStrBufPlain(NULL, StrLength(RecvMsg->IO.IOBuf) * 2);
521
522         StrBufExtract_token(RecvMsg->CurrMsg->MsgUIDL,
523                             RecvMsg->IO.IOBuf, 2, ' ');
524
525         StrBufPrintf(RecvMsg->CurrMsg->MsgUID,
526                      "pop3/%s/%s:%s@%s",
527                      ChrPtr(RecvMsg->RoomName),
528                      ChrPtr(RecvMsg->CurrMsg->MsgUIDL),
529                      RecvMsg->IO.ConnectMe->User,
530                      RecvMsg->IO.ConnectMe->Host);
531         RecvMsg->State --;
532         return eSendReply;
533 }
534
535
536 eNextState POP3C_SendGetOneMsg(pop3aggr *RecvMsg)
537 {
538         AsyncIO *IO = &RecvMsg->IO;
539         long HKLen;
540         const char *HKey;
541         void *vData;
542
543         SetPOP3State(IO, eGetMsg);
544
545         EVP3CM_syslog(LOG_DEBUG, "fast forwarding to the next unknown message");
546
547         RecvMsg->CurrMsg = NULL;
548         while ((RecvMsg->Pos != NULL) && 
549                GetNextHashPos(RecvMsg->MsgNumbers,
550                               RecvMsg->Pos,
551                               &HKLen, &HKey,
552                               &vData) &&
553                (RecvMsg->CurrMsg = (FetchItem*) vData,
554                 RecvMsg->CurrMsg->NeedFetch == 0))
555         {}
556
557         if ((RecvMsg->CurrMsg != NULL ) && (RecvMsg->CurrMsg->NeedFetch == 1))
558         {
559                 EVP3CM_syslog(LOG_DEBUG, "fetching next");
560                 /* Message has not been seen.
561                  * Tell the server to fetch the message... */
562                 StrBufPrintf(RecvMsg->IO.SendBuf.Buf,
563                              "RETR %ld\r\n", RecvMsg->CurrMsg->MSGID);
564                 POP3C_DBG_SEND();
565                 return eReadMessage;
566         }
567         else {
568                 EVP3CM_syslog(LOG_DEBUG, "no more messages to fetch.");
569                 RecvMsg->State = ReadQuitState;
570                 return POP3_C_DispatchWriteDone(&RecvMsg->IO);
571         }
572 }
573
574
575 eNextState POP3C_ReadMessageBodyFollowing(pop3aggr *RecvMsg)
576 {
577         AsyncIO *IO = &RecvMsg->IO;
578         POP3C_DBG_READ();
579         if (!POP3C_OK) return eTerminateConnection;
580         RecvMsg->IO.ReadMsg = NewAsyncMsg(HKEY("."),
581                                           RecvMsg->CurrMsg->MSGSize,
582                                           config.c_maxmsglen,
583                                           NULL, -1,
584                                           1);
585
586         return eReadPayload;
587 }
588
589
590 eNextState POP3C_StoreMsgRead(AsyncIO *IO)
591 {
592         pop3aggr *RecvMsg = (pop3aggr *) IO->Data;
593
594         SetPOP3State(IO, eStoreMsg);
595
596         EVP3CCS_syslog(LOG_DEBUG,
597                        "MARKING: %s as seen: ",
598                        ChrPtr(RecvMsg->CurrMsg->MsgUID));
599         CheckIfAlreadySeen("POP3 Item Seen",
600                            RecvMsg->CurrMsg->MsgUID,
601                            EvGetNow(IO),
602                            EvGetNow(IO) - USETABLE_ANTIEXPIRE,
603                            eWrite,
604                            IO->ID, CCID);
605
606         return DBQueueEventContext(&RecvMsg->IO, POP3_C_ReAttachToFetchMessages);
607 }
608 eNextState POP3C_SaveMsg(AsyncIO *IO)
609 {
610         long msgnum;
611         pop3aggr *RecvMsg = (pop3aggr *) IO->Data;
612
613         /* Do Something With It (tm) */
614         msgnum = CtdlSubmitMsg(RecvMsg->CurrMsg->Msg,
615                                NULL,
616                                ChrPtr(RecvMsg->RoomName),
617                                0);
618         if (msgnum > 0L)
619         {
620                 /* Message has been committed to the store
621                  * write the uidl to the use table
622                  * so we don't fetch this message again
623                  */
624         }
625         CM_Free(RecvMsg->CurrMsg->Msg);
626
627         RecvMsg->count ++;
628         return NextDBOperation(&RecvMsg->IO, POP3C_StoreMsgRead);
629 }
630
631 eNextState POP3C_ReadMessageBody(pop3aggr *RecvMsg)
632 {
633         AsyncIO *IO = &RecvMsg->IO;
634         EVP3CM_syslog(LOG_DEBUG, "Converting message...");
635         RecvMsg->CurrMsg->Msg =
636                 convert_internet_message_buf(&RecvMsg->IO.ReadMsg->MsgBuf);
637         return EventQueueDBOperation(&RecvMsg->IO, POP3C_SaveMsg, 0);
638 }
639
640 eNextState POP3C_SendDelete(pop3aggr *RecvMsg)
641 {
642         AsyncIO *IO = &RecvMsg->IO;
643
644         SetPOP3State(IO, eDelete);
645
646         if (!RecvMsg->keep) {
647                 StrBufPrintf(RecvMsg->IO.SendBuf.Buf,
648                              "DELE %ld\r\n", RecvMsg->CurrMsg->MSGID);
649                 POP3C_DBG_SEND();
650                 return eReadMessage;
651         }
652         else {
653                 RecvMsg->State = ReadMessageBodyFollowing;
654                 return POP3_C_DispatchWriteDone(&RecvMsg->IO);
655         }
656 }
657 eNextState POP3C_ReadDeleteState(pop3aggr *RecvMsg)
658 {
659         AsyncIO *IO = &RecvMsg->IO;
660         POP3C_DBG_READ();
661         RecvMsg->State = GetOneMessageIDState;
662         return POP3_C_DispatchWriteDone(&RecvMsg->IO);
663 }
664
665 eNextState POP3C_SendQuit(pop3aggr *RecvMsg)
666 {
667         AsyncIO *IO = &RecvMsg->IO;
668         SetPOP3State(IO, eQuit);
669
670         /* Log out */
671         StrBufPlain(RecvMsg->IO.SendBuf.Buf,
672                     HKEY("QUIT\r\n3)"));
673         POP3C_DBG_SEND();
674         return eReadMessage;
675 }
676
677
678 eNextState POP3C_ReadQuitState(pop3aggr *RecvMsg)
679 {
680         AsyncIO *IO = &RecvMsg->IO;
681         POP3C_DBG_READ();
682         return eTerminateConnection;
683 }
684
685 const long POP3_C_ConnTimeout = 1000;
686 const long DefaultPOP3Port = 110;
687
688 Pop3ClientHandler POP3C_ReadHandlers[] = {
689         POP3C_ReadGreeting,
690         POP3C_GetUserState,
691         POP3C_GetPassState,
692         POP3C_GetListCommandState,
693         POP3C_GetListOneLine,
694         POP3C_GetOneMessageIDState,
695         POP3C_ReadMessageBodyFollowing,
696         POP3C_ReadMessageBody,
697         POP3C_ReadDeleteState,
698         POP3C_ReadQuitState,
699 };
700
701 const long POP3_C_SendTimeouts[POP3C_MaxRead] = {
702         100,
703         100,
704         100,
705         100,
706         100,
707         100,
708         100,
709         100
710 };
711 const ConstStr POP3C_ReadErrors[POP3C_MaxRead] = {
712         {HKEY("Connection broken during ")},
713         {HKEY("Connection broken during ")},
714         {HKEY("Connection broken during ")},
715         {HKEY("Connection broken during ")},
716         {HKEY("Connection broken during ")},
717         {HKEY("Connection broken during ")},
718         {HKEY("Connection broken during ")},
719         {HKEY("Connection broken during ")}
720 };
721
722 Pop3ClientHandler POP3C_SendHandlers[] = {
723         NULL, /* we don't send a greeting */
724         POP3C_SendUser,
725         POP3C_SendPassword,
726         POP3C_SendListCommand,
727         NULL,
728         POP3C_GetOneMessagID,
729         POP3C_SendGetOneMsg,
730         NULL,
731         POP3C_SendDelete,
732         POP3C_SendQuit
733 };
734
735 const long POP3_C_ReadTimeouts[] = {
736         100,
737         100,
738         100,
739         100,
740         100,
741         100,
742         100,
743         100,
744         100,
745         100
746 };
747 /*****************************************************************************/
748 /*                     POP3 CLIENT DISPATCHER                                */
749 /*****************************************************************************/
750
751 void POP3SetTimeout(eNextState NextTCPState, pop3aggr *pMsg)
752 {
753         AsyncIO *IO = &pMsg->IO;
754         double Timeout = 0.0;
755
756         EVP3C_syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
757
758         switch (NextTCPState) {
759         case eSendFile:
760         case eSendReply:
761         case eSendMore:
762                 Timeout = POP3_C_SendTimeouts[pMsg->State];
763 /*
764   if (pMsg->State == eDATABody) {
765   / * if we're sending a huge message, we need more time. * /
766   Timeout += StrLength(pMsg->msgtext) / 1024;
767   }
768 */
769                 break;
770         case eReadFile:
771         case eReadMessage:
772                 Timeout = POP3_C_ReadTimeouts[pMsg->State];
773 /*
774   if (pMsg->State == eDATATerminateBody) {
775   / *
776   * some mailservers take a nap before accepting the message
777   * content inspection and such.
778   * /
779   Timeout += StrLength(pMsg->msgtext) / 1024;
780   }
781 */
782                 break;
783         case eReadPayload:
784                 Timeout = 100000;
785                 /* TODO!!! */
786                 break;
787         case eSendDNSQuery:
788         case eReadDNSReply:
789         case eConnect:
790         case eTerminateConnection:
791         case eDBQuery:
792         case eAbort:
793         case eReadMore://// TODO
794                 return;
795         }
796         SetNextTimeout(&pMsg->IO, Timeout);
797 }
798 eNextState POP3_C_DispatchReadDone(AsyncIO *IO)
799 {
800 /*      EVP3CCS_syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__); to noisy anyways. */
801         pop3aggr *pMsg = IO->Data;
802         eNextState rc;
803
804         rc = POP3C_ReadHandlers[pMsg->State](pMsg);
805         if (rc != eReadMore)
806             pMsg->State++;
807         POP3SetTimeout(rc, pMsg);
808         return rc;
809 }
810 eNextState POP3_C_DispatchWriteDone(AsyncIO *IO)
811 {
812         pop3aggr *pMsg = IO->Data;
813         eNextState rc;
814
815 /*      EVP3CCS_syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__); to noisy anyways. */
816         rc = POP3C_SendHandlers[pMsg->State](pMsg);
817         POP3SetTimeout(rc, pMsg);
818         return rc;
819 }
820
821
822 /*****************************************************************************/
823 /*                     POP3 CLIENT ERROR CATCHERS                            */
824 /*****************************************************************************/
825 eNextState POP3_C_Terminate(AsyncIO *IO)
826 {
827 ///     pop3aggr *pMsg = (pop3aggr *)IO->Data;
828
829         EVP3CCS_syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
830         FinalizePOP3AggrRun(IO);
831         return eAbort;
832 }
833 eNextState POP3_C_TerminateDB(AsyncIO *IO)
834 {
835 ///     pop3aggr *pMsg = (pop3aggr *)IO->Data;
836
837         EVP3CCS_syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
838         FinalizePOP3AggrRun(IO);
839         return eAbort;
840 }
841 eNextState POP3_C_Timeout(AsyncIO *IO)
842 {
843         pop3aggr *pMsg = IO->Data;
844
845         EVP3CCS_syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
846         StrBufPlain(IO->ErrMsg, CKEY(POP3C_ReadErrors[pMsg->State]));
847         return FailAggregationRun(IO);
848 }
849 eNextState POP3_C_ConnFail(AsyncIO *IO)
850 {
851         pop3aggr *pMsg = (pop3aggr *)IO->Data;
852
853         EVP3CCS_syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
854         StrBufPlain(IO->ErrMsg, CKEY(POP3C_ReadErrors[pMsg->State]));
855         return FailAggregationRun(IO);
856 }
857 eNextState POP3_C_DNSFail(AsyncIO *IO)
858 {
859         pop3aggr *pMsg = (pop3aggr *)IO->Data;
860
861         EVP3CCS_syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
862         StrBufPlain(IO->ErrMsg, CKEY(POP3C_ReadErrors[pMsg->State]));
863         return FailAggregationRun(IO);
864 }
865 eNextState POP3_C_Shutdown(AsyncIO *IO)
866 {
867         EVP3CCS_syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
868 ////    pop3aggr *pMsg = IO->Data;
869
870 ////pMsg->MyQEntry->Status = 3;
871 ///StrBufPlain(pMsg->MyQEntry->StatusMessage, HKEY("server shutdown during message retrieval."));
872         FinalizePOP3AggrRun(IO);
873         return eAbort;
874 }
875
876
877 /**
878  * @brief lineread Handler; understands when to read more POP3 lines,
879  *   and when this is a one-lined reply.
880  */
881 eReadState POP3_C_ReadServerStatus(AsyncIO *IO)
882 {
883         eReadState Finished = eBufferNotEmpty;
884
885         switch (IO->NextState) {
886         case eSendDNSQuery:
887         case eReadDNSReply:
888         case eDBQuery:
889         case eConnect:
890         case eTerminateConnection:
891         case eAbort:
892                 Finished = eReadFail;
893                 break;
894         case eSendFile:
895         case eSendReply:
896         case eSendMore:
897         case eReadMore:
898         case eReadMessage:
899                 Finished = StrBufChunkSipLine(IO->IOBuf, &IO->RecvBuf);
900                 break;
901         case eReadFile:
902         case eReadPayload:
903                 Finished = CtdlReadMessageBodyAsync(IO);
904                 break;
905         }
906         return Finished;
907 }
908
909 /*****************************************************************************
910  * So we connect our Server IP here.                                         *
911  *****************************************************************************/
912 eNextState POP3_C_ReAttachToFetchMessages(AsyncIO *IO)
913 {
914         pop3aggr *cpptr = IO->Data;
915
916         EVP3CCS_syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
917 ////??? cpptr->State ++;
918         if (cpptr->Pos == NULL)
919                 cpptr->Pos = GetNewHashPos(cpptr->MsgNumbers, 0);
920
921         POP3_C_DispatchWriteDone(IO);
922         ReAttachIO(IO, cpptr, 0);
923         IO->NextState = eReadMessage;
924         return IO->NextState;
925 }
926
927 eNextState pop3_connect_ip(AsyncIO *IO)
928 {
929         pop3aggr *cpptr = IO->Data;
930
931         if (cpptr->IOStart == 0.0) /* whith or without DNS? */
932                 cpptr->IOStart = IO->Now;
933
934         EVP3CCS_syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
935
936         return EvConnectSock(IO,
937                              POP3_C_ConnTimeout,
938                              POP3_C_ReadTimeouts[0],
939                              1);
940 }
941
942 eNextState pop3_get_one_host_ip_done(AsyncIO *IO)
943 {
944         pop3aggr *cpptr = IO->Data;
945         struct hostent *hostent;
946
947         QueryCbDone(IO);
948
949         hostent = cpptr->HostLookup.VParsedDNSReply;
950         if ((cpptr->HostLookup.DNSStatus == ARES_SUCCESS) && 
951             (hostent != NULL) ) {
952                 memset(&cpptr->IO.ConnectMe->Addr, 0, sizeof(struct in6_addr));
953                 if (cpptr->IO.ConnectMe->IPv6) {
954                         memcpy(&cpptr->IO.ConnectMe->Addr.sin6_addr.s6_addr, 
955                                &hostent->h_addr_list[0],
956                                sizeof(struct in6_addr));
957
958                         cpptr->IO.ConnectMe->Addr.sin6_family =
959                                 hostent->h_addrtype;
960                         cpptr->IO.ConnectMe->Addr.sin6_port   =
961                                 htons(DefaultPOP3Port);
962                 }
963                 else {
964                         struct sockaddr_in *addr =
965                                 (struct sockaddr_in*)
966                                 &cpptr->IO.ConnectMe->Addr;
967
968                         memcpy(&addr->sin_addr.s_addr,
969                                hostent->h_addr_list[0],
970                                sizeof(uint32_t));
971
972                         addr->sin_family = hostent->h_addrtype;
973                         addr->sin_port   = htons(DefaultPOP3Port);
974                 }
975                 return pop3_connect_ip(IO);
976         }
977         else
978                 return eAbort;
979 }
980
981 eNextState pop3_get_one_host_ip(AsyncIO *IO)
982 {
983         pop3aggr *cpptr = IO->Data;
984
985         cpptr->IOStart = IO->Now;
986
987         EVP3CCS_syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
988
989         EVP3CCS_syslog(LOG_DEBUG, 
990                        "POP3 client[%ld]: looking up %s-Record %s : %d ...\n",
991                        cpptr->n,
992                        (cpptr->IO.ConnectMe->IPv6)? "aaaa": "a",
993                        cpptr->IO.ConnectMe->Host,
994                        cpptr->IO.ConnectMe->Port);
995
996         QueueQuery((cpptr->IO.ConnectMe->IPv6)? ns_t_aaaa : ns_t_a,
997                    cpptr->IO.ConnectMe->Host,
998                    &cpptr->IO,
999                    &cpptr->HostLookup,
1000                    pop3_get_one_host_ip_done);
1001         IO->NextState = eReadDNSReply;
1002         return IO->NextState;
1003 }
1004
1005
1006
1007 int pop3_do_fetching(pop3aggr *cpptr)
1008 {
1009         AsyncIO *IO = &cpptr->IO;
1010
1011         InitIOStruct(IO,
1012                      cpptr,
1013                      eReadMessage,
1014                      POP3_C_ReadServerStatus,
1015                      POP3_C_DNSFail,
1016                      POP3_C_DispatchWriteDone,
1017                      POP3_C_DispatchReadDone,
1018                      POP3_C_Terminate,
1019                      POP3_C_TerminateDB,
1020                      POP3_C_ConnFail,
1021                      POP3_C_Timeout,
1022                      POP3_C_Shutdown);
1023
1024         safestrncpy(((CitContext *)cpptr->IO.CitContext)->cs_host,
1025                     ChrPtr(cpptr->Url),
1026                     sizeof(((CitContext *)cpptr->IO.CitContext)->cs_host));
1027
1028         if (cpptr->IO.ConnectMe->IsIP) {
1029                 QueueEventContext(&cpptr->IO,
1030                                   pop3_connect_ip);
1031         }
1032         else {
1033                 QueueEventContext(&cpptr->IO,
1034                                   pop3_get_one_host_ip);
1035         }
1036         return 1;
1037 }
1038
1039 /*
1040  * Scan a room's netconfig to determine whether it requires POP3 aggregation
1041  */
1042 void pop3client_scan_room(struct ctdlroom *qrbuf, void *data, OneRoomNetCfg *OneRNCFG)
1043 {
1044         const RoomNetCfgLine *pLine;
1045         void *vptr;
1046
1047         pthread_mutex_lock(&POP3QueueMutex);
1048         if (GetHash(POP3QueueRooms, LKEY(qrbuf->QRnumber), &vptr))
1049         {
1050                 pthread_mutex_unlock(&POP3QueueMutex);
1051                 EVP3CQ_syslog(LOG_DEBUG,
1052                               "pop3client: [%ld] %s already in progress.",
1053                               qrbuf->QRnumber,
1054                               qrbuf->QRname);
1055                 return;
1056         }
1057         pthread_mutex_unlock(&POP3QueueMutex);
1058
1059         if (server_shutting_down) return;
1060
1061         pLine = OneRNCFG->NetConfigs[pop3client];
1062
1063         while (pLine != NULL)
1064         {
1065                 pop3aggr *cptr;
1066
1067                 cptr = (pop3aggr *) malloc(sizeof(pop3aggr));
1068                 memset(cptr, 0, sizeof(pop3aggr));
1069                 ///TODO do we need this? cptr->roomlist_parts=1;
1070                 cptr->RoomName = NewStrBufPlain(qrbuf->QRname, -1);
1071                 cptr->pop3user = NewStrBufDup(pLine->Value[1]);
1072                 cptr->pop3pass = NewStrBufDup(pLine->Value[2]);
1073                 cptr->Url = NewStrBuf();
1074                 cptr->Host = NewStrBufDup(pLine->Value[0]);
1075
1076                 cptr->keep = atol(ChrPtr(pLine->Value[3]));
1077                 cptr->interval = atol(ChrPtr(pLine->Value[4]));
1078
1079                 StrBufAppendBufPlain(cptr->Url, HKEY("pop3://"), 0);
1080                 StrBufUrlescUPAppend(cptr->Url, cptr->pop3user, NULL);
1081                 StrBufAppendBufPlain(cptr->Url, HKEY(":"), 0);
1082                 StrBufUrlescUPAppend(cptr->Url, cptr->pop3pass, NULL);
1083                 StrBufAppendBufPlain(cptr->Url, HKEY("@"), 0);
1084                 StrBufAppendBuf(cptr->Url, cptr->Host, 0);
1085                 StrBufAppendBufPlain(cptr->Url, HKEY("/"), 0);
1086                 StrBufUrlescAppend(cptr->Url, cptr->RoomName, NULL);
1087
1088                 ParseURL(&cptr->IO.ConnectMe, cptr->Url, 110);
1089
1090
1091 #if 0
1092 /* todo: we need to reunite the url to be shure. */
1093
1094                 pthread_mutex_lock(&POP3ueueMutex);
1095                 GetHash(POP3FetchUrls, SKEY(ptr->Url), &vptr);
1096                 use_this_cptr = (pop3aggr *)vptr;
1097
1098                 if (use_this_rncptr != NULL)
1099                 {
1100                         /* mustn't attach to an active session */
1101                         if (use_this_cptr->RefCount > 0)
1102                         {
1103                                 DeletePOP3Cfg(cptr);
1104 ///                                             Count->count--;
1105                         }
1106                         else
1107                         {
1108                                 long *QRnumber;
1109                                 StrBufAppendBufPlain(
1110                                         use_this_cptr->rooms,
1111                                         qrbuf->QRname,
1112                                         -1, 0);
1113                                 if (use_this_cptr->roomlist_parts == 1)
1114                                 {
1115                                         use_this_cptr->OtherQRnumbers
1116                                                 = NewHash(1, lFlathash);
1117                                 }
1118                                 QRnumber = (long*)malloc(sizeof(long));
1119                                 *QRnumber = qrbuf->QRnumber;
1120                                 Put(use_this_cptr->OtherQRnumbers,
1121                                     LKEY(qrbuf->QRnumber),
1122                                     QRnumber,
1123                                     NULL);
1124
1125                                 use_this_cptr->roomlist_parts++;
1126                         }
1127                         pthread_mutex_unlock(&POP3QueueMutex);
1128                         continue;
1129                 }
1130                 pthread_mutex_unlock(&RSSQueueMutex);
1131 #endif
1132                 cptr->n = Pop3ClientID++;
1133                 pthread_mutex_lock(&POP3QueueMutex);
1134                 Put(POP3FetchUrls,
1135                     SKEY(cptr->Url),
1136                     cptr,
1137                     DeletePOP3Aggregator);
1138
1139                 pthread_mutex_unlock(&POP3QueueMutex);
1140                 pLine = pLine->next;
1141
1142         }
1143 }
1144
1145 static int doing_pop3client = 0;
1146
1147 void pop3client_scan(void) {
1148         static time_t last_run = 0L;
1149         time_t fastest_scan;
1150         HashPos *it;
1151         long len;
1152         const char *Key;
1153         void *vrptr;
1154         pop3aggr *cptr;
1155
1156         become_session(&pop3_client_CC);
1157
1158         if (config.c_pop3_fastest < config.c_pop3_fetch)
1159                 fastest_scan = config.c_pop3_fastest;
1160         else
1161                 fastest_scan = config.c_pop3_fetch;
1162
1163         /*
1164          * Run POP3 aggregation no more frequently than once every n seconds
1165          */
1166         if ( (time(NULL) - last_run) < fastest_scan ) {
1167                 return;
1168         }
1169
1170         /*
1171          * This is a simple concurrency check to make sure only one pop3client
1172          * run is done at a time.  We could do this with a mutex, but since we
1173          * don't really require extremely fine granularity here, we'll do it
1174          * with a static variable instead.
1175          */
1176         if (doing_pop3client) return;
1177         doing_pop3client = 1;
1178
1179         EVP3CQM_syslog(LOG_DEBUG, "pop3client started");
1180         CtdlForEachNetCfgRoom(pop3client_scan_room, NULL, pop3client);
1181
1182         pthread_mutex_lock(&POP3QueueMutex);
1183         it = GetNewHashPos(POP3FetchUrls, 0);
1184         while (!server_shutting_down &&
1185                GetNextHashPos(POP3FetchUrls, it, &len, &Key, &vrptr) &&
1186                (vrptr != NULL)) {
1187                 cptr = (pop3aggr *)vrptr;
1188                 if (cptr->RefCount == 0)
1189                         if (!pop3_do_fetching(cptr))
1190                                 DeletePOP3Aggregator(cptr);////TODO
1191
1192 /*
1193         if ((palist->interval && time(NULL) > (last_run + palist->interval))
1194                         || (time(NULL) > last_run + config.c_pop3_fetch))
1195                         pop3_do_fetching(palist->roomname, palist->pop3host,
1196                         palist->pop3user, palist->pop3pass, palist->keep);
1197                 pptr = palist;
1198                 palist = palist->next;
1199                 free(pptr);
1200 */
1201         }
1202         DeleteHashPos(&it);
1203         pthread_mutex_unlock(&POP3QueueMutex);
1204
1205         EVP3CQM_syslog(LOG_DEBUG, "pop3client ended");
1206         last_run = time(NULL);
1207         doing_pop3client = 0;
1208 }
1209
1210
1211 void pop3_cleanup(void)
1212 {
1213         /* citthread_mutex_destroy(&POP3QueueMutex); TODO */
1214         while (doing_pop3client != 0) ;
1215         DeleteHash(&POP3FetchUrls);
1216         DeleteHash(&POP3QueueRooms);
1217 }
1218
1219
1220
1221 void LogDebugEnablePOP3Client(const int n)
1222 {
1223         POP3ClientDebugEnabled = n;
1224 }
1225
1226 CTDL_MODULE_INIT(pop3client)
1227 {
1228         if (!threading)
1229         {
1230                 CtdlFillSystemContext(&pop3_client_CC, "POP3aggr");
1231                 CtdlREGISTERRoomCfgType(pop3client, ParseGeneric, 0, 5, SerializeGeneric, DeleteGenericCfgLine);
1232                 pthread_mutex_init(&POP3QueueMutex, NULL);
1233                 POP3QueueRooms = NewHash(1, lFlathash);
1234                 POP3FetchUrls = NewHash(1, NULL);
1235                 CtdlRegisterSessionHook(pop3client_scan, EVT_TIMER, PRIO_AGGR + 50);
1236                 CtdlRegisterEVCleanupHook(pop3_cleanup);
1237                 CtdlRegisterDebugFlagHook(HKEY("pop3client"), LogDebugEnablePOP3Client, &POP3ClientDebugEnabled);
1238         }
1239
1240         /* return our module id for the log */
1241         return "pop3client";
1242 }