EV: stop all watchers before going into other queues - so we can avoid race conditions.
[citadel.git] / citadel / modules / pop3client / serv_pop3client.c
1 /*
2  * Consolidate mail from remote POP3 accounts.
3  *
4  * Copyright (c) 2007-2011 by the citadel.org team
5  *
6  * This program is open source software; you can redistribute it and/or
7  * modify it under the terms of the GNU General Public License as published
8  * by the Free Software Foundation; either version 3 of the License, or
9  * (at your option) any later version.
10  *
11  * This program is distributed in the hope that it will be useful,
12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
14  * GNU General Public License for more details.
15  *
16  * You should have received a copy of the GNU General Public License
17  * along with this program; if not, write to the Free Software
18  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
19  */
20
21 #include <stdlib.h>
22 #include <unistd.h>
23 #include <stdio.h>
24
25 #if TIME_WITH_SYS_TIME
26 # include <sys/time.h>
27 # include <time.h>
28 #else
29 # if HAVE_SYS_TIME_H
30 #  include <sys/time.h>
31 # else
32 #  include <time.h>
33 # endif
34 #endif
35
36 #include <ctype.h>
37 #include <string.h>
38 #include <errno.h>
39 #include <sys/types.h>
40 #include <sys/stat.h>
41 #include <libcitadel.h>
42 #include "citadel.h"
43 #include "server.h"
44 #include "citserver.h"
45 #include "support.h"
46 #include "config.h"
47 #include "ctdl_module.h"
48 #include "clientsocket.h"
49 #include "msgbase.h"
50 #include "internet_addressing.h"
51 #include "database.h"
52 #include "citadel_dirs.h"
53 #include "event_client.h"
54
55
56 #define POP3C_OK (strncasecmp(ChrPtr(RecvMsg->IO.IOBuf), "+OK", 3) == 0)
57 int Pop3ClientID = 0;
58 int POP3ClientDebugEnabled = 0;
59
60 #define N ((pop3aggr*)IO->Data)->n
61
62 #define DBGLOG(LEVEL) if ((LEVEL != LOG_DEBUG) || (POP3ClientDebugEnabled != 0))
63
64 #define EVP3C_syslog(LEVEL, FORMAT, ...)                                \
65         DBGLOG(LEVEL) syslog(LEVEL,                                     \
66                              "IO[%ld]CC[%d][%ld]POP3: " FORMAT,         \
67                              IO->ID, CCID, N, __VA_ARGS__)
68
69 #define EVP3CM_syslog(LEVEL, FORMAT)                                    \
70         DBGLOG(LEVEL) syslog(LEVEL,                                     \
71                              "IO[%ld]CC[%d][%ld]POP3: " FORMAT,         \
72                              IO->ID, CCID, N)
73
74 #define EVP3CQ_syslog(LEVEL, FORMAT, ...)                               \
75         DBGLOG(LEVEL) syslog(LEVEL,                                     \
76                              "P3Q:" FORMAT,                             \
77                              __VA_ARGS__)
78
79 #define EVP3CQM_syslog(LEVEL, FORMAT)                                   \
80         DBGLOG(LEVEL) syslog(LEVEL,                                     \
81                              "P3Q" FORMAT                               \
82                 )
83
84 #define EVP3CCS_syslog(LEVEL, FORMAT, ...)                              \
85         DBGLOG(LEVEL) syslog(LEVEL, "IO[%ld][%ld]POP3: " FORMAT,        \
86                              IO->ID, N, __VA_ARGS__)
87
88 #define EVP3CCSM_syslog(LEVEL, FORMAT)                                  \
89         DBGLOG(LEVEL) syslog(LEVEL, "IO[%ld][%ld]POP3: " FORMAT,        \
90                              IO->ID, N)
91
92 #define POP3C_DBG_SEND()                                                \
93         EVP3C_syslog(LOG_DEBUG,                                         \
94                      "IO[%ld]CC[%d][%ld]POP3: > %s\n",                  \
95                      IO->ID, CCID, N,                                   \
96                      ChrPtr(RecvMsg->IO.SendBuf.Buf))
97
98 #define POP3C_DBG_READ()                                                \
99         EVP3C_syslog(LOG_DEBUG,                                         \
100                      "IO[%ld]CC[%d][%ld]POP3: < %s\n",                  \
101                      IO->ID, CCID, N,                                   \
102                      ChrPtr(RecvMsg->IO.IOBuf))
103
104
105 struct CitContext pop3_client_CC;
106
107 pthread_mutex_t POP3QueueMutex; /* locks the access to the following vars: */
108 HashList *POP3QueueRooms = NULL;
109 HashList *POP3FetchUrls = NULL;
110
111 typedef struct pop3aggr pop3aggr;
112 typedef eNextState(*Pop3ClientHandler)(pop3aggr* RecvMsg);
113
114 eNextState POP3_C_Shutdown(AsyncIO *IO);
115 eNextState POP3_C_Timeout(AsyncIO *IO);
116 eNextState POP3_C_ConnFail(AsyncIO *IO);
117 eNextState POP3_C_DNSFail(AsyncIO *IO);
118 eNextState POP3_C_DispatchReadDone(AsyncIO *IO);
119 eNextState POP3_C_DispatchWriteDone(AsyncIO *IO);
120 eNextState POP3_C_Terminate(AsyncIO *IO);
121 eReadState POP3_C_ReadServerStatus(AsyncIO *IO);
122 eNextState POP3_C_ReAttachToFetchMessages(AsyncIO *IO);
123
124 typedef struct __pop3_room_counter {
125         int count;
126         long QRnumber;
127 }pop3_room_counter;
128
129 typedef enum ePOP3_C_States {
130         ReadGreeting,
131         GetUserState,
132         GetPassState,
133         GetListCommandState,
134         GetListOneLine,
135         GetOneMessageIDState,
136         ReadMessageBodyFollowing,
137         ReadMessageBody,
138         GetDeleteState,
139         ReadQuitState,
140         POP3C_MaxRead
141 }ePOP3_C_States;
142
143
144 typedef struct _FetchItem {
145         long MSGID;
146         long MSGSize;
147         StrBuf *MsgUIDL;
148         StrBuf *MsgUID;
149         int NeedFetch;
150         struct CtdlMessage *Msg;
151 } FetchItem;
152
153 void HfreeFetchItem(void *vItem)
154 {
155         FetchItem *Item = (FetchItem*) vItem;
156         FreeStrBuf(&Item->MsgUIDL);
157         FreeStrBuf(&Item->MsgUID);
158         free(Item);
159 }
160
161
162
163 typedef enum _POP3State {
164         eCreated,
165         eGreeting,
166         eUser,
167         ePassword,
168         eListing,
169         eUseTable,
170         eGetMsgID,
171         eGetMsg,
172         eStoreMsg,
173         eDelete,
174         eQuit
175 } POP3State;
176
177 ConstStr POP3States[] = {
178         {HKEY("Aggregator created")},
179         {HKEY("Reading Greeting")},
180         {HKEY("Sending User")},
181         {HKEY("Sending Password")},
182         {HKEY("Listing")},
183         {HKEY("Fetching Usetable")},
184         {HKEY("Get MSG ID")},
185         {HKEY("Get Message")},
186         {HKEY("Store Msg")},
187         {HKEY("Delete Upstream")},
188         {HKEY("Quit")}
189 };
190
191 static void SetPOP3State(AsyncIO *IO, POP3State State)
192 {
193         CitContext* CCC = IO->CitContext;
194         if (CCC != NULL)
195                 memcpy(CCC->cs_clientname, POP3States[State].Key, POP3States[State].len + 1);
196 }
197
198
199 struct pop3aggr {
200         AsyncIO  IO;
201
202         long n;
203         double IOStart;
204         long count;
205         long RefCount;
206         DNSQueryParts HostLookup;
207
208         long             QRnumber;
209         HashList        *OtherQRnumbers;
210
211         StrBuf          *Url;
212         StrBuf *pop3user;
213         StrBuf *pop3pass;
214         StrBuf *Host;
215         StrBuf *RoomName; // TODO: fill me
216         int keep;
217         time_t interval;
218         ePOP3_C_States State;
219         HashList *MsgNumbers;
220         HashPos *Pos;
221         FetchItem *CurrMsg;
222 };
223
224 void DeletePOP3Aggregator(void *vptr)
225 {
226         pop3aggr *ptr = vptr;
227         DeleteHashPos(&ptr->Pos);
228         DeleteHash(&ptr->MsgNumbers);
229 //      FreeStrBuf(&ptr->rooms);
230         FreeStrBuf(&ptr->pop3user);
231         FreeStrBuf(&ptr->pop3pass);
232         FreeStrBuf(&ptr->Host);
233         FreeStrBuf(&ptr->RoomName);
234         FreeURL(&ptr->IO.ConnectMe);
235         FreeStrBuf(&ptr->Url);
236         FreeStrBuf(&ptr->IO.IOBuf);
237         FreeStrBuf(&ptr->IO.SendBuf.Buf);
238         FreeStrBuf(&ptr->IO.RecvBuf.Buf);
239         DeleteAsyncMsg(&ptr->IO.ReadMsg);
240         if (((struct CitContext*)ptr->IO.CitContext)) {
241                 ((struct CitContext*)ptr->IO.CitContext)->state = CON_IDLE;
242                 ((struct CitContext*)ptr->IO.CitContext)->kill_me = 1;
243         }
244         FreeAsyncIOContents(&ptr->IO);
245         free(ptr);
246 }
247
248 eNextState FinalizePOP3AggrRun(AsyncIO *IO)
249 {
250         HashPos  *It;
251         pop3aggr *cpptr = (pop3aggr *)IO->Data;
252
253         EVP3C_syslog(LOG_INFO,
254                      "%s@%s: fetched %ld new of %d messages in %fs. bye.",
255                      ChrPtr(cpptr->pop3user),
256                      ChrPtr(cpptr->Host),
257                      cpptr->count,
258                      GetCount(cpptr->MsgNumbers), 
259                      IO->Now - cpptr->IOStart 
260                 );
261
262         It = GetNewHashPos(POP3FetchUrls, 0);
263         pthread_mutex_lock(&POP3QueueMutex);
264         {
265                 if (GetHashPosFromKey(POP3FetchUrls, SKEY(cpptr->Url), It))
266                         DeleteEntryFromHash(POP3FetchUrls, It);
267         }
268         pthread_mutex_unlock(&POP3QueueMutex);
269         DeleteHashPos(&It);
270         return eAbort;
271 }
272
273 eNextState FailAggregationRun(AsyncIO *IO)
274 {
275         return eAbort;
276 }
277
278 eNextState POP3C_ReadGreeting(pop3aggr *RecvMsg)
279 {
280         AsyncIO *IO = &RecvMsg->IO;
281         SetPOP3State(IO, eGreeting);
282         POP3C_DBG_READ();
283         /* Read the server greeting */
284         if (!POP3C_OK) return eTerminateConnection;
285         else return eSendReply;
286 }
287
288 eNextState POP3C_SendUser(pop3aggr *RecvMsg)
289 {
290         AsyncIO *IO = &RecvMsg->IO;
291         SetPOP3State(IO, eUser);
292         /* Identify ourselves.  NOTE: we have to append a CR to each command.
293          *  The LF will automatically be appended by sock_puts().  Believe it
294          * or not, leaving out the CR will cause problems if the server happens
295          * to be Exchange, which is so b0rken it actually barfs on
296          * LF-terminated newlines.
297          */
298         StrBufPrintf(RecvMsg->IO.SendBuf.Buf,
299                      "USER %s\r\n", ChrPtr(RecvMsg->pop3user));
300         POP3C_DBG_SEND();
301         return eReadMessage;
302 }
303
304 eNextState POP3C_GetUserState(pop3aggr *RecvMsg)
305 {
306         AsyncIO *IO = &RecvMsg->IO;
307         POP3C_DBG_READ();
308         if (!POP3C_OK) return eTerminateConnection;
309         else return eSendReply;
310 }
311
312 eNextState POP3C_SendPassword(pop3aggr *RecvMsg)
313 {
314         AsyncIO *IO = &RecvMsg->IO;
315         SetPOP3State(IO, ePassword);
316         /* Password */
317         StrBufPrintf(RecvMsg->IO.SendBuf.Buf,
318                      "PASS %s\r\n", ChrPtr(RecvMsg->pop3pass));
319         EVP3CM_syslog(LOG_DEBUG, "<PASS <password>\n");
320 //      POP3C_DBG_SEND(); No, we won't write the passvoid to syslog...
321         return eReadMessage;
322 }
323
324 eNextState POP3C_GetPassState(pop3aggr *RecvMsg)
325 {
326         AsyncIO *IO = &RecvMsg->IO;
327         POP3C_DBG_READ();
328         if (!POP3C_OK) return eTerminateConnection;
329         else return eSendReply;
330 }
331
332 eNextState POP3C_SendListCommand(pop3aggr *RecvMsg)
333 {
334         AsyncIO *IO = &RecvMsg->IO;
335         SetPOP3State(IO, eListing);
336
337         /* Get the list of messages */
338         StrBufPlain(RecvMsg->IO.SendBuf.Buf, HKEY("LIST\r\n"));
339         POP3C_DBG_SEND();
340         return eReadMessage;
341 }
342
343 eNextState POP3C_GetListCommandState(pop3aggr *RecvMsg)
344 {
345         AsyncIO *IO = &RecvMsg->IO;
346         POP3C_DBG_READ();
347         if (!POP3C_OK) return eTerminateConnection;
348         RecvMsg->MsgNumbers = NewHash(1, NULL);
349         RecvMsg->State++;
350         return eReadMore;
351 }
352
353
354 eNextState POP3C_GetListOneLine(pop3aggr *RecvMsg)
355 {
356         AsyncIO *IO = &RecvMsg->IO;
357 #if 0
358         int rc;
359 #endif
360         const char *pch;
361         FetchItem *OneMsg = NULL;
362         POP3C_DBG_READ();
363
364         if ((StrLength(RecvMsg->IO.IOBuf) == 1) &&
365             (ChrPtr(RecvMsg->IO.IOBuf)[0] == '.'))
366         {
367                 if (GetCount(RecvMsg->MsgNumbers) == 0)
368                 {
369                         ////    RecvMsg->Sate = ReadQuitState;
370                 }
371                 else
372                 {
373                         RecvMsg->Pos = GetNewHashPos(RecvMsg->MsgNumbers, 0);
374                 }
375                 return eSendReply;
376
377         }
378
379         /*
380          * work around buggy pop3 servers which send
381          * empty lines in their listings.
382         */
383         if ((StrLength(RecvMsg->IO.IOBuf) == 0) ||
384             !isdigit(ChrPtr(RecvMsg->IO.IOBuf)[0]))
385         {
386                 return eReadMore;
387         }
388
389         OneMsg = (FetchItem*) malloc(sizeof(FetchItem));
390         memset(OneMsg, 0, sizeof(FetchItem));
391         OneMsg->MSGID = atol(ChrPtr(RecvMsg->IO.IOBuf));
392
393         pch = strchr(ChrPtr(RecvMsg->IO.IOBuf), ' ');
394         if (pch != NULL)
395         {
396                 OneMsg->MSGSize = atol(pch + 1);
397         }
398 #if 0
399         rc = TestValidateHash(RecvMsg->MsgNumbers);
400         if (rc != 0)
401                 EVP3CCS_syslog(LOG_DEBUG, "Hash Invalid: %d\n", rc);
402 #endif
403
404         Put(RecvMsg->MsgNumbers, LKEY(OneMsg->MSGID), OneMsg, HfreeFetchItem);
405 #if 0
406         rc = TestValidateHash(RecvMsg->MsgNumbers);
407         if (rc != 0)
408                 EVP3CCS_syslog(LOG_DEBUG, "Hash Invalid: %d\n", rc);
409 #endif
410         //RecvMsg->State --; /* read next Line */
411         return eReadMore;
412 }
413
414 eNextState POP3_FetchNetworkUsetableEntry(AsyncIO *IO)
415 {
416         long HKLen;
417         const char *HKey;
418         void *vData;
419         pop3aggr *RecvMsg = (pop3aggr *) IO->Data;
420
421         SetPOP3State(IO, eUseTable);
422
423         if((RecvMsg->Pos != NULL) &&
424            GetNextHashPos(RecvMsg->MsgNumbers,
425                           RecvMsg->Pos,
426                           &HKLen,
427                           &HKey,
428                           &vData))
429         {
430                 if (server_shutting_down)
431                         return eAbort;
432
433                 if (CheckIfAlreadySeen("POP3 Item Seen",
434                                        RecvMsg->CurrMsg->MsgUID,
435                                        IO->Now,
436                                        IO->Now, //// todo
437                                        eCheckUpdate,
438                                        IO->ID, CCID)
439                     != 0)
440                 {
441                         /* Item has already been seen */
442                         RecvMsg->CurrMsg->NeedFetch = 0;
443                 }
444                 else
445                 {
446                         EVP3CCSM_syslog(LOG_DEBUG, "NO\n");
447                         RecvMsg->CurrMsg->NeedFetch = 1;
448                 }
449                 return NextDBOperation(&RecvMsg->IO,
450                                        POP3_FetchNetworkUsetableEntry);
451         }
452         else
453         {
454                 /* ok, now we know them all,
455                  * continue with reading the actual messages. */
456                 DeleteHashPos(&RecvMsg->Pos);
457                 return DBQueueEventContext(IO, POP3_C_ReAttachToFetchMessages);
458         }
459 }
460
461 eNextState POP3C_GetOneMessagID(pop3aggr *RecvMsg)
462 {
463         AsyncIO *IO = &RecvMsg->IO;
464         long HKLen;
465         const char *HKey;
466         void *vData;
467
468         SetPOP3State(IO, eGetMsgID);
469 #if 0
470         int rc;
471         rc = TestValidateHash(RecvMsg->MsgNumbers);
472         if (rc != 0)
473                 EVP3CCS_syslog(LOG_DEBUG, "Hash Invalid: %d\n", rc);
474 #endif
475         if((RecvMsg->Pos != NULL) &&
476            GetNextHashPos(RecvMsg->MsgNumbers,
477                           RecvMsg->Pos,
478                           &HKLen, &HKey,
479                           &vData))
480         {
481                 RecvMsg->CurrMsg = (FetchItem*) vData;
482                 /* Find out the UIDL of the message,
483                  * to determine whether we've already downloaded it */
484                 StrBufPrintf(RecvMsg->IO.SendBuf.Buf,
485                              "UIDL %ld\r\n", RecvMsg->CurrMsg->MSGID);
486                 POP3C_DBG_SEND();
487         }
488         else
489         {
490                 RecvMsg->State++;
491                 DeleteHashPos(&RecvMsg->Pos);
492                 /// done receiving uidls.. start looking them up now.
493                 RecvMsg->Pos = GetNewHashPos(RecvMsg->MsgNumbers, 0);
494                 return EventQueueDBOperation(&RecvMsg->IO,
495                                              POP3_FetchNetworkUsetableEntry);
496         }
497         return eReadMore; /* TODO */
498 }
499
500 eNextState POP3C_GetOneMessageIDState(pop3aggr *RecvMsg)
501 {
502         AsyncIO *IO = &RecvMsg->IO;
503 #if 0
504         int rc;
505         rc = TestValidateHash(RecvMsg->MsgNumbers);
506         if (rc != 0)
507                 EVP3CCS_syslog(LOG_DEBUG, "Hash Invalid: %d\n", rc);
508 #endif
509
510         POP3C_DBG_READ();
511         if (!POP3C_OK) return eTerminateConnection;
512         RecvMsg->CurrMsg->MsgUIDL =
513                 NewStrBufPlain(NULL, StrLength(RecvMsg->IO.IOBuf));
514         RecvMsg->CurrMsg->MsgUID =
515                 NewStrBufPlain(NULL, StrLength(RecvMsg->IO.IOBuf) * 2);
516
517         StrBufExtract_token(RecvMsg->CurrMsg->MsgUIDL,
518                             RecvMsg->IO.IOBuf, 2, ' ');
519
520         StrBufPrintf(RecvMsg->CurrMsg->MsgUID,
521                      "pop3/%s/%s:%s@%s",
522                      ChrPtr(RecvMsg->RoomName),
523                      ChrPtr(RecvMsg->CurrMsg->MsgUIDL),
524                      RecvMsg->IO.ConnectMe->User,
525                      RecvMsg->IO.ConnectMe->Host);
526         RecvMsg->State --;
527         return eSendReply;
528 }
529
530
531 eNextState POP3C_SendGetOneMsg(pop3aggr *RecvMsg)
532 {
533         AsyncIO *IO = &RecvMsg->IO;
534         long HKLen;
535         const char *HKey;
536         void *vData;
537
538         SetPOP3State(IO, eGetMsg);
539
540         RecvMsg->CurrMsg = NULL;
541         while ((RecvMsg->Pos != NULL) && 
542                GetNextHashPos(RecvMsg->MsgNumbers,
543                               RecvMsg->Pos,
544                               &HKLen, &HKey,
545                               &vData) &&
546                (RecvMsg->CurrMsg = (FetchItem*) vData,
547                 RecvMsg->CurrMsg->NeedFetch == 0))
548         {}
549
550         if ((RecvMsg->CurrMsg != NULL ) && (RecvMsg->CurrMsg->NeedFetch == 1))
551         {
552                 /* Message has not been seen.
553                  * Tell the server to fetch the message... */
554                 StrBufPrintf(RecvMsg->IO.SendBuf.Buf,
555                              "RETR %ld\r\n", RecvMsg->CurrMsg->MSGID);
556                 POP3C_DBG_SEND();
557                 return eReadMessage;
558         }
559         else {
560                 RecvMsg->State = ReadQuitState;
561                 return POP3_C_DispatchWriteDone(&RecvMsg->IO);
562         }
563 }
564
565
566 eNextState POP3C_ReadMessageBodyFollowing(pop3aggr *RecvMsg)
567 {
568         AsyncIO *IO = &RecvMsg->IO;
569         POP3C_DBG_READ();
570         if (!POP3C_OK) return eTerminateConnection;
571         RecvMsg->IO.ReadMsg = NewAsyncMsg(HKEY("."),
572                                           RecvMsg->CurrMsg->MSGSize,
573                                           config.c_maxmsglen,
574                                           NULL, -1,
575                                           1);
576
577         return eReadPayload;
578 }
579
580
581 eNextState POP3C_StoreMsgRead(AsyncIO *IO)
582 {
583         pop3aggr *RecvMsg = (pop3aggr *) IO->Data;
584
585         SetPOP3State(IO, eStoreMsg);
586
587         EVP3CCS_syslog(LOG_DEBUG,
588                        "MARKING: %s as seen: ",
589                        ChrPtr(RecvMsg->CurrMsg->MsgUID));
590         CheckIfAlreadySeen("POP3 Item Seen",
591                            RecvMsg->CurrMsg->MsgUID,
592                            IO->Now,
593                            IO->Now, //// todo
594                            eWrite,
595                            IO->ID, CCID);
596
597         return DBQueueEventContext(&RecvMsg->IO, POP3_C_ReAttachToFetchMessages);
598 }
599 eNextState POP3C_SaveMsg(AsyncIO *IO)
600 {
601         long msgnum;
602         pop3aggr *RecvMsg = (pop3aggr *) IO->Data;
603
604         /* Do Something With It (tm) */
605         msgnum = CtdlSubmitMsg(RecvMsg->CurrMsg->Msg,
606                                NULL,
607                                ChrPtr(RecvMsg->RoomName),
608                                0);
609         if (msgnum > 0L)
610         {
611                 /* Message has been committed to the store
612                  * write the uidl to the use table
613                  * so we don't fetch this message again
614                  */
615         }
616         CtdlFreeMessage(RecvMsg->CurrMsg->Msg);
617
618         RecvMsg->count ++;
619         return NextDBOperation(&RecvMsg->IO, POP3C_StoreMsgRead);
620 }
621
622 eNextState POP3C_ReadMessageBody(pop3aggr *RecvMsg)
623 {
624         AsyncIO *IO = &RecvMsg->IO;
625         EVP3CM_syslog(LOG_DEBUG, "Converting message...");
626         RecvMsg->CurrMsg->Msg =
627                 convert_internet_message_buf(&RecvMsg->IO.ReadMsg->MsgBuf);
628         return EventQueueDBOperation(&RecvMsg->IO, POP3C_SaveMsg);
629 }
630
631 eNextState POP3C_SendDelete(pop3aggr *RecvMsg)
632 {
633         AsyncIO *IO = &RecvMsg->IO;
634
635         SetPOP3State(IO, eDelete);
636
637         if (!RecvMsg->keep) {
638                 StrBufPrintf(RecvMsg->IO.SendBuf.Buf,
639                              "DELE %ld\r\n", RecvMsg->CurrMsg->MSGID);
640                 POP3C_DBG_SEND();
641                 return eReadMessage;
642         }
643         else {
644                 RecvMsg->State = ReadMessageBodyFollowing;
645                 return POP3_C_DispatchWriteDone(&RecvMsg->IO);
646         }
647 }
648 eNextState POP3C_ReadDeleteState(pop3aggr *RecvMsg)
649 {
650         AsyncIO *IO = &RecvMsg->IO;
651         POP3C_DBG_READ();
652         RecvMsg->State = GetOneMessageIDState;
653         return eReadMessage;
654 }
655
656 eNextState POP3C_SendQuit(pop3aggr *RecvMsg)
657 {
658         AsyncIO *IO = &RecvMsg->IO;
659         SetPOP3State(IO, eQuit);
660
661         /* Log out */
662         StrBufPlain(RecvMsg->IO.SendBuf.Buf,
663                     HKEY("QUIT\r\n3)"));
664         POP3C_DBG_SEND();
665         return eReadMessage;
666 }
667
668
669 eNextState POP3C_ReadQuitState(pop3aggr *RecvMsg)
670 {
671         AsyncIO *IO = &RecvMsg->IO;
672         POP3C_DBG_READ();
673         return eTerminateConnection;
674 }
675
676 const long POP3_C_ConnTimeout = 1000;
677 const long DefaultPOP3Port = 110;
678
679 Pop3ClientHandler POP3C_ReadHandlers[] = {
680         POP3C_ReadGreeting,
681         POP3C_GetUserState,
682         POP3C_GetPassState,
683         POP3C_GetListCommandState,
684         POP3C_GetListOneLine,
685         POP3C_GetOneMessageIDState,
686         POP3C_ReadMessageBodyFollowing,
687         POP3C_ReadMessageBody,
688         POP3C_ReadDeleteState,
689         POP3C_ReadQuitState,
690 };
691
692 const long POP3_C_SendTimeouts[POP3C_MaxRead] = {
693         100,
694         100,
695         100,
696         100,
697         100,
698         100,
699         100,
700         100
701 };
702 const ConstStr POP3C_ReadErrors[POP3C_MaxRead] = {
703         {HKEY("Connection broken during ")},
704         {HKEY("Connection broken during ")},
705         {HKEY("Connection broken during ")},
706         {HKEY("Connection broken during ")},
707         {HKEY("Connection broken during ")},
708         {HKEY("Connection broken during ")},
709         {HKEY("Connection broken during ")},
710         {HKEY("Connection broken during ")}
711 };
712
713 Pop3ClientHandler POP3C_SendHandlers[] = {
714         NULL, /* we don't send a greeting */
715         POP3C_SendUser,
716         POP3C_SendPassword,
717         POP3C_SendListCommand,
718         NULL,
719         POP3C_GetOneMessagID,
720         POP3C_SendGetOneMsg,
721         NULL,
722         POP3C_SendDelete,
723         POP3C_SendQuit
724 };
725
726 const long POP3_C_ReadTimeouts[] = {
727         100,
728         100,
729         100,
730         100,
731         100,
732         100,
733         100,
734         100,
735         100,
736         100
737 };
738 /*****************************************************************************/
739 /*                     POP3 CLIENT DISPATCHER                                */
740 /*****************************************************************************/
741
742 void POP3SetTimeout(eNextState NextTCPState, pop3aggr *pMsg)
743 {
744         AsyncIO *IO = &pMsg->IO;
745         double Timeout = 0.0;
746
747         EVP3C_syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
748
749         switch (NextTCPState) {
750         case eSendFile:
751         case eSendReply:
752         case eSendMore:
753                 Timeout = POP3_C_SendTimeouts[pMsg->State];
754 /*
755   if (pMsg->State == eDATABody) {
756   / * if we're sending a huge message, we need more time. * /
757   Timeout += StrLength(pMsg->msgtext) / 1024;
758   }
759 */
760                 break;
761         case eReadFile:
762         case eReadMessage:
763                 Timeout = POP3_C_ReadTimeouts[pMsg->State];
764 /*
765   if (pMsg->State == eDATATerminateBody) {
766   / *
767   * some mailservers take a nap before accepting the message
768   * content inspection and such.
769   * /
770   Timeout += StrLength(pMsg->msgtext) / 1024;
771   }
772 */
773                 break;
774         case eReadPayload:
775                 Timeout = 100000;
776                 /* TODO!!! */
777                 break;
778         case eSendDNSQuery:
779         case eReadDNSReply:
780         case eConnect:
781         case eTerminateConnection:
782         case eDBQuery:
783         case eAbort:
784         case eReadMore://// TODO
785                 return;
786         }
787         SetNextTimeout(&pMsg->IO, Timeout);
788 }
789 eNextState POP3_C_DispatchReadDone(AsyncIO *IO)
790 {
791 /*      EVP3CCS_syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__); to noisy anyways. */
792         pop3aggr *pMsg = IO->Data;
793         eNextState rc;
794
795         rc = POP3C_ReadHandlers[pMsg->State](pMsg);
796         if (rc != eReadMore)
797             pMsg->State++;
798         POP3SetTimeout(rc, pMsg);
799         return rc;
800 }
801 eNextState POP3_C_DispatchWriteDone(AsyncIO *IO)
802 {
803         pop3aggr *pMsg = IO->Data;
804         eNextState rc;
805
806 /*      EVP3CCS_syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__); to noisy anyways. */
807         rc = POP3C_SendHandlers[pMsg->State](pMsg);
808         POP3SetTimeout(rc, pMsg);
809         return rc;
810 }
811
812
813 /*****************************************************************************/
814 /*                     POP3 CLIENT ERROR CATCHERS                            */
815 /*****************************************************************************/
816 eNextState POP3_C_Terminate(AsyncIO *IO)
817 {
818 ///     pop3aggr *pMsg = (pop3aggr *)IO->Data;
819
820         EVP3CCS_syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
821         FinalizePOP3AggrRun(IO);
822         return eAbort;
823 }
824 eNextState POP3_C_TerminateDB(AsyncIO *IO)
825 {
826 ///     pop3aggr *pMsg = (pop3aggr *)IO->Data;
827
828         EVP3CCS_syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
829         FinalizePOP3AggrRun(IO);
830         return eAbort;
831 }
832 eNextState POP3_C_Timeout(AsyncIO *IO)
833 {
834         pop3aggr *pMsg = IO->Data;
835
836         EVP3CCS_syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
837         StrBufPlain(IO->ErrMsg, CKEY(POP3C_ReadErrors[pMsg->State]));
838         return FailAggregationRun(IO);
839 }
840 eNextState POP3_C_ConnFail(AsyncIO *IO)
841 {
842         pop3aggr *pMsg = (pop3aggr *)IO->Data;
843
844         EVP3CCS_syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
845         StrBufPlain(IO->ErrMsg, CKEY(POP3C_ReadErrors[pMsg->State]));
846         return FailAggregationRun(IO);
847 }
848 eNextState POP3_C_DNSFail(AsyncIO *IO)
849 {
850         pop3aggr *pMsg = (pop3aggr *)IO->Data;
851
852         EVP3CCS_syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
853         StrBufPlain(IO->ErrMsg, CKEY(POP3C_ReadErrors[pMsg->State]));
854         return FailAggregationRun(IO);
855 }
856 eNextState POP3_C_Shutdown(AsyncIO *IO)
857 {
858         EVP3CCS_syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
859 ////    pop3aggr *pMsg = IO->Data;
860
861 ////pMsg->MyQEntry->Status = 3;
862 ///StrBufPlain(pMsg->MyQEntry->StatusMessage, HKEY("server shutdown during message retrieval."));
863         FinalizePOP3AggrRun(IO);
864         return eAbort;
865 }
866
867
868 /**
869  * @brief lineread Handler; understands when to read more POP3 lines,
870  *   and when this is a one-lined reply.
871  */
872 eReadState POP3_C_ReadServerStatus(AsyncIO *IO)
873 {
874         eReadState Finished = eBufferNotEmpty;
875
876         switch (IO->NextState) {
877         case eSendDNSQuery:
878         case eReadDNSReply:
879         case eDBQuery:
880         case eConnect:
881         case eTerminateConnection:
882         case eAbort:
883                 Finished = eReadFail;
884                 break;
885         case eSendFile:
886         case eSendReply:
887         case eSendMore:
888         case eReadMore:
889         case eReadMessage:
890                 Finished = StrBufChunkSipLine(IO->IOBuf, &IO->RecvBuf);
891                 break;
892         case eReadFile:
893         case eReadPayload:
894                 Finished = CtdlReadMessageBodyAsync(IO);
895                 break;
896         }
897         return Finished;
898 }
899
900 /*****************************************************************************
901  * So we connect our Server IP here.                                         *
902  *****************************************************************************/
903 eNextState POP3_C_ReAttachToFetchMessages(AsyncIO *IO)
904 {
905         pop3aggr *cpptr = IO->Data;
906
907         EVP3CCS_syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
908 ////??? cpptr->State ++;
909         if (cpptr->Pos == NULL)
910                 cpptr->Pos = GetNewHashPos(cpptr->MsgNumbers, 0);
911
912         POP3_C_DispatchWriteDone(IO);
913         ReAttachIO(IO, cpptr, 0);
914         IO->NextState = eReadMessage;
915         return IO->NextState;
916 }
917
918 eNextState pop3_connect_ip(AsyncIO *IO)
919 {
920         pop3aggr *cpptr = IO->Data;
921
922         if (cpptr->IOStart == 0.0) /* whith or without DNS? */
923                 cpptr->IOStart = IO->Now;
924
925         EVP3CCS_syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
926
927         return EvConnectSock(IO,
928                              POP3_C_ConnTimeout,
929                              POP3_C_ReadTimeouts[0],
930                              1);
931 }
932
933 eNextState pop3_get_one_host_ip_done(AsyncIO *IO)
934 {
935         pop3aggr *cpptr = IO->Data;
936         struct hostent *hostent;
937
938         QueryCbDone(IO);
939
940         hostent = cpptr->HostLookup.VParsedDNSReply;
941         if ((cpptr->HostLookup.DNSStatus == ARES_SUCCESS) && 
942             (hostent != NULL) ) {
943                 memset(&cpptr->IO.ConnectMe->Addr, 0, sizeof(struct in6_addr));
944                 if (cpptr->IO.ConnectMe->IPv6) {
945                         memcpy(&cpptr->IO.ConnectMe->Addr.sin6_addr.s6_addr, 
946                                &hostent->h_addr_list[0],
947                                sizeof(struct in6_addr));
948
949                         cpptr->IO.ConnectMe->Addr.sin6_family =
950                                 hostent->h_addrtype;
951                         cpptr->IO.ConnectMe->Addr.sin6_port   =
952                                 htons(DefaultPOP3Port);
953                 }
954                 else {
955                         struct sockaddr_in *addr =
956                                 (struct sockaddr_in*)
957                                 &cpptr->IO.ConnectMe->Addr;
958
959                         memcpy(&addr->sin_addr.s_addr,
960                                hostent->h_addr_list[0],
961                                sizeof(uint32_t));
962
963                         addr->sin_family = hostent->h_addrtype;
964                         addr->sin_port   = htons(DefaultPOP3Port);
965                 }
966                 return pop3_connect_ip(IO);
967         }
968         else
969                 return eAbort;
970 }
971
972 eNextState pop3_get_one_host_ip(AsyncIO *IO)
973 {
974         pop3aggr *cpptr = IO->Data;
975
976         cpptr->IOStart = IO->Now;
977
978         EVP3CCS_syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
979
980         EVP3CCS_syslog(LOG_DEBUG, 
981                        "POP3 client[%ld]: looking up %s-Record %s : %d ...\n",
982                        cpptr->n,
983                        (cpptr->IO.ConnectMe->IPv6)? "aaaa": "a",
984                        cpptr->IO.ConnectMe->Host,
985                        cpptr->IO.ConnectMe->Port);
986
987         QueueQuery((cpptr->IO.ConnectMe->IPv6)? ns_t_aaaa : ns_t_a,
988                    cpptr->IO.ConnectMe->Host,
989                    &cpptr->IO,
990                    &cpptr->HostLookup,
991                    pop3_get_one_host_ip_done);
992         IO->NextState = eReadDNSReply;
993         return IO->NextState;
994 }
995
996
997
998 int pop3_do_fetching(pop3aggr *cpptr)
999 {
1000         AsyncIO *IO = &cpptr->IO;
1001
1002         InitIOStruct(IO,
1003                      cpptr,
1004                      eReadMessage,
1005                      POP3_C_ReadServerStatus,
1006                      POP3_C_DNSFail,
1007                      POP3_C_DispatchWriteDone,
1008                      POP3_C_DispatchReadDone,
1009                      POP3_C_Terminate,
1010                      POP3_C_TerminateDB,
1011                      POP3_C_ConnFail,
1012                      POP3_C_Timeout,
1013                      POP3_C_Shutdown);
1014
1015         safestrncpy(((CitContext *)cpptr->IO.CitContext)->cs_host,
1016                     ChrPtr(cpptr->Url),
1017                     sizeof(((CitContext *)cpptr->IO.CitContext)->cs_host));
1018
1019         if (cpptr->IO.ConnectMe->IsIP) {
1020                 QueueEventContext(&cpptr->IO,
1021                                   pop3_connect_ip);
1022         }
1023         else {
1024                 QueueEventContext(&cpptr->IO,
1025                                   pop3_get_one_host_ip);
1026         }
1027         return 1;
1028 }
1029
1030 /*
1031  * Scan a room's netconfig to determine whether it requires POP3 aggregation
1032  */
1033 void pop3client_scan_room(struct ctdlroom *qrbuf, void *data, OneRoomNetCfg *OneRNCFG)
1034 {
1035         const RoomNetCfgLine *pLine;
1036         void *vptr;
1037
1038         pthread_mutex_lock(&POP3QueueMutex);
1039         if (GetHash(POP3QueueRooms, LKEY(qrbuf->QRnumber), &vptr))
1040         {
1041                 pthread_mutex_unlock(&POP3QueueMutex);
1042                 EVP3CQ_syslog(LOG_DEBUG,
1043                               "pop3client: [%ld] %s already in progress.",
1044                               qrbuf->QRnumber,
1045                               qrbuf->QRname);
1046                 return;
1047         }
1048         pthread_mutex_unlock(&POP3QueueMutex);
1049
1050         if (server_shutting_down) return;
1051
1052         pLine = OneRNCFG->NetConfigs[pop3client];
1053
1054         while (pLine != NULL)
1055         {
1056                 pop3aggr *cptr;
1057
1058                 cptr = (pop3aggr *) malloc(sizeof(pop3aggr));
1059                 memset(cptr, 0, sizeof(pop3aggr));
1060                 ///TODO do we need this? cptr->roomlist_parts=1;
1061                 cptr->RoomName = NewStrBufPlain(qrbuf->QRname, -1);
1062                 cptr->pop3user = NewStrBufDup(pLine->Value[1]);
1063                 cptr->pop3pass = NewStrBufDup(pLine->Value[2]);
1064                 cptr->Url = NewStrBuf();
1065                 cptr->Host = NewStrBufDup(pLine->Value[0]);
1066
1067                 cptr->keep = atol(ChrPtr(pLine->Value[3]));
1068                 cptr->interval = atol(ChrPtr(pLine->Value[4]));
1069
1070                 StrBufAppendBufPlain(cptr->Url, HKEY("pop3://"), 0);
1071                 StrBufUrlescUPAppend(cptr->Url, cptr->pop3user, NULL);
1072                 StrBufAppendBufPlain(cptr->Url, HKEY(":"), 0);
1073                 StrBufUrlescUPAppend(cptr->Url, cptr->pop3pass, NULL);
1074                 StrBufAppendBufPlain(cptr->Url, HKEY("@"), 0);
1075                 StrBufAppendBuf(cptr->Url, cptr->Host, 0);
1076                 StrBufAppendBufPlain(cptr->Url, HKEY("/"), 0);
1077                 StrBufUrlescAppend(cptr->Url, cptr->RoomName, NULL);
1078
1079                 ParseURL(&cptr->IO.ConnectMe, cptr->Url, 110);
1080
1081
1082 #if 0
1083 /* todo: we need to reunite the url to be shure. */
1084
1085                 pthread_mutex_lock(&POP3ueueMutex);
1086                 GetHash(POP3FetchUrls, SKEY(ptr->Url), &vptr);
1087                 use_this_cptr = (pop3aggr *)vptr;
1088
1089                 if (use_this_rncptr != NULL)
1090                 {
1091                         /* mustn't attach to an active session */
1092                         if (use_this_cptr->RefCount > 0)
1093                         {
1094                                 DeletePOP3Cfg(cptr);
1095 ///                                             Count->count--;
1096                         }
1097                         else
1098                         {
1099                                 long *QRnumber;
1100                                 StrBufAppendBufPlain(
1101                                         use_this_cptr->rooms,
1102                                         qrbuf->QRname,
1103                                         -1, 0);
1104                                 if (use_this_cptr->roomlist_parts == 1)
1105                                 {
1106                                         use_this_cptr->OtherQRnumbers
1107                                                 = NewHash(1, lFlathash);
1108                                 }
1109                                 QRnumber = (long*)malloc(sizeof(long));
1110                                 *QRnumber = qrbuf->QRnumber;
1111                                 Put(use_this_cptr->OtherQRnumbers,
1112                                     LKEY(qrbuf->QRnumber),
1113                                     QRnumber,
1114                                     NULL);
1115
1116                                 use_this_cptr->roomlist_parts++;
1117                         }
1118                         pthread_mutex_unlock(&POP3QueueMutex);
1119                         continue;
1120                 }
1121                 pthread_mutex_unlock(&RSSQueueMutex);
1122 #endif
1123                 cptr->n = Pop3ClientID++;
1124                 pthread_mutex_lock(&POP3QueueMutex);
1125                 Put(POP3FetchUrls,
1126                     SKEY(cptr->Url),
1127                     cptr,
1128                     DeletePOP3Aggregator);
1129
1130                 pthread_mutex_unlock(&POP3QueueMutex);
1131                 pLine = pLine->next;
1132
1133         }
1134 }
1135
1136 static int doing_pop3client = 0;
1137
1138 void pop3client_scan(void) {
1139         static time_t last_run = 0L;
1140         time_t fastest_scan;
1141         HashPos *it;
1142         long len;
1143         const char *Key;
1144         void *vrptr;
1145         pop3aggr *cptr;
1146
1147         become_session(&pop3_client_CC);
1148
1149         if (config.c_pop3_fastest < config.c_pop3_fetch)
1150                 fastest_scan = config.c_pop3_fastest;
1151         else
1152                 fastest_scan = config.c_pop3_fetch;
1153
1154         /*
1155          * Run POP3 aggregation no more frequently than once every n seconds
1156          */
1157         if ( (time(NULL) - last_run) < fastest_scan ) {
1158                 return;
1159         }
1160
1161         /*
1162          * This is a simple concurrency check to make sure only one pop3client
1163          * run is done at a time.  We could do this with a mutex, but since we
1164          * don't really require extremely fine granularity here, we'll do it
1165          * with a static variable instead.
1166          */
1167         if (doing_pop3client) return;
1168         doing_pop3client = 1;
1169
1170         EVP3CQM_syslog(LOG_DEBUG, "pop3client started");
1171         CtdlForEachNetCfgRoom(pop3client_scan_room, NULL, pop3client);
1172
1173         pthread_mutex_lock(&POP3QueueMutex);
1174         it = GetNewHashPos(POP3FetchUrls, 0);
1175         while (!server_shutting_down &&
1176                GetNextHashPos(POP3FetchUrls, it, &len, &Key, &vrptr) &&
1177                (vrptr != NULL)) {
1178                 cptr = (pop3aggr *)vrptr;
1179                 if (cptr->RefCount == 0)
1180                         if (!pop3_do_fetching(cptr))
1181                                 DeletePOP3Aggregator(cptr);////TODO
1182
1183 /*
1184         if ((palist->interval && time(NULL) > (last_run + palist->interval))
1185                         || (time(NULL) > last_run + config.c_pop3_fetch))
1186                         pop3_do_fetching(palist->roomname, palist->pop3host,
1187                         palist->pop3user, palist->pop3pass, palist->keep);
1188                 pptr = palist;
1189                 palist = palist->next;
1190                 free(pptr);
1191 */
1192         }
1193         DeleteHashPos(&it);
1194         pthread_mutex_unlock(&POP3QueueMutex);
1195
1196         EVP3CQM_syslog(LOG_DEBUG, "pop3client ended");
1197         last_run = time(NULL);
1198         doing_pop3client = 0;
1199 }
1200
1201
1202 void pop3_cleanup(void)
1203 {
1204         /* citthread_mutex_destroy(&POP3QueueMutex); TODO */
1205         while (doing_pop3client != 0) ;
1206         DeleteHash(&POP3FetchUrls);
1207         DeleteHash(&POP3QueueRooms);
1208 }
1209
1210
1211
1212 void LogDebugEnablePOP3Client(const int n)
1213 {
1214         POP3ClientDebugEnabled = n;
1215 }
1216
1217 CTDL_MODULE_INIT(pop3client)
1218 {
1219         if (!threading)
1220         {
1221                 CtdlFillSystemContext(&pop3_client_CC, "POP3aggr");
1222                 CtdlREGISTERRoomCfgType(pop3client, ParseGeneric, 0, 5, SerializeGeneric, DeleteGenericCfgLine);
1223                 pthread_mutex_init(&POP3QueueMutex, NULL);
1224                 POP3QueueRooms = NewHash(1, lFlathash);
1225                 POP3FetchUrls = NewHash(1, NULL);
1226                 CtdlRegisterSessionHook(pop3client_scan, EVT_TIMER, PRIO_AGGR + 50);
1227                 CtdlRegisterEVCleanupHook(pop3_cleanup);
1228                 CtdlRegisterDebugFlagHook(HKEY("pop3client"), LogDebugEnablePOP3Client, &POP3ClientDebugEnabled);
1229         }
1230
1231         /* return our module id for the log */
1232         return "pop3client";
1233 }