citadel/modules/pop3client/serv_pop3client.c

   1 /*
   2  * Consolidate mail from remote POP3 accounts.
   3  *
   4  * Copyright (c) 2007-2011 by the citadel.org team
   5  *
   6  * This program is open source software; you can redistribute it and/or
   7  * modify it under the terms of the GNU General Public License as published
   8  * by the Free Software Foundation; either version 3 of the License, or
   9  * (at your option) any later version.
  10  *
  11  * This program is distributed in the hope that it will be useful,
  12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14  * GNU General Public License for more details.
  15  *
  16  * You should have received a copy of the GNU General Public License
  17  * along with this program; if not, write to the Free Software
  18  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
  19  */
  20
  21 #include <stdlib.h>
  22 #include <unistd.h>
  23 #include <stdio.h>
  24
  25 #if TIME_WITH_SYS_TIME
  26 # include <sys/time.h>
  27 # include <time.h>
  28 #else
  29 # if HAVE_SYS_TIME_H
  30 #  include <sys/time.h>
  31 # else
  32 #  include <time.h>
  33 # endif
  34 #endif
  35
  36 #include <ctype.h>
  37 #include <string.h>
  38 #include <errno.h>
  39 #include <sys/types.h>
  40 #include <sys/stat.h>
  41 #include <libcitadel.h>
  42 #include "citadel.h"
  43 #include "server.h"
  44 #include "citserver.h"
  45 #include "support.h"
  46 #include "config.h"
  47 #include "ctdl_module.h"
  48 #include "clientsocket.h"
  49 #include "msgbase.h"
  50 #include "internet_addressing.h"
  51 #include "database.h"
  52 #include "citadel_dirs.h"
  53 #include "event_client.h"
  54
  55
  56 #define POP3C_OK (strncasecmp(ChrPtr(RecvMsg->IO.IOBuf), "+OK", 3) == 0)
  57 int Pop3ClientID = 0;
  58 int POP3ClientDebugEnabled = 0;
  59
  60 #define N ((pop3aggr*)IO->Data)->n
  61
  62 #define DBGLOG(LEVEL) if ((LEVEL != LOG_DEBUG) || (POP3ClientDebugEnabled != 0))
  63
  64 #define EVP3C_syslog(LEVEL, FORMAT, ...)                                \
  65         DBGLOG(LEVEL) syslog(LEVEL,                                     \
  66                              "IO[%ld]CC[%d][%ld]POP3: " FORMAT,         \
  67                              IO->ID, CCID, N, __VA_ARGS__)
  68
  69 #define EVP3CM_syslog(LEVEL, FORMAT)                                    \
  70         DBGLOG(LEVEL) syslog(LEVEL,                                     \
  71                              "IO[%ld]CC[%d][%ld]POP3: " FORMAT,         \
  72                              IO->ID, CCID, N)
  73
  74 #define EVP3CQ_syslog(LEVEL, FORMAT, ...)                               \
  75         DBGLOG(LEVEL) syslog(LEVEL,                                     \
  76                              "P3Q:" FORMAT,                             \
  77                              __VA_ARGS__)
  78
  79 #define EVP3CQM_syslog(LEVEL, FORMAT)                                   \
  80         DBGLOG(LEVEL) syslog(LEVEL,                                     \
  81                              "P3Q" FORMAT                               \
  82                 )
  83
  84 #define EVP3CCS_syslog(LEVEL, FORMAT, ...)                              \
  85         DBGLOG(LEVEL) syslog(LEVEL, "IO[%ld][%ld]POP3: " FORMAT,        \
  86                              IO->ID, N, __VA_ARGS__)
  87
  88 #define EVP3CCSM_syslog(LEVEL, FORMAT)                                  \
  89         DBGLOG(LEVEL) syslog(LEVEL, "IO[%ld][%ld]POP3: " FORMAT,        \
  90                              IO->ID, N)
  91
  92 #define POP3C_DBG_SEND()                                                \
  93         EVP3C_syslog(LOG_DEBUG,                                         \
  94                      "IO[%ld]CC[%d][%ld]POP3: > %s\n",                  \
  95                      IO->ID, CCID, N,                                   \
  96                      ChrPtr(RecvMsg->IO.SendBuf.Buf))
  97
  98 #define POP3C_DBG_READ()                                                \
  99         EVP3C_syslog(LOG_DEBUG,                                         \
 100                      "IO[%ld]CC[%d][%ld]POP3: < %s\n",                  \
 101                      IO->ID, CCID, N,                                   \
 102                      ChrPtr(RecvMsg->IO.IOBuf))
 103
 104
 105 struct CitContext pop3_client_CC;
 106
 107 pthread_mutex_t POP3QueueMutex; /* locks the access to the following vars: */
 108 HashList *POP3QueueRooms = NULL;
 109 HashList *POP3FetchUrls = NULL;
 110
 111 typedef struct pop3aggr pop3aggr;
 112 typedef eNextState(*Pop3ClientHandler)(pop3aggr* RecvMsg);
 113
 114 eNextState POP3_C_Shutdown(AsyncIO *IO);
 115 eNextState POP3_C_Timeout(AsyncIO *IO);
 116 eNextState POP3_C_ConnFail(AsyncIO *IO);
 117 eNextState POP3_C_DNSFail(AsyncIO *IO);
 118 eNextState POP3_C_DispatchReadDone(AsyncIO *IO);
 119 eNextState POP3_C_DispatchWriteDone(AsyncIO *IO);
 120 eNextState POP3_C_Terminate(AsyncIO *IO);
 121 eReadState POP3_C_ReadServerStatus(AsyncIO *IO);
 122 eNextState POP3_C_ReAttachToFetchMessages(AsyncIO *IO);
 123
 124 typedef struct __pop3_room_counter {
 125         int count;
 126         long QRnumber;
 127 }pop3_room_counter;
 128
 129 typedef enum ePOP3_C_States {
 130         ReadGreeting,
 131         GetUserState,
 132         GetPassState,
 133         GetListCommandState,
 134         GetListOneLine,
 135         GetOneMessageIDState,
 136         ReadMessageBodyFollowing,
 137         ReadMessageBody,
 138         GetDeleteState,
 139         ReadQuitState,
 140         POP3C_MaxRead
 141 }ePOP3_C_States;
 142
 143
 144 typedef struct _FetchItem {
 145         long MSGID;
 146         long MSGSize;
 147         StrBuf *MsgUIDL;
 148         StrBuf *MsgUID;
 149         int NeedFetch;
 150         struct CtdlMessage *Msg;
 151 } FetchItem;
 152
 153 void HfreeFetchItem(void *vItem)
 154 {
 155         FetchItem *Item = (FetchItem*) vItem;
 156         FreeStrBuf(&Item->MsgUIDL);
 157         FreeStrBuf(&Item->MsgUID);
 158         free(Item);
 159 }
 160
 161 struct pop3aggr {
 162         AsyncIO  IO;
 163
 164         long n;
 165         double IOStart;
 166         long count;
 167         long RefCount;
 168         DNSQueryParts HostLookup;
 169
 170         long             QRnumber;
 171         HashList        *OtherQRnumbers;
 172
 173         StrBuf          *Url;
 174         StrBuf *pop3user;
 175         StrBuf *pop3pass;
 176         StrBuf *Host;
 177         StrBuf *RoomName; // TODO: fill me
 178         int keep;
 179         time_t interval;
 180         ePOP3_C_States State;
 181         HashList *MsgNumbers;
 182         HashPos *Pos;
 183         FetchItem *CurrMsg;
 184 };
 185
 186 void DeletePOP3Aggregator(void *vptr)
 187 {
 188         pop3aggr *ptr = vptr;
 189         DeleteHashPos(&ptr->Pos);
 190         DeleteHash(&ptr->MsgNumbers);
 191 //      FreeStrBuf(&ptr->rooms);
 192         FreeStrBuf(&ptr->pop3user);
 193         FreeStrBuf(&ptr->pop3pass);
 194         FreeStrBuf(&ptr->Host);
 195         FreeStrBuf(&ptr->RoomName);
 196         FreeURL(&ptr->IO.ConnectMe);
 197         FreeStrBuf(&ptr->Url);
 198         FreeStrBuf(&ptr->IO.IOBuf);
 199         FreeStrBuf(&ptr->IO.SendBuf.Buf);
 200         FreeStrBuf(&ptr->IO.RecvBuf.Buf);
 201         DeleteAsyncMsg(&ptr->IO.ReadMsg);
 202         ((struct CitContext*)ptr->IO.CitContext)->state = CON_IDLE;
 203         ((struct CitContext*)ptr->IO.CitContext)->kill_me = 1;
 204         FreeAsyncIOContents(&ptr->IO);
 205         free(ptr);
 206 }
 207
 208 eNextState FinalizePOP3AggrRun(AsyncIO *IO)
 209 {
 210         HashPos  *It;
 211         pop3aggr *cpptr = (pop3aggr *)IO->Data;
 212
 213         EVP3C_syslog(LOG_INFO,
 214                      "%s@%s: fetched %ld new of %d messages in %fs. bye.",
 215                      ChrPtr(cpptr->pop3user),
 216                      ChrPtr(cpptr->Host),
 217                      cpptr->count,
 218                      GetCount(cpptr->MsgNumbers),
 219                      IO->Now - cpptr->IOStart
 220                 );
 221
 222         It = GetNewHashPos(POP3FetchUrls, 0);
 223         pthread_mutex_lock(&POP3QueueMutex);
 224         {
 225                 if (GetHashPosFromKey(POP3FetchUrls, SKEY(cpptr->Url), It))
 226                         DeleteEntryFromHash(POP3FetchUrls, It);
 227         }
 228         pthread_mutex_unlock(&POP3QueueMutex);
 229         DeleteHashPos(&It);
 230         return eAbort;
 231 }
 232
 233 eNextState FailAggregationRun(AsyncIO *IO)
 234 {
 235         return eAbort;
 236 }
 237
 238 eNextState POP3C_ReadGreeting(pop3aggr *RecvMsg)
 239 {
 240         AsyncIO *IO = &RecvMsg->IO;
 241         POP3C_DBG_READ();
 242         /* Read the server greeting */
 243         if (!POP3C_OK) return eTerminateConnection;
 244         else return eSendReply;
 245 }
 246
 247 eNextState POP3C_SendUser(pop3aggr *RecvMsg)
 248 {
 249         AsyncIO *IO = &RecvMsg->IO;
 250         /* Identify ourselves.  NOTE: we have to append a CR to each command.
 251          *  The LF will automatically be appended by sock_puts().  Believe it
 252          * or not, leaving out the CR will cause problems if the server happens
 253          * to be Exchange, which is so b0rken it actually barfs on
 254          * LF-terminated newlines.
 255          */
 256         StrBufPrintf(RecvMsg->IO.SendBuf.Buf,
 257                      "USER %s\r\n", ChrPtr(RecvMsg->pop3user));
 258         POP3C_DBG_SEND();
 259         return eReadMessage;
 260 }
 261
 262 eNextState POP3C_GetUserState(pop3aggr *RecvMsg)
 263 {
 264         AsyncIO *IO = &RecvMsg->IO;
 265         POP3C_DBG_READ();
 266         if (!POP3C_OK) return eTerminateConnection;
 267         else return eSendReply;
 268 }
 269
 270 eNextState POP3C_SendPassword(pop3aggr *RecvMsg)
 271 {
 272         AsyncIO *IO = &RecvMsg->IO;
 273         /* Password */
 274         StrBufPrintf(RecvMsg->IO.SendBuf.Buf,
 275                      "PASS %s\r\n", ChrPtr(RecvMsg->pop3pass));
 276         EVP3CM_syslog(LOG_DEBUG, "<PASS <password>\n");
 277 //      POP3C_DBG_SEND(); No, we won't write the passvoid to syslog...
 278         return eReadMessage;
 279 }
 280
 281 eNextState POP3C_GetPassState(pop3aggr *RecvMsg)
 282 {
 283         AsyncIO *IO = &RecvMsg->IO;
 284         POP3C_DBG_READ();
 285         if (!POP3C_OK) return eTerminateConnection;
 286         else return eSendReply;
 287 }
 288
 289 eNextState POP3C_SendListCommand(pop3aggr *RecvMsg)
 290 {
 291         AsyncIO *IO = &RecvMsg->IO;
 292         /* Get the list of messages */
 293         StrBufPlain(RecvMsg->IO.SendBuf.Buf, HKEY("LIST\r\n"));
 294         POP3C_DBG_SEND();
 295         return eReadMessage;
 296 }
 297
 298 eNextState POP3C_GetListCommandState(pop3aggr *RecvMsg)
 299 {
 300         AsyncIO *IO = &RecvMsg->IO;
 301         POP3C_DBG_READ();
 302         if (!POP3C_OK) return eTerminateConnection;
 303         RecvMsg->MsgNumbers = NewHash(1, NULL);
 304         RecvMsg->State++;
 305         return eReadMore;
 306 }
 307
 308
 309 eNextState POP3C_GetListOneLine(pop3aggr *RecvMsg)
 310 {
 311         AsyncIO *IO = &RecvMsg->IO;
 312 #if 0
 313         int rc;
 314 #endif
 315         const char *pch;
 316         FetchItem *OneMsg = NULL;
 317         POP3C_DBG_READ();
 318
 319         if ((StrLength(RecvMsg->IO.IOBuf) == 1) &&
 320             (ChrPtr(RecvMsg->IO.IOBuf)[0] == '.'))
 321         {
 322                 if (GetCount(RecvMsg->MsgNumbers) == 0)
 323                 {
 324                         ////    RecvMsg->Sate = ReadQuitState;
 325                 }
 326                 else
 327                 {
 328                         RecvMsg->Pos = GetNewHashPos(RecvMsg->MsgNumbers, 0);
 329                 }
 330                 return eSendReply;
 331
 332         }
 333         OneMsg = (FetchItem*) malloc(sizeof(FetchItem));
 334         memset(OneMsg, 0, sizeof(FetchItem));
 335         OneMsg->MSGID = atol(ChrPtr(RecvMsg->IO.IOBuf));
 336
 337         pch = strchr(ChrPtr(RecvMsg->IO.IOBuf), ' ');
 338         if (pch != NULL)
 339         {
 340                 OneMsg->MSGSize = atol(pch + 1);
 341         }
 342 #if 0
 343         rc = TestValidateHash(RecvMsg->MsgNumbers);
 344         if (rc != 0)
 345                 EVP3CCS_syslog(LOG_DEBUG, "Hash Invalid: %d\n", rc);
 346 #endif
 347
 348         Put(RecvMsg->MsgNumbers, LKEY(OneMsg->MSGID), OneMsg, HfreeFetchItem);
 349 #if 0
 350         rc = TestValidateHash(RecvMsg->MsgNumbers);
 351         if (rc != 0)
 352                 EVP3CCS_syslog(LOG_DEBUG, "Hash Invalid: %d\n", rc);
 353 #endif
 354         //RecvMsg->State --; /* read next Line */
 355         return eReadMore;
 356 }
 357
 358 eNextState POP3_FetchNetworkUsetableEntry(AsyncIO *IO)
 359 {
 360         long HKLen;
 361         const char *HKey;
 362         void *vData;
 363         struct cdbdata *cdbut;
 364         pop3aggr *RecvMsg = (pop3aggr *) IO->Data;
 365
 366         if((RecvMsg->Pos != NULL) &&
 367            GetNextHashPos(RecvMsg->MsgNumbers,
 368                           RecvMsg->Pos,
 369                           &HKLen,
 370                           &HKey,
 371                           &vData))
 372         {
 373                 struct UseTable ut;
 374                 if (server_shutting_down)
 375                         return eAbort;
 376
 377                 RecvMsg->CurrMsg = (FetchItem*) vData;
 378                 EVP3CCS_syslog(LOG_DEBUG,
 379                                "CHECKING: whether %s has already been seen: ",
 380                                ChrPtr(RecvMsg->CurrMsg->MsgUID));
 381
 382                 /* Find out if we've already seen this item */
 383                 safestrncpy(ut.ut_msgid,
 384                             ChrPtr(RecvMsg->CurrMsg->MsgUID),
 385                             sizeof(ut.ut_msgid));
 386                 ut.ut_timestamp = time(NULL);/// TODO: libev timestamp!
 387
 388                 cdbut = cdb_fetch(CDB_USETABLE, SKEY(RecvMsg->CurrMsg->MsgUID));
 389                 if (cdbut != NULL) {
 390                         /* Item has already been seen */
 391                         EVP3CCSM_syslog(LOG_DEBUG, "YES\n");
 392                         cdb_free(cdbut);
 393
 394                         /* rewrite the record anyway, to update the timestamp */
 395                         cdb_store(CDB_USETABLE,
 396                                   SKEY(RecvMsg->CurrMsg->MsgUID),
 397                                   &ut, sizeof(struct UseTable) );
 398                         RecvMsg->CurrMsg->NeedFetch = 0; ////TODO0;
 399                 }
 400                 else
 401                 {
 402                         EVP3CCSM_syslog(LOG_DEBUG, "NO\n");
 403                         RecvMsg->CurrMsg->NeedFetch = 1;
 404                 }
 405                 return NextDBOperation(&RecvMsg->IO,
 406                                        POP3_FetchNetworkUsetableEntry);
 407         }
 408         else
 409         {
 410                 /* ok, now we know them all,
 411                  * continue with reading the actual messages. */
 412                 DeleteHashPos(&RecvMsg->Pos);
 413                 StopDBWatchers(IO);
 414                 return QueueEventContext(IO, POP3_C_ReAttachToFetchMessages);
 415         }
 416 }
 417
 418 eNextState POP3C_GetOneMessagID(pop3aggr *RecvMsg)
 419 {
 420         AsyncIO *IO = &RecvMsg->IO;
 421         long HKLen;
 422         const char *HKey;
 423         void *vData;
 424
 425 #if 0
 426         int rc;
 427         rc = TestValidateHash(RecvMsg->MsgNumbers);
 428         if (rc != 0)
 429                 EVP3CCS_syslog(LOG_DEBUG, "Hash Invalid: %d\n", rc);
 430 #endif
 431         if((RecvMsg->Pos != NULL) &&
 432            GetNextHashPos(RecvMsg->MsgNumbers,
 433                           RecvMsg->Pos,
 434                           &HKLen, &HKey,
 435                           &vData))
 436         {
 437                 RecvMsg->CurrMsg = (FetchItem*) vData;
 438                 /* Find out the UIDL of the message,
 439                  * to determine whether we've already downloaded it */
 440                 StrBufPrintf(RecvMsg->IO.SendBuf.Buf,
 441                              "UIDL %ld\r\n", RecvMsg->CurrMsg->MSGID);
 442                 POP3C_DBG_SEND();
 443         }
 444         else
 445         {
 446                 RecvMsg->State++;
 447                 DeleteHashPos(&RecvMsg->Pos);
 448                 /// done receiving uidls.. start looking them up now.
 449                 RecvMsg->Pos = GetNewHashPos(RecvMsg->MsgNumbers, 0);
 450                 return QueueDBOperation(&RecvMsg->IO,
 451                                         POP3_FetchNetworkUsetableEntry);
 452         }
 453         return eReadMore; /* TODO */
 454 }
 455
 456 eNextState POP3C_GetOneMessageIDState(pop3aggr *RecvMsg)
 457 {
 458         AsyncIO *IO = &RecvMsg->IO;
 459 #if 0
 460         int rc;
 461         rc = TestValidateHash(RecvMsg->MsgNumbers);
 462         if (rc != 0)
 463                 EVP3CCS_syslog(LOG_DEBUG, "Hash Invalid: %d\n", rc);
 464 #endif
 465
 466         POP3C_DBG_READ();
 467         if (!POP3C_OK) return eTerminateConnection;
 468         RecvMsg->CurrMsg->MsgUIDL =
 469                 NewStrBufPlain(NULL, StrLength(RecvMsg->IO.IOBuf));
 470         RecvMsg->CurrMsg->MsgUID =
 471                 NewStrBufPlain(NULL, StrLength(RecvMsg->IO.IOBuf) * 2);
 472
 473         StrBufExtract_token(RecvMsg->CurrMsg->MsgUIDL,
 474                             RecvMsg->IO.IOBuf, 2, ' ');
 475
 476         StrBufPrintf(RecvMsg->CurrMsg->MsgUID,
 477                      "pop3/%s/%s:%s@%s",
 478                      ChrPtr(RecvMsg->RoomName),
 479                      ChrPtr(RecvMsg->CurrMsg->MsgUIDL),
 480                      RecvMsg->IO.ConnectMe->User,
 481                      RecvMsg->IO.ConnectMe->Host);
 482         RecvMsg->State --;
 483         return eSendReply;
 484 }
 485
 486
 487 eNextState POP3C_SendGetOneMsg(pop3aggr *RecvMsg)
 488 {
 489         AsyncIO *IO = &RecvMsg->IO;
 490         long HKLen;
 491         const char *HKey;
 492         void *vData;
 493
 494         RecvMsg->CurrMsg = NULL;
 495         while ((RecvMsg->Pos != NULL) &&
 496                GetNextHashPos(RecvMsg->MsgNumbers,
 497                               RecvMsg->Pos,
 498                               &HKLen, &HKey,
 499                               &vData) &&
 500                (RecvMsg->CurrMsg = (FetchItem*) vData,
 501                 RecvMsg->CurrMsg->NeedFetch == 0))
 502         {}
 503
 504         if ((RecvMsg->CurrMsg != NULL ) && (RecvMsg->CurrMsg->NeedFetch == 1))
 505         {
 506                 /* Message has not been seen.
 507                  * Tell the server to fetch the message... */
 508                 StrBufPrintf(RecvMsg->IO.SendBuf.Buf,
 509                              "RETR %ld\r\n", RecvMsg->CurrMsg->MSGID);
 510                 POP3C_DBG_SEND();
 511                 return eReadMessage;
 512         }
 513         else {
 514                 RecvMsg->State = ReadQuitState;
 515                 return POP3_C_DispatchWriteDone(&RecvMsg->IO);
 516         }
 517 }
 518
 519
 520 eNextState POP3C_ReadMessageBodyFollowing(pop3aggr *RecvMsg)
 521 {
 522         AsyncIO *IO = &RecvMsg->IO;
 523         POP3C_DBG_READ();
 524         if (!POP3C_OK) return eTerminateConnection;
 525         RecvMsg->IO.ReadMsg = NewAsyncMsg(HKEY("."),
 526                                           RecvMsg->CurrMsg->MSGSize,
 527                                           config.c_maxmsglen,
 528                                           NULL, -1,
 529                                           1);
 530
 531         return eReadPayload;
 532 }
 533
 534
 535 eNextState POP3C_StoreMsgRead(AsyncIO *IO)
 536 {
 537         pop3aggr *RecvMsg = (pop3aggr *) IO->Data;
 538         struct UseTable ut;
 539
 540         EVP3CCS_syslog(LOG_DEBUG,
 541                        "MARKING: %s as seen: ",
 542                        ChrPtr(RecvMsg->CurrMsg->MsgUID));
 543
 544         safestrncpy(ut.ut_msgid,
 545                     ChrPtr(RecvMsg->CurrMsg->MsgUID),
 546                     sizeof(ut.ut_msgid));
 547         ut.ut_timestamp = time(NULL); /* TODO: use libev time */
 548         cdb_store(CDB_USETABLE,
 549                   ChrPtr(RecvMsg->CurrMsg->MsgUID),
 550                   StrLength(RecvMsg->CurrMsg->MsgUID),
 551                   &ut,
 552                   sizeof(struct UseTable) );
 553         StopDBWatchers(IO);
 554         return QueueEventContext(&RecvMsg->IO, POP3_C_ReAttachToFetchMessages);
 555 }
 556 eNextState POP3C_SaveMsg(AsyncIO *IO)
 557 {
 558         long msgnum;
 559         pop3aggr *RecvMsg = (pop3aggr *) IO->Data;
 560
 561         /* Do Something With It (tm) */
 562         msgnum = CtdlSubmitMsg(RecvMsg->CurrMsg->Msg,
 563                                NULL,
 564                                ChrPtr(RecvMsg->RoomName),
 565                                0);
 566         if (msgnum > 0L)
 567         {
 568                 /* Message has been committed to the store
 569                  * write the uidl to the use table
 570                  * so we don't fetch this message again
 571                  */
 572         }
 573         CtdlFreeMessage(RecvMsg->CurrMsg->Msg);
 574
 575         RecvMsg->count ++;
 576         return NextDBOperation(&RecvMsg->IO, POP3C_StoreMsgRead);
 577 }
 578
 579 eNextState POP3C_ReadMessageBody(pop3aggr *RecvMsg)
 580 {
 581         AsyncIO *IO = &RecvMsg->IO;
 582         EVP3CM_syslog(LOG_DEBUG, "Converting message...");
 583         RecvMsg->CurrMsg->Msg =
 584                 convert_internet_message_buf(&RecvMsg->IO.ReadMsg->MsgBuf);
 585         StopClientWatchers(IO, 0);
 586         return QueueDBOperation(&RecvMsg->IO, POP3C_SaveMsg);
 587 }
 588
 589 eNextState POP3C_SendDelete(pop3aggr *RecvMsg)
 590 {
 591         AsyncIO *IO = &RecvMsg->IO;
 592         if (!RecvMsg->keep) {
 593                 StrBufPrintf(RecvMsg->IO.SendBuf.Buf,
 594                              "DELE %ld\r\n", RecvMsg->CurrMsg->MSGID);
 595                 POP3C_DBG_SEND();
 596                 return eReadMessage;
 597         }
 598         else {
 599                 RecvMsg->State = ReadMessageBodyFollowing;
 600                 return POP3_C_DispatchWriteDone(&RecvMsg->IO);
 601         }
 602 }
 603 eNextState POP3C_ReadDeleteState(pop3aggr *RecvMsg)
 604 {
 605         AsyncIO *IO = &RecvMsg->IO;
 606         POP3C_DBG_READ();
 607         RecvMsg->State = GetOneMessageIDState;
 608         return eReadMessage;
 609 }
 610
 611 eNextState POP3C_SendQuit(pop3aggr *RecvMsg)
 612 {
 613         AsyncIO *IO = &RecvMsg->IO;
 614         /* Log out */
 615         StrBufPlain(RecvMsg->IO.SendBuf.Buf,
 616                     HKEY("QUIT\r\n3)"));
 617         POP3C_DBG_SEND();
 618         return eReadMessage;
 619 }
 620
 621
 622 eNextState POP3C_ReadQuitState(pop3aggr *RecvMsg)
 623 {
 624         AsyncIO *IO = &RecvMsg->IO;
 625         POP3C_DBG_READ();
 626         return eTerminateConnection;
 627 }
 628
 629 const long POP3_C_ConnTimeout = 1000;
 630 const long DefaultPOP3Port = 110;
 631
 632 Pop3ClientHandler POP3C_ReadHandlers[] = {
 633         POP3C_ReadGreeting,
 634         POP3C_GetUserState,
 635         POP3C_GetPassState,
 636         POP3C_GetListCommandState,
 637         POP3C_GetListOneLine,
 638         POP3C_GetOneMessageIDState,
 639         POP3C_ReadMessageBodyFollowing,
 640         POP3C_ReadMessageBody,
 641         POP3C_ReadDeleteState,
 642         POP3C_ReadQuitState,
 643 };
 644
 645 const long POP3_C_SendTimeouts[POP3C_MaxRead] = {
 646         100,
 647         100,
 648         100,
 649         100,
 650         100,
 651         100,
 652         100,
 653         100
 654 };
 655 const ConstStr POP3C_ReadErrors[POP3C_MaxRead] = {
 656         {HKEY("Connection broken during ")},
 657         {HKEY("Connection broken during ")},
 658         {HKEY("Connection broken during ")},
 659         {HKEY("Connection broken during ")},
 660         {HKEY("Connection broken during ")},
 661         {HKEY("Connection broken during ")},
 662         {HKEY("Connection broken during ")},
 663         {HKEY("Connection broken during ")}
 664 };
 665
 666 Pop3ClientHandler POP3C_SendHandlers[] = {
 667         NULL, /* we don't send a greeting */
 668         POP3C_SendUser,
 669         POP3C_SendPassword,
 670         POP3C_SendListCommand,
 671         NULL,
 672         POP3C_GetOneMessagID,
 673         POP3C_SendGetOneMsg,
 674         NULL,
 675         POP3C_SendDelete,
 676         POP3C_SendQuit
 677 };
 678
 679 const long POP3_C_ReadTimeouts[] = {
 680         100,
 681         100,
 682         100,
 683         100,
 684         100,
 685         100,
 686         100,
 687         100,
 688         100,
 689         100
 690 };
 691 /*****************************************************************************/
 692 /*                     POP3 CLIENT DISPATCHER                                */
 693 /*****************************************************************************/
 694
 695 void POP3SetTimeout(eNextState NextTCPState, pop3aggr *pMsg)
 696 {
 697         AsyncIO *IO = &pMsg->IO;
 698         double Timeout = 0.0;
 699
 700         EVP3C_syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
 701
 702         switch (NextTCPState) {
 703         case eSendFile:
 704         case eSendReply:
 705         case eSendMore:
 706                 Timeout = POP3_C_SendTimeouts[pMsg->State];
 707 /*
 708   if (pMsg->State == eDATABody) {
 709   / * if we're sending a huge message, we need more time. * /
 710   Timeout += StrLength(pMsg->msgtext) / 1024;
 711   }
 712 */
 713                 break;
 714         case eReadFile:
 715         case eReadMessage:
 716                 Timeout = POP3_C_ReadTimeouts[pMsg->State];
 717 /*
 718   if (pMsg->State == eDATATerminateBody) {
 719   / *
 720   * some mailservers take a nap before accepting the message
 721   * content inspection and such.
 722   * /
 723   Timeout += StrLength(pMsg->msgtext) / 1024;
 724   }
 725 */
 726                 break;
 727         case eReadPayload:
 728                 Timeout = 100000;
 729                 /* TODO!!! */
 730                 break;
 731         case eSendDNSQuery:
 732         case eReadDNSReply:
 733         case eConnect:
 734         case eTerminateConnection:
 735         case eDBQuery:
 736         case eAbort:
 737         case eReadMore://// TODO
 738                 return;
 739         }
 740         SetNextTimeout(&pMsg->IO, Timeout);
 741 }
 742 eNextState POP3_C_DispatchReadDone(AsyncIO *IO)
 743 {
 744 /*      EVP3CCS_syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__); to noisy anyways. */
 745         pop3aggr *pMsg = IO->Data;
 746         eNextState rc;
 747
 748         rc = POP3C_ReadHandlers[pMsg->State](pMsg);
 749         if (rc != eReadMore)
 750             pMsg->State++;
 751         POP3SetTimeout(rc, pMsg);
 752         return rc;
 753 }
 754 eNextState POP3_C_DispatchWriteDone(AsyncIO *IO)
 755 {
 756         pop3aggr *pMsg = IO->Data;
 757         eNextState rc;
 758
 759 /*      EVP3CCS_syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__); to noisy anyways. */
 760         rc = POP3C_SendHandlers[pMsg->State](pMsg);
 761         POP3SetTimeout(rc, pMsg);
 762         return rc;
 763 }
 764
 765
 766 /*****************************************************************************/
 767 /*                     POP3 CLIENT ERROR CATCHERS                            */
 768 /*****************************************************************************/
 769 eNextState POP3_C_Terminate(AsyncIO *IO)
 770 {
 771 ///     pop3aggr *pMsg = (pop3aggr *)IO->Data;
 772
 773         EVP3CCS_syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
 774         FinalizePOP3AggrRun(IO);
 775         return eAbort;
 776 }
 777 eNextState POP3_C_TerminateDB(AsyncIO *IO)
 778 {
 779 ///     pop3aggr *pMsg = (pop3aggr *)IO->Data;
 780
 781         EVP3CCS_syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
 782         FinalizePOP3AggrRun(IO);
 783         return eAbort;
 784 }
 785 eNextState POP3_C_Timeout(AsyncIO *IO)
 786 {
 787         pop3aggr *pMsg = IO->Data;
 788
 789         EVP3CCS_syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
 790         StrBufPlain(IO->ErrMsg, CKEY(POP3C_ReadErrors[pMsg->State]));
 791         return FailAggregationRun(IO);
 792 }
 793 eNextState POP3_C_ConnFail(AsyncIO *IO)
 794 {
 795         pop3aggr *pMsg = (pop3aggr *)IO->Data;
 796
 797         EVP3CCS_syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
 798         StrBufPlain(IO->ErrMsg, CKEY(POP3C_ReadErrors[pMsg->State]));
 799         return FailAggregationRun(IO);
 800 }
 801 eNextState POP3_C_DNSFail(AsyncIO *IO)
 802 {
 803         pop3aggr *pMsg = (pop3aggr *)IO->Data;
 804
 805         EVP3CCS_syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
 806         StrBufPlain(IO->ErrMsg, CKEY(POP3C_ReadErrors[pMsg->State]));
 807         return FailAggregationRun(IO);
 808 }
 809 eNextState POP3_C_Shutdown(AsyncIO *IO)
 810 {
 811         EVP3CCS_syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
 812 ////    pop3aggr *pMsg = IO->Data;
 813
 814 ////pMsg->MyQEntry->Status = 3;
 815 ///StrBufPlain(pMsg->MyQEntry->StatusMessage, HKEY("server shutdown during message retrieval."));
 816         FinalizePOP3AggrRun(IO);
 817         return eAbort;
 818 }
 819
 820
 821 /**
 822  * @brief lineread Handler; understands when to read more POP3 lines,
 823  *   and when this is a one-lined reply.
 824  */
 825 eReadState POP3_C_ReadServerStatus(AsyncIO *IO)
 826 {
 827         eReadState Finished = eBufferNotEmpty;
 828
 829         switch (IO->NextState) {
 830         case eSendDNSQuery:
 831         case eReadDNSReply:
 832         case eDBQuery:
 833         case eConnect:
 834         case eTerminateConnection:
 835         case eAbort:
 836                 Finished = eReadFail;
 837                 break;
 838         case eSendFile:
 839         case eSendReply:
 840         case eSendMore:
 841         case eReadMore:
 842         case eReadMessage:
 843                 Finished = StrBufChunkSipLine(IO->IOBuf, &IO->RecvBuf);
 844                 break;
 845         case eReadFile:
 846         case eReadPayload:
 847                 Finished = CtdlReadMessageBodyAsync(IO);
 848                 break;
 849         }
 850         return Finished;
 851 }
 852
 853 /*****************************************************************************
 854  * So we connect our Server IP here.                                         *
 855  *****************************************************************************/
 856 eNextState POP3_C_ReAttachToFetchMessages(AsyncIO *IO)
 857 {
 858         pop3aggr *cpptr = IO->Data;
 859
 860         EVP3CCS_syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
 861 ////??? cpptr->State ++;
 862         if (cpptr->Pos == NULL)
 863                 cpptr->Pos = GetNewHashPos(cpptr->MsgNumbers, 0);
 864
 865         POP3_C_DispatchWriteDone(IO);
 866         ReAttachIO(IO, cpptr, 0);
 867         IO->NextState = eReadMessage;
 868         return IO->NextState;
 869 }
 870
 871 eNextState pop3_connect_ip(AsyncIO *IO)
 872 {
 873         pop3aggr *cpptr = IO->Data;
 874
 875         if (cpptr->IOStart == 0.0) /* whith or without DNS? */
 876                 cpptr->IOStart = IO->Now;
 877
 878         EVP3CCS_syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
 879
 880         return EvConnectSock(IO,
 881                              POP3_C_ConnTimeout,
 882                              POP3_C_ReadTimeouts[0],
 883                              1);
 884 }
 885
 886 eNextState pop3_get_one_host_ip_done(AsyncIO *IO)
 887 {
 888         pop3aggr *cpptr = IO->Data;
 889         struct hostent *hostent;
 890
 891         QueryCbDone(IO);
 892
 893         hostent = cpptr->HostLookup.VParsedDNSReply;
 894         if ((cpptr->HostLookup.DNSStatus == ARES_SUCCESS) &&
 895             (hostent != NULL) ) {
 896                 memset(&cpptr->IO.ConnectMe->Addr, 0, sizeof(struct in6_addr));
 897                 if (cpptr->IO.ConnectMe->IPv6) {
 898                         memcpy(&cpptr->IO.ConnectMe->Addr.sin6_addr.s6_addr,
 899                                &hostent->h_addr_list[0],
 900                                sizeof(struct in6_addr));
 901
 902                         cpptr->IO.ConnectMe->Addr.sin6_family =
 903                                 hostent->h_addrtype;
 904                         cpptr->IO.ConnectMe->Addr.sin6_port   =
 905                                 htons(DefaultPOP3Port);
 906                 }
 907                 else {
 908                         struct sockaddr_in *addr =
 909                                 (struct sockaddr_in*)
 910                                 &cpptr->IO.ConnectMe->Addr;
 911
 912                         memcpy(&addr->sin_addr.s_addr,
 913                                hostent->h_addr_list[0],
 914                                sizeof(uint32_t));
 915
 916                         addr->sin_family = hostent->h_addrtype;
 917                         addr->sin_port   = htons(DefaultPOP3Port);
 918                 }
 919                 return pop3_connect_ip(IO);
 920         }
 921         else
 922                 return eAbort;
 923 }
 924
 925 eNextState pop3_get_one_host_ip(AsyncIO *IO)
 926 {
 927         pop3aggr *cpptr = IO->Data;
 928
 929         cpptr->IOStart = IO->Now;
 930
 931         EVP3CCS_syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
 932
 933         EVP3CCS_syslog(LOG_DEBUG,
 934                        "POP3 client[%ld]: looking up %s-Record %s : %d ...\n",
 935                        cpptr->n,
 936                        (cpptr->IO.ConnectMe->IPv6)? "aaaa": "a",
 937                        cpptr->IO.ConnectMe->Host,
 938                        cpptr->IO.ConnectMe->Port);
 939
 940         QueueQuery((cpptr->IO.ConnectMe->IPv6)? ns_t_aaaa : ns_t_a,
 941                    cpptr->IO.ConnectMe->Host,
 942                    &cpptr->IO,
 943                    &cpptr->HostLookup,
 944                    pop3_get_one_host_ip_done);
 945         IO->NextState = eReadDNSReply;
 946         return IO->NextState;
 947 }
 948
 949
 950
 951 int pop3_do_fetching(pop3aggr *cpptr)
 952 {
 953         AsyncIO *IO = &cpptr->IO;
 954
 955         InitIOStruct(IO,
 956                      cpptr,
 957                      eReadMessage,
 958                      POP3_C_ReadServerStatus,
 959                      POP3_C_DNSFail,
 960                      POP3_C_DispatchWriteDone,
 961                      POP3_C_DispatchReadDone,
 962                      POP3_C_Terminate,
 963                      POP3_C_TerminateDB,
 964                      POP3_C_ConnFail,
 965                      POP3_C_Timeout,
 966                      POP3_C_Shutdown);
 967
 968         safestrncpy(((CitContext *)cpptr->IO.CitContext)->cs_host,
 969                     ChrPtr(cpptr->Url),
 970                     sizeof(((CitContext *)cpptr->IO.CitContext)->cs_host));
 971
 972         if (cpptr->IO.ConnectMe->IsIP) {
 973                 QueueEventContext(&cpptr->IO,
 974                                   pop3_connect_ip);
 975         }
 976         else {
 977                 QueueEventContext(&cpptr->IO,
 978                                   pop3_get_one_host_ip);
 979         }
 980         return 1;
 981 }
 982
 983 /*
 984  * Scan a room's netconfig to determine whether it requires POP3 aggregation
 985  */
 986 void pop3client_scan_room(struct ctdlroom *qrbuf, void *data, OneRoomNetCfg *OneRNCFG)
 987 {
 988         const RoomNetCfgLine *pLine;
 989         void *vptr;
 990
 991         pthread_mutex_lock(&POP3QueueMutex);
 992         if (GetHash(POP3QueueRooms, LKEY(qrbuf->QRnumber), &vptr))
 993         {
 994                 pthread_mutex_unlock(&POP3QueueMutex);
 995                 EVP3CQ_syslog(LOG_DEBUG,
 996                               "pop3client: [%ld] %s already in progress.",
 997                               qrbuf->QRnumber,
 998                               qrbuf->QRname);
 999                 return;
1000         }
1001         pthread_mutex_unlock(&POP3QueueMutex);
1002
1003         if (server_shutting_down) return;
1004
1005         pLine = OneRNCFG->NetConfigs[pop3client];
1006
1007         while (pLine != NULL)
1008         {
1009                 pop3aggr *cptr;
1010
1011                 cptr = (pop3aggr *) malloc(sizeof(pop3aggr));
1012                 memset(cptr, 0, sizeof(pop3aggr));
1013                 ///TODO do we need this? cptr->roomlist_parts=1;
1014                 cptr->RoomName = NewStrBufPlain(qrbuf->QRname, -1);
1015                 cptr->pop3user = NewStrBufDup(pLine->Value[1]);
1016                 cptr->pop3pass = NewStrBufDup(pLine->Value[2]);
1017                 cptr->Url = NewStrBuf();
1018                 cptr->Host = NewStrBufDup(pLine->Value[0]);
1019
1020                 cptr->keep = atol(ChrPtr(pLine->Value[3]));
1021                 cptr->interval = atol(ChrPtr(pLine->Value[4]));
1022
1023                 StrBufAppendBufPlain(cptr->Url, HKEY("pop3://"), 0);
1024                 StrBufUrlescUPAppend(cptr->Url, cptr->pop3user, NULL);
1025                 StrBufAppendBufPlain(cptr->Url, HKEY(":"), 0);
1026                 StrBufUrlescUPAppend(cptr->Url, cptr->pop3pass, NULL);
1027                 StrBufAppendBufPlain(cptr->Url, HKEY("@"), 0);
1028                 StrBufAppendBuf(cptr->Url, cptr->Host, 0);
1029                 StrBufAppendBufPlain(cptr->Url, HKEY("/"), 0);
1030                 StrBufUrlescAppend(cptr->Url, cptr->RoomName, NULL);
1031
1032                 ParseURL(&cptr->IO.ConnectMe, cptr->Url, 110);
1033
1034
1035 #if 0
1036 /* todo: we need to reunite the url to be shure. */
1037
1038                 pthread_mutex_lock(&POP3ueueMutex);
1039                 GetHash(POP3FetchUrls, SKEY(ptr->Url), &vptr);
1040                 use_this_cptr = (pop3aggr *)vptr;
1041
1042                 if (use_this_rncptr != NULL)
1043                 {
1044                         /* mustn't attach to an active session */
1045                         if (use_this_cptr->RefCount > 0)
1046                         {
1047                                 DeletePOP3Cfg(cptr);
1048 ///                                             Count->count--;
1049                         }
1050                         else
1051                         {
1052                                 long *QRnumber;
1053                                 StrBufAppendBufPlain(
1054                                         use_this_cptr->rooms,
1055                                         qrbuf->QRname,
1056                                         -1, 0);
1057                                 if (use_this_cptr->roomlist_parts == 1)
1058                                 {
1059                                         use_this_cptr->OtherQRnumbers
1060                                                 = NewHash(1, lFlathash);
1061                                 }
1062                                 QRnumber = (long*)malloc(sizeof(long));
1063                                 *QRnumber = qrbuf->QRnumber;
1064                                 Put(use_this_cptr->OtherQRnumbers,
1065                                     LKEY(qrbuf->QRnumber),
1066                                     QRnumber,
1067                                     NULL);
1068
1069                                 use_this_cptr->roomlist_parts++;
1070                         }
1071                         pthread_mutex_unlock(&POP3QueueMutex);
1072                         continue;
1073                 }
1074                 pthread_mutex_unlock(&RSSQueueMutex);
1075 #endif
1076                 cptr->n = Pop3ClientID++;
1077                 pthread_mutex_lock(&POP3QueueMutex);
1078                 Put(POP3FetchUrls,
1079                     SKEY(cptr->Url),
1080                     cptr,
1081                     DeletePOP3Aggregator);
1082
1083                 pthread_mutex_unlock(&POP3QueueMutex);
1084
1085         }
1086 }
1087
1088 static int doing_pop3client = 0;
1089
1090 void pop3client_scan(void) {
1091         static time_t last_run = 0L;
1092         time_t fastest_scan;
1093         HashPos *it;
1094         long len;
1095         const char *Key;
1096         void *vrptr;
1097         pop3aggr *cptr;
1098
1099         become_session(&pop3_client_CC);
1100
1101         if (config.c_pop3_fastest < config.c_pop3_fetch)
1102                 fastest_scan = config.c_pop3_fastest;
1103         else
1104                 fastest_scan = config.c_pop3_fetch;
1105
1106         /*
1107          * Run POP3 aggregation no more frequently than once every n seconds
1108          */
1109         if ( (time(NULL) - last_run) < fastest_scan ) {
1110                 return;
1111         }
1112
1113         /*
1114          * This is a simple concurrency check to make sure only one pop3client
1115          * run is done at a time.  We could do this with a mutex, but since we
1116          * don't really require extremely fine granularity here, we'll do it
1117          * with a static variable instead.
1118          */
1119         if (doing_pop3client) return;
1120         doing_pop3client = 1;
1121
1122         EVP3CQM_syslog(LOG_DEBUG, "pop3client started");
1123         CtdlForEachNetCfgRoom(pop3client_scan_room, NULL, pop3client);
1124
1125         pthread_mutex_lock(&POP3QueueMutex);
1126         it = GetNewHashPos(POP3FetchUrls, 0);
1127         while (!server_shutting_down &&
1128                GetNextHashPos(POP3FetchUrls, it, &len, &Key, &vrptr) &&
1129                (vrptr != NULL)) {
1130                 cptr = (pop3aggr *)vrptr;
1131                 if (cptr->RefCount == 0)
1132                         if (!pop3_do_fetching(cptr))
1133                                 DeletePOP3Aggregator(cptr);////TODO
1134
1135 /*
1136         if ((palist->interval && time(NULL) > (last_run + palist->interval))
1137                         || (time(NULL) > last_run + config.c_pop3_fetch))
1138                         pop3_do_fetching(palist->roomname, palist->pop3host,
1139                         palist->pop3user, palist->pop3pass, palist->keep);
1140                 pptr = palist;
1141                 palist = palist->next;
1142                 free(pptr);
1143 */
1144         }
1145         DeleteHashPos(&it);
1146         pthread_mutex_unlock(&POP3QueueMutex);
1147
1148         EVP3CQM_syslog(LOG_DEBUG, "pop3client ended");
1149         last_run = time(NULL);
1150         doing_pop3client = 0;
1151 }
1152
1153
1154 void pop3_cleanup(void)
1155 {
1156         /* citthread_mutex_destroy(&POP3QueueMutex); TODO */
1157         while (doing_pop3client != 0) ;
1158         DeleteHash(&POP3FetchUrls);
1159         DeleteHash(&POP3QueueRooms);
1160 }
1161
1162
1163
1164 void LogDebugEnablePOP3Client(const int n)
1165 {
1166         POP3ClientDebugEnabled = n;
1167 }
1168
1169 CTDL_MODULE_INIT(pop3client)
1170 {
1171         if (!threading)
1172         {
1173                 CtdlFillSystemContext(&pop3_client_CC, "POP3aggr");
1174                 CtdlREGISTERRoomCfgType(pop3client, ParseGeneric, 0, 5, SerializeGeneric, DeleteGenericCfgLine);
1175                 pthread_mutex_init(&POP3QueueMutex, NULL);
1176                 POP3QueueRooms = NewHash(1, lFlathash);
1177                 POP3FetchUrls = NewHash(1, NULL);
1178                 CtdlRegisterSessionHook(pop3client_scan, EVT_TIMER, PRIO_AGGR + 50);
1179                 CtdlRegisterEVCleanupHook(pop3_cleanup);
1180                 CtdlRegisterDebugFlagHook(HKEY("pop3client"), LogDebugEnablePOP3Client, &POP3ClientDebugEnabled);
1181         }
1182
1183         /* return our module id for the log */
1184         return "pop3client";
1185 }