citadel/modules/pop3client/serv_pop3client.c

   1 /*
   2  * Consolidate mail from remote POP3 accounts.
   3  *
   4  * Copyright (c) 2007-2012 by the citadel.org team
   5  *
   6  * This program is open source software; you can redistribute it and/or
   7  * modify it under the terms of the GNU General Public License version 3.
   8  *
   9  * This program is distributed in the hope that it will be useful,
  10  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12  * GNU General Public License for more details.
  13  */
  14
  15 #include <stdlib.h>
  16 #include <unistd.h>
  17 #include <stdio.h>
  18
  19 #if TIME_WITH_SYS_TIME
  20 # include <sys/time.h>
  21 # include <time.h>
  22 #else
  23 # if HAVE_SYS_TIME_H
  24 #  include <sys/time.h>
  25 # else
  26 #  include <time.h>
  27 # endif
  28 #endif
  29
  30 #include <ctype.h>
  31 #include <string.h>
  32 #include <errno.h>
  33 #include <sys/types.h>
  34 #include <sys/stat.h>
  35 #include <libcitadel.h>
  36 #include "citadel.h"
  37 #include "server.h"
  38 #include "citserver.h"
  39 #include "support.h"
  40 #include "config.h"
  41 #include "ctdl_module.h"
  42 #include "clientsocket.h"
  43 #include "msgbase.h"
  44 #include "internet_addressing.h"
  45 #include "database.h"
  46 #include "citadel_dirs.h"
  47 #include "event_client.h"
  48
  49 #define POP3C_OK (strncasecmp(ChrPtr(RecvMsg->IO.IOBuf), "+OK", 3) == 0)
  50
  51 #define POP3C_DBG_SEND()                                        \
  52         syslog(LOG_DEBUG,                                       \
  53                "POP3 client[%ld]: > %s\n",                      \
  54                RecvMsg->n, ChrPtr(RecvMsg->IO.SendBuf.Buf))
  55
  56 #define POP3C_DBG_READ()                                \
  57         syslog(LOG_DEBUG,                               \
  58                "POP3 client[%ld]: < %s\n",              \
  59                RecvMsg->n,                              \
  60                ChrPtr(RecvMsg->IO.IOBuf))
  61
  62 struct CitContext pop3_client_CC;
  63
  64 pthread_mutex_t POP3QueueMutex; /* locks the access to the following vars: */
  65 HashList *POP3QueueRooms = NULL;
  66 HashList *POP3FetchUrls = NULL;
  67
  68 typedef struct pop3aggr pop3aggr;
  69 typedef eNextState(*Pop3ClientHandler)(pop3aggr* RecvMsg);
  70
  71 eNextState POP3_C_Shutdown(AsyncIO *IO);
  72 eNextState POP3_C_Timeout(AsyncIO *IO);
  73 eNextState POP3_C_ConnFail(AsyncIO *IO);
  74 eNextState POP3_C_DNSFail(AsyncIO *IO);
  75 eNextState POP3_C_DispatchReadDone(AsyncIO *IO);
  76 eNextState POP3_C_DispatchWriteDone(AsyncIO *IO);
  77 eNextState POP3_C_Terminate(AsyncIO *IO);
  78 eReadState POP3_C_ReadServerStatus(AsyncIO *IO);
  79 eNextState POP3_C_ReAttachToFetchMessages(AsyncIO *IO);
  80
  81 typedef struct __pop3_room_counter {
  82         int count;
  83         long QRnumber;
  84 }pop3_room_counter;
  85
  86 typedef enum ePOP3_C_States {
  87         ReadGreeting,
  88         GetUserState,
  89         GetPassState,
  90         GetListCommandState,
  91         GetListOneLine,
  92         GetOneMessageIDState,
  93         ReadMessageBodyFollowing,
  94         ReadMessageBody,
  95         GetDeleteState,
  96         ReadQuitState,
  97         POP3C_MaxRead
  98 }ePOP3_C_States;
  99
 100
 101 typedef struct _FetchItem {
 102         long MSGID;
 103         long MSGSize;
 104         StrBuf *MsgUIDL;
 105         StrBuf *MsgUID;
 106         int NeedFetch;
 107         struct CtdlMessage *Msg;
 108 } FetchItem;
 109
 110 void HfreeFetchItem(void *vItem)
 111 {
 112         FetchItem *Item = (FetchItem*) vItem;
 113         FreeStrBuf(&Item->MsgUIDL);
 114         FreeStrBuf(&Item->MsgUID);
 115         free(Item);
 116 }
 117
 118 struct pop3aggr {
 119         AsyncIO  IO;
 120
 121         long n;
 122         long RefCount;
 123         DNSQueryParts HostLookup;
 124
 125         long             QRnumber;
 126         HashList        *OtherQRnumbers;
 127
 128         StrBuf          *Url;
 129         StrBuf *pop3user;
 130         StrBuf *pop3pass;
 131         StrBuf *RoomName; // TODO: fill me
 132         int keep;
 133         time_t interval;
 134         ePOP3_C_States State;
 135         HashList *MsgNumbers;
 136         HashPos *Pos;
 137         FetchItem *CurrMsg;
 138 };
 139
 140 void DeletePOP3Aggregator(void *vptr)
 141 {
 142         pop3aggr *ptr = vptr;
 143         DeleteHashPos(&ptr->Pos);
 144         DeleteHash(&ptr->MsgNumbers);
 145 //      FreeStrBuf(&ptr->rooms);
 146         FreeStrBuf(&ptr->pop3user);
 147         FreeStrBuf(&ptr->pop3pass);
 148         FreeStrBuf(&ptr->RoomName);
 149         FreeURL(&ptr->IO.ConnectMe);
 150         FreeStrBuf(&ptr->Url);
 151         FreeStrBuf(&ptr->IO.IOBuf);
 152         FreeStrBuf(&ptr->IO.SendBuf.Buf);
 153         FreeStrBuf(&ptr->IO.RecvBuf.Buf);
 154         DeleteAsyncMsg(&ptr->IO.ReadMsg);
 155         ((struct CitContext*)ptr->IO.CitContext)->state = CON_IDLE;
 156         ((struct CitContext*)ptr->IO.CitContext)->kill_me = 1;
 157         FreeAsyncIOContents(&ptr->IO);
 158         free(ptr);
 159 }
 160
 161 eNextState FinalizePOP3AggrRun(AsyncIO *IO)
 162 {
 163         HashPos  *It;
 164         pop3aggr *cptr = (pop3aggr *)IO->Data;
 165
 166         syslog(LOG_DEBUG, "Terminating Aggregator; bye.\n");
 167
 168         It = GetNewHashPos(POP3FetchUrls, 0);
 169         pthread_mutex_lock(&POP3QueueMutex);
 170         {
 171                 if (GetHashPosFromKey(POP3FetchUrls, SKEY(cptr->Url), It))
 172                         DeleteEntryFromHash(POP3FetchUrls, It);
 173         }
 174         pthread_mutex_unlock(&POP3QueueMutex);
 175         DeleteHashPos(&It);
 176         return eAbort;
 177 }
 178
 179 eNextState FailAggregationRun(AsyncIO *IO)
 180 {
 181         return eAbort;
 182 }
 183
 184 eNextState POP3C_ReadGreeting(pop3aggr *RecvMsg)
 185 {
 186         POP3C_DBG_READ();
 187         /* Read the server greeting */
 188         if (!POP3C_OK) return eTerminateConnection;
 189         else return eSendReply;
 190 }
 191
 192 eNextState POP3C_SendUser(pop3aggr *RecvMsg)
 193 {
 194         /* Identify ourselves.  NOTE: we have to append a CR to each command.
 195          *  The LF will automatically be appended by sock_puts().  Believe it
 196          * or not, leaving out the CR will cause problems if the server happens
 197          * to be Exchange, which is so b0rken it actually barfs on
 198          * LF-terminated newlines.
 199          */
 200         StrBufPrintf(RecvMsg->IO.SendBuf.Buf,
 201                      "USER %s\r\n", ChrPtr(RecvMsg->pop3user));
 202         POP3C_DBG_SEND();
 203         return eReadMessage;
 204 }
 205
 206 eNextState POP3C_GetUserState(pop3aggr *RecvMsg)
 207 {
 208         POP3C_DBG_READ();
 209         if (!POP3C_OK) return eTerminateConnection;
 210         else return eSendReply;
 211 }
 212
 213 eNextState POP3C_SendPassword(pop3aggr *RecvMsg)
 214 {
 215         /* Password */
 216         StrBufPrintf(RecvMsg->IO.SendBuf.Buf,
 217                      "PASS %s\r\n", ChrPtr(RecvMsg->pop3pass));
 218         syslog(LOG_DEBUG, "<PASS <password>\n");
 219 //      POP3C_DBG_SEND(); No, we won't write the password to syslog...
 220         return eReadMessage;
 221 }
 222
 223 eNextState POP3C_GetPassState(pop3aggr *RecvMsg)
 224 {
 225         POP3C_DBG_READ();
 226         if (!POP3C_OK) return eTerminateConnection;
 227         else return eSendReply;
 228 }
 229
 230 eNextState POP3C_SendListCommand(pop3aggr *RecvMsg)
 231 {
 232         /* Get the list of messages */
 233         StrBufPlain(RecvMsg->IO.SendBuf.Buf, HKEY("LIST\r\n"));
 234         POP3C_DBG_SEND();
 235         return eReadMessage;
 236 }
 237
 238 eNextState POP3C_GetListCommandState(pop3aggr *RecvMsg)
 239 {
 240         POP3C_DBG_READ();
 241         if (!POP3C_OK) return eTerminateConnection;
 242         RecvMsg->MsgNumbers = NewHash(1, NULL);
 243         RecvMsg->State++;
 244         return eReadMore;
 245 }
 246
 247
 248 eNextState POP3C_GetListOneLine(pop3aggr *RecvMsg)
 249 {
 250 #if 0
 251         int rc;
 252 #endif
 253         const char *pch;
 254         FetchItem *OneMsg = NULL;
 255         POP3C_DBG_READ();
 256
 257         if ((StrLength(RecvMsg->IO.IOBuf) == 1) &&
 258             (ChrPtr(RecvMsg->IO.IOBuf)[0] == '.'))
 259         {
 260                 if (GetCount(RecvMsg->MsgNumbers) == 0)
 261                 {
 262                         ////    RecvMsg->Sate = ReadQuitState;
 263                 }
 264                 else
 265                 {
 266                         RecvMsg->Pos = GetNewHashPos(RecvMsg->MsgNumbers, 0);
 267                 }
 268                 return eSendReply;
 269
 270         }
 271         OneMsg = (FetchItem*) malloc(sizeof(FetchItem));
 272         memset(OneMsg, 0, sizeof(FetchItem));
 273         OneMsg->MSGID = atol(ChrPtr(RecvMsg->IO.IOBuf));
 274
 275         pch = strchr(ChrPtr(RecvMsg->IO.IOBuf), ' ');
 276         if (pch != NULL)
 277         {
 278                 OneMsg->MSGSize = atol(pch + 1);
 279         }
 280 #if 0
 281         rc = TestValidateHash(RecvMsg->MsgNumbers);
 282         if (rc != 0)
 283                 syslog(LOG_DEBUG, "Hash Invalid: %d\n", rc);
 284 #endif
 285
 286         Put(RecvMsg->MsgNumbers, LKEY(OneMsg->MSGID), OneMsg, HfreeFetchItem);
 287 #if 0
 288         rc = TestValidateHash(RecvMsg->MsgNumbers);
 289         if (rc != 0)
 290                 syslog(LOG_DEBUG, "Hash Invalid: %d\n", rc);
 291 #endif
 292         //RecvMsg->State --; /* read next Line */
 293         return eReadMore;
 294 }
 295
 296 eNextState POP3_FetchNetworkUsetableEntry(AsyncIO *IO)
 297 {
 298         long HKLen;
 299         const char *HKey;
 300         void *vData;
 301         struct cdbdata *cdbut;
 302         pop3aggr *RecvMsg = (pop3aggr *) IO->Data;
 303
 304         if(GetNextHashPos(RecvMsg->MsgNumbers,
 305                           RecvMsg->Pos,
 306                           &HKLen,
 307                           &HKey,
 308                           &vData))
 309         {
 310                 struct UseTable ut;
 311                 if (server_shutting_down)
 312                         return eAbort;
 313
 314                 RecvMsg->CurrMsg = (FetchItem*) vData;
 315                 syslog(LOG_DEBUG,
 316                        "CHECKING: whether %s has already been seen: ",
 317                        ChrPtr(RecvMsg->CurrMsg->MsgUID));
 318
 319                 /* Find out if we've already seen this item */
 320                 safestrncpy(ut.ut_msgid,
 321                             ChrPtr(RecvMsg->CurrMsg->MsgUID),
 322                             sizeof(ut.ut_msgid));
 323                 ut.ut_timestamp = time(NULL);/// TODO: libev timestamp!
 324
 325                 cdbut = cdb_fetch(CDB_USETABLE, SKEY(RecvMsg->CurrMsg->MsgUID));
 326                 if (cdbut != NULL) {
 327                         /* Item has already been seen */
 328                         syslog(LOG_DEBUG, "YES\n");
 329                         cdb_free(cdbut);
 330
 331                         /* rewrite the record anyway, to update the timestamp */
 332                         cdb_store(CDB_USETABLE,
 333                                   SKEY(RecvMsg->CurrMsg->MsgUID),
 334                                   &ut, sizeof(struct UseTable) );
 335                         RecvMsg->CurrMsg->NeedFetch = 0; ////TODO0;
 336                 }
 337                 else
 338                 {
 339                         syslog(LOG_DEBUG, "NO\n");
 340                         RecvMsg->CurrMsg->NeedFetch = 1;
 341                 }
 342                 return NextDBOperation(&RecvMsg->IO,
 343                                        POP3_FetchNetworkUsetableEntry);
 344         }
 345         else
 346         {
 347                 /* ok, now we know them all,
 348                  * continue with reading the actual messages. */
 349                 DeleteHashPos(&RecvMsg->Pos);
 350
 351                 return QueueEventContext(IO, POP3_C_ReAttachToFetchMessages);
 352         }
 353 }
 354
 355 eNextState POP3C_GetOneMessagID(pop3aggr *RecvMsg)
 356 {
 357         long HKLen;
 358         const char *HKey;
 359         void *vData;
 360
 361 #if 0
 362         int rc;
 363         rc = TestValidateHash(RecvMsg->MsgNumbers);
 364         if (rc != 0)
 365                 syslog(LOG_DEBUG, "Hash Invalid: %d\n", rc);
 366 #endif
 367         if(GetNextHashPos(RecvMsg->MsgNumbers,
 368                           RecvMsg->Pos,
 369                           &HKLen, &HKey,
 370                           &vData))
 371         {
 372                 RecvMsg->CurrMsg = (FetchItem*) vData;
 373                 /* Find out the UIDL of the message,
 374                  * to determine whether we've already downloaded it */
 375                 StrBufPrintf(RecvMsg->IO.SendBuf.Buf,
 376                              "UIDL %ld\r\n", RecvMsg->CurrMsg->MSGID);
 377                 POP3C_DBG_SEND();
 378         }
 379         else
 380         {
 381             RecvMsg->State++;
 382                 DeleteHashPos(&RecvMsg->Pos);
 383                 /// done receiving uidls.. start looking them up now.
 384                 RecvMsg->Pos = GetNewHashPos(RecvMsg->MsgNumbers, 0);
 385                 return QueueDBOperation(&RecvMsg->IO,
 386                                         POP3_FetchNetworkUsetableEntry);
 387         }
 388         return eReadMore; /* TODO */
 389 }
 390
 391 eNextState POP3C_GetOneMessageIDState(pop3aggr *RecvMsg)
 392 {
 393 #if 0
 394         int rc;
 395         rc = TestValidateHash(RecvMsg->MsgNumbers);
 396         if (rc != 0)
 397                 syslog(LOG_DEBUG, "Hash Invalid: %d\n", rc);
 398 #endif
 399
 400         POP3C_DBG_READ();
 401         if (!POP3C_OK) return eTerminateConnection;
 402         RecvMsg->CurrMsg->MsgUIDL =
 403                 NewStrBufPlain(NULL, StrLength(RecvMsg->IO.IOBuf));
 404         RecvMsg->CurrMsg->MsgUID =
 405                 NewStrBufPlain(NULL, StrLength(RecvMsg->IO.IOBuf) * 2);
 406
 407         StrBufExtract_token(RecvMsg->CurrMsg->MsgUIDL,
 408                             RecvMsg->IO.IOBuf, 2, ' ');
 409
 410         StrBufPrintf(RecvMsg->CurrMsg->MsgUID,
 411                      "pop3/%s/%s:%s@%s",
 412                      ChrPtr(RecvMsg->RoomName),
 413                      ChrPtr(RecvMsg->CurrMsg->MsgUIDL),
 414                      RecvMsg->IO.ConnectMe->User,
 415                      RecvMsg->IO.ConnectMe->Host);
 416         RecvMsg->State --;
 417         return eSendReply;
 418 }
 419
 420
 421 eNextState POP3C_SendGetOneMsg(pop3aggr *RecvMsg)
 422 {
 423         long HKLen;
 424         const char *HKey;
 425         void *vData;
 426
 427         RecvMsg->CurrMsg = NULL;
 428         while (GetNextHashPos(RecvMsg->MsgNumbers,
 429                               RecvMsg->Pos,
 430                               &HKLen, &HKey,
 431                               &vData) &&
 432                (RecvMsg->CurrMsg = (FetchItem*) vData,
 433                 RecvMsg->CurrMsg->NeedFetch == 0))
 434         {}
 435
 436         if ((RecvMsg->CurrMsg != NULL ) && (RecvMsg->CurrMsg->NeedFetch == 1))
 437         {
 438                 /* Message has not been seen.
 439                  * Tell the server to fetch the message... */
 440                 StrBufPrintf(RecvMsg->IO.SendBuf.Buf,
 441                              "RETR %ld\r\n", RecvMsg->CurrMsg->MSGID);
 442                 POP3C_DBG_SEND();
 443                 return eReadMessage;
 444         }
 445         else {
 446                 RecvMsg->State = ReadQuitState;
 447                 return POP3_C_DispatchWriteDone(&RecvMsg->IO);
 448         }
 449 }
 450
 451
 452 eNextState POP3C_ReadMessageBodyFollowing(pop3aggr *RecvMsg)
 453 {
 454         POP3C_DBG_READ();
 455         if (!POP3C_OK) return eTerminateConnection;
 456         RecvMsg->IO.ReadMsg = NewAsyncMsg(HKEY("."),
 457                                           RecvMsg->CurrMsg->MSGSize,
 458                                           config.c_maxmsglen,
 459                                           NULL, -1,
 460                                           1);
 461
 462         return eReadPayload;
 463 }
 464
 465
 466 eNextState POP3C_StoreMsgRead(AsyncIO *IO)
 467 {
 468         pop3aggr *RecvMsg = (pop3aggr *) IO->Data;
 469         struct UseTable ut;
 470
 471         syslog(LOG_DEBUG,
 472                "MARKING: %s as seen: ",
 473                ChrPtr(RecvMsg->CurrMsg->MsgUID));
 474
 475         safestrncpy(ut.ut_msgid,
 476                     ChrPtr(RecvMsg->CurrMsg->MsgUID),
 477                     sizeof(ut.ut_msgid));
 478         ut.ut_timestamp = time(NULL); /* TODO: use libev time */
 479         cdb_store(CDB_USETABLE,
 480                   ChrPtr(RecvMsg->CurrMsg->MsgUID),
 481                   StrLength(RecvMsg->CurrMsg->MsgUID),
 482                   &ut,
 483                   sizeof(struct UseTable) );
 484
 485         return QueueEventContext(&RecvMsg->IO, POP3_C_ReAttachToFetchMessages);
 486 }
 487 eNextState POP3C_SaveMsg(AsyncIO *IO)
 488 {
 489         long msgnum;
 490         pop3aggr *RecvMsg = (pop3aggr *) IO->Data;
 491
 492         /* Do Something With It (tm) */
 493         msgnum = CtdlSubmitMsg(RecvMsg->CurrMsg->Msg,
 494                                NULL,
 495                                ChrPtr(RecvMsg->RoomName),
 496                                0);
 497         if (msgnum > 0L)
 498         {
 499                 /* Message has been committed to the store
 500                  * write the uidl to the use table
 501                  * so we don't fetch this message again
 502                  */
 503         }
 504         CtdlFreeMessage(RecvMsg->CurrMsg->Msg);
 505
 506         return NextDBOperation(&RecvMsg->IO, POP3C_StoreMsgRead);
 507 }
 508
 509 eNextState POP3C_ReadMessageBody(pop3aggr *RecvMsg)
 510 {
 511         syslog(LOG_DEBUG, "Converting message...\n");
 512         RecvMsg->CurrMsg->Msg =
 513                 convert_internet_message_buf(&RecvMsg->IO.ReadMsg->MsgBuf);
 514
 515         return QueueDBOperation(&RecvMsg->IO, POP3C_SaveMsg);
 516 }
 517
 518 eNextState POP3C_SendDelete(pop3aggr *RecvMsg)
 519 {
 520         if (!RecvMsg->keep) {
 521                 StrBufPrintf(RecvMsg->IO.SendBuf.Buf,
 522                              "DELE %ld\r\n", RecvMsg->CurrMsg->MSGID);
 523                 POP3C_DBG_SEND();
 524                 return eReadMessage;
 525         }
 526         else {
 527                 RecvMsg->State = ReadMessageBodyFollowing;
 528                 return POP3_C_DispatchWriteDone(&RecvMsg->IO);
 529         }
 530 }
 531 eNextState POP3C_ReadDeleteState(pop3aggr *RecvMsg)
 532 {
 533         POP3C_DBG_READ();
 534         RecvMsg->State = GetOneMessageIDState;
 535         return eReadMessage;
 536 }
 537
 538 eNextState POP3C_SendQuit(pop3aggr *RecvMsg)
 539 {
 540         /* Log out */
 541         StrBufPlain(RecvMsg->IO.SendBuf.Buf,
 542                     HKEY("QUIT\r\n3)"));
 543         POP3C_DBG_SEND();
 544         return eReadMessage;
 545 }
 546
 547
 548 eNextState POP3C_ReadQuitState(pop3aggr *RecvMsg)
 549 {
 550         POP3C_DBG_READ();
 551         return eTerminateConnection;
 552 }
 553
 554 const long POP3_C_ConnTimeout = 1000;
 555 const long DefaultPOP3Port = 110;
 556
 557 Pop3ClientHandler POP3C_ReadHandlers[] = {
 558         POP3C_ReadGreeting,
 559         POP3C_GetUserState,
 560         POP3C_GetPassState,
 561         POP3C_GetListCommandState,
 562         POP3C_GetListOneLine,
 563         POP3C_GetOneMessageIDState,
 564         POP3C_ReadMessageBodyFollowing,
 565         POP3C_ReadMessageBody,
 566         POP3C_ReadDeleteState,
 567         POP3C_ReadQuitState,
 568 };
 569
 570 const long POP3_C_SendTimeouts[POP3C_MaxRead] = {
 571         100,
 572         100,
 573         100,
 574         100,
 575         100,
 576         100,
 577         100,
 578         100
 579 };
 580 const ConstStr POP3C_ReadErrors[POP3C_MaxRead] = {
 581         {HKEY("Connection broken during ")},
 582         {HKEY("Connection broken during ")},
 583         {HKEY("Connection broken during ")},
 584         {HKEY("Connection broken during ")},
 585         {HKEY("Connection broken during ")},
 586         {HKEY("Connection broken during ")},
 587         {HKEY("Connection broken during ")},
 588         {HKEY("Connection broken during ")}
 589 };
 590
 591 Pop3ClientHandler POP3C_SendHandlers[] = {
 592         NULL, /* we don't send a greeting */
 593         POP3C_SendUser,
 594         POP3C_SendPassword,
 595         POP3C_SendListCommand,
 596         NULL,
 597         POP3C_GetOneMessagID,
 598         POP3C_SendGetOneMsg,
 599         NULL,
 600         POP3C_SendDelete,
 601         POP3C_SendQuit
 602 };
 603
 604 const long POP3_C_ReadTimeouts[] = {
 605         100,
 606         100,
 607         100,
 608         100,
 609         100,
 610         100,
 611         100,
 612         100,
 613         100,
 614         100
 615 };
 616 /*****************************************************************************/
 617 /*                     POP3 CLIENT DISPATCHER                                */
 618 /*****************************************************************************/
 619
 620 void POP3SetTimeout(eNextState NextTCPState, pop3aggr *pMsg)
 621 {
 622         double Timeout = 0.0;
 623
 624         syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
 625
 626         switch (NextTCPState) {
 627         case eSendFile:
 628         case eSendReply:
 629         case eSendMore:
 630                 Timeout = POP3_C_SendTimeouts[pMsg->State];
 631 /*
 632   if (pMsg->State == eDATABody) {
 633   / * if we're sending a huge message, we need more time. * /
 634   Timeout += StrLength(pMsg->msgtext) / 1024;
 635   }
 636 */
 637                 break;
 638         case eReadFile:
 639         case eReadMessage:
 640                 Timeout = POP3_C_ReadTimeouts[pMsg->State];
 641 /*
 642   if (pMsg->State == eDATATerminateBody) {
 643   / *
 644   * some mailservers take a nap before accepting the message
 645   * content inspection and such.
 646   * /
 647   Timeout += StrLength(pMsg->msgtext) / 1024;
 648   }
 649 */
 650                 break;
 651         case eReadPayload:
 652                 Timeout = 100000;
 653                 /* TODO!!! */
 654                 break;
 655         case eSendDNSQuery:
 656         case eReadDNSReply:
 657         case eConnect:
 658         case eTerminateConnection:
 659         case eDBQuery:
 660         case eAbort:
 661         case eReadMore://// TODO
 662                 return;
 663         }
 664         SetNextTimeout(&pMsg->IO, Timeout);
 665 }
 666 eNextState POP3_C_DispatchReadDone(AsyncIO *IO)
 667 {
 668         syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
 669         pop3aggr *pMsg = IO->Data;
 670         eNextState rc;
 671
 672         rc = POP3C_ReadHandlers[pMsg->State](pMsg);
 673         if (rc != eReadMore)
 674             pMsg->State++;
 675         POP3SetTimeout(rc, pMsg);
 676         return rc;
 677 }
 678 eNextState POP3_C_DispatchWriteDone(AsyncIO *IO)
 679 {
 680         syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
 681         pop3aggr *pMsg = IO->Data;
 682         eNextState rc;
 683
 684         rc = POP3C_SendHandlers[pMsg->State](pMsg);
 685         POP3SetTimeout(rc, pMsg);
 686         return rc;
 687 }
 688
 689
 690 /*****************************************************************************/
 691 /*                     POP3 CLIENT ERROR CATCHERS                            */
 692 /*****************************************************************************/
 693 eNextState POP3_C_Terminate(AsyncIO *IO)
 694 {
 695 ///     pop3aggr *pMsg = (pop3aggr *)IO->Data;
 696
 697         syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
 698         FinalizePOP3AggrRun(IO);
 699         return eAbort;
 700 }
 701 eNextState POP3_C_TerminateDB(AsyncIO *IO)
 702 {
 703 ///     pop3aggr *pMsg = (pop3aggr *)IO->Data;
 704
 705         syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
 706         FinalizePOP3AggrRun(IO);
 707         return eAbort;
 708 }
 709 eNextState POP3_C_Timeout(AsyncIO *IO)
 710 {
 711         pop3aggr *pMsg = IO->Data;
 712
 713         syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
 714         StrBufPlain(IO->ErrMsg, CKEY(POP3C_ReadErrors[pMsg->State]));
 715         return FailAggregationRun(IO);
 716 }
 717 eNextState POP3_C_ConnFail(AsyncIO *IO)
 718 {
 719         pop3aggr *pMsg = (pop3aggr *)IO->Data;
 720
 721         syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
 722         StrBufPlain(IO->ErrMsg, CKEY(POP3C_ReadErrors[pMsg->State]));
 723         return FailAggregationRun(IO);
 724 }
 725 eNextState POP3_C_DNSFail(AsyncIO *IO)
 726 {
 727         pop3aggr *pMsg = (pop3aggr *)IO->Data;
 728
 729         syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
 730         StrBufPlain(IO->ErrMsg, CKEY(POP3C_ReadErrors[pMsg->State]));
 731         return FailAggregationRun(IO);
 732 }
 733 eNextState POP3_C_Shutdown(AsyncIO *IO)
 734 {
 735         syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
 736 ////    pop3aggr *pMsg = IO->Data;
 737
 738 ////pMsg->MyQEntry->Status = 3;
 739 ///StrBufPlain(pMsg->MyQEntry->StatusMessage, HKEY("server shutdown during message retrieval."));
 740         FinalizePOP3AggrRun(IO);
 741         return eAbort;
 742 }
 743
 744
 745 /*
 746  * lineread Handler; understands when to read more POP3 lines,
 747  * and when this is a one-lined reply.
 748  */
 749 eReadState POP3_C_ReadServerStatus(AsyncIO *IO)
 750 {
 751         eReadState Finished = eBufferNotEmpty;
 752
 753         switch (IO->NextState) {
 754         case eSendDNSQuery:
 755         case eReadDNSReply:
 756         case eDBQuery:
 757         case eConnect:
 758         case eTerminateConnection:
 759         case eAbort:
 760                 Finished = eReadFail;
 761                 break;
 762         case eSendFile:
 763         case eSendReply:
 764         case eSendMore:
 765         case eReadMore:
 766         case eReadMessage:
 767                 Finished = StrBufChunkSipLine(IO->IOBuf, &IO->RecvBuf);
 768                 break;
 769         case eReadFile:
 770         case eReadPayload:
 771                 Finished = CtdlReadMessageBodyAsync(IO);
 772                 break;
 773         }
 774         return Finished;
 775 }
 776
 777 /*****************************************************************************
 778  * So we connect our Server IP here.                                         *
 779  *****************************************************************************
 780  */
 781 eNextState POP3_C_ReAttachToFetchMessages(AsyncIO *IO)
 782 {
 783         pop3aggr *cpptr = IO->Data;
 784
 785         syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
 786 ////??? cpptr->State ++;
 787         if (cpptr->Pos == NULL)
 788                 cpptr->Pos = GetNewHashPos(cpptr->MsgNumbers, 0);
 789
 790         POP3_C_DispatchWriteDone(IO);
 791         ReAttachIO(IO, cpptr, 0);
 792         IO->NextState = eReadMessage;
 793         return IO->NextState;
 794 }
 795
 796 eNextState pop3_connect_ip(AsyncIO *IO)
 797 {
 798         syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
 799
 800         return EvConnectSock(IO,
 801                              POP3_C_ConnTimeout,
 802                              POP3_C_ReadTimeouts[0],
 803                              1);
 804 }
 805
 806 eNextState pop3_get_one_host_ip_done(AsyncIO *IO)
 807 {
 808         pop3aggr *cpptr = IO->Data;
 809         struct hostent *hostent;
 810
 811         QueryCbDone(IO);
 812
 813         hostent = cpptr->HostLookup.VParsedDNSReply;
 814         if ((cpptr->HostLookup.DNSStatus == ARES_SUCCESS) &&
 815             (hostent != NULL) ) {
 816                 memset(&cpptr->IO.ConnectMe->Addr, 0, sizeof(struct in6_addr));
 817                 if (cpptr->IO.ConnectMe->IPv6) {
 818                         memcpy(&cpptr->IO.ConnectMe->Addr.sin6_addr.s6_addr,
 819                                &hostent->h_addr_list[0],
 820                                sizeof(struct in6_addr));
 821
 822                         cpptr->IO.ConnectMe->Addr.sin6_family =
 823                                 hostent->h_addrtype;
 824                         cpptr->IO.ConnectMe->Addr.sin6_port   =
 825                                 htons(DefaultPOP3Port);
 826                 }
 827                 else {
 828                         struct sockaddr_in *addr =
 829                                 (struct sockaddr_in*)
 830                                 &cpptr->IO.ConnectMe->Addr;
 831
 832                         memcpy(&addr->sin_addr.s_addr,
 833                                hostent->h_addr_list[0],
 834                                sizeof(uint32_t));
 835
 836                         addr->sin_family = hostent->h_addrtype;
 837                         addr->sin_port   = htons(DefaultPOP3Port);
 838                 }
 839                 return pop3_connect_ip(IO);
 840         }
 841         else
 842                 return eAbort;
 843 }
 844
 845 eNextState pop3_get_one_host_ip(AsyncIO *IO)
 846 {
 847         pop3aggr *cpptr = IO->Data;
 848
 849         syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
 850
 851         syslog(LOG_DEBUG,
 852                       "POP3 client[%ld]: looking up %s-Record %s : %d ...\n",
 853                       cpptr->n,
 854                       (cpptr->IO.ConnectMe->IPv6)? "aaaa": "a",
 855                       cpptr->IO.ConnectMe->Host,
 856                       cpptr->IO.ConnectMe->Port);
 857
 858         QueueQuery((cpptr->IO.ConnectMe->IPv6)? ns_t_aaaa : ns_t_a,
 859                    cpptr->IO.ConnectMe->Host,
 860                    &cpptr->IO,
 861                    &cpptr->HostLookup,
 862                    pop3_get_one_host_ip_done);
 863         IO->NextState = eReadDNSReply;
 864         return IO->NextState;
 865 }
 866
 867
 868
 869 int pop3_do_fetching(pop3aggr *cpptr)
 870 {
 871         InitIOStruct(&cpptr->IO,
 872                      cpptr,
 873                      eReadMessage,
 874                      POP3_C_ReadServerStatus,
 875                      POP3_C_DNSFail,
 876                      POP3_C_DispatchWriteDone,
 877                      POP3_C_DispatchReadDone,
 878                      POP3_C_Terminate,
 879                      POP3_C_TerminateDB,
 880                      POP3_C_ConnFail,
 881                      POP3_C_Timeout,
 882                      POP3_C_Shutdown);
 883
 884         safestrncpy(((CitContext *)cpptr->IO.CitContext)->cs_host,
 885                     ChrPtr(cpptr->Url),
 886                     sizeof(((CitContext *)cpptr->IO.CitContext)->cs_host));
 887
 888         if (cpptr->IO.ConnectMe->IsIP) {
 889                 QueueEventContext(&cpptr->IO,
 890                                   pop3_connect_ip);
 891         }
 892         else {
 893                 QueueEventContext(&cpptr->IO,
 894                                   pop3_get_one_host_ip);
 895         }
 896         return 1;
 897 }
 898
 899 /*
 900  * Scan a room's netconfig to determine whether it requires POP3 aggregation
 901  */
 902 void pop3client_scan_room(struct ctdlroom *qrbuf, void *data)
 903 {
 904         StrBuf *CfgData;
 905         StrBuf *CfgType;
 906         StrBuf *Line;
 907
 908         struct stat statbuf;
 909         char filename[PATH_MAX];
 910         int  fd;
 911         int Done;
 912         void *vptr;
 913         const char *CfgPtr, *lPtr;
 914         const char *Err;
 915
 916 //      pop3_room_counter *Count = NULL;
 917 //      pop3aggr *cpptr;
 918
 919         pthread_mutex_lock(&POP3QueueMutex);
 920         if (GetHash(POP3QueueRooms, LKEY(qrbuf->QRnumber), &vptr))
 921         {
 922                 syslog(LOG_DEBUG,
 923                               "pop3client: [%ld] %s already in progress.\n",
 924                               qrbuf->QRnumber,
 925                               qrbuf->QRname);
 926                 pthread_mutex_unlock(&POP3QueueMutex);
 927         }
 928         pthread_mutex_unlock(&POP3QueueMutex);
 929
 930         if (server_shutting_down) return;
 931
 932         assoc_file_name(filename, sizeof filename, qrbuf, ctdl_netcfg_dir);
 933
 934         if (server_shutting_down)
 935                 return;
 936
 937         /* Only do net processing for rooms that have netconfigs */
 938         fd = open(filename, 0);
 939         if (fd <= 0) {
 940                 return;
 941         }
 942         if (server_shutting_down)
 943                 return;
 944         if (fstat(fd, &statbuf) == -1) {
 945                 syslog(LOG_DEBUG,
 946                        "ERROR: could not stat configfile '%s' - %s\n",
 947                        filename,
 948                        strerror(errno));
 949                 return;
 950         }
 951         if (server_shutting_down)
 952                 return;
 953         CfgData = NewStrBufPlain(NULL, statbuf.st_size + 1);
 954         if (StrBufReadBLOB(CfgData, &fd, 1, statbuf.st_size, &Err) < 0) {
 955                 close(fd);
 956                 FreeStrBuf(&CfgData);
 957                 syslog(LOG_DEBUG, "ERROR: reading config '%s' - %s<br>\n",
 958                               filename, strerror(errno));
 959                 return;
 960         }
 961         close(fd);
 962         if (server_shutting_down)
 963                 return;
 964
 965         CfgPtr = NULL;
 966         CfgType = NewStrBuf();
 967         Line = NewStrBufPlain(NULL, StrLength(CfgData));
 968         Done = 0;
 969
 970         while (!Done)
 971         {
 972                 Done = StrBufSipLine(Line, CfgData, &CfgPtr) == 0;
 973                 if (StrLength(Line) > 0)
 974                 {
 975                         lPtr = NULL;
 976                         StrBufExtract_NextToken(CfgType, Line, &lPtr, '|');
 977                         if (!strcasecmp("pop3client", ChrPtr(CfgType)))
 978                         {
 979                                 pop3aggr *cptr;
 980                                 StrBuf *Tmp;
 981 /*
 982                                 if (Count == NULL)
 983                                 {
 984                                 Count = malloc(sizeof(pop3_room_counter));
 985                                         Count->count = 0;
 986                                 }
 987                                 Count->count ++;
 988 */
 989                                 cptr = (pop3aggr *) malloc(sizeof(pop3aggr));
 990                                 memset(cptr, 0, sizeof(pop3aggr));
 991                                 ///TODO do we need this? cptr->roomlist_parts=1;
 992                                 cptr->RoomName =
 993                                         NewStrBufPlain(qrbuf->QRname, -1);
 994                                 cptr->pop3user =
 995                                         NewStrBufPlain(NULL, StrLength(Line));
 996                                 cptr->pop3pass =
 997                                         NewStrBufPlain(NULL, StrLength(Line));
 998                                 cptr->Url = NewStrBuf();
 999                                 Tmp = NewStrBuf();
1000
1001                                 StrBufExtract_NextToken(Tmp, Line, &lPtr, '|');
1002                                 StrBufExtract_NextToken(cptr->pop3user,
1003                                                         Line,
1004                                                         &lPtr,
1005                                                         '|');
1006
1007                                 StrBufExtract_NextToken(cptr->pop3pass,
1008                                                         Line,
1009                                                         &lPtr,
1010                                                         '|');
1011
1012                                 cptr->keep = StrBufExtractNext_long(Line,
1013                                                                     &lPtr,
1014                                                                     '|');
1015
1016                                 cptr->interval = StrBufExtractNext_long(Line,
1017                                                                         &lPtr,
1018                                                                         '|');
1019
1020                                 StrBufPrintf(cptr->Url, "pop3://%s:%s@%s/%s",
1021                                              ChrPtr(cptr->pop3user),
1022                                              ChrPtr(cptr->pop3pass),
1023                                              ChrPtr(Tmp),
1024                                              ChrPtr(cptr->RoomName));
1025                                 FreeStrBuf(&Tmp);
1026                                 ParseURL(&cptr->IO.ConnectMe, cptr->Url, 110);
1027
1028
1029 #if 0
1030 /* todo: we need to reunite the url to be shure. */
1031
1032                                 pthread_mutex_lock(&POP3ueueMutex);
1033                                 GetHash(POP3FetchUrls, SKEY(ptr->Url), &vptr);
1034                                 use_this_cptr = (pop3aggr *)vptr;
1035
1036                                 if (use_this_rncptr != NULL)
1037                                 {
1038                                         /* mustn't attach to an active session */
1039                                         if (use_this_cptr->RefCount > 0)
1040                                         {
1041                                                 DeletePOP3Cfg(cptr);
1042 ///                                             Count->count--;
1043                                         }
1044                                         else
1045                                         {
1046                                                 long *QRnumber;
1047                                                 StrBufAppendBufPlain(
1048                                                         use_this_cptr->rooms,
1049                                                         qrbuf->QRname,
1050                                                         -1, 0);
1051                                                 if (use_this_cptr->roomlist_parts == 1)
1052                                                 {
1053                                                         use_this_cptr->OtherQRnumbers
1054                                                                 = NewHash(1, lFlathash);
1055                                                 }
1056                                                 QRnumber = (long*)malloc(sizeof(long));
1057                                                 *QRnumber = qrbuf->QRnumber;
1058                                                 Put(use_this_cptr->OtherQRnumbers,
1059                                                     LKEY(qrbuf->QRnumber),
1060                                                     QRnumber,
1061                                                     NULL);
1062
1063                                                 use_this_cptr->roomlist_parts++;
1064                                         }
1065                                         pthread_mutex_unlock(&POP3QueueMutex);
1066                                         continue;
1067                                 }
1068                                 pthread_mutex_unlock(&RSSQueueMutex);
1069 #endif
1070
1071                                 pthread_mutex_lock(&POP3QueueMutex);
1072                                 Put(POP3FetchUrls,
1073                                     SKEY(cptr->Url),
1074                                     cptr,
1075                                     DeletePOP3Aggregator);
1076
1077                                 pthread_mutex_unlock(&POP3QueueMutex);
1078
1079                         }
1080
1081                 }
1082
1083                 ///fclose(fp);
1084
1085         }
1086         FreeStrBuf(&Line);
1087         FreeStrBuf(&CfgType);
1088         FreeStrBuf(&CfgData);
1089 }
1090
1091 static int doing_pop3client = 0;
1092
1093 void pop3client_scan(void) {
1094         static time_t last_run = 0L;
1095         time_t fastest_scan;
1096         HashPos *it;
1097         long len;
1098         const char *Key;
1099         void *vrptr;
1100         pop3aggr *cptr;
1101
1102         become_session(&pop3_client_CC);
1103
1104         if (config.c_pop3_fastest < config.c_pop3_fetch)
1105                 fastest_scan = config.c_pop3_fastest;
1106         else
1107                 fastest_scan = config.c_pop3_fetch;
1108
1109         /*
1110          * Run POP3 aggregation no more frequently than once every n seconds
1111          */
1112         if ( (time(NULL) - last_run) < fastest_scan ) {
1113                 syslog(LOG_DEBUG,
1114                         "pop3client: polling interval not yet reached; last run was %ldm%lds ago",
1115                         ((time(NULL) - last_run) / 60),
1116                         ((time(NULL) - last_run) % 60)
1117                 );
1118                 return;
1119         }
1120
1121         /*
1122          * This is a simple concurrency check to make sure only one pop3client
1123          * run is done at a time.  We could do this with a mutex, but since we
1124          * don't really require extremely fine granularity here, we'll do it
1125          * with a static variable instead.
1126          */
1127         if (doing_pop3client) {
1128                 syslog(LOG_DEBUG, "pop3client: concurrency check failed; another poll is already running");
1129                 return;
1130         }
1131         doing_pop3client = 1;
1132
1133         syslog(LOG_DEBUG, "pop3client started");
1134         CtdlForEachRoom(pop3client_scan_room, NULL);
1135
1136         pthread_mutex_lock(&POP3QueueMutex);
1137         it = GetNewHashPos(POP3FetchUrls, 0);
1138         while (!server_shutting_down &&
1139                GetNextHashPos(POP3FetchUrls, it, &len, &Key, &vrptr) &&
1140                (vrptr != NULL)) {
1141                 cptr = (pop3aggr *)vrptr;
1142                 if (cptr->RefCount == 0)
1143                         if (!pop3_do_fetching(cptr))
1144                                 DeletePOP3Aggregator(cptr);////TODO
1145
1146 /*
1147         if ((palist->interval && time(NULL) > (last_run + palist->interval))
1148                         || (time(NULL) > last_run + config.c_pop3_fetch))
1149                         pop3_do_fetching(palist->roomname, palist->pop3host,
1150                         palist->pop3user, palist->pop3pass, palist->keep);
1151                 pptr = palist;
1152                 palist = palist->next;
1153                 free(pptr);
1154 */
1155         }
1156         DeleteHashPos(&it);
1157         pthread_mutex_unlock(&POP3QueueMutex);
1158
1159         syslog(LOG_DEBUG, "pop3client ended");
1160         last_run = time(NULL);
1161         doing_pop3client = 0;
1162 }
1163
1164
1165 void pop3_cleanup(void)
1166 {
1167         /* citthread_mutex_destroy(&POP3QueueMutex); TODO */
1168         while (doing_pop3client != 0) ;
1169         DeleteHash(&POP3FetchUrls);
1170         DeleteHash(&POP3QueueRooms);
1171 }
1172
1173 CTDL_MODULE_INIT(pop3client)
1174 {
1175         if (!threading)
1176         {
1177                 CtdlFillSystemContext(&pop3_client_CC, "POP3aggr");
1178                 pthread_mutex_init(&POP3QueueMutex, NULL);
1179                 POP3QueueRooms = NewHash(1, lFlathash);
1180                 POP3FetchUrls = NewHash(1, NULL);
1181                 CtdlRegisterSessionHook(pop3client_scan, EVT_TIMER);
1182                 CtdlRegisterEVCleanupHook(pop3_cleanup);
1183         }
1184
1185         /* return our module id for the log */
1186         return "pop3client";
1187 }