citadel/modules/pop3client/serv_pop3client.c

   1 /*
   2  * Consolidate mail from remote POP3 accounts.
   3  *
   4  * Copyright (c) 2007-2009 by the citadel.org team
   5  *
   6  *  This program is free software; you can redistribute it and/or modify
   7  *  it under the terms of the GNU General Public License as published by
   8  *  the Free Software Foundation; either version 3 of the License, or
   9  *  (at your option) any later version.
  10  *
  11  *  This program is distributed in the hope that it will be useful,
  12  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
  13  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14  *  GNU General Public License for more details.
  15  *
  16  *  You should have received a copy of the GNU General Public License
  17  *  along with this program; if not, write to the Free Software
  18  *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  19  */
  20
  21 #include <stdlib.h>
  22 #include <unistd.h>
  23 #include <stdio.h>
  24
  25 #if TIME_WITH_SYS_TIME
  26 # include <sys/time.h>
  27 # include <time.h>
  28 #else
  29 # if HAVE_SYS_TIME_H
  30 #  include <sys/time.h>
  31 # else
  32 #  include <time.h>
  33 # endif
  34 #endif
  35
  36 #include <ctype.h>
  37 #include <string.h>
  38 #include <errno.h>
  39 #include <sys/types.h>
  40 #include <sys/stat.h>
  41 #include <libcitadel.h>
  42 #include "citadel.h"
  43 #include "server.h"
  44 #include "citserver.h"
  45 #include "support.h"
  46 #include "config.h"
  47 #include "ctdl_module.h"
  48 #include "clientsocket.h"
  49 #include "msgbase.h"
  50 #include "internet_addressing.h"
  51 #include "database.h"
  52 #include "citadel_dirs.h"
  53 #include "event_client.h"
  54
  55
  56
  57 citthread_mutex_t POP3QueueMutex; /* locks the access to the following vars: */
  58 HashList *POP3QueueRooms = NULL; /* rss_room_counter */
  59 HashList *POP3FetchUrls = NULL; /* -> rss_aggregator; ->RefCount access to be locked too. */
  60
  61 typedef struct __pop3_room_counter {
  62         int count;
  63         long QRnumber;
  64 }pop3_room_counter;
  65
  66 typedef enum ePOP3_C_States {
  67         ReadGreeting,
  68         GetUserState,
  69         GetPassState,
  70         GetListCommandState,
  71         GetListOneLine,
  72         GetOneMessageIDState,
  73         ReadMessageBodyFollowing,
  74         ReadMessageBody,
  75         GetDeleteState,
  76         ReadQuitState,
  77         POP3C_MaxRead
  78 }ePOP3_C_States;
  79
  80
  81 typedef struct _FetchItem {
  82         long MSGID;
  83         long MSGSize;
  84         StrBuf *MsgUIDL;
  85         StrBuf *MsgUID;
  86         int NeedFetch;
  87         struct CtdlMessage *Msg;
  88 } FetchItem;
  89
  90 void HfreeFetchItem(void *vItem)
  91 {
  92         FetchItem *Item = (FetchItem*) vItem;
  93         FreeStrBuf(&Item->MsgUIDL);
  94         FreeStrBuf(&Item->MsgUID);
  95         free(Item);
  96 }
  97
  98 typedef struct __pop3aggr {
  99         AsyncIO          IO;
 100
 101         long n;
 102         long RefCount;
 103         ParsedURL Pop3Host;
 104         DNSQueryParts HostLookup;
 105
 106 //      StrBuf          *rooms;
 107         long             QRnumber;
 108         HashList        *OtherQRnumbers;
 109
 110         StrBuf          *Url;
 111 ///     StrBuf *pop3host; -> URL
 112         StrBuf *pop3user;
 113         StrBuf *pop3pass;
 114         StrBuf *RoomName; // TODO: fill me
 115         int keep;
 116         time_t interval;
 117         ePOP3_C_States State;
 118         HashList *MsgNumbers;
 119         HashPos *Pos;
 120         FetchItem *CurrMsg;
 121 } pop3aggr;
 122
 123 void DeletePOP3Aggregator(void *vptr)
 124 {
 125         pop3aggr *ptr = vptr;
 126         DeleteHashPos(&ptr->Pos);
 127         DeleteHash(&ptr->MsgNumbers);
 128 //      FreeStrBuf(&ptr->rooms);
 129         FreeStrBuf(&ptr->pop3user);
 130         FreeStrBuf(&ptr->pop3pass);
 131         FreeStrBuf(&ptr->RoomName);
 132 }
 133
 134
 135 typedef eNextState(*Pop3ClientHandler)(pop3aggr* RecvMsg);
 136
 137 eNextState POP3_C_Shutdown(AsyncIO *IO);
 138 eNextState POP3_C_Timeout(AsyncIO *IO);
 139 eNextState POP3_C_ConnFail(AsyncIO *IO);
 140 eNextState POP3_C_DispatchReadDone(AsyncIO *IO);
 141 eNextState POP3_C_DispatchWriteDone(AsyncIO *IO);
 142 eNextState POP3_C_Terminate(AsyncIO *IO);
 143 eReadState POP3_C_ReadServerStatus(AsyncIO *IO);
 144 eNextState POP3_C_ReAttachToFetchMessages(AsyncIO *IO);
 145
 146 eNextState FinalizePOP3AggrRun(AsyncIO *IO)
 147 {
 148         HashPos  *It;
 149         pop3aggr *cptr = (pop3aggr *)IO->Data;
 150
 151         CtdlLogPrintf(CTDL_DEBUG, "Terminating Aggregator; bye.\n");
 152
 153         It = GetNewHashPos(POP3FetchUrls, 0);
 154         citthread_mutex_lock(&POP3QueueMutex);
 155         {
 156                 GetHashPosFromKey(POP3FetchUrls, SKEY(cptr->Url), It);
 157                 DeleteEntryFromHash(POP3FetchUrls, It);
 158         }
 159         citthread_mutex_unlock(&POP3QueueMutex);
 160         DeleteHashPos(&It);
 161         return eAbort;
 162 }
 163
 164 eNextState FailAggregationRun(AsyncIO *IO)
 165 {
 166         return eAbort;
 167 }
 168
 169 #define POP3C_DBG_SEND() CtdlLogPrintf(CTDL_DEBUG, "POP3 client[%ld]: > %s\n", RecvMsg->n, ChrPtr(RecvMsg->IO.SendBuf.Buf))
 170 #define POP3C_DBG_READ() CtdlLogPrintf(CTDL_DEBUG, "POP3 client[%ld]: < %s\n", RecvMsg->n, ChrPtr(RecvMsg->IO.IOBuf))
 171 #define POP3C_OK (strncasecmp(ChrPtr(RecvMsg->IO.IOBuf), "+OK", 3) == 0)
 172
 173 eNextState POP3C_ReadGreeting(pop3aggr *RecvMsg)
 174 {
 175         POP3C_DBG_READ();
 176         /* Read the server greeting */
 177         if (!POP3C_OK) return eTerminateConnection;
 178         else return eSendReply;
 179 }
 180
 181
 182 eNextState POP3C_SendUser(pop3aggr *RecvMsg)
 183 {
 184         /* Identify ourselves.  NOTE: we have to append a CR to each command.  The LF will
 185          * automatically be appended by sock_puts().  Believe it or not, leaving out the CR
 186          * will cause problems if the server happens to be Exchange, which is so b0rken it
 187          * actually barfs on LF-terminated newlines.
 188          */
 189         StrBufPrintf(RecvMsg->IO.SendBuf.Buf,
 190                      "USER %s\r\n", ChrPtr(RecvMsg->pop3user));
 191         POP3C_DBG_SEND();
 192         return eReadMessage;
 193 }
 194
 195 eNextState POP3C_GetUserState(pop3aggr *RecvMsg)
 196 {
 197         POP3C_DBG_READ();
 198         if (!POP3C_OK) return eTerminateConnection;
 199         else return eSendReply;
 200 }
 201
 202 eNextState POP3C_SendPassword(pop3aggr *RecvMsg)
 203 {
 204         /* Password */
 205         StrBufPrintf(RecvMsg->IO.SendBuf.Buf,
 206                      "PASS %s\r\n", ChrPtr(RecvMsg->pop3pass));
 207         CtdlLogPrintf(CTDL_DEBUG, "<PASS <password>\n");
 208 //      POP3C_DBG_SEND();
 209         return eReadMessage;
 210 }
 211
 212 eNextState POP3C_GetPassState(pop3aggr *RecvMsg)
 213 {
 214         POP3C_DBG_READ();
 215         if (!POP3C_OK) return eTerminateConnection;
 216         else return eSendReply;
 217 }
 218
 219 eNextState POP3C_SendListCommand(pop3aggr *RecvMsg)
 220 {
 221         /* Get the list of messages */
 222         StrBufPlain(RecvMsg->IO.SendBuf.Buf, HKEY("LIST\r\n"));
 223         POP3C_DBG_SEND();
 224         return eReadMessage;
 225 }
 226
 227 eNextState POP3C_GetListCommandState(pop3aggr *RecvMsg)
 228 {
 229         POP3C_DBG_READ();
 230         if (!POP3C_OK) return eTerminateConnection;
 231         RecvMsg->MsgNumbers = NewHash(1, NULL);
 232         RecvMsg->State++;
 233         return eReadMore;
 234 }
 235
 236
 237 eNextState POP3C_GetListOneLine(pop3aggr *RecvMsg)
 238 {
 239         const char *pch;
 240         FetchItem *OneMsg = NULL;
 241         POP3C_DBG_READ();
 242
 243         if ((StrLength(RecvMsg->IO.IOBuf) == 1) &&
 244             (ChrPtr(RecvMsg->IO.IOBuf)[0] == '.'))
 245         {
 246                 if (GetCount(RecvMsg->MsgNumbers) == 0)
 247                 {
 248                         ////    RecvMsg->Sate = ReadQuitState;
 249                 }
 250                 else
 251                 {
 252                         RecvMsg->Pos = GetNewHashPos(RecvMsg->MsgNumbers, 0);
 253                 }
 254                 return eSendReply;
 255
 256         }
 257         OneMsg = (FetchItem*) malloc(sizeof(FetchItem));
 258         memset(OneMsg, 0, sizeof(FetchItem));
 259         OneMsg->MSGID = atol(ChrPtr(RecvMsg->IO.IOBuf));
 260
 261         pch = strchr(ChrPtr(RecvMsg->IO.IOBuf), ' ');
 262         if (pch != NULL)
 263         {
 264                 OneMsg->MSGSize = atol(pch + 1);
 265         }
 266         Put(RecvMsg->MsgNumbers, LKEY(OneMsg->MSGID), OneMsg, HfreeFetchItem);
 267
 268         //RecvMsg->State --; /* read next Line */
 269         return eReadMore;
 270 }
 271
 272 eNextState POP3_FetchNetworkUsetableEntry(AsyncIO *IO)
 273 {
 274         long HKLen;
 275         const char *HKey;
 276         void *vData;
 277         struct cdbdata *cdbut;
 278         pop3aggr *RecvMsg = (pop3aggr *) IO->Data;
 279
 280         if(GetNextHashPos(RecvMsg->MsgNumbers, RecvMsg->Pos, &HKLen, &HKey, &vData))
 281         {
 282                 struct UseTable ut;
 283
 284                 RecvMsg->CurrMsg = (FetchItem*) vData;
 285                 /* Find out if we've already seen this item */
 286                 safestrncpy(ut.ut_msgid,
 287                             ChrPtr(RecvMsg->CurrMsg->MsgUIDL),
 288                             sizeof(ut.ut_msgid));
 289                 ut.ut_timestamp = time(NULL);/// TODO: libev timestamp!
 290
 291                 cdbut = cdb_fetch(CDB_USETABLE, SKEY(RecvMsg->CurrMsg->MsgUIDL));
 292                 if (cdbut != NULL) {
 293                         /* Item has already been seen */
 294                         CtdlLogPrintf(CTDL_DEBUG, "%s has already been seen\n", ChrPtr(RecvMsg->CurrMsg->MsgUIDL));
 295                         cdb_free(cdbut);
 296
 297                         /* rewrite the record anyway, to update the timestamp */
 298                         cdb_store(CDB_USETABLE,
 299                                   SKEY(RecvMsg->CurrMsg->MsgUIDL),
 300                                   &ut, sizeof(struct UseTable) );
 301                         RecvMsg->CurrMsg->NeedFetch = 0;
 302                 }
 303                 else
 304                 {
 305                         RecvMsg->CurrMsg->NeedFetch = 1;
 306                 }
 307                 return NextDBOperation(&RecvMsg->IO, POP3_FetchNetworkUsetableEntry);
 308         }
 309         else
 310         {
 311                 /* ok, now we know them all, continue with reading the actual messages. */
 312                 DeleteHashPos(&RecvMsg->Pos);
 313
 314                 return QueueEventContext(IO, POP3_C_ReAttachToFetchMessages);
 315         }
 316 }
 317
 318 eNextState POP3C_GetOneMessagID(pop3aggr *RecvMsg)
 319 {
 320         long HKLen;
 321         const char *HKey;
 322         void *vData;
 323
 324         if(GetNextHashPos(RecvMsg->MsgNumbers, RecvMsg->Pos, &HKLen, &HKey, &vData))
 325         {
 326                 RecvMsg->CurrMsg = (FetchItem*) vData;
 327                 /* Find out the UIDL of the message, to determine whether we've already downloaded it */
 328                 StrBufPrintf(RecvMsg->IO.SendBuf.Buf,
 329                              "UIDL %ld\r\n", RecvMsg->CurrMsg->MSGID);
 330                 POP3C_DBG_SEND();
 331         }
 332         else
 333         {
 334             RecvMsg->State++;
 335                 DeleteHashPos(&RecvMsg->Pos);
 336                 /// done receiving uidls.. start looking them up now.
 337                 RecvMsg->Pos = GetNewHashPos(RecvMsg->MsgNumbers, 0);
 338                 return QueueDBOperation(&RecvMsg->IO, POP3_FetchNetworkUsetableEntry);
 339         }
 340         return eReadMore; /* TODO */
 341 }
 342
 343 eNextState POP3C_GetOneMessageIDState(pop3aggr *RecvMsg)
 344 {
 345         POP3C_DBG_READ();
 346         if (!POP3C_OK) return eTerminateConnection;
 347         RecvMsg->CurrMsg->MsgUIDL = NewStrBufPlain(NULL, StrLength(RecvMsg->IO.IOBuf));
 348         RecvMsg->CurrMsg->MsgUID = NewStrBufPlain(NULL, StrLength(RecvMsg->IO.IOBuf) * 2);
 349
 350         StrBufExtract_token(RecvMsg->CurrMsg->MsgUIDL, RecvMsg->IO.IOBuf, 2, ' ');
 351         StrBufPrintf(RecvMsg->CurrMsg->MsgUID,
 352                      "pop3/%s/%s@%s",
 353                      ChrPtr(RecvMsg->RoomName),
 354                      ChrPtr(RecvMsg->CurrMsg->MsgUIDL),
 355                      RecvMsg->Pop3Host.Host);
 356         RecvMsg->State --;
 357         return eSendReply;
 358 }
 359
 360 eNextState POP3C_GetOneMessageIDFromUseTable(pop3aggr *RecvMsg)
 361 {
 362
 363         struct cdbdata *cdbut;
 364         struct UseTable ut;
 365
 366         cdbut = cdb_fetch(CDB_USETABLE, SKEY(RecvMsg->CurrMsg->MsgUID));
 367         if (cdbut != NULL) {
 368                 /* message has already been seen */
 369                 CtdlLogPrintf(CTDL_DEBUG, "%s has already been seen\n", ChrPtr(RecvMsg->CurrMsg->MsgUID));
 370                 cdb_free(cdbut);
 371
 372                 /* rewrite the record anyway, to update the timestamp */
 373                 strcpy(ut.ut_msgid, ChrPtr(RecvMsg->CurrMsg->MsgUID));
 374                 ut.ut_timestamp = time(NULL);
 375                 cdb_store(CDB_USETABLE, SKEY(RecvMsg->CurrMsg->MsgUID), &ut, sizeof(struct UseTable) );
 376         }
 377
 378         return eReadMessage;
 379 }
 380
 381 eNextState POP3C_SendGetOneMsg(pop3aggr *RecvMsg)
 382 {
 383         long HKLen;
 384         const char *HKey;
 385         void *vData;
 386
 387         RecvMsg->CurrMsg = NULL;
 388         while (GetNextHashPos(RecvMsg->MsgNumbers, RecvMsg->Pos, &HKLen, &HKey, &vData) &&
 389                (RecvMsg->CurrMsg = (FetchItem*) vData, RecvMsg->CurrMsg->NeedFetch == 0))
 390         {}
 391
 392         if ((RecvMsg->CurrMsg != NULL ) && (RecvMsg->CurrMsg->NeedFetch == 1))
 393         {
 394                 /* Message has not been seen. Tell the server to fetch the message... */
 395                 StrBufPrintf(RecvMsg->IO.SendBuf.Buf,
 396                              "RETR %ld\r\n", RecvMsg->CurrMsg->MSGID);
 397                 POP3C_DBG_SEND();
 398                 return eReadMessage;
 399         }
 400         else {
 401                 RecvMsg->State = ReadQuitState;
 402                 return POP3_C_DispatchWriteDone(&RecvMsg->IO);
 403         }
 404 }
 405
 406
 407 eNextState POP3C_ReadMessageBodyFollowing(pop3aggr *RecvMsg)
 408 {
 409         POP3C_DBG_READ();
 410         if (!POP3C_OK) return eTerminateConnection;
 411         RecvMsg->IO.ReadMsg = NewAsyncMsg(HKEY("."),
 412                                           RecvMsg->CurrMsg->MSGSize,
 413                                           config.c_maxmsglen,
 414                                           NULL, -1,
 415                                           1);
 416
 417         return eReadPayload;
 418 }
 419
 420
 421 eNextState POP3C_StoreMsgRead(AsyncIO *IO)
 422 {
 423         pop3aggr *RecvMsg = (pop3aggr *) IO->Data;
 424         struct UseTable ut;
 425
 426         safestrncpy(ut.ut_msgid,
 427                     ChrPtr(RecvMsg->CurrMsg->MsgUID),
 428                     sizeof(ut.ut_msgid));
 429         ut.ut_timestamp = time(NULL); /* TODO: use libev time */
 430         cdb_store(CDB_USETABLE,
 431                   ChrPtr(RecvMsg->CurrMsg->MsgUID),
 432                   StrLength(RecvMsg->CurrMsg->MsgUID),
 433                   &ut,
 434                   sizeof(struct UseTable) );
 435
 436         return QueueEventContext(&RecvMsg->IO, POP3_C_ReAttachToFetchMessages);
 437 }
 438 eNextState POP3C_SaveMsg(AsyncIO *IO)
 439 {
 440         long msgnum;
 441         pop3aggr *RecvMsg = (pop3aggr *) IO->Data;
 442
 443         /* Do Something With It (tm) */
 444         msgnum = CtdlSubmitMsg(RecvMsg->CurrMsg->Msg,
 445                                NULL,
 446                                ChrPtr(RecvMsg->RoomName),
 447                                0);
 448         if (msgnum > 0L) {
 449                 /* Message has been committed to the store */
 450                 /* write the uidl to the use table so we don't fetch this message again */
 451         }
 452         CtdlFreeMessage(RecvMsg->CurrMsg->Msg);
 453
 454         return NextDBOperation(&RecvMsg->IO, POP3C_StoreMsgRead);
 455         return eReadMessage;
 456 }
 457
 458 eNextState POP3C_ReadMessageBody(pop3aggr *RecvMsg)
 459 {
 460         CtdlLogPrintf(CTDL_DEBUG, "Converting message...\n");
 461         RecvMsg->CurrMsg->Msg = convert_internet_message_buf(&RecvMsg->IO.ReadMsg->MsgBuf);
 462
 463         return QueueDBOperation(&RecvMsg->IO, POP3C_SaveMsg);
 464 }
 465
 466 eNextState POP3C_SendDelete(pop3aggr *RecvMsg)
 467 {
 468         if (!RecvMsg->keep) {
 469                 StrBufPrintf(RecvMsg->IO.SendBuf.Buf,
 470                              "DELE %ld\r\n", RecvMsg->CurrMsg->MSGID);
 471                 POP3C_DBG_SEND();
 472                 return eReadMessage;
 473         }
 474         else {
 475                 RecvMsg->State = ReadMessageBodyFollowing;
 476                 return POP3_C_DispatchWriteDone(&RecvMsg->IO);
 477         }
 478 }
 479 eNextState POP3C_ReadDeleteState(pop3aggr *RecvMsg)
 480 {
 481         POP3C_DBG_READ();
 482         RecvMsg->State = GetOneMessageIDState;
 483         return eReadMessage;
 484 }
 485
 486 eNextState POP3C_SendQuit(pop3aggr *RecvMsg)
 487 {
 488         /* Log out */
 489         StrBufPlain(RecvMsg->IO.SendBuf.Buf,
 490                     HKEY("QUIT\r\n3)"));
 491         POP3C_DBG_SEND();
 492         return eReadMessage;
 493 }
 494
 495
 496 eNextState POP3C_ReadQuitState(pop3aggr *RecvMsg)
 497 {
 498         POP3C_DBG_READ();
 499         return eTerminateConnection;
 500 }
 501
 502 const long POP3_C_ConnTimeout = 1000;
 503 const long DefaultPOP3Port = 110;
 504
 505 Pop3ClientHandler POP3C_ReadHandlers[] = {
 506         POP3C_ReadGreeting,
 507         POP3C_GetUserState,
 508         POP3C_GetPassState,
 509         POP3C_GetListCommandState,
 510         POP3C_GetListOneLine,
 511         POP3C_GetOneMessageIDState,
 512         POP3C_ReadMessageBodyFollowing,
 513         POP3C_ReadMessageBody,
 514         POP3C_ReadDeleteState,
 515         POP3C_ReadQuitState,
 516 };
 517
 518 const long POP3_C_SendTimeouts[POP3C_MaxRead] = {
 519         100,
 520         100,
 521         100,
 522         100,
 523         100,
 524         100,
 525         100,
 526         100
 527 };
 528 const ConstStr POP3C_ReadErrors[POP3C_MaxRead] = {
 529         {HKEY("Connection broken during ")},
 530         {HKEY("Connection broken during ")},
 531         {HKEY("Connection broken during ")},
 532         {HKEY("Connection broken during ")},
 533         {HKEY("Connection broken during ")},
 534         {HKEY("Connection broken during ")},
 535         {HKEY("Connection broken during ")},
 536         {HKEY("Connection broken during ")}
 537 };
 538
 539 Pop3ClientHandler POP3C_SendHandlers[] = {
 540         NULL, /* we don't send a greeting */
 541         POP3C_SendUser,
 542         POP3C_SendPassword,
 543         POP3C_SendListCommand,
 544         NULL,
 545         POP3C_GetOneMessagID,
 546         POP3C_SendGetOneMsg,
 547         NULL,
 548         POP3C_SendDelete,
 549         POP3C_SendQuit
 550 };
 551
 552 const long POP3_C_ReadTimeouts[] = {
 553         100,
 554         100,
 555         100,
 556         100,
 557         100,
 558         100,
 559         100,
 560         100,
 561         100,
 562         100
 563 };
 564 /*****************************************************************************/
 565 /*                     POP3 CLIENT DISPATCHER                                */
 566 /*****************************************************************************/
 567
 568 void POP3SetTimeout(eNextState NextTCPState, pop3aggr *pMsg)
 569 {
 570         double Timeout = 0.0;
 571
 572         CtdlLogPrintf(CTDL_DEBUG, "POP3: %s\n", __FUNCTION__);
 573
 574         switch (NextTCPState) {
 575         case eSendReply:
 576         case eSendMore:
 577                 Timeout = POP3_C_SendTimeouts[pMsg->State];
 578 /*
 579   if (pMsg->State == eDATABody) {
 580   / * if we're sending a huge message, we need more time. * /
 581   Timeout += StrLength(pMsg->msgtext) / 1024;
 582   }
 583 */
 584                 break;
 585         case eReadMessage:
 586                 Timeout = POP3_C_ReadTimeouts[pMsg->State];
 587 /*
 588   if (pMsg->State == eDATATerminateBody) {
 589   / *
 590   * some mailservers take a nap before accepting the message
 591   * content inspection and such.
 592   * /
 593   Timeout += StrLength(pMsg->msgtext) / 1024;
 594   }
 595 */
 596                 break;
 597         case eReadPayload:
 598                 Timeout = 100000;
 599                 /* TODO!!! */
 600                 break;
 601         case eSendDNSQuery:
 602         case eReadDNSReply:
 603         case eConnect:
 604         case eTerminateConnection:
 605         case eDBQuery:
 606         case eAbort:
 607         case eReadMore://// TODO
 608                 return;
 609         }
 610         SetNextTimeout(&pMsg->IO, Timeout);
 611 }
 612 eNextState POP3_C_DispatchReadDone(AsyncIO *IO)
 613 {
 614         CtdlLogPrintf(CTDL_DEBUG, "POP3: %s\n", __FUNCTION__);
 615         pop3aggr *pMsg = IO->Data;
 616         eNextState rc;
 617
 618         rc = POP3C_ReadHandlers[pMsg->State](pMsg);
 619         if (rc != eReadMore)
 620             pMsg->State++;
 621         POP3SetTimeout(rc, pMsg);
 622         return rc;
 623 }
 624 eNextState POP3_C_DispatchWriteDone(AsyncIO *IO)
 625 {
 626         CtdlLogPrintf(CTDL_DEBUG, "POP3: %s\n", __FUNCTION__);
 627         pop3aggr *pMsg = IO->Data;
 628         eNextState rc;
 629
 630         rc = POP3C_SendHandlers[pMsg->State](pMsg);
 631         POP3SetTimeout(rc, pMsg);
 632         return rc;
 633 }
 634
 635
 636 /*****************************************************************************/
 637 /*                     POP3 CLIENT ERROR CATCHERS                            */
 638 /*****************************************************************************/
 639 eNextState POP3_C_Terminate(AsyncIO *IO)
 640 {
 641 ///     pop3aggr *pMsg = (pop3aggr *)IO->Data;
 642
 643         CtdlLogPrintf(CTDL_DEBUG, "POP3: %s\n", __FUNCTION__);
 644         FinalizePOP3AggrRun(IO);
 645         return eAbort;
 646 }
 647 eNextState POP3_C_Timeout(AsyncIO *IO)
 648 {
 649         pop3aggr *pMsg = IO->Data;
 650
 651         CtdlLogPrintf(CTDL_DEBUG, "POP3: %s\n", __FUNCTION__);
 652         StrBufPlain(IO->ErrMsg, CKEY(POP3C_ReadErrors[pMsg->State]));
 653         return FailAggregationRun(IO);
 654 }
 655 eNextState POP3_C_ConnFail(AsyncIO *IO)
 656 {
 657         pop3aggr *pMsg = (pop3aggr *)IO->Data;
 658
 659         CtdlLogPrintf(CTDL_DEBUG, "POP3: %s\n", __FUNCTION__);
 660         StrBufPlain(IO->ErrMsg, CKEY(POP3C_ReadErrors[pMsg->State]));
 661         return FailAggregationRun(IO);
 662 }
 663 eNextState POP3_C_Shutdown(AsyncIO *IO)
 664 {
 665         CtdlLogPrintf(CTDL_DEBUG, "POP3: %s\n", __FUNCTION__);
 666 ////    pop3aggr *pMsg = IO->Data;
 667
 668         ////pMsg->MyQEntry->Status = 3;
 669         ///StrBufPlain(pMsg->MyQEntry->StatusMessage, HKEY("server shutdown during message retrieval."));
 670         FinalizePOP3AggrRun(IO);
 671         return eAbort;
 672 }
 673
 674
 675 /**
 676  * @brief lineread Handler; understands when to read more POP3 lines, and when this is a one-lined reply.
 677  */
 678 eReadState POP3_C_ReadServerStatus(AsyncIO *IO)
 679 {
 680         eReadState Finished = eBufferNotEmpty;
 681
 682         switch (IO->NextState) {
 683         case eSendDNSQuery:
 684         case eReadDNSReply:
 685         case eDBQuery:
 686         case eConnect:
 687         case eTerminateConnection:
 688         case eAbort:
 689                 Finished = eReadFail;
 690                 break;
 691         case eSendReply:
 692         case eSendMore:
 693         case eReadMore:
 694         case eReadMessage:
 695                 Finished = StrBufChunkSipLine(IO->IOBuf, &IO->RecvBuf);
 696                 break;
 697         case eReadPayload:
 698                 Finished = CtdlReadMessageBodyAsync(IO);
 699                 break;
 700         }
 701         return Finished;
 702 }
 703
 704 /*****************************************************************************
 705  * So we connect our Server IP here.                                         *
 706  *****************************************************************************/
 707 eNextState POP3_C_ReAttachToFetchMessages(AsyncIO *IO)
 708 {
 709         pop3aggr *cpptr = IO->Data;
 710
 711         CtdlLogPrintf(CTDL_DEBUG, "POP3: %s\n", __FUNCTION__);
 712 ////??? cpptr->State ++;
 713         if (cpptr->Pos == NULL)
 714                 cpptr->Pos = GetNewHashPos(cpptr->MsgNumbers, 0);
 715
 716         POP3_C_DispatchWriteDone(IO);
 717         ReAttachIO(IO, cpptr, 0);
 718         IO->NextState = eReadMessage;
 719         return IO->NextState;
 720 }
 721
 722 eNextState connect_ip(AsyncIO *IO)
 723 {
 724         pop3aggr *cpptr = IO->Data;
 725
 726         CtdlLogPrintf(CTDL_DEBUG, "POP3: %s\n", __FUNCTION__);
 727
 728 ////    IO->ConnectMe = &cpptr->Pop3Host;
 729         /*  Bypass the ns lookup result like this: IO->Addr.sin_addr.s_addr = inet_addr("127.0.0.1"); */
 730
 731         /////// SetConnectStatus(IO);
 732
 733         return InitEventIO(IO, cpptr,
 734                            POP3_C_ConnTimeout,
 735                            POP3_C_ReadTimeouts[0],
 736                            1);
 737 }
 738
 739 eNextState get_one_host_ip_done(AsyncIO *IO)
 740 {
 741         pop3aggr *cpptr = IO->Data;
 742         struct hostent *hostent;
 743
 744         QueryCbDone(IO);
 745
 746         hostent = cpptr->HostLookup.VParsedDNSReply;
 747         if ((cpptr->HostLookup.DNSStatus == ARES_SUCCESS) &&
 748             (hostent != NULL) ) {
 749                 memset(&cpptr->Pop3Host.Addr, 0, sizeof(struct in6_addr));
 750                 if (cpptr->Pop3Host.IPv6) {
 751                         memcpy(&cpptr->Pop3Host.Addr.sin6_addr.s6_addr,
 752                                &hostent->h_addr_list[0],
 753                                sizeof(struct in6_addr));
 754
 755                         cpptr->Pop3Host.Addr.sin6_family = hostent->h_addrtype;
 756                         cpptr->Pop3Host.Addr.sin6_port   = htons(DefaultPOP3Port);
 757                 }
 758                 else {
 759                         struct sockaddr_in *addr = (struct sockaddr_in*) &cpptr->Pop3Host.Addr;
 760                         /* Bypass the ns lookup result like this: IO->Addr.sin_addr.s_addr = inet_addr("127.0.0.1"); */
 761 //                      addr->sin_addr.s_addr = htonl((uint32_t)&hostent->h_addr_list[0]);
 762                         memcpy(&addr->sin_addr.s_addr,
 763                                hostent->h_addr_list[0],
 764                                sizeof(uint32_t));
 765
 766                         addr->sin_family = hostent->h_addrtype;
 767                         addr->sin_port   = htons(DefaultPOP3Port);
 768
 769                 }
 770                 return connect_ip(IO);
 771         }
 772         else
 773                 return eAbort;
 774 }
 775
 776 eNextState get_one_host_ip(AsyncIO *IO)
 777 {
 778         pop3aggr *cpptr = IO->Data;
 779         /*
 780          * here we start with the lookup of one host. it might be...
 781          * - the relay host *sigh*
 782          * - the direct hostname if there was no mx record
 783          * - one of the mx'es
 784          */
 785
 786         InitC_ares_dns(IO);
 787
 788         CtdlLogPrintf(CTDL_DEBUG, "POP3: %s\n", __FUNCTION__);
 789
 790         CtdlLogPrintf(CTDL_DEBUG,
 791                       "POP3 client[%ld]: looking up %s-Record %s : %d ...\n",
 792                       cpptr->n,
 793                       (cpptr->Pop3Host.IPv6)? "aaaa": "a",
 794                       cpptr->Pop3Host.Host,
 795                       cpptr->Pop3Host.Port);
 796
 797         if (!QueueQuery((cpptr->Pop3Host.IPv6)? ns_t_aaaa : ns_t_a,
 798                         cpptr->Pop3Host.Host,
 799                         &cpptr->IO,
 800                         &cpptr->HostLookup,
 801                         get_one_host_ip_done))
 802         {
 803 //              cpptr->MyQEntry->Status = 5;
 804 //              StrBufPrintf(SendMsg->MyQEntry->StatusMessage,
 805 //                           "No MX hosts found for <%s>", SendMsg->node);
 806                 cpptr->IO.NextState = eTerminateConnection;
 807                 return IO->NextState;
 808         }
 809         IO->NextState = eReadDNSReply;
 810         return IO->NextState;
 811 }
 812
 813
 814
 815 int pop3_do_fetching(pop3aggr *cpptr)
 816 {
 817         CitContext *SubC;
 818
 819         cpptr->IO.Data          = cpptr;
 820
 821         cpptr->IO.SendDone      = POP3_C_DispatchWriteDone;
 822         cpptr->IO.ReadDone      = POP3_C_DispatchReadDone;
 823         cpptr->IO.Terminate     = POP3_C_Terminate;
 824         cpptr->IO.LineReader    = POP3_C_ReadServerStatus;
 825         cpptr->IO.ConnFail      = POP3_C_ConnFail;
 826         cpptr->IO.Timeout       = POP3_C_Timeout;
 827         cpptr->IO.ShutdownAbort = POP3_C_Shutdown;
 828
 829         cpptr->IO.SendBuf.Buf   = NewStrBufPlain(NULL, 1024);
 830         cpptr->IO.RecvBuf.Buf   = NewStrBufPlain(NULL, 1024);
 831         cpptr->IO.IOBuf         = NewStrBuf();
 832
 833         cpptr->IO.NextState     = eReadMessage;
 834 /* TODO
 835    CtdlLogPrintf(CTDL_DEBUG, "POP3: %s %s %s <password>\n", roomname, pop3host, pop3user);
 836    CtdlLogPrintf(CTDL_NOTICE, "Connecting to <%s>\n", pop3host);
 837 */
 838
 839         SubC = CloneContext (CC);
 840         SubC->session_specific_data = (char*) cpptr;
 841         cpptr->IO.CitContext = SubC;
 842
 843         if (cpptr->IO.ConnectMe->IsIP) {
 844                 QueueEventContext(&cpptr->IO,
 845                                   connect_ip);
 846         }
 847         else { /* uneducated admin has chosen to add DNS to the equation... */
 848                 QueueEventContext(&cpptr->IO,
 849                                   get_one_host_ip);
 850         }
 851         return 1;
 852 }
 853
 854 /*
 855  * Scan a room's netconfig to determine whether it requires POP3 aggregation
 856  */
 857 void pop3client_scan_room(struct ctdlroom *qrbuf, void *data)
 858 {
 859         StrBuf *CfgData;
 860         StrBuf *CfgType;
 861         StrBuf *Line;
 862
 863         struct stat statbuf;
 864         char filename[PATH_MAX];
 865         int  fd;
 866         int Done;
 867         void *vptr;
 868         const char *CfgPtr, *lPtr;
 869         const char *Err;
 870
 871         pop3_room_counter *Count = NULL;
 872 //      pop3aggr *cpptr;
 873
 874         citthread_mutex_lock(&POP3QueueMutex);
 875         if (GetHash(POP3QueueRooms, LKEY(qrbuf->QRnumber), &vptr))
 876         {
 877                 CtdlLogPrintf(CTDL_DEBUG,
 878                               "pop3client: [%ld] %s already in progress.\n",
 879                               qrbuf->QRnumber,
 880                               qrbuf->QRname);
 881                 citthread_mutex_unlock(&POP3QueueMutex);
 882                 return;
 883         }
 884         citthread_mutex_unlock(&POP3QueueMutex);
 885
 886         assoc_file_name(filename, sizeof filename, qrbuf, ctdl_netcfg_dir);
 887
 888         if (CtdlThreadCheckStop())
 889                 return;
 890
 891         /* Only do net processing for rooms that have netconfigs */
 892         fd = open(filename, 0);
 893         if (fd <= 0) {
 894                 //CtdlLogPrintf(CTDL_DEBUG, "rssclient: %s no config.\n", qrbuf->QRname);
 895                 return;
 896         }
 897         if (CtdlThreadCheckStop())
 898                 return;
 899         if (fstat(fd, &statbuf) == -1) {
 900                 CtdlLogPrintf(CTDL_DEBUG,  "ERROR: could not stat configfile '%s' - %s\n",
 901                               filename, strerror(errno));
 902                 return;
 903         }
 904         if (CtdlThreadCheckStop())
 905                 return;
 906         CfgData = NewStrBufPlain(NULL, statbuf.st_size + 1);
 907         if (StrBufReadBLOB(CfgData, &fd, 1, statbuf.st_size, &Err) < 0) {
 908                 close(fd);
 909                 FreeStrBuf(&CfgData);
 910                 CtdlLogPrintf(CTDL_DEBUG,  "ERROR: reading config '%s' - %s<br>\n",
 911                               filename, strerror(errno));
 912                 return;
 913         }
 914         close(fd);
 915         if (CtdlThreadCheckStop())
 916                 return;
 917
 918         CfgPtr = NULL;
 919         CfgType = NewStrBuf();
 920         Line = NewStrBufPlain(NULL, StrLength(CfgData));
 921         Done = 0;
 922
 923         while (!Done)
 924         {
 925                 Done = StrBufSipLine(Line, CfgData, &CfgPtr) == 0;
 926                 if (StrLength(Line) > 0)
 927                 {
 928                         lPtr = NULL;
 929                         StrBufExtract_NextToken(CfgType, Line, &lPtr, '|');
 930                         if (!strcasecmp("pop3client", ChrPtr(CfgType)))
 931                         {
 932                                 pop3aggr *cptr;
 933
 934                                 if (Count == NULL)
 935                                 {
 936                                         Count = malloc(sizeof(pop3_room_counter));
 937                                         Count->count = 0;
 938                                 }
 939                                 Count->count ++;
 940                                 cptr = (pop3aggr *) malloc(sizeof(pop3aggr));
 941                                 memset(cptr, 0, sizeof(pop3aggr));
 942                                 /// TODO do we need this? cptr->roomlist_parts = 1;
 943                                 cptr->RoomName = NewStrBufPlain(qrbuf->QRname, -1);
 944                                 cptr->pop3user = NewStrBufPlain(NULL, StrLength(Line));
 945                                 cptr->pop3pass = NewStrBufPlain(NULL, StrLength(Line));
 946                                 cptr->Url = NewStrBuf();
 947
 948                                 StrBufExtract_NextToken(cptr->Url, Line, &lPtr, '|');
 949                                 StrBufExtract_NextToken(cptr->pop3user, Line, &lPtr, '|');
 950                                 StrBufExtract_NextToken(cptr->pop3pass, Line, &lPtr, '|');
 951                                 cptr->keep = StrBufExtractNext_long(Line, &lPtr, '|');
 952                                 cptr->interval = StrBufExtractNext_long(Line, &lPtr, '|');
 953
 954                                 ParseURL(&cptr->IO.ConnectMe, cptr->Url, 110);
 955
 956                                 cptr->IO.ConnectMe->CurlCreds = cptr->pop3user;
 957                                 cptr->IO.ConnectMe->User = ChrPtr(cptr->IO.ConnectMe->CurlCreds);
 958                                 cptr->IO.ConnectMe->UrlWithoutCred = cptr->pop3pass;
 959                                 cptr->IO.ConnectMe->Pass = ChrPtr(cptr->IO.ConnectMe->UrlWithoutCred);
 960
 961
 962
 963 #if 0
 964 /* todo: we need to reunite the url to be shure. */
 965
 966                                 citthread_mutex_lock(&POP3ueueMutex);
 967                                 GetHash(POP3FetchUrls, SKEY(ptr->Url), &vptr);
 968                                 use_this_cptr = (pop3aggr *)vptr;
 969
 970                                 if (use_this_rncptr != NULL)
 971                                 {
 972                                         /* mustn't attach to an active session */
 973                                         if (use_this_cptr->RefCount > 0)
 974                                         {
 975                                                 DeletePOP3Cfg(cptr);
 976                                                 Count->count--;
 977                                         }
 978                                         else
 979                                         {
 980                                                 long *QRnumber;
 981                                                 StrBufAppendBufPlain(use_this_cptr->rooms,
 982                                                                      qrbuf->QRname,
 983                                                                      -1, 0);
 984                                                 if (use_this_cptr->roomlist_parts == 1)
 985                                                 {
 986                                                         use_this_cptr->OtherQRnumbers = NewHash(1, lFlathash);
 987                                                 }
 988                                                 QRnumber = (long*)malloc(sizeof(long));
 989                                                 *QRnumber = qrbuf->QRnumber;
 990                                                 Put(use_this_cptr->OtherQRnumbers, LKEY(qrbuf->QRnumber), QRnumber, NULL);
 991                                                 use_this_cptr->roomlist_parts++;
 992                                         }
 993                                         citthread_mutex_unlock(&POP3QueueMutex);
 994                                         continue;
 995                                 }
 996                                 citthread_mutex_unlock(&RSSQueueMutex);
 997 #endif
 998
 999                                 citthread_mutex_lock(&POP3QueueMutex);
1000                                 Put(POP3FetchUrls, SKEY(cptr->Url), cptr, DeletePOP3Aggregator);
1001                                 citthread_mutex_unlock(&POP3QueueMutex);
1002
1003                         }
1004
1005                 }
1006
1007                 ///fclose(fp);
1008
1009         }
1010 }
1011
1012
1013 void pop3client_scan(void) {
1014         static time_t last_run = 0L;
1015         static int doing_pop3client = 0;
1016 ///     struct pop3aggr *pptr;
1017         time_t fastest_scan;
1018         HashPos *it;
1019         long len;
1020         const char *Key;
1021         void *vrptr;
1022         pop3aggr *cptr;
1023
1024         if (config.c_pop3_fastest < config.c_pop3_fetch)
1025                 fastest_scan = config.c_pop3_fastest;
1026         else
1027                 fastest_scan = config.c_pop3_fetch;
1028
1029         /*
1030          * Run POP3 aggregation no more frequently than once every n seconds
1031          */
1032         if ( (time(NULL) - last_run) < fastest_scan ) {
1033                 return;
1034         }
1035
1036         /*
1037          * This is a simple concurrency check to make sure only one pop3client run
1038          * is done at a time.  We could do this with a mutex, but since we
1039          * don't really require extremely fine granularity here, we'll do it
1040          * with a static variable instead.
1041          */
1042         if (doing_pop3client) return;
1043         doing_pop3client = 1;
1044
1045         CtdlLogPrintf(CTDL_DEBUG, "pop3client started\n");
1046         CtdlForEachRoom(pop3client_scan_room, NULL);
1047
1048
1049         citthread_mutex_lock(&POP3QueueMutex);
1050         it = GetNewHashPos(POP3FetchUrls, 0);
1051         while (GetNextHashPos(POP3FetchUrls, it, &len, &Key, &vrptr) &&
1052                (vrptr != NULL)) {
1053                 cptr = (pop3aggr *)vrptr;
1054                 if (cptr->RefCount == 0)
1055                         if (!pop3_do_fetching(cptr))
1056                                 DeletePOP3Aggregator(cptr);////TODO
1057         }
1058         DeleteHashPos(&it);
1059         citthread_mutex_unlock(&POP3QueueMutex);
1060
1061         CtdlLogPrintf(CTDL_DEBUG, "pop3client ended\n");
1062         last_run = time(NULL);
1063         doing_pop3client = 0;
1064 }
1065
1066
1067 void pop3_cleanup(void)
1068 {
1069         citthread_mutex_destroy(&POP3QueueMutex);
1070         DeleteHash(&POP3FetchUrls);
1071         DeleteHash(&POP3QueueRooms);
1072 }
1073
1074 CTDL_MODULE_INIT(pop3client)
1075 {
1076         if (!threading)
1077         {
1078                 citthread_mutex_init(&POP3QueueMutex, NULL);
1079                 POP3QueueRooms = NewHash(1, lFlathash);
1080                 POP3FetchUrls = NewHash(1, NULL);
1081                 CtdlRegisterSessionHook(pop3client_scan, EVT_TIMER);
1082                 CtdlRegisterCleanupHook(pop3_cleanup);
1083         }
1084         /* return our Subversion id for the Log */
1085         return "pop3client";
1086 }