citadel/modules/pop3client/serv_pop3client.c

   1 /*
   2  * Consolidate mail from remote POP3 accounts.
   3  *
   4  * Copyright (c) 2007-2009 by the citadel.org team
   5  *
   6  *  This program is free software; you can redistribute it and/or modify
   7  *  it under the terms of the GNU General Public License as published by
   8  *  the Free Software Foundation; either version 3 of the License, or
   9  *  (at your option) any later version.
  10  *
  11  *  This program is distributed in the hope that it will be useful,
  12  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
  13  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14  *  GNU General Public License for more details.
  15  *
  16  *  You should have received a copy of the GNU General Public License
  17  *  along with this program; if not, write to the Free Software
  18  *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  19  */
  20
  21 #include <stdlib.h>
  22 #include <unistd.h>
  23 #include <stdio.h>
  24
  25 #if TIME_WITH_SYS_TIME
  26 # include <sys/time.h>
  27 # include <time.h>
  28 #else
  29 # if HAVE_SYS_TIME_H
  30 #  include <sys/time.h>
  31 # else
  32 #  include <time.h>
  33 # endif
  34 #endif
  35
  36 #include <ctype.h>
  37 #include <string.h>
  38 #include <errno.h>
  39 #include <sys/types.h>
  40 #include <sys/stat.h>
  41 #include <libcitadel.h>
  42 #include "citadel.h"
  43 #include "server.h"
  44 #include "citserver.h"
  45 #include "support.h"
  46 #include "config.h"
  47 #include "ctdl_module.h"
  48 #include "clientsocket.h"
  49 #include "msgbase.h"
  50 #include "internet_addressing.h"
  51 #include "database.h"
  52 #include "citadel_dirs.h"
  53 #include "event_client.h"
  54
  55
  56
  57 citthread_mutex_t POP3QueueMutex; /* locks the access to the following vars: */
  58 HashList *POP3QueueRooms = NULL; /* rss_room_counter */
  59 HashList *POP3FetchUrls = NULL; /* -> rss_aggregator; ->RefCount access to be locked too. */
  60
  61 typedef struct __pop3_room_counter {
  62         int count;
  63         long QRnumber;
  64 }pop3_room_counter;
  65
  66 typedef enum ePOP3_C_States {
  67         ReadGreeting,
  68         GetUserState,
  69         GetPassState,
  70         GetListCommandState,
  71         GetListOneLine,
  72         GetOneMessageIDState,
  73         ReadMessageBodyFollowing,
  74         ReadMessageBody,
  75         GetDeleteState,
  76         ReadQuitState,
  77         POP3C_MaxRead
  78 }ePOP3_C_States;
  79
  80
  81 typedef struct _FetchItem {
  82         long MSGID;
  83         long MSGSize;
  84         StrBuf *MsgUIDL;
  85         StrBuf *MsgUID;
  86         int NeedFetch;
  87         struct CtdlMessage *Msg;
  88 } FetchItem;
  89
  90 void HfreeFetchItem(void *vItem)
  91 {
  92         FetchItem *Item = (FetchItem*) vItem;
  93         FreeStrBuf(&Item->MsgUIDL);
  94         FreeStrBuf(&Item->MsgUID);
  95         free(Item);
  96 }
  97
  98 typedef struct __pop3aggr {
  99         AsyncIO          IO;
 100
 101         long n;
 102         long RefCount;
 103         ParsedURL Pop3Host;
 104         DNSQueryParts HostLookup;
 105
 106         StrBuf          *rooms;
 107         long             QRnumber;
 108         HashList        *OtherQRnumbers;
 109
 110         StrBuf          *Url;
 111 ///     StrBuf *pop3host; -> URL
 112         StrBuf *pop3user;
 113         StrBuf *pop3pass;
 114         StrBuf *RoomName; // TODO: fill me
 115         int keep;
 116         time_t interval;
 117         ePOP3_C_States State;
 118         HashList *MsgNumbers;
 119         HashPos *Pos;
 120         FetchItem *CurrMsg;
 121 } pop3aggr;
 122
 123 void DeletePOP3Aggregator(void *vptr)
 124 {
 125         pop3aggr *ptr = vptr;
 126         DeleteHashPos(&ptr->Pos);
 127         DeleteHash(&ptr->MsgNumbers);
 128         FreeStrBuf(&ptr->rooms);
 129         FreeStrBuf(&ptr->pop3user);
 130         FreeStrBuf(&ptr->pop3pass);
 131         FreeStrBuf(&ptr->RoomName);
 132 }
 133
 134
 135 typedef eNextState(*Pop3ClientHandler)(pop3aggr* RecvMsg);
 136
 137 eNextState POP3_C_Shutdown(AsyncIO *IO);
 138 eNextState POP3_C_Timeout(AsyncIO *IO);
 139 eNextState POP3_C_ConnFail(AsyncIO *IO);
 140 eNextState POP3_C_DispatchReadDone(AsyncIO *IO);
 141 eNextState POP3_C_DispatchWriteDone(AsyncIO *IO);
 142 eNextState POP3_C_Terminate(AsyncIO *IO);
 143 eReadState POP3_C_ReadServerStatus(AsyncIO *IO);
 144 eNextState POP3_C_ReAttachToFetchMessages(AsyncIO *IO);
 145
 146 eNextState FinalizePOP3AggrRun(AsyncIO *IO)
 147 {
 148
 149         return eAbort;
 150 }
 151
 152 eNextState FailAggregationRun(AsyncIO *IO)
 153 {
 154         return eAbort;
 155 }
 156
 157 #define POP3C_DBG_SEND() CtdlLogPrintf(CTDL_DEBUG, "POP3 client[%ld]: > %s\n", RecvMsg->n, ChrPtr(RecvMsg->IO.SendBuf.Buf))
 158 #define POP3C_DBG_READ() CtdlLogPrintf(CTDL_DEBUG, "POP3 client[%ld]: < %s\n", RecvMsg->n, ChrPtr(RecvMsg->IO.IOBuf))
 159 #define POP3C_OK (strncasecmp(ChrPtr(RecvMsg->IO.IOBuf), "+OK", 3) == 0)
 160
 161 eNextState POP3C_ReadGreeting(pop3aggr *RecvMsg)
 162 {
 163         POP3C_DBG_READ();
 164         /* Read the server greeting */
 165         if (!POP3C_OK) return eTerminateConnection;
 166         else return eSendReply;
 167 }
 168
 169
 170 eNextState POP3C_SendUser(pop3aggr *RecvMsg)
 171 {
 172         /* Identify ourselves.  NOTE: we have to append a CR to each command.  The LF will
 173          * automatically be appended by sock_puts().  Believe it or not, leaving out the CR
 174          * will cause problems if the server happens to be Exchange, which is so b0rken it
 175          * actually barfs on LF-terminated newlines.
 176          */
 177         StrBufPrintf(RecvMsg->IO.SendBuf.Buf,
 178                      "USER %s\r\n", ChrPtr(RecvMsg->pop3user));
 179         POP3C_DBG_SEND();
 180         return eReadMessage;
 181 }
 182
 183 eNextState POP3C_GetUserState(pop3aggr *RecvMsg)
 184 {
 185         POP3C_DBG_READ();
 186         if (!POP3C_OK) return eTerminateConnection;
 187         else return eSendReply;
 188 }
 189
 190 eNextState POP3C_SendPassword(pop3aggr *RecvMsg)
 191 {
 192         /* Password */
 193         StrBufPrintf(RecvMsg->IO.SendBuf.Buf,
 194                      "PASS %s\r\n", ChrPtr(RecvMsg->pop3pass));
 195         CtdlLogPrintf(CTDL_DEBUG, "<PASS <password>\n");
 196 //      POP3C_DBG_SEND();
 197         return eReadMessage;
 198 }
 199
 200 eNextState POP3C_GetPassState(pop3aggr *RecvMsg)
 201 {
 202         POP3C_DBG_READ();
 203         if (!POP3C_OK) return eTerminateConnection;
 204         else return eSendReply;
 205 }
 206
 207 eNextState POP3C_SendListCommand(pop3aggr *RecvMsg)
 208 {
 209         /* Get the list of messages */
 210         StrBufPlain(RecvMsg->IO.SendBuf.Buf, HKEY("LIST\r\n"));
 211         POP3C_DBG_SEND();
 212         return eReadMessage;
 213 }
 214
 215 eNextState POP3C_GetListCommandState(pop3aggr *RecvMsg)
 216 {
 217         POP3C_DBG_READ();
 218         if (!POP3C_OK) return eTerminateConnection;
 219         RecvMsg->MsgNumbers = NewHash(1, NULL);
 220         RecvMsg->State++;
 221         return eReadMore;
 222 }
 223
 224
 225 eNextState POP3C_GetListOneLine(pop3aggr *RecvMsg)
 226 {
 227         const char *pch;
 228         FetchItem *OneMsg = NULL;
 229         POP3C_DBG_READ();
 230
 231         if ((StrLength(RecvMsg->IO.IOBuf) == 1) &&
 232             (ChrPtr(RecvMsg->IO.IOBuf)[0] == '.'))
 233         {
 234                 if (GetCount(RecvMsg->MsgNumbers) == 0)
 235                 {
 236                         ////    RecvMsg->Sate = ReadQuitState;
 237                 }
 238                 else
 239                 {
 240                         RecvMsg->Pos = GetNewHashPos(RecvMsg->MsgNumbers, 0);
 241                 }
 242                 return eSendReply;
 243
 244         }
 245         OneMsg = (FetchItem*) malloc(sizeof(FetchItem));
 246         memset(OneMsg, 0, sizeof(FetchItem));
 247         OneMsg->MSGID = atol(ChrPtr(RecvMsg->IO.IOBuf));
 248
 249         pch = strchr(ChrPtr(RecvMsg->IO.IOBuf), ' ');
 250         if (pch != NULL)
 251         {
 252                 OneMsg->MSGSize = atol(pch + 1);
 253         }
 254         Put(RecvMsg->MsgNumbers, LKEY(OneMsg->MSGID), OneMsg, HfreeFetchItem);
 255
 256         //RecvMsg->State --; /* read next Line */
 257         return eReadMore;
 258 }
 259
 260 eNextState POP3_FetchNetworkUsetableEntry(AsyncIO *IO)
 261 {
 262         long HKLen;
 263         const char *HKey;
 264         void *vData;
 265         struct cdbdata *cdbut;
 266         pop3aggr *RecvMsg = (pop3aggr *) IO->Data;
 267
 268         if(GetNextHashPos(RecvMsg->MsgNumbers, RecvMsg->Pos, &HKLen, &HKey, &vData))
 269         {
 270                 struct UseTable ut;
 271
 272                 RecvMsg->CurrMsg = (FetchItem*) vData;
 273                 /* Find out if we've already seen this item */
 274                 safestrncpy(ut.ut_msgid,
 275                             ChrPtr(RecvMsg->CurrMsg->MsgUIDL),
 276                             sizeof(ut.ut_msgid));
 277                 ut.ut_timestamp = time(NULL);/// TODO: libev timestamp!
 278
 279                 cdbut = cdb_fetch(CDB_USETABLE, SKEY(RecvMsg->CurrMsg->MsgUIDL));
 280                 if (cdbut != NULL) {
 281                         /* Item has already been seen */
 282                         CtdlLogPrintf(CTDL_DEBUG, "%s has already been seen\n", ChrPtr(RecvMsg->CurrMsg->MsgUIDL));
 283                         cdb_free(cdbut);
 284
 285                         /* rewrite the record anyway, to update the timestamp */
 286                         cdb_store(CDB_USETABLE,
 287                                   SKEY(RecvMsg->CurrMsg->MsgUIDL),
 288                                   &ut, sizeof(struct UseTable) );
 289                         RecvMsg->CurrMsg->NeedFetch = 0;
 290                 }
 291                 else
 292                 {
 293                         RecvMsg->CurrMsg->NeedFetch = 1;
 294                 }
 295                 return NextDBOperation(&RecvMsg->IO, POP3_FetchNetworkUsetableEntry);
 296         }
 297         else
 298         {
 299                 /* ok, now we know them all, continue with reading the actual messages. */
 300                 DeleteHashPos(&RecvMsg->Pos);
 301
 302                 return QueueEventContext(IO, POP3_C_ReAttachToFetchMessages);
 303         }
 304 }
 305
 306 eNextState POP3C_GetOneMessagID(pop3aggr *RecvMsg)
 307 {
 308         long HKLen;
 309         const char *HKey;
 310         void *vData;
 311
 312         if(GetNextHashPos(RecvMsg->MsgNumbers, RecvMsg->Pos, &HKLen, &HKey, &vData))
 313         {
 314                 RecvMsg->CurrMsg = (FetchItem*) vData;
 315                 /* Find out the UIDL of the message, to determine whether we've already downloaded it */
 316                 StrBufPrintf(RecvMsg->IO.SendBuf.Buf,
 317                              "UIDL %ld\r\n", RecvMsg->CurrMsg->MSGID);
 318                 POP3C_DBG_SEND();
 319         }
 320         else
 321         {
 322             RecvMsg->State++;
 323                 DeleteHashPos(&RecvMsg->Pos);
 324                 /// done receiving uidls.. start looking them up now.
 325                 RecvMsg->Pos = GetNewHashPos(RecvMsg->MsgNumbers, 0);
 326                 return QueueDBOperation(&RecvMsg->IO, POP3_FetchNetworkUsetableEntry);
 327         }
 328         return eReadMore; /* TODO */
 329 }
 330
 331 eNextState POP3C_GetOneMessageIDState(pop3aggr *RecvMsg)
 332 {
 333         POP3C_DBG_READ();
 334         if (!POP3C_OK) return eTerminateConnection;
 335         RecvMsg->CurrMsg->MsgUIDL = NewStrBufPlain(NULL, StrLength(RecvMsg->IO.IOBuf));
 336         RecvMsg->CurrMsg->MsgUID = NewStrBufPlain(NULL, StrLength(RecvMsg->IO.IOBuf) * 2);
 337
 338         StrBufExtract_token(RecvMsg->CurrMsg->MsgUIDL, RecvMsg->IO.IOBuf, 2, ' ');
 339         StrBufPrintf(RecvMsg->CurrMsg->MsgUID,
 340                      "pop3/%s/%s@%s",
 341                      ChrPtr(RecvMsg->RoomName),
 342                      ChrPtr(RecvMsg->CurrMsg->MsgUIDL),
 343                      RecvMsg->Pop3Host.Host);
 344         RecvMsg->State --;
 345         return eSendReply;
 346 }
 347
 348 eNextState POP3C_GetOneMessageIDFromUseTable(pop3aggr *RecvMsg)
 349 {
 350
 351         struct cdbdata *cdbut;
 352         struct UseTable ut;
 353
 354         cdbut = cdb_fetch(CDB_USETABLE, SKEY(RecvMsg->CurrMsg->MsgUID));
 355         if (cdbut != NULL) {
 356                 /* message has already been seen */
 357                 CtdlLogPrintf(CTDL_DEBUG, "%s has already been seen\n", ChrPtr(RecvMsg->CurrMsg->MsgUID));
 358                 cdb_free(cdbut);
 359
 360                 /* rewrite the record anyway, to update the timestamp */
 361                 strcpy(ut.ut_msgid, ChrPtr(RecvMsg->CurrMsg->MsgUID));
 362                 ut.ut_timestamp = time(NULL);
 363                 cdb_store(CDB_USETABLE, SKEY(RecvMsg->CurrMsg->MsgUID), &ut, sizeof(struct UseTable) );
 364         }
 365
 366         return eReadMessage;
 367 }
 368
 369 eNextState POP3C_SendGetOneMsg(pop3aggr *RecvMsg)
 370 {
 371         long HKLen;
 372         const char *HKey;
 373         void *vData;
 374
 375         RecvMsg->CurrMsg = NULL;
 376         while (GetNextHashPos(RecvMsg->MsgNumbers, RecvMsg->Pos, &HKLen, &HKey, &vData) &&
 377                (RecvMsg->CurrMsg = (FetchItem*) vData, RecvMsg->CurrMsg->NeedFetch == 0))
 378         {}
 379
 380         if ((RecvMsg->CurrMsg != NULL ) && (RecvMsg->CurrMsg->NeedFetch == 1))
 381         {
 382                 /* Message has not been seen. Tell the server to fetch the message... */
 383                 StrBufPrintf(RecvMsg->IO.SendBuf.Buf,
 384                              "RETR %ld\r\n", RecvMsg->CurrMsg->MSGID);
 385                 POP3C_DBG_SEND();
 386                 return eReadMessage;
 387         }
 388         else {
 389                 RecvMsg->State = ReadQuitState;
 390                 return POP3_C_DispatchWriteDone(&RecvMsg->IO);
 391         }
 392 }
 393
 394
 395 eNextState POP3C_ReadMessageBodyFollowing(pop3aggr *RecvMsg)
 396 {
 397         POP3C_DBG_READ();
 398         if (!POP3C_OK) return eTerminateConnection;
 399         RecvMsg->IO.ReadMsg = NewAsyncMsg(HKEY("."),
 400                                           RecvMsg->CurrMsg->MSGSize,
 401                                           config.c_maxmsglen,
 402                                           NULL, -1,
 403                                           1);
 404
 405         return eReadPayload;
 406 }
 407
 408
 409 eNextState POP3C_StoreMsgRead(AsyncIO *IO)
 410 {
 411         pop3aggr *RecvMsg = (pop3aggr *) IO->Data;
 412         struct UseTable ut;
 413
 414         safestrncpy(ut.ut_msgid,
 415                     ChrPtr(RecvMsg->CurrMsg->MsgUID),
 416                     sizeof(ut.ut_msgid));
 417         ut.ut_timestamp = time(NULL); /* TODO: use libev time */
 418         cdb_store(CDB_USETABLE,
 419                   ChrPtr(RecvMsg->CurrMsg->MsgUID),
 420                   StrLength(RecvMsg->CurrMsg->MsgUID),
 421                   &ut,
 422                   sizeof(struct UseTable) );
 423
 424         return QueueEventContext(&RecvMsg->IO, POP3_C_ReAttachToFetchMessages);
 425 }
 426 eNextState POP3C_SaveMsg(AsyncIO *IO)
 427 {
 428         long msgnum;
 429         pop3aggr *RecvMsg = (pop3aggr *) IO->Data;
 430
 431         /* Do Something With It (tm) */
 432         msgnum = CtdlSubmitMsg(RecvMsg->CurrMsg->Msg,
 433                                NULL,
 434                                ChrPtr(RecvMsg->RoomName),
 435                                0);
 436         if (msgnum > 0L) {
 437                 /* Message has been committed to the store */
 438                 /* write the uidl to the use table so we don't fetch this message again */
 439         }
 440         CtdlFreeMessage(RecvMsg->CurrMsg->Msg);
 441
 442         return NextDBOperation(&RecvMsg->IO, POP3C_StoreMsgRead);
 443         return eReadMessage;
 444 }
 445
 446 eNextState POP3C_ReadMessageBody(pop3aggr *RecvMsg)
 447 {
 448         CtdlLogPrintf(CTDL_DEBUG, "Converting message...\n");
 449         RecvMsg->CurrMsg->Msg = convert_internet_message_buf(&RecvMsg->IO.ReadMsg->MsgBuf);
 450
 451         return QueueDBOperation(&RecvMsg->IO, POP3C_SaveMsg);
 452 }
 453
 454 eNextState POP3C_SendDelete(pop3aggr *RecvMsg)
 455 {
 456         if (!RecvMsg->keep) {
 457                 StrBufPrintf(RecvMsg->IO.SendBuf.Buf,
 458                              "DELE %ld\r\n", RecvMsg->CurrMsg->MSGID);
 459                 POP3C_DBG_SEND();
 460                 return eReadMessage;
 461         }
 462         else {
 463                 RecvMsg->State = ReadMessageBodyFollowing;
 464                 return POP3_C_DispatchWriteDone(&RecvMsg->IO);
 465         }
 466 }
 467 eNextState POP3C_ReadDeleteState(pop3aggr *RecvMsg)
 468 {
 469         POP3C_DBG_READ();
 470         RecvMsg->State = GetOneMessageIDState;
 471         return eReadMessage;
 472 }
 473
 474 eNextState POP3C_SendQuit(pop3aggr *RecvMsg)
 475 {
 476         /* Log out */
 477         StrBufPlain(RecvMsg->IO.SendBuf.Buf,
 478                     HKEY("QUIT\r\n3)"));
 479         POP3C_DBG_SEND();
 480         return eReadMessage;
 481 }
 482
 483
 484 eNextState POP3C_ReadQuitState(pop3aggr *RecvMsg)
 485 {
 486         POP3C_DBG_READ();
 487         return eAbort;
 488 }
 489
 490 const long POP3_C_ConnTimeout = 1000;
 491 const long DefaultPOP3Port = 110;
 492
 493 Pop3ClientHandler POP3C_ReadHandlers[] = {
 494         POP3C_ReadGreeting,
 495         POP3C_GetUserState,
 496         POP3C_GetPassState,
 497         POP3C_GetListCommandState,
 498         POP3C_GetListOneLine,
 499         POP3C_GetOneMessageIDState,
 500         POP3C_ReadMessageBodyFollowing,
 501         POP3C_ReadMessageBody,
 502         POP3C_ReadDeleteState,
 503         POP3C_ReadQuitState,
 504 };
 505
 506 const long POP3_C_SendTimeouts[POP3C_MaxRead] = {
 507         100,
 508         100,
 509         100,
 510         100,
 511         100,
 512         100,
 513         100,
 514         100
 515 };
 516 const ConstStr POP3C_ReadErrors[POP3C_MaxRead] = {
 517         {HKEY("Connection broken during ")},
 518         {HKEY("Connection broken during ")},
 519         {HKEY("Connection broken during ")},
 520         {HKEY("Connection broken during ")},
 521         {HKEY("Connection broken during ")},
 522         {HKEY("Connection broken during ")},
 523         {HKEY("Connection broken during ")},
 524         {HKEY("Connection broken during ")}
 525 };
 526
 527 Pop3ClientHandler POP3C_SendHandlers[] = {
 528         NULL, /* we don't send a greeting */
 529         POP3C_SendUser,
 530         POP3C_SendPassword,
 531         POP3C_SendListCommand,
 532         NULL,
 533         POP3C_GetOneMessagID,
 534         POP3C_SendGetOneMsg,
 535         NULL,
 536         POP3C_SendDelete,
 537         POP3C_SendQuit
 538 };
 539
 540 const long POP3_C_ReadTimeouts[] = {
 541         100,
 542         100,
 543         100,
 544         100,
 545         100,
 546         100,
 547         100,
 548         100,
 549         100,
 550         100
 551 };
 552 /*****************************************************************************/
 553 /*                     POP3 CLIENT DISPATCHER                                */
 554 /*****************************************************************************/
 555
 556 void POP3SetTimeout(eNextState NextTCPState, pop3aggr *pMsg)
 557 {
 558         double Timeout = 0.0;
 559
 560         CtdlLogPrintf(CTDL_DEBUG, "POP3: %s\n", __FUNCTION__);
 561
 562         switch (NextTCPState) {
 563         case eSendReply:
 564         case eSendMore:
 565                 Timeout = POP3_C_SendTimeouts[pMsg->State];
 566 /*
 567   if (pMsg->State == eDATABody) {
 568   / * if we're sending a huge message, we need more time. * /
 569   Timeout += StrLength(pMsg->msgtext) / 1024;
 570   }
 571 */
 572                 break;
 573         case eReadMessage:
 574                 Timeout = POP3_C_ReadTimeouts[pMsg->State];
 575 /*
 576   if (pMsg->State == eDATATerminateBody) {
 577   / *
 578   * some mailservers take a nap before accepting the message
 579   * content inspection and such.
 580   * /
 581   Timeout += StrLength(pMsg->msgtext) / 1024;
 582   }
 583 */
 584                 break;
 585         case eReadPayload:
 586                 Timeout = 100000;
 587                 /* TODO!!! */
 588                 break;
 589         case eSendDNSQuery:
 590         case eReadDNSReply:
 591         case eConnect:
 592         case eTerminateConnection:
 593         case eDBQuery:
 594         case eAbort:
 595         case eReadMore://// TODO
 596                 return;
 597         }
 598         SetNextTimeout(&pMsg->IO, Timeout);
 599 }
 600 eNextState POP3_C_DispatchReadDone(AsyncIO *IO)
 601 {
 602         CtdlLogPrintf(CTDL_DEBUG, "POP3: %s\n", __FUNCTION__);
 603         pop3aggr *pMsg = IO->Data;
 604         eNextState rc;
 605
 606         rc = POP3C_ReadHandlers[pMsg->State](pMsg);
 607         if (rc != eReadMore)
 608             pMsg->State++;
 609         POP3SetTimeout(rc, pMsg);
 610         return rc;
 611 }
 612 eNextState POP3_C_DispatchWriteDone(AsyncIO *IO)
 613 {
 614         CtdlLogPrintf(CTDL_DEBUG, "POP3: %s\n", __FUNCTION__);
 615         pop3aggr *pMsg = IO->Data;
 616         eNextState rc;
 617
 618         rc = POP3C_SendHandlers[pMsg->State](pMsg);
 619         POP3SetTimeout(rc, pMsg);
 620         return rc;
 621 }
 622
 623
 624 /*****************************************************************************/
 625 /*                     POP3 CLIENT ERROR CATCHERS                            */
 626 /*****************************************************************************/
 627 eNextState POP3_C_Terminate(AsyncIO *IO)
 628 {
 629 ///     pop3aggr *pMsg = (pop3aggr *)IO->Data;
 630
 631         CtdlLogPrintf(CTDL_DEBUG, "POP3: %s\n", __FUNCTION__);
 632         FinalizePOP3AggrRun(IO);
 633         return eAbort;
 634 }
 635 eNextState POP3_C_Timeout(AsyncIO *IO)
 636 {
 637         pop3aggr *pMsg = IO->Data;
 638
 639         CtdlLogPrintf(CTDL_DEBUG, "POP3: %s\n", __FUNCTION__);
 640         StrBufPlain(IO->ErrMsg, CKEY(POP3C_ReadErrors[pMsg->State]));
 641         return FailAggregationRun(IO);
 642 }
 643 eNextState POP3_C_ConnFail(AsyncIO *IO)
 644 {
 645         pop3aggr *pMsg = (pop3aggr *)IO->Data;
 646
 647         CtdlLogPrintf(CTDL_DEBUG, "POP3: %s\n", __FUNCTION__);
 648         StrBufPlain(IO->ErrMsg, CKEY(POP3C_ReadErrors[pMsg->State]));
 649         return FailAggregationRun(IO);
 650 }
 651 eNextState POP3_C_Shutdown(AsyncIO *IO)
 652 {
 653         CtdlLogPrintf(CTDL_DEBUG, "POP3: %s\n", __FUNCTION__);
 654 ////    pop3aggr *pMsg = IO->Data;
 655
 656         ////pMsg->MyQEntry->Status = 3;
 657         ///StrBufPlain(pMsg->MyQEntry->StatusMessage, HKEY("server shutdown during message retrieval."));
 658         FinalizePOP3AggrRun(IO);
 659         return eAbort;
 660 }
 661
 662
 663 /**
 664  * @brief lineread Handler; understands when to read more POP3 lines, and when this is a one-lined reply.
 665  */
 666 eReadState POP3_C_ReadServerStatus(AsyncIO *IO)
 667 {
 668         eReadState Finished = eBufferNotEmpty;
 669
 670         switch (IO->NextState) {
 671         case eSendDNSQuery:
 672         case eReadDNSReply:
 673         case eDBQuery:
 674         case eConnect:
 675         case eTerminateConnection:
 676         case eAbort:
 677                 Finished = eReadFail;
 678                 break;
 679         case eSendReply:
 680         case eSendMore:
 681         case eReadMore:
 682         case eReadMessage:
 683                 Finished = StrBufChunkSipLine(IO->IOBuf, &IO->RecvBuf);
 684                 break;
 685         case eReadPayload:
 686                 Finished = CtdlReadMessageBodyAsync(IO);
 687                 break;
 688         }
 689         return Finished;
 690 }
 691
 692 /*****************************************************************************
 693  * So we connect our Server IP here.                                         *
 694  *****************************************************************************/
 695 eNextState POP3_C_ReAttachToFetchMessages(AsyncIO *IO)
 696 {
 697         pop3aggr *cpptr = IO->Data;
 698
 699         CtdlLogPrintf(CTDL_DEBUG, "POP3: %s\n", __FUNCTION__);
 700 ////??? cpptr->State ++;
 701         if (cpptr->Pos == NULL)
 702                 cpptr->Pos = GetNewHashPos(cpptr->MsgNumbers, 0);
 703
 704         POP3_C_DispatchWriteDone(IO);
 705         ReAttachIO(IO, cpptr, 0);
 706         IO->NextState = eReadMessage;
 707         return IO->NextState;
 708 }
 709
 710 eNextState connect_ip(AsyncIO *IO)
 711 {
 712         pop3aggr *cpptr = IO->Data;
 713
 714         CtdlLogPrintf(CTDL_DEBUG, "POP3: %s\n", __FUNCTION__);
 715
 716 ////    IO->ConnectMe = &cpptr->Pop3Host;
 717         /*  Bypass the ns lookup result like this: IO->Addr.sin_addr.s_addr = inet_addr("127.0.0.1"); */
 718
 719         /////// SetConnectStatus(IO);
 720
 721         return InitEventIO(IO, cpptr,
 722                            POP3_C_ConnTimeout,
 723                            POP3_C_ReadTimeouts[0],
 724                            1);
 725 }
 726
 727 eNextState get_one_host_ip_done(AsyncIO *IO)
 728 {
 729         pop3aggr *cpptr = IO->Data;
 730         struct hostent *hostent;
 731
 732         QueryCbDone(IO);
 733
 734         hostent = cpptr->HostLookup.VParsedDNSReply;
 735         if ((cpptr->HostLookup.DNSStatus == ARES_SUCCESS) &&
 736             (hostent != NULL) ) {
 737                 memset(&cpptr->Pop3Host.Addr, 0, sizeof(struct in6_addr));
 738                 if (cpptr->Pop3Host.IPv6) {
 739                         memcpy(&cpptr->Pop3Host.Addr.sin6_addr.s6_addr,
 740                                &hostent->h_addr_list[0],
 741                                sizeof(struct in6_addr));
 742
 743                         cpptr->Pop3Host.Addr.sin6_family = hostent->h_addrtype;
 744                         cpptr->Pop3Host.Addr.sin6_port   = htons(DefaultPOP3Port);
 745                 }
 746                 else {
 747                         struct sockaddr_in *addr = (struct sockaddr_in*) &cpptr->Pop3Host.Addr;
 748                         /* Bypass the ns lookup result like this: IO->Addr.sin_addr.s_addr = inet_addr("127.0.0.1"); */
 749 //                      addr->sin_addr.s_addr = htonl((uint32_t)&hostent->h_addr_list[0]);
 750                         memcpy(&addr->sin_addr.s_addr,
 751                                hostent->h_addr_list[0],
 752                                sizeof(uint32_t));
 753
 754                         addr->sin_family = hostent->h_addrtype;
 755                         addr->sin_port   = htons(DefaultPOP3Port);
 756
 757                 }
 758                 return connect_ip(IO);
 759         }
 760         else
 761                 return eAbort;
 762 }
 763
 764 eNextState get_one_host_ip(AsyncIO *IO)
 765 {
 766         pop3aggr *cpptr = IO->Data;
 767         /*
 768          * here we start with the lookup of one host. it might be...
 769          * - the relay host *sigh*
 770          * - the direct hostname if there was no mx record
 771          * - one of the mx'es
 772          */
 773
 774         InitC_ares_dns(IO);
 775
 776         CtdlLogPrintf(CTDL_DEBUG, "POP3: %s\n", __FUNCTION__);
 777
 778         CtdlLogPrintf(CTDL_DEBUG,
 779                       "POP3 client[%ld]: looking up %s-Record %s : %d ...\n",
 780                       cpptr->n,
 781                       (cpptr->Pop3Host.IPv6)? "aaaa": "a",
 782                       cpptr->Pop3Host.Host,
 783                       cpptr->Pop3Host.Port);
 784
 785         if (!QueueQuery((cpptr->Pop3Host.IPv6)? ns_t_aaaa : ns_t_a,
 786                         cpptr->Pop3Host.Host,
 787                         &cpptr->IO,
 788                         &cpptr->HostLookup,
 789                         get_one_host_ip_done))
 790         {
 791 //              cpptr->MyQEntry->Status = 5;
 792 //              StrBufPrintf(SendMsg->MyQEntry->StatusMessage,
 793 //                           "No MX hosts found for <%s>", SendMsg->node);
 794                 cpptr->IO.NextState = eTerminateConnection;
 795                 return IO->NextState;
 796         }
 797         IO->NextState = eReadDNSReply;
 798         return IO->NextState;
 799 }
 800
 801
 802
 803 int pop3_do_fetching(pop3aggr *cpptr)
 804 {
 805         CitContext *SubC;
 806
 807         cpptr->IO.Data          = cpptr;
 808
 809         cpptr->IO.SendDone      = POP3_C_DispatchWriteDone;
 810         cpptr->IO.ReadDone      = POP3_C_DispatchReadDone;
 811         cpptr->IO.Terminate     = POP3_C_Terminate;
 812         cpptr->IO.LineReader    = POP3_C_ReadServerStatus;
 813         cpptr->IO.ConnFail      = POP3_C_ConnFail;
 814         cpptr->IO.Timeout       = POP3_C_Timeout;
 815         cpptr->IO.ShutdownAbort = POP3_C_Shutdown;
 816
 817         cpptr->IO.SendBuf.Buf   = NewStrBufPlain(NULL, 1024);
 818         cpptr->IO.RecvBuf.Buf   = NewStrBufPlain(NULL, 1024);
 819         cpptr->IO.IOBuf         = NewStrBuf();
 820
 821         cpptr->IO.NextState     = eReadMessage;
 822 /* TODO
 823    CtdlLogPrintf(CTDL_DEBUG, "POP3: %s %s %s <password>\n", roomname, pop3host, pop3user);
 824    CtdlLogPrintf(CTDL_NOTICE, "Connecting to <%s>\n", pop3host);
 825 */
 826
 827         SubC = CloneContext (CC);
 828         SubC->session_specific_data = (char*) cpptr;
 829         cpptr->IO.CitContext = SubC;
 830
 831         if (cpptr->IO.ConnectMe->IsIP) {
 832                 QueueEventContext(&cpptr->IO,
 833                                   connect_ip);
 834         }
 835         else { /* uneducated admin has chosen to add DNS to the equation... */
 836                 QueueEventContext(&cpptr->IO,
 837                                   get_one_host_ip);
 838         }
 839         return 1;
 840 }
 841
 842 /*
 843  * Scan a room's netconfig to determine whether it requires POP3 aggregation
 844  */
 845 void pop3client_scan_room(struct ctdlroom *qrbuf, void *data)
 846 {
 847         StrBuf *CfgData;
 848         StrBuf *CfgType;
 849         StrBuf *Line;
 850
 851         struct stat statbuf;
 852         char filename[PATH_MAX];
 853         int  fd;
 854         int Done;
 855         void *vptr;
 856         const char *CfgPtr, *lPtr;
 857         const char *Err;
 858
 859         pop3_room_counter *Count = NULL;
 860 //      pop3aggr *cpptr;
 861
 862         citthread_mutex_lock(&POP3QueueMutex);
 863         if (GetHash(POP3QueueRooms, LKEY(qrbuf->QRnumber), &vptr))
 864         {
 865                 CtdlLogPrintf(CTDL_DEBUG,
 866                               "pop3client: [%ld] %s already in progress.\n",
 867                               qrbuf->QRnumber,
 868                               qrbuf->QRname);
 869                 citthread_mutex_unlock(&POP3QueueMutex);
 870                 return;
 871         }
 872         citthread_mutex_unlock(&POP3QueueMutex);
 873
 874         assoc_file_name(filename, sizeof filename, qrbuf, ctdl_netcfg_dir);
 875
 876         if (CtdlThreadCheckStop())
 877                 return;
 878
 879         /* Only do net processing for rooms that have netconfigs */
 880         fd = open(filename, 0);
 881         if (fd <= 0) {
 882                 //CtdlLogPrintf(CTDL_DEBUG, "rssclient: %s no config.\n", qrbuf->QRname);
 883                 return;
 884         }
 885         if (CtdlThreadCheckStop())
 886                 return;
 887         if (fstat(fd, &statbuf) == -1) {
 888                 CtdlLogPrintf(CTDL_DEBUG,  "ERROR: could not stat configfile '%s' - %s\n",
 889                               filename, strerror(errno));
 890                 return;
 891         }
 892         if (CtdlThreadCheckStop())
 893                 return;
 894         CfgData = NewStrBufPlain(NULL, statbuf.st_size + 1);
 895         if (StrBufReadBLOB(CfgData, &fd, 1, statbuf.st_size, &Err) < 0) {
 896                 close(fd);
 897                 FreeStrBuf(&CfgData);
 898                 CtdlLogPrintf(CTDL_DEBUG,  "ERROR: reading config '%s' - %s<br>\n",
 899                               filename, strerror(errno));
 900                 return;
 901         }
 902         close(fd);
 903         if (CtdlThreadCheckStop())
 904                 return;
 905
 906         CfgPtr = NULL;
 907         CfgType = NewStrBuf();
 908         Line = NewStrBufPlain(NULL, StrLength(CfgData));
 909         Done = 0;
 910
 911         while (!Done)
 912         {
 913                 Done = StrBufSipLine(Line, CfgData, &CfgPtr) == 0;
 914                 if (StrLength(Line) > 0)
 915                 {
 916                         lPtr = NULL;
 917                         StrBufExtract_NextToken(CfgType, Line, &lPtr, '|');
 918                         if (!strcasecmp("pop3client", ChrPtr(CfgType)))
 919                         {
 920                                 pop3aggr *cptr;
 921
 922                                 if (Count == NULL)
 923                                 {
 924                                         Count = malloc(sizeof(pop3_room_counter));
 925                                         Count->count = 0;
 926                                 }
 927                                 Count->count ++;
 928                                 cptr = (pop3aggr *) malloc(sizeof(pop3aggr));
 929                                 memset(cptr, 0, sizeof(pop3aggr));
 930                                 /// TODO do we need this? cptr->roomlist_parts = 1;
 931                                 cptr->rooms = NewStrBufPlain(qrbuf->QRname, -1);
 932                                 cptr->pop3user = NewStrBufPlain(NULL, StrLength(Line));
 933                                 cptr->pop3pass = NewStrBufPlain(NULL, StrLength(Line));
 934                                 cptr->Url = NewStrBuf();
 935
 936                                 StrBufExtract_NextToken(cptr->Url, Line, &lPtr, '|');
 937                                 StrBufExtract_NextToken(cptr->pop3user, Line, &lPtr, '|');
 938                                 StrBufExtract_NextToken(cptr->pop3pass, Line, &lPtr, '|');
 939                                 cptr->keep = StrBufExtractNext_long(Line, &lPtr, '|');
 940                                 cptr->interval = StrBufExtractNext_long(Line, &lPtr, '|');
 941
 942                                 ParseURL(&cptr->IO.ConnectMe, cptr->Url, 110);
 943
 944                                 cptr->IO.ConnectMe->CurlCreds = cptr->pop3user;
 945                                 cptr->IO.ConnectMe->User = ChrPtr(cptr->IO.ConnectMe->CurlCreds);
 946                                 cptr->IO.ConnectMe->UrlWithoutCred = cptr->pop3pass;
 947                                 cptr->IO.ConnectMe->Pass = ChrPtr(cptr->IO.ConnectMe->UrlWithoutCred);
 948
 949
 950
 951 #if 0
 952 /* todo: we need to reunite the url to be shure. */
 953
 954                                 citthread_mutex_lock(&POP3ueueMutex);
 955                                 GetHash(POP3FetchUrls, SKEY(ptr->Url), &vptr);
 956                                 use_this_cptr = (pop3aggr *)vptr;
 957
 958                                 if (use_this_rncptr != NULL)
 959                                 {
 960                                         /* mustn't attach to an active session */
 961                                         if (use_this_cptr->RefCount > 0)
 962                                         {
 963                                                 DeletePOP3Cfg(cptr);
 964                                                 Count->count--;
 965                                         }
 966                                         else
 967                                         {
 968                                                 long *QRnumber;
 969                                                 StrBufAppendBufPlain(use_this_cptr->rooms,
 970                                                                      qrbuf->QRname,
 971                                                                      -1, 0);
 972                                                 if (use_this_cptr->roomlist_parts == 1)
 973                                                 {
 974                                                         use_this_cptr->OtherQRnumbers = NewHash(1, lFlathash);
 975                                                 }
 976                                                 QRnumber = (long*)malloc(sizeof(long));
 977                                                 *QRnumber = qrbuf->QRnumber;
 978                                                 Put(use_this_cptr->OtherQRnumbers, LKEY(qrbuf->QRnumber), QRnumber, NULL);
 979                                                 use_this_cptr->roomlist_parts++;
 980                                         }
 981                                         citthread_mutex_unlock(&POP3QueueMutex);
 982                                         continue;
 983                                 }
 984                                 citthread_mutex_unlock(&RSSQueueMutex);
 985 #endif
 986
 987                                 citthread_mutex_lock(&POP3QueueMutex);
 988                                 Put(POP3FetchUrls, SKEY(cptr->Url), cptr, DeletePOP3Aggregator);
 989                                 citthread_mutex_unlock(&POP3QueueMutex);
 990
 991                         }
 992
 993                 }
 994
 995                 ///fclose(fp);
 996
 997         }
 998 }
 999
1000
1001 void pop3client_scan(void) {
1002         static time_t last_run = 0L;
1003         static int doing_pop3client = 0;
1004 ///     struct pop3aggr *pptr;
1005         time_t fastest_scan;
1006         HashPos *it;
1007         long len;
1008         const char *Key;
1009         void *vrptr;
1010         pop3aggr *cptr;
1011
1012         if (config.c_pop3_fastest < config.c_pop3_fetch)
1013                 fastest_scan = config.c_pop3_fastest;
1014         else
1015                 fastest_scan = config.c_pop3_fetch;
1016
1017         /*
1018          * Run POP3 aggregation no more frequently than once every n seconds
1019          */
1020         if ( (time(NULL) - last_run) < fastest_scan ) {
1021                 return;
1022         }
1023
1024         /*
1025          * This is a simple concurrency check to make sure only one pop3client run
1026          * is done at a time.  We could do this with a mutex, but since we
1027          * don't really require extremely fine granularity here, we'll do it
1028          * with a static variable instead.
1029          */
1030         if (doing_pop3client) return;
1031         doing_pop3client = 1;
1032
1033         CtdlLogPrintf(CTDL_DEBUG, "pop3client started\n");
1034         CtdlForEachRoom(pop3client_scan_room, NULL);
1035
1036
1037         citthread_mutex_lock(&POP3QueueMutex);
1038         it = GetNewHashPos(POP3FetchUrls, 0);
1039         while (GetNextHashPos(POP3FetchUrls, it, &len, &Key, &vrptr) &&
1040                (vrptr != NULL)) {
1041                 cptr = (pop3aggr *)vrptr;
1042                 if (cptr->RefCount == 0)
1043                         if (!pop3_do_fetching(cptr))
1044                                 DeletePOP3Aggregator(cptr);////TODO
1045         }
1046         DeleteHashPos(&it);
1047         citthread_mutex_unlock(&POP3QueueMutex);
1048
1049         CtdlLogPrintf(CTDL_DEBUG, "pop3client ended\n");
1050         last_run = time(NULL);
1051         doing_pop3client = 0;
1052 }
1053
1054
1055 void pop3_cleanup(void)
1056 {
1057         citthread_mutex_destroy(&POP3QueueMutex);
1058         DeleteHash(&POP3FetchUrls);
1059         DeleteHash(&POP3QueueRooms);
1060 }
1061
1062 CTDL_MODULE_INIT(pop3client)
1063 {
1064         if (!threading)
1065         {
1066                 citthread_mutex_init(&POP3QueueMutex, NULL);
1067                 POP3QueueRooms = NewHash(1, lFlathash);
1068                 POP3FetchUrls = NewHash(1, NULL);
1069                 CtdlRegisterSessionHook(pop3client_scan, EVT_TIMER);
1070                 CtdlRegisterCleanupHook(pop3_cleanup);
1071         }
1072         /* return our Subversion id for the Log */
1073         return "pop3client";
1074 }