citadel/modules/pop3client/serv_pop3client.c

   1 /*
   2  * Consolidate mail from remote POP3 accounts.
   3  *
   4  * Copyright (c) 2007-2009 by the citadel.org team
   5  *
   6  *  This program is free software; you can redistribute it and/or modify
   7  *  it under the terms of the GNU General Public License as published by
   8  *  the Free Software Foundation; either version 3 of the License, or
   9  *  (at your option) any later version.
  10  *
  11  *  This program is distributed in the hope that it will be useful,
  12  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
  13  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14  *  GNU General Public License for more details.
  15  *
  16  *  You should have received a copy of the GNU General Public License
  17  *  along with this program; if not, write to the Free Software
  18  *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  19  */
  20
  21 #include <stdlib.h>
  22 #include <unistd.h>
  23 #include <stdio.h>
  24
  25 #if TIME_WITH_SYS_TIME
  26 # include <sys/time.h>
  27 # include <time.h>
  28 #else
  29 # if HAVE_SYS_TIME_H
  30 #  include <sys/time.h>
  31 # else
  32 #  include <time.h>
  33 # endif
  34 #endif
  35
  36 #include <ctype.h>
  37 #include <string.h>
  38 #include <errno.h>
  39 #include <sys/types.h>
  40 #include <sys/stat.h>
  41 #include <libcitadel.h>
  42 #include "citadel.h"
  43 #include "server.h"
  44 #include "citserver.h"
  45 #include "support.h"
  46 #include "config.h"
  47 #include "ctdl_module.h"
  48 #include "clientsocket.h"
  49 #include "msgbase.h"
  50 #include "internet_addressing.h"
  51 #include "database.h"
  52 #include "citadel_dirs.h"
  53 #include "event_client.h"
  54
  55
  56 struct CitContext pop3_client_CC;
  57
  58 pthread_mutex_t POP3QueueMutex; /* locks the access to the following vars: */
  59 HashList *POP3QueueRooms = NULL; /* rss_room_counter */
  60 HashList *POP3FetchUrls = NULL; /* -> rss_aggregator; ->RefCount access to be locked too. */
  61
  62 typedef struct __pop3_room_counter {
  63         int count;
  64         long QRnumber;
  65 }pop3_room_counter;
  66
  67 typedef enum ePOP3_C_States {
  68         ReadGreeting,
  69         GetUserState,
  70         GetPassState,
  71         GetListCommandState,
  72         GetListOneLine,
  73         GetOneMessageIDState,
  74         ReadMessageBodyFollowing,
  75         ReadMessageBody,
  76         GetDeleteState,
  77         ReadQuitState,
  78         POP3C_MaxRead
  79 }ePOP3_C_States;
  80
  81
  82 typedef struct _FetchItem {
  83         long MSGID;
  84         long MSGSize;
  85         StrBuf *MsgUIDL;
  86         StrBuf *MsgUID;
  87         int NeedFetch;
  88         struct CtdlMessage *Msg;
  89 } FetchItem;
  90
  91 void HfreeFetchItem(void *vItem)
  92 {
  93         FetchItem *Item = (FetchItem*) vItem;
  94         FreeStrBuf(&Item->MsgUIDL);
  95         FreeStrBuf(&Item->MsgUID);
  96         free(Item);
  97 }
  98
  99 typedef struct __pop3aggr {
 100         AsyncIO          IO;
 101
 102         long n;
 103         long RefCount;
 104 ///     ParsedURL *Pop3Host;
 105         DNSQueryParts HostLookup;
 106
 107 //      StrBuf          *rooms;
 108         long             QRnumber;
 109         HashList        *OtherQRnumbers;
 110
 111         StrBuf          *Url;
 112 ///     StrBuf *pop3host; -> URL
 113         StrBuf *pop3user;
 114         StrBuf *pop3pass;
 115         StrBuf *RoomName; // TODO: fill me
 116         int keep;
 117         time_t interval;
 118         ePOP3_C_States State;
 119         HashList *MsgNumbers;
 120         HashPos *Pos;
 121         FetchItem *CurrMsg;
 122 } pop3aggr;
 123
 124 void DeletePOP3Aggregator(void *vptr)
 125 {
 126         pop3aggr *ptr = vptr;
 127         DeleteHashPos(&ptr->Pos);
 128         DeleteHash(&ptr->MsgNumbers);
 129 //      FreeStrBuf(&ptr->rooms);
 130         FreeStrBuf(&ptr->pop3user);
 131         FreeStrBuf(&ptr->pop3pass);
 132         FreeStrBuf(&ptr->RoomName);
 133         FreeURL(&ptr->IO.ConnectMe);
 134         FreeStrBuf(&ptr->Url);
 135         FreeStrBuf(&ptr->IO.IOBuf);
 136         FreeStrBuf(&ptr->IO.SendBuf.Buf);
 137         FreeStrBuf(&ptr->IO.RecvBuf.Buf);
 138         DeleteAsyncMsg(&ptr->IO.ReadMsg);
 139         free(ptr);
 140 }
 141
 142
 143 typedef eNextState(*Pop3ClientHandler)(pop3aggr* RecvMsg);
 144
 145 eNextState POP3_C_Shutdown(AsyncIO *IO);
 146 eNextState POP3_C_Timeout(AsyncIO *IO);
 147 eNextState POP3_C_ConnFail(AsyncIO *IO);
 148 eNextState POP3_C_DispatchReadDone(AsyncIO *IO);
 149 eNextState POP3_C_DispatchWriteDone(AsyncIO *IO);
 150 eNextState POP3_C_Terminate(AsyncIO *IO);
 151 eReadState POP3_C_ReadServerStatus(AsyncIO *IO);
 152 eNextState POP3_C_ReAttachToFetchMessages(AsyncIO *IO);
 153
 154 eNextState FinalizePOP3AggrRun(AsyncIO *IO)
 155 {
 156         HashPos  *It;
 157         pop3aggr *cptr = (pop3aggr *)IO->Data;
 158
 159         syslog(LOG_DEBUG, "Terminating Aggregator; bye.\n");
 160
 161         It = GetNewHashPos(POP3FetchUrls, 0);
 162         pthread_mutex_lock(&POP3QueueMutex);
 163         {
 164                 GetHashPosFromKey(POP3FetchUrls, SKEY(cptr->Url), It);
 165                 DeleteEntryFromHash(POP3FetchUrls, It);
 166         }
 167         pthread_mutex_unlock(&POP3QueueMutex);
 168         DeleteHashPos(&It);
 169         return eAbort;
 170 }
 171
 172 eNextState FailAggregationRun(AsyncIO *IO)
 173 {
 174         return eAbort;
 175 }
 176
 177
 178 #define POP3C_DBG_SEND() syslog(LOG_DEBUG, "POP3 client[%ld]: > %s\n", RecvMsg->n, ChrPtr(RecvMsg->IO.SendBuf.Buf))
 179 #define POP3C_DBG_READ() syslog(LOG_DEBUG, "POP3 client[%ld]: < %s\n", RecvMsg->n, ChrPtr(RecvMsg->IO.IOBuf))
 180 #define POP3C_OK (strncasecmp(ChrPtr(RecvMsg->IO.IOBuf), "+OK", 3) == 0)
 181
 182 eNextState POP3C_ReadGreeting(pop3aggr *RecvMsg)
 183 {
 184         POP3C_DBG_READ();
 185         /* Read the server greeting */
 186         if (!POP3C_OK) return eTerminateConnection;
 187         else return eSendReply;
 188 }
 189
 190 eNextState POP3C_SendUser(pop3aggr *RecvMsg)
 191 {
 192         /* Identify ourselves.  NOTE: we have to append a CR to each command.  The LF will
 193          * automatically be appended by sock_puts().  Believe it or not, leaving out the CR
 194          * will cause problems if the server happens to be Exchange, which is so b0rken it
 195          * actually barfs on LF-terminated newlines.
 196          */
 197         StrBufPrintf(RecvMsg->IO.SendBuf.Buf,
 198                      "USER %s\r\n", ChrPtr(RecvMsg->pop3user));
 199         POP3C_DBG_SEND();
 200         return eReadMessage;
 201 }
 202
 203 eNextState POP3C_GetUserState(pop3aggr *RecvMsg)
 204 {
 205         POP3C_DBG_READ();
 206         if (!POP3C_OK) return eTerminateConnection;
 207         else return eSendReply;
 208 }
 209
 210 eNextState POP3C_SendPassword(pop3aggr *RecvMsg)
 211 {
 212         /* Password */
 213         StrBufPrintf(RecvMsg->IO.SendBuf.Buf,
 214                      "PASS %s\r\n", ChrPtr(RecvMsg->pop3pass));
 215         syslog(LOG_DEBUG, "<PASS <password>\n");
 216 //      POP3C_DBG_SEND();
 217         return eReadMessage;
 218 }
 219
 220 eNextState POP3C_GetPassState(pop3aggr *RecvMsg)
 221 {
 222         POP3C_DBG_READ();
 223         if (!POP3C_OK) return eTerminateConnection;
 224         else return eSendReply;
 225 }
 226
 227 eNextState POP3C_SendListCommand(pop3aggr *RecvMsg)
 228 {
 229         /* Get the list of messages */
 230         StrBufPlain(RecvMsg->IO.SendBuf.Buf, HKEY("LIST\r\n"));
 231         POP3C_DBG_SEND();
 232         return eReadMessage;
 233 }
 234
 235 eNextState POP3C_GetListCommandState(pop3aggr *RecvMsg)
 236 {
 237         POP3C_DBG_READ();
 238         if (!POP3C_OK) return eTerminateConnection;
 239         RecvMsg->MsgNumbers = NewHash(1, NULL);
 240         RecvMsg->State++;
 241         return eReadMore;
 242 }
 243
 244
 245 eNextState POP3C_GetListOneLine(pop3aggr *RecvMsg)
 246 {
 247 #if 0
 248         int rc;
 249 #endif
 250         const char *pch;
 251         FetchItem *OneMsg = NULL;
 252         POP3C_DBG_READ();
 253
 254         if ((StrLength(RecvMsg->IO.IOBuf) == 1) &&
 255             (ChrPtr(RecvMsg->IO.IOBuf)[0] == '.'))
 256         {
 257                 if (GetCount(RecvMsg->MsgNumbers) == 0)
 258                 {
 259                         ////    RecvMsg->Sate = ReadQuitState;
 260                 }
 261                 else
 262                 {
 263                         RecvMsg->Pos = GetNewHashPos(RecvMsg->MsgNumbers, 0);
 264                 }
 265                 return eSendReply;
 266
 267         }
 268         OneMsg = (FetchItem*) malloc(sizeof(FetchItem));
 269         memset(OneMsg, 0, sizeof(FetchItem));
 270         OneMsg->MSGID = atol(ChrPtr(RecvMsg->IO.IOBuf));
 271
 272         pch = strchr(ChrPtr(RecvMsg->IO.IOBuf), ' ');
 273         if (pch != NULL)
 274         {
 275                 OneMsg->MSGSize = atol(pch + 1);
 276         }
 277 #if 0
 278         rc = TestValidateHash(RecvMsg->MsgNumbers);
 279         if (rc != 0)
 280                 syslog(LOG_DEBUG, "Hash Invalid: %d\n", rc);
 281 #endif
 282
 283         Put(RecvMsg->MsgNumbers, LKEY(OneMsg->MSGID), OneMsg, HfreeFetchItem);
 284 #if 0
 285         rc = TestValidateHash(RecvMsg->MsgNumbers);
 286         if (rc != 0)
 287                 syslog(LOG_DEBUG, "Hash Invalid: %d\n", rc);
 288 #endif
 289         //RecvMsg->State --; /* read next Line */
 290         return eReadMore;
 291 }
 292
 293 eNextState POP3_FetchNetworkUsetableEntry(AsyncIO *IO)
 294 {
 295         long HKLen;
 296         const char *HKey;
 297         void *vData;
 298         struct cdbdata *cdbut;
 299         pop3aggr *RecvMsg = (pop3aggr *) IO->Data;
 300
 301         if(GetNextHashPos(RecvMsg->MsgNumbers, RecvMsg->Pos, &HKLen, &HKey, &vData))
 302         {
 303                 struct UseTable ut;
 304                 if (server_shutting_down)
 305                         return eAbort;
 306
 307                 RecvMsg->CurrMsg = (FetchItem*) vData;
 308                 syslog(LOG_DEBUG, "CHECKING: whether %s has already been seen: ", ChrPtr(RecvMsg->CurrMsg->MsgUID));
 309                 /* Find out if we've already seen this item */
 310                 safestrncpy(ut.ut_msgid,
 311                             ChrPtr(RecvMsg->CurrMsg->MsgUID),
 312                             sizeof(ut.ut_msgid));
 313                 ut.ut_timestamp = time(NULL);/// TODO: libev timestamp!
 314
 315                 cdbut = cdb_fetch(CDB_USETABLE, SKEY(RecvMsg->CurrMsg->MsgUID));
 316                 if (cdbut != NULL) {
 317                         /* Item has already been seen */
 318                         syslog(LOG_DEBUG, "YES\n");
 319                         cdb_free(cdbut);
 320
 321                         /* rewrite the record anyway, to update the timestamp */
 322                         cdb_store(CDB_USETABLE,
 323                                   SKEY(RecvMsg->CurrMsg->MsgUID),
 324                                   &ut, sizeof(struct UseTable) );
 325                         RecvMsg->CurrMsg->NeedFetch = 1; ////TODO0;
 326                 }
 327                 else
 328                 {
 329                         syslog(LOG_DEBUG, "NO\n");
 330                         RecvMsg->CurrMsg->NeedFetch = 1;
 331                 }
 332                 return NextDBOperation(&RecvMsg->IO, POP3_FetchNetworkUsetableEntry);
 333         }
 334         else
 335         {
 336                 /* ok, now we know them all, continue with reading the actual messages. */
 337                 DeleteHashPos(&RecvMsg->Pos);
 338
 339                 return QueueEventContext(IO, POP3_C_ReAttachToFetchMessages);
 340         }
 341 }
 342
 343 eNextState POP3C_GetOneMessagID(pop3aggr *RecvMsg)
 344 {
 345         long HKLen;
 346         const char *HKey;
 347         void *vData;
 348
 349 #if 0
 350         int rc;
 351         rc = TestValidateHash(RecvMsg->MsgNumbers);
 352         if (rc != 0)
 353                 syslog(LOG_DEBUG, "Hash Invalid: %d\n", rc);
 354 #endif
 355         if(GetNextHashPos(RecvMsg->MsgNumbers, RecvMsg->Pos, &HKLen, &HKey, &vData))
 356         {
 357                 RecvMsg->CurrMsg = (FetchItem*) vData;
 358                 /* Find out the UIDL of the message, to determine whether we've already downloaded it */
 359                 StrBufPrintf(RecvMsg->IO.SendBuf.Buf,
 360                              "UIDL %ld\r\n", RecvMsg->CurrMsg->MSGID);
 361                 POP3C_DBG_SEND();
 362         }
 363         else
 364         {
 365             RecvMsg->State++;
 366                 DeleteHashPos(&RecvMsg->Pos);
 367                 /// done receiving uidls.. start looking them up now.
 368                 RecvMsg->Pos = GetNewHashPos(RecvMsg->MsgNumbers, 0);
 369                 return QueueDBOperation(&RecvMsg->IO, POP3_FetchNetworkUsetableEntry);
 370         }
 371         return eReadMore; /* TODO */
 372 }
 373
 374 eNextState POP3C_GetOneMessageIDState(pop3aggr *RecvMsg)
 375 {
 376 #if 0
 377         int rc;
 378         rc = TestValidateHash(RecvMsg->MsgNumbers);
 379         if (rc != 0)
 380                 syslog(LOG_DEBUG, "Hash Invalid: %d\n", rc);
 381 #endif
 382
 383         POP3C_DBG_READ();
 384         if (!POP3C_OK) return eTerminateConnection;
 385         RecvMsg->CurrMsg->MsgUIDL = NewStrBufPlain(NULL, StrLength(RecvMsg->IO.IOBuf));
 386         RecvMsg->CurrMsg->MsgUID = NewStrBufPlain(NULL, StrLength(RecvMsg->IO.IOBuf) * 2);
 387
 388         StrBufExtract_token(RecvMsg->CurrMsg->MsgUIDL, RecvMsg->IO.IOBuf, 2, ' ');
 389         StrBufPrintf(RecvMsg->CurrMsg->MsgUID,
 390                      "pop3/%s/%s:%s@%s",
 391                      ChrPtr(RecvMsg->RoomName),
 392                      ChrPtr(RecvMsg->CurrMsg->MsgUIDL),
 393                      RecvMsg->IO.ConnectMe->User,
 394                      RecvMsg->IO.ConnectMe->Host);
 395         RecvMsg->State --;
 396         return eSendReply;
 397 }
 398
 399
 400 eNextState POP3C_SendGetOneMsg(pop3aggr *RecvMsg)
 401 {
 402         long HKLen;
 403         const char *HKey;
 404         void *vData;
 405
 406         RecvMsg->CurrMsg = NULL;
 407         while (GetNextHashPos(RecvMsg->MsgNumbers, RecvMsg->Pos, &HKLen, &HKey, &vData) &&
 408                (RecvMsg->CurrMsg = (FetchItem*) vData, RecvMsg->CurrMsg->NeedFetch == 0))
 409         {}
 410
 411         if ((RecvMsg->CurrMsg != NULL ) && (RecvMsg->CurrMsg->NeedFetch == 1))
 412         {
 413                 /* Message has not been seen. Tell the server to fetch the message... */
 414                 StrBufPrintf(RecvMsg->IO.SendBuf.Buf,
 415                              "RETR %ld\r\n", RecvMsg->CurrMsg->MSGID);
 416                 POP3C_DBG_SEND();
 417                 return eReadMessage;
 418         }
 419         else {
 420                 RecvMsg->State = ReadQuitState;
 421                 return POP3_C_DispatchWriteDone(&RecvMsg->IO);
 422         }
 423 }
 424
 425
 426 eNextState POP3C_ReadMessageBodyFollowing(pop3aggr *RecvMsg)
 427 {
 428         POP3C_DBG_READ();
 429         if (!POP3C_OK) return eTerminateConnection;
 430         RecvMsg->IO.ReadMsg = NewAsyncMsg(HKEY("."),
 431                                           RecvMsg->CurrMsg->MSGSize,
 432                                           config.c_maxmsglen,
 433                                           NULL, -1,
 434                                           1);
 435
 436         return eReadPayload;
 437 }
 438
 439
 440 eNextState POP3C_StoreMsgRead(AsyncIO *IO)
 441 {
 442         pop3aggr *RecvMsg = (pop3aggr *) IO->Data;
 443         struct UseTable ut;
 444
 445         syslog(LOG_DEBUG, "MARKING: %s as seen: ", ChrPtr(RecvMsg->CurrMsg->MsgUID));
 446
 447         safestrncpy(ut.ut_msgid,
 448                     ChrPtr(RecvMsg->CurrMsg->MsgUID),
 449                     sizeof(ut.ut_msgid));
 450         ut.ut_timestamp = time(NULL); /* TODO: use libev time */
 451         cdb_store(CDB_USETABLE,
 452                   ChrPtr(RecvMsg->CurrMsg->MsgUID),
 453                   StrLength(RecvMsg->CurrMsg->MsgUID),
 454                   &ut,
 455                   sizeof(struct UseTable) );
 456
 457         return QueueEventContext(&RecvMsg->IO, POP3_C_ReAttachToFetchMessages);
 458 }
 459 eNextState POP3C_SaveMsg(AsyncIO *IO)
 460 {
 461         long msgnum;
 462         pop3aggr *RecvMsg = (pop3aggr *) IO->Data;
 463
 464         /* Do Something With It (tm) */
 465         msgnum = CtdlSubmitMsg(RecvMsg->CurrMsg->Msg,
 466                                NULL,
 467                                ChrPtr(RecvMsg->RoomName),
 468                                0);
 469         if (msgnum > 0L) {
 470                 /* Message has been committed to the store */
 471                 /* write the uidl to the use table so we don't fetch this message again */
 472         }
 473         CtdlFreeMessage(RecvMsg->CurrMsg->Msg);
 474
 475         return NextDBOperation(&RecvMsg->IO, POP3C_StoreMsgRead);
 476 }
 477
 478 eNextState POP3C_ReadMessageBody(pop3aggr *RecvMsg)
 479 {
 480         syslog(LOG_DEBUG, "Converting message...\n");
 481         RecvMsg->CurrMsg->Msg = convert_internet_message_buf(&RecvMsg->IO.ReadMsg->MsgBuf);
 482
 483         return QueueDBOperation(&RecvMsg->IO, POP3C_SaveMsg);
 484 }
 485
 486 eNextState POP3C_SendDelete(pop3aggr *RecvMsg)
 487 {
 488         if (!RecvMsg->keep) {
 489                 StrBufPrintf(RecvMsg->IO.SendBuf.Buf,
 490                              "DELE %ld\r\n", RecvMsg->CurrMsg->MSGID);
 491                 POP3C_DBG_SEND();
 492                 return eReadMessage;
 493         }
 494         else {
 495                 RecvMsg->State = ReadMessageBodyFollowing;
 496                 return POP3_C_DispatchWriteDone(&RecvMsg->IO);
 497         }
 498 }
 499 eNextState POP3C_ReadDeleteState(pop3aggr *RecvMsg)
 500 {
 501         POP3C_DBG_READ();
 502         RecvMsg->State = GetOneMessageIDState;
 503         return eReadMessage;
 504 }
 505
 506 eNextState POP3C_SendQuit(pop3aggr *RecvMsg)
 507 {
 508         /* Log out */
 509         StrBufPlain(RecvMsg->IO.SendBuf.Buf,
 510                     HKEY("QUIT\r\n3)"));
 511         POP3C_DBG_SEND();
 512         return eReadMessage;
 513 }
 514
 515
 516 eNextState POP3C_ReadQuitState(pop3aggr *RecvMsg)
 517 {
 518         POP3C_DBG_READ();
 519         return eTerminateConnection;
 520 }
 521
 522 const long POP3_C_ConnTimeout = 1000;
 523 const long DefaultPOP3Port = 110;
 524
 525 Pop3ClientHandler POP3C_ReadHandlers[] = {
 526         POP3C_ReadGreeting,
 527         POP3C_GetUserState,
 528         POP3C_GetPassState,
 529         POP3C_GetListCommandState,
 530         POP3C_GetListOneLine,
 531         POP3C_GetOneMessageIDState,
 532         POP3C_ReadMessageBodyFollowing,
 533         POP3C_ReadMessageBody,
 534         POP3C_ReadDeleteState,
 535         POP3C_ReadQuitState,
 536 };
 537
 538 const long POP3_C_SendTimeouts[POP3C_MaxRead] = {
 539         100,
 540         100,
 541         100,
 542         100,
 543         100,
 544         100,
 545         100,
 546         100
 547 };
 548 const ConstStr POP3C_ReadErrors[POP3C_MaxRead] = {
 549         {HKEY("Connection broken during ")},
 550         {HKEY("Connection broken during ")},
 551         {HKEY("Connection broken during ")},
 552         {HKEY("Connection broken during ")},
 553         {HKEY("Connection broken during ")},
 554         {HKEY("Connection broken during ")},
 555         {HKEY("Connection broken during ")},
 556         {HKEY("Connection broken during ")}
 557 };
 558
 559 Pop3ClientHandler POP3C_SendHandlers[] = {
 560         NULL, /* we don't send a greeting */
 561         POP3C_SendUser,
 562         POP3C_SendPassword,
 563         POP3C_SendListCommand,
 564         NULL,
 565         POP3C_GetOneMessagID,
 566         POP3C_SendGetOneMsg,
 567         NULL,
 568         POP3C_SendDelete,
 569         POP3C_SendQuit
 570 };
 571
 572 const long POP3_C_ReadTimeouts[] = {
 573         100,
 574         100,
 575         100,
 576         100,
 577         100,
 578         100,
 579         100,
 580         100,
 581         100,
 582         100
 583 };
 584 /*****************************************************************************/
 585 /*                     POP3 CLIENT DISPATCHER                                */
 586 /*****************************************************************************/
 587
 588 void POP3SetTimeout(eNextState NextTCPState, pop3aggr *pMsg)
 589 {
 590         double Timeout = 0.0;
 591
 592         syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
 593
 594         switch (NextTCPState) {
 595         case eSendReply:
 596         case eSendMore:
 597                 Timeout = POP3_C_SendTimeouts[pMsg->State];
 598 /*
 599   if (pMsg->State == eDATABody) {
 600   / * if we're sending a huge message, we need more time. * /
 601   Timeout += StrLength(pMsg->msgtext) / 1024;
 602   }
 603 */
 604                 break;
 605         case eReadMessage:
 606                 Timeout = POP3_C_ReadTimeouts[pMsg->State];
 607 /*
 608   if (pMsg->State == eDATATerminateBody) {
 609   / *
 610   * some mailservers take a nap before accepting the message
 611   * content inspection and such.
 612   * /
 613   Timeout += StrLength(pMsg->msgtext) / 1024;
 614   }
 615 */
 616                 break;
 617         case eReadPayload:
 618                 Timeout = 100000;
 619                 /* TODO!!! */
 620                 break;
 621         case eSendDNSQuery:
 622         case eReadDNSReply:
 623         case eConnect:
 624         case eTerminateConnection:
 625         case eDBQuery:
 626         case eAbort:
 627         case eReadMore://// TODO
 628                 return;
 629         }
 630         SetNextTimeout(&pMsg->IO, Timeout);
 631 }
 632 eNextState POP3_C_DispatchReadDone(AsyncIO *IO)
 633 {
 634         syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
 635         pop3aggr *pMsg = IO->Data;
 636         eNextState rc;
 637
 638         rc = POP3C_ReadHandlers[pMsg->State](pMsg);
 639         if (rc != eReadMore)
 640             pMsg->State++;
 641         POP3SetTimeout(rc, pMsg);
 642         return rc;
 643 }
 644 eNextState POP3_C_DispatchWriteDone(AsyncIO *IO)
 645 {
 646         syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
 647         pop3aggr *pMsg = IO->Data;
 648         eNextState rc;
 649
 650         rc = POP3C_SendHandlers[pMsg->State](pMsg);
 651         POP3SetTimeout(rc, pMsg);
 652         return rc;
 653 }
 654
 655
 656 /*****************************************************************************/
 657 /*                     POP3 CLIENT ERROR CATCHERS                            */
 658 /*****************************************************************************/
 659 eNextState POP3_C_Terminate(AsyncIO *IO)
 660 {
 661 ///     pop3aggr *pMsg = (pop3aggr *)IO->Data;
 662
 663         syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
 664         FinalizePOP3AggrRun(IO);
 665         return eAbort;
 666 }
 667 eNextState POP3_C_Timeout(AsyncIO *IO)
 668 {
 669         pop3aggr *pMsg = IO->Data;
 670
 671         syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
 672         StrBufPlain(IO->ErrMsg, CKEY(POP3C_ReadErrors[pMsg->State]));
 673         return FailAggregationRun(IO);
 674 }
 675 eNextState POP3_C_ConnFail(AsyncIO *IO)
 676 {
 677         pop3aggr *pMsg = (pop3aggr *)IO->Data;
 678
 679         syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
 680         StrBufPlain(IO->ErrMsg, CKEY(POP3C_ReadErrors[pMsg->State]));
 681         return FailAggregationRun(IO);
 682 }
 683 eNextState POP3_C_Shutdown(AsyncIO *IO)
 684 {
 685         syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
 686 ////    pop3aggr *pMsg = IO->Data;
 687
 688         ////pMsg->MyQEntry->Status = 3;
 689         ///StrBufPlain(pMsg->MyQEntry->StatusMessage, HKEY("server shutdown during message retrieval."));
 690         FinalizePOP3AggrRun(IO);
 691         return eAbort;
 692 }
 693
 694
 695 /**
 696  * @brief lineread Handler; understands when to read more POP3 lines, and when this is a one-lined reply.
 697  */
 698 eReadState POP3_C_ReadServerStatus(AsyncIO *IO)
 699 {
 700         eReadState Finished = eBufferNotEmpty;
 701
 702         switch (IO->NextState) {
 703         case eSendDNSQuery:
 704         case eReadDNSReply:
 705         case eDBQuery:
 706         case eConnect:
 707         case eTerminateConnection:
 708         case eAbort:
 709                 Finished = eReadFail;
 710                 break;
 711         case eSendReply:
 712         case eSendMore:
 713         case eReadMore:
 714         case eReadMessage:
 715                 Finished = StrBufChunkSipLine(IO->IOBuf, &IO->RecvBuf);
 716                 break;
 717         case eReadPayload:
 718                 Finished = CtdlReadMessageBodyAsync(IO);
 719                 break;
 720         }
 721         return Finished;
 722 }
 723
 724 /*****************************************************************************
 725  * So we connect our Server IP here.                                         *
 726  *****************************************************************************/
 727 eNextState POP3_C_ReAttachToFetchMessages(AsyncIO *IO)
 728 {
 729         pop3aggr *cpptr = IO->Data;
 730
 731         syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
 732 ////??? cpptr->State ++;
 733         if (cpptr->Pos == NULL)
 734                 cpptr->Pos = GetNewHashPos(cpptr->MsgNumbers, 0);
 735
 736         POP3_C_DispatchWriteDone(IO);
 737         ReAttachIO(IO, cpptr, 0);
 738         IO->NextState = eReadMessage;
 739         return IO->NextState;
 740 }
 741
 742 eNextState connect_ip(AsyncIO *IO)
 743 {
 744         pop3aggr *cpptr = IO->Data;
 745
 746         syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
 747
 748 ////    IO->ConnectMe = &cpptr->Pop3Host;
 749         /*  Bypass the ns lookup result like this: IO->Addr.sin_addr.s_addr = inet_addr("127.0.0.1"); */
 750
 751         /////// SetConnectStatus(IO);
 752
 753         return InitEventIO(IO, cpptr,
 754                            POP3_C_ConnTimeout,
 755                            POP3_C_ReadTimeouts[0],
 756                            1);
 757 }
 758
 759 eNextState get_one_host_ip_done(AsyncIO *IO)
 760 {
 761         pop3aggr *cpptr = IO->Data;
 762         struct hostent *hostent;
 763
 764         QueryCbDone(IO);
 765
 766         hostent = cpptr->HostLookup.VParsedDNSReply;
 767         if ((cpptr->HostLookup.DNSStatus == ARES_SUCCESS) &&
 768             (hostent != NULL) ) {
 769                 memset(&cpptr->IO.ConnectMe->Addr, 0, sizeof(struct in6_addr));
 770                 if (cpptr->IO.ConnectMe->IPv6) {
 771                         memcpy(&cpptr->IO.ConnectMe->Addr.sin6_addr.s6_addr,
 772                                &hostent->h_addr_list[0],
 773                                sizeof(struct in6_addr));
 774
 775                         cpptr->IO.ConnectMe->Addr.sin6_family = hostent->h_addrtype;
 776                         cpptr->IO.ConnectMe->Addr.sin6_port   = htons(DefaultPOP3Port);
 777                 }
 778                 else {
 779                         struct sockaddr_in *addr = (struct sockaddr_in*) &cpptr->IO.ConnectMe->Addr;
 780                         /* Bypass the ns lookup result like this: IO->Addr.sin_addr.s_addr = inet_addr("127.0.0.1"); */
 781 //                      addr->sin_addr.s_addr = htonl((uint32_t)&hostent->h_addr_list[0]);
 782                         memcpy(&addr->sin_addr.s_addr,
 783                                hostent->h_addr_list[0],
 784                                sizeof(uint32_t));
 785
 786                         addr->sin_family = hostent->h_addrtype;
 787                         addr->sin_port   = htons(DefaultPOP3Port);
 788
 789                 }
 790                 return connect_ip(IO);
 791         }
 792         else
 793                 return eAbort;
 794 }
 795
 796 eNextState get_one_host_ip(AsyncIO *IO)
 797 {
 798         pop3aggr *cpptr = IO->Data;
 799         /*
 800          * here we start with the lookup of one host. it might be...
 801          * - the relay host *sigh*
 802          * - the direct hostname if there was no mx record
 803          * - one of the mx'es
 804          */
 805
 806         InitC_ares_dns(IO);
 807
 808         syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
 809
 810         syslog(LOG_DEBUG,
 811                       "POP3 client[%ld]: looking up %s-Record %s : %d ...\n",
 812                       cpptr->n,
 813                       (cpptr->IO.ConnectMe->IPv6)? "aaaa": "a",
 814                       cpptr->IO.ConnectMe->Host,
 815                       cpptr->IO.ConnectMe->Port);
 816
 817         QueueQuery((cpptr->IO.ConnectMe->IPv6)? ns_t_aaaa : ns_t_a,
 818                    cpptr->IO.ConnectMe->Host,
 819                    &cpptr->IO,
 820                    &cpptr->HostLookup,
 821                    get_one_host_ip_done);
 822         IO->NextState = eReadDNSReply;
 823         return IO->NextState;
 824 }
 825
 826
 827
 828 int pop3_do_fetching(pop3aggr *cpptr)
 829 {
 830         CitContext *SubC;
 831
 832         cpptr->IO.Data          = cpptr;
 833
 834         cpptr->IO.SendDone      = POP3_C_DispatchWriteDone;
 835         cpptr->IO.ReadDone      = POP3_C_DispatchReadDone;
 836         cpptr->IO.Terminate     = POP3_C_Terminate;
 837         cpptr->IO.LineReader    = POP3_C_ReadServerStatus;
 838         cpptr->IO.ConnFail      = POP3_C_ConnFail;
 839         cpptr->IO.Timeout       = POP3_C_Timeout;
 840         cpptr->IO.ShutdownAbort = POP3_C_Shutdown;
 841
 842         cpptr->IO.SendBuf.Buf   = NewStrBufPlain(NULL, 1024);
 843         cpptr->IO.RecvBuf.Buf   = NewStrBufPlain(NULL, 1024);
 844         cpptr->IO.IOBuf         = NewStrBuf();
 845
 846         cpptr->IO.NextState     = eReadMessage;
 847 /* TODO
 848    syslog(LOG_DEBUG, "POP3: %s %s %s <password>\n", roomname, pop3host, pop3user);
 849    syslog(LOG_DEBUG, "Connecting to <%s>\n", pop3host);
 850 */
 851
 852         SubC = CloneContext (&pop3_client_CC);
 853         SubC->session_specific_data = (char*) cpptr;
 854         cpptr->IO.CitContext = SubC;
 855
 856         if (cpptr->IO.ConnectMe->IsIP) {
 857                 QueueEventContext(&cpptr->IO,
 858                                   connect_ip);
 859         }
 860         else { /* uneducated admin has chosen to add DNS to the equation... */
 861                 QueueEventContext(&cpptr->IO,
 862                                   get_one_host_ip);
 863         }
 864         return 1;
 865 }
 866
 867 /*
 868  * Scan a room's netconfig to determine whether it requires POP3 aggregation
 869  */
 870 void pop3client_scan_room(struct ctdlroom *qrbuf, void *data)
 871 {
 872         StrBuf *CfgData;
 873         StrBuf *CfgType;
 874         StrBuf *Line;
 875
 876         struct stat statbuf;
 877         char filename[PATH_MAX];
 878         int  fd;
 879         int Done;
 880         void *vptr;
 881         const char *CfgPtr, *lPtr;
 882         const char *Err;
 883
 884 //      pop3_room_counter *Count = NULL;
 885 //      pop3aggr *cpptr;
 886
 887         pthread_mutex_lock(&POP3QueueMutex);
 888         if (GetHash(POP3QueueRooms, LKEY(qrbuf->QRnumber), &vptr))
 889         {
 890                 syslog(LOG_DEBUG,
 891                               "pop3client: [%ld] %s already in progress.\n",
 892                               qrbuf->QRnumber,
 893                               qrbuf->QRname);
 894                 pthread_mutex_unlock(&POP3QueueMutex);
 895         }
 896         pthread_mutex_unlock(&POP3QueueMutex);
 897
 898         if (server_shutting_down)
 899                 return;
 900
 901         assoc_file_name(filename, sizeof filename, qrbuf, ctdl_netcfg_dir);
 902
 903         if (server_shutting_down)
 904                 return;
 905
 906         /* Only do net processing for rooms that have netconfigs */
 907         fd = open(filename, 0);
 908         if (fd <= 0) {
 909                 //syslog(LOG_DEBUG, "rssclient: %s no config.\n", qrbuf->QRname);
 910                 return;
 911         }
 912         if (server_shutting_down)
 913                 return;
 914         if (fstat(fd, &statbuf) == -1) {
 915                 syslog(LOG_DEBUG, "ERROR: could not stat configfile '%s' - %s\n",
 916                               filename, strerror(errno));
 917                 return;
 918         }
 919         if (server_shutting_down)
 920                 return;
 921         CfgData = NewStrBufPlain(NULL, statbuf.st_size + 1);
 922         if (StrBufReadBLOB(CfgData, &fd, 1, statbuf.st_size, &Err) < 0) {
 923                 close(fd);
 924                 FreeStrBuf(&CfgData);
 925                 syslog(LOG_DEBUG, "ERROR: reading config '%s' - %s<br>\n",
 926                               filename, strerror(errno));
 927                 return;
 928         }
 929         close(fd);
 930         if (server_shutting_down)
 931                 return;
 932
 933         CfgPtr = NULL;
 934         CfgType = NewStrBuf();
 935         Line = NewStrBufPlain(NULL, StrLength(CfgData));
 936         Done = 0;
 937
 938         while (!Done)
 939         {
 940                 Done = StrBufSipLine(Line, CfgData, &CfgPtr) == 0;
 941                 if (StrLength(Line) > 0)
 942                 {
 943                         lPtr = NULL;
 944                         StrBufExtract_NextToken(CfgType, Line, &lPtr, '|');
 945                         if (!strcasecmp("pop3client", ChrPtr(CfgType)))
 946                         {
 947                                 pop3aggr *cptr;
 948                                 StrBuf *Tmp;
 949 /*
 950                                 if (Count == NULL)
 951                                 {
 952                                         Count = malloc(sizeof(pop3_room_counter));
 953                                         Count->count = 0;
 954                                 }
 955                                 Count->count ++;
 956 */
 957                                 cptr = (pop3aggr *) malloc(sizeof(pop3aggr));
 958                                 memset(cptr, 0, sizeof(pop3aggr));
 959                                 /// TODO do we need this? cptr->roomlist_parts = 1;
 960                                 cptr->RoomName = NewStrBufPlain(qrbuf->QRname, -1);
 961                                 cptr->pop3user = NewStrBufPlain(NULL, StrLength(Line));
 962                                 cptr->pop3pass = NewStrBufPlain(NULL, StrLength(Line));
 963                                 cptr->Url = NewStrBuf();
 964                                 Tmp = NewStrBuf();
 965
 966                                 StrBufExtract_NextToken(Tmp, Line, &lPtr, '|');
 967                                 StrBufExtract_NextToken(cptr->pop3user, Line, &lPtr, '|');
 968                                 StrBufExtract_NextToken(cptr->pop3pass, Line, &lPtr, '|');
 969                                 cptr->keep = StrBufExtractNext_long(Line, &lPtr, '|');
 970                                 cptr->interval = StrBufExtractNext_long(Line, &lPtr, '|');
 971
 972                                 StrBufPrintf(cptr->Url, "pop3://%s:%s@%s/%s",
 973                                              ChrPtr(cptr->pop3user),
 974                                              ChrPtr(cptr->pop3pass),
 975                                              ChrPtr(Tmp),
 976                                              ChrPtr(cptr->RoomName));
 977                                 FreeStrBuf(&Tmp);
 978                                 ParseURL(&cptr->IO.ConnectMe, cptr->Url, 110);
 979
 980
 981 #if 0
 982 /* todo: we need to reunite the url to be shure. */
 983
 984                                 pthread_mutex_lock(&POP3ueueMutex);
 985                                 GetHash(POP3FetchUrls, SKEY(ptr->Url), &vptr);
 986                                 use_this_cptr = (pop3aggr *)vptr;
 987
 988                                 if (use_this_rncptr != NULL)
 989                                 {
 990                                         /* mustn't attach to an active session */
 991                                         if (use_this_cptr->RefCount > 0)
 992                                         {
 993                                                 DeletePOP3Cfg(cptr);
 994 ///                                             Count->count--;
 995                                         }
 996                                         else
 997                                         {
 998                                                 long *QRnumber;
 999                                                 StrBufAppendBufPlain(use_this_cptr->rooms,
1000                                                                      qrbuf->QRname,
1001                                                                      -1, 0);
1002                                                 if (use_this_cptr->roomlist_parts == 1)
1003                                                 {
1004                                                         use_this_cptr->OtherQRnumbers = NewHash(1, lFlathash);
1005                                                 }
1006                                                 QRnumber = (long*)malloc(sizeof(long));
1007                                                 *QRnumber = qrbuf->QRnumber;
1008                                                 Put(use_this_cptr->OtherQRnumbers, LKEY(qrbuf->QRnumber), QRnumber, NULL);
1009                                                 use_this_cptr->roomlist_parts++;
1010                                         }
1011                                         pthread_mutex_unlock(&POP3QueueMutex);
1012                                         continue;
1013                                 }
1014                                 pthread_mutex_unlock(&RSSQueueMutex);
1015 #endif
1016
1017                                 pthread_mutex_lock(&POP3QueueMutex);
1018                                 Put(POP3FetchUrls, SKEY(cptr->Url), cptr, DeletePOP3Aggregator);
1019                                 pthread_mutex_unlock(&POP3QueueMutex);
1020
1021                         }
1022
1023                 }
1024
1025                 ///fclose(fp);
1026
1027         }
1028         FreeStrBuf(&Line);
1029         FreeStrBuf(&CfgType);
1030         FreeStrBuf(&CfgData);
1031 }
1032
1033
1034 void pop3client_scan(void) {
1035         static time_t last_run = 0L;
1036         static int doing_pop3client = 0;
1037 ///     struct pop3aggr *pptr;
1038         time_t fastest_scan;
1039         HashPos *it;
1040         long len;
1041         const char *Key;
1042         void *vrptr;
1043         pop3aggr *cptr;
1044
1045         if (config.c_pop3_fastest < config.c_pop3_fetch)
1046                 fastest_scan = config.c_pop3_fastest;
1047         else
1048                 fastest_scan = config.c_pop3_fetch;
1049
1050         /*
1051          * Run POP3 aggregation no more frequently than once every n seconds
1052          */
1053         if ( (time(NULL) - last_run) < fastest_scan ) {
1054                 return;
1055         }
1056
1057         /*
1058          * This is a simple concurrency check to make sure only one pop3client run
1059          * is done at a time.  We could do this with a mutex, but since we
1060          * don't really require extremely fine granularity here, we'll do it
1061          * with a static variable instead.
1062          */
1063         if (doing_pop3client) return;
1064         doing_pop3client = 1;
1065
1066         syslog(LOG_DEBUG, "pop3client started\n");
1067         CtdlForEachRoom(pop3client_scan_room, NULL);
1068
1069         pthread_mutex_lock(&POP3QueueMutex);
1070         it = GetNewHashPos(POP3FetchUrls, 0);
1071         while (!server_shutting_down &&
1072                GetNextHashPos(POP3FetchUrls, it, &len, &Key, &vrptr) &&
1073                (vrptr != NULL)) {
1074                 cptr = (pop3aggr *)vrptr;
1075                 if (cptr->RefCount == 0)
1076                         if (!pop3_do_fetching(cptr))
1077                                 DeletePOP3Aggregator(cptr);////TODO
1078
1079 /*
1080                 if ((palist->interval && time(NULL) > (last_run + palist->interval))
1081                         || (time(NULL) > last_run + config.c_pop3_fetch))
1082                                 pop3_do_fetching(palist->roomname, palist->pop3host,
1083                                         palist->pop3user, palist->pop3pass, palist->keep);
1084                 pptr = palist;
1085                 palist = palist->next;
1086                 free(pptr);
1087 */
1088         }
1089         DeleteHashPos(&it);
1090         pthread_mutex_unlock(&POP3QueueMutex);
1091
1092         syslog(LOG_DEBUG, "pop3client ended\n");
1093         last_run = time(NULL);
1094         doing_pop3client = 0;
1095 }
1096
1097
1098 void pop3_cleanup(void)
1099 {
1100         /* citthread_mutex_destroy(&POP3QueueMutex); TODO */
1101         DeleteHash(&POP3FetchUrls);
1102         DeleteHash(&POP3QueueRooms);
1103 }
1104
1105 CTDL_MODULE_INIT(pop3client)
1106 {
1107         if (!threading)
1108         {
1109                 CtdlFillSystemContext(&pop3_client_CC, "POP3aggr");
1110                 pthread_mutex_init(&POP3QueueMutex, NULL);
1111                 POP3QueueRooms = NewHash(1, lFlathash);
1112                 POP3FetchUrls = NewHash(1, NULL);
1113                 CtdlRegisterSessionHook(pop3client_scan, EVT_TIMER);
1114                 CtdlRegisterCleanupHook(pop3_cleanup);
1115         }
1116         /* return our Subversion id for the Log */
1117         return "pop3client";
1118 }