citadel/modules/pop3client/serv_pop3client.c

   1 /*
   2  * Consolidate mail from remote POP3 accounts.
   3  *
   4  * Copyright (c) 2007-2009 by the citadel.org team
   5  *
   6  *  This program is free software; you can redistribute it and/or modify
   7  *  it under the terms of the GNU General Public License as published by
   8  *  the Free Software Foundation; either version 3 of the License, or
   9  *  (at your option) any later version.
  10  *
  11  *  This program is distributed in the hope that it will be useful,
  12  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
  13  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14  *  GNU General Public License for more details.
  15  *
  16  *  You should have received a copy of the GNU General Public License
  17  *  along with this program; if not, write to the Free Software
  18  *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  19  */
  20
  21 #include <stdlib.h>
  22 #include <unistd.h>
  23 #include <stdio.h>
  24
  25 #if TIME_WITH_SYS_TIME
  26 # include <sys/time.h>
  27 # include <time.h>
  28 #else
  29 # if HAVE_SYS_TIME_H
  30 #  include <sys/time.h>
  31 # else
  32 #  include <time.h>
  33 # endif
  34 #endif
  35
  36 #include <ctype.h>
  37 #include <string.h>
  38 #include <errno.h>
  39 #include <sys/types.h>
  40 #include <sys/stat.h>
  41 #include <libcitadel.h>
  42 #include "citadel.h"
  43 #include "server.h"
  44 #include "citserver.h"
  45 #include "support.h"
  46 #include "config.h"
  47 #include "ctdl_module.h"
  48 #include "clientsocket.h"
  49 #include "msgbase.h"
  50 #include "internet_addressing.h"
  51 #include "database.h"
  52 #include "citadel_dirs.h"
  53 #include "event_client.h"
  54
  55
  56 struct CitContext pop3_client_CC;
  57
  58 citthread_mutex_t POP3QueueMutex; /* locks the access to the following vars: */
  59 HashList *POP3QueueRooms = NULL; /* rss_room_counter */
  60 HashList *POP3FetchUrls = NULL; /* -> rss_aggregator; ->RefCount access to be locked too. */
  61
  62 typedef struct __pop3_room_counter {
  63         int count;
  64         long QRnumber;
  65 }pop3_room_counter;
  66
  67 typedef enum ePOP3_C_States {
  68         ReadGreeting,
  69         GetUserState,
  70         GetPassState,
  71         GetListCommandState,
  72         GetListOneLine,
  73         GetOneMessageIDState,
  74         ReadMessageBodyFollowing,
  75         ReadMessageBody,
  76         GetDeleteState,
  77         ReadQuitState,
  78         POP3C_MaxRead
  79 }ePOP3_C_States;
  80
  81
  82 typedef struct _FetchItem {
  83         long MSGID;
  84         long MSGSize;
  85         StrBuf *MsgUIDL;
  86         StrBuf *MsgUID;
  87         int NeedFetch;
  88         struct CtdlMessage *Msg;
  89 } FetchItem;
  90
  91 void HfreeFetchItem(void *vItem)
  92 {
  93         FetchItem *Item = (FetchItem*) vItem;
  94         FreeStrBuf(&Item->MsgUIDL);
  95         FreeStrBuf(&Item->MsgUID);
  96         free(Item);
  97 }
  98
  99 typedef struct __pop3aggr {
 100         AsyncIO          IO;
 101
 102         long n;
 103         long RefCount;
 104 ///     ParsedURL *Pop3Host;
 105         DNSQueryParts HostLookup;
 106
 107 //      StrBuf          *rooms;
 108         long             QRnumber;
 109         HashList        *OtherQRnumbers;
 110
 111         StrBuf          *Url;
 112 ///     StrBuf *pop3host; -> URL
 113         StrBuf *pop3user;
 114         StrBuf *pop3pass;
 115         StrBuf *RoomName; // TODO: fill me
 116         int keep;
 117         time_t interval;
 118         ePOP3_C_States State;
 119         HashList *MsgNumbers;
 120         HashPos *Pos;
 121         FetchItem *CurrMsg;
 122 } pop3aggr;
 123
 124 void DeletePOP3Aggregator(void *vptr)
 125 {
 126         pop3aggr *ptr = vptr;
 127         DeleteHashPos(&ptr->Pos);
 128         DeleteHash(&ptr->MsgNumbers);
 129 //      FreeStrBuf(&ptr->rooms);
 130         FreeStrBuf(&ptr->pop3user);
 131         FreeStrBuf(&ptr->pop3pass);
 132         FreeStrBuf(&ptr->RoomName);
 133         FreeURL(&ptr->IO.ConnectMe);
 134         FreeStrBuf(&ptr->Url);
 135         FreeStrBuf(&ptr->IO.IOBuf);
 136         FreeStrBuf(&ptr->IO.SendBuf.Buf);
 137         FreeStrBuf(&ptr->IO.RecvBuf.Buf);
 138         free(ptr);
 139 }
 140
 141
 142 typedef eNextState(*Pop3ClientHandler)(pop3aggr* RecvMsg);
 143
 144 eNextState POP3_C_Shutdown(AsyncIO *IO);
 145 eNextState POP3_C_Timeout(AsyncIO *IO);
 146 eNextState POP3_C_ConnFail(AsyncIO *IO);
 147 eNextState POP3_C_DispatchReadDone(AsyncIO *IO);
 148 eNextState POP3_C_DispatchWriteDone(AsyncIO *IO);
 149 eNextState POP3_C_Terminate(AsyncIO *IO);
 150 eReadState POP3_C_ReadServerStatus(AsyncIO *IO);
 151 eNextState POP3_C_ReAttachToFetchMessages(AsyncIO *IO);
 152
 153 eNextState FinalizePOP3AggrRun(AsyncIO *IO)
 154 {
 155         HashPos  *It;
 156         pop3aggr *cptr = (pop3aggr *)IO->Data;
 157
 158         CtdlLogPrintf(CTDL_DEBUG, "Terminating Aggregator; bye.\n");
 159
 160         It = GetNewHashPos(POP3FetchUrls, 0);
 161         citthread_mutex_lock(&POP3QueueMutex);
 162         {
 163                 GetHashPosFromKey(POP3FetchUrls, SKEY(cptr->Url), It);
 164                 DeleteEntryFromHash(POP3FetchUrls, It);
 165         }
 166         citthread_mutex_unlock(&POP3QueueMutex);
 167         DeleteHashPos(&It);
 168         return eAbort;
 169 }
 170
 171 eNextState FailAggregationRun(AsyncIO *IO)
 172 {
 173         return eAbort;
 174 }
 175
 176 #define POP3C_DBG_SEND() CtdlLogPrintf(CTDL_DEBUG, "POP3 client[%ld]: > %s\n", RecvMsg->n, ChrPtr(RecvMsg->IO.SendBuf.Buf))
 177 #define POP3C_DBG_READ() CtdlLogPrintf(CTDL_DEBUG, "POP3 client[%ld]: < %s\n", RecvMsg->n, ChrPtr(RecvMsg->IO.IOBuf))
 178 #define POP3C_OK (strncasecmp(ChrPtr(RecvMsg->IO.IOBuf), "+OK", 3) == 0)
 179
 180 eNextState POP3C_ReadGreeting(pop3aggr *RecvMsg)
 181 {
 182         POP3C_DBG_READ();
 183         /* Read the server greeting */
 184         if (!POP3C_OK) return eTerminateConnection;
 185         else return eSendReply;
 186 }
 187
 188
 189 eNextState POP3C_SendUser(pop3aggr *RecvMsg)
 190 {
 191         /* Identify ourselves.  NOTE: we have to append a CR to each command.  The LF will
 192          * automatically be appended by sock_puts().  Believe it or not, leaving out the CR
 193          * will cause problems if the server happens to be Exchange, which is so b0rken it
 194          * actually barfs on LF-terminated newlines.
 195          */
 196         StrBufPrintf(RecvMsg->IO.SendBuf.Buf,
 197                      "USER %s\r\n", ChrPtr(RecvMsg->pop3user));
 198         POP3C_DBG_SEND();
 199         return eReadMessage;
 200 }
 201
 202 eNextState POP3C_GetUserState(pop3aggr *RecvMsg)
 203 {
 204         POP3C_DBG_READ();
 205         if (!POP3C_OK) return eTerminateConnection;
 206         else return eSendReply;
 207 }
 208
 209 eNextState POP3C_SendPassword(pop3aggr *RecvMsg)
 210 {
 211         /* Password */
 212         StrBufPrintf(RecvMsg->IO.SendBuf.Buf,
 213                      "PASS %s\r\n", ChrPtr(RecvMsg->pop3pass));
 214         CtdlLogPrintf(CTDL_DEBUG, "<PASS <password>\n");
 215 //      POP3C_DBG_SEND();
 216         return eReadMessage;
 217 }
 218
 219 eNextState POP3C_GetPassState(pop3aggr *RecvMsg)
 220 {
 221         POP3C_DBG_READ();
 222         if (!POP3C_OK) return eTerminateConnection;
 223         else return eSendReply;
 224 }
 225
 226 eNextState POP3C_SendListCommand(pop3aggr *RecvMsg)
 227 {
 228         /* Get the list of messages */
 229         StrBufPlain(RecvMsg->IO.SendBuf.Buf, HKEY("LIST\r\n"));
 230         POP3C_DBG_SEND();
 231         return eReadMessage;
 232 }
 233
 234 eNextState POP3C_GetListCommandState(pop3aggr *RecvMsg)
 235 {
 236         POP3C_DBG_READ();
 237         if (!POP3C_OK) return eTerminateConnection;
 238         RecvMsg->MsgNumbers = NewHash(1, NULL);
 239         RecvMsg->State++;
 240         return eReadMore;
 241 }
 242
 243
 244 eNextState POP3C_GetListOneLine(pop3aggr *RecvMsg)
 245 {
 246         int rc;
 247         const char *pch;
 248         FetchItem *OneMsg = NULL;
 249         POP3C_DBG_READ();
 250
 251         if ((StrLength(RecvMsg->IO.IOBuf) == 1) &&
 252             (ChrPtr(RecvMsg->IO.IOBuf)[0] == '.'))
 253         {
 254                 if (GetCount(RecvMsg->MsgNumbers) == 0)
 255                 {
 256                         ////    RecvMsg->Sate = ReadQuitState;
 257                 }
 258                 else
 259                 {
 260                         RecvMsg->Pos = GetNewHashPos(RecvMsg->MsgNumbers, 0);
 261                 }
 262                 return eSendReply;
 263
 264         }
 265         OneMsg = (FetchItem*) malloc(sizeof(FetchItem));
 266         memset(OneMsg, 0, sizeof(FetchItem));
 267         OneMsg->MSGID = atol(ChrPtr(RecvMsg->IO.IOBuf));
 268
 269         pch = strchr(ChrPtr(RecvMsg->IO.IOBuf), ' ');
 270         if (pch != NULL)
 271         {
 272                 OneMsg->MSGSize = atol(pch + 1);
 273         }
 274
 275         rc = TestValidateHash(RecvMsg->MsgNumbers);
 276         if (rc != 0)
 277                 CtdlLogPrintf(CTDL_DEBUG, "Hash Invalid: %d\n", rc);
 278
 279         Put(RecvMsg->MsgNumbers, LKEY(OneMsg->MSGID), OneMsg, HfreeFetchItem);
 280
 281         rc = TestValidateHash(RecvMsg->MsgNumbers);
 282         if (rc != 0)
 283                 CtdlLogPrintf(CTDL_DEBUG, "Hash Invalid: %d\n", rc);
 284         //RecvMsg->State --; /* read next Line */
 285         return eReadMore;
 286 }
 287
 288 eNextState POP3_FetchNetworkUsetableEntry(AsyncIO *IO)
 289 {
 290         long HKLen;
 291         const char *HKey;
 292         void *vData;
 293         struct cdbdata *cdbut;
 294         pop3aggr *RecvMsg = (pop3aggr *) IO->Data;
 295
 296         if(GetNextHashPos(RecvMsg->MsgNumbers, RecvMsg->Pos, &HKLen, &HKey, &vData))
 297         {
 298                 struct UseTable ut;
 299
 300                 RecvMsg->CurrMsg = (FetchItem*) vData;
 301                 CtdlLogPrintf(CTDL_DEBUG, "CHECKING: whether %s has already been seen: ", ChrPtr(RecvMsg->CurrMsg->MsgUID));
 302                 /* Find out if we've already seen this item */
 303                 safestrncpy(ut.ut_msgid,
 304                             ChrPtr(RecvMsg->CurrMsg->MsgUID),
 305                             sizeof(ut.ut_msgid));
 306                 ut.ut_timestamp = time(NULL);/// TODO: libev timestamp!
 307
 308                 cdbut = cdb_fetch(CDB_USETABLE, SKEY(RecvMsg->CurrMsg->MsgUID));
 309                 if (cdbut != NULL) {
 310                         /* Item has already been seen */
 311                         CtdlLogPrintf(CTDL_DEBUG, "YES\n");
 312                         cdb_free(cdbut);
 313
 314                         /* rewrite the record anyway, to update the timestamp */
 315                         cdb_store(CDB_USETABLE,
 316                                   SKEY(RecvMsg->CurrMsg->MsgUID),
 317                                   &ut, sizeof(struct UseTable) );
 318                         RecvMsg->CurrMsg->NeedFetch = 1; ////TODO0;
 319                 }
 320                 else
 321                 {
 322                         CtdlLogPrintf(CTDL_DEBUG, "NO\n");
 323                         RecvMsg->CurrMsg->NeedFetch = 1;
 324                 }
 325                 return NextDBOperation(&RecvMsg->IO, POP3_FetchNetworkUsetableEntry);
 326         }
 327         else
 328         {
 329                 /* ok, now we know them all, continue with reading the actual messages. */
 330                 DeleteHashPos(&RecvMsg->Pos);
 331
 332                 return QueueEventContext(IO, POP3_C_ReAttachToFetchMessages);
 333         }
 334 }
 335
 336 eNextState POP3C_GetOneMessagID(pop3aggr *RecvMsg)
 337 {
 338         long HKLen;
 339         const char *HKey;
 340         void *vData;
 341         int rc;
 342
 343         rc = TestValidateHash(RecvMsg->MsgNumbers);
 344         if (rc != 0)
 345                 CtdlLogPrintf(CTDL_DEBUG, "Hash Invalid: %d\n", rc);
 346
 347         if(GetNextHashPos(RecvMsg->MsgNumbers, RecvMsg->Pos, &HKLen, &HKey, &vData))
 348         {
 349                 RecvMsg->CurrMsg = (FetchItem*) vData;
 350                 /* Find out the UIDL of the message, to determine whether we've already downloaded it */
 351                 StrBufPrintf(RecvMsg->IO.SendBuf.Buf,
 352                              "UIDL %ld\r\n", RecvMsg->CurrMsg->MSGID);
 353                 POP3C_DBG_SEND();
 354         }
 355         else
 356         {
 357             RecvMsg->State++;
 358                 DeleteHashPos(&RecvMsg->Pos);
 359                 /// done receiving uidls.. start looking them up now.
 360                 RecvMsg->Pos = GetNewHashPos(RecvMsg->MsgNumbers, 0);
 361                 return QueueDBOperation(&RecvMsg->IO, POP3_FetchNetworkUsetableEntry);
 362         }
 363         return eReadMore; /* TODO */
 364 }
 365
 366 eNextState POP3C_GetOneMessageIDState(pop3aggr *RecvMsg)
 367 {
 368         int rc;
 369         rc = TestValidateHash(RecvMsg->MsgNumbers);
 370         if (rc != 0)
 371                 CtdlLogPrintf(CTDL_DEBUG, "Hash Invalid: %d\n", rc);
 372
 373         POP3C_DBG_READ();
 374         if (!POP3C_OK) return eTerminateConnection;
 375         RecvMsg->CurrMsg->MsgUIDL = NewStrBufPlain(NULL, StrLength(RecvMsg->IO.IOBuf));
 376         RecvMsg->CurrMsg->MsgUID = NewStrBufPlain(NULL, StrLength(RecvMsg->IO.IOBuf) * 2);
 377
 378         StrBufExtract_token(RecvMsg->CurrMsg->MsgUIDL, RecvMsg->IO.IOBuf, 2, ' ');
 379         StrBufPrintf(RecvMsg->CurrMsg->MsgUID,
 380                      "pop3/%s/%s:%s@%s",
 381                      ChrPtr(RecvMsg->RoomName),
 382                      ChrPtr(RecvMsg->CurrMsg->MsgUIDL),
 383                      RecvMsg->IO.ConnectMe->User,
 384                      RecvMsg->IO.ConnectMe->Host);
 385         RecvMsg->State --;
 386         return eSendReply;
 387 }
 388
 389
 390 eNextState POP3C_SendGetOneMsg(pop3aggr *RecvMsg)
 391 {
 392         long HKLen;
 393         const char *HKey;
 394         void *vData;
 395
 396         RecvMsg->CurrMsg = NULL;
 397         while (GetNextHashPos(RecvMsg->MsgNumbers, RecvMsg->Pos, &HKLen, &HKey, &vData) &&
 398                (RecvMsg->CurrMsg = (FetchItem*) vData, RecvMsg->CurrMsg->NeedFetch == 0))
 399         {}
 400
 401         if ((RecvMsg->CurrMsg != NULL ) && (RecvMsg->CurrMsg->NeedFetch == 1))
 402         {
 403                 /* Message has not been seen. Tell the server to fetch the message... */
 404                 StrBufPrintf(RecvMsg->IO.SendBuf.Buf,
 405                              "RETR %ld\r\n", RecvMsg->CurrMsg->MSGID);
 406                 POP3C_DBG_SEND();
 407                 return eReadMessage;
 408         }
 409         else {
 410                 RecvMsg->State = ReadQuitState;
 411                 return POP3_C_DispatchWriteDone(&RecvMsg->IO);
 412         }
 413 }
 414
 415
 416 eNextState POP3C_ReadMessageBodyFollowing(pop3aggr *RecvMsg)
 417 {
 418         POP3C_DBG_READ();
 419         if (!POP3C_OK) return eTerminateConnection;
 420         RecvMsg->IO.ReadMsg = NewAsyncMsg(HKEY("."),
 421                                           RecvMsg->CurrMsg->MSGSize,
 422                                           config.c_maxmsglen,
 423                                           NULL, -1,
 424                                           1);
 425
 426         return eReadPayload;
 427 }
 428
 429
 430 eNextState POP3C_StoreMsgRead(AsyncIO *IO)
 431 {
 432         pop3aggr *RecvMsg = (pop3aggr *) IO->Data;
 433         struct UseTable ut;
 434
 435         CtdlLogPrintf(CTDL_DEBUG, "MARKING: %s as seen: ", ChrPtr(RecvMsg->CurrMsg->MsgUID));
 436
 437         safestrncpy(ut.ut_msgid,
 438                     ChrPtr(RecvMsg->CurrMsg->MsgUID),
 439                     sizeof(ut.ut_msgid));
 440         ut.ut_timestamp = time(NULL); /* TODO: use libev time */
 441         cdb_store(CDB_USETABLE,
 442                   ChrPtr(RecvMsg->CurrMsg->MsgUID),
 443                   StrLength(RecvMsg->CurrMsg->MsgUID),
 444                   &ut,
 445                   sizeof(struct UseTable) );
 446
 447         return QueueEventContext(&RecvMsg->IO, POP3_C_ReAttachToFetchMessages);
 448 }
 449 eNextState POP3C_SaveMsg(AsyncIO *IO)
 450 {
 451         long msgnum;
 452         pop3aggr *RecvMsg = (pop3aggr *) IO->Data;
 453
 454         /* Do Something With It (tm) */
 455         msgnum = CtdlSubmitMsg(RecvMsg->CurrMsg->Msg,
 456                                NULL,
 457                                ChrPtr(RecvMsg->RoomName),
 458                                0);
 459         if (msgnum > 0L) {
 460                 /* Message has been committed to the store */
 461                 /* write the uidl to the use table so we don't fetch this message again */
 462         }
 463         CtdlFreeMessage(RecvMsg->CurrMsg->Msg);
 464
 465         return NextDBOperation(&RecvMsg->IO, POP3C_StoreMsgRead);
 466         return eReadMessage;
 467 }
 468
 469 eNextState POP3C_ReadMessageBody(pop3aggr *RecvMsg)
 470 {
 471         CtdlLogPrintf(CTDL_DEBUG, "Converting message...\n");
 472         RecvMsg->CurrMsg->Msg = convert_internet_message_buf(&RecvMsg->IO.ReadMsg->MsgBuf);
 473
 474         return QueueDBOperation(&RecvMsg->IO, POP3C_SaveMsg);
 475 }
 476
 477 eNextState POP3C_SendDelete(pop3aggr *RecvMsg)
 478 {
 479         if (!RecvMsg->keep) {
 480                 StrBufPrintf(RecvMsg->IO.SendBuf.Buf,
 481                              "DELE %ld\r\n", RecvMsg->CurrMsg->MSGID);
 482                 POP3C_DBG_SEND();
 483                 return eReadMessage;
 484         }
 485         else {
 486                 RecvMsg->State = ReadMessageBodyFollowing;
 487                 return POP3_C_DispatchWriteDone(&RecvMsg->IO);
 488         }
 489 }
 490 eNextState POP3C_ReadDeleteState(pop3aggr *RecvMsg)
 491 {
 492         POP3C_DBG_READ();
 493         RecvMsg->State = GetOneMessageIDState;
 494         return eReadMessage;
 495 }
 496
 497 eNextState POP3C_SendQuit(pop3aggr *RecvMsg)
 498 {
 499         /* Log out */
 500         StrBufPlain(RecvMsg->IO.SendBuf.Buf,
 501                     HKEY("QUIT\r\n3)"));
 502         POP3C_DBG_SEND();
 503         return eReadMessage;
 504 }
 505
 506
 507 eNextState POP3C_ReadQuitState(pop3aggr *RecvMsg)
 508 {
 509         POP3C_DBG_READ();
 510         return eTerminateConnection;
 511 }
 512
 513 const long POP3_C_ConnTimeout = 1000;
 514 const long DefaultPOP3Port = 110;
 515
 516 Pop3ClientHandler POP3C_ReadHandlers[] = {
 517         POP3C_ReadGreeting,
 518         POP3C_GetUserState,
 519         POP3C_GetPassState,
 520         POP3C_GetListCommandState,
 521         POP3C_GetListOneLine,
 522         POP3C_GetOneMessageIDState,
 523         POP3C_ReadMessageBodyFollowing,
 524         POP3C_ReadMessageBody,
 525         POP3C_ReadDeleteState,
 526         POP3C_ReadQuitState,
 527 };
 528
 529 const long POP3_C_SendTimeouts[POP3C_MaxRead] = {
 530         100,
 531         100,
 532         100,
 533         100,
 534         100,
 535         100,
 536         100,
 537         100
 538 };
 539 const ConstStr POP3C_ReadErrors[POP3C_MaxRead] = {
 540         {HKEY("Connection broken during ")},
 541         {HKEY("Connection broken during ")},
 542         {HKEY("Connection broken during ")},
 543         {HKEY("Connection broken during ")},
 544         {HKEY("Connection broken during ")},
 545         {HKEY("Connection broken during ")},
 546         {HKEY("Connection broken during ")},
 547         {HKEY("Connection broken during ")}
 548 };
 549
 550 Pop3ClientHandler POP3C_SendHandlers[] = {
 551         NULL, /* we don't send a greeting */
 552         POP3C_SendUser,
 553         POP3C_SendPassword,
 554         POP3C_SendListCommand,
 555         NULL,
 556         POP3C_GetOneMessagID,
 557         POP3C_SendGetOneMsg,
 558         NULL,
 559         POP3C_SendDelete,
 560         POP3C_SendQuit
 561 };
 562
 563 const long POP3_C_ReadTimeouts[] = {
 564         100,
 565         100,
 566         100,
 567         100,
 568         100,
 569         100,
 570         100,
 571         100,
 572         100,
 573         100
 574 };
 575 /*****************************************************************************/
 576 /*                     POP3 CLIENT DISPATCHER                                */
 577 /*****************************************************************************/
 578
 579 void POP3SetTimeout(eNextState NextTCPState, pop3aggr *pMsg)
 580 {
 581         double Timeout = 0.0;
 582
 583         CtdlLogPrintf(CTDL_DEBUG, "POP3: %s\n", __FUNCTION__);
 584
 585         switch (NextTCPState) {
 586         case eSendReply:
 587         case eSendMore:
 588                 Timeout = POP3_C_SendTimeouts[pMsg->State];
 589 /*
 590   if (pMsg->State == eDATABody) {
 591   / * if we're sending a huge message, we need more time. * /
 592   Timeout += StrLength(pMsg->msgtext) / 1024;
 593   }
 594 */
 595                 break;
 596         case eReadMessage:
 597                 Timeout = POP3_C_ReadTimeouts[pMsg->State];
 598 /*
 599   if (pMsg->State == eDATATerminateBody) {
 600   / *
 601   * some mailservers take a nap before accepting the message
 602   * content inspection and such.
 603   * /
 604   Timeout += StrLength(pMsg->msgtext) / 1024;
 605   }
 606 */
 607                 break;
 608         case eReadPayload:
 609                 Timeout = 100000;
 610                 /* TODO!!! */
 611                 break;
 612         case eSendDNSQuery:
 613         case eReadDNSReply:
 614         case eConnect:
 615         case eTerminateConnection:
 616         case eDBQuery:
 617         case eAbort:
 618         case eReadMore://// TODO
 619                 return;
 620         }
 621         SetNextTimeout(&pMsg->IO, Timeout);
 622 }
 623 eNextState POP3_C_DispatchReadDone(AsyncIO *IO)
 624 {
 625         CtdlLogPrintf(CTDL_DEBUG, "POP3: %s\n", __FUNCTION__);
 626         pop3aggr *pMsg = IO->Data;
 627         eNextState rc;
 628
 629         rc = POP3C_ReadHandlers[pMsg->State](pMsg);
 630         if (rc != eReadMore)
 631             pMsg->State++;
 632         POP3SetTimeout(rc, pMsg);
 633         return rc;
 634 }
 635 eNextState POP3_C_DispatchWriteDone(AsyncIO *IO)
 636 {
 637         CtdlLogPrintf(CTDL_DEBUG, "POP3: %s\n", __FUNCTION__);
 638         pop3aggr *pMsg = IO->Data;
 639         eNextState rc;
 640
 641         rc = POP3C_SendHandlers[pMsg->State](pMsg);
 642         POP3SetTimeout(rc, pMsg);
 643         return rc;
 644 }
 645
 646
 647 /*****************************************************************************/
 648 /*                     POP3 CLIENT ERROR CATCHERS                            */
 649 /*****************************************************************************/
 650 eNextState POP3_C_Terminate(AsyncIO *IO)
 651 {
 652 ///     pop3aggr *pMsg = (pop3aggr *)IO->Data;
 653
 654         CtdlLogPrintf(CTDL_DEBUG, "POP3: %s\n", __FUNCTION__);
 655         FinalizePOP3AggrRun(IO);
 656         return eAbort;
 657 }
 658 eNextState POP3_C_Timeout(AsyncIO *IO)
 659 {
 660         pop3aggr *pMsg = IO->Data;
 661
 662         CtdlLogPrintf(CTDL_DEBUG, "POP3: %s\n", __FUNCTION__);
 663         StrBufPlain(IO->ErrMsg, CKEY(POP3C_ReadErrors[pMsg->State]));
 664         return FailAggregationRun(IO);
 665 }
 666 eNextState POP3_C_ConnFail(AsyncIO *IO)
 667 {
 668         pop3aggr *pMsg = (pop3aggr *)IO->Data;
 669
 670         CtdlLogPrintf(CTDL_DEBUG, "POP3: %s\n", __FUNCTION__);
 671         StrBufPlain(IO->ErrMsg, CKEY(POP3C_ReadErrors[pMsg->State]));
 672         return FailAggregationRun(IO);
 673 }
 674 eNextState POP3_C_Shutdown(AsyncIO *IO)
 675 {
 676         CtdlLogPrintf(CTDL_DEBUG, "POP3: %s\n", __FUNCTION__);
 677 ////    pop3aggr *pMsg = IO->Data;
 678
 679         ////pMsg->MyQEntry->Status = 3;
 680         ///StrBufPlain(pMsg->MyQEntry->StatusMessage, HKEY("server shutdown during message retrieval."));
 681         FinalizePOP3AggrRun(IO);
 682         return eAbort;
 683 }
 684
 685
 686 /**
 687  * @brief lineread Handler; understands when to read more POP3 lines, and when this is a one-lined reply.
 688  */
 689 eReadState POP3_C_ReadServerStatus(AsyncIO *IO)
 690 {
 691         eReadState Finished = eBufferNotEmpty;
 692
 693         switch (IO->NextState) {
 694         case eSendDNSQuery:
 695         case eReadDNSReply:
 696         case eDBQuery:
 697         case eConnect:
 698         case eTerminateConnection:
 699         case eAbort:
 700                 Finished = eReadFail;
 701                 break;
 702         case eSendReply:
 703         case eSendMore:
 704         case eReadMore:
 705         case eReadMessage:
 706                 Finished = StrBufChunkSipLine(IO->IOBuf, &IO->RecvBuf);
 707                 break;
 708         case eReadPayload:
 709                 Finished = CtdlReadMessageBodyAsync(IO);
 710                 break;
 711         }
 712         return Finished;
 713 }
 714
 715 /*****************************************************************************
 716  * So we connect our Server IP here.                                         *
 717  *****************************************************************************/
 718 eNextState POP3_C_ReAttachToFetchMessages(AsyncIO *IO)
 719 {
 720         pop3aggr *cpptr = IO->Data;
 721
 722         CtdlLogPrintf(CTDL_DEBUG, "POP3: %s\n", __FUNCTION__);
 723 ////??? cpptr->State ++;
 724         if (cpptr->Pos == NULL)
 725                 cpptr->Pos = GetNewHashPos(cpptr->MsgNumbers, 0);
 726
 727         POP3_C_DispatchWriteDone(IO);
 728         ReAttachIO(IO, cpptr, 0);
 729         IO->NextState = eReadMessage;
 730         return IO->NextState;
 731 }
 732
 733 eNextState connect_ip(AsyncIO *IO)
 734 {
 735         pop3aggr *cpptr = IO->Data;
 736
 737         CtdlLogPrintf(CTDL_DEBUG, "POP3: %s\n", __FUNCTION__);
 738
 739 ////    IO->ConnectMe = &cpptr->Pop3Host;
 740         /*  Bypass the ns lookup result like this: IO->Addr.sin_addr.s_addr = inet_addr("127.0.0.1"); */
 741
 742         /////// SetConnectStatus(IO);
 743
 744         return InitEventIO(IO, cpptr,
 745                            POP3_C_ConnTimeout,
 746                            POP3_C_ReadTimeouts[0],
 747                            1);
 748 }
 749
 750 eNextState get_one_host_ip_done(AsyncIO *IO)
 751 {
 752         pop3aggr *cpptr = IO->Data;
 753         struct hostent *hostent;
 754
 755         QueryCbDone(IO);
 756
 757         hostent = cpptr->HostLookup.VParsedDNSReply;
 758         if ((cpptr->HostLookup.DNSStatus == ARES_SUCCESS) &&
 759             (hostent != NULL) ) {
 760                 memset(&cpptr->IO.ConnectMe->Addr, 0, sizeof(struct in6_addr));
 761                 if (cpptr->IO.ConnectMe->IPv6) {
 762                         memcpy(&cpptr->IO.ConnectMe->Addr.sin6_addr.s6_addr,
 763                                &hostent->h_addr_list[0],
 764                                sizeof(struct in6_addr));
 765
 766                         cpptr->IO.ConnectMe->Addr.sin6_family = hostent->h_addrtype;
 767                         cpptr->IO.ConnectMe->Addr.sin6_port   = htons(DefaultPOP3Port);
 768                 }
 769                 else {
 770                         struct sockaddr_in *addr = (struct sockaddr_in*) &cpptr->IO.ConnectMe->Addr;
 771                         /* Bypass the ns lookup result like this: IO->Addr.sin_addr.s_addr = inet_addr("127.0.0.1"); */
 772 //                      addr->sin_addr.s_addr = htonl((uint32_t)&hostent->h_addr_list[0]);
 773                         memcpy(&addr->sin_addr.s_addr,
 774                                hostent->h_addr_list[0],
 775                                sizeof(uint32_t));
 776
 777                         addr->sin_family = hostent->h_addrtype;
 778                         addr->sin_port   = htons(DefaultPOP3Port);
 779
 780                 }
 781                 return connect_ip(IO);
 782         }
 783         else
 784                 return eAbort;
 785 }
 786
 787 eNextState get_one_host_ip(AsyncIO *IO)
 788 {
 789         pop3aggr *cpptr = IO->Data;
 790         /*
 791          * here we start with the lookup of one host. it might be...
 792          * - the relay host *sigh*
 793          * - the direct hostname if there was no mx record
 794          * - one of the mx'es
 795          */
 796
 797         InitC_ares_dns(IO);
 798
 799         CtdlLogPrintf(CTDL_DEBUG, "POP3: %s\n", __FUNCTION__);
 800
 801         CtdlLogPrintf(CTDL_DEBUG,
 802                       "POP3 client[%ld]: looking up %s-Record %s : %d ...\n",
 803                       cpptr->n,
 804                       (cpptr->IO.ConnectMe->IPv6)? "aaaa": "a",
 805                       cpptr->IO.ConnectMe->Host,
 806                       cpptr->IO.ConnectMe->Port);
 807
 808         if (!QueueQuery((cpptr->IO.ConnectMe->IPv6)? ns_t_aaaa : ns_t_a,
 809                         cpptr->IO.ConnectMe->Host,
 810                         &cpptr->IO,
 811                         &cpptr->HostLookup,
 812                         get_one_host_ip_done))
 813         {
 814 //              cpptr->MyQEntry->Status = 5;
 815 //              StrBufPrintf(SendMsg->MyQEntry->StatusMessage,
 816 //                           "No MX hosts found for <%s>", SendMsg->node);
 817                 cpptr->IO.NextState = eTerminateConnection;
 818                 return IO->NextState;
 819         }
 820         IO->NextState = eReadDNSReply;
 821         return IO->NextState;
 822 }
 823
 824
 825
 826 int pop3_do_fetching(pop3aggr *cpptr)
 827 {
 828         CitContext *SubC;
 829
 830         cpptr->IO.Data          = cpptr;
 831
 832         cpptr->IO.SendDone      = POP3_C_DispatchWriteDone;
 833         cpptr->IO.ReadDone      = POP3_C_DispatchReadDone;
 834         cpptr->IO.Terminate     = POP3_C_Terminate;
 835         cpptr->IO.LineReader    = POP3_C_ReadServerStatus;
 836         cpptr->IO.ConnFail      = POP3_C_ConnFail;
 837         cpptr->IO.Timeout       = POP3_C_Timeout;
 838         cpptr->IO.ShutdownAbort = POP3_C_Shutdown;
 839
 840         cpptr->IO.SendBuf.Buf   = NewStrBufPlain(NULL, 1024);
 841         cpptr->IO.RecvBuf.Buf   = NewStrBufPlain(NULL, 1024);
 842         cpptr->IO.IOBuf         = NewStrBuf();
 843
 844         cpptr->IO.NextState     = eReadMessage;
 845 /* TODO
 846    CtdlLogPrintf(CTDL_DEBUG, "POP3: %s %s %s <password>\n", roomname, pop3host, pop3user);
 847    CtdlLogPrintf(CTDL_NOTICE, "Connecting to <%s>\n", pop3host);
 848 */
 849
 850         SubC = CloneContext (&pop3_client_CC);
 851         SubC->session_specific_data = (char*) cpptr;
 852         cpptr->IO.CitContext = SubC;
 853
 854         if (cpptr->IO.ConnectMe->IsIP) {
 855                 QueueEventContext(&cpptr->IO,
 856                                   connect_ip);
 857         }
 858         else { /* uneducated admin has chosen to add DNS to the equation... */
 859                 QueueEventContext(&cpptr->IO,
 860                                   get_one_host_ip);
 861         }
 862         return 1;
 863 }
 864
 865 /*
 866  * Scan a room's netconfig to determine whether it requires POP3 aggregation
 867  */
 868 void pop3client_scan_room(struct ctdlroom *qrbuf, void *data)
 869 {
 870         StrBuf *CfgData;
 871         StrBuf *CfgType;
 872         StrBuf *Line;
 873
 874         struct stat statbuf;
 875         char filename[PATH_MAX];
 876         int  fd;
 877         int Done;
 878         void *vptr;
 879         const char *CfgPtr, *lPtr;
 880         const char *Err;
 881
 882 //      pop3_room_counter *Count = NULL;
 883 //      pop3aggr *cpptr;
 884
 885         citthread_mutex_lock(&POP3QueueMutex);
 886         if (GetHash(POP3QueueRooms, LKEY(qrbuf->QRnumber), &vptr))
 887         {
 888                 CtdlLogPrintf(CTDL_DEBUG,
 889                               "pop3client: [%ld] %s already in progress.\n",
 890                               qrbuf->QRnumber,
 891                               qrbuf->QRname);
 892                 citthread_mutex_unlock(&POP3QueueMutex);
 893                 return;
 894         }
 895         citthread_mutex_unlock(&POP3QueueMutex);
 896
 897         assoc_file_name(filename, sizeof filename, qrbuf, ctdl_netcfg_dir);
 898
 899         if (CtdlThreadCheckStop())
 900                 return;
 901
 902         /* Only do net processing for rooms that have netconfigs */
 903         fd = open(filename, 0);
 904         if (fd <= 0) {
 905                 //CtdlLogPrintf(CTDL_DEBUG, "rssclient: %s no config.\n", qrbuf->QRname);
 906                 return;
 907         }
 908         if (CtdlThreadCheckStop())
 909                 return;
 910         if (fstat(fd, &statbuf) == -1) {
 911                 CtdlLogPrintf(CTDL_DEBUG,  "ERROR: could not stat configfile '%s' - %s\n",
 912                               filename, strerror(errno));
 913                 return;
 914         }
 915         if (CtdlThreadCheckStop())
 916                 return;
 917         CfgData = NewStrBufPlain(NULL, statbuf.st_size + 1);
 918         if (StrBufReadBLOB(CfgData, &fd, 1, statbuf.st_size, &Err) < 0) {
 919                 close(fd);
 920                 FreeStrBuf(&CfgData);
 921                 CtdlLogPrintf(CTDL_DEBUG,  "ERROR: reading config '%s' - %s<br>\n",
 922                               filename, strerror(errno));
 923                 return;
 924         }
 925         close(fd);
 926         if (CtdlThreadCheckStop())
 927                 return;
 928
 929         CfgPtr = NULL;
 930         CfgType = NewStrBuf();
 931         Line = NewStrBufPlain(NULL, StrLength(CfgData));
 932         Done = 0;
 933
 934         while (!Done)
 935         {
 936                 Done = StrBufSipLine(Line, CfgData, &CfgPtr) == 0;
 937                 if (StrLength(Line) > 0)
 938                 {
 939                         lPtr = NULL;
 940                         StrBufExtract_NextToken(CfgType, Line, &lPtr, '|');
 941                         if (!strcasecmp("pop3client", ChrPtr(CfgType)))
 942                         {
 943                                 pop3aggr *cptr;
 944                                 StrBuf *Tmp;
 945 /*
 946                                 if (Count == NULL)
 947                                 {
 948                                         Count = malloc(sizeof(pop3_room_counter));
 949                                         Count->count = 0;
 950                                 }
 951                                 Count->count ++;
 952 */
 953                                 cptr = (pop3aggr *) malloc(sizeof(pop3aggr));
 954                                 memset(cptr, 0, sizeof(pop3aggr));
 955                                 /// TODO do we need this? cptr->roomlist_parts = 1;
 956                                 cptr->RoomName = NewStrBufPlain(qrbuf->QRname, -1);
 957                                 cptr->pop3user = NewStrBufPlain(NULL, StrLength(Line));
 958                                 cptr->pop3pass = NewStrBufPlain(NULL, StrLength(Line));
 959                                 cptr->Url = NewStrBuf();
 960                                 Tmp = NewStrBuf();
 961
 962                                 StrBufExtract_NextToken(Tmp, Line, &lPtr, '|');
 963                                 StrBufExtract_NextToken(cptr->pop3user, Line, &lPtr, '|');
 964                                 StrBufExtract_NextToken(cptr->pop3pass, Line, &lPtr, '|');
 965                                 cptr->keep = StrBufExtractNext_long(Line, &lPtr, '|');
 966                                 cptr->interval = StrBufExtractNext_long(Line, &lPtr, '|');
 967
 968                                 StrBufPrintf(cptr->Url, "pop3://%s:%s@%s/%s",
 969                                              ChrPtr(cptr->pop3user),
 970                                              ChrPtr(cptr->pop3pass),
 971                                              ChrPtr(Tmp),
 972                                              ChrPtr(cptr->RoomName));
 973                                 FreeStrBuf(&Tmp);
 974                                 ParseURL(&cptr->IO.ConnectMe, cptr->Url, 110);
 975
 976
 977 #if 0
 978 /* todo: we need to reunite the url to be shure. */
 979
 980                                 citthread_mutex_lock(&POP3ueueMutex);
 981                                 GetHash(POP3FetchUrls, SKEY(ptr->Url), &vptr);
 982                                 use_this_cptr = (pop3aggr *)vptr;
 983
 984                                 if (use_this_rncptr != NULL)
 985                                 {
 986                                         /* mustn't attach to an active session */
 987                                         if (use_this_cptr->RefCount > 0)
 988                                         {
 989                                                 DeletePOP3Cfg(cptr);
 990 ///                                             Count->count--;
 991                                         }
 992                                         else
 993                                         {
 994                                                 long *QRnumber;
 995                                                 StrBufAppendBufPlain(use_this_cptr->rooms,
 996                                                                      qrbuf->QRname,
 997                                                                      -1, 0);
 998                                                 if (use_this_cptr->roomlist_parts == 1)
 999                                                 {
1000                                                         use_this_cptr->OtherQRnumbers = NewHash(1, lFlathash);
1001                                                 }
1002                                                 QRnumber = (long*)malloc(sizeof(long));
1003                                                 *QRnumber = qrbuf->QRnumber;
1004                                                 Put(use_this_cptr->OtherQRnumbers, LKEY(qrbuf->QRnumber), QRnumber, NULL);
1005                                                 use_this_cptr->roomlist_parts++;
1006                                         }
1007                                         citthread_mutex_unlock(&POP3QueueMutex);
1008                                         continue;
1009                                 }
1010                                 citthread_mutex_unlock(&RSSQueueMutex);
1011 #endif
1012
1013                                 citthread_mutex_lock(&POP3QueueMutex);
1014                                 Put(POP3FetchUrls, SKEY(cptr->Url), cptr, DeletePOP3Aggregator);
1015                                 citthread_mutex_unlock(&POP3QueueMutex);
1016
1017                         }
1018
1019                 }
1020
1021                 ///fclose(fp);
1022
1023         }
1024         FreeStrBuf(&Line);
1025         FreeStrBuf(&CfgType);
1026         FreeStrBuf(&CfgData);
1027 }
1028
1029
1030 void pop3client_scan(void) {
1031         static time_t last_run = 0L;
1032         static int doing_pop3client = 0;
1033 ///     struct pop3aggr *pptr;
1034         time_t fastest_scan;
1035         HashPos *it;
1036         long len;
1037         const char *Key;
1038         void *vrptr;
1039         pop3aggr *cptr;
1040
1041         if (config.c_pop3_fastest < config.c_pop3_fetch)
1042                 fastest_scan = config.c_pop3_fastest;
1043         else
1044                 fastest_scan = config.c_pop3_fetch;
1045
1046         /*
1047          * Run POP3 aggregation no more frequently than once every n seconds
1048          */
1049         if ( (time(NULL) - last_run) < fastest_scan ) {
1050                 return;
1051         }
1052
1053         /*
1054          * This is a simple concurrency check to make sure only one pop3client run
1055          * is done at a time.  We could do this with a mutex, but since we
1056          * don't really require extremely fine granularity here, we'll do it
1057          * with a static variable instead.
1058          */
1059         if (doing_pop3client) return;
1060         doing_pop3client = 1;
1061
1062         CtdlLogPrintf(CTDL_DEBUG, "pop3client started\n");
1063         CtdlForEachRoom(pop3client_scan_room, NULL);
1064
1065
1066         citthread_mutex_lock(&POP3QueueMutex);
1067         it = GetNewHashPos(POP3FetchUrls, 0);
1068         while (GetNextHashPos(POP3FetchUrls, it, &len, &Key, &vrptr) &&
1069                (vrptr != NULL)) {
1070                 cptr = (pop3aggr *)vrptr;
1071                 if (cptr->RefCount == 0)
1072                         if (!pop3_do_fetching(cptr))
1073                                 DeletePOP3Aggregator(cptr);////TODO
1074         }
1075         DeleteHashPos(&it);
1076         citthread_mutex_unlock(&POP3QueueMutex);
1077
1078         CtdlLogPrintf(CTDL_DEBUG, "pop3client ended\n");
1079         last_run = time(NULL);
1080         doing_pop3client = 0;
1081 }
1082
1083
1084 void pop3_cleanup(void)
1085 {
1086         citthread_mutex_destroy(&POP3QueueMutex);
1087         DeleteHash(&POP3FetchUrls);
1088         DeleteHash(&POP3QueueRooms);
1089 }
1090
1091 CTDL_MODULE_INIT(pop3client)
1092 {
1093         if (!threading)
1094         {
1095                 CtdlFillSystemContext(&pop3_client_CC, "POP3aggr");
1096                 citthread_mutex_init(&POP3QueueMutex, NULL);
1097                 POP3QueueRooms = NewHash(1, lFlathash);
1098                 POP3FetchUrls = NewHash(1, NULL);
1099                 CtdlRegisterSessionHook(pop3client_scan, EVT_TIMER);
1100                 CtdlRegisterCleanupHook(pop3_cleanup);
1101         }
1102         /* return our Subversion id for the Log */
1103         return "pop3client";
1104 }