citadel/modules/pop3client/serv_pop3client.c

   1 /*
   2  * Consolidate mail from remote POP3 accounts.
   3  *
   4  * Copyright (c) 2007-2011 by the citadel.org team
   5  *
   6  * This program is open source software; you can redistribute it and/or
   7  * modify it under the terms of the GNU General Public License as published
   8  * by the Free Software Foundation; either version 3 of the License, or
   9  * (at your option) any later version.
  10  *
  11  * This program is distributed in the hope that it will be useful,
  12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14  * GNU General Public License for more details.
  15  *
  16  * You should have received a copy of the GNU General Public License
  17  * along with this program; if not, write to the Free Software
  18  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
  19  */
  20
  21 #include <stdlib.h>
  22 #include <unistd.h>
  23 #include <stdio.h>
  24
  25 #if TIME_WITH_SYS_TIME
  26 # include <sys/time.h>
  27 # include <time.h>
  28 #else
  29 # if HAVE_SYS_TIME_H
  30 #  include <sys/time.h>
  31 # else
  32 #  include <time.h>
  33 # endif
  34 #endif
  35
  36 #include <ctype.h>
  37 #include <string.h>
  38 #include <errno.h>
  39 #include <sys/types.h>
  40 #include <sys/stat.h>
  41 #include <libcitadel.h>
  42 #include "citadel.h"
  43 #include "server.h"
  44 #include "citserver.h"
  45 #include "support.h"
  46 #include "config.h"
  47 #include "ctdl_module.h"
  48 #include "clientsocket.h"
  49 #include "msgbase.h"
  50 #include "internet_addressing.h"
  51 #include "database.h"
  52 #include "citadel_dirs.h"
  53 #include "event_client.h"
  54
  55
  56 struct CitContext pop3_client_CC;
  57
  58 pthread_mutex_t POP3QueueMutex; /* locks the access to the following vars: */
  59 HashList *POP3QueueRooms = NULL; /* rss_room_counter */
  60 HashList *POP3FetchUrls = NULL; /* -> rss_aggregator; ->RefCount access to be locked too. */
  61
  62 typedef struct __pop3_room_counter {
  63         int count;
  64         long QRnumber;
  65 }pop3_room_counter;
  66
  67 typedef enum ePOP3_C_States {
  68         ReadGreeting,
  69         GetUserState,
  70         GetPassState,
  71         GetListCommandState,
  72         GetListOneLine,
  73         GetOneMessageIDState,
  74         ReadMessageBodyFollowing,
  75         ReadMessageBody,
  76         GetDeleteState,
  77         ReadQuitState,
  78         POP3C_MaxRead
  79 }ePOP3_C_States;
  80
  81
  82 typedef struct _FetchItem {
  83         long MSGID;
  84         long MSGSize;
  85         StrBuf *MsgUIDL;
  86         StrBuf *MsgUID;
  87         int NeedFetch;
  88         struct CtdlMessage *Msg;
  89 } FetchItem;
  90
  91 void HfreeFetchItem(void *vItem)
  92 {
  93         FetchItem *Item = (FetchItem*) vItem;
  94         FreeStrBuf(&Item->MsgUIDL);
  95         FreeStrBuf(&Item->MsgUID);
  96         free(Item);
  97 }
  98
  99 typedef struct __pop3aggr {
 100         AsyncIO          IO;
 101
 102         long n;
 103         long RefCount;
 104 ///     ParsedURL *Pop3Host;
 105         DNSQueryParts HostLookup;
 106
 107 //      StrBuf          *rooms;
 108         long             QRnumber;
 109         HashList        *OtherQRnumbers;
 110
 111         StrBuf          *Url;
 112 ///     StrBuf *pop3host; -> URL
 113         StrBuf *pop3user;
 114         StrBuf *pop3pass;
 115         StrBuf *RoomName; // TODO: fill me
 116         int keep;
 117         time_t interval;
 118         ePOP3_C_States State;
 119         HashList *MsgNumbers;
 120         HashPos *Pos;
 121         FetchItem *CurrMsg;
 122 } pop3aggr;
 123
 124 void DeletePOP3Aggregator(void *vptr)
 125 {
 126         pop3aggr *ptr = vptr;
 127         DeleteHashPos(&ptr->Pos);
 128         DeleteHash(&ptr->MsgNumbers);
 129 //      FreeStrBuf(&ptr->rooms);
 130         FreeStrBuf(&ptr->pop3user);
 131         FreeStrBuf(&ptr->pop3pass);
 132         FreeStrBuf(&ptr->RoomName);
 133         FreeURL(&ptr->IO.ConnectMe);
 134         FreeStrBuf(&ptr->Url);
 135         FreeStrBuf(&ptr->IO.IOBuf);
 136         FreeStrBuf(&ptr->IO.SendBuf.Buf);
 137         FreeStrBuf(&ptr->IO.RecvBuf.Buf);
 138         DeleteAsyncMsg(&ptr->IO.ReadMsg);
 139         FreeAsyncIOContents(&ptr->IO);
 140         free(ptr);
 141 }
 142
 143
 144 typedef eNextState(*Pop3ClientHandler)(pop3aggr* RecvMsg);
 145
 146 eNextState POP3_C_Shutdown(AsyncIO *IO);
 147 eNextState POP3_C_Timeout(AsyncIO *IO);
 148 eNextState POP3_C_ConnFail(AsyncIO *IO);
 149 eNextState POP3_C_DNSFail(AsyncIO *IO);
 150 eNextState POP3_C_DispatchReadDone(AsyncIO *IO);
 151 eNextState POP3_C_DispatchWriteDone(AsyncIO *IO);
 152 eNextState POP3_C_Terminate(AsyncIO *IO);
 153 eReadState POP3_C_ReadServerStatus(AsyncIO *IO);
 154 eNextState POP3_C_ReAttachToFetchMessages(AsyncIO *IO);
 155
 156 eNextState FinalizePOP3AggrRun(AsyncIO *IO)
 157 {
 158         HashPos  *It;
 159         pop3aggr *cptr = (pop3aggr *)IO->Data;
 160
 161         syslog(LOG_DEBUG, "Terminating Aggregator; bye.\n");
 162
 163         It = GetNewHashPos(POP3FetchUrls, 0);
 164         pthread_mutex_lock(&POP3QueueMutex);
 165         {
 166                 GetHashPosFromKey(POP3FetchUrls, SKEY(cptr->Url), It);
 167                 DeleteEntryFromHash(POP3FetchUrls, It);
 168         }
 169         pthread_mutex_unlock(&POP3QueueMutex);
 170         DeleteHashPos(&It);
 171         return eAbort;
 172 }
 173
 174 eNextState FailAggregationRun(AsyncIO *IO)
 175 {
 176         return eAbort;
 177 }
 178
 179
 180 #define POP3C_DBG_SEND() syslog(LOG_DEBUG, "POP3 client[%ld]: > %s\n", RecvMsg->n, ChrPtr(RecvMsg->IO.SendBuf.Buf))
 181 #define POP3C_DBG_READ() syslog(LOG_DEBUG, "POP3 client[%ld]: < %s\n", RecvMsg->n, ChrPtr(RecvMsg->IO.IOBuf))
 182 #define POP3C_OK (strncasecmp(ChrPtr(RecvMsg->IO.IOBuf), "+OK", 3) == 0)
 183
 184 eNextState POP3C_ReadGreeting(pop3aggr *RecvMsg)
 185 {
 186         POP3C_DBG_READ();
 187         /* Read the server greeting */
 188         if (!POP3C_OK) return eTerminateConnection;
 189         else return eSendReply;
 190 }
 191
 192 eNextState POP3C_SendUser(pop3aggr *RecvMsg)
 193 {
 194         /* Identify ourselves.  NOTE: we have to append a CR to each command.  The LF will
 195          * automatically be appended by sock_puts().  Believe it or not, leaving out the CR
 196          * will cause problems if the server happens to be Exchange, which is so b0rken it
 197          * actually barfs on LF-terminated newlines.
 198          */
 199         StrBufPrintf(RecvMsg->IO.SendBuf.Buf,
 200                      "USER %s\r\n", ChrPtr(RecvMsg->pop3user));
 201         POP3C_DBG_SEND();
 202         return eReadMessage;
 203 }
 204
 205 eNextState POP3C_GetUserState(pop3aggr *RecvMsg)
 206 {
 207         POP3C_DBG_READ();
 208         if (!POP3C_OK) return eTerminateConnection;
 209         else return eSendReply;
 210 }
 211
 212 eNextState POP3C_SendPassword(pop3aggr *RecvMsg)
 213 {
 214         /* Password */
 215         StrBufPrintf(RecvMsg->IO.SendBuf.Buf,
 216                      "PASS %s\r\n", ChrPtr(RecvMsg->pop3pass));
 217         syslog(LOG_DEBUG, "<PASS <password>\n");
 218 //      POP3C_DBG_SEND();
 219         return eReadMessage;
 220 }
 221
 222 eNextState POP3C_GetPassState(pop3aggr *RecvMsg)
 223 {
 224         POP3C_DBG_READ();
 225         if (!POP3C_OK) return eTerminateConnection;
 226         else return eSendReply;
 227 }
 228
 229 eNextState POP3C_SendListCommand(pop3aggr *RecvMsg)
 230 {
 231         /* Get the list of messages */
 232         StrBufPlain(RecvMsg->IO.SendBuf.Buf, HKEY("LIST\r\n"));
 233         POP3C_DBG_SEND();
 234         return eReadMessage;
 235 }
 236
 237 eNextState POP3C_GetListCommandState(pop3aggr *RecvMsg)
 238 {
 239         POP3C_DBG_READ();
 240         if (!POP3C_OK) return eTerminateConnection;
 241         RecvMsg->MsgNumbers = NewHash(1, NULL);
 242         RecvMsg->State++;
 243         return eReadMore;
 244 }
 245
 246
 247 eNextState POP3C_GetListOneLine(pop3aggr *RecvMsg)
 248 {
 249 #if 0
 250         int rc;
 251 #endif
 252         const char *pch;
 253         FetchItem *OneMsg = NULL;
 254         POP3C_DBG_READ();
 255
 256         if ((StrLength(RecvMsg->IO.IOBuf) == 1) &&
 257             (ChrPtr(RecvMsg->IO.IOBuf)[0] == '.'))
 258         {
 259                 if (GetCount(RecvMsg->MsgNumbers) == 0)
 260                 {
 261                         ////    RecvMsg->Sate = ReadQuitState;
 262                 }
 263                 else
 264                 {
 265                         RecvMsg->Pos = GetNewHashPos(RecvMsg->MsgNumbers, 0);
 266                 }
 267                 return eSendReply;
 268
 269         }
 270         OneMsg = (FetchItem*) malloc(sizeof(FetchItem));
 271         memset(OneMsg, 0, sizeof(FetchItem));
 272         OneMsg->MSGID = atol(ChrPtr(RecvMsg->IO.IOBuf));
 273
 274         pch = strchr(ChrPtr(RecvMsg->IO.IOBuf), ' ');
 275         if (pch != NULL)
 276         {
 277                 OneMsg->MSGSize = atol(pch + 1);
 278         }
 279 #if 0
 280         rc = TestValidateHash(RecvMsg->MsgNumbers);
 281         if (rc != 0)
 282                 syslog(LOG_DEBUG, "Hash Invalid: %d\n", rc);
 283 #endif
 284
 285         Put(RecvMsg->MsgNumbers, LKEY(OneMsg->MSGID), OneMsg, HfreeFetchItem);
 286 #if 0
 287         rc = TestValidateHash(RecvMsg->MsgNumbers);
 288         if (rc != 0)
 289                 syslog(LOG_DEBUG, "Hash Invalid: %d\n", rc);
 290 #endif
 291         //RecvMsg->State --; /* read next Line */
 292         return eReadMore;
 293 }
 294
 295 eNextState POP3_FetchNetworkUsetableEntry(AsyncIO *IO)
 296 {
 297         long HKLen;
 298         const char *HKey;
 299         void *vData;
 300         struct cdbdata *cdbut;
 301         pop3aggr *RecvMsg = (pop3aggr *) IO->Data;
 302
 303         if(GetNextHashPos(RecvMsg->MsgNumbers, RecvMsg->Pos, &HKLen, &HKey, &vData))
 304         {
 305                 struct UseTable ut;
 306                 if (server_shutting_down)
 307                         return eAbort;
 308
 309                 RecvMsg->CurrMsg = (FetchItem*) vData;
 310                 syslog(LOG_DEBUG, "CHECKING: whether %s has already been seen: ", ChrPtr(RecvMsg->CurrMsg->MsgUID));
 311                 /* Find out if we've already seen this item */
 312                 safestrncpy(ut.ut_msgid,
 313                             ChrPtr(RecvMsg->CurrMsg->MsgUID),
 314                             sizeof(ut.ut_msgid));
 315                 ut.ut_timestamp = time(NULL);/// TODO: libev timestamp!
 316
 317                 cdbut = cdb_fetch(CDB_USETABLE, SKEY(RecvMsg->CurrMsg->MsgUID));
 318                 if (cdbut != NULL) {
 319                         /* Item has already been seen */
 320                         syslog(LOG_DEBUG, "YES\n");
 321                         cdb_free(cdbut);
 322
 323                         /* rewrite the record anyway, to update the timestamp */
 324                         cdb_store(CDB_USETABLE,
 325                                   SKEY(RecvMsg->CurrMsg->MsgUID),
 326                                   &ut, sizeof(struct UseTable) );
 327                         RecvMsg->CurrMsg->NeedFetch = 0; ////TODO0;
 328                 }
 329                 else
 330                 {
 331                         syslog(LOG_DEBUG, "NO\n");
 332                         RecvMsg->CurrMsg->NeedFetch = 1;
 333                 }
 334                 return NextDBOperation(&RecvMsg->IO, POP3_FetchNetworkUsetableEntry);
 335         }
 336         else
 337         {
 338                 /* ok, now we know them all, continue with reading the actual messages. */
 339                 DeleteHashPos(&RecvMsg->Pos);
 340
 341                 return QueueEventContext(IO, POP3_C_ReAttachToFetchMessages);
 342         }
 343 }
 344
 345 eNextState POP3C_GetOneMessagID(pop3aggr *RecvMsg)
 346 {
 347         long HKLen;
 348         const char *HKey;
 349         void *vData;
 350
 351 #if 0
 352         int rc;
 353         rc = TestValidateHash(RecvMsg->MsgNumbers);
 354         if (rc != 0)
 355                 syslog(LOG_DEBUG, "Hash Invalid: %d\n", rc);
 356 #endif
 357         if(GetNextHashPos(RecvMsg->MsgNumbers, RecvMsg->Pos, &HKLen, &HKey, &vData))
 358         {
 359                 RecvMsg->CurrMsg = (FetchItem*) vData;
 360                 /* Find out the UIDL of the message, to determine whether we've already downloaded it */
 361                 StrBufPrintf(RecvMsg->IO.SendBuf.Buf,
 362                              "UIDL %ld\r\n", RecvMsg->CurrMsg->MSGID);
 363                 POP3C_DBG_SEND();
 364         }
 365         else
 366         {
 367             RecvMsg->State++;
 368                 DeleteHashPos(&RecvMsg->Pos);
 369                 /// done receiving uidls.. start looking them up now.
 370                 RecvMsg->Pos = GetNewHashPos(RecvMsg->MsgNumbers, 0);
 371                 return QueueDBOperation(&RecvMsg->IO, POP3_FetchNetworkUsetableEntry);
 372         }
 373         return eReadMore; /* TODO */
 374 }
 375
 376 eNextState POP3C_GetOneMessageIDState(pop3aggr *RecvMsg)
 377 {
 378 #if 0
 379         int rc;
 380         rc = TestValidateHash(RecvMsg->MsgNumbers);
 381         if (rc != 0)
 382                 syslog(LOG_DEBUG, "Hash Invalid: %d\n", rc);
 383 #endif
 384
 385         POP3C_DBG_READ();
 386         if (!POP3C_OK) return eTerminateConnection;
 387         RecvMsg->CurrMsg->MsgUIDL = NewStrBufPlain(NULL, StrLength(RecvMsg->IO.IOBuf));
 388         RecvMsg->CurrMsg->MsgUID = NewStrBufPlain(NULL, StrLength(RecvMsg->IO.IOBuf) * 2);
 389
 390         StrBufExtract_token(RecvMsg->CurrMsg->MsgUIDL, RecvMsg->IO.IOBuf, 2, ' ');
 391         StrBufPrintf(RecvMsg->CurrMsg->MsgUID,
 392                      "pop3/%s/%s:%s@%s",
 393                      ChrPtr(RecvMsg->RoomName),
 394                      ChrPtr(RecvMsg->CurrMsg->MsgUIDL),
 395                      RecvMsg->IO.ConnectMe->User,
 396                      RecvMsg->IO.ConnectMe->Host);
 397         RecvMsg->State --;
 398         return eSendReply;
 399 }
 400
 401
 402 eNextState POP3C_SendGetOneMsg(pop3aggr *RecvMsg)
 403 {
 404         long HKLen;
 405         const char *HKey;
 406         void *vData;
 407
 408         RecvMsg->CurrMsg = NULL;
 409         while (GetNextHashPos(RecvMsg->MsgNumbers, RecvMsg->Pos, &HKLen, &HKey, &vData) &&
 410                (RecvMsg->CurrMsg = (FetchItem*) vData, RecvMsg->CurrMsg->NeedFetch == 0))
 411         {}
 412
 413         if ((RecvMsg->CurrMsg != NULL ) && (RecvMsg->CurrMsg->NeedFetch == 1))
 414         {
 415                 /* Message has not been seen. Tell the server to fetch the message... */
 416                 StrBufPrintf(RecvMsg->IO.SendBuf.Buf,
 417                              "RETR %ld\r\n", RecvMsg->CurrMsg->MSGID);
 418                 POP3C_DBG_SEND();
 419                 return eReadMessage;
 420         }
 421         else {
 422                 RecvMsg->State = ReadQuitState;
 423                 return POP3_C_DispatchWriteDone(&RecvMsg->IO);
 424         }
 425 }
 426
 427
 428 eNextState POP3C_ReadMessageBodyFollowing(pop3aggr *RecvMsg)
 429 {
 430         POP3C_DBG_READ();
 431         if (!POP3C_OK) return eTerminateConnection;
 432         RecvMsg->IO.ReadMsg = NewAsyncMsg(HKEY("."),
 433                                           RecvMsg->CurrMsg->MSGSize,
 434                                           config.c_maxmsglen,
 435                                           NULL, -1,
 436                                           1);
 437
 438         return eReadPayload;
 439 }
 440
 441
 442 eNextState POP3C_StoreMsgRead(AsyncIO *IO)
 443 {
 444         pop3aggr *RecvMsg = (pop3aggr *) IO->Data;
 445         struct UseTable ut;
 446
 447         syslog(LOG_DEBUG, "MARKING: %s as seen: ", ChrPtr(RecvMsg->CurrMsg->MsgUID));
 448
 449         safestrncpy(ut.ut_msgid,
 450                     ChrPtr(RecvMsg->CurrMsg->MsgUID),
 451                     sizeof(ut.ut_msgid));
 452         ut.ut_timestamp = time(NULL); /* TODO: use libev time */
 453         cdb_store(CDB_USETABLE,
 454                   ChrPtr(RecvMsg->CurrMsg->MsgUID),
 455                   StrLength(RecvMsg->CurrMsg->MsgUID),
 456                   &ut,
 457                   sizeof(struct UseTable) );
 458
 459         return QueueEventContext(&RecvMsg->IO, POP3_C_ReAttachToFetchMessages);
 460 }
 461 eNextState POP3C_SaveMsg(AsyncIO *IO)
 462 {
 463         long msgnum;
 464         pop3aggr *RecvMsg = (pop3aggr *) IO->Data;
 465
 466         /* Do Something With It (tm) */
 467         msgnum = CtdlSubmitMsg(RecvMsg->CurrMsg->Msg,
 468                                NULL,
 469                                ChrPtr(RecvMsg->RoomName),
 470                                0);
 471         if (msgnum > 0L) {
 472                 /* Message has been committed to the store */
 473                 /* write the uidl to the use table so we don't fetch this message again */
 474         }
 475         CtdlFreeMessage(RecvMsg->CurrMsg->Msg);
 476
 477         return NextDBOperation(&RecvMsg->IO, POP3C_StoreMsgRead);
 478 }
 479
 480 eNextState POP3C_ReadMessageBody(pop3aggr *RecvMsg)
 481 {
 482         syslog(LOG_DEBUG, "Converting message...\n");
 483         RecvMsg->CurrMsg->Msg = convert_internet_message_buf(&RecvMsg->IO.ReadMsg->MsgBuf);
 484
 485         return QueueDBOperation(&RecvMsg->IO, POP3C_SaveMsg);
 486 }
 487
 488 eNextState POP3C_SendDelete(pop3aggr *RecvMsg)
 489 {
 490         if (!RecvMsg->keep) {
 491                 StrBufPrintf(RecvMsg->IO.SendBuf.Buf,
 492                              "DELE %ld\r\n", RecvMsg->CurrMsg->MSGID);
 493                 POP3C_DBG_SEND();
 494                 return eReadMessage;
 495         }
 496         else {
 497                 RecvMsg->State = ReadMessageBodyFollowing;
 498                 return POP3_C_DispatchWriteDone(&RecvMsg->IO);
 499         }
 500 }
 501 eNextState POP3C_ReadDeleteState(pop3aggr *RecvMsg)
 502 {
 503         POP3C_DBG_READ();
 504         RecvMsg->State = GetOneMessageIDState;
 505         return eReadMessage;
 506 }
 507
 508 eNextState POP3C_SendQuit(pop3aggr *RecvMsg)
 509 {
 510         /* Log out */
 511         StrBufPlain(RecvMsg->IO.SendBuf.Buf,
 512                     HKEY("QUIT\r\n3)"));
 513         POP3C_DBG_SEND();
 514         return eReadMessage;
 515 }
 516
 517
 518 eNextState POP3C_ReadQuitState(pop3aggr *RecvMsg)
 519 {
 520         POP3C_DBG_READ();
 521         return eTerminateConnection;
 522 }
 523
 524 const long POP3_C_ConnTimeout = 1000;
 525 const long DefaultPOP3Port = 110;
 526
 527 Pop3ClientHandler POP3C_ReadHandlers[] = {
 528         POP3C_ReadGreeting,
 529         POP3C_GetUserState,
 530         POP3C_GetPassState,
 531         POP3C_GetListCommandState,
 532         POP3C_GetListOneLine,
 533         POP3C_GetOneMessageIDState,
 534         POP3C_ReadMessageBodyFollowing,
 535         POP3C_ReadMessageBody,
 536         POP3C_ReadDeleteState,
 537         POP3C_ReadQuitState,
 538 };
 539
 540 const long POP3_C_SendTimeouts[POP3C_MaxRead] = {
 541         100,
 542         100,
 543         100,
 544         100,
 545         100,
 546         100,
 547         100,
 548         100
 549 };
 550 const ConstStr POP3C_ReadErrors[POP3C_MaxRead] = {
 551         {HKEY("Connection broken during ")},
 552         {HKEY("Connection broken during ")},
 553         {HKEY("Connection broken during ")},
 554         {HKEY("Connection broken during ")},
 555         {HKEY("Connection broken during ")},
 556         {HKEY("Connection broken during ")},
 557         {HKEY("Connection broken during ")},
 558         {HKEY("Connection broken during ")}
 559 };
 560
 561 Pop3ClientHandler POP3C_SendHandlers[] = {
 562         NULL, /* we don't send a greeting */
 563         POP3C_SendUser,
 564         POP3C_SendPassword,
 565         POP3C_SendListCommand,
 566         NULL,
 567         POP3C_GetOneMessagID,
 568         POP3C_SendGetOneMsg,
 569         NULL,
 570         POP3C_SendDelete,
 571         POP3C_SendQuit
 572 };
 573
 574 const long POP3_C_ReadTimeouts[] = {
 575         100,
 576         100,
 577         100,
 578         100,
 579         100,
 580         100,
 581         100,
 582         100,
 583         100,
 584         100
 585 };
 586 /*****************************************************************************/
 587 /*                     POP3 CLIENT DISPATCHER                                */
 588 /*****************************************************************************/
 589
 590 void POP3SetTimeout(eNextState NextTCPState, pop3aggr *pMsg)
 591 {
 592         double Timeout = 0.0;
 593
 594         syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
 595
 596         switch (NextTCPState) {
 597         case eSendReply:
 598         case eSendMore:
 599                 Timeout = POP3_C_SendTimeouts[pMsg->State];
 600 /*
 601   if (pMsg->State == eDATABody) {
 602   / * if we're sending a huge message, we need more time. * /
 603   Timeout += StrLength(pMsg->msgtext) / 1024;
 604   }
 605 */
 606                 break;
 607         case eReadMessage:
 608                 Timeout = POP3_C_ReadTimeouts[pMsg->State];
 609 /*
 610   if (pMsg->State == eDATATerminateBody) {
 611   / *
 612   * some mailservers take a nap before accepting the message
 613   * content inspection and such.
 614   * /
 615   Timeout += StrLength(pMsg->msgtext) / 1024;
 616   }
 617 */
 618                 break;
 619         case eReadPayload:
 620                 Timeout = 100000;
 621                 /* TODO!!! */
 622                 break;
 623         case eSendDNSQuery:
 624         case eReadDNSReply:
 625         case eConnect:
 626         case eTerminateConnection:
 627         case eDBQuery:
 628         case eAbort:
 629         case eReadMore://// TODO
 630                 return;
 631         }
 632         SetNextTimeout(&pMsg->IO, Timeout);
 633 }
 634 eNextState POP3_C_DispatchReadDone(AsyncIO *IO)
 635 {
 636         syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
 637         pop3aggr *pMsg = IO->Data;
 638         eNextState rc;
 639
 640         rc = POP3C_ReadHandlers[pMsg->State](pMsg);
 641         if (rc != eReadMore)
 642             pMsg->State++;
 643         POP3SetTimeout(rc, pMsg);
 644         return rc;
 645 }
 646 eNextState POP3_C_DispatchWriteDone(AsyncIO *IO)
 647 {
 648         syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
 649         pop3aggr *pMsg = IO->Data;
 650         eNextState rc;
 651
 652         rc = POP3C_SendHandlers[pMsg->State](pMsg);
 653         POP3SetTimeout(rc, pMsg);
 654         return rc;
 655 }
 656
 657
 658 /*****************************************************************************/
 659 /*                     POP3 CLIENT ERROR CATCHERS                            */
 660 /*****************************************************************************/
 661 eNextState POP3_C_Terminate(AsyncIO *IO)
 662 {
 663 ///     pop3aggr *pMsg = (pop3aggr *)IO->Data;
 664
 665         syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
 666         FinalizePOP3AggrRun(IO);
 667         return eAbort;
 668 }
 669 eNextState POP3_C_Timeout(AsyncIO *IO)
 670 {
 671         pop3aggr *pMsg = IO->Data;
 672
 673         syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
 674         StrBufPlain(IO->ErrMsg, CKEY(POP3C_ReadErrors[pMsg->State]));
 675         return FailAggregationRun(IO);
 676 }
 677 eNextState POP3_C_ConnFail(AsyncIO *IO)
 678 {
 679         pop3aggr *pMsg = (pop3aggr *)IO->Data;
 680
 681         syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
 682         StrBufPlain(IO->ErrMsg, CKEY(POP3C_ReadErrors[pMsg->State]));
 683         return FailAggregationRun(IO);
 684 }
 685 eNextState POP3_C_DNSFail(AsyncIO *IO)
 686 {
 687         pop3aggr *pMsg = (pop3aggr *)IO->Data;
 688
 689         syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
 690         StrBufPlain(IO->ErrMsg, CKEY(POP3C_ReadErrors[pMsg->State]));
 691         return FailAggregationRun(IO);
 692 }
 693 eNextState POP3_C_Shutdown(AsyncIO *IO)
 694 {
 695         syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
 696 ////    pop3aggr *pMsg = IO->Data;
 697
 698         ////pMsg->MyQEntry->Status = 3;
 699         ///StrBufPlain(pMsg->MyQEntry->StatusMessage, HKEY("server shutdown during message retrieval."));
 700         FinalizePOP3AggrRun(IO);
 701         return eAbort;
 702 }
 703
 704
 705 /**
 706  * @brief lineread Handler; understands when to read more POP3 lines, and when this is a one-lined reply.
 707  */
 708 eReadState POP3_C_ReadServerStatus(AsyncIO *IO)
 709 {
 710         eReadState Finished = eBufferNotEmpty;
 711
 712         switch (IO->NextState) {
 713         case eSendDNSQuery:
 714         case eReadDNSReply:
 715         case eDBQuery:
 716         case eConnect:
 717         case eTerminateConnection:
 718         case eAbort:
 719                 Finished = eReadFail;
 720                 break;
 721         case eSendReply:
 722         case eSendMore:
 723         case eReadMore:
 724         case eReadMessage:
 725                 Finished = StrBufChunkSipLine(IO->IOBuf, &IO->RecvBuf);
 726                 break;
 727         case eReadPayload:
 728                 Finished = CtdlReadMessageBodyAsync(IO);
 729                 break;
 730         }
 731         return Finished;
 732 }
 733
 734 /*****************************************************************************
 735  * So we connect our Server IP here.                                         *
 736  *****************************************************************************/
 737 eNextState POP3_C_ReAttachToFetchMessages(AsyncIO *IO)
 738 {
 739         pop3aggr *cpptr = IO->Data;
 740
 741         syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
 742 ////??? cpptr->State ++;
 743         if (cpptr->Pos == NULL)
 744                 cpptr->Pos = GetNewHashPos(cpptr->MsgNumbers, 0);
 745
 746         POP3_C_DispatchWriteDone(IO);
 747         ReAttachIO(IO, cpptr, 0);
 748         IO->NextState = eReadMessage;
 749         return IO->NextState;
 750 }
 751
 752 eNextState pop3_connect_ip(AsyncIO *IO)
 753 {
 754         pop3aggr *cpptr = IO->Data;
 755
 756         syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
 757
 758 ////    IO->ConnectMe = &cpptr->Pop3Host;
 759         /*  Bypass the ns lookup result like this: IO->Addr.sin_addr.s_addr = inet_addr("127.0.0.1"); */
 760
 761         /////// SetConnectStatus(IO);
 762
 763         return InitEventIO(IO, cpptr,
 764                            POP3_C_ConnTimeout,
 765                            POP3_C_ReadTimeouts[0],
 766                            1);
 767 }
 768
 769 eNextState pop3_get_one_host_ip_done(AsyncIO *IO)
 770 {
 771         pop3aggr *cpptr = IO->Data;
 772         struct hostent *hostent;
 773
 774         QueryCbDone(IO);
 775
 776         hostent = cpptr->HostLookup.VParsedDNSReply;
 777         if ((cpptr->HostLookup.DNSStatus == ARES_SUCCESS) &&
 778             (hostent != NULL) ) {
 779                 memset(&cpptr->IO.ConnectMe->Addr, 0, sizeof(struct in6_addr));
 780                 if (cpptr->IO.ConnectMe->IPv6) {
 781                         memcpy(&cpptr->IO.ConnectMe->Addr.sin6_addr.s6_addr,
 782                                &hostent->h_addr_list[0],
 783                                sizeof(struct in6_addr));
 784
 785                         cpptr->IO.ConnectMe->Addr.sin6_family = hostent->h_addrtype;
 786                         cpptr->IO.ConnectMe->Addr.sin6_port   = htons(DefaultPOP3Port);
 787                 }
 788                 else {
 789                         struct sockaddr_in *addr = (struct sockaddr_in*) &cpptr->IO.ConnectMe->Addr;
 790                         /* Bypass the ns lookup result like this: IO->Addr.sin_addr.s_addr = inet_addr("127.0.0.1"); */
 791 //                      addr->sin_addr.s_addr = htonl((uint32_t)&hostent->h_addr_list[0]);
 792                         memcpy(&addr->sin_addr.s_addr,
 793                                hostent->h_addr_list[0],
 794                                sizeof(uint32_t));
 795
 796                         addr->sin_family = hostent->h_addrtype;
 797                         addr->sin_port   = htons(DefaultPOP3Port);
 798
 799                 }
 800                 return pop3_connect_ip(IO);
 801         }
 802         else
 803                 return eAbort;
 804 }
 805
 806 eNextState pop3_get_one_host_ip(AsyncIO *IO)
 807 {
 808         pop3aggr *cpptr = IO->Data;
 809         /*
 810          * here we start with the lookup of one host. it might be...
 811          * - the relay host *sigh*
 812          * - the direct hostname if there was no mx record
 813          * - one of the mx'es
 814          */
 815
 816         InitC_ares_dns(IO);
 817
 818         syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
 819
 820         syslog(LOG_DEBUG,
 821                       "POP3 client[%ld]: looking up %s-Record %s : %d ...\n",
 822                       cpptr->n,
 823                       (cpptr->IO.ConnectMe->IPv6)? "aaaa": "a",
 824                       cpptr->IO.ConnectMe->Host,
 825                       cpptr->IO.ConnectMe->Port);
 826
 827         QueueQuery((cpptr->IO.ConnectMe->IPv6)? ns_t_aaaa : ns_t_a,
 828                    cpptr->IO.ConnectMe->Host,
 829                    &cpptr->IO,
 830                    &cpptr->HostLookup,
 831                    pop3_get_one_host_ip_done);
 832         IO->NextState = eReadDNSReply;
 833         return IO->NextState;
 834 }
 835
 836
 837
 838 int pop3_do_fetching(pop3aggr *cpptr)
 839 {
 840         CitContext *SubC;
 841
 842         cpptr->IO.Data          = cpptr;
 843
 844         cpptr->IO.SendDone      = POP3_C_DispatchWriteDone;
 845         cpptr->IO.ReadDone      = POP3_C_DispatchReadDone;
 846         cpptr->IO.Terminate     = POP3_C_Terminate;
 847         cpptr->IO.LineReader    = POP3_C_ReadServerStatus;
 848         cpptr->IO.ConnFail      = POP3_C_ConnFail;
 849         cpptr->IO.DNSFail       = POP3_C_DNSFail;
 850         cpptr->IO.Timeout       = POP3_C_Timeout;
 851         cpptr->IO.ShutdownAbort = POP3_C_Shutdown;
 852
 853         cpptr->IO.SendBuf.Buf   = NewStrBufPlain(NULL, 1024);
 854         cpptr->IO.RecvBuf.Buf   = NewStrBufPlain(NULL, 1024);
 855         cpptr->IO.IOBuf         = NewStrBuf();
 856
 857         cpptr->IO.NextState     = eReadMessage;
 858 /* TODO
 859    syslog(LOG_DEBUG, "POP3: %s %s %s <password>\n", roomname, pop3host, pop3user);
 860    syslog(LOG_DEBUG, "Connecting to <%s>\n", pop3host);
 861 */
 862
 863         SubC = CloneContext (&pop3_client_CC);
 864         SubC->session_specific_data = (char*) cpptr;
 865         cpptr->IO.CitContext = SubC;
 866
 867         if (cpptr->IO.ConnectMe->IsIP) {
 868                 QueueEventContext(&cpptr->IO,
 869                                   pop3_connect_ip);
 870         }
 871         else { /* uneducated admin has chosen to add DNS to the equation... */
 872                 QueueEventContext(&cpptr->IO,
 873                                   pop3_get_one_host_ip);
 874         }
 875         return 1;
 876 }
 877
 878 /*
 879  * Scan a room's netconfig to determine whether it requires POP3 aggregation
 880  */
 881 void pop3client_scan_room(struct ctdlroom *qrbuf, void *data)
 882 {
 883         StrBuf *CfgData;
 884         StrBuf *CfgType;
 885         StrBuf *Line;
 886
 887         struct stat statbuf;
 888         char filename[PATH_MAX];
 889         int  fd;
 890         int Done;
 891         void *vptr;
 892         const char *CfgPtr, *lPtr;
 893         const char *Err;
 894
 895 //      pop3_room_counter *Count = NULL;
 896 //      pop3aggr *cpptr;
 897
 898         pthread_mutex_lock(&POP3QueueMutex);
 899         if (GetHash(POP3QueueRooms, LKEY(qrbuf->QRnumber), &vptr))
 900         {
 901                 syslog(LOG_DEBUG,
 902                               "pop3client: [%ld] %s already in progress.\n",
 903                               qrbuf->QRnumber,
 904                               qrbuf->QRname);
 905                 pthread_mutex_unlock(&POP3QueueMutex);
 906         }
 907         pthread_mutex_unlock(&POP3QueueMutex);
 908
 909         if (server_shutting_down) return;
 910
 911         assoc_file_name(filename, sizeof filename, qrbuf, ctdl_netcfg_dir);
 912
 913         if (server_shutting_down)
 914                 return;
 915
 916         /* Only do net processing for rooms that have netconfigs */
 917         fd = open(filename, 0);
 918         if (fd <= 0) {
 919                 //syslog(LOG_DEBUG, "rssclient: %s no config.\n", qrbuf->QRname);
 920                 return;
 921         }
 922         if (server_shutting_down)
 923                 return;
 924         if (fstat(fd, &statbuf) == -1) {
 925                 syslog(LOG_DEBUG, "ERROR: could not stat configfile '%s' - %s\n",
 926                               filename, strerror(errno));
 927                 return;
 928         }
 929         if (server_shutting_down)
 930                 return;
 931         CfgData = NewStrBufPlain(NULL, statbuf.st_size + 1);
 932         if (StrBufReadBLOB(CfgData, &fd, 1, statbuf.st_size, &Err) < 0) {
 933                 close(fd);
 934                 FreeStrBuf(&CfgData);
 935                 syslog(LOG_DEBUG, "ERROR: reading config '%s' - %s<br>\n",
 936                               filename, strerror(errno));
 937                 return;
 938         }
 939         close(fd);
 940         if (server_shutting_down)
 941                 return;
 942
 943         CfgPtr = NULL;
 944         CfgType = NewStrBuf();
 945         Line = NewStrBufPlain(NULL, StrLength(CfgData));
 946         Done = 0;
 947
 948         while (!Done)
 949         {
 950                 Done = StrBufSipLine(Line, CfgData, &CfgPtr) == 0;
 951                 if (StrLength(Line) > 0)
 952                 {
 953                         lPtr = NULL;
 954                         StrBufExtract_NextToken(CfgType, Line, &lPtr, '|');
 955                         if (!strcasecmp("pop3client", ChrPtr(CfgType)))
 956                         {
 957                                 pop3aggr *cptr;
 958                                 StrBuf *Tmp;
 959 /*
 960                                 if (Count == NULL)
 961                                 {
 962                                         Count = malloc(sizeof(pop3_room_counter));
 963                                         Count->count = 0;
 964                                 }
 965                                 Count->count ++;
 966 */
 967                                 cptr = (pop3aggr *) malloc(sizeof(pop3aggr));
 968                                 memset(cptr, 0, sizeof(pop3aggr));
 969                                 /// TODO do we need this? cptr->roomlist_parts = 1;
 970                                 cptr->RoomName = NewStrBufPlain(qrbuf->QRname, -1);
 971                                 cptr->pop3user = NewStrBufPlain(NULL, StrLength(Line));
 972                                 cptr->pop3pass = NewStrBufPlain(NULL, StrLength(Line));
 973                                 cptr->Url = NewStrBuf();
 974                                 Tmp = NewStrBuf();
 975
 976                                 StrBufExtract_NextToken(Tmp, Line, &lPtr, '|');
 977                                 StrBufExtract_NextToken(cptr->pop3user, Line, &lPtr, '|');
 978                                 StrBufExtract_NextToken(cptr->pop3pass, Line, &lPtr, '|');
 979                                 cptr->keep = StrBufExtractNext_long(Line, &lPtr, '|');
 980                                 cptr->interval = StrBufExtractNext_long(Line, &lPtr, '|');
 981
 982                                 StrBufPrintf(cptr->Url, "pop3://%s:%s@%s/%s",
 983                                              ChrPtr(cptr->pop3user),
 984                                              ChrPtr(cptr->pop3pass),
 985                                              ChrPtr(Tmp),
 986                                              ChrPtr(cptr->RoomName));
 987                                 FreeStrBuf(&Tmp);
 988                                 ParseURL(&cptr->IO.ConnectMe, cptr->Url, 110);
 989
 990
 991 #if 0
 992 /* todo: we need to reunite the url to be shure. */
 993
 994                                 pthread_mutex_lock(&POP3ueueMutex);
 995                                 GetHash(POP3FetchUrls, SKEY(ptr->Url), &vptr);
 996                                 use_this_cptr = (pop3aggr *)vptr;
 997
 998                                 if (use_this_rncptr != NULL)
 999                                 {
1000                                         /* mustn't attach to an active session */
1001                                         if (use_this_cptr->RefCount > 0)
1002                                         {
1003                                                 DeletePOP3Cfg(cptr);
1004 ///                                             Count->count--;
1005                                         }
1006                                         else
1007                                         {
1008                                                 long *QRnumber;
1009                                                 StrBufAppendBufPlain(use_this_cptr->rooms,
1010                                                                      qrbuf->QRname,
1011                                                                      -1, 0);
1012                                                 if (use_this_cptr->roomlist_parts == 1)
1013                                                 {
1014                                                         use_this_cptr->OtherQRnumbers = NewHash(1, lFlathash);
1015                                                 }
1016                                                 QRnumber = (long*)malloc(sizeof(long));
1017                                                 *QRnumber = qrbuf->QRnumber;
1018                                                 Put(use_this_cptr->OtherQRnumbers, LKEY(qrbuf->QRnumber), QRnumber, NULL);
1019                                                 use_this_cptr->roomlist_parts++;
1020                                         }
1021                                         pthread_mutex_unlock(&POP3QueueMutex);
1022                                         continue;
1023                                 }
1024                                 pthread_mutex_unlock(&RSSQueueMutex);
1025 #endif
1026
1027                                 pthread_mutex_lock(&POP3QueueMutex);
1028                                 Put(POP3FetchUrls, SKEY(cptr->Url), cptr, DeletePOP3Aggregator);
1029                                 pthread_mutex_unlock(&POP3QueueMutex);
1030
1031                         }
1032
1033                 }
1034
1035                 ///fclose(fp);
1036
1037         }
1038         FreeStrBuf(&Line);
1039         FreeStrBuf(&CfgType);
1040         FreeStrBuf(&CfgData);
1041 }
1042
1043
1044 void pop3client_scan(void) {
1045         static time_t last_run = 0L;
1046         static int doing_pop3client = 0;
1047 ///     struct pop3aggr *pptr;
1048         time_t fastest_scan;
1049         HashPos *it;
1050         long len;
1051         const char *Key;
1052         void *vrptr;
1053         pop3aggr *cptr;
1054
1055         if (config.c_pop3_fastest < config.c_pop3_fetch)
1056                 fastest_scan = config.c_pop3_fastest;
1057         else
1058                 fastest_scan = config.c_pop3_fetch;
1059
1060         /*
1061          * Run POP3 aggregation no more frequently than once every n seconds
1062          */
1063         if ( (time(NULL) - last_run) < fastest_scan ) {
1064                 return;
1065         }
1066
1067         /*
1068          * This is a simple concurrency check to make sure only one pop3client run
1069          * is done at a time.  We could do this with a mutex, but since we
1070          * don't really require extremely fine granularity here, we'll do it
1071          * with a static variable instead.
1072          */
1073         if (doing_pop3client) return;
1074         doing_pop3client = 1;
1075
1076         syslog(LOG_DEBUG, "pop3client started");
1077         CtdlForEachRoom(pop3client_scan_room, NULL);
1078
1079         pthread_mutex_lock(&POP3QueueMutex);
1080         it = GetNewHashPos(POP3FetchUrls, 0);
1081         while (!server_shutting_down &&
1082                GetNextHashPos(POP3FetchUrls, it, &len, &Key, &vrptr) &&
1083                (vrptr != NULL)) {
1084                 cptr = (pop3aggr *)vrptr;
1085                 if (cptr->RefCount == 0)
1086                         if (!pop3_do_fetching(cptr))
1087                                 DeletePOP3Aggregator(cptr);////TODO
1088
1089 /*
1090                 if ((palist->interval && time(NULL) > (last_run + palist->interval))
1091                         || (time(NULL) > last_run + config.c_pop3_fetch))
1092                                 pop3_do_fetching(palist->roomname, palist->pop3host,
1093                                         palist->pop3user, palist->pop3pass, palist->keep);
1094                 pptr = palist;
1095                 palist = palist->next;
1096                 free(pptr);
1097 */
1098         }
1099         DeleteHashPos(&it);
1100         pthread_mutex_unlock(&POP3QueueMutex);
1101
1102         syslog(LOG_DEBUG, "pop3client ended");
1103         last_run = time(NULL);
1104         doing_pop3client = 0;
1105 }
1106
1107
1108 void pop3_cleanup(void)
1109 {
1110         /* citthread_mutex_destroy(&POP3QueueMutex); TODO */
1111 //      DeleteHash(&POP3FetchUrls);
1112         DeleteHash(&POP3QueueRooms);
1113 }
1114
1115 CTDL_MODULE_INIT(pop3client)
1116 {
1117         if (!threading)
1118         {
1119                 CtdlFillSystemContext(&pop3_client_CC, "POP3aggr");
1120                 pthread_mutex_init(&POP3QueueMutex, NULL);
1121                 POP3QueueRooms = NewHash(1, lFlathash);
1122                 POP3FetchUrls = NewHash(1, NULL);
1123                 CtdlRegisterSessionHook(pop3client_scan, EVT_TIMER);
1124                 CtdlRegisterCleanupHook(pop3_cleanup);
1125         }
1126
1127         /* return our module id for the log */
1128         return "pop3client";
1129 }