citadel/modules/pop3client/serv_pop3client.c

   1 /*
   2  * Consolidate mail from remote POP3 accounts.
   3  *
   4  * Copyright (c) 2007-2011 by the citadel.org team
   5  *
   6  * This program is open source software; you can redistribute it and/or
   7  * modify it under the terms of the GNU General Public License as published
   8  * by the Free Software Foundation; either version 3 of the License, or
   9  * (at your option) any later version.
  10  *
  11  * This program is distributed in the hope that it will be useful,
  12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14  * GNU General Public License for more details.
  15  *
  16  * You should have received a copy of the GNU General Public License
  17  * along with this program; if not, write to the Free Software
  18  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
  19  */
  20
  21 #include <stdlib.h>
  22 #include <unistd.h>
  23 #include <stdio.h>
  24
  25 #if TIME_WITH_SYS_TIME
  26 # include <sys/time.h>
  27 # include <time.h>
  28 #else
  29 # if HAVE_SYS_TIME_H
  30 #  include <sys/time.h>
  31 # else
  32 #  include <time.h>
  33 # endif
  34 #endif
  35
  36 #include <ctype.h>
  37 #include <string.h>
  38 #include <errno.h>
  39 #include <sys/types.h>
  40 #include <sys/stat.h>
  41 #include <libcitadel.h>
  42 #include "citadel.h"
  43 #include "server.h"
  44 #include "citserver.h"
  45 #include "support.h"
  46 #include "config.h"
  47 #include "ctdl_module.h"
  48 #include "clientsocket.h"
  49 #include "msgbase.h"
  50 #include "internet_addressing.h"
  51 #include "database.h"
  52 #include "citadel_dirs.h"
  53 #include "event_client.h"
  54
  55
  56 struct CitContext pop3_client_CC;
  57
  58 pthread_mutex_t POP3QueueMutex; /* locks the access to the following vars: */
  59 HashList *POP3QueueRooms = NULL; /* rss_room_counter */
  60 HashList *POP3FetchUrls = NULL; /* -> rss_aggregator; ->RefCount access to be locked too. */
  61
  62 typedef struct __pop3_room_counter {
  63         int count;
  64         long QRnumber;
  65 }pop3_room_counter;
  66
  67 typedef enum ePOP3_C_States {
  68         ReadGreeting,
  69         GetUserState,
  70         GetPassState,
  71         GetListCommandState,
  72         GetListOneLine,
  73         GetOneMessageIDState,
  74         ReadMessageBodyFollowing,
  75         ReadMessageBody,
  76         GetDeleteState,
  77         ReadQuitState,
  78         POP3C_MaxRead
  79 }ePOP3_C_States;
  80
  81
  82 typedef struct _FetchItem {
  83         long MSGID;
  84         long MSGSize;
  85         StrBuf *MsgUIDL;
  86         StrBuf *MsgUID;
  87         int NeedFetch;
  88         struct CtdlMessage *Msg;
  89 } FetchItem;
  90
  91 void HfreeFetchItem(void *vItem)
  92 {
  93         FetchItem *Item = (FetchItem*) vItem;
  94         FreeStrBuf(&Item->MsgUIDL);
  95         FreeStrBuf(&Item->MsgUID);
  96         free(Item);
  97 }
  98
  99 typedef struct __pop3aggr {
 100         AsyncIO          IO;
 101
 102         long n;
 103         long RefCount;
 104 ///     ParsedURL *Pop3Host;
 105         DNSQueryParts HostLookup;
 106
 107 //      StrBuf          *rooms;
 108         long             QRnumber;
 109         HashList        *OtherQRnumbers;
 110
 111         StrBuf          *Url;
 112 ///     StrBuf *pop3host; -> URL
 113         StrBuf *pop3user;
 114         StrBuf *pop3pass;
 115         StrBuf *RoomName; // TODO: fill me
 116         int keep;
 117         time_t interval;
 118         ePOP3_C_States State;
 119         HashList *MsgNumbers;
 120         HashPos *Pos;
 121         FetchItem *CurrMsg;
 122 } pop3aggr;
 123
 124 void DeletePOP3Aggregator(void *vptr)
 125 {
 126         pop3aggr *ptr = vptr;
 127         DeleteHashPos(&ptr->Pos);
 128         DeleteHash(&ptr->MsgNumbers);
 129 //      FreeStrBuf(&ptr->rooms);
 130         FreeStrBuf(&ptr->pop3user);
 131         FreeStrBuf(&ptr->pop3pass);
 132         FreeStrBuf(&ptr->RoomName);
 133         FreeURL(&ptr->IO.ConnectMe);
 134         FreeStrBuf(&ptr->Url);
 135         FreeStrBuf(&ptr->IO.IOBuf);
 136         FreeStrBuf(&ptr->IO.SendBuf.Buf);
 137         FreeStrBuf(&ptr->IO.RecvBuf.Buf);
 138         DeleteAsyncMsg(&ptr->IO.ReadMsg);
 139         ((struct CitContext*)ptr->IO.CitContext)->state = CON_IDLE;
 140         ((struct CitContext*)ptr->IO.CitContext)->kill_me = 1;
 141         FreeAsyncIOContents(&ptr->IO);
 142         free(ptr);
 143 }
 144
 145
 146 typedef eNextState(*Pop3ClientHandler)(pop3aggr* RecvMsg);
 147
 148 eNextState POP3_C_Shutdown(AsyncIO *IO);
 149 eNextState POP3_C_Timeout(AsyncIO *IO);
 150 eNextState POP3_C_ConnFail(AsyncIO *IO);
 151 eNextState POP3_C_DNSFail(AsyncIO *IO);
 152 eNextState POP3_C_DispatchReadDone(AsyncIO *IO);
 153 eNextState POP3_C_DispatchWriteDone(AsyncIO *IO);
 154 eNextState POP3_C_Terminate(AsyncIO *IO);
 155 eReadState POP3_C_ReadServerStatus(AsyncIO *IO);
 156 eNextState POP3_C_ReAttachToFetchMessages(AsyncIO *IO);
 157
 158 eNextState FinalizePOP3AggrRun(AsyncIO *IO)
 159 {
 160         HashPos  *It;
 161         pop3aggr *cptr = (pop3aggr *)IO->Data;
 162
 163         syslog(LOG_DEBUG, "Terminating Aggregator; bye.\n");
 164
 165         It = GetNewHashPos(POP3FetchUrls, 0);
 166         pthread_mutex_lock(&POP3QueueMutex);
 167         {
 168                 if (GetHashPosFromKey(POP3FetchUrls, SKEY(cptr->Url), It))
 169                         DeleteEntryFromHash(POP3FetchUrls, It);
 170         }
 171         pthread_mutex_unlock(&POP3QueueMutex);
 172         DeleteHashPos(&It);
 173         return eAbort;
 174 }
 175
 176 eNextState FailAggregationRun(AsyncIO *IO)
 177 {
 178         return eAbort;
 179 }
 180
 181
 182 #define POP3C_DBG_SEND() syslog(LOG_DEBUG, "POP3 client[%ld]: > %s\n", RecvMsg->n, ChrPtr(RecvMsg->IO.SendBuf.Buf))
 183 #define POP3C_DBG_READ() syslog(LOG_DEBUG, "POP3 client[%ld]: < %s\n", RecvMsg->n, ChrPtr(RecvMsg->IO.IOBuf))
 184 #define POP3C_OK (strncasecmp(ChrPtr(RecvMsg->IO.IOBuf), "+OK", 3) == 0)
 185
 186 eNextState POP3C_ReadGreeting(pop3aggr *RecvMsg)
 187 {
 188         POP3C_DBG_READ();
 189         /* Read the server greeting */
 190         if (!POP3C_OK) return eTerminateConnection;
 191         else return eSendReply;
 192 }
 193
 194 eNextState POP3C_SendUser(pop3aggr *RecvMsg)
 195 {
 196         /* Identify ourselves.  NOTE: we have to append a CR to each command.  The LF will
 197          * automatically be appended by sock_puts().  Believe it or not, leaving out the CR
 198          * will cause problems if the server happens to be Exchange, which is so b0rken it
 199          * actually barfs on LF-terminated newlines.
 200          */
 201         StrBufPrintf(RecvMsg->IO.SendBuf.Buf,
 202                      "USER %s\r\n", ChrPtr(RecvMsg->pop3user));
 203         POP3C_DBG_SEND();
 204         return eReadMessage;
 205 }
 206
 207 eNextState POP3C_GetUserState(pop3aggr *RecvMsg)
 208 {
 209         POP3C_DBG_READ();
 210         if (!POP3C_OK) return eTerminateConnection;
 211         else return eSendReply;
 212 }
 213
 214 eNextState POP3C_SendPassword(pop3aggr *RecvMsg)
 215 {
 216         /* Password */
 217         StrBufPrintf(RecvMsg->IO.SendBuf.Buf,
 218                      "PASS %s\r\n", ChrPtr(RecvMsg->pop3pass));
 219         syslog(LOG_DEBUG, "<PASS <password>\n");
 220 //      POP3C_DBG_SEND();
 221         return eReadMessage;
 222 }
 223
 224 eNextState POP3C_GetPassState(pop3aggr *RecvMsg)
 225 {
 226         POP3C_DBG_READ();
 227         if (!POP3C_OK) return eTerminateConnection;
 228         else return eSendReply;
 229 }
 230
 231 eNextState POP3C_SendListCommand(pop3aggr *RecvMsg)
 232 {
 233         /* Get the list of messages */
 234         StrBufPlain(RecvMsg->IO.SendBuf.Buf, HKEY("LIST\r\n"));
 235         POP3C_DBG_SEND();
 236         return eReadMessage;
 237 }
 238
 239 eNextState POP3C_GetListCommandState(pop3aggr *RecvMsg)
 240 {
 241         POP3C_DBG_READ();
 242         if (!POP3C_OK) return eTerminateConnection;
 243         RecvMsg->MsgNumbers = NewHash(1, NULL);
 244         RecvMsg->State++;
 245         return eReadMore;
 246 }
 247
 248
 249 eNextState POP3C_GetListOneLine(pop3aggr *RecvMsg)
 250 {
 251 #if 0
 252         int rc;
 253 #endif
 254         const char *pch;
 255         FetchItem *OneMsg = NULL;
 256         POP3C_DBG_READ();
 257
 258         if ((StrLength(RecvMsg->IO.IOBuf) == 1) &&
 259             (ChrPtr(RecvMsg->IO.IOBuf)[0] == '.'))
 260         {
 261                 if (GetCount(RecvMsg->MsgNumbers) == 0)
 262                 {
 263                         ////    RecvMsg->Sate = ReadQuitState;
 264                 }
 265                 else
 266                 {
 267                         RecvMsg->Pos = GetNewHashPos(RecvMsg->MsgNumbers, 0);
 268                 }
 269                 return eSendReply;
 270
 271         }
 272         OneMsg = (FetchItem*) malloc(sizeof(FetchItem));
 273         memset(OneMsg, 0, sizeof(FetchItem));
 274         OneMsg->MSGID = atol(ChrPtr(RecvMsg->IO.IOBuf));
 275
 276         pch = strchr(ChrPtr(RecvMsg->IO.IOBuf), ' ');
 277         if (pch != NULL)
 278         {
 279                 OneMsg->MSGSize = atol(pch + 1);
 280         }
 281 #if 0
 282         rc = TestValidateHash(RecvMsg->MsgNumbers);
 283         if (rc != 0)
 284                 syslog(LOG_DEBUG, "Hash Invalid: %d\n", rc);
 285 #endif
 286
 287         Put(RecvMsg->MsgNumbers, LKEY(OneMsg->MSGID), OneMsg, HfreeFetchItem);
 288 #if 0
 289         rc = TestValidateHash(RecvMsg->MsgNumbers);
 290         if (rc != 0)
 291                 syslog(LOG_DEBUG, "Hash Invalid: %d\n", rc);
 292 #endif
 293         //RecvMsg->State --; /* read next Line */
 294         return eReadMore;
 295 }
 296
 297 eNextState POP3_FetchNetworkUsetableEntry(AsyncIO *IO)
 298 {
 299         long HKLen;
 300         const char *HKey;
 301         void *vData;
 302         struct cdbdata *cdbut;
 303         pop3aggr *RecvMsg = (pop3aggr *) IO->Data;
 304
 305         if(GetNextHashPos(RecvMsg->MsgNumbers, RecvMsg->Pos, &HKLen, &HKey, &vData))
 306         {
 307                 struct UseTable ut;
 308                 if (server_shutting_down)
 309                         return eAbort;
 310
 311                 RecvMsg->CurrMsg = (FetchItem*) vData;
 312                 syslog(LOG_DEBUG, "CHECKING: whether %s has already been seen: ", ChrPtr(RecvMsg->CurrMsg->MsgUID));
 313                 /* Find out if we've already seen this item */
 314                 safestrncpy(ut.ut_msgid,
 315                             ChrPtr(RecvMsg->CurrMsg->MsgUID),
 316                             sizeof(ut.ut_msgid));
 317                 ut.ut_timestamp = time(NULL);/// TODO: libev timestamp!
 318
 319                 cdbut = cdb_fetch(CDB_USETABLE, SKEY(RecvMsg->CurrMsg->MsgUID));
 320                 if (cdbut != NULL) {
 321                         /* Item has already been seen */
 322                         syslog(LOG_DEBUG, "YES\n");
 323                         cdb_free(cdbut);
 324
 325                         /* rewrite the record anyway, to update the timestamp */
 326                         cdb_store(CDB_USETABLE,
 327                                   SKEY(RecvMsg->CurrMsg->MsgUID),
 328                                   &ut, sizeof(struct UseTable) );
 329                         RecvMsg->CurrMsg->NeedFetch = 0; ////TODO0;
 330                 }
 331                 else
 332                 {
 333                         syslog(LOG_DEBUG, "NO\n");
 334                         RecvMsg->CurrMsg->NeedFetch = 1;
 335                 }
 336                 return NextDBOperation(&RecvMsg->IO, POP3_FetchNetworkUsetableEntry);
 337         }
 338         else
 339         {
 340                 /* ok, now we know them all, continue with reading the actual messages. */
 341                 DeleteHashPos(&RecvMsg->Pos);
 342
 343                 return QueueEventContext(IO, POP3_C_ReAttachToFetchMessages);
 344         }
 345 }
 346
 347 eNextState POP3C_GetOneMessagID(pop3aggr *RecvMsg)
 348 {
 349         long HKLen;
 350         const char *HKey;
 351         void *vData;
 352
 353 #if 0
 354         int rc;
 355         rc = TestValidateHash(RecvMsg->MsgNumbers);
 356         if (rc != 0)
 357                 syslog(LOG_DEBUG, "Hash Invalid: %d\n", rc);
 358 #endif
 359         if(GetNextHashPos(RecvMsg->MsgNumbers, RecvMsg->Pos, &HKLen, &HKey, &vData))
 360         {
 361                 RecvMsg->CurrMsg = (FetchItem*) vData;
 362                 /* Find out the UIDL of the message, to determine whether we've already downloaded it */
 363                 StrBufPrintf(RecvMsg->IO.SendBuf.Buf,
 364                              "UIDL %ld\r\n", RecvMsg->CurrMsg->MSGID);
 365                 POP3C_DBG_SEND();
 366         }
 367         else
 368         {
 369             RecvMsg->State++;
 370                 DeleteHashPos(&RecvMsg->Pos);
 371                 /// done receiving uidls.. start looking them up now.
 372                 RecvMsg->Pos = GetNewHashPos(RecvMsg->MsgNumbers, 0);
 373                 return QueueDBOperation(&RecvMsg->IO, POP3_FetchNetworkUsetableEntry);
 374         }
 375         return eReadMore; /* TODO */
 376 }
 377
 378 eNextState POP3C_GetOneMessageIDState(pop3aggr *RecvMsg)
 379 {
 380 #if 0
 381         int rc;
 382         rc = TestValidateHash(RecvMsg->MsgNumbers);
 383         if (rc != 0)
 384                 syslog(LOG_DEBUG, "Hash Invalid: %d\n", rc);
 385 #endif
 386
 387         POP3C_DBG_READ();
 388         if (!POP3C_OK) return eTerminateConnection;
 389         RecvMsg->CurrMsg->MsgUIDL = NewStrBufPlain(NULL, StrLength(RecvMsg->IO.IOBuf));
 390         RecvMsg->CurrMsg->MsgUID = NewStrBufPlain(NULL, StrLength(RecvMsg->IO.IOBuf) * 2);
 391
 392         StrBufExtract_token(RecvMsg->CurrMsg->MsgUIDL, RecvMsg->IO.IOBuf, 2, ' ');
 393         StrBufPrintf(RecvMsg->CurrMsg->MsgUID,
 394                      "pop3/%s/%s:%s@%s",
 395                      ChrPtr(RecvMsg->RoomName),
 396                      ChrPtr(RecvMsg->CurrMsg->MsgUIDL),
 397                      RecvMsg->IO.ConnectMe->User,
 398                      RecvMsg->IO.ConnectMe->Host);
 399         RecvMsg->State --;
 400         return eSendReply;
 401 }
 402
 403
 404 eNextState POP3C_SendGetOneMsg(pop3aggr *RecvMsg)
 405 {
 406         long HKLen;
 407         const char *HKey;
 408         void *vData;
 409
 410         RecvMsg->CurrMsg = NULL;
 411         while (GetNextHashPos(RecvMsg->MsgNumbers, RecvMsg->Pos, &HKLen, &HKey, &vData) &&
 412                (RecvMsg->CurrMsg = (FetchItem*) vData, RecvMsg->CurrMsg->NeedFetch == 0))
 413         {}
 414
 415         if ((RecvMsg->CurrMsg != NULL ) && (RecvMsg->CurrMsg->NeedFetch == 1))
 416         {
 417                 /* Message has not been seen. Tell the server to fetch the message... */
 418                 StrBufPrintf(RecvMsg->IO.SendBuf.Buf,
 419                              "RETR %ld\r\n", RecvMsg->CurrMsg->MSGID);
 420                 POP3C_DBG_SEND();
 421                 return eReadMessage;
 422         }
 423         else {
 424                 RecvMsg->State = ReadQuitState;
 425                 return POP3_C_DispatchWriteDone(&RecvMsg->IO);
 426         }
 427 }
 428
 429
 430 eNextState POP3C_ReadMessageBodyFollowing(pop3aggr *RecvMsg)
 431 {
 432         POP3C_DBG_READ();
 433         if (!POP3C_OK) return eTerminateConnection;
 434         RecvMsg->IO.ReadMsg = NewAsyncMsg(HKEY("."),
 435                                           RecvMsg->CurrMsg->MSGSize,
 436                                           config.c_maxmsglen,
 437                                           NULL, -1,
 438                                           1);
 439
 440         return eReadPayload;
 441 }
 442
 443
 444 eNextState POP3C_StoreMsgRead(AsyncIO *IO)
 445 {
 446         pop3aggr *RecvMsg = (pop3aggr *) IO->Data;
 447         struct UseTable ut;
 448
 449         syslog(LOG_DEBUG, "MARKING: %s as seen: ", ChrPtr(RecvMsg->CurrMsg->MsgUID));
 450
 451         safestrncpy(ut.ut_msgid,
 452                     ChrPtr(RecvMsg->CurrMsg->MsgUID),
 453                     sizeof(ut.ut_msgid));
 454         ut.ut_timestamp = time(NULL); /* TODO: use libev time */
 455         cdb_store(CDB_USETABLE,
 456                   ChrPtr(RecvMsg->CurrMsg->MsgUID),
 457                   StrLength(RecvMsg->CurrMsg->MsgUID),
 458                   &ut,
 459                   sizeof(struct UseTable) );
 460
 461         return QueueEventContext(&RecvMsg->IO, POP3_C_ReAttachToFetchMessages);
 462 }
 463 eNextState POP3C_SaveMsg(AsyncIO *IO)
 464 {
 465         long msgnum;
 466         pop3aggr *RecvMsg = (pop3aggr *) IO->Data;
 467
 468         /* Do Something With It (tm) */
 469         msgnum = CtdlSubmitMsg(RecvMsg->CurrMsg->Msg,
 470                                NULL,
 471                                ChrPtr(RecvMsg->RoomName),
 472                                0);
 473         if (msgnum > 0L) {
 474                 /* Message has been committed to the store */
 475                 /* write the uidl to the use table so we don't fetch this message again */
 476         }
 477         CtdlFreeMessage(RecvMsg->CurrMsg->Msg);
 478
 479         return NextDBOperation(&RecvMsg->IO, POP3C_StoreMsgRead);
 480 }
 481
 482 eNextState POP3C_ReadMessageBody(pop3aggr *RecvMsg)
 483 {
 484         syslog(LOG_DEBUG, "Converting message...\n");
 485         RecvMsg->CurrMsg->Msg = convert_internet_message_buf(&RecvMsg->IO.ReadMsg->MsgBuf);
 486
 487         return QueueDBOperation(&RecvMsg->IO, POP3C_SaveMsg);
 488 }
 489
 490 eNextState POP3C_SendDelete(pop3aggr *RecvMsg)
 491 {
 492         if (!RecvMsg->keep) {
 493                 StrBufPrintf(RecvMsg->IO.SendBuf.Buf,
 494                              "DELE %ld\r\n", RecvMsg->CurrMsg->MSGID);
 495                 POP3C_DBG_SEND();
 496                 return eReadMessage;
 497         }
 498         else {
 499                 RecvMsg->State = ReadMessageBodyFollowing;
 500                 return POP3_C_DispatchWriteDone(&RecvMsg->IO);
 501         }
 502 }
 503 eNextState POP3C_ReadDeleteState(pop3aggr *RecvMsg)
 504 {
 505         POP3C_DBG_READ();
 506         RecvMsg->State = GetOneMessageIDState;
 507         return eReadMessage;
 508 }
 509
 510 eNextState POP3C_SendQuit(pop3aggr *RecvMsg)
 511 {
 512         /* Log out */
 513         StrBufPlain(RecvMsg->IO.SendBuf.Buf,
 514                     HKEY("QUIT\r\n3)"));
 515         POP3C_DBG_SEND();
 516         return eReadMessage;
 517 }
 518
 519
 520 eNextState POP3C_ReadQuitState(pop3aggr *RecvMsg)
 521 {
 522         POP3C_DBG_READ();
 523         return eTerminateConnection;
 524 }
 525
 526 const long POP3_C_ConnTimeout = 1000;
 527 const long DefaultPOP3Port = 110;
 528
 529 Pop3ClientHandler POP3C_ReadHandlers[] = {
 530         POP3C_ReadGreeting,
 531         POP3C_GetUserState,
 532         POP3C_GetPassState,
 533         POP3C_GetListCommandState,
 534         POP3C_GetListOneLine,
 535         POP3C_GetOneMessageIDState,
 536         POP3C_ReadMessageBodyFollowing,
 537         POP3C_ReadMessageBody,
 538         POP3C_ReadDeleteState,
 539         POP3C_ReadQuitState,
 540 };
 541
 542 const long POP3_C_SendTimeouts[POP3C_MaxRead] = {
 543         100,
 544         100,
 545         100,
 546         100,
 547         100,
 548         100,
 549         100,
 550         100
 551 };
 552 const ConstStr POP3C_ReadErrors[POP3C_MaxRead] = {
 553         {HKEY("Connection broken during ")},
 554         {HKEY("Connection broken during ")},
 555         {HKEY("Connection broken during ")},
 556         {HKEY("Connection broken during ")},
 557         {HKEY("Connection broken during ")},
 558         {HKEY("Connection broken during ")},
 559         {HKEY("Connection broken during ")},
 560         {HKEY("Connection broken during ")}
 561 };
 562
 563 Pop3ClientHandler POP3C_SendHandlers[] = {
 564         NULL, /* we don't send a greeting */
 565         POP3C_SendUser,
 566         POP3C_SendPassword,
 567         POP3C_SendListCommand,
 568         NULL,
 569         POP3C_GetOneMessagID,
 570         POP3C_SendGetOneMsg,
 571         NULL,
 572         POP3C_SendDelete,
 573         POP3C_SendQuit
 574 };
 575
 576 const long POP3_C_ReadTimeouts[] = {
 577         100,
 578         100,
 579         100,
 580         100,
 581         100,
 582         100,
 583         100,
 584         100,
 585         100,
 586         100
 587 };
 588 /*****************************************************************************/
 589 /*                     POP3 CLIENT DISPATCHER                                */
 590 /*****************************************************************************/
 591
 592 void POP3SetTimeout(eNextState NextTCPState, pop3aggr *pMsg)
 593 {
 594         double Timeout = 0.0;
 595
 596         syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
 597
 598         switch (NextTCPState) {
 599         case eSendFile:
 600         case eSendReply:
 601         case eSendMore:
 602                 Timeout = POP3_C_SendTimeouts[pMsg->State];
 603 /*
 604   if (pMsg->State == eDATABody) {
 605   / * if we're sending a huge message, we need more time. * /
 606   Timeout += StrLength(pMsg->msgtext) / 1024;
 607   }
 608 */
 609                 break;
 610         case eReadFile:
 611         case eReadMessage:
 612                 Timeout = POP3_C_ReadTimeouts[pMsg->State];
 613 /*
 614   if (pMsg->State == eDATATerminateBody) {
 615   / *
 616   * some mailservers take a nap before accepting the message
 617   * content inspection and such.
 618   * /
 619   Timeout += StrLength(pMsg->msgtext) / 1024;
 620   }
 621 */
 622                 break;
 623         case eReadPayload:
 624                 Timeout = 100000;
 625                 /* TODO!!! */
 626                 break;
 627         case eSendDNSQuery:
 628         case eReadDNSReply:
 629         case eConnect:
 630         case eTerminateConnection:
 631         case eDBQuery:
 632         case eAbort:
 633         case eReadMore://// TODO
 634                 return;
 635         }
 636         SetNextTimeout(&pMsg->IO, Timeout);
 637 }
 638 eNextState POP3_C_DispatchReadDone(AsyncIO *IO)
 639 {
 640         syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
 641         pop3aggr *pMsg = IO->Data;
 642         eNextState rc;
 643
 644         rc = POP3C_ReadHandlers[pMsg->State](pMsg);
 645         if (rc != eReadMore)
 646             pMsg->State++;
 647         POP3SetTimeout(rc, pMsg);
 648         return rc;
 649 }
 650 eNextState POP3_C_DispatchWriteDone(AsyncIO *IO)
 651 {
 652         syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
 653         pop3aggr *pMsg = IO->Data;
 654         eNextState rc;
 655
 656         rc = POP3C_SendHandlers[pMsg->State](pMsg);
 657         POP3SetTimeout(rc, pMsg);
 658         return rc;
 659 }
 660
 661
 662 /*****************************************************************************/
 663 /*                     POP3 CLIENT ERROR CATCHERS                            */
 664 /*****************************************************************************/
 665 eNextState POP3_C_Terminate(AsyncIO *IO)
 666 {
 667 ///     pop3aggr *pMsg = (pop3aggr *)IO->Data;
 668
 669         syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
 670         FinalizePOP3AggrRun(IO);
 671         return eAbort;
 672 }
 673 eNextState POP3_C_Timeout(AsyncIO *IO)
 674 {
 675         pop3aggr *pMsg = IO->Data;
 676
 677         syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
 678         StrBufPlain(IO->ErrMsg, CKEY(POP3C_ReadErrors[pMsg->State]));
 679         return FailAggregationRun(IO);
 680 }
 681 eNextState POP3_C_ConnFail(AsyncIO *IO)
 682 {
 683         pop3aggr *pMsg = (pop3aggr *)IO->Data;
 684
 685         syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
 686         StrBufPlain(IO->ErrMsg, CKEY(POP3C_ReadErrors[pMsg->State]));
 687         return FailAggregationRun(IO);
 688 }
 689 eNextState POP3_C_DNSFail(AsyncIO *IO)
 690 {
 691         pop3aggr *pMsg = (pop3aggr *)IO->Data;
 692
 693         syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
 694         StrBufPlain(IO->ErrMsg, CKEY(POP3C_ReadErrors[pMsg->State]));
 695         return FailAggregationRun(IO);
 696 }
 697 eNextState POP3_C_Shutdown(AsyncIO *IO)
 698 {
 699         syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
 700 ////    pop3aggr *pMsg = IO->Data;
 701
 702         ////pMsg->MyQEntry->Status = 3;
 703         ///StrBufPlain(pMsg->MyQEntry->StatusMessage, HKEY("server shutdown during message retrieval."));
 704         FinalizePOP3AggrRun(IO);
 705         return eAbort;
 706 }
 707
 708
 709 /**
 710  * @brief lineread Handler; understands when to read more POP3 lines, and when this is a one-lined reply.
 711  */
 712 eReadState POP3_C_ReadServerStatus(AsyncIO *IO)
 713 {
 714         eReadState Finished = eBufferNotEmpty;
 715
 716         switch (IO->NextState) {
 717         case eSendDNSQuery:
 718         case eReadDNSReply:
 719         case eDBQuery:
 720         case eConnect:
 721         case eTerminateConnection:
 722         case eAbort:
 723                 Finished = eReadFail;
 724                 break;
 725         case eSendFile:
 726         case eSendReply:
 727         case eSendMore:
 728         case eReadMore:
 729         case eReadMessage:
 730                 Finished = StrBufChunkSipLine(IO->IOBuf, &IO->RecvBuf);
 731                 break;
 732         case eReadFile:
 733         case eReadPayload:
 734                 Finished = CtdlReadMessageBodyAsync(IO);
 735                 break;
 736         }
 737         return Finished;
 738 }
 739
 740 /*****************************************************************************
 741  * So we connect our Server IP here.                                         *
 742  *****************************************************************************/
 743 eNextState POP3_C_ReAttachToFetchMessages(AsyncIO *IO)
 744 {
 745         pop3aggr *cpptr = IO->Data;
 746
 747         syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
 748 ////??? cpptr->State ++;
 749         if (cpptr->Pos == NULL)
 750                 cpptr->Pos = GetNewHashPos(cpptr->MsgNumbers, 0);
 751
 752         POP3_C_DispatchWriteDone(IO);
 753         ReAttachIO(IO, cpptr, 0);
 754         IO->NextState = eReadMessage;
 755         return IO->NextState;
 756 }
 757
 758 eNextState pop3_connect_ip(AsyncIO *IO)
 759 {
 760         pop3aggr *cpptr = IO->Data;
 761
 762         syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
 763
 764 ////    IO->ConnectMe = &cpptr->Pop3Host;
 765         /*  Bypass the ns lookup result like this: IO->Addr.sin_addr.s_addr = inet_addr("127.0.0.1"); */
 766
 767         /////// SetConnectStatus(IO);
 768
 769         return EvConnectSock(IO, cpptr,
 770                              POP3_C_ConnTimeout,
 771                              POP3_C_ReadTimeouts[0],
 772                              1);
 773 }
 774
 775 eNextState pop3_get_one_host_ip_done(AsyncIO *IO)
 776 {
 777         pop3aggr *cpptr = IO->Data;
 778         struct hostent *hostent;
 779
 780         QueryCbDone(IO);
 781
 782         hostent = cpptr->HostLookup.VParsedDNSReply;
 783         if ((cpptr->HostLookup.DNSStatus == ARES_SUCCESS) &&
 784             (hostent != NULL) ) {
 785                 memset(&cpptr->IO.ConnectMe->Addr, 0, sizeof(struct in6_addr));
 786                 if (cpptr->IO.ConnectMe->IPv6) {
 787                         memcpy(&cpptr->IO.ConnectMe->Addr.sin6_addr.s6_addr,
 788                                &hostent->h_addr_list[0],
 789                                sizeof(struct in6_addr));
 790
 791                         cpptr->IO.ConnectMe->Addr.sin6_family = hostent->h_addrtype;
 792                         cpptr->IO.ConnectMe->Addr.sin6_port   = htons(DefaultPOP3Port);
 793                 }
 794                 else {
 795                         struct sockaddr_in *addr = (struct sockaddr_in*) &cpptr->IO.ConnectMe->Addr;
 796                         /* Bypass the ns lookup result like this: IO->Addr.sin_addr.s_addr = inet_addr("127.0.0.1"); */
 797 //                      addr->sin_addr.s_addr = htonl((uint32_t)&hostent->h_addr_list[0]);
 798                         memcpy(&addr->sin_addr.s_addr,
 799                                hostent->h_addr_list[0],
 800                                sizeof(uint32_t));
 801
 802                         addr->sin_family = hostent->h_addrtype;
 803                         addr->sin_port   = htons(DefaultPOP3Port);
 804
 805                 }
 806                 return pop3_connect_ip(IO);
 807         }
 808         else
 809                 return eAbort;
 810 }
 811
 812 eNextState pop3_get_one_host_ip(AsyncIO *IO)
 813 {
 814         pop3aggr *cpptr = IO->Data;
 815         /*
 816          * here we start with the lookup of one host. it might be...
 817          * - the relay host *sigh*
 818          * - the direct hostname if there was no mx record
 819          * - one of the mx'es
 820          */
 821
 822         InitC_ares_dns(IO);
 823
 824         syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
 825
 826         syslog(LOG_DEBUG,
 827                       "POP3 client[%ld]: looking up %s-Record %s : %d ...\n",
 828                       cpptr->n,
 829                       (cpptr->IO.ConnectMe->IPv6)? "aaaa": "a",
 830                       cpptr->IO.ConnectMe->Host,
 831                       cpptr->IO.ConnectMe->Port);
 832
 833         QueueQuery((cpptr->IO.ConnectMe->IPv6)? ns_t_aaaa : ns_t_a,
 834                    cpptr->IO.ConnectMe->Host,
 835                    &cpptr->IO,
 836                    &cpptr->HostLookup,
 837                    pop3_get_one_host_ip_done);
 838         IO->NextState = eReadDNSReply;
 839         return IO->NextState;
 840 }
 841
 842
 843
 844 int pop3_do_fetching(pop3aggr *cpptr)
 845 {
 846         CitContext *SubC;
 847
 848         cpptr->IO.Data          = cpptr;
 849
 850         cpptr->IO.SendDone      = POP3_C_DispatchWriteDone;
 851         cpptr->IO.ReadDone      = POP3_C_DispatchReadDone;
 852         cpptr->IO.Terminate     = POP3_C_Terminate;
 853         cpptr->IO.LineReader    = POP3_C_ReadServerStatus;
 854         cpptr->IO.ConnFail      = POP3_C_ConnFail;
 855         cpptr->IO.DNS.Fail      = POP3_C_DNSFail;
 856         cpptr->IO.Timeout       = POP3_C_Timeout;
 857         cpptr->IO.ShutdownAbort = POP3_C_Shutdown;
 858
 859         cpptr->IO.SendBuf.Buf   = NewStrBufPlain(NULL, 1024);
 860         cpptr->IO.RecvBuf.Buf   = NewStrBufPlain(NULL, 1024);
 861         cpptr->IO.IOBuf         = NewStrBuf();
 862
 863         cpptr->IO.NextState     = eReadMessage;
 864 /* TODO
 865    syslog(LOG_DEBUG, "POP3: %s %s %s <password>\n", roomname, pop3host, pop3user);
 866    syslog(LOG_DEBUG, "Connecting to <%s>\n", pop3host);
 867 */
 868
 869         SubC = CloneContext (&pop3_client_CC);
 870         SubC->session_specific_data = (char*) cpptr;
 871         cpptr->IO.CitContext = SubC;
 872         safestrncpy(SubC->cs_host,
 873                     ChrPtr(cpptr->Url),
 874                     sizeof(SubC->cs_host));
 875
 876         if (cpptr->IO.ConnectMe->IsIP) {
 877                 QueueEventContext(&cpptr->IO,
 878                                   pop3_connect_ip);
 879         }
 880         else { /* uneducated admin has chosen to add DNS to the equation... */
 881                 QueueEventContext(&cpptr->IO,
 882                                   pop3_get_one_host_ip);
 883         }
 884         return 1;
 885 }
 886
 887 /*
 888  * Scan a room's netconfig to determine whether it requires POP3 aggregation
 889  */
 890 void pop3client_scan_room(struct ctdlroom *qrbuf, void *data)
 891 {
 892         StrBuf *CfgData;
 893         StrBuf *CfgType;
 894         StrBuf *Line;
 895
 896         struct stat statbuf;
 897         char filename[PATH_MAX];
 898         int  fd;
 899         int Done;
 900         void *vptr;
 901         const char *CfgPtr, *lPtr;
 902         const char *Err;
 903
 904 //      pop3_room_counter *Count = NULL;
 905 //      pop3aggr *cpptr;
 906
 907         pthread_mutex_lock(&POP3QueueMutex);
 908         if (GetHash(POP3QueueRooms, LKEY(qrbuf->QRnumber), &vptr))
 909         {
 910                 syslog(LOG_DEBUG,
 911                               "pop3client: [%ld] %s already in progress.\n",
 912                               qrbuf->QRnumber,
 913                               qrbuf->QRname);
 914                 pthread_mutex_unlock(&POP3QueueMutex);
 915         }
 916         pthread_mutex_unlock(&POP3QueueMutex);
 917
 918         if (server_shutting_down) return;
 919
 920         assoc_file_name(filename, sizeof filename, qrbuf, ctdl_netcfg_dir);
 921
 922         if (server_shutting_down)
 923                 return;
 924
 925         /* Only do net processing for rooms that have netconfigs */
 926         fd = open(filename, 0);
 927         if (fd <= 0) {
 928                 //syslog(LOG_DEBUG, "rssclient: %s no config.\n", qrbuf->QRname);
 929                 return;
 930         }
 931         if (server_shutting_down)
 932                 return;
 933         if (fstat(fd, &statbuf) == -1) {
 934                 syslog(LOG_DEBUG, "ERROR: could not stat configfile '%s' - %s\n",
 935                               filename, strerror(errno));
 936                 return;
 937         }
 938         if (server_shutting_down)
 939                 return;
 940         CfgData = NewStrBufPlain(NULL, statbuf.st_size + 1);
 941         if (StrBufReadBLOB(CfgData, &fd, 1, statbuf.st_size, &Err) < 0) {
 942                 close(fd);
 943                 FreeStrBuf(&CfgData);
 944                 syslog(LOG_DEBUG, "ERROR: reading config '%s' - %s<br>\n",
 945                               filename, strerror(errno));
 946                 return;
 947         }
 948         close(fd);
 949         if (server_shutting_down)
 950                 return;
 951
 952         CfgPtr = NULL;
 953         CfgType = NewStrBuf();
 954         Line = NewStrBufPlain(NULL, StrLength(CfgData));
 955         Done = 0;
 956
 957         while (!Done)
 958         {
 959                 Done = StrBufSipLine(Line, CfgData, &CfgPtr) == 0;
 960                 if (StrLength(Line) > 0)
 961                 {
 962                         lPtr = NULL;
 963                         StrBufExtract_NextToken(CfgType, Line, &lPtr, '|');
 964                         if (!strcasecmp("pop3client", ChrPtr(CfgType)))
 965                         {
 966                                 pop3aggr *cptr;
 967                                 StrBuf *Tmp;
 968 /*
 969                                 if (Count == NULL)
 970                                 {
 971                                         Count = malloc(sizeof(pop3_room_counter));
 972                                         Count->count = 0;
 973                                 }
 974                                 Count->count ++;
 975 */
 976                                 cptr = (pop3aggr *) malloc(sizeof(pop3aggr));
 977                                 memset(cptr, 0, sizeof(pop3aggr));
 978                                 /// TODO do we need this? cptr->roomlist_parts = 1;
 979                                 cptr->RoomName = NewStrBufPlain(qrbuf->QRname, -1);
 980                                 cptr->pop3user = NewStrBufPlain(NULL, StrLength(Line));
 981                                 cptr->pop3pass = NewStrBufPlain(NULL, StrLength(Line));
 982                                 cptr->Url = NewStrBuf();
 983                                 Tmp = NewStrBuf();
 984
 985                                 StrBufExtract_NextToken(Tmp, Line, &lPtr, '|');
 986                                 StrBufExtract_NextToken(cptr->pop3user, Line, &lPtr, '|');
 987                                 StrBufExtract_NextToken(cptr->pop3pass, Line, &lPtr, '|');
 988                                 cptr->keep = StrBufExtractNext_long(Line, &lPtr, '|');
 989                                 cptr->interval = StrBufExtractNext_long(Line, &lPtr, '|');
 990
 991                                 StrBufPrintf(cptr->Url, "pop3://%s:%s@%s/%s",
 992                                              ChrPtr(cptr->pop3user),
 993                                              ChrPtr(cptr->pop3pass),
 994                                              ChrPtr(Tmp),
 995                                              ChrPtr(cptr->RoomName));
 996                                 FreeStrBuf(&Tmp);
 997                                 ParseURL(&cptr->IO.ConnectMe, cptr->Url, 110);
 998
 999
1000 #if 0
1001 /* todo: we need to reunite the url to be shure. */
1002
1003                                 pthread_mutex_lock(&POP3ueueMutex);
1004                                 GetHash(POP3FetchUrls, SKEY(ptr->Url), &vptr);
1005                                 use_this_cptr = (pop3aggr *)vptr;
1006
1007                                 if (use_this_rncptr != NULL)
1008                                 {
1009                                         /* mustn't attach to an active session */
1010                                         if (use_this_cptr->RefCount > 0)
1011                                         {
1012                                                 DeletePOP3Cfg(cptr);
1013 ///                                             Count->count--;
1014                                         }
1015                                         else
1016                                         {
1017                                                 long *QRnumber;
1018                                                 StrBufAppendBufPlain(use_this_cptr->rooms,
1019                                                                      qrbuf->QRname,
1020                                                                      -1, 0);
1021                                                 if (use_this_cptr->roomlist_parts == 1)
1022                                                 {
1023                                                         use_this_cptr->OtherQRnumbers = NewHash(1, lFlathash);
1024                                                 }
1025                                                 QRnumber = (long*)malloc(sizeof(long));
1026                                                 *QRnumber = qrbuf->QRnumber;
1027                                                 Put(use_this_cptr->OtherQRnumbers, LKEY(qrbuf->QRnumber), QRnumber, NULL);
1028                                                 use_this_cptr->roomlist_parts++;
1029                                         }
1030                                         pthread_mutex_unlock(&POP3QueueMutex);
1031                                         continue;
1032                                 }
1033                                 pthread_mutex_unlock(&RSSQueueMutex);
1034 #endif
1035
1036                                 pthread_mutex_lock(&POP3QueueMutex);
1037                                 Put(POP3FetchUrls, SKEY(cptr->Url), cptr, DeletePOP3Aggregator);
1038                                 pthread_mutex_unlock(&POP3QueueMutex);
1039
1040                         }
1041
1042                 }
1043
1044                 ///fclose(fp);
1045
1046         }
1047         FreeStrBuf(&Line);
1048         FreeStrBuf(&CfgType);
1049         FreeStrBuf(&CfgData);
1050 }
1051
1052 static int doing_pop3client = 0;
1053
1054 void pop3client_scan(void) {
1055         static time_t last_run = 0L;
1056 ///     struct pop3aggr *pptr;
1057         time_t fastest_scan;
1058         HashPos *it;
1059         long len;
1060         const char *Key;
1061         void *vrptr;
1062         pop3aggr *cptr;
1063
1064         if (config.c_pop3_fastest < config.c_pop3_fetch)
1065                 fastest_scan = config.c_pop3_fastest;
1066         else
1067                 fastest_scan = config.c_pop3_fetch;
1068
1069         /*
1070          * Run POP3 aggregation no more frequently than once every n seconds
1071          */
1072         if ( (time(NULL) - last_run) < fastest_scan ) {
1073                 return;
1074         }
1075
1076         /*
1077          * This is a simple concurrency check to make sure only one pop3client run
1078          * is done at a time.  We could do this with a mutex, but since we
1079          * don't really require extremely fine granularity here, we'll do it
1080          * with a static variable instead.
1081          */
1082         if (doing_pop3client) return;
1083         doing_pop3client = 1;
1084
1085         syslog(LOG_DEBUG, "pop3client started");
1086         CtdlForEachRoom(pop3client_scan_room, NULL);
1087
1088         pthread_mutex_lock(&POP3QueueMutex);
1089         it = GetNewHashPos(POP3FetchUrls, 0);
1090         while (!server_shutting_down &&
1091                GetNextHashPos(POP3FetchUrls, it, &len, &Key, &vrptr) &&
1092                (vrptr != NULL)) {
1093                 cptr = (pop3aggr *)vrptr;
1094                 if (cptr->RefCount == 0)
1095                         if (!pop3_do_fetching(cptr))
1096                                 DeletePOP3Aggregator(cptr);////TODO
1097
1098 /*
1099                 if ((palist->interval && time(NULL) > (last_run + palist->interval))
1100                         || (time(NULL) > last_run + config.c_pop3_fetch))
1101                                 pop3_do_fetching(palist->roomname, palist->pop3host,
1102                                         palist->pop3user, palist->pop3pass, palist->keep);
1103                 pptr = palist;
1104                 palist = palist->next;
1105                 free(pptr);
1106 */
1107         }
1108         DeleteHashPos(&it);
1109         pthread_mutex_unlock(&POP3QueueMutex);
1110
1111         syslog(LOG_DEBUG, "pop3client ended");
1112         last_run = time(NULL);
1113         doing_pop3client = 0;
1114 }
1115
1116
1117 void pop3_cleanup(void)
1118 {
1119         /* citthread_mutex_destroy(&POP3QueueMutex); TODO */
1120         while (doing_pop3client != 0) ;
1121         DeleteHash(&POP3FetchUrls);
1122         DeleteHash(&POP3QueueRooms);
1123 }
1124
1125 CTDL_MODULE_INIT(pop3client)
1126 {
1127         if (!threading)
1128         {
1129                 CtdlFillSystemContext(&pop3_client_CC, "POP3aggr");
1130                 pthread_mutex_init(&POP3QueueMutex, NULL);
1131                 POP3QueueRooms = NewHash(1, lFlathash);
1132                 POP3FetchUrls = NewHash(1, NULL);
1133                 CtdlRegisterSessionHook(pop3client_scan, EVT_TIMER);
1134                 CtdlRegisterCleanupHook(pop3_cleanup);
1135         }
1136
1137         /* return our module id for the log */
1138         return "pop3client";
1139 }