citadel/modules/pop3client/serv_pop3client.c

   1 /*
   2  * Consolidate mail from remote POP3 accounts.
   3  *
   4  * Copyright (c) 2007-2009 by the citadel.org team
   5  *
   6  *  This program is free software; you can redistribute it and/or modify
   7  *  it under the terms of the GNU General Public License as published by
   8  *  the Free Software Foundation; either version 3 of the License, or
   9  *  (at your option) any later version.
  10  *
  11  *  This program is distributed in the hope that it will be useful,
  12  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
  13  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14  *  GNU General Public License for more details.
  15  *
  16  *  You should have received a copy of the GNU General Public License
  17  *  along with this program; if not, write to the Free Software
  18  *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  19  */
  20
  21 #include <stdlib.h>
  22 #include <unistd.h>
  23 #include <stdio.h>
  24
  25 #if TIME_WITH_SYS_TIME
  26 # include <sys/time.h>
  27 # include <time.h>
  28 #else
  29 # if HAVE_SYS_TIME_H
  30 #  include <sys/time.h>
  31 # else
  32 #  include <time.h>
  33 # endif
  34 #endif
  35
  36 #include <ctype.h>
  37 #include <string.h>
  38 #include <errno.h>
  39 #include <sys/types.h>
  40 #include <sys/stat.h>
  41 #include <libcitadel.h>
  42 #include "citadel.h"
  43 #include "server.h"
  44 #include "citserver.h"
  45 #include "support.h"
  46 #include "config.h"
  47 #include "ctdl_module.h"
  48 #include "clientsocket.h"
  49 #include "msgbase.h"
  50 #include "internet_addressing.h"
  51 #include "database.h"
  52 #include "citadel_dirs.h"
  53 #include "event_client.h"
  54
  55
  56 struct CitContext pop3_client_CC;
  57
  58 citthread_mutex_t POP3QueueMutex; /* locks the access to the following vars: */
  59 HashList *POP3QueueRooms = NULL; /* rss_room_counter */
  60 HashList *POP3FetchUrls = NULL; /* -> rss_aggregator; ->RefCount access to be locked too. */
  61
  62 typedef struct __pop3_room_counter {
  63         int count;
  64         long QRnumber;
  65 }pop3_room_counter;
  66
  67 typedef enum ePOP3_C_States {
  68         ReadGreeting,
  69         GetUserState,
  70         GetPassState,
  71         GetListCommandState,
  72         GetListOneLine,
  73         GetOneMessageIDState,
  74         ReadMessageBodyFollowing,
  75         ReadMessageBody,
  76         GetDeleteState,
  77         ReadQuitState,
  78         POP3C_MaxRead
  79 }ePOP3_C_States;
  80
  81
  82 typedef struct _FetchItem {
  83         long MSGID;
  84         long MSGSize;
  85         StrBuf *MsgUIDL;
  86         StrBuf *MsgUID;
  87         int NeedFetch;
  88         struct CtdlMessage *Msg;
  89 } FetchItem;
  90
  91 void HfreeFetchItem(void *vItem)
  92 {
  93         FetchItem *Item = (FetchItem*) vItem;
  94         FreeStrBuf(&Item->MsgUIDL);
  95         FreeStrBuf(&Item->MsgUID);
  96         free(Item);
  97 }
  98
  99 typedef struct __pop3aggr {
 100         AsyncIO          IO;
 101
 102         long n;
 103         long RefCount;
 104 ///     ParsedURL *Pop3Host;
 105         DNSQueryParts HostLookup;
 106
 107 //      StrBuf          *rooms;
 108         long             QRnumber;
 109         HashList        *OtherQRnumbers;
 110
 111         StrBuf          *Url;
 112 ///     StrBuf *pop3host; -> URL
 113         StrBuf *pop3user;
 114         StrBuf *pop3pass;
 115         StrBuf *RoomName; // TODO: fill me
 116         int keep;
 117         time_t interval;
 118         ePOP3_C_States State;
 119         HashList *MsgNumbers;
 120         HashPos *Pos;
 121         FetchItem *CurrMsg;
 122 } pop3aggr;
 123
 124 void DeletePOP3Aggregator(void *vptr)
 125 {
 126         pop3aggr *ptr = vptr;
 127         DeleteHashPos(&ptr->Pos);
 128         DeleteHash(&ptr->MsgNumbers);
 129 //      FreeStrBuf(&ptr->rooms);
 130         FreeStrBuf(&ptr->pop3user);
 131         FreeStrBuf(&ptr->pop3pass);
 132         FreeStrBuf(&ptr->RoomName);
 133         FreeURL(&ptr->IO.ConnectMe);
 134         FreeStrBuf(&ptr->Url);
 135         FreeStrBuf(&ptr->IO.IOBuf);
 136         FreeStrBuf(&ptr->IO.SendBuf.Buf);
 137         FreeStrBuf(&ptr->IO.RecvBuf.Buf);
 138         free(ptr);
 139 }
 140
 141
 142 typedef eNextState(*Pop3ClientHandler)(pop3aggr* RecvMsg);
 143
 144 eNextState POP3_C_Shutdown(AsyncIO *IO);
 145 eNextState POP3_C_Timeout(AsyncIO *IO);
 146 eNextState POP3_C_ConnFail(AsyncIO *IO);
 147 eNextState POP3_C_DispatchReadDone(AsyncIO *IO);
 148 eNextState POP3_C_DispatchWriteDone(AsyncIO *IO);
 149 eNextState POP3_C_Terminate(AsyncIO *IO);
 150 eReadState POP3_C_ReadServerStatus(AsyncIO *IO);
 151 eNextState POP3_C_ReAttachToFetchMessages(AsyncIO *IO);
 152
 153 eNextState FinalizePOP3AggrRun(AsyncIO *IO)
 154 {
 155         HashPos  *It;
 156         pop3aggr *cptr = (pop3aggr *)IO->Data;
 157
 158         CtdlLogPrintf(CTDL_DEBUG, "Terminating Aggregator; bye.\n");
 159
 160         It = GetNewHashPos(POP3FetchUrls, 0);
 161         citthread_mutex_lock(&POP3QueueMutex);
 162         {
 163                 GetHashPosFromKey(POP3FetchUrls, SKEY(cptr->Url), It);
 164                 DeleteEntryFromHash(POP3FetchUrls, It);
 165         }
 166         citthread_mutex_unlock(&POP3QueueMutex);
 167         DeleteHashPos(&It);
 168         return eAbort;
 169 }
 170
 171 eNextState FailAggregationRun(AsyncIO *IO)
 172 {
 173         return eAbort;
 174 }
 175
 176 #define POP3C_DBG_SEND() CtdlLogPrintf(CTDL_DEBUG, "POP3 client[%ld]: > %s\n", RecvMsg->n, ChrPtr(RecvMsg->IO.SendBuf.Buf))
 177 #define POP3C_DBG_READ() CtdlLogPrintf(CTDL_DEBUG, "POP3 client[%ld]: < %s\n", RecvMsg->n, ChrPtr(RecvMsg->IO.IOBuf))
 178 #define POP3C_OK (strncasecmp(ChrPtr(RecvMsg->IO.IOBuf), "+OK", 3) == 0)
 179
 180 eNextState POP3C_ReadGreeting(pop3aggr *RecvMsg)
 181 {
 182         POP3C_DBG_READ();
 183         /* Read the server greeting */
 184         if (!POP3C_OK) return eTerminateConnection;
 185         else return eSendReply;
 186 }
 187
 188
 189 eNextState POP3C_SendUser(pop3aggr *RecvMsg)
 190 {
 191         /* Identify ourselves.  NOTE: we have to append a CR to each command.  The LF will
 192          * automatically be appended by sock_puts().  Believe it or not, leaving out the CR
 193          * will cause problems if the server happens to be Exchange, which is so b0rken it
 194          * actually barfs on LF-terminated newlines.
 195          */
 196         StrBufPrintf(RecvMsg->IO.SendBuf.Buf,
 197                      "USER %s\r\n", ChrPtr(RecvMsg->pop3user));
 198         POP3C_DBG_SEND();
 199         return eReadMessage;
 200 }
 201
 202 eNextState POP3C_GetUserState(pop3aggr *RecvMsg)
 203 {
 204         POP3C_DBG_READ();
 205         if (!POP3C_OK) return eTerminateConnection;
 206         else return eSendReply;
 207 }
 208
 209 eNextState POP3C_SendPassword(pop3aggr *RecvMsg)
 210 {
 211         /* Password */
 212         StrBufPrintf(RecvMsg->IO.SendBuf.Buf,
 213                      "PASS %s\r\n", ChrPtr(RecvMsg->pop3pass));
 214         CtdlLogPrintf(CTDL_DEBUG, "<PASS <password>\n");
 215 //      POP3C_DBG_SEND();
 216         return eReadMessage;
 217 }
 218
 219 eNextState POP3C_GetPassState(pop3aggr *RecvMsg)
 220 {
 221         POP3C_DBG_READ();
 222         if (!POP3C_OK) return eTerminateConnection;
 223         else return eSendReply;
 224 }
 225
 226 eNextState POP3C_SendListCommand(pop3aggr *RecvMsg)
 227 {
 228         /* Get the list of messages */
 229         StrBufPlain(RecvMsg->IO.SendBuf.Buf, HKEY("LIST\r\n"));
 230         POP3C_DBG_SEND();
 231         return eReadMessage;
 232 }
 233
 234 eNextState POP3C_GetListCommandState(pop3aggr *RecvMsg)
 235 {
 236         POP3C_DBG_READ();
 237         if (!POP3C_OK) return eTerminateConnection;
 238         RecvMsg->MsgNumbers = NewHash(1, NULL);
 239         RecvMsg->State++;
 240         return eReadMore;
 241 }
 242
 243
 244 eNextState POP3C_GetListOneLine(pop3aggr *RecvMsg)
 245 {
 246         const char *pch;
 247         FetchItem *OneMsg = NULL;
 248         POP3C_DBG_READ();
 249
 250         if ((StrLength(RecvMsg->IO.IOBuf) == 1) &&
 251             (ChrPtr(RecvMsg->IO.IOBuf)[0] == '.'))
 252         {
 253                 if (GetCount(RecvMsg->MsgNumbers) == 0)
 254                 {
 255                         ////    RecvMsg->Sate = ReadQuitState;
 256                 }
 257                 else
 258                 {
 259                         RecvMsg->Pos = GetNewHashPos(RecvMsg->MsgNumbers, 0);
 260                 }
 261                 return eSendReply;
 262
 263         }
 264         OneMsg = (FetchItem*) malloc(sizeof(FetchItem));
 265         memset(OneMsg, 0, sizeof(FetchItem));
 266         OneMsg->MSGID = atol(ChrPtr(RecvMsg->IO.IOBuf));
 267
 268         pch = strchr(ChrPtr(RecvMsg->IO.IOBuf), ' ');
 269         if (pch != NULL)
 270         {
 271                 OneMsg->MSGSize = atol(pch + 1);
 272         }
 273         Put(RecvMsg->MsgNumbers, LKEY(OneMsg->MSGID), OneMsg, HfreeFetchItem);
 274
 275         //RecvMsg->State --; /* read next Line */
 276         return eReadMore;
 277 }
 278
 279 eNextState POP3_FetchNetworkUsetableEntry(AsyncIO *IO)
 280 {
 281         long HKLen;
 282         const char *HKey;
 283         void *vData;
 284         struct cdbdata *cdbut;
 285         pop3aggr *RecvMsg = (pop3aggr *) IO->Data;
 286
 287         if(GetNextHashPos(RecvMsg->MsgNumbers, RecvMsg->Pos, &HKLen, &HKey, &vData))
 288         {
 289                 struct UseTable ut;
 290
 291                 RecvMsg->CurrMsg = (FetchItem*) vData;
 292                 CtdlLogPrintf(CTDL_DEBUG, "CHECKING: whether %s has already been seen: ", ChrPtr(RecvMsg->CurrMsg->MsgUID));
 293                 /* Find out if we've already seen this item */
 294                 safestrncpy(ut.ut_msgid,
 295                             ChrPtr(RecvMsg->CurrMsg->MsgUID),
 296                             sizeof(ut.ut_msgid));
 297                 ut.ut_timestamp = time(NULL);/// TODO: libev timestamp!
 298
 299                 cdbut = cdb_fetch(CDB_USETABLE, SKEY(RecvMsg->CurrMsg->MsgUID));
 300                 if (cdbut != NULL) {
 301                         /* Item has already been seen */
 302                         CtdlLogPrintf(CTDL_DEBUG, "YES\n");
 303                         cdb_free(cdbut);
 304
 305                         /* rewrite the record anyway, to update the timestamp */
 306                         cdb_store(CDB_USETABLE,
 307                                   SKEY(RecvMsg->CurrMsg->MsgUID),
 308                                   &ut, sizeof(struct UseTable) );
 309                         RecvMsg->CurrMsg->NeedFetch = 0;
 310                 }
 311                 else
 312                 {
 313                         CtdlLogPrintf(CTDL_DEBUG, "NO\n");
 314                         RecvMsg->CurrMsg->NeedFetch = 1;
 315                 }
 316                 return NextDBOperation(&RecvMsg->IO, POP3_FetchNetworkUsetableEntry);
 317         }
 318         else
 319         {
 320                 /* ok, now we know them all, continue with reading the actual messages. */
 321                 DeleteHashPos(&RecvMsg->Pos);
 322
 323                 return QueueEventContext(IO, POP3_C_ReAttachToFetchMessages);
 324         }
 325 }
 326
 327 eNextState POP3C_GetOneMessagID(pop3aggr *RecvMsg)
 328 {
 329         long HKLen;
 330         const char *HKey;
 331         void *vData;
 332
 333         if(GetNextHashPos(RecvMsg->MsgNumbers, RecvMsg->Pos, &HKLen, &HKey, &vData))
 334         {
 335                 RecvMsg->CurrMsg = (FetchItem*) vData;
 336                 /* Find out the UIDL of the message, to determine whether we've already downloaded it */
 337                 StrBufPrintf(RecvMsg->IO.SendBuf.Buf,
 338                              "UIDL %ld\r\n", RecvMsg->CurrMsg->MSGID);
 339                 POP3C_DBG_SEND();
 340         }
 341         else
 342         {
 343             RecvMsg->State++;
 344                 DeleteHashPos(&RecvMsg->Pos);
 345                 /// done receiving uidls.. start looking them up now.
 346                 RecvMsg->Pos = GetNewHashPos(RecvMsg->MsgNumbers, 0);
 347                 return QueueDBOperation(&RecvMsg->IO, POP3_FetchNetworkUsetableEntry);
 348         }
 349         return eReadMore; /* TODO */
 350 }
 351
 352 eNextState POP3C_GetOneMessageIDState(pop3aggr *RecvMsg)
 353 {
 354         POP3C_DBG_READ();
 355         if (!POP3C_OK) return eTerminateConnection;
 356         RecvMsg->CurrMsg->MsgUIDL = NewStrBufPlain(NULL, StrLength(RecvMsg->IO.IOBuf));
 357         RecvMsg->CurrMsg->MsgUID = NewStrBufPlain(NULL, StrLength(RecvMsg->IO.IOBuf) * 2);
 358
 359         StrBufExtract_token(RecvMsg->CurrMsg->MsgUIDL, RecvMsg->IO.IOBuf, 2, ' ');
 360         StrBufPrintf(RecvMsg->CurrMsg->MsgUID,
 361                      "pop3/%s/%s:%s@%s",
 362                      ChrPtr(RecvMsg->RoomName),
 363                      ChrPtr(RecvMsg->CurrMsg->MsgUIDL),
 364                      RecvMsg->IO.ConnectMe->User,
 365                      RecvMsg->IO.ConnectMe->Host);
 366         RecvMsg->State --;
 367         return eSendReply;
 368 }
 369
 370
 371 eNextState POP3C_SendGetOneMsg(pop3aggr *RecvMsg)
 372 {
 373         long HKLen;
 374         const char *HKey;
 375         void *vData;
 376
 377         RecvMsg->CurrMsg = NULL;
 378         while (GetNextHashPos(RecvMsg->MsgNumbers, RecvMsg->Pos, &HKLen, &HKey, &vData) &&
 379                (RecvMsg->CurrMsg = (FetchItem*) vData, RecvMsg->CurrMsg->NeedFetch == 0))
 380         {}
 381
 382         if ((RecvMsg->CurrMsg != NULL ) && (RecvMsg->CurrMsg->NeedFetch == 1))
 383         {
 384                 /* Message has not been seen. Tell the server to fetch the message... */
 385                 StrBufPrintf(RecvMsg->IO.SendBuf.Buf,
 386                              "RETR %ld\r\n", RecvMsg->CurrMsg->MSGID);
 387                 POP3C_DBG_SEND();
 388                 return eReadMessage;
 389         }
 390         else {
 391                 RecvMsg->State = ReadQuitState;
 392                 return POP3_C_DispatchWriteDone(&RecvMsg->IO);
 393         }
 394 }
 395
 396
 397 eNextState POP3C_ReadMessageBodyFollowing(pop3aggr *RecvMsg)
 398 {
 399         POP3C_DBG_READ();
 400         if (!POP3C_OK) return eTerminateConnection;
 401         RecvMsg->IO.ReadMsg = NewAsyncMsg(HKEY("."),
 402                                           RecvMsg->CurrMsg->MSGSize,
 403                                           config.c_maxmsglen,
 404                                           NULL, -1,
 405                                           1);
 406
 407         return eReadPayload;
 408 }
 409
 410
 411 eNextState POP3C_StoreMsgRead(AsyncIO *IO)
 412 {
 413         pop3aggr *RecvMsg = (pop3aggr *) IO->Data;
 414         struct UseTable ut;
 415
 416         CtdlLogPrintf(CTDL_DEBUG, "MARKING: %s as seen: ", ChrPtr(RecvMsg->CurrMsg->MsgUID));
 417
 418         safestrncpy(ut.ut_msgid,
 419                     ChrPtr(RecvMsg->CurrMsg->MsgUID),
 420                     sizeof(ut.ut_msgid));
 421         ut.ut_timestamp = time(NULL); /* TODO: use libev time */
 422         cdb_store(CDB_USETABLE,
 423                   ChrPtr(RecvMsg->CurrMsg->MsgUID),
 424                   StrLength(RecvMsg->CurrMsg->MsgUID),
 425                   &ut,
 426                   sizeof(struct UseTable) );
 427
 428         return QueueEventContext(&RecvMsg->IO, POP3_C_ReAttachToFetchMessages);
 429 }
 430 eNextState POP3C_SaveMsg(AsyncIO *IO)
 431 {
 432         long msgnum;
 433         pop3aggr *RecvMsg = (pop3aggr *) IO->Data;
 434
 435         /* Do Something With It (tm) */
 436         msgnum = CtdlSubmitMsg(RecvMsg->CurrMsg->Msg,
 437                                NULL,
 438                                ChrPtr(RecvMsg->RoomName),
 439                                0);
 440         if (msgnum > 0L) {
 441                 /* Message has been committed to the store */
 442                 /* write the uidl to the use table so we don't fetch this message again */
 443         }
 444         CtdlFreeMessage(RecvMsg->CurrMsg->Msg);
 445
 446         return NextDBOperation(&RecvMsg->IO, POP3C_StoreMsgRead);
 447         return eReadMessage;
 448 }
 449
 450 eNextState POP3C_ReadMessageBody(pop3aggr *RecvMsg)
 451 {
 452         CtdlLogPrintf(CTDL_DEBUG, "Converting message...\n");
 453         RecvMsg->CurrMsg->Msg = convert_internet_message_buf(&RecvMsg->IO.ReadMsg->MsgBuf);
 454
 455         return QueueDBOperation(&RecvMsg->IO, POP3C_SaveMsg);
 456 }
 457
 458 eNextState POP3C_SendDelete(pop3aggr *RecvMsg)
 459 {
 460         if (!RecvMsg->keep) {
 461                 StrBufPrintf(RecvMsg->IO.SendBuf.Buf,
 462                              "DELE %ld\r\n", RecvMsg->CurrMsg->MSGID);
 463                 POP3C_DBG_SEND();
 464                 return eReadMessage;
 465         }
 466         else {
 467                 RecvMsg->State = ReadMessageBodyFollowing;
 468                 return POP3_C_DispatchWriteDone(&RecvMsg->IO);
 469         }
 470 }
 471 eNextState POP3C_ReadDeleteState(pop3aggr *RecvMsg)
 472 {
 473         POP3C_DBG_READ();
 474         RecvMsg->State = GetOneMessageIDState;
 475         return eReadMessage;
 476 }
 477
 478 eNextState POP3C_SendQuit(pop3aggr *RecvMsg)
 479 {
 480         /* Log out */
 481         StrBufPlain(RecvMsg->IO.SendBuf.Buf,
 482                     HKEY("QUIT\r\n3)"));
 483         POP3C_DBG_SEND();
 484         return eReadMessage;
 485 }
 486
 487
 488 eNextState POP3C_ReadQuitState(pop3aggr *RecvMsg)
 489 {
 490         POP3C_DBG_READ();
 491         return eTerminateConnection;
 492 }
 493
 494 const long POP3_C_ConnTimeout = 1000;
 495 const long DefaultPOP3Port = 110;
 496
 497 Pop3ClientHandler POP3C_ReadHandlers[] = {
 498         POP3C_ReadGreeting,
 499         POP3C_GetUserState,
 500         POP3C_GetPassState,
 501         POP3C_GetListCommandState,
 502         POP3C_GetListOneLine,
 503         POP3C_GetOneMessageIDState,
 504         POP3C_ReadMessageBodyFollowing,
 505         POP3C_ReadMessageBody,
 506         POP3C_ReadDeleteState,
 507         POP3C_ReadQuitState,
 508 };
 509
 510 const long POP3_C_SendTimeouts[POP3C_MaxRead] = {
 511         100,
 512         100,
 513         100,
 514         100,
 515         100,
 516         100,
 517         100,
 518         100
 519 };
 520 const ConstStr POP3C_ReadErrors[POP3C_MaxRead] = {
 521         {HKEY("Connection broken during ")},
 522         {HKEY("Connection broken during ")},
 523         {HKEY("Connection broken during ")},
 524         {HKEY("Connection broken during ")},
 525         {HKEY("Connection broken during ")},
 526         {HKEY("Connection broken during ")},
 527         {HKEY("Connection broken during ")},
 528         {HKEY("Connection broken during ")}
 529 };
 530
 531 Pop3ClientHandler POP3C_SendHandlers[] = {
 532         NULL, /* we don't send a greeting */
 533         POP3C_SendUser,
 534         POP3C_SendPassword,
 535         POP3C_SendListCommand,
 536         NULL,
 537         POP3C_GetOneMessagID,
 538         POP3C_SendGetOneMsg,
 539         NULL,
 540         POP3C_SendDelete,
 541         POP3C_SendQuit
 542 };
 543
 544 const long POP3_C_ReadTimeouts[] = {
 545         100,
 546         100,
 547         100,
 548         100,
 549         100,
 550         100,
 551         100,
 552         100,
 553         100,
 554         100
 555 };
 556 /*****************************************************************************/
 557 /*                     POP3 CLIENT DISPATCHER                                */
 558 /*****************************************************************************/
 559
 560 void POP3SetTimeout(eNextState NextTCPState, pop3aggr *pMsg)
 561 {
 562         double Timeout = 0.0;
 563
 564         CtdlLogPrintf(CTDL_DEBUG, "POP3: %s\n", __FUNCTION__);
 565
 566         switch (NextTCPState) {
 567         case eSendReply:
 568         case eSendMore:
 569                 Timeout = POP3_C_SendTimeouts[pMsg->State];
 570 /*
 571   if (pMsg->State == eDATABody) {
 572   / * if we're sending a huge message, we need more time. * /
 573   Timeout += StrLength(pMsg->msgtext) / 1024;
 574   }
 575 */
 576                 break;
 577         case eReadMessage:
 578                 Timeout = POP3_C_ReadTimeouts[pMsg->State];
 579 /*
 580   if (pMsg->State == eDATATerminateBody) {
 581   / *
 582   * some mailservers take a nap before accepting the message
 583   * content inspection and such.
 584   * /
 585   Timeout += StrLength(pMsg->msgtext) / 1024;
 586   }
 587 */
 588                 break;
 589         case eReadPayload:
 590                 Timeout = 100000;
 591                 /* TODO!!! */
 592                 break;
 593         case eSendDNSQuery:
 594         case eReadDNSReply:
 595         case eConnect:
 596         case eTerminateConnection:
 597         case eDBQuery:
 598         case eAbort:
 599         case eReadMore://// TODO
 600                 return;
 601         }
 602         SetNextTimeout(&pMsg->IO, Timeout);
 603 }
 604 eNextState POP3_C_DispatchReadDone(AsyncIO *IO)
 605 {
 606         CtdlLogPrintf(CTDL_DEBUG, "POP3: %s\n", __FUNCTION__);
 607         pop3aggr *pMsg = IO->Data;
 608         eNextState rc;
 609
 610         rc = POP3C_ReadHandlers[pMsg->State](pMsg);
 611         if (rc != eReadMore)
 612             pMsg->State++;
 613         POP3SetTimeout(rc, pMsg);
 614         return rc;
 615 }
 616 eNextState POP3_C_DispatchWriteDone(AsyncIO *IO)
 617 {
 618         CtdlLogPrintf(CTDL_DEBUG, "POP3: %s\n", __FUNCTION__);
 619         pop3aggr *pMsg = IO->Data;
 620         eNextState rc;
 621
 622         rc = POP3C_SendHandlers[pMsg->State](pMsg);
 623         POP3SetTimeout(rc, pMsg);
 624         return rc;
 625 }
 626
 627
 628 /*****************************************************************************/
 629 /*                     POP3 CLIENT ERROR CATCHERS                            */
 630 /*****************************************************************************/
 631 eNextState POP3_C_Terminate(AsyncIO *IO)
 632 {
 633 ///     pop3aggr *pMsg = (pop3aggr *)IO->Data;
 634
 635         CtdlLogPrintf(CTDL_DEBUG, "POP3: %s\n", __FUNCTION__);
 636         FinalizePOP3AggrRun(IO);
 637         return eAbort;
 638 }
 639 eNextState POP3_C_Timeout(AsyncIO *IO)
 640 {
 641         pop3aggr *pMsg = IO->Data;
 642
 643         CtdlLogPrintf(CTDL_DEBUG, "POP3: %s\n", __FUNCTION__);
 644         StrBufPlain(IO->ErrMsg, CKEY(POP3C_ReadErrors[pMsg->State]));
 645         return FailAggregationRun(IO);
 646 }
 647 eNextState POP3_C_ConnFail(AsyncIO *IO)
 648 {
 649         pop3aggr *pMsg = (pop3aggr *)IO->Data;
 650
 651         CtdlLogPrintf(CTDL_DEBUG, "POP3: %s\n", __FUNCTION__);
 652         StrBufPlain(IO->ErrMsg, CKEY(POP3C_ReadErrors[pMsg->State]));
 653         return FailAggregationRun(IO);
 654 }
 655 eNextState POP3_C_Shutdown(AsyncIO *IO)
 656 {
 657         CtdlLogPrintf(CTDL_DEBUG, "POP3: %s\n", __FUNCTION__);
 658 ////    pop3aggr *pMsg = IO->Data;
 659
 660         ////pMsg->MyQEntry->Status = 3;
 661         ///StrBufPlain(pMsg->MyQEntry->StatusMessage, HKEY("server shutdown during message retrieval."));
 662         FinalizePOP3AggrRun(IO);
 663         return eAbort;
 664 }
 665
 666
 667 /**
 668  * @brief lineread Handler; understands when to read more POP3 lines, and when this is a one-lined reply.
 669  */
 670 eReadState POP3_C_ReadServerStatus(AsyncIO *IO)
 671 {
 672         eReadState Finished = eBufferNotEmpty;
 673
 674         switch (IO->NextState) {
 675         case eSendDNSQuery:
 676         case eReadDNSReply:
 677         case eDBQuery:
 678         case eConnect:
 679         case eTerminateConnection:
 680         case eAbort:
 681                 Finished = eReadFail;
 682                 break;
 683         case eSendReply:
 684         case eSendMore:
 685         case eReadMore:
 686         case eReadMessage:
 687                 Finished = StrBufChunkSipLine(IO->IOBuf, &IO->RecvBuf);
 688                 break;
 689         case eReadPayload:
 690                 Finished = CtdlReadMessageBodyAsync(IO);
 691                 break;
 692         }
 693         return Finished;
 694 }
 695
 696 /*****************************************************************************
 697  * So we connect our Server IP here.                                         *
 698  *****************************************************************************/
 699 eNextState POP3_C_ReAttachToFetchMessages(AsyncIO *IO)
 700 {
 701         pop3aggr *cpptr = IO->Data;
 702
 703         CtdlLogPrintf(CTDL_DEBUG, "POP3: %s\n", __FUNCTION__);
 704 ////??? cpptr->State ++;
 705         if (cpptr->Pos == NULL)
 706                 cpptr->Pos = GetNewHashPos(cpptr->MsgNumbers, 0);
 707
 708         POP3_C_DispatchWriteDone(IO);
 709         ReAttachIO(IO, cpptr, 0);
 710         IO->NextState = eReadMessage;
 711         return IO->NextState;
 712 }
 713
 714 eNextState connect_ip(AsyncIO *IO)
 715 {
 716         pop3aggr *cpptr = IO->Data;
 717
 718         CtdlLogPrintf(CTDL_DEBUG, "POP3: %s\n", __FUNCTION__);
 719
 720 ////    IO->ConnectMe = &cpptr->Pop3Host;
 721         /*  Bypass the ns lookup result like this: IO->Addr.sin_addr.s_addr = inet_addr("127.0.0.1"); */
 722
 723         /////// SetConnectStatus(IO);
 724
 725         return InitEventIO(IO, cpptr,
 726                            POP3_C_ConnTimeout,
 727                            POP3_C_ReadTimeouts[0],
 728                            1);
 729 }
 730
 731 eNextState get_one_host_ip_done(AsyncIO *IO)
 732 {
 733         pop3aggr *cpptr = IO->Data;
 734         struct hostent *hostent;
 735
 736         QueryCbDone(IO);
 737
 738         hostent = cpptr->HostLookup.VParsedDNSReply;
 739         if ((cpptr->HostLookup.DNSStatus == ARES_SUCCESS) &&
 740             (hostent != NULL) ) {
 741                 memset(&cpptr->IO.ConnectMe->Addr, 0, sizeof(struct in6_addr));
 742                 if (cpptr->IO.ConnectMe->IPv6) {
 743                         memcpy(&cpptr->IO.ConnectMe->Addr.sin6_addr.s6_addr,
 744                                &hostent->h_addr_list[0],
 745                                sizeof(struct in6_addr));
 746
 747                         cpptr->IO.ConnectMe->Addr.sin6_family = hostent->h_addrtype;
 748                         cpptr->IO.ConnectMe->Addr.sin6_port   = htons(DefaultPOP3Port);
 749                 }
 750                 else {
 751                         struct sockaddr_in *addr = (struct sockaddr_in*) &cpptr->IO.ConnectMe->Addr;
 752                         /* Bypass the ns lookup result like this: IO->Addr.sin_addr.s_addr = inet_addr("127.0.0.1"); */
 753 //                      addr->sin_addr.s_addr = htonl((uint32_t)&hostent->h_addr_list[0]);
 754                         memcpy(&addr->sin_addr.s_addr,
 755                                hostent->h_addr_list[0],
 756                                sizeof(uint32_t));
 757
 758                         addr->sin_family = hostent->h_addrtype;
 759                         addr->sin_port   = htons(DefaultPOP3Port);
 760
 761                 }
 762                 return connect_ip(IO);
 763         }
 764         else
 765                 return eAbort;
 766 }
 767
 768 eNextState get_one_host_ip(AsyncIO *IO)
 769 {
 770         pop3aggr *cpptr = IO->Data;
 771         /*
 772          * here we start with the lookup of one host. it might be...
 773          * - the relay host *sigh*
 774          * - the direct hostname if there was no mx record
 775          * - one of the mx'es
 776          */
 777
 778         InitC_ares_dns(IO);
 779
 780         CtdlLogPrintf(CTDL_DEBUG, "POP3: %s\n", __FUNCTION__);
 781
 782         CtdlLogPrintf(CTDL_DEBUG,
 783                       "POP3 client[%ld]: looking up %s-Record %s : %d ...\n",
 784                       cpptr->n,
 785                       (cpptr->IO.ConnectMe->IPv6)? "aaaa": "a",
 786                       cpptr->IO.ConnectMe->Host,
 787                       cpptr->IO.ConnectMe->Port);
 788
 789         if (!QueueQuery((cpptr->IO.ConnectMe->IPv6)? ns_t_aaaa : ns_t_a,
 790                         cpptr->IO.ConnectMe->Host,
 791                         &cpptr->IO,
 792                         &cpptr->HostLookup,
 793                         get_one_host_ip_done))
 794         {
 795 //              cpptr->MyQEntry->Status = 5;
 796 //              StrBufPrintf(SendMsg->MyQEntry->StatusMessage,
 797 //                           "No MX hosts found for <%s>", SendMsg->node);
 798                 cpptr->IO.NextState = eTerminateConnection;
 799                 return IO->NextState;
 800         }
 801         IO->NextState = eReadDNSReply;
 802         return IO->NextState;
 803 }
 804
 805
 806
 807 int pop3_do_fetching(pop3aggr *cpptr)
 808 {
 809         CitContext *SubC;
 810
 811         cpptr->IO.Data          = cpptr;
 812
 813         cpptr->IO.SendDone      = POP3_C_DispatchWriteDone;
 814         cpptr->IO.ReadDone      = POP3_C_DispatchReadDone;
 815         cpptr->IO.Terminate     = POP3_C_Terminate;
 816         cpptr->IO.LineReader    = POP3_C_ReadServerStatus;
 817         cpptr->IO.ConnFail      = POP3_C_ConnFail;
 818         cpptr->IO.Timeout       = POP3_C_Timeout;
 819         cpptr->IO.ShutdownAbort = POP3_C_Shutdown;
 820
 821         cpptr->IO.SendBuf.Buf   = NewStrBufPlain(NULL, 1024);
 822         cpptr->IO.RecvBuf.Buf   = NewStrBufPlain(NULL, 1024);
 823         cpptr->IO.IOBuf         = NewStrBuf();
 824
 825         cpptr->IO.NextState     = eReadMessage;
 826 /* TODO
 827    CtdlLogPrintf(CTDL_DEBUG, "POP3: %s %s %s <password>\n", roomname, pop3host, pop3user);
 828    CtdlLogPrintf(CTDL_NOTICE, "Connecting to <%s>\n", pop3host);
 829 */
 830
 831         SubC = CloneContext (&pop3_client_CC);
 832         SubC->session_specific_data = (char*) cpptr;
 833         cpptr->IO.CitContext = SubC;
 834
 835         if (cpptr->IO.ConnectMe->IsIP) {
 836                 QueueEventContext(&cpptr->IO,
 837                                   connect_ip);
 838         }
 839         else { /* uneducated admin has chosen to add DNS to the equation... */
 840                 QueueEventContext(&cpptr->IO,
 841                                   get_one_host_ip);
 842         }
 843         return 1;
 844 }
 845
 846 /*
 847  * Scan a room's netconfig to determine whether it requires POP3 aggregation
 848  */
 849 void pop3client_scan_room(struct ctdlroom *qrbuf, void *data)
 850 {
 851         StrBuf *CfgData;
 852         StrBuf *CfgType;
 853         StrBuf *Line;
 854
 855         struct stat statbuf;
 856         char filename[PATH_MAX];
 857         int  fd;
 858         int Done;
 859         void *vptr;
 860         const char *CfgPtr, *lPtr;
 861         const char *Err;
 862
 863 //      pop3_room_counter *Count = NULL;
 864 //      pop3aggr *cpptr;
 865
 866         citthread_mutex_lock(&POP3QueueMutex);
 867         if (GetHash(POP3QueueRooms, LKEY(qrbuf->QRnumber), &vptr))
 868         {
 869                 CtdlLogPrintf(CTDL_DEBUG,
 870                               "pop3client: [%ld] %s already in progress.\n",
 871                               qrbuf->QRnumber,
 872                               qrbuf->QRname);
 873                 citthread_mutex_unlock(&POP3QueueMutex);
 874                 return;
 875         }
 876         citthread_mutex_unlock(&POP3QueueMutex);
 877
 878         assoc_file_name(filename, sizeof filename, qrbuf, ctdl_netcfg_dir);
 879
 880         if (CtdlThreadCheckStop())
 881                 return;
 882
 883         /* Only do net processing for rooms that have netconfigs */
 884         fd = open(filename, 0);
 885         if (fd <= 0) {
 886                 //CtdlLogPrintf(CTDL_DEBUG, "rssclient: %s no config.\n", qrbuf->QRname);
 887                 return;
 888         }
 889         if (CtdlThreadCheckStop())
 890                 return;
 891         if (fstat(fd, &statbuf) == -1) {
 892                 CtdlLogPrintf(CTDL_DEBUG,  "ERROR: could not stat configfile '%s' - %s\n",
 893                               filename, strerror(errno));
 894                 return;
 895         }
 896         if (CtdlThreadCheckStop())
 897                 return;
 898         CfgData = NewStrBufPlain(NULL, statbuf.st_size + 1);
 899         if (StrBufReadBLOB(CfgData, &fd, 1, statbuf.st_size, &Err) < 0) {
 900                 close(fd);
 901                 FreeStrBuf(&CfgData);
 902                 CtdlLogPrintf(CTDL_DEBUG,  "ERROR: reading config '%s' - %s<br>\n",
 903                               filename, strerror(errno));
 904                 return;
 905         }
 906         close(fd);
 907         if (CtdlThreadCheckStop())
 908                 return;
 909
 910         CfgPtr = NULL;
 911         CfgType = NewStrBuf();
 912         Line = NewStrBufPlain(NULL, StrLength(CfgData));
 913         Done = 0;
 914
 915         while (!Done)
 916         {
 917                 Done = StrBufSipLine(Line, CfgData, &CfgPtr) == 0;
 918                 if (StrLength(Line) > 0)
 919                 {
 920                         lPtr = NULL;
 921                         StrBufExtract_NextToken(CfgType, Line, &lPtr, '|');
 922                         if (!strcasecmp("pop3client", ChrPtr(CfgType)))
 923                         {
 924                                 pop3aggr *cptr;
 925                                 StrBuf *Tmp;
 926 /*
 927                                 if (Count == NULL)
 928                                 {
 929                                         Count = malloc(sizeof(pop3_room_counter));
 930                                         Count->count = 0;
 931                                 }
 932                                 Count->count ++;
 933 */
 934                                 cptr = (pop3aggr *) malloc(sizeof(pop3aggr));
 935                                 memset(cptr, 0, sizeof(pop3aggr));
 936                                 /// TODO do we need this? cptr->roomlist_parts = 1;
 937                                 cptr->RoomName = NewStrBufPlain(qrbuf->QRname, -1);
 938                                 cptr->pop3user = NewStrBufPlain(NULL, StrLength(Line));
 939                                 cptr->pop3pass = NewStrBufPlain(NULL, StrLength(Line));
 940                                 cptr->Url = NewStrBuf();
 941                                 Tmp = NewStrBuf();
 942
 943                                 StrBufExtract_NextToken(Tmp, Line, &lPtr, '|');
 944                                 StrBufExtract_NextToken(cptr->pop3user, Line, &lPtr, '|');
 945                                 StrBufExtract_NextToken(cptr->pop3pass, Line, &lPtr, '|');
 946                                 cptr->keep = StrBufExtractNext_long(Line, &lPtr, '|');
 947                                 cptr->interval = StrBufExtractNext_long(Line, &lPtr, '|');
 948
 949                                 StrBufPrintf(cptr->Url, "pop3://%s:%s@%s/%s",
 950                                              ChrPtr(cptr->pop3user),
 951                                              ChrPtr(cptr->pop3pass),
 952                                              ChrPtr(Tmp),
 953                                              ChrPtr(cptr->RoomName));
 954                                 FreeStrBuf(&Tmp);
 955                                 ParseURL(&cptr->IO.ConnectMe, cptr->Url, 110);
 956
 957
 958 #if 0
 959 /* todo: we need to reunite the url to be shure. */
 960
 961                                 citthread_mutex_lock(&POP3ueueMutex);
 962                                 GetHash(POP3FetchUrls, SKEY(ptr->Url), &vptr);
 963                                 use_this_cptr = (pop3aggr *)vptr;
 964
 965                                 if (use_this_rncptr != NULL)
 966                                 {
 967                                         /* mustn't attach to an active session */
 968                                         if (use_this_cptr->RefCount > 0)
 969                                         {
 970                                                 DeletePOP3Cfg(cptr);
 971 ///                                             Count->count--;
 972                                         }
 973                                         else
 974                                         {
 975                                                 long *QRnumber;
 976                                                 StrBufAppendBufPlain(use_this_cptr->rooms,
 977                                                                      qrbuf->QRname,
 978                                                                      -1, 0);
 979                                                 if (use_this_cptr->roomlist_parts == 1)
 980                                                 {
 981                                                         use_this_cptr->OtherQRnumbers = NewHash(1, lFlathash);
 982                                                 }
 983                                                 QRnumber = (long*)malloc(sizeof(long));
 984                                                 *QRnumber = qrbuf->QRnumber;
 985                                                 Put(use_this_cptr->OtherQRnumbers, LKEY(qrbuf->QRnumber), QRnumber, NULL);
 986                                                 use_this_cptr->roomlist_parts++;
 987                                         }
 988                                         citthread_mutex_unlock(&POP3QueueMutex);
 989                                         continue;
 990                                 }
 991                                 citthread_mutex_unlock(&RSSQueueMutex);
 992 #endif
 993
 994                                 citthread_mutex_lock(&POP3QueueMutex);
 995                                 Put(POP3FetchUrls, SKEY(cptr->Url), cptr, DeletePOP3Aggregator);
 996                                 citthread_mutex_unlock(&POP3QueueMutex);
 997
 998                         }
 999
1000                 }
1001
1002                 ///fclose(fp);
1003
1004         }
1005         FreeStrBuf(&Line);
1006         FreeStrBuf(&CfgType);
1007         FreeStrBuf(&CfgData);
1008 }
1009
1010
1011 void pop3client_scan(void) {
1012         static time_t last_run = 0L;
1013         static int doing_pop3client = 0;
1014 ///     struct pop3aggr *pptr;
1015         time_t fastest_scan;
1016         HashPos *it;
1017         long len;
1018         const char *Key;
1019         void *vrptr;
1020         pop3aggr *cptr;
1021
1022         if (config.c_pop3_fastest < config.c_pop3_fetch)
1023                 fastest_scan = config.c_pop3_fastest;
1024         else
1025                 fastest_scan = config.c_pop3_fetch;
1026
1027         /*
1028          * Run POP3 aggregation no more frequently than once every n seconds
1029          */
1030         if ( (time(NULL) - last_run) < fastest_scan ) {
1031                 return;
1032         }
1033
1034         /*
1035          * This is a simple concurrency check to make sure only one pop3client run
1036          * is done at a time.  We could do this with a mutex, but since we
1037          * don't really require extremely fine granularity here, we'll do it
1038          * with a static variable instead.
1039          */
1040         if (doing_pop3client) return;
1041         doing_pop3client = 1;
1042
1043         CtdlLogPrintf(CTDL_DEBUG, "pop3client started\n");
1044         CtdlForEachRoom(pop3client_scan_room, NULL);
1045
1046
1047         citthread_mutex_lock(&POP3QueueMutex);
1048         it = GetNewHashPos(POP3FetchUrls, 0);
1049         while (GetNextHashPos(POP3FetchUrls, it, &len, &Key, &vrptr) &&
1050                (vrptr != NULL)) {
1051                 cptr = (pop3aggr *)vrptr;
1052                 if (cptr->RefCount == 0)
1053                         if (!pop3_do_fetching(cptr))
1054                                 DeletePOP3Aggregator(cptr);////TODO
1055         }
1056         DeleteHashPos(&it);
1057         citthread_mutex_unlock(&POP3QueueMutex);
1058
1059         CtdlLogPrintf(CTDL_DEBUG, "pop3client ended\n");
1060         last_run = time(NULL);
1061         doing_pop3client = 0;
1062 }
1063
1064
1065 void pop3_cleanup(void)
1066 {
1067         citthread_mutex_destroy(&POP3QueueMutex);
1068         DeleteHash(&POP3FetchUrls);
1069         DeleteHash(&POP3QueueRooms);
1070 }
1071
1072 CTDL_MODULE_INIT(pop3client)
1073 {
1074         if (!threading)
1075         {
1076                 CtdlFillSystemContext(&pop3_client_CC, "POP3aggr");
1077                 citthread_mutex_init(&POP3QueueMutex, NULL);
1078                 POP3QueueRooms = NewHash(1, lFlathash);
1079                 POP3FetchUrls = NewHash(1, NULL);
1080                 CtdlRegisterSessionHook(pop3client_scan, EVT_TIMER);
1081                 CtdlRegisterCleanupHook(pop3_cleanup);
1082         }
1083         /* return our Subversion id for the Log */
1084         return "pop3client";
1085 }