citadel/modules/pop3client/serv_pop3client.c

   1 /*
   2  * Consolidate mail from remote POP3 accounts.
   3  *
   4  * Copyright (c) 2007-2009 by the citadel.org team
   5  *
   6  *  This program is free software; you can redistribute it and/or modify
   7  *  it under the terms of the GNU General Public License as published by
   8  *  the Free Software Foundation; either version 3 of the License, or
   9  *  (at your option) any later version.
  10  *
  11  *  This program is distributed in the hope that it will be useful,
  12  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
  13  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14  *  GNU General Public License for more details.
  15  *
  16  *  You should have received a copy of the GNU General Public License
  17  *  along with this program; if not, write to the Free Software
  18  *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  19  */
  20
  21 #include <stdlib.h>
  22 #include <unistd.h>
  23 #include <stdio.h>
  24
  25 #if TIME_WITH_SYS_TIME
  26 # include <sys/time.h>
  27 # include <time.h>
  28 #else
  29 # if HAVE_SYS_TIME_H
  30 #  include <sys/time.h>
  31 # else
  32 #  include <time.h>
  33 # endif
  34 #endif
  35
  36 #include <ctype.h>
  37 #include <string.h>
  38 #include <errno.h>
  39 #include <sys/types.h>
  40 #include <sys/stat.h>
  41 #include <libcitadel.h>
  42 #include "citadel.h"
  43 #include "server.h"
  44 #include "citserver.h"
  45 #include "support.h"
  46 #include "config.h"
  47 #include "ctdl_module.h"
  48 #include "clientsocket.h"
  49 #include "msgbase.h"
  50 #include "internet_addressing.h"
  51 #include "database.h"
  52 #include "citadel_dirs.h"
  53 #include "event_client.h"
  54
  55
  56
  57 citthread_mutex_t POP3QueueMutex; /* locks the access to the following vars: */
  58 HashList *POP3QueueRooms = NULL; /* rss_room_counter */
  59 HashList *POP3FetchUrls = NULL; /* -> rss_aggregator; ->RefCount access to be locked too. */
  60
  61 typedef struct __pop3_room_counter {
  62         int count;
  63         long QRnumber;
  64 }pop3_room_counter;
  65
  66 typedef enum ePOP3_C_States {
  67         ReadGreeting,
  68         GetUserState,
  69         GetPassState,
  70         GetListCommandState,
  71         GetOneMessageIDState,
  72         ReadMessageBodyFollowing,
  73         ReadMessageBody,
  74         ReadQuitState,
  75         POP3C_MaxRead
  76 }ePOP3_C_States;
  77
  78
  79 typedef struct _FetchItem {
  80         long MSGID;
  81         StrBuf *MsgUIDL;
  82         StrBuf *MsgUID;
  83         int NeedFetch;
  84 } FetchItem;
  85
  86 void HfreeFetchItem(void *vItem)
  87 {
  88         FetchItem *Item = (FetchItem*) vItem;
  89         FreeStrBuf(&Item->MsgUIDL);
  90         FreeStrBuf(&Item->MsgUID);
  91         free(Item);
  92 }
  93
  94 typedef struct __pop3aggr {
  95         AsyncIO          IO;
  96
  97         long n;
  98         long RefCount;
  99         ParsedURL Pop3Host;
 100         DNSQueryParts HostLookup;
 101
 102         StrBuf          *rooms;
 103         long             QRnumber;
 104         HashList        *OtherQRnumbers;
 105
 106         StrBuf          *Url;
 107 ///     StrBuf *pop3host; -> URL
 108         StrBuf *pop3user;
 109         StrBuf *pop3pass;
 110         StrBuf *RoomName; // TODO: fill me
 111         int keep;
 112         time_t interval;
 113         ePOP3_C_States State;
 114         HashList *MsgNumbers;
 115         HashPos *Pos;
 116         FetchItem *CurrMsg;
 117 } pop3aggr;
 118
 119 void DeletePOP3Aggregator(void *vptr)
 120 {
 121         pop3aggr *ptr = vptr;
 122         DeleteHashPos(&ptr->Pos);
 123         DeleteHash(&ptr->MsgNumbers);
 124         FreeStrBuf(&ptr->rooms);
 125         FreeStrBuf(&ptr->pop3user);
 126         FreeStrBuf(&ptr->pop3pass);
 127         FreeStrBuf(&ptr->RoomName);
 128 }
 129
 130
 131 typedef eNextState(*Pop3ClientHandler)(pop3aggr* RecvMsg);
 132
 133 eNextState POP3_C_Shutdown(AsyncIO *IO);
 134 eNextState POP3_C_Timeout(AsyncIO *IO);
 135 eNextState POP3_C_ConnFail(AsyncIO *IO);
 136 eNextState POP3_C_DispatchReadDone(AsyncIO *IO);
 137 eNextState POP3_C_DispatchWriteDone(AsyncIO *IO);
 138 eNextState POP3_C_Terminate(AsyncIO *IO);
 139 eReadState POP3_C_ReadServerStatus(AsyncIO *IO);
 140
 141 eNextState FinalizePOP3AggrRun(AsyncIO *IO)
 142 {
 143
 144         return eAbort;
 145 }
 146
 147 eNextState FailAggregationRun(AsyncIO *IO)
 148 {
 149         return eAbort;
 150 }
 151
 152 #define POP3C_DBG_SEND() CtdlLogPrintf(CTDL_DEBUG, "POP3 client[%ld]: > %s\n", RecvMsg->n, ChrPtr(RecvMsg->IO.SendBuf.Buf))
 153 #define POP3C_DBG_READ() CtdlLogPrintf(CTDL_DEBUG, "POP3 client[%ld]: < %s\n", RecvMsg->n, ChrPtr(RecvMsg->IO.IOBuf))
 154 #define POP3C_OK (strncasecmp(ChrPtr(RecvMsg->IO.IOBuf), "+OK", 3) == 0)
 155
 156 eNextState POP3C_ReadGreeting(pop3aggr *RecvMsg)
 157 {
 158         POP3C_DBG_READ();
 159         /* Read the server greeting */
 160         if (!POP3C_OK) return eTerminateConnection;
 161         else return eSendReply;
 162 }
 163
 164
 165 eNextState POP3C_SendUser(pop3aggr *RecvMsg)
 166 {
 167         /* Identify ourselves.  NOTE: we have to append a CR to each command.  The LF will
 168          * automatically be appended by sock_puts().  Believe it or not, leaving out the CR
 169          * will cause problems if the server happens to be Exchange, which is so b0rken it
 170          * actually barfs on LF-terminated newlines.
 171          */
 172         StrBufPrintf(RecvMsg->IO.SendBuf.Buf,
 173                      "USER %s\r\n", ChrPtr(RecvMsg->pop3user));
 174         POP3C_DBG_SEND();
 175         return eReadMessage;
 176 }
 177
 178 eNextState POP3C_GetUserState(pop3aggr *RecvMsg)
 179 {
 180         POP3C_DBG_READ();
 181         if (!POP3C_OK) return eTerminateConnection;
 182         else return eSendReply;
 183 }
 184
 185 eNextState POP3C_SendPassword(pop3aggr *RecvMsg)
 186 {
 187         /* Password */
 188         StrBufPrintf(RecvMsg->IO.SendBuf.Buf,
 189                      "PASS %s\r\n", ChrPtr(RecvMsg->pop3pass));
 190         CtdlLogPrintf(CTDL_DEBUG, "<PASS <password>\n");
 191 //      POP3C_DBG_SEND();
 192         return eReadMessage;
 193 }
 194
 195 eNextState POP3C_GetPassState(pop3aggr *RecvMsg)
 196 {
 197         POP3C_DBG_READ();
 198         if (!POP3C_OK) return eTerminateConnection;
 199         else return eSendReply;
 200 }
 201
 202 eNextState POP3C_SendListCommand(pop3aggr *RecvMsg)
 203 {
 204         /* Get the list of messages */
 205         StrBufPlain(RecvMsg->IO.SendBuf.Buf, HKEY("LIST\r\n"));
 206         POP3C_DBG_SEND();
 207         return eReadMessage;
 208 }
 209
 210 eNextState POP3C_GetListCommandState(pop3aggr *RecvMsg)
 211 {
 212         POP3C_DBG_READ();
 213         if (!POP3C_OK) return eTerminateConnection;
 214         RecvMsg->MsgNumbers = NewHash(1, NULL);
 215         return eReadMore;
 216 }
 217
 218
 219 eNextState POP3C_GetListOneLine(pop3aggr *RecvMsg)
 220 {
 221         FetchItem *OneMsg = NULL;
 222         POP3C_DBG_READ();
 223
 224         if ((StrLength(RecvMsg->IO.IOBuf) == 1) &&
 225             (ChrPtr(RecvMsg->IO.IOBuf)[0] == '.'))
 226         {
 227                 if (GetCount(RecvMsg->MsgNumbers) == 0)
 228                 {
 229                         ////    RecvMsg->Sate = ReadQuitState;
 230                 }
 231                 else
 232                 {
 233                         RecvMsg->Pos = GetNewHashPos(RecvMsg->MsgNumbers, 0);
 234                 }
 235                 return eSendReply;
 236
 237         }
 238         OneMsg = (FetchItem*) malloc(sizeof(FetchItem));
 239         memset(OneMsg, 0, sizeof(FetchItem));
 240         OneMsg->MSGID = atoi(ChrPtr(RecvMsg->IO.IOBuf));
 241         Put(RecvMsg->MsgNumbers, LKEY(OneMsg->MSGID), OneMsg, HfreeFetchItem);
 242
 243         //RecvMsg->State --; /* read next Line */
 244         return eReadMore;
 245 }
 246
 247 eNextState POP3C_GetOneMessagID(pop3aggr *RecvMsg)
 248 {
 249         long HKLen;
 250         const char *HKey;
 251         void *vData;
 252
 253         if(GetNextHashPos(RecvMsg->MsgNumbers, RecvMsg->Pos, &HKLen, &HKey, &vData))
 254         {
 255                 RecvMsg->CurrMsg = (FetchItem*) vData;
 256                 /* Find out the UIDL of the message, to determine whether we've already downloaded it */
 257                 StrBufPrintf(RecvMsg->IO.SendBuf.Buf,
 258                              "UIDL %ld\r\n", RecvMsg->CurrMsg->MSGID);
 259                 POP3C_DBG_SEND();
 260         }
 261         else
 262         {
 263                 DeleteHashPos(&RecvMsg->Pos);
 264                 /// done receiving uidls.. start looking them up now.
 265                 RecvMsg->Pos = GetNewHashPos(RecvMsg->MsgNumbers, 0);
 266
 267         }
 268         return eReadMore; /* TODO */
 269 }
 270
 271 #if 0
 272 eNextState FetchNetworkUsetableEntry(AsyncIO *IO)
 273 {
 274         struct cdbdata *cdbut;
 275         networker_save_message *Ctx = (networker_save_message *) IO->Data;
 276
 277         if(GetNextHashPos(RecvMsg->MsgNumbers, RecvMsg->Pos, &HKLen, &HKey, &vData))
 278         {
 279
 280                 /* Find out if we've already seen this item */
 281                 strcpy(Ctx->ut.ut_msgid, ChrPtr(Ctx->MsgGUID)); /// TODO
 282                 Ctx->ut.ut_timestamp = time(NULL);
 283
 284                 cdbut = cdb_fetch(CDB_USETABLE, SKEY(Ctx->MsgGUID));
 285                 if (cdbut != NULL) {
 286                         /* Item has already been seen */
 287                         CtdlLogPrintf(CTDL_DEBUG, "%s has already been seen\n", ChrPtr(Ctx->MsgGUID));
 288                         cdb_free(cdbut);
 289
 290                         /* rewrite the record anyway, to update the timestamp */
 291                         cdb_store(CDB_USETABLE,
 292                                   SKEY(Ctx->MsgGUID),
 293                                   &Ctx->ut, sizeof(struct UseTable) );
 294                         return eAbort;
 295                 }
 296                 else
 297                 {
 298                         NextDBOperation(IO, RSSSaveMessage);
 299                         return eSendMore;
 300                 }
 301         }
 302         return eReadMessage;
 303 }
 304 #endif
 305
 306
 307 eNextState POP3C_GetOneMessageIDState(pop3aggr *RecvMsg)
 308 {
 309         POP3C_DBG_READ();
 310         if (!POP3C_OK) return eTerminateConnection;
 311         RecvMsg->CurrMsg->MsgUIDL = NewStrBufPlain(NULL, StrLength(RecvMsg->IO.IOBuf));
 312         RecvMsg->CurrMsg->MsgUID = NewStrBufPlain(NULL, StrLength(RecvMsg->IO.IOBuf) * 2);
 313
 314         StrBufExtract_token(RecvMsg->CurrMsg->MsgUIDL, RecvMsg->IO.IOBuf, 2, ' ');
 315         StrBufPrintf(RecvMsg->CurrMsg->MsgUID,
 316                      "pop3/%s/%s@%s",
 317                      ChrPtr(RecvMsg->RoomName),
 318                      ChrPtr(RecvMsg->CurrMsg->MsgUIDL),
 319                      RecvMsg->Pop3Host.Host);
 320         return eReadMessage;
 321 }
 322
 323 eNextState POP3C_GetOneMessageIDFromUseTable(pop3aggr *RecvMsg)
 324 {
 325
 326         struct cdbdata *cdbut;
 327         struct UseTable ut;
 328
 329         cdbut = cdb_fetch(CDB_USETABLE, SKEY(RecvMsg->CurrMsg->MsgUID));
 330         if (cdbut != NULL) {
 331                 /* message has already been seen */
 332                 CtdlLogPrintf(CTDL_DEBUG, "%s has already been seen\n", ChrPtr(RecvMsg->CurrMsg->MsgUID));
 333                 cdb_free(cdbut);
 334
 335                 /* rewrite the record anyway, to update the timestamp */
 336                 strcpy(ut.ut_msgid, ChrPtr(RecvMsg->CurrMsg->MsgUID));
 337                 ut.ut_timestamp = time(NULL);
 338                 cdb_store(CDB_USETABLE, SKEY(RecvMsg->CurrMsg->MsgUID), &ut, sizeof(struct UseTable) );
 339         }
 340
 341         return eReadMessage;
 342 }
 343
 344 eNextState POP3C_SendGetOneMsg(pop3aggr *RecvMsg)
 345 {
 346         /* Message has not been seen. Tell the server to fetch the message... */
 347         StrBufPrintf(RecvMsg->IO.SendBuf.Buf,
 348                      "RETR %ld\r\n", RecvMsg->CurrMsg->MSGID);
 349         POP3C_DBG_SEND();
 350         return eReadMessage;
 351 }
 352
 353
 354 eNextState POP3C_ReadMessageBodyFollowing(pop3aggr *RecvMsg)
 355 {
 356         POP3C_DBG_READ();
 357         if (!POP3C_OK) return eTerminateConnection;
 358         else return eSendReply;
 359 }
 360
 361
 362
 363 eNextState POP3C_ReadMessageBody(pop3aggr *RecvMsg)
 364 {
 365 #if 0
 366 //TODO
 367         /* If we get to this point, the message is on its way.  Read it. */
 368         body = CtdlReadMessageBody(HKEY("."), config.c_maxmsglen, NULL, 1, &sock);
 369         if (body == NULL) goto bail;
 370
 371         CtdlLogPrintf(CTDL_DEBUG, "Converting message...\n");
 372         msg = convert_internet_message(body);
 373         body = NULL;    /* yes, this should be dereferenced, NOT freed */
 374
 375                         /* Do Something With It (tm) */
 376         msgnum = CtdlSubmitMsg(msg, NULL, roomname, 0);
 377         if (msgnum > 0L) {
 378                 /* Message has been committed to the store */
 379                 /* write the uidl to the use table so we don't fetch this message again */
 380         }
 381         CtdlFreeMessage(msg);
 382 #endif
 383         return eReadMessage;
 384 }
 385
 386 eNextState POP3C_StoreMsgRead(pop3aggr *RecvMsg)
 387 {
 388 #if 0
 389         strcpy(ut.ut_msgid, utmsgid);
 390         ut.ut_timestamp = time(NULL);
 391         cdb_store(CDB_USETABLE, utmsgid, strlen(utmsgid),
 392                   &ut, sizeof(struct UseTable) );
 393 #endif
 394         return eReadMessage;/// TODO
 395 }
 396 eNextState POP3C_SendDelete(pop3aggr *RecvMsg)
 397 {
 398         if (!RecvMsg->keep) {
 399                 StrBufPrintf(RecvMsg->IO.SendBuf.Buf,
 400                              "DELE %ld\r\n", RecvMsg->CurrMsg->MSGID);
 401                 POP3C_DBG_SEND();
 402         }
 403         return eReadMessage;
 404 }
 405 eNextState POP3C_ReadDeleteState(pop3aggr *RecvMsg)
 406 {
 407         POP3C_DBG_READ();
 408         return eReadMessage;
 409 }
 410
 411 eNextState POP3C_SendQuit(pop3aggr *RecvMsg)
 412 {
 413         /* Log out */
 414         StrBufPlain(RecvMsg->IO.SendBuf.Buf,
 415                     HKEY("QUIT\r\n3)"));
 416         POP3C_DBG_SEND();
 417         return eReadMessage;
 418 }
 419
 420
 421 eNextState POP3C_ReadQuitState(pop3aggr *RecvMsg)
 422 {
 423         POP3C_DBG_READ();
 424         return eAbort;
 425 }
 426
 427 const long POP3_C_ConnTimeout = 1000;
 428 const long DefaultPOP3Port = 110;
 429
 430 Pop3ClientHandler POP3C_ReadHandlers[] = {
 431         POP3C_ReadGreeting,
 432         POP3C_GetUserState,
 433         POP3C_GetPassState,
 434         POP3C_GetListCommandState,
 435         POP3C_GetOneMessageIDState,
 436         POP3C_ReadMessageBodyFollowing,
 437         POP3C_ReadMessageBody,
 438         POP3C_ReadQuitState,
 439 };
 440
 441 const long POP3_C_SendTimeouts[POP3C_MaxRead] = {
 442         100,
 443         100,
 444         100,
 445         100,
 446         100,
 447         100,
 448         100,
 449         100
 450 };
 451 const ConstStr POP3C_ReadErrors[POP3C_MaxRead] = {
 452         {HKEY("Connection broken during ")},
 453         {HKEY("Connection broken during ")},
 454         {HKEY("Connection broken during ")},
 455         {HKEY("Connection broken during ")},
 456         {HKEY("Connection broken during ")},
 457         {HKEY("Connection broken during ")},
 458         {HKEY("Connection broken during ")},
 459         {HKEY("Connection broken during ")}
 460 };
 461
 462 Pop3ClientHandler POP3C_SendHandlers[] = {
 463         NULL, /* we don't send a greeting */
 464         POP3C_SendUser,
 465         POP3C_SendPassword,
 466         POP3C_SendListCommand,
 467         POP3C_GetListOneLine,
 468         POP3C_GetOneMessagID,
 469         POP3C_SendGetOneMsg,
 470         POP3C_SendDelete,
 471         POP3C_SendQuit
 472 };
 473
 474 const long POP3_C_ReadTimeouts[] = {
 475         100,
 476         100,
 477         100,
 478         100,
 479         100,
 480         100,
 481         100,
 482         100,
 483         100,
 484         100
 485 };
 486 /*****************************************************************************/
 487 /*                     POP3 CLIENT DISPATCHER                                */
 488 /*****************************************************************************/
 489
 490 void POP3SetTimeout(eNextState NextTCPState, pop3aggr *pMsg)
 491 {
 492         double Timeout = 0.0;
 493
 494         CtdlLogPrintf(CTDL_DEBUG, "POP3: %s\n", __FUNCTION__);
 495
 496         switch (NextTCPState) {
 497         case eSendReply:
 498         case eSendMore:
 499                 Timeout = POP3_C_SendTimeouts[pMsg->State];
 500 /*
 501   if (pMsg->State == eDATABody) {
 502   / * if we're sending a huge message, we need more time. * /
 503   Timeout += StrLength(pMsg->msgtext) / 1024;
 504   }
 505 */
 506                 break;
 507         case eReadMessage:
 508                 Timeout = POP3_C_ReadTimeouts[pMsg->State];
 509 /*
 510   if (pMsg->State == eDATATerminateBody) {
 511   / *
 512   * some mailservers take a nap before accepting the message
 513   * content inspection and such.
 514   * /
 515   Timeout += StrLength(pMsg->msgtext) / 1024;
 516   }
 517 */
 518                 break;
 519         case eSendDNSQuery:
 520         case eReadDNSReply:
 521         case eConnect:
 522         case eTerminateConnection:
 523         case eAbort:
 524         case eReadMore://// TODO
 525                 return;
 526         }
 527         SetNextTimeout(&pMsg->IO, Timeout);
 528 }
 529 eNextState POP3_C_DispatchReadDone(AsyncIO *IO)
 530 {
 531         CtdlLogPrintf(CTDL_DEBUG, "POP3: %s\n", __FUNCTION__);
 532         pop3aggr *pMsg = IO->Data;
 533         eNextState rc;
 534
 535         rc = POP3C_ReadHandlers[pMsg->State](pMsg);
 536         pMsg->State++;
 537         POP3SetTimeout(rc, pMsg);
 538         return rc;
 539 }
 540 eNextState POP3_C_DispatchWriteDone(AsyncIO *IO)
 541 {
 542         CtdlLogPrintf(CTDL_DEBUG, "POP3: %s\n", __FUNCTION__);
 543         pop3aggr *pMsg = IO->Data;
 544         eNextState rc;
 545
 546         rc = POP3C_SendHandlers[pMsg->State](pMsg);
 547         POP3SetTimeout(rc, pMsg);
 548         return rc;
 549 }
 550
 551
 552 /*****************************************************************************/
 553 /*                     POP3 CLIENT ERROR CATCHERS                            */
 554 /*****************************************************************************/
 555 eNextState POP3_C_Terminate(AsyncIO *IO)
 556 {
 557 ///     pop3aggr *pMsg = (pop3aggr *)IO->Data;
 558
 559         CtdlLogPrintf(CTDL_DEBUG, "POP3: %s\n", __FUNCTION__);
 560         FinalizePOP3AggrRun(IO);
 561         return eAbort;
 562 }
 563 eNextState POP3_C_Timeout(AsyncIO *IO)
 564 {
 565         pop3aggr *pMsg = IO->Data;
 566
 567         CtdlLogPrintf(CTDL_DEBUG, "POP3: %s\n", __FUNCTION__);
 568         StrBufPlain(IO->ErrMsg, CKEY(POP3C_ReadErrors[pMsg->State]));
 569         return FailAggregationRun(IO);
 570 }
 571 eNextState POP3_C_ConnFail(AsyncIO *IO)
 572 {
 573         pop3aggr *pMsg = (pop3aggr *)IO->Data;
 574
 575         CtdlLogPrintf(CTDL_DEBUG, "POP3: %s\n", __FUNCTION__);
 576         StrBufPlain(IO->ErrMsg, CKEY(POP3C_ReadErrors[pMsg->State]));
 577         return FailAggregationRun(IO);
 578 }
 579 eNextState POP3_C_Shutdown(AsyncIO *IO)
 580 {
 581         CtdlLogPrintf(CTDL_DEBUG, "POP3: %s\n", __FUNCTION__);
 582 ////    pop3aggr *pMsg = IO->Data;
 583
 584         ////pMsg->MyQEntry->Status = 3;
 585         ///StrBufPlain(pMsg->MyQEntry->StatusMessage, HKEY("server shutdown during message retrieval."));
 586         FinalizePOP3AggrRun(IO);
 587         return eAbort;
 588 }
 589
 590
 591 /**
 592  * @brief lineread Handler; understands when to read more POP3 lines, and when this is a one-lined reply.
 593  */
 594 eReadState POP3_C_ReadServerStatus(AsyncIO *IO)
 595 {
 596         eReadState Finished = eBufferNotEmpty;
 597
 598         while (Finished == eBufferNotEmpty) {
 599                 Finished = StrBufChunkSipLine(IO->IOBuf, &IO->RecvBuf);
 600
 601                 switch (Finished) {
 602                 case eMustReadMore: /// read new from socket...
 603                         return Finished;
 604                         break;
 605                 case eBufferNotEmpty: /* shouldn't happen... */
 606                 case eReadSuccess: /// done for now...
 607                         if (StrLength(IO->IOBuf) < 4)
 608                                 continue;
 609                         if (ChrPtr(IO->IOBuf)[3] == '-')
 610                                 Finished = eBufferNotEmpty;
 611                         else
 612                                 return Finished;
 613                         break;
 614                 case eReadFail: /// WHUT?
 615                         ///todo: shut down!
 616                         break;
 617                 }
 618         }
 619         return Finished;
 620 }
 621
 622 /*****************************************************************************
 623  * So we connect our Server IP here.                                         *
 624  *****************************************************************************/
 625 eNextState connect_ip(AsyncIO *IO)
 626 {
 627         pop3aggr *cpptr = IO->Data;
 628
 629         CtdlLogPrintf(CTDL_DEBUG, "POP3: %s\n", __FUNCTION__);
 630
 631 ////    IO->ConnectMe = &cpptr->Pop3Host;
 632         /*  Bypass the ns lookup result like this: IO->Addr.sin_addr.s_addr = inet_addr("127.0.0.1"); */
 633
 634         /////// SetConnectStatus(IO);
 635
 636         return InitEventIO(IO, cpptr,
 637                            POP3_C_ConnTimeout,
 638                            POP3_C_ReadTimeouts[0],
 639                            1);
 640 }
 641
 642 eNextState get_one_host_ip_done(AsyncIO *IO)
 643 {
 644         pop3aggr *cpptr = IO->Data;
 645         struct hostent *hostent;
 646
 647         QueryCbDone(IO);
 648
 649         hostent = cpptr->HostLookup.VParsedDNSReply;
 650         if ((cpptr->HostLookup.DNSStatus == ARES_SUCCESS) &&
 651             (hostent != NULL) ) {
 652                 memset(&cpptr->Pop3Host.Addr, 0, sizeof(struct in6_addr));
 653                 if (cpptr->Pop3Host.IPv6) {
 654                         memcpy(&cpptr->Pop3Host.Addr.sin6_addr.s6_addr,
 655                                &hostent->h_addr_list[0],
 656                                sizeof(struct in6_addr));
 657
 658                         cpptr->Pop3Host.Addr.sin6_family = hostent->h_addrtype;
 659                         cpptr->Pop3Host.Addr.sin6_port   = htons(DefaultPOP3Port);
 660                 }
 661                 else {
 662                         struct sockaddr_in *addr = (struct sockaddr_in*) &cpptr->Pop3Host.Addr;
 663                         /* Bypass the ns lookup result like this: IO->Addr.sin_addr.s_addr = inet_addr("127.0.0.1"); */
 664 //                      addr->sin_addr.s_addr = htonl((uint32_t)&hostent->h_addr_list[0]);
 665                         memcpy(&addr->sin_addr.s_addr,
 666                                hostent->h_addr_list[0],
 667                                sizeof(uint32_t));
 668
 669                         addr->sin_family = hostent->h_addrtype;
 670                         addr->sin_port   = htons(DefaultPOP3Port);
 671
 672                 }
 673                 return connect_ip(IO);
 674         }
 675         else
 676                 return eAbort;
 677 }
 678
 679 eNextState get_one_host_ip(AsyncIO *IO)
 680 {
 681         pop3aggr *cpptr = IO->Data;
 682         /*
 683          * here we start with the lookup of one host. it might be...
 684          * - the relay host *sigh*
 685          * - the direct hostname if there was no mx record
 686          * - one of the mx'es
 687          */
 688
 689         InitC_ares_dns(IO);
 690
 691         CtdlLogPrintf(CTDL_DEBUG, "POP3: %s\n", __FUNCTION__);
 692
 693         CtdlLogPrintf(CTDL_DEBUG,
 694                       "POP3 client[%ld]: looking up %s-Record %s : %d ...\n",
 695                       cpptr->n,
 696                       (cpptr->Pop3Host.IPv6)? "aaaa": "a",
 697                       cpptr->Pop3Host.Host,
 698                       cpptr->Pop3Host.Port);
 699
 700         if (!QueueQuery((cpptr->Pop3Host.IPv6)? ns_t_aaaa : ns_t_a,
 701                         cpptr->Pop3Host.Host,
 702                         &cpptr->IO,
 703                         &cpptr->HostLookup,
 704                         get_one_host_ip_done))
 705         {
 706 //              cpptr->MyQEntry->Status = 5;
 707 //              StrBufPrintf(SendMsg->MyQEntry->StatusMessage,
 708 //                           "No MX hosts found for <%s>", SendMsg->node);
 709                 cpptr->IO.NextState = eTerminateConnection;
 710                 return IO->NextState;
 711         }
 712         IO->NextState = eReadDNSReply;
 713         return IO->NextState;
 714 }
 715
 716
 717
 718 int pop3_do_fetching(pop3aggr *cpptr)
 719 {
 720         CitContext *SubC;
 721
 722         cpptr->IO.Data          = cpptr;
 723
 724         cpptr->IO.SendDone      = POP3_C_DispatchWriteDone;
 725         cpptr->IO.ReadDone      = POP3_C_DispatchReadDone;
 726         cpptr->IO.Terminate     = POP3_C_Terminate;
 727         cpptr->IO.LineReader    = POP3_C_ReadServerStatus;
 728         cpptr->IO.ConnFail      = POP3_C_ConnFail;
 729         cpptr->IO.Timeout       = POP3_C_Timeout;
 730         cpptr->IO.ShutdownAbort = POP3_C_Shutdown;
 731
 732         cpptr->IO.SendBuf.Buf   = NewStrBufPlain(NULL, 1024);
 733         cpptr->IO.RecvBuf.Buf   = NewStrBufPlain(NULL, 1024);
 734         cpptr->IO.IOBuf         = NewStrBuf();
 735
 736         cpptr->IO.NextState     = eReadMessage;
 737 /* TODO
 738    CtdlLogPrintf(CTDL_DEBUG, "POP3: %s %s %s <password>\n", roomname, pop3host, pop3user);
 739    CtdlLogPrintf(CTDL_NOTICE, "Connecting to <%s>\n", pop3host);
 740 */
 741
 742         SubC = CloneContext (CC);
 743         SubC->session_specific_data = (char*) cpptr;
 744         cpptr->IO.CitContext = SubC;
 745
 746         if (cpptr->IO.ConnectMe->IsIP) {
 747                 QueueEventContext(&cpptr->IO,
 748                                   connect_ip);
 749         }
 750         else { /* uneducated admin has chosen to add DNS to the equation... */
 751                 QueueEventContext(&cpptr->IO,
 752                                   get_one_host_ip);
 753         }
 754         return 1;
 755 }
 756
 757 /*
 758  * Scan a room's netconfig to determine whether it requires POP3 aggregation
 759  */
 760 void pop3client_scan_room(struct ctdlroom *qrbuf, void *data)
 761 {
 762         StrBuf *CfgData;
 763         StrBuf *CfgType;
 764         StrBuf *Line;
 765
 766         struct stat statbuf;
 767         char filename[PATH_MAX];
 768         int  fd;
 769         int Done;
 770         void *vptr;
 771         const char *CfgPtr, *lPtr;
 772         const char *Err;
 773
 774         pop3_room_counter *Count = NULL;
 775 //      pop3aggr *cpptr;
 776
 777         citthread_mutex_lock(&POP3QueueMutex);
 778         if (GetHash(POP3QueueRooms, LKEY(qrbuf->QRnumber), &vptr))
 779         {
 780                 CtdlLogPrintf(CTDL_DEBUG,
 781                               "pop3client: [%ld] %s already in progress.\n",
 782                               qrbuf->QRnumber,
 783                               qrbuf->QRname);
 784                 citthread_mutex_unlock(&POP3QueueMutex);
 785                 return;
 786         }
 787         citthread_mutex_unlock(&POP3QueueMutex);
 788
 789         assoc_file_name(filename, sizeof filename, qrbuf, ctdl_netcfg_dir);
 790
 791         if (CtdlThreadCheckStop())
 792                 return;
 793
 794         /* Only do net processing for rooms that have netconfigs */
 795         fd = open(filename, 0);
 796         if (fd <= 0) {
 797                 //CtdlLogPrintf(CTDL_DEBUG, "rssclient: %s no config.\n", qrbuf->QRname);
 798                 return;
 799         }
 800         if (CtdlThreadCheckStop())
 801                 return;
 802         if (fstat(fd, &statbuf) == -1) {
 803                 CtdlLogPrintf(CTDL_DEBUG,  "ERROR: could not stat configfile '%s' - %s\n",
 804                               filename, strerror(errno));
 805                 return;
 806         }
 807         if (CtdlThreadCheckStop())
 808                 return;
 809         CfgData = NewStrBufPlain(NULL, statbuf.st_size + 1);
 810         if (StrBufReadBLOB(CfgData, &fd, 1, statbuf.st_size, &Err) < 0) {
 811                 close(fd);
 812                 FreeStrBuf(&CfgData);
 813                 CtdlLogPrintf(CTDL_DEBUG,  "ERROR: reading config '%s' - %s<br>\n",
 814                               filename, strerror(errno));
 815                 return;
 816         }
 817         close(fd);
 818         if (CtdlThreadCheckStop())
 819                 return;
 820
 821         CfgPtr = NULL;
 822         CfgType = NewStrBuf();
 823         Line = NewStrBufPlain(NULL, StrLength(CfgData));
 824         Done = 0;
 825
 826         while (!Done)
 827         {
 828                 Done = StrBufSipLine(Line, CfgData, &CfgPtr) == 0;
 829                 if (StrLength(Line) > 0)
 830                 {
 831                         lPtr = NULL;
 832                         StrBufExtract_NextToken(CfgType, Line, &lPtr, '|');
 833                         if (!strcasecmp("pop3client", ChrPtr(CfgType)))
 834                         {
 835                                 pop3aggr *cptr;
 836
 837                                 if (Count == NULL)
 838                                 {
 839                                         Count = malloc(sizeof(pop3_room_counter));
 840                                         Count->count = 0;
 841                                 }
 842                                 Count->count ++;
 843                                 cptr = (pop3aggr *) malloc(sizeof(pop3aggr));
 844                                 memset(cptr, 0, sizeof(pop3aggr));
 845                                 /// TODO do we need this? cptr->roomlist_parts = 1;
 846                                 cptr->rooms = NewStrBufPlain(qrbuf->QRname, -1);
 847                                 cptr->pop3user = NewStrBufPlain(NULL, StrLength(Line));
 848                                 cptr->pop3pass = NewStrBufPlain(NULL, StrLength(Line));
 849                                 cptr->Url = NewStrBuf();
 850
 851                                 StrBufExtract_NextToken(cptr->Url, Line, &lPtr, '|');
 852                                 StrBufExtract_NextToken(cptr->pop3user, Line, &lPtr, '|');
 853                                 StrBufExtract_NextToken(cptr->pop3pass, Line, &lPtr, '|');
 854                                 cptr->keep = StrBufExtractNext_long(Line, &lPtr, '|');
 855                                 cptr->interval = StrBufExtractNext_long(Line, &lPtr, '|');
 856
 857                                 ParseURL(&cptr->IO.ConnectMe, cptr->Url, 110);
 858
 859                                 cptr->IO.ConnectMe->CurlCreds = cptr->pop3user;
 860                                 cptr->IO.ConnectMe->User = ChrPtr(cptr->IO.ConnectMe->CurlCreds);
 861                                 cptr->IO.ConnectMe->UrlWithoutCred = cptr->pop3pass;
 862                                 cptr->IO.ConnectMe->Pass = ChrPtr(cptr->IO.ConnectMe->UrlWithoutCred);
 863
 864
 865
 866 #if 0
 867 /* todo: we need to reunite the url to be shure. */
 868
 869                                 citthread_mutex_lock(&POP3ueueMutex);
 870                                 GetHash(POP3FetchUrls, SKEY(ptr->Url), &vptr);
 871                                 use_this_cptr = (pop3aggr *)vptr;
 872
 873                                 if (use_this_rncptr != NULL)
 874                                 {
 875                                         /* mustn't attach to an active session */
 876                                         if (use_this_cptr->RefCount > 0)
 877                                         {
 878                                                 DeletePOP3Cfg(cptr);
 879                                                 Count->count--;
 880                                         }
 881                                         else
 882                                         {
 883                                                 long *QRnumber;
 884                                                 StrBufAppendBufPlain(use_this_cptr->rooms,
 885                                                                      qrbuf->QRname,
 886                                                                      -1, 0);
 887                                                 if (use_this_cptr->roomlist_parts == 1)
 888                                                 {
 889                                                         use_this_cptr->OtherQRnumbers = NewHash(1, lFlathash);
 890                                                 }
 891                                                 QRnumber = (long*)malloc(sizeof(long));
 892                                                 *QRnumber = qrbuf->QRnumber;
 893                                                 Put(use_this_cptr->OtherQRnumbers, LKEY(qrbuf->QRnumber), QRnumber, NULL);
 894                                                 use_this_cptr->roomlist_parts++;
 895                                         }
 896                                         citthread_mutex_unlock(&POP3QueueMutex);
 897                                         continue;
 898                                 }
 899                                 citthread_mutex_unlock(&RSSQueueMutex);
 900 #endif
 901
 902                                 citthread_mutex_lock(&POP3QueueMutex);
 903                                 Put(POP3FetchUrls, SKEY(cptr->Url), cptr, DeletePOP3Aggregator);
 904                                 citthread_mutex_unlock(&POP3QueueMutex);
 905
 906                         }
 907
 908                 }
 909
 910                 ///fclose(fp);
 911
 912         }
 913 }
 914
 915
 916 void pop3client_scan(void) {
 917         static time_t last_run = 0L;
 918         static int doing_pop3client = 0;
 919 ///     struct pop3aggr *pptr;
 920         time_t fastest_scan;
 921         HashPos *it;
 922         long len;
 923         const char *Key;
 924         void *vrptr;
 925         pop3aggr *cptr;
 926
 927         if (config.c_pop3_fastest < config.c_pop3_fetch)
 928                 fastest_scan = config.c_pop3_fastest;
 929         else
 930                 fastest_scan = config.c_pop3_fetch;
 931
 932         /*
 933          * Run POP3 aggregation no more frequently than once every n seconds
 934          */
 935         if ( (time(NULL) - last_run) < fastest_scan ) {
 936                 return;
 937         }
 938
 939         /*
 940          * This is a simple concurrency check to make sure only one pop3client run
 941          * is done at a time.  We could do this with a mutex, but since we
 942          * don't really require extremely fine granularity here, we'll do it
 943          * with a static variable instead.
 944          */
 945         if (doing_pop3client) return;
 946         doing_pop3client = 1;
 947
 948         CtdlLogPrintf(CTDL_DEBUG, "pop3client started\n");
 949         CtdlForEachRoom(pop3client_scan_room, NULL);
 950
 951
 952         citthread_mutex_lock(&POP3QueueMutex);
 953         it = GetNewHashPos(POP3FetchUrls, 0);
 954         while (GetNextHashPos(POP3FetchUrls, it, &len, &Key, &vrptr) &&
 955                (vrptr != NULL)) {
 956                 cptr = (pop3aggr *)vrptr;
 957                 if (cptr->RefCount == 0)
 958                         if (!pop3_do_fetching(cptr))
 959                                 DeletePOP3Aggregator(cptr);////TODO
 960         }
 961         DeleteHashPos(&it);
 962         citthread_mutex_unlock(&POP3QueueMutex);
 963
 964         CtdlLogPrintf(CTDL_DEBUG, "pop3client ended\n");
 965         last_run = time(NULL);
 966         doing_pop3client = 0;
 967 }
 968
 969
 970 void pop3_cleanup(void)
 971 {
 972         citthread_mutex_destroy(&POP3QueueMutex);
 973         DeleteHash(&POP3FetchUrls);
 974         DeleteHash(&POP3QueueRooms);
 975 }
 976
 977 CTDL_MODULE_INIT(pop3client)
 978 {
 979         if (!threading)
 980         {
 981                 citthread_mutex_init(&POP3QueueMutex, NULL);
 982                 POP3QueueRooms = NewHash(1, lFlathash);
 983                 POP3FetchUrls = NewHash(1, NULL);
 984                 CtdlRegisterSessionHook(pop3client_scan, EVT_TIMER);
 985                 CtdlRegisterCleanupHook(pop3_cleanup);
 986         }
 987         /* return our Subversion id for the Log */
 988         return "pop3client";
 989 }