citadel/modules/pop3client/serv_pop3client.c

   1 /*
   2  * Consolidate mail from remote POP3 accounts.
   3  *
   4  * Copyright (c) 2007-2011 by the citadel.org team
   5  *
   6  * This program is open source software; you can redistribute it and/or
   7  * modify it under the terms of the GNU General Public License as published
   8  * by the Free Software Foundation; either version 3 of the License, or
   9  * (at your option) any later version.
  10  *
  11  * This program is distributed in the hope that it will be useful,
  12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14  * GNU General Public License for more details.
  15  *
  16  * You should have received a copy of the GNU General Public License
  17  * along with this program; if not, write to the Free Software
  18  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
  19  */
  20
  21 #include <stdlib.h>
  22 #include <unistd.h>
  23 #include <stdio.h>
  24
  25 #if TIME_WITH_SYS_TIME
  26 # include <sys/time.h>
  27 # include <time.h>
  28 #else
  29 # if HAVE_SYS_TIME_H
  30 #  include <sys/time.h>
  31 # else
  32 #  include <time.h>
  33 # endif
  34 #endif
  35
  36 #include <ctype.h>
  37 #include <string.h>
  38 #include <errno.h>
  39 #include <sys/types.h>
  40 #include <sys/stat.h>
  41 #include <libcitadel.h>
  42 #include "citadel.h"
  43 #include "server.h"
  44 #include "citserver.h"
  45 #include "support.h"
  46 #include "config.h"
  47 #include "ctdl_module.h"
  48 #include "clientsocket.h"
  49 #include "msgbase.h"
  50 #include "internet_addressing.h"
  51 #include "database.h"
  52 #include "citadel_dirs.h"
  53 #include "event_client.h"
  54
  55
  56 #define POP3C_OK (strncasecmp(ChrPtr(RecvMsg->IO.IOBuf), "+OK", 3) == 0)
  57
  58 #define POP3C_DBG_SEND()                                        \
  59         syslog(LOG_DEBUG,                                       \
  60                "POP3 client[%ld]: > %s\n",                      \
  61                RecvMsg->n, ChrPtr(RecvMsg->IO.SendBuf.Buf))
  62
  63 #define POP3C_DBG_READ()                                \
  64         syslog(LOG_DEBUG,                               \
  65                "POP3 client[%ld]: < %s\n",              \
  66                RecvMsg->n,                              \
  67                ChrPtr(RecvMsg->IO.IOBuf))
  68
  69
  70 struct CitContext pop3_client_CC;
  71
  72 pthread_mutex_t POP3QueueMutex; /* locks the access to the following vars: */
  73 HashList *POP3QueueRooms = NULL;
  74 HashList *POP3FetchUrls = NULL;
  75
  76 typedef struct pop3aggr pop3aggr;
  77 typedef eNextState(*Pop3ClientHandler)(pop3aggr* RecvMsg);
  78
  79 eNextState POP3_C_Shutdown(AsyncIO *IO);
  80 eNextState POP3_C_Timeout(AsyncIO *IO);
  81 eNextState POP3_C_ConnFail(AsyncIO *IO);
  82 eNextState POP3_C_DNSFail(AsyncIO *IO);
  83 eNextState POP3_C_DispatchReadDone(AsyncIO *IO);
  84 eNextState POP3_C_DispatchWriteDone(AsyncIO *IO);
  85 eNextState POP3_C_Terminate(AsyncIO *IO);
  86 eReadState POP3_C_ReadServerStatus(AsyncIO *IO);
  87 eNextState POP3_C_ReAttachToFetchMessages(AsyncIO *IO);
  88
  89 typedef struct __pop3_room_counter {
  90         int count;
  91         long QRnumber;
  92 }pop3_room_counter;
  93
  94 typedef enum ePOP3_C_States {
  95         ReadGreeting,
  96         GetUserState,
  97         GetPassState,
  98         GetListCommandState,
  99         GetListOneLine,
 100         GetOneMessageIDState,
 101         ReadMessageBodyFollowing,
 102         ReadMessageBody,
 103         GetDeleteState,
 104         ReadQuitState,
 105         POP3C_MaxRead
 106 }ePOP3_C_States;
 107
 108
 109 typedef struct _FetchItem {
 110         long MSGID;
 111         long MSGSize;
 112         StrBuf *MsgUIDL;
 113         StrBuf *MsgUID;
 114         int NeedFetch;
 115         struct CtdlMessage *Msg;
 116 } FetchItem;
 117
 118 void HfreeFetchItem(void *vItem)
 119 {
 120         FetchItem *Item = (FetchItem*) vItem;
 121         FreeStrBuf(&Item->MsgUIDL);
 122         FreeStrBuf(&Item->MsgUID);
 123         free(Item);
 124 }
 125
 126 struct pop3aggr {
 127         AsyncIO  IO;
 128
 129         long n;
 130         long RefCount;
 131         DNSQueryParts HostLookup;
 132
 133         long             QRnumber;
 134         HashList        *OtherQRnumbers;
 135
 136         StrBuf          *Url;
 137         StrBuf *pop3user;
 138         StrBuf *pop3pass;
 139         StrBuf *RoomName; // TODO: fill me
 140         int keep;
 141         time_t interval;
 142         ePOP3_C_States State;
 143         HashList *MsgNumbers;
 144         HashPos *Pos;
 145         FetchItem *CurrMsg;
 146 };
 147
 148 void DeletePOP3Aggregator(void *vptr)
 149 {
 150         pop3aggr *ptr = vptr;
 151         DeleteHashPos(&ptr->Pos);
 152         DeleteHash(&ptr->MsgNumbers);
 153 //      FreeStrBuf(&ptr->rooms);
 154         FreeStrBuf(&ptr->pop3user);
 155         FreeStrBuf(&ptr->pop3pass);
 156         FreeStrBuf(&ptr->RoomName);
 157         FreeURL(&ptr->IO.ConnectMe);
 158         FreeStrBuf(&ptr->Url);
 159         FreeStrBuf(&ptr->IO.IOBuf);
 160         FreeStrBuf(&ptr->IO.SendBuf.Buf);
 161         FreeStrBuf(&ptr->IO.RecvBuf.Buf);
 162         DeleteAsyncMsg(&ptr->IO.ReadMsg);
 163         ((struct CitContext*)ptr->IO.CitContext)->state = CON_IDLE;
 164         ((struct CitContext*)ptr->IO.CitContext)->kill_me = 1;
 165         FreeAsyncIOContents(&ptr->IO);
 166         free(ptr);
 167 }
 168
 169 eNextState FinalizePOP3AggrRun(AsyncIO *IO)
 170 {
 171         HashPos  *It;
 172         pop3aggr *cptr = (pop3aggr *)IO->Data;
 173
 174         syslog(LOG_DEBUG, "Terminating Aggregator; bye.\n");
 175
 176         It = GetNewHashPos(POP3FetchUrls, 0);
 177         pthread_mutex_lock(&POP3QueueMutex);
 178         {
 179                 if (GetHashPosFromKey(POP3FetchUrls, SKEY(cptr->Url), It))
 180                         DeleteEntryFromHash(POP3FetchUrls, It);
 181         }
 182         pthread_mutex_unlock(&POP3QueueMutex);
 183         DeleteHashPos(&It);
 184         return eAbort;
 185 }
 186
 187 eNextState FailAggregationRun(AsyncIO *IO)
 188 {
 189         return eAbort;
 190 }
 191
 192 eNextState POP3C_ReadGreeting(pop3aggr *RecvMsg)
 193 {
 194         POP3C_DBG_READ();
 195         /* Read the server greeting */
 196         if (!POP3C_OK) return eTerminateConnection;
 197         else return eSendReply;
 198 }
 199
 200 eNextState POP3C_SendUser(pop3aggr *RecvMsg)
 201 {
 202         /* Identify ourselves.  NOTE: we have to append a CR to each command.
 203          *  The LF will automatically be appended by sock_puts().  Believe it
 204          * or not, leaving out the CR will cause problems if the server happens
 205          * to be Exchange, which is so b0rken it actually barfs on
 206          * LF-terminated newlines.
 207          */
 208         StrBufPrintf(RecvMsg->IO.SendBuf.Buf,
 209                      "USER %s\r\n", ChrPtr(RecvMsg->pop3user));
 210         POP3C_DBG_SEND();
 211         return eReadMessage;
 212 }
 213
 214 eNextState POP3C_GetUserState(pop3aggr *RecvMsg)
 215 {
 216         POP3C_DBG_READ();
 217         if (!POP3C_OK) return eTerminateConnection;
 218         else return eSendReply;
 219 }
 220
 221 eNextState POP3C_SendPassword(pop3aggr *RecvMsg)
 222 {
 223         /* Password */
 224         StrBufPrintf(RecvMsg->IO.SendBuf.Buf,
 225                      "PASS %s\r\n", ChrPtr(RecvMsg->pop3pass));
 226         syslog(LOG_DEBUG, "<PASS <password>\n");
 227 //      POP3C_DBG_SEND(); No, we won't write the passvoid to syslog...
 228         return eReadMessage;
 229 }
 230
 231 eNextState POP3C_GetPassState(pop3aggr *RecvMsg)
 232 {
 233         POP3C_DBG_READ();
 234         if (!POP3C_OK) return eTerminateConnection;
 235         else return eSendReply;
 236 }
 237
 238 eNextState POP3C_SendListCommand(pop3aggr *RecvMsg)
 239 {
 240         /* Get the list of messages */
 241         StrBufPlain(RecvMsg->IO.SendBuf.Buf, HKEY("LIST\r\n"));
 242         POP3C_DBG_SEND();
 243         return eReadMessage;
 244 }
 245
 246 eNextState POP3C_GetListCommandState(pop3aggr *RecvMsg)
 247 {
 248         POP3C_DBG_READ();
 249         if (!POP3C_OK) return eTerminateConnection;
 250         RecvMsg->MsgNumbers = NewHash(1, NULL);
 251         RecvMsg->State++;
 252         return eReadMore;
 253 }
 254
 255
 256 eNextState POP3C_GetListOneLine(pop3aggr *RecvMsg)
 257 {
 258 #if 0
 259         int rc;
 260 #endif
 261         const char *pch;
 262         FetchItem *OneMsg = NULL;
 263         POP3C_DBG_READ();
 264
 265         if ((StrLength(RecvMsg->IO.IOBuf) == 1) &&
 266             (ChrPtr(RecvMsg->IO.IOBuf)[0] == '.'))
 267         {
 268                 if (GetCount(RecvMsg->MsgNumbers) == 0)
 269                 {
 270                         ////    RecvMsg->Sate = ReadQuitState;
 271                 }
 272                 else
 273                 {
 274                         RecvMsg->Pos = GetNewHashPos(RecvMsg->MsgNumbers, 0);
 275                 }
 276                 return eSendReply;
 277
 278         }
 279         OneMsg = (FetchItem*) malloc(sizeof(FetchItem));
 280         memset(OneMsg, 0, sizeof(FetchItem));
 281         OneMsg->MSGID = atol(ChrPtr(RecvMsg->IO.IOBuf));
 282
 283         pch = strchr(ChrPtr(RecvMsg->IO.IOBuf), ' ');
 284         if (pch != NULL)
 285         {
 286                 OneMsg->MSGSize = atol(pch + 1);
 287         }
 288 #if 0
 289         rc = TestValidateHash(RecvMsg->MsgNumbers);
 290         if (rc != 0)
 291                 syslog(LOG_DEBUG, "Hash Invalid: %d\n", rc);
 292 #endif
 293
 294         Put(RecvMsg->MsgNumbers, LKEY(OneMsg->MSGID), OneMsg, HfreeFetchItem);
 295 #if 0
 296         rc = TestValidateHash(RecvMsg->MsgNumbers);
 297         if (rc != 0)
 298                 syslog(LOG_DEBUG, "Hash Invalid: %d\n", rc);
 299 #endif
 300         //RecvMsg->State --; /* read next Line */
 301         return eReadMore;
 302 }
 303
 304 eNextState POP3_FetchNetworkUsetableEntry(AsyncIO *IO)
 305 {
 306         long HKLen;
 307         const char *HKey;
 308         void *vData;
 309         struct cdbdata *cdbut;
 310         pop3aggr *RecvMsg = (pop3aggr *) IO->Data;
 311
 312         if(GetNextHashPos(RecvMsg->MsgNumbers,
 313                           RecvMsg->Pos,
 314                           &HKLen,
 315                           &HKey,
 316                           &vData))
 317         {
 318                 struct UseTable ut;
 319                 if (server_shutting_down)
 320                         return eAbort;
 321
 322                 RecvMsg->CurrMsg = (FetchItem*) vData;
 323                 syslog(LOG_DEBUG,
 324                        "CHECKING: whether %s has already been seen: ",
 325                        ChrPtr(RecvMsg->CurrMsg->MsgUID));
 326
 327                 /* Find out if we've already seen this item */
 328                 safestrncpy(ut.ut_msgid,
 329                             ChrPtr(RecvMsg->CurrMsg->MsgUID),
 330                             sizeof(ut.ut_msgid));
 331                 ut.ut_timestamp = time(NULL);/// TODO: libev timestamp!
 332
 333                 cdbut = cdb_fetch(CDB_USETABLE, SKEY(RecvMsg->CurrMsg->MsgUID));
 334                 if (cdbut != NULL) {
 335                         /* Item has already been seen */
 336                         syslog(LOG_DEBUG, "YES\n");
 337                         cdb_free(cdbut);
 338
 339                         /* rewrite the record anyway, to update the timestamp */
 340                         cdb_store(CDB_USETABLE,
 341                                   SKEY(RecvMsg->CurrMsg->MsgUID),
 342                                   &ut, sizeof(struct UseTable) );
 343                         RecvMsg->CurrMsg->NeedFetch = 0; ////TODO0;
 344                 }
 345                 else
 346                 {
 347                         syslog(LOG_DEBUG, "NO\n");
 348                         RecvMsg->CurrMsg->NeedFetch = 1;
 349                 }
 350                 return NextDBOperation(&RecvMsg->IO,
 351                                        POP3_FetchNetworkUsetableEntry);
 352         }
 353         else
 354         {
 355                 /* ok, now we know them all,
 356                  * continue with reading the actual messages. */
 357                 DeleteHashPos(&RecvMsg->Pos);
 358
 359                 return QueueEventContext(IO, POP3_C_ReAttachToFetchMessages);
 360         }
 361 }
 362
 363 eNextState POP3C_GetOneMessagID(pop3aggr *RecvMsg)
 364 {
 365         long HKLen;
 366         const char *HKey;
 367         void *vData;
 368
 369 #if 0
 370         int rc;
 371         rc = TestValidateHash(RecvMsg->MsgNumbers);
 372         if (rc != 0)
 373                 syslog(LOG_DEBUG, "Hash Invalid: %d\n", rc);
 374 #endif
 375         if(GetNextHashPos(RecvMsg->MsgNumbers,
 376                           RecvMsg->Pos,
 377                           &HKLen, &HKey,
 378                           &vData))
 379         {
 380                 RecvMsg->CurrMsg = (FetchItem*) vData;
 381                 /* Find out the UIDL of the message,
 382                  * to determine whether we've already downloaded it */
 383                 StrBufPrintf(RecvMsg->IO.SendBuf.Buf,
 384                              "UIDL %ld\r\n", RecvMsg->CurrMsg->MSGID);
 385                 POP3C_DBG_SEND();
 386         }
 387         else
 388         {
 389             RecvMsg->State++;
 390                 DeleteHashPos(&RecvMsg->Pos);
 391                 /// done receiving uidls.. start looking them up now.
 392                 RecvMsg->Pos = GetNewHashPos(RecvMsg->MsgNumbers, 0);
 393                 return QueueDBOperation(&RecvMsg->IO,
 394                                         POP3_FetchNetworkUsetableEntry);
 395         }
 396         return eReadMore; /* TODO */
 397 }
 398
 399 eNextState POP3C_GetOneMessageIDState(pop3aggr *RecvMsg)
 400 {
 401 #if 0
 402         int rc;
 403         rc = TestValidateHash(RecvMsg->MsgNumbers);
 404         if (rc != 0)
 405                 syslog(LOG_DEBUG, "Hash Invalid: %d\n", rc);
 406 #endif
 407
 408         POP3C_DBG_READ();
 409         if (!POP3C_OK) return eTerminateConnection;
 410         RecvMsg->CurrMsg->MsgUIDL =
 411                 NewStrBufPlain(NULL, StrLength(RecvMsg->IO.IOBuf));
 412         RecvMsg->CurrMsg->MsgUID =
 413                 NewStrBufPlain(NULL, StrLength(RecvMsg->IO.IOBuf) * 2);
 414
 415         StrBufExtract_token(RecvMsg->CurrMsg->MsgUIDL,
 416                             RecvMsg->IO.IOBuf, 2, ' ');
 417
 418         StrBufPrintf(RecvMsg->CurrMsg->MsgUID,
 419                      "pop3/%s/%s:%s@%s",
 420                      ChrPtr(RecvMsg->RoomName),
 421                      ChrPtr(RecvMsg->CurrMsg->MsgUIDL),
 422                      RecvMsg->IO.ConnectMe->User,
 423                      RecvMsg->IO.ConnectMe->Host);
 424         RecvMsg->State --;
 425         return eSendReply;
 426 }
 427
 428
 429 eNextState POP3C_SendGetOneMsg(pop3aggr *RecvMsg)
 430 {
 431         long HKLen;
 432         const char *HKey;
 433         void *vData;
 434
 435         RecvMsg->CurrMsg = NULL;
 436         while (GetNextHashPos(RecvMsg->MsgNumbers,
 437                               RecvMsg->Pos,
 438                               &HKLen, &HKey,
 439                               &vData) &&
 440                (RecvMsg->CurrMsg = (FetchItem*) vData,
 441                 RecvMsg->CurrMsg->NeedFetch == 0))
 442         {}
 443
 444         if ((RecvMsg->CurrMsg != NULL ) && (RecvMsg->CurrMsg->NeedFetch == 1))
 445         {
 446                 /* Message has not been seen.
 447                  * Tell the server to fetch the message... */
 448                 StrBufPrintf(RecvMsg->IO.SendBuf.Buf,
 449                              "RETR %ld\r\n", RecvMsg->CurrMsg->MSGID);
 450                 POP3C_DBG_SEND();
 451                 return eReadMessage;
 452         }
 453         else {
 454                 RecvMsg->State = ReadQuitState;
 455                 return POP3_C_DispatchWriteDone(&RecvMsg->IO);
 456         }
 457 }
 458
 459
 460 eNextState POP3C_ReadMessageBodyFollowing(pop3aggr *RecvMsg)
 461 {
 462         POP3C_DBG_READ();
 463         if (!POP3C_OK) return eTerminateConnection;
 464         RecvMsg->IO.ReadMsg = NewAsyncMsg(HKEY("."),
 465                                           RecvMsg->CurrMsg->MSGSize,
 466                                           config.c_maxmsglen,
 467                                           NULL, -1,
 468                                           1);
 469
 470         return eReadPayload;
 471 }
 472
 473
 474 eNextState POP3C_StoreMsgRead(AsyncIO *IO)
 475 {
 476         pop3aggr *RecvMsg = (pop3aggr *) IO->Data;
 477         struct UseTable ut;
 478
 479         syslog(LOG_DEBUG,
 480                "MARKING: %s as seen: ",
 481                ChrPtr(RecvMsg->CurrMsg->MsgUID));
 482
 483         safestrncpy(ut.ut_msgid,
 484                     ChrPtr(RecvMsg->CurrMsg->MsgUID),
 485                     sizeof(ut.ut_msgid));
 486         ut.ut_timestamp = time(NULL); /* TODO: use libev time */
 487         cdb_store(CDB_USETABLE,
 488                   ChrPtr(RecvMsg->CurrMsg->MsgUID),
 489                   StrLength(RecvMsg->CurrMsg->MsgUID),
 490                   &ut,
 491                   sizeof(struct UseTable) );
 492
 493         return QueueEventContext(&RecvMsg->IO, POP3_C_ReAttachToFetchMessages);
 494 }
 495 eNextState POP3C_SaveMsg(AsyncIO *IO)
 496 {
 497         long msgnum;
 498         pop3aggr *RecvMsg = (pop3aggr *) IO->Data;
 499
 500         /* Do Something With It (tm) */
 501         msgnum = CtdlSubmitMsg(RecvMsg->CurrMsg->Msg,
 502                                NULL,
 503                                ChrPtr(RecvMsg->RoomName),
 504                                0);
 505         if (msgnum > 0L)
 506         {
 507                 /* Message has been committed to the store
 508                  * write the uidl to the use table
 509                  * so we don't fetch this message again
 510                  */
 511         }
 512         CtdlFreeMessage(RecvMsg->CurrMsg->Msg);
 513
 514         return NextDBOperation(&RecvMsg->IO, POP3C_StoreMsgRead);
 515 }
 516
 517 eNextState POP3C_ReadMessageBody(pop3aggr *RecvMsg)
 518 {
 519         syslog(LOG_DEBUG, "Converting message...\n");
 520         RecvMsg->CurrMsg->Msg =
 521                 convert_internet_message_buf(&RecvMsg->IO.ReadMsg->MsgBuf);
 522
 523         return QueueDBOperation(&RecvMsg->IO, POP3C_SaveMsg);
 524 }
 525
 526 eNextState POP3C_SendDelete(pop3aggr *RecvMsg)
 527 {
 528         if (!RecvMsg->keep) {
 529                 StrBufPrintf(RecvMsg->IO.SendBuf.Buf,
 530                              "DELE %ld\r\n", RecvMsg->CurrMsg->MSGID);
 531                 POP3C_DBG_SEND();
 532                 return eReadMessage;
 533         }
 534         else {
 535                 RecvMsg->State = ReadMessageBodyFollowing;
 536                 return POP3_C_DispatchWriteDone(&RecvMsg->IO);
 537         }
 538 }
 539 eNextState POP3C_ReadDeleteState(pop3aggr *RecvMsg)
 540 {
 541         POP3C_DBG_READ();
 542         RecvMsg->State = GetOneMessageIDState;
 543         return eReadMessage;
 544 }
 545
 546 eNextState POP3C_SendQuit(pop3aggr *RecvMsg)
 547 {
 548         /* Log out */
 549         StrBufPlain(RecvMsg->IO.SendBuf.Buf,
 550                     HKEY("QUIT\r\n3)"));
 551         POP3C_DBG_SEND();
 552         return eReadMessage;
 553 }
 554
 555
 556 eNextState POP3C_ReadQuitState(pop3aggr *RecvMsg)
 557 {
 558         POP3C_DBG_READ();
 559         return eTerminateConnection;
 560 }
 561
 562 const long POP3_C_ConnTimeout = 1000;
 563 const long DefaultPOP3Port = 110;
 564
 565 Pop3ClientHandler POP3C_ReadHandlers[] = {
 566         POP3C_ReadGreeting,
 567         POP3C_GetUserState,
 568         POP3C_GetPassState,
 569         POP3C_GetListCommandState,
 570         POP3C_GetListOneLine,
 571         POP3C_GetOneMessageIDState,
 572         POP3C_ReadMessageBodyFollowing,
 573         POP3C_ReadMessageBody,
 574         POP3C_ReadDeleteState,
 575         POP3C_ReadQuitState,
 576 };
 577
 578 const long POP3_C_SendTimeouts[POP3C_MaxRead] = {
 579         100,
 580         100,
 581         100,
 582         100,
 583         100,
 584         100,
 585         100,
 586         100
 587 };
 588 const ConstStr POP3C_ReadErrors[POP3C_MaxRead] = {
 589         {HKEY("Connection broken during ")},
 590         {HKEY("Connection broken during ")},
 591         {HKEY("Connection broken during ")},
 592         {HKEY("Connection broken during ")},
 593         {HKEY("Connection broken during ")},
 594         {HKEY("Connection broken during ")},
 595         {HKEY("Connection broken during ")},
 596         {HKEY("Connection broken during ")}
 597 };
 598
 599 Pop3ClientHandler POP3C_SendHandlers[] = {
 600         NULL, /* we don't send a greeting */
 601         POP3C_SendUser,
 602         POP3C_SendPassword,
 603         POP3C_SendListCommand,
 604         NULL,
 605         POP3C_GetOneMessagID,
 606         POP3C_SendGetOneMsg,
 607         NULL,
 608         POP3C_SendDelete,
 609         POP3C_SendQuit
 610 };
 611
 612 const long POP3_C_ReadTimeouts[] = {
 613         100,
 614         100,
 615         100,
 616         100,
 617         100,
 618         100,
 619         100,
 620         100,
 621         100,
 622         100
 623 };
 624 /*****************************************************************************/
 625 /*                     POP3 CLIENT DISPATCHER                                */
 626 /*****************************************************************************/
 627
 628 void POP3SetTimeout(eNextState NextTCPState, pop3aggr *pMsg)
 629 {
 630         double Timeout = 0.0;
 631
 632         syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
 633
 634         switch (NextTCPState) {
 635         case eSendFile:
 636         case eSendReply:
 637         case eSendMore:
 638                 Timeout = POP3_C_SendTimeouts[pMsg->State];
 639 /*
 640   if (pMsg->State == eDATABody) {
 641   / * if we're sending a huge message, we need more time. * /
 642   Timeout += StrLength(pMsg->msgtext) / 1024;
 643   }
 644 */
 645                 break;
 646         case eReadFile:
 647         case eReadMessage:
 648                 Timeout = POP3_C_ReadTimeouts[pMsg->State];
 649 /*
 650   if (pMsg->State == eDATATerminateBody) {
 651   / *
 652   * some mailservers take a nap before accepting the message
 653   * content inspection and such.
 654   * /
 655   Timeout += StrLength(pMsg->msgtext) / 1024;
 656   }
 657 */
 658                 break;
 659         case eReadPayload:
 660                 Timeout = 100000;
 661                 /* TODO!!! */
 662                 break;
 663         case eSendDNSQuery:
 664         case eReadDNSReply:
 665         case eConnect:
 666         case eTerminateConnection:
 667         case eDBQuery:
 668         case eAbort:
 669         case eReadMore://// TODO
 670                 return;
 671         }
 672         SetNextTimeout(&pMsg->IO, Timeout);
 673 }
 674 eNextState POP3_C_DispatchReadDone(AsyncIO *IO)
 675 {
 676         syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
 677         pop3aggr *pMsg = IO->Data;
 678         eNextState rc;
 679
 680         rc = POP3C_ReadHandlers[pMsg->State](pMsg);
 681         if (rc != eReadMore)
 682             pMsg->State++;
 683         POP3SetTimeout(rc, pMsg);
 684         return rc;
 685 }
 686 eNextState POP3_C_DispatchWriteDone(AsyncIO *IO)
 687 {
 688         syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
 689         pop3aggr *pMsg = IO->Data;
 690         eNextState rc;
 691
 692         rc = POP3C_SendHandlers[pMsg->State](pMsg);
 693         POP3SetTimeout(rc, pMsg);
 694         return rc;
 695 }
 696
 697
 698 /*****************************************************************************/
 699 /*                     POP3 CLIENT ERROR CATCHERS                            */
 700 /*****************************************************************************/
 701 eNextState POP3_C_Terminate(AsyncIO *IO)
 702 {
 703 ///     pop3aggr *pMsg = (pop3aggr *)IO->Data;
 704
 705         syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
 706         FinalizePOP3AggrRun(IO);
 707         return eAbort;
 708 }
 709 eNextState POP3_C_TerminateDB(AsyncIO *IO)
 710 {
 711 ///     pop3aggr *pMsg = (pop3aggr *)IO->Data;
 712
 713         syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
 714         FinalizePOP3AggrRun(IO);
 715         return eAbort;
 716 }
 717 eNextState POP3_C_Timeout(AsyncIO *IO)
 718 {
 719         pop3aggr *pMsg = IO->Data;
 720
 721         syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
 722         StrBufPlain(IO->ErrMsg, CKEY(POP3C_ReadErrors[pMsg->State]));
 723         return FailAggregationRun(IO);
 724 }
 725 eNextState POP3_C_ConnFail(AsyncIO *IO)
 726 {
 727         pop3aggr *pMsg = (pop3aggr *)IO->Data;
 728
 729         syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
 730         StrBufPlain(IO->ErrMsg, CKEY(POP3C_ReadErrors[pMsg->State]));
 731         return FailAggregationRun(IO);
 732 }
 733 eNextState POP3_C_DNSFail(AsyncIO *IO)
 734 {
 735         pop3aggr *pMsg = (pop3aggr *)IO->Data;
 736
 737         syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
 738         StrBufPlain(IO->ErrMsg, CKEY(POP3C_ReadErrors[pMsg->State]));
 739         return FailAggregationRun(IO);
 740 }
 741 eNextState POP3_C_Shutdown(AsyncIO *IO)
 742 {
 743         syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
 744 ////    pop3aggr *pMsg = IO->Data;
 745
 746 ////pMsg->MyQEntry->Status = 3;
 747 ///StrBufPlain(pMsg->MyQEntry->StatusMessage, HKEY("server shutdown during message retrieval."));
 748         FinalizePOP3AggrRun(IO);
 749         return eAbort;
 750 }
 751
 752
 753 /**
 754  * @brief lineread Handler; understands when to read more POP3 lines,
 755  *   and when this is a one-lined reply.
 756  */
 757 eReadState POP3_C_ReadServerStatus(AsyncIO *IO)
 758 {
 759         eReadState Finished = eBufferNotEmpty;
 760
 761         switch (IO->NextState) {
 762         case eSendDNSQuery:
 763         case eReadDNSReply:
 764         case eDBQuery:
 765         case eConnect:
 766         case eTerminateConnection:
 767         case eAbort:
 768                 Finished = eReadFail;
 769                 break;
 770         case eSendFile:
 771         case eSendReply:
 772         case eSendMore:
 773         case eReadMore:
 774         case eReadMessage:
 775                 Finished = StrBufChunkSipLine(IO->IOBuf, &IO->RecvBuf);
 776                 break;
 777         case eReadFile:
 778         case eReadPayload:
 779                 Finished = CtdlReadMessageBodyAsync(IO);
 780                 break;
 781         }
 782         return Finished;
 783 }
 784
 785 /*****************************************************************************
 786  * So we connect our Server IP here.                                         *
 787  *****************************************************************************/
 788 eNextState POP3_C_ReAttachToFetchMessages(AsyncIO *IO)
 789 {
 790         pop3aggr *cpptr = IO->Data;
 791
 792         syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
 793 ////??? cpptr->State ++;
 794         if (cpptr->Pos == NULL)
 795                 cpptr->Pos = GetNewHashPos(cpptr->MsgNumbers, 0);
 796
 797         POP3_C_DispatchWriteDone(IO);
 798         ReAttachIO(IO, cpptr, 0);
 799         IO->NextState = eReadMessage;
 800         return IO->NextState;
 801 }
 802
 803 eNextState pop3_connect_ip(AsyncIO *IO)
 804 {
 805         syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
 806
 807         return EvConnectSock(IO,
 808                              POP3_C_ConnTimeout,
 809                              POP3_C_ReadTimeouts[0],
 810                              1);
 811 }
 812
 813 eNextState pop3_get_one_host_ip_done(AsyncIO *IO)
 814 {
 815         pop3aggr *cpptr = IO->Data;
 816         struct hostent *hostent;
 817
 818         QueryCbDone(IO);
 819
 820         hostent = cpptr->HostLookup.VParsedDNSReply;
 821         if ((cpptr->HostLookup.DNSStatus == ARES_SUCCESS) &&
 822             (hostent != NULL) ) {
 823                 memset(&cpptr->IO.ConnectMe->Addr, 0, sizeof(struct in6_addr));
 824                 if (cpptr->IO.ConnectMe->IPv6) {
 825                         memcpy(&cpptr->IO.ConnectMe->Addr.sin6_addr.s6_addr,
 826                                &hostent->h_addr_list[0],
 827                                sizeof(struct in6_addr));
 828
 829                         cpptr->IO.ConnectMe->Addr.sin6_family =
 830                                 hostent->h_addrtype;
 831                         cpptr->IO.ConnectMe->Addr.sin6_port   =
 832                                 htons(DefaultPOP3Port);
 833                 }
 834                 else {
 835                         struct sockaddr_in *addr =
 836                                 (struct sockaddr_in*)
 837                                 &cpptr->IO.ConnectMe->Addr;
 838
 839                         memcpy(&addr->sin_addr.s_addr,
 840                                hostent->h_addr_list[0],
 841                                sizeof(uint32_t));
 842
 843                         addr->sin_family = hostent->h_addrtype;
 844                         addr->sin_port   = htons(DefaultPOP3Port);
 845                 }
 846                 return pop3_connect_ip(IO);
 847         }
 848         else
 849                 return eAbort;
 850 }
 851
 852 eNextState pop3_get_one_host_ip(AsyncIO *IO)
 853 {
 854         pop3aggr *cpptr = IO->Data;
 855
 856         syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
 857
 858         syslog(LOG_DEBUG,
 859                       "POP3 client[%ld]: looking up %s-Record %s : %d ...\n",
 860                       cpptr->n,
 861                       (cpptr->IO.ConnectMe->IPv6)? "aaaa": "a",
 862                       cpptr->IO.ConnectMe->Host,
 863                       cpptr->IO.ConnectMe->Port);
 864
 865         QueueQuery((cpptr->IO.ConnectMe->IPv6)? ns_t_aaaa : ns_t_a,
 866                    cpptr->IO.ConnectMe->Host,
 867                    &cpptr->IO,
 868                    &cpptr->HostLookup,
 869                    pop3_get_one_host_ip_done);
 870         IO->NextState = eReadDNSReply;
 871         return IO->NextState;
 872 }
 873
 874
 875
 876 int pop3_do_fetching(pop3aggr *cpptr)
 877 {
 878         InitIOStruct(&cpptr->IO,
 879                      cpptr,
 880                      eReadMessage,
 881                      POP3_C_ReadServerStatus,
 882                      POP3_C_DNSFail,
 883                      POP3_C_DispatchWriteDone,
 884                      POP3_C_DispatchReadDone,
 885                      POP3_C_Terminate,
 886                      POP3_C_TerminateDB,
 887                      POP3_C_ConnFail,
 888                      POP3_C_Timeout,
 889                      POP3_C_Shutdown);
 890
 891         safestrncpy(((CitContext *)cpptr->IO.CitContext)->cs_host,
 892                     ChrPtr(cpptr->Url),
 893                     sizeof(((CitContext *)cpptr->IO.CitContext)->cs_host));
 894
 895         if (cpptr->IO.ConnectMe->IsIP) {
 896                 QueueEventContext(&cpptr->IO,
 897                                   pop3_connect_ip);
 898         }
 899         else {
 900                 QueueEventContext(&cpptr->IO,
 901                                   pop3_get_one_host_ip);
 902         }
 903         return 1;
 904 }
 905
 906 /*
 907  * Scan a room's netconfig to determine whether it requires POP3 aggregation
 908  */
 909 void pop3client_scan_room(struct ctdlroom *qrbuf, void *data)
 910 {
 911         StrBuf *CfgData;
 912         StrBuf *CfgType;
 913         StrBuf *Line;
 914
 915         struct stat statbuf;
 916         char filename[PATH_MAX];
 917         int  fd;
 918         int Done;
 919         void *vptr;
 920         const char *CfgPtr, *lPtr;
 921         const char *Err;
 922
 923 //      pop3_room_counter *Count = NULL;
 924 //      pop3aggr *cpptr;
 925
 926         pthread_mutex_lock(&POP3QueueMutex);
 927         if (GetHash(POP3QueueRooms, LKEY(qrbuf->QRnumber), &vptr))
 928         {
 929                 syslog(LOG_DEBUG,
 930                               "pop3client: [%ld] %s already in progress.\n",
 931                               qrbuf->QRnumber,
 932                               qrbuf->QRname);
 933                 pthread_mutex_unlock(&POP3QueueMutex);
 934         }
 935         pthread_mutex_unlock(&POP3QueueMutex);
 936
 937         if (server_shutting_down) return;
 938
 939         assoc_file_name(filename, sizeof filename, qrbuf, ctdl_netcfg_dir);
 940
 941         if (server_shutting_down)
 942                 return;
 943
 944         /* Only do net processing for rooms that have netconfigs */
 945         fd = open(filename, 0);
 946         if (fd <= 0) {
 947                 return;
 948         }
 949         if (server_shutting_down)
 950                 return;
 951         if (fstat(fd, &statbuf) == -1) {
 952                 syslog(LOG_DEBUG,
 953                        "ERROR: could not stat configfile '%s' - %s\n",
 954                        filename,
 955                        strerror(errno));
 956                 return;
 957         }
 958         if (server_shutting_down)
 959                 return;
 960         CfgData = NewStrBufPlain(NULL, statbuf.st_size + 1);
 961         if (StrBufReadBLOB(CfgData, &fd, 1, statbuf.st_size, &Err) < 0) {
 962                 close(fd);
 963                 FreeStrBuf(&CfgData);
 964                 syslog(LOG_DEBUG, "ERROR: reading config '%s' - %s<br>\n",
 965                               filename, strerror(errno));
 966                 return;
 967         }
 968         close(fd);
 969         if (server_shutting_down)
 970                 return;
 971
 972         CfgPtr = NULL;
 973         CfgType = NewStrBuf();
 974         Line = NewStrBufPlain(NULL, StrLength(CfgData));
 975         Done = 0;
 976
 977         while (!Done)
 978         {
 979                 Done = StrBufSipLine(Line, CfgData, &CfgPtr) == 0;
 980                 if (StrLength(Line) > 0)
 981                 {
 982                         lPtr = NULL;
 983                         StrBufExtract_NextToken(CfgType, Line, &lPtr, '|');
 984                         if (!strcasecmp("pop3client", ChrPtr(CfgType)))
 985                         {
 986                                 pop3aggr *cptr;
 987                                 StrBuf *Tmp;
 988 /*
 989                                 if (Count == NULL)
 990                                 {
 991                                 Count = malloc(sizeof(pop3_room_counter));
 992                                         Count->count = 0;
 993                                 }
 994                                 Count->count ++;
 995 */
 996                                 cptr = (pop3aggr *) malloc(sizeof(pop3aggr));
 997                                 memset(cptr, 0, sizeof(pop3aggr));
 998                                 ///TODO do we need this? cptr->roomlist_parts=1;
 999                                 cptr->RoomName =
1000                                         NewStrBufPlain(qrbuf->QRname, -1);
1001                                 cptr->pop3user =
1002                                         NewStrBufPlain(NULL, StrLength(Line));
1003                                 cptr->pop3pass =
1004                                         NewStrBufPlain(NULL, StrLength(Line));
1005                                 cptr->Url = NewStrBuf();
1006                                 Tmp = NewStrBuf();
1007
1008                                 StrBufExtract_NextToken(Tmp, Line, &lPtr, '|');
1009                                 StrBufExtract_NextToken(cptr->pop3user,
1010                                                         Line,
1011                                                         &lPtr,
1012                                                         '|');
1013
1014                                 StrBufExtract_NextToken(cptr->pop3pass,
1015                                                         Line,
1016                                                         &lPtr,
1017                                                         '|');
1018
1019                                 cptr->keep = StrBufExtractNext_long(Line,
1020                                                                     &lPtr,
1021                                                                     '|');
1022
1023                                 cptr->interval = StrBufExtractNext_long(Line,
1024                                                                         &lPtr,
1025                                                                         '|');
1026
1027                                 StrBufPrintf(cptr->Url, "pop3://%s:%s@%s/%s",
1028                                              ChrPtr(cptr->pop3user),
1029                                              ChrPtr(cptr->pop3pass),
1030                                              ChrPtr(Tmp),
1031                                              ChrPtr(cptr->RoomName));
1032                                 FreeStrBuf(&Tmp);
1033                                 ParseURL(&cptr->IO.ConnectMe, cptr->Url, 110);
1034
1035
1036 #if 0
1037 /* todo: we need to reunite the url to be shure. */
1038
1039                                 pthread_mutex_lock(&POP3ueueMutex);
1040                                 GetHash(POP3FetchUrls, SKEY(ptr->Url), &vptr);
1041                                 use_this_cptr = (pop3aggr *)vptr;
1042
1043                                 if (use_this_rncptr != NULL)
1044                                 {
1045                                         /* mustn't attach to an active session */
1046                                         if (use_this_cptr->RefCount > 0)
1047                                         {
1048                                                 DeletePOP3Cfg(cptr);
1049 ///                                             Count->count--;
1050                                         }
1051                                         else
1052                                         {
1053                                                 long *QRnumber;
1054                                                 StrBufAppendBufPlain(
1055                                                         use_this_cptr->rooms,
1056                                                         qrbuf->QRname,
1057                                                         -1, 0);
1058                                                 if (use_this_cptr->roomlist_parts == 1)
1059                                                 {
1060                                                         use_this_cptr->OtherQRnumbers
1061                                                                 = NewHash(1, lFlathash);
1062                                                 }
1063                                                 QRnumber = (long*)malloc(sizeof(long));
1064                                                 *QRnumber = qrbuf->QRnumber;
1065                                                 Put(use_this_cptr->OtherQRnumbers,
1066                                                     LKEY(qrbuf->QRnumber),
1067                                                     QRnumber,
1068                                                     NULL);
1069
1070                                                 use_this_cptr->roomlist_parts++;
1071                                         }
1072                                         pthread_mutex_unlock(&POP3QueueMutex);
1073                                         continue;
1074                                 }
1075                                 pthread_mutex_unlock(&RSSQueueMutex);
1076 #endif
1077
1078                                 pthread_mutex_lock(&POP3QueueMutex);
1079                                 Put(POP3FetchUrls,
1080                                     SKEY(cptr->Url),
1081                                     cptr,
1082                                     DeletePOP3Aggregator);
1083
1084                                 pthread_mutex_unlock(&POP3QueueMutex);
1085
1086                         }
1087
1088                 }
1089
1090                 ///fclose(fp);
1091
1092         }
1093         FreeStrBuf(&Line);
1094         FreeStrBuf(&CfgType);
1095         FreeStrBuf(&CfgData);
1096 }
1097
1098 static int doing_pop3client = 0;
1099
1100 void pop3client_scan(void) {
1101         static time_t last_run = 0L;
1102         time_t fastest_scan;
1103         HashPos *it;
1104         long len;
1105         const char *Key;
1106         void *vrptr;
1107         pop3aggr *cptr;
1108
1109         become_session(&pop3_client_CC);
1110
1111         if (config.c_pop3_fastest < config.c_pop3_fetch)
1112                 fastest_scan = config.c_pop3_fastest;
1113         else
1114                 fastest_scan = config.c_pop3_fetch;
1115
1116         /*
1117          * Run POP3 aggregation no more frequently than once every n seconds
1118          */
1119         if ( (time(NULL) - last_run) < fastest_scan ) {
1120                 return;
1121         }
1122
1123         /*
1124          * This is a simple concurrency check to make sure only one pop3client
1125          * run is done at a time.  We could do this with a mutex, but since we
1126          * don't really require extremely fine granularity here, we'll do it
1127          * with a static variable instead.
1128          */
1129         if (doing_pop3client) return;
1130         doing_pop3client = 1;
1131
1132         syslog(LOG_DEBUG, "pop3client started");
1133         CtdlForEachRoom(pop3client_scan_room, NULL);
1134
1135         pthread_mutex_lock(&POP3QueueMutex);
1136         it = GetNewHashPos(POP3FetchUrls, 0);
1137         while (!server_shutting_down &&
1138                GetNextHashPos(POP3FetchUrls, it, &len, &Key, &vrptr) &&
1139                (vrptr != NULL)) {
1140                 cptr = (pop3aggr *)vrptr;
1141                 if (cptr->RefCount == 0)
1142                         if (!pop3_do_fetching(cptr))
1143                                 DeletePOP3Aggregator(cptr);////TODO
1144
1145 /*
1146         if ((palist->interval && time(NULL) > (last_run + palist->interval))
1147                         || (time(NULL) > last_run + config.c_pop3_fetch))
1148                         pop3_do_fetching(palist->roomname, palist->pop3host,
1149                         palist->pop3user, palist->pop3pass, palist->keep);
1150                 pptr = palist;
1151                 palist = palist->next;
1152                 free(pptr);
1153 */
1154         }
1155         DeleteHashPos(&it);
1156         pthread_mutex_unlock(&POP3QueueMutex);
1157
1158         syslog(LOG_DEBUG, "pop3client ended");
1159         last_run = time(NULL);
1160         doing_pop3client = 0;
1161 }
1162
1163
1164 void pop3_cleanup(void)
1165 {
1166         /* citthread_mutex_destroy(&POP3QueueMutex); TODO */
1167         while (doing_pop3client != 0) ;
1168         DeleteHash(&POP3FetchUrls);
1169         DeleteHash(&POP3QueueRooms);
1170 }
1171
1172 CTDL_MODULE_INIT(pop3client)
1173 {
1174         if (!threading)
1175         {
1176                 CtdlFillSystemContext(&pop3_client_CC, "POP3aggr");
1177                 pthread_mutex_init(&POP3QueueMutex, NULL);
1178                 POP3QueueRooms = NewHash(1, lFlathash);
1179                 POP3FetchUrls = NewHash(1, NULL);
1180                 CtdlRegisterSessionHook(pop3client_scan, EVT_TIMER);
1181                 CtdlRegisterEVCleanupHook(pop3_cleanup);
1182         }
1183
1184         /* return our module id for the log */
1185         return "pop3client";
1186 }