citadel/modules/pop3client/serv_pop3client.c

   1 /*
   2  * Consolidate mail from remote POP3 accounts.
   3  *
   4  * Copyright (c) 2007-2011 by the citadel.org team
   5  *
   6  * This program is open source software; you can redistribute it and/or
   7  * modify it under the terms of the GNU General Public License as published
   8  * by the Free Software Foundation; either version 3 of the License, or
   9  * (at your option) any later version.
  10  *
  11  * This program is distributed in the hope that it will be useful,
  12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14  * GNU General Public License for more details.
  15  *
  16  * You should have received a copy of the GNU General Public License
  17  * along with this program; if not, write to the Free Software
  18  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
  19  */
  20
  21 #include <stdlib.h>
  22 #include <unistd.h>
  23 #include <stdio.h>
  24
  25 #if TIME_WITH_SYS_TIME
  26 # include <sys/time.h>
  27 # include <time.h>
  28 #else
  29 # if HAVE_SYS_TIME_H
  30 #  include <sys/time.h>
  31 # else
  32 #  include <time.h>
  33 # endif
  34 #endif
  35
  36 #include <ctype.h>
  37 #include <string.h>
  38 #include <errno.h>
  39 #include <sys/types.h>
  40 #include <sys/stat.h>
  41 #include <libcitadel.h>
  42 #include "citadel.h"
  43 #include "server.h"
  44 #include "citserver.h"
  45 #include "support.h"
  46 #include "config.h"
  47 #include "ctdl_module.h"
  48 #include "clientsocket.h"
  49 #include "msgbase.h"
  50 #include "internet_addressing.h"
  51 #include "database.h"
  52 #include "citadel_dirs.h"
  53 #include "event_client.h"
  54
  55
  56 #define POP3C_OK (strncasecmp(ChrPtr(RecvMsg->IO.IOBuf), "+OK", 3) == 0)
  57
  58 #define POP3C_DBG_SEND()                                        \
  59         syslog(LOG_DEBUG,                                       \
  60                "POP3 client[%ld]: > %s\n",                      \
  61                RecvMsg->n, ChrPtr(RecvMsg->IO.SendBuf.Buf))
  62
  63 #define POP3C_DBG_READ()                                \
  64         syslog(LOG_DEBUG,                               \
  65                "POP3 client[%ld]: < %s\n",              \
  66                RecvMsg->n,                              \
  67                ChrPtr(RecvMsg->IO.IOBuf))
  68
  69
  70 struct CitContext pop3_client_CC;
  71
  72 pthread_mutex_t POP3QueueMutex; /* locks the access to the following vars: */
  73 HashList *POP3QueueRooms = NULL;
  74 HashList *POP3FetchUrls = NULL;
  75
  76 typedef struct pop3aggr pop3aggr;
  77 typedef eNextState(*Pop3ClientHandler)(pop3aggr* RecvMsg);
  78
  79 eNextState POP3_C_Shutdown(AsyncIO *IO);
  80 eNextState POP3_C_Timeout(AsyncIO *IO);
  81 eNextState POP3_C_ConnFail(AsyncIO *IO);
  82 eNextState POP3_C_DNSFail(AsyncIO *IO);
  83 eNextState POP3_C_DispatchReadDone(AsyncIO *IO);
  84 eNextState POP3_C_DispatchWriteDone(AsyncIO *IO);
  85 eNextState POP3_C_Terminate(AsyncIO *IO);
  86 eReadState POP3_C_ReadServerStatus(AsyncIO *IO);
  87 eNextState POP3_C_ReAttachToFetchMessages(AsyncIO *IO);
  88
  89 typedef struct __pop3_room_counter {
  90         int count;
  91         long QRnumber;
  92 }pop3_room_counter;
  93
  94 typedef enum ePOP3_C_States {
  95         ReadGreeting,
  96         GetUserState,
  97         GetPassState,
  98         GetListCommandState,
  99         GetListOneLine,
 100         GetOneMessageIDState,
 101         ReadMessageBodyFollowing,
 102         ReadMessageBody,
 103         GetDeleteState,
 104         ReadQuitState,
 105         POP3C_MaxRead
 106 }ePOP3_C_States;
 107
 108
 109 typedef struct _FetchItem {
 110         long MSGID;
 111         long MSGSize;
 112         StrBuf *MsgUIDL;
 113         StrBuf *MsgUID;
 114         int NeedFetch;
 115         struct CtdlMessage *Msg;
 116 } FetchItem;
 117
 118 void HfreeFetchItem(void *vItem)
 119 {
 120         FetchItem *Item = (FetchItem*) vItem;
 121         FreeStrBuf(&Item->MsgUIDL);
 122         FreeStrBuf(&Item->MsgUID);
 123         free(Item);
 124 }
 125
 126 struct pop3aggr {
 127         AsyncIO  IO;
 128
 129         long n;
 130         long RefCount;
 131         DNSQueryParts HostLookup;
 132
 133         long             QRnumber;
 134         HashList        *OtherQRnumbers;
 135
 136         StrBuf          *Url;
 137         StrBuf *pop3user;
 138         StrBuf *pop3pass;
 139         StrBuf *RoomName; // TODO: fill me
 140         int keep;
 141         time_t interval;
 142         ePOP3_C_States State;
 143         HashList *MsgNumbers;
 144         HashPos *Pos;
 145         FetchItem *CurrMsg;
 146 };
 147
 148 void DeletePOP3Aggregator(void *vptr)
 149 {
 150         pop3aggr *ptr = vptr;
 151         DeleteHashPos(&ptr->Pos);
 152         DeleteHash(&ptr->MsgNumbers);
 153 //      FreeStrBuf(&ptr->rooms);
 154         FreeStrBuf(&ptr->pop3user);
 155         FreeStrBuf(&ptr->pop3pass);
 156         FreeStrBuf(&ptr->RoomName);
 157         FreeURL(&ptr->IO.ConnectMe);
 158         FreeStrBuf(&ptr->Url);
 159         FreeStrBuf(&ptr->IO.IOBuf);
 160         FreeStrBuf(&ptr->IO.SendBuf.Buf);
 161         FreeStrBuf(&ptr->IO.RecvBuf.Buf);
 162         DeleteAsyncMsg(&ptr->IO.ReadMsg);
 163         ((struct CitContext*)ptr->IO.CitContext)->state = CON_IDLE;
 164         ((struct CitContext*)ptr->IO.CitContext)->kill_me = 1;
 165         FreeAsyncIOContents(&ptr->IO);
 166         free(ptr);
 167 }
 168
 169 eNextState FinalizePOP3AggrRun(AsyncIO *IO)
 170 {
 171         HashPos  *It;
 172         pop3aggr *cptr = (pop3aggr *)IO->Data;
 173
 174         syslog(LOG_DEBUG, "Terminating Aggregator; bye.\n");
 175
 176         It = GetNewHashPos(POP3FetchUrls, 0);
 177         pthread_mutex_lock(&POP3QueueMutex);
 178         {
 179                 if (GetHashPosFromKey(POP3FetchUrls, SKEY(cptr->Url), It))
 180                         DeleteEntryFromHash(POP3FetchUrls, It);
 181         }
 182         pthread_mutex_unlock(&POP3QueueMutex);
 183         DeleteHashPos(&It);
 184         return eAbort;
 185 }
 186
 187 eNextState FailAggregationRun(AsyncIO *IO)
 188 {
 189         return eAbort;
 190 }
 191
 192 eNextState POP3C_ReadGreeting(pop3aggr *RecvMsg)
 193 {
 194         POP3C_DBG_READ();
 195         /* Read the server greeting */
 196         if (!POP3C_OK) return eTerminateConnection;
 197         else return eSendReply;
 198 }
 199
 200 eNextState POP3C_SendUser(pop3aggr *RecvMsg)
 201 {
 202         /* Identify ourselves.  NOTE: we have to append a CR to each command.
 203          *  The LF will automatically be appended by sock_puts().  Believe it
 204          * or not, leaving out the CR will cause problems if the server happens
 205          * to be Exchange, which is so b0rken it actually barfs on
 206          * LF-terminated newlines.
 207          */
 208         StrBufPrintf(RecvMsg->IO.SendBuf.Buf,
 209                      "USER %s\r\n", ChrPtr(RecvMsg->pop3user));
 210         POP3C_DBG_SEND();
 211         return eReadMessage;
 212 }
 213
 214 eNextState POP3C_GetUserState(pop3aggr *RecvMsg)
 215 {
 216         POP3C_DBG_READ();
 217         if (!POP3C_OK) return eTerminateConnection;
 218         else return eSendReply;
 219 }
 220
 221 eNextState POP3C_SendPassword(pop3aggr *RecvMsg)
 222 {
 223         /* Password */
 224         StrBufPrintf(RecvMsg->IO.SendBuf.Buf,
 225                      "PASS %s\r\n", ChrPtr(RecvMsg->pop3pass));
 226         syslog(LOG_DEBUG, "<PASS <password>\n");
 227 //      POP3C_DBG_SEND(); No, we won't write the passvoid to syslog...
 228         return eReadMessage;
 229 }
 230
 231 eNextState POP3C_GetPassState(pop3aggr *RecvMsg)
 232 {
 233         POP3C_DBG_READ();
 234         if (!POP3C_OK) return eTerminateConnection;
 235         else return eSendReply;
 236 }
 237
 238 eNextState POP3C_SendListCommand(pop3aggr *RecvMsg)
 239 {
 240         /* Get the list of messages */
 241         StrBufPlain(RecvMsg->IO.SendBuf.Buf, HKEY("LIST\r\n"));
 242         POP3C_DBG_SEND();
 243         return eReadMessage;
 244 }
 245
 246 eNextState POP3C_GetListCommandState(pop3aggr *RecvMsg)
 247 {
 248         POP3C_DBG_READ();
 249         if (!POP3C_OK) return eTerminateConnection;
 250         RecvMsg->MsgNumbers = NewHash(1, NULL);
 251         RecvMsg->State++;
 252         return eReadMore;
 253 }
 254
 255
 256 eNextState POP3C_GetListOneLine(pop3aggr *RecvMsg)
 257 {
 258 #if 0
 259         int rc;
 260 #endif
 261         const char *pch;
 262         FetchItem *OneMsg = NULL;
 263         POP3C_DBG_READ();
 264
 265         if ((StrLength(RecvMsg->IO.IOBuf) == 1) &&
 266             (ChrPtr(RecvMsg->IO.IOBuf)[0] == '.'))
 267         {
 268                 if (GetCount(RecvMsg->MsgNumbers) == 0)
 269                 {
 270                         ////    RecvMsg->Sate = ReadQuitState;
 271                 }
 272                 else
 273                 {
 274                         RecvMsg->Pos = GetNewHashPos(RecvMsg->MsgNumbers, 0);
 275                 }
 276                 return eSendReply;
 277
 278         }
 279         OneMsg = (FetchItem*) malloc(sizeof(FetchItem));
 280         memset(OneMsg, 0, sizeof(FetchItem));
 281         OneMsg->MSGID = atol(ChrPtr(RecvMsg->IO.IOBuf));
 282
 283         pch = strchr(ChrPtr(RecvMsg->IO.IOBuf), ' ');
 284         if (pch != NULL)
 285         {
 286                 OneMsg->MSGSize = atol(pch + 1);
 287         }
 288 #if 0
 289         rc = TestValidateHash(RecvMsg->MsgNumbers);
 290         if (rc != 0)
 291                 syslog(LOG_DEBUG, "Hash Invalid: %d\n", rc);
 292 #endif
 293
 294         Put(RecvMsg->MsgNumbers, LKEY(OneMsg->MSGID), OneMsg, HfreeFetchItem);
 295 #if 0
 296         rc = TestValidateHash(RecvMsg->MsgNumbers);
 297         if (rc != 0)
 298                 syslog(LOG_DEBUG, "Hash Invalid: %d\n", rc);
 299 #endif
 300         //RecvMsg->State --; /* read next Line */
 301         return eReadMore;
 302 }
 303
 304 eNextState POP3_FetchNetworkUsetableEntry(AsyncIO *IO)
 305 {
 306         long HKLen;
 307         const char *HKey;
 308         void *vData;
 309         struct cdbdata *cdbut;
 310         pop3aggr *RecvMsg = (pop3aggr *) IO->Data;
 311
 312         if(GetNextHashPos(RecvMsg->MsgNumbers,
 313                           RecvMsg->Pos,
 314                           &HKLen,
 315                           &HKey,
 316                           &vData))
 317         {
 318                 struct UseTable ut;
 319                 if (server_shutting_down)
 320                         return eAbort;
 321
 322                 RecvMsg->CurrMsg = (FetchItem*) vData;
 323                 syslog(LOG_DEBUG,
 324                        "CHECKING: whether %s has already been seen: ",
 325                        ChrPtr(RecvMsg->CurrMsg->MsgUID));
 326
 327                 /* Find out if we've already seen this item */
 328                 safestrncpy(ut.ut_msgid,
 329                             ChrPtr(RecvMsg->CurrMsg->MsgUID),
 330                             sizeof(ut.ut_msgid));
 331                 ut.ut_timestamp = time(NULL);/// TODO: libev timestamp!
 332
 333                 cdbut = cdb_fetch(CDB_USETABLE, SKEY(RecvMsg->CurrMsg->MsgUID));
 334                 if (cdbut != NULL) {
 335                         /* Item has already been seen */
 336                         syslog(LOG_DEBUG, "YES\n");
 337                         cdb_free(cdbut);
 338
 339                         /* rewrite the record anyway, to update the timestamp */
 340                         cdb_store(CDB_USETABLE,
 341                                   SKEY(RecvMsg->CurrMsg->MsgUID),
 342                                   &ut, sizeof(struct UseTable) );
 343                         RecvMsg->CurrMsg->NeedFetch = 0; ////TODO0;
 344                 }
 345                 else
 346                 {
 347                         syslog(LOG_DEBUG, "NO\n");
 348                         RecvMsg->CurrMsg->NeedFetch = 1;
 349                 }
 350                 return NextDBOperation(&RecvMsg->IO,
 351                                        POP3_FetchNetworkUsetableEntry);
 352         }
 353         else
 354         {
 355                 /* ok, now we know them all,
 356                  * continue with reading the actual messages. */
 357                 DeleteHashPos(&RecvMsg->Pos);
 358
 359                 return QueueEventContext(IO, POP3_C_ReAttachToFetchMessages);
 360         }
 361 }
 362
 363 eNextState POP3C_GetOneMessagID(pop3aggr *RecvMsg)
 364 {
 365         long HKLen;
 366         const char *HKey;
 367         void *vData;
 368
 369 #if 0
 370         int rc;
 371         rc = TestValidateHash(RecvMsg->MsgNumbers);
 372         if (rc != 0)
 373                 syslog(LOG_DEBUG, "Hash Invalid: %d\n", rc);
 374 #endif
 375         if(GetNextHashPos(RecvMsg->MsgNumbers,
 376                           RecvMsg->Pos,
 377                           &HKLen, &HKey,
 378                           &vData))
 379         {
 380                 RecvMsg->CurrMsg = (FetchItem*) vData;
 381                 /* Find out the UIDL of the message,
 382                  * to determine whether we've already downloaded it */
 383                 StrBufPrintf(RecvMsg->IO.SendBuf.Buf,
 384                              "UIDL %ld\r\n", RecvMsg->CurrMsg->MSGID);
 385                 POP3C_DBG_SEND();
 386         }
 387         else
 388         {
 389             RecvMsg->State++;
 390                 DeleteHashPos(&RecvMsg->Pos);
 391                 /// done receiving uidls.. start looking them up now.
 392                 RecvMsg->Pos = GetNewHashPos(RecvMsg->MsgNumbers, 0);
 393                 return QueueDBOperation(&RecvMsg->IO,
 394                                         POP3_FetchNetworkUsetableEntry);
 395         }
 396         return eReadMore; /* TODO */
 397 }
 398
 399 eNextState POP3C_GetOneMessageIDState(pop3aggr *RecvMsg)
 400 {
 401 #if 0
 402         int rc;
 403         rc = TestValidateHash(RecvMsg->MsgNumbers);
 404         if (rc != 0)
 405                 syslog(LOG_DEBUG, "Hash Invalid: %d\n", rc);
 406 #endif
 407
 408         POP3C_DBG_READ();
 409         if (!POP3C_OK) return eTerminateConnection;
 410         RecvMsg->CurrMsg->MsgUIDL =
 411                 NewStrBufPlain(NULL, StrLength(RecvMsg->IO.IOBuf));
 412         RecvMsg->CurrMsg->MsgUID =
 413                 NewStrBufPlain(NULL, StrLength(RecvMsg->IO.IOBuf) * 2);
 414
 415         StrBufExtract_token(RecvMsg->CurrMsg->MsgUIDL,
 416                             RecvMsg->IO.IOBuf, 2, ' ');
 417
 418         StrBufPrintf(RecvMsg->CurrMsg->MsgUID,
 419                      "pop3/%s/%s:%s@%s",
 420                      ChrPtr(RecvMsg->RoomName),
 421                      ChrPtr(RecvMsg->CurrMsg->MsgUIDL),
 422                      RecvMsg->IO.ConnectMe->User,
 423                      RecvMsg->IO.ConnectMe->Host);
 424         RecvMsg->State --;
 425         return eSendReply;
 426 }
 427
 428
 429 eNextState POP3C_SendGetOneMsg(pop3aggr *RecvMsg)
 430 {
 431         long HKLen;
 432         const char *HKey;
 433         void *vData;
 434
 435         RecvMsg->CurrMsg = NULL;
 436         while (GetNextHashPos(RecvMsg->MsgNumbers,
 437                               RecvMsg->Pos,
 438                               &HKLen, &HKey,
 439                               &vData) &&
 440                (RecvMsg->CurrMsg = (FetchItem*) vData,
 441                 RecvMsg->CurrMsg->NeedFetch == 0))
 442         {}
 443
 444         if ((RecvMsg->CurrMsg != NULL ) && (RecvMsg->CurrMsg->NeedFetch == 1))
 445         {
 446                 /* Message has not been seen.
 447                  * Tell the server to fetch the message... */
 448                 StrBufPrintf(RecvMsg->IO.SendBuf.Buf,
 449                              "RETR %ld\r\n", RecvMsg->CurrMsg->MSGID);
 450                 POP3C_DBG_SEND();
 451                 return eReadMessage;
 452         }
 453         else {
 454                 RecvMsg->State = ReadQuitState;
 455                 return POP3_C_DispatchWriteDone(&RecvMsg->IO);
 456         }
 457 }
 458
 459
 460 eNextState POP3C_ReadMessageBodyFollowing(pop3aggr *RecvMsg)
 461 {
 462         POP3C_DBG_READ();
 463         if (!POP3C_OK) return eTerminateConnection;
 464         RecvMsg->IO.ReadMsg = NewAsyncMsg(HKEY("."),
 465                                           RecvMsg->CurrMsg->MSGSize,
 466                                           config.c_maxmsglen,
 467                                           NULL, -1,
 468                                           1);
 469
 470         return eReadPayload;
 471 }
 472
 473
 474 eNextState POP3C_StoreMsgRead(AsyncIO *IO)
 475 {
 476         pop3aggr *RecvMsg = (pop3aggr *) IO->Data;
 477         struct UseTable ut;
 478
 479         syslog(LOG_DEBUG,
 480                "MARKING: %s as seen: ",
 481                ChrPtr(RecvMsg->CurrMsg->MsgUID));
 482
 483         safestrncpy(ut.ut_msgid,
 484                     ChrPtr(RecvMsg->CurrMsg->MsgUID),
 485                     sizeof(ut.ut_msgid));
 486         ut.ut_timestamp = time(NULL); /* TODO: use libev time */
 487         cdb_store(CDB_USETABLE,
 488                   ChrPtr(RecvMsg->CurrMsg->MsgUID),
 489                   StrLength(RecvMsg->CurrMsg->MsgUID),
 490                   &ut,
 491                   sizeof(struct UseTable) );
 492
 493         return QueueEventContext(&RecvMsg->IO, POP3_C_ReAttachToFetchMessages);
 494 }
 495 eNextState POP3C_SaveMsg(AsyncIO *IO)
 496 {
 497         long msgnum;
 498         pop3aggr *RecvMsg = (pop3aggr *) IO->Data;
 499
 500         /* Do Something With It (tm) */
 501         msgnum = CtdlSubmitMsg(RecvMsg->CurrMsg->Msg,
 502                                NULL,
 503                                ChrPtr(RecvMsg->RoomName),
 504                                0);
 505         if (msgnum > 0L)
 506         {
 507                 /* Message has been committed to the store
 508                  * write the uidl to the use table
 509                  * so we don't fetch this message again
 510                  */
 511         }
 512         CtdlFreeMessage(RecvMsg->CurrMsg->Msg);
 513
 514         return NextDBOperation(&RecvMsg->IO, POP3C_StoreMsgRead);
 515 }
 516
 517 eNextState POP3C_ReadMessageBody(pop3aggr *RecvMsg)
 518 {
 519         syslog(LOG_DEBUG, "Converting message...\n");
 520         RecvMsg->CurrMsg->Msg =
 521                 convert_internet_message_buf(&RecvMsg->IO.ReadMsg->MsgBuf);
 522
 523         return QueueDBOperation(&RecvMsg->IO, POP3C_SaveMsg);
 524 }
 525
 526 eNextState POP3C_SendDelete(pop3aggr *RecvMsg)
 527 {
 528         if (!RecvMsg->keep) {
 529                 StrBufPrintf(RecvMsg->IO.SendBuf.Buf,
 530                              "DELE %ld\r\n", RecvMsg->CurrMsg->MSGID);
 531                 POP3C_DBG_SEND();
 532                 return eReadMessage;
 533         }
 534         else {
 535                 RecvMsg->State = ReadMessageBodyFollowing;
 536                 return POP3_C_DispatchWriteDone(&RecvMsg->IO);
 537         }
 538 }
 539 eNextState POP3C_ReadDeleteState(pop3aggr *RecvMsg)
 540 {
 541         POP3C_DBG_READ();
 542         RecvMsg->State = GetOneMessageIDState;
 543         return eReadMessage;
 544 }
 545
 546 eNextState POP3C_SendQuit(pop3aggr *RecvMsg)
 547 {
 548         /* Log out */
 549         StrBufPlain(RecvMsg->IO.SendBuf.Buf,
 550                     HKEY("QUIT\r\n3)"));
 551         POP3C_DBG_SEND();
 552         return eReadMessage;
 553 }
 554
 555
 556 eNextState POP3C_ReadQuitState(pop3aggr *RecvMsg)
 557 {
 558         POP3C_DBG_READ();
 559         return eTerminateConnection;
 560 }
 561
 562 const long POP3_C_ConnTimeout = 1000;
 563 const long DefaultPOP3Port = 110;
 564
 565 Pop3ClientHandler POP3C_ReadHandlers[] = {
 566         POP3C_ReadGreeting,
 567         POP3C_GetUserState,
 568         POP3C_GetPassState,
 569         POP3C_GetListCommandState,
 570         POP3C_GetListOneLine,
 571         POP3C_GetOneMessageIDState,
 572         POP3C_ReadMessageBodyFollowing,
 573         POP3C_ReadMessageBody,
 574         POP3C_ReadDeleteState,
 575         POP3C_ReadQuitState,
 576 };
 577
 578 const long POP3_C_SendTimeouts[POP3C_MaxRead] = {
 579         100,
 580         100,
 581         100,
 582         100,
 583         100,
 584         100,
 585         100,
 586         100
 587 };
 588 const ConstStr POP3C_ReadErrors[POP3C_MaxRead] = {
 589         {HKEY("Connection broken during ")},
 590         {HKEY("Connection broken during ")},
 591         {HKEY("Connection broken during ")},
 592         {HKEY("Connection broken during ")},
 593         {HKEY("Connection broken during ")},
 594         {HKEY("Connection broken during ")},
 595         {HKEY("Connection broken during ")},
 596         {HKEY("Connection broken during ")}
 597 };
 598
 599 Pop3ClientHandler POP3C_SendHandlers[] = {
 600         NULL, /* we don't send a greeting */
 601         POP3C_SendUser,
 602         POP3C_SendPassword,
 603         POP3C_SendListCommand,
 604         NULL,
 605         POP3C_GetOneMessagID,
 606         POP3C_SendGetOneMsg,
 607         NULL,
 608         POP3C_SendDelete,
 609         POP3C_SendQuit
 610 };
 611
 612 const long POP3_C_ReadTimeouts[] = {
 613         100,
 614         100,
 615         100,
 616         100,
 617         100,
 618         100,
 619         100,
 620         100,
 621         100,
 622         100
 623 };
 624 /*****************************************************************************/
 625 /*                     POP3 CLIENT DISPATCHER                                */
 626 /*****************************************************************************/
 627
 628 void POP3SetTimeout(eNextState NextTCPState, pop3aggr *pMsg)
 629 {
 630         double Timeout = 0.0;
 631
 632         syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
 633
 634         switch (NextTCPState) {
 635         case eSendFile:
 636         case eSendReply:
 637         case eSendMore:
 638                 Timeout = POP3_C_SendTimeouts[pMsg->State];
 639 /*
 640   if (pMsg->State == eDATABody) {
 641   / * if we're sending a huge message, we need more time. * /
 642   Timeout += StrLength(pMsg->msgtext) / 1024;
 643   }
 644 */
 645                 break;
 646         case eReadFile:
 647         case eReadMessage:
 648                 Timeout = POP3_C_ReadTimeouts[pMsg->State];
 649 /*
 650   if (pMsg->State == eDATATerminateBody) {
 651   / *
 652   * some mailservers take a nap before accepting the message
 653   * content inspection and such.
 654   * /
 655   Timeout += StrLength(pMsg->msgtext) / 1024;
 656   }
 657 */
 658                 break;
 659         case eReadPayload:
 660                 Timeout = 100000;
 661                 /* TODO!!! */
 662                 break;
 663         case eSendDNSQuery:
 664         case eReadDNSReply:
 665         case eConnect:
 666         case eTerminateConnection:
 667         case eDBQuery:
 668         case eAbort:
 669         case eReadMore://// TODO
 670                 return;
 671         }
 672         SetNextTimeout(&pMsg->IO, Timeout);
 673 }
 674 eNextState POP3_C_DispatchReadDone(AsyncIO *IO)
 675 {
 676         syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
 677         pop3aggr *pMsg = IO->Data;
 678         eNextState rc;
 679
 680         rc = POP3C_ReadHandlers[pMsg->State](pMsg);
 681         if (rc != eReadMore)
 682             pMsg->State++;
 683         POP3SetTimeout(rc, pMsg);
 684         return rc;
 685 }
 686 eNextState POP3_C_DispatchWriteDone(AsyncIO *IO)
 687 {
 688         syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
 689         pop3aggr *pMsg = IO->Data;
 690         eNextState rc;
 691
 692         rc = POP3C_SendHandlers[pMsg->State](pMsg);
 693         POP3SetTimeout(rc, pMsg);
 694         return rc;
 695 }
 696
 697
 698 /*****************************************************************************/
 699 /*                     POP3 CLIENT ERROR CATCHERS                            */
 700 /*****************************************************************************/
 701 eNextState POP3_C_Terminate(AsyncIO *IO)
 702 {
 703 ///     pop3aggr *pMsg = (pop3aggr *)IO->Data;
 704
 705         syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
 706         FinalizePOP3AggrRun(IO);
 707         return eAbort;
 708 }
 709 eNextState POP3_C_Timeout(AsyncIO *IO)
 710 {
 711         pop3aggr *pMsg = IO->Data;
 712
 713         syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
 714         StrBufPlain(IO->ErrMsg, CKEY(POP3C_ReadErrors[pMsg->State]));
 715         return FailAggregationRun(IO);
 716 }
 717 eNextState POP3_C_ConnFail(AsyncIO *IO)
 718 {
 719         pop3aggr *pMsg = (pop3aggr *)IO->Data;
 720
 721         syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
 722         StrBufPlain(IO->ErrMsg, CKEY(POP3C_ReadErrors[pMsg->State]));
 723         return FailAggregationRun(IO);
 724 }
 725 eNextState POP3_C_DNSFail(AsyncIO *IO)
 726 {
 727         pop3aggr *pMsg = (pop3aggr *)IO->Data;
 728
 729         syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
 730         StrBufPlain(IO->ErrMsg, CKEY(POP3C_ReadErrors[pMsg->State]));
 731         return FailAggregationRun(IO);
 732 }
 733 eNextState POP3_C_Shutdown(AsyncIO *IO)
 734 {
 735         syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
 736 ////    pop3aggr *pMsg = IO->Data;
 737
 738 ////pMsg->MyQEntry->Status = 3;
 739 ///StrBufPlain(pMsg->MyQEntry->StatusMessage, HKEY("server shutdown during message retrieval."));
 740         FinalizePOP3AggrRun(IO);
 741         return eAbort;
 742 }
 743
 744
 745 /**
 746  * @brief lineread Handler; understands when to read more POP3 lines,
 747  *   and when this is a one-lined reply.
 748  */
 749 eReadState POP3_C_ReadServerStatus(AsyncIO *IO)
 750 {
 751         eReadState Finished = eBufferNotEmpty;
 752
 753         switch (IO->NextState) {
 754         case eSendDNSQuery:
 755         case eReadDNSReply:
 756         case eDBQuery:
 757         case eConnect:
 758         case eTerminateConnection:
 759         case eAbort:
 760                 Finished = eReadFail;
 761                 break;
 762         case eSendFile:
 763         case eSendReply:
 764         case eSendMore:
 765         case eReadMore:
 766         case eReadMessage:
 767                 Finished = StrBufChunkSipLine(IO->IOBuf, &IO->RecvBuf);
 768                 break;
 769         case eReadFile:
 770         case eReadPayload:
 771                 Finished = CtdlReadMessageBodyAsync(IO);
 772                 break;
 773         }
 774         return Finished;
 775 }
 776
 777 /*****************************************************************************
 778  * So we connect our Server IP here.                                         *
 779  *****************************************************************************/
 780 eNextState POP3_C_ReAttachToFetchMessages(AsyncIO *IO)
 781 {
 782         pop3aggr *cpptr = IO->Data;
 783
 784         syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
 785 ////??? cpptr->State ++;
 786         if (cpptr->Pos == NULL)
 787                 cpptr->Pos = GetNewHashPos(cpptr->MsgNumbers, 0);
 788
 789         POP3_C_DispatchWriteDone(IO);
 790         ReAttachIO(IO, cpptr, 0);
 791         IO->NextState = eReadMessage;
 792         return IO->NextState;
 793 }
 794
 795 eNextState pop3_connect_ip(AsyncIO *IO)
 796 {
 797         syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
 798
 799         return EvConnectSock(IO,
 800                              POP3_C_ConnTimeout,
 801                              POP3_C_ReadTimeouts[0],
 802                              1);
 803 }
 804
 805 eNextState pop3_get_one_host_ip_done(AsyncIO *IO)
 806 {
 807         pop3aggr *cpptr = IO->Data;
 808         struct hostent *hostent;
 809
 810         QueryCbDone(IO);
 811
 812         hostent = cpptr->HostLookup.VParsedDNSReply;
 813         if ((cpptr->HostLookup.DNSStatus == ARES_SUCCESS) &&
 814             (hostent != NULL) ) {
 815                 memset(&cpptr->IO.ConnectMe->Addr, 0, sizeof(struct in6_addr));
 816                 if (cpptr->IO.ConnectMe->IPv6) {
 817                         memcpy(&cpptr->IO.ConnectMe->Addr.sin6_addr.s6_addr,
 818                                &hostent->h_addr_list[0],
 819                                sizeof(struct in6_addr));
 820
 821                         cpptr->IO.ConnectMe->Addr.sin6_family =
 822                                 hostent->h_addrtype;
 823                         cpptr->IO.ConnectMe->Addr.sin6_port   =
 824                                 htons(DefaultPOP3Port);
 825                 }
 826                 else {
 827                         struct sockaddr_in *addr =
 828                                 (struct sockaddr_in*)
 829                                 &cpptr->IO.ConnectMe->Addr;
 830
 831                         memcpy(&addr->sin_addr.s_addr,
 832                                hostent->h_addr_list[0],
 833                                sizeof(uint32_t));
 834
 835                         addr->sin_family = hostent->h_addrtype;
 836                         addr->sin_port   = htons(DefaultPOP3Port);
 837                 }
 838                 return pop3_connect_ip(IO);
 839         }
 840         else
 841                 return eAbort;
 842 }
 843
 844 eNextState pop3_get_one_host_ip(AsyncIO *IO)
 845 {
 846         pop3aggr *cpptr = IO->Data;
 847         /*
 848          * here we start with the lookup of one host. it might be...
 849          * - the relay host *sigh*
 850          * - the direct hostname if there was no mx record
 851          * - one of the mx'es
 852          */
 853
 854         InitC_ares_dns(IO);
 855
 856         syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
 857
 858         syslog(LOG_DEBUG,
 859                       "POP3 client[%ld]: looking up %s-Record %s : %d ...\n",
 860                       cpptr->n,
 861                       (cpptr->IO.ConnectMe->IPv6)? "aaaa": "a",
 862                       cpptr->IO.ConnectMe->Host,
 863                       cpptr->IO.ConnectMe->Port);
 864
 865         QueueQuery((cpptr->IO.ConnectMe->IPv6)? ns_t_aaaa : ns_t_a,
 866                    cpptr->IO.ConnectMe->Host,
 867                    &cpptr->IO,
 868                    &cpptr->HostLookup,
 869                    pop3_get_one_host_ip_done);
 870         IO->NextState = eReadDNSReply;
 871         return IO->NextState;
 872 }
 873
 874
 875
 876 int pop3_do_fetching(pop3aggr *cpptr)
 877 {
 878         InitIOStruct(&cpptr->IO,
 879                      cpptr,
 880                      eReadMessage,
 881                      POP3_C_ReadServerStatus,
 882                      POP3_C_DNSFail,
 883                      POP3_C_DispatchWriteDone,
 884                      POP3_C_DispatchReadDone,
 885                      POP3_C_Terminate,
 886                      POP3_C_ConnFail,
 887                      POP3_C_Timeout,
 888                      POP3_C_Shutdown);
 889
 890         safestrncpy(((CitContext *)cpptr->IO.CitContext)->cs_host,
 891                     ChrPtr(cpptr->Url),
 892                     sizeof(((CitContext *)cpptr->IO.CitContext)->cs_host));
 893
 894         if (cpptr->IO.ConnectMe->IsIP) {
 895                 QueueEventContext(&cpptr->IO,
 896                                   pop3_connect_ip);
 897         }
 898         else {
 899                 QueueEventContext(&cpptr->IO,
 900                                   pop3_get_one_host_ip);
 901         }
 902         return 1;
 903 }
 904
 905 /*
 906  * Scan a room's netconfig to determine whether it requires POP3 aggregation
 907  */
 908 void pop3client_scan_room(struct ctdlroom *qrbuf, void *data)
 909 {
 910         StrBuf *CfgData;
 911         StrBuf *CfgType;
 912         StrBuf *Line;
 913
 914         struct stat statbuf;
 915         char filename[PATH_MAX];
 916         int  fd;
 917         int Done;
 918         void *vptr;
 919         const char *CfgPtr, *lPtr;
 920         const char *Err;
 921
 922 //      pop3_room_counter *Count = NULL;
 923 //      pop3aggr *cpptr;
 924
 925         pthread_mutex_lock(&POP3QueueMutex);
 926         if (GetHash(POP3QueueRooms, LKEY(qrbuf->QRnumber), &vptr))
 927         {
 928                 syslog(LOG_DEBUG,
 929                               "pop3client: [%ld] %s already in progress.\n",
 930                               qrbuf->QRnumber,
 931                               qrbuf->QRname);
 932                 pthread_mutex_unlock(&POP3QueueMutex);
 933         }
 934         pthread_mutex_unlock(&POP3QueueMutex);
 935
 936         if (server_shutting_down) return;
 937
 938         assoc_file_name(filename, sizeof filename, qrbuf, ctdl_netcfg_dir);
 939
 940         if (server_shutting_down)
 941                 return;
 942
 943         /* Only do net processing for rooms that have netconfigs */
 944         fd = open(filename, 0);
 945         if (fd <= 0) {
 946                 return;
 947         }
 948         if (server_shutting_down)
 949                 return;
 950         if (fstat(fd, &statbuf) == -1) {
 951                 syslog(LOG_DEBUG,
 952                        "ERROR: could not stat configfile '%s' - %s\n",
 953                        filename,
 954                        strerror(errno));
 955                 return;
 956         }
 957         if (server_shutting_down)
 958                 return;
 959         CfgData = NewStrBufPlain(NULL, statbuf.st_size + 1);
 960         if (StrBufReadBLOB(CfgData, &fd, 1, statbuf.st_size, &Err) < 0) {
 961                 close(fd);
 962                 FreeStrBuf(&CfgData);
 963                 syslog(LOG_DEBUG, "ERROR: reading config '%s' - %s<br>\n",
 964                               filename, strerror(errno));
 965                 return;
 966         }
 967         close(fd);
 968         if (server_shutting_down)
 969                 return;
 970
 971         CfgPtr = NULL;
 972         CfgType = NewStrBuf();
 973         Line = NewStrBufPlain(NULL, StrLength(CfgData));
 974         Done = 0;
 975
 976         while (!Done)
 977         {
 978                 Done = StrBufSipLine(Line, CfgData, &CfgPtr) == 0;
 979                 if (StrLength(Line) > 0)
 980                 {
 981                         lPtr = NULL;
 982                         StrBufExtract_NextToken(CfgType, Line, &lPtr, '|');
 983                         if (!strcasecmp("pop3client", ChrPtr(CfgType)))
 984                         {
 985                                 pop3aggr *cptr;
 986                                 StrBuf *Tmp;
 987 /*
 988                                 if (Count == NULL)
 989                                 {
 990                                 Count = malloc(sizeof(pop3_room_counter));
 991                                         Count->count = 0;
 992                                 }
 993                                 Count->count ++;
 994 */
 995                                 cptr = (pop3aggr *) malloc(sizeof(pop3aggr));
 996                                 memset(cptr, 0, sizeof(pop3aggr));
 997                                 ///TODO do we need this? cptr->roomlist_parts=1;
 998                                 cptr->RoomName =
 999                                         NewStrBufPlain(qrbuf->QRname, -1);
1000                                 cptr->pop3user =
1001                                         NewStrBufPlain(NULL, StrLength(Line));
1002                                 cptr->pop3pass =
1003                                         NewStrBufPlain(NULL, StrLength(Line));
1004                                 cptr->Url = NewStrBuf();
1005                                 Tmp = NewStrBuf();
1006
1007                                 StrBufExtract_NextToken(Tmp, Line, &lPtr, '|');
1008                                 StrBufExtract_NextToken(cptr->pop3user,
1009                                                         Line,
1010                                                         &lPtr,
1011                                                         '|');
1012
1013                                 StrBufExtract_NextToken(cptr->pop3pass,
1014                                                         Line,
1015                                                         &lPtr,
1016                                                         '|');
1017
1018                                 cptr->keep = StrBufExtractNext_long(Line,
1019                                                                     &lPtr,
1020                                                                     '|');
1021
1022                                 cptr->interval = StrBufExtractNext_long(Line,
1023                                                                         &lPtr,
1024                                                                         '|');
1025
1026                                 StrBufPrintf(cptr->Url, "pop3://%s:%s@%s/%s",
1027                                              ChrPtr(cptr->pop3user),
1028                                              ChrPtr(cptr->pop3pass),
1029                                              ChrPtr(Tmp),
1030                                              ChrPtr(cptr->RoomName));
1031                                 FreeStrBuf(&Tmp);
1032                                 ParseURL(&cptr->IO.ConnectMe, cptr->Url, 110);
1033
1034
1035 #if 0
1036 /* todo: we need to reunite the url to be shure. */
1037
1038                                 pthread_mutex_lock(&POP3ueueMutex);
1039                                 GetHash(POP3FetchUrls, SKEY(ptr->Url), &vptr);
1040                                 use_this_cptr = (pop3aggr *)vptr;
1041
1042                                 if (use_this_rncptr != NULL)
1043                                 {
1044                                         /* mustn't attach to an active session */
1045                                         if (use_this_cptr->RefCount > 0)
1046                                         {
1047                                                 DeletePOP3Cfg(cptr);
1048 ///                                             Count->count--;
1049                                         }
1050                                         else
1051                                         {
1052                                                 long *QRnumber;
1053                                                 StrBufAppendBufPlain(
1054                                                         use_this_cptr->rooms,
1055                                                         qrbuf->QRname,
1056                                                         -1, 0);
1057                                                 if (use_this_cptr->roomlist_parts == 1)
1058                                                 {
1059                                                         use_this_cptr->OtherQRnumbers
1060                                                                 = NewHash(1, lFlathash);
1061                                                 }
1062                                                 QRnumber = (long*)malloc(sizeof(long));
1063                                                 *QRnumber = qrbuf->QRnumber;
1064                                                 Put(use_this_cptr->OtherQRnumbers,
1065                                                     LKEY(qrbuf->QRnumber),
1066                                                     QRnumber,
1067                                                     NULL);
1068
1069                                                 use_this_cptr->roomlist_parts++;
1070                                         }
1071                                         pthread_mutex_unlock(&POP3QueueMutex);
1072                                         continue;
1073                                 }
1074                                 pthread_mutex_unlock(&RSSQueueMutex);
1075 #endif
1076
1077                                 pthread_mutex_lock(&POP3QueueMutex);
1078                                 Put(POP3FetchUrls,
1079                                     SKEY(cptr->Url),
1080                                     cptr,
1081                                     DeletePOP3Aggregator);
1082
1083                                 pthread_mutex_unlock(&POP3QueueMutex);
1084
1085                         }
1086
1087                 }
1088
1089                 ///fclose(fp);
1090
1091         }
1092         FreeStrBuf(&Line);
1093         FreeStrBuf(&CfgType);
1094         FreeStrBuf(&CfgData);
1095 }
1096
1097 static int doing_pop3client = 0;
1098
1099 void pop3client_scan(void) {
1100         static time_t last_run = 0L;
1101         time_t fastest_scan;
1102         HashPos *it;
1103         long len;
1104         const char *Key;
1105         void *vrptr;
1106         pop3aggr *cptr;
1107
1108         become_session(&pop3_client_CC);
1109
1110         if (config.c_pop3_fastest < config.c_pop3_fetch)
1111                 fastest_scan = config.c_pop3_fastest;
1112         else
1113                 fastest_scan = config.c_pop3_fetch;
1114
1115         /*
1116          * Run POP3 aggregation no more frequently than once every n seconds
1117          */
1118         if ( (time(NULL) - last_run) < fastest_scan ) {
1119                 return;
1120         }
1121
1122         /*
1123          * This is a simple concurrency check to make sure only one pop3client
1124          * run is done at a time.  We could do this with a mutex, but since we
1125          * don't really require extremely fine granularity here, we'll do it
1126          * with a static variable instead.
1127          */
1128         if (doing_pop3client) return;
1129         doing_pop3client = 1;
1130
1131         syslog(LOG_DEBUG, "pop3client started");
1132         CtdlForEachRoom(pop3client_scan_room, NULL);
1133
1134         pthread_mutex_lock(&POP3QueueMutex);
1135         it = GetNewHashPos(POP3FetchUrls, 0);
1136         while (!server_shutting_down &&
1137                GetNextHashPos(POP3FetchUrls, it, &len, &Key, &vrptr) &&
1138                (vrptr != NULL)) {
1139                 cptr = (pop3aggr *)vrptr;
1140                 if (cptr->RefCount == 0)
1141                         if (!pop3_do_fetching(cptr))
1142                                 DeletePOP3Aggregator(cptr);////TODO
1143
1144 /*
1145         if ((palist->interval && time(NULL) > (last_run + palist->interval))
1146                         || (time(NULL) > last_run + config.c_pop3_fetch))
1147                         pop3_do_fetching(palist->roomname, palist->pop3host,
1148                         palist->pop3user, palist->pop3pass, palist->keep);
1149                 pptr = palist;
1150                 palist = palist->next;
1151                 free(pptr);
1152 */
1153         }
1154         DeleteHashPos(&it);
1155         pthread_mutex_unlock(&POP3QueueMutex);
1156
1157         syslog(LOG_DEBUG, "pop3client ended");
1158         last_run = time(NULL);
1159         doing_pop3client = 0;
1160 }
1161
1162
1163 void pop3_cleanup(void)
1164 {
1165         /* citthread_mutex_destroy(&POP3QueueMutex); TODO */
1166         while (doing_pop3client != 0) ;
1167         DeleteHash(&POP3FetchUrls);
1168         DeleteHash(&POP3QueueRooms);
1169 }
1170
1171 CTDL_MODULE_INIT(pop3client)
1172 {
1173         if (!threading)
1174         {
1175                 CtdlFillSystemContext(&pop3_client_CC, "POP3aggr");
1176                 pthread_mutex_init(&POP3QueueMutex, NULL);
1177                 POP3QueueRooms = NewHash(1, lFlathash);
1178                 POP3FetchUrls = NewHash(1, NULL);
1179                 CtdlRegisterSessionHook(pop3client_scan, EVT_TIMER);
1180                 CtdlRegisterCleanupHook(pop3_cleanup);
1181         }
1182
1183         /* return our module id for the log */
1184         return "pop3client";
1185 }