citadel/modules/pop3client/serv_pop3client.c

   1 /*
   2  * Consolidate mail from remote POP3 accounts.
   3  *
   4  * Copyright (c) 2007-2017 by the citadel.org team
   5  *
   6  * This program is open source software; you can redistribute it and/or
   7  * modify it under the terms of the GNU General Public License as published
   8  * by the Free Software Foundation; either version 3 of the License, or
   9  * (at your option) any later version.
  10  *
  11  * This program is distributed in the hope that it will be useful,
  12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14  * GNU General Public License for more details.
  15  */
  16
  17 #include <stdlib.h>
  18 #include <unistd.h>
  19 #include <stdio.h>
  20 #include <sysconfig.h>
  21
  22 #if TIME_WITH_SYS_TIME
  23 # include <sys/time.h>
  24 # include <time.h>
  25 #else
  26 # if HAVE_SYS_TIME_H
  27 #  include <sys/time.h>
  28 # else
  29 #  include <time.h>
  30 # endif
  31 #endif
  32
  33 #include <ctype.h>
  34 #include <string.h>
  35 #include <errno.h>
  36 #include <sys/types.h>
  37 #include <sys/stat.h>
  38 #include <libcitadel.h>
  39 #include "citadel.h"
  40 #include "server.h"
  41 #include "citserver.h"
  42 #include "support.h"
  43 #include "config.h"
  44 #include "ctdl_module.h"
  45 #include "clientsocket.h"
  46 #include "msgbase.h"
  47 #include "internet_addressing.h"
  48 #include "database.h"
  49 #include "citadel_dirs.h"
  50 #include "event_client.h"
  51
  52
  53 #define POP3C_OK (strncasecmp(ChrPtr(RecvMsg->IO.IOBuf), "+OK", 3) == 0)
  54 int Pop3ClientID = 0;
  55
  56 #define N ((pop3aggr*)IO->Data)->n
  57
  58 struct CitContext pop3_client_CC;
  59
  60 pthread_mutex_t POP3QueueMutex; /* locks the access to the following vars: */
  61 HashList *POP3QueueRooms = NULL;
  62 HashList *POP3FetchUrls = NULL;
  63
  64 typedef struct pop3aggr pop3aggr;
  65 typedef eNextState(*Pop3ClientHandler)(pop3aggr* RecvMsg);
  66
  67 eNextState POP3_C_Shutdown(AsyncIO *IO);
  68 eNextState POP3_C_Timeout(AsyncIO *IO);
  69 eNextState POP3_C_ConnFail(AsyncIO *IO);
  70 eNextState POP3_C_DNSFail(AsyncIO *IO);
  71 eNextState POP3_C_DispatchReadDone(AsyncIO *IO);
  72 eNextState POP3_C_DispatchWriteDone(AsyncIO *IO);
  73 eNextState POP3_C_Terminate(AsyncIO *IO);
  74 eReadState POP3_C_ReadServerStatus(AsyncIO *IO);
  75 eNextState POP3_C_ReAttachToFetchMessages(AsyncIO *IO);
  76
  77 typedef struct __pop3_room_counter {
  78         int count;
  79         long QRnumber;
  80 }pop3_room_counter;
  81
  82 typedef enum ePOP3_C_States {
  83         ReadGreeting,
  84         GetUserState,
  85         GetPassState,
  86         GetListCommandState,
  87         GetListOneLine,
  88         GetOneMessageIDState,
  89         ReadMessageBodyFollowing,
  90         ReadMessageBody,
  91         GetDeleteState,
  92         ReadQuitState,
  93         POP3C_MaxRead
  94 }ePOP3_C_States;
  95
  96
  97 typedef struct _FetchItem {
  98         long MSGID;
  99         long MSGSize;
 100         StrBuf *MsgUIDL;
 101         StrBuf *MsgUID;
 102         int NeedFetch;
 103         struct CtdlMessage *Msg;
 104 } FetchItem;
 105
 106 void HfreeFetchItem(void *vItem)
 107 {
 108         FetchItem *Item = (FetchItem*) vItem;
 109         FreeStrBuf(&Item->MsgUIDL);
 110         FreeStrBuf(&Item->MsgUID);
 111         free(Item);
 112 }
 113
 114
 115
 116 typedef enum _POP3State {
 117         eCreated,
 118         eGreeting,
 119         eUser,
 120         ePassword,
 121         eListing,
 122         eUseTable,
 123         eGetMsgID,
 124         eGetMsg,
 125         eStoreMsg,
 126         eDelete,
 127         eQuit
 128 } POP3State;
 129
 130 ConstStr POP3States[] = {
 131         {HKEY("Aggregator created")},
 132         {HKEY("Reading Greeting")},
 133         {HKEY("Sending User")},
 134         {HKEY("Sending Password")},
 135         {HKEY("Listing")},
 136         {HKEY("Fetching Usetable")},
 137         {HKEY("Get MSG ID")},
 138         {HKEY("Get Message")},
 139         {HKEY("Store Msg")},
 140         {HKEY("Delete Upstream")},
 141         {HKEY("Quit")}
 142 };
 143
 144 static void SetPOP3State(AsyncIO *IO, POP3State State)
 145 {
 146         CitContext* CCC = IO->CitContext;
 147         if (CCC != NULL)
 148                 memcpy(CCC->cs_clientname, POP3States[State].Key, POP3States[State].len + 1);
 149 }
 150
 151
 152 struct pop3aggr {
 153         AsyncIO  IO;
 154
 155         long n;
 156         double IOStart;
 157         long count;
 158         long RefCount;
 159         DNSQueryParts HostLookup;
 160
 161         long             QRnumber;
 162         HashList        *OtherQRnumbers;
 163
 164         StrBuf          *Url;
 165         StrBuf *pop3user;
 166         StrBuf *pop3pass;
 167         StrBuf *Host;
 168         StrBuf *RoomName; // TODO: fill me
 169         int keep;
 170         time_t interval;
 171         ePOP3_C_States State;
 172         HashList *MsgNumbers;
 173         HashPos *Pos;
 174         FetchItem *CurrMsg;
 175 };
 176
 177 void DeletePOP3Aggregator(void *vptr)
 178 {
 179         pop3aggr *ptr = vptr;
 180         DeleteHashPos(&ptr->Pos);
 181         DeleteHash(&ptr->MsgNumbers);
 182 //      FreeStrBuf(&ptr->rooms);
 183         FreeStrBuf(&ptr->pop3user);
 184         FreeStrBuf(&ptr->pop3pass);
 185         FreeStrBuf(&ptr->Host);
 186         FreeStrBuf(&ptr->RoomName);
 187         FreeURL(&ptr->IO.ConnectMe);
 188         FreeStrBuf(&ptr->Url);
 189         FreeStrBuf(&ptr->IO.IOBuf);
 190         FreeStrBuf(&ptr->IO.SendBuf.Buf);
 191         FreeStrBuf(&ptr->IO.RecvBuf.Buf);
 192         DeleteAsyncMsg(&ptr->IO.ReadMsg);
 193         if (((struct CitContext*)ptr->IO.CitContext)) {
 194                 ((struct CitContext*)ptr->IO.CitContext)->state = CON_IDLE;
 195                 ((struct CitContext*)ptr->IO.CitContext)->kill_me = 1;
 196         }
 197         FreeAsyncIOContents(&ptr->IO);
 198         free(ptr);
 199 }
 200
 201 eNextState FinalizePOP3AggrRun(AsyncIO *IO)
 202 {
 203         HashPos  *It;
 204         pop3aggr *cpptr = (pop3aggr *)IO->Data;
 205
 206         syslog(LOG_INFO,
 207                      "%s@%s: fetched %ld new of %d messages in %fs. bye.",
 208                      ChrPtr(cpptr->pop3user),
 209                      ChrPtr(cpptr->Host),
 210                      cpptr->count,
 211                      GetCount(cpptr->MsgNumbers),
 212                      IO->Now - cpptr->IOStart
 213                 );
 214
 215         It = GetNewHashPos(POP3FetchUrls, 0);
 216         pthread_mutex_lock(&POP3QueueMutex);
 217         {
 218                 if (GetHashPosFromKey(POP3FetchUrls, SKEY(cpptr->Url), It))
 219                         DeleteEntryFromHash(POP3FetchUrls, It);
 220         }
 221         pthread_mutex_unlock(&POP3QueueMutex);
 222         DeleteHashPos(&It);
 223         return eAbort;
 224 }
 225
 226 eNextState FailAggregationRun(AsyncIO *IO)
 227 {
 228         return eAbort;
 229 }
 230
 231 eNextState POP3C_ReadGreeting(pop3aggr *RecvMsg)
 232 {
 233         AsyncIO *IO = &RecvMsg->IO;
 234         SetPOP3State(IO, eGreeting);
 235         /* Read the server greeting */
 236         if (!POP3C_OK) return eTerminateConnection;
 237         else return eSendReply;
 238 }
 239
 240 eNextState POP3C_SendUser(pop3aggr *RecvMsg)
 241 {
 242         AsyncIO *IO = &RecvMsg->IO;
 243         SetPOP3State(IO, eUser);
 244         /* Identify ourselves.  NOTE: we have to append a CR to each command.
 245          *  The LF will automatically be appended by sock_puts().  Believe it
 246          * or not, leaving out the CR will cause problems if the server happens
 247          * to be Exchange, which is so b0rken it actually barfs on
 248          * LF-terminated newlines.
 249          */
 250         StrBufPrintf(RecvMsg->IO.SendBuf.Buf, "USER %s\r\n", ChrPtr(RecvMsg->pop3user));
 251         return eReadMessage;
 252 }
 253
 254 eNextState POP3C_GetUserState(pop3aggr *RecvMsg)
 255 {
 256         if (!POP3C_OK) return eTerminateConnection;
 257         else return eSendReply;
 258 }
 259
 260 eNextState POP3C_SendPassword(pop3aggr *RecvMsg)
 261 {
 262         AsyncIO *IO = &RecvMsg->IO;
 263         SetPOP3State(IO, ePassword);
 264         /* Password */
 265         StrBufPrintf(RecvMsg->IO.SendBuf.Buf, "PASS %s\r\n", ChrPtr(RecvMsg->pop3pass));
 266         syslog(LOG_DEBUG, "<PASS <password>\n");
 267         return eReadMessage;
 268 }
 269
 270 eNextState POP3C_GetPassState(pop3aggr *RecvMsg)
 271 {
 272         if (!POP3C_OK) return eTerminateConnection;
 273         else return eSendReply;
 274 }
 275
 276 eNextState POP3C_SendListCommand(pop3aggr *RecvMsg)
 277 {
 278         AsyncIO *IO = &RecvMsg->IO;
 279         SetPOP3State(IO, eListing);
 280
 281         /* Get the list of messages */
 282         StrBufPlain(RecvMsg->IO.SendBuf.Buf, HKEY("LIST\r\n"));
 283         return eReadMessage;
 284 }
 285
 286 eNextState POP3C_GetListCommandState(pop3aggr *RecvMsg)
 287 {
 288         if (!POP3C_OK) return eTerminateConnection;
 289         RecvMsg->MsgNumbers = NewHash(1, NULL);
 290         RecvMsg->State++;
 291         return eReadMore;
 292 }
 293
 294
 295 eNextState POP3C_GetListOneLine(pop3aggr *RecvMsg)
 296 {
 297 #if 0
 298         int rc;
 299 #endif
 300         const char *pch;
 301         FetchItem *OneMsg = NULL;
 302
 303         if ((StrLength(RecvMsg->IO.IOBuf) == 1) &&
 304             (ChrPtr(RecvMsg->IO.IOBuf)[0] == '.'))
 305         {
 306                 if (GetCount(RecvMsg->MsgNumbers) == 0)
 307                 {
 308                         ////    RecvMsg->Sate = ReadQuitState;
 309                 }
 310                 else
 311                 {
 312                         RecvMsg->Pos = GetNewHashPos(RecvMsg->MsgNumbers, 0);
 313                 }
 314                 return eSendReply;
 315
 316         }
 317
 318         /*
 319          * work around buggy pop3 servers which send
 320          * empty lines in their listings.
 321         */
 322         if ((StrLength(RecvMsg->IO.IOBuf) == 0) ||
 323             !isdigit(ChrPtr(RecvMsg->IO.IOBuf)[0]))
 324         {
 325                 return eReadMore;
 326         }
 327
 328         OneMsg = (FetchItem*) malloc(sizeof(FetchItem));
 329         memset(OneMsg, 0, sizeof(FetchItem));
 330         OneMsg->MSGID = atol(ChrPtr(RecvMsg->IO.IOBuf));
 331
 332         pch = strchr(ChrPtr(RecvMsg->IO.IOBuf), ' ');
 333         if (pch != NULL)
 334         {
 335                 OneMsg->MSGSize = atol(pch + 1);
 336         }
 337 #if 0
 338         rc = TestValidateHash(RecvMsg->MsgNumbers);
 339         if (rc != 0)
 340                 syslog(LOG_DEBUG, "Hash Invalid: %d\n", rc);
 341 #endif
 342
 343         Put(RecvMsg->MsgNumbers, LKEY(OneMsg->MSGID), OneMsg, HfreeFetchItem);
 344 #if 0
 345         rc = TestValidateHash(RecvMsg->MsgNumbers);
 346         if (rc != 0)
 347                 syslog(LOG_DEBUG, "Hash Invalid: %d\n", rc);
 348 #endif
 349         //RecvMsg->State --; /* read next Line */
 350         return eReadMore;
 351 }
 352
 353 eNextState POP3_FetchNetworkUsetableEntry(AsyncIO *IO)
 354 {
 355         long HKLen;
 356         const char *HKey;
 357         void *vData;
 358         pop3aggr *RecvMsg = (pop3aggr *) IO->Data;
 359         time_t seenstamp = 0;
 360
 361         SetPOP3State(IO, eUseTable);
 362
 363         if((RecvMsg->Pos != NULL) &&
 364            GetNextHashPos(RecvMsg->MsgNumbers,
 365                           RecvMsg->Pos,
 366                           &HKLen,
 367                           &HKey,
 368                           &vData))
 369         {
 370                 if (server_shutting_down)
 371                         return eAbort;
 372
 373                 RecvMsg->CurrMsg = (FetchItem*)vData;
 374
 375                 seenstamp = CheckIfAlreadySeen(RecvMsg->CurrMsg->MsgUID,
 376                         EvGetNow(IO),
 377                         EvGetNow(IO) - USETABLE_ANTIEXPIRE,
 378                         eCheckUpdate
 379                 );
 380                 if (seenstamp != 0)
 381                 {
 382                         /* Item has already been seen */
 383                         RecvMsg->CurrMsg->NeedFetch = 0;
 384                 }
 385                 else
 386                 {
 387                         syslog(LOG_DEBUG, "NO\n");
 388                         RecvMsg->CurrMsg->NeedFetch = 1;
 389                 }
 390                 return NextDBOperation(&RecvMsg->IO,
 391                                        POP3_FetchNetworkUsetableEntry);
 392         }
 393         else
 394         {
 395                 /* ok, now we know them all,
 396                  * continue with reading the actual messages. */
 397                 DeleteHashPos(&RecvMsg->Pos);
 398                 return DBQueueEventContext(IO, POP3_C_ReAttachToFetchMessages);
 399         }
 400 }
 401
 402 eNextState POP3C_GetOneMessagID(pop3aggr *RecvMsg)
 403 {
 404         AsyncIO *IO = &RecvMsg->IO;
 405         long HKLen;
 406         const char *HKey;
 407         void *vData;
 408
 409         SetPOP3State(IO, eGetMsgID);
 410 #if 0
 411         int rc;
 412         rc = TestValidateHash(RecvMsg->MsgNumbers);
 413         if (rc != 0)
 414                 syslog(LOG_DEBUG, "Hash Invalid: %d\n", rc);
 415 #endif
 416         if((RecvMsg->Pos != NULL) &&
 417            GetNextHashPos(RecvMsg->MsgNumbers,
 418                           RecvMsg->Pos,
 419                           &HKLen, &HKey,
 420                           &vData))
 421         {
 422                 RecvMsg->CurrMsg = (FetchItem*) vData;
 423                 /* Find out the UIDL of the message,
 424                  * to determine whether we've already downloaded it */
 425                 StrBufPrintf(RecvMsg->IO.SendBuf.Buf,
 426                              "UIDL %ld\r\n", RecvMsg->CurrMsg->MSGID);
 427         }
 428         else
 429         {
 430                 RecvMsg->State++;
 431                 DeleteHashPos(&RecvMsg->Pos);
 432                 /// done receiving uidls.. start looking them up now.
 433                 RecvMsg->Pos = GetNewHashPos(RecvMsg->MsgNumbers, 0);
 434                 return EventQueueDBOperation(&RecvMsg->IO,
 435                                              POP3_FetchNetworkUsetableEntry,
 436                                              0);
 437         }
 438         return eReadMore; /* TODO */
 439 }
 440
 441 eNextState POP3C_GetOneMessageIDState(pop3aggr *RecvMsg)
 442 {
 443 #if 0
 444         int rc;
 445         rc = TestValidateHash(RecvMsg->MsgNumbers);
 446         if (rc != 0)
 447                 syslog(LOG_DEBUG, "Hash Invalid: %d\n", rc);
 448 #endif
 449
 450         if (!POP3C_OK) return eTerminateConnection;
 451         RecvMsg->CurrMsg->MsgUIDL = NewStrBufPlain(NULL, StrLength(RecvMsg->IO.IOBuf));
 452         RecvMsg->CurrMsg->MsgUID = NewStrBufPlain(NULL, StrLength(RecvMsg->IO.IOBuf) * 2);
 453
 454         StrBufExtract_token(RecvMsg->CurrMsg->MsgUIDL, RecvMsg->IO.IOBuf, 2, ' ');
 455
 456         StrBufPrintf(RecvMsg->CurrMsg->MsgUID,
 457                      "pop3/%s/%s:%s@%s",
 458                      ChrPtr(RecvMsg->RoomName),
 459                      ChrPtr(RecvMsg->CurrMsg->MsgUIDL),
 460                      RecvMsg->IO.ConnectMe->User,
 461                      RecvMsg->IO.ConnectMe->Host
 462         );
 463         RecvMsg->State --;
 464         return eSendReply;
 465 }
 466
 467
 468 eNextState POP3C_SendGetOneMsg(pop3aggr *RecvMsg)
 469 {
 470         AsyncIO *IO = &RecvMsg->IO;
 471         long HKLen;
 472         const char *HKey;
 473         void *vData;
 474
 475         SetPOP3State(IO, eGetMsg);
 476
 477         syslog(LOG_DEBUG, "fast forwarding to the next unknown message");
 478
 479         RecvMsg->CurrMsg = NULL;
 480         while ((RecvMsg->Pos != NULL) &&
 481                GetNextHashPos(RecvMsg->MsgNumbers,
 482                               RecvMsg->Pos,
 483                               &HKLen, &HKey,
 484                               &vData) &&
 485                (RecvMsg->CurrMsg = (FetchItem*) vData,
 486                 RecvMsg->CurrMsg->NeedFetch == 0))
 487         {}
 488
 489         if ((RecvMsg->CurrMsg != NULL ) && (RecvMsg->CurrMsg->NeedFetch == 1))
 490         {
 491                 syslog(LOG_DEBUG, "fetching next");
 492                 /* Message has not been seen.
 493                  * Tell the server to fetch the message... */
 494                 StrBufPrintf(RecvMsg->IO.SendBuf.Buf, "RETR %ld\r\n", RecvMsg->CurrMsg->MSGID);
 495                 return eReadMessage;
 496         }
 497         else {
 498                 syslog(LOG_DEBUG, "no more messages to fetch.");
 499                 RecvMsg->State = ReadQuitState;
 500                 return POP3_C_DispatchWriteDone(&RecvMsg->IO);
 501         }
 502 }
 503
 504
 505 eNextState POP3C_ReadMessageBodyFollowing(pop3aggr *RecvMsg)
 506 {
 507         if (!POP3C_OK) return eTerminateConnection;
 508         RecvMsg->IO.ReadMsg = NewAsyncMsg(HKEY("."),
 509                 RecvMsg->CurrMsg->MSGSize,
 510                 CtdlGetConfigLong("c_maxmsglen"),
 511                 NULL, -1,
 512                 1
 513         );
 514
 515         return eReadPayload;
 516 }
 517
 518
 519 eNextState POP3C_StoreMsgRead(AsyncIO *IO)
 520 {
 521         pop3aggr *RecvMsg = (pop3aggr *) IO->Data;
 522
 523         SetPOP3State(IO, eStoreMsg);
 524
 525         syslog(LOG_DEBUG, "MARKING: %s as seen: ", ChrPtr(RecvMsg->CurrMsg->MsgUID));
 526         CheckIfAlreadySeen(RecvMsg->CurrMsg->MsgUID, EvGetNow(IO), EvGetNow(IO) - USETABLE_ANTIEXPIRE, eWrite);
 527
 528         return DBQueueEventContext(&RecvMsg->IO, POP3_C_ReAttachToFetchMessages);
 529 }
 530
 531
 532 eNextState POP3C_SaveMsg(AsyncIO *IO)
 533 {
 534         long msgnum;
 535         pop3aggr *RecvMsg = (pop3aggr *) IO->Data;
 536
 537         /* Do Something With It (tm) */
 538         msgnum = CtdlSubmitMsg(RecvMsg->CurrMsg->Msg,
 539                                NULL,
 540                                ChrPtr(RecvMsg->RoomName),
 541                                0);
 542         if (msgnum > 0L)
 543         {
 544                 /* Message has been committed to the store
 545                  * write the uidl to the use table
 546                  * so we don't fetch this message again
 547                  */
 548         }
 549         CM_Free(RecvMsg->CurrMsg->Msg);
 550
 551         RecvMsg->count ++;
 552         return NextDBOperation(&RecvMsg->IO, POP3C_StoreMsgRead);
 553 }
 554
 555
 556 eNextState POP3C_ReadMessageBody(pop3aggr *RecvMsg)
 557 {
 558         syslog(LOG_DEBUG, "Converting message...");
 559         RecvMsg->CurrMsg->Msg = convert_internet_message_buf(&RecvMsg->IO.ReadMsg->MsgBuf);
 560         return EventQueueDBOperation(&RecvMsg->IO, POP3C_SaveMsg, 0);
 561 }
 562
 563
 564 eNextState POP3C_SendDelete(pop3aggr *RecvMsg)
 565 {
 566         AsyncIO *IO = &RecvMsg->IO;
 567
 568         SetPOP3State(IO, eDelete);
 569
 570         if (!RecvMsg->keep) {
 571                 StrBufPrintf(RecvMsg->IO.SendBuf.Buf, "DELE %ld\r\n", RecvMsg->CurrMsg->MSGID);
 572                 return eReadMessage;
 573         }
 574         else {
 575                 RecvMsg->State = ReadMessageBodyFollowing;
 576                 return POP3_C_DispatchWriteDone(&RecvMsg->IO);
 577         }
 578 }
 579
 580
 581 eNextState POP3C_ReadDeleteState(pop3aggr *RecvMsg)
 582 {
 583         RecvMsg->State = GetOneMessageIDState;
 584         return POP3_C_DispatchWriteDone(&RecvMsg->IO);
 585 }
 586
 587
 588 eNextState POP3C_SendQuit(pop3aggr *RecvMsg)
 589 {
 590         AsyncIO *IO = &RecvMsg->IO;
 591         SetPOP3State(IO, eQuit);
 592
 593         /* Log out */
 594         StrBufPlain(RecvMsg->IO.SendBuf.Buf,
 595                     HKEY("QUIT\r\n3)"));
 596         return eReadMessage;
 597 }
 598
 599
 600 eNextState POP3C_ReadQuitState(pop3aggr *RecvMsg)
 601 {
 602         return eTerminateConnection;
 603 }
 604
 605
 606 const long POP3_C_ConnTimeout = 1000;
 607 const long DefaultPOP3Port = 110;
 608
 609 Pop3ClientHandler POP3C_ReadHandlers[] = {
 610         POP3C_ReadGreeting,
 611         POP3C_GetUserState,
 612         POP3C_GetPassState,
 613         POP3C_GetListCommandState,
 614         POP3C_GetListOneLine,
 615         POP3C_GetOneMessageIDState,
 616         POP3C_ReadMessageBodyFollowing,
 617         POP3C_ReadMessageBody,
 618         POP3C_ReadDeleteState,
 619         POP3C_ReadQuitState,
 620 };
 621
 622 const long POP3_C_SendTimeouts[POP3C_MaxRead] = {
 623         100,
 624         100,
 625         100,
 626         100,
 627         100,
 628         100,
 629         100,
 630         100
 631 };
 632 const ConstStr POP3C_ReadErrors[POP3C_MaxRead] = {
 633         {HKEY("Connection broken during ")},
 634         {HKEY("Connection broken during ")},
 635         {HKEY("Connection broken during ")},
 636         {HKEY("Connection broken during ")},
 637         {HKEY("Connection broken during ")},
 638         {HKEY("Connection broken during ")},
 639         {HKEY("Connection broken during ")},
 640         {HKEY("Connection broken during ")}
 641 };
 642
 643 Pop3ClientHandler POP3C_SendHandlers[] = {
 644         NULL, /* we don't send a greeting */
 645         POP3C_SendUser,
 646         POP3C_SendPassword,
 647         POP3C_SendListCommand,
 648         NULL,
 649         POP3C_GetOneMessagID,
 650         POP3C_SendGetOneMsg,
 651         NULL,
 652         POP3C_SendDelete,
 653         POP3C_SendQuit
 654 };
 655
 656 const long POP3_C_ReadTimeouts[] = {
 657         100,
 658         100,
 659         100,
 660         100,
 661         100,
 662         100,
 663         100,
 664         100,
 665         100,
 666         100
 667 };
 668 /*****************************************************************************/
 669 /*                     POP3 CLIENT DISPATCHER                                */
 670 /*****************************************************************************/
 671
 672 void POP3SetTimeout(eNextState NextTCPState, pop3aggr *pMsg)
 673 {
 674         double Timeout = 0.0;
 675
 676         syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
 677
 678         switch (NextTCPState) {
 679         case eSendFile:
 680         case eSendReply:
 681         case eSendMore:
 682                 Timeout = POP3_C_SendTimeouts[pMsg->State];
 683 /*
 684   if (pMsg->State == eDATABody) {
 685   / * if we're sending a huge message, we need more time. * /
 686   Timeout += StrLength(pMsg->msgtext) / 1024;
 687   }
 688 */
 689                 break;
 690         case eReadFile:
 691         case eReadMessage:
 692                 Timeout = POP3_C_ReadTimeouts[pMsg->State];
 693 /*
 694   if (pMsg->State == eDATATerminateBody) {
 695   / *
 696   * some mailservers take a nap before accepting the message
 697   * content inspection and such.
 698   * /
 699   Timeout += StrLength(pMsg->msgtext) / 1024;
 700   }
 701 */
 702                 break;
 703         case eReadPayload:
 704                 Timeout = 100000;
 705                 /* TODO!!! */
 706                 break;
 707         case eSendDNSQuery:
 708         case eReadDNSReply:
 709         case eConnect:
 710         case eTerminateConnection:
 711         case eDBQuery:
 712         case eAbort:
 713         case eReadMore://// TODO
 714                 return;
 715         }
 716         SetNextTimeout(&pMsg->IO, Timeout);
 717 }
 718 eNextState POP3_C_DispatchReadDone(AsyncIO *IO)
 719 {
 720 /*      syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__); to noisy anyways. */
 721         pop3aggr *pMsg = IO->Data;
 722         eNextState rc;
 723
 724         rc = POP3C_ReadHandlers[pMsg->State](pMsg);
 725         if (rc != eReadMore)
 726             pMsg->State++;
 727         POP3SetTimeout(rc, pMsg);
 728         return rc;
 729 }
 730 eNextState POP3_C_DispatchWriteDone(AsyncIO *IO)
 731 {
 732         pop3aggr *pMsg = IO->Data;
 733         eNextState rc;
 734
 735 /*      syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__); to noisy anyways. */
 736         rc = POP3C_SendHandlers[pMsg->State](pMsg);
 737         POP3SetTimeout(rc, pMsg);
 738         return rc;
 739 }
 740
 741
 742 /*****************************************************************************/
 743 /*                     POP3 CLIENT ERROR CATCHERS                            */
 744 /*****************************************************************************/
 745 eNextState POP3_C_Terminate(AsyncIO *IO)
 746 {
 747         syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
 748         FinalizePOP3AggrRun(IO);
 749         return eAbort;
 750 }
 751 eNextState POP3_C_TerminateDB(AsyncIO *IO)
 752 {
 753         syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
 754         FinalizePOP3AggrRun(IO);
 755         return eAbort;
 756 }
 757 eNextState POP3_C_Timeout(AsyncIO *IO)
 758 {
 759         pop3aggr *pMsg = IO->Data;
 760
 761         syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
 762         StrBufPlain(IO->ErrMsg, CKEY(POP3C_ReadErrors[pMsg->State]));
 763         return FailAggregationRun(IO);
 764 }
 765 eNextState POP3_C_ConnFail(AsyncIO *IO)
 766 {
 767         pop3aggr *pMsg = (pop3aggr *)IO->Data;
 768
 769         syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
 770         StrBufPlain(IO->ErrMsg, CKEY(POP3C_ReadErrors[pMsg->State]));
 771         return FailAggregationRun(IO);
 772 }
 773 eNextState POP3_C_DNSFail(AsyncIO *IO)
 774 {
 775         pop3aggr *pMsg = (pop3aggr *)IO->Data;
 776
 777         syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
 778         StrBufPlain(IO->ErrMsg, CKEY(POP3C_ReadErrors[pMsg->State]));
 779         return FailAggregationRun(IO);
 780 }
 781 eNextState POP3_C_Shutdown(AsyncIO *IO)
 782 {
 783         syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
 784         FinalizePOP3AggrRun(IO);
 785         return eAbort;
 786 }
 787
 788
 789 /*
 790  * lineread Handler; understands when to read more POP3 lines, and when this is a one-lined reply.
 791  */
 792 eReadState POP3_C_ReadServerStatus(AsyncIO *IO)
 793 {
 794         eReadState Finished = eBufferNotEmpty;
 795
 796         switch (IO->NextState) {
 797         case eSendDNSQuery:
 798         case eReadDNSReply:
 799         case eDBQuery:
 800         case eConnect:
 801         case eTerminateConnection:
 802         case eAbort:
 803                 Finished = eReadFail;
 804                 break;
 805         case eSendFile:
 806         case eSendReply:
 807         case eSendMore:
 808         case eReadMore:
 809         case eReadMessage:
 810                 Finished = StrBufChunkSipLine(IO->IOBuf, &IO->RecvBuf);
 811                 break;
 812         case eReadFile:
 813         case eReadPayload:
 814                 Finished = CtdlReadMessageBodyAsync(IO);
 815                 break;
 816         }
 817         return Finished;
 818 }
 819
 820 /*****************************************************************************
 821  * So we connect our Server IP here.                                         *
 822  *****************************************************************************/
 823 eNextState POP3_C_ReAttachToFetchMessages(AsyncIO *IO)
 824 {
 825         pop3aggr *cpptr = IO->Data;
 826
 827         syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
 828 ////??? cpptr->State ++;
 829         if (cpptr->Pos == NULL)
 830                 cpptr->Pos = GetNewHashPos(cpptr->MsgNumbers, 0);
 831
 832         POP3_C_DispatchWriteDone(IO);
 833         ReAttachIO(IO, cpptr, 0);
 834         IO->NextState = eReadMessage;
 835         return IO->NextState;
 836 }
 837
 838 eNextState pop3_connect_ip(AsyncIO *IO)
 839 {
 840         pop3aggr *cpptr = IO->Data;
 841
 842         if (cpptr->IOStart == 0.0) /* whith or without DNS? */
 843                 cpptr->IOStart = IO->Now;
 844
 845         syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
 846
 847         return EvConnectSock(IO,
 848                              POP3_C_ConnTimeout,
 849                              POP3_C_ReadTimeouts[0],
 850                              1);
 851 }
 852
 853 eNextState pop3_get_one_host_ip_done(AsyncIO *IO)
 854 {
 855         pop3aggr *cpptr = IO->Data;
 856         struct hostent *hostent;
 857
 858         QueryCbDone(IO);
 859
 860         hostent = cpptr->HostLookup.VParsedDNSReply;
 861         if ((cpptr->HostLookup.DNSStatus == ARES_SUCCESS) &&
 862             (hostent != NULL) ) {
 863                 memset(&cpptr->IO.ConnectMe->Addr, 0, sizeof(struct in6_addr));
 864                 if (cpptr->IO.ConnectMe->IPv6) {
 865                         memcpy(&cpptr->IO.ConnectMe->Addr.sin6_addr.s6_addr,
 866                                &hostent->h_addr_list[0],
 867                                sizeof(struct in6_addr));
 868
 869                         cpptr->IO.ConnectMe->Addr.sin6_family =
 870                                 hostent->h_addrtype;
 871                         cpptr->IO.ConnectMe->Addr.sin6_port   =
 872                                 htons(DefaultPOP3Port);
 873                 }
 874                 else {
 875                         struct sockaddr_in *addr =
 876                                 (struct sockaddr_in*)
 877                                 &cpptr->IO.ConnectMe->Addr;
 878
 879                         memcpy(&addr->sin_addr.s_addr,
 880                                hostent->h_addr_list[0],
 881                                sizeof(uint32_t));
 882
 883                         addr->sin_family = hostent->h_addrtype;
 884                         addr->sin_port   = htons(DefaultPOP3Port);
 885                 }
 886                 return pop3_connect_ip(IO);
 887         }
 888         else
 889                 return eAbort;
 890 }
 891
 892 eNextState pop3_get_one_host_ip(AsyncIO *IO)
 893 {
 894         pop3aggr *cpptr = IO->Data;
 895
 896         cpptr->IOStart = IO->Now;
 897
 898         syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
 899
 900         syslog(LOG_DEBUG,
 901                        "POP3 client[%ld]: looking up %s-Record %s : %d ...\n",
 902                        cpptr->n,
 903                        (cpptr->IO.ConnectMe->IPv6)? "aaaa": "a",
 904                        cpptr->IO.ConnectMe->Host,
 905                        cpptr->IO.ConnectMe->Port);
 906
 907         QueueQuery((cpptr->IO.ConnectMe->IPv6)? ns_t_aaaa : ns_t_a,
 908                    cpptr->IO.ConnectMe->Host,
 909                    &cpptr->IO,
 910                    &cpptr->HostLookup,
 911                    pop3_get_one_host_ip_done);
 912         IO->NextState = eReadDNSReply;
 913         return IO->NextState;
 914 }
 915
 916
 917
 918 int pop3_do_fetching(pop3aggr *cpptr)
 919 {
 920         AsyncIO *IO = &cpptr->IO;
 921
 922         InitIOStruct(IO,
 923                      cpptr,
 924                      eReadMessage,
 925                      POP3_C_ReadServerStatus,
 926                      POP3_C_DNSFail,
 927                      POP3_C_DispatchWriteDone,
 928                      POP3_C_DispatchReadDone,
 929                      POP3_C_Terminate,
 930                      POP3_C_TerminateDB,
 931                      POP3_C_ConnFail,
 932                      POP3_C_Timeout,
 933                      POP3_C_Shutdown);
 934
 935         safestrncpy(((CitContext *)cpptr->IO.CitContext)->cs_host,
 936                     ChrPtr(cpptr->Url),
 937                     sizeof(((CitContext *)cpptr->IO.CitContext)->cs_host));
 938
 939         if (cpptr->IO.ConnectMe->IsIP) {
 940                 QueueEventContext(&cpptr->IO,
 941                                   pop3_connect_ip);
 942         }
 943         else {
 944                 QueueEventContext(&cpptr->IO,
 945                                   pop3_get_one_host_ip);
 946         }
 947         return 1;
 948 }
 949
 950 /*
 951  * Scan a room's netconfig to determine whether it requires POP3 aggregation
 952  */
 953 void pop3client_scan_room(struct ctdlroom *qrbuf, void *data, OneRoomNetCfg *OneRNCFG)
 954 {
 955         const RoomNetCfgLine *pLine;
 956         void *vptr;
 957
 958         pthread_mutex_lock(&POP3QueueMutex);
 959         if (GetHash(POP3QueueRooms, LKEY(qrbuf->QRnumber), &vptr))
 960         {
 961                 pthread_mutex_unlock(&POP3QueueMutex);
 962                 syslog(LOG_DEBUG,
 963                               "pop3client: [%ld] %s already in progress.",
 964                               qrbuf->QRnumber,
 965                               qrbuf->QRname);
 966                 return;
 967         }
 968         pthread_mutex_unlock(&POP3QueueMutex);
 969
 970         if (server_shutting_down) return;
 971
 972         pLine = OneRNCFG->NetConfigs[pop3client];
 973
 974         while (pLine != NULL)
 975         {
 976
 977
 978                 // BEGIN diagnostics for ajc 2017mar08
 979                 syslog(LOG_DEBUG, "pop3client: room=<%s> host=<%s> user=<%s> pass=<%s> keep=<%d> interval=<%ld>",
 980                         qrbuf->QRname,
 981                         ChrPtr(pLine->Value[0]),
 982                         ChrPtr(pLine->Value[1]),
 983                         ChrPtr(pLine->Value[2]),
 984                         atoi(ChrPtr(pLine->Value[3])),
 985                         atol(ChrPtr(pLine->Value[4]))
 986                 );
 987                 // END diagnostics for ajc 2017mar08
 988
 989                 pop3aggr *cptr;
 990
 991                 cptr = (pop3aggr *) malloc(sizeof(pop3aggr));
 992                 memset(cptr, 0, sizeof(pop3aggr));
 993                 cptr->RoomName = NewStrBufPlain(qrbuf->QRname, -1);
 994                 cptr->pop3user = NewStrBufDup(pLine->Value[1]);
 995                 cptr->pop3pass = NewStrBufDup(pLine->Value[2]);
 996                 cptr->Url = NewStrBuf();
 997                 cptr->Host = NewStrBufDup(pLine->Value[0]);
 998
 999                 cptr->keep = atol(ChrPtr(pLine->Value[3]));
1000                 cptr->interval = atol(ChrPtr(pLine->Value[4]));
1001
1002                 StrBufAppendBufPlain(cptr->Url, HKEY("pop3://"), 0);
1003                 StrBufUrlescUPAppend(cptr->Url, cptr->pop3user, NULL);
1004                 StrBufAppendBufPlain(cptr->Url, HKEY(":"), 0);
1005                 StrBufUrlescUPAppend(cptr->Url, cptr->pop3pass, NULL);
1006                 StrBufAppendBufPlain(cptr->Url, HKEY("@"), 0);
1007                 StrBufAppendBuf(cptr->Url, cptr->Host, 0);
1008                 StrBufAppendBufPlain(cptr->Url, HKEY("/"), 0);
1009                 StrBufUrlescAppend(cptr->Url, cptr->RoomName, NULL);
1010
1011                 ParseURL(&cptr->IO.ConnectMe, cptr->Url, 110);
1012
1013                 cptr->n = Pop3ClientID++;
1014                 pthread_mutex_lock(&POP3QueueMutex);
1015                 Put(POP3FetchUrls,
1016                     SKEY(cptr->Url),
1017                     cptr,
1018                     DeletePOP3Aggregator);
1019
1020                 pthread_mutex_unlock(&POP3QueueMutex);
1021                 pLine = pLine->next;
1022
1023         }
1024 }
1025
1026 static int doing_pop3client = 0;
1027
1028 void pop3client_scan(void) {
1029         static time_t last_run = 0L;
1030         time_t fastest_scan;
1031         HashPos *it;
1032         long len;
1033         const char *Key;
1034         void *vrptr;
1035         pop3aggr *cptr;
1036
1037         become_session(&pop3_client_CC);
1038
1039         if (CtdlGetConfigLong("c_pop3_fastest") < CtdlGetConfigLong("c_pop3_fetch"))
1040                 fastest_scan = CtdlGetConfigLong("c_pop3_fastest");
1041         else
1042                 fastest_scan = CtdlGetConfigLong("c_pop3_fetch");
1043
1044         /*
1045          * Run POP3 aggregation no more frequently than once every n seconds
1046          */
1047         if ( (time(NULL) - last_run) < fastest_scan ) {
1048                 return;
1049         }
1050
1051         /*
1052          * This is a simple concurrency check to make sure only one pop3client
1053          * run is done at a time.  We could do this with a mutex, but since we
1054          * don't really require extremely fine granularity here, we'll do it
1055          * with a static variable instead.
1056          */
1057         if (doing_pop3client) return;
1058         doing_pop3client = 1;
1059
1060         syslog(LOG_DEBUG, "pop3client started");
1061         CtdlForEachNetCfgRoom(pop3client_scan_room, NULL);
1062
1063         pthread_mutex_lock(&POP3QueueMutex);
1064         it = GetNewHashPos(POP3FetchUrls, 0);
1065         while (!server_shutting_down &&
1066                GetNextHashPos(POP3FetchUrls, it, &len, &Key, &vrptr) &&
1067                (vrptr != NULL)) {
1068                 cptr = (pop3aggr *)vrptr;
1069                 if (cptr->RefCount == 0) {
1070                         if (!pop3_do_fetching(cptr)) {
1071                                 DeletePOP3Aggregator(cptr);
1072                         }
1073                 }
1074         }
1075         DeleteHashPos(&it);
1076         pthread_mutex_unlock(&POP3QueueMutex);
1077
1078         syslog(LOG_DEBUG, "pop3client ended");
1079         last_run = time(NULL);
1080         doing_pop3client = 0;
1081 }
1082
1083
1084 void pop3_cleanup(void)
1085 {
1086         while (doing_pop3client != 0) ;
1087         DeleteHash(&POP3FetchUrls);
1088         DeleteHash(&POP3QueueRooms);
1089 }
1090
1091
1092
1093 CTDL_MODULE_INIT(pop3client)
1094 {
1095         if (!threading)
1096         {
1097                 CtdlFillSystemContext(&pop3_client_CC, "POP3aggr");
1098                 CtdlREGISTERRoomCfgType(pop3client, ParseGeneric, 0, 5, SerializeGeneric, DeleteGenericCfgLine);
1099                 pthread_mutex_init(&POP3QueueMutex, NULL);
1100                 POP3QueueRooms = NewHash(1, lFlathash);
1101                 POP3FetchUrls = NewHash(1, NULL);
1102                 CtdlRegisterSessionHook(pop3client_scan, EVT_TIMER, PRIO_AGGR + 50);
1103                 CtdlRegisterEVCleanupHook(pop3_cleanup);
1104         }
1105
1106         /* return our module id for the log */
1107         return "pop3client";
1108 }