citadel/modules/pop3client/serv_pop3client.c

   1 /*
   2  * Consolidate mail from remote POP3 accounts.
   3  *
   4  * Copyright (c) 2007-2015 by the citadel.org team
   5  *
   6  * This program is open source software; you can redistribute it and/or
   7  * modify it under the terms of the GNU General Public License as published
   8  * by the Free Software Foundation; either version 3 of the License, or
   9  * (at your option) any later version.
  10  *
  11  * This program is distributed in the hope that it will be useful,
  12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14  * GNU General Public License for more details.
  15  */
  16
  17 #include <stdlib.h>
  18 #include <unistd.h>
  19 #include <stdio.h>
  20 #include <sysconfig.h>
  21
  22 #if TIME_WITH_SYS_TIME
  23 # include <sys/time.h>
  24 # include <time.h>
  25 #else
  26 # if HAVE_SYS_TIME_H
  27 #  include <sys/time.h>
  28 # else
  29 #  include <time.h>
  30 # endif
  31 #endif
  32
  33 #include <ctype.h>
  34 #include <string.h>
  35 #include <errno.h>
  36 #include <sys/types.h>
  37 #include <sys/stat.h>
  38 #include <libcitadel.h>
  39 #include "citadel.h"
  40 #include "server.h"
  41 #include "citserver.h"
  42 #include "support.h"
  43 #include "config.h"
  44 #include "ctdl_module.h"
  45 #include "clientsocket.h"
  46 #include "msgbase.h"
  47 #include "internet_addressing.h"
  48 #include "database.h"
  49 #include "citadel_dirs.h"
  50 #include "event_client.h"
  51
  52
  53 #define POP3C_OK (strncasecmp(ChrPtr(RecvMsg->IO.IOBuf), "+OK", 3) == 0)
  54 int Pop3ClientID = 0;
  55
  56 #define N ((pop3aggr*)IO->Data)->n
  57
  58 struct CitContext pop3_client_CC;
  59
  60 pthread_mutex_t POP3QueueMutex; /* locks the access to the following vars: */
  61 HashList *POP3QueueRooms = NULL;
  62 HashList *POP3FetchUrls = NULL;
  63
  64 typedef struct pop3aggr pop3aggr;
  65 typedef eNextState(*Pop3ClientHandler)(pop3aggr* RecvMsg);
  66
  67 eNextState POP3_C_Shutdown(AsyncIO *IO);
  68 eNextState POP3_C_Timeout(AsyncIO *IO);
  69 eNextState POP3_C_ConnFail(AsyncIO *IO);
  70 eNextState POP3_C_DNSFail(AsyncIO *IO);
  71 eNextState POP3_C_DispatchReadDone(AsyncIO *IO);
  72 eNextState POP3_C_DispatchWriteDone(AsyncIO *IO);
  73 eNextState POP3_C_Terminate(AsyncIO *IO);
  74 eReadState POP3_C_ReadServerStatus(AsyncIO *IO);
  75 eNextState POP3_C_ReAttachToFetchMessages(AsyncIO *IO);
  76
  77 typedef struct __pop3_room_counter {
  78         int count;
  79         long QRnumber;
  80 }pop3_room_counter;
  81
  82 typedef enum ePOP3_C_States {
  83         ReadGreeting,
  84         GetUserState,
  85         GetPassState,
  86         GetListCommandState,
  87         GetListOneLine,
  88         GetOneMessageIDState,
  89         ReadMessageBodyFollowing,
  90         ReadMessageBody,
  91         GetDeleteState,
  92         ReadQuitState,
  93         POP3C_MaxRead
  94 }ePOP3_C_States;
  95
  96
  97 typedef struct _FetchItem {
  98         long MSGID;
  99         long MSGSize;
 100         StrBuf *MsgUIDL;
 101         StrBuf *MsgUID;
 102         int NeedFetch;
 103         struct CtdlMessage *Msg;
 104 } FetchItem;
 105
 106 void HfreeFetchItem(void *vItem)
 107 {
 108         FetchItem *Item = (FetchItem*) vItem;
 109         FreeStrBuf(&Item->MsgUIDL);
 110         FreeStrBuf(&Item->MsgUID);
 111         free(Item);
 112 }
 113
 114
 115
 116 typedef enum _POP3State {
 117         eCreated,
 118         eGreeting,
 119         eUser,
 120         ePassword,
 121         eListing,
 122         eUseTable,
 123         eGetMsgID,
 124         eGetMsg,
 125         eStoreMsg,
 126         eDelete,
 127         eQuit
 128 } POP3State;
 129
 130 ConstStr POP3States[] = {
 131         {HKEY("Aggregator created")},
 132         {HKEY("Reading Greeting")},
 133         {HKEY("Sending User")},
 134         {HKEY("Sending Password")},
 135         {HKEY("Listing")},
 136         {HKEY("Fetching Usetable")},
 137         {HKEY("Get MSG ID")},
 138         {HKEY("Get Message")},
 139         {HKEY("Store Msg")},
 140         {HKEY("Delete Upstream")},
 141         {HKEY("Quit")}
 142 };
 143
 144 static void SetPOP3State(AsyncIO *IO, POP3State State)
 145 {
 146         CitContext* CCC = IO->CitContext;
 147         if (CCC != NULL)
 148                 memcpy(CCC->cs_clientname, POP3States[State].Key, POP3States[State].len + 1);
 149 }
 150
 151
 152 struct pop3aggr {
 153         AsyncIO  IO;
 154
 155         long n;
 156         double IOStart;
 157         long count;
 158         long RefCount;
 159         DNSQueryParts HostLookup;
 160
 161         long             QRnumber;
 162         HashList        *OtherQRnumbers;
 163
 164         StrBuf          *Url;
 165         StrBuf *pop3user;
 166         StrBuf *pop3pass;
 167         StrBuf *Host;
 168         StrBuf *RoomName; // TODO: fill me
 169         int keep;
 170         time_t interval;
 171         ePOP3_C_States State;
 172         HashList *MsgNumbers;
 173         HashPos *Pos;
 174         FetchItem *CurrMsg;
 175 };
 176
 177 void DeletePOP3Aggregator(void *vptr)
 178 {
 179         pop3aggr *ptr = vptr;
 180         DeleteHashPos(&ptr->Pos);
 181         DeleteHash(&ptr->MsgNumbers);
 182 //      FreeStrBuf(&ptr->rooms);
 183         FreeStrBuf(&ptr->pop3user);
 184         FreeStrBuf(&ptr->pop3pass);
 185         FreeStrBuf(&ptr->Host);
 186         FreeStrBuf(&ptr->RoomName);
 187         FreeURL(&ptr->IO.ConnectMe);
 188         FreeStrBuf(&ptr->Url);
 189         FreeStrBuf(&ptr->IO.IOBuf);
 190         FreeStrBuf(&ptr->IO.SendBuf.Buf);
 191         FreeStrBuf(&ptr->IO.RecvBuf.Buf);
 192         DeleteAsyncMsg(&ptr->IO.ReadMsg);
 193         if (((struct CitContext*)ptr->IO.CitContext)) {
 194                 ((struct CitContext*)ptr->IO.CitContext)->state = CON_IDLE;
 195                 ((struct CitContext*)ptr->IO.CitContext)->kill_me = 1;
 196         }
 197         FreeAsyncIOContents(&ptr->IO);
 198         free(ptr);
 199 }
 200
 201 eNextState FinalizePOP3AggrRun(AsyncIO *IO)
 202 {
 203         HashPos  *It;
 204         pop3aggr *cpptr = (pop3aggr *)IO->Data;
 205
 206         syslog(LOG_INFO,
 207                      "%s@%s: fetched %ld new of %d messages in %fs. bye.",
 208                      ChrPtr(cpptr->pop3user),
 209                      ChrPtr(cpptr->Host),
 210                      cpptr->count,
 211                      GetCount(cpptr->MsgNumbers),
 212                      IO->Now - cpptr->IOStart
 213                 );
 214
 215         It = GetNewHashPos(POP3FetchUrls, 0);
 216         pthread_mutex_lock(&POP3QueueMutex);
 217         {
 218                 if (GetHashPosFromKey(POP3FetchUrls, SKEY(cpptr->Url), It))
 219                         DeleteEntryFromHash(POP3FetchUrls, It);
 220         }
 221         pthread_mutex_unlock(&POP3QueueMutex);
 222         DeleteHashPos(&It);
 223         return eAbort;
 224 }
 225
 226 eNextState FailAggregationRun(AsyncIO *IO)
 227 {
 228         return eAbort;
 229 }
 230
 231 eNextState POP3C_ReadGreeting(pop3aggr *RecvMsg)
 232 {
 233         AsyncIO *IO = &RecvMsg->IO;
 234         SetPOP3State(IO, eGreeting);
 235         /* Read the server greeting */
 236         if (!POP3C_OK) return eTerminateConnection;
 237         else return eSendReply;
 238 }
 239
 240 eNextState POP3C_SendUser(pop3aggr *RecvMsg)
 241 {
 242         AsyncIO *IO = &RecvMsg->IO;
 243         SetPOP3State(IO, eUser);
 244         /* Identify ourselves.  NOTE: we have to append a CR to each command.
 245          *  The LF will automatically be appended by sock_puts().  Believe it
 246          * or not, leaving out the CR will cause problems if the server happens
 247          * to be Exchange, which is so b0rken it actually barfs on
 248          * LF-terminated newlines.
 249          */
 250         StrBufPrintf(RecvMsg->IO.SendBuf.Buf,
 251                      "USER %s\r\n", ChrPtr(RecvMsg->pop3user));
 252         return eReadMessage;
 253 }
 254
 255 eNextState POP3C_GetUserState(pop3aggr *RecvMsg)
 256 {
 257         if (!POP3C_OK) return eTerminateConnection;
 258         else return eSendReply;
 259 }
 260
 261 eNextState POP3C_SendPassword(pop3aggr *RecvMsg)
 262 {
 263         AsyncIO *IO = &RecvMsg->IO;
 264         SetPOP3State(IO, ePassword);
 265         /* Password */
 266         StrBufPrintf(RecvMsg->IO.SendBuf.Buf,
 267                      "PASS %s\r\n", ChrPtr(RecvMsg->pop3pass));
 268         syslog(LOG_DEBUG, "<PASS <password>\n");
 269         return eReadMessage;
 270 }
 271
 272 eNextState POP3C_GetPassState(pop3aggr *RecvMsg)
 273 {
 274         if (!POP3C_OK) return eTerminateConnection;
 275         else return eSendReply;
 276 }
 277
 278 eNextState POP3C_SendListCommand(pop3aggr *RecvMsg)
 279 {
 280         AsyncIO *IO = &RecvMsg->IO;
 281         SetPOP3State(IO, eListing);
 282
 283         /* Get the list of messages */
 284         StrBufPlain(RecvMsg->IO.SendBuf.Buf, HKEY("LIST\r\n"));
 285         return eReadMessage;
 286 }
 287
 288 eNextState POP3C_GetListCommandState(pop3aggr *RecvMsg)
 289 {
 290         if (!POP3C_OK) return eTerminateConnection;
 291         RecvMsg->MsgNumbers = NewHash(1, NULL);
 292         RecvMsg->State++;
 293         return eReadMore;
 294 }
 295
 296
 297 eNextState POP3C_GetListOneLine(pop3aggr *RecvMsg)
 298 {
 299 #if 0
 300         int rc;
 301 #endif
 302         const char *pch;
 303         FetchItem *OneMsg = NULL;
 304
 305         if ((StrLength(RecvMsg->IO.IOBuf) == 1) &&
 306             (ChrPtr(RecvMsg->IO.IOBuf)[0] == '.'))
 307         {
 308                 if (GetCount(RecvMsg->MsgNumbers) == 0)
 309                 {
 310                         ////    RecvMsg->Sate = ReadQuitState;
 311                 }
 312                 else
 313                 {
 314                         RecvMsg->Pos = GetNewHashPos(RecvMsg->MsgNumbers, 0);
 315                 }
 316                 return eSendReply;
 317
 318         }
 319
 320         /*
 321          * work around buggy pop3 servers which send
 322          * empty lines in their listings.
 323         */
 324         if ((StrLength(RecvMsg->IO.IOBuf) == 0) ||
 325             !isdigit(ChrPtr(RecvMsg->IO.IOBuf)[0]))
 326         {
 327                 return eReadMore;
 328         }
 329
 330         OneMsg = (FetchItem*) malloc(sizeof(FetchItem));
 331         memset(OneMsg, 0, sizeof(FetchItem));
 332         OneMsg->MSGID = atol(ChrPtr(RecvMsg->IO.IOBuf));
 333
 334         pch = strchr(ChrPtr(RecvMsg->IO.IOBuf), ' ');
 335         if (pch != NULL)
 336         {
 337                 OneMsg->MSGSize = atol(pch + 1);
 338         }
 339 #if 0
 340         rc = TestValidateHash(RecvMsg->MsgNumbers);
 341         if (rc != 0)
 342                 syslog(LOG_DEBUG, "Hash Invalid: %d\n", rc);
 343 #endif
 344
 345         Put(RecvMsg->MsgNumbers, LKEY(OneMsg->MSGID), OneMsg, HfreeFetchItem);
 346 #if 0
 347         rc = TestValidateHash(RecvMsg->MsgNumbers);
 348         if (rc != 0)
 349                 syslog(LOG_DEBUG, "Hash Invalid: %d\n", rc);
 350 #endif
 351         //RecvMsg->State --; /* read next Line */
 352         return eReadMore;
 353 }
 354
 355 eNextState POP3_FetchNetworkUsetableEntry(AsyncIO *IO)
 356 {
 357         long HKLen;
 358         const char *HKey;
 359         void *vData;
 360         pop3aggr *RecvMsg = (pop3aggr *) IO->Data;
 361         time_t seenstamp = 0;
 362
 363         SetPOP3State(IO, eUseTable);
 364
 365         if((RecvMsg->Pos != NULL) &&
 366            GetNextHashPos(RecvMsg->MsgNumbers,
 367                           RecvMsg->Pos,
 368                           &HKLen,
 369                           &HKey,
 370                           &vData))
 371         {
 372                 if (server_shutting_down)
 373                         return eAbort;
 374
 375                 RecvMsg->CurrMsg = (FetchItem*)vData;
 376
 377                 seenstamp = CheckIfAlreadySeen("POP3 Item Seen",
 378                                                RecvMsg->CurrMsg->MsgUID,
 379                                                EvGetNow(IO),
 380                                                EvGetNow(IO) - USETABLE_ANTIEXPIRE,
 381                                                eCheckUpdate,
 382                                                IO->ID, CCID);
 383                 if (seenstamp != 0)
 384                 {
 385                         /* Item has already been seen */
 386                         RecvMsg->CurrMsg->NeedFetch = 0;
 387                 }
 388                 else
 389                 {
 390                         syslog(LOG_DEBUG, "NO\n");
 391                         RecvMsg->CurrMsg->NeedFetch = 1;
 392                 }
 393                 return NextDBOperation(&RecvMsg->IO,
 394                                        POP3_FetchNetworkUsetableEntry);
 395         }
 396         else
 397         {
 398                 /* ok, now we know them all,
 399                  * continue with reading the actual messages. */
 400                 DeleteHashPos(&RecvMsg->Pos);
 401                 return DBQueueEventContext(IO, POP3_C_ReAttachToFetchMessages);
 402         }
 403 }
 404
 405 eNextState POP3C_GetOneMessagID(pop3aggr *RecvMsg)
 406 {
 407         AsyncIO *IO = &RecvMsg->IO;
 408         long HKLen;
 409         const char *HKey;
 410         void *vData;
 411
 412         SetPOP3State(IO, eGetMsgID);
 413 #if 0
 414         int rc;
 415         rc = TestValidateHash(RecvMsg->MsgNumbers);
 416         if (rc != 0)
 417                 syslog(LOG_DEBUG, "Hash Invalid: %d\n", rc);
 418 #endif
 419         if((RecvMsg->Pos != NULL) &&
 420            GetNextHashPos(RecvMsg->MsgNumbers,
 421                           RecvMsg->Pos,
 422                           &HKLen, &HKey,
 423                           &vData))
 424         {
 425                 RecvMsg->CurrMsg = (FetchItem*) vData;
 426                 /* Find out the UIDL of the message,
 427                  * to determine whether we've already downloaded it */
 428                 StrBufPrintf(RecvMsg->IO.SendBuf.Buf,
 429                              "UIDL %ld\r\n", RecvMsg->CurrMsg->MSGID);
 430         }
 431         else
 432         {
 433                 RecvMsg->State++;
 434                 DeleteHashPos(&RecvMsg->Pos);
 435                 /// done receiving uidls.. start looking them up now.
 436                 RecvMsg->Pos = GetNewHashPos(RecvMsg->MsgNumbers, 0);
 437                 return EventQueueDBOperation(&RecvMsg->IO,
 438                                              POP3_FetchNetworkUsetableEntry,
 439                                              0);
 440         }
 441         return eReadMore; /* TODO */
 442 }
 443
 444 eNextState POP3C_GetOneMessageIDState(pop3aggr *RecvMsg)
 445 {
 446 #if 0
 447         int rc;
 448         rc = TestValidateHash(RecvMsg->MsgNumbers);
 449         if (rc != 0)
 450                 syslog(LOG_DEBUG, "Hash Invalid: %d\n", rc);
 451 #endif
 452
 453         if (!POP3C_OK) return eTerminateConnection;
 454         RecvMsg->CurrMsg->MsgUIDL =
 455                 NewStrBufPlain(NULL, StrLength(RecvMsg->IO.IOBuf));
 456         RecvMsg->CurrMsg->MsgUID =
 457                 NewStrBufPlain(NULL, StrLength(RecvMsg->IO.IOBuf) * 2);
 458
 459         StrBufExtract_token(RecvMsg->CurrMsg->MsgUIDL,
 460                             RecvMsg->IO.IOBuf, 2, ' ');
 461
 462         StrBufPrintf(RecvMsg->CurrMsg->MsgUID,
 463                      "pop3/%s/%s:%s@%s",
 464                      ChrPtr(RecvMsg->RoomName),
 465                      ChrPtr(RecvMsg->CurrMsg->MsgUIDL),
 466                      RecvMsg->IO.ConnectMe->User,
 467                      RecvMsg->IO.ConnectMe->Host);
 468         RecvMsg->State --;
 469         return eSendReply;
 470 }
 471
 472
 473 eNextState POP3C_SendGetOneMsg(pop3aggr *RecvMsg)
 474 {
 475         AsyncIO *IO = &RecvMsg->IO;
 476         long HKLen;
 477         const char *HKey;
 478         void *vData;
 479
 480         SetPOP3State(IO, eGetMsg);
 481
 482         syslog(LOG_DEBUG, "fast forwarding to the next unknown message");
 483
 484         RecvMsg->CurrMsg = NULL;
 485         while ((RecvMsg->Pos != NULL) &&
 486                GetNextHashPos(RecvMsg->MsgNumbers,
 487                               RecvMsg->Pos,
 488                               &HKLen, &HKey,
 489                               &vData) &&
 490                (RecvMsg->CurrMsg = (FetchItem*) vData,
 491                 RecvMsg->CurrMsg->NeedFetch == 0))
 492         {}
 493
 494         if ((RecvMsg->CurrMsg != NULL ) && (RecvMsg->CurrMsg->NeedFetch == 1))
 495         {
 496                 syslog(LOG_DEBUG, "fetching next");
 497                 /* Message has not been seen.
 498                  * Tell the server to fetch the message... */
 499                 StrBufPrintf(RecvMsg->IO.SendBuf.Buf,
 500                              "RETR %ld\r\n", RecvMsg->CurrMsg->MSGID);
 501                 return eReadMessage;
 502         }
 503         else {
 504                 syslog(LOG_DEBUG, "no more messages to fetch.");
 505                 RecvMsg->State = ReadQuitState;
 506                 return POP3_C_DispatchWriteDone(&RecvMsg->IO);
 507         }
 508 }
 509
 510
 511 eNextState POP3C_ReadMessageBodyFollowing(pop3aggr *RecvMsg)
 512 {
 513         if (!POP3C_OK) return eTerminateConnection;
 514         RecvMsg->IO.ReadMsg = NewAsyncMsg(HKEY("."),
 515                                           RecvMsg->CurrMsg->MSGSize,
 516                                           CtdlGetConfigLong("c_maxmsglen"),
 517                                           NULL, -1,
 518                                           1);
 519
 520         return eReadPayload;
 521 }
 522
 523
 524 eNextState POP3C_StoreMsgRead(AsyncIO *IO)
 525 {
 526         pop3aggr *RecvMsg = (pop3aggr *) IO->Data;
 527
 528         SetPOP3State(IO, eStoreMsg);
 529
 530         syslog(LOG_DEBUG,
 531                        "MARKING: %s as seen: ",
 532                        ChrPtr(RecvMsg->CurrMsg->MsgUID));
 533         CheckIfAlreadySeen("POP3 Item Seen",
 534                            RecvMsg->CurrMsg->MsgUID,
 535                            EvGetNow(IO),
 536                            EvGetNow(IO) - USETABLE_ANTIEXPIRE,
 537                            eWrite,
 538                            IO->ID, CCID);
 539
 540         return DBQueueEventContext(&RecvMsg->IO, POP3_C_ReAttachToFetchMessages);
 541 }
 542 eNextState POP3C_SaveMsg(AsyncIO *IO)
 543 {
 544         long msgnum;
 545         pop3aggr *RecvMsg = (pop3aggr *) IO->Data;
 546
 547         /* Do Something With It (tm) */
 548         msgnum = CtdlSubmitMsg(RecvMsg->CurrMsg->Msg,
 549                                NULL,
 550                                ChrPtr(RecvMsg->RoomName),
 551                                0);
 552         if (msgnum > 0L)
 553         {
 554                 /* Message has been committed to the store
 555                  * write the uidl to the use table
 556                  * so we don't fetch this message again
 557                  */
 558         }
 559         CM_Free(RecvMsg->CurrMsg->Msg);
 560
 561         RecvMsg->count ++;
 562         return NextDBOperation(&RecvMsg->IO, POP3C_StoreMsgRead);
 563 }
 564
 565 eNextState POP3C_ReadMessageBody(pop3aggr *RecvMsg)
 566 {
 567         syslog(LOG_DEBUG, "Converting message...");
 568         RecvMsg->CurrMsg->Msg =
 569                 convert_internet_message_buf(&RecvMsg->IO.ReadMsg->MsgBuf);
 570         return EventQueueDBOperation(&RecvMsg->IO, POP3C_SaveMsg, 0);
 571 }
 572
 573 eNextState POP3C_SendDelete(pop3aggr *RecvMsg)
 574 {
 575         AsyncIO *IO = &RecvMsg->IO;
 576
 577         SetPOP3State(IO, eDelete);
 578
 579         if (!RecvMsg->keep) {
 580                 StrBufPrintf(RecvMsg->IO.SendBuf.Buf,
 581                              "DELE %ld\r\n", RecvMsg->CurrMsg->MSGID);
 582                 return eReadMessage;
 583         }
 584         else {
 585                 RecvMsg->State = ReadMessageBodyFollowing;
 586                 return POP3_C_DispatchWriteDone(&RecvMsg->IO);
 587         }
 588 }
 589 eNextState POP3C_ReadDeleteState(pop3aggr *RecvMsg)
 590 {
 591         RecvMsg->State = GetOneMessageIDState;
 592         return POP3_C_DispatchWriteDone(&RecvMsg->IO);
 593 }
 594
 595 eNextState POP3C_SendQuit(pop3aggr *RecvMsg)
 596 {
 597         AsyncIO *IO = &RecvMsg->IO;
 598         SetPOP3State(IO, eQuit);
 599
 600         /* Log out */
 601         StrBufPlain(RecvMsg->IO.SendBuf.Buf,
 602                     HKEY("QUIT\r\n3)"));
 603         return eReadMessage;
 604 }
 605
 606
 607 eNextState POP3C_ReadQuitState(pop3aggr *RecvMsg)
 608 {
 609         return eTerminateConnection;
 610 }
 611
 612 const long POP3_C_ConnTimeout = 1000;
 613 const long DefaultPOP3Port = 110;
 614
 615 Pop3ClientHandler POP3C_ReadHandlers[] = {
 616         POP3C_ReadGreeting,
 617         POP3C_GetUserState,
 618         POP3C_GetPassState,
 619         POP3C_GetListCommandState,
 620         POP3C_GetListOneLine,
 621         POP3C_GetOneMessageIDState,
 622         POP3C_ReadMessageBodyFollowing,
 623         POP3C_ReadMessageBody,
 624         POP3C_ReadDeleteState,
 625         POP3C_ReadQuitState,
 626 };
 627
 628 const long POP3_C_SendTimeouts[POP3C_MaxRead] = {
 629         100,
 630         100,
 631         100,
 632         100,
 633         100,
 634         100,
 635         100,
 636         100
 637 };
 638 const ConstStr POP3C_ReadErrors[POP3C_MaxRead] = {
 639         {HKEY("Connection broken during ")},
 640         {HKEY("Connection broken during ")},
 641         {HKEY("Connection broken during ")},
 642         {HKEY("Connection broken during ")},
 643         {HKEY("Connection broken during ")},
 644         {HKEY("Connection broken during ")},
 645         {HKEY("Connection broken during ")},
 646         {HKEY("Connection broken during ")}
 647 };
 648
 649 Pop3ClientHandler POP3C_SendHandlers[] = {
 650         NULL, /* we don't send a greeting */
 651         POP3C_SendUser,
 652         POP3C_SendPassword,
 653         POP3C_SendListCommand,
 654         NULL,
 655         POP3C_GetOneMessagID,
 656         POP3C_SendGetOneMsg,
 657         NULL,
 658         POP3C_SendDelete,
 659         POP3C_SendQuit
 660 };
 661
 662 const long POP3_C_ReadTimeouts[] = {
 663         100,
 664         100,
 665         100,
 666         100,
 667         100,
 668         100,
 669         100,
 670         100,
 671         100,
 672         100
 673 };
 674 /*****************************************************************************/
 675 /*                     POP3 CLIENT DISPATCHER                                */
 676 /*****************************************************************************/
 677
 678 void POP3SetTimeout(eNextState NextTCPState, pop3aggr *pMsg)
 679 {
 680         double Timeout = 0.0;
 681
 682         syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
 683
 684         switch (NextTCPState) {
 685         case eSendFile:
 686         case eSendReply:
 687         case eSendMore:
 688                 Timeout = POP3_C_SendTimeouts[pMsg->State];
 689 /*
 690   if (pMsg->State == eDATABody) {
 691   / * if we're sending a huge message, we need more time. * /
 692   Timeout += StrLength(pMsg->msgtext) / 1024;
 693   }
 694 */
 695                 break;
 696         case eReadFile:
 697         case eReadMessage:
 698                 Timeout = POP3_C_ReadTimeouts[pMsg->State];
 699 /*
 700   if (pMsg->State == eDATATerminateBody) {
 701   / *
 702   * some mailservers take a nap before accepting the message
 703   * content inspection and such.
 704   * /
 705   Timeout += StrLength(pMsg->msgtext) / 1024;
 706   }
 707 */
 708                 break;
 709         case eReadPayload:
 710                 Timeout = 100000;
 711                 /* TODO!!! */
 712                 break;
 713         case eSendDNSQuery:
 714         case eReadDNSReply:
 715         case eConnect:
 716         case eTerminateConnection:
 717         case eDBQuery:
 718         case eAbort:
 719         case eReadMore://// TODO
 720                 return;
 721         }
 722         SetNextTimeout(&pMsg->IO, Timeout);
 723 }
 724 eNextState POP3_C_DispatchReadDone(AsyncIO *IO)
 725 {
 726 /*      syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__); to noisy anyways. */
 727         pop3aggr *pMsg = IO->Data;
 728         eNextState rc;
 729
 730         rc = POP3C_ReadHandlers[pMsg->State](pMsg);
 731         if (rc != eReadMore)
 732             pMsg->State++;
 733         POP3SetTimeout(rc, pMsg);
 734         return rc;
 735 }
 736 eNextState POP3_C_DispatchWriteDone(AsyncIO *IO)
 737 {
 738         pop3aggr *pMsg = IO->Data;
 739         eNextState rc;
 740
 741 /*      syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__); to noisy anyways. */
 742         rc = POP3C_SendHandlers[pMsg->State](pMsg);
 743         POP3SetTimeout(rc, pMsg);
 744         return rc;
 745 }
 746
 747
 748 /*****************************************************************************/
 749 /*                     POP3 CLIENT ERROR CATCHERS                            */
 750 /*****************************************************************************/
 751 eNextState POP3_C_Terminate(AsyncIO *IO)
 752 {
 753 ///     pop3aggr *pMsg = (pop3aggr *)IO->Data;
 754
 755         syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
 756         FinalizePOP3AggrRun(IO);
 757         return eAbort;
 758 }
 759 eNextState POP3_C_TerminateDB(AsyncIO *IO)
 760 {
 761 ///     pop3aggr *pMsg = (pop3aggr *)IO->Data;
 762
 763         syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
 764         FinalizePOP3AggrRun(IO);
 765         return eAbort;
 766 }
 767 eNextState POP3_C_Timeout(AsyncIO *IO)
 768 {
 769         pop3aggr *pMsg = IO->Data;
 770
 771         syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
 772         StrBufPlain(IO->ErrMsg, CKEY(POP3C_ReadErrors[pMsg->State]));
 773         return FailAggregationRun(IO);
 774 }
 775 eNextState POP3_C_ConnFail(AsyncIO *IO)
 776 {
 777         pop3aggr *pMsg = (pop3aggr *)IO->Data;
 778
 779         syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
 780         StrBufPlain(IO->ErrMsg, CKEY(POP3C_ReadErrors[pMsg->State]));
 781         return FailAggregationRun(IO);
 782 }
 783 eNextState POP3_C_DNSFail(AsyncIO *IO)
 784 {
 785         pop3aggr *pMsg = (pop3aggr *)IO->Data;
 786
 787         syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
 788         StrBufPlain(IO->ErrMsg, CKEY(POP3C_ReadErrors[pMsg->State]));
 789         return FailAggregationRun(IO);
 790 }
 791 eNextState POP3_C_Shutdown(AsyncIO *IO)
 792 {
 793         syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
 794 ////    pop3aggr *pMsg = IO->Data;
 795
 796 ////pMsg->MyQEntry->Status = 3;
 797 ///StrBufPlain(pMsg->MyQEntry->StatusMessage, HKEY("server shutdown during message retrieval."));
 798         FinalizePOP3AggrRun(IO);
 799         return eAbort;
 800 }
 801
 802
 803 /**
 804  * @brief lineread Handler; understands when to read more POP3 lines,
 805  *   and when this is a one-lined reply.
 806  */
 807 eReadState POP3_C_ReadServerStatus(AsyncIO *IO)
 808 {
 809         eReadState Finished = eBufferNotEmpty;
 810
 811         switch (IO->NextState) {
 812         case eSendDNSQuery:
 813         case eReadDNSReply:
 814         case eDBQuery:
 815         case eConnect:
 816         case eTerminateConnection:
 817         case eAbort:
 818                 Finished = eReadFail;
 819                 break;
 820         case eSendFile:
 821         case eSendReply:
 822         case eSendMore:
 823         case eReadMore:
 824         case eReadMessage:
 825                 Finished = StrBufChunkSipLine(IO->IOBuf, &IO->RecvBuf);
 826                 break;
 827         case eReadFile:
 828         case eReadPayload:
 829                 Finished = CtdlReadMessageBodyAsync(IO);
 830                 break;
 831         }
 832         return Finished;
 833 }
 834
 835 /*****************************************************************************
 836  * So we connect our Server IP here.                                         *
 837  *****************************************************************************/
 838 eNextState POP3_C_ReAttachToFetchMessages(AsyncIO *IO)
 839 {
 840         pop3aggr *cpptr = IO->Data;
 841
 842         syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
 843 ////??? cpptr->State ++;
 844         if (cpptr->Pos == NULL)
 845                 cpptr->Pos = GetNewHashPos(cpptr->MsgNumbers, 0);
 846
 847         POP3_C_DispatchWriteDone(IO);
 848         ReAttachIO(IO, cpptr, 0);
 849         IO->NextState = eReadMessage;
 850         return IO->NextState;
 851 }
 852
 853 eNextState pop3_connect_ip(AsyncIO *IO)
 854 {
 855         pop3aggr *cpptr = IO->Data;
 856
 857         if (cpptr->IOStart == 0.0) /* whith or without DNS? */
 858                 cpptr->IOStart = IO->Now;
 859
 860         syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
 861
 862         return EvConnectSock(IO,
 863                              POP3_C_ConnTimeout,
 864                              POP3_C_ReadTimeouts[0],
 865                              1);
 866 }
 867
 868 eNextState pop3_get_one_host_ip_done(AsyncIO *IO)
 869 {
 870         pop3aggr *cpptr = IO->Data;
 871         struct hostent *hostent;
 872
 873         QueryCbDone(IO);
 874
 875         hostent = cpptr->HostLookup.VParsedDNSReply;
 876         if ((cpptr->HostLookup.DNSStatus == ARES_SUCCESS) &&
 877             (hostent != NULL) ) {
 878                 memset(&cpptr->IO.ConnectMe->Addr, 0, sizeof(struct in6_addr));
 879                 if (cpptr->IO.ConnectMe->IPv6) {
 880                         memcpy(&cpptr->IO.ConnectMe->Addr.sin6_addr.s6_addr,
 881                                &hostent->h_addr_list[0],
 882                                sizeof(struct in6_addr));
 883
 884                         cpptr->IO.ConnectMe->Addr.sin6_family =
 885                                 hostent->h_addrtype;
 886                         cpptr->IO.ConnectMe->Addr.sin6_port   =
 887                                 htons(DefaultPOP3Port);
 888                 }
 889                 else {
 890                         struct sockaddr_in *addr =
 891                                 (struct sockaddr_in*)
 892                                 &cpptr->IO.ConnectMe->Addr;
 893
 894                         memcpy(&addr->sin_addr.s_addr,
 895                                hostent->h_addr_list[0],
 896                                sizeof(uint32_t));
 897
 898                         addr->sin_family = hostent->h_addrtype;
 899                         addr->sin_port   = htons(DefaultPOP3Port);
 900                 }
 901                 return pop3_connect_ip(IO);
 902         }
 903         else
 904                 return eAbort;
 905 }
 906
 907 eNextState pop3_get_one_host_ip(AsyncIO *IO)
 908 {
 909         pop3aggr *cpptr = IO->Data;
 910
 911         cpptr->IOStart = IO->Now;
 912
 913         syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
 914
 915         syslog(LOG_DEBUG,
 916                        "POP3 client[%ld]: looking up %s-Record %s : %d ...\n",
 917                        cpptr->n,
 918                        (cpptr->IO.ConnectMe->IPv6)? "aaaa": "a",
 919                        cpptr->IO.ConnectMe->Host,
 920                        cpptr->IO.ConnectMe->Port);
 921
 922         QueueQuery((cpptr->IO.ConnectMe->IPv6)? ns_t_aaaa : ns_t_a,
 923                    cpptr->IO.ConnectMe->Host,
 924                    &cpptr->IO,
 925                    &cpptr->HostLookup,
 926                    pop3_get_one_host_ip_done);
 927         IO->NextState = eReadDNSReply;
 928         return IO->NextState;
 929 }
 930
 931
 932
 933 int pop3_do_fetching(pop3aggr *cpptr)
 934 {
 935         AsyncIO *IO = &cpptr->IO;
 936
 937         InitIOStruct(IO,
 938                      cpptr,
 939                      eReadMessage,
 940                      POP3_C_ReadServerStatus,
 941                      POP3_C_DNSFail,
 942                      POP3_C_DispatchWriteDone,
 943                      POP3_C_DispatchReadDone,
 944                      POP3_C_Terminate,
 945                      POP3_C_TerminateDB,
 946                      POP3_C_ConnFail,
 947                      POP3_C_Timeout,
 948                      POP3_C_Shutdown);
 949
 950         safestrncpy(((CitContext *)cpptr->IO.CitContext)->cs_host,
 951                     ChrPtr(cpptr->Url),
 952                     sizeof(((CitContext *)cpptr->IO.CitContext)->cs_host));
 953
 954         if (cpptr->IO.ConnectMe->IsIP) {
 955                 QueueEventContext(&cpptr->IO,
 956                                   pop3_connect_ip);
 957         }
 958         else {
 959                 QueueEventContext(&cpptr->IO,
 960                                   pop3_get_one_host_ip);
 961         }
 962         return 1;
 963 }
 964
 965 /*
 966  * Scan a room's netconfig to determine whether it requires POP3 aggregation
 967  */
 968 void pop3client_scan_room(struct ctdlroom *qrbuf, void *data, OneRoomNetCfg *OneRNCFG)
 969 {
 970         const RoomNetCfgLine *pLine;
 971         void *vptr;
 972
 973         pthread_mutex_lock(&POP3QueueMutex);
 974         if (GetHash(POP3QueueRooms, LKEY(qrbuf->QRnumber), &vptr))
 975         {
 976                 pthread_mutex_unlock(&POP3QueueMutex);
 977                 syslog(LOG_DEBUG,
 978                               "pop3client: [%ld] %s already in progress.",
 979                               qrbuf->QRnumber,
 980                               qrbuf->QRname);
 981                 return;
 982         }
 983         pthread_mutex_unlock(&POP3QueueMutex);
 984
 985         if (server_shutting_down) return;
 986
 987         pLine = OneRNCFG->NetConfigs[pop3client];
 988
 989         while (pLine != NULL)
 990         {
 991                 pop3aggr *cptr;
 992
 993                 cptr = (pop3aggr *) malloc(sizeof(pop3aggr));
 994                 memset(cptr, 0, sizeof(pop3aggr));
 995                 ///TODO do we need this? cptr->roomlist_parts=1;
 996                 cptr->RoomName = NewStrBufPlain(qrbuf->QRname, -1);
 997                 cptr->pop3user = NewStrBufDup(pLine->Value[1]);
 998                 cptr->pop3pass = NewStrBufDup(pLine->Value[2]);
 999                 cptr->Url = NewStrBuf();
1000                 cptr->Host = NewStrBufDup(pLine->Value[0]);
1001
1002                 cptr->keep = atol(ChrPtr(pLine->Value[3]));
1003                 cptr->interval = atol(ChrPtr(pLine->Value[4]));
1004
1005                 StrBufAppendBufPlain(cptr->Url, HKEY("pop3://"), 0);
1006                 StrBufUrlescUPAppend(cptr->Url, cptr->pop3user, NULL);
1007                 StrBufAppendBufPlain(cptr->Url, HKEY(":"), 0);
1008                 StrBufUrlescUPAppend(cptr->Url, cptr->pop3pass, NULL);
1009                 StrBufAppendBufPlain(cptr->Url, HKEY("@"), 0);
1010                 StrBufAppendBuf(cptr->Url, cptr->Host, 0);
1011                 StrBufAppendBufPlain(cptr->Url, HKEY("/"), 0);
1012                 StrBufUrlescAppend(cptr->Url, cptr->RoomName, NULL);
1013
1014                 ParseURL(&cptr->IO.ConnectMe, cptr->Url, 110);
1015
1016
1017 #if 0
1018 /* todo: we need to reunite the url to be shure. */
1019
1020                 pthread_mutex_lock(&POP3ueueMutex);
1021                 GetHash(POP3FetchUrls, SKEY(ptr->Url), &vptr);
1022                 use_this_cptr = (pop3aggr *)vptr;
1023
1024                 if (use_this_rncptr != NULL)
1025                 {
1026                         /* mustn't attach to an active session */
1027                         if (use_this_cptr->RefCount > 0)
1028                         {
1029                                 DeletePOP3Cfg(cptr);
1030 ///                                             Count->count--;
1031                         }
1032                         else
1033                         {
1034                                 long *QRnumber;
1035                                 StrBufAppendBufPlain(
1036                                         use_this_cptr->rooms,
1037                                         qrbuf->QRname,
1038                                         -1, 0);
1039                                 if (use_this_cptr->roomlist_parts == 1)
1040                                 {
1041                                         use_this_cptr->OtherQRnumbers
1042                                                 = NewHash(1, lFlathash);
1043                                 }
1044                                 QRnumber = (long*)malloc(sizeof(long));
1045                                 *QRnumber = qrbuf->QRnumber;
1046                                 Put(use_this_cptr->OtherQRnumbers,
1047                                     LKEY(qrbuf->QRnumber),
1048                                     QRnumber,
1049                                     NULL);
1050
1051                                 use_this_cptr->roomlist_parts++;
1052                         }
1053                         pthread_mutex_unlock(&POP3QueueMutex);
1054                         continue;
1055                 }
1056                 pthread_mutex_unlock(&RSSQueueMutex);
1057 #endif
1058                 cptr->n = Pop3ClientID++;
1059                 pthread_mutex_lock(&POP3QueueMutex);
1060                 Put(POP3FetchUrls,
1061                     SKEY(cptr->Url),
1062                     cptr,
1063                     DeletePOP3Aggregator);
1064
1065                 pthread_mutex_unlock(&POP3QueueMutex);
1066                 pLine = pLine->next;
1067
1068         }
1069 }
1070
1071 static int doing_pop3client = 0;
1072
1073 void pop3client_scan(void) {
1074         static time_t last_run = 0L;
1075         time_t fastest_scan;
1076         HashPos *it;
1077         long len;
1078         const char *Key;
1079         void *vrptr;
1080         pop3aggr *cptr;
1081
1082         become_session(&pop3_client_CC);
1083
1084         if (CtdlGetConfigLong("c_pop3_fastest") < CtdlGetConfigLong("c_pop3_fetch"))
1085                 fastest_scan = CtdlGetConfigLong("c_pop3_fastest");
1086         else
1087                 fastest_scan = CtdlGetConfigLong("c_pop3_fetch");
1088
1089         /*
1090          * Run POP3 aggregation no more frequently than once every n seconds
1091          */
1092         if ( (time(NULL) - last_run) < fastest_scan ) {
1093                 return;
1094         }
1095
1096         /*
1097          * This is a simple concurrency check to make sure only one pop3client
1098          * run is done at a time.  We could do this with a mutex, but since we
1099          * don't really require extremely fine granularity here, we'll do it
1100          * with a static variable instead.
1101          */
1102         if (doing_pop3client) return;
1103         doing_pop3client = 1;
1104
1105         syslog(LOG_DEBUG, "pop3client started");
1106         CtdlForEachNetCfgRoom(pop3client_scan_room, NULL);
1107
1108         pthread_mutex_lock(&POP3QueueMutex);
1109         it = GetNewHashPos(POP3FetchUrls, 0);
1110         while (!server_shutting_down &&
1111                GetNextHashPos(POP3FetchUrls, it, &len, &Key, &vrptr) &&
1112                (vrptr != NULL)) {
1113                 cptr = (pop3aggr *)vrptr;
1114                 if (cptr->RefCount == 0)
1115                         if (!pop3_do_fetching(cptr))
1116                                 DeletePOP3Aggregator(cptr);////TODO
1117
1118 /*
1119         if ((palist->interval && time(NULL) > (last_run + palist->interval))
1120                         || (time(NULL) > last_run + CtdlGetConfigLong("c_pop3_fetch")))
1121                         pop3_do_fetching(palist->roomname, palist->pop3host,
1122                         palist->pop3user, palist->pop3pass, palist->keep);
1123                 pptr = palist;
1124                 palist = palist->next;
1125                 free(pptr);
1126 */
1127         }
1128         DeleteHashPos(&it);
1129         pthread_mutex_unlock(&POP3QueueMutex);
1130
1131         syslog(LOG_DEBUG, "pop3client ended");
1132         last_run = time(NULL);
1133         doing_pop3client = 0;
1134 }
1135
1136
1137 void pop3_cleanup(void)
1138 {
1139         /* citthread_mutex_destroy(&POP3QueueMutex); TODO */
1140         while (doing_pop3client != 0) ;
1141         DeleteHash(&POP3FetchUrls);
1142         DeleteHash(&POP3QueueRooms);
1143 }
1144
1145
1146
1147 CTDL_MODULE_INIT(pop3client)
1148 {
1149         if (!threading)
1150         {
1151                 CtdlFillSystemContext(&pop3_client_CC, "POP3aggr");
1152                 CtdlREGISTERRoomCfgType(pop3client, ParseGeneric, 0, 5, SerializeGeneric, DeleteGenericCfgLine);
1153                 pthread_mutex_init(&POP3QueueMutex, NULL);
1154                 POP3QueueRooms = NewHash(1, lFlathash);
1155                 POP3FetchUrls = NewHash(1, NULL);
1156                 CtdlRegisterSessionHook(pop3client_scan, EVT_TIMER, PRIO_AGGR + 50);
1157                 CtdlRegisterEVCleanupHook(pop3_cleanup);
1158         }
1159
1160         /* return our module id for the log */
1161         return "pop3client";
1162 }