citadel/modules/pop3client/serv_pop3client.c

   1 /*
   2  * Consolidate mail from remote POP3 accounts.
   3  *
   4  * Copyright (c) 2007-2011 by the citadel.org team
   5  *
   6  * This program is open source software; you can redistribute it and/or
   7  * modify it under the terms of the GNU General Public License as published
   8  * by the Free Software Foundation; either version 3 of the License, or
   9  * (at your option) any later version.
  10  *
  11  * This program is distributed in the hope that it will be useful,
  12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14  * GNU General Public License for more details.
  15  *
  16  * You should have received a copy of the GNU General Public License
  17  * along with this program; if not, write to the Free Software
  18  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
  19  */
  20
  21 #include <stdlib.h>
  22 #include <unistd.h>
  23 #include <stdio.h>
  24
  25 #if TIME_WITH_SYS_TIME
  26 # include <sys/time.h>
  27 # include <time.h>
  28 #else
  29 # if HAVE_SYS_TIME_H
  30 #  include <sys/time.h>
  31 # else
  32 #  include <time.h>
  33 # endif
  34 #endif
  35
  36 #include <ctype.h>
  37 #include <string.h>
  38 #include <errno.h>
  39 #include <sys/types.h>
  40 #include <sys/stat.h>
  41 #include <libcitadel.h>
  42 #include "citadel.h"
  43 #include "server.h"
  44 #include "citserver.h"
  45 #include "support.h"
  46 #include "config.h"
  47 #include "ctdl_module.h"
  48 #include "clientsocket.h"
  49 #include "msgbase.h"
  50 #include "internet_addressing.h"
  51 #include "database.h"
  52 #include "citadel_dirs.h"
  53 #include "event_client.h"
  54
  55
  56 #define POP3C_OK (strncasecmp(ChrPtr(RecvMsg->IO.IOBuf), "+OK", 3) == 0)
  57 int Pop3ClientID = 0;
  58 int POP3ClientDebugEnabled = 0;
  59
  60 #define N ((pop3aggr*)IO->Data)->n
  61
  62 #define DBGLOG(LEVEL) if ((LEVEL != LOG_DEBUG) || (POP3ClientDebugEnabled != 0))
  63
  64 #define EVP3C_syslog(LEVEL, FORMAT, ...)                                \
  65         DBGLOG(LEVEL) syslog(LEVEL,                                     \
  66                              "IO[%ld]CC[%d][%ld]POP3: " FORMAT,         \
  67                              IO->ID, CCID, N, __VA_ARGS__)
  68
  69 #define EVP3CM_syslog(LEVEL, FORMAT)                                    \
  70         DBGLOG(LEVEL) syslog(LEVEL,                                     \
  71                              "IO[%ld]CC[%d][%ld]POP3: " FORMAT,         \
  72                              IO->ID, CCID, N)
  73
  74 #define EVP3CQ_syslog(LEVEL, FORMAT, ...)                               \
  75         DBGLOG(LEVEL) syslog(LEVEL,                                     \
  76                              "P3Q:" FORMAT,                             \
  77                              __VA_ARGS__)
  78
  79 #define EVP3CQM_syslog(LEVEL, FORMAT)                                   \
  80         DBGLOG(LEVEL) syslog(LEVEL,                                     \
  81                              "P3Q" FORMAT                               \
  82                 )
  83
  84 #define EVP3CCS_syslog(LEVEL, FORMAT, ...)                              \
  85         DBGLOG(LEVEL) syslog(LEVEL, "IO[%ld][%ld]POP3: " FORMAT,        \
  86                              IO->ID, N, __VA_ARGS__)
  87
  88 #define EVP3CCSM_syslog(LEVEL, FORMAT)                                  \
  89         DBGLOG(LEVEL) syslog(LEVEL, "IO[%ld][%ld]POP3: " FORMAT,        \
  90                              IO->ID, N)
  91
  92 #define POP3C_DBG_SEND()                                                \
  93         EVP3C_syslog(LOG_DEBUG,                                         \
  94                      "IO[%ld]CC[%d][%ld]POP3: > %s\n",                  \
  95                      IO->ID, CCID, N,                                   \
  96                      ChrPtr(RecvMsg->IO.SendBuf.Buf))
  97
  98 #define POP3C_DBG_READ()                                                \
  99         EVP3C_syslog(LOG_DEBUG,                                         \
 100                      "IO[%ld]CC[%d][%ld]POP3: < %s\n",                  \
 101                      IO->ID, CCID, N,                                   \
 102                      ChrPtr(RecvMsg->IO.IOBuf))
 103
 104
 105 struct CitContext pop3_client_CC;
 106
 107 pthread_mutex_t POP3QueueMutex; /* locks the access to the following vars: */
 108 HashList *POP3QueueRooms = NULL;
 109 HashList *POP3FetchUrls = NULL;
 110
 111 typedef struct pop3aggr pop3aggr;
 112 typedef eNextState(*Pop3ClientHandler)(pop3aggr* RecvMsg);
 113
 114 eNextState POP3_C_Shutdown(AsyncIO *IO);
 115 eNextState POP3_C_Timeout(AsyncIO *IO);
 116 eNextState POP3_C_ConnFail(AsyncIO *IO);
 117 eNextState POP3_C_DNSFail(AsyncIO *IO);
 118 eNextState POP3_C_DispatchReadDone(AsyncIO *IO);
 119 eNextState POP3_C_DispatchWriteDone(AsyncIO *IO);
 120 eNextState POP3_C_Terminate(AsyncIO *IO);
 121 eReadState POP3_C_ReadServerStatus(AsyncIO *IO);
 122 eNextState POP3_C_ReAttachToFetchMessages(AsyncIO *IO);
 123
 124 typedef struct __pop3_room_counter {
 125         int count;
 126         long QRnumber;
 127 }pop3_room_counter;
 128
 129 typedef enum ePOP3_C_States {
 130         ReadGreeting,
 131         GetUserState,
 132         GetPassState,
 133         GetListCommandState,
 134         GetListOneLine,
 135         GetOneMessageIDState,
 136         ReadMessageBodyFollowing,
 137         ReadMessageBody,
 138         GetDeleteState,
 139         ReadQuitState,
 140         POP3C_MaxRead
 141 }ePOP3_C_States;
 142
 143
 144 typedef struct _FetchItem {
 145         long MSGID;
 146         long MSGSize;
 147         StrBuf *MsgUIDL;
 148         StrBuf *MsgUID;
 149         int NeedFetch;
 150         struct CtdlMessage *Msg;
 151 } FetchItem;
 152
 153 void HfreeFetchItem(void *vItem)
 154 {
 155         FetchItem *Item = (FetchItem*) vItem;
 156         FreeStrBuf(&Item->MsgUIDL);
 157         FreeStrBuf(&Item->MsgUID);
 158         free(Item);
 159 }
 160
 161 struct pop3aggr {
 162         AsyncIO  IO;
 163
 164         long n;
 165         double IOStart;
 166         long count;
 167         long RefCount;
 168         DNSQueryParts HostLookup;
 169
 170         long             QRnumber;
 171         HashList        *OtherQRnumbers;
 172
 173         StrBuf          *Url;
 174         StrBuf *pop3user;
 175         StrBuf *pop3pass;
 176         StrBuf *Host;
 177         StrBuf *RoomName; // TODO: fill me
 178         int keep;
 179         time_t interval;
 180         ePOP3_C_States State;
 181         HashList *MsgNumbers;
 182         HashPos *Pos;
 183         FetchItem *CurrMsg;
 184 };
 185
 186 void DeletePOP3Aggregator(void *vptr)
 187 {
 188         pop3aggr *ptr = vptr;
 189         DeleteHashPos(&ptr->Pos);
 190         DeleteHash(&ptr->MsgNumbers);
 191 //      FreeStrBuf(&ptr->rooms);
 192         FreeStrBuf(&ptr->pop3user);
 193         FreeStrBuf(&ptr->pop3pass);
 194         FreeStrBuf(&ptr->Host);
 195         FreeStrBuf(&ptr->RoomName);
 196         FreeURL(&ptr->IO.ConnectMe);
 197         FreeStrBuf(&ptr->Url);
 198         FreeStrBuf(&ptr->IO.IOBuf);
 199         FreeStrBuf(&ptr->IO.SendBuf.Buf);
 200         FreeStrBuf(&ptr->IO.RecvBuf.Buf);
 201         DeleteAsyncMsg(&ptr->IO.ReadMsg);
 202         if (((struct CitContext*)ptr->IO.CitContext)) {
 203                 ((struct CitContext*)ptr->IO.CitContext)->state = CON_IDLE;
 204                 ((struct CitContext*)ptr->IO.CitContext)->kill_me = 1;
 205         }
 206         FreeAsyncIOContents(&ptr->IO);
 207         free(ptr);
 208 }
 209
 210 eNextState FinalizePOP3AggrRun(AsyncIO *IO)
 211 {
 212         HashPos  *It;
 213         pop3aggr *cpptr = (pop3aggr *)IO->Data;
 214
 215         EVP3C_syslog(LOG_INFO,
 216                      "%s@%s: fetched %ld new of %d messages in %fs. bye.",
 217                      ChrPtr(cpptr->pop3user),
 218                      ChrPtr(cpptr->Host),
 219                      cpptr->count,
 220                      GetCount(cpptr->MsgNumbers),
 221                      IO->Now - cpptr->IOStart
 222                 );
 223
 224         It = GetNewHashPos(POP3FetchUrls, 0);
 225         pthread_mutex_lock(&POP3QueueMutex);
 226         {
 227                 if (GetHashPosFromKey(POP3FetchUrls, SKEY(cpptr->Url), It))
 228                         DeleteEntryFromHash(POP3FetchUrls, It);
 229         }
 230         pthread_mutex_unlock(&POP3QueueMutex);
 231         DeleteHashPos(&It);
 232         return eAbort;
 233 }
 234
 235 eNextState FailAggregationRun(AsyncIO *IO)
 236 {
 237         return eAbort;
 238 }
 239
 240 eNextState POP3C_ReadGreeting(pop3aggr *RecvMsg)
 241 {
 242         AsyncIO *IO = &RecvMsg->IO;
 243         POP3C_DBG_READ();
 244         /* Read the server greeting */
 245         if (!POP3C_OK) return eTerminateConnection;
 246         else return eSendReply;
 247 }
 248
 249 eNextState POP3C_SendUser(pop3aggr *RecvMsg)
 250 {
 251         AsyncIO *IO = &RecvMsg->IO;
 252         /* Identify ourselves.  NOTE: we have to append a CR to each command.
 253          *  The LF will automatically be appended by sock_puts().  Believe it
 254          * or not, leaving out the CR will cause problems if the server happens
 255          * to be Exchange, which is so b0rken it actually barfs on
 256          * LF-terminated newlines.
 257          */
 258         StrBufPrintf(RecvMsg->IO.SendBuf.Buf,
 259                      "USER %s\r\n", ChrPtr(RecvMsg->pop3user));
 260         POP3C_DBG_SEND();
 261         return eReadMessage;
 262 }
 263
 264 eNextState POP3C_GetUserState(pop3aggr *RecvMsg)
 265 {
 266         AsyncIO *IO = &RecvMsg->IO;
 267         POP3C_DBG_READ();
 268         if (!POP3C_OK) return eTerminateConnection;
 269         else return eSendReply;
 270 }
 271
 272 eNextState POP3C_SendPassword(pop3aggr *RecvMsg)
 273 {
 274         AsyncIO *IO = &RecvMsg->IO;
 275         /* Password */
 276         StrBufPrintf(RecvMsg->IO.SendBuf.Buf,
 277                      "PASS %s\r\n", ChrPtr(RecvMsg->pop3pass));
 278         EVP3CM_syslog(LOG_DEBUG, "<PASS <password>\n");
 279 //      POP3C_DBG_SEND(); No, we won't write the passvoid to syslog...
 280         return eReadMessage;
 281 }
 282
 283 eNextState POP3C_GetPassState(pop3aggr *RecvMsg)
 284 {
 285         AsyncIO *IO = &RecvMsg->IO;
 286         POP3C_DBG_READ();
 287         if (!POP3C_OK) return eTerminateConnection;
 288         else return eSendReply;
 289 }
 290
 291 eNextState POP3C_SendListCommand(pop3aggr *RecvMsg)
 292 {
 293         AsyncIO *IO = &RecvMsg->IO;
 294         /* Get the list of messages */
 295         StrBufPlain(RecvMsg->IO.SendBuf.Buf, HKEY("LIST\r\n"));
 296         POP3C_DBG_SEND();
 297         return eReadMessage;
 298 }
 299
 300 eNextState POP3C_GetListCommandState(pop3aggr *RecvMsg)
 301 {
 302         AsyncIO *IO = &RecvMsg->IO;
 303         POP3C_DBG_READ();
 304         if (!POP3C_OK) return eTerminateConnection;
 305         RecvMsg->MsgNumbers = NewHash(1, NULL);
 306         RecvMsg->State++;
 307         return eReadMore;
 308 }
 309
 310
 311 eNextState POP3C_GetListOneLine(pop3aggr *RecvMsg)
 312 {
 313         AsyncIO *IO = &RecvMsg->IO;
 314 #if 0
 315         int rc;
 316 #endif
 317         const char *pch;
 318         FetchItem *OneMsg = NULL;
 319         POP3C_DBG_READ();
 320
 321         if ((StrLength(RecvMsg->IO.IOBuf) == 1) &&
 322             (ChrPtr(RecvMsg->IO.IOBuf)[0] == '.'))
 323         {
 324                 if (GetCount(RecvMsg->MsgNumbers) == 0)
 325                 {
 326                         ////    RecvMsg->Sate = ReadQuitState;
 327                 }
 328                 else
 329                 {
 330                         RecvMsg->Pos = GetNewHashPos(RecvMsg->MsgNumbers, 0);
 331                 }
 332                 return eSendReply;
 333
 334         }
 335
 336         /*
 337          * work around buggy pop3 servers which send
 338          * empty lines in their listings.
 339         */
 340         if ((StrLength(RecvMsg->IO.IOBuf) == 0) ||
 341             !isdigit(ChrPtr(RecvMsg->IO.IOBuf)[0]))
 342         {
 343                 return eReadMore;
 344         }
 345
 346         OneMsg = (FetchItem*) malloc(sizeof(FetchItem));
 347         memset(OneMsg, 0, sizeof(FetchItem));
 348         OneMsg->MSGID = atol(ChrPtr(RecvMsg->IO.IOBuf));
 349
 350         pch = strchr(ChrPtr(RecvMsg->IO.IOBuf), ' ');
 351         if (pch != NULL)
 352         {
 353                 OneMsg->MSGSize = atol(pch + 1);
 354         }
 355 #if 0
 356         rc = TestValidateHash(RecvMsg->MsgNumbers);
 357         if (rc != 0)
 358                 EVP3CCS_syslog(LOG_DEBUG, "Hash Invalid: %d\n", rc);
 359 #endif
 360
 361         Put(RecvMsg->MsgNumbers, LKEY(OneMsg->MSGID), OneMsg, HfreeFetchItem);
 362 #if 0
 363         rc = TestValidateHash(RecvMsg->MsgNumbers);
 364         if (rc != 0)
 365                 EVP3CCS_syslog(LOG_DEBUG, "Hash Invalid: %d\n", rc);
 366 #endif
 367         //RecvMsg->State --; /* read next Line */
 368         return eReadMore;
 369 }
 370
 371 eNextState POP3_FetchNetworkUsetableEntry(AsyncIO *IO)
 372 {
 373         long HKLen;
 374         const char *HKey;
 375         void *vData;
 376         pop3aggr *RecvMsg = (pop3aggr *) IO->Data;
 377
 378         if((RecvMsg->Pos != NULL) &&
 379            GetNextHashPos(RecvMsg->MsgNumbers,
 380                           RecvMsg->Pos,
 381                           &HKLen,
 382                           &HKey,
 383                           &vData))
 384         {
 385                 if (server_shutting_down)
 386                         return eAbort;
 387
 388                 if (CheckIfAlreadySeen("POP3 Item Seen",
 389                                        RecvMsg->CurrMsg->MsgUID,
 390                                        IO->Now,
 391                                        IO->Now, //// todo
 392                                        eCheckUpdate,
 393                                        IO->ID, CCID)
 394                     != 0)
 395                 {
 396                         /* Item has already been seen */
 397                         RecvMsg->CurrMsg->NeedFetch = 0;
 398                 }
 399                 else
 400                 {
 401                         EVP3CCSM_syslog(LOG_DEBUG, "NO\n");
 402                         RecvMsg->CurrMsg->NeedFetch = 1;
 403                 }
 404                 return NextDBOperation(&RecvMsg->IO,
 405                                        POP3_FetchNetworkUsetableEntry);
 406         }
 407         else
 408         {
 409                 /* ok, now we know them all,
 410                  * continue with reading the actual messages. */
 411                 DeleteHashPos(&RecvMsg->Pos);
 412                 StopDBWatchers(IO);
 413                 return QueueEventContext(IO, POP3_C_ReAttachToFetchMessages);
 414         }
 415 }
 416
 417 eNextState POP3C_GetOneMessagID(pop3aggr *RecvMsg)
 418 {
 419         AsyncIO *IO = &RecvMsg->IO;
 420         long HKLen;
 421         const char *HKey;
 422         void *vData;
 423
 424 #if 0
 425         int rc;
 426         rc = TestValidateHash(RecvMsg->MsgNumbers);
 427         if (rc != 0)
 428                 EVP3CCS_syslog(LOG_DEBUG, "Hash Invalid: %d\n", rc);
 429 #endif
 430         if((RecvMsg->Pos != NULL) &&
 431            GetNextHashPos(RecvMsg->MsgNumbers,
 432                           RecvMsg->Pos,
 433                           &HKLen, &HKey,
 434                           &vData))
 435         {
 436                 RecvMsg->CurrMsg = (FetchItem*) vData;
 437                 /* Find out the UIDL of the message,
 438                  * to determine whether we've already downloaded it */
 439                 StrBufPrintf(RecvMsg->IO.SendBuf.Buf,
 440                              "UIDL %ld\r\n", RecvMsg->CurrMsg->MSGID);
 441                 POP3C_DBG_SEND();
 442         }
 443         else
 444         {
 445                 RecvMsg->State++;
 446                 DeleteHashPos(&RecvMsg->Pos);
 447                 /// done receiving uidls.. start looking them up now.
 448                 RecvMsg->Pos = GetNewHashPos(RecvMsg->MsgNumbers, 0);
 449                 return QueueDBOperation(&RecvMsg->IO,
 450                                         POP3_FetchNetworkUsetableEntry);
 451         }
 452         return eReadMore; /* TODO */
 453 }
 454
 455 eNextState POP3C_GetOneMessageIDState(pop3aggr *RecvMsg)
 456 {
 457         AsyncIO *IO = &RecvMsg->IO;
 458 #if 0
 459         int rc;
 460         rc = TestValidateHash(RecvMsg->MsgNumbers);
 461         if (rc != 0)
 462                 EVP3CCS_syslog(LOG_DEBUG, "Hash Invalid: %d\n", rc);
 463 #endif
 464
 465         POP3C_DBG_READ();
 466         if (!POP3C_OK) return eTerminateConnection;
 467         RecvMsg->CurrMsg->MsgUIDL =
 468                 NewStrBufPlain(NULL, StrLength(RecvMsg->IO.IOBuf));
 469         RecvMsg->CurrMsg->MsgUID =
 470                 NewStrBufPlain(NULL, StrLength(RecvMsg->IO.IOBuf) * 2);
 471
 472         StrBufExtract_token(RecvMsg->CurrMsg->MsgUIDL,
 473                             RecvMsg->IO.IOBuf, 2, ' ');
 474
 475         StrBufPrintf(RecvMsg->CurrMsg->MsgUID,
 476                      "pop3/%s/%s:%s@%s",
 477                      ChrPtr(RecvMsg->RoomName),
 478                      ChrPtr(RecvMsg->CurrMsg->MsgUIDL),
 479                      RecvMsg->IO.ConnectMe->User,
 480                      RecvMsg->IO.ConnectMe->Host);
 481         RecvMsg->State --;
 482         return eSendReply;
 483 }
 484
 485
 486 eNextState POP3C_SendGetOneMsg(pop3aggr *RecvMsg)
 487 {
 488         AsyncIO *IO = &RecvMsg->IO;
 489         long HKLen;
 490         const char *HKey;
 491         void *vData;
 492
 493         RecvMsg->CurrMsg = NULL;
 494         while ((RecvMsg->Pos != NULL) &&
 495                GetNextHashPos(RecvMsg->MsgNumbers,
 496                               RecvMsg->Pos,
 497                               &HKLen, &HKey,
 498                               &vData) &&
 499                (RecvMsg->CurrMsg = (FetchItem*) vData,
 500                 RecvMsg->CurrMsg->NeedFetch == 0))
 501         {}
 502
 503         if ((RecvMsg->CurrMsg != NULL ) && (RecvMsg->CurrMsg->NeedFetch == 1))
 504         {
 505                 /* Message has not been seen.
 506                  * Tell the server to fetch the message... */
 507                 StrBufPrintf(RecvMsg->IO.SendBuf.Buf,
 508                              "RETR %ld\r\n", RecvMsg->CurrMsg->MSGID);
 509                 POP3C_DBG_SEND();
 510                 return eReadMessage;
 511         }
 512         else {
 513                 RecvMsg->State = ReadQuitState;
 514                 return POP3_C_DispatchWriteDone(&RecvMsg->IO);
 515         }
 516 }
 517
 518
 519 eNextState POP3C_ReadMessageBodyFollowing(pop3aggr *RecvMsg)
 520 {
 521         AsyncIO *IO = &RecvMsg->IO;
 522         POP3C_DBG_READ();
 523         if (!POP3C_OK) return eTerminateConnection;
 524         RecvMsg->IO.ReadMsg = NewAsyncMsg(HKEY("."),
 525                                           RecvMsg->CurrMsg->MSGSize,
 526                                           config.c_maxmsglen,
 527                                           NULL, -1,
 528                                           1);
 529
 530         return eReadPayload;
 531 }
 532
 533
 534 eNextState POP3C_StoreMsgRead(AsyncIO *IO)
 535 {
 536         pop3aggr *RecvMsg = (pop3aggr *) IO->Data;
 537
 538         EVP3CCS_syslog(LOG_DEBUG,
 539                        "MARKING: %s as seen: ",
 540                        ChrPtr(RecvMsg->CurrMsg->MsgUID));
 541         CheckIfAlreadySeen("POP3 Item Seen",
 542                            RecvMsg->CurrMsg->MsgUID,
 543                            IO->Now,
 544                            IO->Now, //// todo
 545                            eWrite,
 546                            IO->ID, CCID);
 547
 548         StopDBWatchers(IO);
 549         return QueueEventContext(&RecvMsg->IO, POP3_C_ReAttachToFetchMessages);
 550 }
 551 eNextState POP3C_SaveMsg(AsyncIO *IO)
 552 {
 553         long msgnum;
 554         pop3aggr *RecvMsg = (pop3aggr *) IO->Data;
 555
 556         /* Do Something With It (tm) */
 557         msgnum = CtdlSubmitMsg(RecvMsg->CurrMsg->Msg,
 558                                NULL,
 559                                ChrPtr(RecvMsg->RoomName),
 560                                0);
 561         if (msgnum > 0L)
 562         {
 563                 /* Message has been committed to the store
 564                  * write the uidl to the use table
 565                  * so we don't fetch this message again
 566                  */
 567         }
 568         CtdlFreeMessage(RecvMsg->CurrMsg->Msg);
 569
 570         RecvMsg->count ++;
 571         return NextDBOperation(&RecvMsg->IO, POP3C_StoreMsgRead);
 572 }
 573
 574 eNextState POP3C_ReadMessageBody(pop3aggr *RecvMsg)
 575 {
 576         AsyncIO *IO = &RecvMsg->IO;
 577         EVP3CM_syslog(LOG_DEBUG, "Converting message...");
 578         RecvMsg->CurrMsg->Msg =
 579                 convert_internet_message_buf(&RecvMsg->IO.ReadMsg->MsgBuf);
 580         StopClientWatchers(IO, 0);
 581         return QueueDBOperation(&RecvMsg->IO, POP3C_SaveMsg);
 582 }
 583
 584 eNextState POP3C_SendDelete(pop3aggr *RecvMsg)
 585 {
 586         AsyncIO *IO = &RecvMsg->IO;
 587         if (!RecvMsg->keep) {
 588                 StrBufPrintf(RecvMsg->IO.SendBuf.Buf,
 589                              "DELE %ld\r\n", RecvMsg->CurrMsg->MSGID);
 590                 POP3C_DBG_SEND();
 591                 return eReadMessage;
 592         }
 593         else {
 594                 RecvMsg->State = ReadMessageBodyFollowing;
 595                 return POP3_C_DispatchWriteDone(&RecvMsg->IO);
 596         }
 597 }
 598 eNextState POP3C_ReadDeleteState(pop3aggr *RecvMsg)
 599 {
 600         AsyncIO *IO = &RecvMsg->IO;
 601         POP3C_DBG_READ();
 602         RecvMsg->State = GetOneMessageIDState;
 603         return eReadMessage;
 604 }
 605
 606 eNextState POP3C_SendQuit(pop3aggr *RecvMsg)
 607 {
 608         AsyncIO *IO = &RecvMsg->IO;
 609         /* Log out */
 610         StrBufPlain(RecvMsg->IO.SendBuf.Buf,
 611                     HKEY("QUIT\r\n3)"));
 612         POP3C_DBG_SEND();
 613         return eReadMessage;
 614 }
 615
 616
 617 eNextState POP3C_ReadQuitState(pop3aggr *RecvMsg)
 618 {
 619         AsyncIO *IO = &RecvMsg->IO;
 620         POP3C_DBG_READ();
 621         return eTerminateConnection;
 622 }
 623
 624 const long POP3_C_ConnTimeout = 1000;
 625 const long DefaultPOP3Port = 110;
 626
 627 Pop3ClientHandler POP3C_ReadHandlers[] = {
 628         POP3C_ReadGreeting,
 629         POP3C_GetUserState,
 630         POP3C_GetPassState,
 631         POP3C_GetListCommandState,
 632         POP3C_GetListOneLine,
 633         POP3C_GetOneMessageIDState,
 634         POP3C_ReadMessageBodyFollowing,
 635         POP3C_ReadMessageBody,
 636         POP3C_ReadDeleteState,
 637         POP3C_ReadQuitState,
 638 };
 639
 640 const long POP3_C_SendTimeouts[POP3C_MaxRead] = {
 641         100,
 642         100,
 643         100,
 644         100,
 645         100,
 646         100,
 647         100,
 648         100
 649 };
 650 const ConstStr POP3C_ReadErrors[POP3C_MaxRead] = {
 651         {HKEY("Connection broken during ")},
 652         {HKEY("Connection broken during ")},
 653         {HKEY("Connection broken during ")},
 654         {HKEY("Connection broken during ")},
 655         {HKEY("Connection broken during ")},
 656         {HKEY("Connection broken during ")},
 657         {HKEY("Connection broken during ")},
 658         {HKEY("Connection broken during ")}
 659 };
 660
 661 Pop3ClientHandler POP3C_SendHandlers[] = {
 662         NULL, /* we don't send a greeting */
 663         POP3C_SendUser,
 664         POP3C_SendPassword,
 665         POP3C_SendListCommand,
 666         NULL,
 667         POP3C_GetOneMessagID,
 668         POP3C_SendGetOneMsg,
 669         NULL,
 670         POP3C_SendDelete,
 671         POP3C_SendQuit
 672 };
 673
 674 const long POP3_C_ReadTimeouts[] = {
 675         100,
 676         100,
 677         100,
 678         100,
 679         100,
 680         100,
 681         100,
 682         100,
 683         100,
 684         100
 685 };
 686 /*****************************************************************************/
 687 /*                     POP3 CLIENT DISPATCHER                                */
 688 /*****************************************************************************/
 689
 690 void POP3SetTimeout(eNextState NextTCPState, pop3aggr *pMsg)
 691 {
 692         AsyncIO *IO = &pMsg->IO;
 693         double Timeout = 0.0;
 694
 695         EVP3C_syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
 696
 697         switch (NextTCPState) {
 698         case eSendFile:
 699         case eSendReply:
 700         case eSendMore:
 701                 Timeout = POP3_C_SendTimeouts[pMsg->State];
 702 /*
 703   if (pMsg->State == eDATABody) {
 704   / * if we're sending a huge message, we need more time. * /
 705   Timeout += StrLength(pMsg->msgtext) / 1024;
 706   }
 707 */
 708                 break;
 709         case eReadFile:
 710         case eReadMessage:
 711                 Timeout = POP3_C_ReadTimeouts[pMsg->State];
 712 /*
 713   if (pMsg->State == eDATATerminateBody) {
 714   / *
 715   * some mailservers take a nap before accepting the message
 716   * content inspection and such.
 717   * /
 718   Timeout += StrLength(pMsg->msgtext) / 1024;
 719   }
 720 */
 721                 break;
 722         case eReadPayload:
 723                 Timeout = 100000;
 724                 /* TODO!!! */
 725                 break;
 726         case eSendDNSQuery:
 727         case eReadDNSReply:
 728         case eConnect:
 729         case eTerminateConnection:
 730         case eDBQuery:
 731         case eAbort:
 732         case eReadMore://// TODO
 733                 return;
 734         }
 735         SetNextTimeout(&pMsg->IO, Timeout);
 736 }
 737 eNextState POP3_C_DispatchReadDone(AsyncIO *IO)
 738 {
 739 /*      EVP3CCS_syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__); to noisy anyways. */
 740         pop3aggr *pMsg = IO->Data;
 741         eNextState rc;
 742
 743         rc = POP3C_ReadHandlers[pMsg->State](pMsg);
 744         if (rc != eReadMore)
 745             pMsg->State++;
 746         POP3SetTimeout(rc, pMsg);
 747         return rc;
 748 }
 749 eNextState POP3_C_DispatchWriteDone(AsyncIO *IO)
 750 {
 751         pop3aggr *pMsg = IO->Data;
 752         eNextState rc;
 753
 754 /*      EVP3CCS_syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__); to noisy anyways. */
 755         rc = POP3C_SendHandlers[pMsg->State](pMsg);
 756         POP3SetTimeout(rc, pMsg);
 757         return rc;
 758 }
 759
 760
 761 /*****************************************************************************/
 762 /*                     POP3 CLIENT ERROR CATCHERS                            */
 763 /*****************************************************************************/
 764 eNextState POP3_C_Terminate(AsyncIO *IO)
 765 {
 766 ///     pop3aggr *pMsg = (pop3aggr *)IO->Data;
 767
 768         EVP3CCS_syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
 769         FinalizePOP3AggrRun(IO);
 770         return eAbort;
 771 }
 772 eNextState POP3_C_TerminateDB(AsyncIO *IO)
 773 {
 774 ///     pop3aggr *pMsg = (pop3aggr *)IO->Data;
 775
 776         EVP3CCS_syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
 777         FinalizePOP3AggrRun(IO);
 778         return eAbort;
 779 }
 780 eNextState POP3_C_Timeout(AsyncIO *IO)
 781 {
 782         pop3aggr *pMsg = IO->Data;
 783
 784         EVP3CCS_syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
 785         StrBufPlain(IO->ErrMsg, CKEY(POP3C_ReadErrors[pMsg->State]));
 786         return FailAggregationRun(IO);
 787 }
 788 eNextState POP3_C_ConnFail(AsyncIO *IO)
 789 {
 790         pop3aggr *pMsg = (pop3aggr *)IO->Data;
 791
 792         EVP3CCS_syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
 793         StrBufPlain(IO->ErrMsg, CKEY(POP3C_ReadErrors[pMsg->State]));
 794         return FailAggregationRun(IO);
 795 }
 796 eNextState POP3_C_DNSFail(AsyncIO *IO)
 797 {
 798         pop3aggr *pMsg = (pop3aggr *)IO->Data;
 799
 800         EVP3CCS_syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
 801         StrBufPlain(IO->ErrMsg, CKEY(POP3C_ReadErrors[pMsg->State]));
 802         return FailAggregationRun(IO);
 803 }
 804 eNextState POP3_C_Shutdown(AsyncIO *IO)
 805 {
 806         EVP3CCS_syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
 807 ////    pop3aggr *pMsg = IO->Data;
 808
 809 ////pMsg->MyQEntry->Status = 3;
 810 ///StrBufPlain(pMsg->MyQEntry->StatusMessage, HKEY("server shutdown during message retrieval."));
 811         FinalizePOP3AggrRun(IO);
 812         return eAbort;
 813 }
 814
 815
 816 /**
 817  * @brief lineread Handler; understands when to read more POP3 lines,
 818  *   and when this is a one-lined reply.
 819  */
 820 eReadState POP3_C_ReadServerStatus(AsyncIO *IO)
 821 {
 822         eReadState Finished = eBufferNotEmpty;
 823
 824         switch (IO->NextState) {
 825         case eSendDNSQuery:
 826         case eReadDNSReply:
 827         case eDBQuery:
 828         case eConnect:
 829         case eTerminateConnection:
 830         case eAbort:
 831                 Finished = eReadFail;
 832                 break;
 833         case eSendFile:
 834         case eSendReply:
 835         case eSendMore:
 836         case eReadMore:
 837         case eReadMessage:
 838                 Finished = StrBufChunkSipLine(IO->IOBuf, &IO->RecvBuf);
 839                 break;
 840         case eReadFile:
 841         case eReadPayload:
 842                 Finished = CtdlReadMessageBodyAsync(IO);
 843                 break;
 844         }
 845         return Finished;
 846 }
 847
 848 /*****************************************************************************
 849  * So we connect our Server IP here.                                         *
 850  *****************************************************************************/
 851 eNextState POP3_C_ReAttachToFetchMessages(AsyncIO *IO)
 852 {
 853         pop3aggr *cpptr = IO->Data;
 854
 855         EVP3CCS_syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
 856 ////??? cpptr->State ++;
 857         if (cpptr->Pos == NULL)
 858                 cpptr->Pos = GetNewHashPos(cpptr->MsgNumbers, 0);
 859
 860         POP3_C_DispatchWriteDone(IO);
 861         ReAttachIO(IO, cpptr, 0);
 862         IO->NextState = eReadMessage;
 863         return IO->NextState;
 864 }
 865
 866 eNextState pop3_connect_ip(AsyncIO *IO)
 867 {
 868         pop3aggr *cpptr = IO->Data;
 869
 870         if (cpptr->IOStart == 0.0) /* whith or without DNS? */
 871                 cpptr->IOStart = IO->Now;
 872
 873         EVP3CCS_syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
 874
 875         return EvConnectSock(IO,
 876                              POP3_C_ConnTimeout,
 877                              POP3_C_ReadTimeouts[0],
 878                              1);
 879 }
 880
 881 eNextState pop3_get_one_host_ip_done(AsyncIO *IO)
 882 {
 883         pop3aggr *cpptr = IO->Data;
 884         struct hostent *hostent;
 885
 886         QueryCbDone(IO);
 887
 888         hostent = cpptr->HostLookup.VParsedDNSReply;
 889         if ((cpptr->HostLookup.DNSStatus == ARES_SUCCESS) &&
 890             (hostent != NULL) ) {
 891                 memset(&cpptr->IO.ConnectMe->Addr, 0, sizeof(struct in6_addr));
 892                 if (cpptr->IO.ConnectMe->IPv6) {
 893                         memcpy(&cpptr->IO.ConnectMe->Addr.sin6_addr.s6_addr,
 894                                &hostent->h_addr_list[0],
 895                                sizeof(struct in6_addr));
 896
 897                         cpptr->IO.ConnectMe->Addr.sin6_family =
 898                                 hostent->h_addrtype;
 899                         cpptr->IO.ConnectMe->Addr.sin6_port   =
 900                                 htons(DefaultPOP3Port);
 901                 }
 902                 else {
 903                         struct sockaddr_in *addr =
 904                                 (struct sockaddr_in*)
 905                                 &cpptr->IO.ConnectMe->Addr;
 906
 907                         memcpy(&addr->sin_addr.s_addr,
 908                                hostent->h_addr_list[0],
 909                                sizeof(uint32_t));
 910
 911                         addr->sin_family = hostent->h_addrtype;
 912                         addr->sin_port   = htons(DefaultPOP3Port);
 913                 }
 914                 return pop3_connect_ip(IO);
 915         }
 916         else
 917                 return eAbort;
 918 }
 919
 920 eNextState pop3_get_one_host_ip(AsyncIO *IO)
 921 {
 922         pop3aggr *cpptr = IO->Data;
 923
 924         cpptr->IOStart = IO->Now;
 925
 926         EVP3CCS_syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
 927
 928         EVP3CCS_syslog(LOG_DEBUG,
 929                        "POP3 client[%ld]: looking up %s-Record %s : %d ...\n",
 930                        cpptr->n,
 931                        (cpptr->IO.ConnectMe->IPv6)? "aaaa": "a",
 932                        cpptr->IO.ConnectMe->Host,
 933                        cpptr->IO.ConnectMe->Port);
 934
 935         QueueQuery((cpptr->IO.ConnectMe->IPv6)? ns_t_aaaa : ns_t_a,
 936                    cpptr->IO.ConnectMe->Host,
 937                    &cpptr->IO,
 938                    &cpptr->HostLookup,
 939                    pop3_get_one_host_ip_done);
 940         IO->NextState = eReadDNSReply;
 941         return IO->NextState;
 942 }
 943
 944
 945
 946 int pop3_do_fetching(pop3aggr *cpptr)
 947 {
 948         AsyncIO *IO = &cpptr->IO;
 949
 950         InitIOStruct(IO,
 951                      cpptr,
 952                      eReadMessage,
 953                      POP3_C_ReadServerStatus,
 954                      POP3_C_DNSFail,
 955                      POP3_C_DispatchWriteDone,
 956                      POP3_C_DispatchReadDone,
 957                      POP3_C_Terminate,
 958                      POP3_C_TerminateDB,
 959                      POP3_C_ConnFail,
 960                      POP3_C_Timeout,
 961                      POP3_C_Shutdown);
 962
 963         safestrncpy(((CitContext *)cpptr->IO.CitContext)->cs_host,
 964                     ChrPtr(cpptr->Url),
 965                     sizeof(((CitContext *)cpptr->IO.CitContext)->cs_host));
 966
 967         if (cpptr->IO.ConnectMe->IsIP) {
 968                 QueueEventContext(&cpptr->IO,
 969                                   pop3_connect_ip);
 970         }
 971         else {
 972                 QueueEventContext(&cpptr->IO,
 973                                   pop3_get_one_host_ip);
 974         }
 975         return 1;
 976 }
 977
 978 /*
 979  * Scan a room's netconfig to determine whether it requires POP3 aggregation
 980  */
 981 void pop3client_scan_room(struct ctdlroom *qrbuf, void *data, OneRoomNetCfg *OneRNCFG)
 982 {
 983         const RoomNetCfgLine *pLine;
 984         void *vptr;
 985
 986         pthread_mutex_lock(&POP3QueueMutex);
 987         if (GetHash(POP3QueueRooms, LKEY(qrbuf->QRnumber), &vptr))
 988         {
 989                 pthread_mutex_unlock(&POP3QueueMutex);
 990                 EVP3CQ_syslog(LOG_DEBUG,
 991                               "pop3client: [%ld] %s already in progress.",
 992                               qrbuf->QRnumber,
 993                               qrbuf->QRname);
 994                 return;
 995         }
 996         pthread_mutex_unlock(&POP3QueueMutex);
 997
 998         if (server_shutting_down) return;
 999
1000         pLine = OneRNCFG->NetConfigs[pop3client];
1001
1002         while (pLine != NULL)
1003         {
1004                 pop3aggr *cptr;
1005
1006                 cptr = (pop3aggr *) malloc(sizeof(pop3aggr));
1007                 memset(cptr, 0, sizeof(pop3aggr));
1008                 ///TODO do we need this? cptr->roomlist_parts=1;
1009                 cptr->RoomName = NewStrBufPlain(qrbuf->QRname, -1);
1010                 cptr->pop3user = NewStrBufDup(pLine->Value[1]);
1011                 cptr->pop3pass = NewStrBufDup(pLine->Value[2]);
1012                 cptr->Url = NewStrBuf();
1013                 cptr->Host = NewStrBufDup(pLine->Value[0]);
1014
1015                 cptr->keep = atol(ChrPtr(pLine->Value[3]));
1016                 cptr->interval = atol(ChrPtr(pLine->Value[4]));
1017
1018                 StrBufAppendBufPlain(cptr->Url, HKEY("pop3://"), 0);
1019                 StrBufUrlescUPAppend(cptr->Url, cptr->pop3user, NULL);
1020                 StrBufAppendBufPlain(cptr->Url, HKEY(":"), 0);
1021                 StrBufUrlescUPAppend(cptr->Url, cptr->pop3pass, NULL);
1022                 StrBufAppendBufPlain(cptr->Url, HKEY("@"), 0);
1023                 StrBufAppendBuf(cptr->Url, cptr->Host, 0);
1024                 StrBufAppendBufPlain(cptr->Url, HKEY("/"), 0);
1025                 StrBufUrlescAppend(cptr->Url, cptr->RoomName, NULL);
1026
1027                 ParseURL(&cptr->IO.ConnectMe, cptr->Url, 110);
1028
1029
1030 #if 0
1031 /* todo: we need to reunite the url to be shure. */
1032
1033                 pthread_mutex_lock(&POP3ueueMutex);
1034                 GetHash(POP3FetchUrls, SKEY(ptr->Url), &vptr);
1035                 use_this_cptr = (pop3aggr *)vptr;
1036
1037                 if (use_this_rncptr != NULL)
1038                 {
1039                         /* mustn't attach to an active session */
1040                         if (use_this_cptr->RefCount > 0)
1041                         {
1042                                 DeletePOP3Cfg(cptr);
1043 ///                                             Count->count--;
1044                         }
1045                         else
1046                         {
1047                                 long *QRnumber;
1048                                 StrBufAppendBufPlain(
1049                                         use_this_cptr->rooms,
1050                                         qrbuf->QRname,
1051                                         -1, 0);
1052                                 if (use_this_cptr->roomlist_parts == 1)
1053                                 {
1054                                         use_this_cptr->OtherQRnumbers
1055                                                 = NewHash(1, lFlathash);
1056                                 }
1057                                 QRnumber = (long*)malloc(sizeof(long));
1058                                 *QRnumber = qrbuf->QRnumber;
1059                                 Put(use_this_cptr->OtherQRnumbers,
1060                                     LKEY(qrbuf->QRnumber),
1061                                     QRnumber,
1062                                     NULL);
1063
1064                                 use_this_cptr->roomlist_parts++;
1065                         }
1066                         pthread_mutex_unlock(&POP3QueueMutex);
1067                         continue;
1068                 }
1069                 pthread_mutex_unlock(&RSSQueueMutex);
1070 #endif
1071                 cptr->n = Pop3ClientID++;
1072                 pthread_mutex_lock(&POP3QueueMutex);
1073                 Put(POP3FetchUrls,
1074                     SKEY(cptr->Url),
1075                     cptr,
1076                     DeletePOP3Aggregator);
1077
1078                 pthread_mutex_unlock(&POP3QueueMutex);
1079                 pLine = pLine->next;
1080
1081         }
1082 }
1083
1084 static int doing_pop3client = 0;
1085
1086 void pop3client_scan(void) {
1087         static time_t last_run = 0L;
1088         time_t fastest_scan;
1089         HashPos *it;
1090         long len;
1091         const char *Key;
1092         void *vrptr;
1093         pop3aggr *cptr;
1094
1095         become_session(&pop3_client_CC);
1096
1097         if (config.c_pop3_fastest < config.c_pop3_fetch)
1098                 fastest_scan = config.c_pop3_fastest;
1099         else
1100                 fastest_scan = config.c_pop3_fetch;
1101
1102         /*
1103          * Run POP3 aggregation no more frequently than once every n seconds
1104          */
1105         if ( (time(NULL) - last_run) < fastest_scan ) {
1106                 return;
1107         }
1108
1109         /*
1110          * This is a simple concurrency check to make sure only one pop3client
1111          * run is done at a time.  We could do this with a mutex, but since we
1112          * don't really require extremely fine granularity here, we'll do it
1113          * with a static variable instead.
1114          */
1115         if (doing_pop3client) return;
1116         doing_pop3client = 1;
1117
1118         EVP3CQM_syslog(LOG_DEBUG, "pop3client started");
1119         CtdlForEachNetCfgRoom(pop3client_scan_room, NULL, pop3client);
1120
1121         pthread_mutex_lock(&POP3QueueMutex);
1122         it = GetNewHashPos(POP3FetchUrls, 0);
1123         while (!server_shutting_down &&
1124                GetNextHashPos(POP3FetchUrls, it, &len, &Key, &vrptr) &&
1125                (vrptr != NULL)) {
1126                 cptr = (pop3aggr *)vrptr;
1127                 if (cptr->RefCount == 0)
1128                         if (!pop3_do_fetching(cptr))
1129                                 DeletePOP3Aggregator(cptr);////TODO
1130
1131 /*
1132         if ((palist->interval && time(NULL) > (last_run + palist->interval))
1133                         || (time(NULL) > last_run + config.c_pop3_fetch))
1134                         pop3_do_fetching(palist->roomname, palist->pop3host,
1135                         palist->pop3user, palist->pop3pass, palist->keep);
1136                 pptr = palist;
1137                 palist = palist->next;
1138                 free(pptr);
1139 */
1140         }
1141         DeleteHashPos(&it);
1142         pthread_mutex_unlock(&POP3QueueMutex);
1143
1144         EVP3CQM_syslog(LOG_DEBUG, "pop3client ended");
1145         last_run = time(NULL);
1146         doing_pop3client = 0;
1147 }
1148
1149
1150 void pop3_cleanup(void)
1151 {
1152         /* citthread_mutex_destroy(&POP3QueueMutex); TODO */
1153         while (doing_pop3client != 0) ;
1154         DeleteHash(&POP3FetchUrls);
1155         DeleteHash(&POP3QueueRooms);
1156 }
1157
1158
1159
1160 void LogDebugEnablePOP3Client(const int n)
1161 {
1162         POP3ClientDebugEnabled = n;
1163 }
1164
1165 CTDL_MODULE_INIT(pop3client)
1166 {
1167         if (!threading)
1168         {
1169                 CtdlFillSystemContext(&pop3_client_CC, "POP3aggr");
1170                 CtdlREGISTERRoomCfgType(pop3client, ParseGeneric, 0, 5, SerializeGeneric, DeleteGenericCfgLine);
1171                 pthread_mutex_init(&POP3QueueMutex, NULL);
1172                 POP3QueueRooms = NewHash(1, lFlathash);
1173                 POP3FetchUrls = NewHash(1, NULL);
1174                 CtdlRegisterSessionHook(pop3client_scan, EVT_TIMER, PRIO_AGGR + 50);
1175                 CtdlRegisterEVCleanupHook(pop3_cleanup);
1176                 CtdlRegisterDebugFlagHook(HKEY("pop3client"), LogDebugEnablePOP3Client, &POP3ClientDebugEnabled);
1177         }
1178
1179         /* return our module id for the log */
1180         return "pop3client";
1181 }