citadel/modules/pop3client/serv_pop3client.c

   1 /*
   2  * Consolidate mail from remote POP3 accounts.
   3  *
   4  * Copyright (c) 2007-2011 by the citadel.org team
   5  *
   6  * This program is open source software; you can redistribute it and/or
   7  * modify it under the terms of the GNU General Public License as published
   8  * by the Free Software Foundation; either version 3 of the License, or
   9  * (at your option) any later version.
  10  *
  11  * This program is distributed in the hope that it will be useful,
  12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14  * GNU General Public License for more details.
  15  *
  16  * You should have received a copy of the GNU General Public License
  17  * along with this program; if not, write to the Free Software
  18  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
  19  */
  20
  21 #include <stdlib.h>
  22 #include <unistd.h>
  23 #include <stdio.h>
  24
  25 #if TIME_WITH_SYS_TIME
  26 # include <sys/time.h>
  27 # include <time.h>
  28 #else
  29 # if HAVE_SYS_TIME_H
  30 #  include <sys/time.h>
  31 # else
  32 #  include <time.h>
  33 # endif
  34 #endif
  35
  36 #include <ctype.h>
  37 #include <string.h>
  38 #include <errno.h>
  39 #include <sys/types.h>
  40 #include <sys/stat.h>
  41 #include <libcitadel.h>
  42 #include "citadel.h"
  43 #include "server.h"
  44 #include "citserver.h"
  45 #include "support.h"
  46 #include "config.h"
  47 #include "ctdl_module.h"
  48 #include "clientsocket.h"
  49 #include "msgbase.h"
  50 #include "internet_addressing.h"
  51 #include "database.h"
  52 #include "citadel_dirs.h"
  53 #include "event_client.h"
  54
  55
  56 #define POP3C_OK (strncasecmp(ChrPtr(RecvMsg->IO.IOBuf), "+OK", 3) == 0)
  57 int Pop3ClientID = 0;
  58 int POP3ClientDebugEnabled = 0;
  59
  60 #define N ((pop3aggr*)IO->Data)->n
  61
  62 #define DBGLOG(LEVEL) if ((LEVEL != LOG_DEBUG) || (POP3ClientDebugEnabled != 0))
  63
  64 #define EVP3C_syslog(LEVEL, FORMAT, ...)                                \
  65         DBGLOG(LEVEL) syslog(LEVEL,                                     \
  66                              "IO[%ld]CC[%d][%ld]POP3: " FORMAT,         \
  67                              IO->ID, CCID, N, __VA_ARGS__)
  68
  69 #define EVP3CM_syslog(LEVEL, FORMAT)                                    \
  70         DBGLOG(LEVEL) syslog(LEVEL,                                     \
  71                              "IO[%ld]CC[%d][%ld]POP3: " FORMAT,         \
  72                              IO->ID, CCID, N)
  73
  74 #define EVP3CQ_syslog(LEVEL, FORMAT, ...)                               \
  75         DBGLOG(LEVEL) syslog(LEVEL,                                     \
  76                              "P3Q:" FORMAT,                             \
  77                              __VA_ARGS__)
  78
  79 #define EVP3CQM_syslog(LEVEL, FORMAT)                                   \
  80         DBGLOG(LEVEL) syslog(LEVEL,                                     \
  81                              "P3Q" FORMAT                               \
  82                 )
  83
  84 #define EVP3CCS_syslog(LEVEL, FORMAT, ...)                              \
  85         DBGLOG(LEVEL) syslog(LEVEL, "IO[%ld][%ld]POP3: " FORMAT,        \
  86                              IO->ID, N, __VA_ARGS__)
  87
  88 #define EVP3CCSM_syslog(LEVEL, FORMAT)                                  \
  89         DBGLOG(LEVEL) syslog(LEVEL, "IO[%ld][%ld]POP3: " FORMAT,        \
  90                              IO->ID, N)
  91
  92 #define POP3C_DBG_SEND()                                                \
  93         EVP3C_syslog(LOG_DEBUG,                                         \
  94                      "IO[%ld]CC[%d][%ld]POP3: > %s\n",                  \
  95                      IO->ID, CCID, N,                                   \
  96                      ChrPtr(RecvMsg->IO.SendBuf.Buf))
  97
  98 #define POP3C_DBG_READ()                                                \
  99         EVP3C_syslog(LOG_DEBUG,                                         \
 100                      "IO[%ld]CC[%d][%ld]POP3: < %s\n",                  \
 101                      IO->ID, CCID, N,                                   \
 102                      ChrPtr(RecvMsg->IO.IOBuf))
 103
 104
 105 struct CitContext pop3_client_CC;
 106
 107 pthread_mutex_t POP3QueueMutex; /* locks the access to the following vars: */
 108 HashList *POP3QueueRooms = NULL;
 109 HashList *POP3FetchUrls = NULL;
 110
 111 typedef struct pop3aggr pop3aggr;
 112 typedef eNextState(*Pop3ClientHandler)(pop3aggr* RecvMsg);
 113
 114 eNextState POP3_C_Shutdown(AsyncIO *IO);
 115 eNextState POP3_C_Timeout(AsyncIO *IO);
 116 eNextState POP3_C_ConnFail(AsyncIO *IO);
 117 eNextState POP3_C_DNSFail(AsyncIO *IO);
 118 eNextState POP3_C_DispatchReadDone(AsyncIO *IO);
 119 eNextState POP3_C_DispatchWriteDone(AsyncIO *IO);
 120 eNextState POP3_C_Terminate(AsyncIO *IO);
 121 eReadState POP3_C_ReadServerStatus(AsyncIO *IO);
 122 eNextState POP3_C_ReAttachToFetchMessages(AsyncIO *IO);
 123
 124 typedef struct __pop3_room_counter {
 125         int count;
 126         long QRnumber;
 127 }pop3_room_counter;
 128
 129 typedef enum ePOP3_C_States {
 130         ReadGreeting,
 131         GetUserState,
 132         GetPassState,
 133         GetListCommandState,
 134         GetListOneLine,
 135         GetOneMessageIDState,
 136         ReadMessageBodyFollowing,
 137         ReadMessageBody,
 138         GetDeleteState,
 139         ReadQuitState,
 140         POP3C_MaxRead
 141 }ePOP3_C_States;
 142
 143
 144 typedef struct _FetchItem {
 145         long MSGID;
 146         long MSGSize;
 147         StrBuf *MsgUIDL;
 148         StrBuf *MsgUID;
 149         int NeedFetch;
 150         struct CtdlMessage *Msg;
 151 } FetchItem;
 152
 153 void HfreeFetchItem(void *vItem)
 154 {
 155         FetchItem *Item = (FetchItem*) vItem;
 156         FreeStrBuf(&Item->MsgUIDL);
 157         FreeStrBuf(&Item->MsgUID);
 158         free(Item);
 159 }
 160
 161 struct pop3aggr {
 162         AsyncIO  IO;
 163
 164         long n;
 165         double IOStart;
 166         long count;
 167         long RefCount;
 168         DNSQueryParts HostLookup;
 169
 170         long             QRnumber;
 171         HashList        *OtherQRnumbers;
 172
 173         StrBuf          *Url;
 174         StrBuf *pop3user;
 175         StrBuf *pop3pass;
 176         StrBuf *Host;
 177         StrBuf *RoomName; // TODO: fill me
 178         int keep;
 179         time_t interval;
 180         ePOP3_C_States State;
 181         HashList *MsgNumbers;
 182         HashPos *Pos;
 183         FetchItem *CurrMsg;
 184 };
 185
 186 void DeletePOP3Aggregator(void *vptr)
 187 {
 188         pop3aggr *ptr = vptr;
 189         DeleteHashPos(&ptr->Pos);
 190         DeleteHash(&ptr->MsgNumbers);
 191 //      FreeStrBuf(&ptr->rooms);
 192         FreeStrBuf(&ptr->pop3user);
 193         FreeStrBuf(&ptr->pop3pass);
 194         FreeStrBuf(&ptr->Host);
 195         FreeStrBuf(&ptr->RoomName);
 196         FreeURL(&ptr->IO.ConnectMe);
 197         FreeStrBuf(&ptr->Url);
 198         FreeStrBuf(&ptr->IO.IOBuf);
 199         FreeStrBuf(&ptr->IO.SendBuf.Buf);
 200         FreeStrBuf(&ptr->IO.RecvBuf.Buf);
 201         DeleteAsyncMsg(&ptr->IO.ReadMsg);
 202         if (((struct CitContext*)ptr->IO.CitContext)) {
 203                 ((struct CitContext*)ptr->IO.CitContext)->state = CON_IDLE;
 204                 ((struct CitContext*)ptr->IO.CitContext)->kill_me = 1;
 205         }
 206         FreeAsyncIOContents(&ptr->IO);
 207         free(ptr);
 208 }
 209
 210 eNextState FinalizePOP3AggrRun(AsyncIO *IO)
 211 {
 212         HashPos  *It;
 213         pop3aggr *cpptr = (pop3aggr *)IO->Data;
 214
 215         EVP3C_syslog(LOG_INFO,
 216                      "%s@%s: fetched %ld new of %d messages in %fs. bye.",
 217                      ChrPtr(cpptr->pop3user),
 218                      ChrPtr(cpptr->Host),
 219                      cpptr->count,
 220                      GetCount(cpptr->MsgNumbers),
 221                      IO->Now - cpptr->IOStart
 222                 );
 223
 224         It = GetNewHashPos(POP3FetchUrls, 0);
 225         pthread_mutex_lock(&POP3QueueMutex);
 226         {
 227                 if (GetHashPosFromKey(POP3FetchUrls, SKEY(cpptr->Url), It))
 228                         DeleteEntryFromHash(POP3FetchUrls, It);
 229         }
 230         pthread_mutex_unlock(&POP3QueueMutex);
 231         DeleteHashPos(&It);
 232         return eAbort;
 233 }
 234
 235 eNextState FailAggregationRun(AsyncIO *IO)
 236 {
 237         return eAbort;
 238 }
 239
 240 eNextState POP3C_ReadGreeting(pop3aggr *RecvMsg)
 241 {
 242         AsyncIO *IO = &RecvMsg->IO;
 243         POP3C_DBG_READ();
 244         /* Read the server greeting */
 245         if (!POP3C_OK) return eTerminateConnection;
 246         else return eSendReply;
 247 }
 248
 249 eNextState POP3C_SendUser(pop3aggr *RecvMsg)
 250 {
 251         AsyncIO *IO = &RecvMsg->IO;
 252         /* Identify ourselves.  NOTE: we have to append a CR to each command.
 253          *  The LF will automatically be appended by sock_puts().  Believe it
 254          * or not, leaving out the CR will cause problems if the server happens
 255          * to be Exchange, which is so b0rken it actually barfs on
 256          * LF-terminated newlines.
 257          */
 258         StrBufPrintf(RecvMsg->IO.SendBuf.Buf,
 259                      "USER %s\r\n", ChrPtr(RecvMsg->pop3user));
 260         POP3C_DBG_SEND();
 261         return eReadMessage;
 262 }
 263
 264 eNextState POP3C_GetUserState(pop3aggr *RecvMsg)
 265 {
 266         AsyncIO *IO = &RecvMsg->IO;
 267         POP3C_DBG_READ();
 268         if (!POP3C_OK) return eTerminateConnection;
 269         else return eSendReply;
 270 }
 271
 272 eNextState POP3C_SendPassword(pop3aggr *RecvMsg)
 273 {
 274         AsyncIO *IO = &RecvMsg->IO;
 275         /* Password */
 276         StrBufPrintf(RecvMsg->IO.SendBuf.Buf,
 277                      "PASS %s\r\n", ChrPtr(RecvMsg->pop3pass));
 278         EVP3CM_syslog(LOG_DEBUG, "<PASS <password>\n");
 279 //      POP3C_DBG_SEND(); No, we won't write the passvoid to syslog...
 280         return eReadMessage;
 281 }
 282
 283 eNextState POP3C_GetPassState(pop3aggr *RecvMsg)
 284 {
 285         AsyncIO *IO = &RecvMsg->IO;
 286         POP3C_DBG_READ();
 287         if (!POP3C_OK) return eTerminateConnection;
 288         else return eSendReply;
 289 }
 290
 291 eNextState POP3C_SendListCommand(pop3aggr *RecvMsg)
 292 {
 293         AsyncIO *IO = &RecvMsg->IO;
 294         /* Get the list of messages */
 295         StrBufPlain(RecvMsg->IO.SendBuf.Buf, HKEY("LIST\r\n"));
 296         POP3C_DBG_SEND();
 297         return eReadMessage;
 298 }
 299
 300 eNextState POP3C_GetListCommandState(pop3aggr *RecvMsg)
 301 {
 302         AsyncIO *IO = &RecvMsg->IO;
 303         POP3C_DBG_READ();
 304         if (!POP3C_OK) return eTerminateConnection;
 305         RecvMsg->MsgNumbers = NewHash(1, NULL);
 306         RecvMsg->State++;
 307         return eReadMore;
 308 }
 309
 310
 311 eNextState POP3C_GetListOneLine(pop3aggr *RecvMsg)
 312 {
 313         AsyncIO *IO = &RecvMsg->IO;
 314 #if 0
 315         int rc;
 316 #endif
 317         const char *pch;
 318         FetchItem *OneMsg = NULL;
 319         POP3C_DBG_READ();
 320
 321         if ((StrLength(RecvMsg->IO.IOBuf) == 1) &&
 322             (ChrPtr(RecvMsg->IO.IOBuf)[0] == '.'))
 323         {
 324                 if (GetCount(RecvMsg->MsgNumbers) == 0)
 325                 {
 326                         ////    RecvMsg->Sate = ReadQuitState;
 327                 }
 328                 else
 329                 {
 330                         RecvMsg->Pos = GetNewHashPos(RecvMsg->MsgNumbers, 0);
 331                 }
 332                 return eSendReply;
 333
 334         }
 335         OneMsg = (FetchItem*) malloc(sizeof(FetchItem));
 336         memset(OneMsg, 0, sizeof(FetchItem));
 337         OneMsg->MSGID = atol(ChrPtr(RecvMsg->IO.IOBuf));
 338
 339         pch = strchr(ChrPtr(RecvMsg->IO.IOBuf), ' ');
 340         if (pch != NULL)
 341         {
 342                 OneMsg->MSGSize = atol(pch + 1);
 343         }
 344 #if 0
 345         rc = TestValidateHash(RecvMsg->MsgNumbers);
 346         if (rc != 0)
 347                 EVP3CCS_syslog(LOG_DEBUG, "Hash Invalid: %d\n", rc);
 348 #endif
 349
 350         Put(RecvMsg->MsgNumbers, LKEY(OneMsg->MSGID), OneMsg, HfreeFetchItem);
 351 #if 0
 352         rc = TestValidateHash(RecvMsg->MsgNumbers);
 353         if (rc != 0)
 354                 EVP3CCS_syslog(LOG_DEBUG, "Hash Invalid: %d\n", rc);
 355 #endif
 356         //RecvMsg->State --; /* read next Line */
 357         return eReadMore;
 358 }
 359
 360 eNextState POP3_FetchNetworkUsetableEntry(AsyncIO *IO)
 361 {
 362         long HKLen;
 363         const char *HKey;
 364         void *vData;
 365         struct cdbdata *cdbut;
 366         pop3aggr *RecvMsg = (pop3aggr *) IO->Data;
 367
 368         if((RecvMsg->Pos != NULL) &&
 369            GetNextHashPos(RecvMsg->MsgNumbers,
 370                           RecvMsg->Pos,
 371                           &HKLen,
 372                           &HKey,
 373                           &vData))
 374         {
 375                 struct UseTable ut;
 376                 if (server_shutting_down)
 377                         return eAbort;
 378
 379                 RecvMsg->CurrMsg = (FetchItem*) vData;
 380                 EVP3CCS_syslog(LOG_DEBUG,
 381                                "CHECKING: whether %s has already been seen: ",
 382                                ChrPtr(RecvMsg->CurrMsg->MsgUID));
 383
 384                 /* Find out if we've already seen this item */
 385                 safestrncpy(ut.ut_msgid,
 386                             ChrPtr(RecvMsg->CurrMsg->MsgUID),
 387                             sizeof(ut.ut_msgid));
 388                 ut.ut_timestamp = time(NULL);/// TODO: libev timestamp!
 389
 390                 cdbut = cdb_fetch(CDB_USETABLE, SKEY(RecvMsg->CurrMsg->MsgUID));
 391                 if (cdbut != NULL) {
 392                         /* Item has already been seen */
 393                         EVP3CCSM_syslog(LOG_DEBUG, "YES\n");
 394                         cdb_free(cdbut);
 395
 396                         /* rewrite the record anyway, to update the timestamp */
 397                         cdb_store(CDB_USETABLE,
 398                                   SKEY(RecvMsg->CurrMsg->MsgUID),
 399                                   &ut, sizeof(struct UseTable) );
 400                         RecvMsg->CurrMsg->NeedFetch = 0; ////TODO0;
 401                 }
 402                 else
 403                 {
 404                         EVP3CCSM_syslog(LOG_DEBUG, "NO\n");
 405                         RecvMsg->CurrMsg->NeedFetch = 1;
 406                 }
 407                 return NextDBOperation(&RecvMsg->IO,
 408                                        POP3_FetchNetworkUsetableEntry);
 409         }
 410         else
 411         {
 412                 /* ok, now we know them all,
 413                  * continue with reading the actual messages. */
 414                 DeleteHashPos(&RecvMsg->Pos);
 415                 StopDBWatchers(IO);
 416                 return QueueEventContext(IO, POP3_C_ReAttachToFetchMessages);
 417         }
 418 }
 419
 420 eNextState POP3C_GetOneMessagID(pop3aggr *RecvMsg)
 421 {
 422         AsyncIO *IO = &RecvMsg->IO;
 423         long HKLen;
 424         const char *HKey;
 425         void *vData;
 426
 427 #if 0
 428         int rc;
 429         rc = TestValidateHash(RecvMsg->MsgNumbers);
 430         if (rc != 0)
 431                 EVP3CCS_syslog(LOG_DEBUG, "Hash Invalid: %d\n", rc);
 432 #endif
 433         if((RecvMsg->Pos != NULL) &&
 434            GetNextHashPos(RecvMsg->MsgNumbers,
 435                           RecvMsg->Pos,
 436                           &HKLen, &HKey,
 437                           &vData))
 438         {
 439                 RecvMsg->CurrMsg = (FetchItem*) vData;
 440                 /* Find out the UIDL of the message,
 441                  * to determine whether we've already downloaded it */
 442                 StrBufPrintf(RecvMsg->IO.SendBuf.Buf,
 443                              "UIDL %ld\r\n", RecvMsg->CurrMsg->MSGID);
 444                 POP3C_DBG_SEND();
 445         }
 446         else
 447         {
 448                 RecvMsg->State++;
 449                 DeleteHashPos(&RecvMsg->Pos);
 450                 /// done receiving uidls.. start looking them up now.
 451                 RecvMsg->Pos = GetNewHashPos(RecvMsg->MsgNumbers, 0);
 452                 return QueueDBOperation(&RecvMsg->IO,
 453                                         POP3_FetchNetworkUsetableEntry);
 454         }
 455         return eReadMore; /* TODO */
 456 }
 457
 458 eNextState POP3C_GetOneMessageIDState(pop3aggr *RecvMsg)
 459 {
 460         AsyncIO *IO = &RecvMsg->IO;
 461 #if 0
 462         int rc;
 463         rc = TestValidateHash(RecvMsg->MsgNumbers);
 464         if (rc != 0)
 465                 EVP3CCS_syslog(LOG_DEBUG, "Hash Invalid: %d\n", rc);
 466 #endif
 467
 468         POP3C_DBG_READ();
 469         if (!POP3C_OK) return eTerminateConnection;
 470         RecvMsg->CurrMsg->MsgUIDL =
 471                 NewStrBufPlain(NULL, StrLength(RecvMsg->IO.IOBuf));
 472         RecvMsg->CurrMsg->MsgUID =
 473                 NewStrBufPlain(NULL, StrLength(RecvMsg->IO.IOBuf) * 2);
 474
 475         StrBufExtract_token(RecvMsg->CurrMsg->MsgUIDL,
 476                             RecvMsg->IO.IOBuf, 2, ' ');
 477
 478         StrBufPrintf(RecvMsg->CurrMsg->MsgUID,
 479                      "pop3/%s/%s:%s@%s",
 480                      ChrPtr(RecvMsg->RoomName),
 481                      ChrPtr(RecvMsg->CurrMsg->MsgUIDL),
 482                      RecvMsg->IO.ConnectMe->User,
 483                      RecvMsg->IO.ConnectMe->Host);
 484         RecvMsg->State --;
 485         return eSendReply;
 486 }
 487
 488
 489 eNextState POP3C_SendGetOneMsg(pop3aggr *RecvMsg)
 490 {
 491         AsyncIO *IO = &RecvMsg->IO;
 492         long HKLen;
 493         const char *HKey;
 494         void *vData;
 495
 496         RecvMsg->CurrMsg = NULL;
 497         while ((RecvMsg->Pos != NULL) &&
 498                GetNextHashPos(RecvMsg->MsgNumbers,
 499                               RecvMsg->Pos,
 500                               &HKLen, &HKey,
 501                               &vData) &&
 502                (RecvMsg->CurrMsg = (FetchItem*) vData,
 503                 RecvMsg->CurrMsg->NeedFetch == 0))
 504         {}
 505
 506         if ((RecvMsg->CurrMsg != NULL ) && (RecvMsg->CurrMsg->NeedFetch == 1))
 507         {
 508                 /* Message has not been seen.
 509                  * Tell the server to fetch the message... */
 510                 StrBufPrintf(RecvMsg->IO.SendBuf.Buf,
 511                              "RETR %ld\r\n", RecvMsg->CurrMsg->MSGID);
 512                 POP3C_DBG_SEND();
 513                 return eReadMessage;
 514         }
 515         else {
 516                 RecvMsg->State = ReadQuitState;
 517                 return POP3_C_DispatchWriteDone(&RecvMsg->IO);
 518         }
 519 }
 520
 521
 522 eNextState POP3C_ReadMessageBodyFollowing(pop3aggr *RecvMsg)
 523 {
 524         AsyncIO *IO = &RecvMsg->IO;
 525         POP3C_DBG_READ();
 526         if (!POP3C_OK) return eTerminateConnection;
 527         RecvMsg->IO.ReadMsg = NewAsyncMsg(HKEY("."),
 528                                           RecvMsg->CurrMsg->MSGSize,
 529                                           config.c_maxmsglen,
 530                                           NULL, -1,
 531                                           1);
 532
 533         return eReadPayload;
 534 }
 535
 536
 537 eNextState POP3C_StoreMsgRead(AsyncIO *IO)
 538 {
 539         pop3aggr *RecvMsg = (pop3aggr *) IO->Data;
 540         struct UseTable ut;
 541
 542         EVP3CCS_syslog(LOG_DEBUG,
 543                        "MARKING: %s as seen: ",
 544                        ChrPtr(RecvMsg->CurrMsg->MsgUID));
 545
 546         safestrncpy(ut.ut_msgid,
 547                     ChrPtr(RecvMsg->CurrMsg->MsgUID),
 548                     sizeof(ut.ut_msgid));
 549         ut.ut_timestamp = time(NULL); /* TODO: use libev time */
 550         cdb_store(CDB_USETABLE,
 551                   ChrPtr(RecvMsg->CurrMsg->MsgUID),
 552                   StrLength(RecvMsg->CurrMsg->MsgUID),
 553                   &ut,
 554                   sizeof(struct UseTable) );
 555         StopDBWatchers(IO);
 556         return QueueEventContext(&RecvMsg->IO, POP3_C_ReAttachToFetchMessages);
 557 }
 558 eNextState POP3C_SaveMsg(AsyncIO *IO)
 559 {
 560         long msgnum;
 561         pop3aggr *RecvMsg = (pop3aggr *) IO->Data;
 562
 563         /* Do Something With It (tm) */
 564         msgnum = CtdlSubmitMsg(RecvMsg->CurrMsg->Msg,
 565                                NULL,
 566                                ChrPtr(RecvMsg->RoomName),
 567                                0);
 568         if (msgnum > 0L)
 569         {
 570                 /* Message has been committed to the store
 571                  * write the uidl to the use table
 572                  * so we don't fetch this message again
 573                  */
 574         }
 575         CtdlFreeMessage(RecvMsg->CurrMsg->Msg);
 576
 577         RecvMsg->count ++;
 578         return NextDBOperation(&RecvMsg->IO, POP3C_StoreMsgRead);
 579 }
 580
 581 eNextState POP3C_ReadMessageBody(pop3aggr *RecvMsg)
 582 {
 583         AsyncIO *IO = &RecvMsg->IO;
 584         EVP3CM_syslog(LOG_DEBUG, "Converting message...");
 585         RecvMsg->CurrMsg->Msg =
 586                 convert_internet_message_buf(&RecvMsg->IO.ReadMsg->MsgBuf);
 587         StopClientWatchers(IO, 0);
 588         return QueueDBOperation(&RecvMsg->IO, POP3C_SaveMsg);
 589 }
 590
 591 eNextState POP3C_SendDelete(pop3aggr *RecvMsg)
 592 {
 593         AsyncIO *IO = &RecvMsg->IO;
 594         if (!RecvMsg->keep) {
 595                 StrBufPrintf(RecvMsg->IO.SendBuf.Buf,
 596                              "DELE %ld\r\n", RecvMsg->CurrMsg->MSGID);
 597                 POP3C_DBG_SEND();
 598                 return eReadMessage;
 599         }
 600         else {
 601                 RecvMsg->State = ReadMessageBodyFollowing;
 602                 return POP3_C_DispatchWriteDone(&RecvMsg->IO);
 603         }
 604 }
 605 eNextState POP3C_ReadDeleteState(pop3aggr *RecvMsg)
 606 {
 607         AsyncIO *IO = &RecvMsg->IO;
 608         POP3C_DBG_READ();
 609         RecvMsg->State = GetOneMessageIDState;
 610         return eReadMessage;
 611 }
 612
 613 eNextState POP3C_SendQuit(pop3aggr *RecvMsg)
 614 {
 615         AsyncIO *IO = &RecvMsg->IO;
 616         /* Log out */
 617         StrBufPlain(RecvMsg->IO.SendBuf.Buf,
 618                     HKEY("QUIT\r\n3)"));
 619         POP3C_DBG_SEND();
 620         return eReadMessage;
 621 }
 622
 623
 624 eNextState POP3C_ReadQuitState(pop3aggr *RecvMsg)
 625 {
 626         AsyncIO *IO = &RecvMsg->IO;
 627         POP3C_DBG_READ();
 628         return eTerminateConnection;
 629 }
 630
 631 const long POP3_C_ConnTimeout = 1000;
 632 const long DefaultPOP3Port = 110;
 633
 634 Pop3ClientHandler POP3C_ReadHandlers[] = {
 635         POP3C_ReadGreeting,
 636         POP3C_GetUserState,
 637         POP3C_GetPassState,
 638         POP3C_GetListCommandState,
 639         POP3C_GetListOneLine,
 640         POP3C_GetOneMessageIDState,
 641         POP3C_ReadMessageBodyFollowing,
 642         POP3C_ReadMessageBody,
 643         POP3C_ReadDeleteState,
 644         POP3C_ReadQuitState,
 645 };
 646
 647 const long POP3_C_SendTimeouts[POP3C_MaxRead] = {
 648         100,
 649         100,
 650         100,
 651         100,
 652         100,
 653         100,
 654         100,
 655         100
 656 };
 657 const ConstStr POP3C_ReadErrors[POP3C_MaxRead] = {
 658         {HKEY("Connection broken during ")},
 659         {HKEY("Connection broken during ")},
 660         {HKEY("Connection broken during ")},
 661         {HKEY("Connection broken during ")},
 662         {HKEY("Connection broken during ")},
 663         {HKEY("Connection broken during ")},
 664         {HKEY("Connection broken during ")},
 665         {HKEY("Connection broken during ")}
 666 };
 667
 668 Pop3ClientHandler POP3C_SendHandlers[] = {
 669         NULL, /* we don't send a greeting */
 670         POP3C_SendUser,
 671         POP3C_SendPassword,
 672         POP3C_SendListCommand,
 673         NULL,
 674         POP3C_GetOneMessagID,
 675         POP3C_SendGetOneMsg,
 676         NULL,
 677         POP3C_SendDelete,
 678         POP3C_SendQuit
 679 };
 680
 681 const long POP3_C_ReadTimeouts[] = {
 682         100,
 683         100,
 684         100,
 685         100,
 686         100,
 687         100,
 688         100,
 689         100,
 690         100,
 691         100
 692 };
 693 /*****************************************************************************/
 694 /*                     POP3 CLIENT DISPATCHER                                */
 695 /*****************************************************************************/
 696
 697 void POP3SetTimeout(eNextState NextTCPState, pop3aggr *pMsg)
 698 {
 699         AsyncIO *IO = &pMsg->IO;
 700         double Timeout = 0.0;
 701
 702         EVP3C_syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
 703
 704         switch (NextTCPState) {
 705         case eSendFile:
 706         case eSendReply:
 707         case eSendMore:
 708                 Timeout = POP3_C_SendTimeouts[pMsg->State];
 709 /*
 710   if (pMsg->State == eDATABody) {
 711   / * if we're sending a huge message, we need more time. * /
 712   Timeout += StrLength(pMsg->msgtext) / 1024;
 713   }
 714 */
 715                 break;
 716         case eReadFile:
 717         case eReadMessage:
 718                 Timeout = POP3_C_ReadTimeouts[pMsg->State];
 719 /*
 720   if (pMsg->State == eDATATerminateBody) {
 721   / *
 722   * some mailservers take a nap before accepting the message
 723   * content inspection and such.
 724   * /
 725   Timeout += StrLength(pMsg->msgtext) / 1024;
 726   }
 727 */
 728                 break;
 729         case eReadPayload:
 730                 Timeout = 100000;
 731                 /* TODO!!! */
 732                 break;
 733         case eSendDNSQuery:
 734         case eReadDNSReply:
 735         case eConnect:
 736         case eTerminateConnection:
 737         case eDBQuery:
 738         case eAbort:
 739         case eReadMore://// TODO
 740                 return;
 741         }
 742         SetNextTimeout(&pMsg->IO, Timeout);
 743 }
 744 eNextState POP3_C_DispatchReadDone(AsyncIO *IO)
 745 {
 746 /*      EVP3CCS_syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__); to noisy anyways. */
 747         pop3aggr *pMsg = IO->Data;
 748         eNextState rc;
 749
 750         rc = POP3C_ReadHandlers[pMsg->State](pMsg);
 751         if (rc != eReadMore)
 752             pMsg->State++;
 753         POP3SetTimeout(rc, pMsg);
 754         return rc;
 755 }
 756 eNextState POP3_C_DispatchWriteDone(AsyncIO *IO)
 757 {
 758         pop3aggr *pMsg = IO->Data;
 759         eNextState rc;
 760
 761 /*      EVP3CCS_syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__); to noisy anyways. */
 762         rc = POP3C_SendHandlers[pMsg->State](pMsg);
 763         POP3SetTimeout(rc, pMsg);
 764         return rc;
 765 }
 766
 767
 768 /*****************************************************************************/
 769 /*                     POP3 CLIENT ERROR CATCHERS                            */
 770 /*****************************************************************************/
 771 eNextState POP3_C_Terminate(AsyncIO *IO)
 772 {
 773 ///     pop3aggr *pMsg = (pop3aggr *)IO->Data;
 774
 775         EVP3CCS_syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
 776         FinalizePOP3AggrRun(IO);
 777         return eAbort;
 778 }
 779 eNextState POP3_C_TerminateDB(AsyncIO *IO)
 780 {
 781 ///     pop3aggr *pMsg = (pop3aggr *)IO->Data;
 782
 783         EVP3CCS_syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
 784         FinalizePOP3AggrRun(IO);
 785         return eAbort;
 786 }
 787 eNextState POP3_C_Timeout(AsyncIO *IO)
 788 {
 789         pop3aggr *pMsg = IO->Data;
 790
 791         EVP3CCS_syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
 792         StrBufPlain(IO->ErrMsg, CKEY(POP3C_ReadErrors[pMsg->State]));
 793         return FailAggregationRun(IO);
 794 }
 795 eNextState POP3_C_ConnFail(AsyncIO *IO)
 796 {
 797         pop3aggr *pMsg = (pop3aggr *)IO->Data;
 798
 799         EVP3CCS_syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
 800         StrBufPlain(IO->ErrMsg, CKEY(POP3C_ReadErrors[pMsg->State]));
 801         return FailAggregationRun(IO);
 802 }
 803 eNextState POP3_C_DNSFail(AsyncIO *IO)
 804 {
 805         pop3aggr *pMsg = (pop3aggr *)IO->Data;
 806
 807         EVP3CCS_syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
 808         StrBufPlain(IO->ErrMsg, CKEY(POP3C_ReadErrors[pMsg->State]));
 809         return FailAggregationRun(IO);
 810 }
 811 eNextState POP3_C_Shutdown(AsyncIO *IO)
 812 {
 813         EVP3CCS_syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
 814 ////    pop3aggr *pMsg = IO->Data;
 815
 816 ////pMsg->MyQEntry->Status = 3;
 817 ///StrBufPlain(pMsg->MyQEntry->StatusMessage, HKEY("server shutdown during message retrieval."));
 818         FinalizePOP3AggrRun(IO);
 819         return eAbort;
 820 }
 821
 822
 823 /**
 824  * @brief lineread Handler; understands when to read more POP3 lines,
 825  *   and when this is a one-lined reply.
 826  */
 827 eReadState POP3_C_ReadServerStatus(AsyncIO *IO)
 828 {
 829         eReadState Finished = eBufferNotEmpty;
 830
 831         switch (IO->NextState) {
 832         case eSendDNSQuery:
 833         case eReadDNSReply:
 834         case eDBQuery:
 835         case eConnect:
 836         case eTerminateConnection:
 837         case eAbort:
 838                 Finished = eReadFail;
 839                 break;
 840         case eSendFile:
 841         case eSendReply:
 842         case eSendMore:
 843         case eReadMore:
 844         case eReadMessage:
 845                 Finished = StrBufChunkSipLine(IO->IOBuf, &IO->RecvBuf);
 846                 break;
 847         case eReadFile:
 848         case eReadPayload:
 849                 Finished = CtdlReadMessageBodyAsync(IO);
 850                 break;
 851         }
 852         return Finished;
 853 }
 854
 855 /*****************************************************************************
 856  * So we connect our Server IP here.                                         *
 857  *****************************************************************************/
 858 eNextState POP3_C_ReAttachToFetchMessages(AsyncIO *IO)
 859 {
 860         pop3aggr *cpptr = IO->Data;
 861
 862         EVP3CCS_syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
 863 ////??? cpptr->State ++;
 864         if (cpptr->Pos == NULL)
 865                 cpptr->Pos = GetNewHashPos(cpptr->MsgNumbers, 0);
 866
 867         POP3_C_DispatchWriteDone(IO);
 868         ReAttachIO(IO, cpptr, 0);
 869         IO->NextState = eReadMessage;
 870         return IO->NextState;
 871 }
 872
 873 eNextState pop3_connect_ip(AsyncIO *IO)
 874 {
 875         pop3aggr *cpptr = IO->Data;
 876
 877         if (cpptr->IOStart == 0.0) /* whith or without DNS? */
 878                 cpptr->IOStart = IO->Now;
 879
 880         EVP3CCS_syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
 881
 882         return EvConnectSock(IO,
 883                              POP3_C_ConnTimeout,
 884                              POP3_C_ReadTimeouts[0],
 885                              1);
 886 }
 887
 888 eNextState pop3_get_one_host_ip_done(AsyncIO *IO)
 889 {
 890         pop3aggr *cpptr = IO->Data;
 891         struct hostent *hostent;
 892
 893         QueryCbDone(IO);
 894
 895         hostent = cpptr->HostLookup.VParsedDNSReply;
 896         if ((cpptr->HostLookup.DNSStatus == ARES_SUCCESS) &&
 897             (hostent != NULL) ) {
 898                 memset(&cpptr->IO.ConnectMe->Addr, 0, sizeof(struct in6_addr));
 899                 if (cpptr->IO.ConnectMe->IPv6) {
 900                         memcpy(&cpptr->IO.ConnectMe->Addr.sin6_addr.s6_addr,
 901                                &hostent->h_addr_list[0],
 902                                sizeof(struct in6_addr));
 903
 904                         cpptr->IO.ConnectMe->Addr.sin6_family =
 905                                 hostent->h_addrtype;
 906                         cpptr->IO.ConnectMe->Addr.sin6_port   =
 907                                 htons(DefaultPOP3Port);
 908                 }
 909                 else {
 910                         struct sockaddr_in *addr =
 911                                 (struct sockaddr_in*)
 912                                 &cpptr->IO.ConnectMe->Addr;
 913
 914                         memcpy(&addr->sin_addr.s_addr,
 915                                hostent->h_addr_list[0],
 916                                sizeof(uint32_t));
 917
 918                         addr->sin_family = hostent->h_addrtype;
 919                         addr->sin_port   = htons(DefaultPOP3Port);
 920                 }
 921                 return pop3_connect_ip(IO);
 922         }
 923         else
 924                 return eAbort;
 925 }
 926
 927 eNextState pop3_get_one_host_ip(AsyncIO *IO)
 928 {
 929         pop3aggr *cpptr = IO->Data;
 930
 931         cpptr->IOStart = IO->Now;
 932
 933         EVP3CCS_syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
 934
 935         EVP3CCS_syslog(LOG_DEBUG,
 936                        "POP3 client[%ld]: looking up %s-Record %s : %d ...\n",
 937                        cpptr->n,
 938                        (cpptr->IO.ConnectMe->IPv6)? "aaaa": "a",
 939                        cpptr->IO.ConnectMe->Host,
 940                        cpptr->IO.ConnectMe->Port);
 941
 942         QueueQuery((cpptr->IO.ConnectMe->IPv6)? ns_t_aaaa : ns_t_a,
 943                    cpptr->IO.ConnectMe->Host,
 944                    &cpptr->IO,
 945                    &cpptr->HostLookup,
 946                    pop3_get_one_host_ip_done);
 947         IO->NextState = eReadDNSReply;
 948         return IO->NextState;
 949 }
 950
 951
 952
 953 int pop3_do_fetching(pop3aggr *cpptr)
 954 {
 955         AsyncIO *IO = &cpptr->IO;
 956
 957         InitIOStruct(IO,
 958                      cpptr,
 959                      eReadMessage,
 960                      POP3_C_ReadServerStatus,
 961                      POP3_C_DNSFail,
 962                      POP3_C_DispatchWriteDone,
 963                      POP3_C_DispatchReadDone,
 964                      POP3_C_Terminate,
 965                      POP3_C_TerminateDB,
 966                      POP3_C_ConnFail,
 967                      POP3_C_Timeout,
 968                      POP3_C_Shutdown);
 969
 970         safestrncpy(((CitContext *)cpptr->IO.CitContext)->cs_host,
 971                     ChrPtr(cpptr->Url),
 972                     sizeof(((CitContext *)cpptr->IO.CitContext)->cs_host));
 973
 974         if (cpptr->IO.ConnectMe->IsIP) {
 975                 QueueEventContext(&cpptr->IO,
 976                                   pop3_connect_ip);
 977         }
 978         else {
 979                 QueueEventContext(&cpptr->IO,
 980                                   pop3_get_one_host_ip);
 981         }
 982         return 1;
 983 }
 984
 985 /*
 986  * Scan a room's netconfig to determine whether it requires POP3 aggregation
 987  */
 988 void pop3client_scan_room(struct ctdlroom *qrbuf, void *data, OneRoomNetCfg *OneRNCFG)
 989 {
 990         const RoomNetCfgLine *pLine;
 991         void *vptr;
 992
 993         pthread_mutex_lock(&POP3QueueMutex);
 994         if (GetHash(POP3QueueRooms, LKEY(qrbuf->QRnumber), &vptr))
 995         {
 996                 pthread_mutex_unlock(&POP3QueueMutex);
 997                 EVP3CQ_syslog(LOG_DEBUG,
 998                               "pop3client: [%ld] %s already in progress.",
 999                               qrbuf->QRnumber,
1000                               qrbuf->QRname);
1001                 return;
1002         }
1003         pthread_mutex_unlock(&POP3QueueMutex);
1004
1005         if (server_shutting_down) return;
1006
1007         pLine = OneRNCFG->NetConfigs[pop3client];
1008
1009         while (pLine != NULL)
1010         {
1011                 pop3aggr *cptr;
1012
1013                 cptr = (pop3aggr *) malloc(sizeof(pop3aggr));
1014                 memset(cptr, 0, sizeof(pop3aggr));
1015                 ///TODO do we need this? cptr->roomlist_parts=1;
1016                 cptr->RoomName = NewStrBufPlain(qrbuf->QRname, -1);
1017                 cptr->pop3user = NewStrBufDup(pLine->Value[1]);
1018                 cptr->pop3pass = NewStrBufDup(pLine->Value[2]);
1019                 cptr->Url = NewStrBuf();
1020                 cptr->Host = NewStrBufDup(pLine->Value[0]);
1021
1022                 cptr->keep = atol(ChrPtr(pLine->Value[3]));
1023                 cptr->interval = atol(ChrPtr(pLine->Value[4]));
1024
1025                 StrBufAppendBufPlain(cptr->Url, HKEY("pop3://"), 0);
1026                 StrBufUrlescUPAppend(cptr->Url, cptr->pop3user, NULL);
1027                 StrBufAppendBufPlain(cptr->Url, HKEY(":"), 0);
1028                 StrBufUrlescUPAppend(cptr->Url, cptr->pop3pass, NULL);
1029                 StrBufAppendBufPlain(cptr->Url, HKEY("@"), 0);
1030                 StrBufAppendBuf(cptr->Url, cptr->Host, 0);
1031                 StrBufAppendBufPlain(cptr->Url, HKEY("/"), 0);
1032                 StrBufUrlescAppend(cptr->Url, cptr->RoomName, NULL);
1033
1034                 ParseURL(&cptr->IO.ConnectMe, cptr->Url, 110);
1035
1036
1037 #if 0
1038 /* todo: we need to reunite the url to be shure. */
1039
1040                 pthread_mutex_lock(&POP3ueueMutex);
1041                 GetHash(POP3FetchUrls, SKEY(ptr->Url), &vptr);
1042                 use_this_cptr = (pop3aggr *)vptr;
1043
1044                 if (use_this_rncptr != NULL)
1045                 {
1046                         /* mustn't attach to an active session */
1047                         if (use_this_cptr->RefCount > 0)
1048                         {
1049                                 DeletePOP3Cfg(cptr);
1050 ///                                             Count->count--;
1051                         }
1052                         else
1053                         {
1054                                 long *QRnumber;
1055                                 StrBufAppendBufPlain(
1056                                         use_this_cptr->rooms,
1057                                         qrbuf->QRname,
1058                                         -1, 0);
1059                                 if (use_this_cptr->roomlist_parts == 1)
1060                                 {
1061                                         use_this_cptr->OtherQRnumbers
1062                                                 = NewHash(1, lFlathash);
1063                                 }
1064                                 QRnumber = (long*)malloc(sizeof(long));
1065                                 *QRnumber = qrbuf->QRnumber;
1066                                 Put(use_this_cptr->OtherQRnumbers,
1067                                     LKEY(qrbuf->QRnumber),
1068                                     QRnumber,
1069                                     NULL);
1070
1071                                 use_this_cptr->roomlist_parts++;
1072                         }
1073                         pthread_mutex_unlock(&POP3QueueMutex);
1074                         continue;
1075                 }
1076                 pthread_mutex_unlock(&RSSQueueMutex);
1077 #endif
1078                 cptr->n = Pop3ClientID++;
1079                 pthread_mutex_lock(&POP3QueueMutex);
1080                 Put(POP3FetchUrls,
1081                     SKEY(cptr->Url),
1082                     cptr,
1083                     DeletePOP3Aggregator);
1084
1085                 pthread_mutex_unlock(&POP3QueueMutex);
1086
1087         }
1088 }
1089
1090 static int doing_pop3client = 0;
1091
1092 void pop3client_scan(void) {
1093         static time_t last_run = 0L;
1094         time_t fastest_scan;
1095         HashPos *it;
1096         long len;
1097         const char *Key;
1098         void *vrptr;
1099         pop3aggr *cptr;
1100
1101         become_session(&pop3_client_CC);
1102
1103         if (config.c_pop3_fastest < config.c_pop3_fetch)
1104                 fastest_scan = config.c_pop3_fastest;
1105         else
1106                 fastest_scan = config.c_pop3_fetch;
1107
1108         /*
1109          * Run POP3 aggregation no more frequently than once every n seconds
1110          */
1111         if ( (time(NULL) - last_run) < fastest_scan ) {
1112                 return;
1113         }
1114
1115         /*
1116          * This is a simple concurrency check to make sure only one pop3client
1117          * run is done at a time.  We could do this with a mutex, but since we
1118          * don't really require extremely fine granularity here, we'll do it
1119          * with a static variable instead.
1120          */
1121         if (doing_pop3client) return;
1122         doing_pop3client = 1;
1123
1124         EVP3CQM_syslog(LOG_DEBUG, "pop3client started");
1125         CtdlForEachNetCfgRoom(pop3client_scan_room, NULL, pop3client);
1126
1127         pthread_mutex_lock(&POP3QueueMutex);
1128         it = GetNewHashPos(POP3FetchUrls, 0);
1129         while (!server_shutting_down &&
1130                GetNextHashPos(POP3FetchUrls, it, &len, &Key, &vrptr) &&
1131                (vrptr != NULL)) {
1132                 cptr = (pop3aggr *)vrptr;
1133                 if (cptr->RefCount == 0)
1134                         if (!pop3_do_fetching(cptr))
1135                                 DeletePOP3Aggregator(cptr);////TODO
1136
1137 /*
1138         if ((palist->interval && time(NULL) > (last_run + palist->interval))
1139                         || (time(NULL) > last_run + config.c_pop3_fetch))
1140                         pop3_do_fetching(palist->roomname, palist->pop3host,
1141                         palist->pop3user, palist->pop3pass, palist->keep);
1142                 pptr = palist;
1143                 palist = palist->next;
1144                 free(pptr);
1145 */
1146         }
1147         DeleteHashPos(&it);
1148         pthread_mutex_unlock(&POP3QueueMutex);
1149
1150         EVP3CQM_syslog(LOG_DEBUG, "pop3client ended");
1151         last_run = time(NULL);
1152         doing_pop3client = 0;
1153 }
1154
1155
1156 void pop3_cleanup(void)
1157 {
1158         /* citthread_mutex_destroy(&POP3QueueMutex); TODO */
1159         while (doing_pop3client != 0) ;
1160         DeleteHash(&POP3FetchUrls);
1161         DeleteHash(&POP3QueueRooms);
1162 }
1163
1164
1165
1166 void LogDebugEnablePOP3Client(const int n)
1167 {
1168         POP3ClientDebugEnabled = n;
1169 }
1170
1171 CTDL_MODULE_INIT(pop3client)
1172 {
1173         if (!threading)
1174         {
1175                 CtdlFillSystemContext(&pop3_client_CC, "POP3aggr");
1176                 CtdlREGISTERRoomCfgType(pop3client, ParseGeneric, 0, 5, SerializeGeneric, DeleteGenericCfgLine);
1177                 pthread_mutex_init(&POP3QueueMutex, NULL);
1178                 POP3QueueRooms = NewHash(1, lFlathash);
1179                 POP3FetchUrls = NewHash(1, NULL);
1180                 CtdlRegisterSessionHook(pop3client_scan, EVT_TIMER, PRIO_AGGR + 50);
1181                 CtdlRegisterEVCleanupHook(pop3_cleanup);
1182                 CtdlRegisterDebugFlagHook(HKEY("pop3client"), LogDebugEnablePOP3Client, &POP3ClientDebugEnabled);
1183         }
1184
1185         /* return our module id for the log */
1186         return "pop3client";
1187 }