citadel/modules/pop3client/serv_pop3client.c

   1 /*
   2  * Consolidate mail from remote POP3 accounts.
   3  *
   4  * Copyright (c) 2007-2011 by the citadel.org team
   5  *
   6  * This program is open source software; you can redistribute it and/or
   7  * modify it under the terms of the GNU General Public License as published
   8  * by the Free Software Foundation; either version 3 of the License, or
   9  * (at your option) any later version.
  10  *
  11  * This program is distributed in the hope that it will be useful,
  12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14  * GNU General Public License for more details.
  15  *
  16  * You should have received a copy of the GNU General Public License
  17  * along with this program; if not, write to the Free Software
  18  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
  19  */
  20
  21 #include <stdlib.h>
  22 #include <unistd.h>
  23 #include <stdio.h>
  24
  25 #if TIME_WITH_SYS_TIME
  26 # include <sys/time.h>
  27 # include <time.h>
  28 #else
  29 # if HAVE_SYS_TIME_H
  30 #  include <sys/time.h>
  31 # else
  32 #  include <time.h>
  33 # endif
  34 #endif
  35
  36 #include <ctype.h>
  37 #include <string.h>
  38 #include <errno.h>
  39 #include <sys/types.h>
  40 #include <sys/stat.h>
  41 #include <libcitadel.h>
  42 #include "citadel.h"
  43 #include "server.h"
  44 #include "citserver.h"
  45 #include "support.h"
  46 #include "config.h"
  47 #include "ctdl_module.h"
  48 #include "clientsocket.h"
  49 #include "msgbase.h"
  50 #include "internet_addressing.h"
  51 #include "database.h"
  52 #include "citadel_dirs.h"
  53 #include "event_client.h"
  54
  55
  56 #define POP3C_OK (strncasecmp(ChrPtr(RecvMsg->IO.IOBuf), "+OK", 3) == 0)
  57 int Pop3ClientID = 0;
  58 int POP3ClientDebugEnabled = 0;
  59
  60 #define N ((pop3aggr*)IO->Data)->n
  61
  62 #define DBGLOG(LEVEL) if ((LEVEL != LOG_DEBUG) || (POP3ClientDebugEnabled != 0))
  63
  64 #define EVP3C_syslog(LEVEL, FORMAT, ...)                                \
  65         DBGLOG(LEVEL) syslog(LEVEL,                                     \
  66                              "IO[%ld]CC[%d][%ld]" FORMAT,               \
  67                              IO->ID, CCID, N, __VA_ARGS__)
  68
  69 #define EVP3CM_syslog(LEVEL, FORMAT)                                    \
  70         DBGLOG(LEVEL) syslog(LEVEL,                                     \
  71                              "IO[%ld]CC[%d][%ld]" FORMAT,               \
  72                              IO->ID, CCID, N)
  73
  74 #define EVP3CCS_syslog(LEVEL, FORMAT, ...)                              \
  75         DBGLOG(LEVEL) syslog(LEVEL, "IO[%ld][%ld]" FORMAT,              \
  76                              IO->ID, N, __VA_ARGS__)
  77
  78 #define EVP3CCSM_syslog(LEVEL, FORMAT)                          \
  79         DBGLOG(LEVEL) syslog(LEVEL, "IO[%ld][%ld]" FORMAT,      \
  80                              IO->ID, N)
  81
  82 #define POP3C_DBG_SEND()                                                \
  83         EVP3C_syslog(LOG_DEBUG,                                         \
  84                      "POP3: > %s\n",                                    \
  85                      ChrPtr(RecvMsg->IO.SendBuf.Buf))
  86
  87 #define POP3C_DBG_READ()                                \
  88         EVP3C_syslog(LOG_DEBUG,                         \
  89                      "POP3: < %s\n",                    \
  90                      ChrPtr(RecvMsg->IO.IOBuf))
  91
  92
  93 struct CitContext pop3_client_CC;
  94
  95 pthread_mutex_t POP3QueueMutex; /* locks the access to the following vars: */
  96 HashList *POP3QueueRooms = NULL;
  97 HashList *POP3FetchUrls = NULL;
  98
  99 typedef struct pop3aggr pop3aggr;
 100 typedef eNextState(*Pop3ClientHandler)(pop3aggr* RecvMsg);
 101
 102 eNextState POP3_C_Shutdown(AsyncIO *IO);
 103 eNextState POP3_C_Timeout(AsyncIO *IO);
 104 eNextState POP3_C_ConnFail(AsyncIO *IO);
 105 eNextState POP3_C_DNSFail(AsyncIO *IO);
 106 eNextState POP3_C_DispatchReadDone(AsyncIO *IO);
 107 eNextState POP3_C_DispatchWriteDone(AsyncIO *IO);
 108 eNextState POP3_C_Terminate(AsyncIO *IO);
 109 eReadState POP3_C_ReadServerStatus(AsyncIO *IO);
 110 eNextState POP3_C_ReAttachToFetchMessages(AsyncIO *IO);
 111
 112 typedef struct __pop3_room_counter {
 113         int count;
 114         long QRnumber;
 115 }pop3_room_counter;
 116
 117 typedef enum ePOP3_C_States {
 118         ReadGreeting,
 119         GetUserState,
 120         GetPassState,
 121         GetListCommandState,
 122         GetListOneLine,
 123         GetOneMessageIDState,
 124         ReadMessageBodyFollowing,
 125         ReadMessageBody,
 126         GetDeleteState,
 127         ReadQuitState,
 128         POP3C_MaxRead
 129 }ePOP3_C_States;
 130
 131
 132 typedef struct _FetchItem {
 133         long MSGID;
 134         long MSGSize;
 135         StrBuf *MsgUIDL;
 136         StrBuf *MsgUID;
 137         int NeedFetch;
 138         struct CtdlMessage *Msg;
 139 } FetchItem;
 140
 141 void HfreeFetchItem(void *vItem)
 142 {
 143         FetchItem *Item = (FetchItem*) vItem;
 144         FreeStrBuf(&Item->MsgUIDL);
 145         FreeStrBuf(&Item->MsgUID);
 146         free(Item);
 147 }
 148
 149 struct pop3aggr {
 150         AsyncIO  IO;
 151
 152         long n;
 153         long RefCount;
 154         DNSQueryParts HostLookup;
 155
 156         long             QRnumber;
 157         HashList        *OtherQRnumbers;
 158
 159         StrBuf          *Url;
 160         StrBuf *pop3user;
 161         StrBuf *pop3pass;
 162         StrBuf *RoomName; // TODO: fill me
 163         int keep;
 164         time_t interval;
 165         ePOP3_C_States State;
 166         HashList *MsgNumbers;
 167         HashPos *Pos;
 168         FetchItem *CurrMsg;
 169 };
 170
 171 void DeletePOP3Aggregator(void *vptr)
 172 {
 173         pop3aggr *ptr = vptr;
 174         DeleteHashPos(&ptr->Pos);
 175         DeleteHash(&ptr->MsgNumbers);
 176 //      FreeStrBuf(&ptr->rooms);
 177         FreeStrBuf(&ptr->pop3user);
 178         FreeStrBuf(&ptr->pop3pass);
 179         FreeStrBuf(&ptr->RoomName);
 180         FreeURL(&ptr->IO.ConnectMe);
 181         FreeStrBuf(&ptr->Url);
 182         FreeStrBuf(&ptr->IO.IOBuf);
 183         FreeStrBuf(&ptr->IO.SendBuf.Buf);
 184         FreeStrBuf(&ptr->IO.RecvBuf.Buf);
 185         DeleteAsyncMsg(&ptr->IO.ReadMsg);
 186         ((struct CitContext*)ptr->IO.CitContext)->state = CON_IDLE;
 187         ((struct CitContext*)ptr->IO.CitContext)->kill_me = 1;
 188         FreeAsyncIOContents(&ptr->IO);
 189         free(ptr);
 190 }
 191
 192 eNextState FinalizePOP3AggrRun(AsyncIO *IO)
 193 {
 194         HashPos  *It;
 195         pop3aggr *cptr = (pop3aggr *)IO->Data;
 196
 197         EVP3CM_syslog(LOG_DEBUG, "Terminating Aggregator; bye.\n");
 198
 199         It = GetNewHashPos(POP3FetchUrls, 0);
 200         pthread_mutex_lock(&POP3QueueMutex);
 201         {
 202                 if (GetHashPosFromKey(POP3FetchUrls, SKEY(cptr->Url), It))
 203                         DeleteEntryFromHash(POP3FetchUrls, It);
 204         }
 205         pthread_mutex_unlock(&POP3QueueMutex);
 206         DeleteHashPos(&It);
 207         return eAbort;
 208 }
 209
 210 eNextState FailAggregationRun(AsyncIO *IO)
 211 {
 212         return eAbort;
 213 }
 214
 215 eNextState POP3C_ReadGreeting(pop3aggr *RecvMsg)
 216 {
 217         AsyncIO *IO = &RecvMsg->IO;
 218         POP3C_DBG_READ();
 219         /* Read the server greeting */
 220         if (!POP3C_OK) return eTerminateConnection;
 221         else return eSendReply;
 222 }
 223
 224 eNextState POP3C_SendUser(pop3aggr *RecvMsg)
 225 {
 226         AsyncIO *IO = &RecvMsg->IO;
 227         /* Identify ourselves.  NOTE: we have to append a CR to each command.
 228          *  The LF will automatically be appended by sock_puts().  Believe it
 229          * or not, leaving out the CR will cause problems if the server happens
 230          * to be Exchange, which is so b0rken it actually barfs on
 231          * LF-terminated newlines.
 232          */
 233         StrBufPrintf(RecvMsg->IO.SendBuf.Buf,
 234                      "USER %s\r\n", ChrPtr(RecvMsg->pop3user));
 235         POP3C_DBG_SEND();
 236         return eReadMessage;
 237 }
 238
 239 eNextState POP3C_GetUserState(pop3aggr *RecvMsg)
 240 {
 241         AsyncIO *IO = &RecvMsg->IO;
 242         POP3C_DBG_READ();
 243         if (!POP3C_OK) return eTerminateConnection;
 244         else return eSendReply;
 245 }
 246
 247 eNextState POP3C_SendPassword(pop3aggr *RecvMsg)
 248 {
 249         AsyncIO *IO = &RecvMsg->IO;
 250         /* Password */
 251         StrBufPrintf(RecvMsg->IO.SendBuf.Buf,
 252                      "PASS %s\r\n", ChrPtr(RecvMsg->pop3pass));
 253         EVP3CM_syslog(LOG_DEBUG, "<PASS <password>\n");
 254 //      POP3C_DBG_SEND(); No, we won't write the passvoid to syslog...
 255         return eReadMessage;
 256 }
 257
 258 eNextState POP3C_GetPassState(pop3aggr *RecvMsg)
 259 {
 260         AsyncIO *IO = &RecvMsg->IO;
 261         POP3C_DBG_READ();
 262         if (!POP3C_OK) return eTerminateConnection;
 263         else return eSendReply;
 264 }
 265
 266 eNextState POP3C_SendListCommand(pop3aggr *RecvMsg)
 267 {
 268         AsyncIO *IO = &RecvMsg->IO;
 269         /* Get the list of messages */
 270         StrBufPlain(RecvMsg->IO.SendBuf.Buf, HKEY("LIST\r\n"));
 271         POP3C_DBG_SEND();
 272         return eReadMessage;
 273 }
 274
 275 eNextState POP3C_GetListCommandState(pop3aggr *RecvMsg)
 276 {
 277         AsyncIO *IO = &RecvMsg->IO;
 278         POP3C_DBG_READ();
 279         if (!POP3C_OK) return eTerminateConnection;
 280         RecvMsg->MsgNumbers = NewHash(1, NULL);
 281         RecvMsg->State++;
 282         return eReadMore;
 283 }
 284
 285
 286 eNextState POP3C_GetListOneLine(pop3aggr *RecvMsg)
 287 {
 288         AsyncIO *IO = &RecvMsg->IO;
 289 #if 0
 290         int rc;
 291 #endif
 292         const char *pch;
 293         FetchItem *OneMsg = NULL;
 294         POP3C_DBG_READ();
 295
 296         if ((StrLength(RecvMsg->IO.IOBuf) == 1) &&
 297             (ChrPtr(RecvMsg->IO.IOBuf)[0] == '.'))
 298         {
 299                 if (GetCount(RecvMsg->MsgNumbers) == 0)
 300                 {
 301                         ////    RecvMsg->Sate = ReadQuitState;
 302                 }
 303                 else
 304                 {
 305                         RecvMsg->Pos = GetNewHashPos(RecvMsg->MsgNumbers, 0);
 306                 }
 307                 return eSendReply;
 308
 309         }
 310         OneMsg = (FetchItem*) malloc(sizeof(FetchItem));
 311         memset(OneMsg, 0, sizeof(FetchItem));
 312         OneMsg->MSGID = atol(ChrPtr(RecvMsg->IO.IOBuf));
 313
 314         pch = strchr(ChrPtr(RecvMsg->IO.IOBuf), ' ');
 315         if (pch != NULL)
 316         {
 317                 OneMsg->MSGSize = atol(pch + 1);
 318         }
 319 #if 0
 320         rc = TestValidateHash(RecvMsg->MsgNumbers);
 321         if (rc != 0)
 322                 syslog(LOG_DEBUG, "Hash Invalid: %d\n", rc);
 323 #endif
 324
 325         Put(RecvMsg->MsgNumbers, LKEY(OneMsg->MSGID), OneMsg, HfreeFetchItem);
 326 #if 0
 327         rc = TestValidateHash(RecvMsg->MsgNumbers);
 328         if (rc != 0)
 329                 syslog(LOG_DEBUG, "Hash Invalid: %d\n", rc);
 330 #endif
 331         //RecvMsg->State --; /* read next Line */
 332         return eReadMore;
 333 }
 334
 335 eNextState POP3_FetchNetworkUsetableEntry(AsyncIO *IO)
 336 {
 337         long HKLen;
 338         const char *HKey;
 339         void *vData;
 340         struct cdbdata *cdbut;
 341         pop3aggr *RecvMsg = (pop3aggr *) IO->Data;
 342
 343         if(GetNextHashPos(RecvMsg->MsgNumbers,
 344                           RecvMsg->Pos,
 345                           &HKLen,
 346                           &HKey,
 347                           &vData))
 348         {
 349                 struct UseTable ut;
 350                 if (server_shutting_down)
 351                         return eAbort;
 352
 353                 RecvMsg->CurrMsg = (FetchItem*) vData;
 354                 syslog(LOG_DEBUG,
 355                        "CHECKING: whether %s has already been seen: ",
 356                        ChrPtr(RecvMsg->CurrMsg->MsgUID));
 357
 358                 /* Find out if we've already seen this item */
 359                 safestrncpy(ut.ut_msgid,
 360                             ChrPtr(RecvMsg->CurrMsg->MsgUID),
 361                             sizeof(ut.ut_msgid));
 362                 ut.ut_timestamp = time(NULL);/// TODO: libev timestamp!
 363
 364                 cdbut = cdb_fetch(CDB_USETABLE, SKEY(RecvMsg->CurrMsg->MsgUID));
 365                 if (cdbut != NULL) {
 366                         /* Item has already been seen */
 367                         syslog(LOG_DEBUG, "YES\n");
 368                         cdb_free(cdbut);
 369
 370                         /* rewrite the record anyway, to update the timestamp */
 371                         cdb_store(CDB_USETABLE,
 372                                   SKEY(RecvMsg->CurrMsg->MsgUID),
 373                                   &ut, sizeof(struct UseTable) );
 374                         RecvMsg->CurrMsg->NeedFetch = 0; ////TODO0;
 375                 }
 376                 else
 377                 {
 378                         syslog(LOG_DEBUG, "NO\n");
 379                         RecvMsg->CurrMsg->NeedFetch = 1;
 380                 }
 381                 return NextDBOperation(&RecvMsg->IO,
 382                                        POP3_FetchNetworkUsetableEntry);
 383         }
 384         else
 385         {
 386                 /* ok, now we know them all,
 387                  * continue with reading the actual messages. */
 388                 DeleteHashPos(&RecvMsg->Pos);
 389
 390                 return QueueEventContext(IO, POP3_C_ReAttachToFetchMessages);
 391         }
 392 }
 393
 394 eNextState POP3C_GetOneMessagID(pop3aggr *RecvMsg)
 395 {
 396         AsyncIO *IO = &RecvMsg->IO;
 397         long HKLen;
 398         const char *HKey;
 399         void *vData;
 400
 401 #if 0
 402         int rc;
 403         rc = TestValidateHash(RecvMsg->MsgNumbers);
 404         if (rc != 0)
 405                 syslog(LOG_DEBUG, "Hash Invalid: %d\n", rc);
 406 #endif
 407         if(GetNextHashPos(RecvMsg->MsgNumbers,
 408                           RecvMsg->Pos,
 409                           &HKLen, &HKey,
 410                           &vData))
 411         {
 412                 RecvMsg->CurrMsg = (FetchItem*) vData;
 413                 /* Find out the UIDL of the message,
 414                  * to determine whether we've already downloaded it */
 415                 StrBufPrintf(RecvMsg->IO.SendBuf.Buf,
 416                              "UIDL %ld\r\n", RecvMsg->CurrMsg->MSGID);
 417                 POP3C_DBG_SEND();
 418         }
 419         else
 420         {
 421             RecvMsg->State++;
 422                 DeleteHashPos(&RecvMsg->Pos);
 423                 /// done receiving uidls.. start looking them up now.
 424                 RecvMsg->Pos = GetNewHashPos(RecvMsg->MsgNumbers, 0);
 425                 return QueueDBOperation(&RecvMsg->IO,
 426                                         POP3_FetchNetworkUsetableEntry);
 427         }
 428         return eReadMore; /* TODO */
 429 }
 430
 431 eNextState POP3C_GetOneMessageIDState(pop3aggr *RecvMsg)
 432 {
 433         AsyncIO *IO = &RecvMsg->IO;
 434 #if 0
 435         int rc;
 436         rc = TestValidateHash(RecvMsg->MsgNumbers);
 437         if (rc != 0)
 438                 syslog(LOG_DEBUG, "Hash Invalid: %d\n", rc);
 439 #endif
 440
 441         POP3C_DBG_READ();
 442         if (!POP3C_OK) return eTerminateConnection;
 443         RecvMsg->CurrMsg->MsgUIDL =
 444                 NewStrBufPlain(NULL, StrLength(RecvMsg->IO.IOBuf));
 445         RecvMsg->CurrMsg->MsgUID =
 446                 NewStrBufPlain(NULL, StrLength(RecvMsg->IO.IOBuf) * 2);
 447
 448         StrBufExtract_token(RecvMsg->CurrMsg->MsgUIDL,
 449                             RecvMsg->IO.IOBuf, 2, ' ');
 450
 451         StrBufPrintf(RecvMsg->CurrMsg->MsgUID,
 452                      "pop3/%s/%s:%s@%s",
 453                      ChrPtr(RecvMsg->RoomName),
 454                      ChrPtr(RecvMsg->CurrMsg->MsgUIDL),
 455                      RecvMsg->IO.ConnectMe->User,
 456                      RecvMsg->IO.ConnectMe->Host);
 457         RecvMsg->State --;
 458         return eSendReply;
 459 }
 460
 461
 462 eNextState POP3C_SendGetOneMsg(pop3aggr *RecvMsg)
 463 {
 464         AsyncIO *IO = &RecvMsg->IO;
 465         long HKLen;
 466         const char *HKey;
 467         void *vData;
 468
 469         RecvMsg->CurrMsg = NULL;
 470         while (GetNextHashPos(RecvMsg->MsgNumbers,
 471                               RecvMsg->Pos,
 472                               &HKLen, &HKey,
 473                               &vData) &&
 474                (RecvMsg->CurrMsg = (FetchItem*) vData,
 475                 RecvMsg->CurrMsg->NeedFetch == 0))
 476         {}
 477
 478         if ((RecvMsg->CurrMsg != NULL ) && (RecvMsg->CurrMsg->NeedFetch == 1))
 479         {
 480                 /* Message has not been seen.
 481                  * Tell the server to fetch the message... */
 482                 StrBufPrintf(RecvMsg->IO.SendBuf.Buf,
 483                              "RETR %ld\r\n", RecvMsg->CurrMsg->MSGID);
 484                 POP3C_DBG_SEND();
 485                 return eReadMessage;
 486         }
 487         else {
 488                 RecvMsg->State = ReadQuitState;
 489                 return POP3_C_DispatchWriteDone(&RecvMsg->IO);
 490         }
 491 }
 492
 493
 494 eNextState POP3C_ReadMessageBodyFollowing(pop3aggr *RecvMsg)
 495 {
 496         AsyncIO *IO = &RecvMsg->IO;
 497         POP3C_DBG_READ();
 498         if (!POP3C_OK) return eTerminateConnection;
 499         RecvMsg->IO.ReadMsg = NewAsyncMsg(HKEY("."),
 500                                           RecvMsg->CurrMsg->MSGSize,
 501                                           config.c_maxmsglen,
 502                                           NULL, -1,
 503                                           1);
 504
 505         return eReadPayload;
 506 }
 507
 508
 509 eNextState POP3C_StoreMsgRead(AsyncIO *IO)
 510 {
 511         pop3aggr *RecvMsg = (pop3aggr *) IO->Data;
 512         struct UseTable ut;
 513
 514         syslog(LOG_DEBUG,
 515                "MARKING: %s as seen: ",
 516                ChrPtr(RecvMsg->CurrMsg->MsgUID));
 517
 518         safestrncpy(ut.ut_msgid,
 519                     ChrPtr(RecvMsg->CurrMsg->MsgUID),
 520                     sizeof(ut.ut_msgid));
 521         ut.ut_timestamp = time(NULL); /* TODO: use libev time */
 522         cdb_store(CDB_USETABLE,
 523                   ChrPtr(RecvMsg->CurrMsg->MsgUID),
 524                   StrLength(RecvMsg->CurrMsg->MsgUID),
 525                   &ut,
 526                   sizeof(struct UseTable) );
 527
 528         return QueueEventContext(&RecvMsg->IO, POP3_C_ReAttachToFetchMessages);
 529 }
 530 eNextState POP3C_SaveMsg(AsyncIO *IO)
 531 {
 532         long msgnum;
 533         pop3aggr *RecvMsg = (pop3aggr *) IO->Data;
 534
 535         /* Do Something With It (tm) */
 536         msgnum = CtdlSubmitMsg(RecvMsg->CurrMsg->Msg,
 537                                NULL,
 538                                ChrPtr(RecvMsg->RoomName),
 539                                0);
 540         if (msgnum > 0L)
 541         {
 542                 /* Message has been committed to the store
 543                  * write the uidl to the use table
 544                  * so we don't fetch this message again
 545                  */
 546         }
 547         CtdlFreeMessage(RecvMsg->CurrMsg->Msg);
 548
 549         return NextDBOperation(&RecvMsg->IO, POP3C_StoreMsgRead);
 550 }
 551
 552 eNextState POP3C_ReadMessageBody(pop3aggr *RecvMsg)
 553 {
 554         AsyncIO *IO = &RecvMsg->IO;
 555         EVP3CM_syslog(LOG_DEBUG, "Converting message...");
 556         RecvMsg->CurrMsg->Msg =
 557                 convert_internet_message_buf(&RecvMsg->IO.ReadMsg->MsgBuf);
 558
 559         return QueueDBOperation(&RecvMsg->IO, POP3C_SaveMsg);
 560 }
 561
 562 eNextState POP3C_SendDelete(pop3aggr *RecvMsg)
 563 {
 564         AsyncIO *IO = &RecvMsg->IO;
 565         if (!RecvMsg->keep) {
 566                 StrBufPrintf(RecvMsg->IO.SendBuf.Buf,
 567                              "DELE %ld\r\n", RecvMsg->CurrMsg->MSGID);
 568                 POP3C_DBG_SEND();
 569                 return eReadMessage;
 570         }
 571         else {
 572                 RecvMsg->State = ReadMessageBodyFollowing;
 573                 return POP3_C_DispatchWriteDone(&RecvMsg->IO);
 574         }
 575 }
 576 eNextState POP3C_ReadDeleteState(pop3aggr *RecvMsg)
 577 {
 578         AsyncIO *IO = &RecvMsg->IO;
 579         POP3C_DBG_READ();
 580         RecvMsg->State = GetOneMessageIDState;
 581         return eReadMessage;
 582 }
 583
 584 eNextState POP3C_SendQuit(pop3aggr *RecvMsg)
 585 {
 586         AsyncIO *IO = &RecvMsg->IO;
 587         /* Log out */
 588         StrBufPlain(RecvMsg->IO.SendBuf.Buf,
 589                     HKEY("QUIT\r\n3)"));
 590         POP3C_DBG_SEND();
 591         return eReadMessage;
 592 }
 593
 594
 595 eNextState POP3C_ReadQuitState(pop3aggr *RecvMsg)
 596 {
 597         AsyncIO *IO = &RecvMsg->IO;
 598         POP3C_DBG_READ();
 599         return eTerminateConnection;
 600 }
 601
 602 const long POP3_C_ConnTimeout = 1000;
 603 const long DefaultPOP3Port = 110;
 604
 605 Pop3ClientHandler POP3C_ReadHandlers[] = {
 606         POP3C_ReadGreeting,
 607         POP3C_GetUserState,
 608         POP3C_GetPassState,
 609         POP3C_GetListCommandState,
 610         POP3C_GetListOneLine,
 611         POP3C_GetOneMessageIDState,
 612         POP3C_ReadMessageBodyFollowing,
 613         POP3C_ReadMessageBody,
 614         POP3C_ReadDeleteState,
 615         POP3C_ReadQuitState,
 616 };
 617
 618 const long POP3_C_SendTimeouts[POP3C_MaxRead] = {
 619         100,
 620         100,
 621         100,
 622         100,
 623         100,
 624         100,
 625         100,
 626         100
 627 };
 628 const ConstStr POP3C_ReadErrors[POP3C_MaxRead] = {
 629         {HKEY("Connection broken during ")},
 630         {HKEY("Connection broken during ")},
 631         {HKEY("Connection broken during ")},
 632         {HKEY("Connection broken during ")},
 633         {HKEY("Connection broken during ")},
 634         {HKEY("Connection broken during ")},
 635         {HKEY("Connection broken during ")},
 636         {HKEY("Connection broken during ")}
 637 };
 638
 639 Pop3ClientHandler POP3C_SendHandlers[] = {
 640         NULL, /* we don't send a greeting */
 641         POP3C_SendUser,
 642         POP3C_SendPassword,
 643         POP3C_SendListCommand,
 644         NULL,
 645         POP3C_GetOneMessagID,
 646         POP3C_SendGetOneMsg,
 647         NULL,
 648         POP3C_SendDelete,
 649         POP3C_SendQuit
 650 };
 651
 652 const long POP3_C_ReadTimeouts[] = {
 653         100,
 654         100,
 655         100,
 656         100,
 657         100,
 658         100,
 659         100,
 660         100,
 661         100,
 662         100
 663 };
 664 /*****************************************************************************/
 665 /*                     POP3 CLIENT DISPATCHER                                */
 666 /*****************************************************************************/
 667
 668 void POP3SetTimeout(eNextState NextTCPState, pop3aggr *pMsg)
 669 {
 670         AsyncIO *IO = &pMsg->IO;
 671         double Timeout = 0.0;
 672
 673         EVP3C_syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
 674
 675         switch (NextTCPState) {
 676         case eSendFile:
 677         case eSendReply:
 678         case eSendMore:
 679                 Timeout = POP3_C_SendTimeouts[pMsg->State];
 680 /*
 681   if (pMsg->State == eDATABody) {
 682   / * if we're sending a huge message, we need more time. * /
 683   Timeout += StrLength(pMsg->msgtext) / 1024;
 684   }
 685 */
 686                 break;
 687         case eReadFile:
 688         case eReadMessage:
 689                 Timeout = POP3_C_ReadTimeouts[pMsg->State];
 690 /*
 691   if (pMsg->State == eDATATerminateBody) {
 692   / *
 693   * some mailservers take a nap before accepting the message
 694   * content inspection and such.
 695   * /
 696   Timeout += StrLength(pMsg->msgtext) / 1024;
 697   }
 698 */
 699                 break;
 700         case eReadPayload:
 701                 Timeout = 100000;
 702                 /* TODO!!! */
 703                 break;
 704         case eSendDNSQuery:
 705         case eReadDNSReply:
 706         case eConnect:
 707         case eTerminateConnection:
 708         case eDBQuery:
 709         case eAbort:
 710         case eReadMore://// TODO
 711                 return;
 712         }
 713         SetNextTimeout(&pMsg->IO, Timeout);
 714 }
 715 eNextState POP3_C_DispatchReadDone(AsyncIO *IO)
 716 {
 717         syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
 718         pop3aggr *pMsg = IO->Data;
 719         eNextState rc;
 720
 721         rc = POP3C_ReadHandlers[pMsg->State](pMsg);
 722         if (rc != eReadMore)
 723             pMsg->State++;
 724         POP3SetTimeout(rc, pMsg);
 725         return rc;
 726 }
 727 eNextState POP3_C_DispatchWriteDone(AsyncIO *IO)
 728 {
 729         syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
 730         pop3aggr *pMsg = IO->Data;
 731         eNextState rc;
 732
 733         rc = POP3C_SendHandlers[pMsg->State](pMsg);
 734         POP3SetTimeout(rc, pMsg);
 735         return rc;
 736 }
 737
 738
 739 /*****************************************************************************/
 740 /*                     POP3 CLIENT ERROR CATCHERS                            */
 741 /*****************************************************************************/
 742 eNextState POP3_C_Terminate(AsyncIO *IO)
 743 {
 744 ///     pop3aggr *pMsg = (pop3aggr *)IO->Data;
 745
 746         syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
 747         FinalizePOP3AggrRun(IO);
 748         return eAbort;
 749 }
 750 eNextState POP3_C_TerminateDB(AsyncIO *IO)
 751 {
 752 ///     pop3aggr *pMsg = (pop3aggr *)IO->Data;
 753
 754         syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
 755         FinalizePOP3AggrRun(IO);
 756         return eAbort;
 757 }
 758 eNextState POP3_C_Timeout(AsyncIO *IO)
 759 {
 760         pop3aggr *pMsg = IO->Data;
 761
 762         syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
 763         StrBufPlain(IO->ErrMsg, CKEY(POP3C_ReadErrors[pMsg->State]));
 764         return FailAggregationRun(IO);
 765 }
 766 eNextState POP3_C_ConnFail(AsyncIO *IO)
 767 {
 768         pop3aggr *pMsg = (pop3aggr *)IO->Data;
 769
 770         syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
 771         StrBufPlain(IO->ErrMsg, CKEY(POP3C_ReadErrors[pMsg->State]));
 772         return FailAggregationRun(IO);
 773 }
 774 eNextState POP3_C_DNSFail(AsyncIO *IO)
 775 {
 776         pop3aggr *pMsg = (pop3aggr *)IO->Data;
 777
 778         syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
 779         StrBufPlain(IO->ErrMsg, CKEY(POP3C_ReadErrors[pMsg->State]));
 780         return FailAggregationRun(IO);
 781 }
 782 eNextState POP3_C_Shutdown(AsyncIO *IO)
 783 {
 784         syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
 785 ////    pop3aggr *pMsg = IO->Data;
 786
 787 ////pMsg->MyQEntry->Status = 3;
 788 ///StrBufPlain(pMsg->MyQEntry->StatusMessage, HKEY("server shutdown during message retrieval."));
 789         FinalizePOP3AggrRun(IO);
 790         return eAbort;
 791 }
 792
 793
 794 /**
 795  * @brief lineread Handler; understands when to read more POP3 lines,
 796  *   and when this is a one-lined reply.
 797  */
 798 eReadState POP3_C_ReadServerStatus(AsyncIO *IO)
 799 {
 800         eReadState Finished = eBufferNotEmpty;
 801
 802         switch (IO->NextState) {
 803         case eSendDNSQuery:
 804         case eReadDNSReply:
 805         case eDBQuery:
 806         case eConnect:
 807         case eTerminateConnection:
 808         case eAbort:
 809                 Finished = eReadFail;
 810                 break;
 811         case eSendFile:
 812         case eSendReply:
 813         case eSendMore:
 814         case eReadMore:
 815         case eReadMessage:
 816                 Finished = StrBufChunkSipLine(IO->IOBuf, &IO->RecvBuf);
 817                 break;
 818         case eReadFile:
 819         case eReadPayload:
 820                 Finished = CtdlReadMessageBodyAsync(IO);
 821                 break;
 822         }
 823         return Finished;
 824 }
 825
 826 /*****************************************************************************
 827  * So we connect our Server IP here.                                         *
 828  *****************************************************************************/
 829 eNextState POP3_C_ReAttachToFetchMessages(AsyncIO *IO)
 830 {
 831         pop3aggr *cpptr = IO->Data;
 832
 833         syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
 834 ////??? cpptr->State ++;
 835         if (cpptr->Pos == NULL)
 836                 cpptr->Pos = GetNewHashPos(cpptr->MsgNumbers, 0);
 837
 838         POP3_C_DispatchWriteDone(IO);
 839         ReAttachIO(IO, cpptr, 0);
 840         IO->NextState = eReadMessage;
 841         return IO->NextState;
 842 }
 843
 844 eNextState pop3_connect_ip(AsyncIO *IO)
 845 {
 846         syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
 847
 848         return EvConnectSock(IO,
 849                              POP3_C_ConnTimeout,
 850                              POP3_C_ReadTimeouts[0],
 851                              1);
 852 }
 853
 854 eNextState pop3_get_one_host_ip_done(AsyncIO *IO)
 855 {
 856         pop3aggr *cpptr = IO->Data;
 857         struct hostent *hostent;
 858
 859         QueryCbDone(IO);
 860
 861         hostent = cpptr->HostLookup.VParsedDNSReply;
 862         if ((cpptr->HostLookup.DNSStatus == ARES_SUCCESS) &&
 863             (hostent != NULL) ) {
 864                 memset(&cpptr->IO.ConnectMe->Addr, 0, sizeof(struct in6_addr));
 865                 if (cpptr->IO.ConnectMe->IPv6) {
 866                         memcpy(&cpptr->IO.ConnectMe->Addr.sin6_addr.s6_addr,
 867                                &hostent->h_addr_list[0],
 868                                sizeof(struct in6_addr));
 869
 870                         cpptr->IO.ConnectMe->Addr.sin6_family =
 871                                 hostent->h_addrtype;
 872                         cpptr->IO.ConnectMe->Addr.sin6_port   =
 873                                 htons(DefaultPOP3Port);
 874                 }
 875                 else {
 876                         struct sockaddr_in *addr =
 877                                 (struct sockaddr_in*)
 878                                 &cpptr->IO.ConnectMe->Addr;
 879
 880                         memcpy(&addr->sin_addr.s_addr,
 881                                hostent->h_addr_list[0],
 882                                sizeof(uint32_t));
 883
 884                         addr->sin_family = hostent->h_addrtype;
 885                         addr->sin_port   = htons(DefaultPOP3Port);
 886                 }
 887                 return pop3_connect_ip(IO);
 888         }
 889         else
 890                 return eAbort;
 891 }
 892
 893 eNextState pop3_get_one_host_ip(AsyncIO *IO)
 894 {
 895         pop3aggr *cpptr = IO->Data;
 896
 897         syslog(LOG_DEBUG, "POP3: %s\n", __FUNCTION__);
 898
 899         syslog(LOG_DEBUG,
 900                       "POP3 client[%ld]: looking up %s-Record %s : %d ...\n",
 901                       cpptr->n,
 902                       (cpptr->IO.ConnectMe->IPv6)? "aaaa": "a",
 903                       cpptr->IO.ConnectMe->Host,
 904                       cpptr->IO.ConnectMe->Port);
 905
 906         QueueQuery((cpptr->IO.ConnectMe->IPv6)? ns_t_aaaa : ns_t_a,
 907                    cpptr->IO.ConnectMe->Host,
 908                    &cpptr->IO,
 909                    &cpptr->HostLookup,
 910                    pop3_get_one_host_ip_done);
 911         IO->NextState = eReadDNSReply;
 912         return IO->NextState;
 913 }
 914
 915
 916
 917 int pop3_do_fetching(pop3aggr *cpptr)
 918 {
 919         AsyncIO *IO = &cpptr->IO;
 920
 921         InitIOStruct(IO,
 922                      cpptr,
 923                      eReadMessage,
 924                      POP3_C_ReadServerStatus,
 925                      POP3_C_DNSFail,
 926                      POP3_C_DispatchWriteDone,
 927                      POP3_C_DispatchReadDone,
 928                      POP3_C_Terminate,
 929                      POP3_C_TerminateDB,
 930                      POP3_C_ConnFail,
 931                      POP3_C_Timeout,
 932                      POP3_C_Shutdown);
 933
 934         safestrncpy(((CitContext *)cpptr->IO.CitContext)->cs_host,
 935                     ChrPtr(cpptr->Url),
 936                     sizeof(((CitContext *)cpptr->IO.CitContext)->cs_host));
 937
 938         if (cpptr->IO.ConnectMe->IsIP) {
 939                 QueueEventContext(&cpptr->IO,
 940                                   pop3_connect_ip);
 941         }
 942         else {
 943                 QueueEventContext(&cpptr->IO,
 944                                   pop3_get_one_host_ip);
 945         }
 946         return 1;
 947 }
 948
 949 /*
 950  * Scan a room's netconfig to determine whether it requires POP3 aggregation
 951  */
 952 void pop3client_scan_room(struct ctdlroom *qrbuf, void *data)
 953 {
 954         StrBuf *CfgData;
 955         StrBuf *CfgType;
 956         StrBuf *Line;
 957
 958         struct stat statbuf;
 959         char filename[PATH_MAX];
 960         int  fd;
 961         int Done;
 962         void *vptr;
 963         const char *CfgPtr, *lPtr;
 964         const char *Err;
 965
 966 //      pop3_room_counter *Count = NULL;
 967 //      pop3aggr *cpptr;
 968
 969         pthread_mutex_lock(&POP3QueueMutex);
 970         if (GetHash(POP3QueueRooms, LKEY(qrbuf->QRnumber), &vptr))
 971         {
 972                 syslog(LOG_DEBUG,
 973                               "pop3client: [%ld] %s already in progress.\n",
 974                               qrbuf->QRnumber,
 975                               qrbuf->QRname);
 976                 pthread_mutex_unlock(&POP3QueueMutex);
 977         }
 978         pthread_mutex_unlock(&POP3QueueMutex);
 979
 980         if (server_shutting_down) return;
 981
 982         assoc_file_name(filename, sizeof filename, qrbuf, ctdl_netcfg_dir);
 983
 984         if (server_shutting_down)
 985                 return;
 986
 987         /* Only do net processing for rooms that have netconfigs */
 988         fd = open(filename, 0);
 989         if (fd <= 0) {
 990                 return;
 991         }
 992         if (server_shutting_down)
 993                 return;
 994         if (fstat(fd, &statbuf) == -1) {
 995                 syslog(LOG_DEBUG,
 996                        "ERROR: could not stat configfile '%s' - %s\n",
 997                        filename,
 998                        strerror(errno));
 999                 return;
1000         }
1001         if (server_shutting_down)
1002                 return;
1003         CfgData = NewStrBufPlain(NULL, statbuf.st_size + 1);
1004         if (StrBufReadBLOB(CfgData, &fd, 1, statbuf.st_size, &Err) < 0) {
1005                 close(fd);
1006                 FreeStrBuf(&CfgData);
1007                 syslog(LOG_DEBUG, "ERROR: reading config '%s' - %s<br>\n",
1008                               filename, strerror(errno));
1009                 return;
1010         }
1011         close(fd);
1012         if (server_shutting_down)
1013                 return;
1014
1015         CfgPtr = NULL;
1016         CfgType = NewStrBuf();
1017         Line = NewStrBufPlain(NULL, StrLength(CfgData));
1018         Done = 0;
1019
1020         while (!Done)
1021         {
1022                 Done = StrBufSipLine(Line, CfgData, &CfgPtr) == 0;
1023                 if (StrLength(Line) > 0)
1024                 {
1025                         lPtr = NULL;
1026                         StrBufExtract_NextToken(CfgType, Line, &lPtr, '|');
1027                         if (!strcasecmp("pop3client", ChrPtr(CfgType)))
1028                         {
1029                                 pop3aggr *cptr;
1030                                 StrBuf *Tmp;
1031 /*
1032                                 if (Count == NULL)
1033                                 {
1034                                 Count = malloc(sizeof(pop3_room_counter));
1035                                         Count->count = 0;
1036                                 }
1037                                 Count->count ++;
1038 */
1039                                 cptr = (pop3aggr *) malloc(sizeof(pop3aggr));
1040                                 memset(cptr, 0, sizeof(pop3aggr));
1041                                 ///TODO do we need this? cptr->roomlist_parts=1;
1042                                 cptr->RoomName =
1043                                         NewStrBufPlain(qrbuf->QRname, -1);
1044                                 cptr->pop3user =
1045                                         NewStrBufPlain(NULL, StrLength(Line));
1046                                 cptr->pop3pass =
1047                                         NewStrBufPlain(NULL, StrLength(Line));
1048                                 cptr->Url = NewStrBuf();
1049                                 Tmp = NewStrBuf();
1050
1051                                 StrBufExtract_NextToken(Tmp, Line, &lPtr, '|');
1052                                 StrBufExtract_NextToken(cptr->pop3user,
1053                                                         Line,
1054                                                         &lPtr,
1055                                                         '|');
1056
1057                                 StrBufExtract_NextToken(cptr->pop3pass,
1058                                                         Line,
1059                                                         &lPtr,
1060                                                         '|');
1061
1062                                 cptr->keep = StrBufExtractNext_long(Line,
1063                                                                     &lPtr,
1064                                                                     '|');
1065
1066                                 cptr->interval = StrBufExtractNext_long(Line,
1067                                                                         &lPtr,
1068                                                                         '|');
1069
1070                                 StrBufPrintf(cptr->Url, "pop3://%s:%s@%s/%s",
1071                                              ChrPtr(cptr->pop3user),
1072                                              ChrPtr(cptr->pop3pass),
1073                                              ChrPtr(Tmp),
1074                                              ChrPtr(cptr->RoomName));
1075                                 FreeStrBuf(&Tmp);
1076                                 ParseURL(&cptr->IO.ConnectMe, cptr->Url, 110);
1077
1078
1079 #if 0
1080 /* todo: we need to reunite the url to be shure. */
1081
1082                                 pthread_mutex_lock(&POP3ueueMutex);
1083                                 GetHash(POP3FetchUrls, SKEY(ptr->Url), &vptr);
1084                                 use_this_cptr = (pop3aggr *)vptr;
1085
1086                                 if (use_this_rncptr != NULL)
1087                                 {
1088                                         /* mustn't attach to an active session */
1089                                         if (use_this_cptr->RefCount > 0)
1090                                         {
1091                                                 DeletePOP3Cfg(cptr);
1092 ///                                             Count->count--;
1093                                         }
1094                                         else
1095                                         {
1096                                                 long *QRnumber;
1097                                                 StrBufAppendBufPlain(
1098                                                         use_this_cptr->rooms,
1099                                                         qrbuf->QRname,
1100                                                         -1, 0);
1101                                                 if (use_this_cptr->roomlist_parts == 1)
1102                                                 {
1103                                                         use_this_cptr->OtherQRnumbers
1104                                                                 = NewHash(1, lFlathash);
1105                                                 }
1106                                                 QRnumber = (long*)malloc(sizeof(long));
1107                                                 *QRnumber = qrbuf->QRnumber;
1108                                                 Put(use_this_cptr->OtherQRnumbers,
1109                                                     LKEY(qrbuf->QRnumber),
1110                                                     QRnumber,
1111                                                     NULL);
1112
1113                                                 use_this_cptr->roomlist_parts++;
1114                                         }
1115                                         pthread_mutex_unlock(&POP3QueueMutex);
1116                                         continue;
1117                                 }
1118                                 pthread_mutex_unlock(&RSSQueueMutex);
1119 #endif
1120                                 cptr->n = Pop3ClientID++;
1121                                 pthread_mutex_lock(&POP3QueueMutex);
1122                                 Put(POP3FetchUrls,
1123                                     SKEY(cptr->Url),
1124                                     cptr,
1125                                     DeletePOP3Aggregator);
1126
1127                                 pthread_mutex_unlock(&POP3QueueMutex);
1128
1129                         }
1130
1131                 }
1132
1133                 ///fclose(fp);
1134
1135         }
1136         FreeStrBuf(&Line);
1137         FreeStrBuf(&CfgType);
1138         FreeStrBuf(&CfgData);
1139 }
1140
1141 static int doing_pop3client = 0;
1142
1143 void pop3client_scan(void) {
1144         static time_t last_run = 0L;
1145         time_t fastest_scan;
1146         HashPos *it;
1147         long len;
1148         const char *Key;
1149         void *vrptr;
1150         pop3aggr *cptr;
1151
1152         become_session(&pop3_client_CC);
1153
1154         if (config.c_pop3_fastest < config.c_pop3_fetch)
1155                 fastest_scan = config.c_pop3_fastest;
1156         else
1157                 fastest_scan = config.c_pop3_fetch;
1158
1159         /*
1160          * Run POP3 aggregation no more frequently than once every n seconds
1161          */
1162         if ( (time(NULL) - last_run) < fastest_scan ) {
1163                 return;
1164         }
1165
1166         /*
1167          * This is a simple concurrency check to make sure only one pop3client
1168          * run is done at a time.  We could do this with a mutex, but since we
1169          * don't really require extremely fine granularity here, we'll do it
1170          * with a static variable instead.
1171          */
1172         if (doing_pop3client) return;
1173         doing_pop3client = 1;
1174
1175         syslog(LOG_DEBUG, "pop3client started");
1176         CtdlForEachRoom(pop3client_scan_room, NULL);
1177
1178         pthread_mutex_lock(&POP3QueueMutex);
1179         it = GetNewHashPos(POP3FetchUrls, 0);
1180         while (!server_shutting_down &&
1181                GetNextHashPos(POP3FetchUrls, it, &len, &Key, &vrptr) &&
1182                (vrptr != NULL)) {
1183                 cptr = (pop3aggr *)vrptr;
1184                 if (cptr->RefCount == 0)
1185                         if (!pop3_do_fetching(cptr))
1186                                 DeletePOP3Aggregator(cptr);////TODO
1187
1188 /*
1189         if ((palist->interval && time(NULL) > (last_run + palist->interval))
1190                         || (time(NULL) > last_run + config.c_pop3_fetch))
1191                         pop3_do_fetching(palist->roomname, palist->pop3host,
1192                         palist->pop3user, palist->pop3pass, palist->keep);
1193                 pptr = palist;
1194                 palist = palist->next;
1195                 free(pptr);
1196 */
1197         }
1198         DeleteHashPos(&it);
1199         pthread_mutex_unlock(&POP3QueueMutex);
1200
1201         syslog(LOG_DEBUG, "pop3client ended");
1202         last_run = time(NULL);
1203         doing_pop3client = 0;
1204 }
1205
1206
1207 void pop3_cleanup(void)
1208 {
1209         /* citthread_mutex_destroy(&POP3QueueMutex); TODO */
1210         while (doing_pop3client != 0) ;
1211         DeleteHash(&POP3FetchUrls);
1212         DeleteHash(&POP3QueueRooms);
1213 }
1214
1215
1216
1217 void LogDebugEnablePOP3Client(void)
1218 {
1219         POP3ClientDebugEnabled = 1;
1220 }
1221
1222 CTDL_MODULE_INIT(pop3client)
1223 {
1224         if (!threading)
1225         {
1226                 CtdlFillSystemContext(&pop3_client_CC, "POP3aggr");
1227                 pthread_mutex_init(&POP3QueueMutex, NULL);
1228                 POP3QueueRooms = NewHash(1, lFlathash);
1229                 POP3FetchUrls = NewHash(1, NULL);
1230                 CtdlRegisterSessionHook(pop3client_scan, EVT_TIMER);
1231                 CtdlRegisterEVCleanupHook(pop3_cleanup);
1232                 CtdlRegisterDebugFlagHook(HKEY("pop3client"), LogDebugEnablePOP3Client);
1233         }
1234
1235         /* return our module id for the log */
1236         return "pop3client";
1237 }