fix pop3 Aggregator
[citadel.git] / citadel / modules / pop3client / serv_pop3client.c
index 25c835c7561022bd276c5e8093b879101a1ce8d6..58cf69094c070980402524949f11fa00615e851e 100644 (file)
@@ -21,6 +21,7 @@
 #include <stdlib.h>
 #include <unistd.h>
 #include <stdio.h>
+#include <sysconfig.h>
 
 #if TIME_WITH_SYS_TIME
 # include <sys/time.h>
@@ -63,42 +64,42 @@ int POP3ClientDebugEnabled = 0;
 
 #define EVP3C_syslog(LEVEL, FORMAT, ...)                               \
        DBGLOG(LEVEL) syslog(LEVEL,                                     \
-                            "IO[%ld]CC[%d][%ld]POP3: " FORMAT,         \
-                            IO->ID, CCID, N, __VA_ARGS__)
+                            "%s[%ld]CC[%d][%ld]POP3: " FORMAT,         \
+                            IOSTR, IO->ID, CCID, N, __VA_ARGS__)
 
 #define EVP3CM_syslog(LEVEL, FORMAT)                                   \
        DBGLOG(LEVEL) syslog(LEVEL,                                     \
-                            "IO[%ld]CC[%d][%ld]POP3: " FORMAT,         \
-                            IO->ID, CCID, N)
+                            "%s[%ld]CC[%d][%ld]POP3: " FORMAT,         \
+                            IOSTR, IO->ID, CCID, N)
 
 #define EVP3CQ_syslog(LEVEL, FORMAT, ...)                              \
        DBGLOG(LEVEL) syslog(LEVEL,                                     \
-                            "P3Q:" FORMAT,                             \
-                            __VA_ARGS__)
+                            "%s P3Q:" FORMAT,                          \
+                            IOSTR, __VA_ARGS__)
 
 #define EVP3CQM_syslog(LEVEL, FORMAT)                                  \
        DBGLOG(LEVEL) syslog(LEVEL,                                     \
-                            "P3Q" FORMAT                               \
-               )
+                            "%s P3Q" FORMAT,                           \
+                            IOSTR)
 
 #define EVP3CCS_syslog(LEVEL, FORMAT, ...)                             \
-       DBGLOG(LEVEL) syslog(LEVEL, "IO[%ld][%ld]POP3: " FORMAT,        \
-                            IO->ID, N, __VA_ARGS__)
+       DBGLOG(LEVEL) syslog(LEVEL, "%s[%ld][%ld]POP3: " FORMAT,        \
+                            IOSTR, IO->ID, N, __VA_ARGS__)
 
 #define EVP3CCSM_syslog(LEVEL, FORMAT)                                 \
-       DBGLOG(LEVEL) syslog(LEVEL, "IO[%ld][%ld]POP3: " FORMAT,        \
-                            IO->ID, N)
+       DBGLOG(LEVEL) syslog(LEVEL, "%s[%ld][%ld]POP3: " FORMAT,        \
+                            IOSTR, IO->ID, N)
 
 #define POP3C_DBG_SEND()                                               \
        EVP3C_syslog(LOG_DEBUG,                                         \
-                    "IO[%ld]CC[%d][%ld]POP3: > %s\n",                  \
-                    IO->ID, CCID, N,                                   \
+                    "%s[%ld]CC[%d][%ld]POP3: > %s\n",                  \
+                    IOSTR, IO->ID, CCID, N,                            \
                     ChrPtr(RecvMsg->IO.SendBuf.Buf))
 
 #define POP3C_DBG_READ()                                               \
        EVP3C_syslog(LOG_DEBUG,                                         \
-                    "IO[%ld]CC[%d][%ld]POP3: < %s\n",                  \
-                    IO->ID, CCID, N,                                   \
+                    "%s[%ld]CC[%d][%ld]POP3: < %s\n",                  \
+                    IOSTR, IO->ID, CCID, N,                            \
                     ChrPtr(RecvMsg->IO.IOBuf))
 
 
@@ -158,6 +159,44 @@ void HfreeFetchItem(void *vItem)
        free(Item);
 }
 
+
+
+typedef enum _POP3State {
+       eCreated,
+       eGreeting,
+       eUser,
+       ePassword,
+       eListing,
+       eUseTable,
+       eGetMsgID,
+       eGetMsg,
+       eStoreMsg,
+       eDelete,
+       eQuit
+} POP3State;
+
+ConstStr POP3States[] = {
+       {HKEY("Aggregator created")},
+       {HKEY("Reading Greeting")},
+       {HKEY("Sending User")},
+       {HKEY("Sending Password")},
+       {HKEY("Listing")},
+       {HKEY("Fetching Usetable")},
+       {HKEY("Get MSG ID")},
+       {HKEY("Get Message")},
+       {HKEY("Store Msg")},
+       {HKEY("Delete Upstream")},
+       {HKEY("Quit")}
+};
+
+static void SetPOP3State(AsyncIO *IO, POP3State State)
+{
+       CitContext* CCC = IO->CitContext;
+       if (CCC != NULL)
+               memcpy(CCC->cs_clientname, POP3States[State].Key, POP3States[State].len + 1);
+}
+
+
 struct pop3aggr {
        AsyncIO  IO;
 
@@ -199,8 +238,10 @@ void DeletePOP3Aggregator(void *vptr)
        FreeStrBuf(&ptr->IO.SendBuf.Buf);
        FreeStrBuf(&ptr->IO.RecvBuf.Buf);
        DeleteAsyncMsg(&ptr->IO.ReadMsg);
-       ((struct CitContext*)ptr->IO.CitContext)->state = CON_IDLE;
-       ((struct CitContext*)ptr->IO.CitContext)->kill_me = 1;
+       if (((struct CitContext*)ptr->IO.CitContext)) {
+               ((struct CitContext*)ptr->IO.CitContext)->state = CON_IDLE;
+               ((struct CitContext*)ptr->IO.CitContext)->kill_me = 1;
+       }
        FreeAsyncIOContents(&ptr->IO);
        free(ptr);
 }
@@ -238,6 +279,7 @@ eNextState FailAggregationRun(AsyncIO *IO)
 eNextState POP3C_ReadGreeting(pop3aggr *RecvMsg)
 {
        AsyncIO *IO = &RecvMsg->IO;
+       SetPOP3State(IO, eGreeting);
        POP3C_DBG_READ();
        /* Read the server greeting */
        if (!POP3C_OK) return eTerminateConnection;
@@ -247,6 +289,7 @@ eNextState POP3C_ReadGreeting(pop3aggr *RecvMsg)
 eNextState POP3C_SendUser(pop3aggr *RecvMsg)
 {
        AsyncIO *IO = &RecvMsg->IO;
+       SetPOP3State(IO, eUser);
        /* Identify ourselves.  NOTE: we have to append a CR to each command.
         *  The LF will automatically be appended by sock_puts().  Believe it
         * or not, leaving out the CR will cause problems if the server happens
@@ -270,6 +313,7 @@ eNextState POP3C_GetUserState(pop3aggr *RecvMsg)
 eNextState POP3C_SendPassword(pop3aggr *RecvMsg)
 {
        AsyncIO *IO = &RecvMsg->IO;
+       SetPOP3State(IO, ePassword);
        /* Password */
        StrBufPrintf(RecvMsg->IO.SendBuf.Buf,
                     "PASS %s\r\n", ChrPtr(RecvMsg->pop3pass));
@@ -289,6 +333,8 @@ eNextState POP3C_GetPassState(pop3aggr *RecvMsg)
 eNextState POP3C_SendListCommand(pop3aggr *RecvMsg)
 {
        AsyncIO *IO = &RecvMsg->IO;
+       SetPOP3State(IO, eListing);
+
        /* Get the list of messages */
        StrBufPlain(RecvMsg->IO.SendBuf.Buf, HKEY("LIST\r\n"));
        POP3C_DBG_SEND();
@@ -330,6 +376,17 @@ eNextState POP3C_GetListOneLine(pop3aggr *RecvMsg)
                return eSendReply;
 
        }
+
+       /*
+        * work around buggy pop3 servers which send
+        * empty lines in their listings.
+       */
+       if ((StrLength(RecvMsg->IO.IOBuf) == 0) ||
+           !isdigit(ChrPtr(RecvMsg->IO.IOBuf)[0]))
+       {
+               return eReadMore;
+       }
+
        OneMsg = (FetchItem*) malloc(sizeof(FetchItem));
        memset(OneMsg, 0, sizeof(FetchItem));
        OneMsg->MSGID = atol(ChrPtr(RecvMsg->IO.IOBuf));
@@ -360,8 +417,10 @@ eNextState POP3_FetchNetworkUsetableEntry(AsyncIO *IO)
        long HKLen;
        const char *HKey;
        void *vData;
-       struct cdbdata *cdbut;
        pop3aggr *RecvMsg = (pop3aggr *) IO->Data;
+       time_t seenstamp = 0;
+
+       SetPOP3State(IO, eUseTable);
 
        if((RecvMsg->Pos != NULL) &&
           GetNextHashPos(RecvMsg->MsgNumbers,
@@ -370,32 +429,21 @@ eNextState POP3_FetchNetworkUsetableEntry(AsyncIO *IO)
                          &HKey,
                          &vData))
        {
-               struct UseTable ut;
                if (server_shutting_down)
                        return eAbort;
 
-               RecvMsg->CurrMsg = (FetchItem*) vData;
-               EVP3CCS_syslog(LOG_DEBUG,
-                              "CHECKING: whether %s has already been seen: ",
-                              ChrPtr(RecvMsg->CurrMsg->MsgUID));
-
-               /* Find out if we've already seen this item */
-               safestrncpy(ut.ut_msgid,
-                           ChrPtr(RecvMsg->CurrMsg->MsgUID),
-                           sizeof(ut.ut_msgid));
-               ut.ut_timestamp = time(NULL);/// TODO: libev timestamp!
-
-               cdbut = cdb_fetch(CDB_USETABLE, SKEY(RecvMsg->CurrMsg->MsgUID));
-               if (cdbut != NULL) {
+               RecvMsg->CurrMsg = (FetchItem*)vData;
+
+               seenstamp = CheckIfAlreadySeen("POP3 Item Seen",
+                                              RecvMsg->CurrMsg->MsgUID,
+                                              EvGetNow(IO),
+                                              EvGetNow(IO) - USETABLE_ANTIEXPIRE,
+                                              eCheckUpdate,
+                                              IO->ID, CCID);
+               if (seenstamp != 0)
+               {
                        /* Item has already been seen */
-                       EVP3CCSM_syslog(LOG_DEBUG, "YES\n");
-                       cdb_free(cdbut);
-
-                       /* rewrite the record anyway, to update the timestamp */
-                       cdb_store(CDB_USETABLE,
-                                 SKEY(RecvMsg->CurrMsg->MsgUID),
-                                 &ut, sizeof(struct UseTable) );
-                       RecvMsg->CurrMsg->NeedFetch = 0; ////TODO0;
+                       RecvMsg->CurrMsg->NeedFetch = 0;
                }
                else
                {
@@ -410,8 +458,7 @@ eNextState POP3_FetchNetworkUsetableEntry(AsyncIO *IO)
                /* ok, now we know them all,
                 * continue with reading the actual messages. */
                DeleteHashPos(&RecvMsg->Pos);
-               StopDBWatchers(IO);
-               return QueueEventContext(IO, POP3_C_ReAttachToFetchMessages);
+               return DBQueueEventContext(IO, POP3_C_ReAttachToFetchMessages);
        }
 }
 
@@ -422,6 +469,7 @@ eNextState POP3C_GetOneMessagID(pop3aggr *RecvMsg)
        const char *HKey;
        void *vData;
 
+       SetPOP3State(IO, eGetMsgID);
 #if 0
        int rc;
        rc = TestValidateHash(RecvMsg->MsgNumbers);
@@ -447,8 +495,9 @@ eNextState POP3C_GetOneMessagID(pop3aggr *RecvMsg)
                DeleteHashPos(&RecvMsg->Pos);
                /// done receiving uidls.. start looking them up now.
                RecvMsg->Pos = GetNewHashPos(RecvMsg->MsgNumbers, 0);
-               return QueueDBOperation(&RecvMsg->IO,
-                                       POP3_FetchNetworkUsetableEntry);
+               return EventQueueDBOperation(&RecvMsg->IO,
+                                            POP3_FetchNetworkUsetableEntry,
+                                            0);
        }
        return eReadMore; /* TODO */
 }
@@ -491,6 +540,10 @@ eNextState POP3C_SendGetOneMsg(pop3aggr *RecvMsg)
        const char *HKey;
        void *vData;
 
+       SetPOP3State(IO, eGetMsg);
+
+       EVP3CM_syslog(LOG_DEBUG, "fast forwarding to the next unknown message");
+
        RecvMsg->CurrMsg = NULL;
        while ((RecvMsg->Pos != NULL) && 
               GetNextHashPos(RecvMsg->MsgNumbers,
@@ -503,6 +556,7 @@ eNextState POP3C_SendGetOneMsg(pop3aggr *RecvMsg)
 
        if ((RecvMsg->CurrMsg != NULL ) && (RecvMsg->CurrMsg->NeedFetch == 1))
        {
+               EVP3CM_syslog(LOG_DEBUG, "fetching next");
                /* Message has not been seen.
                 * Tell the server to fetch the message... */
                StrBufPrintf(RecvMsg->IO.SendBuf.Buf,
@@ -511,6 +565,7 @@ eNextState POP3C_SendGetOneMsg(pop3aggr *RecvMsg)
                return eReadMessage;
        }
        else {
+               EVP3CM_syslog(LOG_DEBUG, "no more messages to fetch.");
                RecvMsg->State = ReadQuitState;
                return POP3_C_DispatchWriteDone(&RecvMsg->IO);
        }
@@ -535,23 +590,20 @@ eNextState POP3C_ReadMessageBodyFollowing(pop3aggr *RecvMsg)
 eNextState POP3C_StoreMsgRead(AsyncIO *IO)
 {
        pop3aggr *RecvMsg = (pop3aggr *) IO->Data;
-       struct UseTable ut;
+
+       SetPOP3State(IO, eStoreMsg);
 
        EVP3CCS_syslog(LOG_DEBUG,
                       "MARKING: %s as seen: ",
                       ChrPtr(RecvMsg->CurrMsg->MsgUID));
-
-       safestrncpy(ut.ut_msgid,
-                   ChrPtr(RecvMsg->CurrMsg->MsgUID),
-                   sizeof(ut.ut_msgid));
-       ut.ut_timestamp = time(NULL); /* TODO: use libev time */
-       cdb_store(CDB_USETABLE,
-                 ChrPtr(RecvMsg->CurrMsg->MsgUID),
-                 StrLength(RecvMsg->CurrMsg->MsgUID),
-                 &ut,
-                 sizeof(struct UseTable) );
-       StopDBWatchers(IO);
-       return QueueEventContext(&RecvMsg->IO, POP3_C_ReAttachToFetchMessages);
+       CheckIfAlreadySeen("POP3 Item Seen",
+                          RecvMsg->CurrMsg->MsgUID,
+                          EvGetNow(IO),
+                          EvGetNow(IO) - USETABLE_ANTIEXPIRE,
+                          eWrite,
+                          IO->ID, CCID);
+
+       return DBQueueEventContext(&RecvMsg->IO, POP3_C_ReAttachToFetchMessages);
 }
 eNextState POP3C_SaveMsg(AsyncIO *IO)
 {
@@ -570,7 +622,7 @@ eNextState POP3C_SaveMsg(AsyncIO *IO)
                 * so we don't fetch this message again
                 */
        }
-       CtdlFreeMessage(RecvMsg->CurrMsg->Msg);
+       CM_Free(RecvMsg->CurrMsg->Msg);
 
        RecvMsg->count ++;
        return NextDBOperation(&RecvMsg->IO, POP3C_StoreMsgRead);
@@ -582,13 +634,15 @@ eNextState POP3C_ReadMessageBody(pop3aggr *RecvMsg)
        EVP3CM_syslog(LOG_DEBUG, "Converting message...");
        RecvMsg->CurrMsg->Msg =
                convert_internet_message_buf(&RecvMsg->IO.ReadMsg->MsgBuf);
-       StopClientWatchers(IO, 0);
-       return QueueDBOperation(&RecvMsg->IO, POP3C_SaveMsg);
+       return EventQueueDBOperation(&RecvMsg->IO, POP3C_SaveMsg, 0);
 }
 
 eNextState POP3C_SendDelete(pop3aggr *RecvMsg)
 {
        AsyncIO *IO = &RecvMsg->IO;
+
+       SetPOP3State(IO, eDelete);
+
        if (!RecvMsg->keep) {
                StrBufPrintf(RecvMsg->IO.SendBuf.Buf,
                             "DELE %ld\r\n", RecvMsg->CurrMsg->MSGID);
@@ -605,12 +659,14 @@ eNextState POP3C_ReadDeleteState(pop3aggr *RecvMsg)
        AsyncIO *IO = &RecvMsg->IO;
        POP3C_DBG_READ();
        RecvMsg->State = GetOneMessageIDState;
-       return eReadMessage;
+       return POP3_C_DispatchWriteDone(&RecvMsg->IO);
 }
 
 eNextState POP3C_SendQuit(pop3aggr *RecvMsg)
 {
        AsyncIO *IO = &RecvMsg->IO;
+       SetPOP3State(IO, eQuit);
+
        /* Log out */
        StrBufPlain(RecvMsg->IO.SendBuf.Buf,
                    HKEY("QUIT\r\n3)"));
@@ -1081,6 +1137,7 @@ void pop3client_scan_room(struct ctdlroom *qrbuf, void *data, OneRoomNetCfg *One
                    DeletePOP3Aggregator);
 
                pthread_mutex_unlock(&POP3QueueMutex);
+               pLine = pLine->next;
 
        }
 }