X-Git-Url: https://code.citadel.org/?a=blobdiff_plain;f=citadel%2Fmodules%2Fpop3client%2Fserv_pop3client.c;h=2d4c02530adc6470a0b9ed460abfc766c5ed0b8d;hb=66f09fdd7c3f58e13ed01431dd5148302327ca6e;hp=8c051ed4d83ad543a6c37ff9a54e76751b69cdc0;hpb=83f482e23f47fb8931916539a411b33a838d12df;p=citadel.git diff --git a/citadel/modules/pop3client/serv_pop3client.c b/citadel/modules/pop3client/serv_pop3client.c index 8c051ed4d..2d4c02530 100644 --- a/citadel/modules/pop3client/serv_pop3client.c +++ b/citadel/modules/pop3client/serv_pop3client.c @@ -63,12 +63,12 @@ int POP3ClientDebugEnabled = 0; #define EVP3C_syslog(LEVEL, FORMAT, ...) \ DBGLOG(LEVEL) syslog(LEVEL, \ - "IO[%ld]CC[%d][%ld]" FORMAT, \ + "IO[%ld]CC[%d][%ld]POP3: " FORMAT, \ IO->ID, CCID, N, __VA_ARGS__) #define EVP3CM_syslog(LEVEL, FORMAT) \ DBGLOG(LEVEL) syslog(LEVEL, \ - "IO[%ld]CC[%d][%ld]" FORMAT, \ + "IO[%ld]CC[%d][%ld]POP3: " FORMAT, \ IO->ID, CCID, N) #define EVP3CQ_syslog(LEVEL, FORMAT, ...) \ @@ -82,21 +82,23 @@ int POP3ClientDebugEnabled = 0; ) #define EVP3CCS_syslog(LEVEL, FORMAT, ...) \ - DBGLOG(LEVEL) syslog(LEVEL, "IO[%ld][%ld]" FORMAT, \ + DBGLOG(LEVEL) syslog(LEVEL, "IO[%ld][%ld]POP3: " FORMAT, \ IO->ID, N, __VA_ARGS__) -#define EVP3CCSM_syslog(LEVEL, FORMAT) \ - DBGLOG(LEVEL) syslog(LEVEL, "IO[%ld][%ld]" FORMAT, \ +#define EVP3CCSM_syslog(LEVEL, FORMAT) \ + DBGLOG(LEVEL) syslog(LEVEL, "IO[%ld][%ld]POP3: " FORMAT, \ IO->ID, N) #define POP3C_DBG_SEND() \ EVP3C_syslog(LOG_DEBUG, \ - "POP3: > %s\n", \ + "IO[%ld]CC[%d][%ld]POP3: > %s\n", \ + IO->ID, CCID, N, \ ChrPtr(RecvMsg->IO.SendBuf.Buf)) -#define POP3C_DBG_READ() \ - EVP3C_syslog(LOG_DEBUG, \ - "POP3: < %s\n", \ +#define POP3C_DBG_READ() \ + EVP3C_syslog(LOG_DEBUG, \ + "IO[%ld]CC[%d][%ld]POP3: < %s\n", \ + IO->ID, CCID, N, \ ChrPtr(RecvMsg->IO.IOBuf)) @@ -171,6 +173,7 @@ struct pop3aggr { StrBuf *Url; StrBuf *pop3user; StrBuf *pop3pass; + StrBuf *Host; StrBuf *RoomName; // TODO: fill me int keep; time_t interval; @@ -188,6 +191,7 @@ void DeletePOP3Aggregator(void *vptr) // FreeStrBuf(&ptr->rooms); FreeStrBuf(&ptr->pop3user); FreeStrBuf(&ptr->pop3pass); + FreeStrBuf(&ptr->Host); FreeStrBuf(&ptr->RoomName); FreeURL(&ptr->IO.ConnectMe); FreeStrBuf(&ptr->Url); @@ -195,8 +199,10 @@ void DeletePOP3Aggregator(void *vptr) FreeStrBuf(&ptr->IO.SendBuf.Buf); FreeStrBuf(&ptr->IO.RecvBuf.Buf); DeleteAsyncMsg(&ptr->IO.ReadMsg); - ((struct CitContext*)ptr->IO.CitContext)->state = CON_IDLE; - ((struct CitContext*)ptr->IO.CitContext)->kill_me = 1; + if (((struct CitContext*)ptr->IO.CitContext)) { + ((struct CitContext*)ptr->IO.CitContext)->state = CON_IDLE; + ((struct CitContext*)ptr->IO.CitContext)->kill_me = 1; + } FreeAsyncIOContents(&ptr->IO); free(ptr); } @@ -209,7 +215,7 @@ eNextState FinalizePOP3AggrRun(AsyncIO *IO) EVP3C_syslog(LOG_INFO, "%s@%s: fetched %ld new of %d messages in %fs. bye.", ChrPtr(cpptr->pop3user), - ChrPtr(cpptr->pop3pass), + ChrPtr(cpptr->Host), cpptr->count, GetCount(cpptr->MsgNumbers), IO->Now - cpptr->IOStart @@ -326,6 +332,17 @@ eNextState POP3C_GetListOneLine(pop3aggr *RecvMsg) return eSendReply; } + + /* + * work around buggy pop3 servers which send + * empty lines in their listings. + */ + if ((StrLength(RecvMsg->IO.IOBuf) == 0) || + !isdigit(ChrPtr(RecvMsg->IO.IOBuf)[0])) + { + return eReadMore; + } + OneMsg = (FetchItem*) malloc(sizeof(FetchItem)); memset(OneMsg, 0, sizeof(FetchItem)); OneMsg->MSGID = atol(ChrPtr(RecvMsg->IO.IOBuf)); @@ -356,7 +373,6 @@ eNextState POP3_FetchNetworkUsetableEntry(AsyncIO *IO) long HKLen; const char *HKey; void *vData; - struct cdbdata *cdbut; pop3aggr *RecvMsg = (pop3aggr *) IO->Data; if((RecvMsg->Pos != NULL) && @@ -366,32 +382,19 @@ eNextState POP3_FetchNetworkUsetableEntry(AsyncIO *IO) &HKey, &vData)) { - struct UseTable ut; if (server_shutting_down) return eAbort; - RecvMsg->CurrMsg = (FetchItem*) vData; - EVP3CCS_syslog(LOG_DEBUG, - "CHECKING: whether %s has already been seen: ", - ChrPtr(RecvMsg->CurrMsg->MsgUID)); - - /* Find out if we've already seen this item */ - safestrncpy(ut.ut_msgid, - ChrPtr(RecvMsg->CurrMsg->MsgUID), - sizeof(ut.ut_msgid)); - ut.ut_timestamp = time(NULL);/// TODO: libev timestamp! - - cdbut = cdb_fetch(CDB_USETABLE, SKEY(RecvMsg->CurrMsg->MsgUID)); - if (cdbut != NULL) { + if (CheckIfAlreadySeen("POP3 Item Seen", + RecvMsg->CurrMsg->MsgUID, + IO->Now, + IO->Now, //// todo + eCheckUpdate, + IO->ID, CCID) + != 0) + { /* Item has already been seen */ - EVP3CCSM_syslog(LOG_DEBUG, "YES\n"); - cdb_free(cdbut); - - /* rewrite the record anyway, to update the timestamp */ - cdb_store(CDB_USETABLE, - SKEY(RecvMsg->CurrMsg->MsgUID), - &ut, sizeof(struct UseTable) ); - RecvMsg->CurrMsg->NeedFetch = 0; ////TODO0; + RecvMsg->CurrMsg->NeedFetch = 0; } else { @@ -439,7 +442,7 @@ eNextState POP3C_GetOneMessagID(pop3aggr *RecvMsg) } else { - RecvMsg->State++; + RecvMsg->State++; DeleteHashPos(&RecvMsg->Pos); /// done receiving uidls.. start looking them up now. RecvMsg->Pos = GetNewHashPos(RecvMsg->MsgNumbers, 0); @@ -531,21 +534,17 @@ eNextState POP3C_ReadMessageBodyFollowing(pop3aggr *RecvMsg) eNextState POP3C_StoreMsgRead(AsyncIO *IO) { pop3aggr *RecvMsg = (pop3aggr *) IO->Data; - struct UseTable ut; EVP3CCS_syslog(LOG_DEBUG, "MARKING: %s as seen: ", ChrPtr(RecvMsg->CurrMsg->MsgUID)); + CheckIfAlreadySeen("POP3 Item Seen", + RecvMsg->CurrMsg->MsgUID, + IO->Now, + IO->Now, //// todo + eWrite, + IO->ID, CCID); - safestrncpy(ut.ut_msgid, - ChrPtr(RecvMsg->CurrMsg->MsgUID), - sizeof(ut.ut_msgid)); - ut.ut_timestamp = time(NULL); /* TODO: use libev time */ - cdb_store(CDB_USETABLE, - ChrPtr(RecvMsg->CurrMsg->MsgUID), - StrLength(RecvMsg->CurrMsg->MsgUID), - &ut, - sizeof(struct UseTable) ); StopDBWatchers(IO); return QueueEventContext(&RecvMsg->IO, POP3_C_ReAttachToFetchMessages); } @@ -979,22 +978,10 @@ int pop3_do_fetching(pop3aggr *cpptr) /* * Scan a room's netconfig to determine whether it requires POP3 aggregation */ -void pop3client_scan_room(struct ctdlroom *qrbuf, void *data) +void pop3client_scan_room(struct ctdlroom *qrbuf, void *data, OneRoomNetCfg *OneRNCFG) { - StrBuf *CfgData; - StrBuf *CfgType; - StrBuf *Line; - - struct stat statbuf; - char filename[PATH_MAX]; - int fd; - int Done; + const RoomNetCfgLine *pLine; void *vptr; - const char *CfgPtr, *lPtr; - const char *Err; - -// pop3_room_counter *Count = NULL; -// pop3aggr *cpptr; pthread_mutex_lock(&POP3QueueMutex); if (GetHash(POP3QueueRooms, LKEY(qrbuf->QRnumber), &vptr)) @@ -1010,168 +997,88 @@ void pop3client_scan_room(struct ctdlroom *qrbuf, void *data) if (server_shutting_down) return; - assoc_file_name(filename, sizeof filename, qrbuf, ctdl_netcfg_dir); + pLine = OneRNCFG->NetConfigs[pop3client]; - if (server_shutting_down) - return; + while (pLine != NULL) + { + pop3aggr *cptr; - /* Only do net processing for rooms that have netconfigs */ - fd = open(filename, 0); - if (fd <= 0) { - return; - } - if (server_shutting_down) - return; - if (fstat(fd, &statbuf) == -1) { - EVP3CQ_syslog(LOG_INFO, - "ERROR: could not stat configfile '%s' - %s", - filename, - strerror(errno)); - return; - } - if (server_shutting_down) - return; - CfgData = NewStrBufPlain(NULL, statbuf.st_size + 1); - if (StrBufReadBLOB(CfgData, &fd, 1, statbuf.st_size, &Err) < 0) { - close(fd); - FreeStrBuf(&CfgData); - EVP3CQ_syslog(LOG_INFO, - "ERROR: reading config '%s' - %s", - filename, strerror(errno)); - return; - } - close(fd); - if (server_shutting_down) - return; + cptr = (pop3aggr *) malloc(sizeof(pop3aggr)); + memset(cptr, 0, sizeof(pop3aggr)); + ///TODO do we need this? cptr->roomlist_parts=1; + cptr->RoomName = NewStrBufPlain(qrbuf->QRname, -1); + cptr->pop3user = NewStrBufDup(pLine->Value[1]); + cptr->pop3pass = NewStrBufDup(pLine->Value[2]); + cptr->Url = NewStrBuf(); + cptr->Host = NewStrBufDup(pLine->Value[0]); - CfgPtr = NULL; - CfgType = NewStrBuf(); - Line = NewStrBufPlain(NULL, StrLength(CfgData)); - Done = 0; + cptr->keep = atol(ChrPtr(pLine->Value[3])); + cptr->interval = atol(ChrPtr(pLine->Value[4])); - while (!Done) - { - Done = StrBufSipLine(Line, CfgData, &CfgPtr) == 0; - if (StrLength(Line) > 0) - { - lPtr = NULL; - StrBufExtract_NextToken(CfgType, Line, &lPtr, '|'); - if (!strcasecmp("pop3client", ChrPtr(CfgType))) - { - pop3aggr *cptr; - StrBuf *Tmp; -/* - if (Count == NULL) - { - Count = malloc(sizeof(pop3_room_counter)); - Count->count = 0; - } - Count->count ++; -*/ - cptr = (pop3aggr *) malloc(sizeof(pop3aggr)); - memset(cptr, 0, sizeof(pop3aggr)); - ///TODO do we need this? cptr->roomlist_parts=1; - cptr->RoomName = - NewStrBufPlain(qrbuf->QRname, -1); - cptr->pop3user = - NewStrBufPlain(NULL, StrLength(Line)); - cptr->pop3pass = - NewStrBufPlain(NULL, StrLength(Line)); - cptr->Url = NewStrBuf(); - Tmp = NewStrBuf(); - - StrBufExtract_NextToken(Tmp, Line, &lPtr, '|'); - StrBufExtract_NextToken(cptr->pop3user, - Line, - &lPtr, - '|'); - - StrBufExtract_NextToken(cptr->pop3pass, - Line, - &lPtr, - '|'); - - cptr->keep = StrBufExtractNext_long(Line, - &lPtr, - '|'); - - cptr->interval = StrBufExtractNext_long(Line, - &lPtr, - '|'); - - StrBufAppendBufPlain(cptr->Url, HKEY("pop3://"), 0); - StrBufUrlescUPAppend(cptr->Url, cptr->pop3user, NULL); - StrBufAppendBufPlain(cptr->Url, HKEY(":"), 0); - StrBufUrlescUPAppend(cptr->Url, cptr->pop3pass, NULL); - StrBufAppendBufPlain(cptr->Url, HKEY("@"), 0); - StrBufAppendBuf(cptr->Url, Tmp, 0); - StrBufAppendBufPlain(cptr->Url, HKEY("/"), 0); - StrBufUrlescAppend(cptr->Url, cptr->RoomName, NULL); - - FreeStrBuf(&Tmp); - ParseURL(&cptr->IO.ConnectMe, cptr->Url, 110); + StrBufAppendBufPlain(cptr->Url, HKEY("pop3://"), 0); + StrBufUrlescUPAppend(cptr->Url, cptr->pop3user, NULL); + StrBufAppendBufPlain(cptr->Url, HKEY(":"), 0); + StrBufUrlescUPAppend(cptr->Url, cptr->pop3pass, NULL); + StrBufAppendBufPlain(cptr->Url, HKEY("@"), 0); + StrBufAppendBuf(cptr->Url, cptr->Host, 0); + StrBufAppendBufPlain(cptr->Url, HKEY("/"), 0); + StrBufUrlescAppend(cptr->Url, cptr->RoomName, NULL); + + ParseURL(&cptr->IO.ConnectMe, cptr->Url, 110); #if 0 /* todo: we need to reunite the url to be shure. */ - pthread_mutex_lock(&POP3ueueMutex); - GetHash(POP3FetchUrls, SKEY(ptr->Url), &vptr); - use_this_cptr = (pop3aggr *)vptr; + pthread_mutex_lock(&POP3ueueMutex); + GetHash(POP3FetchUrls, SKEY(ptr->Url), &vptr); + use_this_cptr = (pop3aggr *)vptr; - if (use_this_rncptr != NULL) - { - /* mustn't attach to an active session */ - if (use_this_cptr->RefCount > 0) - { - DeletePOP3Cfg(cptr); + if (use_this_rncptr != NULL) + { + /* mustn't attach to an active session */ + if (use_this_cptr->RefCount > 0) + { + DeletePOP3Cfg(cptr); /// Count->count--; - } - else - { - long *QRnumber; - StrBufAppendBufPlain( - use_this_cptr->rooms, - qrbuf->QRname, - -1, 0); - if (use_this_cptr->roomlist_parts == 1) - { - use_this_cptr->OtherQRnumbers - = NewHash(1, lFlathash); - } - QRnumber = (long*)malloc(sizeof(long)); - *QRnumber = qrbuf->QRnumber; - Put(use_this_cptr->OtherQRnumbers, - LKEY(qrbuf->QRnumber), - QRnumber, - NULL); - - use_this_cptr->roomlist_parts++; - } - pthread_mutex_unlock(&POP3QueueMutex); - continue; + } + else + { + long *QRnumber; + StrBufAppendBufPlain( + use_this_cptr->rooms, + qrbuf->QRname, + -1, 0); + if (use_this_cptr->roomlist_parts == 1) + { + use_this_cptr->OtherQRnumbers + = NewHash(1, lFlathash); } - pthread_mutex_unlock(&RSSQueueMutex); -#endif - cptr->n = Pop3ClientID++; - pthread_mutex_lock(&POP3QueueMutex); - Put(POP3FetchUrls, - SKEY(cptr->Url), - cptr, - DeletePOP3Aggregator); - - pthread_mutex_unlock(&POP3QueueMutex); - + QRnumber = (long*)malloc(sizeof(long)); + *QRnumber = qrbuf->QRnumber; + Put(use_this_cptr->OtherQRnumbers, + LKEY(qrbuf->QRnumber), + QRnumber, + NULL); + + use_this_cptr->roomlist_parts++; } - + pthread_mutex_unlock(&POP3QueueMutex); + continue; } + pthread_mutex_unlock(&RSSQueueMutex); +#endif + cptr->n = Pop3ClientID++; + pthread_mutex_lock(&POP3QueueMutex); + Put(POP3FetchUrls, + SKEY(cptr->Url), + cptr, + DeletePOP3Aggregator); - ///fclose(fp); + pthread_mutex_unlock(&POP3QueueMutex); + pLine = pLine->next; } - FreeStrBuf(&Line); - FreeStrBuf(&CfgType); - FreeStrBuf(&CfgData); } static int doing_pop3client = 0; @@ -1209,7 +1116,7 @@ void pop3client_scan(void) { doing_pop3client = 1; EVP3CQM_syslog(LOG_DEBUG, "pop3client started"); - CtdlForEachRoom(pop3client_scan_room, NULL); + CtdlForEachNetCfgRoom(pop3client_scan_room, NULL, pop3client); pthread_mutex_lock(&POP3QueueMutex); it = GetNewHashPos(POP3FetchUrls, 0); @@ -1260,6 +1167,7 @@ CTDL_MODULE_INIT(pop3client) if (!threading) { CtdlFillSystemContext(&pop3_client_CC, "POP3aggr"); + CtdlREGISTERRoomCfgType(pop3client, ParseGeneric, 0, 5, SerializeGeneric, DeleteGenericCfgLine); pthread_mutex_init(&POP3QueueMutex, NULL); POP3QueueRooms = NewHash(1, lFlathash); POP3FetchUrls = NewHash(1, NULL);