X-Git-Url: https://code.citadel.org/?p=citadel.git;a=blobdiff_plain;f=citadel%2Fmodules%2Frssclient%2Fserv_rssclient.c;h=b8d1b4495ec9cdd260d13ff61b0caf50094f09e6;hp=703387da0a9e1c132ebff234338d1717ecdffd4f;hb=91acc8a0f1f9c4638613c6d2c177e28db1027366;hpb=a7ab5f292afc141361382a11f777b3ff5703ca20 diff --git a/citadel/modules/rssclient/serv_rssclient.c b/citadel/modules/rssclient/serv_rssclient.c index 703387da0..b8d1b4495 100644 --- a/citadel/modules/rssclient/serv_rssclient.c +++ b/citadel/modules/rssclient/serv_rssclient.c @@ -1,8 +1,21 @@ /* - * $Id$ - * * Bring external RSS feeds into rooms. * + * Copyright (c) 2007-2010 by the citadel.org team + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ #include @@ -34,335 +47,373 @@ #include "support.h" #include "config.h" #include "threads.h" -#include "room_ops.h" #include "ctdl_module.h" -#include "clientsocket.h" #include "msgbase.h" #include "parsedate.h" #include "database.h" #include "citadel_dirs.h" #include "md5.h" +#include "context.h" +#include "event_client.h" +#include "rss_atom_parser.h" -struct rssnetcfg { - struct rssnetcfg *next; - char url[256]; - char *rooms; -}; - -struct rss_item { - char *chardata; - int chardata_len; - char *roomlist; - int done_parsing; - char *guid; - char *title; - char *link; - char *description; - time_t pubdate; - char channel_title[256]; - int item_tag_nesting; - char *author_or_creator; -}; +#define TMP_MSGDATA 0xFF +#define TMP_SHORTER_URL_OFFSET 0xFE +#define TMP_SHORTER_URLS 0xFD -struct rssnetcfg *rnclist = NULL; +time_t last_run = 0L; +pthread_mutex_t RSSQueueMutex; /* locks the access to the following vars: */ +HashList *RSSQueueRooms = NULL; /* rss_room_counter */ +HashList *RSSFetchUrls = NULL; /* -> rss_aggregator; ->RefCount access to be locked too. */ -/* - * Commit a fetched and parsed RSS item to disk - */ -void rss_save_item(struct rss_item *ri) { +eNextState RSSAggregatorTerminate(AsyncIO *IO); - struct MD5Context md5context; - u_char rawdigest[MD5_DIGEST_LEN]; - int i; - char utmsgid[SIZ]; - struct cdbdata *cdbut; - struct UseTable ut; - struct CtdlMessage *msg; - struct recptypes *recp = NULL; - int msglen = 0; +struct CitContext rss_CC; - recp = (struct recptypes *) malloc(sizeof(struct recptypes)); - if (recp == NULL) return; - memset(recp, 0, sizeof(struct recptypes)); - recp->recp_room = strdup(ri->roomlist); - recp->num_room = num_tokens(ri->roomlist, '|'); - recp->recptypes_magic = RECPTYPES_MAGIC; - - /* Construct a GUID to use in the S_USETABLE table. - * If one is not present in the item itself, make one up. - */ - if (ri->guid != NULL) { - snprintf(utmsgid, sizeof utmsgid, "rss/%s", ri->guid); - } - else { - MD5Init(&md5context); - if (ri->title != NULL) { - MD5Update(&md5context, (unsigned char*)ri->title, strlen(ri->title)); - } - if (ri->link != NULL) { - MD5Update(&md5context, (unsigned char*)ri->link, strlen(ri->link)); - } - MD5Final(rawdigest, &md5context); - for (i=0; i 0) + { + StrBufAppendBufPlain(Message, HKEY(""), 0); + if (StrLength(LinkTitle) > 0) + StrBufAppendBuf(Message, LinkTitle, 0); + else if ((Title != NULL) && !IsEmptyStr(Title)) + StrBufAppendBufPlain(Message, Title, -1, 0); + else + StrBufAppendBuf(Message, link, 0); + StrBufAppendBufPlain(Message, HKEY("
\n"), 0); } +} +typedef struct __networker_save_message { + AsyncIO IO; + struct CtdlMessage *Msg; + struct recptypes *recp; + rss_aggregator *Cfg; + StrBuf *MsgGUID; + StrBuf *Message; + struct UseTable ut; +} networker_save_message; - /* Find out if we've already seen this item */ - cdbut = cdb_fetch(CDB_USETABLE, utmsgid, strlen(utmsgid)); - if (cdbut != NULL) { - /* Item has already been seen */ - CtdlLogPrintf(CTDL_DEBUG, "%s has already been seen\n", utmsgid); - cdb_free(cdbut); - /* rewrite the record anyway, to update the timestamp */ - strcpy(ut.ut_msgid, utmsgid); - ut.ut_timestamp = time(NULL); - cdb_store(CDB_USETABLE, utmsgid, strlen(utmsgid), &ut, sizeof(struct UseTable) ); - } - else { - /* Item has not been seen, so save it. */ +void DeleteRoomReference(long QRnumber) +{ + HashPos *At; + long HKLen; + const char *HK; + void *vData = NULL; + rss_room_counter *pRoomC; - if (ri->description == NULL) ri->description = strdup(""); - for (i=strlen(ri->description); i>=0; --i) { - if (isspace(ri->description[i])) { - ri->description[i] = ' '; - } - } + At = GetNewHashPos(RSSQueueRooms, 0); - msg = malloc(sizeof(struct CtdlMessage)); - memset(msg, 0, sizeof(struct CtdlMessage)); - msg->cm_magic = CTDLMESSAGE_MAGIC; - msg->cm_anon_type = MES_NORMAL; - msg->cm_format_type = FMT_RFC822; + GetHashPosFromKey(RSSQueueRooms, LKEY(QRnumber), At); + GetHashPos(RSSQueueRooms, At, &HKLen, &HK, &vData); + if (vData != NULL) + { + pRoomC = (rss_room_counter *) vData; + pRoomC->count --; + if (pRoomC->count == 0) + DeleteEntryFromHash(RSSQueueRooms, At); + } + DeleteHashPos(&At); +} - if (ri->author_or_creator != NULL) { - msg->cm_fields['A'] = html_to_ascii(ri->author_or_creator, - strlen(ri->author_or_creator), 512, 0); - striplt(msg->cm_fields['A']); - } - else { - msg->cm_fields['A'] = strdup("rss"); +void UnlinkRooms(rss_aggregator *Cfg) +{ + + DeleteRoomReference(Cfg->QRnumber); + if (Cfg->OtherQRnumbers != NULL) + { + long HKLen; + const char *HK; + HashPos *At; + void *vData; + + At = GetNewHashPos(Cfg->OtherQRnumbers, 0); + while (GetNextHashPos(Cfg->OtherQRnumbers, At, &HKLen, &HK, &vData) && + (vData != NULL)) + { + long *lData = (long*) vData; + DeleteRoomReference(*lData); } +/* + if (server_shutting_down) + break; / * TODO */ - msg->cm_fields['N'] = strdup(NODENAME); - if (ri->title != NULL) { - msg->cm_fields['U'] = html_to_ascii(ri->title, strlen(ri->title), 512, 0); - striplt(msg->cm_fields['U']); - } - msg->cm_fields['T'] = malloc(64); - snprintf(msg->cm_fields['T'], 64, "%ld", ri->pubdate); - if (ri->channel_title != NULL) { - if (!IsEmptyStr(ri->channel_title)) { - msg->cm_fields['O'] = strdup(ri->channel_title); - } - } - if (ri->link == NULL) - ri->link = strdup(""); - msglen += 1024 + strlen(ri->link) + strlen(ri->description) ; - msg->cm_fields['M'] = malloc(msglen); - snprintf(msg->cm_fields['M'], msglen, - "Content-type: text/html\r\n\r\n" - "\n" - "%s

\n" - "%s\n" - "\n" - , - ri->description, - ri->link, ri->link - ); - - CtdlSubmitMsg(msg, recp, NULL, 0); - CtdlFreeMessage(msg); - - /* write the uidl to the use table so we don't store this item again */ - strcpy(ut.ut_msgid, utmsgid); - ut.ut_timestamp = time(NULL); - cdb_store(CDB_USETABLE, utmsgid, strlen(utmsgid), &ut, sizeof(struct UseTable) ); + DeleteHashPos(&At); } - free_recipients(recp); } +void UnlinkRSSAggregator(rss_aggregator *Cfg) +{ + HashPos *At; + UnlinkRooms(Cfg); -/* - * Convert an RDF/RSS datestamp into a time_t - */ -time_t rdf_parsedate(char *p) + At = GetNewHashPos(RSSFetchUrls, 0); + if (GetHashPosFromKey(RSSFetchUrls, SKEY(Cfg->Url), At) == 0) + { + DeleteEntryFromHash(RSSFetchUrls, At); + } + DeleteHashPos(&At); + last_run = time(NULL); +} + +eNextState FreeNetworkSaveMessage (AsyncIO *IO) { - struct tm tm; - time_t t = 0; + networker_save_message *Ctx = (networker_save_message *) IO->Data; - if (!p) return 0L; - if (strlen(p) < 10) return 0L; + pthread_mutex_lock(&RSSQueueMutex); + Ctx->Cfg->RefCount --; - memset(&tm, 0, sizeof tm); + if (Ctx->Cfg->RefCount == 0) + { + UnlinkRSSAggregator(Ctx->Cfg); - /* YYYY-MM-DDTHH:MM format... - */ - if ( (p[4] == '-') && (p[7] == '-') ) { - tm.tm_year = atoi(&p[0]) - 1900; - tm.tm_mon = atoi(&p[5]) - 1; - tm.tm_mday = atoi(&p[8]); - if ( (p[10] == 'T') && (p[13] == ':') ) { - tm.tm_hour = atoi(&p[11]); - tm.tm_min = atoi(&p[14]); - } - return mktime(&tm); } + pthread_mutex_unlock(&RSSQueueMutex); - /* hmm... try RFC822 date stamp format */ + CtdlFreeMessage(Ctx->Msg); + free_recipients(Ctx->recp); + FreeStrBuf(&Ctx->Message); + FreeStrBuf(&Ctx->MsgGUID); + free(Ctx); + last_run = time(NULL); + return eAbort; +} + +eNextState AbortNetworkSaveMessage (AsyncIO *IO) +{ + return eAbort; ///TODO +} + +eNextState RSSSaveMessage(AsyncIO *IO) +{ + networker_save_message *Ctx = (networker_save_message *) IO->Data; - t = parsedate(p); - if (t > 0) return(t); + Ctx->Msg->cm_fields['M'] = SmashStrBuf(&Ctx->Message); - /* yeesh. ok, just return the current date and time. */ - return(time(NULL)); + CtdlSubmitMsg(Ctx->Msg, Ctx->recp, NULL, 0); + + /* write the uidl to the use table so we don't store this item again */ + cdb_store(CDB_USETABLE, SKEY(Ctx->MsgGUID), &Ctx->ut, sizeof(struct UseTable) ); + + return eTerminateConnection; } +eNextState RSS_FetchNetworkUsetableEntry(AsyncIO *IO) +{ + struct cdbdata *cdbut; + networker_save_message *Ctx = (networker_save_message *) IO->Data; + /* Find out if we've already seen this item */ + strcpy(Ctx->ut.ut_msgid, ChrPtr(Ctx->MsgGUID)); /// TODO + Ctx->ut.ut_timestamp = time(NULL); -void rss_xml_start(void *data, const char *supplied_el, const char **attr) { - struct rss_item *ri = (struct rss_item *) data; - char el[256]; - char *sep = NULL; + cdbut = cdb_fetch(CDB_USETABLE, SKEY(Ctx->MsgGUID)); +#ifndef DEBUG_RSS + if (cdbut != NULL) { + /* Item has already been seen */ + syslog(LOG_DEBUG, "%s has already been seen\n", ChrPtr(Ctx->MsgGUID)); + cdb_free(cdbut); - /* Axe the namespace, we don't care about it */ - safestrncpy(el, supplied_el, sizeof el); - while (sep = strchr(el, ':'), sep) { - strcpy(el, ++sep); + /* rewrite the record anyway, to update the timestamp */ + cdb_store(CDB_USETABLE, + SKEY(Ctx->MsgGUID), + &Ctx->ut, sizeof(struct UseTable) ); + return eAbort; } - - if (!strcasecmp(el, "item")) { - ++ri->item_tag_nesting; - - /* Initialize the feed item data structure */ - if (ri->guid != NULL) free(ri->guid); - ri->guid = NULL; - if (ri->title != NULL) free(ri->title); - ri->title = NULL; - if (ri->link != NULL) free(ri->link); - ri->link = NULL; - if (ri->author_or_creator != NULL) free(ri->author_or_creator); - ri->author_or_creator = NULL; - if (ri->description != NULL) free(ri->description); - ri->description = NULL; - - /* Throw away any existing character data */ - if (ri->chardata_len > 0) { - free(ri->chardata); - ri->chardata = 0; - ri->chardata_len = 0; - } + else +#endif + { + NextDBOperation(IO, RSSSaveMessage); + return eSendMore; } +} +void RSSQueueSaveMessage(struct CtdlMessage *Msg, struct recptypes *recp, StrBuf *MsgGUID, StrBuf *MessageBody, rss_aggregator *Cfg) +{ + networker_save_message *Ctx; + Ctx = (networker_save_message *) malloc(sizeof(networker_save_message)); + memset(Ctx, 0, sizeof(networker_save_message)); + + Ctx->MsgGUID = MsgGUID; + Ctx->Message = MessageBody; + Ctx->Msg = Msg; + Ctx->Cfg = Cfg; + Ctx->recp = recp; + Ctx->IO.Data = Ctx; + Ctx->IO.CitContext = CloneContext(&rss_CC); + Ctx->IO.Terminate = FreeNetworkSaveMessage; + Ctx->IO.ShutdownAbort = AbortNetworkSaveMessage; + QueueDBOperation(&Ctx->IO, RSS_FetchNetworkUsetableEntry); +} -} +/* + * Commit a fetched and parsed RSS item to disk + */ +void rss_save_item(rss_item *ri, rss_aggregator *Cfg) +{ -void rss_xml_end(void *data, const char *supplied_el) { - struct rss_item *ri = (struct rss_item *) data; - char el[256]; - char *sep = NULL; + struct MD5Context md5context; + u_char rawdigest[MD5_DIGEST_LEN]; + struct CtdlMessage *msg; + struct recptypes *recp = NULL; + int msglen = 0; + StrBuf *Message; + StrBuf *guid; + StrBuf *Buf; - /* Axe the namespace, we don't care about it */ - safestrncpy(el, supplied_el, sizeof el); - while (sep = strchr(el, ':'), sep) { - strcpy(el, ++sep); + recp = (struct recptypes *) malloc(sizeof(struct recptypes)); + if (recp == NULL) return; + memset(recp, 0, sizeof(struct recptypes)); + Buf = NewStrBufDup(Cfg->rooms); + recp->recp_room = SmashStrBuf(&Buf); + recp->num_room = Cfg->roomlist_parts; + recp->recptypes_magic = RECPTYPES_MAGIC; + + Cfg->RefCount ++; + /* Construct a GUID to use in the S_USETABLE table. + * If one is not present in the item itself, make one up. + */ + if (ri->guid != NULL) { + StrBufSpaceToBlank(ri->guid); + StrBufTrim(ri->guid); + guid = NewStrBufPlain(HKEY("rss/")); + StrBufAppendBuf(guid, ri->guid, 0); } - - if ( (!strcasecmp(el, "title")) && (ri->item_tag_nesting == 0) && (ri->chardata != NULL) ) { - safestrncpy(ri->channel_title, ri->chardata, sizeof ri->channel_title); - striplt(ri->channel_title); + else { + MD5Init(&md5context); + if (ri->title != NULL) { + MD5Update(&md5context, (const unsigned char*)ChrPtr(ri->title), StrLength(ri->title)); + } + if (ri->link != NULL) { + MD5Update(&md5context, (const unsigned char*)ChrPtr(ri->link), StrLength(ri->link)); + } + MD5Final(rawdigest, &md5context); + guid = NewStrBufPlain(NULL, MD5_DIGEST_LEN * 2 + 12 /* _rss2ctdl*/); + StrBufHexEscAppend(guid, NULL, rawdigest, MD5_DIGEST_LEN); + StrBufAppendBufPlain(guid, HKEY("_rss2ctdl"), 0); } - if ( (!strcasecmp(el, "guid")) && (ri->chardata != NULL) ) { - if (ri->guid != NULL) free(ri->guid); - striplt(ri->chardata); - ri->guid = strdup(ri->chardata); - } + /* translate Item into message. */ + syslog(LOG_DEBUG, "RSS: translating item...\n"); + if (ri->description == NULL) ri->description = NewStrBufPlain(HKEY("")); + StrBufSpaceToBlank(ri->description); + msg = malloc(sizeof(struct CtdlMessage)); + memset(msg, 0, sizeof(struct CtdlMessage)); + msg->cm_magic = CTDLMESSAGE_MAGIC; + msg->cm_anon_type = MES_NORMAL; + msg->cm_format_type = FMT_RFC822; - if ( (!strcasecmp(el, "title")) && (ri->chardata != NULL) ) { - if (ri->title != NULL) free(ri->title); - striplt(ri->chardata); - ri->title = strdup(ri->chardata); + if (ri->guid != NULL) { + msg->cm_fields['E'] = strdup(ChrPtr(ri->guid)); } - if ( (!strcasecmp(el, "link")) && (ri->chardata != NULL) ) { - if (ri->link != NULL) free(ri->link); - striplt(ri->chardata); - ri->link = strdup(ri->chardata); - } + if (ri->author_or_creator != NULL) { + char *From; + StrBuf *Encoded = NULL; + int FromAt; + + From = html_to_ascii(ChrPtr(ri->author_or_creator), + StrLength(ri->author_or_creator), + 512, 0); + StrBufPlain(ri->author_or_creator, From, -1); + StrBufTrim(ri->author_or_creator); + free(From); + + FromAt = strchr(ChrPtr(ri->author_or_creator), '@') != NULL; + if (!FromAt && StrLength (ri->author_email) > 0) + { + StrBufRFC2047encode(&Encoded, ri->author_or_creator); + msg->cm_fields['A'] = SmashStrBuf(&Encoded); + msg->cm_fields['P'] = SmashStrBuf(&ri->author_email); + } + else + { + if (FromAt) + { + msg->cm_fields['A'] = SmashStrBuf(&ri->author_or_creator); + msg->cm_fields['P'] = strdup(msg->cm_fields['A']); + } + else + { + StrBufRFC2047encode(&Encoded, ri->author_or_creator); + msg->cm_fields['A'] = SmashStrBuf(&Encoded); + msg->cm_fields['P'] = strdup("rss@localhost"); - if ( (!strcasecmp(el, "description")) && (ri->chardata != NULL) ) { - if (ri->description != NULL) free(ri->description); - ri->description = strdup(ri->chardata); + } + if (ri->pubdate <= 0) { + ri->pubdate = time(NULL); + } + } } - - if ( ((!strcasecmp(el, "pubdate")) || (!strcasecmp(el, "date"))) && (ri->chardata != NULL) ) { - striplt(ri->chardata); - ri->pubdate = rdf_parsedate(ri->chardata); + else { + msg->cm_fields['A'] = strdup("rss"); } - if ( ((!strcasecmp(el, "author")) || (!strcasecmp(el, "creator"))) && (ri->chardata != NULL) ) { - if (ri->author_or_creator != NULL) free(ri->author_or_creator); - striplt(ri->chardata); - ri->author_or_creator = strdup(ri->chardata); - } + msg->cm_fields['N'] = strdup(NODENAME); + if (ri->title != NULL) { + long len; + char *Sbj; + StrBuf *Encoded, *QPEncoded; + + QPEncoded = NULL; + StrBufSpaceToBlank(ri->title); + len = StrLength(ri->title); + Sbj = html_to_ascii(ChrPtr(ri->title), len, 512, 0); + len = strlen(Sbj); + if (Sbj[len - 1] == '\n') + { + len --; + Sbj[len] = '\0'; + } + Encoded = NewStrBufPlain(Sbj, len); + free(Sbj); - if (!strcasecmp(el, "item")) { - --ri->item_tag_nesting; - rss_save_item(ri); - } + StrBufTrim(Encoded); + StrBufRFC2047encode(&QPEncoded, Encoded); - if ( (!strcasecmp(el, "rss")) || (!strcasecmp(el, "rdf")) ) { - CtdlLogPrintf(CTDL_DEBUG, "End of feed detected. Closing parser.\n"); - ri->done_parsing = 1; + msg->cm_fields['U'] = SmashStrBuf(&QPEncoded); + FreeStrBuf(&Encoded); } - - if (ri->chardata_len > 0) { - free(ri->chardata); - ri->chardata = 0; - ri->chardata_len = 0; + msg->cm_fields['T'] = malloc(64); + snprintf(msg->cm_fields['T'], 64, "%ld", ri->pubdate); + if (ri->channel_title != NULL) { + if (StrLength(ri->channel_title) > 0) { + msg->cm_fields['O'] = strdup(ChrPtr(ri->channel_title)); + } } + if (ri->link == NULL) + ri->link = NewStrBufPlain(HKEY("")); -} +#if 0 /* temporarily disable shorter urls. */ + msg->cm_fields[TMP_SHORTER_URLS] = GetShorterUrls(ri->description); +#endif + msglen += 1024 + StrLength(ri->link) + StrLength(ri->description) ; -/* - * This callback stores up the data which appears in between tags. - */ -void rss_xml_chardata(void *data, const XML_Char *s, int len) { - struct rss_item *ri = (struct rss_item *) data; - int old_len; - int new_len; - char *new_buffer; - - old_len = ri->chardata_len; - new_len = old_len + len; - new_buffer = realloc(ri->chardata, new_len + 1); - if (new_buffer != NULL) { - memcpy(&new_buffer[old_len], s, len); - new_buffer[new_len] = 0; - ri->chardata = new_buffer; - ri->chardata_len = new_len; - } -} + Message = NewStrBufPlain(NULL, StrLength(ri->description)); + StrBufPlain(Message, HKEY( + "Content-type: text/html; charset=\"UTF-8\"\r\n\r\n" + "\n")); +#if 0 /* disable shorter url for now. */ + msg->cm_fields[TMP_SHORTER_URL_OFFSET] = StrLength(Message); +#endif + StrBufAppendBuf(Message, ri->description, 0); + StrBufAppendBufPlain(Message, HKEY("

\n"), 0); + AppendLink(Message, ri->link, ri->linkTitle, NULL); + AppendLink(Message, ri->reLink, ri->reLinkTitle, "Reply to this"); + StrBufAppendBufPlain(Message, HKEY("\n"), 0); -/* - * Callback function for passing libcurl's output to expat for parsing - */ -size_t rss_libcurl_callback(void *ptr, size_t size, size_t nmemb, void *stream) -{ - XML_Parse((XML_Parser)stream, ptr, (size * nmemb), 0); - return (size*nmemb); + RSSQueueSaveMessage(msg, recp, guid, Message, Cfg); } @@ -370,190 +421,277 @@ size_t rss_libcurl_callback(void *ptr, size_t size, size_t nmemb, void *stream) /* * Begin a feed parse */ -void rss_do_fetching(char *url, char *rooms) { - struct rss_item ri; - XML_Parser xp; +int rss_do_fetching(rss_aggregator *Cfg) +{ + rss_item *ri; + + time_t now; + AsyncIO *IO; - CURL *curl; - CURLcode res; - char errmsg[1024] = ""; + now = time(NULL); - CtdlLogPrintf(CTDL_DEBUG, "Fetching RSS feed <%s>\n", url); + if ((Cfg->next_poll != 0) && (now < Cfg->next_poll)) + return 0; + Cfg->RefCount = 1; - curl = curl_easy_init(); - if (!curl) { - CtdlLogPrintf(CTDL_ALERT, "Unable to initialize libcurl.\n"); - return; - } + ri = (rss_item*) malloc(sizeof(rss_item)); + memset(ri, 0, sizeof(rss_item)); + Cfg->Item = ri; + IO = &Cfg->IO; + IO->CitContext = CloneContext(&rss_CC); + IO->Data = Cfg; - xp = XML_ParserCreateNS("UTF-8", ':'); - if (!xp) { - CtdlLogPrintf(CTDL_ALERT, "Cannot create XML parser!\n"); - curl_easy_cleanup(curl); - return; - } - curl_easy_setopt(curl, CURLOPT_URL, url); - curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 0); - curl_easy_setopt(curl, CURLOPT_SSL_VERIFYHOST, 0); - curl_easy_setopt(curl, CURLOPT_WRITEDATA, xp); - curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, rss_libcurl_callback); - curl_easy_setopt(curl, CURLOPT_ERRORBUFFER, errmsg); - curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1); -#ifdef CURLOPT_HTTP_CONTENT_DECODING - curl_easy_setopt(curl, CURLOPT_HTTP_CONTENT_DECODING, 1); - curl_easy_setopt(curl, CURLOPT_ENCODING, ""); -#endif - curl_easy_setopt(curl, CURLOPT_USERAGENT, CITADEL); - curl_easy_setopt(curl, CURLOPT_TIMEOUT, 180); /* die after 180 seconds */ - if (!IsEmptyStr(config.c_ip_addr)) { - curl_easy_setopt(curl, CURLOPT_INTERFACE, config.c_ip_addr); + syslog(LOG_DEBUG, "Fetching RSS feed <%s>\n", ChrPtr(Cfg->Url)); + ParseURL(&IO->ConnectMe, Cfg->Url, 80); + CurlPrepareURL(IO->ConnectMe); + + if (! evcurl_init(IO, +// Ctx, + NULL, + "Citadel RSS Client", + ParseRSSReply, + RSSAggregatorTerminate)) + { + syslog(LOG_DEBUG, "Unable to initialize libcurl.\n"); + return 0; } - memset(&ri, 0, sizeof(struct rss_item)); - ri.roomlist = rooms; - XML_SetElementHandler(xp, rss_xml_start, rss_xml_end); - XML_SetCharacterDataHandler(xp, rss_xml_chardata); - XML_SetUserData(xp, &ri); + evcurl_handle_start(IO); + return 1; +} + + +void DeleteRssCfg(void *vptr) +{ + rss_aggregator *rncptr = (rss_aggregator *)vptr; - if (CtdlThreadCheckStop()) + FreeStrBuf(&rncptr->Url); + FreeStrBuf(&rncptr->rooms); + FreeStrBuf(&rncptr->CData); + FreeStrBuf(&rncptr->Key); + FreeStrBuf(&rncptr->IO.HttpReq.ReplyData); + DeleteHash(&rncptr->OtherQRnumbers); + FreeURL(&rncptr->IO.ConnectMe); + + if (rncptr->Item != NULL) { - XML_ParserFree(xp); - curl_easy_cleanup(curl); - return; + FreeStrBuf(&rncptr->Item->guid); + FreeStrBuf(&rncptr->Item->title); + FreeStrBuf(&rncptr->Item->link); + FreeStrBuf(&rncptr->Item->linkTitle); + FreeStrBuf(&rncptr->Item->reLink); + FreeStrBuf(&rncptr->Item->reLinkTitle); + FreeStrBuf(&rncptr->Item->description); + FreeStrBuf(&rncptr->Item->channel_title); + FreeStrBuf(&rncptr->Item->author_or_creator); + FreeStrBuf(&rncptr->Item->author_url); + FreeStrBuf(&rncptr->Item->author_email); + + free(rncptr->Item); } - - if (CtdlThreadCheckStop()) - goto shutdown ; + free(rncptr); +} - res = curl_easy_perform(curl); - if (res) { - CtdlLogPrintf(CTDL_ALERT, "libcurl error %d: %s\n", res, errmsg); - } +eNextState RSSAggregatorTerminate(AsyncIO *IO) +{ + rss_aggregator *rncptr = (rss_aggregator *)IO->Data; + /* + HashPos *At; + long HKLen; + const char *HK; + void *vData; + */ + pthread_mutex_lock(&RSSQueueMutex); + rncptr->RefCount --; + if (rncptr->RefCount == 0) + { + UnlinkRSSAggregator(rncptr); - if (CtdlThreadCheckStop()) - goto shutdown ; - - if (ri.done_parsing == 0) XML_Parse(xp, "", 0, 1); - - -shutdown: - curl_easy_cleanup(curl); - XML_ParserFree(xp); - - /* Free the feed item data structure */ - if (ri.guid != NULL) free(ri.guid); - ri.guid = NULL; - if (ri.title != NULL) free(ri.title); - ri.title = NULL; - if (ri.link != NULL) free(ri.link); - ri.link = NULL; - if (ri.author_or_creator != NULL) free(ri.author_or_creator); - ri.author_or_creator = NULL; - if (ri.description != NULL) free(ri.description); - ri.description = NULL; - if (ri.chardata_len > 0) { - free(ri.chardata); - ri.chardata = 0; - ri.chardata_len = 0; } -} + pthread_mutex_unlock(&RSSQueueMutex); +/* + At = GetNewHashPos(RSSFetchUrls, 0); + pthread_mutex_lock(&RSSQueueMutex); + GetHashPosFromKey(RSSFetchUrls, SKEY(rncptr->Url), At); + GetHashPos(RSSFetchUrls, At, &HKLen, &HK, &vData); + DeleteEntryFromHash(RSSFetchUrls, At); + pthread_mutex_unlock(&RSSQueueMutex); + + DeleteHashPos(&At); +*/ + return eAbort; +} /* * Scan a room's netconfig to determine whether it is requesting any RSS feeds */ void rssclient_scan_room(struct ctdlroom *qrbuf, void *data) { + StrBuf *CfgData=NULL; + StrBuf *CfgType; + StrBuf *Line; + rss_room_counter *Count = NULL; + struct stat statbuf; char filename[PATH_MAX]; - char buf[1024]; - char instr[32]; - FILE *fp; - char feedurl[256]; - struct rssnetcfg *rncptr = NULL; - struct rssnetcfg *use_this_rncptr = NULL; - int len = 0; - char *ptr = NULL; + int fd; + int Done; + rss_aggregator *rncptr = NULL; + rss_aggregator *use_this_rncptr = NULL; + void *vptr; + const char *CfgPtr, *lPtr; + const char *Err; + + pthread_mutex_lock(&RSSQueueMutex); + if (GetHash(RSSQueueRooms, LKEY(qrbuf->QRnumber), &vptr)) + { + syslog(LOG_DEBUG, + "rssclient: [%ld] %s already in progress.\n", + qrbuf->QRnumber, + qrbuf->QRname); + pthread_mutex_unlock(&RSSQueueMutex); + return; + } + pthread_mutex_unlock(&RSSQueueMutex); assoc_file_name(filename, sizeof filename, qrbuf, ctdl_netcfg_dir); - if (CtdlThreadCheckStop()) + if (server_shutting_down) return; /* Only do net processing for rooms that have netconfigs */ - fp = fopen(filename, "r"); - if (fp == NULL) { + fd = open(filename, 0); + if (fd <= 0) { + //syslog(LOG_DEBUG, "rssclient: %s no config.\n", qrbuf->QRname); return; } - while (fgets(buf, sizeof buf, fp) != NULL && !CtdlThreadCheckStop()) { - buf[strlen(buf)-1] = 0; - - extract_token(instr, buf, 0, '|', sizeof instr); - if (!strcasecmp(instr, "rssclient")) { - - use_this_rncptr = NULL; + if (server_shutting_down) + return; - extract_token(feedurl, buf, 1, '|', sizeof feedurl); + if (fstat(fd, &statbuf) == -1) { + syslog(LOG_DEBUG, "ERROR: could not stat configfile '%s' - %s\n", + filename, strerror(errno)); + return; + } - /* If any other rooms have requested the same feed, then we will just add this - * room to the target list for that client request. - */ - for (rncptr=rnclist; rncptr!=NULL; rncptr=rncptr->next) { - if (!strcmp(rncptr->url, feedurl)) { - use_this_rncptr = rncptr; - } - } + if (server_shutting_down) + return; - /* Otherwise create a new client request */ - if (use_this_rncptr == NULL) { - rncptr = (struct rssnetcfg *) malloc(sizeof(struct rssnetcfg)); - if (rncptr != NULL) { - rncptr->next = rnclist; - safestrncpy(rncptr->url, feedurl, sizeof rncptr->url); - rncptr->rooms = NULL; - rnclist = rncptr; - use_this_rncptr = rncptr; - } - } + CfgData = NewStrBufPlain(NULL, statbuf.st_size + 1); - /* Add the room name to the request */ - if (use_this_rncptr != NULL) { - if (use_this_rncptr->rooms == NULL) { - rncptr->rooms = strdup(qrbuf->QRname); - } - else { - len = strlen(use_this_rncptr->rooms) + strlen(qrbuf->QRname) + 5; - ptr = realloc(use_this_rncptr->rooms, len); - if (ptr != NULL) { - strcat(ptr, "|"); - strcat(ptr, qrbuf->QRname); - use_this_rncptr->rooms = ptr; - } - } - } + if (StrBufReadBLOB(CfgData, &fd, 1, statbuf.st_size, &Err) < 0) { + close(fd); + FreeStrBuf(&CfgData); + syslog(LOG_DEBUG, "ERROR: reading config '%s' - %s
\n", + filename, strerror(errno)); + return; + } + close(fd); + if (server_shutting_down) + return; + + CfgPtr = NULL; + CfgType = NewStrBuf(); + Line = NewStrBufPlain(NULL, StrLength(CfgData)); + Done = 0; + while (!Done) + { + Done = StrBufSipLine(Line, CfgData, &CfgPtr) == 0; + if (StrLength(Line) > 0) + { + lPtr = NULL; + StrBufExtract_NextToken(CfgType, Line, &lPtr, '|'); + if (!strcasecmp("rssclient", ChrPtr(CfgType))) + { + if (Count == NULL) + { + Count = malloc(sizeof(rss_room_counter)); + Count->count = 0; + } + Count->count ++; + rncptr = (rss_aggregator *) malloc(sizeof(rss_aggregator)); + memset (rncptr, 0, sizeof(rss_aggregator)); + rncptr->roomlist_parts = 1; + rncptr->Url = NewStrBuf(); + StrBufExtract_NextToken(rncptr->Url, Line, &lPtr, '|'); + + pthread_mutex_lock(&RSSQueueMutex); + GetHash(RSSFetchUrls, SKEY(rncptr->Url), &vptr); + use_this_rncptr = (rss_aggregator *)vptr; + if (use_this_rncptr != NULL) + { + /* mustn't attach to an active session */ + if (use_this_rncptr->RefCount > 0) + { + DeleteRssCfg(rncptr); + Count->count--; + } + else + { + long *QRnumber; + StrBufAppendBufPlain(use_this_rncptr->rooms, + qrbuf->QRname, + -1, 0); + if (use_this_rncptr->roomlist_parts == 1) + { + use_this_rncptr->OtherQRnumbers = NewHash(1, lFlathash); + } + QRnumber = (long*)malloc(sizeof(long)); + *QRnumber = qrbuf->QRnumber; + Put(use_this_rncptr->OtherQRnumbers, LKEY(qrbuf->QRnumber), QRnumber, NULL); + use_this_rncptr->roomlist_parts++; + } + pthread_mutex_unlock(&RSSQueueMutex); + + + FreeStrBuf(&rncptr->Url); + free(rncptr); + rncptr = NULL; + continue; + } + pthread_mutex_unlock(&RSSQueueMutex); + + rncptr->ItemType = RSS_UNSET; + + rncptr->rooms = NewStrBufPlain(qrbuf->QRname, -1); + + pthread_mutex_lock(&RSSQueueMutex); + Put(RSSFetchUrls, SKEY(rncptr->Url), rncptr, DeleteRssCfg); + pthread_mutex_unlock(&RSSQueueMutex); } - + } } - - fclose(fp); - + if (Count != NULL) + { + Count->QRnumber = qrbuf->QRnumber; + pthread_mutex_lock(&RSSQueueMutex); + syslog(LOG_DEBUG, "rssclient: [%ld] %s now starting.\n", + qrbuf->QRnumber, qrbuf->QRname); + Put(RSSQueueRooms, LKEY(qrbuf->QRnumber), Count, NULL); + pthread_mutex_unlock(&RSSQueueMutex); + } + FreeStrBuf(&CfgData); + FreeStrBuf(&CfgType); + FreeStrBuf(&Line); } /* * Scan for rooms that have RSS client requests configured */ -void *rssclient_scan(void *args) { - static time_t last_run = 0L; +void rssclient_scan(void) { static int doing_rssclient = 0; - struct rssnetcfg *rptr = NULL; - struct CitContext rssclientCC; - - /* Give this thread its own private CitContext */ - CtdlFillSystemContext(&rssclientCC, "rssclient"); - citthread_setspecific(MyConKey, (void *)&rssclientCC ); - - CtdlThreadAllocTSD(); + rss_aggregator *rptr = NULL; + void *vrptr = NULL; + HashPos *it; + long len; + const char *Key; + + /* Run no more than once every 15 minutes. */ + if ((time(NULL) - last_run) < 900) { + return; + } /* * This is a simple concurrency check to make sure only one rssclient run @@ -561,28 +699,36 @@ void *rssclient_scan(void *args) { * don't really require extremely fine granularity here, we'll do it * with a static variable instead. */ - if (doing_rssclient) return NULL; + if (doing_rssclient) return; doing_rssclient = 1; - CtdlLogPrintf(CTDL_DEBUG, "rssclient started\n"); - ForEachRoom(rssclient_scan_room, NULL); + syslog(LOG_DEBUG, "rssclient started\n"); + CtdlForEachRoom(rssclient_scan_room, NULL); - while (rnclist != NULL && !CtdlThreadCheckStop()) { - rss_do_fetching(rnclist->url, rnclist->rooms); - rptr = rnclist; - rnclist = rnclist->next; - if (rptr->rooms != NULL) free(rptr->rooms); - free(rptr); + pthread_mutex_lock(&RSSQueueMutex); + + it = GetNewHashPos(RSSFetchUrls, 0); + while (!server_shutting_down && + GetNextHashPos(RSSFetchUrls, it, &len, &Key, &vrptr) && + (vrptr != NULL)) { + rptr = (rss_aggregator *)vrptr; + if (rptr->RefCount == 0) + if (!rss_do_fetching(rptr)) + UnlinkRSSAggregator(rptr); } + DeleteHashPos(&it); + pthread_mutex_unlock(&RSSQueueMutex); - CtdlLogPrintf(CTDL_DEBUG, "rssclient ended\n"); - last_run = time(NULL); + syslog(LOG_DEBUG, "rssclient ended\n"); doing_rssclient = 0; - if (!CtdlThreadCheckStop()) - CtdlThreadSchedule ("RSS Client", CTDLTHREAD_BIGSTACK, rssclient_scan, NULL, last_run + config.c_net_freq); - else - CtdlLogPrintf(CTDL_DEBUG, "rssclient: Task STOPPED.\n"); - return NULL; + return; +} + +void rss_cleanup(void) +{ + /* citthread_mutex_destroy(&RSSQueueMutex); TODO */ + DeleteHash(&RSSFetchUrls); + DeleteHash(&RSSQueueRooms); } @@ -590,9 +736,13 @@ CTDL_MODULE_INIT(rssclient) { if (threading) { - CtdlLogPrintf(CTDL_INFO, "%s\n", curl_version()); - CtdlThreadSchedule ("RSS Client", CTDLTHREAD_BIGSTACK, rssclient_scan, NULL, 0); + CtdlFillSystemContext(&rss_CC, "rssclient"); + pthread_mutex_init(&RSSQueueMutex, NULL); + RSSQueueRooms = NewHash(1, lFlathash); + RSSFetchUrls = NewHash(1, NULL); + syslog(LOG_INFO, "%s\n", curl_version()); + CtdlRegisterSessionHook(rssclient_scan, EVT_TIMER); + CtdlRegisterCleanupHook(rss_cleanup); } - /* return our Subversion id for the Log */ - return "$Id$"; + return "rssclient"; }