From a1166fed0ac7e484831ff5cd04584e1fef99ed2e Mon Sep 17 00:00:00 2001 From: Wilfried Goesgens Date: Sun, 12 Sep 2010 23:25:32 +0200 Subject: [PATCH] * rewrite rss parser: * handle links more properly * properly catch CDATA stuff * have hash of namespaces which we "understand" --- citadel/modules/rssclient/serv_rssclient.c | 789 +++++++++++++++------ 1 file changed, 554 insertions(+), 235 deletions(-) diff --git a/citadel/modules/rssclient/serv_rssclient.c b/citadel/modules/rssclient/serv_rssclient.c index 50ab007f2..de3a6c2db 100644 --- a/citadel/modules/rssclient/serv_rssclient.c +++ b/citadel/modules/rssclient/serv_rssclient.c @@ -57,7 +57,6 @@ #include "context.h" typedef struct rssnetcfg rssnetcfg; - struct rssnetcfg { rssnetcfg *next; char url[256]; @@ -66,37 +65,70 @@ struct rssnetcfg { int ItemType; }; -#define RSS_UNSET 0 -#define RSS_RSS 1 -#define RSS_ATOM 2 +#define RSS_UNSET (1<<0) +#define RSS_RSS (1<<1) +#define RSS_ATOM (1<<2) +#define RSS_REQUIRE_BUF (1<<3) typedef struct _rss_item { - char *chardata; - int chardata_len; char *roomlist; int done_parsing; - char *guid; - char *title; - char *link; - char *description; + StrBuf *guid; + StrBuf *title; + StrBuf *link; + StrBuf *linkTitle; + StrBuf *reLink; + StrBuf *reLinkTitle; + StrBuf *description; time_t pubdate; - char channel_title[256]; + StrBuf *channel_title; int item_tag_nesting; - char *author_or_creator; - char *author_url; - StrBuf *CData; + StrBuf *author_or_creator; + StrBuf *author_url; }rss_item; +typedef void (*rss_handler_func)(StrBuf *CData, + rss_item *ri, + rssnetcfg *Cfg, + const char** Attr); + +typedef struct __rss_xml_handler { + int Flags; + rss_handler_func Handler; +}rss_xml_handler; + + typedef struct _rsscollection { + StrBuf *CData; + StrBuf *Key; + rss_item *Item; rssnetcfg *Cfg; - + rss_xml_handler *Current; } rsscollection; struct rssnetcfg *rnclist = NULL; - +HashList *StartHandlers; +HashList *EndHandlers; +HashList *KnownNameSpaces; +void AddRSSStartHandler(rss_handler_func Handler, int Flags, const char *key, long len) +{ + rss_xml_handler *h; + h = (rss_xml_handler*) malloc(sizeof (rss_xml_handler)); + h->Flags = Flags; + h->Handler = Handler; + Put(StartHandlers, key, len, h, NULL); +} +void AddRSSEndHandler(rss_handler_func Handler, int Flags, const char *key, long len) +{ + rss_xml_handler *h; + h = (rss_xml_handler*) malloc(sizeof (rss_xml_handler)); + h->Flags = Flags; + h->Handler = Handler; + Put(EndHandlers, key, len, h, NULL); +} #if 0 #ifdef HAVE_ICONV @@ -275,7 +307,8 @@ handle_unknown_xml_encoding (void *encodingHandleData, /* * Commit a fetched and parsed RSS item to disk */ -void rss_save_item(rsscollection *rssc) { +void rss_save_item(rss_item *ri) +{ struct MD5Context md5context; u_char rawdigest[MD5_DIGEST_LEN]; @@ -286,7 +319,6 @@ void rss_save_item(rsscollection *rssc) { struct CtdlMessage *msg; struct recptypes *recp = NULL; int msglen = 0; - rss_item *ri = rssc->Item; recp = (struct recptypes *) malloc(sizeof(struct recptypes)); if (recp == NULL) return; @@ -300,15 +332,17 @@ void rss_save_item(rsscollection *rssc) { * If one is not present in the item itself, make one up. */ if (ri->guid != NULL) { - snprintf(utmsgid, sizeof utmsgid, "rss/%s", ri->guid); + StrBufSpaceToBlank(ri->guid); + StrBufTrim(ri->guid); + snprintf(utmsgid, sizeof utmsgid, "rss/%s", ChrPtr(ri->guid)); } else { MD5Init(&md5context); if (ri->title != NULL) { - MD5Update(&md5context, (unsigned char*)ri->title, strlen(ri->title)); + MD5Update(&md5context, (const unsigned char*)ChrPtr(ri->title), StrLength(ri->title)); } if (ri->link != NULL) { - MD5Update(&md5context, (unsigned char*)ri->link, strlen(ri->link)); + MD5Update(&md5context, (const unsigned char*)ChrPtr(ri->link), StrLength(ri->link)); } MD5Final(rawdigest, &md5context); for (i=0; idescription == NULL) ri->description = strdup(""); - for (i=strlen(ri->description); i>=0; --i) { - if (isspace(ri->description[i])) { - ri->description[i] = ' '; - } - } - + if (ri->description == NULL) ri->description = NewStrBufPlain(HKEY("")); + StrBufSpaceToBlank(ri->description); msg = malloc(sizeof(struct CtdlMessage)); memset(msg, 0, sizeof(struct CtdlMessage)); msg->cm_magic = CTDLMESSAGE_MAGIC; @@ -349,7 +381,7 @@ void rss_save_item(rsscollection *rssc) { msg->cm_format_type = FMT_RFC822; if (ri->guid != NULL) { - msg->cm_fields['E'] = strdup(ri->guid); + msg->cm_fields['E'] = strdup(ChrPtr(ri->guid)); } if (ri->author_or_creator != NULL) { @@ -362,9 +394,9 @@ void rss_save_item(rsscollection *rssc) { UserName = NewStrBuf(); EmailAddress = NewStrBuf(); EncBuf = NewStrBuf(); - - From = html_to_ascii(ri->author_or_creator, - strlen(ri->author_or_creator), +////TODO! + From = html_to_ascii(ChrPtr(ri->author_or_creator), + StrLength(ri->author_or_creator), 512, 0); Encoded = NewStrBufPlain(From, -1); @@ -390,8 +422,9 @@ void rss_save_item(rsscollection *rssc) { StrBuf *Encoded, *QPEncoded; QPEncoded = NULL; - len = strlen(ri->title); - Sbj = html_to_ascii(ri->title, len, 512, 0); + StrBufSpaceToBlank(ri->title); + len = StrLength(ri->title); + Sbj = html_to_ascii(ChrPtr(ri->title), len, 512, 0); Encoded = NewStrBufPlain(Sbj, -1); free(Sbj); @@ -404,23 +437,27 @@ void rss_save_item(rsscollection *rssc) { msg->cm_fields['T'] = malloc(64); snprintf(msg->cm_fields['T'], 64, "%ld", ri->pubdate); if (ri->channel_title != NULL) { - if (!IsEmptyStr(ri->channel_title)) { - msg->cm_fields['O'] = strdup(ri->channel_title); + if (StrLength(ri->channel_title) > 0) { + msg->cm_fields['O'] = strdup(ChrPtr(ri->channel_title)); } } if (ri->link == NULL) - ri->link = strdup(""); - msglen += 1024 + strlen(ri->link) + strlen(ri->description) ; + ri->link = NewStrBufPlain(HKEY("")); + msglen += 1024 + StrLength(ri->link) + StrLength(ri->description) ; msg->cm_fields['M'] = malloc(msglen); snprintf(msg->cm_fields['M'], msglen, - "Content-type: text/html; charset=\"UTF-8\"\r\n\r\n" - "\n" - "%s

\n" - "%s\n" - "\n" - , - ri->description, - ri->link, ri->link + "Content-type: text/html; charset=\"UTF-8\"\r\n\r\n" + "\n" + "%s

\n" + "%s
\n" + "%s\n" + "\n" + , + ChrPtr(ri->description), + ChrPtr(ri->link), + (StrLength(ri->linkTitle)>0)?ChrPtr(ri->linkTitle):ChrPtr(ri->link), + ChrPtr(ri->reLink), + (StrLength(ri->reLinkTitle)>0)?ChrPtr(ri->reLinkTitle):"Reply to this" ); CtdlSubmitMsg(msg, recp, NULL, 0); @@ -439,7 +476,7 @@ void rss_save_item(rsscollection *rssc) { /* * Convert an RDF/RSS datestamp into a time_t */ -time_t rdf_parsedate(char *p) +time_t rdf_parsedate(const char *p) { struct tm tm; time_t t = 0; @@ -476,255 +513,459 @@ time_t rdf_parsedate(char *p) return(time(NULL)); } -#define RSS_UNSET 0 -#define RSS_RSS 1 -#define RSS_ATOM 2 - -void flush_rss_ite(rss_item *ri) +void flush_rss_item(rss_item *ri) { /* Initialize the feed item data structure */ - if (ri->guid != NULL) free(ri->guid); - ri->guid = NULL; - if (ri->title != NULL) free(ri->title); - ri->title = NULL; - if (ri->link != NULL) free(ri->link); - ri->link = NULL; - if (ri->author_or_creator != NULL) free(ri->author_or_creator); - ri->author_or_creator = NULL; - if (ri->author_url != NULL) free(ri->author_url); - ri->author_url = NULL; - if (ri->description != NULL) free(ri->description); - ri->description = NULL; - /* Throw away any existing character data */ - if (ri->chardata_len > 0) { - free(ri->chardata); - ri->chardata = 0; - ri->chardata_len = 0; - } - FreeStrBuf(&ri->CData); -} - -void rss_xml_start(void *data, const char *supplied_el, const char **attr) { - rsscollection *rssc = (rsscollection*) data; - rss_item *ri = rssc->Item; - char el[256]; - char *sep = NULL; + FreeStrBuf(&ri->guid); + FreeStrBuf(&ri->title); + FreeStrBuf(&ri->link); + FreeStrBuf(&ri->author_or_creator); + FreeStrBuf(&ri->author_url); + FreeStrBuf(&ri->description); +} + +void rss_xml_start(void *data, const char *supplied_el, const char **attr) +{ + rss_xml_handler *h; + rsscollection *rssc = (rsscollection*) data; + rssnetcfg *Cfg = rssc->Cfg; + rss_item *ri = rssc->Item; + void *pv; + const char *pel; + char *sep = NULL; /* Axe the namespace, we don't care about it */ /// CtdlLogPrintf(0, "RSS: supplied el %d: %s...\n", rssc->Cfg->ItemType, supplied_el); - safestrncpy(el, supplied_el, sizeof el); - while (sep = strchr(el, ':'), sep) { - strcpy(el, ++sep); + pel = supplied_el; + while (sep = strchr(pel, ':'), sep) { + pel = sep + 1; } - if ((rssc->Cfg->ItemType == RSS_UNSET) && !strcasecmp(el, "rss")) + if (pel != supplied_el) { - CtdlLogPrintf(9, "RSS: This is an RSS feed.\n"); - rssc->Cfg->ItemType = RSS_RSS; + void *v; + + if (!GetHash(KnownNameSpaces, + supplied_el, + pel - supplied_el - 1, + &v)) + { + CtdlLogPrintf(0, "RSS: START ignoring because of wrong namespace [%s] = [%s]\n", + supplied_el); + return; + } } - if ((rssc->Cfg->ItemType == RSS_UNSET) && !strcasecmp(el, "rdf")) + + StrBufPlain(rssc->Key, pel, -1); + StrBufLowerCase(rssc->Key); + if (GetHash(StartHandlers, SKEY(rssc->Key), &pv)) { - CtdlLogPrintf(9, "RSS: This is an RDF feed.\n"); - rssc->Cfg->ItemType = RSS_RSS; + rssc->Current = h = (rss_xml_handler*) pv; + + if (((h->Flags & RSS_UNSET) != 0) && + (Cfg->ItemType == RSS_UNSET)) + { + h->Handler(rssc->CData, ri, Cfg, attr); + } + else if (((h->Flags & RSS_RSS) != 0) && + (Cfg->ItemType == RSS_RSS)) + { + h->Handler(rssc->CData, ri, Cfg, attr); + } + else if (((h->Flags & RSS_ATOM) != 0) && + (Cfg->ItemType == RSS_ATOM)) + { + h->Handler(rssc->CData, ri, Cfg, attr); + } + else + CtdlLogPrintf(0, "RSS: START unhandled: [%s] [%s]...\n", pel, supplied_el); } - else if ((rssc->Cfg->ItemType == RSS_UNSET) && !strcasecmp(el, "feed")) - { - CtdlLogPrintf(9, "RSS: This is an ATOM feed.\n"); - rssc->Cfg->ItemType = RSS_ATOM; + else + CtdlLogPrintf(0, "RSS: START unhandled: [%s] [%s]...\n", pel, supplied_el); + +} + +void rss_xml_end(void *data, const char *supplied_el) +{ + rss_xml_handler *h; + rsscollection *rssc = (rsscollection*) data; + rssnetcfg *Cfg = rssc->Cfg; + rss_item *ri = rssc->Item; + const char *pel; + char *sep = NULL; + void *pv; + + /* Axe the namespace, we don't care about it */ + pel = supplied_el; + while (sep = strchr(pel, ':'), sep) { + pel = sep + 1; } - else if ((rssc->Cfg->ItemType == RSS_RSS) && - !strcasecmp(el, "item")) +// CtdlLogPrintf(0, "RSS: END %s...\n", el); + if (pel != supplied_el) { - ri->item_tag_nesting ++ ; - flush_rss_ite(ri); - } - else if ( (rssc->Cfg->ItemType == RSS_ATOM) && - !strcasecmp(el, "entry")) - { /* Atom feed... */ - ++ri->item_tag_nesting; - flush_rss_ite(ri); + void *v; + + if (!GetHash(KnownNameSpaces, + supplied_el, + pel - supplied_el - 1, + &v)) + { + CtdlLogPrintf(0, "RSS: END ignoring because of wrong namespace [%s] = [%s]\n", + supplied_el, ChrPtr(rssc->CData)); + FlushStrBuf(rssc->CData); + return; + } } - else if ((rssc->Cfg->ItemType == RSS_ATOM) && - !strcasecmp(el, "link")) + + StrBufPlain(rssc->Key, pel, -1); + StrBufLowerCase(rssc->Key); + if (GetHash(EndHandlers, SKEY(rssc->Key), &pv)) { - int found ; - int i; + h = (rss_xml_handler*) pv; - for (found = 0, i = 0;!found && attr[i] != NULL; i+=2) + if (((h->Flags & RSS_UNSET) != 0) && + (Cfg->ItemType == RSS_UNSET)) { - if (!strcmp(attr[i], "href")) - { - found = 1; - if (ri->link != NULL) - free(ri->link); - ri->link = strdup(attr[i+1]); - striplt(ri->link); - } + h->Handler(rssc->CData, ri, Cfg, NULL); } - + else if (((h->Flags & RSS_RSS) != 0) && + (Cfg->ItemType == RSS_RSS)) + { + h->Handler(rssc->CData, ri, Cfg, NULL); + } + else if (((h->Flags & RSS_ATOM) != 0) && + (Cfg->ItemType == RSS_ATOM)) + { + h->Handler(rssc->CData, ri, Cfg, NULL); + } + else + CtdlLogPrintf(0, "RSS: END unhandled: [%s] [%s] = [%s]...\n", pel, supplied_el, ChrPtr(rssc->CData)); + } + else + CtdlLogPrintf(0, "RSS: END unhandled: [%s] [%s] = [%s]...\n", pel, supplied_el, ChrPtr(rssc->CData)); + FlushStrBuf(rssc->CData); + rssc->Current = NULL; +} + + + + + + + + + + + + + + + +void RSS_item_rss_start (StrBuf *CData, rss_item *ri, rssnetcfg *Cfg, const char** Attr) +{ + CtdlLogPrintf(CTDL_DEBUG, "RSS: This is an RSS feed.\n"); + Cfg->ItemType = RSS_RSS; } -void rss_xml_end(void *data, const char *supplied_el) { - rsscollection *rssc = (rsscollection*) data; - rss_item *ri = rssc->Item; - char el[256]; - char *sep = NULL; +void RSS_item_rdf_start(StrBuf *CData, rss_item *ri, rssnetcfg *Cfg, const char** Attr) +{ + CtdlLogPrintf(CTDL_DEBUG, "RSS: This is an RDF feed.\n"); + Cfg->ItemType = RSS_RSS; +} - /* Axe the namespace, we don't care about it */ - safestrncpy(el, supplied_el, sizeof el); - while (sep = strchr(el, ':'), sep) { - strcpy(el, ++sep); +void ATOM_item_feed_start(StrBuf *CData, rss_item *ri, rssnetcfg *Cfg, const char** Attr) +{ + CtdlLogPrintf(CTDL_DEBUG, "RSS: This is an ATOM feed.\n"); + Cfg->ItemType = RSS_ATOM; +} + + +void RSS_item_item_start(StrBuf *CData, rss_item *ri, rssnetcfg *Cfg, const char** Attr) +{ + ri->item_tag_nesting ++; + flush_rss_item(ri); +} + +void ATOM_item_entry_start(StrBuf *CData, rss_item *ri, rssnetcfg *Cfg, const char** Attr) +{ +/* Atom feed... */ + ri->item_tag_nesting ++; + flush_rss_item(ri); +} + +void ATOM_item_link_start (StrBuf *CData, rss_item *ri, rssnetcfg *Cfg, const char** Attr) +{ + int i; + const char *pHref = NULL; + const char *pType = NULL; + const char *pRel = NULL; + const char *pTitle = NULL; + + for (i = 0; Attr[i] != NULL; i+=2) + { + if (!strcmp(Attr[i], "href")) + { + pHref = Attr[i+1]; + } + else if (!strcmp(Attr[i], "rel")) + { + pRel = Attr[i+1]; + } + else if (!strcmp(Attr[i], "type")) + { + pType = Attr[i+1]; + } + else if (!strcmp(Attr[i], "title")) + { + pTitle = Attr[i+1]; + } } -// CtdlLogPrintf(0, "RSS: END %s...\n", el); + if (pHref == NULL) + return; /* WHUT? Pointing... where? */ + if ((pType != NULL) && !strcasecmp(pType, "application/atom+xml")) + return; /* these just point to other rss resources, we're not interested in them. */ + if (pRel != NULL) + { + if (!strcasecmp (pRel, "replies")) + { + NewStrBufDupAppendFlush(&ri->reLink, NULL, pHref, -1); + StrBufTrim(ri->link); + NewStrBufDupAppendFlush(&ri->reLinkTitle, NULL, pTitle, -1); + } + else if (!strcasecmp(pRel, "alternate")) /* Alternative representation of this Item... */ + { + NewStrBufDupAppendFlush(&ri->link, NULL, pHref, -1); + StrBufTrim(ri->link); + NewStrBufDupAppendFlush(&ri->linkTitle, NULL, pTitle, -1); - if ( (!strcasecmp(el, "title")) && (ri->item_tag_nesting == 0) && (ri->chardata != NULL) ) { - safestrncpy(ri->channel_title, ri->chardata, sizeof ri->channel_title); - striplt(ri->channel_title); + } +#if 0 /* these are also defined, but dunno what to do with them.. */ + else if (!strcasecmp(pRel, "related")) + { + } + else if (!strcasecmp(pRel, "self")) + { + } + else if (!strcasecmp(pRel, "enclosure")) + {/* this reference can get big, and is probably the full article... */ + } + else if (!strcasecmp(pRel, "via")) + {/* this article was provided via... */ + } +#endif + } + else if (StrLength(ri->link) == 0) + { + NewStrBufDupAppendFlush(&ri->link, NULL, pHref, -1); + StrBufTrim(ri->link); + NewStrBufDupAppendFlush(&ri->linkTitle, NULL, pTitle, -1); } +} + + - if ( (rssc->Cfg->ItemType == RSS_RSS) && - (!strcasecmp(el, "guid")) && (ri->chardata != NULL) ) { - if (ri->guid != NULL) free(ri->guid); - striplt(ri->chardata); - ri->guid = strdup(ri->chardata); + +void ATOMRSS_item_title_end(StrBuf *CData, rss_item *ri, rssnetcfg *Cfg, const char** Attr) +{ + if ((ri->item_tag_nesting == 0) && (StrLength(CData) > 0)) { + NewStrBufDupAppendFlush(&ri->channel_title, CData, NULL, 0); + StrBufTrim(ri->channel_title); } - else if ( (rssc->Cfg->ItemType == RSS_ATOM) && - (!strcasecmp(el, "id")) && (ri->chardata != NULL) ) { - if (ri->guid != NULL) free(ri->guid); - striplt(ri->chardata); - ri->guid = strdup(ri->chardata); +} + +void RSS_item_guid_end(StrBuf *CData, rss_item *ri, rssnetcfg *Cfg, const char** Attr) +{ + if (StrLength(CData) > 0) { + NewStrBufDupAppendFlush(&ri->guid, CData, NULL, 0); } +} - else if ( (rssc->Cfg->ItemType == RSS_RSS) && (!strcasecmp(el, "link")) && (ri->chardata != NULL) ) { - if (ri->link != NULL) free(ri->link); - striplt(ri->chardata); - ri->link = strdup(ri->chardata); +void ATOM_item_id_end(StrBuf *CData, rss_item *ri, rssnetcfg *Cfg, const char** Attr) +{ + if (StrLength(CData) > 0) { + NewStrBufDupAppendFlush(&ri->guid, CData, NULL, 0); } +} + - else if ( (!strcasecmp(el, "title")) && (ri->chardata != NULL) ) { - if (ri->title != NULL) free(ri->title); - striplt(ri->chardata); - ri->title = strdup(ri->chardata); +void RSS_item_link_end (StrBuf *CData, rss_item *ri, rssnetcfg *Cfg, const char** Attr) +{ + if (StrLength(CData) > 0) { + NewStrBufDupAppendFlush(&ri->link, CData, NULL, 0); + StrBufTrim(ri->link); } +} - else if ((rssc->Cfg->ItemType == RSS_ATOM) && - (!strcasecmp(el, "content")) && - (ri->chardata != NULL) ) { - if (ri->description != NULL) free(ri->description); - ri->description = strdup(ri->chardata); +void RSSATOM_item_title_end (StrBuf *CData, rss_item *ri, rssnetcfg *Cfg, const char** Attr) +{ + if (StrLength(CData) > 0) { + NewStrBufDupAppendFlush(&ri->title, CData, NULL, 0); + StrBufTrim(ri->title); } - else if ( (rssc->Cfg->ItemType == RSS_RSS) && - (!strcasecmp(el, "description")) && - (ri->chardata != NULL) ) { - if (ri->description != NULL) free(ri->description); - ri->description = strdup(ri->chardata); +} + +void ATOM_item_content_end (StrBuf *CData, rss_item *ri, rssnetcfg *Cfg, const char** Attr) +{ + if (StrLength(CData) > 0) { + NewStrBufDupAppendFlush(&ri->description, CData, NULL, 0); + StrBufTrim(ri->description); } - - else if ((rssc->Cfg->ItemType == RSS_ATOM) && - ((!strcasecmp(el, "published")) || - (!strcasecmp(el, "updated"))) && - (ri->chardata != NULL) ) { - striplt(ri->chardata); - ri->pubdate = rdf_parsedate(ri->chardata); +} +void ATOM_item_summary_end (StrBuf *CData, rss_item *ri, rssnetcfg *Cfg, const char** Attr) +{ + /* this can contain an abstract of the article. but we don't want to verwrite a full document if we already have it. */ + if ((StrLength(CData) > 0) && (StrLength(ri->description) == 0)) + { + NewStrBufDupAppendFlush(&ri->description, CData, NULL, 0); + StrBufTrim(ri->description); } +} +void RSS_item_description_end (StrBuf *CData, rss_item *ri, rssnetcfg *Cfg, const char** Attr) +{ + if (StrLength(CData) > 0) { + NewStrBufDupAppendFlush(&ri->description, CData, NULL, 0); + } +} - else if ((rssc->Cfg->ItemType == RSS_RSS) && - ((!strcasecmp(el, "pubdate")) || - (!strcasecmp(el, "date"))) && - (ri->chardata != NULL) ) { - striplt(ri->chardata); - ri->pubdate = rdf_parsedate(ri->chardata); +void ATOM_item_published_end (StrBuf *CData, rss_item *ri, rssnetcfg *Cfg, const char** Attr) +{ + if (StrLength(CData) > 0) { + StrBufTrim(CData); + ri->pubdate = rdf_parsedate(ChrPtr(CData)); } +} - else if ((rssc->Cfg->ItemType == RSS_RSS) && - ((!strcasecmp(el, "author")) || - (!strcasecmp(el, "creator"))) && - (ri->chardata != NULL) ) { - if (ri->author_or_creator != NULL) free(ri->author_or_creator); - striplt(ri->chardata); - ri->author_or_creator = strdup(ri->chardata); +void ATOM_item_updated_end (StrBuf *CData, rss_item *ri, rssnetcfg *Cfg, const char** Attr) +{ + if (StrLength(CData) > 0) { + StrBufTrim(CData); + ri->pubdate = rdf_parsedate(ChrPtr(CData)); } +} - else if ((rssc->Cfg->ItemType == RSS_ATOM) && - (!strcasecmp(el, "name")) && - (ri->chardata != NULL) ) { - if (ri->author_or_creator != NULL) free(ri->author_or_creator); - striplt(ri->chardata); - ri->author_or_creator = strdup(ri->chardata); +void RSS_item_pubdate_end (StrBuf *CData, rss_item *ri, rssnetcfg *Cfg, const char** Attr) +{ + if (StrLength(CData) > 0) { + StrBufTrim(CData); + ri->pubdate = rdf_parsedate(ChrPtr(CData)); } - else if ((rssc->Cfg->ItemType == RSS_ATOM) && - (!strcasecmp(el, "uri")) && - (ri->chardata != NULL) ) { - if (ri->author_url != NULL) free(ri->author_url); - striplt(ri->chardata); - ri->author_url = strdup(ri->chardata); +} + + +void RSS_item_date_end (StrBuf *CData, rss_item *ri, rssnetcfg *Cfg, const char** Attr) +{ + if (StrLength(CData) > 0) { + StrBufTrim(CData); + ri->pubdate = rdf_parsedate(ChrPtr(CData)); } +} - else if ((rssc->Cfg->ItemType == RSS_RSS) && - !strcasecmp(el, "item")) { - --ri->item_tag_nesting; - rss_save_item(rssc); + + +void RSS_item_author_end(StrBuf *CData, rss_item *ri, rssnetcfg *Cfg, const char** Attr) +{ + if (StrLength(CData) > 0) { + NewStrBufDupAppendFlush(&ri->author_or_creator, CData, NULL, 0); + StrBufTrim(ri->author_or_creator); } - else if ((rssc->Cfg->ItemType == RSS_ATOM) && - !strcasecmp(el, "entry")) { - --ri->item_tag_nesting; - rss_save_item(rssc); +} + + +void ATOM_item_name_end(StrBuf *CData, rss_item *ri, rssnetcfg *Cfg, const char** Attr) +{ + if (StrLength(CData) > 0) { + NewStrBufDupAppendFlush(&ri->author_or_creator, CData, NULL, 0); + StrBufTrim(ri->author_or_creator); } +} - else if ( (!strcasecmp(el, "rss")) || - (!strcasecmp(el, "rdf")) ) { -// CtdlLogPrintf(CTDL_DEBUG, "End of feed detected. Closing parser.\n"); - ri->done_parsing = 1; +void RSS_item_creator_end(StrBuf *CData, rss_item *ri, rssnetcfg *Cfg, const char** Attr) +{ + if ((StrLength(CData) > 0) && + (StrLength(ri->author_or_creator) == 0)) + { + NewStrBufDupAppendFlush(&ri->author_or_creator, CData, NULL, 0); + StrBufTrim(ri->author_or_creator); } +} + - if (ri->chardata_len > 0) { - free(ri->chardata); - ri->chardata = 0; - ri->chardata_len = 0; +void ATOM_item_uri_end(StrBuf *CData, rss_item *ri, rssnetcfg *Cfg, const char** Attr) +{ + if (StrLength(CData) > 0) { + NewStrBufDupAppendFlush(&ri->author_url, CData, NULL, 0); + StrBufTrim(ri->author_url); } +} + +void RSS_item_item_end(StrBuf *CData, rss_item *ri, rssnetcfg *Cfg, const char** Attr) +{ + --ri->item_tag_nesting; + rss_save_item(ri); +} + + +void ATOM_item_entry_end(StrBuf *CData, rss_item *ri, rssnetcfg *Cfg, const char** Attr) +{ + --ri->item_tag_nesting; + rss_save_item(ri); +} + +void RSS_item_rss_end(StrBuf *CData, rss_item *ri, rssnetcfg *Cfg, const char** Attr) +{ +// CtdlLogPrintf(CTDL_DEBUG, "End of feed detected. Closing parser.\n"); + ri->done_parsing = 1; + +} +void RSS_item_rdf_end(StrBuf *CData, rss_item *ri, rssnetcfg *Cfg, const char** Attr) +{ +// CtdlLogPrintf(CTDL_DEBUG, "End of feed detected. Closing parser.\n"); + ri->done_parsing = 1; +} + +void RSSATOM_item_ignore(StrBuf *CData, rss_item *ri, rssnetcfg *Cfg, const char** Attr) +{ } + + + + + + + + + /* * This callback stores up the data which appears in between tags. */ -void rss_xml_cdata_start(void *data) { +void rss_xml_cdata_start(void *data) +{ rsscollection *rssc = (rsscollection*) data; - rss_item *ri = rssc->Item; - ri->CData = NewStrBuf(); + FlushStrBuf(rssc->CData); } -void rss_xml_cdata_end(void *data) { - rsscollection *rssc = (rsscollection*) data; - rss_item *ri = rssc->Item; +void rss_xml_cdata_end(void *data) +{ + //rsscollection *rssc = (rsscollection*) data; + //rss_item *ri = rssc->Item; - ri->description = SmashStrBuf(&ri->CData); + /* Hm, it seems as if in some CDATA cases expat doesn't call the handler... * / + if (rssc->Current == NULL) + NewStrBufDupAppendFlush(&ri->description, rssc->CData, NULL, 0); + */ } -void rss_xml_chardata(void *data, const XML_Char *s, int len) { +void rss_xml_chardata(void *data, const XML_Char *s, int len) +{ rsscollection *rssc = (rsscollection*) data; - rss_item *ri = rssc->Item; - int old_len; - int new_len; - char *new_buffer; - - if (ri->CData != NULL) { - StrBufAppendBufPlain (ri->CData, s, len, 0); - } else { - old_len = ri->chardata_len; - new_len = old_len + len; - new_buffer = realloc(ri->chardata, new_len + 1); - if (new_buffer != NULL) { - memcpy(&new_buffer[old_len], s, len); - new_buffer[new_len] = 0; - ri->chardata = new_buffer; - ri->chardata_len = new_len; - } - } + + StrBufAppendBufPlain (rssc->CData, s, len, 0); } /* @@ -750,6 +991,7 @@ void rss_do_fetching(rssnetcfg *Cfg) { CURLcode res; char errmsg[1024] = ""; + memset(&ri, 0, sizeof(rss_item)); rssc.Item = &ri; rssc.Cfg = Cfg; @@ -767,7 +1009,8 @@ void rss_do_fetching(rssnetcfg *Cfg) { curl_easy_cleanup(curl); return; } - + rssc.CData = NewStrBufPlain(NULL, SIZ); + rssc.Key = NewStrBuf(); curl_easy_setopt(curl, CURLOPT_URL, Cfg->url); curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 0); curl_easy_setopt(curl, CURLOPT_SSL_VERIFYHOST, 0); @@ -835,7 +1078,9 @@ shutdown: curl_easy_cleanup(curl); XML_ParserFree(xp); - flush_rss_ite(&ri); + flush_rss_item(&ri); + FreeStrBuf(&rssc.CData); + FreeStrBuf(&rssc.Key); } @@ -972,6 +1217,80 @@ CTDL_MODULE_INIT(rssclient) CtdlLogPrintf(CTDL_INFO, "%s\n", curl_version()); CtdlThreadSchedule ("RSS Client", CTDLTHREAD_BIGSTACK, rssclient_scan, NULL, 0); } - /* return our Subversion id for the Log */ - return "rssclient"; + + StartHandlers = NewHash(1, NULL); + EndHandlers = NewHash(1, NULL); + + AddRSSStartHandler(RSS_item_rss_start, RSS_UNSET, HKEY("rss")); + AddRSSStartHandler(RSS_item_rdf_start, RSS_UNSET, HKEY("rdf")); + AddRSSStartHandler(ATOM_item_feed_start, RSS_UNSET, HKEY("feed")); + AddRSSStartHandler(RSS_item_item_start, RSS_RSS, HKEY("item")); + AddRSSStartHandler(ATOM_item_entry_start, RSS_ATOM, HKEY("entry")); + AddRSSStartHandler(ATOM_item_link_start, RSS_ATOM, HKEY("link")); + + AddRSSEndHandler(ATOMRSS_item_title_end, RSS_ATOM|RSS_RSS|RSS_REQUIRE_BUF, HKEY("title")); + AddRSSEndHandler(RSS_item_guid_end, RSS_RSS|RSS_REQUIRE_BUF, HKEY("guid")); + AddRSSEndHandler(ATOM_item_id_end, RSS_ATOM|RSS_REQUIRE_BUF, HKEY("id")); + AddRSSEndHandler(RSS_item_link_end, RSS_RSS|RSS_REQUIRE_BUF, HKEY("link")); + AddRSSEndHandler(RSSATOM_item_title_end, RSS_ATOM|RSS_RSS|RSS_REQUIRE_BUF, HKEY("title")); + AddRSSEndHandler(ATOM_item_content_end, RSS_ATOM|RSS_REQUIRE_BUF, HKEY("content")); + AddRSSEndHandler(RSS_item_description_end, RSS_RSS|RSS_REQUIRE_BUF, HKEY("encoded")); + AddRSSEndHandler(ATOM_item_summary_end, RSS_ATOM|RSS_REQUIRE_BUF, HKEY("summary")); + AddRSSEndHandler(RSS_item_description_end, RSS_RSS|RSS_REQUIRE_BUF, HKEY("description")); + AddRSSEndHandler(ATOM_item_published_end, RSS_ATOM|RSS_REQUIRE_BUF, HKEY("published")); + AddRSSEndHandler(ATOM_item_updated_end, RSS_ATOM|RSS_REQUIRE_BUF, HKEY("updated")); + AddRSSEndHandler(RSS_item_pubdate_end, RSS_RSS|RSS_REQUIRE_BUF, HKEY("pubdate")); + AddRSSEndHandler(RSS_item_date_end, RSS_RSS|RSS_REQUIRE_BUF, HKEY("date")); + AddRSSEndHandler(RSS_item_author_end, RSS_RSS|RSS_REQUIRE_BUF, HKEY("author")); + AddRSSEndHandler(RSS_item_creator_end, RSS_RSS|RSS_REQUIRE_BUF, HKEY("creator")); + AddRSSEndHandler(ATOM_item_name_end, RSS_ATOM|RSS_REQUIRE_BUF, HKEY("name")); + AddRSSEndHandler(ATOM_item_uri_end, RSS_ATOM|RSS_REQUIRE_BUF, HKEY("uri")); + + AddRSSEndHandler(RSS_item_item_end, RSS_RSS, HKEY("item")); + AddRSSEndHandler(RSS_item_rss_end, RSS_RSS, HKEY("rss")); + AddRSSEndHandler(RSS_item_rdf_end, RSS_RSS, HKEY("rdf")); + AddRSSEndHandler(ATOM_item_entry_end, RSS_ATOM, HKEY("entry")); + + +/* at the start of atoms:
  • link to resource
    •  ignore them. */ + AddRSSStartHandler(RSSATOM_item_ignore, RSS_RSS|RSS_ATOM, HKEY("seq")); + AddRSSEndHandler (RSSATOM_item_ignore, RSS_RSS|RSS_ATOM, HKEY("seq")); + AddRSSStartHandler(RSSATOM_item_ignore, RSS_RSS|RSS_ATOM, HKEY("li")); + AddRSSEndHandler (RSSATOM_item_ignore, RSS_RSS|RSS_ATOM, HKEY("li")); + +/* links to other feed generators... */ + AddRSSStartHandler(RSSATOM_item_ignore, RSS_RSS|RSS_ATOM, HKEY("feedflare")); + AddRSSEndHandler (RSSATOM_item_ignore, RSS_RSS|RSS_ATOM, HKEY("feedflare")); + AddRSSStartHandler(RSSATOM_item_ignore, RSS_RSS|RSS_ATOM, HKEY("browserfriendly")); + AddRSSEndHandler (RSSATOM_item_ignore, RSS_RSS|RSS_ATOM, HKEY("browserfriendly")); + + KnownNameSpaces = NewHash(1, NULL); + Put(KnownNameSpaces, HKEY("http://a9.com/-/spec/opensearch/1.1/"), NULL, reference_free_handler); + Put(KnownNameSpaces, HKEY("http://a9.com/-/spec/opensearchrss/1.0/"), NULL, reference_free_handler); + Put(KnownNameSpaces, HKEY("http://backend.userland.com/creativeCommonsRssModule"), NULL, reference_free_handler); + Put(KnownNameSpaces, HKEY("http://purl.org/atom/ns#"), NULL, reference_free_handler); + Put(KnownNameSpaces, HKEY("http://purl.org/dc/elements/1.1/"), NULL, reference_free_handler); + Put(KnownNameSpaces, HKEY("http://purl.org/rss/1.0/"), NULL, reference_free_handler); + Put(KnownNameSpaces, HKEY("http://purl.org/rss/1.0/modules/content/"), NULL, reference_free_handler); + Put(KnownNameSpaces, HKEY("http://purl.org/rss/1.0/modules/slash/"), NULL, reference_free_handler); + Put(KnownNameSpaces, HKEY("http://purl.org/rss/1.0/modules/syndication/"), NULL, reference_free_handler); + Put(KnownNameSpaces, HKEY("http://purl.org/rss/1.0/"), NULL, reference_free_handler); + Put(KnownNameSpaces, HKEY("http://purl.org/syndication/thread/1.0"), NULL, reference_free_handler); + Put(KnownNameSpaces, HKEY("http://rssnamespace.org/feedburner/ext/1.0"), NULL, reference_free_handler); + Put(KnownNameSpaces, HKEY("http://schemas.google.com/g/2005"), NULL, reference_free_handler); + Put(KnownNameSpaces, HKEY("http://webns.net/mvcb/"), NULL, reference_free_handler); + Put(KnownNameSpaces, HKEY("http://web.resource.org/cc/"), NULL, reference_free_handler); + Put(KnownNameSpaces, HKEY("http://wellformedweb.org/CommentAPI/"), NULL, reference_free_handler); + Put(KnownNameSpaces, HKEY("http://www.georss.org/georss"), NULL, reference_free_handler); + Put(KnownNameSpaces, HKEY("http://www.w3.org/1999/xhtml"), NULL, reference_free_handler); + Put(KnownNameSpaces, HKEY("http://www.w3.org/1999/02/22-rdf-syntax-ns#"), NULL, reference_free_handler); + Put(KnownNameSpaces, HKEY("http://www.w3.org/1999/02/22-rdf-syntax-ns#"), NULL, reference_free_handler); + Put(KnownNameSpaces, HKEY("http://www.w3.org/2003/01/geo/wgs84_pos#"), NULL, reference_free_handler); + Put(KnownNameSpaces, HKEY("http://www.w3.org/2005/Atom"), NULL, reference_free_handler); + Put(KnownNameSpaces, HKEY("urn:flickr:"), NULL, reference_free_handler); +#if 0 + /* we don't like these namespaces because of they shadow our usefull parameters. */ + Put(KnownNameSpaces, HKEY("http://search.yahoo.com/mrss/"), NULL, reference_free_handler); +#endif + return "rssclient"; } -- 2.39.2