+void rss_remember_item(rss_item *ri, rss_aggregator *RSSAggr)
+{
+ networker_save_message *SaveMsg;
+ struct MD5Context md5context;
+ u_char rawdigest[MD5_DIGEST_LEN];
+ StrBuf *guid;
+ AsyncIO *IO = &RSSAggr->IO;
+ int n;
+
+
+ SaveMsg = (networker_save_message *) malloc(
+ sizeof(networker_save_message));
+ memset(SaveMsg, 0, sizeof(networker_save_message));
+
+ /* Construct a GUID to use in the S_USETABLE table.
+ * If one is not present in the item itself, make one up.
+ */
+ if (ri->guid != NULL) {
+ StrBufSpaceToBlank(ri->guid);
+ StrBufTrim(ri->guid);
+ guid = NewStrBufPlain(HKEY("rss/"));
+ StrBufAppendBuf(guid, ri->guid, 0);
+ }
+ else {
+ MD5Init(&md5context);
+ if (ri->title != NULL) {
+ MD5Update(&md5context,
+ (const unsigned char*)SKEY(ri->title));
+ }
+ if (ri->link != NULL) {
+ MD5Update(&md5context,
+ (const unsigned char*)SKEY(ri->link));
+ }
+ MD5Final(rawdigest, &md5context);
+ guid = NewStrBufPlain(NULL,
+ MD5_DIGEST_LEN * 2 + 12 /* _rss2ctdl*/);
+ StrBufHexEscAppend(guid, NULL, rawdigest, MD5_DIGEST_LEN);
+ StrBufAppendBufPlain(guid, HKEY("_rss2ctdl"), 0);
+ }
+
+ /* translate Item into message. */
+ EVRSSATOMM_syslog(LOG_DEBUG, "RSS: translating item...\n");
+ if (ri->description == NULL) ri->description = NewStrBufPlain(HKEY(""));
+ StrBufSpaceToBlank(ri->description);
+ SaveMsg->Msg.cm_magic = CTDLMESSAGE_MAGIC;
+ SaveMsg->Msg.cm_anon_type = MES_NORMAL;
+ SaveMsg->Msg.cm_format_type = FMT_RFC822;
+
+ /* gather the cheaply computed information now... */
+
+ if (ri->guid != NULL) {
+ CM_SetField(&SaveMsg->Msg, eExclusiveID, SKEY(ri->guid));
+ }
+
+ SaveMsg->MsgGUID = guid;
+
+ if (ri->pubdate <= 0) {
+ ri->pubdate = time(NULL); /// TODO: use event time!
+ }
+ CM_SetFieldLONG(&SaveMsg->Msg, eTimestamp, ri->pubdate);
+ if (ri->channel_title != NULL) {
+ if (StrLength(ri->channel_title) > 0) {
+ CM_SetField(&SaveMsg->Msg, eOriginalRoom, SKEY(ri->channel_title));
+ }
+ }
+
+ /* remember the ones for defferred processing to save computing power after we know if we realy need it. */
+
+ SaveMsg->author_or_creator = ri->author_or_creator;
+ ri->author_or_creator = NULL;
+
+ SaveMsg->author_email = ri->author_email;
+ ri->author_email = NULL;
+
+ SaveMsg->title = ri->title;
+ ri->title = NULL;
+
+ SaveMsg->link = ri->link;
+ ri->link = NULL;
+
+ SaveMsg->description = ri->description;
+ ri->description = NULL;
+
+ SaveMsg->linkTitle = ri->linkTitle;
+ ri->linkTitle = NULL;
+
+ SaveMsg->reLink = ri->reLink;
+ ri->reLink = NULL;
+
+ SaveMsg->reLinkTitle = ri->reLinkTitle;
+ ri->reLinkTitle = NULL;
+
+ n = GetCount(RSSAggr->Messages) + 1;
+ Put(RSSAggr->Messages, IKEY(n), SaveMsg, FreeNetworkSaveMessage);
+}
+
+
+
+void rss_xml_start(void *data, const char *supplied_el, const char **attr)
+{
+ rss_xml_handler *h;
+ rss_aggregator *RSSAggr = (rss_aggregator*) data;
+ AsyncIO *IO = &RSSAggr->IO;
+ rss_item *ri = RSSAggr->Item;
+ void *pv;
+ const char *pel;
+ char *sep = NULL;
+
+ /* Axe the namespace, we don't care about it */
+ /*
+ syslog(LOG_DEBUG,
+ "RSS: supplied el %d: %s\n", RSSAggr->RSSAggr->ItemType, supplied_el);
+ */
+ pel = supplied_el;
+ while (sep = strchr(pel, ':'), sep) {
+ pel = sep + 1;
+ }
+
+ if (pel != supplied_el)
+ {
+ void *v;
+
+ if (!GetHash(KnownNameSpaces,
+ supplied_el,
+ pel - supplied_el - 1,
+ &v))
+ {
+ EVRSSATOM_syslog(LOG_DEBUG,
+ "RSS: START ignoring "
+ "because of wrong namespace [%s]\n",
+ supplied_el);
+ return;
+ }
+ }
+
+ StrBufPlain(RSSAggr->Key, pel, -1);
+ StrBufLowerCase(RSSAggr->Key);
+ if (GetHash(StartHandlers, SKEY(RSSAggr->Key), &pv))
+ {
+ h = (rss_xml_handler*) pv;
+
+ if (((h->Flags & RSS_UNSET) != 0) &&
+ (RSSAggr->ItemType == RSS_UNSET))
+ {
+ h->Handler(RSSAggr->CData, ri, RSSAggr, attr);
+ }
+ else if (((h->Flags & RSS_RSS) != 0) &&
+ (RSSAggr->ItemType == RSS_RSS))
+ {
+ h->Handler(RSSAggr->CData, ri, RSSAggr, attr);
+ }
+ else if (((h->Flags & RSS_ATOM) != 0) &&
+ (RSSAggr->ItemType == RSS_ATOM))
+ {
+ h->Handler(RSSAggr->CData,
+ ri,
+ RSSAggr,
+ attr);
+ }
+ else
+ EVRSSATOM_syslog(LOG_DEBUG,
+ "RSS: START unhandled: [%s] [%s]...\n",
+ pel,
+ supplied_el);
+ }
+ else
+ EVRSSATOM_syslog(LOG_DEBUG,
+ "RSS: START unhandled: [%s] [%s]...\n",
+ pel,
+ supplied_el);
+}
+
+void rss_xml_end(void *data, const char *supplied_el)
+{
+ rss_xml_handler *h;
+ rss_aggregator *RSSAggr = (rss_aggregator*) data;
+ AsyncIO *IO = &RSSAggr->IO;
+ rss_item *ri = RSSAggr->Item;
+ const char *pel;
+ char *sep = NULL;
+ void *pv;
+
+ /* Axe the namespace, we don't care about it */
+ pel = supplied_el;
+ while (sep = strchr(pel, ':'), sep) {
+ pel = sep + 1;
+ }
+ EVRSSATOM_syslog(LOG_DEBUG, "RSS: END %s...\n", supplied_el);
+ if (pel != supplied_el)
+ {
+ void *v;
+
+ if (!GetHash(KnownNameSpaces,
+ supplied_el,
+ pel - supplied_el - 1,
+ &v))
+ {
+ EVRSSATOM_syslog(LOG_DEBUG,
+ "RSS: END ignoring because of wrong namespace"
+ "[%s] = [%s]\n",
+ supplied_el,
+ ChrPtr(RSSAggr->CData));
+ FlushStrBuf(RSSAggr->CData);
+ return;
+ }
+ }
+
+ StrBufPlain(RSSAggr->Key, pel, -1);
+ StrBufLowerCase(RSSAggr->Key);
+ if (GetHash(EndHandlers, SKEY(RSSAggr->Key), &pv))
+ {
+ h = (rss_xml_handler*) pv;
+
+ if (((h->Flags & RSS_UNSET) != 0) &&
+ (RSSAggr->ItemType == RSS_UNSET))
+ {
+ h->Handler(RSSAggr->CData, ri, RSSAggr, NULL);
+ }
+ else if (((h->Flags & RSS_RSS) != 0) &&
+ (RSSAggr->ItemType == RSS_RSS))
+ {
+ h->Handler(RSSAggr->CData, ri, RSSAggr, NULL);
+ }
+ else if (((h->Flags & RSS_ATOM) != 0) &&
+ (RSSAggr->ItemType == RSS_ATOM))
+ {
+ h->Handler(RSSAggr->CData, ri, RSSAggr, NULL);
+ }
+ else
+ EVRSSATOM_syslog(LOG_DEBUG,
+ "RSS: END unhandled: [%s] [%s] = [%s]...\n",
+ pel,
+ supplied_el,
+ ChrPtr(RSSAggr->CData));
+ }
+ else
+ EVRSSATOM_syslog(LOG_DEBUG,
+ "RSS: END unhandled: [%s] [%s] = [%s]...\n",
+ pel,
+ supplied_el,
+ ChrPtr(RSSAggr->CData));
+ FlushStrBuf(RSSAggr->CData);
+}
+
+
+
+/*
+ * Callback function for passing libcurl's output to expat for parsing
+ * we don't do streamed parsing so expat can handle non-utf8 documents