more work on libev´ing the rss-reader.

diff --git a/citadel/event_client.h b/citadel/event_client.h
index a3d467db072ca2b3aff780ba54e48c81b40633e8..23fdf68dbb34971da133c287f08adad1347391d7 100644 (file)
--- a/citadel/event_client.h
+++ b/citadel/event_client.h
@@ -92,6 +92,10 @@ struct AsyncIO {
        
        evcurl_request_data HttpReq;
 
+       /* Saving / loading a message async from / to disk */
+
+       struct CtdlMessage *AsyncMsg;
+       struct recptypes AsyncRcp;
        /* Custom data; its expected to contain  AsyncIO so we can save malloc()s... */
        void *Data;        /* application specific data */
        void *CitContext;  /* Citadel Session context... */

diff --git a/citadel/modules/rssclient/serv_rssclient.c b/citadel/modules/rssclient/serv_rssclient.c
index 13a887e0f5a87d2806f433628b1fcb61fb0aaa70..a01a92f1c46fefcbee702e9dd46768dd1065ecce 100644 (file)
--- a/citadel/modules/rssclient/serv_rssclient.c
+++ b/citadel/modules/rssclient/serv_rssclient.c
@@ -58,6 +58,11 @@
 #include "rss_atom_parser.h"
 
 
+#define TMP_MSGDATA 0xFF
+#define TMP_SHORTER_URL_OFFSET 0xFE
+#define TMP_SHORTER_URLS 0xFD
+
+
 struct rssnetcfg *rnclist = NULL;
 void AppendLink(StrBuf *Message, StrBuf *link, StrBuf *LinkTitle, const char *Title)
 {
@@ -75,6 +80,25 @@ void AppendLink(StrBuf *Message, StrBuf *link, StrBuf *LinkTitle, const char *Ti
                StrBufAppendBufPlain(Message, HKEY("</a><br>\n"), 0);
        }
 }
+
+void RSSSaveMessage(struct CtdlMessage *Msg, rss_item *ri, struct UseTable *ut)
+{
+
+       CtdlSubmitMsg(msg, recp, NULL, 0);
+       CtdlFreeMessage(msg);
+
+       /* write the uidl to the use table so we don't store this item again */
+       cdb_store(CDB_USETABLE, utmsgid, strlen(utmsgid), &ut, sizeof(struct UseTable) );
+       free(ut);
+}
+
+
+rss_save_msg(msg, recp)
+{
+
+
+}
+
 /*
  * Commit a fetched and parsed RSS item to disk
  */
@@ -91,6 +115,7 @@ void rss_save_item(rss_item *ri)
        struct recptypes *recp = NULL;
        int msglen = 0;
        StrBuf *Message;
+       AsyncIO *OtherIO;
 
        recp = (struct recptypes *) malloc(sizeof(struct recptypes));
        if (recp == NULL) return;
@@ -125,138 +150,150 @@ void rss_save_item(rss_item *ri)
                strcat(utmsgid, "_rss2ctdl");
        }
 
-       /* Find out if we've already seen this item */
+       /* translate Item into message. */
+       CtdlLogPrintf(CTDL_DEBUG, "RSS: translating item...\n");
+       if (ri->description == NULL) ri->description = NewStrBufPlain(HKEY(""));
+       StrBufSpaceToBlank(ri->description);
+       msg = malloc(sizeof(struct CtdlMessage));
+       memset(msg, 0, sizeof(struct CtdlMessage));
+       msg->cm_magic = CTDLMESSAGE_MAGIC;
+       msg->cm_anon_type = MES_NORMAL;
+       msg->cm_format_type = FMT_RFC822;
 
-       cdbut = cdb_fetch(CDB_USETABLE, utmsgid, strlen(utmsgid));
-#ifndef DEBUG_RSS
-       if (cdbut != NULL) {
-               /* Item has already been seen */
-               CtdlLogPrintf(CTDL_DEBUG, "%s has already been seen\n", utmsgid);
-               cdb_free(cdbut);
-
-               /* rewrite the record anyway, to update the timestamp */
-               strcpy(ut.ut_msgid, utmsgid);
-               ut.ut_timestamp = time(NULL);
-               cdb_store(CDB_USETABLE, utmsgid, strlen(utmsgid), &ut, sizeof(struct UseTable) );
+       if (ri->guid != NULL) {
+               msg->cm_fields['E'] = strdup(ChrPtr(ri->guid));
        }
-       else
-#endif
-{
-               /* Item has not been seen, so save it. */
-               CtdlLogPrintf(CTDL_DEBUG, "RSS: saving item...\n");
-               if (ri->description == NULL) ri->description = NewStrBufPlain(HKEY(""));
-               StrBufSpaceToBlank(ri->description);
-               msg = malloc(sizeof(struct CtdlMessage));
-               memset(msg, 0, sizeof(struct CtdlMessage));
-               msg->cm_magic = CTDLMESSAGE_MAGIC;
-               msg->cm_anon_type = MES_NORMAL;
-               msg->cm_format_type = FMT_RFC822;
-
-               if (ri->guid != NULL) {
-                       msg->cm_fields['E'] = strdup(ChrPtr(ri->guid));
-               }
 
-               if (ri->author_or_creator != NULL) {
-                       char *From;
-                       StrBuf *Encoded = NULL;
-                       int FromAt;
+       if (ri->author_or_creator != NULL) {
+               char *From;
+               StrBuf *Encoded = NULL;
+               int FromAt;
                        
-                       From = html_to_ascii(ChrPtr(ri->author_or_creator),
-                                            StrLength(ri->author_or_creator), 
-                                            512, 0);
-                       StrBufPlain(ri->author_or_creator, From, -1);
-                       StrBufTrim(ri->author_or_creator);
-                       free(From);
-
-                       FromAt = strchr(ChrPtr(ri->author_or_creator), '@') != NULL;
-                       if (!FromAt && StrLength (ri->author_email) > 0)
+               From = html_to_ascii(ChrPtr(ri->author_or_creator),
+                                    StrLength(ri->author_or_creator), 
+                                    512, 0);
+               StrBufPlain(ri->author_or_creator, From, -1);
+               StrBufTrim(ri->author_or_creator);
+               free(From);
+
+               FromAt = strchr(ChrPtr(ri->author_or_creator), '@') != NULL;
+               if (!FromAt && StrLength (ri->author_email) > 0)
+               {
+                       StrBufRFC2047encode(&Encoded, ri->author_or_creator);
+                       msg->cm_fields['A'] = SmashStrBuf(&Encoded);
+                       msg->cm_fields['P'] = SmashStrBuf(&ri->author_email);
+               }
+               else
+               {
+                       if (FromAt)
+                               msg->cm_fields['P'] = SmashStrBuf(&ri->author_or_creator);
+                       else 
                        {
                                StrBufRFC2047encode(&Encoded, ri->author_or_creator);
                                msg->cm_fields['A'] = SmashStrBuf(&Encoded);
-                               msg->cm_fields['P'] = SmashStrBuf(&ri->author_email);
-                       }
-                       else
-                       {
-                               if (FromAt)
-                                       msg->cm_fields['P'] = SmashStrBuf(&ri->author_or_creator);
-                               else 
-                               {
-                                       StrBufRFC2047encode(&Encoded, ri->author_or_creator);
-                                       msg->cm_fields['A'] = SmashStrBuf(&Encoded);
-                                       msg->cm_fields['P'] = strdup("rss@localhost");
-                               }
+                               msg->cm_fields['P'] = strdup("rss@localhost");
                        }
                }
-               else {
-                       msg->cm_fields['A'] = strdup("rss");
-               }
+       }
+       else {
+               msg->cm_fields['A'] = strdup("rss");
+       }
 
-               msg->cm_fields['N'] = strdup(NODENAME);
-               if (ri->title != NULL) {
-                       long len;
-                       char *Sbj;
-                       StrBuf *Encoded, *QPEncoded;
-
-                       QPEncoded = NULL;
-                       StrBufSpaceToBlank(ri->title);
-                       len = StrLength(ri->title);
-                       Sbj = html_to_ascii(ChrPtr(ri->title), len, 512, 0);
-                       len = strlen(Sbj);
-                       if (Sbj[len - 1] == '\n')
-                       {
-                               len --;
-                               Sbj[len] = '\0';
-                       }
-                       Encoded = NewStrBufPlain(Sbj, len);
-                       free(Sbj);
+       msg->cm_fields['N'] = strdup(NODENAME);
+       if (ri->title != NULL) {
+               long len;
+               char *Sbj;
+               StrBuf *Encoded, *QPEncoded;
+
+               QPEncoded = NULL;
+               StrBufSpaceToBlank(ri->title);
+               len = StrLength(ri->title);
+               Sbj = html_to_ascii(ChrPtr(ri->title), len, 512, 0);
+               len = strlen(Sbj);
+               if (Sbj[len - 1] == '\n')
+               {
+                       len --;
+                       Sbj[len] = '\0';
+               }
+               Encoded = NewStrBufPlain(Sbj, len);
+               free(Sbj);
 
-                       StrBufTrim(Encoded);
-                       StrBufRFC2047encode(&QPEncoded, Encoded);
+               StrBufTrim(Encoded);
+               StrBufRFC2047encode(&QPEncoded, Encoded);
 
-                       msg->cm_fields['U'] = SmashStrBuf(&QPEncoded);
-                       FreeStrBuf(&Encoded);
-               }
-               msg->cm_fields['T'] = malloc(64);
-               snprintf(msg->cm_fields['T'], 64, "%ld", ri->pubdate);
-               if (ri->channel_title != NULL) {
-                       if (StrLength(ri->channel_title) > 0) {
-                               msg->cm_fields['O'] = strdup(ChrPtr(ri->channel_title));
-                       }
+               msg->cm_fields['U'] = SmashStrBuf(&QPEncoded);
+               FreeStrBuf(&Encoded);
+       }
+       msg->cm_fields['T'] = malloc(64);
+       snprintf(msg->cm_fields['T'], 64, "%ld", ri->pubdate);
+       if (ri->channel_title != NULL) {
+               if (StrLength(ri->channel_title) > 0) {
+                       msg->cm_fields['O'] = strdup(ChrPtr(ri->channel_title));
                }
-               if (ri->link == NULL) 
-                       ri->link = NewStrBufPlain(HKEY(""));
-               // TODO: reenable me    ExpandShortUrls(ri->description);
-               msglen += 1024 + StrLength(ri->link) + StrLength(ri->description) ;
+       }
+       if (ri->link == NULL) 
+               ri->link = NewStrBufPlain(HKEY(""));
 
-               Message = NewStrBufPlain(NULL, StrLength(ri->description));
 
-               StrBufPlain(Message, HKEY(
-                        "Content-type: text/html; charset=\"UTF-8\"\r\n\r\n"
-                        "<html><body>\n"));
+       msg->cm_fields[TMP_SHORTER_URLS] = GetShorterUrls(ri->description);
 
-               StrBufAppendBuf(Message, ri->description, 0);
-               StrBufAppendBufPlain(Message, HKEY("<br><br>\n"), 0);
+       strcpy(ut->ut_msgid, utmsgid);
+       ut->ut_timestamp = time(NULL);
 
-               AppendLink(Message, ri->link, ri->linkTitle, NULL);
-               AppendLink(Message, ri->reLink, ri->reLinkTitle, "Reply to this");
-               StrBufAppendBufPlain(Message, HKEY("</body></html>\n"), 0);
+       msglen += 1024 + StrLength(ri->link) + StrLength(ri->description) ;
 
-               msg->cm_fields['M'] = SmashStrBuf(&Message);
+       Message = NewStrBufPlain(NULL, StrLength(ri->description));
 
-               CtdlSubmitMsg(msg, recp, NULL, 0);
-               CtdlFreeMessage(msg);
+       StrBufPlain(Message, HKEY(
+                           "Content-type: text/html; charset=\"UTF-8\"\r\n\r\n"
+                           "<html><body>\n"));
+       msg->cm_fields[TMP_SHORTER_URL_OFFSET] = StrLength(Message);
+       StrBufAppendBuf(Message, ri->description, 0);
+       StrBufAppendBufPlain(Message, HKEY("<br><br>\n"), 0);
 
-               /* write the uidl to the use table so we don't store this item again */
-               strcpy(ut.ut_msgid, utmsgid);
-               ut.ut_timestamp = time(NULL);
-               cdb_store(CDB_USETABLE, utmsgid, strlen(utmsgid), &ut, sizeof(struct UseTable) );
-       }
-       free_recipients(recp);
+       AppendLink(Message, ri->link, ri->linkTitle, NULL);
+       AppendLink(Message, ri->reLink, ri->reLinkTitle, "Reply to this");
+       StrBufAppendBufPlain(Message, HKEY("</body></html>\n"), 0);
+
+
+       msg->cm_fields[TMP_MSGDATA] = Message;
+       
+
+       OtherIO = malloc(sizeof(AsyncIO));
+       memset(OtherIO, 0, sizeof(AsyncIO));
+       OtherIO->AsyncMsg = msg;
+       OtherIO->AsyncRcp = recp;
+
+       rss_save_msg(msg, recp);
+//     msg->cm_fields['M'] = SmashStrBuf(&Message);
+
+       // TODO: reenable me    ExpandShortUrls(ri->description);
+
+///    free_recipients(recp);
 }
 
 
 
 
+       /* Find out if we've already seen this item * /
+
+       cdbut = cdb_fetch(CDB_USETABLE, utmsgid, strlen(utmsgid));
+#ifndef DEBUG_RSS
+       if (cdbut != NULL) {
+               /* Item has already been seen * /
+               CtdlLogPrintf(CTDL_DEBUG, "%s has already been seen\n", utmsgid);
+               cdb_free(cdbut);
+
+               /* rewrite the record anyway, to update the timestamp * /
+               strcpy(ut.ut_msgid, utmsgid);
+               ut.ut_timestamp = time(NULL);
+               cdb_store(CDB_USETABLE, utmsgid, strlen(utmsgid), &ut, sizeof(struct UseTable) );
+       }
+       else
+#endif
+       {
+*/
+
 
 /*
  * Begin a feed parse

diff --git a/citadel/modules/urldeshortener/serv_expand_shorter_urls.c b/citadel/modules/urldeshortener/serv_expand_shorter_urls.c
index f7dac2a7a0c6c75d36c7b821f0b76636b8f7899a..ea37d4034b65ef422f5fd2054522d18fbcc4ad35 100644 (file)
--- a/citadel/modules/urldeshortener/serv_expand_shorter_urls.c
+++ b/citadel/modules/urldeshortener/serv_expand_shorter_urls.c
@@ -191,77 +191,84 @@ int SortConstStrByPosition(const void *Item1, const void *Item2)
        return -1;
 }
 
-void ExpandShortUrls(StrBuf *Message)
+HashList GetShorterUrls(StrBuf Message)
 {
-       StrBuf *Shadow;
        HashList *pUrls;
-       ConstStr *pCUrl;
-       const char *pch;
-       const char *pche;
-
        /* we just suspect URL shorteners to be inside of feeds from twitter
         * or other short content messages, so don't crawl through real blogs.
         */
        if (StrLength(Message) > 500)
-               return;
+               return NULL;
 
        pUrls = NewHash(1, Flathash);
        CrawlMessageForShorterUrls(pUrls, Message);
 
        if (GetCount(pUrls) > 0)
-       {
-               StrBuf *ShorterUrlStr;
-               HashPos *Pos;
-               const char *Key;
-               void *pv;
-               long len;
+               return pURLs;
+       else 
+               return NULL;
 
-               Shadow = NewStrBufPlain(NULL, StrLength(Message));
-               SortByPayload (pUrls, SortConstStrByPosition);
+}
 
-               ShorterUrlStr = NewStrBufPlain(NULL, StrLength(Message));
+void ExpandShortUrls(StrBuf *Message, HashList *pUrls, int Callback)
+{
+       StrBuf *Shadow;
+       ConstStr *pCUrl;
+       const char *pch;
+       const char *pche;
 
-               pch = ChrPtr(Message);
-               pche = pch + StrLength(Message);
-               Pos = GetNewHashPos(pUrls, 1);
-               while (GetNextHashPos(pUrls, Pos, &len, &Key, &pv))
-               {
-                       pCUrl = (ConstStr*) pv;
+       StrBuf *ShorterUrlStr;
+       HashPos *Pos;
+       const char *Key;
+       void *pv;
+       long len;
+       
+       Shadow = NewStrBufPlain(NULL, StrLength(Message));
+       SortByPayload (pUrls, SortConstStrByPosition);
+               
+       ShorterUrlStr = NewStrBufPlain(NULL, StrLength(Message));
+               
+       pch = ChrPtr(Message);
+       pche = pch + StrLength(Message);
+       Pos = GetNewHashPos(pUrls, 1);
+       while (GetNextHashPos(pUrls, Pos, &len, &Key, &pv))
+       {
+               pCUrl = (ConstStr*) pv;
 
-                       if (pch != pCUrl->Key)
-                               StrBufAppendBufPlain(Shadow, pch, pCUrl->Key - pch, 0);
+               if (pch != pCUrl->Key)
+                       StrBufAppendBufPlain(Shadow, pch, pCUrl->Key - pch, 0);
                        
-                       StrBufPlain(ShorterUrlStr, CKEY(*pCUrl));
-                       if (LookupUrl(ShorterUrlStr))
-                       {
-                               StrBufAppendBufPlain(Shadow, HKEY("<a href=\""), 0);
-                               StrBufAppendBuf(Shadow, ShorterUrlStr, 0);
-                               StrBufAppendBufPlain(Shadow, HKEY("\">"), 0);
-                               StrBufAppendBuf(Shadow, ShorterUrlStr, 0);
-                               StrBufAppendBufPlain(Shadow, HKEY("["), 0);
-                               StrBufAppendBufPlain(Shadow, pCUrl->Key, pCUrl->len, 0);
-                               StrBufAppendBufPlain(Shadow, HKEY("]</a>"), 0);
-                       }
-                       else
-                       {
-                               StrBufAppendBufPlain(Shadow, HKEY("<a href=\""), 0);
-                               StrBufAppendBufPlain(Shadow, pCUrl->Key, pCUrl->len, 0);
-                               StrBufAppendBufPlain(Shadow, HKEY("\">"), 0);
-                               StrBufAppendBufPlain(Shadow, pCUrl->Key, pCUrl->len, 0);
-                               StrBufAppendBufPlain(Shadow, HKEY("</a>"), 0);
-                       }
-                       pch = pCUrl->Key + pCUrl->len + 1;
-
+               StrBufPlain(ShorterUrlStr, CKEY(*pCUrl));
+               if (LookupUrl(ShorterUrlStr))
+               {
+                       StrBufAppendBufPlain(Shadow, HKEY("<a href=\""), 0);
+                       StrBufAppendBuf(Shadow, ShorterUrlStr, 0);
+                       StrBufAppendBufPlain(Shadow, HKEY("\">"), 0);
+                       StrBufAppendBuf(Shadow, ShorterUrlStr, 0);
+                       StrBufAppendBufPlain(Shadow, HKEY("["), 0);
+                       StrBufAppendBufPlain(Shadow, pCUrl->Key, pCUrl->len, 0);
+                       StrBufAppendBufPlain(Shadow, HKEY("]</a>"), 0);
                }
-               if (pch < pche)
-                       StrBufAppendBufPlain(Shadow, pch, pche - pch, 0);
-               FlushStrBuf(Message);
-               StrBufAppendBuf(Message, Shadow, 0);
-
-               FreeStrBuf(&ShorterUrlStr);
-               FreeStrBuf(&Shadow);
-               DeleteHashPos(&Pos);
+               else
+               {
+                       StrBufAppendBufPlain(Shadow, HKEY("<a href=\""), 0);
+                       StrBufAppendBufPlain(Shadow, pCUrl->Key, pCUrl->len, 0);
+                       StrBufAppendBufPlain(Shadow, HKEY("\">"), 0);
+                       StrBufAppendBufPlain(Shadow, pCUrl->Key, pCUrl->len, 0);
+                       StrBufAppendBufPlain(Shadow, HKEY("</a>"), 0);
+               }
+               pch = pCUrl->Key + pCUrl->len + 1;
+
        }
+       if (pch < pche)
+               StrBufAppendBufPlain(Shadow, pch, pche - pch, 0);
+       FlushStrBuf(Message);
+       StrBufAppendBuf(Message, Shadow, 0);
+
+       FreeStrBuf(&ShorterUrlStr);
+       FreeStrBuf(&Shadow);
+       DeleteHashPos(&Pos);
+       
 
        DeleteHash(&pUrls);
 }

diff --git a/citadel/modules/urldeshortener/serv_expand_shorter_urls.h b/citadel/modules/urldeshortener/serv_expand_shorter_urls.h
index 477f6b5a3381a7ef198f683f7d6babae100ccb3f..4efcc26d04247699c63971a4af0855d43ac04adc 100644 (file)
--- a/citadel/modules/urldeshortener/serv_expand_shorter_urls.h
+++ b/citadel/modules/urldeshortener/serv_expand_shorter_urls.h
@@ -1 +1,3 @@
-void ExpandShortUrls(StrBuf *Message, Callback)
+void ExpandShortUrls(StrBuf *Message, Callback);
+
+HashList GetShorterUrls(StrBuf Message);