Began working on the 'clean shutdown' code for the new thread architecture.
[citadel.git] / citadel / modules / rssclient / serv_rssclient.c
1 /*
2  * Bring external RSS feeds into rooms.
3  *
4  * Copyright (c) 2007-2010 by the citadel.org team
5  *
6  * This program is free software; you can redistribute it and/or modify
7  * it under the terms of the GNU General Public License as published by
8  * the Free Software Foundation; either version 3 of the License, or
9  * (at your option) any later version.
10  *
11  * This program is distributed in the hope that it will be useful,
12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
14  * GNU General Public License for more details.
15  *
16  * You should have received a copy of the GNU General Public License
17  * along with this program; if not, write to the Free Software
18  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
19  */
20
21 #include <stdlib.h>
22 #include <unistd.h>
23 #include <stdio.h>
24
25 #if TIME_WITH_SYS_TIME
26 # include <sys/time.h>
27 # include <time.h>
28 #else
29 # if HAVE_SYS_TIME_H
30 #  include <sys/time.h>
31 # else
32 #  include <time.h>
33 # endif
34 #endif
35
36 #include <ctype.h>
37 #include <string.h>
38 #include <errno.h>
39 #include <sys/types.h>
40 #include <sys/stat.h>
41 #include <expat.h>
42 #include <curl/curl.h>
43 #include <libcitadel.h>
44 #include "citadel.h"
45 #include "server.h"
46 #include "citserver.h"
47 #include "support.h"
48 #include "config.h"
49 #include "threads.h"
50 #include "ctdl_module.h"
51 #include "clientsocket.h"
52 #include "msgbase.h"
53 #include "parsedate.h"
54 #include "database.h"
55 #include "citadel_dirs.h"
56 #include "md5.h"
57 #include "context.h"
58
59
60 typedef struct rssnetcfg rssnetcfg;
61 struct rssnetcfg {
62         rssnetcfg *next;
63         char url[256];
64         char *rooms;
65         time_t last_error_when;
66         int ItemType;
67         time_t next_poll;
68 };
69
70 #define RSS_UNSET       (1<<0)
71 #define RSS_RSS         (1<<1)
72 #define RSS_ATOM        (1<<2)
73 #define RSS_REQUIRE_BUF (1<<3)
74
75 typedef struct _rss_item {
76         char *roomlist;
77         int done_parsing;
78         StrBuf *guid;
79         StrBuf *title;
80         StrBuf *link;
81         StrBuf *linkTitle;
82         StrBuf *reLink;
83         StrBuf *reLinkTitle;
84         StrBuf *description;
85         time_t pubdate;
86         StrBuf *channel_title;
87         int item_tag_nesting;
88         StrBuf *author_or_creator;
89         StrBuf *author_url;
90         StrBuf *author_email;
91 }rss_item;
92
93
94 typedef void (*rss_handler_func)(StrBuf *CData, 
95                                  rss_item *ri, 
96                                  rssnetcfg *Cfg, 
97                                  const char** Attr);
98
99 typedef struct __rss_xml_handler {
100         int Flags;
101         rss_handler_func Handler;
102 }rss_xml_handler;
103
104
105 typedef struct _rsscollection {
106         StrBuf *CData;
107         StrBuf *Key;
108
109         rss_item *Item;
110         rssnetcfg *Cfg;
111         
112         rss_xml_handler *Current;
113 } rsscollection;
114
115 struct rssnetcfg *rnclist = NULL;
116 HashList *StartHandlers = NULL;
117 HashList *EndHandlers = NULL;
118 HashList *KnownNameSpaces = NULL;
119 HashList *UrlShorteners = NULL;
120 void AddRSSStartHandler(rss_handler_func Handler, int Flags, const char *key, long len)
121 {
122         rss_xml_handler *h;
123         h = (rss_xml_handler*) malloc(sizeof (rss_xml_handler));
124         h->Flags = Flags;
125         h->Handler = Handler;
126         Put(StartHandlers, key, len, h, NULL);
127 }
128 void AddRSSEndHandler(rss_handler_func Handler, int Flags, const char *key, long len)
129 {
130         rss_xml_handler *h;
131         h = (rss_xml_handler*) malloc(sizeof (rss_xml_handler));
132         h->Flags = Flags;
133         h->Handler = Handler;
134         Put(EndHandlers, key, len, h, NULL);
135 }
136
137 #if 0
138 //#ifdef HAVE_ICONV
139 #include <iconv.h>
140
141
142 /* 
143  * dug this out of the trashcan of the midgard project, lets see if it works for us.
144  * original code by Alexander Bokovoy <bokovoy@avilink.ne> distributed under GPL V2 or later
145  */
146
147 /* Returns: 
148  >= 0 - successfull, 0 means conversion doesn't use multibyte sequences 
149    -1 - error during iconv_open call 
150    -2 - error during iconv_close call 
151    ---------------------------------- 
152    This function expects that multibyte encoding in 'charset' wouldn't have 
153    characters with more than 3 bytes. It is not intended to convert UTF-8 because 
154    we'll never receive UTF-8 in our handler (it is handled by Exat itself). 
155 */ 
156 static int 
157 fill_encoding_info (const char *charset, XML_Encoding * info) 
158
159   iconv_t cd = (iconv_t)(-1); 
160   int flag; 
161         syslog(LOG_EMERG, "RSS: fill encoding info ...\n");
162  
163 #if G_BYTE_ORDER == G_LITTLE_ENDIAN 
164   cd = iconv_open ("UCS-2LE", charset); 
165 #else 
166   cd = iconv_open ("UCS-2BE", charset); 
167 #endif 
168  
169   if (cd == (iconv_t) (-1)) 
170     { 
171       return -1; 
172     } 
173  
174   { 
175     unsigned short out = 0; 
176     unsigned char buf[4]; 
177     unsigned int i0, i1, i2; 
178     int result = 0; 
179     flag = 0; 
180     for (i0 = 0; i0 < 0x100; i0++) 
181       { 
182         buf[0] = i0; 
183         info->map[i0] = 0; 
184         //result = try (cd, buf, 1, &out); 
185         if (result < 0) 
186           { 
187           } 
188         else if (result > 0) 
189           { 
190             info->map[i0] = out; 
191           } 
192         else 
193           { 
194             for (i1 = 0; i1 < 0x100; i1++) 
195               { 
196                 buf[1] = i1; 
197                 ///result = try (cd, buf, 2, &out); 
198                 if (result < 0) 
199                   { 
200                   } 
201                 else if (result > 0) 
202                   { 
203                     flag++; 
204                     info->map[i0] = -2; 
205                   } 
206                 else 
207                   { 
208                     for (i2 = 0; i2 < 0x100; i2++) 
209                       { 
210                         buf[2] = i2; 
211                         ////result = try (cd, buf, 3, &out); 
212                         if (result < 0) 
213                           { 
214                           } 
215                         else if (result > 0) 
216                           { 
217                             flag++; 
218                             info->map[i0] = -3; 
219                           } 
220                       } 
221                   } 
222               } 
223           } 
224       } 
225   } 
226  
227   if (iconv_close (cd) < 0) 
228     { 
229       return -2; 
230     } 
231   return flag; 
232
233
234 static int 
235 iconv_convertor (void *data, const char *s) 
236
237   XML_Encoding *info = data; 
238   int res; 
239         syslog(LOG_EMERG, "RSS: Converting ...\n");
240
241   if (s == NULL) 
242     return -1; 
243 /*
244   GByteArray *result; 
245   result = g_byte_array_new (); 
246   if (process_block (info->data, (char *) s, strlen (s), result) == 0) 
247     { 
248       res = *(result->data); 
249       g_byte_array_free (result, TRUE); 
250       return res; 
251     } 
252   g_byte_array_free (result, TRUE); 
253 */
254   return -1; 
255
256
257 static void 
258 my_release (void *data) 
259
260   iconv_t cd = (iconv_t) data; 
261   if (iconv_close (cd) != 0) 
262     { 
263 /// TODO: uh no.      exit (1); 
264     } 
265
266 int 
267 handle_unknown_xml_encoding (void *encodingHandleData, 
268                              const XML_Char * name, 
269                              XML_Encoding * info) 
270
271   int result; 
272   syslog(LOG_EMERG, "RSS: unknown encoding ...\n");
273   result = fill_encoding_info (name, info); 
274   if (result >= 0) 
275     { 
276       /*  
277         Special case: client asked for reverse conversion, we'll provide him with 
278         iconv descriptor which handles it. Client should release it by himself. 
279       */ 
280       if(encodingHandleData != NULL) 
281             *((iconv_t *)encodingHandleData) = iconv_open(name, "UTF-8"); 
282       /*  
283          Optimization: we do not need conversion function if encoding is one-to-one,  
284          info->map table will be enough  
285        */ 
286       if (result == 0) 
287         { 
288           info->data = NULL; 
289           info->convert = NULL; 
290           info->release = NULL; 
291           return 1; 
292         } 
293       /*  
294          We do need conversion function because this encoding uses multibyte sequences 
295        */ 
296       info->data = (void *) iconv_open ("UTF-8", name); 
297       if ((int)info->data == -1) 
298         return -1; 
299       info->convert = iconv_convertor; 
300       info->release = my_release; 
301       return 1; 
302     } 
303   if(encodingHandleData != NULL)  
304     *(iconv_t *)encodingHandleData = NULL; 
305   return 0; 
306
307
308 ///#endif
309 #endif
310 size_t GetLocationString( void *ptr, size_t size, size_t nmemb, void *userdata)
311 {
312 #define LOCATION "location"
313         if (strncasecmp((char*)ptr, LOCATION, sizeof(LOCATION) - 1) == 0)
314         {
315                 StrBuf *pURL = (StrBuf*) userdata;
316                 char *pch = (char*) ptr;
317                 char *pche;
318                 
319                 pche = pch + (size * nmemb);
320                 pch += sizeof(LOCATION);
321                 
322                 while (isspace(*pch) || (*pch == ':'))
323                         pch ++;
324
325                 while (isspace(*pche) || (*pche == '\0'))
326                         pche--;
327                 
328                 FlushStrBuf(pURL);
329                 StrBufPlain(pURL, pch, pche - pch + 1); 
330         }
331         return size * nmemb;
332 }
333
334 int LookupUrl(StrBuf *ShorterUrlStr)
335 {
336         CURL *curl;
337         char errmsg[1024] = "";
338         StrBuf *Answer;
339         int rc = 0;
340
341         curl = curl_easy_init();
342         if (!curl) {
343                 syslog(LOG_ALERT, "Unable to initialize libcurl.\n");
344                 return 0;
345         }
346         Answer = NewStrBufPlain(NULL, SIZ);
347
348         curl_easy_setopt(curl, CURLOPT_URL, ChrPtr(ShorterUrlStr));
349         curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 0);
350         curl_easy_setopt(curl, CURLOPT_SSL_VERIFYHOST, 0);
351         curl_easy_setopt(curl, CURLOPT_WRITEDATA, Answer);
352 //      curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, rss_libcurl_callback);
353         curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, CurlFillStrBuf_callback);
354         curl_easy_setopt(curl, CURLOPT_ERRORBUFFER, errmsg);
355         curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1);
356 #ifdef CURLOPT_HTTP_CONTENT_DECODING
357         curl_easy_setopt(curl, CURLOPT_HTTP_CONTENT_DECODING, 1);
358         curl_easy_setopt(curl, CURLOPT_ENCODING, "");
359 #endif
360         curl_easy_setopt(curl, CURLOPT_USERAGENT, CITADEL);
361         curl_easy_setopt(curl, CURLOPT_TIMEOUT, 180);           /* die after 180 seconds */
362         curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 0);
363
364         curl_easy_setopt(curl, CURLOPT_HEADERFUNCTION , GetLocationString);
365         curl_easy_setopt(curl, CURLOPT_WRITEHEADER, ShorterUrlStr);
366
367
368         if (
369                 (!IsEmptyStr(config.c_ip_addr))
370                 && (strcmp(config.c_ip_addr, "*"))
371                 && (strcmp(config.c_ip_addr, "::"))
372                 && (strcmp(config.c_ip_addr, "0.0.0.0"))
373         ) {
374                 curl_easy_setopt(curl, CURLOPT_INTERFACE, config.c_ip_addr);
375         }
376
377         if (server_shutting_down)
378                 goto shutdown ;
379
380         rc = curl_easy_perform(curl);
381         if (rc) {
382                 syslog(LOG_ALERT, "libcurl error %d: %s\n", rc, errmsg);
383                 rc = 0;
384         }
385         else 
386                 rc = 1;
387
388 shutdown:
389         curl_easy_cleanup(curl);
390
391         return rc;
392
393 }
394
395
396
397 void CrawlMessageForShorterUrls(HashList *pUrls, StrBuf *Message)
398 {
399         int nHits = 0;
400         void *pv;
401         int nShorter = 0;
402         const char *pch;
403         const char *pUrl;
404         ConstStr *pCUrl;
405
406         while (GetHash(UrlShorteners, IKEY(nShorter), &pv))
407         {
408                 nShorter++;
409                 pch = ChrPtr(Message);
410                 pUrl = strstr(pch, ChrPtr((StrBuf*)pv));
411                 while ((pUrl != NULL) && (nHits < 99))
412                 {
413                         pCUrl = malloc(sizeof(ConstStr));
414
415                         pCUrl->Key = pUrl;
416                         pch = pUrl + StrLength((StrBuf*)pv);
417                         while (isalnum(*pch)||(*pch == '-')||(*pch == '/'))
418                                 pch++;
419                         pCUrl->len = pch - pCUrl->Key;
420
421                         Put(pUrls, IKEY(nHits), pCUrl, NULL);
422                         nHits ++;
423                         pUrl = strstr(pch, ChrPtr((StrBuf*)pv));
424                 }
425         }
426 }
427
428 int SortConstStrByPosition(const void *Item1, const void *Item2)
429 {
430         const ConstStr *p1, *p2;
431         p1 = (const ConstStr*) Item1;
432         p2 = (const ConstStr*) Item2;
433         if (p1->Key == p2->Key)
434                 return 0;
435         if (p1->Key > p2->Key)
436                 return 1;
437         return -1;
438 }
439
440 void ExpandShortUrls(StrBuf *Message)
441 {
442         StrBuf *Shadow;
443         HashList *pUrls;
444         ConstStr *pCUrl;
445         const char *pch;
446         const char *pche;
447
448         /* we just suspect URL shorteners to be inside of feeds from twitter
449          * or other short content messages, so don't crawl through real blogs.
450          */
451         if (StrLength(Message) > 500)
452                 return;
453
454         pUrls = NewHash(1, Flathash);
455         CrawlMessageForShorterUrls(pUrls, Message);
456
457         if (GetCount(pUrls) > 0)
458         {
459                 StrBuf *ShorterUrlStr;
460                 HashPos *Pos;
461                 const char *RetrKey;
462                 void *pv;
463                 long len;
464
465                 Shadow = NewStrBufPlain(NULL, StrLength(Message));
466                 SortByPayload (pUrls, SortConstStrByPosition);
467
468                 ShorterUrlStr = NewStrBufPlain(NULL, StrLength(Message));
469
470                 pch = ChrPtr(Message);
471                 pche = pch + StrLength(Message);
472                 Pos = GetNewHashPos(pUrls, 1);
473                 while (GetNextHashPos(pUrls, Pos, &len, &RetrKey, &pv))
474                 {
475                         pCUrl = (ConstStr*) pv;
476
477                         if (pch != pCUrl->Key)
478                                 StrBufAppendBufPlain(Shadow, pch, pCUrl->Key - pch, 0);
479                         
480                         StrBufPlain(ShorterUrlStr, CKEY(*pCUrl));
481                         if (LookupUrl(ShorterUrlStr))
482                         {
483                                 StrBufAppendBufPlain(Shadow, HKEY("<a href=\""), 0);
484                                 StrBufAppendBuf(Shadow, ShorterUrlStr, 0);
485                                 StrBufAppendBufPlain(Shadow, HKEY("\">"), 0);
486                                 StrBufAppendBuf(Shadow, ShorterUrlStr, 0);
487                                 StrBufAppendBufPlain(Shadow, HKEY("["), 0);
488                                 StrBufAppendBufPlain(Shadow, pCUrl->Key, pCUrl->len, 0);
489                                 StrBufAppendBufPlain(Shadow, HKEY("]</a>"), 0);
490                         }
491                         else
492                         {
493                                 StrBufAppendBufPlain(Shadow, HKEY("<a href=\""), 0);
494                                 StrBufAppendBufPlain(Shadow, pCUrl->Key, pCUrl->len, 0);
495                                 StrBufAppendBufPlain(Shadow, HKEY("\">"), 0);
496                                 StrBufAppendBufPlain(Shadow, pCUrl->Key, pCUrl->len, 0);
497                                 StrBufAppendBufPlain(Shadow, HKEY("</a>"), 0);
498                         }
499                         pch = pCUrl->Key + pCUrl->len + 1;
500
501                 }
502                 if (pch < pche)
503                         StrBufAppendBufPlain(Shadow, pch, pche - pch, 0);
504                 FlushStrBuf(Message);
505                 StrBufAppendBuf(Message, Shadow, 0);
506
507                 FreeStrBuf(&ShorterUrlStr);
508                 FreeStrBuf(&Shadow);
509                 DeleteHashPos(&Pos);
510         }
511
512         DeleteHash(&pUrls);
513 }
514
515
516 void AppendLink(StrBuf *Message, StrBuf *link, StrBuf *LinkTitle, const char *Title)
517 {
518         if (StrLength(link) > 0)
519         {
520                 StrBufAppendBufPlain(Message, HKEY("<a href=\""), 0);
521                 StrBufAppendBuf(Message, link, 0);
522                 StrBufAppendBufPlain(Message, HKEY("\">"), 0);
523                 if (StrLength(LinkTitle) > 0)
524                         StrBufAppendBuf(Message, LinkTitle, 0);
525                 else if ((Title != NULL) && !IsEmptyStr(Title))
526                         StrBufAppendBufPlain(Message, Title, -1, 0);
527                 else
528                         StrBufAppendBuf(Message, link, 0);
529                 StrBufAppendBufPlain(Message, HKEY("</a><br>\n"), 0);
530         }
531 }
532 /*
533  * Commit a fetched and parsed RSS item to disk
534  */
535 void rss_save_item(rss_item *ri)
536 {
537
538         struct MD5Context md5context;
539         u_char rawdigest[MD5_DIGEST_LEN];
540         int i;
541         char utmsgid[SIZ];
542         struct cdbdata *cdbut;
543         struct UseTable ut;
544         struct CtdlMessage *msg;
545         struct recptypes *recp = NULL;
546         int msglen = 0;
547         StrBuf *Message;
548
549         recp = (struct recptypes *) malloc(sizeof(struct recptypes));
550         if (recp == NULL) return;
551         memset(recp, 0, sizeof(struct recptypes));
552         memset(&ut, 0, sizeof(struct UseTable));
553         recp->recp_room = strdup(ri->roomlist);
554         recp->num_room = num_tokens(ri->roomlist, '|');
555         recp->recptypes_magic = RECPTYPES_MAGIC;
556    
557         /* Construct a GUID to use in the S_USETABLE table.
558          * If one is not present in the item itself, make one up.
559          */
560         if (ri->guid != NULL) {
561                 StrBufSpaceToBlank(ri->guid);
562                 StrBufTrim(ri->guid);
563                 snprintf(utmsgid, sizeof utmsgid, "rss/%s", ChrPtr(ri->guid));
564         }
565         else {
566                 MD5Init(&md5context);
567                 if (ri->title != NULL) {
568                         MD5Update(&md5context, (const unsigned char*)ChrPtr(ri->title), StrLength(ri->title));
569                 }
570                 if (ri->link != NULL) {
571                         MD5Update(&md5context, (const unsigned char*)ChrPtr(ri->link), StrLength(ri->link));
572                 }
573                 MD5Final(rawdigest, &md5context);
574                 for (i=0; i<MD5_DIGEST_LEN; i++) {
575                         sprintf(&utmsgid[i*2], "%02X", (unsigned char) (rawdigest[i] & 0xff));
576                         utmsgid[i*2] = tolower(utmsgid[i*2]);
577                         utmsgid[(i*2)+1] = tolower(utmsgid[(i*2)+1]);
578                 }
579                 strcat(utmsgid, "_rss2ctdl");
580         }
581
582         /* Find out if we've already seen this item */
583
584         cdbut = cdb_fetch(CDB_USETABLE, utmsgid, strlen(utmsgid));
585 #ifndef DEBUG_RSS
586         if (cdbut != NULL) {
587                 /* Item has already been seen */
588                 syslog(LOG_DEBUG, "%s has already been seen\n", utmsgid);
589                 cdb_free(cdbut);
590
591                 /* rewrite the record anyway, to update the timestamp */
592                 strcpy(ut.ut_msgid, utmsgid);
593                 ut.ut_timestamp = time(NULL);
594                 cdb_store(CDB_USETABLE, utmsgid, strlen(utmsgid), &ut, sizeof(struct UseTable) );
595         }
596         else
597 #endif
598 {
599                 /* Item has not been seen, so save it. */
600                 syslog(LOG_DEBUG, "RSS: saving item...\n");
601                 if (ri->description == NULL) ri->description = NewStrBufPlain(HKEY(""));
602                 StrBufSpaceToBlank(ri->description);
603                 msg = malloc(sizeof(struct CtdlMessage));
604                 memset(msg, 0, sizeof(struct CtdlMessage));
605                 msg->cm_magic = CTDLMESSAGE_MAGIC;
606                 msg->cm_anon_type = MES_NORMAL;
607                 msg->cm_format_type = FMT_RFC822;
608
609                 if (ri->guid != NULL) {
610                         msg->cm_fields['E'] = strdup(ChrPtr(ri->guid));
611                 }
612
613                 if (ri->author_or_creator != NULL) {
614                         char *From;
615                         StrBuf *Encoded = NULL;
616                         int FromAt;
617                         
618                         From = html_to_ascii(ChrPtr(ri->author_or_creator),
619                                              StrLength(ri->author_or_creator), 
620                                              512, 0);
621                         StrBufPlain(ri->author_or_creator, From, -1);
622                         StrBufTrim(ri->author_or_creator);
623                         free(From);
624
625                         FromAt = strchr(ChrPtr(ri->author_or_creator), '@') != NULL;
626                         if (!FromAt && StrLength (ri->author_email) > 0)
627                         {
628                                 StrBufRFC2047encode(&Encoded, ri->author_or_creator);
629                                 msg->cm_fields['A'] = SmashStrBuf(&Encoded);
630                                 msg->cm_fields['P'] = SmashStrBuf(&ri->author_email);
631                         }
632                         else
633                         {
634                                 if (FromAt)
635                                 {
636                                         msg->cm_fields['A'] = SmashStrBuf(&ri->author_or_creator);
637                                         msg->cm_fields['P'] = strdup(msg->cm_fields['A']);
638                                 }
639                                 else 
640                                 {
641                                         StrBufRFC2047encode(&Encoded, ri->author_or_creator);
642                                         msg->cm_fields['A'] = SmashStrBuf(&Encoded);
643                                         msg->cm_fields['P'] = strdup("rss@localhost");
644                                 }
645                         }
646                 }
647                 else {
648                         msg->cm_fields['A'] = strdup("rss");
649                 }
650
651                 msg->cm_fields['N'] = strdup(NODENAME);
652                 if (ri->title != NULL) {
653                         long len;
654                         char *Sbj;
655                         StrBuf *Encoded, *QPEncoded;
656
657                         QPEncoded = NULL;
658                         StrBufSpaceToBlank(ri->title);
659                         len = StrLength(ri->title);
660                         Sbj = html_to_ascii(ChrPtr(ri->title), len, 512, 0);
661                         len = strlen(Sbj);
662                         if (Sbj[len - 1] == '\n')
663                         {
664                                 len --;
665                                 Sbj[len] = '\0';
666                         }
667                         Encoded = NewStrBufPlain(Sbj, len);
668                         free(Sbj);
669
670                         StrBufTrim(Encoded);
671                         StrBufRFC2047encode(&QPEncoded, Encoded);
672
673                         msg->cm_fields['U'] = SmashStrBuf(&QPEncoded);
674                         FreeStrBuf(&Encoded);
675                 }
676                 msg->cm_fields['T'] = malloc(64);
677                 snprintf(msg->cm_fields['T'], 64, "%ld", ri->pubdate);
678                 if (ri->channel_title != NULL) {
679                         if (StrLength(ri->channel_title) > 0) {
680                                 msg->cm_fields['O'] = strdup(ChrPtr(ri->channel_title));
681                         }
682                 }
683                 if (ri->link == NULL) 
684                         ri->link = NewStrBufPlain(HKEY(""));
685                 ExpandShortUrls(ri->description);
686                 msglen += 1024 + StrLength(ri->link) + StrLength(ri->description) ;
687
688                 Message = NewStrBufPlain(NULL, StrLength(ri->description));
689
690                 StrBufPlain(Message, HKEY(
691                          "Content-type: text/html; charset=\"UTF-8\"\r\n\r\n"
692                          "<html><body>\n"));
693
694                 StrBufAppendBuf(Message, ri->description, 0);
695                 StrBufAppendBufPlain(Message, HKEY("<br><br>\n"), 0);
696
697                 AppendLink(Message, ri->link, ri->linkTitle, NULL);
698                 AppendLink(Message, ri->reLink, ri->reLinkTitle, "Reply to this");
699                 StrBufAppendBufPlain(Message, HKEY("</body></html>\n"), 0);
700
701                 msg->cm_fields['M'] = SmashStrBuf(&Message);
702
703                 CtdlSubmitMsg(msg, recp, NULL, 0);
704                 CtdlFreeMessage(msg);
705
706                 /* write the uidl to the use table so we don't store this item again */
707                 strcpy(ut.ut_msgid, utmsgid);
708                 ut.ut_timestamp = time(NULL);
709                 cdb_store(CDB_USETABLE, utmsgid, strlen(utmsgid), &ut, sizeof(struct UseTable) );
710         }
711         free_recipients(recp);
712 }
713
714
715
716 /*
717  * Convert an RDF/RSS datestamp into a time_t
718  */
719 time_t rdf_parsedate(const char *p)
720 {
721         struct tm tm;
722         time_t t = 0;
723
724         if (!p) return 0L;
725         if (strlen(p) < 10) return 0L;
726
727         memset(&tm, 0, sizeof tm);
728
729         /*
730          * If the timestamp appears to be in W3C datetime format, try to
731          * parse it.  See also: http://www.w3.org/TR/NOTE-datetime
732          *
733          * This code, along with parsedate.c, is a potential candidate for
734          * moving into libcitadel.
735          */
736         if ( (p[4] == '-') && (p[7] == '-') ) {
737                 tm.tm_year = atoi(&p[0]) - 1900;
738                 tm.tm_mon = atoi(&p[5]) - 1;
739                 tm.tm_mday = atoi(&p[8]);
740                 if ( (p[10] == 'T') && (p[13] == ':') ) {
741                         tm.tm_hour = atoi(&p[11]);
742                         tm.tm_min = atoi(&p[14]);
743                 }
744                 return mktime(&tm);
745         }
746
747         /* hmm... try RFC822 date stamp format */
748
749         t = parsedate(p);
750         if (t > 0) return(t);
751
752         /* yeesh.  ok, just return the current date and time. */
753         return(time(NULL));
754 }
755
756 void flush_rss_item(rss_item *ri)
757 {
758         /* Initialize the feed item data structure */
759         FreeStrBuf(&ri->guid);
760         FreeStrBuf(&ri->title);
761         FreeStrBuf(&ri->link);
762         FreeStrBuf(&ri->linkTitle);
763         FreeStrBuf(&ri->reLink);
764         FreeStrBuf(&ri->reLinkTitle);
765         FreeStrBuf(&ri->description);
766         FreeStrBuf(&ri->channel_title);
767         FreeStrBuf(&ri->author_or_creator);
768         FreeStrBuf(&ri->author_url);
769         FreeStrBuf(&ri->author_email);
770 }
771
772 void rss_xml_start(void *data, const char *supplied_el, const char **attr)
773 {
774         rss_xml_handler *h;
775         rsscollection   *rssc = (rsscollection*) data;
776         rssnetcfg       *Cfg = rssc->Cfg;
777         rss_item        *ri = rssc->Item;
778         void            *pv;
779         const char      *pel;
780         char            *sep = NULL;
781
782         /* Axe the namespace, we don't care about it */
783 ///     syslog(LOG_EMERG, "RSS: supplied el %d: %s...\n", rssc->Cfg->ItemType, supplied_el);
784         pel = supplied_el;
785         while (sep = strchr(pel, ':'), sep) {
786                 pel = sep + 1;
787         }
788
789         if (pel != supplied_el)
790         {
791                 void *v;
792                 
793                 if (!GetHash(KnownNameSpaces, 
794                              supplied_el, 
795                              pel - supplied_el - 1,
796                              &v))
797                 {
798 #ifdef DEBUG_RSS
799                         syslog(LOG_EMERG, "RSS: START ignoring because of wrong namespace [%s] = [%s]\n", 
800                                       supplied_el);
801 #endif
802                         return;
803                 }
804         }
805
806         StrBufPlain(rssc->Key, pel, -1);
807         StrBufLowerCase(rssc->Key);
808         if (GetHash(StartHandlers, SKEY(rssc->Key), &pv))
809         {
810                 rssc->Current = h = (rss_xml_handler*) pv;
811
812                 if (((h->Flags & RSS_UNSET) != 0) && 
813                     (Cfg->ItemType == RSS_UNSET))
814                 {
815                         h->Handler(rssc->CData, ri, Cfg, attr);
816                 }
817                 else if (((h->Flags & RSS_RSS) != 0) &&
818                     (Cfg->ItemType == RSS_RSS))
819                 {
820                         h->Handler(rssc->CData, ri, Cfg, attr);
821                 }
822                 else if (((h->Flags & RSS_ATOM) != 0) &&
823                          (Cfg->ItemType == RSS_ATOM))
824                 {
825                         h->Handler(rssc->CData, ri, Cfg, attr);                 
826                 }
827 #ifdef DEBUG_RSS
828                 else 
829                         syslog(LOG_EMERG, "RSS: START unhandled: [%s] [%s]...\n", pel, supplied_el);
830 #endif
831         }
832 #ifdef DEBUG_RSS
833         else 
834                 syslog(LOG_EMERG, "RSS: START unhandled: [%s] [%s]...\n", pel,  supplied_el);
835 #endif
836 }
837
838 void rss_xml_end(void *data, const char *supplied_el)
839 {
840         rss_xml_handler *h;
841         rsscollection   *rssc = (rsscollection*) data;
842         rssnetcfg       *Cfg = rssc->Cfg;
843         rss_item        *ri = rssc->Item;
844         const char      *pel;
845         char            *sep = NULL;
846         void            *pv;
847
848         /* Axe the namespace, we don't care about it */
849         pel = supplied_el;
850         while (sep = strchr(pel, ':'), sep) {
851                 pel = sep + 1;
852         }
853 //      syslog(LOG_EMERG, "RSS: END %s...\n", el);
854         if (pel != supplied_el)
855         {
856                 void *v;
857                 
858                 if (!GetHash(KnownNameSpaces, 
859                              supplied_el, 
860                              pel - supplied_el - 1,
861                              &v))
862                 {
863 #ifdef DEBUG_RSS
864                         syslog(LOG_EMERG, "RSS: END ignoring because of wrong namespace [%s] = [%s]\n", 
865                                       supplied_el, ChrPtr(rssc->CData));
866 #endif
867                         FlushStrBuf(rssc->CData);
868                         return;
869                 }
870         }
871
872         StrBufPlain(rssc->Key, pel, -1);
873         StrBufLowerCase(rssc->Key);
874         if (GetHash(EndHandlers, SKEY(rssc->Key), &pv))
875         {
876                 h = (rss_xml_handler*) pv;
877
878                 if (((h->Flags & RSS_UNSET) != 0) && 
879                     (Cfg->ItemType == RSS_UNSET))
880                 {
881                         h->Handler(rssc->CData, ri, Cfg, NULL);
882                 }
883                 else if (((h->Flags & RSS_RSS) != 0) &&
884                     (Cfg->ItemType == RSS_RSS))
885                 {
886                         h->Handler(rssc->CData, ri, Cfg, NULL);
887                 }
888                 else if (((h->Flags & RSS_ATOM) != 0) &&
889                          (Cfg->ItemType == RSS_ATOM))
890                 {
891                         h->Handler(rssc->CData, ri, Cfg, NULL);
892                 }
893 #ifdef DEBUG_RSS
894                 else 
895                         syslog(LOG_EMERG, "RSS: END   unhandled: [%s]  [%s] = [%s]...\n", pel, supplied_el, ChrPtr(rssc->CData));
896 #endif
897         }
898 #ifdef DEBUG_RSS
899         else 
900                 syslog(LOG_EMERG, "RSS: END   unhandled: [%s]  [%s] = [%s]...\n", pel, supplied_el, ChrPtr(rssc->CData));
901 #endif
902         FlushStrBuf(rssc->CData);
903         rssc->Current = NULL;
904 }
905
906
907
908
909
910 void RSS_item_rss_start (StrBuf *CData, rss_item *ri, rssnetcfg *Cfg, const char** Attr)
911 {
912         syslog(LOG_DEBUG, "RSS: This is an RSS feed.\n");
913         Cfg->ItemType = RSS_RSS;
914 }
915
916 void RSS_item_rdf_start(StrBuf *CData, rss_item *ri, rssnetcfg *Cfg, const char** Attr)
917 {
918         syslog(LOG_DEBUG, "RSS: This is an RDF feed.\n");
919         Cfg->ItemType = RSS_RSS;
920 }
921
922 void ATOM_item_feed_start(StrBuf *CData, rss_item *ri, rssnetcfg *Cfg, const char** Attr)
923 {
924         syslog(LOG_DEBUG, "RSS: This is an ATOM feed.\n");
925         Cfg->ItemType = RSS_ATOM;
926 }
927
928
929 void RSS_item_item_start(StrBuf *CData, rss_item *ri, rssnetcfg *Cfg, const char** Attr)
930 {
931         ri->item_tag_nesting ++;
932         flush_rss_item(ri);
933 }
934
935 void ATOM_item_entry_start(StrBuf *CData, rss_item *ri, rssnetcfg *Cfg, const char** Attr)
936 {
937 /* Atom feed... */
938         ri->item_tag_nesting ++;
939         flush_rss_item(ri);
940 }
941
942 void ATOM_item_link_start (StrBuf *CData, rss_item *ri, rssnetcfg *Cfg, const char** Attr)
943 {
944         int i;
945         const char *pHref = NULL;
946         const char *pType = NULL;
947         const char *pRel = NULL;
948         const char *pTitle = NULL;
949
950         for (i = 0; Attr[i] != NULL; i+=2)
951         {
952                 if (!strcmp(Attr[i], "href"))
953                 {
954                         pHref = Attr[i+1];
955                 }
956                 else if (!strcmp(Attr[i], "rel"))
957                 {
958                         pRel = Attr[i+1];
959                 }
960                 else if (!strcmp(Attr[i], "type"))
961                 {
962                         pType = Attr[i+1];
963                 }
964                 else if (!strcmp(Attr[i], "title"))
965                 {
966                         pTitle = Attr[i+1];
967                 }
968         }
969         if (pHref == NULL)
970                 return; /* WHUT? Pointing... where? */
971         if ((pType != NULL) && !strcasecmp(pType, "application/atom+xml"))
972                 return; /* these just point to other rss resources, we're not interested in them. */
973         if (pRel != NULL)
974         {
975                 if (!strcasecmp (pRel, "replies"))
976                 {
977                         NewStrBufDupAppendFlush(&ri->reLink, NULL, pHref, -1);
978                         StrBufTrim(ri->link);
979                         NewStrBufDupAppendFlush(&ri->reLinkTitle, NULL, pTitle, -1);
980                 }
981                 else if (!strcasecmp(pRel, "alternate")) /* Alternative representation of this Item... */
982                 {
983                         NewStrBufDupAppendFlush(&ri->link, NULL, pHref, -1);
984                         StrBufTrim(ri->link);
985                         NewStrBufDupAppendFlush(&ri->linkTitle, NULL, pTitle, -1);
986
987                 }
988 #if 0 /* these are also defined, but dunno what to do with them.. */
989                 else if (!strcasecmp(pRel, "related"))
990                 {
991                 }
992                 else if (!strcasecmp(pRel, "self"))
993                 {
994                 }
995                 else if (!strcasecmp(pRel, "enclosure"))
996                 {/* this reference can get big, and is probably the full article... */
997                 }
998                 else if (!strcasecmp(pRel, "via"))
999                 {/* this article was provided via... */
1000                 }
1001 #endif
1002         }
1003         else if (StrLength(ri->link) == 0)
1004         {
1005                 NewStrBufDupAppendFlush(&ri->link, NULL, pHref, -1);
1006                 StrBufTrim(ri->link);
1007                 NewStrBufDupAppendFlush(&ri->linkTitle, NULL, pTitle, -1);
1008         }
1009 }
1010
1011
1012
1013
1014 void ATOMRSS_item_title_end(StrBuf *CData, rss_item *ri, rssnetcfg *Cfg, const char** Attr)
1015 {
1016         if ((ri->item_tag_nesting == 0) && (StrLength(CData) > 0)) {
1017                 NewStrBufDupAppendFlush(&ri->channel_title, CData, NULL, 0);
1018                 StrBufTrim(ri->channel_title);
1019         }
1020 }
1021
1022 void RSS_item_guid_end(StrBuf *CData, rss_item *ri, rssnetcfg *Cfg, const char** Attr)
1023 {
1024         if (StrLength(CData) > 0) {
1025                 NewStrBufDupAppendFlush(&ri->guid, CData, NULL, 0);
1026         }
1027 }
1028
1029 void ATOM_item_id_end(StrBuf *CData, rss_item *ri, rssnetcfg *Cfg, const char** Attr)
1030 {
1031         if (StrLength(CData) > 0) {
1032                 NewStrBufDupAppendFlush(&ri->guid, CData, NULL, 0);
1033         }
1034 }
1035
1036
1037 void RSS_item_link_end (StrBuf *CData, rss_item *ri, rssnetcfg *Cfg, const char** Attr)
1038 {
1039         if (StrLength(CData) > 0) {
1040                 NewStrBufDupAppendFlush(&ri->link, CData, NULL, 0);
1041                 StrBufTrim(ri->link);
1042         }
1043 }
1044 void RSS_item_relink_end(StrBuf *CData, rss_item *ri, rssnetcfg *Cfg, const char** Attr)
1045 {
1046         if (StrLength(CData) > 0) {
1047                 NewStrBufDupAppendFlush(&ri->reLink, CData, NULL, 0);
1048                 StrBufTrim(ri->reLink);
1049         }
1050 }
1051
1052 void RSSATOM_item_title_end (StrBuf *CData, rss_item *ri, rssnetcfg *Cfg, const char** Attr)
1053 {
1054         if (StrLength(CData) > 0) {
1055                 NewStrBufDupAppendFlush(&ri->title, CData, NULL, 0);
1056                 StrBufTrim(ri->title);
1057         }
1058 }
1059
1060 void ATOM_item_content_end (StrBuf *CData, rss_item *ri, rssnetcfg *Cfg, const char** Attr)
1061 {
1062         long olen = StrLength (ri->description);
1063         long clen = StrLength (CData);
1064         if (clen > 0) 
1065         {
1066                 if (olen == 0) {
1067                         NewStrBufDupAppendFlush(&ri->description, CData, NULL, 0);
1068                         StrBufTrim(ri->description);
1069                 }
1070                 else if (olen < clen) {
1071                         FlushStrBuf(ri->description);
1072                         NewStrBufDupAppendFlush(&ri->description, CData, NULL, 0);
1073                         StrBufTrim(ri->description);
1074                 }
1075         }
1076 }
1077 void ATOM_item_summary_end (StrBuf *CData, rss_item *ri, rssnetcfg *Cfg, const char** Attr)
1078 {
1079         /* this can contain an abstract of the article. but we don't want to verwrite a full document if we already have it. */
1080         if ((StrLength(CData) > 0) && (StrLength(ri->description) == 0))
1081         {
1082                 NewStrBufDupAppendFlush(&ri->description, CData, NULL, 0);
1083                 StrBufTrim(ri->description);
1084         }
1085 }
1086
1087 void RSS_item_description_end (StrBuf *CData, rss_item *ri, rssnetcfg *Cfg, const char** Attr)
1088 {
1089         long olen = StrLength (ri->description);
1090         long clen = StrLength (CData);
1091         if (clen > 0) 
1092         {
1093                 if (olen == 0) {
1094                         NewStrBufDupAppendFlush(&ri->description, CData, NULL, 0);
1095                         StrBufTrim(ri->description);
1096                 }
1097                 else if (olen < clen) {
1098                         FlushStrBuf(ri->description);
1099                         NewStrBufDupAppendFlush(&ri->description, CData, NULL, 0);
1100                         StrBufTrim(ri->description);
1101                 }
1102         }
1103 }
1104
1105 void ATOM_item_published_end (StrBuf *CData, rss_item *ri, rssnetcfg *Cfg, const char** Attr)
1106 {                 
1107         if (StrLength(CData) > 0) {
1108                 StrBufTrim(CData);
1109                 ri->pubdate = rdf_parsedate(ChrPtr(CData));
1110         }
1111 }
1112
1113 void ATOM_item_updated_end (StrBuf *CData, rss_item *ri, rssnetcfg *Cfg, const char** Attr)
1114 {
1115         if (StrLength(CData) > 0) {
1116                 StrBufTrim(CData);
1117                 ri->pubdate = rdf_parsedate(ChrPtr(CData));
1118         }
1119 }
1120
1121 void RSS_item_pubdate_end (StrBuf *CData, rss_item *ri, rssnetcfg *Cfg, const char** Attr)
1122 {
1123         if (StrLength(CData) > 0) {
1124                 StrBufTrim(CData);
1125                 ri->pubdate = rdf_parsedate(ChrPtr(CData));
1126         }
1127 }
1128
1129
1130 void RSS_item_date_end (StrBuf *CData, rss_item *ri, rssnetcfg *Cfg, const char** Attr)
1131 {
1132         if (StrLength(CData) > 0) {
1133                 StrBufTrim(CData);
1134                 ri->pubdate = rdf_parsedate(ChrPtr(CData));
1135         }
1136 }
1137
1138
1139
1140 void RSS_item_author_end(StrBuf *CData, rss_item *ri, rssnetcfg *Cfg, const char** Attr)
1141 {
1142         if (StrLength(CData) > 0) {
1143                 NewStrBufDupAppendFlush(&ri->author_or_creator, CData, NULL, 0);
1144                 StrBufTrim(ri->author_or_creator);
1145         }
1146 }
1147
1148
1149 void ATOM_item_name_end(StrBuf *CData, rss_item *ri, rssnetcfg *Cfg, const char** Attr)
1150 {
1151         if (StrLength(CData) > 0) {
1152                 NewStrBufDupAppendFlush(&ri->author_or_creator, CData, NULL, 0);
1153                 StrBufTrim(ri->author_or_creator);
1154         }
1155 }
1156
1157 void ATOM_item_email_end(StrBuf *CData, rss_item *ri, rssnetcfg *Cfg, const char** Attr)
1158 {
1159         if (StrLength(CData) > 0) {
1160                 NewStrBufDupAppendFlush(&ri->author_email, CData, NULL, 0);
1161                 StrBufTrim(ri->author_email);
1162         }
1163 }
1164
1165 void RSS_item_creator_end(StrBuf *CData, rss_item *ri, rssnetcfg *Cfg, const char** Attr)
1166 {
1167         if ((StrLength(CData) > 0) && 
1168             (StrLength(ri->author_or_creator) == 0))
1169         {
1170                 NewStrBufDupAppendFlush(&ri->author_or_creator, CData, NULL, 0);
1171                 StrBufTrim(ri->author_or_creator);
1172         }
1173 }
1174
1175
1176 void ATOM_item_uri_end(StrBuf *CData, rss_item *ri, rssnetcfg *Cfg, const char** Attr)
1177 {
1178         if (StrLength(CData) > 0) {
1179                 NewStrBufDupAppendFlush(&ri->author_url, CData, NULL, 0);
1180                 StrBufTrim(ri->author_url);
1181         }
1182 }
1183
1184 void RSS_item_item_end(StrBuf *CData, rss_item *ri, rssnetcfg *Cfg, const char** Attr)
1185 {
1186         --ri->item_tag_nesting;
1187         rss_save_item(ri);
1188 }
1189
1190
1191 void ATOM_item_entry_end(StrBuf *CData, rss_item *ri, rssnetcfg *Cfg, const char** Attr)
1192 {
1193         --ri->item_tag_nesting;
1194         rss_save_item(ri);
1195 }
1196
1197 void RSS_item_rss_end(StrBuf *CData, rss_item *ri, rssnetcfg *Cfg, const char** Attr)
1198 {
1199 //              syslog(LOG_DEBUG, "End of feed detected.  Closing parser.\n");
1200         ri->done_parsing = 1;
1201         
1202 }
1203 void RSS_item_rdf_end(StrBuf *CData, rss_item *ri, rssnetcfg *Cfg, const char** Attr)
1204 {
1205 //              syslog(LOG_DEBUG, "End of feed detected.  Closing parser.\n");
1206         ri->done_parsing = 1;
1207 }
1208
1209
1210 void RSSATOM_item_ignore(StrBuf *CData, rss_item *ri, rssnetcfg *Cfg, const char** Attr)
1211 {
1212 }
1213
1214
1215
1216 /*
1217  * This callback stores up the data which appears in between tags.
1218  */
1219 void rss_xml_cdata_start(void *data) 
1220 {
1221         rsscollection *rssc = (rsscollection*) data;
1222
1223         FlushStrBuf(rssc->CData);
1224 }
1225
1226 void rss_xml_cdata_end(void *data) 
1227 {
1228 }
1229 void rss_xml_chardata(void *data, const XML_Char *s, int len) 
1230 {
1231         rsscollection *rssc = (rsscollection*) data;
1232
1233         StrBufAppendBufPlain (rssc->CData, s, len, 0);
1234 }
1235
1236 /*
1237  * Callback function for passing libcurl's output to expat for parsing
1238  */
1239 size_t rss_libcurl_callback(void *ptr, size_t size, size_t nmemb, void *stream)
1240 {
1241         XML_Parse((XML_Parser)stream, ptr, (size * nmemb), 0);
1242         return (size*nmemb);
1243 }
1244
1245
1246
1247 /*
1248  * Begin a feed parse
1249  */
1250 void rss_do_fetching(rssnetcfg *Cfg) {
1251         rsscollection rssc;
1252         rss_item ri;
1253         XML_Parser xp = NULL;
1254         StrBuf *Answer;
1255
1256         CURL *curl;
1257         CURLcode res;
1258         char errmsg[1024] = "";
1259         char *ptr;
1260         const char *at;
1261         long len;
1262
1263         time_t now;
1264
1265         now = time(NULL);
1266
1267         if ((Cfg->next_poll != 0) && (now < Cfg->next_poll))
1268                 return;
1269         memset(&ri, 0, sizeof(rss_item));
1270         rssc.Item = &ri;
1271         rssc.Cfg = Cfg;
1272
1273         syslog(LOG_DEBUG, "Fetching RSS feed <%s>\n", Cfg->url);
1274
1275         curl = curl_easy_init();
1276         if (!curl) {
1277                 syslog(LOG_ALERT, "Unable to initialize libcurl.\n");
1278                 return;
1279         }
1280         Answer = NewStrBufPlain(NULL, SIZ);
1281
1282         curl_easy_setopt(curl, CURLOPT_URL, Cfg->url);
1283         curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 0);
1284         curl_easy_setopt(curl, CURLOPT_SSL_VERIFYHOST, 0);
1285         curl_easy_setopt(curl, CURLOPT_WRITEDATA, Answer);
1286 //      curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, rss_libcurl_callback);
1287         curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, CurlFillStrBuf_callback);
1288         curl_easy_setopt(curl, CURLOPT_ERRORBUFFER, errmsg);
1289         curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1);
1290 #ifdef CURLOPT_HTTP_CONTENT_DECODING
1291         curl_easy_setopt(curl, CURLOPT_HTTP_CONTENT_DECODING, 1);
1292         curl_easy_setopt(curl, CURLOPT_ENCODING, "");
1293 #endif
1294         curl_easy_setopt(curl, CURLOPT_USERAGENT, CITADEL);
1295         curl_easy_setopt(curl, CURLOPT_TIMEOUT, 180);           /* die after 180 seconds */
1296         if (
1297                 (!IsEmptyStr(config.c_ip_addr))
1298                 && (strcmp(config.c_ip_addr, "*"))
1299                 && (strcmp(config.c_ip_addr, "::"))
1300                 && (strcmp(config.c_ip_addr, "0.0.0.0"))
1301         ) {
1302                 curl_easy_setopt(curl, CURLOPT_INTERFACE, config.c_ip_addr);
1303         }
1304
1305         if (server_shutting_down)
1306         {
1307                 curl_easy_cleanup(curl);
1308                 return;
1309         }
1310         
1311         if (server_shutting_down)
1312                 goto shutdown ;
1313
1314         res = curl_easy_perform(curl);
1315         if (res) {
1316                 syslog(LOG_ALERT, "libcurl error %d: %s\n", res, errmsg);
1317         }
1318
1319         if (server_shutting_down)
1320                 goto shutdown ;
1321
1322
1323
1324
1325         memset(&ri, 0, sizeof(rss_item));
1326         ri.roomlist = Cfg->rooms;
1327         rssc.CData = NewStrBufPlain(NULL, SIZ);
1328         rssc.Key = NewStrBuf();
1329         at = NULL;
1330         StrBufSipLine(rssc.Key, Answer, &at);
1331         ptr = NULL;
1332
1333 #define encoding "encoding=\""
1334         ptr = strstr(ChrPtr(rssc.Key), encoding);
1335         if (ptr != NULL)
1336         {
1337                 char *pche;
1338
1339                 ptr += sizeof (encoding) - 1;
1340                 pche = strchr(ptr, '"');
1341                 if (pche != NULL)
1342                         StrBufCutAt(rssc.Key, -1, pche);
1343                 else 
1344                         ptr = "UTF-8";
1345         }
1346         else
1347                 ptr = "UTF-8";
1348
1349
1350         xp = XML_ParserCreateNS(ptr, ':');
1351         if (!xp) {
1352                 syslog(LOG_ALERT, "Cannot create XML parser!\n");
1353                 goto shutdown;
1354         }
1355         FlushStrBuf(rssc.Key);
1356 //#ifdef HAVE_ICONV
1357 #if 0
1358         XML_SetUnknownEncodingHandler(xp,
1359                                       handle_unknown_xml_encoding,
1360                                       &rssc);
1361 #endif
1362 //#endif
1363         XML_SetElementHandler(xp, rss_xml_start, rss_xml_end);
1364         XML_SetCharacterDataHandler(xp, rss_xml_chardata);
1365         XML_SetUserData(xp, &rssc);
1366         XML_SetCdataSectionHandler(xp,
1367                                    rss_xml_cdata_start,
1368                                    rss_xml_cdata_end);
1369
1370
1371         len = StrLength(Answer);
1372         ptr = SmashStrBuf(&Answer);
1373         XML_Parse(xp, ptr, len, 0);
1374         free (ptr);
1375         if (ri.done_parsing == 0)
1376                 XML_Parse(xp, "", 0, 1);
1377
1378
1379         syslog(LOG_ALERT, "RSS: XML Status [%s] \n", 
1380                       XML_ErrorString(
1381                               XML_GetErrorCode(xp)));
1382
1383 shutdown:
1384         FreeStrBuf(&Answer);
1385         curl_easy_cleanup(curl);
1386         XML_ParserFree(xp);
1387
1388         flush_rss_item(&ri);
1389         FreeStrBuf(&rssc.CData);
1390         FreeStrBuf(&rssc.Key);
1391
1392         Cfg->next_poll = time(NULL) + config.c_net_freq; 
1393 }
1394
1395
1396 /*
1397  * Scan a room's netconfig to determine whether it is requesting any RSS feeds
1398  */
1399 void rssclient_scan_room(struct ctdlroom *qrbuf, void *data)
1400 {
1401         char filename[PATH_MAX];
1402         char buf[1024];
1403         char instr[32];
1404         FILE *fp;
1405         char feedurl[256];
1406         rssnetcfg *rncptr = NULL;
1407         rssnetcfg *use_this_rncptr = NULL;
1408         int len = 0;
1409         char *ptr = NULL;
1410
1411         assoc_file_name(filename, sizeof filename, qrbuf, ctdl_netcfg_dir);
1412
1413         if (server_shutting_down)
1414                 return;
1415                 
1416         /* Only do net processing for rooms that have netconfigs */
1417         fp = fopen(filename, "r");
1418         if (fp == NULL) {
1419                 return;
1420         }
1421
1422         while (fgets(buf, sizeof buf, fp) != NULL && !server_shutting_down) {
1423                 buf[strlen(buf)-1] = 0;
1424
1425                 extract_token(instr, buf, 0, '|', sizeof instr);
1426                 if (!strcasecmp(instr, "rssclient")) {
1427
1428                         use_this_rncptr = NULL;
1429
1430                         extract_token(feedurl, buf, 1, '|', sizeof feedurl);
1431
1432                         /* If any other rooms have requested the same feed, then we will just add this
1433                          * room to the target list for that client request.
1434                          */
1435                         for (rncptr=rnclist; rncptr!=NULL; rncptr=rncptr->next) {
1436                                 if (!strcmp(rncptr->url, feedurl)) {
1437                                         use_this_rncptr = rncptr;
1438                                 }
1439                         }
1440
1441                         /* Otherwise create a new client request */
1442                         if (use_this_rncptr == NULL) {
1443                                 rncptr = (rssnetcfg *) malloc(sizeof(rssnetcfg));
1444                                 memset(rncptr, 0, sizeof(rssnetcfg));
1445                                 rncptr->ItemType = RSS_UNSET;
1446                                 if (rncptr != NULL) {
1447                                         rncptr->next = rnclist;
1448                                         safestrncpy(rncptr->url, feedurl, sizeof rncptr->url);
1449                                         rncptr->rooms = NULL;
1450                                         rnclist = rncptr;
1451                                         use_this_rncptr = rncptr;
1452                                 }
1453                         }
1454
1455                         /* Add the room name to the request */
1456                         if (use_this_rncptr != NULL) {
1457                                 if (use_this_rncptr->rooms == NULL) {
1458                                         rncptr->rooms = strdup(qrbuf->QRname);
1459                                 }
1460                                 else {
1461                                         len = strlen(use_this_rncptr->rooms) + strlen(qrbuf->QRname) + 5;
1462                                         ptr = realloc(use_this_rncptr->rooms, len);
1463                                         if (ptr != NULL) {
1464                                                 strcat(ptr, "|");
1465                                                 strcat(ptr, qrbuf->QRname);
1466                                                 use_this_rncptr->rooms = ptr;
1467                                         }
1468                                 }
1469                         }
1470                 }
1471
1472         }
1473
1474         fclose(fp);
1475
1476 }
1477
1478 /*
1479  * Scan for rooms that have RSS client requests configured
1480  */
1481 void rssclient_scan(void) {
1482         static time_t last_run = 0L;
1483         static int doing_rssclient = 0;
1484         rssnetcfg *rptr = NULL;
1485
1486         /* Run no more than once every 15 minutes. */
1487         if ((time(NULL) - last_run) < 900) {
1488                 return;
1489         }
1490
1491         /*
1492          * This is a simple concurrency check to make sure only one rssclient run
1493          * is done at a time.  We could do this with a mutex, but since we
1494          * don't really require extremely fine granularity here, we'll do it
1495          * with a static variable instead.
1496          */
1497         if (doing_rssclient) return;
1498         doing_rssclient = 1;
1499
1500         syslog(LOG_DEBUG, "rssclient started\n");
1501         CtdlForEachRoom(rssclient_scan_room, NULL);
1502
1503         while (rnclist != NULL && !server_shutting_down) {
1504                 rss_do_fetching(rnclist);
1505                 rptr = rnclist;
1506                 rnclist = rnclist->next;
1507                 if (rptr->rooms != NULL) free(rptr->rooms);
1508                 free(rptr);
1509         }
1510
1511         syslog(LOG_DEBUG, "rssclient ended\n");
1512         last_run = time(NULL);
1513         doing_rssclient = 0;
1514         return;
1515 }
1516
1517 void LoadUrlShorteners(void)
1518 {
1519         int i = 0;
1520         int fd;
1521         const char *POS = NULL;
1522         const char *Err = NULL;
1523         StrBuf *Content, *Line;
1524
1525
1526         UrlShorteners = NewHash(0, Flathash);
1527
1528         fd = open(file_citadel_urlshorteners, 0);
1529
1530         if (fd != 0)
1531         {
1532                 Content = NewStrBufPlain(NULL, SIZ);
1533                 Line = NewStrBuf();
1534                 while (POS != StrBufNOTNULL)
1535                 {
1536                         StrBufTCP_read_buffered_line_fast (Line, Content, &POS, &fd, 1, 1, &Err);
1537                         StrBufTrim(Line);
1538                         if ((*ChrPtr(Line) != '#') && (StrLength(Line) > 0))
1539                         {
1540                                 Put(UrlShorteners, IKEY(i), Line, HFreeStrBuf);
1541                                 i++;
1542                                 Line = NewStrBuf();
1543                         }
1544                         else
1545                                 FlushStrBuf(Line);
1546                         if (POS == NULL)
1547                                 POS = StrBufNOTNULL;
1548                 }
1549                 FreeStrBuf(&Line);
1550                 FreeStrBuf(&Content);
1551         }
1552         close(fd);
1553 }
1554
1555 void rss_cleanup(void)
1556 {
1557         DeleteHash(&StartHandlers);
1558         DeleteHash(&EndHandlers);
1559         DeleteHash(&UrlShorteners);
1560         DeleteHash(&KnownNameSpaces);
1561 }
1562
1563 CTDL_MODULE_INIT(rssclient)
1564 {
1565         if (threading)
1566         {
1567                 syslog(LOG_INFO, "%s\n", curl_version());
1568                 CtdlRegisterSessionHook(rssclient_scan, EVT_TIMER);
1569         }
1570         else 
1571         {
1572                 LoadUrlShorteners ();
1573
1574                 StartHandlers = NewHash(1, NULL);
1575                 EndHandlers = NewHash(1, NULL);
1576
1577                 AddRSSStartHandler(RSS_item_rss_start,     RSS_UNSET, HKEY("rss"));
1578                 AddRSSStartHandler(RSS_item_rdf_start,     RSS_UNSET, HKEY("rdf"));
1579                 AddRSSStartHandler(ATOM_item_feed_start,    RSS_UNSET, HKEY("feed"));
1580                 AddRSSStartHandler(RSS_item_item_start,    RSS_RSS, HKEY("item"));
1581                 AddRSSStartHandler(ATOM_item_entry_start,  RSS_ATOM, HKEY("entry"));
1582                 AddRSSStartHandler(ATOM_item_link_start,   RSS_ATOM, HKEY("link"));
1583
1584                 AddRSSEndHandler(ATOMRSS_item_title_end,   RSS_ATOM|RSS_RSS|RSS_REQUIRE_BUF, HKEY("title"));
1585                 AddRSSEndHandler(RSS_item_guid_end,        RSS_RSS|RSS_REQUIRE_BUF, HKEY("guid"));
1586                 AddRSSEndHandler(ATOM_item_id_end,         RSS_ATOM|RSS_REQUIRE_BUF, HKEY("id"));
1587                 AddRSSEndHandler(RSS_item_link_end,        RSS_RSS|RSS_REQUIRE_BUF, HKEY("link"));
1588 #if 0 
1589 // hm, rss to the comments of that blog, might be interesting in future, but... 
1590                 AddRSSEndHandler(RSS_item_relink_end,      RSS_RSS|RSS_REQUIRE_BUF, HKEY("commentrss"));
1591 // comment count...
1592                 AddRSSEndHandler(RSS_item_relink_end,      RSS_RSS|RSS_REQUIRE_BUF, HKEY("comments"));
1593 #endif
1594                 AddRSSEndHandler(RSSATOM_item_title_end,   RSS_ATOM|RSS_RSS|RSS_REQUIRE_BUF, HKEY("title"));
1595                 AddRSSEndHandler(ATOM_item_content_end,    RSS_ATOM|RSS_REQUIRE_BUF, HKEY("content"));
1596                 AddRSSEndHandler(RSS_item_description_end, RSS_RSS|RSS_ATOM|RSS_REQUIRE_BUF, HKEY("encoded"));
1597                 AddRSSEndHandler(ATOM_item_summary_end,    RSS_ATOM|RSS_REQUIRE_BUF, HKEY("summary"));
1598                 AddRSSEndHandler(RSS_item_description_end, RSS_RSS|RSS_REQUIRE_BUF, HKEY("description"));
1599                 AddRSSEndHandler(ATOM_item_published_end,  RSS_ATOM|RSS_REQUIRE_BUF, HKEY("published"));
1600                 AddRSSEndHandler(ATOM_item_updated_end,    RSS_ATOM|RSS_REQUIRE_BUF, HKEY("updated"));
1601                 AddRSSEndHandler(RSS_item_pubdate_end,     RSS_RSS|RSS_REQUIRE_BUF, HKEY("pubdate"));
1602                 AddRSSEndHandler(RSS_item_date_end,        RSS_RSS|RSS_REQUIRE_BUF, HKEY("date"));
1603                 AddRSSEndHandler(RSS_item_author_end,      RSS_RSS|RSS_REQUIRE_BUF, HKEY("author"));
1604                 AddRSSEndHandler(RSS_item_creator_end,     RSS_RSS|RSS_REQUIRE_BUF, HKEY("creator"));
1605 /* <author> */
1606                 AddRSSEndHandler(ATOM_item_email_end,      RSS_ATOM|RSS_REQUIRE_BUF, HKEY("email"));
1607                 AddRSSEndHandler(ATOM_item_name_end,       RSS_ATOM|RSS_REQUIRE_BUF, HKEY("name"));
1608                 AddRSSEndHandler(ATOM_item_uri_end,        RSS_ATOM|RSS_REQUIRE_BUF, HKEY("uri"));
1609 /* </author> */
1610                 AddRSSEndHandler(RSS_item_item_end,        RSS_RSS, HKEY("item"));
1611                 AddRSSEndHandler(RSS_item_rss_end,         RSS_RSS, HKEY("rss"));
1612                 AddRSSEndHandler(RSS_item_rdf_end,         RSS_RSS, HKEY("rdf"));
1613                 AddRSSEndHandler(ATOM_item_entry_end,      RSS_ATOM, HKEY("entry"));
1614
1615
1616 /* at the start of atoms: <seq> <li>link to resource</li></seq> ignore them. */
1617                 AddRSSStartHandler(RSSATOM_item_ignore,      RSS_RSS|RSS_ATOM, HKEY("seq"));
1618                 AddRSSEndHandler  (RSSATOM_item_ignore,      RSS_RSS|RSS_ATOM, HKEY("seq"));
1619                 AddRSSStartHandler(RSSATOM_item_ignore,      RSS_RSS|RSS_ATOM, HKEY("li"));
1620                 AddRSSEndHandler  (RSSATOM_item_ignore,      RSS_RSS|RSS_ATOM, HKEY("li"));
1621
1622 /* links to other feed generators... */
1623                 AddRSSStartHandler(RSSATOM_item_ignore,      RSS_RSS|RSS_ATOM, HKEY("feedflare"));
1624                 AddRSSEndHandler  (RSSATOM_item_ignore,      RSS_RSS|RSS_ATOM, HKEY("feedflare"));
1625                 AddRSSStartHandler(RSSATOM_item_ignore,      RSS_RSS|RSS_ATOM, HKEY("browserfriendly"));
1626                 AddRSSEndHandler  (RSSATOM_item_ignore,      RSS_RSS|RSS_ATOM, HKEY("browserfriendly"));
1627
1628                 KnownNameSpaces = NewHash(1, NULL);
1629                 Put(KnownNameSpaces, HKEY("http://a9.com/-/spec/opensearch/1.1/"), NULL, reference_free_handler);
1630                 Put(KnownNameSpaces, HKEY("http://a9.com/-/spec/opensearchrss/1.0/"), NULL, reference_free_handler);
1631                 Put(KnownNameSpaces, HKEY("http://backend.userland.com/creativeCommonsRssModule"), NULL, reference_free_handler);
1632                 Put(KnownNameSpaces, HKEY("http://purl.org/atom/ns#"), NULL, reference_free_handler);
1633                 Put(KnownNameSpaces, HKEY("http://purl.org/dc/elements/1.1/"), NULL, reference_free_handler);
1634                 Put(KnownNameSpaces, HKEY("http://purl.org/rss/1.0/"), NULL, reference_free_handler);
1635                 Put(KnownNameSpaces, HKEY("http://purl.org/rss/1.0/modules/content/"), NULL, reference_free_handler);
1636                 Put(KnownNameSpaces, HKEY("http://purl.org/rss/1.0/modules/slash/"), NULL, reference_free_handler);
1637                 Put(KnownNameSpaces, HKEY("http://purl.org/rss/1.0/modules/syndication/"), NULL, reference_free_handler);
1638                 Put(KnownNameSpaces, HKEY("http://purl.org/rss/1.0/"), NULL, reference_free_handler);
1639                 Put(KnownNameSpaces, HKEY("http://purl.org/syndication/thread/1.0"), NULL, reference_free_handler);
1640                 Put(KnownNameSpaces, HKEY("http://rssnamespace.org/feedburner/ext/1.0"), NULL, reference_free_handler);
1641                 Put(KnownNameSpaces, HKEY("http://schemas.google.com/g/2005"), NULL, reference_free_handler);
1642                 Put(KnownNameSpaces, HKEY("http://webns.net/mvcb/"), NULL, reference_free_handler);
1643                 Put(KnownNameSpaces, HKEY("http://web.resource.org/cc/"), NULL, reference_free_handler);
1644                 Put(KnownNameSpaces, HKEY("http://wellformedweb.org/CommentAPI/"), NULL, reference_free_handler);
1645                 Put(KnownNameSpaces, HKEY("http://www.georss.org/georss"), NULL, reference_free_handler);
1646                 Put(KnownNameSpaces, HKEY("http://www.w3.org/1999/xhtml"), NULL, reference_free_handler);
1647                 Put(KnownNameSpaces, HKEY("http://www.w3.org/1999/02/22-rdf-syntax-ns#"), NULL, reference_free_handler);
1648                 Put(KnownNameSpaces, HKEY("http://www.w3.org/1999/02/22-rdf-syntax-ns#"), NULL, reference_free_handler);
1649                 Put(KnownNameSpaces, HKEY("http://www.w3.org/2003/01/geo/wgs84_pos#"), NULL, reference_free_handler);
1650                 Put(KnownNameSpaces, HKEY("http://www.w3.org/2005/Atom"), NULL, reference_free_handler);
1651                 Put(KnownNameSpaces, HKEY("urn:flickr:"), NULL, reference_free_handler);
1652 #if 0
1653                 /* we don't like these namespaces because of they shadow our usefull parameters. */
1654                 Put(KnownNameSpaces, HKEY("http://search.yahoo.com/mrss/"), NULL, reference_free_handler);
1655 #endif
1656                 CtdlRegisterCleanupHook(rss_cleanup);
1657         }
1658         return "rssclient";
1659 }