Did away with lprintf all together now its called CtdlLogPrintf()
[citadel.git] / citadel / modules / rssclient / serv_rssclient.c
1 /*
2  * $Id: serv_rssclient.c 5652 2007-10-29 20:14:48Z ajc $
3  *
4  * Bring external RSS feeds into rooms.
5  *
6  */
7
8 #include <stdlib.h>
9 #include <unistd.h>
10 #include <stdio.h>
11
12 #if TIME_WITH_SYS_TIME
13 # include <sys/time.h>
14 # include <time.h>
15 #else
16 # if HAVE_SYS_TIME_H
17 #  include <sys/time.h>
18 # else
19 #  include <time.h>
20 # endif
21 #endif
22
23 #include <ctype.h>
24 #include <string.h>
25 #include <errno.h>
26 #include <sys/types.h>
27 #include <sys/stat.h>
28 #include <expat.h>
29 #include <libcitadel.h>
30 #include "citadel.h"
31 #include "server.h"
32 #include "citserver.h"
33 #include "support.h"
34 #include "config.h"
35 #include "threads.h"
36 #include "room_ops.h"
37 #include "ctdl_module.h"
38 #include "clientsocket.h"
39 #include "msgbase.h"
40 #include "parsedate.h"
41 #include "database.h"
42 #include "citadel_dirs.h"
43 #include "md5.h"
44
45
46 struct rssnetcfg {
47         struct rssnetcfg *next;
48         char url[256];
49         char *rooms;
50 };
51
52 struct rss_item {
53         char *chardata;
54         int chardata_len;
55         char *roomlist;
56         int done_parsing;
57         char *guid;
58         char *title;
59         char *link;
60         char *description;
61         time_t pubdate;
62         char channel_title[256];
63         int item_tag_nesting;
64 };
65
66 struct rssnetcfg *rnclist = NULL;
67
68
69 /*
70  * Commit a fetched and parsed RSS item to disk
71  */
72 void rss_save_item(struct rss_item *ri) {
73
74         struct MD5Context md5context;
75         u_char rawdigest[MD5_DIGEST_LEN];
76         int i;
77         char utmsgid[SIZ];
78         struct cdbdata *cdbut;
79         struct UseTable ut;
80         struct CtdlMessage *msg;
81         struct recptypes *recp = NULL;
82         int msglen = 0;
83
84         recp = (struct recptypes *) malloc(sizeof(struct recptypes));
85         if (recp == NULL) return;
86         memset(recp, 0, sizeof(struct recptypes));
87         recp->recp_room = strdup(ri->roomlist);
88         recp->num_room = num_tokens(ri->roomlist, '|');
89         recp->recptypes_magic = RECPTYPES_MAGIC;
90    
91         /* Construct a GUID to use in the S_USETABLE table.
92          * If one is not present in the item itself, make one up.
93          */
94         if (ri->guid != NULL) {
95                 snprintf(utmsgid, sizeof utmsgid, "rss/%s", ri->guid);
96         }
97         else {
98                 MD5Init(&md5context);
99                 if (ri->title != NULL) {
100                         MD5Update(&md5context, ri->title, strlen(ri->title));
101                 }
102                 if (ri->link != NULL) {
103                         MD5Update(&md5context, ri->link, strlen(ri->link));
104                 }
105                 MD5Final(rawdigest, &md5context);
106                 for (i=0; i<MD5_DIGEST_LEN; i++) {
107                         sprintf(&utmsgid[i*2], "%02X", (unsigned char) (rawdigest[i] & 0xff));
108                         utmsgid[i*2] = tolower(utmsgid[i*2]);
109                         utmsgid[(i*2)+1] = tolower(utmsgid[(i*2)+1]);
110                 }
111                 strcat(utmsgid, "_rss2ctdl");
112         }
113
114         /* Find out if we've already seen this item */
115         cdbut = cdb_fetch(CDB_USETABLE, utmsgid, strlen(utmsgid));
116         if (cdbut != NULL) {
117                 /* Item has already been seen */
118                 CtdlLogPrintf(CTDL_DEBUG, "%s has already been seen\n", utmsgid);
119                 cdb_free(cdbut);
120
121                 /* rewrite the record anyway, to update the timestamp */
122                 strcpy(ut.ut_msgid, utmsgid);
123                 ut.ut_timestamp = time(NULL);
124                 cdb_store(CDB_USETABLE, utmsgid, strlen(utmsgid), &ut, sizeof(struct UseTable) );
125         }
126         else {
127                 /* Item has not been seen, so save it. */
128
129                 if (ri->description == NULL) ri->description = strdup("");
130                 for (i=strlen(ri->description); i>=0; --i) {
131                         if (isspace(ri->description[i])) {
132                                 ri->description[i] = ' ';
133                         }
134                 }
135
136                 msg = malloc(sizeof(struct CtdlMessage));
137                 memset(msg, 0, sizeof(struct CtdlMessage));
138                 msg->cm_magic = CTDLMESSAGE_MAGIC;
139                 msg->cm_anon_type = MES_NORMAL;
140                 msg->cm_format_type = FMT_RFC822;
141                 msg->cm_fields['A'] = strdup("rss");
142                 msg->cm_fields['N'] = strdup(NODENAME);
143                 msg->cm_fields['U'] = strdup(ri->title);
144                 msg->cm_fields['T'] = malloc(64);
145                 snprintf(msg->cm_fields['T'], 64, "%ld", ri->pubdate);
146                 if (!IsEmptyStr(ri->channel_title)) {
147                         msg->cm_fields['O'] = strdup(ri->channel_title);
148                 }
149
150                 msglen = 1024 + strlen(ri->link) + strlen(ri->description) ;
151                 msg->cm_fields['M'] = malloc(msglen);
152                 snprintf(msg->cm_fields['M'], msglen,
153                         "Content-type: text/html\r\n\r\n"
154                         "<html><body>\n"
155                         "%s<br><br>\n"
156                         "<a href=\"%s\">%s</a>\n"
157                         "</body></html>\n"
158                         ,
159                         ri->description,
160                         ri->link, ri->link
161                 );
162
163                 CtdlSubmitMsg(msg, recp, NULL);
164                 CtdlFreeMessage(msg);
165
166                 /* write the uidl to the use table so we don't store this item again */
167                 strcpy(ut.ut_msgid, utmsgid);
168                 ut.ut_timestamp = time(NULL);
169                 cdb_store(CDB_USETABLE, utmsgid, strlen(utmsgid), &ut, sizeof(struct UseTable) );
170         }
171         free_recipients(recp);
172 }
173
174
175
176 /*
177  * Convert an RDF/RSS datestamp into a time_t
178  */
179 time_t rdf_parsedate(char *p)
180 {
181         struct tm tm;
182         time_t t = 0;
183
184         if (!p) return 0L;
185         if (strlen(p) < 10) return 0L;
186
187         memset(&tm, 0, sizeof tm);
188
189         /* YYYY-MM-DDTHH:MM format...
190          */
191         if ( (p[4] == '-') && (p[7] == '-') ) {
192                 tm.tm_year = atoi(&p[0]) - 1900;
193                 tm.tm_mon = atoi(&p[5]) - 1;
194                 tm.tm_mday = atoi(&p[8]);
195                 if ( (p[10] == 'T') && (p[13] == ':') ) {
196                         tm.tm_hour = atoi(&p[11]);
197                         tm.tm_min = atoi(&p[14]);
198                 }
199                 return mktime(&tm);
200         }
201
202         /* hmm... try RFC822 date stamp format */
203
204         t = parsedate(p);
205         if (t > 0) return(t);
206
207         /* yeesh.  ok, just return the current date and time. */
208         return(time(NULL));
209 }
210
211
212
213 void rss_xml_start(void *data, const char *supplied_el, const char **attr) {
214         struct rss_item *ri = (struct rss_item *) data;
215         char el[256];
216         char *sep = NULL;
217
218         /* Axe the namespace, we don't care about it */
219         safestrncpy(el, supplied_el, sizeof el);
220         while (sep = strchr(el, ':'), sep) {
221                 strcpy(el, ++sep);
222         }
223
224         if (!strcasecmp(el, "item")) {
225                 ++ri->item_tag_nesting;
226
227                 /* Initialize the feed item data structure */
228                 if (ri->guid != NULL) free(ri->guid);
229                 ri->guid = NULL;
230                 if (ri->title != NULL) free(ri->title);
231                 ri->title = NULL;
232                 if (ri->link != NULL) free(ri->link);
233                 ri->link = NULL;
234                 if (ri->description != NULL) free(ri->description);
235                 ri->description = NULL;
236
237                 /* Throw away any existing character data */
238                 if (ri->chardata_len > 0) {
239                         free(ri->chardata);
240                         ri->chardata = 0;
241                         ri->chardata_len = 0;
242                 }
243         }
244
245
246
247 }
248
249 void rss_xml_end(void *data, const char *supplied_el) {
250         struct rss_item *ri = (struct rss_item *) data;
251         char el[256];
252         char *sep = NULL;
253
254         /* Axe the namespace, we don't care about it */
255         safestrncpy(el, supplied_el, sizeof el);
256         while (sep = strchr(el, ':'), sep) {
257                 strcpy(el, ++sep);
258         }
259
260         if ( (!strcasecmp(el, "title")) && (ri->item_tag_nesting == 0) && (ri->chardata != NULL) ) {
261                 safestrncpy(ri->channel_title, ri->chardata, sizeof ri->channel_title);
262                 striplt(ri->channel_title);
263         }
264
265         if ( (!strcasecmp(el, "guid")) && (ri->chardata != NULL) ) {
266                 if (ri->guid != NULL) free(ri->guid);
267                 striplt(ri->chardata);
268                 ri->guid = strdup(ri->chardata);
269         }
270
271         if ( (!strcasecmp(el, "title")) && (ri->chardata != NULL) ) {
272                 if (ri->title != NULL) free(ri->title);
273                 striplt(ri->chardata);
274                 ri->title = strdup(ri->chardata);
275         }
276
277         if ( (!strcasecmp(el, "link")) && (ri->chardata != NULL) ) {
278                 if (ri->link != NULL) free(ri->link);
279                 striplt(ri->chardata);
280                 ri->link = strdup(ri->chardata);
281         }
282
283         if ( (!strcasecmp(el, "description")) && (ri->chardata != NULL) ) {
284                 if (ri->description != NULL) free(ri->description);
285                 ri->description = strdup(ri->chardata);
286         }
287
288         if ( ((!strcasecmp(el, "pubdate")) || (!strcasecmp(el, "date"))) && (ri->chardata != NULL) ) {
289                 striplt(ri->chardata);
290                 ri->pubdate = rdf_parsedate(ri->chardata);
291         }
292
293         if (!strcasecmp(el, "item")) {
294                 --ri->item_tag_nesting;
295                 rss_save_item(ri);
296         }
297
298         if ( (!strcasecmp(el, "rss")) || (!strcasecmp(el, "rdf")) ) {
299                 CtdlLogPrintf(CTDL_DEBUG, "End of feed detected.  Closing parser.\n");
300                 ri->done_parsing = 1;
301         }
302
303         if (ri->chardata_len > 0) {
304                 free(ri->chardata);
305                 ri->chardata = 0;
306                 ri->chardata_len = 0;
307         }
308
309 }
310
311
312 /*
313  * This callback stores up the data which appears in between tags.
314  */
315 void rss_xml_chardata(void *data, const XML_Char *s, int len) {
316         struct rss_item *ri = (struct rss_item *) data;
317         int old_len;
318         int new_len;
319         char *new_buffer;
320
321         old_len = ri->chardata_len;
322         new_len = old_len + len;
323         new_buffer = realloc(ri->chardata, new_len + 1);
324         if (new_buffer != NULL) {
325                 memcpy(&new_buffer[old_len], s, len);
326                 new_buffer[new_len] = 0;
327                 ri->chardata = new_buffer;
328                 ri->chardata_len = new_len;
329         }
330 }
331
332
333
334 /* 
335  * Parse a URL into host, port number, and resource identifier.
336  */
337 int parse_url(char *url, char *hostname, int *port, char *identifier)
338 {
339         char protocol[1024];
340         char scratch[1024];
341         char *ptr = NULL;
342         char *nptr = NULL;
343         
344         strcpy(scratch, url);
345         ptr = (char *)strchr(scratch, ':');
346         if (!ptr) {
347                 return(1);      /* no protocol specified */
348         }
349
350         strcpy(ptr, "");
351         strcpy(protocol, scratch);
352         if (strcmp(protocol, "http")) {
353                 return(2);      /* not HTTP */
354         }
355
356         strcpy(scratch, url);
357         ptr = (char *) strstr(scratch, "//");
358         if (!ptr) {
359                 return(3);      /* no server specified */
360         }
361         ptr += 2;
362
363         strcpy(hostname, ptr);
364         nptr = (char *)strchr(ptr, ':');
365         if (!nptr) {
366                 *port = 80;     /* default */
367                 nptr = (char *)strchr(hostname, '/');
368         }
369         else {
370                 sscanf(nptr, ":%d", port);
371                 nptr = (char *)strchr(hostname, ':');
372         }
373
374         if (nptr) {
375                 *nptr = '\0';
376         }
377
378         nptr = (char *)strchr(ptr, '/');
379         
380         if (!nptr) {
381                 return(4);      /* no url specified */
382         }
383         
384         strcpy(identifier, nptr);
385         return(0);
386 }
387
388
389 /*
390  * Begin a feed parse
391  */
392 void rss_do_fetching(char *url, char *rooms) {
393         char buf[1024];
394         char rsshost[1024];
395         int rssport = 80;
396         char rssurl[1024];
397         struct rss_item ri;
398         XML_Parser xp;
399         int sock = (-1);
400         int got_bytes = (-1);
401         int redirect_count = 0;
402
403         /* Parse the URL */
404         if (parse_url(url, rsshost, &rssport, rssurl) != 0) {
405                 CtdlLogPrintf(CTDL_ALERT, "Invalid URL: %s\n", url);
406         }
407         
408         if (CtdlThreadCheckStop())
409                 return;
410
411         xp = XML_ParserCreateNS("UTF-8", ':');
412         if (!xp) {
413                 CtdlLogPrintf(CTDL_ALERT, "Cannot create XML parser!\n");
414                 return;
415         }
416
417         memset(&ri, 0, sizeof(struct rss_item));
418         ri.roomlist = rooms;
419         XML_SetElementHandler(xp, rss_xml_start, rss_xml_end);
420         XML_SetCharacterDataHandler(xp, rss_xml_chardata);
421         XML_SetUserData(xp, &ri);
422
423         if (CtdlThreadCheckStop())
424         {
425                 XML_ParserFree(xp);
426                 return;
427         }
428         
429 retry:  CtdlLogPrintf(CTDL_NOTICE, "Connecting to <%s>\n", rsshost);
430         sprintf(buf, "%d", rssport);
431         sock = sock_connect(rsshost, buf, "tcp");
432         if (sock >= 0) {
433                 CtdlLogPrintf(CTDL_DEBUG, "Connected!\n");
434
435                 if (CtdlThreadCheckStop())
436                         goto shutdown ;
437                         
438                 snprintf(buf, sizeof buf, "GET %s HTTP/1.0", rssurl);
439                 CtdlLogPrintf(CTDL_DEBUG, "<%s\n", buf);
440                 sock_puts(sock, buf);
441
442                 if (CtdlThreadCheckStop())
443                         goto shutdown ;
444                         
445                 snprintf(buf, sizeof buf, "Host: %s", rsshost);
446                 CtdlLogPrintf(CTDL_DEBUG, "<%s\n", buf);
447                 sock_puts(sock, buf);
448
449                 if (CtdlThreadCheckStop())
450                         goto shutdown ;
451                         
452                 snprintf(buf, sizeof buf, "User-Agent: %s", CITADEL);
453                 CtdlLogPrintf(CTDL_DEBUG, "<%s\n", buf);
454                 sock_puts(sock, buf);
455
456                 if (CtdlThreadCheckStop())
457                         goto shutdown ;
458                         
459                 snprintf(buf, sizeof buf, "Accept: */*");
460                 CtdlLogPrintf(CTDL_DEBUG, "<%s\n", buf);
461                 sock_puts(sock, buf);
462
463                 if (CtdlThreadCheckStop())
464                         goto shutdown ;
465                         
466                 sock_puts(sock, "");
467
468                 if (CtdlThreadCheckStop())
469                         goto shutdown ;
470                         
471                 if (sock_getln(sock, buf, sizeof buf) >= 0) {
472                         CtdlLogPrintf(CTDL_DEBUG, ">%s\n", buf);
473                         remove_token(buf, 0, ' ');
474
475                         /* 200 OK */
476                         if (buf[0] == '2') {
477
478                                 while (got_bytes = sock_getln(sock, buf, sizeof buf),
479                                       (got_bytes >= 0 && (strcmp(buf, "")) && (strcmp(buf, "\r"))) ) {
480                                         if (CtdlThreadCheckStop())
481                                                 goto shutdown ;
482                                         /* discard headers */
483                                 }
484
485                                 while (got_bytes = sock_read(sock, buf, sizeof buf, 0),
486                                       ((got_bytes>=0) && (ri.done_parsing == 0)) ) {
487                                         if (CtdlThreadCheckStop())
488                                                 goto shutdown ;
489                                         XML_Parse(xp, buf, got_bytes, 0);
490                                 }
491                                 if (ri.done_parsing == 0) XML_Parse(xp, "", 0, 1);
492                         }
493
494                         /* 30X redirect */
495                         else if ( (!strncmp(buf, "30", 2)) && (redirect_count < 16) ) {
496                                 while (got_bytes = sock_getln(sock, buf, sizeof buf),
497                                       (got_bytes >= 0 && (strcmp(buf, "")) && (strcmp(buf, "\r"))) ) {
498                                         if (CtdlThreadCheckStop())
499                                                 goto shutdown ;
500                                         if (!strncasecmp(buf, "Location:", 9)) {
501                                                 ++redirect_count;
502                                                 strcpy(buf, &buf[9]);
503                                                 striplt(buf);
504                                                 if (parse_url(buf, rsshost, &rssport, rssurl) == 0) {
505                                                         sock_close(sock);
506                                                         goto retry;
507                                                 }
508                                                 else {
509                                                         CtdlLogPrintf(CTDL_ALERT, "Invalid URL: %s\n", buf);
510                                                 }
511                                         }
512                                 }
513                         }
514
515                 }
516 shutdown:
517                 sock_close(sock);
518         }
519         else {
520                 CtdlLogPrintf(CTDL_ERR, "Could not connect: %s\n", strerror(errno));
521         }
522
523         XML_ParserFree(xp);
524
525         /* Free the feed item data structure */
526         if (ri.guid != NULL) free(ri.guid);
527         ri.guid = NULL;
528         if (ri.title != NULL) free(ri.title);
529         ri.title = NULL;
530         if (ri.link != NULL) free(ri.link);
531         ri.link = NULL;
532         if (ri.description != NULL) free(ri.description);
533         ri.description = NULL;
534         if (ri.chardata_len > 0) {
535                 free(ri.chardata);
536                 ri.chardata = 0;
537                 ri.chardata_len = 0;
538         }
539 }
540
541
542 /*
543  * Scan a room's netconfig to determine whether it is requesting any RSS feeds
544  */
545 void rssclient_scan_room(struct ctdlroom *qrbuf, void *data)
546 {
547         char filename[PATH_MAX];
548         char buf[1024];
549         char instr[32];
550         FILE *fp;
551         char feedurl[256];
552         struct rssnetcfg *rncptr = NULL;
553         struct rssnetcfg *use_this_rncptr = NULL;
554         int len = 0;
555         char *ptr = NULL;
556
557         assoc_file_name(filename, sizeof filename, qrbuf, ctdl_netcfg_dir);
558
559         if (CtdlThreadCheckStop())
560                 return;
561                 
562         /* Only do net processing for rooms that have netconfigs */
563         fp = fopen(filename, "r");
564         if (fp == NULL) {
565                 return;
566         }
567
568         while (fgets(buf, sizeof buf, fp) != NULL && !CtdlThreadCheckStop()) {
569                 buf[strlen(buf)-1] = 0;
570
571                 extract_token(instr, buf, 0, '|', sizeof instr);
572                 if (!strcasecmp(instr, "rssclient")) {
573
574                         use_this_rncptr = NULL;
575
576                         extract_token(feedurl, buf, 1, '|', sizeof feedurl);
577
578                         /* If any other rooms have requested the same feed, then we will just add this
579                          * room to the target list for that client request.
580                          */
581                         for (rncptr=rnclist; rncptr!=NULL; rncptr=rncptr->next) {
582                                 if (!strcmp(rncptr->url, feedurl)) {
583                                         use_this_rncptr = rncptr;
584                                 }
585                         }
586
587                         /* Otherwise create a new client request */
588                         if (use_this_rncptr == NULL) {
589                                 rncptr = (struct rssnetcfg *) malloc(sizeof(struct rssnetcfg));
590                                 if (rncptr != NULL) {
591                                         rncptr->next = rnclist;
592                                         safestrncpy(rncptr->url, feedurl, sizeof rncptr->url);
593                                         rncptr->rooms = NULL;
594                                         rnclist = rncptr;
595                                         use_this_rncptr = rncptr;
596                                 }
597                         }
598
599                         /* Add the room name to the request */
600                         if (use_this_rncptr != NULL) {
601                                 if (use_this_rncptr->rooms == NULL) {
602                                         rncptr->rooms = strdup(qrbuf->QRname);
603                                 }
604                                 else {
605                                         len = strlen(use_this_rncptr->rooms) + strlen(qrbuf->QRname) + 5;
606                                         ptr = realloc(use_this_rncptr->rooms, len);
607                                         if (ptr != NULL) {
608                                                 strcat(ptr, "|");
609                                                 strcat(ptr, qrbuf->QRname);
610                                                 use_this_rncptr->rooms = ptr;
611                                         }
612                                 }
613                         }
614                 }
615
616         }
617
618         fclose(fp);
619
620 }
621
622 /*
623  * Scan for rooms that have RSS client requests configured
624  */
625 void *rssclient_scan(void *args) {
626         static time_t last_run = 0L;
627         static int doing_rssclient = 0;
628         struct rssnetcfg *rptr = NULL;
629         struct CitContext rssclientCC;
630
631         /* Give this thread its own private CitContext */
632         memset(&rssclientCC, 0, sizeof(struct CitContext));
633         rssclientCC.internal_pgm = 1;
634         rssclientCC.cs_pid = 0;
635         pthread_setspecific(MyConKey, (void *)&rssclientCC );
636
637         CtdlThreadAllocTSD();
638
639         /*
640          * This is a simple concurrency check to make sure only one rssclient run
641          * is done at a time.  We could do this with a mutex, but since we
642          * don't really require extremely fine granularity here, we'll do it
643          * with a static variable instead.
644          */
645         if (doing_rssclient) return NULL;
646         doing_rssclient = 1;
647
648         CtdlLogPrintf(CTDL_DEBUG, "rssclient started\n");
649         ForEachRoom(rssclient_scan_room, NULL);
650
651         while (rnclist != NULL && !CtdlThreadCheckStop()) {
652                 rss_do_fetching(rnclist->url, rnclist->rooms);
653                 rptr = rnclist;
654                 rnclist = rnclist->next;
655                 if (rptr->rooms != NULL) free(rptr->rooms);
656                 free(rptr);
657         }
658
659         CtdlLogPrintf(CTDL_DEBUG, "rssclient ended\n");
660         last_run = time(NULL);
661         doing_rssclient = 0;
662         if (!CtdlThreadCheckStop())
663                 CtdlThreadSchedule ("RSS Client", CTDLTHREAD_BIGSTACK, rssclient_scan, NULL, last_run + config.c_net_freq);
664         else
665                 CtdlLogPrintf(CTDL_DEBUG, "rssclient: Task STOPPED.\n");
666         return NULL;
667 }
668
669
670 CTDL_MODULE_INIT(rssclient)
671 {
672         if (threading)
673         {
674                 CtdlThreadSchedule ("RSS Client", CTDLTHREAD_BIGSTACK, rssclient_scan, NULL, 0);
675         }
676         /* return our Subversion id for the Log */
677         return "$Id: serv_rssclient.c 5652 2007-10-29 20:14:48Z ajc $";
678 }