fix pointer indirections so we now can successfully fetch RSS feeds again.
[citadel.git] / citadel / modules / urldeshortener / serv_expand_shorter_urls.c
1 /*
2  *
3  * Copyright (c) 1998-2009 by the citadel.org team
4  *
5  *  This program is free software; you can redistribute it and/or modify
6  *  it under the terms of the GNU General Public License as published by
7  *  the Free Software Foundation; either version 3 of the License, or
8  *  (at your option) any later version.
9  *
10  *  This program is distributed in the hope that it will be useful,
11  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
12  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
13  *  GNU General Public License for more details.
14  *
15  *  You should have received a copy of the GNU General Public License
16  *  along with this program; if not, write to the Free Software
17  *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
18  */
19
20 #include "sysdep.h"
21 #include <stdlib.h>
22 #include <unistd.h>
23 #include <stdio.h>
24 #include <termios.h>
25 #include <fcntl.h>
26 #include <signal.h>
27 #include <pwd.h>
28 #include <errno.h>
29 #include <sys/types.h>
30 #include <syslog.h>
31
32 #if TIME_WITH_SYS_TIME
33 # include <sys/time.h>
34 # include <time.h>
35 #else
36 # if HAVE_SYS_TIME_H
37 #  include <sys/time.h>
38 # else
39 #  include <time.h>
40 # endif
41 #endif
42 #include <sys/wait.h>
43 #include <ctype.h>
44 #include <string.h>
45 #include <limits.h>
46 #include <sys/socket.h>
47 #include <netinet/in.h>
48 #include <arpa/inet.h>
49 #include <assert.h>
50
51 #include <libcitadel.h>
52 #include "citadel.h"
53 #include "server.h"
54 #include "citserver.h"
55 #include "support.h"
56 #include "config.h"
57 #include "control.h"
58 #include "user_ops.h"
59 #include "database.h"
60 #include "msgbase.h"
61 #include "internet_addressing.h"
62 #include "genstamp.h"
63 #include "domain.h"
64 #include "ctdl_module.h"
65 #include "locate_host.h"
66 #include "citadel_dirs.h"
67
68 #include "event_client.h"
69
70 HashList *UrlShorteners = NULL;
71
72 size_t GetLocationString( void *ptr, size_t size, size_t nmemb, void *userdata)
73 {
74 #define LOCATION "location"
75         if (strncasecmp((char*)ptr, LOCATION, sizeof(LOCATION) - 1) == 0)
76         {
77                 StrBuf *pURL = (StrBuf*) userdata;
78                 char *pch = (char*) ptr;
79                 char *pche;
80                 
81                 pche = pch + (size * nmemb);
82                 pch += sizeof(LOCATION);
83                 
84                 while (isspace(*pch) || (*pch == ':'))
85                         pch ++;
86
87                 while (isspace(*pche) || (*pche == '\0'))
88                         pche--;
89                 
90                 FlushStrBuf(pURL);
91                 StrBufPlain(pURL, pch, pche - pch + 1); 
92         }
93         return size * nmemb;
94 }
95
96
97 eNextState LookupUrlResult(AsyncIO *IO)
98 {
99         return eTerminateConnection; /// /TODO
100 }
101
102 int LookupUrl(StrBuf *ShorterUrlStr)
103 {
104         CURLcode sta;
105         int rc = 0;
106         CURL *chnd;
107         AsyncIO *IO;
108
109
110         IO = (AsyncIO*) malloc(sizeof(AsyncIO));
111         memset(IO, 0, sizeof(AsyncIO));
112         IO->CitContext = CloneContext(CC);
113
114         ParseURL(&IO->ConnectMe, ShorterUrlStr, 80);
115         CurlPrepareURL(IO->ConnectMe);
116         if (! evcurl_init(IO, 
117 //                        Ctx, 
118                           NULL,
119                           "Citadel RSS ShorterURL Expander",
120                           LookupUrlResult))
121         {
122                 CtdlLogPrintf(CTDL_ALERT, "Unable to initialize libcurl.\n");
123                 goto shutdown;
124         }
125         chnd = IO->HttpReq.chnd;
126
127         OPT(SSL_VERIFYPEER, 0);
128         OPT(SSL_VERIFYHOST, 0);
129         OPT(FOLLOWLOCATION, 10);
130 #ifdef CURLOPT_HTTP_CONTENT_DECODING
131         OPT(HTTP_CONTENT_DECODING, 1);
132         OPT(ENCODING, "");
133 #endif 
134         OPT(HEADERFUNCTION , GetLocationString);
135         OPT(WRITEHEADER, ShorterUrlStr);
136
137
138         if (CtdlThreadCheckStop())
139                 goto shutdown ;
140
141         evcurl_handle_start(IO);
142
143 shutdown:
144
145         return rc;
146
147 }
148
149
150
151 void CrawlMessageForShorterUrls(HashList *pUrls, StrBuf *Message)
152 {
153         int nHits = 0;
154         void *pv;
155         int nShorter = 0;
156         const char *pch;
157         const char *pUrl;
158         ConstStr *pCUrl;
159
160         while (GetHash(UrlShorteners, IKEY(nShorter), &pv))
161         {
162                 nShorter++;
163                 pch = ChrPtr(Message);
164                 pUrl = strstr(pch, ChrPtr((StrBuf*)pv));
165                 while ((pUrl != NULL) && (nHits < 99))
166                 {
167                         pCUrl = malloc(sizeof(ConstStr));
168
169                         pCUrl->Key = pUrl;
170                         pch = pUrl + StrLength((StrBuf*)pv);
171                         while (isalnum(*pch)||(*pch == '-')||(*pch == '/'))
172                                 pch++;
173                         pCUrl->len = pch - pCUrl->Key;
174
175                         Put(pUrls, IKEY(nHits), pCUrl, NULL);
176                         nHits ++;
177                         pUrl = strstr(pch, ChrPtr((StrBuf*)pv));
178                 }
179         }
180 }
181
182 int SortConstStrByPosition(const void *Item1, const void *Item2)
183 {
184         const ConstStr *p1, *p2;
185         p1 = (const ConstStr*) Item1;
186         p2 = (const ConstStr*) Item2;
187         if (p1->Key == p2->Key)
188                 return 0;
189         if (p1->Key > p2->Key)
190                 return 1;
191         return -1;
192 }
193
194 void ExpandShortUrls(StrBuf *Message)
195 {
196         StrBuf *Shadow;
197         HashList *pUrls;
198         ConstStr *pCUrl;
199         const char *pch;
200         const char *pche;
201
202         /* we just suspect URL shorteners to be inside of feeds from twitter
203          * or other short content messages, so don't crawl through real blogs.
204          */
205         if (StrLength(Message) > 500)
206                 return;
207
208         pUrls = NewHash(1, Flathash);
209         CrawlMessageForShorterUrls(pUrls, Message);
210
211         if (GetCount(pUrls) > 0)
212         {
213                 StrBuf *ShorterUrlStr;
214                 HashPos *Pos;
215                 const char *Key;
216                 void *pv;
217                 long len;
218
219                 Shadow = NewStrBufPlain(NULL, StrLength(Message));
220                 SortByPayload (pUrls, SortConstStrByPosition);
221
222                 ShorterUrlStr = NewStrBufPlain(NULL, StrLength(Message));
223
224                 pch = ChrPtr(Message);
225                 pche = pch + StrLength(Message);
226                 Pos = GetNewHashPos(pUrls, 1);
227                 while (GetNextHashPos(pUrls, Pos, &len, &Key, &pv))
228                 {
229                         pCUrl = (ConstStr*) pv;
230
231                         if (pch != pCUrl->Key)
232                                 StrBufAppendBufPlain(Shadow, pch, pCUrl->Key - pch, 0);
233                         
234                         StrBufPlain(ShorterUrlStr, CKEY(*pCUrl));
235                         if (LookupUrl(ShorterUrlStr))
236                         {
237                                 StrBufAppendBufPlain(Shadow, HKEY("<a href=\""), 0);
238                                 StrBufAppendBuf(Shadow, ShorterUrlStr, 0);
239                                 StrBufAppendBufPlain(Shadow, HKEY("\">"), 0);
240                                 StrBufAppendBuf(Shadow, ShorterUrlStr, 0);
241                                 StrBufAppendBufPlain(Shadow, HKEY("["), 0);
242                                 StrBufAppendBufPlain(Shadow, pCUrl->Key, pCUrl->len, 0);
243                                 StrBufAppendBufPlain(Shadow, HKEY("]</a>"), 0);
244                         }
245                         else
246                         {
247                                 StrBufAppendBufPlain(Shadow, HKEY("<a href=\""), 0);
248                                 StrBufAppendBufPlain(Shadow, pCUrl->Key, pCUrl->len, 0);
249                                 StrBufAppendBufPlain(Shadow, HKEY("\">"), 0);
250                                 StrBufAppendBufPlain(Shadow, pCUrl->Key, pCUrl->len, 0);
251                                 StrBufAppendBufPlain(Shadow, HKEY("</a>"), 0);
252                         }
253                         pch = pCUrl->Key + pCUrl->len + 1;
254
255                 }
256                 if (pch < pche)
257                         StrBufAppendBufPlain(Shadow, pch, pche - pch, 0);
258                 FlushStrBuf(Message);
259                 StrBufAppendBuf(Message, Shadow, 0);
260
261                 FreeStrBuf(&ShorterUrlStr);
262                 FreeStrBuf(&Shadow);
263                 DeleteHashPos(&Pos);
264         }
265
266         DeleteHash(&pUrls);
267 }
268
269 void LoadUrlShorteners(void)
270 {
271         int i = 0;
272         int fd;
273         const char *POS = NULL;
274         const char *Err = NULL;
275         StrBuf *Content, *Line;
276
277
278         UrlShorteners = NewHash(0, Flathash);
279
280         fd = open(file_citadel_urlshorteners, 0);
281
282         if (fd != 0)
283         {
284                 Content = NewStrBufPlain(NULL, SIZ);
285                 Line = NewStrBuf();
286                 while (POS != StrBufNOTNULL)
287                 {
288                         StrBufTCP_read_buffered_line_fast (Line, Content, &POS, &fd, 1, 1, &Err);
289                         StrBufTrim(Line);
290                         if ((*ChrPtr(Line) != '#') && (StrLength(Line) > 0))
291                         {
292                                 Put(UrlShorteners, IKEY(i), Line, HFreeStrBuf);
293                                 i++;
294                                 Line = NewStrBuf();
295                         }
296                         else
297                                 FlushStrBuf(Line);
298                         if (POS == NULL)
299                                 POS = StrBufNOTNULL;
300                 }
301                 FreeStrBuf(&Line);
302                 FreeStrBuf(&Content);
303         }
304         close(fd);
305 }
306
307 void shorter_url_cleanup(void)
308 {
309         DeleteHash(&UrlShorteners);
310 }
311
312
313 CTDL_MODULE_INIT(urldeshortener)
314 {
315         if (threading)
316         {
317                 CtdlLogPrintf(CTDL_INFO, "%s\n", curl_version());
318         }
319         else 
320         {
321                 LoadUrlShorteners ();
322                 CtdlRegisterCleanupHook(shorter_url_cleanup);
323         }
324         return "UrlShortener";
325 }