citadel/modules/urldeshortener/serv_expand_shorter_urls.c

   1 /*
   2  *
   3  * Copyright (c) 1998-2009 by the citadel.org team
   4  *
   5  *  This program is free software; you can redistribute it and/or modify
   6  *  it under the terms of the GNU General Public License as published by
   7  *  the Free Software Foundation; either version 3 of the License, or
   8  *  (at your option) any later version.
   9  *
  10  *  This program is distributed in the hope that it will be useful,
  11  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
  12  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  13  *  GNU General Public License for more details.
  14  *
  15  *  You should have received a copy of the GNU General Public License
  16  *  along with this program; if not, write to the Free Software
  17  *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  18  */
  19
  20 #include "sysdep.h"
  21 #include <stdlib.h>
  22 #include <unistd.h>
  23 #include <stdio.h>
  24 #include <termios.h>
  25 #include <fcntl.h>
  26 #include <signal.h>
  27 #include <pwd.h>
  28 #include <errno.h>
  29 #include <sys/types.h>
  30 #include <syslog.h>
  31
  32 #if TIME_WITH_SYS_TIME
  33 # include <sys/time.h>
  34 # include <time.h>
  35 #else
  36 # if HAVE_SYS_TIME_H
  37 #  include <sys/time.h>
  38 # else
  39 #  include <time.h>
  40 # endif
  41 #endif
  42 #include <sys/wait.h>
  43 #include <ctype.h>
  44 #include <string.h>
  45 #include <limits.h>
  46 #include <sys/socket.h>
  47 #include <netinet/in.h>
  48 #include <arpa/inet.h>
  49 #include <assert.h>
  50
  51 #include <libcitadel.h>
  52 #include "citadel.h"
  53 #include "server.h"
  54 #include "citserver.h"
  55 #include "support.h"
  56 #include "config.h"
  57 #include "control.h"
  58 #include "user_ops.h"
  59 #include "database.h"
  60 #include "msgbase.h"
  61 #include "internet_addressing.h"
  62 #include "genstamp.h"
  63 #include "domain.h"
  64 #include "ctdl_module.h"
  65 #include "locate_host.h"
  66 #include "citadel_dirs.h"
  67
  68 #include "event_client.h"
  69
  70 HashList *UrlShorteners = NULL;
  71
  72 size_t GetLocationString( void *ptr, size_t size, size_t nmemb, void *userdata)
  73 {
  74 #define LOCATION "location"
  75         if (strncasecmp((char*)ptr, LOCATION, sizeof(LOCATION) - 1) == 0)
  76         {
  77                 StrBuf *pURL = (StrBuf*) userdata;
  78                 char *pch = (char*) ptr;
  79                 char *pche;
  80
  81                 pche = pch + (size * nmemb);
  82                 pch += sizeof(LOCATION);
  83
  84                 while (isspace(*pch) || (*pch == ':'))
  85                         pch ++;
  86
  87                 while (isspace(*pche) || (*pche == '\0'))
  88                         pche--;
  89
  90                 FlushStrBuf(pURL);
  91                 StrBufPlain(pURL, pch, pche - pch + 1);
  92         }
  93         return size * nmemb;
  94 }
  95
  96
  97 eNextState LookupUrlResult(AsyncIO *IO)
  98 {
  99         return eTerminateConnection; /// /TODO
 100 }
 101
 102 int LookupUrl(StrBuf *ShorterUrlStr)
 103 {
 104         CURLcode sta;
 105         int rc = 0;
 106         CURL *chnd;
 107         AsyncIO *IO;
 108
 109
 110         IO = (AsyncIO*) malloc(sizeof(AsyncIO));
 111         memset(IO, 0, sizeof(AsyncIO));
 112         IO->CitContext = CloneContext(CC);
 113
 114         ParseURL(&IO->ConnectMe, ShorterUrlStr, 80);
 115         CurlPrepareURL(IO->ConnectMe);
 116         if (! evcurl_init(IO,
 117 //                        Ctx,
 118                           NULL,
 119                           "Citadel RSS ShorterURL Expander",
 120                           LookupUrlResult))
 121         {
 122                 CtdlLogPrintf(CTDL_ALERT, "Unable to initialize libcurl.\n");
 123                 goto shutdown;
 124         }
 125         chnd = IO->HttpReq.chnd;
 126
 127         OPT(SSL_VERIFYPEER, 0);
 128         OPT(SSL_VERIFYHOST, 0);
 129         OPT(FOLLOWLOCATION, 10);
 130 #ifdef CURLOPT_HTTP_CONTENT_DECODING
 131         OPT(HTTP_CONTENT_DECODING, 1);
 132         OPT(ENCODING, "");
 133 #endif
 134         OPT(HEADERFUNCTION , GetLocationString);
 135         OPT(WRITEHEADER, ShorterUrlStr);
 136
 137
 138         if (CtdlThreadCheckStop())
 139                 goto shutdown ;
 140
 141         evcurl_handle_start(IO);
 142
 143 shutdown:
 144
 145         return rc;
 146
 147 }
 148
 149
 150
 151 void CrawlMessageForShorterUrls(HashList *pUrls, StrBuf *Message)
 152 {
 153         int nHits = 0;
 154         void *pv;
 155         int nShorter = 0;
 156         const char *pch;
 157         const char *pUrl;
 158         ConstStr *pCUrl;
 159
 160         while (GetHash(UrlShorteners, IKEY(nShorter), &pv))
 161         {
 162                 nShorter++;
 163                 pch = ChrPtr(Message);
 164                 pUrl = strstr(pch, ChrPtr((StrBuf*)pv));
 165                 while ((pUrl != NULL) && (nHits < 99))
 166                 {
 167                         pCUrl = malloc(sizeof(ConstStr));
 168
 169                         pCUrl->Key = pUrl;
 170                         pch = pUrl + StrLength((StrBuf*)pv);
 171                         while (isalnum(*pch)||(*pch == '-')||(*pch == '/'))
 172                                 pch++;
 173                         pCUrl->len = pch - pCUrl->Key;
 174
 175                         Put(pUrls, IKEY(nHits), pCUrl, NULL);
 176                         nHits ++;
 177                         pUrl = strstr(pch, ChrPtr((StrBuf*)pv));
 178                 }
 179         }
 180 }
 181
 182 int SortConstStrByPosition(const void *Item1, const void *Item2)
 183 {
 184         const ConstStr *p1, *p2;
 185         p1 = (const ConstStr*) Item1;
 186         p2 = (const ConstStr*) Item2;
 187         if (p1->Key == p2->Key)
 188                 return 0;
 189         if (p1->Key > p2->Key)
 190                 return 1;
 191         return -1;
 192 }
 193
 194 HashList GetShorterUrls(StrBuf Message)
 195 {
 196         HashList *pUrls;
 197         /* we just suspect URL shorteners to be inside of feeds from twitter
 198          * or other short content messages, so don't crawl through real blogs.
 199          */
 200         if (StrLength(Message) > 500)
 201                 return NULL;
 202
 203         pUrls = NewHash(1, Flathash);
 204         CrawlMessageForShorterUrls(pUrls, Message);
 205
 206         if (GetCount(pUrls) > 0)
 207                 return pURLs;
 208         else
 209                 return NULL;
 210
 211 }
 212
 213 void ExpandShortUrls(StrBuf *Message, HashList *pUrls, int Callback)
 214 {
 215         StrBuf *Shadow;
 216         ConstStr *pCUrl;
 217         const char *pch;
 218         const char *pche;
 219
 220         StrBuf *ShorterUrlStr;
 221         HashPos *Pos;
 222         const char *Key;
 223         void *pv;
 224         long len;
 225
 226         Shadow = NewStrBufPlain(NULL, StrLength(Message));
 227         SortByPayload (pUrls, SortConstStrByPosition);
 228
 229         ShorterUrlStr = NewStrBufPlain(NULL, StrLength(Message));
 230
 231         pch = ChrPtr(Message);
 232         pche = pch + StrLength(Message);
 233         Pos = GetNewHashPos(pUrls, 1);
 234         while (GetNextHashPos(pUrls, Pos, &len, &Key, &pv))
 235         {
 236                 pCUrl = (ConstStr*) pv;
 237
 238                 if (pch != pCUrl->Key)
 239                         StrBufAppendBufPlain(Shadow, pch, pCUrl->Key - pch, 0);
 240
 241                 StrBufPlain(ShorterUrlStr, CKEY(*pCUrl));
 242                 if (LookupUrl(ShorterUrlStr))
 243                 {
 244                         StrBufAppendBufPlain(Shadow, HKEY("<a href=\""), 0);
 245                         StrBufAppendBuf(Shadow, ShorterUrlStr, 0);
 246                         StrBufAppendBufPlain(Shadow, HKEY("\">"), 0);
 247                         StrBufAppendBuf(Shadow, ShorterUrlStr, 0);
 248                         StrBufAppendBufPlain(Shadow, HKEY("["), 0);
 249                         StrBufAppendBufPlain(Shadow, pCUrl->Key, pCUrl->len, 0);
 250                         StrBufAppendBufPlain(Shadow, HKEY("]</a>"), 0);
 251                 }
 252                 else
 253                 {
 254                         StrBufAppendBufPlain(Shadow, HKEY("<a href=\""), 0);
 255                         StrBufAppendBufPlain(Shadow, pCUrl->Key, pCUrl->len, 0);
 256                         StrBufAppendBufPlain(Shadow, HKEY("\">"), 0);
 257                         StrBufAppendBufPlain(Shadow, pCUrl->Key, pCUrl->len, 0);
 258                         StrBufAppendBufPlain(Shadow, HKEY("</a>"), 0);
 259                 }
 260                 pch = pCUrl->Key + pCUrl->len + 1;
 261
 262         }
 263         if (pch < pche)
 264                 StrBufAppendBufPlain(Shadow, pch, pche - pch, 0);
 265         FlushStrBuf(Message);
 266         StrBufAppendBuf(Message, Shadow, 0);
 267
 268         FreeStrBuf(&ShorterUrlStr);
 269         FreeStrBuf(&Shadow);
 270         DeleteHashPos(&Pos);
 271
 272
 273         DeleteHash(&pUrls);
 274 }
 275
 276 void LoadUrlShorteners(void)
 277 {
 278         int i = 0;
 279         int fd;
 280         const char *POS = NULL;
 281         const char *Err = NULL;
 282         StrBuf *Content, *Line;
 283
 284
 285         UrlShorteners = NewHash(0, Flathash);
 286
 287         fd = open(file_citadel_urlshorteners, 0);
 288
 289         if (fd != 0)
 290         {
 291                 Content = NewStrBufPlain(NULL, SIZ);
 292                 Line = NewStrBuf();
 293                 while (POS != StrBufNOTNULL)
 294                 {
 295                         StrBufTCP_read_buffered_line_fast (Line, Content, &POS, &fd, 1, 1, &Err);
 296                         StrBufTrim(Line);
 297                         if ((*ChrPtr(Line) != '#') && (StrLength(Line) > 0))
 298                         {
 299                                 Put(UrlShorteners, IKEY(i), Line, HFreeStrBuf);
 300                                 i++;
 301                                 Line = NewStrBuf();
 302                         }
 303                         else
 304                                 FlushStrBuf(Line);
 305                         if (POS == NULL)
 306                                 POS = StrBufNOTNULL;
 307                 }
 308                 FreeStrBuf(&Line);
 309                 FreeStrBuf(&Content);
 310         }
 311         close(fd);
 312 }
 313
 314 void shorter_url_cleanup(void)
 315 {
 316         DeleteHash(&UrlShorteners);
 317 }
 318
 319
 320 CTDL_MODULE_INIT(urldeshortener)
 321 {
 322         if (threading)
 323         {
 324                 CtdlLogPrintf(CTDL_INFO, "%s\n", curl_version());
 325         }
 326         else
 327         {
 328                 LoadUrlShorteners ();
 329                 CtdlRegisterCleanupHook(shorter_url_cleanup);
 330         }
 331         return "UrlShortener";
 332 }