citadel/modules/curl/serv_curl.c

   1 /*
   2  * Copyright (c) 1998-2009 by the citadel.org team
   3  *
   4  *  This program is free software; you can redistribute it and/or modify
   5  *  it under the terms of the GNU General Public License as published by
   6  *  the Free Software Foundation; either version 3 of the License, or
   7  *  (at your option) any later version.
   8  *
   9  *  This program is distributed in the hope that it will be useful,
  10  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
  11  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12  *  GNU General Public License for more details.
  13  *
  14  *  You should have received a copy of the GNU General Public License
  15  *  along with this program; if not, write to the Free Software
  16  *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  17  *
  18  *  thanks to some guy in #libev
  19  */
  20
  21 #include "sysdep.h"
  22 #include <stdlib.h>
  23 #include <unistd.h>
  24 #include <stdio.h>
  25 #include <termios.h>
  26 #include <fcntl.h>
  27 #include <signal.h>
  28 #include <pwd.h>
  29 #include <errno.h>
  30 #include <sys/types.h>
  31 #include <syslog.h>
  32
  33 #if TIME_WITH_SYS_TIME
  34 # include <sys/time.h>
  35 # include <time.h>
  36 #else
  37 # if HAVE_SYS_TIME_H
  38 #  include <sys/time.h>
  39 # else
  40 #  include <time.h>
  41 # endif
  42 #endif
  43 #include <sys/wait.h>
  44 #include <ctype.h>
  45 #include <string.h>
  46 #include <limits.h>
  47 #include <sys/socket.h>
  48 #include <netinet/in.h>
  49 #include <arpa/inet.h>
  50
  51 #include <libcitadel.h>
  52 #include "citadel.h"
  53 #include "server.h"
  54 #include "citserver.h"
  55 #include "support.h"
  56
  57 #include "ctdl_module.h"
  58 #include "serv_curl.h"
  59
  60 evcurl_global_data global;
  61
  62
  63 extern struct ev_loop *event_base;
  64
  65 void SockStateCb(void *data, int sock, int read, int write);
  66
  67 #define MOPT(s, v) \
  68         do {                                                        \
  69         sta = curl_multi_setopt(mhnd, (CURLMOPT_##s), (v));         \
  70         if (sta) {                                                  \
  71                 CtdlLogPrintf(CTDL_ERR, "error setting option " #s " on curl multi handle: %s\n", curl_easy_strerror(sta)); \
  72                 exit (1);                                           \
  73         }                                                           \
  74 } while (0)
  75
  76
  77 void ____InitC_ares_dns(AsyncIO *IO)
  78 {
  79         int optmask = 0;
  80         if (IO->DNSChannel == NULL) {
  81                 optmask |= ARES_OPT_SOCK_STATE_CB;
  82                 IO->DNSOptions.sock_state_cb = SockStateCb;
  83                 IO->DNSOptions.sock_state_cb_data = IO;
  84                 ares_init_options(&IO->DNSChannel, &IO->DNSOptions, optmask);
  85         }
  86 }
  87
  88
  89
  90 /*****************************************************************************
  91  *                   libevent / c-ares integration                           *
  92  *****************************************************************************/
  93 static void ____DNS_send_callback(struct ev_loop *loop, ev_io *watcher, int revents)
  94 {
  95         AsyncIO *IO = watcher->data;
  96
  97         ares_process_fd(IO->DNSChannel, ARES_SOCKET_BAD, IO->dns_send_event.fd);
  98 }
  99 static void ____DNS_recv_callback(struct ev_loop *loop, ev_io *watcher, int revents)
 100 {
 101         AsyncIO *IO = watcher->data;
 102
 103         ares_process_fd(IO->DNSChannel, IO->dns_recv_event.fd, ARES_SOCKET_BAD);
 104 }
 105
 106 void ____SockStateCb(void *data, int sock, int read, int write)
 107 {
 108 /*
 109         struct timeval tvbuf, maxtv, *ret;
 110
 111         int64_t time = 10;
 112 */
 113         AsyncIO *IO = data;
 114 /* already inside of the event queue. */
 115
 116         if (read) {
 117                 if ((IO->dns_recv_event.fd != sock) &&
 118                     (IO->dns_recv_event.fd != 0)) {
 119                         ev_io_stop(event_base, &IO->dns_recv_event);
 120                 }
 121                 IO->dns_recv_event.fd = sock;
 122                 ev_io_init(&IO->dns_recv_event, ____DNS_recv_callback, IO->dns_recv_event.fd, EV_READ);
 123                 IO->dns_recv_event.data = IO;
 124                 ev_io_start(event_base, &IO->dns_recv_event);
 125         }
 126         if (write) {
 127                 if ((IO->dns_send_event.fd != sock) &&
 128                     (IO->dns_send_event.fd != 0)) {
 129                         ev_io_stop(event_base, &IO->dns_send_event);
 130                 }
 131                 IO->dns_send_event.fd = sock;
 132                 ev_io_init(&IO->dns_send_event, ____DNS_send_callback, IO->dns_send_event.fd, EV_WRITE);
 133                 IO->dns_send_event.data = IO;
 134                 ev_io_start(event_base, &IO->dns_send_event);
 135         }
 136 /*
 137
 138                 ev_io_start(event_base, &IO->dns_io_event);
 139
 140                 maxtv.tv_sec = time/1000;
 141                 maxtv.tv_usec = (time % 1000) * 1000;
 142
 143                 ret = ares_timeout(IO->DNSChannel, &maxtv, &tvbuf);
 144         }
 145 */
 146         if ((read == 0) && (write == 0)) {
 147                 ev_io_stop(event_base, &IO->dns_recv_event);
 148                 ev_io_stop(event_base, &IO->dns_send_event);
 149         }
 150 }
 151
 152
 153
 154
 155
 156
 157 static void
 158 gotstatus(evcurl_global_data *global, int nnrun)
 159 {
 160         CURLM *mhnd;
 161         CURLMsg *msg;
 162         int nmsg;
 163 /*
 164         if (EVCURL_GLOBAL_MAGIC != global.magic)
 165         {
 166                 CtdlLogPrintf(CTDL_ERR, "internal error: gotstatus on wrong struct");
 167                 return;
 168         }
 169 */
 170         global->nrun = nnrun;
 171         mhnd = global->mhnd;
 172
 173         CtdlLogPrintf(CTDL_ERR, "about to call curl_multi_info_read\n");
 174         while ((msg = curl_multi_info_read(mhnd, &nmsg))) {
 175                 CtdlLogPrintf(CTDL_ERR, "got curl multi_info message msg=%d", msg->msg);
 176                 if (CURLMSG_DONE == msg->msg) {
 177                         CtdlLogPrintf(CTDL_ERR, "request complete\n");
 178                         CURL *chnd = msg->easy_handle;
 179                         char *chandle;
 180                         CURLcode sta = curl_easy_getinfo(chnd, CURLINFO_PRIVATE, &chandle);
 181                         if (sta)
 182                                 CtdlLogPrintf(CTDL_ERR, "error asking curl for private cookie of curl handle: %s\n", curl_easy_strerror(sta));
 183                         evcurl_request_data  *handle = (void *)chandle;
 184                         if (global != handle->global || chnd != handle->chnd)
 185                                 CtdlLogPrintf(CTDL_ERR, "internal evcurl error: unknown curl handle completed\n");
 186                         sta = msg->data.result;
 187                         if (sta) {
 188                                 CtdlLogPrintf(CTDL_ERR, "error description: %s\n", handle->errdesc);
 189                                 CtdlLogPrintf(CTDL_ERR, "error performing request: %s\n", curl_easy_strerror(sta));
 190                         }
 191                         long httpcode;
 192                         sta = curl_easy_getinfo(chnd, CURLINFO_RESPONSE_CODE, &httpcode);
 193                         if (sta)
 194                                 CtdlLogPrintf(CTDL_ERR, "error asking curl for response code from request: %s\n", curl_easy_strerror(sta));
 195                         CtdlLogPrintf(CTDL_ERR, "http response code was %ld\n", (long)httpcode);
 196                         CURLMcode msta = curl_multi_remove_handle(mhnd, chnd);
 197                         if (msta)
 198                                 CtdlLogPrintf(CTDL_ERR, "warning problem detaching completed handle from curl multi: %s\n", curl_multi_strerror(msta));
 199                         handle->attached = 0;
 200                 }
 201         }
 202         if (0 == nnrun) {
 203                 ev_unloop(EV_DEFAULT, EVUNLOOP_ONE); /* remove in production */
 204         }
 205 }
 206
 207 static void
 208 stepmulti(evcurl_global_data *global, curl_socket_t fd) {
 209         int nnrun;
 210         CURLMcode msta = curl_multi_socket_action(global->mhnd, fd, 0, &nnrun);
 211         if (msta)
 212                 CtdlLogPrintf(CTDL_ERR, "error in curl processing events on multi handle, fd %d: %s\n", (int)fd, curl_multi_strerror(msta));
 213         if (global->nrun != nnrun)
 214                 gotstatus(global, nnrun);
 215 }
 216
 217 static void
 218 gottime(struct ev_loop *loop, ev_timer *timeev, int events) {
 219         CtdlLogPrintf(CTDL_ERR,"waking up curl for timeout\n");
 220         evcurl_global_data *global = (void *)timeev->data;
 221         stepmulti(global, CURL_SOCKET_TIMEOUT);
 222 }
 223
 224 static void
 225 gotio(struct ev_loop *loop, ev_io *ioev, int events) {
 226         CtdlLogPrintf(CTDL_ERR,"waking up curl for io on fd %d\n", (int)ioev->fd);
 227         sockwatcher_data *sockwatcher = (void *)ioev->data;
 228         stepmulti(sockwatcher->global, ioev->fd);
 229 }
 230
 231 static size_t
 232 gotdata(void *data, size_t size, size_t nmemb, void *cglobal) {
 233         evcurl_request_data *D = (evcurl_request_data*) data;
 234         return CurlFillStrBuf_callback(D->ReplyData, size, nmemb, cglobal);
 235 }
 236
 237 static int
 238 gotwatchtime(CURLM *multi, long tblock_ms, void *cglobal) {
 239         CtdlLogPrintf(CTDL_ERR,"gotwatchtime called %ld ms\n", tblock_ms);
 240         evcurl_global_data *global = cglobal;
 241         ev_timer_stop(EV_DEFAULT, &global->timeev);
 242         if (tblock_ms < 0 || 14000 < tblock_ms)
 243                 tblock_ms = 14000;
 244         ev_timer_set(&global->timeev, 0.5e-3 + 1.0e-3 * tblock_ms, 14.0);
 245         ev_timer_start(EV_DEFAULT_UC, &global->timeev);
 246         return 0;
 247 }
 248
 249 static int
 250 gotwatchsock(CURL *easy, curl_socket_t fd, int action, void *cglobal, void *csockwatcher) {
 251         evcurl_global_data *global = cglobal;
 252         CURLM *mhnd = global->mhnd;
 253         CtdlLogPrintf(CTDL_ERR,"gotwatchsock called fd=%d action=%d\n", (int)fd, action);
 254         sockwatcher_data *sockwatcher = csockwatcher;
 255         if (!sockwatcher) {
 256                 CtdlLogPrintf(CTDL_ERR,"called first time to register this sockwatcker\n");
 257                 sockwatcher = malloc(sizeof(sockwatcher_data));
 258                 sockwatcher->global = global;
 259                 ev_init(&sockwatcher->ioev, &gotio);
 260                 sockwatcher->ioev.data = (void *)sockwatcher;
 261                 curl_multi_assign(mhnd, fd, sockwatcher);
 262         }
 263         if (CURL_POLL_REMOVE == action) {
 264                 CtdlLogPrintf(CTDL_ERR,"called last time to unregister this sockwatcher\n");
 265                 free(sockwatcher);
 266         } else {
 267                 int events = (action & CURL_POLL_IN ? EV_READ : 0) | (action & CURL_POLL_OUT ? EV_WRITE : 0);
 268                 ev_io_stop(EV_DEFAULT, &sockwatcher->ioev);
 269                 if (events) {
 270                         ev_io_set(&sockwatcher->ioev, fd, events);
 271                         ev_io_start(EV_DEFAULT, &sockwatcher->ioev);
 272                 }
 273         }
 274         return 0;
 275 }
 276
 277 void curl_init_connectionpool(void)
 278 {
 279         CURLM *mhnd ;
 280 //      global.magic = EVCURL_GLOBAL_MAGIC;
 281
 282         ev_timer_init(&global.timeev, &gottime, 14.0, 14.0);
 283         global.timeev.data = (void *)&global;
 284         global.nrun = -1;
 285         CURLcode sta = curl_global_init(CURL_GLOBAL_ALL);
 286                 /* note: probably replace with curl_global_init_mem if used with perl */
 287         if (sta)
 288         {
 289                 CtdlLogPrintf(CTDL_ERR,"error initializing curl library: %s\n", curl_easy_strerror(sta));
 290                 exit(1);
 291         }
 292 /*
 293         if (!ev_default_loop(EVFLAG_AUTO))
 294         {
 295                 CtdlLogPrintf(CTDL_ERR,"error initializing libev\n");
 296                 exit(2);
 297         }
 298 */
 299         mhnd = global.mhnd = curl_multi_init();
 300         if (!mhnd)
 301         {
 302                 CtdlLogPrintf(CTDL_ERR,"error initializing curl multi handle\n");
 303                 exit(3);
 304         }
 305
 306         MOPT(SOCKETFUNCTION, &gotwatchsock);
 307         MOPT(SOCKETDATA, (void *)&global);
 308         MOPT(TIMERFUNCTION, &gotwatchtime);
 309         MOPT(TIMERDATA, (void *)&global);
 310
 311         /* well, just there to fire the sample request?*/
 312         ev_timer_start(EV_DEFAULT, &global.timeev);
 313         return;
 314 }
 315
 316
 317
 318
 319 int evcurl_init(evcurl_request_data *handle,
 320                 void *CustomData,
 321                 const char* Desc,
 322                 int CallBack)
 323 {
 324         CURLcode sta;
 325         CURL *chnd;
 326
 327         handle->global = &global;
 328         handle->attached = 0;
 329         chnd = handle->chnd = curl_easy_init();
 330         if (!chnd)
 331         {
 332                 CtdlLogPrintf(CTDL_ERR, "error initializing curl handle\n");
 333                 return 1;
 334         }
 335
 336         strcpy(handle->errdesc, Desc);
 337
 338         OPT(VERBOSE, (long)1);
 339                 /* unset in production */
 340         OPT(NOPROGRESS, (long)1);
 341         OPT(NOSIGNAL, (long)1);
 342         OPT(FAILONERROR, (long)1);
 343         OPT(ENCODING, "");
 344         OPT(FOLLOWLOCATION, (long)1);
 345         OPT(MAXREDIRS, (long)7);
 346         OPT(USERAGENT, CITADEL);
 347
 348         OPT(TIMEOUT, (long)1800);
 349         OPT(LOW_SPEED_LIMIT, (long)64);
 350         OPT(LOW_SPEED_TIME, (long)600);
 351         OPT(CONNECTTIMEOUT, (long)600);
 352         OPT(PRIVATE, (void *)handle);
 353
 354
 355         OPT(WRITEFUNCTION, &gotdata);
 356         OPT(WRITEDATA, (void *)handle);
 357         OPT(ERRORBUFFER, handle->errdesc);
 358
 359                 /* point to a structure that points back to the perl structure and stuff */
 360         OPT(URL, handle->URL->PlainUrl);
 361         if (StrLength(handle->URL->CurlCreds))
 362         {
 363                 OPT(HTTPAUTH, (long)CURLAUTH_BASIC);
 364                 OPT(USERPWD, ChrPtr(handle->URL->CurlCreds));
 365         }
 366 #ifdef CURLOPT_HTTP_CONTENT_DECODING
 367         OPT(HTTP_CONTENT_DECODING, 1);
 368         OPT(ENCODING, "");
 369 #endif
 370         if (StrLength(handle->PostData) > 0)
 371         {
 372                 OPT(POSTFIELDS, ChrPtr(handle->PostData));
 373                 OPT(POSTFIELDSIZE, StrLength(handle->PostData));
 374
 375         }
 376         else if ((handle->PlainPostDataLen != 0) && (handle->PlainPostData != NULL))
 377         {
 378                 OPT(POSTFIELDS, handle->PlainPostData);
 379                 OPT(POSTFIELDSIZE, handle->PlainPostDataLen);
 380         }
 381
 382         if (handle->headers != NULL)
 383                 OPT(HTTPHEADER, handle->headers);
 384
 385         return 1;
 386 }
 387
 388 void
 389 evcurl_handle_start(evcurl_request_data *handle)
 390 {
 391         CURLMcode msta = curl_multi_add_handle(handle->global->mhnd, handle->chnd);
 392         if (msta)
 393                 CtdlLogPrintf(CTDL_ERR, "error attaching to curl multi handle: %s\n", curl_multi_strerror(msta));
 394         handle->attached = 1;
 395 }
 396
 397
 398
 399 CTDL_MODULE_INIT(curl_client)
 400 {
 401         if (!threading)
 402         {
 403                 curl_init_connectionpool();
 404 /*
 405                 int r = ares_library_init(ARES_LIB_INIT_ALL);
 406                 if (0 != r) {
 407                         // TODO
 408                         // ThrowException(Exception::Error(String::New(ares_strerror(r))));
 409 ////                    assert(r == 0);
 410                 }
 411 */      }
 412         return "curl";
 413 }