citadel/event_client.c

   1 /*
   2  * Copyright (c) 1998-2012 by the citadel.org team
   3  *
   4  * This program is open source software; you can redistribute it and/or modify
   5  * it under the terms of the GNU General Public License, version 3.
   6  *
   7  * This program is distributed in the hope that it will be useful,
   8  * but WITHOUT ANY WARRANTY; without even the implied warranty of
   9  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  10  * GNU General Public License for more details.
  11  */
  12
  13 #include "sysdep.h"
  14 #include <stdlib.h>
  15 #include <unistd.h>
  16 #include <stdio.h>
  17 #include <termios.h>
  18 #include <fcntl.h>
  19 #include <signal.h>
  20 #include <pwd.h>
  21 #include <errno.h>
  22 #include <sys/types.h>
  23 #include <syslog.h>
  24
  25 #if TIME_WITH_SYS_TIME
  26 # include <sys/time.h>
  27 # include <time.h>
  28 #else
  29 # if HAVE_SYS_TIME_H
  30 #  include <sys/time.h>
  31 # else
  32 #  include <time.h>
  33 # endif
  34 #endif
  35 #include <sys/wait.h>
  36 #include <ctype.h>
  37 #include <string.h>
  38 #include <limits.h>
  39 #include <sys/socket.h>
  40 #include <netinet/in.h>
  41 #include <arpa/inet.h>
  42 #include <assert.h>
  43 #if HAVE_BACKTRACE
  44 #include <execinfo.h>
  45 #endif
  46
  47 #include <libcitadel.h>
  48 #include "citadel.h"
  49 #include "server.h"
  50 #include "citserver.h"
  51 #include "support.h"
  52 #include "config.h"
  53 #include "control.h"
  54 #include "user_ops.h"
  55 #include "database.h"
  56 #include "msgbase.h"
  57 #include "internet_addressing.h"
  58 #include "genstamp.h"
  59 #include "domain.h"
  60 #include "clientsocket.h"
  61 #include "locate_host.h"
  62 #include "citadel_dirs.h"
  63
  64 #include "event_client.h"
  65 #include "ctdl_module.h"
  66
  67
  68 ConstStr IOStates[] = {
  69         {HKEY("DB Queue")},
  70         {HKEY("DB Q Next")},
  71         {HKEY("DB Attach")},
  72         {HKEY("DB Next")},
  73         {HKEY("DB Stop")},
  74         {HKEY("DB Exit")},
  75         {HKEY("DB Terminate")},
  76         {HKEY("IO Queue")},
  77         {HKEY("IO Attach")},
  78         {HKEY("IO Connect Socket")},
  79         {HKEY("IO Abort")},
  80         {HKEY("IO Timeout")},
  81         {HKEY("IO ConnFail")},
  82         {HKEY("IO ConnFail Now")},
  83         {HKEY("IO Conn Now")},
  84         {HKEY("IO Conn Wait")},
  85         {HKEY("Curl Q")},
  86         {HKEY("Curl Start")},
  87         {HKEY("Curl Shotdown")},
  88         {HKEY("Curl More IO")},
  89         {HKEY("Curl Got IO")},
  90         {HKEY("Curl Got Data")},
  91         {HKEY("Curl Got Status")},
  92         {HKEY("C-Ares Start")},
  93         {HKEY("C-Ares IO Done")},
  94         {HKEY("C-Ares Finished")},
  95         {HKEY("C-Ares exit")},
  96         {HKEY("Killing")},
  97         {HKEY("Exit")}
  98 };
  99
 100 void SetEVState(AsyncIO *IO, eIOState State)
 101 {
 102
 103         CitContext* CCC = IO->CitContext;
 104         if (CCC != NULL)
 105                 memcpy(CCC->lastcmdname, IOStates[State].Key, IOStates[State].len + 1);
 106
 107 }
 108
 109
 110 static void IO_Timeout_callback(struct ev_loop *loop, ev_timer *watcher, int revents);
 111 static void IO_abort_shutdown_callback(struct ev_loop *loop,
 112                                        ev_cleanup *watcher,
 113                                        int revents);
 114
 115
 116 /*------------------------------------------------------------------------------
 117  *                              Server DB IO
 118  *----------------------------------------------------------------------------*/
 119 extern int evdb_count;
 120 extern pthread_mutex_t DBEventQueueMutex;
 121 extern pthread_mutex_t DBEventExitQueueMutex;
 122 extern HashList *DBInboundEventQueue;
 123 extern struct ev_loop *event_db;
 124 extern ev_async DBAddJob;
 125 extern ev_async DBExitEventLoop;
 126
 127 eNextState QueueDBOperation(AsyncIO *IO, IO_CallBack CB)
 128 {
 129         IOAddHandler *h;
 130         int i;
 131
 132         SetEVState(IO, eDBQ);
 133         h = (IOAddHandler*)malloc(sizeof(IOAddHandler));
 134         h->IO = IO;
 135         h->EvAttch = CB;
 136         ev_cleanup_init(&IO->db_abort_by_shutdown,
 137                         IO_abort_shutdown_callback);
 138         IO->db_abort_by_shutdown.data = IO;
 139
 140         pthread_mutex_lock(&DBEventQueueMutex);
 141         if (DBInboundEventQueue == NULL)
 142         {
 143                 /* shutting down... */
 144                 free(h);
 145                 EVM_syslog(LOG_DEBUG, "DBEVENT Q exiting.\n");
 146                 pthread_mutex_unlock(&DBEventQueueMutex);
 147                 return eAbort;
 148         }
 149         EVM_syslog(LOG_DEBUG, "DBEVENT Q\n");
 150         i = ++evdb_count ;
 151         Put(DBInboundEventQueue, IKEY(i), h, NULL);
 152         pthread_mutex_unlock(&DBEventQueueMutex);
 153
 154         pthread_mutex_lock(&DBEventExitQueueMutex);
 155         if (event_db == NULL)
 156         {
 157                 pthread_mutex_unlock(&DBEventExitQueueMutex);
 158                 return eAbort;
 159         }
 160         ev_async_send (event_db, &DBAddJob);
 161         pthread_mutex_unlock(&DBEventExitQueueMutex);
 162
 163         EVM_syslog(LOG_DEBUG, "DBEVENT Q Done.\n");
 164         return eDBQuery;
 165 }
 166
 167 void StopDBWatchers(AsyncIO *IO)
 168 {
 169         SetEVState(IO, eDBStop);
 170         ev_cleanup_stop(event_db, &IO->db_abort_by_shutdown);
 171         ev_idle_stop(event_db, &IO->db_unwind_stack);
 172 }
 173
 174 void ShutDownDBCLient(AsyncIO *IO)
 175 {
 176         CitContext *Ctx =IO->CitContext;
 177         become_session(Ctx);
 178
 179         SetEVState(IO, eDBTerm);
 180         EVM_syslog(LOG_DEBUG, "DBEVENT Terminating.\n");
 181         StopDBWatchers(IO);
 182
 183         assert(IO->DBTerminate);
 184         IO->DBTerminate(IO);
 185 }
 186
 187 void
 188 DB_PerformNext(struct ev_loop *loop, ev_idle *watcher, int revents)
 189 {
 190         AsyncIO *IO = watcher->data;
 191
 192         SetEVState(IO, eDBNext);
 193         IO->Now = ev_now(event_db);
 194         EV_syslog(LOG_DEBUG, "%s()", __FUNCTION__);
 195         become_session(IO->CitContext);
 196
 197         ev_idle_stop(event_db, &IO->db_unwind_stack);
 198
 199         assert(IO->NextDBOperation);
 200         switch (IO->NextDBOperation(IO))
 201         {
 202         case eDBQuery:
 203                 break;
 204         case eSendDNSQuery:
 205         case eReadDNSReply:
 206         case eConnect:
 207         case eSendReply:
 208         case eSendMore:
 209         case eSendFile:
 210         case eReadMessage:
 211         case eReadMore:
 212         case eReadPayload:
 213         case eReadFile:
 214                 ev_cleanup_stop(loop, &IO->db_abort_by_shutdown);
 215                 break;
 216         case eTerminateConnection:
 217         case eAbort:
 218                 ev_idle_stop(event_db, &IO->db_unwind_stack);
 219                 ev_cleanup_stop(loop, &IO->db_abort_by_shutdown);
 220                 ShutDownDBCLient(IO);
 221         }
 222 }
 223
 224 eNextState NextDBOperation(AsyncIO *IO, IO_CallBack CB)
 225 {
 226         SetEVState(IO, eQDBNext);
 227         IO->NextDBOperation = CB;
 228         ev_idle_init(&IO->db_unwind_stack,
 229                      DB_PerformNext);
 230         IO->db_unwind_stack.data = IO;
 231         ev_idle_start(event_db, &IO->db_unwind_stack);
 232         return eDBQuery;
 233 }
 234
 235 /*------------------------------------------------------------------------------
 236  *                      Client IO
 237  *----------------------------------------------------------------------------*/
 238 extern int evbase_count;
 239 extern pthread_mutex_t EventQueueMutex;
 240 extern pthread_mutex_t EventExitQueueMutex;
 241 extern HashList *InboundEventQueue;
 242 extern struct ev_loop *event_base;
 243 extern ev_async AddJob;
 244 extern ev_async ExitEventLoop;
 245
 246 static void IO_abort_shutdown_callback(struct ev_loop *loop,
 247                                        ev_cleanup *watcher,
 248                                        int revents)
 249 {
 250         AsyncIO *IO = watcher->data;
 251
 252         SetEVState(IO, eIOAbort);
 253         EV_syslog(LOG_DEBUG, "EVENT Q: %s\n", __FUNCTION__);
 254         IO->Now = ev_now(event_base);
 255         assert(IO->ShutdownAbort);
 256         IO->ShutdownAbort(IO);
 257 }
 258
 259
 260 eNextState QueueEventContext(AsyncIO *IO, IO_CallBack CB)
 261 {
 262         IOAddHandler *h;
 263         int i;
 264
 265         SetEVState(IO, eIOQ);
 266         h = (IOAddHandler*)malloc(sizeof(IOAddHandler));
 267         h->IO = IO;
 268         h->EvAttch = CB;
 269         ev_cleanup_init(&IO->abort_by_shutdown,
 270                         IO_abort_shutdown_callback);
 271         IO->abort_by_shutdown.data = IO;
 272
 273         pthread_mutex_lock(&EventQueueMutex);
 274         if (InboundEventQueue == NULL)
 275         {
 276                 free(h);
 277                 /* shutting down... */
 278                 EVM_syslog(LOG_DEBUG, "EVENT Q exiting.\n");
 279                 pthread_mutex_unlock(&EventQueueMutex);
 280                 return eAbort;
 281         }
 282         EVM_syslog(LOG_DEBUG, "EVENT Q\n");
 283         i = ++evbase_count;
 284         Put(InboundEventQueue, IKEY(i), h, NULL);
 285         pthread_mutex_unlock(&EventQueueMutex);
 286
 287         pthread_mutex_lock(&EventExitQueueMutex);
 288         if (event_base == NULL) {
 289                 pthread_mutex_unlock(&EventExitQueueMutex);
 290                 return eAbort;
 291         }
 292         ev_async_send (event_base, &AddJob);
 293         pthread_mutex_unlock(&EventExitQueueMutex);
 294         EVM_syslog(LOG_DEBUG, "EVENT Q Done.\n");
 295         return eSendReply;
 296 }
 297
 298 eNextState EventQueueDBOperation(AsyncIO *IO, IO_CallBack CB)
 299 {
 300         StopClientWatchers(IO, 0);
 301         return QueueDBOperation(IO, CB);
 302 }
 303 eNextState DBQueueEventContext(AsyncIO *IO, IO_CallBack CB)
 304 {
 305         StopDBWatchers(IO);
 306         return QueueEventContext(IO, CB);
 307 }
 308
 309 extern eNextState evcurl_handle_start(AsyncIO *IO);
 310
 311 eNextState QueueCurlContext(AsyncIO *IO)
 312 {
 313         IOAddHandler *h;
 314         int i;
 315
 316         SetEVState(IO, eCurlQ);
 317         h = (IOAddHandler*)malloc(sizeof(IOAddHandler));
 318         h->IO = IO;
 319         h->EvAttch = evcurl_handle_start;
 320
 321         pthread_mutex_lock(&EventQueueMutex);
 322         if (InboundEventQueue == NULL)
 323         {
 324                 /* shutting down... */
 325                 free(h);
 326                 EVM_syslog(LOG_DEBUG, "EVENT Q exiting.\n");
 327                 pthread_mutex_unlock(&EventQueueMutex);
 328                 return eAbort;
 329         }
 330
 331         EVM_syslog(LOG_DEBUG, "EVENT Q\n");
 332         i = ++evbase_count;
 333         Put(InboundEventQueue, IKEY(i), h, NULL);
 334         pthread_mutex_unlock(&EventQueueMutex);
 335
 336         pthread_mutex_lock(&EventExitQueueMutex);
 337         if (event_base == NULL) {
 338                 pthread_mutex_unlock(&EventExitQueueMutex);
 339                 return eAbort;
 340         }
 341         ev_async_send (event_base, &AddJob);
 342         pthread_mutex_unlock(&EventExitQueueMutex);
 343
 344         EVM_syslog(LOG_DEBUG, "EVENT Q Done.\n");
 345         return eSendReply;
 346 }
 347
 348 eNextState CurlQueueDBOperation(AsyncIO *IO, IO_CallBack CB)
 349 {
 350         StopCurlWatchers(IO);
 351         return QueueDBOperation(IO, CB);
 352 }
 353
 354
 355 void DestructCAres(AsyncIO *IO);
 356 void FreeAsyncIOContents(AsyncIO *IO)
 357 {
 358         CitContext *Ctx = IO->CitContext;
 359
 360         FreeStrBuf(&IO->IOBuf);
 361         FreeStrBuf(&IO->SendBuf.Buf);
 362         FreeStrBuf(&IO->RecvBuf.Buf);
 363
 364         DestructCAres(IO);
 365
 366         FreeURL(&IO->ConnectMe);
 367         FreeStrBuf(&IO->HttpReq.ReplyData);
 368
 369         if (Ctx) {
 370                 Ctx->state = CON_IDLE;
 371                 Ctx->kill_me = 1;
 372                 IO->CitContext = NULL;
 373         }
 374 }
 375
 376
 377 void StopClientWatchers(AsyncIO *IO, int CloseFD)
 378 {
 379         ev_timer_stop (event_base, &IO->rw_timeout);
 380         ev_timer_stop(event_base, &IO->conn_fail);
 381         ev_idle_stop(event_base, &IO->unwind_stack);
 382         ev_cleanup_stop(event_base, &IO->abort_by_shutdown);
 383
 384         ev_io_stop(event_base, &IO->conn_event);
 385         ev_io_stop(event_base, &IO->send_event);
 386         ev_io_stop(event_base, &IO->recv_event);
 387
 388         if (CloseFD && (IO->SendBuf.fd > 0)) {
 389                 close(IO->SendBuf.fd);
 390                 IO->SendBuf.fd = 0;
 391                 IO->RecvBuf.fd = 0;
 392         }
 393 }
 394
 395 void StopCurlWatchers(AsyncIO *IO)
 396 {
 397         EVM_syslog(LOG_DEBUG, "EVENT StopCurlWatchers \n");
 398
 399         ev_timer_stop (event_base, &IO->rw_timeout);
 400         ev_timer_stop(event_base, &IO->conn_fail);
 401         ev_idle_stop(event_base, &IO->unwind_stack);
 402         ev_cleanup_stop(event_base, &IO->abort_by_shutdown);
 403
 404         ev_io_stop(event_base, &IO->conn_event);
 405         ev_io_stop(event_base, &IO->send_event);
 406         ev_io_stop(event_base, &IO->recv_event);
 407
 408         curl_easy_cleanup(IO->HttpReq.chnd);
 409         IO->HttpReq.chnd = NULL;
 410
 411         if (IO->SendBuf.fd != 0) {
 412                 close(IO->SendBuf.fd);
 413         }
 414         IO->SendBuf.fd = 0;
 415         IO->RecvBuf.fd = 0;
 416 }
 417
 418 void ShutDownCLient(AsyncIO *IO)
 419 {
 420         CitContext *Ctx =IO->CitContext;
 421
 422         SetEVState(IO, eExit);
 423         become_session(Ctx);
 424
 425         EVM_syslog(LOG_DEBUG, "EVENT Terminating \n");
 426
 427         StopClientWatchers(IO, 1);
 428
 429         if (IO->DNS.Channel != NULL) {
 430                 ares_destroy(IO->DNS.Channel);
 431                 EV_DNS_LOG_STOP(DNS.recv_event);
 432                 EV_DNS_LOG_STOP(DNS.send_event);
 433                 ev_io_stop(event_base, &IO->DNS.recv_event);
 434                 ev_io_stop(event_base, &IO->DNS.send_event);
 435                 IO->DNS.Channel = NULL;
 436         }
 437         assert(IO->Terminate);
 438         IO->Terminate(IO);
 439 }
 440
 441 void PostInbound(AsyncIO *IO)
 442 {
 443         switch (IO->NextState) {
 444         case eSendFile:
 445                 ev_io_start(event_base, &IO->send_event);
 446                 break;
 447         case eSendReply:
 448         case eSendMore:
 449                 assert(IO->SendDone);
 450                 IO->NextState = IO->SendDone(IO);
 451                 switch (IO->NextState)
 452                 {
 453                 case eSendFile:
 454                 case eSendReply:
 455                 case eSendMore:
 456                 case eReadMessage:
 457                 case eReadPayload:
 458                 case eReadMore:
 459                 case eReadFile:
 460                         ev_io_start(event_base, &IO->send_event);
 461                         break;
 462                 case eDBQuery:
 463                         StopClientWatchers(IO, 0);
 464                 default:
 465                         break;
 466                 }
 467                 break;
 468         case eReadPayload:
 469         case eReadMore:
 470         case eReadFile:
 471                 ev_io_start(event_base, &IO->recv_event);
 472                 break;
 473         case eTerminateConnection:
 474                 ShutDownCLient(IO);
 475                 break;
 476         case eAbort:
 477                 ShutDownCLient(IO);
 478                 break;
 479         case eSendDNSQuery:
 480         case eReadDNSReply:
 481         case eDBQuery:
 482         case eConnect:
 483         case eReadMessage:
 484                 break;
 485         }
 486 }
 487 eReadState HandleInbound(AsyncIO *IO)
 488 {
 489         const char *Err = NULL;
 490         eReadState Finished = eBufferNotEmpty;
 491
 492         become_session(IO->CitContext);
 493
 494         while ((Finished == eBufferNotEmpty) &&
 495                ((IO->NextState == eReadMessage)||
 496                 (IO->NextState == eReadMore)||
 497                 (IO->NextState == eReadFile)||
 498                 (IO->NextState == eReadPayload)))
 499         {
 500                 /* Reading lines...
 501                  * lex line reply in callback,
 502                  * or do it ourselves.
 503                  * i.e. as nnn-blabla means continue reading in SMTP
 504                  */
 505                 if ((IO->NextState == eReadFile) &&
 506                     (Finished == eBufferNotEmpty))
 507                 {
 508                         Finished = WriteIOBAlreadyRead(&IO->IOB, &Err);
 509                         if (Finished == eReadSuccess)
 510                         {
 511                                 IO->NextState = eSendReply;
 512                         }
 513                 }
 514                 else if (IO->LineReader)
 515                         Finished = IO->LineReader(IO);
 516                 else
 517                         Finished = StrBufChunkSipLine(IO->IOBuf,
 518                                                       &IO->RecvBuf);
 519
 520                 switch (Finished) {
 521                 case eMustReadMore: /// read new from socket...
 522                         break;
 523                 case eBufferNotEmpty: /* shouldn't happen... */
 524                 case eReadSuccess: /// done for now...
 525                         break;
 526                 case eReadFail: /// WHUT?
 527                                 ///todo: shut down!
 528                         break;
 529                 }
 530
 531                 if (Finished != eMustReadMore) {
 532                         assert(IO->ReadDone);
 533                         ev_io_stop(event_base, &IO->recv_event);
 534                         IO->NextState = IO->ReadDone(IO);
 535                         Finished = StrBufCheckBuffer(&IO->RecvBuf);
 536                 }
 537         }
 538
 539         PostInbound(IO);
 540
 541         return Finished;
 542 }
 543
 544
 545 static void
 546 IO_send_callback(struct ev_loop *loop, ev_io *watcher, int revents)
 547 {
 548         int rc;
 549         AsyncIO *IO = watcher->data;
 550         const char *errmsg = NULL;
 551
 552         IO->Now = ev_now(event_base);
 553         become_session(IO->CitContext);
 554 #ifdef BIGBAD_IODBG
 555         {
 556                 int rv = 0;
 557                 char fn [SIZ];
 558                 FILE *fd;
 559                 const char *pch = ChrPtr(IO->SendBuf.Buf);
 560                 const char *pchh = IO->SendBuf.ReadWritePointer;
 561                 long nbytes;
 562
 563                 if (pchh == NULL)
 564                         pchh = pch;
 565
 566                 nbytes = StrLength(IO->SendBuf.Buf) - (pchh - pch);
 567                 snprintf(fn, SIZ, "/tmp/foolog_ev_%s.%d",
 568                          ((CitContext*)(IO->CitContext))->ServiceName,
 569                          IO->SendBuf.fd);
 570
 571                 fd = fopen(fn, "a+");
 572                 fprintf(fd, "Send: BufSize: %ld BufContent: [",
 573                         nbytes);
 574                 rv = fwrite(pchh, nbytes, 1, fd);
 575                 if (!rv) printf("failed to write debug to %s!\n", fn);
 576                 fprintf(fd, "]\n");
 577 #endif
 578                 switch (IO->NextState) {
 579                 case eSendFile:
 580                         rc = FileSendChunked(&IO->IOB, &errmsg);
 581                         if (rc < 0)
 582                                 StrBufPlain(IO->ErrMsg, errmsg, -1);
 583                         break;
 584                 default:
 585                         rc = StrBuf_write_one_chunk_callback(IO->SendBuf.fd,
 586                                                              0,
 587                                                              &IO->SendBuf);
 588                 }
 589
 590 #ifdef BIGBAD_IODBG
 591                 fprintf(fd, "Sent: BufSize: %d bytes.\n", rc);
 592                 fclose(fd);
 593         }
 594 #endif
 595         if (rc == 0)
 596         {
 597                 ev_io_stop(event_base, &IO->send_event);
 598                 switch (IO->NextState) {
 599                 case eSendMore:
 600                         assert(IO->SendDone);
 601                         IO->NextState = IO->SendDone(IO);
 602
 603                         if ((IO->NextState == eTerminateConnection) ||
 604                             (IO->NextState == eAbort) )
 605                                 ShutDownCLient(IO);
 606                         else {
 607                                 ev_io_start(event_base, &IO->send_event);
 608                         }
 609                         break;
 610                 case eSendFile:
 611                         if (IO->IOB.ChunkSendRemain > 0) {
 612                                 ev_io_start(event_base, &IO->recv_event);
 613                                 SetNextTimeout(IO, 100.0);
 614
 615                         } else {
 616                                 assert(IO->ReadDone);
 617                                 IO->NextState = IO->ReadDone(IO);
 618                                 switch(IO->NextState) {
 619                                 case eSendDNSQuery:
 620                                 case eReadDNSReply:
 621                                 case eDBQuery:
 622                                 case eConnect:
 623                                         break;
 624                                 case eSendReply:
 625                                 case eSendMore:
 626                                 case eSendFile:
 627                                         ev_io_start(event_base,
 628                                                     &IO->send_event);
 629                                         break;
 630                                 case eReadMessage:
 631                                 case eReadMore:
 632                                 case eReadPayload:
 633                                 case eReadFile:
 634                                         break;
 635                                 case eTerminateConnection:
 636                                 case eAbort:
 637                                         break;
 638                                 }
 639                         }
 640                         break;
 641                 case eSendReply:
 642                     if (StrBufCheckBuffer(&IO->SendBuf) != eReadSuccess)
 643                         break;
 644                     IO->NextState = eReadMore;
 645                 case eReadMore:
 646                 case eReadMessage:
 647                 case eReadPayload:
 648                 case eReadFile:
 649                         if (StrBufCheckBuffer(&IO->RecvBuf) == eBufferNotEmpty)
 650                         {
 651                                 HandleInbound(IO);
 652                         }
 653                         else {
 654                                 ev_io_start(event_base, &IO->recv_event);
 655                         }
 656
 657                         break;
 658                 case eDBQuery:
 659                         /*
 660                          * we now live in another queue,
 661                          * so we have to unregister.
 662                          */
 663                         ev_cleanup_stop(loop, &IO->abort_by_shutdown);
 664                         break;
 665                 case eSendDNSQuery:
 666                 case eReadDNSReply:
 667                 case eConnect:
 668                 case eTerminateConnection:
 669                 case eAbort:
 670                         break;
 671                 }
 672         }
 673         else if (rc < 0) {
 674                 if (errno != EAGAIN) {
 675                         StopClientWatchers(IO, 1);
 676                         EV_syslog(LOG_DEBUG,
 677                                   "IO_send_callback(): Socket Invalid! [%d] [%s] [%d]\n",
 678                                   errno, strerror(errno), IO->SendBuf.fd);
 679                         StrBufPrintf(IO->ErrMsg,
 680                                      "Socket Invalid! [%s]",
 681                                      strerror(errno));
 682                         SetNextTimeout(IO, 0.01);
 683                 }
 684         }
 685         /* else : must write more. */
 686 }
 687 static void
 688 set_start_callback(struct ev_loop *loop, AsyncIO *IO, int revents)
 689 {
 690         ev_timer_stop(event_base, &IO->conn_fail);
 691         ev_timer_start(event_base, &IO->rw_timeout);
 692
 693         switch(IO->NextState) {
 694         case eReadMore:
 695         case eReadMessage:
 696         case eReadFile:
 697                 StrBufAppendBufPlain(IO->ErrMsg, HKEY("[while waiting for greeting]"), 0);
 698                 ev_io_start(event_base, &IO->recv_event);
 699                 break;
 700         case eSendReply:
 701         case eSendMore:
 702         case eReadPayload:
 703         case eSendFile:
 704                 become_session(IO->CitContext);
 705                 IO_send_callback(loop, &IO->send_event, revents);
 706                 break;
 707         case eDBQuery:
 708         case eSendDNSQuery:
 709         case eReadDNSReply:
 710         case eConnect:
 711         case eTerminateConnection:
 712         case eAbort:
 713                 /// TODO: WHUT?
 714                 break;
 715         }
 716 }
 717
 718 static void
 719 IO_Timeout_callback(struct ev_loop *loop, ev_timer *watcher, int revents)
 720 {
 721         AsyncIO *IO = watcher->data;
 722
 723         SetEVState(IO, eIOTimeout);
 724         IO->Now = ev_now(event_base);
 725         ev_timer_stop (event_base, &IO->rw_timeout);
 726         become_session(IO->CitContext);
 727
 728         if (IO->SendBuf.fd != 0)
 729         {
 730                 ev_io_stop(event_base, &IO->send_event);
 731                 ev_io_stop(event_base, &IO->recv_event);
 732                 ev_timer_stop (event_base, &IO->rw_timeout);
 733                 close(IO->SendBuf.fd);
 734                 IO->SendBuf.fd = IO->RecvBuf.fd = 0;
 735         }
 736
 737         assert(IO->Timeout);
 738         switch (IO->Timeout(IO))
 739         {
 740         case eAbort:
 741                 ShutDownCLient(IO);
 742         default:
 743                 break;
 744         }
 745 }
 746
 747 static void
 748 IO_connfail_callback(struct ev_loop *loop, ev_timer *watcher, int revents)
 749 {
 750         AsyncIO *IO = watcher->data;
 751
 752         SetEVState(IO, eIOConnfail);
 753         IO->Now = ev_now(event_base);
 754         ev_timer_stop (event_base, &IO->conn_fail);
 755
 756         if (IO->SendBuf.fd != 0)
 757         {
 758                 ev_io_stop(loop, &IO->conn_event);
 759                 ev_io_stop(event_base, &IO->send_event);
 760                 ev_io_stop(event_base, &IO->recv_event);
 761                 ev_timer_stop (event_base, &IO->rw_timeout);
 762                 close(IO->SendBuf.fd);
 763                 IO->SendBuf.fd = IO->RecvBuf.fd = 0;
 764         }
 765         become_session(IO->CitContext);
 766
 767         assert(IO->ConnFail);
 768         switch (IO->ConnFail(IO))
 769         {
 770         case eAbort:
 771                 ShutDownCLient(IO);
 772         default:
 773                 break;
 774
 775         }
 776 }
 777
 778 static void
 779 IO_connfailimmediate_callback(struct ev_loop *loop,
 780                               ev_idle *watcher,
 781                               int revents)
 782 {
 783         AsyncIO *IO = watcher->data;
 784
 785         SetEVState(IO, eIOConnfailNow);
 786         IO->Now = ev_now(event_base);
 787         ev_idle_stop (event_base, &IO->conn_fail_immediate);
 788
 789         if (IO->SendBuf.fd != 0)
 790         {
 791                 close(IO->SendBuf.fd);
 792                 IO->SendBuf.fd = IO->RecvBuf.fd = 0;
 793         }
 794         become_session(IO->CitContext);
 795
 796         assert(IO->ConnFail);
 797         switch (IO->ConnFail(IO))
 798         {
 799         case eAbort:
 800                 ShutDownCLient(IO);
 801         default:
 802                 break;
 803
 804         }
 805 }
 806
 807 static void
 808 IO_connestd_callback(struct ev_loop *loop, ev_io *watcher, int revents)
 809 {
 810         AsyncIO *IO = watcher->data;
 811         int             so_err = 0;
 812         socklen_t       lon = sizeof(so_err);
 813         int             err;
 814
 815         SetEVState(IO, eIOConnNow);
 816         IO->Now = ev_now(event_base);
 817         EVM_syslog(LOG_DEBUG, "connect() succeeded.\n");
 818
 819         ev_io_stop(loop, &IO->conn_event);
 820         ev_timer_stop(event_base, &IO->conn_fail);
 821
 822         err = getsockopt(IO->SendBuf.fd,
 823                          SOL_SOCKET,
 824                          SO_ERROR,
 825                          (void*)&so_err,
 826                          &lon);
 827
 828         if ((err == 0) && (so_err != 0))
 829         {
 830                 EV_syslog(LOG_DEBUG, "connect() failed [%d][%s]\n",
 831                           so_err,
 832                           strerror(so_err));
 833                 IO_connfail_callback(loop, &IO->conn_fail, revents);
 834
 835         }
 836         else
 837         {
 838                 EVM_syslog(LOG_DEBUG, "connect() succeeded\n");
 839                 set_start_callback(loop, IO, revents);
 840         }
 841 }
 842
 843 static void
 844 IO_recv_callback(struct ev_loop *loop, ev_io *watcher, int revents)
 845 {
 846         const char *errmsg;
 847         ssize_t nbytes;
 848         AsyncIO *IO = watcher->data;
 849
 850         IO->Now = ev_now(event_base);
 851         switch (IO->NextState) {
 852         case eReadFile:
 853                 nbytes = FileRecvChunked(&IO->IOB, &errmsg);
 854                 if (nbytes < 0)
 855                         StrBufPlain(IO->ErrMsg, errmsg, -1);
 856                 else
 857                 {
 858                         if (IO->IOB.ChunkSendRemain == 0)
 859                         {
 860                                 IO->NextState = eSendReply;
 861                                 assert(IO->ReadDone);
 862                                 ev_io_stop(event_base, &IO->recv_event);
 863                                 PostInbound(IO);
 864                                 return;
 865                         }
 866                         else
 867                                 return;
 868                 }
 869                 break;
 870         default:
 871                 nbytes = StrBuf_read_one_chunk_callback(IO->RecvBuf.fd,
 872                                                         0,
 873                                                         &IO->RecvBuf);
 874                 break;
 875         }
 876
 877 #ifdef BIGBAD_IODBG
 878         {
 879                 long nbytes;
 880                 int rv = 0;
 881                 char fn [SIZ];
 882                 FILE *fd;
 883                 const char *pch = ChrPtr(IO->RecvBuf.Buf);
 884                 const char *pchh = IO->RecvBuf.ReadWritePointer;
 885
 886                 if (pchh == NULL)
 887                         pchh = pch;
 888
 889                 nbytes = StrLength(IO->RecvBuf.Buf) - (pchh - pch);
 890                 snprintf(fn, SIZ, "/tmp/foolog_ev_%s.%d",
 891                          ((CitContext*)(IO->CitContext))->ServiceName,
 892                          IO->SendBuf.fd);
 893
 894                 fd = fopen(fn, "a+");
 895                 fprintf(fd, "Read: BufSize: %ld BufContent: [",
 896                         nbytes);
 897                 rv = fwrite(pchh, nbytes, 1, fd);
 898                 if (!rv) printf("failed to write debug to %s!\n", fn);
 899                 fprintf(fd, "]\n");
 900                 fclose(fd);
 901         }
 902 #endif
 903         if (nbytes > 0) {
 904                 HandleInbound(IO);
 905         } else if (nbytes == 0) {
 906                 StopClientWatchers(IO, 1);
 907                 SetNextTimeout(IO, 0.01);
 908                 return;
 909         } else if (nbytes == -1) {
 910                 if (errno != EAGAIN) {
 911                         // FD is gone. kick it.
 912                         StopClientWatchers(IO, 1);
 913                         EV_syslog(LOG_DEBUG,
 914                                   "IO_recv_callback(): Socket Invalid! [%d] [%s] [%d]\n",
 915                                   errno, strerror(errno), IO->SendBuf.fd);
 916                         StrBufPrintf(IO->ErrMsg,
 917                                      "Socket Invalid! [%s]",
 918                                      strerror(errno));
 919                         SetNextTimeout(IO, 0.01);
 920                 }
 921                 return;
 922         }
 923 }
 924
 925 void
 926 IO_postdns_callback(struct ev_loop *loop, ev_idle *watcher, int revents)
 927 {
 928         AsyncIO *IO = watcher->data;
 929
 930         SetEVState(IO, eCaresFinished);
 931         IO->Now = ev_now(event_base);
 932         EV_syslog(LOG_DEBUG, "event: %s\n", __FUNCTION__);
 933         become_session(IO->CitContext);
 934         assert(IO->DNS.Query->PostDNS);
 935         switch (IO->DNS.Query->PostDNS(IO))
 936         {
 937         case eAbort:
 938                 assert(IO->DNS.Fail);
 939                 switch (IO->DNS.Fail(IO)) {
 940                 case eAbort:
 941 ////                    StopClientWatchers(IO);
 942                         ShutDownCLient(IO);
 943                 default:
 944                         break;
 945                 }
 946         default:
 947                 break;
 948         }
 949 }
 950
 951
 952 eNextState EvConnectSock(AsyncIO *IO,
 953                          double conn_timeout,
 954                          double first_rw_timeout,
 955                          int ReadFirst)
 956 {
 957         struct sockaddr_in egress_sin;
 958         int fdflags;
 959         int rc = -1;
 960
 961         SetEVState(IO, eIOConnectSock);
 962         become_session(IO->CitContext);
 963
 964         if (ReadFirst) {
 965                 IO->NextState = eReadMessage;
 966         }
 967         else {
 968                 IO->NextState = eSendReply;
 969         }
 970
 971         IO->SendBuf.fd = IO->RecvBuf.fd =
 972                 socket(
 973                         (IO->ConnectMe->IPv6)?PF_INET6:PF_INET,
 974                         SOCK_STREAM,
 975                         IPPROTO_TCP);
 976
 977         if (IO->SendBuf.fd < 0) {
 978                 EV_syslog(LOG_ERR,
 979                           "EVENT: socket() failed: %s\n",
 980                           strerror(errno));
 981
 982                 StrBufPrintf(IO->ErrMsg,
 983                              "Failed to create socket: %s",
 984                              strerror(errno));
 985                 IO->SendBuf.fd = IO->RecvBuf.fd = 0;
 986                 return eAbort;
 987         }
 988         fdflags = fcntl(IO->SendBuf.fd, F_GETFL);
 989         if (fdflags < 0) {
 990                 EV_syslog(LOG_ERR,
 991                           "EVENT: unable to get socket %d flags! %s \n",
 992                           IO->SendBuf.fd,
 993                           strerror(errno));
 994                 StrBufPrintf(IO->ErrMsg,
 995                              "Failed to get socket %d flags: %s",
 996                              IO->SendBuf.fd,
 997                              strerror(errno));
 998                 close(IO->SendBuf.fd);
 999                 IO->SendBuf.fd = IO->RecvBuf.fd = 0;
1000                 return eAbort;
1001         }
1002         fdflags = fdflags | O_NONBLOCK;
1003         if (fcntl(IO->SendBuf.fd, F_SETFL, fdflags) < 0) {
1004                 EV_syslog(
1005                         LOG_ERR,
1006                         "EVENT: unable to set socket %d nonblocking flags! %s \n",
1007                         IO->SendBuf.fd,
1008                         strerror(errno));
1009                 StrBufPrintf(IO->ErrMsg,
1010                              "Failed to set socket flags: %s",
1011                              strerror(errno));
1012                 close(IO->SendBuf.fd);
1013                 IO->SendBuf.fd = IO->RecvBuf.fd = 0;
1014                 return eAbort;
1015         }
1016 /* TODO: maye we could use offsetof() to calc the position of data...
1017  * http://doc.dvgu.ru/devel/ev.html#associating_custom_data_with_a_watcher
1018  */
1019         ev_io_init(&IO->recv_event, IO_recv_callback, IO->RecvBuf.fd, EV_READ);
1020         IO->recv_event.data = IO;
1021         ev_io_init(&IO->send_event, IO_send_callback, IO->SendBuf.fd, EV_WRITE);
1022         IO->send_event.data = IO;
1023
1024         ev_timer_init(&IO->conn_fail, IO_connfail_callback, conn_timeout, 0);
1025         IO->conn_fail.data = IO;
1026         ev_timer_init(&IO->rw_timeout, IO_Timeout_callback, first_rw_timeout,0);
1027         IO->rw_timeout.data = IO;
1028
1029
1030
1031
1032         /* for debugging you may bypass it like this:
1033          * IO->Addr.sin_addr.s_addr = inet_addr("127.0.0.1");
1034          * ((struct sockaddr_in)IO->ConnectMe->Addr).sin_addr.s_addr =
1035          *   inet_addr("127.0.0.1");
1036          */
1037         if (IO->ConnectMe->IPv6) {
1038                 rc = connect(IO->SendBuf.fd,
1039                              &IO->ConnectMe->Addr,
1040                              sizeof(struct sockaddr_in6));
1041         }
1042         else {
1043                 /* If citserver is bound to a specific IP address on the host, make
1044                  * sure we use that address for outbound connections.
1045                  */
1046
1047                 memset(&egress_sin, 0, sizeof(egress_sin));
1048                 egress_sin.sin_family = AF_INET;
1049                 if (!IsEmptyStr(config.c_ip_addr)) {
1050                         egress_sin.sin_addr.s_addr = inet_addr(config.c_ip_addr);
1051                         if (egress_sin.sin_addr.s_addr == !INADDR_ANY) {
1052                                 egress_sin.sin_addr.s_addr = INADDR_ANY;
1053                         }
1054
1055                         /* If this bind fails, no problem; we can still use INADDR_ANY */
1056                         bind(IO->SendBuf.fd, (struct sockaddr *)&egress_sin, sizeof(egress_sin));
1057                 }
1058                 rc = connect(IO->SendBuf.fd,
1059                              (struct sockaddr_in *)&IO->ConnectMe->Addr,
1060                              sizeof(struct sockaddr_in));
1061         }
1062
1063         if (rc >= 0){
1064                 SetEVState(IO, eIOConnNow);
1065                 EV_syslog(LOG_DEBUG, "connect() = %d immediate success.\n", IO->SendBuf.fd);
1066                 set_start_callback(event_base, IO, 0);
1067                 return IO->NextState;
1068         }
1069         else if (errno == EINPROGRESS) {
1070                 SetEVState(IO, eIOConnWait);
1071                 EV_syslog(LOG_DEBUG, "connect() = %d have to wait now.\n", IO->SendBuf.fd);
1072
1073                 ev_io_init(&IO->conn_event,
1074                            IO_connestd_callback,
1075                            IO->SendBuf.fd,
1076                            EV_READ|EV_WRITE);
1077
1078                 IO->conn_event.data = IO;
1079
1080                 ev_io_start(event_base, &IO->conn_event);
1081                 ev_timer_start(event_base, &IO->conn_fail);
1082                 return IO->NextState;
1083         }
1084         else {
1085                 SetEVState(IO, eIOConnfail);
1086                 ev_idle_init(&IO->conn_fail_immediate,
1087                              IO_connfailimmediate_callback);
1088                 IO->conn_fail_immediate.data = IO;
1089                 ev_idle_start(event_base, &IO->conn_fail_immediate);
1090
1091                 EV_syslog(LOG_ERR,
1092                           "connect() = %d failed: %s\n",
1093                           IO->SendBuf.fd,
1094                           strerror(errno));
1095
1096                 StrBufPrintf(IO->ErrMsg,
1097                              "Failed to connect: %s",
1098                              strerror(errno));
1099                 return IO->NextState;
1100         }
1101         return IO->NextState;
1102 }
1103
1104 void SetNextTimeout(AsyncIO *IO, double timeout)
1105 {
1106         IO->rw_timeout.repeat = timeout;
1107         ev_timer_again (event_base,  &IO->rw_timeout);
1108 }
1109
1110
1111 eNextState ReAttachIO(AsyncIO *IO,
1112                       void *pData,
1113                       int ReadFirst)
1114 {
1115         SetEVState(IO, eIOAttach);
1116         IO->Data = pData;
1117         become_session(IO->CitContext);
1118         ev_cleanup_start(event_base, &IO->abort_by_shutdown);
1119         if (ReadFirst) {
1120                 IO->NextState = eReadMessage;
1121         }
1122         else {
1123                 IO->NextState = eSendReply;
1124         }
1125         set_start_callback(event_base, IO, 0);
1126
1127         return IO->NextState;
1128 }
1129
1130 void InitIOStruct(AsyncIO *IO,
1131                   void *Data,
1132                   eNextState NextState,
1133                   IO_LineReaderCallback LineReader,
1134                   IO_CallBack DNS_Fail,
1135                   IO_CallBack SendDone,
1136                   IO_CallBack ReadDone,
1137                   IO_CallBack Terminate,
1138                   IO_CallBack DBTerminate,
1139                   IO_CallBack ConnFail,
1140                   IO_CallBack Timeout,
1141                   IO_CallBack ShutdownAbort)
1142 {
1143         IO->Data          = Data;
1144
1145         IO->CitContext    = CloneContext(CC);
1146         IO->CitContext->session_specific_data = Data;
1147         IO->CitContext->IO = IO;
1148
1149         IO->NextState     = NextState;
1150
1151         IO->SendDone      = SendDone;
1152         IO->ReadDone      = ReadDone;
1153         IO->Terminate     = Terminate;
1154         IO->DBTerminate   = DBTerminate;
1155         IO->LineReader    = LineReader;
1156         IO->ConnFail      = ConnFail;
1157         IO->Timeout       = Timeout;
1158         IO->ShutdownAbort = ShutdownAbort;
1159
1160         IO->DNS.Fail      = DNS_Fail;
1161
1162         IO->SendBuf.Buf   = NewStrBufPlain(NULL, 1024);
1163         IO->RecvBuf.Buf   = NewStrBufPlain(NULL, 1024);
1164         IO->IOBuf         = NewStrBuf();
1165         EV_syslog(LOG_DEBUG,
1166                   "EVENT: Session lives at %p IO at %p \n",
1167                   Data, IO);
1168
1169 }
1170
1171 extern int evcurl_init(AsyncIO *IO);
1172
1173 int InitcURLIOStruct(AsyncIO *IO,
1174                      void *Data,
1175                      const char* Desc,
1176                      IO_CallBack SendDone,
1177                      IO_CallBack Terminate,
1178                      IO_CallBack DBTerminate,
1179                      IO_CallBack ShutdownAbort)
1180 {
1181         IO->Data          = Data;
1182
1183         IO->CitContext    = CloneContext(CC);
1184         IO->CitContext->session_specific_data = Data;
1185         IO->CitContext->IO = IO;
1186
1187         IO->SendDone      = SendDone;
1188         IO->Terminate     = Terminate;
1189         IO->DBTerminate   = DBTerminate;
1190         IO->ShutdownAbort = ShutdownAbort;
1191
1192         strcpy(IO->HttpReq.errdesc, Desc);
1193
1194
1195         return  evcurl_init(IO);
1196
1197 }
1198
1199
1200 typedef struct KillOtherSessionContext {
1201         AsyncIO IO;
1202         AsyncIO *OtherOne;
1203 }KillOtherSessionContext;
1204
1205 eNextState KillTerminate(AsyncIO *IO)
1206 {
1207         long id;
1208         KillOtherSessionContext *Ctx = (KillOtherSessionContext*)IO->Data;
1209         EV_syslog(LOG_DEBUG, "%s Exit\n", __FUNCTION__);
1210         id = IO->ID;
1211         FreeAsyncIOContents(IO);
1212         memset(Ctx, 0, sizeof(KillOtherSessionContext));
1213         IO->ID = id; /* just for the case we want to analyze it in a coredump */
1214         free(Ctx);
1215         return eAbort;
1216
1217 }
1218
1219 eNextState KillShutdown(AsyncIO *IO)
1220 {
1221         return eTerminateConnection;
1222 }
1223
1224 eNextState KillOtherContextNow(AsyncIO *IO)
1225 {
1226         KillOtherSessionContext *Ctx = IO->Data;
1227
1228         SetEVState(IO, eKill);
1229
1230         if (Ctx->OtherOne->ShutdownAbort != NULL)
1231                 Ctx->OtherOne->ShutdownAbort(Ctx->OtherOne);
1232         return eTerminateConnection;
1233 }
1234
1235 void KillAsyncIOContext(AsyncIO *IO)
1236 {
1237         KillOtherSessionContext *Ctx;
1238
1239         Ctx = (KillOtherSessionContext*) malloc(sizeof(KillOtherSessionContext));
1240         memset(Ctx, 0, sizeof(KillOtherSessionContext));
1241
1242         InitIOStruct(&Ctx->IO,
1243                      Ctx,
1244                      eReadMessage,
1245                      NULL,
1246                      NULL,
1247                      NULL,
1248                      NULL,
1249                      KillTerminate,
1250                      NULL,
1251                      NULL,
1252                      NULL,
1253                      KillShutdown);
1254
1255         Ctx->OtherOne = IO;
1256
1257         switch(IO->NextState) {
1258         case eSendDNSQuery:
1259         case eReadDNSReply:
1260
1261         case eConnect:
1262         case eSendReply:
1263         case eSendMore:
1264         case eSendFile:
1265
1266         case eReadMessage:
1267         case eReadMore:
1268         case eReadPayload:
1269         case eReadFile:
1270                 QueueEventContext(&Ctx->IO, KillOtherContextNow);
1271                 break;
1272         case eDBQuery:
1273                 QueueDBOperation(&Ctx->IO, KillOtherContextNow);
1274                 break;
1275         case eTerminateConnection:
1276         case eAbort:
1277                 /*hm, its already dying, dunno which Queue its in... */
1278                 free(Ctx);
1279         }
1280
1281 }
1282
1283 extern int DebugEventLoopBacktrace;
1284 void EV_backtrace(AsyncIO *IO)
1285 {
1286 #ifdef HAVE_BACKTRACE
1287         void *stack_frames[50];
1288         size_t size, i;
1289         char **strings;
1290
1291         if ((IO == NULL) || (DebugEventLoopBacktrace == 0))
1292                 return;
1293         size = backtrace(stack_frames, sizeof(stack_frames) / sizeof(void*));
1294         strings = backtrace_symbols(stack_frames, size);
1295         for (i = 0; i < size; i++) {
1296                 if (strings != NULL) {
1297                         EV_syslog(LOG_ALERT, " BT %s\n", strings[i]);
1298                 }
1299                 else {
1300                         EV_syslog(LOG_ALERT, " BT %p\n", stack_frames[i]);
1301                 }
1302         }
1303         free(strings);
1304 #endif
1305 }
1306
1307
1308 ev_tstamp ctdl_ev_now (void)
1309 {
1310         return ev_now(event_base);
1311 }