citadel/event_client.c

   1 /*
   2  * Copyright (c) 1998-2012 by the citadel.org team
   3  *
   4  * This program is open source software; you can redistribute it and/or modify
   5  * it under the terms of the GNU General Public License, version 3.
   6  *
   7  * This program is distributed in the hope that it will be useful,
   8  * but WITHOUT ANY WARRANTY; without even the implied warranty of
   9  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  10  * GNU General Public License for more details.
  11  */
  12 #include "sysdep.h"
  13
  14 #include <stdio.h>
  15 #include <string.h>
  16 #include <syslog.h>
  17 #include <assert.h>
  18 #include <sys/socket.h>
  19 #include <netinet/in.h>
  20 #include <arpa/inet.h>
  21 #if HAVE_BACKTRACE
  22 #include <execinfo.h>
  23 #endif
  24
  25 #include <libcitadel.h>
  26
  27 #include "ctdl_module.h"
  28 #include "event_client.h"
  29
  30 ConstStr IOStates[] = {
  31         {HKEY("DB Queue")},
  32         {HKEY("DB Q Next")},
  33         {HKEY("DB Attach")},
  34         {HKEY("DB Next")},
  35         {HKEY("DB Stop")},
  36         {HKEY("DB Exit")},
  37         {HKEY("DB Terminate")},
  38         {HKEY("IO Queue")},
  39         {HKEY("IO Attach")},
  40         {HKEY("IO Connect Socket")},
  41         {HKEY("IO Abort")},
  42         {HKEY("IO Timeout")},
  43         {HKEY("IO ConnFail")},
  44         {HKEY("IO ConnFail Now")},
  45         {HKEY("IO Conn Now")},
  46         {HKEY("IO Conn Wait")},
  47         {HKEY("Curl Q")},
  48         {HKEY("Curl Start")},
  49         {HKEY("Curl Shotdown")},
  50         {HKEY("Curl More IO")},
  51         {HKEY("Curl Got IO")},
  52         {HKEY("Curl Got Data")},
  53         {HKEY("Curl Got Status")},
  54         {HKEY("C-Ares Start")},
  55         {HKEY("C-Ares IO Done")},
  56         {HKEY("C-Ares Finished")},
  57         {HKEY("C-Ares exit")},
  58         {HKEY("Killing")},
  59         {HKEY("Exit")}
  60 };
  61
  62 void SetEVState(AsyncIO *IO, eIOState State)
  63 {
  64
  65         CitContext* CCC = IO->CitContext;
  66         if (CCC != NULL)
  67                 memcpy(CCC->lastcmdname, IOStates[State].Key, IOStates[State].len + 1);
  68
  69 }
  70
  71 eNextState QueueAnEventContext(AsyncIO *IO);
  72 static void IO_Timeout_callback(struct ev_loop *loop, ev_timer *watcher, int revents);
  73 static void IO_abort_shutdown_callback(struct ev_loop *loop,
  74                                        ev_cleanup *watcher,
  75                                        int revents);
  76
  77
  78 /*------------------------------------------------------------------------------
  79  *                              Server DB IO
  80  *----------------------------------------------------------------------------*/
  81 extern int evdb_count;
  82 extern pthread_mutex_t DBEventQueueMutex;
  83 extern pthread_mutex_t DBEventExitQueueMutex;
  84 extern HashList *DBInboundEventQueue;
  85 extern struct ev_loop *event_db;
  86 extern ev_async DBAddJob;
  87 extern ev_async DBExitEventLoop;
  88
  89 eNextState QueueAnDBOperation(AsyncIO *IO)
  90 {
  91         IOAddHandler *h;
  92         int i;
  93
  94         SetEVState(IO, eDBQ);
  95         h = (IOAddHandler*)malloc(sizeof(IOAddHandler));
  96         h->IO = IO;
  97
  98         assert(IO->ReAttachCB != NULL);
  99
 100         h->EvAttch = IO->ReAttachCB;
 101         ev_cleanup_init(&IO->db_abort_by_shutdown,
 102                         IO_abort_shutdown_callback);
 103         IO->db_abort_by_shutdown.data = IO;
 104
 105         pthread_mutex_lock(&DBEventQueueMutex);
 106         if (DBInboundEventQueue == NULL)
 107         {
 108                 /* shutting down... */
 109                 free(h);
 110                 EVM_syslog(LOG_DEBUG, "DBEVENT Q exiting.\n");
 111                 pthread_mutex_unlock(&DBEventQueueMutex);
 112                 return eAbort;
 113         }
 114         EVM_syslog(LOG_DEBUG, "DBEVENT Q\n");
 115         i = ++evdb_count ;
 116         Put(DBInboundEventQueue, IKEY(i), h, NULL);
 117         pthread_mutex_unlock(&DBEventQueueMutex);
 118
 119         pthread_mutex_lock(&DBEventExitQueueMutex);
 120         if (event_db == NULL)
 121         {
 122                 pthread_mutex_unlock(&DBEventExitQueueMutex);
 123                 return eAbort;
 124         }
 125         ev_async_send (event_db, &DBAddJob);
 126         pthread_mutex_unlock(&DBEventExitQueueMutex);
 127
 128         EVQM_syslog(LOG_DEBUG, "DBEVENT Q Done.\n");
 129         return eDBQuery;
 130 }
 131
 132 void StopDBWatchers(AsyncIO *IO)
 133 {
 134         SetEVState(IO, eDBStop);
 135         ev_cleanup_stop(event_db, &IO->db_abort_by_shutdown);
 136         ev_idle_stop(event_db, &IO->db_unwind_stack);
 137 }
 138
 139 void ShutDownDBCLient(AsyncIO *IO)
 140 {
 141         CitContext *Ctx =IO->CitContext;
 142         become_session(Ctx);
 143
 144         SetEVState(IO, eDBTerm);
 145         EVM_syslog(LOG_DEBUG, "DBEVENT Terminating.\n");
 146         StopDBWatchers(IO);
 147
 148         assert(IO->DBTerminate);
 149         IO->DBTerminate(IO);
 150 }
 151
 152 void
 153 DB_PerformNext(struct ev_loop *loop, ev_idle *watcher, int revents)
 154 {
 155         AsyncIO *IO = watcher->data;
 156
 157         SetEVState(IO, eDBNext);
 158         IO->Now = ev_now(event_db);
 159         EV_syslog(LOG_DEBUG, "%s()", __FUNCTION__);
 160         become_session(IO->CitContext);
 161
 162         ev_idle_stop(event_db, &IO->db_unwind_stack);
 163
 164         assert(IO->NextDBOperation);
 165         switch (IO->NextDBOperation(IO))
 166         {
 167         case eSendReply:
 168                 ev_cleanup_stop(loop, &IO->db_abort_by_shutdown);
 169                 QueueAnEventContext(IO);
 170                 break;
 171         case eDBQuery:
 172                 break;
 173         case eSendDNSQuery:
 174         case eReadDNSReply:
 175         case eConnect:
 176         case eSendMore:
 177         case eSendFile:
 178         case eReadMessage:
 179         case eReadMore:
 180         case eReadPayload:
 181         case eReadFile:
 182                 ev_cleanup_stop(loop, &IO->db_abort_by_shutdown);
 183                 break;
 184         case eTerminateConnection:
 185         case eAbort:
 186                 ev_idle_stop(event_db, &IO->db_unwind_stack);
 187                 ev_cleanup_stop(loop, &IO->db_abort_by_shutdown);
 188                 ShutDownDBCLient(IO);
 189         }
 190 }
 191
 192 eNextState NextDBOperation(AsyncIO *IO, IO_CallBack CB)
 193 {
 194         SetEVState(IO, eQDBNext);
 195         IO->NextDBOperation = CB;
 196         ev_idle_init(&IO->db_unwind_stack,
 197                      DB_PerformNext);
 198         IO->db_unwind_stack.data = IO;
 199         ev_idle_start(event_db, &IO->db_unwind_stack);
 200         return eDBQuery;
 201 }
 202
 203 /*------------------------------------------------------------------------------
 204  *                      Client IO
 205  *----------------------------------------------------------------------------*/
 206 extern int evbase_count;
 207 extern pthread_mutex_t EventQueueMutex;
 208 extern pthread_mutex_t EventExitQueueMutex;
 209 extern HashList *InboundEventQueue;
 210 extern struct ev_loop *event_base;
 211 extern ev_async AddJob;
 212 extern ev_async ExitEventLoop;
 213
 214 static void IO_abort_shutdown_callback(struct ev_loop *loop,
 215                                        ev_cleanup *watcher,
 216                                        int revents)
 217 {
 218         AsyncIO *IO = watcher->data;
 219
 220         SetEVState(IO, eIOAbort);
 221         EV_syslog(LOG_DEBUG, "EVENT Q: %s\n", __FUNCTION__);
 222         IO->Now = ev_now(event_base);
 223         assert(IO->ShutdownAbort);
 224         IO->ShutdownAbort(IO);
 225 }
 226
 227
 228 eNextState QueueAnEventContext(AsyncIO *IO)
 229 {
 230         IOAddHandler *h;
 231         int i;
 232
 233         SetEVState(IO, eIOQ);
 234         h = (IOAddHandler*)malloc(sizeof(IOAddHandler));
 235         h->IO = IO;
 236
 237         assert(IO->ReAttachCB != NULL);
 238
 239         h->EvAttch = IO->ReAttachCB;
 240
 241         ev_cleanup_init(&IO->abort_by_shutdown,
 242                         IO_abort_shutdown_callback);
 243         IO->abort_by_shutdown.data = IO;
 244
 245         pthread_mutex_lock(&EventQueueMutex);
 246         if (InboundEventQueue == NULL)
 247         {
 248                 free(h);
 249                 /* shutting down... */
 250                 EVM_syslog(LOG_DEBUG, "EVENT Q exiting.\n");
 251                 pthread_mutex_unlock(&EventQueueMutex);
 252                 return eAbort;
 253         }
 254         EVM_syslog(LOG_DEBUG, "EVENT Q\n");
 255         i = ++evbase_count;
 256         Put(InboundEventQueue, IKEY(i), h, NULL);
 257         pthread_mutex_unlock(&EventQueueMutex);
 258
 259         pthread_mutex_lock(&EventExitQueueMutex);
 260         if (event_base == NULL) {
 261                 pthread_mutex_unlock(&EventExitQueueMutex);
 262                 return eAbort;
 263         }
 264         ev_async_send (event_base, &AddJob);
 265         pthread_mutex_unlock(&EventExitQueueMutex);
 266         EVM_syslog(LOG_DEBUG, "EVENT Q Done.\n");
 267         return eSendReply;
 268 }
 269
 270 eNextState EventQueueDBOperation(AsyncIO *IO, IO_CallBack CB, int CloseFDs)
 271 {
 272         StopClientWatchers(IO, CloseFDs);
 273         IO->ReAttachCB = CB;
 274         return eDBQuery;
 275 }
 276 eNextState DBQueueEventContext(AsyncIO *IO, IO_CallBack CB)
 277 {
 278         StopDBWatchers(IO);
 279         IO->ReAttachCB = CB;
 280         return eSendReply;
 281 }
 282
 283 eNextState QueueEventContext(AsyncIO *IO, IO_CallBack CB)
 284 {
 285         IO->ReAttachCB = CB;
 286         return QueueAnEventContext(IO);
 287 }
 288
 289 extern eNextState evcurl_handle_start(AsyncIO *IO);
 290
 291 eNextState QueueCurlContext(AsyncIO *IO)
 292 {
 293         IOAddHandler *h;
 294         int i;
 295
 296         SetEVState(IO, eCurlQ);
 297         h = (IOAddHandler*)malloc(sizeof(IOAddHandler));
 298         h->IO = IO;
 299         h->EvAttch = evcurl_handle_start;
 300
 301         pthread_mutex_lock(&EventQueueMutex);
 302         if (InboundEventQueue == NULL)
 303         {
 304                 /* shutting down... */
 305                 free(h);
 306                 EVM_syslog(LOG_DEBUG, "EVENT Q exiting.\n");
 307                 pthread_mutex_unlock(&EventQueueMutex);
 308                 return eAbort;
 309         }
 310
 311         EVM_syslog(LOG_DEBUG, "EVENT Q\n");
 312         i = ++evbase_count;
 313         Put(InboundEventQueue, IKEY(i), h, NULL);
 314         pthread_mutex_unlock(&EventQueueMutex);
 315
 316         pthread_mutex_lock(&EventExitQueueMutex);
 317         if (event_base == NULL) {
 318                 pthread_mutex_unlock(&EventExitQueueMutex);
 319                 return eAbort;
 320         }
 321         ev_async_send (event_base, &AddJob);
 322         pthread_mutex_unlock(&EventExitQueueMutex);
 323
 324         EVM_syslog(LOG_DEBUG, "EVENT Q Done.\n");
 325         return eSendReply;
 326 }
 327
 328 eNextState CurlQueueDBOperation(AsyncIO *IO, IO_CallBack CB)
 329 {
 330         StopCurlWatchers(IO);
 331         IO->ReAttachCB = CB;
 332         return eDBQuery;
 333 }
 334
 335
 336 void FreeAsyncIOContents(AsyncIO *IO)
 337 {
 338         CitContext *Ctx = IO->CitContext;
 339
 340         FreeStrBuf(&IO->IOBuf);
 341         FreeStrBuf(&IO->SendBuf.Buf);
 342         FreeStrBuf(&IO->RecvBuf.Buf);
 343
 344         FreeURL(&IO->ConnectMe);
 345         FreeStrBuf(&IO->HttpReq.ReplyData);
 346
 347         if (Ctx) {
 348                 Ctx->state = CON_IDLE;
 349                 Ctx->kill_me = 1;
 350                 IO->CitContext = NULL;
 351         }
 352 }
 353
 354
 355 void DestructCAres(AsyncIO *IO);
 356 void StopClientWatchers(AsyncIO *IO, int CloseFD)
 357 {
 358         EVM_syslog(LOG_DEBUG, "EVENT StopClientWatchers");
 359
 360         DestructCAres(IO);
 361
 362         ev_timer_stop (event_base, &IO->rw_timeout);
 363         ev_timer_stop(event_base, &IO->conn_fail);
 364         ev_idle_stop(event_base, &IO->unwind_stack);
 365         ev_cleanup_stop(event_base, &IO->abort_by_shutdown);
 366
 367         ev_io_stop(event_base, &IO->conn_event);
 368         ev_io_stop(event_base, &IO->send_event);
 369         ev_io_stop(event_base, &IO->recv_event);
 370
 371         if (CloseFD && (IO->SendBuf.fd > 0)) {
 372                 close(IO->SendBuf.fd);
 373                 IO->SendBuf.fd = 0;
 374                 IO->RecvBuf.fd = 0;
 375         }
 376 }
 377
 378 void StopCurlWatchers(AsyncIO *IO)
 379 {
 380         EVM_syslog(LOG_DEBUG, "EVENT StopCurlWatchers \n");
 381
 382         ev_timer_stop (event_base, &IO->rw_timeout);
 383         ev_timer_stop(event_base, &IO->conn_fail);
 384         ev_idle_stop(event_base, &IO->unwind_stack);
 385         ev_cleanup_stop(event_base, &IO->abort_by_shutdown);
 386
 387         ev_io_stop(event_base, &IO->conn_event);
 388         ev_io_stop(event_base, &IO->send_event);
 389         ev_io_stop(event_base, &IO->recv_event);
 390
 391         curl_easy_cleanup(IO->HttpReq.chnd);
 392         IO->HttpReq.chnd = NULL;
 393
 394         if (IO->SendBuf.fd != 0) {
 395                 close(IO->SendBuf.fd);
 396         }
 397         IO->SendBuf.fd = 0;
 398         IO->RecvBuf.fd = 0;
 399 }
 400
 401 eNextState ShutDownCLient(AsyncIO *IO)
 402 {
 403         CitContext *Ctx =IO->CitContext;
 404
 405         SetEVState(IO, eExit);
 406         become_session(Ctx);
 407
 408         EVM_syslog(LOG_DEBUG, "EVENT Terminating \n");
 409
 410         StopClientWatchers(IO, 1);
 411
 412         if (IO->DNS.Channel != NULL) {
 413                 ares_destroy(IO->DNS.Channel);
 414                 EV_DNS_LOG_STOP(DNS.recv_event);
 415                 EV_DNS_LOG_STOP(DNS.send_event);
 416                 ev_io_stop(event_base, &IO->DNS.recv_event);
 417                 ev_io_stop(event_base, &IO->DNS.send_event);
 418                 IO->DNS.Channel = NULL;
 419         }
 420         assert(IO->Terminate);
 421         return IO->Terminate(IO);
 422 }
 423
 424 void PostInbound(AsyncIO *IO)
 425 {
 426
 427         switch (IO->NextState) {
 428         case eSendFile:
 429                 ev_io_start(event_base, &IO->send_event);
 430                 break;
 431         case eSendReply:
 432         case eSendMore:
 433                 assert(IO->SendDone);
 434                 IO->NextState = IO->SendDone(IO);
 435                 switch (IO->NextState)
 436                 {
 437                 case eSendFile:
 438                 case eSendReply:
 439                 case eSendMore:
 440                 case eReadMessage:
 441                 case eReadPayload:
 442                 case eReadMore:
 443                 case eReadFile:
 444                         ev_io_start(event_base, &IO->send_event);
 445                         break;
 446                 case eDBQuery:
 447                         StopClientWatchers(IO, 0);
 448                         QueueAnDBOperation(IO);
 449                 default:
 450                         break;
 451                 }
 452                 break;
 453         case eReadPayload:
 454         case eReadMore:
 455         case eReadFile:
 456                 ev_io_start(event_base, &IO->recv_event);
 457                 break;
 458         case eTerminateConnection:
 459         case eAbort:
 460                 if (ShutDownCLient(IO) == eDBQuery) {
 461                         QueueAnDBOperation(IO);
 462                 }
 463                 break;
 464         case eSendDNSQuery:
 465         case eReadDNSReply:
 466         case eConnect:
 467         case eReadMessage:
 468                 break;
 469         case eDBQuery:
 470                 QueueAnDBOperation(IO);
 471         }
 472 }
 473 eReadState HandleInbound(AsyncIO *IO)
 474 {
 475         const char *Err = NULL;
 476         eReadState Finished = eBufferNotEmpty;
 477
 478         become_session(IO->CitContext);
 479
 480         while ((Finished == eBufferNotEmpty) &&
 481                ((IO->NextState == eReadMessage)||
 482                 (IO->NextState == eReadMore)||
 483                 (IO->NextState == eReadFile)||
 484                 (IO->NextState == eReadPayload)))
 485         {
 486                 /* Reading lines...
 487                  * lex line reply in callback,
 488                  * or do it ourselves.
 489                  * i.e. as nnn-blabla means continue reading in SMTP
 490                  */
 491                 if ((IO->NextState == eReadFile) &&
 492                     (Finished == eBufferNotEmpty))
 493                 {
 494                         Finished = WriteIOBAlreadyRead(&IO->IOB, &Err);
 495                         if (Finished == eReadSuccess)
 496                         {
 497                                 IO->NextState = eSendReply;
 498                         }
 499                 }
 500                 else if (IO->LineReader)
 501                         Finished = IO->LineReader(IO);
 502                 else
 503                         Finished = StrBufChunkSipLine(IO->IOBuf,
 504                                                       &IO->RecvBuf);
 505
 506                 switch (Finished) {
 507                 case eMustReadMore: /// read new from socket...
 508                         break;
 509                 case eBufferNotEmpty: /* shouldn't happen... */
 510                 case eReadSuccess: /// done for now...
 511                         break;
 512                 case eReadFail: /// WHUT?
 513                                 ///todo: shut down!
 514                         break;
 515                 }
 516
 517                 if (Finished != eMustReadMore) {
 518                         ev_io_stop(event_base, &IO->recv_event);
 519                         IO->NextState = IO->ReadDone(IO);
 520                         if  (IO->NextState == eDBQuery) {
 521                                 if (QueueAnDBOperation(IO) == eAbort)
 522                                         return eReadFail;
 523                                 else
 524                                         return eReadSuccess;
 525                         }
 526                         else {
 527                                 Finished = StrBufCheckBuffer(&IO->RecvBuf);
 528                         }
 529                 }
 530         }
 531
 532         PostInbound(IO);
 533
 534         return Finished;
 535 }
 536
 537
 538 static void
 539 IO_send_callback(struct ev_loop *loop, ev_io *watcher, int revents)
 540 {
 541         int rc;
 542         AsyncIO *IO = watcher->data;
 543         const char *errmsg = NULL;
 544
 545         IO->Now = ev_now(event_base);
 546         become_session(IO->CitContext);
 547 #ifdef BIGBAD_IODBG
 548         {
 549                 int rv = 0;
 550                 char fn [SIZ];
 551                 FILE *fd;
 552                 const char *pch = ChrPtr(IO->SendBuf.Buf);
 553                 const char *pchh = IO->SendBuf.ReadWritePointer;
 554                 long nbytes;
 555
 556                 if (pchh == NULL)
 557                         pchh = pch;
 558
 559                 nbytes = StrLength(IO->SendBuf.Buf) - (pchh - pch);
 560                 snprintf(fn, SIZ, "/tmp/foolog_ev_%s.%d",
 561                          ((CitContext*)(IO->CitContext))->ServiceName,
 562                          IO->SendBuf.fd);
 563
 564                 fd = fopen(fn, "a+");
 565                 fprintf(fd, "Send: BufSize: %ld BufContent: [",
 566                         nbytes);
 567                 rv = fwrite(pchh, nbytes, 1, fd);
 568                 if (!rv) printf("failed to write debug to %s!\n", fn);
 569                 fprintf(fd, "]\n");
 570 #endif
 571                 switch (IO->NextState) {
 572                 case eSendFile:
 573                         rc = FileSendChunked(&IO->IOB, &errmsg);
 574                         if (rc < 0)
 575                                 StrBufPlain(IO->ErrMsg, errmsg, -1);
 576                         break;
 577                 default:
 578                         rc = StrBuf_write_one_chunk_callback(IO->SendBuf.fd,
 579                                                              0,
 580                                                              &IO->SendBuf);
 581                 }
 582
 583 #ifdef BIGBAD_IODBG
 584                 fprintf(fd, "Sent: BufSize: %d bytes.\n", rc);
 585                 fclose(fd);
 586         }
 587 #endif
 588         if (rc == 0)
 589         {
 590                 ev_io_stop(event_base, &IO->send_event);
 591                 switch (IO->NextState) {
 592                 case eSendMore:
 593                         assert(IO->SendDone);
 594                         IO->NextState = IO->SendDone(IO);
 595
 596                         if ((IO->NextState == eTerminateConnection) ||
 597                             (IO->NextState == eAbort) )
 598                                 ShutDownCLient(IO);
 599                         else {
 600                                 ev_io_start(event_base, &IO->send_event);
 601                         }
 602                         break;
 603                 case eSendFile:
 604                         if (IO->IOB.ChunkSendRemain > 0) {
 605                                 ev_io_start(event_base, &IO->recv_event);
 606                                 SetNextTimeout(IO, 100.0);
 607
 608                         } else {
 609                                 assert(IO->ReadDone);
 610                                 IO->NextState = IO->ReadDone(IO);
 611                                 switch(IO->NextState) {
 612                                 case eSendDNSQuery:
 613                                 case eReadDNSReply:
 614                                 case eDBQuery:
 615                                 case eConnect:
 616                                         break;
 617                                 case eSendReply:
 618                                 case eSendMore:
 619                                 case eSendFile:
 620                                         ev_io_start(event_base,
 621                                                     &IO->send_event);
 622                                         break;
 623                                 case eReadMessage:
 624                                 case eReadMore:
 625                                 case eReadPayload:
 626                                 case eReadFile:
 627                                         break;
 628                                 case eTerminateConnection:
 629                                 case eAbort:
 630                                         break;
 631                                 }
 632                         }
 633                         break;
 634                 case eSendReply:
 635                     if (StrBufCheckBuffer(&IO->SendBuf) != eReadSuccess)
 636                         break;
 637                     IO->NextState = eReadMore;
 638                 case eReadMore:
 639                 case eReadMessage:
 640                 case eReadPayload:
 641                 case eReadFile:
 642                         if (StrBufCheckBuffer(&IO->RecvBuf) == eBufferNotEmpty)
 643                         {
 644                                 HandleInbound(IO);
 645                         }
 646                         else {
 647                                 ev_io_start(event_base, &IO->recv_event);
 648                         }
 649
 650                         break;
 651                 case eDBQuery:
 652                         /*
 653                          * we now live in another queue,
 654                          * so we have to unregister.
 655                          */
 656                         ev_cleanup_stop(loop, &IO->abort_by_shutdown);
 657                         break;
 658                 case eSendDNSQuery:
 659                 case eReadDNSReply:
 660                 case eConnect:
 661                 case eTerminateConnection:
 662                 case eAbort:
 663                         break;
 664                 }
 665         }
 666         else if (rc < 0) {
 667                 if (errno != EAGAIN) {
 668                         StopClientWatchers(IO, 1);
 669                         EV_syslog(LOG_DEBUG,
 670                                   "IO_send_callback(): Socket Invalid! [%d] [%s] [%d]\n",
 671                                   errno, strerror(errno), IO->SendBuf.fd);
 672                         StrBufPrintf(IO->ErrMsg,
 673                                      "Socket Invalid! [%s]",
 674                                      strerror(errno));
 675                         SetNextTimeout(IO, 0.01);
 676                 }
 677         }
 678         /* else : must write more. */
 679 }
 680 static void
 681 set_start_callback(struct ev_loop *loop, AsyncIO *IO, int revents)
 682 {
 683         ev_timer_stop(event_base, &IO->conn_fail);
 684         ev_timer_start(event_base, &IO->rw_timeout);
 685
 686         switch(IO->NextState) {
 687         case eReadMore:
 688         case eReadMessage:
 689         case eReadFile:
 690                 StrBufAppendBufPlain(IO->ErrMsg, HKEY("[while waiting for greeting]"), 0);
 691                 ev_io_start(event_base, &IO->recv_event);
 692                 break;
 693         case eSendReply:
 694         case eSendMore:
 695         case eReadPayload:
 696         case eSendFile:
 697                 become_session(IO->CitContext);
 698                 IO_send_callback(loop, &IO->send_event, revents);
 699                 break;
 700         case eDBQuery:
 701         case eSendDNSQuery:
 702         case eReadDNSReply:
 703         case eConnect:
 704         case eTerminateConnection:
 705         case eAbort:
 706                 /// TODO: WHUT?
 707                 break;
 708         }
 709 }
 710
 711 static void
 712 IO_Timeout_callback(struct ev_loop *loop, ev_timer *watcher, int revents)
 713 {
 714         AsyncIO *IO = watcher->data;
 715
 716         SetEVState(IO, eIOTimeout);
 717         IO->Now = ev_now(event_base);
 718         ev_timer_stop (event_base, &IO->rw_timeout);
 719         become_session(IO->CitContext);
 720
 721         if (IO->SendBuf.fd != 0)
 722         {
 723                 ev_io_stop(event_base, &IO->send_event);
 724                 ev_io_stop(event_base, &IO->recv_event);
 725                 ev_timer_stop (event_base, &IO->rw_timeout);
 726                 close(IO->SendBuf.fd);
 727                 IO->SendBuf.fd = IO->RecvBuf.fd = 0;
 728         }
 729
 730         assert(IO->Timeout);
 731         switch (IO->Timeout(IO))
 732         {
 733         case eAbort:
 734                 ShutDownCLient(IO);
 735         default:
 736                 break;
 737         }
 738 }
 739
 740 static void
 741 IO_connfail_callback(struct ev_loop *loop, ev_timer *watcher, int revents)
 742 {
 743         AsyncIO *IO = watcher->data;
 744
 745         SetEVState(IO, eIOConnfail);
 746         IO->Now = ev_now(event_base);
 747         ev_timer_stop (event_base, &IO->conn_fail);
 748
 749         if (IO->SendBuf.fd != 0)
 750         {
 751                 ev_io_stop(loop, &IO->conn_event);
 752                 ev_io_stop(event_base, &IO->send_event);
 753                 ev_io_stop(event_base, &IO->recv_event);
 754                 ev_timer_stop (event_base, &IO->rw_timeout);
 755                 close(IO->SendBuf.fd);
 756                 IO->SendBuf.fd = IO->RecvBuf.fd = 0;
 757         }
 758         become_session(IO->CitContext);
 759
 760         assert(IO->ConnFail);
 761         switch (IO->ConnFail(IO))
 762         {
 763         case eAbort:
 764                 ShutDownCLient(IO);
 765         default:
 766                 break;
 767
 768         }
 769 }
 770
 771 static void
 772 IO_connfailimmediate_callback(struct ev_loop *loop,
 773                               ev_idle *watcher,
 774                               int revents)
 775 {
 776         AsyncIO *IO = watcher->data;
 777
 778         SetEVState(IO, eIOConnfailNow);
 779         IO->Now = ev_now(event_base);
 780         ev_idle_stop (event_base, &IO->conn_fail_immediate);
 781
 782         if (IO->SendBuf.fd != 0)
 783         {
 784                 close(IO->SendBuf.fd);
 785                 IO->SendBuf.fd = IO->RecvBuf.fd = 0;
 786         }
 787         become_session(IO->CitContext);
 788
 789         assert(IO->ConnFail);
 790         switch (IO->ConnFail(IO))
 791         {
 792         case eAbort:
 793                 ShutDownCLient(IO);
 794         default:
 795                 break;
 796
 797         }
 798 }
 799
 800 static void
 801 IO_connestd_callback(struct ev_loop *loop, ev_io *watcher, int revents)
 802 {
 803         AsyncIO *IO = watcher->data;
 804         int             so_err = 0;
 805         socklen_t       lon = sizeof(so_err);
 806         int             err;
 807
 808         SetEVState(IO, eIOConnNow);
 809         IO->Now = ev_now(event_base);
 810         EVM_syslog(LOG_DEBUG, "connect() succeeded.\n");
 811
 812         ev_io_stop(loop, &IO->conn_event);
 813         ev_timer_stop(event_base, &IO->conn_fail);
 814
 815         err = getsockopt(IO->SendBuf.fd,
 816                          SOL_SOCKET,
 817                          SO_ERROR,
 818                          (void*)&so_err,
 819                          &lon);
 820
 821         if ((err == 0) && (so_err != 0))
 822         {
 823                 EV_syslog(LOG_DEBUG, "connect() failed [%d][%s]\n",
 824                           so_err,
 825                           strerror(so_err));
 826                 IO_connfail_callback(loop, &IO->conn_fail, revents);
 827
 828         }
 829         else
 830         {
 831                 EVM_syslog(LOG_DEBUG, "connect() succeeded\n");
 832                 set_start_callback(loop, IO, revents);
 833         }
 834 }
 835
 836 static void
 837 IO_recv_callback(struct ev_loop *loop, ev_io *watcher, int revents)
 838 {
 839         const char *errmsg;
 840         ssize_t nbytes;
 841         AsyncIO *IO = watcher->data;
 842
 843         IO->Now = ev_now(event_base);
 844         switch (IO->NextState) {
 845         case eReadFile:
 846                 nbytes = FileRecvChunked(&IO->IOB, &errmsg);
 847                 if (nbytes < 0)
 848                         StrBufPlain(IO->ErrMsg, errmsg, -1);
 849                 else
 850                 {
 851                         if (IO->IOB.ChunkSendRemain == 0)
 852                         {
 853                                 IO->NextState = eSendReply;
 854                                 assert(IO->ReadDone);
 855                                 ev_io_stop(event_base, &IO->recv_event);
 856                                 PostInbound(IO);
 857                                 return;
 858                         }
 859                         else
 860                                 return;
 861                 }
 862                 break;
 863         default:
 864                 nbytes = StrBuf_read_one_chunk_callback(IO->RecvBuf.fd,
 865                                                         0,
 866                                                         &IO->RecvBuf);
 867                 break;
 868         }
 869
 870 #ifdef BIGBAD_IODBG
 871         {
 872                 long nbytes;
 873                 int rv = 0;
 874                 char fn [SIZ];
 875                 FILE *fd;
 876                 const char *pch = ChrPtr(IO->RecvBuf.Buf);
 877                 const char *pchh = IO->RecvBuf.ReadWritePointer;
 878
 879                 if (pchh == NULL)
 880                         pchh = pch;
 881
 882                 nbytes = StrLength(IO->RecvBuf.Buf) - (pchh - pch);
 883                 snprintf(fn, SIZ, "/tmp/foolog_ev_%s.%d",
 884                          ((CitContext*)(IO->CitContext))->ServiceName,
 885                          IO->SendBuf.fd);
 886
 887                 fd = fopen(fn, "a+");
 888                 fprintf(fd, "Read: BufSize: %ld BufContent: [",
 889                         nbytes);
 890                 rv = fwrite(pchh, nbytes, 1, fd);
 891                 if (!rv) printf("failed to write debug to %s!\n", fn);
 892                 fprintf(fd, "]\n");
 893                 fclose(fd);
 894         }
 895 #endif
 896         if (nbytes > 0) {
 897                 HandleInbound(IO);
 898         } else if (nbytes == 0) {
 899                 StopClientWatchers(IO, 1);
 900                 SetNextTimeout(IO, 0.01);
 901                 return;
 902         } else if (nbytes == -1) {
 903                 if (errno != EAGAIN) {
 904                         // FD is gone. kick it.
 905                         StopClientWatchers(IO, 1);
 906                         EV_syslog(LOG_DEBUG,
 907                                   "IO_recv_callback(): Socket Invalid! [%d] [%s] [%d]\n",
 908                                   errno, strerror(errno), IO->SendBuf.fd);
 909                         StrBufPrintf(IO->ErrMsg,
 910                                      "Socket Invalid! [%s]",
 911                                      strerror(errno));
 912                         SetNextTimeout(IO, 0.01);
 913                 }
 914                 return;
 915         }
 916 }
 917
 918 void
 919 IO_postdns_callback(struct ev_loop *loop, ev_idle *watcher, int revents)
 920 {
 921         AsyncIO *IO = watcher->data;
 922
 923         SetEVState(IO, eCaresFinished);
 924         IO->Now = ev_now(event_base);
 925         EV_syslog(LOG_DEBUG, "event: %s\n", __FUNCTION__);
 926         become_session(IO->CitContext);
 927         assert(IO->DNS.Query->PostDNS);
 928         switch (IO->DNS.Query->PostDNS(IO))
 929         {
 930         case eAbort:
 931                 assert(IO->DNS.Fail);
 932                 switch (IO->DNS.Fail(IO)) {
 933                 case eAbort:
 934 ////                    StopClientWatchers(IO);
 935                         ShutDownCLient(IO);
 936                         break;
 937                 case eDBQuery:
 938                         StopClientWatchers(IO, 0);
 939                         QueueAnDBOperation(IO);
 940                         break;
 941                 default:
 942                         break;
 943                 }
 944         case eDBQuery:
 945                 StopClientWatchers(IO, 0);
 946                 QueueAnDBOperation(IO);
 947                 break;
 948         default:
 949                 break;
 950         }
 951 }
 952
 953
 954 eNextState EvConnectSock(AsyncIO *IO,
 955                          double conn_timeout,
 956                          double first_rw_timeout,
 957                          int ReadFirst)
 958 {
 959         struct sockaddr_in egress_sin;
 960         int fdflags;
 961         int rc = -1;
 962
 963         SetEVState(IO, eIOConnectSock);
 964         become_session(IO->CitContext);
 965
 966         if (ReadFirst) {
 967                 IO->NextState = eReadMessage;
 968         }
 969         else {
 970                 IO->NextState = eSendReply;
 971         }
 972
 973         IO->SendBuf.fd = IO->RecvBuf.fd =
 974                 socket(
 975                         (IO->ConnectMe->IPv6)?PF_INET6:PF_INET,
 976                         SOCK_STREAM,
 977                         IPPROTO_TCP);
 978
 979         if (IO->SendBuf.fd < 0) {
 980                 EV_syslog(LOG_ERR,
 981                           "EVENT: socket() failed: %s\n",
 982                           strerror(errno));
 983
 984                 StrBufPrintf(IO->ErrMsg,
 985                              "Failed to create socket: %s",
 986                              strerror(errno));
 987                 IO->SendBuf.fd = IO->RecvBuf.fd = 0;
 988                 return eAbort;
 989         }
 990         fdflags = fcntl(IO->SendBuf.fd, F_GETFL);
 991         if (fdflags < 0) {
 992                 EV_syslog(LOG_ERR,
 993                           "EVENT: unable to get socket %d flags! %s \n",
 994                           IO->SendBuf.fd,
 995                           strerror(errno));
 996                 StrBufPrintf(IO->ErrMsg,
 997                              "Failed to get socket %d flags: %s",
 998                              IO->SendBuf.fd,
 999                              strerror(errno));
1000                 close(IO->SendBuf.fd);
1001                 IO->SendBuf.fd = IO->RecvBuf.fd = 0;
1002                 return eAbort;
1003         }
1004         fdflags = fdflags | O_NONBLOCK;
1005         if (fcntl(IO->SendBuf.fd, F_SETFL, fdflags) < 0) {
1006                 EV_syslog(
1007                         LOG_ERR,
1008                         "EVENT: unable to set socket %d nonblocking flags! %s \n",
1009                         IO->SendBuf.fd,
1010                         strerror(errno));
1011                 StrBufPrintf(IO->ErrMsg,
1012                              "Failed to set socket flags: %s",
1013                              strerror(errno));
1014                 close(IO->SendBuf.fd);
1015                 IO->SendBuf.fd = IO->RecvBuf.fd = 0;
1016                 return eAbort;
1017         }
1018 /* TODO: maye we could use offsetof() to calc the position of data...
1019  * http://doc.dvgu.ru/devel/ev.html#associating_custom_data_with_a_watcher
1020  */
1021         ev_io_init(&IO->recv_event, IO_recv_callback, IO->RecvBuf.fd, EV_READ);
1022         IO->recv_event.data = IO;
1023         ev_io_init(&IO->send_event, IO_send_callback, IO->SendBuf.fd, EV_WRITE);
1024         IO->send_event.data = IO;
1025
1026         ev_timer_init(&IO->conn_fail, IO_connfail_callback, conn_timeout, 0);
1027         IO->conn_fail.data = IO;
1028         ev_timer_init(&IO->rw_timeout, IO_Timeout_callback, first_rw_timeout,0);
1029         IO->rw_timeout.data = IO;
1030
1031
1032
1033
1034         /* for debugging you may bypass it like this:
1035          * IO->Addr.sin_addr.s_addr = inet_addr("127.0.0.1");
1036          * ((struct sockaddr_in)IO->ConnectMe->Addr).sin_addr.s_addr =
1037          *   inet_addr("127.0.0.1");
1038          */
1039         if (IO->ConnectMe->IPv6) {
1040                 rc = connect(IO->SendBuf.fd,
1041                              &IO->ConnectMe->Addr,
1042                              sizeof(struct sockaddr_in6));
1043         }
1044         else {
1045                 /* If citserver is bound to a specific IP address on the host, make
1046                  * sure we use that address for outbound connections.
1047                  */
1048
1049                 memset(&egress_sin, 0, sizeof(egress_sin));
1050                 egress_sin.sin_family = AF_INET;
1051                 if (!IsEmptyStr(config.c_ip_addr)) {
1052                         egress_sin.sin_addr.s_addr = inet_addr(config.c_ip_addr);
1053                         if (egress_sin.sin_addr.s_addr == !INADDR_ANY) {
1054                                 egress_sin.sin_addr.s_addr = INADDR_ANY;
1055                         }
1056
1057                         /* If this bind fails, no problem; we can still use INADDR_ANY */
1058                         bind(IO->SendBuf.fd, (struct sockaddr *)&egress_sin, sizeof(egress_sin));
1059                 }
1060                 rc = connect(IO->SendBuf.fd,
1061                              (struct sockaddr_in *)&IO->ConnectMe->Addr,
1062                              sizeof(struct sockaddr_in));
1063         }
1064
1065         if (rc >= 0){
1066                 SetEVState(IO, eIOConnNow);
1067                 EV_syslog(LOG_DEBUG, "connect() = %d immediate success.\n", IO->SendBuf.fd);
1068                 set_start_callback(event_base, IO, 0);
1069                 return IO->NextState;
1070         }
1071         else if (errno == EINPROGRESS) {
1072                 SetEVState(IO, eIOConnWait);
1073                 EV_syslog(LOG_DEBUG, "connect() = %d have to wait now.\n", IO->SendBuf.fd);
1074
1075                 ev_io_init(&IO->conn_event,
1076                            IO_connestd_callback,
1077                            IO->SendBuf.fd,
1078                            EV_READ|EV_WRITE);
1079
1080                 IO->conn_event.data = IO;
1081
1082                 ev_io_start(event_base, &IO->conn_event);
1083                 ev_timer_start(event_base, &IO->conn_fail);
1084                 return IO->NextState;
1085         }
1086         else {
1087                 SetEVState(IO, eIOConnfail);
1088                 ev_idle_init(&IO->conn_fail_immediate,
1089                              IO_connfailimmediate_callback);
1090                 IO->conn_fail_immediate.data = IO;
1091                 ev_idle_start(event_base, &IO->conn_fail_immediate);
1092
1093                 EV_syslog(LOG_ERR,
1094                           "connect() = %d failed: %s\n",
1095                           IO->SendBuf.fd,
1096                           strerror(errno));
1097
1098                 StrBufPrintf(IO->ErrMsg,
1099                              "Failed to connect: %s",
1100                              strerror(errno));
1101                 return IO->NextState;
1102         }
1103         return IO->NextState;
1104 }
1105
1106 void SetNextTimeout(AsyncIO *IO, double timeout)
1107 {
1108         IO->rw_timeout.repeat = timeout;
1109         ev_timer_again (event_base,  &IO->rw_timeout);
1110 }
1111
1112
1113 eNextState ReAttachIO(AsyncIO *IO,
1114                       void *pData,
1115                       int ReadFirst)
1116 {
1117         SetEVState(IO, eIOAttach);
1118         IO->Data = pData;
1119         become_session(IO->CitContext);
1120         ev_cleanup_start(event_base, &IO->abort_by_shutdown);
1121         if (ReadFirst) {
1122                 IO->NextState = eReadMessage;
1123         }
1124         else {
1125                 IO->NextState = eSendReply;
1126         }
1127         set_start_callback(event_base, IO, 0);
1128
1129         return IO->NextState;
1130 }
1131
1132 void InitIOStruct(AsyncIO *IO,
1133                   void *Data,
1134                   eNextState NextState,
1135                   IO_LineReaderCallback LineReader,
1136                   IO_CallBack DNS_Fail,
1137                   IO_CallBack SendDone,
1138                   IO_CallBack ReadDone,
1139                   IO_CallBack Terminate,
1140                   IO_CallBack DBTerminate,
1141                   IO_CallBack ConnFail,
1142                   IO_CallBack Timeout,
1143                   IO_CallBack ShutdownAbort)
1144 {
1145         IO->Data          = Data;
1146
1147         IO->CitContext    = CloneContext(CC);
1148         IO->CitContext->session_specific_data = Data;
1149         IO->CitContext->IO = IO;
1150
1151         IO->NextState     = NextState;
1152
1153         IO->SendDone      = SendDone;
1154         IO->ReadDone      = ReadDone;
1155         IO->Terminate     = Terminate;
1156         IO->DBTerminate   = DBTerminate;
1157         IO->LineReader    = LineReader;
1158         IO->ConnFail      = ConnFail;
1159         IO->Timeout       = Timeout;
1160         IO->ShutdownAbort = ShutdownAbort;
1161
1162         IO->DNS.Fail      = DNS_Fail;
1163
1164         IO->SendBuf.Buf   = NewStrBufPlain(NULL, 1024);
1165         IO->RecvBuf.Buf   = NewStrBufPlain(NULL, 1024);
1166         IO->IOBuf         = NewStrBuf();
1167         EV_syslog(LOG_DEBUG,
1168                   "EVENT: Session lives at %p IO at %p \n",
1169                   Data, IO);
1170
1171 }
1172
1173 extern int evcurl_init(AsyncIO *IO);
1174
1175 int InitcURLIOStruct(AsyncIO *IO,
1176                      void *Data,
1177                      const char* Desc,
1178                      IO_CallBack SendDone,
1179                      IO_CallBack Terminate,
1180                      IO_CallBack DBTerminate,
1181                      IO_CallBack ShutdownAbort)
1182 {
1183         IO->Data          = Data;
1184
1185         IO->CitContext    = CloneContext(CC);
1186         IO->CitContext->session_specific_data = Data;
1187         IO->CitContext->IO = IO;
1188
1189         IO->SendDone      = SendDone;
1190         IO->Terminate     = Terminate;
1191         IO->DBTerminate   = DBTerminate;
1192         IO->ShutdownAbort = ShutdownAbort;
1193
1194         strcpy(IO->HttpReq.errdesc, Desc);
1195
1196
1197         return  evcurl_init(IO);
1198
1199 }
1200
1201
1202 typedef struct KillOtherSessionContext {
1203         AsyncIO IO;
1204         AsyncIO *OtherOne;
1205 }KillOtherSessionContext;
1206
1207 eNextState KillTerminate(AsyncIO *IO)
1208 {
1209         long id;
1210         KillOtherSessionContext *Ctx = (KillOtherSessionContext*)IO->Data;
1211         EV_syslog(LOG_DEBUG, "%s Exit\n", __FUNCTION__);
1212         id = IO->ID;
1213         FreeAsyncIOContents(IO);
1214         memset(Ctx, 0, sizeof(KillOtherSessionContext));
1215         IO->ID = id; /* just for the case we want to analyze it in a coredump */
1216         free(Ctx);
1217         return eAbort;
1218
1219 }
1220
1221 eNextState KillShutdown(AsyncIO *IO)
1222 {
1223         return eTerminateConnection;
1224 }
1225
1226 eNextState KillOtherContextNow(AsyncIO *IO)
1227 {
1228         KillOtherSessionContext *Ctx = IO->Data;
1229
1230         SetEVState(IO, eKill);
1231
1232         if (Ctx->OtherOne->ShutdownAbort != NULL) {
1233                 Ctx->OtherOne->NextState = eAbort;
1234                 if (Ctx->OtherOne->ShutdownAbort(Ctx->OtherOne) == eDBQuery) {
1235                         StopClientWatchers(Ctx->OtherOne, 0);
1236                         QueueAnDBOperation(Ctx->OtherOne);
1237                 }
1238         }
1239         return eTerminateConnection;
1240 }
1241
1242 void KillAsyncIOContext(AsyncIO *IO)
1243 {
1244         KillOtherSessionContext *Ctx;
1245
1246         Ctx = (KillOtherSessionContext*) malloc(sizeof(KillOtherSessionContext));
1247         memset(Ctx, 0, sizeof(KillOtherSessionContext));
1248
1249         InitIOStruct(&Ctx->IO,
1250                      Ctx,
1251                      eReadMessage,
1252                      NULL,
1253                      NULL,
1254                      NULL,
1255                      NULL,
1256                      KillTerminate,
1257                      NULL,
1258                      NULL,
1259                      NULL,
1260                      KillShutdown);
1261
1262         Ctx->OtherOne = IO;
1263
1264         switch(IO->NextState) {
1265         case eSendDNSQuery:
1266         case eReadDNSReply:
1267
1268         case eConnect:
1269         case eSendReply:
1270         case eSendMore:
1271         case eSendFile:
1272
1273         case eReadMessage:
1274         case eReadMore:
1275         case eReadPayload:
1276         case eReadFile:
1277                 Ctx->IO.ReAttachCB = KillOtherContextNow;
1278                 QueueAnEventContext(&Ctx->IO);
1279                 break;
1280         case eDBQuery:
1281                 Ctx->IO.ReAttachCB = KillOtherContextNow;
1282                 QueueAnDBOperation(&Ctx->IO);
1283                 break;
1284         case eTerminateConnection:
1285         case eAbort:
1286                 /*hm, its already dying, dunno which Queue its in... */
1287                 free(Ctx);
1288         }
1289
1290 }
1291
1292 extern int DebugEventLoopBacktrace;
1293 void EV_backtrace(AsyncIO *IO)
1294 {
1295 #ifdef HAVE_BACKTRACE
1296         void *stack_frames[50];
1297         size_t size, i;
1298         char **strings;
1299
1300         if ((IO == NULL) || (DebugEventLoopBacktrace == 0))
1301                 return;
1302         size = backtrace(stack_frames, sizeof(stack_frames) / sizeof(void*));
1303         strings = backtrace_symbols(stack_frames, size);
1304         for (i = 0; i < size; i++) {
1305                 if (strings != NULL) {
1306                         EV_syslog(LOG_ALERT, " BT %s\n", strings[i]);
1307                 }
1308                 else {
1309                         EV_syslog(LOG_ALERT, " BT %p\n", stack_frames[i]);
1310                 }
1311         }
1312         free(strings);
1313 #endif
1314 }
1315
1316
1317 ev_tstamp ctdl_ev_now (void)
1318 {
1319         return ev_now(event_base);
1320 }