citadel/event_client.c

   1 /*
   2  * Copyright (c) 1998-2012 by the citadel.org team
   3  *
   4  * This program is open source software; you can redistribute it and/or modify
   5  * it under the terms of the GNU General Public License, version 3.
   6  *
   7  * This program is distributed in the hope that it will be useful,
   8  * but WITHOUT ANY WARRANTY; without even the implied warranty of
   9  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  10  * GNU General Public License for more details.
  11  */
  12 #include "sysdep.h"
  13
  14 #include <stdio.h>
  15 #include <string.h>
  16 #include <syslog.h>
  17 #include <assert.h>
  18 #include <sys/socket.h>
  19 #include <netinet/in.h>
  20 #include <arpa/inet.h>
  21 #if HAVE_BACKTRACE
  22 #include <execinfo.h>
  23 #endif
  24
  25 #include <libcitadel.h>
  26
  27 #include "ctdl_module.h"
  28 #include "event_client.h"
  29
  30 ConstStr IOStates[] = {
  31         {HKEY("DB Queue")},
  32         {HKEY("DB Q Next")},
  33         {HKEY("DB Attach")},
  34         {HKEY("DB Next")},
  35         {HKEY("DB Stop")},
  36         {HKEY("DB Exit")},
  37         {HKEY("DB Terminate")},
  38         {HKEY("IO Queue")},
  39         {HKEY("IO Attach")},
  40         {HKEY("IO Connect Socket")},
  41         {HKEY("IO Abort")},
  42         {HKEY("IO Timeout")},
  43         {HKEY("IO ConnFail")},
  44         {HKEY("IO ConnFail Now")},
  45         {HKEY("IO Conn Now")},
  46         {HKEY("IO Conn Wait")},
  47         {HKEY("Curl Q")},
  48         {HKEY("Curl Start")},
  49         {HKEY("Curl Shotdown")},
  50         {HKEY("Curl More IO")},
  51         {HKEY("Curl Got IO")},
  52         {HKEY("Curl Got Data")},
  53         {HKEY("Curl Got Status")},
  54         {HKEY("C-Ares Start")},
  55         {HKEY("C-Ares IO Done")},
  56         {HKEY("C-Ares Finished")},
  57         {HKEY("C-Ares exit")},
  58         {HKEY("Killing")},
  59         {HKEY("Exit")}
  60 };
  61
  62 void SetEVState(AsyncIO *IO, eIOState State)
  63 {
  64
  65         CitContext* CCC = IO->CitContext;
  66         if (CCC != NULL)
  67                 memcpy(CCC->lastcmdname, IOStates[State].Key, IOStates[State].len + 1);
  68
  69 }
  70
  71
  72 static void IO_Timeout_callback(struct ev_loop *loop, ev_timer *watcher, int revents);
  73 static void IO_abort_shutdown_callback(struct ev_loop *loop,
  74                                        ev_cleanup *watcher,
  75                                        int revents);
  76
  77
  78 /*------------------------------------------------------------------------------
  79  *                              Server DB IO
  80  *----------------------------------------------------------------------------*/
  81 extern int evdb_count;
  82 extern pthread_mutex_t DBEventQueueMutex;
  83 extern pthread_mutex_t DBEventExitQueueMutex;
  84 extern HashList *DBInboundEventQueue;
  85 extern struct ev_loop *event_db;
  86 extern ev_async DBAddJob;
  87 extern ev_async DBExitEventLoop;
  88
  89 eNextState QueueDBOperation(AsyncIO *IO, IO_CallBack CB)
  90 {
  91         IOAddHandler *h;
  92         int i;
  93
  94         SetEVState(IO, eDBQ);
  95         h = (IOAddHandler*)malloc(sizeof(IOAddHandler));
  96         h->IO = IO;
  97         h->EvAttch = CB;
  98         ev_cleanup_init(&IO->db_abort_by_shutdown,
  99                         IO_abort_shutdown_callback);
 100         IO->db_abort_by_shutdown.data = IO;
 101
 102         pthread_mutex_lock(&DBEventQueueMutex);
 103         if (DBInboundEventQueue == NULL)
 104         {
 105                 /* shutting down... */
 106                 free(h);
 107                 EVM_syslog(LOG_DEBUG, "DBEVENT Q exiting.\n");
 108                 pthread_mutex_unlock(&DBEventQueueMutex);
 109                 return eAbort;
 110         }
 111         EVM_syslog(LOG_DEBUG, "DBEVENT Q\n");
 112         i = ++evdb_count ;
 113         Put(DBInboundEventQueue, IKEY(i), h, NULL);
 114         pthread_mutex_unlock(&DBEventQueueMutex);
 115
 116         pthread_mutex_lock(&DBEventExitQueueMutex);
 117         if (event_db == NULL)
 118         {
 119                 pthread_mutex_unlock(&DBEventExitQueueMutex);
 120                 return eAbort;
 121         }
 122         ev_async_send (event_db, &DBAddJob);
 123         pthread_mutex_unlock(&DBEventExitQueueMutex);
 124
 125         EVM_syslog(LOG_DEBUG, "DBEVENT Q Done.\n");
 126         return eDBQuery;
 127 }
 128
 129 void StopDBWatchers(AsyncIO *IO)
 130 {
 131         SetEVState(IO, eDBStop);
 132         ev_cleanup_stop(event_db, &IO->db_abort_by_shutdown);
 133         ev_idle_stop(event_db, &IO->db_unwind_stack);
 134 }
 135
 136 void ShutDownDBCLient(AsyncIO *IO)
 137 {
 138         CitContext *Ctx =IO->CitContext;
 139         become_session(Ctx);
 140
 141         SetEVState(IO, eDBTerm);
 142         EVM_syslog(LOG_DEBUG, "DBEVENT Terminating.\n");
 143         StopDBWatchers(IO);
 144
 145         assert(IO->DBTerminate);
 146         IO->DBTerminate(IO);
 147 }
 148
 149 void
 150 DB_PerformNext(struct ev_loop *loop, ev_idle *watcher, int revents)
 151 {
 152         AsyncIO *IO = watcher->data;
 153
 154         SetEVState(IO, eDBNext);
 155         IO->Now = ev_now(event_db);
 156         EV_syslog(LOG_DEBUG, "%s()", __FUNCTION__);
 157         become_session(IO->CitContext);
 158
 159         ev_idle_stop(event_db, &IO->db_unwind_stack);
 160
 161         assert(IO->NextDBOperation);
 162         switch (IO->NextDBOperation(IO))
 163         {
 164         case eDBQuery:
 165                 break;
 166         case eSendDNSQuery:
 167         case eReadDNSReply:
 168         case eConnect:
 169         case eSendReply:
 170         case eSendMore:
 171         case eSendFile:
 172         case eReadMessage:
 173         case eReadMore:
 174         case eReadPayload:
 175         case eReadFile:
 176                 ev_cleanup_stop(loop, &IO->db_abort_by_shutdown);
 177                 break;
 178         case eTerminateConnection:
 179         case eAbort:
 180                 ev_idle_stop(event_db, &IO->db_unwind_stack);
 181                 ev_cleanup_stop(loop, &IO->db_abort_by_shutdown);
 182                 ShutDownDBCLient(IO);
 183         }
 184 }
 185
 186 eNextState NextDBOperation(AsyncIO *IO, IO_CallBack CB)
 187 {
 188         SetEVState(IO, eQDBNext);
 189         IO->NextDBOperation = CB;
 190         ev_idle_init(&IO->db_unwind_stack,
 191                      DB_PerformNext);
 192         IO->db_unwind_stack.data = IO;
 193         ev_idle_start(event_db, &IO->db_unwind_stack);
 194         return eDBQuery;
 195 }
 196
 197 /*------------------------------------------------------------------------------
 198  *                      Client IO
 199  *----------------------------------------------------------------------------*/
 200 extern int evbase_count;
 201 extern pthread_mutex_t EventQueueMutex;
 202 extern pthread_mutex_t EventExitQueueMutex;
 203 extern HashList *InboundEventQueue;
 204 extern struct ev_loop *event_base;
 205 extern ev_async AddJob;
 206 extern ev_async ExitEventLoop;
 207
 208 static void IO_abort_shutdown_callback(struct ev_loop *loop,
 209                                        ev_cleanup *watcher,
 210                                        int revents)
 211 {
 212         AsyncIO *IO = watcher->data;
 213
 214         SetEVState(IO, eIOAbort);
 215         EV_syslog(LOG_DEBUG, "EVENT Q: %s\n", __FUNCTION__);
 216         IO->Now = ev_now(event_base);
 217         assert(IO->ShutdownAbort);
 218         IO->ShutdownAbort(IO);
 219 }
 220
 221
 222 eNextState QueueEventContext(AsyncIO *IO, IO_CallBack CB)
 223 {
 224         IOAddHandler *h;
 225         int i;
 226
 227         SetEVState(IO, eIOQ);
 228         h = (IOAddHandler*)malloc(sizeof(IOAddHandler));
 229         h->IO = IO;
 230         h->EvAttch = CB;
 231         ev_cleanup_init(&IO->abort_by_shutdown,
 232                         IO_abort_shutdown_callback);
 233         IO->abort_by_shutdown.data = IO;
 234
 235         pthread_mutex_lock(&EventQueueMutex);
 236         if (InboundEventQueue == NULL)
 237         {
 238                 free(h);
 239                 /* shutting down... */
 240                 EVM_syslog(LOG_DEBUG, "EVENT Q exiting.\n");
 241                 pthread_mutex_unlock(&EventQueueMutex);
 242                 return eAbort;
 243         }
 244         EVM_syslog(LOG_DEBUG, "EVENT Q\n");
 245         i = ++evbase_count;
 246         Put(InboundEventQueue, IKEY(i), h, NULL);
 247         pthread_mutex_unlock(&EventQueueMutex);
 248
 249         pthread_mutex_lock(&EventExitQueueMutex);
 250         if (event_base == NULL) {
 251                 pthread_mutex_unlock(&EventExitQueueMutex);
 252                 return eAbort;
 253         }
 254         ev_async_send (event_base, &AddJob);
 255         pthread_mutex_unlock(&EventExitQueueMutex);
 256         EVM_syslog(LOG_DEBUG, "EVENT Q Done.\n");
 257         return eSendReply;
 258 }
 259
 260 eNextState EventQueueDBOperation(AsyncIO *IO, IO_CallBack CB)
 261 {
 262         StopClientWatchers(IO, 0);
 263         return QueueDBOperation(IO, CB);
 264 }
 265 eNextState DBQueueEventContext(AsyncIO *IO, IO_CallBack CB)
 266 {
 267         StopDBWatchers(IO);
 268         return QueueEventContext(IO, CB);
 269 }
 270
 271 extern eNextState evcurl_handle_start(AsyncIO *IO);
 272
 273 eNextState QueueCurlContext(AsyncIO *IO)
 274 {
 275         IOAddHandler *h;
 276         int i;
 277
 278         SetEVState(IO, eCurlQ);
 279         h = (IOAddHandler*)malloc(sizeof(IOAddHandler));
 280         h->IO = IO;
 281         h->EvAttch = evcurl_handle_start;
 282
 283         pthread_mutex_lock(&EventQueueMutex);
 284         if (InboundEventQueue == NULL)
 285         {
 286                 /* shutting down... */
 287                 free(h);
 288                 EVM_syslog(LOG_DEBUG, "EVENT Q exiting.\n");
 289                 pthread_mutex_unlock(&EventQueueMutex);
 290                 return eAbort;
 291         }
 292
 293         EVM_syslog(LOG_DEBUG, "EVENT Q\n");
 294         i = ++evbase_count;
 295         Put(InboundEventQueue, IKEY(i), h, NULL);
 296         pthread_mutex_unlock(&EventQueueMutex);
 297
 298         pthread_mutex_lock(&EventExitQueueMutex);
 299         if (event_base == NULL) {
 300                 pthread_mutex_unlock(&EventExitQueueMutex);
 301                 return eAbort;
 302         }
 303         ev_async_send (event_base, &AddJob);
 304         pthread_mutex_unlock(&EventExitQueueMutex);
 305
 306         EVM_syslog(LOG_DEBUG, "EVENT Q Done.\n");
 307         return eSendReply;
 308 }
 309
 310 eNextState CurlQueueDBOperation(AsyncIO *IO, IO_CallBack CB)
 311 {
 312         StopCurlWatchers(IO);
 313         return QueueDBOperation(IO, CB);
 314 }
 315
 316
 317 void DestructCAres(AsyncIO *IO);
 318 void FreeAsyncIOContents(AsyncIO *IO)
 319 {
 320         CitContext *Ctx = IO->CitContext;
 321
 322         FreeStrBuf(&IO->IOBuf);
 323         FreeStrBuf(&IO->SendBuf.Buf);
 324         FreeStrBuf(&IO->RecvBuf.Buf);
 325
 326         DestructCAres(IO);
 327
 328         FreeURL(&IO->ConnectMe);
 329         FreeStrBuf(&IO->HttpReq.ReplyData);
 330
 331         if (Ctx) {
 332                 Ctx->state = CON_IDLE;
 333                 Ctx->kill_me = 1;
 334                 IO->CitContext = NULL;
 335         }
 336 }
 337
 338
 339 void StopClientWatchers(AsyncIO *IO, int CloseFD)
 340 {
 341         ev_timer_stop (event_base, &IO->rw_timeout);
 342         ev_timer_stop(event_base, &IO->conn_fail);
 343         ev_idle_stop(event_base, &IO->unwind_stack);
 344         ev_cleanup_stop(event_base, &IO->abort_by_shutdown);
 345
 346         ev_io_stop(event_base, &IO->conn_event);
 347         ev_io_stop(event_base, &IO->send_event);
 348         ev_io_stop(event_base, &IO->recv_event);
 349
 350         if (CloseFD && (IO->SendBuf.fd > 0)) {
 351                 close(IO->SendBuf.fd);
 352                 IO->SendBuf.fd = 0;
 353                 IO->RecvBuf.fd = 0;
 354         }
 355 }
 356
 357 void StopCurlWatchers(AsyncIO *IO)
 358 {
 359         EVM_syslog(LOG_DEBUG, "EVENT StopCurlWatchers \n");
 360
 361         ev_timer_stop (event_base, &IO->rw_timeout);
 362         ev_timer_stop(event_base, &IO->conn_fail);
 363         ev_idle_stop(event_base, &IO->unwind_stack);
 364         ev_cleanup_stop(event_base, &IO->abort_by_shutdown);
 365
 366         ev_io_stop(event_base, &IO->conn_event);
 367         ev_io_stop(event_base, &IO->send_event);
 368         ev_io_stop(event_base, &IO->recv_event);
 369
 370         curl_easy_cleanup(IO->HttpReq.chnd);
 371         IO->HttpReq.chnd = NULL;
 372
 373         if (IO->SendBuf.fd != 0) {
 374                 close(IO->SendBuf.fd);
 375         }
 376         IO->SendBuf.fd = 0;
 377         IO->RecvBuf.fd = 0;
 378 }
 379
 380 void ShutDownCLient(AsyncIO *IO)
 381 {
 382         CitContext *Ctx =IO->CitContext;
 383
 384         SetEVState(IO, eExit);
 385         become_session(Ctx);
 386
 387         EVM_syslog(LOG_DEBUG, "EVENT Terminating \n");
 388
 389         StopClientWatchers(IO, 1);
 390
 391         if (IO->DNS.Channel != NULL) {
 392                 ares_destroy(IO->DNS.Channel);
 393                 EV_DNS_LOG_STOP(DNS.recv_event);
 394                 EV_DNS_LOG_STOP(DNS.send_event);
 395                 ev_io_stop(event_base, &IO->DNS.recv_event);
 396                 ev_io_stop(event_base, &IO->DNS.send_event);
 397                 IO->DNS.Channel = NULL;
 398         }
 399         assert(IO->Terminate);
 400         IO->Terminate(IO);
 401 }
 402
 403 void PostInbound(AsyncIO *IO)
 404 {
 405         switch (IO->NextState) {
 406         case eSendFile:
 407                 ev_io_start(event_base, &IO->send_event);
 408                 break;
 409         case eSendReply:
 410         case eSendMore:
 411                 assert(IO->SendDone);
 412                 IO->NextState = IO->SendDone(IO);
 413                 switch (IO->NextState)
 414                 {
 415                 case eSendFile:
 416                 case eSendReply:
 417                 case eSendMore:
 418                 case eReadMessage:
 419                 case eReadPayload:
 420                 case eReadMore:
 421                 case eReadFile:
 422                         ev_io_start(event_base, &IO->send_event);
 423                         break;
 424                 case eDBQuery:
 425                         StopClientWatchers(IO, 0);
 426                 default:
 427                         break;
 428                 }
 429                 break;
 430         case eReadPayload:
 431         case eReadMore:
 432         case eReadFile:
 433                 ev_io_start(event_base, &IO->recv_event);
 434                 break;
 435         case eTerminateConnection:
 436                 ShutDownCLient(IO);
 437                 break;
 438         case eAbort:
 439                 ShutDownCLient(IO);
 440                 break;
 441         case eSendDNSQuery:
 442         case eReadDNSReply:
 443         case eDBQuery:
 444         case eConnect:
 445         case eReadMessage:
 446                 break;
 447         }
 448 }
 449 eReadState HandleInbound(AsyncIO *IO)
 450 {
 451         const char *Err = NULL;
 452         eReadState Finished = eBufferNotEmpty;
 453
 454         become_session(IO->CitContext);
 455
 456         while ((Finished == eBufferNotEmpty) &&
 457                ((IO->NextState == eReadMessage)||
 458                 (IO->NextState == eReadMore)||
 459                 (IO->NextState == eReadFile)||
 460                 (IO->NextState == eReadPayload)))
 461         {
 462                 /* Reading lines...
 463                  * lex line reply in callback,
 464                  * or do it ourselves.
 465                  * i.e. as nnn-blabla means continue reading in SMTP
 466                  */
 467                 if ((IO->NextState == eReadFile) &&
 468                     (Finished == eBufferNotEmpty))
 469                 {
 470                         Finished = WriteIOBAlreadyRead(&IO->IOB, &Err);
 471                         if (Finished == eReadSuccess)
 472                         {
 473                                 IO->NextState = eSendReply;
 474                         }
 475                 }
 476                 else if (IO->LineReader)
 477                         Finished = IO->LineReader(IO);
 478                 else
 479                         Finished = StrBufChunkSipLine(IO->IOBuf,
 480                                                       &IO->RecvBuf);
 481
 482                 switch (Finished) {
 483                 case eMustReadMore: /// read new from socket...
 484                         break;
 485                 case eBufferNotEmpty: /* shouldn't happen... */
 486                 case eReadSuccess: /// done for now...
 487                         break;
 488                 case eReadFail: /// WHUT?
 489                                 ///todo: shut down!
 490                         break;
 491                 }
 492
 493                 if (Finished != eMustReadMore) {
 494                         assert(IO->ReadDone);
 495                         ev_io_stop(event_base, &IO->recv_event);
 496                         IO->NextState = IO->ReadDone(IO);
 497                         Finished = StrBufCheckBuffer(&IO->RecvBuf);
 498                 }
 499         }
 500
 501         PostInbound(IO);
 502
 503         return Finished;
 504 }
 505
 506
 507 static void
 508 IO_send_callback(struct ev_loop *loop, ev_io *watcher, int revents)
 509 {
 510         int rc;
 511         AsyncIO *IO = watcher->data;
 512         const char *errmsg = NULL;
 513
 514         IO->Now = ev_now(event_base);
 515         become_session(IO->CitContext);
 516 #ifdef BIGBAD_IODBG
 517         {
 518                 int rv = 0;
 519                 char fn [SIZ];
 520                 FILE *fd;
 521                 const char *pch = ChrPtr(IO->SendBuf.Buf);
 522                 const char *pchh = IO->SendBuf.ReadWritePointer;
 523                 long nbytes;
 524
 525                 if (pchh == NULL)
 526                         pchh = pch;
 527
 528                 nbytes = StrLength(IO->SendBuf.Buf) - (pchh - pch);
 529                 snprintf(fn, SIZ, "/tmp/foolog_ev_%s.%d",
 530                          ((CitContext*)(IO->CitContext))->ServiceName,
 531                          IO->SendBuf.fd);
 532
 533                 fd = fopen(fn, "a+");
 534                 fprintf(fd, "Send: BufSize: %ld BufContent: [",
 535                         nbytes);
 536                 rv = fwrite(pchh, nbytes, 1, fd);
 537                 if (!rv) printf("failed to write debug to %s!\n", fn);
 538                 fprintf(fd, "]\n");
 539 #endif
 540                 switch (IO->NextState) {
 541                 case eSendFile:
 542                         rc = FileSendChunked(&IO->IOB, &errmsg);
 543                         if (rc < 0)
 544                                 StrBufPlain(IO->ErrMsg, errmsg, -1);
 545                         break;
 546                 default:
 547                         rc = StrBuf_write_one_chunk_callback(IO->SendBuf.fd,
 548                                                              0,
 549                                                              &IO->SendBuf);
 550                 }
 551
 552 #ifdef BIGBAD_IODBG
 553                 fprintf(fd, "Sent: BufSize: %d bytes.\n", rc);
 554                 fclose(fd);
 555         }
 556 #endif
 557         if (rc == 0)
 558         {
 559                 ev_io_stop(event_base, &IO->send_event);
 560                 switch (IO->NextState) {
 561                 case eSendMore:
 562                         assert(IO->SendDone);
 563                         IO->NextState = IO->SendDone(IO);
 564
 565                         if ((IO->NextState == eTerminateConnection) ||
 566                             (IO->NextState == eAbort) )
 567                                 ShutDownCLient(IO);
 568                         else {
 569                                 ev_io_start(event_base, &IO->send_event);
 570                         }
 571                         break;
 572                 case eSendFile:
 573                         if (IO->IOB.ChunkSendRemain > 0) {
 574                                 ev_io_start(event_base, &IO->recv_event);
 575                                 SetNextTimeout(IO, 100.0);
 576
 577                         } else {
 578                                 assert(IO->ReadDone);
 579                                 IO->NextState = IO->ReadDone(IO);
 580                                 switch(IO->NextState) {
 581                                 case eSendDNSQuery:
 582                                 case eReadDNSReply:
 583                                 case eDBQuery:
 584                                 case eConnect:
 585                                         break;
 586                                 case eSendReply:
 587                                 case eSendMore:
 588                                 case eSendFile:
 589                                         ev_io_start(event_base,
 590                                                     &IO->send_event);
 591                                         break;
 592                                 case eReadMessage:
 593                                 case eReadMore:
 594                                 case eReadPayload:
 595                                 case eReadFile:
 596                                         break;
 597                                 case eTerminateConnection:
 598                                 case eAbort:
 599                                         break;
 600                                 }
 601                         }
 602                         break;
 603                 case eSendReply:
 604                     if (StrBufCheckBuffer(&IO->SendBuf) != eReadSuccess)
 605                         break;
 606                     IO->NextState = eReadMore;
 607                 case eReadMore:
 608                 case eReadMessage:
 609                 case eReadPayload:
 610                 case eReadFile:
 611                         if (StrBufCheckBuffer(&IO->RecvBuf) == eBufferNotEmpty)
 612                         {
 613                                 HandleInbound(IO);
 614                         }
 615                         else {
 616                                 ev_io_start(event_base, &IO->recv_event);
 617                         }
 618
 619                         break;
 620                 case eDBQuery:
 621                         /*
 622                          * we now live in another queue,
 623                          * so we have to unregister.
 624                          */
 625                         ev_cleanup_stop(loop, &IO->abort_by_shutdown);
 626                         break;
 627                 case eSendDNSQuery:
 628                 case eReadDNSReply:
 629                 case eConnect:
 630                 case eTerminateConnection:
 631                 case eAbort:
 632                         break;
 633                 }
 634         }
 635         else if (rc < 0) {
 636                 if (errno != EAGAIN) {
 637                         StopClientWatchers(IO, 1);
 638                         EV_syslog(LOG_DEBUG,
 639                                   "IO_send_callback(): Socket Invalid! [%d] [%s] [%d]\n",
 640                                   errno, strerror(errno), IO->SendBuf.fd);
 641                         StrBufPrintf(IO->ErrMsg,
 642                                      "Socket Invalid! [%s]",
 643                                      strerror(errno));
 644                         SetNextTimeout(IO, 0.01);
 645                 }
 646         }
 647         /* else : must write more. */
 648 }
 649 static void
 650 set_start_callback(struct ev_loop *loop, AsyncIO *IO, int revents)
 651 {
 652         ev_timer_stop(event_base, &IO->conn_fail);
 653         ev_timer_start(event_base, &IO->rw_timeout);
 654
 655         switch(IO->NextState) {
 656         case eReadMore:
 657         case eReadMessage:
 658         case eReadFile:
 659                 StrBufAppendBufPlain(IO->ErrMsg, HKEY("[while waiting for greeting]"), 0);
 660                 ev_io_start(event_base, &IO->recv_event);
 661                 break;
 662         case eSendReply:
 663         case eSendMore:
 664         case eReadPayload:
 665         case eSendFile:
 666                 become_session(IO->CitContext);
 667                 IO_send_callback(loop, &IO->send_event, revents);
 668                 break;
 669         case eDBQuery:
 670         case eSendDNSQuery:
 671         case eReadDNSReply:
 672         case eConnect:
 673         case eTerminateConnection:
 674         case eAbort:
 675                 /// TODO: WHUT?
 676                 break;
 677         }
 678 }
 679
 680 static void
 681 IO_Timeout_callback(struct ev_loop *loop, ev_timer *watcher, int revents)
 682 {
 683         AsyncIO *IO = watcher->data;
 684
 685         SetEVState(IO, eIOTimeout);
 686         IO->Now = ev_now(event_base);
 687         ev_timer_stop (event_base, &IO->rw_timeout);
 688         become_session(IO->CitContext);
 689
 690         if (IO->SendBuf.fd != 0)
 691         {
 692                 ev_io_stop(event_base, &IO->send_event);
 693                 ev_io_stop(event_base, &IO->recv_event);
 694                 ev_timer_stop (event_base, &IO->rw_timeout);
 695                 close(IO->SendBuf.fd);
 696                 IO->SendBuf.fd = IO->RecvBuf.fd = 0;
 697         }
 698
 699         assert(IO->Timeout);
 700         switch (IO->Timeout(IO))
 701         {
 702         case eAbort:
 703                 ShutDownCLient(IO);
 704         default:
 705                 break;
 706         }
 707 }
 708
 709 static void
 710 IO_connfail_callback(struct ev_loop *loop, ev_timer *watcher, int revents)
 711 {
 712         AsyncIO *IO = watcher->data;
 713
 714         SetEVState(IO, eIOConnfail);
 715         IO->Now = ev_now(event_base);
 716         ev_timer_stop (event_base, &IO->conn_fail);
 717
 718         if (IO->SendBuf.fd != 0)
 719         {
 720                 ev_io_stop(loop, &IO->conn_event);
 721                 ev_io_stop(event_base, &IO->send_event);
 722                 ev_io_stop(event_base, &IO->recv_event);
 723                 ev_timer_stop (event_base, &IO->rw_timeout);
 724                 close(IO->SendBuf.fd);
 725                 IO->SendBuf.fd = IO->RecvBuf.fd = 0;
 726         }
 727         become_session(IO->CitContext);
 728
 729         assert(IO->ConnFail);
 730         switch (IO->ConnFail(IO))
 731         {
 732         case eAbort:
 733                 ShutDownCLient(IO);
 734         default:
 735                 break;
 736
 737         }
 738 }
 739
 740 static void
 741 IO_connfailimmediate_callback(struct ev_loop *loop,
 742                               ev_idle *watcher,
 743                               int revents)
 744 {
 745         AsyncIO *IO = watcher->data;
 746
 747         SetEVState(IO, eIOConnfailNow);
 748         IO->Now = ev_now(event_base);
 749         ev_idle_stop (event_base, &IO->conn_fail_immediate);
 750
 751         if (IO->SendBuf.fd != 0)
 752         {
 753                 close(IO->SendBuf.fd);
 754                 IO->SendBuf.fd = IO->RecvBuf.fd = 0;
 755         }
 756         become_session(IO->CitContext);
 757
 758         assert(IO->ConnFail);
 759         switch (IO->ConnFail(IO))
 760         {
 761         case eAbort:
 762                 ShutDownCLient(IO);
 763         default:
 764                 break;
 765
 766         }
 767 }
 768
 769 static void
 770 IO_connestd_callback(struct ev_loop *loop, ev_io *watcher, int revents)
 771 {
 772         AsyncIO *IO = watcher->data;
 773         int             so_err = 0;
 774         socklen_t       lon = sizeof(so_err);
 775         int             err;
 776
 777         SetEVState(IO, eIOConnNow);
 778         IO->Now = ev_now(event_base);
 779         EVM_syslog(LOG_DEBUG, "connect() succeeded.\n");
 780
 781         ev_io_stop(loop, &IO->conn_event);
 782         ev_timer_stop(event_base, &IO->conn_fail);
 783
 784         err = getsockopt(IO->SendBuf.fd,
 785                          SOL_SOCKET,
 786                          SO_ERROR,
 787                          (void*)&so_err,
 788                          &lon);
 789
 790         if ((err == 0) && (so_err != 0))
 791         {
 792                 EV_syslog(LOG_DEBUG, "connect() failed [%d][%s]\n",
 793                           so_err,
 794                           strerror(so_err));
 795                 IO_connfail_callback(loop, &IO->conn_fail, revents);
 796
 797         }
 798         else
 799         {
 800                 EVM_syslog(LOG_DEBUG, "connect() succeeded\n");
 801                 set_start_callback(loop, IO, revents);
 802         }
 803 }
 804
 805 static void
 806 IO_recv_callback(struct ev_loop *loop, ev_io *watcher, int revents)
 807 {
 808         const char *errmsg;
 809         ssize_t nbytes;
 810         AsyncIO *IO = watcher->data;
 811
 812         IO->Now = ev_now(event_base);
 813         switch (IO->NextState) {
 814         case eReadFile:
 815                 nbytes = FileRecvChunked(&IO->IOB, &errmsg);
 816                 if (nbytes < 0)
 817                         StrBufPlain(IO->ErrMsg, errmsg, -1);
 818                 else
 819                 {
 820                         if (IO->IOB.ChunkSendRemain == 0)
 821                         {
 822                                 IO->NextState = eSendReply;
 823                                 assert(IO->ReadDone);
 824                                 ev_io_stop(event_base, &IO->recv_event);
 825                                 PostInbound(IO);
 826                                 return;
 827                         }
 828                         else
 829                                 return;
 830                 }
 831                 break;
 832         default:
 833                 nbytes = StrBuf_read_one_chunk_callback(IO->RecvBuf.fd,
 834                                                         0,
 835                                                         &IO->RecvBuf);
 836                 break;
 837         }
 838
 839 #ifdef BIGBAD_IODBG
 840         {
 841                 long nbytes;
 842                 int rv = 0;
 843                 char fn [SIZ];
 844                 FILE *fd;
 845                 const char *pch = ChrPtr(IO->RecvBuf.Buf);
 846                 const char *pchh = IO->RecvBuf.ReadWritePointer;
 847
 848                 if (pchh == NULL)
 849                         pchh = pch;
 850
 851                 nbytes = StrLength(IO->RecvBuf.Buf) - (pchh - pch);
 852                 snprintf(fn, SIZ, "/tmp/foolog_ev_%s.%d",
 853                          ((CitContext*)(IO->CitContext))->ServiceName,
 854                          IO->SendBuf.fd);
 855
 856                 fd = fopen(fn, "a+");
 857                 fprintf(fd, "Read: BufSize: %ld BufContent: [",
 858                         nbytes);
 859                 rv = fwrite(pchh, nbytes, 1, fd);
 860                 if (!rv) printf("failed to write debug to %s!\n", fn);
 861                 fprintf(fd, "]\n");
 862                 fclose(fd);
 863         }
 864 #endif
 865         if (nbytes > 0) {
 866                 HandleInbound(IO);
 867         } else if (nbytes == 0) {
 868                 StopClientWatchers(IO, 1);
 869                 SetNextTimeout(IO, 0.01);
 870                 return;
 871         } else if (nbytes == -1) {
 872                 if (errno != EAGAIN) {
 873                         // FD is gone. kick it.
 874                         StopClientWatchers(IO, 1);
 875                         EV_syslog(LOG_DEBUG,
 876                                   "IO_recv_callback(): Socket Invalid! [%d] [%s] [%d]\n",
 877                                   errno, strerror(errno), IO->SendBuf.fd);
 878                         StrBufPrintf(IO->ErrMsg,
 879                                      "Socket Invalid! [%s]",
 880                                      strerror(errno));
 881                         SetNextTimeout(IO, 0.01);
 882                 }
 883                 return;
 884         }
 885 }
 886
 887 void
 888 IO_postdns_callback(struct ev_loop *loop, ev_idle *watcher, int revents)
 889 {
 890         AsyncIO *IO = watcher->data;
 891
 892         SetEVState(IO, eCaresFinished);
 893         IO->Now = ev_now(event_base);
 894         EV_syslog(LOG_DEBUG, "event: %s\n", __FUNCTION__);
 895         become_session(IO->CitContext);
 896         assert(IO->DNS.Query->PostDNS);
 897         switch (IO->DNS.Query->PostDNS(IO))
 898         {
 899         case eAbort:
 900                 assert(IO->DNS.Fail);
 901                 switch (IO->DNS.Fail(IO)) {
 902                 case eAbort:
 903 ////                    StopClientWatchers(IO);
 904                         ShutDownCLient(IO);
 905                 default:
 906                         break;
 907                 }
 908         default:
 909                 break;
 910         }
 911 }
 912
 913
 914 eNextState EvConnectSock(AsyncIO *IO,
 915                          double conn_timeout,
 916                          double first_rw_timeout,
 917                          int ReadFirst)
 918 {
 919         struct sockaddr_in egress_sin;
 920         int fdflags;
 921         int rc = -1;
 922
 923         SetEVState(IO, eIOConnectSock);
 924         become_session(IO->CitContext);
 925
 926         if (ReadFirst) {
 927                 IO->NextState = eReadMessage;
 928         }
 929         else {
 930                 IO->NextState = eSendReply;
 931         }
 932
 933         IO->SendBuf.fd = IO->RecvBuf.fd =
 934                 socket(
 935                         (IO->ConnectMe->IPv6)?PF_INET6:PF_INET,
 936                         SOCK_STREAM,
 937                         IPPROTO_TCP);
 938
 939         if (IO->SendBuf.fd < 0) {
 940                 EV_syslog(LOG_ERR,
 941                           "EVENT: socket() failed: %s\n",
 942                           strerror(errno));
 943
 944                 StrBufPrintf(IO->ErrMsg,
 945                              "Failed to create socket: %s",
 946                              strerror(errno));
 947                 IO->SendBuf.fd = IO->RecvBuf.fd = 0;
 948                 return eAbort;
 949         }
 950         fdflags = fcntl(IO->SendBuf.fd, F_GETFL);
 951         if (fdflags < 0) {
 952                 EV_syslog(LOG_ERR,
 953                           "EVENT: unable to get socket %d flags! %s \n",
 954                           IO->SendBuf.fd,
 955                           strerror(errno));
 956                 StrBufPrintf(IO->ErrMsg,
 957                              "Failed to get socket %d flags: %s",
 958                              IO->SendBuf.fd,
 959                              strerror(errno));
 960                 close(IO->SendBuf.fd);
 961                 IO->SendBuf.fd = IO->RecvBuf.fd = 0;
 962                 return eAbort;
 963         }
 964         fdflags = fdflags | O_NONBLOCK;
 965         if (fcntl(IO->SendBuf.fd, F_SETFL, fdflags) < 0) {
 966                 EV_syslog(
 967                         LOG_ERR,
 968                         "EVENT: unable to set socket %d nonblocking flags! %s \n",
 969                         IO->SendBuf.fd,
 970                         strerror(errno));
 971                 StrBufPrintf(IO->ErrMsg,
 972                              "Failed to set socket flags: %s",
 973                              strerror(errno));
 974                 close(IO->SendBuf.fd);
 975                 IO->SendBuf.fd = IO->RecvBuf.fd = 0;
 976                 return eAbort;
 977         }
 978 /* TODO: maye we could use offsetof() to calc the position of data...
 979  * http://doc.dvgu.ru/devel/ev.html#associating_custom_data_with_a_watcher
 980  */
 981         ev_io_init(&IO->recv_event, IO_recv_callback, IO->RecvBuf.fd, EV_READ);
 982         IO->recv_event.data = IO;
 983         ev_io_init(&IO->send_event, IO_send_callback, IO->SendBuf.fd, EV_WRITE);
 984         IO->send_event.data = IO;
 985
 986         ev_timer_init(&IO->conn_fail, IO_connfail_callback, conn_timeout, 0);
 987         IO->conn_fail.data = IO;
 988         ev_timer_init(&IO->rw_timeout, IO_Timeout_callback, first_rw_timeout,0);
 989         IO->rw_timeout.data = IO;
 990
 991
 992
 993
 994         /* for debugging you may bypass it like this:
 995          * IO->Addr.sin_addr.s_addr = inet_addr("127.0.0.1");
 996          * ((struct sockaddr_in)IO->ConnectMe->Addr).sin_addr.s_addr =
 997          *   inet_addr("127.0.0.1");
 998          */
 999         if (IO->ConnectMe->IPv6) {
1000                 rc = connect(IO->SendBuf.fd,
1001                              &IO->ConnectMe->Addr,
1002                              sizeof(struct sockaddr_in6));
1003         }
1004         else {
1005                 /* If citserver is bound to a specific IP address on the host, make
1006                  * sure we use that address for outbound connections.
1007                  */
1008
1009                 memset(&egress_sin, 0, sizeof(egress_sin));
1010                 egress_sin.sin_family = AF_INET;
1011                 if (!IsEmptyStr(config.c_ip_addr)) {
1012                         egress_sin.sin_addr.s_addr = inet_addr(config.c_ip_addr);
1013                         if (egress_sin.sin_addr.s_addr == !INADDR_ANY) {
1014                                 egress_sin.sin_addr.s_addr = INADDR_ANY;
1015                         }
1016
1017                         /* If this bind fails, no problem; we can still use INADDR_ANY */
1018                         bind(IO->SendBuf.fd, (struct sockaddr *)&egress_sin, sizeof(egress_sin));
1019                 }
1020                 rc = connect(IO->SendBuf.fd,
1021                              (struct sockaddr_in *)&IO->ConnectMe->Addr,
1022                              sizeof(struct sockaddr_in));
1023         }
1024
1025         if (rc >= 0){
1026                 SetEVState(IO, eIOConnNow);
1027                 EV_syslog(LOG_DEBUG, "connect() = %d immediate success.\n", IO->SendBuf.fd);
1028                 set_start_callback(event_base, IO, 0);
1029                 return IO->NextState;
1030         }
1031         else if (errno == EINPROGRESS) {
1032                 SetEVState(IO, eIOConnWait);
1033                 EV_syslog(LOG_DEBUG, "connect() = %d have to wait now.\n", IO->SendBuf.fd);
1034
1035                 ev_io_init(&IO->conn_event,
1036                            IO_connestd_callback,
1037                            IO->SendBuf.fd,
1038                            EV_READ|EV_WRITE);
1039
1040                 IO->conn_event.data = IO;
1041
1042                 ev_io_start(event_base, &IO->conn_event);
1043                 ev_timer_start(event_base, &IO->conn_fail);
1044                 return IO->NextState;
1045         }
1046         else {
1047                 SetEVState(IO, eIOConnfail);
1048                 ev_idle_init(&IO->conn_fail_immediate,
1049                              IO_connfailimmediate_callback);
1050                 IO->conn_fail_immediate.data = IO;
1051                 ev_idle_start(event_base, &IO->conn_fail_immediate);
1052
1053                 EV_syslog(LOG_ERR,
1054                           "connect() = %d failed: %s\n",
1055                           IO->SendBuf.fd,
1056                           strerror(errno));
1057
1058                 StrBufPrintf(IO->ErrMsg,
1059                              "Failed to connect: %s",
1060                              strerror(errno));
1061                 return IO->NextState;
1062         }
1063         return IO->NextState;
1064 }
1065
1066 void SetNextTimeout(AsyncIO *IO, double timeout)
1067 {
1068         IO->rw_timeout.repeat = timeout;
1069         ev_timer_again (event_base,  &IO->rw_timeout);
1070 }
1071
1072
1073 eNextState ReAttachIO(AsyncIO *IO,
1074                       void *pData,
1075                       int ReadFirst)
1076 {
1077         SetEVState(IO, eIOAttach);
1078         IO->Data = pData;
1079         become_session(IO->CitContext);
1080         ev_cleanup_start(event_base, &IO->abort_by_shutdown);
1081         if (ReadFirst) {
1082                 IO->NextState = eReadMessage;
1083         }
1084         else {
1085                 IO->NextState = eSendReply;
1086         }
1087         set_start_callback(event_base, IO, 0);
1088
1089         return IO->NextState;
1090 }
1091
1092 void InitIOStruct(AsyncIO *IO,
1093                   void *Data,
1094                   eNextState NextState,
1095                   IO_LineReaderCallback LineReader,
1096                   IO_CallBack DNS_Fail,
1097                   IO_CallBack SendDone,
1098                   IO_CallBack ReadDone,
1099                   IO_CallBack Terminate,
1100                   IO_CallBack DBTerminate,
1101                   IO_CallBack ConnFail,
1102                   IO_CallBack Timeout,
1103                   IO_CallBack ShutdownAbort)
1104 {
1105         IO->Data          = Data;
1106
1107         IO->CitContext    = CloneContext(CC);
1108         IO->CitContext->session_specific_data = Data;
1109         IO->CitContext->IO = IO;
1110
1111         IO->NextState     = NextState;
1112
1113         IO->SendDone      = SendDone;
1114         IO->ReadDone      = ReadDone;
1115         IO->Terminate     = Terminate;
1116         IO->DBTerminate   = DBTerminate;
1117         IO->LineReader    = LineReader;
1118         IO->ConnFail      = ConnFail;
1119         IO->Timeout       = Timeout;
1120         IO->ShutdownAbort = ShutdownAbort;
1121
1122         IO->DNS.Fail      = DNS_Fail;
1123
1124         IO->SendBuf.Buf   = NewStrBufPlain(NULL, 1024);
1125         IO->RecvBuf.Buf   = NewStrBufPlain(NULL, 1024);
1126         IO->IOBuf         = NewStrBuf();
1127         EV_syslog(LOG_DEBUG,
1128                   "EVENT: Session lives at %p IO at %p \n",
1129                   Data, IO);
1130
1131 }
1132
1133 extern int evcurl_init(AsyncIO *IO);
1134
1135 int InitcURLIOStruct(AsyncIO *IO,
1136                      void *Data,
1137                      const char* Desc,
1138                      IO_CallBack SendDone,
1139                      IO_CallBack Terminate,
1140                      IO_CallBack DBTerminate,
1141                      IO_CallBack ShutdownAbort)
1142 {
1143         IO->Data          = Data;
1144
1145         IO->CitContext    = CloneContext(CC);
1146         IO->CitContext->session_specific_data = Data;
1147         IO->CitContext->IO = IO;
1148
1149         IO->SendDone      = SendDone;
1150         IO->Terminate     = Terminate;
1151         IO->DBTerminate   = DBTerminate;
1152         IO->ShutdownAbort = ShutdownAbort;
1153
1154         strcpy(IO->HttpReq.errdesc, Desc);
1155
1156
1157         return  evcurl_init(IO);
1158
1159 }
1160
1161
1162 typedef struct KillOtherSessionContext {
1163         AsyncIO IO;
1164         AsyncIO *OtherOne;
1165 }KillOtherSessionContext;
1166
1167 eNextState KillTerminate(AsyncIO *IO)
1168 {
1169         long id;
1170         KillOtherSessionContext *Ctx = (KillOtherSessionContext*)IO->Data;
1171         EV_syslog(LOG_DEBUG, "%s Exit\n", __FUNCTION__);
1172         id = IO->ID;
1173         FreeAsyncIOContents(IO);
1174         memset(Ctx, 0, sizeof(KillOtherSessionContext));
1175         IO->ID = id; /* just for the case we want to analyze it in a coredump */
1176         free(Ctx);
1177         return eAbort;
1178
1179 }
1180
1181 eNextState KillShutdown(AsyncIO *IO)
1182 {
1183         return eTerminateConnection;
1184 }
1185
1186 eNextState KillOtherContextNow(AsyncIO *IO)
1187 {
1188         KillOtherSessionContext *Ctx = IO->Data;
1189
1190         SetEVState(IO, eKill);
1191
1192         if (Ctx->OtherOne->ShutdownAbort != NULL)
1193                 Ctx->OtherOne->ShutdownAbort(Ctx->OtherOne);
1194         return eTerminateConnection;
1195 }
1196
1197 void KillAsyncIOContext(AsyncIO *IO)
1198 {
1199         KillOtherSessionContext *Ctx;
1200
1201         Ctx = (KillOtherSessionContext*) malloc(sizeof(KillOtherSessionContext));
1202         memset(Ctx, 0, sizeof(KillOtherSessionContext));
1203
1204         InitIOStruct(&Ctx->IO,
1205                      Ctx,
1206                      eReadMessage,
1207                      NULL,
1208                      NULL,
1209                      NULL,
1210                      NULL,
1211                      KillTerminate,
1212                      NULL,
1213                      NULL,
1214                      NULL,
1215                      KillShutdown);
1216
1217         Ctx->OtherOne = IO;
1218
1219         switch(IO->NextState) {
1220         case eSendDNSQuery:
1221         case eReadDNSReply:
1222
1223         case eConnect:
1224         case eSendReply:
1225         case eSendMore:
1226         case eSendFile:
1227
1228         case eReadMessage:
1229         case eReadMore:
1230         case eReadPayload:
1231         case eReadFile:
1232                 QueueEventContext(&Ctx->IO, KillOtherContextNow);
1233                 break;
1234         case eDBQuery:
1235                 QueueDBOperation(&Ctx->IO, KillOtherContextNow);
1236                 break;
1237         case eTerminateConnection:
1238         case eAbort:
1239                 /*hm, its already dying, dunno which Queue its in... */
1240                 free(Ctx);
1241         }
1242
1243 }
1244
1245 extern int DebugEventLoopBacktrace;
1246 void EV_backtrace(AsyncIO *IO)
1247 {
1248 #ifdef HAVE_BACKTRACE
1249         void *stack_frames[50];
1250         size_t size, i;
1251         char **strings;
1252
1253         if ((IO == NULL) || (DebugEventLoopBacktrace == 0))
1254                 return;
1255         size = backtrace(stack_frames, sizeof(stack_frames) / sizeof(void*));
1256         strings = backtrace_symbols(stack_frames, size);
1257         for (i = 0; i < size; i++) {
1258                 if (strings != NULL) {
1259                         EV_syslog(LOG_ALERT, " BT %s\n", strings[i]);
1260                 }
1261                 else {
1262                         EV_syslog(LOG_ALERT, " BT %p\n", stack_frames[i]);
1263                 }
1264         }
1265         free(strings);
1266 #endif
1267 }
1268
1269
1270 ev_tstamp ctdl_ev_now (void)
1271 {
1272         return ev_now(event_base);
1273 }