citadel/event_client.c

   1 /*
   2  * Copyright (c) 1998-2012 by the citadel.org team
   3  *
   4  * This program is open source software; you can redistribute it and/or modify
   5  * it under the terms of the GNU General Public License, version 3.
   6  *
   7  * This program is distributed in the hope that it will be useful,
   8  * but WITHOUT ANY WARRANTY; without even the implied warranty of
   9  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  10  * GNU General Public License for more details.
  11  */
  12 #include "sysdep.h"
  13
  14 #include <stdio.h>
  15 #include <string.h>
  16 #include <syslog.h>
  17 #include <assert.h>
  18 #include <sys/socket.h>
  19 #include <netinet/in.h>
  20 #include <arpa/inet.h>
  21 #if HAVE_BACKTRACE
  22 #include <execinfo.h>
  23 #endif
  24
  25 #include <libcitadel.h>
  26
  27 #include "ctdl_module.h"
  28 #include "event_client.h"
  29 #include "citserver.h"
  30
  31 ConstStr IOStates[] = {
  32         {HKEY("DB Queue")},
  33         {HKEY("DB Q Next")},
  34         {HKEY("DB Attach")},
  35         {HKEY("DB Next")},
  36         {HKEY("DB Stop")},
  37         {HKEY("DB Exit")},
  38         {HKEY("DB Terminate")},
  39         {HKEY("IO Queue")},
  40         {HKEY("IO Attach")},
  41         {HKEY("IO Connect Socket")},
  42         {HKEY("IO Abort")},
  43         {HKEY("IO Timeout")},
  44         {HKEY("IO ConnFail")},
  45         {HKEY("IO ConnFail Now")},
  46         {HKEY("IO Conn Now")},
  47         {HKEY("IO Conn Wait")},
  48         {HKEY("Curl Q")},
  49         {HKEY("Curl Start")},
  50         {HKEY("Curl Shotdown")},
  51         {HKEY("Curl More IO")},
  52         {HKEY("Curl Got IO")},
  53         {HKEY("Curl Got Data")},
  54         {HKEY("Curl Got Status")},
  55         {HKEY("C-Ares Start")},
  56         {HKEY("C-Ares IO Done")},
  57         {HKEY("C-Ares Finished")},
  58         {HKEY("C-Ares exit")},
  59         {HKEY("Killing")},
  60         {HKEY("Exit")}
  61 };
  62
  63 void SetEVState(AsyncIO *IO, eIOState State)
  64 {
  65
  66         CitContext* CCC = IO->CitContext;
  67         if (CCC != NULL)
  68                 memcpy(CCC->lastcmdname, IOStates[State].Key, IOStates[State].len + 1);
  69
  70 }
  71
  72 eNextState QueueAnEventContext(AsyncIO *IO);
  73 static void IO_Timeout_callback(struct ev_loop *loop, ev_timer *watcher, int revents);
  74 static void IO_abort_shutdown_callback(struct ev_loop *loop,
  75                                        ev_cleanup *watcher,
  76                                        int revents);
  77
  78
  79 /*------------------------------------------------------------------------------
  80  *                              Server DB IO
  81  *----------------------------------------------------------------------------*/
  82 extern int evdb_count;
  83 extern pthread_mutex_t DBEventQueueMutex;
  84 extern pthread_mutex_t DBEventExitQueueMutex;
  85 extern HashList *DBInboundEventQueue;
  86 extern struct ev_loop *event_db;
  87 extern ev_async DBAddJob;
  88 extern ev_async DBExitEventLoop;
  89
  90 eNextState QueueAnDBOperation(AsyncIO *IO)
  91 {
  92         IOAddHandler *h;
  93         int i;
  94
  95         SetEVState(IO, eDBQ);
  96         h = (IOAddHandler*)malloc(sizeof(IOAddHandler));
  97         h->IO = IO;
  98
  99         assert(IO->ReAttachCB != NULL);
 100
 101         h->EvAttch = IO->ReAttachCB;
 102         ev_cleanup_init(&IO->db_abort_by_shutdown,
 103                         IO_abort_shutdown_callback);
 104         IO->db_abort_by_shutdown.data = IO;
 105
 106         pthread_mutex_lock(&DBEventQueueMutex);
 107         if (DBInboundEventQueue == NULL)
 108         {
 109                 /* shutting down... */
 110                 free(h);
 111                 EVM_syslog(LOG_DEBUG, "DBEVENT Q exiting.\n");
 112                 pthread_mutex_unlock(&DBEventQueueMutex);
 113                 return eAbort;
 114         }
 115         EVM_syslog(LOG_DEBUG, "DBEVENT Q\n");
 116         i = ++evdb_count ;
 117         Put(DBInboundEventQueue, IKEY(i), h, NULL);
 118         pthread_mutex_unlock(&DBEventQueueMutex);
 119
 120         pthread_mutex_lock(&DBEventExitQueueMutex);
 121         if (event_db == NULL)
 122         {
 123                 pthread_mutex_unlock(&DBEventExitQueueMutex);
 124                 return eAbort;
 125         }
 126         ev_async_send (event_db, &DBAddJob);
 127         pthread_mutex_unlock(&DBEventExitQueueMutex);
 128
 129         EVQM_syslog(LOG_DEBUG, "DBEVENT Q Done.\n");
 130         return eDBQuery;
 131 }
 132
 133 void StopDBWatchers(AsyncIO *IO)
 134 {
 135         SetEVState(IO, eDBStop);
 136         ev_cleanup_stop(event_db, &IO->db_abort_by_shutdown);
 137         ev_idle_stop(event_db, &IO->db_unwind_stack);
 138 }
 139
 140 void ShutDownDBCLient(AsyncIO *IO)
 141 {
 142         CitContext *Ctx =IO->CitContext;
 143         become_session(Ctx);
 144
 145         SetEVState(IO, eDBTerm);
 146         EVM_syslog(LOG_DEBUG, "DBEVENT Terminating.\n");
 147         StopDBWatchers(IO);
 148
 149         assert(IO->DBTerminate);
 150         IO->DBTerminate(IO);
 151 }
 152
 153 void
 154 DB_PerformNext(struct ev_loop *loop, ev_idle *watcher, int revents)
 155 {
 156         AsyncIO *IO = watcher->data;
 157
 158         SetEVState(IO, eDBNext);
 159         IO->Now = ev_now(event_db);
 160         EV_syslog(LOG_DEBUG, "%s()", __FUNCTION__);
 161         become_session(IO->CitContext);
 162
 163         ev_idle_stop(event_db, &IO->db_unwind_stack);
 164
 165         assert(IO->NextDBOperation);
 166         switch (IO->NextDBOperation(IO))
 167         {
 168         case eSendReply:
 169                 ev_cleanup_stop(loop, &IO->db_abort_by_shutdown);
 170                 QueueAnEventContext(IO);
 171                 break;
 172         case eDBQuery:
 173                 break;
 174         case eSendDNSQuery:
 175         case eReadDNSReply:
 176         case eConnect:
 177         case eSendMore:
 178         case eSendFile:
 179         case eReadMessage:
 180         case eReadMore:
 181         case eReadPayload:
 182         case eReadFile:
 183                 ev_cleanup_stop(loop, &IO->db_abort_by_shutdown);
 184                 break;
 185         case eTerminateConnection:
 186         case eAbort:
 187                 ev_idle_stop(event_db, &IO->db_unwind_stack);
 188                 ev_cleanup_stop(loop, &IO->db_abort_by_shutdown);
 189                 ShutDownDBCLient(IO);
 190         }
 191 }
 192
 193 eNextState NextDBOperation(AsyncIO *IO, IO_CallBack CB)
 194 {
 195         SetEVState(IO, eQDBNext);
 196         IO->NextDBOperation = CB;
 197         ev_idle_init(&IO->db_unwind_stack,
 198                      DB_PerformNext);
 199         IO->db_unwind_stack.data = IO;
 200         ev_idle_start(event_db, &IO->db_unwind_stack);
 201         return eDBQuery;
 202 }
 203
 204 /*------------------------------------------------------------------------------
 205  *                      Client IO
 206  *----------------------------------------------------------------------------*/
 207 extern int evbase_count;
 208 extern pthread_mutex_t EventQueueMutex;
 209 extern pthread_mutex_t EventExitQueueMutex;
 210 extern HashList *InboundEventQueue;
 211 extern struct ev_loop *event_base;
 212 extern ev_async AddJob;
 213 extern ev_async ExitEventLoop;
 214
 215 static void IO_abort_shutdown_callback(struct ev_loop *loop,
 216                                        ev_cleanup *watcher,
 217                                        int revents)
 218 {
 219         AsyncIO *IO = watcher->data;
 220
 221         SetEVState(IO, eIOAbort);
 222         EV_syslog(LOG_DEBUG, "EVENT Q: %s\n", __FUNCTION__);
 223         IO->Now = ev_now(event_base);
 224         assert(IO->ShutdownAbort);
 225         IO->ShutdownAbort(IO);
 226 }
 227
 228
 229 eNextState QueueAnEventContext(AsyncIO *IO)
 230 {
 231         IOAddHandler *h;
 232         int i;
 233
 234         SetEVState(IO, eIOQ);
 235         h = (IOAddHandler*)malloc(sizeof(IOAddHandler));
 236         h->IO = IO;
 237
 238         assert(IO->ReAttachCB != NULL);
 239
 240         h->EvAttch = IO->ReAttachCB;
 241
 242         ev_cleanup_init(&IO->abort_by_shutdown,
 243                         IO_abort_shutdown_callback);
 244         IO->abort_by_shutdown.data = IO;
 245
 246         pthread_mutex_lock(&EventQueueMutex);
 247         if (InboundEventQueue == NULL)
 248         {
 249                 free(h);
 250                 /* shutting down... */
 251                 EVM_syslog(LOG_DEBUG, "EVENT Q exiting.\n");
 252                 pthread_mutex_unlock(&EventQueueMutex);
 253                 return eAbort;
 254         }
 255         EVM_syslog(LOG_DEBUG, "EVENT Q\n");
 256         i = ++evbase_count;
 257         Put(InboundEventQueue, IKEY(i), h, NULL);
 258         pthread_mutex_unlock(&EventQueueMutex);
 259
 260         pthread_mutex_lock(&EventExitQueueMutex);
 261         if (event_base == NULL) {
 262                 pthread_mutex_unlock(&EventExitQueueMutex);
 263                 return eAbort;
 264         }
 265         ev_async_send (event_base, &AddJob);
 266         pthread_mutex_unlock(&EventExitQueueMutex);
 267         EVM_syslog(LOG_DEBUG, "EVENT Q Done.\n");
 268         return eSendReply;
 269 }
 270
 271 eNextState EventQueueDBOperation(AsyncIO *IO, IO_CallBack CB, int CloseFDs)
 272 {
 273         StopClientWatchers(IO, CloseFDs);
 274         IO->ReAttachCB = CB;
 275         return eDBQuery;
 276 }
 277 eNextState DBQueueEventContext(AsyncIO *IO, IO_CallBack CB)
 278 {
 279         StopDBWatchers(IO);
 280         IO->ReAttachCB = CB;
 281         return eSendReply;
 282 }
 283
 284 eNextState QueueEventContext(AsyncIO *IO, IO_CallBack CB)
 285 {
 286         IO->ReAttachCB = CB;
 287         return QueueAnEventContext(IO);
 288 }
 289
 290 extern eNextState evcurl_handle_start(AsyncIO *IO);
 291
 292 eNextState QueueCurlContext(AsyncIO *IO)
 293 {
 294         IOAddHandler *h;
 295         int i;
 296
 297         SetEVState(IO, eCurlQ);
 298         h = (IOAddHandler*)malloc(sizeof(IOAddHandler));
 299         h->IO = IO;
 300         h->EvAttch = evcurl_handle_start;
 301
 302         pthread_mutex_lock(&EventQueueMutex);
 303         if (InboundEventQueue == NULL)
 304         {
 305                 /* shutting down... */
 306                 free(h);
 307                 EVM_syslog(LOG_DEBUG, "EVENT Q exiting.\n");
 308                 pthread_mutex_unlock(&EventQueueMutex);
 309                 return eAbort;
 310         }
 311
 312         EVM_syslog(LOG_DEBUG, "EVENT Q\n");
 313         i = ++evbase_count;
 314         Put(InboundEventQueue, IKEY(i), h, NULL);
 315         pthread_mutex_unlock(&EventQueueMutex);
 316
 317         pthread_mutex_lock(&EventExitQueueMutex);
 318         if (event_base == NULL) {
 319                 pthread_mutex_unlock(&EventExitQueueMutex);
 320                 return eAbort;
 321         }
 322         ev_async_send (event_base, &AddJob);
 323         pthread_mutex_unlock(&EventExitQueueMutex);
 324
 325         EVM_syslog(LOG_DEBUG, "EVENT Q Done.\n");
 326         return eSendReply;
 327 }
 328
 329 eNextState CurlQueueDBOperation(AsyncIO *IO, IO_CallBack CB)
 330 {
 331         StopCurlWatchers(IO);
 332         IO->ReAttachCB = CB;
 333         return eDBQuery;
 334 }
 335
 336
 337 void FreeAsyncIOContents(AsyncIO *IO)
 338 {
 339         CitContext *Ctx = IO->CitContext;
 340
 341         FreeStrBuf(&IO->IOBuf);
 342         FreeStrBuf(&IO->SendBuf.Buf);
 343         FreeStrBuf(&IO->RecvBuf.Buf);
 344
 345         FreeURL(&IO->ConnectMe);
 346         FreeStrBuf(&IO->HttpReq.ReplyData);
 347
 348         if (Ctx) {
 349                 Ctx->state = CON_IDLE;
 350                 Ctx->kill_me = 1;
 351                 IO->CitContext = NULL;
 352         }
 353 }
 354
 355
 356 void DestructCAres(AsyncIO *IO);
 357 void StopClientWatchers(AsyncIO *IO, int CloseFD)
 358 {
 359         EVM_syslog(LOG_DEBUG, "EVENT StopClientWatchers");
 360
 361         DestructCAres(IO);
 362
 363         ev_timer_stop (event_base, &IO->rw_timeout);
 364         ev_timer_stop(event_base, &IO->conn_fail);
 365         ev_idle_stop(event_base, &IO->unwind_stack);
 366         ev_cleanup_stop(event_base, &IO->abort_by_shutdown);
 367
 368         ev_io_stop(event_base, &IO->conn_event);
 369         ev_io_stop(event_base, &IO->send_event);
 370         ev_io_stop(event_base, &IO->recv_event);
 371
 372         if (CloseFD && (IO->SendBuf.fd > 0)) {
 373                 close(IO->SendBuf.fd);
 374                 IO->SendBuf.fd = 0;
 375                 IO->RecvBuf.fd = 0;
 376         }
 377 }
 378
 379 void StopCurlWatchers(AsyncIO *IO)
 380 {
 381         EVM_syslog(LOG_DEBUG, "EVENT StopCurlWatchers \n");
 382
 383         ev_timer_stop (event_base, &IO->rw_timeout);
 384         ev_timer_stop(event_base, &IO->conn_fail);
 385         ev_idle_stop(event_base, &IO->unwind_stack);
 386         ev_cleanup_stop(event_base, &IO->abort_by_shutdown);
 387
 388         ev_io_stop(event_base, &IO->conn_event);
 389         ev_io_stop(event_base, &IO->send_event);
 390         ev_io_stop(event_base, &IO->recv_event);
 391
 392         curl_easy_cleanup(IO->HttpReq.chnd);
 393         IO->HttpReq.chnd = NULL;
 394
 395         if (IO->SendBuf.fd != 0) {
 396                 close(IO->SendBuf.fd);
 397         }
 398         IO->SendBuf.fd = 0;
 399         IO->RecvBuf.fd = 0;
 400 }
 401
 402 eNextState ShutDownCLient(AsyncIO *IO)
 403 {
 404         CitContext *Ctx =IO->CitContext;
 405
 406         SetEVState(IO, eExit);
 407         become_session(Ctx);
 408
 409         EVM_syslog(LOG_DEBUG, "EVENT Terminating \n");
 410
 411         StopClientWatchers(IO, 1);
 412
 413         if (IO->DNS.Channel != NULL) {
 414                 ares_destroy(IO->DNS.Channel);
 415                 EV_DNS_LOG_STOP(DNS.recv_event);
 416                 EV_DNS_LOG_STOP(DNS.send_event);
 417                 ev_io_stop(event_base, &IO->DNS.recv_event);
 418                 ev_io_stop(event_base, &IO->DNS.send_event);
 419                 IO->DNS.Channel = NULL;
 420         }
 421         assert(IO->Terminate);
 422         return IO->Terminate(IO);
 423 }
 424
 425 void PostInbound(AsyncIO *IO)
 426 {
 427
 428         switch (IO->NextState) {
 429         case eSendFile:
 430                 ev_io_start(event_base, &IO->send_event);
 431                 break;
 432         case eSendReply:
 433         case eSendMore:
 434                 assert(IO->SendDone);
 435                 IO->NextState = IO->SendDone(IO);
 436                 switch (IO->NextState)
 437                 {
 438                 case eSendFile:
 439                 case eSendReply:
 440                 case eSendMore:
 441                 case eReadMessage:
 442                 case eReadPayload:
 443                 case eReadMore:
 444                 case eReadFile:
 445                         ev_io_start(event_base, &IO->send_event);
 446                         break;
 447                 case eDBQuery:
 448                         StopClientWatchers(IO, 0);
 449                         QueueAnDBOperation(IO);
 450                 default:
 451                         break;
 452                 }
 453                 break;
 454         case eReadPayload:
 455         case eReadMore:
 456         case eReadFile:
 457                 ev_io_start(event_base, &IO->recv_event);
 458                 break;
 459         case eTerminateConnection:
 460         case eAbort:
 461                 if (ShutDownCLient(IO) == eDBQuery) {
 462                         QueueAnDBOperation(IO);
 463                 }
 464                 break;
 465         case eSendDNSQuery:
 466         case eReadDNSReply:
 467         case eConnect:
 468         case eReadMessage:
 469                 break;
 470         case eDBQuery:
 471                 QueueAnDBOperation(IO);
 472         }
 473 }
 474 eReadState HandleInbound(AsyncIO *IO)
 475 {
 476         const char *Err = NULL;
 477         eReadState Finished = eBufferNotEmpty;
 478
 479         become_session(IO->CitContext);
 480
 481         while ((Finished == eBufferNotEmpty) &&
 482                ((IO->NextState == eReadMessage)||
 483                 (IO->NextState == eReadMore)||
 484                 (IO->NextState == eReadFile)||
 485                 (IO->NextState == eReadPayload)))
 486         {
 487                 /* Reading lines...
 488                  * lex line reply in callback,
 489                  * or do it ourselves.
 490                  * i.e. as nnn-blabla means continue reading in SMTP
 491                  */
 492                 if ((IO->NextState == eReadFile) &&
 493                     (Finished == eBufferNotEmpty))
 494                 {
 495                         Finished = WriteIOBAlreadyRead(&IO->IOB, &Err);
 496                         if (Finished == eReadSuccess)
 497                         {
 498                                 IO->NextState = eSendReply;
 499                         }
 500                 }
 501                 else if (IO->LineReader)
 502                         Finished = IO->LineReader(IO);
 503                 else
 504                         Finished = StrBufChunkSipLine(IO->IOBuf,
 505                                                       &IO->RecvBuf);
 506
 507                 switch (Finished) {
 508                 case eMustReadMore: /// read new from socket...
 509                         break;
 510                 case eBufferNotEmpty: /* shouldn't happen... */
 511                 case eReadSuccess: /// done for now...
 512                         break;
 513                 case eReadFail: /// WHUT?
 514                                 ///todo: shut down!
 515                         break;
 516                 }
 517
 518                 if (Finished != eMustReadMore) {
 519                         ev_io_stop(event_base, &IO->recv_event);
 520                         IO->NextState = IO->ReadDone(IO);
 521                         if  (IO->NextState == eDBQuery) {
 522                                 if (QueueAnDBOperation(IO) == eAbort)
 523                                         return eReadFail;
 524                                 else
 525                                         return eReadSuccess;
 526                         }
 527                         else {
 528                                 Finished = StrBufCheckBuffer(&IO->RecvBuf);
 529                         }
 530                 }
 531         }
 532
 533         PostInbound(IO);
 534
 535         return Finished;
 536 }
 537
 538
 539 static void
 540 IO_send_callback(struct ev_loop *loop, ev_io *watcher, int revents)
 541 {
 542         int rc;
 543         AsyncIO *IO = watcher->data;
 544         const char *errmsg = NULL;
 545
 546         IO->Now = ev_now(event_base);
 547         become_session(IO->CitContext);
 548 #ifdef BIGBAD_IODBG
 549         {
 550                 int rv = 0;
 551                 char fn [SIZ];
 552                 FILE *fd;
 553                 const char *pch = ChrPtr(IO->SendBuf.Buf);
 554                 const char *pchh = IO->SendBuf.ReadWritePointer;
 555                 long nbytes;
 556
 557                 if (pchh == NULL)
 558                         pchh = pch;
 559
 560                 nbytes = StrLength(IO->SendBuf.Buf) - (pchh - pch);
 561                 snprintf(fn, SIZ, "/tmp/foolog_ev_%s.%d",
 562                          ((CitContext*)(IO->CitContext))->ServiceName,
 563                          IO->SendBuf.fd);
 564
 565                 fd = fopen(fn, "a+");
 566                 if (fd == NULL) {
 567                         syslog(LOG_EMERG, "failed to open file %s: %s", fn, strerror(errno));
 568                         cit_backtrace();
 569                         exit(1);
 570                 }
 571                 fprintf(fd, "Send: BufSize: %ld BufContent: [",
 572                         nbytes);
 573                 rv = fwrite(pchh, nbytes, 1, fd);
 574                 if (!rv) printf("failed to write debug to %s!\n", fn);
 575                 fprintf(fd, "]\n");
 576 #endif
 577                 switch (IO->NextState) {
 578                 case eSendFile:
 579                         rc = FileSendChunked(&IO->IOB, &errmsg);
 580                         if (rc < 0)
 581                                 StrBufPlain(IO->ErrMsg, errmsg, -1);
 582                         break;
 583                 default:
 584                         rc = StrBuf_write_one_chunk_callback(IO->SendBuf.fd,
 585                                                              0,
 586                                                              &IO->SendBuf);
 587                 }
 588
 589 #ifdef BIGBAD_IODBG
 590                 fprintf(fd, "Sent: BufSize: %d bytes.\n", rc);
 591                 fclose(fd);
 592         }
 593 #endif
 594         if (rc == 0)
 595         {
 596                 ev_io_stop(event_base, &IO->send_event);
 597                 switch (IO->NextState) {
 598                 case eSendMore:
 599                         assert(IO->SendDone);
 600                         IO->NextState = IO->SendDone(IO);
 601
 602                         if ((IO->NextState == eTerminateConnection) ||
 603                             (IO->NextState == eAbort) )
 604                                 ShutDownCLient(IO);
 605                         else {
 606                                 ev_io_start(event_base, &IO->send_event);
 607                         }
 608                         break;
 609                 case eSendFile:
 610                         if (IO->IOB.ChunkSendRemain > 0) {
 611                                 ev_io_start(event_base, &IO->recv_event);
 612                                 SetNextTimeout(IO, 100.0);
 613
 614                         } else {
 615                                 assert(IO->ReadDone);
 616                                 IO->NextState = IO->ReadDone(IO);
 617                                 switch(IO->NextState) {
 618                                 case eSendDNSQuery:
 619                                 case eReadDNSReply:
 620                                 case eDBQuery:
 621                                 case eConnect:
 622                                         break;
 623                                 case eSendReply:
 624                                 case eSendMore:
 625                                 case eSendFile:
 626                                         ev_io_start(event_base,
 627                                                     &IO->send_event);
 628                                         break;
 629                                 case eReadMessage:
 630                                 case eReadMore:
 631                                 case eReadPayload:
 632                                 case eReadFile:
 633                                         break;
 634                                 case eTerminateConnection:
 635                                 case eAbort:
 636                                         break;
 637                                 }
 638                         }
 639                         break;
 640                 case eSendReply:
 641                     if (StrBufCheckBuffer(&IO->SendBuf) != eReadSuccess)
 642                         break;
 643                     IO->NextState = eReadMore;
 644                 case eReadMore:
 645                 case eReadMessage:
 646                 case eReadPayload:
 647                 case eReadFile:
 648                         if (StrBufCheckBuffer(&IO->RecvBuf) == eBufferNotEmpty)
 649                         {
 650                                 HandleInbound(IO);
 651                         }
 652                         else {
 653                                 ev_io_start(event_base, &IO->recv_event);
 654                         }
 655
 656                         break;
 657                 case eDBQuery:
 658                         /*
 659                          * we now live in another queue,
 660                          * so we have to unregister.
 661                          */
 662                         ev_cleanup_stop(loop, &IO->abort_by_shutdown);
 663                         break;
 664                 case eSendDNSQuery:
 665                 case eReadDNSReply:
 666                 case eConnect:
 667                 case eTerminateConnection:
 668                 case eAbort:
 669                         break;
 670                 }
 671         }
 672         else if (rc < 0) {
 673                 if (errno != EAGAIN) {
 674                         StopClientWatchers(IO, 1);
 675                         EV_syslog(LOG_DEBUG,
 676                                   "IO_send_callback(): Socket Invalid! [%d] [%s] [%d]\n",
 677                                   errno, strerror(errno), IO->SendBuf.fd);
 678                         StrBufPrintf(IO->ErrMsg,
 679                                      "Socket Invalid! [%s]",
 680                                      strerror(errno));
 681                         SetNextTimeout(IO, 0.01);
 682                 }
 683         }
 684         /* else : must write more. */
 685 }
 686 static void
 687 set_start_callback(struct ev_loop *loop, AsyncIO *IO, int revents)
 688 {
 689         ev_timer_stop(event_base, &IO->conn_fail);
 690         ev_timer_start(event_base, &IO->rw_timeout);
 691
 692         switch(IO->NextState) {
 693         case eReadMore:
 694         case eReadMessage:
 695         case eReadFile:
 696                 StrBufAppendBufPlain(IO->ErrMsg, HKEY("[while waiting for greeting]"), 0);
 697                 ev_io_start(event_base, &IO->recv_event);
 698                 break;
 699         case eSendReply:
 700         case eSendMore:
 701         case eReadPayload:
 702         case eSendFile:
 703                 become_session(IO->CitContext);
 704                 IO_send_callback(loop, &IO->send_event, revents);
 705                 break;
 706         case eDBQuery:
 707         case eSendDNSQuery:
 708         case eReadDNSReply:
 709         case eConnect:
 710         case eTerminateConnection:
 711         case eAbort:
 712                 /// TODO: WHUT?
 713                 break;
 714         }
 715 }
 716
 717 static void
 718 IO_Timeout_callback(struct ev_loop *loop, ev_timer *watcher, int revents)
 719 {
 720         AsyncIO *IO = watcher->data;
 721
 722         SetEVState(IO, eIOTimeout);
 723         IO->Now = ev_now(event_base);
 724         ev_timer_stop (event_base, &IO->rw_timeout);
 725         become_session(IO->CitContext);
 726
 727         if (IO->SendBuf.fd != 0)
 728         {
 729                 ev_io_stop(event_base, &IO->send_event);
 730                 ev_io_stop(event_base, &IO->recv_event);
 731                 ev_timer_stop (event_base, &IO->rw_timeout);
 732                 close(IO->SendBuf.fd);
 733                 IO->SendBuf.fd = IO->RecvBuf.fd = 0;
 734         }
 735
 736         assert(IO->Timeout);
 737         switch (IO->Timeout(IO))
 738         {
 739         case eAbort:
 740                 ShutDownCLient(IO);
 741         default:
 742                 break;
 743         }
 744 }
 745
 746 static void
 747 IO_connfail_callback(struct ev_loop *loop, ev_timer *watcher, int revents)
 748 {
 749         AsyncIO *IO = watcher->data;
 750
 751         SetEVState(IO, eIOConnfail);
 752         IO->Now = ev_now(event_base);
 753         ev_timer_stop (event_base, &IO->conn_fail);
 754
 755         if (IO->SendBuf.fd != 0)
 756         {
 757                 ev_io_stop(loop, &IO->conn_event);
 758                 ev_io_stop(event_base, &IO->send_event);
 759                 ev_io_stop(event_base, &IO->recv_event);
 760                 ev_timer_stop (event_base, &IO->rw_timeout);
 761                 close(IO->SendBuf.fd);
 762                 IO->SendBuf.fd = IO->RecvBuf.fd = 0;
 763         }
 764         become_session(IO->CitContext);
 765
 766         assert(IO->ConnFail);
 767         switch (IO->ConnFail(IO))
 768         {
 769         case eAbort:
 770                 ShutDownCLient(IO);
 771         default:
 772                 break;
 773
 774         }
 775 }
 776
 777 static void
 778 IO_connfailimmediate_callback(struct ev_loop *loop,
 779                               ev_idle *watcher,
 780                               int revents)
 781 {
 782         AsyncIO *IO = watcher->data;
 783
 784         SetEVState(IO, eIOConnfailNow);
 785         IO->Now = ev_now(event_base);
 786         ev_idle_stop (event_base, &IO->conn_fail_immediate);
 787
 788         if (IO->SendBuf.fd != 0)
 789         {
 790                 close(IO->SendBuf.fd);
 791                 IO->SendBuf.fd = IO->RecvBuf.fd = 0;
 792         }
 793         become_session(IO->CitContext);
 794
 795         assert(IO->ConnFail);
 796         switch (IO->ConnFail(IO))
 797         {
 798         case eAbort:
 799                 ShutDownCLient(IO);
 800         default:
 801                 break;
 802
 803         }
 804 }
 805
 806 static void
 807 IO_connestd_callback(struct ev_loop *loop, ev_io *watcher, int revents)
 808 {
 809         AsyncIO *IO = watcher->data;
 810         int             so_err = 0;
 811         socklen_t       lon = sizeof(so_err);
 812         int             err;
 813
 814         SetEVState(IO, eIOConnNow);
 815         IO->Now = ev_now(event_base);
 816         EVM_syslog(LOG_DEBUG, "connect() succeeded.\n");
 817
 818         ev_io_stop(loop, &IO->conn_event);
 819         ev_timer_stop(event_base, &IO->conn_fail);
 820
 821         err = getsockopt(IO->SendBuf.fd,
 822                          SOL_SOCKET,
 823                          SO_ERROR,
 824                          (void*)&so_err,
 825                          &lon);
 826
 827         if ((err == 0) && (so_err != 0))
 828         {
 829                 EV_syslog(LOG_DEBUG, "connect() failed [%d][%s]\n",
 830                           so_err,
 831                           strerror(so_err));
 832                 IO_connfail_callback(loop, &IO->conn_fail, revents);
 833
 834         }
 835         else
 836         {
 837                 EVM_syslog(LOG_DEBUG, "connect() succeeded\n");
 838                 set_start_callback(loop, IO, revents);
 839         }
 840 }
 841
 842 static void
 843 IO_recv_callback(struct ev_loop *loop, ev_io *watcher, int revents)
 844 {
 845         const char *errmsg;
 846         ssize_t nbytes;
 847         AsyncIO *IO = watcher->data;
 848
 849         IO->Now = ev_now(event_base);
 850         switch (IO->NextState) {
 851         case eReadFile:
 852                 nbytes = FileRecvChunked(&IO->IOB, &errmsg);
 853                 if (nbytes < 0)
 854                         StrBufPlain(IO->ErrMsg, errmsg, -1);
 855                 else
 856                 {
 857                         if (IO->IOB.ChunkSendRemain == 0)
 858                         {
 859                                 IO->NextState = eSendReply;
 860                                 assert(IO->ReadDone);
 861                                 ev_io_stop(event_base, &IO->recv_event);
 862                                 PostInbound(IO);
 863                                 return;
 864                         }
 865                         else
 866                                 return;
 867                 }
 868                 break;
 869         default:
 870                 nbytes = StrBuf_read_one_chunk_callback(IO->RecvBuf.fd,
 871                                                         0,
 872                                                         &IO->RecvBuf);
 873                 break;
 874         }
 875
 876 #ifdef BIGBAD_IODBG
 877         {
 878                 long nbytes;
 879                 int rv = 0;
 880                 char fn [SIZ];
 881                 FILE *fd;
 882                 const char *pch = ChrPtr(IO->RecvBuf.Buf);
 883                 const char *pchh = IO->RecvBuf.ReadWritePointer;
 884
 885                 if (pchh == NULL)
 886                         pchh = pch;
 887
 888                 nbytes = StrLength(IO->RecvBuf.Buf) - (pchh - pch);
 889                 snprintf(fn, SIZ, "/tmp/foolog_ev_%s.%d",
 890                          ((CitContext*)(IO->CitContext))->ServiceName,
 891                          IO->SendBuf.fd);
 892
 893                 fd = fopen(fn, "a+");
 894                 if (fd == NULL) {
 895                         syslog(LOG_EMERG, "failed to open file %s: %s", fn, strerror(errno));
 896                         cit_backtrace();
 897                         exit(1);
 898                 }
 899                 fprintf(fd, "Read: BufSize: %ld BufContent: [",
 900                         nbytes);
 901                 rv = fwrite(pchh, nbytes, 1, fd);
 902                 if (!rv) printf("failed to write debug to %s!\n", fn);
 903                 fprintf(fd, "]\n");
 904                 fclose(fd);
 905         }
 906 #endif
 907         if (nbytes > 0) {
 908                 HandleInbound(IO);
 909         } else if (nbytes == 0) {
 910                 StopClientWatchers(IO, 1);
 911                 SetNextTimeout(IO, 0.01);
 912                 return;
 913         } else if (nbytes == -1) {
 914                 if (errno != EAGAIN) {
 915                         // FD is gone. kick it.
 916                         StopClientWatchers(IO, 1);
 917                         EV_syslog(LOG_DEBUG,
 918                                   "IO_recv_callback(): Socket Invalid! [%d] [%s] [%d]\n",
 919                                   errno, strerror(errno), IO->SendBuf.fd);
 920                         StrBufPrintf(IO->ErrMsg,
 921                                      "Socket Invalid! [%s]",
 922                                      strerror(errno));
 923                         SetNextTimeout(IO, 0.01);
 924                 }
 925                 return;
 926         }
 927 }
 928
 929 void
 930 IO_postdns_callback(struct ev_loop *loop, ev_idle *watcher, int revents)
 931 {
 932         AsyncIO *IO = watcher->data;
 933
 934         SetEVState(IO, eCaresFinished);
 935         IO->Now = ev_now(event_base);
 936         EV_syslog(LOG_DEBUG, "event: %s\n", __FUNCTION__);
 937         become_session(IO->CitContext);
 938         assert(IO->DNS.Query->PostDNS);
 939         switch (IO->DNS.Query->PostDNS(IO))
 940         {
 941         case eAbort:
 942                 assert(IO->DNS.Fail);
 943                 switch (IO->DNS.Fail(IO)) {
 944                 case eAbort:
 945 ////                    StopClientWatchers(IO);
 946                         ShutDownCLient(IO);
 947                         break;
 948                 case eDBQuery:
 949                         StopClientWatchers(IO, 0);
 950                         QueueAnDBOperation(IO);
 951                         break;
 952                 default:
 953                         break;
 954                 }
 955         case eDBQuery:
 956                 StopClientWatchers(IO, 0);
 957                 QueueAnDBOperation(IO);
 958                 break;
 959         default:
 960                 break;
 961         }
 962 }
 963
 964
 965 eNextState EvConnectSock(AsyncIO *IO,
 966                          double conn_timeout,
 967                          double first_rw_timeout,
 968                          int ReadFirst)
 969 {
 970         struct sockaddr_in egress_sin;
 971         int fdflags;
 972         int rc = -1;
 973
 974         SetEVState(IO, eIOConnectSock);
 975         become_session(IO->CitContext);
 976
 977         if (ReadFirst) {
 978                 IO->NextState = eReadMessage;
 979         }
 980         else {
 981                 IO->NextState = eSendReply;
 982         }
 983
 984         IO->SendBuf.fd = IO->RecvBuf.fd =
 985                 socket(
 986                         (IO->ConnectMe->IPv6)?PF_INET6:PF_INET,
 987                         SOCK_STREAM,
 988                         IPPROTO_TCP);
 989
 990         if (IO->SendBuf.fd < 0) {
 991                 EV_syslog(LOG_ERR,
 992                           "EVENT: socket() failed: %s\n",
 993                           strerror(errno));
 994
 995                 StrBufPrintf(IO->ErrMsg,
 996                              "Failed to create socket: %s",
 997                              strerror(errno));
 998                 IO->SendBuf.fd = IO->RecvBuf.fd = 0;
 999                 return eAbort;
1000         }
1001         fdflags = fcntl(IO->SendBuf.fd, F_GETFL);
1002         if (fdflags < 0) {
1003                 EV_syslog(LOG_ERR,
1004                           "EVENT: unable to get socket %d flags! %s \n",
1005                           IO->SendBuf.fd,
1006                           strerror(errno));
1007                 StrBufPrintf(IO->ErrMsg,
1008                              "Failed to get socket %d flags: %s",
1009                              IO->SendBuf.fd,
1010                              strerror(errno));
1011                 close(IO->SendBuf.fd);
1012                 IO->SendBuf.fd = IO->RecvBuf.fd = 0;
1013                 return eAbort;
1014         }
1015         fdflags = fdflags | O_NONBLOCK;
1016         if (fcntl(IO->SendBuf.fd, F_SETFL, fdflags) < 0) {
1017                 EV_syslog(
1018                         LOG_ERR,
1019                         "EVENT: unable to set socket %d nonblocking flags! %s \n",
1020                         IO->SendBuf.fd,
1021                         strerror(errno));
1022                 StrBufPrintf(IO->ErrMsg,
1023                              "Failed to set socket flags: %s",
1024                              strerror(errno));
1025                 close(IO->SendBuf.fd);
1026                 IO->SendBuf.fd = IO->RecvBuf.fd = 0;
1027                 return eAbort;
1028         }
1029 /* TODO: maye we could use offsetof() to calc the position of data...
1030  * http://doc.dvgu.ru/devel/ev.html#associating_custom_data_with_a_watcher
1031  */
1032         ev_io_init(&IO->recv_event, IO_recv_callback, IO->RecvBuf.fd, EV_READ);
1033         IO->recv_event.data = IO;
1034         ev_io_init(&IO->send_event, IO_send_callback, IO->SendBuf.fd, EV_WRITE);
1035         IO->send_event.data = IO;
1036
1037         ev_timer_init(&IO->conn_fail, IO_connfail_callback, conn_timeout, 0);
1038         IO->conn_fail.data = IO;
1039         ev_timer_init(&IO->rw_timeout, IO_Timeout_callback, first_rw_timeout,0);
1040         IO->rw_timeout.data = IO;
1041
1042
1043
1044
1045         /* for debugging you may bypass it like this:
1046          * IO->Addr.sin_addr.s_addr = inet_addr("127.0.0.1");
1047          * ((struct sockaddr_in)IO->ConnectMe->Addr).sin_addr.s_addr =
1048          *   inet_addr("127.0.0.1");
1049          */
1050         if (IO->ConnectMe->IPv6) {
1051                 rc = connect(IO->SendBuf.fd,
1052                              &IO->ConnectMe->Addr,
1053                              sizeof(struct sockaddr_in6));
1054         }
1055         else {
1056                 /* If citserver is bound to a specific IP address on the host, make
1057                  * sure we use that address for outbound connections.
1058                  */
1059
1060                 memset(&egress_sin, 0, sizeof(egress_sin));
1061                 egress_sin.sin_family = AF_INET;
1062                 if (!IsEmptyStr(config.c_ip_addr)) {
1063                         egress_sin.sin_addr.s_addr = inet_addr(config.c_ip_addr);
1064                         if (egress_sin.sin_addr.s_addr == !INADDR_ANY) {
1065                                 egress_sin.sin_addr.s_addr = INADDR_ANY;
1066                         }
1067
1068                         /* If this bind fails, no problem; we can still use INADDR_ANY */
1069                         bind(IO->SendBuf.fd, (struct sockaddr *)&egress_sin, sizeof(egress_sin));
1070                 }
1071                 rc = connect(IO->SendBuf.fd,
1072                              (struct sockaddr_in *)&IO->ConnectMe->Addr,
1073                              sizeof(struct sockaddr_in));
1074         }
1075
1076         if (rc >= 0){
1077                 SetEVState(IO, eIOConnNow);
1078                 EV_syslog(LOG_DEBUG, "connect() = %d immediate success.\n", IO->SendBuf.fd);
1079                 set_start_callback(event_base, IO, 0);
1080                 return IO->NextState;
1081         }
1082         else if (errno == EINPROGRESS) {
1083                 SetEVState(IO, eIOConnWait);
1084                 EV_syslog(LOG_DEBUG, "connect() = %d have to wait now.\n", IO->SendBuf.fd);
1085
1086                 ev_io_init(&IO->conn_event,
1087                            IO_connestd_callback,
1088                            IO->SendBuf.fd,
1089                            EV_READ|EV_WRITE);
1090
1091                 IO->conn_event.data = IO;
1092
1093                 ev_io_start(event_base, &IO->conn_event);
1094                 ev_timer_start(event_base, &IO->conn_fail);
1095                 return IO->NextState;
1096         }
1097         else {
1098                 SetEVState(IO, eIOConnfail);
1099                 ev_idle_init(&IO->conn_fail_immediate,
1100                              IO_connfailimmediate_callback);
1101                 IO->conn_fail_immediate.data = IO;
1102                 ev_idle_start(event_base, &IO->conn_fail_immediate);
1103
1104                 EV_syslog(LOG_ERR,
1105                           "connect() = %d failed: %s\n",
1106                           IO->SendBuf.fd,
1107                           strerror(errno));
1108
1109                 StrBufPrintf(IO->ErrMsg,
1110                              "Failed to connect: %s",
1111                              strerror(errno));
1112                 return IO->NextState;
1113         }
1114         return IO->NextState;
1115 }
1116
1117 void SetNextTimeout(AsyncIO *IO, double timeout)
1118 {
1119         IO->rw_timeout.repeat = timeout;
1120         ev_timer_again (event_base,  &IO->rw_timeout);
1121 }
1122
1123
1124 eNextState ReAttachIO(AsyncIO *IO,
1125                       void *pData,
1126                       int ReadFirst)
1127 {
1128         SetEVState(IO, eIOAttach);
1129         IO->Data = pData;
1130         become_session(IO->CitContext);
1131         ev_cleanup_start(event_base, &IO->abort_by_shutdown);
1132         if (ReadFirst) {
1133                 IO->NextState = eReadMessage;
1134         }
1135         else {
1136                 IO->NextState = eSendReply;
1137         }
1138         set_start_callback(event_base, IO, 0);
1139
1140         return IO->NextState;
1141 }
1142
1143 void InitIOStruct(AsyncIO *IO,
1144                   void *Data,
1145                   eNextState NextState,
1146                   IO_LineReaderCallback LineReader,
1147                   IO_CallBack DNS_Fail,
1148                   IO_CallBack SendDone,
1149                   IO_CallBack ReadDone,
1150                   IO_CallBack Terminate,
1151                   IO_CallBack DBTerminate,
1152                   IO_CallBack ConnFail,
1153                   IO_CallBack Timeout,
1154                   IO_CallBack ShutdownAbort)
1155 {
1156         IO->Data          = Data;
1157
1158         IO->CitContext    = CloneContext(CC);
1159         IO->CitContext->session_specific_data = Data;
1160         IO->CitContext->IO = IO;
1161
1162         IO->NextState     = NextState;
1163
1164         IO->SendDone      = SendDone;
1165         IO->ReadDone      = ReadDone;
1166         IO->Terminate     = Terminate;
1167         IO->DBTerminate   = DBTerminate;
1168         IO->LineReader    = LineReader;
1169         IO->ConnFail      = ConnFail;
1170         IO->Timeout       = Timeout;
1171         IO->ShutdownAbort = ShutdownAbort;
1172
1173         IO->DNS.Fail      = DNS_Fail;
1174
1175         IO->SendBuf.Buf   = NewStrBufPlain(NULL, 1024);
1176         IO->RecvBuf.Buf   = NewStrBufPlain(NULL, 1024);
1177         IO->IOBuf         = NewStrBuf();
1178         EV_syslog(LOG_DEBUG,
1179                   "EVENT: Session lives at %p IO at %p \n",
1180                   Data, IO);
1181
1182 }
1183
1184 extern int evcurl_init(AsyncIO *IO);
1185
1186 int InitcURLIOStruct(AsyncIO *IO,
1187                      void *Data,
1188                      const char* Desc,
1189                      IO_CallBack SendDone,
1190                      IO_CallBack Terminate,
1191                      IO_CallBack DBTerminate,
1192                      IO_CallBack ShutdownAbort)
1193 {
1194         IO->Data          = Data;
1195
1196         IO->CitContext    = CloneContext(CC);
1197         IO->CitContext->session_specific_data = Data;
1198         IO->CitContext->IO = IO;
1199
1200         IO->SendDone      = SendDone;
1201         IO->Terminate     = Terminate;
1202         IO->DBTerminate   = DBTerminate;
1203         IO->ShutdownAbort = ShutdownAbort;
1204
1205         strcpy(IO->HttpReq.errdesc, Desc);
1206
1207
1208         return  evcurl_init(IO);
1209
1210 }
1211
1212
1213 typedef struct KillOtherSessionContext {
1214         AsyncIO IO;
1215         AsyncIO *OtherOne;
1216 }KillOtherSessionContext;
1217
1218 eNextState KillTerminate(AsyncIO *IO)
1219 {
1220         long id;
1221         KillOtherSessionContext *Ctx = (KillOtherSessionContext*)IO->Data;
1222         EV_syslog(LOG_DEBUG, "%s Exit\n", __FUNCTION__);
1223         id = IO->ID;
1224         FreeAsyncIOContents(IO);
1225         memset(Ctx, 0, sizeof(KillOtherSessionContext));
1226         IO->ID = id; /* just for the case we want to analyze it in a coredump */
1227         free(Ctx);
1228         return eAbort;
1229
1230 }
1231
1232 eNextState KillShutdown(AsyncIO *IO)
1233 {
1234         return eTerminateConnection;
1235 }
1236
1237 eNextState KillOtherContextNow(AsyncIO *IO)
1238 {
1239         KillOtherSessionContext *Ctx = IO->Data;
1240
1241         SetEVState(IO, eKill);
1242
1243         if (Ctx->OtherOne->ShutdownAbort != NULL) {
1244                 Ctx->OtherOne->NextState = eAbort;
1245                 if (Ctx->OtherOne->ShutdownAbort(Ctx->OtherOne) == eDBQuery) {
1246                         StopClientWatchers(Ctx->OtherOne, 0);
1247                         QueueAnDBOperation(Ctx->OtherOne);
1248                 }
1249         }
1250         return eTerminateConnection;
1251 }
1252
1253 void KillAsyncIOContext(AsyncIO *IO)
1254 {
1255         KillOtherSessionContext *Ctx;
1256
1257         Ctx = (KillOtherSessionContext*) malloc(sizeof(KillOtherSessionContext));
1258         memset(Ctx, 0, sizeof(KillOtherSessionContext));
1259
1260         InitIOStruct(&Ctx->IO,
1261                      Ctx,
1262                      eReadMessage,
1263                      NULL,
1264                      NULL,
1265                      NULL,
1266                      NULL,
1267                      KillTerminate,
1268                      NULL,
1269                      NULL,
1270                      NULL,
1271                      KillShutdown);
1272
1273         Ctx->OtherOne = IO;
1274
1275         switch(IO->NextState) {
1276         case eSendDNSQuery:
1277         case eReadDNSReply:
1278
1279         case eConnect:
1280         case eSendReply:
1281         case eSendMore:
1282         case eSendFile:
1283
1284         case eReadMessage:
1285         case eReadMore:
1286         case eReadPayload:
1287         case eReadFile:
1288                 Ctx->IO.ReAttachCB = KillOtherContextNow;
1289                 QueueAnEventContext(&Ctx->IO);
1290                 break;
1291         case eDBQuery:
1292                 Ctx->IO.ReAttachCB = KillOtherContextNow;
1293                 QueueAnDBOperation(&Ctx->IO);
1294                 break;
1295         case eTerminateConnection:
1296         case eAbort:
1297                 /*hm, its already dying, dunno which Queue its in... */
1298                 free(Ctx);
1299         }
1300
1301 }
1302
1303 extern int DebugEventLoopBacktrace;
1304 void EV_backtrace(AsyncIO *IO)
1305 {
1306 #ifdef HAVE_BACKTRACE
1307         void *stack_frames[50];
1308         size_t size, i;
1309         char **strings;
1310
1311         if ((IO == NULL) || (DebugEventLoopBacktrace == 0))
1312                 return;
1313         size = backtrace(stack_frames, sizeof(stack_frames) / sizeof(void*));
1314         strings = backtrace_symbols(stack_frames, size);
1315         for (i = 0; i < size; i++) {
1316                 if (strings != NULL) {
1317                         EV_syslog(LOG_ALERT, " BT %s\n", strings[i]);
1318                 }
1319                 else {
1320                         EV_syslog(LOG_ALERT, " BT %p\n", stack_frames[i]);
1321                 }
1322         }
1323         free(strings);
1324 #endif
1325 }
1326
1327
1328 ev_tstamp ctdl_ev_now (void)
1329 {
1330         return ev_now(event_base);
1331 }