citadel/event_client.c

   1 /*
   2  * Copyright (c) 1998-2012 by the citadel.org team
   3  *
   4  * This program is open source software; you can redistribute it and/or modify
   5  * it under the terms of the GNU General Public License, version 3.
   6  *
   7  * This program is distributed in the hope that it will be useful,
   8  * but WITHOUT ANY WARRANTY; without even the implied warranty of
   9  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  10  * GNU General Public License for more details.
  11  */
  12 #include "sysdep.h"
  13
  14 #include <stdio.h>
  15 #include <string.h>
  16 #include <syslog.h>
  17 #include <assert.h>
  18 #include <sys/socket.h>
  19 #include <netinet/in.h>
  20 #include <arpa/inet.h>
  21
  22 #include <libcitadel.h>
  23
  24 #include "ctdl_module.h"
  25 #include "event_client.h"
  26
  27 ConstStr IOStates[] = {
  28         {HKEY("DB Queue")},
  29         {HKEY("DB Q Next")},
  30         {HKEY("DB Attach")},
  31         {HKEY("DB Next")},
  32         {HKEY("DB Stop")},
  33         {HKEY("DB Exit")},
  34         {HKEY("DB Terminate")},
  35         {HKEY("IO Queue")},
  36         {HKEY("IO Attach")},
  37         {HKEY("IO Connect Socket")},
  38         {HKEY("IO Abort")},
  39         {HKEY("IO Timeout")},
  40         {HKEY("IO ConnFail")},
  41         {HKEY("IO ConnFail Now")},
  42         {HKEY("IO Conn Now")},
  43         {HKEY("IO Conn Wait")},
  44         {HKEY("Curl Q")},
  45         {HKEY("Curl Start")},
  46         {HKEY("Curl Shotdown")},
  47         {HKEY("Curl More IO")},
  48         {HKEY("Curl Got IO")},
  49         {HKEY("Curl Got Data")},
  50         {HKEY("Curl Got Status")},
  51         {HKEY("C-Ares Start")},
  52         {HKEY("C-Ares IO Done")},
  53         {HKEY("C-Ares Finished")},
  54         {HKEY("C-Ares exit")},
  55         {HKEY("Killing")},
  56         {HKEY("Exit")}
  57 };
  58
  59 void SetEVState(AsyncIO *IO, eIOState State)
  60 {
  61
  62         CitContext* CCC = IO->CitContext;
  63         if (CCC != NULL)
  64                 memcpy(CCC->lastcmdname, IOStates[State].Key, IOStates[State].len + 1);
  65
  66 }
  67
  68
  69 static void IO_Timeout_callback(struct ev_loop *loop, ev_timer *watcher, int revents);
  70 static void IO_abort_shutdown_callback(struct ev_loop *loop,
  71                                        ev_cleanup *watcher,
  72                                        int revents);
  73
  74
  75 /*------------------------------------------------------------------------------
  76  *                              Server DB IO
  77  *----------------------------------------------------------------------------*/
  78 extern int evdb_count;
  79 extern pthread_mutex_t DBEventQueueMutex;
  80 extern pthread_mutex_t DBEventExitQueueMutex;
  81 extern HashList *DBInboundEventQueue;
  82 extern struct ev_loop *event_db;
  83 extern ev_async DBAddJob;
  84 extern ev_async DBExitEventLoop;
  85
  86 eNextState QueueDBOperation(AsyncIO *IO, IO_CallBack CB)
  87 {
  88         IOAddHandler *h;
  89         int i;
  90
  91         SetEVState(IO, eDBQ);
  92         h = (IOAddHandler*)malloc(sizeof(IOAddHandler));
  93         h->IO = IO;
  94         h->EvAttch = CB;
  95         ev_cleanup_init(&IO->db_abort_by_shutdown,
  96                         IO_abort_shutdown_callback);
  97         IO->db_abort_by_shutdown.data = IO;
  98
  99         pthread_mutex_lock(&DBEventQueueMutex);
 100         if (DBInboundEventQueue == NULL)
 101         {
 102                 /* shutting down... */
 103                 free(h);
 104                 EVM_syslog(LOG_DEBUG, "DBEVENT Q exiting.\n");
 105                 pthread_mutex_unlock(&DBEventQueueMutex);
 106                 return eAbort;
 107         }
 108         EVM_syslog(LOG_DEBUG, "DBEVENT Q\n");
 109         i = ++evdb_count ;
 110         Put(DBInboundEventQueue, IKEY(i), h, NULL);
 111         pthread_mutex_unlock(&DBEventQueueMutex);
 112
 113         pthread_mutex_lock(&DBEventExitQueueMutex);
 114         if (event_db == NULL)
 115         {
 116                 pthread_mutex_unlock(&DBEventExitQueueMutex);
 117                 return eAbort;
 118         }
 119         ev_async_send (event_db, &DBAddJob);
 120         pthread_mutex_unlock(&DBEventExitQueueMutex);
 121
 122         EVM_syslog(LOG_DEBUG, "DBEVENT Q Done.\n");
 123         return eDBQuery;
 124 }
 125
 126 void StopDBWatchers(AsyncIO *IO)
 127 {
 128         SetEVState(IO, eDBStop);
 129         ev_cleanup_stop(event_db, &IO->db_abort_by_shutdown);
 130         ev_idle_stop(event_db, &IO->db_unwind_stack);
 131 }
 132
 133 void ShutDownDBCLient(AsyncIO *IO)
 134 {
 135         CitContext *Ctx =IO->CitContext;
 136         become_session(Ctx);
 137
 138         SetEVState(IO, eDBTerm);
 139         EVM_syslog(LOG_DEBUG, "DBEVENT Terminating.\n");
 140         StopDBWatchers(IO);
 141
 142         assert(IO->DBTerminate);
 143         IO->DBTerminate(IO);
 144 }
 145
 146 void
 147 DB_PerformNext(struct ev_loop *loop, ev_idle *watcher, int revents)
 148 {
 149         AsyncIO *IO = watcher->data;
 150
 151         SetEVState(IO, eDBNext);
 152         IO->Now = ev_now(event_db);
 153         EV_syslog(LOG_DEBUG, "%s()", __FUNCTION__);
 154         become_session(IO->CitContext);
 155
 156         ev_idle_stop(event_db, &IO->db_unwind_stack);
 157
 158         assert(IO->NextDBOperation);
 159         switch (IO->NextDBOperation(IO))
 160         {
 161         case eDBQuery:
 162                 break;
 163         case eSendDNSQuery:
 164         case eReadDNSReply:
 165         case eConnect:
 166         case eSendReply:
 167         case eSendMore:
 168         case eSendFile:
 169         case eReadMessage:
 170         case eReadMore:
 171         case eReadPayload:
 172         case eReadFile:
 173                 ev_cleanup_stop(loop, &IO->db_abort_by_shutdown);
 174                 break;
 175         case eTerminateConnection:
 176         case eAbort:
 177                 ev_idle_stop(event_db, &IO->db_unwind_stack);
 178                 ev_cleanup_stop(loop, &IO->db_abort_by_shutdown);
 179                 ShutDownDBCLient(IO);
 180         }
 181 }
 182
 183 eNextState NextDBOperation(AsyncIO *IO, IO_CallBack CB)
 184 {
 185         SetEVState(IO, eQDBNext);
 186         IO->NextDBOperation = CB;
 187         ev_idle_init(&IO->db_unwind_stack,
 188                      DB_PerformNext);
 189         IO->db_unwind_stack.data = IO;
 190         ev_idle_start(event_db, &IO->db_unwind_stack);
 191         return eDBQuery;
 192 }
 193
 194 /*------------------------------------------------------------------------------
 195  *                      Client IO
 196  *----------------------------------------------------------------------------*/
 197 extern int evbase_count;
 198 extern pthread_mutex_t EventQueueMutex;
 199 extern pthread_mutex_t EventExitQueueMutex;
 200 extern HashList *InboundEventQueue;
 201 extern struct ev_loop *event_base;
 202 extern ev_async AddJob;
 203 extern ev_async ExitEventLoop;
 204
 205 static void IO_abort_shutdown_callback(struct ev_loop *loop,
 206                                        ev_cleanup *watcher,
 207                                        int revents)
 208 {
 209         AsyncIO *IO = watcher->data;
 210
 211         SetEVState(IO, eIOAbort);
 212         EV_syslog(LOG_DEBUG, "EVENT Q: %s\n", __FUNCTION__);
 213         IO->Now = ev_now(event_base);
 214         assert(IO->ShutdownAbort);
 215         IO->ShutdownAbort(IO);
 216 }
 217
 218
 219 eNextState QueueEventContext(AsyncIO *IO, IO_CallBack CB)
 220 {
 221         IOAddHandler *h;
 222         int i;
 223
 224         SetEVState(IO, eIOQ);
 225         h = (IOAddHandler*)malloc(sizeof(IOAddHandler));
 226         h->IO = IO;
 227         h->EvAttch = CB;
 228         ev_cleanup_init(&IO->abort_by_shutdown,
 229                         IO_abort_shutdown_callback);
 230         IO->abort_by_shutdown.data = IO;
 231
 232         pthread_mutex_lock(&EventQueueMutex);
 233         if (InboundEventQueue == NULL)
 234         {
 235                 free(h);
 236                 /* shutting down... */
 237                 EVM_syslog(LOG_DEBUG, "EVENT Q exiting.\n");
 238                 pthread_mutex_unlock(&EventQueueMutex);
 239                 return eAbort;
 240         }
 241         EVM_syslog(LOG_DEBUG, "EVENT Q\n");
 242         i = ++evbase_count;
 243         Put(InboundEventQueue, IKEY(i), h, NULL);
 244         pthread_mutex_unlock(&EventQueueMutex);
 245
 246         pthread_mutex_lock(&EventExitQueueMutex);
 247         if (event_base == NULL) {
 248                 pthread_mutex_unlock(&EventExitQueueMutex);
 249                 return eAbort;
 250         }
 251         ev_async_send (event_base, &AddJob);
 252         pthread_mutex_unlock(&EventExitQueueMutex);
 253         EVM_syslog(LOG_DEBUG, "EVENT Q Done.\n");
 254         return eSendReply;
 255 }
 256
 257 eNextState EventQueueDBOperation(AsyncIO *IO, IO_CallBack CB)
 258 {
 259         StopClientWatchers(IO, 0);
 260         return QueueDBOperation(IO, CB);
 261 }
 262 eNextState DBQueueEventContext(AsyncIO *IO, IO_CallBack CB)
 263 {
 264         StopDBWatchers(IO);
 265         return QueueEventContext(IO, CB);
 266 }
 267
 268 extern eNextState evcurl_handle_start(AsyncIO *IO);
 269
 270 eNextState QueueCurlContext(AsyncIO *IO)
 271 {
 272         IOAddHandler *h;
 273         int i;
 274
 275         SetEVState(IO, eCurlQ);
 276         h = (IOAddHandler*)malloc(sizeof(IOAddHandler));
 277         h->IO = IO;
 278         h->EvAttch = evcurl_handle_start;
 279
 280         pthread_mutex_lock(&EventQueueMutex);
 281         if (InboundEventQueue == NULL)
 282         {
 283                 /* shutting down... */
 284                 free(h);
 285                 EVM_syslog(LOG_DEBUG, "EVENT Q exiting.\n");
 286                 pthread_mutex_unlock(&EventQueueMutex);
 287                 return eAbort;
 288         }
 289
 290         EVM_syslog(LOG_DEBUG, "EVENT Q\n");
 291         i = ++evbase_count;
 292         Put(InboundEventQueue, IKEY(i), h, NULL);
 293         pthread_mutex_unlock(&EventQueueMutex);
 294
 295         pthread_mutex_lock(&EventExitQueueMutex);
 296         if (event_base == NULL) {
 297                 pthread_mutex_unlock(&EventExitQueueMutex);
 298                 return eAbort;
 299         }
 300         ev_async_send (event_base, &AddJob);
 301         pthread_mutex_unlock(&EventExitQueueMutex);
 302
 303         EVM_syslog(LOG_DEBUG, "EVENT Q Done.\n");
 304         return eSendReply;
 305 }
 306
 307 eNextState CurlQueueDBOperation(AsyncIO *IO, IO_CallBack CB)
 308 {
 309         StopCurlWatchers(IO);
 310         return QueueDBOperation(IO, CB);
 311 }
 312
 313
 314 void DestructCAres(AsyncIO *IO);
 315 void FreeAsyncIOContents(AsyncIO *IO)
 316 {
 317         CitContext *Ctx = IO->CitContext;
 318
 319         FreeStrBuf(&IO->IOBuf);
 320         FreeStrBuf(&IO->SendBuf.Buf);
 321         FreeStrBuf(&IO->RecvBuf.Buf);
 322
 323         DestructCAres(IO);
 324
 325         FreeURL(&IO->ConnectMe);
 326         FreeStrBuf(&IO->HttpReq.ReplyData);
 327
 328         if (Ctx) {
 329                 Ctx->state = CON_IDLE;
 330                 Ctx->kill_me = 1;
 331                 IO->CitContext = NULL;
 332         }
 333 }
 334
 335
 336 void StopClientWatchers(AsyncIO *IO, int CloseFD)
 337 {
 338         ev_timer_stop (event_base, &IO->rw_timeout);
 339         ev_timer_stop(event_base, &IO->conn_fail);
 340         ev_idle_stop(event_base, &IO->unwind_stack);
 341         ev_cleanup_stop(event_base, &IO->abort_by_shutdown);
 342
 343         ev_io_stop(event_base, &IO->conn_event);
 344         ev_io_stop(event_base, &IO->send_event);
 345         ev_io_stop(event_base, &IO->recv_event);
 346
 347         if (CloseFD && (IO->SendBuf.fd > 0)) {
 348                 close(IO->SendBuf.fd);
 349                 IO->SendBuf.fd = 0;
 350                 IO->RecvBuf.fd = 0;
 351         }
 352 }
 353
 354 void StopCurlWatchers(AsyncIO *IO)
 355 {
 356         EVM_syslog(LOG_DEBUG, "EVENT StopCurlWatchers \n");
 357
 358         ev_timer_stop (event_base, &IO->rw_timeout);
 359         ev_timer_stop(event_base, &IO->conn_fail);
 360         ev_idle_stop(event_base, &IO->unwind_stack);
 361         ev_cleanup_stop(event_base, &IO->abort_by_shutdown);
 362
 363         ev_io_stop(event_base, &IO->conn_event);
 364         ev_io_stop(event_base, &IO->send_event);
 365         ev_io_stop(event_base, &IO->recv_event);
 366
 367         curl_easy_cleanup(IO->HttpReq.chnd);
 368         IO->HttpReq.chnd = NULL;
 369
 370         if (IO->SendBuf.fd != 0) {
 371                 close(IO->SendBuf.fd);
 372         }
 373         IO->SendBuf.fd = 0;
 374         IO->RecvBuf.fd = 0;
 375 }
 376
 377 void ShutDownCLient(AsyncIO *IO)
 378 {
 379         CitContext *Ctx =IO->CitContext;
 380
 381         SetEVState(IO, eExit);
 382         become_session(Ctx);
 383
 384         EVM_syslog(LOG_DEBUG, "EVENT Terminating \n");
 385
 386         StopClientWatchers(IO, 1);
 387
 388         if (IO->DNS.Channel != NULL) {
 389                 ares_destroy(IO->DNS.Channel);
 390                 EV_DNS_LOG_STOP(DNS.recv_event);
 391                 EV_DNS_LOG_STOP(DNS.send_event);
 392                 ev_io_stop(event_base, &IO->DNS.recv_event);
 393                 ev_io_stop(event_base, &IO->DNS.send_event);
 394                 IO->DNS.Channel = NULL;
 395         }
 396         assert(IO->Terminate);
 397         IO->Terminate(IO);
 398 }
 399
 400 void PostInbound(AsyncIO *IO)
 401 {
 402         switch (IO->NextState) {
 403         case eSendFile:
 404                 ev_io_start(event_base, &IO->send_event);
 405                 break;
 406         case eSendReply:
 407         case eSendMore:
 408                 assert(IO->SendDone);
 409                 IO->NextState = IO->SendDone(IO);
 410                 switch (IO->NextState)
 411                 {
 412                 case eSendFile:
 413                 case eSendReply:
 414                 case eSendMore:
 415                 case eReadMessage:
 416                 case eReadPayload:
 417                 case eReadMore:
 418                 case eReadFile:
 419                         ev_io_start(event_base, &IO->send_event);
 420                         break;
 421                 case eDBQuery:
 422                         StopClientWatchers(IO, 0);
 423                 default:
 424                         break;
 425                 }
 426                 break;
 427         case eReadPayload:
 428         case eReadMore:
 429         case eReadFile:
 430                 ev_io_start(event_base, &IO->recv_event);
 431                 break;
 432         case eTerminateConnection:
 433                 ShutDownCLient(IO);
 434                 break;
 435         case eAbort:
 436                 ShutDownCLient(IO);
 437                 break;
 438         case eSendDNSQuery:
 439         case eReadDNSReply:
 440         case eDBQuery:
 441         case eConnect:
 442         case eReadMessage:
 443                 break;
 444         }
 445 }
 446 eReadState HandleInbound(AsyncIO *IO)
 447 {
 448         const char *Err = NULL;
 449         eReadState Finished = eBufferNotEmpty;
 450
 451         become_session(IO->CitContext);
 452
 453         while ((Finished == eBufferNotEmpty) &&
 454                ((IO->NextState == eReadMessage)||
 455                 (IO->NextState == eReadMore)||
 456                 (IO->NextState == eReadFile)||
 457                 (IO->NextState == eReadPayload)))
 458         {
 459                 /* Reading lines...
 460                  * lex line reply in callback,
 461                  * or do it ourselves.
 462                  * i.e. as nnn-blabla means continue reading in SMTP
 463                  */
 464                 if ((IO->NextState == eReadFile) &&
 465                     (Finished == eBufferNotEmpty))
 466                 {
 467                         Finished = WriteIOBAlreadyRead(&IO->IOB, &Err);
 468                         if (Finished == eReadSuccess)
 469                         {
 470                                 IO->NextState = eSendReply;
 471                         }
 472                 }
 473                 else if (IO->LineReader)
 474                         Finished = IO->LineReader(IO);
 475                 else
 476                         Finished = StrBufChunkSipLine(IO->IOBuf,
 477                                                       &IO->RecvBuf);
 478
 479                 switch (Finished) {
 480                 case eMustReadMore: /// read new from socket...
 481                         break;
 482                 case eBufferNotEmpty: /* shouldn't happen... */
 483                 case eReadSuccess: /// done for now...
 484                         break;
 485                 case eReadFail: /// WHUT?
 486                                 ///todo: shut down!
 487                         break;
 488                 }
 489
 490                 if (Finished != eMustReadMore) {
 491                         assert(IO->ReadDone);
 492                         ev_io_stop(event_base, &IO->recv_event);
 493                         IO->NextState = IO->ReadDone(IO);
 494                         Finished = StrBufCheckBuffer(&IO->RecvBuf);
 495                 }
 496         }
 497
 498         PostInbound(IO);
 499
 500         return Finished;
 501 }
 502
 503
 504 static void
 505 IO_send_callback(struct ev_loop *loop, ev_io *watcher, int revents)
 506 {
 507         int rc;
 508         AsyncIO *IO = watcher->data;
 509         const char *errmsg = NULL;
 510
 511         IO->Now = ev_now(event_base);
 512         become_session(IO->CitContext);
 513 #ifdef BIGBAD_IODBG
 514         {
 515                 int rv = 0;
 516                 char fn [SIZ];
 517                 FILE *fd;
 518                 const char *pch = ChrPtr(IO->SendBuf.Buf);
 519                 const char *pchh = IO->SendBuf.ReadWritePointer;
 520                 long nbytes;
 521
 522                 if (pchh == NULL)
 523                         pchh = pch;
 524
 525                 nbytes = StrLength(IO->SendBuf.Buf) - (pchh - pch);
 526                 snprintf(fn, SIZ, "/tmp/foolog_ev_%s.%d",
 527                          ((CitContext*)(IO->CitContext))->ServiceName,
 528                          IO->SendBuf.fd);
 529
 530                 fd = fopen(fn, "a+");
 531                 fprintf(fd, "Send: BufSize: %ld BufContent: [",
 532                         nbytes);
 533                 rv = fwrite(pchh, nbytes, 1, fd);
 534                 if (!rv) printf("failed to write debug to %s!\n", fn);
 535                 fprintf(fd, "]\n");
 536 #endif
 537                 switch (IO->NextState) {
 538                 case eSendFile:
 539                         rc = FileSendChunked(&IO->IOB, &errmsg);
 540                         if (rc < 0)
 541                                 StrBufPlain(IO->ErrMsg, errmsg, -1);
 542                         break;
 543                 default:
 544                         rc = StrBuf_write_one_chunk_callback(IO->SendBuf.fd,
 545                                                              0,
 546                                                              &IO->SendBuf);
 547                 }
 548
 549 #ifdef BIGBAD_IODBG
 550                 fprintf(fd, "Sent: BufSize: %d bytes.\n", rc);
 551                 fclose(fd);
 552         }
 553 #endif
 554         if (rc == 0)
 555         {
 556                 ev_io_stop(event_base, &IO->send_event);
 557                 switch (IO->NextState) {
 558                 case eSendMore:
 559                         assert(IO->SendDone);
 560                         IO->NextState = IO->SendDone(IO);
 561
 562                         if ((IO->NextState == eTerminateConnection) ||
 563                             (IO->NextState == eAbort) )
 564                                 ShutDownCLient(IO);
 565                         else {
 566                                 ev_io_start(event_base, &IO->send_event);
 567                         }
 568                         break;
 569                 case eSendFile:
 570                         if (IO->IOB.ChunkSendRemain > 0) {
 571                                 ev_io_start(event_base, &IO->recv_event);
 572                                 SetNextTimeout(IO, 100.0);
 573
 574                         } else {
 575                                 assert(IO->ReadDone);
 576                                 IO->NextState = IO->ReadDone(IO);
 577                                 switch(IO->NextState) {
 578                                 case eSendDNSQuery:
 579                                 case eReadDNSReply:
 580                                 case eDBQuery:
 581                                 case eConnect:
 582                                         break;
 583                                 case eSendReply:
 584                                 case eSendMore:
 585                                 case eSendFile:
 586                                         ev_io_start(event_base,
 587                                                     &IO->send_event);
 588                                         break;
 589                                 case eReadMessage:
 590                                 case eReadMore:
 591                                 case eReadPayload:
 592                                 case eReadFile:
 593                                         break;
 594                                 case eTerminateConnection:
 595                                 case eAbort:
 596                                         break;
 597                                 }
 598                         }
 599                         break;
 600                 case eSendReply:
 601                     if (StrBufCheckBuffer(&IO->SendBuf) != eReadSuccess)
 602                         break;
 603                     IO->NextState = eReadMore;
 604                 case eReadMore:
 605                 case eReadMessage:
 606                 case eReadPayload:
 607                 case eReadFile:
 608                         if (StrBufCheckBuffer(&IO->RecvBuf) == eBufferNotEmpty)
 609                         {
 610                                 HandleInbound(IO);
 611                         }
 612                         else {
 613                                 ev_io_start(event_base, &IO->recv_event);
 614                         }
 615
 616                         break;
 617                 case eDBQuery:
 618                         /*
 619                          * we now live in another queue,
 620                          * so we have to unregister.
 621                          */
 622                         ev_cleanup_stop(loop, &IO->abort_by_shutdown);
 623                         break;
 624                 case eSendDNSQuery:
 625                 case eReadDNSReply:
 626                 case eConnect:
 627                 case eTerminateConnection:
 628                 case eAbort:
 629                         break;
 630                 }
 631         }
 632         else if (rc < 0) {
 633                 if (errno != EAGAIN) {
 634                         StopClientWatchers(IO, 1);
 635                         EV_syslog(LOG_DEBUG,
 636                                   "IO_send_callback(): Socket Invalid! [%d] [%s] [%d]\n",
 637                                   errno, strerror(errno), IO->SendBuf.fd);
 638                         StrBufPrintf(IO->ErrMsg,
 639                                      "Socket Invalid! [%s]",
 640                                      strerror(errno));
 641                         SetNextTimeout(IO, 0.01);
 642                 }
 643         }
 644         /* else : must write more. */
 645 }
 646 static void
 647 set_start_callback(struct ev_loop *loop, AsyncIO *IO, int revents)
 648 {
 649         ev_timer_stop(event_base, &IO->conn_fail);
 650         ev_timer_start(event_base, &IO->rw_timeout);
 651
 652         switch(IO->NextState) {
 653         case eReadMore:
 654         case eReadMessage:
 655         case eReadFile:
 656                 StrBufAppendBufPlain(IO->ErrMsg, HKEY("[while waiting for greeting]"), 0);
 657                 ev_io_start(event_base, &IO->recv_event);
 658                 break;
 659         case eSendReply:
 660         case eSendMore:
 661         case eReadPayload:
 662         case eSendFile:
 663                 become_session(IO->CitContext);
 664                 IO_send_callback(loop, &IO->send_event, revents);
 665                 break;
 666         case eDBQuery:
 667         case eSendDNSQuery:
 668         case eReadDNSReply:
 669         case eConnect:
 670         case eTerminateConnection:
 671         case eAbort:
 672                 /// TODO: WHUT?
 673                 break;
 674         }
 675 }
 676
 677 static void
 678 IO_Timeout_callback(struct ev_loop *loop, ev_timer *watcher, int revents)
 679 {
 680         AsyncIO *IO = watcher->data;
 681
 682         SetEVState(IO, eIOTimeout);
 683         IO->Now = ev_now(event_base);
 684         ev_timer_stop (event_base, &IO->rw_timeout);
 685         become_session(IO->CitContext);
 686
 687         if (IO->SendBuf.fd != 0)
 688         {
 689                 ev_io_stop(event_base, &IO->send_event);
 690                 ev_io_stop(event_base, &IO->recv_event);
 691                 ev_timer_stop (event_base, &IO->rw_timeout);
 692                 close(IO->SendBuf.fd);
 693                 IO->SendBuf.fd = IO->RecvBuf.fd = 0;
 694         }
 695
 696         assert(IO->Timeout);
 697         switch (IO->Timeout(IO))
 698         {
 699         case eAbort:
 700                 ShutDownCLient(IO);
 701         default:
 702                 break;
 703         }
 704 }
 705
 706 static void
 707 IO_connfail_callback(struct ev_loop *loop, ev_timer *watcher, int revents)
 708 {
 709         AsyncIO *IO = watcher->data;
 710
 711         SetEVState(IO, eIOConnfail);
 712         IO->Now = ev_now(event_base);
 713         ev_timer_stop (event_base, &IO->conn_fail);
 714
 715         if (IO->SendBuf.fd != 0)
 716         {
 717                 ev_io_stop(loop, &IO->conn_event);
 718                 ev_io_stop(event_base, &IO->send_event);
 719                 ev_io_stop(event_base, &IO->recv_event);
 720                 ev_timer_stop (event_base, &IO->rw_timeout);
 721                 close(IO->SendBuf.fd);
 722                 IO->SendBuf.fd = IO->RecvBuf.fd = 0;
 723         }
 724         become_session(IO->CitContext);
 725
 726         assert(IO->ConnFail);
 727         switch (IO->ConnFail(IO))
 728         {
 729         case eAbort:
 730                 ShutDownCLient(IO);
 731         default:
 732                 break;
 733
 734         }
 735 }
 736
 737 static void
 738 IO_connfailimmediate_callback(struct ev_loop *loop,
 739                               ev_idle *watcher,
 740                               int revents)
 741 {
 742         AsyncIO *IO = watcher->data;
 743
 744         SetEVState(IO, eIOConnfailNow);
 745         IO->Now = ev_now(event_base);
 746         ev_idle_stop (event_base, &IO->conn_fail_immediate);
 747
 748         if (IO->SendBuf.fd != 0)
 749         {
 750                 close(IO->SendBuf.fd);
 751                 IO->SendBuf.fd = IO->RecvBuf.fd = 0;
 752         }
 753         become_session(IO->CitContext);
 754
 755         assert(IO->ConnFail);
 756         switch (IO->ConnFail(IO))
 757         {
 758         case eAbort:
 759                 ShutDownCLient(IO);
 760         default:
 761                 break;
 762
 763         }
 764 }
 765
 766 static void
 767 IO_connestd_callback(struct ev_loop *loop, ev_io *watcher, int revents)
 768 {
 769         AsyncIO *IO = watcher->data;
 770         int             so_err = 0;
 771         socklen_t       lon = sizeof(so_err);
 772         int             err;
 773
 774         SetEVState(IO, eIOConnNow);
 775         IO->Now = ev_now(event_base);
 776         EVM_syslog(LOG_DEBUG, "connect() succeeded.\n");
 777
 778         ev_io_stop(loop, &IO->conn_event);
 779         ev_timer_stop(event_base, &IO->conn_fail);
 780
 781         err = getsockopt(IO->SendBuf.fd,
 782                          SOL_SOCKET,
 783                          SO_ERROR,
 784                          (void*)&so_err,
 785                          &lon);
 786
 787         if ((err == 0) && (so_err != 0))
 788         {
 789                 EV_syslog(LOG_DEBUG, "connect() failed [%d][%s]\n",
 790                           so_err,
 791                           strerror(so_err));
 792                 IO_connfail_callback(loop, &IO->conn_fail, revents);
 793
 794         }
 795         else
 796         {
 797                 EVM_syslog(LOG_DEBUG, "connect() succeeded\n");
 798                 set_start_callback(loop, IO, revents);
 799         }
 800 }
 801
 802 static void
 803 IO_recv_callback(struct ev_loop *loop, ev_io *watcher, int revents)
 804 {
 805         const char *errmsg;
 806         ssize_t nbytes;
 807         AsyncIO *IO = watcher->data;
 808
 809         IO->Now = ev_now(event_base);
 810         switch (IO->NextState) {
 811         case eReadFile:
 812                 nbytes = FileRecvChunked(&IO->IOB, &errmsg);
 813                 if (nbytes < 0)
 814                         StrBufPlain(IO->ErrMsg, errmsg, -1);
 815                 else
 816                 {
 817                         if (IO->IOB.ChunkSendRemain == 0)
 818                         {
 819                                 IO->NextState = eSendReply;
 820                                 assert(IO->ReadDone);
 821                                 ev_io_stop(event_base, &IO->recv_event);
 822                                 PostInbound(IO);
 823                                 return;
 824                         }
 825                         else
 826                                 return;
 827                 }
 828                 break;
 829         default:
 830                 nbytes = StrBuf_read_one_chunk_callback(IO->RecvBuf.fd,
 831                                                         0,
 832                                                         &IO->RecvBuf);
 833                 break;
 834         }
 835
 836 #ifdef BIGBAD_IODBG
 837         {
 838                 long nbytes;
 839                 int rv = 0;
 840                 char fn [SIZ];
 841                 FILE *fd;
 842                 const char *pch = ChrPtr(IO->RecvBuf.Buf);
 843                 const char *pchh = IO->RecvBuf.ReadWritePointer;
 844
 845                 if (pchh == NULL)
 846                         pchh = pch;
 847
 848                 nbytes = StrLength(IO->RecvBuf.Buf) - (pchh - pch);
 849                 snprintf(fn, SIZ, "/tmp/foolog_ev_%s.%d",
 850                          ((CitContext*)(IO->CitContext))->ServiceName,
 851                          IO->SendBuf.fd);
 852
 853                 fd = fopen(fn, "a+");
 854                 fprintf(fd, "Read: BufSize: %ld BufContent: [",
 855                         nbytes);
 856                 rv = fwrite(pchh, nbytes, 1, fd);
 857                 if (!rv) printf("failed to write debug to %s!\n", fn);
 858                 fprintf(fd, "]\n");
 859                 fclose(fd);
 860         }
 861 #endif
 862         if (nbytes > 0) {
 863                 HandleInbound(IO);
 864         } else if (nbytes == 0) {
 865                 StopClientWatchers(IO, 1);
 866                 SetNextTimeout(IO, 0.01);
 867                 return;
 868         } else if (nbytes == -1) {
 869                 if (errno != EAGAIN) {
 870                         // FD is gone. kick it.
 871                         StopClientWatchers(IO, 1);
 872                         EV_syslog(LOG_DEBUG,
 873                                   "IO_recv_callback(): Socket Invalid! [%d] [%s] [%d]\n",
 874                                   errno, strerror(errno), IO->SendBuf.fd);
 875                         StrBufPrintf(IO->ErrMsg,
 876                                      "Socket Invalid! [%s]",
 877                                      strerror(errno));
 878                         SetNextTimeout(IO, 0.01);
 879                 }
 880                 return;
 881         }
 882 }
 883
 884 void
 885 IO_postdns_callback(struct ev_loop *loop, ev_idle *watcher, int revents)
 886 {
 887         AsyncIO *IO = watcher->data;
 888
 889         SetEVState(IO, eCaresFinished);
 890         IO->Now = ev_now(event_base);
 891         EV_syslog(LOG_DEBUG, "event: %s\n", __FUNCTION__);
 892         become_session(IO->CitContext);
 893         assert(IO->DNS.Query->PostDNS);
 894         switch (IO->DNS.Query->PostDNS(IO))
 895         {
 896         case eAbort:
 897                 assert(IO->DNS.Fail);
 898                 switch (IO->DNS.Fail(IO)) {
 899                 case eAbort:
 900 ////                    StopClientWatchers(IO);
 901                         ShutDownCLient(IO);
 902                 default:
 903                         break;
 904                 }
 905         default:
 906                 break;
 907         }
 908 }
 909
 910
 911 eNextState EvConnectSock(AsyncIO *IO,
 912                          double conn_timeout,
 913                          double first_rw_timeout,
 914                          int ReadFirst)
 915 {
 916         struct sockaddr_in egress_sin;
 917         int fdflags;
 918         int rc = -1;
 919
 920         SetEVState(IO, eIOConnectSock);
 921         become_session(IO->CitContext);
 922
 923         if (ReadFirst) {
 924                 IO->NextState = eReadMessage;
 925         }
 926         else {
 927                 IO->NextState = eSendReply;
 928         }
 929
 930         IO->SendBuf.fd = IO->RecvBuf.fd =
 931                 socket(
 932                         (IO->ConnectMe->IPv6)?PF_INET6:PF_INET,
 933                         SOCK_STREAM,
 934                         IPPROTO_TCP);
 935
 936         if (IO->SendBuf.fd < 0) {
 937                 EV_syslog(LOG_ERR,
 938                           "EVENT: socket() failed: %s\n",
 939                           strerror(errno));
 940
 941                 StrBufPrintf(IO->ErrMsg,
 942                              "Failed to create socket: %s",
 943                              strerror(errno));
 944                 IO->SendBuf.fd = IO->RecvBuf.fd = 0;
 945                 return eAbort;
 946         }
 947         fdflags = fcntl(IO->SendBuf.fd, F_GETFL);
 948         if (fdflags < 0) {
 949                 EV_syslog(LOG_ERR,
 950                           "EVENT: unable to get socket %d flags! %s \n",
 951                           IO->SendBuf.fd,
 952                           strerror(errno));
 953                 StrBufPrintf(IO->ErrMsg,
 954                              "Failed to get socket %d flags: %s",
 955                              IO->SendBuf.fd,
 956                              strerror(errno));
 957                 close(IO->SendBuf.fd);
 958                 IO->SendBuf.fd = IO->RecvBuf.fd = 0;
 959                 return eAbort;
 960         }
 961         fdflags = fdflags | O_NONBLOCK;
 962         if (fcntl(IO->SendBuf.fd, F_SETFL, fdflags) < 0) {
 963                 EV_syslog(
 964                         LOG_ERR,
 965                         "EVENT: unable to set socket %d nonblocking flags! %s \n",
 966                         IO->SendBuf.fd,
 967                         strerror(errno));
 968                 StrBufPrintf(IO->ErrMsg,
 969                              "Failed to set socket flags: %s",
 970                              strerror(errno));
 971                 close(IO->SendBuf.fd);
 972                 IO->SendBuf.fd = IO->RecvBuf.fd = 0;
 973                 return eAbort;
 974         }
 975 /* TODO: maye we could use offsetof() to calc the position of data...
 976  * http://doc.dvgu.ru/devel/ev.html#associating_custom_data_with_a_watcher
 977  */
 978         ev_io_init(&IO->recv_event, IO_recv_callback, IO->RecvBuf.fd, EV_READ);
 979         IO->recv_event.data = IO;
 980         ev_io_init(&IO->send_event, IO_send_callback, IO->SendBuf.fd, EV_WRITE);
 981         IO->send_event.data = IO;
 982
 983         ev_timer_init(&IO->conn_fail, IO_connfail_callback, conn_timeout, 0);
 984         IO->conn_fail.data = IO;
 985         ev_timer_init(&IO->rw_timeout, IO_Timeout_callback, first_rw_timeout,0);
 986         IO->rw_timeout.data = IO;
 987
 988
 989
 990
 991         /* for debugging you may bypass it like this:
 992          * IO->Addr.sin_addr.s_addr = inet_addr("127.0.0.1");
 993          * ((struct sockaddr_in)IO->ConnectMe->Addr).sin_addr.s_addr =
 994          *   inet_addr("127.0.0.1");
 995          */
 996         if (IO->ConnectMe->IPv6) {
 997                 rc = connect(IO->SendBuf.fd,
 998                              &IO->ConnectMe->Addr,
 999                              sizeof(struct sockaddr_in6));
1000         }
1001         else {
1002                 /* If citserver is bound to a specific IP address on the host, make
1003                  * sure we use that address for outbound connections.
1004                  */
1005
1006                 memset(&egress_sin, 0, sizeof(egress_sin));
1007                 egress_sin.sin_family = AF_INET;
1008                 if (!IsEmptyStr(config.c_ip_addr)) {
1009                         egress_sin.sin_addr.s_addr = inet_addr(config.c_ip_addr);
1010                         if (egress_sin.sin_addr.s_addr == !INADDR_ANY) {
1011                                 egress_sin.sin_addr.s_addr = INADDR_ANY;
1012                         }
1013
1014                         /* If this bind fails, no problem; we can still use INADDR_ANY */
1015                         bind(IO->SendBuf.fd, (struct sockaddr *)&egress_sin, sizeof(egress_sin));
1016                 }
1017                 rc = connect(IO->SendBuf.fd,
1018                              (struct sockaddr_in *)&IO->ConnectMe->Addr,
1019                              sizeof(struct sockaddr_in));
1020         }
1021
1022         if (rc >= 0){
1023                 SetEVState(IO, eIOConnNow);
1024                 EV_syslog(LOG_DEBUG, "connect() = %d immediate success.\n", IO->SendBuf.fd);
1025                 set_start_callback(event_base, IO, 0);
1026                 return IO->NextState;
1027         }
1028         else if (errno == EINPROGRESS) {
1029                 SetEVState(IO, eIOConnWait);
1030                 EV_syslog(LOG_DEBUG, "connect() = %d have to wait now.\n", IO->SendBuf.fd);
1031
1032                 ev_io_init(&IO->conn_event,
1033                            IO_connestd_callback,
1034                            IO->SendBuf.fd,
1035                            EV_READ|EV_WRITE);
1036
1037                 IO->conn_event.data = IO;
1038
1039                 ev_io_start(event_base, &IO->conn_event);
1040                 ev_timer_start(event_base, &IO->conn_fail);
1041                 return IO->NextState;
1042         }
1043         else {
1044                 SetEVState(IO, eIOConnfail);
1045                 ev_idle_init(&IO->conn_fail_immediate,
1046                              IO_connfailimmediate_callback);
1047                 IO->conn_fail_immediate.data = IO;
1048                 ev_idle_start(event_base, &IO->conn_fail_immediate);
1049
1050                 EV_syslog(LOG_ERR,
1051                           "connect() = %d failed: %s\n",
1052                           IO->SendBuf.fd,
1053                           strerror(errno));
1054
1055                 StrBufPrintf(IO->ErrMsg,
1056                              "Failed to connect: %s",
1057                              strerror(errno));
1058                 return IO->NextState;
1059         }
1060         return IO->NextState;
1061 }
1062
1063 void SetNextTimeout(AsyncIO *IO, double timeout)
1064 {
1065         IO->rw_timeout.repeat = timeout;
1066         ev_timer_again (event_base,  &IO->rw_timeout);
1067 }
1068
1069
1070 eNextState ReAttachIO(AsyncIO *IO,
1071                       void *pData,
1072                       int ReadFirst)
1073 {
1074         SetEVState(IO, eIOAttach);
1075         IO->Data = pData;
1076         become_session(IO->CitContext);
1077         ev_cleanup_start(event_base, &IO->abort_by_shutdown);
1078         if (ReadFirst) {
1079                 IO->NextState = eReadMessage;
1080         }
1081         else {
1082                 IO->NextState = eSendReply;
1083         }
1084         set_start_callback(event_base, IO, 0);
1085
1086         return IO->NextState;
1087 }
1088
1089 void InitIOStruct(AsyncIO *IO,
1090                   void *Data,
1091                   eNextState NextState,
1092                   IO_LineReaderCallback LineReader,
1093                   IO_CallBack DNS_Fail,
1094                   IO_CallBack SendDone,
1095                   IO_CallBack ReadDone,
1096                   IO_CallBack Terminate,
1097                   IO_CallBack DBTerminate,
1098                   IO_CallBack ConnFail,
1099                   IO_CallBack Timeout,
1100                   IO_CallBack ShutdownAbort)
1101 {
1102         IO->Data          = Data;
1103
1104         IO->CitContext    = CloneContext(CC);
1105         IO->CitContext->session_specific_data = Data;
1106         IO->CitContext->IO = IO;
1107
1108         IO->NextState     = NextState;
1109
1110         IO->SendDone      = SendDone;
1111         IO->ReadDone      = ReadDone;
1112         IO->Terminate     = Terminate;
1113         IO->DBTerminate   = DBTerminate;
1114         IO->LineReader    = LineReader;
1115         IO->ConnFail      = ConnFail;
1116         IO->Timeout       = Timeout;
1117         IO->ShutdownAbort = ShutdownAbort;
1118
1119         IO->DNS.Fail      = DNS_Fail;
1120
1121         IO->SendBuf.Buf   = NewStrBufPlain(NULL, 1024);
1122         IO->RecvBuf.Buf   = NewStrBufPlain(NULL, 1024);
1123         IO->IOBuf         = NewStrBuf();
1124         EV_syslog(LOG_DEBUG,
1125                   "EVENT: Session lives at %p IO at %p \n",
1126                   Data, IO);
1127
1128 }
1129
1130 extern int evcurl_init(AsyncIO *IO);
1131
1132 int InitcURLIOStruct(AsyncIO *IO,
1133                      void *Data,
1134                      const char* Desc,
1135                      IO_CallBack SendDone,
1136                      IO_CallBack Terminate,
1137                      IO_CallBack DBTerminate,
1138                      IO_CallBack ShutdownAbort)
1139 {
1140         IO->Data          = Data;
1141
1142         IO->CitContext    = CloneContext(CC);
1143         IO->CitContext->session_specific_data = Data;
1144         IO->CitContext->IO = IO;
1145
1146         IO->SendDone      = SendDone;
1147         IO->Terminate     = Terminate;
1148         IO->DBTerminate   = DBTerminate;
1149         IO->ShutdownAbort = ShutdownAbort;
1150
1151         strcpy(IO->HttpReq.errdesc, Desc);
1152
1153
1154         return  evcurl_init(IO);
1155
1156 }
1157
1158
1159 typedef struct KillOtherSessionContext {
1160         AsyncIO IO;
1161         AsyncIO *OtherOne;
1162 }KillOtherSessionContext;
1163
1164 eNextState KillTerminate(AsyncIO *IO)
1165 {
1166         long id;
1167         KillOtherSessionContext *Ctx = (KillOtherSessionContext*)IO->Data;
1168         EV_syslog(LOG_DEBUG, "%s Exit\n", __FUNCTION__);
1169         id = IO->ID;
1170         FreeAsyncIOContents(IO);
1171         memset(Ctx, 0, sizeof(KillOtherSessionContext));
1172         IO->ID = id; /* just for the case we want to analyze it in a coredump */
1173         free(Ctx);
1174         return eAbort;
1175
1176 }
1177
1178 eNextState KillShutdown(AsyncIO *IO)
1179 {
1180         return eTerminateConnection;
1181 }
1182
1183 eNextState KillOtherContextNow(AsyncIO *IO)
1184 {
1185         KillOtherSessionContext *Ctx = IO->Data;
1186
1187         SetEVState(IO, eKill);
1188
1189         if (Ctx->OtherOne->ShutdownAbort != NULL)
1190                 Ctx->OtherOne->ShutdownAbort(Ctx->OtherOne);
1191         return eTerminateConnection;
1192 }
1193
1194 void KillAsyncIOContext(AsyncIO *IO)
1195 {
1196         KillOtherSessionContext *Ctx;
1197
1198         Ctx = (KillOtherSessionContext*) malloc(sizeof(KillOtherSessionContext));
1199         memset(Ctx, 0, sizeof(KillOtherSessionContext));
1200
1201         InitIOStruct(&Ctx->IO,
1202                      Ctx,
1203                      eReadMessage,
1204                      NULL,
1205                      NULL,
1206                      NULL,
1207                      NULL,
1208                      KillTerminate,
1209                      NULL,
1210                      NULL,
1211                      NULL,
1212                      KillShutdown);
1213
1214         Ctx->OtherOne = IO;
1215
1216         switch(IO->NextState) {
1217         case eSendDNSQuery:
1218         case eReadDNSReply:
1219
1220         case eConnect:
1221         case eSendReply:
1222         case eSendMore:
1223         case eSendFile:
1224
1225         case eReadMessage:
1226         case eReadMore:
1227         case eReadPayload:
1228         case eReadFile:
1229                 QueueEventContext(&Ctx->IO, KillOtherContextNow);
1230                 break;
1231         case eDBQuery:
1232                 QueueDBOperation(&Ctx->IO, KillOtherContextNow);
1233                 break;
1234         case eTerminateConnection:
1235         case eAbort:
1236                 /*hm, its already dying, dunno which Queue its in... */
1237                 free(Ctx);
1238         }
1239
1240 }
1241
1242 extern int DebugEventLoopBacktrace;
1243 void EV_backtrace(AsyncIO *IO)
1244 {
1245 #ifdef HAVE_BACKTRACE
1246         void *stack_frames[50];
1247         size_t size, i;
1248         char **strings;
1249
1250         if ((IO == NULL) || (DebugEventLoopBacktrace == 0))
1251                 return;
1252         size = backtrace(stack_frames, sizeof(stack_frames) / sizeof(void*));
1253         strings = backtrace_symbols(stack_frames, size);
1254         for (i = 0; i < size; i++) {
1255                 if (strings != NULL) {
1256                         EV_syslog(LOG_ALERT, " BT %s\n", strings[i]);
1257                 }
1258                 else {
1259                         EV_syslog(LOG_ALERT, " BT %p\n", stack_frames[i]);
1260                 }
1261         }
1262         free(strings);
1263 #endif
1264 }
1265
1266
1267 ev_tstamp ctdl_ev_now (void)
1268 {
1269         return ev_now(event_base);
1270 }