citadel/event_client.c

   1 /*
   2  * Copyright (c) 1998-2012 by the citadel.org team
   3  *
   4  * This program is open source software; you can redistribute it and/or modify
   5  * it under the terms of the GNU General Public License, version 3.
   6  *
   7  * This program is distributed in the hope that it will be useful,
   8  * but WITHOUT ANY WARRANTY; without even the implied warranty of
   9  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  10  * GNU General Public License for more details.
  11  */
  12
  13 #include "sysdep.h"
  14 #include <stdlib.h>
  15 #include <unistd.h>
  16 #include <stdio.h>
  17 #include <termios.h>
  18 #include <fcntl.h>
  19 #include <signal.h>
  20 #include <pwd.h>
  21 #include <errno.h>
  22 #include <sys/types.h>
  23 #include <syslog.h>
  24
  25 #if TIME_WITH_SYS_TIME
  26 # include <sys/time.h>
  27 # include <time.h>
  28 #else
  29 # if HAVE_SYS_TIME_H
  30 #  include <sys/time.h>
  31 # else
  32 #  include <time.h>
  33 # endif
  34 #endif
  35 #include <sys/wait.h>
  36 #include <ctype.h>
  37 #include <string.h>
  38 #include <limits.h>
  39 #include <sys/socket.h>
  40 #include <netinet/in.h>
  41 #include <arpa/inet.h>
  42 #include <assert.h>
  43 #if HAVE_BACKTRACE
  44 #include <execinfo.h>
  45 #endif
  46
  47 #include <libcitadel.h>
  48 #include "citadel.h"
  49 #include "server.h"
  50 #include "citserver.h"
  51 #include "support.h"
  52 #include "config.h"
  53 #include "control.h"
  54 #include "user_ops.h"
  55 #include "database.h"
  56 #include "msgbase.h"
  57 #include "internet_addressing.h"
  58 #include "genstamp.h"
  59 #include "domain.h"
  60 #include "clientsocket.h"
  61 #include "locate_host.h"
  62 #include "citadel_dirs.h"
  63
  64 #include "event_client.h"
  65 #include "ctdl_module.h"
  66
  67
  68 ConstStr IOStates[] = {
  69         {HKEY("DB Queue")},
  70         {HKEY("DB Q Next")},
  71         {HKEY("DB Attach")},
  72         {HKEY("DB Next")},
  73         {HKEY("DB Stop")},
  74         {HKEY("DB Exit")},
  75         {HKEY("DB Terminate")},
  76         {HKEY("IO Queue")},
  77         {HKEY("IO Attach")},
  78         {HKEY("IO Connect Socket")},
  79         {HKEY("IO Abort")},
  80         {HKEY("IO Timeout")},
  81         {HKEY("IO ConnFail")},
  82         {HKEY("IO ConnFail Now")},
  83         {HKEY("IO Conn Now")},
  84         {HKEY("IO Conn Wait")},
  85         {HKEY("Curl Q")},
  86         {HKEY("Curl Start")},
  87         {HKEY("Curl Shotdown")},
  88         {HKEY("Curl More IO")},
  89         {HKEY("Curl Got IO")},
  90         {HKEY("Curl Got Data")},
  91         {HKEY("Curl Got Status")},
  92         {HKEY("C-Ares Start")},
  93         {HKEY("C-Ares IO Done")},
  94         {HKEY("C-Ares Finished")},
  95         {HKEY("C-Ares exit")},
  96         {HKEY("Killing")},
  97         {HKEY("Exit")}
  98 };
  99
 100 void SetEVState(AsyncIO *IO, eIOState State)
 101 {
 102
 103         CitContext* CCC = IO->CitContext;
 104         memcpy(CCC->lastcmdname, IOStates[State].Key, IOStates[State].len + 1);
 105
 106 }
 107
 108
 109 static void IO_Timeout_callback(struct ev_loop *loop, ev_timer *watcher, int revents);
 110 static void IO_abort_shutdown_callback(struct ev_loop *loop,
 111                                        ev_cleanup *watcher,
 112                                        int revents);
 113
 114
 115 /*------------------------------------------------------------------------------
 116  *                              Server DB IO
 117  *----------------------------------------------------------------------------*/
 118 extern int evdb_count;
 119 extern pthread_mutex_t DBEventQueueMutex;
 120 extern pthread_mutex_t DBEventExitQueueMutex;
 121 extern HashList *DBInboundEventQueue;
 122 extern struct ev_loop *event_db;
 123 extern ev_async DBAddJob;
 124 extern ev_async DBExitEventLoop;
 125
 126 eNextState QueueDBOperation(AsyncIO *IO, IO_CallBack CB)
 127 {
 128         IOAddHandler *h;
 129         int i;
 130
 131         SetEVState(IO, eDBQ);
 132         h = (IOAddHandler*)malloc(sizeof(IOAddHandler));
 133         h->IO = IO;
 134         h->EvAttch = CB;
 135         ev_cleanup_init(&IO->db_abort_by_shutdown,
 136                         IO_abort_shutdown_callback);
 137         IO->db_abort_by_shutdown.data = IO;
 138
 139         pthread_mutex_lock(&DBEventQueueMutex);
 140         if (DBInboundEventQueue == NULL)
 141         {
 142                 /* shutting down... */
 143                 free(h);
 144                 EVM_syslog(LOG_DEBUG, "DBEVENT Q exiting.\n");
 145                 pthread_mutex_unlock(&DBEventQueueMutex);
 146                 return eAbort;
 147         }
 148         EVM_syslog(LOG_DEBUG, "DBEVENT Q\n");
 149         i = ++evdb_count ;
 150         Put(DBInboundEventQueue, IKEY(i), h, NULL);
 151         pthread_mutex_unlock(&DBEventQueueMutex);
 152
 153         pthread_mutex_lock(&DBEventExitQueueMutex);
 154         if (event_db == NULL)
 155         {
 156                 pthread_mutex_unlock(&DBEventExitQueueMutex);
 157                 return eAbort;
 158         }
 159         ev_async_send (event_db, &DBAddJob);
 160         pthread_mutex_unlock(&DBEventExitQueueMutex);
 161
 162         EVM_syslog(LOG_DEBUG, "DBEVENT Q Done.\n");
 163         return eDBQuery;
 164 }
 165
 166 void StopDBWatchers(AsyncIO *IO)
 167 {
 168         SetEVState(IO, eDBStop);
 169         ev_cleanup_stop(event_db, &IO->db_abort_by_shutdown);
 170         ev_idle_stop(event_db, &IO->db_unwind_stack);
 171 }
 172
 173 void ShutDownDBCLient(AsyncIO *IO)
 174 {
 175         CitContext *Ctx =IO->CitContext;
 176         become_session(Ctx);
 177
 178         SetEVState(IO, eDBTerm);
 179         EVM_syslog(LOG_DEBUG, "DBEVENT Terminating.\n");
 180         StopDBWatchers(IO);
 181
 182         assert(IO->DBTerminate);
 183         IO->DBTerminate(IO);
 184 }
 185
 186 void
 187 DB_PerformNext(struct ev_loop *loop, ev_idle *watcher, int revents)
 188 {
 189         AsyncIO *IO = watcher->data;
 190
 191         SetEVState(IO, eDBNext);
 192         IO->Now = ev_now(event_db);
 193         EV_syslog(LOG_DEBUG, "%s()", __FUNCTION__);
 194         become_session(IO->CitContext);
 195
 196         ev_idle_stop(event_db, &IO->db_unwind_stack);
 197
 198         assert(IO->NextDBOperation);
 199         switch (IO->NextDBOperation(IO))
 200         {
 201         case eDBQuery:
 202                 break;
 203         case eSendDNSQuery:
 204         case eReadDNSReply:
 205         case eConnect:
 206         case eSendReply:
 207         case eSendMore:
 208         case eSendFile:
 209         case eReadMessage:
 210         case eReadMore:
 211         case eReadPayload:
 212         case eReadFile:
 213                 ev_cleanup_stop(loop, &IO->db_abort_by_shutdown);
 214                 break;
 215         case eTerminateConnection:
 216         case eAbort:
 217                 ev_idle_stop(event_db, &IO->db_unwind_stack);
 218                 ev_cleanup_stop(loop, &IO->db_abort_by_shutdown);
 219                 ShutDownDBCLient(IO);
 220         }
 221 }
 222
 223 eNextState NextDBOperation(AsyncIO *IO, IO_CallBack CB)
 224 {
 225         SetEVState(IO, eQDBNext);
 226         IO->NextDBOperation = CB;
 227         ev_idle_init(&IO->db_unwind_stack,
 228                      DB_PerformNext);
 229         IO->db_unwind_stack.data = IO;
 230         ev_idle_start(event_db, &IO->db_unwind_stack);
 231         return eDBQuery;
 232 }
 233
 234 /*------------------------------------------------------------------------------
 235  *                      Client IO
 236  *----------------------------------------------------------------------------*/
 237 extern int evbase_count;
 238 extern pthread_mutex_t EventQueueMutex;
 239 extern pthread_mutex_t EventExitQueueMutex;
 240 extern HashList *InboundEventQueue;
 241 extern struct ev_loop *event_base;
 242 extern ev_async AddJob;
 243 extern ev_async ExitEventLoop;
 244
 245 static void IO_abort_shutdown_callback(struct ev_loop *loop,
 246                                        ev_cleanup *watcher,
 247                                        int revents)
 248 {
 249         AsyncIO *IO = watcher->data;
 250
 251         SetEVState(IO, eIOAbort);
 252         EV_syslog(LOG_DEBUG, "EVENT Q: %s\n", __FUNCTION__);
 253         IO->Now = ev_now(event_base);
 254         assert(IO->ShutdownAbort);
 255         IO->ShutdownAbort(IO);
 256 }
 257
 258
 259 eNextState QueueEventContext(AsyncIO *IO, IO_CallBack CB)
 260 {
 261         IOAddHandler *h;
 262         int i;
 263
 264         SetEVState(IO, eIOQ);
 265         h = (IOAddHandler*)malloc(sizeof(IOAddHandler));
 266         h->IO = IO;
 267         h->EvAttch = CB;
 268         ev_cleanup_init(&IO->abort_by_shutdown,
 269                         IO_abort_shutdown_callback);
 270         IO->abort_by_shutdown.data = IO;
 271
 272         pthread_mutex_lock(&EventQueueMutex);
 273         if (InboundEventQueue == NULL)
 274         {
 275                 free(h);
 276                 /* shutting down... */
 277                 EVM_syslog(LOG_DEBUG, "EVENT Q exiting.\n");
 278                 pthread_mutex_unlock(&EventQueueMutex);
 279                 return eAbort;
 280         }
 281         EVM_syslog(LOG_DEBUG, "EVENT Q\n");
 282         i = ++evbase_count;
 283         Put(InboundEventQueue, IKEY(i), h, NULL);
 284         pthread_mutex_unlock(&EventQueueMutex);
 285
 286         pthread_mutex_lock(&EventExitQueueMutex);
 287         if (event_base == NULL) {
 288                 pthread_mutex_unlock(&EventExitQueueMutex);
 289                 return eAbort;
 290         }
 291         ev_async_send (event_base, &AddJob);
 292         pthread_mutex_unlock(&EventExitQueueMutex);
 293         EVM_syslog(LOG_DEBUG, "EVENT Q Done.\n");
 294         return eSendReply;
 295 }
 296
 297 extern eNextState evcurl_handle_start(AsyncIO *IO);
 298
 299 eNextState QueueCurlContext(AsyncIO *IO)
 300 {
 301         IOAddHandler *h;
 302         int i;
 303
 304         SetEVState(IO, eCurlQ);
 305         h = (IOAddHandler*)malloc(sizeof(IOAddHandler));
 306         h->IO = IO;
 307         h->EvAttch = evcurl_handle_start;
 308
 309         pthread_mutex_lock(&EventQueueMutex);
 310         if (InboundEventQueue == NULL)
 311         {
 312                 /* shutting down... */
 313                 free(h);
 314                 EVM_syslog(LOG_DEBUG, "EVENT Q exiting.\n");
 315                 pthread_mutex_unlock(&EventQueueMutex);
 316                 return eAbort;
 317         }
 318
 319         EVM_syslog(LOG_DEBUG, "EVENT Q\n");
 320         i = ++evbase_count;
 321         Put(InboundEventQueue, IKEY(i), h, NULL);
 322         pthread_mutex_unlock(&EventQueueMutex);
 323
 324         pthread_mutex_lock(&EventExitQueueMutex);
 325         if (event_base == NULL) {
 326                 pthread_mutex_unlock(&EventExitQueueMutex);
 327                 return eAbort;
 328         }
 329         ev_async_send (event_base, &AddJob);
 330         pthread_mutex_unlock(&EventExitQueueMutex);
 331
 332         EVM_syslog(LOG_DEBUG, "EVENT Q Done.\n");
 333         return eSendReply;
 334 }
 335
 336 void DestructCAres(AsyncIO *IO);
 337 void FreeAsyncIOContents(AsyncIO *IO)
 338 {
 339         CitContext *Ctx = IO->CitContext;
 340
 341         FreeStrBuf(&IO->IOBuf);
 342         FreeStrBuf(&IO->SendBuf.Buf);
 343         FreeStrBuf(&IO->RecvBuf.Buf);
 344
 345         DestructCAres(IO);
 346
 347         FreeURL(&IO->ConnectMe);
 348         FreeStrBuf(&IO->HttpReq.ReplyData);
 349
 350         if (Ctx) {
 351                 Ctx->state = CON_IDLE;
 352                 Ctx->kill_me = 1;
 353                 IO->CitContext = NULL;
 354         }
 355 }
 356
 357
 358 void StopClientWatchers(AsyncIO *IO, int CloseFD)
 359 {
 360         ev_timer_stop (event_base, &IO->rw_timeout);
 361         ev_timer_stop(event_base, &IO->conn_fail);
 362         ev_idle_stop(event_base, &IO->unwind_stack);
 363         ev_cleanup_stop(event_base, &IO->abort_by_shutdown);
 364
 365         ev_io_stop(event_base, &IO->conn_event);
 366         ev_io_stop(event_base, &IO->send_event);
 367         ev_io_stop(event_base, &IO->recv_event);
 368
 369         if (CloseFD && (IO->SendBuf.fd > 0)) {
 370                 close(IO->SendBuf.fd);
 371                 IO->SendBuf.fd = 0;
 372                 IO->RecvBuf.fd = 0;
 373         }
 374 }
 375
 376 void StopCurlWatchers(AsyncIO *IO)
 377 {
 378         ev_timer_stop (event_base, &IO->rw_timeout);
 379         ev_timer_stop(event_base, &IO->conn_fail);
 380         ev_idle_stop(event_base, &IO->unwind_stack);
 381         ev_cleanup_stop(event_base, &IO->abort_by_shutdown);
 382
 383         ev_io_stop(event_base, &IO->conn_event);
 384         ev_io_stop(event_base, &IO->send_event);
 385         ev_io_stop(event_base, &IO->recv_event);
 386
 387         if (IO->SendBuf.fd != 0) {
 388                 close(IO->SendBuf.fd);
 389         }
 390         IO->SendBuf.fd = 0;
 391         IO->RecvBuf.fd = 0;
 392 }
 393
 394 void ShutDownCLient(AsyncIO *IO)
 395 {
 396         CitContext *Ctx =IO->CitContext;
 397
 398         SetEVState(IO, eExit);
 399         become_session(Ctx);
 400
 401         EVM_syslog(LOG_DEBUG, "EVENT Terminating \n");
 402
 403         StopClientWatchers(IO, 1);
 404
 405         if (IO->DNS.Channel != NULL) {
 406                 ares_destroy(IO->DNS.Channel);
 407                 EV_DNS_LOG_STOP(DNS.recv_event);
 408                 EV_DNS_LOG_STOP(DNS.send_event);
 409                 ev_io_stop(event_base, &IO->DNS.recv_event);
 410                 ev_io_stop(event_base, &IO->DNS.send_event);
 411                 IO->DNS.Channel = NULL;
 412         }
 413         assert(IO->Terminate);
 414         IO->Terminate(IO);
 415 }
 416
 417 void PostInbound(AsyncIO *IO)
 418 {
 419         switch (IO->NextState) {
 420         case eSendFile:
 421                 ev_io_start(event_base, &IO->send_event);
 422                 break;
 423         case eSendReply:
 424         case eSendMore:
 425                 assert(IO->SendDone);
 426                 IO->NextState = IO->SendDone(IO);
 427                 switch (IO->NextState)
 428                 {
 429                 case eSendFile:
 430                 case eSendReply:
 431                 case eSendMore:
 432                 case eReadMessage:
 433                 case eReadPayload:
 434                 case eReadMore:
 435                 case eReadFile:
 436                         ev_io_start(event_base, &IO->send_event);
 437                         break;
 438                 case eDBQuery:
 439                         StopClientWatchers(IO, 0);
 440                 default:
 441                         break;
 442                 }
 443                 break;
 444         case eReadPayload:
 445         case eReadMore:
 446         case eReadFile:
 447                 ev_io_start(event_base, &IO->recv_event);
 448                 break;
 449         case eTerminateConnection:
 450                 ShutDownCLient(IO);
 451                 break;
 452         case eAbort:
 453                 ShutDownCLient(IO);
 454                 break;
 455         case eSendDNSQuery:
 456         case eReadDNSReply:
 457         case eDBQuery:
 458         case eConnect:
 459         case eReadMessage:
 460                 break;
 461         }
 462 }
 463 eReadState HandleInbound(AsyncIO *IO)
 464 {
 465         const char *Err = NULL;
 466         eReadState Finished = eBufferNotEmpty;
 467
 468         become_session(IO->CitContext);
 469
 470         while ((Finished == eBufferNotEmpty) &&
 471                ((IO->NextState == eReadMessage)||
 472                 (IO->NextState == eReadMore)||
 473                 (IO->NextState == eReadFile)||
 474                 (IO->NextState == eReadPayload)))
 475         {
 476                 /* Reading lines...
 477                  * lex line reply in callback,
 478                  * or do it ourselves.
 479                  * i.e. as nnn-blabla means continue reading in SMTP
 480                  */
 481                 if ((IO->NextState == eReadFile) &&
 482                     (Finished == eBufferNotEmpty))
 483                 {
 484                         Finished = WriteIOBAlreadyRead(&IO->IOB, &Err);
 485                         if (Finished == eReadSuccess)
 486                         {
 487                                 IO->NextState = eSendReply;
 488                         }
 489                 }
 490                 else if (IO->LineReader)
 491                         Finished = IO->LineReader(IO);
 492                 else
 493                         Finished = StrBufChunkSipLine(IO->IOBuf,
 494                                                       &IO->RecvBuf);
 495
 496                 switch (Finished) {
 497                 case eMustReadMore: /// read new from socket...
 498                         break;
 499                 case eBufferNotEmpty: /* shouldn't happen... */
 500                 case eReadSuccess: /// done for now...
 501                         break;
 502                 case eReadFail: /// WHUT?
 503                                 ///todo: shut down!
 504                         break;
 505                 }
 506
 507                 if (Finished != eMustReadMore) {
 508                         assert(IO->ReadDone);
 509                         ev_io_stop(event_base, &IO->recv_event);
 510                         IO->NextState = IO->ReadDone(IO);
 511                         Finished = StrBufCheckBuffer(&IO->RecvBuf);
 512                 }
 513         }
 514
 515         PostInbound(IO);
 516
 517         return Finished;
 518 }
 519
 520
 521 static void
 522 IO_send_callback(struct ev_loop *loop, ev_io *watcher, int revents)
 523 {
 524         int rc;
 525         AsyncIO *IO = watcher->data;
 526         const char *errmsg = NULL;
 527
 528         IO->Now = ev_now(event_base);
 529         become_session(IO->CitContext);
 530 #ifdef BIGBAD_IODBG
 531         {
 532                 int rv = 0;
 533                 char fn [SIZ];
 534                 FILE *fd;
 535                 const char *pch = ChrPtr(IO->SendBuf.Buf);
 536                 const char *pchh = IO->SendBuf.ReadWritePointer;
 537                 long nbytes;
 538
 539                 if (pchh == NULL)
 540                         pchh = pch;
 541
 542                 nbytes = StrLength(IO->SendBuf.Buf) - (pchh - pch);
 543                 snprintf(fn, SIZ, "/tmp/foolog_ev_%s.%d",
 544                          ((CitContext*)(IO->CitContext))->ServiceName,
 545                          IO->SendBuf.fd);
 546
 547                 fd = fopen(fn, "a+");
 548                 fprintf(fd, "Send: BufSize: %ld BufContent: [",
 549                         nbytes);
 550                 rv = fwrite(pchh, nbytes, 1, fd);
 551                 if (!rv) printf("failed to write debug to %s!\n", fn);
 552                 fprintf(fd, "]\n");
 553 #endif
 554                 switch (IO->NextState) {
 555                 case eSendFile:
 556                         rc = FileSendChunked(&IO->IOB, &errmsg);
 557                         if (rc < 0)
 558                                 StrBufPlain(IO->ErrMsg, errmsg, -1);
 559                         break;
 560                 default:
 561                         rc = StrBuf_write_one_chunk_callback(IO->SendBuf.fd,
 562                                                              0,
 563                                                              &IO->SendBuf);
 564                 }
 565
 566 #ifdef BIGBAD_IODBG
 567                 fprintf(fd, "Sent: BufSize: %d bytes.\n", rc);
 568                 fclose(fd);
 569         }
 570 #endif
 571         if (rc == 0)
 572         {
 573                 ev_io_stop(event_base, &IO->send_event);
 574                 switch (IO->NextState) {
 575                 case eSendMore:
 576                         assert(IO->SendDone);
 577                         IO->NextState = IO->SendDone(IO);
 578
 579                         if ((IO->NextState == eTerminateConnection) ||
 580                             (IO->NextState == eAbort) )
 581                                 ShutDownCLient(IO);
 582                         else {
 583                                 ev_io_start(event_base, &IO->send_event);
 584                         }
 585                         break;
 586                 case eSendFile:
 587                         if (IO->IOB.ChunkSendRemain > 0) {
 588                                 ev_io_start(event_base, &IO->recv_event);
 589                                 SetNextTimeout(IO, 100.0);
 590
 591                         } else {
 592                                 assert(IO->ReadDone);
 593                                 IO->NextState = IO->ReadDone(IO);
 594                                 switch(IO->NextState) {
 595                                 case eSendDNSQuery:
 596                                 case eReadDNSReply:
 597                                 case eDBQuery:
 598                                 case eConnect:
 599                                         break;
 600                                 case eSendReply:
 601                                 case eSendMore:
 602                                 case eSendFile:
 603                                         ev_io_start(event_base,
 604                                                     &IO->send_event);
 605                                         break;
 606                                 case eReadMessage:
 607                                 case eReadMore:
 608                                 case eReadPayload:
 609                                 case eReadFile:
 610                                         break;
 611                                 case eTerminateConnection:
 612                                 case eAbort:
 613                                         break;
 614                                 }
 615                         }
 616                         break;
 617                 case eSendReply:
 618                     if (StrBufCheckBuffer(&IO->SendBuf) != eReadSuccess)
 619                         break;
 620                     IO->NextState = eReadMore;
 621                 case eReadMore:
 622                 case eReadMessage:
 623                 case eReadPayload:
 624                 case eReadFile:
 625                         if (StrBufCheckBuffer(&IO->RecvBuf) == eBufferNotEmpty)
 626                         {
 627                                 HandleInbound(IO);
 628                         }
 629                         else {
 630                                 ev_io_start(event_base, &IO->recv_event);
 631                         }
 632
 633                         break;
 634                 case eDBQuery:
 635                         /*
 636                          * we now live in another queue,
 637                          * so we have to unregister.
 638                          */
 639                         ev_cleanup_stop(loop, &IO->abort_by_shutdown);
 640                         break;
 641                 case eSendDNSQuery:
 642                 case eReadDNSReply:
 643                 case eConnect:
 644                 case eTerminateConnection:
 645                 case eAbort:
 646                         break;
 647                 }
 648         }
 649         else if (rc < 0) {
 650                 if (errno != EAGAIN) {
 651                         StopClientWatchers(IO, 1);
 652                         EV_syslog(LOG_DEBUG,
 653                                   "IO_send_callback(): Socket Invalid! [%d] [%s] [%d]\n",
 654                                   errno, strerror(errno), IO->SendBuf.fd);
 655                         StrBufPrintf(IO->ErrMsg,
 656                                      "Socket Invalid! [%s]",
 657                                      strerror(errno));
 658                         SetNextTimeout(IO, 0.01);
 659                 }
 660         }
 661         /* else : must write more. */
 662 }
 663 static void
 664 set_start_callback(struct ev_loop *loop, AsyncIO *IO, int revents)
 665 {
 666         ev_timer_stop(event_base, &IO->conn_fail);
 667         ev_timer_start(event_base, &IO->rw_timeout);
 668
 669         switch(IO->NextState) {
 670         case eReadMore:
 671         case eReadMessage:
 672         case eReadFile:
 673                 StrBufAppendBufPlain(IO->ErrMsg, HKEY("[while waiting for greeting]"), 0);
 674                 ev_io_start(event_base, &IO->recv_event);
 675                 break;
 676         case eSendReply:
 677         case eSendMore:
 678         case eReadPayload:
 679         case eSendFile:
 680                 become_session(IO->CitContext);
 681                 IO_send_callback(loop, &IO->send_event, revents);
 682                 break;
 683         case eDBQuery:
 684         case eSendDNSQuery:
 685         case eReadDNSReply:
 686         case eConnect:
 687         case eTerminateConnection:
 688         case eAbort:
 689                 /// TODO: WHUT?
 690                 break;
 691         }
 692 }
 693
 694 static void
 695 IO_Timeout_callback(struct ev_loop *loop, ev_timer *watcher, int revents)
 696 {
 697         AsyncIO *IO = watcher->data;
 698
 699         SetEVState(IO, eIOTimeout);
 700         IO->Now = ev_now(event_base);
 701         ev_timer_stop (event_base, &IO->rw_timeout);
 702         become_session(IO->CitContext);
 703
 704         if (IO->SendBuf.fd != 0)
 705         {
 706                 ev_io_stop(event_base, &IO->send_event);
 707                 ev_io_stop(event_base, &IO->recv_event);
 708                 ev_timer_stop (event_base, &IO->rw_timeout);
 709                 close(IO->SendBuf.fd);
 710                 IO->SendBuf.fd = IO->RecvBuf.fd = 0;
 711         }
 712
 713         assert(IO->Timeout);
 714         switch (IO->Timeout(IO))
 715         {
 716         case eAbort:
 717                 ShutDownCLient(IO);
 718         default:
 719                 break;
 720         }
 721 }
 722
 723 static void
 724 IO_connfail_callback(struct ev_loop *loop, ev_timer *watcher, int revents)
 725 {
 726         AsyncIO *IO = watcher->data;
 727
 728         SetEVState(IO, eIOConnfail);
 729         IO->Now = ev_now(event_base);
 730         ev_timer_stop (event_base, &IO->conn_fail);
 731
 732         if (IO->SendBuf.fd != 0)
 733         {
 734                 ev_io_stop(loop, &IO->conn_event);
 735                 ev_io_stop(event_base, &IO->send_event);
 736                 ev_io_stop(event_base, &IO->recv_event);
 737                 ev_timer_stop (event_base, &IO->rw_timeout);
 738                 close(IO->SendBuf.fd);
 739                 IO->SendBuf.fd = IO->RecvBuf.fd = 0;
 740         }
 741         become_session(IO->CitContext);
 742
 743         assert(IO->ConnFail);
 744         switch (IO->ConnFail(IO))
 745         {
 746         case eAbort:
 747                 ShutDownCLient(IO);
 748         default:
 749                 break;
 750
 751         }
 752 }
 753
 754 static void
 755 IO_connfailimmediate_callback(struct ev_loop *loop,
 756                               ev_idle *watcher,
 757                               int revents)
 758 {
 759         AsyncIO *IO = watcher->data;
 760
 761         SetEVState(IO, eIOConnfailNow);
 762         IO->Now = ev_now(event_base);
 763         ev_idle_stop (event_base, &IO->conn_fail_immediate);
 764
 765         if (IO->SendBuf.fd != 0)
 766         {
 767                 close(IO->SendBuf.fd);
 768                 IO->SendBuf.fd = IO->RecvBuf.fd = 0;
 769         }
 770         become_session(IO->CitContext);
 771
 772         assert(IO->ConnFail);
 773         switch (IO->ConnFail(IO))
 774         {
 775         case eAbort:
 776                 ShutDownCLient(IO);
 777         default:
 778                 break;
 779
 780         }
 781 }
 782
 783 static void
 784 IO_connestd_callback(struct ev_loop *loop, ev_io *watcher, int revents)
 785 {
 786         AsyncIO *IO = watcher->data;
 787         int             so_err = 0;
 788         socklen_t       lon = sizeof(so_err);
 789         int             err;
 790
 791         SetEVState(IO, eIOConnNow);
 792         IO->Now = ev_now(event_base);
 793         EVM_syslog(LOG_DEBUG, "connect() succeeded.\n");
 794
 795         ev_io_stop(loop, &IO->conn_event);
 796         ev_timer_stop(event_base, &IO->conn_fail);
 797
 798         err = getsockopt(IO->SendBuf.fd,
 799                          SOL_SOCKET,
 800                          SO_ERROR,
 801                          (void*)&so_err,
 802                          &lon);
 803
 804         if ((err == 0) && (so_err != 0))
 805         {
 806                 EV_syslog(LOG_DEBUG, "connect() failed [%d][%s]\n",
 807                           so_err,
 808                           strerror(so_err));
 809                 IO_connfail_callback(loop, &IO->conn_fail, revents);
 810
 811         }
 812         else
 813         {
 814                 EVM_syslog(LOG_DEBUG, "connect() succeeded\n");
 815                 set_start_callback(loop, IO, revents);
 816         }
 817 }
 818
 819 static void
 820 IO_recv_callback(struct ev_loop *loop, ev_io *watcher, int revents)
 821 {
 822         const char *errmsg;
 823         ssize_t nbytes;
 824         AsyncIO *IO = watcher->data;
 825
 826         IO->Now = ev_now(event_base);
 827         switch (IO->NextState) {
 828         case eReadFile:
 829                 nbytes = FileRecvChunked(&IO->IOB, &errmsg);
 830                 if (nbytes < 0)
 831                         StrBufPlain(IO->ErrMsg, errmsg, -1);
 832                 else
 833                 {
 834                         if (IO->IOB.ChunkSendRemain == 0)
 835                         {
 836                                 IO->NextState = eSendReply;
 837                                 assert(IO->ReadDone);
 838                                 ev_io_stop(event_base, &IO->recv_event);
 839                                 PostInbound(IO);
 840                                 return;
 841                         }
 842                         else
 843                                 return;
 844                 }
 845                 break;
 846         default:
 847                 nbytes = StrBuf_read_one_chunk_callback(IO->RecvBuf.fd,
 848                                                         0,
 849                                                         &IO->RecvBuf);
 850                 break;
 851         }
 852
 853 #ifdef BIGBAD_IODBG
 854         {
 855                 long nbytes;
 856                 int rv = 0;
 857                 char fn [SIZ];
 858                 FILE *fd;
 859                 const char *pch = ChrPtr(IO->RecvBuf.Buf);
 860                 const char *pchh = IO->RecvBuf.ReadWritePointer;
 861
 862                 if (pchh == NULL)
 863                         pchh = pch;
 864
 865                 nbytes = StrLength(IO->RecvBuf.Buf) - (pchh - pch);
 866                 snprintf(fn, SIZ, "/tmp/foolog_ev_%s.%d",
 867                          ((CitContext*)(IO->CitContext))->ServiceName,
 868                          IO->SendBuf.fd);
 869
 870                 fd = fopen(fn, "a+");
 871                 fprintf(fd, "Read: BufSize: %ld BufContent: [",
 872                         nbytes);
 873                 rv = fwrite(pchh, nbytes, 1, fd);
 874                 if (!rv) printf("failed to write debug to %s!\n", fn);
 875                 fprintf(fd, "]\n");
 876                 fclose(fd);
 877         }
 878 #endif
 879         if (nbytes > 0) {
 880                 HandleInbound(IO);
 881         } else if (nbytes == 0) {
 882                 StopClientWatchers(IO, 1);
 883                 SetNextTimeout(IO, 0.01);
 884                 return;
 885         } else if (nbytes == -1) {
 886                 if (errno != EAGAIN) {
 887                         // FD is gone. kick it.
 888                         StopClientWatchers(IO, 1);
 889                         EV_syslog(LOG_DEBUG,
 890                                   "IO_recv_callback(): Socket Invalid! [%d] [%s] [%d]\n",
 891                                   errno, strerror(errno), IO->SendBuf.fd);
 892                         StrBufPrintf(IO->ErrMsg,
 893                                      "Socket Invalid! [%s]",
 894                                      strerror(errno));
 895                         SetNextTimeout(IO, 0.01);
 896                 }
 897                 return;
 898         }
 899 }
 900
 901 void
 902 IO_postdns_callback(struct ev_loop *loop, ev_idle *watcher, int revents)
 903 {
 904         AsyncIO *IO = watcher->data;
 905
 906         SetEVState(IO, eCaresFinished);
 907         IO->Now = ev_now(event_base);
 908         EV_syslog(LOG_DEBUG, "event: %s\n", __FUNCTION__);
 909         become_session(IO->CitContext);
 910         assert(IO->DNS.Query->PostDNS);
 911         switch (IO->DNS.Query->PostDNS(IO))
 912         {
 913         case eAbort:
 914                 assert(IO->DNS.Fail);
 915                 switch (IO->DNS.Fail(IO)) {
 916                 case eAbort:
 917 ////                    StopClientWatchers(IO);
 918                         ShutDownCLient(IO);
 919                 default:
 920                         break;
 921                 }
 922         default:
 923                 break;
 924         }
 925 }
 926
 927
 928 eNextState EvConnectSock(AsyncIO *IO,
 929                          double conn_timeout,
 930                          double first_rw_timeout,
 931                          int ReadFirst)
 932 {
 933         struct sockaddr_in egress_sin;
 934         int fdflags;
 935         int rc = -1;
 936
 937         SetEVState(IO, eIOConnectSock);
 938         become_session(IO->CitContext);
 939
 940         if (ReadFirst) {
 941                 IO->NextState = eReadMessage;
 942         }
 943         else {
 944                 IO->NextState = eSendReply;
 945         }
 946
 947         IO->SendBuf.fd = IO->RecvBuf.fd =
 948                 socket(
 949                         (IO->ConnectMe->IPv6)?PF_INET6:PF_INET,
 950                         SOCK_STREAM,
 951                         IPPROTO_TCP);
 952
 953         if (IO->SendBuf.fd < 0) {
 954                 EV_syslog(LOG_ERR,
 955                           "EVENT: socket() failed: %s\n",
 956                           strerror(errno));
 957
 958                 StrBufPrintf(IO->ErrMsg,
 959                              "Failed to create socket: %s",
 960                              strerror(errno));
 961                 IO->SendBuf.fd = IO->RecvBuf.fd = 0;
 962                 return eAbort;
 963         }
 964         fdflags = fcntl(IO->SendBuf.fd, F_GETFL);
 965         if (fdflags < 0) {
 966                 EV_syslog(LOG_ERR,
 967                           "EVENT: unable to get socket %d flags! %s \n",
 968                           IO->SendBuf.fd,
 969                           strerror(errno));
 970                 StrBufPrintf(IO->ErrMsg,
 971                              "Failed to get socket %d flags: %s",
 972                              IO->SendBuf.fd,
 973                              strerror(errno));
 974                 close(IO->SendBuf.fd);
 975                 IO->SendBuf.fd = IO->RecvBuf.fd = 0;
 976                 return eAbort;
 977         }
 978         fdflags = fdflags | O_NONBLOCK;
 979         if (fcntl(IO->SendBuf.fd, F_SETFL, fdflags) < 0) {
 980                 EV_syslog(
 981                         LOG_ERR,
 982                         "EVENT: unable to set socket %d nonblocking flags! %s \n",
 983                         IO->SendBuf.fd,
 984                         strerror(errno));
 985                 StrBufPrintf(IO->ErrMsg,
 986                              "Failed to set socket flags: %s",
 987                              strerror(errno));
 988                 close(IO->SendBuf.fd);
 989                 IO->SendBuf.fd = IO->RecvBuf.fd = 0;
 990                 return eAbort;
 991         }
 992 /* TODO: maye we could use offsetof() to calc the position of data...
 993  * http://doc.dvgu.ru/devel/ev.html#associating_custom_data_with_a_watcher
 994  */
 995         ev_io_init(&IO->recv_event, IO_recv_callback, IO->RecvBuf.fd, EV_READ);
 996         IO->recv_event.data = IO;
 997         ev_io_init(&IO->send_event, IO_send_callback, IO->SendBuf.fd, EV_WRITE);
 998         IO->send_event.data = IO;
 999
1000         ev_timer_init(&IO->conn_fail, IO_connfail_callback, conn_timeout, 0);
1001         IO->conn_fail.data = IO;
1002         ev_timer_init(&IO->rw_timeout, IO_Timeout_callback, first_rw_timeout,0);
1003         IO->rw_timeout.data = IO;
1004
1005
1006
1007
1008         /* for debugging you may bypass it like this:
1009          * IO->Addr.sin_addr.s_addr = inet_addr("127.0.0.1");
1010          * ((struct sockaddr_in)IO->ConnectMe->Addr).sin_addr.s_addr =
1011          *   inet_addr("127.0.0.1");
1012          */
1013         if (IO->ConnectMe->IPv6) {
1014                 rc = connect(IO->SendBuf.fd,
1015                              &IO->ConnectMe->Addr,
1016                              sizeof(struct sockaddr_in6));
1017         }
1018         else {
1019                 /* If citserver is bound to a specific IP address on the host, make
1020                  * sure we use that address for outbound connections.
1021                  */
1022
1023                 memset(&egress_sin, 0, sizeof(egress_sin));
1024                 egress_sin.sin_family = AF_INET;
1025                 if (!IsEmptyStr(config.c_ip_addr)) {
1026                         egress_sin.sin_addr.s_addr = inet_addr(config.c_ip_addr);
1027                         if (egress_sin.sin_addr.s_addr == !INADDR_ANY) {
1028                                 egress_sin.sin_addr.s_addr = INADDR_ANY;
1029                         }
1030
1031                         /* If this bind fails, no problem; we can still use INADDR_ANY */
1032                         bind(IO->SendBuf.fd, (struct sockaddr *)&egress_sin, sizeof(egress_sin));
1033                 }
1034                 rc = connect(IO->SendBuf.fd,
1035                              (struct sockaddr_in *)&IO->ConnectMe->Addr,
1036                              sizeof(struct sockaddr_in));
1037         }
1038
1039         if (rc >= 0){
1040                 SetEVState(IO, eIOConnNow);
1041                 EV_syslog(LOG_DEBUG, "connect() = %d immediate success.\n", IO->SendBuf.fd);
1042                 set_start_callback(event_base, IO, 0);
1043                 return IO->NextState;
1044         }
1045         else if (errno == EINPROGRESS) {
1046                 SetEVState(IO, eIOConnWait);
1047                 EV_syslog(LOG_DEBUG, "connect() = %d have to wait now.\n", IO->SendBuf.fd);
1048
1049                 ev_io_init(&IO->conn_event,
1050                            IO_connestd_callback,
1051                            IO->SendBuf.fd,
1052                            EV_READ|EV_WRITE);
1053
1054                 IO->conn_event.data = IO;
1055
1056                 ev_io_start(event_base, &IO->conn_event);
1057                 ev_timer_start(event_base, &IO->conn_fail);
1058                 return IO->NextState;
1059         }
1060         else {
1061                 SetEVState(IO, eIOConnfail);
1062                 ev_idle_init(&IO->conn_fail_immediate,
1063                              IO_connfailimmediate_callback);
1064                 IO->conn_fail_immediate.data = IO;
1065                 ev_idle_start(event_base, &IO->conn_fail_immediate);
1066
1067                 EV_syslog(LOG_ERR,
1068                           "connect() = %d failed: %s\n",
1069                           IO->SendBuf.fd,
1070                           strerror(errno));
1071
1072                 StrBufPrintf(IO->ErrMsg,
1073                              "Failed to connect: %s",
1074                              strerror(errno));
1075                 return IO->NextState;
1076         }
1077         return IO->NextState;
1078 }
1079
1080 void SetNextTimeout(AsyncIO *IO, double timeout)
1081 {
1082         IO->rw_timeout.repeat = timeout;
1083         ev_timer_again (event_base,  &IO->rw_timeout);
1084 }
1085
1086
1087 eNextState ReAttachIO(AsyncIO *IO,
1088                       void *pData,
1089                       int ReadFirst)
1090 {
1091         SetEVState(IO, eIOAttach);
1092         IO->Data = pData;
1093         become_session(IO->CitContext);
1094         ev_cleanup_start(event_base, &IO->abort_by_shutdown);
1095         if (ReadFirst) {
1096                 IO->NextState = eReadMessage;
1097         }
1098         else {
1099                 IO->NextState = eSendReply;
1100         }
1101         set_start_callback(event_base, IO, 0);
1102
1103         return IO->NextState;
1104 }
1105
1106 void InitIOStruct(AsyncIO *IO,
1107                   void *Data,
1108                   eNextState NextState,
1109                   IO_LineReaderCallback LineReader,
1110                   IO_CallBack DNS_Fail,
1111                   IO_CallBack SendDone,
1112                   IO_CallBack ReadDone,
1113                   IO_CallBack Terminate,
1114                   IO_CallBack DBTerminate,
1115                   IO_CallBack ConnFail,
1116                   IO_CallBack Timeout,
1117                   IO_CallBack ShutdownAbort)
1118 {
1119         IO->Data          = Data;
1120
1121         IO->CitContext    = CloneContext(CC);
1122         IO->CitContext->session_specific_data = Data;
1123         IO->CitContext->IO = IO;
1124
1125         IO->NextState     = NextState;
1126
1127         IO->SendDone      = SendDone;
1128         IO->ReadDone      = ReadDone;
1129         IO->Terminate     = Terminate;
1130         IO->DBTerminate   = DBTerminate;
1131         IO->LineReader    = LineReader;
1132         IO->ConnFail      = ConnFail;
1133         IO->Timeout       = Timeout;
1134         IO->ShutdownAbort = ShutdownAbort;
1135
1136         IO->DNS.Fail      = DNS_Fail;
1137
1138         IO->SendBuf.Buf   = NewStrBufPlain(NULL, 1024);
1139         IO->RecvBuf.Buf   = NewStrBufPlain(NULL, 1024);
1140         IO->IOBuf         = NewStrBuf();
1141         EV_syslog(LOG_DEBUG,
1142                   "EVENT: Session lives at %p IO at %p \n",
1143                   Data, IO);
1144
1145 }
1146
1147 extern int evcurl_init(AsyncIO *IO);
1148
1149 int InitcURLIOStruct(AsyncIO *IO,
1150                      void *Data,
1151                      const char* Desc,
1152                      IO_CallBack SendDone,
1153                      IO_CallBack Terminate,
1154                      IO_CallBack DBTerminate,
1155                      IO_CallBack ShutdownAbort)
1156 {
1157         IO->Data          = Data;
1158
1159         IO->CitContext    = CloneContext(CC);
1160         IO->CitContext->session_specific_data = Data;
1161         IO->CitContext->IO = IO;
1162
1163         IO->SendDone      = SendDone;
1164         IO->Terminate     = Terminate;
1165         IO->DBTerminate   = DBTerminate;
1166         IO->ShutdownAbort = ShutdownAbort;
1167
1168         strcpy(IO->HttpReq.errdesc, Desc);
1169
1170
1171         return  evcurl_init(IO);
1172
1173 }
1174
1175
1176 typedef struct KillOtherSessionContext {
1177         AsyncIO IO;
1178         AsyncIO *OtherOne;
1179 }KillOtherSessionContext;
1180
1181 eNextState KillTerminate(AsyncIO *IO)
1182 {
1183         long id;
1184         KillOtherSessionContext *Ctx = (KillOtherSessionContext*)IO->Data;
1185         EV_syslog(LOG_DEBUG, "%s Exit\n", __FUNCTION__);
1186         id = IO->ID;
1187         FreeAsyncIOContents(IO);
1188         memset(Ctx, 0, sizeof(KillOtherSessionContext));
1189         IO->ID = id; /* just for the case we want to analyze it in a coredump */
1190         free(Ctx);
1191         return eAbort;
1192
1193 }
1194
1195 eNextState KillShutdown(AsyncIO *IO)
1196 {
1197         return eTerminateConnection;
1198 }
1199
1200 eNextState KillOtherContextNow(AsyncIO *IO)
1201 {
1202         KillOtherSessionContext *Ctx = IO->Data;
1203
1204         SetEVState(IO, eKill);
1205
1206         if (Ctx->OtherOne->ShutdownAbort != NULL)
1207                 Ctx->OtherOne->ShutdownAbort(Ctx->OtherOne);
1208         return eTerminateConnection;
1209 }
1210
1211 void KillAsyncIOContext(AsyncIO *IO)
1212 {
1213         KillOtherSessionContext *Ctx;
1214
1215         Ctx = (KillOtherSessionContext*) malloc(sizeof(KillOtherSessionContext));
1216         memset(Ctx, 0, sizeof(KillOtherSessionContext));
1217
1218         InitIOStruct(&Ctx->IO,
1219                      Ctx,
1220                      eReadMessage,
1221                      NULL,
1222                      NULL,
1223                      NULL,
1224                      NULL,
1225                      KillTerminate,
1226                      NULL,
1227                      NULL,
1228                      NULL,
1229                      KillShutdown);
1230
1231         Ctx->OtherOne = IO;
1232
1233         switch(IO->NextState) {
1234         case eSendDNSQuery:
1235         case eReadDNSReply:
1236
1237         case eConnect:
1238         case eSendReply:
1239         case eSendMore:
1240         case eSendFile:
1241
1242         case eReadMessage:
1243         case eReadMore:
1244         case eReadPayload:
1245         case eReadFile:
1246                 QueueEventContext(&Ctx->IO, KillOtherContextNow);
1247                 break;
1248         case eDBQuery:
1249                 QueueDBOperation(&Ctx->IO, KillOtherContextNow);
1250                 break;
1251         case eTerminateConnection:
1252         case eAbort:
1253                 /*hm, its already dying, dunno which Queue its in... */
1254                 free(Ctx);
1255         }
1256
1257 }
1258
1259 extern int DebugEventLoopBacktrace;
1260 void EV_backtrace(AsyncIO *IO)
1261 {
1262 #ifdef HAVE_BACKTRACE
1263         void *stack_frames[50];
1264         size_t size, i;
1265         char **strings;
1266
1267         if ((IO == NULL) || (DebugEventLoopBacktrace == 0))
1268                 return;
1269         size = backtrace(stack_frames, sizeof(stack_frames) / sizeof(void*));
1270         strings = backtrace_symbols(stack_frames, size);
1271         for (i = 0; i < size; i++) {
1272                 if (strings != NULL) {
1273                         EV_syslog(LOG_ALERT, " BT %s\n", strings[i]);
1274                 }
1275                 else {
1276                         EV_syslog(LOG_ALERT, " BT %p\n", stack_frames[i]);
1277                 }
1278         }
1279         free(strings);
1280 #endif
1281 }
1282
1283
1284 ev_tstamp ctdl_ev_now (void)
1285 {
1286         return ev_now(event_base);
1287 }