90c5e5dd986619f4f9f5426bea3393133a9cb7c1
[citadel] / citadel / event_client.c
1 /*
2  * Copyright (c) 1998-2012 by the citadel.org team
3  *
4  * This program is open source software; you can redistribute it and/or modify
5  * it under the terms of the GNU General Public License, version 3.
6  *
7  * This program is distributed in the hope that it will be useful,
8  * but WITHOUT ANY WARRANTY; without even the implied warranty of
9  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
10  * GNU General Public License for more details.
11  */
12
13 #include "sysdep.h"
14 #include <stdlib.h>
15 #include <unistd.h>
16 #include <stdio.h>
17 #include <termios.h>
18 #include <fcntl.h>
19 #include <signal.h>
20 #include <pwd.h>
21 #include <errno.h>
22 #include <sys/types.h>
23 #include <syslog.h>
24
25 #if TIME_WITH_SYS_TIME
26 # include <sys/time.h>
27 # include <time.h>
28 #else
29 # if HAVE_SYS_TIME_H
30 #  include <sys/time.h>
31 # else
32 #  include <time.h>
33 # endif
34 #endif
35 #include <sys/wait.h>
36 #include <ctype.h>
37 #include <string.h>
38 #include <limits.h>
39 #include <sys/socket.h>
40 #include <netinet/in.h>
41 #include <arpa/inet.h>
42 #include <assert.h>
43 #if HAVE_BACKTRACE
44 #include <execinfo.h>
45 #endif
46
47 #include <libcitadel.h>
48 #include "citadel.h"
49 #include "server.h"
50 #include "citserver.h"
51 #include "support.h"
52 #include "config.h"
53 #include "control.h"
54 #include "user_ops.h"
55 #include "database.h"
56 #include "msgbase.h"
57 #include "internet_addressing.h"
58 #include "genstamp.h"
59 #include "domain.h"
60 #include "clientsocket.h"
61 #include "locate_host.h"
62 #include "citadel_dirs.h"
63
64 #include "event_client.h"
65 #include "ctdl_module.h"
66
67 static void IO_Timeout_callback(struct ev_loop *loop, ev_timer *watcher, int revents);
68 static void IO_abort_shutdown_callback(struct ev_loop *loop,
69                                        ev_cleanup *watcher,
70                                        int revents);
71
72
73 /*------------------------------------------------------------------------------
74  *                              Server DB IO
75  *----------------------------------------------------------------------------*/
76 extern int evdb_count;
77 extern pthread_mutex_t DBEventQueueMutex;
78 extern pthread_mutex_t DBEventExitQueueMutex;
79 extern HashList *DBInboundEventQueue;
80 extern struct ev_loop *event_db;
81 extern ev_async DBAddJob;
82 extern ev_async DBExitEventLoop;
83
84 eNextState QueueDBOperation(AsyncIO *IO, IO_CallBack CB)
85 {
86         IOAddHandler *h;
87         int i;
88
89         h = (IOAddHandler*)malloc(sizeof(IOAddHandler));
90         h->IO = IO;
91         h->EvAttch = CB;
92         ev_cleanup_init(&IO->db_abort_by_shutdown,
93                         IO_abort_shutdown_callback);
94         IO->db_abort_by_shutdown.data = IO;
95
96         pthread_mutex_lock(&DBEventQueueMutex);
97         if (DBInboundEventQueue == NULL)
98         {
99                 /* shutting down... */
100                 free(h);
101                 EVM_syslog(LOG_DEBUG, "DBEVENT Q exiting.\n");
102                 pthread_mutex_unlock(&DBEventQueueMutex);
103                 return eAbort;
104         }
105         EVM_syslog(LOG_DEBUG, "DBEVENT Q\n");
106         i = ++evdb_count ;
107         Put(DBInboundEventQueue, IKEY(i), h, NULL);
108         pthread_mutex_unlock(&DBEventQueueMutex);
109
110         pthread_mutex_lock(&DBEventExitQueueMutex);
111         if (event_db == NULL)
112         {
113                 pthread_mutex_unlock(&DBEventExitQueueMutex);
114                 return eAbort;
115         }
116         ev_async_send (event_db, &DBAddJob);
117         pthread_mutex_unlock(&DBEventExitQueueMutex);
118
119         EVM_syslog(LOG_DEBUG, "DBEVENT Q Done.\n");
120         return eDBQuery;
121 }
122
123 void StopDBWatchers(AsyncIO *IO)
124 {
125         ev_cleanup_stop(event_db, &IO->db_abort_by_shutdown);
126         ev_idle_stop(event_db, &IO->db_unwind_stack);
127 }
128
129 void ShutDownDBCLient(AsyncIO *IO)
130 {
131         CitContext *Ctx =IO->CitContext;
132         become_session(Ctx);
133
134         EVM_syslog(LOG_DEBUG, "DBEVENT Terminating.\n");
135         StopDBWatchers(IO);
136
137         assert(IO->DBTerminate);
138         IO->DBTerminate(IO);
139 }
140
141 void
142 DB_PerformNext(struct ev_loop *loop, ev_idle *watcher, int revents)
143 {
144         AsyncIO *IO = watcher->data;
145         IO->Now = ev_now(event_db);
146         EV_syslog(LOG_DEBUG, "%s()", __FUNCTION__);
147         become_session(IO->CitContext);
148
149         ev_idle_stop(event_db, &IO->db_unwind_stack);
150
151         assert(IO->NextDBOperation);
152         switch (IO->NextDBOperation(IO))
153         {
154         case eDBQuery:
155                 break;
156         case eSendDNSQuery:
157         case eReadDNSReply:
158         case eConnect:
159         case eSendReply:
160         case eSendMore:
161         case eSendFile:
162         case eReadMessage:
163         case eReadMore:
164         case eReadPayload:
165         case eReadFile:
166                 ev_cleanup_stop(loop, &IO->db_abort_by_shutdown);
167                 break;
168         case eTerminateConnection:
169         case eAbort:
170                 ev_idle_stop(event_db, &IO->db_unwind_stack);
171                 ev_cleanup_stop(loop, &IO->db_abort_by_shutdown);
172                 ShutDownDBCLient(IO);
173         }
174 }
175
176 eNextState NextDBOperation(AsyncIO *IO, IO_CallBack CB)
177 {
178         IO->NextDBOperation = CB;
179         ev_idle_init(&IO->db_unwind_stack,
180                      DB_PerformNext);
181         IO->db_unwind_stack.data = IO;
182         ev_idle_start(event_db, &IO->db_unwind_stack);
183         return eDBQuery;
184 }
185
186 /*------------------------------------------------------------------------------
187  *                      Client IO
188  *----------------------------------------------------------------------------*/
189 extern int evbase_count;
190 extern pthread_mutex_t EventQueueMutex;
191 extern pthread_mutex_t EventExitQueueMutex; 
192 extern HashList *InboundEventQueue;
193 extern struct ev_loop *event_base;
194 extern ev_async AddJob;
195 extern ev_async ExitEventLoop;
196
197 static void IO_abort_shutdown_callback(struct ev_loop *loop,
198                                        ev_cleanup *watcher,
199                                        int revents)
200 {
201         AsyncIO *IO = watcher->data;
202         EV_syslog(LOG_DEBUG, "EVENT Q: %s\n", __FUNCTION__);
203         IO->Now = ev_now(event_base);
204         assert(IO->ShutdownAbort);
205         IO->ShutdownAbort(IO);
206 }
207
208
209 eNextState QueueEventContext(AsyncIO *IO, IO_CallBack CB)
210 {
211         IOAddHandler *h;
212         int i;
213
214         h = (IOAddHandler*)malloc(sizeof(IOAddHandler));
215         h->IO = IO;
216         h->EvAttch = CB;
217         ev_cleanup_init(&IO->abort_by_shutdown,
218                         IO_abort_shutdown_callback);
219         IO->abort_by_shutdown.data = IO;
220
221         pthread_mutex_lock(&EventQueueMutex);
222         if (InboundEventQueue == NULL)
223         {
224                 free(h);
225                 /* shutting down... */
226                 EVM_syslog(LOG_DEBUG, "EVENT Q exiting.\n");
227                 pthread_mutex_unlock(&EventQueueMutex);
228                 return eAbort;
229         }
230         EVM_syslog(LOG_DEBUG, "EVENT Q\n");
231         i = ++evbase_count;
232         Put(InboundEventQueue, IKEY(i), h, NULL);
233         pthread_mutex_unlock(&EventQueueMutex);
234
235         pthread_mutex_lock(&EventExitQueueMutex);
236         if (event_base == NULL) {
237                 pthread_mutex_unlock(&EventExitQueueMutex);
238                 return eAbort;
239         }
240         ev_async_send (event_base, &AddJob);
241         pthread_mutex_unlock(&EventExitQueueMutex);
242         EVM_syslog(LOG_DEBUG, "EVENT Q Done.\n");
243         return eSendReply;
244 }
245
246 extern eNextState evcurl_handle_start(AsyncIO *IO);
247
248 eNextState QueueCurlContext(AsyncIO *IO)
249 {
250         IOAddHandler *h;
251         int i;
252
253         h = (IOAddHandler*)malloc(sizeof(IOAddHandler));
254         h->IO = IO;
255         h->EvAttch = evcurl_handle_start;
256
257         pthread_mutex_lock(&EventQueueMutex);
258         if (InboundEventQueue == NULL)
259         {
260                 /* shutting down... */
261                 free(h);
262                 EVM_syslog(LOG_DEBUG, "EVENT Q exiting.\n");
263                 pthread_mutex_unlock(&EventQueueMutex);
264                 return eAbort;
265         }
266
267         EVM_syslog(LOG_DEBUG, "EVENT Q\n");
268         i = ++evbase_count;
269         Put(InboundEventQueue, IKEY(i), h, NULL);
270         pthread_mutex_unlock(&EventQueueMutex);
271
272         pthread_mutex_lock(&EventExitQueueMutex);
273         if (event_base == NULL) {
274                 pthread_mutex_unlock(&EventExitQueueMutex);
275                 return eAbort;
276         }
277         ev_async_send (event_base, &AddJob);
278         pthread_mutex_unlock(&EventExitQueueMutex);
279
280         EVM_syslog(LOG_DEBUG, "EVENT Q Done.\n");
281         return eSendReply;
282 }
283
284 void DestructCAres(AsyncIO *IO);
285 void FreeAsyncIOContents(AsyncIO *IO)
286 {
287         CitContext *Ctx = IO->CitContext;
288
289         FreeStrBuf(&IO->IOBuf);
290         FreeStrBuf(&IO->SendBuf.Buf);
291         FreeStrBuf(&IO->RecvBuf.Buf);
292
293         DestructCAres(IO);
294
295         FreeURL(&IO->ConnectMe);
296         FreeStrBuf(&IO->HttpReq.ReplyData);
297
298         if (Ctx) {
299                 Ctx->state = CON_IDLE;
300                 Ctx->kill_me = 1;
301                 IO->CitContext = NULL;
302         }
303 }
304
305
306 void StopClientWatchers(AsyncIO *IO, int CloseFD)
307 {
308         ev_timer_stop (event_base, &IO->rw_timeout);
309         ev_timer_stop(event_base, &IO->conn_fail);
310         ev_idle_stop(event_base, &IO->unwind_stack);
311         ev_cleanup_stop(event_base, &IO->abort_by_shutdown);
312
313         ev_io_stop(event_base, &IO->conn_event);
314         ev_io_stop(event_base, &IO->send_event);
315         ev_io_stop(event_base, &IO->recv_event);
316
317         if (CloseFD && (IO->SendBuf.fd > 0)) {
318                 close(IO->SendBuf.fd);
319                 IO->SendBuf.fd = 0;
320                 IO->RecvBuf.fd = 0;
321         }
322 }
323
324 void StopCurlWatchers(AsyncIO *IO)
325 {
326         ev_timer_stop (event_base, &IO->rw_timeout);
327         ev_timer_stop(event_base, &IO->conn_fail);
328         ev_idle_stop(event_base, &IO->unwind_stack);
329         ev_cleanup_stop(event_base, &IO->abort_by_shutdown);
330
331         ev_io_stop(event_base, &IO->conn_event);
332         ev_io_stop(event_base, &IO->send_event);
333         ev_io_stop(event_base, &IO->recv_event);
334
335         if (IO->SendBuf.fd != 0) {
336                 close(IO->SendBuf.fd);
337         }
338         IO->SendBuf.fd = 0;
339         IO->RecvBuf.fd = 0;
340 }
341
342 void ShutDownCLient(AsyncIO *IO)
343 {
344         CitContext *Ctx =IO->CitContext;
345         become_session(Ctx);
346
347         EVM_syslog(LOG_DEBUG, "EVENT Terminating \n");
348
349         StopClientWatchers(IO, 1);
350
351         if (IO->DNS.Channel != NULL) {
352                 ares_destroy(IO->DNS.Channel);
353                 EV_DNS_LOG_STOP(DNS.recv_event);
354                 EV_DNS_LOG_STOP(DNS.send_event);
355                 ev_io_stop(event_base, &IO->DNS.recv_event);
356                 ev_io_stop(event_base, &IO->DNS.send_event);
357                 IO->DNS.Channel = NULL;
358         }
359         assert(IO->Terminate);
360         IO->Terminate(IO);
361 }
362
363 void PostInbound(AsyncIO *IO)
364 {
365         switch (IO->NextState) {
366         case eSendFile:
367                 ev_io_start(event_base, &IO->send_event);
368                 break;
369         case eSendReply:
370         case eSendMore:
371                 assert(IO->SendDone);
372                 IO->NextState = IO->SendDone(IO);
373                 switch (IO->NextState)
374                 {
375                 case eSendFile:
376                 case eSendReply:
377                 case eSendMore:
378                 case eReadMessage:
379                 case eReadPayload:
380                 case eReadMore:
381                 case eReadFile:
382                         ev_io_start(event_base, &IO->send_event);
383                         break;
384                 case eDBQuery:
385                         StopClientWatchers(IO, 0);
386                 default:
387                         break;
388                 }
389                 break;
390         case eReadPayload:
391         case eReadMore:
392         case eReadFile:
393                 ev_io_start(event_base, &IO->recv_event);
394                 break;
395         case eTerminateConnection:
396                 ShutDownCLient(IO);
397                 break;
398         case eAbort:
399                 ShutDownCLient(IO);
400                 break;
401         case eSendDNSQuery:
402         case eReadDNSReply:
403         case eDBQuery:
404         case eConnect:
405         case eReadMessage:
406                 break;
407         }
408 }
409 eReadState HandleInbound(AsyncIO *IO)
410 {
411         const char *Err = NULL;
412         eReadState Finished = eBufferNotEmpty;
413
414         become_session(IO->CitContext);
415
416         while ((Finished == eBufferNotEmpty) &&
417                ((IO->NextState == eReadMessage)||
418                 (IO->NextState == eReadMore)||
419                 (IO->NextState == eReadFile)||
420                 (IO->NextState == eReadPayload)))
421         {
422                 /* Reading lines...
423                  * lex line reply in callback,
424                  * or do it ourselves.
425                  * i.e. as nnn-blabla means continue reading in SMTP
426                  */
427                 if ((IO->NextState == eReadFile) &&
428                     (Finished == eBufferNotEmpty))
429                 {
430                         Finished = WriteIOBAlreadyRead(&IO->IOB, &Err);
431                         if (Finished == eReadSuccess)
432                         {
433                                 IO->NextState = eSendReply;
434                         }
435                 }
436                 else if (IO->LineReader)
437                         Finished = IO->LineReader(IO);
438                 else
439                         Finished = StrBufChunkSipLine(IO->IOBuf,
440                                                       &IO->RecvBuf);
441
442                 switch (Finished) {
443                 case eMustReadMore: /// read new from socket...
444                         break;
445                 case eBufferNotEmpty: /* shouldn't happen... */
446                 case eReadSuccess: /// done for now...
447                         break;
448                 case eReadFail: /// WHUT?
449                                 ///todo: shut down!
450                         break;
451                 }
452
453                 if (Finished != eMustReadMore) {
454                         assert(IO->ReadDone);
455                         ev_io_stop(event_base, &IO->recv_event);
456                         IO->NextState = IO->ReadDone(IO);
457                         Finished = StrBufCheckBuffer(&IO->RecvBuf);
458                 }
459         }
460
461         PostInbound(IO);
462
463         return Finished;
464 }
465
466
467 static void
468 IO_send_callback(struct ev_loop *loop, ev_io *watcher, int revents)
469 {
470         int rc;
471         AsyncIO *IO = watcher->data;
472         const char *errmsg = NULL;
473
474         IO->Now = ev_now(event_base);
475         become_session(IO->CitContext);
476 #ifdef BIGBAD_IODBG
477         {
478                 int rv = 0;
479                 char fn [SIZ];
480                 FILE *fd;
481                 const char *pch = ChrPtr(IO->SendBuf.Buf);
482                 const char *pchh = IO->SendBuf.ReadWritePointer;
483                 long nbytes;
484
485                 if (pchh == NULL)
486                         pchh = pch;
487
488                 nbytes = StrLength(IO->SendBuf.Buf) - (pchh - pch);
489                 snprintf(fn, SIZ, "/tmp/foolog_ev_%s.%d",
490                          ((CitContext*)(IO->CitContext))->ServiceName,
491                          IO->SendBuf.fd);
492
493                 fd = fopen(fn, "a+");
494                 fprintf(fd, "Send: BufSize: %ld BufContent: [",
495                         nbytes);
496                 rv = fwrite(pchh, nbytes, 1, fd);
497                 if (!rv) printf("failed to write debug to %s!\n", fn);
498                 fprintf(fd, "]\n");
499 #endif
500                 switch (IO->NextState) {
501                 case eSendFile:
502                         rc = FileSendChunked(&IO->IOB, &errmsg);
503                         if (rc < 0)
504                                 StrBufPlain(IO->ErrMsg, errmsg, -1);
505                         break;
506                 default:
507                         rc = StrBuf_write_one_chunk_callback(IO->SendBuf.fd,
508                                                              0,
509                                                              &IO->SendBuf);
510                 }
511
512 #ifdef BIGBAD_IODBG
513                 fprintf(fd, "Sent: BufSize: %d bytes.\n", rc);
514                 fclose(fd);
515         }
516 #endif
517         if (rc == 0)
518         {
519                 ev_io_stop(event_base, &IO->send_event);
520                 switch (IO->NextState) {
521                 case eSendMore:
522                         assert(IO->SendDone);
523                         IO->NextState = IO->SendDone(IO);
524
525                         if ((IO->NextState == eTerminateConnection) ||
526                             (IO->NextState == eAbort) )
527                                 ShutDownCLient(IO);
528                         else {
529                                 ev_io_start(event_base, &IO->send_event);
530                         }
531                         break;
532                 case eSendFile:
533                         if (IO->IOB.ChunkSendRemain > 0) {
534                                 ev_io_start(event_base, &IO->recv_event);
535                                 SetNextTimeout(IO, 100.0);
536
537                         } else {
538                                 assert(IO->ReadDone);
539                                 IO->NextState = IO->ReadDone(IO);
540                                 switch(IO->NextState) {
541                                 case eSendDNSQuery:
542                                 case eReadDNSReply:
543                                 case eDBQuery:
544                                 case eConnect:
545                                         break;
546                                 case eSendReply:
547                                 case eSendMore:
548                                 case eSendFile:
549                                         ev_io_start(event_base,
550                                                     &IO->send_event);
551                                         break;
552                                 case eReadMessage:
553                                 case eReadMore:
554                                 case eReadPayload:
555                                 case eReadFile:
556                                         break;
557                                 case eTerminateConnection:
558                                 case eAbort:
559                                         break;
560                                 }
561                         }
562                         break;
563                 case eSendReply:
564                     if (StrBufCheckBuffer(&IO->SendBuf) != eReadSuccess)
565                         break;
566                     IO->NextState = eReadMore;
567                 case eReadMore:
568                 case eReadMessage:
569                 case eReadPayload:
570                 case eReadFile:
571                         if (StrBufCheckBuffer(&IO->RecvBuf) == eBufferNotEmpty)
572                         {
573                                 HandleInbound(IO);
574                         }
575                         else {
576                                 ev_io_start(event_base, &IO->recv_event);
577                         }
578
579                         break;
580                 case eDBQuery:
581                         /*
582                          * we now live in another queue,
583                          * so we have to unregister.
584                          */
585                         ev_cleanup_stop(loop, &IO->abort_by_shutdown);
586                         break;
587                 case eSendDNSQuery:
588                 case eReadDNSReply:
589                 case eConnect:
590                 case eTerminateConnection:
591                 case eAbort:
592                         break;
593                 }
594         }
595         else if (rc < 0) {
596                 if (errno != EAGAIN) {
597                         StopClientWatchers(IO, 1);
598                         EV_syslog(LOG_DEBUG,
599                                   "IO_send_callback(): Socket Invalid! [%d] [%s] [%d]\n",
600                                   errno, strerror(errno), IO->SendBuf.fd);
601                         StrBufPrintf(IO->ErrMsg,
602                                      "Socket Invalid! [%s]",
603                                      strerror(errno));
604                         SetNextTimeout(IO, 0.01);
605                 }
606         }
607         /* else : must write more. */
608 }
609 static void
610 set_start_callback(struct ev_loop *loop, AsyncIO *IO, int revents)
611 {
612         ev_timer_stop(event_base, &IO->conn_fail);
613         ev_timer_start(event_base, &IO->rw_timeout);
614
615         switch(IO->NextState) {
616         case eReadMore:
617         case eReadMessage:
618         case eReadFile:
619                 StrBufAppendBufPlain(IO->ErrMsg, HKEY("[while waiting for greeting]"), 0);
620                 ev_io_start(event_base, &IO->recv_event);
621                 break;
622         case eSendReply:
623         case eSendMore:
624         case eReadPayload:
625         case eSendFile:
626                 become_session(IO->CitContext);
627                 IO_send_callback(loop, &IO->send_event, revents);
628                 break;
629         case eDBQuery:
630         case eSendDNSQuery:
631         case eReadDNSReply:
632         case eConnect:
633         case eTerminateConnection:
634         case eAbort:
635                 /// TODO: WHUT?
636                 break;
637         }
638 }
639
640 static void
641 IO_Timeout_callback(struct ev_loop *loop, ev_timer *watcher, int revents)
642 {
643         AsyncIO *IO = watcher->data;
644
645         IO->Now = ev_now(event_base);
646         ev_timer_stop (event_base, &IO->rw_timeout);
647         become_session(IO->CitContext);
648
649         if (IO->SendBuf.fd != 0)
650         {
651                 ev_io_stop(event_base, &IO->send_event);
652                 ev_io_stop(event_base, &IO->recv_event);
653                 ev_timer_stop (event_base, &IO->rw_timeout);
654                 close(IO->SendBuf.fd);
655                 IO->SendBuf.fd = IO->RecvBuf.fd = 0;
656         }
657
658         assert(IO->Timeout);
659         switch (IO->Timeout(IO))
660         {
661         case eAbort:
662                 ShutDownCLient(IO);
663         default:
664                 break;
665         }
666 }
667
668 static void
669 IO_connfail_callback(struct ev_loop *loop, ev_timer *watcher, int revents)
670 {
671         AsyncIO *IO = watcher->data;
672
673         IO->Now = ev_now(event_base);
674         ev_timer_stop (event_base, &IO->conn_fail);
675
676         if (IO->SendBuf.fd != 0)
677         {
678                 ev_io_stop(loop, &IO->conn_event);
679                 ev_io_stop(event_base, &IO->send_event);
680                 ev_io_stop(event_base, &IO->recv_event);
681                 ev_timer_stop (event_base, &IO->rw_timeout);
682                 close(IO->SendBuf.fd);
683                 IO->SendBuf.fd = IO->RecvBuf.fd = 0;
684         }
685         become_session(IO->CitContext);
686
687         assert(IO->ConnFail);
688         switch (IO->ConnFail(IO))
689         {
690         case eAbort:
691                 ShutDownCLient(IO);
692         default:
693                 break;
694
695         }
696 }
697
698 static void
699 IO_connfailimmediate_callback(struct ev_loop *loop,
700                               ev_idle *watcher,
701                               int revents)
702 {
703         AsyncIO *IO = watcher->data;
704
705         IO->Now = ev_now(event_base);
706         ev_idle_stop (event_base, &IO->conn_fail_immediate);
707
708         if (IO->SendBuf.fd != 0)
709         {
710                 close(IO->SendBuf.fd);
711                 IO->SendBuf.fd = IO->RecvBuf.fd = 0;
712         }
713         become_session(IO->CitContext);
714
715         assert(IO->ConnFail);
716         switch (IO->ConnFail(IO))
717         {
718         case eAbort:
719                 ShutDownCLient(IO);
720         default:
721                 break;
722
723         }
724 }
725
726 static void
727 IO_connestd_callback(struct ev_loop *loop, ev_io *watcher, int revents)
728 {
729         AsyncIO *IO = watcher->data;
730         int             so_err = 0;
731         socklen_t       lon = sizeof(so_err);
732         int             err;
733
734         IO->Now = ev_now(event_base);
735         EVM_syslog(LOG_DEBUG, "connect() succeeded.\n");
736
737         ev_io_stop(loop, &IO->conn_event);
738         ev_timer_stop(event_base, &IO->conn_fail);
739
740         err = getsockopt(IO->SendBuf.fd,
741                          SOL_SOCKET,
742                          SO_ERROR,
743                          (void*)&so_err,
744                          &lon);
745
746         if ((err == 0) && (so_err != 0))
747         {
748                 EV_syslog(LOG_DEBUG, "connect() failed [%d][%s]\n",
749                           so_err,
750                           strerror(so_err));
751                 IO_connfail_callback(loop, &IO->conn_fail, revents);
752
753         }
754         else
755         {
756                 EVM_syslog(LOG_DEBUG, "connect() succeeded\n");
757                 set_start_callback(loop, IO, revents);
758         }
759 }
760
761 static void
762 IO_recv_callback(struct ev_loop *loop, ev_io *watcher, int revents)
763 {
764         const char *errmsg;
765         ssize_t nbytes;
766         AsyncIO *IO = watcher->data;
767
768         IO->Now = ev_now(event_base);
769         switch (IO->NextState) {
770         case eReadFile:
771                 nbytes = FileRecvChunked(&IO->IOB, &errmsg);
772                 if (nbytes < 0)
773                         StrBufPlain(IO->ErrMsg, errmsg, -1);
774                 else
775                 {
776                         if (IO->IOB.ChunkSendRemain == 0)
777                         {
778                                 IO->NextState = eSendReply;
779                                 assert(IO->ReadDone);
780                                 ev_io_stop(event_base, &IO->recv_event);
781                                 PostInbound(IO);
782                                 return;
783                         }
784                         else
785                                 return;
786                 }
787                 break;
788         default:
789                 nbytes = StrBuf_read_one_chunk_callback(IO->RecvBuf.fd,
790                                                         0,
791                                                         &IO->RecvBuf);
792                 break;
793         }
794
795 #ifdef BIGBAD_IODBG
796         {
797                 long nbytes;
798                 int rv = 0;
799                 char fn [SIZ];
800                 FILE *fd;
801                 const char *pch = ChrPtr(IO->RecvBuf.Buf);
802                 const char *pchh = IO->RecvBuf.ReadWritePointer;
803
804                 if (pchh == NULL)
805                         pchh = pch;
806
807                 nbytes = StrLength(IO->RecvBuf.Buf) - (pchh - pch);
808                 snprintf(fn, SIZ, "/tmp/foolog_ev_%s.%d",
809                          ((CitContext*)(IO->CitContext))->ServiceName,
810                          IO->SendBuf.fd);
811
812                 fd = fopen(fn, "a+");
813                 fprintf(fd, "Read: BufSize: %ld BufContent: [",
814                         nbytes);
815                 rv = fwrite(pchh, nbytes, 1, fd);
816                 if (!rv) printf("failed to write debug to %s!\n", fn);
817                 fprintf(fd, "]\n");
818                 fclose(fd);
819         }
820 #endif
821         if (nbytes > 0) {
822                 HandleInbound(IO);
823         } else if (nbytes == 0) {
824                 StopClientWatchers(IO, 1);
825                 SetNextTimeout(IO, 0.01);
826                 return;
827         } else if (nbytes == -1) {
828                 if (errno != EAGAIN) {
829                         // FD is gone. kick it. 
830                         StopClientWatchers(IO, 1);
831                         EV_syslog(LOG_DEBUG,
832                                   "IO_recv_callback(): Socket Invalid! [%d] [%s] [%d]\n",
833                                   errno, strerror(errno), IO->SendBuf.fd);
834                         StrBufPrintf(IO->ErrMsg,
835                                      "Socket Invalid! [%s]",
836                                      strerror(errno));
837                         SetNextTimeout(IO, 0.01);
838                 }
839                 return;
840         }
841 }
842
843 void
844 IO_postdns_callback(struct ev_loop *loop, ev_idle *watcher, int revents)
845 {
846         AsyncIO *IO = watcher->data;
847         IO->Now = ev_now(event_base);
848         EV_syslog(LOG_DEBUG, "event: %s\n", __FUNCTION__);
849         become_session(IO->CitContext);
850         assert(IO->DNS.Query->PostDNS);
851         switch (IO->DNS.Query->PostDNS(IO))
852         {
853         case eAbort:
854                 assert(IO->DNS.Fail);
855                 switch (IO->DNS.Fail(IO)) {
856                 case eAbort:
857 ////                    StopClientWatchers(IO);
858                         ShutDownCLient(IO);
859                 default:
860                         break;
861                 }
862         default:
863                 break;
864         }
865 }
866
867
868 eNextState EvConnectSock(AsyncIO *IO,
869                          double conn_timeout,
870                          double first_rw_timeout,
871                          int ReadFirst)
872 {
873         struct sockaddr_in egress_sin;
874         int fdflags;
875         int rc = -1;
876
877         become_session(IO->CitContext);
878
879         if (ReadFirst) {
880                 IO->NextState = eReadMessage;
881         }
882         else {
883                 IO->NextState = eSendReply;
884         }
885
886         IO->SendBuf.fd = IO->RecvBuf.fd =
887                 socket(
888                         (IO->ConnectMe->IPv6)?PF_INET6:PF_INET,
889                         SOCK_STREAM,
890                         IPPROTO_TCP);
891
892         if (IO->SendBuf.fd < 0) {
893                 EV_syslog(LOG_ERR,
894                           "EVENT: socket() failed: %s\n",
895                           strerror(errno));
896
897                 StrBufPrintf(IO->ErrMsg,
898                              "Failed to create socket: %s",
899                              strerror(errno));
900                 IO->SendBuf.fd = IO->RecvBuf.fd = 0;
901                 return eAbort;
902         }
903         fdflags = fcntl(IO->SendBuf.fd, F_GETFL);
904         if (fdflags < 0) {
905                 EV_syslog(LOG_ERR,
906                           "EVENT: unable to get socket %d flags! %s \n",
907                           IO->SendBuf.fd,
908                           strerror(errno));
909                 StrBufPrintf(IO->ErrMsg,
910                              "Failed to get socket %d flags: %s",
911                              IO->SendBuf.fd,
912                              strerror(errno));
913                 close(IO->SendBuf.fd);
914                 IO->SendBuf.fd = IO->RecvBuf.fd = 0;
915                 return eAbort;
916         }
917         fdflags = fdflags | O_NONBLOCK;
918         if (fcntl(IO->SendBuf.fd, F_SETFL, fdflags) < 0) {
919                 EV_syslog(
920                         LOG_ERR,
921                         "EVENT: unable to set socket %d nonblocking flags! %s \n",
922                         IO->SendBuf.fd,
923                         strerror(errno));
924                 StrBufPrintf(IO->ErrMsg,
925                              "Failed to set socket flags: %s",
926                              strerror(errno));
927                 close(IO->SendBuf.fd);
928                 IO->SendBuf.fd = IO->RecvBuf.fd = 0;
929                 return eAbort;
930         }
931 /* TODO: maye we could use offsetof() to calc the position of data...
932  * http://doc.dvgu.ru/devel/ev.html#associating_custom_data_with_a_watcher
933  */
934         ev_io_init(&IO->recv_event, IO_recv_callback, IO->RecvBuf.fd, EV_READ);
935         IO->recv_event.data = IO;
936         ev_io_init(&IO->send_event, IO_send_callback, IO->SendBuf.fd, EV_WRITE);
937         IO->send_event.data = IO;
938
939         ev_timer_init(&IO->conn_fail, IO_connfail_callback, conn_timeout, 0);
940         IO->conn_fail.data = IO;
941         ev_timer_init(&IO->rw_timeout, IO_Timeout_callback, first_rw_timeout,0);
942         IO->rw_timeout.data = IO;
943
944
945
946
947         /* for debugging you may bypass it like this:
948          * IO->Addr.sin_addr.s_addr = inet_addr("127.0.0.1");
949          * ((struct sockaddr_in)IO->ConnectMe->Addr).sin_addr.s_addr =
950          *   inet_addr("127.0.0.1");
951          */
952         if (IO->ConnectMe->IPv6) {
953                 rc = connect(IO->SendBuf.fd,
954                              &IO->ConnectMe->Addr,
955                              sizeof(struct sockaddr_in6));
956         }
957         else {
958                 /* If citserver is bound to a specific IP address on the host, make
959                  * sure we use that address for outbound connections.
960                  */
961         
962                 memset(&egress_sin, 0, sizeof(egress_sin));
963                 egress_sin.sin_family = AF_INET;
964                 if (!IsEmptyStr(config.c_ip_addr)) {
965                         egress_sin.sin_addr.s_addr = inet_addr(config.c_ip_addr);
966                         if (egress_sin.sin_addr.s_addr == !INADDR_ANY) {
967                                 egress_sin.sin_addr.s_addr = INADDR_ANY;
968                         }
969
970                         /* If this bind fails, no problem; we can still use INADDR_ANY */
971                         bind(IO->SendBuf.fd, (struct sockaddr *)&egress_sin, sizeof(egress_sin));
972                 }
973                 rc = connect(IO->SendBuf.fd,
974                              (struct sockaddr_in *)&IO->ConnectMe->Addr,
975                              sizeof(struct sockaddr_in));
976         }
977
978         if (rc >= 0){
979                 EV_syslog(LOG_DEBUG, "connect() = %d immediate success.\n", IO->SendBuf.fd);
980                 set_start_callback(event_base, IO, 0);
981                 return IO->NextState;
982         }
983         else if (errno == EINPROGRESS) {
984                 EV_syslog(LOG_DEBUG, "connect() = %d have to wait now.\n", IO->SendBuf.fd);
985
986                 ev_io_init(&IO->conn_event,
987                            IO_connestd_callback,
988                            IO->SendBuf.fd,
989                            EV_READ|EV_WRITE);
990
991                 IO->conn_event.data = IO;
992
993                 ev_io_start(event_base, &IO->conn_event);
994                 ev_timer_start(event_base, &IO->conn_fail);
995                 return IO->NextState;
996         }
997         else {
998                 ev_idle_init(&IO->conn_fail_immediate,
999                              IO_connfailimmediate_callback);
1000                 IO->conn_fail_immediate.data = IO;
1001                 ev_idle_start(event_base, &IO->conn_fail_immediate);
1002
1003                 EV_syslog(LOG_ERR,
1004                           "connect() = %d failed: %s\n",
1005                           IO->SendBuf.fd,
1006                           strerror(errno));
1007
1008                 StrBufPrintf(IO->ErrMsg,
1009                              "Failed to connect: %s",
1010                              strerror(errno));
1011                 return IO->NextState;
1012         }
1013         return IO->NextState;
1014 }
1015
1016 void SetNextTimeout(AsyncIO *IO, double timeout)
1017 {
1018         IO->rw_timeout.repeat = timeout;
1019         ev_timer_again (event_base,  &IO->rw_timeout);
1020 }
1021
1022
1023 eNextState ReAttachIO(AsyncIO *IO,
1024                       void *pData,
1025                       int ReadFirst)
1026 {
1027         IO->Data = pData;
1028         become_session(IO->CitContext);
1029         ev_cleanup_start(event_base, &IO->abort_by_shutdown);
1030         if (ReadFirst) {
1031                 IO->NextState = eReadMessage;
1032         }
1033         else {
1034                 IO->NextState = eSendReply;
1035         }
1036         set_start_callback(event_base, IO, 0);
1037
1038         return IO->NextState;
1039 }
1040
1041 void InitIOStruct(AsyncIO *IO,
1042                   void *Data,
1043                   eNextState NextState,
1044                   IO_LineReaderCallback LineReader,
1045                   IO_CallBack DNS_Fail,
1046                   IO_CallBack SendDone,
1047                   IO_CallBack ReadDone,
1048                   IO_CallBack Terminate,
1049                   IO_CallBack DBTerminate,
1050                   IO_CallBack ConnFail,
1051                   IO_CallBack Timeout,
1052                   IO_CallBack ShutdownAbort)
1053 {
1054         IO->Data          = Data;
1055
1056         IO->CitContext    = CloneContext(CC);
1057         IO->CitContext->session_specific_data = Data;
1058         IO->CitContext->IO = IO;
1059
1060         IO->NextState     = NextState;
1061
1062         IO->SendDone      = SendDone;
1063         IO->ReadDone      = ReadDone;
1064         IO->Terminate     = Terminate;
1065         IO->DBTerminate   = DBTerminate;
1066         IO->LineReader    = LineReader;
1067         IO->ConnFail      = ConnFail;
1068         IO->Timeout       = Timeout;
1069         IO->ShutdownAbort = ShutdownAbort;
1070
1071         IO->DNS.Fail      = DNS_Fail;
1072
1073         IO->SendBuf.Buf   = NewStrBufPlain(NULL, 1024);
1074         IO->RecvBuf.Buf   = NewStrBufPlain(NULL, 1024);
1075         IO->IOBuf         = NewStrBuf();
1076         EV_syslog(LOG_DEBUG,
1077                   "EVENT: Session lives at %p IO at %p \n",
1078                   Data, IO);
1079
1080 }
1081
1082 extern int evcurl_init(AsyncIO *IO);
1083
1084 int InitcURLIOStruct(AsyncIO *IO,
1085                      void *Data,
1086                      const char* Desc,
1087                      IO_CallBack SendDone,
1088                      IO_CallBack Terminate,
1089                      IO_CallBack DBTerminate,
1090                      IO_CallBack ShutdownAbort)
1091 {
1092         IO->Data          = Data;
1093
1094         IO->CitContext    = CloneContext(CC);
1095         IO->CitContext->session_specific_data = Data;
1096         IO->CitContext->IO = IO;
1097
1098         IO->SendDone      = SendDone;
1099         IO->Terminate     = Terminate;
1100         IO->DBTerminate   = DBTerminate;
1101         IO->ShutdownAbort = ShutdownAbort;
1102
1103         strcpy(IO->HttpReq.errdesc, Desc);
1104
1105
1106         return  evcurl_init(IO);
1107
1108 }
1109
1110
1111 typedef struct KillOtherSessionContext {
1112         AsyncIO IO;
1113         AsyncIO *OtherOne;
1114 }KillOtherSessionContext;
1115
1116 eNextState KillTerminate(AsyncIO *IO)
1117 {
1118         long id;
1119         KillOtherSessionContext *Ctx = (KillOtherSessionContext*)IO->Data;
1120         EV_syslog(LOG_DEBUG, "%s Exit\n", __FUNCTION__);
1121         id = IO->ID;
1122         FreeAsyncIOContents(IO);
1123         memset(Ctx, 0, sizeof(KillOtherSessionContext));
1124         IO->ID = id; /* just for the case we want to analyze it in a coredump */
1125         free(Ctx);
1126         return eAbort;
1127
1128 }
1129
1130 eNextState KillShutdown(AsyncIO *IO)
1131 {
1132         return eTerminateConnection;
1133 }
1134
1135 eNextState KillOtherContextNow(AsyncIO *IO)
1136 {
1137         KillOtherSessionContext *Ctx = IO->Data;
1138
1139         if (Ctx->OtherOne->ShutdownAbort != NULL)
1140                 Ctx->OtherOne->ShutdownAbort(Ctx->OtherOne);
1141         return eTerminateConnection;
1142 }
1143
1144 void KillAsyncIOContext(AsyncIO *IO)
1145 {
1146         KillOtherSessionContext *Ctx;
1147
1148         Ctx = (KillOtherSessionContext*) malloc(sizeof(KillOtherSessionContext));
1149         memset(Ctx, 0, sizeof(KillOtherSessionContext));
1150         
1151         InitIOStruct(&Ctx->IO,
1152                      Ctx,
1153                      eReadMessage,
1154                      NULL,
1155                      NULL,
1156                      NULL,
1157                      NULL,
1158                      KillTerminate,
1159                      NULL,
1160                      NULL,
1161                      NULL,
1162                      KillShutdown);
1163
1164         Ctx->OtherOne = IO;
1165
1166         switch(IO->NextState) {
1167         case eSendDNSQuery:
1168         case eReadDNSReply:
1169
1170         case eConnect:
1171         case eSendReply:
1172         case eSendMore:
1173         case eSendFile:
1174
1175         case eReadMessage:
1176         case eReadMore:
1177         case eReadPayload:
1178         case eReadFile:
1179                 QueueEventContext(&Ctx->IO, KillOtherContextNow);
1180                 break;
1181         case eDBQuery:
1182                 QueueDBOperation(&Ctx->IO, KillOtherContextNow);
1183                 break;
1184         case eTerminateConnection:
1185         case eAbort:
1186                 /*hm, its already dying, dunno which Queue its in... */
1187                 free(Ctx);
1188         }
1189         
1190 }
1191
1192 extern int DebugEventLoopBacktrace;
1193 void EV_backtrace(AsyncIO *IO)
1194 {
1195 #ifdef HAVE_BACKTRACE
1196         void *stack_frames[50];
1197         size_t size, i;
1198         char **strings;
1199
1200         if ((IO == NULL) || (DebugEventLoopBacktrace == 0))
1201                 return;
1202         size = backtrace(stack_frames, sizeof(stack_frames) / sizeof(void*));
1203         strings = backtrace_symbols(stack_frames, size);
1204         for (i = 0; i < size; i++) {
1205                 if (strings != NULL) {
1206                         EV_syslog(LOG_ALERT, " BT %s\n", strings[i]);
1207                 }
1208                 else {
1209                         EV_syslog(LOG_ALERT, " BT %p\n", stack_frames[i]);
1210                 }
1211         }
1212         free(strings);
1213 #endif
1214 }
1215
1216
1217 ev_tstamp ctdl_ev_now (void)
1218 {
1219         return ev_now(event_base);
1220 }