citadel/modules/networkclient/serv_networkclient.c

   1 /*
   2  * This module handles shared rooms, inter-Citadel mail, and outbound
   3  * mailing list processing.
   4  *
   5  * Copyright (c) 2000-2012 by the citadel.org team
   6  *
   7  *  This program is open source software; you can redistribute it and/or modify
   8  *  it under the terms of the GNU General Public License, version 3.
   9  *
  10  *  This program is distributed in the hope that it will be useful,
  11  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
  12  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  13  *  GNU General Public License for more details.
  14  *
  15  * ** NOTE **   A word on the S_NETCONFIGS semaphore:
  16  * This is a fairly high-level type of critical section.  It ensures that no
  17  * two threads work on the netconfigs files at the same time.  Since we do
  18  * so many things inside these, here are the rules:
  19  *  1. begin_critical_section(S_NETCONFIGS) *before* begin_ any others.
  20  *  2. Do *not* perform any I/O with the client during these sections.
  21  *
  22  */
  23
  24
  25 #include "sysdep.h"
  26 #include <stdlib.h>
  27 #include <unistd.h>
  28 #include <stdio.h>
  29 #include <fcntl.h>
  30 #include <ctype.h>
  31 #include <signal.h>
  32 #include <pwd.h>
  33 #include <errno.h>
  34 #include <sys/stat.h>
  35 #include <sys/types.h>
  36 #include <dirent.h>
  37 #if TIME_WITH_SYS_TIME
  38 # include <sys/time.h>
  39 # include <time.h>
  40 #else
  41 # if HAVE_SYS_TIME_H
  42 #  include <sys/time.h>
  43 # else
  44 #  include <time.h>
  45 # endif
  46 #endif
  47 #ifdef HAVE_SYSCALL_H
  48 # include <syscall.h>
  49 #else
  50 # if HAVE_SYS_SYSCALL_H
  51 #  include <sys/syscall.h>
  52 # endif
  53 #endif
  54
  55 #include <sys/wait.h>
  56 #include <string.h>
  57 #include <limits.h>
  58 #include <libcitadel.h>
  59 #include "citadel.h"
  60 #include "server.h"
  61 #include "citserver.h"
  62 #include "support.h"
  63 #include "config.h"
  64 #include "user_ops.h"
  65 #include "database.h"
  66 #include "msgbase.h"
  67 #include "internet_addressing.h"
  68 #include "clientsocket.h"
  69 #include "citadel_dirs.h"
  70 #include "threads.h"
  71 #include "context.h"
  72 #include "ctdl_module.h"
  73
  74 struct CitContext networker_client_CC;
  75
  76 #define NODE ChrPtr(((AsyncNetworker*)IO->Data)->node)
  77 #define N ((AsyncNetworker*)IO->Data)->n
  78
  79 int NetworkClientDebugEnabled = 0;
  80
  81 #define NCDBGLOG(LEVEL) if ((LEVEL != LOG_DEBUG) || (NetworkClientDebugEnabled != 0))
  82
  83 #define EVN_syslog(LEVEL, FORMAT, ...)                                  \
  84         NCDBGLOG(LEVEL) syslog(LEVEL,                                   \
  85                                "%s[%ld]CC[%d]NW[%s][%ld]" FORMAT,       \
  86                                IOSTR, IO->ID, CCID, NODE, N, __VA_ARGS__)
  87
  88 #define EVNM_syslog(LEVEL, FORMAT)                                      \
  89         NCDBGLOG(LEVEL) syslog(LEVEL,                                   \
  90                                "%s[%ld]CC[%d]NW[%s][%ld]" FORMAT,       \
  91                                IOSTR, IO->ID, CCID, NODE, N)
  92
  93 #define EVNCS_syslog(LEVEL, FORMAT, ...) \
  94         NCDBGLOG(LEVEL) syslog(LEVEL, "%s[%ld]NW[%s][%ld]" FORMAT,      \
  95                                IOSTR, IO->ID, NODE, N, __VA_ARGS__)
  96
  97 #define EVNCSM_syslog(LEVEL, FORMAT) \
  98         NCDBGLOG(LEVEL) syslog(LEVEL, "%s[%ld]NW[%s][%ld]" FORMAT,      \
  99                                IOSTR, IO->ID, NODE, N)
 100
 101
 102 typedef enum _eNWCState {
 103         eGreating,
 104         eAuth,
 105         eNDOP,
 106         eREAD,
 107         eReadBLOB,
 108         eCLOS,
 109         eNUOP,
 110         eWRIT,
 111         eWriteBLOB,
 112         eUCLS,
 113         eQUIT
 114 }eNWCState;
 115
 116 typedef enum _eNWCVState {
 117         eNWCVSLookup,
 118         eNWCVSConnecting,
 119         eNWCVSConnFail,
 120         eNWCVSGreating,
 121         eNWCVSAuth,
 122         eNWCVSAuthFailNTT,
 123         eNWCVSAuthFail,
 124         eNWCVSNDOP,
 125         eNWCVSNDOPDone,
 126         eNWCVSNUOP,
 127         eNWCVSNUOPDone,
 128         eNWCVSFail
 129 }eNWCVState;
 130
 131 ConstStr NWCStateStr[] = {
 132         {HKEY("Looking up Host")},
 133         {HKEY("Connecting host")},
 134         {HKEY("Failed to connect")},
 135         {HKEY("Rread Greeting")},
 136         {HKEY("Authenticating")},
 137         {HKEY("Auth failed by NTT")},
 138         {HKEY("Auth failed")},
 139         {HKEY("Downloading")},
 140         {HKEY("Downloading Success")},
 141         {HKEY("Uploading Spoolfile")},
 142         {HKEY("Uploading done")},
 143         {HKEY("failed")}
 144 };
 145
 146 void SetNWCState(AsyncIO *IO, eNWCVState State)
 147 {
 148         CitContext* CCC = IO->CitContext;
 149         memcpy(CCC->cs_clientname, NWCStateStr[State].Key, NWCStateStr[State].len + 1);
 150 }
 151
 152 typedef struct _async_networker {
 153         AsyncIO IO;
 154         DNSQueryParts HostLookup;
 155         eNWCState State;
 156         long n;
 157         StrBuf *SpoolFileName;
 158         StrBuf *tempFileName;
 159         StrBuf *node;
 160         StrBuf *host;
 161         StrBuf *port;
 162         StrBuf *secret;
 163         StrBuf          *Url;
 164 } AsyncNetworker;
 165
 166 typedef eNextState(*NWClientHandler)(AsyncNetworker* NW);
 167 eNextState nwc_get_one_host_ip(AsyncIO *IO);
 168
 169 eNextState nwc_connect_ip(AsyncIO *IO);
 170
 171 eNextState NWC_SendQUIT(AsyncNetworker *NW);
 172 eNextState NWC_DispatchWriteDone(AsyncIO *IO);
 173
 174 void DeleteNetworker(void *vptr)
 175 {
 176         AsyncNetworker *NW = (AsyncNetworker *)vptr;
 177         FreeStrBuf(&NW->SpoolFileName);
 178         FreeStrBuf(&NW->tempFileName);
 179         FreeStrBuf(&NW->node);
 180         FreeStrBuf(&NW->host);
 181         FreeStrBuf(&NW->port);
 182         FreeStrBuf(&NW->secret);
 183         FreeStrBuf(&NW->Url);
 184         FreeStrBuf(&NW->IO.ErrMsg);
 185         FreeAsyncIOContents(&NW->IO);
 186         if (NW->HostLookup.VParsedDNSReply != NULL) {
 187                 NW->HostLookup.DNSReplyFree(NW->HostLookup.VParsedDNSReply);
 188                 NW->HostLookup.VParsedDNSReply = NULL;
 189         }
 190         free(NW);
 191 }
 192
 193 #define NWC_DBG_SEND() EVN_syslog(LOG_DEBUG, ": > %s", ChrPtr(NW->IO.SendBuf.Buf))
 194 #define NWC_DBG_READ() EVN_syslog(LOG_DEBUG, ": < %s\n", ChrPtr(NW->IO.IOBuf))
 195 #define NWC_OK (strncasecmp(ChrPtr(NW->IO.IOBuf), "+OK", 3) == 0)
 196
 197 eNextState NWC_SendFailureMessage(AsyncIO *IO)
 198 {
 199         AsyncNetworker *NW = IO->Data;
 200         long lens[2];
 201         const char *strs[2];
 202
 203         EVN_syslog(LOG_DEBUG, "NWC: %s\n", __FUNCTION__);
 204
 205         strs[0] = ChrPtr(NW->node);
 206         lens[0] = StrLength(NW->node);
 207
 208         strs[1] = ChrPtr(NW->IO.ErrMsg);
 209         lens[1] = StrLength(NW->IO.ErrMsg);
 210         CtdlAideFPMessage(
 211                 ChrPtr(NW->IO.ErrMsg),
 212                 "Networker error",
 213                 2, strs, (long*) &lens,
 214                 CCID, IO->ID,
 215                 EvGetNow(IO));
 216
 217         return eAbort;
 218 }
 219
 220 eNextState FinalizeNetworker(AsyncIO *IO)
 221 {
 222         AsyncNetworker *NW = (AsyncNetworker *)IO->Data;
 223
 224         CtdlNetworkTalkingTo(SKEY(NW->node), NTT_REMOVE);
 225
 226         DeleteNetworker(IO->Data);
 227         return eAbort;
 228 }
 229
 230 eNextState NWC_ReadGreeting(AsyncNetworker *NW)
 231 {
 232         char connected_to[SIZ];
 233         AsyncIO *IO = &NW->IO;
 234         SetNWCState(IO, eNWCVSGreating);
 235         NWC_DBG_READ();
 236         /* Read the server greeting */
 237         /* Check that the remote is who we think it is and warn the Aide if not */
 238         extract_token (connected_to, ChrPtr(NW->IO.IOBuf), 1, ' ', sizeof connected_to);
 239         if (strcmp(connected_to, ChrPtr(NW->node)) != 0)
 240         {
 241                 if (NW->IO.ErrMsg == NULL)
 242                         NW->IO.ErrMsg = NewStrBuf();
 243                 StrBufPrintf(NW->IO.ErrMsg,
 244                              "Connected to node \"%s\" but I was expecting to connect to node \"%s\".",
 245                              connected_to, ChrPtr(NW->node));
 246                 EVN_syslog(LOG_ERR, "%s\n", ChrPtr(NW->IO.ErrMsg));
 247
 248                 return EventQueueDBOperation(IO, NWC_SendFailureMessage, 1);
 249         }
 250         return eSendReply;
 251 }
 252
 253 eNextState NWC_SendAuth(AsyncNetworker *NW)
 254 {
 255         AsyncIO *IO = &NW->IO;
 256         SetNWCState(IO, eNWCVSAuth);
 257         /* We're talking to the correct node.  Now identify ourselves. */
 258         StrBufPrintf(NW->IO.SendBuf.Buf, "NETP %s|%s\n",
 259                      config.c_nodename,
 260                      ChrPtr(NW->secret));
 261         NWC_DBG_SEND();
 262         return eSendReply;
 263 }
 264
 265 eNextState NWC_ReadAuthReply(AsyncNetworker *NW)
 266 {
 267         AsyncIO *IO = &NW->IO;
 268         NWC_DBG_READ();
 269         if (ChrPtr(NW->IO.IOBuf)[0] == '2')
 270         {
 271                 return eSendReply;
 272         }
 273         else
 274         {
 275                 int Error = atol(ChrPtr(NW->IO.IOBuf));
 276                 if (NW->IO.ErrMsg == NULL)
 277                         NW->IO.ErrMsg = NewStrBuf();
 278                 StrBufPrintf(NW->IO.ErrMsg,
 279                              "Connected to node \"%s\" but my secret wasn't accurate.\nReason was:%s\n",
 280                              ChrPtr(NW->node), ChrPtr(NW->IO.IOBuf) + 4);
 281                 if (Error == 552) {
 282                         SetNWCState(IO, eNWCVSAuthFailNTT);
 283                         EVN_syslog(LOG_INFO,
 284                                    "Already talking to %s; skipping this time.\n",
 285                                    ChrPtr(NW->node));
 286
 287                 }
 288                 else {
 289                         SetNWCState(IO, eNWCVSAuthFailNTT);
 290                         EVN_syslog(LOG_ERR, "%s\n", ChrPtr(NW->IO.ErrMsg));
 291                         return EventQueueDBOperation(IO, NWC_SendFailureMessage, 1);
 292                 }
 293                 return eAbort;
 294         }
 295 }
 296
 297 eNextState NWC_SendNDOP(AsyncNetworker *NW)
 298 {
 299         AsyncIO *IO = &NW->IO;
 300         SetNWCState(IO, eNWCVSNDOP);
 301         NW->tempFileName = NewStrBuf();
 302         NW->SpoolFileName = NewStrBuf();
 303         StrBufPrintf(NW->SpoolFileName,
 304                      "%s/%s.%lx%x",
 305                      ctdl_netin_dir,
 306                      ChrPtr(NW->node),
 307                      time(NULL),// TODO: get time from libev
 308                      rand());
 309         StrBufStripSlashes(NW->SpoolFileName, 1);
 310         StrBufPrintf(NW->tempFileName,
 311                      "%s/%s.%lx%x",
 312                      ctdl_nettmp_dir,
 313                      ChrPtr(NW->node),
 314                      time(NULL),// TODO: get time from libev
 315                      rand());
 316         StrBufStripSlashes(NW->tempFileName, 1);
 317         /* We're talking to the correct node.  Now identify ourselves. */
 318         StrBufPlain(NW->IO.SendBuf.Buf, HKEY("NDOP\n"));
 319         NWC_DBG_SEND();
 320         return eSendReply;
 321 }
 322
 323 eNextState NWC_ReadNDOPReply(AsyncNetworker *NW)
 324 {
 325         AsyncIO *IO = &NW->IO;
 326         int TotalSendSize;
 327         NWC_DBG_READ();
 328         if (ChrPtr(NW->IO.IOBuf)[0] == '2')
 329         {
 330                 int LogLevel = LOG_DEBUG;
 331
 332                 NW->IO.IOB.TotalSentAlready = 0;
 333
 334                 TotalSendSize = atol (ChrPtr(NW->IO.IOBuf) + 4);
 335
 336                 if (TotalSendSize > 0)
 337                         LogLevel = LOG_INFO;
 338
 339                 EVN_syslog(LogLevel,
 340                            "Expecting to transfer %d bytes to %s\n",
 341                            TotalSendSize,
 342                            ChrPtr(NW->tempFileName));
 343
 344                 if (TotalSendSize <= 0) {
 345                         NW->State = eNUOP - 1;
 346                 }
 347                 else {
 348                         int fd;
 349                         fd = open(ChrPtr(NW->tempFileName),
 350                                   O_EXCL|O_CREAT|O_NONBLOCK|O_WRONLY,
 351                                   S_IRUSR|S_IWUSR);
 352                         if (fd < 0)
 353                         {
 354                                 SetNWCState(IO, eNWCVSFail);
 355                                 EVN_syslog(LOG_CRIT,
 356                                        "cannot open %s: %s\n",
 357                                        ChrPtr(NW->tempFileName),
 358                                        strerror(errno));
 359
 360                                 NW->State = eQUIT - 1;
 361                                 return eAbort;
 362                         }
 363                         FDIOBufferInit(&NW->IO.IOB, &NW->IO.RecvBuf, fd, TotalSendSize);
 364                 }
 365                 return eSendReply;
 366         }
 367         else
 368         {
 369                 SetNWCState(IO, eNWCVSFail);
 370                 return eAbort;
 371         }
 372 }
 373
 374 eNextState NWC_SendREAD(AsyncNetworker *NW)
 375 {
 376         AsyncIO *IO = &NW->IO;
 377         eNextState rc;
 378
 379         if (NW->IO.IOB.TotalSentAlready < NW->IO.IOB.TotalSendSize)
 380         {
 381                 /*
 382                  * If shutting down we can exit here and unlink the temp file.
 383                  * this shouldn't loose us any messages.
 384                  */
 385                 if (server_shutting_down)
 386                 {
 387                         FDIOBufferDelete(&NW->IO.IOB);
 388                         unlink(ChrPtr(NW->tempFileName));
 389                         FDIOBufferDelete(&IO->IOB);
 390                         SetNWCState(IO, eNWCVSFail);
 391                         return eAbort;
 392                 }
 393                 StrBufPrintf(NW->IO.SendBuf.Buf, "READ "LOFF_T_FMT"|%ld\n",
 394                              NW->IO.IOB.TotalSentAlready,
 395                              NW->IO.IOB.TotalSendSize);
 396 /*
 397                              ((NW->IO.IOB.TotalSendSize - NW->IO.IOB.TotalSentAlready > IGNET_PACKET_SIZE)
 398                               ? IGNET_PACKET_SIZE :
 399                               (NW->IO.IOB.TotalSendSize - NW->IO.IOB.TotalSentAlready))
 400                         );
 401 */
 402                 NWC_DBG_SEND();
 403                 return eSendReply;
 404         }
 405         else
 406         {
 407                 NW->State = eCLOS;
 408                 rc = NWC_DispatchWriteDone(&NW->IO);
 409                 NWC_DBG_SEND();
 410
 411                 return rc;
 412         }
 413 }
 414
 415 eNextState NWC_ReadREADState(AsyncNetworker *NW)
 416 {
 417         AsyncIO *IO = &NW->IO;
 418         NWC_DBG_READ();
 419         if (ChrPtr(NW->IO.IOBuf)[0] == '6')
 420         {
 421                 NW->IO.IOB.ChunkSendRemain =
 422                         NW->IO.IOB.ChunkSize = atol(ChrPtr(NW->IO.IOBuf)+4);
 423                 return eReadFile;
 424         }
 425         FDIOBufferDelete(&IO->IOB);
 426         return eAbort;
 427 }
 428 eNextState NWC_ReadREADBlobDone(AsyncNetworker *NW);
 429 eNextState NWC_ReadREADBlob(AsyncNetworker *NW)
 430 {
 431         eNextState rc;
 432         AsyncIO *IO = &NW->IO;
 433         NWC_DBG_READ();
 434         if (NW->IO.IOB.TotalSendSize == NW->IO.IOB.TotalSentAlready)
 435         {
 436                 NW->State ++;
 437
 438                 FDIOBufferDelete(&NW->IO.IOB);
 439
 440                 if (link(ChrPtr(NW->tempFileName), ChrPtr(NW->SpoolFileName)) != 0) {
 441                         EVN_syslog(LOG_ALERT,
 442                                "Could not link %s to %s: %s\n",
 443                                ChrPtr(NW->tempFileName),
 444                                ChrPtr(NW->SpoolFileName),
 445                                strerror(errno));
 446                 }
 447                 else {
 448                         EVN_syslog(LOG_INFO,
 449                                "moved %s to %s\n",
 450                                ChrPtr(NW->tempFileName),
 451                                ChrPtr(NW->SpoolFileName));
 452                 }
 453
 454                 unlink(ChrPtr(NW->tempFileName));
 455                 rc = NWC_DispatchWriteDone(&NW->IO);
 456                 NW->State --;
 457                 return rc;
 458         }
 459         else {
 460                 NW->State --;
 461                 NW->IO.IOB.ChunkSendRemain = NW->IO.IOB.ChunkSize;
 462                 return eSendReply; //NWC_DispatchWriteDone(&NW->IO);
 463         }
 464 }
 465
 466 eNextState NWC_ReadREADBlobDone(AsyncNetworker *NW)
 467 {
 468         eNextState rc;
 469         AsyncIO *IO = &NW->IO;
 470 /* we don't have any data to debug print here. */
 471         if (NW->IO.IOB.TotalSentAlready >= NW->IO.IOB.TotalSendSize)
 472         {
 473                 NW->State ++;
 474
 475                 FDIOBufferDelete(&NW->IO.IOB);
 476                 if (link(ChrPtr(NW->tempFileName), ChrPtr(NW->SpoolFileName)) != 0) {
 477                         EVN_syslog(LOG_ALERT,
 478                                "Could not link %s to %s: %s\n",
 479                                ChrPtr(NW->tempFileName),
 480                                ChrPtr(NW->SpoolFileName),
 481                                strerror(errno));
 482                 }
 483                 else {
 484                         EVN_syslog(LOG_INFO,
 485                                "moved %s to %s\n",
 486                                ChrPtr(NW->tempFileName),
 487                                ChrPtr(NW->SpoolFileName));
 488                 }
 489
 490                 unlink(ChrPtr(NW->tempFileName));
 491                 rc = NWC_DispatchWriteDone(&NW->IO);
 492                 NW->State --;
 493                 return rc;
 494         }
 495         else {
 496                 NW->State --;
 497                 NW->IO.IOB.ChunkSendRemain = NW->IO.IOB.ChunkSize;
 498                 return NWC_DispatchWriteDone(&NW->IO);
 499         }
 500 }
 501 eNextState NWC_SendCLOS(AsyncNetworker *NW)
 502 {
 503         AsyncIO *IO = &NW->IO;
 504         SetNWCState(IO, eNWCVSNDOPDone);
 505         StrBufPlain(NW->IO.SendBuf.Buf, HKEY("CLOS\n"));
 506         NWC_DBG_SEND();
 507         return eSendReply;
 508 }
 509
 510 eNextState NWC_ReadCLOSReply(AsyncNetworker *NW)
 511 {
 512         AsyncIO *IO = &NW->IO;
 513         NWC_DBG_READ();
 514         FDIOBufferDelete(&IO->IOB);
 515         if (ChrPtr(NW->IO.IOBuf)[0] != '2')
 516                 return eTerminateConnection;
 517         return eSendReply;
 518 }
 519
 520
 521 eNextState NWC_SendNUOP(AsyncNetworker *NW)
 522 {
 523         AsyncIO *IO = &NW->IO;
 524         eNextState rc;
 525         long TotalSendSize;
 526         struct stat statbuf;
 527         int fd;
 528
 529         SetNWCState(IO, eNWCVSNUOP);
 530         StrBufPrintf(NW->SpoolFileName,
 531                      "%s/%s",
 532                      ctdl_netout_dir,
 533                      ChrPtr(NW->node));
 534         StrBufStripSlashes(NW->SpoolFileName, 1);
 535
 536         fd = open(ChrPtr(NW->SpoolFileName), O_EXCL|O_NONBLOCK|O_RDONLY);
 537         if (fd < 0) {
 538                 if (errno != ENOENT) {
 539                         EVN_syslog(LOG_CRIT,
 540                                "cannot open %s: %s\n",
 541                                ChrPtr(NW->SpoolFileName),
 542                                strerror(errno));
 543                 }
 544                 NW->State = eQUIT;
 545                 rc = NWC_SendQUIT(NW);
 546                 NWC_DBG_SEND();
 547                 return rc;
 548         }
 549
 550         if (fstat(fd, &statbuf) == -1) {
 551                 EVN_syslog(LOG_CRIT, "FSTAT FAILED %s [%s]--\n",
 552                            ChrPtr(NW->SpoolFileName),
 553                            strerror(errno));
 554                 if (fd > 0) close(fd);
 555                 return eAbort;
 556         }
 557         TotalSendSize = statbuf.st_size;
 558         if (TotalSendSize == 0) {
 559                 EVNM_syslog(LOG_DEBUG,
 560                        "Nothing to send.\n");
 561                 NW->State = eQUIT;
 562                 rc = NWC_SendQUIT(NW);
 563                 NWC_DBG_SEND();
 564                 if (fd > 0) close(fd);
 565                 return rc;
 566         }
 567         else
 568         {
 569                 EVN_syslog(LOG_INFO,
 570                            "sending %s to %s\n",
 571                            ChrPtr(NW->SpoolFileName),
 572                            ChrPtr(NW->node));
 573         }
 574
 575         FDIOBufferInit(&NW->IO.IOB, &NW->IO.SendBuf, fd, TotalSendSize);
 576
 577         StrBufPlain(NW->IO.SendBuf.Buf, HKEY("NUOP\n"));
 578         NWC_DBG_SEND();
 579         return eSendReply;
 580
 581 }
 582 eNextState NWC_ReadNUOPReply(AsyncNetworker *NW)
 583 {
 584         AsyncIO *IO = &NW->IO;
 585         NWC_DBG_READ();
 586         if (ChrPtr(NW->IO.IOBuf)[0] != '2') {
 587                 FDIOBufferDelete(&IO->IOB);
 588                 return eAbort;
 589         }
 590         return eSendReply;
 591 }
 592
 593 eNextState NWC_SendWRIT(AsyncNetworker *NW)
 594 {
 595         AsyncIO *IO = &NW->IO;
 596         StrBufPrintf(NW->IO.SendBuf.Buf, "WRIT "LOFF_T_FMT"\n",
 597                      NW->IO.IOB.TotalSendSize - NW->IO.IOB.TotalSentAlready);
 598         NWC_DBG_SEND();
 599         return eSendReply;
 600 }
 601 eNextState NWC_ReadWRITReply(AsyncNetworker *NW)
 602 {
 603         AsyncIO *IO = &NW->IO;
 604         NWC_DBG_READ();
 605         if (ChrPtr(NW->IO.IOBuf)[0] != '7')
 606         {
 607                 FDIOBufferDelete(&IO->IOB);
 608                 return eAbort;
 609         }
 610
 611         NW->IO.IOB.ChunkSendRemain =
 612                 NW->IO.IOB.ChunkSize = atol(ChrPtr(NW->IO.IOBuf)+4);
 613         return eSendFile;
 614 }
 615
 616 eNextState NWC_SendBlobDone(AsyncNetworker *NW)
 617 {
 618         AsyncIO *IO = &NW->IO;
 619         eNextState rc;
 620         if (NW->IO.IOB.TotalSentAlready >= IO->IOB.TotalSendSize)
 621         {
 622                 NW->State ++;
 623
 624                 FDIOBufferDelete(&IO->IOB);
 625                 rc =  NWC_DispatchWriteDone(IO);
 626                 NW->State --;
 627                 return rc;
 628         }
 629         else {
 630                 NW->State --;
 631                 IO->IOB.ChunkSendRemain = IO->IOB.ChunkSize;
 632                 rc = NWC_DispatchWriteDone(IO);
 633                 NW->State --;
 634                 return rc;
 635         }
 636 }
 637
 638 eNextState NWC_SendUCLS(AsyncNetworker *NW)
 639 {
 640         AsyncIO *IO = &NW->IO;
 641         StrBufPlain(NW->IO.SendBuf.Buf, HKEY("UCLS 1\n"));
 642         NWC_DBG_SEND();
 643         return eSendReply;
 644
 645 }
 646 eNextState NWC_ReadUCLS(AsyncNetworker *NW)
 647 {
 648         AsyncIO *IO = &NW->IO;
 649         NWC_DBG_READ();
 650
 651         EVN_syslog(LOG_NOTICE,
 652                    "Sent %s [%ld] octets to <%s>\n",
 653                    ChrPtr(NW->SpoolFileName),
 654                    NW->IO.IOB.ChunkSize,
 655                    ChrPtr(NW->node));
 656
 657         if (ChrPtr(NW->IO.IOBuf)[0] == '2') {
 658                 EVN_syslog(LOG_DEBUG, "Removing <%s>\n", ChrPtr(NW->SpoolFileName));
 659                 unlink(ChrPtr(NW->SpoolFileName));
 660         }
 661         FDIOBufferDelete(&IO->IOB);
 662         SetNWCState(IO, eNWCVSNUOPDone);
 663         return eSendReply;
 664 }
 665
 666 eNextState NWC_SendQUIT(AsyncNetworker *NW)
 667 {
 668         AsyncIO *IO = &NW->IO;
 669         StrBufPlain(NW->IO.SendBuf.Buf, HKEY("QUIT\n"));
 670
 671         NWC_DBG_SEND();
 672         return eSendReply;
 673 }
 674
 675 eNextState NWC_ReadQUIT(AsyncNetworker *NW)
 676 {
 677         AsyncIO *IO = &NW->IO;
 678         NWC_DBG_READ();
 679
 680         return eAbort;
 681 }
 682
 683
 684 NWClientHandler NWC_ReadHandlers[] = {
 685         NWC_ReadGreeting,
 686         NWC_ReadAuthReply,
 687         NWC_ReadNDOPReply,
 688         NWC_ReadREADState,
 689         NWC_ReadREADBlob,
 690         NWC_ReadCLOSReply,
 691         NWC_ReadNUOPReply,
 692         NWC_ReadWRITReply,
 693         NWC_SendBlobDone,
 694         NWC_ReadUCLS,
 695         NWC_ReadQUIT};
 696
 697 long NWC_ConnTimeout = 100;
 698
 699 const long NWC_SendTimeouts[] = {
 700         100,
 701         100,
 702         100,
 703         100,
 704         100,
 705         100,
 706         100,
 707         100
 708 };
 709 const ConstStr NWC[] = {
 710         {HKEY("Connection broken during ")},
 711         {HKEY("Connection broken during ")},
 712         {HKEY("Connection broken during ")},
 713         {HKEY("Connection broken during ")},
 714         {HKEY("Connection broken during ")},
 715         {HKEY("Connection broken during ")},
 716         {HKEY("Connection broken during ")},
 717         {HKEY("Connection broken during ")}
 718 };
 719
 720 NWClientHandler NWC_SendHandlers[] = {
 721         NULL,
 722         NWC_SendAuth,
 723         NWC_SendNDOP,
 724         NWC_SendREAD,
 725         NWC_ReadREADBlobDone,
 726         NWC_SendCLOS,
 727         NWC_SendNUOP,
 728         NWC_SendWRIT,
 729         NWC_SendBlobDone,
 730         NWC_SendUCLS,
 731         NWC_SendQUIT
 732 };
 733
 734 const long NWC_ReadTimeouts[] = {
 735         100,
 736         100,
 737         100,
 738         100,
 739         100,
 740         100,
 741         100,
 742         100,
 743         100,
 744         100
 745 };
 746
 747
 748
 749
 750 eNextState nwc_get_one_host_ip_done(AsyncIO *IO)
 751 {
 752         AsyncNetworker *NW = IO->Data;
 753         struct hostent *hostent;
 754
 755         QueryCbDone(IO);
 756
 757         hostent = NW->HostLookup.VParsedDNSReply;
 758         if ((NW->HostLookup.DNSStatus == ARES_SUCCESS) &&
 759             (hostent != NULL) ) {
 760                 memset(&NW->IO.ConnectMe->Addr, 0, sizeof(struct in6_addr));
 761                 if (NW->IO.ConnectMe->IPv6) {
 762                         memcpy(&NW->IO.ConnectMe->Addr.sin6_addr.s6_addr,
 763                                &hostent->h_addr_list[0],
 764                                sizeof(struct in6_addr));
 765
 766                         NW->IO.ConnectMe->Addr.sin6_family = hostent->h_addrtype;
 767                         NW->IO.ConnectMe->Addr.sin6_port   = htons(atol(ChrPtr(NW->port)));//// TODO use the one from the URL.
 768                 }
 769                 else {
 770                         struct sockaddr_in *addr = (struct sockaddr_in*) &NW->IO.ConnectMe->Addr;
 771                         /* Bypass the ns lookup result like this: IO->Addr.sin_addr.s_addr = inet_addr("127.0.0.1"); */
 772 //                      addr->sin_addr.s_addr = htonl((uint32_t)&hostent->h_addr_list[0]);
 773                         memcpy(&addr->sin_addr.s_addr,
 774                                hostent->h_addr_list[0],
 775                                sizeof(uint32_t));
 776
 777                         addr->sin_family = hostent->h_addrtype;
 778                         addr->sin_port   = htons(504);/// default citadel port
 779
 780                 }
 781                 return nwc_connect_ip(IO);
 782         }
 783         else
 784                 return eAbort;
 785 }
 786
 787
 788 eNextState nwc_get_one_host_ip(AsyncIO *IO)
 789 {
 790         AsyncNetworker *NW = IO->Data;
 791         /*
 792          * here we start with the lookup of one host.
 793          */
 794
 795         EVN_syslog(LOG_DEBUG, "NWC: %s\n", __FUNCTION__);
 796
 797         EVN_syslog(LOG_DEBUG,
 798                    "NWC client[%ld]: looking up %s-Record %s : %d ...\n",
 799                    NW->n,
 800                    (NW->IO.ConnectMe->IPv6)? "aaaa": "a",
 801                    NW->IO.ConnectMe->Host,
 802                    NW->IO.ConnectMe->Port);
 803
 804         QueueQuery((NW->IO.ConnectMe->IPv6)? ns_t_aaaa : ns_t_a,
 805                    NW->IO.ConnectMe->Host,
 806                    &NW->IO,
 807                    &NW->HostLookup,
 808                    nwc_get_one_host_ip_done);
 809         IO->NextState = eReadDNSReply;
 810         return IO->NextState;
 811 }
 812 /**
 813  * @brief lineread Handler; understands when to read more POP3 lines, and when this is a one-lined reply.
 814  */
 815 eReadState NWC_ReadServerStatus(AsyncIO *IO)
 816 {
 817 //      AsyncNetworker *NW = IO->Data;
 818         eReadState Finished = eBufferNotEmpty;
 819
 820         switch (IO->NextState) {
 821         case eSendDNSQuery:
 822         case eReadDNSReply:
 823         case eDBQuery:
 824         case eConnect:
 825         case eTerminateConnection:
 826         case eAbort:
 827                 Finished = eReadFail;
 828                 break;
 829         case eSendReply:
 830         case eSendMore:
 831         case eReadMore:
 832         case eReadMessage:
 833                 Finished = StrBufChunkSipLine(IO->IOBuf, &IO->RecvBuf);
 834                 break;
 835         case eReadFile:
 836         case eSendFile:
 837         case eReadPayload:
 838                 break;
 839         }
 840         return Finished;
 841 }
 842
 843
 844
 845 eNextState NWC_FailNetworkConnection(AsyncIO *IO)
 846 {
 847         SetNWCState(IO, eNWCVSConnFail);
 848         return EventQueueDBOperation(IO, NWC_SendFailureMessage, 1);
 849 }
 850
 851 void NWC_SetTimeout(eNextState NextTCPState, AsyncNetworker *NW)
 852 {
 853         double Timeout = 0.0;
 854
 855         //EVN_syslog(LOG_DEBUG, "%s - %d\n", __FUNCTION__, NextTCPState);
 856
 857         switch (NextTCPState) {
 858         case eSendMore:
 859         case eSendReply:
 860         case eReadMessage:
 861                 Timeout = NWC_ReadTimeouts[NW->State];
 862                 break;
 863         case eReadFile:
 864         case eSendFile:
 865         case eReadPayload:
 866                 Timeout = 100000;
 867                 break;
 868         case eSendDNSQuery:
 869         case eReadDNSReply:
 870         case eDBQuery:
 871         case eReadMore:
 872         case eConnect:
 873         case eTerminateConnection:
 874         case eAbort:
 875                 return;
 876         }
 877         if (Timeout > 0) {
 878                 AsyncIO *IO = &NW->IO;
 879                 EVN_syslog(LOG_DEBUG,
 880                            "%s - %d %f\n",
 881                            __FUNCTION__,
 882                            NextTCPState,
 883                            Timeout);
 884                 SetNextTimeout(&NW->IO, Timeout*100);
 885         }
 886 }
 887
 888
 889 eNextState NWC_DispatchReadDone(AsyncIO *IO)
 890 {
 891         EVN_syslog(LOG_DEBUG, "%s\n", __FUNCTION__);
 892         AsyncNetworker *NW = IO->Data;
 893         eNextState rc;
 894
 895         rc = NWC_ReadHandlers[NW->State](NW);
 896
 897         if ((rc != eReadMore) &&
 898             (rc != eAbort) &&
 899             (rc != eDBQuery)) {
 900                 NW->State++;
 901         }
 902
 903         NWC_SetTimeout(rc, NW);
 904
 905         return rc;
 906 }
 907 eNextState NWC_DispatchWriteDone(AsyncIO *IO)
 908 {
 909         EVN_syslog(LOG_DEBUG, "%s\n", __FUNCTION__);
 910         AsyncNetworker *NW = IO->Data;
 911         eNextState rc;
 912
 913         rc = NWC_SendHandlers[NW->State](NW);
 914         NWC_SetTimeout(rc, NW);
 915         return rc;
 916 }
 917
 918 /*****************************************************************************/
 919 /*                     Networker CLIENT ERROR CATCHERS                       */
 920 /*****************************************************************************/
 921 eNextState NWC_Terminate(AsyncIO *IO)
 922 {
 923         EVN_syslog(LOG_DEBUG, "%s\n", __FUNCTION__);
 924         FinalizeNetworker(IO);
 925         return eAbort;
 926 }
 927
 928 eNextState NWC_TerminateDB(AsyncIO *IO)
 929 {
 930         EVN_syslog(LOG_DEBUG, "%s\n", __FUNCTION__);
 931         FinalizeNetworker(IO);
 932         return eAbort;
 933 }
 934
 935 eNextState NWC_Timeout(AsyncIO *IO)
 936 {
 937         AsyncNetworker *NW = IO->Data;
 938         EVN_syslog(LOG_DEBUG, "%s\n", __FUNCTION__);
 939
 940         if (NW->IO.ErrMsg == NULL)
 941                 NW->IO.ErrMsg = NewStrBuf();
 942         StrBufPrintf(NW->IO.ErrMsg, "Timeout while talking to %s \r\n", ChrPtr(NW->host));
 943         return NWC_FailNetworkConnection(IO);
 944 }
 945 eNextState NWC_ConnFail(AsyncIO *IO)
 946 {
 947         AsyncNetworker *NW = IO->Data;
 948
 949         EVN_syslog(LOG_DEBUG, "%s\n", __FUNCTION__);
 950         if (NW->IO.ErrMsg == NULL)
 951                 NW->IO.ErrMsg = NewStrBuf();
 952         StrBufPrintf(NW->IO.ErrMsg, "failed to connect %s \r\n", ChrPtr(NW->host));
 953
 954         return NWC_FailNetworkConnection(IO);
 955 }
 956 eNextState NWC_DNSFail(AsyncIO *IO)
 957 {
 958         AsyncNetworker *NW = IO->Data;
 959
 960         EVN_syslog(LOG_DEBUG, "%s\n", __FUNCTION__);
 961         if (NW->IO.ErrMsg == NULL)
 962                 NW->IO.ErrMsg = NewStrBuf();
 963         StrBufPrintf(NW->IO.ErrMsg, "failed to look up %s \r\n", ChrPtr(NW->host));
 964
 965         return NWC_FailNetworkConnection(IO);
 966 }
 967 eNextState NWC_Shutdown(AsyncIO *IO)
 968 {
 969         EVN_syslog(LOG_DEBUG, "%s\n", __FUNCTION__);
 970
 971         FinalizeNetworker(IO);
 972         return eAbort;
 973 }
 974
 975
 976 eNextState nwc_connect_ip(AsyncIO *IO)
 977 {
 978         AsyncNetworker *NW = IO->Data;
 979
 980         SetNWCState(&NW->IO, eNWCVSConnecting);
 981         EVN_syslog(LOG_DEBUG, "%s\n", __FUNCTION__);
 982         EVN_syslog(LOG_NOTICE, "Connecting to <%s> at %s:%s\n",
 983                    ChrPtr(NW->node),
 984                    ChrPtr(NW->host),
 985                    ChrPtr(NW->port));
 986
 987         return EvConnectSock(IO,
 988                              NWC_ConnTimeout,
 989                              NWC_ReadTimeouts[0],
 990                              1);
 991 }
 992
 993 static int NetworkerCount = 0;
 994 void RunNetworker(AsyncNetworker *NW)
 995 {
 996         NW->n = NetworkerCount++;
 997         CtdlNetworkTalkingTo(SKEY(NW->node), NTT_ADD);
 998         syslog(LOG_DEBUG, "NW[%s][%ld]: polling\n", ChrPtr(NW->node), NW->n);
 999         ParseURL(&NW->IO.ConnectMe, NW->Url, 504);
1000
1001         InitIOStruct(&NW->IO,
1002                      NW,
1003                      eReadMessage,
1004                      NWC_ReadServerStatus,
1005                      NWC_DNSFail,
1006                      NWC_DispatchWriteDone,
1007                      NWC_DispatchReadDone,
1008                      NWC_Terminate,
1009                      NWC_TerminateDB,
1010                      NWC_ConnFail,
1011                      NWC_Timeout,
1012                      NWC_Shutdown);
1013
1014         safestrncpy(((CitContext *)NW->IO.CitContext)->cs_host,
1015                     ChrPtr(NW->host),
1016                     sizeof(((CitContext *)NW->IO.CitContext)->cs_host));
1017
1018         if (NW->IO.ConnectMe->IsIP) {
1019                 SetNWCState(&NW->IO, eNWCVSLookup);
1020                 QueueEventContext(&NW->IO,
1021                                   nwc_connect_ip);
1022         }
1023         else { /* uneducated admin has chosen to add DNS to the equation... */
1024                 SetNWCState(&NW->IO, eNWCVSConnecting);
1025                 QueueEventContext(&NW->IO,
1026                                   nwc_get_one_host_ip);
1027         }
1028 }
1029
1030 /*
1031  * Poll other Citadel nodes and transfer inbound/outbound network data.
1032  * Set "full" to nonzero to force a poll of every node, or to zero to poll
1033  * only nodes to which we have data to send.
1034  */
1035 void network_poll_other_citadel_nodes(int full_poll, HashList *ignetcfg)
1036 {
1037         const char *key;
1038         long len;
1039         HashPos *Pos;
1040         void *vCfg;
1041         AsyncNetworker *NW;
1042         StrBuf *SpoolFileName;
1043
1044         int poll = 0;
1045
1046         if (GetCount(ignetcfg) ==0) {
1047                 syslog(LOG_DEBUG, "network: no neighbor nodes are configured - not polling.\n");
1048                 return;
1049         }
1050         become_session(&networker_client_CC);
1051
1052         SpoolFileName = NewStrBufPlain(ctdl_netout_dir, -1);
1053
1054         Pos = GetNewHashPos(ignetcfg, 0);
1055
1056         while (GetNextHashPos(ignetcfg, Pos, &len, &key, &vCfg))
1057         {
1058                 /* Use the string tokenizer to grab one line at a time */
1059                 if(server_shutting_down)
1060                         return;/* TODO free stuff*/
1061                 CtdlNodeConf *pNode = (CtdlNodeConf*) vCfg;
1062                 poll = 0;
1063                 NW = (AsyncNetworker*)malloc(sizeof(AsyncNetworker));
1064                 memset(NW, 0, sizeof(AsyncNetworker));
1065
1066                 NW->node = NewStrBufDup(pNode->NodeName);
1067                 NW->host = NewStrBufDup(pNode->Host);
1068                 NW->port = NewStrBufDup(pNode->Port);
1069                 NW->secret = NewStrBufDup(pNode->Secret);
1070
1071                 if ( (StrLength(NW->node) != 0) &&
1072                      (StrLength(NW->secret) != 0) &&
1073                      (StrLength(NW->host) != 0) &&
1074                      (StrLength(NW->port) != 0))
1075                 {
1076                         poll = full_poll;
1077                         if (poll == 0)
1078                         {
1079                                 StrBufAppendBufPlain(SpoolFileName, HKEY("/"), 0);
1080                                 StrBufAppendBuf(SpoolFileName, NW->node, 0);
1081                                 StrBufStripSlashes(SpoolFileName, 1);
1082
1083                                 if (access(ChrPtr(SpoolFileName), R_OK) == 0) {
1084                                         poll = 1;
1085                                 }
1086                         }
1087                 }
1088                 if (poll &&
1089                     (StrLength(NW->host) > 0) &&
1090                     strcmp("0.0.0.0", ChrPtr(NW->host)))
1091                 {
1092                         NW->Url = NewStrBuf();
1093                         StrBufPrintf(NW->Url, "citadel://%s@%s:%s",
1094                                      ChrPtr(NW->secret),
1095                                      ChrPtr(NW->host),
1096                                      ChrPtr(NW->port));
1097                         if (!CtdlNetworkTalkingTo(SKEY(NW->node), NTT_CHECK))
1098                         {
1099                                 RunNetworker(NW);
1100                                 continue;
1101                         }
1102                 }
1103                 DeleteNetworker(NW);
1104         }
1105         FreeStrBuf(&SpoolFileName);
1106         DeleteHashPos(&Pos);
1107 }
1108
1109
1110 void network_do_clientqueue(void)
1111 {
1112         HashList *working_ignetcfg;
1113         int full_processing = 1;
1114         static time_t last_run = 0L;
1115
1116         /*
1117          * Run the full set of processing tasks no more frequently
1118          * than once every n seconds
1119          */
1120         if ( (time(NULL) - last_run) < config.c_net_freq ) {
1121                 full_processing = 0;
1122                 syslog(LOG_DEBUG, "Network full processing in %ld seconds.\n",
1123                         config.c_net_freq - (time(NULL)- last_run)
1124                 );
1125         }
1126
1127         working_ignetcfg = CtdlLoadIgNetCfg();
1128         /*
1129          * Poll other Citadel nodes.  Maybe.  If "full_processing" is set
1130          * then we poll everyone.  Otherwise we only poll nodes we have stuff
1131          * to send to.
1132          */
1133         network_poll_other_citadel_nodes(full_processing, working_ignetcfg);
1134         DeleteHash(&working_ignetcfg);
1135 }
1136
1137 void LogDebugEnableNetworkClient(const int n)
1138 {
1139         NetworkClientDebugEnabled = n;
1140 }
1141 /*
1142  * Module entry point
1143  */
1144 CTDL_MODULE_INIT(network_client)
1145 {
1146         if (!threading)
1147         {
1148                 CtdlFillSystemContext(&networker_client_CC, "CitNetworker");
1149
1150                 CtdlRegisterSessionHook(network_do_clientqueue, EVT_TIMER, PRIO_SEND + 10);
1151                 CtdlRegisterDebugFlagHook(HKEY("networkclient"), LogDebugEnableNetworkClient, &NetworkClientDebugEnabled);
1152
1153         }
1154         return "networkclient";
1155 }