citadel/modules/network/serv_netspool.c

   1 /*
   2  * This module handles shared rooms, inter-Citadel mail, and outbound
   3  * mailing list processing.
   4  *
   5  * Copyright (c) 2000-2012 by the citadel.org team
   6  *
   7  *  This program is open source software; you can redistribute it and/or modify
   8  *  it under the terms of the GNU General Public License, version 3.
   9  *
  10  *  This program is distributed in the hope that it will be useful,
  11  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
  12  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  13  *  GNU General Public License for more details.
  14  *
  15  * ** NOTE **   A word on the S_NETCONFIGS semaphore:
  16  * This is a fairly high-level type of critical section.  It ensures that no
  17  * two threads work on the netconfigs files at the same time.  Since we do
  18  * so many things inside these, here are the rules:
  19  *  1. begin_critical_section(S_NETCONFIGS) *before* begin_ any others.
  20  *  2. Do *not* perform any I/O with the client during these sections.
  21  *
  22  */
  23
  24 /*
  25  * Duration of time (in seconds) after which pending list subscribe/unsubscribe
  26  * requests that have not been confirmed will be deleted.
  27  */
  28 #define EXP     259200  /* three days */
  29
  30 #include "sysdep.h"
  31 #include <stdlib.h>
  32 #include <unistd.h>
  33 #include <stdio.h>
  34 #include <fcntl.h>
  35 #include <ctype.h>
  36 #include <signal.h>
  37 #include <pwd.h>
  38 #include <errno.h>
  39 #include <sys/stat.h>
  40 #include <sys/types.h>
  41 #include <dirent.h>
  42 #if TIME_WITH_SYS_TIME
  43 # include <sys/time.h>
  44 # include <time.h>
  45 #else
  46 # if HAVE_SYS_TIME_H
  47 #  include <sys/time.h>
  48 # else
  49 #  include <time.h>
  50 # endif
  51 #endif
  52 #ifdef HAVE_SYSCALL_H
  53 # include <syscall.h>
  54 #else
  55 # if HAVE_SYS_SYSCALL_H
  56 #  include <sys/syscall.h>
  57 # endif
  58 #endif
  59
  60 #include <sys/wait.h>
  61 #include <string.h>
  62 #include <limits.h>
  63 #include <libcitadel.h>
  64 #include "citadel.h"
  65 #include "server.h"
  66 #include "citserver.h"
  67 #include "support.h"
  68 #include "config.h"
  69 #include "user_ops.h"
  70 #include "database.h"
  71 #include "msgbase.h"
  72 #include "internet_addressing.h"
  73 #include "serv_network.h"
  74 #include "clientsocket.h"
  75 #include "file_ops.h"
  76 #include "citadel_dirs.h"
  77 #include "threads.h"
  78 #include "context.h"
  79
  80 #include "ctdl_module.h"
  81
  82 #include "netspool.h"
  83 #include "netmail.h"
  84
  85
  86 #ifndef DT_UNKNOWN
  87 #define DT_UNKNOWN     0
  88 #define DT_DIR         4
  89 #define DT_REG         8
  90 #define DT_LNK         10
  91
  92 #define IFTODT(mode)   (((mode) & 0170000) >> 12)
  93 #define DTTOIF(dirtype)        ((dirtype) << 12)
  94 #endif
  95
  96
  97 void ParseLastSent(const CfgLineType *ThisOne, StrBuf *Line, const char *LinePos, OneRoomNetCfg *OneRNCFG)
  98 {
  99         RoomNetCfgLine *nptr;
 100         nptr = (RoomNetCfgLine *)
 101                 malloc(sizeof(RoomNetCfgLine));
 102         memset(nptr, 0, sizeof(RoomNetCfgLine));
 103         OneRNCFG->lastsent = extract_long(LinePos, 0);
 104         OneRNCFG->NetConfigs[ThisOne->C] = nptr;
 105 }
 106
 107 void ParseRoomAlias(const CfgLineType *ThisOne, StrBuf *Line, const char *LinePos, OneRoomNetCfg *rncfg)
 108 {
 109 /*
 110         if (rncfg->RNCfg->sender != NULL)
 111                 continue; / * just one alowed... * /
 112         extract_token(nptr->name, buf, 1, '|', sizeof nptr->name);
 113         rncfg->RNCfg->sender = nptr;
 114 */
 115 }
 116
 117 void ParseSubPendingLine(const CfgLineType *ThisOne, StrBuf *Line, const char *LinePos, OneRoomNetCfg *OneRNCFG)
 118 {
 119         if (time(NULL) - extract_long(LinePos, 3) > EXP)
 120                 return; /* expired subscription... */
 121
 122         ParseGeneric(ThisOne, Line, LinePos, OneRNCFG);
 123 }
 124 void ParseUnSubPendingLine(const CfgLineType *ThisOne, StrBuf *Line, const char *LinePos, OneRoomNetCfg *OneRNCFG)
 125 {
 126         if (time(NULL) - extract_long(LinePos, 2) > EXP)
 127                 return; /* expired subscription... */
 128
 129         ParseGeneric(ThisOne, Line, LinePos, OneRNCFG);
 130 }
 131
 132
 133 void SerializeLastSent(const CfgLineType *ThisOne, StrBuf *OutputBuffer, OneRoomNetCfg *RNCfg, RoomNetCfgLine *data)
 134 {
 135         StrBufAppendBufPlain(OutputBuffer, CKEY(ThisOne->Str), 0);
 136         StrBufAppendPrintf(OutputBuffer, "|%ld\n", RNCfg->lastsent);
 137 }
 138
 139 void DeleteLastSent(const CfgLineType *ThisOne, RoomNetCfgLine **data)
 140 {
 141         free(*data);
 142         *data = NULL;
 143 }
 144
 145 static const RoomNetCfg SpoolCfgs [4] = {
 146         listrecp,
 147         digestrecp,
 148         participate,
 149         ignet_push_share
 150 };
 151
 152 static const long SpoolCfgsCopyN [4] = {
 153         1, 1, 1, 2
 154 };
 155
 156 int HaveSpoolConfig(OneRoomNetCfg* RNCfg)
 157 {
 158         int i;
 159         int interested = 0;
 160         for (i=0; i < 4; i++) if (RNCfg->NetConfigs[SpoolCfgs[i]] == NULL) interested = 1;
 161         return interested;
 162 }
 163
 164
 165
 166 void InspectQueuedRoom(SpoolControl **pSC,
 167                        RoomProcList *room_to_spool,
 168                        HashList *working_ignetcfg,
 169                        HashList *the_netmap)
 170 {
 171         SpoolControl *sc;
 172         int i = 0;
 173
 174         sc = (SpoolControl*)malloc(sizeof(SpoolControl));
 175         memset(sc, 0, sizeof(SpoolControl));
 176         sc->RNCfg = room_to_spool->OneRNCfg;
 177         sc->lastsent = room_to_spool->lastsent;
 178         sc->working_ignetcfg = working_ignetcfg;
 179         sc->the_netmap = the_netmap;
 180
 181         /*
 182          * If the room doesn't exist, don't try to perform its networking tasks.
 183          * Normally this should never happen, but once in a while maybe a room gets
 184          * queued for networking and then deleted before it can happen.
 185          */
 186         if (CtdlGetRoom(&sc->room, room_to_spool->name) != 0) {
 187                 syslog(LOG_CRIT, "ERROR: cannot load <%s>\n", room_to_spool->name);
 188                 free(sc);
 189                 return;
 190         }
 191         if (sc->room.QRhighest <= sc->lastsent)
 192         {
 193                 syslog(LOG_DEBUG, "nothing to do for <%s>\n", room_to_spool->name);
 194                 free(sc);
 195                 return;
 196         }
 197
 198         if (sc->RNCfg == NULL)
 199                 sc->RNCfg = CtdlGetNetCfgForRoom(sc->room.QRnumber);
 200
 201         if (!HaveSpoolConfig(sc->RNCfg))
 202         {
 203                 free(sc);
 204                 /* nothing to do for this room... */
 205                 return;
 206         }
 207
 208         /* Now lets remember whats needed for the actual work... */
 209
 210         for (i=0; i < 4; i++)
 211         {
 212                 aggregate_recipients(&sc->Users[SpoolCfgs[i]],
 213                                      SpoolCfgs[i],
 214                                      sc->RNCfg,
 215                                      SpoolCfgsCopyN[i]);
 216         }
 217
 218         if (StrLength(sc->RNCfg->Sender) > 0)
 219                 sc->Users[roommailalias] = NewStrBufDup(sc->RNCfg->Sender);
 220
 221         sc->next = *pSC;
 222         *pSC = sc;
 223
 224 }
 225
 226 void CalcListID(SpoolControl *sc)
 227 {
 228         const char *err;
 229         int fd;
 230         struct CitContext *CCC = CC;
 231         char filename[PATH_MAX];
 232 #define MAX_LISTIDLENGTH 150
 233
 234         assoc_file_name(filename, sizeof filename, &sc->room, ctdl_info_dir);
 235         fd = open(filename, 0);
 236
 237         if (fd != 0) {
 238                 struct stat stbuf;
 239
 240                 if ((fstat(fd, &stbuf) == 0) &&
 241                     (stbuf.st_size > 0))
 242                 {
 243                         sc->RoomInfo = NewStrBufPlain(NULL, stbuf.st_size + 1);
 244                         StrBufReadBLOB(sc->RoomInfo, &fd, 0, stbuf.st_size, &err);
 245                 }
 246                 close(fd);
 247         }
 248
 249         sc->ListID = NewStrBufPlain(NULL, 1024);
 250         if (StrLength(sc->RoomInfo) > 0)
 251         {
 252                 const char *Pos = NULL;
 253                 StrBufSipLine(sc->ListID, sc->RoomInfo, &Pos);
 254
 255                 if (StrLength(sc->ListID) > MAX_LISTIDLENGTH)
 256                 {
 257                         StrBufCutAt(sc->ListID, MAX_LISTIDLENGTH, NULL);
 258                         StrBufAppendBufPlain(sc->ListID, HKEY("..."), 0);
 259                 }
 260                 StrBufAsciify(sc->ListID, ' ');
 261         }
 262         else
 263         {
 264                 StrBufAppendBufPlain(sc->ListID, CCC->room.QRname, -1, 0);
 265         }
 266
 267         StrBufAppendBufPlain(sc->ListID, HKEY("<"), 0);
 268
 269         if (StrLength(sc->Users[roommailalias]) > 0)
 270         {
 271                 long Pos;
 272                 const char *AtPos;
 273
 274                 Pos = StrLength(sc->ListID);
 275                 StrBufAppendBuf(sc->ListID, sc->Users[roommailalias], 0);
 276                 AtPos = strchr(ChrPtr(sc->ListID) + Pos, '@');
 277
 278                 if (AtPos != NULL)
 279                 {
 280                         StrBufPeek(sc->ListID, AtPos, 0, '.');
 281                 }
 282         }
 283         else
 284         {
 285                 StrBufAppendBufPlain(sc->ListID, HKEY("room_"), 0);
 286                 StrBufAppendBufPlain(sc->ListID, sc->room.QRname, -1, 0);
 287                 StrBufAppendBufPlain(sc->ListID, HKEY("."), 0);
 288                 StrBufAppendBufPlain(sc->ListID, config.c_fqdn, -1, 0);
 289                 /*
 290                  * this used to be:
 291                  * roomname <Room-Number.list-id.fqdn>
 292                  * according to rfc2919.txt it only has to be a uniq identifier
 293                  * under the domain of the system;
 294                  * in general MUAs use it to calculate the reply address nowadays.
 295                  */
 296         }
 297         StrBufAppendBufPlain(sc->ListID, HKEY(">"), 0);
 298
 299         if (StrLength(sc->Users[roommailalias]) == 0)
 300         {
 301                 sc->Users[roommailalias] = NewStrBuf();
 302
 303                 StrBufPrintf(sc->Users[roommailalias],
 304                              "room_%s@%s",
 305                              CCC->room.QRname,
 306                              config.c_fqdn);
 307
 308                 StrBufAsciify(sc->Users[roommailalias], '_');
 309                 StrBufLowerCase(sc->Users[roommailalias]);
 310         }
 311
 312 }
 313
 314
 315 /*
 316  * Batch up and send all outbound traffic from the current room
 317  */
 318 void network_spoolout_room(SpoolControl *sc)
 319 {
 320         struct CitContext *CCC = CC;
 321         char buf[SIZ];
 322         int i;
 323         long lastsent;
 324
 325         /*
 326          * If the room doesn't exist, don't try to perform its networking tasks.
 327          * Normally this should never happen, but once in a while maybe a room gets
 328          * queued for networking and then deleted before it can happen.
 329          */
 330         memcpy (&CCC->room, &sc->room, sizeof(ctdlroom));
 331
 332         syslog(LOG_INFO, "Networking started for <%s>\n", CCC->room.QRname);
 333
 334         /* If there are digest recipients, we have to build a digest */
 335         if (sc->Users[digestrecp] != NULL) {
 336                 sc->digestfp = tmpfile();
 337                 fprintf(sc->digestfp, "Content-type: text/plain\n\n");
 338         }
 339
 340         CalcListID(sc);
 341
 342         /* remember where we started... */
 343         lastsent = sc->lastsent;
 344
 345         /* Do something useful */
 346         CtdlForEachMessage(MSGS_GT, sc->lastsent, NULL, NULL, NULL,
 347                 network_spool_msg, sc);
 348
 349         /* If we wrote a digest, deliver it and then close it */
 350         if (StrLength(sc->Users[roommailalias]) > 0)
 351         {
 352                 long len;
 353                 len = StrLength(sc->Users[roommailalias]);
 354                 if (len + 1 > sizeof(buf))
 355                         len = sizeof(buf) - 1;
 356                 memcpy(buf, ChrPtr(sc->Users[roommailalias]), len);
 357                 buf[len] = '\0';
 358         }
 359         else
 360         {
 361                 snprintf(buf, sizeof buf, "room_%s@%s",
 362                          CCC->room.QRname, config.c_fqdn);
 363         }
 364
 365         for (i=0; buf[i]; ++i) {
 366                 buf[i] = tolower(buf[i]);
 367                 if (isspace(buf[i])) buf[i] = '_';
 368         }
 369         if (sc->digestfp != NULL) {
 370                 fprintf(sc->digestfp,
 371                         " -----------------------------------"
 372                         "------------------------------------"
 373                         "-------\n"
 374                         "You are subscribed to the '%s' "
 375                         "list.\n"
 376                         "To post to the list: %s\n",
 377                         CCC->room.QRname, buf
 378                 );
 379                 network_deliver_digest(sc);     /* deliver and close */
 380         }
 381
 382         /* Now rewrite the config file */
 383         if (sc->lastsent != lastsent)
 384         {
 385                 sc->RNCfg = CtdlGetNetCfgForRoom(sc->room.QRnumber);
 386
 387                 sc->RNCfg->lastsent = sc->lastsent;
 388                 sc->RNCfg->changed = 1;
 389         }
 390         end_critical_section(S_NETCONFIGS);
 391 }
 392
 393 /*
 394  * Process a buffer containing a single message from a single file
 395  * from the inbound queue
 396  */
 397 void network_process_buffer(char *buffer, long size, HashList *working_ignetcfg, HashList *the_netmap, int *netmap_changed)
 398 {
 399         struct CitContext *CCC = CC;
 400         StrBuf *Buf = NULL;
 401         struct CtdlMessage *msg = NULL;
 402         long pos;
 403         int field;
 404         struct recptypes *recp = NULL;
 405         char target_room[ROOMNAMELEN];
 406         struct ser_ret sermsg;
 407         char *oldpath = NULL;
 408         char filename[PATH_MAX];
 409         FILE *fp;
 410         const StrBuf *nexthop = NULL;
 411         unsigned char firstbyte;
 412         unsigned char lastbyte;
 413
 414         QN_syslog(LOG_DEBUG, "network_process_buffer() processing %ld bytes\n", size);
 415
 416         /* Validate just a little bit.  First byte should be FF and * last byte should be 00. */
 417         firstbyte = buffer[0];
 418         lastbyte = buffer[size-1];
 419         if ( (firstbyte != 255) || (lastbyte != 0) ) {
 420                 QN_syslog(LOG_ERR, "Corrupt message ignored.  Length=%ld, firstbyte = %d, lastbyte = %d\n",
 421                           size, firstbyte, lastbyte);
 422                 return;
 423         }
 424
 425         /* Set default target room to trash */
 426         strcpy(target_room, TWITROOM);
 427
 428         /* Load the message into memory */
 429         msg = (struct CtdlMessage *) malloc(sizeof(struct CtdlMessage));
 430         memset(msg, 0, sizeof(struct CtdlMessage));
 431         msg->cm_magic = CTDLMESSAGE_MAGIC;
 432         msg->cm_anon_type = buffer[1];
 433         msg->cm_format_type = buffer[2];
 434
 435         for (pos = 3; pos < size; ++pos) {
 436                 field = buffer[pos];
 437                 msg->cm_fields[field] = strdup(&buffer[pos+1]);
 438                 pos = pos + strlen(&buffer[(int)pos]);
 439         }
 440
 441         /* Check for message routing */
 442         if (msg->cm_fields['D'] != NULL) {
 443                 if (strcasecmp(msg->cm_fields['D'], config.c_nodename)) {
 444
 445                         /* route the message */
 446                         Buf = NewStrBufPlain(msg->cm_fields['D'], -1);
 447                         if (CtdlIsValidNode(&nexthop,
 448                                             NULL,
 449                                             Buf,
 450                                             working_ignetcfg,
 451                                             the_netmap) == 0)
 452                         {
 453                                 /* prepend our node to the path */
 454                                 if (msg->cm_fields['P'] != NULL) {
 455                                         oldpath = msg->cm_fields['P'];
 456                                         msg->cm_fields['P'] = NULL;
 457                                 }
 458                                 else {
 459                                         oldpath = strdup("unknown_user");
 460                                 }
 461                                 size = strlen(oldpath) + SIZ;
 462                                 msg->cm_fields['P'] = malloc(size);
 463                                 snprintf(msg->cm_fields['P'], size, "%s!%s",
 464                                         config.c_nodename, oldpath);
 465                                 free(oldpath);
 466
 467                                 /* serialize the message */
 468                                 serialize_message(&sermsg, msg);
 469
 470                                 /* now send it */
 471                                 if (StrLength(nexthop) == 0) {
 472                                         nexthop = Buf;
 473                                 }
 474                                 snprintf(filename,
 475                                          sizeof filename,
 476                                          "%s/%s@%lx%x",
 477                                          ctdl_netout_dir,
 478                                          ChrPtr(nexthop),
 479                                          time(NULL),
 480                                          rand()
 481                                 );
 482                                 QN_syslog(LOG_DEBUG, "Appending to %s\n", filename);
 483                                 fp = fopen(filename, "ab");
 484                                 if (fp != NULL) {
 485                                         fwrite(sermsg.ser, sermsg.len, 1, fp);
 486                                         fclose(fp);
 487                                 }
 488                                 else {
 489                                         QN_syslog(LOG_ERR, "%s: %s\n", filename, strerror(errno));
 490                                 }
 491                                 free(sermsg.ser);
 492                                 CtdlFreeMessage(msg);
 493                                 FreeStrBuf(&Buf);
 494                                 return;
 495                         }
 496
 497                         else {  /* invalid destination node name */
 498                                 FreeStrBuf(&Buf);
 499
 500                                 network_bounce(msg,
 501 "A message you sent could not be delivered due to an invalid destination node"
 502 " name.  Please check the address and try sending the message again.\n");
 503                                 msg = NULL;
 504                                 return;
 505
 506                         }
 507                 }
 508         }
 509
 510         /*
 511          * Check to see if we already have a copy of this message, and
 512          * abort its processing if so.  (We used to post a warning to Aide>
 513          * every time this happened, but the network is now so densely
 514          * connected that it's inevitable.)
 515          */
 516         if (network_usetable(msg) != 0) {
 517                 CtdlFreeMessage(msg);
 518                 return;
 519         }
 520
 521         /* Learn network topology from the path */
 522         if ((msg->cm_fields['N'] != NULL) && (msg->cm_fields['P'] != NULL)) {
 523                 NetworkLearnTopology(msg->cm_fields['N'],
 524                                      msg->cm_fields['P'],
 525                                      the_netmap,
 526                                      netmap_changed);
 527         }
 528
 529         /* Is the sending node giving us a very persuasive suggestion about
 530          * which room this message should be saved in?  If so, go with that.
 531          */
 532         if (msg->cm_fields['C'] != NULL) {
 533                 safestrncpy(target_room, msg->cm_fields['C'], sizeof target_room);
 534         }
 535
 536         /* Otherwise, does it have a recipient?  If so, validate it... */
 537         else if (msg->cm_fields['R'] != NULL) {
 538                 recp = validate_recipients(msg->cm_fields['R'], NULL, 0);
 539                 if (recp != NULL) if (recp->num_error != 0) {
 540                         network_bounce(msg,
 541                                 "A message you sent could not be delivered due to an invalid address.\n"
 542                                 "Please check the address and try sending the message again.\n");
 543                         msg = NULL;
 544                         free_recipients(recp);
 545                         QNM_syslog(LOG_DEBUG, "Bouncing message due to invalid recipient address.\n");
 546                         return;
 547                 }
 548                 strcpy(target_room, "");        /* no target room if mail */
 549         }
 550
 551         /* Our last shot at finding a home for this message is to see if
 552          * it has the O field (Originating room) set.
 553          */
 554         else if (msg->cm_fields['O'] != NULL) {
 555                 safestrncpy(target_room, msg->cm_fields['O'], sizeof target_room);
 556         }
 557
 558         /* Strip out fields that are only relevant during transit */
 559         if (msg->cm_fields['D'] != NULL) {
 560                 free(msg->cm_fields['D']);
 561                 msg->cm_fields['D'] = NULL;
 562         }
 563         if (msg->cm_fields['C'] != NULL) {
 564                 free(msg->cm_fields['C']);
 565                 msg->cm_fields['C'] = NULL;
 566         }
 567
 568         /* save the message into a room */
 569         if (PerformNetprocHooks(msg, target_room) == 0) {
 570                 msg->cm_flags = CM_SKIP_HOOKS;
 571                 CtdlSubmitMsg(msg, recp, target_room, 0);
 572         }
 573         CtdlFreeMessage(msg);
 574         free_recipients(recp);
 575 }
 576
 577
 578 /*
 579  * Process a single message from a single file from the inbound queue
 580  */
 581 void network_process_message(FILE *fp,
 582                              long msgstart,
 583                              long msgend,
 584                              HashList *working_ignetcfg,
 585                              HashList *the_netmap,
 586                              int *netmap_changed)
 587 {
 588         long hold_pos;
 589         long size;
 590         char *buffer;
 591
 592         hold_pos = ftell(fp);
 593         size = msgend - msgstart + 1;
 594         buffer = malloc(size);
 595         if (buffer != NULL) {
 596                 fseek(fp, msgstart, SEEK_SET);
 597                 if (fread(buffer, size, 1, fp) > 0) {
 598                         network_process_buffer(buffer,
 599                                                size,
 600                                                working_ignetcfg,
 601                                                the_netmap,
 602                                                netmap_changed);
 603                 }
 604                 free(buffer);
 605         }
 606
 607         fseek(fp, hold_pos, SEEK_SET);
 608 }
 609
 610
 611 /*
 612  * Process a single file from the inbound queue
 613  */
 614 void network_process_file(char *filename,
 615                           HashList *working_ignetcfg,
 616                           HashList *the_netmap,
 617                           int *netmap_changed)
 618 {
 619         struct CitContext *CCC = CC;
 620         FILE *fp;
 621         long msgstart = (-1L);
 622         long msgend = (-1L);
 623         long msgcur = 0L;
 624         int ch;
 625
 626
 627         fp = fopen(filename, "rb");
 628         if (fp == NULL) {
 629                 QN_syslog(LOG_CRIT, "Error opening %s: %s\n", filename, strerror(errno));
 630                 return;
 631         }
 632
 633         fseek(fp, 0L, SEEK_END);
 634         QN_syslog(LOG_INFO, "network: processing %ld bytes from %s\n", ftell(fp), filename);
 635         rewind(fp);
 636
 637         /* Look for messages in the data stream and break them out */
 638         while (ch = getc(fp), ch >= 0) {
 639
 640                 if (ch == 255) {
 641                         if (msgstart >= 0L) {
 642                                 msgend = msgcur - 1;
 643                                 network_process_message(fp,
 644                                                         msgstart,
 645                                                         msgend,
 646                                                         working_ignetcfg,
 647                                                         the_netmap,
 648                                                         netmap_changed);
 649                         }
 650                         msgstart = msgcur;
 651                 }
 652
 653                 ++msgcur;
 654         }
 655
 656         msgend = msgcur - 1;
 657         if (msgstart >= 0L) {
 658                 network_process_message(fp,
 659                                         msgstart,
 660                                         msgend,
 661                                         working_ignetcfg,
 662                                         the_netmap,
 663                                         netmap_changed);
 664         }
 665
 666         fclose(fp);
 667         unlink(filename);
 668 }
 669
 670
 671 /*
 672  * Process anything in the inbound queue
 673  */
 674 void network_do_spoolin(HashList *working_ignetcfg, HashList *the_netmap, int *netmap_changed)
 675 {
 676         struct CitContext *CCC = CC;
 677         DIR *dp;
 678         struct dirent *d;
 679         struct dirent *filedir_entry;
 680         struct stat statbuf;
 681         char filename[PATH_MAX];
 682         static time_t last_spoolin_mtime = 0L;
 683         int d_type = 0;
 684         int d_namelen;
 685
 686         /*
 687          * Check the spoolin directory's modification time.  If it hasn't
 688          * been touched, we don't need to scan it.
 689          */
 690         if (stat(ctdl_netin_dir, &statbuf)) return;
 691         if (statbuf.st_mtime == last_spoolin_mtime) {
 692                 QNM_syslog(LOG_DEBUG, "network: nothing in inbound queue\n");
 693                 return;
 694         }
 695         last_spoolin_mtime = statbuf.st_mtime;
 696         QNM_syslog(LOG_DEBUG, "network: processing inbound queue\n");
 697
 698         /*
 699          * Ok, there's something interesting in there, so scan it.
 700          */
 701         dp = opendir(ctdl_netin_dir);
 702         if (dp == NULL) return;
 703
 704         d = (struct dirent *)malloc(offsetof(struct dirent, d_name) + PATH_MAX + 1);
 705         if (d == NULL) {
 706                 closedir(dp);
 707                 return;
 708         }
 709
 710         while ((readdir_r(dp, d, &filedir_entry) == 0) &&
 711                (filedir_entry != NULL))
 712         {
 713 #ifdef _DIRENT_HAVE_D_NAMLEN
 714                 d_namelen = filedir_entry->d_namelen;
 715
 716 #else
 717                 d_namelen = strlen(filedir_entry->d_name);
 718 #endif
 719
 720 #ifdef _DIRENT_HAVE_D_TYPE
 721                 d_type = filedir_entry->d_type;
 722 #else
 723                 d_type = DT_UNKNOWN;
 724 #endif
 725                 if ((d_namelen > 1) && filedir_entry->d_name[d_namelen - 1] == '~')
 726                         continue; /* Ignore backup files... */
 727
 728                 if ((d_namelen == 1) &&
 729                     (filedir_entry->d_name[0] == '.'))
 730                         continue;
 731
 732                 if ((d_namelen == 2) &&
 733                     (filedir_entry->d_name[0] == '.') &&
 734                     (filedir_entry->d_name[1] == '.'))
 735                         continue;
 736
 737                 if (d_type == DT_UNKNOWN) {
 738                         struct stat s;
 739                         char path[PATH_MAX];
 740
 741                         snprintf(path,
 742                                  PATH_MAX,
 743                                  "%s/%s",
 744                                  ctdl_netin_dir,
 745                                  filedir_entry->d_name);
 746
 747                         if (lstat(path, &s) == 0) {
 748                                 d_type = IFTODT(s.st_mode);
 749                         }
 750                 }
 751
 752                 switch (d_type)
 753                 {
 754                 case DT_DIR:
 755                         break;
 756                 case DT_LNK: /* TODO: check whether its a file or a directory */
 757                 case DT_REG:
 758                         snprintf(filename,
 759                                 sizeof filename,
 760                                 "%s/%s",
 761                                 ctdl_netin_dir,
 762                                 d->d_name
 763                         );
 764                         network_process_file(filename,
 765                                              working_ignetcfg,
 766                                              the_netmap,
 767                                              netmap_changed);
 768                 }
 769         }
 770
 771         closedir(dp);
 772         free(d);
 773 }
 774
 775 /*
 776  * Step 1: consolidate files in the outbound queue into one file per neighbor node
 777  * Step 2: delete any files in the outbound queue that were for neighbors who no longer exist.
 778  */
 779 void network_consolidate_spoolout(HashList *working_ignetcfg, HashList *the_netmap)
 780 {
 781         struct CitContext *CCC = CC;
 782         IOBuffer IOB;
 783         FDIOBuffer FDIO;
 784         int d_namelen;
 785         DIR *dp;
 786         struct dirent *d;
 787         struct dirent *filedir_entry;
 788         const char *pch;
 789         char spooloutfilename[PATH_MAX];
 790         char filename[PATH_MAX];
 791         const StrBuf *nexthop;
 792         StrBuf *NextHop;
 793         int i;
 794         struct stat statbuf;
 795         int nFailed = 0;
 796         int d_type = 0;
 797
 798
 799         /* Step 1: consolidate files in the outbound queue into one file per neighbor node */
 800         d = (struct dirent *)malloc(offsetof(struct dirent, d_name) + PATH_MAX + 1);
 801         if (d == NULL)  return;
 802
 803         dp = opendir(ctdl_netout_dir);
 804         if (dp == NULL) {
 805                 free(d);
 806                 return;
 807         }
 808
 809         NextHop = NewStrBuf();
 810         memset(&IOB, 0, sizeof(IOBuffer));
 811         memset(&FDIO, 0, sizeof(FDIOBuffer));
 812         FDIO.IOB = &IOB;
 813
 814         while ((readdir_r(dp, d, &filedir_entry) == 0) &&
 815                (filedir_entry != NULL))
 816         {
 817 #ifdef _DIRENT_HAVE_D_NAMLEN
 818                 d_namelen = filedir_entry->d_namelen;
 819
 820 #else
 821                 d_namelen = strlen(filedir_entry->d_name);
 822 #endif
 823
 824 #ifdef _DIRENT_HAVE_D_TYPE
 825                 d_type = filedir_entry->d_type;
 826 #else
 827                 d_type = DT_UNKNOWN;
 828 #endif
 829                 if (d_type == DT_DIR)
 830                         continue;
 831
 832                 if ((d_namelen > 1) && filedir_entry->d_name[d_namelen - 1] == '~')
 833                         continue; /* Ignore backup files... */
 834
 835                 if ((d_namelen == 1) &&
 836                     (filedir_entry->d_name[0] == '.'))
 837                         continue;
 838
 839                 if ((d_namelen == 2) &&
 840                     (filedir_entry->d_name[0] == '.') &&
 841                     (filedir_entry->d_name[1] == '.'))
 842                         continue;
 843
 844                 pch = strchr(filedir_entry->d_name, '@');
 845                 if (pch == NULL)
 846                         continue;
 847
 848                 snprintf(filename,
 849                          sizeof filename,
 850                          "%s/%s",
 851                          ctdl_netout_dir,
 852                          filedir_entry->d_name);
 853
 854                 StrBufPlain(NextHop,
 855                             filedir_entry->d_name,
 856                             pch - filedir_entry->d_name);
 857
 858                 snprintf(spooloutfilename,
 859                          sizeof spooloutfilename,
 860                          "%s/%s",
 861                          ctdl_netout_dir,
 862                          ChrPtr(NextHop));
 863
 864                 QN_syslog(LOG_DEBUG, "Consolidate %s to %s\n", filename, ChrPtr(NextHop));
 865                 if (CtdlNetworkTalkingTo(SKEY(NextHop), NTT_CHECK)) {
 866                         nFailed++;
 867                         QN_syslog(LOG_DEBUG,
 868                                   "Currently online with %s - skipping for now\n",
 869                                   ChrPtr(NextHop)
 870                                 );
 871                 }
 872                 else {
 873                         size_t dsize;
 874                         size_t fsize;
 875                         int infd, outfd;
 876                         const char *err = NULL;
 877                         CtdlNetworkTalkingTo(SKEY(NextHop), NTT_ADD);
 878
 879                         infd = open(filename, O_RDONLY);
 880                         if (infd == -1) {
 881                                 nFailed++;
 882                                 QN_syslog(LOG_ERR,
 883                                           "failed to open %s for reading due to %s; skipping.\n",
 884                                           filename, strerror(errno)
 885                                         );
 886                                 CtdlNetworkTalkingTo(SKEY(NextHop), NTT_REMOVE);
 887                                 continue;
 888                         }
 889
 890                         outfd = open(spooloutfilename,
 891                                   O_EXCL|O_CREAT|O_NONBLOCK|O_WRONLY,
 892                                   S_IRUSR|S_IWUSR);
 893                         if (outfd == -1)
 894                         {
 895                                 outfd = open(spooloutfilename,
 896                                              O_EXCL|O_NONBLOCK|O_WRONLY,
 897                                              S_IRUSR | S_IWUSR);
 898                         }
 899                         if (outfd == -1) {
 900                                 nFailed++;
 901                                 QN_syslog(LOG_ERR,
 902                                           "failed to open %s for reading due to %s; skipping.\n",
 903                                           spooloutfilename, strerror(errno)
 904                                         );
 905                                 close(infd);
 906                                 CtdlNetworkTalkingTo(SKEY(NextHop), NTT_REMOVE);
 907                                 continue;
 908                         }
 909
 910                         dsize = lseek(outfd, 0, SEEK_END);
 911                         lseek(outfd, -dsize, SEEK_SET);
 912
 913                         fstat(infd, &statbuf);
 914                         fsize = statbuf.st_size;
 915 /*
 916                         fsize = lseek(infd, 0, SEEK_END);
 917 */
 918                         IOB.fd = infd;
 919                         FDIOBufferInit(&FDIO, &IOB, outfd, fsize + dsize);
 920                         FDIO.ChunkSendRemain = fsize;
 921                         FDIO.TotalSentAlready = dsize;
 922                         err = NULL;
 923                         errno = 0;
 924                         do {} while ((FileMoveChunked(&FDIO, &err) > 0) && (err == NULL));
 925                         if (err == NULL) {
 926                                 unlink(filename);
 927                         }
 928                         else {
 929                                 nFailed++;
 930                                 QN_syslog(LOG_ERR,
 931                                           "failed to append to %s [%s]; rolling back..\n",
 932                                           spooloutfilename, strerror(errno)
 933                                         );
 934                                 /* whoops partial append?? truncate spooloutfilename again! */
 935                                 ftruncate(outfd, dsize);
 936                         }
 937                         FDIOBufferDelete(&FDIO);
 938                         close(infd);
 939                         close(outfd);
 940                         CtdlNetworkTalkingTo(SKEY(NextHop), NTT_REMOVE);
 941                 }
 942         }
 943         closedir(dp);
 944
 945         if (nFailed > 0) {
 946                 FreeStrBuf(&NextHop);
 947                 QN_syslog(LOG_INFO,
 948                           "skipping Spoolcleanup because of %d files unprocessed.\n",
 949                           nFailed
 950                         );
 951
 952                 return;
 953         }
 954
 955         /* Step 2: delete any files in the outbound queue that were for neighbors who no longer exist */
 956         dp = opendir(ctdl_netout_dir);
 957         if (dp == NULL) {
 958                 FreeStrBuf(&NextHop);
 959                 free(d);
 960                 return;
 961         }
 962
 963         while ((readdir_r(dp, d, &filedir_entry) == 0) &&
 964                (filedir_entry != NULL))
 965         {
 966 #ifdef _DIRENT_HAVE_D_NAMLEN
 967                 d_namelen = filedir_entry->d_namelen;
 968
 969 #else
 970                 d_namelen = strlen(filedir_entry->d_name);
 971 #endif
 972
 973 #ifdef _DIRENT_HAVE_D_TYPE
 974                 d_type = filedir_entry->d_type;
 975 #else
 976                 d_type = DT_UNKNOWN;
 977 #endif
 978                 if (d_type == DT_DIR)
 979                         continue;
 980
 981                 if ((d_namelen == 1) &&
 982                     (filedir_entry->d_name[0] == '.'))
 983                         continue;
 984
 985                 if ((d_namelen == 2) &&
 986                     (filedir_entry->d_name[0] == '.') &&
 987                     (filedir_entry->d_name[1] == '.'))
 988                         continue;
 989
 990                 pch = strchr(filedir_entry->d_name, '@');
 991                 if (pch == NULL) /* no @ in name? consolidated file. */
 992                         continue;
 993
 994                 StrBufPlain(NextHop,
 995                             filedir_entry->d_name,
 996                             pch - filedir_entry->d_name);
 997
 998                 snprintf(filename,
 999                         sizeof filename,
1000                         "%s/%s",
1001                         ctdl_netout_dir,
1002                         filedir_entry->d_name
1003                 );
1004
1005                 i = CtdlIsValidNode(&nexthop,
1006                                     NULL,
1007                                     NextHop,
1008                                     working_ignetcfg,
1009                                     the_netmap);
1010
1011                 if ( (i != 0) || (StrLength(nexthop) > 0) ) {
1012                         unlink(filename);
1013                 }
1014         }
1015         FreeStrBuf(&NextHop);
1016         free(d);
1017         closedir(dp);
1018 }
1019
1020 void free_spoolcontrol_struct(SpoolControl **sc)
1021 {
1022         free_spoolcontrol_struct_members(*sc);
1023         free(*sc);
1024         *sc = NULL;
1025 }
1026
1027 void free_spoolcontrol_struct_members(SpoolControl *sc)
1028 {
1029         int i;
1030         FreeStrBuf(&sc->RoomInfo);
1031         FreeStrBuf(&sc->ListID);
1032         for (i = 0; i < maxRoomNetCfg; i++)
1033                 FreeStrBuf(&sc->Users[i]);
1034 }
1035
1036
1037
1038 /*
1039  * It's ok if these directories already exist.  Just fail silently.
1040  */
1041 void create_spool_dirs(void) {
1042         if ((mkdir(ctdl_spool_dir, 0700) != 0) && (errno != EEXIST))
1043                 syslog(LOG_EMERG, "unable to create directory [%s]: %s", ctdl_spool_dir, strerror(errno));
1044         if (chown(ctdl_spool_dir, CTDLUID, (-1)) != 0)
1045                 syslog(LOG_EMERG, "unable to set the access rights for [%s]: %s", ctdl_spool_dir, strerror(errno));
1046         if ((mkdir(ctdl_netin_dir, 0700) != 0) && (errno != EEXIST))
1047                 syslog(LOG_EMERG, "unable to create directory [%s]: %s", ctdl_netin_dir, strerror(errno));
1048         if (chown(ctdl_netin_dir, CTDLUID, (-1)) != 0)
1049                 syslog(LOG_EMERG, "unable to set the access rights for [%s]: %s", ctdl_netin_dir, strerror(errno));
1050         if ((mkdir(ctdl_nettmp_dir, 0700) != 0) && (errno != EEXIST))
1051                 syslog(LOG_EMERG, "unable to create directory [%s]: %s", ctdl_nettmp_dir, strerror(errno));
1052         if (chown(ctdl_nettmp_dir, CTDLUID, (-1)) != 0)
1053                 syslog(LOG_EMERG, "unable to set the access rights for [%s]: %s", ctdl_nettmp_dir, strerror(errno));
1054         if ((mkdir(ctdl_netout_dir, 0700) != 0) && (errno != EEXIST))
1055                 syslog(LOG_EMERG, "unable to create directory [%s]: %s", ctdl_netout_dir, strerror(errno));
1056         if (chown(ctdl_netout_dir, CTDLUID, (-1)) != 0)
1057                 syslog(LOG_EMERG, "unable to set the access rights for [%s]: %s", ctdl_netout_dir, strerror(errno));
1058 }
1059
1060 /*
1061  * Module entry point
1062  */
1063 CTDL_MODULE_INIT(network_spool)
1064 {
1065         if (!threading)
1066         {
1067                 CtdlREGISTERRoomCfgType(subpending,       ParseSubPendingLine,   0, 5, SerializeGeneric,  DeleteGenericCfgLine); /// todo: move this to mailinglist manager
1068                 CtdlREGISTERRoomCfgType(unsubpending,     ParseUnSubPendingLine, 0, 4, SerializeGeneric,  DeleteGenericCfgLine); /// todo: move this to mailinglist manager
1069                 CtdlREGISTERRoomCfgType(lastsent,         ParseLastSent,         1, 1, SerializeLastSent, DeleteLastSent);
1070                 CtdlREGISTERRoomCfgType(ignet_push_share, ParseGeneric,          0, 2, SerializeGeneric,  DeleteGenericCfgLine); // [remotenode|remoteroomname (optional)]// todo: move this to the ignet client
1071                 CtdlREGISTERRoomCfgType(listrecp,         ParseGeneric,          0, 1, SerializeGeneric,  DeleteGenericCfgLine);
1072                 CtdlREGISTERRoomCfgType(digestrecp,       ParseGeneric,          0, 1, SerializeGeneric,  DeleteGenericCfgLine);
1073                 CtdlREGISTERRoomCfgType(participate,      ParseGeneric,          0, 1, SerializeGeneric,  DeleteGenericCfgLine);
1074                 CtdlREGISTERRoomCfgType(roommailalias,    ParseRoomAlias,        0, 1, SerializeGeneric,  DeleteGenericCfgLine);
1075
1076                 create_spool_dirs();
1077 //////todo              CtdlRegisterCleanupHook(destroy_network_queue_room);
1078         }
1079         return "network_spool";
1080 }