citadel/modules/network/serv_network.c

   1 /*
   2  * This module handles shared rooms, inter-Citadel mail, and outbound
   3  * mailing list processing.
   4  *
   5  * Copyright (c) 2000-2012 by the citadel.org team
   6  *
   7  *  This program is open source software; you can redistribute it and/or modify
   8  *  it under the terms of the GNU General Public License, version 3.
   9  *
  10  *  This program is distributed in the hope that it will be useful,
  11  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
  12  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  13  *  GNU General Public License for more details.
  14  *
  15  * ** NOTE **   A word on the S_NETCONFIGS semaphore:
  16  * This is a fairly high-level type of critical section.  It ensures that no
  17  * two threads work on the netconfigs files at the same time.  Since we do
  18  * so many things inside these, here are the rules:
  19  *  1. begin_critical_section(S_NETCONFIGS) *before* begin_ any others.
  20  *  2. Do *not* perform any I/O with the client during these sections.
  21  *
  22  */
  23
  24 /*
  25  * Duration of time (in seconds) after which pending list subscribe/unsubscribe
  26  * requests that have not been confirmed will be deleted.
  27  */
  28 #define EXP     259200  /* three days */
  29
  30 #include "sysdep.h"
  31 #include <stdlib.h>
  32 #include <unistd.h>
  33 #include <stdio.h>
  34 #include <fcntl.h>
  35 #include <ctype.h>
  36 #include <signal.h>
  37 #include <pwd.h>
  38 #include <errno.h>
  39 #include <sys/stat.h>
  40 #include <sys/types.h>
  41 #include <dirent.h>
  42 #if TIME_WITH_SYS_TIME
  43 # include <sys/time.h>
  44 # include <time.h>
  45 #else
  46 # if HAVE_SYS_TIME_H
  47 #  include <sys/time.h>
  48 # else
  49 #  include <time.h>
  50 # endif
  51 #endif
  52 #ifdef HAVE_SYSCALL_H
  53 # include <syscall.h>
  54 #else
  55 # if HAVE_SYS_SYSCALL_H
  56 #  include <sys/syscall.h>
  57 # endif
  58 #endif
  59
  60 #include <sys/wait.h>
  61 #include <string.h>
  62 #include <limits.h>
  63 #include <libcitadel.h>
  64 #include "citadel.h"
  65 #include "server.h"
  66 #include "citserver.h"
  67 #include "support.h"
  68 #include "config.h"
  69 #include "user_ops.h"
  70 #include "database.h"
  71 #include "msgbase.h"
  72 #include "internet_addressing.h"
  73 #include "serv_network.h"
  74 #include "clientsocket.h"
  75 #include "file_ops.h"
  76 #include "citadel_dirs.h"
  77 #include "threads.h"
  78 #include "context.h"
  79 #include "ctdl_module.h"
  80 #include "netspool.h"
  81 #include "netmail.h"
  82
  83 int NetQDebugEnabled = 0;
  84 struct CitContext networker_spool_CC;
  85
  86 /* comes from lookup3.c from libcitadel... */
  87 extern uint32_t hashlittle( const void *key, size_t length, uint32_t initval);
  88
  89 typedef struct __roomlists {
  90         RoomProcList *rplist;
  91         HashList *RoomsInterestedIn;
  92 }roomlists;
  93 /*
  94  * When we do network processing, it's accomplished in two passes; one to
  95  * gather a list of rooms and one to actually do them.  It's ok that rplist
  96  * is global; we have a mutex that keeps it safe.
  97  */
  98 struct RoomProcList *rplist = NULL;
  99
 100 int GetNetworkedRoomNumbers(const char *DirName, HashList *DirList)
 101 {
 102         DIR *filedir = NULL;
 103         struct dirent *d;
 104         struct dirent *filedir_entry;
 105         long RoomNR;
 106         long Count = 0;
 107
 108         filedir = opendir (DirName);
 109         if (filedir == NULL) {
 110                 return 0;
 111         }
 112
 113         d = (struct dirent *)malloc(offsetof(struct dirent, d_name) + PATH_MAX + 1);
 114         if (d == NULL) {
 115                 return 0;
 116         }
 117
 118         while ((readdir_r(filedir, d, &filedir_entry) == 0) &&
 119                (filedir_entry != NULL))
 120         {
 121                 RoomNR = atol(filedir_entry->d_name);
 122                 if (RoomNR != 0) {
 123                         Count++;
 124                         Put(DirList, LKEY(RoomNR), &Count, reference_free_handler);
 125                 }
 126         }
 127         free(d);
 128         closedir(filedir);
 129         return Count;
 130 }
 131
 132
 133
 134
 135 /*
 136  * Check the use table.  This is a list of messages which have recently
 137  * arrived on the system.  It is maintained and queried to prevent the same
 138  * message from being entered into the database multiple times if it happens
 139  * to arrive multiple times by accident.
 140  */
 141 int network_usetable(struct CtdlMessage *msg)
 142 {
 143         struct CitContext *CCC = CC;
 144         char msgid[SIZ];
 145         struct cdbdata *cdbut;
 146         struct UseTable ut;
 147
 148         /* Bail out if we can't generate a message ID */
 149         if (msg == NULL) {
 150                 return(0);
 151         }
 152         if (msg->cm_fields['I'] == NULL) {
 153                 return(0);
 154         }
 155         if (IsEmptyStr(msg->cm_fields['I'])) {
 156                 return(0);
 157         }
 158
 159         /* Generate the message ID */
 160         strcpy(msgid, msg->cm_fields['I']);
 161         if (haschar(msgid, '@') == 0) {
 162                 strcat(msgid, "@");
 163                 if (msg->cm_fields['N'] != NULL) {
 164                         strcat(msgid, msg->cm_fields['N']);
 165                 }
 166                 else {
 167                         return(0);
 168                 }
 169         }
 170
 171         cdbut = cdb_fetch(CDB_USETABLE, msgid, strlen(msgid));
 172         if (cdbut != NULL) {
 173                 cdb_free(cdbut);
 174                 QN_syslog(LOG_DEBUG, "network_usetable() : we already have %s\n", msgid);
 175                 return(1);
 176         }
 177
 178         /* If we got to this point, it's unique: add it. */
 179         strcpy(ut.ut_msgid, msgid);
 180         ut.ut_timestamp = time(NULL);
 181         cdb_store(CDB_USETABLE, msgid, strlen(msgid), &ut, sizeof(struct UseTable) );
 182         return(0);
 183 }
 184
 185
 186
 187 /*
 188  * Send the *entire* contents of the current room to one specific network node,
 189  * ignoring anything we know about which messages have already undergone
 190  * network processing.  This can be used to bring a new node into sync.
 191  */
 192 int network_sync_to(char *target_node, long len)
 193 {
 194         struct CitContext *CCC = CC;
 195         const OneRoomNetCfg *OneRNCFG;
 196         const RoomNetCfgLine *pCfgLine;
 197         SpoolControl sc;
 198         int num_spooled = 0;
 199
 200         /* Grab the configuration line we're looking for */
 201         begin_critical_section(S_NETCONFIGS);
 202         OneRNCFG = CtdlGetNetCfgForRoom(CCC->room.QRnumber);
 203         if ((OneRNCFG == NULL) ||
 204             (OneRNCFG->NetConfigs[ignet_push_share] == NULL))
 205         {
 206                 return -1;
 207         }
 208
 209         pCfgLine = OneRNCFG->NetConfigs[ignet_push_share];
 210         while (pCfgLine != NULL)
 211         {
 212                 if (strcmp(ChrPtr(pCfgLine->Value[0]), target_node))
 213                         break;
 214                 pCfgLine = pCfgLine->next;
 215         }
 216         if (pCfgLine == NULL)
 217         {
 218                 return -1;
 219         }
 220         memset(&sc, 0, sizeof(SpoolControl));
 221
 222         sc.NetConfigs[ignet_push_share] = DuplicateOneGenericCfgLine(pCfgLine);
 223
 224         end_critical_section(S_NETCONFIGS);
 225
 226         sc.working_ignetcfg = CtdlLoadIgNetCfg();
 227         sc.the_netmap = CtdlReadNetworkMap();
 228
 229         /* Send ALL messages */
 230         num_spooled = CtdlForEachMessage(MSGS_ALL, 0L, NULL, NULL, NULL,
 231                 network_spool_msg, &sc);
 232
 233         /* Concise cleanup because we know there's only one node in the sc */
 234         DeleteGenericCfgLine(NULL/*TODO*/, &sc.NetConfigs[ignet_push_share]);
 235
 236         DeleteHash(&sc.working_ignetcfg);
 237         DeleteHash(&sc.the_netmap);
 238
 239         QN_syslog(LOG_NOTICE, "Synchronized %d messages to <%s>\n",
 240                   num_spooled, target_node);
 241         return(num_spooled);
 242 }
 243
 244
 245 /*
 246  * Implements the NSYN command
 247  */
 248 void cmd_nsyn(char *argbuf) {
 249         int num_spooled;
 250         long len;
 251         char target_node[256];
 252
 253         if (CtdlAccessCheck(ac_aide)) return;
 254
 255         len = extract_token(target_node, argbuf, 0, '|', sizeof target_node);
 256         num_spooled = network_sync_to(target_node, len);
 257         if (num_spooled >= 0) {
 258                 cprintf("%d Spooled %d messages.\n", CIT_OK, num_spooled);
 259         }
 260         else {
 261                 cprintf("%d No such room/node share exists.\n",
 262                         ERROR + ROOM_NOT_FOUND);
 263         }
 264 }
 265
 266
 267
 268 /*
 269  * Batch up and send all outbound traffic from the current room
 270  */
 271 void network_queue_interesting_rooms(struct ctdlroom *qrbuf, void *data) {
 272         int i;
 273         struct RoomProcList *ptr;
 274         long QRNum = qrbuf->QRnumber;
 275         void *v;
 276         roomlists *RP = (roomlists*) data;
 277
 278         if (!GetHash(RP->RoomsInterestedIn, LKEY(QRNum), &v))
 279                 return;
 280
 281         ptr = (struct RoomProcList *) malloc(sizeof (struct RoomProcList));
 282         if (ptr == NULL) return;
 283
 284         ptr->namelen = strlen(qrbuf->QRname);
 285         if (ptr->namelen > ROOMNAMELEN)
 286                 ptr->namelen = ROOMNAMELEN - 1;
 287
 288         memcpy (ptr->name, qrbuf->QRname, ptr->namelen);
 289         ptr->name[ptr->namelen] = '\0';
 290         ptr->QRNum = qrbuf->QRnumber;
 291
 292         for (i = 0; i < ptr->namelen; i++)
 293         {
 294                 ptr->lcname[i] = tolower(ptr->name[i]);
 295         }
 296
 297         ptr->lcname[ptr->namelen] = '\0';
 298         ptr->key = hashlittle(ptr->lcname, ptr->namelen, 9872345);
 299         ptr->next = RP->rplist;
 300         RP->rplist = ptr;
 301 }
 302
 303 /*
 304  * Batch up and send all outbound traffic from the current room
 305  */
 306 void network_queue_room(struct ctdlroom *qrbuf, void *data) {
 307         int i;
 308         struct RoomProcList *ptr;
 309
 310         if (qrbuf->QRdefaultview == VIEW_QUEUE)
 311                 return;
 312         ptr = (struct RoomProcList *) malloc(sizeof (struct RoomProcList));
 313         if (ptr == NULL) return;
 314
 315         ptr->namelen = strlen(qrbuf->QRname);
 316         if (ptr->namelen > ROOMNAMELEN)
 317                 ptr->namelen = ROOMNAMELEN - 1;
 318
 319         memcpy (ptr->name, qrbuf->QRname, ptr->namelen);
 320         ptr->name[ptr->namelen] = '\0';
 321         ptr->QRNum = qrbuf->QRnumber;
 322
 323         for (i = 0; i < ptr->namelen; i++)
 324         {
 325                 ptr->lcname[i] = tolower(ptr->name[i]);
 326         }
 327         ptr->lcname[ptr->namelen] = '\0';
 328         ptr->key = hashlittle(ptr->lcname, ptr->namelen, 9872345);
 329
 330         begin_critical_section(S_RPLIST);
 331         ptr->next = rplist;
 332         rplist = ptr;
 333         end_critical_section(S_RPLIST);
 334 }
 335
 336 void destroy_network_queue_room(RoomProcList *rplist)
 337 {
 338         struct RoomProcList *cur, *p;
 339
 340         cur = rplist;
 341         while (cur != NULL)
 342         {
 343                 p = cur->next;
 344                 free (cur);
 345                 cur = p;
 346         }
 347 }
 348
 349 void destroy_network_queue_room_locked (void)
 350 {
 351         begin_critical_section(S_RPLIST);
 352         destroy_network_queue_room(rplist);
 353         end_critical_section(S_RPLIST);
 354 }
 355
 356
 357
 358 /*
 359  * Bounce a message back to the sender
 360  */
 361 void network_bounce(struct CtdlMessage *msg, char *reason)
 362 {
 363         struct CitContext *CCC = CC;
 364         char *oldpath = NULL;
 365         char buf[SIZ];
 366         char bouncesource[SIZ];
 367         char recipient[SIZ];
 368         struct recptypes *valid = NULL;
 369         char force_room[ROOMNAMELEN];
 370         static int serialnum = 0;
 371         size_t size;
 372
 373         QNM_syslog(LOG_DEBUG, "entering network_bounce()\n");
 374
 375         if (msg == NULL) return;
 376
 377         snprintf(bouncesource, sizeof bouncesource, "%s@%s", BOUNCESOURCE, config.c_nodename);
 378
 379         /*
 380          * Give it a fresh message ID
 381          */
 382         if (msg->cm_fields['I'] != NULL) {
 383                 free(msg->cm_fields['I']);
 384         }
 385         snprintf(buf, sizeof buf, "%ld.%04lx.%04x@%s",
 386                 (long)time(NULL), (long)getpid(), ++serialnum, config.c_fqdn);
 387         msg->cm_fields['I'] = strdup(buf);
 388
 389         /*
 390          * FIXME ... right now we're just sending a bounce; we really want to
 391          * include the text of the bounced message.
 392          */
 393         if (msg->cm_fields['M'] != NULL) {
 394                 free(msg->cm_fields['M']);
 395         }
 396         msg->cm_fields['M'] = strdup(reason);
 397         msg->cm_format_type = 0;
 398
 399         /*
 400          * Turn the message around
 401          */
 402         if (msg->cm_fields['R'] == NULL) {
 403                 free(msg->cm_fields['R']);
 404         }
 405
 406         if (msg->cm_fields['D'] == NULL) {
 407                 free(msg->cm_fields['D']);
 408         }
 409
 410         snprintf(recipient, sizeof recipient, "%s@%s",
 411                 msg->cm_fields['A'], msg->cm_fields['N']);
 412
 413         if (msg->cm_fields['A'] == NULL) {
 414                 free(msg->cm_fields['A']);
 415         }
 416
 417         if (msg->cm_fields['N'] == NULL) {
 418                 free(msg->cm_fields['N']);
 419         }
 420
 421         if (msg->cm_fields['U'] == NULL) {
 422                 free(msg->cm_fields['U']);
 423         }
 424
 425         msg->cm_fields['A'] = strdup(BOUNCESOURCE);
 426         msg->cm_fields['N'] = strdup(config.c_nodename);
 427         msg->cm_fields['U'] = strdup("Delivery Status Notification (Failure)");
 428
 429         /* prepend our node to the path */
 430         if (msg->cm_fields['P'] != NULL) {
 431                 oldpath = msg->cm_fields['P'];
 432                 msg->cm_fields['P'] = NULL;
 433         }
 434         else {
 435                 oldpath = strdup("unknown_user");
 436         }
 437         size = strlen(oldpath) + SIZ;
 438         msg->cm_fields['P'] = malloc(size);
 439         snprintf(msg->cm_fields['P'], size, "%s!%s", config.c_nodename, oldpath);
 440         free(oldpath);
 441
 442         /* Now submit the message */
 443         valid = validate_recipients(recipient, NULL, 0);
 444         if (valid != NULL) if (valid->num_error != 0) {
 445                 free_recipients(valid);
 446                 valid = NULL;
 447         }
 448         if ( (valid == NULL) || (!strcasecmp(recipient, bouncesource)) ) {
 449                 strcpy(force_room, config.c_aideroom);
 450         }
 451         else {
 452                 strcpy(force_room, "");
 453         }
 454         if ( (valid == NULL) && IsEmptyStr(force_room) ) {
 455                 strcpy(force_room, config.c_aideroom);
 456         }
 457         CtdlSubmitMsg(msg, valid, force_room, 0);
 458
 459         /* Clean up */
 460         if (valid != NULL) free_recipients(valid);
 461         CtdlFreeMessage(msg);
 462         QNM_syslog(LOG_DEBUG, "leaving network_bounce()\n");
 463 }
 464
 465
 466
 467
 468
 469
 470
 471 /*
 472  * network_do_queue()
 473  *
 474  * Run through the rooms doing various types of network stuff.
 475  */
 476 void network_do_queue(void)
 477 {
 478         struct CitContext *CCC = CC;
 479         static int doing_queue = 0;
 480         static time_t last_run = 0L;
 481         int full_processing = 1;
 482         HashList *working_ignetcfg;
 483         HashList *the_netmap = NULL;
 484         int netmap_changed = 0;
 485         roomlists RL;
 486
 487         /*
 488          * Run the full set of processing tasks no more frequently
 489          * than once every n seconds
 490          */
 491         if ( (time(NULL) - last_run) < config.c_net_freq ) {
 492                 full_processing = 0;
 493                 syslog(LOG_DEBUG, "Network full processing in %ld seconds.\n",
 494                        config.c_net_freq - (time(NULL)- last_run)
 495                 );
 496         }
 497
 498         /*
 499          * This is a simple concurrency check to make sure only one queue run
 500          * is done at a time.  We could do this with a mutex, but since we
 501          * don't really require extremely fine granularity here, we'll do it
 502          * with a static variable instead.
 503          */
 504         if (doing_queue) {
 505                 return;
 506         }
 507         doing_queue = 1;
 508
 509         become_session(&networker_spool_CC);
 510         begin_critical_section(S_RPLIST);
 511         RL.rplist = rplist;
 512         rplist = NULL;
 513         end_critical_section(S_RPLIST);
 514
 515         RL.RoomsInterestedIn = NewHash(1, lFlathash);
 516         if (full_processing &&
 517             (GetNetworkedRoomNumbers(ctdl_netcfg_dir, RL.RoomsInterestedIn)==0))
 518         {
 519                 doing_queue = 0;
 520                 DeleteHash(&RL.RoomsInterestedIn);
 521                 if (RL.rplist == NULL)
 522                         return;
 523         }
 524         /* Load the IGnet Configuration into memory */
 525         working_ignetcfg = CtdlLoadIgNetCfg();
 526
 527         /*
 528          * Load the network map and filter list into memory.
 529          */
 530         if (!server_shutting_down)
 531                 the_netmap = CtdlReadNetworkMap();
 532         if (!server_shutting_down)
 533                 load_network_filter_list();
 534
 535         /*
 536          * Go ahead and run the queue
 537          */
 538         if (full_processing && !server_shutting_down) {
 539                 QNM_syslog(LOG_DEBUG, "network: loading outbound queue");
 540                 CtdlForEachRoom(network_queue_interesting_rooms, &RL);
 541         }
 542
 543         if ((RL.rplist != NULL) && (!server_shutting_down)) {
 544                 RoomProcList *ptr, *cmp;
 545                 ptr = RL.rplist;
 546                 QNM_syslog(LOG_DEBUG, "network: running outbound queue");
 547                 while (ptr != NULL && !server_shutting_down) {
 548
 549                         cmp = ptr->next;
 550
 551                         while (cmp != NULL) {
 552                                 if ((cmp->namelen > 0) &&
 553                                     (cmp->key == ptr->key) &&
 554                                     (cmp->namelen == ptr->namelen) &&
 555                                     (strcmp(cmp->lcname, ptr->lcname) == 0))
 556                                 {
 557                                         cmp->namelen = 0;
 558                                 }
 559                                 cmp = cmp->next;
 560                         }
 561
 562                         if (ptr->namelen > 0) {
 563                                 network_spoolout_room(ptr,
 564                                                       working_ignetcfg,
 565                                                       the_netmap);
 566                         }
 567                         ptr = ptr->next;
 568                 }
 569         }
 570
 571         /* If there is anything in the inbound queue, process it */
 572         if (!server_shutting_down) {
 573                 network_do_spoolin(working_ignetcfg,
 574                                    the_netmap,
 575                                    &netmap_changed);
 576         }
 577
 578         /* Free the filter list in memory */
 579         free_netfilter_list();
 580
 581         /* Save the network map back to disk */
 582         if (netmap_changed) {
 583                 StrBuf *MapStr = CtdlSerializeNetworkMap(the_netmap);
 584                 CtdlPutSysConfig(IGNETMAP, SmashStrBuf(&MapStr));
 585         }
 586
 587         /* combine singe message files into one spool entry per remote node. */
 588         network_consolidate_spoolout(working_ignetcfg, the_netmap);
 589
 590         /* shut down. */
 591
 592         DeleteHash(&the_netmap);
 593
 594         DeleteHash(&working_ignetcfg);
 595
 596         QNM_syslog(LOG_DEBUG, "network: queue run completed");
 597
 598         if (full_processing) {
 599                 last_run = time(NULL);
 600         }
 601         DeleteHash(&RL.RoomsInterestedIn);
 602         destroy_network_queue_room(RL.rplist);
 603         doing_queue = 0;
 604 }
 605
 606
 607
 608
 609 int network_room_handler (struct ctdlroom *room)
 610 {
 611         network_queue_room(room, NULL);
 612         return 0;
 613 }
 614
 615
 616
 617
 618 void network_logout_hook(void)
 619 {
 620         CitContext *CCC = MyContext();
 621
 622         /*
 623          * If we were talking to a network node, we're not anymore...
 624          */
 625         if (!IsEmptyStr(CCC->net_node)) {
 626                 CtdlNetworkTalkingTo(CCC->net_node, strlen(CCC->net_node), NTT_REMOVE);
 627                 CCC->net_node[0] = '\0';
 628         }
 629 }
 630 void network_cleanup_function(void)
 631 {
 632         struct CitContext *CCC = CC;
 633
 634         if (!IsEmptyStr(CCC->net_node)) {
 635                 CtdlNetworkTalkingTo(CCC->net_node, strlen(CCC->net_node), NTT_REMOVE);
 636                 CCC->net_node[0] = '\0';
 637         }
 638 }
 639
 640
 641 /*
 642  * Module entry point
 643  */
 644
 645 void SetNetQDebugEnabled(const int n)
 646 {
 647         NetQDebugEnabled = n;
 648 }
 649
 650 CTDL_MODULE_INIT(network)
 651 {
 652         if (!threading)
 653         {
 654                 CtdlFillSystemContext(&networker_spool_CC, "CitNetSpool");
 655                 CtdlRegisterDebugFlagHook(HKEY("networkqueue"), SetNetQDebugEnabled, &NetQDebugEnabled);
 656                 CtdlRegisterSessionHook(network_cleanup_function, EVT_STOP, PRIO_STOP + 30);
 657                 CtdlRegisterSessionHook(network_logout_hook, EVT_LOGOUT, PRIO_LOGOUT + 10);
 658                 CtdlRegisterProtoHook(cmd_nsyn, "NSYN", "Synchronize room to node");
 659                 CtdlRegisterRoomHook(network_room_handler);
 660                 CtdlRegisterCleanupHook(destroy_network_queue_room_locked);
 661                 CtdlRegisterSessionHook(network_do_queue, EVT_TIMER, PRIO_QUEUE + 10);
 662         }
 663         return "network";
 664 }