# config.mk is generated by ./configure
include config.mk
-all := citserver setup sendcommand citmail chkpw chkpwd ctdldump ctdlload
+all := ctdldump ctdlload citserver setup sendcommand citmail chkpw chkpwd
all: $(all)
SRCDIRS := $(wildcard server server/modules/*)
cc ${CFLAGS} ${LDFLAGS} utils/chkpwd.c utils/auth.c -lcrypt -o chkpwd
ctdldump: utils/ctdldump.c utils/*.h server/*.h
- cc ${CFLAGS} ${LDFLAGS} utils/ctdldump.c -lcitadel -lz -ldb -o ctdldump
+ cc ${CFLAGS} ${LDFLAGS} utils/ctdldump.c -lcitadel -lz -ldb -lpthread -o ctdldump
ctdlload: utils/ctdlload.c server/makeuserkey.c utils/*.h server/*.h
- cc ${CFLAGS} ${LDFLAGS} utils/ctdlload.c server/makeuserkey.c -lcitadel -lz -ldb -o ctdlload
+ cc ${CFLAGS} ${LDFLAGS} utils/ctdlload.c server/makeuserkey.c -lcitadel -lz -ldb -lpthread -o ctdlload
config.mk: configure
./configure
#include "typesize.h"
#include "ipcdef.h"
-#define REV_LEVEL 980 // This version
+#define REV_LEVEL 988 // This version
#define REV_MIN 591 // Oldest compatible database
#define EXPORT_REV_MIN 931 // Oldest compatible export files
#define LIBCITADEL_MIN 951 // Minimum required version of libcitadel
-// Copyright (c) 1987-2022 by the citadel.org team
+// Copyright (c) 1987-2023 by the citadel.org team
//
// This program is open source software. Use, duplication, or disclosure
// is subject to the terms of the GNU General Public License, version 3.
#define file_pid_paniclog "panic.log"
#define file_crpt_file_key "keys/citadel.key"
#define file_crpt_file_cer "keys/citadel.cer"
-#define file_chkpwd CTDLDIR "chkpwd"
-#define file_guesstimezone CTDLDIR "guesstimezone.sh"
+#define file_chkpwd CTDLDIR "/chkpwd"
+#define file_guesstimezone CTDLDIR "/guesstimezone.sh"
// externs
int i;
int ret;
+ static int closing = 0;
+ while (closing == 1) {
+ syslog(LOG_INFO, "db: already closing");
+ }
+ closing = 1;
+
syslog(LOG_INFO, "db: performing final checkpoint");
if ((ret = dbenv->txn_checkpoint(dbenv, 0, 0, 0))) {
syslog(LOG_ERR, "db: txn_checkpoint: %s", db_strerror(ret));
if (ret) {
syslog(LOG_ERR, "db: db_close: %s", db_strerror(ret));
}
-
}
- // This seemed nifty at the time but did anyone really look at it?
- // #ifdef DB_STAT_ALL
- // dbenv->lock_stat_print(dbenv, DB_STAT_ALL);
- // #endif
-
// Close the handle.
- ret = dbenv->close(dbenv, 0);
+ ret = dbenv->close(dbenv, DB_FORCESYNC);
if (ret) {
syslog(LOG_ERR, "db: DBENV->close: %s", db_strerror(ret));
}
#include <stdio.h>
#include <sys/stat.h>
#include <libcitadel.h>
+#include <string.h>
#include "config.h"
#include "user_ops.h"
return;
}
- char *t = malloc(strlen(response));
- if (!t) {
- return;
- }
- t[0] = 0;
-
char *p;
for (p = response; *p != 0; ++p) {
if ( (*p != '\n') && (!isprint(*p)) ) { // expunge any nonprintables except for newlines
static RETSIGTYPE signal_cleanup(int signum) {
syslog(LOG_DEBUG, "sysdep: caught signal %d", signum);
+ signal(SIGINT, SIG_DFL);
+ signal(SIGHUP, SIG_DFL);
+ signal(SIGTERM, SIG_DFL);
+ signal(SIGSEGV, SIG_DFL);
exit_signal = signum;
server_shutting_down = 1;
+ master_cleanup(signum);
}
sigaddset(&set, SIGINT);
sigaddset(&set, SIGHUP);
sigaddset(&set, SIGTERM);
+ sigaddset(&set, SIGSEGV);
sigprocmask(SIG_UNBLOCK, &set, NULL);
signal(SIGINT, signal_cleanup);
signal(SIGHUP, signal_cleanup);
signal(SIGTERM, signal_cleanup);
+ signal(SIGSEGV, signal_cleanup);
// Do not shut down the server on broken pipe signals, otherwise the
// whole Citadel service would come down whenever a single client
pid_t current_child;
-void graceful_shutdown(int signum) {
+void supervisor_process_shutdown(int signum) {
kill(current_child, signum);
unlink(file_pid_file);
exit(0);
do {
current_child = fork();
- signal(SIGTERM, graceful_shutdown);
+ signal(SIGTERM, supervisor_process_shutdown);
if (current_child < 0) {
perror("fork");
exit(errno);
void *p = realloc(ptr, size);
if (!p) {
fprintf(stderr, "realloc() failed to resize %p to %ld bytes, error: %m\n", ptr, size);
- exit(1);
+ abort();
}
return p;
}
char *b64out(void *data, size_t len) {
static char *outbuf = NULL;
static size_t outlen = 0;
- int i;
- char ch;
if ((outbuf == NULL) || (outlen < (len * 2))) {
outbuf = reallok(outbuf, (len * 2));
}
-// export function for a msglist or a fulltext index record
-// (both are indexed by a long and the data is arrays of longs)
+// export function for a msglist
+// (indexed by a long and the data is arrays of longs)
void export_msglist(int which_cdb, DBT *in_key, DBT *in_data) {
int i;
int num_msgs;
}
+// export function for a full text search index record
+// (indexed by an int and the data is arrays of longs)
+void export_fulltext(int which_cdb, DBT *in_key, DBT *in_data) {
+ int i;
+ int num_msgs;
+ long msg;
+
+ // records are indexed by a single "int" and contains an array of zero or more "long"s
+ int indexnum;
+ memcpy(&indexnum, in_key->data, sizeof(int));
+
+ printf("fulltext|%d|", indexnum);
+
+ if (in_data->size > 0) {
+ num_msgs = in_data->size / sizeof(long);
+ for (i=0; i<num_msgs; ++i) {
+ memcpy(&msg, (in_data->data + (i * sizeof(long))), sizeof(long));
+ if (i != 0) {
+ printf(",");
+ }
+ printf("%ld", msg);
+ }
+ }
+ printf("|\n");
+}
+
+
// export function for a visit record
void export_visit(int which_cdb, DBT *in_key, DBT *in_data) {
struct visit *visit = (struct visit *)in_data->data;
+ int i, len;
+
+ // If there is corrupt data in the "seen" array, cut that out before exporting
+ len = strlen(visit->v_seen);
+ for (i=0; i<len; ++i) {
+ if (!isprint(visit->v_seen[i])) {
+ visit->v_seen[i] = 0;
+ }
+ }
+
+ // If there is corrupt data in the "answered" array, cut that out before exporting
+ len = strlen(visit->v_answered);
+ for (i=0; i<len; ++i) {
+ if (!isprint(visit->v_answered[i])) {
+ visit->v_answered[i] = 0;
+ }
+ }
+
+ // output the record
printf("visit|%ld|%ld|%ld|%ld|%u|%s|%s|%d|\n",
visit->v_roomnum,
visit->v_roomgen,
// export function for a directory record
-// (This is a secondary index -- should we just regenerate the data after import?)
-// void export_dir(int which_cdb, DBT *in_key, DBT *in_data) {
- // printf("dir|");
- // fwrite(in_key->data, in_key->size, 1, stdout);
- // printf("|%s|\n", (char *)in_data->data);
-// }
+void export_dir(int which_cdb, DBT *in_key, DBT *in_data) {
+ printf("dir|");
+ fwrite(in_key->data, in_key->size, 1, stdout);
+ printf("|%s|\n", (char *)in_data->data);
+}
// export function for a use table record
// export function for EUID Index records
-//void export_euidindex(int which_cdb, DBT *in_key, DBT *in_data) {
+void export_euidindex(int which_cdb, DBT *in_key, DBT *in_data) {
// The structure of an euidindex record *key* is:
// |----room_number----|----------EUID-------------|
// |-----msg_number----|----room_number----|----------EUID-------------|
// (sizeof long) (sizeof long) (actual length of euid)
- //long msgnum, roomnum;
- //char *euid;
+ long msgnum, roomnum;
+ char *euid;
- //memcpy(&msgnum, in_data->data, sizeof(long));
- //memcpy(&roomnum, in_data->data+sizeof(long), sizeof(msgnum));
- //euid = in_data->data+(sizeof(long)*2);
-//
- //printf("euidindex|%ld|%ld|%s|\n", msgnum, roomnum, euid);
-//}
+ memcpy(&msgnum, in_data->data, sizeof(long));
+ memcpy(&roomnum, in_data->data+sizeof(long), sizeof(msgnum));
+ euid = in_data->data+(sizeof(long)*2);
+
+ printf("euidindex|%ld|%ld|%s|\n", msgnum, roomnum, euid);
+}
// export users-by-number records
// (This is a secondary index -- should we just regenerate the data after import?)
-//void export_usersbynumber(int which_cdb, DBT *in_key, DBT *in_data) {
+void export_usersbynumber(int which_cdb, DBT *in_key, DBT *in_data) {
// key is a long
- //long usernum;
- //memcpy(&usernum, in_key->data, sizeof(usernum));
+ long usernum;
+ memcpy(&usernum, in_key->data, sizeof(usernum));
// value is a string
- //printf("usersbynumber|%ld|%s|\n", usernum, (char *)in_data->data);
-//}
+ printf("usersbynumber|%ld|%s|\n", usernum, (char *)in_data->data);
+}
// export function for a config record
void (*export_functions[])(int which_cdb, DBT *in_key, DBT *in_data) = {
- export_msgmain, // CDB_MSGMAIN
- export_user, // CDB_USERS
- export_room, // CDB_ROOMS
- export_floor, // CDB_FLOORTAB
- export_msglist, // CDB_MSGLISTS
- export_visit, // CDB_VISIT
- zero_function, // CDB_DIRECTORY (regenerate this on the server)
- export_usetable, // CDB_USETABLE
- export_bigmsg, // CDB_BIGMSGS
- zero_function, // CDB_FULLTEXT (regenerate this on the server)
- zero_function, // CDB_EUIDINDEX (regenerate this on the server)
- zero_function, // CDB_USERSBYNUMBER (regenerate this on the server)
- zero_function, // CDB_UNUSED1 (obsolete)
- export_config // CDB_CONFIG
+ export_msgmain, // 00 CDB_MSGMAIN
+ export_user, // 01 CDB_USERS
+ export_room, // 02 CDB_ROOMS
+ export_floor, // 03 CDB_FLOORTAB
+ export_msglist, // 04 CDB_MSGLISTS
+ export_visit, // 05 CDB_VISIT
+ export_dir, // 06 CDB_DIRECTORY
+ export_usetable, // 07 CDB_USETABLE
+ export_bigmsg, // 08 CDB_BIGMSGS
+ export_fulltext, // 09 CDB_FULLTEXT
+ export_euidindex, // 0a CDB_EUIDINDEX
+ export_usersbynumber, // 0b CDB_USERSBYNUMBER
+ zero_function, // 0c CDB_UNUSED1 (obsolete)
+ export_config // 0d CDB_CONFIG
};
// open the file containing the source table
ret = src_dbp->open(src_dbp, NULL, dbfilename, NULL, DB_BTREE, 0, 0600);
if (ret) {
- fprintf(stderr, "ctdldump: db_open: %s\n", db_strerror(ret));
- fprintf(stderr, "ctdldump: exit code %d\n", ret);
- exit(CTDLEXIT_DB);
+ fprintf(stderr, "ctdldump: db_open(%s): %s (skipping empty or missing table)\n", dbfilename, db_strerror(ret));
}
+ else {
- // Acquire a cursor to read the source table
- if ((ret = src_dbp->cursor(src_dbp, NULL, &src_dbcp, 0)) != 0) {
- fprintf(stderr, "ctdldump: db_cursor: %s\n", db_strerror(ret));
- fprintf(stderr, "ctdldump: exit code %d\n", ret);
- exit(CTDLEXIT_DB);
- }
-
- // Zero out these database keys
- memset(&in_key, 0, sizeof(DBT)); // input
- memset(&in_data, 0, sizeof(DBT));
- memset(&uncomp_data, 0, sizeof(DBT)); // decompressed input (the key doesn't change)
-
- // Walk through the database, calling export functions as we go and clearing buffers before each call.
- while (ret = src_dbcp->get(src_dbcp, &in_key, &in_data, DB_NEXT) == 0) {
-
- // If either the key or data are zero length, skip this record
- if ((in_key.size == 0) || (in_data.size == 0)) {
- ++num_bad_rows;
+ // Acquire a cursor to read the source table
+ if ((ret = src_dbp->cursor(src_dbp, NULL, &src_dbcp, 0)) != 0) {
+ fprintf(stderr, "ctdldump: db_cursor: %s\n", db_strerror(ret));
+ fprintf(stderr, "ctdldump: exit code %d\n", ret);
+ exit(CTDLEXIT_DB);
}
-
- else { // Both key and data are >0 length so we're good to go
-
- // Do we need to decompress?
- static int32_t magic = COMPRESS_MAGIC;
- compressed = 0;
- if ((in_data.size >= sizeof(struct CtdlCompressHeader)) && (!memcmp(in_data.data, &magic, sizeof(magic)))) {
- // yes, we need to decompress
- compressed = 1;
- struct CtdlCompressHeader comp;
- memcpy(&comp, in_data.data, sizeof(struct CtdlCompressHeader));
- uncomp_data.size = comp.uncompressed_len;
- uncomp_data.data = reallok(uncomp_data.data, uncomp_data.size);
- destLen = (uLongf)comp.uncompressed_len;
+ // Zero out these database keys
+ memset(&in_key, 0, sizeof(DBT)); // input
+ memset(&in_data, 0, sizeof(DBT));
+ memset(&uncomp_data, 0, sizeof(DBT)); // decompressed input (the key doesn't change)
- ret = uncompress((Bytef *)uncomp_data.data, (uLongf *)&destLen,
- (const Bytef *)in_data.data+sizeof(struct CtdlCompressHeader),
- (uLong)comp.compressed_len);
- if (ret != Z_OK) {
- fprintf(stderr, "ctdldump: uncompress() error %d\n", ret);
- exit(CTDLEXIT_DB);
- }
+ // Walk through the database, calling export functions as we go and clearing buffers before each call.
+ while (ret = src_dbcp->get(src_dbcp, &in_key, &in_data, DB_NEXT) == 0) {
+
+ // If either the key or data are zero length, skip this record
+ if ((in_key.size == 0) || (in_data.size == 0)) {
+ ++num_bad_rows;
}
- // Call the export function registered to this table
- export_functions[which_cdb](which_cdb, &in_key, (compressed ? &uncomp_data : &in_data));
+ else { // Both key and data are >0 length so we're good to go
- // Knowing the total number of rows isn't critical to the program. It's just for the user to know.
- fflush(stdout);
+ // Do we need to decompress?
+ static int32_t magic = COMPRESS_MAGIC;
+ compressed = 0;
+ if ((in_data.size >= sizeof(struct CtdlCompressHeader)) && (!memcmp(in_data.data, &magic, sizeof(magic)))) {
+
+ // yes, we need to decompress
+ compressed = 1;
+ struct CtdlCompressHeader comp;
+ memcpy(&comp, in_data.data, sizeof(struct CtdlCompressHeader));
+ uncomp_data.size = comp.uncompressed_len;
+ uncomp_data.data = reallok(uncomp_data.data, uncomp_data.size);
+ destLen = (uLongf)comp.uncompressed_len;
+
+ ret = uncompress((Bytef *)uncomp_data.data, (uLongf *)&destLen,
+ (const Bytef *)in_data.data+sizeof(struct CtdlCompressHeader),
+ (uLong)comp.compressed_len);
+ if (ret != Z_OK) {
+ fprintf(stderr, "ctdldump: uncompress() error %d\n", ret);
+ exit(CTDLEXIT_DB);
+ }
+ }
+
+ // Call the export function registered to this table
+ export_functions[which_cdb](which_cdb, &in_key, (compressed ? &uncomp_data : &in_data));
+
+ // Knowing the total number of rows isn't critical to the program. It's just for the user to know.
+ fflush(stdout);
+ }
}
}
int main(int argc, char **argv) {
int i = 0;
- char *src_dir = NULL;
- char *dst_dir = NULL;
+ char src_dir[PATH_MAX];
int confirmed = 0;
static DB_ENV *src_dbenv; // Source DB environment (global)
// display the greeting
- fprintf(stderr, "\033[44m\033[30m \033[K\033[0m\n"
- "\033[44m\033[30m DB Dump utility for Citadel \033[K\033[0m\n"
- "\033[44m\033[30m Copyright (c) 2023 by citadel.org et al. \033[K\033[0m\n"
- "\033[44m\033[30m This program is open source software. Use, duplication, or disclosure \033[K\033[0m\n"
- "\033[44m\033[30m is subject to the terms of the GNU General Public license v3. \033[K\033[0m\n"
- "\033[44m\033[30m \033[K\033[0m\n");
+ fprintf(stderr, "\033[44m\033[33m\033[1m \033[K\033[0m\n"
+ "\033[44m\033[33m\033[1m DB Dump utility for Citadel \033[K\033[0m\n"
+ "\033[44m\033[33m\033[1m Copyright (c) 2023 by citadel.org et al. \033[K\033[0m\n"
+ "\033[44m\033[33m\033[1m This program is open source software. Use, duplication, or disclosure \033[K\033[0m\n"
+ "\033[44m\033[33m\033[1m is subject to the terms of the GNU General Public license v3. \033[K\033[0m\n"
+ "\033[44m\033[33m\033[1m \033[K\033[0m\n");
+
+ // Default source directory unless overridden
+ snprintf(src_dir, sizeof(src_dir), "%s/data", CTDLDIR);
// Parse command line
int a;
while ((a = getopt(argc, argv, "h:y")) != EOF) {
switch (a) {
case 'h':
- src_dir = optarg;
+ snprintf(src_dir, sizeof(src_dir), "%s/data", optarg);
break;
case 'y':
confirmed = 1;
break;
default:
- fprintf(stderr, "%s: usage: %s -s source_dir [>dumpfile]\n", argv[0], argv[0]);
+ fprintf(stderr, "%s: usage: %s -s citadel_dir [>dumpfile]\n", argv[0], argv[0]);
exit(2);
}
}
+ if (src_dir == NULL) {
+ fprintf(stderr, "ctdldump: no source directory was specified.\n");
+ exit(1);
+ }
+
if (confirmed == 1) {
fprintf(stderr, "ctdldump: You have specified the [-y] flag, so processing will continue.\n");
}
else {
- fprintf(stderr, "ctdldump: Please read [ https://www.citadel.org/dump-and-load.html ] to learn how to proceed.\n");
- exit(0);
+ fprintf(stderr, "ctdldump: usage: ctdldump -y -h[citadel_dir] >[dump_file]\n");
+ fprintf(stderr, " [citadel_dir] is your database directory, usually /usr/local/citadel\n");
+ fprintf(stderr, " Please read [ https://www.citadel.org/dump-and-load.html ] to learn how to proceed.\n");
+ exit(1);
}
src_dbenv = open_dbenv(src_dir);
printf("begin|\n");
for (i = 0; i < MAXCDB; ++i) {
export_table(i, src_dbenv);
- if (i == CDB_CONFIG) {
- printf("config|regenerate_secondary_indices|1|\n"); // Force citserver to rebuild those tables
- printf("config|MM_fulltext_wordbreaker|0|\n"); // Burn the full text search index
- }
}
close_dbenv(src_dbenv);
printf("end|\n");
+ fprintf(stderr, "ctdldump: \033[32m\033[1mfinished\033[0m\n");
exit(0);
}
void *p = realloc(ptr, size);
if (!p) {
fprintf(stderr, "realloc() failed to resize %p to %ld bytes, error: %m\n", ptr, size);
- exit(1);
+ abort();
}
return p;
}
// Convert a "msgtext" record to a message on disk. NOT THREADSAFE
// This also works for "bigmsg" records.
-int convert_msgtext(char *line, DBT *out_key, DBT *out_data) {
+int import_msgtext(char *line, DBT *out_key, DBT *out_data) {
static char *b64_decoded_msg = NULL;
static size_t b64_decoded_alloc = 0;
// Convert a "msgmeta" record to a message metadata record on disk. NOT THREADSAFE
-int convert_msgmeta(char *line, DBT *out_key, DBT *out_data) {
+int import_msgmeta(char *line, DBT *out_key, DBT *out_data) {
char *token;
struct MetaData *m = malloc(sizeof(struct MetaData));
- token = strtok(line, "|");
- m->meta_msgnum = atol(strtok(NULL, "|"));
- m->meta_refcount = atoi(strtok(NULL, "|"));
- strncpy(m->meta_content_type, strtok(NULL, "|"), sizeof(m->meta_content_type));
- m->meta_rfc822_length = atol(strtok(NULL, "|"));
+
+ memset(m, 0, sizeof(struct MetaData));
+ char *p = line;
+
+ for (int i=0; (token = strsep(&p, "|")); ++i) {
+ switch(i) {
+ case 1:
+ m->meta_msgnum = atol(token);
+ break;
+ case 2:
+ m->meta_refcount = atoi(token);
+ break;
+ case 3:
+ strncpy(m->meta_content_type, token, sizeof(m->meta_content_type));
+ break;
+ case 4:
+ m->meta_rfc822_length = atol(token);
+ break;
+ }
+ }
// metadata records are stored in the CDB_MSGMAIN table,
// but with the index being the *negative* of the message number.
// Convert a "user" record to a record on disk. (Source string is unusable after this function is called.)
-int convert_user(char *line, DBT *out_key, DBT *out_data) {
+int import_user(char *line, DBT *out_key, DBT *out_data) {
char userkey[USERNAME_SIZE];
char *token;
struct ctdluser *u = malloc(sizeof(struct ctdluser));
// Convert a "room" record to a record on disk. (Source string is unusable after this function is called.)
-int convert_room(char *line, DBT *out_key, DBT *out_data) {
+int import_room(char *line, DBT *out_key, DBT *out_data) {
char *token;
struct ctdlroom *r = malloc(sizeof(struct ctdlroom));
// Convert a floor record to a record on disk.
-int convert_floor(char *line, DBT *out_key, DBT *out_data) {
+int import_floor(char *line, DBT *out_key, DBT *out_data) {
char *token;
struct floor *f = malloc(sizeof(struct floor));
int floor_num;
// Import a msglist record
// msglist|26|32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51|
-int convert_msglist(char *line, DBT *out_key, DBT *out_data) {
+int import_msglist(char *line, DBT *out_key, DBT *out_data) {
long roomnum;
- char *token;
+ char *token, *mtoken;
char *p = line;
+ char *q = NULL;
int num_msgs = 0;
long *msglist = NULL;
roomnum = atol(token);
break;
case 2:
- char *q = token;
- char *mtoken;
+ q = token;
for (int j=0; (mtoken = strsep(&q, ",")); ++j) {
msglist = realloc(msglist, (num_msgs+1) * sizeof(long));
msglist[num_msgs++] = atol(mtoken);
// Convert a "visit" record to a record on disk.
-int convert_visit(char *line, DBT *out_key, DBT *out_data) {
+int import_visit(char *line, DBT *out_key, DBT *out_data) {
char *token;
struct visit *v = malloc(sizeof(struct visit));
}
+// Convert a "dir" record to a record on disk.
+int import_dir(char *line, DBT *out_key, DBT *out_data) {
+ char dirkey[SIZ];
+ char username[USERNAME_SIZE];
+ char *token;
+
+ char *p = line;
+ for (int i=0; (token = strsep(&p, "|")); ++i) {
+ switch(i) {
+ case 1:
+ strncpy(dirkey, token, sizeof(dirkey));
+ break;
+ case 2:
+ strncpy(username, token, sizeof(username));
+ break;
+ }
+ }
+
+ out_key->size = strlen(dirkey);
+ out_key->data = reallok(NULL, out_key->size);
+ memcpy(out_key->data, dirkey, strlen(dirkey));
+
+ out_data->size = strlen(username) + 1;
+ out_data->data = strdup(username);
+
+ return(1);
+}
+
+
// Convert a "usetable" record to a record on disk.
-int convert_usetable(char *line, DBT *out_key, DBT *out_data) {
+int import_usetable(char *line, DBT *out_key, DBT *out_data) {
char *token;
struct UseTable *u = malloc(sizeof(struct UseTable));
}
+// Import a full text search index record.
+// It's just like a msglists record: a key and a list of message numbers, but the key is "int" instead of "long"
+int import_fulltext(char *line, DBT *out_key, DBT *out_data) {
+ int indexnum;
+ char *token, *mtoken;
+ char *p = line;
+ char *q = NULL;
+ int num_msgs = 0;
+ long *msglist = NULL;
+
+ for (int i=0; (token = strsep(&p, "|")); ++i) {
+ switch(i) {
+ case 1:
+ indexnum = atoi(token);
+ break;
+ case 2:
+ q = token;
+ for (int j=0; (mtoken = strsep(&q, ",")); ++j) {
+ msglist = realloc(msglist, (num_msgs+1) * sizeof(long));
+ msglist[num_msgs++] = atol(mtoken);
+ }
+ break;
+ }
+ }
+
+ out_key->size = sizeof(int);
+ out_key->data = malloc(out_key->size);
+ memcpy(out_key->data, &indexnum, out_key->size);
+
+ out_data->size = num_msgs * sizeof(long);
+ out_data->data = msglist;
+
+ return(1);
+}
+
+
+// Import an EUID Index record
+// euidindex|msgnum|roomnum|euid|
+int import_euidindex(char *line, DBT *out_key, DBT *out_data) {
+ char euid[SIZ];
+ long msgnum;
+ long roomnum;
+ char *token;
+
+ char *p = line;
+ for (int i=0; (token = strsep(&p, "|")); ++i) {
+ switch(i) {
+ case 1:
+ msgnum = atol(token);
+ break;
+ case 2:
+ roomnum = atol(token);
+ break;
+ case 3:
+ strncpy(euid, token, sizeof(euid));
+ break;
+ }
+ }
+
+ // The structure of an euidindex record *key* is:
+ // |----room_number----|----------EUID-------------|
+ // (sizeof long) (actual length of euid)
+ out_key->size = sizeof(long) + strlen(euid) + 1;
+ out_key->data = reallok(NULL, out_key->size);
+ memcpy(out_key->data, &roomnum, sizeof(long));
+ strcpy(out_key->data + sizeof(long), euid);
+
+ // The structure of an euidindex record *value* is:
+ // |-----msg_number----|----room_number----|----------EUID-------------|
+ // (sizeof long) (sizeof long) (actual length of euid)
+ out_data->size = sizeof(long) + sizeof(long) + strlen(euid) + 1;
+ out_data->data = reallok(NULL, out_data->size);
+ memcpy(out_data->data, &msgnum, sizeof(long));
+ memcpy(out_data->data + sizeof(long), &roomnum, sizeof(long));
+ strcpy(out_data->data + sizeof(long) + sizeof(long), euid);
+
+ return(1);
+}
+
+
+// Import a "users by number" (secondary index) record
+// The key is a "long"
+int import_usersbynumber(char *line, DBT *out_key, DBT *out_data) {
+ char *token;
+ long usernum;
+
+ char *p = line;
+ for (int i=0; (token = strsep(&p, "|")); ++i) {
+ switch(i) {
+ case 1:
+ usernum = atol(token);
+ break;
+ case 2:
+ out_key->size = sizeof(long);
+ out_key->data = reallok(NULL, sizeof(long));
+ memcpy(out_key->data, &usernum, out_key->size);
+ out_data->data = strdup(token);
+ out_data->size = strlen(out_data->data) + 1;
+ return(1);
+ }
+ }
+
+ return(0); // should never get here unless it's a bad record
+}
+
+
// Import a config record
// The key is the config key
// The data is the config key, a null, the value, and another null
-int convert_config(char *line, DBT *out_key, DBT *out_data) {
+int import_config(char *line, DBT *out_key, DBT *out_data) {
char *token;
char *p = line;
char *k, *v;
fflush(stderr);
}
- // Identify the record type we are currently working with
+ // Clear out our record buffer
+ memset(&out_key, 0, sizeof(DBT));
+ memset(&out_data, 0, sizeof(DBT));
+ row_was_good = 0;
+
+ // Identify the record type we are currently working with,
+ // then call the correct conversion function to load up our record buffer.
extract_token(record_type, line, 0, '|', sizeof record_type);
- if (!strcasecmp(record_type, "msgtext")) current_cdb = CDB_MSGMAIN;
- else if (!strcasecmp(record_type, "msgmeta")) current_cdb = CDB_MSGMAIN;
- else if (!strcasecmp(record_type, "user")) current_cdb = CDB_USERS;
- else if (!strcasecmp(record_type, "room")) current_cdb = CDB_ROOMS;
- else if (!strcasecmp(record_type, "floor")) current_cdb = CDB_FLOORTAB;
- else if (!strcasecmp(record_type, "msglist")) current_cdb = CDB_MSGLISTS;
- else if (!strcasecmp(record_type, "visit")) current_cdb = CDB_VISIT;
- else if (!strcasecmp(record_type, "dir")) current_cdb = CDB_DIRECTORY;
- else if (!strcasecmp(record_type, "use")) current_cdb = CDB_USETABLE;
- else if (!strcasecmp(record_type, "bigmsg")) current_cdb = CDB_BIGMSGS;
- else if (!strcasecmp(record_type, "euidindex")) current_cdb = CDB_EUIDINDEX;
- else if (!strcasecmp(record_type, "usersbynumber")) current_cdb = CDB_USERSBYNUMBER;
- else if (!strcasecmp(record_type, "config")) current_cdb = CDB_CONFIG;
- else current_cdb = -1 ;
+ if (!strcasecmp(record_type, "msgtext")) {
+ current_cdb = CDB_MSGMAIN;
+ row_was_good = import_msgtext(line, &out_key, &out_data);
+ }
+ else if (!strcasecmp(record_type, "msgmeta")) {
+ current_cdb = CDB_MSGMAIN;
+ row_was_good = import_msgmeta(line, &out_key, &out_data);
+ }
+ else if (!strcasecmp(record_type, "user")) {
+ current_cdb = CDB_USERS;
+ row_was_good = import_user(line, &out_key, &out_data);
+ }
+ else if (!strcasecmp(record_type, "room")) {
+ current_cdb = CDB_ROOMS;
+ row_was_good = import_room(line, &out_key, &out_data);
+ }
+ else if (!strcasecmp(record_type, "floor")) {
+ current_cdb = CDB_FLOORTAB;
+ row_was_good = import_floor(line, &out_key, &out_data);
+ }
+ else if (!strcasecmp(record_type, "msglist")) {
+ current_cdb = CDB_MSGLISTS;
+ row_was_good = import_msglist(line, &out_key, &out_data);
+ }
+ else if (!strcasecmp(record_type, "visit")) {
+ current_cdb = CDB_VISIT;
+ row_was_good = import_visit(line, &out_key, &out_data);
+ }
+ else if (!strcasecmp(record_type, "dir")) {
+ current_cdb = CDB_DIRECTORY;
+ row_was_good = import_dir(line, &out_key, &out_data);
+ }
+ else if (!strcasecmp(record_type, "use")) {
+ current_cdb = CDB_USETABLE;
+ row_was_good = import_usetable(line, &out_key, &out_data);
+ }
+ else if (!strcasecmp(record_type, "bigmsg")) {
+ current_cdb = CDB_BIGMSGS;
+ row_was_good = import_msgtext(line, &out_key, &out_data);
+ }
+ else if (!strcasecmp(record_type, "fulltext")) {
+ current_cdb = CDB_FULLTEXT;
+ row_was_good = import_fulltext(line, &out_key, &out_data);
+ }
+ else if (!strcasecmp(record_type, "euidindex")) {
+ current_cdb = CDB_EUIDINDEX;
+ row_was_good = import_euidindex(line, &out_key, &out_data);
+ }
+ else if (!strcasecmp(record_type, "usersbynumber")) {
+ current_cdb = CDB_USERSBYNUMBER;
+ row_was_good = import_usersbynumber(line, &out_key, &out_data);
+ }
+ else if (!strcasecmp(record_type, "config")) {
+ current_cdb = CDB_CONFIG;
+ row_was_good = import_config(line, &out_key, &out_data);
+ }
+ else {
+ current_cdb = -1 ;
+ }
+ // If the current record is a different table than the previous record,
+ // then close the other table and open the correct one.
if (current_cdb != previous_cdb) {
if (previous_cdb >= 0) {
fprintf(stderr, "\n");
// open the file containing the destination table
ret = dst_dbp->open(dst_dbp, NULL, dbfilename, NULL, DB_BTREE, (DB_CREATE | DB_TRUNCATE), 0600);
if (ret) {
- fprintf(stderr, "db: db_open: %s\n", db_strerror(ret));
+ fprintf(stderr, "db: db_open(%s): %s\n", dbfilename, db_strerror(ret));
fprintf(stderr, "db: exit code %d\n", ret);
exit(CTDLEXIT_DB);
}
previous_cdb = current_cdb;
}
- // If we have a valid record type and a target database open, dispatch the correct record type handler.
- memset(&out_key, 0, sizeof(DBT));
- memset(&out_data, 0, sizeof(DBT));
- row_was_good = 0;
- if (!strcasecmp(record_type, "msgtext")) row_was_good = convert_msgtext(line, &out_key, &out_data);
- else if (!strcasecmp(record_type, "msgmeta")) row_was_good = convert_msgmeta(line, &out_key, &out_data);
- else if (!strcasecmp(record_type, "user")) row_was_good = convert_user(line, &out_key, &out_data);
- else if (!strcasecmp(record_type, "room")) row_was_good = convert_room(line, &out_key, &out_data);
- else if (!strcasecmp(record_type, "floor")) row_was_good = convert_floor(line, &out_key, &out_data);
- else if (!strcasecmp(record_type, "msglist")) row_was_good = convert_msglist(line, &out_key, &out_data);
- else if (!strcasecmp(record_type, "visit")) row_was_good = convert_visit(line, &out_key, &out_data);
- else if (!strcasecmp(record_type, "use")) row_was_good = convert_usetable(line, &out_key, &out_data);
- else if (!strcasecmp(record_type, "bigmsg")) row_was_good = convert_msgtext(line, &out_key, &out_data);
- else if (!strcasecmp(record_type, "config")) row_was_good = convert_config(line, &out_key, &out_data);
- else row_was_good = 0;
-
+ // If the conversion function was successful, write the record to the database.
if (row_was_good) {
++good_rows;
ret = dst_dbp->put(dst_dbp, NULL, &out_key, &out_data, 0);
static size_t line_alloc = 1;
static char *line;
static size_t line_len = 0;
- char ch;
+ int ch;
fprintf(stderr, "\033[7mtable\033[0m \033[7mgood_rows\033[0m \033[7mbad_rows\033[0m\n");
line = reallok(NULL, line_alloc);
// Main entry point
int main(int argc, char **argv) {
- char *dst_dir = NULL;
+ char dst_dir[PATH_MAX];
int confirmed = 0;
- static DB_ENV *dst_dbenv; // Source DB environment (global)
+ static DB_ENV *dst_dbenv; // Target DB environment
// display the greeting
- fprintf(stderr, "\033[42m\033[30m \033[K\033[0m\n"
- "\033[42m\033[30m DB Load utility for Citadel \033[K\033[0m\n"
- "\033[42m\033[30m Copyright (c) 2023 by citadel.org et al. \033[K\033[0m\n"
- "\033[42m\033[30m This program is open source software. Use, duplication, or disclosure \033[K\033[0m\n"
- "\033[42m\033[30m is subject to the terms of the GNU General Public license v3. \033[K\033[0m\n"
- "\033[42m\033[30m \033[K\033[0m\n");
+ fprintf(stderr, "\033[44m\033[33m\033[1m \033[K\033[0m\n"
+ "\033[44m\033[33m\033[1m DB Load utility for Citadel \033[K\033[0m\n"
+ "\033[44m\033[33m\033[1m Copyright (c) 2023 by citadel.org et al. \033[K\033[0m\n"
+ "\033[44m\033[33m\033[1m This program is open source software. Use, duplication, or disclosure \033[K\033[0m\n"
+ "\033[44m\033[33m\033[1m is subject to the terms of the GNU General Public license v3. \033[K\033[0m\n"
+ "\033[44m\033[33m\033[1m \033[K\033[0m\n");
+
+ // Default destination directory unless overridden
+ snprintf(dst_dir, sizeof(dst_dir), "%s/data", CTDLDIR);
// Parse command line
int a;
while ((a = getopt(argc, argv, "h:y")) != EOF) {
switch (a) {
case 'h':
- dst_dir = optarg;
+ snprintf(dst_dir, sizeof(dst_dir), "%s/data", optarg);
break;
case 'y':
confirmed = 1;
}
}
+ if (dst_dir == NULL) {
+ fprintf(stderr, "ctdlload: no destination directory was specified.\n");
+ exit(1);
+ }
+
if (confirmed == 1) {
fprintf(stderr, "ctdlload: You have specified the [-y] flag, so processing will continue.\n");
}
else {
- fprintf(stderr, "ctdlload: Please read [ https://www.citadel.org/dump-and-load.html ] to learn how to proceed.\n");
- exit(0);
+ fprintf(stderr, "ctdlload: usage: ctdlload -y -h[data_dir] <[dump_file]\n");
+ fprintf(stderr, " [data_dir] is your database directory, usually /usr/local/citadel/data\n");
+ fprintf(stderr, " Please read [ https://www.citadel.org/dump-and-load.html ] to learn how to proceed.\n");
+ exit(1);
}
- char cmd[1024];
+ // Remove any database that is already in the target directory (yes, delete it, be careful)
+ char cmd[PATH_MAX];
snprintf(cmd, sizeof cmd, "rm -fv %s/cdb.* %s/log.*", dst_dir, dst_dir);
system(cmd);
ingest(dst_dbenv);
close_dbenv(dst_dbenv);
+ fprintf(stderr, "ctdlload: \033[32m\033[1mfinished\033[0m\n");
exit(0);
}
#include <sys/types.h>
#include <netinet/in.h>
-#define LIBCITADEL_VERSION_NUMBER 980
+#define LIBCITADEL_VERSION_NUMBER 988
/*
* Here's a bunch of stupid magic to make the MIME parser portable.
void vnote_free(struct vnote *v);
char *vnote_serialize(struct vnote *v);
void vnote_serialize_output_field(char *append_to, char *field, char *label);
+char b64unalphabet(char ch);
#define UDS "_UDS_"
#define DEFAULT_HOST "localhost"
#define DEFAULT_PORT "504"
-#define CLIENT_VERSION 980
+#define CLIENT_VERSION 988
#define CLIENT_TYPE 0
// commands we can send to the stty_ctdl() routine
#define PORT_NUM 80 /* port number to listen on */
#define DEVELOPER_ID 0
#define CLIENT_ID 4
-#define CLIENT_VERSION 980 /* This version of WebCit */
+#define CLIENT_VERSION 988 /* This version of WebCit */
#define MINIMUM_CIT_VERSION 931 /* Minimum required version of Citadel server */
#define LIBCITADEL_MIN 931 /* Minimum required version of libcitadel */
#define DEFAULT_CTDLDIR "/usr/local/citadel" /* Default Citadel server directory */
projects / citadel.git / commitdiff