1 // Load (restore) the Citadel database from a flat file created by ctdldump
3 // Copyright (c) 2023 by Art Cancro citadel.org
5 // This program is open source software. Use, duplication, or disclosure
6 // is subject to the terms of the GNU General Public License, version 3.
13 #include <sys/types.h>
14 #include <sys/socket.h>
23 #include <libcitadel.h>
25 #include "../server/sysdep.h"
26 #include "../server/citadel_defs.h"
27 #include "../server/server.h"
28 #include "../server/makeuserkey.h"
29 #include "../server/citadel_dirs.h"
30 #include "../server/database.h"
34 // Wrapper for realloc() that crashes and burns if the call fails.
35 void *reallok(void *ptr, size_t size) {
36 void *p = realloc(ptr, size);
38 fprintf(stderr, "realloc() failed to resize %p to %ld bytes, error: %m\n", ptr, size);
45 // Convert a "msgtext" record to a message on disk. NOT THREADSAFE
46 // This also works for "bigmsg" records.
47 int import_msgtext(char *line, struct cdbkeyval *kv) {
49 static char *b64_decoded_msg = NULL;
50 static size_t b64_decoded_alloc = 0;
54 token = strtok(line, "|");
55 msgnum = atol(strtok(NULL, "|"));
56 token = strtok(NULL, "|");
58 // The record key will be the message number
59 kv->key.len = sizeof(long);
60 kv->key.ptr = reallok(kv->key.ptr, kv->key.len);
61 memcpy(kv->key.ptr, &msgnum, kv->key.len);
63 // The record data will be the decoded message text.
64 // We are allocating more memory than we need, but BDB will only write the number of bytes we tell it to.
65 kv->val.ptr = (char *) reallok(kv->val.ptr, strlen(token));
66 kv->val.len = CtdlDecodeBase64(kv->val.ptr, token, strlen(token));
71 // Convert a "msgmeta" record to a message metadata record on disk. NOT THREADSAFE
72 int import_msgmeta(char *line, struct cdbkeyval *kv) {
74 struct MetaData *m = malloc(sizeof(struct MetaData));
76 memset(m, 0, sizeof(struct MetaData));
79 for (int i=0; (token = strsep(&p, "|")); ++i) {
82 m->meta_msgnum = atol(token);
85 m->meta_refcount = atoi(token);
88 strncpy(m->meta_content_type, token, sizeof(m->meta_content_type));
91 m->meta_rfc822_length = atol(token);
96 // metadata records are stored in the CDB_MSGMAIN table,
97 // but with the index being the *negative* of the message number.
98 long index = 0 - m->meta_msgnum;
99 kv->key.len = sizeof(long);
100 kv->key.ptr = reallok(NULL, kv->key.len);
101 memcpy(kv->key.ptr, &index, kv->key.len);
104 kv->val.len = sizeof(struct MetaData);
105 kv->val.ptr = (char *) m; // out_data owns this memory now
111 // Convert a "user" record to a record on disk. (Source string is unusable after this function is called.)
112 int import_user(char *line, struct cdbkeyval *kv) {
113 char userkey[USERNAME_SIZE];
115 struct ctdluser *u = malloc(sizeof(struct ctdluser));
117 memset(u, 0, sizeof(struct ctdluser));
120 for (int i=0; (token = strsep(&p, "|")); ++i) {
123 u->version = atoi(token);
126 u->uid = atoi(token);
129 strncpy(u->password, token, sizeof(u->password));
132 u->flags = atoi(token);
135 u->axlevel = atoi(token);
138 u->usernum = atol(token);
141 u->lastcall = atol(token);
144 u->USuserpurge = atoi(token);
147 strncpy(u->fullname, token, sizeof(u->fullname));
150 u->msgnum_bio = atol(token);
153 u->msgnum_pic = atol(token);
156 CtdlDecodeBase64(u->emailaddrs, token, strlen(token));
159 u->msgnum_inboxrules = atol(token);
162 u->lastproc_inboxrules = atol(token);
167 makeuserkey(userkey, u->fullname);
168 kv->key.len = strlen(userkey);
169 kv->key.ptr = strdup(userkey);
170 kv->val.len = sizeof(struct ctdluser);
171 kv->val.ptr = (char *) u;
176 // Convert a "room" record to a record on disk. (Source string is unusable after this function is called.)
177 int import_room(char *line, struct cdbkeyval *kv) {
179 struct ctdlroom *r = malloc(sizeof(struct ctdlroom));
181 memset(r, 0, sizeof(struct ctdlroom));
184 for (int i=0; (token = strsep(&p, "|")); ++i) {
187 strncpy(r->QRname, token, sizeof(r->QRname));
190 strncpy(r->QRpasswd, token, sizeof (r->QRpasswd));
193 r->QRroomaide = atol(token);
196 r->QRhighest = atol(token);
199 r->QRgen = atol(token);
202 r->QRflags = atoi(token);
205 strncpy(r->QRdirname, token, sizeof(r->QRdirname));
208 r->msgnum_info = atol(token);
211 r->QRfloor = atoi(token);
214 r->QRmtime = atol(token);
217 r->QRep.expire_mode = atoi(token);
220 r->QRep.expire_value = atoi(token);
223 r->QRnumber = atol(token);
226 r->QRorder = atoi(token);
229 r->QRflags2 = atoi(token);
232 r->QRdefaultview = atoi(token);
235 r->msgnum_pic = atol(token);
240 // The key is the room name in all lower case
241 kv->key.len = strlen(r->QRname);
242 kv->key.ptr = strdup(r->QRname);
243 char *k = (char *)kv->key.ptr;
244 for (int i=0; i<=kv->key.len; ++i) {
245 k[i] = tolower(k[i]);
248 kv->val.len = sizeof(struct ctdlroom);
249 kv->val.ptr = (char *) r;
254 // Convert a floor record to a record on disk.
255 int import_floor(char *line, struct cdbkeyval *kv) {
257 struct floor *f = malloc(sizeof(struct floor));
260 memset(f, 0, sizeof(struct floor));
263 for (int i=0; (token = strsep(&p, "|")); ++i) {
266 floor_num = atoi(token);
269 f->f_flags = atoi(token);
272 strncpy(f->f_name, token, sizeof(f->f_name));
275 f->f_ref_count = atoi(token);
278 f->f_ep.expire_mode = atoi(token);
281 f->f_ep.expire_value = atoi(token);
286 kv->key.len = sizeof(int);
287 kv->key.ptr = malloc(kv->key.len);
288 memcpy(kv->key.ptr, &floor_num, kv->key.len);
290 kv->val.len = sizeof(struct floor);
291 kv->val.ptr = (char *) f;
296 // Import a msglist record
297 // msglist|26|32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51|
298 int import_msglist(char *line, struct cdbkeyval *kv) {
300 char *token, *mtoken;
304 long *msglist = NULL;
306 for (int i=0; (token = strsep(&p, "|")); ++i) {
309 roomnum = atol(token);
313 for (int j=0; (mtoken = strsep(&q, ",")); ++j) {
314 msglist = realloc(msglist, (num_msgs+1) * sizeof(long));
315 msglist[num_msgs++] = atol(mtoken);
321 kv->key.len = sizeof(long);
322 kv->key.ptr = malloc(kv->key.len);
323 memcpy(kv->key.ptr, &roomnum, kv->key.len);
325 kv->val.len = num_msgs * sizeof(long);
326 kv->val.ptr = (char *) msglist;
332 // Convert a "visit" record to a record on disk.
333 int import_visit(char *line, struct cdbkeyval *kv) {
335 struct visit *v = malloc(sizeof(struct visit));
337 memset(v, 0, sizeof(struct visit));
340 for (int i=0; (token = strsep(&p, "|")); ++i) {
343 v->v_roomnum = atol(token);
346 v->v_roomgen = atol(token);
349 v->v_usernum = atol(token);
352 v->v_lastseen = atoi(token);
355 v->v_flags = atoi(token);
358 strncpy(v->v_seen, token, sizeof(v->v_seen));
361 strncpy(v->v_answered, token, sizeof(v->v_answered));
364 v->v_view = atoi(token);
369 // The key is the same as the first three data members (3 x long)
370 kv->key.len = sizeof(long) * 3;
371 kv->key.ptr = reallok(NULL, kv->key.len);
372 memcpy(kv->key.ptr, v, kv->key.len);
374 kv->val.len = sizeof(struct visit);
375 kv->val.ptr = (char *) v;
380 // Convert a "dir" record to a record on disk.
381 int import_dir(char *line, struct cdbkeyval *kv) {
383 char username[USERNAME_SIZE];
387 for (int i=0; (token = strsep(&p, "|")); ++i) {
390 strncpy(dirkey, token, sizeof(dirkey));
393 strncpy(username, token, sizeof(username));
398 kv->key.len = strlen(dirkey);
399 kv->key.ptr = reallok(NULL, kv->key.len);
400 memcpy(kv->key.ptr, dirkey, strlen(dirkey));
402 kv->val.len = strlen(username) + 1;
403 kv->val.ptr = strdup(username);
409 // Convert a "usetable" record to a record on disk.
410 int import_usetable(char *line, struct cdbkeyval *kv) {
412 struct UseTable *u = malloc(sizeof(struct UseTable));
414 memset(u, 0, sizeof(struct UseTable));
417 for (int i=0; (token = strsep(&p, "|")); ++i) {
420 u->hash = atoi(token);
423 u->timestamp = atol(token);
428 // the key is just an int (the hash)
429 kv->key.len = sizeof(int);
430 kv->key.ptr = reallok(NULL, kv->key.len);
431 memcpy(kv->key.ptr, &u->hash, kv->key.len);
433 kv->val.len = sizeof(struct UseTable);
434 kv->val.ptr = (char *) u;
439 // Import a full text search index record.
440 // It's just like a msglists record: a key and a list of message numbers, but the key is "int" instead of "long"
441 int import_fulltext(char *line, struct cdbkeyval *kv) {
443 char *token, *mtoken;
447 long *msglist = NULL;
449 for (int i=0; (token = strsep(&p, "|")); ++i) {
452 indexnum = atoi(token);
456 for (int j=0; (mtoken = strsep(&q, ",")); ++j) {
457 msglist = realloc(msglist, (num_msgs+1) * sizeof(long));
458 msglist[num_msgs++] = atol(mtoken);
464 kv->key.len = sizeof(int);
465 kv->key.ptr = malloc(kv->key.len);
466 memcpy(kv->key.ptr, &indexnum, kv->key.len);
468 kv->val.len = num_msgs * sizeof(long);
469 kv->val.ptr = (char *) msglist;
475 // Import an EUID Index record
476 // euidindex|msgnum|roomnum|euid|
477 int import_euidindex(char *line, struct cdbkeyval *kv) {
484 for (int i=0; (token = strsep(&p, "|")); ++i) {
487 msgnum = atol(token);
490 roomnum = atol(token);
493 strncpy(euid, token, sizeof(euid));
498 // The structure of an euidindex record *key* is:
499 // |----room_number----|----------EUID-------------|
500 // (sizeof long) (actual length of euid)
501 kv->key.len = sizeof(long) + strlen(euid) + 1;
502 kv->key.ptr = reallok(NULL, kv->key.len);
503 memcpy(kv->key.ptr, &roomnum, sizeof(long));
504 strcpy(kv->key.ptr + sizeof(long), euid);
506 // The structure of an euidindex record *value* is:
507 // |-----msg_number----|----room_number----|----------EUID-------------|
508 // (sizeof long) (sizeof long) (actual length of euid)
509 kv->val.len = sizeof(long) + sizeof(long) + strlen(euid) + 1;
510 kv->val.ptr = (char *) reallok(NULL, kv->val.len);
511 memcpy(kv->val.ptr, &msgnum, sizeof(long));
512 memcpy(kv->val.ptr + sizeof(long), &roomnum, sizeof(long));
513 strcpy(kv->val.ptr + sizeof(long) + sizeof(long), euid);
519 // Import a "users by number" (secondary index) record
520 // The key is a "long"
521 int import_usersbynumber(char *line, struct cdbkeyval *kv) {
526 for (int i=0; (token = strsep(&p, "|")); ++i) {
529 usernum = atol(token);
532 kv->key.len = sizeof(long);
533 kv->key.ptr = reallok(NULL, sizeof(long));
534 memcpy(kv->key.ptr, &usernum, kv->key.len);
535 kv->val.ptr = (char *) strdup(token);
536 kv->val.len = strlen(kv->val.ptr) + 1;
541 return(0); // should never get here unless it's a bad record
545 // Import a config record
546 // The key is the config key
547 // The data is the config key, a null, the value, and another null
548 int import_config(char *line, struct cdbkeyval *kv) {
553 for (int i=0; (token = strsep(&p, "|")); ++i) {
557 kv->key.len = strlen(token);
558 kv->key.ptr = strdup(token);
562 kv->val.len = strlen(k) + strlen(v) + 2;
563 kv->val.ptr = (char *) reallok(NULL, kv->val.len);
564 strcpy(kv->val.ptr, k);
565 strcpy(kv->val.ptr + strlen(k) + 1, v);
574 // Ingest one line of dump data. NOT REENTRANT
575 void ingest_one(char *line, struct cdbkeyval *kv) {
577 static int good_rows = 0;
578 static int bad_rows = 0;
579 static int current_cdb = -1 ;
580 char record_type[32];
582 static time_t last_diag = 0;
584 // Clear out our record buffer
585 memset(kv, 0, sizeof(struct cdbkeyval));
588 // Identify the record type we are currently working with,
589 // then call the correct conversion function to load up our record buffer.
590 extract_token(record_type, line, 0, '|', sizeof record_type);
591 if (!strcasecmp(record_type, "msgtext")) {
592 current_cdb = CDB_MSGMAIN;
593 row_was_good = import_msgtext(line, kv);
595 else if (!strcasecmp(record_type, "msgmeta")) {
596 current_cdb = CDB_MSGMAIN;
597 row_was_good = import_msgmeta(line, kv);
599 else if (!strcasecmp(record_type, "user")) {
600 current_cdb = CDB_USERS;
601 row_was_good = import_user(line, kv);
603 else if (!strcasecmp(record_type, "room")) {
604 current_cdb = CDB_ROOMS;
605 row_was_good = import_room(line, kv);
607 else if (!strcasecmp(record_type, "floor")) {
608 current_cdb = CDB_FLOORTAB;
609 row_was_good = import_floor(line, kv);
611 else if (!strcasecmp(record_type, "msglist")) {
612 current_cdb = CDB_MSGLISTS;
613 row_was_good = import_msglist(line, kv);
615 else if (!strcasecmp(record_type, "visit")) {
616 current_cdb = CDB_VISIT;
617 row_was_good = import_visit(line, kv);
619 else if (!strcasecmp(record_type, "dir")) {
620 current_cdb = CDB_DIRECTORY;
621 row_was_good = import_dir(line, kv);
623 else if (!strcasecmp(record_type, "use")) {
624 current_cdb = CDB_USETABLE;
625 row_was_good = import_usetable(line, kv);
627 else if (!strcasecmp(record_type, "bigmsg")) {
628 current_cdb = CDB_BIGMSGS;
629 row_was_good = import_msgtext(line, kv);
631 else if (!strcasecmp(record_type, "fulltext")) {
632 current_cdb = CDB_FULLTEXT;
633 row_was_good = import_fulltext(line, kv);
635 else if (!strcasecmp(record_type, "euidindex")) {
636 current_cdb = CDB_EUIDINDEX;
637 row_was_good = import_euidindex(line, kv);
639 else if (!strcasecmp(record_type, "usersbynumber")) {
640 current_cdb = CDB_USERSBYNUMBER;
641 row_was_good = import_usersbynumber(line, kv);
643 else if (!strcasecmp(record_type, "config")) {
644 current_cdb = CDB_CONFIG;
645 row_was_good = import_config(line, kv);
651 // If the conversion function was successful, write the record to the database.
652 if ( (row_was_good) && (current_cdb >= 0) ) {
653 cdb_store(current_cdb, kv->key.ptr, kv->key.len, kv->val.ptr, kv->val.len);
659 if (time(NULL) > last_diag) {
660 fprintf(stderr, " \033[32m%9d\033[0m / \033[31m%8d\033[0m\r", good_rows, bad_rows);
662 last_diag = time(NULL);
668 // This is the loop that loads the dump data. NOT REENTRANT
670 static size_t line_alloc = 1;
672 static size_t line_len = 0;
678 memset(&kv, 0, sizeof(struct cdbkeyval));
680 line = reallok(NULL, line_alloc);
685 while (ch = getc(stdin), ((ch != '\n') && (ch > 0))) {
686 if ((line_len+2) > line_alloc) {
688 line = reallok(line, line_alloc);
690 line[line_len++] = ch;
699 if ( (begin_found) && (!end_found) ) {
700 ingest_one(line, &kv);
702 if (!strncasecmp(line, HKEY("begin|"))) {
704 fprintf(stderr, " good rows / bad rows:\n");
706 if (!strncasecmp(line, HKEY("end|"))) {
707 fprintf(stderr, "\n");
715 fprintf(stderr, "\033[31mWARNING: \"begin\" line was not found in the loaded data.\033[0m\n");
718 fprintf(stderr, "\033[31mWARNING: \"end\" line was not found in the loaded data.\033[0m\n");
724 int main(int argc, char **argv) {
726 char *ctdldir = CTDLDIR;
728 // display the greeting
729 fprintf(stderr, "\033[44m\033[33m\033[1m \033[K\033[0m\n"
730 "\033[44m\033[33m\033[1m DB Load utility for Citadel \033[K\033[0m\n"
731 "\033[44m\033[33m\033[1m Copyright (c) 2023 by citadel.org et al. \033[K\033[0m\n"
732 "\033[44m\033[33m\033[1m This program is open source software. Use, duplication, or disclosure \033[K\033[0m\n"
733 "\033[44m\033[33m\033[1m is subject to the terms of the GNU General Public license v3. \033[K\033[0m\n"
734 "\033[44m\033[33m\033[1m \033[K\033[0m\n");
736 // Parse command line
738 while ((a = getopt(argc, argv, "h:y")) != EOF) {
747 fprintf(stderr, "%s: usage: %s -h citadel_dir [<dumpfile]\n", argv[0], argv[0]);
752 if (confirmed == 1) {
753 fprintf(stderr, "ctdlload: You have specified the [-y] flag, so processing will continue.\n");
756 fprintf(stderr, "ctdlload: usage: ctdlload -y -h[citadel_dir] <[dump_file]\n");
757 fprintf(stderr, " [citadel_dir] is your server directory, usually /usr/local/citadel\n");
758 fprintf(stderr, " Please read [ https://www.citadel.org/dump-and-load.html ] to learn how to proceed.\n");
762 if (chdir(ctdldir) != 0) {
763 fprintf(stderr, "ctdlload: unable to change directory to [%s]: %m", ctdldir);
767 // backend modules use syslog -- redirect to stderr
768 openlog("ctdlload", LOG_PERROR , LOG_DAEMON);
770 // Remove any database that is already in the target directory (yes, delete it, be careful)
771 system("rm -fv ./data/*");
773 // initialize the database backend
775 cdb_open_databases();
780 cdb_close_databases();
782 fprintf(stderr, "ctdlload: \033[32m\033[1mfinished\033[0m\n");