1 // Dump the Citadel database to a flat file that can be restored by ctdlload on any architecture
3 // Copyright (c) 2023 by Art Cancro citadel.org
5 // This program is open source software. Use, duplication, or disclosure
6 // is subject to the terms of the GNU General Public License, version 3.
13 #include <sys/types.h>
14 #include <sys/socket.h>
23 #include <libcitadel.h>
25 #include "../server/sysdep.h"
26 #include "../server/citadel_defs.h"
27 #include "../server/server.h"
28 #include "../server/citadel_dirs.h"
29 #include "../server/database.h"
33 // Wrapper for realloc() that crashes and burns if the call fails.
34 void *reallok(void *ptr, size_t size) {
35 void *p = realloc(ptr, size);
37 fprintf(stderr, "realloc() failed to resize %p to %ld bytes, error: %m\n", ptr, size);
44 // convert a binary blob to base64 (non-reentrant!)
45 char *b64out(void *data, size_t len) {
46 static char *outbuf = NULL;
47 static size_t outlen = 0;
49 if ((outbuf == NULL) || (outlen < (len * 2))) {
50 outbuf = reallok(outbuf, (len * 2));
54 CtdlEncodeBase64(outbuf, data, len, 0);
59 // export function for a message in msgmain
60 void export_msgmain(int which_cdb, struct cdbkeyval kv) {
63 memcpy(&in_msgnum, kv.key.ptr, sizeof(in_msgnum));
65 // If the msgnum is negative, we are looking at METADATA
67 struct MetaData *meta = (struct MetaData *)kv.val.ptr;
68 printf("msgmeta|%ld|%d|%s|%ld|\n",
71 meta->meta_content_type,
72 meta->meta_rfc822_length
76 // If the msgnum is positive, we are looking at a MESSAGE
77 else if (in_msgnum > 0) {
78 printf("msgtext|%ld|%s|\n", in_msgnum, b64out(kv.val.ptr, kv.val.len));
81 // If the msgnum is 0 it's probably not a valid record.
85 // export function for a user record
86 void export_user(int which_cdb, struct cdbkeyval kv) {
88 struct ctdluser *user = (struct ctdluser *)kv.val.ptr;
90 printf("user|%d|%d|%s|%u|%d|%ld|%ld|%d|%s|%ld|%ld|%s|%ld|%ld|\n",
102 b64out(user->emailaddrs, strlen(user->emailaddrs)),
103 user->msgnum_inboxrules,
104 user->lastproc_inboxrules
109 // export function for a room record
110 void export_room(int which_cdb, struct cdbkeyval kv) {
112 struct ctdlroom *room = (struct ctdlroom *)kv.val.ptr;
114 printf("room|%s|%s|%ld|%ld|%ld|%u|%s|%ld|%d|%ld|%d|%d|%ld|%d|%u|%d|%ld|\n",
125 room->QRep.expire_mode,
126 room->QRep.expire_value,
136 // export function for a floor record
137 void export_floor(int which_cdb, struct cdbkeyval kv) {
140 memcpy(&floor_num, kv.key.ptr, sizeof(int));
142 struct floor *floor = (struct floor *)kv.val.ptr;
144 printf("floor|%d|%u|%s|%d|%d|%d|\n",
149 floor->f_ep.expire_mode,
150 floor->f_ep.expire_value
155 // export function for a msglist
156 // (indexed by a long and the data is arrays of longs)
157 void export_msglist(int which_cdb, struct cdbkeyval kv) {
162 // records are indexed by a single "long" and contains an array of zero or more "long"s
164 memcpy(&roomnum, kv.key.ptr, sizeof(long));
166 printf("msglist|%ld|", roomnum);
168 if (kv.val.len > 0) {
169 num_msgs = kv.val.len / sizeof(long);
170 for (i=0; i<num_msgs; ++i) {
171 memcpy(&msg, (kv.val.ptr + (i * sizeof(long))), sizeof(long));
182 // export function for a full text search index record
183 // (indexed by an int and the data is arrays of longs)
184 void export_fulltext(int which_cdb, struct cdbkeyval kv) {
189 // records are indexed by a single "int" and contains an array of zero or more "long"s
191 memcpy(&indexnum, kv.key.ptr, sizeof(int));
193 printf("fulltext|%d|", indexnum);
195 if (kv.val.len > 0) {
196 num_msgs = kv.val.len / sizeof(long);
197 for (i=0; i<num_msgs; ++i) {
198 memcpy(&msg, (kv.val.ptr + (i * sizeof(long))), sizeof(long));
209 // export function for a visit record
210 void export_visit(int which_cdb, struct cdbkeyval kv) {
211 struct visit *visit = (struct visit *)kv.val.ptr;
214 // If there is corrupt data in the "seen" array, cut that out before exporting
215 len = strlen(visit->v_seen);
216 for (i=0; i<len; ++i) {
217 if (!isprint(visit->v_seen[i])) {
218 visit->v_seen[i] = 0;
222 // If there is corrupt data in the "answered" array, cut that out before exporting
223 len = strlen(visit->v_answered);
224 for (i=0; i<len; ++i) {
225 if (!isprint(visit->v_answered[i])) {
226 visit->v_answered[i] = 0;
231 printf("visit|%ld|%ld|%ld|%ld|%u|%s|%s|%d|\n",
244 // export function for a directory record
245 void export_dir(int which_cdb, struct cdbkeyval kv) {
247 fwrite(kv.key.ptr, kv.key.len, 1, stdout);
248 printf("|%s|\n", (char *)kv.val.ptr);
252 // export function for a use table record
253 void export_usetable(int which_cdb, struct cdbkeyval kv) {
254 struct UseTable *u = (struct UseTable *)kv.val.ptr;
255 printf("use|%d|%ld|\n", u->hash, u->timestamp);
259 // export function for large message texts
260 void export_bigmsg(int which_cdb, struct cdbkeyval kv) {
263 memcpy(&msgnum, kv.key.ptr, sizeof(msgnum));
264 printf("bigmsg|%ld|%s|\n", msgnum, b64out(kv.val.ptr, kv.val.len));
268 // export function for EUID Index records
269 void export_euidindex(int which_cdb, struct cdbkeyval kv) {
271 // The structure of an euidindex record *key* is:
272 // |----room_number----|----------EUID-------------|
273 // (sizeof long) (actual length of euid)
275 // The structure of an euidindex record *value* is:
276 // |-----msg_number----|----room_number----|----------EUID-------------|
277 // (sizeof long) (sizeof long) (actual length of euid)
279 long msgnum, roomnum;
282 memcpy(&msgnum, kv.val.ptr, sizeof(long));
283 memcpy(&roomnum, kv.val.ptr+sizeof(long), sizeof(msgnum));
284 euid = kv.val.ptr+(sizeof(long)*2);
286 printf("euidindex|%ld|%ld|%s|\n", msgnum, roomnum, euid);
290 // export users-by-number records
291 // (This is a secondary index -- should we just regenerate the data after import?)
292 void export_usersbynumber(int which_cdb, struct cdbkeyval kv) {
296 memcpy(&usernum, kv.key.ptr, sizeof(usernum));
299 printf("usersbynumber|%ld|%s|\n", usernum, (char *)kv.val.ptr);
303 // export function for a config record
304 void export_config(int which_cdb, struct cdbkeyval kv) {
306 printf("config|%s|%s|\n",
308 (char *)kv.val.ptr + strlen(kv.val.ptr) + 1
314 // For obsolete databases, zero all the output
315 void zero_function(int which_cdb, struct cdbkeyval kv) {
320 void (*export_functions[])(int which_cdb, struct cdbkeyval kv) = {
321 export_msgmain, // 00 CDB_MSGMAIN
322 export_user, // 01 CDB_USERS
323 export_room, // 02 CDB_ROOMS
324 export_floor, // 03 CDB_FLOORTAB
325 export_msglist, // 04 CDB_MSGLISTS
326 export_visit, // 05 CDB_VISIT
327 export_dir, // 06 CDB_DIRECTORY
328 export_usetable, // 07 CDB_USETABLE
329 export_bigmsg, // 08 CDB_BIGMSGS
330 export_fulltext, // 09 CDB_FULLTEXT
331 export_euidindex, // 0a CDB_EUIDINDEX
332 export_usersbynumber, // 0b CDB_USERSBYNUMBER
333 zero_function, // 0c CDB_UNUSED1 (obsolete)
334 export_config // 0d CDB_CONFIG
338 void export_table(int which_cdb) {
340 struct cdbkeyval ckv;
342 int num_good_rows = 0;
343 int num_bad_rows = 0;
345 cdb_rewind(which_cdb);
346 while (ckv = cdb_next_item(which_cdb), ckv.val.ptr!=NULL) { // always read through to the end
347 // Call the export function registered to this table
348 export_functions[which_cdb](which_cdb, ckv);
351 // Knowing the total number of rows isn't critical to the program. It's just for the user to know.
356 int main(int argc, char **argv) {
359 char *ctdldir = CTDLDIR;
361 // display the greeting
362 fprintf(stderr, "\033[44m\033[33m\033[1m \033[K\033[0m\n"
363 "\033[44m\033[33m\033[1m DB Dump utility for Citadel \033[K\033[0m\n"
364 "\033[44m\033[33m\033[1m Copyright (c) 2023 by citadel.org et al. \033[K\033[0m\n"
365 "\033[44m\033[33m\033[1m This program is open source software. Use, duplication, or disclosure \033[K\033[0m\n"
366 "\033[44m\033[33m\033[1m is subject to the terms of the GNU General Public license v3. \033[K\033[0m\n"
367 "\033[44m\033[33m\033[1m \033[K\033[0m\n");
369 // Parse command line
371 while ((a = getopt(argc, argv, "h:y")) != EOF) {
380 fprintf(stderr, "%s: usage: %s -s citadel_dir [>dumpfile]\n", argv[0], argv[0]);
385 if (confirmed == 1) {
386 fprintf(stderr, "ctdldump: You have specified the [-y] flag, so processing will continue.\n");
389 fprintf(stderr, "ctdldump: usage: ctdldump -y -h[citadel_dir] >[dump_file]\n");
390 fprintf(stderr, " [citadel_dir] is your server directory, usually /usr/local/citadel\n");
391 fprintf(stderr, " Please read [ https://www.citadel.org/dump-and-load.html ] to learn how to proceed.\n");
395 if (chdir(ctdldir) != 0) {
396 fprintf(stderr, "ctdlload: unable to change directory to [%s]: %m", ctdldir);
400 // backend modules use syslog -- redirect to stderr
401 openlog("ctdldump", LOG_PERROR , LOG_DAEMON);
403 // initialize the database backend
405 cdb_open_databases();
408 for (i = 0; i < MAXCDB; ++i) {
415 cdb_close_databases();
417 fprintf(stderr, "ctdldump: \033[32m\033[1mfinished\033[0m\n");