1 // This module dumps and/or loads the Citadel database in XML format.
3 // Copyright (c) 1987-2023 by the citadel.org team
5 // This program is open source software; you can redistribute it and/or modify
6 // it under the terms of the GNU General Public License version 3.
8 // This program is distributed in the hope that it will be useful,
9 // but WITHOUT ANY WARRANTY; without even the implied warranty of
10 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 // GNU General Public License for more details.
13 // Explanation of <progress> tags:
16 // 2% finished exporting configuration
17 // 7% finished exporting users
19 // 17% finished exporting rooms
20 // 18% finished exporting floors
21 // 25% finished exporting visits
22 // 26-99% exporting messages
23 // 100% finished exporting messages
25 // These tags are inserted into the XML stream to give the reader an approximation of its progress.
27 #include "../../sysdep.h"
35 #include <sys/types.h>
42 #include <libcitadel.h>
43 #include "../../citadel_defs.h"
44 #include "../../server.h"
45 #include "../../citserver.h"
46 #include "../../support.h"
47 #include "../../config.h"
48 #include "../../database.h"
49 #include "../../msgbase.h"
50 #include "../../user_ops.h"
51 #include "../../euidindex.h"
52 #include "../../internet_addressing.h"
53 #include "../../ctdl_module.h"
55 char migr_tempfilename1[PATH_MAX];
56 char migr_tempfilename2[PATH_MAX];
57 FILE *migr_global_message_list;
59 char *ikey = NULL; // If we're importing a config key we store it here.
61 //*****************************************************************************
62 //* Code which implements the export appears in this section *
63 //*****************************************************************************
65 // Output a string to the client with these characters escaped: & < > " '
66 void xml_strout(char *str) {
76 client_write(HKEY("""));
78 else if (*c == '\'') {
79 client_write(HKEY("'"));
82 client_write(HKEY("<"));
85 client_write(HKEY(">"));
88 client_write(HKEY("&"));
98 // Export a user record as XML
99 void migr_export_users_backend(char *username, void *data) {
102 if (CtdlGetUser(&u, username) != 0) {
106 client_write(HKEY("<user>\n"));
107 cprintf("<u_version>%d</u_version>\n", u.version);
108 cprintf("<u_uid>%ld</u_uid>\n", (long)u.uid);
109 client_write(HKEY("<u_password>")); xml_strout(u.password); client_write(HKEY("</u_password>\n"));
110 cprintf("<u_flags>%u</u_flags>\n", u.flags);
111 cprintf("<u_timescalled>%ld</u_timescalled>\n", u.timescalled);
112 cprintf("<u_posted>%ld</u_posted>\n", u.posted);
113 cprintf("<u_axlevel>%d</u_axlevel>\n", u.axlevel);
114 cprintf("<u_usernum>%ld</u_usernum>\n", u.usernum);
115 cprintf("<u_lastcall>%ld</u_lastcall>\n", (long)u.lastcall);
116 cprintf("<u_USuserpurge>%d</u_USuserpurge>\n", u.USuserpurge);
117 client_write(HKEY("<u_fullname>")); xml_strout(u.fullname); client_write(HKEY("</u_fullname>\n"));
118 cprintf("<u_msgnum_bio>%ld</u_msgnum_bio>\n", u.msgnum_bio);
119 cprintf("<u_msgnum_pic>%ld</u_msgnum_pic>\n", u.msgnum_pic);
120 cprintf("<u_emailaddrs>%s</u_emailaddrs>\n", u.emailaddrs);
121 cprintf("<u_msgnum_inboxrules>%ld</u_msgnum_inboxrules>\n", u.msgnum_inboxrules);
122 cprintf("<u_lastproc_inboxrules>%ld</u_lastproc_inboxrules>\n", u.lastproc_inboxrules);
123 client_write(HKEY("</user>\n"));
127 void migr_export_users(void) {
128 ForEachUser(migr_export_users_backend, NULL);
132 void migr_export_room_msg(long msgnum, void *userdata) {
133 static int count = 0;
135 cprintf("%ld,", msgnum);
139 fprintf(migr_global_message_list, "%ld\n", msgnum);
143 void migr_export_rooms_backend(struct ctdlroom *buf, void *data) {
144 client_write(HKEY("<room>\n"));
145 client_write(HKEY("<QRname>"));
146 xml_strout(buf->QRname);
147 client_write(HKEY("</QRname>\n"));
148 client_write(HKEY("<QRpasswd>"));
149 xml_strout(buf->QRpasswd);
150 client_write(HKEY("</QRpasswd>\n"));
151 cprintf("<QRroomaide>%ld</QRroomaide>\n", buf->QRroomaide);
152 cprintf("<QRhighest>%ld</QRhighest>\n", buf->QRhighest);
153 cprintf("<QRgen>%ld</QRgen>\n", (long)buf->QRgen);
154 cprintf("<QRflags>%u</QRflags>\n", buf->QRflags);
155 if (buf->QRflags & QR_DIRECTORY) {
156 client_write(HKEY("<QRdirname>"));
157 xml_strout(buf->QRdirname);
158 client_write(HKEY("</QRdirname>\n"));
160 cprintf("<QRfloor>%d</QRfloor>\n", buf->QRfloor);
161 cprintf("<QRmtime>%ld</QRmtime>\n", (long)buf->QRmtime);
162 cprintf("<QRexpire_mode>%d</QRexpire_mode>\n", buf->QRep.expire_mode);
163 cprintf("<QRexpire_value>%d</QRexpire_value>\n", buf->QRep.expire_value);
164 cprintf("<QRnumber>%ld</QRnumber>\n", buf->QRnumber);
165 cprintf("<QRorder>%d</QRorder>\n", buf->QRorder);
166 cprintf("<QRflags2>%u</QRflags2>\n", buf->QRflags2);
167 cprintf("<QRdefaultview>%d</QRdefaultview>\n", buf->QRdefaultview);
168 cprintf("<msgnum_info>%ld</msgnum_info>\n", buf->msgnum_info);
169 cprintf("<msgnum_pic>%ld</msgnum_pic>\n", buf->msgnum_pic);
170 client_write(HKEY("</room>\n"));
172 // message list goes inside this tag
173 CtdlGetRoom(&CC->room, buf->QRname);
174 client_write(HKEY("<room_messages>"));
175 client_write(HKEY("<FRname>"));
176 xml_strout(buf->QRname); // buf->QRname rather than CC->room.QRname to guarantee consistency
177 client_write(HKEY("</FRname>\n"));
178 client_write(HKEY("<FRmsglist>\n"));
179 CtdlForEachMessage(MSGS_ALL, 0L, NULL, NULL, NULL, migr_export_room_msg, NULL);
180 client_write(HKEY("</FRmsglist>\n"));
181 client_write(HKEY("</room_messages>\n"));
185 void migr_export_rooms(void) {
187 migr_global_message_list = fopen(migr_tempfilename1, "w");
188 if (migr_global_message_list != NULL) {
189 CtdlForEachRoom(migr_export_rooms_backend, NULL);
190 fclose(migr_global_message_list);
193 // Process the 'global' message list. (Sort it and remove dups.
194 // Dups are ok because a message may be in more than one room, but
195 // this will be handled by exporting the reference count, not by
196 // exporting the message multiple times.)
197 snprintf(cmd, sizeof cmd, "sort -n <%s >%s", migr_tempfilename1, migr_tempfilename2);
198 if (system(cmd) != 0) {
199 syslog(LOG_ERR, "migrate: error %d", errno);
201 snprintf(cmd, sizeof cmd, "uniq <%s >%s", migr_tempfilename2, migr_tempfilename1);
202 if (system(cmd) != 0) {
203 syslog(LOG_ERR, "migrate: error %d", errno);
206 snprintf(cmd, sizeof cmd, "wc -l %s", migr_tempfilename1);
207 FILE *fp = popen(cmd, "r");
209 fgets(cmd, sizeof cmd, fp);
211 total_msgs = atoi(cmd);
214 total_msgs = 1; // any nonzero just to keep it from barfing
216 syslog(LOG_DEBUG, "migrate: total messages to be exported: %d", total_msgs);
220 void migr_export_floors(void) {
221 struct floor qfbuf, *buf;
224 for (i=0; i < MAXFLOORS; ++i) {
225 client_write(HKEY("<floor>\n"));
226 cprintf("<f_num>%d</f_num>\n", i);
227 CtdlGetFloor(&qfbuf, i);
229 cprintf("<f_flags>%u</f_flags>\n", buf->f_flags);
230 client_write(HKEY("<f_name>")); xml_strout(buf->f_name); client_write(HKEY("</f_name>\n"));
231 cprintf("<f_ref_count>%d</f_ref_count>\n", buf->f_ref_count);
232 cprintf("<f_ep_expire_mode>%d</f_ep_expire_mode>\n", buf->f_ep.expire_mode);
233 cprintf("<f_ep_expire_value>%d</f_ep_expire_value>\n", buf->f_ep.expire_value);
234 client_write(HKEY("</floor>\n"));
239 // Return nonzero if the supplied string contains only characters which are valid in a sequence set.
240 int is_sequence_set(char *s) {
245 while (ch = *c++, ch) {
246 if (!strchr("0123456789*,:", ch)) {
254 // Traverse the visits file...
255 void migr_export_visits(void) {
257 struct cdbdata *cdbv;
259 cdb_rewind(CDB_VISIT);
261 while (cdbv = cdb_next_item(CDB_VISIT), cdbv != NULL) {
262 memset(&vbuf, 0, sizeof(struct visit));
263 memcpy(&vbuf, cdbv->ptr,
264 ((cdbv->len > sizeof(struct visit)) ?
265 sizeof(struct visit) : cdbv->len));
268 client_write(HKEY("<visit>\n"));
269 cprintf("<v_roomnum>%ld</v_roomnum>\n", vbuf.v_roomnum);
270 cprintf("<v_roomgen>%ld</v_roomgen>\n", vbuf.v_roomgen);
271 cprintf("<v_usernum>%ld</v_usernum>\n", vbuf.v_usernum);
273 client_write(HKEY("<v_seen>"));
274 if ( (!IsEmptyStr(vbuf.v_seen)) && (is_sequence_set(vbuf.v_seen)) ) {
275 xml_strout(vbuf.v_seen);
278 cprintf("%ld", vbuf.v_lastseen);
280 client_write(HKEY("</v_seen>"));
282 if ( (!IsEmptyStr(vbuf.v_answered)) && (is_sequence_set(vbuf.v_answered)) ) {
283 client_write(HKEY("<v_answered>"));
284 xml_strout(vbuf.v_answered);
285 client_write(HKEY("</v_answered>\n"));
288 cprintf("<v_flags>%u</v_flags>\n", vbuf.v_flags);
289 cprintf("<v_view>%d</v_view>\n", vbuf.v_view);
290 client_write(HKEY("</visit>\n"));
295 void migr_export_message(long msgnum) {
297 struct CtdlMessage *msg;
299 long bytes_written = 0;
302 // We can use a static buffer here because there will never be more than
303 // one of this operation happening at any given time, and it's really best
304 // to just keep it allocated once instead of torturing malloc/free.
305 // Call this function with msgnum "-1" to free the buffer when finished.
307 static int encoded_alloc = 0;
308 static char *encoded_msg = NULL;
311 if ((encoded_alloc == 0) && (encoded_msg != NULL)) {
319 // Ok, here we go ...
321 msg = CtdlFetchMessage(msgnum, 1);
322 if (msg == NULL) return; // fail silently
324 client_write(HKEY("<message>\n"));
325 GetMetaData(&smi, msgnum);
326 cprintf("<msg_msgnum>%ld</msg_msgnum>\n", msgnum);
327 cprintf("<msg_meta_refcount>%d</msg_meta_refcount>\n", smi.meta_refcount);
328 cprintf("<msg_meta_rfc822_length>%ld</msg_meta_rfc822_length>\n", smi.meta_rfc822_length);
329 client_write(HKEY("<msg_meta_content_type>")); xml_strout(smi.meta_content_type); client_write(HKEY("</msg_meta_content_type>\n"));
331 client_write(HKEY("<msg_text>"));
332 CtdlSerializeMessage(&smr, msg);
335 // Predict the buffer size we need. Expand the buffer if necessary.
336 int encoded_len = smr.len * 15 / 10 ;
337 if (encoded_len > encoded_alloc) {
338 encoded_alloc = encoded_len;
339 encoded_msg = realloc(encoded_msg, encoded_alloc);
342 if (encoded_msg == NULL) {
343 // Questionable hack that hopes it'll work next time and we only lose one message
347 // Once we do the encoding we know the exact size
348 encoded_len = CtdlEncodeBase64(encoded_msg, (char *)smr.ser, smr.len, BASE64_YES_LINEBREAKS);
350 // Careful now. If the message is gargantuan, trying to write multiple gigamegs in one
351 // big write operation can make our transport unhappy. So we'll chunk it up 10 KB at a time.
353 while ( (bytes_written < encoded_len) && (!server_shutting_down) ) {
354 this_block = encoded_len - bytes_written;
355 if (this_block > 10240) {
358 client_write(&encoded_msg[bytes_written], this_block);
359 bytes_written += this_block;
365 client_write(HKEY("</msg_text>\n"));
366 client_write(HKEY("</message>\n"));
370 void migr_export_configs(void) {
371 struct cdbdata *cdbcfg;
376 cdb_rewind(CDB_CONFIG);
377 while (cdbcfg = cdb_next_item(CDB_CONFIG), cdbcfg != NULL) {
379 keylen = strlen(cdbcfg->ptr);
381 value = cdbcfg->ptr + keylen + 1;
383 client_write("<config key=\"", 13);
385 client_write("\">", 2);
387 client_write("</config>\n", 10);
393 void migr_export_messages(void) {
398 int prev_progress = 0;
402 migr_global_message_list = fopen(migr_tempfilename1, "r");
403 if (migr_global_message_list != NULL) {
404 syslog(LOG_INFO, "migrate: opened %s", migr_tempfilename1);
405 while ((Ctx->kill_me == 0) &&
406 (fgets(buf, sizeof(buf), migr_global_message_list) != NULL)) {
409 migr_export_message(msgnum);
412 progress = (count * 74 / total_msgs) + 25 ;
413 if ((progress > prev_progress) && (progress < 100)) {
414 cprintf("<progress>%d</progress>\n", progress);
416 prev_progress = progress;
418 fclose(migr_global_message_list);
420 if (Ctx->kill_me == 0) {
421 syslog(LOG_INFO, "migrate: exported %d messages.", count);
424 syslog(LOG_ERR, "migrate: export aborted due to client disconnect!");
427 migr_export_message(-1L); // This frees the encoding buffer
431 void migr_do_export(void) {
435 cprintf("%d Exporting all Citadel databases.\n", LISTING_FOLLOWS);
438 client_write(HKEY("<?xml version=\"1.0\" encoding=\"UTF-8\" ?>\n"));
439 client_write(HKEY("<citadel_migrate_data>\n"));
440 cprintf("<version>%d</version>\n", REV_LEVEL);
441 cprintf("<progress>%d</progress>\n", 0);
443 // export the configuration database
444 migr_export_configs();
445 cprintf("<progress>%d</progress>\n", 2);
447 if (Ctx->kill_me == 0) migr_export_users();
448 cprintf("<progress>%d</progress>\n", 7);
449 cprintf("<progress>%d</progress>\n", 12);
450 if (Ctx->kill_me == 0) migr_export_rooms();
451 cprintf("<progress>%d</progress>\n", 17);
452 if (Ctx->kill_me == 0) migr_export_floors();
453 cprintf("<progress>%d</progress>\n", 18);
454 if (Ctx->kill_me == 0) migr_export_visits();
455 cprintf("<progress>%d</progress>\n", 25);
456 if (Ctx->kill_me == 0) migr_export_messages();
457 cprintf("<progress>%d</progress>\n", 100);
458 client_write(HKEY("</citadel_migrate_data>\n"));
459 client_write(HKEY("000\n"));
465 // Here's the code that implements the import side. It's going to end up
466 // being one big loop with lots of global variables. I don't care.
467 // You wouldn't run multiple concurrent imports anyway. If this offends your
468 // delicate sensibilities then go rewrite it in Ruby on Rails or something.
471 int citadel_migrate_data = 0; // Are we inside a <citadel_migrate_data> tag pair?
472 StrBuf *migr_chardata = NULL;
473 StrBuf *migr_MsgData = NULL;
474 struct ctdluser usbuf;
475 struct ctdlroom qrbuf;
476 char FRname[ROOMNAMELEN];
481 long import_msgnum = 0;
483 // This callback stores up the data which appears in between tags.
484 void migr_xml_chardata(void *data, const XML_Char *s, int len) {
485 StrBufAppendBufPlain(migr_chardata, s, len, 0);
489 void migr_xml_start(void *data, const char *el, const char **attr) {
492 // *** GENERAL STUFF ***
494 // Reset chardata_len to zero and init buffer
495 FlushStrBuf(migr_chardata);
496 FlushStrBuf(migr_MsgData);
498 if (!strcasecmp(el, "citadel_migrate_data")) {
499 ++citadel_migrate_data;
501 // As soon as it looks like the input data is a genuine Citadel XML export,
502 // whack the existing database on disk to make room for the new one.
503 if (citadel_migrate_data == 1) {
504 syslog(LOG_INFO, "migrate: erasing existing databases so we can receive the incoming import");
505 for (i = 0; i < MAXCDB; ++i) {
512 if (citadel_migrate_data != 1) {
513 syslog(LOG_ERR, "migrate: out-of-sequence tag <%s> detected. Warning: ODD-DATA!", el);
517 // When we begin receiving XML for one of these record types, clear out the associated
518 // buffer so we don't accidentally carry over any data from a previous record.
519 if (!strcasecmp(el, "user")) memset(&usbuf, 0, sizeof(struct ctdluser));
520 else if (!strcasecmp(el, "room")) memset(&qrbuf, 0, sizeof(struct ctdlroom));
521 else if (!strcasecmp(el, "room_messages")) memset(FRname, 0, sizeof FRname);
522 else if (!strcasecmp(el, "floor")) memset(&flbuf, 0, sizeof(struct floor));
523 else if (!strcasecmp(el, "visit")) memset(&vbuf, 0, sizeof(struct visit));
525 else if (!strcasecmp(el, "message")) {
526 memset(&smi, 0, sizeof (struct MetaData));
529 else if (!strcasecmp(el, "config")) {
535 if (!strcasecmp(attr[0], "key")) {
536 ikey = strdup(attr[1]);
545 int migr_userrecord(void *data, const char *el) {
546 if (!strcasecmp(el, "u_version")) usbuf.version = atoi(ChrPtr(migr_chardata));
547 else if (!strcasecmp(el, "u_uid")) usbuf.uid = atol(ChrPtr(migr_chardata));
548 else if (!strcasecmp(el, "u_password")) safestrncpy(usbuf.password, ChrPtr(migr_chardata), sizeof usbuf.password);
549 else if (!strcasecmp(el, "u_flags")) usbuf.flags = atoi(ChrPtr(migr_chardata));
550 else if (!strcasecmp(el, "u_timescalled")) usbuf.timescalled = atol(ChrPtr(migr_chardata));
551 else if (!strcasecmp(el, "u_posted")) usbuf.posted = atol(ChrPtr(migr_chardata));
552 else if (!strcasecmp(el, "u_axlevel")) usbuf.axlevel = atoi(ChrPtr(migr_chardata));
553 else if (!strcasecmp(el, "u_usernum")) usbuf.usernum = atol(ChrPtr(migr_chardata));
554 else if (!strcasecmp(el, "u_lastcall")) usbuf.lastcall = atol(ChrPtr(migr_chardata));
555 else if (!strcasecmp(el, "u_USuserpurge")) usbuf.USuserpurge = atoi(ChrPtr(migr_chardata));
556 else if (!strcasecmp(el, "u_fullname")) safestrncpy(usbuf.fullname, ChrPtr(migr_chardata), sizeof usbuf.fullname);
557 else if (!strcasecmp(el, "u_msgnum_bio")) usbuf.msgnum_bio = atol(ChrPtr(migr_chardata));
558 else if (!strcasecmp(el, "u_msgnum_pic")) usbuf.msgnum_pic = atol(ChrPtr(migr_chardata));
559 else if (!strcasecmp(el, "u_emailaddrs")) safestrncpy(usbuf.emailaddrs, ChrPtr(migr_chardata), sizeof usbuf.emailaddrs);
560 else if (!strcasecmp(el, "u_msgnum_inboxrules")) usbuf.msgnum_inboxrules = atol(ChrPtr(migr_chardata));
561 else if (!strcasecmp(el, "u_lastproc_inboxrules")) usbuf.lastproc_inboxrules = atol(ChrPtr(migr_chardata));
567 int migr_roomrecord(void *data, const char *el) {
568 if (!strcasecmp(el, "QRname")) safestrncpy(qrbuf.QRname, ChrPtr(migr_chardata), sizeof qrbuf.QRname);
569 else if (!strcasecmp(el, "QRpasswd")) safestrncpy(qrbuf.QRpasswd, ChrPtr(migr_chardata), sizeof qrbuf.QRpasswd);
570 else if (!strcasecmp(el, "QRroomaide")) qrbuf.QRroomaide = atol(ChrPtr(migr_chardata));
571 else if (!strcasecmp(el, "QRhighest")) qrbuf.QRhighest = atol(ChrPtr(migr_chardata));
572 else if (!strcasecmp(el, "QRgen")) qrbuf.QRgen = atol(ChrPtr(migr_chardata));
573 else if (!strcasecmp(el, "QRflags")) qrbuf.QRflags = atoi(ChrPtr(migr_chardata));
574 else if (!strcasecmp(el, "QRdirname")) safestrncpy(qrbuf.QRdirname, ChrPtr(migr_chardata), sizeof qrbuf.QRdirname);
575 else if (!strcasecmp(el, "QRfloor")) qrbuf.QRfloor = atoi(ChrPtr(migr_chardata));
576 else if (!strcasecmp(el, "QRmtime")) qrbuf.QRmtime = atol(ChrPtr(migr_chardata));
577 else if (!strcasecmp(el, "QRexpire_mode")) qrbuf.QRep.expire_mode = atoi(ChrPtr(migr_chardata));
578 else if (!strcasecmp(el, "QRexpire_value")) qrbuf.QRep.expire_value = atoi(ChrPtr(migr_chardata));
579 else if (!strcasecmp(el, "QRnumber")) qrbuf.QRnumber = atol(ChrPtr(migr_chardata));
580 else if (!strcasecmp(el, "QRorder")) qrbuf.QRorder = atoi(ChrPtr(migr_chardata));
581 else if (!strcasecmp(el, "QRflags2")) qrbuf.QRflags2 = atoi(ChrPtr(migr_chardata));
582 else if (!strcasecmp(el, "QRdefaultview")) qrbuf.QRdefaultview = atoi(ChrPtr(migr_chardata));
583 else if (!strcasecmp(el, "msgnum_info")) qrbuf.msgnum_info = atol(ChrPtr(migr_chardata));
584 else if (!strcasecmp(el, "msgnum_pic")) qrbuf.msgnum_pic = atol(ChrPtr(migr_chardata));
590 int migr_floorrecord(void *data, const char *el) {
591 if (!strcasecmp(el, "f_num")) floornum = atoi(ChrPtr(migr_chardata));
592 else if (!strcasecmp(el, "f_flags")) flbuf.f_flags = atoi(ChrPtr(migr_chardata));
593 else if (!strcasecmp(el, "f_name")) safestrncpy(flbuf.f_name, ChrPtr(migr_chardata), sizeof flbuf.f_name);
594 else if (!strcasecmp(el, "f_ref_count")) flbuf.f_ref_count = atoi(ChrPtr(migr_chardata));
595 else if (!strcasecmp(el, "f_ep_expire_mode")) flbuf.f_ep.expire_mode = atoi(ChrPtr(migr_chardata));
596 else if (!strcasecmp(el, "f_ep_expire_value")) flbuf.f_ep.expire_value = atoi(ChrPtr(migr_chardata));
602 int migr_visitrecord(void *data, const char *el) {
603 if (!strcasecmp(el, "v_roomnum")) vbuf.v_roomnum = atol(ChrPtr(migr_chardata));
604 else if (!strcasecmp(el, "v_roomgen")) vbuf.v_roomgen = atol(ChrPtr(migr_chardata));
605 else if (!strcasecmp(el, "v_usernum")) vbuf.v_usernum = atol(ChrPtr(migr_chardata));
607 else if (!strcasecmp(el, "v_seen")) {
608 int is_textual_seen = 0;
610 int max = StrLength(migr_chardata);
612 vbuf.v_lastseen = atol(ChrPtr(migr_chardata));
614 for (i=0; i < max; ++i)
615 if (!isdigit(ChrPtr(migr_chardata)[i]))
618 safestrncpy(vbuf.v_seen, ChrPtr(migr_chardata), sizeof vbuf.v_seen);
621 else if (!strcasecmp(el, "v_answered")) safestrncpy(vbuf.v_answered, ChrPtr(migr_chardata), sizeof vbuf.v_answered);
622 else if (!strcasecmp(el, "v_flags")) vbuf.v_flags = atoi(ChrPtr(migr_chardata));
623 else if (!strcasecmp(el, "v_view")) vbuf.v_view = atoi(ChrPtr(migr_chardata));
629 void migr_xml_end(void *data, const char *el) {
633 long *msglist = NULL;
634 int msglist_alloc = 0;
635 // *** GENERAL STUFF ***
637 if (!strcasecmp(el, "citadel_migrate_data")) {
638 --citadel_migrate_data;
642 if (citadel_migrate_data != 1) {
643 syslog(LOG_ERR, "migrate: out-of-sequence tag <%s> detected. Warning: ODD-DATA!", el);
647 // syslog(LOG_DEBUG, "END TAG: <%s> DATA: <%s>\n", el, (migr_chardata_len ? migr_chardata : ""));
651 if (!strcasecmp(el, "config")) {
652 syslog(LOG_DEBUG, "migrate: imported config key=%s", ikey);
655 CtdlSetConfigStr(ikey, (char *)ChrPtr(migr_chardata));
660 syslog(LOG_INFO, "migrate: closed a <config> tag but no key name was supplied.");
665 else if ((!strncasecmp(el, HKEY("u_"))) &&
666 migr_userrecord(data, el))
667 ; /* Nothing to do anymore */
668 else if (!strcasecmp(el, "user")) {
670 syslog(LOG_INFO, "migrate: imported user: %s", usbuf.fullname);
674 else if ((!strncasecmp(el, HKEY("QR"))) &&
675 migr_roomrecord(data, el))
676 ; // Nothing to do anymore
677 else if (!strcasecmp(el, "room")) {
679 syslog(LOG_INFO, "migrate: imported room: %s", qrbuf.QRname);
682 // *** ROOM MESSAGE POINTERS ***
684 else if (!strcasecmp(el, "FRname")) {
685 safestrncpy(FRname, ChrPtr(migr_chardata), sizeof FRname);
688 else if (!strcasecmp(el, "FRmsglist")) {
689 if (!IsEmptyStr(FRname)) {
691 msglist_alloc = 1000;
692 msglist = malloc(sizeof(long) * msglist_alloc);
694 syslog(LOG_DEBUG, "migrate: message list for: %s", FRname);
696 ptr = ChrPtr(migr_chardata);
698 while ((*ptr != 0) && (!isdigit(*ptr))) {
701 if ((*ptr != 0) && (isdigit(*ptr))) {
704 if (msgcount >= msglist_alloc) {
706 msglist = realloc(msglist, sizeof(long) * msglist_alloc);
708 msglist[msgcount++] = msgnum;
711 while ((*ptr != 0) && (isdigit(*ptr))) {
717 CtdlSaveMsgPointersInRoom(FRname, msglist, msgcount, 0, NULL, 1);
722 syslog(LOG_DEBUG, "migrate: imported %d messages.", msgcount);
723 if (server_shutting_down) {
729 else if ((!strncasecmp(el, HKEY("f_"))) &&
730 migr_floorrecord(data, el))
731 ; // Nothing to do anymore
733 else if (!strcasecmp(el, "floor")) {
734 CtdlPutFloor(&flbuf, floornum);
735 syslog(LOG_INFO, "migrate: imported floor #%d (%s)", floornum, flbuf.f_name);
739 else if ((!strncasecmp(el, HKEY("v_"))) && migr_visitrecord(data, el)) {
740 ; // Nothing to do anymore
742 else if (!strcasecmp(el, "visit")) {
744 syslog(LOG_INFO, "migrate: imported visit: %ld/%ld/%ld", vbuf.v_roomnum, vbuf.v_roomgen, vbuf.v_usernum);
749 else if (!strcasecmp(el, "msg_msgnum")) smi.meta_msgnum = import_msgnum = atol(ChrPtr(migr_chardata));
750 else if (!strcasecmp(el, "msg_meta_refcount")) smi.meta_refcount = atoi(ChrPtr(migr_chardata));
751 else if (!strcasecmp(el, "msg_meta_rfc822_length")) smi.meta_rfc822_length = atoi(ChrPtr(migr_chardata));
752 else if (!strcasecmp(el, "msg_meta_content_type")) safestrncpy(smi.meta_content_type, ChrPtr(migr_chardata), sizeof smi.meta_content_type);
754 else if (!strcasecmp(el, "msg_text")) {
756 struct CtdlMessage *msg;
758 FlushStrBuf(migr_MsgData);
759 StrBufDecodeBase64To(migr_chardata, migr_MsgData);
761 msg = CtdlDeserializeMessage(import_msgnum, -1,
762 ChrPtr(migr_MsgData),
763 StrLength(migr_MsgData));
765 rc = CtdlSaveThisMessage(msg, import_msgnum, 0);
776 "migrate: %s message #%ld, size=%d, ref=%d, body=%ld, content-type: %s",
777 (rc!= 0)?"failed to import":"imported",
779 StrLength(migr_MsgData),
781 smi.meta_rfc822_length,
782 smi.meta_content_type
784 memset(&smi, 0, sizeof(smi));
787 // *** MORE GENERAL STUFF ***
789 FlushStrBuf(migr_chardata);
793 // Import begins here
794 void migr_do_import(void) {
799 migr_chardata = NewStrBufPlain(NULL, SIZ * 20);
800 migr_MsgData = NewStrBufPlain(NULL, SIZ * 20);
801 xp = XML_ParserCreate(NULL);
803 cprintf("%d Failed to create XML parser instance\n", ERROR+INTERNAL_ERROR);
806 XML_SetElementHandler(xp, migr_xml_start, migr_xml_end);
807 XML_SetCharacterDataHandler(xp, migr_xml_chardata);
811 cprintf("%d sock it to me\n", SEND_LISTING);
814 client_set_inbound_buf(SIZ * 10);
816 while (client_getln(buf, sizeof buf) >= 0 && strcmp(buf, "000")) {
817 XML_Parse(xp, buf, strlen(buf), 0);
818 if (server_shutting_down)
819 break; // Should we break or return?
822 XML_Parse(xp, "", 0, 1);
824 FreeStrBuf(&migr_chardata);
825 FreeStrBuf(&migr_MsgData);
826 rebuild_euid_index();
827 rebuild_usersbynumber();
828 CtdlSetConfigInt("MM_fulltext_wordbreaker", -1); // Set an invalid wordbreaker to force re-indexing
829 CtdlRebuildDirectoryIndex(); // Force a rebuild of the directory index
834 // ******************************************************************************
835 // * Dispatcher, Common code *
836 // ******************************************************************************
838 // Dump out the pathnames of directories which can be copied "as is"
839 void migr_do_listdirs(void) {
840 cprintf("%d Don't forget these:\n", LISTING_FOLLOWS);
841 cprintf("files|%s\n", ctdl_file_dir);
842 cprintf("messages|%s\n", ctdl_message_dir);
843 cprintf("keys|%s\n", ctdl_key_dir);
848 // ******************************************************************************
849 // * Repair database integrity *
850 // ******************************************************************************
852 StrBuf *PlainMessageBuf = NULL;
853 HashList *UsedMessageIDS = NULL;
855 int migr_restore_message_metadata(long msgnum, int refcount) {
857 struct CtdlMessage *msg;
860 // We can use a static buffer here because there will never be more than
861 // one of this operation happening at any given time, and it's really best
862 // to just keep it allocated once instead of torturing malloc/free.
863 // Call this function with msgnum "-1" to free the buffer when finished.
864 static int encoded_alloc = 0;
865 static char *encoded_msg = NULL;
868 if ((encoded_alloc == 0) && (encoded_msg != NULL)) {
872 // todo FreeStrBuf(&PlainMessageBuf); PlainMessageBuf = NULL;
877 if (PlainMessageBuf == NULL) {
878 PlainMessageBuf = NewStrBufPlain(NULL, 10*SIZ);
881 // Ok, here we go ...
883 msg = CtdlFetchMessage(msgnum, 1);
888 GetMetaData(&smi, msgnum);
889 smi.meta_msgnum = msgnum;
890 smi.meta_refcount = refcount;
892 // restore the content type from the message body:
893 mptr = bmstrcasestr(msg->cm_fields[eMesageText], "Content-type:");
896 safestrncpy(smi.meta_content_type, &mptr[13], sizeof smi.meta_content_type);
897 string_trim(smi.meta_content_type);
898 aptr = smi.meta_content_type;
899 while (!IsEmptyStr(aptr)) {
904 memset(aptr, 0, sizeof(smi.meta_content_type) - (aptr - smi.meta_content_type));
910 CC->redirect_buffer = PlainMessageBuf;
911 CtdlOutputPreLoadedMsg(msg, MT_RFC822, HEADERS_ALL, 0, 1, QP_EADDR);
912 smi.meta_rfc822_length = StrLength(CC->redirect_buffer);
913 CC->redirect_buffer = NULL;
916 "migrate: setting message #%ld meta data to: refcount=%d, bodylength=%ld, content-type: %s",
919 smi.meta_rfc822_length,
920 smi.meta_content_type
929 void migr_check_room_msg(long msgnum, void *userdata) {
930 fprintf(migr_global_message_list, "%ld %s\n", msgnum, CC->room.QRname);
934 void migr_check_rooms_backend(struct ctdlroom *buf, void *data) {
935 // message list goes inside this tag
936 CtdlGetRoom(&CC->room, buf->QRname);
937 CtdlForEachMessage(MSGS_ALL, 0L, NULL, NULL, NULL, migr_check_room_msg, NULL);
941 void RemoveMessagesFromRooms(StrBuf *RoomNameVec, long msgnum) {
943 const char *Pos = NULL;
944 StrBuf *oneRoom = NewStrBuf();
946 syslog(LOG_INFO, "migrate: removing message pointer %ld from these rooms: %s", msgnum, ChrPtr(RoomNameVec));
948 while (Pos != StrBufNOTNULL){
949 StrBufExtract_NextToken(oneRoom, RoomNameVec, &Pos, '|');
950 CtdlDeleteMessages(ChrPtr(oneRoom), &msgnum, 1, "");
952 GetMetaData(&smi, msgnum);
953 AdjRefCount(msgnum, -smi.meta_refcount);
957 void migr_do_restore_meta(void) {
968 migr_global_message_list = fopen(migr_tempfilename1, "w");
969 if (migr_global_message_list != NULL) {
970 CtdlForEachRoom(migr_check_rooms_backend, NULL);
971 fclose(migr_global_message_list);
974 // Process the 'global' message list. (Sort it and remove dups.
975 // Dups are ok because a message may be in more than one room, but
976 // this will be handled by exporting the reference count, not by
977 // exporting the message multiple times.)
978 snprintf(cmd, sizeof cmd, "sort -n <%s >%s", migr_tempfilename1, migr_tempfilename2);
979 if (system(cmd) != 0) {
980 syslog(LOG_ERR, "migrate: error %d", errno);
983 RoomNames = NewStrBuf();
985 migr_global_message_list = fopen(migr_tempfilename2, "r");
986 if (migr_global_message_list != NULL) {
987 syslog(LOG_INFO, "migrate: opened %s", migr_tempfilename1);
988 while ((Ctx->kill_me == 0) &&
989 (fgets(buf, sizeof(buf), migr_global_message_list) != NULL)) {
996 prn = strchr(buf, ' ');
997 if (lastnum != msgnum) {
998 failGetMessage = migr_restore_message_metadata(lastnum, refcount);
999 if (failGetMessage) {
1000 RemoveMessagesFromRooms(RoomNames, lastnum);
1005 StrBufPlain(RoomNames, prn + 1, -1);
1007 StrBufTrim(RoomNames);
1011 if (StrLength(RoomNames) > 0) {
1012 StrBufAppendBufPlain(RoomNames, HKEY("|"), 0);
1014 StrBufAppendBufPlain(RoomNames, prn, -1, 1);
1015 StrBufTrim(RoomNames);
1021 failGetMessage = migr_restore_message_metadata(msgnum, refcount);
1022 if (failGetMessage) {
1023 RemoveMessagesFromRooms(RoomNames, lastnum);
1025 fclose(migr_global_message_list);
1028 migr_restore_message_metadata(-1L, -1); // This frees the encoding buffer
1029 cprintf("%d system analysis completed", CIT_OK);
1034 // ******************************************************************************
1035 // * Dispatcher, Common code *
1036 // ******************************************************************************
1037 void cmd_migr(char *cmdbuf) {
1040 if (CtdlAccessCheck(ac_aide)) return;
1042 if (CtdlTrySingleUser()) {
1043 CtdlDisableHouseKeeping();
1044 CtdlMakeTempFileName(migr_tempfilename1, sizeof migr_tempfilename1);
1045 CtdlMakeTempFileName(migr_tempfilename2, sizeof migr_tempfilename2);
1047 extract_token(cmd, cmdbuf, 0, '|', sizeof cmd);
1048 if (!strcasecmp(cmd, "export")) {
1051 else if (!strcasecmp(cmd, "import")) {
1054 else if (!strcasecmp(cmd, "listdirs")) {
1057 else if (!strcasecmp(cmd, "restoremeta")) {
1058 migr_do_restore_meta();
1061 cprintf("%d illegal command\n", ERROR + ILLEGAL_VALUE);
1064 unlink(migr_tempfilename1);
1065 unlink(migr_tempfilename2);
1067 CtdlEnableHouseKeeping();
1068 CtdlEndSingleUser();
1071 cprintf("%d The migrator is already running.\n", ERROR + RESOURCE_BUSY);
1076 // Initialization function, called from modules_init.c
1077 char *ctdl_module_init_migrate() {
1079 CtdlRegisterProtoHook(cmd_migr, "MIGR", "Across-the-wire migration");
1082 // return our module name for the log