2 * This module dumps and/or loads the Citadel database in XML format.
4 * Copyright (c) 1987-2021 by the citadel.org team
6 * This program is open source software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License version 3.
9 * This program is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 * GNU General Public License for more details.
16 * Explanation of <progress> tags:
19 * 2% finished exporting configuration
20 * 7% finished exporting users
21 * 12% finished exporting openids
22 * 17% finished exporting rooms
23 * 18% finished exporting floors
24 * 25% finished exporting visits
25 * 26-99% exporting messages
26 * 100% finished exporting messages
28 * These tags are inserted into the XML stream to give the reader an approximation of its progress.
39 #include <sys/types.h>
46 #include <libcitadel.h>
49 #include "citserver.h"
55 #include "euidindex.h"
56 #include "ctdl_module.h"
58 char migr_tempfilename1[PATH_MAX];
59 char migr_tempfilename2[PATH_MAX];
60 FILE *migr_global_message_list;
62 char *ikey = NULL; // If we're importing a config key we store it here.
64 /******************************************************************************
65 * Code which implements the export appears in this section *
66 ******************************************************************************/
69 * Output a string to the client with these characters escaped: & < > " '
71 void xml_strout(char *str) {
81 client_write(HKEY("""));
83 else if (*c == '\'') {
84 client_write(HKEY("'"));
87 client_write(HKEY("<"));
90 client_write(HKEY(">"));
93 client_write(HKEY("&"));
104 * Export a user record as XML
106 void migr_export_users_backend(char *username, void *data) {
109 if (CtdlGetUser(&u, username) != 0) {
113 client_write(HKEY("<user>\n"));
114 cprintf("<u_version>%d</u_version>\n", u.version);
115 cprintf("<u_uid>%ld</u_uid>\n", (long)u.uid);
116 client_write(HKEY("<u_password>")); xml_strout(u.password); client_write(HKEY("</u_password>\n"));
117 cprintf("<u_flags>%u</u_flags>\n", u.flags);
118 cprintf("<u_timescalled>%ld</u_timescalled>\n", u.timescalled);
119 cprintf("<u_posted>%ld</u_posted>\n", u.posted);
120 cprintf("<u_axlevel>%d</u_axlevel>\n", u.axlevel);
121 cprintf("<u_usernum>%ld</u_usernum>\n", u.usernum);
122 cprintf("<u_lastcall>%ld</u_lastcall>\n", (long)u.lastcall);
123 cprintf("<u_USuserpurge>%d</u_USuserpurge>\n", u.USuserpurge);
124 client_write(HKEY("<u_fullname>")); xml_strout(u.fullname); client_write(HKEY("</u_fullname>\n"));
125 cprintf("<u_msgnum_bio>%ld</u_msgnum_bio>\n", u.msgnum_bio);
126 cprintf("<u_msgnum_pic>%ld</u_msgnum_pic>\n", u.msgnum_pic);
127 cprintf("<u_emailaddrs>%s</u_emailaddrs>\n", u.emailaddrs);
128 cprintf("<u_msgnum_inboxrules>%ld</u_msgnum_inboxrules>\n", u.msgnum_inboxrules);
129 cprintf("<u_lastproc_inboxrules>%ld</u_lastproc_inboxrules>\n", u.lastproc_inboxrules);
130 client_write(HKEY("</user>\n"));
134 void migr_export_users(void) {
135 ForEachUser(migr_export_users_backend, NULL);
139 void migr_export_room_msg(long msgnum, void *userdata) {
140 static int count = 0;
142 cprintf("%ld,", msgnum);
146 fprintf(migr_global_message_list, "%ld\n", msgnum);
150 void migr_export_rooms_backend(struct ctdlroom *buf, void *data) {
151 client_write(HKEY("<room>\n"));
152 client_write(HKEY("<QRname>"));
153 xml_strout(buf->QRname);
154 client_write(HKEY("</QRname>\n"));
155 client_write(HKEY("<QRpasswd>"));
156 xml_strout(buf->QRpasswd);
157 client_write(HKEY("</QRpasswd>\n"));
158 cprintf("<QRroomaide>%ld</QRroomaide>\n", buf->QRroomaide);
159 cprintf("<QRhighest>%ld</QRhighest>\n", buf->QRhighest);
160 cprintf("<QRgen>%ld</QRgen>\n", (long)buf->QRgen);
161 cprintf("<QRflags>%u</QRflags>\n", buf->QRflags);
162 if (buf->QRflags & QR_DIRECTORY) {
163 client_write(HKEY("<QRdirname>"));
164 xml_strout(buf->QRdirname);
165 client_write(HKEY("</QRdirname>\n"));
167 cprintf("<QRfloor>%d</QRfloor>\n", buf->QRfloor);
168 cprintf("<QRmtime>%ld</QRmtime>\n", (long)buf->QRmtime);
169 cprintf("<QRexpire_mode>%d</QRexpire_mode>\n", buf->QRep.expire_mode);
170 cprintf("<QRexpire_value>%d</QRexpire_value>\n", buf->QRep.expire_value);
171 cprintf("<QRnumber>%ld</QRnumber>\n", buf->QRnumber);
172 cprintf("<QRorder>%d</QRorder>\n", buf->QRorder);
173 cprintf("<QRflags2>%u</QRflags2>\n", buf->QRflags2);
174 cprintf("<QRdefaultview>%d</QRdefaultview>\n", buf->QRdefaultview);
175 cprintf("<msgnum_info>%ld</msgnum_info>\n", buf->msgnum_info);
176 cprintf("<msgnum_pic>%ld</msgnum_pic>\n", buf->msgnum_pic);
177 client_write(HKEY("</room>\n"));
179 /* message list goes inside this tag */
180 CtdlGetRoom(&CC->room, buf->QRname);
181 client_write(HKEY("<room_messages>"));
182 client_write(HKEY("<FRname>"));
183 xml_strout(buf->QRname); // buf->QRname rather than CC->room.QRname to guarantee consistency
184 client_write(HKEY("</FRname>\n"));
185 client_write(HKEY("<FRmsglist>\n"));
186 CtdlForEachMessage(MSGS_ALL, 0L, NULL, NULL, NULL, migr_export_room_msg, NULL);
187 client_write(HKEY("</FRmsglist>\n"));
188 client_write(HKEY("</room_messages>\n"));
192 void migr_export_rooms(void) {
194 migr_global_message_list = fopen(migr_tempfilename1, "w");
195 if (migr_global_message_list != NULL) {
196 CtdlForEachRoom(migr_export_rooms_backend, NULL);
197 fclose(migr_global_message_list);
201 * Process the 'global' message list. (Sort it and remove dups.
202 * Dups are ok because a message may be in more than one room, but
203 * this will be handled by exporting the reference count, not by
204 * exporting the message multiple times.)
206 snprintf(cmd, sizeof cmd, "sort -n <%s >%s", migr_tempfilename1, migr_tempfilename2);
207 if (system(cmd) != 0) {
208 syslog(LOG_ERR, "migrate: error %d", errno);
210 snprintf(cmd, sizeof cmd, "uniq <%s >%s", migr_tempfilename2, migr_tempfilename1);
211 if (system(cmd) != 0) {
212 syslog(LOG_ERR, "migrate: error %d", errno);
215 snprintf(cmd, sizeof cmd, "wc -l %s", migr_tempfilename1);
216 FILE *fp = popen(cmd, "r");
218 fgets(cmd, sizeof cmd, fp);
220 total_msgs = atoi(cmd);
223 total_msgs = 1; // any nonzero just to keep it from barfing
225 syslog(LOG_DEBUG, "migrate: total messages to be exported: %d", total_msgs);
229 void migr_export_floors(void) {
230 struct floor qfbuf, *buf;
233 for (i=0; i < MAXFLOORS; ++i) {
234 client_write(HKEY("<floor>\n"));
235 cprintf("<f_num>%d</f_num>\n", i);
236 CtdlGetFloor(&qfbuf, i);
238 cprintf("<f_flags>%u</f_flags>\n", buf->f_flags);
239 client_write(HKEY("<f_name>")); xml_strout(buf->f_name); client_write(HKEY("</f_name>\n"));
240 cprintf("<f_ref_count>%d</f_ref_count>\n", buf->f_ref_count);
241 cprintf("<f_ep_expire_mode>%d</f_ep_expire_mode>\n", buf->f_ep.expire_mode);
242 cprintf("<f_ep_expire_value>%d</f_ep_expire_value>\n", buf->f_ep.expire_value);
243 client_write(HKEY("</floor>\n"));
249 * Return nonzero if the supplied string contains only characters which are valid in a sequence set.
251 int is_sequence_set(char *s) {
256 while (ch = *c++, ch) {
257 if (!strchr("0123456789*,:", ch)) {
266 * Traverse the visits file...
268 void migr_export_visits(void) {
270 struct cdbdata *cdbv;
272 cdb_rewind(CDB_VISIT);
274 while (cdbv = cdb_next_item(CDB_VISIT), cdbv != NULL) {
275 memset(&vbuf, 0, sizeof(visit));
276 memcpy(&vbuf, cdbv->ptr,
277 ((cdbv->len > sizeof(visit)) ?
278 sizeof(visit) : cdbv->len));
281 client_write(HKEY("<visit>\n"));
282 cprintf("<v_roomnum>%ld</v_roomnum>\n", vbuf.v_roomnum);
283 cprintf("<v_roomgen>%ld</v_roomgen>\n", vbuf.v_roomgen);
284 cprintf("<v_usernum>%ld</v_usernum>\n", vbuf.v_usernum);
286 client_write(HKEY("<v_seen>"));
287 if ( (!IsEmptyStr(vbuf.v_seen)) && (is_sequence_set(vbuf.v_seen)) ) {
288 xml_strout(vbuf.v_seen);
291 cprintf("%ld", vbuf.v_lastseen);
293 client_write(HKEY("</v_seen>"));
295 if ( (!IsEmptyStr(vbuf.v_answered)) && (is_sequence_set(vbuf.v_answered)) ) {
296 client_write(HKEY("<v_answered>"));
297 xml_strout(vbuf.v_answered);
298 client_write(HKEY("</v_answered>\n"));
301 cprintf("<v_flags>%u</v_flags>\n", vbuf.v_flags);
302 cprintf("<v_view>%d</v_view>\n", vbuf.v_view);
303 client_write(HKEY("</visit>\n"));
308 void migr_export_message(long msgnum) {
310 struct CtdlMessage *msg;
312 long bytes_written = 0;
315 // We can use a static buffer here because there will never be more than
316 // one of this operation happening at any given time, and it's really best
317 // to just keep it allocated once instead of torturing malloc/free.
318 // Call this function with msgnum "-1" to free the buffer when finished.
320 static int encoded_alloc = 0;
321 static char *encoded_msg = NULL;
324 if ((encoded_alloc == 0) && (encoded_msg != NULL)) {
332 // Ok, here we go ...
334 msg = CtdlFetchMessage(msgnum, 1);
335 if (msg == NULL) return; // fail silently
337 client_write(HKEY("<message>\n"));
338 GetMetaData(&smi, msgnum);
339 cprintf("<msg_msgnum>%ld</msg_msgnum>\n", msgnum);
340 cprintf("<msg_meta_refcount>%d</msg_meta_refcount>\n", smi.meta_refcount);
341 cprintf("<msg_meta_rfc822_length>%ld</msg_meta_rfc822_length>\n", smi.meta_rfc822_length);
342 client_write(HKEY("<msg_meta_content_type>")); xml_strout(smi.meta_content_type); client_write(HKEY("</msg_meta_content_type>\n"));
344 client_write(HKEY("<msg_text>"));
345 CtdlSerializeMessage(&smr, msg);
348 // Predict the buffer size we need. Expand the buffer if necessary.
349 int encoded_len = smr.len * 15 / 10 ;
350 if (encoded_len > encoded_alloc) {
351 encoded_alloc = encoded_len;
352 encoded_msg = realloc(encoded_msg, encoded_alloc);
355 if (encoded_msg == NULL) {
356 // Questionable hack that hopes it'll work next time and we only lose one message
360 // Once we do the encoding we know the exact size
361 encoded_len = CtdlEncodeBase64(encoded_msg, (char *)smr.ser, smr.len, 1);
363 // Careful now. If the message is gargantuan, trying to write multiple gigamegs in one
364 // big write operation can make our transport unhappy. So we'll chunk it up 10 KB at a time.
366 while ( (bytes_written < encoded_len) && (!server_shutting_down) ) {
367 this_block = encoded_len - bytes_written;
368 if (this_block > 10240) {
371 client_write(&encoded_msg[bytes_written], this_block);
372 bytes_written += this_block;
378 client_write(HKEY("</msg_text>\n"));
379 client_write(HKEY("</message>\n"));
383 void migr_export_openids(void) {
384 struct cdbdata *cdboi;
388 cdb_rewind(CDB_EXTAUTH);
389 while (cdboi = cdb_next_item(CDB_EXTAUTH), cdboi != NULL) {
390 if (cdboi->len > sizeof(long)) {
391 client_write(HKEY("<openid>\n"));
392 memcpy(&usernum, cdboi->ptr, sizeof(long));
393 snprintf(url, sizeof url, "%s", (cdboi->ptr)+sizeof(long) );
394 client_write(HKEY("<oid_url>"));
396 client_write(HKEY("</oid_url>\n"));
397 cprintf("<oid_usernum>%ld</oid_usernum>\n", usernum);
398 client_write(HKEY("</openid>\n"));
405 void migr_export_configs(void) {
406 struct cdbdata *cdbcfg;
411 cdb_rewind(CDB_CONFIG);
412 while (cdbcfg = cdb_next_item(CDB_CONFIG), cdbcfg != NULL) {
414 keylen = strlen(cdbcfg->ptr);
416 value = cdbcfg->ptr + keylen + 1;
418 client_write("<config key=\"", 13);
420 client_write("\">", 2);
422 client_write("</config>\n", 10);
428 void migr_export_messages(void) {
433 int prev_progress = 0;
437 migr_global_message_list = fopen(migr_tempfilename1, "r");
438 if (migr_global_message_list != NULL) {
439 syslog(LOG_INFO, "migrate: opened %s", migr_tempfilename1);
440 while ((Ctx->kill_me == 0) &&
441 (fgets(buf, sizeof(buf), migr_global_message_list) != NULL)) {
444 migr_export_message(msgnum);
447 progress = (count * 74 / total_msgs) + 25 ;
448 if ((progress > prev_progress) && (progress < 100)) {
449 cprintf("<progress>%d</progress>\n", progress);
451 prev_progress = progress;
453 fclose(migr_global_message_list);
455 if (Ctx->kill_me == 0) {
456 syslog(LOG_INFO, "migrate: exported %d messages.", count);
459 syslog(LOG_ERR, "migrate: export aborted due to client disconnect!");
462 migr_export_message(-1L); /* This frees the encoding buffer */
466 void migr_do_export(void) {
470 cprintf("%d Exporting all Citadel databases.\n", LISTING_FOLLOWS);
473 client_write(HKEY("<?xml version=\"1.0\" encoding=\"UTF-8\" ?>\n"));
474 client_write(HKEY("<citadel_migrate_data>\n"));
475 cprintf("<version>%d</version>\n", REV_LEVEL);
476 cprintf("<progress>%d</progress>\n", 0);
478 /* export the configuration database */
479 migr_export_configs();
480 cprintf("<progress>%d</progress>\n", 2);
482 if (Ctx->kill_me == 0) migr_export_users();
483 cprintf("<progress>%d</progress>\n", 7);
484 if (Ctx->kill_me == 0) migr_export_openids();
485 cprintf("<progress>%d</progress>\n", 12);
486 if (Ctx->kill_me == 0) migr_export_rooms();
487 cprintf("<progress>%d</progress>\n", 17);
488 if (Ctx->kill_me == 0) migr_export_floors();
489 cprintf("<progress>%d</progress>\n", 18);
490 if (Ctx->kill_me == 0) migr_export_visits();
491 cprintf("<progress>%d</progress>\n", 25);
492 if (Ctx->kill_me == 0) migr_export_messages();
493 cprintf("<progress>%d</progress>\n", 100);
494 client_write(HKEY("</citadel_migrate_data>\n"));
495 client_write(HKEY("000\n"));
500 /******************************************************************************
502 * Here's the code that implements the import side. It's going to end up *
503 * being one big loop with lots of global variables. I don't care. *
504 * You wouldn't run multiple concurrent imports anyway. If this offends your *
505 * delicate sensibilities then go rewrite it in Ruby on Rails or something. *
506 ******************************************************************************/
509 int citadel_migrate_data = 0; /* Are we inside a <citadel_migrate_data> tag pair? */
510 StrBuf *migr_chardata = NULL;
511 StrBuf *migr_MsgData = NULL;
512 struct ctdluser usbuf;
513 struct ctdlroom qrbuf;
514 char openid_url[512];
515 long openid_usernum = 0;
516 char FRname[ROOMNAMELEN];
521 long import_msgnum = 0;
524 * This callback stores up the data which appears in between tags.
526 void migr_xml_chardata(void *data, const XML_Char *s, int len) {
527 StrBufAppendBufPlain(migr_chardata, s, len, 0);
531 void migr_xml_start(void *data, const char *el, const char **attr) {
534 /*** GENERAL STUFF ***/
536 /* Reset chardata_len to zero and init buffer */
537 FlushStrBuf(migr_chardata);
538 FlushStrBuf(migr_MsgData);
540 if (!strcasecmp(el, "citadel_migrate_data")) {
541 ++citadel_migrate_data;
543 /* As soon as it looks like the input data is a genuine Citadel XML export,
544 * whack the existing database on disk to make room for the new one.
546 if (citadel_migrate_data == 1) {
547 for (i = 0; i < MAXCDB; ++i) {
554 if (citadel_migrate_data != 1) {
555 syslog(LOG_ERR, "migrate: out-of-sequence tag <%s> detected. Warning: ODD-DATA!", el);
559 /* When we begin receiving XML for one of these record types, clear out the associated
560 * buffer so we don't accidentally carry over any data from a previous record.
562 if (!strcasecmp(el, "user")) memset(&usbuf, 0, sizeof (struct ctdluser));
563 else if (!strcasecmp(el, "openid")) memset(openid_url, 0, sizeof openid_url);
564 else if (!strcasecmp(el, "room")) memset(&qrbuf, 0, sizeof (struct ctdlroom));
565 else if (!strcasecmp(el, "room_messages")) memset(FRname, 0, sizeof FRname);
566 else if (!strcasecmp(el, "floor")) memset(&flbuf, 0, sizeof (struct floor));
567 else if (!strcasecmp(el, "visit")) memset(&vbuf, 0, sizeof (visit));
569 else if (!strcasecmp(el, "message")) {
570 memset(&smi, 0, sizeof (struct MetaData));
573 else if (!strcasecmp(el, "config")) {
579 if (!strcasecmp(attr[0], "key")) {
580 ikey = strdup(attr[1]);
589 int migr_userrecord(void *data, const char *el) {
590 if (!strcasecmp(el, "u_version")) usbuf.version = atoi(ChrPtr(migr_chardata));
591 else if (!strcasecmp(el, "u_uid")) usbuf.uid = atol(ChrPtr(migr_chardata));
592 else if (!strcasecmp(el, "u_password")) safestrncpy(usbuf.password, ChrPtr(migr_chardata), sizeof usbuf.password);
593 else if (!strcasecmp(el, "u_flags")) usbuf.flags = atoi(ChrPtr(migr_chardata));
594 else if (!strcasecmp(el, "u_timescalled")) usbuf.timescalled = atol(ChrPtr(migr_chardata));
595 else if (!strcasecmp(el, "u_posted")) usbuf.posted = atol(ChrPtr(migr_chardata));
596 else if (!strcasecmp(el, "u_axlevel")) usbuf.axlevel = atoi(ChrPtr(migr_chardata));
597 else if (!strcasecmp(el, "u_usernum")) usbuf.usernum = atol(ChrPtr(migr_chardata));
598 else if (!strcasecmp(el, "u_lastcall")) usbuf.lastcall = atol(ChrPtr(migr_chardata));
599 else if (!strcasecmp(el, "u_USuserpurge")) usbuf.USuserpurge = atoi(ChrPtr(migr_chardata));
600 else if (!strcasecmp(el, "u_fullname")) safestrncpy(usbuf.fullname, ChrPtr(migr_chardata), sizeof usbuf.fullname);
601 else if (!strcasecmp(el, "u_msgnum_bio")) usbuf.msgnum_bio = atol(ChrPtr(migr_chardata));
602 else if (!strcasecmp(el, "u_msgnum_pic")) usbuf.msgnum_pic = atol(ChrPtr(migr_chardata));
603 else if (!strcasecmp(el, "u_emailaddrs")) safestrncpy(usbuf.emailaddrs, ChrPtr(migr_chardata), sizeof usbuf.emailaddrs);
604 else if (!strcasecmp(el, "u_msgnum_inboxrules")) usbuf.msgnum_inboxrules = atol(ChrPtr(migr_chardata));
605 else if (!strcasecmp(el, "u_lastproc_inboxrules")) usbuf.lastproc_inboxrules = atol(ChrPtr(migr_chardata));
611 int migr_roomrecord(void *data, const char *el) {
612 if (!strcasecmp(el, "QRname")) safestrncpy(qrbuf.QRname, ChrPtr(migr_chardata), sizeof qrbuf.QRname);
613 else if (!strcasecmp(el, "QRpasswd")) safestrncpy(qrbuf.QRpasswd, ChrPtr(migr_chardata), sizeof qrbuf.QRpasswd);
614 else if (!strcasecmp(el, "QRroomaide")) qrbuf.QRroomaide = atol(ChrPtr(migr_chardata));
615 else if (!strcasecmp(el, "QRhighest")) qrbuf.QRhighest = atol(ChrPtr(migr_chardata));
616 else if (!strcasecmp(el, "QRgen")) qrbuf.QRgen = atol(ChrPtr(migr_chardata));
617 else if (!strcasecmp(el, "QRflags")) qrbuf.QRflags = atoi(ChrPtr(migr_chardata));
618 else if (!strcasecmp(el, "QRdirname")) safestrncpy(qrbuf.QRdirname, ChrPtr(migr_chardata), sizeof qrbuf.QRdirname);
619 else if (!strcasecmp(el, "QRfloor")) qrbuf.QRfloor = atoi(ChrPtr(migr_chardata));
620 else if (!strcasecmp(el, "QRmtime")) qrbuf.QRmtime = atol(ChrPtr(migr_chardata));
621 else if (!strcasecmp(el, "QRexpire_mode")) qrbuf.QRep.expire_mode = atoi(ChrPtr(migr_chardata));
622 else if (!strcasecmp(el, "QRexpire_value")) qrbuf.QRep.expire_value = atoi(ChrPtr(migr_chardata));
623 else if (!strcasecmp(el, "QRnumber")) qrbuf.QRnumber = atol(ChrPtr(migr_chardata));
624 else if (!strcasecmp(el, "QRorder")) qrbuf.QRorder = atoi(ChrPtr(migr_chardata));
625 else if (!strcasecmp(el, "QRflags2")) qrbuf.QRflags2 = atoi(ChrPtr(migr_chardata));
626 else if (!strcasecmp(el, "QRdefaultview")) qrbuf.QRdefaultview = atoi(ChrPtr(migr_chardata));
627 else if (!strcasecmp(el, "msgnum_info")) qrbuf.msgnum_info = atol(ChrPtr(migr_chardata));
628 else if (!strcasecmp(el, "msgnum_pic")) qrbuf.msgnum_pic = atol(ChrPtr(migr_chardata));
634 int migr_floorrecord(void *data, const char *el) {
635 if (!strcasecmp(el, "f_num")) floornum = atoi(ChrPtr(migr_chardata));
636 else if (!strcasecmp(el, "f_flags")) flbuf.f_flags = atoi(ChrPtr(migr_chardata));
637 else if (!strcasecmp(el, "f_name")) safestrncpy(flbuf.f_name, ChrPtr(migr_chardata), sizeof flbuf.f_name);
638 else if (!strcasecmp(el, "f_ref_count")) flbuf.f_ref_count = atoi(ChrPtr(migr_chardata));
639 else if (!strcasecmp(el, "f_ep_expire_mode")) flbuf.f_ep.expire_mode = atoi(ChrPtr(migr_chardata));
640 else if (!strcasecmp(el, "f_ep_expire_value")) flbuf.f_ep.expire_value = atoi(ChrPtr(migr_chardata));
646 int migr_visitrecord(void *data, const char *el) {
647 if (!strcasecmp(el, "v_roomnum")) vbuf.v_roomnum = atol(ChrPtr(migr_chardata));
648 else if (!strcasecmp(el, "v_roomgen")) vbuf.v_roomgen = atol(ChrPtr(migr_chardata));
649 else if (!strcasecmp(el, "v_usernum")) vbuf.v_usernum = atol(ChrPtr(migr_chardata));
651 else if (!strcasecmp(el, "v_seen")) {
652 int is_textual_seen = 0;
654 int max = StrLength(migr_chardata);
656 vbuf.v_lastseen = atol(ChrPtr(migr_chardata));
658 for (i=0; i < max; ++i)
659 if (!isdigit(ChrPtr(migr_chardata)[i]))
662 safestrncpy(vbuf.v_seen, ChrPtr(migr_chardata), sizeof vbuf.v_seen);
665 else if (!strcasecmp(el, "v_answered")) safestrncpy(vbuf.v_answered, ChrPtr(migr_chardata), sizeof vbuf.v_answered);
666 else if (!strcasecmp(el, "v_flags")) vbuf.v_flags = atoi(ChrPtr(migr_chardata));
667 else if (!strcasecmp(el, "v_view")) vbuf.v_view = atoi(ChrPtr(migr_chardata));
673 void migr_xml_end(void *data, const char *el) {
677 long *msglist = NULL;
678 int msglist_alloc = 0;
679 /*** GENERAL STUFF ***/
681 if (!strcasecmp(el, "citadel_migrate_data")) {
682 --citadel_migrate_data;
686 if (citadel_migrate_data != 1) {
687 syslog(LOG_ERR, "migrate: out-of-sequence tag <%s> detected. Warning: ODD-DATA!", el);
691 // syslog(LOG_DEBUG, "END TAG: <%s> DATA: <%s>\n", el, (migr_chardata_len ? migr_chardata : ""));
695 if (!strcasecmp(el, "config")) {
696 syslog(LOG_DEBUG, "migrate: imported config key=%s", ikey);
699 CtdlSetConfigStr(ikey, (char *)ChrPtr(migr_chardata));
704 syslog(LOG_INFO, "migrate: closed a <config> tag but no key name was supplied.");
709 else if ((!strncasecmp(el, HKEY("u_"))) &&
710 migr_userrecord(data, el))
711 ; /* Nothing to do anymore */
712 else if (!strcasecmp(el, "user")) {
714 syslog(LOG_INFO, "migrate: imported user: %s", usbuf.fullname);
719 else if (!strcasecmp(el, "oid_url")) safestrncpy(openid_url, ChrPtr(migr_chardata), sizeof openid_url);
720 else if (!strcasecmp(el, "oid_usernum")) openid_usernum = atol(ChrPtr(migr_chardata));
722 else if (!strcasecmp(el, "openid")) { /* see serv_openid_rp.c for a description of the record format */
725 oid_data_len = sizeof(long) + strlen(openid_url) + 1;
726 oid_data = malloc(oid_data_len);
727 memcpy(oid_data, &openid_usernum, sizeof(long));
728 memcpy(&oid_data[sizeof(long)], openid_url, strlen(openid_url) + 1);
729 cdb_store(CDB_EXTAUTH, openid_url, strlen(openid_url), oid_data, oid_data_len);
731 syslog(LOG_INFO, "migrate: imported OpenID: %s (%ld)", openid_url, openid_usernum);
735 else if ((!strncasecmp(el, HKEY("QR"))) &&
736 migr_roomrecord(data, el))
737 ; /* Nothing to do anymore */
738 else if (!strcasecmp(el, "room")) {
740 syslog(LOG_INFO, "migrate: imported room: %s", qrbuf.QRname);
743 /*** ROOM MESSAGE POINTERS ***/
745 else if (!strcasecmp(el, "FRname")) {
746 safestrncpy(FRname, ChrPtr(migr_chardata), sizeof FRname);
749 else if (!strcasecmp(el, "FRmsglist")) {
750 if (!IsEmptyStr(FRname)) {
752 msglist_alloc = 1000;
753 msglist = malloc(sizeof(long) * msglist_alloc);
755 syslog(LOG_DEBUG, "migrate: message list for: %s", FRname);
757 ptr = ChrPtr(migr_chardata);
759 while ((*ptr != 0) && (!isdigit(*ptr))) {
762 if ((*ptr != 0) && (isdigit(*ptr))) {
765 if (msgcount >= msglist_alloc) {
767 msglist = realloc(msglist, sizeof(long) * msglist_alloc);
769 msglist[msgcount++] = msgnum;
772 while ((*ptr != 0) && (isdigit(*ptr))) {
778 CtdlSaveMsgPointersInRoom(FRname, msglist, msgcount, 0, NULL, 1);
783 syslog(LOG_DEBUG, "migrate: imported %d messages.", msgcount);
784 if (server_shutting_down) {
790 else if ((!strncasecmp(el, HKEY("f_"))) &&
791 migr_floorrecord(data, el))
792 ; /* Nothing to do anymore */
794 else if (!strcasecmp(el, "floor")) {
795 CtdlPutFloor(&flbuf, floornum);
796 syslog(LOG_INFO, "migrate: imported floor #%d (%s)", floornum, flbuf.f_name);
800 else if ((!strncasecmp(el, HKEY("v_"))) && migr_visitrecord(data, el)) {
801 ; /* Nothing to do anymore */
803 else if (!strcasecmp(el, "visit")) {
805 syslog(LOG_INFO, "migrate: imported visit: %ld/%ld/%ld", vbuf.v_roomnum, vbuf.v_roomgen, vbuf.v_usernum);
810 else if (!strcasecmp(el, "msg_msgnum")) smi.meta_msgnum = import_msgnum = atol(ChrPtr(migr_chardata));
811 else if (!strcasecmp(el, "msg_meta_refcount")) smi.meta_refcount = atoi(ChrPtr(migr_chardata));
812 else if (!strcasecmp(el, "msg_meta_rfc822_length")) smi.meta_rfc822_length = atoi(ChrPtr(migr_chardata));
813 else if (!strcasecmp(el, "msg_meta_content_type")) safestrncpy(smi.meta_content_type, ChrPtr(migr_chardata), sizeof smi.meta_content_type);
815 else if (!strcasecmp(el, "msg_text")) {
817 struct CtdlMessage *msg;
819 FlushStrBuf(migr_MsgData);
820 StrBufDecodeBase64To(migr_chardata, migr_MsgData);
822 msg = CtdlDeserializeMessage(import_msgnum, -1,
823 ChrPtr(migr_MsgData),
824 StrLength(migr_MsgData));
826 rc = CtdlSaveThisMessage(msg, import_msgnum, 0);
837 "migrate: %s message #%ld, size=%d, refcount=%d, bodylength=%ld, content-type: %s",
838 (rc!= 0)?"failed to import ":"Imported ",
840 StrLength(migr_MsgData),
842 smi.meta_rfc822_length,
843 smi.meta_content_type
845 memset(&smi, 0, sizeof(smi));
848 /*** MORE GENERAL STUFF ***/
850 FlushStrBuf(migr_chardata);
857 void migr_do_import(void) {
862 migr_chardata = NewStrBufPlain(NULL, SIZ * 20);
863 migr_MsgData = NewStrBufPlain(NULL, SIZ * 20);
864 xp = XML_ParserCreate(NULL);
866 cprintf("%d Failed to create XML parser instance\n", ERROR+INTERNAL_ERROR);
869 XML_SetElementHandler(xp, migr_xml_start, migr_xml_end);
870 XML_SetCharacterDataHandler(xp, migr_xml_chardata);
874 cprintf("%d sock it to me\n", SEND_LISTING);
877 client_set_inbound_buf(SIZ * 10);
879 while (client_getln(buf, sizeof buf) >= 0 && strcmp(buf, "000")) {
880 XML_Parse(xp, buf, strlen(buf), 0);
881 if (server_shutting_down)
882 break; // Should we break or return?
885 XML_Parse(xp, "", 0, 1);
887 FreeStrBuf(&migr_chardata);
888 FreeStrBuf(&migr_MsgData);
889 rebuild_euid_index();
890 rebuild_usersbynumber();
891 CtdlSetConfigInt("MM_fulltext_wordbreaker", -1); // Set an invalid wordbreaker to force re-indexing
896 /******************************************************************************
897 * Dispatcher, Common code *
898 ******************************************************************************/
900 * Dump out the pathnames of directories which can be copied "as is"
902 void migr_do_listdirs(void) {
903 cprintf("%d Don't forget these:\n", LISTING_FOLLOWS);
904 cprintf("files|%s\n", ctdl_file_dir);
905 cprintf("messages|%s\n", ctdl_message_dir);
906 cprintf("keys|%s\n", ctdl_key_dir);
911 /******************************************************************************
912 * Repair database integrity *
913 ******************************************************************************/
915 StrBuf *PlainMessageBuf = NULL;
916 HashList *UsedMessageIDS = NULL;
918 int migr_restore_message_metadata(long msgnum, int refcount) {
920 struct CtdlMessage *msg;
923 /* We can use a static buffer here because there will never be more than
924 * one of this operation happening at any given time, and it's really best
925 * to just keep it allocated once instead of torturing malloc/free.
926 * Call this function with msgnum "-1" to free the buffer when finished.
928 static int encoded_alloc = 0;
929 static char *encoded_msg = NULL;
932 if ((encoded_alloc == 0) && (encoded_msg != NULL)) {
936 // todo FreeStrBuf(&PlainMessageBuf); PlainMessageBuf = NULL;
941 if (PlainMessageBuf == NULL) {
942 PlainMessageBuf = NewStrBufPlain(NULL, 10*SIZ);
945 /* Ok, here we go ... */
947 msg = CtdlFetchMessage(msgnum, 1);
952 GetMetaData(&smi, msgnum);
953 smi.meta_msgnum = msgnum;
954 smi.meta_refcount = refcount;
956 /* restore the content type from the message body: */
957 mptr = bmstrcasestr(msg->cm_fields[eMesageText], "Content-type:");
960 safestrncpy(smi.meta_content_type, &mptr[13], sizeof smi.meta_content_type);
961 striplt(smi.meta_content_type);
962 aptr = smi.meta_content_type;
963 while (!IsEmptyStr(aptr)) {
968 memset(aptr, 0, sizeof(smi.meta_content_type) - (aptr - smi.meta_content_type));
974 CC->redirect_buffer = PlainMessageBuf;
975 CtdlOutputPreLoadedMsg(msg, MT_RFC822, HEADERS_ALL, 0, 1, QP_EADDR);
976 smi.meta_rfc822_length = StrLength(CC->redirect_buffer);
977 CC->redirect_buffer = NULL;
980 "migrate: setting message #%ld meta data to: refcount=%d, bodylength=%ld, content-type: %s",
983 smi.meta_rfc822_length,
984 smi.meta_content_type
993 void migr_check_room_msg(long msgnum, void *userdata) {
994 fprintf(migr_global_message_list, "%ld %s\n", msgnum, CC->room.QRname);
998 void migr_check_rooms_backend(struct ctdlroom *buf, void *data) {
999 /* message list goes inside this tag */
1000 CtdlGetRoom(&CC->room, buf->QRname);
1001 CtdlForEachMessage(MSGS_ALL, 0L, NULL, NULL, NULL, migr_check_room_msg, NULL);
1005 void RemoveMessagesFromRooms(StrBuf *RoomNameVec, long msgnum) {
1006 struct MetaData smi;
1007 const char *Pos = NULL;
1008 StrBuf *oneRoom = NewStrBuf();
1010 syslog(LOG_INFO, "migrate: removing message pointer %ld from these rooms: %s", msgnum, ChrPtr(RoomNameVec));
1012 while (Pos != StrBufNOTNULL){
1013 StrBufExtract_NextToken(oneRoom, RoomNameVec, &Pos, '|');
1014 CtdlDeleteMessages(ChrPtr(oneRoom), &msgnum, 1, "");
1016 GetMetaData(&smi, msgnum);
1017 AdjRefCount(msgnum, -smi.meta_refcount);
1021 void migr_do_restore_meta(void) {
1032 migr_global_message_list = fopen(migr_tempfilename1, "w");
1033 if (migr_global_message_list != NULL) {
1034 CtdlForEachRoom(migr_check_rooms_backend, NULL);
1035 fclose(migr_global_message_list);
1039 * Process the 'global' message list. (Sort it and remove dups.
1040 * Dups are ok because a message may be in more than one room, but
1041 * this will be handled by exporting the reference count, not by
1042 * exporting the message multiple times.)
1044 snprintf(cmd, sizeof cmd, "sort -n <%s >%s", migr_tempfilename1, migr_tempfilename2);
1045 if (system(cmd) != 0) {
1046 syslog(LOG_ERR, "migrate: error %d", errno);
1049 RoomNames = NewStrBuf();
1051 migr_global_message_list = fopen(migr_tempfilename2, "r");
1052 if (migr_global_message_list != NULL) {
1053 syslog(LOG_INFO, "migrate: opened %s", migr_tempfilename1);
1054 while ((Ctx->kill_me == 0) &&
1055 (fgets(buf, sizeof(buf), migr_global_message_list) != NULL)) {
1062 prn = strchr(buf, ' ');
1063 if (lastnum != msgnum) {
1064 failGetMessage = migr_restore_message_metadata(lastnum, refcount);
1065 if (failGetMessage) {
1066 RemoveMessagesFromRooms(RoomNames, lastnum);
1071 StrBufPlain(RoomNames, prn + 1, -1);
1073 StrBufTrim(RoomNames);
1077 if (StrLength(RoomNames) > 0) {
1078 StrBufAppendBufPlain(RoomNames, HKEY("|"), 0);
1080 StrBufAppendBufPlain(RoomNames, prn, -1, 1);
1081 StrBufTrim(RoomNames);
1087 failGetMessage = migr_restore_message_metadata(msgnum, refcount);
1088 if (failGetMessage) {
1089 RemoveMessagesFromRooms(RoomNames, lastnum);
1091 fclose(migr_global_message_list);
1094 migr_restore_message_metadata(-1L, -1); /* This frees the encoding buffer */
1095 cprintf("%d system analysis completed", CIT_OK);
1100 /******************************************************************************
1101 * Dispatcher, Common code *
1102 ******************************************************************************/
1103 void cmd_migr(char *cmdbuf) {
1106 if (CtdlAccessCheck(ac_internal)) return;
1108 if (CtdlTrySingleUser()) {
1109 CtdlDisableHouseKeeping();
1110 CtdlMakeTempFileName(migr_tempfilename1, sizeof migr_tempfilename1);
1111 CtdlMakeTempFileName(migr_tempfilename2, sizeof migr_tempfilename2);
1113 extract_token(cmd, cmdbuf, 0, '|', sizeof cmd);
1114 if (!strcasecmp(cmd, "export")) {
1117 else if (!strcasecmp(cmd, "import")) {
1120 else if (!strcasecmp(cmd, "listdirs")) {
1123 else if (!strcasecmp(cmd, "restoremeta")) {
1124 migr_do_restore_meta();
1127 cprintf("%d illegal command\n", ERROR + ILLEGAL_VALUE);
1130 unlink(migr_tempfilename1);
1131 unlink(migr_tempfilename2);
1133 CtdlEnableHouseKeeping();
1134 CtdlEndSingleUser();
1137 cprintf("%d The migrator is already running.\n", ERROR + RESOURCE_BUSY);
1142 /******************************************************************************
1144 ******************************************************************************/
1146 CTDL_MODULE_INIT(migrate)
1150 CtdlRegisterProtoHook(cmd_migr, "MIGR", "Across-the-wire migration");
1153 /* return our module name for the log */