2 * This module dumps and/or loads the Citadel database in XML format.
4 * Copyright (c) 1987-2015 by the citadel.org team
6 * This program is open source software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License version 3.
9 * This program is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 * GNU General Public License for more details.
16 * Explanation of <progress> tags:
19 * 2% finished exporting configuration
20 * 7% finished exporting users
21 * 12% finished exporting openids
22 * 17% finished exporting rooms
23 * 18% finished exporting floors
24 * 25% finished exporting visits
25 * 100% finished exporting messages
36 #include <sys/types.h>
38 #if TIME_WITH_SYS_TIME
39 # include <sys/time.h>
43 # include <sys/time.h>
54 #include <libcitadel.h>
57 #include "citserver.h"
63 #include "euidindex.h"
64 #include "ctdl_module.h"
66 #define END_OF_MESSAGE "---eom---dbd---"
68 char migr_tempfilename1[PATH_MAX];
69 char migr_tempfilename2[PATH_MAX];
70 FILE *migr_global_message_list;
74 /******************************************************************************
75 * Code which implements the export appears in this section *
76 ******************************************************************************/
79 * Output a string to the client with these characters escaped: & < > " '
81 void xml_strout(char *str) {
91 client_write(HKEY("""));
93 else if (*c == '\'') {
94 client_write(HKEY("'"));
97 client_write(HKEY("<"));
100 client_write(HKEY(">"));
102 else if (*c == '&') {
103 client_write(HKEY("&"));
114 * Export a user record as XML
116 void migr_export_users_backend(struct ctdluser *buf, void *data) {
117 client_write(HKEY("<user>\n"));
118 cprintf("<u_version>%d</u_version>\n", buf->version);
119 cprintf("<u_uid>%ld</u_uid>\n", (long)buf->uid);
120 client_write(HKEY("<u_password>")); xml_strout(buf->password); client_write(HKEY("</u_password>\n"));
121 cprintf("<u_flags>%u</u_flags>\n", buf->flags);
122 cprintf("<u_timescalled>%ld</u_timescalled>\n", buf->timescalled);
123 cprintf("<u_posted>%ld</u_posted>\n", buf->posted);
124 cprintf("<u_axlevel>%d</u_axlevel>\n", buf->axlevel);
125 cprintf("<u_usernum>%ld</u_usernum>\n", buf->usernum);
126 cprintf("<u_lastcall>%ld</u_lastcall>\n", (long)buf->lastcall);
127 cprintf("<u_USuserpurge>%d</u_USuserpurge>\n", buf->USuserpurge);
128 client_write(HKEY("<u_fullname>")); xml_strout(buf->fullname); client_write(HKEY("</u_fullname>\n"));
129 client_write(HKEY("</user>\n"));
133 void migr_export_users(void) {
134 ForEachUser(migr_export_users_backend, NULL);
138 void migr_export_room_msg(long msgnum, void *userdata) {
139 cprintf("%ld\n", msgnum);
140 fprintf(migr_global_message_list, "%ld\n", msgnum);
144 void migr_export_rooms_backend(struct ctdlroom *buf, void *data) {
145 client_write(HKEY("<room>\n"));
146 client_write(HKEY("<QRname>")); xml_strout(buf->QRname); client_write(HKEY("</QRname>\n"));
147 client_write(HKEY("<QRpasswd>")); xml_strout(buf->QRpasswd); client_write(HKEY("</QRpasswd>\n"));
148 cprintf("<QRroomaide>%ld</QRroomaide>\n", buf->QRroomaide);
149 cprintf("<QRhighest>%ld</QRhighest>\n", buf->QRhighest);
150 cprintf("<QRgen>%ld</QRgen>\n", (long)buf->QRgen);
151 cprintf("<QRflags>%u</QRflags>\n", buf->QRflags);
152 if (buf->QRflags & QR_DIRECTORY) {
153 client_write(HKEY("<QRdirname>"));
154 xml_strout(buf->QRdirname);
155 client_write(HKEY("</QRdirname>\n"));
157 cprintf("<QRinfo>%ld</QRinfo>\n", buf->QRinfo);
158 cprintf("<QRfloor>%d</QRfloor>\n", buf->QRfloor);
159 cprintf("<QRmtime>%ld</QRmtime>\n", (long)buf->QRmtime);
160 cprintf("<QRexpire_mode>%d</QRexpire_mode>\n", buf->QRep.expire_mode);
161 cprintf("<QRexpire_value>%d</QRexpire_value>\n", buf->QRep.expire_value);
162 cprintf("<QRnumber>%ld</QRnumber>\n", buf->QRnumber);
163 cprintf("<QRorder>%d</QRorder>\n", buf->QRorder);
164 cprintf("<QRflags2>%u</QRflags2>\n", buf->QRflags2);
165 cprintf("<QRdefaultview>%d</QRdefaultview>\n", buf->QRdefaultview);
166 client_write(HKEY("</room>\n"));
168 /* message list goes inside this tag */
170 CtdlGetRoom(&CC->room, buf->QRname);
171 client_write(HKEY("<room_messages>"));
172 client_write(HKEY("<FRname>")); xml_strout(CC->room.QRname); client_write(HKEY("</FRname>\n"));
173 client_write(HKEY("<FRmsglist>"));
174 CtdlForEachMessage(MSGS_ALL, 0L, NULL, NULL, NULL, migr_export_room_msg, NULL);
175 client_write(HKEY("</FRmsglist>\n"));
176 client_write(HKEY("</room_messages>\n"));
182 void migr_export_rooms(void) {
184 migr_global_message_list = fopen(migr_tempfilename1, "w");
185 if (migr_global_message_list != NULL) {
186 CtdlForEachRoom(migr_export_rooms_backend, NULL);
187 fclose(migr_global_message_list);
191 * Process the 'global' message list. (Sort it and remove dups.
192 * Dups are ok because a message may be in more than one room, but
193 * this will be handled by exporting the reference count, not by
194 * exporting the message multiple times.)
196 snprintf(cmd, sizeof cmd, "sort -n <%s >%s", migr_tempfilename1, migr_tempfilename2);
197 if (system(cmd) != 0) syslog(LOG_ALERT, "Error %d", errno);
198 snprintf(cmd, sizeof cmd, "uniq <%s >%s", migr_tempfilename2, migr_tempfilename1);
199 if (system(cmd) != 0) syslog(LOG_ALERT, "Error %d", errno);
202 snprintf(cmd, sizeof cmd, "wc -l %s", migr_tempfilename1);
203 FILE *fp = popen(cmd, "r");
205 fgets(cmd, sizeof cmd, fp);
207 total_msgs = atoi(cmd);
210 total_msgs = 1; // any nonzero just to keep it from barfing
212 syslog(LOG_DEBUG, "Total messages to be exported: %d", total_msgs);
216 void migr_export_floors(void) {
217 struct floor qfbuf, *buf;
220 for (i=0; i < MAXFLOORS; ++i) {
221 client_write(HKEY("<floor>\n"));
222 cprintf("<f_num>%d</f_num>\n", i);
223 CtdlGetFloor(&qfbuf, i);
225 cprintf("<f_flags>%u</f_flags>\n", buf->f_flags);
226 client_write(HKEY("<f_name>")); xml_strout(buf->f_name); client_write(HKEY("</f_name>\n"));
227 cprintf("<f_ref_count>%d</f_ref_count>\n", buf->f_ref_count);
228 cprintf("<f_ep_expire_mode>%d</f_ep_expire_mode>\n", buf->f_ep.expire_mode);
229 cprintf("<f_ep_expire_value>%d</f_ep_expire_value>\n", buf->f_ep.expire_value);
230 client_write(HKEY("</floor>\n"));
236 * Return nonzero if the supplied string contains only characters which are valid in a sequence set.
238 int is_sequence_set(char *s) {
243 while (ch = *c++, ch) {
244 if (!strchr("0123456789*,:", ch)) {
254 * Traverse the visits file...
256 void migr_export_visits(void) {
258 struct cdbdata *cdbv;
260 cdb_rewind(CDB_VISIT);
262 while (cdbv = cdb_next_item(CDB_VISIT), cdbv != NULL) {
263 memset(&vbuf, 0, sizeof(visit));
264 memcpy(&vbuf, cdbv->ptr,
265 ((cdbv->len > sizeof(visit)) ?
266 sizeof(visit) : cdbv->len));
269 client_write(HKEY("<visit>\n"));
270 cprintf("<v_roomnum>%ld</v_roomnum>\n", vbuf.v_roomnum);
271 cprintf("<v_roomgen>%ld</v_roomgen>\n", vbuf.v_roomgen);
272 cprintf("<v_usernum>%ld</v_usernum>\n", vbuf.v_usernum);
274 client_write(HKEY("<v_seen>"));
275 if ( (!IsEmptyStr(vbuf.v_seen)) && (is_sequence_set(vbuf.v_seen)) ) {
276 xml_strout(vbuf.v_seen);
279 cprintf("%ld", vbuf.v_lastseen);
281 client_write(HKEY("</v_seen>"));
283 if ( (!IsEmptyStr(vbuf.v_answered)) && (is_sequence_set(vbuf.v_answered)) ) {
284 client_write(HKEY("<v_answered>"));
285 xml_strout(vbuf.v_answered);
286 client_write(HKEY("</v_answered>\n"));
289 cprintf("<v_flags>%u</v_flags>\n", vbuf.v_flags);
290 cprintf("<v_view>%d</v_view>\n", vbuf.v_view);
291 client_write(HKEY("</visit>\n"));
296 void migr_export_message(long msgnum) {
298 struct CtdlMessage *msg;
301 /* We can use a static buffer here because there will never be more than
302 * one of this operation happening at any given time, and it's really best
303 * to just keep it allocated once instead of torturing malloc/free.
304 * Call this function with msgnum "-1" to free the buffer when finished.
306 static int encoded_alloc = 0;
307 static char *encoded_msg = NULL;
310 if ((encoded_alloc == 0) && (encoded_msg != NULL)) {
318 /* Ok, here we go ... */
320 msg = CtdlFetchMessage(msgnum, 1, 0);
321 if (msg == NULL) return; /* fail silently */
323 client_write(HKEY("<message>\n"));
324 GetMetaData(&smi, msgnum);
325 cprintf("<msg_msgnum>%ld</msg_msgnum>\n", msgnum);
326 cprintf("<msg_meta_refcount>%d</msg_meta_refcount>\n", smi.meta_refcount);
327 cprintf("<msg_meta_rfc822_length>%ld</msg_meta_rfc822_length>\n", smi.meta_rfc822_length);
328 client_write(HKEY("<msg_meta_content_type>")); xml_strout(smi.meta_content_type); client_write(HKEY("</msg_meta_content_type>\n"));
329 client_write(HKEY("<msg_mimetype>")); xml_strout(smi.mimetype); client_write(HKEY("</msg_mimetype>\n"));
331 client_write(HKEY("<msg_text>"));
332 CtdlSerializeMessage(&smr, msg);
335 /* Predict the buffer size we need. Expand the buffer if necessary. */
336 int encoded_len = smr.len * 15 / 10 ;
337 if (encoded_len > encoded_alloc) {
338 encoded_alloc = encoded_len;
339 encoded_msg = realloc(encoded_msg, encoded_alloc);
342 if (encoded_msg == NULL) {
343 /* Questionable hack that hopes it'll work next time and we only lose one message */
347 /* Once we do the encoding we know the exact size */
348 encoded_len = CtdlEncodeBase64(encoded_msg, (char *)smr.ser, smr.len, 1);
349 client_write(encoded_msg, encoded_len);
354 client_write(HKEY("</msg_text>\n"));
355 client_write(HKEY("</message>\n"));
360 void migr_export_openids(void) {
361 struct cdbdata *cdboi;
365 cdb_rewind(CDB_OPENID);
366 while (cdboi = cdb_next_item(CDB_OPENID), cdboi != NULL) {
367 if (cdboi->len > sizeof(long)) {
368 client_write(HKEY("<openid>\n"));
369 memcpy(&usernum, cdboi->ptr, sizeof(long));
370 snprintf(url, sizeof url, "%s", (cdboi->ptr)+sizeof(long) );
371 client_write(HKEY("<oid_url>"));
373 client_write(HKEY("</oid_url>\n"));
374 cprintf("<oid_usernum>%ld</oid_usernum>\n", usernum);
375 client_write(HKEY("</openid>\n"));
382 void migr_export_configs(void) {
383 struct cdbdata *cdbcfg;
388 cdb_rewind(CDB_CONFIG);
389 while (cdbcfg = cdb_next_item(CDB_CONFIG), cdbcfg != NULL) {
391 keylen = strlen(cdbcfg->ptr);
393 value = cdbcfg->ptr + keylen + 1;
395 client_write("<config key=\"", 13);
397 client_write("\">", 2);
399 client_write("</config>\n", 10);
407 void migr_export_messages(void) {
412 int prev_progress = 0;
416 migr_global_message_list = fopen(migr_tempfilename1, "r");
417 if (migr_global_message_list != NULL) {
418 syslog(LOG_INFO, "Opened %s", migr_tempfilename1);
419 while ((Ctx->kill_me == 0) &&
420 (fgets(buf, sizeof(buf), migr_global_message_list) != NULL)) {
423 migr_export_message(msgnum);
426 progress = (count * 74 / total_msgs) + 25 ;
427 if ((progress > prev_progress) && (progress < 100)) {
428 cprintf("<progress>%d</progress>\n", progress);
430 prev_progress = progress;
432 fclose(migr_global_message_list);
434 if (Ctx->kill_me == 0)
435 syslog(LOG_INFO, "Exported %d messages.", count);
437 syslog(LOG_ERR, "Export aborted due to client disconnect!");
439 migr_export_message(-1L); /* This frees the encoding buffer */
444 void migr_do_export(void) {
448 cprintf("%d Exporting all Citadel databases.\n", LISTING_FOLLOWS);
451 client_write(HKEY("<?xml version=\"1.0\" encoding=\"UTF-8\" ?>\n"));
452 client_write(HKEY("<citadel_migrate_data>\n"));
453 cprintf("<version>%d</version>\n", REV_LEVEL);
454 cprintf("<progress>%d</progress>\n", 0);
456 /* export the configuration database */
457 migr_export_configs();
458 cprintf("<progress>%d</progress>\n", 2);
460 if (Ctx->kill_me == 0) migr_export_users();
461 cprintf("<progress>%d</progress>\n", 7);
462 if (Ctx->kill_me == 0) migr_export_openids();
463 cprintf("<progress>%d</progress>\n", 12);
464 if (Ctx->kill_me == 0) migr_export_rooms();
465 cprintf("<progress>%d</progress>\n", 17);
466 if (Ctx->kill_me == 0) migr_export_floors();
467 cprintf("<progress>%d</progress>\n", 18);
468 if (Ctx->kill_me == 0) migr_export_visits();
469 cprintf("<progress>%d</progress>\n", 25);
470 if (Ctx->kill_me == 0) migr_export_messages();
471 client_write(HKEY("</citadel_migrate_data>\n"));
472 cprintf("<progress>%d</progress>\n", 100);
473 client_write(HKEY("000\n"));
480 /******************************************************************************
482 * Here's the code that implements the import side. It's going to end up *
483 * being one big loop with lots of global variables. I don't care. *
484 * You wouldn't run multiple concurrent imports anyway. If this offends your *
485 * delicate sensibilities then go rewrite it in Ruby on Rails or something. *
486 ******************************************************************************/
489 int citadel_migrate_data = 0; /* Are we inside a <citadel_migrate_data> tag pair? */
490 StrBuf *migr_chardata = NULL;
491 StrBuf *migr_MsgData = NULL;
492 struct ctdluser usbuf;
493 struct ctdlroom qrbuf;
494 char openid_url[512];
495 long openid_usernum = 0;
496 char FRname[ROOMNAMELEN];
501 long import_msgnum = 0;
504 * This callback stores up the data which appears in between tags.
506 void migr_xml_chardata(void *data, const XML_Char *s, int len)
508 StrBufAppendBufPlain(migr_chardata, s, len, 0);
512 void migr_xml_start(void *data, const char *el, const char **attr) {
515 /*** GENERAL STUFF ***/
517 /* Reset chardata_len to zero and init buffer */
518 FlushStrBuf(migr_chardata);
519 FlushStrBuf(migr_MsgData);
521 if (!strcasecmp(el, "citadel_migrate_data")) {
522 ++citadel_migrate_data;
524 /* As soon as it looks like the input data is a genuine Citadel XML export,
525 * whack the existing database on disk to make room for the new one.
527 if (citadel_migrate_data == 1) {
528 for (i = 0; i < MAXCDB; ++i) {
535 if (citadel_migrate_data != 1) {
536 syslog(LOG_ALERT, "Out-of-sequence tag <%s> detected. Warning: ODD-DATA!", el);
540 /* When we begin receiving XML for one of these record types, clear out the associated
541 * buffer so we don't accidentally carry over any data from a previous record.
543 if (!strcasecmp(el, "user")) memset(&usbuf, 0, sizeof (struct ctdluser));
544 else if (!strcasecmp(el, "openid")) memset(openid_url, 0, sizeof openid_url);
545 else if (!strcasecmp(el, "room")) memset(&qrbuf, 0, sizeof (struct ctdlroom));
546 else if (!strcasecmp(el, "room_messages")) memset(FRname, 0, sizeof FRname);
547 else if (!strcasecmp(el, "floor")) memset(&flbuf, 0, sizeof (struct floor));
548 else if (!strcasecmp(el, "visit")) memset(&vbuf, 0, sizeof (visit));
550 else if (!strcasecmp(el, "message")) {
551 memset(&smi, 0, sizeof (struct MetaData));
554 else if (!strcasecmp(el, "config")) {
555 syslog(LOG_DEBUG, "\033[31m IMPORT OF CONFIG START ELEMENT FIXME\033\0m");
561 int migr_userrecord(void *data, const char *el)
563 if (!strcasecmp(el, "u_version")) usbuf.version = atoi(ChrPtr(migr_chardata));
564 else if (!strcasecmp(el, "u_uid")) usbuf.uid = atol(ChrPtr(migr_chardata));
565 else if (!strcasecmp(el, "u_password")) safestrncpy(usbuf.password, ChrPtr(migr_chardata), sizeof usbuf.password);
566 else if (!strcasecmp(el, "u_flags")) usbuf.flags = atoi(ChrPtr(migr_chardata));
567 else if (!strcasecmp(el, "u_timescalled")) usbuf.timescalled = atol(ChrPtr(migr_chardata));
568 else if (!strcasecmp(el, "u_posted")) usbuf.posted = atol(ChrPtr(migr_chardata));
569 else if (!strcasecmp(el, "u_axlevel")) usbuf.axlevel = atoi(ChrPtr(migr_chardata));
570 else if (!strcasecmp(el, "u_usernum")) usbuf.usernum = atol(ChrPtr(migr_chardata));
571 else if (!strcasecmp(el, "u_lastcall")) usbuf.lastcall = atol(ChrPtr(migr_chardata));
572 else if (!strcasecmp(el, "u_USuserpurge")) usbuf.USuserpurge = atoi(ChrPtr(migr_chardata));
573 else if (!strcasecmp(el, "u_fullname")) safestrncpy(usbuf.fullname, ChrPtr(migr_chardata), sizeof usbuf.fullname);
578 int migr_roomrecord(void *data, const char *el)
580 if (!strcasecmp(el, "QRname")) safestrncpy(qrbuf.QRname, ChrPtr(migr_chardata), sizeof qrbuf.QRname);
581 else if (!strcasecmp(el, "QRpasswd")) safestrncpy(qrbuf.QRpasswd, ChrPtr(migr_chardata), sizeof qrbuf.QRpasswd);
582 else if (!strcasecmp(el, "QRroomaide")) qrbuf.QRroomaide = atol(ChrPtr(migr_chardata));
583 else if (!strcasecmp(el, "QRhighest")) qrbuf.QRhighest = atol(ChrPtr(migr_chardata));
584 else if (!strcasecmp(el, "QRgen")) qrbuf.QRgen = atol(ChrPtr(migr_chardata));
585 else if (!strcasecmp(el, "QRflags")) qrbuf.QRflags = atoi(ChrPtr(migr_chardata));
586 else if (!strcasecmp(el, "QRdirname")) safestrncpy(qrbuf.QRdirname, ChrPtr(migr_chardata), sizeof qrbuf.QRdirname);
587 else if (!strcasecmp(el, "QRinfo")) qrbuf.QRinfo = atol(ChrPtr(migr_chardata));
588 else if (!strcasecmp(el, "QRfloor")) qrbuf.QRfloor = atoi(ChrPtr(migr_chardata));
589 else if (!strcasecmp(el, "QRmtime")) qrbuf.QRmtime = atol(ChrPtr(migr_chardata));
590 else if (!strcasecmp(el, "QRexpire_mode")) qrbuf.QRep.expire_mode = atoi(ChrPtr(migr_chardata));
591 else if (!strcasecmp(el, "QRexpire_value")) qrbuf.QRep.expire_value = atoi(ChrPtr(migr_chardata));
592 else if (!strcasecmp(el, "QRnumber")) qrbuf.QRnumber = atol(ChrPtr(migr_chardata));
593 else if (!strcasecmp(el, "QRorder")) qrbuf.QRorder = atoi(ChrPtr(migr_chardata));
594 else if (!strcasecmp(el, "QRflags2")) qrbuf.QRflags2 = atoi(ChrPtr(migr_chardata));
595 else if (!strcasecmp(el, "QRdefaultview")) qrbuf.QRdefaultview = atoi(ChrPtr(migr_chardata));
600 int migr_floorrecord(void *data, const char *el)
602 if (!strcasecmp(el, "f_num")) floornum = atoi(ChrPtr(migr_chardata));
603 else if (!strcasecmp(el, "f_flags")) flbuf.f_flags = atoi(ChrPtr(migr_chardata));
604 else if (!strcasecmp(el, "f_name")) safestrncpy(flbuf.f_name, ChrPtr(migr_chardata), sizeof flbuf.f_name);
605 else if (!strcasecmp(el, "f_ref_count")) flbuf.f_ref_count = atoi(ChrPtr(migr_chardata));
606 else if (!strcasecmp(el, "f_ep_expire_mode")) flbuf.f_ep.expire_mode = atoi(ChrPtr(migr_chardata));
607 else if (!strcasecmp(el, "f_ep_expire_value")) flbuf.f_ep.expire_value = atoi(ChrPtr(migr_chardata));
612 int migr_visitrecord(void *data, const char *el)
614 if (!strcasecmp(el, "v_roomnum")) vbuf.v_roomnum = atol(ChrPtr(migr_chardata));
615 else if (!strcasecmp(el, "v_roomgen")) vbuf.v_roomgen = atol(ChrPtr(migr_chardata));
616 else if (!strcasecmp(el, "v_usernum")) vbuf.v_usernum = atol(ChrPtr(migr_chardata));
618 else if (!strcasecmp(el, "v_seen")) {
619 int is_textual_seen = 0;
621 int max = StrLength(migr_chardata);
623 vbuf.v_lastseen = atol(ChrPtr(migr_chardata));
625 for (i=0; i < max; ++i)
626 if (!isdigit(ChrPtr(migr_chardata)[i]))
629 safestrncpy(vbuf.v_seen, ChrPtr(migr_chardata), sizeof vbuf.v_seen);
632 else if (!strcasecmp(el, "v_answered")) safestrncpy(vbuf.v_answered, ChrPtr(migr_chardata), sizeof vbuf.v_answered);
633 else if (!strcasecmp(el, "v_flags")) vbuf.v_flags = atoi(ChrPtr(migr_chardata));
634 else if (!strcasecmp(el, "v_view")) vbuf.v_view = atoi(ChrPtr(migr_chardata));
640 void migr_xml_end(void *data, const char *el)
645 long *msglist = NULL;
646 int msglist_alloc = 0;
647 /*** GENERAL STUFF ***/
649 if (!strcasecmp(el, "citadel_migrate_data")) {
650 --citadel_migrate_data;
654 if (citadel_migrate_data != 1) {
655 syslog(LOG_ALERT, "Out-of-sequence tag <%s> detected. Warning: ODD-DATA!", el);
659 // syslog(LOG_DEBUG, "END TAG: <%s> DATA: <%s>\n", el, (migr_chardata_len ? migr_chardata : ""));
663 if (!strcasecmp(el, "config"))
665 syslog(LOG_DEBUG, "\033[31m IMPORT OF CONFIG END ELEMENT FIXME\033\0m");
666 CtdlSetConfigInt("c_enable_fulltext", 0); /* always disable FIXME put this somewhere more appropriate */
670 else if ((!strncasecmp(el, HKEY("u_"))) &&
671 migr_userrecord(data, el))
672 ; /* Nothing to do anymore */
673 else if (!strcasecmp(el, "user")) {
675 syslog(LOG_INFO, "Imported user: %s", usbuf.fullname);
680 else if (!strcasecmp(el, "oid_url")) safestrncpy(openid_url, ChrPtr(migr_chardata), sizeof openid_url);
681 else if (!strcasecmp(el, "oid_usernum")) openid_usernum = atol(ChrPtr(migr_chardata));
683 else if (!strcasecmp(el, "openid")) { /* see serv_openid_rp.c for a description of the record format */
686 oid_data_len = sizeof(long) + strlen(openid_url) + 1;
687 oid_data = malloc(oid_data_len);
688 memcpy(oid_data, &openid_usernum, sizeof(long));
689 memcpy(&oid_data[sizeof(long)], openid_url, strlen(openid_url) + 1);
690 cdb_store(CDB_OPENID, openid_url, strlen(openid_url), oid_data, oid_data_len);
692 syslog(LOG_INFO, "Imported OpenID: %s (%ld)", openid_url, openid_usernum);
696 else if ((!strncasecmp(el, HKEY("QR"))) &&
697 migr_roomrecord(data, el))
698 ; /* Nothing to do anymore */
699 else if (!strcasecmp(el, "room")) {
701 syslog(LOG_INFO, "Imported room: %s", qrbuf.QRname);
704 /*** ROOM MESSAGE POINTERS ***/
706 else if (!strcasecmp(el, "FRname")) safestrncpy(FRname, ChrPtr(migr_chardata), sizeof FRname);
708 else if (!strcasecmp(el, "FRmsglist")) {
709 if (!IsEmptyStr(FRname)) {
711 msglist_alloc = 1000;
712 msglist = malloc(sizeof(long) * msglist_alloc);
714 syslog(LOG_DEBUG, "Message list for: %s", FRname);
716 ptr = ChrPtr(migr_chardata);
718 while ((*ptr != 0) && (!isdigit(*ptr))) {
721 if ((*ptr != 0) && (isdigit(*ptr))) {
724 if (msgcount >= msglist_alloc) {
726 msglist = realloc(msglist, sizeof(long) * msglist_alloc);
728 msglist[msgcount++] = msgnum;
731 while ((*ptr != 0) && (isdigit(*ptr))) {
737 CtdlSaveMsgPointersInRoom(FRname, msglist, msgcount, 0, NULL, 1);
742 syslog(LOG_DEBUG, "Imported %d messages.", msgcount);
743 if (server_shutting_down) {
749 else if ((!strncasecmp(el, HKEY("f_"))) &&
750 migr_floorrecord(data, el))
751 ; /* Nothing to do anymore */
753 else if (!strcasecmp(el, "floor")) {
754 CtdlPutFloor(&flbuf, floornum);
755 syslog(LOG_INFO, "Imported floor #%d (%s)", floornum, flbuf.f_name);
759 else if ((!strncasecmp(el, HKEY("v_"))) &&
760 migr_visitrecord(data, el))
761 ; /* Nothing to do anymore */
762 else if (!strcasecmp(el, "visit")) {
764 syslog(LOG_INFO, "Imported visit: %ld/%ld/%ld", vbuf.v_roomnum, vbuf.v_roomgen, vbuf.v_usernum);
769 else if (!strcasecmp(el, "msg_msgnum")) smi.meta_msgnum = import_msgnum = atol(ChrPtr(migr_chardata));
770 else if (!strcasecmp(el, "msg_meta_refcount")) smi.meta_refcount = atoi(ChrPtr(migr_chardata));
771 else if (!strcasecmp(el, "msg_meta_rfc822_length")) smi.meta_rfc822_length = atoi(ChrPtr(migr_chardata));
772 else if (!strcasecmp(el, "msg_meta_content_type")) safestrncpy(smi.meta_content_type, ChrPtr(migr_chardata), sizeof smi.meta_content_type);
773 else if (!strcasecmp(el, "msg_mimetype")) safestrncpy(smi.mimetype, ChrPtr(migr_chardata), sizeof smi.mimetype);
775 else if (!strcasecmp(el, "msg_text"))
778 struct CtdlMessage *msg;
780 FlushStrBuf(migr_MsgData);
781 StrBufDecodeBase64To(migr_chardata, migr_MsgData);
783 msg = CtdlDeserializeMessage(import_msgnum, -1,
784 ChrPtr(migr_MsgData),
785 StrLength(migr_MsgData));
787 rc = CtdlSaveThisMessage(msg, import_msgnum, 0);
798 "%s message #%ld, size=%d, refcount=%d, bodylength=%ld, content-type: %s / %s",
799 (rc!= 0)?"failed to import ":"Imported ",
801 StrLength(migr_MsgData),
803 smi.meta_rfc822_length,
804 smi.meta_content_type,
806 memset(&smi, 0, sizeof(smi));
809 /*** MORE GENERAL STUFF ***/
811 FlushStrBuf(migr_chardata);
820 void migr_do_import(void) {
826 migr_chardata = NewStrBufPlain(NULL, SIZ * 20);
827 migr_MsgData = NewStrBufPlain(NULL, SIZ * 20);
828 Buf = NewStrBufPlain(NULL, SIZ);
829 xp = XML_ParserCreate(NULL);
831 cprintf("%d Failed to create XML parser instance\n", ERROR+INTERNAL_ERROR);
834 XML_SetElementHandler(xp, migr_xml_start, migr_xml_end);
835 XML_SetCharacterDataHandler(xp, migr_xml_chardata);
839 cprintf("%d sock it to me\n", SEND_LISTING);
842 client_set_inbound_buf(SIZ * 10);
844 while (!Finished && client_read_random_blob(Buf, -1) >= 0) {
845 if ((StrLength(Buf) > 4) &&
846 !strcmp(ChrPtr(Buf) + StrLength(Buf) - 4, "000\n"))
849 StrBufCutAt(Buf, StrLength(Buf) - 4, NULL);
851 if (server_shutting_down)
852 break; // Should we break or return?
854 if (StrLength(Buf) == 0)
857 XML_Parse(xp, ChrPtr(Buf), StrLength(Buf), 0);
861 XML_Parse(xp, "", 0, 1);
864 FreeStrBuf(&migr_chardata);
865 FreeStrBuf(&migr_MsgData);
866 rebuild_euid_index();
867 rebuild_usersbynumber();
873 /******************************************************************************
874 * Dispatcher, Common code *
875 ******************************************************************************/
877 * Dump out the pathnames of directories which can be copied "as is"
879 void migr_do_listdirs(void) {
880 cprintf("%d Don't forget these:\n", LISTING_FOLLOWS);
881 cprintf("bio|%s\n", ctdl_bio_dir);
882 cprintf("files|%s\n", ctdl_file_dir);
883 cprintf("userpics|%s\n", ctdl_usrpic_dir);
884 cprintf("messages|%s\n", ctdl_message_dir);
885 cprintf("netconfigs|%s\n", ctdl_netcfg_dir);
886 cprintf("keys|%s\n", ctdl_key_dir);
887 cprintf("images|%s\n", ctdl_image_dir);
888 cprintf("info|%s\n", ctdl_info_dir);
892 /******************************************************************************
893 * Repair database integrity *
894 ******************************************************************************/
896 StrBuf *PlainMessageBuf = NULL;
897 HashList *UsedMessageIDS = NULL;
899 int migr_restore_message_metadata(long msgnum, int refcount)
901 CitContext *CCC = MyContext();
903 struct CtdlMessage *msg;
906 /* We can use a static buffer here because there will never be more than
907 * one of this operation happening at any given time, and it's really best
908 * to just keep it allocated once instead of torturing malloc/free.
909 * Call this function with msgnum "-1" to free the buffer when finished.
911 static int encoded_alloc = 0;
912 static char *encoded_msg = NULL;
915 if ((encoded_alloc == 0) && (encoded_msg != NULL)) {
919 // todo FreeStrBuf(&PlainMessageBuf); PlainMessageBuf = NULL;
924 if (PlainMessageBuf == NULL) {
925 PlainMessageBuf = NewStrBufPlain(NULL, 10*SIZ);
928 /* Ok, here we go ... */
930 msg = CtdlFetchMessage(msgnum, 1, 0);
935 GetMetaData(&smi, msgnum);
936 smi.meta_msgnum = msgnum;
937 smi.meta_refcount = refcount;
939 /* restore the content type from the message body: */
940 mptr = bmstrcasestr(msg->cm_fields[eMesageText], "Content-type:");
943 safestrncpy(smi.meta_content_type, &mptr[13], sizeof smi.meta_content_type);
944 striplt(smi.meta_content_type);
945 aptr = smi.meta_content_type;
946 while (!IsEmptyStr(aptr)) {
951 memset(aptr, 0, sizeof(smi.meta_content_type) - (aptr - smi.meta_content_type));
957 CCC->redirect_buffer = PlainMessageBuf;
958 CtdlOutputPreLoadedMsg(msg, MT_RFC822, HEADERS_ALL, 0, 1, QP_EADDR);
959 smi.meta_rfc822_length = StrLength(CCC->redirect_buffer);
960 CCC->redirect_buffer = NULL;
964 "Setting message #%ld meta data to: refcount=%d, bodylength=%ld, content-type: %s / %s",
967 smi.meta_rfc822_length,
968 smi.meta_content_type,
978 void migr_check_room_msg(long msgnum, void *userdata) {
979 fprintf(migr_global_message_list, "%ld %s\n", msgnum, CC->room.QRname);
983 void migr_check_rooms_backend(struct ctdlroom *buf, void *data) {
985 /* message list goes inside this tag */
987 CtdlGetRoom(&CC->room, buf->QRname);
988 CtdlForEachMessage(MSGS_ALL, 0L, NULL, NULL, NULL, migr_check_room_msg, NULL);
991 void RemoveMessagesFromRooms(StrBuf *RoomNameVec, long msgnum) {
993 const char *Pos = NULL;
994 StrBuf *oneRoom = NewStrBuf();
996 syslog(LOG_INFO, "removing message pointer %ld from these rooms: %s", msgnum, ChrPtr(RoomNameVec));
998 while (Pos != StrBufNOTNULL){
999 StrBufExtract_NextToken(oneRoom, RoomNameVec, &Pos, '|');
1000 CtdlDeleteMessages(ChrPtr(oneRoom), &msgnum, 1, "");
1002 GetMetaData(&smi, msgnum);
1003 TDAP_AdjRefCount(msgnum, -smi.meta_refcount);
1006 void migr_do_restore_meta(void) {
1017 migr_global_message_list = fopen(migr_tempfilename1, "w");
1018 if (migr_global_message_list != NULL) {
1019 CtdlForEachRoom(migr_check_rooms_backend, NULL);
1020 fclose(migr_global_message_list);
1024 * Process the 'global' message list. (Sort it and remove dups.
1025 * Dups are ok because a message may be in more than one room, but
1026 * this will be handled by exporting the reference count, not by
1027 * exporting the message multiple times.)
1029 snprintf(cmd, sizeof cmd, "sort -n <%s >%s", migr_tempfilename1, migr_tempfilename2);
1030 if (system(cmd) != 0) syslog(LOG_ALERT, "Error %d", errno);
1032 RoomNames = NewStrBuf();
1034 migr_global_message_list = fopen(migr_tempfilename2, "r");
1035 if (migr_global_message_list != NULL) {
1036 syslog(LOG_INFO, "Opened %s", migr_tempfilename1);
1037 while ((Ctx->kill_me == 0) &&
1038 (fgets(buf, sizeof(buf), migr_global_message_list) != NULL)) {
1045 prn = strchr(buf, ' ');
1046 if (lastnum != msgnum) {
1047 failGetMessage = migr_restore_message_metadata(lastnum, refcount);
1048 if (failGetMessage) {
1049 RemoveMessagesFromRooms(RoomNames, lastnum);
1054 StrBufPlain(RoomNames, prn + 1, -1);
1055 StrBufTrim(RoomNames);
1059 if (StrLength(RoomNames) > 0)
1060 StrBufAppendBufPlain(RoomNames, HKEY("|"), 0);
1061 StrBufAppendBufPlain(RoomNames, prn, -1, 1);
1062 StrBufTrim(RoomNames);
1068 failGetMessage = migr_restore_message_metadata(msgnum, refcount);
1069 if (failGetMessage) {
1070 RemoveMessagesFromRooms(RoomNames, lastnum);
1072 fclose(migr_global_message_list);
1075 migr_restore_message_metadata(-1L, -1); /* This frees the encoding buffer */
1076 cprintf("%d system analysis completed", CIT_OK);
1083 /******************************************************************************
1084 * Dispatcher, Common code *
1085 ******************************************************************************/
1086 void cmd_migr(char *cmdbuf) {
1089 if (CtdlAccessCheck(ac_internal)) return;
1091 if (CtdlTrySingleUser())
1093 CtdlDisableHouseKeeping();
1094 CtdlMakeTempFileName(migr_tempfilename1, sizeof migr_tempfilename1);
1095 CtdlMakeTempFileName(migr_tempfilename2, sizeof migr_tempfilename2);
1097 extract_token(cmd, cmdbuf, 0, '|', sizeof cmd);
1098 if (!strcasecmp(cmd, "export")) {
1101 else if (!strcasecmp(cmd, "import")) {
1104 else if (!strcasecmp(cmd, "listdirs")) {
1107 else if (!strcasecmp(cmd, "restoremeta")) {
1108 migr_do_restore_meta();
1111 cprintf("%d illegal command\n", ERROR + ILLEGAL_VALUE);
1114 unlink(migr_tempfilename1);
1115 unlink(migr_tempfilename2);
1117 CtdlEnableHouseKeeping();
1118 CtdlEndSingleUser();
1122 cprintf("%d The migrator is already running.\n", ERROR + RESOURCE_BUSY);
1126 /******************************************************************************
1128 ******************************************************************************/
1130 CTDL_MODULE_INIT(migrate)
1134 CtdlRegisterProtoHook(cmd_migr, "MIGR", "Across-the-wire migration");
1137 /* return our module name for the log */