]> code.citadel.org Git - citadel.git/blob - citadel/server/modules/migrate/serv_migrate.c
Webcit coredumping when body of message is blank
[citadel.git] / citadel / server / modules / migrate / serv_migrate.c
1 // This module dumps and/or loads the Citadel database in XML format.
2 //
3 // Copyright (c) 1987-2023 by the citadel.org team
4 //
5 // This program is open source software; you can redistribute it and/or modify
6 // it under the terms of the GNU General Public License version 3.
7 //
8 // This program is distributed in the hope that it will be useful,
9 // but WITHOUT ANY WARRANTY; without even the implied warranty of
10 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
11 // GNU General Public License for more details.
12 //
13 // Explanation of <progress> tags:
14 //
15 // 0%           started
16 // 2%           finished exporting configuration
17 // 7%           finished exporting users
18 // 12%
19 // 17%          finished exporting rooms
20 // 18%          finished exporting floors
21 // 25%          finished exporting visits
22 // 26-99%       exporting messages
23 // 100%         finished exporting messages
24 //
25 // These tags are inserted into the XML stream to give the reader an approximation of its progress.
26
27 #include "../../sysdep.h"
28 #include <stdlib.h>
29 #include <unistd.h>
30 #include <stdio.h>
31 #include <fcntl.h>
32 #include <signal.h>
33 #include <pwd.h>
34 #include <errno.h>
35 #include <sys/types.h>
36 #include <time.h>
37 #include <sys/wait.h>
38 #include <string.h>
39 #include <ctype.h>
40 #include <limits.h>
41 #include <expat.h>
42 #include <libcitadel.h>
43 #include "../../citadel_defs.h"
44 #include "../../server.h"
45 #include "../../citserver.h"
46 #include "../../support.h"
47 #include "../../config.h"
48 #include "../../database.h"
49 #include "../../msgbase.h"
50 #include "../../user_ops.h"
51 #include "../../euidindex.h"
52 #include "../../internet_addressing.h"
53 #include "../../ctdl_module.h"
54
55 char migr_tempfilename1[PATH_MAX];
56 char migr_tempfilename2[PATH_MAX];
57 FILE *migr_global_message_list;
58 int total_msgs = 0;
59 char *ikey = NULL;                      // If we're importing a config key we store it here.
60
61 //*****************************************************************************
62 //*       Code which implements the export appears in this section            *
63 //*****************************************************************************
64
65 // Output a string to the client with these characters escaped:  & < > " '
66 void xml_strout(char *str) {
67
68         char *c = str;
69
70         if (str == NULL) {
71                 return;
72         }
73
74         while (*c != 0) {
75                 if (*c == '\"') {
76                         client_write(HKEY("&quot;"));
77                 }
78                 else if (*c == '\'') {
79                         client_write(HKEY("&apos;"));
80                 }
81                 else if (*c == '<') {
82                         client_write(HKEY("&lt;"));
83                 }
84                 else if (*c == '>') {
85                         client_write(HKEY("&gt;"));
86                 }
87                 else if (*c == '&') {
88                         client_write(HKEY("&amp;"));
89                 }
90                 else {
91                         client_write(c, 1);
92                 }
93                 ++c;
94         }
95 }
96
97
98 // Export a user record as XML
99 void migr_export_users_backend(char *username, void *data) {
100
101         struct ctdluser u;
102         if (CtdlGetUser(&u, username) != 0) {
103                 return;
104         }
105
106         client_write(HKEY("<user>\n"));
107         cprintf("<u_version>%d</u_version>\n", u.version);
108         cprintf("<u_uid>%ld</u_uid>\n", (long)u.uid);
109         client_write(HKEY("<u_password>"));     xml_strout(u.password);         client_write(HKEY("</u_password>\n"));
110         cprintf("<u_flags>%u</u_flags>\n", u.flags);
111         cprintf("<u_timescalled>%ld</u_timescalled>\n", u.timescalled);
112         cprintf("<u_posted>%ld</u_posted>\n", u.posted);
113         cprintf("<u_axlevel>%d</u_axlevel>\n", u.axlevel);
114         cprintf("<u_usernum>%ld</u_usernum>\n", u.usernum);
115         cprintf("<u_lastcall>%ld</u_lastcall>\n", (long)u.lastcall);
116         cprintf("<u_USuserpurge>%d</u_USuserpurge>\n", u.USuserpurge);
117         client_write(HKEY("<u_fullname>"));     xml_strout(u.fullname);         client_write(HKEY("</u_fullname>\n"));
118         cprintf("<u_msgnum_bio>%ld</u_msgnum_bio>\n", u.msgnum_bio);
119         cprintf("<u_msgnum_pic>%ld</u_msgnum_pic>\n", u.msgnum_pic);
120         cprintf("<u_emailaddrs>%s</u_emailaddrs>\n", u.emailaddrs);
121         cprintf("<u_msgnum_inboxrules>%ld</u_msgnum_inboxrules>\n", u.msgnum_inboxrules);
122         cprintf("<u_lastproc_inboxrules>%ld</u_lastproc_inboxrules>\n", u.lastproc_inboxrules);
123         client_write(HKEY("</user>\n"));
124 }
125
126
127 void migr_export_users(void) {
128         ForEachUser(migr_export_users_backend, NULL);
129 }
130
131
132 void migr_export_room_msg(long msgnum, void *userdata) {
133         static int count = 0;
134
135         cprintf("%ld,", msgnum);
136         if (++count%10==0) {
137                 cprintf("\n");
138         }
139         fprintf(migr_global_message_list, "%ld\n", msgnum);
140 }
141
142
143 void migr_export_rooms_backend(struct ctdlroom *buf, void *data) {
144         client_write(HKEY("<room>\n"));
145         client_write(HKEY("<QRname>"));
146         xml_strout(buf->QRname);
147         client_write(HKEY("</QRname>\n"));
148         client_write(HKEY("<QRpasswd>"));
149         xml_strout(buf->QRpasswd);
150         client_write(HKEY("</QRpasswd>\n"));
151         cprintf("<QRroomaide>%ld</QRroomaide>\n", buf->QRroomaide);
152         cprintf("<QRhighest>%ld</QRhighest>\n", buf->QRhighest);
153         cprintf("<QRgen>%ld</QRgen>\n", (long)buf->QRgen);
154         cprintf("<QRflags>%u</QRflags>\n", buf->QRflags);
155         if (buf->QRflags & QR_DIRECTORY) {
156                 client_write(HKEY("<QRdirname>"));
157                 xml_strout(buf->QRdirname);
158                 client_write(HKEY("</QRdirname>\n"));
159         }
160         cprintf("<QRfloor>%d</QRfloor>\n", buf->QRfloor);
161         cprintf("<QRmtime>%ld</QRmtime>\n", (long)buf->QRmtime);
162         cprintf("<QRexpire_mode>%d</QRexpire_mode>\n", buf->QRep.expire_mode);
163         cprintf("<QRexpire_value>%d</QRexpire_value>\n", buf->QRep.expire_value);
164         cprintf("<QRnumber>%ld</QRnumber>\n", buf->QRnumber);
165         cprintf("<QRorder>%d</QRorder>\n", buf->QRorder);
166         cprintf("<QRflags2>%u</QRflags2>\n", buf->QRflags2);
167         cprintf("<QRdefaultview>%d</QRdefaultview>\n", buf->QRdefaultview);
168         cprintf("<msgnum_info>%ld</msgnum_info>\n", buf->msgnum_info);
169         cprintf("<msgnum_pic>%ld</msgnum_pic>\n", buf->msgnum_pic);
170         client_write(HKEY("</room>\n"));
171
172         // message list goes inside this tag
173         CtdlGetRoom(&CC->room, buf->QRname);
174         client_write(HKEY("<room_messages>"));
175         client_write(HKEY("<FRname>"));
176         xml_strout(buf->QRname);                        // buf->QRname rather than CC->room.QRname to guarantee consistency
177         client_write(HKEY("</FRname>\n"));
178         client_write(HKEY("<FRmsglist>\n"));
179         CtdlForEachMessage(MSGS_ALL, 0L, NULL, NULL, NULL, migr_export_room_msg, NULL);
180         client_write(HKEY("</FRmsglist>\n"));
181         client_write(HKEY("</room_messages>\n"));
182 }
183
184
185 void migr_export_rooms(void) {
186         char cmd[SIZ];
187         migr_global_message_list = fopen(migr_tempfilename1, "w");
188         if (migr_global_message_list != NULL) {
189                 CtdlForEachRoom(migr_export_rooms_backend, NULL);
190                 fclose(migr_global_message_list);
191         }
192
193         // Process the 'global' message list.  (Sort it and remove dups.
194         // Dups are ok because a message may be in more than one room, but
195         // this will be handled by exporting the reference count, not by
196         // exporting the message multiple times.)
197         snprintf(cmd, sizeof cmd, "sort -n <%s >%s", migr_tempfilename1, migr_tempfilename2);
198         if (system(cmd) != 0) {
199                 syslog(LOG_ERR, "migrate: error %d", errno);
200         }
201         snprintf(cmd, sizeof cmd, "uniq <%s >%s", migr_tempfilename2, migr_tempfilename1);
202         if (system(cmd) != 0) {
203                 syslog(LOG_ERR, "migrate: error %d", errno);
204         }
205
206         snprintf(cmd, sizeof cmd, "wc -l %s", migr_tempfilename1);
207         FILE *fp = popen(cmd, "r");
208         if (fp) {
209                 fgets(cmd, sizeof cmd, fp);
210                 pclose(fp);
211                 total_msgs = atoi(cmd);
212         }
213         else {
214                 total_msgs = 1; // any nonzero just to keep it from barfing
215         }
216         syslog(LOG_DEBUG, "migrate: total messages to be exported: %d", total_msgs);
217 }
218
219
220 void migr_export_floors(void) {
221         struct floor qfbuf, *buf;
222         int i;
223
224         for (i=0; i < MAXFLOORS; ++i) {
225                 client_write(HKEY("<floor>\n"));
226                 cprintf("<f_num>%d</f_num>\n", i);
227                 CtdlGetFloor(&qfbuf, i);
228                 buf = &qfbuf;
229                 cprintf("<f_flags>%u</f_flags>\n", buf->f_flags);
230                 client_write(HKEY("<f_name>")); xml_strout(buf->f_name); client_write(HKEY("</f_name>\n"));
231                 cprintf("<f_ref_count>%d</f_ref_count>\n", buf->f_ref_count);
232                 cprintf("<f_ep_expire_mode>%d</f_ep_expire_mode>\n", buf->f_ep.expire_mode);
233                 cprintf("<f_ep_expire_value>%d</f_ep_expire_value>\n", buf->f_ep.expire_value);
234                 client_write(HKEY("</floor>\n"));
235         }
236 }
237
238
239 // Return nonzero if the supplied string contains only characters which are valid in a sequence set.
240 int is_sequence_set(char *s) {
241         if (!s) return(0);
242
243         char *c = s;
244         char ch;
245         while (ch = *c++, ch) {
246                 if (!strchr("0123456789*,:", ch)) {
247                         return(0);
248                 }
249         }
250         return(1);
251 }
252
253
254 // Traverse the visits file...
255 void migr_export_visits(void) {
256         struct visit vbuf;
257         struct cdbdata *cdbv;
258
259         cdb_rewind(CDB_VISIT);
260
261         while (cdbv = cdb_next_item(CDB_VISIT), cdbv != NULL) {
262                 memset(&vbuf, 0, sizeof(struct visit));
263                 memcpy(&vbuf, cdbv->ptr,
264                        ((cdbv->len > sizeof(struct visit)) ?
265                         sizeof(struct visit) : cdbv->len));
266                 cdb_free(cdbv);
267
268                 client_write(HKEY("<visit>\n"));
269                 cprintf("<v_roomnum>%ld</v_roomnum>\n", vbuf.v_roomnum);
270                 cprintf("<v_roomgen>%ld</v_roomgen>\n", vbuf.v_roomgen);
271                 cprintf("<v_usernum>%ld</v_usernum>\n", vbuf.v_usernum);
272
273                 client_write(HKEY("<v_seen>"));
274                 if ( (!IsEmptyStr(vbuf.v_seen)) && (is_sequence_set(vbuf.v_seen)) ) {
275                         xml_strout(vbuf.v_seen);
276                 }
277                 else {
278                         cprintf("%ld", vbuf.v_lastseen);
279                 }
280                 client_write(HKEY("</v_seen>"));
281
282                 if ( (!IsEmptyStr(vbuf.v_answered)) && (is_sequence_set(vbuf.v_answered)) ) {
283                         client_write(HKEY("<v_answered>"));
284                         xml_strout(vbuf.v_answered);
285                         client_write(HKEY("</v_answered>\n"));
286                 }
287
288                 cprintf("<v_flags>%u</v_flags>\n", vbuf.v_flags);
289                 cprintf("<v_view>%d</v_view>\n", vbuf.v_view);
290                 client_write(HKEY("</visit>\n"));
291         }
292 }
293
294
295 void migr_export_message(long msgnum) {
296         struct MetaData smi;
297         struct CtdlMessage *msg;
298         struct ser_ret smr;
299         long bytes_written = 0;
300         long this_block = 0;
301
302         // We can use a static buffer here because there will never be more than
303         // one of this operation happening at any given time, and it's really best
304         // to just keep it allocated once instead of torturing malloc/free.
305         // Call this function with msgnum "-1" to free the buffer when finished.
306
307         static int encoded_alloc = 0;
308         static char *encoded_msg = NULL;
309
310         if (msgnum < 0) {
311                 if ((encoded_alloc == 0) && (encoded_msg != NULL)) {
312                         free(encoded_msg);
313                         encoded_alloc = 0;
314                         encoded_msg = NULL;
315                 }
316                 return;
317         }
318
319         // Ok, here we go ...
320
321         msg = CtdlFetchMessage(msgnum, 1);
322         if (msg == NULL) return;                                // fail silently
323
324         client_write(HKEY("<message>\n"));
325         GetMetaData(&smi, msgnum);
326         cprintf("<msg_msgnum>%ld</msg_msgnum>\n", msgnum);
327         cprintf("<msg_meta_refcount>%d</msg_meta_refcount>\n", smi.meta_refcount);
328         cprintf("<msg_meta_rfc822_length>%ld</msg_meta_rfc822_length>\n", smi.meta_rfc822_length);
329         client_write(HKEY("<msg_meta_content_type>")); xml_strout(smi.meta_content_type); client_write(HKEY("</msg_meta_content_type>\n"));
330
331         client_write(HKEY("<msg_text>"));
332         CtdlSerializeMessage(&smr, msg);
333         CM_Free(msg);
334
335         // Predict the buffer size we need.  Expand the buffer if necessary.
336         int encoded_len = smr.len * 15 / 10 ;
337         if (encoded_len > encoded_alloc) {
338                 encoded_alloc = encoded_len;
339                 encoded_msg = realloc(encoded_msg, encoded_alloc);
340         }
341
342         if (encoded_msg == NULL) {
343                 // Questionable hack that hopes it'll work next time and we only lose one message
344                 encoded_alloc = 0;
345         }
346         else {
347                 // Once we do the encoding we know the exact size
348                 encoded_len = CtdlEncodeBase64(encoded_msg, (char *)smr.ser, smr.len, BASE64_YES_LINEBREAKS);
349
350                 // Careful now.  If the message is gargantuan, trying to write multiple gigamegs in one
351                 // big write operation can make our transport unhappy.  So we'll chunk it up 10 KB at a time.
352                 bytes_written = 0;
353                 while ( (bytes_written < encoded_len) && (!server_shutting_down) ) {
354                         this_block = encoded_len - bytes_written;
355                         if (this_block > 10240) {
356                                 this_block = 10240;
357                         }
358                         client_write(&encoded_msg[bytes_written], this_block);
359                         bytes_written += this_block;
360                 }
361         }
362
363         free(smr.ser);
364
365         client_write(HKEY("</msg_text>\n"));
366         client_write(HKEY("</message>\n"));
367 }
368
369
370 void migr_export_configs(void) {
371         struct cdbdata *cdbcfg;
372         int keylen = 0;
373         char *key = NULL;
374         char *value = NULL;
375
376         cdb_rewind(CDB_CONFIG);
377         while (cdbcfg = cdb_next_item(CDB_CONFIG), cdbcfg != NULL) {
378
379                 keylen = strlen(cdbcfg->ptr);
380                 key = cdbcfg->ptr;
381                 value = cdbcfg->ptr + keylen + 1;
382
383                 client_write("<config key=\"", 13);
384                 xml_strout(key);
385                 client_write("\">", 2);
386                 xml_strout(value);
387                 client_write("</config>\n", 10);
388                 cdb_free(cdbcfg);
389         }
390 }
391
392
393 void migr_export_messages(void) {
394         char buf[SIZ];
395         long msgnum;
396         int count = 0;
397         int progress = 0;
398         int prev_progress = 0;
399         CitContext *Ctx;
400
401         Ctx = CC;
402         migr_global_message_list = fopen(migr_tempfilename1, "r");
403         if (migr_global_message_list != NULL) {
404                 syslog(LOG_INFO, "migrate: opened %s", migr_tempfilename1);
405                 while ((Ctx->kill_me == 0) && 
406                        (fgets(buf, sizeof(buf), migr_global_message_list) != NULL)) {
407                         msgnum = atol(buf);
408                         if (msgnum > 0L) {
409                                 migr_export_message(msgnum);
410                                 ++count;
411                         }
412                         progress = (count * 74 / total_msgs) + 25 ;
413                         if ((progress > prev_progress) && (progress < 100)) {
414                                 cprintf("<progress>%d</progress>\n", progress);
415                         }
416                         prev_progress = progress;
417                 }
418                 fclose(migr_global_message_list);
419         }
420         if (Ctx->kill_me == 0) {
421                 syslog(LOG_INFO, "migrate: exported %d messages.", count);
422         }
423         else {
424                 syslog(LOG_ERR, "migrate: export aborted due to client disconnect!");
425         }
426
427         migr_export_message(-1L);       // This frees the encoding buffer
428 }
429
430
431 void migr_do_export(void) {
432         CitContext *Ctx;
433
434         Ctx = CC;
435         cprintf("%d Exporting all Citadel databases.\n", LISTING_FOLLOWS);
436         Ctx->dont_term = 1;
437
438         client_write(HKEY("<?xml version=\"1.0\" encoding=\"UTF-8\" ?>\n"));
439         client_write(HKEY("<citadel_migrate_data>\n"));
440         cprintf("<version>%d</version>\n", REV_LEVEL);
441         cprintf("<progress>%d</progress>\n", 0);
442
443         // export the configuration database
444         migr_export_configs();
445         cprintf("<progress>%d</progress>\n", 2);
446         
447         if (Ctx->kill_me == 0)  migr_export_users();
448         cprintf("<progress>%d</progress>\n", 7);
449         cprintf("<progress>%d</progress>\n", 12);
450         if (Ctx->kill_me == 0)  migr_export_rooms();
451         cprintf("<progress>%d</progress>\n", 17);
452         if (Ctx->kill_me == 0)  migr_export_floors();
453         cprintf("<progress>%d</progress>\n", 18);
454         if (Ctx->kill_me == 0)  migr_export_visits();
455         cprintf("<progress>%d</progress>\n", 25);
456         if (Ctx->kill_me == 0)  migr_export_messages();
457         cprintf("<progress>%d</progress>\n", 100);
458         client_write(HKEY("</citadel_migrate_data>\n"));
459         client_write(HKEY("000\n"));
460         Ctx->dont_term = 0;
461 }
462
463
464 //                              Import code
465 //   Here's the code that implements the import side.  It's going to end up
466 //       being one big loop with lots of global variables.  I don't care.
467 // You wouldn't run multiple concurrent imports anyway.  If this offends your
468 //  delicate sensibilities  then go rewrite it in Ruby on Rails or something.
469
470
471 int citadel_migrate_data = 0;           // Are we inside a <citadel_migrate_data> tag pair?
472 StrBuf *migr_chardata = NULL;
473 StrBuf *migr_MsgData = NULL;
474 struct ctdluser usbuf;
475 struct ctdlroom qrbuf;
476 char FRname[ROOMNAMELEN];
477 struct floor flbuf;
478 int floornum = 0;
479 struct visit vbuf;
480 struct MetaData smi;
481 long import_msgnum = 0;
482
483 // This callback stores up the data which appears in between tags.
484 void migr_xml_chardata(void *data, const XML_Char *s, int len) {
485         StrBufAppendBufPlain(migr_chardata, s, len, 0);
486 }
487
488
489 void migr_xml_start(void *data, const char *el, const char **attr) {
490         int i;
491
492         //  *** GENERAL STUFF ***
493
494         // Reset chardata_len to zero and init buffer
495         FlushStrBuf(migr_chardata);
496         FlushStrBuf(migr_MsgData);
497
498         if (!strcasecmp(el, "citadel_migrate_data")) {
499                 ++citadel_migrate_data;
500
501                 // As soon as it looks like the input data is a genuine Citadel XML export,
502                 // whack the existing database on disk to make room for the new one.
503                 if (citadel_migrate_data == 1) {
504                         syslog(LOG_INFO, "migrate: erasing existing databases so we can receive the incoming import");
505                         for (i = 0; i < MAXCDB; ++i) {
506                                 cdb_trunc(i);
507                         }
508                 }
509                 return;
510         }
511
512         if (citadel_migrate_data != 1) {
513                 syslog(LOG_ERR, "migrate: out-of-sequence tag <%s> detected.  Warning: ODD-DATA!", el);
514                 return;
515         }
516
517         // When we begin receiving XML for one of these record types, clear out the associated
518         // buffer so we don't accidentally carry over any data from a previous record.
519         if (!strcasecmp(el, "user"))                    memset(&usbuf, 0, sizeof(struct ctdluser));
520         else if (!strcasecmp(el, "room"))               memset(&qrbuf, 0, sizeof(struct ctdlroom));
521         else if (!strcasecmp(el, "room_messages"))      memset(FRname, 0, sizeof FRname);
522         else if (!strcasecmp(el, "floor"))              memset(&flbuf, 0, sizeof(struct floor));
523         else if (!strcasecmp(el, "visit"))              memset(&vbuf, 0, sizeof(struct visit));
524
525         else if (!strcasecmp(el, "message")) {
526                 memset(&smi, 0, sizeof (struct MetaData));
527                 import_msgnum = 0;
528         }
529         else if (!strcasecmp(el, "config")) {
530                 if (ikey != NULL) {
531                         free(ikey);
532                         ikey = NULL;
533                 }
534                 while (*attr) {
535                         if (!strcasecmp(attr[0], "key")) {
536                                 ikey = strdup(attr[1]);
537                         }
538                         attr += 2;
539                 }
540         }
541
542 }
543
544
545 int migr_userrecord(void *data, const char *el) {
546         if (!strcasecmp(el, "u_version"))                       usbuf.version = atoi(ChrPtr(migr_chardata));
547         else if (!strcasecmp(el, "u_uid"))                      usbuf.uid = atol(ChrPtr(migr_chardata));
548         else if (!strcasecmp(el, "u_password"))                 safestrncpy(usbuf.password, ChrPtr(migr_chardata), sizeof usbuf.password);
549         else if (!strcasecmp(el, "u_flags"))                    usbuf.flags = atoi(ChrPtr(migr_chardata));
550         else if (!strcasecmp(el, "u_timescalled"))              usbuf.timescalled = atol(ChrPtr(migr_chardata));
551         else if (!strcasecmp(el, "u_posted"))                   usbuf.posted = atol(ChrPtr(migr_chardata));
552         else if (!strcasecmp(el, "u_axlevel"))                  usbuf.axlevel = atoi(ChrPtr(migr_chardata));
553         else if (!strcasecmp(el, "u_usernum"))                  usbuf.usernum = atol(ChrPtr(migr_chardata));
554         else if (!strcasecmp(el, "u_lastcall"))                 usbuf.lastcall = atol(ChrPtr(migr_chardata));
555         else if (!strcasecmp(el, "u_USuserpurge"))              usbuf.USuserpurge = atoi(ChrPtr(migr_chardata));
556         else if (!strcasecmp(el, "u_fullname"))                 safestrncpy(usbuf.fullname, ChrPtr(migr_chardata), sizeof usbuf.fullname);
557         else if (!strcasecmp(el, "u_msgnum_bio"))               usbuf.msgnum_bio = atol(ChrPtr(migr_chardata));
558         else if (!strcasecmp(el, "u_msgnum_pic"))               usbuf.msgnum_pic = atol(ChrPtr(migr_chardata));
559         else if (!strcasecmp(el, "u_emailaddrs"))               safestrncpy(usbuf.emailaddrs, ChrPtr(migr_chardata), sizeof usbuf.emailaddrs);
560         else if (!strcasecmp(el, "u_msgnum_inboxrules"))        usbuf.msgnum_inboxrules = atol(ChrPtr(migr_chardata));
561         else if (!strcasecmp(el, "u_lastproc_inboxrules"))      usbuf.lastproc_inboxrules = atol(ChrPtr(migr_chardata));
562         else return 0;
563         return 1;
564 }
565
566
567 int migr_roomrecord(void *data, const char *el) {
568         if (!strcasecmp(el, "QRname"))                          safestrncpy(qrbuf.QRname, ChrPtr(migr_chardata), sizeof qrbuf.QRname);
569         else if (!strcasecmp(el, "QRpasswd"))                   safestrncpy(qrbuf.QRpasswd, ChrPtr(migr_chardata), sizeof qrbuf.QRpasswd);
570         else if (!strcasecmp(el, "QRroomaide"))                 qrbuf.QRroomaide = atol(ChrPtr(migr_chardata));
571         else if (!strcasecmp(el, "QRhighest"))                  qrbuf.QRhighest = atol(ChrPtr(migr_chardata));
572         else if (!strcasecmp(el, "QRgen"))                      qrbuf.QRgen = atol(ChrPtr(migr_chardata));
573         else if (!strcasecmp(el, "QRflags"))                    qrbuf.QRflags = atoi(ChrPtr(migr_chardata));
574         else if (!strcasecmp(el, "QRdirname"))                  safestrncpy(qrbuf.QRdirname, ChrPtr(migr_chardata), sizeof qrbuf.QRdirname);
575         else if (!strcasecmp(el, "QRfloor"))                    qrbuf.QRfloor = atoi(ChrPtr(migr_chardata));
576         else if (!strcasecmp(el, "QRmtime"))                    qrbuf.QRmtime = atol(ChrPtr(migr_chardata));
577         else if (!strcasecmp(el, "QRexpire_mode"))              qrbuf.QRep.expire_mode = atoi(ChrPtr(migr_chardata));
578         else if (!strcasecmp(el, "QRexpire_value"))             qrbuf.QRep.expire_value = atoi(ChrPtr(migr_chardata));
579         else if (!strcasecmp(el, "QRnumber"))                   qrbuf.QRnumber = atol(ChrPtr(migr_chardata));
580         else if (!strcasecmp(el, "QRorder"))                    qrbuf.QRorder = atoi(ChrPtr(migr_chardata));
581         else if (!strcasecmp(el, "QRflags2"))                   qrbuf.QRflags2 = atoi(ChrPtr(migr_chardata));
582         else if (!strcasecmp(el, "QRdefaultview"))              qrbuf.QRdefaultview = atoi(ChrPtr(migr_chardata));
583         else if (!strcasecmp(el, "msgnum_info"))                qrbuf.msgnum_info = atol(ChrPtr(migr_chardata));
584         else if (!strcasecmp(el, "msgnum_pic"))                 qrbuf.msgnum_pic = atol(ChrPtr(migr_chardata));
585         else return 0;
586         return 1;
587 }
588
589
590 int migr_floorrecord(void *data, const char *el) {
591         if (!strcasecmp(el, "f_num"))                           floornum = atoi(ChrPtr(migr_chardata));
592         else if (!strcasecmp(el, "f_flags"))                    flbuf.f_flags = atoi(ChrPtr(migr_chardata));
593         else if (!strcasecmp(el, "f_name"))                     safestrncpy(flbuf.f_name, ChrPtr(migr_chardata), sizeof flbuf.f_name);
594         else if (!strcasecmp(el, "f_ref_count"))                flbuf.f_ref_count = atoi(ChrPtr(migr_chardata));
595         else if (!strcasecmp(el, "f_ep_expire_mode"))           flbuf.f_ep.expire_mode = atoi(ChrPtr(migr_chardata));
596         else if (!strcasecmp(el, "f_ep_expire_value"))          flbuf.f_ep.expire_value = atoi(ChrPtr(migr_chardata));
597         else return 0;
598         return 1;
599 }
600
601
602 int migr_visitrecord(void *data, const char *el) {
603         if (!strcasecmp(el, "v_roomnum"))                       vbuf.v_roomnum = atol(ChrPtr(migr_chardata));
604         else if (!strcasecmp(el, "v_roomgen"))                  vbuf.v_roomgen = atol(ChrPtr(migr_chardata));
605         else if (!strcasecmp(el, "v_usernum"))                  vbuf.v_usernum = atol(ChrPtr(migr_chardata));
606
607         else if (!strcasecmp(el, "v_seen")) {
608                 int is_textual_seen = 0;
609                 int i;
610                 int max = StrLength(migr_chardata);
611
612                 vbuf.v_lastseen = atol(ChrPtr(migr_chardata));
613                 is_textual_seen = 0;
614                 for (i=0; i < max; ++i) 
615                         if (!isdigit(ChrPtr(migr_chardata)[i]))
616                                 is_textual_seen = 1;
617                 if (is_textual_seen)
618                         safestrncpy(vbuf.v_seen, ChrPtr(migr_chardata), sizeof vbuf.v_seen);
619         }
620
621         else if (!strcasecmp(el, "v_answered"))                 safestrncpy(vbuf.v_answered, ChrPtr(migr_chardata), sizeof vbuf.v_answered);
622         else if (!strcasecmp(el, "v_flags"))                    vbuf.v_flags = atoi(ChrPtr(migr_chardata));
623         else if (!strcasecmp(el, "v_view"))                     vbuf.v_view = atoi(ChrPtr(migr_chardata));
624         else return 0;
625         return 1;
626 }
627
628
629 void migr_xml_end(void *data, const char *el) {
630         const char *ptr;
631         int msgcount = 0;
632         long msgnum = 0L;
633         long *msglist = NULL;
634         int msglist_alloc = 0;
635         // *** GENERAL STUFF ***
636
637         if (!strcasecmp(el, "citadel_migrate_data")) {
638                 --citadel_migrate_data;
639                 return;
640         }
641
642         if (citadel_migrate_data != 1) {
643                 syslog(LOG_ERR, "migrate: out-of-sequence tag <%s> detected.  Warning: ODD-DATA!", el);
644                 return;
645         }
646
647         // syslog(LOG_DEBUG, "END TAG: <%s> DATA: <%s>\n", el, (migr_chardata_len ? migr_chardata : ""));
648
649         // *** CONFIG ***
650
651         if (!strcasecmp(el, "config")) {
652                 syslog(LOG_DEBUG, "migrate: imported config key=%s", ikey);
653
654                 if (ikey != NULL) {
655                         CtdlSetConfigStr(ikey, (char *)ChrPtr(migr_chardata));
656                         free(ikey);
657                         ikey = NULL;
658                 }
659                 else {
660                         syslog(LOG_INFO, "migrate: closed a <config> tag but no key name was supplied.");
661                 }
662         }
663
664         // *** USER ***
665         else if ((!strncasecmp(el, HKEY("u_"))) && 
666                  migr_userrecord(data, el))
667                 ; /* Nothing to do anymore */
668         else if (!strcasecmp(el, "user")) {
669                 CtdlPutUser(&usbuf);
670                 syslog(LOG_INFO, "migrate: imported user: %s", usbuf.fullname);
671         }
672
673         // *** ROOM ***
674         else if ((!strncasecmp(el, HKEY("QR"))) && 
675                  migr_roomrecord(data, el))
676                 ; // Nothing to do anymore
677         else if (!strcasecmp(el, "room")) {
678                 CtdlPutRoom(&qrbuf);
679                 syslog(LOG_INFO, "migrate: imported room: %s", qrbuf.QRname);
680         }
681
682         // *** ROOM MESSAGE POINTERS ***
683
684         else if (!strcasecmp(el, "FRname")) {
685                 safestrncpy(FRname, ChrPtr(migr_chardata), sizeof FRname);
686         }
687
688         else if (!strcasecmp(el, "FRmsglist")) {
689                 if (!IsEmptyStr(FRname)) {
690                         msgcount = 0;
691                         msglist_alloc = 1000;
692                         msglist = malloc(sizeof(long) * msglist_alloc);
693
694                         syslog(LOG_DEBUG, "migrate: message list for: %s", FRname);
695
696                         ptr = ChrPtr(migr_chardata);
697                         while (*ptr != 0) {
698                                 while ((*ptr != 0) && (!isdigit(*ptr))) {
699                                         ++ptr;
700                                 }
701                                 if ((*ptr != 0) && (isdigit(*ptr))) {
702                                         msgnum = atol(ptr);
703                                         if (msgnum > 0L) {
704                                                 if (msgcount >= msglist_alloc) {
705                                                         msglist_alloc *= 2;
706                                                         msglist = realloc(msglist, sizeof(long) * msglist_alloc);
707                                                 }
708                                                 msglist[msgcount++] = msgnum;
709                                                 }
710                                         }
711                                         while ((*ptr != 0) && (isdigit(*ptr))) {
712                                                 ++ptr;
713                                         }
714                                 }
715                         }
716                         if (msgcount > 0) {
717                                 CtdlSaveMsgPointersInRoom(FRname, msglist, msgcount, 0, NULL, 1);
718                         }
719                         free(msglist);
720                         msglist = NULL;
721                         msglist_alloc = 0;
722                         syslog(LOG_DEBUG, "migrate: imported %d messages.", msgcount);
723                         if (server_shutting_down) {
724                                 return;
725                 }
726         }
727
728         // *** FLOORS ***
729         else if ((!strncasecmp(el, HKEY("f_"))) && 
730                  migr_floorrecord(data, el))
731                 ; // Nothing to do anymore
732
733         else if (!strcasecmp(el, "floor")) {
734                 CtdlPutFloor(&flbuf, floornum);
735                 syslog(LOG_INFO, "migrate: imported floor #%d (%s)", floornum, flbuf.f_name);
736         }
737
738         // *** VISITS ***
739         else if ((!strncasecmp(el, HKEY("v_"))) && migr_visitrecord(data, el)) {
740                 ; // Nothing to do anymore
741         }
742         else if (!strcasecmp(el, "visit")) {
743                 put_visit(&vbuf);
744                 syslog(LOG_INFO, "migrate: imported visit: %ld/%ld/%ld", vbuf.v_roomnum, vbuf.v_roomgen, vbuf.v_usernum);
745         }
746
747         // *** MESSAGES ***
748         
749         else if (!strcasecmp(el, "msg_msgnum"))                 smi.meta_msgnum = import_msgnum = atol(ChrPtr(migr_chardata));
750         else if (!strcasecmp(el, "msg_meta_refcount"))          smi.meta_refcount = atoi(ChrPtr(migr_chardata));
751         else if (!strcasecmp(el, "msg_meta_rfc822_length"))     smi.meta_rfc822_length = atoi(ChrPtr(migr_chardata));
752         else if (!strcasecmp(el, "msg_meta_content_type"))      safestrncpy(smi.meta_content_type, ChrPtr(migr_chardata), sizeof smi.meta_content_type);
753
754         else if (!strcasecmp(el, "msg_text")) {
755                 long rc;
756                 struct CtdlMessage *msg;
757
758                 FlushStrBuf(migr_MsgData);
759                 StrBufDecodeBase64To(migr_chardata, migr_MsgData);
760
761                 msg = CtdlDeserializeMessage(import_msgnum, -1,
762                                              ChrPtr(migr_MsgData), 
763                                              StrLength(migr_MsgData));
764                 if (msg != NULL) {
765                         rc = CtdlSaveThisMessage(msg, import_msgnum, 0);
766                         if (rc == 0) {
767                                 PutMetaData(&smi);
768                         }
769                         CM_Free(msg);
770                 }
771                 else {
772                         rc = -1;
773                 }
774
775                 syslog(LOG_INFO,
776                        "migrate: %s message #%ld, size=%d, ref=%d, body=%ld, content-type: %s",
777                        (rc!= 0)?"failed to import":"imported",
778                        import_msgnum,
779                        StrLength(migr_MsgData),
780                        smi.meta_refcount,
781                        smi.meta_rfc822_length,
782                        smi.meta_content_type
783                 );
784                 memset(&smi, 0, sizeof(smi));
785         }
786
787         // *** MORE GENERAL STUFF ***
788
789         FlushStrBuf(migr_chardata);
790 }
791
792
793 // Import begins here
794 void migr_do_import(void) {
795         XML_Parser xp;
796         char buf[SIZ];
797         
798         unbuffer_output();
799         migr_chardata = NewStrBufPlain(NULL, SIZ * 20);
800         migr_MsgData = NewStrBufPlain(NULL, SIZ * 20);
801         xp = XML_ParserCreate(NULL);
802         if (!xp) {
803                 cprintf("%d Failed to create XML parser instance\n", ERROR+INTERNAL_ERROR);
804                 return;
805         }
806         XML_SetElementHandler(xp, migr_xml_start, migr_xml_end);
807         XML_SetCharacterDataHandler(xp, migr_xml_chardata);
808
809         CC->dont_term = 1;
810
811         cprintf("%d sock it to me\n", SEND_LISTING);
812         unbuffer_output();
813
814         client_set_inbound_buf(SIZ * 10);
815
816         while (client_getln(buf, sizeof buf) >= 0 && strcmp(buf, "000")) {
817                 XML_Parse(xp, buf, strlen(buf), 0);
818                 if (server_shutting_down)
819                         break;  // Should we break or return?
820         }
821
822         XML_Parse(xp, "", 0, 1);
823         XML_ParserFree(xp);
824         FreeStrBuf(&migr_chardata);
825         FreeStrBuf(&migr_MsgData);
826         rebuild_euid_index();
827         rebuild_usersbynumber();
828         CtdlSetConfigInt("MM_fulltext_wordbreaker", -1);        // Set an invalid wordbreaker to force re-indexing
829         CtdlRebuildDirectoryIndex();                            // Force a rebuild of the directory index
830         CC->dont_term = 0;
831 }
832
833
834 // ******************************************************************************
835 // *                         Dispatcher, Common code                            *
836 // ******************************************************************************
837
838 // Dump out the pathnames of directories which can be copied "as is"
839 void migr_do_listdirs(void) {
840         cprintf("%d Don't forget these:\n", LISTING_FOLLOWS);
841         cprintf("files|%s\n",           ctdl_file_dir);
842         cprintf("messages|%s\n",        ctdl_message_dir);
843         cprintf("keys|%s\n",            ctdl_key_dir);
844         cprintf("000\n");
845 }
846
847
848 // ******************************************************************************
849 // *                    Repair database integrity                               *
850 // ******************************************************************************
851
852 StrBuf *PlainMessageBuf = NULL;
853 HashList *UsedMessageIDS = NULL;
854
855 int migr_restore_message_metadata(long msgnum, int refcount) {
856         struct MetaData smi;
857         struct CtdlMessage *msg;
858         char *mptr = NULL;
859
860         // We can use a static buffer here because there will never be more than
861         // one of this operation happening at any given time, and it's really best
862         // to just keep it allocated once instead of torturing malloc/free.
863         // Call this function with msgnum "-1" to free the buffer when finished.
864         static int encoded_alloc = 0;
865         static char *encoded_msg = NULL;
866
867         if (msgnum < 0) {
868                 if ((encoded_alloc == 0) && (encoded_msg != NULL)) {
869                         free(encoded_msg);
870                         encoded_alloc = 0;
871                         encoded_msg = NULL;
872                         // todo FreeStrBuf(&PlainMessageBuf); PlainMessageBuf = NULL;
873                 }
874                 return 0;
875         }
876
877         if (PlainMessageBuf == NULL) {
878                 PlainMessageBuf = NewStrBufPlain(NULL, 10*SIZ);
879         }
880
881         // Ok, here we go ...
882
883         msg = CtdlFetchMessage(msgnum, 1);
884         if (msg == NULL) {
885                 return 1;
886         }
887
888         GetMetaData(&smi, msgnum);
889         smi.meta_msgnum = msgnum;
890         smi.meta_refcount = refcount;
891         
892         // restore the content type from the message body:
893         mptr = bmstrcasestr(msg->cm_fields[eMesageText], "Content-type:");
894         if (mptr != NULL) {
895                 char *aptr;
896                 safestrncpy(smi.meta_content_type, &mptr[13], sizeof smi.meta_content_type);
897                 string_trim(smi.meta_content_type);
898                 aptr = smi.meta_content_type;
899                 while (!IsEmptyStr(aptr)) {
900                         if ((*aptr == ';')
901                             || (*aptr == ' ')
902                             || (*aptr == 13)
903                             || (*aptr == 10)) {
904                                 memset(aptr, 0, sizeof(smi.meta_content_type) - (aptr - smi.meta_content_type));
905                         }
906                         else aptr++;
907                 }
908         }
909
910         CC->redirect_buffer = PlainMessageBuf;
911         CtdlOutputPreLoadedMsg(msg, MT_RFC822, HEADERS_ALL, 0, 1, QP_EADDR);
912         smi.meta_rfc822_length = StrLength(CC->redirect_buffer);
913         CC->redirect_buffer = NULL;
914
915         syslog(LOG_INFO,
916                "migrate: setting message #%ld meta data to: refcount=%d, bodylength=%ld, content-type: %s",
917                smi.meta_msgnum,
918                smi.meta_refcount,
919                smi.meta_rfc822_length,
920                smi.meta_content_type
921         );
922
923         PutMetaData(&smi);
924         CM_Free(msg);
925         return 0;
926 }
927
928
929 void migr_check_room_msg(long msgnum, void *userdata) {
930         fprintf(migr_global_message_list, "%ld %s\n", msgnum, CC->room.QRname);
931 }
932
933
934 void migr_check_rooms_backend(struct ctdlroom *buf, void *data) {
935         // message list goes inside this tag
936         CtdlGetRoom(&CC->room, buf->QRname);
937         CtdlForEachMessage(MSGS_ALL, 0L, NULL, NULL, NULL, migr_check_room_msg, NULL);
938 }
939
940
941 void RemoveMessagesFromRooms(StrBuf *RoomNameVec, long msgnum) {
942         struct MetaData smi;
943         const char *Pos = NULL;
944         StrBuf *oneRoom = NewStrBuf();
945
946         syslog(LOG_INFO, "migrate: removing message pointer %ld from these rooms: %s", msgnum, ChrPtr(RoomNameVec));
947
948         while (Pos != StrBufNOTNULL){
949                 StrBufExtract_NextToken(oneRoom, RoomNameVec, &Pos, '|');
950                 CtdlDeleteMessages(ChrPtr(oneRoom), &msgnum, 1, "");
951         };
952         GetMetaData(&smi, msgnum);
953         AdjRefCount(msgnum, -smi.meta_refcount);
954 }
955
956
957 void migr_do_restore_meta(void) {
958         char buf[SIZ];
959         int failGetMessage;
960         long msgnum = 0;
961         int lastnum = 0;
962         int refcount = 0;
963         CitContext *Ctx;
964         char *prn;
965         StrBuf *RoomNames;
966         char cmd[SIZ];
967
968         migr_global_message_list = fopen(migr_tempfilename1, "w");
969         if (migr_global_message_list != NULL) {
970                 CtdlForEachRoom(migr_check_rooms_backend, NULL);
971                 fclose(migr_global_message_list);
972         }
973
974         // Process the 'global' message list.  (Sort it and remove dups.
975         // Dups are ok because a message may be in more than one room, but
976         // this will be handled by exporting the reference count, not by
977         // exporting the message multiple times.)
978         snprintf(cmd, sizeof cmd, "sort -n <%s >%s", migr_tempfilename1, migr_tempfilename2);
979         if (system(cmd) != 0) {
980                 syslog(LOG_ERR, "migrate: error %d", errno);
981         }
982
983         RoomNames = NewStrBuf();
984         Ctx = CC;
985         migr_global_message_list = fopen(migr_tempfilename2, "r");
986         if (migr_global_message_list != NULL) {
987                 syslog(LOG_INFO, "migrate: opened %s", migr_tempfilename1);
988                 while ((Ctx->kill_me == 0) && 
989                        (fgets(buf, sizeof(buf), migr_global_message_list) != NULL)) {
990                         msgnum = atol(buf);
991                         if (msgnum == 0L) 
992                                 continue;
993                         if (lastnum == 0) {
994                                 lastnum = msgnum;
995                         }
996                         prn = strchr(buf, ' ');
997                         if (lastnum != msgnum) {
998                                 failGetMessage = migr_restore_message_metadata(lastnum, refcount);
999                                 if (failGetMessage) {
1000                                         RemoveMessagesFromRooms(RoomNames, lastnum);
1001                                 }
1002                                 refcount = 1;
1003                                 lastnum = msgnum;
1004                                 if (prn != NULL) {
1005                                         StrBufPlain(RoomNames, prn + 1, -1);
1006                                 }
1007                                 StrBufTrim(RoomNames);
1008                         }
1009                         else {
1010                                 if (prn != NULL) {
1011                                         if (StrLength(RoomNames) > 0) {
1012                                                 StrBufAppendBufPlain(RoomNames, HKEY("|"), 0);
1013                                         }
1014                                         StrBufAppendBufPlain(RoomNames, prn, -1, 1);
1015                                         StrBufTrim(RoomNames);
1016                                 }
1017                                 refcount ++;
1018                         }
1019                         lastnum = msgnum;
1020                 }
1021                 failGetMessage = migr_restore_message_metadata(msgnum, refcount);
1022                 if (failGetMessage) {
1023                         RemoveMessagesFromRooms(RoomNames, lastnum);
1024                 }
1025                 fclose(migr_global_message_list);
1026         }
1027
1028         migr_restore_message_metadata(-1L, -1); // This frees the encoding buffer
1029         cprintf("%d system analysis completed", CIT_OK);
1030         Ctx->kill_me = 1;
1031 }
1032
1033
1034 // ******************************************************************************
1035 // *                         Dispatcher, Common code                            *
1036 // ******************************************************************************
1037 void cmd_migr(char *cmdbuf) {
1038         char cmd[32];
1039         
1040         if (CtdlAccessCheck(ac_aide)) return;
1041         
1042         if (CtdlTrySingleUser()) {
1043                 CtdlDisableHouseKeeping();
1044                 CtdlMakeTempFileName(migr_tempfilename1, sizeof migr_tempfilename1);
1045                 CtdlMakeTempFileName(migr_tempfilename2, sizeof migr_tempfilename2);
1046
1047                 extract_token(cmd, cmdbuf, 0, '|', sizeof cmd);
1048                 if (!strcasecmp(cmd, "export")) {
1049                         migr_do_export();
1050                 }
1051                 else if (!strcasecmp(cmd, "import")) {
1052                         migr_do_import();
1053                 }
1054                 else if (!strcasecmp(cmd, "listdirs")) {
1055                         migr_do_listdirs();
1056                 }
1057                 else if (!strcasecmp(cmd, "restoremeta")) {
1058                         migr_do_restore_meta();
1059                 }
1060                 else {
1061                         cprintf("%d illegal command\n", ERROR + ILLEGAL_VALUE);
1062                 }
1063
1064                 unlink(migr_tempfilename1);
1065                 unlink(migr_tempfilename2);
1066
1067                 CtdlEnableHouseKeeping();
1068                 CtdlEndSingleUser();
1069         }
1070         else {
1071                 cprintf("%d The migrator is already running.\n", ERROR + RESOURCE_BUSY);
1072         }
1073 }
1074
1075
1076 // Initialization function, called from modules_init.c
1077 char *ctdl_module_init_migrate() {
1078         if (!threading) {
1079                 CtdlRegisterProtoHook(cmd_migr, "MIGR", "Across-the-wire migration");
1080         }
1081         
1082         // return our module name for the log
1083         return "migrate";
1084 }