]> code.citadel.org Git - citadel.git/blob - citadel/server/modules/migrate/serv_migrate.c
citserver: remove openid support
[citadel.git] / citadel / server / modules / migrate / serv_migrate.c
1 // This module dumps and/or loads the Citadel database in XML format.
2 //
3 // Copyright (c) 1987-2022 by the citadel.org team
4 //
5 // This program is open source software; you can redistribute it and/or modify
6 // it under the terms of the GNU General Public License version 3.
7 //
8 // This program is distributed in the hope that it will be useful,
9 // but WITHOUT ANY WARRANTY; without even the implied warranty of
10 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
11 // GNU General Public License for more details.
12 //
13 // Explanation of <progress> tags:
14 //
15 // 0%           started
16 // 2%           finished exporting configuration
17 // 7%           finished exporting users
18 // 12%          (openid is no longer supported)
19 // 17%          finished exporting rooms
20 // 18%          finished exporting floors
21 // 25%          finished exporting visits
22 // 26-99%       exporting messages
23 // 100%         finished exporting messages
24 //
25 // These tags are inserted into the XML stream to give the reader an approximation of its progress.
26
27 #include "../../sysdep.h"
28 #include <stdlib.h>
29 #include <unistd.h>
30 #include <stdio.h>
31 #include <fcntl.h>
32 #include <signal.h>
33 #include <pwd.h>
34 #include <errno.h>
35 #include <sys/types.h>
36 #include <time.h>
37 #include <sys/wait.h>
38 #include <string.h>
39 #include <ctype.h>
40 #include <limits.h>
41 #include <expat.h>
42 #include <libcitadel.h>
43 #include "../../citadel_defs.h"
44 #include "../../server.h"
45 #include "../../citserver.h"
46 #include "../../support.h"
47 #include "../../config.h"
48 #include "../../database.h"
49 #include "../../msgbase.h"
50 #include "../../user_ops.h"
51 #include "../../euidindex.h"
52 #include "../../internet_addressing.h"
53 #include "../../ctdl_module.h"
54
55 char migr_tempfilename1[PATH_MAX];
56 char migr_tempfilename2[PATH_MAX];
57 FILE *migr_global_message_list;
58 int total_msgs = 0;
59 char *ikey = NULL;                      // If we're importing a config key we store it here.
60
61 //*****************************************************************************
62 //*       Code which implements the export appears in this section            *
63 //*****************************************************************************
64
65 // Output a string to the client with these characters escaped:  & < > " '
66 void xml_strout(char *str) {
67
68         char *c = str;
69
70         if (str == NULL) {
71                 return;
72         }
73
74         while (*c != 0) {
75                 if (*c == '\"') {
76                         client_write(HKEY("&quot;"));
77                 }
78                 else if (*c == '\'') {
79                         client_write(HKEY("&apos;"));
80                 }
81                 else if (*c == '<') {
82                         client_write(HKEY("&lt;"));
83                 }
84                 else if (*c == '>') {
85                         client_write(HKEY("&gt;"));
86                 }
87                 else if (*c == '&') {
88                         client_write(HKEY("&amp;"));
89                 }
90                 else {
91                         client_write(c, 1);
92                 }
93                 ++c;
94         }
95 }
96
97
98 // Export a user record as XML
99 void migr_export_users_backend(char *username, void *data) {
100
101         struct ctdluser u;
102         if (CtdlGetUser(&u, username) != 0) {
103                 return;
104         }
105
106         client_write(HKEY("<user>\n"));
107         cprintf("<u_version>%d</u_version>\n", u.version);
108         cprintf("<u_uid>%ld</u_uid>\n", (long)u.uid);
109         client_write(HKEY("<u_password>"));     xml_strout(u.password);         client_write(HKEY("</u_password>\n"));
110         cprintf("<u_flags>%u</u_flags>\n", u.flags);
111         cprintf("<u_timescalled>%ld</u_timescalled>\n", u.timescalled);
112         cprintf("<u_posted>%ld</u_posted>\n", u.posted);
113         cprintf("<u_axlevel>%d</u_axlevel>\n", u.axlevel);
114         cprintf("<u_usernum>%ld</u_usernum>\n", u.usernum);
115         cprintf("<u_lastcall>%ld</u_lastcall>\n", (long)u.lastcall);
116         cprintf("<u_USuserpurge>%d</u_USuserpurge>\n", u.USuserpurge);
117         client_write(HKEY("<u_fullname>"));     xml_strout(u.fullname);         client_write(HKEY("</u_fullname>\n"));
118         cprintf("<u_msgnum_bio>%ld</u_msgnum_bio>\n", u.msgnum_bio);
119         cprintf("<u_msgnum_pic>%ld</u_msgnum_pic>\n", u.msgnum_pic);
120         cprintf("<u_emailaddrs>%s</u_emailaddrs>\n", u.emailaddrs);
121         cprintf("<u_msgnum_inboxrules>%ld</u_msgnum_inboxrules>\n", u.msgnum_inboxrules);
122         cprintf("<u_lastproc_inboxrules>%ld</u_lastproc_inboxrules>\n", u.lastproc_inboxrules);
123         client_write(HKEY("</user>\n"));
124 }
125
126
127 void migr_export_users(void) {
128         ForEachUser(migr_export_users_backend, NULL);
129 }
130
131
132 void migr_export_room_msg(long msgnum, void *userdata) {
133         static int count = 0;
134
135         cprintf("%ld,", msgnum);
136         if (++count%10==0) {
137                 cprintf("\n");
138         }
139         fprintf(migr_global_message_list, "%ld\n", msgnum);
140 }
141
142
143 void migr_export_rooms_backend(struct ctdlroom *buf, void *data) {
144         client_write(HKEY("<room>\n"));
145         client_write(HKEY("<QRname>"));
146         xml_strout(buf->QRname);
147         client_write(HKEY("</QRname>\n"));
148         client_write(HKEY("<QRpasswd>"));
149         xml_strout(buf->QRpasswd);
150         client_write(HKEY("</QRpasswd>\n"));
151         cprintf("<QRroomaide>%ld</QRroomaide>\n", buf->QRroomaide);
152         cprintf("<QRhighest>%ld</QRhighest>\n", buf->QRhighest);
153         cprintf("<QRgen>%ld</QRgen>\n", (long)buf->QRgen);
154         cprintf("<QRflags>%u</QRflags>\n", buf->QRflags);
155         if (buf->QRflags & QR_DIRECTORY) {
156                 client_write(HKEY("<QRdirname>"));
157                 xml_strout(buf->QRdirname);
158                 client_write(HKEY("</QRdirname>\n"));
159         }
160         cprintf("<QRfloor>%d</QRfloor>\n", buf->QRfloor);
161         cprintf("<QRmtime>%ld</QRmtime>\n", (long)buf->QRmtime);
162         cprintf("<QRexpire_mode>%d</QRexpire_mode>\n", buf->QRep.expire_mode);
163         cprintf("<QRexpire_value>%d</QRexpire_value>\n", buf->QRep.expire_value);
164         cprintf("<QRnumber>%ld</QRnumber>\n", buf->QRnumber);
165         cprintf("<QRorder>%d</QRorder>\n", buf->QRorder);
166         cprintf("<QRflags2>%u</QRflags2>\n", buf->QRflags2);
167         cprintf("<QRdefaultview>%d</QRdefaultview>\n", buf->QRdefaultview);
168         cprintf("<msgnum_info>%ld</msgnum_info>\n", buf->msgnum_info);
169         cprintf("<msgnum_pic>%ld</msgnum_pic>\n", buf->msgnum_pic);
170         client_write(HKEY("</room>\n"));
171
172         // message list goes inside this tag
173         CtdlGetRoom(&CC->room, buf->QRname);
174         client_write(HKEY("<room_messages>"));
175         client_write(HKEY("<FRname>"));
176         xml_strout(buf->QRname);                        // buf->QRname rather than CC->room.QRname to guarantee consistency
177         client_write(HKEY("</FRname>\n"));
178         client_write(HKEY("<FRmsglist>\n"));
179         CtdlForEachMessage(MSGS_ALL, 0L, NULL, NULL, NULL, migr_export_room_msg, NULL);
180         client_write(HKEY("</FRmsglist>\n"));
181         client_write(HKEY("</room_messages>\n"));
182 }
183
184
185 void migr_export_rooms(void) {
186         char cmd[SIZ];
187         migr_global_message_list = fopen(migr_tempfilename1, "w");
188         if (migr_global_message_list != NULL) {
189                 CtdlForEachRoom(migr_export_rooms_backend, NULL);
190                 fclose(migr_global_message_list);
191         }
192
193         // Process the 'global' message list.  (Sort it and remove dups.
194         // Dups are ok because a message may be in more than one room, but
195         // this will be handled by exporting the reference count, not by
196         // exporting the message multiple times.)
197         snprintf(cmd, sizeof cmd, "sort -n <%s >%s", migr_tempfilename1, migr_tempfilename2);
198         if (system(cmd) != 0) {
199                 syslog(LOG_ERR, "migrate: error %d", errno);
200         }
201         snprintf(cmd, sizeof cmd, "uniq <%s >%s", migr_tempfilename2, migr_tempfilename1);
202         if (system(cmd) != 0) {
203                 syslog(LOG_ERR, "migrate: error %d", errno);
204         }
205
206         snprintf(cmd, sizeof cmd, "wc -l %s", migr_tempfilename1);
207         FILE *fp = popen(cmd, "r");
208         if (fp) {
209                 fgets(cmd, sizeof cmd, fp);
210                 pclose(fp);
211                 total_msgs = atoi(cmd);
212         }
213         else {
214                 total_msgs = 1; // any nonzero just to keep it from barfing
215         }
216         syslog(LOG_DEBUG, "migrate: total messages to be exported: %d", total_msgs);
217 }
218
219
220 void migr_export_floors(void) {
221         struct floor qfbuf, *buf;
222         int i;
223
224         for (i=0; i < MAXFLOORS; ++i) {
225                 client_write(HKEY("<floor>\n"));
226                 cprintf("<f_num>%d</f_num>\n", i);
227                 CtdlGetFloor(&qfbuf, i);
228                 buf = &qfbuf;
229                 cprintf("<f_flags>%u</f_flags>\n", buf->f_flags);
230                 client_write(HKEY("<f_name>")); xml_strout(buf->f_name); client_write(HKEY("</f_name>\n"));
231                 cprintf("<f_ref_count>%d</f_ref_count>\n", buf->f_ref_count);
232                 cprintf("<f_ep_expire_mode>%d</f_ep_expire_mode>\n", buf->f_ep.expire_mode);
233                 cprintf("<f_ep_expire_value>%d</f_ep_expire_value>\n", buf->f_ep.expire_value);
234                 client_write(HKEY("</floor>\n"));
235         }
236 }
237
238
239 // Return nonzero if the supplied string contains only characters which are valid in a sequence set.
240 int is_sequence_set(char *s) {
241         if (!s) return(0);
242
243         char *c = s;
244         char ch;
245         while (ch = *c++, ch) {
246                 if (!strchr("0123456789*,:", ch)) {
247                         return(0);
248                 }
249         }
250         return(1);
251 }
252
253
254 // Traverse the visits file...
255 void migr_export_visits(void) {
256         struct visit vbuf;
257         struct cdbdata *cdbv;
258
259         cdb_rewind(CDB_VISIT);
260
261         while (cdbv = cdb_next_item(CDB_VISIT), cdbv != NULL) {
262                 memset(&vbuf, 0, sizeof(struct visit));
263                 memcpy(&vbuf, cdbv->ptr,
264                        ((cdbv->len > sizeof(struct visit)) ?
265                         sizeof(struct visit) : cdbv->len));
266                 cdb_free(cdbv);
267
268                 client_write(HKEY("<visit>\n"));
269                 cprintf("<v_roomnum>%ld</v_roomnum>\n", vbuf.v_roomnum);
270                 cprintf("<v_roomgen>%ld</v_roomgen>\n", vbuf.v_roomgen);
271                 cprintf("<v_usernum>%ld</v_usernum>\n", vbuf.v_usernum);
272
273                 client_write(HKEY("<v_seen>"));
274                 if ( (!IsEmptyStr(vbuf.v_seen)) && (is_sequence_set(vbuf.v_seen)) ) {
275                         xml_strout(vbuf.v_seen);
276                 }
277                 else {
278                         cprintf("%ld", vbuf.v_lastseen);
279                 }
280                 client_write(HKEY("</v_seen>"));
281
282                 if ( (!IsEmptyStr(vbuf.v_answered)) && (is_sequence_set(vbuf.v_answered)) ) {
283                         client_write(HKEY("<v_answered>"));
284                         xml_strout(vbuf.v_answered);
285                         client_write(HKEY("</v_answered>\n"));
286                 }
287
288                 cprintf("<v_flags>%u</v_flags>\n", vbuf.v_flags);
289                 cprintf("<v_view>%d</v_view>\n", vbuf.v_view);
290                 client_write(HKEY("</visit>\n"));
291         }
292 }
293
294
295 void migr_export_message(long msgnum) {
296         struct MetaData smi;
297         struct CtdlMessage *msg;
298         struct ser_ret smr;
299         long bytes_written = 0;
300         long this_block = 0;
301
302         // We can use a static buffer here because there will never be more than
303         // one of this operation happening at any given time, and it's really best
304         // to just keep it allocated once instead of torturing malloc/free.
305         // Call this function with msgnum "-1" to free the buffer when finished.
306
307         static int encoded_alloc = 0;
308         static char *encoded_msg = NULL;
309
310         if (msgnum < 0) {
311                 if ((encoded_alloc == 0) && (encoded_msg != NULL)) {
312                         free(encoded_msg);
313                         encoded_alloc = 0;
314                         encoded_msg = NULL;
315                 }
316                 return;
317         }
318
319         // Ok, here we go ...
320
321         msg = CtdlFetchMessage(msgnum, 1);
322         if (msg == NULL) return;                                // fail silently
323
324         client_write(HKEY("<message>\n"));
325         GetMetaData(&smi, msgnum);
326         cprintf("<msg_msgnum>%ld</msg_msgnum>\n", msgnum);
327         cprintf("<msg_meta_refcount>%d</msg_meta_refcount>\n", smi.meta_refcount);
328         cprintf("<msg_meta_rfc822_length>%ld</msg_meta_rfc822_length>\n", smi.meta_rfc822_length);
329         client_write(HKEY("<msg_meta_content_type>")); xml_strout(smi.meta_content_type); client_write(HKEY("</msg_meta_content_type>\n"));
330
331         client_write(HKEY("<msg_text>"));
332         CtdlSerializeMessage(&smr, msg);
333         CM_Free(msg);
334
335         // Predict the buffer size we need.  Expand the buffer if necessary.
336         int encoded_len = smr.len * 15 / 10 ;
337         if (encoded_len > encoded_alloc) {
338                 encoded_alloc = encoded_len;
339                 encoded_msg = realloc(encoded_msg, encoded_alloc);
340         }
341
342         if (encoded_msg == NULL) {
343                 // Questionable hack that hopes it'll work next time and we only lose one message
344                 encoded_alloc = 0;
345         }
346         else {
347                 // Once we do the encoding we know the exact size
348                 encoded_len = CtdlEncodeBase64(encoded_msg, (char *)smr.ser, smr.len, BASE64_YES_LINEBREAKS);
349
350                 // Careful now.  If the message is gargantuan, trying to write multiple gigamegs in one
351                 // big write operation can make our transport unhappy.  So we'll chunk it up 10 KB at a time.
352                 bytes_written = 0;
353                 while ( (bytes_written < encoded_len) && (!server_shutting_down) ) {
354                         this_block = encoded_len - bytes_written;
355                         if (this_block > 10240) {
356                                 this_block = 10240;
357                         }
358                         client_write(&encoded_msg[bytes_written], this_block);
359                         bytes_written += this_block;
360                 }
361         }
362
363         free(smr.ser);
364
365         client_write(HKEY("</msg_text>\n"));
366         client_write(HKEY("</message>\n"));
367 }
368
369
370 void migr_export_configs(void) {
371         struct cdbdata *cdbcfg;
372         int keylen = 0;
373         char *key = NULL;
374         char *value = NULL;
375
376         cdb_rewind(CDB_CONFIG);
377         while (cdbcfg = cdb_next_item(CDB_CONFIG), cdbcfg != NULL) {
378
379                 keylen = strlen(cdbcfg->ptr);
380                 key = cdbcfg->ptr;
381                 value = cdbcfg->ptr + keylen + 1;
382
383                 client_write("<config key=\"", 13);
384                 xml_strout(key);
385                 client_write("\">", 2);
386                 xml_strout(value);
387                 client_write("</config>\n", 10);
388                 cdb_free(cdbcfg);
389         }
390 }
391
392
393 void migr_export_messages(void) {
394         char buf[SIZ];
395         long msgnum;
396         int count = 0;
397         int progress = 0;
398         int prev_progress = 0;
399         CitContext *Ctx;
400
401         Ctx = CC;
402         migr_global_message_list = fopen(migr_tempfilename1, "r");
403         if (migr_global_message_list != NULL) {
404                 syslog(LOG_INFO, "migrate: opened %s", migr_tempfilename1);
405                 while ((Ctx->kill_me == 0) && 
406                        (fgets(buf, sizeof(buf), migr_global_message_list) != NULL)) {
407                         msgnum = atol(buf);
408                         if (msgnum > 0L) {
409                                 migr_export_message(msgnum);
410                                 ++count;
411                         }
412                         progress = (count * 74 / total_msgs) + 25 ;
413                         if ((progress > prev_progress) && (progress < 100)) {
414                                 cprintf("<progress>%d</progress>\n", progress);
415                         }
416                         prev_progress = progress;
417                 }
418                 fclose(migr_global_message_list);
419         }
420         if (Ctx->kill_me == 0) {
421                 syslog(LOG_INFO, "migrate: exported %d messages.", count);
422         }
423         else {
424                 syslog(LOG_ERR, "migrate: export aborted due to client disconnect!");
425         }
426
427         migr_export_message(-1L);       // This frees the encoding buffer
428 }
429
430
431 void migr_do_export(void) {
432         CitContext *Ctx;
433
434         Ctx = CC;
435         cprintf("%d Exporting all Citadel databases.\n", LISTING_FOLLOWS);
436         Ctx->dont_term = 1;
437
438         client_write(HKEY("<?xml version=\"1.0\" encoding=\"UTF-8\" ?>\n"));
439         client_write(HKEY("<citadel_migrate_data>\n"));
440         cprintf("<version>%d</version>\n", REV_LEVEL);
441         cprintf("<progress>%d</progress>\n", 0);
442
443         // export the configuration database
444         migr_export_configs();
445         cprintf("<progress>%d</progress>\n", 2);
446         
447         if (Ctx->kill_me == 0)  migr_export_users();
448         cprintf("<progress>%d</progress>\n", 7);
449         if (Ctx->kill_me == 0)  migr_export_rooms();
450         cprintf("<progress>%d</progress>\n", 17);
451         if (Ctx->kill_me == 0)  migr_export_floors();
452         cprintf("<progress>%d</progress>\n", 18);
453         if (Ctx->kill_me == 0)  migr_export_visits();
454         cprintf("<progress>%d</progress>\n", 25);
455         if (Ctx->kill_me == 0)  migr_export_messages();
456         cprintf("<progress>%d</progress>\n", 100);
457         client_write(HKEY("</citadel_migrate_data>\n"));
458         client_write(HKEY("000\n"));
459         Ctx->dont_term = 0;
460 }
461
462
463 //                              Import code
464 //   Here's the code that implements the import side.  It's going to end up
465 //       being one big loop with lots of global variables.  I don't care.
466 // You wouldn't run multiple concurrent imports anyway.  If this offends your
467 //  delicate sensibilities  then go rewrite it in Ruby on Rails or something.
468
469
470 int citadel_migrate_data = 0;           // Are we inside a <citadel_migrate_data> tag pair?
471 StrBuf *migr_chardata = NULL;
472 StrBuf *migr_MsgData = NULL;
473 struct ctdluser usbuf;
474 struct ctdlroom qrbuf;
475 char FRname[ROOMNAMELEN];
476 struct floor flbuf;
477 int floornum = 0;
478 struct visit vbuf;
479 struct MetaData smi;
480 long import_msgnum = 0;
481
482 // This callback stores up the data which appears in between tags.
483 void migr_xml_chardata(void *data, const XML_Char *s, int len) {
484         StrBufAppendBufPlain(migr_chardata, s, len, 0);
485 }
486
487
488 void migr_xml_start(void *data, const char *el, const char **attr) {
489         int i;
490
491         //  *** GENERAL STUFF ***
492
493         // Reset chardata_len to zero and init buffer
494         FlushStrBuf(migr_chardata);
495         FlushStrBuf(migr_MsgData);
496
497         if (!strcasecmp(el, "citadel_migrate_data")) {
498                 ++citadel_migrate_data;
499
500                 // As soon as it looks like the input data is a genuine Citadel XML export,
501                 // whack the existing database on disk to make room for the new one.
502                 if (citadel_migrate_data == 1) {
503                         syslog(LOG_INFO, "migrate: erasing existing databases so we can receive the incoming import");
504                         for (i = 0; i < MAXCDB; ++i) {
505                                 cdb_trunc(i);
506                         }
507                 }
508                 return;
509         }
510
511         if (citadel_migrate_data != 1) {
512                 syslog(LOG_ERR, "migrate: out-of-sequence tag <%s> detected.  Warning: ODD-DATA!", el);
513                 return;
514         }
515
516         // When we begin receiving XML for one of these record types, clear out the associated
517         // buffer so we don't accidentally carry over any data from a previous record.
518         if (!strcasecmp(el, "user"))                    memset(&usbuf, 0, sizeof(struct ctdluser));
519         else if (!strcasecmp(el, "room"))               memset(&qrbuf, 0, sizeof(struct ctdlroom));
520         else if (!strcasecmp(el, "room_messages"))      memset(FRname, 0, sizeof FRname);
521         else if (!strcasecmp(el, "floor"))              memset(&flbuf, 0, sizeof(struct floor));
522         else if (!strcasecmp(el, "visit"))              memset(&vbuf, 0, sizeof(struct visit));
523
524         else if (!strcasecmp(el, "message")) {
525                 memset(&smi, 0, sizeof (struct MetaData));
526                 import_msgnum = 0;
527         }
528         else if (!strcasecmp(el, "config")) {
529                 if (ikey != NULL) {
530                         free(ikey);
531                         ikey = NULL;
532                 }
533                 while (*attr) {
534                         if (!strcasecmp(attr[0], "key")) {
535                                 ikey = strdup(attr[1]);
536                         }
537                         attr += 2;
538                 }
539         }
540
541 }
542
543
544 int migr_userrecord(void *data, const char *el) {
545         if (!strcasecmp(el, "u_version"))                       usbuf.version = atoi(ChrPtr(migr_chardata));
546         else if (!strcasecmp(el, "u_uid"))                      usbuf.uid = atol(ChrPtr(migr_chardata));
547         else if (!strcasecmp(el, "u_password"))                 safestrncpy(usbuf.password, ChrPtr(migr_chardata), sizeof usbuf.password);
548         else if (!strcasecmp(el, "u_flags"))                    usbuf.flags = atoi(ChrPtr(migr_chardata));
549         else if (!strcasecmp(el, "u_timescalled"))              usbuf.timescalled = atol(ChrPtr(migr_chardata));
550         else if (!strcasecmp(el, "u_posted"))                   usbuf.posted = atol(ChrPtr(migr_chardata));
551         else if (!strcasecmp(el, "u_axlevel"))                  usbuf.axlevel = atoi(ChrPtr(migr_chardata));
552         else if (!strcasecmp(el, "u_usernum"))                  usbuf.usernum = atol(ChrPtr(migr_chardata));
553         else if (!strcasecmp(el, "u_lastcall"))                 usbuf.lastcall = atol(ChrPtr(migr_chardata));
554         else if (!strcasecmp(el, "u_USuserpurge"))              usbuf.USuserpurge = atoi(ChrPtr(migr_chardata));
555         else if (!strcasecmp(el, "u_fullname"))                 safestrncpy(usbuf.fullname, ChrPtr(migr_chardata), sizeof usbuf.fullname);
556         else if (!strcasecmp(el, "u_msgnum_bio"))               usbuf.msgnum_bio = atol(ChrPtr(migr_chardata));
557         else if (!strcasecmp(el, "u_msgnum_pic"))               usbuf.msgnum_pic = atol(ChrPtr(migr_chardata));
558         else if (!strcasecmp(el, "u_emailaddrs"))               safestrncpy(usbuf.emailaddrs, ChrPtr(migr_chardata), sizeof usbuf.emailaddrs);
559         else if (!strcasecmp(el, "u_msgnum_inboxrules"))        usbuf.msgnum_inboxrules = atol(ChrPtr(migr_chardata));
560         else if (!strcasecmp(el, "u_lastproc_inboxrules"))      usbuf.lastproc_inboxrules = atol(ChrPtr(migr_chardata));
561         else return 0;
562         return 1;
563 }
564
565
566 int migr_roomrecord(void *data, const char *el) {
567         if (!strcasecmp(el, "QRname"))                          safestrncpy(qrbuf.QRname, ChrPtr(migr_chardata), sizeof qrbuf.QRname);
568         else if (!strcasecmp(el, "QRpasswd"))                   safestrncpy(qrbuf.QRpasswd, ChrPtr(migr_chardata), sizeof qrbuf.QRpasswd);
569         else if (!strcasecmp(el, "QRroomaide"))                 qrbuf.QRroomaide = atol(ChrPtr(migr_chardata));
570         else if (!strcasecmp(el, "QRhighest"))                  qrbuf.QRhighest = atol(ChrPtr(migr_chardata));
571         else if (!strcasecmp(el, "QRgen"))                      qrbuf.QRgen = atol(ChrPtr(migr_chardata));
572         else if (!strcasecmp(el, "QRflags"))                    qrbuf.QRflags = atoi(ChrPtr(migr_chardata));
573         else if (!strcasecmp(el, "QRdirname"))                  safestrncpy(qrbuf.QRdirname, ChrPtr(migr_chardata), sizeof qrbuf.QRdirname);
574         else if (!strcasecmp(el, "QRfloor"))                    qrbuf.QRfloor = atoi(ChrPtr(migr_chardata));
575         else if (!strcasecmp(el, "QRmtime"))                    qrbuf.QRmtime = atol(ChrPtr(migr_chardata));
576         else if (!strcasecmp(el, "QRexpire_mode"))              qrbuf.QRep.expire_mode = atoi(ChrPtr(migr_chardata));
577         else if (!strcasecmp(el, "QRexpire_value"))             qrbuf.QRep.expire_value = atoi(ChrPtr(migr_chardata));
578         else if (!strcasecmp(el, "QRnumber"))                   qrbuf.QRnumber = atol(ChrPtr(migr_chardata));
579         else if (!strcasecmp(el, "QRorder"))                    qrbuf.QRorder = atoi(ChrPtr(migr_chardata));
580         else if (!strcasecmp(el, "QRflags2"))                   qrbuf.QRflags2 = atoi(ChrPtr(migr_chardata));
581         else if (!strcasecmp(el, "QRdefaultview"))              qrbuf.QRdefaultview = atoi(ChrPtr(migr_chardata));
582         else if (!strcasecmp(el, "msgnum_info"))                qrbuf.msgnum_info = atol(ChrPtr(migr_chardata));
583         else if (!strcasecmp(el, "msgnum_pic"))                 qrbuf.msgnum_pic = atol(ChrPtr(migr_chardata));
584         else return 0;
585         return 1;
586 }
587
588
589 int migr_floorrecord(void *data, const char *el) {
590         if (!strcasecmp(el, "f_num"))                           floornum = atoi(ChrPtr(migr_chardata));
591         else if (!strcasecmp(el, "f_flags"))                    flbuf.f_flags = atoi(ChrPtr(migr_chardata));
592         else if (!strcasecmp(el, "f_name"))                     safestrncpy(flbuf.f_name, ChrPtr(migr_chardata), sizeof flbuf.f_name);
593         else if (!strcasecmp(el, "f_ref_count"))                flbuf.f_ref_count = atoi(ChrPtr(migr_chardata));
594         else if (!strcasecmp(el, "f_ep_expire_mode"))           flbuf.f_ep.expire_mode = atoi(ChrPtr(migr_chardata));
595         else if (!strcasecmp(el, "f_ep_expire_value"))          flbuf.f_ep.expire_value = atoi(ChrPtr(migr_chardata));
596         else return 0;
597         return 1;
598 }
599
600
601 int migr_visitrecord(void *data, const char *el) {
602         if (!strcasecmp(el, "v_roomnum"))                       vbuf.v_roomnum = atol(ChrPtr(migr_chardata));
603         else if (!strcasecmp(el, "v_roomgen"))                  vbuf.v_roomgen = atol(ChrPtr(migr_chardata));
604         else if (!strcasecmp(el, "v_usernum"))                  vbuf.v_usernum = atol(ChrPtr(migr_chardata));
605
606         else if (!strcasecmp(el, "v_seen")) {
607                 int is_textual_seen = 0;
608                 int i;
609                 int max = StrLength(migr_chardata);
610
611                 vbuf.v_lastseen = atol(ChrPtr(migr_chardata));
612                 is_textual_seen = 0;
613                 for (i=0; i < max; ++i) 
614                         if (!isdigit(ChrPtr(migr_chardata)[i]))
615                                 is_textual_seen = 1;
616                 if (is_textual_seen)
617                         safestrncpy(vbuf.v_seen, ChrPtr(migr_chardata), sizeof vbuf.v_seen);
618         }
619
620         else if (!strcasecmp(el, "v_answered"))                 safestrncpy(vbuf.v_answered, ChrPtr(migr_chardata), sizeof vbuf.v_answered);
621         else if (!strcasecmp(el, "v_flags"))                    vbuf.v_flags = atoi(ChrPtr(migr_chardata));
622         else if (!strcasecmp(el, "v_view"))                     vbuf.v_view = atoi(ChrPtr(migr_chardata));
623         else return 0;
624         return 1;
625 }
626
627
628 void migr_xml_end(void *data, const char *el) {
629         const char *ptr;
630         int msgcount = 0;
631         long msgnum = 0L;
632         long *msglist = NULL;
633         int msglist_alloc = 0;
634         // *** GENERAL STUFF ***
635
636         if (!strcasecmp(el, "citadel_migrate_data")) {
637                 --citadel_migrate_data;
638                 return;
639         }
640
641         if (citadel_migrate_data != 1) {
642                 syslog(LOG_ERR, "migrate: out-of-sequence tag <%s> detected.  Warning: ODD-DATA!", el);
643                 return;
644         }
645
646         // syslog(LOG_DEBUG, "END TAG: <%s> DATA: <%s>\n", el, (migr_chardata_len ? migr_chardata : ""));
647
648         // *** CONFIG ***
649
650         if (!strcasecmp(el, "config")) {
651                 syslog(LOG_DEBUG, "migrate: imported config key=%s", ikey);
652
653                 if (ikey != NULL) {
654                         CtdlSetConfigStr(ikey, (char *)ChrPtr(migr_chardata));
655                         free(ikey);
656                         ikey = NULL;
657                 }
658                 else {
659                         syslog(LOG_INFO, "migrate: closed a <config> tag but no key name was supplied.");
660                 }
661         }
662
663         // *** USER ***
664         else if ((!strncasecmp(el, HKEY("u_"))) && 
665                  migr_userrecord(data, el))
666                 ; /* Nothing to do anymore */
667         else if (!strcasecmp(el, "user")) {
668                 CtdlPutUser(&usbuf);
669                 syslog(LOG_INFO, "migrate: imported user: %s", usbuf.fullname);
670         }
671
672         // *** ROOM ***
673         else if ((!strncasecmp(el, HKEY("QR"))) && 
674                  migr_roomrecord(data, el))
675                 ; // Nothing to do anymore
676         else if (!strcasecmp(el, "room")) {
677                 CtdlPutRoom(&qrbuf);
678                 syslog(LOG_INFO, "migrate: imported room: %s", qrbuf.QRname);
679         }
680
681         // *** ROOM MESSAGE POINTERS ***
682
683         else if (!strcasecmp(el, "FRname")) {
684                 safestrncpy(FRname, ChrPtr(migr_chardata), sizeof FRname);
685         }
686
687         else if (!strcasecmp(el, "FRmsglist")) {
688                 if (!IsEmptyStr(FRname)) {
689                         msgcount = 0;
690                         msglist_alloc = 1000;
691                         msglist = malloc(sizeof(long) * msglist_alloc);
692
693                         syslog(LOG_DEBUG, "migrate: message list for: %s", FRname);
694
695                         ptr = ChrPtr(migr_chardata);
696                         while (*ptr != 0) {
697                                 while ((*ptr != 0) && (!isdigit(*ptr))) {
698                                         ++ptr;
699                                 }
700                                 if ((*ptr != 0) && (isdigit(*ptr))) {
701                                         msgnum = atol(ptr);
702                                         if (msgnum > 0L) {
703                                                 if (msgcount >= msglist_alloc) {
704                                                         msglist_alloc *= 2;
705                                                         msglist = realloc(msglist, sizeof(long) * msglist_alloc);
706                                                 }
707                                                 msglist[msgcount++] = msgnum;
708                                                 }
709                                         }
710                                         while ((*ptr != 0) && (isdigit(*ptr))) {
711                                                 ++ptr;
712                                         }
713                                 }
714                         }
715                         if (msgcount > 0) {
716                                 CtdlSaveMsgPointersInRoom(FRname, msglist, msgcount, 0, NULL, 1);
717                         }
718                         free(msglist);
719                         msglist = NULL;
720                         msglist_alloc = 0;
721                         syslog(LOG_DEBUG, "migrate: imported %d messages.", msgcount);
722                         if (server_shutting_down) {
723                                 return;
724                 }
725         }
726
727         // *** FLOORS ***
728         else if ((!strncasecmp(el, HKEY("f_"))) && 
729                  migr_floorrecord(data, el))
730                 ; // Nothing to do anymore
731
732         else if (!strcasecmp(el, "floor")) {
733                 CtdlPutFloor(&flbuf, floornum);
734                 syslog(LOG_INFO, "migrate: imported floor #%d (%s)", floornum, flbuf.f_name);
735         }
736
737         // *** VISITS ***
738         else if ((!strncasecmp(el, HKEY("v_"))) && migr_visitrecord(data, el)) {
739                 ; // Nothing to do anymore
740         }
741         else if (!strcasecmp(el, "visit")) {
742                 put_visit(&vbuf);
743                 syslog(LOG_INFO, "migrate: imported visit: %ld/%ld/%ld", vbuf.v_roomnum, vbuf.v_roomgen, vbuf.v_usernum);
744         }
745
746         // *** MESSAGES ***
747         
748         else if (!strcasecmp(el, "msg_msgnum"))                 smi.meta_msgnum = import_msgnum = atol(ChrPtr(migr_chardata));
749         else if (!strcasecmp(el, "msg_meta_refcount"))          smi.meta_refcount = atoi(ChrPtr(migr_chardata));
750         else if (!strcasecmp(el, "msg_meta_rfc822_length"))     smi.meta_rfc822_length = atoi(ChrPtr(migr_chardata));
751         else if (!strcasecmp(el, "msg_meta_content_type"))      safestrncpy(smi.meta_content_type, ChrPtr(migr_chardata), sizeof smi.meta_content_type);
752
753         else if (!strcasecmp(el, "msg_text")) {
754                 long rc;
755                 struct CtdlMessage *msg;
756
757                 FlushStrBuf(migr_MsgData);
758                 StrBufDecodeBase64To(migr_chardata, migr_MsgData);
759
760                 msg = CtdlDeserializeMessage(import_msgnum, -1,
761                                              ChrPtr(migr_MsgData), 
762                                              StrLength(migr_MsgData));
763                 if (msg != NULL) {
764                         rc = CtdlSaveThisMessage(msg, import_msgnum, 0);
765                         if (rc == 0) {
766                                 PutMetaData(&smi);
767                         }
768                         CM_Free(msg);
769                 }
770                 else {
771                         rc = -1;
772                 }
773
774                 syslog(LOG_INFO,
775                        "migrate: %s message #%ld, size=%d, ref=%d, body=%ld, content-type: %s",
776                        (rc!= 0)?"failed to import":"imported",
777                        import_msgnum,
778                        StrLength(migr_MsgData),
779                        smi.meta_refcount,
780                        smi.meta_rfc822_length,
781                        smi.meta_content_type
782                 );
783                 memset(&smi, 0, sizeof(smi));
784         }
785
786         // *** MORE GENERAL STUFF ***
787
788         FlushStrBuf(migr_chardata);
789 }
790
791
792 // Import begins here
793 void migr_do_import(void) {
794         XML_Parser xp;
795         char buf[SIZ];
796         
797         unbuffer_output();
798         migr_chardata = NewStrBufPlain(NULL, SIZ * 20);
799         migr_MsgData = NewStrBufPlain(NULL, SIZ * 20);
800         xp = XML_ParserCreate(NULL);
801         if (!xp) {
802                 cprintf("%d Failed to create XML parser instance\n", ERROR+INTERNAL_ERROR);
803                 return;
804         }
805         XML_SetElementHandler(xp, migr_xml_start, migr_xml_end);
806         XML_SetCharacterDataHandler(xp, migr_xml_chardata);
807
808         CC->dont_term = 1;
809
810         cprintf("%d sock it to me\n", SEND_LISTING);
811         unbuffer_output();
812
813         client_set_inbound_buf(SIZ * 10);
814
815         while (client_getln(buf, sizeof buf) >= 0 && strcmp(buf, "000")) {
816                 XML_Parse(xp, buf, strlen(buf), 0);
817                 if (server_shutting_down)
818                         break;  // Should we break or return?
819         }
820
821         XML_Parse(xp, "", 0, 1);
822         XML_ParserFree(xp);
823         FreeStrBuf(&migr_chardata);
824         FreeStrBuf(&migr_MsgData);
825         rebuild_euid_index();
826         rebuild_usersbynumber();
827         CtdlSetConfigInt("MM_fulltext_wordbreaker", -1);        // Set an invalid wordbreaker to force re-indexing
828         CtdlRebuildDirectoryIndex();                            // Force a rebuild of the directory index
829         CC->dont_term = 0;
830 }
831
832
833 // ******************************************************************************
834 // *                         Dispatcher, Common code                            *
835 // ******************************************************************************
836
837 // Dump out the pathnames of directories which can be copied "as is"
838 void migr_do_listdirs(void) {
839         cprintf("%d Don't forget these:\n", LISTING_FOLLOWS);
840         cprintf("files|%s\n",           ctdl_file_dir);
841         cprintf("messages|%s\n",        ctdl_message_dir);
842         cprintf("keys|%s\n",            ctdl_key_dir);
843         cprintf("000\n");
844 }
845
846
847 // ******************************************************************************
848 // *                    Repair database integrity                               *
849 // ******************************************************************************
850
851 StrBuf *PlainMessageBuf = NULL;
852 HashList *UsedMessageIDS = NULL;
853
854 int migr_restore_message_metadata(long msgnum, int refcount) {
855         struct MetaData smi;
856         struct CtdlMessage *msg;
857         char *mptr = NULL;
858
859         // We can use a static buffer here because there will never be more than
860         // one of this operation happening at any given time, and it's really best
861         // to just keep it allocated once instead of torturing malloc/free.
862         // Call this function with msgnum "-1" to free the buffer when finished.
863         static int encoded_alloc = 0;
864         static char *encoded_msg = NULL;
865
866         if (msgnum < 0) {
867                 if ((encoded_alloc == 0) && (encoded_msg != NULL)) {
868                         free(encoded_msg);
869                         encoded_alloc = 0;
870                         encoded_msg = NULL;
871                         // todo FreeStrBuf(&PlainMessageBuf); PlainMessageBuf = NULL;
872                 }
873                 return 0;
874         }
875
876         if (PlainMessageBuf == NULL) {
877                 PlainMessageBuf = NewStrBufPlain(NULL, 10*SIZ);
878         }
879
880         // Ok, here we go ...
881
882         msg = CtdlFetchMessage(msgnum, 1);
883         if (msg == NULL) {
884                 return 1;
885         }
886
887         GetMetaData(&smi, msgnum);
888         smi.meta_msgnum = msgnum;
889         smi.meta_refcount = refcount;
890         
891         // restore the content type from the message body:
892         mptr = bmstrcasestr(msg->cm_fields[eMesageText], "Content-type:");
893         if (mptr != NULL) {
894                 char *aptr;
895                 safestrncpy(smi.meta_content_type, &mptr[13], sizeof smi.meta_content_type);
896                 string_trim(smi.meta_content_type);
897                 aptr = smi.meta_content_type;
898                 while (!IsEmptyStr(aptr)) {
899                         if ((*aptr == ';')
900                             || (*aptr == ' ')
901                             || (*aptr == 13)
902                             || (*aptr == 10)) {
903                                 memset(aptr, 0, sizeof(smi.meta_content_type) - (aptr - smi.meta_content_type));
904                         }
905                         else aptr++;
906                 }
907         }
908
909         CC->redirect_buffer = PlainMessageBuf;
910         CtdlOutputPreLoadedMsg(msg, MT_RFC822, HEADERS_ALL, 0, 1, QP_EADDR);
911         smi.meta_rfc822_length = StrLength(CC->redirect_buffer);
912         CC->redirect_buffer = NULL;
913
914         syslog(LOG_INFO,
915                "migrate: setting message #%ld meta data to: refcount=%d, bodylength=%ld, content-type: %s",
916                smi.meta_msgnum,
917                smi.meta_refcount,
918                smi.meta_rfc822_length,
919                smi.meta_content_type
920         );
921
922         PutMetaData(&smi);
923         CM_Free(msg);
924         return 0;
925 }
926
927
928 void migr_check_room_msg(long msgnum, void *userdata) {
929         fprintf(migr_global_message_list, "%ld %s\n", msgnum, CC->room.QRname);
930 }
931
932
933 void migr_check_rooms_backend(struct ctdlroom *buf, void *data) {
934         // message list goes inside this tag
935         CtdlGetRoom(&CC->room, buf->QRname);
936         CtdlForEachMessage(MSGS_ALL, 0L, NULL, NULL, NULL, migr_check_room_msg, NULL);
937 }
938
939
940 void RemoveMessagesFromRooms(StrBuf *RoomNameVec, long msgnum) {
941         struct MetaData smi;
942         const char *Pos = NULL;
943         StrBuf *oneRoom = NewStrBuf();
944
945         syslog(LOG_INFO, "migrate: removing message pointer %ld from these rooms: %s", msgnum, ChrPtr(RoomNameVec));
946
947         while (Pos != StrBufNOTNULL){
948                 StrBufExtract_NextToken(oneRoom, RoomNameVec, &Pos, '|');
949                 CtdlDeleteMessages(ChrPtr(oneRoom), &msgnum, 1, "");
950         };
951         GetMetaData(&smi, msgnum);
952         AdjRefCount(msgnum, -smi.meta_refcount);
953 }
954
955
956 void migr_do_restore_meta(void) {
957         char buf[SIZ];
958         int failGetMessage;
959         long msgnum = 0;
960         int lastnum = 0;
961         int refcount = 0;
962         CitContext *Ctx;
963         char *prn;
964         StrBuf *RoomNames;
965         char cmd[SIZ];
966
967         migr_global_message_list = fopen(migr_tempfilename1, "w");
968         if (migr_global_message_list != NULL) {
969                 CtdlForEachRoom(migr_check_rooms_backend, NULL);
970                 fclose(migr_global_message_list);
971         }
972
973         // Process the 'global' message list.  (Sort it and remove dups.
974         // Dups are ok because a message may be in more than one room, but
975         // this will be handled by exporting the reference count, not by
976         // exporting the message multiple times.)
977         snprintf(cmd, sizeof cmd, "sort -n <%s >%s", migr_tempfilename1, migr_tempfilename2);
978         if (system(cmd) != 0) {
979                 syslog(LOG_ERR, "migrate: error %d", errno);
980         }
981
982         RoomNames = NewStrBuf();
983         Ctx = CC;
984         migr_global_message_list = fopen(migr_tempfilename2, "r");
985         if (migr_global_message_list != NULL) {
986                 syslog(LOG_INFO, "migrate: opened %s", migr_tempfilename1);
987                 while ((Ctx->kill_me == 0) && 
988                        (fgets(buf, sizeof(buf), migr_global_message_list) != NULL)) {
989                         msgnum = atol(buf);
990                         if (msgnum == 0L) 
991                                 continue;
992                         if (lastnum == 0) {
993                                 lastnum = msgnum;
994                         }
995                         prn = strchr(buf, ' ');
996                         if (lastnum != msgnum) {
997                                 failGetMessage = migr_restore_message_metadata(lastnum, refcount);
998                                 if (failGetMessage) {
999                                         RemoveMessagesFromRooms(RoomNames, lastnum);
1000                                 }
1001                                 refcount = 1;
1002                                 lastnum = msgnum;
1003                                 if (prn != NULL) {
1004                                         StrBufPlain(RoomNames, prn + 1, -1);
1005                                 }
1006                                 StrBufTrim(RoomNames);
1007                         }
1008                         else {
1009                                 if (prn != NULL) {
1010                                         if (StrLength(RoomNames) > 0) {
1011                                                 StrBufAppendBufPlain(RoomNames, HKEY("|"), 0);
1012                                         }
1013                                         StrBufAppendBufPlain(RoomNames, prn, -1, 1);
1014                                         StrBufTrim(RoomNames);
1015                                 }
1016                                 refcount ++;
1017                         }
1018                         lastnum = msgnum;
1019                 }
1020                 failGetMessage = migr_restore_message_metadata(msgnum, refcount);
1021                 if (failGetMessage) {
1022                         RemoveMessagesFromRooms(RoomNames, lastnum);
1023                 }
1024                 fclose(migr_global_message_list);
1025         }
1026
1027         migr_restore_message_metadata(-1L, -1); // This frees the encoding buffer
1028         cprintf("%d system analysis completed", CIT_OK);
1029         Ctx->kill_me = 1;
1030 }
1031
1032
1033 // ******************************************************************************
1034 // *                         Dispatcher, Common code                            *
1035 // ******************************************************************************
1036 void cmd_migr(char *cmdbuf) {
1037         char cmd[32];
1038         
1039         if (CtdlAccessCheck(ac_aide)) return;
1040         
1041         if (CtdlTrySingleUser()) {
1042                 CtdlDisableHouseKeeping();
1043                 CtdlMakeTempFileName(migr_tempfilename1, sizeof migr_tempfilename1);
1044                 CtdlMakeTempFileName(migr_tempfilename2, sizeof migr_tempfilename2);
1045
1046                 extract_token(cmd, cmdbuf, 0, '|', sizeof cmd);
1047                 if (!strcasecmp(cmd, "export")) {
1048                         migr_do_export();
1049                 }
1050                 else if (!strcasecmp(cmd, "import")) {
1051                         migr_do_import();
1052                 }
1053                 else if (!strcasecmp(cmd, "listdirs")) {
1054                         migr_do_listdirs();
1055                 }
1056                 else if (!strcasecmp(cmd, "restoremeta")) {
1057                         migr_do_restore_meta();
1058                 }
1059                 else {
1060                         cprintf("%d illegal command\n", ERROR + ILLEGAL_VALUE);
1061                 }
1062
1063                 unlink(migr_tempfilename1);
1064                 unlink(migr_tempfilename2);
1065
1066                 CtdlEnableHouseKeeping();
1067                 CtdlEndSingleUser();
1068         }
1069         else {
1070                 cprintf("%d The migrator is already running.\n", ERROR + RESOURCE_BUSY);
1071         }
1072 }
1073
1074
1075 // Initialization function, called from modules_init.c
1076 char *ctdl_module_init_migrate() {
1077         if (!threading) {
1078                 CtdlRegisterProtoHook(cmd_migr, "MIGR", "Across-the-wire migration");
1079         }
1080         
1081         // return our module name for the log
1082         return "migrate";
1083 }