Removed the caching of netconfig entries. The caching of configdb entries now serves...
[citadel.git] / citadel / modules / migrate / serv_migrate.c
1 /*
2  * This module dumps and/or loads the Citadel database in XML format.
3  *
4  * Copyright (c) 1987-2016 by the citadel.org team
5  *
6  * This program is open source software; you can redistribute it and/or modify
7  * it under the terms of the GNU General Public License version 3.
8  *
9  * This program is distributed in the hope that it will be useful,
10  * but WITHOUT ANY WARRANTY; without even the implied warranty of
11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
12  * GNU General Public License for more details.
13  */
14
15 /*
16  * Explanation of <progress> tags:
17  *
18  * 0%              nothing
19  * 2%              finished exporting configuration
20  * 7%              finished exporting users
21  * 12%             finished exporting openids
22  * 17%             finished exporting rooms
23  * 18%             finished exporting floors
24  * 25%             finished exporting visits
25  * 100%            finished exporting messages
26  */
27
28 #include "sysdep.h"
29 #include <stdlib.h>
30 #include <unistd.h>
31 #include <stdio.h>
32 #include <fcntl.h>
33 #include <signal.h>
34 #include <pwd.h>
35 #include <errno.h>
36 #include <sys/types.h>
37
38 #if TIME_WITH_SYS_TIME
39 # include <sys/time.h>
40 # include <time.h>
41 #else
42 # if HAVE_SYS_TIME_H
43 #  include <sys/time.h>
44 # else
45 #  include <time.h>
46 # endif
47 #endif
48
49 #include <sys/wait.h>
50 #include <string.h>
51 #include <ctype.h>
52 #include <limits.h>
53 #include <expat.h>
54 #include <libcitadel.h>
55 #include "citadel.h"
56 #include "server.h"
57 #include "citserver.h"
58 #include "support.h"
59 #include "config.h"
60 #include "database.h"
61 #include "msgbase.h"
62 #include "user_ops.h"
63 #include "euidindex.h"
64 #include "ctdl_module.h"
65
66 #define END_OF_MESSAGE  "---eom---dbd---"
67
68 char migr_tempfilename1[PATH_MAX];
69 char migr_tempfilename2[PATH_MAX];
70 FILE *migr_global_message_list;
71 int total_msgs = 0;
72 char *ikey = NULL;                      // If we're importing a config key we store it here.
73
74
75 /******************************************************************************
76  *        Code which implements the export appears in this section            *
77  ******************************************************************************/
78
79 /*
80  * Output a string to the client with these characters escaped:  & < > " '
81  */
82 void xml_strout(char *str) {
83
84         char *c = str;
85
86         if (str == NULL) {
87                 return;
88         }
89
90         while (*c != 0) {
91                 if (*c == '\"') {
92                         client_write(HKEY("&quot;"));
93                 }
94                 else if (*c == '\'') {
95                         client_write(HKEY("&apos;"));
96                 }
97                 else if (*c == '<') {
98                         client_write(HKEY("&lt;"));
99                 }
100                 else if (*c == '>') {
101                         client_write(HKEY("&gt;"));
102                 }
103                 else if (*c == '&') {
104                         client_write(HKEY("&amp;"));
105                 }
106                 else {
107                         client_write(c, 1);
108                 }
109                 ++c;
110         }
111 }
112
113
114 /*
115  * Export a user record as XML
116  */
117 void migr_export_users_backend(struct ctdluser *buf, void *data) {
118         client_write(HKEY("<user>\n"));
119         cprintf("<u_version>%d</u_version>\n", buf->version);
120         cprintf("<u_uid>%ld</u_uid>\n", (long)buf->uid);
121         client_write(HKEY("<u_password>"));     xml_strout(buf->password);              client_write(HKEY("</u_password>\n"));
122         cprintf("<u_flags>%u</u_flags>\n", buf->flags);
123         cprintf("<u_timescalled>%ld</u_timescalled>\n", buf->timescalled);
124         cprintf("<u_posted>%ld</u_posted>\n", buf->posted);
125         cprintf("<u_axlevel>%d</u_axlevel>\n", buf->axlevel);
126         cprintf("<u_usernum>%ld</u_usernum>\n", buf->usernum);
127         cprintf("<u_lastcall>%ld</u_lastcall>\n", (long)buf->lastcall);
128         cprintf("<u_USuserpurge>%d</u_USuserpurge>\n", buf->USuserpurge);
129         client_write(HKEY("<u_fullname>"));     xml_strout(buf->fullname);              client_write(HKEY("</u_fullname>\n"));
130         client_write(HKEY("</user>\n"));
131 }
132
133
134 void migr_export_users(void) {
135         ForEachUser(migr_export_users_backend, NULL);
136 }
137
138
139 void migr_export_room_msg(long msgnum, void *userdata) {
140         cprintf("%ld\n", msgnum);
141         fprintf(migr_global_message_list, "%ld\n", msgnum);
142 }
143
144
145 void migr_export_rooms_backend(struct ctdlroom *buf, void *data) {
146         client_write(HKEY("<room>\n"));
147         client_write(HKEY("<QRname>")); xml_strout(buf->QRname);        client_write(HKEY("</QRname>\n"));
148         client_write(HKEY("<QRpasswd>"));       xml_strout(buf->QRpasswd);      client_write(HKEY("</QRpasswd>\n"));
149         cprintf("<QRroomaide>%ld</QRroomaide>\n", buf->QRroomaide);
150         cprintf("<QRhighest>%ld</QRhighest>\n", buf->QRhighest);
151         cprintf("<QRgen>%ld</QRgen>\n", (long)buf->QRgen);
152         cprintf("<QRflags>%u</QRflags>\n", buf->QRflags);
153         if (buf->QRflags & QR_DIRECTORY) {
154                 client_write(HKEY("<QRdirname>"));
155                 xml_strout(buf->QRdirname);
156                 client_write(HKEY("</QRdirname>\n"));
157         }
158         cprintf("<QRinfo>%ld</QRinfo>\n", buf->QRinfo);
159         cprintf("<QRfloor>%d</QRfloor>\n", buf->QRfloor);
160         cprintf("<QRmtime>%ld</QRmtime>\n", (long)buf->QRmtime);
161         cprintf("<QRexpire_mode>%d</QRexpire_mode>\n", buf->QRep.expire_mode);
162         cprintf("<QRexpire_value>%d</QRexpire_value>\n", buf->QRep.expire_value);
163         cprintf("<QRnumber>%ld</QRnumber>\n", buf->QRnumber);
164         cprintf("<QRorder>%d</QRorder>\n", buf->QRorder);
165         cprintf("<QRflags2>%u</QRflags2>\n", buf->QRflags2);
166         cprintf("<QRdefaultview>%d</QRdefaultview>\n", buf->QRdefaultview);
167         client_write(HKEY("</room>\n"));
168
169         /* message list goes inside this tag */
170
171         CtdlGetRoom(&CC->room, buf->QRname);
172         client_write(HKEY("<room_messages>"));
173         client_write(HKEY("<FRname>")); xml_strout(CC->room.QRname);    client_write(HKEY("</FRname>\n"));
174         client_write(HKEY("<FRmsglist>"));
175         CtdlForEachMessage(MSGS_ALL, 0L, NULL, NULL, NULL, migr_export_room_msg, NULL);
176         client_write(HKEY("</FRmsglist>\n"));
177         client_write(HKEY("</room_messages>\n"));
178
179
180 }
181
182
183 void migr_export_rooms(void) {
184         char cmd[SIZ];
185         migr_global_message_list = fopen(migr_tempfilename1, "w");
186         if (migr_global_message_list != NULL) {
187                 CtdlForEachRoom(migr_export_rooms_backend, NULL);
188                 fclose(migr_global_message_list);
189         }
190
191         /*
192          * Process the 'global' message list.  (Sort it and remove dups.
193          * Dups are ok because a message may be in more than one room, but
194          * this will be handled by exporting the reference count, not by
195          * exporting the message multiple times.)
196          */
197         snprintf(cmd, sizeof cmd, "sort -n <%s >%s", migr_tempfilename1, migr_tempfilename2);
198         if (system(cmd) != 0) syslog(LOG_ALERT, "Error %d", errno);
199         snprintf(cmd, sizeof cmd, "uniq <%s >%s", migr_tempfilename2, migr_tempfilename1);
200         if (system(cmd) != 0) syslog(LOG_ALERT, "Error %d", errno);
201
202
203         snprintf(cmd, sizeof cmd, "wc -l %s", migr_tempfilename1);
204         FILE *fp = popen(cmd, "r");
205         if (fp) {
206                 fgets(cmd, sizeof cmd, fp);
207                 pclose(fp);
208                 total_msgs = atoi(cmd);
209         }
210         else {
211                 total_msgs = 1; // any nonzero just to keep it from barfing
212         }
213         syslog(LOG_DEBUG, "Total messages to be exported: %d", total_msgs);
214 }
215
216
217 void migr_export_floors(void) {
218         struct floor qfbuf, *buf;
219         int i;
220
221         for (i=0; i < MAXFLOORS; ++i) {
222                 client_write(HKEY("<floor>\n"));
223                 cprintf("<f_num>%d</f_num>\n", i);
224                 CtdlGetFloor(&qfbuf, i);
225                 buf = &qfbuf;
226                 cprintf("<f_flags>%u</f_flags>\n", buf->f_flags);
227                 client_write(HKEY("<f_name>")); xml_strout(buf->f_name); client_write(HKEY("</f_name>\n"));
228                 cprintf("<f_ref_count>%d</f_ref_count>\n", buf->f_ref_count);
229                 cprintf("<f_ep_expire_mode>%d</f_ep_expire_mode>\n", buf->f_ep.expire_mode);
230                 cprintf("<f_ep_expire_value>%d</f_ep_expire_value>\n", buf->f_ep.expire_value);
231                 client_write(HKEY("</floor>\n"));
232         }
233 }
234
235
236 /*
237  * Return nonzero if the supplied string contains only characters which are valid in a sequence set.
238  */
239 int is_sequence_set(char *s) {
240         if (!s) return(0);
241
242         char *c = s;
243         char ch;
244         while (ch = *c++, ch) {
245                 if (!strchr("0123456789*,:", ch)) {
246                         return(0);
247                 }
248         }
249         return(1);
250 }
251
252
253
254 /* 
255  *  Traverse the visits file...
256  */
257 void migr_export_visits(void) {
258         visit vbuf;
259         struct cdbdata *cdbv;
260
261         cdb_rewind(CDB_VISIT);
262
263         while (cdbv = cdb_next_item(CDB_VISIT), cdbv != NULL) {
264                 memset(&vbuf, 0, sizeof(visit));
265                 memcpy(&vbuf, cdbv->ptr,
266                        ((cdbv->len > sizeof(visit)) ?
267                         sizeof(visit) : cdbv->len));
268                 cdb_free(cdbv);
269
270                 client_write(HKEY("<visit>\n"));
271                 cprintf("<v_roomnum>%ld</v_roomnum>\n", vbuf.v_roomnum);
272                 cprintf("<v_roomgen>%ld</v_roomgen>\n", vbuf.v_roomgen);
273                 cprintf("<v_usernum>%ld</v_usernum>\n", vbuf.v_usernum);
274
275                 client_write(HKEY("<v_seen>"));
276                 if ( (!IsEmptyStr(vbuf.v_seen)) && (is_sequence_set(vbuf.v_seen)) ) {
277                         xml_strout(vbuf.v_seen);
278                 }
279                 else {
280                         cprintf("%ld", vbuf.v_lastseen);
281                 }
282                 client_write(HKEY("</v_seen>"));
283
284                 if ( (!IsEmptyStr(vbuf.v_answered)) && (is_sequence_set(vbuf.v_answered)) ) {
285                         client_write(HKEY("<v_answered>"));
286                         xml_strout(vbuf.v_answered);
287                         client_write(HKEY("</v_answered>\n"));
288                 }
289
290                 cprintf("<v_flags>%u</v_flags>\n", vbuf.v_flags);
291                 cprintf("<v_view>%d</v_view>\n", vbuf.v_view);
292                 client_write(HKEY("</visit>\n"));
293         }
294 }
295
296
297 void migr_export_message(long msgnum) {
298         struct MetaData smi;
299         struct CtdlMessage *msg;
300         struct ser_ret smr;
301
302         /* We can use a static buffer here because there will never be more than
303          * one of this operation happening at any given time, and it's really best
304          * to just keep it allocated once instead of torturing malloc/free.
305          * Call this function with msgnum "-1" to free the buffer when finished.
306          */
307         static int encoded_alloc = 0;
308         static char *encoded_msg = NULL;
309
310         if (msgnum < 0) {
311                 if ((encoded_alloc == 0) && (encoded_msg != NULL)) {
312                         free(encoded_msg);
313                         encoded_alloc = 0;
314                         encoded_msg = NULL;
315                 }
316                 return;
317         }
318
319         /* Ok, here we go ... */
320
321         msg = CtdlFetchMessage(msgnum, 1, 0);
322         if (msg == NULL) return;        /* fail silently */
323
324         client_write(HKEY("<message>\n"));
325         GetMetaData(&smi, msgnum);
326         cprintf("<msg_msgnum>%ld</msg_msgnum>\n", msgnum);
327         cprintf("<msg_meta_refcount>%d</msg_meta_refcount>\n", smi.meta_refcount);
328         cprintf("<msg_meta_rfc822_length>%ld</msg_meta_rfc822_length>\n", smi.meta_rfc822_length);
329         client_write(HKEY("<msg_meta_content_type>")); xml_strout(smi.meta_content_type); client_write(HKEY("</msg_meta_content_type>\n"));
330         client_write(HKEY("<msg_mimetype>")); xml_strout(smi.mimetype); client_write(HKEY("</msg_mimetype>\n"));
331
332         client_write(HKEY("<msg_text>"));
333         CtdlSerializeMessage(&smr, msg);
334         CM_Free(msg);
335
336         /* Predict the buffer size we need.  Expand the buffer if necessary. */
337         int encoded_len = smr.len * 15 / 10 ;
338         if (encoded_len > encoded_alloc) {
339                 encoded_alloc = encoded_len;
340                 encoded_msg = realloc(encoded_msg, encoded_alloc);
341         }
342
343         if (encoded_msg == NULL) {
344                 /* Questionable hack that hopes it'll work next time and we only lose one message */
345                 encoded_alloc = 0;
346         }
347         else {
348                 /* Once we do the encoding we know the exact size */
349                 encoded_len = CtdlEncodeBase64(encoded_msg, (char *)smr.ser, smr.len, 1);
350                 client_write(encoded_msg, encoded_len);
351         }
352
353         free(smr.ser);
354
355         client_write(HKEY("</msg_text>\n"));
356         client_write(HKEY("</message>\n"));
357 }
358
359
360
361 void migr_export_openids(void) {
362         struct cdbdata *cdboi;
363         long usernum;
364         char url[512];
365
366         cdb_rewind(CDB_OPENID);
367         while (cdboi = cdb_next_item(CDB_OPENID), cdboi != NULL) {
368                 if (cdboi->len > sizeof(long)) {
369                         client_write(HKEY("<openid>\n"));
370                         memcpy(&usernum, cdboi->ptr, sizeof(long));
371                         snprintf(url, sizeof url, "%s", (cdboi->ptr)+sizeof(long) );
372                         client_write(HKEY("<oid_url>"));
373                         xml_strout(url);
374                         client_write(HKEY("</oid_url>\n"));
375                         cprintf("<oid_usernum>%ld</oid_usernum>\n", usernum);
376                         client_write(HKEY("</openid>\n"));
377                 }
378                 cdb_free(cdboi);
379         }
380 }
381
382
383 void migr_export_configs(void) {
384         struct cdbdata *cdbcfg;
385         int keylen = 0;
386         char *key = NULL;
387         char *value = NULL;
388
389         cdb_rewind(CDB_CONFIG);
390         while (cdbcfg = cdb_next_item(CDB_CONFIG), cdbcfg != NULL) {
391
392                 keylen = strlen(cdbcfg->ptr);
393                 key = cdbcfg->ptr;
394                 value = cdbcfg->ptr + keylen + 1;
395
396                 client_write("<config key=\"", 13);
397                 xml_strout(key);
398                 client_write("\">", 2);
399                 xml_strout(value);
400                 client_write("</config>\n", 10);
401                 cdb_free(cdbcfg);
402         }
403 }
404
405
406
407
408 void migr_export_messages(void) {
409         char buf[SIZ];
410         long msgnum;
411         int count = 0;
412         int progress = 0;
413         int prev_progress = 0;
414         CitContext *Ctx;
415
416         Ctx = CC;
417         migr_global_message_list = fopen(migr_tempfilename1, "r");
418         if (migr_global_message_list != NULL) {
419                 syslog(LOG_INFO, "Opened %s", migr_tempfilename1);
420                 while ((Ctx->kill_me == 0) && 
421                        (fgets(buf, sizeof(buf), migr_global_message_list) != NULL)) {
422                         msgnum = atol(buf);
423                         if (msgnum > 0L) {
424                                 migr_export_message(msgnum);
425                                 ++count;
426                         }
427                         progress = (count * 74 / total_msgs) + 25 ;
428                         if ((progress > prev_progress) && (progress < 100)) {
429                                 cprintf("<progress>%d</progress>\n", progress);
430                         }
431                         prev_progress = progress;
432                 }
433                 fclose(migr_global_message_list);
434         }
435         if (Ctx->kill_me == 0)
436                 syslog(LOG_INFO, "Exported %d messages.", count);
437         else
438                 syslog(LOG_ERR, "Export aborted due to client disconnect!");
439
440         migr_export_message(-1L);       /* This frees the encoding buffer */
441 }
442
443
444
445 void migr_do_export(void) {
446         CitContext *Ctx;
447
448         Ctx = CC;
449         cprintf("%d Exporting all Citadel databases.\n", LISTING_FOLLOWS);
450         Ctx->dont_term = 1;
451
452         client_write(HKEY("<?xml version=\"1.0\" encoding=\"UTF-8\" ?>\n"));
453         client_write(HKEY("<citadel_migrate_data>\n"));
454         cprintf("<version>%d</version>\n", REV_LEVEL);
455         cprintf("<progress>%d</progress>\n", 0);
456
457         /* export the configuration database */
458         migr_export_configs();
459         cprintf("<progress>%d</progress>\n", 2);
460         
461         if (Ctx->kill_me == 0)  migr_export_users();
462         cprintf("<progress>%d</progress>\n", 7);
463         if (Ctx->kill_me == 0)  migr_export_openids();
464         cprintf("<progress>%d</progress>\n", 12);
465         if (Ctx->kill_me == 0)  migr_export_rooms();
466         cprintf("<progress>%d</progress>\n", 17);
467         if (Ctx->kill_me == 0)  migr_export_floors();
468         cprintf("<progress>%d</progress>\n", 18);
469         if (Ctx->kill_me == 0)  migr_export_visits();
470         cprintf("<progress>%d</progress>\n", 25);
471         if (Ctx->kill_me == 0)  migr_export_messages();
472         client_write(HKEY("</citadel_migrate_data>\n"));
473         cprintf("<progress>%d</progress>\n", 100);
474         client_write(HKEY("000\n"));
475         Ctx->dont_term = 0;
476 }
477
478
479
480
481 /******************************************************************************
482  *                              Import code                                   *
483  *    Here's the code that implements the import side.  It's going to end up  *
484  *        being one big loop with lots of global variables.  I don't care.    *
485  * You wouldn't run multiple concurrent imports anyway.  If this offends your *
486  * delicate sensibilities  then go rewrite it in Ruby on Rails or something.  *
487  ******************************************************************************/
488
489
490 int citadel_migrate_data = 0;           /* Are we inside a <citadel_migrate_data> tag pair? */
491 StrBuf *migr_chardata = NULL;
492 StrBuf *migr_MsgData = NULL;
493 struct ctdluser usbuf;
494 struct ctdlroom qrbuf;
495 char openid_url[512];
496 long openid_usernum = 0;
497 char FRname[ROOMNAMELEN];
498 struct floor flbuf;
499 int floornum = 0;
500 visit vbuf;
501 struct MetaData smi;
502 long import_msgnum = 0;
503
504 /*
505  * This callback stores up the data which appears in between tags.
506  */
507 void migr_xml_chardata(void *data, const XML_Char *s, int len)
508 {
509         StrBufAppendBufPlain(migr_chardata, s, len, 0);
510 }
511
512
513 void migr_xml_start(void *data, const char *el, const char **attr) {
514         int i;
515
516         /*** GENERAL STUFF ***/
517
518         /* Reset chardata_len to zero and init buffer */
519         FlushStrBuf(migr_chardata);
520         FlushStrBuf(migr_MsgData);
521
522         if (!strcasecmp(el, "citadel_migrate_data")) {
523                 ++citadel_migrate_data;
524
525                 /* As soon as it looks like the input data is a genuine Citadel XML export,
526                  * whack the existing database on disk to make room for the new one.
527                  */
528                 if (citadel_migrate_data == 1) {
529                         for (i = 0; i < MAXCDB; ++i) {
530                                 cdb_trunc(i);
531                         }
532                 }
533                 return;
534         }
535
536         if (citadel_migrate_data != 1) {
537                 syslog(LOG_ALERT, "Out-of-sequence tag <%s> detected.  Warning: ODD-DATA!", el);
538                 return;
539         }
540
541         /* When we begin receiving XML for one of these record types, clear out the associated
542          * buffer so we don't accidentally carry over any data from a previous record.
543          */
544         if (!strcasecmp(el, "user"))                    memset(&usbuf, 0, sizeof (struct ctdluser));
545         else if (!strcasecmp(el, "openid"))             memset(openid_url, 0, sizeof openid_url);
546         else if (!strcasecmp(el, "room"))               memset(&qrbuf, 0, sizeof (struct ctdlroom));
547         else if (!strcasecmp(el, "room_messages"))      memset(FRname, 0, sizeof FRname);
548         else if (!strcasecmp(el, "floor"))              memset(&flbuf, 0, sizeof (struct floor));
549         else if (!strcasecmp(el, "visit"))              memset(&vbuf, 0, sizeof (visit));
550
551         else if (!strcasecmp(el, "message")) {
552                 memset(&smi, 0, sizeof (struct MetaData));
553                 import_msgnum = 0;
554         }
555         else if (!strcasecmp(el, "config")) {
556                 if (ikey != NULL) {
557                         free(ikey);
558                         ikey = NULL;
559                 }
560                 while (*attr) {
561                         if (!strcasecmp(attr[0], "key")) {
562                                 ikey = strdup(attr[1]);
563                         }
564                         attr += 2;
565                 }
566         }
567
568 }
569
570
571 int migr_userrecord(void *data, const char *el)
572 {
573         if (!strcasecmp(el, "u_version"))                       usbuf.version = atoi(ChrPtr(migr_chardata));
574         else if (!strcasecmp(el, "u_uid"))                      usbuf.uid = atol(ChrPtr(migr_chardata));
575         else if (!strcasecmp(el, "u_password"))                 safestrncpy(usbuf.password, ChrPtr(migr_chardata), sizeof usbuf.password);
576         else if (!strcasecmp(el, "u_flags"))                    usbuf.flags = atoi(ChrPtr(migr_chardata));
577         else if (!strcasecmp(el, "u_timescalled"))              usbuf.timescalled = atol(ChrPtr(migr_chardata));
578         else if (!strcasecmp(el, "u_posted"))                   usbuf.posted = atol(ChrPtr(migr_chardata));
579         else if (!strcasecmp(el, "u_axlevel"))                  usbuf.axlevel = atoi(ChrPtr(migr_chardata));
580         else if (!strcasecmp(el, "u_usernum"))                  usbuf.usernum = atol(ChrPtr(migr_chardata));
581         else if (!strcasecmp(el, "u_lastcall"))                 usbuf.lastcall = atol(ChrPtr(migr_chardata));
582         else if (!strcasecmp(el, "u_USuserpurge"))              usbuf.USuserpurge = atoi(ChrPtr(migr_chardata));
583         else if (!strcasecmp(el, "u_fullname"))                 safestrncpy(usbuf.fullname, ChrPtr(migr_chardata), sizeof usbuf.fullname);
584         else return 0;
585         return 1;
586 }
587
588 int migr_roomrecord(void *data, const char *el)
589 {
590         if (!strcasecmp(el, "QRname"))                  safestrncpy(qrbuf.QRname, ChrPtr(migr_chardata), sizeof qrbuf.QRname);
591         else if (!strcasecmp(el, "QRpasswd"))                   safestrncpy(qrbuf.QRpasswd, ChrPtr(migr_chardata), sizeof qrbuf.QRpasswd);
592         else if (!strcasecmp(el, "QRroomaide"))                 qrbuf.QRroomaide = atol(ChrPtr(migr_chardata));
593         else if (!strcasecmp(el, "QRhighest"))                  qrbuf.QRhighest = atol(ChrPtr(migr_chardata));
594         else if (!strcasecmp(el, "QRgen"))                      qrbuf.QRgen = atol(ChrPtr(migr_chardata));
595         else if (!strcasecmp(el, "QRflags"))                    qrbuf.QRflags = atoi(ChrPtr(migr_chardata));
596         else if (!strcasecmp(el, "QRdirname"))                  safestrncpy(qrbuf.QRdirname, ChrPtr(migr_chardata), sizeof qrbuf.QRdirname);
597         else if (!strcasecmp(el, "QRinfo"))                     qrbuf.QRinfo = atol(ChrPtr(migr_chardata));
598         else if (!strcasecmp(el, "QRfloor"))                    qrbuf.QRfloor = atoi(ChrPtr(migr_chardata));
599         else if (!strcasecmp(el, "QRmtime"))                    qrbuf.QRmtime = atol(ChrPtr(migr_chardata));
600         else if (!strcasecmp(el, "QRexpire_mode"))              qrbuf.QRep.expire_mode = atoi(ChrPtr(migr_chardata));
601         else if (!strcasecmp(el, "QRexpire_value"))             qrbuf.QRep.expire_value = atoi(ChrPtr(migr_chardata));
602         else if (!strcasecmp(el, "QRnumber"))                   qrbuf.QRnumber = atol(ChrPtr(migr_chardata));
603         else if (!strcasecmp(el, "QRorder"))                    qrbuf.QRorder = atoi(ChrPtr(migr_chardata));
604         else if (!strcasecmp(el, "QRflags2"))                   qrbuf.QRflags2 = atoi(ChrPtr(migr_chardata));
605         else if (!strcasecmp(el, "QRdefaultview"))              qrbuf.QRdefaultview = atoi(ChrPtr(migr_chardata));
606         else return 0;
607         return 1;
608 }
609
610 int migr_floorrecord(void *data, const char *el)
611 {
612         if (!strcasecmp(el, "f_num"))                   floornum = atoi(ChrPtr(migr_chardata));
613         else if (!strcasecmp(el, "f_flags"))                    flbuf.f_flags = atoi(ChrPtr(migr_chardata));
614         else if (!strcasecmp(el, "f_name"))                     safestrncpy(flbuf.f_name, ChrPtr(migr_chardata), sizeof flbuf.f_name);
615         else if (!strcasecmp(el, "f_ref_count"))                flbuf.f_ref_count = atoi(ChrPtr(migr_chardata));
616         else if (!strcasecmp(el, "f_ep_expire_mode"))           flbuf.f_ep.expire_mode = atoi(ChrPtr(migr_chardata));
617         else if (!strcasecmp(el, "f_ep_expire_value"))          flbuf.f_ep.expire_value = atoi(ChrPtr(migr_chardata));
618         else return 0;
619         return 1;
620 }
621
622 int migr_visitrecord(void *data, const char *el)
623 {
624         if (!strcasecmp(el, "v_roomnum"))                       vbuf.v_roomnum = atol(ChrPtr(migr_chardata));
625         else if (!strcasecmp(el, "v_roomgen"))                  vbuf.v_roomgen = atol(ChrPtr(migr_chardata));
626         else if (!strcasecmp(el, "v_usernum"))                  vbuf.v_usernum = atol(ChrPtr(migr_chardata));
627
628         else if (!strcasecmp(el, "v_seen")) {
629                 int is_textual_seen = 0;
630                 int i;
631                 int max = StrLength(migr_chardata);
632
633                 vbuf.v_lastseen = atol(ChrPtr(migr_chardata));
634                 is_textual_seen = 0;
635                 for (i=0; i < max; ++i) 
636                         if (!isdigit(ChrPtr(migr_chardata)[i]))
637                                 is_textual_seen = 1;
638                 if (is_textual_seen)
639                         safestrncpy(vbuf.v_seen, ChrPtr(migr_chardata), sizeof vbuf.v_seen);
640         }
641
642         else if (!strcasecmp(el, "v_answered"))                 safestrncpy(vbuf.v_answered, ChrPtr(migr_chardata), sizeof vbuf.v_answered);
643         else if (!strcasecmp(el, "v_flags"))                    vbuf.v_flags = atoi(ChrPtr(migr_chardata));
644         else if (!strcasecmp(el, "v_view"))                     vbuf.v_view = atoi(ChrPtr(migr_chardata));
645         else return 0;
646         return 1;
647 }
648
649
650 void migr_xml_end(void *data, const char *el)
651 {
652         const char *ptr;
653         int msgcount = 0;
654         long msgnum = 0L;
655         long *msglist = NULL;
656         int msglist_alloc = 0;
657         /*** GENERAL STUFF ***/
658
659         if (!strcasecmp(el, "citadel_migrate_data")) {
660                 --citadel_migrate_data;
661                 return;
662         }
663
664         if (citadel_migrate_data != 1) {
665                 syslog(LOG_ALERT, "Out-of-sequence tag <%s> detected.  Warning: ODD-DATA!", el);
666                 return;
667         }
668
669         // syslog(LOG_DEBUG, "END TAG: <%s> DATA: <%s>\n", el, (migr_chardata_len ? migr_chardata : ""));
670
671         /*** CONFIG ***/
672
673         if (!strcasecmp(el, "config"))
674         {
675                 syslog(LOG_DEBUG, "Imported config key=%s", ikey);
676
677                 if (ikey != NULL) {
678                         CtdlSetConfigStr(ikey, ChrPtr(migr_chardata));
679                         free(ikey);
680                         ikey = NULL;
681                 }
682                 else {
683                         syslog(LOG_INFO, "Closed a <config> tag but no key name was supplied.");
684                 }
685         }
686
687         /*** USER ***/
688         else if ((!strncasecmp(el, HKEY("u_"))) && 
689                  migr_userrecord(data, el))
690                 ; /* Nothing to do anymore */
691         else if (!strcasecmp(el, "user")) {
692                 CtdlPutUser(&usbuf);
693                 syslog(LOG_INFO, "Imported user: %s", usbuf.fullname);
694         }
695
696         /*** OPENID ***/
697
698         else if (!strcasecmp(el, "oid_url"))                    safestrncpy(openid_url, ChrPtr(migr_chardata), sizeof openid_url);
699         else if (!strcasecmp(el, "oid_usernum"))                openid_usernum = atol(ChrPtr(migr_chardata));
700
701         else if (!strcasecmp(el, "openid")) {                   /* see serv_openid_rp.c for a description of the record format */
702                 char *oid_data;
703                 int oid_data_len;
704                 oid_data_len = sizeof(long) + strlen(openid_url) + 1;
705                 oid_data = malloc(oid_data_len);
706                 memcpy(oid_data, &openid_usernum, sizeof(long));
707                 memcpy(&oid_data[sizeof(long)], openid_url, strlen(openid_url) + 1);
708                 cdb_store(CDB_OPENID, openid_url, strlen(openid_url), oid_data, oid_data_len);
709                 free(oid_data);
710                 syslog(LOG_INFO, "Imported OpenID: %s (%ld)", openid_url, openid_usernum);
711         }
712
713         /*** ROOM ***/
714         else if ((!strncasecmp(el, HKEY("QR"))) && 
715                  migr_roomrecord(data, el))
716                 ; /* Nothing to do anymore */
717         else if (!strcasecmp(el, "room")) {
718                 CtdlPutRoom(&qrbuf);
719                 syslog(LOG_INFO, "Imported room: %s", qrbuf.QRname);
720         }
721
722         /*** ROOM MESSAGE POINTERS ***/
723
724         else if (!strcasecmp(el, "FRname"))                     safestrncpy(FRname, ChrPtr(migr_chardata), sizeof FRname);
725
726         else if (!strcasecmp(el, "FRmsglist")) {
727                 if (!IsEmptyStr(FRname)) {
728                         msgcount = 0;
729                         msglist_alloc = 1000;
730                         msglist = malloc(sizeof(long) * msglist_alloc);
731
732                         syslog(LOG_DEBUG, "Message list for: %s", FRname);
733
734                         ptr = ChrPtr(migr_chardata);
735                         while (*ptr != 0) {
736                                 while ((*ptr != 0) && (!isdigit(*ptr))) {
737                                         ++ptr;
738                                 }
739                                 if ((*ptr != 0) && (isdigit(*ptr))) {
740                                         msgnum = atol(ptr);
741                                         if (msgnum > 0L) {
742                                                 if (msgcount >= msglist_alloc) {
743                                                         msglist_alloc *= 2;
744                                                         msglist = realloc(msglist, sizeof(long) * msglist_alloc);
745                                                 }
746                                                 msglist[msgcount++] = msgnum;
747                                                 }
748                                         }
749                                         while ((*ptr != 0) && (isdigit(*ptr))) {
750                                                 ++ptr;
751                                         }
752                                 }
753                         }
754                         if (msgcount > 0) {
755                                 CtdlSaveMsgPointersInRoom(FRname, msglist, msgcount, 0, NULL, 1);
756                         }
757                         free(msglist);
758                         msglist = NULL;
759                         msglist_alloc = 0;
760                         syslog(LOG_DEBUG, "Imported %d messages.", msgcount);
761                         if (server_shutting_down) {
762                                 return;
763                 }
764         }
765
766         /*** FLOORS ***/
767         else if ((!strncasecmp(el, HKEY("f_"))) && 
768                  migr_floorrecord(data, el))
769                 ; /* Nothing to do anymore */
770
771         else if (!strcasecmp(el, "floor")) {
772                 CtdlPutFloor(&flbuf, floornum);
773                 syslog(LOG_INFO, "Imported floor #%d (%s)", floornum, flbuf.f_name);
774         }
775
776         /*** VISITS ***/
777         else if ((!strncasecmp(el, HKEY("v_"))) && 
778                  migr_visitrecord(data, el))
779                 ; /* Nothing to do anymore */
780         else if (!strcasecmp(el, "visit")) {
781                 put_visit(&vbuf);
782                 syslog(LOG_INFO, "Imported visit: %ld/%ld/%ld", vbuf.v_roomnum, vbuf.v_roomgen, vbuf.v_usernum);
783         }
784
785         /*** MESSAGES ***/
786         
787         else if (!strcasecmp(el, "msg_msgnum"))                 smi.meta_msgnum = import_msgnum = atol(ChrPtr(migr_chardata));
788         else if (!strcasecmp(el, "msg_meta_refcount"))          smi.meta_refcount = atoi(ChrPtr(migr_chardata));
789         else if (!strcasecmp(el, "msg_meta_rfc822_length"))     smi.meta_rfc822_length = atoi(ChrPtr(migr_chardata));
790         else if (!strcasecmp(el, "msg_meta_content_type"))      safestrncpy(smi.meta_content_type, ChrPtr(migr_chardata), sizeof smi.meta_content_type);
791         else if (!strcasecmp(el, "msg_mimetype"))               safestrncpy(smi.mimetype, ChrPtr(migr_chardata), sizeof smi.mimetype);
792
793         else if (!strcasecmp(el, "msg_text"))
794         {
795                 long rc;
796                 struct CtdlMessage *msg;
797
798                 FlushStrBuf(migr_MsgData);
799                 StrBufDecodeBase64To(migr_chardata, migr_MsgData);
800
801                 msg = CtdlDeserializeMessage(import_msgnum, -1,
802                                              ChrPtr(migr_MsgData), 
803                                              StrLength(migr_MsgData));
804                 if (msg != NULL) {
805                         rc = CtdlSaveThisMessage(msg, import_msgnum, 0);
806                         if (rc == 0) {
807                                 PutMetaData(&smi);
808                         }
809                         CM_Free(msg);
810                 }
811                 else {
812                         rc = -1;
813                 }
814
815                 syslog(LOG_INFO,
816                        "%s message #%ld, size=%d, refcount=%d, bodylength=%ld, content-type: %s / %s",
817                        (rc!= 0)?"failed to import ":"Imported ",
818                        import_msgnum,
819                        StrLength(migr_MsgData),
820                        smi.meta_refcount,
821                        smi.meta_rfc822_length,
822                        smi.meta_content_type,
823                        smi.mimetype);
824                 memset(&smi, 0, sizeof(smi));
825         }
826
827         /*** MORE GENERAL STUFF ***/
828
829         FlushStrBuf(migr_chardata);
830 }
831
832
833
834
835 /*
836  * Import begins here
837  */
838 void migr_do_import(void) {
839         StrBuf *Buf;
840         XML_Parser xp;
841         int Finished = 0;
842         
843         unbuffer_output();
844         migr_chardata = NewStrBufPlain(NULL, SIZ * 20);
845         migr_MsgData = NewStrBufPlain(NULL, SIZ * 20);
846         Buf = NewStrBufPlain(NULL, SIZ);
847         xp = XML_ParserCreate(NULL);
848         if (!xp) {
849                 cprintf("%d Failed to create XML parser instance\n", ERROR+INTERNAL_ERROR);
850                 return;
851         }
852         XML_SetElementHandler(xp, migr_xml_start, migr_xml_end);
853         XML_SetCharacterDataHandler(xp, migr_xml_chardata);
854
855         CC->dont_term = 1;
856
857         cprintf("%d sock it to me\n", SEND_LISTING);
858         unbuffer_output();
859
860         client_set_inbound_buf(SIZ * 10);
861
862         while (!Finished && client_read_random_blob(Buf, -1) >= 0) {
863                 if ((StrLength(Buf) > 4) &&
864                     !strcmp(ChrPtr(Buf) + StrLength(Buf) - 4, "000\n"))
865                 {
866                         Finished = 1;
867                         StrBufCutAt(Buf, StrLength(Buf) - 4, NULL);
868                 }
869                 if (server_shutting_down)
870                         break;  // Should we break or return?
871                 
872                 if (StrLength(Buf) == 0)
873                         continue;
874
875                 XML_Parse(xp, ChrPtr(Buf), StrLength(Buf), 0);
876                 FlushStrBuf(Buf);
877         }
878
879         XML_Parse(xp, "", 0, 1);
880         XML_ParserFree(xp);
881         FreeStrBuf(&Buf);
882         FreeStrBuf(&migr_chardata);
883         FreeStrBuf(&migr_MsgData);
884         rebuild_euid_index();
885         rebuild_usersbynumber();
886         CtdlSetConfigInt("MM_fulltext_wordbreaker", -1);        // Set an invalid wordbreaker to force re-indexing
887         CC->dont_term = 0;
888 }
889
890
891
892 /******************************************************************************
893  *                         Dispatcher, Common code                            *
894  ******************************************************************************/
895 /*
896  * Dump out the pathnames of directories which can be copied "as is"
897  */
898 void migr_do_listdirs(void) {
899         cprintf("%d Don't forget these:\n", LISTING_FOLLOWS);
900         cprintf("bio|%s\n",             ctdl_bio_dir);
901         cprintf("files|%s\n",           ctdl_file_dir);
902         cprintf("userpics|%s\n",        ctdl_usrpic_dir);
903         cprintf("messages|%s\n",        ctdl_message_dir);
904         cprintf("keys|%s\n",            ctdl_key_dir);
905         cprintf("images|%s\n",          ctdl_image_dir);
906         cprintf("info|%s\n",            ctdl_info_dir);
907         cprintf("000\n");
908 }
909
910 /******************************************************************************
911  *                    Repair database integrity                               *
912  ******************************************************************************/
913
914 StrBuf *PlainMessageBuf = NULL;
915 HashList *UsedMessageIDS = NULL;
916
917 int migr_restore_message_metadata(long msgnum, int refcount)
918 {
919         CitContext *CCC = MyContext();
920         struct MetaData smi;
921         struct CtdlMessage *msg;
922         char *mptr = NULL;
923
924         /* We can use a static buffer here because there will never be more than
925          * one of this operation happening at any given time, and it's really best
926          * to just keep it allocated once instead of torturing malloc/free.
927          * Call this function with msgnum "-1" to free the buffer when finished.
928          */
929         static int encoded_alloc = 0;
930         static char *encoded_msg = NULL;
931
932         if (msgnum < 0) {
933                 if ((encoded_alloc == 0) && (encoded_msg != NULL)) {
934                         free(encoded_msg);
935                         encoded_alloc = 0;
936                         encoded_msg = NULL;
937                         // todo FreeStrBuf(&PlainMessageBuf); PlainMessageBuf = NULL;
938                 }
939                 return 0;
940         }
941
942         if (PlainMessageBuf == NULL) {
943                 PlainMessageBuf = NewStrBufPlain(NULL, 10*SIZ);
944         }
945
946         /* Ok, here we go ... */
947
948         msg = CtdlFetchMessage(msgnum, 1, 0);
949         if (msg == NULL) {
950                 return 1;
951         }
952
953         GetMetaData(&smi, msgnum);
954         smi.meta_msgnum = msgnum;
955         smi.meta_refcount = refcount;
956         
957         /* restore the content type from the message body: */
958         mptr = bmstrcasestr(msg->cm_fields[eMesageText], "Content-type:");
959         if (mptr != NULL) {
960                 char *aptr;
961                 safestrncpy(smi.meta_content_type, &mptr[13], sizeof smi.meta_content_type);
962                 striplt(smi.meta_content_type);
963                 aptr = smi.meta_content_type;
964                 while (!IsEmptyStr(aptr)) {
965                         if ((*aptr == ';')
966                             || (*aptr == ' ')
967                             || (*aptr == 13)
968                             || (*aptr == 10)) {
969                                 memset(aptr, 0, sizeof(smi.meta_content_type) - (aptr - smi.meta_content_type));
970                         }
971                         else aptr++;
972                 }
973         }
974
975         CCC->redirect_buffer = PlainMessageBuf;
976         CtdlOutputPreLoadedMsg(msg, MT_RFC822, HEADERS_ALL, 0, 1, QP_EADDR);
977         smi.meta_rfc822_length = StrLength(CCC->redirect_buffer);
978         CCC->redirect_buffer = NULL;
979
980
981         syslog(LOG_INFO,
982                "Setting message #%ld meta data to: refcount=%d, bodylength=%ld, content-type: %s / %s",
983                smi.meta_msgnum,
984                smi.meta_refcount,
985                smi.meta_rfc822_length,
986                smi.meta_content_type,
987                smi.mimetype);
988
989         PutMetaData(&smi);
990
991         CM_Free(msg);
992
993         return 0;
994 }
995
996 void migr_check_room_msg(long msgnum, void *userdata) {
997         fprintf(migr_global_message_list, "%ld %s\n", msgnum, CC->room.QRname);
998 }
999
1000
1001 void migr_check_rooms_backend(struct ctdlroom *buf, void *data) {
1002
1003         /* message list goes inside this tag */
1004
1005         CtdlGetRoom(&CC->room, buf->QRname);
1006         CtdlForEachMessage(MSGS_ALL, 0L, NULL, NULL, NULL, migr_check_room_msg, NULL);
1007 }
1008
1009 void RemoveMessagesFromRooms(StrBuf *RoomNameVec, long msgnum) {
1010         struct MetaData smi;
1011         const char *Pos = NULL;
1012         StrBuf *oneRoom = NewStrBuf();
1013
1014         syslog(LOG_INFO, "removing message pointer %ld from these rooms: %s", msgnum, ChrPtr(RoomNameVec));
1015
1016         while (Pos != StrBufNOTNULL){
1017                 StrBufExtract_NextToken(oneRoom, RoomNameVec, &Pos, '|');
1018                 CtdlDeleteMessages(ChrPtr(oneRoom), &msgnum, 1, "");
1019         };
1020         GetMetaData(&smi, msgnum);
1021         TDAP_AdjRefCount(msgnum, -smi.meta_refcount);
1022 }
1023
1024 void migr_do_restore_meta(void) {
1025         char buf[SIZ];
1026         int failGetMessage;
1027         long msgnum = 0;
1028         int lastnum = 0;
1029         int refcount = 0;
1030         CitContext *Ctx;
1031         char *prn;
1032         StrBuf *RoomNames;
1033         char cmd[SIZ];
1034
1035         migr_global_message_list = fopen(migr_tempfilename1, "w");
1036         if (migr_global_message_list != NULL) {
1037                 CtdlForEachRoom(migr_check_rooms_backend, NULL);
1038                 fclose(migr_global_message_list);
1039         }
1040
1041         /*
1042          * Process the 'global' message list.  (Sort it and remove dups.
1043          * Dups are ok because a message may be in more than one room, but
1044          * this will be handled by exporting the reference count, not by
1045          * exporting the message multiple times.)
1046          */
1047         snprintf(cmd, sizeof cmd, "sort -n <%s >%s", migr_tempfilename1, migr_tempfilename2);
1048         if (system(cmd) != 0) syslog(LOG_ALERT, "Error %d", errno);
1049
1050         RoomNames = NewStrBuf();
1051         Ctx = CC;
1052         migr_global_message_list = fopen(migr_tempfilename2, "r");
1053         if (migr_global_message_list != NULL) {
1054                 syslog(LOG_INFO, "Opened %s", migr_tempfilename1);
1055                 while ((Ctx->kill_me == 0) && 
1056                        (fgets(buf, sizeof(buf), migr_global_message_list) != NULL)) {
1057                         msgnum = atol(buf);
1058                         if (msgnum == 0L) 
1059                                 continue;
1060                         if (lastnum == 0) {
1061                                 lastnum = msgnum;
1062                         }
1063                         prn = strchr(buf, ' ');
1064                         if (lastnum != msgnum) {
1065                                 failGetMessage = migr_restore_message_metadata(lastnum, refcount);
1066                                 if (failGetMessage) {
1067                                         RemoveMessagesFromRooms(RoomNames, lastnum);
1068                                 }
1069                                 refcount = 1;
1070                                 lastnum = msgnum;
1071                                 if (prn != NULL)
1072                                         StrBufPlain(RoomNames, prn + 1, -1);
1073                                 StrBufTrim(RoomNames);
1074                         }
1075                         else {
1076                                 if (prn != NULL) {
1077                                         if (StrLength(RoomNames) > 0)
1078                                                 StrBufAppendBufPlain(RoomNames, HKEY("|"), 0);
1079                                         StrBufAppendBufPlain(RoomNames, prn, -1, 1);
1080                                         StrBufTrim(RoomNames);
1081                                 }
1082                                 refcount ++;
1083                         }
1084                         lastnum = msgnum;
1085                 }
1086                 failGetMessage = migr_restore_message_metadata(msgnum, refcount);
1087                 if (failGetMessage) {
1088                         RemoveMessagesFromRooms(RoomNames, lastnum);
1089                 }
1090                 fclose(migr_global_message_list);
1091         }
1092
1093         migr_restore_message_metadata(-1L, -1); /* This frees the encoding buffer */
1094         cprintf("%d system analysis completed", CIT_OK);
1095         Ctx->kill_me = 1;
1096 }
1097
1098
1099
1100
1101 /******************************************************************************
1102  *                         Dispatcher, Common code                            *
1103  ******************************************************************************/
1104 void cmd_migr(char *cmdbuf) {
1105         char cmd[32];
1106         
1107         if (CtdlAccessCheck(ac_internal)) return;
1108         
1109         if (CtdlTrySingleUser())
1110         {
1111                 CtdlDisableHouseKeeping();
1112                 CtdlMakeTempFileName(migr_tempfilename1, sizeof migr_tempfilename1);
1113                 CtdlMakeTempFileName(migr_tempfilename2, sizeof migr_tempfilename2);
1114
1115                 extract_token(cmd, cmdbuf, 0, '|', sizeof cmd);
1116                 if (!strcasecmp(cmd, "export")) {
1117                         migr_do_export();
1118                 }
1119                 else if (!strcasecmp(cmd, "import")) {
1120                         migr_do_import();
1121                 }
1122                 else if (!strcasecmp(cmd, "listdirs")) {
1123                         migr_do_listdirs();
1124                 }
1125                 else if (!strcasecmp(cmd, "restoremeta")) {
1126                         migr_do_restore_meta();
1127                 }
1128                 else {
1129                         cprintf("%d illegal command\n", ERROR + ILLEGAL_VALUE);
1130                 }
1131
1132                 unlink(migr_tempfilename1);
1133                 unlink(migr_tempfilename2);
1134
1135                 CtdlEnableHouseKeeping();
1136                 CtdlEndSingleUser();
1137         }
1138         else
1139         {
1140                 cprintf("%d The migrator is already running.\n", ERROR + RESOURCE_BUSY);
1141         }
1142 }
1143
1144 /******************************************************************************
1145  *                              Module Hook                                  *
1146  ******************************************************************************/
1147
1148 CTDL_MODULE_INIT(migrate)
1149 {
1150         if (!threading)
1151         {
1152                 CtdlRegisterProtoHook(cmd_migr, "MIGR", "Across-the-wire migration");
1153         }
1154         
1155         /* return our module name for the log */
1156         return "migrate";
1157 }