X-Git-Url: https://code.citadel.org/?a=blobdiff_plain;f=citadel%2Fmodules%2Fmigrate%2Fserv_migrate.c;h=7cc8fc041fde63275598c34fd4c85e202cd5bd09;hb=79d6eafd2a8bb379b3e13ff071a000d9bf36771b;hp=8b431c8afbfdf021b9f290da5c5746ad85688e08;hpb=63262a64633e72e4096aaeedda6950c02fb4d990;p=citadel.git diff --git a/citadel/modules/migrate/serv_migrate.c b/citadel/modules/migrate/serv_migrate.c index 8b431c8af..7cc8fc041 100644 --- a/citadel/modules/migrate/serv_migrate.c +++ b/citadel/modules/migrate/serv_migrate.c @@ -1,23 +1,31 @@ /* - * $Id$ - * * This module dumps and/or loads the Citadel database in XML format. * - * Copyright (c) 1987-2010 by the citadel.org team + * Copyright (c) 1987-2021 by the citadel.org team * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 3 of the License, or - * (at your option) any later version. + * This program is open source software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 3. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. + */ + +/* + * Explanation of tags: * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + * 0% started + * 2% finished exporting configuration + * 7% finished exporting users + * 12% finished exporting openids + * 17% finished exporting rooms + * 18% finished exporting floors + * 25% finished exporting visits + * 26-99% exporting messages + * 100% finished exporting messages + * + * These tags are inserted into the XML stream to give the reader an approximation of its progress. */ #include "sysdep.h" @@ -29,18 +37,7 @@ #include #include #include - -#if TIME_WITH_SYS_TIME -# include -# include -#else -# if HAVE_SYS_TIME_H -# include -# else -# include -# endif -#endif - +#include #include #include #include @@ -55,45 +52,45 @@ #include "database.h" #include "msgbase.h" #include "user_ops.h" -#include "control.h" #include "euidindex.h" - - #include "ctdl_module.h" - - -#define END_OF_MESSAGE "---eom---dbd---" - char migr_tempfilename1[PATH_MAX]; char migr_tempfilename2[PATH_MAX]; FILE *migr_global_message_list; +int total_msgs = 0; +char *ikey = NULL; // If we're importing a config key we store it here. - - +/****************************************************************************** + * Code which implements the export appears in this section * + ******************************************************************************/ /* - * Code which implements the export appears in this section - */ - - - -/* - * Output a string to the client with these characters escaped: & < > + * Output a string to the client with these characters escaped: & < > " ' */ void xml_strout(char *str) { char *c = str; + if (str == NULL) { + return; + } + while (*c != 0) { - if (*c == '&') { - client_write("&", 5); + if (*c == '\"') { + client_write(HKEY(""")); + } + else if (*c == '\'') { + client_write(HKEY("'")); } else if (*c == '<') { - client_write("<", 4); + client_write(HKEY("<")); } else if (*c == '>') { - client_write(">", 4); + client_write(HKEY(">")); + } + else if (*c == '&') { + client_write(HKEY("&")); } else { client_write(c, 1); @@ -106,20 +103,31 @@ void xml_strout(char *str) { /* * Export a user record as XML */ -void migr_export_users_backend(struct ctdluser *buf, void *data) { - client_write("\n", 7); - cprintf("%d\n", buf->version); - cprintf("%ld\n", (long)buf->uid); - client_write("", 12); xml_strout(buf->password); client_write("\n", 14); - cprintf("%u\n", buf->flags); - cprintf("%ld\n", buf->timescalled); - cprintf("%ld\n", buf->posted); - cprintf("%d\n", buf->axlevel); - cprintf("%ld\n", buf->usernum); - cprintf("%ld\n", (long)buf->lastcall); - cprintf("%d\n", buf->USuserpurge); - client_write("", 12); xml_strout(buf->fullname); client_write("\n", 14); - client_write("\n", 8); +void migr_export_users_backend(char *username, void *data) { + + struct ctdluser u; + if (CtdlGetUser(&u, username) != 0) { + return; + } + + client_write(HKEY("\n")); + cprintf("%d\n", u.version); + cprintf("%ld\n", (long)u.uid); + client_write(HKEY("")); xml_strout(u.password); client_write(HKEY("\n")); + cprintf("%u\n", u.flags); + cprintf("%ld\n", u.timescalled); + cprintf("%ld\n", u.posted); + cprintf("%d\n", u.axlevel); + cprintf("%ld\n", u.usernum); + cprintf("%ld\n", (long)u.lastcall); + cprintf("%d\n", u.USuserpurge); + client_write(HKEY("")); xml_strout(u.fullname); client_write(HKEY("\n")); + cprintf("%ld\n", u.msgnum_bio); + cprintf("%ld\n", u.msgnum_pic); + cprintf("%s\n", u.emailaddrs); + cprintf("%ld\n", u.msgnum_inboxrules); + cprintf("%ld\n", u.lastproc_inboxrules); + client_write(HKEY("\n")); } @@ -129,21 +137,33 @@ void migr_export_users(void) { void migr_export_room_msg(long msgnum, void *userdata) { - cprintf("%ld\n", msgnum); + static int count = 0; + + cprintf("%ld,", msgnum); + if (++count%10==0) { + cprintf("\n"); + } fprintf(migr_global_message_list, "%ld\n", msgnum); } void migr_export_rooms_backend(struct ctdlroom *buf, void *data) { - client_write("\n", 7); - client_write("", 8); xml_strout(buf->QRname); client_write("\n", 10); - client_write("", 10); xml_strout(buf->QRpasswd); client_write("\n", 12); + client_write(HKEY("\n")); + client_write(HKEY("")); + xml_strout(buf->QRname); + client_write(HKEY("\n")); + client_write(HKEY("")); + xml_strout(buf->QRpasswd); + client_write(HKEY("\n")); cprintf("%ld\n", buf->QRroomaide); cprintf("%ld\n", buf->QRhighest); cprintf("%ld\n", (long)buf->QRgen); cprintf("%u\n", buf->QRflags); - client_write("", 11); xml_strout(buf->QRdirname); client_write("\n", 13); - cprintf("%ld\n", buf->QRinfo); + if (buf->QRflags & QR_DIRECTORY) { + client_write(HKEY("")); + xml_strout(buf->QRdirname); + client_write(HKEY("\n")); + } cprintf("%d\n", buf->QRfloor); cprintf("%ld\n", (long)buf->QRmtime); cprintf("%d\n", buf->QRep.expire_mode); @@ -152,19 +172,20 @@ void migr_export_rooms_backend(struct ctdlroom *buf, void *data) { cprintf("%d\n", buf->QRorder); cprintf("%u\n", buf->QRflags2); cprintf("%d\n", buf->QRdefaultview); - client_write("\n", 8); + cprintf("%ld\n", buf->msgnum_info); + cprintf("%ld\n", buf->msgnum_pic); + client_write(HKEY("\n")); /* message list goes inside this tag */ - CtdlGetRoom(&CC->room, buf->QRname); - client_write("", 15); - client_write("", 8); xml_strout(CC->room.QRname); client_write("\n", 10); - client_write("", 11); + client_write(HKEY("")); + client_write(HKEY("")); + xml_strout(buf->QRname); // buf->QRname rather than CC->room.QRname to guarantee consistency + client_write(HKEY("\n")); + client_write(HKEY("\n")); CtdlForEachMessage(MSGS_ALL, 0L, NULL, NULL, NULL, migr_export_room_msg, NULL); - client_write("\n", 13); - client_write("\n", 17); - - + client_write(HKEY("\n")); + client_write(HKEY("\n")); } @@ -182,10 +203,26 @@ void migr_export_rooms(void) { * this will be handled by exporting the reference count, not by * exporting the message multiple times.) */ - snprintf(cmd, sizeof cmd, "sort <%s >%s", migr_tempfilename1, migr_tempfilename2); - if (system(cmd) != 0) CtdlLogPrintf(CTDL_ALERT, "Error %d\n", errno); + snprintf(cmd, sizeof cmd, "sort -n <%s >%s", migr_tempfilename1, migr_tempfilename2); + if (system(cmd) != 0) { + syslog(LOG_ERR, "migrate: error %d", errno); + } snprintf(cmd, sizeof cmd, "uniq <%s >%s", migr_tempfilename2, migr_tempfilename1); - if (system(cmd) != 0) CtdlLogPrintf(CTDL_ALERT, "Error %d\n", errno); + if (system(cmd) != 0) { + syslog(LOG_ERR, "migrate: error %d", errno); + } + + snprintf(cmd, sizeof cmd, "wc -l %s", migr_tempfilename1); + FILE *fp = popen(cmd, "r"); + if (fp) { + fgets(cmd, sizeof cmd, fp); + pclose(fp); + total_msgs = atoi(cmd); + } + else { + total_msgs = 1; // any nonzero just to keep it from barfing + } + syslog(LOG_DEBUG, "migrate: total messages to be exported: %d", total_msgs); } @@ -194,20 +231,36 @@ void migr_export_floors(void) { int i; for (i=0; i < MAXFLOORS; ++i) { - client_write("\n", 8); + client_write(HKEY("\n")); cprintf("%d\n", i); CtdlGetFloor(&qfbuf, i); buf = &qfbuf; cprintf("%u\n", buf->f_flags); - client_write("", 8); xml_strout(buf->f_name); client_write("\n", 10); + client_write(HKEY("")); xml_strout(buf->f_name); client_write(HKEY("\n")); cprintf("%d\n", buf->f_ref_count); cprintf("%d\n", buf->f_ep.expire_mode); cprintf("%d\n", buf->f_ep.expire_value); - client_write("\n", 9); + client_write(HKEY("\n")); } } +/* + * Return nonzero if the supplied string contains only characters which are valid in a sequence set. + */ +int is_sequence_set(char *s) { + if (!s) return(0); + + char *c = s; + char ch; + while (ch = *c++, ch) { + if (!strchr("0123456789*,:", ch)) { + return(0); + } + } + return(1); +} + /* * Traverse the visits file... @@ -225,24 +278,29 @@ void migr_export_visits(void) { sizeof(visit) : cdbv->len)); cdb_free(cdbv); - client_write("\n", 8); + client_write(HKEY("\n")); cprintf("%ld\n", vbuf.v_roomnum); cprintf("%ld\n", vbuf.v_roomgen); cprintf("%ld\n", vbuf.v_usernum); - client_write("", 8); - if (!IsEmptyStr(vbuf.v_seen)) { + client_write(HKEY("")); + if ( (!IsEmptyStr(vbuf.v_seen)) && (is_sequence_set(vbuf.v_seen)) ) { xml_strout(vbuf.v_seen); } else { cprintf("%ld", vbuf.v_lastseen); } - client_write("", 9); + client_write(HKEY("")); + + if ( (!IsEmptyStr(vbuf.v_answered)) && (is_sequence_set(vbuf.v_answered)) ) { + client_write(HKEY("")); + xml_strout(vbuf.v_answered); + client_write(HKEY("\n")); + } - client_write("", 12); xml_strout(vbuf.v_answered); client_write("\n", 14); cprintf("%u\n", vbuf.v_flags); cprintf("%d\n", vbuf.v_view); - client_write("\n", 9); + client_write(HKEY("\n")); } } @@ -251,12 +309,14 @@ void migr_export_message(long msgnum) { struct MetaData smi; struct CtdlMessage *msg; struct ser_ret smr; + long bytes_written = 0; + long this_block = 0; + + // We can use a static buffer here because there will never be more than + // one of this operation happening at any given time, and it's really best + // to just keep it allocated once instead of torturing malloc/free. + // Call this function with msgnum "-1" to free the buffer when finished. - /* We can use a static buffer here because there will never be more than - * one of this operation happening at any given time, and it's really best - * to just keep it allocated once instead of torturing malloc/free. - * Call this function with msgnum "-1" to free the buffer when finished. - */ static int encoded_alloc = 0; static char *encoded_msg = NULL; @@ -269,22 +329,23 @@ void migr_export_message(long msgnum) { return; } - /* Ok, here we go ... */ + // Ok, here we go ... msg = CtdlFetchMessage(msgnum, 1); - if (msg == NULL) return; /* fail silently */ + if (msg == NULL) return; // fail silently - client_write("\n", 10); + client_write(HKEY("\n")); GetMetaData(&smi, msgnum); cprintf("%ld\n", msgnum); cprintf("%d\n", smi.meta_refcount); - client_write("", 23); xml_strout(smi.meta_content_type); client_write("\n", 25); + cprintf("%ld\n", smi.meta_rfc822_length); + client_write(HKEY("")); xml_strout(smi.meta_content_type); client_write(HKEY("\n")); - client_write("", 10); - serialize_message(&smr, msg); - CtdlFreeMessage(msg); + client_write(HKEY("")); + CtdlSerializeMessage(&smr, msg); + CM_Free(msg); - /* Predict the buffer size we need. Expand the buffer if necessary. */ + // Predict the buffer size we need. Expand the buffer if necessary. int encoded_len = smr.len * 15 / 10 ; if (encoded_len > encoded_alloc) { encoded_alloc = encoded_len; @@ -292,204 +353,162 @@ void migr_export_message(long msgnum) { } if (encoded_msg == NULL) { - /* Questionable hack that hopes it'll work next time and we only lose one message */ + // Questionable hack that hopes it'll work next time and we only lose one message encoded_alloc = 0; } else { - /* Once we do the encoding we know the exact size */ + // Once we do the encoding we know the exact size encoded_len = CtdlEncodeBase64(encoded_msg, (char *)smr.ser, smr.len, 1); - client_write(encoded_msg, encoded_len); + + // Careful now. If the message is gargantuan, trying to write multiple gigamegs in one + // big write operation can make our transport unhappy. So we'll chunk it up 10 KB at a time. + bytes_written = 0; + while ( (bytes_written < encoded_len) && (!server_shutting_down) ) { + this_block = encoded_len - bytes_written; + if (this_block > 10240) { + this_block = 10240; + } + client_write(&encoded_msg[bytes_written], this_block); + bytes_written += this_block; + } } free(smr.ser); - client_write("\n", 12); - client_write("\n", 11); + client_write(HKEY("\n")); + client_write(HKEY("\n")); } - void migr_export_openids(void) { struct cdbdata *cdboi; long usernum; char url[512]; - cdb_rewind(CDB_OPENID); - while (cdboi = cdb_next_item(CDB_OPENID), cdboi != NULL) { + cdb_rewind(CDB_EXTAUTH); + while (cdboi = cdb_next_item(CDB_EXTAUTH), cdboi != NULL) { if (cdboi->len > sizeof(long)) { - client_write("\n", 9); + client_write(HKEY("\n")); memcpy(&usernum, cdboi->ptr, sizeof(long)); snprintf(url, sizeof url, "%s", (cdboi->ptr)+sizeof(long) ); - client_write("", 9); + client_write(HKEY("")); xml_strout(url); - client_write("\n", 11); + client_write(HKEY("\n")); cprintf("%ld\n", usernum); - client_write("\n", 10); + client_write(HKEY("\n")); } cdb_free(cdboi); } } +void migr_export_configs(void) { + struct cdbdata *cdbcfg; + int keylen = 0; + char *key = NULL; + char *value = NULL; + + cdb_rewind(CDB_CONFIG); + while (cdbcfg = cdb_next_item(CDB_CONFIG), cdbcfg != NULL) { + + keylen = strlen(cdbcfg->ptr); + key = cdbcfg->ptr; + value = cdbcfg->ptr + keylen + 1; + + client_write("", 2); + xml_strout(value); + client_write("\n", 10); + cdb_free(cdbcfg); + } +} void migr_export_messages(void) { char buf[SIZ]; long msgnum; int count = 0; + int progress = 0; + int prev_progress = 0; CitContext *Ctx; Ctx = CC; migr_global_message_list = fopen(migr_tempfilename1, "r"); if (migr_global_message_list != NULL) { - CtdlLogPrintf(CTDL_INFO, "Opened %s\n", migr_tempfilename1); - while ((Ctx->kill_me != 1) && + syslog(LOG_INFO, "migrate: opened %s", migr_tempfilename1); + while ((Ctx->kill_me == 0) && (fgets(buf, sizeof(buf), migr_global_message_list) != NULL)) { msgnum = atol(buf); if (msgnum > 0L) { migr_export_message(msgnum); ++count; } + progress = (count * 74 / total_msgs) + 25 ; + if ((progress > prev_progress) && (progress < 100)) { + cprintf("%d\n", progress); + } + prev_progress = progress; } fclose(migr_global_message_list); } - if (Ctx->kill_me != 1) - CtdlLogPrintf(CTDL_INFO, "Exported %d messages.\n", count); - else - CtdlLogPrintf(CTDL_ERR, "Export aborted due to client disconnect! \n"); + if (Ctx->kill_me == 0) { + syslog(LOG_INFO, "migrate: exported %d messages.", count); + } + else { + syslog(LOG_ERR, "migrate: export aborted due to client disconnect!"); + } migr_export_message(-1L); /* This frees the encoding buffer */ } - void migr_do_export(void) { - struct config *buf; - buf = &config; CitContext *Ctx; Ctx = CC; cprintf("%d Exporting all Citadel databases.\n", LISTING_FOLLOWS); Ctx->dont_term = 1; - client_write("\n", 40); - client_write("\n", 23); + client_write(HKEY("\n")); + client_write(HKEY("\n")); cprintf("%d\n", REV_LEVEL); + cprintf("%d\n", 0); - /* export the config file (this is done using x-macros) */ - client_write("\n", 9); - client_write("", 12); xml_strout(config.c_nodename); client_write("\n", 14); - client_write("", 8); xml_strout(config.c_fqdn); client_write("\n", 10); - client_write("", 13); xml_strout(config.c_humannode); client_write("\n", 15); - client_write("", 12); xml_strout(config.c_phonenum); client_write("\n", 14); - cprintf("%d\n", config.c_ctdluid); - cprintf("%d\n", config.c_creataide); - cprintf("%d\n", config.c_sleeping); - cprintf("%d\n", config.c_initax); - cprintf("%d\n", config.c_regiscall); - cprintf("%d\n", config.c_twitdetect); - client_write("", 12); xml_strout(config.c_twitroom); client_write("\n", 14); - client_write("", 14); xml_strout(config.c_moreprompt); client_write("\n", 16); - cprintf("%d\n", config.c_restrict); - client_write("", 17); xml_strout(config.c_site_location); client_write("\n", 19); - client_write("", 10); xml_strout(config.c_sysadm); client_write("\n", 12); - cprintf("%d\n", config.c_setup_level); - cprintf("%d\n", config.c_maxsessions); - client_write("", 11); xml_strout(config.c_ip_addr); client_write("\n", 13); - cprintf("%d\n", config.c_port_number); - cprintf("%d\n", config.c_ep.expire_mode); - cprintf("%d\n", config.c_ep.expire_value); - cprintf("%d\n", config.c_userpurge); - cprintf("%d\n", config.c_roompurge); - client_write("", 12); xml_strout(config.c_logpages); client_write("\n", 14); - cprintf("%d\n", config.c_createax); - cprintf("%ld\n", config.c_maxmsglen); - cprintf("%d\n", config.c_min_workers); - cprintf("%d\n", config.c_max_workers); - cprintf("%d\n", config.c_pop3_port); - cprintf("%d\n", config.c_smtp_port); - cprintf("%d\n", config.c_rfc822_strict_from); - cprintf("%d\n", config.c_aide_zap); - cprintf("%d\n", config.c_imap_port); - cprintf("%ld\n", config.c_net_freq); - cprintf("%d\n", config.c_disable_newu); - cprintf("%d\n", config.c_enable_fulltext); - client_write("", 12); xml_strout(config.c_baseroom); client_write("\n", 14); - client_write("", 12); xml_strout(config.c_aideroom); client_write("\n", 14); - cprintf("%d\n", config.c_purge_hour); - cprintf("%d\n", config.c_mbxep.expire_mode); - cprintf("%d\n", config.c_mbxep.expire_value); - client_write("", 13); xml_strout(config.c_ldap_host); client_write("\n", 15); - cprintf("%d\n", config.c_ldap_port); - client_write("", 16); xml_strout(config.c_ldap_base_dn); client_write("\n", 18); - client_write("", 16); xml_strout(config.c_ldap_bind_dn); client_write("\n", 18); - client_write("", 16); xml_strout(config.c_ldap_bind_pw); client_write("\n", 18); - cprintf("%d\n", config.c_msa_port); - cprintf("%d\n", config.c_imaps_port); - cprintf("%d\n", config.c_pop3s_port); - cprintf("%d\n", config.c_smtps_port); - cprintf("%d\n", config.c_auto_cull); - cprintf("%d\n", config.c_instant_expunge); - cprintf("%d\n", config.c_allow_spoofing); - cprintf("%d\n", config.c_journal_email); - cprintf("%d\n", config.c_journal_pubmsgs); - client_write("", 16); xml_strout(config.c_journal_dest); client_write("\n", 18); - client_write("", 20); xml_strout(config.c_default_cal_zone); client_write("\n", 22); - cprintf("%d\n", config.c_pftcpdict_port); - cprintf("%d\n", config.c_managesieve_port); - cprintf("%d\n", config.c_auth_mode); - client_write("", 17); xml_strout(config.c_funambol_host); client_write("\n", 19); - cprintf("%d\n", config.c_funambol_port); - client_write("", 19); xml_strout(config.c_funambol_source); client_write("\n", 21); - client_write("", 17); xml_strout(config.c_funambol_auth); client_write("\n", 19); - cprintf("%d\n", config.c_rbl_at_greeting); - client_write("", 15); xml_strout(config.c_master_user); client_write("\n", 17); - client_write("", 15); xml_strout(config.c_master_pass); client_write("\n", 17); - client_write("", 17); xml_strout(config.c_pager_program); client_write("\n", 19); - cprintf("%d\n", config.c_imap_keep_from); - cprintf("%d\n", config.c_xmpp_c2s_port); - cprintf("%d\n", config.c_xmpp_s2s_port); - cprintf("%ld\n", config.c_pop3_fetch); - cprintf("%ld\n", config.c_pop3_fastest); - cprintf("%d\n", config.c_spam_flag_only); - client_write("\n", 10); + /* export the configuration database */ + migr_export_configs(); + cprintf("%d\n", 2); - /* Export the control file */ - get_control(); - client_write("\n", 10); - cprintf("%ld\n", CitControl.MMhighest); - cprintf("%u\n", CitControl.MMflags); - cprintf("%ld\n", CitControl.MMnextuser); - cprintf("%ld\n", CitControl.MMnextroom); - cprintf("%d\n", CitControl.version); - client_write("\n", 11); - - if (Ctx->kill_me != 1) migr_export_users(); - if (Ctx->kill_me != 1) migr_export_openids(); - if (Ctx->kill_me != 1) migr_export_rooms(); - if (Ctx->kill_me != 1) migr_export_floors(); - if (Ctx->kill_me != 1) migr_export_visits(); - if (Ctx->kill_me != 1) migr_export_messages(); - client_write("\n", 24); - client_write("000\n", 4); + if (Ctx->kill_me == 0) migr_export_users(); + cprintf("%d\n", 7); + if (Ctx->kill_me == 0) migr_export_openids(); + cprintf("%d\n", 12); + if (Ctx->kill_me == 0) migr_export_rooms(); + cprintf("%d\n", 17); + if (Ctx->kill_me == 0) migr_export_floors(); + cprintf("%d\n", 18); + if (Ctx->kill_me == 0) migr_export_visits(); + cprintf("%d\n", 25); + if (Ctx->kill_me == 0) migr_export_messages(); + cprintf("%d\n", 100); + client_write(HKEY("\n")); + client_write(HKEY("000\n")); Ctx->dont_term = 0; } - - - -/* - * Here's the code that implements the import side. It's going to end up being - * one big loop with lots of global variables. I don't care. You wouldn't run - * multiple concurrent imports anyway. If this offends your delicate sensibilities - * then go rewrite it in Ruby on Rails or something. - */ +/****************************************************************************** + * Import code * + * Here's the code that implements the import side. It's going to end up * + * being one big loop with lots of global variables. I don't care. * + * You wouldn't run multiple concurrent imports anyway. If this offends your * + * delicate sensibilities then go rewrite it in Ruby on Rails or something. * + ******************************************************************************/ int citadel_migrate_data = 0; /* Are we inside a tag pair? */ -char *migr_chardata = NULL; -int migr_chardata_len = 0; +StrBuf *migr_chardata = NULL; +StrBuf *migr_MsgData = NULL; struct ctdluser usbuf; struct ctdlroom qrbuf; char openid_url[512]; @@ -500,25 +519,12 @@ int floornum = 0; visit vbuf; struct MetaData smi; long import_msgnum = 0; -char *decoded_msg = NULL; /* * This callback stores up the data which appears in between tags. */ void migr_xml_chardata(void *data, const XML_Char *s, int len) { - int old_len; - int new_len; - char *new_buffer; - - old_len = migr_chardata_len; - new_len = old_len + len; - new_buffer = realloc(migr_chardata, new_len + 1); - if (new_buffer != NULL) { - memcpy(&new_buffer[old_len], s, len); - new_buffer[new_len] = 0; - migr_chardata = new_buffer; - migr_chardata_len = new_len; - } + StrBufAppendBufPlain(migr_chardata, s, len, 0); } @@ -527,12 +533,9 @@ void migr_xml_start(void *data, const char *el, const char **attr) { /*** GENERAL STUFF ***/ - /* Throw away any existing chardata */ - if (migr_chardata != NULL) { - free(migr_chardata); - migr_chardata = NULL; - migr_chardata_len = 0; - } + /* Reset chardata_len to zero and init buffer */ + FlushStrBuf(migr_chardata); + FlushStrBuf(migr_MsgData); if (!strcasecmp(el, "citadel_migrate_data")) { ++citadel_migrate_data; @@ -549,7 +552,7 @@ void migr_xml_start(void *data, const char *el, const char **attr) { } if (citadel_migrate_data != 1) { - CtdlLogPrintf(CTDL_ALERT, "Out-of-sequence tag <%s> detected. Warning: ODD-DATA!\n"); + syslog(LOG_ERR, "migrate: out-of-sequence tag <%s> detected. Warning: ODD-DATA!", el); return; } @@ -566,26 +569,113 @@ void migr_xml_start(void *data, const char *el, const char **attr) { else if (!strcasecmp(el, "message")) { memset(&smi, 0, sizeof (struct MetaData)); import_msgnum = 0; - if (decoded_msg != NULL) { - free(decoded_msg); - decoded_msg = NULL; + } + else if (!strcasecmp(el, "config")) { + if (ikey != NULL) { + free(ikey); + ikey = NULL; + } + while (*attr) { + if (!strcasecmp(attr[0], "key")) { + ikey = strdup(attr[1]); + } + attr += 2; } } } +int migr_userrecord(void *data, const char *el) { + if (!strcasecmp(el, "u_version")) usbuf.version = atoi(ChrPtr(migr_chardata)); + else if (!strcasecmp(el, "u_uid")) usbuf.uid = atol(ChrPtr(migr_chardata)); + else if (!strcasecmp(el, "u_password")) safestrncpy(usbuf.password, ChrPtr(migr_chardata), sizeof usbuf.password); + else if (!strcasecmp(el, "u_flags")) usbuf.flags = atoi(ChrPtr(migr_chardata)); + else if (!strcasecmp(el, "u_timescalled")) usbuf.timescalled = atol(ChrPtr(migr_chardata)); + else if (!strcasecmp(el, "u_posted")) usbuf.posted = atol(ChrPtr(migr_chardata)); + else if (!strcasecmp(el, "u_axlevel")) usbuf.axlevel = atoi(ChrPtr(migr_chardata)); + else if (!strcasecmp(el, "u_usernum")) usbuf.usernum = atol(ChrPtr(migr_chardata)); + else if (!strcasecmp(el, "u_lastcall")) usbuf.lastcall = atol(ChrPtr(migr_chardata)); + else if (!strcasecmp(el, "u_USuserpurge")) usbuf.USuserpurge = atoi(ChrPtr(migr_chardata)); + else if (!strcasecmp(el, "u_fullname")) safestrncpy(usbuf.fullname, ChrPtr(migr_chardata), sizeof usbuf.fullname); + else if (!strcasecmp(el, "u_msgnum_bio")) usbuf.msgnum_bio = atol(ChrPtr(migr_chardata)); + else if (!strcasecmp(el, "u_msgnum_pic")) usbuf.msgnum_pic = atol(ChrPtr(migr_chardata)); + else if (!strcasecmp(el, "u_emailaddrs")) safestrncpy(usbuf.emailaddrs, ChrPtr(migr_chardata), sizeof usbuf.emailaddrs); + else if (!strcasecmp(el, "u_msgnum_inboxrules")) usbuf.msgnum_inboxrules = atol(ChrPtr(migr_chardata)); + else if (!strcasecmp(el, "u_lastproc_inboxrules")) usbuf.lastproc_inboxrules = atol(ChrPtr(migr_chardata)); + else return 0; + return 1; +} + + +int migr_roomrecord(void *data, const char *el) { + if (!strcasecmp(el, "QRname")) safestrncpy(qrbuf.QRname, ChrPtr(migr_chardata), sizeof qrbuf.QRname); + else if (!strcasecmp(el, "QRpasswd")) safestrncpy(qrbuf.QRpasswd, ChrPtr(migr_chardata), sizeof qrbuf.QRpasswd); + else if (!strcasecmp(el, "QRroomaide")) qrbuf.QRroomaide = atol(ChrPtr(migr_chardata)); + else if (!strcasecmp(el, "QRhighest")) qrbuf.QRhighest = atol(ChrPtr(migr_chardata)); + else if (!strcasecmp(el, "QRgen")) qrbuf.QRgen = atol(ChrPtr(migr_chardata)); + else if (!strcasecmp(el, "QRflags")) qrbuf.QRflags = atoi(ChrPtr(migr_chardata)); + else if (!strcasecmp(el, "QRdirname")) safestrncpy(qrbuf.QRdirname, ChrPtr(migr_chardata), sizeof qrbuf.QRdirname); + else if (!strcasecmp(el, "QRfloor")) qrbuf.QRfloor = atoi(ChrPtr(migr_chardata)); + else if (!strcasecmp(el, "QRmtime")) qrbuf.QRmtime = atol(ChrPtr(migr_chardata)); + else if (!strcasecmp(el, "QRexpire_mode")) qrbuf.QRep.expire_mode = atoi(ChrPtr(migr_chardata)); + else if (!strcasecmp(el, "QRexpire_value")) qrbuf.QRep.expire_value = atoi(ChrPtr(migr_chardata)); + else if (!strcasecmp(el, "QRnumber")) qrbuf.QRnumber = atol(ChrPtr(migr_chardata)); + else if (!strcasecmp(el, "QRorder")) qrbuf.QRorder = atoi(ChrPtr(migr_chardata)); + else if (!strcasecmp(el, "QRflags2")) qrbuf.QRflags2 = atoi(ChrPtr(migr_chardata)); + else if (!strcasecmp(el, "QRdefaultview")) qrbuf.QRdefaultview = atoi(ChrPtr(migr_chardata)); + else if (!strcasecmp(el, "msgnum_info")) qrbuf.msgnum_info = atol(ChrPtr(migr_chardata)); + else if (!strcasecmp(el, "msgnum_pic")) qrbuf.msgnum_pic = atol(ChrPtr(migr_chardata)); + else return 0; + return 1; +} + + +int migr_floorrecord(void *data, const char *el) { + if (!strcasecmp(el, "f_num")) floornum = atoi(ChrPtr(migr_chardata)); + else if (!strcasecmp(el, "f_flags")) flbuf.f_flags = atoi(ChrPtr(migr_chardata)); + else if (!strcasecmp(el, "f_name")) safestrncpy(flbuf.f_name, ChrPtr(migr_chardata), sizeof flbuf.f_name); + else if (!strcasecmp(el, "f_ref_count")) flbuf.f_ref_count = atoi(ChrPtr(migr_chardata)); + else if (!strcasecmp(el, "f_ep_expire_mode")) flbuf.f_ep.expire_mode = atoi(ChrPtr(migr_chardata)); + else if (!strcasecmp(el, "f_ep_expire_value")) flbuf.f_ep.expire_value = atoi(ChrPtr(migr_chardata)); + else return 0; + return 1; +} + + +int migr_visitrecord(void *data, const char *el) { + if (!strcasecmp(el, "v_roomnum")) vbuf.v_roomnum = atol(ChrPtr(migr_chardata)); + else if (!strcasecmp(el, "v_roomgen")) vbuf.v_roomgen = atol(ChrPtr(migr_chardata)); + else if (!strcasecmp(el, "v_usernum")) vbuf.v_usernum = atol(ChrPtr(migr_chardata)); + + else if (!strcasecmp(el, "v_seen")) { + int is_textual_seen = 0; + int i; + int max = StrLength(migr_chardata); + + vbuf.v_lastseen = atol(ChrPtr(migr_chardata)); + is_textual_seen = 0; + for (i=0; i < max; ++i) + if (!isdigit(ChrPtr(migr_chardata)[i])) + is_textual_seen = 1; + if (is_textual_seen) + safestrncpy(vbuf.v_seen, ChrPtr(migr_chardata), sizeof vbuf.v_seen); + } + + else if (!strcasecmp(el, "v_answered")) safestrncpy(vbuf.v_answered, ChrPtr(migr_chardata), sizeof vbuf.v_answered); + else if (!strcasecmp(el, "v_flags")) vbuf.v_flags = atoi(ChrPtr(migr_chardata)); + else if (!strcasecmp(el, "v_view")) vbuf.v_view = atoi(ChrPtr(migr_chardata)); + else return 0; + return 1; +} + + void migr_xml_end(void *data, const char *el) { - char *ptr; + const char *ptr; int msgcount = 0; long msgnum = 0L; long *msglist = NULL; int msglist_alloc = 0; - int i; - int is_textual_seen = 0; - long msglen; - int len; - /*** GENERAL STUFF ***/ if (!strcasecmp(el, "citadel_migrate_data")) { @@ -594,137 +684,40 @@ void migr_xml_end(void *data, const char *el) { } if (citadel_migrate_data != 1) { - CtdlLogPrintf(CTDL_ALERT, "Out-of-sequence tag <%s> detected. Warning: ODD-DATA!\n"); + syslog(LOG_ERR, "migrate: out-of-sequence tag <%s> detected. Warning: ODD-DATA!", el); return; } - if (migr_chardata == NULL) { /* If NULL chardata, substitute an empty string instead. */ - migr_chardata = strdup(""); - migr_chardata_len = 0; - } - - // CtdlLogPrintf(CTDL_DEBUG, "END TAG: <%s> DATA: <%s>\n", el, (migr_chardata_len ? migr_chardata : "")); + // syslog(LOG_DEBUG, "END TAG: <%s> DATA: <%s>\n", el, (migr_chardata_len ? migr_chardata : "")); /*** CONFIG ***/ if (!strcasecmp(el, "config")) { - config.c_enable_fulltext = 0; /* always disable */ - put_config(); - CtdlLogPrintf(CTDL_INFO, "Completed import of server configuration\n"); - } - - else if (!strcasecmp(el, "c_nodename")) safestrncpy(config.c_nodename, migr_chardata, sizeof config.c_nodename); - else if (!strcasecmp(el, "c_fqdn")) safestrncpy(config.c_fqdn, migr_chardata, sizeof config.c_fqdn); - else if (!strcasecmp(el, "c_humannode")) safestrncpy(config.c_humannode, migr_chardata, sizeof config.c_humannode); - else if (!strcasecmp(el, "c_phonenum")) safestrncpy(config.c_phonenum, migr_chardata, sizeof config.c_phonenum); - else if (!strcasecmp(el, "c_ctdluid")) config.c_ctdluid = atoi(migr_chardata); - else if (!strcasecmp(el, "c_creataide")) config.c_creataide = atoi(migr_chardata); - else if (!strcasecmp(el, "c_sleeping")) config.c_sleeping = atoi(migr_chardata); - else if (!strcasecmp(el, "c_initax")) config.c_initax = atoi(migr_chardata); - else if (!strcasecmp(el, "c_regiscall")) config.c_regiscall = atoi(migr_chardata); - else if (!strcasecmp(el, "c_twitdetect")) config.c_twitdetect = atoi(migr_chardata); - else if (!strcasecmp(el, "c_twitroom")) safestrncpy(config.c_twitroom, migr_chardata, sizeof config.c_twitroom); - else if (!strcasecmp(el, "c_moreprompt")) safestrncpy(config.c_moreprompt, migr_chardata, sizeof config.c_moreprompt); - else if (!strcasecmp(el, "c_restrict")) config.c_restrict = atoi(migr_chardata); - else if (!strcasecmp(el, "c_site_location")) safestrncpy(config.c_site_location, migr_chardata, sizeof config.c_site_location); - else if (!strcasecmp(el, "c_sysadm")) safestrncpy(config.c_sysadm, migr_chardata, sizeof config.c_sysadm); - else if (!strcasecmp(el, "c_setup_level")) config.c_setup_level = atoi(migr_chardata); - else if (!strcasecmp(el, "c_maxsessions")) config.c_maxsessions = atoi(migr_chardata); - else if (!strcasecmp(el, "c_ip_addr")) safestrncpy(config.c_ip_addr, migr_chardata, sizeof config.c_ip_addr); - else if (!strcasecmp(el, "c_port_number")) config.c_port_number = atoi(migr_chardata); - else if (!strcasecmp(el, "c_ep_expire_mode")) config.c_ep.expire_mode = atoi(migr_chardata); - else if (!strcasecmp(el, "c_ep_expire_value")) config.c_ep.expire_value = atoi(migr_chardata); - else if (!strcasecmp(el, "c_userpurge")) config.c_userpurge = atoi(migr_chardata); - else if (!strcasecmp(el, "c_roompurge")) config.c_roompurge = atoi(migr_chardata); - else if (!strcasecmp(el, "c_logpages")) safestrncpy(config.c_logpages, migr_chardata, sizeof config.c_logpages); - else if (!strcasecmp(el, "c_createax")) config.c_createax = atoi(migr_chardata); - else if (!strcasecmp(el, "c_maxmsglen")) config.c_maxmsglen = atol(migr_chardata); - else if (!strcasecmp(el, "c_min_workers")) config.c_min_workers = atoi(migr_chardata); - else if (!strcasecmp(el, "c_max_workers")) config.c_max_workers = atoi(migr_chardata); - else if (!strcasecmp(el, "c_pop3_port")) config.c_pop3_port = atoi(migr_chardata); - else if (!strcasecmp(el, "c_smtp_port")) config.c_smtp_port = atoi(migr_chardata); - else if (!strcasecmp(el, "c_rfc822_strict_from")) config.c_rfc822_strict_from = atoi(migr_chardata); - else if (!strcasecmp(el, "c_aide_zap")) config.c_aide_zap = atoi(migr_chardata); - else if (!strcasecmp(el, "c_imap_port")) config.c_imap_port = atoi(migr_chardata); - else if (!strcasecmp(el, "c_net_freq")) config.c_net_freq = atol(migr_chardata); - else if (!strcasecmp(el, "c_disable_newu")) config.c_disable_newu = atoi(migr_chardata); - else if (!strcasecmp(el, "c_enable_fulltext")) config.c_enable_fulltext = atoi(migr_chardata); - else if (!strcasecmp(el, "c_baseroom")) safestrncpy(config.c_baseroom, migr_chardata, sizeof config.c_baseroom); - else if (!strcasecmp(el, "c_aideroom")) safestrncpy(config.c_aideroom, migr_chardata, sizeof config.c_aideroom); - else if (!strcasecmp(el, "c_purge_hour")) config.c_purge_hour = atoi(migr_chardata); - else if (!strcasecmp(el, "c_mbxep_expire_mode")) config.c_mbxep.expire_mode = atoi(migr_chardata); - else if (!strcasecmp(el, "c_mbxep_expire_value")) config.c_mbxep.expire_value = atoi(migr_chardata); - else if (!strcasecmp(el, "c_ldap_host")) safestrncpy(config.c_ldap_host, migr_chardata, sizeof config.c_ldap_host); - else if (!strcasecmp(el, "c_ldap_port")) config.c_ldap_port = atoi(migr_chardata); - else if (!strcasecmp(el, "c_ldap_base_dn")) safestrncpy(config.c_ldap_base_dn, migr_chardata, sizeof config.c_ldap_base_dn); - else if (!strcasecmp(el, "c_ldap_bind_dn")) safestrncpy(config.c_ldap_bind_dn, migr_chardata, sizeof config.c_ldap_bind_dn); - else if (!strcasecmp(el, "c_ldap_bind_pw")) safestrncpy(config.c_ldap_bind_pw, migr_chardata, sizeof config.c_ldap_bind_pw); - else if (!strcasecmp(el, "c_msa_port")) config.c_msa_port = atoi(migr_chardata); - else if (!strcasecmp(el, "c_imaps_port")) config.c_imaps_port = atoi(migr_chardata); - else if (!strcasecmp(el, "c_pop3s_port")) config.c_pop3s_port = atoi(migr_chardata); - else if (!strcasecmp(el, "c_smtps_port")) config.c_smtps_port = atoi(migr_chardata); - else if (!strcasecmp(el, "c_auto_cull")) config.c_auto_cull = atoi(migr_chardata); - else if (!strcasecmp(el, "c_instant_expunge")) config.c_instant_expunge = atoi(migr_chardata); - else if (!strcasecmp(el, "c_allow_spoofing")) config.c_allow_spoofing = atoi(migr_chardata); - else if (!strcasecmp(el, "c_journal_email")) config.c_journal_email = atoi(migr_chardata); - else if (!strcasecmp(el, "c_journal_pubmsgs")) config.c_journal_pubmsgs = atoi(migr_chardata); - else if (!strcasecmp(el, "c_journal_dest")) safestrncpy(config.c_journal_dest, migr_chardata, sizeof config.c_journal_dest); - else if (!strcasecmp(el, "c_default_cal_zone")) safestrncpy(config.c_default_cal_zone, migr_chardata, sizeof config.c_default_cal_zone); - else if (!strcasecmp(el, "c_pftcpdict_port")) config.c_pftcpdict_port = atoi(migr_chardata); - else if (!strcasecmp(el, "c_managesieve_port")) config.c_managesieve_port = atoi(migr_chardata); - else if (!strcasecmp(el, "c_auth_mode")) config.c_auth_mode = atoi(migr_chardata); - else if (!strcasecmp(el, "c_funambol_host")) safestrncpy(config.c_funambol_host, migr_chardata, sizeof config.c_funambol_host); - else if (!strcasecmp(el, "c_funambol_port")) config.c_funambol_port = atoi(migr_chardata); - else if (!strcasecmp(el, "c_funambol_source")) safestrncpy(config.c_funambol_source, migr_chardata, sizeof config.c_funambol_source); - else if (!strcasecmp(el, "c_funambol_auth")) safestrncpy(config.c_funambol_auth, migr_chardata, sizeof config.c_funambol_auth); - else if (!strcasecmp(el, "c_rbl_at_greeting")) config.c_rbl_at_greeting = atoi(migr_chardata); - else if (!strcasecmp(el, "c_master_user")) safestrncpy(config.c_master_user, migr_chardata, sizeof config.c_master_user); - else if (!strcasecmp(el, "c_master_pass")) safestrncpy(config.c_master_pass, migr_chardata, sizeof config.c_master_pass); - else if (!strcasecmp(el, "c_pager_program")) safestrncpy(config.c_pager_program, migr_chardata, sizeof config.c_pager_program); - else if (!strcasecmp(el, "c_imap_keep_from")) config.c_imap_keep_from = atoi(migr_chardata); - else if (!strcasecmp(el, "c_xmpp_c2s_port")) config.c_xmpp_c2s_port = atoi(migr_chardata); - else if (!strcasecmp(el, "c_xmpp_s2s_port")) config.c_xmpp_s2s_port = atoi(migr_chardata); - else if (!strcasecmp(el, "c_pop3_fetch")) config.c_pop3_fetch = atol(migr_chardata); - else if (!strcasecmp(el, "c_pop3_fastest")) config.c_pop3_fastest = atol(migr_chardata); - else if (!strcasecmp(el, "c_spam_flag_only")) config.c_spam_flag_only = atoi(migr_chardata); - - /*** CONTROL ***/ - - else if (!strcasecmp(el, "control_highest")) CitControl.MMhighest = atol(migr_chardata); - else if (!strcasecmp(el, "control_flags")) CitControl.MMflags = atoi(migr_chardata); - else if (!strcasecmp(el, "control_nextuser")) CitControl.MMnextuser = atol(migr_chardata); - else if (!strcasecmp(el, "control_nextroom")) CitControl.MMnextroom = atol(migr_chardata); - else if (!strcasecmp(el, "control_version")) CitControl.version = atoi(migr_chardata); - - else if (!strcasecmp(el, "control")) { - CitControl.MMfulltext = (-1L); /* always flush */ - put_control(); - CtdlLogPrintf(CTDL_INFO, "Completed import of control record\n"); + syslog(LOG_DEBUG, "migrate: imported config key=%s", ikey); + + if (ikey != NULL) { + CtdlSetConfigStr(ikey, (char *)ChrPtr(migr_chardata)); + free(ikey); + ikey = NULL; + } + else { + syslog(LOG_INFO, "migrate: closed a tag but no key name was supplied."); + } } /*** USER ***/ - - else if (!strcasecmp(el, "u_version")) usbuf.version = atoi(migr_chardata); - else if (!strcasecmp(el, "u_uid")) usbuf.uid = atol(migr_chardata); - else if (!strcasecmp(el, "u_password")) safestrncpy(usbuf.password, migr_chardata, sizeof usbuf.password); - else if (!strcasecmp(el, "u_flags")) usbuf.flags = atoi(migr_chardata); - else if (!strcasecmp(el, "u_timescalled")) usbuf.timescalled = atol(migr_chardata); - else if (!strcasecmp(el, "u_posted")) usbuf.posted = atol(migr_chardata); - else if (!strcasecmp(el, "u_axlevel")) usbuf.axlevel = atoi(migr_chardata); - else if (!strcasecmp(el, "u_usernum")) usbuf.usernum = atol(migr_chardata); - else if (!strcasecmp(el, "u_lastcall")) usbuf.lastcall = atol(migr_chardata); - else if (!strcasecmp(el, "u_USuserpurge")) usbuf.USuserpurge = atoi(migr_chardata); - else if (!strcasecmp(el, "u_fullname")) safestrncpy(usbuf.fullname, migr_chardata, sizeof usbuf.fullname); - + else if ((!strncasecmp(el, HKEY("u_"))) && + migr_userrecord(data, el)) + ; /* Nothing to do anymore */ else if (!strcasecmp(el, "user")) { CtdlPutUser(&usbuf); - CtdlLogPrintf(CTDL_INFO, "Imported user: %s\n", usbuf.fullname); + syslog(LOG_INFO, "migrate: imported user: %s", usbuf.fullname); } /*** OPENID ***/ - else if (!strcasecmp(el, "oid_url")) safestrncpy(openid_url, migr_chardata, sizeof openid_url); - else if (!strcasecmp(el, "oid_usernum")) openid_usernum = atol(migr_chardata); + else if (!strcasecmp(el, "oid_url")) safestrncpy(openid_url, ChrPtr(migr_chardata), sizeof openid_url); + else if (!strcasecmp(el, "oid_usernum")) openid_usernum = atol(ChrPtr(migr_chardata)); else if (!strcasecmp(el, "openid")) { /* see serv_openid_rp.c for a description of the record format */ char *oid_data; @@ -733,38 +726,25 @@ void migr_xml_end(void *data, const char *el) { oid_data = malloc(oid_data_len); memcpy(oid_data, &openid_usernum, sizeof(long)); memcpy(&oid_data[sizeof(long)], openid_url, strlen(openid_url) + 1); - cdb_store(CDB_OPENID, openid_url, strlen(openid_url), oid_data, oid_data_len); + cdb_store(CDB_EXTAUTH, openid_url, strlen(openid_url), oid_data, oid_data_len); free(oid_data); - CtdlLogPrintf(CTDL_INFO, "Imported OpenID: %s (%ld)\n", openid_url, openid_usernum); + syslog(LOG_INFO, "migrate: imported OpenID: %s (%ld)", openid_url, openid_usernum); } /*** ROOM ***/ - - else if (!strcasecmp(el, "QRname")) safestrncpy(qrbuf.QRname, migr_chardata, sizeof qrbuf.QRname); - else if (!strcasecmp(el, "QRpasswd")) safestrncpy(qrbuf.QRpasswd, migr_chardata, sizeof qrbuf.QRpasswd); - else if (!strcasecmp(el, "QRroomaide")) qrbuf.QRroomaide = atol(migr_chardata); - else if (!strcasecmp(el, "QRhighest")) qrbuf.QRhighest = atol(migr_chardata); - else if (!strcasecmp(el, "QRgen")) qrbuf.QRgen = atol(migr_chardata); - else if (!strcasecmp(el, "QRflags")) qrbuf.QRflags = atoi(migr_chardata); - else if (!strcasecmp(el, "QRdirname")) safestrncpy(qrbuf.QRdirname, migr_chardata, sizeof qrbuf.QRdirname); - else if (!strcasecmp(el, "QRinfo")) qrbuf.QRinfo = atol(migr_chardata); - else if (!strcasecmp(el, "QRfloor")) qrbuf.QRfloor = atoi(migr_chardata); - else if (!strcasecmp(el, "QRmtime")) qrbuf.QRmtime = atol(migr_chardata); - else if (!strcasecmp(el, "QRexpire_mode")) qrbuf.QRep.expire_mode = atoi(migr_chardata); - else if (!strcasecmp(el, "QRexpire_value")) qrbuf.QRep.expire_value = atoi(migr_chardata); - else if (!strcasecmp(el, "QRnumber")) qrbuf.QRnumber = atol(migr_chardata); - else if (!strcasecmp(el, "QRorder")) qrbuf.QRorder = atoi(migr_chardata); - else if (!strcasecmp(el, "QRflags2")) qrbuf.QRflags2 = atoi(migr_chardata); - else if (!strcasecmp(el, "QRdefaultview")) qrbuf.QRdefaultview = atoi(migr_chardata); - + else if ((!strncasecmp(el, HKEY("QR"))) && + migr_roomrecord(data, el)) + ; /* Nothing to do anymore */ else if (!strcasecmp(el, "room")) { CtdlPutRoom(&qrbuf); - CtdlLogPrintf(CTDL_INFO, "Imported room: %s\n", qrbuf.QRname); + syslog(LOG_INFO, "migrate: imported room: %s", qrbuf.QRname); } /*** ROOM MESSAGE POINTERS ***/ - else if (!strcasecmp(el, "FRname")) safestrncpy(FRname, migr_chardata, sizeof FRname); + else if (!strcasecmp(el, "FRname")) { + safestrncpy(FRname, ChrPtr(migr_chardata), sizeof FRname); + } else if (!strcasecmp(el, "FRmsglist")) { if (!IsEmptyStr(FRname)) { @@ -772,9 +752,9 @@ void migr_xml_end(void *data, const char *el) { msglist_alloc = 1000; msglist = malloc(sizeof(long) * msglist_alloc); - CtdlLogPrintf(CTDL_DEBUG, "Message list for: %s\n", FRname); + syslog(LOG_DEBUG, "migrate: message list for: %s", FRname); - ptr = migr_chardata; + ptr = ChrPtr(migr_chardata); while (*ptr != 0) { while ((*ptr != 0) && (!isdigit(*ptr))) { ++ptr; @@ -800,91 +780,87 @@ void migr_xml_end(void *data, const char *el) { free(msglist); msglist = NULL; msglist_alloc = 0; - CtdlLogPrintf(CTDL_DEBUG, "Imported %d messages.\n", msgcount); - if (CtdlThreadCheckStop()) { + syslog(LOG_DEBUG, "migrate: imported %d messages.", msgcount); + if (server_shutting_down) { return; } } /*** FLOORS ***/ - - else if (!strcasecmp(el, "f_num")) floornum = atoi(migr_chardata); - else if (!strcasecmp(el, "f_flags")) flbuf.f_flags = atoi(migr_chardata); - else if (!strcasecmp(el, "f_name")) safestrncpy(flbuf.f_name, migr_chardata, sizeof flbuf.f_name); - else if (!strcasecmp(el, "f_ref_count")) flbuf.f_ref_count = atoi(migr_chardata); - else if (!strcasecmp(el, "f_ep_expire_mode")) flbuf.f_ep.expire_mode = atoi(migr_chardata); - else if (!strcasecmp(el, "f_ep_expire_value")) flbuf.f_ep.expire_value = atoi(migr_chardata); + else if ((!strncasecmp(el, HKEY("f_"))) && + migr_floorrecord(data, el)) + ; /* Nothing to do anymore */ else if (!strcasecmp(el, "floor")) { CtdlPutFloor(&flbuf, floornum); - CtdlLogPrintf(CTDL_INFO, "Imported floor #%d (%s)\n", floornum, flbuf.f_name); + syslog(LOG_INFO, "migrate: imported floor #%d (%s)", floornum, flbuf.f_name); } /*** VISITS ***/ - - else if (!strcasecmp(el, "v_roomnum")) vbuf.v_roomnum = atol(migr_chardata); - else if (!strcasecmp(el, "v_roomgen")) vbuf.v_roomgen = atol(migr_chardata); - else if (!strcasecmp(el, "v_usernum")) vbuf.v_usernum = atol(migr_chardata); - - else if (!strcasecmp(el, "v_seen")) { - vbuf.v_lastseen = atol(migr_chardata); - is_textual_seen = 0; - for (i=0; migr_chardata[i]; ++i) if (!isdigit(migr_chardata[i])) is_textual_seen = 1; - if (is_textual_seen) safestrncpy(vbuf.v_seen, migr_chardata, sizeof vbuf.v_seen); + else if ((!strncasecmp(el, HKEY("v_"))) && migr_visitrecord(data, el)) { + ; /* Nothing to do anymore */ } - - else if (!strcasecmp(el, "v_answered")) safestrncpy(vbuf.v_answered, migr_chardata, sizeof vbuf.v_answered); - else if (!strcasecmp(el, "v_flags")) vbuf.v_flags = atoi(migr_chardata); - else if (!strcasecmp(el, "v_view")) vbuf.v_view = atoi(migr_chardata); - else if (!strcasecmp(el, "visit")) { put_visit(&vbuf); - CtdlLogPrintf(CTDL_INFO, "Imported visit: %ld/%ld/%ld\n", vbuf.v_roomnum, vbuf.v_roomgen, vbuf.v_usernum); + syslog(LOG_INFO, "migrate: imported visit: %ld/%ld/%ld", vbuf.v_roomnum, vbuf.v_roomgen, vbuf.v_usernum); } /*** MESSAGES ***/ - - else if (!strcasecmp(el, "msg_msgnum")) import_msgnum = atol(migr_chardata); - else if (!strcasecmp(el, "msg_meta_refcount")) smi.meta_refcount = atoi(migr_chardata); - else if (!strcasecmp(el, "msg_meta_content_type")) safestrncpy(smi.meta_content_type, migr_chardata, sizeof smi.meta_content_type); + + else if (!strcasecmp(el, "msg_msgnum")) smi.meta_msgnum = import_msgnum = atol(ChrPtr(migr_chardata)); + else if (!strcasecmp(el, "msg_meta_refcount")) smi.meta_refcount = atoi(ChrPtr(migr_chardata)); + else if (!strcasecmp(el, "msg_meta_rfc822_length")) smi.meta_rfc822_length = atoi(ChrPtr(migr_chardata)); + else if (!strcasecmp(el, "msg_meta_content_type")) safestrncpy(smi.meta_content_type, ChrPtr(migr_chardata), sizeof smi.meta_content_type); else if (!strcasecmp(el, "msg_text")) { - if (decoded_msg != NULL) { - free(decoded_msg); + long rc; + struct CtdlMessage *msg; + + FlushStrBuf(migr_MsgData); + StrBufDecodeBase64To(migr_chardata, migr_MsgData); + + msg = CtdlDeserializeMessage(import_msgnum, -1, + ChrPtr(migr_MsgData), + StrLength(migr_MsgData)); + if (msg != NULL) { + rc = CtdlSaveThisMessage(msg, import_msgnum, 0); + if (rc == 0) { + PutMetaData(&smi); + } + CM_Free(msg); + } + else { + rc = -1; } - len = strlen(migr_chardata); - decoded_msg = malloc(len); - msglen = CtdlDecodeBase64(decoded_msg, migr_chardata, len); - cdb_store(CDB_MSGMAIN, &import_msgnum, sizeof(long), decoded_msg, msglen); - free(decoded_msg); - decoded_msg = NULL; - PutMetaData(&smi); - CtdlLogPrintf(CTDL_INFO, "Imported message #%ld, size=%ld, refcount=%d, content-type: %s\n", - import_msgnum, msglen, smi.meta_refcount, smi.meta_content_type); + + syslog(LOG_INFO, + "migrate: %s message #%ld, size=%d, refcount=%d, bodylength=%ld, content-type: %s", + (rc!= 0)?"failed to import ":"Imported ", + import_msgnum, + StrLength(migr_MsgData), + smi.meta_refcount, + smi.meta_rfc822_length, + smi.meta_content_type + ); + memset(&smi, 0, sizeof(smi)); } /*** MORE GENERAL STUFF ***/ - if (migr_chardata != NULL) { - free(migr_chardata); - migr_chardata = NULL; - migr_chardata_len = 0; - } + FlushStrBuf(migr_chardata); } - - /* * Import begins here */ void migr_do_import(void) { - char buf[SIZ]; XML_Parser xp; - int linelen; + char buf[SIZ]; unbuffer_output(); - + migr_chardata = NewStrBufPlain(NULL, SIZ * 20); + migr_MsgData = NewStrBufPlain(NULL, SIZ * 20); xp = XML_ParserCreate(NULL); if (!xp) { cprintf("%d Failed to create XML parser instance\n", ERROR+INTERNAL_ERROR); @@ -898,58 +874,239 @@ void migr_do_import(void) { cprintf("%d sock it to me\n", SEND_LISTING); unbuffer_output(); - while (client_getln(buf, sizeof buf) >= 0 && strcmp(buf, "000")) { - linelen = strlen(buf); - strcpy(&buf[linelen++], "\n"); + client_set_inbound_buf(SIZ * 10); - if (CtdlThreadCheckStop()) + while (client_getln(buf, sizeof buf) >= 0 && strcmp(buf, "000")) { + XML_Parse(xp, buf, strlen(buf), 0); + if (server_shutting_down) break; // Should we break or return? - - if (buf[0] == '\0') - continue; - - XML_Parse(xp, buf, linelen, 0); } XML_Parse(xp, "", 0, 1); XML_ParserFree(xp); - + FreeStrBuf(&migr_chardata); + FreeStrBuf(&migr_MsgData); rebuild_euid_index(); rebuild_usersbynumber(); + CtdlSetConfigInt("MM_fulltext_wordbreaker", -1); // Set an invalid wordbreaker to force re-indexing CC->dont_term = 0; } - +/****************************************************************************** + * Dispatcher, Common code * + ******************************************************************************/ /* * Dump out the pathnames of directories which can be copied "as is" */ void migr_do_listdirs(void) { cprintf("%d Don't forget these:\n", LISTING_FOLLOWS); - cprintf("bio|%s\n", ctdl_bio_dir); cprintf("files|%s\n", ctdl_file_dir); - cprintf("userpics|%s\n", ctdl_usrpic_dir); cprintf("messages|%s\n", ctdl_message_dir); - cprintf("netconfigs|%s\n", ctdl_netcfg_dir); cprintf("keys|%s\n", ctdl_key_dir); - cprintf("images|%s\n", ctdl_image_dir); - cprintf("info|%s\n", ctdl_info_dir); cprintf("000\n"); } -/* - * Common code appears in this section - */ +/****************************************************************************** + * Repair database integrity * + ******************************************************************************/ + +StrBuf *PlainMessageBuf = NULL; +HashList *UsedMessageIDS = NULL; + +int migr_restore_message_metadata(long msgnum, int refcount) { + struct MetaData smi; + struct CtdlMessage *msg; + char *mptr = NULL; + + /* We can use a static buffer here because there will never be more than + * one of this operation happening at any given time, and it's really best + * to just keep it allocated once instead of torturing malloc/free. + * Call this function with msgnum "-1" to free the buffer when finished. + */ + static int encoded_alloc = 0; + static char *encoded_msg = NULL; + + if (msgnum < 0) { + if ((encoded_alloc == 0) && (encoded_msg != NULL)) { + free(encoded_msg); + encoded_alloc = 0; + encoded_msg = NULL; + // todo FreeStrBuf(&PlainMessageBuf); PlainMessageBuf = NULL; + } + return 0; + } + + if (PlainMessageBuf == NULL) { + PlainMessageBuf = NewStrBufPlain(NULL, 10*SIZ); + } + + /* Ok, here we go ... */ + + msg = CtdlFetchMessage(msgnum, 1); + if (msg == NULL) { + return 1; + } + + GetMetaData(&smi, msgnum); + smi.meta_msgnum = msgnum; + smi.meta_refcount = refcount; + + /* restore the content type from the message body: */ + mptr = bmstrcasestr(msg->cm_fields[eMesageText], "Content-type:"); + if (mptr != NULL) { + char *aptr; + safestrncpy(smi.meta_content_type, &mptr[13], sizeof smi.meta_content_type); + striplt(smi.meta_content_type); + aptr = smi.meta_content_type; + while (!IsEmptyStr(aptr)) { + if ((*aptr == ';') + || (*aptr == ' ') + || (*aptr == 13) + || (*aptr == 10)) { + memset(aptr, 0, sizeof(smi.meta_content_type) - (aptr - smi.meta_content_type)); + } + else aptr++; + } + } + + CC->redirect_buffer = PlainMessageBuf; + CtdlOutputPreLoadedMsg(msg, MT_RFC822, HEADERS_ALL, 0, 1, QP_EADDR); + smi.meta_rfc822_length = StrLength(CC->redirect_buffer); + CC->redirect_buffer = NULL; + + syslog(LOG_INFO, + "migrate: setting message #%ld meta data to: refcount=%d, bodylength=%ld, content-type: %s", + smi.meta_msgnum, + smi.meta_refcount, + smi.meta_rfc822_length, + smi.meta_content_type + ); + + PutMetaData(&smi); + CM_Free(msg); + return 0; +} +void migr_check_room_msg(long msgnum, void *userdata) { + fprintf(migr_global_message_list, "%ld %s\n", msgnum, CC->room.QRname); +} + + +void migr_check_rooms_backend(struct ctdlroom *buf, void *data) { + /* message list goes inside this tag */ + CtdlGetRoom(&CC->room, buf->QRname); + CtdlForEachMessage(MSGS_ALL, 0L, NULL, NULL, NULL, migr_check_room_msg, NULL); +} + + +void RemoveMessagesFromRooms(StrBuf *RoomNameVec, long msgnum) { + struct MetaData smi; + const char *Pos = NULL; + StrBuf *oneRoom = NewStrBuf(); + + syslog(LOG_INFO, "migrate: removing message pointer %ld from these rooms: %s", msgnum, ChrPtr(RoomNameVec)); + + while (Pos != StrBufNOTNULL){ + StrBufExtract_NextToken(oneRoom, RoomNameVec, &Pos, '|'); + CtdlDeleteMessages(ChrPtr(oneRoom), &msgnum, 1, ""); + }; + GetMetaData(&smi, msgnum); + AdjRefCount(msgnum, -smi.meta_refcount); +} + + +void migr_do_restore_meta(void) { + char buf[SIZ]; + int failGetMessage; + long msgnum = 0; + int lastnum = 0; + int refcount = 0; + CitContext *Ctx; + char *prn; + StrBuf *RoomNames; + char cmd[SIZ]; + + migr_global_message_list = fopen(migr_tempfilename1, "w"); + if (migr_global_message_list != NULL) { + CtdlForEachRoom(migr_check_rooms_backend, NULL); + fclose(migr_global_message_list); + } + + /* + * Process the 'global' message list. (Sort it and remove dups. + * Dups are ok because a message may be in more than one room, but + * this will be handled by exporting the reference count, not by + * exporting the message multiple times.) + */ + snprintf(cmd, sizeof cmd, "sort -n <%s >%s", migr_tempfilename1, migr_tempfilename2); + if (system(cmd) != 0) { + syslog(LOG_ERR, "migrate: error %d", errno); + } + + RoomNames = NewStrBuf(); + Ctx = CC; + migr_global_message_list = fopen(migr_tempfilename2, "r"); + if (migr_global_message_list != NULL) { + syslog(LOG_INFO, "migrate: opened %s", migr_tempfilename1); + while ((Ctx->kill_me == 0) && + (fgets(buf, sizeof(buf), migr_global_message_list) != NULL)) { + msgnum = atol(buf); + if (msgnum == 0L) + continue; + if (lastnum == 0) { + lastnum = msgnum; + } + prn = strchr(buf, ' '); + if (lastnum != msgnum) { + failGetMessage = migr_restore_message_metadata(lastnum, refcount); + if (failGetMessage) { + RemoveMessagesFromRooms(RoomNames, lastnum); + } + refcount = 1; + lastnum = msgnum; + if (prn != NULL) { + StrBufPlain(RoomNames, prn + 1, -1); + } + StrBufTrim(RoomNames); + } + else { + if (prn != NULL) { + if (StrLength(RoomNames) > 0) { + StrBufAppendBufPlain(RoomNames, HKEY("|"), 0); + } + StrBufAppendBufPlain(RoomNames, prn, -1, 1); + StrBufTrim(RoomNames); + } + refcount ++; + } + lastnum = msgnum; + } + failGetMessage = migr_restore_message_metadata(msgnum, refcount); + if (failGetMessage) { + RemoveMessagesFromRooms(RoomNames, lastnum); + } + fclose(migr_global_message_list); + } + + migr_restore_message_metadata(-1L, -1); /* This frees the encoding buffer */ + cprintf("%d system analysis completed", CIT_OK); + Ctx->kill_me = 1; +} + + +/****************************************************************************** + * Dispatcher, Common code * + ******************************************************************************/ void cmd_migr(char *cmdbuf) { char cmd[32]; if (CtdlAccessCheck(ac_internal)) return; - if (CtdlTrySingleUser()) - { + if (CtdlTrySingleUser()) { + CtdlDisableHouseKeeping(); CtdlMakeTempFileName(migr_tempfilename1, sizeof migr_tempfilename1); CtdlMakeTempFileName(migr_tempfilename2, sizeof migr_tempfilename2); @@ -963,30 +1120,36 @@ void cmd_migr(char *cmdbuf) { else if (!strcasecmp(cmd, "listdirs")) { migr_do_listdirs(); } + else if (!strcasecmp(cmd, "restoremeta")) { + migr_do_restore_meta(); + } else { cprintf("%d illegal command\n", ERROR + ILLEGAL_VALUE); } unlink(migr_tempfilename1); unlink(migr_tempfilename2); - + + CtdlEnableHouseKeeping(); CtdlEndSingleUser(); } - else - { + else { cprintf("%d The migrator is already running.\n", ERROR + RESOURCE_BUSY); } } +/****************************************************************************** + * Module Hook * + ******************************************************************************/ + CTDL_MODULE_INIT(migrate) { if (!threading) { CtdlRegisterProtoHook(cmd_migr, "MIGR", "Across-the-wire migration"); - CtdlRegisterProtoHook(cmd_migr, "ARTV", "Across-the-wire migration (legacy syntax)"); } - /* return our Subversion id for the Log */ - return "$Id$"; + /* return our module name for the log */ + return "migrate"; }