X-Git-Url: https://code.citadel.org/?a=blobdiff_plain;f=citadel%2Fmodules%2Fmigrate%2Fserv_migrate.c;h=7cc8fc041fde63275598c34fd4c85e202cd5bd09;hb=79d6eafd2a8bb379b3e13ff071a000d9bf36771b;hp=f2f95c0bd437d8a5c52b9dd3c5d505d3fbd56fc3;hpb=fa2dd842abb5feedea3e2253255722fcaecc3c6f;p=citadel.git diff --git a/citadel/modules/migrate/serv_migrate.c b/citadel/modules/migrate/serv_migrate.c index f2f95c0bd..7cc8fc041 100644 --- a/citadel/modules/migrate/serv_migrate.c +++ b/citadel/modules/migrate/serv_migrate.c @@ -204,10 +204,13 @@ void migr_export_rooms(void) { * exporting the message multiple times.) */ snprintf(cmd, sizeof cmd, "sort -n <%s >%s", migr_tempfilename1, migr_tempfilename2); - if (system(cmd) != 0) syslog(LOG_ERR, "Error %d", errno); + if (system(cmd) != 0) { + syslog(LOG_ERR, "migrate: error %d", errno); + } snprintf(cmd, sizeof cmd, "uniq <%s >%s", migr_tempfilename2, migr_tempfilename1); - if (system(cmd) != 0) syslog(LOG_ERR, "Error %d", errno); - + if (system(cmd) != 0) { + syslog(LOG_ERR, "migrate: error %d", errno); + } snprintf(cmd, sizeof cmd, "wc -l %s", migr_tempfilename1); FILE *fp = popen(cmd, "r"); @@ -219,7 +222,7 @@ void migr_export_rooms(void) { else { total_msgs = 1; // any nonzero just to keep it from barfing } - syslog(LOG_DEBUG, "Total messages to be exported: %d", total_msgs); + syslog(LOG_DEBUG, "migrate: total messages to be exported: %d", total_msgs); } @@ -306,12 +309,14 @@ void migr_export_message(long msgnum) { struct MetaData smi; struct CtdlMessage *msg; struct ser_ret smr; + long bytes_written = 0; + long this_block = 0; + + // We can use a static buffer here because there will never be more than + // one of this operation happening at any given time, and it's really best + // to just keep it allocated once instead of torturing malloc/free. + // Call this function with msgnum "-1" to free the buffer when finished. - /* We can use a static buffer here because there will never be more than - * one of this operation happening at any given time, and it's really best - * to just keep it allocated once instead of torturing malloc/free. - * Call this function with msgnum "-1" to free the buffer when finished. - */ static int encoded_alloc = 0; static char *encoded_msg = NULL; @@ -324,10 +329,10 @@ void migr_export_message(long msgnum) { return; } - /* Ok, here we go ... */ + // Ok, here we go ... msg = CtdlFetchMessage(msgnum, 1); - if (msg == NULL) return; /* fail silently */ + if (msg == NULL) return; // fail silently client_write(HKEY("\n")); GetMetaData(&smi, msgnum); @@ -340,7 +345,7 @@ void migr_export_message(long msgnum) { CtdlSerializeMessage(&smr, msg); CM_Free(msg); - /* Predict the buffer size we need. Expand the buffer if necessary. */ + // Predict the buffer size we need. Expand the buffer if necessary. int encoded_len = smr.len * 15 / 10 ; if (encoded_len > encoded_alloc) { encoded_alloc = encoded_len; @@ -348,13 +353,24 @@ void migr_export_message(long msgnum) { } if (encoded_msg == NULL) { - /* Questionable hack that hopes it'll work next time and we only lose one message */ + // Questionable hack that hopes it'll work next time and we only lose one message encoded_alloc = 0; } else { - /* Once we do the encoding we know the exact size */ + // Once we do the encoding we know the exact size encoded_len = CtdlEncodeBase64(encoded_msg, (char *)smr.ser, smr.len, 1); - client_write(encoded_msg, encoded_len); + + // Careful now. If the message is gargantuan, trying to write multiple gigamegs in one + // big write operation can make our transport unhappy. So we'll chunk it up 10 KB at a time. + bytes_written = 0; + while ( (bytes_written < encoded_len) && (!server_shutting_down) ) { + this_block = encoded_len - bytes_written; + if (this_block > 10240) { + this_block = 10240; + } + client_write(&encoded_msg[bytes_written], this_block); + bytes_written += this_block; + } } free(smr.ser); @@ -420,7 +436,7 @@ void migr_export_messages(void) { Ctx = CC; migr_global_message_list = fopen(migr_tempfilename1, "r"); if (migr_global_message_list != NULL) { - syslog(LOG_INFO, "Opened %s", migr_tempfilename1); + syslog(LOG_INFO, "migrate: opened %s", migr_tempfilename1); while ((Ctx->kill_me == 0) && (fgets(buf, sizeof(buf), migr_global_message_list) != NULL)) { msgnum = atol(buf); @@ -437,10 +453,10 @@ void migr_export_messages(void) { fclose(migr_global_message_list); } if (Ctx->kill_me == 0) { - syslog(LOG_INFO, "Exported %d messages.", count); + syslog(LOG_INFO, "migrate: exported %d messages.", count); } else { - syslog(LOG_ERR, "Export aborted due to client disconnect!"); + syslog(LOG_ERR, "migrate: export aborted due to client disconnect!"); } migr_export_message(-1L); /* This frees the encoding buffer */ @@ -508,12 +524,6 @@ long import_msgnum = 0; * This callback stores up the data which appears in between tags. */ void migr_xml_chardata(void *data, const XML_Char *s, int len) { - - char aaa[65536]; - memcpy(aaa, s, len); - aaa[len] = 0; - syslog(LOG_DEBUG, "chardata <\033[35m%s\033[0m>", aaa); - StrBufAppendBufPlain(migr_chardata, s, len, 0); } @@ -542,7 +552,7 @@ void migr_xml_start(void *data, const char *el, const char **attr) { } if (citadel_migrate_data != 1) { - syslog(LOG_ERR, "Out-of-sequence tag <%s> detected. Warning: ODD-DATA!", el); + syslog(LOG_ERR, "migrate: out-of-sequence tag <%s> detected. Warning: ODD-DATA!", el); return; } @@ -674,7 +684,7 @@ void migr_xml_end(void *data, const char *el) { } if (citadel_migrate_data != 1) { - syslog(LOG_ERR, "Out-of-sequence tag <%s> detected. Warning: ODD-DATA!", el); + syslog(LOG_ERR, "migrate: out-of-sequence tag <%s> detected. Warning: ODD-DATA!", el); return; } @@ -683,7 +693,7 @@ void migr_xml_end(void *data, const char *el) { /*** CONFIG ***/ if (!strcasecmp(el, "config")) { - syslog(LOG_DEBUG, "Imported config key=%s", ikey); + syslog(LOG_DEBUG, "migrate: imported config key=%s", ikey); if (ikey != NULL) { CtdlSetConfigStr(ikey, (char *)ChrPtr(migr_chardata)); @@ -691,7 +701,7 @@ void migr_xml_end(void *data, const char *el) { ikey = NULL; } else { - syslog(LOG_INFO, "Closed a tag but no key name was supplied."); + syslog(LOG_INFO, "migrate: closed a tag but no key name was supplied."); } } @@ -701,7 +711,7 @@ void migr_xml_end(void *data, const char *el) { ; /* Nothing to do anymore */ else if (!strcasecmp(el, "user")) { CtdlPutUser(&usbuf); - syslog(LOG_INFO, "Imported user: %s", usbuf.fullname); + syslog(LOG_INFO, "migrate: imported user: %s", usbuf.fullname); } /*** OPENID ***/ @@ -718,7 +728,7 @@ void migr_xml_end(void *data, const char *el) { memcpy(&oid_data[sizeof(long)], openid_url, strlen(openid_url) + 1); cdb_store(CDB_EXTAUTH, openid_url, strlen(openid_url), oid_data, oid_data_len); free(oid_data); - syslog(LOG_INFO, "Imported OpenID: %s (%ld)", openid_url, openid_usernum); + syslog(LOG_INFO, "migrate: imported OpenID: %s (%ld)", openid_url, openid_usernum); } /*** ROOM ***/ @@ -727,7 +737,7 @@ void migr_xml_end(void *data, const char *el) { ; /* Nothing to do anymore */ else if (!strcasecmp(el, "room")) { CtdlPutRoom(&qrbuf); - syslog(LOG_INFO, "Imported room: %s", qrbuf.QRname); + syslog(LOG_INFO, "migrate: imported room: %s", qrbuf.QRname); } /*** ROOM MESSAGE POINTERS ***/ @@ -742,10 +752,9 @@ void migr_xml_end(void *data, const char *el) { msglist_alloc = 1000; msglist = malloc(sizeof(long) * msglist_alloc); - syslog(LOG_DEBUG, "Message list for: %s", FRname); + syslog(LOG_DEBUG, "migrate: message list for: %s", FRname); ptr = ChrPtr(migr_chardata); - syslog(LOG_DEBUG, "\033[31m%s\033[0m", ptr); while (*ptr != 0) { while ((*ptr != 0) && (!isdigit(*ptr))) { ++ptr; @@ -758,7 +767,6 @@ void migr_xml_end(void *data, const char *el) { msglist = realloc(msglist, sizeof(long) * msglist_alloc); } msglist[msgcount++] = msgnum; - syslog(LOG_DEBUG, "\033[32mmsgnum %ld , count is now %d\033[0m", msgnum, msgcount); } } while ((*ptr != 0) && (isdigit(*ptr))) { @@ -772,7 +780,7 @@ void migr_xml_end(void *data, const char *el) { free(msglist); msglist = NULL; msglist_alloc = 0; - syslog(LOG_DEBUG, "Imported %d messages.", msgcount); + syslog(LOG_DEBUG, "migrate: imported %d messages.", msgcount); if (server_shutting_down) { return; } @@ -785,7 +793,7 @@ void migr_xml_end(void *data, const char *el) { else if (!strcasecmp(el, "floor")) { CtdlPutFloor(&flbuf, floornum); - syslog(LOG_INFO, "Imported floor #%d (%s)", floornum, flbuf.f_name); + syslog(LOG_INFO, "migrate: imported floor #%d (%s)", floornum, flbuf.f_name); } /*** VISITS ***/ @@ -794,7 +802,7 @@ void migr_xml_end(void *data, const char *el) { } else if (!strcasecmp(el, "visit")) { put_visit(&vbuf); - syslog(LOG_INFO, "Imported visit: %ld/%ld/%ld", vbuf.v_roomnum, vbuf.v_roomgen, vbuf.v_usernum); + syslog(LOG_INFO, "migrate: imported visit: %ld/%ld/%ld", vbuf.v_roomnum, vbuf.v_roomgen, vbuf.v_usernum); } /*** MESSAGES ***/ @@ -826,7 +834,7 @@ void migr_xml_end(void *data, const char *el) { } syslog(LOG_INFO, - "%s message #%ld, size=%d, refcount=%d, bodylength=%ld, content-type: %s", + "migrate: %s message #%ld, size=%d, refcount=%d, bodylength=%ld, content-type: %s", (rc!= 0)?"failed to import ":"Imported ", import_msgnum, StrLength(migr_MsgData), @@ -969,7 +977,7 @@ int migr_restore_message_metadata(long msgnum, int refcount) { CC->redirect_buffer = NULL; syslog(LOG_INFO, - "Setting message #%ld meta data to: refcount=%d, bodylength=%ld, content-type: %s", + "migrate: setting message #%ld meta data to: refcount=%d, bodylength=%ld, content-type: %s", smi.meta_msgnum, smi.meta_refcount, smi.meta_rfc822_length, @@ -999,7 +1007,7 @@ void RemoveMessagesFromRooms(StrBuf *RoomNameVec, long msgnum) { const char *Pos = NULL; StrBuf *oneRoom = NewStrBuf(); - syslog(LOG_INFO, "removing message pointer %ld from these rooms: %s", msgnum, ChrPtr(RoomNameVec)); + syslog(LOG_INFO, "migrate: removing message pointer %ld from these rooms: %s", msgnum, ChrPtr(RoomNameVec)); while (Pos != StrBufNOTNULL){ StrBufExtract_NextToken(oneRoom, RoomNameVec, &Pos, '|'); @@ -1034,13 +1042,15 @@ void migr_do_restore_meta(void) { * exporting the message multiple times.) */ snprintf(cmd, sizeof cmd, "sort -n <%s >%s", migr_tempfilename1, migr_tempfilename2); - if (system(cmd) != 0) syslog(LOG_ERR, "Error %d", errno); + if (system(cmd) != 0) { + syslog(LOG_ERR, "migrate: error %d", errno); + } RoomNames = NewStrBuf(); Ctx = CC; migr_global_message_list = fopen(migr_tempfilename2, "r"); if (migr_global_message_list != NULL) { - syslog(LOG_INFO, "Opened %s", migr_tempfilename1); + syslog(LOG_INFO, "migrate: opened %s", migr_tempfilename1); while ((Ctx->kill_me == 0) && (fgets(buf, sizeof(buf), migr_global_message_list) != NULL)) { msgnum = atol(buf);