From 0fcfc4d81b4def0bd0f7562465b5bf7ef4b7857b Mon Sep 17 00:00:00 2001 From: Art Cancro Date: Wed, 3 Mar 2021 20:19:04 -0500 Subject: [PATCH] ctdlmigrate now uses a direct socket connection to the local server instead of sendcommand. Also removed the use of buffered I/O from serv_migrate.c because it was locking up. --- citadel/ctdlmigrate.c | 269 +++++++++++++++---------- citadel/modules/migrate/serv_migrate.c | 25 +-- citadel/sendcommand.c | 18 +- 3 files changed, 176 insertions(+), 136 deletions(-) diff --git a/citadel/ctdlmigrate.c b/citadel/ctdlmigrate.c index 21dad4b24..7dbf5d389 100644 --- a/citadel/ctdlmigrate.c +++ b/citadel/ctdlmigrate.c @@ -35,6 +35,8 @@ #include #include #include +#include +#include #include #include "citadel.h" #include "axdefs.h" @@ -63,13 +65,103 @@ void getz(char *buf) { } +int uds_connectsock(char *sockpath) { + int s; + struct sockaddr_un addr; + memset(&addr, 0, sizeof(addr)); + addr.sun_family = AF_UNIX; + strncpy(addr.sun_path, sockpath, sizeof addr.sun_path); + + s = socket(AF_UNIX, SOCK_STREAM, 0); + if (s < 0) { + fprintf(stderr, "sendcommand: Can't create socket: %s\n", strerror(errno)); + exit(3); + } + + if (connect(s, (struct sockaddr *) &addr, sizeof(addr)) < 0) { + fprintf(stderr, "sendcommand: can't connect: %s\n", strerror(errno)); + close(s); + exit(3); + } + + return s; +} + + +/* + * input binary data from socket + */ +void serv_read(int serv_sock, char *buf, int bytes) { + int len, rlen; + + len = 0; + while (len < bytes) { + rlen = read(serv_sock, &buf[len], bytes - len); + if (rlen < 1) { + return; + } + len = len + rlen; + } +} + + +/* + * send binary to server + */ +void serv_write(int serv_sock, char *buf, int nbytes) { + int bytes_written = 0; + int retval; + while (bytes_written < nbytes) { + retval = write(serv_sock, &buf[bytes_written], nbytes - bytes_written); + if (retval < 1) { + return; + } + bytes_written = bytes_written + retval; + } +} + + +/* + * input string from socket - implemented in terms of serv_read() + */ +void serv_gets(int serv_sock, char *buf) { + int i; + + /* Read one character at a time. + */ + for (i = 0;; i++) { + serv_read(serv_sock, &buf[i], 1); + if (buf[i] == '\n' || i == (SIZ-1)) + break; + } + + /* If we got a long line, discard characters until the newline. + */ + if (i == (SIZ-1)) { + while (buf[i] != '\n') { + serv_read(serv_sock, &buf[i], 1); + } + } + + /* Strip all trailing nonprintables (crlf) + */ + buf[i] = 0; +} + + +/* + * send line to server - implemented in terms of serv_write() + */ +void serv_puts(int serv_sock, char *buf) { + serv_write(serv_sock, buf, strlen(buf)); + serv_write(serv_sock, "\n", 1); +} int main(int argc, char *argv[]) { char ctdldir[PATH_MAX]=CTDLDIR; char yesno[5]; - char sendcommand[PATH_MAX]; int cmdexit; char cmd[PATH_MAX]; char buf[PATH_MAX]; @@ -78,18 +170,27 @@ int main(int argc, char *argv[]) { char remote_host[SIZ]; char remote_sendcommand[PATH_MAX]; FILE *sourcefp = NULL; - FILE *targetfp = NULL; int linecount = 0; - char spinning[4] = "-\\|/" ; - int exitcode = 0; - - CtdlMakeTempFileName(socket_path, sizeof socket_path); + int a; + int local_admin_socket = (-1); + + /* Parse command line */ + while ((a = getopt(argc, argv, "h:")) != EOF) { + switch (a) { + case 'h': + strncpy(ctdldir, optarg, sizeof ctdldir); + break; + default: + fprintf(stderr, "sendcommand: usage: ctdlmigrate [-h server_dir]\n"); + return(1); + } + } + if (chdir(ctdldir) != 0) { fprintf(stderr, "sendcommand: %s: %s\n", ctdldir, strerror(errno)); exit(errno); } - printf( "\033[2J\033[H\n" "-------------------------------------------\n" "Over-the-wire migration utility for Citadel\n" @@ -104,11 +205,12 @@ int main(int argc, char *argv[]) { "the 'rsync' utility, and OpenSSH v4 or newer.\n" "\n" "You must run this utility on the TARGET SYSTEM. Any existing data\n" - "on this system will be ERASED.\n" + "on this system will be ERASED. Your target system will be on this\n" + "host in %s.\n" "\n" "Do you wish to continue? " , - EXPORT_REV_MIN + EXPORT_REV_MIN, ctdldir ); if ((fgets(yesno, sizeof yesno, stdin) == NULL) || (tolower(yesno[0]) != 'y')) { @@ -118,15 +220,22 @@ int main(int argc, char *argv[]) { printf("\n\nGreat! First we will check some things out here on our target\n" "system to make sure it is ready to receive data.\n\n"); - printf("Locating 'sendcommand' and checking connectivity to Citadel...\n"); - snprintf(sendcommand, sizeof sendcommand, "%s/sendcommand", ctdl_utilbin_dir); - snprintf(cmd, sizeof cmd, "%s 'NOOP'", sendcommand); - cmdexit = system(cmd); - if (cmdexit != 0) { - printf("\nctdlmigrate was unable to attach to the Citadel server\n" - "here on the target system. Is Citadel running?\n\n"); + printf("Checking connectivity to Citadel in %s...\n", ctdldir); + local_admin_socket = uds_connectsock("citadel-admin.socket"); + + + serv_gets(local_admin_socket, buf); + puts(buf); + if (buf[0] != '2') { exit(1); } + serv_puts(local_admin_socket, "ECHO Connection to Citadel Server succeeded."); + serv_gets(local_admin_socket, buf); + puts(buf); + if (buf[0] != '2') { + exit(1); + } + printf("\nOK, this side is ready to go. Now we must connect to the source system.\n\n"); printf("Enter the host name or IP address of the source system\n" @@ -134,23 +243,23 @@ int main(int argc, char *argv[]) { "--> "); getz(remote_host); -get_remote_user: - printf("\nEnter the name of a user on %s who has full access to Citadel files\n" - "(usually root)\n--> ", - remote_host); - getz(remote_user); - if (IsEmptyStr(remote_user)) - goto get_remote_user; + while (IsEmptyStr(remote_user)) { + printf("\nEnter the name of a user on %s who has full access to Citadel files\n" + "(usually root)\n--> ", + remote_host); + getz(remote_user); + } printf("\nEstablishing an SSH connection to the source system...\n\n"); + sprintf(socket_path, "/tmp/ctdlmigrate.XXXXXX"); + mktemp(socket_path); unlink(socket_path); snprintf(cmd, sizeof cmd, "ssh -MNf -S %s %s@%s", socket_path, remote_user, remote_host); cmdexit = system(cmd); printf("\n"); if (cmdexit != 0) { printf("This program was unable to establish an SSH session to the source system.\n\n"); - exitcode = cmdexit; - goto THEEND; + exit(cmdexit); } printf("\nTesting a command over the connection...\n\n"); @@ -160,8 +269,7 @@ get_remote_user: printf("\n"); if (cmdexit != 0) { printf("Remote commands are not succeeding.\n\n"); - exitcode = cmdexit; - goto THEEND; + exit(cmdexit); } printf("\nLocating the remote 'sendcommand' and Citadel installation...\n"); @@ -189,100 +297,51 @@ get_remote_user: printf("\n"); if (cmdexit != 0) { printf("ctdlmigrate was unable to attach to the remote Citadel system.\n\n"); - exitcode = cmdexit; - goto THEEND; + exit(cmdexit); } - printf("ctdlmigrate will now begin a database migration...\n"); - printf(" if the system doesn't start working, \n"); - printf(" have a look at the syslog for pending jobs needing to be terminated.\n"); + printf("\033[2J\n"); + printf("\033[2;0H\033[33mMigrating from %s\033[0m\n\n", remote_host); - snprintf(cmd, sizeof cmd, "ssh -S %s %s@%s %s -w3600 MIGR export", - socket_path, remote_user, remote_host, remote_sendcommand); + snprintf(cmd, sizeof cmd, "ssh -S %s %s@%s %s -w3600 MIGR export", socket_path, remote_user, remote_host, remote_sendcommand); sourcefp = popen(cmd, "r"); if (!sourcefp) { printf("\n%s\n\n", strerror(errno)); - exitcode = 2; - goto THEEND; + exit(2); } - snprintf(cmd, sizeof cmd, "%s -w3600 MIGR import", sendcommand); - targetfp = popen(cmd, "w"); - if (!targetfp) { - printf("\n%s\n\n", strerror(errno)); - exitcode = 3; - goto THEEND; + serv_puts(local_admin_socket, "MIGR import"); + serv_gets(local_admin_socket, buf); + if (buf[0] != '4') { + printf("\n%s\n", buf); + exit(3); } - while (fgets(buf, sizeof buf, sourcefp) != NULL) { - if (fwrite(buf, strlen(buf), 1, targetfp) < 1) { - exitcode = 4; - printf("%s\n", strerror(errno)); - goto FAIL; - } + char *ptr; + time_t time_started = time(NULL); + time_t last_update = time(NULL); + while (ptr = fgets(buf, SIZ, sourcefp), (ptr != NULL)) { + ptr = strchr(buf, '\n'); + if (ptr) *ptr = 0; ++linecount; - if ((linecount % 100) == 0) { - printf("%c\r", spinning[((linecount / 100) % 4)]); - fflush(stdout); + if (!strncasecmp(buf, "", 10)) { + printf("\033[11;0HPercent complete: \033[32m%d\033[0m\n", atoi(&buf[10])); } + if (time(NULL) != last_update) { + last_update = time(NULL); + printf("\033[10;0H Lines received: \033[32m%d\033[0m\n", linecount); + } + serv_puts(local_admin_socket, buf); } -FAIL: if (sourcefp) pclose(sourcefp); - if (targetfp) pclose(targetfp); - if (exitcode != 0) goto THEEND; - - /* We need to copy a bunch of other stuff, and will do so using rsync */ + serv_puts(local_admin_socket, "000"); - snprintf(cmd, sizeof cmd, "ssh -S %s %s@%s %s MIGR listdirs", - socket_path, remote_user, remote_host, remote_sendcommand); - sourcefp = popen(cmd, "r"); - if (!sourcefp) { - printf("\n%s\n\n", strerror(errno)); - exitcode = 2; - goto THEEND; - } - while ((fgets(buf, sizeof buf, sourcefp)) && (strcmp(buf, "000"))) { - buf[strlen(buf)-1] = 0; + // FIXME restart the local server now - if (!strncasecmp(buf, "files|", 6)) { - snprintf(cmd, sizeof cmd, "rsync -va --rsh='ssh -S %s' %s@%s:%s/ %s/", - socket_path, remote_user, remote_host, &buf[6], ctdl_file_dir); - } - else if (!strncasecmp(buf, "messages|", 9)) { - snprintf(cmd, sizeof cmd, "rsync -va --rsh='ssh -S %s' %s@%s:%s/ %s/", - socket_path, remote_user, remote_host, &buf[9], ctdl_message_dir); - } - else if (!strncasecmp(buf, "keys|", 5)) { - snprintf(cmd, sizeof cmd, "rsync -va --rsh='ssh -S %s' %s@%s:%s/ %s/", - socket_path, remote_user, remote_host, &buf[5], ctdl_key_dir); - } - else { - strcpy(cmd, "false"); /* cheap and sleazy way to throw an error */ - } - printf("%s\n", cmd); - cmdexit = system(cmd); - if (cmdexit != 0) { - exitcode += cmdexit; - } - } pclose(sourcefp); - -THEEND: if (exitcode == 0) { - printf("\n\n *** Citadel migration was successful! *** \n\n"); - } - else { - printf("\n\n *** Citadel migration was unsuccessful. *** \n\n"); - } - printf("\nShutting down the socket connection...\n\n"); - snprintf(cmd, sizeof cmd, "ssh -S %s -N -O exit %s@%s", - socket_path, remote_user, remote_host); - cmdexit = system(cmd); - printf("\n"); - if (cmdexit != 0) { - printf("There was an error shutting down the socket.\n\n"); - exitcode = cmdexit; - } - + printf("\nShutting down the socket connection...\n\n"); + snprintf(cmd, sizeof cmd, "ssh -S %s -N -O exit %s@%s", socket_path, remote_user, remote_host); + system(cmd); unlink(socket_path); - exit(exitcode); + exit(0); } diff --git a/citadel/modules/migrate/serv_migrate.c b/citadel/modules/migrate/serv_migrate.c index 99377ee91..aad6615f5 100644 --- a/citadel/modules/migrate/serv_migrate.c +++ b/citadel/modules/migrate/serv_migrate.c @@ -834,14 +834,12 @@ void migr_xml_end(void *data, const char *el) * Import begins here */ void migr_do_import(void) { - StrBuf *Buf; XML_Parser xp; - int Finished = 0; + char buf[SIZ]; unbuffer_output(); migr_chardata = NewStrBufPlain(NULL, SIZ * 20); migr_MsgData = NewStrBufPlain(NULL, SIZ * 20); - Buf = NewStrBufPlain(NULL, SIZ); xp = XML_ParserCreate(NULL); if (!xp) { cprintf("%d Failed to create XML parser instance\n", ERROR+INTERNAL_ERROR); @@ -857,24 +855,14 @@ void migr_do_import(void) { client_set_inbound_buf(SIZ * 10); - while (!Finished && client_read_random_blob(Buf, -1) >= 0) { - if ((StrLength(Buf) > 4) && !strcmp(ChrPtr(Buf) + StrLength(Buf) - 4, "000\n")) { - Finished = 1; - StrBufCutAt(Buf, StrLength(Buf) - 4, NULL); - } + while (client_getln(buf, sizeof buf) >= 0 && strcmp(buf, "000")) { + XML_Parse(xp, buf, strlen(buf), 0); if (server_shutting_down) break; // Should we break or return? - - if (StrLength(Buf) == 0) - continue; - - XML_Parse(xp, ChrPtr(Buf), StrLength(Buf), 0); - FlushStrBuf(Buf); } XML_Parse(xp, "", 0, 1); XML_ParserFree(xp); - FreeStrBuf(&Buf); FreeStrBuf(&migr_chardata); FreeStrBuf(&migr_MsgData); rebuild_euid_index(); @@ -908,7 +896,6 @@ HashList *UsedMessageIDS = NULL; int migr_restore_message_metadata(long msgnum, int refcount) { - CitContext *CCC = MyContext(); struct MetaData smi; struct CtdlMessage *msg; char *mptr = NULL; @@ -964,10 +951,10 @@ int migr_restore_message_metadata(long msgnum, int refcount) } } - CCC->redirect_buffer = PlainMessageBuf; + CC->redirect_buffer = PlainMessageBuf; CtdlOutputPreLoadedMsg(msg, MT_RFC822, HEADERS_ALL, 0, 1, QP_EADDR); - smi.meta_rfc822_length = StrLength(CCC->redirect_buffer); - CCC->redirect_buffer = NULL; + smi.meta_rfc822_length = StrLength(CC->redirect_buffer); + CC->redirect_buffer = NULL; syslog(LOG_INFO, "Setting message #%ld meta data to: refcount=%d, bodylength=%ld, content-type: %s", diff --git a/citadel/sendcommand.c b/citadel/sendcommand.c index caca87fd9..bd87ff3b4 100644 --- a/citadel/sendcommand.c +++ b/citadel/sendcommand.c @@ -32,8 +32,7 @@ int serv_sock = (-1); -int uds_connectsock(char *sockpath) -{ +int uds_connectsock(char *sockpath) { int s; struct sockaddr_un addr; @@ -60,8 +59,7 @@ int uds_connectsock(char *sockpath) /* * input binary data from socket */ -void serv_read(char *buf, int bytes) -{ +void serv_read(char *buf, int bytes) { int len, rlen; len = 0; @@ -78,8 +76,7 @@ void serv_read(char *buf, int bytes) /* * send binary to server */ -void serv_write(char *buf, int nbytes) -{ +void serv_write(char *buf, int nbytes) { int bytes_written = 0; int retval; while (bytes_written < nbytes) { @@ -95,8 +92,7 @@ void serv_write(char *buf, int nbytes) /* * input string from socket - implemented in terms of serv_read() */ -void serv_gets(char *buf) -{ +void serv_gets(char *buf) { int i; /* Read one character at a time. @@ -124,8 +120,7 @@ void serv_gets(char *buf) /* * send line to server - implemented in terms of serv_write() */ -void serv_puts(char *buf) -{ +void serv_puts(char *buf) { serv_write(buf, strlen(buf)); serv_write("\n", 1); } @@ -134,8 +129,7 @@ void serv_puts(char *buf) /* * Main loop. Do things and have fun. */ -int main(int argc, char **argv) -{ +int main(int argc, char **argv) { int a; int watchdog = 60; char buf[SIZ]; -- 2.30.2