ctdlmigrate now uses a direct socket connection to the local server instead of sendco...

diff --git a/citadel/ctdlmigrate.c b/citadel/ctdlmigrate.c
index 21dad4b24f87f331956542a95ced05e13db28ace..7dbf5d389a8d85f29a7fe2b325a5f98594e2d687 100644 (file)
--- a/citadel/ctdlmigrate.c
+++ b/citadel/ctdlmigrate.c
@@ -35,6 +35,8 @@
 #include <limits.h>
 #include <pwd.h>
 #include <time.h>
 #include <limits.h>
 #include <pwd.h>
 #include <time.h>

+#include <sys/socket.h>
+#include <sys/un.h>

 #include <libcitadel.h>
 #include "citadel.h"
 #include "axdefs.h"
 #include <libcitadel.h>
 #include "citadel.h"
 #include "axdefs.h"


@@ -63,13 +65,103 @@ void getz(char *buf) {
 }
 
 
 }
 
 

+int uds_connectsock(char *sockpath) {
+       int s;
+       struct sockaddr_un addr;

 
 

+       memset(&addr, 0, sizeof(addr));
+       addr.sun_family = AF_UNIX;
+       strncpy(addr.sun_path, sockpath, sizeof addr.sun_path);
+
+       s = socket(AF_UNIX, SOCK_STREAM, 0);
+       if (s < 0) {
+               fprintf(stderr, "sendcommand: Can't create socket: %s\n", strerror(errno));
+               exit(3);
+       }
+
+       if (connect(s, (struct sockaddr *) &addr, sizeof(addr)) < 0) {
+               fprintf(stderr, "sendcommand: can't connect: %s\n", strerror(errno));
+               close(s);
+               exit(3);
+       }
+
+       return s;
+}
+
+
+/*
+ * input binary data from socket
+ */
+void serv_read(int serv_sock, char *buf, int bytes) {
+       int len, rlen;
+
+       len = 0;
+       while (len < bytes) {
+               rlen = read(serv_sock, &buf[len], bytes - len);
+               if (rlen < 1) {
+                       return;
+               }
+               len = len + rlen;
+       }
+}
+
+
+/*
+ * send binary to server
+ */
+void serv_write(int serv_sock, char *buf, int nbytes) {
+       int bytes_written = 0;
+       int retval;
+       while (bytes_written < nbytes) {
+               retval = write(serv_sock, &buf[bytes_written], nbytes - bytes_written);
+               if (retval < 1) {
+                       return;
+               }
+               bytes_written = bytes_written + retval;
+       }
+}
+
+
+/*
+ * input string from socket - implemented in terms of serv_read()
+ */
+void serv_gets(int serv_sock, char *buf) {
+       int i;
+
+       /* Read one character at a time.
+        */
+       for (i = 0;; i++) {
+               serv_read(serv_sock, &buf[i], 1);
+               if (buf[i] == '\n' || i == (SIZ-1))
+                       break;
+       }
+
+       /* If we got a long line, discard characters until the newline.
+        */
+       if (i == (SIZ-1)) {
+               while (buf[i] != '\n') {
+                       serv_read(serv_sock, &buf[i], 1);
+               }
+       }
+
+       /* Strip all trailing nonprintables (crlf)
+        */
+       buf[i] = 0;
+}
+
+
+/*
+ * send line to server - implemented in terms of serv_write()
+ */
+void serv_puts(int serv_sock, char *buf) {
+       serv_write(serv_sock, buf, strlen(buf));
+       serv_write(serv_sock, "\n", 1);
+}

 
 
 int main(int argc, char *argv[]) {
        char ctdldir[PATH_MAX]=CTDLDIR;
        char yesno[5];
 
 
 int main(int argc, char *argv[]) {
        char ctdldir[PATH_MAX]=CTDLDIR;
        char yesno[5];

-       char sendcommand[PATH_MAX];

        int cmdexit;
        char cmd[PATH_MAX];
        char buf[PATH_MAX];
        int cmdexit;
        char cmd[PATH_MAX];
        char buf[PATH_MAX];


@@ -78,18 +170,27 @@ int main(int argc, char *argv[]) {
        char remote_host[SIZ];
        char remote_sendcommand[PATH_MAX];
        FILE *sourcefp = NULL;
        char remote_host[SIZ];
        char remote_sendcommand[PATH_MAX];
        FILE *sourcefp = NULL;

-       FILE *targetfp = NULL;

        int linecount = 0;
        int linecount = 0;

-       char spinning[4] = "-\\|/" ;
-       int exitcode = 0;
-       
-       CtdlMakeTempFileName(socket_path, sizeof socket_path);
+       int a;
+       int local_admin_socket = (-1);
+
+       /* Parse command line */
+       while ((a = getopt(argc, argv, "h:")) != EOF) {
+               switch (a) {
+               case 'h':
+                       strncpy(ctdldir, optarg, sizeof ctdldir);
+                       break;
+               default:
+                       fprintf(stderr, "sendcommand: usage: ctdlmigrate [-h server_dir]\n");
+                       return(1);
+               }
+       }
+

        if (chdir(ctdldir) != 0) {
                fprintf(stderr, "sendcommand: %s: %s\n", ctdldir, strerror(errno));
                exit(errno);
        }
 
        if (chdir(ctdldir) != 0) {
                fprintf(stderr, "sendcommand: %s: %s\n", ctdldir, strerror(errno));
                exit(errno);
        }
 

-

        printf( "\033[2J\033[H\n"
                "-------------------------------------------\n"
                "Over-the-wire migration utility for Citadel\n"
        printf( "\033[2J\033[H\n"
                "-------------------------------------------\n"
                "Over-the-wire migration utility for Citadel\n"


@@ -104,11 +205,12 @@ int main(int argc, char *argv[]) {
                "the 'rsync' utility, and OpenSSH v4 or newer.\n"
                "\n"
                "You must run this utility on the TARGET SYSTEM.  Any existing data\n"
                "the 'rsync' utility, and OpenSSH v4 or newer.\n"
                "\n"
                "You must run this utility on the TARGET SYSTEM.  Any existing data\n"

-               "on this system will be ERASED.\n"
+               "on this system will be ERASED.  Your target system will be on this\n"
+               "host in %s.\n"

                "\n"
                "Do you wish to continue? "
                ,
                "\n"
                "Do you wish to continue? "
                ,

-               EXPORT_REV_MIN
+               EXPORT_REV_MIN, ctdldir

        );
 
        if ((fgets(yesno, sizeof yesno, stdin) == NULL) || (tolower(yesno[0]) != 'y')) {
        );
 
        if ((fgets(yesno, sizeof yesno, stdin) == NULL) || (tolower(yesno[0]) != 'y')) {


@@ -118,15 +220,22 @@ int main(int argc, char *argv[]) {
        printf("\n\nGreat!  First we will check some things out here on our target\n"
                "system to make sure it is ready to receive data.\n\n");
 
        printf("\n\nGreat!  First we will check some things out here on our target\n"
                "system to make sure it is ready to receive data.\n\n");
 

-       printf("Locating 'sendcommand' and checking connectivity to Citadel...\n");
-       snprintf(sendcommand, sizeof sendcommand, "%s/sendcommand", ctdl_utilbin_dir);
-       snprintf(cmd, sizeof cmd, "%s 'NOOP'", sendcommand);
-       cmdexit = system(cmd);
-       if (cmdexit != 0) {
-               printf("\nctdlmigrate was unable to attach to the Citadel server\n"
-                       "here on the target system.  Is Citadel running?\n\n");
+       printf("Checking connectivity to Citadel in %s...\n", ctdldir);
+       local_admin_socket = uds_connectsock("citadel-admin.socket");
+
+
+       serv_gets(local_admin_socket, buf);
+       puts(buf);
+       if (buf[0] != '2') {

                exit(1);
        }
                exit(1);
        }

+       serv_puts(local_admin_socket, "ECHO Connection to Citadel Server succeeded.");
+       serv_gets(local_admin_socket, buf);
+       puts(buf);
+       if (buf[0] != '2') {
+               exit(1);
+       }
+

        printf("\nOK, this side is ready to go.  Now we must connect to the source system.\n\n");
 
        printf("Enter the host name or IP address of the source system\n"
        printf("\nOK, this side is ready to go.  Now we must connect to the source system.\n\n");
 
        printf("Enter the host name or IP address of the source system\n"


@@ -134,23 +243,23 @@ int main(int argc, char *argv[]) {
                "--> ");
        getz(remote_host);
 
                "--> ");
        getz(remote_host);
 

-get_remote_user:
-       printf("\nEnter the name of a user on %s who has full access to Citadel files\n"
-               "(usually root)\n--> ",
-               remote_host);
-       getz(remote_user);
-       if (IsEmptyStr(remote_user))
-               goto get_remote_user;
+       while (IsEmptyStr(remote_user)) {
+               printf("\nEnter the name of a user on %s who has full access to Citadel files\n"
+                       "(usually root)\n--> ",
+                       remote_host);
+               getz(remote_user);
+       }

 
        printf("\nEstablishing an SSH connection to the source system...\n\n");
 
        printf("\nEstablishing an SSH connection to the source system...\n\n");

+       sprintf(socket_path, "/tmp/ctdlmigrate.XXXXXX");
+       mktemp(socket_path);

        unlink(socket_path);
        snprintf(cmd, sizeof cmd, "ssh -MNf -S %s %s@%s", socket_path, remote_user, remote_host);
        cmdexit = system(cmd);
        printf("\n");
        if (cmdexit != 0) {
                printf("This program was unable to establish an SSH session to the source system.\n\n");
        unlink(socket_path);
        snprintf(cmd, sizeof cmd, "ssh -MNf -S %s %s@%s", socket_path, remote_user, remote_host);
        cmdexit = system(cmd);
        printf("\n");
        if (cmdexit != 0) {
                printf("This program was unable to establish an SSH session to the source system.\n\n");

-               exitcode = cmdexit;
-               goto THEEND;
+               exit(cmdexit);

        }
 
        printf("\nTesting a command over the connection...\n\n");
        }
 
        printf("\nTesting a command over the connection...\n\n");


@@ -160,8 +269,7 @@ get_remote_user:
        printf("\n");
        if (cmdexit != 0) {
                printf("Remote commands are not succeeding.\n\n");
        printf("\n");
        if (cmdexit != 0) {
                printf("Remote commands are not succeeding.\n\n");

-               exitcode = cmdexit;
-               goto THEEND;
+               exit(cmdexit);

        }
 
        printf("\nLocating the remote 'sendcommand' and Citadel installation...\n");
        }
 
        printf("\nLocating the remote 'sendcommand' and Citadel installation...\n");


@@ -189,100 +297,51 @@ get_remote_user:
        printf("\n");
        if (cmdexit != 0) {
                printf("ctdlmigrate was unable to attach to the remote Citadel system.\n\n");
        printf("\n");
        if (cmdexit != 0) {
                printf("ctdlmigrate was unable to attach to the remote Citadel system.\n\n");

-               exitcode = cmdexit;
-               goto THEEND;
+               exit(cmdexit);

        }
 
        }
 

-       printf("ctdlmigrate will now begin a database migration...\n");
-       printf("  if the system doesn't start working, \n");
-       printf("  have a look at the syslog for pending jobs needing to be terminated.\n");
+       printf("\033[2J\n");
+       printf("\033[2;0H\033[33mMigrating from %s\033[0m\n\n", remote_host);

 
 

-       snprintf(cmd, sizeof cmd, "ssh -S %s %s@%s %s -w3600 MIGR export",
-               socket_path, remote_user, remote_host, remote_sendcommand);
+       snprintf(cmd, sizeof cmd, "ssh -S %s %s@%s %s -w3600 MIGR export", socket_path, remote_user, remote_host, remote_sendcommand);

        sourcefp = popen(cmd, "r");
        if (!sourcefp) {
                printf("\n%s\n\n", strerror(errno));
        sourcefp = popen(cmd, "r");
        if (!sourcefp) {
                printf("\n%s\n\n", strerror(errno));

-               exitcode = 2;
-               goto THEEND;
+               exit(2);

        }
 
        }
 

-       snprintf(cmd, sizeof cmd, "%s -w3600 MIGR import", sendcommand);
-       targetfp = popen(cmd, "w");
-       if (!targetfp) {
-               printf("\n%s\n\n", strerror(errno));
-               exitcode = 3;
-               goto THEEND;
+       serv_puts(local_admin_socket, "MIGR import");
+       serv_gets(local_admin_socket, buf);
+       if (buf[0] != '4') {
+               printf("\n%s\n", buf);
+               exit(3);

        }
 
        }
 

-       while (fgets(buf, sizeof buf, sourcefp) != NULL) {
-               if (fwrite(buf, strlen(buf), 1, targetfp) < 1) {
-                       exitcode = 4;
-                       printf("%s\n", strerror(errno));
-                       goto FAIL;
-               }
+       char *ptr;
+       time_t time_started = time(NULL);
+       time_t last_update = time(NULL);
+       while (ptr = fgets(buf, SIZ, sourcefp), (ptr != NULL)) {
+               ptr = strchr(buf, '\n');
+               if (ptr) *ptr = 0;

                ++linecount;
                ++linecount;

-               if ((linecount % 100) == 0) {
-                       printf("%c\r", spinning[((linecount / 100) % 4)]);
-                       fflush(stdout);
+               if (!strncasecmp(buf, "<progress>", 10)) {
+                       printf("\033[11;0HPercent complete: \033[32m%d\033[0m\n", atoi(&buf[10]));

                }
                }

+               if (time(NULL) != last_update) {
+                       last_update = time(NULL);
+                       printf("\033[10;0H  Lines received: \033[32m%d\033[0m\n", linecount);
+               }
+               serv_puts(local_admin_socket, buf);

        }
 
        }
 

-FAIL:  if (sourcefp) pclose(sourcefp);
-       if (targetfp) pclose(targetfp);
-       if (exitcode != 0) goto THEEND;
-
-       /* We need to copy a bunch of other stuff, and will do so using rsync */
+       serv_puts(local_admin_socket, "000");

 
 

-       snprintf(cmd, sizeof cmd, "ssh -S %s %s@%s %s MIGR listdirs",
-               socket_path, remote_user, remote_host, remote_sendcommand);
-       sourcefp = popen(cmd, "r");
-       if (!sourcefp) {
-               printf("\n%s\n\n", strerror(errno));
-               exitcode = 2;
-               goto THEEND;
-       }
-       while ((fgets(buf, sizeof buf, sourcefp)) && (strcmp(buf, "000"))) {
-               buf[strlen(buf)-1] = 0;
+       // FIXME restart the local server now

 
 

-               if (!strncasecmp(buf, "files|", 6)) {
-                       snprintf(cmd, sizeof cmd, "rsync -va --rsh='ssh -S %s' %s@%s:%s/ %s/",
-                               socket_path, remote_user, remote_host, &buf[6], ctdl_file_dir);
-               }
-               else if (!strncasecmp(buf, "messages|", 9)) {
-                       snprintf(cmd, sizeof cmd, "rsync -va --rsh='ssh -S %s' %s@%s:%s/ %s/",
-                               socket_path, remote_user, remote_host, &buf[9], ctdl_message_dir);
-               }
-               else if (!strncasecmp(buf, "keys|", 5)) {
-                       snprintf(cmd, sizeof cmd, "rsync -va --rsh='ssh -S %s' %s@%s:%s/ %s/",
-                               socket_path, remote_user, remote_host, &buf[5], ctdl_key_dir);
-               }
-               else {
-                       strcpy(cmd, "false");   /* cheap and sleazy way to throw an error */
-               }
-               printf("%s\n", cmd);
-               cmdexit = system(cmd);
-               if (cmdexit != 0) {
-                       exitcode += cmdexit;
-               }
-       }

        pclose(sourcefp);
        pclose(sourcefp);

-
-THEEND:        if (exitcode == 0) {
-               printf("\n\n *** Citadel migration was successful! *** \n\n");
-       }
-       else {
-               printf("\n\n *** Citadel migration was unsuccessful. *** \n\n");
-       }
-       printf("\nShutting down the socket connection...\n\n");
-       snprintf(cmd, sizeof cmd, "ssh -S %s -N -O exit %s@%s",
-               socket_path, remote_user, remote_host);
-       cmdexit = system(cmd);
-       printf("\n");
-       if (cmdexit != 0) {
-               printf("There was an error shutting down the socket.\n\n");
-               exitcode = cmdexit;
-       }
-
+       printf("\nShutting down the socket connection...\n\n");
+       snprintf(cmd, sizeof cmd, "ssh -S %s -N -O exit %s@%s", socket_path, remote_user, remote_host);
+       system(cmd);

        unlink(socket_path);
        unlink(socket_path);

-       exit(exitcode);
+       exit(0);

 }
 }

diff --git a/citadel/modules/migrate/serv_migrate.c b/citadel/modules/migrate/serv_migrate.c
index 99377ee914ec874c482ea3b79f7ab5b7214bb02c..aad6615f553dce0659d0f3954edbbd6810411f6d 100644 (file)
--- a/citadel/modules/migrate/serv_migrate.c
+++ b/citadel/modules/migrate/serv_migrate.c
@@ -834,14 +834,12 @@ void migr_xml_end(void *data, const char *el)
  * Import begins here
  */
 void migr_do_import(void) {
  * Import begins here
  */
 void migr_do_import(void) {

-       StrBuf *Buf;

        XML_Parser xp;
        XML_Parser xp;

-       int Finished = 0;
+       char buf[SIZ];

        
        unbuffer_output();
        migr_chardata = NewStrBufPlain(NULL, SIZ * 20);
        migr_MsgData = NewStrBufPlain(NULL, SIZ * 20);
        
        unbuffer_output();
        migr_chardata = NewStrBufPlain(NULL, SIZ * 20);
        migr_MsgData = NewStrBufPlain(NULL, SIZ * 20);

-       Buf = NewStrBufPlain(NULL, SIZ);

        xp = XML_ParserCreate(NULL);
        if (!xp) {
                cprintf("%d Failed to create XML parser instance\n", ERROR+INTERNAL_ERROR);
        xp = XML_ParserCreate(NULL);
        if (!xp) {
                cprintf("%d Failed to create XML parser instance\n", ERROR+INTERNAL_ERROR);


@@ -857,24 +855,14 @@ void migr_do_import(void) {
 
        client_set_inbound_buf(SIZ * 10);
 
 
        client_set_inbound_buf(SIZ * 10);
 

-       while (!Finished && client_read_random_blob(Buf, -1) >= 0) {
-               if ((StrLength(Buf) > 4) && !strcmp(ChrPtr(Buf) + StrLength(Buf) - 4, "000\n")) {
-                       Finished = 1;
-                       StrBufCutAt(Buf, StrLength(Buf) - 4, NULL);
-               }
+       while (client_getln(buf, sizeof buf) >= 0 && strcmp(buf, "000")) {
+               XML_Parse(xp, buf, strlen(buf), 0);

                if (server_shutting_down)
                        break;  // Should we break or return?
                if (server_shutting_down)
                        break;  // Should we break or return?

-               
-               if (StrLength(Buf) == 0)
-                       continue;
-
-               XML_Parse(xp, ChrPtr(Buf), StrLength(Buf), 0);
-               FlushStrBuf(Buf);

        }
 
        XML_Parse(xp, "", 0, 1);
        XML_ParserFree(xp);
        }
 
        XML_Parse(xp, "", 0, 1);
        XML_ParserFree(xp);

-       FreeStrBuf(&Buf);

        FreeStrBuf(&migr_chardata);
        FreeStrBuf(&migr_MsgData);
        rebuild_euid_index();
        FreeStrBuf(&migr_chardata);
        FreeStrBuf(&migr_MsgData);
        rebuild_euid_index();


@@ -908,7 +896,6 @@ HashList *UsedMessageIDS = NULL;
 
 int migr_restore_message_metadata(long msgnum, int refcount)
 {
 
 int migr_restore_message_metadata(long msgnum, int refcount)
 {

-       CitContext *CCC = MyContext();

        struct MetaData smi;
        struct CtdlMessage *msg;
        char *mptr = NULL;
        struct MetaData smi;
        struct CtdlMessage *msg;
        char *mptr = NULL;


@@ -964,10 +951,10 @@ int migr_restore_message_metadata(long msgnum, int refcount)
                }
        }
 
                }
        }
 

-       CCC->redirect_buffer = PlainMessageBuf;
+       CC->redirect_buffer = PlainMessageBuf;

        CtdlOutputPreLoadedMsg(msg, MT_RFC822, HEADERS_ALL, 0, 1, QP_EADDR);
        CtdlOutputPreLoadedMsg(msg, MT_RFC822, HEADERS_ALL, 0, 1, QP_EADDR);

-       smi.meta_rfc822_length = StrLength(CCC->redirect_buffer);
-       CCC->redirect_buffer = NULL;
+       smi.meta_rfc822_length = StrLength(CC->redirect_buffer);
+       CC->redirect_buffer = NULL;

 
        syslog(LOG_INFO,
               "Setting message #%ld meta data to: refcount=%d, bodylength=%ld, content-type: %s",
 
        syslog(LOG_INFO,
               "Setting message #%ld meta data to: refcount=%d, bodylength=%ld, content-type: %s",

diff --git a/citadel/sendcommand.c b/citadel/sendcommand.c
index caca87fd9a534ff09c7887aad60da0be7b048f11..bd87ff3b48cf3be2f5c251cfb3afd5ddb0a17e3e 100644 (file)
--- a/citadel/sendcommand.c
+++ b/citadel/sendcommand.c
@@ -32,8 +32,7 @@
 
 int serv_sock = (-1);
 
 
 int serv_sock = (-1);
 

-int uds_connectsock(char *sockpath)
-{
+int uds_connectsock(char *sockpath) {

        int s;
        struct sockaddr_un addr;
 
        int s;
        struct sockaddr_un addr;
 


@@ -60,8 +59,7 @@ int uds_connectsock(char *sockpath)
 /*
  * input binary data from socket
  */
 /*
  * input binary data from socket
  */

-void serv_read(char *buf, int bytes)
-{
+void serv_read(char *buf, int bytes) {

        int len, rlen;
 
        len = 0;
        int len, rlen;
 
        len = 0;


@@ -78,8 +76,7 @@ void serv_read(char *buf, int bytes)
 /*
  * send binary to server
  */
 /*
  * send binary to server
  */

-void serv_write(char *buf, int nbytes)
-{
+void serv_write(char *buf, int nbytes) {

        int bytes_written = 0;
        int retval;
        while (bytes_written < nbytes) {
        int bytes_written = 0;
        int retval;
        while (bytes_written < nbytes) {


@@ -95,8 +92,7 @@ void serv_write(char *buf, int nbytes)
 /*
  * input string from socket - implemented in terms of serv_read()
  */
 /*
  * input string from socket - implemented in terms of serv_read()
  */

-void serv_gets(char *buf)
-{
+void serv_gets(char *buf) {

        int i;
 
        /* Read one character at a time.
        int i;
 
        /* Read one character at a time.


@@ -124,8 +120,7 @@ void serv_gets(char *buf)
 /*
  * send line to server - implemented in terms of serv_write()
  */
 /*
  * send line to server - implemented in terms of serv_write()
  */

-void serv_puts(char *buf)
-{
+void serv_puts(char *buf) {

        serv_write(buf, strlen(buf));
        serv_write("\n", 1);
 }
        serv_write(buf, strlen(buf));
        serv_write("\n", 1);
 }


@@ -134,8 +129,7 @@ void serv_puts(char *buf)
 /*
  * Main loop.  Do things and have fun.
  */
 /*
  * Main loop.  Do things and have fun.
  */

-int main(int argc, char **argv)
-{
+int main(int argc, char **argv) {

        int a;
        int watchdog = 60;
        char buf[SIZ];
        int a;
        int watchdog = 60;
        char buf[SIZ];