My 'ctdlload' operation kept failing because there were a few user
records dumped that had insane data in them. CtdlDecodeBase64() was
silently doing buffer overruns because it doesn't know the size of
the target buffer.
While noodling a workaround, I realized that you can pass the same
pointer to CtdlDecodeBase64() as both the source *and* target buffer.
This works because the decoded data pointer will ALWAYS be behind the
encoded data pointer. This allows a "decode-in-place" operation!
By performing a decode-in-place and then using safestrncpy() to copy
the results to the finitely-sized string buffer in the target struct,
we guarantee no buffer overruns. Valgrind no longer bitches about it
and the program does not crash.
// Dump the Citadel database to a flat file that can be restored by ctdlload on any architecture
//
// Dump the Citadel database to a flat file that can be restored by ctdlload on any architecture
//
-// Copyright (c) 2024 by Art Cancro citadel.org
+// Copyright (c) 2023-2024 by Art Cancro citadel.org
//
// This program is open source software. Use, duplication, or disclosure
// is subject to the terms of the GNU General Public License, version 3.
//
// This program is open source software. Use, duplication, or disclosure
// is subject to the terms of the GNU General Public License, version 3.
// Load (restore) the Citadel database from a flat file created by ctdldump
//
// Load (restore) the Citadel database from a flat file created by ctdldump
//
-// Copyright (c) 2024 by Art Cancro citadel.org
+// Copyright (c) 2023-2024 by Art Cancro citadel.org
//
// This program is open source software. Use, duplication, or disclosure
// is subject to the terms of the GNU General Public License, version 3.
//
// This program is open source software. Use, duplication, or disclosure
// is subject to the terms of the GNU General Public License, version 3.
// Convert a "msgmeta" record to a message metadata record on disk. NOT THREADSAFE
int import_msgmeta(char *line, struct cdbkeyval *kv) {
char *token;
// Convert a "msgmeta" record to a message metadata record on disk. NOT THREADSAFE
int import_msgmeta(char *line, struct cdbkeyval *kv) {
char *token;
- struct MetaData *m = malloc(sizeof(struct MetaData));
+ m = malloc(sizeof(struct MetaData));
+ if (!m) {
+ fprintf(stderr, "import_msgmeta: malloc: %s\n", strerror(errno));
+ return(0);
+ }
memset(m, 0, sizeof(struct MetaData));
char *p = line;
memset(m, 0, sizeof(struct MetaData));
char *p = line;
int import_user(char *line, struct cdbkeyval *kv) {
char userkey[USERNAME_SIZE];
char *token;
int import_user(char *line, struct cdbkeyval *kv) {
char userkey[USERNAME_SIZE];
char *token;
- struct ctdluser *u = malloc(sizeof(struct ctdluser));
+ u = malloc(sizeof(struct ctdluser));
+ if (!u) {
+ fprintf(stderr, "malloc failed\n");
+ return(0);
+ }
memset(u, 0, sizeof(struct ctdluser));
char *p = line;
memset(u, 0, sizeof(struct ctdluser));
char *p = line;
u->uid = atoi(token);
break;
case 3:
u->uid = atoi(token);
break;
case 3:
- strncpy(u->password, token, sizeof(u->password));
+ safestrncpy(u->password, token, sizeof(u->password));
break;
case 4:
u->flags = atoi(token);
break;
case 4:
u->flags = atoi(token);
u->USuserpurge = atoi(token);
break;
case 9:
u->USuserpurge = atoi(token);
break;
case 9:
- strncpy(u->fullname, token, sizeof(u->fullname));
+ safestrncpy(u->fullname, token, sizeof(u->fullname));
break;
case 10:
u->msgnum_bio = atol(token);
break;
case 10:
u->msgnum_bio = atol(token);
u->msgnum_pic = atol(token);
break;
case 12:
u->msgnum_pic = atol(token);
break;
case 12:
- CtdlDecodeBase64(u->emailaddrs, token, strlen(token));
+ CtdlDecodeBase64(token, token, strlen(token));
+ safestrncpy(u->emailaddrs, token, sizeof(u->emailaddrs));
break;
case 13:
u->msgnum_inboxrules = atol(token);
break;
case 13:
u->msgnum_inboxrules = atol(token);
+
+ // reject any user records without names
+ if (strlen(u->fullname) == 0) {
+ free(u);
+ return(0);
+ }
makeuserkey(userkey, u->fullname);
kv->key.len = strlen(userkey);
kv->key.ptr = strdup(userkey);
makeuserkey(userkey, u->fullname);
kv->key.len = strlen(userkey);
kv->key.ptr = strdup(userkey);
// Convert a "room" record to a record on disk. (Source string is unusable after this function is called.)
int import_room(char *line, struct cdbkeyval *kv) {
char *token;
// Convert a "room" record to a record on disk. (Source string is unusable after this function is called.)
int import_room(char *line, struct cdbkeyval *kv) {
char *token;
- struct ctdlroom *r = malloc(sizeof(struct ctdlroom));
+ r = malloc(sizeof(struct ctdlroom));
+ if (!r) {
+ fprintf(stderr, "import_room: malloc: %s\n", strerror(errno));
+ return(0);
+ }
memset(r, 0, sizeof(struct ctdlroom));
char *p = line;
memset(r, 0, sizeof(struct ctdlroom));
char *p = line;
// Convert a floor record to a record on disk.
int import_floor(char *line, struct cdbkeyval *kv) {
char *token;
// Convert a floor record to a record on disk.
int import_floor(char *line, struct cdbkeyval *kv) {
char *token;
- struct floor *f = malloc(sizeof(struct floor));
+ f = malloc(sizeof(struct floor));
+ if (!f) {
+ fprintf(stderr, "import_floor: malloc: %s\n", strerror(errno));
+ return(0);
+ }
memset(f, 0, sizeof(struct floor));
char *p = line;
memset(f, 0, sizeof(struct floor));
char *p = line;
// Convert a "visit" record to a record on disk.
int import_visit(char *line, struct cdbkeyval *kv) {
char *token;
// Convert a "visit" record to a record on disk.
int import_visit(char *line, struct cdbkeyval *kv) {
char *token;
- struct visit *v = malloc(sizeof(struct visit));
+ v = malloc(sizeof(struct visit));
+ if (!v) {
+ fprintf(stderr, "import_visit: malloc: %s\n", strerror(errno));
+ return(0);
+ }
memset(v, 0, sizeof(struct visit));
char *p = line;
memset(v, 0, sizeof(struct visit));
char *p = line;
// Convert a "usetable" record to a record on disk.
int import_usetable(char *line, struct cdbkeyval *kv) {
char *token;
// Convert a "usetable" record to a record on disk.
int import_usetable(char *line, struct cdbkeyval *kv) {
char *token;
- struct UseTable *u = malloc(sizeof(struct UseTable));
+ u = malloc(sizeof(struct UseTable));
+ if (!u) {
+ fprintf(stderr, "import_usetable: malloc: %s\n", strerror(errno));
+ return(0);
+ }
memset(u, 0, sizeof(struct UseTable));
char *p = line;
memset(u, 0, sizeof(struct UseTable));
char *p = line;
static int current_cdb = -1 ;
char record_type[32];
int row_was_good;
static int current_cdb = -1 ;
char record_type[32];
int row_was_good;
- static time_t last_diag = 0;
// Clear out our record buffer
memset(kv, 0, sizeof(struct cdbkeyval));
// Clear out our record buffer
memset(kv, 0, sizeof(struct cdbkeyval));
- if (time(NULL) > last_diag) {
- fprintf(stderr, " \033[32m%9d\033[0m / \033[31m%8d\033[0m\r", good_rows, bad_rows);
- fflush(stderr);
- last_diag = time(NULL);
- }
+
+ fprintf(stderr, "\r \033[32m%9d\033[0m / \033[31m%8d\033[0m ", good_rows, bad_rows);
+ fflush(stderr);
char *ctdldir = CTDLDIR;
// display the greeting
char *ctdldir = CTDLDIR;
// display the greeting
- fprintf(stderr, "\033[44m\033[1m"
- "╔════════════════════════════════════════════════════════════════════════╗\n"
- "║ DB Load utility for Citadel ║\n"
- "║ Copyright (c) 2023-2024 by citadel.org et al. ║\n"
- "║ This program is open source software. Use, duplication, or disclosure ║\n"
- "║ is subject to the terms of the GNU General Public license v3. ║\n"
- "╚════════════════════════════════════════════════════════════════════════╝\033[0m\n"
+ fprintf(stderr,
+ "\033[44m\033[1m╔════════════════════════════════════════════════════════════════════════╗\033[0m\n"
+ "\033[44m\033[1m║ DB Load utility for Citadel ║\033[0m\n"
+ "\033[44m\033[1m║ Copyright (c) 2023-2024 by citadel.org et al. ║\033[0m\n"
+ "\033[44m\033[1m║ This program is open source software. Use, duplication, or disclosure ║\033[0m\n"
+ "\033[44m\033[1m║ is subject to the terms of the GNU General Public license v3. ║\033[0m\n"
+ "\033[44m\033[1m╚════════════════════════════════════════════════════════════════════════╝\033[0m\n"
cdb_init_backends();
cdb_open_databases();
cdb_init_backends();
cdb_open_databases();
+ // Load rows from the input source
ingest();
// close databases
ingest();
// close databases
// and I don't want to fix it again. I don't care how many nanoseconds you think
// you can shave off the execution time. Don't fucking touch it.
//
// and I don't want to fix it again. I don't care how many nanoseconds you think
// you can shave off the execution time. Don't fucking touch it.
//
-// Copyright (c) 1987-2022 by the citadel.org team
+// Copyright (c) 1987-2024 by the citadel.org team
//
// This program is open source software. Use, duplication, or disclosure
// is subject to the terms of the GNU General Public License, version 3.
//
// This program is open source software. Use, duplication, or disclosure
// is subject to the terms of the GNU General Public License, version 3.
// source Source base64-encoded buffer
// source_len Stop after parsing this many bytes
// return value Decoded length
// source Source base64-encoded buffer
// source_len Stop after parsing this many bytes
// return value Decoded length
+
+// AWESOME SAUCE ALERT:
+// It is legal to specify the same pointer for the source and destination buffers.
+// If you do so, the string will be "decoded in place".
+
size_t CtdlDecodeBase64(char *dest, const char *source, size_t source_len) {
size_t bytes_read = 0;
size_t bytes_decoded = 0;
size_t CtdlDecodeBase64(char *dest, const char *source, size_t source_len) {
size_t bytes_read = 0;
size_t bytes_decoded = 0;
BASE64_NO_LINEBREAKS = 0,
BASE64_YES_LINEBREAKS = 1
};
BASE64_NO_LINEBREAKS = 0,
BASE64_YES_LINEBREAKS = 1
};
-size_t CtdlDecodeBase64(char *dest, const char *source, size_t length);
+size_t CtdlDecodeBase64(char *dest, const char *source, size_t source_len);
unsigned int decode_hex(char *Source);
int CtdlDecodeQuotedPrintable(char *decoded, char *encoded, int sourcelen);
void StripSlashes(char *Dir, int TrailingSlash);
unsigned int decode_hex(char *Source);
int CtdlDecodeQuotedPrintable(char *decoded, char *encoded, int sourcelen);
void StripSlashes(char *Dir, int TrailingSlash);
int safestrncpy(char *dest, const char *src, size_t n) {
int i = 0;
int safestrncpy(char *dest, const char *src, size_t n) {
int i = 0;
- if (dest == NULL || src == NULL)
- {
+ if (dest == NULL || src == NULL) {
fprintf(stderr, "safestrncpy: NULL argument\n");
abort();
}
fprintf(stderr, "safestrncpy: NULL argument\n");
abort();
}