From: Art Cancro Date: Mon, 3 Sep 2018 19:25:19 +0000 (-0400) Subject: AdjRefCount() now runs synchronously, eliminating the refcount_adjustments.dat queue... X-Git-Tag: v939~351 X-Git-Url: https://code.citadel.org/?p=citadel.git;a=commitdiff_plain;h=e047988e5cb07d89edeec13afea5c25a52ef26c6 AdjRefCount() now runs synchronously, eliminating the refcount_adjustments.dat queue file. This is the last piece of server state not kept in the database, except for files/ and keys/ Increment version to 922 --- diff --git a/citadel/citadel.h b/citadel/citadel.h index aa108c30c..3a4ddf246 100644 --- a/citadel/citadel.h +++ b/citadel/citadel.h @@ -35,7 +35,7 @@ extern "C" { */ #define CITADEL PACKAGE_STRING -#define REV_LEVEL 921 // This version +#define REV_LEVEL 922 // This version #define REV_MIN 591 // Oldest compatible database #define EXPORT_REV_MIN 760 // Oldest compatible export files #define LIBCITADEL_MIN 920 // Minimum required version of libcitadel diff --git a/citadel/modules/expire/serv_expire.c b/citadel/modules/expire/serv_expire.c index a275ca8a4..879975496 100644 --- a/citadel/modules/expire/serv_expire.c +++ b/citadel/modules/expire/serv_expire.c @@ -3,7 +3,7 @@ * * You might also see this module affectionately referred to as the DAP (the Dreaded Auto-Purger). * - * Copyright (c) 1988-2017 by citadel.org (Art Cancro, Wilifried Goesgens, and others) + * Copyright (c) 1988-2018 by citadel.org (Art Cancro, Wilifried Goesgens, and others) * * This program is open source software; you can redistribute it and/or * modify it under the terms of the GNU General Public License as published @@ -898,11 +898,10 @@ void purge_databases(void) syslog(LOG_NOTICE, "Purged %d stale external auth associations.", retval); } - if (!server_shutting_down) - { - retval = TDAP_ProcessAdjRefCountQueue(); - syslog(LOG_NOTICE, "Processed %d message reference count adjustments.", retval); - } + //if (!server_shutting_down) + //{ + // FIXME this is where we could do a non-interactive delete of zero-refcount messages + //} if ( (!server_shutting_down) && (CtdlGetConfigInt("c_shrink_db_files") != 0) ) { diff --git a/citadel/modules/migrate/serv_migrate.c b/citadel/modules/migrate/serv_migrate.c index 96ff75057..d55f8e9de 100644 --- a/citadel/modules/migrate/serv_migrate.c +++ b/citadel/modules/migrate/serv_migrate.c @@ -253,7 +253,6 @@ int is_sequence_set(char *s) { } - /* * Traverse the visits file... */ @@ -359,7 +358,6 @@ void migr_export_message(long msgnum) { } - void migr_export_openids(void) { struct cdbdata *cdboi; long usernum; @@ -405,8 +403,6 @@ void migr_export_configs(void) { } - - void migr_export_messages(void) { char buf[SIZ]; long msgnum; @@ -434,16 +430,17 @@ void migr_export_messages(void) { } fclose(migr_global_message_list); } - if (Ctx->kill_me == 0) + if (Ctx->kill_me == 0) { syslog(LOG_INFO, "Exported %d messages.", count); - else + } + else { syslog(LOG_ERR, "Export aborted due to client disconnect!"); + } migr_export_message(-1L); /* This frees the encoding buffer */ } - void migr_do_export(void) { CitContext *Ctx; @@ -478,8 +475,6 @@ void migr_do_export(void) { } - - /****************************************************************************** * Import code * * Here's the code that implements the import side. It's going to end up * @@ -613,9 +608,10 @@ int migr_roomrecord(void *data, const char *el) return 1; } + int migr_floorrecord(void *data, const char *el) { - if (!strcasecmp(el, "f_num")) floornum = atoi(ChrPtr(migr_chardata)); + if (!strcasecmp(el, "f_num")) floornum = atoi(ChrPtr(migr_chardata)); else if (!strcasecmp(el, "f_flags")) flbuf.f_flags = atoi(ChrPtr(migr_chardata)); else if (!strcasecmp(el, "f_name")) safestrncpy(flbuf.f_name, ChrPtr(migr_chardata), sizeof flbuf.f_name); else if (!strcasecmp(el, "f_ref_count")) flbuf.f_ref_count = atoi(ChrPtr(migr_chardata)); @@ -625,6 +621,7 @@ int migr_floorrecord(void *data, const char *el) return 1; } + int migr_visitrecord(void *data, const char *el) { if (!strcasecmp(el, "v_roomnum")) vbuf.v_roomnum = atol(ChrPtr(migr_chardata)); @@ -780,9 +777,9 @@ void migr_xml_end(void *data, const char *el) } /*** VISITS ***/ - else if ((!strncasecmp(el, HKEY("v_"))) && - migr_visitrecord(data, el)) + else if ((!strncasecmp(el, HKEY("v_"))) && migr_visitrecord(data, el)) { ; /* Nothing to do anymore */ + } else if (!strcasecmp(el, "visit")) { put_visit(&vbuf); syslog(LOG_INFO, "Imported visit: %ld/%ld/%ld", vbuf.v_roomnum, vbuf.v_roomgen, vbuf.v_usernum); @@ -835,8 +832,6 @@ void migr_xml_end(void *data, const char *el) } - - /* * Import begins here */ @@ -865,9 +860,7 @@ void migr_do_import(void) { client_set_inbound_buf(SIZ * 10); while (!Finished && client_read_random_blob(Buf, -1) >= 0) { - if ((StrLength(Buf) > 4) && - !strcmp(ChrPtr(Buf) + StrLength(Buf) - 4, "000\n")) - { + if ((StrLength(Buf) > 4) && !strcmp(ChrPtr(Buf) + StrLength(Buf) - 4, "000\n")) { Finished = 1; StrBufCutAt(Buf, StrLength(Buf) - 4, NULL); } @@ -978,7 +971,6 @@ int migr_restore_message_metadata(long msgnum, int refcount) smi.meta_rfc822_length = StrLength(CCC->redirect_buffer); CCC->redirect_buffer = NULL; - syslog(LOG_INFO, "Setting message #%ld meta data to: refcount=%d, bodylength=%ld, content-type: %s", smi.meta_msgnum, @@ -994,6 +986,7 @@ int migr_restore_message_metadata(long msgnum, int refcount) return 0; } + void migr_check_room_msg(long msgnum, void *userdata) { fprintf(migr_global_message_list, "%ld %s\n", msgnum, CC->room.QRname); } @@ -1007,6 +1000,7 @@ void migr_check_rooms_backend(struct ctdlroom *buf, void *data) { CtdlForEachMessage(MSGS_ALL, 0L, NULL, NULL, NULL, migr_check_room_msg, NULL); } + void RemoveMessagesFromRooms(StrBuf *RoomNameVec, long msgnum) { struct MetaData smi; const char *Pos = NULL; @@ -1019,9 +1013,10 @@ void RemoveMessagesFromRooms(StrBuf *RoomNameVec, long msgnum) { CtdlDeleteMessages(ChrPtr(oneRoom), &msgnum, 1, ""); }; GetMetaData(&smi, msgnum); - TDAP_AdjRefCount(msgnum, -smi.meta_refcount); + AdjRefCount(msgnum, -smi.meta_refcount); } + void migr_do_restore_meta(void) { char buf[SIZ]; int failGetMessage; @@ -1069,14 +1064,16 @@ void migr_do_restore_meta(void) { } refcount = 1; lastnum = msgnum; - if (prn != NULL) + if (prn != NULL) { StrBufPlain(RoomNames, prn + 1, -1); + } StrBufTrim(RoomNames); } else { if (prn != NULL) { - if (StrLength(RoomNames) > 0) + if (StrLength(RoomNames) > 0) { StrBufAppendBufPlain(RoomNames, HKEY("|"), 0); + } StrBufAppendBufPlain(RoomNames, prn, -1, 1); StrBufTrim(RoomNames); } @@ -1097,8 +1094,6 @@ void migr_do_restore_meta(void) { } - - /****************************************************************************** * Dispatcher, Common code * ******************************************************************************/ @@ -1107,8 +1102,7 @@ void cmd_migr(char *cmdbuf) { if (CtdlAccessCheck(ac_internal)) return; - if (CtdlTrySingleUser()) - { + if (CtdlTrySingleUser()) { CtdlDisableHouseKeeping(); CtdlMakeTempFileName(migr_tempfilename1, sizeof migr_tempfilename1); CtdlMakeTempFileName(migr_tempfilename2, sizeof migr_tempfilename2); @@ -1136,12 +1130,12 @@ void cmd_migr(char *cmdbuf) { CtdlEnableHouseKeeping(); CtdlEndSingleUser(); } - else - { + else { cprintf("%d The migrator is already running.\n", ERROR + RESOURCE_BUSY); } } + /****************************************************************************** * Module Hook * ******************************************************************************/ diff --git a/citadel/modules/upgrade/serv_upgrade.c b/citadel/modules/upgrade/serv_upgrade.c index 94c1406f4..35216e126 100644 --- a/citadel/modules/upgrade/serv_upgrade.c +++ b/citadel/modules/upgrade/serv_upgrade.c @@ -4,7 +4,7 @@ * guesses about what kind of data format changes need to be applied, and * we apply them transparently. * - * Copyright (c) 1987-2017 by the citadel.org team + * Copyright (c) 1987-2018 by the citadel.org team * * This program is open source software; you can redistribute it and/or * modify it under the terms of the GNU General Public License version 3. @@ -501,6 +501,38 @@ void miafvtur_backend(struct ctdluser *usbuf, void *data) { } +/* + * If our system still has a "refcount_adjustments.dat" sitting around from an old version, ingest it now. + */ +int ProcessOldStyleAdjRefCountQueue(void) +{ + int r; + FILE *fp; + struct arcq arcq_rec; + int num_records_processed = 0; + + fp = fopen(file_arcq, "rb"); + if (fp == NULL) { + return(num_records_processed); + } + + syslog(LOG_INFO, "msgbase: ingesting %s", file_arcq); + + while (fread(&arcq_rec, sizeof(struct arcq), 1, fp) == 1) { + AdjRefCount(arcq_rec.arcq_msgnum, arcq_rec.arcq_delta); + ++num_records_processed; + } + + fclose(fp); + r = unlink(file_arcq); + if (r != 0) { + syslog(LOG_ERR, "%s: %m", file_arcq); + } + + return(num_records_processed); +} + + /* * Prior to version 912 we kept a user's various Internet email addresses in their vCards. * This function moves them over to the user record, which is where we keep them now. @@ -599,6 +631,10 @@ void post_startup_upgrades(void) { if ((oldver > 000) && (oldver < 912)) { move_inet_addrs_from_vcards_to_user_records(); } + + if ((oldver > 000) && (oldver < 922)) { + ProcessOldStyleAdjRefCountQueue(); + } } diff --git a/citadel/msgbase.c b/citadel/msgbase.c index 901d18673..a88a76369 100644 --- a/citadel/msgbase.c +++ b/citadel/msgbase.c @@ -35,10 +35,6 @@ struct addresses_to_be_filed *atbf = NULL; -/* This temp file holds the queue of operations for AdjRefCount() */ -static FILE *arcfp = NULL; -void AdjRefCountList(long *msgnum, long nmsg, int incr); - /* * These are the four-character field headers we use when outputting * messages in Citadel format (as opposed to RFC822 format). @@ -3530,168 +3526,22 @@ void PutMetaData(struct MetaData *smibuf) /* - * AdjRefCount - submit an adjustment to the reference count for a message. - * (These are just queued -- we actually process them later.) + * Convenience function to process a big block of AdjRefCount() operations */ -void AdjRefCount(long msgnum, int incr) -{ - struct arcq new_arcq; - int rv = 0; - - syslog(LOG_DEBUG, "msgbase: AdjRefCount() msg %ld ref count delta %+d", msgnum, incr); - - begin_critical_section(S_SUPPMSGMAIN); - if (arcfp == NULL) { - arcfp = fopen(file_arcq, "ab+"); - chown(file_arcq, CTDLUID, (-1)); - chmod(file_arcq, 0600); - } - end_critical_section(S_SUPPMSGMAIN); - - /* msgnum < 0 means that we're trying to close the file */ - if (msgnum < 0) { - syslog(LOG_DEBUG, "msgbase: closing the AdjRefCount queue file"); - begin_critical_section(S_SUPPMSGMAIN); - if (arcfp != NULL) { - fclose(arcfp); - arcfp = NULL; - } - end_critical_section(S_SUPPMSGMAIN); - return; - } - - /* - * If we can't open the queue, perform the operation synchronously. - */ - if (arcfp == NULL) { - TDAP_AdjRefCount(msgnum, incr); - return; - } - - new_arcq.arcq_msgnum = msgnum; - new_arcq.arcq_delta = incr; - rv = fwrite(&new_arcq, sizeof(struct arcq), 1, arcfp); - if (rv == -1) { - syslog(LOG_EMERG, "%s: %m", file_arcq); - } - fflush(arcfp); - - return; -} - - void AdjRefCountList(long *msgnum, long nmsg, int incr) { - long i, the_size, offset; - struct arcq *new_arcq; - int rv = 0; - - begin_critical_section(S_SUPPMSGMAIN); - if (arcfp == NULL) { - arcfp = fopen(file_arcq, "ab+"); - chown(file_arcq, CTDLUID, (-1)); - chmod(file_arcq, 0600); - } - end_critical_section(S_SUPPMSGMAIN); - - /* - * If we can't open the queue, perform the operation synchronously. - */ - if (arcfp == NULL) { - for (i = 0; i < nmsg; i++) - TDAP_AdjRefCount(msgnum[i], incr); - return; - } + long i; - the_size = sizeof(struct arcq) * nmsg; - new_arcq = malloc(the_size); for (i = 0; i < nmsg; i++) { - syslog(LOG_DEBUG, "msgbase: AdjRefCountList() msg %ld ref count delta %+d", msgnum[i], incr); - new_arcq[i].arcq_msgnum = msgnum[i]; - new_arcq[i].arcq_delta = incr; + AdjRefCount(msgnum[i], incr); } - rv = 0; - offset = 0; - while ((rv >= 0) && (offset < the_size)) - { - rv = fwrite(new_arcq + offset, 1, the_size - offset, arcfp); - if (rv == -1) { - syslog(LOG_ERR, "%s: %m", file_arcq); - } - else { - offset += rv; - } - } - free(new_arcq); - fflush(arcfp); - - return; } /* - * TDAP_ProcessAdjRefCountQueue() - * - * Process the queue of message count adjustments that was created by calls - * to AdjRefCount() ... by reading the queue and calling TDAP_AdjRefCount() - * for each one. This should be an "off hours" operation. + * AdjRefCount - adjust the reference count for a message. We need to delete from disk any message whose reference count reaches zero. */ -int TDAP_ProcessAdjRefCountQueue(void) -{ - char file_arcq_temp[PATH_MAX]; - int r; - FILE *fp; - struct arcq arcq_rec; - int num_records_processed = 0; - - snprintf(file_arcq_temp, sizeof file_arcq_temp, "%s.%04x", file_arcq, rand()); - - begin_critical_section(S_SUPPMSGMAIN); - if (arcfp != NULL) { - fclose(arcfp); - arcfp = NULL; - } - - r = link(file_arcq, file_arcq_temp); - if (r != 0) { - syslog(LOG_ERR, "%s: %m", file_arcq_temp); - end_critical_section(S_SUPPMSGMAIN); - return(num_records_processed); - } - - unlink(file_arcq); - end_critical_section(S_SUPPMSGMAIN); - - fp = fopen(file_arcq_temp, "rb"); - if (fp == NULL) { - syslog(LOG_ERR, "%s: %m", file_arcq_temp); - return(num_records_processed); - } - - while (fread(&arcq_rec, sizeof(struct arcq), 1, fp) == 1) { - TDAP_AdjRefCount(arcq_rec.arcq_msgnum, arcq_rec.arcq_delta); - ++num_records_processed; - } - - fclose(fp); - r = unlink(file_arcq_temp); - if (r != 0) { - syslog(LOG_ERR, "%s: %m", file_arcq_temp); - } - - return(num_records_processed); -} - - -/* - * TDAP_AdjRefCount - adjust the reference count for a message. - * This one does it "for real" because it's called by - * the autopurger function that processes the queue - * created by AdjRefCount(). If a message's reference - * count becomes zero, we also delete the message from - * disk and de-index it. - */ -void TDAP_AdjRefCount(long msgnum, int incr) +void AdjRefCount(long msgnum, int incr) { struct MetaData smi; long delnum; @@ -3705,10 +3555,9 @@ void TDAP_AdjRefCount(long msgnum, int incr) smi.meta_refcount += incr; PutMetaData(&smi); end_critical_section(S_SUPPMSGMAIN); - syslog(LOG_DEBUG, "msgbase: TDAP_AdjRefCount() msg %ld ref count delta %+d, is now %d", msgnum, incr, smi.meta_refcount); + syslog(LOG_DEBUG, "msgbase: AdjRefCount() msg %ld ref count delta %+d, is now %d", msgnum, incr, smi.meta_refcount); - /* If the reference count is now zero, delete the message - * (and its supplementary record as well). + /* If the reference count is now zero, delete both the message and its metadata record. */ if (smi.meta_refcount == 0) { syslog(LOG_DEBUG, "msgbase: deleting message <%ld>", msgnum); diff --git a/citadel/msgbase.h b/citadel/msgbase.h index ec22c2020..6ba7603ef 100644 --- a/citadel/msgbase.h +++ b/citadel/msgbase.h @@ -226,5 +226,6 @@ struct CtdlMessage *CtdlMakeMessageLen( long reflen ); +void AdjRefCountList(long *msgnum, long nmsg, int incr); #endif /* MSGBASE_H */