From: Art Cancro Date: Fri, 22 Dec 2006 22:58:06 +0000 (+0000) Subject: * Reference count adjustments are now deferred by queuing X-Git-Tag: v7.86~3762 X-Git-Url: https://code.citadel.org/?p=citadel.git;a=commitdiff_plain;h=02d9dbca862ffa00b2041287596119a54867762c * Reference count adjustments are now deferred by queuing them, and the queue is processed by THE DREADED AUTO-PURGER. --- diff --git a/citadel/citadel_dirs.c b/citadel/citadel_dirs.c index 5f507f1dd..79d0d2321 100644 --- a/citadel/citadel_dirs.c +++ b/citadel/citadel_dirs.c @@ -53,6 +53,7 @@ char file_citadel_rc[PATH_MAX]=""; char file_citadel_config[PATH_MAX]=""; char file_lmtp_socket[PATH_MAX]=""; char file_lmtp_unfiltered_socket[PATH_MAX]=""; +char file_arcq[PATH_MAX]=""; char file_citadel_socket[PATH_MAX]=""; char file_mail_aliases[PATH_MAX]=""; @@ -168,6 +169,11 @@ void calc_dirs_n_files(int relh, int home, const char *relhome,const char *ctdl "%scitadel.socket", ctdl_run_dir); + snprintf(file_arcq, + sizeof file_arcq, + "%srefcount_adjustments.dat", + ctdl_run_dir); + /* * DIRTY HACK FOLLOWS! due to configs in the network dir in the * legacy installations, we need to calculate ifdeffed here. diff --git a/citadel/citadel_dirs.h b/citadel/citadel_dirs.h index fce8008d7..a4ebbf4ae 100644 --- a/citadel/citadel_dirs.h +++ b/citadel/citadel_dirs.h @@ -36,6 +36,7 @@ extern char file_citadel_rc[PATH_MAX]; extern char file_citadel_config[PATH_MAX]; extern char file_lmtp_socket[PATH_MAX]; extern char file_lmtp_unfiltered_socket[PATH_MAX]; +extern char file_arcq[PATH_MAX]; extern char file_citadel_socket[PATH_MAX]; extern char file_mail_aliases[PATH_MAX]; diff --git a/citadel/citserver.c b/citadel/citserver.c index 1a949939e..54e57695d 100644 --- a/citadel/citserver.c +++ b/citadel/citserver.c @@ -109,27 +109,6 @@ void master_startup(void) { lputroom(&qrbuf); } - /* - * Create a room in which we can deposit "deleted" messages for - * deferred deletion. This will silently fail if the room already - * exists, and that's perfectly ok, because we want it to exist. - */ - create_room(DELETED_MSGS_ROOM, 3, "", 0, 1, 0, VIEW_MAILBOX); - - /* - * Make sure it's set to be a "system room" so it doesn't show up - * in the nown rooms list for Aides. Also set the message expire - * policy to "by count, 1 message" so everything gets deleted all - * the time (we can't set it to 0 because that's invalid, so we keep - * a single message around). - */ - if (lgetroom(&qrbuf, DELETED_MSGS_ROOM) == 0) { - qrbuf.QRflags2 |= QR2_SYSTEM; - qrbuf.QRep.expire_mode = EXPIRE_NUMMSGS; - qrbuf.QRep.expire_value = 1; - lputroom(&qrbuf); - } - lprintf(CTDL_INFO, "Seeding the pseudo-random number generator...\n"); urandom = fopen("/dev/urandom", "r"); if (urandom != NULL) { @@ -166,6 +145,9 @@ void master_cleanup(int exitcode) { (*fcn->h_function_pointer)(); } + /* Close the AdjRefCount queue file */ + AdjRefCount(-1, 0); + /* Shut down the indexer thread */ lprintf(CTDL_INFO, "Waiting for the indexer thread to shut down\n"); pthread_join(indexer_thread_tid, NULL); diff --git a/citadel/msgbase.c b/citadel/msgbase.c index c7b7ce6b8..c5280e809 100644 --- a/citadel/msgbase.c +++ b/citadel/msgbase.c @@ -61,6 +61,9 @@ long config_msgnum; struct addresses_to_be_filed *atbf = NULL; +/* This temp file holds the queue of operations for AdjRefCount() */ +static FILE *arcfp = NULL; + /* * This really belongs in serv_network.c, but I don't know how to export * symbols between modules. @@ -2318,7 +2321,7 @@ void ReplicationChecks(struct CtdlMessage *msg) { old_msgnum = locate_message_by_euid(msg->cm_fields['E'], &CC->room); if (old_msgnum > 0L) { lprintf(CTDL_DEBUG, "ReplicationChecks() replacing message %ld\n", old_msgnum); - CtdlDeleteMessages(CC->room.QRname, &old_msgnum, 1, "", 0); + CtdlDeleteMessages(CC->room.QRname, &old_msgnum, 1, ""); } } @@ -3478,8 +3481,7 @@ void cmd_ent0(char *entargs) int CtdlDeleteMessages(char *room_name, /* which room */ long *dmsgnums, /* array of msg numbers to be deleted */ int num_dmsgnums, /* number of msgs to be deleted, or 0 for "any" */ - char *content_type, /* or "" for any */ - int deferred /* let TDAP sweep it later */ + char *content_type /* or "" for any */ ) { @@ -3493,8 +3495,8 @@ int CtdlDeleteMessages(char *room_name, /* which room */ int delete_this; struct MetaData smi; - lprintf(CTDL_DEBUG, "CtdlDeleteMessages(%s, %d msgs, %s, %d)\n", - room_name, num_dmsgnums, content_type, deferred); + lprintf(CTDL_DEBUG, "CtdlDeleteMessages(%s, %d msgs, %s)\n", + room_name, num_dmsgnums, content_type); /* get room record, obtaining a lock... */ if (lgetroom(&qrbuf, room_name) != 0) { @@ -3556,20 +3558,6 @@ int CtdlDeleteMessages(char *room_name, /* which room */ } lputroom(&qrbuf); - /* - * If the delete operation is "deferred" (and technically, any delete - * operation not performed by THE DREADED AUTO-PURGER ought to be - * a deferred delete) then we save a pointer to the message in the - * DELETED_MSGS_ROOM. This will cause the reference count to remain - * at least 1, which will save the user from having to synchronously - * wait for various disk-intensive operations to complete. - * - * Slick -- we now use the new bulk API for moving messages. - */ - if ( (deferred) && (num_deleted) ) { - CtdlCopyMsgsToRoom(dellist, num_deleted, DELETED_MSGS_ROOM); - } - /* Go through the messages we pulled out of the index, and decrement * their reference counts by 1. If this is the only room the message * was in, the reference count will reach zero and the message will @@ -3643,7 +3631,7 @@ void cmd_dele(char *args) msgs[i] = atol(msgtok); } - num_deleted = CtdlDeleteMessages(CC->room.QRname, msgs, num_msgs, "", 1); + num_deleted = CtdlDeleteMessages(CC->room.QRname, msgs, num_msgs, ""); free(msgs); if (num_deleted) { @@ -3762,7 +3750,7 @@ void cmd_move(char *args) * if this is a 'move' rather than a 'copy' operation. */ if (is_copy == 0) { - CtdlDeleteMessages(CC->room.QRname, msgs, num_msgs, "", 0); + CtdlDeleteMessages(CC->room.QRname, msgs, num_msgs, ""); } free(msgs); @@ -3819,10 +3807,112 @@ void PutMetaData(struct MetaData *smibuf) } /* - * AdjRefCount - change the reference count for a message; - * delete the message if it reaches zero + * AdjRefCount - submit an adjustment to the reference count for a message. + * (These are just queued -- we actually process them later.) */ void AdjRefCount(long msgnum, int incr) +{ + struct arcq new_arcq; + + begin_critical_section(S_SUPPMSGMAIN); + if (arcfp == NULL) { + arcfp = fopen(file_arcq, "ab+"); + } + end_critical_section(S_SUPPMSGMAIN); + + /* msgnum < 0 means that we're trying to close the file */ + if (msgnum < 0) { + lprintf(CTDL_DEBUG, "Closing the AdjRefCount queue file\n"); + begin_critical_section(S_SUPPMSGMAIN); + if (arcfp != NULL) { + fclose(arcfp); + arcfp = NULL; + } + end_critical_section(S_SUPPMSGMAIN); + return; + } + + /* + * If we can't open the queue, perform the operation synchronously. + */ + if (arcfp == NULL) { + TDAP_AdjRefCount(msgnum, incr); + return; + } + + new_arcq.arcq_msgnum = msgnum; + new_arcq.arcq_delta = incr; + fwrite(&new_arcq, sizeof(struct arcq), 1, arcfp); + fflush(arcfp); + + return; +} + + +/* + * TDAP_ProcessAdjRefCountQueue() + * + * Process the queue of message count adjustments that was created by calls + * to AdjRefCount() ... by reading the queue and calling TDAP_AdjRefCount() + * for each one. This should be an "off hours" operation. + */ +int TDAP_ProcessAdjRefCountQueue(void) +{ + char file_arcq_temp[PATH_MAX]; + int r; + FILE *fp; + struct arcq arcq_rec; + int num_records_processed = 0; + + snprintf(file_arcq_temp, sizeof file_arcq_temp, "%s2", file_arcq); + + begin_critical_section(S_SUPPMSGMAIN); + if (arcfp != NULL) { + fclose(arcfp); + arcfp = NULL; + } + + r = link(file_arcq, file_arcq_temp); + if (r != 0) { + lprintf(CTDL_CRIT, "%s: %s\n", file_arcq_temp, strerror(errno)); + end_critical_section(S_SUPPMSGMAIN); + return(num_records_processed); + } + + unlink(file_arcq); + end_critical_section(S_SUPPMSGMAIN); + + fp = fopen(file_arcq_temp, "rb"); + if (fp == NULL) { + lprintf(CTDL_CRIT, "%s: %s\n", file_arcq_temp, strerror(errno)); + return(num_records_processed); + } + + while (fread(&arcq_rec, sizeof(struct arcq), 1, fp) == 1) { + TDAP_AdjRefCount(arcq_rec.arcq_msgnum, arcq_rec.arcq_delta); + ++num_records_processed; + } + + fclose(fp); + r = unlink(file_arcq_temp); + if (r != 0) { + lprintf(CTDL_CRIT, "%s: %s\n", file_arcq_temp, strerror(errno)); + } + + return(num_records_processed); +} + + + +/* + * TDAP_AdjRefCount - adjust the reference count for a message. + * This one does it "for real" because it's called by + * the autopurger function that processes the queue + * created by AdjRefCount(). If a message's reference + * count becomes zero, we also delete the message from + * disk and de-index it. + */ +void TDAP_AdjRefCount(long msgnum, int incr) { struct MetaData smi; @@ -3860,6 +3950,7 @@ void AdjRefCount(long msgnum, int incr) delnum = (0L - msgnum); cdb_delete(CDB_MSGMAIN, &delnum, (int)sizeof(long)); } + } /* @@ -3973,7 +4064,7 @@ void CtdlWriteObject(char *req_room, /* Room to stuff it in */ */ if (is_unique) { lprintf(CTDL_DEBUG, "Deleted %d other msgs of this type\n", - CtdlDeleteMessages(roomname, NULL, 0, content_type, 0) + CtdlDeleteMessages(roomname, NULL, 0, content_type) ); } /* Now write the data */ diff --git a/citadel/msgbase.h b/citadel/msgbase.h index 33f67b9ef..026908fa3 100644 --- a/citadel/msgbase.h +++ b/citadel/msgbase.h @@ -102,6 +102,8 @@ void cmd_move (char *args); void GetMetaData(struct MetaData *, long); void PutMetaData(struct MetaData *); void AdjRefCount(long, int); +void TDAP_AdjRefCount(long, int); +int TDAP_ProcessAdjRefCountQueue(void); void simple_listing(long, void *); int CtdlMsgCmp(struct CtdlMessage *msg, struct CtdlMessage *template); int CtdlForEachMessage(int mode, @@ -111,7 +113,7 @@ int CtdlForEachMessage(int mode, struct CtdlMessage *compare, void (*CallBack) (long, void *), void *userdata); -int CtdlDeleteMessages(char *, long *, int, char *, int); +int CtdlDeleteMessages(char *, long *, int, char *); void CtdlWriteObject(char *, char *, char *, struct ctdluser *, int, int, unsigned int); struct CtdlMessage *CtdlFetchMessage(long msgnum, int with_body); diff --git a/citadel/room_ops.c b/citadel/room_ops.c index d6ec55e4a..05bced87c 100644 --- a/citadel/room_ops.c +++ b/citadel/room_ops.c @@ -1607,7 +1607,7 @@ void delete_room(struct ctdlroom *qrbuf) /* Delete the messages in the room * (Careful: this opens an S_ROOMS critical section!) */ - CtdlDeleteMessages(qrbuf->QRname, NULL, 0, "", 0); + CtdlDeleteMessages(qrbuf->QRname, NULL, 0, ""); /* Flag the room record as not in use */ lgetroom(qrbuf, qrbuf->QRname); diff --git a/citadel/serv_calendar.c b/citadel/serv_calendar.c index a847f2000..cbb3f84c4 100644 --- a/citadel/serv_calendar.c +++ b/citadel/serv_calendar.c @@ -430,7 +430,7 @@ void ical_respond(long msgnum, char *partnum, char *action) { /* Now that we've processed this message, we don't need it * anymore. So delete it. */ - CtdlDeleteMessages(CC->room.QRname, &msgnum, 1, "", 1); + CtdlDeleteMessages(CC->room.QRname, &msgnum, 1, ""); /* Free the memory we allocated and return a response. */ icalcomponent_free(ird.cal); @@ -777,7 +777,7 @@ void ical_handle_rsvp(long msgnum, char *partnum, char *action) { /* Now that we've processed this message, we don't need it * anymore. So delete it. (Maybe make this optional?) */ - CtdlDeleteMessages(CC->room.QRname, &msgnum, 1, "", 1); + CtdlDeleteMessages(CC->room.QRname, &msgnum, 1, ""); /* Free the memory we allocated and return a response. */ icalcomponent_free(ird.cal); @@ -1474,7 +1474,7 @@ void ical_putics(void) * the entire calendar with an entire new (or updated) calendar. * (Careful: this opens an S_ROOMS critical section!) */ - CtdlDeleteMessages(CC->room.QRname, NULL, 0, "", 0); + CtdlDeleteMessages(CC->room.QRname, NULL, 0, ""); /* If the top-level component is *not* a VCALENDAR, we can drop it right * in. This will almost never happen. diff --git a/citadel/serv_expire.c b/citadel/serv_expire.c index bb78d96bb..e357ea615 100644 --- a/citadel/serv_expire.c +++ b/citadel/serv_expire.c @@ -212,7 +212,7 @@ void DoPurgeMessages(FILE *purgelist) { if (!strncasecmp(buf, "m=", 2)) { msgnum = atol(&buf[2]); if (msgnum > 0L) { - CtdlDeleteMessages(roomname, &msgnum, 1, "", 0); + CtdlDeleteMessages(roomname, &msgnum, 1, ""); } } } @@ -726,6 +726,9 @@ void purge_databases(void) { retval = PurgeEuidIndexTable(); lprintf(CTDL_NOTICE, "Purged %d entries from the EUID index.\n", retval); + retval = TDAP_ProcessAdjRefCountQueue(); + lprintf(CTDL_NOTICE, "Processed %d message reference count adjustments.\n", retval); + lprintf(CTDL_INFO, "Auto-purger: finished.\n"); last_purge = now; /* So we don't do it again soon */ @@ -793,9 +796,7 @@ void cmd_fsck(char *argbuf) { if ( (smi.meta_refcount != realcount) || (realcount == 0) ) { - smi.meta_refcount = realcount; - PutMetaData(&smi); - AdjRefCount(msgnum, 0); /* deletes if needed */ + AdjRefCount(msgnum, (smi.meta_refcount - realcount)); } } diff --git a/citadel/serv_imap.c b/citadel/serv_imap.c index 7862a298b..9c17ccae7 100644 --- a/citadel/serv_imap.c +++ b/citadel/serv_imap.c @@ -727,7 +727,7 @@ int imap_do_expunge(void) } } if (num_delmsgs > 0) { - CtdlDeleteMessages(CC->room.QRname, delmsgs, num_delmsgs, "", 1); + CtdlDeleteMessages(CC->room.QRname, delmsgs, num_delmsgs, ""); } num_expunged += num_delmsgs; free(delmsgs); @@ -1396,7 +1396,9 @@ void imap_command_loop(void) char cmdbuf[SIZ]; char *parms[SIZ]; int num_parms; + struct timeval tv1, tv2; + gettimeofday(&tv1, NULL); CC->lastcmd = time(NULL); memset(cmdbuf, 0, sizeof cmdbuf); /* Clear it, just in case */ flush_output(); @@ -1613,6 +1615,12 @@ void imap_command_loop(void) /* If the client transmitted a message we can free it now */ imap_free_transmitted_message(); + + gettimeofday(&tv2, NULL); + lprintf(CTDL_DEBUG, "IMAP %s took %ld microseconds\n", + parms[1], + (tv2.tv_usec + (tv2.tv_sec * 1000000)) - (tv1.tv_usec + (tv1.tv_sec * 1000000)) + ); } diff --git a/citadel/serv_network.c b/citadel/serv_network.c index a3b98a4f8..2b4ca3e12 100644 --- a/citadel/serv_network.c +++ b/citadel/serv_network.c @@ -864,7 +864,7 @@ void network_spool_msg(long msgnum, void *userdata) { /* Delete this message if delete-after-send is set */ if (delete_after_send) { - CtdlDeleteMessages(CC->room.QRname, &msgnum, 1, "", 0); + CtdlDeleteMessages(CC->room.QRname, &msgnum, 1, ""); } } diff --git a/citadel/serv_pop3.c b/citadel/serv_pop3.c index db1689fb8..17255d1d1 100644 --- a/citadel/serv_pop3.c +++ b/citadel/serv_pop3.c @@ -478,7 +478,7 @@ void pop3_update(void) { } } if (num_deletemsgs > 0) { - CtdlDeleteMessages(MAILROOM, deletemsgs, num_deletemsgs, "", 1); + CtdlDeleteMessages(MAILROOM, deletemsgs, num_deletemsgs, ""); } free(deletemsgs); } diff --git a/citadel/serv_sieve.c b/citadel/serv_sieve.c index 8bfd7828b..fbb826416 100644 --- a/citadel/serv_sieve.c +++ b/citadel/serv_sieve.c @@ -596,7 +596,7 @@ void sieve_do_msg(long msgnum, void *userdata) { */ if ( (!my.keep) && (my.cancel_implicit_keep) ) { lprintf(CTDL_DEBUG, "keep is 0 -- deleting message from inbox\n"); - CtdlDeleteMessages(CC->room.QRname, &msgnum, 1, "", 0); + CtdlDeleteMessages(CC->room.QRname, &msgnum, 1, ""); } lprintf(CTDL_DEBUG, "Completed sieve processing on msg <%ld>\n", msgnum); @@ -758,7 +758,7 @@ void rewrite_ctdl_sieve_config(struct sdm_userdata *u, int yes_write_to_disk) { /* And delete the old one */ if (u->config_msgnum > 0) { - CtdlDeleteMessages(u->config_roomname, &u->config_msgnum, 1, "", 0); + CtdlDeleteMessages(u->config_roomname, &u->config_msgnum, 1, ""); } } diff --git a/citadel/serv_smtp.c b/citadel/serv_smtp.c index dcad73001..255dfcf2a 100644 --- a/citadel/serv_smtp.c +++ b/citadel/serv_smtp.c @@ -1659,7 +1659,7 @@ void smtp_do_procmsg(long msgnum, void *userdata) { long delmsgs[2]; delmsgs[0] = msgnum; delmsgs[1] = text_msgid; - CtdlDeleteMessages(SMTP_SPOOLOUT_ROOM, delmsgs, 2, "", 0); + CtdlDeleteMessages(SMTP_SPOOLOUT_ROOM, delmsgs, 2, ""); } /* @@ -1667,7 +1667,7 @@ void smtp_do_procmsg(long msgnum, void *userdata) { * instructions and replace with the updated ones. */ if (incomplete_deliveries_remaining > 0) { - CtdlDeleteMessages(SMTP_SPOOLOUT_ROOM, &msgnum, 1, "", 0); + CtdlDeleteMessages(SMTP_SPOOLOUT_ROOM, &msgnum, 1, ""); msg = malloc(sizeof(struct CtdlMessage)); memset(msg, 0, sizeof(struct CtdlMessage)); msg->cm_magic = CTDLMESSAGE_MAGIC; diff --git a/citadel/serv_vcard.c b/citadel/serv_vcard.c index f2205b687..ffa723f3f 100644 --- a/citadel/serv_vcard.c +++ b/citadel/serv_vcard.c @@ -322,8 +322,7 @@ int vcard_upload_beforesave(struct CtdlMessage *msg) { * want to make sure there is absolutely only one * vCard in the user's config room at all times. */ - CtdlDeleteMessages(CC->room.QRname, - NULL, 0, "text/x-vcard", 1); + CtdlDeleteMessages(CC->room.QRname, NULL, 0, "text/x-vcard"); /* Make the author of the message the name of the user. */ diff --git a/citadel/server.h b/citadel/server.h index e76fe1ac8..fa424e460 100644 --- a/citadel/server.h +++ b/citadel/server.h @@ -457,6 +457,15 @@ struct MetaData { long meta_rfc822_length; /* Cache of RFC822-translated msg length */ }; +/* Calls to AdjRefCount() are queued and deferred, so the user doesn't + * have to wait for various disk-intensive operations to complete synchronously. + * This is the record format. + */ +struct arcq { + long arcq_msgnum; /* Message number being adjusted */ + int arcq_delta; /* Adjustment ( usually 1 or -1 ) */ +}; + /* * Serialization routines use this struct to return a pointer and a length diff --git a/citadel/sysconfig.h b/citadel/sysconfig.h index 92dbee7a5..2db5f367c 100644 --- a/citadel/sysconfig.h +++ b/citadel/sysconfig.h @@ -112,7 +112,6 @@ #define PAGELOGROOM "Sent/Received Pages" #define SYSCONFIGROOM "Local System Configuration" #define SMTP_SPOOLOUT_ROOM "__CitadelSMTPspoolout__" -#define DELETED_MSGS_ROOM "__CitadelDeletedMessages__" /* * Where we keep messages containing the vCards that source our directory. It