4 * This module handles fulltext indexing of the message base.
17 #include <sys/types.h>
19 #if TIME_WITH_SYS_TIME
20 # include <sys/time.h>
24 # include <sys/time.h>
35 #include "sysdep_decls.h"
36 #include "citserver.h"
39 #include "serv_extensions.h"
45 #include "serv_fulltext.h"
46 #include "ft_wordbreaker.h"
49 long ft_newhighest = 0L;
50 long *ft_newmsgs = NULL;
56 * Index or de-index a message. (op == 1 to index, 0 to de-index)
58 void ft_index_message(long msgnum, int op) {
59 struct CtdlMessage *msg;
63 struct cdbdata *cdb_bucket;
66 msg = CtdlFetchMessage(msgnum, 1);
67 if (msg == NULL) return;
69 if (msg->cm_fields['M'] != NULL) {
70 wordbreaker(msg->cm_fields['M'], &num_tokens, &tokens);
75 for (i=0; i<num_tokens; ++i) {
77 /* Add the message to the relevant token bucket */
78 lprintf(CTDL_DEBUG, "msg %ld, token %d\n", msgnum, tokens[i]);
80 /* FIXME lock the file */
81 cdb_bucket = cdb_fetch(CDB_FULLTEXT, &tokens[i], sizeof(long));
82 if (cdb_bucket == NULL) {
83 cdb_bucket = malloc(sizeof(struct cdbdata));
85 cdb_bucket->ptr = malloc(sizeof(long));
87 num_msgs = cdb_bucket->len / sizeof(long);
89 /* FIXME finish this */
93 /* FIXME unlock the file */
103 * Add a message to the list of those to be indexed.
105 void ft_index_msg(long msgnum, void *userdata) {
107 if ((msgnum > CitControl.MMfulltext) && (msgnum <= ft_newhighest)) {
109 if (ft_num_msgs > ft_num_alloc) {
110 ft_num_alloc += 1024;
111 ft_newmsgs = realloc(ft_newmsgs,
112 (ft_num_alloc * sizeof(long)));
114 ft_newmsgs[ft_num_msgs - 1] = msgnum;
120 * Scan a room for messages to index.
122 void ft_index_room(struct ctdlroom *qrbuf, void *data)
124 getroom(&CC->room, qrbuf->QRname);
125 CtdlForEachMessage(MSGS_ALL, 0L, NULL, NULL, ft_index_msg, NULL);
132 int longcmp(const void *rec1, const void *rec2) {
135 i1 = *(const long *)rec1;
136 i2 = *(const long *)rec2;
138 if (i1 > i2) return(1);
139 if (i1 < i2) return(-1);
145 * Begin the fulltext indexing process. (Called as an EVT_TIMER event)
147 void do_fulltext_indexing(void) {
149 static time_t last_index = 0L;
152 * Make sure we don't run the indexer too frequently.
153 * FIXME move the setting into config
155 if ( (time(NULL) - last_index) < 300L) {
160 * Check to see whether the fulltext index is up to date; if there
161 * are no messages to index, don't waste any more time trying.
163 lprintf(CTDL_DEBUG, "CitControl.MMhighest = %ld\n",
164 CitControl.MMhighest);
165 lprintf(CTDL_DEBUG, "CitControl.MMfulltext = %ld\n",
166 CitControl.MMfulltext);
167 if (CitControl.MMfulltext >= CitControl.MMhighest) {
172 lprintf(CTDL_DEBUG, "do_fulltext_indexing() started\n");
175 * If we've switched wordbreaker modules, burn the index and start
176 * over. FIXME write this...
178 if (CitControl.fulltext_wordbreaker != FT_WORDBREAKER_ID) {
179 lprintf(CTDL_INFO, "(re)initializing full text index\n");
180 cdb_trunc(CDB_FULLTEXT);
181 CitControl.MMfulltext = 0L;
186 * Now go through each room and find messages to index.
188 ft_newhighest = CitControl.MMhighest;
189 ForEachRoom(ft_index_room, NULL); /* load all msg pointers */
191 if (ft_num_msgs > 0) {
192 qsort(ft_newmsgs, ft_num_msgs, sizeof(long), longcmp);
193 if (i>1) for (i=0; i<(ft_num_msgs-1); ++i) { /* purge dups */
194 if (ft_newmsgs[i] == ft_newmsgs[i+1]) {
195 memmove(&ft_newmsgs[i], &ft_newmsgs[i+1],
196 ((ft_num_msgs - i)*sizeof(long)));
201 /* Here it is ... do each message! */
202 for (i=0; i<ft_num_msgs; ++i) {
203 ft_index_message(ft_newmsgs[i], 1);
212 lprintf(CTDL_DEBUG, "do_fulltext_indexing() finished\n");
217 /*****************************************************************************/
219 char *serv_fulltext_init(void)
221 CtdlRegisterSessionHook(do_fulltext_indexing, EVT_TIMER);