4 * This module handles fulltext indexing of the message base.
17 #include <sys/types.h>
19 #if TIME_WITH_SYS_TIME
20 # include <sys/time.h>
24 # include <sys/time.h>
35 #include "sysdep_decls.h"
36 #include "citserver.h"
39 #include "serv_extensions.h"
45 #include "serv_fulltext.h"
46 #include "ft_wordbreaker.h"
49 long ft_newhighest = 0L;
50 long *ft_newmsgs = NULL;
56 * Index or de-index a message. (op == 1 to index, 0 to de-index)
58 void ft_index_message(long msgnum, int op) {
59 struct CtdlMessage *msg;
63 struct cdbdata *cdb_bucket;
67 msg = CtdlFetchMessage(msgnum, 1);
68 if (msg == NULL) return;
70 if (msg->cm_fields['M'] != NULL) {
71 wordbreaker(msg->cm_fields['M'], &num_tokens, &tokens);
76 for (i=0; i<num_tokens; ++i) {
78 /* Add the message to the relevant token bucket */
80 /* FIXME do "if op=1" ... */
82 /* FIXME lock the file */
83 cdb_bucket = cdb_fetch(CDB_FULLTEXT, &tokens[i], sizeof(int));
84 if (cdb_bucket == NULL) {
85 cdb_bucket = malloc(sizeof(struct cdbdata));
87 cdb_bucket->ptr = malloc(sizeof(long));
89 num_msgs = cdb_bucket->len / sizeof(long);
92 cdb_bucket->ptr = realloc(cdb_bucket->ptr, num_msgs*sizeof(long) );
93 msgs = (long *) cdb_bucket->ptr;
94 msgs[num_msgs - 1] = msgnum;
96 cdb_store(CDB_FULLTEXT, &tokens[i], sizeof(int),
97 cdb_bucket->ptr, num_msgs*sizeof(long) );
101 /* FIXME unlock the file */
111 * Add a message to the list of those to be indexed.
113 void ft_index_msg(long msgnum, void *userdata) {
115 if ((msgnum > CitControl.MMfulltext) && (msgnum <= ft_newhighest)) {
117 if (ft_num_msgs > ft_num_alloc) {
118 ft_num_alloc += 1024;
119 ft_newmsgs = realloc(ft_newmsgs,
120 (ft_num_alloc * sizeof(long)));
122 ft_newmsgs[ft_num_msgs - 1] = msgnum;
128 * Scan a room for messages to index.
130 void ft_index_room(struct ctdlroom *qrbuf, void *data)
132 getroom(&CC->room, qrbuf->QRname);
133 CtdlForEachMessage(MSGS_ALL, 0L, NULL, NULL, ft_index_msg, NULL);
140 int longcmp(const void *rec1, const void *rec2) {
143 i1 = *(const long *)rec1;
144 i2 = *(const long *)rec2;
146 if (i1 > i2) return(1);
147 if (i1 < i2) return(-1);
153 * Begin the fulltext indexing process. (Called as an EVT_TIMER event)
155 void do_fulltext_indexing(void) {
157 static time_t last_index = 0L;
160 * Make sure we don't run the indexer too frequently.
161 * FIXME move the setting into config
163 if ( (time(NULL) - last_index) < 300L) {
168 * Check to see whether the fulltext index is up to date; if there
169 * are no messages to index, don't waste any more time trying.
171 lprintf(CTDL_DEBUG, "CitControl.MMhighest = %ld\n",
172 CitControl.MMhighest);
173 lprintf(CTDL_DEBUG, "CitControl.MMfulltext = %ld\n",
174 CitControl.MMfulltext);
175 if (CitControl.MMfulltext >= CitControl.MMhighest) {
180 lprintf(CTDL_DEBUG, "do_fulltext_indexing() started\n");
183 * If we've switched wordbreaker modules, burn the index and start
184 * over. FIXME write this...
186 if (CitControl.fulltext_wordbreaker != FT_WORDBREAKER_ID) {
187 lprintf(CTDL_INFO, "(re)initializing full text index\n");
188 cdb_trunc(CDB_FULLTEXT);
189 CitControl.MMfulltext = 0L;
194 * Now go through each room and find messages to index.
196 ft_newhighest = CitControl.MMhighest;
197 ForEachRoom(ft_index_room, NULL); /* load all msg pointers */
199 if (ft_num_msgs > 0) {
200 qsort(ft_newmsgs, ft_num_msgs, sizeof(long), longcmp);
201 if (i>1) for (i=0; i<(ft_num_msgs-1); ++i) { /* purge dups */
202 if (ft_newmsgs[i] == ft_newmsgs[i+1]) {
203 memmove(&ft_newmsgs[i], &ft_newmsgs[i+1],
204 ((ft_num_msgs - i)*sizeof(long)));
209 /* Here it is ... do each message! */
210 for (i=0; i<ft_num_msgs; ++i) {
211 ft_index_message(ft_newmsgs[i], 1);
220 /* Save our place so we don't have to do this again */
221 CitControl.MMfulltext = ft_newhighest;
222 CitControl.fulltext_wordbreaker = FT_WORDBREAKER_ID;
224 last_index = time(NULL);
226 lprintf(CTDL_DEBUG, "do_fulltext_indexing() finished\n");
232 * Tentative form of our search command
234 void cmd_srch(char *argbuf) {
235 char search_string[256];
240 if (CtdlAccessCheck(ac_logged_in)) return;
241 extract_token(search_string, argbuf, 0, '|', sizeof search_string);
242 wordbreaker(search_string, &num_tokens, &tokens);
244 cprintf("%d msgs matching search words:\n", LISTING_FOLLOWS);
245 if (num_tokens > 0) {
246 for (i=0; i<num_tokens; ++i) {
248 cprintf("FIXME search for token %d\n", tokens[i]);
257 /*****************************************************************************/
259 char *serv_fulltext_init(void)
261 CtdlRegisterSessionHook(do_fulltext_indexing, EVT_TIMER);
262 CtdlRegisterProtoHook(cmd_srch, "SRCH", "Full text search");