4 * This module handles fulltext indexing of the message base.
17 #include <sys/types.h>
19 #if TIME_WITH_SYS_TIME
20 # include <sys/time.h>
24 # include <sys/time.h>
35 #include "sysdep_decls.h"
36 #include "citserver.h"
39 #include "serv_extensions.h"
45 #include "serv_fulltext.h"
46 #include "ft_wordbreaker.h"
49 long ft_newhighest = 0L;
50 long *ft_newmsgs = NULL;
56 * Index or de-index a message. (op == 1 to index, 0 to de-index)
58 void ft_index_message(long msgnum, int op) {
59 struct CtdlMessage *msg;
63 struct cdbdata *cdb_bucket;
67 msg = CtdlFetchMessage(msgnum, 1);
68 if (msg == NULL) return;
70 if (msg->cm_fields['M'] != NULL) {
71 wordbreaker(msg->cm_fields['M'], &num_tokens, &tokens);
76 for (i=0; i<num_tokens; ++i) {
78 /* Add the message to the relevant token bucket */
80 /* FIXME do "if op=1" ... */
82 /* FIXME lock the file */
83 cdb_bucket = cdb_fetch(CDB_FULLTEXT, &tokens[i], sizeof(int));
84 if (cdb_bucket == NULL) {
85 cdb_bucket = malloc(sizeof(struct cdbdata));
87 cdb_bucket->ptr = malloc(sizeof(long));
91 num_msgs = cdb_bucket->len / sizeof(long);
95 cdb_bucket->ptr = realloc(cdb_bucket->ptr, num_msgs*sizeof(long) );
96 msgs = (long *) cdb_bucket->ptr;
97 msgs[num_msgs - 1] = msgnum;
99 cdb_store(CDB_FULLTEXT, &tokens[i], sizeof(int),
100 cdb_bucket->ptr, num_msgs*sizeof(long) );
102 cdb_free(cdb_bucket);
104 /* FIXME unlock the file */
114 * Add a message to the list of those to be indexed.
116 void ft_index_msg(long msgnum, void *userdata) {
118 if ((msgnum > CitControl.MMfulltext) && (msgnum <= ft_newhighest)) {
120 if (ft_num_msgs > ft_num_alloc) {
121 ft_num_alloc += 1024;
122 ft_newmsgs = realloc(ft_newmsgs,
123 (ft_num_alloc * sizeof(long)));
125 ft_newmsgs[ft_num_msgs - 1] = msgnum;
131 * Scan a room for messages to index.
133 void ft_index_room(struct ctdlroom *qrbuf, void *data)
135 getroom(&CC->room, qrbuf->QRname);
136 CtdlForEachMessage(MSGS_ALL, 0L, NULL, NULL, ft_index_msg, NULL);
143 int longcmp(const void *rec1, const void *rec2) {
146 i1 = *(const long *)rec1;
147 i2 = *(const long *)rec2;
149 if (i1 > i2) return(1);
150 if (i1 < i2) return(-1);
156 * Begin the fulltext indexing process. (Called as an EVT_TIMER event)
158 void do_fulltext_indexing(void) {
160 static time_t last_index = 0L;
163 * Make sure we don't run the indexer too frequently.
164 * FIXME move the setting into config
166 if ( (time(NULL) - last_index) < 300L) {
171 * Check to see whether the fulltext index is up to date; if there
172 * are no messages to index, don't waste any more time trying.
174 lprintf(CTDL_DEBUG, "CitControl.MMhighest = %ld\n",
175 CitControl.MMhighest);
176 lprintf(CTDL_DEBUG, "CitControl.MMfulltext = %ld\n",
177 CitControl.MMfulltext);
178 if (CitControl.MMfulltext >= CitControl.MMhighest) {
183 lprintf(CTDL_DEBUG, "do_fulltext_indexing() started\n");
186 * If we've switched wordbreaker modules, burn the index and start
187 * over. FIXME write this...
189 if (CitControl.fulltext_wordbreaker != FT_WORDBREAKER_ID) {
190 lprintf(CTDL_INFO, "(re)initializing full text index\n");
191 cdb_trunc(CDB_FULLTEXT);
192 CitControl.MMfulltext = 0L;
197 * Now go through each room and find messages to index.
199 ft_newhighest = CitControl.MMhighest;
200 ForEachRoom(ft_index_room, NULL); /* load all msg pointers */
202 if (ft_num_msgs > 0) {
203 qsort(ft_newmsgs, ft_num_msgs, sizeof(long), longcmp);
204 if (i>1) for (i=0; i<(ft_num_msgs-1); ++i) { /* purge dups */
205 if (ft_newmsgs[i] == ft_newmsgs[i+1]) {
206 memmove(&ft_newmsgs[i], &ft_newmsgs[i+1],
207 ((ft_num_msgs - i)*sizeof(long)));
212 /* Here it is ... do each message! */
213 for (i=0; i<ft_num_msgs; ++i) {
214 ft_index_message(ft_newmsgs[i], 1);
223 /* Save our place so we don't have to do this again */
224 CitControl.MMfulltext = ft_newhighest;
225 CitControl.fulltext_wordbreaker = FT_WORDBREAKER_ID;
227 last_index = time(NULL);
229 lprintf(CTDL_DEBUG, "do_fulltext_indexing() finished\n");
235 * Tentative form of our search command
237 void cmd_srch(char *argbuf) {
238 char search_string[256];
242 struct cdbdata *cdb_bucket;
246 if (CtdlAccessCheck(ac_logged_in)) return;
247 extract_token(search_string, argbuf, 0, '|', sizeof search_string);
248 wordbreaker(search_string, &num_tokens, &tokens);
250 cprintf("%d msgs matching search words:\n", LISTING_FOLLOWS);
251 if (num_tokens > 0) {
252 for (i=0; i<num_tokens; ++i) {
254 /* search for tokens[i] */
255 cdb_bucket = cdb_fetch(CDB_FULLTEXT, &tokens[i], sizeof(int));
256 if (cdb_bucket != NULL) {
257 num_msgs = cdb_bucket->len / sizeof(long);
258 msgs = (long *)cdb_bucket->ptr;
259 cdb_free(cdb_bucket);
260 for (j=0; j<num_msgs; ++j) {
261 cprintf("Token <%d> is in msg <%ld>\n", tokens[i], msgs[j]);
272 /*****************************************************************************/
274 char *serv_fulltext_init(void)
276 CtdlRegisterSessionHook(do_fulltext_indexing, EVT_TIMER);
277 CtdlRegisterProtoHook(cmd_srch, "SRCH", "Full text search");