MSGS command can now do full text search on the room

[citadel.git] / citadel / serv_fulltext.c

diff --git a/citadel/serv_fulltext.c b/citadel/serv_fulltext.c
index 29ea751cec6d7ca0286695ee4e2059f82fa35ae8..ab78a7428c2ac08b40b5de6eb8c3ace0aea8baaf 100644 (file)
--- a/citadel/serv_fulltext.c
+++ b/citadel/serv_fulltext.c
@@ -127,7 +127,7 @@ void ft_index_message(long msgnum, int op) {
        wordbreaker(msgtext, &num_tokens, &tokens);
        free(msgtext);
 
-       lprintf(CTDL_DEBUG, "Indexing message %ld...\n", msgnum);
+       lprintf(CTDL_DEBUG, "Indexing message %ld [%d tokens]\n", msgnum, num_tokens);
        if (num_tokens > 0) {
                for (i=0; i<num_tokens; ++i) {
 
@@ -136,40 +136,45 @@ void ft_index_message(long msgnum, int op) {
                        /* search for tokens[i] */
                        tok = tokens[i];
 
-                       /* fetch the bucket, Liza */
-                       if (ftc_msgs[tok] == NULL) {
-                               cdb_bucket = cdb_fetch(CDB_FULLTEXT, &tok, sizeof(int));
-                               if (cdb_bucket != NULL) {
-                                       ftc_num_msgs[tok] = cdb_bucket->len / sizeof(long);
-                                       ftc_msgs[tok] = (long *)cdb_bucket->ptr;
-                                       cdb_bucket->ptr = NULL;
-                                       cdb_free(cdb_bucket);
+                       if ( (tok >= 0) && (tok <= 65535) ) {
+                               /* fetch the bucket, Liza */
+                               if (ftc_msgs[tok] == NULL) {
+                                       cdb_bucket = cdb_fetch(CDB_FULLTEXT, &tok, sizeof(int));
+                                       if (cdb_bucket != NULL) {
+                                               ftc_num_msgs[tok] = cdb_bucket->len / sizeof(long);
+                                               ftc_msgs[tok] = (long *)cdb_bucket->ptr;
+                                               cdb_bucket->ptr = NULL;
+                                               cdb_free(cdb_bucket);
+                                       }
+                                       else {
+                                               ftc_num_msgs[tok] = 0;
+                                               ftc_msgs[tok] = malloc(sizeof(long));
+                                       }
                                }
-                               else {
-                                       ftc_num_msgs[tok] = 0;
-                                       ftc_msgs[tok] = malloc(sizeof(long));
+       
+       
+                               if (op == 1) {  /* add to index */
+                                       ++ftc_num_msgs[tok];
+                                       ftc_msgs[tok] = realloc(ftc_msgs[tok],
+                                                               ftc_num_msgs[tok]*sizeof(long));
+                                       ftc_msgs[tok][ftc_num_msgs[tok] - 1] = msgnum;
                                }
-                       }
-
-
-                       if (op == 1) {  /* add to index */
-                               ++ftc_num_msgs[tok];
-                               ftc_msgs[tok] = realloc(ftc_msgs[tok],
-                                                       ftc_num_msgs[tok]*sizeof(long));
-                               ftc_msgs[tok][ftc_num_msgs[tok] - 1] = msgnum;
-                       }
-
-                       if (op == 0) {  /* remove from index */
-                               if (ftc_num_msgs[tok] >= 1) {
-                                       for (j=0; j<ftc_num_msgs[tok]; ++j) {
-                                               if (ftc_msgs[tok][j] == msgnum) {
-                                                       memmove(&ftc_msgs[tok][j], &ftc_msgs[tok][j+1], ((ftc_num_msgs[tok] - j - 1)*sizeof(long)));
-                                                       --ftc_num_msgs[tok];
-                                                       --j;
+       
+                               if (op == 0) {  /* remove from index */
+                                       if (ftc_num_msgs[tok] >= 1) {
+                                               for (j=0; j<ftc_num_msgs[tok]; ++j) {
+                                                       if (ftc_msgs[tok][j] == msgnum) {
+                                                               memmove(&ftc_msgs[tok][j], &ftc_msgs[tok][j+1], ((ftc_num_msgs[tok] - j - 1)*sizeof(long)));
+                                                               --ftc_num_msgs[tok];
+                                                               --j;
+                                                       }
                                                }
                                        }
                                }
                        }
+                       else {
+                               lprintf(CTDL_ALERT, "Invalid token %d !!\n", tok);
+                       }
 
                        /* FIXME do we need to unlock the file here? */
                }
@@ -203,7 +208,7 @@ void ft_index_msg(long msgnum, void *userdata) {
 void ft_index_room(struct ctdlroom *qrbuf, void *data)
 {
        getroom(&CC->room, qrbuf->QRname);
-       CtdlForEachMessage(MSGS_ALL, 0L, NULL, NULL, ft_index_msg, NULL);
+       CtdlForEachMessage(MSGS_ALL, 0L, NULL, NULL, NULL, ft_index_msg, NULL);
 }
 
 
@@ -244,12 +249,16 @@ void do_fulltext_indexing(void) {
         * If we've switched wordbreaker modules, burn the index and start
         * over.
         */
+       begin_critical_section(S_CONTROL);
+       lprintf(CTDL_DEBUG, "wb ver on disk = %d, code ver = %d\n",
+                       CitControl.fulltext_wordbreaker, FT_WORDBREAKER_ID);
        if (CitControl.fulltext_wordbreaker != FT_WORDBREAKER_ID) {
                lprintf(CTDL_INFO, "(re)initializing full text index\n");
                cdb_trunc(CDB_FULLTEXT);
                CitControl.MMfulltext = 0L;
                put_control();
        }
+       end_critical_section(S_CONTROL);
 
        /*
         * Now go through each room and find messages to index.
@@ -287,6 +296,13 @@ void do_fulltext_indexing(void) {
                                break;
                        }
 
+                       /* Check to see if we have to maybe flush to disk */
+                       if (i >= FT_MAX_CACHE) {
+                               lprintf(CTDL_DEBUG, "Time to flush.\n");
+                               ft_newhighest = ft_newmsgs[i];
+                               break;
+                       }
+
                }
 
                free(ft_newmsgs);
@@ -297,9 +313,11 @@ void do_fulltext_indexing(void) {
 
        /* Save our place so we don't have to do this again */
        ft_flush_cache();
+       begin_critical_section(S_CONTROL);
        CitControl.MMfulltext = ft_newhighest;
        CitControl.fulltext_wordbreaker = FT_WORDBREAKER_ID;
        put_control();
+       end_critical_section(S_CONTROL);
        last_index = time(NULL);
 
        lprintf(CTDL_DEBUG, "do_fulltext_indexing() finished\n");
@@ -410,7 +428,7 @@ void ft_search(int *fts_num_msgs, long **fts_msgs, char *search_string) {
 
 
 /*
- * Tentative form of a search command
+ * This search command is for diagnostic purposes and may be removed or replaced.
  */
 void cmd_srch(char *argbuf) {
        int num_msgs = 0;
@@ -423,6 +441,7 @@ void cmd_srch(char *argbuf) {
        if (!config.c_enable_fulltext) {
                cprintf("%d Full text index is not enabled on this server.\n",
                        ERROR + CMD_NOT_SUPPORTED);
+               return;
        }
 
        extract_token(search_string, argbuf, 0, '|', sizeof search_string);