Rewrite of the indexer is complete!

No more search hooks.  Modules can call each other now and that's a lot simpler.
We also eliminated the index cache, because it made things a lot more complex, and
a modern database is going to mmap most of the index as it gets used anyway.

diff --git a/citadel/server/modules/autocompletion/serv_autocompletion.c b/citadel/server/modules/autocompletion/serv_autocompletion.c
index 3f025521fe76b4250c3d8851ddf68671758a7ac5..d8e04cb798166d4d84e900b9e2f436e200a9862e 100644 (file)
--- a/citadel/server/modules/autocompletion/serv_autocompletion.c
+++ b/citadel/server/modules/autocompletion/serv_autocompletion.c
@@ -9,6 +9,7 @@
 #include "serv_autocompletion.h"
 #include "../../config.h"
 #include "../../room_ops.h"
+#include "../fulltext/serv_fulltext.h"
 
 
 // Convert a structured name into a friendly name.  Caller must free the
@@ -109,13 +110,12 @@ void cmd_auto(char *argbuf) {
        char search_string[256];
        long *msglist = NULL;
        int num_msgs = 0;
-       long *fts_msgs = NULL;
-       int fts_num_msgs = 0;
        int r = 0;
        int i = 0;
        int j = 0;
        int search_match = 0;
        char *rooms_to_try[] = { USERCONTACTSROOM, ADDRESS_BOOK_ROOM };
+       Array *fts = NULL;
                
        if (CtdlAccessCheck(ac_logged_in)) return;
        extract_token(search_string, argbuf, 0, '|', sizeof search_string);
@@ -134,21 +134,23 @@ void cmd_auto(char *argbuf) {
 
                        // Search-reduce the results if we have the full text index available
                        if (CtdlGetConfigInt("c_enable_fulltext")) {
-                               CtdlModuleDoSearch(&fts_num_msgs, &fts_msgs, search_string, "fulltext");
-                               if (fts_msgs) {
+                               fts = CtdlFullTextSearch(search_string);
+                               if (fts) {
                                        for (i=0; i<num_msgs; ++i) {
                                                search_match = 0;
-                                               for (j=0; j<fts_num_msgs; ++j) {
-                                                       if (msglist[i] == fts_msgs[j]) {
+                                               for (j=0; j<array_len(fts); ++j) {
+                                                       long smsgnum;
+                                                       memcpy(&smsgnum, array_get_element_at(fts, j), sizeof(long));
+                                                       if (msglist[i] == smsgnum) {
                                                                search_match = 1;
-                                                               j = fts_num_msgs + 1;   // end the search 
+                                                               j = array_len(fts) + 1; // end the search 
                                                        }
                                                }
                                                if (!search_match) {
                                                        msglist[i] = 0;                 // invalidate this result
                                                }
                                        }
-                                       free(fts_msgs);
+                                       array_free(fts);
                                }
                                else {
                                        // If no results, invalidate the whole list

diff --git a/citadel/server/modules/fulltext/serv_fulltext.c b/citadel/server/modules/fulltext/serv_fulltext.c
index ede6fb35592584070a91a06635d8f9d718995c12..7f7dd7959b2a984eb91eb41e1873fed92bcf45ab 100644 (file)
--- a/citadel/server/modules/fulltext/serv_fulltext.c
+++ b/citadel/server/modules/fulltext/serv_fulltext.c
@@ -249,7 +249,7 @@ void do_fulltext_indexing(void) {
 // API call to perform searches.
 // (This one does the "all of these words" search.)
 // Caller is responsible for freeing the message list.
-Array *ft_search(const char *search_string) {
+Array *CtdlFullTextSearch(const char *search_string) {
        int i, j, tok;
        struct cdbdata cdb_bucket;
        long msgnum, smsgnum;
@@ -313,7 +313,7 @@ void cmd_srch(char *argbuf) {
        }
 
        extract_token(search_string, argbuf, 0, '|', sizeof search_string);
-       matches = ft_search(search_string);
+       matches = CtdlFullTextSearch(search_string);
 
        cprintf("%d %d msgs match all search words:\n", LISTING_FOLLOWS, array_len(matches));
        if ((matches != NULL) && (array_len(matches) > 0)) {
@@ -344,7 +344,6 @@ char *ctdl_module_init_fulltext(void) {
        if (!threading) {
                CtdlRegisterProtoHook(cmd_srch, "SRCH", "Full text search");
                CtdlRegisterDeleteHook(ft_delete_remove);
-               //CtdlRegisterSearchFuncHook(ft_search, "fulltext");
                CtdlRegisterSessionHook(do_fulltext_indexing, EVT_TIMER, PRIO_CLEANUP + 300);
        }
        // return our module name for the log

diff --git a/citadel/server/modules/fulltext/serv_fulltext.h b/citadel/server/modules/fulltext/serv_fulltext.h
index 449a4b8e6b7fb3421208d9a8373537c0884e7bb0..e96a8f60c6945b0ffabb180c77e72233e1aa143d 100644 (file)
--- a/citadel/server/modules/fulltext/serv_fulltext.h
+++ b/citadel/server/modules/fulltext/serv_fulltext.h
@@ -4,4 +4,4 @@
 // is subject to the terms of the GNU General Public License, version 3.
 
 void ft_index_message(long msgnum, int op);
-Array *ft_search(const char *search_string);
+Array *CtdlFullTextSearch(const char *search_string);

diff --git a/citadel/server/modules/imap/imap_search.c b/citadel/server/modules/imap/imap_search.c
index 03678d296db47ab9819d4b22036f1d407d01fa13..1c70de8abdaa8b129c23bcc28af6b38dbe72bae0 100644 (file)
--- a/citadel/server/modules/imap/imap_search.c
+++ b/citadel/server/modules/imap/imap_search.c
@@ -13,8 +13,6 @@
  */
 
 #include "../../ctdl_module.h"
-
-
 #include "../../sysdep.h"
 #include <stdlib.h>
 #include <unistd.h>
@@ -45,6 +43,7 @@
 #include "imap_fetch.h"
 #include "imap_search.h"
 #include "../../genstamp.h"
+#include "../fulltext/serv_fulltext.h"
 
 
 /*
@@ -525,10 +524,9 @@ int imap_do_search_msg(int seq, struct CtdlMessage *supplied_msg,
 void imap_do_search(int num_items, ConstStr *itemlist, int is_uid) {
        citimap *Imap = IMAP;
        int i, j, k;
-       int fts_num_msgs = 0;
-       long *fts_msgs = NULL;
        int is_in_list = 0;
        int num_results = 0;
+       Array *fts = NULL;
 
        /* Strip parentheses.  We realize that this method will not work
         * in all cases, but it seems to work with all currently available
@@ -549,13 +547,15 @@ void imap_do_search(int num_items, ConstStr *itemlist, int is_uid) {
         */
        if (CtdlGetConfigInt("c_enable_fulltext")) for (i=0; i<(num_items-1); ++i) {
                if (!strcasecmp(itemlist[i].Key, "BODY")) {
-                       CtdlModuleDoSearch(&fts_num_msgs, &fts_msgs, itemlist[i+1].Key, "fulltext");
-                       if (fts_num_msgs > 0) {
+                       fts = CtdlFullTextSearch(itemlist[i+1].Key);
+                       if ((fts) && (array_len(fts) > 0)) {
                                for (j=0; j < Imap->num_msgs; ++j) {
                                        if (Imap->flags[j] & IMAP_SELECTED) {
                                                is_in_list = 0;
-                                               for (k=0; k<fts_num_msgs; ++k) {
-                                                       if (Imap->msgids[j] == fts_msgs[k]) {
+                                               for (k=0; k<array_len(fts); ++k) {
+                                                       long smsgnum;
+                                                       memcpy(&smsgnum, array_get_element_at(fts, k), sizeof(long));
+                                                       if (Imap->msgids[j] == smsgnum) {
                                                                ++is_in_list;
                                                        }
                                                }
@@ -570,8 +570,8 @@ void imap_do_search(int num_items, ConstStr *itemlist, int is_uid) {
                                        Imap->flags[j] = Imap->flags[j] & ~IMAP_SELECTED;
                                }
                        }
-                       if (fts_msgs) {
-                               free(fts_msgs);
+                       if (fts) {
+                               array_free(fts);
                        }
                }
        }

diff --git a/citadel/server/msgbase.c b/citadel/server/msgbase.c
index 60e1140cf3b9f05a6c3958b587e85f07c96c1d0c..66a56da97d9eca29cae2e29fef0a3ff07e3e4fcf 100644 (file)
--- a/citadel/server/msgbase.c
+++ b/citadel/server/msgbase.c
@@ -24,6 +24,7 @@
 #include "euidindex.h"
 #include "msgbase.h"
 #include "journaling.h"
+#include "modules/fulltext/serv_fulltext.h"
 
 struct addresses_to_be_filed *atbf = NULL;
 
@@ -615,11 +616,10 @@ int CtdlForEachMessage(int mode, long ref, char *search_string,
        int is_seen = 0;
        long lastold = 0L;
        int printed_lastold = 0;
-       int num_search_msgs = 0;
-       long *search_msgs = NULL;
        regex_t re;
        int need_to_free_re = 0;
        regmatch_t pm;
+       Array *search = NULL;
 
        if ((content_type) && (!IsEmptyStr(content_type))) {
                regcomp(&re, content_type, 0);
@@ -719,18 +719,19 @@ int CtdlForEachMessage(int mode, long ref, char *search_string,
                 * NULL means use any search function available.
                 * otherwise replace with a char * to name of search routine
                 */
-               CtdlModuleDoSearch(search_string, "fulltext");
-               //FIXME FOOFOO DO ARRAY
+               search = CtdlFullTextSearch(search_string);
 
-               if (num_search_msgs > 0) {
+               if (array_len(search) > 0) {
        
                        int orig_num_msgs;
 
                        orig_num_msgs = num_msgs;
                        num_msgs = 0;
                        for (i=0; i<orig_num_msgs; ++i) {
-                               for (j=0; j<num_search_msgs; ++j) {
-                                       if (msglist[i] == search_msgs[j]) {
+                               for (j=0; j<array_len(search); ++j) {
+                                       long smsgnum;
+                                       memcpy(&smsgnum, array_get_element_at(search, j), sizeof(long));
+                                       if (msglist[i] == smsgnum) {
                                                msglist[num_msgs++] = msglist[i];
                                        }
                                }
@@ -739,7 +740,7 @@ int CtdlForEachMessage(int mode, long ref, char *search_string,
                else {
                        num_msgs = 0;   /* No messages qualify */
                }
-               if (search_msgs != NULL) free(search_msgs);
+               if (search != NULL) array_free(search);
 
                /* Now that we've purged messages which don't contain the search
                 * string, treat a MSGS_SEARCH just like a MSGS_ALL from this