* More glue code for the fulltext indexer.

diff --git a/citadel/ChangeLog b/citadel/ChangeLog
index e860cbfb5ec3920428d7bf48dd4154d13929ef45..7bf15f790661f32a893aaef980141aa0dfa3c1b9 100644 (file)
--- a/citadel/ChangeLog
+++ b/citadel/ChangeLog
@@ -1,4 +1,7 @@
  $Log$
+ Revision 647.5  2005/05/17 04:04:46  ajc
+ * More glue code for the fulltext indexer.
+
  Revision 647.4  2005/05/16 20:03:33  ajc
  * definition of struct CitControl moved from citadel.h to server.h
 
@@ -6699,3 +6702,4 @@ Sat Jul 11 00:20:48 EDT 1998 Nathan Bryant <bryant@cs.usm.maine.edu>
 
 Fri Jul 10 1998 Art Cancro <ajc@uncensored.citadel.org>
        * Initial CVS import
+

diff --git a/citadel/ft_wordbreaker.c b/citadel/ft_wordbreaker.c
index c9d9db259ca33f9ca1ea2891a9f45643e29f769e..4fc3f3aace3373f878e69762eb281fa25a80d337 100644 (file)
--- a/citadel/ft_wordbreaker.c
+++ b/citadel/ft_wordbreaker.c
@@ -44,3 +44,20 @@
 #include "ft_wordbreaker.h"
 
 
+void wordbreaker(char *text, int *num_tokens, int **tokens) {
+
+       int wb_num_tokens = 0;
+       int wb_num_alloc = 0;
+       int *wb_tokens = NULL;
+
+       wb_num_tokens = 3;
+       wb_tokens = malloc(wb_num_tokens * sizeof(int));
+
+       wb_tokens[0] = 6;
+       wb_tokens[1] = 7;       /* FIXME this obviously isn't a wordbreaker */
+       wb_tokens[2] = 8;
+
+       *num_tokens = wb_num_tokens;
+       *tokens = wb_tokens;
+}
+

diff --git a/citadel/ft_wordbreaker.h b/citadel/ft_wordbreaker.h
index d093489f514800f77e4fbd61804a1bd2e7f1fdca..97628357461eebe8cd6c8c09b9345c520f0278da 100644 (file)
--- a/citadel/ft_wordbreaker.h
+++ b/citadel/ft_wordbreaker.h
@@ -10,3 +10,5 @@
  * system knows it needs to throw away the existing index and rebuild it.
  */
 #define        FT_WORDBREAKER_ID       0x0001
+
+void wordbreaker(char *text, int *num_tokens, int **tokens);

diff --git a/citadel/ft_wordbreaker.o b/citadel/ft_wordbreaker.o
index 6e341fe906e8d87c0e104316d0843316b4cf30e0..67f2f693286b91545517729c16c4bcaa47c9ed25 100644 (file)
Binary files a/citadel/ft_wordbreaker.o and b/citadel/ft_wordbreaker.o differ

diff --git a/citadel/serv_fulltext.c b/citadel/serv_fulltext.c
index 6597988b649a8e8e81c22425d5f17ff241bcfb7f..37bfc3cd2a7940eb24d3764bba04187f2294cf19 100644 (file)
--- a/citadel/serv_fulltext.c
+++ b/citadel/serv_fulltext.c
@@ -52,13 +52,46 @@ int ft_num_msgs = 0;
 int ft_num_alloc = 0;
 
 
+/*
+ * Index or de-index a message.  (op == 1 to index, 0 to de-index)
+ */
+void ft_index_message(long msgnum, int op) {
+       struct CtdlMessage *msg;
+       int num_tokens = 0;
+       int *tokens = NULL;
+       int i;
+
+       msg = CtdlFetchMessage(msgnum, 1);
+       if (msg == NULL) return;
+
+       if (msg->cm_fields['M'] != NULL) {
+               wordbreaker(msg->cm_fields['M'], &num_tokens, &tokens);
+       }
+       CtdlFreeMessage(msg);
+
+       if (num_tokens > 0) {
+               for (i=0; i<num_tokens; ++i) {
+                       /* FIXME do something with this */
+                       lprintf(CTDL_DEBUG, "msg %ld, token %d\n",
+                               msgnum, tokens[i]);
+                       }
+               free(tokens);
+       }
+}
+
+
+
+/*
+ * Add a message to the list of those to be indexed.
+ */
 void ft_index_msg(long msgnum, void *userdata) {
 
        if ((msgnum > CitControl.MMfulltext) && (msgnum <= ft_newhighest)) {
                ++ft_num_msgs;
                if (ft_num_msgs > ft_num_alloc) {
                        ft_num_alloc += 1024;
-                       ft_newmsgs = realloc(ft_newmsgs, (ft_num_alloc * sizeof(long)));
+                       ft_newmsgs = realloc(ft_newmsgs,
+                               (ft_num_alloc * sizeof(long)));
                }
                ft_newmsgs[ft_num_msgs - 1] = msgnum;
        }
@@ -90,7 +123,9 @@ int longcmp(const void *rec1, const void *rec2) {
 }
 
 
-
+/*
+ * Begin the fulltext indexing process.  (Called as an EVT_TIMER event)
+ */
 void do_fulltext_indexing(void) {
        int i;
 
@@ -100,8 +135,10 @@ void do_fulltext_indexing(void) {
         * Check to see whether the fulltext index is up to date; if there
         * are no messages to index, don't waste any more time trying.
         */
-       lprintf(CTDL_DEBUG, "CitControl.MMhighest  = %ld\n", CitControl.MMhighest);
-       lprintf(CTDL_DEBUG, "CitControl.MMfulltext = %ld\n", CitControl.MMfulltext);
+       lprintf(CTDL_DEBUG, "CitControl.MMhighest  = %ld\n",
+               CitControl.MMhighest);
+       lprintf(CTDL_DEBUG, "CitControl.MMfulltext = %ld\n",
+               CitControl.MMfulltext);
        if (CitControl.MMfulltext >= CitControl.MMhighest) {
                lprintf(CTDL_DEBUG, "Nothing to do!\n");
                return;
@@ -121,25 +158,27 @@ void do_fulltext_indexing(void) {
         * Now go through each room and find messages to index.
         */
        ft_newhighest = CitControl.MMhighest;
-       ForEachRoom(ft_index_room, NULL);                               /* merge ptrs */
+       ForEachRoom(ft_index_room, NULL);       /* load all msg pointers */
 
        if (ft_num_msgs > 0) {
-               qsort(ft_newmsgs, ft_num_msgs, sizeof(long), longcmp);  /* sort */
-               if (i>1) for (i=0; i<(ft_num_msgs-1); ++i) {            /* purge dups */
+               qsort(ft_newmsgs, ft_num_msgs, sizeof(long), longcmp);
+               if (i>1) for (i=0; i<(ft_num_msgs-1); ++i) { /* purge dups */
                        if (ft_newmsgs[i] == ft_newmsgs[i+1]) {
-                               memmove(&ft_newmsgs[i], &ft_newmsgs[i+1], ((ft_num_msgs - i)*sizeof(long)));
+                               memmove(&ft_newmsgs[i], &ft_newmsgs[i+1],
+                                       ((ft_num_msgs - i)*sizeof(long)));
                                --ft_num_msgs;
                        }
                }
 
                /* Here it is ... do each message! */
                for (i=0; i<ft_num_msgs; ++i) {
-                       lprintf(CTDL_DEBUG, "FIXME INDEX %ld\n", ft_newmsgs[i]);
+                       ft_index_message(ft_newmsgs[i], 1);
                }
 
                free(ft_newmsgs);
                ft_num_msgs = 0;
                ft_num_alloc = 0;
+               ft_newmsgs = NULL;
        }
 
        lprintf(CTDL_DEBUG, "do_fulltext_indexing() finished\n");

diff --git a/citadel/serv_fulltext.o b/citadel/serv_fulltext.o
index e133d6b0b50a9c858f002c2c1bc5b4f9672d3dce..7661e3c19b4ddac642600165ef187258012ecae2 100644 (file)
Binary files a/citadel/serv_fulltext.o and b/citadel/serv_fulltext.o differ