* Finished the indexer and the first part of the search function...

diff --git a/citadel/ChangeLog b/citadel/ChangeLog
index 08e13c69a1b24ce52baf6a3503200b9c6f57f9f8..e33dff34956d1d1b77363ad25d6c2c875cc71ce4 100644 (file)
--- a/citadel/ChangeLog
+++ b/citadel/ChangeLog
@@ -1,4 +1,7 @@
  $Log$
+ Revision 647.8  2005/05/18 03:22:27  ajc
+ * Finished the indexer and the first part of the search function...
+
  Revision 647.7  2005/05/17 20:36:48  ajc
  * Indexer is completed; also began work on the search function itself.
    Still need to add de-indexing so deleted messages are removed from index.
@@ -6709,4 +6712,3 @@ Sat Jul 11 00:20:48 EDT 1998 Nathan Bryant <bryant@cs.usm.maine.edu>
 
 Fri Jul 10 1998 Art Cancro <ajc@uncensored.citadel.org>
        * Initial CVS import
-

diff --git a/citadel/crc16.o b/citadel/crc16.o
index bf0808ca264cb5ed450ecc82cd0d43d0f190142d..b079e8a093c7088976842a377501c370cf057ee4 100644 (file)
Binary files a/citadel/crc16.o and b/citadel/crc16.o differ

diff --git a/citadel/ft_wordbreaker.c b/citadel/ft_wordbreaker.c
index f330ac573b162dc189eb70c5e990de639cfa1f1a..fc59ee0b7edb9bb04d9f2950b31b379e4d82d8cc 100644 (file)
--- a/citadel/ft_wordbreaker.c
+++ b/citadel/ft_wordbreaker.c
@@ -45,6 +45,20 @@
 #include "ft_wordbreaker.h"
 #include "crc16.h"
 
+/*
+ * Compare function
+ */
+int intcmp(const void *rec1, const void *rec2) {
+       int i1, i2;
+
+       i1 = *(const int *)rec1;
+       i2 = *(const int *)rec2;
+
+       if (i1 > i2) return(1);
+       if (i1 < i2) return(-1);
+       return(0);
+}
+
 
 void wordbreaker(char *text, int *num_tokens, int **tokens) {
 
@@ -113,6 +127,18 @@ void wordbreaker(char *text, int *num_tokens, int **tokens) {
                }
        }
 
+       /* sort and purge dups */
+       if (wb_num_tokens > 1) {
+               qsort(wb_tokens, wb_num_tokens, sizeof(int), intcmp);
+               for (i=0; i<(wb_num_tokens-1); ++i) {
+                       if (wb_tokens[i] == wb_tokens[i+1]) {
+                               memmove(&wb_tokens[i], &wb_tokens[i+1],
+                                       ((wb_num_tokens - i)*sizeof(int)));
+                               --wb_num_tokens;
+                       }
+               }
+       }
+
        *num_tokens = wb_num_tokens;
        *tokens = wb_tokens;
 }

diff --git a/citadel/ft_wordbreaker.h b/citadel/ft_wordbreaker.h
index 843c72a72ec82940e18e5657638de3967df1c0c8..15166cf1112d0f52f9e080f2fab6b12aa2dd1bff 100644 (file)
--- a/citadel/ft_wordbreaker.h
+++ b/citadel/ft_wordbreaker.h
@@ -9,7 +9,7 @@
  * later on, or even if we update this one, we can use a different ID so the
  * system knows it needs to throw away the existing index and rebuild it.
  */
-#define        FT_WORDBREAKER_ID       0x0003
+#define        FT_WORDBREAKER_ID       0x0008
 
 /*
  * Minimum and maximum length of words to index

diff --git a/citadel/ft_wordbreaker.o b/citadel/ft_wordbreaker.o
index 9c10c42c68f8ec3967df2e577f6a30d1cfb85a59..c4988d38e7c0719a5890f42438ca676d04e412d3 100644 (file)
Binary files a/citadel/ft_wordbreaker.o and b/citadel/ft_wordbreaker.o differ

diff --git a/citadel/serv_fulltext.c b/citadel/serv_fulltext.c
index 8c5f720f8d89132b2e7bb5488d4f8c9bc0fe9627..0b29872d596c81b393d806798bb7dfe7fca4efda 100644 (file)
--- a/citadel/serv_fulltext.c
+++ b/citadel/serv_fulltext.c
@@ -52,6 +52,23 @@ int ft_num_msgs = 0;
 int ft_num_alloc = 0;
 
 
+/*
+ * Compare function
+ */
+int longcmp(const void *rec1, const void *rec2) {
+       long i1, i2;
+
+       i1 = *(const long *)rec1;
+       i2 = *(const long *)rec2;
+
+       if (i1 > i2) return(1);
+       if (i1 < i2) return(-1);
+       return(0);
+}
+
+
+
+
 /*
  * Index or de-index a message.  (op == 1 to index, 0 to de-index)
  */
@@ -59,7 +76,7 @@ void ft_index_message(long msgnum, int op) {
        struct CtdlMessage *msg;
        int num_tokens = 0;
        int *tokens = NULL;
-       int i;
+       int i, j;
        struct cdbdata *cdb_bucket;
        int num_msgs;
        long *msgs;
@@ -84,7 +101,7 @@ void ft_index_message(long msgnum, int op) {
                        if (cdb_bucket == NULL) {
                                cdb_bucket = malloc(sizeof(struct cdbdata));
                                cdb_bucket->len = 0;
-                               cdb_bucket->ptr = malloc(sizeof(long));
+                               cdb_bucket->ptr = NULL;
                                num_msgs = 0;
                        }
                        else {
@@ -96,8 +113,23 @@ void ft_index_message(long msgnum, int op) {
                        msgs = (long *) cdb_bucket->ptr;
                        msgs[num_msgs - 1] = msgnum;
 
+                       /* lprintf(CTDL_DEBUG, "bucket <%5d> position <%2d> msg <%ld>\n",
+                               tokens[i], num_msgs-1, msgnum); */
+
+                       /* sort and purge dups */
+                       if (num_msgs > 1) {
+                               qsort(msgs, num_msgs, sizeof(long), longcmp);
+                               for (j=0; j<(num_msgs-1); ++j) {
+                                       if (msgs[j] == msgs[j+1]) {
+                                               memmove(&msgs[j], &msgs[j+1],
+                                                       ((num_msgs - j)*sizeof(long)));
+                                               --num_msgs;
+                                       }
+                               }
+                       }
+
                        cdb_store(CDB_FULLTEXT, &tokens[i], sizeof(int),
-                               cdb_bucket->ptr, num_msgs*sizeof(long) );
+                               msgs, (num_msgs*sizeof(long)) );
 
                        cdb_free(cdb_bucket);
 
@@ -137,21 +169,6 @@ void ft_index_room(struct ctdlroom *qrbuf, void *data)
 }
 
 
-/*
- * Compare function
- */
-int longcmp(const void *rec1, const void *rec2) {
-       long i1, i2;
-
-       i1 = *(const long *)rec1;
-       i2 = *(const long *)rec2;
-
-       if (i1 > i2) return(1);
-       if (i1 < i2) return(-1);
-       return(0);
-}
-
-
 /*
  * Begin the fulltext indexing process.  (Called as an EVT_TIMER event)
  */
@@ -171,10 +188,8 @@ void do_fulltext_indexing(void) {
         * Check to see whether the fulltext index is up to date; if there
         * are no messages to index, don't waste any more time trying.
         */
-       lprintf(CTDL_DEBUG, "CitControl.MMhighest  = %ld\n",
-               CitControl.MMhighest);
-       lprintf(CTDL_DEBUG, "CitControl.MMfulltext = %ld\n",
-               CitControl.MMfulltext);
+       lprintf(CTDL_DEBUG, "CitControl.MMhighest  = %ld\n", CitControl.MMhighest);
+       lprintf(CTDL_DEBUG, "CitControl.MMfulltext = %ld\n", CitControl.MMfulltext);
        if (CitControl.MMfulltext >= CitControl.MMhighest) {
                /* nothing to do! */
                return;
@@ -201,7 +216,7 @@ void do_fulltext_indexing(void) {
 
        if (ft_num_msgs > 0) {
                qsort(ft_newmsgs, ft_num_msgs, sizeof(long), longcmp);
-               if (i>1) for (i=0; i<(ft_num_msgs-1); ++i) { /* purge dups */
+               for (i=0; i<(ft_num_msgs-1); ++i) { /* purge dups */
                        if (ft_newmsgs[i] == ft_newmsgs[i+1]) {
                                memmove(&ft_newmsgs[i], &ft_newmsgs[i+1],
                                        ((ft_num_msgs - i)*sizeof(long)));
@@ -256,10 +271,10 @@ void cmd_srch(char *argbuf) {
                        if (cdb_bucket != NULL) {
                                num_msgs = cdb_bucket->len / sizeof(long);
                                msgs = (long *)cdb_bucket->ptr;
-                               cdb_free(cdb_bucket);
                                for (j=0; j<num_msgs; ++j) {
                                        cprintf("Token <%d> is in msg <%ld>\n", tokens[i], msgs[j]);
                                }
+                               cdb_free(cdb_bucket);
                        }
 
                }

diff --git a/citadel/serv_fulltext.o b/citadel/serv_fulltext.o
index 50c60e42710120918fe13704c0546abaa2cc962d..d5444317a3346b6380a5a4eab90a99def2d93333 100644 (file)
Binary files a/citadel/serv_fulltext.o and b/citadel/serv_fulltext.o differ