2 * Copyright (c) 2005-2012 by the citadel.org team
4 * This program is open source software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License version 3.
9 * This program is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 * GNU General Public License for more details.
21 * This is an ID for the wordbreaker module. If we do pluggable wordbreakers
22 * later on, or even if we update this one, we can use a different ID so the
23 * system knows it needs to throw away the existing index and rebuild it.
25 #define FT_WORDBREAKER_ID 0x0021
28 * Minimum and maximum length of words to index
30 #define WB_MIN 4 // nothing with 3 or less chars
33 void wordbreaker(const char *text, int *num_tokens, int **tokens);
35 void initialize_noise_words(void);
36 void noise_word_cleanup(void);
39 typedef struct noise_word noise_word;