5 * Copyright (c) 2005-2009 by the citadel.org team
7 * This program is free software; you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License as published by
9 * the Free Software Foundation; either version 3 of the License, or
10 * (at your option) any later version.
12 * This program is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 * GNU General Public License for more details.
17 * You should have received a copy of the GNU General Public License
18 * along with this program; if not, write to the Free Software
19 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
24 * This is an ID for the wordbreaker module. If we do pluggable wordbreakers
25 * later on, or even if we update this one, we can use a different ID so the
26 * system knows it needs to throw away the existing index and rebuild it.
28 #define FT_WORDBREAKER_ID 0x0021
31 * Minimum and maximum length of words to index
33 #define WB_MIN 4 // nothing with 3 or less chars
36 void wordbreaker(char *text, int *num_tokens, int **tokens);
38 void initialize_noise_words(void);
39 void noise_word_cleanup(void);
42 typedef struct noise_word noise_word;