8 * This is an ID for the wordbreaker module. If we do pluggable wordbreakers
9 * later on, or even if we update this one, we can use a different ID so the
10 * system knows it needs to throw away the existing index and rebuild it.
12 #define FT_WORDBREAKER_ID 0x0021
15 * Minimum and maximum length of words to index
17 #define WB_MIN 4 // nothing with 3 or less chars
20 void wordbreaker(char *text, int *num_tokens, int **tokens);