/*
* Default wordbreaker module for full text indexing.
*
- * Copyright (c) 2005-2012 by the citadel.org team
+ * Copyright (c) 2005-2017 by the citadel.org team
*
- * This program is open source software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 3.
- *
- *
+ * This program is open source software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 3.
*
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- *
- *
- *
- *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
*/
-
#include "sysdep.h"
#include <stdlib.h>
#include <unistd.h>
int i;
noise_word *cur, *next;
- syslog(LOG_INFO, "Cleaning up fulltext noise words.\n");
+ syslog(LOG_INFO, "wordbreaker: cleaning up fulltext noise words");
for (i = 0 ; i < 26 ; i++)
{
ch = *ptr;
if ( (!isalnum(ch)) && (word_start) ) {
word_end = ptr;
-// --word_end;
/* extract the word */
word_len = word_end - word_start;
if (word_len >= sizeof word) {
- syslog(LOG_DEBUG, "Invalid word length: %d\n", word_len);
+ syslog(LOG_DEBUG, "wordbreaker: invalid word length: %d", word_len);
safestrncpy(word, word_start, sizeof word);
word[(sizeof word) - 1] = 0;
}
/*
* This module handles fulltext indexing of the message base.
- * Copyright (c) 2005-2015 by the citadel.org team
+ * Copyright (c) 2005-2017 by the citadel.org team
*
* This program is open source software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License as published
for (i=0; i<65536; ++i) {
if ((time(NULL) - last_update) >= 10) {
syslog(LOG_INFO,
- "Flushing index cache to disk (%d%% complete)",
+ "fulltext: flushing index cache to disk (%d%% complete)",
(i * 100 / 65536)
);
last_update = time(NULL);
ftc_msgs[i] = NULL;
}
}
- syslog(LOG_INFO, "Flushed index cache to disk (100%% complete)");
+ syslog(LOG_INFO, "fulltext: flushed index cache to disk (100%% complete)");
}
msg = CtdlFetchMessage(msgnum, 1, 1);
if (msg == NULL) {
- syslog(LOG_ERR, "ft_index_message() could not load msg %ld", msgnum);
+ syslog(LOG_ERR, "fulltext: ft_index_message() could not load msg %ld", msgnum);
return;
}
if (!CM_IsEmpty(msg, eSuppressIdx)) {
- syslog(LOG_DEBUG, "ft_index_message() excluded msg %ld", msgnum);
+ syslog(LOG_DEBUG, "fulltext: ft_index_message() excluded msg %ld", msgnum);
CM_Free(msg);
return;
}
- syslog(LOG_DEBUG, "ft_index_message() %s msg %ld", (op ? "adding" : "removing") , msgnum);
+ syslog(LOG_DEBUG, "fulltext: ft_index_message() %s msg %ld", (op ? "adding" : "removing") , msgnum);
/* Output the message as text before indexing it, so we don't end up
* indexing a bunch of encoded base64, etc.
msgtext = CC->redirect_buffer;
CC->redirect_buffer = NULL;
if (msgtext != NULL) {
- syslog(LOG_DEBUG, "Wordbreaking message %ld (%d bytes)", msgnum, StrLength(msgtext));
+ syslog(LOG_DEBUG, "fulltext: wordbreaking message %ld (%d bytes)", msgnum, StrLength(msgtext));
}
txt = SmashStrBuf(&msgtext);
wordbreaker(txt, &num_tokens, &tokens);
free(txt);
- syslog(LOG_DEBUG, "Indexing message %ld [%d tokens]", msgnum, num_tokens);
+ syslog(LOG_DEBUG, "fulltext: indexing message %ld [%d tokens]", msgnum, num_tokens);
if (num_tokens > 0) {
for (i=0; i<num_tokens; ++i) {
}
}
else {
- syslog(LOG_ALERT, "Invalid token %d !!", tok);
+ syslog(LOG_ALERT, "fulltext: invalid token %d !!", tok);
}
}
int i;
static time_t last_index = 0L;
static time_t last_progress = 0L;
- time_t run_time = 0L;
- time_t end_time = 0L;
static int is_running = 0;
if (is_running) return; /* Concurrency check - only one can run */
is_running = 1;
* Make sure we don't run the indexer too frequently.
* FIXME move the setting into config
*/
-
if ( (time(NULL) - last_index) < 300L) {
return;
}
return; /* nothing to do! */
}
- run_time = time(NULL);
- syslog(LOG_DEBUG, "do_fulltext_indexing() started (%ld)", run_time);
-
/*
* If we've switched wordbreaker modules, burn the index and start
* over.
*/
begin_critical_section(S_CONTROL);
if (CtdlGetConfigInt("MM_fulltext_wordbreaker") != FT_WORDBREAKER_ID) {
- syslog(LOG_DEBUG, "wb ver on disk = %d, code ver = %d",
+ syslog(LOG_DEBUG, "fulltext: wb ver on disk = %d, code ver = %d",
CtdlGetConfigInt("MM_fulltext_wordbreaker"), FT_WORDBREAKER_ID
);
- syslog(LOG_INFO, "(re)initializing full text index");
+ syslog(LOG_INFO, "fulltext: (re)initializing index");
cdb_trunc(CDB_FULLTEXT);
CtdlSetConfigLong("MMfulltext", 0);
}
for (i=0; i<ft_num_msgs; ++i) {
if (time(NULL) != last_progress) {
syslog(LOG_DEBUG,
- "Indexed %d of %d messages (%d%%)",
+ "fulltext: indexed %d of %d messages (%d%%)",
i, ft_num_msgs,
((i*100) / ft_num_msgs)
);
/* Check to see if we need to quit early */
if (server_shutting_down) {
- syslog(LOG_DEBUG, "Indexer quitting early");
+ syslog(LOG_DEBUG, "fulltext: indexer quitting early");
ft_newhighest = ft_newmsgs[i];
break;
}
/* Check to see if we have to maybe flush to disk */
if (i >= FT_MAX_CACHE) {
- syslog(LOG_DEBUG, "Time to flush.");
+ syslog(LOG_DEBUG, "fulltext: time to flush.");
ft_newhighest = ft_newmsgs[i];
break;
}
ft_num_alloc = 0;
ft_newmsgs = NULL;
}
- end_time = time(NULL);
if (server_shutting_down) {
is_running = 0;
return;
}
- syslog(LOG_DEBUG, "do_fulltext_indexing() duration (%ld)", end_time - run_time);
-
/* Save our place so we don't have to do this again */
ft_flush_cache();
begin_critical_section(S_CONTROL);
end_critical_section(S_CONTROL);
last_index = time(NULL);
- syslog(LOG_DEBUG, "do_fulltext_indexing() finished");
+ syslog(LOG_DEBUG, "fulltext: indexing finished");
is_running = 0;
return;
}
res = curl_easy_perform(curl);
if (res == CURLE_OK) {
} else {
- syslog(LOG_DEBUG, "POP3S client failed: %s , trying POP3 next", curl_easy_strerror(res));
+ syslog(LOG_DEBUG, "pop3client: POP3S connection failed: %s , trying POP3 next", curl_easy_strerror(res));
snprintf(url, sizeof url, "pop3://%s", host); // try unencrypted next
curl_easy_setopt(curl, CURLOPT_URL, url);
FlushStrBuf(Uidls);
}
if (res != CURLE_OK) {
- syslog(LOG_DEBUG, "pop3 client failed: %s", curl_easy_strerror(res));
+ syslog(LOG_DEBUG, "pop3client: POP3 connection failed: %s", curl_easy_strerror(res));
curl_easy_cleanup(curl);
FreeStrBuf(&Uidls);
return;
// Now go through the UIDL list and look for messages.
int num_msgs = num_tokens(ChrPtr(Uidls), '\n');
- syslog(LOG_DEBUG, "There are %d messages.", num_msgs);
+ syslog(LOG_DEBUG, "pop3client: there are %d messages", num_msgs);
for (i=0; i<num_msgs; ++i) {
char oneuidl[1024];
extract_token(oneuidl, ChrPtr(Uidls), i, '\n', sizeof oneuidl);
}
}
else {
- syslog(LOG_DEBUG, "%s has already been retrieved", oneuidl);
+ syslog(LOG_DEBUG, "pop3client: %s has already been retrieved", oneuidl);
}
}
}
if (doing_pop3client) return;
doing_pop3client = 1;
- syslog(LOG_DEBUG, "pop3client scan started");
+ syslog(LOG_DEBUG, "pop3client: scan started");
CtdlForEachNetCfgRoom(pop3client_scan_room, NULL);
/*
* We have to queue and process in separate phases, otherwise we leave a cursor open
*/
- syslog(LOG_DEBUG, "pop3client processing started");
+ syslog(LOG_DEBUG, "pop3client: processing started");
while (p3cq != NULL) {
pptr = p3cq;
p3cq = p3cq->next;
free(pptr);
}
- syslog(LOG_DEBUG, "pop3client ended");
+ syslog(LOG_DEBUG, "pop3client: ended");
last_run = time(NULL);
doing_pop3client = 0;
}
else {
CtdlSaveMsgPointerInRoom(rr->room, msgnum, 0, NULL);
}
- syslog(LOG_DEBUG, "Saved message %ld to %s", msgnum, rr->room);
+ syslog(LOG_DEBUG, "rssclient: saved message %ld to %s", msgnum, rr->room);
}
}
else {
- syslog(LOG_DEBUG, "%s was already seen", r->item_id);
+ syslog(LOG_DEBUG, "rssclient: already seen %s", r->item_id);
}
CM_Free(r->msg);
struct rssurl *thisone = NULL;
struct rssroom *newroom = NULL;
- syslog(LOG_DEBUG, "rssclient_push_todo(%s, %s)", rssurl, roomname);
+ syslog(LOG_DEBUG, "rssclient: will fetch %s to %s", rssurl, roomname);
for (r=rsstodo; r!=NULL; r=r->next) {
if (!strcasecmp(r->url, rssurl)) {
CURLcode res;
StrBuf *Downloaded = NULL;
- syslog(LOG_DEBUG, "rss_pull_one_feed(%s)", url->url);
+ syslog(LOG_DEBUG, "rssclient: fetching %s", url->url);
curl = curl_easy_init();
if (!curl) {
curl_easy_setopt(curl, CURLOPT_TIMEOUT, 20L); // Time out after 20 seconds
res = curl_easy_perform(curl); // Perform the request
if (res != CURLE_OK) {
- syslog(LOG_WARNING, "Failed to load feed: %s", curl_easy_strerror(res));
+ syslog(LOG_WARNING, "rssclient: failed to load feed: %s", curl_easy_strerror(res));
}
curl_easy_cleanup(curl);
/* Run no more than once every 15 minutes. */
if ((now - last_run) < 900) {
syslog(LOG_DEBUG,
- "Client: polling interval not yet reached; last run was %ldm%lds ago",
+ "rssclient: polling interval not yet reached; last run was %ldm%lds ago",
((now - last_run) / 60),
((now - last_run) % 60)
);
}
become_session(&rss_CC);
- syslog(LOG_DEBUG, "rssclient started");
+ syslog(LOG_DEBUG, "rssclient: started");
CtdlForEachRoom(rssclient_scan_room, NULL);
rss_pull_feeds();
- syslog(LOG_DEBUG, "rssclient ended");
+ syslog(LOG_DEBUG, "rssclient: ended");
last_run = time(NULL);
return;
}
{
if (!threading)
{
- syslog(LOG_INFO, "%s", curl_version());
+ syslog(LOG_INFO, "rssclient: using %s", curl_version());
CtdlRegisterSessionHook(rssclient_scan, EVT_TIMER, PRIO_AGGR + 300);
}
else