* rework imap tokenizer, we no longer copy the stuff around, we keep a reference...
[citadel.git] / citadel / modules / imap / imap_fetch.c
1 /*
2  * $Id$
3  *
4  * Implements the FETCH command in IMAP.
5  * This is a good example of the protocol's gratuitous complexity.
6  *
7  * Copyright (c) 2001-2009 by the citadel.org team
8  *
9  *  This program is free software; you can redistribute it and/or modify
10  *  it under the terms of the GNU General Public License as published by
11  *  the Free Software Foundation; either version 3 of the License, or
12  *  (at your option) any later version.
13  *
14  *  This program is distributed in the hope that it will be useful,
15  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
16  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
17  *  GNU General Public License for more details.
18  *
19  *  You should have received a copy of the GNU General Public License
20  *  along with this program; if not, write to the Free Software
21  *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
22  */
23
24
25 #include "sysdep.h"
26 #include <stdlib.h>
27 #include <unistd.h>
28 #include <stdio.h>
29 #include <fcntl.h>
30 #include <signal.h>
31 #include <pwd.h>
32 #include <errno.h>
33 #include <sys/types.h>
34
35 #if TIME_WITH_SYS_TIME
36 # include <sys/time.h>
37 # include <time.h>
38 #else
39 # if HAVE_SYS_TIME_H
40 #  include <sys/time.h>
41 # else
42 #  include <time.h>
43 # endif
44 #endif
45
46 #include <sys/wait.h>
47 #include <ctype.h>
48 #include <string.h>
49 #include <limits.h>
50 #include <libcitadel.h>
51 #include "citadel.h"
52 #include "server.h"
53 #include "sysdep_decls.h"
54 #include "citserver.h"
55 #include "support.h"
56 #include "config.h"
57 #include "user_ops.h"
58 #include "policy.h"
59 #include "database.h"
60 #include "msgbase.h"
61 #include "internet_addressing.h"
62 #include "imap_tools.h"
63 #include "serv_imap.h"
64 #include "imap_fetch.h"
65 #include "genstamp.h"
66 #include "ctdl_module.h"
67
68
69
70 /*
71  * Individual field functions for imap_do_fetch_msg() ...
72  */
73
74 void imap_fetch_uid(int seq) {
75         cprintf("UID %ld", IMAP->msgids[seq-1]);
76 }
77
78 void imap_fetch_flags(int seq) {
79         int num_flags_printed = 0;
80         cprintf("FLAGS (");
81         if (IMAP->flags[seq] & IMAP_DELETED) {
82                 if (num_flags_printed > 0) cprintf(" ");
83                 cprintf("\\Deleted");
84                 ++num_flags_printed;
85         }
86         if (IMAP->flags[seq] & IMAP_SEEN) {
87                 if (num_flags_printed > 0) cprintf(" ");
88                 cprintf("\\Seen");
89                 ++num_flags_printed;
90         }
91         if (IMAP->flags[seq] & IMAP_ANSWERED) {
92                 if (num_flags_printed > 0) cprintf(" ");
93                 cprintf("\\Answered");
94                 ++num_flags_printed;
95         }
96         if (IMAP->flags[seq] & IMAP_RECENT) {
97                 if (num_flags_printed > 0) cprintf(" ");
98                 cprintf("\\Recent");
99                 ++num_flags_printed;
100         }
101         cprintf(")");
102 }
103
104
105 void imap_fetch_internaldate(struct CtdlMessage *msg) {
106         char buf[SIZ];
107         time_t msgdate;
108
109         if (!msg) return;
110         if (msg->cm_fields['T'] != NULL) {
111                 msgdate = atol(msg->cm_fields['T']);
112         }
113         else {
114                 msgdate = time(NULL);
115         }
116
117         datestring(buf, sizeof buf, msgdate, DATESTRING_IMAP);
118         cprintf("INTERNALDATE \"%s\"", buf);
119 }
120
121
122 /*
123  * Fetch RFC822-formatted messages.
124  *
125  * 'whichfmt' should be set to one of:
126  *      "RFC822"        entire message
127  *      "RFC822.HEADER" headers only (with trailing blank line)
128  *      "RFC822.SIZE"   size of translated message
129  *      "RFC822.TEXT"   body only (without leading blank line)
130  */
131 void imap_fetch_rfc822(long msgnum, char *whichfmt) {
132         char buf[SIZ];
133         const char *ptr = NULL;
134         size_t headers_size, text_size, total_size;
135         size_t bytes_to_send = 0;
136         struct MetaData smi;
137         int need_to_rewrite_metadata = 0;
138         int need_body = 0;
139
140         /* Determine whether this particular fetch operation requires
141          * us to fetch the message body from disk.  If not, we can save
142          * on some disk operations...
143          */
144         if ( (!strcasecmp(whichfmt, "RFC822"))
145            || (!strcasecmp(whichfmt, "RFC822.TEXT")) ) {
146                 need_body = 1;
147         }
148
149         /* If this is an RFC822.SIZE fetch, first look in the message's
150          * metadata record to see if we've saved that information.
151          */
152         if (!strcasecmp(whichfmt, "RFC822.SIZE")) {
153                 GetMetaData(&smi, msgnum);
154                 if (smi.meta_rfc822_length > 0L) {
155                         cprintf("RFC822.SIZE %ld", smi.meta_rfc822_length);
156                         return;
157                 }
158                 need_to_rewrite_metadata = 1;
159                 need_body = 1;
160         }
161         
162         /* Cache the most recent RFC822 FETCH because some clients like to
163          * fetch in pieces, and we don't want to have to go back to the
164          * message store for each piece.  We also burn the cache if the
165          * client requests something that involves reading the message
166          * body, but we haven't fetched the body yet.
167          */
168         if ((IMAP->cached_rfc822_data != NULL)
169            && (IMAP->cached_rfc822_msgnum == msgnum)
170            && (IMAP->cached_rfc822_withbody || (!need_body)) ) {
171                 /* Good to go! */
172         }
173         else if (IMAP->cached_rfc822_data != NULL) {
174                 /* Some other message is cached -- free it */
175                 free(IMAP->cached_rfc822_data);
176                 IMAP->cached_rfc822_data = NULL;
177                 IMAP->cached_rfc822_msgnum = (-1);
178                 IMAP->cached_rfc822_len = 0;
179         }
180
181         /* At this point, we now can fetch and convert the message iff it's not
182          * the one we had cached.
183          */
184         if (IMAP->cached_rfc822_data == NULL) {
185                 /*
186                  * Load the message into memory for translation & measurement
187                  */
188                 CC->redirect_buffer = malloc(SIZ);
189                 CC->redirect_len = 0;
190                 CC->redirect_alloc = SIZ;
191                 CtdlOutputMsg(msgnum, MT_RFC822,
192                         (need_body ? HEADERS_ALL : HEADERS_FAST),
193                         0, 1, NULL, SUPPRESS_ENV_TO
194                 );
195                 if (!need_body) cprintf("\r\n");        /* extra trailing newline */
196                 IMAP->cached_rfc822_data = CC->redirect_buffer;
197                 IMAP->cached_rfc822_len = CC->redirect_len;
198                 IMAP->cached_rfc822_msgnum = msgnum;
199                 IMAP->cached_rfc822_withbody = need_body;
200                 CC->redirect_buffer = NULL;
201                 CC->redirect_len = 0;
202                 CC->redirect_alloc = 0;
203                 if ( (need_to_rewrite_metadata) && (IMAP->cached_rfc822_len > 0) ) {
204                         smi.meta_rfc822_length = (long)IMAP->cached_rfc822_len;
205                         PutMetaData(&smi);
206                 }
207         }
208
209         /*
210          * Now figure out where the headers/text break is.  IMAP considers the
211          * intervening blank line to be part of the headers, not the text.
212          */
213         headers_size = 0;
214         text_size = 0;
215         total_size = 0;
216
217         if (need_body) {
218                 ptr = IMAP->cached_rfc822_data;
219                 do {
220                         ptr = memreadline(ptr, buf, sizeof buf);
221                         if (*ptr != 0) {
222                                 striplt(buf);
223                                 if (IsEmptyStr(buf)) {
224                                         headers_size = ptr - IMAP->cached_rfc822_data;
225                                 }
226                         }
227                 } while ( (headers_size == 0) && (*ptr != 0) );
228
229                 total_size = IMAP->cached_rfc822_len;
230                 text_size = total_size - headers_size;
231         }
232         else {
233                 headers_size = IMAP->cached_rfc822_len;
234                 total_size = IMAP->cached_rfc822_len;
235                 text_size = 0;
236         }
237
238         CtdlLogPrintf(CTDL_DEBUG, 
239                 "RFC822: headers=" SIZE_T_FMT 
240                 ", text=" SIZE_T_FMT
241                 ", total=" SIZE_T_FMT "\n",
242                 headers_size, text_size, total_size);
243
244         if (!strcasecmp(whichfmt, "RFC822.SIZE")) {
245                 cprintf("RFC822.SIZE " SIZE_T_FMT, total_size);
246                 return;
247         }
248
249         else if (!strcasecmp(whichfmt, "RFC822")) {
250                 ptr = IMAP->cached_rfc822_data;
251                 bytes_to_send = total_size;
252         }
253
254         else if (!strcasecmp(whichfmt, "RFC822.HEADER")) {
255                 ptr = IMAP->cached_rfc822_data;
256                 bytes_to_send = headers_size;
257         }
258
259         else if (!strcasecmp(whichfmt, "RFC822.TEXT")) {
260                 ptr = &IMAP->cached_rfc822_data[headers_size];
261                 bytes_to_send = text_size;
262         }
263
264         cprintf("%s {" SIZE_T_FMT "}\r\n", whichfmt, bytes_to_send);
265         client_write(ptr, bytes_to_send);
266 }
267
268
269
270 /*
271  * Load a specific part of a message into the temp file to be output to a
272  * client.  FIXME we can handle parts like "2" and "2.1" and even "2.MIME"
273  * but we still can't handle "2.HEADER" (which might not be a problem).
274  *
275  * Note: mime_parser() was called with dont_decode set to 1, so we have the
276  * luxury of simply spewing without having to re-encode.
277  */
278 void imap_load_part(char *name, char *filename, char *partnum, char *disp,
279                     void *content, char *cbtype, char *cbcharset, size_t length, char *encoding,
280                     char *cbid, void *cbuserdata)
281 {
282         char mbuf2[SIZ];
283         char *desired_section;
284
285         desired_section = (char *)cbuserdata;
286
287         if (!strcasecmp(partnum, desired_section)) {
288                 client_write(content, length);
289         }
290
291         snprintf(mbuf2, sizeof mbuf2, "%s.MIME", partnum);
292
293         if (!strcasecmp(desired_section, mbuf2)) {
294                 cprintf("Content-type: %s", cbtype);
295                 if (!IsEmptyStr(cbcharset))
296                         cprintf("; charset=\"%s\"", cbcharset);
297                 if (!IsEmptyStr(name))
298                         cprintf("; name=\"%s\"", name);
299                 cprintf("\r\n");
300                 if (!IsEmptyStr(encoding))
301                         cprintf("Content-Transfer-Encoding: %s\r\n", encoding);
302                 if (!IsEmptyStr(encoding)) {
303                         cprintf("Content-Disposition: %s", disp);
304                         if (!IsEmptyStr(filename)) {
305                                 cprintf("; filename=\"%s\"", filename);
306                         }
307                         cprintf("\r\n");
308                 }
309                 cprintf("Content-Length: %ld\r\n", (long)length);
310                 cprintf("\r\n");
311         }
312                         
313
314 }
315
316
317 /* 
318  * Called by imap_fetch_envelope() to output the "From" field.
319  * This is in its own function because its logic is kind of complex.  We
320  * really need to make this suck less.
321  */
322 void imap_output_envelope_from(struct CtdlMessage *msg) {
323         char user[SIZ], node[SIZ], name[SIZ];
324
325         if (!msg) return;
326
327         /* For anonymous messages, it's so easy! */
328         if (!is_room_aide() && (msg->cm_anon_type == MES_ANONONLY)) {
329                 cprintf("((\"----\" NIL \"x\" \"x.org\")) ");
330                 return;
331         }
332         if (!is_room_aide() && (msg->cm_anon_type == MES_ANONOPT)) {
333                 cprintf("((\"anonymous\" NIL \"x\" \"x.org\")) ");
334                 return;
335         }
336
337         /* For everything else, we do stuff. */
338         cprintf("((");                          /* open double-parens */
339         plain_imap_strout(msg->cm_fields['A']); /* personal name */
340         cprintf(" NIL ");                       /* source route (not used) */
341
342
343         if (msg->cm_fields['F'] != NULL) {
344                 process_rfc822_addr(msg->cm_fields['F'], user, node, name);
345                 plain_imap_strout(user);                /* mailbox name (user id) */
346                 cprintf(" ");
347                 if (!strcasecmp(node, config.c_nodename)) {
348                         plain_imap_strout(config.c_fqdn);
349                 }
350                 else {
351                         plain_imap_strout(node);                /* host name */
352                 }
353         }
354         else {
355                 plain_imap_strout(msg->cm_fields['A']); /* mailbox name (user id) */
356                 cprintf(" ");
357                 plain_imap_strout(msg->cm_fields['N']); /* host name */
358         }
359         
360         cprintf(")) ");                         /* close double-parens */
361 }
362
363
364
365 /*
366  * Output an envelope address (or set of addresses) in the official,
367  * convoluted, braindead format.  (Note that we can't use this for
368  * the "From" address because its data may come from a number of different
369  * fields.  But we can use it for "To" and possibly others.
370  */
371 void imap_output_envelope_addr(char *addr) {
372         char individual_addr[256];
373         int num_addrs;
374         int i;
375         char user[256];
376         char node[256];
377         char name[256];
378
379         if (addr == NULL) {
380                 cprintf("NIL ");
381                 return;
382         }
383
384         if (IsEmptyStr(addr)) {
385                 cprintf("NIL ");
386                 return;
387         }
388
389         cprintf("(");
390
391         /* How many addresses are listed here? */
392         num_addrs = num_tokens(addr, ',');
393
394         /* Output them one by one. */
395         for (i=0; i<num_addrs; ++i) {
396                 extract_token(individual_addr, addr, i, ',', sizeof individual_addr);
397                 striplt(individual_addr);
398                 process_rfc822_addr(individual_addr, user, node, name);
399                 cprintf("(");
400                 plain_imap_strout(name);
401                 cprintf(" NIL ");
402                 plain_imap_strout(user);
403                 cprintf(" ");
404                 plain_imap_strout(node);
405                 cprintf(")");
406                 if (i < (num_addrs-1)) cprintf(" ");
407         }
408
409         cprintf(") ");
410 }
411
412
413 /*
414  * Implements the ENVELOPE fetch item
415  * 
416  * Note that the imap_strout() function can cleverly output NULL fields as NIL,
417  * so we don't have to check for that condition like we do elsewhere.
418  */
419 void imap_fetch_envelope(struct CtdlMessage *msg) {
420         char datestringbuf[SIZ];
421         time_t msgdate;
422         char *fieldptr = NULL;
423
424         if (!msg) return;
425
426         /* Parse the message date into an IMAP-format date string */
427         if (msg->cm_fields['T'] != NULL) {
428                 msgdate = atol(msg->cm_fields['T']);
429         }
430         else {
431                 msgdate = time(NULL);
432         }
433         datestring(datestringbuf, sizeof datestringbuf,
434                 msgdate, DATESTRING_IMAP);
435
436         /* Now start spewing data fields.  The order is important, as it is
437          * defined by the protocol specification.  Nonexistent fields must
438          * be output as NIL, existent fields must be quoted or literalled.
439          * The imap_strout() function conveniently does all this for us.
440          */
441         cprintf("ENVELOPE (");
442
443         /* Date */
444         plain_imap_strout(datestringbuf);
445         cprintf(" ");
446
447         /* Subject */
448         plain_imap_strout(msg->cm_fields['U']);
449         cprintf(" ");
450
451         /* From */
452         imap_output_envelope_from(msg);
453
454         /* Sender (default to same as 'From' if not present) */
455         fieldptr = rfc822_fetch_field(msg->cm_fields['M'], "Sender");
456         if (fieldptr != NULL) {
457                 imap_output_envelope_addr(fieldptr);
458                 free(fieldptr);
459         }
460         else {
461                 imap_output_envelope_from(msg);
462         }
463
464         /* Reply-to */
465         fieldptr = rfc822_fetch_field(msg->cm_fields['M'], "Reply-to");
466         if (fieldptr != NULL) {
467                 imap_output_envelope_addr(fieldptr);
468                 free(fieldptr);
469         }
470         else {
471                 imap_output_envelope_from(msg);
472         }
473
474         /* To */
475         imap_output_envelope_addr(msg->cm_fields['R']);
476
477         /* Cc (we do it this way because there might be a legacy non-Citadel Cc: field present) */
478         fieldptr = msg->cm_fields['Y'];
479         if (fieldptr != NULL) {
480                 imap_output_envelope_addr(fieldptr);
481         }
482         else {
483                 fieldptr = rfc822_fetch_field(msg->cm_fields['M'], "Cc");
484                 imap_output_envelope_addr(fieldptr);
485                 if (fieldptr != NULL) free(fieldptr);
486         }
487
488         /* Bcc */
489         fieldptr = rfc822_fetch_field(msg->cm_fields['M'], "Bcc");
490         imap_output_envelope_addr(fieldptr);
491         if (fieldptr != NULL) free(fieldptr);
492
493         /* In-reply-to */
494         fieldptr = rfc822_fetch_field(msg->cm_fields['M'], "In-reply-to");
495         plain_imap_strout(fieldptr);
496         cprintf(" ");
497         if (fieldptr != NULL) free(fieldptr);
498
499         /* message ID */
500         plain_imap_strout(msg->cm_fields['I']);
501
502         cprintf(")");
503 }
504
505 /*
506  * This function is called only when CC->redirect_buffer contains a set of
507  * RFC822 headers with no body attached.  Its job is to strip that set of
508  * headers down to *only* the ones we're interested in.
509  */
510 void imap_strip_headers(char *section) {
511         char buf[SIZ];
512         char *which_fields = NULL;
513         int doing_headers = 0;
514         int headers_not = 0;
515         char *parms[SIZ];
516         int num_parms = 0;
517         int i;
518         char *boiled_headers = NULL;
519         int ok = 0;
520         int done_headers = 0;
521         const char *ptr = NULL;
522
523         if (CC->redirect_buffer == NULL) return;
524
525         which_fields = strdup(section);
526
527         if (!strncasecmp(which_fields, "HEADER.FIELDS", 13))
528                 doing_headers = 1;
529         if (!strncasecmp(which_fields, "HEADER.FIELDS.NOT", 17))
530                 headers_not = 1;
531
532         for (i=0; which_fields[i]; ++i) {
533                 if (which_fields[i]=='(')
534                         strcpy(which_fields, &which_fields[i+1]);
535         }
536         for (i=0; which_fields[i]; ++i) {
537                 if (which_fields[i]==')') {
538                         which_fields[i] = 0;
539                         break;
540                 }
541         }
542         num_parms = old_imap_parameterize(parms, which_fields);
543
544         boiled_headers = malloc(CC->redirect_alloc);
545         strcpy(boiled_headers, "");
546
547         ptr = CC->redirect_buffer;
548         ok = 0;
549         do {
550                 ptr = memreadline(ptr, buf, sizeof buf);
551                 if (!isspace(buf[0])) {
552                         ok = 0;
553                         if (doing_headers == 0) ok = 1;
554                         else {
555                                 if (headers_not) ok = 1;
556                                 else ok = 0;
557                                 for (i=0; i<num_parms; ++i) {
558                                         if ( (!strncasecmp(buf, parms[i],
559                                            strlen(parms[i]))) &&
560                                            (buf[strlen(parms[i])]==':') ) {
561                                                 if (headers_not) ok = 0;
562                                                 else ok = 1;
563                                         }
564                                 }
565                         }
566                 }
567
568                 if (ok) {
569                         strcat(boiled_headers, buf);
570                         strcat(boiled_headers, "\r\n");
571                 }
572
573                 if (IsEmptyStr(buf)) done_headers = 1;
574                 if (buf[0]=='\r') done_headers = 1;
575                 if (buf[0]=='\n') done_headers = 1;
576                 if (*ptr == 0) done_headers = 1;
577         } while (!done_headers);
578
579         strcat(boiled_headers, "\r\n");
580
581         /* Now save it back (it'll always be smaller) */
582         strcpy(CC->redirect_buffer, boiled_headers);
583         CC->redirect_len = strlen(boiled_headers);
584
585         free(which_fields);
586         free(boiled_headers);
587 }
588
589
590 /*
591  * Implements the BODY and BODY.PEEK fetch items
592  */
593 void imap_fetch_body(long msgnum, char *item, int is_peek) {
594         struct CtdlMessage *msg = NULL;
595         char section[SIZ];
596         char partial[SIZ];
597         int is_partial = 0;
598         size_t pstart, pbytes;
599         int loading_body_now = 0;
600         int need_body = 1;
601         int burn_the_cache = 0;
602
603         /* extract section */
604         safestrncpy(section, item, sizeof section);
605         if (strchr(section, '[') != NULL) {
606                 stripallbut(section, '[', ']');
607         }
608         CtdlLogPrintf(CTDL_DEBUG, "Section is: %s%s\n", 
609                 section, 
610                 IsEmptyStr(section) ? "(empty)" : ""
611         );
612
613         /* Burn the cache if we don't have the same section of the 
614          * same message again.
615          */
616         if (IMAP->cached_body != NULL) {
617                 if (IMAP->cached_bodymsgnum != msgnum) {
618                         burn_the_cache = 1;
619                 }
620                 else if ( (!IMAP->cached_body_withbody) && (need_body) ) {
621                         burn_the_cache = 1;
622                 }
623                 else if (strcasecmp(IMAP->cached_bodypart, section)) {
624                         burn_the_cache = 1;
625                 }
626                 if (burn_the_cache) {
627                         /* Yup, go ahead and burn the cache. */
628                         free(IMAP->cached_body);
629                         IMAP->cached_body_len = 0;
630                         IMAP->cached_body = NULL;
631                         IMAP->cached_bodymsgnum = (-1);
632                         strcpy(IMAP->cached_bodypart, "");
633                 }
634         }
635
636         /* extract partial */
637         safestrncpy(partial, item, sizeof partial);
638         if (strchr(partial, '<') != NULL) {
639                 stripallbut(partial, '<', '>');
640                 is_partial = 1;
641         }
642         if (is_partial == 0) strcpy(partial, "");
643         /* if (!IsEmptyStr(partial)) CtdlLogPrintf(CTDL_DEBUG, "Partial is %s\n", partial); */
644
645         if (IMAP->cached_body == NULL) {
646                 CC->redirect_buffer = malloc(SIZ);
647                 CC->redirect_len = 0;
648                 CC->redirect_alloc = SIZ;
649                 loading_body_now = 1;
650                 msg = CtdlFetchMessage(msgnum, (need_body ? 1 : 0));
651         }
652
653         /* Now figure out what the client wants, and get it */
654
655         if (!loading_body_now) {
656                 /* What we want is already in memory */
657         }
658
659         else if ( (!strcmp(section, "1")) && (msg->cm_format_type != 4) ) {
660                 CtdlOutputPreLoadedMsg(msg, MT_RFC822, HEADERS_NONE, 0, 1, SUPPRESS_ENV_TO);
661         }
662
663         else if (!strcmp(section, "")) {
664                 CtdlOutputPreLoadedMsg(msg, MT_RFC822, HEADERS_ALL, 0, 1, SUPPRESS_ENV_TO);
665         }
666
667         /*
668          * If the client asked for just headers, or just particular header
669          * fields, strip it down.
670          */
671         else if (!strncasecmp(section, "HEADER", 6)) {
672                 /* This used to work with HEADERS_FAST, but then Apple got stupid with their
673                  * IMAP library and this broke Mail.App and iPhone Mail, so we had to change it
674                  * to HEADERS_ONLY so the trendy hipsters with their iPhones can read mail.
675                  */
676                 CtdlOutputPreLoadedMsg(msg, MT_RFC822, HEADERS_ONLY, 0, 1, SUPPRESS_ENV_TO);
677                 imap_strip_headers(section);
678         }
679
680         /*
681          * Strip it down if the client asked for everything _except_ headers.
682          */
683         else if (!strncasecmp(section, "TEXT", 4)) {
684                 CtdlOutputPreLoadedMsg(msg, MT_RFC822, HEADERS_NONE, 0, 1, SUPPRESS_ENV_TO);
685         }
686
687         /*
688          * Anything else must be a part specifier.
689          * (Note value of 1 passed as 'dont_decode' so client gets it encoded)
690          */
691         else {
692                 mime_parser(msg->cm_fields['M'], NULL,
693                                 *imap_load_part, NULL, NULL,
694                                 section,
695                                 1);
696         }
697
698         if (loading_body_now) {
699                 IMAP->cached_body = CC->redirect_buffer;
700                 IMAP->cached_body_len = CC->redirect_len;
701                 IMAP->cached_bodymsgnum = msgnum;
702                 IMAP->cached_body_withbody = need_body;
703                 strcpy(IMAP->cached_bodypart, section);
704                 CC->redirect_buffer = NULL;
705                 CC->redirect_len = 0;
706                 CC->redirect_alloc = 0;
707         }
708
709         if (is_partial == 0) {
710                 cprintf("BODY[%s] {" SIZE_T_FMT "}\r\n", section, IMAP->cached_body_len);
711                 pstart = 0;
712                 pbytes = IMAP->cached_body_len;
713         }
714         else {
715                 sscanf(partial, SIZE_T_FMT "." SIZE_T_FMT, &pstart, &pbytes);
716                 if (pbytes > (IMAP->cached_body_len - pstart)) {
717                         pbytes = IMAP->cached_body_len - pstart;
718                 }
719                 cprintf("BODY[%s]<" SIZE_T_FMT "> {" SIZE_T_FMT "}\r\n", section, pstart, pbytes);
720         }
721
722         /* Here we go -- output it */
723         client_write(&IMAP->cached_body[pstart], pbytes);
724
725         if (msg != NULL) {
726                 CtdlFreeMessage(msg);
727         }
728
729         /* Mark this message as "seen" *unless* this is a "peek" operation */
730         if (is_peek == 0) {
731                 CtdlSetSeen(&msgnum, 1, 1, ctdlsetseen_seen, NULL, NULL);
732         }
733 }
734
735 /*
736  * Called immediately before outputting a multipart bodystructure
737  */
738 void imap_fetch_bodystructure_pre(
739                 char *name, char *filename, char *partnum, char *disp,
740                 void *content, char *cbtype, char *cbcharset, size_t length, char *encoding,
741                 char *cbid, void *cbuserdata
742                 ) {
743
744         cprintf("(");
745 }
746
747
748
749 /*
750  * Called immediately after outputting a multipart bodystructure
751  */
752 void imap_fetch_bodystructure_post(
753                 char *name, char *filename, char *partnum, char *disp,
754                 void *content, char *cbtype, char *cbcharset, size_t length, char *encoding,
755                 char *cbid, void *cbuserdata
756                 ) {
757
758         char subtype[128];
759
760         cprintf(" ");
761
762         /* disposition */
763         extract_token(subtype, cbtype, 1, '/', sizeof subtype);
764         plain_imap_strout(subtype);
765
766         /* body language */
767         /* cprintf(" NIL"); We thought we needed this at one point, but maybe we don't... */
768
769         cprintf(")");
770 }
771
772
773
774 /*
775  * Output the info for a MIME part in the format required by BODYSTRUCTURE.
776  *
777  */
778 void imap_fetch_bodystructure_part(
779                 char *name, char *filename, char *partnum, char *disp,
780                 void *content, char *cbtype, char *cbcharset, size_t length, char *encoding,
781                 char *cbid, void *cbuserdata
782                 ) {
783
784         int have_cbtype = 0;
785         int have_encoding = 0;
786         int lines = 0;
787         size_t i;
788         char cbmaintype[128];
789         char cbsubtype[128];
790
791         if (cbtype != NULL) if (!IsEmptyStr(cbtype)) have_cbtype = 1;
792         if (have_cbtype) {
793                 extract_token(cbmaintype, cbtype, 0, '/', sizeof cbmaintype);
794                 extract_token(cbsubtype, cbtype, 1, '/', sizeof cbsubtype);
795         }
796         else {
797                 strcpy(cbmaintype, "TEXT");
798                 strcpy(cbsubtype, "PLAIN");
799         }
800
801         cprintf("(");
802         plain_imap_strout(cbmaintype);                                  /* body type */
803         cprintf(" ");
804         plain_imap_strout(cbsubtype);                                           /* body subtype */
805         cprintf(" ");
806
807         cprintf("(");                                                   /* begin body parameter list */
808
809         /* "NAME" must appear as the first parameter.  This is not required by IMAP,
810          * but the Asterisk voicemail application blindly assumes that NAME will be in
811          * the first position.  If it isn't, it rejects the message.
812          */
813         if (name != NULL) if (!IsEmptyStr(name)) {
814                 cprintf("\"NAME\" ");
815                 plain_imap_strout(name);
816                 cprintf(" ");
817         }
818
819         cprintf("\"CHARSET\" ");
820         if (cbcharset == NULL) {
821                 plain_imap_strout("US-ASCII");
822         }
823         else if (cbcharset[0] == 0) {
824                 plain_imap_strout("US-ASCII");
825         }
826         else {
827                 plain_imap_strout(cbcharset);
828         }
829         cprintf(") ");                                                  /* end body parameter list */
830
831         cprintf("NIL ");                                                /* Body ID */
832         cprintf("NIL ");                                                /* Body description */
833
834         if (encoding != NULL) if (encoding[0] != 0)  have_encoding = 1;
835         if (have_encoding) {
836                 plain_imap_strout(encoding);
837         }
838         else {
839                 plain_imap_strout("7BIT");
840         }
841         cprintf(" ");
842
843         /* The next field is the size of the part in bytes. */
844         cprintf("%ld ", (long)length);  /* bytes */
845
846         /* The next field is the number of lines in the part, if and only
847          * if the part is TEXT.  More gratuitous complexity.
848          */
849         if (!strcasecmp(cbmaintype, "TEXT")) {
850                 if (length) for (i=0; i<length; ++i) {
851                         if (((char *)content)[i] == '\n') ++lines;
852                 }
853                 cprintf("%d ", lines);
854         }
855
856         /* More gratuitous complexity */
857         if ((!strcasecmp(cbmaintype, "MESSAGE"))
858            && (!strcasecmp(cbsubtype, "RFC822"))) {
859                 /* FIXME: message/rfc822 also needs to output the envelope structure,
860                  * body structure, and line count of the encapsulated message.  Fortunately
861                  * there are not yet any clients depending on this, so we can get away
862                  * with not implementing it for now.
863                  */
864         }
865
866         /* MD5 value of body part; we can get away with NIL'ing this */
867         cprintf("NIL ");
868
869         /* Disposition */
870         if (disp == NULL) {
871                 cprintf("NIL");
872         }
873         else if (IsEmptyStr(disp)) {
874                 cprintf("NIL");
875         }
876         else {
877                 cprintf("(");
878                 plain_imap_strout(disp);
879                 if (filename != NULL) if (!IsEmptyStr(filename)) {
880                         cprintf(" (\"FILENAME\" ");
881                         plain_imap_strout(filename);
882                         cprintf(")");
883                 }
884                 cprintf(")");
885         }
886
887         /* Body language (not defined yet) */
888         cprintf(" NIL)");
889 }
890
891
892
893 /*
894  * Spew the BODYSTRUCTURE data for a message.
895  *
896  */
897 void imap_fetch_bodystructure (long msgnum, char *item,
898                 struct CtdlMessage *msg) {
899         const char *rfc822 = NULL;
900         const char *rfc822_body = NULL;
901         size_t rfc822_len;
902         size_t rfc822_headers_len;
903         size_t rfc822_body_len;
904         const char *ptr = NULL;
905         char *pch;
906         char buf[SIZ];
907         int lines = 0;
908
909         /* Handle NULL message gracefully */
910         if (msg == NULL) {
911                 cprintf("BODYSTRUCTURE (\"TEXT\" \"PLAIN\" "
912                         "(\"CHARSET\" \"US-ASCII\") NIL NIL "
913                         "\"7BIT\" 0 0)");
914                 return;
915         }
916
917         /* For non-RFC822 (ordinary Citadel) messages, this is short and
918          * sweet...
919          */
920         if (msg->cm_format_type != FMT_RFC822) {
921
922                 /* *sigh* We have to RFC822-format the message just to be able
923                  * to measure it.  FIXME use smi cached fields if possible
924                  */
925
926                 CC->redirect_buffer = malloc(SIZ);
927                 CC->redirect_len = 0;
928                 CC->redirect_alloc = SIZ;
929                 CtdlOutputPreLoadedMsg(msg, MT_RFC822, 0, 0, 1, SUPPRESS_ENV_TO);
930                 rfc822 = pch = CC->redirect_buffer;
931                 rfc822_len = CC->redirect_len;
932                 CC->redirect_buffer = NULL;
933                 CC->redirect_len = 0;
934                 CC->redirect_alloc = 0;
935
936                 ptr = rfc822;
937                 do {
938                         ptr = memreadline(ptr, buf, sizeof buf);
939                         ++lines;
940                         if ((IsEmptyStr(buf)) && (rfc822_body == NULL)) {
941                                 rfc822_body = ptr;
942                         }
943                 } while (*ptr != 0);
944
945                 rfc822_headers_len = rfc822_body - rfc822;
946                 rfc822_body_len = rfc822_len - rfc822_headers_len;
947                 free(pch);
948
949                 cprintf("BODYSTRUCTURE (\"TEXT\" \"PLAIN\" "
950                         "(\"CHARSET\" \"US-ASCII\") NIL NIL "
951                         "\"7BIT\" " SIZE_T_FMT " %d)", rfc822_body_len, lines);
952
953                 return;
954         }
955
956         /* For messages already stored in RFC822 format, we have to parse. */
957         cprintf("BODYSTRUCTURE ");
958         mime_parser(msg->cm_fields['M'],
959                         NULL,
960                         *imap_fetch_bodystructure_part, /* part */
961                         *imap_fetch_bodystructure_pre,  /* pre-multi */
962                         *imap_fetch_bodystructure_post, /* post-multi */
963                         NULL,
964                         1);     /* don't decode -- we want it as-is */
965 }
966
967
968 /*
969  * imap_do_fetch() calls imap_do_fetch_msg() to output the data of an
970  * individual message, once it has been selected for output.
971  */
972 void imap_do_fetch_msg(int seq, int num_items, char **itemlist) {
973         int i;
974         struct CtdlMessage *msg = NULL;
975         int body_loaded = 0;
976
977         /* Don't attempt to fetch bogus messages or UID's */
978         if (seq < 1) return;
979         if (IMAP->msgids[seq-1] < 1L) return;
980
981         buffer_output();
982         cprintf("* %d FETCH (", seq);
983
984         for (i=0; i<num_items; ++i) {
985
986                 /* Fetchable without going to the message store at all */
987                 if (!strcasecmp(itemlist[i], "UID")) {
988                         imap_fetch_uid(seq);
989                 }
990                 else if (!strcasecmp(itemlist[i], "FLAGS")) {
991                         imap_fetch_flags(seq-1);
992                 }
993
994                 /* Potentially fetchable from cache, if the client requests
995                  * stuff from the same message several times in a row.
996                  */
997                 else if (!strcasecmp(itemlist[i], "RFC822")) {
998                         imap_fetch_rfc822(IMAP->msgids[seq-1], itemlist[i]);
999                 }
1000                 else if (!strcasecmp(itemlist[i], "RFC822.HEADER")) {
1001                         imap_fetch_rfc822(IMAP->msgids[seq-1], itemlist[i]);
1002                 }
1003                 else if (!strcasecmp(itemlist[i], "RFC822.SIZE")) {
1004                         imap_fetch_rfc822(IMAP->msgids[seq-1], itemlist[i]);
1005                 }
1006                 else if (!strcasecmp(itemlist[i], "RFC822.TEXT")) {
1007                         imap_fetch_rfc822(IMAP->msgids[seq-1], itemlist[i]);
1008                 }
1009
1010                 /* BODY fetches do their own fetching and caching too. */
1011                 else if (!strncasecmp(itemlist[i], "BODY[", 5)) {
1012                         imap_fetch_body(IMAP->msgids[seq-1], itemlist[i], 0);
1013                 }
1014                 else if (!strncasecmp(itemlist[i], "BODY.PEEK[", 10)) {
1015                         imap_fetch_body(IMAP->msgids[seq-1], itemlist[i], 1);
1016                 }
1017
1018                 /* Otherwise, load the message into memory.
1019                  */
1020                 else if (!strcasecmp(itemlist[i], "BODYSTRUCTURE")) {
1021                         if ((msg != NULL) && (!body_loaded)) {
1022                                 CtdlFreeMessage(msg);   /* need the whole thing */
1023                                 msg = NULL;
1024                         }
1025                         if (msg == NULL) {
1026                                 msg = CtdlFetchMessage(IMAP->msgids[seq-1], 1);
1027                                 body_loaded = 1;
1028                         }
1029                         imap_fetch_bodystructure(IMAP->msgids[seq-1],
1030                                         itemlist[i], msg);
1031                 }
1032                 else if (!strcasecmp(itemlist[i], "ENVELOPE")) {
1033                         if (msg == NULL) {
1034                                 msg = CtdlFetchMessage(IMAP->msgids[seq-1], 0);
1035                                 body_loaded = 0;
1036                         }
1037                         imap_fetch_envelope(msg);
1038                 }
1039                 else if (!strcasecmp(itemlist[i], "INTERNALDATE")) {
1040                         if (msg == NULL) {
1041                                 msg = CtdlFetchMessage(IMAP->msgids[seq-1], 0);
1042                                 body_loaded = 0;
1043                         }
1044                         imap_fetch_internaldate(msg);
1045                 }
1046
1047                 if (i != num_items-1) cprintf(" ");
1048         }
1049
1050         cprintf(")\r\n");
1051         unbuffer_output();
1052         if (msg != NULL) {
1053                 CtdlFreeMessage(msg);
1054         }
1055 }
1056
1057
1058
1059 /*
1060  * imap_fetch() calls imap_do_fetch() to do its actual work, once it's
1061  * validated and boiled down the request a bit.
1062  */
1063 void imap_do_fetch(int num_items, char **itemlist) {
1064         int i;
1065
1066         if (IMAP->num_msgs > 0) {
1067                 for (i = 0; i < IMAP->num_msgs; ++i) {
1068
1069                         /* Abort the fetch loop if the session breaks.
1070                          * This is important for users who keep mailboxes
1071                          * that are too big *and* are too impatient to
1072                          * let them finish loading.  :)
1073                          */
1074                         if (CC->kill_me) return;
1075
1076                         /* Get any message marked for fetch. */
1077                         if (IMAP->flags[i] & IMAP_SELECTED) {
1078                                 imap_do_fetch_msg(i+1, num_items, itemlist);
1079                         }
1080                 }
1081         }
1082 }
1083
1084
1085
1086 /*
1087  * Back end for imap_handle_macros()
1088  * Note that this function *only* looks at the beginning of the string.  It
1089  * is not a generic search-and-replace function.
1090  */
1091 void imap_macro_replace(char *str, char *find, char *replace) {
1092         char holdbuf[SIZ];
1093         int findlen;
1094
1095         findlen = strlen(find);
1096
1097         if (!strncasecmp(str, find, findlen)) {
1098                 if (str[findlen]==' ') {
1099                         strcpy(holdbuf, &str[findlen+1]);
1100                         strcpy(str, replace);
1101                         strcat(str, " ");
1102                         strcat(str, holdbuf);
1103                 }
1104                 if (str[findlen]==0) {
1105                         strcpy(holdbuf, &str[findlen+1]);
1106                         strcpy(str, replace);
1107                 }
1108         }
1109 }
1110
1111
1112
1113 /*
1114  * Handle macros embedded in FETCH data items.
1115  * (What the heck are macros doing in a wire protocol?  Are we trying to save
1116  * the computer at the other end the trouble of typing a lot of characters?)
1117  */
1118 void imap_handle_macros(char *str) {
1119         int i;
1120         int nest = 0;
1121
1122         for (i=0; str[i]; ++i) {
1123                 if (str[i]=='(') ++nest;
1124                 if (str[i]=='[') ++nest;
1125                 if (str[i]=='<') ++nest;
1126                 if (str[i]=='{') ++nest;
1127                 if (str[i]==')') --nest;
1128                 if (str[i]==']') --nest;
1129                 if (str[i]=='>') --nest;
1130                 if (str[i]=='}') --nest;
1131
1132                 if (nest <= 0) {
1133                         imap_macro_replace(&str[i],
1134                                 "ALL",
1135                                 "FLAGS INTERNALDATE RFC822.SIZE ENVELOPE"
1136                         );
1137                         imap_macro_replace(&str[i],
1138                                 "BODY",
1139                                 "BODYSTRUCTURE"
1140                         );
1141                         imap_macro_replace(&str[i],
1142                                 "FAST",
1143                                 "FLAGS INTERNALDATE RFC822.SIZE"
1144                         );
1145                         imap_macro_replace(&str[i],
1146                                 "FULL",
1147                                 "FLAGS INTERNALDATE RFC822.SIZE ENVELOPE BODY"
1148                         );
1149                 }
1150         }
1151 }
1152
1153
1154 /*
1155  * Break out the data items requested, possibly a parenthesized list.
1156  * Returns the number of data items, or -1 if the list is invalid.
1157  * NOTE: this function alters the string it is fed, and uses it as a buffer
1158  * to hold the data for the pointers it returns.
1159  */
1160 int imap_extract_data_items(char **argv, char *items) {
1161         int num_items = 0;
1162         int nest = 0;
1163         int i;
1164         char *start;
1165         long initial_len;
1166
1167         /* Convert all whitespace to ordinary space characters. */
1168         for (i=0; items[i]; ++i) {
1169                 if (isspace(items[i])) items[i]=' ';
1170         }
1171
1172         /* Strip leading and trailing whitespace, then strip leading and
1173          * trailing parentheses if it's a list
1174          */
1175         striplt(items);
1176         if ( (items[0]=='(') && (items[strlen(items)-1]==')') ) {
1177                 items[strlen(items)-1] = 0;
1178                 strcpy(items, &items[1]);
1179                 striplt(items);
1180         }
1181
1182         /* Parse any macro data items */
1183         imap_handle_macros(items);
1184
1185         /*
1186          * Now break out the data items.  We throw in one trailing space in
1187          * order to avoid having to break out the last one manually.
1188          */
1189         strcat(items, " ");
1190         start = items;
1191         initial_len = strlen(items);
1192         for (i=0; i<initial_len; ++i) {
1193                 if (items[i]=='(') ++nest;
1194                 if (items[i]=='[') ++nest;
1195                 if (items[i]=='<') ++nest;
1196                 if (items[i]=='{') ++nest;
1197                 if (items[i]==')') --nest;
1198                 if (items[i]==']') --nest;
1199                 if (items[i]=='>') --nest;
1200                 if (items[i]=='}') --nest;
1201
1202                 if (nest <= 0) if (items[i]==' ') {
1203                         items[i] = 0;
1204                         argv[num_items++] = start;
1205                         start = &items[i+1];
1206                 }
1207         }
1208
1209         return(num_items);
1210
1211 }
1212
1213
1214 /*
1215  * One particularly hideous aspect of IMAP is that we have to allow the client
1216  * to specify arbitrary ranges and/or sets of messages to fetch.  Citadel IMAP
1217  * handles this by setting the IMAP_SELECTED flag for each message specified in
1218  * the ranges/sets, then looping through the message array, outputting messages
1219  * with the flag set.  We don't bother returning an error if an out-of-range
1220  * number is specified (we just return quietly) because any client braindead
1221  * enough to request a bogus message number isn't going to notice the
1222  * difference anyway.
1223  *
1224  * This function clears out the IMAP_SELECTED bits, then sets that bit for each
1225  * message included in the specified range.
1226  *
1227  * Set is_uid to 1 to fetch by UID instead of sequence number.
1228  */
1229 void imap_pick_range(const char *supplied_range, int is_uid) {
1230         int i;
1231         int num_sets;
1232         int s;
1233         char setstr[SIZ], lostr[SIZ], histr[SIZ];
1234         long lo, hi;
1235         char actual_range[SIZ];
1236         citimap *Imap;
1237
1238         /* 
1239          * Handle the "ALL" macro
1240          */
1241         if (!strcasecmp(supplied_range, "ALL")) {
1242                 safestrncpy(actual_range, "1:*", sizeof actual_range);
1243         }
1244         else {
1245                 safestrncpy(actual_range, supplied_range, sizeof actual_range);
1246         }
1247
1248         Imap = IMAP;
1249         /*
1250          * Clear out the IMAP_SELECTED flags for all messages.
1251          */
1252         for (i = 0; i < Imap->num_msgs; ++i) {
1253                 Imap->flags[i] = Imap->flags[i] & ~IMAP_SELECTED;
1254         }
1255
1256         /*
1257          * Now set it for all specified messages.
1258          */
1259         num_sets = num_tokens(actual_range, ',');
1260         for (s=0; s<num_sets; ++s) {
1261                 extract_token(setstr, actual_range, s, ',', sizeof setstr);
1262
1263                 extract_token(lostr, setstr, 0, ':', sizeof lostr);
1264                 if (num_tokens(setstr, ':') >= 2) {
1265                         extract_token(histr, setstr, 1, ':', sizeof histr);
1266                         if (!strcmp(histr, "*")) snprintf(histr, sizeof histr, "%ld", LONG_MAX);
1267                 } 
1268                 else {
1269                         safestrncpy(histr, lostr, sizeof histr);
1270                 }
1271                 lo = atol(lostr);
1272                 hi = atol(histr);
1273
1274                 /* Loop through the array, flipping bits where appropriate */
1275                 for (i = 1; i <= Imap->num_msgs; ++i) {
1276                         if (is_uid) {   /* fetch by sequence number */
1277                                 if ( (Imap->msgids[i-1]>=lo)
1278                                    && (Imap->msgids[i-1]<=hi)) {
1279                                         Imap->flags[i-1] |= IMAP_SELECTED;
1280                                 }
1281                         }
1282                         else {          /* fetch by uid */
1283                                 if ( (i>=lo) && (i<=hi)) {
1284                                         Imap->flags[i-1] |= IMAP_SELECTED;
1285                                 }
1286                         }
1287                 }
1288         }
1289 }
1290
1291
1292
1293 /*
1294  * This function is called by the main command loop.
1295  */
1296 void imap_fetch(int num_parms, ConstStr *Params) {
1297         char items[SIZ];
1298         char *itemlist[512];
1299         int num_items;
1300         int i;
1301
1302         if (num_parms < 4) {
1303                 cprintf("%s BAD invalid parameters\r\n", Params[0].Key);
1304                 return;
1305         }
1306
1307         imap_pick_range(Params[2].Key, 0);
1308
1309         strcpy(items, "");
1310         for (i=3; i<num_parms; ++i) {
1311                 strcat(items, Params[i].Key);
1312                 if (i < (num_parms-1)) strcat(items, " ");
1313         }
1314
1315         num_items = imap_extract_data_items(itemlist, items);
1316         if (num_items < 1) {
1317                 cprintf("%s BAD invalid data item list\r\n", Params[0].Key);
1318                 return;
1319         }
1320
1321         imap_do_fetch(num_items, itemlist);
1322         cprintf("%s OK FETCH completed\r\n", Params[0].Key);
1323 }
1324
1325 /*
1326  * This function is called by the main command loop.
1327  */
1328 void imap_uidfetch(int num_parms, ConstStr *Params) {
1329         char items[SIZ];
1330         char *itemlist[512];
1331         int num_items;
1332         int i;
1333         int have_uid_item = 0;
1334
1335         if (num_parms < 5) {
1336                 cprintf("%s BAD invalid parameters\r\n", Params[0].Key);
1337                 return;
1338         }
1339
1340         imap_pick_range(Params[3].Key, 1);
1341
1342         strcpy(items, "");
1343         for (i=4; i<num_parms; ++i) {
1344                 strcat(items, Params[i].Key);
1345                 if (i < (num_parms-1)) strcat(items, " ");
1346         }
1347
1348         num_items = imap_extract_data_items(itemlist, items);
1349         if (num_items < 1) {
1350                 cprintf("%s BAD invalid data item list\r\n", Params[0].Key);
1351                 return;
1352         }
1353
1354         /* If the "UID" item was not included, we include it implicitly
1355          * (at the beginning) because this is a UID FETCH command
1356          */
1357         for (i=0; i<num_items; ++i) {
1358                 if (!strcasecmp(itemlist[i], "UID")) ++have_uid_item;
1359         }
1360         if (have_uid_item == 0) {
1361                 memmove(&itemlist[1], &itemlist[0], (sizeof(itemlist[0]) * num_items));
1362                 ++num_items;
1363                 itemlist[0] = "UID";
1364         }
1365
1366         imap_do_fetch(num_items, itemlist);
1367         cprintf("%s OK UID FETCH completed\r\n", Params[0].Key);
1368 }
1369
1370