* Fixed bug #467, possibly at the cost of poorer performance.
[citadel.git] / citadel / modules / imap / imap_fetch.c
1 /*
2  * $Id$
3  *
4  * Implements the FETCH command in IMAP.
5  * This is a good example of the protocol's gratuitous complexity.
6  *
7  */
8
9
10 #include "sysdep.h"
11 #include <stdlib.h>
12 #include <unistd.h>
13 #include <stdio.h>
14 #include <fcntl.h>
15 #include <signal.h>
16 #include <pwd.h>
17 #include <errno.h>
18 #include <sys/types.h>
19
20 #if TIME_WITH_SYS_TIME
21 # include <sys/time.h>
22 # include <time.h>
23 #else
24 # if HAVE_SYS_TIME_H
25 #  include <sys/time.h>
26 # else
27 #  include <time.h>
28 # endif
29 #endif
30
31 #include <sys/wait.h>
32 #include <ctype.h>
33 #include <string.h>
34 #include <limits.h>
35 #include <libcitadel.h>
36 #include "citadel.h"
37 #include "server.h"
38 #include "sysdep_decls.h"
39 #include "citserver.h"
40 #include "support.h"
41 #include "config.h"
42 #include "room_ops.h"
43 #include "user_ops.h"
44 #include "policy.h"
45 #include "database.h"
46 #include "msgbase.h"
47 #include "internet_addressing.h"
48 #include "serv_imap.h"
49 #include "imap_tools.h"
50 #include "imap_fetch.h"
51 #include "genstamp.h"
52 #include "ctdl_module.h"
53
54
55
56 /*
57  * Individual field functions for imap_do_fetch_msg() ...
58  */
59
60 void imap_fetch_uid(int seq) {
61         cprintf("UID %ld", IMAP->msgids[seq-1]);
62 }
63
64 void imap_fetch_flags(int seq) {
65         int num_flags_printed = 0;
66         cprintf("FLAGS (");
67         if (IMAP->flags[seq] & IMAP_DELETED) {
68                 if (num_flags_printed > 0) cprintf(" ");
69                 cprintf("\\Deleted");
70                 ++num_flags_printed;
71         }
72         if (IMAP->flags[seq] & IMAP_SEEN) {
73                 if (num_flags_printed > 0) cprintf(" ");
74                 cprintf("\\Seen");
75                 ++num_flags_printed;
76         }
77         if (IMAP->flags[seq] & IMAP_ANSWERED) {
78                 if (num_flags_printed > 0) cprintf(" ");
79                 cprintf("\\Answered");
80                 ++num_flags_printed;
81         }
82         if (IMAP->flags[seq] & IMAP_RECENT) {
83                 if (num_flags_printed > 0) cprintf(" ");
84                 cprintf("\\Recent");
85                 ++num_flags_printed;
86         }
87         cprintf(")");
88 }
89
90 void imap_fetch_internaldate(struct CtdlMessage *msg) {
91         char buf[SIZ];
92         time_t msgdate;
93
94         if (!msg) return;
95         if (msg->cm_fields['T'] != NULL) {
96                 msgdate = atol(msg->cm_fields['T']);
97         }
98         else {
99                 msgdate = time(NULL);
100         }
101
102         datestring(buf, sizeof buf, msgdate, DATESTRING_IMAP);
103         cprintf("INTERNALDATE \"%s\"", buf);
104 }
105
106
107 /*
108  * Fetch RFC822-formatted messages.
109  *
110  * 'whichfmt' should be set to one of:
111  *      "RFC822"        entire message
112  *      "RFC822.HEADER" headers only (with trailing blank line)
113  *      "RFC822.SIZE"   size of translated message
114  *      "RFC822.TEXT"   body only (without leading blank line)
115  */
116 void imap_fetch_rfc822(long msgnum, char *whichfmt) {
117         char buf[SIZ];
118         char *ptr = NULL;
119         size_t headers_size, text_size, total_size;
120         size_t bytes_to_send = 0;
121         struct MetaData smi;
122         int need_to_rewrite_metadata = 0;
123         int need_body = 0;
124
125         /* Determine whether this particular fetch operation requires
126          * us to fetch the message body from disk.  If not, we can save
127          * on some disk operations...
128          */
129         if ( (!strcasecmp(whichfmt, "RFC822"))
130            || (!strcasecmp(whichfmt, "RFC822.TEXT")) ) {
131                 need_body = 1;
132         }
133
134         /* If this is an RFC822.SIZE fetch, first look in the message's
135          * metadata record to see if we've saved that information.
136          */
137         if (!strcasecmp(whichfmt, "RFC822.SIZE")) {
138                 GetMetaData(&smi, msgnum);
139                 if (smi.meta_rfc822_length > 0L) {
140                         cprintf("RFC822.SIZE %ld", smi.meta_rfc822_length);
141                         return;
142                 }
143                 need_to_rewrite_metadata = 1;
144                 need_body = 1;
145         }
146         
147         /* Cache the most recent RFC822 FETCH because some clients like to
148          * fetch in pieces, and we don't want to have to go back to the
149          * message store for each piece.  We also burn the cache if the
150          * client requests something that involves reading the message
151          * body, but we haven't fetched the body yet.
152          */
153         if ((IMAP->cached_rfc822_data != NULL)
154            && (IMAP->cached_rfc822_msgnum == msgnum)
155            && (IMAP->cached_rfc822_withbody || (!need_body)) ) {
156                 /* Good to go! */
157         }
158         else if (IMAP->cached_rfc822_data != NULL) {
159                 /* Some other message is cached -- free it */
160                 free(IMAP->cached_rfc822_data);
161                 IMAP->cached_rfc822_data = NULL;
162                 IMAP->cached_rfc822_msgnum = (-1);
163                 IMAP->cached_rfc822_len = 0;
164         }
165
166         /* At this point, we now can fetch and convert the message iff it's not
167          * the one we had cached.
168          */
169         if (IMAP->cached_rfc822_data == NULL) {
170                 /*
171                  * Load the message into memory for translation & measurement
172                  */
173                 CC->redirect_buffer = malloc(SIZ);
174                 CC->redirect_len = 0;
175                 CC->redirect_alloc = SIZ;
176                 CtdlOutputMsg(msgnum, MT_RFC822,
177                         (need_body ? HEADERS_ALL : HEADERS_FAST),
178                               0, 1, NULL, 0);
179                 if (!need_body) cprintf("\r\n");        /* extra trailing newline */
180                 IMAP->cached_rfc822_data = CC->redirect_buffer;
181                 IMAP->cached_rfc822_len = CC->redirect_len;
182                 IMAP->cached_rfc822_msgnum = msgnum;
183                 IMAP->cached_rfc822_withbody = need_body;
184                 CC->redirect_buffer = NULL;
185                 CC->redirect_len = 0;
186                 CC->redirect_alloc = 0;
187                 if ( (need_to_rewrite_metadata) && (IMAP->cached_rfc822_len > 0) ) {
188                         smi.meta_rfc822_length = (long)IMAP->cached_rfc822_len;
189                         PutMetaData(&smi);
190                 }
191         }
192
193         /*
194          * Now figure out where the headers/text break is.  IMAP considers the
195          * intervening blank line to be part of the headers, not the text.
196          */
197         headers_size = 0;
198         text_size = 0;
199         total_size = 0;
200
201         if (need_body) {
202                 ptr = IMAP->cached_rfc822_data;
203                 do {
204                         ptr = memreadline(ptr, buf, sizeof buf);
205                         if (*ptr != 0) {
206                                 striplt(buf);
207                                 if (IsEmptyStr(buf)) {
208                                         headers_size = ptr - IMAP->cached_rfc822_data;
209                                 }
210                         }
211                 } while ( (headers_size == 0) && (*ptr != 0) );
212
213                 total_size = IMAP->cached_rfc822_len;
214                 text_size = total_size - headers_size;
215         }
216         else {
217                 headers_size = IMAP->cached_rfc822_len;
218                 total_size = IMAP->cached_rfc822_len;
219                 text_size = 0;
220         }
221
222         CtdlLogPrintf(CTDL_DEBUG, 
223                 "RFC822: headers=" SIZE_T_FMT 
224                 ", text=" SIZE_T_FMT
225                 ", total=" SIZE_T_FMT "\n",
226                 headers_size, text_size, total_size);
227
228         if (!strcasecmp(whichfmt, "RFC822.SIZE")) {
229                 cprintf("RFC822.SIZE " SIZE_T_FMT, total_size);
230                 return;
231         }
232
233         else if (!strcasecmp(whichfmt, "RFC822")) {
234                 ptr = IMAP->cached_rfc822_data;
235                 bytes_to_send = total_size;
236         }
237
238         else if (!strcasecmp(whichfmt, "RFC822.HEADER")) {
239                 ptr = IMAP->cached_rfc822_data;
240                 bytes_to_send = headers_size;
241         }
242
243         else if (!strcasecmp(whichfmt, "RFC822.TEXT")) {
244                 ptr = &IMAP->cached_rfc822_data[headers_size];
245                 bytes_to_send = text_size;
246         }
247
248         cprintf("%s {" SIZE_T_FMT "}\r\n", whichfmt, bytes_to_send);
249         client_write(ptr, bytes_to_send);
250 }
251
252
253
254 /*
255  * Load a specific part of a message into the temp file to be output to a
256  * client.  FIXME we can handle parts like "2" and "2.1" and even "2.MIME"
257  * but we still can't handle "2.HEADER" (which might not be a problem).
258  *
259  * Note: mime_parser() was called with dont_decode set to 1, so we have the
260  * luxury of simply spewing without having to re-encode.
261  */
262 void imap_load_part(char *name, char *filename, char *partnum, char *disp,
263                     void *content, char *cbtype, char *cbcharset, size_t length, char *encoding,
264                     char *cbid, void *cbuserdata)
265 {
266         char mbuf2[SIZ];
267         char *desired_section;
268
269         desired_section = (char *)cbuserdata;
270
271         if (!strcasecmp(partnum, desired_section)) {
272                 client_write(content, length);
273         }
274
275         snprintf(mbuf2, sizeof mbuf2, "%s.MIME", partnum);
276
277         if (!strcasecmp(desired_section, mbuf2)) {
278                 cprintf("Content-type: %s", cbtype);
279                 if (!IsEmptyStr(cbcharset))
280                         cprintf("; charset=\"%s\"", cbcharset);
281                 if (!IsEmptyStr(name))
282                         cprintf("; name=\"%s\"", name);
283                 cprintf("\r\n");
284                 if (!IsEmptyStr(encoding))
285                         cprintf("Content-Transfer-Encoding: %s\r\n", encoding);
286                 if (!IsEmptyStr(encoding)) {
287                         cprintf("Content-Disposition: %s", disp);
288                         if (!IsEmptyStr(filename)) {
289                                 cprintf("; filename=\"%s\"", filename);
290                         }
291                         cprintf("\r\n");
292                 }
293                 cprintf("Content-Length: %ld\r\n", (long)length);
294                 cprintf("\r\n");
295         }
296                         
297
298 }
299
300
301 /* 
302  * Called by imap_fetch_envelope() to output the "From" field.
303  * This is in its own function because its logic is kind of complex.  We
304  * really need to make this suck less.
305  */
306 void imap_output_envelope_from(struct CtdlMessage *msg) {
307         char user[SIZ], node[SIZ], name[SIZ];
308
309         if (!msg) return;
310
311         /* For anonymous messages, it's so easy! */
312         if (!is_room_aide() && (msg->cm_anon_type == MES_ANONONLY)) {
313                 cprintf("((\"----\" NIL \"x\" \"x.org\")) ");
314                 return;
315         }
316         if (!is_room_aide() && (msg->cm_anon_type == MES_ANONOPT)) {
317                 cprintf("((\"anonymous\" NIL \"x\" \"x.org\")) ");
318                 return;
319         }
320
321         /* For everything else, we do stuff. */
322         cprintf("((");                          /* open double-parens */
323         imap_strout(msg->cm_fields['A']);       /* personal name */
324         cprintf(" NIL ");                       /* source route (not used) */
325
326
327         if (msg->cm_fields['F'] != NULL) {
328                 process_rfc822_addr(msg->cm_fields['F'], user, node, name);
329                 imap_strout(user);              /* mailbox name (user id) */
330                 cprintf(" ");
331                 if (!strcasecmp(node, config.c_nodename)) {
332                         imap_strout(config.c_fqdn);
333                 }
334                 else {
335                         imap_strout(node);              /* host name */
336                 }
337         }
338         else {
339                 imap_strout(msg->cm_fields['A']); /* mailbox name (user id) */
340                 cprintf(" ");
341                 imap_strout(msg->cm_fields['N']);       /* host name */
342         }
343         
344         cprintf(")) ");                         /* close double-parens */
345 }
346
347
348
349 /*
350  * Output an envelope address (or set of addresses) in the official,
351  * convoluted, braindead format.  (Note that we can't use this for
352  * the "From" address because its data may come from a number of different
353  * fields.  But we can use it for "To" and possibly others.
354  */
355 void imap_output_envelope_addr(char *addr) {
356         char individual_addr[256];
357         int num_addrs;
358         int i;
359         char user[256];
360         char node[256];
361         char name[256];
362
363         if (addr == NULL) {
364                 cprintf("NIL ");
365                 return;
366         }
367
368         if (IsEmptyStr(addr)) {
369                 cprintf("NIL ");
370                 return;
371         }
372
373         cprintf("(");
374
375         /* How many addresses are listed here? */
376         num_addrs = num_tokens(addr, ',');
377
378         /* Output them one by one. */
379         for (i=0; i<num_addrs; ++i) {
380                 extract_token(individual_addr, addr, i, ',', sizeof individual_addr);
381                 striplt(individual_addr);
382                 process_rfc822_addr(individual_addr, user, node, name);
383                 cprintf("(");
384                 imap_strout(name);
385                 cprintf(" NIL ");
386                 imap_strout(user);
387                 cprintf(" ");
388                 imap_strout(node);
389                 cprintf(")");
390                 if (i < (num_addrs-1)) cprintf(" ");
391         }
392
393         cprintf(") ");
394 }
395
396
397 /*
398  * Implements the ENVELOPE fetch item
399  * 
400  * Note that the imap_strout() function can cleverly output NULL fields as NIL,
401  * so we don't have to check for that condition like we do elsewhere.
402  */
403 void imap_fetch_envelope(struct CtdlMessage *msg) {
404         char datestringbuf[SIZ];
405         time_t msgdate;
406         char *fieldptr = NULL;
407
408         if (!msg) return;
409
410         /* Parse the message date into an IMAP-format date string */
411         if (msg->cm_fields['T'] != NULL) {
412                 msgdate = atol(msg->cm_fields['T']);
413         }
414         else {
415                 msgdate = time(NULL);
416         }
417         datestring(datestringbuf, sizeof datestringbuf,
418                 msgdate, DATESTRING_IMAP);
419
420         /* Now start spewing data fields.  The order is important, as it is
421          * defined by the protocol specification.  Nonexistent fields must
422          * be output as NIL, existent fields must be quoted or literalled.
423          * The imap_strout() function conveniently does all this for us.
424          */
425         cprintf("ENVELOPE (");
426
427         /* Date */
428         imap_strout(datestringbuf);
429         cprintf(" ");
430
431         /* Subject */
432         imap_strout(msg->cm_fields['U']);
433         cprintf(" ");
434
435         /* From */
436         imap_output_envelope_from(msg);
437
438         /* Sender (default to same as 'From' if not present) */
439         fieldptr = rfc822_fetch_field(msg->cm_fields['M'], "Sender");
440         if (fieldptr != NULL) {
441                 imap_output_envelope_addr(fieldptr);
442                 free(fieldptr);
443         }
444         else {
445                 imap_output_envelope_from(msg);
446         }
447
448         /* Reply-to */
449         fieldptr = rfc822_fetch_field(msg->cm_fields['M'], "Reply-to");
450         if (fieldptr != NULL) {
451                 imap_output_envelope_addr(fieldptr);
452                 free(fieldptr);
453         }
454         else {
455                 imap_output_envelope_from(msg);
456         }
457
458         /* To */
459         imap_output_envelope_addr(msg->cm_fields['R']);
460
461         /* Cc (we do it this way because there might be a legacy non-Citadel Cc: field present) */
462         fieldptr = msg->cm_fields['Y'];
463         if (fieldptr != NULL) {
464                 imap_output_envelope_addr(fieldptr);
465         }
466         else {
467                 fieldptr = rfc822_fetch_field(msg->cm_fields['M'], "Cc");
468                 imap_output_envelope_addr(fieldptr);
469                 if (fieldptr != NULL) free(fieldptr);
470         }
471
472         /* Bcc */
473         fieldptr = rfc822_fetch_field(msg->cm_fields['M'], "Bcc");
474         imap_output_envelope_addr(fieldptr);
475         if (fieldptr != NULL) free(fieldptr);
476
477         /* In-reply-to */
478         fieldptr = rfc822_fetch_field(msg->cm_fields['M'], "In-reply-to");
479         imap_strout(fieldptr);
480         cprintf(" ");
481         if (fieldptr != NULL) free(fieldptr);
482
483         /* message ID */
484         imap_strout(msg->cm_fields['I']);
485
486         cprintf(")");
487 }
488
489 /*
490  * This function is called only when CC->redirect_buffer contains a set of
491  * RFC822 headers with no body attached.  Its job is to strip that set of
492  * headers down to *only* the ones we're interested in.
493  */
494 void imap_strip_headers(char *section) {
495         char buf[SIZ];
496         char *which_fields = NULL;
497         int doing_headers = 0;
498         int headers_not = 0;
499         char *parms[SIZ];
500         int num_parms = 0;
501         int i;
502         char *boiled_headers = NULL;
503         int ok = 0;
504         int done_headers = 0;
505         char *ptr = NULL;
506
507         if (CC->redirect_buffer == NULL) return;
508
509         which_fields = strdup(section);
510
511         if (!strncasecmp(which_fields, "HEADER.FIELDS", 13))
512                 doing_headers = 1;
513         if (!strncasecmp(which_fields, "HEADER.FIELDS.NOT", 17))
514                 headers_not = 1;
515
516         for (i=0; which_fields[i]; ++i) {
517                 if (which_fields[i]=='(')
518                         strcpy(which_fields, &which_fields[i+1]);
519         }
520         for (i=0; which_fields[i]; ++i) {
521                 if (which_fields[i]==')') {
522                         which_fields[i] = 0;
523                         break;
524                 }
525         }
526         num_parms = imap_parameterize(parms, which_fields);
527
528         boiled_headers = malloc(CC->redirect_alloc);
529         strcpy(boiled_headers, "");
530
531         ptr = CC->redirect_buffer;
532         ok = 0;
533         do {
534                 ptr = memreadline(ptr, buf, sizeof buf);
535                 if (!isspace(buf[0])) {
536                         ok = 0;
537                         if (doing_headers == 0) ok = 1;
538                         else {
539                                 if (headers_not) ok = 1;
540                                 else ok = 0;
541                                 for (i=0; i<num_parms; ++i) {
542                                         if ( (!strncasecmp(buf, parms[i],
543                                            strlen(parms[i]))) &&
544                                            (buf[strlen(parms[i])]==':') ) {
545                                                 if (headers_not) ok = 0;
546                                                 else ok = 1;
547                                         }
548                                 }
549                         }
550                 }
551
552                 if (ok) {
553                         strcat(boiled_headers, buf);
554                         strcat(boiled_headers, "\r\n");
555                 }
556
557                 if (IsEmptyStr(buf)) done_headers = 1;
558                 if (buf[0]=='\r') done_headers = 1;
559                 if (buf[0]=='\n') done_headers = 1;
560                 if (*ptr == 0) done_headers = 1;
561         } while (!done_headers);
562
563         strcat(boiled_headers, "\r\n");
564
565         /* Now save it back (it'll always be smaller) */
566         strcpy(CC->redirect_buffer, boiled_headers);
567         CC->redirect_len = strlen(boiled_headers);
568
569         free(which_fields);
570         free(boiled_headers);
571 }
572
573
574 /*
575  * Implements the BODY and BODY.PEEK fetch items
576  */
577 void imap_fetch_body(long msgnum, char *item, int is_peek) {
578         struct CtdlMessage *msg = NULL;
579         char section[SIZ];
580         char partial[SIZ];
581         int is_partial = 0;
582         size_t pstart, pbytes;
583         int loading_body_now = 0;
584         int need_body = 1;
585         int burn_the_cache = 0;
586
587         /* extract section */
588         safestrncpy(section, item, sizeof section);
589         if (strchr(section, '[') != NULL) {
590                 stripallbut(section, '[', ']');
591         }
592         CtdlLogPrintf(CTDL_DEBUG, "Section is: %s%s\n", 
593                 section, 
594                 IsEmptyStr(section) ? "(empty)" : "");
595
596         /* Burn the cache if we don't have the same section of the 
597          * same message again.
598          */
599         if (IMAP->cached_body != NULL) {
600                 if (IMAP->cached_bodymsgnum != msgnum) {
601                         burn_the_cache = 1;
602                 }
603                 else if ( (!IMAP->cached_body_withbody) && (need_body) ) {
604                         burn_the_cache = 1;
605                 }
606                 else if (strcasecmp(IMAP->cached_bodypart, section)) {
607                         burn_the_cache = 1;
608                 }
609                 if (burn_the_cache) {
610                         /* Yup, go ahead and burn the cache. */
611                         free(IMAP->cached_body);
612                         IMAP->cached_body_len = 0;
613                         IMAP->cached_body = NULL;
614                         IMAP->cached_bodymsgnum = (-1);
615                         strcpy(IMAP->cached_bodypart, "");
616                 }
617         }
618
619         /* extract partial */
620         safestrncpy(partial, item, sizeof partial);
621         if (strchr(partial, '<') != NULL) {
622                 stripallbut(partial, '<', '>');
623                 is_partial = 1;
624         }
625         if (is_partial == 0) strcpy(partial, "");
626         /* if (!IsEmptyStr(partial)) CtdlLogPrintf(CTDL_DEBUG, "Partial is %s\n", partial); */
627
628         if (IMAP->cached_body == NULL) {
629                 CC->redirect_buffer = malloc(SIZ);
630                 CC->redirect_len = 0;
631                 CC->redirect_alloc = SIZ;
632                 loading_body_now = 1;
633                 msg = CtdlFetchMessage(msgnum, (need_body ? 1 : 0));
634         }
635
636         /* Now figure out what the client wants, and get it */
637
638         if (!loading_body_now) {
639                 /* What we want is already in memory */
640         }
641
642         else if ( (!strcmp(section, "1")) && (msg->cm_format_type != 4) ) {
643                 CtdlOutputPreLoadedMsg(msg, MT_RFC822, HEADERS_NONE, 0, 1, 0);
644         }
645
646         else if (!strcmp(section, "")) {
647                 CtdlOutputPreLoadedMsg(msg, MT_RFC822, HEADERS_ALL, 0, 1, 0);
648         }
649
650         /*
651          * If the client asked for just headers, or just particular header
652          * fields, strip it down.
653          */
654         else if (!strncasecmp(section, "HEADER", 6)) {
655                 /* This used to work with HEADERS_FAST, but then Apple got stupid with their
656                  * IMAP library and this broke Mail.App and iPhone Mail, so we had to change it
657                  * to HEADERS_ONLY so the trendy hipsters with their iPhones can read mail.
658                  */
659                 CtdlOutputPreLoadedMsg(msg, MT_RFC822, HEADERS_ONLY, 0, 1, 0);
660                 imap_strip_headers(section);
661         }
662
663         /*
664          * Strip it down if the client asked for everything _except_ headers.
665          */
666         else if (!strncasecmp(section, "TEXT", 4)) {
667                 CtdlOutputPreLoadedMsg(msg, MT_RFC822, HEADERS_NONE, 0, 1, 0);
668         }
669
670         /*
671          * Anything else must be a part specifier.
672          * (Note value of 1 passed as 'dont_decode' so client gets it encoded)
673          */
674         else {
675                 mime_parser(msg->cm_fields['M'], NULL,
676                                 *imap_load_part, NULL, NULL,
677                                 section,
678                                 1);
679         }
680
681         if (loading_body_now) {
682                 IMAP->cached_body = CC->redirect_buffer;
683                 IMAP->cached_body_len = CC->redirect_len;
684                 IMAP->cached_bodymsgnum = msgnum;
685                 IMAP->cached_body_withbody = need_body;
686                 strcpy(IMAP->cached_bodypart, section);
687                 CC->redirect_buffer = NULL;
688                 CC->redirect_len = 0;
689                 CC->redirect_alloc = 0;
690         }
691
692         if (is_partial == 0) {
693                 cprintf("BODY[%s] {" SIZE_T_FMT "}\r\n", section, IMAP->cached_body_len);
694                 pstart = 0;
695                 pbytes = IMAP->cached_body_len;
696         }
697         else {
698                 sscanf(partial, SIZE_T_FMT "." SIZE_T_FMT, &pstart, &pbytes);
699                 if (pbytes > (IMAP->cached_body_len - pstart)) {
700                         pbytes = IMAP->cached_body_len - pstart;
701                 }
702                 cprintf("BODY[%s]<" SIZE_T_FMT "> {" SIZE_T_FMT "}\r\n", section, pstart, pbytes);
703         }
704
705         /* Here we go -- output it */
706         client_write(&IMAP->cached_body[pstart], pbytes);
707
708         if (msg != NULL) {
709                 CtdlFreeMessage(msg);
710         }
711
712         /* Mark this message as "seen" *unless* this is a "peek" operation */
713         if (is_peek == 0) {
714                 CtdlSetSeen(&msgnum, 1, 1, ctdlsetseen_seen, NULL, NULL);
715         }
716 }
717
718 /*
719  * Called immediately before outputting a multipart bodystructure
720  */
721 void imap_fetch_bodystructure_pre(
722                 char *name, char *filename, char *partnum, char *disp,
723                 void *content, char *cbtype, char *cbcharset, size_t length, char *encoding,
724                 char *cbid, void *cbuserdata
725                 ) {
726
727         cprintf("(");
728 }
729
730
731
732 /*
733  * Called immediately after outputting a multipart bodystructure
734  */
735 void imap_fetch_bodystructure_post(
736                 char *name, char *filename, char *partnum, char *disp,
737                 void *content, char *cbtype, char *cbcharset, size_t length, char *encoding,
738                 char *cbid, void *cbuserdata
739                 ) {
740
741         char subtype[128];
742
743         cprintf(" ");
744
745         /* disposition */
746         extract_token(subtype, cbtype, 1, '/', sizeof subtype);
747         imap_strout(subtype);
748
749         /* body language */
750         /* cprintf(" NIL"); We thought we needed this at one point, but maybe we don't... */
751
752         cprintf(")");
753 }
754
755
756
757 /*
758  * Output the info for a MIME part in the format required by BODYSTRUCTURE.
759  *
760  */
761 void imap_fetch_bodystructure_part(
762                 char *name, char *filename, char *partnum, char *disp,
763                 void *content, char *cbtype, char *cbcharset, size_t length, char *encoding,
764                 char *cbid, void *cbuserdata
765                 ) {
766
767         int have_cbtype = 0;
768         int have_encoding = 0;
769         int lines = 0;
770         size_t i;
771         char cbmaintype[128];
772         char cbsubtype[128];
773
774         if (cbtype != NULL) if (!IsEmptyStr(cbtype)) have_cbtype = 1;
775         if (have_cbtype) {
776                 extract_token(cbmaintype, cbtype, 0, '/', sizeof cbmaintype);
777                 extract_token(cbsubtype, cbtype, 1, '/', sizeof cbsubtype);
778         }
779         else {
780                 strcpy(cbmaintype, "TEXT");
781                 strcpy(cbsubtype, "PLAIN");
782         }
783
784         cprintf("(");
785         imap_strout(cbmaintype);                                        /* body type */
786         cprintf(" ");
787         imap_strout(cbsubtype);                                         /* body subtype */
788         cprintf(" ");
789
790         cprintf("(");                                                   /* begin body parameter list */
791
792         /* "NAME" must appear as the first parameter.  This is not required by IMAP,
793          * but the Asterisk voicemail application blindly assumes that NAME will be in
794          * the first position.  If it isn't, it rejects the message.
795          */
796         if (name != NULL) if (!IsEmptyStr(name)) {
797                 cprintf("\"NAME\" ");
798                 imap_strout(name);
799                 cprintf(" ");
800         }
801
802         cprintf("\"CHARSET\" ");
803         if (cbcharset == NULL) {
804                 imap_strout("US-ASCII");
805         }
806         else if (cbcharset[0] == 0) {
807                 imap_strout("US-ASCII");
808         }
809         else {
810                 imap_strout(cbcharset);
811         }
812         cprintf(") ");                                                  /* end body parameter list */
813
814         cprintf("NIL ");                                                /* Body ID */
815         cprintf("NIL ");                                                /* Body description */
816
817         if (encoding != NULL) if (encoding[0] != 0)  have_encoding = 1;
818         if (have_encoding) {
819                 imap_strout(encoding);
820         }
821         else {
822                 imap_strout("7BIT");
823         }
824         cprintf(" ");
825
826         /* The next field is the size of the part in bytes. */
827         cprintf("%ld ", (long)length);  /* bytes */
828
829         /* The next field is the number of lines in the part, if and only
830          * if the part is TEXT.  More gratuitous complexity.
831          */
832         if (!strcasecmp(cbmaintype, "TEXT")) {
833                 if (length) for (i=0; i<length; ++i) {
834                         if (((char *)content)[i] == '\n') ++lines;
835                 }
836                 cprintf("%d ", lines);
837         }
838
839         /* More gratuitous complexity */
840         if ((!strcasecmp(cbmaintype, "MESSAGE"))
841            && (!strcasecmp(cbsubtype, "RFC822"))) {
842                 /* FIXME: message/rfc822 also needs to output the envelope structure,
843                  * body structure, and line count of the encapsulated message.  Fortunately
844                  * there are not yet any clients depending on this, so we can get away
845                  * with not implementing it for now.
846                  */
847         }
848
849         /* MD5 value of body part; we can get away with NIL'ing this */
850         cprintf("NIL ");
851
852         /* Disposition */
853         if (disp == NULL) {
854                 cprintf("NIL");
855         }
856         else if (IsEmptyStr(disp)) {
857                 cprintf("NIL");
858         }
859         else {
860                 cprintf("(");
861                 imap_strout(disp);
862                 if (filename != NULL) if (!IsEmptyStr(filename)) {
863                         cprintf(" (\"FILENAME\" ");
864                         imap_strout(filename);
865                         cprintf(")");
866                 }
867                 cprintf(")");
868         }
869
870         /* Body language (not defined yet) */
871         cprintf(" NIL)");
872 }
873
874
875
876 /*
877  * Spew the BODYSTRUCTURE data for a message.
878  *
879  */
880 void imap_fetch_bodystructure (long msgnum, char *item,
881                 struct CtdlMessage *msg) {
882         char *rfc822 = NULL;
883         char *rfc822_body = NULL;
884         size_t rfc822_len;
885         size_t rfc822_headers_len;
886         size_t rfc822_body_len;
887         char *ptr = NULL;
888         char buf[SIZ];
889         int lines = 0;
890
891         /* Handle NULL message gracefully */
892         if (msg == NULL) {
893                 cprintf("BODYSTRUCTURE (\"TEXT\" \"PLAIN\" "
894                         "(\"CHARSET\" \"US-ASCII\") NIL NIL "
895                         "\"7BIT\" 0 0)");
896                 return;
897         }
898
899         /* For non-RFC822 (ordinary Citadel) messages, this is short and
900          * sweet...
901          */
902         if (msg->cm_format_type != FMT_RFC822) {
903
904                 /* *sigh* We have to RFC822-format the message just to be able
905                  * to measure it.  FIXME use smi cached fields if possible
906                  */
907
908                 CC->redirect_buffer = malloc(SIZ);
909                 CC->redirect_len = 0;
910                 CC->redirect_alloc = SIZ;
911                 CtdlOutputPreLoadedMsg(msg, MT_RFC822, 0, 0, 1, 0);
912                 rfc822 = CC->redirect_buffer;
913                 rfc822_len = CC->redirect_len;
914                 CC->redirect_buffer = NULL;
915                 CC->redirect_len = 0;
916                 CC->redirect_alloc = 0;
917
918                 ptr = rfc822;
919                 do {
920                         ptr = memreadline(ptr, buf, sizeof buf);
921                         ++lines;
922                         if ((IsEmptyStr(buf)) && (rfc822_body == NULL)) {
923                                 rfc822_body = ptr;
924                         }
925                 } while (*ptr != 0);
926
927                 rfc822_headers_len = rfc822_body - rfc822;
928                 rfc822_body_len = rfc822_len - rfc822_headers_len;
929                 free(rfc822);
930
931                 cprintf("BODYSTRUCTURE (\"TEXT\" \"PLAIN\" "
932                         "(\"CHARSET\" \"US-ASCII\") NIL NIL "
933                         "\"7BIT\" " SIZE_T_FMT " %d)", rfc822_body_len, lines);
934
935                 return;
936         }
937
938         /* For messages already stored in RFC822 format, we have to parse. */
939         cprintf("BODYSTRUCTURE ");
940         mime_parser(msg->cm_fields['M'],
941                         NULL,
942                         *imap_fetch_bodystructure_part, /* part */
943                         *imap_fetch_bodystructure_pre,  /* pre-multi */
944                         *imap_fetch_bodystructure_post, /* post-multi */
945                         NULL,
946                         1);     /* don't decode -- we want it as-is */
947 }
948
949
950 /*
951  * imap_do_fetch() calls imap_do_fetch_msg() to output the data of an
952  * individual message, once it has been selected for output.
953  */
954 void imap_do_fetch_msg(int seq, int num_items, char **itemlist) {
955         int i;
956         struct CtdlMessage *msg = NULL;
957         int body_loaded = 0;
958
959         /* Don't attempt to fetch bogus messages or UID's */
960         if (seq < 1) return;
961         if (IMAP->msgids[seq-1] < 1L) return;
962
963         buffer_output();
964         cprintf("* %d FETCH (", seq);
965
966         for (i=0; i<num_items; ++i) {
967
968                 /* Fetchable without going to the message store at all */
969                 if (!strcasecmp(itemlist[i], "UID")) {
970                         imap_fetch_uid(seq);
971                 }
972                 else if (!strcasecmp(itemlist[i], "FLAGS")) {
973                         imap_fetch_flags(seq-1);
974                 }
975
976                 /* Potentially fetchable from cache, if the client requests
977                  * stuff from the same message several times in a row.
978                  */
979                 else if (!strcasecmp(itemlist[i], "RFC822")) {
980                         imap_fetch_rfc822(IMAP->msgids[seq-1], itemlist[i]);
981                 }
982                 else if (!strcasecmp(itemlist[i], "RFC822.HEADER")) {
983                         imap_fetch_rfc822(IMAP->msgids[seq-1], itemlist[i]);
984                 }
985                 else if (!strcasecmp(itemlist[i], "RFC822.SIZE")) {
986                         imap_fetch_rfc822(IMAP->msgids[seq-1], itemlist[i]);
987                 }
988                 else if (!strcasecmp(itemlist[i], "RFC822.TEXT")) {
989                         imap_fetch_rfc822(IMAP->msgids[seq-1], itemlist[i]);
990                 }
991
992                 /* BODY fetches do their own fetching and caching too. */
993                 else if (!strncasecmp(itemlist[i], "BODY[", 5)) {
994                         imap_fetch_body(IMAP->msgids[seq-1], itemlist[i], 0);
995                 }
996                 else if (!strncasecmp(itemlist[i], "BODY.PEEK[", 10)) {
997                         imap_fetch_body(IMAP->msgids[seq-1], itemlist[i], 1);
998                 }
999
1000                 /* Otherwise, load the message into memory.
1001                  */
1002                 else if (!strcasecmp(itemlist[i], "BODYSTRUCTURE")) {
1003                         if ((msg != NULL) && (!body_loaded)) {
1004                                 CtdlFreeMessage(msg);   /* need the whole thing */
1005                                 msg = NULL;
1006                         }
1007                         if (msg == NULL) {
1008                                 msg = CtdlFetchMessage(IMAP->msgids[seq-1], 1);
1009                                 body_loaded = 1;
1010                         }
1011                         imap_fetch_bodystructure(IMAP->msgids[seq-1],
1012                                         itemlist[i], msg);
1013                 }
1014                 else if (!strcasecmp(itemlist[i], "ENVELOPE")) {
1015                         if (msg == NULL) {
1016                                 msg = CtdlFetchMessage(IMAP->msgids[seq-1], 0);
1017                                 body_loaded = 0;
1018                         }
1019                         imap_fetch_envelope(msg);
1020                 }
1021                 else if (!strcasecmp(itemlist[i], "INTERNALDATE")) {
1022                         if (msg == NULL) {
1023                                 msg = CtdlFetchMessage(IMAP->msgids[seq-1], 0);
1024                                 body_loaded = 0;
1025                         }
1026                         imap_fetch_internaldate(msg);
1027                 }
1028
1029                 if (i != num_items-1) cprintf(" ");
1030         }
1031
1032         cprintf(")\r\n");
1033         unbuffer_output();
1034         if (msg != NULL) {
1035                 CtdlFreeMessage(msg);
1036         }
1037 }
1038
1039
1040
1041 /*
1042  * imap_fetch() calls imap_do_fetch() to do its actual work, once it's
1043  * validated and boiled down the request a bit.
1044  */
1045 void imap_do_fetch(int num_items, char **itemlist) {
1046         int i;
1047
1048         if (IMAP->num_msgs > 0) {
1049                 for (i = 0; i < IMAP->num_msgs; ++i) {
1050
1051                         /* Abort the fetch loop if the session breaks.
1052                          * This is important for users who keep mailboxes
1053                          * that are too big *and* are too impatient to
1054                          * let them finish loading.  :)
1055                          */
1056                         if (CC->kill_me) return;
1057
1058                         /* Get any message marked for fetch. */
1059                         if (IMAP->flags[i] & IMAP_SELECTED) {
1060                                 imap_do_fetch_msg(i+1, num_items, itemlist);
1061                         }
1062                 }
1063         }
1064 }
1065
1066
1067
1068 /*
1069  * Back end for imap_handle_macros()
1070  * Note that this function *only* looks at the beginning of the string.  It
1071  * is not a generic search-and-replace function.
1072  */
1073 void imap_macro_replace(char *str, char *find, char *replace) {
1074         char holdbuf[SIZ];
1075         int findlen;
1076
1077         findlen = strlen(find);
1078
1079         if (!strncasecmp(str, find, findlen)) {
1080                 if (str[findlen]==' ') {
1081                         strcpy(holdbuf, &str[findlen+1]);
1082                         strcpy(str, replace);
1083                         strcat(str, " ");
1084                         strcat(str, holdbuf);
1085                 }
1086                 if (str[findlen]==0) {
1087                         strcpy(holdbuf, &str[findlen+1]);
1088                         strcpy(str, replace);
1089                 }
1090         }
1091 }
1092
1093
1094
1095 /*
1096  * Handle macros embedded in FETCH data items.
1097  * (What the heck are macros doing in a wire protocol?  Are we trying to save
1098  * the computer at the other end the trouble of typing a lot of characters?)
1099  */
1100 void imap_handle_macros(char *str) {
1101         int i;
1102         int nest = 0;
1103
1104         for (i=0; str[i]; ++i) {
1105                 if (str[i]=='(') ++nest;
1106                 if (str[i]=='[') ++nest;
1107                 if (str[i]=='<') ++nest;
1108                 if (str[i]=='{') ++nest;
1109                 if (str[i]==')') --nest;
1110                 if (str[i]==']') --nest;
1111                 if (str[i]=='>') --nest;
1112                 if (str[i]=='}') --nest;
1113
1114                 if (nest <= 0) {
1115                         imap_macro_replace(&str[i],
1116                                 "ALL",
1117                                 "FLAGS INTERNALDATE RFC822.SIZE ENVELOPE"
1118                         );
1119                         imap_macro_replace(&str[i],
1120                                 "BODY",
1121                                 "BODYSTRUCTURE"
1122                         );
1123                         imap_macro_replace(&str[i],
1124                                 "FAST",
1125                                 "FLAGS INTERNALDATE RFC822.SIZE"
1126                         );
1127                         imap_macro_replace(&str[i],
1128                                 "FULL",
1129                                 "FLAGS INTERNALDATE RFC822.SIZE ENVELOPE BODY"
1130                         );
1131                 }
1132         }
1133 }
1134
1135
1136 /*
1137  * Break out the data items requested, possibly a parenthesized list.
1138  * Returns the number of data items, or -1 if the list is invalid.
1139  * NOTE: this function alters the string it is fed, and uses it as a buffer
1140  * to hold the data for the pointers it returns.
1141  */
1142 int imap_extract_data_items(char **argv, char *items) {
1143         int num_items = 0;
1144         int nest = 0;
1145         int i;
1146         char *start;
1147         long initial_len;
1148
1149         /* Convert all whitespace to ordinary space characters. */
1150         for (i=0; items[i]; ++i) {
1151                 if (isspace(items[i])) items[i]=' ';
1152         }
1153
1154         /* Strip leading and trailing whitespace, then strip leading and
1155          * trailing parentheses if it's a list
1156          */
1157         striplt(items);
1158         if ( (items[0]=='(') && (items[strlen(items)-1]==')') ) {
1159                 items[strlen(items)-1] = 0;
1160                 strcpy(items, &items[1]);
1161                 striplt(items);
1162         }
1163
1164         /* Parse any macro data items */
1165         imap_handle_macros(items);
1166
1167         /*
1168          * Now break out the data items.  We throw in one trailing space in
1169          * order to avoid having to break out the last one manually.
1170          */
1171         strcat(items, " ");
1172         start = items;
1173         initial_len = strlen(items);
1174         for (i=0; i<initial_len; ++i) {
1175                 if (items[i]=='(') ++nest;
1176                 if (items[i]=='[') ++nest;
1177                 if (items[i]=='<') ++nest;
1178                 if (items[i]=='{') ++nest;
1179                 if (items[i]==')') --nest;
1180                 if (items[i]==']') --nest;
1181                 if (items[i]=='>') --nest;
1182                 if (items[i]=='}') --nest;
1183
1184                 if (nest <= 0) if (items[i]==' ') {
1185                         items[i] = 0;
1186                         argv[num_items++] = start;
1187                         start = &items[i+1];
1188                 }
1189         }
1190
1191         return(num_items);
1192
1193 }
1194
1195
1196 /*
1197  * One particularly hideous aspect of IMAP is that we have to allow the client
1198  * to specify arbitrary ranges and/or sets of messages to fetch.  Citadel IMAP
1199  * handles this by setting the IMAP_SELECTED flag for each message specified in
1200  * the ranges/sets, then looping through the message array, outputting messages
1201  * with the flag set.  We don't bother returning an error if an out-of-range
1202  * number is specified (we just return quietly) because any client braindead
1203  * enough to request a bogus message number isn't going to notice the
1204  * difference anyway.
1205  *
1206  * This function clears out the IMAP_SELECTED bits, then sets that bit for each
1207  * message included in the specified range.
1208  *
1209  * Set is_uid to 1 to fetch by UID instead of sequence number.
1210  */
1211 void imap_pick_range(char *supplied_range, int is_uid) {
1212         int i;
1213         int num_sets;
1214         int s;
1215         char setstr[SIZ], lostr[SIZ], histr[SIZ];
1216         long lo, hi;
1217         char actual_range[SIZ];
1218         struct citimap *Imap;
1219
1220         /* 
1221          * Handle the "ALL" macro
1222          */
1223         if (!strcasecmp(supplied_range, "ALL")) {
1224                 safestrncpy(actual_range, "1:*", sizeof actual_range);
1225         }
1226         else {
1227                 safestrncpy(actual_range, supplied_range, sizeof actual_range);
1228         }
1229
1230         Imap = IMAP;
1231         /*
1232          * Clear out the IMAP_SELECTED flags for all messages.
1233          */
1234         for (i = 0; i < Imap->num_msgs; ++i) {
1235                 Imap->flags[i] = Imap->flags[i] & ~IMAP_SELECTED;
1236         }
1237
1238         /*
1239          * Now set it for all specified messages.
1240          */
1241         num_sets = num_tokens(actual_range, ',');
1242         for (s=0; s<num_sets; ++s) {
1243                 extract_token(setstr, actual_range, s, ',', sizeof setstr);
1244
1245                 extract_token(lostr, setstr, 0, ':', sizeof lostr);
1246                 if (num_tokens(setstr, ':') >= 2) {
1247                         extract_token(histr, setstr, 1, ':', sizeof histr);
1248                         if (!strcmp(histr, "*")) snprintf(histr, sizeof histr, "%ld", LONG_MAX);
1249                 } 
1250                 else {
1251                         safestrncpy(histr, lostr, sizeof histr);
1252                 }
1253                 lo = atol(lostr);
1254                 hi = atol(histr);
1255
1256                 /* Loop through the array, flipping bits where appropriate */
1257                 for (i = 1; i <= Imap->num_msgs; ++i) {
1258                         if (is_uid) {   /* fetch by sequence number */
1259                                 if ( (Imap->msgids[i-1]>=lo)
1260                                    && (Imap->msgids[i-1]<=hi)) {
1261                                         Imap->flags[i-1] |= IMAP_SELECTED;
1262                                 }
1263                         }
1264                         else {          /* fetch by uid */
1265                                 if ( (i>=lo) && (i<=hi)) {
1266                                         Imap->flags[i-1] |= IMAP_SELECTED;
1267                                 }
1268                         }
1269                 }
1270         }
1271
1272 }
1273
1274
1275
1276 /*
1277  * This function is called by the main command loop.
1278  */
1279 void imap_fetch(int num_parms, char *parms[]) {
1280         char items[SIZ];
1281         char *itemlist[512];
1282         int num_items;
1283         int i;
1284
1285         if (num_parms < 4) {
1286                 cprintf("%s BAD invalid parameters\r\n", parms[0]);
1287                 return;
1288         }
1289
1290         imap_pick_range(parms[2], 0);
1291
1292         strcpy(items, "");
1293         for (i=3; i<num_parms; ++i) {
1294                 strcat(items, parms[i]);
1295                 if (i < (num_parms-1)) strcat(items, " ");
1296         }
1297
1298         num_items = imap_extract_data_items(itemlist, items);
1299         if (num_items < 1) {
1300                 cprintf("%s BAD invalid data item list\r\n", parms[0]);
1301                 return;
1302         }
1303
1304         imap_do_fetch(num_items, itemlist);
1305         cprintf("%s OK FETCH completed\r\n", parms[0]);
1306 }
1307
1308 /*
1309  * This function is called by the main command loop.
1310  */
1311 void imap_uidfetch(int num_parms, char *parms[]) {
1312         char items[SIZ];
1313         char *itemlist[512];
1314         int num_items;
1315         int i;
1316         int have_uid_item = 0;
1317
1318         if (num_parms < 5) {
1319                 cprintf("%s BAD invalid parameters\r\n", parms[0]);
1320                 return;
1321         }
1322
1323         imap_pick_range(parms[3], 1);
1324
1325         strcpy(items, "");
1326         for (i=4; i<num_parms; ++i) {
1327                 strcat(items, parms[i]);
1328                 if (i < (num_parms-1)) strcat(items, " ");
1329         }
1330
1331         num_items = imap_extract_data_items(itemlist, items);
1332         if (num_items < 1) {
1333                 cprintf("%s BAD invalid data item list\r\n", parms[0]);
1334                 return;
1335         }
1336
1337         /* If the "UID" item was not included, we include it implicitly
1338          * (at the beginning) because this is a UID FETCH command
1339          */
1340         for (i=0; i<num_items; ++i) {
1341                 if (!strcasecmp(itemlist[i], "UID")) ++have_uid_item;
1342         }
1343         if (have_uid_item == 0) {
1344                 memmove(&itemlist[1], &itemlist[0], (sizeof(itemlist[0]) * num_items));
1345                 ++num_items;
1346                 itemlist[0] = "UID";
1347         }
1348
1349         imap_do_fetch(num_items, itemlist);
1350         cprintf("%s OK UID FETCH completed\r\n", parms[0]);
1351 }
1352
1353