[fetchmail-devel] [PATCH] imap: introduce imapsearchmax parameter

[Andrea R. <rig...@gm...>]

To retrieve new messages from a IMAP folder we use the IMAP SEARCH
command, but we limit the messages processed by the query to 1000.
This limit is hard-coded and it cannot be changed at runtime.

This value is not the best option in some cases, especially for large
folders, because we need to submit a query each 1000 messages.

The following patch introduces a new option (imapsearchmax) that can be
used to change this limit at runtime.

So, for example this is what happens with a 3000 messages folder:
...
fetchmail: IMAP< A0003 OK [READ-WRITE] INBOX selected. (Success)
fetchmail: IMAP> A0004 SEARCH 1:1000 UNSEEN UNDELETED
fetchmail: IMAP< * SEARCH
fetchmail: IMAP< A0004 OK SEARCH completed (Success)
fetchmail: IMAP> A0005 SEARCH 1001:2000 UNSEEN UNDELETED
fetchmail: IMAP< * SEARCH
fetchmail: IMAP< A0005 OK SEARCH completed (Success)
fetchmail: IMAP> A0006 SEARCH 2001:3000 UNSEEN UNDELETED
...

With imapsearchmax we could increase the SEARCH limit to a higher value
(i.e., 5000) and submit a single query to the server:
...
fetchmail: IMAP< A0003 OK [READ-WRITE] INBOX selected. (Success)
fetchmail: IMAP> A0004 SEARCH 1:5000 UNSEEN UNDELETED
...

I did some tests with my >500000 messages gmail INBOX. The results are
reported below:

 - without the patch:

 $ time fetchmail -s

 real	7m1.338s
 user	0m0.070s
 sys	0m0.100s

 - with the patch (using imapsearchmax=1000000)

 $ time fetchmail -s --imapsearchmax=1000000

 real	0m3.099s
 user	0m0.000s
 sys	0m0.040s

Signed-off-by: Andrea Righi <rig...@gm...>
---
 conf.c           |    1 +
 fetchmail.c      |    7 +++++++
 fetchmail.h      |    1 +
 fetchmail.man    |    7 +++++++
 fetchmailconf.py |    6 ++++++
 imap.c           |   13 +++++++------
 options.c        |    7 +++++++
 rcfile_l.l       |    1 +
 rcfile_y.y       |    3 ++-
 9 files changed, 39 insertions(+), 7 deletions(-)

diff --git a/conf.c b/conf.c
index 5758138..0f968d0 100644
--- a/conf.c
+++ b/conf.c
@@ -375,6 +375,7 @@ void dump_config(struct runctl *runp, struct query *querylist)
 	numdump("warnings", ctl->warnings);
 	numdump("fetchlimit", ctl->fetchlimit);
 	numdump("fetchsizelimit", ctl->fetchsizelimit);
+	numdump("imapsearchmax", ctl->imapsearchmax);
 	numdump("fastuidl", ctl->fastuidl);
 	numdump("batchlimit", ctl->batchlimit);
 #ifdef SSL_ENABLE
diff --git a/fetchmail.c b/fetchmail.c
index fc39936..0a05bd8 100644
--- a/fetchmail.c
+++ b/fetchmail.c
@@ -977,6 +977,7 @@ static void optmerge(struct query *h2, struct query *h1, int force)
     FLAG_MERGE(warnings);
     FLAG_MERGE(fetchlimit);
     FLAG_MERGE(fetchsizelimit);
+    FLAG_MERGE(imapsearchmax);
     FLAG_MERGE(fastuidl);
     FLAG_MERGE(batchlimit);
 #ifdef	SSL_ENABLE
@@ -1023,6 +1024,7 @@ static int load_params(int argc, char **argv, int optind)
     def_opts.remotename = user;
     def_opts.listener = SMTP_MODE;
     def_opts.fetchsizelimit = 100;
+    def_opts.imapsearchmax = 1000;
     def_opts.fastuidl = 4;
 
     /* get the location of rcfile */
@@ -1761,6 +1763,11 @@ static void dump_params (struct runctl *runp,
 		       ctl->fetchsizelimit, ctl->fetchsizelimit);
 	    else if (outlevel >= O_VERBOSE)
 		printf(GT_("  No fetch message size limit (--fetchsizelimit 0).\n"));
+	    if (NUM_NONZERO(ctl->imapsearchmax))
+		printf(GT_("   Max number of messages we can process in IMAP SEARCH response is %d (--imapsearchmax %d).\n"),
+		       ctl->imapsearchmax, ctl->imapsearchmax);
+	    else if (outlevel >= O_VERBOSE)
+		printf(GT_("  No max number of messages for IMAP SEARCH (--imapsearchmax 0).\n"));
 	    if (NUM_NONZERO(ctl->fastuidl) && MAILBOX_PROTOCOL(ctl))
 	    {
 		if (ctl->fastuidl == 1)
diff --git a/fetchmail.h b/fetchmail.h
index f6c6a4e..dce4d7d 100644
--- a/fetchmail.h
+++ b/fetchmail.h
@@ -360,6 +360,7 @@ struct query
     int warnings;		/* size warning interval */
     int	fetchlimit;		/* max # msgs to get in single poll */
     int fetchsizelimit;		/* max # msg sizes to get in a request */
+    int imapsearchmax;		/* max # msg we can process in "SEARCH" response */
     int fastuidl;		/* do binary search for new UIDLs? */
     int fastuidlcount;		/* internal count for frequency of binary search */
     int	batchlimit;		/* max # msgs to pass in single SMTP session */
diff --git a/fetchmail.man b/fetchmail.man
index 69aa887..4cca19c 100644
--- a/fetchmail.man
+++ b/fetchmail.man
@@ -811,6 +811,13 @@ messages are downloaded at the start.
 This option does not work with ETRN or ODMR.  For POP3, the only valid
 non-zero value is 1.
 .TP
+.B \-\-imapsearchmax <number>
+(Keyword: imapsearchmax)
+.br
+Limit the maximum number of messages processed by the IMAP SEARCH query to an
+IMAP server.  By default, the limit is 1000. If set to 0, the IMAP SEARCH is
+applied to all the messages present in the target IMAP folder.
+.TP
 .B \-\-fastuidl <number>
 (Keyword: fastuidl)
 .br
diff --git a/fetchmailconf.py b/fetchmailconf.py
index 2dc02d8..8eec6db 100755
--- a/fetchmailconf.py
+++ b/fetchmailconf.py
@@ -261,6 +261,7 @@ class User:
 	self.warnings = 3600	# Size warning interval (see tunable.h)
 	self.fetchlimit = 0	# Max messages fetched per batch
 	self.fetchsizelimit = 100	# Max message sizes fetched per transaction
+	self.imapsearchmax = 1000	# Max number of messages processed by "IMAP SEARCH"
 	self.fastuidl = 4	# Do fast uidl 3 out of 4 times
 	self.batchlimit = 0	# Max message forwarded per batch
 	self.expunge = 0	# Interval between expunges (IMAP)
@@ -302,6 +303,7 @@ class User:
 	    ('warnings',    'Int'),
 	    ('fetchlimit',  'Int'),
 	    ('fetchsizelimit',	'Int'),
+	    ('imapsearchmax',	'Int'),
 	    ('fastuidl',    'Int'),
 	    ('batchlimit',  'Int'),
 	    ('expunge',     'Int'),
@@ -369,6 +371,8 @@ class User:
 	    res = res + " fetchlimit " + `self.fetchlimit`
 	if self.fetchsizelimit != UserDefaults.fetchsizelimit:
 	    res = res + " fetchsizelimit " + `self.fetchsizelimit`
+	if self.imapsearchmax != UserDefaults.imapsearchmax:
+	    res = res + " imapsearchmax " + `self.imapsearchmax`
 	if self.fastuidl != UserDefaults.fastuidl:
 	    res = res + " fastuidl " + `self.fastuidl`
 	if self.batchlimit != UserDefaults.batchlimit:
@@ -1761,6 +1765,8 @@ class UserEdit(Frame, MyWidget):
 		      self.fetchlimit, '30').pack(side=TOP, fill=X)
 	    LabeledEntry(limwin, 'Max message sizes to fetch per transaction:',
 		      self.fetchsizelimit, '30').pack(side=TOP, fill=X)
+	    LabeledEntry(limwin, 'Max number of messages processed by the IMAP SEARCH query:',
+		      self.imapsearchmax, '30').pack(side=TOP, fill=X)
 	    if self.parent.server.protocol not in ('ETRN', 'ODMR'):
 		LabeledEntry(limwin, 'Use fast UIDL:',
 			self.fastuidl, '30').pack(side=TOP, fill=X)
diff --git a/imap.c b/imap.c
index 1364027..9bd9221 100644
--- a/imap.c
+++ b/imap.c
@@ -754,14 +754,15 @@ static int imap_idle(int sock)
     return(ok);
 }
 
-/* maximum number of numbers we can process in "SEARCH" response */
-# define IMAP_SEARCH_MAX 1000
-
 static int imap_search(int sock, struct query *ctl, int count)
 /* search for unseen messages */
 {
     int ok, first, last;
     char buf[MSGBUFSIZE+1], *cp;
+    int stride = ctl->imapsearchmax;
+
+    if (stride < 0)
+        stride = INT_MAX;
 
     /* Don't count deleted messages. Enabled only for IMAP4 servers or
      * higher and only when keeping mails. This flag will have an
@@ -771,15 +772,15 @@ static int imap_search(int sock, struct query *ctl, int count)
     const char *undeleted;
 
     /* Skip range search if there are less than or equal to
-     * IMAP_SEARCH_MAX mails. */
-    flag skiprangesearch = (count <= IMAP_SEARCH_MAX);
+     * imap_search_max mails. */
+    flag skiprangesearch = (count <= stride);
 
     /* startcount is higher than count so that if there are no
      * unseen messages, imap_getsizes() will not need to do
      * anything! */
     startcount = count + 1;
 
-    for (first = 1, last = IMAP_SEARCH_MAX; first <= count; first += IMAP_SEARCH_MAX, last += IMAP_SEARCH_MAX)
+    for (first = 1, last = stride; first <= count; first += stride, last += stride)
     {
 	if (last > count)
 	    last = count;
diff --git a/options.c b/options.c
index aee616b..6c7c612 100644
--- a/options.c
+++ b/options.c
@@ -50,6 +50,7 @@ enum {
     LA_SSLCOMMONNAME,
     LA_SSLFINGERPRINT,
     LA_FETCHSIZELIMIT,
+    LA_IMAPSEARCHMAX,
     LA_FASTUIDL,
     LA_LIMITFLUSH,
     LA_IDLE,
@@ -120,6 +121,7 @@ static const struct option longoptions[] = {
   {"batchlimit",required_argument, (int *) 0, 'b' },
   {"fetchlimit",required_argument, (int *) 0, 'B' },
   {"fetchsizelimit",required_argument, (int *) 0, LA_FETCHSIZELIMIT },
+  {"imapsearchmax",required_argument, (int *) 0, LA_IMAPSEARCHMAX},
   {"fastuidl",	required_argument, (int *) 0, LA_FASTUIDL },
   {"expunge",	required_argument, (int *) 0, 'e' },
   {"mda",	required_argument, (int *) 0, 'm' },
@@ -506,6 +508,10 @@ int parsecmdline (int argc /** argument count */,
 	    c = xatoi(optarg, &errflag);
 	    ctl->fetchsizelimit = NUM_VALUE_IN(c);
 	    break;
+	case LA_IMAPSEARCHMAX:
+	    c = xatoi(optarg, &errflag);
+	    ctl->imapsearchmax = NUM_VALUE_IN(c);
+	    break;
 	case LA_FASTUIDL:
 	    c = xatoi(optarg, &errflag);
 	    ctl->fastuidl = NUM_VALUE_IN(c);
@@ -687,6 +693,7 @@ int parsecmdline (int argc /** argument count */,
 	P(GT_("  -b, --batchlimit  set batch limit for SMTP connections\n"));
 	P(GT_("  -B, --fetchlimit  set fetch limit for server connections\n"));
 	P(GT_("      --fetchsizelimit set fetch message size limit\n"));
+	P(GT_("      --imapsearchmax set max number of imap search messages\n"));
 	P(GT_("      --fastuidl    do a binary search for UIDLs\n"));
 	P(GT_("  -e, --expunge     set max deletions between expunges\n"));
 	P(GT_("  -m, --mda         set MDA to use for forwarding\n"));
diff --git a/rcfile_l.l b/rcfile_l.l
index c7e49fe..6c80749 100644
--- a/rcfile_l.l
+++ b/rcfile_l.l
@@ -119,6 +119,7 @@ plugout		{ return PLUGOUT; }
 batchlimit	{ return BATCHLIMIT; }
 fetchlimit	{ return FETCHLIMIT; }
 fetchsizelimit	{ return FETCHSIZELIMIT; }
+imapsearchmax   { return IMAPSEARCHMAX; }
 fastuidl	{ return FASTUIDL; }
 expunge		{ return EXPUNGE; }
 properties	{ return PROPERTIES; }
diff --git a/rcfile_y.y b/rcfile_y.y
index b32a6b0..5ae05f8 100644
--- a/rcfile_y.y
+++ b/rcfile_y.y
@@ -67,7 +67,7 @@ extern char * yytext;
 %token SMTPADDRESS SMTPNAME SPAMRESPONSE PRECONNECT POSTCONNECT LIMIT WARNINGS
 %token INTERFACE MONITOR PLUGIN PLUGOUT
 %token IS HERE THERE TO MAP
-%token BATCHLIMIT FETCHLIMIT FETCHSIZELIMIT FASTUIDL EXPUNGE PROPERTIES
+%token BATCHLIMIT FETCHLIMIT FETCHSIZELIMIT IMAPSEARCHMAX FASTUIDL EXPUNGE PROPERTIES
 %token SET LOGFILE DAEMON SYSLOG IDFILE PIDFILE INVISIBLE POSTMASTER BOUNCEMAIL
 %token SPAMBOUNCE SOFTBOUNCE SHOWDOTS
 %token BADHEADER ACCEPT REJECT_
@@ -368,6 +368,7 @@ user_option	: TO mapping_list HERE
 		| WARNINGS NUMBER	{current.warnings    = NUM_VALUE_IN($2);}
 		| FETCHLIMIT NUMBER	{current.fetchlimit  = NUM_VALUE_IN($2);}
 		| FETCHSIZELIMIT NUMBER	{current.fetchsizelimit = NUM_VALUE_IN($2);}
+		| IMAPSEARCHMAX NUMBER	{current.imapsearchmax = NUM_VALUE_IN($2);}
 		| FASTUIDL NUMBER	{current.fastuidl    = NUM_VALUE_IN($2);}
 		| BATCHLIMIT NUMBER	{current.batchlimit  = NUM_VALUE_IN($2);}
 		| EXPUNGE NUMBER	{current.expunge     = NUM_VALUE_IN($2);}
-- 
1.7.1