From 3305d8cd4d01a877a044412a6eda94dbbfd5ad56 Mon Sep 17 00:00:00 2001 From: John Goerzen Date: Mon, 1 Oct 2007 22:20:37 +0100 Subject: [PATCH] Daniel Jacobowitz patches fixes deb#433732 Date: Sun, 30 Sep 2007 13:54:56 -0400 From: Daniel Jacobowitz To: offlineimap@complete.org Subject: Assorted patches Here's the result of a lazy Sunday hacking on offlineimap. Sorry for not breaking this into multiple patches. They're mostly logically independent so just ask if that would make a difference. First, a new -q (quick) option. The quick option means to only update folders that seem to have had significant changes. For Maildir, any change to any message UID or flags is significant, because checking the flags doesn't add a significant cost. For IMAP, only a change to the total number of messages or a change in the UID of the most recent message is significant. This should catch everything except for flags changes. The difference in bandwidth is astonishing: a quick sync takes 80K instead of 5.3MB, and 28 seconds instead of 90. There's a configuration variable that lets you say every tenth sync should update flags, but let all the intervening ones be lighter. Second, a fix to the UID validity problems many people have been reporting with Courier. As discussed in Debian bug #433732, I changed the UID validity check to use SELECT unless the server complains that the folder is read-only. This avoids the Courier bug (see the Debian log for more details). This won't fix existing validity errors, you need to remove the local status and validity files by hand and resync. Third, some speedups in Maildir checking. It's still pretty slow due to a combination of poor performance in os.listdir (never reads more than 4K of directory entries at a time) and some semaphore that leads to lots of futex wake operations, but at least this saves 20% or so of the CPU time running offlineimap on a single folder: Time with quick refresh and md5 in loop: 4.75s user 0.46s system 12% cpu 41.751 total Time with quick refresh and md5 out of loop: 4.38s user 0.50s system 14% cpu 34.799 total Time using string compare to check folder: 4.11s user 0.47s system 13% cpu 34.788 total And fourth, some display fixes for Curses.Blinkenlights. I made warnings more visible, made the new quick sync message cyan, and made all not explicitly colored messages grey. That last one was really bugging me. Any time OfflineIMAP printed a warning in this UI, it had even odds of coming out black on black! Anyway, I hope these are useful. I'm happy to revise them if you see a problem. -- Daniel Jacobowitz CodeSourcery --- offlineimap.conf | 10 +++++++ offlineimap.sgml | 5 ++++ offlineimap/accounts.py | 38 ++++++++++++++++++----- offlineimap/folder/IMAP.py | 53 ++++++++++++++++++++++++++++++++- offlineimap/folder/Maildir.py | 24 +++++++++++---- offlineimap/init.py | 6 +++- offlineimap/ui/Blinkenlights.py | 11 +++++++ offlineimap/ui/Curses.py | 2 ++ offlineimap/ui/UIBase.py | 5 ++++ offlineimap/version.py | 5 +++- 10 files changed, 142 insertions(+), 17 deletions(-) diff --git a/offlineimap.conf b/offlineimap.conf index 4351d0d..ce6d17b 100644 --- a/offlineimap.conf +++ b/offlineimap.conf @@ -159,6 +159,16 @@ remoterepository = RemoteExample # autorefresh = 5 +# You can tell offlineimap to do a number of quicker synchronizations +# between full updates. A quick synchronization only synchronizes +# if a Maildir folder has changed, or if an IMAP folder has received +# new messages or had messages deleted. It does not update if the +# only changes were to IMAP flags. Specify 0 to never do quick updates, +# -1 to always do quick updates, or a positive integer to do that many +# quick updates between each full synchronization (requires autorefresh). + +# quick = 10 + [Repository LocalExample] # This is one of the two repositories that you'll work with given the diff --git a/offlineimap.sgml b/offlineimap.sgml index 6fc8d52..5b5d148 100644 --- a/offlineimap.sgml +++ b/offlineimap.sgml @@ -389,6 +389,11 @@ cd offlineimap-x.y.z file. + -q + Run only quick synchronizations. Ignore any flag + updates on IMAP servers. + + -h --help Show summary of options. diff --git a/offlineimap/accounts.py b/offlineimap/accounts.py index d05928c..9f224d0 100644 --- a/offlineimap/accounts.py +++ b/offlineimap/accounts.py @@ -45,6 +45,7 @@ class Account(CustomConfig.ConfigHelperMixin): self.localeval = config.getlocaleval() self.ui = UIBase.getglobalui() self.refreshperiod = self.getconffloat('autorefresh', 0.0) + self.quicknum = 0 if self.refreshperiod == 0.0: self.refreshperiod = None @@ -125,6 +126,20 @@ class AccountSynchronizationMixin: def sync(self): # We don't need an account lock because syncitall() goes through # each account once, then waits for all to finish. + + quickconfig = self.getconfint('quick', 0) + if quickconfig < 0: + quick = True + elif quickconfig > 0: + if self.quicknum == 0 or self.quicknum > quickconfig: + self.quicknum = 1 + quick = False + else: + self.quicknum = self.quicknum + 1 + quick = True + else: + quick = False + try: remoterepos = self.remoterepos localrepos = self.localrepos @@ -140,7 +155,7 @@ class AccountSynchronizationMixin: name = "Folder sync %s[%s]" % \ (self.name, remotefolder.getvisiblename()), args = (self.name, remoterepos, remotefolder, localrepos, - statusrepos)) + statusrepos, quick)) thread.setDaemon(1) thread.start() folderthreads.append(thread) @@ -157,7 +172,7 @@ class SyncableAccount(Account, AccountSynchronizationMixin): pass def syncfolder(accountname, remoterepos, remotefolder, localrepos, - statusrepos): + statusrepos, quick): global mailboxes ui = UIBase.getglobalui() ui.registerthread(accountname) @@ -167,12 +182,6 @@ def syncfolder(accountname, remoterepos, remotefolder, localrepos, replace(remoterepos.getsep(), localrepos.getsep())) # Write the mailboxes mbnames.add(accountname, localfolder.getvisiblename()) - # Load local folder - ui.syncingfolder(remoterepos, remotefolder, localrepos, localfolder) - ui.loadmessagelist(localrepos, localfolder) - localfolder.cachemessagelist() - ui.messagelistloaded(localrepos, localfolder, len(localfolder.getmessagelist().keys())) - # Load status folder. statusfolder = statusrepos.getfolder(remotefolder.getvisiblename().\ @@ -185,6 +194,19 @@ def syncfolder(accountname, remoterepos, remotefolder, localrepos, statusfolder.cachemessagelist() + if quick: + if not localfolder.quickchanged(statusfolder) \ + and not remotefolder.quickchanged(statusfolder): + ui.skippingfolder(remotefolder) + localrepos.restore_atime() + return + + # Load local folder + ui.syncingfolder(remoterepos, remotefolder, localrepos, localfolder) + ui.loadmessagelist(localrepos, localfolder) + localfolder.cachemessagelist() + ui.messagelistloaded(localrepos, localfolder, len(localfolder.getmessagelist().keys())) + # If either the local or the status folder has messages and there is a UID # validity problem, warn and abort. If there are no messages, UW IMAPd # loses UIDVALIDITY. But we don't really need it if both local folders are diff --git a/offlineimap/folder/IMAP.py b/offlineimap/folder/IMAP.py index 8772c12..12a9660 100644 --- a/offlineimap/folder/IMAP.py +++ b/offlineimap/folder/IMAP.py @@ -41,6 +41,16 @@ class IMAPFolder(BaseFolder): self.randomgenerator = random.Random() BaseFolder.__init__(self) + def selectro(self, imapobj): + """Select this folder when we do not need write access. + Prefer SELECT to EXAMINE if we can, since some servers + (Courier) do not stabilize UID validity until the folder is + selected.""" + try: + imapobj.select(self.getfullname()) + except imapobj.readonly: + imapobj.select(self.getfullname(), readonly = 1) + def getaccountname(self): return self.accountname @@ -60,11 +70,52 @@ class IMAPFolder(BaseFolder): imapobj = self.imapserver.acquireconnection() try: # Primes untagged_responses - imapobj.select(self.getfullname(), readonly = 1) + self.selectro(imapobj) return long(imapobj.untagged_responses['UIDVALIDITY'][0]) finally: self.imapserver.releaseconnection(imapobj) + def quickchanged(self, statusfolder): + # An IMAP folder has definitely changed if the number of + # messages or the UID of the last message have changed. Otherwise + # only flag changes could have occurred. + imapobj = self.imapserver.acquireconnection() + try: + # Primes untagged_responses + imapobj.select(self.getfullname(), readonly = 1, force = 1) + try: + # Some mail servers do not return an EXISTS response if + # the folder is empty. + maxmsgid = long(imapobj.untagged_responses['EXISTS'][0]) + except KeyError: + return True + + # Different number of messages than last time? + if maxmsgid != len(statusfolder.getmessagelist()): + return True + + if maxmsgid < 1: + # No messages; return + return False + + # Now, get the UID for the last message. + response = imapobj.fetch('%d' % maxmsgid, '(UID)')[1] + finally: + self.imapserver.releaseconnection(imapobj) + + # Discard the message number. + messagestr = string.split(response[0], maxsplit = 1)[1] + options = imaputil.flags2hash(messagestr) + if not options.has_key('UID'): + return True + uid = long(options['UID']) + saveduids = statusfolder.getmessagelist().keys() + saveduids.sort() + if uid != saveduids[-1]: + return True + + return False + def cachemessagelist(self): imapobj = self.imapserver.acquireconnection() self.messagelist = {} diff --git a/offlineimap/folder/Maildir.py b/offlineimap/folder/Maildir.py index 643f57a..56a63e4 100644 --- a/offlineimap/folder/Maildir.py +++ b/offlineimap/folder/Maildir.py @@ -22,7 +22,6 @@ from offlineimap.ui import UIBase from threading import Lock import os.path, os, re, time, socket, md5 -foldermatchre = re.compile(',FMD5=([0-9a-f]{32})') uidmatchre = re.compile(',U=(\d+)') flagmatchre = re.compile(':.*2,([A-Z]+)') @@ -78,16 +77,16 @@ class MaildirFolder(BaseFolder): files = [] nouidcounter = -1 # Messages without UIDs get # negative UID numbers. + foldermd5 = md5.new(self.getvisiblename()).hexdigest() + folderstr = ',FMD5=' + foldermd5 for dirannex in ['new', 'cur']: fulldirname = os.path.join(self.getfullname(), dirannex) files.extend([os.path.join(fulldirname, filename) for filename in os.listdir(fulldirname)]) for file in files: messagename = os.path.basename(file) - foldermatch = foldermatchre.search(messagename) - if (not foldermatch) or \ - md5.new(self.getvisiblename()).hexdigest() \ - != foldermatch.group(1): + foldermatch = messagename.find(folderstr) != -1 + if not foldermatch: # If there is no folder MD5 specified, or if it mismatches, # assume it is a foreign (new) message and generate a # negative uid for it @@ -111,8 +110,21 @@ class MaildirFolder(BaseFolder): 'filename': file} return retval + def quickchanged(self, statusfolder): + self.cachemessagelist() + savedmessages = statusfolder.getmessagelist() + if len(self.messagelist) != len(savedmessages): + return True + for uid in self.messagelist.keys(): + if uid not in savedmessages: + return True + if self.messagelist[uid]['flags'] != savedmessages[uid]['flags']: + return True + return False + def cachemessagelist(self): - self.messagelist = self._scanfolder() + if self.messagelist is None: + self.messagelist = self._scanfolder() def getmessagelist(self): return self.messagelist diff --git a/offlineimap/init.py b/offlineimap/init.py index 480778f..9824f05 100644 --- a/offlineimap/init.py +++ b/offlineimap/init.py @@ -53,7 +53,7 @@ def startup(versionno): sys.stdout.write(version.getcmdhelp() + "\n") sys.exit(0) - for optlist in getopt(sys.argv[1:], 'P:1oa:c:d:l:u:h')[0]: + for optlist in getopt(sys.argv[1:], 'P:1oqa:c:d:l:u:h')[0]: options[optlist[0]] = optlist[1] if options.has_key('-h'): @@ -100,6 +100,10 @@ def startup(versionno): for section in accounts.getaccountlist(config): config.remove_option('Account ' + section, "autorefresh") + if options.has_key('-q'): + for section in accounts.getaccountlist(config): + config.set('Account ' + section, "quick", '-1') + lock(config, ui) try: diff --git a/offlineimap/ui/Blinkenlights.py b/offlineimap/ui/Blinkenlights.py index 717e81b..6982351 100644 --- a/offlineimap/ui/Blinkenlights.py +++ b/offlineimap/ui/Blinkenlights.py @@ -42,6 +42,10 @@ class BlinkenBase: s.gettf().setcolor('cyan') s.__class__.__bases__[-1].syncingfolder(s, srcrepos, srcfolder, destrepos, destfolder) + def skippingfolder(s, folder): + s.gettf().setcolor('cyan') + s.__class__.__bases__[-1].skippingfolder(s, folder) + def loadmessagelist(s, repos, folder): s.gettf().setcolor('green') s._msg("Scanning folder [%s/%s]" % (s.getnicename(repos), @@ -71,6 +75,13 @@ class BlinkenBase: s.gettf().setcolor('pink') s.__class__.__bases__[-1].deletingflags(s, uidlist, flags, destlist) + def warn(s, msg, minor = 0): + if minor: + s.gettf().setcolor('pink') + else: + s.gettf().setcolor('red') + s.__class__.__bases__[-1].warn(s, msg, minor) + def init_banner(s): s.availablethreadframes = {} s.threadframes = {} diff --git a/offlineimap/ui/Curses.py b/offlineimap/ui/Curses.py index 4768ea0..4cf9066 100644 --- a/offlineimap/ui/Curses.py +++ b/offlineimap/ui/Curses.py @@ -511,6 +511,8 @@ class Blinkenlights(BlinkenBase, UIBase): return if color: s.gettf().setcolor(color) + elif s.gettf().getcolor() == 'black': + s.gettf().setcolor('gray') s._addline(msg, s.gettf().getcolorpair()) s.logwindow.refresh() finally: diff --git a/offlineimap/ui/UIBase.py b/offlineimap/ui/UIBase.py index 2f7b634..d0e92e6 100644 --- a/offlineimap/ui/UIBase.py +++ b/offlineimap/ui/UIBase.py @@ -208,6 +208,11 @@ class UIBase: s.getnicename(srcrepos), s.getnicename(destrepos))) + def skippingfolder(s, folder): + """Called when a folder sync operation is started.""" + if s.verbose >= 0: + s._msg("Skipping %s (not changed)" % folder.getname()) + def validityproblem(s, folder): s.warn("UID validity problem for folder %s (repo %s) (saved %d; got %d); skipping it" % \ (folder.getname(), folder.getrepository().getname(), diff --git a/offlineimap/version.py b/offlineimap/version.py index 1a1a1eb..33620f7 100644 --- a/offlineimap/version.py +++ b/offlineimap/version.py @@ -43,7 +43,7 @@ def getcmdhelp(): return """ offlineimap [ -1 ] [ -P profiledir ] [ -a accountlist ] [ -c configfile ] [ -d debugtype[,debugtype...] ] [ -o ] [ - -u interface ] + -u interface ] [ -q ] offlineimap -h | --help @@ -97,6 +97,9 @@ def getcmdhelp(): -o Run only once, ignoring any autorefresh setting in the config file. + -q Run only quick synchronizations. Ignore any flag + updates on IMAP servers. + -h, --help Show summary of options.