From c84d23b65670f3480691259a4c686e3903d2ca92 Mon Sep 17 00:00:00 2001 From: Ilias Tsitsimpis Date: Sat, 5 Mar 2016 19:07:07 +0200 Subject: [PATCH] Identify and fix messages with FMD5 inconsistencies MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Introduce the '--migrate-fmd5-using-nametrans' option which migrates the FMD5 hashes from versions prior to 6.3.5. It seems that commit 'Apply nametrans to all Foldertypes' (6b2ec956cf) introduced a regression because it changed the FMD5 part of the filename calculated by OfflineIMAP. Thus, OfflineIMAP believes that the messages has been removed and adds them back. For more information, see: http://www.offlineimap.org/configuration/2016/02/12/debian-upgrade-from-jessie-to-stretch.html Bug-Debian: https://bugs.debian.org/812108 Reported-by: François Signed-off-by: Ilias Tsitsimpis Signed-off-by: Nicolas Sebrecht --- docs/offlineimap.txt | 14 ++++++++++++++ offlineimap/folder/Maildir.py | 34 ++++++++++++++++++++++++++++++++++ offlineimap/init.py | 27 ++++++++++++++++++++++++++- 3 files changed, 74 insertions(+), 1 deletion(-) diff --git a/docs/offlineimap.txt b/docs/offlineimap.txt index 33451cc..0c66489 100644 --- a/docs/offlineimap.txt +++ b/docs/offlineimap.txt @@ -163,6 +163,20 @@ blinkenlights, machineui. This option is only applicable in non-verbose mode. +--migrate-fmd5-using-nametrans:: + Migrate FMD5 hashes from versions prior to 6.3.5. ++ +The way that FMD5 hashes are calculated was changed in version 6.3.5 (now using +the nametrans folder name) introducing a regression which may lead to +re-uploading all messages. Try and fix the above regression by calculating the +correct FMD5 values and renaming the corresponding messages. + +CAUTION: Since the FMD5 part of the filename changes, this may lead to UID +conflicts. Ensure to dispose a proper backup of both the cache and the Maildir +before running this fix as well as verify the results using the `--dry-run' +flag first. + + Synchronization Performance --------------------------- diff --git a/offlineimap/folder/Maildir.py b/offlineimap/folder/Maildir.py index 7d188b5..77a774b 100644 --- a/offlineimap/folder/Maildir.py +++ b/offlineimap/folder/Maildir.py @@ -485,3 +485,37 @@ class MaildirFolder(BaseFolder): os.unlink(filepath) # Yep -- return. del(self.messagelist[uid]) + + def migratefmd5(self, dryrun=False): + """Migrate FMD5 hashes from versions prior to 6.3.5 + + :param dryrun: Run in dry run mode + :type fix: Boolean + :return: None + """ + oldfmd5 = md5(self.name).hexdigest() + msglist = self._scanfolder() + for mkey, mvalue in msglist.iteritems(): + filename = os.path.join(self.getfullname(), mvalue['filename']) + match = re.search("FMD5=([a-fA-F0-9]+)", filename) + if match is None: + self.ui.debug("maildir", + "File `%s' doesn't have an FMD5 assigned" + % filename) + elif match.group(1) == oldfmd5: + self.ui.info("Migrating file `%s' to FMD5 `%s'" + % (filename, self._foldermd5)) + if not dryrun: + newfilename = filename.replace( + "FMD5=" + match.group(1), "FMD5=" + self._foldermd5) + try: + os.rename(filename, newfilename) + except OSError as e: + raise OfflineImapError( + "Can't rename file '%s' to '%s': %s" % ( + filename, newfilename, e[1]), + OfflineImapError.ERROR.FOLDER), None, exc_info()[2] + elif match.group(1) != self._foldermd5: + self.ui.warn(("Inconsistent FMD5 for file `%s':" + " Neither `%s' nor `%s' found") + % (filename, oldfmd5, self._foldermd5)) diff --git a/offlineimap/init.py b/offlineimap/init.py index 7ed3b6b..636db5d 100644 --- a/offlineimap/init.py +++ b/offlineimap/init.py @@ -25,11 +25,12 @@ import logging from optparse import OptionParser import offlineimap -from offlineimap import accounts, threadutil, syncmaster +from offlineimap import accounts, threadutil, syncmaster, folder from offlineimap import globals from offlineimap.ui import UI_LIST, setglobalui, getglobalui from offlineimap.CustomConfig import CustomConfigParser from offlineimap.utils import stacktrace +from offlineimap.repository import Repository import traceback import collections @@ -50,6 +51,8 @@ class OfflineImap: options, args = self.__parse_cmd_options() if options.diagnostics: self.__serverdiagnostics(options) + elif options.migrate_fmd5: + self.__migratefmd5(options) else: return self.__sync(options) @@ -120,6 +123,10 @@ class OfflineImap: help="specifies an alternative user interface" " (quiet, basic, syslog, ttyui, blinkenlights, machineui)") + parser.add_option("--migrate-fmd5-using-nametrans", + action="store_true", dest="migrate_fmd5", default=False, + help="migrate FMD5 hashes from versions prior to 6.3.5") + (options, args) = parser.parse_args() globals.set_options (options) @@ -427,3 +434,21 @@ class OfflineImap: for account in allaccounts: if account.name not in activeaccounts: continue account.serverdiagnostics() + + def __migratefmd5(self, options): + activeaccounts = self.config.get("general", "accounts") + if options.accounts: + activeaccounts = options.accounts + activeaccounts = activeaccounts.replace(" ", "") + activeaccounts = activeaccounts.split(",") + allaccounts = accounts.AccountListGenerator(self.config) + + for account in allaccounts: + if account.name not in activeaccounts: + continue + localrepo = Repository(account, 'local') + if localrepo.getfoldertype() != folder.Maildir.MaildirFolder: + continue + folders = localrepo.getfolders() + for f in folders: + f.migratefmd5(options.dryrun)