fix: when called with -a, mbnames must not erase entries of other accounts

Make mbnames to work with intermediate files, one per account, in the JSON
format. The mbnames target is built from those intermediate files.

Github-Fix: https://github.com/OfflineIMAP/offlineimap/issues/66
Signed-off-by: Nicolas Sebrecht <nicolas.s-dev@laposte.net>
This commit is contained in:
Nicolas Sebrecht 2016-06-23 00:20:57 +02:00
parent 52120beb27
commit 092264c8e7
5 changed files with 197 additions and 82 deletions

View File

@ -369,9 +369,7 @@ class SyncableAccount(Account):
# wait for all threads to finish # wait for all threads to finish
for thr in folderthreads: for thr in folderthreads:
thr.join() thr.join()
# Write out mailbox names if required and not in dry-run mode mbnames.writeIntermediateFile(self.name) # Write out mailbox names.
if not self.dryrun:
mbnames.write(False)
localrepos.forgetfolders() localrepos.forgetfolders()
remoterepos.forgetfolders() remoterepos.forgetfolders()
except: except:
@ -512,9 +510,9 @@ def syncfolder(account, remotefolder, quick):
# Load local folder. # Load local folder.
localfolder = account.get_local_folder(remotefolder) localfolder = account.get_local_folder(remotefolder)
# Write the mailboxes # Add the folder to the mbnames mailboxes.
mbnames.add(account.name, localfolder.getname(), mbnames.add(account.name, localrepos.getlocalroot(),
localrepos.getlocalroot()) localfolder.getname())
# Load status folder. # Load status folder.
statusfolder = statusrepos.getfolder(remotefolder.getvisiblename(). statusfolder = statusrepos.getfolder(remotefolder.getvisiblename().

View File

@ -8,6 +8,6 @@ from offlineimap.utils import const
options = const.ConstProxy() options = const.ConstProxy()
def set_options(source): def set_options(source):
""" Sets the source for options variable """ """Sets the source for options variable."""
options.set_source(source) options.set_source(source)

View File

@ -25,8 +25,7 @@ import logging
from optparse import OptionParser from optparse import OptionParser
import offlineimap import offlineimap
from offlineimap import accounts, threadutil, folder from offlineimap import globals, accounts, threadutil, folder, mbnames
from offlineimap import globals
from offlineimap.ui import UI_LIST, setglobalui, getglobalui from offlineimap.ui import UI_LIST, setglobalui, getglobalui
from offlineimap.CustomConfig import CustomConfigParser from offlineimap.CustomConfig import CustomConfigParser
from offlineimap.utils import stacktrace from offlineimap.utils import stacktrace
@ -423,7 +422,7 @@ class OfflineImap(object):
signal.signal(signal.SIGQUIT, sig_handler) signal.signal(signal.SIGQUIT, sig_handler)
# Various initializations that need to be performed: # Various initializations that need to be performed:
offlineimap.mbnames.init(self.config, syncaccounts) mbnames.init(self.config, self.ui, options.dryrun)
if options.singlethreading: if options.singlethreading:
# Singlethreaded. # Singlethreaded.
@ -440,9 +439,8 @@ class OfflineImap(object):
t.start() t.start()
threadutil.monitor() threadutil.monitor()
if not options.dryrun: # All sync are done.
offlineimap.mbnames.write(True) mbnames.write()
self.ui.terminate() self.ui.terminate()
return 0 return 0
except (SystemExit): except (SystemExit):

View File

@ -1,6 +1,5 @@
# Mailbox name generator # Mailbox name generator
# # Copyright (C) 2002-2016 John Goerzen & contributors
# Copyright (C) 2002-2015 John Goerzen & contributors
# #
# This program is free software; you can redistribute it and/or modify # This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by # it under the terms of the GNU General Public License as published by
@ -16,80 +15,200 @@
# along with this program; if not, write to the Free Software # along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
import os.path
import re # for folderfilter import re # For folderfilter.
import json
from threading import Lock from threading import Lock
from os import listdir, makedirs, path
boxes = {} from sys import exc_info
localroots = {}
config = None
accounts = None
mblock = Lock()
def init(conf, accts):
global config, accounts
config = conf
accounts = accts
def add(accountname, foldername, localfolders):
if not accountname in boxes:
boxes[accountname] = []
localroots[accountname] = localfolders
if not foldername in boxes[accountname]:
boxes[accountname].append(foldername)
def write(allcomplete):
incremental = config.getdefaultboolean("mbnames", "incremental", False)
# Skip writing if we don't want incremental writing and we're not done.
if not incremental and not allcomplete:
return
# Skip writing if we want incremental writing and we're done.
if incremental and allcomplete:
return
# See if we're ready to write it out.
for account in accounts:
if account not in boxes:
return
__genmbnames()
def __genmbnames():
"""Takes a configparser object and a boxlist, which is a list of hashes
containing 'accountname' and 'foldername' keys."""
xforms = [os.path.expanduser, os.path.expandvars]
mblock.acquire()
try: try:
localeval = config.getlocaleval() import UserDict
if not config.getdefaultboolean("mbnames", "enabled", 0): except ImportError:
# Py3
from collections import UserDict
_mbLock = Lock()
_mbnames = None
def add(accountname, folder_root, foldername):
global _mbnames
if _mbnames is None:
return return
path = config.apply_xforms(config.get("mbnames", "filename"), xforms)
file = open(path, "wt") with _mbLock:
file.write(localeval.eval(config.get("mbnames", "header"))) _mbnames.addAccountFolder(accountname, folder_root, foldername)
folderfilter = lambda accountname, foldername: 1
if config.has_option("mbnames", "folderfilter"): def init(conf, ui, dry_run):
folderfilter = localeval.eval(config.get("mbnames", "folderfilter"), global _mbnames
{'re': re}) enabled = conf.getdefaultboolean("mbnames", "enabled", False)
mb_sort_keyfunc = lambda d: (d['accountname'], d['foldername']) if enabled is True and _mbnames is None:
if config.has_option("mbnames", "sort_keyfunc"): _mbnames = _Mbnames(conf, ui, dry_run)
mb_sort_keyfunc = localeval.eval(config.get("mbnames", "sort_keyfunc"),
{'re': re}) def write():
"""Write the mbnames file."""
global _mbnames
if _mbnames is None:
return
if _mbnames.get_incremental() is not True:
_mbnames.write()
def writeIntermediateFile(accountname):
"""Write intermediate mbnames file."""
global _mbnames
if _mbnames is None:
return
_mbnames.writeIntermediateFile(accountname)
if _mbnames.get_incremental() is True:
_mbnames.write()
class _IntermediateMbnames(object):
"""mbnames data for one account."""
def __init__(self, accountname, folder_root, mbnamesdir, folderfilter,
dry_run):
self._foldernames = []
self._accountname = accountname
self._folder_root = folder_root
self._folderfilter = folderfilter
self._path = path.join(mbnamesdir, "%s.yml"% accountname)
self._dryrun = dry_run
def add(self, foldername):
self._foldernames.append(foldername)
def get_folder_root(self):
return self._folder_root
def write(self):
"""Write intermediate mbnames file in JSON format."""
itemlist = [] itemlist = []
for accountname in boxes.keys():
localroot = localroots[accountname] for foldername in self._foldernames:
for foldername in boxes[accountname]: if self._folderfilter(self._accountname, foldername):
if folderfilter(accountname, foldername): itemlist.append({
itemlist.append({'accountname': accountname, 'accountname': self._accountname,
'foldername': foldername, 'foldername': foldername,
'localfolders': localroot}) 'localfolders': self._folder_root,
itemlist.sort(key = mb_sort_keyfunc) })
format_string = config.get("mbnames", "peritem", raw=1)
itemlist = [format_string % d for d in itemlist] if not self._dryrun:
file.write(localeval.eval(config.get("mbnames", "sep")).join(itemlist)) with open(self._path, "wt") as intermediateFile:
file.write(localeval.eval(config.get("mbnames", "footer"))) json.dump(itemlist, intermediateFile)
file.close()
finally:
mblock.release() class _Mbnames(object):
def __init__(self, config, ui, dry_run):
self._config = config
self._dryrun = dry_run
self.ui = ui
# Keys: accountname, values: _IntermediateMbnames instance
self._intermediates = {}
self._incremental = None
self._mbnamesdir = None
self._path = None
self._folderfilter = lambda accountname, foldername: True
self._func_sortkey = lambda d: (d['accountname'], d['foldername'])
self._peritem = self._config.get("mbnames", "peritem", raw=1)
localeval = config.getlocaleval()
self._header = localeval.eval(config.get("mbnames", "header"))
self._sep = localeval.eval(config.get("mbnames", "sep"))
self._footer = localeval.eval(config.get("mbnames", "footer"))
mbnamesdir = path.join(config.getmetadatadir(), "mbnames")
try:
if not self._dryrun:
makedirs(mbnamesdir)
except OSError:
pass
self._mbnamesdir = mbnamesdir
xforms = [path.expanduser, path.expandvars]
self._path = config.apply_xforms(
config.get("mbnames", "filename"), xforms)
if self._config.has_option("mbnames", "sort_keyfunc"):
self._func_sortkey = localeval.eval(
self._config.get("mbnames", "sort_keyfunc"), {'re': re})
if self._config.has_option("mbnames", "folderfilter"):
self._folderfilter = localeval.eval(
self._config.get("mbnames", "folderfilter"), {'re': re})
def addAccountFolder(self, accountname, folder_root, foldername):
"""Add foldername entry for an account."""
if accountname not in self._intermediates:
self._intermediates[accountname] = _IntermediateMbnames(
accountname,
folder_root,
self._mbnamesdir,
self._folderfilter,
self._dryrun,
)
self._intermediates[accountname].add(foldername)
def get_incremental(self):
if self._incremental is None:
self._incremental = self._config.getdefaultboolean(
"mbnames", "incremental", False)
return self._incremental
def write(self):
itemlist = []
try:
for foo in listdir(self._mbnamesdir):
foo = path.join(self._mbnamesdir, foo)
if path.isfile(foo) and foo[-5:] == '.json':
try:
with open(foo, 'rt') as intermediateFile:
for item in json.load(intermediateFile):
itemlist.append(item)
except Exception as e:
self.ui.error(
e,
exc_info()[2],
"intermediate mbnames file %s not properly read"% foo
)
except OSError:
pass
itemlist.sort(key=self._func_sortkey)
itemlist = [self._peritem % d for d in itemlist]
if not self._dryrun:
try:
with open(self._path, 'wt') as mbnamesFile:
mbnamesFile.write(self._header)
mbnamesFile.write(self._sep.join(itemlist))
mbnamesFile.write(self._footer)
except (OSError, IOError) as e:
self.ui.error(
e,
exc_info()[2],
"mbnames file %s not properly written"% self._path
)
def writeIntermediateFile(self, accountname):
try:
self._intermediates[accountname].write()
except (OSError, IOError) as e:
self.ui.error(
e,
exc_info()[2],
"intermediate mbnames file %s not properly written"% self._path
)

View File

@ -1,5 +1,5 @@
# UI base class # UI base class
# Copyright (C) 2002-2015 John Goerzen & contributors # Copyright (C) 2002-2016 John Goerzen & contributors
# #
# This program is free software; you can redistribute it and/or modify # This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by # it under the terms of the GNU General Public License as published by