add new config option filename_use_mail_timestamp

If this value is true, use (if possible) a timestamp based on message
Date or Delivery-date headers. The current system time is used
otherwise.

filename_use_mail_timestamp and utime_from_header are now completely
separated option that do not interfere one with other.

To handle this feature in a multithread context we use a hash to count
the number of mail with the same timestamp. This method is more accurate
than using the old lasttime and timeseq variables.

Signed-off-by: Sébastien Gross <seb•ɑƬ•chezwam•ɖɵʈ•org>
Signed-off-by: Nicolas Sebrecht <nicolas.s-dev@laposte.net>
This commit is contained in:
Sébastien Gross 2015-10-28 10:56:16 +01:00 committed by Nicolas Sebrecht
parent b7fec93661
commit 3b30c4aa93
2 changed files with 39 additions and 14 deletions

View File

@ -60,6 +60,13 @@ class BaseFolder(object):
self._utime_from_header = self.config.getdefaultboolean(repo,
"utime_from_header", utime_from_header_global)
# Do we need to use mail timestamp for filename prefix?
filename_use_mail_timestamp_global = self.config.getdefaultboolean(
"general", "filename_use_mail_timestamp", False)
repo = "Repository " + repository.name
self._filename_use_mail_timestamp = self.config.getdefaultboolean(repo,
"filename_use_mail_timestamp", filename_use_mail_timestamp_global)
# Determine if we're running static or dynamic folder filtering
# and check filtering status
self._dynamic_folderfilter = self.config.getdefaultboolean(

View File

@ -38,22 +38,20 @@ re_uidmatch = re.compile(',U=(\d+)')
# Find a numeric timestamp in a string (filename prefix)
re_timestampmatch = re.compile('(\d+)');
timeseq = 0
lasttime = 0
timehash = {}
timelock = Lock()
def _gettimeseq():
global lasttime, timeseq, timelock
def _gettimeseq(date=None):
global timehash, timelock
timelock.acquire()
try:
thistime = long(time.time())
if thistime == lasttime:
timeseq += 1
return (thistime, timeseq)
if date is None:
date = long(time.time())
if timehash.has_key(date):
timehash[date] += 1
else:
lasttime = thistime
timeseq = 0
return (thistime, timeseq)
timehash[date] = 0
return (date, timehash[date])
finally:
timelock.release()
@ -269,14 +267,14 @@ class MaildirFolder(BaseFolder):
filepath = os.path.join(self.getfullname(), filename)
return os.path.getmtime(filepath)
def new_message_filename(self, uid, flags=set()):
def new_message_filename(self, uid, flags=set(), date=None):
"""Creates a new unique Maildir filename
:param uid: The UID`None`, or a set of maildir flags
:param flags: A set of maildir flags
:returns: String containing unique message filename"""
timeval, timeseq = _gettimeseq()
timeval, timeseq = _gettimeseq(date)
return '%d_%d.%d.%s,U=%d,FMD5=%s%s2,%s'% \
(timeval, timeseq, os.getpid(), socket.gethostname(),
uid, self._foldermd5, self.infosep, ''.join(sorted(flags)))
@ -346,7 +344,27 @@ class MaildirFolder(BaseFolder):
# Otherwise, save the message in tmp/ and then call savemessageflags()
# to give it a permanent home.
tmpdir = os.path.join(self.getfullname(), 'tmp')
messagename = self.new_message_filename(uid, flags)
# use the mail timestamp given by either Date or Delivery-date mail
# headers.
message_timestamp = None
if self._filename_use_mail_timestamp:
try:
message_timestamp = emailutil.get_message_date(content, 'Date')
if message_timestamp is None:
# Give a try with Delivery-date
date = emailutil.get_message_date(content, 'Delivery-date')
except:
# This should never happen
from email.Parser import Parser
from offlineimap.ui import getglobalui
datestr = Parser().parsestr(content, True).get("Date")
ui = getglobalui()
ui.warn("UID %d has invalid date %s: %s\n"
"Not using message timestamp as file prefix" % (uid, datestr, e))
# No need to check if date is None here since it would
# be overridden by _gettimeseq.
messagename = self.new_message_filename(uid, flags, date=message_timestamp)
tmpname = self.save_to_tmp_file(messagename, content)
if self.utime_from_header: