From c52ca6687401639b716155cc90097f190b2739fa Mon Sep 17 00:00:00 2001 From: Igor Almeida Date: Fri, 20 Nov 2015 16:09:14 -0300 Subject: [PATCH 1/7] Maildir folder: extract lower-case letters (custom flags) from filename Remove filtering that was previously done to avoid errors in flag handling. Signed-off-by: Igor Almeida Signed-off-by: Nicolas Sebrecht --- offlineimap/folder/Maildir.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/offlineimap/folder/Maildir.py b/offlineimap/folder/Maildir.py index a7dbf26..19a9ecf 100644 --- a/offlineimap/folder/Maildir.py +++ b/offlineimap/folder/Maildir.py @@ -135,9 +135,7 @@ class MaildirFolder(BaseFolder): uid = long(uidmatch.group(1)) flagmatch = self.re_flagmatch.search(filename) if flagmatch: - # Filter out all lowercase (custom maildir) flags. We don't - # handle them yet. - flags = set((c for c in flagmatch.group(1) if not c.islower())) + flags = set((c for c in flagmatch.group(1))) return prefix, uid, fmd5, flags def _scanfolder(self, min_date=None, min_uid=None): @@ -149,7 +147,7 @@ class MaildirFolder(BaseFolder): with similar UID's (e.g. the UID was reassigned much later). Maildir flags are: R (replied) S (seen) T (trashed) D (draft) F - (flagged). + (flagged), plus lower-case letters for custom flags. :returns: dict that can be used as self.messagelist. """ @@ -414,8 +412,7 @@ class MaildirFolder(BaseFolder): if flags != self.messagelist[uid]['flags']: # Flags have actually changed, construct new filename Strip - # off existing infostring (possibly discarding small letter - # flags that dovecot uses TODO) + # off existing infostring infomatch = self.re_flagmatch.search(filename) if infomatch: filename = filename[:-len(infomatch.group())] #strip off From 4e2de8f58a45181e7073990e62ff7c60286203e5 Mon Sep 17 00:00:00 2001 From: Igor Almeida Date: Fri, 20 Nov 2015 16:09:09 -0300 Subject: [PATCH 2/7] Maildir repository: add config keys for IMAP keywords This commit assembles a dictionary mapping user-specified IMAP keywords to Maildir lower-case flags, similar to Dovecot's format http://wiki2.dovecot.org/MailboxFormat/Maildir Configuration example: [Repository Local] type = Maildir localfolders = ~/Maildir/ customflag_a = $label1 customflag_b = $Forwarded customflag_c = Junk Signed-off-by: Igor Almeida Signed-off-by: Nicolas Sebrecht --- offlineimap/repository/Base.py | 3 +++ offlineimap/repository/Maildir.py | 11 +++++++++++ 2 files changed, 14 insertions(+) diff --git a/offlineimap/repository/Base.py b/offlineimap/repository/Base.py index adae16d..8634628 100644 --- a/offlineimap/repository/Base.py +++ b/offlineimap/repository/Base.py @@ -133,6 +133,9 @@ class BaseRepository(CustomConfig.ConfigHelperMixin, object): def getsep(self): raise NotImplementedError + def getkeywordmap(self): + raise NotImplementedError + def should_sync_folder(self, fname): """Should this folder be synced?""" diff --git a/offlineimap/repository/Maildir.py b/offlineimap/repository/Maildir.py index 0262ba2..fef57f3 100644 --- a/offlineimap/repository/Maildir.py +++ b/offlineimap/repository/Maildir.py @@ -39,6 +39,14 @@ class MaildirRepository(BaseRepository): if not os.path.isdir(self.root): os.mkdir(self.root, 0o700) + # Create the keyword->char mapping + self.keyword2char = dict() + for c in 'abcdefghijklmnopqrstuvwxyz': + confkey = 'customflag_' + c + keyword = self.getconf(confkey, None) + if keyword is not None: + self.keyword2char[keyword] = c + def _append_folder_atimes(self, foldername): """Store the atimes of a folder's new|cur in self.folder_atimes""" @@ -72,6 +80,9 @@ class MaildirRepository(BaseRepository): def getsep(self): return self.getconf('sep', '.').strip() + def getkeywordmap(self): + return self.keyword2char + def makefolder(self, foldername): """Create new Maildir folder if necessary From 73a3767d11614d95e9468af64d7c5af4d3349c6b Mon Sep 17 00:00:00 2001 From: Igor Almeida Date: Fri, 20 Nov 2015 16:09:10 -0300 Subject: [PATCH 3/7] IMAP folder: expose the message keywords The keywords are in the flag string, so imaputil can just strip the usual \Flags. Signed-off-by: Igor Almeida Signed-off-by: Nicolas Sebrecht --- offlineimap/folder/Base.py | 5 +++++ offlineimap/folder/IMAP.py | 8 +++++++- offlineimap/imaputil.py | 8 ++++++++ 3 files changed, 20 insertions(+), 1 deletion(-) diff --git a/offlineimap/folder/Base.py b/offlineimap/folder/Base.py index d2a0706..5031a40 100644 --- a/offlineimap/folder/Base.py +++ b/offlineimap/folder/Base.py @@ -420,6 +420,11 @@ class BaseFolder(object): raise NotImplementedError + def getmessagekeywords(self, uid): + """Returns the keywords for the specified message.""" + + raise NotImplementedError + def savemessageflags(self, uid, flags): """Sets the specified message's flags to the given set. diff --git a/offlineimap/folder/IMAP.py b/offlineimap/folder/IMAP.py index 60b5301..7f0fd6d 100644 --- a/offlineimap/folder/IMAP.py +++ b/offlineimap/folder/IMAP.py @@ -251,8 +251,10 @@ class IMAPFolder(BaseFolder): uid = long(options['UID']) self.messagelist[uid] = self.msglist_item_initializer(uid) flags = imaputil.flagsimap2maildir(options['FLAGS']) + keywords = imaputil.flagsimap2keywords(options['FLAGS']) rtime = imaplibutil.Internaldate2epoch(messagestr) - self.messagelist[uid] = {'uid': uid, 'flags': flags, 'time': rtime} + self.messagelist[uid] = {'uid': uid, 'flags': flags, 'time': rtime, + 'keywords': keywords} self.ui.messagelistloaded(self.repository, self, self.getmessagecount()) def dropmessagelistcache(self): @@ -309,6 +311,10 @@ class IMAPFolder(BaseFolder): def getmessageflags(self, uid): return self.messagelist[uid]['flags'] + # Interface from BaseFolder + def getmessagekeywords(self, uid): + return self.messagelist[uid]['keywords'] + def __generate_randomheader(self, content): """Returns a unique X-OfflineIMAP header diff --git a/offlineimap/imaputil.py b/offlineimap/imaputil.py index 6d8b444..6a18732 100644 --- a/offlineimap/imaputil.py +++ b/offlineimap/imaputil.py @@ -195,6 +195,14 @@ def flagsimap2maildir(flagstring): retval.add(maildirflag) return retval +def flagsimap2keywords(flagstring): + """Convert string '(\\Draft \\Deleted somekeyword otherkeyword)' into a + keyword set (somekeyword otherkeyword).""" + + imapflagset = set(flagstring[1:-1].split()) + serverflagset = set([flag for (flag, c) in flagmap]) + return imapflagset - serverflagset + def flagsmaildir2imap(maildirflaglist): """Convert set of flags ([DR]) into a string '(\\Deleted \\Draft)'.""" From 61ee6e783e6b6faf3258f7b847fb8c870ab4395e Mon Sep 17 00:00:00 2001 From: Igor Almeida Date: Fri, 20 Nov 2015 16:09:11 -0300 Subject: [PATCH 4/7] __syncmessagesto_flags: store keywords This uses the destination folder's keyword mapping to translate the message's keywords into some appropriate format. Tested only with local Maildir. Signed-off-by: Igor Almeida Signed-off-by: Nicolas Sebrecht --- offlineimap/folder/Base.py | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/offlineimap/folder/Base.py b/offlineimap/folder/Base.py index 5031a40..b48afbe 100644 --- a/offlineimap/folder/Base.py +++ b/offlineimap/folder/Base.py @@ -937,6 +937,32 @@ class BaseFolder(object): else: statusflags = set() + #keywords: if there is a keyword map, use it to figure out what + #other 'flags' we should add + try: + keywordmap = dstfolder.getrepository().getkeywordmap() + knownkeywords = set(keywordmap.keys()) + + selfkeywords = self.getmessagekeywords(uid) + + if not knownkeywords >= selfkeywords: + #some of the message's keywords are not in the mapping, so + #skip them + + skipped_keywords = list(selfkeywords - knownkeywords) + selfkeywords &= knownkeywords + + self.ui.warn("Unknown keywords skipped: %s\n" + "You may want to change your configuration to include " + "those\n" % (skipped_keywords)) + + keywordletterset = set([keywordmap[keyw] for keyw in selfkeywords]) + + #add the lower-case letters to the list of message flags + selfflags |= keywordletterset + except NotImplementedError: + pass + addflags = selfflags - statusflags delflags = statusflags - selfflags From 58a6f8b401cf8608f020711be6b4f63b0083fcda Mon Sep 17 00:00:00 2001 From: Igor Almeida Date: Fri, 20 Nov 2015 16:09:12 -0300 Subject: [PATCH 5/7] __syncmessagesto_flags: refactor for readability Extract the flag/keyword translation and combination logic to a function. Signed-off-by: Igor Almeida Signed-off-by: Nicolas Sebrecht --- offlineimap/folder/Base.py | 64 ++++++++++++++++++++++---------------- 1 file changed, 37 insertions(+), 27 deletions(-) diff --git a/offlineimap/folder/Base.py b/offlineimap/folder/Base.py index b48afbe..d77a16c 100644 --- a/offlineimap/folder/Base.py +++ b/offlineimap/folder/Base.py @@ -908,6 +908,42 @@ class BaseFolder(object): return #don't delete messages in dry-run mode dstfolder.deletemessages(deletelist) + def combine_flags_and_keywords(self, uid, dstfolder): + """Combine the message's flags and keywords using the mapping for the + destination folder.""" + + # Take a copy of the message flag set, otherwise + # __syncmessagesto_flags() will fail because statusflags is actually a + # reference to selfflags (which it should not, but I don't have time to + # debug THAT). + selfflags = set(self.getmessageflags(uid)) + + try: + keywordmap = dstfolder.getrepository().getkeywordmap() + knownkeywords = set(keywordmap.keys()) + + selfkeywords = self.getmessagekeywords(uid) + + if not knownkeywords >= selfkeywords: + #some of the message's keywords are not in the mapping, so + #skip them + + skipped_keywords = list(selfkeywords - knownkeywords) + selfkeywords &= knownkeywords + + self.ui.warn("Unknown keywords skipped: %s\n" + "You may want to change your configuration to include " + "those\n" % (skipped_keywords)) + + keywordletterset = set([keywordmap[keyw] for keyw in selfkeywords]) + + #add the mapped keywords to the list of message flags + selfflags |= keywordletterset + except NotImplementedError: + pass + + return selfflags + def __syncmessagesto_flags(self, dstfolder, statusfolder): """Pass 3: Flag synchronization. @@ -930,38 +966,12 @@ class BaseFolder(object): if uid < 0 or not dstfolder.uidexists(uid): continue - selfflags = self.getmessageflags(uid) - if statusfolder.uidexists(uid): statusflags = statusfolder.getmessageflags(uid) else: statusflags = set() - #keywords: if there is a keyword map, use it to figure out what - #other 'flags' we should add - try: - keywordmap = dstfolder.getrepository().getkeywordmap() - knownkeywords = set(keywordmap.keys()) - - selfkeywords = self.getmessagekeywords(uid) - - if not knownkeywords >= selfkeywords: - #some of the message's keywords are not in the mapping, so - #skip them - - skipped_keywords = list(selfkeywords - knownkeywords) - selfkeywords &= knownkeywords - - self.ui.warn("Unknown keywords skipped: %s\n" - "You may want to change your configuration to include " - "those\n" % (skipped_keywords)) - - keywordletterset = set([keywordmap[keyw] for keyw in selfkeywords]) - - #add the lower-case letters to the list of message flags - selfflags |= keywordletterset - except NotImplementedError: - pass + selfflags = self.combine_flags_and_keywords(uid, dstfolder) addflags = selfflags - statusflags delflags = statusflags - selfflags From def087eeeab072a7cb0925edbe90a3827310bf3a Mon Sep 17 00:00:00 2001 From: Igor Almeida Date: Fri, 20 Nov 2015 16:09:13 -0300 Subject: [PATCH 6/7] IMAP keyword sync: add Maildir configuration example Signed-off-by: Igor Almeida Signed-off-by: Nicolas Sebrecht --- offlineimap.conf | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/offlineimap.conf b/offlineimap.conf index 2d01221..4be3ce5 100644 --- a/offlineimap.conf +++ b/offlineimap.conf @@ -536,6 +536,31 @@ localfolders = ~/Test # #filename_use_mail_timestamp = no +# This option stands in the [Repository LocalExample] section. +# +# Map IMAP [user-defined] keywords to lowercase letters, similar to Dovecot's +# format described in http://wiki2.dovecot.org/MailboxFormat/Maildir . This +# option makes sense for the Maildir type, only. +# +# Configuration example: +# customflag_x = some_keyword +# +# With the configuration example above enabled, all IMAP messages that have +# 'some_keyword' in their FLAGS field will have an 'x' in the flags part of the +# maildir filename: +# 1234567890.M20046P2137.mailserver,S=4542,W=4642:2,Sx +# +# Valid fields are customflag_[a-z], valid values are whatever the IMAP server +# allows. +# +# Comparison in offlineimap is case-sensitive. +# +# This option is EXPERIMENTAL. +# +#customflag_a = some_keyword +#customflag_b = $OtherKeyword +#customflag_c = NonJunk +#customflag_d = ToDo [Repository GmailLocalExample] From d6077a09cf8a7e4c30635a8cec2ac2e278c8da5a Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Sun, 22 Nov 2015 19:52:32 +0100 Subject: [PATCH 7/7] Keywords: avoid warning at each message when no keywords are used This fix does not apply when any keyword in configured which is already harmless. Written-by: Igor Almeida Signed-off-by: Nicolas Sebrecht --- offlineimap/folder/Base.py | 3 +++ offlineimap/repository/Maildir.py | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/offlineimap/folder/Base.py b/offlineimap/folder/Base.py index d77a16c..14b0867 100644 --- a/offlineimap/folder/Base.py +++ b/offlineimap/folder/Base.py @@ -920,6 +920,9 @@ class BaseFolder(object): try: keywordmap = dstfolder.getrepository().getkeywordmap() + if keywordmap is None: + return selfflags + knownkeywords = set(keywordmap.keys()) selfkeywords = self.getmessagekeywords(uid) diff --git a/offlineimap/repository/Maildir.py b/offlineimap/repository/Maildir.py index fef57f3..10085e7 100644 --- a/offlineimap/repository/Maildir.py +++ b/offlineimap/repository/Maildir.py @@ -81,7 +81,7 @@ class MaildirRepository(BaseRepository): return self.getconf('sep', '.').strip() def getkeywordmap(self): - return self.keyword2char + return self.keyword2char if len(self.keyword2char) > 0 else None def makefolder(self, foldername): """Create new Maildir folder if necessary