/offlineimap/head: changeset 442

Moved account-sep branch to head
This commit is contained in:
jgoerzen 2003-04-18 03:18:34 +01:00
parent 0f81229c68
commit faf26007b1
22 changed files with 650 additions and 290 deletions

View File

@ -18,4 +18,4 @@
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
from offlineimap import init
init.startup('3.99.10')
init.startup('3.99.12')

View File

@ -1,3 +1,38 @@
offlineimap (3.99.12) unstable; urgency=low
* This is a 4.0 TRACK release, and may be unstable or in flux!
* Big news: OfflineIMAP can now sync two remote IMAP servers to each
other, with no need to write a Maildir at all.
* WARNING: the format of the configuration file *AND* the local
status area changes with this release!
* Major reworking of internal management of accounts. Previously, the
account defined a local Maildir and a remote IMAP server. Now, the
account is simply a connection between two repositories. For
traditional ones, an account basically specifies a refresh interval,
a Maildir repository, and an IMAP repository.
* Added a notion of a repository to the configuration file. Repositories
currently available are IMAP and Maildir. Combined with the new account
system, this lets the user define powerful combinations without
duplicating information.
* When uploading messages to an IMAP server, OfflineIMAP generates its
own X-OfflineIMAP header rather than trying to guess the new message UID
based on the Message-Id header. This leads to greater reliability when
uploading messages, especially when dealing with duplicate messages.
This change was required to permit reliable IMAP-to-IMAP syncing, but
helps with regular IMAP-to-Maildir syncing as well.
* Local status area under ~/.offlineimap revamped. It now contains
separate subdirectories for each account and repository, and they
contain UID validity information, UID mapping (for IMAP-to-IMAP
syncing). UID validity information is no longer stored in the Maildir
itself.
* New debug type: "thread" to debug multithreading.
* preauth tunnels no longer require remoteuser, remotepass, host,
or port in the configuration file.
* Logging for preauth tunnels is more sensible.
* Fixed a logic error for syncs with a reference that returns no folders.
-- John Goerzen <jgoerzen@complete.org> Thu, 17 Apr 2003 17:20:08 -0500
offlineimap (3.99.11) unstable; urgency=low
* Curses interface can now be resized. Closes: #176342.

View File

@ -147,16 +147,52 @@ fontsize = 8
# This is an account definition clause. You'll have one of these
# for each account listed in general/accounts above.
[Test]
[Account Test]
########## Basic settings
# These settings specify the two folders that you will be syncing.
# You'll need to have a "Repository ..." section for each one.
localrepository = LocalExample
remoterepository = RemoteExample
########## Advanced settings
# You can have offlineimap continue running indefinately, automatically
# syncing your mail periodically. If you want that, specify how
# frequently to do that (in minutes) here.
# autorefresh = 5
[Repository LocalExample]
# This is one of the two repositories that you'll work with given the
# above example. Each repository requires a "type" declaration.
#
# The types supported are Maildir and IMAP.
#
type = Maildir
# Specify local repository. Your IMAP folders will be synchronized
# to maildirs created under this path. OfflineIMAP will create the
# maildirs for you as needed.
localfolders = ~/Test
# You can specify the "path separator character" used for your Maildir
# folders. This is inserted in-between the components of the tree.
# It defaults to ".". If you want your Maildir folders to be nested,
# set it to "/".
sep = .
[Repository RemoteExample]
# And this is the remote repository. For now, we only support IMAP here.
type = IMAP
# Specify the remote hostname.
remotehost = examplehost
@ -198,13 +234,6 @@ remoteuser = username
########## Advanced settings
# You can have offlineimap continue running indefinately, automatically
# syncing your mail periodically. If you want that, specify how
# frequently to do that (in minutes) here.
# autorefresh = 5
# Some IMAP servers need a "reference" which often refers to the
# "folder root". This is most commonly needed with UW IMAP, where
# you might need to specify the directory in which your mail is
@ -212,6 +241,40 @@ remoteuser = username
#
# reference = Mail
# OfflineIMAP can use multiple connections to the server in order
# to perform multiple synchronization actions simultaneously.
# This may place a higher burden on the server. In most cases,
# setting this value to 2 or 3 will speed up the sync, but in some
# cases, it may slow things down. The safe answer is 1. You should
# probably never set it to a value more than 5.
maxconnections = 1
# OfflineIMAP normally closes IMAP server connections between refreshes if
# the global option autorefresh is specified. If you wish it to keep the
# connection open, set this to true. If not specified, the default is
# false. Keeping the connection open means a faster sync start the
# next time and may use fewer server resources on connection, but uses
# more server memory. This setting has no effect if autorefresh is not set.
holdconnectionopen = no
# If you want to have "keepalives" sent while waiting between syncs,
# specify the amount of time IN SECONDS between keepalives here. Note that
# sometimes more than this amount of time might pass, so don't make it
# tight. This setting has no effect if autorefresh and holdconnectionopen
# are not both set.
# keepalive = 60
# Normally, OfflineIMAP will expunge deleted messages from the server.
# You can disable that if you wish. This means that OfflineIMAP will
# mark them deleted on the server, but not actually delete them.
# You must use some other IMAP client to delete them if you use this
# setting; otherwise, the messgaes will just pile up there forever.
# Therefore, this setting is definately NOT recommended.
#
# expunge = no
# You can specify a folder translator. This must be a eval-able
# Python expression that takes a foldername arg and returns the new
# value. I suggest a lambda. This example below will remove "INBOX." from
@ -290,44 +353,3 @@ remoteuser = username
#
# foldersort = lambda x, y: -cmp(x, y)
# OfflineIMAP can use multiple connections to the server in order
# to perform multiple synchronization actions simultaneously.
# This may place a higher burden on the server. In most cases,
# setting this value to 2 or 3 will speed up the sync, but in some
# cases, it may slow things down. The safe answer is 1. You should
# probably never set it to a value more than 5.
maxconnections = 1
# OfflineIMAP normally closes IMAP server connections between refreshes if
# the global option autorefresh is specified. If you wish it to keep the
# connection open, set this to true. If not specified, the default is
# false. Keeping the connection open means a faster sync start the
# next time and may use fewer server resources on connection, but uses
# more server memory. This setting has no effect if autorefresh is not set.
holdconnectionopen = no
# If you want to have "keepalives" sent while waiting between syncs,
# specify the amount of time IN SECONDS between keepalives here. Note that
# sometimes more than this amount of time might pass, so don't make it
# tight. This setting has no effect if autorefresh and holdconnectionopen
# are not both set.
# keepalive = 60
# You can specify the "path separator character" used for your Maildir
# folders. This is inserted in-between the components of the tree.
# It defaults to ".". If you want your Maildir folders to be nested,
# set it to "/".
sep = .
# Normally, OfflineIMAP will expunge deleted messages from the server.
# You can disable that if you wish. This means that OfflineIMAP will
# mark them deleted on the server, but not actually delete them.
# You must use some other IMAP client to delete them if you use this
# setting; otherwise, the messgaes will just pile up there forever.
# Therefore, this setting is definately NOT recommended.
#
# expunge = no

View File

@ -4,7 +4,15 @@
[general]
accounts = Test
[Test]
[Account Test]
localrepository = Main
remoterepository = Example
[Repository Main]
type = Maildir
localfolders = ~/Test
[Repository Example]
type = IMAP
remotehost = examplehost
remoteuser = jgoerzen

View File

@ -18,4 +18,4 @@
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
from offlineimap import init
init.startup('3.99.10')
init.startup('3.99.12')

View File

@ -1,4 +1,3 @@
<!-- -*- DocBook -*- -->
<!DOCTYPE reference PUBLIC "-//OASIS//DTD DocBook V4.1//EN" [
<!ENTITY OfflineIMAP "<application>OfflineIMAP</application>">
]>
@ -11,7 +10,7 @@
<refentryinfo>
<address><email>jgoerzen@complete.org</email></address>
<author><firstname>John</firstname><surname>Goerzen</surname></author>
<date> $Date: 2003-04-16 09:23:45 -0500 (Wed, 16 Apr 2003) $ </date>
<date> $Date: 2003-04-17 13:25:30 -0500 (Thu, 17 Apr 2003) $ </date>
</refentryinfo>
<refmeta>
@ -333,13 +332,15 @@ cd offlineimap-x.y.z</ProgramListing>
<para><option>-d</option> requires one or more debugtypes,
separated by commas. These define what exactly will be
debugged, and include two options: <property>imap</property>
and <property>maildir</property>. The <property>imap</property>
debugged, and include three options: <property>imap</property>,
<property>maildir</property>, and <property>thread</property>.
The <property>imap</property>
option will enable IMAP protocol stream and parsing debugging. Note
that the output may contain passwords, so take care to remove that
from the debugging output before sending it to anyone else. The
<property>maildir</property> option will enable debugging for
certain Maildir operations.
certain Maildir operations. And <property>thread</property>
will debug the threading model.
</para></listitem>
</varlistentry>
<varlistentry><term>-o</term>
@ -1032,3 +1033,10 @@ rm -r ~/.offlineimap/AccountName/INBOX</programlisting>
</refsect1>
</refentry>
</reference>
<!--
Local Variables:
mode: sgml
sgml-set-face: T
End:
-->

View File

@ -54,6 +54,40 @@ class CustomConfigParser(ConfigParser):
path = None
return LocalEval(path)
def getaccountlist(self):
return [x for x in self.sections() if x != 'general']
def getsectionlist(self, key):
"""Returns a list of sections that start with key + " ". That is,
if key is "Account", returns all section names that start with
"Account ", but strips off the "Account ". For instance, for
"Account Test", returns "Test"."""
key = key + ' '
return [x[len(key):] for x in self.sections() \
if x.startswith(key)]
def CustomConfigDefault():
"""Just a sample constant that won't occur anywhere else to use for the
default."""
pass
class ConfigHelperMixin:
def _confighelper_runner(self, option, default, defaultfunc, mainfunc):
if default != CustomConfigDefault:
return apply(defaultfunc, [self.getsection(), option, default])
else:
return apply(mainfunc, [self.getsection(), option])
def getconf(self, option, default = CustomConfigDefault):
return self._confighelper_runner(option, default,
self.getconfig().getdefault,
self.getconfig().get)
def getconfboolean(self, option, default = CustomConfigDefault):
return self._confighelper_runner(option, default,
self.getconfig().getdefaultboolean,
self.getconfig().getboolean)
def getconfint(self, option, default = CustomConfigDefault):
return self._confighelper_runner(option, default,
self.getconfig().getdefaultint,
self.getconfig().getint)

View File

@ -15,39 +15,49 @@
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
from offlineimap import imapserver, repository, threadutil, mbnames
from offlineimap import repository, threadutil, mbnames, CustomConfig
from offlineimap.ui import UIBase
from offlineimap.threadutil import InstanceLimitedThread, ExitNotifyThread
from threading import Event
import os
def getaccountlist(customconfig):
return customconfig.getsectionlist('Account')
def AccountListGenerator(customconfig):
return [Account(customconfig, accountname)
for accountname in getaccountlist(customconfig)]
def AccountHashGenerator(customconfig):
retval = {}
for item in AccountListGenerator(customconfig):
retval[item.getname()] = item
return retval
mailboxes = []
class Account:
class Account(CustomConfig.ConfigHelperMixin):
def __init__(self, config, name):
self.config = config
self.name = name
self.metadatadir = config.getmetadatadir()
self.localeval = config.getlocaleval()
self.server = imapserver.ConfigedIMAPServer(config, self.name)
self.ui = UIBase.getglobalui()
if self.config.has_option(self.name, 'autorefresh'):
self.refreshperiod = self.config.getint(self.name, 'autorefresh')
else:
self.refreshperiod = self.getconfint('autorefresh', 0)
if self.refreshperiod == 0:
self.refreshperiod = None
self.hold = self.config.has_option(self.name, 'holdconnectionopen') \
and self.config.getboolean(self.name, 'holdconnectionopen')
if self.config.has_option(self.name, 'keepalive'):
self.keepalive = self.config.getint(self.name, 'keepalive')
else:
self.keepalive = None
def getconf(self, option, default = None):
if default != None:
return self.config.get(self.name, option)
else:
return self.config.getdefault(self.name, option,
default)
def getlocaleval(self):
return self.localeval
def getconfig(self):
return self.config
def getname(self):
return self.name
def getsection(self):
return 'Account ' + self.getname()
def sleeper(self):
"""Sleep handler. Returns same value as UIBase.sleep:
@ -58,31 +68,46 @@ class Account:
if not self.refreshperiod:
return 100
kaobjs = []
if hasattr(self, 'localrepos'):
kaobjs.append(self.localrepos)
if hasattr(self, 'remoterepos'):
kaobjs.append(self.remoterepos)
for item in kaobjs:
item.startkeepalive()
refreshperiod = self.refreshperiod * 60
if self.keepalive:
kaevent = Event()
kathread = ExitNotifyThread(target = self.server.keepalive,
name = "Keep alive " + self.name,
args = (self.keepalive, kaevent))
kathread.setDaemon(1)
kathread.start()
sleepresult = self.ui.sleep(refreshperiod)
if sleepresult == 2:
# Cancel keep-alive, but don't bother terminating threads
if self.keepalive:
kaevent.set()
for item in kaobjs:
item.stopkeepalive(abrupt = 1)
return sleepresult
else:
# Cancel keep-alive and wait for thread to terminate.
if self.keepalive:
kaevent.set()
kathread.join()
for item in kaobjs:
item.stopkeepalive(abrupt = 0)
return sleepresult
class AccountSynchronizationMixin:
def syncrunner(self):
self.ui.registerthread(self.name)
self.ui.acct(self.name)
accountmetadata = self.getaccountmeta()
if not os.path.exists(accountmetadata):
os.mkdir(accountmetadata, 0700)
self.remoterepos = repository.Base.LoadRepository(self.getconf('remoterepository'), self, 'remote')
# Connect to the local repository.
self.localrepos = repository.Base.LoadRepository(self.getconf('localrepository'), self, 'local')
# Connect to the local cache.
self.statusrepos = repository.LocalStatus.LocalStatusRepository(self.getconf('localrepository'), self)
if not self.refreshperiod:
self.sync()
self.ui.acctdone(self.name)
@ -93,32 +118,23 @@ class AccountSynchronizationMixin:
looping = self.sleeper() != 2
self.ui.acctdone(self.name)
def getaccountmeta(self):
return os.path.join(self.metadatadir, 'Account-' + self.name)
def sync(self):
# We don't need an account lock because syncitall() goes through
# each account once, then waits for all to finish.
try:
accountmetadata = os.path.join(self.metadatadir, self.name)
if not os.path.exists(accountmetadata):
os.mkdir(accountmetadata, 0700)
remoterepos = repository.IMAP.IMAPRepository(self.config,
self.localeval,
self.name,
self.server)
# Connect to the Maildirs.
localrepos = repository.Maildir.MaildirRepository(os.path.expanduser(self.config.get(self.name, "localfolders")), self.name, self.config)
# Connect to the local cache.
statusrepos = repository.LocalStatus.LocalStatusRepository(accountmetadata, self.name)
remoterepos = self.remoterepos
localrepos = self.localrepos
statusrepos = self.statusrepos
self.ui.syncfolders(remoterepos, localrepos)
remoterepos.syncfoldersto(localrepos)
folderthreads = []
for remotefolder in remoterepos.getfolders():
thread = InstanceLimitedThread(\
instancename = 'FOLDER_' + self.name,
instancename = 'FOLDER_' + self.remoterepos.getname(),
target = syncfolder,
name = "Folder sync %s[%s]" % \
(self.name, remotefolder.getvisiblename()),
@ -129,8 +145,8 @@ class AccountSynchronizationMixin:
folderthreads.append(thread)
threadutil.threadsreset(folderthreads)
mbnames.write()
if not self.hold:
self.server.close()
localrepos.holdordropconnections()
remoterepos.holdordropconnections()
finally:
pass
@ -166,19 +182,22 @@ def syncfolder(accountname, remoterepos, remotefolder, localrepos,
statusfolder.cachemessagelist()
# If either the local or the status folder has messages and
# there is a UID validity problem, warn and abort.
# If there are no messages, UW IMAPd loses UIDVALIDITY.
# But we don't really need it if both local folders are empty.
# So, in that case, save it off.
if (len(localfolder.getmessagelist()) or \
len(statusfolder.getmessagelist())) and \
not localfolder.isuidvalidityok(remotefolder):
ui.validityproblem(remotefolder)
return
# If either the local or the status folder has messages and there is a UID
# validity problem, warn and abort. If there are no messages, UW IMAPd
# loses UIDVALIDITY. But we don't really need it if both local folders are
# empty. So, in that case, just save it off.
if len(localfolder.getmessagelist()) or len(statusfolder.getmessagelist()):
if not localfolder.isuidvalidityok():
ui.validityproblem(localfolder, localfolder.getsaveduidvalidity(),
localfolder.getuidvalidity())
return
if not remotefolder.isuidvalidityok():
ui.validityproblem(remotefolder, remotefolder.getsaveduidvalidity(),
remotefolder.getuidvalidity())
return
else:
localfolder.saveuidvalidity(remotefolder.getuidvalidity())
localfolder.saveuidvalidity()
remotefolder.saveuidvalidity()
# Load remote folder.
ui.loadmessagelist(remoterepos, remotefolder)

View File

@ -20,8 +20,12 @@ from threading import *
from offlineimap import threadutil
from offlineimap.threadutil import InstanceLimitedThread
from offlineimap.ui import UIBase
import os.path, re
class BaseFolder:
def __init__(self):
self.uidlock = Lock()
def getname(self):
"""Returns name"""
return self.name
@ -68,15 +72,52 @@ class BaseFolder:
else:
return self.getname()
def isuidvalidityok(self, remotefolder):
raise NotImplementedException
def getfolderbasename(self):
foldername = self.getname()
foldername = foldername.replace(self.repository.getsep(), '.')
foldername = re.sub('/\.$', '/dot', foldername)
foldername = re.sub('^\.$', 'dot', foldername)
return foldername
def isuidvalidityok(self):
if self.getsaveduidvalidity() != None:
return self.getsaveduidvalidity() == self.getuidvalidity()
else:
self.saveuidvalidity()
return 1
def _getuidfilename(self):
return os.path.join(self.repository.getuiddir(),
self.getfolderbasename())
def getsaveduidvalidity(self):
if hasattr(self, '_base_saved_uidvalidity'):
return self._base_saved_uidvalidity
uidfilename = self._getuidfilename()
if not os.path.exists(uidfilename):
self._base_saved_uidvalidity = None
else:
file = open(uidfilename, "rt")
self._base_saved_uidvalidity = long(file.readline().strip())
file.close()
return self._base_saved_uidvalidity
def saveuidvalidity(self):
newval = self.getuidvalidity()
uidfilename = self._getuidfilename()
self.uidlock.acquire()
try:
file = open(uidfilename + ".tmp", "wt")
file.write("%d\n" % newval)
file.close()
os.rename(uidfilename + ".tmp", uidfilename)
self._base_saved_uidvalidity = newval
finally:
self.uidlock.release()
def getuidvalidity(self):
raise NotImplementedException
def saveuidvalidity(self, newval):
raise NotImplementedException
def cachemessagelist(self):
"""Reads the message list from disk or network and stores it in
memory for later use. This list will not be re-read from disk or

View File

@ -19,7 +19,7 @@
from Base import BaseFolder
from offlineimap import imaputil, imaplib
from offlineimap.ui import UIBase
import rfc822, time, string
import rfc822, time, string, random, binascii
from StringIO import StringIO
from copy import copy
@ -27,9 +27,7 @@ from copy import copy
class IMAPFolder(BaseFolder):
def __init__(self, imapserver, name, visiblename, accountname, repository):
self.config = imapserver.config
self.expunge = 1
if self.config.has_option(accountname, 'expunge'):
self.expunge = self.config.getboolean(accountname, 'expunge')
self.expunge = repository.getexpunge()
self.name = imaputil.dequote(name)
self.root = None # imapserver.root
self.sep = imapserver.delim
@ -38,6 +36,8 @@ class IMAPFolder(BaseFolder):
self.visiblename = visiblename
self.accountname = accountname
self.repository = repository
self.randomgenerator = random.Random()
BaseFolder.__init__(self)
def getaccountname(self):
return self.accountname
@ -49,7 +49,7 @@ class IMAPFolder(BaseFolder):
self.imapserver.connectionwait()
def getcopyinstancelimit(self):
return 'MSGCOPY_' + self.accountname
return 'MSGCOPY_' + self.repository.getname()
def getvisiblename(self):
return self.visiblename
@ -70,7 +70,12 @@ class IMAPFolder(BaseFolder):
try:
# Primes untagged_responses
imapobj.select(self.getfullname(), readonly = 1)
maxmsgid = long(imapobj.untagged_responses['EXISTS'][0])
try:
# Some mail servers do not return an EXISTS response if
# the folder is empty.
maxmsgid = long(imapobj.untagged_responses['EXISTS'][0])
except KeyError:
return
if maxmsgid < 1:
# No messages; return
return
@ -107,7 +112,37 @@ class IMAPFolder(BaseFolder):
def getmessageflags(self, uid):
return self.messagelist[uid]['flags']
def savemessage_getnewheader(self, content):
headername = 'X-OfflineIMAP-%s-' % str(binascii.crc32(content)).replace('-', 'x')
headername += binascii.hexlify(self.repository.getname()) + '-'
headername += binascii.hexlify(self.getname())
headervalue= '%d-' % long(time.time())
headervalue += str(self.randomgenerator.random()).replace('.', '')
return (headername, headervalue)
def savemessage_addheader(self, content, headername, headervalue):
insertionpoint = content.find("\r\n")
leader = content[0:insertionpoint]
newline = "\r\n%s: %s" % (headername, headervalue)
trailer = content[insertionpoint:]
return leader + newline + trailer
def savemessage_searchforheader(self, imapobj, headername, headervalue):
# Now find the UID it got.
headervalue = imapobj._quote(headervalue)
try:
matchinguids = imapobj.uid('search', None,
'(HEADER %s %s)' % (headername, headervalue))[1][0]
except imapobj.error:
# IMAP server doesn't implement search or had a problem.
return 0
matchinguids = matchinguids.split(' ')
if len(matchinguids) != 1 or matchinguids[0] == None:
raise ValueError, "While attempting to find UID for message with header %s, got wrong-sized matchinguids of %s" % (headername, str(matchinguids))
matchinguids.sort()
return long(matchinguids[0])
def savemessage(self, uid, content, flags):
imapobj = self.imapserver.acquireconnection()
try:
@ -123,9 +158,6 @@ class IMAPFolder(BaseFolder):
# In order to get the new uid, we need to save off the message ID.
message = rfc822.Message(StringIO(content))
mid = message.getheader('Message-Id')
if mid != None:
mid = imapobj._quote(mid)
datetuple = rfc822.parsedate(message.getheader('Date'))
# Will be None if missing or not in a valid format.
if datetuple == None:
@ -145,35 +177,31 @@ class IMAPFolder(BaseFolder):
if content.find("\r\n") == -1: # Convert line endings if not already
content = content.replace("\n", "\r\n")
(headername, headervalue) = self.savemessage_getnewheader(content)
content = self.savemessage_addheader(content, headername,
headervalue)
assert(imapobj.append(self.getfullname(),
imaputil.flagsmaildir2imap(flags),
date, content)[0] == 'OK')
# Checkpoint. Let it write out the messages, etc.
assert(imapobj.check()[0] == 'OK')
if mid == None:
# No message ID in original message -- no sense trying to
# search for it.
return 0
# Now find the UID it got.
# Keep trying until we get the UID.
try:
matchinguids = imapobj.uid('search', None,
'(HEADER Message-Id %s)' % mid)[1][0]
except imapobj.error:
# IMAP server doesn't implement search or had a problem.
return 0
matchinguids = matchinguids.split(' ')
if len(matchinguids) != 1 or matchinguids[0] == None:
return 0
matchinguids.sort()
try:
uid = long(matchinguids[-1])
uid = self.savemessage_searchforheader(imapobj, headername,
headervalue)
except ValueError:
return 0
self.messagelist[uid] = {'uid': uid, 'flags': flags}
return uid
assert(imapobj.noop()[0] == 'OK')
uid = self.savemessage_searchforheader(imapobj, headername,
headervalue)
finally:
self.imapserver.releaseconnection(imapobj)
self.messagelist[uid] = {'uid': uid, 'flags': flags}
return uid
def savemessageflags(self, uid, flags):
imapobj = self.imapserver.acquireconnection()
try:
@ -197,9 +225,15 @@ class IMAPFolder(BaseFolder):
def addmessageflags(self, uid, flags):
self.addmessagesflags([uid], flags)
def addmessagesflags(self, uidlist, flags):
def addmessagesflags_noconvert(self, uidlist, flags):
self.processmessagesflags('+', uidlist, flags)
def addmessagesflags(self, uidlist, flags):
"""This is here for the sake of UIDMaps.py -- deletemessages must
add flags and get a converted UID, and if we don't have noconvert,
then UIDMaps will try to convert it twice."""
self.addmessagesflags_noconvert(uidlist, flags)
def deletemessageflags(self, uid, flags):
self.deletemessagesflags([uid], flags)
@ -254,15 +288,18 @@ class IMAPFolder(BaseFolder):
self.messagelist[uid]['flags'].remove(flag)
def deletemessage(self, uid):
self.deletemessages([uid])
self.deletemessages_noconvert([uid])
def deletemessages(self, uidlist):
self.deletemessages_noconvert(uidlist)
def deletemessages_noconvert(self, uidlist):
# Weed out ones not in self.messagelist
uidlist = [uid for uid in uidlist if uid in self.messagelist]
if not len(uidlist):
return
self.addmessagesflags(uidlist, ['T'])
self.addmessagesflags_noconvert(uidlist, ['T'])
imapobj = self.imapserver.acquireconnection()
try:
try:

View File

@ -27,11 +27,13 @@ class LocalStatusFolder(BaseFolder):
self.root = root
self.sep = '.'
self.filename = os.path.join(root, name)
self.filename = repository.getfolderfilename(name)
self.messagelist = None
self.repository = repository
self.savelock = threading.Lock()
self.doautosave = 1
self.accountname = accountname
BaseFolder.__init__(self)
def getaccountname(self):
return self.accountname

View File

@ -43,10 +43,10 @@ class MaildirFolder(BaseFolder):
self.name = name
self.root = root
self.sep = sep
self.uidfilename = os.path.join(self.getfullname(), "offlineimap.uidvalidity")
self.messagelist = None
self.repository = repository
self.accountname = accountname
BaseFolder.__init__(self)
def getaccountname(self):
return self.accountname
@ -55,31 +55,10 @@ class MaildirFolder(BaseFolder):
return os.path.join(self.getroot(), self.getname())
def getuidvalidity(self):
if hasattr(self, 'uidvalidity'):
return self.uidvalidity
if not os.path.exists(self.uidfilename):
self.uidvalidity = None
else:
file = open(self.uidfilename, "rt")
self.uidvalidity = long(file.readline().strip())
file.close()
return self.uidvalidity
"""Maildirs have no notion of uidvalidity, so we just return a magic
token."""
return 42
def saveuidvalidity(self, newval):
file = open(self.uidfilename + ".tmp", "wt")
file.write("%d\n" % newval)
file.close()
os.rename(self.uidfilename + ".tmp", self.uidfilename)
self.uidvalidity = newval
def isuidvalidityok(self, remotefolder):
myval = self.getuidvalidity()
if myval != None:
return myval == remotefolder.getuidvalidity()
else:
self.saveuidvalidity(remotefolder.getuidvalidity())
return 1
def _scanfolder(self):
"""Cache the message list. Maildir flags are:
R (replied)

View File

@ -1,5 +1,5 @@
# IMAP server support
# Copyright (C) 2002 John Goerzen
# Copyright (C) 2002, 2003 John Goerzen
# <jgoerzen@complete.org>
#
# This program is free software; you can redistribute it and/or modify
@ -48,11 +48,11 @@ class UsefulIMAP4_SSL(UsefulIMAPMixIn, imaplib.IMAP4_SSL): pass
class UsefulIMAP4_Tunnel(UsefulIMAPMixIn, imaplib.IMAP4_Tunnel): pass
class IMAPServer:
def __init__(self, config, accountname,
def __init__(self, config, reposname,
username = None, password = None, hostname = None,
port = None, ssl = 1, maxconnections = 1, tunnel = None,
reference = '""'):
self.account = accountname
self.reposname = reposname
self.config = config
self.username = username
self.password = password
@ -80,7 +80,8 @@ class IMAPServer:
if self.password != None and self.passworderror == None:
return self.password
self.password = UIBase.getglobalui().getpass(self.account, self.config,
self.password = UIBase.getglobalui().getpass(self.reposname,
self.config,
self.passworderror)
self.passworderror = None
@ -152,17 +153,18 @@ class IMAPServer:
self.connectionlock.release() # Release until need to modify data
UIBase.getglobalui().connecting(self.hostname, self.port)
success = 0
while not success:
# Generate a new connection.
if self.tunnel:
UIBase.getglobalui().connecting('tunnel', self.tunnel)
imapobj = UsefulIMAP4_Tunnel(self.tunnel)
success = 1
elif self.usessl:
UIBase.getglobalui().connecting(self.hostname, self.port)
imapobj = UsefulIMAP4_SSL(self.hostname, self.port)
else:
UIBase.getglobalui().connecting(self.hostname, self.port)
imapobj = UsefulIMAP4(self.hostname, self.port)
if not self.tunnel:
@ -258,39 +260,34 @@ class ConfigedIMAPServer(IMAPServer):
object and an account name. The passwordhash is used if
passwords for certain accounts are known. If the password for this
account is listed, it will be obtained from there."""
def __init__(self, config, accountname, passwordhash = {}):
def __init__(self, repository, passwordhash = {}):
"""Initialize the object. If the account is not a tunnel,
the password is required."""
host = config.get(accountname, "remotehost")
user = config.get(accountname, "remoteuser")
port = None
if config.has_option(accountname, "remoteport"):
port = config.getint(accountname, "remoteport")
ssl = config.getdefaultboolean(accountname, "ssl", 0)
usetunnel = config.has_option(accountname, "preauthtunnel")
reference = '""'
if config.has_option(accountname, "reference"):
reference = config.get(accountname, "reference")
self.repos = repository
self.config = self.repos.getconfig()
usetunnel = self.repos.getpreauthtunnel()
if not usetunnel:
host = self.repos.gethost()
user = self.repos.getuser()
port = self.repos.getport()
ssl = self.repos.getssl()
reference = self.repos.getreference()
server = None
password = None
if accountname in passwordhash:
password = passwordhash[accountname]
if repository.getname() in passwordhash:
password = passwordhash[repository.getname()]
# Connect to the remote server.
if usetunnel:
IMAPServer.__init__(self, config, accountname,
tunnel = config.get(accountname, "preauthtunnel"),
IMAPServer.__init__(self, self.config, self.repos.getname(),
tunnel = usetunnel,
reference = reference,
maxconnections = config.getint(accountname, "maxconnections"))
maxconnections = self.repos.getmaxconnections())
else:
if not password:
if config.has_option(accountname, 'remotepass'):
password = config.get(accountname, 'remotepass')
elif config.has_option(accountname, 'remotepassfile'):
passfile = open(os.path.expanduser(config.get(accountname, "remotepassfile")))
password = passfile.readline().strip()
passfile.close()
IMAPServer.__init__(self, config, accountname,
password = self.repos.getpassword()
IMAPServer.__init__(self, self.config, self.repos.getname(),
user, password, host, port, ssl,
config.getdefaultint(accountname, "maxconnections", 1),
self.repos.getmaxconnections(),
reference = reference)

View File

@ -1,5 +1,5 @@
# OfflineIMAP initialization code
# Copyright (C) 2002 John Goerzen
# Copyright (C) 2002, 2003 John Goerzen
# <jgoerzen@complete.org>
#
# This program is free software; you can redistribute it and/or modify
@ -16,13 +16,14 @@
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
from offlineimap import imaplib, imapserver, repository, folder, mbnames, threadutil, version, syncmaster
from offlineimap import imaplib, imapserver, repository, folder, mbnames, threadutil, version, syncmaster, accounts
from offlineimap.localeval import LocalEval
from offlineimap.threadutil import InstanceLimitedThread, ExitNotifyThread
from offlineimap.ui import UIBase
import re, os, os.path, offlineimap, sys, fcntl
from offlineimap.CustomConfig import CustomConfigParser
from threading import *
import threading
from getopt import getopt
lockfd = None
@ -78,18 +79,26 @@ def startup(versionno):
ui.add_debug(debugtype.strip())
if debugtype == 'imap':
imaplib.Debug = 5
if debugtype == 'thread':
threading._VERBOSE = 1
if '-o' in options:
for section in config.getaccountlist():
config.remove_option(section, "autorefresh")
# FIXME: maybe need a better
for section in accounts.getaccountlist(config):
config.remove_option('Account ' + section, "autorefresh")
lock(config, ui)
accounts = config.get("general", "accounts")
activeaccounts = config.get("general", "accounts")
if '-a' in options:
accounts = options['-a']
accounts = accounts.replace(" ", "")
accounts = accounts.split(",")
activeaccounts = options['-a']
activeaccounts = activeaccounts.replace(" ", "")
activeaccounts = activeaccounts.split(",")
allaccounts = accounts.AccountHashGenerator(config)
syncaccounts = {}
for account in activeaccounts:
syncaccounts[account] = allaccounts[account]
server = None
remoterepos = None
@ -101,18 +110,19 @@ def startup(versionno):
threadutil.initInstanceLimit("ACCOUNTLIMIT",
config.getdefaultint("general", "maxsyncaccounts", 1))
for account in accounts:
for instancename in ["FOLDER_" + account, "MSGCOPY_" + account]:
for reposname in config.getsectionlist('Repository'):
for instancename in ["FOLDER_" + reposname,
"MSGCOPY_" + reposname]:
if '-1' in options:
threadutil.initInstanceLimit(instancename, 1)
else:
threadutil.initInstanceLimit(instancename,
config.getdefaultint(account, "maxconnections", 1))
config.getdefaultint('Repository ' + reposname, "maxconnections", 1))
threadutil.initexitnotify()
t = ExitNotifyThread(target=syncmaster.syncitall,
name='Sync Runner',
kwargs = {'accounts': accounts,
kwargs = {'accounts': syncaccounts,
'config': config})
t.setDaemon(1)
t.start()

View File

@ -1,5 +1,5 @@
# Base repository support
# Copyright (C) 2002 John Goerzen
# Copyright (C) 2002, 2003 John Goerzen
# <jgoerzen@complete.org>
#
# This program is free software; you can redistribute it and/or modify
@ -16,7 +16,71 @@
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
class BaseRepository:
from offlineimap import CustomConfig
import os.path
def LoadRepository(name, account, reqtype):
from offlineimap.repository.IMAP import IMAPRepository, MappedIMAPRepository
from offlineimap.repository.Maildir import MaildirRepository
if reqtype == 'remote':
# For now, we don't support Maildirs on the remote side.
typemap = {'IMAP': IMAPRepository}
elif reqtype == 'local':
typemap = {'IMAP': MappedIMAPRepository,
'Maildir': MaildirRepository}
else:
raise ValueError, "Request type %s not supported" % reqtype
config = account.getconfig()
repostype = config.get('Repository ' + name, 'type').strip()
return typemap[repostype](name, account)
class BaseRepository(CustomConfig.ConfigHelperMixin):
def __init__(self, reposname, account):
self.account = account
self.config = account.getconfig()
self.name = reposname
self.localeval = account.getlocaleval()
self.accountname = self.account.getname()
self.uiddir = os.path.join(self.config.getmetadatadir(), 'Repository-' + self.name)
if not os.path.exists(self.uiddir):
os.mkdir(self.uiddir, 0700)
self.mapdir = os.path.join(self.uiddir, 'UIDMapping')
if not os.path.exists(self.mapdir):
os.mkdir(self.mapdir, 0700)
self.uiddir = os.path.join(self.uiddir, 'FolderValidity')
if not os.path.exists(self.uiddir):
os.mkdir(self.uiddir, 0700)
def holdordropconnections(self):
pass
def dropconnections(self):
pass
def getaccount(self):
return self.account
def getname(self):
return self.name
def getuiddir(self):
return self.uiddir
def getmapdir(self):
return self.mapdir
def getaccountname(self):
return self.accountname
def getsection(self):
return 'Repository ' + self.name
def getconfig(self):
return self.config
def getlocaleval(self):
return self.account.getlocaleval()
def getfolders(self):
"""Returns a list of ALL folders on this server."""
return []
@ -68,3 +132,14 @@ class BaseRepository:
# if not key in srchash:
# dest.deletefolder(key)
##### Keepalive
def startkeepalive(self):
"""The default implementation will do nothing."""
pass
def stopkeepalive(self, abrupt = 0):
"""Stop keep alive. If abrupt is 1, stop it but don't bother waiting
for the threads to terminate."""
pass

View File

@ -17,38 +17,116 @@
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
from Base import BaseRepository
from offlineimap import folder, imaputil
from offlineimap import folder, imaputil, imapserver
from offlineimap.folder.UIDMaps import MappedIMAPFolder
from offlineimap.threadutil import ExitNotifyThread
import re, types
from threading import *
class IMAPRepository(BaseRepository):
def __init__(self, config, localeval, accountname, imapserver):
"""Initialize an IMAPRepository object. Takes an IMAPServer
object."""
self.imapserver = imapserver
self.config = config
self.accountname = accountname
def __init__(self, reposname, account):
"""Initialize an IMAPRepository object."""
BaseRepository.__init__(self, reposname, account)
self.imapserver = imapserver.ConfigedIMAPServer(self)
self.folders = None
self.nametrans = lambda foldername: foldername
self.folderfilter = lambda foldername: 1
self.folderincludes = []
self.foldersort = cmp
if config.has_option(accountname, 'nametrans'):
self.nametrans = localeval.eval(config.get(accountname, 'nametrans'), {'re': re})
if config.has_option(accountname, 'folderfilter'):
self.folderfilter = localeval.eval(config.get(accountname, 'folderfilter'), {'re': re})
if config.has_option(accountname, 'folderincludes'):
self.folderincludes = localeval.eval(config.get(accountname, 'folderincludes'), {'re': re})
if config.has_option(accountname, 'foldersort'):
self.foldersort = localeval.eval(config.get(accountname, 'foldersort'), {'re': re})
localeval = self.localeval
if self.config.has_option(self.getsection(), 'nametrans'):
self.nametrans = localeval.eval(self.getconf('nametrans'),
{'re': re})
if self.config.has_option(self.getsection(), 'folderfilter'):
self.folderfilter = localeval.eval(self.getconf('folderfilter'),
{'re': re})
if self.config.has_option(self.getsection(), 'folderincludes'):
self.folderincludes = localeval.eval(self.getconf('folderincludes'),
{'re': re})
if self.config.has_option(self.getsection(), 'foldersort'):
self.foldersort = localeval.eval(self.getconf('foldersort'),
{'re': re})
def startkeepalive(self):
keepalivetime = self.getkeepalive()
if not keepalivetime: return
self.kaevent = Event()
self.kathread = ExitNotifyThread(target = self.imapserver.keepalive,
name = "Keep alive " + self.getname(),
args = (keepalivetime, self.kaevent))
self.kathread.setDaemon(1)
self.kathread.start()
def stopkeepalive(self, abrupt = 0):
if not hasattr(self, 'kaevent'):
# Keepalive is not active.
return
self.kaevent.set()
if not abrupt:
self.kathread.join()
del self.kathread
del self.kaevent
def holdordropconnections(self):
if not self.getholdconnectionopen():
self.dropconnections()
def dropconnections(self):
self.imapserver.close()
def getholdconnectionopen(self):
return self.getconfboolean("holdconnectionopen", 0)
def getkeepalive(self):
return self.getconfint("keepalive", 0)
def getsep(self):
return self.imapserver.delim
def gethost(self):
return self.getconf('remotehost')
def getuser(self):
return self.getconf('remoteuser')
def getport(self):
return self.getconfint('remoteport', None)
def getssl(self):
return self.getconfboolean('ssl', 0)
def getpreauthtunnel(self):
return self.getconf('preauthtunnel', None)
def getreference(self):
return self.getconf('reference', '""')
def getmaxconnections(self):
return self.getconfint('maxconnections', 1)
def getexpunge(self):
return self.getconfboolean('expunge', 1)
def getpassword(self):
password = self.getconf('remotepass', None)
if password != None:
return password
passfile = self.getconf('remotepassfile', None)
if passfile != None:
fd = open(os.path.expanduser(passfile))
password = passfile.readline().strip()
passfile.close()
return password
return None
def getfolder(self, foldername):
return folder.IMAP.IMAPFolder(self.imapserver, foldername,
self.nametrans(foldername),
accountname, self)
return self.getfoldertype()(self.imapserver, foldername,
self.nametrans(foldername),
self.accountname, self)
def getfoldertype(self):
return folder.IMAP.IMAPFolder
def getfolders(self):
if self.folders != None:
@ -60,7 +138,8 @@ class IMAPRepository(BaseRepository):
finally:
self.imapserver.releaseconnection(imapobj)
for string in listresult:
if type(string) == types.StringType and string == '':
if string == None or \
(type(string) == types.StringType and string == ''):
# Bug in imaplib: empty strings in results from
# literals.
continue
@ -71,13 +150,31 @@ class IMAPRepository(BaseRepository):
foldername = imaputil.dequote(name)
if not self.folderfilter(foldername):
continue
retval.append(folder.IMAP.IMAPFolder(self.imapserver, foldername,
self.nametrans(foldername),
self.accountname, self))
retval.append(self.getfoldertype()(self.imapserver, foldername,
self.nametrans(foldername),
self.accountname, self))
for foldername in self.folderincludes:
retval.append(folder.IMAP.IMAPFolder(self.imapserver, foldername,
self.nametrans(foldername),
self.accountname, self))
retval.append(self.getfoldertype()(self.imapserver, foldername,
self.nametrans(foldername),
self.accountname, self))
retval.sort(lambda x, y: self.foldersort(x.getvisiblename(), y.getvisiblename()))
self.folders = retval
return retval
def makefolder(self, foldername):
#if self.getreference() != '""':
# newname = self.getreference() + self.getsep() + foldername
#else:
# newname = foldername
newname = foldername
imapobj = self.imapserver.acquireconnection()
try:
result = imapobj.create(newname)
if result[0] != 'OK':
raise RuntimeError, "Repository %s could not create folder %s: %s" % (self.getname(), foldername, str(result))
finally:
self.imapserver.releaseconnection(imapobj)
class MappedIMAPRepository(IMAPRepository):
def getfoldertype(self):
return MappedIMAPFolder

View File

@ -18,18 +18,22 @@
from Base import BaseRepository
from offlineimap import folder
import os
import os, re
class LocalStatusRepository(BaseRepository):
def __init__(self, directory, accountname):
self.directory = directory
def __init__(self, reposname, account):
BaseRepository.__init__(self, reposname, account)
self.directory = os.path.join(account.getaccountmeta(), 'LocalStatus')
if not os.path.exists(self.directory):
os.mkdir(self.directory, 0700)
self.folders = None
self.accountname = accountname
def getsep(self):
return '.'
def getfolderfilename(self, foldername):
foldername = re.sub('/\.$', '/dot', foldername)
foldername = re.sub('^\.$', 'dot', foldername)
return os.path.join(self.directory, foldername)
def makefolder(self, foldername):

View File

@ -23,25 +23,24 @@ from mailbox import Maildir
import os
class MaildirRepository(BaseRepository):
def __init__(self, root, accountname, config):
def __init__(self, reposname, account):
"""Initialize a MaildirRepository object. Takes a path name
to the directory holding all the Maildir directories."""
BaseRepository.__init__(self, reposname, account)
self.root = root
self.root = self.getlocalroot()
self.folders = None
self.accountname = accountname
self.config = config
self.ui = UIBase.getglobalui()
self.debug("MaildirRepository initialized, sep is " + repr(self.getsep()))
def getlocalroot(self):
return os.path.expanduser(self.getconf('localfolders'))
def debug(self, msg):
self.ui.debug('maildir', msg)
def getsep(self):
if self.config.has_option(self.accountname, 'sep'):
return self.config.get(self.accountname, 'sep').strip()
else:
return '.'
return self.getconf('sep', '.').strip()
def makefolder(self, foldername):
self.debug("makefolder called with arg " + repr(foldername))
@ -65,7 +64,8 @@ class MaildirRepository(BaseRepository):
# makedirs will fail because the higher-up dir already exists.
# So, check to see if this is indeed the case.
if self.getsep() == '/' and os.path.isdir(foldername):
if (self.getsep() == '/' or self.getconfboolean('existsok', 0)) \
and os.path.isdir(foldername):
self.debug("makefolder: %s already is a directory" % foldername)
# Already exists. Sanity-check that it's not a Maildir.
for subdir in ['cur', 'new', 'tmp']:

View File

@ -1,5 +1,5 @@
# OfflineIMAP synchronization master code
# Copyright (C) 2002 John Goerzen
# Copyright (C) 2002, 2003 John Goerzen
# <jgoerzen@complete.org>
#
# This program is free software; you can redistribute it and/or modify
@ -25,21 +25,16 @@ import re, os, os.path, offlineimap, sys
from ConfigParser import ConfigParser
from threading import *
def syncaccount(threads, config, accountname):
def syncaccount(config, accountname):
account = SyncableAccount(config, accountname)
thread = InstanceLimitedThread(instancename = 'ACCOUNTLIMIT',
target = account.syncrunner,
name = "Account sync %s" % accountname)
thread.setDaemon(1)
thread.start()
threads.add(thread)
def syncitall(accounts, config):
currentThread().setExitMessage('SYNC_WITH_TIMER_TERMINATE')
ui = UIBase.getglobalui()
threads = threadutil.threadlist()
mbnames.init(config, accounts)
for accountname in accounts:
syncaccount(threads, config, accountname)
# Wait for the threads to finish.
threads.reset()
syncaccount(config, accountname)

View File

@ -1,4 +1,4 @@
# Copyright (C) 2002 John Goerzen
# Copyright (C) 2002, 2003 John Goerzen
# Thread support module
# <jgoerzen@complete.org>
#
@ -113,12 +113,14 @@ def exitnotifymonitorloop(callback):
global exitcondition, exitthreads
while 1: # Loop forever.
exitcondition.acquire()
while not len(exitthreads):
exitcondition.wait(1)
try:
while not len(exitthreads):
exitcondition.wait(1)
while len(exitthreads):
callback(exitthreads.pop(0)) # Pull off in order added!
exitcondition.release()
while len(exitthreads):
callback(exitthreads.pop(0)) # Pull off in order added!
finally:
exitcondition.release()
def threadexited(thread):
"""Called when a thread exits."""
@ -132,11 +134,6 @@ def threadexited(thread):
ui.threadException(thread) # Expected to terminate
sys.exit(100) # Just in case...
os._exit(100)
elif thread.getExitMessage() == 'SYNC_WITH_TIMER_TERMINATE':
ui.terminate()
# Just in case...
sys.exit(100)
os._exit(100)
else:
ui.threadExited(thread)

View File

@ -155,7 +155,7 @@ class UIBase:
if hostname == None:
hostname = ''
if port != None:
port = ":%d" % port
port = ":%s" % str(port)
displaystr = ' to %s%s.' % (hostname, port)
if hostname == '' and port == None:
displaystr = '.'
@ -182,9 +182,9 @@ class UIBase:
s.getnicename(srcrepos),
s.getnicename(destrepos)))
def validityproblem(s, folder):
s.warn("UID validity problem for folder %s; skipping it" % \
folder.getname())
def validityproblem(s, folder, saved, new):
s.warn("UID validity problem for folder %s (saved %d; got %d); skipping it" % \
(folder.getname(), saved, new))
def loadmessagelist(s, repos, folder):
if s.verbose > 0:

View File

@ -1,8 +1,8 @@
productname = 'OfflineIMAP'
versionstr = "3.99.10"
revno = long('$Rev: 367 $'[6:-2])
versionstr = "3.99.12"
revno = long('$Rev: 439 $'[6:-2])
revstr = "Rev %d" % revno
datestr = '$Date: 2003-04-16 09:23:45 -0500 (Wed, 16 Apr 2003) $'
datestr = '$Date: 2003-04-17 16:16:00 -0500 (Thu, 17 Apr 2003) $'
versionlist = versionstr.split(".")
major = versionlist[0]