Add IDLE support from James Bunton

Merge branch 'bunton'

offlineimap.conf

@@ -307,6 +307,19 @@ remoteuser = username
 #
 # reference = Mail
 
+# In between synchronisations, OfflineIMAP can monitor mailboxes for new
+# messages using the IDLE command. If you want to enable this, specify here
+# the folders you wish to monitor. Note that the IMAP protocol requires a
+# separate connection for each folder monitored in this way, so setting
+# this option will force settings for:
+#     maxconnections - to be at least the number of folders you give
+#     holdconnectionopen - to be true
+#     keepalive - to be 29 minutes unless you specify otherwise
+# This option should return a Python list. For example
+#
+# idlefolders = ['INBOX', 'INBOX.Alerts']
+#
+
 # OfflineIMAP can use multiple connections to the server in order
 # to perform multiple synchronization actions simultaneously.
 # This may place a higher burden on the server.  In most cases,

offlineimap/folder/Gmail.py

@@ -20,8 +20,7 @@
 """
 
 from IMAP import IMAPFolder
-import imaplib
+from offlineimap import imaplib2, imaputil, imaplibutil
-from offlineimap import imaputil, imaplibutil
 from offlineimap.ui import UIBase
 from copy import copy
 

offlineimap/folder/IMAP.py

@@ -17,8 +17,7 @@
 #    Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
 
 from Base import BaseFolder
-import imaplib
+from offlineimap import imaplib2, imaputil, imaplibutil
-from offlineimap import imaputil, imaplibutil
 from offlineimap.ui import UIBase
 from offlineimap.version import versionstr
 import rfc822, time, string, random, binascii, re
@@ -271,13 +270,13 @@ class IMAPFolder(BaseFolder):
                     raise ValueError
 
                 # This could raise a value error if it's not a valid format.
-                date = imaplib.Time2Internaldate(datetuple) 
+                date = imaplib2.Time2Internaldate(datetuple) 
             except (ValueError, OverflowError):
                 # Argh, sometimes it's a valid format but year is 0102
                 # or something.  Argh.  It seems that Time2Internaldate
                 # will rause a ValueError if the year is 0102 but not 1902,
                 # but some IMAP servers nonetheless choke on 1902.
-                date = imaplib.Time2Internaldate(time.localtime())
+                date = imaplib2.Time2Internaldate(time.localtime())
 
             ui.debug('imap', 'savemessage: using date ' + str(date))
             content = re.sub("(?<!\r)\n", "\r\n", content)

offlineimap/imaplibutil.py

@@ -16,12 +16,12 @@
 #    along with this program; if not, write to the Free Software
 #    Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
 
-import re, string, types, binascii, socket, time, random, subprocess, sys, os
+import os, re, socket, time, subprocess, sys, threading
 from offlineimap.ui import UIBase
-from imaplib import *
+from offlineimap.imaplib2 import *
 
 # Import the symbols we need that aren't exported by default
-from imaplib import IMAP4_PORT, IMAP4_SSL_PORT, InternalDate, Mon2num
+from offlineimap.imaplib2 import IMAP4_PORT, IMAP4_SSL_PORT, InternalDate, Mon2num
 
 
 class IMAP4_Tunnel(IMAP4):
@@ -40,12 +40,10 @@ class IMAP4_Tunnel(IMAP4):
         self.process = subprocess.Popen(host, shell=True, close_fds=True,
                         stdin=subprocess.PIPE, stdout=subprocess.PIPE)
         (self.outfd, self.infd) = (self.process.stdin, self.process.stdout)
+        self.read_fd = self.infd.fileno()
 
     def read(self, size):
-        retval = ''
+        return os.read(self.read_fd, size)
-        while len(retval) < size:
-            retval += self.infd.read(size - len(retval))
-        return retval
 
     def readline(self):
         return self.infd.readline()
@@ -97,78 +95,22 @@ class sslwrapper:
             else:
                 retval += linebuf
 
-def new_mesg(self, s, secs=None):
+def new_mesg(self, s, tn=None, secs=None):
             if secs is None:
                 secs = time.time()
+            if tn is None:
+                tn = threading.currentThread().getName()
             tm = time.strftime('%M:%S', time.localtime(secs))
-            UIBase.getglobalui().debug('imap', '  %s.%02d %s' % (tm, (secs*100)%100, s))
+            UIBase.getglobalui().debug('imap', '  %s.%02d %s %s' % (tm, (secs*100)%100, tn, s))
 
 class WrappedIMAP4_SSL(IMAP4_SSL):
-    def open(self, host = '', port = IMAP4_SSL_PORT):
+    def open(self, host=None, port=None):
         IMAP4_SSL.open(self, host, port)
         self.sslobj = sslwrapper(self.sslobj)
 
     def readline(self):
         return self.sslobj.readline()
 
-def new_open(self, host = '', port = IMAP4_PORT):
-        """Setup connection to remote server on "host:port"
-            (default: localhost:standard IMAP4 port).
-        This connection will be used by the routines:
-            read, readline, send, shutdown.
-        """
-        self.host = host
-        self.port = port
-        res = socket.getaddrinfo(host, port, socket.AF_UNSPEC,
-                                 socket.SOCK_STREAM)
-
-        # Try each address returned by getaddrinfo in turn until we
-        # manage to connect to one.
-        # Try all the addresses in turn until we connect()
-        last_error = 0
-        for remote in res:
-            af, socktype, proto, canonname, sa = remote
-            self.sock = socket.socket(af, socktype, proto)
-            last_error = self.sock.connect_ex(sa)
-            if last_error == 0:
-                break
-            else:
-                self.sock.close()
-        if last_error != 0:
-            # FIXME
-            raise socket.error(last_error)
-        self.file = self.sock.makefile('rb')
-
-def new_open_ssl(self, host = '', port = IMAP4_SSL_PORT):
-        """Setup connection to remote server on "host:port".
-            (default: localhost:standard IMAP4 SSL port).
-        This connection will be used by the routines:
-            read, readline, send, shutdown.
-        """
-        self.host = host
-        self.port = port
-        #This connects to the first ip found ipv4/ipv6
-        #Added by Adriaan Peeters <apeeters@lashout.net> based on a socket
-        #example from the python documentation:
-        #http://www.python.org/doc/lib/socket-example.html
-        res = socket.getaddrinfo(host, port, socket.AF_UNSPEC,
-                                 socket.SOCK_STREAM)
-        # Try all the addresses in turn until we connect()
-        last_error = 0
-        for remote in res:
-            af, socktype, proto, canonname, sa = remote
-            self.sock = socket.socket(af, socktype, proto)
-            last_error = self.sock.connect_ex(sa)
-            if last_error == 0:
-                break
-            else:
-                self.sock.close()
-        if last_error != 0:
-            # FIXME
-            raise socket.error(last_error)
-        self.sslobj = socket.ssl(self.sock, self.keyfile, self.certfile)
-        self.sslobj = sslwrapper(self.sslobj)
-
 mustquote = re.compile(r"[^\w!#$%&'+,.:;<=>?^`|~-]")
 
 def Internaldate2epoch(resp):

offlineimap/imapserver.py

@@ -16,9 +16,9 @@
 #    along with this program; if not, write to the Free Software
 #    Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
 
-import imaplib
+from offlineimap import imaplib2, imaplibutil, imaputil, threadutil
-from offlineimap import imaplibutil, imaputil, threadutil
 from offlineimap.ui import UIBase
+from offlineimap.accounts import syncfolder
 from threading import *
 import thread, hmac, os, time
 import base64
@@ -56,13 +56,10 @@ class UsefulIMAPMixIn:
         else:
             self.selectedfolder = None
 
-    def _mesg(self, s, secs=None):
+    def _mesg(self, s, tn=None, secs=None):
-        imaplibutil.new_mesg(self, s, secs)
+        imaplibutil.new_mesg(self, s, tn, secs)
-
-class UsefulIMAP4(UsefulIMAPMixIn, imaplib.IMAP4):
-    def open(self, host = '', port = imaplib.IMAP4_PORT):
-        imaplibutil.new_open(self, host, port)
 
+class UsefulIMAP4(UsefulIMAPMixIn, imaplib2.IMAP4):
     # This is a hack around Darwin's implementation of realloc() (which
     # Python uses inside the socket code). On Darwin, we split the
     # message into 100k chunks, which should be small enough - smaller
@@ -73,17 +70,17 @@ class UsefulIMAP4(UsefulIMAPMixIn, imaplib.IMAP4):
             read = 0
             io = StringIO()
             while read < size:
-                data = imaplib.IMAP4.read (self, min(size-read,8192))
+                sz = min(size-read, 8192)
+                data = imaplib2.IMAP4.read (self, sz)
                 read += len(data)
                 io.write(data)
+                if len(data) < sz:
+                    break
             return io.getvalue()
         else:
-            return imaplib.IMAP4.read (self, size)
+            return imaplib2.IMAP4.read (self, size)
 
 class UsefulIMAP4_SSL(UsefulIMAPMixIn, imaplibutil.WrappedIMAP4_SSL):
-    def open(self, host = '', port = imaplib.IMAP4_SSL_PORT):
-        imaplibutil.new_open_ssl(self, host, port)
-
     # This is the same hack as above, to be used in the case of an SSL
     # connexion.
 
@@ -92,9 +89,12 @@ class UsefulIMAP4_SSL(UsefulIMAPMixIn, imaplibutil.WrappedIMAP4_SSL):
             read = 0
             io = StringIO()
             while read < size:
-                data = imaplibutil.WrappedIMAP4_SSL.read (self, min(size-read,8192))
+                sz = min(size-read,8192)
+                data = imaplibutil.WrappedIMAP4_SSL.read (self, sz)
                 read += len(data)
                 io.write(data)
+                if len(data) < sz:
+                    break
             return io.getvalue()
         else:
             return imaplibutil.WrappedIMAP4_SSL.read (self,size)
@@ -107,7 +107,8 @@ class IMAPServer:
     def __init__(self, config, reposname,
                  username = None, password = None, hostname = None,
                  port = None, ssl = 1, maxconnections = 1, tunnel = None,
-                 reference = '""', sslclientcert = None, sslclientkey = None):
+                 reference = '""', sslclientcert = None, sslclientkey = None,
+                 idlefolders = []):
         self.reposname = reposname
         self.config = config
         self.username = username
@@ -134,6 +135,7 @@ class IMAPServer:
         self.semaphore = BoundedSemaphore(self.maxconnections)
         self.connectionlock = Lock()
         self.reference = reference
+        self.idlefolders = idlefolders
         self.gss_step = self.GSS_STATE_STEP
         self.gss_vc = None
         self.gssapi = False
@@ -344,8 +346,6 @@ class IMAPServer:
         ui.debug('imap', 'keepalive thread started')
         while 1:
             ui.debug('imap', 'keepalive: top of loop')
-            time.sleep(timeout)
-            ui.debug('imap', 'keepalive: after wait')
             if event.isSet():
                 ui.debug('imap', 'keepalive: event is set; exiting')
                 return
@@ -356,32 +356,91 @@ class IMAPServer:
             self.connectionlock.release()
             ui.debug('imap', 'keepalive: connectionlock released')
             threads = []
-            imapobjs = []
         
             for i in range(numconnections):
                 ui.debug('imap', 'keepalive: processing connection %d of %d' % (i, numconnections))
-                imapobj = self.acquireconnection()
+                if len(self.idlefolders) > i:
-                ui.debug('imap', 'keepalive: connection %d acquired' % i)
+                    idler = IdleThread(self, self.idlefolders[i])
-                imapobjs.append(imapobj)
+                else:
-                thr = threadutil.ExitNotifyThread(target = imapobj.noop)
+                    idler = IdleThread(self)
-                thr.setDaemon(1)
+                idler.start()
-                thr.start()
+                threads.append(idler)
-                threads.append(thr)
                 ui.debug('imap', 'keepalive: thread started')
 
+            ui.debug('imap', 'keepalive: waiting for timeout')
+            event.wait(timeout)
+
             ui.debug('imap', 'keepalive: joining threads')
 
-            for thr in threads:
+            for idler in threads:
                 # Make sure all the commands have completed.
-                thr.join()
+                idler.stop()
-
+                idler.join()
-            ui.debug('imap', 'keepalive: releasing connections')
-
-            for imapobj in imapobjs:
-                self.releaseconnection(imapobj)
 
             ui.debug('imap', 'keepalive: bottom of loop')
 
+class IdleThread(object):
+    def __init__(self, parent, folder=None):
+        self.parent = parent
+        self.folder = folder
+        self.event = Event()
+        if folder is None:
+            self.thread = Thread(target=self.noop)
+        else:
+            self.thread = Thread(target=self.idle)
+        self.thread.setDaemon(1)
+
+    def start(self):
+        self.thread.start()
+
+    def stop(self):
+        self.event.set()
+
+    def join(self):
+        self.thread.join()
+
+    def noop(self):
+        imapobj = self.parent.acquireconnection()
+        imapobj.noop()
+        self.event.wait()
+        self.parent.releaseconnection(imapobj)
+
+    def dosync(self):
+        remoterepos = self.parent.repos
+        account = remoterepos.account
+        localrepos = account.localrepos
+        remoterepos = account.remoterepos
+        statusrepos = account.statusrepos
+        remotefolder = remoterepos.getfolder(self.folder)
+        syncfolder(account.name, remoterepos, remotefolder, localrepos, statusrepos, quick=False)
+        ui = UIBase.getglobalui()
+        ui.unregisterthread(currentThread())
+
+    def idle(self):
+        imapobj = self.parent.acquireconnection()
+        imapobj.select(self.folder)
+        self.parent.releaseconnection(imapobj)
+        while True:
+            if self.event.isSet():
+                return
+            self.needsync = False
+            def callback(args):
+                if not self.event.isSet():
+                    self.needsync = True
+                    self.event.set()
+            imapobj = self.parent.acquireconnection()
+            if "IDLE" in imapobj.capabilities:
+                imapobj.idle(callback=callback)
+            else:
+                imapobj.noop()
+            self.event.wait()
+            if self.event.isSet():
+                imapobj.noop()
+            self.parent.releaseconnection(imapobj)
+            if self.needsync:
+                self.event.clear()
+                self.dosync()
+
 class ConfigedIMAPServer(IMAPServer):
     """This class is designed for easier initialization given a ConfigParser
     object and an account name.  The passwordhash is used if
@@ -401,6 +460,7 @@ class ConfigedIMAPServer(IMAPServer):
             sslclientcert = self.repos.getsslclientcert()
             sslclientkey = self.repos.getsslclientkey()
         reference = self.repos.getreference()
+        idlefolders = self.repos.getidlefolders()
         server = None
         password = None
         
@@ -412,6 +472,7 @@ class ConfigedIMAPServer(IMAPServer):
             IMAPServer.__init__(self, self.config, self.repos.getname(),
                                 tunnel = usetunnel,
                                 reference = reference,
+                                idlefolders = idlefolders,
                                 maxconnections = self.repos.getmaxconnections())
         else:
             if not password:
@@ -420,5 +481,6 @@ class ConfigedIMAPServer(IMAPServer):
                                 user, password, host, port, ssl,
                                 self.repos.getmaxconnections(),
                                 reference = reference,
+                                idlefolders = idlefolders,
                                 sslclientcert = sslclientcert,
                                 sslclientkey = sslclientkey)

offlineimap/init.py

@@ -16,8 +16,7 @@
 #    along with this program; if not, write to the Free Software
 #    Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
 
-import imaplib
+from offlineimap import imaplib2, imapserver, repository, folder, mbnames, threadutil, version, syncmaster, accounts
-from offlineimap import imapserver, repository, folder, mbnames, threadutil, version, syncmaster, accounts
 from offlineimap.localeval import LocalEval
 from offlineimap.threadutil import InstanceLimitedThread, ExitNotifyThread
 from offlineimap.ui import UIBase
@@ -103,7 +102,7 @@ def startup(versionno):
         for debugtype in options['-d'].split(','):
             ui.add_debug(debugtype.strip())
             if debugtype == 'imap':
-                imaplib.Debug = 5
+                imaplib2.Debug = 5
             if debugtype == 'thread':
                 threading._VERBOSE = 1
 

offlineimap/repository/IMAP.py

@@ -74,10 +74,16 @@ class IMAPRepository(BaseRepository):
         self.imapserver.close()
 
     def getholdconnectionopen(self):
+        if self.getidlefolders():
+            return 1
         return self.getconfboolean("holdconnectionopen", 0)
 
     def getkeepalive(self):
-        return self.getconfint("keepalive", 0)
+        num = self.getconfint("keepalive", 0)
+        if num == 0 and self.getidlefolders():
+            return 29*60
+        else:
+            return num
 
     def getsep(self):
         return self.imapserver.delim
@@ -145,8 +151,14 @@ class IMAPRepository(BaseRepository):
     def getreference(self):
         return self.getconf('reference', '""')
 
+    def getidlefolders(self):
+        localeval = self.localeval
+        return localeval.eval(self.getconf('idlefolders', '[]'))
+
     def getmaxconnections(self):
-        return self.getconfint('maxconnections', 1)
+        num1 = len(self.getidlefolders())
+        num2 = self.getconfint('maxconnections', 1)
+        return max(num1, num2)
 
     def getexpunge(self):
         return self.getconfboolean('expunge', 1)

offlineimap/syncmaster.py

@@ -16,8 +16,7 @@
 #    along with this program; if not, write to the Free Software
 #    Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
 
-import imaplib
+from offlineimap import imaplib2, imapserver, repository, folder, mbnames, threadutil, version
-from offlineimap import imapserver, repository, folder, mbnames, threadutil, version
 from offlineimap.threadutil import InstanceLimitedThread, ExitNotifyThread
 import offlineimap.accounts
 from offlineimap.accounts import SyncableAccount, SigListener