Make postponing SQLite transaction committing possible.

This should significantly improve performance when used to write large
amounts of messages.

This addresses #390 (although it doesn't necessarily fix all instances
of that problem yet).

offlineimap/folder/Base.py

@@ -103,6 +103,16 @@ class BaseFolder(object):
         # fails if the str is utf-8
         return self.name.decode('utf-8')
 
+    def __enter__(self):
+        """Starts a transaction. This will postpone (guaranteed) saving to disk
+        of all messages saved inside this transaction until its committed."""
+        pass
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        """Commits a transaction, all messages saved inside this transaction
+        will only now be persisted to disk."""
+        pass
+
     @property
     def accountname(self):
         """Account name as string"""
@@ -891,38 +901,39 @@ class BaseFolder(object):
             )
             return
 
-        for num, uid in enumerate(copylist):
-            # Bail out on CTRL-C or SIGTERM.
-            if offlineimap.accounts.Account.abort_NOW_signal.is_set():
-                break
+        with self:
+            for num, uid in enumerate(copylist):
+                # Bail out on CTRL-C or SIGTERM.
+                if offlineimap.accounts.Account.abort_NOW_signal.is_set():
+                    break
 
-            if uid == 0:
-                self.ui.warn("Assertion that UID != 0 failed; ignoring message.")
-                continue
+                if uid == 0:
+                    self.ui.warn("Assertion that UID != 0 failed; ignoring message.")
+                    continue
 
-            if uid > 0 and dstfolder.uidexists(uid):
-                # dstfolder has message with that UID already, only update status.
-                flags = self.getmessageflags(uid)
-                rtime = self.getmessagetime(uid)
-                statusfolder.savemessage(uid, None, flags, rtime)
-                continue
+                if uid > 0 and dstfolder.uidexists(uid):
+                    # dstfolder has message with that UID already, only update status.
+                    flags = self.getmessageflags(uid)
+                    rtime = self.getmessagetime(uid)
+                    statusfolder.savemessage(uid, None, flags, rtime)
+                    continue
 
-            self.ui.copyingmessage(uid, num+1, num_to_copy, self, dstfolder)
-            # Exceptions are caught in copymessageto().
-            if self.suggeststhreads():
-                self.waitforthread()
-                thread = threadutil.InstanceLimitedThread(
-                    self.getinstancelimitnamespace(),
-                    target=self.copymessageto,
-                    name="Copy message from %s:%s"% (self.repository, self),
-                    args=(uid, dstfolder, statusfolder)
-                )
-                thread.start()
-                threads.append(thread)
-            else:
-                self.copymessageto(uid, dstfolder, statusfolder, register=0)
-        for thread in threads:
-            thread.join() # Block until all "copy" threads are done.
+                self.ui.copyingmessage(uid, num+1, num_to_copy, self, dstfolder)
+                # Exceptions are caught in copymessageto().
+                if self.suggeststhreads():
+                    self.waitforthread()
+                    thread = threadutil.InstanceLimitedThread(
+                        self.getinstancelimitnamespace(),
+                        target=self.copymessageto,
+                        name="Copy message from %s:%s"% (self.repository, self),
+                        args=(uid, dstfolder, statusfolder)
+                    )
+                    thread.start()
+                    threads.append(thread)
+                else:
+                    self.copymessageto(uid, dstfolder, statusfolder, register=0)
+            for thread in threads:
+                thread.join() # Block until all "copy" threads are done.
 
         # Execute new mail hook if we have new mail.
         if self.have_newmail:

offlineimap/folder/LocalStatusSQLite.py

@@ -92,6 +92,19 @@ class LocalStatusSQLiteFolder(BaseFolder):
             LocalStatusSQLiteFolder.locks[self.filename] = DatabaseFileLock()
         self._databaseFileLock = LocalStatusSQLiteFolder.locks[self.filename]
 
+        self._in_transactions = 0
+
+    def __enter__(self):
+        assert self.connection is not None
+        self._in_transactions += 1
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        assert self._in_transactions > 0
+
+        self._in_transactions -= 1
+        if not self._in_transactions:
+            self.connection.commit()
+
     def openfiles(self):
         # Make sure sqlite is in multithreading SERIALIZE mode.
         assert sqlite.threadsafety == 1, 'Your sqlite is not multithreading safe.'
@@ -169,7 +182,8 @@ class LocalStatusSQLiteFolder(BaseFolder):
                         else:
                             self.connection.execute(sql, args)
                     success = True
-                    self.connection.commit()
+                    if not self._in_transactions:
+                        self.connection.commit()
             except sqlite.OperationalError as e:
                 if e.args[0] == 'cannot commit - no transaction is active':
                     pass