Improvements to the SQlite-based local status folder

* Do not inherit LocalStatusSQLiteFolder class from the plaintext
    one.

  * Use some functions already in BaseFolder in both, plaintext and
    sqlite classes.

  * Add a saveall method. The idea is that saveall dumps the entire
    messagelist to disk, while save only commits the uncommited
    changes. Right now, save is noop for sqlite, and equivalent to
    saveall for plaintext, but it enables to be more clever on when we
    commit to disk in the future.

Signed-off-by: Eygene Ryabinkin <rea@codelabs.ru>

offlineimap/folder/Base.py

@@ -36,6 +36,7 @@ class BaseFolder(object):
         # Save original name for folderfilter operations
         self.ffilter_name = name
         # Top level dir name is always ''
+        self.root = None
         self.name = name if not name == self.getsep() else ''
         self.repository = repository
         self.visiblename = repository.nametrans(name)

offlineimap/folder/LocalStatus.py

@@ -29,6 +29,7 @@ class LocalStatusFolder(BaseFolder):
     def __init__(self, name, repository):
         self.sep = '.' #needs to be set before super.__init__()
         super(LocalStatusFolder, self).__init__(name, repository)
+        self.root = repository.root
         self.filename = os.path.join(self.getroot(), self.getfolderbasename())
         self.messagelist = {}
         self.savelock = threading.Lock()
@@ -47,14 +48,6 @@ class LocalStatusFolder(BaseFolder):
     def getname(self):
         return self.name
 
-    # Interface from BaseFolder
-    def getroot(self):
-        return self.repository.root
-
-    # Interface from BaseFolder
-    def getsep(self):
-        return self.sep
-
     # Interface from BaseFolder
     def getfullname(self):
         return self.filename
@@ -178,6 +171,11 @@ class LocalStatusFolder(BaseFolder):
 
 
     def save(self):
+        """Save changed data to disk. For this backend it is the same as saveall"""
+        self.saveall()
+
+    def saveall(self):
+        """Saves the entire messagelist to disk"""
         with self.savelock:
             file = open(self.filename + ".tmp", "wt")
             file.write((self.magicline % self.cur_version) + "\n")

offlineimap/folder/LocalStatusSQLite.py

@@ -17,14 +17,14 @@
 import os.path
 import re
 from threading import Lock
-from .LocalStatus import LocalStatusFolder
+from .Base import BaseFolder
 try:
     import sqlite3 as sqlite
 except:
     pass #fail only if needed later on, not on import
 
 
-class LocalStatusSQLiteFolder(LocalStatusFolder):
+class LocalStatusSQLiteFolder(BaseFolder):
     """LocalStatus backend implemented with an SQLite database
 
     As python-sqlite currently does not allow to access the same sqlite
@@ -43,9 +43,17 @@ class LocalStatusSQLiteFolder(LocalStatusFolder):
     cur_version = 2
 
     def __init__(self, name, repository):
+        self.sep = '.' #needs to be set before super.__init__()
         super(LocalStatusSQLiteFolder, self).__init__(name, repository)
+        self.root = repository.root
+        self.filename = os.path.join(self.getroot(), self.getfolderbasename())
+        self.messagelist = {}
+
+        self._newfolder = False        # flag if the folder is new
+
         # dblock protects against concurrent writes in same connection
         self._dblock = Lock()
+
         #Try to establish connection, no need for threadsafety in __init__
         try:
             self.connection = sqlite.connect(self.filename, check_same_thread = False)
@@ -62,13 +70,35 @@ class LocalStatusSQLiteFolder(LocalStatusFolder):
             cursor = self.connection.execute("SELECT value from metadata WHERE key='db_version'")
         except sqlite.DatabaseError:
             #db file missing or corrupt, recreate it.
-            self.__upgrade_db(0)
+            self.__create_db()
         else:
             # fetch db version and upgrade if needed
             version = int(cursor.fetchone()[0])
             if version < LocalStatusSQLiteFolder.cur_version:
                 self.__upgrade_db(version)
 
+
+    def storesmessages(self):
+        return False
+
+    def getname(self):
+        return self.name
+
+    def getfullname(self):
+        return self.filename
+
+
+    # Interface from LocalStatusFolder
+    def isnewfolder(self):
+        return self._newfolder
+
+
+    # Interface from LocalStatusFolder
+    def deletemessagelist(self):
+        """delete all messages in the db"""
+        self.__sql_write('DELETE FROM status')
+
+
     def __sql_write(self, sql, vars=None, executemany=False):
         """Execute some SQL, retrying if the db was locked.
 
@@ -123,6 +153,7 @@ class LocalStatusSQLiteFolder(LocalStatusFolder):
                 'LocalStatus',
                 self.getfolderbasename())
             # MIGRATE from plaintext if needed
+            # TODO: adopt for plain-text v2
             if os.path.exists(plaintextfilename):
                 self.ui._msg('Migrating LocalStatus cache from plain text '
                              'to sqlite database for %s:%s' %\
@@ -168,22 +199,12 @@ class LocalStatusSQLiteFolder(LocalStatusFolder):
                          % (self.repository, self))
         self.connection.executescript("""
         CREATE TABLE metadata (key VARCHAR(50) PRIMARY KEY, value VARCHAR(128));
-        INSERT INTO metadata VALUES('db_version', '1');
+        INSERT INTO metadata VALUES('db_version', '2');
         CREATE TABLE status (id INTEGER PRIMARY KEY, flags VARCHAR(50), mtime INTEGER, labels VARCHAR(256));
         """)
         self.connection.commit()
+        self._newfolder = True
 
-    # Interface from LocalStatusFolder
-    def isnewfolder(self):
-        # testing the existence of the db file won't work. It is created
-        # as soon as this class instance was intitiated. So say it is a
-        # new folder when there are no messages at all recorded in it.
-        return self.getmessagecount() > 0
-
-    # Interface from LocalStatusFolder
-    def deletemessagelist(self):
-        """delete all messages in the db"""
-        self.__sql_write('DELETE FROM status')
 
     # Interface from BaseFolder
     def cachemessagelist(self):
@@ -196,8 +217,21 @@ class LocalStatusSQLiteFolder(LocalStatusFolder):
 
     # Interface from LocalStatusFolder
     def save(self):
-        #Noop in this backend
         pass
+        # Noop. every transaction commits to database!
+
+    def saveall(self):
+        """Saves the entire messagelist to the database."""
+        data = []
+        for uid, msg in self.messagelist.items():
+            mtime = msg['mtime']
+            flags = ''.join(sorted(msg['flags']))
+            labels = ', '.join(sorted(msg['labels']))
+            data.append((uid, flags, mtime, labels))
+
+        self.__sql_write('INSERT OR REPLACE INTO status (id,flags,mtime,labels) VALUES (?,?,?,?)',
+                                    data, executemany=True)
+
 
     # Following some pure SQLite functions, where we chose to use
     # BaseFolder() methods instead. Doing those on the in-memory list is
@@ -235,6 +269,12 @@ class LocalStatusSQLiteFolder(LocalStatusFolder):
     #            return flags
     #        assert False,"getmessageflags() called on non-existing message"
 
+
+    # Interface from BaseFolder
+    def getmessagelist(self):
+        return self.messagelist
+
+
     # Interface from BaseFolder
     def savemessage(self, uid, content, flags, rtime, mtime=0, labels=set()):
         """