* <many>: (WIP) use ~/mu/xapian as the database with an embedded version tag

- add checks in the code to make sure the database is up to date,
    if not, warn the user.

Makefile.am

@@ -18,7 +18,7 @@ SUBDIRS=src man
 
 # calculate the cyclomatic-complexity, which should not go over 10
 cc10:
-	@pmccabe `find . -regex '.*\.\(c\|cc\|h\)$'`  | sort -nr | awk '($$1 > 10)'
+	@pmccabe `find . -regex '.*\.\(c\|cc\|h\)$'` | sort -nr | awk '($$1 > 10)'
 
 EXTRA_DIST=	\
 	TODO	\

configure.ac

@@ -14,20 +14,20 @@
 ## along with this program; if not, write to the Free Software Foundation,
 ## Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.  
 
-AC_INIT([mu],[0.6],[http://www.djcbsoftware.nl/code/mu])
+AC_INIT([mu],[0.7dev],[http://www.djcbsoftware.nl/code/mu])
 AC_CONFIG_HEADERS([config.h])
 AC_CONFIG_SRCDIR([src/mu.c])
 AM_INIT_AUTOMAKE([dist-bzip2])
 
 # we set the set the version of the Xapian database layout here; it
-# will become part of the db name, so we can automtically recreate the
+# will become part of the db name, so we can automatically recreate the
 # database when we incompatible have changes.
 #
 # note that MU_XAPIAN_DB_VERSION does not necessarily follow MU
 # versioning, as we hopefully don't have updates for each version;
 # also, this has nothing to do with the version of the Xapian library
 #
-AC_DEFINE(MU_XAPIAN_DB_VERSION,["0.6"], [Schema version of the database])
+AC_DEFINE(MU_XAPIAN_DB_VERSION,["6.99"], [Schema version of the database])
 
 AC_PROG_LIBTOOL
 
@@ -100,6 +100,10 @@ src/Makefile
 man/Makefile
 ])
 
+echo
+echo "Note: the Xapian database is no longer stored as <muhome>/xapian-0.6"
+echo "but instead simply as <muhome>/xapian. You can remove the older"
+echo "<muhome>xapian-0.6 directory to save some disk space"
 
 echo
 echo "mu configuration is complete."

src/Makefile.am

@@ -38,7 +38,6 @@ mu_SOURCES=			\
 	mu-msg-gmime.h		\
 	mu-msg-str.c		\
 	mu-msg-str.h		\
-	mu-msg-xapian-priv.hh	\
 	mu-msg-xapian.cc	\
 	mu-msg-xapian.h		\
 	mu-msg.h		\
@@ -49,6 +48,8 @@ mu_SOURCES=			\
 	mu-store-xapian.h	\
 	mu-util.h		\
 	mu-util.c		\
+	mu-util-xapian.cc	\
+	mu-util-xapian.h	\
 	mu.c
 
 mu_LDADD=			\

src/mu-cmd.c

@@ -32,6 +32,7 @@
 #include "mu-msg-str.h"
 #include "mu-cmd.h"
 #include "mu-util.h"
+#include "mu-util-xapian.h"
 
 static MuCmd 
 cmd_from_string (const char* cmd)
@@ -62,6 +63,30 @@ cmd_from_string (const char* cmd)
 	return MU_CMD_UNKNOWN;
 }
 
+static gboolean
+database_needs_reindex (MuConfigOptions *opts)
+{
+	gchar *version;
+	gboolean reindex;
+	
+	version = mu_util_xapian_db_version (opts->xpath);		
+	if (!version || strcmp (version, MU_XAPIAN_DB_VERSION) != 0) {
+		g_warning ("expected database version %s, "
+			   "but current version is %s",
+			   MU_XAPIAN_DB_VERSION,
+			   version ? version : "<none>");
+		g_message ("please run `mu index --reindex' for your full "
+			   "maildir");
+		reindex = TRUE;
+	} else
+		reindex = FALSE;
+
+	g_free (version);
+	return reindex;
+}
+
+
+
 
 static gboolean
 print_query (MuQueryXapian *xapian, const gchar *query)
@@ -131,9 +156,6 @@ sort_field_from_string (const char* fieldstr)
 	return field;
 }
 
-
-
-
 static gboolean
 print_rows (MuQueryXapian *xapian, const gchar *query, MuConfigOptions *opts)
 {
@@ -316,6 +338,9 @@ cmd_find (MuConfigOptions *opts)
 		
 	if (!query_params_valid (opts))
 		return FALSE;
+
+	if (database_needs_reindex(opts))
+		return FALSE;
 	
 	/* first param is 'query', search params are after that */
 	params = (const gchar**)&opts->params[1];
@@ -391,10 +416,14 @@ cmd_index (MuConfigOptions *opts)
 	if (!check_index_params (opts))
 		return FALSE;
 
+	if (!opts->reindex && database_needs_reindex(opts))
+		return FALSE;
+	
 	mu_msg_gmime_init ();
 	{
 		MuIndex *midx;
 		MuIndexStats stats;
+		gboolean reindex;
 		
 		mu_index_stats_clear (&stats);
 		midx = mu_index_new (opts->xpath);

src/mu-config.c

@@ -183,8 +183,9 @@ mu_config_init (MuConfigOptions *opts, int *argcp, char ***argvp)
 	/* set dirmode before, because '0000' is a valid mode */
 	opts->dirmode = 0755;
 
-	if (!parse_params (opts, argcp, argvp))
-		return FALSE;
+	if (argcp && argvp)
+		if (!parse_params (opts, argcp, argvp))
+			return FALSE;
 	
 	if (!opts->muhome)
 		opts->muhome = guess_muhome ();

src/mu-index.c

@@ -30,9 +30,11 @@
 #include "mu-index.h"
 #include "mu-store-xapian.h"
 #include "mu-util.h"
+#include "mu-util-xapian.h"
 
 struct _MuIndex {
 	MuStoreXapian  *_xapian;
+	gboolean _needs_reindex;
 };
 
 MuIndex* 
@@ -51,7 +53,11 @@ mu_index_new (const char *xpath)
 		g_free (index);
 		return NULL;
 	}
-	
+
+	/* see we need to reindex the database */
+	index->_needs_reindex = 
+		mu_util_xapian_db_version_up_to_date (xpath) ? FALSE : TRUE;
+		
 	return index;
 }
 
@@ -66,6 +72,8 @@ mu_index_destroy (MuIndex *index)
 	g_free (index);
 }
 
+
+
 struct _MuIndexCallbackData {
 	MuIndexMsgCallback    _idx_msg_cb;
 	MuIndexDirCallback    _idx_dir_cb;
@@ -227,12 +235,18 @@ mu_index_run (MuIndex *index, const char* path,
 {
 	MuIndexCallbackData cb_data;
 	MuResult rv;
+	gchar *version;
 	
 	g_return_val_if_fail (index && index->_xapian, MU_ERROR);
 	
 	if (!check_path (path))
 		return MU_ERROR;
-
+	
+	if (!reindex && index->_needs_reindex) {
+		g_warning ("database not up-to-date; needs full reindex");
+		return MU_ERROR;
+	}
+	
 	if (stats)
 		memset (stats, 0, sizeof(MuIndexStats));
 	
@@ -242,16 +256,21 @@ mu_index_run (MuIndex *index, const char* path,
 	cb_data._user_data = user_data;
 	cb_data._xapian    = index->_xapian;
 	cb_data._stats     = stats;
+	
 	cb_data._reindex   = reindex;
-
 	cb_data._dirstamp  = 0;
 
 	rv = mu_maildir_walk (path,
 			      (MuMaildirWalkMsgCallback)on_run_maildir_msg,
 			      (MuMaildirWalkDirCallback)on_run_maildir_dir,
 			      &cb_data);
+	if (rv == MU_OK) {
+		if (!mu_store_xapian_set_version (index->_xapian,
+						  MU_XAPIAN_DB_VERSION))
+			g_warning ("failed to set database version");
+	}
+	
 	mu_store_xapian_flush (index->_xapian);
-
 	return rv;
 }
 

src/mu-index.h

@@ -57,6 +57,7 @@ MuIndex* mu_index_new (const char* muhome);
 void mu_index_destroy (MuIndex *index);
 
 
+
 /** 
  * callback function for mu_index_(run|stats|cleanup), for each message
  * 

src/mu-maildir.c

@@ -530,12 +530,10 @@ clear_links (const gchar* dirname, DIR *dir)
 			continue;
 		}
 
-		if (entry->d_type == DT_LNK) {
-			if (unlink (fullpath) != 0) {
-				g_warning ("error unlinking %s: %s",
-					   fullpath, strerror(errno));
-				rv = FALSE;
-			}
+		if (entry->d_type == DT_LNK && unlink (fullpath) != 0) {
+			g_warning ("error unlinking %s: %s",
+				   fullpath, strerror(errno));
+			rv = FALSE;
 		} else /* == DT_DIR, see check before*/
 			rv = mu_maildir_clear_links (fullpath);
 	}

src/mu-query-xapian.cc

@@ -29,6 +29,7 @@
 #include "mu-msg-xapian-priv.hh"
 
 #include "mu-util.h"
+#include "mu-util-xapian.h"
 
 
 static void add_prefix (const MuMsgField* field,
@@ -132,11 +133,16 @@ mu_query_xapian_new (const char* xpath)
 	g_return_val_if_fail (xpath, NULL);
 
 	if (!mu_util_check_dir (xpath, TRUE, FALSE)) {
-		g_warning ("'%s' is not a readable xapian dir",
-			   xpath);
+		g_warning ("'%s' is not a readable xapian dir", xpath);
 		return NULL;
 	}
 
+	if (!mu_util_xapian_db_version_up_to_date (xpath)) {
+		g_warning ("%s is not up-to-date, needs full reindex",
+			   xpath);
+		return NULL;
+	}
+	
 	mqx = g_new (MuQueryXapian, 1);
 
 	if (!init_mu_query_xapian (mqx, xpath)) {
@@ -149,6 +155,7 @@ mu_query_xapian_new (const char* xpath)
 }
 
 
+
 void
 mu_query_xapian_destroy (MuQueryXapian *self)
 {
@@ -168,6 +175,7 @@ mu_query_xapian_run (MuQueryXapian *self, const char* searchexpr,
 	g_return_val_if_fail (searchexpr, NULL);
 		
 	try {
+	
 		Xapian::Query q(get_query(self, searchexpr));
 		Xapian::Enquire enq (*self->_db);
 		

src/mu-query-xapian.h

@@ -48,7 +48,17 @@ MuQueryXapian  *mu_query_xapian_new  (const char* path) G_GNUC_WARN_UNUSED_RESUL
  * 
  * @param self a MuQueryXapian instance, or NULL
  */
-void            mu_query_xapian_destroy  (MuQueryXapian *self);
+void mu_query_xapian_destroy  (MuQueryXapian *self);
+
+
+/** 
+ * get a version string for the database
+ * 
+ * @param store a valid MuQueryXapian
+ * 
+ * @return the version string (free with g_free), or NULL in case of error
+ */
+char* mu_query_xapian_version (MuQueryXapian *store);
 
 
 /** 

src/mu-store-xapian.cc

@@ -53,7 +53,7 @@ mu_store_xapian_new  (const char* xpath)
 	
 	try {
 		store = g_new0(MuStoreXapian,1);
-		store->_db = new Xapian::WritableDatabase
+		store->_db = new Xapian::WritableDatabase 
 			(xpath, Xapian::DB_CREATE_OR_OPEN);
 		
 		/* keep count of processed docs */
@@ -66,15 +66,45 @@ mu_store_xapian_new  (const char* xpath)
 
 	} MU_XAPIAN_CATCH_BLOCK;		
 
-	try {
-		delete store->_db;
-	} MU_XAPIAN_CATCH_BLOCK;
+	try { delete store->_db; } MU_XAPIAN_CATCH_BLOCK;
 	
 	g_free (store);
 	return NULL;
 }
 
 
+char*
+mu_store_xapian_version (MuStoreXapian *store)
+{
+	g_return_val_if_fail (store, NULL);
+
+	try {
+		const std::string version (
+			store->_db->get_metadata (MU_XAPIAN_VERSION_KEY));
+		
+		return version.empty() ? NULL : g_strdup (version.c_str());
+		
+	} MU_XAPIAN_CATCH_BLOCK;
+
+	return NULL;
+}
+
+gboolean
+mu_store_xapian_set_version (MuStoreXapian *store, const char* version)
+{
+	g_return_val_if_fail (store, FALSE);
+	g_return_val_if_fail (version, FALSE);
+	
+	try {
+		store->_db->set_metadata (MU_XAPIAN_VERSION_KEY, version);
+		return TRUE;
+		
+	} MU_XAPIAN_CATCH_BLOCK;
+
+	return FALSE;
+}
+
+
 static void
 begin_trx_if (MuStoreXapian *store, gboolean cond)
 {

src/mu-store-xapian.h

@@ -31,7 +31,6 @@ G_BEGIN_DECLS
 struct _MuStoreXapian;
 typedef struct _MuStoreXapian MuStoreXapian;
 
-
 /** 
  * create a new Xapian store, a place to store documents
  * 
@@ -49,6 +48,28 @@ MuStoreXapian*    mu_store_xapian_new     (const char* path);
 void              mu_store_xapian_destroy (MuStoreXapian *store);
 
 
+
+/** 
+ * get a version string for the database
+ * 
+ * @param store a valid MuStoreXapian
+ * 
+ * @return the version string (free with g_free), or NULL in case of error
+ */
+char* mu_store_xapian_version (MuStoreXapian *store);
+
+/** 
+ * set the version string for the database
+ * 
+ * @param store a valid MuStoreXapian
+ * @param version the version string (non-NULL)
+ *
+ * @return TRUE if setting the version succeeded, FALSE otherwise  
+ */
+gboolean  mu_store_xapian_set_version (MuStoreXapian *store,
+				       const char* version);
+
+
 /** 
  * try to flush/commit all outstanding work
  * 

src/mu-util-xapian.cc

@@ -0,0 +1,61 @@
+/*
+** Copyright (C) 2010 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl>
+**
+** This program is free software; you can redistribute it and/or modify
+** it under the terms of the GNU General Public License as published by
+** the Free Software Foundation; either version 3 of the License, or
+** (at your option) any later version.
+**
+** This program is distributed in the hope that it will be useful,
+** but WITHOUT ANY WARRANTY; without even the implied warranty of
+** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+** GNU General Public License for more details.
+**
+** You should have received a copy of the GNU General Public License
+** along with this program; if not, write to the Free Software Foundation,
+** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+**
+*/
+
+#include "config.h"
+#include <xapian.h>
+
+#include <cstring>
+#include "mu-util.h"
+#include "mu-util-xapian.h"
+
+char*
+mu_util_xapian_db_version (const gchar *xpath)
+{
+	try {
+		Xapian::Database db (xpath);
+		const std::string version
+			(db.get_metadata (MU_XAPIAN_VERSION_KEY));
+		
+		return version.empty() ? NULL : g_strdup (version.c_str());
+		
+	} MU_XAPIAN_CATCH_BLOCK;
+
+	return NULL;
+}
+
+
+gboolean
+mu_util_xapian_db_version_up_to_date (const gchar *xpath)
+{
+	gchar *version;
+	gboolean uptodate;
+	
+	version = mu_util_xapian_db_version (xpath);
+	if (!version)
+		return FALSE;
+	
+	uptodate = (std::strcmp (version, MU_XAPIAN_DB_VERSION) == 0); 
+	g_free (version);
+
+	return uptodate;
+}
+
+
+
+

src/mu-util-xapian.h

@@ -0,0 +1,52 @@
+/*
+** Copyright (C) 2010 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl>
+**
+** This program is free software; you can redistribute it and/or modify
+** it under the terms of the GNU General Public License as published by
+** the Free Software Foundation; either version 3 of the License, or
+** (at your option) any later version.
+**
+** This program is distributed in the hope that it will be useful,
+** but WITHOUT ANY WARRANTY; without even the implied warranty of
+** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+** GNU General Public License for more details.
+**
+** You should have received a copy of the GNU General Public License
+** along with this program; if not, write to the Free Software Foundation,
+** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+**
+*/
+
+
+#ifndef __MU_UTIL_XAPIAN_H__
+#define __MU_UTIL_XAPIAN_H__
+
+#include <glib.h>
+
+G_BEGIN_DECLS
+
+/** 
+ * get the version of the xapian database (ie., the version of the
+ * 'schema' we are using). If this version != MU_XAPIAN_DB_VERSION,
+ * it's means we need to a full reindex.
+ * 
+ * @param xpath path to the xapian database
+ * 
+ * @return the version of the database as a newly allocated string
+ * (free with g_free); if there is no version yet, it will return NULL
+ */
+char* mu_util_xapian_db_version (const gchar *xpath);
+
+
+/** 
+ * check if the 'schema' of the current database is up-to-date
+ * 
+ * @param xpath path to the xapian database
+ * 
+ * @return TRUE if it's up-to-date, FALSE otherwise
+ */
+gboolean mu_util_xapian_db_version_up_to_date (const gchar *xpath);
+
+G_END_DECLS
+
+#endif /*__MU_UTIL_XAPIAN_H__*/

src/mu-util.h

@@ -95,7 +95,8 @@ gboolean mu_util_check_dir (const gchar* path, gboolean readable,
         }
 
 /* the name of the (leaf) dir which has the xapian database */
-#define MU_XAPIAN_DIR_NAME "xapian-" MU_XAPIAN_DB_VERSION
+#define MU_XAPIAN_DIR_NAME    "xapian"
+#define MU_XAPIAN_VERSION_KEY "db_version"
 
 /** 
  * log something in the log file; note, we use G_LOG_LEVEL_INFO