From 83d6484f864f74dae3a8ee25e47a16f704c76dd4 Mon Sep 17 00:00:00 2001 From: "Dirk-Jan C. Binnema" Date: Sun, 28 Jul 2019 14:12:06 +0300 Subject: [PATCH] lib/mu-store: rework implementation * mu-store.h, mu-store-read.cc, mu-store-write.cc, mu-store-priv.hh have been reworked in mu-store.{cc,hh}, it the mix of c/c++ improved * update all the dependent modules * make it easier to upgrade an database in place (without user intervention) * remove the xbatch-size option --- lib/Makefile.am | 5 +- lib/mu-contacts.cc | 8 +- lib/mu-contacts.hh | 12 +- lib/mu-index.c | 9 - lib/mu-index.h | 12 +- lib/mu-msg-file.c | 5 +- lib/mu-query.cc | 6 - lib/mu-query.h | 2 +- lib/mu-store-priv.hh | 240 -------- lib/mu-store-read.cc | 274 --------- lib/mu-store-write.cc | 806 ------------------------- lib/mu-store.cc | 1343 +++++++++++++++++++++++++++++++++++++++-- lib/mu-store.h | 419 ------------- lib/mu-util.h | 199 +++--- 14 files changed, 1393 insertions(+), 1947 deletions(-) delete mode 100644 lib/mu-store-priv.hh delete mode 100644 lib/mu-store-read.cc delete mode 100644 lib/mu-store-write.cc delete mode 100644 lib/mu-store.h diff --git a/lib/Makefile.am b/lib/Makefile.am index cf563602..d36d9093 100644 --- a/lib/Makefile.am +++ b/lib/Makefile.am @@ -102,10 +102,7 @@ libmu_la_SOURCES= \ mu-script.c \ mu-script.h \ mu-store.cc \ - mu-store.h \ - mu-store-read.cc \ - mu-store-write.cc \ - mu-store-priv.hh \ + mu-store.hh \ mu-str.c \ mu-str.h \ mu-threader.c \ diff --git a/lib/mu-contacts.cc b/lib/mu-contacts.cc index 0d736c69..740bf6a6 100644 --- a/lib/mu-contacts.cc +++ b/lib/mu-contacts.cc @@ -263,23 +263,23 @@ Contacts::for_each(const EachContactFunc& each_contact) const /// C binding size_t -mu_contacts_count (MuContacts *self) +mu_contacts_count (const MuContacts *self) { g_return_val_if_fail (self, 0); - auto myself = reinterpret_cast(self); + auto myself = reinterpret_cast(self); return myself->size(); } gboolean -mu_contacts_foreach (MuContacts *self, MuContactsForeachFunc func, +mu_contacts_foreach (const MuContacts *self, MuContactsForeachFunc func, gpointer user_data) { g_return_val_if_fail (self, FALSE); g_return_val_if_fail (func, FALSE); - auto myself = reinterpret_cast(self); + auto myself = reinterpret_cast(self); myself->for_each([&](const ContactInfo& ci) { g_return_if_fail (!ci.email.empty()); diff --git a/lib/mu-contacts.hh b/lib/mu-contacts.hh index 2ab33ff9..49a86a1e 100644 --- a/lib/mu-contacts.hh +++ b/lib/mu-contacts.hh @@ -71,11 +71,11 @@ struct ContactInfo { class Contacts { public: /** - * Construct a new contacts object + * Construct a new contacts objects * * @param serialized serialized contacts */ - Contacts (const std::string& serialized); + Contacts (const std::string& serialized = ""); /** * DTOR @@ -147,7 +147,9 @@ public: * * @return a MuContacts* refering to this. */ - MuContacts* mu_contacts() { return reinterpret_cast(this); } + const MuContacts* mu_contacts() const { + return reinterpret_cast(this); + } @@ -170,7 +172,7 @@ G_BEGIN_DECLS * * @return the number of contacts */ -size_t mu_contacts_count (MuContacts *self); +size_t mu_contacts_count (const MuContacts *self); /** * Function called for mu_contacts_foreach; returns the e-mail address, name @@ -196,7 +198,7 @@ typedef void (*MuContactsForeachFunc) (const char *full_address, * @return TRUE if the function succeeded, or FALSE if the provide regular * expression was invalid (and not NULL) */ -gboolean mu_contacts_foreach (MuContacts *self, +gboolean mu_contacts_foreach (const MuContacts *self, MuContactsForeachFunc func, gpointer user_data); diff --git a/lib/mu-index.c b/lib/mu-index.c index 1a96bcee..73a25c35 100644 --- a/lib/mu-index.c +++ b/lib/mu-index.c @@ -33,7 +33,6 @@ #include #include "mu-maildir.h" -#include "mu-store.h" #include "mu-util.h" #define MU_LAST_USED_MAILDIR_KEY "last_used_maildir" @@ -331,14 +330,6 @@ mu_index_set_max_msg_size (MuIndex *index, guint max_size) index->_max_filesize = max_size; } -void -mu_index_set_xbatch_size (MuIndex *index, guint xbatchsize) -{ - g_return_if_fail (index); - mu_store_set_batch_size (index->_store, xbatchsize); -} - - MuError mu_index_run (MuIndex *index, const char *path, diff --git a/lib/mu-index.h b/lib/mu-index.h index bbce5360..72673e5b 100644 --- a/lib/mu-index.h +++ b/lib/mu-index.h @@ -25,7 +25,7 @@ #include #include #include /* for MuResult */ -#include +#include G_BEGIN_DECLS @@ -77,16 +77,6 @@ void mu_index_destroy (MuIndex *index); void mu_index_set_max_msg_size (MuIndex *index, guint max_size); -/** - * change batch size for Xapian store transaction (see - * 'mu_store_set_batch_size') - * - * @param index a mu index object - * @param max_size the batch size, or 0 to reset to the default - */ -void mu_index_set_xbatch_size (MuIndex *index, guint xbatchsize); - - /** * callback function for mu_index_(run|stats|cleanup), for each message * diff --git a/lib/mu-msg-file.c b/lib/mu-msg-file.c index 1a86dced..01b2d339 100644 --- a/lib/mu-msg-file.c +++ b/lib/mu-msg-file.c @@ -25,12 +25,13 @@ #include #include #include +#include #include #include "mu-util.h" #include "mu-str.h" #include "mu-maildir.h" -#include "mu-store.h" +#include "mu-store.hh" #include "mu-msg-priv.h" static gboolean init_file_metadata (MuMsgFile *self, const char* path, @@ -650,7 +651,7 @@ get_msgid (MuMsgFile *self, gboolean *do_free) return (char*)msgid; } else { /* if there is none, fake it */ *do_free = TRUE; - return g_strdup_printf ("%s@fake-msgid", + return g_strdup_printf ("%016 " PRIx64 "@fake-msgid", mu_util_get_hash (self->_path)); } } diff --git a/lib/mu-query.cc b/lib/mu-query.cc index a7e0584f..a1ce42de 100644 --- a/lib/mu-query.cc +++ b/lib/mu-query.cc @@ -215,12 +215,6 @@ mu_query_new (MuStore *store, GError **err) { g_return_val_if_fail (store, NULL); - if (mu_store_count (store, err) == 0) { - g_set_error (err, MU_ERROR_DOMAIN, MU_ERROR_XAPIAN_IS_EMPTY, - "database is empty"); - return 0; - } - try { return new MuQuery (store); } MU_XAPIAN_CATCH_BLOCK_G_ERROR_RETURN (err, MU_ERROR_XAPIAN, 0); diff --git a/lib/mu-query.h b/lib/mu-query.h index 25f0ea91..078657d7 100644 --- a/lib/mu-query.h +++ b/lib/mu-query.h @@ -21,7 +21,7 @@ #define __MU_QUERY_H__ #include -#include +#include #include #include diff --git a/lib/mu-store-priv.hh b/lib/mu-store-priv.hh deleted file mode 100644 index 182e8a09..00000000 --- a/lib/mu-store-priv.hh +++ /dev/null @@ -1,240 +0,0 @@ -/* -*-mode: c++; tab-width: 8; indent-tabs-mode: t; c-basic-offset: 8-*- */ -/* -** Copyright (C) 2011-2016 -** -** This program is free software; you can redistribute it and/or modify it -** under the terms of the GNU General Public License as published by the -** Free Software Foundation; either version 3, or (at your option) any -** later version. -** -** This program is distributed in the hope that it will be useful, -** but WITHOUT ANY WARRANTY; without even the implied warranty of -** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -** GNU General Public License for more details. -** -** You should have received a copy of the GNU General Public License -** along with this program; if not, write to the Free Software Foundation, -** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -** -*/ - -#ifndef __MU_STORE_PRIV_HH__ -#define __MU_STORE_PRIV_HH__ - -#include "config.h" - -#include -#include -#include -#include -#include - -#include - -#include "mu-store.h" -#include "mu-contacts.hh" -#include "mu-str.h" - -class MuStoreError { -public: - MuStoreError (MuError err, const std::string& whatarg): - _err (err), _what(whatarg) {} - MuError mu_error () const { return _err; } - const std::string& what() const { return _what; } -private: - MuError _err; - const std::string _what; -}; - -#define MU_CONTACTS_CACHE "contacts-cache" - -struct _MuStore { -public: - /* create a read-write MuStore */ - _MuStore (const char *patharg, bool rebuild) { - - if (rebuild) - _db = std::make_unique - (patharg, Xapian::DB_CREATE_OR_OVERWRITE); - else - _db = std::make_unique - (patharg, Xapian::DB_CREATE_OR_OPEN); - - init (patharg, rebuild, false); - check_set_version (); - - MU_WRITE_LOG ("%s: opened %s (batch size: %u) for read-write", - __func__, this->path(), (unsigned)batch_size()); - } - - /* create a read-only MuStore */ - _MuStore (const char *patharg) { - - _db = std::make_unique(patharg); - - init (patharg, false, false); - if (!mu_store_versions_match(this)) { - char *errstr = - g_strdup_printf ("db version: %s, but we need %s; " - "database rebuild is required", - mu_store_version (this), - MU_STORE_SCHEMA_VERSION); - - MuStoreError exc (MU_ERROR_XAPIAN_VERSION_MISMATCH, errstr); - g_free (errstr); - throw exc; - } - MU_WRITE_LOG ("%s: opened %s read-only", __func__, this->path()); - } - - void init (const char *patharg, bool rebuild, bool read_only) { - - _my_addresses = NULL; - _batch_size = DEFAULT_BATCH_SIZE; - _in_transaction = false; - _path = patharg; - _processed = 0; - _read_only = read_only; - _ref_count = 1; - _version = NULL; - - _contacts = std::make_unique( - _db->get_metadata(MU_CONTACTS_CACHE)); - } - - void set_my_addresses (const char **addrs) { - - if (_my_addresses) { - mu_str_free_list (_my_addresses); - _my_addresses = NULL; - } - - while (addrs && *addrs) { - _my_addresses = g_slist_prepend - (_my_addresses, g_strdup (*addrs)); - ++addrs; - } - } - - void check_set_version () { - if (_version) - return; - _version = mu_store_get_metadata (this, MU_STORE_VERSION_KEY, NULL); - if (!_version) { - mu_store_set_metadata (this, MU_STORE_VERSION_KEY, - MU_STORE_SCHEMA_VERSION, NULL); - _version = mu_store_get_metadata (this, MU_STORE_VERSION_KEY, NULL); - - } else if (g_strcmp0 (_version, MU_STORE_SCHEMA_VERSION) != 0) - throw MuStoreError (MU_ERROR_XAPIAN_VERSION_MISMATCH, - "the database needs a rebuild"); - } - - ~_MuStore () { - try { - if (_ref_count != 0) - g_warning ("ref count != 0"); - - if (!_read_only && db_writable()) - mu_store_flush (this); - - g_free (_version); - mu_str_free_list (_my_addresses); - - MU_WRITE_LOG ("closing xapian database with %d document(s)", - (int)db_read_only()->get_doccount()); - - } MU_XAPIAN_CATCH_BLOCK; - } - - /* close the old database, and write an empty one on top of it */ - void clear () { - if (is_read_only()) - throw std::runtime_error ("database is read-only"); - - // clear the database - db_writable()->close (); - _db = std::make_unique - (path(), Xapian::DB_CREATE_OR_OVERWRITE); - } - - // not re-entrant; stays valid until called again - const char *get_uid_term (const char *path) const; - - Mu::Contacts* contacts() { return _contacts.get(); } - - const char *version () const { - if (!_version) - _version = mu_store_get_metadata (this, MU_STORE_VERSION_KEY, NULL); - return _version; - } - - void set_version (const char *vers) { - g_free (_version); - _version = NULL; - mu_store_set_metadata (this, MU_STORE_VERSION_KEY, vers, NULL); - } - - static unsigned max_term_length() { return MU_STORE_MAX_TERM_LENGTH; } - - void begin_transaction (); - void commit_transaction (); - void rollback_transaction (); - - Xapian::WritableDatabase* db_writable() { - if (G_UNLIKELY(is_read_only())) - throw std::runtime_error ("database is read-only"); - return dynamic_cast(_db.get()); - } - - Xapian::Database* db_read_only() const { - return dynamic_cast(_db.get()); } - - const char* path () const { return _path.c_str(); } - bool is_read_only () const { return _read_only; } - - size_t batch_size () const { return _batch_size;} - size_t set_batch_size (size_t n) { - return _batch_size = ( n == 0) ? DEFAULT_BATCH_SIZE : n; - } - - bool in_transaction () const { return _in_transaction; } - bool in_transaction (bool in_tx) { return _in_transaction = in_tx; } - - int processed () const { return _processed; } - int set_processed (int n) { return _processed = n;} - int inc_processed () { return ++_processed; } - - /* MuStore is ref-counted */ - guint ref () { return ++_ref_count; } - guint unref () { - if (_ref_count < 1) - g_critical ("ref count error"); - return --_ref_count; - } - - GSList *my_addresses () { return _my_addresses; } - - /* by default, use transactions of 30000 messages */ - static const unsigned DEFAULT_BATCH_SIZE = 30000; -private: - /* transaction handling */ - bool _in_transaction; - int _processed; - size_t _batch_size; /* batch size of a xapian transaction */ - - /* contacts object to cache all the contact information */ - std::string _path; - mutable char *_version; - - std::unique_ptr _db; - std::unique_ptr _contacts; - - bool _read_only; - guint _ref_count; - - GSList *_my_addresses; -}; - - -#endif /*__MU_STORE_PRIV_HH__*/ diff --git a/lib/mu-store-read.cc b/lib/mu-store-read.cc deleted file mode 100644 index 1668a165..00000000 --- a/lib/mu-store-read.cc +++ /dev/null @@ -1,274 +0,0 @@ -/* -*-mode: c++; tab-width: 8; indent-tabs-mode: t; c-basic-offset: 8-*- */ -/* -** Copyright (C) 2008-2013 Dirk-Jan C. Binnema -** -** This program is free software; you can redistribute it and/or modify it -** under the terms of the GNU General Public License as published by the -** Free Software Foundation; either version 3, or (at your option) any -** later version. -** -** This program is distributed in the hope that it will be useful, -** but WITHOUT ANY WARRANTY; without even the implied warranty of -** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -** GNU General Public License for more details. -** -** You should have received a copy of the GNU General Public License -** along with this program; if not, write to the Free Software Foundation, -** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -** -*/ - -#if HAVE_CONFIG_H -#include "config.h" -#endif /*HAVE_CONFIG_H*/ - -#include -#include -#include -#include -#include -#include -#include - -#include "mu-store.h" -#include "mu-store-priv.hh" /* _MuStore */ - -#include "mu-msg.h" -#include "mu-msg-part.h" -#include "mu-store.h" -#include "mu-util.h" -#include "mu-str.h" -#include "mu-date.h" -#include "mu-flags.h" -#include "mu-contacts.hh" - - -// note: not re-entrant -const char* -_MuStore::get_uid_term (const char* path) const -{ - static char uid_term[64] = { '\0' }; - if (G_UNLIKELY(uid_term[0] == '\0')) - uid_term[0] = mu_msg_field_xapian_prefix(MU_MSG_FIELD_ID_UID); - - strncpy (uid_term + 1, mu_util_get_hash (path), sizeof(uid_term) - 1); - - return uid_term; -} - - -MuStore* -mu_store_new_read_only (const char* xpath, GError **err) -{ - g_return_val_if_fail (xpath, NULL); - - try { - return new _MuStore (xpath); - } catch (const MuStoreError& merr) { - mu_util_g_set_error (err, merr.mu_error(), "%s", - merr.what().c_str()); - - } MU_XAPIAN_CATCH_BLOCK_G_ERROR(err, MU_ERROR_XAPIAN); - - return NULL; -} - - -gboolean -mu_store_is_read_only (const MuStore *store) -{ - g_return_val_if_fail (store, FALSE); - - try { - return store->is_read_only() ? TRUE : FALSE; - - } MU_XAPIAN_CATCH_BLOCK_RETURN(FALSE); - -} - - -MuContacts* -mu_store_contacts (MuStore *store) -{ - g_return_val_if_fail (store, FALSE); - - try { - return store->contacts()->mu_contacts(); - - } MU_XAPIAN_CATCH_BLOCK_RETURN(FALSE); -} - - - -unsigned -mu_store_count (const MuStore *store, GError **err) -{ - g_return_val_if_fail (store, (unsigned)-1); - - try { - return store->db_read_only()->get_doccount(); - - } MU_XAPIAN_CATCH_BLOCK_G_ERROR_RETURN(err, MU_ERROR_XAPIAN, - (unsigned)-1); -} - - -const char* -mu_store_version (const MuStore *store) -{ - g_return_val_if_fail (store, NULL); - - return store->version(); -} - - -gboolean -mu_store_versions_match (const MuStore *store) -{ - g_return_val_if_fail (store, TRUE); - - return g_strcmp0 (mu_store_version (store), - MU_STORE_SCHEMA_VERSION) == 0; -} - - -char* -mu_store_get_metadata (const MuStore *store, const char *key, GError **err) -{ - g_return_val_if_fail (store, NULL); - g_return_val_if_fail (store->db_read_only(), NULL); - g_return_val_if_fail (key, NULL); - - try { - std::string val; - - val = store->db_read_only()->get_metadata (key); - if (!val.empty()) - return g_strdup (val.c_str()); - else - return NULL; - - } MU_XAPIAN_CATCH_BLOCK_G_ERROR_RETURN(err, MU_ERROR_XAPIAN, NULL); -} - - -XapianDatabase* -mu_store_get_read_only_database (MuStore *store) -{ - g_return_val_if_fail (store, NULL); - return (XapianWritableDatabase*)store->db_read_only(); -} - - - -gboolean -mu_store_contains_message (const MuStore *store, const char* path, GError **err) -{ - g_return_val_if_fail (store, FALSE); - g_return_val_if_fail (path, FALSE); - - try { - const std::string term (store->get_uid_term(path)); - return store->db_read_only()->term_exists (term) ? TRUE: FALSE; - - } MU_XAPIAN_CATCH_BLOCK_G_ERROR_RETURN(err, MU_ERROR_XAPIAN, FALSE); - -} - - -unsigned -mu_store_get_docid_for_path (const MuStore *store, const char* path, GError **err) -{ - g_return_val_if_fail (store, FALSE); - g_return_val_if_fail (path, FALSE); - - try { - const std::string term (store->get_uid_term(path)); - Xapian::Query query (term); - Xapian::Enquire enq (*store->db_read_only()); - - enq.set_query (query); - - Xapian::MSet mset (enq.get_mset (0,1)); - if (mset.empty()) - throw MuStoreError (MU_ERROR_NO_MATCHES, - "message not found"); - - return *mset.begin(); - - } MU_XAPIAN_CATCH_BLOCK_G_ERROR_RETURN(err, MU_ERROR_XAPIAN, - MU_STORE_INVALID_DOCID); -} - - - -time_t -mu_store_get_timestamp (const MuStore *store, const char *msgpath, GError **err) -{ - char *stampstr; - time_t rv; - - g_return_val_if_fail (store, 0); - g_return_val_if_fail (msgpath, 0); - - stampstr = mu_store_get_metadata (store, msgpath, err); - if (!stampstr) - return (time_t)0; - - rv = (time_t) g_ascii_strtoull (stampstr, NULL, 10); - g_free (stampstr); - - return rv; -} - - - -MuError -mu_store_foreach (MuStore *self, - MuStoreForeachFunc func, void *user_data, GError **err) -{ - g_return_val_if_fail (self, MU_ERROR); - g_return_val_if_fail (func, MU_ERROR); - - try { - Xapian::Enquire enq (*self->db_read_only()); - - enq.set_query (Xapian::Query::MatchAll); - enq.set_cutoff (0,0); - - Xapian::MSet matches - (enq.get_mset (0, self->db_read_only()->get_doccount())); - if (matches.empty()) - return MU_OK; /* database is empty */ - - for (Xapian::MSet::iterator iter = matches.begin(); - iter != matches.end(); ++iter) { - Xapian::Document doc (iter.get_document()); - const std::string path(doc.get_value(MU_MSG_FIELD_ID_PATH)); - MuError res = func (path.c_str(), user_data); - if (res != MU_OK) - return res; - } - - } MU_XAPIAN_CATCH_BLOCK_G_ERROR_RETURN(err, MU_ERROR_XAPIAN, - MU_ERROR_XAPIAN); - - return MU_OK; -} - - - -MuMsg* -mu_store_get_msg (const MuStore *self, unsigned docid, GError **err) -{ - g_return_val_if_fail (self, NULL); - g_return_val_if_fail (docid != 0, NULL); - - try { - Xapian::Document *doc = - new Xapian::Document - (self->db_read_only()->get_document (docid)); - return mu_msg_new_from_doc ((XapianDocument*)doc, err); - - } MU_XAPIAN_CATCH_BLOCK_G_ERROR_RETURN (err, MU_ERROR_XAPIAN, 0); -} diff --git a/lib/mu-store-write.cc b/lib/mu-store-write.cc deleted file mode 100644 index c8a0f933..00000000 --- a/lib/mu-store-write.cc +++ /dev/null @@ -1,806 +0,0 @@ -/* -*-mode: c++; tab-width: 8; indent-tabs-mode: t; c-basic-offset: 8-*- */ -/* -** Copyright (C) 2008-2017 Dirk-Jan C. Binnema -** -** This program is free software; you can redistribute it and/or modify it -** under the terms of the GNU General Public License as published by the -** Free Software Foundation; either version 3, or (at your option) any -** later version. -** -** This program is distributed in the hope that it will be useful, -** but WITHOUT ANY WARRANTY; without even the implied warranty of -** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -** GNU General Public License for more details. -** -** You should have received a copy of the GNU General Public License -** along with this program; if not, write to the Free Software Foundation, -** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -** -*/ - -#if HAVE_CONFIG_H -#include "config.h" -#endif /*HAVE_CONFIG_H*/ - -#include -#include -#include -#include -#include - -#include - -#include "mu-store.h" -#include "mu-store-priv.hh" /* _MuStore */ - -#include "mu-msg.h" -#include "mu-msg-part.h" -#include "mu-store.h" -#include "mu-util.h" -#include "mu-str.h" -#include "mu-date.h" -#include "mu-flags.h" -#include "mu-contacts.hh" - -void -_MuStore::begin_transaction () -{ - try { - db_writable()->begin_transaction(); - in_transaction (true); - } MU_XAPIAN_CATCH_BLOCK; -} - - -void -_MuStore::commit_transaction () { - try { - in_transaction (false); - db_writable()->commit_transaction(); - } MU_XAPIAN_CATCH_BLOCK; -} - -void -_MuStore::rollback_transaction () { - try { - in_transaction (false); - db_writable()->cancel_transaction(); - } MU_XAPIAN_CATCH_BLOCK; -} - - -/* we cache these prefix strings, so we don't have to allocate them all - * the time; this should save 10-20 string allocs per message */ -G_GNUC_CONST static const std::string& -prefix (MuMsgFieldId mfid) -{ - static std::string fields[MU_MSG_FIELD_ID_NUM]; - static bool initialized = false; - - if (G_UNLIKELY(!initialized)) { - for (int i = 0; i != MU_MSG_FIELD_ID_NUM; ++i) - fields[i] = std::string (1, mu_msg_field_xapian_prefix - ((MuMsgFieldId)i)); - initialized = true; - } - - return fields[mfid]; -} - - -static void -add_synonym_for_flag (MuFlags flag, Xapian::WritableDatabase *db) -{ - static const std::string pfx(prefix(MU_MSG_FIELD_ID_FLAGS)); - - db->clear_synonyms (pfx + mu_flag_name (flag)); - db->add_synonym (pfx + mu_flag_name (flag), pfx + - (std::string(1, (char)(tolower(mu_flag_char(flag)))))); -} - - -static void -add_synonym_for_prio (MuMsgPrio prio, Xapian::WritableDatabase *db) -{ - static const std::string pfx (prefix(MU_MSG_FIELD_ID_PRIO)); - - std::string s1 (pfx + mu_msg_prio_name (prio)); - std::string s2 (pfx + (std::string(1, mu_msg_prio_char (prio)))); - - db->clear_synonyms (s1); - db->clear_synonyms (s2); - - db->add_synonym (s1, s2); -} - - -static void -add_synonyms (MuStore *store) -{ - mu_flags_foreach ((MuFlagsForeachFunc)add_synonym_for_flag, - store->db_writable()); - mu_msg_prio_foreach ((MuMsgPrioForeachFunc)add_synonym_for_prio, - store->db_writable()); -} - - -MuStore* -mu_store_new_writable (const char* xpath, gboolean rebuild, GError **err) -{ - g_return_val_if_fail (xpath, NULL); - - try { - try { - MuStore *store; - store = new _MuStore (xpath, rebuild ? true : false); - add_synonyms (store); - return store; - - } MU_STORE_CATCH_BLOCK_RETURN(err,NULL); - - } MU_XAPIAN_CATCH_BLOCK_G_ERROR_RETURN (err, MU_ERROR_XAPIAN, NULL); -} - - - -void -mu_store_set_batch_size (MuStore *store, guint batchsize) -{ - g_return_if_fail (store); - store->set_batch_size (batchsize); -} - - -gboolean -mu_store_set_metadata (MuStore *store, const char *key, const char *val, - GError **err) -{ - g_return_val_if_fail (store, FALSE); - g_return_val_if_fail (key, FALSE); - g_return_val_if_fail (val, FALSE); - - try { - try { - store->db_writable()->set_metadata (key, val); - return TRUE; - - } MU_STORE_CATCH_BLOCK_RETURN(err, FALSE); - - } MU_XAPIAN_CATCH_BLOCK_G_ERROR_RETURN(err, MU_ERROR_XAPIAN, FALSE); - -} - - -gboolean -mu_store_clear (MuStore *store, GError **err) -{ - g_return_val_if_fail (store, FALSE); - - try { - try { - store->clear(); - return TRUE; - - } MU_STORE_CATCH_BLOCK_RETURN(err, FALSE); - - } MU_XAPIAN_CATCH_BLOCK_RETURN(FALSE); -} - - -void -mu_store_flush (MuStore *store) try { - - g_return_if_fail (store); - - if (store->in_transaction()) - store->commit_transaction (); - store->db_writable()->commit (); - - if (store->contacts()) - store->db_writable()->set_metadata(MU_CONTACTS_CACHE, - store->contacts()->serialize()); - -} MU_XAPIAN_CATCH_BLOCK; - -static void -add_terms_values_date (Xapian::Document& doc, MuMsg *msg, MuMsgFieldId mfid) -{ - const auto dstr = Mux::date_to_time_t_string ( - (time_t)mu_msg_get_field_numeric (msg, mfid)); - - doc.add_value ((Xapian::valueno)mfid, dstr); -} - -static void -add_terms_values_size (Xapian::Document& doc, MuMsg *msg, MuMsgFieldId mfid) -{ - const auto szstr = - Mux::size_to_string (mu_msg_get_field_numeric (msg, mfid)); - doc.add_value ((Xapian::valueno)mfid, szstr); -} - - - -G_GNUC_CONST -static const std::string& -flag_val (char flagchar) -{ - static const std::string - pfx (prefix(MU_MSG_FIELD_ID_FLAGS)), - draftstr (pfx + (char)tolower(mu_flag_char(MU_FLAG_DRAFT))), - flaggedstr (pfx + (char)tolower(mu_flag_char(MU_FLAG_FLAGGED))), - passedstr (pfx + (char)tolower(mu_flag_char(MU_FLAG_PASSED))), - repliedstr (pfx + (char)tolower(mu_flag_char(MU_FLAG_REPLIED))), - seenstr (pfx + (char)tolower(mu_flag_char(MU_FLAG_SEEN))), - trashedstr (pfx + (char)tolower(mu_flag_char(MU_FLAG_TRASHED))), - newstr (pfx + (char)tolower(mu_flag_char(MU_FLAG_NEW))), - signedstr (pfx + (char)tolower(mu_flag_char(MU_FLAG_SIGNED))), - cryptstr (pfx + (char)tolower(mu_flag_char(MU_FLAG_ENCRYPTED))), - attachstr (pfx + (char)tolower(mu_flag_char(MU_FLAG_HAS_ATTACH))), - unreadstr (pfx + (char)tolower(mu_flag_char(MU_FLAG_UNREAD))), - liststr (pfx + (char)tolower(mu_flag_char(MU_FLAG_LIST))); - - switch (flagchar) { - - case 'D': return draftstr; - case 'F': return flaggedstr; - case 'P': return passedstr; - case 'R': return repliedstr; - case 'S': return seenstr; - case 'T': return trashedstr; - - case 'N': return newstr; - - case 'z': return signedstr; - case 'x': return cryptstr; - case 'a': return attachstr; - case 'l': return liststr; - - case 'u': return unreadstr; - - default: - g_return_val_if_reached (flaggedstr); - return flaggedstr; - } -} - -/* pre-calculate; optimization */ -G_GNUC_CONST static const std::string& -prio_val (MuMsgPrio prio) -{ - static const std::string pfx (prefix(MU_MSG_FIELD_ID_PRIO)); - - static const std::string - low (pfx + std::string(1, mu_msg_prio_char(MU_MSG_PRIO_LOW))), - norm (pfx + std::string(1, mu_msg_prio_char(MU_MSG_PRIO_NORMAL))), - high (pfx + std::string(1, mu_msg_prio_char(MU_MSG_PRIO_HIGH))); - - switch (prio) { - case MU_MSG_PRIO_LOW: return low; - case MU_MSG_PRIO_NORMAL: return norm; - case MU_MSG_PRIO_HIGH: return high; - default: - g_return_val_if_reached (norm); - return norm; - } -} - - -static void // add term, truncate if needed. -add_term (Xapian::Document& doc, const std::string& term) -{ - if (term.length() < MU_STORE_MAX_TERM_LENGTH) - doc.add_term(term); - else - doc.add_term(term.substr(0, MU_STORE_MAX_TERM_LENGTH)); -} - - - -static void -add_terms_values_number (Xapian::Document& doc, MuMsg *msg, MuMsgFieldId mfid) -{ - gint64 num = mu_msg_get_field_numeric (msg, mfid); - - const std::string numstr (Xapian::sortable_serialise((double)num)); - doc.add_value ((Xapian::valueno)mfid, numstr); - - if (mfid == MU_MSG_FIELD_ID_FLAGS) { - const char *cur = mu_flags_to_str_s - ((MuFlags)num,(MuFlagType)MU_FLAG_TYPE_ANY); - g_return_if_fail (cur); - while (*cur) { - add_term (doc, flag_val(*cur)); - ++cur; - } - - } else if (mfid == MU_MSG_FIELD_ID_PRIO) - add_term (doc, prio_val((MuMsgPrio)num)); -} - - -/* for string and string-list */ -static void -add_terms_values_str (Xapian::Document& doc, const char *val, MuMsgFieldId mfid) -{ - const auto flat = Mux::utf8_flatten (val); - - if (mu_msg_field_xapian_index (mfid)) { - Xapian::TermGenerator termgen; - termgen.set_document (doc); - termgen.index_text (flat, 1, prefix(mfid)); - } - - if (mu_msg_field_xapian_term(mfid)) - add_term(doc, prefix(mfid) + flat); - -} - -static void -add_terms_values_string (Xapian::Document& doc, MuMsg *msg, MuMsgFieldId mfid) -{ - const char *orig; - - if (!(orig = mu_msg_get_field_string (msg, mfid))) - return; /* nothing to do */ - - /* the value is what we display in search results; the - * unchanged original */ - if (mu_msg_field_xapian_value(mfid)) - doc.add_value ((Xapian::valueno)mfid, orig); - - add_terms_values_str (doc, orig, mfid); -} - -static void -add_terms_values_string_list (Xapian::Document& doc, MuMsg *msg, - MuMsgFieldId mfid) -{ - const GSList *lst; - - lst = mu_msg_get_field_string_list (msg, mfid); - if (!lst) - return; - - if (mu_msg_field_xapian_value (mfid)) { - gchar *str; - str = mu_str_from_list (lst, ','); - if (str) - doc.add_value ((Xapian::valueno)mfid, str); - g_free (str); - } - - if (mu_msg_field_xapian_term (mfid)) { - for (; lst; lst = g_slist_next ((GSList*)lst)) - add_terms_values_str (doc, (const gchar*)lst->data, - mfid); - } -} - - -struct PartData { - PartData (Xapian::Document& doc, MuMsgFieldId mfid): - _doc (doc), _mfid(mfid) {} - Xapian::Document _doc; - MuMsgFieldId _mfid; -}; - -/* index non-body text parts */ -static void -maybe_index_text_part (MuMsg *msg, MuMsgPart *part, PartData *pdata) -{ - char *txt; - Xapian::TermGenerator termgen; - - /* only deal with attachments/messages; inlines are indexed as - * body parts */ - if (!(part->part_type & MU_MSG_PART_TYPE_ATTACHMENT) && - !(part->part_type & MU_MSG_PART_TYPE_MESSAGE)) - return; - - txt = mu_msg_part_get_text (msg, part, MU_MSG_OPTION_NONE); - if (!txt) - return; - - termgen.set_document(pdata->_doc); - const auto str = Mux::utf8_flatten (txt); - g_free (txt); - - termgen.index_text (str, 1, prefix(MU_MSG_FIELD_ID_EMBEDDED_TEXT)); -} - - -static void -each_part (MuMsg *msg, MuMsgPart *part, PartData *pdata) -{ - char *fname; - static const std::string - file (prefix(MU_MSG_FIELD_ID_FILE)), - mime (prefix(MU_MSG_FIELD_ID_MIME)); - - /* save the mime type of any part */ - if (part->type) { - char ctype[MU_STORE_MAX_TERM_LENGTH + 1]; - snprintf(ctype, sizeof(ctype), "%s/%s", part->type, part->subtype); - add_term(pdata->_doc, mime + ctype); - } - - if ((fname = mu_msg_part_get_filename (part, FALSE))) { - const auto flat = Mux::utf8_flatten (fname); - g_free (fname); - add_term(pdata->_doc, file + flat); - } - - maybe_index_text_part (msg, part, pdata); -} - - -static void -add_terms_values_attach (Xapian::Document& doc, MuMsg *msg, - MuMsgFieldId mfid) -{ - PartData pdata (doc, mfid); - mu_msg_part_foreach (msg, MU_MSG_OPTION_RECURSE_RFC822, - (MuMsgPartForeachFunc)each_part, &pdata); -} - - -static void -add_terms_values_body (Xapian::Document& doc, MuMsg *msg, - MuMsgFieldId mfid) -{ - if (mu_msg_get_flags(msg) & MU_FLAG_ENCRYPTED) - return; /* ignore encrypted bodies */ - - auto str = mu_msg_get_body_text (msg, MU_MSG_OPTION_NONE); - if (!str) /* FIXME: html->txt fallback needed */ - str = mu_msg_get_body_html (msg, MU_MSG_OPTION_NONE); - if (!str) - return; /* no body... */ - - Xapian::TermGenerator termgen; - termgen.set_document(doc); - - const auto flat = Mux::utf8_flatten(str); - termgen.index_text (flat, 1, prefix(mfid)); -} - -struct _MsgDoc { - Xapian::Document *_doc; - MuMsg *_msg; - MuStore *_store; - - /* callback data, to determine whether this message is 'personal' */ - gboolean _personal; - GSList *_my_addresses; -}; -typedef struct _MsgDoc MsgDoc; - - -static void -add_terms_values_default (MuMsgFieldId mfid, MsgDoc *msgdoc) -{ - if (mu_msg_field_is_numeric (mfid)) - add_terms_values_number - (*msgdoc->_doc, msgdoc->_msg, mfid); - else if (mu_msg_field_is_string (mfid)) - add_terms_values_string - (*msgdoc->_doc, msgdoc->_msg, mfid); - else if (mu_msg_field_is_string_list(mfid)) - add_terms_values_string_list - (*msgdoc->_doc, msgdoc->_msg, mfid); - else - g_return_if_reached (); - -} - -static void -add_terms_values (MuMsgFieldId mfid, MsgDoc* msgdoc) -{ - /* note: contact-stuff (To/Cc/From) will handled in - * each_contact_info, not here */ - if (!mu_msg_field_xapian_index(mfid) && - !mu_msg_field_xapian_term(mfid) && - !mu_msg_field_xapian_value(mfid)) - return; - - switch (mfid) { - case MU_MSG_FIELD_ID_DATE: - add_terms_values_date (*msgdoc->_doc, msgdoc->_msg, mfid); - break; - case MU_MSG_FIELD_ID_SIZE: - add_terms_values_size (*msgdoc->_doc, msgdoc->_msg, mfid); - break; - case MU_MSG_FIELD_ID_BODY_TEXT: - add_terms_values_body (*msgdoc->_doc, msgdoc->_msg, mfid); - break; - /* note: add_terms_values_attach handles _FILE, _MIME and - * _ATTACH_TEXT msgfields */ - case MU_MSG_FIELD_ID_FILE: - add_terms_values_attach (*msgdoc->_doc, msgdoc->_msg, mfid); - break; - case MU_MSG_FIELD_ID_MIME: - case MU_MSG_FIELD_ID_EMBEDDED_TEXT: - break; - case MU_MSG_FIELD_ID_THREAD_ID: - case MU_MSG_FIELD_ID_UID: - break; /* already taken care of elsewhere */ - default: - return add_terms_values_default (mfid, msgdoc); - } -} - - -static const std::string& -xapian_pfx (MuMsgContact *contact) -{ - static const std::string empty; - - /* use ptr to string to prevent copy... */ - switch (contact->type) { - case MU_MSG_CONTACT_TYPE_TO: - return prefix(MU_MSG_FIELD_ID_TO); - case MU_MSG_CONTACT_TYPE_FROM: - return prefix(MU_MSG_FIELD_ID_FROM); - case MU_MSG_CONTACT_TYPE_CC: - return prefix(MU_MSG_FIELD_ID_CC); - case MU_MSG_CONTACT_TYPE_BCC: - return prefix(MU_MSG_FIELD_ID_BCC); - default: - g_warning ("unsupported contact type %u", - (unsigned)contact->type); - return empty; - } -} - - -static void -add_address_subfields (Xapian::Document& doc, const char *addr, - const std::string& pfx) -{ - const char *at, *domain_part; - char *name_part; - - /* add "foo" and "bar.com" as terms as well for - * "foo@bar.com" */ - if (G_UNLIKELY(!(at = (g_strstr_len (addr, -1, "@"))))) - return; - - name_part = g_strndup(addr, at - addr); // foo - domain_part = at + 1; - - add_term(doc, pfx + name_part); - add_term(doc, pfx + domain_part); - - g_free (name_part); -} - -static gboolean -each_contact_info (MuMsgContact *contact, MsgDoc *msgdoc) -{ - /* for now, don't store reply-to addresses */ - if (mu_msg_contact_type (contact) == MU_MSG_CONTACT_TYPE_REPLY_TO) - return TRUE; - - const std::string pfx (xapian_pfx(contact)); - if (pfx.empty()) - return TRUE; /* unsupported contact type */ - - if (!mu_str_is_empty(contact->name)) { - Xapian::TermGenerator termgen; - termgen.set_document (*msgdoc->_doc); - const auto flat = Mux::utf8_flatten(contact->name); - termgen.index_text (flat, 1, pfx); - } - - if (!mu_str_is_empty(contact->email)) { - const auto flat = Mux::utf8_flatten(contact->email); - add_term(*msgdoc->_doc, pfx + flat); - add_address_subfields (*msgdoc->_doc, contact->email, pfx); - /* store it also in our contacts cache */ - auto contacts = msgdoc->_store->contacts(); - if (contacts) - contacts->add(Mu::ContactInfo(contact->full_address, - contact->email, - contact->name ? contact->name : "", - msgdoc->_personal, - mu_msg_get_date(msgdoc->_msg))); - } - - return TRUE; -} - - -static gboolean -each_contact_check_if_personal (MuMsgContact *contact, MsgDoc *msgdoc) -{ - GSList *cur; - - if (msgdoc->_personal || !contact->email) - return TRUE; - - for (cur = msgdoc->_my_addresses; cur; cur = g_slist_next (cur)) { - if (g_ascii_strcasecmp ( - contact->email, (const char*)cur->data) == 0) { - msgdoc->_personal = TRUE; - break; - } - } - - return TRUE; -} - -static Xapian::Document -new_doc_from_message (MuStore *store, MuMsg *msg) -{ - Xapian::Document doc; - MsgDoc docinfo = {&doc, msg, store, 0, FALSE}; - - mu_msg_field_foreach ((MuMsgFieldForeachFunc)add_terms_values, &docinfo); - - /* determine whether this is 'personal' email, ie. one of my - * e-mail addresses is explicitly mentioned -- it's not a - * mailing list message. Callback will update docinfo->_personal */ - if (store->my_addresses()) { - docinfo._my_addresses = store->my_addresses(); - mu_msg_contact_foreach - (msg, - (MuMsgContactForeachFunc)each_contact_check_if_personal, - &docinfo); - } - - /* also store the contact-info as separate terms, and add it - * to the cache */ - mu_msg_contact_foreach (msg, (MuMsgContactForeachFunc)each_contact_info, - &docinfo); - - // g_printerr ("\n--%s\n--\n", doc.serialise().c_str()); - - return doc; -} - -static void -update_threading_info (Xapian::WritableDatabase* db, - MuMsg *msg, Xapian::Document& doc) -{ - const GSList *refs; - - // refs contains a list of parent messages, with the oldest - // one first until the last one, which is the direct parent of - // the current message. of course, it may be empty. - // - // NOTE: there may be cases where the list is truncated; we happily - // ignore that case. - refs = mu_msg_get_references (msg); - - std::string thread_id; - if (refs) - thread_id = mu_util_get_hash ((const char*)refs->data); - else - thread_id = mu_util_get_hash (mu_msg_get_msgid (msg)); - - add_term (doc, prefix(MU_MSG_FIELD_ID_THREAD_ID) + thread_id); - doc.add_value((Xapian::valueno)MU_MSG_FIELD_ID_THREAD_ID, thread_id); -} - - -static unsigned -add_or_update_msg (MuStore *store, unsigned docid, MuMsg *msg, GError **err) -{ - g_return_val_if_fail (store, MU_STORE_INVALID_DOCID); - g_return_val_if_fail (msg, MU_STORE_INVALID_DOCID); - - try { - Xapian::docid id; - Xapian::Document doc (new_doc_from_message(store, msg)); - const std::string term (store->get_uid_term (mu_msg_get_path(msg))); - - if (!store->in_transaction()) - store->begin_transaction(); - - add_term (doc, term); - - // update the threading info if this message has a message id - if (mu_msg_get_msgid (msg)) - update_threading_info (store->db_writable(), msg, doc); - - if (docid == 0) - id = store->db_writable()->replace_document (term, doc); - else { - store->db_writable()->replace_document (docid, doc); - id = docid; - } - - if (store->inc_processed() % store->batch_size() == 0) - store->commit_transaction(); - - return id; - - } MU_XAPIAN_CATCH_BLOCK_G_ERROR (err, MU_ERROR_XAPIAN_STORE_FAILED); - - if (store->in_transaction()) - store->rollback_transaction(); - - return MU_STORE_INVALID_DOCID; -} - -unsigned -mu_store_add_msg (MuStore *store, MuMsg *msg, GError **err) -{ - g_return_val_if_fail (store, MU_STORE_INVALID_DOCID); - g_return_val_if_fail (msg, MU_STORE_INVALID_DOCID); - - return add_or_update_msg (store, 0, msg, err); -} - -unsigned -mu_store_update_msg (MuStore *store, unsigned docid, MuMsg *msg, GError **err) -{ - g_return_val_if_fail (store, MU_STORE_INVALID_DOCID); - g_return_val_if_fail (msg, MU_STORE_INVALID_DOCID); - g_return_val_if_fail (docid != 0, MU_STORE_INVALID_DOCID); - - return add_or_update_msg (store, docid, msg, err); -} - -unsigned -mu_store_add_path (MuStore *store, const char *path, const char *maildir, - GError **err) -{ - MuMsg *msg; - unsigned docid; - - g_return_val_if_fail (store, FALSE); - g_return_val_if_fail (path, FALSE); - - msg = mu_msg_new_from_file (path, maildir, err); - if (!msg) - return MU_STORE_INVALID_DOCID; - - docid = add_or_update_msg (store, 0, msg, err); - mu_msg_unref (msg); - - return docid; -} - - -XapianWritableDatabase* -mu_store_get_writable_database (MuStore *store) -{ - g_return_val_if_fail (store, NULL); - - return (XapianWritableDatabase*)store->db_writable(); -} - - -gboolean -mu_store_remove_path (MuStore *store, const char *msgpath) -{ - g_return_val_if_fail (store, FALSE); - g_return_val_if_fail (msgpath, FALSE); - - try { - const std::string term - (store->get_uid_term(msgpath)); - - store->db_writable()->delete_document (term); - store->inc_processed(); - - return TRUE; - - } MU_XAPIAN_CATCH_BLOCK_RETURN (FALSE); -} - - -gboolean -mu_store_set_timestamp (MuStore *store, const char* msgpath, - time_t stamp, GError **err) -{ - char buf[21]; - - g_return_val_if_fail (store, FALSE); - g_return_val_if_fail (msgpath, FALSE); - - sprintf (buf, "%" G_GUINT64_FORMAT, (guint64)stamp); - return mu_store_set_metadata (store, msgpath, buf, err); -} diff --git a/lib/mu-store.cc b/lib/mu-store.cc index ea78395f..af6537ad 100644 --- a/lib/mu-store.cc +++ b/lib/mu-store.cc @@ -1,6 +1,5 @@ -/* -*-mode: c++; tab-width: 8; indent-tabs-mode: t; c-basic-offset: 8-*- */ /* -** Copyright (C) 2008-2017 Dirk-Jan C. Binnema +** Copyright (C) 2019 Dirk-Jan C. Binnema ** ** This program is free software; you can redistribute it and/or modify it ** under the terms of the GNU General Public License as published by the @@ -18,105 +17,1323 @@ ** */ -#if HAVE_CONFIG_H #include "config.h" -#endif /*HAVE_CONFIG_H*/ -#include +#include +#include +#include #include -#include -#include -#include +#include +#include -#include - -#include "mu-store.h" -#include "mu-store-priv.hh" /* _MuStore */ - -#include "mu-msg.h" -#include "mu-msg-part.h" -#include "mu-store.h" -#include "mu-util.h" +#include "mu-store.hh" #include "mu-str.h" -#include "mu-date.h" -#include "mu-flags.h" -#include "mu-contacts.hh" +#include "mu-msg-part.h" +#include "parser/utils.hh" +using namespace Mu; +constexpr auto SchemaVersionKey = "schema-version"; +constexpr auto MaildirKey = "maildir"; +constexpr auto ContactsKey = "contacts"; +constexpr auto PersonalAddressesKey = "personal-addresses"; +constexpr auto CreatedKey = "created"; -MuStore* -mu_store_ref (MuStore *store) +constexpr auto ExpectedSchemaVersion = MU_STORE_SCHEMA_VERSION; + +using Addresses = Store::Addresses; + +/* we cache these prefix strings, so we don't have to allocate them all + * the time; this should save 10-20 string allocs per message */ +G_GNUC_CONST static const std::string& +prefix (MuMsgFieldId mfid) { - g_return_val_if_fail (store, NULL); - store->ref(); - return store; -} + static std::string fields[MU_MSG_FIELD_ID_NUM]; + static bool initialized = false; -MuStore* -mu_store_unref (MuStore *store) -{ - g_return_val_if_fail (store, NULL); - - if (store->unref() == 0) { - try { delete store; } MU_XAPIAN_CATCH_BLOCK; + if (G_UNLIKELY(!initialized)) { + for (int i = 0; i != MU_MSG_FIELD_ID_NUM; ++i) + fields[i] = std::string (1, mu_msg_field_xapian_prefix + ((MuMsgFieldId)i)); + initialized = true; } - return NULL; + return fields[mfid]; +} + +static void +add_synonym_for_flag (MuFlags flag, Xapian::WritableDatabase *db) +{ + static const std::string pfx(prefix(MU_MSG_FIELD_ID_FLAGS)); + + db->clear_synonyms (pfx + mu_flag_name (flag)); + db->add_synonym (pfx + mu_flag_name (flag), pfx + + (std::string(1, (char)(tolower(mu_flag_char(flag)))))); } -static char* -xapian_get_metadata (const gchar *xpath, const gchar *key) +static void +add_synonym_for_prio (MuMsgPrio prio, Xapian::WritableDatabase *db) +{ + static const std::string pfx (prefix(MU_MSG_FIELD_ID_PRIO)); + + std::string s1 (pfx + mu_msg_prio_name (prio)); + std::string s2 (pfx + (std::string(1, mu_msg_prio_char (prio)))); + + db->clear_synonyms (s1); + db->clear_synonyms (s2); + + db->add_synonym (s1, s2); +} + +struct Store::Private { + +#define LOCKED std::lock_guard l(lock_); + + Private (const std::string& path, bool readonly): + db_path_{path}, + db_{readonly? + std::make_shared(db_path_) : + std::make_shared(db_path_, Xapian::DB_OPEN)}, + maildir_{db()->get_metadata(MaildirKey)}, + created_{atoll(db()->get_metadata(CreatedKey).c_str())}, + schema_version_{db()->get_metadata(SchemaVersionKey)}, + personal_addresses_{Mux::split(db()->get_metadata(PersonalAddressesKey),",")}, + contacts_{db()->get_metadata(ContactsKey)} { + } + + Private (const std::string& path, const std::string& maildir): + db_path_{path}, + db_{std::make_shared( + db_path_, Xapian::DB_CREATE_OR_OVERWRITE)}, + maildir_{maildir}, + created_{time({})}, + schema_version_{MU_STORE_SCHEMA_VERSION} { + + writable_db()->set_metadata(SchemaVersionKey, schema_version_); + writable_db()->set_metadata(MaildirKey, maildir_); + writable_db()->set_metadata(CreatedKey, + Mux::format("%" PRId64, (int64_t)created_)); + } + + ~Private() { + if (wdb()) + wdb()->set_metadata (ContactsKey, contacts_.serialize()); + } + + std::shared_ptr db() const { + if (!db_) + throw std::runtime_error ("no db"); + return db_; + } + + std::shared_ptr wdb() const { + return std::dynamic_pointer_cast(db_); + } + + std::shared_ptr writable_db() const { + auto w_db{wdb()}; + if (!w_db) + throw std::runtime_error ("database is read-only"); + else + return w_db; + } + + void set_personal_addresses (const Addresses& addresses) { + + std::string all_addresses; + for (const auto& addr : addresses) { + // very basic check; just ensure there's no ',' in the address. + // we don't insist on full RFC5322 + if (addr.find(",") != std::string::npos) + throw std::runtime_error ("e-mail address with ',': " + addr); + if (!all_addresses.empty()) + all_addresses += ','; + all_addresses += addr; + } + writable_db()->set_metadata (PersonalAddressesKey, all_addresses); + } + + void add_synonyms () { + mu_flags_foreach ((MuFlagsForeachFunc)add_synonym_for_flag, + writable_db().get()); + mu_msg_prio_foreach ((MuMsgPrioForeachFunc)add_synonym_for_prio, + writable_db().get()); + } + + + time_t metadata_time_t (const std::string& key) const { + const auto ts = db()->get_metadata(key); + return (time_t)atoll(db()->get_metadata(key).c_str()); + } + + const std::string db_path_; + std::shared_ptr db_; + const std::string maildir_; + const time_t created_{}; + const std::string schema_version_; + const Addresses personal_addresses_; + Contacts contacts_; + + bool in_transaction_{}; + std::mutex lock_; + + mutable std::atomic ref_count_{1}; +}; + + +#undef LOCKED +#define LOCKED std::lock_guard l(priv_->lock_); + +struct NeedsReIndex: public std::runtime_error { + using std::runtime_error::runtime_error; +}; + +Store::Store (const std::string& path, bool readonly): + priv_{std::make_unique(path, readonly)} +{ + if (ExpectedSchemaVersion == schema_version()) + return; // All is good; nothing further to do + + if (readonly || maildir().empty()) + throw NeedsReIndex("database needs reindexing"); + + g_debug ("upgrading database"); + const auto addresses{personal_addresses()}; + const auto mdir{maildir()}; + + priv_.reset(); + priv_ = std::make_unique (path, mdir); + set_personal_addresses (addresses); +} + +Store::Store (const std::string& path, const std::string& maildir): + priv_{std::make_unique(path, maildir)} +{} + +Store::~Store() = default; + +bool +Store::read_only() const +{ + LOCKED; + return !priv_->wdb(); +} + +const std::string& +Store::maildir () const +{ + LOCKED; + return priv_->maildir_; +} + +void +Store::set_personal_addresses(const Store::Addresses& addresses) +{ + LOCKED; + priv_->set_personal_addresses (addresses); +} + +const Store::Addresses& +Store::personal_addresses(void) const +{ + LOCKED; + return priv_->personal_addresses_; +} + +const std::string& +Store::database_path() const +{ + LOCKED; + return priv_->db_path_; +} + +const Contacts& +Store::contacts() const +{ + LOCKED; + return priv_->contacts_; +} + +std::size_t +Store::size() const +{ + LOCKED; + + return priv_->db()->get_doccount(); +} + +bool +Store::empty() const +{ + return size() == 0; +} + + +const std::string& +Store::schema_version() const +{ + LOCKED; + + return priv_->schema_version_; +} + +time_t +Store::created() const +{ + LOCKED; + + return priv_->created_; +} + +time_t +Store::path_tstamp (const std::string& path) const +{ + LOCKED; + + const auto ts = priv_->db()->get_metadata(path); + if (ts.empty()) + return 0; + else + return (time_t)strtoll(ts.c_str(), NULL, 16); +} + +void +Store::set_path_tstamp (const std::string& path, time_t tstamp) +{ + LOCKED; + + std::array data{}; + const std::size_t len = snprintf (data.data(), data.size(), "%x", tstamp); + + priv_->writable_db()->set_metadata(path, std::string{data.data(), len}); +} + +void +Store::begin_transaction () try +{ + LOCKED; + + priv_->wdb()->begin_transaction(); + priv_->in_transaction_ = true; + +} MU_XAPIAN_CATCH_BLOCK; + +void +Store::commit_transaction () try +{ + LOCKED; + + priv_->in_transaction_ = false; + priv_->wdb()->commit_transaction(); + +} MU_XAPIAN_CATCH_BLOCK; + +void +Store::cancel_transaction () try +{ + LOCKED; + + priv_->in_transaction_ = false; + priv_->wdb()->cancel_transaction(); + +} MU_XAPIAN_CATCH_BLOCK; + +bool +Store::in_transaction () const +{ + LOCKED; + + return priv_->in_transaction_; +} + + +//////////////////////////////////////////////////////////////////////////////// +// C compat +extern "C" { + + +struct MuStore_ { Mu::Store* self; }; + + +static const Mu::Store* +self (const MuStore *store) +{ + if (!store) { + g_error ("invalid store"); // terminates + return {}; + } + + return reinterpret_cast(store); +} + +static Mu::Store* +mutable_self (MuStore *store) +{ + if (!store) { + g_error ("invalid store"); // terminates + return {}; + } + + auto s = reinterpret_cast(store); + if (s->read_only()) { + g_error ("store is read-only"); // terminates + return {}; + } + + return s; +} + + +static void +hash_str (char *buf, size_t buf_size, const char *data) +{ + snprintf(buf, buf_size, "016%" PRIx64, mu_util_get_hash(data)); +} + + +static std::string +get_uid_term (const char* path) +{ + char uid_term[1 + 16 + 1] = {'\0'}; + uid_term[0] = mu_msg_field_xapian_prefix(MU_MSG_FIELD_ID_UID); + hash_str(uid_term + 1, sizeof(uid_term)-1, path); + + return std::string{uid_term, sizeof(uid_term)}; +} + + +MuStore* +mu_store_new_readable (const char* xpath, GError **err) { g_return_val_if_fail (xpath, NULL); - g_return_val_if_fail (key, NULL); - if (access(xpath, F_OK) != 0) { - g_warning ("cannot access %s: %s", xpath, strerror(errno)); - return NULL; - } + g_debug ("opening database at %s (read-only)", xpath); try { - Xapian::Database db (xpath); - const std::string val(db.get_metadata (key)); - return val.empty() ? NULL : g_strdup (val.c_str()); + return reinterpret_cast(new Store (xpath)); - } MU_XAPIAN_CATCH_BLOCK; + } catch (const NeedsReIndex& nri) { + g_set_error (err, MU_ERROR_DOMAIN, MU_ERROR_XAPIAN_NEEDS_REINDEX, + "database @ %s needs (re)indexing", xpath); + } catch (const Xapian::DatabaseNotFoundError& dbe) { + g_set_error (err, MU_ERROR_DOMAIN, MU_ERROR_XAPIAN_NEEDS_REINDEX, + "database @ %s not found", xpath); + } catch (const Xapian::DatabaseOpeningError& dbe) { + g_set_error (err, MU_ERROR_DOMAIN, MU_ERROR_XAPIAN, + "failed to open database @ %s", xpath); + } catch (...) { + g_set_error (err, MU_ERROR_DOMAIN, MU_ERROR_XAPIAN, + "error opening database @ %s", xpath); + } - return NULL; + return NULL; } -char* -mu_store_database_version (const gchar *xpath) +MuStore* +mu_store_new_writable (const char* xpath, GError **err) { g_return_val_if_fail (xpath, NULL); - return xapian_get_metadata (xpath, MU_STORE_VERSION_KEY); + g_debug ("opening database at %s (writable)", xpath); + + try { + return reinterpret_cast(new Store (xpath, false/*!readonly*/)); + } catch (const NeedsReIndex& nri) { + g_set_error (err, MU_ERROR_DOMAIN, MU_ERROR_XAPIAN_NEEDS_REINDEX, + "database @ %s needs (re)indexing", xpath); + } catch (const Xapian::DatabaseNotFoundError& dbe) { + g_set_error (err, MU_ERROR_DOMAIN, MU_ERROR_XAPIAN_NEEDS_REINDEX, + "database @ %s not found", xpath); + } catch (const Xapian::DatabaseOpeningError& dbe) { + g_set_error (err, MU_ERROR_DOMAIN, MU_ERROR_XAPIAN, + "failed to open database @ %s", xpath); + } catch (const Xapian::DatabaseLockError& dle) { + g_set_error (err, MU_ERROR_DOMAIN, MU_ERROR_XAPIAN_CANNOT_GET_WRITELOCK, + "database @ %s is write-locked already", xpath); + } catch (...) { + g_set_error (err, MU_ERROR_DOMAIN, MU_ERROR_XAPIAN, + "error opening database @ %s", xpath); + } + + return NULL; } +MuStore* +mu_store_new_create (const char* xpath, const char *maildir, GError **err) +{ + g_return_val_if_fail (xpath, NULL); + g_return_val_if_fail (maildir, NULL); + + g_debug ("create database at %s (maildir=%s)", xpath, maildir); + + try { + return reinterpret_cast( + new Store (xpath, std::string{maildir})); + + } catch (const Xapian::DatabaseLockError& dle) { + g_set_error (err, MU_ERROR_DOMAIN, MU_ERROR_XAPIAN_CANNOT_GET_WRITELOCK, + "database @ %s is write-locked already", xpath); + } catch (...) { + g_set_error (err, MU_ERROR_DOMAIN, MU_ERROR_XAPIAN, + "error opening database @ %s", xpath); + } + + return NULL; +} + + +MuStore* +mu_store_ref (MuStore* store) +{ + g_return_val_if_fail (store, NULL); + g_return_val_if_fail (self(store)->priv()->ref_count_ > 0, NULL); + + ++self(store)->priv()->ref_count_; + return store; +} + + +MuStore* +mu_store_unref (MuStore* store) +{ + g_return_val_if_fail (store, NULL); + g_return_val_if_fail (self(store)->priv()->ref_count_ > 0, NULL); + + auto me = reinterpret_cast(store); + + if (--me->priv()->ref_count_ == 0) + delete me; + + return NULL; +} gboolean -mu_store_database_is_locked (const gchar *xpath) +mu_store_is_read_only (const MuStore *store) { - g_return_val_if_fail (xpath, FALSE); + g_return_val_if_fail (store, FALSE); try { - Xapian::WritableDatabase db (xpath, Xapian::DB_OPEN); - } catch (const Xapian::DatabaseLockError& xer) { - return TRUE; - } catch (const Xapian::Error &xer) { - g_warning ("%s: error: %s", __func__, - xer.get_msg().c_str()); - } + return self(store)->read_only() ? TRUE : FALSE; - return FALSE; + } MU_XAPIAN_CATCH_BLOCK_RETURN(FALSE); + +} + +gboolean +mu_store_clear (MuStore *store, GError **err) +{ + g_return_val_if_fail (store, FALSE); + + // FIXME: implement + return TRUE; +} + + +const MuContacts* +mu_store_contacts (MuStore *store) +{ + g_return_val_if_fail (store, FALSE); + + try { + return self(store)->contacts().mu_contacts(); + + } MU_XAPIAN_CATCH_BLOCK_RETURN(FALSE); +} + +unsigned +mu_store_count (const MuStore *store, GError **err) +{ + g_return_val_if_fail (store, (unsigned)-1); + + try { + return self(store)->size(); + + } MU_XAPIAN_CATCH_BLOCK_G_ERROR_RETURN(err, MU_ERROR_XAPIAN, + (unsigned)-1); +} + + +const char* +mu_store_schema_version (const MuStore *store) +{ + g_return_val_if_fail (store, NULL); + + return self(store)->schema_version().c_str(); +} + +XapianDatabase* +mu_store_get_read_only_database (MuStore *store) +{ + g_return_val_if_fail (store, NULL); + return (XapianWritableDatabase*)self(store)->priv()->db().get(); +} + +gboolean +mu_store_contains_message (const MuStore *store, const char* path, GError **err) +{ + g_return_val_if_fail (store, FALSE); + g_return_val_if_fail (path, FALSE); + + try { + const std::string term (get_uid_term(path)); + return self(store)->priv()->db()->term_exists (term) ? TRUE: FALSE; + + } MU_XAPIAN_CATCH_BLOCK_G_ERROR_RETURN(err, MU_ERROR_XAPIAN, FALSE); +} + +unsigned +mu_store_get_docid_for_path (const MuStore *store, const char* path, GError **err) +{ + g_return_val_if_fail (store, FALSE); + g_return_val_if_fail (path, FALSE); + + try { + const std::string term (get_uid_term(path)); + Xapian::Query query (term); + Xapian::Enquire enq (*self(store)->priv()->db().get()); + + enq.set_query (query); + + Xapian::MSet mset (enq.get_mset (0,1)); + if (mset.empty()) + throw std::runtime_error ("message not found"); + + return *mset.begin(); + + } MU_XAPIAN_CATCH_BLOCK_G_ERROR_RETURN(err, MU_ERROR_XAPIAN, + MU_STORE_INVALID_DOCID); +} + + +MuError +mu_store_foreach (MuStore *store, + MuStoreForeachFunc func, void *user_data, GError **err) +{ + g_return_val_if_fail (self, MU_ERROR); + g_return_val_if_fail (func, MU_ERROR); + + try { + Xapian::Enquire enq (*self(store)->priv()->db().get()); + + enq.set_query (Xapian::Query::MatchAll); + enq.set_cutoff (0,0); + + Xapian::MSet matches(enq.get_mset (0, self(store)->size())); + if (matches.empty()) + return MU_OK; /* database is empty */ + + for (Xapian::MSet::iterator iter = matches.begin(); + iter != matches.end(); ++iter) { + Xapian::Document doc (iter.get_document()); + const std::string path(doc.get_value(MU_MSG_FIELD_ID_PATH)); + MuError res = func (path.c_str(), user_data); + if (res != MU_OK) + return res; + } + + } MU_XAPIAN_CATCH_BLOCK_G_ERROR_RETURN(err, MU_ERROR_XAPIAN, + MU_ERROR_XAPIAN); + + return MU_OK; +} + + +MuMsg* +mu_store_get_msg (const MuStore *store, unsigned docid, GError **err) +{ + g_return_val_if_fail (store, NULL); + g_return_val_if_fail (docid != 0, NULL); + + try { + Xapian::Document *doc = + new Xapian::Document + (self(store)->priv()->db()->get_document (docid)); + return mu_msg_new_from_doc ((XapianDocument*)doc, err); + + } MU_XAPIAN_CATCH_BLOCK_G_ERROR_RETURN (err, MU_ERROR_XAPIAN, 0); +} + + +const char* +mu_store_database_path (const MuStore *store) +{ + g_return_val_if_fail (store, NULL); + + return self(store)->database_path().c_str(); +} + + +const char* +mu_store_maildir (const MuStore *store) +{ + g_return_val_if_fail (store, NULL); + + return self(store)->maildir().c_str(); +} + + +time_t +mu_store_created (const MuStore *store) +{ + g_return_val_if_fail (store, (time_t)0); + + return self(store)->created(); +} + + + +void +mu_store_set_personal_addresses (MuStore *store, const char **my_addresses) +{ + g_return_if_fail (store); + + if (!my_addresses) + return; + + Store::Addresses addrs; + for (auto i = 0; my_addresses[i]; ++i) + addrs.emplace_back(my_addresses[i]); + + mutable_self(store)->set_personal_addresses (addrs); +} + + +char** +mu_store_personal_addresses (const MuStore *store) +{ + g_return_val_if_fail (store, NULL); + + const auto size = self(store)->personal_addresses().size(); + auto addrs = g_new0 (char*, 1 + size); + for (auto i = 0; i != size; ++i) + addrs[i] = g_strdup(self(store)->personal_addresses()[i].c_str()); + + return addrs; } void -mu_store_set_my_addresses (MuStore *store, const char **my_addresses) -{ - g_return_if_fail (store); +mu_store_flush (MuStore *store) try { + + g_return_if_fail (store); + + if (self(store)->priv()->in_transaction_) + mutable_self(store)->commit_transaction (); + + mutable_self(store)->priv()->wdb()->set_metadata( + ContactsKey, self(store)->priv()->contacts_.serialize()); + +} MU_XAPIAN_CATCH_BLOCK; + +static void +add_terms_values_date (Xapian::Document& doc, MuMsg *msg, MuMsgFieldId mfid) +{ + const auto dstr = Mux::date_to_time_t_string ( + (time_t)mu_msg_get_field_numeric (msg, mfid)); + + doc.add_value ((Xapian::valueno)mfid, dstr); +} + +static void +add_terms_values_size (Xapian::Document& doc, MuMsg *msg, MuMsgFieldId mfid) +{ + const auto szstr = + Mux::size_to_string (mu_msg_get_field_numeric (msg, mfid)); + doc.add_value ((Xapian::valueno)mfid, szstr); +} + +G_GNUC_CONST +static const std::string& +flag_val (char flagchar) +{ + static const std::string + pfx (prefix(MU_MSG_FIELD_ID_FLAGS)), + draftstr (pfx + (char)tolower(mu_flag_char(MU_FLAG_DRAFT))), + flaggedstr (pfx + (char)tolower(mu_flag_char(MU_FLAG_FLAGGED))), + passedstr (pfx + (char)tolower(mu_flag_char(MU_FLAG_PASSED))), + repliedstr (pfx + (char)tolower(mu_flag_char(MU_FLAG_REPLIED))), + seenstr (pfx + (char)tolower(mu_flag_char(MU_FLAG_SEEN))), + trashedstr (pfx + (char)tolower(mu_flag_char(MU_FLAG_TRASHED))), + newstr (pfx + (char)tolower(mu_flag_char(MU_FLAG_NEW))), + signedstr (pfx + (char)tolower(mu_flag_char(MU_FLAG_SIGNED))), + cryptstr (pfx + (char)tolower(mu_flag_char(MU_FLAG_ENCRYPTED))), + attachstr (pfx + (char)tolower(mu_flag_char(MU_FLAG_HAS_ATTACH))), + unreadstr (pfx + (char)tolower(mu_flag_char(MU_FLAG_UNREAD))), + liststr (pfx + (char)tolower(mu_flag_char(MU_FLAG_LIST))); + + switch (flagchar) { + + case 'D': return draftstr; + case 'F': return flaggedstr; + case 'P': return passedstr; + case 'R': return repliedstr; + case 'S': return seenstr; + case 'T': return trashedstr; + + case 'N': return newstr; + + case 'z': return signedstr; + case 'x': return cryptstr; + case 'a': return attachstr; + case 'l': return liststr; + + case 'u': return unreadstr; + + default: + g_return_val_if_reached (flaggedstr); + return flaggedstr; + } +} + +/* pre-calculate; optimization */ +G_GNUC_CONST static const std::string& +prio_val (MuMsgPrio prio) +{ + static const std::string pfx (prefix(MU_MSG_FIELD_ID_PRIO)); + + static const std::string + low (pfx + std::string(1, mu_msg_prio_char(MU_MSG_PRIO_LOW))), + norm (pfx + std::string(1, mu_msg_prio_char(MU_MSG_PRIO_NORMAL))), + high (pfx + std::string(1, mu_msg_prio_char(MU_MSG_PRIO_HIGH))); + + switch (prio) { + case MU_MSG_PRIO_LOW: return low; + case MU_MSG_PRIO_NORMAL: return norm; + case MU_MSG_PRIO_HIGH: return high; + default: + g_return_val_if_reached (norm); + return norm; + } +} + + +static void // add term, truncate if needed. +add_term (Xapian::Document& doc, const std::string& term) +{ + if (term.length() < MU_STORE_MAX_TERM_LENGTH) + doc.add_term(term); + else + doc.add_term(term.substr(0, MU_STORE_MAX_TERM_LENGTH)); +} + + + +static void +add_terms_values_number (Xapian::Document& doc, MuMsg *msg, MuMsgFieldId mfid) +{ + gint64 num = mu_msg_get_field_numeric (msg, mfid); + + const std::string numstr (Xapian::sortable_serialise((double)num)); + doc.add_value ((Xapian::valueno)mfid, numstr); + + if (mfid == MU_MSG_FIELD_ID_FLAGS) { + const char *cur = mu_flags_to_str_s + ((MuFlags)num,(MuFlagType)MU_FLAG_TYPE_ANY); + g_return_if_fail (cur); + while (*cur) { + add_term (doc, flag_val(*cur)); + ++cur; + } + + } else if (mfid == MU_MSG_FIELD_ID_PRIO) + add_term (doc, prio_val((MuMsgPrio)num)); +} + + +/* for string and string-list */ +static void +add_terms_values_str (Xapian::Document& doc, const char *val, MuMsgFieldId mfid) +{ + const auto flat = Mux::utf8_flatten (val); + + if (mu_msg_field_xapian_index (mfid)) { + Xapian::TermGenerator termgen; + termgen.set_document (doc); + termgen.index_text (flat, 1, prefix(mfid)); + } + + if (mu_msg_field_xapian_term(mfid)) + add_term(doc, prefix(mfid) + flat); + +} + +static void +add_terms_values_string (Xapian::Document& doc, MuMsg *msg, MuMsgFieldId mfid) +{ + const char *orig; + + if (!(orig = mu_msg_get_field_string (msg, mfid))) + return; /* nothing to do */ + + /* the value is what we display in search results; the + * unchanged original */ + if (mu_msg_field_xapian_value(mfid)) + doc.add_value ((Xapian::valueno)mfid, orig); + + add_terms_values_str (doc, orig, mfid); +} + +static void +add_terms_values_string_list (Xapian::Document& doc, MuMsg *msg, + MuMsgFieldId mfid) +{ + const GSList *lst; + + lst = mu_msg_get_field_string_list (msg, mfid); + if (!lst) + return; + + if (mu_msg_field_xapian_value (mfid)) { + gchar *str; + str = mu_str_from_list (lst, ','); + if (str) + doc.add_value ((Xapian::valueno)mfid, str); + g_free (str); + } + + if (mu_msg_field_xapian_term (mfid)) { + for (; lst; lst = g_slist_next ((GSList*)lst)) + add_terms_values_str (doc, (const gchar*)lst->data, + mfid); + } +} + + +struct PartData { + PartData (Xapian::Document& doc, MuMsgFieldId mfid): + _doc (doc), _mfid(mfid) {} + Xapian::Document _doc; + MuMsgFieldId _mfid; +}; + +/* index non-body text parts */ +static void +maybe_index_text_part (MuMsg *msg, MuMsgPart *part, PartData *pdata) +{ + char *txt; + Xapian::TermGenerator termgen; + + /* only deal with attachments/messages; inlines are indexed as + * body parts */ + if (!(part->part_type & MU_MSG_PART_TYPE_ATTACHMENT) && + !(part->part_type & MU_MSG_PART_TYPE_MESSAGE)) + return; + + txt = mu_msg_part_get_text (msg, part, MU_MSG_OPTION_NONE); + if (!txt) + return; + + termgen.set_document(pdata->_doc); + const auto str = Mux::utf8_flatten (txt); + g_free (txt); + + termgen.index_text (str, 1, prefix(MU_MSG_FIELD_ID_EMBEDDED_TEXT)); +} + + +static void +each_part (MuMsg *msg, MuMsgPart *part, PartData *pdata) +{ + char *fname; + static const std::string + file (prefix(MU_MSG_FIELD_ID_FILE)), + mime (prefix(MU_MSG_FIELD_ID_MIME)); + + /* save the mime type of any part */ + if (part->type) { + char ctype[MU_STORE_MAX_TERM_LENGTH + 1]; + snprintf(ctype, sizeof(ctype), "%s/%s", part->type, part->subtype); + add_term(pdata->_doc, mime + ctype); + } + + if ((fname = mu_msg_part_get_filename (part, FALSE))) { + const auto flat = Mux::utf8_flatten (fname); + g_free (fname); + add_term(pdata->_doc, file + flat); + } + + maybe_index_text_part (msg, part, pdata); +} + + +static void +add_terms_values_attach (Xapian::Document& doc, MuMsg *msg, + MuMsgFieldId mfid) +{ + PartData pdata (doc, mfid); + mu_msg_part_foreach (msg, MU_MSG_OPTION_RECURSE_RFC822, + (MuMsgPartForeachFunc)each_part, &pdata); +} + + +static void +add_terms_values_body (Xapian::Document& doc, MuMsg *msg, + MuMsgFieldId mfid) +{ + if (mu_msg_get_flags(msg) & MU_FLAG_ENCRYPTED) + return; /* ignore encrypted bodies */ + + auto str = mu_msg_get_body_text (msg, MU_MSG_OPTION_NONE); + if (!str) /* FIXME: html->txt fallback needed */ + str = mu_msg_get_body_html (msg, MU_MSG_OPTION_NONE); + if (!str) + return; /* no body... */ + + Xapian::TermGenerator termgen; + termgen.set_document(doc); + + const auto flat = Mux::utf8_flatten(str); + termgen.index_text (flat, 1, prefix(mfid)); +} + +struct MsgDoc { + Xapian::Document *_doc; + MuMsg *_msg; + Store *_store; + /* callback data, to determine whether this message is 'personal' */ + gboolean _personal; + GSList *_my_addresses; +}; + + +static void +add_terms_values_default (MuMsgFieldId mfid, MsgDoc *msgdoc) +{ + if (mu_msg_field_is_numeric (mfid)) + add_terms_values_number + (*msgdoc->_doc, msgdoc->_msg, mfid); + else if (mu_msg_field_is_string (mfid)) + add_terms_values_string + (*msgdoc->_doc, msgdoc->_msg, mfid); + else if (mu_msg_field_is_string_list(mfid)) + add_terms_values_string_list + (*msgdoc->_doc, msgdoc->_msg, mfid); + else + g_return_if_reached (); +} + +static void +add_terms_values (MuMsgFieldId mfid, MsgDoc* msgdoc) +{ + /* note: contact-stuff (To/Cc/From) will handled in + * each_contact_info, not here */ + if (!mu_msg_field_xapian_index(mfid) && + !mu_msg_field_xapian_term(mfid) && + !mu_msg_field_xapian_value(mfid)) + return; + + switch (mfid) { + case MU_MSG_FIELD_ID_DATE: + add_terms_values_date (*msgdoc->_doc, msgdoc->_msg, mfid); + break; + case MU_MSG_FIELD_ID_SIZE: + add_terms_values_size (*msgdoc->_doc, msgdoc->_msg, mfid); + break; + case MU_MSG_FIELD_ID_BODY_TEXT: + add_terms_values_body (*msgdoc->_doc, msgdoc->_msg, mfid); + break; + /* note: add_terms_values_attach handles _FILE, _MIME and + * _ATTACH_TEXT msgfields */ + case MU_MSG_FIELD_ID_FILE: + add_terms_values_attach (*msgdoc->_doc, msgdoc->_msg, mfid); + break; + case MU_MSG_FIELD_ID_MIME: + case MU_MSG_FIELD_ID_EMBEDDED_TEXT: + break; + case MU_MSG_FIELD_ID_THREAD_ID: + case MU_MSG_FIELD_ID_UID: + break; /* already taken care of elsewhere */ + default: + return add_terms_values_default (mfid, msgdoc); + } +} + + +static const std::string& +xapian_pfx (MuMsgContact *contact) +{ + static const std::string empty; + + /* use ptr to string to prevent copy... */ + switch (contact->type) { + case MU_MSG_CONTACT_TYPE_TO: + return prefix(MU_MSG_FIELD_ID_TO); + case MU_MSG_CONTACT_TYPE_FROM: + return prefix(MU_MSG_FIELD_ID_FROM); + case MU_MSG_CONTACT_TYPE_CC: + return prefix(MU_MSG_FIELD_ID_CC); + case MU_MSG_CONTACT_TYPE_BCC: + return prefix(MU_MSG_FIELD_ID_BCC); + default: + g_warning ("unsupported contact type %u", + (unsigned)contact->type); + return empty; + } +} + + +static void +add_address_subfields (Xapian::Document& doc, const char *addr, + const std::string& pfx) +{ + const char *at, *domain_part; + char *name_part; + + /* add "foo" and "bar.com" as terms as well for + * "foo@bar.com" */ + if (G_UNLIKELY(!(at = (g_strstr_len (addr, -1, "@"))))) + return; + + name_part = g_strndup(addr, at - addr); // foo + domain_part = at + 1; + + add_term(doc, pfx + name_part); + add_term(doc, pfx + domain_part); + + g_free (name_part); +} + +static gboolean +each_contact_info (MuMsgContact *contact, MsgDoc *msgdoc) +{ + /* for now, don't store reply-to addresses */ + if (mu_msg_contact_type (contact) == MU_MSG_CONTACT_TYPE_REPLY_TO) + return TRUE; + + const std::string pfx (xapian_pfx(contact)); + if (pfx.empty()) + return TRUE; /* unsupported contact type */ + + if (!mu_str_is_empty(contact->name)) { + Xapian::TermGenerator termgen; + termgen.set_document (*msgdoc->_doc); + const auto flat = Mux::utf8_flatten(contact->name); + termgen.index_text (flat, 1, pfx); + } + + if (!mu_str_is_empty(contact->email)) { + const auto flat = Mux::utf8_flatten(contact->email); + add_term(*msgdoc->_doc, pfx + flat); + add_address_subfields (*msgdoc->_doc, contact->email, pfx); + /* store it also in our contacts cache */ + auto& contacts = msgdoc->_store->priv()->contacts_; + contacts.add(Mu::ContactInfo(contact->full_address, + contact->email, + contact->name ? contact->name : "", + msgdoc->_personal, + mu_msg_get_date(msgdoc->_msg))); + } + + return TRUE; +} + + +static gboolean +each_contact_check_if_personal (MuMsgContact *contact, MsgDoc *msgdoc) +{ + GSList *cur; + + if (msgdoc->_personal || !contact->email) + return TRUE; + + for (cur = msgdoc->_my_addresses; cur; cur = g_slist_next (cur)) { + if (g_ascii_strcasecmp ( + contact->email, (const char*)cur->data) == 0) { + msgdoc->_personal = TRUE; + break; + } + } + + return TRUE; +} + +static Xapian::Document +new_doc_from_message (MuStore *store, MuMsg *msg) +{ + Xapian::Document doc; + MsgDoc docinfo = {&doc, msg, mutable_self(store), 0, FALSE}; + + mu_msg_field_foreach ((MuMsgFieldForeachFunc)add_terms_values, &docinfo); + + /* determine whether this is 'personal' email, ie. one of my + * e-mail addresses is explicitly mentioned -- it's not a + * mailing list message. Callback will update docinfo->_personal */ + // FIXME + // if (store->my_addresses()) { + // docinfo._my_addresses = store->my_addresses(); + // mu_msg_contact_foreach + // (msg, + // (MuMsgContactForeachFunc)each_contact_check_if_personal, + // &docinfo); + // } + + /* also store the contact-info as separate terms, and add it + * to the cache */ + mu_msg_contact_foreach (msg, (MuMsgContactForeachFunc)each_contact_info, + &docinfo); + + // g_printerr ("\n--%s\n--\n", doc.serialise().c_str()); + + return doc; +} + +static void +update_threading_info (Xapian::WritableDatabase* db, + MuMsg *msg, Xapian::Document& doc) +{ + const GSList *refs; + + // refs contains a list of parent messages, with the oldest + // one first until the last one, which is the direct parent of + // the current message. of course, it may be empty. + // + // NOTE: there may be cases where the list is truncated; we happily + // ignore that case. + refs = mu_msg_get_references (msg); + + char thread_id[16+1]; + hash_str(thread_id, sizeof(thread_id), + refs ? (const char*)refs->data : mu_msg_get_msgid (msg)); + + add_term (doc, prefix(MU_MSG_FIELD_ID_THREAD_ID) + thread_id); + doc.add_value((Xapian::valueno)MU_MSG_FIELD_ID_THREAD_ID, thread_id); +} + + +static unsigned +add_or_update_msg (MuStore *store, unsigned docid, MuMsg *msg, GError **err) +{ + g_return_val_if_fail (store, MU_STORE_INVALID_DOCID); + g_return_val_if_fail (msg, MU_STORE_INVALID_DOCID); + + try { + Xapian::docid id; + Xapian::Document doc (new_doc_from_message(store, msg)); + const std::string term (get_uid_term (mu_msg_get_path(msg))); + + auto self = mutable_self(store); + auto wdb = self->priv()->wdb(); + + if (!self->in_transaction()) + self->begin_transaction(); + + add_term (doc, term); + + // update the threading info if this message has a message id + if (mu_msg_get_msgid (msg)) + update_threading_info (wdb.get(), msg, doc); + + if (docid == 0) + id = wdb->replace_document (term, doc); + else { + wdb->replace_document (docid, doc); + id = docid; + } + + // FIXME + // if (self->inc_processed() % store->batch_size() == 0) + // self->commit_transaction(); + + return id; + + } MU_XAPIAN_CATCH_BLOCK_G_ERROR (err, MU_ERROR_XAPIAN_STORE_FAILED); + + // FIXME + // if (store->in_transaction()) + // store->rollback_transaction(); + + return MU_STORE_INVALID_DOCID; +} + +unsigned +mu_store_add_msg (MuStore *store, MuMsg *msg, GError **err) +{ + g_return_val_if_fail (store, MU_STORE_INVALID_DOCID); + g_return_val_if_fail (msg, MU_STORE_INVALID_DOCID); + + return add_or_update_msg (store, 0, msg, err); +} + +unsigned +mu_store_update_msg (MuStore *store, unsigned docid, MuMsg *msg, GError **err) +{ + g_return_val_if_fail (store, MU_STORE_INVALID_DOCID); + g_return_val_if_fail (msg, MU_STORE_INVALID_DOCID); + g_return_val_if_fail (docid != 0, MU_STORE_INVALID_DOCID); + + return add_or_update_msg (store, docid, msg, err); +} + +unsigned +mu_store_add_path (MuStore *store, const char *path, const char *maildir, + GError **err) +{ + MuMsg *msg; + unsigned docid; + + g_return_val_if_fail (store, FALSE); + g_return_val_if_fail (path, FALSE); + + msg = mu_msg_new_from_file (path, maildir, err); + if (!msg) + return MU_STORE_INVALID_DOCID; + + docid = add_or_update_msg (store, 0, msg, err); + mu_msg_unref (msg); + + return docid; +} + + +XapianWritableDatabase* +mu_store_get_writable_database (MuStore *store) +{ + g_return_val_if_fail (store, NULL); + + return (XapianWritableDatabase*)mutable_self(store)->priv()->wdb().get(); +} + + +gboolean +mu_store_remove_path (MuStore *store, const char *msgpath) +{ + g_return_val_if_fail (store, FALSE); + g_return_val_if_fail (msgpath, FALSE); + + try { + const std::string term{(get_uid_term(msgpath))}; + auto wdb = mutable_self(store)->priv()->wdb(); + + wdb->delete_document (term); + //store->inc_processed(); + + return TRUE; + + } MU_XAPIAN_CATCH_BLOCK_RETURN (FALSE); +} + + +gboolean +mu_store_set_timestamp (MuStore *store, const char* msgpath, + time_t stamp, GError **err) +{ + g_return_val_if_fail (store, FALSE); + g_return_val_if_fail (msgpath, FALSE); + + mutable_self(store)->set_path_tstamp(msgpath, stamp); + + return TRUE; +} + +time_t +mu_store_get_timestamp (const MuStore *store, const char *msgpath, GError **err) +{ + g_return_val_if_fail (store, 0); + g_return_val_if_fail (msgpath, 0); + + return self(store)->path_tstamp(msgpath); +} + - store->set_my_addresses (my_addresses); } diff --git a/lib/mu-store.h b/lib/mu-store.h deleted file mode 100644 index ed40c22c..00000000 --- a/lib/mu-store.h +++ /dev/null @@ -1,419 +0,0 @@ -/* -** Copyright (C) 2008-2017 Dirk-Jan C. Binnema -** -** This program is free software; you can redistribute it and/or modify it -** under the terms of the GNU General Public License as published by the -** Free Software Foundation; either version 3, or (at your option) any -** later version. -** -** This program is distributed in the hope that it will be useful, -** but WITHOUT ANY WARRANTY; without even the implied warranty of -** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -** GNU General Public License for more details. -** -** You should have received a copy of the GNU General Public License -** along with this program; if not, write to the Free Software Foundation, -** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -** -*/ - -#ifndef __MU_STORE_H__ -#define __MU_STORE_H__ - -#include -#include -#include -#include /* for MuError, MuError */ -#include - -G_BEGIN_DECLS - -struct _MuStore; -typedef struct _MuStore MuStore; - -/* http://article.gmane.org/gmane.comp.search.xapian.general/3656 */ -#define MU_STORE_MAX_TERM_LENGTH (240) - -/** - * create a new writable Xapian store, a place to store documents - * - * @param path the path to the database - * @param err to receive error info or NULL. err->code is MuError value - * - * @return a new MuStore object with ref count == 1, or NULL in case - * of error; free with mu_store_unref - */ -MuStore* mu_store_new_writable (const char *xpath, - gboolean rebuild, GError **err) - G_GNUC_MALLOC G_GNUC_WARN_UNUSED_RESULT; - - -/** - * create a new read-only Xapian store, for querying documents - * - * @param path the path to the database - * @param err to receive error info or NULL. err->code is MuError value - * - * @return a new MuStore object with ref count == 1, or NULL in case - * of error; free with mu_store_unref - */ -MuStore* mu_store_new_read_only (const char* xpath, GError **err) - G_GNUC_MALLOC G_GNUC_WARN_UNUSED_RESULT; - - - -/** - * increase the reference count for this store with 1 - * - * @param store a valid store object - * - * @return the same store with increased ref count, or NULL in case of - * error - */ -MuStore* mu_store_ref (MuStore *store); - -/** - * decrease the reference count for this store with 1 - * - * @param store a valid store object - * - * @return NULL - */ -MuStore* mu_store_unref (MuStore *store); - - -/** - * we need this when using Xapian::WritableDatabase* from C - */ -typedef gpointer XapianWritableDatabase; - - - -/** - * get the underlying writable database object for this store; not - * that this pointer becomes in valid after mu_store_destroy - * - * @param store a valid store - * - * @return a Xapian::WritableDatabase (you'll need to cast in C++), or - * NULL in case of error. - */ -XapianWritableDatabase* mu_store_get_writable_database (MuStore *store); - - -/** - * we need this when using Xapian::WritableDatabase* from C - */ -typedef gpointer XapianDatabase; - - -/** - * get the underlying read-only database object for this store; not that this - * pointer becomes in valid after mu_store_destroy - * - * @param store a valid store - * - * @return a Xapian::Database (you'll need to cast in C++), or - * NULL in case of error. - */ -XapianDatabase* mu_store_get_read_only_database (MuStore *store); - - -/** - * set the Xapian batch size for this store. Normally, there's no need - * to use this function as the default is good enough; however, if you - * use mu in a very memory-constrained environment, you can set the - * batchsize to e.g. 1000 at the cost of significant slow-down. - * - * @param store a valid store object - * @param batchsize the new batch size; or 0 to reset to - * the default batch size - */ -void mu_store_set_batch_size (MuStore *store, guint batchsize); - - -/** - * register a char** of email addresses as 'my' addresses, ie. mark - * message that have these addresses in one of the address fields as - * 'personal' (e.g., in mu-contacts). calling this function overrides - * any 'my addresses' that were set before, using this function or - * through mu_store_new_writable - * - * @param store a valid store object - * @param my_addresses a char** of email addresses - */ -void mu_store_set_my_addresses (MuStore *store, const char **my_addresses); - - -/** - * Get the a MuContacts* ptr for this store. - * - * @param store a store - * - * @return the contacts ptr - */ -MuContacts* mu_store_contacts (MuStore *store); - - -/** - * get the numbers of documents in the database - * - * @param index a valid MuStore instance - * @param err to receive error info or NULL. err->code is MuError value - * - * @return the number of documents in the database; (unsigned)-1 in - * case of error - */ -unsigned mu_store_count (const MuStore *store, GError **err); - -/** - * get a version string for the database; it's a const string, which - * is valid as long MuStore exists and mu_store_version is not called - * again. - * - * @param store a valid MuStore - * - * @return the version string or NULL in case of error - */ -const char* mu_store_version (const MuStore *store); - - -/** - * try to flush/commit all outstanding work to the database and the contacts - * cache. - * - * @param store a valid xapian store - */ -void mu_store_flush (MuStore *store); - -#define MU_STORE_INVALID_DOCID 0 - -/** - * store an email message in the XapianStore - * - * @param store a valid store - * @param msg a valid message - * @param err receives error information, if any, or NULL - * - * @return the docid of the stored message, or 0 - * (MU_STORE_INVALID_DOCID) in case of error - */ -unsigned mu_store_add_msg (MuStore *store, MuMsg *msg, GError **err); - - -/** - * update an email message in the XapianStore - * - * @param store a valid store - * @param the docid for the message - * @param msg a valid message - * @param err receives error information, if any, or NULL - * - * @return the docid of the stored message, or 0 - * (MU_STORE_INVALID_DOCID) in case of error - */ -unsigned mu_store_update_msg (MuStore *store, unsigned docid, MuMsg *msg, - GError **err); - - -/** - * store an email message in the XapianStore; similar to - * mu_store_store, but instead takes a path as parameter instead of a - * MuMsg* - * - * @param store a valid store - * @param path full filesystem path to a valid message - * @param maildir set the maildir (e.g. "/drafts") for this message, or NULL - * note that you cannot mu_msg_move_msg_to_maildir unless maildir is set. - * @param err receives error information, if any, or NULL - * - * @return the docid of the stored message, or 0 - * (MU_STORE_INVALID_DOCID) in case of error - */ -unsigned mu_store_add_path (MuStore *store, const char *path, - const char* maildir, GError **err); - -/** - * remove a message from the database based on its path - * - * @param store a valid store - * @param msgpath path of the message (note, this is only used to - * *identify* the message; a common use of this function is to remove - * a message from the database, for which there is no message anymore - * in the filesystem. - * - * @return TRUE if it succeeded, FALSE otherwise - */ -gboolean mu_store_remove_path (MuStore *store, const char* msgpath); - - -/** - * does a certain message exist in the database already? - * - * @param store a store - * @param path the message path - * @param err to receive error info or NULL. err->code is MuError value - * - * @return TRUE if the message exists, FALSE otherwise - */ -gboolean mu_store_contains_message (const MuStore *store, const char* path, - GError **err); - - - -/** - * get the docid for message at path - * - * @param store a store - * @param path the message path - * @param err to receive error info or NULL. err->code is MuError value - * - * @return the docid if the message was found, MU_STORE_INVALID_DOCID (0) otherwise - * */ -unsigned mu_store_get_docid_for_path (const MuStore *store, const char* path, GError **err); - -/** - * store a timestamp for a directory - * - * @param store a valid store - * @param msgpath path to a maildir - * @param stamp a timestamp - * @param err to receive error info or NULL. err->code is MuError value - * - * @return TRUE if setting the timestamp succeeded, FALSE otherwise - */ -gboolean mu_store_set_timestamp (MuStore *store, const char* msgpath, - time_t stamp, GError **err); - -/** - * get the timestamp for a directory - * - * @param store a valid store - * @param msgpath path to a maildir - * @param err to receive error info or NULL. err->code is MuError value - * - * @return the timestamp, or 0 in case of error - */ -time_t mu_store_get_timestamp (const MuStore *store, const char* msgpath, - GError **err); - - -/** - * check whether this store is read-only - * - * @param store a store - * - * @return TRUE if the store is read-only, FALSE otherwise (and in - * case of error) - */ -gboolean mu_store_is_read_only (const MuStore *store); - - -/** - * call a function for each document in the database - * - * @param self a valid store - * @param func a callback function to to call for each document - * @param user_data a user pointer passed to the callback function - * @param err to receive error info or NULL. err->code is MuError value - * - * @return MU_OK if all went well, MU_STOP if the foreach was interrupted, - * MU_ERROR in case of error - */ -typedef MuError (*MuStoreForeachFunc) (const char* path, gpointer user_data); -MuError mu_store_foreach (MuStore *self, MuStoreForeachFunc func, - void *user_data, GError **err); - -/** - * set metadata for this MuStore - * - * @param store a store - * @param key metadata key - * @param val metadata value - * @param err to receive error info or NULL. err->code is the MuError value - * - * @return TRUE if succeeded, FALSE otherwise - */ -gboolean mu_store_set_metadata (MuStore *store, const char *key, const char *val, - GError **err); - -/** - * get metadata for this MuStore - * - * @param store a store - * @param key the metadata key - * @param err to receive error info or NULL. err->code is MuError value - * - * @return the value of the metadata (gfree when done with it), or - * NULL in case of error - */ -char* mu_store_get_metadata (const MuStore *store, const char *key, GError **err) - G_GNUC_WARN_UNUSED_RESULT; - - -/** -" * get the version of the xapian database (ie., the version of the - * 'schema' we are using). If this version != MU_STORE_SCHEMA_VERSION, - * it's means we need to a full reindex. - * - * @param xpath path to the xapian database - * - * @return the version of the database as a newly allocated string - * (free with g_free); if there is no version yet, it will return NULL - */ -gchar* mu_store_database_version (const gchar *xpath) G_GNUC_WARN_UNUSED_RESULT; - - -/** - * check whether the database schema's version is the same as the one - * that the current mu uses. If they are not the same, we'll need a - * database rebuild. - * - * @param store a MuStore instance - * - * @return TRUE if the versions are the same, FALSE otherwise. - */ -gboolean mu_store_versions_match (const MuStore *store); - -/** - * clear the database, ie., remove all of the contents. This is a - * destructive operation, but the database can be restored be doing a - * full scan of the maildirs. Also, clear the contacts cache file - * - * @param store a MuStore object - * @param err to receive error info or NULL. err->code is MuError value - * - * @return TRUE if the clearing succeeded, FALSE otherwise. - */ -gboolean mu_store_clear (MuStore *store, GError **err); - - -/** - * check if the database is locked for writing - * - * @param xpath path to a xapian database - * - * @return TRUE if it is locked, FALSE otherwise (or in case of error) - */ -gboolean mu_store_database_is_locked (const gchar *xpath); - - - -/** - * get a specific message, based on its Xapian docid - * - * @param self a valid MuQuery instance - * @param docid the Xapian docid for the wanted message - * @param err receives error information, or NULL - * - * @return a MuMsg instance (use mu_msg_unref when done with it), or - * NULL in case of error - */ -MuMsg* mu_store_get_msg (const MuStore *self, unsigned docid, GError **err) - G_GNUC_WARN_UNUSED_RESULT; - - - -G_END_DECLS - -#endif /*__MU_STORE_H__*/ diff --git a/lib/mu-util.h b/lib/mu-util.h index 302db23c..f65735d9 100644 --- a/lib/mu-util.h +++ b/lib/mu-util.h @@ -1,5 +1,3 @@ -/* -*-mode: c; tab-width: 8; indent-tabs-mode: t; c-basic-offset: 8 -*-*/ - /* ** Copyright (C) 2008-2013 Dirk-Jan C. Binnema ** @@ -284,15 +282,15 @@ typedef gpointer XapianEnquire; /* print a warning for a GError, and free it */ -#define MU_HANDLE_G_ERROR(GE) \ - do { \ - if (!(GE)) \ - g_warning ("%s:%u: an error occured in %s", \ - __FILE__, __LINE__, __func__); \ - else { \ - g_warning ("error %u: %s", (GE)->code, (GE)->message); \ - g_error_free ((GE)); \ - } \ +#define MU_HANDLE_G_ERROR(GE) \ + do { \ + if (!(GE)) \ + g_warning ("%s:%u: an error occured in %s", \ + __FILE__, __LINE__, __func__); \ + else { \ + g_warning ("error %u: %s", (GE)->code, (GE)->message); \ + g_error_free ((GE)); \ + } \ } while (0) @@ -311,73 +309,71 @@ typedef gpointer XapianEnquire; } \ -#define MU_XAPIAN_CATCH_BLOCK \ - catch (const Xapian::Error &xerr) { \ - g_critical ("%s: xapian error '%s'", \ - __func__, xerr.get_msg().c_str()); \ - } catch (...) { \ - g_critical ("%s: caught exception", __func__); \ - } - -#define MU_XAPIAN_CATCH_BLOCK_G_ERROR(GE,E) \ - catch (const Xapian::DatabaseLockError &xerr) { \ - mu_util_g_set_error ((GE), \ - MU_ERROR_XAPIAN_CANNOT_GET_WRITELOCK, \ - "%s: xapian error '%s'", \ - __func__, xerr.get_msg().c_str()); \ - } catch (const Xapian::DatabaseCorruptError &xerr) { \ - mu_util_g_set_error ((GE), \ - MU_ERROR_XAPIAN_CORRUPTION, \ - "%s: xapian error '%s'", \ - __func__, xerr.get_msg().c_str()); \ - } catch (const Xapian::DatabaseError &xerr) { \ - mu_util_g_set_error ((GE),MU_ERROR_XAPIAN, \ - "%s: xapian error '%s'", \ - __func__, xerr.get_msg().c_str()); \ - } catch (const Xapian::Error &xerr) { \ - mu_util_g_set_error ((GE),(E), \ - "%s: xapian error '%s'", \ - __func__, xerr.get_msg().c_str()); \ - } catch (...) { \ - mu_util_g_set_error ((GE),(MU_ERROR_INTERNAL), \ - "%s: caught exception", __func__); \ - } - - -#define MU_XAPIAN_CATCH_BLOCK_RETURN(R) \ - catch (const Xapian::Error &xerr) { \ - g_critical ("%s: xapian error '%s'", \ - __func__, xerr.get_msg().c_str()); \ - return (R); \ - } catch (...) { \ - g_critical ("%s: caught exception", __func__); \ - return (R); \ - } - -#define MU_XAPIAN_CATCH_BLOCK_G_ERROR_RETURN(GE,E,R) \ +#define MU_XAPIAN_CATCH_BLOCK \ catch (const Xapian::Error &xerr) { \ - mu_util_g_set_error ((GE),(E), \ - "%s: xapian error '%s'", \ - __func__, xerr.get_msg().c_str()); \ - return (R); \ + g_critical ("%s: xapian error '%s'", \ + __func__, xerr.get_msg().c_str()); \ + } catch (const std::runtime_error& re) { \ + g_critical ("%s: error: %s", __func__, re.what()); \ } catch (...) { \ - if ((GE)&&!(*(GE))) \ - mu_util_g_set_error ((GE), \ - (MU_ERROR_INTERNAL), \ + g_critical ("%s: caught exception", __func__); \ + } + +#define MU_XAPIAN_CATCH_BLOCK_G_ERROR(GE,E) \ + catch (const Xapian::DatabaseLockError &xerr) { \ + mu_util_g_set_error ((GE), \ + MU_ERROR_XAPIAN_CANNOT_GET_WRITELOCK, \ + "%s: xapian error '%s'", \ + __func__, xerr.get_msg().c_str()); \ + } catch (const Xapian::DatabaseError &xerr) { \ + mu_util_g_set_error ((GE),MU_ERROR_XAPIAN, \ + "%s: xapian error '%s'", \ + __func__, xerr.get_msg().c_str()); \ + } catch (const Xapian::Error &xerr) { \ + mu_util_g_set_error ((GE),(E), \ + "%s: xapian error '%s'", \ + __func__, xerr.get_msg().c_str()); \ + } catch (const std::runtime_error& ex) { \ + mu_util_g_set_error ((GE),(MU_ERROR_INTERNAL), \ + "%s: error: %s", __func__, ex.what()); \ + \ + } catch (...) { \ + mu_util_g_set_error ((GE),(MU_ERROR_INTERNAL), \ + "%s: caught exception", __func__); \ + } + + +#define MU_XAPIAN_CATCH_BLOCK_RETURN(R) \ + catch (const Xapian::Error &xerr) { \ + g_critical ("%s: xapian error '%s'", \ + __func__, xerr.get_msg().c_str()); \ + return (R); \ + } catch (const std::runtime_error& ex) { \ + g_critical("%s: error: %s", __func__, ex.what()); \ + return (R); \ + } catch (...) { \ + g_critical ("%s: caught exception", __func__); \ + return (R); \ + } + +#define MU_XAPIAN_CATCH_BLOCK_G_ERROR_RETURN(GE,E,R) \ + catch (const Xapian::Error &xerr) { \ + mu_util_g_set_error ((GE),(E), \ + "%s: xapian error '%s'", \ + __func__, xerr.get_msg().c_str()); \ + return (R); \ + } catch (const std::runtime_error& ex) { \ + mu_util_g_set_error ((GE),(MU_ERROR_INTERNAL), \ + "%s: error: %s", __func__, ex.what()); \ + return (R); \ + } catch (...) { \ + if ((GE)&&!(*(GE))) \ + mu_util_g_set_error ((GE), \ + (MU_ERROR_INTERNAL), \ "%s: caught exception", __func__); \ - return (R); \ + return (R); \ } -/* the name of the (leaf) dir which has the xapian database */ -#define MU_XAPIAN_DIR_NAME "xapian" - -/* name of the bookmark file */ -#define MU_BOOKMARK_FILENAME "bookmarks" - -/* metadata key for the xapian 'schema' version */ -#define MU_STORE_VERSION_KEY "db_version" - - /** * log something in the log file; note, we use G_LOG_LEVEL_INFO * for such messages @@ -414,24 +410,20 @@ enum _MuError { /* (parsing) error in the query */ MU_ERROR_XAPIAN_QUERY = 13, - /* xapian dir is not accessible */ - MU_ERROR_XAPIAN_DIR_NOT_ACCESSIBLE = 14, - /* database version is not up-to-date */ - MU_ERROR_XAPIAN_VERSION_MISMATCH = 15, + /* missing data for a document */ - MU_ERROR_XAPIAN_MISSING_DATA = 16, - /* database corruption */ - MU_ERROR_XAPIAN_CORRUPTION = 17, + MU_ERROR_XAPIAN_MISSING_DATA = 17, /* can't get write lock */ - MU_ERROR_XAPIAN_CANNOT_GET_WRITELOCK = 18, - /* database is empty */ - MU_ERROR_XAPIAN_IS_EMPTY = 19, - /* could not write */ - MU_ERROR_XAPIAN_STORE_FAILED = 20, + MU_ERROR_XAPIAN_CANNOT_GET_WRITELOCK = 19, + /* could not write */ + MU_ERROR_XAPIAN_STORE_FAILED = 21, /* could not remove */ - MU_ERROR_XAPIAN_REMOVE_FAILED = 21, + MU_ERROR_XAPIAN_REMOVE_FAILED = 22, /* database was modified; reload */ - MU_ERROR_XAPIAN_MODIFIED = 22, + MU_ERROR_XAPIAN_MODIFIED = 23, + /* database was modified; reload */ + MU_ERROR_XAPIAN_NEEDS_REINDEX = 24, + /* GMime related errors */ @@ -481,38 +473,39 @@ typedef enum _MuError MuError; gboolean mu_util_g_set_error (GError **err, MuError errcode, const char *frm, ...) G_GNUC_PRINTF(3,4); - /** - * calculate a 64-bit hash for the given string, based on a - * combination of the DJB and BKDR hash functions + * calculate a 64-bit hash for the given string, based on a combination of the + * DJB and BKDR hash functions. * * @param a string * - * @return the hash as a static string, which stays valid until this - * function is called again. + * @return the hash */ -static inline const char* +static inline guint64 mu_util_get_hash (const char* str) { - unsigned djbhash, bkdrhash, bkdrseed; - unsigned u; - static char hex[18]; + guint32 djbhash; + guint32 bkdrhash; + guint32 bkdrseed; - djbhash = 5381; - bkdrhash = 0; - bkdrseed = 1313; + djbhash = 5381; + bkdrhash = 0; + bkdrseed = 1313; - for(u = 0; str[u]; ++u) { + for(unsigned u = 0U; str[u]; ++u) { djbhash = ((djbhash << 5) + djbhash) + str[u]; bkdrhash = bkdrhash * bkdrseed + str[u]; } - snprintf (hex, sizeof(hex), "%08x%08x", djbhash, bkdrhash); - - return hex; + guint64 hash = djbhash; + return (hash<<32) | bkdrhash; } + + + + #define MU_COLOR_RED "\x1b[31m" #define MU_COLOR_GREEN "\x1b[32m" #define MU_COLOR_YELLOW "\x1b[33m"