From 5fc8a8f83ef53c17986abc4bd7ad689856bcf79a Mon Sep 17 00:00:00 2001 From: "Dirk-Jan C. Binnema" Date: Sun, 30 Jan 2022 14:28:30 +0200 Subject: [PATCH] store/query: access query only through store Make Mu::Query only accessible through store, so we can lock the db for the duration of a (full, multipass) query. --- lib/mu-query.cc | 24 +++++++---- lib/mu-query.hh | 44 ++++++++++--------- lib/mu-store.cc | 110 +++++++++++++++++++++++++++++++++++------------- lib/mu-store.hh | 39 +++++++++++++++++ 4 files changed, 158 insertions(+), 59 deletions(-) diff --git a/lib/mu-query.cc b/lib/mu-query.cc index 03684d89..e97e4ba7 100644 --- a/lib/mu-query.cc +++ b/lib/mu-query.cc @@ -38,12 +38,13 @@ using namespace Mu; struct Query::Private { - Private(const Store& store) : store_{store}, parser_{store_} {} + Private(const Store& store) + : store_{store}, parser_{store_} {} // New // bool calculate_threads (Xapian::Enquire& enq, size maxnum); Xapian::Enquire - make_enquire(const std::string& expr, MuMsgFieldId sortfieldid, QueryFlags qflags) const; + make_enquire(const std::string& expr, MuMsgFieldId sortfieldid, QueryFlags qflags) const; Xapian::Enquire make_related_enquire(const StringSet& thread_ids, MuMsgFieldId sortfieldid, QueryFlags qflags) const; @@ -65,11 +66,17 @@ struct Query::Private { QueryFlags qflags, size_t maxnum) const; + size_t store_size() const + { + return store_.database().get_doccount(); + } + const Store& store_; const Parser parser_; }; -Query::Query(const Store& store) : priv_{std::make_unique(store)} {} +Query::Query(const Store& store) + : priv_{std::make_unique(store)} {} Query::Query(Query&& other) = default; @@ -123,7 +130,8 @@ Query::Private::make_related_enquire(const StringSet& thread_ids, } struct ThreadKeyMaker : public Xapian::KeyMaker { - ThreadKeyMaker(const QueryMatches& matches) : match_info_(matches) {} + ThreadKeyMaker(const QueryMatches& matches) + : match_info_(matches) {} std::string operator()(const Xapian::Document& doc) const override { const auto it{match_info_.find(doc.get_docid())}; @@ -229,9 +237,9 @@ Query::Private::run_related(const std::string& expr, // is unlimited and the sorting happens during threading. auto r_enq{make_related_enquire(minfo.thread_ids, threading ? MU_MSG_FIELD_ID_NONE : sortfieldid, - qflags)}; + qflags)}; const auto r_mset{r_enq.get_mset(0, - threading ? store_.size() : maxnum, + threading ? store_size() : maxnum, {}, make_related_decider(qflags, minfo).get())}; auto qres{QueryResults{r_mset, std::move(minfo.matches)}}; @@ -244,7 +252,7 @@ Query::Private::run(const std::string& expr, QueryFlags qflags, size_t maxnum) const { - const auto eff_maxnum{maxnum == 0 ? store_.size() : maxnum}; + const auto eff_maxnum{maxnum == 0 ? store_size() : maxnum}; #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wextra" const auto eff_sortfield{sortfieldid == MU_MSG_FIELD_ID_NONE ? MU_MSG_FIELD_ID_DATE @@ -283,7 +291,7 @@ Query::count(const std::string& expr) const return xapian_try( [&] { const auto enq{priv_->make_enquire(expr, MU_MSG_FIELD_ID_NONE, {})}; - auto mset{enq.get_mset(0, priv_->store_.size())}; + auto mset{enq.get_mset(0, priv_->store_size())}; mset.fetch(); return mset.size(); }, diff --git a/lib/mu-query.hh b/lib/mu-query.hh index c6c7e727..9f1ac5b6 100644 --- a/lib/mu-query.hh +++ b/lib/mu-query.hh @@ -30,26 +30,7 @@ namespace Mu { class Query { - public: - /** - * Construct a new Query instance. - * - * @param store a MuStore object - */ - Query(const Store& store); - /** - * DTOR - * - */ - ~Query(); - - /** - * Move CTOR - * - * @param other - */ - Query(Query&& other); - +public: /** * Run a query on the store * @@ -87,7 +68,28 @@ class Query { */ std::string parse(const std::string& expr, bool xapian) const; - private: +private: + friend class Store; + + /** + * Construct a new Query instance. + * + * @param store a MuStore object + */ + Query(const Store& store); + /** + * DTOR + * + */ + ~Query(); + + /** + * Move CTOR + * + * @param other + */ + Query(Query&& other); + struct Private; std::unique_ptr priv_; }; diff --git a/lib/mu-store.cc b/lib/mu-store.cc index 7883cda1..bce18875 100644 --- a/lib/mu-store.cc +++ b/lib/mu-store.cc @@ -34,6 +34,7 @@ #include #include "mu-store.hh" +#include "mu-query.hh" #include "utils/mu-str.h" #include "utils/mu-error.hh" @@ -100,8 +101,6 @@ add_synonym_for_prio(MuMsgPrio prio, Xapian::WritableDatabase* db) } struct Store::Private { -#define LOCKED std::lock_guard l(lock_); - enum struct XapianOpts { ReadOnly, Open, CreateOverwrite, @@ -306,9 +305,6 @@ get_uid_term(const char* path) return std::string{uid_term, sizeof(uid_term)}; } -#undef LOCKED -#define LOCKED std::lock_guard l__(priv_->lock_) - Store::Store(const std::string& path, bool readonly) : priv_{std::make_unique(path, readonly)} { @@ -362,7 +358,7 @@ Store::writable_database() Indexer& Store::indexer() { - LOCKED; + std::lock_guard guard{priv_->lock_}; if (metadata().read_only) throw Error{Error::Code::Store, "no indexer for read-only store"}; @@ -375,7 +371,7 @@ Store::indexer() std::size_t Store::size() const { - LOCKED; + std::lock_guard guard{priv_->lock_}; return priv_->db().get_doccount(); } @@ -415,7 +411,7 @@ maildir_from_path(const std::string& root, const std::string& path) unsigned Store::add_message(const std::string& path, bool use_transaction) { - LOCKED; + std::lock_guard guard{priv_->lock_}; GError* gerr{}; const auto maildir{maildir_from_path(metadata().root_maildir, path)}; @@ -461,7 +457,7 @@ Store::remove_message(const std::string& path) { return xapian_try( [&] { - LOCKED; + std::lock_guard guard{priv_->lock_}; const std::string term{(get_uid_term(path.c_str()))}; priv_->writable_db().delete_document(term); @@ -475,7 +471,7 @@ Store::remove_message(const std::string& path) void Store::remove_messages(const std::vector& ids) { - LOCKED; + std::lock_guard guard{priv_->lock_}; priv_->transaction_inc(); @@ -491,7 +487,7 @@ Store::remove_messages(const std::vector& ids) time_t Store::dirstamp(const std::string& path) const { - LOCKED; + std::lock_guard guard{priv_->lock_}; constexpr auto epoch = static_cast(0); return xapian_try([&] { @@ -507,10 +503,12 @@ Store::dirstamp(const std::string& path) const void Store::set_dirstamp(const std::string& path, time_t tstamp) { - LOCKED; + std::lock_guard guard{priv_->lock_}; std::array data{}; - const auto len = static_cast(g_snprintf(data.data(), data.size(), "%zx", tstamp)); + + const auto len = static_cast( + g_snprintf(data.data(), data.size(), "%zx", tstamp)); xapian_try([&] { priv_->writable_db().set_metadata(path, std::string{data.data(), len}); @@ -522,10 +520,11 @@ Store::find_message(unsigned docid) const { return xapian_try( [&] { - LOCKED; + std::lock_guard guard{priv_->lock_}; Xapian::Document* doc{new Xapian::Document{priv_->db().get_document(docid)}}; GError* gerr{}; - auto msg{mu_msg_new_from_doc(reinterpret_cast(doc), &gerr)}; + auto msg{mu_msg_new_from_doc( + reinterpret_cast(doc), &gerr)}; if (!msg) { g_warning("could not create message: %s", gerr ? gerr->message : "something went wrong"); @@ -541,7 +540,7 @@ Store::contains_message(const std::string& path) const { return xapian_try( [&] { - LOCKED; + std::lock_guard guard{priv_->lock_}; const std::string term(get_uid_term(path.c_str())); return priv_->db().term_exists(term); }, @@ -554,8 +553,9 @@ Store::for_each_message_path(Store::ForEachMessageFunc msg_func) const size_t n{}; xapian_try([&] { - LOCKED; + std::lock_guard guard{priv_->lock_}; Xapian::Enquire enq{priv_->db()}; + enq.set_query(Xapian::Query::MatchAll); enq.set_cutoff(0, 0); @@ -571,7 +571,7 @@ Store::for_each_message_path(Store::ForEachMessageFunc msg_func) const void Store::commit() { - LOCKED; + std::lock_guard guard{priv_->lock_}; priv_->transaction_maybe_commit(true /*force*/); } @@ -596,8 +596,8 @@ Store::for_each_term(const std::string& field, Store::ForEachTermFunc func) cons size_t n{}; xapian_try([&] { - LOCKED; - const auto id = field_id(field.c_str()); + std::lock_guard guard{priv_->lock_}; + const auto id = field_id(field.c_str()); if (id == MU_MSG_FIELD_ID_NONE) return; @@ -613,6 +613,43 @@ Store::for_each_term(const std::string& field, Store::ForEachTermFunc func) cons return n; } +Option +Store::run_query(const std::string& expr, MuMsgFieldId sortfieldid, + QueryFlags flags, size_t maxnum) const +{ + return xapian_try([&] { + std::lock_guard guard{priv_->lock_}; + Query q{*this}; + + return q.run(expr, sortfieldid, flags, maxnum); + }, + Nothing); +} + +size_t +Store::count_query(const std::string& expr) const +{ + return xapian_try([&] { + std::lock_guard guard{priv_->lock_}; + Query q{*this}; + + return q.count(expr); + }, + 0); +} + +std::string +Store::parse_query(const std::string& expr, bool xapian) const +{ + return xapian_try([&] { + std::lock_guard guard{priv_->lock_}; + Query q{*this}; + + return q.parse(expr, xapian); + }, + std::string{}); +} + static void add_terms_values_date(Xapian::Document& doc, MuMsg* msg, MuMsgFieldId mfid) { @@ -884,8 +921,12 @@ add_terms_values(MuMsgFieldId mfid, MsgDoc* msgdoc) return; switch (mfid) { - case MU_MSG_FIELD_ID_DATE: add_terms_values_date(*msgdoc->_doc, msgdoc->_msg, mfid); break; - case MU_MSG_FIELD_ID_SIZE: add_terms_values_size(*msgdoc->_doc, msgdoc->_msg, mfid); break; + case MU_MSG_FIELD_ID_DATE: + add_terms_values_date(*msgdoc->_doc, msgdoc->_msg, mfid); + break; + case MU_MSG_FIELD_ID_SIZE: + add_terms_values_size(*msgdoc->_doc, msgdoc->_msg, mfid); + break; case MU_MSG_FIELD_ID_BODY_TEXT: add_terms_values_body(*msgdoc->_doc, msgdoc->_msg, mfid); break; @@ -895,10 +936,13 @@ add_terms_values(MuMsgFieldId mfid, MsgDoc* msgdoc) add_terms_values_attach(*msgdoc->_doc, msgdoc->_msg, mfid); break; case MU_MSG_FIELD_ID_MIME: - case MU_MSG_FIELD_ID_EMBEDDED_TEXT: break; + case MU_MSG_FIELD_ID_EMBEDDED_TEXT: + break; case MU_MSG_FIELD_ID_THREAD_ID: - case MU_MSG_FIELD_ID_UID: break; /* already taken care of elsewhere */ - default: return add_terms_values_default(mfid, msgdoc); + case MU_MSG_FIELD_ID_UID: + break; /* already taken care of elsewhere */ + default: + return add_terms_values_default(mfid, msgdoc); } } @@ -909,11 +953,17 @@ xapian_pfx(MuMsgContact* contact) /* use ptr to string to prevent copy... */ switch (contact->type) { - case MU_MSG_CONTACT_TYPE_TO: return prefix(MU_MSG_FIELD_ID_TO); - case MU_MSG_CONTACT_TYPE_FROM: return prefix(MU_MSG_FIELD_ID_FROM); - case MU_MSG_CONTACT_TYPE_CC: return prefix(MU_MSG_FIELD_ID_CC); - case MU_MSG_CONTACT_TYPE_BCC: return prefix(MU_MSG_FIELD_ID_BCC); - default: g_warning("unsupported contact type %u", (unsigned)contact->type); return empty; + case MU_MSG_CONTACT_TYPE_TO: + return prefix(MU_MSG_FIELD_ID_TO); + case MU_MSG_CONTACT_TYPE_FROM: + return prefix(MU_MSG_FIELD_ID_FROM); + case MU_MSG_CONTACT_TYPE_CC: + return prefix(MU_MSG_FIELD_ID_CC); + case MU_MSG_CONTACT_TYPE_BCC: + return prefix(MU_MSG_FIELD_ID_BCC); + default: + g_warning("unsupported contact type %u", (unsigned)contact->type); + return empty; } } diff --git a/lib/mu-store.hh b/lib/mu-store.hh index 2722ec35..f79fe930 100644 --- a/lib/mu-store.hh +++ b/lib/mu-store.hh @@ -32,6 +32,8 @@ #include #include +#include +#include namespace Mu { @@ -135,6 +137,43 @@ public: */ Indexer& indexer(); + /** + * Run a query; see the `mu-query` man page for the syntax. + * + * @param expr the search expression + * @param sortfieldid the sortfield-id. If the field is NONE, sort by DATE + * @param flags query flags + * @param maxnum maximum number of results to return. 0 for 'no limit' + * + * @return the query-results, or Nothing in case of error. + */ + Option run_query(const std::string& expr = "", + MuMsgFieldId sortfieldid = MU_MSG_FIELD_ID_NONE, + QueryFlags flags = QueryFlags::None, + size_t maxnum = 0) const; + + /** + * run a Xapian query merely to count the number of matches; for the + * syntax, please refer to the mu-query manpage + * + * @param expr the search expression; use "" to match all messages + * + * @return the number of matches + */ + size_t count_query(const std::string& expr = "") const; + + /** + * For debugging, get the internal string representation of the parsed + * query + * + * @param expr a xapian search expression + * @param xapian if true, show Xapian's internal representation, + * otherwise, mu's. + + * @return the string representation of the query + */ + std::string parse_query(const std::string& expr, bool xapian) const; + /** * Add a message to the store. When planning to write many messages, * it's much faster to do so in a transaction. If so, set