From e51240f43f44565bface74ec8b1b16b97f035e42 Mon Sep 17 00:00:00 2001 From: "Dirk-Jan C. Binnema" Date: Fri, 31 Jan 2020 00:20:34 +0200 Subject: [PATCH] mu: lib: update store API. update some dependents Implement add_message / remove_message. Rename path_tstamp -> dirstamp. Rename maildir -> root_maildir Update dependents. --- lib/Makefile.am | 5 +- lib/mu-index.c | 8 +- lib/mu-maildir.c | 1 + lib/mu-store.cc | 175 ++++++++++++++++++++++++++++++++------------ lib/mu-store.hh | 78 +++++++++++++------- lib/test-mu-store.c | 8 +- 6 files changed, 192 insertions(+), 83 deletions(-) diff --git a/lib/Makefile.am b/lib/Makefile.am index 6b9c7525..fdc040c0 100644 --- a/lib/Makefile.am +++ b/lib/Makefile.am @@ -41,7 +41,7 @@ AM_CFLAGS= \ -DMU_TESTMAILDIR3=\"${abs_srcdir}/testdir3\" \ -DMU_TESTMAILDIR4=\"${abs_srcdir}/testdir4\" \ -DABS_CURDIR=\"${abs_builddir}\" \ - -DABS_SRCDIR=\"${abs_srcdir}\" + -DABS_SRCDIR=\"${abs_srcdir}\" \ -Wno-format-nonliteral \ -Wno-switch-enum \ -Wno-deprecated-declarations \ @@ -56,7 +56,8 @@ AM_CXXFLAGS= \ $(WARN_CXXFLAGS) \ $(XAPIAN_CXXFLAGS) \ $(ASAN_CXXFLAGS) \ - $(CODE_COVERAGE_CFLAGS) + $(CODE_COVERAGE_CFLAGS) \ + -DMU_TESTMAILDIR=\"${abs_srcdir}/testdir\" AM_CPPFLAGS= \ $(CODE_COVERAGE_CPPFLAGS) diff --git a/lib/mu-index.c b/lib/mu-index.c index b789c88c..670588f0 100644 --- a/lib/mu-index.c +++ b/lib/mu-index.c @@ -112,8 +112,8 @@ needs_index (MuIndexCallbackData *data, const char *fullpath, if (data->_reindex) return TRUE; - /* it's not in the database yet (FIXME: GError)*/ - if (!mu_store_contains_message (data->_store, fullpath, NULL)) + /* it's not in the database yet */ + if (!mu_store_contains_message (data->_store, fullpath)) return TRUE; /* it's there, but it's not up to date */ @@ -242,7 +242,7 @@ on_run_maildir_dir (const char* fullpath, gboolean enter, */ if (enter) { data->_dirstamp = - mu_store_get_timestamp (data->_store, fullpath, &err); + mu_store_get_dirstamp (data->_store, fullpath, &err); /* in 'lazy' mode, we only check the dir timestamp, and if it's * up to date, we don't bother with this dir. This fails to * account for messages below this dir that have merely @@ -257,7 +257,7 @@ on_run_maildir_dir (const char* fullpath, gboolean enter, } g_debug ("entering %s", fullpath); } else { - mu_store_set_timestamp (data->_store, fullpath, + mu_store_set_dirstamp (data->_store, fullpath, time(NULL), &err); g_debug ("leaving %s", fullpath); } diff --git a/lib/mu-maildir.c b/lib/mu-maildir.c index 03724fae..c8c74f41 100644 --- a/lib/mu-maildir.c +++ b/lib/mu-maildir.c @@ -19,6 +19,7 @@ ** */ + #if HAVE_CONFIG_H #include "config.h" #endif /*HAVE_CONFIG_H*/ diff --git a/lib/mu-store.cc b/lib/mu-store.cc index 139f2a74..0f6bd804 100644 --- a/lib/mu-store.cc +++ b/lib/mu-store.cc @@ -36,7 +36,7 @@ using namespace Mu; constexpr auto SchemaVersionKey = "schema-version"; -constexpr auto MaildirKey = "maildir"; +constexpr auto RootMaildirKey = "maildir"; // XXX: make this 'root-maildir' constexpr auto ContactsKey = "contacts"; constexpr auto PersonalAddressesKey = "personal-addresses"; constexpr auto CreatedKey = "created"; @@ -45,6 +45,10 @@ constexpr auto ExpectedSchemaVersion = MU_STORE_SCHEMA_VERSION; using Addresses = Store::Addresses; +extern "C" { +static unsigned add_or_update_msg (MuStore *store, unsigned docid, MuMsg *msg, GError **err); +} + /* we cache these prefix strings, so we don't have to allocate them all * the time; this should save 10-20 string allocs per message */ G_GNUC_CONST static const std::string& @@ -97,23 +101,23 @@ struct Store::Private { db_{readonly? std::make_shared(db_path_) : std::make_shared(db_path_, Xapian::DB_OPEN)}, - maildir_{db()->get_metadata(MaildirKey)}, + root_maildir_{db()->get_metadata(RootMaildirKey)}, created_{atoll(db()->get_metadata(CreatedKey).c_str())}, schema_version_{db()->get_metadata(SchemaVersionKey)}, personal_addresses_{Mu::split(db()->get_metadata(PersonalAddressesKey),",")}, contacts_{db()->get_metadata(ContactsKey)} { } - Private (const std::string& path, const std::string& maildir): + Private (const std::string& path, const std::string& root_maildir): db_path_{path}, db_{std::make_shared( db_path_, Xapian::DB_CREATE_OR_OVERWRITE)}, - maildir_{maildir}, + root_maildir_{root_maildir}, created_{time({})}, schema_version_{MU_STORE_SCHEMA_VERSION} { writable_db()->set_metadata(SchemaVersionKey, schema_version_); - writable_db()->set_metadata(MaildirKey, maildir_); + writable_db()->set_metadata(RootMaildirKey, root_maildir_); writable_db()->set_metadata(CreatedKey, Mu::format("%" PRId64, (int64_t)created_)); } @@ -174,7 +178,7 @@ struct Store::Private { const std::string db_path_; std::shared_ptr db_; - const std::string maildir_; + const std::string root_maildir_; const time_t created_{}; const std::string schema_version_; Addresses personal_addresses_; @@ -187,6 +191,24 @@ struct Store::Private { }; +static void +hash_str (char *buf, size_t buf_size, const char *data) +{ + g_snprintf(buf, buf_size, "016%" PRIx64, mu_util_get_hash(data)); +} + + +static std::string +get_uid_term (const char* path) +{ + char uid_term[1 + 16 + 1] = {'\0'}; + uid_term[0] = mu_msg_field_xapian_prefix(MU_MSG_FIELD_ID_UID); + hash_str(uid_term + 1, sizeof(uid_term)-1, path); + + return std::string{uid_term, sizeof(uid_term)}; +} + + #undef LOCKED #define LOCKED std::lock_guard l(priv_->lock_); @@ -196,15 +218,15 @@ Store::Store (const std::string& path, bool readonly): if (ExpectedSchemaVersion == schema_version()) return; // All is good; nothing further to do - if (readonly || maildir().empty()) + if (readonly || root_maildir().empty()) throw Mu::Error(Error::Code::SchemaMismatch, "database needs reindexing"); g_debug ("upgrading database"); const auto addresses{personal_addresses()}; - const auto mdir{maildir()}; + const auto root_mdir{root_maildir()}; priv_.reset(); - priv_ = std::make_unique (path, mdir); + priv_ = std::make_unique (path, root_mdir); set_personal_addresses (addresses); } @@ -222,10 +244,10 @@ Store::read_only() const } const std::string& -Store::maildir () const +Store::root_maildir () const { LOCKED; - return priv_->maildir_; + return priv_->root_maildir_; } void @@ -287,8 +309,77 @@ Store::created() const return priv_->created_; } +static std::string +maildir_from_path (const std::string& root, const std::string& path) +{ + if (G_UNLIKELY(root.empty()) || root.length() >= path.length() || + path.find(root) != 0) + throw Mu::Error{Error::Code::InvalidArgument, + "root '%s' is not a proper suffix of path '%s'", + root.c_str(), path.c_str()}; + + auto mdir{path.substr(root.length())}; + auto slash{mdir.rfind('/')}; + + if (G_UNLIKELY(slash == std::string::npos) || slash < 4) + throw Mu::Error{Error::Code::InvalidArgument, + "invalid path: %s", path.c_str()}; + mdir.erase(slash); + auto subdir=mdir.data()+slash-4; + if (G_UNLIKELY(strncmp(subdir, "/cur", 4) != 0 && + strncmp(subdir, "/new", 4))) + throw Mu::Error{Error::Code::InvalidArgument, + "cannot find '/new' or '/cur' - invalid path: %s", path.c_str()}; + if (mdir.length() == 4) + return "/"; + + mdir.erase(mdir.length()-4); + return mdir; +} + + +unsigned +Store::add_message (const std::string& path) +{ + LOCKED; + + GError *gerr{}; + const auto maildir{ maildir_from_path(root_maildir(), path)}; + auto msg{mu_msg_new_from_file (path.c_str(), maildir.c_str(), &gerr)}; + if (G_UNLIKELY(!msg)) + throw Error{Error::Code::Message, "failed to create message: %s", + gerr ? gerr->message : "something went wrong"}; + + auto store{reinterpret_cast(this)}; // yuk. + const auto docid{add_or_update_msg (store, 0, msg, &gerr)}; + mu_msg_unref (msg); + if (G_UNLIKELY(docid == MU_STORE_INVALID_DOCID)) + throw Error{Error::Code::Message, "failed to store message: %s", + gerr ? gerr->message : "something went wrong"}; + + return docid; +} + +bool +Store::remove_message (const std::string& path) +{ + LOCKED; + + try { + const std::string term{(get_uid_term(path.c_str()))}; + auto wdb{priv()->wdb()}; + + wdb->delete_document (term); + + } MU_XAPIAN_CATCH_BLOCK_RETURN (false); + + return true; +} + + + time_t -Store::path_tstamp (const std::string& path) const +Store::dirstamp (const std::string& path) const { LOCKED; @@ -300,16 +391,29 @@ Store::path_tstamp (const std::string& path) const } void -Store::set_path_tstamp (const std::string& path, time_t tstamp) +Store::set_dirstamp (const std::string& path, time_t tstamp) { LOCKED; std::array data{}; - const std::size_t len = g_snprintf (data.data(), data.size(), "%x", tstamp); + const std::size_t len = g_snprintf (data.data(), data.size(), "%zx", tstamp); priv_->writable_db()->set_metadata(path, std::string{data.data(), len}); } + +bool +Store::contains_message (const std::string& path) const +{ + LOCKED; + + try { + const std::string term (get_uid_term(path.c_str())); + return priv_->db()->term_exists (term); + + } MU_XAPIAN_CATCH_BLOCK_RETURN(false); +} + void Store::begin_transaction () try { @@ -386,24 +490,6 @@ mutable_self (MuStore *store) } -static void -hash_str (char *buf, size_t buf_size, const char *data) -{ - g_snprintf(buf, buf_size, "016%" PRIx64, mu_util_get_hash(data)); -} - - -static std::string -get_uid_term (const char* path) -{ - char uid_term[1 + 16 + 1] = {'\0'}; - uid_term[0] = mu_msg_field_xapian_prefix(MU_MSG_FIELD_ID_UID); - hash_str(uid_term + 1, sizeof(uid_term)-1, path); - - return std::string{uid_term, sizeof(uid_term)}; -} - - MuStore* mu_store_new_readable (const char* xpath, GError **err) { @@ -579,16 +665,15 @@ mu_store_get_read_only_database (MuStore *store) } gboolean -mu_store_contains_message (const MuStore *store, const char* path, GError **err) +mu_store_contains_message (const MuStore *store, const char* path) { g_return_val_if_fail (store, FALSE); g_return_val_if_fail (path, FALSE); try { - const std::string term (get_uid_term(path)); - return self(store)->priv()->db()->term_exists (term) ? TRUE: FALSE; + return self(store)->contains_message(path) ? TRUE : FALSE; - } MU_XAPIAN_CATCH_BLOCK_G_ERROR_RETURN(err, MU_ERROR_XAPIAN, FALSE); + } MU_XAPIAN_CATCH_BLOCK_RETURN(FALSE); } unsigned @@ -679,7 +764,7 @@ mu_store_maildir (const MuStore *store) { g_return_val_if_fail (store, NULL); - return self(store)->maildir().c_str(); + return self(store)->root_maildir().c_str(); } @@ -1222,7 +1307,7 @@ add_or_update_msg (MuStore *store, unsigned docid, MuMsg *msg, GError **err) const std::string term (get_uid_term (mu_msg_get_path(msg))); auto self = mutable_self(store); - auto wdb = self->priv()->wdb(); + auto wdb = self->priv()->wdb(); if (!self->in_transaction()) self->begin_transaction(); @@ -1324,24 +1409,24 @@ mu_store_remove_path (MuStore *store, const char *msgpath) gboolean -mu_store_set_timestamp (MuStore *store, const char* msgpath, - time_t stamp, GError **err) +mu_store_set_dirstamp (MuStore *store, const char* dirpath, + time_t stamp, GError **err) { g_return_val_if_fail (store, FALSE); - g_return_val_if_fail (msgpath, FALSE); + g_return_val_if_fail (dirpath, FALSE); - mutable_self(store)->set_path_tstamp(msgpath, stamp); + mutable_self(store)->set_dirstamp(dirpath, stamp); return TRUE; } time_t -mu_store_get_timestamp (const MuStore *store, const char *msgpath, GError **err) +mu_store_get_dirstamp (const MuStore *store, const char *dirpath, GError **err) { g_return_val_if_fail (store, 0); - g_return_val_if_fail (msgpath, 0); + g_return_val_if_fail (dirpath, 0); - return self(store)->path_tstamp(msgpath); + return self(store)->dirstamp(dirpath); } diff --git a/lib/mu-store.hh b/lib/mu-store.hh index 77f14f75..72a28257 100644 --- a/lib/mu-store.hh +++ b/lib/mu-store.hh @@ -75,7 +75,7 @@ public: * * @return the maildir */ - const std::string& maildir() const; + const std::string& root_maildir() const; /** * Version of the database-schema @@ -117,6 +117,51 @@ public: */ const Contacts& contacts() const; + /** + * Add a message to the store. + * + * @param path the message path. + * + * @return the doc id of the added message + */ + unsigned add_message (const std::string& path); + + /** + * Add a message to the store. + * + * @param path the message path. + * + * @return true if removing happened; false otherwise. + */ + bool remove_message (const std::string& path); + + + /** + * does a certain message exist in the store already? + * + * @param path the message path + * + * @return true if the message exists in the store, false otherwise + */ + bool contains_message (const std::string& path) const; + + /** + * Get the timestamp for some directory + * + * @param path the path + * + * @return the timestamp, or 0 if not found + */ + time_t dirstamp (const std::string& path) const; + + /** + * Set the timestamp for some directory + * + * @param path a filesystem path + * @param tstamp the timestamp for that path + */ + void set_dirstamp (const std::string& path, time_t tstamp); + /** * Get the number of documents in the document database * @@ -131,24 +176,6 @@ public: */ bool empty() const; - /** - * Get the timestamp for a given path. - * - * @param path the path - * - * @return the timestamp, or 0 if not found - */ - time_t path_tstamp (const std::string& path) const; - - /** - * Set the timestamp for some path - * - * @param path a filesystem path - * @param tstamp the timestamp for that path - */ - void set_path_tstamp (const std::string& path, time_t tstamp); - - /** * Begin a database transaction */ @@ -451,18 +478,15 @@ unsigned mu_store_add_path (MuStore *store, const char *path, */ gboolean mu_store_remove_path (MuStore *store, const char* msgpath); - /** * does a certain message exist in the database already? * * @param store a store * @param path the message path - * @param err to receive error info or NULL. err->code is MuError value * * @return TRUE if the message exists, FALSE otherwise */ -gboolean mu_store_contains_message (const MuStore *store, const char* path, - GError **err); +gboolean mu_store_contains_message (const MuStore *store, const char* path); /** * get the docid for message at path @@ -480,25 +504,25 @@ unsigned mu_store_get_docid_for_path (const MuStore *store, const char* path, * store a timestamp for a directory * * @param store a valid store - * @param msgpath path to a maildir + * @param dirpath path to some directory * @param stamp a timestamp * @param err to receive error info or NULL. err->code is MuError value * * @return TRUE if setting the timestamp succeeded, FALSE otherwise */ -gboolean mu_store_set_timestamp (MuStore *store, const char* msgpath, +gboolean mu_store_set_dirstamp (MuStore *store, const char* dirpath, time_t stamp, GError **err); /** * get the timestamp for a directory * * @param store a valid store - * @param msgpath path to a maildir + * @param msgpath path to some directory * @param err to receive error info or NULL. err->code is MuError value * * @return the timestamp, or 0 in case of error */ -time_t mu_store_get_timestamp (const MuStore *store, const char* msgpath, +time_t mu_store_get_dirstamp (const MuStore *store, const char* dirpath, GError **err); /** diff --git a/lib/test-mu-store.c b/lib/test-mu-store.c index 083bcdc8..ae7bfb06 100644 --- a/lib/test-mu-store.c +++ b/lib/test-mu-store.c @@ -112,8 +112,7 @@ test_mu_store_store_msg_and_count (void) g_assert_cmpuint (1,==,mu_store_count (store, NULL)); g_assert_cmpuint (TRUE,==,mu_store_contains_message (store, - MU_TESTMAILDIR "/cur/1283599333.1840_11.cthulhu!2,", - NULL)); + MU_TESTMAILDIR "/cur/1283599333.1840_11.cthulhu!2,")); mu_msg_unref (msg); /* add another one */ @@ -125,7 +124,7 @@ test_mu_store_store_msg_and_count (void) g_assert_cmpuint (2,==,mu_store_count (store, NULL)); g_assert_cmpuint (TRUE,==, mu_store_contains_message (store, MU_TESTMAILDIR2 - "/bar/cur/mail3", NULL)); + "/bar/cur/mail3")); mu_msg_unref (msg); /* try to add the first one again. count should be 2 still */ @@ -175,8 +174,7 @@ test_mu_store_store_msg_remove_and_count (void) g_assert_cmpuint (0,==,mu_store_count (store, NULL)); g_assert_cmpuint (FALSE,==,mu_store_contains_message (store, - MU_TESTMAILDIR "/cur/1283599333.1840_11.cthulhu!2,", - NULL)); + MU_TESTMAILDIR "/cur/1283599333.1840_11.cthulhu!2,")); g_free (tmpdir); mu_store_unref (store); }