mirror of https://github.com/djcb/mu.git
lib/store: rework to use xapian-db / config
Simplifies the implementation.
This commit is contained in:
parent
c28fde9155
commit
3791d0c375
|
@ -23,6 +23,7 @@ lib_mu=static_library(
|
|||
'mu',
|
||||
[
|
||||
'mu-bookmarks.cc',
|
||||
'mu-config.cc',
|
||||
'mu-contacts-cache.cc',
|
||||
'mu-maildir.cc',
|
||||
'mu-parser.cc',
|
||||
|
@ -33,7 +34,8 @@ lib_mu=static_library(
|
|||
'mu-server.cc',
|
||||
'mu-store.cc',
|
||||
'mu-tokenizer.cc',
|
||||
'mu-xapian.cc'
|
||||
'mu-xapian.cc',
|
||||
'mu-xapian-db.cc'
|
||||
],
|
||||
dependencies: [
|
||||
glib_dep,
|
||||
|
@ -66,15 +68,22 @@ tokenize = executable(
|
|||
|
||||
test('test-threads',
|
||||
executable('test-threads',
|
||||
'mu-query-threads.cc',
|
||||
install: false,
|
||||
cpp_args: ['-DBUILD_TESTS'],
|
||||
dependencies: [glib_dep, lib_mu_dep]))
|
||||
'mu-query-threads.cc',
|
||||
install: false,
|
||||
cpp_args: ['-DBUILD_TESTS'],
|
||||
dependencies: [glib_dep, lib_mu_dep]))
|
||||
test('test-contacts-cache',
|
||||
executable('test-contacts-cache',
|
||||
'mu-contacts-cache.cc',
|
||||
install: false,
|
||||
cpp_args: ['-DBUILD_TESTS'],
|
||||
dependencies: [glib_dep, lib_mu_dep]))
|
||||
'mu-contacts-cache.cc',
|
||||
install: false,
|
||||
cpp_args: ['-DBUILD_TESTS'],
|
||||
dependencies: [glib_dep, lib_mu_dep]))
|
||||
|
||||
test('test-config',
|
||||
executable('test-config',
|
||||
'mu-config.cc',
|
||||
install: false,
|
||||
cpp_args: ['-DBUILD_TESTS'],
|
||||
dependencies: [glib_dep, lib_mu_dep]))
|
||||
|
||||
subdir('tests')
|
||||
|
|
449
lib/mu-store.cc
449
lib/mu-store.cc
|
@ -1,5 +1,5 @@
|
|||
/*
|
||||
** Copyright (C) 2021-2022 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl>
|
||||
** Copyright (C) 2021-2023 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl>
|
||||
**
|
||||
** This program is free software; you can redistribute it and/or modify it
|
||||
** under the terms of the GNU General Public License as published by the
|
||||
|
@ -38,130 +38,52 @@
|
|||
#include "mu-maildir.hh"
|
||||
#include "mu-store.hh"
|
||||
#include "mu-query.hh"
|
||||
#include "mu-xapian-db.hh"
|
||||
|
||||
#include "utils/mu-error.hh"
|
||||
|
||||
#include "utils/mu-utils.hh"
|
||||
#include <utils/mu-utils-file.hh>
|
||||
#include "utils/mu-xapian-utils.hh"
|
||||
|
||||
using namespace Mu;
|
||||
|
||||
static_assert(std::is_same<Store::Id, Xapian::docid>::value, "wrong type for Store::Id");
|
||||
|
||||
// Properties
|
||||
constexpr auto SchemaVersionKey = "schema-version";
|
||||
constexpr auto RootMaildirKey = "maildir"; // XXX: make this 'root-maildir'
|
||||
constexpr auto ContactsKey = "contacts";
|
||||
constexpr auto PersonalAddressesKey = "personal-addresses";
|
||||
constexpr auto CreatedKey = "created";
|
||||
constexpr auto BatchSizeKey = "batch-size";
|
||||
constexpr auto DefaultBatchSize = 250'000U;
|
||||
|
||||
constexpr auto MaxMessageSizeKey = "max-message-size";
|
||||
constexpr auto DefaultMaxMessageSize = 100'000'000U;
|
||||
constexpr auto ExpectedSchemaVersion = MU_STORE_SCHEMA_VERSION;
|
||||
|
||||
// Stats.
|
||||
constexpr auto ChangedKey = "changed";
|
||||
constexpr auto IndexedKey = "indexed";
|
||||
|
||||
|
||||
static std::string
|
||||
tstamp_to_string(::time_t t)
|
||||
{
|
||||
char buf[17];
|
||||
::snprintf(buf, sizeof(buf), "%" PRIx64, static_cast<int64_t>(t));
|
||||
return std::string(buf);
|
||||
}
|
||||
|
||||
static ::time_t
|
||||
string_to_tstamp(const std::string& str)
|
||||
{
|
||||
return static_cast<::time_t>(::strtoll(str.c_str(), {}, 16));
|
||||
}
|
||||
|
||||
struct Store::Private {
|
||||
enum struct XapianOpts { ReadOnly, Open, CreateOverwrite };
|
||||
|
||||
Private(const std::string& path, bool readonly)
|
||||
: read_only_{readonly}, db_{make_xapian_db(path,
|
||||
read_only_ ? XapianOpts::ReadOnly
|
||||
: XapianOpts::Open)},
|
||||
properties_{make_properties(path)},
|
||||
contacts_cache_{db().get_metadata(ContactsKey),
|
||||
properties_.personal_addresses} {
|
||||
}
|
||||
Private(const std::string& path, bool readonly):
|
||||
xapian_db_{make_db(path, readonly ? XapianDb::Flavor::ReadOnly
|
||||
: XapianDb::Flavor::Open)},
|
||||
config_{xapian_db_},
|
||||
contacts_cache_{config_},
|
||||
root_maildir_{config_.get<Config::Id::RootMaildir>()}
|
||||
{}
|
||||
|
||||
Private(const std::string& path,
|
||||
const std::string& root_maildir,
|
||||
const StringVec& personal_addresses,
|
||||
const Store::Config& conf)
|
||||
: read_only_{false}, db_{make_xapian_db(path, XapianOpts::CreateOverwrite)},
|
||||
properties_{init_metadata(conf, path, root_maildir, personal_addresses)},
|
||||
contacts_cache_{"", properties_.personal_addresses} {
|
||||
}
|
||||
Private(const std::string& path, const std::string& root_maildir,
|
||||
Option<const Config&> conf):
|
||||
xapian_db_{make_db(path, XapianDb::Flavor::CreateOverwrite)},
|
||||
config_{make_config(xapian_db_, root_maildir, conf)},
|
||||
contacts_cache_{config_},
|
||||
root_maildir_{config_.get<Config::Id::RootMaildir>()}
|
||||
{}
|
||||
|
||||
~Private() try {
|
||||
|
||||
g_debug("closing store @ %s", properties_.database_path.c_str());
|
||||
if (!read_only_) {
|
||||
g_debug("closing store @ %s", xapian_db_.path().c_str());
|
||||
if (!xapian_db_.read_only()) {
|
||||
transaction_maybe_commit(true /*force*/);
|
||||
}
|
||||
} catch (...) {
|
||||
g_critical("caught exception in store dtor");
|
||||
}
|
||||
|
||||
std::unique_ptr<Xapian::Database> make_xapian_db(const std::string db_path, XapianOpts opts)
|
||||
try {
|
||||
/* we do our own flushing, set Xapian's internal one as the
|
||||
* backstop*/
|
||||
g_setenv("XAPIAN_FLUSH_THRESHOLD", "500000", 1);
|
||||
|
||||
if (g_mkdir_with_parents(db_path.c_str(), 0700) != 0)
|
||||
throw Mu::Error(Error::Code::Internal,
|
||||
"failed to create database dir %s: %s",
|
||||
db_path.c_str(), ::strerror(errno));
|
||||
|
||||
switch (opts) {
|
||||
case XapianOpts::ReadOnly:
|
||||
return std::make_unique<Xapian::Database>(db_path);
|
||||
case XapianOpts::Open:
|
||||
return std::make_unique<Xapian::WritableDatabase>(db_path, Xapian::DB_OPEN);
|
||||
case XapianOpts::CreateOverwrite:
|
||||
return std::make_unique<Xapian::WritableDatabase>(
|
||||
db_path,
|
||||
Xapian::DB_CREATE_OR_OVERWRITE);
|
||||
default:
|
||||
throw std::logic_error("invalid xapian options");
|
||||
}
|
||||
|
||||
} catch (const Xapian::DatabaseLockError& xde) {
|
||||
throw Mu::Error(Error::Code::StoreLock,
|
||||
"%s", xde.get_msg().c_str());
|
||||
} catch (const Xapian::DatabaseError& xde) {
|
||||
throw Mu::Error(Error::Code::Store,
|
||||
"%s", xde.get_msg().c_str());
|
||||
} catch (const Mu::Error& me) {
|
||||
throw;
|
||||
} catch (...) {
|
||||
throw Mu::Error(Error::Code::Internal,
|
||||
"something went wrong when opening store @ %s",
|
||||
db_path.c_str());
|
||||
}
|
||||
|
||||
const Xapian::Database& db() const { return *db_.get(); }
|
||||
|
||||
Xapian::WritableDatabase& writable_db() {
|
||||
if (read_only_)
|
||||
throw Mu::Error(Error::Code::AccessDenied, "database is read-only");
|
||||
return dynamic_cast<Xapian::WritableDatabase&>(*db_.get());
|
||||
}
|
||||
|
||||
// If not started yet, start a transaction. Otherwise, just update the transaction size.
|
||||
void transaction_inc() noexcept {
|
||||
if (transaction_size_ == 0) {
|
||||
g_debug("starting transaction");
|
||||
xapian_try([this] { writable_db().begin_transaction(); });
|
||||
xapian_db_.begin_transaction();
|
||||
}
|
||||
++transaction_size_;
|
||||
}
|
||||
|
@ -169,80 +91,46 @@ struct Store::Private {
|
|||
// Opportunistically commit a transaction if the transaction size
|
||||
// filled up a batch, or with force.
|
||||
void transaction_maybe_commit(bool force = false) noexcept {
|
||||
if (force || transaction_size_ >= properties_.batch_size) {
|
||||
if (contacts_cache_.dirty()) {
|
||||
xapian_try([&] {
|
||||
writable_db().set_metadata(ContactsKey,
|
||||
contacts_cache_.serialize());
|
||||
});
|
||||
}
|
||||
static auto batch_size = config_.get<Config::Id::BatchSize>();
|
||||
if (force || transaction_size_ >= batch_size) {
|
||||
contacts_cache_.serialize();
|
||||
|
||||
if (indexer_) { // save last index time.
|
||||
if (indexer_) // save last index time.
|
||||
if (auto&& t{indexer_->completed()}; t != 0)
|
||||
writable_db().set_metadata(
|
||||
IndexedKey, tstamp_to_string(t));
|
||||
}
|
||||
config_.set<Config::Id::LastIndex>(::time({}));
|
||||
|
||||
if (transaction_size_ == 0)
|
||||
return; // nothing more to do here.
|
||||
|
||||
g_debug("committing transaction (n=%zu,%zu)",
|
||||
transaction_size_, metadata_cache_.size());
|
||||
xapian_try([this] {
|
||||
writable_db().commit_transaction();
|
||||
for (auto&& mdata : metadata_cache_)
|
||||
writable_db().set_metadata(mdata.first, mdata.second);
|
||||
transaction_size_ = 0;
|
||||
});
|
||||
g_debug("committing transaction (n=%zu)", transaction_size_);
|
||||
xapian_db_.commit_transaction();
|
||||
transaction_size_ = 0;
|
||||
}
|
||||
}
|
||||
|
||||
time_t metadata_time_t(const std::string& key) const {
|
||||
const auto ts = db().get_metadata(key);
|
||||
return (time_t)atoll(db().get_metadata(key).c_str());
|
||||
return static_cast<time_t>(::atoll(xapian_db_.metadata(key).c_str()));
|
||||
}
|
||||
|
||||
Store::Properties make_properties(const std::string& db_path) {
|
||||
Store::Properties props;
|
||||
|
||||
props.database_path = db_path;
|
||||
props.schema_version = db().get_metadata(SchemaVersionKey);
|
||||
props.created = string_to_tstamp(db().get_metadata(CreatedKey));
|
||||
props.batch_size = ::atoll(db().get_metadata(BatchSizeKey).c_str());
|
||||
props.max_message_size = ::atoll(db().get_metadata(MaxMessageSizeKey).c_str());
|
||||
props.root_maildir = db().get_metadata(RootMaildirKey);
|
||||
props.personal_addresses = Mu::split(db().get_metadata(PersonalAddressesKey), ",");
|
||||
|
||||
return props;
|
||||
XapianDb make_db(const std::string& path, XapianDb::Flavor flavor) {
|
||||
if (auto&& res{XapianDb::make(path, flavor)}; res)
|
||||
return std::move(res.value());
|
||||
else
|
||||
throw res.error();
|
||||
}
|
||||
|
||||
Store::Properties init_metadata(const Store::Config& conf,
|
||||
const std::string& path,
|
||||
const std::string& root_maildir,
|
||||
const StringVec& personal_addresses) {
|
||||
Config make_config(XapianDb& xapian_db, const std::string& root_maildir,
|
||||
Option<const Config&> conf) {
|
||||
|
||||
writable_db().set_metadata(SchemaVersionKey, ExpectedSchemaVersion);
|
||||
writable_db().set_metadata(CreatedKey, tstamp_to_string(::time({})));
|
||||
Config config{xapian_db};
|
||||
|
||||
const size_t batch_size = conf.batch_size ? conf.batch_size : DefaultBatchSize;
|
||||
writable_db().set_metadata(BatchSizeKey, Mu::format("%zu", batch_size));
|
||||
const size_t max_msg_size = conf.max_message_size ? conf.max_message_size
|
||||
: DefaultMaxMessageSize;
|
||||
writable_db().set_metadata(MaxMessageSizeKey, Mu::format("%zu", max_msg_size));
|
||||
if (conf)
|
||||
config.import_configurable(*conf);
|
||||
|
||||
writable_db().set_metadata(RootMaildirKey, canonicalize_filename(root_maildir, {}));
|
||||
config.set<Config::Id::RootMaildir>(root_maildir);
|
||||
config.set<Config::Id::SchemaVersion>(ExpectedSchemaVersion);
|
||||
|
||||
std::string addrs;
|
||||
for (const auto& addr : personal_addresses) { // _very_ minimal check.
|
||||
if (addr.find(",") != std::string::npos)
|
||||
throw Mu::Error(Error::Code::InvalidArgument,
|
||||
"e-mail address '%s' contains comma",
|
||||
addr.c_str());
|
||||
addrs += (addrs.empty() ? "" : ",") + addr;
|
||||
}
|
||||
writable_db().set_metadata(PersonalAddressesKey, addrs);
|
||||
|
||||
return make_properties(path);
|
||||
return config;
|
||||
}
|
||||
|
||||
Option<Message> find_message_unlocked(Store::Id docid) const;
|
||||
|
@ -252,56 +140,43 @@ struct Store::Private {
|
|||
Option<const std::string&> target_mdir,
|
||||
Option<Flags> new_flags,
|
||||
MoveOptions opts);
|
||||
|
||||
/* metadata to write as part of a transaction commit */
|
||||
std::unordered_map<std::string, std::string> metadata_cache_;
|
||||
|
||||
const bool read_only_{};
|
||||
std::unique_ptr<Xapian::Database> db_;
|
||||
|
||||
const Store::Properties properties_;
|
||||
XapianDb xapian_db_;
|
||||
Config config_;
|
||||
ContactsCache contacts_cache_;
|
||||
std::unique_ptr<Indexer> indexer_;
|
||||
|
||||
const std::string root_maildir_;
|
||||
|
||||
size_t transaction_size_{};
|
||||
std::mutex lock_;
|
||||
};
|
||||
|
||||
Result<Store::Id>
|
||||
Store::Private::update_message_unlocked(Message& msg, Store::Id docid)
|
||||
Result<Store::Id>Store::Private::update_message_unlocked(Message& msg, Store::Id docid)
|
||||
{
|
||||
return xapian_try_result([&]{
|
||||
writable_db().replace_document(docid, msg.document().xapian_document());
|
||||
g_debug("updated message @ %s; docid = %u", msg.path().c_str(), docid);
|
||||
//g_info("%s", msg.sexp().to_string().c_str());
|
||||
writable_db().set_metadata(ChangedKey, tstamp_to_string(::time({})));
|
||||
return Ok(std::move(docid));
|
||||
});
|
||||
xapian_db_.replace_document(docid, msg.document().xapian_document());
|
||||
g_debug("updated message @ %s; docid = %u", msg.path().c_str(), docid);
|
||||
|
||||
return Ok(std::move(docid));
|
||||
}
|
||||
|
||||
Result<Store::Id>
|
||||
Store::Private::update_message_unlocked(Message& msg, const std::string& path_to_replace)
|
||||
{
|
||||
return xapian_try_result([&]{
|
||||
auto id = writable_db().replace_document(
|
||||
field_from_id(Field::Id::Path).xapian_term(path_to_replace),
|
||||
msg.document().xapian_document());
|
||||
|
||||
writable_db().set_metadata(ChangedKey, tstamp_to_string(::time({})));
|
||||
return Ok(std::move(id));
|
||||
});
|
||||
auto id = xapian_db_.replace_document(
|
||||
field_from_id(Field::Id::Path).xapian_term(path_to_replace),
|
||||
msg.document().xapian_document());
|
||||
return Ok(std::move(id));
|
||||
}
|
||||
|
||||
Option<Message>
|
||||
Store::Private::find_message_unlocked(Store::Id docid) const
|
||||
{
|
||||
return xapian_try([&]()->Option<Message> {
|
||||
auto res = Message::make_from_document(db().get_document(docid));
|
||||
if (res)
|
||||
return Some(std::move(res.value()));
|
||||
else
|
||||
return Nothing;
|
||||
}, Nothing);
|
||||
if (auto&& doc{xapian_db_.document(docid)}; !doc)
|
||||
return Nothing;
|
||||
else if (auto&& msg{Message::make_from_document(std::move(*doc))}; !msg)
|
||||
return Nothing;
|
||||
else
|
||||
return Some(std::move(*msg));
|
||||
}
|
||||
|
||||
|
||||
|
@ -313,35 +188,37 @@ Store::Store(const std::string& path, Store::Options opts)
|
|||
throw Mu::Error(Error::Code::InvalidArgument,
|
||||
"Options::ReInit requires Options::Writable");
|
||||
|
||||
const auto s_version{config().get<Config::Id::SchemaVersion>()};
|
||||
if (any_of(opts & Store::Options::ReInit)) {
|
||||
/* user wants to re-initialize an existing store */
|
||||
Config conf{};
|
||||
conf.batch_size = properties().batch_size;
|
||||
conf.max_message_size = properties().max_message_size;
|
||||
const auto root_maildir{properties().root_maildir};
|
||||
const auto addrs{properties().personal_addresses};
|
||||
/* close the old one */
|
||||
/* don't try to recover from version with an incompatible scheme */
|
||||
if (s_version < 500)
|
||||
throw Mu::Error(Error::Code::CannotReinit,
|
||||
"old schema (%zu) is too old to re-initialize from",
|
||||
s_version);
|
||||
const auto old_root_maildir{root_maildir()};
|
||||
|
||||
MemDb mem_db;
|
||||
Config old_config(mem_db);
|
||||
old_config.import_configurable(config());
|
||||
|
||||
this->priv_.reset();
|
||||
/* and create a new one. */
|
||||
Store new_store(path, root_maildir, addrs, conf);
|
||||
/* and create a new one "in place" */
|
||||
Store new_store(path, old_root_maildir, old_config);
|
||||
this->priv_ = std::move(new_store.priv_);
|
||||
}
|
||||
|
||||
/* otherwise, the schema version should match. */
|
||||
if (properties().schema_version != ExpectedSchemaVersion)
|
||||
if (s_version != ExpectedSchemaVersion)
|
||||
throw Mu::Error(Error::Code::SchemaMismatch,
|
||||
"expected schema-version %s, but got %s",
|
||||
ExpectedSchemaVersion,
|
||||
properties().schema_version.c_str());
|
||||
"expected schema-version %zu, but got %zu",
|
||||
ExpectedSchemaVersion, s_version);
|
||||
}
|
||||
|
||||
Store::Store(const std::string& path,
|
||||
const std::string& maildir,
|
||||
const StringVec& personal_addresses,
|
||||
const Store::Config& conf)
|
||||
: priv_{std::make_unique<Private>(path, maildir, personal_addresses, conf)}
|
||||
{
|
||||
}
|
||||
Store::Store(const std::string& path,
|
||||
const std::string& root_maildir,
|
||||
Option<const Config&> conf):
|
||||
priv_{std::make_unique<Private>(path, root_maildir, conf)}
|
||||
{}
|
||||
|
||||
Store::Store(Store&& other)
|
||||
{
|
||||
|
@ -351,42 +228,59 @@ Store::Store(Store&& other)
|
|||
|
||||
Store::~Store() = default;
|
||||
|
||||
const Store::Properties&
|
||||
Store::properties() const
|
||||
{
|
||||
return priv_->properties_;
|
||||
}
|
||||
|
||||
Store::Statistics
|
||||
Store::statistics() const
|
||||
{
|
||||
Statistics stats{};
|
||||
|
||||
stats.size = size();
|
||||
stats.last_change = string_to_tstamp(priv_->db().get_metadata(ChangedKey));
|
||||
stats.last_index = string_to_tstamp(priv_->db().get_metadata(IndexedKey));
|
||||
stats.last_change = config().get<Config::Id::LastChange>();
|
||||
stats.last_index = config().get<Config::Id::LastIndex>();
|
||||
|
||||
return stats;
|
||||
}
|
||||
|
||||
const XapianDb&
|
||||
Store::xapian_db() const
|
||||
{
|
||||
return priv_->xapian_db_;
|
||||
}
|
||||
|
||||
XapianDb&
|
||||
Store::xapian_db()
|
||||
{
|
||||
return priv_->xapian_db_;
|
||||
}
|
||||
|
||||
const Config&
|
||||
Store::config() const
|
||||
{
|
||||
return priv_->config_;
|
||||
}
|
||||
|
||||
Config&
|
||||
Store::config()
|
||||
{
|
||||
return priv_->config_;
|
||||
}
|
||||
|
||||
const std::string&
|
||||
Store::root_maildir() const {
|
||||
return priv_->root_maildir_;
|
||||
}
|
||||
|
||||
const ContactsCache&
|
||||
Store::contacts_cache() const
|
||||
{
|
||||
return priv_->contacts_cache_;
|
||||
}
|
||||
|
||||
const Xapian::Database&
|
||||
Store::database() const
|
||||
{
|
||||
return priv_->db();
|
||||
}
|
||||
|
||||
Indexer&
|
||||
Store::indexer()
|
||||
{
|
||||
std::lock_guard guard{priv_->lock_};
|
||||
|
||||
if (read_only())
|
||||
if (xapian_db().read_only())
|
||||
throw Error{Error::Code::Store, "no indexer for read-only store"};
|
||||
else if (!priv_->indexer_)
|
||||
priv_->indexer_ = std::make_unique<Indexer>(*this);
|
||||
|
@ -394,20 +288,6 @@ Store::indexer()
|
|||
return *priv_->indexer_.get();
|
||||
}
|
||||
|
||||
std::size_t
|
||||
Store::size() const
|
||||
{
|
||||
std::lock_guard guard{priv_->lock_};
|
||||
return priv_->db().get_doccount();
|
||||
}
|
||||
|
||||
bool
|
||||
Store::read_only() const
|
||||
{
|
||||
return priv_->read_only_;
|
||||
}
|
||||
|
||||
|
||||
Result<Store::Id>
|
||||
Store::add_message(const std::string& path, bool use_transaction)
|
||||
{
|
||||
|
@ -423,7 +303,7 @@ Store::add_message(Message& msg, bool use_transaction)
|
|||
std::lock_guard guard{priv_->lock_};
|
||||
|
||||
const auto mdir{maildir_from_path(msg.path(),
|
||||
properties().root_maildir)};
|
||||
root_maildir())};
|
||||
if (!mdir)
|
||||
return Err(mdir.error());
|
||||
|
||||
|
@ -457,7 +337,6 @@ Store::add_message(Message& msg, bool use_transaction)
|
|||
return res;
|
||||
}
|
||||
|
||||
|
||||
Result<Store::Id>
|
||||
Store::update_message(Message& msg, Store::Id docid)
|
||||
{
|
||||
|
@ -469,18 +348,12 @@ Store::update_message(Message& msg, Store::Id docid)
|
|||
bool
|
||||
Store::remove_message(const std::string& path)
|
||||
{
|
||||
return xapian_try(
|
||||
[&] {
|
||||
std::lock_guard guard{priv_->lock_};
|
||||
const auto term{field_from_id(Field::Id::Path).xapian_term(path)};
|
||||
priv_->writable_db().delete_document(term);
|
||||
priv_->writable_db().set_metadata(
|
||||
ChangedKey, tstamp_to_string(::time({})));
|
||||
g_debug("deleted message @ %s from store", path.c_str());
|
||||
|
||||
return true;
|
||||
},
|
||||
false);
|
||||
std::lock_guard guard{priv_->lock_};
|
||||
const auto term{field_from_id(Field::Id::Path).xapian_term(path)};
|
||||
xapian_db().delete_document(term);
|
||||
config().set<Config::Id::LastChange>(::time({}));
|
||||
g_debug("deleted message @ %s from store", path.c_str());
|
||||
return true;
|
||||
}
|
||||
|
||||
void
|
||||
|
@ -490,14 +363,10 @@ Store::remove_messages(const std::vector<Store::Id>& ids)
|
|||
|
||||
priv_->transaction_inc();
|
||||
|
||||
xapian_try([&] {
|
||||
for (auto&& id : ids) {
|
||||
priv_->writable_db().delete_document(id);
|
||||
}
|
||||
priv_->writable_db().set_metadata(
|
||||
ChangedKey, tstamp_to_string(::time({})));
|
||||
});
|
||||
for (auto&& id : ids)
|
||||
xapian_db().delete_document(id);
|
||||
|
||||
config().set<Config::Id::LastChange>(::time({}));
|
||||
priv_->transaction_maybe_commit(true /*force*/);
|
||||
}
|
||||
|
||||
|
@ -534,7 +403,7 @@ Store::Private::move_message_unlocked(Message&& msg,
|
|||
|
||||
/* 1. first determine the file system path of the target */
|
||||
const auto target_path =
|
||||
maildir_determine_target(msg.path(), properties_.root_maildir,
|
||||
maildir_determine_target(msg.path(), root_maildir_,
|
||||
target_maildir, target_flags,
|
||||
any_of(opts & MoveOptions::ChangeName));
|
||||
if (!target_path)
|
||||
|
@ -650,39 +519,11 @@ Store::move_message(Store::Id id,
|
|||
return Ok(std::move(imvec));
|
||||
}
|
||||
|
||||
std::string
|
||||
Store::metadata(const std::string& key) const
|
||||
{
|
||||
// get metadata either from the (uncommitted) cache or from the store.
|
||||
|
||||
std::lock_guard guard{priv_->lock_};
|
||||
|
||||
const auto it = priv_->metadata_cache_.find(key);
|
||||
if (it != priv_->metadata_cache_.end())
|
||||
return it->second;
|
||||
else
|
||||
return xapian_try([&] {
|
||||
return priv_->db().get_metadata(key);
|
||||
}, "");
|
||||
}
|
||||
|
||||
void
|
||||
Store::set_metadata(const std::string& key, const std::string& val)
|
||||
{
|
||||
// get metadata either from the (uncommitted) cache or from the store.
|
||||
|
||||
std::lock_guard guard{priv_->lock_};
|
||||
|
||||
priv_->metadata_cache_.erase(key);
|
||||
priv_->metadata_cache_.emplace(key, val);
|
||||
}
|
||||
|
||||
|
||||
time_t
|
||||
Store::dirstamp(const std::string& path) const
|
||||
{
|
||||
constexpr auto epoch = static_cast<time_t>(0);
|
||||
const auto ts{metadata(path)};
|
||||
const auto ts{xapian_db().metadata(path)};
|
||||
if (ts.empty())
|
||||
return epoch;
|
||||
else
|
||||
|
@ -696,19 +537,13 @@ Store::set_dirstamp(const std::string& path, time_t tstamp)
|
|||
const auto len = static_cast<size_t>(
|
||||
g_snprintf(data.data(), data.size(), "%zx", tstamp));
|
||||
|
||||
set_metadata(path, std::string{data.data(), len});
|
||||
xapian_db().set_metadata(path, std::string{data.data(), len});
|
||||
}
|
||||
|
||||
bool
|
||||
Store::contains_message(const std::string& path) const
|
||||
{
|
||||
return xapian_try(
|
||||
[&] {
|
||||
std::lock_guard guard{priv_->lock_};
|
||||
const auto term{field_from_id(Field::Id::Path).xapian_term(path)};
|
||||
return priv_->db().term_exists(term);
|
||||
},
|
||||
false);
|
||||
return xapian_db().term_exists(field_from_id(Field::Id::Path).xapian_term(path));
|
||||
}
|
||||
|
||||
std::size_t
|
||||
|
@ -718,12 +553,12 @@ Store::for_each_message_path(Store::ForEachMessageFunc msg_func) const
|
|||
|
||||
xapian_try([&] {
|
||||
std::lock_guard guard{priv_->lock_};
|
||||
Xapian::Enquire enq{priv_->db()};
|
||||
auto enq{xapian_db().enquire()};
|
||||
|
||||
enq.set_query(Xapian::Query::MatchAll);
|
||||
enq.set_cutoff(0, 0);
|
||||
|
||||
Xapian::MSet matches(enq.get_mset(0, priv_->db().get_doccount()));
|
||||
Xapian::MSet matches(enq.get_mset(0, xapian_db().size()));
|
||||
constexpr auto path_no{field_from_id(Field::Id::Path).value_no()};
|
||||
for (auto&& it = matches.begin(); it != matches.end(); ++it, ++n)
|
||||
if (!msg_func(*it, it.get_document().get_value(path_no)))
|
||||
|
@ -744,24 +579,7 @@ Store::commit()
|
|||
std::size_t
|
||||
Store::for_each_term(Field::Id field_id, Store::ForEachTermFunc func) const
|
||||
{
|
||||
size_t n{};
|
||||
|
||||
xapian_try([&] {
|
||||
/*
|
||||
* Do _not_ take a lock; this is only called from
|
||||
* the message parser which already has the lock
|
||||
*/
|
||||
std::vector<std::string> terms;
|
||||
const auto prefix{field_from_id(field_id).xapian_term()};
|
||||
for (auto it = priv_->db().allterms_begin(prefix);
|
||||
it != priv_->db().allterms_end(prefix); ++it) {
|
||||
++n;
|
||||
if (!func(*it))
|
||||
break;
|
||||
}
|
||||
});
|
||||
|
||||
return n;
|
||||
return xapian_db().all_terms(field_from_id(field_id).xapian_term(), func);
|
||||
}
|
||||
|
||||
std::mutex&
|
||||
|
@ -795,6 +613,5 @@ Store::parse_query(const std::string& expr, bool xapian) const
|
|||
Query q{*this};
|
||||
|
||||
return q.parse(expr, xapian);
|
||||
},
|
||||
std::string{});
|
||||
}, std::string{});
|
||||
}
|
||||
|
|
133
lib/mu-store.hh
133
lib/mu-store.hh
|
@ -1,5 +1,5 @@
|
|||
/*
|
||||
** Copyright (C) 2022 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl>
|
||||
** Copyright (C) 2023 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl>
|
||||
**
|
||||
** This program is free software; you can redistribute it and/or modify it
|
||||
** under the terms of the GNU General Public License as published by the
|
||||
|
@ -26,6 +26,8 @@
|
|||
#include <ctime>
|
||||
|
||||
#include "mu-contacts-cache.hh"
|
||||
#include "mu-xapian-db.hh"
|
||||
#include "mu-config.hh"
|
||||
#include <xapian.h>
|
||||
|
||||
#include <utils/mu-utils.hh>
|
||||
|
@ -73,31 +75,19 @@ public:
|
|||
}
|
||||
/* LCOV_EXCL_STOP */
|
||||
|
||||
|
||||
struct Config {
|
||||
size_t max_message_size{};
|
||||
/**< maximum size (in bytes) for a message, or 0 for default */
|
||||
size_t batch_size{};
|
||||
/**< size of batches before committing, or 0 for default */
|
||||
};
|
||||
|
||||
/**
|
||||
* Construct a store for a not-yet-existing document database
|
||||
*
|
||||
* @param path path to the database
|
||||
* @param maildir maildir to use for this store
|
||||
* @param personal_addresses addresses that should be recognized as
|
||||
* 'personal' for identifying personal messages.
|
||||
* @param root_maildir maildir to use for this store
|
||||
* @param config a configuration object
|
||||
*
|
||||
* @return a store or an error
|
||||
*/
|
||||
static Result<Store> make_new(const std::string& path,
|
||||
const std::string& maildir,
|
||||
const StringVec& personal_addresses,
|
||||
const Config& conf) noexcept try {
|
||||
|
||||
return Ok(Store(path, maildir, personal_addresses, conf));
|
||||
const std::string& root_maildir,
|
||||
Option<const Config&> conf={}) noexcept try {
|
||||
return Ok(Store(path, root_maildir, conf));
|
||||
|
||||
} catch (const Mu::Error& me) {
|
||||
return Err(me);
|
||||
|
@ -108,6 +98,7 @@ public:
|
|||
}
|
||||
/* LCOV_EXCL_STOP */
|
||||
|
||||
|
||||
/**
|
||||
* Move CTOR
|
||||
*
|
||||
|
@ -119,30 +110,6 @@ public:
|
|||
*/
|
||||
~Store();
|
||||
|
||||
/**
|
||||
* Store properties
|
||||
*/
|
||||
struct Properties {
|
||||
std::string database_path; /**< Full path to the Xapian database */
|
||||
std::string schema_version; /**< Database schema version */
|
||||
std::time_t created; /**< database creation time */
|
||||
|
||||
size_t batch_size; /**< Maximum database transaction batch size */
|
||||
|
||||
std::string root_maildir; /**< Absolute path to the top-level maildir */
|
||||
|
||||
StringVec personal_addresses; /**< Personal e-mail addresses */
|
||||
size_t max_message_size; /**< Maximus allowed message size */
|
||||
};
|
||||
|
||||
/**
|
||||
* Get properties about this store.
|
||||
*
|
||||
* @return the metadata
|
||||
*/
|
||||
const Properties& properties() const;
|
||||
|
||||
|
||||
/**
|
||||
* Store statistics. Unlike the properties, these can change
|
||||
* during the lifetime of a store.
|
||||
|
@ -161,27 +128,21 @@ public:
|
|||
*/
|
||||
Statistics statistics() const;
|
||||
|
||||
/**
|
||||
* Get the underlying xapian db object
|
||||
*
|
||||
* @return the XapianDb for this store
|
||||
*/
|
||||
const XapianDb& xapian_db() const;
|
||||
XapianDb& xapian_db();
|
||||
|
||||
/**
|
||||
* Get the number of documents in the document database
|
||||
* Get the Config for this store
|
||||
*
|
||||
* @return the number
|
||||
* @return the Config
|
||||
*/
|
||||
std::size_t size() const;
|
||||
|
||||
/**
|
||||
* Is the database empty?
|
||||
*
|
||||
* @return true or false
|
||||
*/
|
||||
bool empty() const { return size() == 0; }
|
||||
|
||||
/**
|
||||
* Is the database read-only?
|
||||
*
|
||||
* @return true or false
|
||||
*/
|
||||
bool read_only() const;
|
||||
const Config& config() const;
|
||||
Config& config();
|
||||
|
||||
/**
|
||||
* Get the ContactsCache object for this store
|
||||
|
@ -190,13 +151,6 @@ public:
|
|||
*/
|
||||
const ContactsCache& contacts_cache() const;
|
||||
|
||||
/**
|
||||
* Get the underlying Xapian database for this store.
|
||||
*
|
||||
* @return the database
|
||||
*/
|
||||
const Xapian::Database& database() const;
|
||||
|
||||
/**
|
||||
* Get the Indexer associated with this store. It is an error to call
|
||||
* this on a read-only store.
|
||||
|
@ -449,6 +403,44 @@ public:
|
|||
*/
|
||||
void commit();
|
||||
|
||||
/*
|
||||
*
|
||||
* Some convenience
|
||||
*
|
||||
*/
|
||||
|
||||
/**
|
||||
* Get the Xapian database-path for this store
|
||||
*
|
||||
* @return the path
|
||||
*/
|
||||
const std::string& path() const { return xapian_db().path(); }
|
||||
|
||||
/**
|
||||
* Get the root-maildir for this store
|
||||
*
|
||||
* @return the root-maildir
|
||||
*/
|
||||
const std::string& root_maildir() const;
|
||||
|
||||
/**
|
||||
* Get the number of messages in the store
|
||||
*
|
||||
* @return the number
|
||||
*/
|
||||
size_t size() const { return xapian_db().size(); }
|
||||
|
||||
/**
|
||||
* Is the store empty?
|
||||
*
|
||||
* @return true or false
|
||||
*/
|
||||
size_t empty() const { return xapian_db().empty(); }
|
||||
|
||||
/*
|
||||
* _almost_ private
|
||||
*/
|
||||
|
||||
/**
|
||||
* Get a reference to the private data. For internal use.
|
||||
*
|
||||
|
@ -467,21 +459,14 @@ private:
|
|||
*/
|
||||
Store(const std::string& path, Options opts=Options::None);
|
||||
|
||||
|
||||
/**
|
||||
* Construct a store for a not-yet-existing document database
|
||||
*
|
||||
* @param path path to the database
|
||||
* @param maildir maildir to use for this store
|
||||
* @param personal_addresses addresses that should be recognized as
|
||||
* 'personal' for identifying personal messages.
|
||||
* @param config a configuration object
|
||||
*/
|
||||
Store(const std::string& path,
|
||||
const std::string& maildir,
|
||||
const StringVec& personal_addresses,
|
||||
const Config& conf);
|
||||
|
||||
Store(const std::string& path, const std::string& root_maildir,
|
||||
Option<const Config&> conf);
|
||||
|
||||
std::unique_ptr<Private> priv_;
|
||||
};
|
||||
|
|
|
@ -17,7 +17,7 @@
|
|||
################################################################################
|
||||
# project setup
|
||||
project('mu', ['c', 'cpp'],
|
||||
version: '1.11.7',
|
||||
version: '1.11.8',
|
||||
meson_version: '>= 0.56.0',
|
||||
license: 'GPL-3.0-or-later',
|
||||
default_options : [
|
||||
|
@ -86,7 +86,7 @@ cxx.check_header('charconv', required:true)
|
|||
# config.h setup
|
||||
#
|
||||
config_h_data=configuration_data()
|
||||
config_h_data.set_quoted('MU_STORE_SCHEMA_VERSION', '467')
|
||||
config_h_data.set('MU_STORE_SCHEMA_VERSION', 500)
|
||||
config_h_data.set_quoted('PACKAGE_VERSION', meson.project_version())
|
||||
config_h_data.set_quoted('PACKAGE_STRING', meson.project_name() + ' ' +
|
||||
meson.project_version())
|
||||
|
|
Loading…
Reference in New Issue