2009-11-25 21:55:06 +01:00
|
|
|
/*
|
2023-06-30 22:19:17 +02:00
|
|
|
** Copyright (C) 2021-2023 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl>
|
2009-11-25 21:55:06 +01:00
|
|
|
**
|
|
|
|
** This program is free software; you can redistribute it and/or modify it
|
|
|
|
** under the terms of the GNU General Public License as published by the
|
|
|
|
** Free Software Foundation; either version 3, or (at your option) any
|
|
|
|
** later version.
|
|
|
|
**
|
|
|
|
** This program is distributed in the hope that it will be useful,
|
|
|
|
** but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
|
|
** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
|
|
** GNU General Public License for more details.
|
|
|
|
**
|
|
|
|
** You should have received a copy of the GNU General Public License
|
|
|
|
** along with this program; if not, write to the Free Software Foundation,
|
|
|
|
** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
|
|
|
|
**
|
|
|
|
*/
|
|
|
|
|
|
|
|
#include "config.h"
|
2023-09-10 07:45:24 +02:00
|
|
|
#include "mu-store.hh"
|
2009-11-25 21:55:06 +01:00
|
|
|
|
2020-06-27 10:47:34 +02:00
|
|
|
#include <chrono>
|
2019-07-28 13:12:06 +02:00
|
|
|
#include <mutex>
|
|
|
|
#include <array>
|
|
|
|
#include <cstdlib>
|
2020-06-27 10:47:34 +02:00
|
|
|
#include <stdexcept>
|
2019-07-28 13:12:06 +02:00
|
|
|
#include <unordered_map>
|
|
|
|
#include <atomic>
|
2020-06-27 10:47:34 +02:00
|
|
|
#include <type_traits>
|
2020-02-06 19:24:38 +01:00
|
|
|
#include <iostream>
|
2020-02-29 13:40:22 +01:00
|
|
|
#include <cstring>
|
2020-06-27 10:47:34 +02:00
|
|
|
|
2022-04-22 07:05:08 +02:00
|
|
|
#include "mu-maildir.hh"
|
2022-01-30 13:28:30 +01:00
|
|
|
#include "mu-query.hh"
|
2023-06-30 22:19:17 +02:00
|
|
|
#include "mu-xapian-db.hh"
|
2023-09-10 07:45:24 +02:00
|
|
|
#include "mu-scanner.hh"
|
2023-06-30 22:19:17 +02:00
|
|
|
|
2019-12-30 21:28:53 +01:00
|
|
|
#include "utils/mu-error.hh"
|
|
|
|
|
2019-12-16 21:41:17 +01:00
|
|
|
#include "utils/mu-utils.hh"
|
2023-01-14 16:11:36 +01:00
|
|
|
#include <utils/mu-utils-file.hh>
|
2011-08-29 22:35:12 +02:00
|
|
|
|
2019-07-28 13:12:06 +02:00
|
|
|
using namespace Mu;
|
|
|
|
|
2021-10-20 11:18:15 +02:00
|
|
|
static_assert(std::is_same<Store::Id, Xapian::docid>::value, "wrong type for Store::Id");
|
2020-06-27 10:47:34 +02:00
|
|
|
|
2022-05-10 07:16:47 +02:00
|
|
|
// Properties
|
2020-06-27 10:47:34 +02:00
|
|
|
constexpr auto ExpectedSchemaVersion = MU_STORE_SCHEMA_VERSION;
|
2020-03-02 22:19:34 +01:00
|
|
|
|
2023-07-06 20:33:01 +02:00
|
|
|
static std::string
|
|
|
|
remove_slash(const std::string& str)
|
|
|
|
{
|
|
|
|
auto clean{str};
|
|
|
|
while (clean[clean.length() - 1] == '/')
|
|
|
|
clean.pop_back();
|
|
|
|
|
|
|
|
return clean;
|
|
|
|
}
|
|
|
|
|
2019-07-28 13:12:06 +02:00
|
|
|
struct Store::Private {
|
2021-10-20 11:18:15 +02:00
|
|
|
|
2023-06-30 22:19:17 +02:00
|
|
|
Private(const std::string& path, bool readonly):
|
2023-07-09 22:17:08 +02:00
|
|
|
xapian_db_{XapianDb(path, readonly ? XapianDb::Flavor::ReadOnly
|
|
|
|
: XapianDb::Flavor::Open)},
|
2023-06-30 22:19:17 +02:00
|
|
|
config_{xapian_db_},
|
|
|
|
contacts_cache_{config_},
|
2023-09-09 10:57:05 +02:00
|
|
|
root_maildir_{remove_slash(config_.get<Config::Id::RootMaildir>())},
|
|
|
|
message_opts_{make_message_options(config_)}
|
2023-06-30 22:19:17 +02:00
|
|
|
{}
|
|
|
|
|
|
|
|
Private(const std::string& path, const std::string& root_maildir,
|
|
|
|
Option<const Config&> conf):
|
2023-07-09 22:17:08 +02:00
|
|
|
xapian_db_{XapianDb(path, XapianDb::Flavor::CreateOverwrite)},
|
2023-06-30 22:19:17 +02:00
|
|
|
config_{make_config(xapian_db_, root_maildir, conf)},
|
|
|
|
contacts_cache_{config_},
|
2023-09-09 10:57:05 +02:00
|
|
|
root_maildir_{remove_slash(config_.get<Config::Id::RootMaildir>())},
|
|
|
|
message_opts_{make_message_options(config_)}
|
2023-06-30 22:19:17 +02:00
|
|
|
{}
|
2021-10-20 11:18:15 +02:00
|
|
|
|
2022-05-18 17:07:19 +02:00
|
|
|
~Private() try {
|
2023-07-05 22:10:13 +02:00
|
|
|
mu_debug("closing store @ {}", xapian_db_.path());
|
2023-06-30 22:19:17 +02:00
|
|
|
if (!xapian_db_.read_only()) {
|
2021-11-09 21:43:11 +01:00
|
|
|
transaction_maybe_commit(true /*force*/);
|
2021-10-20 11:18:15 +02:00
|
|
|
}
|
2022-02-07 21:34:49 +01:00
|
|
|
} catch (...) {
|
2023-07-05 22:10:13 +02:00
|
|
|
mu_critical("caught exception in store dtor");
|
2021-10-20 11:18:15 +02:00
|
|
|
}
|
2020-11-03 08:58:59 +01:00
|
|
|
|
2021-11-09 21:43:11 +01:00
|
|
|
// If not started yet, start a transaction. Otherwise, just update the transaction size.
|
2023-04-23 19:26:16 +02:00
|
|
|
void transaction_inc() noexcept {
|
2021-11-09 21:43:11 +01:00
|
|
|
if (transaction_size_ == 0) {
|
2023-07-05 22:10:13 +02:00
|
|
|
mu_debug("starting transaction");
|
2023-06-30 22:19:17 +02:00
|
|
|
xapian_db_.begin_transaction();
|
2021-11-09 21:43:11 +01:00
|
|
|
}
|
|
|
|
++transaction_size_;
|
2021-10-18 11:22:26 +02:00
|
|
|
}
|
2021-10-20 11:18:15 +02:00
|
|
|
|
2021-11-09 21:43:11 +01:00
|
|
|
// Opportunistically commit a transaction if the transaction size
|
|
|
|
// filled up a batch, or with force.
|
2022-05-05 00:27:08 +02:00
|
|
|
void transaction_maybe_commit(bool force = false) noexcept {
|
2023-06-30 22:19:17 +02:00
|
|
|
static auto batch_size = config_.get<Config::Id::BatchSize>();
|
|
|
|
if (force || transaction_size_ >= batch_size) {
|
|
|
|
contacts_cache_.serialize();
|
|
|
|
|
|
|
|
if (indexer_) // save last index time.
|
2022-05-18 19:16:48 +02:00
|
|
|
if (auto&& t{indexer_->completed()}; t != 0)
|
2023-06-30 22:19:17 +02:00
|
|
|
config_.set<Config::Id::LastIndex>(::time({}));
|
2022-05-18 19:16:48 +02:00
|
|
|
|
2022-04-22 07:05:08 +02:00
|
|
|
if (transaction_size_ == 0)
|
|
|
|
return; // nothing more to do here.
|
|
|
|
|
2023-07-05 22:10:13 +02:00
|
|
|
mu_debug("committing transaction (n={})", transaction_size_);
|
2023-06-30 22:19:17 +02:00
|
|
|
xapian_db_.commit_transaction();
|
|
|
|
transaction_size_ = 0;
|
2021-11-09 21:43:11 +01:00
|
|
|
}
|
2021-10-20 11:18:15 +02:00
|
|
|
}
|
2020-06-27 10:47:34 +02:00
|
|
|
|
2023-06-30 22:19:17 +02:00
|
|
|
Config make_config(XapianDb& xapian_db, const std::string& root_maildir,
|
|
|
|
Option<const Config&> conf) {
|
2022-02-13 13:52:41 +01:00
|
|
|
|
2023-06-30 22:19:17 +02:00
|
|
|
Config config{xapian_db};
|
2022-02-13 13:52:41 +01:00
|
|
|
|
2023-06-30 22:19:17 +02:00
|
|
|
if (conf)
|
|
|
|
config.import_configurable(*conf);
|
2020-06-27 10:47:34 +02:00
|
|
|
|
2023-07-06 20:33:01 +02:00
|
|
|
config.set<Config::Id::RootMaildir>(remove_slash(root_maildir));
|
2023-06-30 22:19:17 +02:00
|
|
|
config.set<Config::Id::SchemaVersion>(ExpectedSchemaVersion);
|
2021-10-20 11:18:15 +02:00
|
|
|
|
2023-06-30 22:19:17 +02:00
|
|
|
return config;
|
2021-10-20 11:18:15 +02:00
|
|
|
}
|
2020-03-02 22:19:34 +01:00
|
|
|
|
2023-09-09 10:57:05 +02:00
|
|
|
Message::Options make_message_options(const Config& conf) {
|
|
|
|
if (conf.get<Config::Id::SupportNgrams>())
|
|
|
|
return Message::Options::SupportNgrams;
|
|
|
|
else
|
|
|
|
return Message::Options::None;
|
|
|
|
}
|
|
|
|
|
2022-05-05 00:27:08 +02:00
|
|
|
Option<Message> find_message_unlocked(Store::Id docid) const;
|
2023-08-09 19:09:46 +02:00
|
|
|
Store::IdVec find_duplicates_unlocked(const Store& store,
|
|
|
|
const std::string& message_id) const;
|
|
|
|
|
2023-07-25 22:52:22 +02:00
|
|
|
Result<Store::Id> add_message_unlocked(Message& msg);
|
2022-05-05 00:27:08 +02:00
|
|
|
Result<Store::Id> update_message_unlocked(Message& msg, Store::Id docid);
|
|
|
|
Result<Store::Id> update_message_unlocked(Message& msg, const std::string& old_path);
|
2023-08-09 19:09:46 +02:00
|
|
|
|
2022-12-03 15:42:19 +01:00
|
|
|
Result<Message> move_message_unlocked(Message&& msg,
|
|
|
|
Option<const std::string&> target_mdir,
|
|
|
|
Option<Flags> new_flags,
|
|
|
|
MoveOptions opts);
|
2023-06-30 22:19:17 +02:00
|
|
|
XapianDb xapian_db_;
|
|
|
|
Config config_;
|
2022-02-19 21:08:54 +01:00
|
|
|
ContactsCache contacts_cache_;
|
2021-10-20 11:18:15 +02:00
|
|
|
std::unique_ptr<Indexer> indexer_;
|
2019-07-28 13:12:06 +02:00
|
|
|
|
2023-09-09 10:57:05 +02:00
|
|
|
const std::string root_maildir_;
|
|
|
|
const Message::Options message_opts_;
|
2023-06-30 22:19:17 +02:00
|
|
|
|
2021-11-09 21:43:11 +01:00
|
|
|
size_t transaction_size_{};
|
|
|
|
std::mutex lock_;
|
2019-07-28 13:12:06 +02:00
|
|
|
};
|
|
|
|
|
2023-07-25 22:52:22 +02:00
|
|
|
|
|
|
|
Result<Store::Id>
|
|
|
|
Store::Private::add_message_unlocked(Message& msg)
|
|
|
|
{
|
2023-09-23 08:27:46 +02:00
|
|
|
auto&& docid{xapian_db_.add_document(msg.document().xapian_document())};
|
|
|
|
if (docid)
|
|
|
|
mu_debug("added message @ {}; docid = {}", msg.path(), *docid);
|
2023-07-25 22:52:22 +02:00
|
|
|
|
2023-09-23 08:27:46 +02:00
|
|
|
return docid;
|
2023-07-25 22:52:22 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
Result<Store::Id>
|
|
|
|
Store::Private::update_message_unlocked(Message& msg, Store::Id docid)
|
2023-06-30 22:19:17 +02:00
|
|
|
{
|
2023-09-23 08:27:46 +02:00
|
|
|
auto&& res{xapian_db_.replace_document(docid, msg.document().xapian_document())};
|
|
|
|
if (res)
|
|
|
|
mu_debug("updated message @ {}; docid = {}", msg.path(), *res);
|
2023-06-30 22:19:17 +02:00
|
|
|
|
2023-09-23 08:27:46 +02:00
|
|
|
return res;
|
2022-05-05 00:27:08 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
Result<Store::Id>
|
|
|
|
Store::Private::update_message_unlocked(Message& msg, const std::string& path_to_replace)
|
|
|
|
{
|
2023-09-23 08:27:46 +02:00
|
|
|
return xapian_db_.replace_document(
|
2023-06-30 22:19:17 +02:00
|
|
|
field_from_id(Field::Id::Path).xapian_term(path_to_replace),
|
|
|
|
msg.document().xapian_document());
|
2022-05-05 00:27:08 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
Option<Message>
|
|
|
|
Store::Private::find_message_unlocked(Store::Id docid) const
|
|
|
|
{
|
2023-06-30 22:19:17 +02:00
|
|
|
if (auto&& doc{xapian_db_.document(docid)}; !doc)
|
|
|
|
return Nothing;
|
|
|
|
else if (auto&& msg{Message::make_from_document(std::move(*doc))}; !msg)
|
|
|
|
return Nothing;
|
|
|
|
else
|
|
|
|
return Some(std::move(*msg));
|
2022-05-05 00:27:08 +02:00
|
|
|
}
|
|
|
|
|
2023-08-09 19:09:46 +02:00
|
|
|
Store::IdVec
|
|
|
|
Store::Private::find_duplicates_unlocked(const Store& store,
|
|
|
|
const std::string& message_id) const
|
|
|
|
{
|
|
|
|
if (message_id.empty() || message_id.size() > MaxTermLength) {
|
|
|
|
mu_warning("invalid message-id '{}'", message_id);
|
|
|
|
return {};
|
|
|
|
}
|
|
|
|
|
|
|
|
auto expr{mu_format("{}:{}",
|
|
|
|
field_from_id(Field::Id::MessageId).shortcut,
|
|
|
|
message_id)};
|
|
|
|
if (auto&& res{store.run_query(expr)}; !res) {
|
|
|
|
mu_warning("error finding message-ids: {}", res.error().what());
|
|
|
|
return {};
|
|
|
|
|
|
|
|
} else {
|
|
|
|
Store::IdVec ids;
|
|
|
|
ids.reserve(res->size());
|
|
|
|
for (auto&& mi: *res)
|
|
|
|
ids.emplace_back(mi.doc_id());
|
|
|
|
return ids;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2022-05-05 00:27:08 +02:00
|
|
|
|
2022-05-09 19:58:35 +02:00
|
|
|
Store::Store(const std::string& path, Store::Options opts)
|
|
|
|
: priv_{std::make_unique<Private>(path, none_of(opts & Store::Options::Writable))}
|
2019-07-28 13:12:06 +02:00
|
|
|
{
|
2022-09-28 21:38:06 +02:00
|
|
|
if (none_of(opts & Store::Options::Writable) &&
|
|
|
|
any_of(opts & Store::Options::ReInit))
|
|
|
|
throw Mu::Error(Error::Code::InvalidArgument,
|
|
|
|
"Options::ReInit requires Options::Writable");
|
|
|
|
|
2023-06-30 22:19:17 +02:00
|
|
|
const auto s_version{config().get<Config::Id::SchemaVersion>()};
|
2022-09-28 21:38:06 +02:00
|
|
|
if (any_of(opts & Store::Options::ReInit)) {
|
2023-06-30 22:19:17 +02:00
|
|
|
/* don't try to recover from version with an incompatible scheme */
|
|
|
|
if (s_version < 500)
|
|
|
|
throw Mu::Error(Error::Code::CannotReinit,
|
2023-07-05 22:10:13 +02:00
|
|
|
"old schema ({}) is too old to re-initialize from",
|
2023-09-16 10:07:03 +02:00
|
|
|
s_version).add_hint("Invoke 'mu init' without '--reinit'; "
|
|
|
|
"see mu-init(1) for details");
|
2023-06-30 22:19:17 +02:00
|
|
|
const auto old_root_maildir{root_maildir()};
|
|
|
|
|
|
|
|
MemDb mem_db;
|
|
|
|
Config old_config(mem_db);
|
|
|
|
old_config.import_configurable(config());
|
|
|
|
|
2022-09-28 21:38:06 +02:00
|
|
|
this->priv_.reset();
|
2023-06-30 22:19:17 +02:00
|
|
|
/* and create a new one "in place" */
|
|
|
|
Store new_store(path, old_root_maildir, old_config);
|
2022-09-28 21:38:06 +02:00
|
|
|
this->priv_ = std::move(new_store.priv_);
|
2022-05-09 19:58:35 +02:00
|
|
|
}
|
|
|
|
|
2022-09-28 21:38:06 +02:00
|
|
|
/* otherwise, the schema version should match. */
|
2023-06-30 22:19:17 +02:00
|
|
|
if (s_version != ExpectedSchemaVersion)
|
2022-05-09 19:58:35 +02:00
|
|
|
throw Mu::Error(Error::Code::SchemaMismatch,
|
2023-07-05 22:10:13 +02:00
|
|
|
"expected schema-version {}, but got {}",
|
2023-09-16 10:07:03 +02:00
|
|
|
ExpectedSchemaVersion, s_version).
|
|
|
|
add_hint("Please (re)initialize with 'mu init'; see mu-init(1) for details");
|
2019-07-28 13:12:06 +02:00
|
|
|
}
|
|
|
|
|
2023-06-30 22:19:17 +02:00
|
|
|
Store::Store(const std::string& path,
|
|
|
|
const std::string& root_maildir,
|
|
|
|
Option<const Config&> conf):
|
|
|
|
priv_{std::make_unique<Private>(path, root_maildir, conf)}
|
|
|
|
{}
|
2021-02-16 18:32:15 +01:00
|
|
|
|
2022-10-26 20:51:53 +02:00
|
|
|
Store::Store(Store&& other)
|
|
|
|
{
|
|
|
|
priv_ = std::move(other.priv_);
|
|
|
|
priv_->indexer_.reset();
|
|
|
|
}
|
2022-05-09 19:58:35 +02:00
|
|
|
|
2019-07-28 13:12:06 +02:00
|
|
|
Store::~Store() = default;
|
|
|
|
|
2022-05-10 07:16:47 +02:00
|
|
|
Store::Statistics
|
|
|
|
Store::statistics() const
|
|
|
|
{
|
|
|
|
Statistics stats{};
|
|
|
|
|
|
|
|
stats.size = size();
|
2023-06-30 22:19:17 +02:00
|
|
|
stats.last_change = config().get<Config::Id::LastChange>();
|
|
|
|
stats.last_index = config().get<Config::Id::LastIndex>();
|
2022-05-10 07:16:47 +02:00
|
|
|
|
|
|
|
return stats;
|
|
|
|
}
|
|
|
|
|
2023-06-30 22:19:17 +02:00
|
|
|
const XapianDb&
|
|
|
|
Store::xapian_db() const
|
2019-07-28 13:12:06 +02:00
|
|
|
{
|
2023-06-30 22:19:17 +02:00
|
|
|
return priv_->xapian_db_;
|
|
|
|
}
|
|
|
|
|
|
|
|
XapianDb&
|
|
|
|
Store::xapian_db()
|
|
|
|
{
|
|
|
|
return priv_->xapian_db_;
|
2019-07-28 13:12:06 +02:00
|
|
|
}
|
|
|
|
|
2023-06-30 22:19:17 +02:00
|
|
|
const Config&
|
|
|
|
Store::config() const
|
2020-11-03 08:58:59 +01:00
|
|
|
{
|
2023-06-30 22:19:17 +02:00
|
|
|
return priv_->config_;
|
|
|
|
}
|
|
|
|
|
|
|
|
Config&
|
|
|
|
Store::config()
|
|
|
|
{
|
|
|
|
return priv_->config_;
|
|
|
|
}
|
|
|
|
|
|
|
|
const std::string&
|
|
|
|
Store::root_maildir() const {
|
|
|
|
return priv_->root_maildir_;
|
|
|
|
}
|
|
|
|
|
|
|
|
const ContactsCache&
|
|
|
|
Store::contacts_cache() const
|
|
|
|
{
|
|
|
|
return priv_->contacts_cache_;
|
2020-11-03 08:58:59 +01:00
|
|
|
}
|
|
|
|
|
2020-06-27 10:47:34 +02:00
|
|
|
Indexer&
|
|
|
|
Store::indexer()
|
2019-07-28 13:12:06 +02:00
|
|
|
{
|
2022-01-30 13:28:30 +01:00
|
|
|
std::lock_guard guard{priv_->lock_};
|
2019-07-28 13:12:06 +02:00
|
|
|
|
2023-06-30 22:19:17 +02:00
|
|
|
if (xapian_db().read_only())
|
2021-10-20 11:18:15 +02:00
|
|
|
throw Error{Error::Code::Store, "no indexer for read-only store"};
|
|
|
|
else if (!priv_->indexer_)
|
|
|
|
priv_->indexer_ = std::make_unique<Indexer>(*this);
|
2019-07-28 13:12:06 +02:00
|
|
|
|
2021-10-20 11:18:15 +02:00
|
|
|
return *priv_->indexer_.get();
|
2019-07-28 13:12:06 +02:00
|
|
|
}
|
|
|
|
|
2022-04-28 21:59:03 +02:00
|
|
|
Result<Store::Id>
|
2023-07-25 22:52:22 +02:00
|
|
|
Store::add_message(Message& msg, bool use_transaction, bool is_new)
|
2020-01-30 23:20:34 +01:00
|
|
|
{
|
2022-05-09 18:34:07 +02:00
|
|
|
const auto mdir{maildir_from_path(msg.path(),
|
2023-06-30 22:19:17 +02:00
|
|
|
root_maildir())};
|
2022-04-28 21:59:03 +02:00
|
|
|
if (!mdir)
|
|
|
|
return Err(mdir.error());
|
|
|
|
if (auto&& res = msg.set_maildir(mdir.value()); !res)
|
|
|
|
return Err(res.error());
|
2023-07-25 22:52:22 +02:00
|
|
|
|
2023-09-09 10:57:05 +02:00
|
|
|
// we shouldn't mix ngrams/non-ngrams messages.
|
|
|
|
if (any_of(msg.options() & Message::Options::SupportNgrams) !=
|
|
|
|
any_of(message_options() & Message::Options::SupportNgrams))
|
2023-09-16 10:07:03 +02:00
|
|
|
return Err(Error::Code::InvalidArgument, "incompatible message options");
|
2023-09-09 10:57:05 +02:00
|
|
|
|
2022-05-01 10:18:57 +02:00
|
|
|
/* add contacts from this message to cache; this cache
|
|
|
|
* also determines whether those contacts are _personal_, i.e. match
|
|
|
|
* our personal addresses.
|
|
|
|
*
|
|
|
|
* if a message has any personal contacts, mark it as personal; do
|
|
|
|
* this by updating the message flags.
|
|
|
|
*/
|
|
|
|
bool is_personal{};
|
|
|
|
priv_->contacts_cache_.add(msg.all_contacts(), is_personal);
|
|
|
|
if (is_personal)
|
|
|
|
msg.set_flags(msg.flags() | Flags::Personal);
|
|
|
|
|
2023-07-25 22:52:22 +02:00
|
|
|
std::lock_guard guard{priv_->lock_};
|
2022-05-05 00:27:08 +02:00
|
|
|
if (use_transaction)
|
|
|
|
priv_->transaction_inc();
|
2022-04-28 21:59:03 +02:00
|
|
|
|
2023-07-25 22:52:22 +02:00
|
|
|
auto&& res = is_new ?
|
|
|
|
priv_->add_message_unlocked(msg) :
|
|
|
|
priv_->update_message_unlocked(msg, msg.path());
|
2022-05-05 00:27:08 +02:00
|
|
|
if (!res)
|
|
|
|
return Err(res.error());
|
2021-11-09 21:43:11 +01:00
|
|
|
|
2022-05-05 00:27:08 +02:00
|
|
|
if (use_transaction) /* commit if batch is full */
|
|
|
|
priv_->transaction_maybe_commit();
|
2020-01-30 23:20:34 +01:00
|
|
|
|
2023-07-25 22:52:22 +02:00
|
|
|
mu_debug("added {}message @ {}; docid = {}",
|
|
|
|
is_personal ? "personal " : "", msg.path(), *res);
|
2020-06-10 16:41:00 +02:00
|
|
|
|
2022-05-05 00:27:08 +02:00
|
|
|
return res;
|
2020-01-30 23:20:34 +01:00
|
|
|
}
|
|
|
|
|
2023-09-09 10:57:05 +02:00
|
|
|
Result<Store::Id>
|
|
|
|
Store::add_message(const std::string& path, bool use_transaction, bool is_new)
|
|
|
|
{
|
|
|
|
if (auto msg{Message::make_from_path(path, priv_->message_opts_)}; !msg)
|
|
|
|
return Err(msg.error());
|
|
|
|
else
|
|
|
|
return add_message(msg.value(), use_transaction, is_new);
|
|
|
|
}
|
|
|
|
|
|
|
|
|
2020-01-30 23:20:34 +01:00
|
|
|
bool
|
2021-10-20 11:18:15 +02:00
|
|
|
Store::remove_message(const std::string& path)
|
2020-01-30 23:20:34 +01:00
|
|
|
{
|
2023-06-30 22:19:17 +02:00
|
|
|
const auto term{field_from_id(Field::Id::Path).xapian_term(path)};
|
2023-07-25 22:52:22 +02:00
|
|
|
|
|
|
|
std::lock_guard guard{priv_->lock_};
|
|
|
|
|
2023-06-30 22:19:17 +02:00
|
|
|
xapian_db().delete_document(term);
|
2023-07-25 22:52:22 +02:00
|
|
|
mu_debug("deleted message @ {} from store", path);
|
2023-06-30 22:19:17 +02:00
|
|
|
return true;
|
2020-01-30 23:20:34 +01:00
|
|
|
}
|
|
|
|
|
2020-06-27 10:47:34 +02:00
|
|
|
void
|
2021-10-20 11:18:15 +02:00
|
|
|
Store::remove_messages(const std::vector<Store::Id>& ids)
|
2020-06-27 10:47:34 +02:00
|
|
|
{
|
2022-01-30 13:28:30 +01:00
|
|
|
std::lock_guard guard{priv_->lock_};
|
2021-11-09 21:43:11 +01:00
|
|
|
|
|
|
|
priv_->transaction_inc();
|
2021-10-20 16:18:51 +02:00
|
|
|
|
2023-06-30 22:19:17 +02:00
|
|
|
for (auto&& id : ids)
|
|
|
|
xapian_db().delete_document(id);
|
2021-10-20 16:18:51 +02:00
|
|
|
|
2021-11-09 21:43:11 +01:00
|
|
|
priv_->transaction_maybe_commit(true /*force*/);
|
2020-06-27 10:47:34 +02:00
|
|
|
}
|
2020-01-30 23:20:34 +01:00
|
|
|
|
2022-02-13 14:22:09 +01:00
|
|
|
|
2022-05-05 00:27:08 +02:00
|
|
|
Option<Message>
|
|
|
|
Store::find_message(Store::Id docid) const
|
|
|
|
{
|
|
|
|
std::lock_guard guard{priv_->lock_};
|
|
|
|
|
|
|
|
return priv_->find_message_unlocked(docid);
|
|
|
|
}
|
|
|
|
|
2023-08-09 19:09:46 +02:00
|
|
|
Store::IdMessageVec
|
|
|
|
Store::find_messages(IdVec ids) const
|
|
|
|
{
|
|
|
|
std::lock_guard guard{priv_->lock_};
|
|
|
|
|
|
|
|
IdMessageVec id_msgs;
|
|
|
|
for (auto&& id: ids) {
|
|
|
|
if (auto&& msg{priv_->find_message_unlocked(id)}; msg)
|
|
|
|
id_msgs.emplace_back(std::make_pair(id, std::move(*msg)));
|
|
|
|
}
|
|
|
|
|
|
|
|
return id_msgs;
|
|
|
|
}
|
|
|
|
|
2022-12-03 15:42:19 +01:00
|
|
|
/**
|
|
|
|
* Move a message in store and filesystem.
|
|
|
|
*
|
|
|
|
* Lock is assumed taken already
|
|
|
|
*
|
|
|
|
* @param id message id
|
2023-08-09 19:09:46 +02:00
|
|
|
* @param target_mdir target_mdir (or Nothing for current)
|
|
|
|
* @param new_flags new flags (or Nothing)
|
|
|
|
* @param opts move_options
|
2022-12-03 15:42:19 +01:00
|
|
|
*
|
|
|
|
* @return the Message after the moving, or an Error
|
|
|
|
*/
|
2022-04-22 07:05:08 +02:00
|
|
|
Result<Message>
|
2022-12-03 15:42:19 +01:00
|
|
|
Store::Private::move_message_unlocked(Message&& msg,
|
|
|
|
Option<const std::string&> target_mdir,
|
|
|
|
Option<Flags> new_flags,
|
|
|
|
MoveOptions opts)
|
2022-04-22 07:05:08 +02:00
|
|
|
{
|
2023-08-09 19:09:46 +02:00
|
|
|
const auto old_path = msg.path();
|
|
|
|
const auto target_flags = new_flags.value_or(msg.flags());
|
|
|
|
const auto target_maildir = target_mdir.value_or(msg.maildir());
|
2022-04-22 07:05:08 +02:00
|
|
|
|
|
|
|
/* 1. first determine the file system path of the target */
|
|
|
|
const auto target_path =
|
2023-06-30 22:19:17 +02:00
|
|
|
maildir_determine_target(msg.path(), root_maildir_,
|
2022-12-03 15:42:19 +01:00
|
|
|
target_maildir, target_flags,
|
|
|
|
any_of(opts & MoveOptions::ChangeName));
|
2022-04-22 07:05:08 +02:00
|
|
|
if (!target_path)
|
|
|
|
return Err(target_path.error());
|
|
|
|
|
|
|
|
/* 2. let's move it */
|
2022-12-03 15:42:19 +01:00
|
|
|
if (const auto res = maildir_move_message(msg.path(), target_path.value()); !res)
|
2022-05-05 00:27:08 +02:00
|
|
|
return Err(res.error());
|
2022-04-22 07:05:08 +02:00
|
|
|
|
|
|
|
/* 3. file move worked, now update the message with the new info.*/
|
2022-12-03 15:42:19 +01:00
|
|
|
if (auto&& res = msg.update_after_move(
|
2022-05-05 00:27:08 +02:00
|
|
|
target_path.value(), target_maildir, target_flags); !res)
|
|
|
|
return Err(res.error());
|
2022-04-22 07:05:08 +02:00
|
|
|
|
|
|
|
/* 4. update message worked; re-store it */
|
2022-12-03 15:42:19 +01:00
|
|
|
if (auto&& res = update_message_unlocked(msg, old_path); !res)
|
2022-05-05 00:27:08 +02:00
|
|
|
return Err(res.error());
|
2022-04-22 07:05:08 +02:00
|
|
|
|
2022-05-05 00:27:08 +02:00
|
|
|
/* 6. Profit! */
|
2022-12-03 15:42:19 +01:00
|
|
|
return Ok(std::move(msg));
|
|
|
|
}
|
|
|
|
|
2023-08-09 19:09:46 +02:00
|
|
|
Store::IdVec
|
|
|
|
Store::find_duplicates(const std::string& message_id) const
|
2023-07-09 22:20:32 +02:00
|
|
|
{
|
|
|
|
std::lock_guard guard{priv_->lock_};
|
|
|
|
|
2023-08-09 19:09:46 +02:00
|
|
|
return priv_->find_duplicates_unlocked(*this, message_id);
|
2023-07-09 22:20:32 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
|
2022-12-03 15:42:19 +01:00
|
|
|
|
2023-08-09 19:09:46 +02:00
|
|
|
Result<Store::IdVec>
|
2022-12-03 15:42:19 +01:00
|
|
|
Store::move_message(Store::Id id,
|
|
|
|
Option<const std::string&> target_mdir,
|
|
|
|
Option<Flags> new_flags,
|
|
|
|
MoveOptions opts)
|
|
|
|
{
|
2023-08-09 19:09:46 +02:00
|
|
|
auto filter_dup_flags=[](Flags old_flags, Flags new_flags) -> Flags {
|
|
|
|
new_flags = flags_keep_unmutable(old_flags, new_flags, Flags::Draft);
|
|
|
|
new_flags = flags_keep_unmutable(old_flags, new_flags, Flags::Flagged);
|
|
|
|
new_flags = flags_keep_unmutable(old_flags, new_flags, Flags::Trashed);
|
|
|
|
return new_flags;
|
|
|
|
};
|
|
|
|
|
2022-12-03 15:42:19 +01:00
|
|
|
std::lock_guard guard{priv_->lock_};
|
|
|
|
|
|
|
|
auto msg{priv_->find_message_unlocked(id)};
|
|
|
|
if (!msg)
|
2023-07-05 22:10:13 +02:00
|
|
|
return Err(Error::Code::Store, "cannot find message <{}>", id);
|
2022-12-03 15:42:19 +01:00
|
|
|
|
2023-08-09 19:09:46 +02:00
|
|
|
const auto message_id{msg->message_id()};
|
|
|
|
auto res{priv_->move_message_unlocked(std::move(*msg),
|
|
|
|
target_mdir, new_flags, opts)};
|
2022-12-03 15:42:19 +01:00
|
|
|
if (!res)
|
|
|
|
return Err(res.error());
|
|
|
|
|
2023-08-09 19:09:46 +02:00
|
|
|
IdVec ids{id};
|
|
|
|
if (none_of(opts & Store::MoveOptions::DupFlags) || message_id.empty() || !new_flags)
|
|
|
|
return Ok(std::move(ids));
|
2022-12-03 15:42:19 +01:00
|
|
|
|
|
|
|
/* handle the dupflags case; i.e. apply (a subset of) the flags to
|
|
|
|
* all messages with the same message-id as well */
|
2023-08-09 19:09:46 +02:00
|
|
|
auto dups{priv_->find_duplicates_unlocked(*this, message_id)};
|
|
|
|
for (auto&& dupid: dups) {
|
2022-12-03 15:42:19 +01:00
|
|
|
|
2023-08-09 19:09:46 +02:00
|
|
|
if (dupid == id)
|
2022-12-03 15:42:19 +01:00
|
|
|
continue; // already
|
|
|
|
|
2023-08-09 19:09:46 +02:00
|
|
|
auto dup_msg{priv_->find_message_unlocked(dupid)};
|
|
|
|
if (!dup_msg)
|
|
|
|
continue; // no such message
|
2022-12-03 15:42:19 +01:00
|
|
|
|
2023-08-09 19:09:46 +02:00
|
|
|
/* For now, don't change Draft/Flagged/Trashed */
|
|
|
|
const auto dup_flags{filter_dup_flags(dup_msg->flags(), *new_flags)};
|
2022-12-03 15:42:19 +01:00
|
|
|
/* use the updated new_flags and default MoveOptions (so we don't recurse, nor do we
|
|
|
|
* change the base-name of moved messages) */
|
2023-08-09 19:09:46 +02:00
|
|
|
if (auto dup_res = priv_->move_message_unlocked(
|
|
|
|
std::move(*dup_msg), Nothing,
|
|
|
|
dup_flags,
|
|
|
|
Store::MoveOptions::None); !dup_res)
|
2023-07-05 22:10:13 +02:00
|
|
|
mu_warning("failed to move dup: {}", dup_res.error().what());
|
2023-08-09 19:09:46 +02:00
|
|
|
else
|
|
|
|
ids.emplace_back(dupid);
|
2022-12-03 15:42:19 +01:00
|
|
|
}
|
|
|
|
|
2023-08-09 19:09:46 +02:00
|
|
|
return Ok(std::move(ids));
|
2022-04-22 07:05:08 +02:00
|
|
|
}
|
|
|
|
|
2022-02-13 14:22:09 +01:00
|
|
|
time_t
|
|
|
|
Store::dirstamp(const std::string& path) const
|
|
|
|
{
|
2023-07-11 21:52:59 +02:00
|
|
|
std::string ts;
|
|
|
|
|
|
|
|
{
|
|
|
|
std::unique_lock lock{priv_->lock_};
|
|
|
|
ts = xapian_db().metadata(path);
|
|
|
|
}
|
|
|
|
|
|
|
|
return ts.empty() ? 0 /*epoch*/ : ::strtoll(ts.c_str(), {}, 16);
|
2022-02-13 14:22:09 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
void
|
|
|
|
Store::set_dirstamp(const std::string& path, time_t tstamp)
|
|
|
|
{
|
2023-07-11 21:52:59 +02:00
|
|
|
std::unique_lock lock{priv_->lock_};
|
2019-07-28 13:12:06 +02:00
|
|
|
|
2023-07-11 21:52:59 +02:00
|
|
|
xapian_db().set_metadata(path, mu_format("{:x}", tstamp));
|
2019-07-28 13:12:06 +02:00
|
|
|
}
|
|
|
|
|
2020-01-30 23:20:34 +01:00
|
|
|
bool
|
2021-10-20 11:18:15 +02:00
|
|
|
Store::contains_message(const std::string& path) const
|
2020-01-30 23:20:34 +01:00
|
|
|
{
|
2023-07-11 21:52:59 +02:00
|
|
|
std::unique_lock lock{priv_->lock_};
|
|
|
|
|
2023-06-30 22:19:17 +02:00
|
|
|
return xapian_db().term_exists(field_from_id(Field::Id::Path).xapian_term(path));
|
2020-01-30 23:20:34 +01:00
|
|
|
}
|
|
|
|
|
2020-06-27 10:47:34 +02:00
|
|
|
std::size_t
|
2021-10-20 11:18:15 +02:00
|
|
|
Store::for_each_message_path(Store::ForEachMessageFunc msg_func) const
|
2019-07-28 13:12:06 +02:00
|
|
|
{
|
2021-10-18 11:22:26 +02:00
|
|
|
size_t n{};
|
2021-10-20 11:18:15 +02:00
|
|
|
|
|
|
|
xapian_try([&] {
|
2022-01-30 13:28:30 +01:00
|
|
|
std::lock_guard guard{priv_->lock_};
|
2023-06-30 22:19:17 +02:00
|
|
|
auto enq{xapian_db().enquire()};
|
2022-01-30 13:28:30 +01:00
|
|
|
|
2021-10-20 11:18:15 +02:00
|
|
|
enq.set_query(Xapian::Query::MatchAll);
|
|
|
|
enq.set_cutoff(0, 0);
|
2019-07-28 13:12:06 +02:00
|
|
|
|
2023-06-30 22:19:17 +02:00
|
|
|
Xapian::MSet matches(enq.get_mset(0, xapian_db().size()));
|
2022-03-20 13:12:41 +01:00
|
|
|
constexpr auto path_no{field_from_id(Field::Id::Path).value_no()};
|
2021-10-20 11:18:15 +02:00
|
|
|
for (auto&& it = matches.begin(); it != matches.end(); ++it, ++n)
|
2022-03-04 23:29:21 +01:00
|
|
|
if (!msg_func(*it, it.get_document().get_value(path_no)))
|
2021-10-20 11:18:15 +02:00
|
|
|
break;
|
|
|
|
});
|
2020-06-27 10:47:34 +02:00
|
|
|
|
2021-10-18 11:22:26 +02:00
|
|
|
return n;
|
2019-07-28 13:12:06 +02:00
|
|
|
}
|
2011-08-29 22:35:12 +02:00
|
|
|
|
2021-11-09 21:43:11 +01:00
|
|
|
void
|
|
|
|
Store::commit()
|
|
|
|
{
|
2022-01-30 13:28:30 +01:00
|
|
|
std::lock_guard guard{priv_->lock_};
|
2021-11-09 21:43:11 +01:00
|
|
|
priv_->transaction_maybe_commit(true /*force*/);
|
|
|
|
}
|
|
|
|
|
2022-05-10 07:16:47 +02:00
|
|
|
|
2020-11-03 08:58:59 +01:00
|
|
|
std::size_t
|
2022-03-20 13:12:41 +01:00
|
|
|
Store::for_each_term(Field::Id field_id, Store::ForEachTermFunc func) const
|
2020-11-03 08:58:59 +01:00
|
|
|
{
|
2023-06-30 22:19:17 +02:00
|
|
|
return xapian_db().all_terms(field_from_id(field_id).xapian_term(), func);
|
2020-11-03 08:58:59 +01:00
|
|
|
}
|
|
|
|
|
2022-02-17 22:45:04 +01:00
|
|
|
std::mutex&
|
|
|
|
Store::lock() const
|
|
|
|
{
|
|
|
|
return priv_->lock_;
|
|
|
|
}
|
|
|
|
|
2022-04-28 21:59:03 +02:00
|
|
|
Result<QueryResults>
|
2022-03-04 23:29:21 +01:00
|
|
|
Store::run_query(const std::string& expr,
|
2022-04-28 21:59:03 +02:00
|
|
|
Field::Id sortfield_id,
|
2022-02-19 18:00:47 +01:00
|
|
|
QueryFlags flags, size_t maxnum) const
|
2022-01-30 13:28:30 +01:00
|
|
|
{
|
2022-04-28 21:59:03 +02:00
|
|
|
return Query{*this}.run(expr, sortfield_id, flags, maxnum);
|
2022-01-30 13:28:30 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
size_t
|
|
|
|
Store::count_query(const std::string& expr) const
|
|
|
|
{
|
|
|
|
return xapian_try([&] {
|
|
|
|
std::lock_guard guard{priv_->lock_};
|
|
|
|
Query q{*this};
|
2022-03-04 23:29:21 +01:00
|
|
|
return q.count(expr); }, 0);
|
2022-01-30 13:28:30 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
std::string
|
|
|
|
Store::parse_query(const std::string& expr, bool xapian) const
|
|
|
|
{
|
|
|
|
return xapian_try([&] {
|
|
|
|
std::lock_guard guard{priv_->lock_};
|
|
|
|
Query q{*this};
|
|
|
|
|
|
|
|
return q.parse(expr, xapian);
|
2023-06-30 22:19:17 +02:00
|
|
|
}, std::string{});
|
2022-01-30 13:28:30 +01:00
|
|
|
}
|
2023-08-13 08:44:20 +02:00
|
|
|
|
|
|
|
|
|
|
|
std::vector<std::string>
|
|
|
|
Store::maildirs() const
|
|
|
|
{
|
|
|
|
std::vector<std::string> mdirs;
|
|
|
|
const auto prefix_size = root_maildir().size();
|
|
|
|
Scanner::Handler handler = [&](const std::string& path, auto&& _1, auto&& _2) {
|
|
|
|
mdirs.emplace_back(path.substr(prefix_size));
|
|
|
|
return true;
|
|
|
|
};
|
|
|
|
|
|
|
|
Scanner scanner{root_maildir(), handler, Scanner::Mode::MaildirsOnly};
|
|
|
|
scanner.start();
|
|
|
|
std::sort(mdirs.begin(), mdirs.end());
|
|
|
|
|
|
|
|
return mdirs;
|
|
|
|
}
|
2023-09-09 10:57:05 +02:00
|
|
|
|
|
|
|
Message::Options
|
|
|
|
Store::message_options() const
|
|
|
|
{
|
|
|
|
return priv_->message_opts_;
|
|
|
|
}
|