store/info: Gather some usage statistics

Keep track of the latest-change/latest-index.
This commit is contained in:
Dirk-Jan C. Binnema 2022-05-10 08:16:47 +03:00
parent c8e995ed15
commit 2e9666af0b
5 changed files with 117 additions and 59 deletions

View File

@ -1,5 +1,5 @@
/*
** Copyright (C) 2020 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl>
** Copyright (C) 2020-2022 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl>
**
** This program is free software; you can redistribute it and/or modify it
** under the terms of the GNU General Public License as published by the
@ -57,17 +57,14 @@ struct IndexState {
}
}
bool operator==(State rhs) const
{
bool operator==(State rhs) const {
return state_ == rhs;
}
bool operator!=(State rhs) const
{
bool operator!=(State rhs) const {
return state_ != rhs;
}
void change_to(State new_state)
{
void change_to(State new_state) {
g_debug("changing indexer state %s->%s", name((State)state_),
name((State)new_state));
state_ = new_state;
@ -332,11 +329,14 @@ Indexer::Private::scan_worker()
if (conf_.cleanup) {
g_debug("starting cleanup");
state_.change_to(IndexState::Cleaning);
cleanup();
g_debug("cleanup finished");
}
leave:
store_.index_complete();
state_.change_to(IndexState::Idle);
}
@ -377,6 +377,7 @@ Indexer::Private::stop()
if (scanner_worker_.joinable())
scanner_worker_.join();
store_.index_complete();
state_.change_to(IndexState::Idle);
for (auto&& w : workers_)
if (w.joinable())

View File

@ -48,6 +48,7 @@ using namespace Mu;
static_assert(std::is_same<Store::Id, Xapian::docid>::value, "wrong type for Store::Id");
// Properties
constexpr auto SchemaVersionKey = "schema-version";
constexpr auto RootMaildirKey = "maildir"; // XXX: make this 'root-maildir'
constexpr auto ContactsKey = "contacts";
@ -58,9 +59,26 @@ constexpr auto DefaultBatchSize = 250'000U;
constexpr auto MaxMessageSizeKey = "max-message-size";
constexpr auto DefaultMaxMessageSize = 100'000'000U;
constexpr auto ExpectedSchemaVersion = MU_STORE_SCHEMA_VERSION;
// Stats.
constexpr auto ChangedKey = "changed";
constexpr auto IndexedKey = "indexed";
static std::string
tstamp_to_string(::time_t t)
{
char buf[17];
::snprintf(buf, sizeof(buf), "%" PRIx64, static_cast<int64_t>(t));
return std::string(buf);
}
static ::time_t
string_to_tstamp(const std::string& str)
{
return static_cast<::time_t>(::strtoll(str.c_str(), {}, 16));
}
struct Store::Private {
enum struct XapianOpts { ReadOnly, Open, CreateOverwrite };
@ -81,20 +99,6 @@ struct Store::Private {
: read_only_{false}, db_{make_xapian_db(path, XapianOpts::CreateOverwrite)},
properties_{init_metadata(conf, path, root_maildir, personal_addresses)},
contacts_cache_{"", properties_.personal_addresses} {
/* add synonym */
for (auto&& info: AllMessageFlagInfos) {
constexpr auto field{field_from_id(Field::Id::Flags)};
const auto s1{field.xapian_term(info.name)};
const auto s2{field.xapian_term(info.shortcut)};
writable_db().add_synonym(s1, s2);
}
for (auto&& prio : AllMessagePriorities) {
constexpr auto field{field_from_id(Field::Id::Priority)};
const auto s1{field.xapian_term(to_string(prio))};
const auto s2{field.xapian_term(to_char(prio))};
writable_db().add_synonym(s1, s2);
}
}
~Private()
@ -193,7 +197,7 @@ struct Store::Private {
props.database_path = db_path;
props.schema_version = db().get_metadata(SchemaVersionKey);
props.created = ::atoll(db().get_metadata(CreatedKey).c_str());
props.created = string_to_tstamp(db().get_metadata(CreatedKey));
props.read_only = read_only_;
props.batch_size = ::atoll(db().get_metadata(BatchSizeKey).c_str());
props.max_message_size = ::atoll(db().get_metadata(MaxMessageSizeKey).c_str());
@ -209,7 +213,7 @@ struct Store::Private {
const StringVec& personal_addresses) {
writable_db().set_metadata(SchemaVersionKey, ExpectedSchemaVersion);
writable_db().set_metadata(CreatedKey, Mu::format("%" PRId64, (int64_t)::time({})));
writable_db().set_metadata(CreatedKey, tstamp_to_string(::time({})));
const size_t batch_size = conf.batch_size ? conf.batch_size : DefaultBatchSize;
writable_db().set_metadata(BatchSizeKey, Mu::format("%zu", batch_size));
@ -258,6 +262,7 @@ Store::Private::update_message_unlocked(Message& msg, Store::Id docid)
return xapian_try_result([&]{
writable_db().replace_document(docid, msg.document().xapian_document());
g_debug("updated message @ %s; docid = %u", msg.path().c_str(), docid);
writable_db().set_metadata(ChangedKey, tstamp_to_string(::time({})));
return Ok(std::move(docid));
});
}
@ -272,6 +277,7 @@ Store::Private::update_message_unlocked(Message& msg, const std::string& path_to
field_from_id(Field::Id::Path).xapian_term(path_to_replace),
msg.document().xapian_document());
writable_db().set_metadata(ChangedKey, tstamp_to_string(::time({})));
return Ok(std::move(id));
});
}
@ -289,8 +295,6 @@ Store::Private::find_message_unlocked(Store::Id docid) const
}
Store::Store(const std::string& path, Store::Options opts)
: priv_{std::make_unique<Private>(path, none_of(opts & Store::Options::Writable))}
{
@ -352,6 +356,20 @@ Store::properties() const
return priv_->properties_;
}
Store::Statistics
Store::statistics() const
{
Statistics stats{};
stats.size = size();
stats.last_change = string_to_tstamp(priv_->db().get_metadata(ChangedKey));
stats.last_index = string_to_tstamp(priv_->db().get_metadata(IndexedKey));
return stats;
}
const ContactsCache&
Store::contacts_cache() const
{
@ -455,6 +473,8 @@ Store::remove_message(const std::string& path)
std::lock_guard guard{priv_->lock_};
const auto term{field_from_id(Field::Id::Path).xapian_term(path)};
priv_->writable_db().delete_document(term);
priv_->writable_db().set_metadata(
ChangedKey, tstamp_to_string(::time({})));
g_debug("deleted message @ %s from store", path.c_str());
return true;
@ -473,6 +493,8 @@ Store::remove_messages(const std::vector<Store::Id>& ids)
for (auto&& id : ids) {
priv_->writable_db().delete_document(id);
}
priv_->writable_db().set_metadata(
ChangedKey, tstamp_to_string(::time({})));
});
priv_->transaction_maybe_commit(true /*force*/);
@ -618,6 +640,15 @@ Store::commit()
priv_->transaction_maybe_commit(true /*force*/);
}
void
Store::index_complete()
{
g_debug("marking index complete");
priv_->writable_db().set_metadata(IndexedKey, tstamp_to_string(::time({})));
}
std::size_t
Store::for_each_term(Field::Id field_id, Store::ForEachTermFunc func) const
{

View File

@ -138,6 +138,27 @@ public:
* @return the metadata
*/
const Properties& properties() const;
/**
* Store statistics. Unlike the properties, these can change
* during the lifetime of a store.
*
*/
struct Statistics {
size_t size; /**< number of messages in store */
::time_t last_change; /**< last time any update happened */
::time_t last_index; /**< last time an indexing op was performed */
};
/**
* Get store statistics
*
* @return statistics
*/
Statistics statistics() const;
/**
* Get the ContactsCache object for this store
*
@ -434,6 +455,13 @@ private:
const StringVec& personal_addresses,
const Config& conf);
/**
* Call with indexing has completed to update metadata.
*/
friend class Indexer;
void index_complete();
std::unique_ptr<Private> priv_;
};

View File

@ -1,4 +1,4 @@
.TH MU-INFO 1 "February 2020" "User Manuals"
.TH MU-INFO 1 "May 2022" "User Manuals"
.SH NAME
@ -11,7 +11,8 @@ mu info \- show information about the mu database
.SH DESCRIPTION
\fBmu info\fR is the \fBmu\fR command for getting information about the mu
database.
database. Note that while running (e.g. \fBmu4e\fR), some of the information
may be slightly delayed due to database caching.
.SH OPTIONS

View File

@ -471,47 +471,44 @@ cmd_info(const Mu::Store& store, const MuConfig* opts, GError** err)
{
using namespace tabulate;
auto colorify = [](Table& table) {
for (auto&& row: table) {
if (row.cells().size() < 2)
continue;
row.cells().at(0)->format().font_style({FontStyle::bold})
.font_color({Color::green});
row.cells().at(1)->format().font_color({Color::blue});
}
};
auto tstamp = [](::time_t t)->std::string {
if (t == 0)
return "never";
else
return time_to_string("%c", t);
};
Table info;
//Mu::MaybeAnsi col{!opts->nocolor};
info.add_row({"maildir", store.properties().root_maildir});
info.add_row({"database-path", store.properties().database_path});
info.add_row({"schema-version", store.properties().schema_version});
info.add_row({"max-message-size", format("%zu", store.properties().max_message_size)});
info.add_row({"batch-size", format("%zu", store.properties().batch_size)});
info.add_row({"messages in store", format("%zu", store.size())});
const auto created{store.properties().created};
const auto tstamp{::localtime(&created)};
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wformat-y2k"
char tbuf[64];
strftime(tbuf, sizeof(tbuf), "%c", tstamp);
#pragma GCC diagnostic pop
info.add_row({"created", tbuf});
const auto addrs{store.properties().personal_addresses};
if (addrs.empty())
info.add_row({"personal-address", "<none>"});
else
for (auto&& c : addrs)
info.add_row({"created", tstamp(store.properties().created)});
for (auto&& c : store.properties().personal_addresses)
info.add_row({"personal-address", c});
if (!opts->nocolor) {
for (auto&& row: info) {
row.cells().at(0)->format().font_style({FontStyle::bold})
.font_color({Color::green});
row.cells().at(1)->format().font_color({Color::blue});
}
}
std::cout << info << std::endl;
info.add_row({"messages in store", format("%zu", store.size())});
info.add_row({"last-change", tstamp(store.statistics().last_change)});
info.add_row({"last-index", tstamp(store.statistics().last_index)});
if (!opts->nocolor)
colorify(info);
std::cout << info << '\n';
return MU_OK;
}