mirror of https://github.com/djcb/mu.git
mu-server: implement temp-file optimization
It can be faster to feed big mu -> mu4e data, such as contacts are message headers through a temp-file instead directly though stdout; implement this, and add the server parameter --allow-temp-file. Implement this the "contacts" and "find" commands.
This commit is contained in:
parent
0ace413f80
commit
924bb2145e
|
@ -22,7 +22,7 @@
|
||||||
#include "message/mu-message.hh"
|
#include "message/mu-message.hh"
|
||||||
#include "mu-server.hh"
|
#include "mu-server.hh"
|
||||||
|
|
||||||
#include <iostream>
|
#include <fstream>
|
||||||
#include <string>
|
#include <string>
|
||||||
#include <algorithm>
|
#include <algorithm>
|
||||||
#include <atomic>
|
#include <atomic>
|
||||||
|
@ -51,16 +51,19 @@ using namespace Mu;
|
||||||
|
|
||||||
/// @brief object to manage the server-context for all commands.
|
/// @brief object to manage the server-context for all commands.
|
||||||
struct Server::Private {
|
struct Server::Private {
|
||||||
Private(Store& store, Output output)
|
Private(Store& store, const Server::Options& opts, Output output)
|
||||||
: store_{store}, output_{output},
|
: store_{store}, options_{opts}, output_{output},
|
||||||
command_handler_{make_command_map()},
|
command_handler_{make_command_map()},
|
||||||
keep_going_{true}
|
keep_going_{true},
|
||||||
|
tmp_dir_{unwrap(make_temp_dir())}
|
||||||
{}
|
{}
|
||||||
|
|
||||||
~Private() {
|
~Private() {
|
||||||
indexer().stop();
|
indexer().stop();
|
||||||
if (index_thread_.joinable())
|
if (index_thread_.joinable())
|
||||||
index_thread_.join();
|
index_thread_.join();
|
||||||
|
if (!tmp_dir_.empty())
|
||||||
|
remove_directory(tmp_dir_);
|
||||||
}
|
}
|
||||||
//
|
//
|
||||||
// construction helpers
|
// construction helpers
|
||||||
|
@ -88,6 +91,7 @@ struct Server::Private {
|
||||||
}
|
}
|
||||||
|
|
||||||
size_t output_results(const QueryResults& qres, size_t batch_size) const;
|
size_t output_results(const QueryResults& qres, size_t batch_size) const;
|
||||||
|
size_t output_results_temp_file(const QueryResults& qres, size_t batch_size) const;
|
||||||
|
|
||||||
//
|
//
|
||||||
// handlers for various commands.
|
// handlers for various commands.
|
||||||
|
@ -125,11 +129,15 @@ private:
|
||||||
|
|
||||||
void view_mark_as_read(Store::Id docid, Message&& msg, bool rename);
|
void view_mark_as_read(Store::Id docid, Message&& msg, bool rename);
|
||||||
|
|
||||||
|
std::pair<std::ofstream, std::string> make_temp_file_stream() const;
|
||||||
|
|
||||||
Store& store_;
|
Store& store_;
|
||||||
|
Server::Options options_;
|
||||||
Server::Output output_;
|
Server::Output output_;
|
||||||
const CommandHandler command_handler_;
|
const CommandHandler command_handler_;
|
||||||
std::atomic<bool> keep_going_{};
|
std::atomic<bool> keep_going_{};
|
||||||
std::thread index_thread_;
|
std::thread index_thread_;
|
||||||
|
std::string tmp_dir_;
|
||||||
};
|
};
|
||||||
|
|
||||||
static Sexp
|
static Sexp
|
||||||
|
@ -220,7 +228,8 @@ Server::Private::make_command_map()
|
||||||
{":after",
|
{":after",
|
||||||
ArgInfo{Type::String, false, "only contacts seen after time_t string"}},
|
ArgInfo{Type::String, false, "only contacts seen after time_t string"}},
|
||||||
{":tstamp", ArgInfo{Type::String, false, "return changes since tstamp"}},
|
{":tstamp", ArgInfo{Type::String, false, "return changes since tstamp"}},
|
||||||
{":maxnum", ArgInfo{Type::Number, false, "max number of contacts to return"}}},
|
{":maxnum", ArgInfo{Type::Number, false, "max number of contacts to return"}}
|
||||||
|
},
|
||||||
"get contact information",
|
"get contact information",
|
||||||
[&](const auto& params) { contacts_handler(params); }});
|
[&](const auto& params) { contacts_handler(params); }});
|
||||||
cmap.emplace(
|
cmap.emplace(
|
||||||
|
@ -492,6 +501,21 @@ Server::Private::compose_handler(const Command& cmd)
|
||||||
output_sexp(comp_lst);
|
output_sexp(comp_lst);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// create pair of ostream / name
|
||||||
|
std::pair<std::ofstream, std::string>
|
||||||
|
Server::Private::make_temp_file_stream() const
|
||||||
|
{
|
||||||
|
auto tmp_eld{join_paths(tmp_dir_, mu_format("mu-{}.eld", g_get_monotonic_time()))};
|
||||||
|
std::ofstream output{tmp_eld, std::ios::out};
|
||||||
|
if (!output.good())
|
||||||
|
throw Mu::Error{Error::Code::File, "failed to create temp-file"};
|
||||||
|
|
||||||
|
return make_pair<std::ofstream, std::string>(std::move(output),
|
||||||
|
std::move(tmp_eld));
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
void
|
void
|
||||||
Server::Private::contacts_handler(const Command& cmd)
|
Server::Private::contacts_handler(const Command& cmd)
|
||||||
{
|
{
|
||||||
|
@ -520,20 +544,24 @@ Server::Private::contacts_handler(const Command& cmd)
|
||||||
/* only include newer-than-x contacts */
|
/* only include newer-than-x contacts */
|
||||||
if (after > ci.message_date)
|
if (after > ci.message_date)
|
||||||
return true;
|
return true;
|
||||||
|
|
||||||
n++;
|
n++;
|
||||||
|
|
||||||
contacts.add(ci.display_name());
|
contacts.add(ci.display_name());
|
||||||
return maxnum == 0 || n < maxnum;
|
return maxnum == 0 || n < maxnum;
|
||||||
});
|
});
|
||||||
|
|
||||||
Sexp seq;
|
|
||||||
seq.put_props(":contacts", contacts,
|
|
||||||
":tstamp", mu_format("{}", g_get_monotonic_time()));
|
|
||||||
|
|
||||||
/* dump the contacts cache as a giant sexp */
|
/* dump the contacts cache as a giant sexp */
|
||||||
mu_debug("sending {} of {} contact(s)", n, store().contacts_cache().size());
|
mu_debug("sending {} of {} contact(s)", n, store().contacts_cache().size());
|
||||||
output_sexp(seq, Server::OutputFlags::SplitList);
|
if (options_.allow_temp_file) {
|
||||||
|
auto&& [output, tmp_eld] = make_temp_file_stream();
|
||||||
|
output << contacts;
|
||||||
|
output_sexp(Sexp{":tstamp"_sym, mu_format("{}", g_get_monotonic_time()),
|
||||||
|
":contacts-temp-file"_sym, tmp_eld});
|
||||||
|
} else {
|
||||||
|
Sexp seq;
|
||||||
|
seq.put_props(":contacts", contacts,
|
||||||
|
":tstamp", mu_format("{}", g_get_monotonic_time()));
|
||||||
|
output_sexp(seq, Server::OutputFlags::SplitList);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/*
|
/*
|
||||||
|
@ -587,7 +615,6 @@ Server::Private::output_results(const QueryResults& qres, size_t batch_size) con
|
||||||
if (!msg)
|
if (!msg)
|
||||||
continue;
|
continue;
|
||||||
++n;
|
++n;
|
||||||
|
|
||||||
// construct sexp for a single header.
|
// construct sexp for a single header.
|
||||||
auto qm{mi.query_match()};
|
auto qm{mi.query_match()};
|
||||||
auto msgsexp{build_message_sexp(*msg, mi.doc_id(), qm)};
|
auto msgsexp{build_message_sexp(*msg, mi.doc_id(), qm)};
|
||||||
|
@ -608,13 +635,47 @@ Server::Private::output_results(const QueryResults& qres, size_t batch_size) con
|
||||||
return n;
|
return n;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
size_t
|
||||||
|
Server::Private::output_results_temp_file(const QueryResults& qres, size_t batch_size) const
|
||||||
|
{
|
||||||
|
// create an output stream with a file name
|
||||||
|
size_t n{};
|
||||||
|
|
||||||
|
auto&& [output, tmp_eld] = make_temp_file_stream();
|
||||||
|
output << '(';
|
||||||
|
for(auto&& mi: qres) {
|
||||||
|
|
||||||
|
auto msg{mi.message()};
|
||||||
|
if (!msg)
|
||||||
|
continue;
|
||||||
|
|
||||||
|
auto qm{mi.query_match()}; // construct sexp for a single header.
|
||||||
|
output << build_message_sexp(*msg, mi.doc_id(), qm);
|
||||||
|
++n;
|
||||||
|
|
||||||
|
if (n % batch_size == 0) {
|
||||||
|
output << ')';
|
||||||
|
output_sexp(Sexp{":headers-temp-file"_sym, tmp_eld});
|
||||||
|
auto new_stream{make_temp_file_stream()};
|
||||||
|
output = std::move(new_stream.first);
|
||||||
|
tmp_eld = std::move(new_stream.second);
|
||||||
|
output << '(';
|
||||||
|
}
|
||||||
|
}
|
||||||
|
output << ')';
|
||||||
|
output_sexp(Sexp{":headers-temp-file"_sym, tmp_eld});
|
||||||
|
|
||||||
|
return n;
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
void
|
void
|
||||||
Server::Private::find_handler(const Command& cmd)
|
Server::Private::find_handler(const Command& cmd)
|
||||||
{
|
{
|
||||||
const auto q{cmd.string_arg(":query").value_or("")};
|
const auto q{cmd.string_arg(":query").value_or("")};
|
||||||
const auto threads{cmd.boolean_arg(":threads")};
|
const auto threads{cmd.boolean_arg(":threads")};
|
||||||
// perhaps let mu4e set this as frame-lines of the appropriate frame.
|
// perhaps let mu4e set this as frame-lines of the appropriate frame.
|
||||||
const auto batch_size{cmd.number_arg(":batch-size").value_or(110)};
|
const auto batch_size{cmd.number_arg(":batch-size").value_or(200)};
|
||||||
const auto descending{cmd.boolean_arg(":descending")};
|
const auto descending{cmd.boolean_arg(":descending")};
|
||||||
const auto maxnum{cmd.number_arg(":maxnum").value_or(-1) /*unlimited*/};
|
const auto maxnum{cmd.number_arg(":maxnum").value_or(-1) /*unlimited*/};
|
||||||
const auto skip_dups{cmd.boolean_arg(":skip-dups")};
|
const auto skip_dups{cmd.boolean_arg(":skip-dups")};
|
||||||
|
@ -659,7 +720,9 @@ Server::Private::find_handler(const Command& cmd)
|
||||||
* knows it should erase the headers buffer. this will ensure that the
|
* knows it should erase the headers buffer. this will ensure that the
|
||||||
* output of two finds will not be mixed. */
|
* output of two finds will not be mixed. */
|
||||||
output_sexp(Sexp().put_props(":erase", Sexp::t_sym));
|
output_sexp(Sexp().put_props(":erase", Sexp::t_sym));
|
||||||
const auto foundnum{output_results(*qres, static_cast<size_t>(batch_size))};
|
const auto bsize{static_cast<size_t>(batch_size)};
|
||||||
|
const auto foundnum =options_.allow_temp_file ?
|
||||||
|
output_results_temp_file(*qres, bsize) : output_results(*qres, bsize);
|
||||||
output_sexp(Sexp().put_props(":found", foundnum));
|
output_sexp(Sexp().put_props(":found", foundnum));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -899,7 +962,6 @@ Server::Private::ping_handler(const Command& cmd)
|
||||||
":doccount", storecount)));
|
":doccount", storecount)));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
void
|
void
|
||||||
Server::Private::queries_handler(const Command& cmd)
|
Server::Private::queries_handler(const Command& cmd)
|
||||||
{
|
{
|
||||||
|
@ -1011,8 +1073,8 @@ Server::Private::view_handler(const Command& cmd)
|
||||||
/* otherwise, mark message and and possible dups as read */
|
/* otherwise, mark message and and possible dups as read */
|
||||||
}
|
}
|
||||||
|
|
||||||
Server::Server(Store& store, Server::Output output)
|
Server::Server(Store& store, const Server::Options& opts, Server::Output output)
|
||||||
: priv_{std::make_unique<Private>(store, output)}
|
: priv_{std::make_unique<Private>(store, opts, output)}
|
||||||
{}
|
{}
|
||||||
|
|
||||||
Server::~Server() = default;
|
Server::~Server() = default;
|
||||||
|
|
|
@ -50,13 +50,17 @@ public:
|
||||||
*/
|
*/
|
||||||
using Output = std::function<void(const Sexp& sexp, OutputFlags flags)>;
|
using Output = std::function<void(const Sexp& sexp, OutputFlags flags)>;
|
||||||
|
|
||||||
|
struct Options {
|
||||||
|
bool allow_temp_file; /**< temp file optimization allowed? */
|
||||||
|
};
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Construct a new server
|
* Construct a new server
|
||||||
*
|
*
|
||||||
* @param store a message store object
|
* @param store a message store object
|
||||||
* @param output callable for the server responses.
|
* @param output callable for the server responses.
|
||||||
*/
|
*/
|
||||||
Server(Store& store, Output output);
|
Server(Store& store, const Options& opts, Output output);
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* DTOR
|
* DTOR
|
||||||
|
|
|
@ -16,10 +16,6 @@ database. The output uses s-expressions. *mu server* is not meant for use by
|
||||||
humans, except for debugging purposes. Instead, it is designed specifically for
|
humans, except for debugging purposes. Instead, it is designed specifically for
|
||||||
the *mu4e* e-mail client.
|
the *mu4e* e-mail client.
|
||||||
|
|
||||||
In this man-page, we document the commands *mu server* accepts, as well as their
|
|
||||||
responses. In general, the commands sent to the server are s-expressions of the
|
|
||||||
form:
|
|
||||||
|
|
||||||
#+begin_example
|
#+begin_example
|
||||||
(<command-name> :param1 value1 :param2 value2)
|
(<command-name> :param1 value1 :param2 value2)
|
||||||
#+end_example
|
#+end_example
|
||||||
|
@ -60,6 +56,14 @@ List available commands (and try with ~--verbose~)
|
||||||
|
|
||||||
Evaluate a mu4e server s-expression
|
Evaluate a mu4e server s-expression
|
||||||
|
|
||||||
|
** --allow-temp-file
|
||||||
|
|
||||||
|
If set, allow for the output of some commands to use temp-files rather than
|
||||||
|
directly through the emacs process input/output. This is noticeably faster for
|
||||||
|
commands with a lot of output, esp. when the the temp-file uses a in-memory
|
||||||
|
file-system.
|
||||||
|
|
||||||
|
|
||||||
#+include: "muhome.inc" :minlevel 2
|
#+include: "muhome.inc" :minlevel 2
|
||||||
|
|
||||||
#+include: "common-options.inc" :minlevel 1
|
#+include: "common-options.inc" :minlevel 1
|
||||||
|
|
|
@ -117,7 +117,10 @@ Mu::mu_cmd_server(const Mu::Options& opts) try {
|
||||||
if (!store)
|
if (!store)
|
||||||
return Err(store.error());
|
return Err(store.error());
|
||||||
|
|
||||||
Server server{*store, output_sexp_stdout};
|
Server::Options sopts{};
|
||||||
|
sopts.allow_temp_file = opts.server.allow_temp_file;
|
||||||
|
|
||||||
|
Server server{*store, sopts, output_sexp_stdout};
|
||||||
mu_message("created server with store @ {}; maildir @ {}; debug-mode {};"
|
mu_message("created server with store @ {}; maildir @ {}; debug-mode {};"
|
||||||
"readline: {}",
|
"readline: {}",
|
||||||
store->path(), store->root_maildir(),
|
store->path(), store->root_maildir(),
|
||||||
|
|
|
@ -498,6 +498,10 @@ sub_server(CLI::App& sub, Options& opts)
|
||||||
sub.add_option("--eval", opts.server.eval,
|
sub.add_option("--eval", opts.server.eval,
|
||||||
"Evaluate mu server expression")
|
"Evaluate mu server expression")
|
||||||
->excludes("--commands");
|
->excludes("--commands");
|
||||||
|
sub.add_flag("--allow-temp-file", opts.server.allow_temp_file,
|
||||||
|
"Allow for the temp-file optimization")
|
||||||
|
->excludes("--commands");
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
static void
|
static void
|
||||||
|
|
|
@ -222,6 +222,7 @@ struct Options {
|
||||||
struct Server {
|
struct Server {
|
||||||
bool commands; /**< dump docs for commands */
|
bool commands; /**< dump docs for commands */
|
||||||
std::string eval; /**< command to evaluate */
|
std::string eval; /**< command to evaluate */
|
||||||
|
bool allow_temp_file; /**< temp-file optimization allowed? */
|
||||||
} server;
|
} server;
|
||||||
|
|
||||||
/*
|
/*
|
||||||
|
|
Loading…
Reference in New Issue