From 503d7224e0ad69146aae7062f0f813ba90fe3a56 Mon Sep 17 00:00:00 2001 From: "Dirk-Jan C. Binnema" Date: Wed, 10 Nov 2021 21:50:43 +0200 Subject: [PATCH] mu: update the index 'processed' -> 'checked' The 'processed' statistic for indexing was more-or-less synonymous for 'updated'; let's change to something more useful, 'checked' which roughly means the number of messages checked for updates (typically a cheap timestamp check). --- lib/index/mu-indexer.cc | 15 +++++++++------ lib/index/mu-indexer.hh | 12 ++++++------ lib/mu-server.cc | 4 ++-- mu/mu-cmd-index.cc | 2 +- 4 files changed, 18 insertions(+), 15 deletions(-) diff --git a/lib/index/mu-indexer.cc b/lib/index/mu-indexer.cc index 2a8f83c9..38ae1f3a 100644 --- a/lib/index/mu-indexer.cc +++ b/lib/index/mu-indexer.cc @@ -124,7 +124,10 @@ Indexer::Private::handler(const std::string& fullpath, dirstamp_ = store_.dirstamp(fullpath); if (conf_.lazy_check && dirstamp_ >= statbuf->st_mtime && htype == Scanner::HandleType::EnterNewCur) { - g_debug("skip %s (seems up-to-date)", fullpath.c_str()); + g_debug("skip %s (seems up-to-date: %s >= %s)", + fullpath.c_str(), + time_to_string("%FT%T", dirstamp_).c_str(), + time_to_string("%FT%T", statbuf->st_mtime).c_str()); return false; } @@ -145,7 +148,7 @@ Indexer::Private::handler(const std::string& fullpath, } } - g_debug("process %s", fullpath.c_str()); + g_debug("checked %s", fullpath.c_str()); return true; } case Scanner::HandleType::LeaveDir: { @@ -154,6 +157,8 @@ Indexer::Private::handler(const std::string& fullpath, } case Scanner::HandleType::File: { + ++progress_.checked; + if ((size_t)statbuf->st_size > max_message_size_) { g_debug("skip %s (too big: %" G_GINT64_FORMAT " bytes)", fullpath.c_str(), @@ -168,6 +173,8 @@ Indexer::Private::handler(const std::string& fullpath, return false; } + // push the remaining messages to our "todo" queue for + // (re)parsing and adding/updating to the database. fq_.push(std::string{fullpath}); return true; } @@ -194,10 +201,6 @@ Indexer::Private::worker() while (state_ == IndexState::Scanning || !fq_.empty()) { if (!fq_.pop(item, 250ms)) continue; - - // g_debug ("popped (n=%zu) path %s", fq_.size(), item.c_str()); - ++progress_.processed; - try { std::unique_lock lock{lock_}; diff --git a/lib/index/mu-indexer.hh b/lib/index/mu-indexer.hh index d79ed7cb..78527445 100644 --- a/lib/index/mu-indexer.hh +++ b/lib/index/mu-indexer.hh @@ -29,7 +29,7 @@ class Store; /// An object abstracting the index process. class Indexer { - public: +public: /** * Construct an indexer object * @@ -86,10 +86,10 @@ class Indexer { // Object describing current progress struct Progress { - bool running{}; /**< Is an index operation in progress? */ - size_t processed{}; /**< Number of messages processed */ - size_t updated{}; /**< Number of messages added/updated to store */ - size_t removed{}; /**< Number of message removed from store */ + bool running{}; /**< Is an index operation in progress? */ + size_t checked{}; /**< Number of messages checked for changes */ + size_t updated{}; /**< Number of messages (re)parsed & added/updated to store */ + size_t removed{}; /**< Number of message removed from store */ }; /** @@ -101,7 +101,7 @@ class Indexer { */ Progress progress() const; - private: +private: struct Private; std::unique_ptr priv_; }; diff --git a/lib/mu-server.cc b/lib/mu-server.cc index af966095..13779dfc 100644 --- a/lib/mu-server.cc +++ b/lib/mu-server.cc @@ -104,7 +104,7 @@ struct Server::Private { void sent_handler(const Parameters& params); void view_handler(const Parameters& params); - private: +private: // helpers Sexp build_message_sexp(MuMsg* msg, unsigned docid, @@ -754,7 +754,7 @@ get_stats(const Indexer::Progress& stats, const std::string& state) lst.add_prop(":info", Sexp::make_symbol("index")); lst.add_prop(":status", Sexp::make_symbol(std::string{state})); - lst.add_prop(":processed", Sexp::make_number(stats.processed)); + lst.add_prop(":checked", Sexp::make_number(stats.checked)); lst.add_prop(":updated", Sexp::make_number(stats.updated)); lst.add_prop(":cleaned-up", Sexp::make_number(stats.removed)); diff --git a/mu/mu-cmd-index.cc b/mu/mu-cmd-index.cc index 8c262652..d7788c29 100644 --- a/mu/mu-cmd-index.cc +++ b/mu/mu-cmd-index.cc @@ -72,7 +72,7 @@ print_stats(const Indexer::Progress& stats, bool color) using Color = MaybeAnsi::Color; std::cout << col.fg(Color::Yellow) << kars[++i % 4] << col.reset() << " indexing messages; " - << "processed: " << col.fg(Color::Green) << stats.processed << col.reset() + << "checked: " << col.fg(Color::Green) << stats.checked << col.reset() << "; updated/new: " << col.fg(Color::Green) << stats.updated << col.reset() << "; cleaned-up: " << col.fg(Color::Green) << stats.removed << col.reset(); }