mu4e: use faster count queries, document differences

Use faster queries for counting read/unread messages; document why the
results might differ from what you get doing a normal search.

lib/mu-query.cc

@@ -474,27 +474,11 @@ mu_query_count_run (MuQuery *self, const char *searchexpr) try
 	g_return_val_if_fail (self, 0);
 	g_return_val_if_fail (searchexpr, 0);
 
-        // XXX: this _should_ work and be a bit faster, but gives incorrect
-        // results.
-        // find out why.
-        // const auto enq{get_enquire(self, searchexpr,MU_MSG_FIELD_ID_NONE, false, false, NULL)};
-        // auto mset(enq.get_mset(0, self->db().get_doccount()));
-        // mset.fetch();
-        // return mset.size();
+        const auto enq{get_enquire(self, searchexpr,MU_MSG_FIELD_ID_NONE, false, false, NULL)};
+        auto mset(enq.get_mset(0, self->db().get_doccount()));
+        mset.fetch();
 
-        auto msgiter{mu_query_run(self, searchexpr, MU_MSG_FIELD_ID_NONE, -1,
-                                  MU_QUERY_FLAG_NONE, NULL)};
-        if (!msgiter)
-                return 0;
-
-        size_t num{};
-        while (!mu_msg_iter_is_done(msgiter))  {
-                ++num;
-                mu_msg_iter_next(msgiter);
-        }
-        mu_msg_iter_destroy (msgiter);
-
-        return num;
+        return mset.size();
 
 } MU_XAPIAN_CATCH_BLOCK_RETURN (0);
 

lib/mu-query.h

@@ -99,7 +99,6 @@ MuMsgIter* mu_query_run (MuQuery *self, const char* expr,
  */
 size_t mu_query_count_run (MuQuery *self, const char *searchexpr);
 
-
 /**
  * get Xapian's internal string representation of the query
  *

mu/mu-cmd-server.cc

@@ -1045,16 +1045,17 @@ ping_handler (Context& context, const Parameters& params)
                 throw Error{Error::Code::Store, &gerr, "failed to read store"};
 
         const auto queries  = get_string_vec (params, "queries");
-        const auto qresults= [&]() -> std::string {
+        const auto qresults = [&]() -> std::string {
                 if (queries.empty())
                         return {};
 
                 std::string res{":queries ("};
                 for (auto&& q: queries) {
                         const auto count{mu_query_count_run (context.query, q.c_str())};
-                        const auto unreadq{format("(%s) AND flag:unread", q.c_str())};
+                        const auto unreadq{format("flag:unread AND (%s)", q.c_str())};
                         const auto unread{mu_query_count_run (context.query, unreadq.c_str())};
-                        res += format("(:query %s :count %zu :unread %zu)", quote(q).c_str(), count, unread);
+                        res += format("(:query %s :count %zu :unread %zu)", quote(q).c_str(),
+                                      count, unread);
                 }
                 return res + ")";
         }();
@@ -1252,7 +1253,9 @@ make_command_map (Context& context)
       cmap.emplace("ping",
                    CommandInfo{
                            ArgMap{ {"queries",  ArgInfo{Type::List, false,
-                                                   "queries for which to get read/unread numbers"}}},
+                                                   "queries for which to get read/unread numbers"}},
+                                   {"skip-dups",  ArgInfo{Type::Symbol, false,
+                                            "whether to exclude messages with duplicate message-ids"}},},
                            "ping the mu-server and get information in response",
                           [&](const auto& params){ping_handler(context, params);}});
 

mu4e/mu4e-headers.el

@@ -132,27 +132,12 @@ messages that were found in the first set (the \"leaders\").
 (make-obsolete-variable 'mu4e-search-results-limit
                         'mu4e-headers-results-limit "0.9.9.5-dev6")
 
-(defcustom mu4e-headers-skip-duplicates t
-  "With this option set to non-nil, show only one of duplicate
-messages. This is useful when you have multiple copies of the same
-message, which is a common occurrence for example when using Gmail
-and offlineimap."
-  :type 'boolean
-  :group 'mu4e-headers)
-
 (defcustom mu4e-headers-advance-after-mark t
   "With this option set to non-nil, automatically advance to the
 next mail after marking a message in header view."
   :type 'boolean
   :group 'mu4e-headers)
 
-(defcustom mu4e-headers-include-related t
-  "With this option set to non-nil, not just return the matches for
-a searches, but also messages that are related (through their
-references) to these messages. This can be useful e.g. to include
-sent messages into message threads."
-  :type 'boolean
-  :group 'mu4e-headers)
 
 (defvar mu4e-headers-hide-predicate nil
   "Predicate function applied to headers before they are shown;

mu4e/mu4e-vars.el

@@ -140,6 +140,22 @@ some specific setting.")
   :type 'boolean
   :group 'mu4e)
 
+(defcustom mu4e-headers-include-related t
+  "With this option set to non-nil, not just return the matches for
+a searches, but also messages that are related (through their
+references) to these messages. This can be useful e.g. to include
+sent messages into message threads."
+  :type 'boolean
+  :group 'mu4e-headers)
+
+(defcustom mu4e-headers-skip-duplicates t
+  "With this option set to non-nil, show only one of duplicate
+messages. This is useful when you have multiple copies of the same
+message, which is a common occurrence for example when using Gmail
+and offlineimap."
+  :type 'boolean
+  :group 'mu4e-headers)
+
 (defcustom mu4e-change-filenames-when-moving nil
   "Change message file names when moving them.
 When moving messages to different folders, normally mu/mu4e keep

mu4e/mu4e.texi

@@ -3862,18 +3862,18 @@ answers.
 @section General
 
 @subsection Results from @command{mu} and @t{mu4e} differ - why?
+@node mu-mu4e-differ
 In general, the same queries for @command{mu} and @t{mu4e} should
-yield the same results. If they differ, this is usually because one
-of the following reasons:
+yield the same results. If they differ, this is usually because one of
+the following reasons:
 @itemize
 @item different default options:
 mu4e defaults to having @t{mu4e-headers-include-related} and
-@t{mu4e-headers-skip-duplicates} enabled and
 @t{mu4e-headers-results-limit} set to 500. However, the command-line
-@command{mu find}'s corresponding @t{--include-related} and
-@t{--skip-dups} are false, and there's no limit (@t{--maxnum}).
+@command{mu find}'s corresponding @t{--include-related} is false, and
+there's no limit (@t{--maxnum}).
 @item reverse sorting:
-The results amy be different when only one @t{mu4e} and @command{mu
+The results may be different when only one @t{mu4e} and @command{mu
 find} do not both sort their results in the same direction.
 @item shell quoting issues:
 Depending on the shell, various shell metacharacters in search query
@@ -3882,11 +3882,16 @@ sees them, and the query may not be what you think it is. Quoting is
 necessary.
 @end itemize
 
+@subsection The unread/all counts in the main-screen differ from the 'real' numbers - what's going on?
+For speed reasons, the counts do not exclude messages that no longer
+exist in the file-system, nor does it exclude duplicate messages. See
+@ref{mu-mu4e-differ}.
+
 @subsection How can I quickly delete/move/trash a lot of messages?
-You can select ('mark' in Emacs-speak) the messages like you
-would select text in a buffer; the actions you then take (e.g.,
-@key{DEL} for delete, @key{m} for move and @key{t} for trash) apply to
-all selected messages. You can also use functions like
+You can select ('mark' in Emacs-speak) the messages like you would
+select text in a buffer; the actions you then take (e.g., @key{DEL}
+for delete, @key{m} for move and @key{t} for trash) apply to all
+selected messages. You can also use functions like
 @code{mu4e-headers-mark-thread} (@key{T}),
 @code{mu4e-headers-mark-subthread} (@key{t}) to mark whole threads at
 the same time, and @code{mu4e-headers-mark-pattern} (@key{%}) to mark