emacs
/
mu
mirror of https://github.com/djcb/mu.git
1
0
Fork 0
Code Issues Releases Wiki Activity

lib/query,parser: update to use mu-message-fields

This commit is contained in:
Dirk-Jan C. Binnema 2022-03-04 00:06:31 +02:00
parent 7c185590e4
commit 0df7a6959a
8 changed files with 229 additions and 250 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
lib/mu-data.hh
View File

@ -24,6 +24,7 @@
#include <iostream> #include <iostream>
#include <regex> #include <regex>
#include <mu-message.hh>
#include <utils/mu-utils.hh> #include <utils/mu-utils.hh>
namespace Mu { namespace Mu {
@ -34,13 +35,14 @@ struct Data {
enum class Type { Value, Range }; enum class Type { Value, Range };
virtual ~Data() = default; virtual ~Data() = default;
Type type; /**< type of data */ Type type; /**< type of data */
std::string field; /**< full name of the field */ std::string field; /**< full name of the field */
std::string prefix; /**< Xapian prefix for thef field */ std::string prefix; /**< Xapian prefix for thef field */
unsigned id; /**< Xapian value no for the field */ Message::Field::Id id; /**< Xapian value no for the field */
protected: protected:
Data(Type _type, const std::string& _field, const std::string& _prefix, unsigned _id) Data(Type _type, const std::string& _field, const std::string& _prefix,
Message::Field::Id _id)
: type(_type), field(_field), prefix(_prefix), id(_id) : type(_type), field(_field), prefix(_prefix), id(_id)
{ {
} }
@ -80,7 +82,7 @@ struct Range : public Data {
*/ */
Range(const std::string& _field, Range(const std::string& _field,
const std::string& _prefix, const std::string& _prefix,
unsigned _id, Message::Field::Id _id,
const std::string& _lower, const std::string& _lower,
const std::string& _upper) const std::string& _upper)
: :
@ -103,12 +105,12 @@ struct Value : public Data {
* *
* @param _field the field * @param _field the field
* @param _prefix the xapian prefix * @param _prefix the xapian prefix
* @param _id xapian value number * @param _id field id
* @param _value the value * @param _value the value
*/ */
Value(const std::string& _field, Value(const std::string& _field,
const std::string& _prefix, const std::string& _prefix,
unsigned _id, Message::Field::Id _id,
const std::string& _value, const std::string& _value,
bool _phrase = false) bool _phrase = false)
: Data(Value::Type::Value, _field, _prefix, _id), value(_value), phrase(_phrase) : Data(Value::Type::Value, _field, _prefix, _id), value(_value), phrase(_phrase)

230
lib/mu-parser.cc
View File

@ -17,12 +17,18 @@
** 02110-1301, USA. ** 02110-1301, USA.
*/ */
#include "mu-parser.hh" #include "mu-parser.hh"
#include <algorithm>
#include <optional>
#include "mu-tokenizer.hh" #include "mu-tokenizer.hh"
#include "utils/mu-utils.hh" #include "utils/mu-utils.hh"
#include "utils/mu-error.hh" #include "utils/mu-error.hh"
#include <algorithm> #include "mu-message.hh"
using namespace Mu; using namespace Mu;
using namespace Mu::Message;
// 3 precedence levels: units (NOT,()) > factors (OR) > terms (AND) // 3 precedence levels: units (NOT,()) > factors (OR) > terms (AND)
@ -52,17 +58,14 @@ struct FieldInfo {
const std::string field; const std::string field;
const std::string prefix; const std::string prefix;
bool supports_phrase; bool supports_phrase;
unsigned id; Field::Id id;
}; };
using FieldInfoVec = std::vector<FieldInfo>; using FieldInfoVec = std::vector<FieldInfo>;
using Flags = Parser::Flags;
struct Parser::Private { struct Parser::Private {
Private(const Store& store, Flags flags) : store_{store}, flags_{flags} {} Private(const Store& store, Parser::Flags flags) : store_{store}, flags_{flags} {}
std::vector<std::string> process_regex(const std::string& field, std::vector<std::string> process_regex(const std::string& field,
const std::regex& rx) const; const std::regex& rx) const;
Mu::Tree term_1(Mu::Tokens& tokens, WarningVec& warnings) const; Mu::Tree term_1(Mu::Tokens& tokens, WarningVec& warnings) const;
Mu::Tree term_2(Mu::Tokens& tokens, Node::Type& op, WarningVec& warnings) const; Mu::Tree term_2(Mu::Tokens& tokens, Node::Type& op, WarningVec& warnings) const;
@ -71,118 +74,92 @@ struct Parser::Private {
Mu::Tree unit(Mu::Tokens& tokens, WarningVec& warnings) const; Mu::Tree unit(Mu::Tokens& tokens, WarningVec& warnings) const;
Mu::Tree data(Mu::Tokens& tokens, WarningVec& warnings) const; Mu::Tree data(Mu::Tokens& tokens, WarningVec& warnings) const;
Mu::Tree range(const FieldInfoVec& fields, Mu::Tree range(const FieldInfoVec& fields,
const std::string& lower, const std::string& lower,
const std::string& upper, const std::string& upper,
size_t pos, size_t pos,
WarningVec& warnings) const; WarningVec& warnings) const;
Mu::Tree regex(const FieldInfoVec& fields, Mu::Tree regex(const FieldInfoVec& fields,
const std::string& v, const std::string& v,
size_t pos, size_t pos,
WarningVec& warnings) const; WarningVec& warnings) const;
Mu::Tree value(const FieldInfoVec& fields, Mu::Tree value(const FieldInfoVec& fields,
const std::string& v, const std::string& v,
size_t pos, size_t pos,
WarningVec& warnings) const; WarningVec& warnings) const;
private: private:
const Store& store_; const Store& store_;
const Flags flags_; const Parser::Flags flags_;
}; };
static MuMsgFieldId
field_id(const std::string& field)
{
if (field.empty())
return MU_MSG_FIELD_ID_NONE;
MuMsgFieldId id = mu_msg_field_id_from_name(field.c_str(), FALSE);
if (id != MU_MSG_FIELD_ID_NONE)
return id;
else if (field.length() == 1)
return mu_msg_field_id_from_shortcut(field[0], FALSE);
else
return MU_MSG_FIELD_ID_NONE;
}
static std::string static std::string
process_value(const std::string& field, const std::string& value) process_value(const std::string& field, const std::string& value)
{ {
const auto id = field_id(field); const auto id_opt{message_field_id(field)};
if (id == MU_MSG_FIELD_ID_NONE) if (id_opt) {
return value; switch (*id_opt) {
switch (id) { case Field::Id::Priority: {
case MU_MSG_FIELD_ID_PRIO: { if (!value.empty())
if (!value.empty()) return std::string(1, value[0]);
return std::string(1, value[0]); } break;
} break; case Field::Id::Flags:
if (const auto info{message_flag_info(value)}; info)
case MU_MSG_FIELD_ID_FLAGS: return std::string(1, info->shortcut_lower());
if (const auto info{message_flag_info(value)}; info) break;
return std::string(1, info->shortcut_lower()); default:
break; break;
}
default:
break;
} }
return value; // XXX prio/flags, etc. alias return value; // XXX prio/flags, etc. alias
} }
static void static void
add_field(std::vector<FieldInfo>& fields, MuMsgFieldId id) add_field(std::vector<FieldInfo>& fields, Field::Id field_id)
{ {
const auto shortcut = mu_msg_field_shortcut(id); const auto field{message_field(field_id)};
if (!shortcut) if (!field.shortcut)
return; // can't be searched return; // can't be searched
const auto name = mu_msg_field_name(id); fields.emplace_back(FieldInfo{std::string{field.name}, field.xapian_term(),
const auto pfx = mu_msg_field_xapian_prefix(id); field.is_full_text(), field_id});
if (!name || !pfx)
return;
fields.push_back({{name}, {pfx}, (bool)mu_msg_field_xapian_index(id), id});
} }
static std::vector<FieldInfo> static std::vector<FieldInfo>
process_field(const std::string& field, Flags flags) process_field(const std::string& field_str, Parser::Flags flags)
{ {
std::vector<FieldInfo> fields; std::vector<FieldInfo> fields;
if (any_of(flags & Flags::UnitTest)) { if (any_of(flags & Parser::Flags::UnitTest)) {
add_field(fields, MU_MSG_FIELD_ID_MSGID); add_field(fields, Field::Id::MessageId);
return fields; return fields;
} }
if (field == "contact" || field == "recip") { // multi fields if (field_str == "contact" || field_str == "recip") { // multi fields
add_field(fields, MU_MSG_FIELD_ID_TO); add_field(fields, Field::Id::To);
add_field(fields, MU_MSG_FIELD_ID_CC); add_field(fields, Field::Id::Cc);
add_field(fields, MU_MSG_FIELD_ID_BCC); add_field(fields, Field::Id::Bcc);
if (field == "contact") if (field_str == "contact")
add_field(fields, MU_MSG_FIELD_ID_FROM); add_field(fields, Field::Id::From);
} else if (field == "") { } else if (field_str.empty()) {
add_field(fields, MU_MSG_FIELD_ID_TO); add_field(fields, Field::Id::To);
add_field(fields, MU_MSG_FIELD_ID_CC); add_field(fields, Field::Id::Cc);
add_field(fields, MU_MSG_FIELD_ID_BCC); add_field(fields, Field::Id::Bcc);
add_field(fields, MU_MSG_FIELD_ID_FROM); add_field(fields, Field::Id::From);
add_field(fields, MU_MSG_FIELD_ID_SUBJECT); add_field(fields, Field::Id::Subject);
add_field(fields, MU_MSG_FIELD_ID_BODY_TEXT); add_field(fields, Field::Id::BodyText);
} else { } else if (const auto id_opt{message_field_id(field_str)}; id_opt)
const auto id = field_id(field); add_field(fields, *id_opt);
if (id != MU_MSG_FIELD_ID_NONE)
add_field(fields, id);
}
return fields; return fields;
} }
static bool static bool
is_range_field(const std::string& field) is_range_field(const std::string& field_str)
{ {
const auto id = field_id(field); if (const auto field_id_opt{message_field_id(field_str)}; !field_id_opt)
if (id == MU_MSG_FIELD_ID_NONE)
return false; return false;
else else
return mu_msg_field_is_range_field(id); return message_field(*field_id_opt).is_range();
} }
struct MyRange { struct MyRange {
@ -191,19 +168,20 @@ struct MyRange {
}; };
static MyRange static MyRange
process_range(const std::string& field, const std::string& lower, const std::string& upper) process_range(const std::string& field_str,
const std::string& lower, const std::string& upper)
{ {
const auto id = field_id(field); const auto id_opt{message_field_id(field_str)};
if (id == MU_MSG_FIELD_ID_NONE) if (!id_opt)
return {lower, upper}; return {lower, upper};
std::string l2 = lower; std::string l2 = lower;
std::string u2 = upper; std::string u2 = upper;
if (id == MU_MSG_FIELD_ID_DATE) { if (*id_opt == Field::Id::Date) {
l2 = Mu::date_to_time_t_string(lower, true); l2 = Mu::date_to_time_t_string(lower, true);
u2 = Mu::date_to_time_t_string(upper, false); u2 = Mu::date_to_time_t_string(upper, false);
} else if (id == MU_MSG_FIELD_ID_SIZE) { } else if (*id_opt == Field::Id::Size) {
l2 = Mu::size_to_string(lower, true); l2 = Mu::size_to_string(lower, true);
u2 = Mu::size_to_string(upper, false); u2 = Mu::size_to_string(upper, false);
} }
@ -212,16 +190,17 @@ process_range(const std::string& field, const std::string& lower, const std::str
} }
std::vector<std::string> std::vector<std::string>
Parser::Private::process_regex(const std::string& field, const std::regex& rx) const Parser::Private::process_regex(const std::string& field_str,
const std::regex& rx) const
{ {
const auto id = field_id(field); const auto id_opt{message_field_id(field_str)};
if (id == MU_MSG_FIELD_ID_NONE) if (!id_opt)
return {}; return {};
char pfx[] = {mu_msg_field_shortcut(id), '\0'}; const auto field{message_field(*id_opt)};
const auto prefix{field.xapian_term()};
std::vector<std::string> terms; std::vector<std::string> terms;
store_.for_each_term(pfx, [&](auto&& str) { store_.for_each_term(prefix, [&](auto&& str) {
if (std::regex_search(str.c_str() + 1, rx)) // avoid copy if (std::regex_search(str.c_str() + 1, rx)) // avoid copy
terms.emplace_back(str); terms.emplace_back(str);
return true; return true;
@ -244,9 +223,9 @@ empty()
Mu::Tree Mu::Tree
Parser::Private::value(const FieldInfoVec& fields, Parser::Private::value(const FieldInfoVec& fields,
const std::string& v, const std::string& v,
size_t pos, size_t pos,
WarningVec& warnings) const WarningVec& warnings) const
{ {
auto val = utf8_flatten(v); auto val = utf8_flatten(v);
@ -256,30 +235,30 @@ Parser::Private::value(const FieldInfoVec& fields,
if (fields.size() == 1) { if (fields.size() == 1) {
const auto item = fields.front(); const auto item = fields.front();
return Tree({Node::Type::Value, return Tree({Node::Type::Value,
std::make_unique<Value>(item.field, std::make_unique<Value>(item.field,
item.prefix, item.prefix,
item.id, item.id,
process_value(item.field, val), process_value(item.field, val),
item.supports_phrase)}); item.supports_phrase)});
} }
// a 'multi-field' such as "recip:" // a 'multi-field' such as "recip:"
Tree tree(Node{Node::Type::OpOr}); Tree tree(Node{Node::Type::OpOr});
for (const auto& item : fields) for (const auto& item : fields)
tree.add_child(Tree({Node::Type::Value, tree.add_child(Tree({Node::Type::Value,
std::make_unique<Value>(item.field, std::make_unique<Value>(item.field,
item.prefix, item.prefix,
item.id, item.id,
process_value(item.field, val), process_value(item.field, val),
item.supports_phrase)})); item.supports_phrase)}));
return tree; return tree;
} }
Mu::Tree Mu::Tree
Parser::Private::regex(const FieldInfoVec& fields, Parser::Private::regex(const FieldInfoVec& fields,
const std::string& v, const std::string& v,
size_t pos, size_t pos,
WarningVec& warnings) const WarningVec& warnings) const
{ {
if (v.length() < 2) if (v.length() < 2)
throw BUG("expected regexp, got '%s'", v.c_str()); throw BUG("expected regexp, got '%s'", v.c_str());
@ -312,10 +291,10 @@ Parser::Private::regex(const FieldInfoVec& fields,
Mu::Tree Mu::Tree
Parser::Private::range(const FieldInfoVec& fields, Parser::Private::range(const FieldInfoVec& fields,
const std::string& lower, const std::string& lower,
const std::string& upper, const std::string& upper,
size_t pos, size_t pos,
WarningVec& warnings) const WarningVec& warnings) const
{ {
if (fields.empty()) if (fields.empty())
throw BUG("expected field"); throw BUG("expected field");
@ -329,11 +308,11 @@ Parser::Private::range(const FieldInfoVec& fields,
prange = process_range(field.field, upper, lower); prange = process_range(field.field, upper, lower);
return Tree({Node::Type::Range, return Tree({Node::Type::Range,
std::make_unique<Range>(field.field, std::make_unique<Range>(field.field,
field.prefix, field.prefix,
field.id, field.id,
prange.lower, prange.lower,
prange.upper)}); prange.upper)});
} }
Mu::Tree Mu::Tree
@ -370,10 +349,10 @@ Parser::Private::data(Mu::Tokens& tokens, WarningVec& warnings) const
const auto dotdot = val.find(".."); const auto dotdot = val.find("..");
if (dotdot != std::string::npos) if (dotdot != std::string::npos)
return range(fields, return range(fields,
val.substr(0, dotdot), val.substr(0, dotdot),
val.substr(dotdot + 2), val.substr(dotdot + 2),
token.pos, token.pos,
warnings); warnings);
else if (is_range_field(fields.front().field)) { else if (is_range_field(fields.front().field)) {
// range field without a range - treat as field:val..val // range field without a range - treat as field:val..val
return range(fields, val, val, token.pos, warnings); return range(fields, val, val, token.pos, warnings);
@ -511,7 +490,8 @@ Parser::Private::term_1(Mu::Tokens& tokens, WarningVec& warnings) const
} }
} }
Mu::Parser::Parser(const Store& store, Flags flags) : priv_{std::make_unique<Private>(store, flags)} Mu::Parser::Parser(const Store& store, Parser::Flags flags) :
priv_{std::make_unique<Private>(store, flags)}
{ {
} }

17
lib/mu-query-match-deciders.cc
View File

@ -23,6 +23,7 @@
#include "utils/mu-option.hh" #include "utils/mu-option.hh"
using namespace Mu; using namespace Mu;
using namespace Mu::Message;
// We use a MatchDecider to gather information about the matches, and decide // We use a MatchDecider to gather information about the matches, and decide
// whether to include them in the results. // whether to include them in the results.
@ -47,12 +48,12 @@ struct MatchDecider : public Xapian::MatchDecider {
{ {
QueryMatch qm{}; QueryMatch qm{};
auto msgid{opt_string(doc, MU_MSG_FIELD_ID_MSGID) auto msgid{opt_string(doc, Field::Id::MessageId)
.value_or(*opt_string(doc, MU_MSG_FIELD_ID_PATH))}; .value_or(*opt_string(doc, Field::Id::Path))};
if (!decider_info_.message_ids.emplace(std::move(msgid)).second) if (!decider_info_.message_ids.emplace(std::move(msgid)).second)
qm.flags |= QueryMatch::Flags::Duplicate; qm.flags |= QueryMatch::Flags::Duplicate;
const auto path{opt_string(doc, MU_MSG_FIELD_ID_PATH)}; const auto path{opt_string(doc, Field::Id::Path)};
if (!path || ::access(path->c_str(), R_OK) != 0) if (!path || ::access(path->c_str(), R_OK) != 0)
qm.flags |= QueryMatch::Flags::Unreadable; qm.flags |= QueryMatch::Flags::Unreadable;
@ -86,7 +87,7 @@ struct MatchDecider : public Xapian::MatchDecider {
*/ */
void gather_thread_ids(const Xapian::Document& doc) const void gather_thread_ids(const Xapian::Document& doc) const
{ {
auto thread_id{opt_string(doc, MU_MSG_FIELD_ID_THREAD_ID)}; auto thread_id{opt_string(doc, Field::Id::ThreadId)};
if (thread_id) if (thread_id)
decider_info_.thread_ids.emplace(std::move(*thread_id)); decider_info_.thread_ids.emplace(std::move(*thread_id));
} }
@ -95,10 +96,10 @@ struct MatchDecider : public Xapian::MatchDecider {
const QueryFlags qflags_; const QueryFlags qflags_;
DeciderInfo& decider_info_; DeciderInfo& decider_info_;
private: private:
Option<std::string> opt_string(const Xapian::Document& doc, MuMsgFieldId id) const noexcept Option<std::string> opt_string(const Xapian::Document& doc, Field::Id id) const noexcept {
{ const auto value_no{message_field(id).value_no()};
std::string val = xapian_try([&] { return doc.get_value(id); }, std::string{""}); std::string val = xapian_try([&] { return doc.get_value(value_no); }, std::string{""});
if (val.empty()) if (val.empty())
return Nothing; return Nothing;
else else

8
lib/mu-query-matches.hh
View File

@ -73,7 +73,7 @@ class QueryResultsIterator {
QueryResultsIterator(Xapian::MSetIterator it, QueryResultsIterator(Xapian::MSetIterator it,
size_t max_num, size_t max_num,
MuMsgFieldId sort_field, Message::Field::Id sort_field,
MuMsgIterFlags flags, MuMsgIterFlags flags,
MatchInfo& minfo) MatchInfo& minfo)
: it_{it}, match_info_{minfo} : it_{it}, match_info_{minfo}
@ -124,7 +124,7 @@ class QueryResultsIterator {
std::string message_id() const std::string message_id() const
{ {
g_return_val_if_fail(it_ != Xapian::MSetIterator::end(), ""); g_return_val_if_fail(it_ != Xapian::MSetIterator::end(), "");
return document().get_value(MU_MSG_FIELD_ID_MSGID); return document().get_value(Field::Id::MessageId);
} }
/** /**
@ -136,7 +136,7 @@ class QueryResultsIterator {
std::string path() const std::string path() const
{ {
g_return_val_if_fail(it_ != Xapian::MSetIterator::end(), ""); g_return_val_if_fail(it_ != Xapian::MSetIterator::end(), "");
return document().get_value(MU_MSG_FIELD_ID_PATH); return document().get_value(Field::Id::Path);
} }
/** /**
@ -149,7 +149,7 @@ class QueryResultsIterator {
std::vector<std::string> references() const std::vector<std::string> references() const
{ {
g_return_val_if_fail(it_ != Xapian::MSetIterator::end(), {}); g_return_val_if_fail(it_ != Xapian::MSetIterator::end(), {});
return split(document().get_value(MU_MSG_FIELD_ID_REFS), ","); return split(document().get_value(Field::Id::References), ",");
} }
private: private:

25
lib/mu-query-results.hh
View File

@ -59,7 +59,7 @@ enum struct QueryFlags {
Threading = 1 << 4, /**< calculate threading info */ Threading = 1 << 4, /**< calculate threading info */
// internal // internal
Leader = 1 << 5, /**< This is the leader query (for internal use Leader = 1 << 5, /**< This is the leader query (for internal use
* only)*/ * only)*/
}; };
MU_ENABLE_BITOPS(QueryFlags); MU_ENABLE_BITOPS(QueryFlags);
@ -219,7 +219,7 @@ public:
*/ */
Option<std::string> message_id() const noexcept Option<std::string> message_id() const noexcept
{ {
return opt_string(MU_MSG_FIELD_ID_MSGID); return opt_string(Message::Field::Id::MessageId);
} }
/** /**
@ -230,7 +230,7 @@ public:
*/ */
Option<std::string> thread_id() const noexcept Option<std::string> thread_id() const noexcept
{ {
return opt_string(MU_MSG_FIELD_ID_THREAD_ID); return opt_string(Message::Field::Id::ThreadId);
} }
/** /**
@ -239,7 +239,7 @@ public:
* *
* @return a filesystem path * @return a filesystem path
*/ */
Option<std::string> path() const noexcept { return opt_string(MU_MSG_FIELD_ID_PATH); } Option<std::string> path() const noexcept { return opt_string(Message::Field::Id::Path); }
/** /**
* Get the date for the document (message) the iterator is pointing at. * Get the date for the document (message) the iterator is pointing at.
@ -247,7 +247,7 @@ public:
* *
* @return a filesystem path * @return a filesystem path
*/ */
Option<std::string> date() const noexcept { return opt_string(MU_MSG_FIELD_ID_DATE); } Option<std::string> date() const noexcept { return opt_string(Message::Field::Id::Date); }
/** /**
* Get the file-system path for the document (message) this iterator is * Get the file-system path for the document (message) this iterator is
@ -255,7 +255,7 @@ public:
* *
* @return the subject * @return the subject
*/ */
Option<std::string> subject() const noexcept { return opt_string(MU_MSG_FIELD_ID_SUBJECT); } Option<std::string> subject() const noexcept { return opt_string(Message::Field::Id::Subject); }
/** /**
* Get the references for the document (messages) this is iterator is * Get the references for the document (messages) this is iterator is
@ -266,7 +266,7 @@ public:
*/ */
std::vector<std::string> references() const noexcept std::vector<std::string> references() const noexcept
{ {
return split(document().get_value(MU_MSG_FIELD_ID_REFS), ","); return split(opt_string(Message::Field::Id::References).value_or(""), ",");
} }
/** /**
@ -276,10 +276,11 @@ public:
* *
* @return the value * @return the value
*/ */
Option<std::string> opt_string(MuMsgFieldId id) const noexcept Option<std::string> opt_string(Message::Field::Id id) const noexcept
{ {
std::string empty; std::string empty;
std::string val = xapian_try([&] { return document().get_value(id); }, empty); const auto value_no{message_field(id).value_no()};
std::string val = xapian_try([&] {return document().get_value(value_no);}, empty);
if (val.empty()) if (val.empty())
return Nothing; return Nothing;
else else
@ -314,14 +315,14 @@ public:
return xapian_try( return xapian_try(
[&] { [&] {
auto docp{reinterpret_cast<XapianDocument*>( auto docp{reinterpret_cast<XapianDocument*>(
new Xapian::Document(document()))}; new Xapian::Document(document()))};
GError* err{}; GError* err{};
g_clear_pointer(&msg_, mu_msg_unref); g_clear_pointer(&msg_, mu_msg_unref);
if (!(msg_ = mu_msg_new_from_doc(docp, &err))) { if (!(msg_ = mu_msg_new_from_doc(docp, &err))) {
delete docp; delete docp;
g_warning("failed to crate message for %s: %s", g_warning("failed to crate message for %s: %s",
path().value_or("<none>").c_str(), path().value_or("<none>").c_str(),
err ? err->message : "somethng went wrong"); err ? err->message : "somethng went wrong");
g_clear_error(&err); g_clear_error(&err);
} }
return msg_; return msg_;

2
lib/mu-query-threads.cc
View File

@ -18,7 +18,7 @@
*/ */
#include "mu-query-threads.hh" #include "mu-query-threads.hh"
#include "mu-msg-fields.h" #include "mu-message-fields.hh"
#include <set> #include <set>
#include <unordered_set> #include <unordered_set>

168
lib/mu-query.cc
View File

@ -29,72 +29,64 @@
#include <xapian.h> #include <xapian.h>
#include <glib/gstdio.h> #include <glib/gstdio.h>
#include "mu-msg-fields.h"
#include "mu-query-results.hh" #include "mu-query-results.hh"
#include "mu-query-match-deciders.hh" #include "mu-query-match-deciders.hh"
#include "mu-query-threads.hh" #include "mu-query-threads.hh"
#include <mu-xapian.hh> #include <mu-xapian.hh>
using namespace Mu; using namespace Mu;
using namespace Mu::Message;
struct Query::Private { struct Query::Private {
Private(const Store& store) Private(const Store& store) : store_{store}, parser_{store_} {}
: store_{store}, parser_{store_} {}
// New // New
// bool calculate_threads (Xapian::Enquire& enq, size maxnum); // bool calculate_threads (Xapian::Enquire& enq, size maxnum);
Xapian::Enquire Xapian::Enquire make_enquire(const std::string& expr,
make_enquire(const std::string& expr, MuMsgFieldId sortfieldid, QueryFlags qflags) const; std::optional<Field::Id> sortfield_id,
Xapian::Enquire make_related_enquire(const StringSet& thread_ids, QueryFlags qflags) const;
MuMsgFieldId sortfieldid, Xapian::Enquire make_related_enquire(const StringSet& thread_ids,
QueryFlags qflags) const; std::optional<Field::Id> sortfield_id,
QueryFlags qflags) const;
Option<QueryResults> run_threaded(QueryResults&& qres, Option<QueryResults> run_threaded(QueryResults&& qres, Xapian::Enquire& enq,
Xapian::Enquire& enq, QueryFlags qflags, size_t max_size) const;
QueryFlags qflags, Option<QueryResults> run_singular(const std::string& expr,
size_t max_size) const; std::optional<Field::Id> sortfield_id,
Option<QueryResults> run_singular(const std::string& expr, QueryFlags qflags, size_t maxnum) const;
MuMsgFieldId sortfieldid, Option<QueryResults> run_related(const std::string& expr,
QueryFlags qflags, std::optional<Field::Id> sortfield_id,
size_t maxnum) const; QueryFlags qflags, size_t maxnum) const;
Option<QueryResults> run_related(const std::string& expr,
MuMsgFieldId sortfieldid,
QueryFlags qflags,
size_t maxnum) const;
Option<QueryResults> run(const std::string& expr,
MuMsgFieldId sortfieldid,
QueryFlags qflags,
size_t maxnum) const;
size_t store_size() const Option<QueryResults> run(const std::string& expr,
{ std::optional<Field::Id> sortfield_id, QueryFlags qflags,
return store_.database().get_doccount(); size_t maxnum) const;
}
size_t store_size() const { return store_.database().get_doccount(); }
const Store& store_; const Store& store_;
const Parser parser_; const Parser parser_;
}; };
Query::Query(const Store& store) Query::Query(const Store& store) : priv_{std::make_unique<Private>(store)} {}
: priv_{std::make_unique<Private>(store)} {}
Query::Query(Query&& other) = default; Query::Query(Query&& other) = default;
Query::~Query() = default; Query::~Query() = default;
static Xapian::Enquire& static Xapian::Enquire&
maybe_sort(Xapian::Enquire& enq, MuMsgFieldId sortfieldid, QueryFlags qflags) sort_enquire(Xapian::Enquire& enq, Field::Id sortfield_id, QueryFlags qflags)
{ {
if (sortfieldid != MU_MSG_FIELD_ID_NONE) const auto value_no{message_field(sortfield_id).value_no()};
enq.set_sort_by_value(static_cast<Xapian::valueno>(sortfieldid), enq.set_sort_by_value(value_no, any_of(qflags & QueryFlags::Descending));
any_of(qflags & QueryFlags::Descending));
return enq; return enq;
} }
Xapian::Enquire Xapian::Enquire
Query::Private::make_enquire(const std::string& expr, Query::Private::make_enquire(const std::string& expr,
MuMsgFieldId sortfieldid, std::optional<Field::Id> sortfield_id,
QueryFlags qflags) const QueryFlags qflags) const
{ {
Xapian::Enquire enq{store_.database()}; Xapian::Enquire enq{store_.database()};
@ -109,29 +101,33 @@ Query::Private::make_enquire(const std::string& expr,
g_debug("qtree: %s", to_string(tree).c_str()); g_debug("qtree: %s", to_string(tree).c_str());
} }
return maybe_sort(enq, sortfieldid, qflags); if (sortfield_id)
sort_enquire(enq, *sortfield_id, qflags);
return enq;
} }
Xapian::Enquire Xapian::Enquire
Query::Private::make_related_enquire(const StringSet& thread_ids, Query::Private::make_related_enquire(const StringSet& thread_ids,
MuMsgFieldId sortfieldid, std::optional<Field::Id> sortfield_id,
QueryFlags qflags) const QueryFlags qflags) const
{ {
Xapian::Enquire enq{store_.database()}; Xapian::Enquire enq{store_.database()};
static std::string pfx(1, mu_msg_field_xapian_prefix(MU_MSG_FIELD_ID_THREAD_ID));
std::vector<Xapian::Query> qvec; std::vector<Xapian::Query> qvec;
for (auto&& t : thread_ids) for (auto&& t : thread_ids)
qvec.emplace_back(pfx + t); qvec.emplace_back(message_field(Field::Id::ThreadId).xapian_term(t));
Xapian::Query qr{Xapian::Query::OP_OR, qvec.begin(), qvec.end()}; Xapian::Query qr{Xapian::Query::OP_OR, qvec.begin(), qvec.end()};
enq.set_query(qr); enq.set_query(qr);
return maybe_sort(enq, sortfieldid, qflags); if (sortfield_id)
sort_enquire(enq, *sortfield_id, qflags);
return enq;
} }
struct ThreadKeyMaker : public Xapian::KeyMaker { struct ThreadKeyMaker : public Xapian::KeyMaker {
ThreadKeyMaker(const QueryMatches& matches) ThreadKeyMaker(const QueryMatches& matches) : match_info_(matches) {}
: match_info_(matches) {}
std::string operator()(const Xapian::Document& doc) const override std::string operator()(const Xapian::Document& doc) const override
{ {
const auto it{match_info_.find(doc.get_docid())}; const auto it{match_info_.find(doc.get_docid())};
@ -141,10 +137,8 @@ struct ThreadKeyMaker : public Xapian::KeyMaker {
}; };
Option<QueryResults> Option<QueryResults>
Query::Private::run_threaded(QueryResults&& qres, Query::Private::run_threaded(QueryResults&& qres, Xapian::Enquire& enq, QueryFlags qflags,
Xapian::Enquire& enq, size_t maxnum) const
QueryFlags qflags,
size_t maxnum) const
{ {
const auto descending{any_of(qflags & QueryFlags::Descending)}; const auto descending{any_of(qflags & QueryFlags::Descending)};
@ -163,9 +157,8 @@ Query::Private::run_threaded(QueryResults&& qres,
Option<QueryResults> Option<QueryResults>
Query::Private::run_singular(const std::string& expr, Query::Private::run_singular(const std::string& expr,
MuMsgFieldId sortfieldid, std::optional<Field::Id> sortfield_id,
QueryFlags qflags, QueryFlags qflags, size_t maxnum) const
size_t maxnum) const
{ {
// i.e. a query _without_ related messages, but still possibly // i.e. a query _without_ related messages, but still possibly
// with threading. // with threading.
@ -179,10 +172,11 @@ Query::Private::run_singular(const std::string& expr,
DeciderInfo minfo{}; DeciderInfo minfo{};
#pragma GCC diagnostic push #pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wextra" #pragma GCC diagnostic ignored "-Wextra"
auto enq{make_enquire(expr, threading ? MU_MSG_FIELD_ID_DATE : sortfieldid, qflags)}; auto enq{make_enquire(expr, threading ? Field::Id::Date : sortfield_id, qflags)};
#pragma GCC diagnostic ignored "-Wswitch-default" #pragma GCC diagnostic ignored "-Wswitch-default"
#pragma GCC diagnostic pop #pragma GCC diagnostic pop
auto mset{enq.get_mset(0, maxnum, {}, make_leader_decider(singular_qflags, minfo).get())}; auto mset{enq.get_mset(0, maxnum, {},
make_leader_decider(singular_qflags, minfo).get())};
mset.fetch(); mset.fetch();
auto qres{QueryResults{mset, std::move(minfo.matches)}}; auto qres{QueryResults{mset, std::move(minfo.matches)}};
@ -191,9 +185,11 @@ Query::Private::run_singular(const std::string& expr,
} }
static Option<std::string> static Option<std::string>
opt_string(const Xapian::Document& doc, MuMsgFieldId id) noexcept opt_string(const Xapian::Document& doc, Field::Id id) noexcept
{ {
std::string val = xapian_try([&] { return doc.get_value(id); }, std::string{""}); const auto value_no{message_field(id).value_no()};
std::string val =
xapian_try([&] { return doc.get_value(value_no); }, std::string{""});
if (val.empty()) if (val.empty())
return Nothing; return Nothing;
else else
@ -202,9 +198,8 @@ opt_string(const Xapian::Document& doc, MuMsgFieldId id) noexcept
Option<QueryResults> Option<QueryResults>
Query::Private::run_related(const std::string& expr, Query::Private::run_related(const std::string& expr,
MuMsgFieldId sortfieldid, std::optional<Field::Id> sortfield_id,
QueryFlags qflags, QueryFlags qflags, size_t maxnum) const
size_t maxnum) const
{ {
// i.e. a query _with_ related messages and possibly with threading. // i.e. a query _with_ related messages and possibly with threading.
// //
@ -218,14 +213,14 @@ Query::Private::run_related(const std::string& expr,
// Run our first, "leader" query // Run our first, "leader" query
DeciderInfo minfo{}; DeciderInfo minfo{};
auto enq{make_enquire(expr, MU_MSG_FIELD_ID_DATE, leader_qflags)}; auto enq{make_enquire(expr, Field::Id::Date, leader_qflags)};
const auto mset{ const auto mset{
enq.get_mset(0, maxnum, {}, make_leader_decider(leader_qflags, minfo).get())}; enq.get_mset(0, maxnum, {}, make_leader_decider(leader_qflags, minfo).get())};
// Gather the thread-ids we found // Gather the thread-ids we found
mset.fetch(); mset.fetch();
for (auto it = mset.begin(); it != mset.end(); ++it) { for (auto it = mset.begin(); it != mset.end(); ++it) {
auto thread_id{opt_string(it.get_document(), MU_MSG_FIELD_ID_THREAD_ID)}; auto thread_id{opt_string(it.get_document(), Field::Id::ThreadId)};
if (thread_id) if (thread_id)
minfo.thread_ids.emplace(std::move(*thread_id)); minfo.thread_ids.emplace(std::move(*thread_id));
} }
@ -235,28 +230,28 @@ Query::Private::run_related(const std::string& expr,
// In the threaded-case, we search among _all_ messages, since complete // In the threaded-case, we search among _all_ messages, since complete
// threads are preferred; no need to sort in that case since the search // threads are preferred; no need to sort in that case since the search
// is unlimited and the sorting happens during threading. // is unlimited and the sorting happens during threading.
auto r_enq{make_related_enquire(minfo.thread_ids, auto r_enq = std::invoke([&]{
threading ? MU_MSG_FIELD_ID_NONE : sortfieldid, if (threading)
qflags)}; return make_related_enquire(minfo.thread_ids, std::nullopt, qflags);
const auto r_mset{r_enq.get_mset(0, else
threading ? store_size() : maxnum, return make_related_enquire(minfo.thread_ids, sortfield_id, qflags);
{}, });
make_related_decider(qflags, minfo).get())};
const auto r_mset{r_enq.get_mset(0, threading ? store_size() : maxnum, {},
make_related_decider(qflags, minfo).get())};
auto qres{QueryResults{r_mset, std::move(minfo.matches)}}; auto qres{QueryResults{r_mset, std::move(minfo.matches)}};
return threading ? run_threaded(std::move(qres), r_enq, qflags, maxnum) : qres; return threading ? run_threaded(std::move(qres), r_enq, qflags, maxnum) : qres;
} }
Option<QueryResults> Option<QueryResults>
Query::Private::run(const std::string& expr, Query::Private::run(const std::string& expr,
MuMsgFieldId sortfieldid, std::optional<Message::Field::Id> sortfield_id, QueryFlags qflags,
QueryFlags qflags, size_t maxnum) const
size_t maxnum) const
{ {
const auto eff_maxnum{maxnum == 0 ? store_size() : maxnum}; const auto eff_maxnum{maxnum == 0 ? store_size() : maxnum};
#pragma GCC diagnostic push #pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wextra" #pragma GCC diagnostic ignored "-Wextra"
const auto eff_sortfield{sortfieldid == MU_MSG_FIELD_ID_NONE ? MU_MSG_FIELD_ID_DATE const auto eff_sortfield{sortfield_id.value_or(Field::Id::Date)};
: sortfieldid};
#pragma GCC diagnostic pop #pragma GCC diagnostic pop
if (any_of(qflags & QueryFlags::IncludeRelated)) if (any_of(qflags & QueryFlags::IncludeRelated))
return run_related(expr, eff_sortfield, qflags, eff_maxnum); return run_related(expr, eff_sortfield, qflags, eff_maxnum);
@ -265,21 +260,18 @@ Query::Private::run(const std::string& expr,
} }
Option<QueryResults> Option<QueryResults>
Query::run(const std::string& expr, Query::run(const std::string& expr, std::optional<Message::Field::Id> sortfield_id,
MuMsgFieldId sortfieldid, QueryFlags qflags, size_t maxnum) const
QueryFlags qflags,
size_t maxnum) const
try { try {
// some flags are for internal use only. // some flags are for internal use only.
g_return_val_if_fail(none_of(qflags & QueryFlags::Leader), Nothing); g_return_val_if_fail(none_of(qflags & QueryFlags::Leader), Nothing);
StopWatch sw{format("ran query '%s'; related: %s; threads: %s; max-size: %zu", StopWatch sw{format(
expr.c_str(), "ran query '%s'; related: %s; threads: %s; max-size: %zu", expr.c_str(),
any_of(qflags & QueryFlags::IncludeRelated) ? "yes" : "no", any_of(qflags & QueryFlags::IncludeRelated) ? "yes" : "no",
any_of(qflags & QueryFlags::Threading) ? "yes" : "no", any_of(qflags & QueryFlags::Threading) ? "yes" : "no", maxnum)};
maxnum)};
return priv_->run(expr, sortfieldid, qflags, maxnum); return priv_->run(expr, sortfield_id, qflags, maxnum);
} catch (...) { } catch (...) {
return Nothing; return Nothing;
@ -290,7 +282,7 @@ Query::count(const std::string& expr) const
{ {
return xapian_try( return xapian_try(
[&] { [&] {
const auto enq{priv_->make_enquire(expr, MU_MSG_FIELD_ID_NONE, {})}; const auto enq{priv_->make_enquire(expr, {}, {})};
auto mset{enq.get_mset(0, priv_->store_size())}; auto mset{enq.get_mset(0, priv_->store_size())};
mset.fetch(); mset.fetch();
return mset.size(); return mset.size();

11
lib/mu-query.hh
View File

@ -21,11 +21,13 @@
#define __MU_QUERY_HH__ #define __MU_QUERY_HH__
#include <memory> #include <memory>
#include <optional>
#include <glib.h> #include <glib.h>
#include <mu-store.hh> #include <mu-store.hh>
#include <mu-query-results.hh> #include <mu-query-results.hh>
#include <utils/mu-utils.hh> #include <utils/mu-utils.hh>
#include <mu-message.hh>
namespace Mu { namespace Mu {
@ -41,10 +43,11 @@ public:
* *
* @return the query-results, or Nothing in case of error. * @return the query-results, or Nothing in case of error.
*/ */
Option<QueryResults> run(const std::string& expr = "",
MuMsgFieldId sortfieldid = MU_MSG_FIELD_ID_NONE, Option<QueryResults> run(const std::string& expr = "",
QueryFlags flags = QueryFlags::None, std::optional<Message::Field::Id> sortfield_id = {},
size_t maxnum = 0) const; QueryFlags flags = QueryFlags::None,
size_t maxnum = 0) const;
/** /**
* run a Xapian query to count the number of matches; for the syntax, please * run a Xapian query to count the number of matches; for the syntax, please