diff --git a/lib/Makefile.am b/lib/Makefile.am index 276bd257..5e31e1c7 100644 --- a/lib/Makefile.am +++ b/lib/Makefile.am @@ -18,7 +18,7 @@ # before descending into tests/ include $(top_srcdir)/gtest.mk -SUBDIRS= utils query index +SUBDIRS= utils index TESTDEFS= \ -DMU_TESTMAILDIR=\"${abs_srcdir}/testdir\" \ @@ -34,9 +34,7 @@ AM_CFLAGS= \ $(GMIME_CFLAGS) \ $(GLIB_CFLAGS) \ $(GUILE_CFLAGS) \ - $(JSON_GLIB_CFLAGS) \ $(ASAN_CFLAGS) \ - $(json_flag) \ $(CODE_COVERAGE_CFLAGS) \ $(TESTDEFS) \ -Wno-format-nonliteral \ @@ -48,8 +46,6 @@ AM_CXXFLAGS= \ $(GMIME_CFLAGS) \ $(GLIB_CFLAGS) \ $(GUILE_CFLAGS) \ - $(JSON_GLIB_CFLAGS) \ - $(json_flag) \ $(WARN_CXXFLAGS) \ $(XAPIAN_CXXFLAGS) \ $(ASAN_CXXFLAGS) \ @@ -59,13 +55,6 @@ AM_CXXFLAGS= \ AM_CPPFLAGS= \ $(CODE_COVERAGE_CPPFLAGS) -# don't use -Werror, as it might break on other compilers -# use -Wno-unused-parameters, because some callbacks may not -# really need all the params they get -# AM_CFLAGS=-Wall -Wextra -Wno-unused-parameter \ -# -Wdeclaration-after-statement -Wno-variadic-macros -# AM_CXXFLAGS=-Wall -Wextra -Wno-unused-parameter - noinst_LTLIBRARIES= \ libmu.la @@ -76,8 +65,9 @@ libmu_la_SOURCES= \ mu-contacts.hh \ mu-container.c \ mu-container.h \ - mu-flags.h \ + mu-data.hh \ mu-flags.c \ + mu-flags.h \ mu-maildir.c \ mu-maildir.h \ mu-msg-crypto.c \ @@ -89,7 +79,6 @@ libmu_la_SOURCES= \ mu-msg-file.h \ mu-msg-iter.cc \ mu-msg-iter.h \ - $(json_srcs) \ mu-msg-part.c \ mu-msg-part.h \ mu-msg-prio.c \ @@ -99,8 +88,10 @@ libmu_la_SOURCES= \ mu-msg.c \ mu-msg.h \ mu-msg.h \ + mu-parser.cc \ + mu-parser.hh \ mu-query.cc \ - mu-query.h \ + mu-query.hh \ mu-runtime.cc \ mu-runtime.h \ mu-script.c \ @@ -110,40 +101,54 @@ libmu_la_SOURCES= \ mu-store.cc \ mu-store.hh \ mu-threader.c \ - mu-threader.h + mu-threader.h \ + mu-tokenizer.cc \ + mu-tokenizer.hh \ + mu-tree.hh \ + mu-xapian.cc \ + mu-xapian.hh libmu_la_LIBADD= \ $(XAPIAN_LIBS) \ $(GMIME_LIBS) \ $(GLIB_LIBS) \ $(GUILE_LIBS) \ - $(JSON_GLIB_LIBS) \ - ${builddir}/query/libmu-query.la \ ${builddir}/index/libmu-index.la \ $(CODE_COVERAGE_LIBS) libmu_la_LDFLAGS= \ $(ASAN_LDFLAGS) +noinst_PROGRAMS= \ + tokenize + +tokenize_SOURCES= \ + tokenize.cc + +tokenize_LDADD= \ + $(WARN_LDFLAGS) \ + libmu.la \ + utils/libmu-utils.la + EXTRA_DIST= \ mu-msg-crypto.c \ doxyfile.in -noinst_PROGRAMS= $(TEST_PROGS) +noinst_PROGRAMS+=$(TEST_PROGS) noinst_LTLIBRARIES+= \ libtestmucommon.la TEST_PROGS += test-mu-maildir -test_mu_maildir_SOURCES= test-mu-maildir.c dummy.cc +test_mu_maildir_SOURCES= test-mu-maildir.cc test_mu_maildir_LDADD= libtestmucommon.la TEST_PROGS += test-mu-msg-fields -test_mu_msg_fields_SOURCES= test-mu-msg-fields.c dummy.cc +test_mu_msg_fields_SOURCES= test-mu-msg-fields.cc test_mu_msg_fields_LDADD= libtestmucommon.la TEST_PROGS += test-mu-msg -test_mu_msg_SOURCES= test-mu-msg.c dummy.cc +test_mu_msg_SOURCES= test-mu-msg.cc test_mu_msg_LDADD= libtestmucommon.la TEST_PROGS += test-mu-store @@ -151,29 +156,32 @@ test_mu_store_SOURCES= test-mu-store.cc test_mu_store_LDADD= libtestmucommon.la TEST_PROGS += test-mu-flags -test_mu_flags_SOURCES= test-mu-flags.c dummy.cc +test_mu_flags_SOURCES= test-mu-flags.cc test_mu_flags_LDADD= libtestmucommon.la TEST_PROGS += test-mu-container -test_mu_container_SOURCES= test-mu-container.c dummy.cc +test_mu_container_SOURCES= test-mu-container.cc test_mu_container_LDADD= libtestmucommon.la TEST_PROGS += test-mu-contacts test_mu_contacts_SOURCES= test-mu-contacts.cc test_mu_contacts_LDADD= libtestmucommon.la -# we need to use dummy.cc to enforce c++ linking... -BUILT_SOURCES= \ - dummy.cc +TEST_PROGS+=test-mu-tokenizer +test_mu_tokenizer_SOURCES=test-tokenizer.cc +test_mu_tokenizer_LDADD=libtestmucommon.la -dummy.cc: - touch dummy.cc +# TEST_PROGS+=test-mu-parser +# test_mu_parser_SOURCES=test-parser.cc +# test_mu_parser_LDADD=libtestmucommon.la libtestmucommon_la_SOURCES= \ - test-mu-common.c \ - test-mu-common.h + test-mu-common.cc \ + test-mu-common.hh + libtestmucommon_la_LIBADD= \ - libmu.la + libmu.la \ + utils/libmu-utils.la # note the question marks; make does not like files with ':', so we # use the (also supported) version with '!' instead. We could escape diff --git a/lib/index/mu-indexer.cc b/lib/index/mu-indexer.cc index 8875958f..fbb52741 100644 --- a/lib/index/mu-indexer.cc +++ b/lib/index/mu-indexer.cc @@ -205,7 +205,7 @@ Indexer::Private::cleanup() g_debug ("starting cleanup"); std::vector orphans_; // store messages without files. - store_.for_each([&](Store::Id id, const std::string &path) { + store_.for_each_message_path([&](Store::Id id, const std::string &path) { if (clean_done_) return false; diff --git a/lib/mu-container.h b/lib/mu-container.h index 69c19507..628e0ac4 100644 --- a/lib/mu-container.h +++ b/lib/mu-container.h @@ -23,6 +23,8 @@ #include #include +G_BEGIN_DECLS + enum _MuContainerFlag { MU_CONTAINER_FLAG_NONE = 0, MU_CONTAINER_FLAG_DELETE = 1 << 0, @@ -221,4 +223,6 @@ MuContainer* mu_container_sort (MuContainer *c, MuMsgFieldId mfid, GHashTable* mu_container_thread_info_hash_new (MuContainer *root_set, size_t matchnum); +G_END_DECLS + #endif /*__MU_CONTAINER_H__*/ diff --git a/lib/query/mu-data.hh b/lib/mu-data.hh similarity index 100% rename from lib/query/mu-data.hh rename to lib/mu-data.hh diff --git a/lib/mu-parser.cc b/lib/mu-parser.cc new file mode 100644 index 00000000..3d9622d7 --- /dev/null +++ b/lib/mu-parser.cc @@ -0,0 +1,527 @@ +/* +** Copyright (C) 2020 Dirk-Jan C. Binnema +** +** This library is free software; you can redistribute it and/or +** modify it under the terms of the GNU Lesser General Public License +** as published by the Free Software Foundation; either version 2.1 +** of the License, or (at your option) any later version. +** +** This library is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +** Lesser General Public License for more details. +** +** You should have received a copy of the GNU Lesser General Public +** License along with this library; if not, write to the Free +** Software Foundation, 51 Franklin Street, Fifth Floor, Boston, MA +** 02110-1301, USA. +*/ +#include "mu-parser.hh" +#include "mu-tokenizer.hh" +#include "utils/mu-utils.hh" +#include "utils/mu-error.hh" +#include + +using namespace Mu; + +// 3 precedence levels: units (NOT,()) > factors (OR) > terms (AND) + +// query -> | ε +// -> | ε +// -> OR|XOR | ε +// -> | ε +// -> [AND]|AND NOT | ε +// -> [NOT] | ( ) | +// -> | | +// -> [field:]value +// -> [field:][lower]..[upper] +// -> [field:]/regex/ + + +#define BUG(...) Mu::Error (Error::Code::Internal, format("%u: BUG: ",__LINE__) \ + + format(__VA_ARGS__)) + + + +/** + * Get the "shortcut"/internal fields for the the given fieldstr or empty if there is none + * + * @param fieldstr a fieldstr, e.g "subject" or "s" for the subject field + * + * @return a vector with "exploded" values, with a code and a fullname. E.g. "s" might map + * to [<"S","subject">], while "recip" could map to [<"to", "T">, <"cc", "C">, <"bcc", "B">] + */ +struct FieldInfo { + const std::string field; + const std::string prefix; + bool supports_phrase; + unsigned id; +}; +using FieldInfoVec = std::vector; + +struct Parser::Private { + Private(const Store& store): store_{store} {} + + std::vector process_regex (const std::string& field, + const std::regex& rx) const; + + Mu::Tree term_1 (Mu::Tokens& tokens, WarningVec& warnings) const; + Mu::Tree term_2 (Mu::Tokens& tokens, Node::Type& op, WarningVec& warnings) const; + Mu::Tree factor_1 (Mu::Tokens& tokens, WarningVec& warnings) const; + Mu::Tree factor_2 (Mu::Tokens& tokens, Node::Type& op, WarningVec& warnings) const; + Mu::Tree unit (Mu::Tokens& tokens, WarningVec& warnings) const; + Mu::Tree data (Mu::Tokens& tokens, WarningVec& warnings) const; + Mu::Tree range (const FieldInfoVec& fields, const std::string& lower, + const std::string& upper, size_t pos, WarningVec& warnings) const; + Mu::Tree regex (const FieldInfoVec& fields, const std::string& v, + size_t pos, WarningVec& warnings) const; + Mu::Tree value (const FieldInfoVec& fields, const std::string& v, + size_t pos, WarningVec& warnings) const; +private: + const Store& store_; +}; + +static MuMsgFieldId +field_id (const std::string& field) +{ + + if (field.empty()) + return MU_MSG_FIELD_ID_NONE; + + MuMsgFieldId id = mu_msg_field_id_from_name (field.c_str(), FALSE); + if (id != MU_MSG_FIELD_ID_NONE) + return id; + else if (field.length() == 1) + return mu_msg_field_id_from_shortcut (field[0], FALSE); + else + return MU_MSG_FIELD_ID_NONE; +} + +static std::string +process_value (const std::string& field, const std::string& value) +{ + const auto id = field_id (field); + if (id == MU_MSG_FIELD_ID_NONE) + return value; + switch(id) { + case MU_MSG_FIELD_ID_PRIO: { + if (!value.empty()) + return std::string(1, value[0]); + } break; + + case MU_MSG_FIELD_ID_FLAGS: { + const auto flag = mu_flag_char_from_name (value.c_str()); + if (flag) + return std::string(1, tolower(flag)); + } break; + + default: + break; + } + + return value; // XXX prio/flags, etc. alias +} + +static void +add_field (std::vector& fields, MuMsgFieldId id) +{ + const auto shortcut = mu_msg_field_shortcut(id); + if (!shortcut) + return; // can't be searched + + const auto name = mu_msg_field_name (id); + const auto pfx = mu_msg_field_xapian_prefix (id); + + if (!name || !pfx) + return; + + fields.push_back ({{name}, {pfx}, + (bool)mu_msg_field_xapian_index(id), + id}); +} + +static std::vector +process_field (const std::string& field) +{ + + std::vector fields; + + if (field == "contact" || field == "recip") { // multi fields + add_field (fields, MU_MSG_FIELD_ID_TO); + add_field (fields, MU_MSG_FIELD_ID_CC); + add_field (fields, MU_MSG_FIELD_ID_BCC); + if (field == "contact") + add_field (fields, MU_MSG_FIELD_ID_FROM); + } else if (field == "") { + add_field (fields, MU_MSG_FIELD_ID_TO); + add_field (fields, MU_MSG_FIELD_ID_CC); + add_field (fields, MU_MSG_FIELD_ID_BCC); + add_field (fields, MU_MSG_FIELD_ID_FROM); + add_field (fields, MU_MSG_FIELD_ID_SUBJECT); + add_field (fields, MU_MSG_FIELD_ID_BODY_TEXT); + } else { + const auto id = field_id (field.c_str()); + if (id != MU_MSG_FIELD_ID_NONE) + add_field (fields, id); + } + + return fields; +} + +static bool +is_range_field (const std::string& field) +{ + const auto id = field_id (field.c_str()); + if (id == MU_MSG_FIELD_ID_NONE) + return false; + else + return mu_msg_field_is_range_field (id); +} + +struct MyRange { + std::string lower; + std::string upper; +}; + +static MyRange +process_range (const std::string& field, const std::string& lower, + const std::string& upper) +{ + const auto id = field_id (field.c_str()); + if (id == MU_MSG_FIELD_ID_NONE) + return { lower, upper }; + + std::string l2 = lower; + std::string u2 = upper; + + if (id == MU_MSG_FIELD_ID_DATE) { + l2 = Mu::date_to_time_t_string (lower, true); + u2 = Mu::date_to_time_t_string (upper, false); + } else if (id == MU_MSG_FIELD_ID_SIZE) { + l2 = Mu::size_to_string (lower, true); + u2 = Mu::size_to_string (upper, false); + } + + return { l2, u2 }; +} + +std::vector +Parser::Private::process_regex (const std::string& field, const std::regex& rx) const +{ + const auto id = field_id (field.c_str()); + if (id == MU_MSG_FIELD_ID_NONE) + return {}; + + char pfx[] = { mu_msg_field_xapian_prefix(id), '\0' }; + + std::vector terms; + store_.for_each_term(pfx,[&](auto&& str){ + if (std::regex_search(str.c_str() + 1, rx)) // avoid copy + terms.emplace_back(str); + return true; + }); + + return terms; +} + +static Token +look_ahead (const Mu::Tokens& tokens) +{ + return tokens.front(); +} + +static Mu::Tree +empty() +{ + return {{Node::Type::Empty}}; +} + +Mu::Tree +Parser::Private::value (const FieldInfoVec& fields, const std::string& v, + size_t pos, WarningVec& warnings) const +{ + auto val = utf8_flatten(v); + + if (fields.empty()) + throw BUG("expected one or more fields"); + + if (fields.size() == 1) { + const auto item = fields.front(); + return Tree({Node::Type::Value, + std::make_unique( + item.field, item.prefix, item.id, + process_value(item.field, val), + item.supports_phrase)}); + } + + // a 'multi-field' such as "recip:" + Tree tree(Node{Node::Type::OpOr}); + for (const auto& item: fields) + tree.add_child (Tree({Node::Type::Value, + std::make_unique( + item.field, item.prefix, item.id, + process_value(item.field, val), + item.supports_phrase)})); + return tree; +} + +Mu::Tree +Parser::Private::regex (const FieldInfoVec& fields, const std::string& v, + size_t pos, WarningVec& warnings) const +{ + if (v.length() < 2) + throw BUG("expected regexp, got '%s'", v.c_str()); + + const auto rxstr = utf8_flatten(v.substr(1, v.length()-2)); + + try { + Tree tree(Node{Node::Type::OpOr}); + const auto rx = std::regex (rxstr); + for (const auto& field: fields) { + const auto terms = process_regex (field.field, rx); + for (const auto& term: terms) { + tree.add_child (Tree( + {Node::Type::Value, + std::make_unique(field.field, "", + field.id, term)})); + } + } + + if (tree.children.empty()) + return empty(); + else + return tree; + + } catch (...) { + // fallback + warnings.push_back ({pos, "invalid regexp"}); + return value (fields, v, pos, warnings); + } +} + + + +Mu::Tree +Parser::Private::range (const FieldInfoVec& fields, const std::string& lower, + const std::string& upper, size_t pos, WarningVec& warnings) const +{ + if (fields.empty()) + throw BUG("expected field"); + + const auto& field = fields.front(); + if (!is_range_field(field.field)) + return value (fields, lower + ".." + upper, pos, warnings); + + auto prange = process_range (field.field, lower, upper); + if (prange.lower > prange.upper) + prange = process_range (field.field, upper, lower); + + return Tree({Node::Type::Range, + std::make_unique(field.field, field.prefix, field.id, + prange.lower, prange.upper)}); +} + +Mu::Tree +Parser::Private::data (Mu::Tokens& tokens, WarningVec& warnings) const +{ + const auto token = look_ahead(tokens); + if (token.type != Token::Type::Data) + warnings.push_back ({token.pos, "expected: value"}); + + tokens.pop_front(); + + std::string field, val; + const auto col = token.str.find (":"); + if (col != 0 && col != std::string::npos && col != token.str.length()-1) { + field = token.str.substr(0, col); + val = token.str.substr(col + 1); + } else + val = token.str; + + auto fields = process_field (field); + if (fields.empty()) {// not valid field... + warnings.push_back ({token.pos, format ("invalid field '%s'", field.c_str())}); + fields = process_field (""); + // fallback, treat the whole of foo:bar as a value + return value (fields, field + ":" + val, token.pos, warnings); + } + + // does it look like a regexp? + if (val.length() >=2 ) + if (val[0] == '/' && val[val.length()-1] == '/') + return regex (fields, val, token.pos, warnings); + + // does it look like a range? + const auto dotdot = val.find(".."); + if (dotdot != std::string::npos) + return range(fields, val.substr(0, dotdot), val.substr(dotdot + 2), + token.pos, warnings); + else if (is_range_field(fields.front().field)) { + // range field without a range - treat as field:val..val + return range (fields, val, val, token.pos, warnings); + } + + // if nothing else, it's a value. + return value (fields, val, token.pos, warnings); +} + +Mu::Tree +Parser::Private::unit (Mu::Tokens& tokens, WarningVec& warnings) const +{ + if (tokens.empty()) { + warnings.push_back ({0, "expected: unit"}); + return empty(); + } + + const auto token = look_ahead (tokens); + + if (token.type == Token::Type::Not) { + tokens.pop_front(); + Tree tree{{Node::Type::OpNot}}; + tree.add_child(unit (tokens, warnings)); + return tree; + } + + if (token.type == Token::Type::Open) { + tokens.pop_front(); + auto tree = term_1 (tokens, warnings); + if (tokens.empty()) + warnings.push_back({token.pos, "expected: ')'"}); + else { + const auto token2 = look_ahead(tokens); + if (token2.type == Token::Type::Close) + tokens.pop_front(); + else { + warnings.push_back( + {token2.pos, + std::string("expected: ')' but got ") + + token2.str}); + } + + } + return tree; + } + + return data (tokens, warnings); +} + +Mu::Tree +Parser::Private::factor_2 (Mu::Tokens& tokens, Node::Type& op, + WarningVec& warnings) const +{ + if (tokens.empty()) + return empty(); + + const auto token = look_ahead(tokens); + +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wswitch-enum" + switch (token.type) { + case Token::Type::And: { + tokens.pop_front(); + op = Node::Type::OpAnd; + } break; + + case Token::Type::Open: + case Token::Type::Data: + case Token::Type::Not: + op = Node::Type::OpAnd; // implicit AND + break; + + default: + return empty(); + } +#pragma GCC diagnostic pop + + return factor_1 (tokens, warnings); +} + + +Mu::Tree +Parser::Private::factor_1 (Mu::Tokens& tokens, WarningVec& warnings) const +{ + Node::Type op { Node::Type::Invalid }; + + auto t = unit (tokens, warnings); + auto a2 = factor_2 (tokens, op, warnings); + + if (a2.empty()) + return t; + + Tree tree {{op}}; + tree.add_child(std::move(t)); + tree.add_child(std::move(a2)); + + return tree; +} + + +Mu::Tree +Parser::Private::term_2 (Mu::Tokens& tokens, Node::Type& op, WarningVec& warnings) const +{ + if (tokens.empty()) + return empty(); + + const auto token = look_ahead (tokens); + +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wswitch-enum" + switch (token.type) { + case Token::Type::Or: + op = Node::Type::OpOr; + break; + case Token::Type::Xor: + op = Node::Type::OpXor; + break; + default: + if (token.type != Token::Type::Close) + warnings.push_back({token.pos, "expected OR|XOR"}); + return empty(); + } +#pragma GCC diagnostic pop + + tokens.pop_front(); + + return term_1 (tokens, warnings); +} + +Mu::Tree +Parser::Private::term_1 (Mu::Tokens& tokens, WarningVec& warnings) const +{ + Node::Type op { Node::Type::Invalid }; + + auto t = factor_1 (tokens, warnings); + auto o2 = term_2 (tokens, op, warnings); + + if (o2.empty()) + return t; + else { + Tree tree {{op}}; + tree.add_child(std::move(t)); + tree.add_child(std::move(o2)); + return tree; + } +} + +Mu::Parser::Parser(const Store& store): + priv_{std::make_unique(store)} +{} + +Mu::Parser::~Parser() = default; + + +Mu::Tree +Mu::Parser::parse (const std::string& expr, WarningVec& warnings) const +{ + try { + auto tokens = tokenize (expr); + if (tokens.empty()) + return empty (); + else + return priv_->term_1 (tokens, warnings); + + } catch (const std::runtime_error& ex) { + std::cerr << ex.what() << std::endl; + return empty(); + } +} diff --git a/lib/query/mu-parser.hh b/lib/mu-parser.hh similarity index 72% rename from lib/query/mu-parser.hh rename to lib/mu-parser.hh index 0c2ffe9e..42d223ee 100644 --- a/lib/query/mu-parser.hh +++ b/lib/mu-parser.hh @@ -25,9 +25,9 @@ #include #include -#include -#include -#include +#include +#include +#include // A simple recursive-descent parser for queries. Follows the Xapian syntax, // but better handles non-alphanum; also implements regexp @@ -53,7 +53,7 @@ struct Warning { return pos == rhs.pos && msg == rhs.msg; } }; - +using WarningVec=std::vector; /** * operator<< @@ -70,19 +70,34 @@ operator<< (std::ostream& os, const Warning& w) return os; } -/** - * Parse a query string - * - * @param query a query string - * @param warnings vec to receive warnings - * @param proc a Processor object - * - * @return a parse-tree - */ -using WarningVec=std::vector; -using ProcPtr = const std::unique_ptr&; -Tree parse (const std::string& query, WarningVec& warnings, - ProcPtr proc = std::make_unique()); +class Parser { +public: + /** + * Construct a query parser object + * + * @param store a store object ptr, or none + */ + Parser(const Store& store); + /** + * DTOR + * + */ + ~Parser(); + + /** + * Parse a query string + * + * @param query a query string + * @param warnings vec to receive warnings + * + * @return a parse-tree + */ + + Tree parse (const std::string& query, WarningVec& warnings) const; +private: + struct Private; + std::unique_ptr priv_; +}; } // namespace Mu diff --git a/lib/mu-query.cc b/lib/mu-query.cc index 7719f8e7..55ce1538 100644 --- a/lib/mu-query.cc +++ b/lib/mu-query.cc @@ -1,5 +1,5 @@ /* -** Copyright (C) 2008-2017 Dirk-Jan C. Binnema +** Copyright (C) 2008-2020 Dirk-Jan C. Binnema ** ** This program is free software; you can redistribute it and/or modify ** it under the terms of the GNU General Public License as published by @@ -16,6 +16,7 @@ ** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. ** */ +#include #include #include @@ -27,7 +28,6 @@ #include #include -#include "mu-query.h" #include "mu-msg-fields.h" #include "mu-msg-iter.h" @@ -36,273 +36,94 @@ #include "utils/mu-date.h" #include -#include -#include +#include using namespace Mu; -struct MuProc: public Mu::ProcIface { +struct Query::Private { + Private(const Store& store): store_{store}, + parser_{store_} {} - MuProc (const Xapian::Database& db): db_{db} {} + Xapian::Query make_query (const std::string& expr, GError **err) const; + Xapian::Enquire make_enquire (const std::string& expr, MuMsgFieldId sortfieldid, + bool descending, GError **err) const; + GHashTable* find_thread_ids (MuMsgIter *iter, GHashTable **orig_set) const; - static MuMsgFieldId field_id (const std::string& field) { + Xapian::Query make_related_query (MuMsgIter *iter, GHashTable **orig_set) const; - if (field.empty()) - return MU_MSG_FIELD_ID_NONE; + void find_related_messages (MuMsgIter **iter, int maxnum, + MuMsgFieldId sortfieldid, Query::Flags flags, + Xapian::Query orig_query) const; - MuMsgFieldId id = mu_msg_field_id_from_name (field.c_str(), FALSE); - if (id != MU_MSG_FIELD_ID_NONE) - return id; - else if (field.length() == 1) - return mu_msg_field_id_from_shortcut (field[0], FALSE); - else - return MU_MSG_FIELD_ID_NONE; - } - - std::string - process_value (const std::string& field, - const std::string& value) const override { - const auto id = field_id (field); - if (id == MU_MSG_FIELD_ID_NONE) - return value; - switch(id) { - case MU_MSG_FIELD_ID_PRIO: { - if (!value.empty()) - return std::string(1, value[0]); - } break; - - case MU_MSG_FIELD_ID_FLAGS: { - const auto flag = mu_flag_char_from_name (value.c_str()); - if (flag) - return std::string(1, tolower(flag)); - } break; - - default: - break; - } - - return value; // XXX prio/flags, etc. alias - } - - void add_field (std::vector& fields, MuMsgFieldId id) const { - - const auto shortcut = mu_msg_field_shortcut(id); - if (!shortcut) - return; // can't be searched - - const auto name = mu_msg_field_name (id); - const auto pfx = mu_msg_field_xapian_prefix (id); - - if (!name || !pfx) - return; - - fields.push_back ({{name}, {pfx}, - (bool)mu_msg_field_xapian_index(id), - id}); - } - - std::vector - process_field (const std::string& field) const override { - - std::vector fields; - - if (field == "contact" || field == "recip") { // multi fields - add_field (fields, MU_MSG_FIELD_ID_TO); - add_field (fields, MU_MSG_FIELD_ID_CC); - add_field (fields, MU_MSG_FIELD_ID_BCC); - if (field == "contact") - add_field (fields, MU_MSG_FIELD_ID_FROM); - } else if (field == "") { - add_field (fields, MU_MSG_FIELD_ID_TO); - add_field (fields, MU_MSG_FIELD_ID_CC); - add_field (fields, MU_MSG_FIELD_ID_BCC); - add_field (fields, MU_MSG_FIELD_ID_FROM); - add_field (fields, MU_MSG_FIELD_ID_SUBJECT); - add_field (fields, MU_MSG_FIELD_ID_BODY_TEXT); - } else { - const auto id = field_id (field.c_str()); - if (id != MU_MSG_FIELD_ID_NONE) - add_field (fields, id); - } - - return fields; - } - - bool is_range_field (const std::string& field) const override { - const auto id = field_id (field.c_str()); - if (id == MU_MSG_FIELD_ID_NONE) - return false; - else - return mu_msg_field_is_range_field (id); - } - - Range process_range (const std::string& field, const std::string& lower, - const std::string& upper) const override { - - const auto id = field_id (field.c_str()); - if (id == MU_MSG_FIELD_ID_NONE) - return { lower, upper }; - - std::string l2 = lower; - std::string u2 = upper; - - if (id == MU_MSG_FIELD_ID_DATE) { - l2 = Mu::date_to_time_t_string (lower, true); - u2 = Mu::date_to_time_t_string (upper, false); - } else if (id == MU_MSG_FIELD_ID_SIZE) { - l2 = Mu::size_to_string (lower, true); - u2 = Mu::size_to_string (upper, false); - } - - return { l2, u2 }; - } - - std::vector - process_regex (const std::string& field, const std::regex& rx) const override { - - const auto id = field_id (field.c_str()); - if (id == MU_MSG_FIELD_ID_NONE) - return {}; - - char pfx[] = { mu_msg_field_xapian_prefix(id), '\0' }; - - std::vector terms; - for (auto it = db_.allterms_begin(pfx); it != db_.allterms_end(pfx); ++it) { - if (std::regex_search((*it).c_str() + 1, rx)) // avoid copy - terms.push_back(*it); - } - - return terms; - } - - const Xapian::Database& db_; + const Store& store_; + const Parser parser_; }; -struct _MuQuery { -public: - _MuQuery (MuStore *store): _store(mu_store_ref(store)) {} - ~_MuQuery () { mu_store_unref (_store); } - Xapian::Database& db() const { - const auto db = reinterpret_cast - (mu_store_get_read_only_database (_store)); - if (!db) - throw Mu::Error(Error::Code::NotFound, "no database"); - return *db; - } -private: - MuStore *_store; -}; - -static const Xapian::Query -get_query (MuQuery *mqx, const char* searchexpr, bool raw, GError **err) try { - - Mu::WarningVec warns; - const auto tree = Mu::parse (searchexpr, warns, - std::make_unique(mqx->db())); - for (auto&& w: warns) - std::cerr << w << std::endl; - - return Mu::xapian_query (tree); - -} catch (...) { - mu_util_g_set_error (err,MU_ERROR_XAPIAN_QUERY, - "parse error in query"); - throw; -} - -MuQuery* -mu_query_new (MuStore *store, GError **err) +static constexpr MuMsgIterFlags +msg_iter_flags (Query::Flags flags) { - g_return_val_if_fail (store, NULL); + MuMsgIterFlags iflags{MU_MSG_ITER_FLAG_NONE}; - try { - return new MuQuery (store); - } MU_XAPIAN_CATCH_BLOCK_G_ERROR_RETURN (err, MU_ERROR_XAPIAN, 0); - - return 0; -} - -void -mu_query_destroy (MuQuery *self) -{ - try { delete self; } MU_XAPIAN_CATCH_BLOCK; -} - - -/* this function is for handling the case where a DatabaseModified - * exception is raised. We try to reopen the database, and run the - * query again. */ -static MuMsgIter * -try_requery (MuQuery *self, const char* searchexpr, MuMsgFieldId sortfieldid, - int maxnum, MuQueryFlags flags, GError **err) -{ - try { - /* let's assume that infinite regression is - * impossible */ - self->db().reopen(); - g_message ("reopening db after modification"); - return mu_query_run (self, searchexpr, sortfieldid, - maxnum, flags, err); - - } MU_XAPIAN_CATCH_BLOCK_G_ERROR_RETURN (err, MU_ERROR_XAPIAN, 0); -} - - -static MuMsgIterFlags -msg_iter_flags (MuQueryFlags flags) -{ - MuMsgIterFlags iflags; - - iflags = MU_MSG_ITER_FLAG_NONE; - - if (flags & MU_QUERY_FLAG_DESCENDING) + if (any_of(flags & Query::Flags::Descending)) iflags |= MU_MSG_ITER_FLAG_DESCENDING; - if (flags & MU_QUERY_FLAG_SKIP_UNREADABLE) + if (any_of(flags & Query::Flags::SkipUnreadable)) iflags |= MU_MSG_ITER_FLAG_SKIP_UNREADABLE; - if (flags & MU_QUERY_FLAG_SKIP_DUPS) + if (any_of(flags & Query::Flags::SkipDups)) iflags |= MU_MSG_ITER_FLAG_SKIP_DUPS; - if (flags & MU_QUERY_FLAG_THREADS) + if (any_of(flags & Query::Flags::Threading)) iflags |= MU_MSG_ITER_FLAG_THREADS; return iflags; } +Xapian::Query +Query::Private::make_query (const std::string& expr, GError **err) const try { + + Mu::WarningVec warns; + const auto tree{parser_.parse(expr, warns)}; + for (auto&& w: warns) + g_warning ("query warning: %s", to_string(w).c_str()); + + return Mu::xapian_query (tree); + +} catch (...) { + mu_util_g_set_error (err, MU_ERROR_XAPIAN_QUERY, + "parse error in query"); + throw; +} -static Xapian::Enquire -get_enquire (MuQuery *self, const char *searchexpr, MuMsgFieldId sortfieldid, - bool descending, bool raw, GError **err) +Xapian::Enquire +Query::Private::make_enquire (const std::string& expr, MuMsgFieldId sortfieldid, + bool descending, GError **err) const { - Xapian::Enquire enq (self->db()); + Xapian::Enquire enq{store_.database()}; - try { - if (raw) - enq.set_query(Xapian::Query(Xapian::Query(searchexpr))); - else if (!mu_str_is_empty(searchexpr) && - g_strcmp0 (searchexpr, "\"\"") != 0) /* NULL or "" or """" */ - enq.set_query(get_query (self, searchexpr, raw, err)); + try { + if (!expr.empty() && expr != R"("")") + enq.set_query(make_query (expr, err)); else/* empty or "" means "matchall" */ enq.set_query(Xapian::Query::MatchAll); } catch (...) { - mu_util_g_set_error (err, MU_ERROR_XAPIAN_QUERY, - "parse error in query"); + mu_util_g_set_error (err, MU_ERROR_XAPIAN_QUERY, "parse error in query"); throw; } enq.set_cutoff(0,0); - return enq; + + return enq; } /* - * record all threadids for the messages; also 'orig_set' receives all + * record all thread-ids for the messages; also 'orig_set' receives all * original matches (a map msgid-->docid), so we can make sure the * originals are not seen as 'duplicates' later (when skipping * duplicates). We want to favor the originals over the related * messages, when skipping duplicates. */ -static GHashTable* -get_thread_ids (MuMsgIter *iter, GHashTable **orig_set) +GHashTable* +Query::Private::find_thread_ids (MuMsgIter *iter, GHashTable **orig_set) const { GHashTable *ids; @@ -332,8 +153,8 @@ get_thread_ids (MuMsgIter *iter, GHashTable **orig_set) } -static Xapian::Query -get_related_query (MuMsgIter *iter, GHashTable **orig_set) +Xapian::Query +Query::Private::make_related_query (MuMsgIter *iter, GHashTable **orig_set) const { GHashTable *hash; GList *id_list, *cur; @@ -343,7 +164,7 @@ get_related_query (MuMsgIter *iter, GHashTable **orig_set) /* orig_set receives the hash msgid->docid of the set of * original matches */ - hash = get_thread_ids (iter, orig_set); + hash = find_thread_ids (iter, orig_set); /* id_list now gets a list of all thread-ids seen in the query * results; either in the Message-Id field or in * References. */ @@ -363,18 +184,18 @@ get_related_query (MuMsgIter *iter, GHashTable **orig_set) } -static void -get_related_messages (MuQuery *self, MuMsgIter **iter, int maxnum, - MuMsgFieldId sortfieldid, MuQueryFlags flags, - Xapian::Query orig_query) +void +Query::Private::find_related_messages (MuMsgIter **iter, int maxnum, + MuMsgFieldId sortfieldid, Query::Flags flags, + Xapian::Query orig_query) const { GHashTable *orig_set; - Xapian::Enquire enq (self->db()); + Xapian::Enquire enq{store_.database()}; MuMsgIter *rel_iter; - const bool inc_related = flags & MU_QUERY_FLAG_INCLUDE_RELATED; + const bool inc_related{any_of(flags & Query::Flags::IncludeRelated)}; orig_set = NULL; - Xapian::Query new_query = get_related_query (*iter, &orig_set); + Xapian::Query new_query{make_related_query (*iter, &orig_set)}; /* If related message are not desired, filter out messages which would not have matched the original query. */ @@ -402,25 +223,28 @@ get_related_messages (MuQuery *self, MuMsgIter **iter, int maxnum, *iter = rel_iter; } +Query::Query(const Store& store): + priv_{std::make_unique(store)} +{} + +Query::Query(Query&& other) = default; + +Query::~Query() = default; + MuMsgIter* -mu_query_run (MuQuery *self, const char *searchexpr, MuMsgFieldId sortfieldid, - int maxnum, MuQueryFlags flags, GError **err) +Query::run (const std::string& expr, MuMsgFieldId sortfieldid, Query::Flags flags, + size_t maxnum, GError **err) const { - g_return_val_if_fail (self, NULL); - g_return_val_if_fail (searchexpr, NULL); g_return_val_if_fail (mu_msg_field_id_is_valid (sortfieldid) || sortfieldid == MU_MSG_FIELD_ID_NONE, NULL); try { MuMsgIter *iter; - MuQueryFlags first_flags; - const bool threads = flags & MU_QUERY_FLAG_THREADS; - const bool inc_related = flags & MU_QUERY_FLAG_INCLUDE_RELATED; - const bool descending = flags & MU_QUERY_FLAG_DESCENDING; - const bool raw = flags & MU_QUERY_FLAG_RAW; - Xapian::Enquire enq (get_enquire(self, searchexpr, sortfieldid, - descending, raw, err)); + const bool threads = any_of(flags & Flags::Threading); + const bool inc_related = any_of(flags & Flags::IncludeRelated); + const bool descending = any_of(flags & Flags::Descending); + Xapian::Enquire enq (priv_->make_enquire(expr, sortfieldid, descending, err)); /* when we're doing a 'include-related query', wea're actually * doing /two/ queries; one to get the initial matches, and @@ -429,12 +253,13 @@ mu_query_run (MuQuery *self, const char *searchexpr, MuMsgFieldId sortfieldid, */ /* get the 'real' maxnum if it was specified as < 0 */ - maxnum = maxnum < 0 ? self->db().get_doccount() : maxnum; + maxnum = maxnum == 0 ? priv_->store_.size(): maxnum; /* Calculating threads involves two queries, so do the calculation only in * the second query instead of in both. */ + Query::Flags first_flags{}; if (threads) - first_flags = (MuQueryFlags)(flags & ~MU_QUERY_FLAG_THREADS); + first_flags = flags & ~Flags::Threading; else first_flags = flags; /* Perform the initial query, returning up to max num results. @@ -454,68 +279,47 @@ mu_query_run (MuQuery *self, const char *searchexpr, MuMsgFieldId sortfieldid, * the undesired related messages later. */ if(threads||inc_related) - get_related_messages (self, &iter, maxnum, sortfieldid, flags, - enq.get_query()); + priv_->find_related_messages (&iter, maxnum, sortfieldid, flags, + enq.get_query()); - if (err && *err && (*err)->code == MU_ERROR_XAPIAN_MODIFIED) { - g_clear_error (err); - return try_requery (self, searchexpr, sortfieldid, - maxnum, flags, err); - } else - return iter; + return iter; } MU_XAPIAN_CATCH_BLOCK_G_ERROR_RETURN (err, MU_ERROR_XAPIAN, 0); } size_t -mu_query_count_run (MuQuery *self, const char *searchexpr) try +Query::count (const std::string& expr) const try { - g_return_val_if_fail (self, 0); - g_return_val_if_fail (searchexpr, 0); - - const auto enq{get_enquire(self, searchexpr,MU_MSG_FIELD_ID_NONE, false, false, NULL)}; - auto mset(enq.get_mset(0, self->db().get_doccount())); + const auto enq{priv_->make_enquire(expr, MU_MSG_FIELD_ID_NONE, false, nullptr)}; + auto mset{enq.get_mset(0, priv_->store_.size())}; mset.fetch(); return mset.size(); -} MU_XAPIAN_CATCH_BLOCK_RETURN (0); +}MU_XAPIAN_CATCH_BLOCK_RETURN (0); -char* -mu_query_internal_xapian (MuQuery *self, const char *searchexpr, GError **err) + + +std::string +Query::parse(const std::string& expr, bool xapian) const try { - g_return_val_if_fail (self, NULL); - g_return_val_if_fail (searchexpr, NULL); - - try { - Xapian::Query query (get_query(self, searchexpr, false, err)); - return g_strdup(query.get_description().c_str()); - - } MU_XAPIAN_CATCH_BLOCK_RETURN(NULL); -} - - -char* -mu_query_internal (MuQuery *self, const char *searchexpr, - gboolean warn, GError **err) -{ - g_return_val_if_fail (self, NULL); - g_return_val_if_fail (searchexpr, NULL); - - try { + if (xapian) { + GError *err{}; + const auto descr{priv_->make_query(expr, &err).get_description()}; + if (err) { + g_warning ("query error: %s", err->message); + g_clear_error(&err); + } + return descr; + } else { Mu::WarningVec warns; - const auto tree = Mu::parse (searchexpr, warns, - std::make_unique(self->db())); - std::stringstream ss; - ss << tree; + const auto tree = priv_->parser_.parse (expr, warns); + for (auto&& w: warns) + g_warning ("query error: %s", to_string(w).c_str()); - if (warn) { - for (auto&& w: warns) - std::cerr << w << std::endl; - } + return to_string(tree); - return g_strdup(ss.str().c_str()); + } - } MU_XAPIAN_CATCH_BLOCK_RETURN(NULL); -} +} MU_XAPIAN_CATCH_BLOCK_RETURN(""); diff --git a/lib/mu-query.h b/lib/mu-query.h deleted file mode 100644 index 26cdb0f6..00000000 --- a/lib/mu-query.h +++ /dev/null @@ -1,136 +0,0 @@ -/* -** Copyright (C) 2008-2017 Dirk-Jan C. Binnema -** -** This program is free software; you can redistribute it and/or modify -** it under the terms of the GNU General Public License as published by -** the Free Software Foundation; either version 3 of the License, or -** (at your option) any later version. -** -** This program is distributed in the hope that it will be useful, -** but WITHOUT ANY WARRANTY; without even the implied warranty of -** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -** GNU General Public License for more details. -** -** You should have received a copy of the GNU General Public License -** along with this program; if not, write to the Free Software Foundation, -** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -** -*/ - -#ifndef __MU_QUERY_H__ -#define __MU_QUERY_H__ - -#include -#include -#include -#include - -G_BEGIN_DECLS - -struct _MuQuery; -typedef struct _MuQuery MuQuery; - -/** - * create a new MuQuery instance. - * - * @param store a MuStore object - * @param err receives error information (if there is any); if - * function returns non-NULL, err will _not_be set. err can be NULL - * possible errors (err->code) are MU_ERROR_XAPIAN_DIR and - * MU_ERROR_XAPIAN_NOT_UPTODATE - * - * @return a new MuQuery instance, or NULL in case of error. - * when the instance is no longer needed, use mu_query_destroy - * to free it - */ -MuQuery* mu_query_new (MuStore *store, GError **err) - G_GNUC_MALLOC G_GNUC_WARN_UNUSED_RESULT; - -/** - * destroy the MuQuery instance - * - * @param self a MuQuery instance, or NULL - */ -void mu_query_destroy (MuQuery *self); - - -typedef enum { - MU_QUERY_FLAG_NONE = 0 << 0, /**< no flags */ - MU_QUERY_FLAG_DESCENDING = 1 << 0, /**< sort z->a */ - MU_QUERY_FLAG_SKIP_UNREADABLE = 1 << 1, /**< skip unreadable msgs */ - MU_QUERY_FLAG_SKIP_DUPS = 1 << 2, /**< skip duplicate msgs */ - MU_QUERY_FLAG_INCLUDE_RELATED = 1 << 3, /**< include related msgs */ - MU_QUERY_FLAG_THREADS = 1 << 4, /**< calculate threading info */ - MU_QUERY_FLAG_RAW = 1 << 5 /**< don't parse the query */ -} MuQueryFlags; - -/** - * run a Xapian query; for the syntax, please refer to the mu-query - * manpage - * - * @param self a valid MuQuery instance - * @param expr the search expression; use "" to match all messages - * @param sortfield the field id to sort by or MU_MSG_FIELD_ID_NONE if - * sorting is not desired - * @param maxnum maximum number of search results to return, or <= 0 for - * unlimited - * @param flags bitwise OR'd flags to influence the query (see MuQueryFlags) - * @param err receives error information (if there is any); if - * function returns non-NULL, err will _not_be set. err can be NULL - * possible error (err->code) is MU_ERROR_QUERY, - * - * @return a MuMsgIter instance you can iterate over, or NULL in - * case of error - */ -MuMsgIter* mu_query_run (MuQuery *self, const char* expr, - MuMsgFieldId sortfieldid, int maxnum, - MuQueryFlags flags, GError **err) - G_GNUC_MALLOC G_GNUC_WARN_UNUSED_RESULT; - - -/** - * run a Xapian query to count the number of matches; for the syntax, please - * refer to the mu-query manpage - * - * @param self a valid MuQuery instance - * @param expr the search expression; use "" to match all messages - * - * @return the number of matches - */ -size_t mu_query_count_run (MuQuery *self, const char *searchexpr); - -/** - * get Xapian's internal string representation of the query - * - * @param self a MuQuery instance - * @param searchexpr a xapian search expression - * @param warn print warnings to stderr - * @param err receives error information (if there is any); if - * function returns non-NULL, err will _not_be set. err can be NULL - * - * @return the string representation of the xapian query, or NULL in case of - * error; free the returned value with g_free - */ -char* mu_query_internal (MuQuery *self, const char *searchexpr, - gboolean warn, GError **err) - G_GNUC_MALLOC G_GNUC_WARN_UNUSED_RESULT; - -/** - * get Xapian's internal string representation of the query - * - * @param self a MuQuery instance - * @param searchexpr a xapian search expression - * @param err receives error information (if there is any); if - * function returns non-NULL, err will _not_be set. err can be NULL - * - * @return the string representation of the xapian query, or NULL in case of - * error; free the returned value with g_free - */ -char* mu_query_internal_xapian (MuQuery *self, const char* searchexpr, - GError **err) - G_GNUC_MALLOC G_GNUC_WARN_UNUSED_RESULT; - - -G_END_DECLS - -#endif /*__MU_QUERY_H__*/ diff --git a/lib/mu-query.hh b/lib/mu-query.hh new file mode 100644 index 00000000..8c25cdf4 --- /dev/null +++ b/lib/mu-query.hh @@ -0,0 +1,120 @@ +/* +** Copyright (C) 2008-2020 Dirk-Jan C. Binnema +** +** This program is free software; you can redistribute it and/or modify +** it under the terms of the GNU General Public License as published by +** the Free Software Foundation; either version 3 of the License, or +** (at your option) any later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program; if not, write to the Free Software Foundation, +** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +** +*/ + +#ifndef __MU_QUERY_HH__ +#define __MU_QUERY_HH__ + +#include + +#include +#include +#include +#include + +namespace Mu { + +class Query { +public: + /** + * Construct a new Query instance. + * + * @param store a MuStore object + */ + Query (const Store& store); + /** + * DTOR + * + */ + ~Query (); + + + /** + * Move CTOR + * + * @param other + */ + Query(Query&& other); + + + enum struct Flags { + None = 0, /**< no flags */ + Descending = 1 << 0, /**< sort z->a */ + SkipUnreadable = 1 << 1, /**< skip unreadable msgs */ + SkipDups = 1 << 2, /**< skip duplicate msgs */ + IncludeRelated = 1 << 3, /**< include related msgs */ + Threading = 1 << 4, /**< calculate threading info */ + }; + + + /** + * run a query; for the syntax, please refer to the mu-query manpage + * + * @param expr the search expression; use "" to match all messages + * @param sortfield the field id to sort by or MU_MSG_FIELD_ID_NONE if + * sorting is not desired + * @param flags bitwise OR'd flags to influence the query (see MuQueryFlags) + * @param maxnum maximum number of search results to return, or 0 for + * unlimited + * @param err receives error information (if there is any); if + * function returns non-NULL, err will _not_be set. err can be NULL + * possible error (err->code) is MU_ERROR_QUERY, + * + * @return a MuMsgIter instance you can iterate over, or NULL in + * case of error + */ + MuMsgIter* run (const std::string& expr="", + MuMsgFieldId sortfieldid=MU_MSG_FIELD_ID_NONE, + Flags flags=Flags::None, + size_t maxnum=0, + GError **err=nullptr) const + G_GNUC_MALLOC G_GNUC_WARN_UNUSED_RESULT; + + + /** + * run a Xapian query to count the number of matches; for the syntax, please + * refer to the mu-query manpage + * + * @param expr the search expression; use "" to match all messages + * + * @return the number of matches + */ + size_t count (const std::string& expr="") const; + + /** + * For debugging, get the internal string representation of the parsed + * query + * + * @param expr a xapian search expression + * @param xapian if true, show Xapian's internal representation, + * otherwise, mu's. + + * @return the string representation of the query + */ + std::string parse (const std::string& expr, bool xapian) const; + +private: + struct Private; + std::unique_ptr priv_; + +}; +MU_ENABLE_BITOPS(Query::Flags); + +} + +#endif /*__MU_QUERY_HH__*/ diff --git a/lib/mu-server.cc b/lib/mu-server.cc index a887dbbe..604ea5d2 100644 --- a/lib/mu-server.cc +++ b/lib/mu-server.cc @@ -19,6 +19,7 @@ #include "config.h" +#include "mu-msg-fields.h" #include "mu-server.hh" #include @@ -36,7 +37,7 @@ #include "mu-msg.h" #include "mu-runtime.h" #include "mu-maildir.h" -#include "mu-query.h" +#include "mu-query.hh" #include "index/mu-indexer.hh" #include "mu-store.hh" #include "mu-msg-part.h" @@ -59,21 +60,12 @@ struct Server::Private { store_{store}, output_{output}, command_map_{make_command_map()}, - query_{make_query(store_)}, - keep_going_{true} { - if (!query_) - throw Error(Error::Code::Query, "failed to create server"); - } - - ~Private() { - g_clear_pointer(&query_, mu_query_destroy); - } - + query_{store_}, + keep_going_{true} {} // // construction helpers // CommandMap make_command_map(); - MuQuery* make_query(Store& store) const; // // acccessors @@ -81,7 +73,7 @@ struct Server::Private { const Store& store() const { return store_; } Indexer& indexer() { return store().indexer(); } const CommandMap& command_map() const { return command_map_; } - MuQuery* query() { return query_; } + const Query& query() const { return query_; } // // invoke @@ -122,24 +114,11 @@ private: Store& store_; Server::Output output_; const CommandMap command_map_; - MuQuery *query_{}; + const Query query_; std::atomic keep_going_{}; }; -MuQuery* -Server::Private::make_query (Store& store) const -{ - GError *gerr{}; - auto q{mu_query_new (reinterpret_cast(&store), &gerr)}; - if (!q) { - g_critical("failed to create query: %s", - gerr ? gerr->message : "something went wrong"); - g_clear_error(&gerr); - } - - return q; -} CommandMap Server::Private::make_command_map () @@ -624,22 +603,22 @@ Server::Private::extract_handler (const Parameters& params) /* get a *list* of all messages with the given message id */ static std::vector -docids_for_msgid (MuQuery *query, const std::string& msgid, size_t max=100) +docids_for_msgid (const Query& q, const std::string& msgid, size_t max=100) { if (msgid.size() > MU_STORE_MAX_TERM_LENGTH - 1) { throw Error(Error::Code::InvalidArgument, "invalid message-id '%s'", msgid.c_str()); } - const auto xprefix{mu_msg_field_xapian_prefix(MU_MSG_FIELD_ID_MSGID)}; + const auto xprefix{mu_msg_field_shortcut(MU_MSG_FIELD_ID_MSGID)}; /*XXX this is a bit dodgy */ auto tmp{g_ascii_strdown(msgid.c_str(), -1)}; - auto rawq{g_strdup_printf("%c%s", xprefix, tmp)}; + auto expr{g_strdup_printf("%c:%s", xprefix, tmp)}; g_free(tmp); GError *gerr{}; - auto iter{mu_query_run (query, rawq, MU_MSG_FIELD_ID_NONE, max, MU_QUERY_FLAG_RAW, &gerr)}; - g_free (rawq); + auto iter{q.run(expr , MU_MSG_FIELD_ID_NONE, Query::Flags::None, max)}; + g_free (expr); if (!iter) throw Error(Error::Code::Store, &gerr, "failed to run msgid-query"); if (mu_msg_iter_is_done (iter)) @@ -680,7 +659,7 @@ path_from_docid (const Store& store, unsigned docid) static std::vector -determine_docids (MuQuery *query, const Parameters& params) +determine_docids (const Query& q, const Parameters& params) { auto docid{get_int_or(params, ":docid", 0)}; const auto msgid{get_string_or(params, ":msgid")}; @@ -692,7 +671,7 @@ determine_docids (MuQuery *query, const Parameters& params) if (docid != 0) return { (unsigned)docid }; else - return docids_for_msgid (query, msgid.c_str()); + return docids_for_msgid (q, msgid.c_str()); } @@ -739,19 +718,18 @@ Server::Private::find_handler (const Parameters& params) sortfieldstr.c_str()}; } - int qflags{MU_QUERY_FLAG_NONE/*UNREADABLE*/}; + auto qflags{Query::Flags::None}; if (descending) - qflags |= MU_QUERY_FLAG_DESCENDING; + qflags |= Query::Flags::Descending; if (skip_dups) - qflags |= MU_QUERY_FLAG_SKIP_DUPS; + qflags |= Query::Flags::SkipDups; if (include_related) - qflags |= MU_QUERY_FLAG_INCLUDE_RELATED; + qflags |= Query::Flags::IncludeRelated; if (threads) - qflags |= MU_QUERY_FLAG_THREADS; + qflags |= Query::Flags::Threading; GError *gerr{}; - auto miter{mu_query_run(query(), q.c_str(), sort_field, maxnum, - (MuQueryFlags)qflags, &gerr)}; + auto miter{query().run(q, sort_field, qflags, maxnum, &gerr)}; if (!miter) throw Error(Error::Code::Query, &gerr, "failed to run query"); @@ -1023,9 +1001,9 @@ Server::Private::ping_handler (const Parameters& params) Sexp::List qresults; for (auto&& q: queries) { - const auto count{mu_query_count_run (query(), q.c_str())}; + const auto count{query().count(q)}; const auto unreadq{format("flag:unread AND (%s)", q.c_str())}; - const auto unread{mu_query_count_run (query(), unreadq.c_str())}; + const auto unread{query().count(unreadq)}; Sexp::List lst; lst.add_prop(":query", Sexp::make_string(q)); diff --git a/lib/mu-store.cc b/lib/mu-store.cc index 90e9e511..ac4e3497 100644 --- a/lib/mu-store.cc +++ b/lib/mu-store.cc @@ -20,6 +20,7 @@ #include "config.h" #include +#include #include #include #include @@ -112,43 +113,42 @@ struct Store::Private { enum struct XapianOpts {ReadOnly, Open, CreateOverwrite }; Private (const std::string& path, bool readonly): - db_{make_xapian(path, readonly ? XapianOpts::ReadOnly : XapianOpts::Open)}, + read_only_{readonly}, + db_{make_xapian_db(path, read_only_ ? XapianOpts::ReadOnly : XapianOpts::Open)}, mdata_{make_metadata(path)}, - contacts_{db()->get_metadata(ContactsKey), mdata_.personal_addresses} { + contacts_{db().get_metadata(ContactsKey), mdata_.personal_addresses} { if (!readonly) - wdb()->begin_transaction(); + writable_db().begin_transaction(); } Private (const std::string& path, const std::string& root_maildir, const StringVec& personal_addresses, const Store::Config& conf): - db_{make_xapian(path, XapianOpts::CreateOverwrite)}, + read_only_{false}, + db_{make_xapian_db(path, XapianOpts::CreateOverwrite)}, mdata_{init_metadata(conf, path, root_maildir, personal_addresses)}, contacts_{"", mdata_.personal_addresses} { - wdb()->begin_transaction(); + writable_db().begin_transaction(); } ~Private() try { - LOCKED; g_debug("closing store @ %s", mdata_.database_path.c_str()); - if (wdb()) { - wdb()->set_metadata (ContactsKey, contacts_.serialize()); + if (!read_only_) { + writable_db().set_metadata (ContactsKey, contacts_.serialize()); commit(); } } MU_XAPIAN_CATCH_BLOCK; - std::shared_ptr make_xapian (const std::string db_path, - XapianOpts opts) try { + std::unique_ptr make_xapian_db (const std::string db_path, XapianOpts opts) try { + switch (opts) { case XapianOpts::ReadOnly: - return std::make_shared(db_path); + return std::make_unique(db_path); case XapianOpts::Open: - return std::make_shared( - db_path, Xapian::DB_OPEN); + return std::make_unique(db_path, Xapian::DB_OPEN); case XapianOpts::CreateOverwrite: - return std::make_shared( - db_path, Xapian::DB_CREATE_OR_OVERWRITE); + return std::make_unique(db_path, Xapian::DB_CREATE_OR_OVERWRITE); default: throw std::logic_error ("invalid xapian options"); } @@ -162,22 +162,12 @@ struct Store::Private { db_path.c_str()); } - std::shared_ptr db() const { - if (!db_) - throw Mu::Error(Error::Code::NotFound, "no database found"); - return db_; - } + const Xapian::Database& db() const { return *db_.get(); } - std::shared_ptr wdb() const { - return std::dynamic_pointer_cast(db_); - } - - std::shared_ptr writable_db() const { - auto w_db{wdb()}; - if (!w_db) + Xapian::WritableDatabase& writable_db() { + if (read_only_) throw Mu::Error(Error::Code::AccessDenied, "database is read-only"); - else - return w_db; + return dynamic_cast(*db_.get()); } void dirty () try { @@ -188,35 +178,35 @@ struct Store::Private { void commit () try { g_debug("committing %zu modification(s)", dirtiness_); dirtiness_ = 0; - wdb()->commit_transaction(); - wdb()->begin_transaction(); + writable_db().commit_transaction(); + writable_db().begin_transaction(); } MU_XAPIAN_CATCH_BLOCK; void add_synonyms () { mu_flags_foreach ((MuFlagsForeachFunc)add_synonym_for_flag, - writable_db().get()); + &writable_db()); mu_msg_prio_foreach ((MuMsgPrioForeachFunc)add_synonym_for_prio, - writable_db().get()); + &writable_db()); } time_t metadata_time_t (const std::string& key) const { - const auto ts = db()->get_metadata(key); - return (time_t)atoll(db()->get_metadata(key).c_str()); + const auto ts = db().get_metadata(key); + return (time_t)atoll(db().get_metadata(key).c_str()); } Store::Metadata make_metadata(const std::string& db_path) { Store::Metadata mdata; mdata.database_path = db_path; - mdata.schema_version = db()->get_metadata(SchemaVersionKey); - mdata.created = ::atoll(db()->get_metadata(CreatedKey).c_str()); - mdata.read_only = !wdb(); + mdata.schema_version = db().get_metadata(SchemaVersionKey); + mdata.created = ::atoll(db().get_metadata(CreatedKey).c_str()); + mdata.read_only = read_only_; - mdata.batch_size = ::atoll(db()->get_metadata(BatchSizeKey).c_str()); - mdata.max_message_size = ::atoll(db()->get_metadata(MaxMessageSizeKey).c_str()); + mdata.batch_size = ::atoll(db().get_metadata(BatchSizeKey).c_str()); + mdata.max_message_size = ::atoll(db().get_metadata(MaxMessageSizeKey).c_str()); - mdata.root_maildir = db()->get_metadata(RootMaildirKey); - mdata.personal_addresses = Mu::split(db()->get_metadata(PersonalAddressesKey),","); + mdata.root_maildir = db().get_metadata(RootMaildirKey); + mdata.personal_addresses = Mu::split(db().get_metadata(PersonalAddressesKey),","); return mdata; } @@ -225,17 +215,17 @@ struct Store::Private { const std::string& path, const std::string& root_maildir, const StringVec& personal_addresses) { - wdb()->set_metadata(SchemaVersionKey, ExpectedSchemaVersion); - wdb()->set_metadata(CreatedKey, Mu::format("%" PRId64, (int64_t)::time({}))); + writable_db().set_metadata(SchemaVersionKey, ExpectedSchemaVersion); + writable_db().set_metadata(CreatedKey, Mu::format("%" PRId64, (int64_t)::time({}))); const size_t batch_size = conf.batch_size ? conf.batch_size : DefaultBatchSize; - wdb()->set_metadata(BatchSizeKey, Mu::format("%zu", batch_size)); + writable_db().set_metadata(BatchSizeKey, Mu::format("%zu", batch_size)); const size_t max_msg_size = conf.max_message_size ? conf.max_message_size : DefaultMaxMessageSize; - wdb()->set_metadata(MaxMessageSizeKey, Mu::format("%zu", max_msg_size)); + writable_db().set_metadata(MaxMessageSizeKey, Mu::format("%zu", max_msg_size)); - wdb()->set_metadata(RootMaildirKey, root_maildir); + writable_db().set_metadata(RootMaildirKey, root_maildir); std::string addrs; for (const auto& addr : personal_addresses) { // _very_ minimal check. @@ -244,15 +234,17 @@ struct Store::Private { "e-mail address '%s' contains comma", addr.c_str()); addrs += (addrs.empty() ? "": ",") + addr; } - wdb()->set_metadata (PersonalAddressesKey, addrs); + writable_db().set_metadata (PersonalAddressesKey, addrs); return make_metadata(path); } - std::shared_ptr db_; - const Store::Metadata mdata_; - Contacts contacts_; - std::unique_ptr indexer_; + const bool read_only_{}; + std::unique_ptr db_; + + const Store::Metadata mdata_; + Contacts contacts_; + std::unique_ptr indexer_; std::atomic in_transaction_{}; std::mutex lock_; @@ -311,6 +303,20 @@ Store::contacts() const return priv_->contacts_; } + +const Xapian::Database& +Store::database() const +{ + return priv_->db(); + +} + +Xapian::WritableDatabase& +Store::writable_database() +{ + return priv_->writable_db(); +} + Indexer& Store::indexer() { @@ -328,7 +334,7 @@ std::size_t Store::size() const { LOCKED; - return priv_->db()->get_doccount(); + return priv_->db().get_doccount(); } bool @@ -419,9 +425,7 @@ Store::remove_message (const std::string& path) try { const std::string term{(get_uid_term(path.c_str()))}; - auto wdb{priv()->wdb()}; - - wdb->delete_document (term); + priv()->writable_db().delete_document(term); } MU_XAPIAN_CATCH_BLOCK_RETURN (false); @@ -439,7 +443,7 @@ Store::remove_messages (const std::vector& ids) try { for (auto&& id: ids) { - priv()->wdb()->delete_document(id); + priv()->writable_db().delete_document(id); priv_->dirty(); } @@ -451,7 +455,7 @@ Store::dirstamp (const std::string& path) const { LOCKED; - const auto ts = priv_->db()->get_metadata(path); + const auto ts = priv_->db().get_metadata(path); if (ts.empty()) return 0; else @@ -466,7 +470,7 @@ Store::set_dirstamp (const std::string& path, time_t tstamp) std::array data{}; const std::size_t len = g_snprintf (data.data(), data.size(), "%zx", tstamp); - priv_->writable_db()->set_metadata(path, std::string{data.data(), len}); + priv_->writable_db().set_metadata(path, std::string{data.data(), len}); priv_->dirty(); } @@ -477,7 +481,7 @@ Store::find_message (unsigned docid) const LOCKED; try { - Xapian::Document *doc{new Xapian::Document{priv_->db()->get_document (docid)}}; + Xapian::Document *doc{new Xapian::Document{priv_->db().get_document (docid)}}; GError *gerr{}; auto msg{mu_msg_new_from_doc (reinterpret_cast(doc), &gerr)}; if (!msg) { @@ -499,26 +503,26 @@ Store::contains_message (const std::string& path) const try { const std::string term (get_uid_term(path.c_str())); - return priv_->db()->term_exists (term); + return priv_->db().term_exists (term); } MU_XAPIAN_CATCH_BLOCK_RETURN(false); } std::size_t -Store::for_each (Store::ForEachFunc func) +Store::for_each_message_path (Store::ForEachMessageFunc func) const { LOCKED; size_t n{}; try { - Xapian::Enquire enq (*priv_->db().get()); + Xapian::Enquire enq{priv_->db()}; enq.set_query (Xapian::Query::MatchAll); enq.set_cutoff (0,0); - Xapian::MSet matches(enq.get_mset (0, priv_->db()->get_doccount())); + Xapian::MSet matches(enq.get_mset (0, priv_->db().get_doccount())); for (auto&& it = matches.begin(); it != matches.end(); ++it, ++n) if (!func (*it, it.get_document().get_value(MU_MSG_FIELD_ID_PATH))) @@ -529,6 +533,53 @@ Store::for_each (Store::ForEachFunc func) return n; } +static MuMsgFieldId +field_id (const std::string& field) +{ + + if (field.empty()) + return MU_MSG_FIELD_ID_NONE; + + MuMsgFieldId id = mu_msg_field_id_from_name (field.c_str(), FALSE); + if (id != MU_MSG_FIELD_ID_NONE) + return id; + else if (field.length() == 1) + return mu_msg_field_id_from_shortcut (field[0], FALSE); + else + return MU_MSG_FIELD_ID_NONE; +} + + + +std::size_t +Store::for_each_term (const std::string& field, Store::ForEachTermFunc func) const +{ + LOCKED; + + size_t n{}; + + try { + const auto id = field_id (field.c_str()); + if (id == MU_MSG_FIELD_ID_NONE) + return {}; + + char pfx[] = { mu_msg_field_xapian_prefix(id), '\0' }; + + std::vector terms; + for (auto it = priv_->db().allterms_begin(pfx); + it != priv_->db().allterms_end(pfx); ++it) { + if (!func(*it)) + break; + } + + } MU_XAPIAN_CATCH_BLOCK; + + return n; +} + + + + void Store::commit () try { @@ -643,13 +694,6 @@ mu_store_schema_version (const MuStore *store) return self(store)->metadata().schema_version.c_str(); } -XapianDatabase* -mu_store_get_read_only_database (MuStore *store) -{ - g_return_val_if_fail (store, NULL); - return (XapianDatabase*)self(store)->priv()->db().get(); -} - static void add_terms_values_date (Xapian::Document& doc, MuMsg *msg, MuMsgFieldId mfid) { @@ -1085,8 +1129,7 @@ new_doc_from_message (MuStore *store, MuMsg *msg) } static void -update_threading_info (Xapian::WritableDatabase* db, - MuMsg *msg, Xapian::Document& doc) +update_threading_info (MuMsg *msg, Xapian::Document& doc) { const GSList *refs; @@ -1119,18 +1162,18 @@ add_or_update_msg (MuStore *store, unsigned docid, MuMsg *msg, GError **err) const std::string term (get_uid_term (mu_msg_get_path(msg))); auto self = mutable_self(store); - auto wdb = self->priv()->wdb(); + auto wdb = self->priv()->writable_db(); add_term (doc, term); // update the threading info if this message has a message id if (mu_msg_get_msgid (msg)) - update_threading_info (wdb.get(), msg, doc); + update_threading_info (msg, doc); if (docid == 0) - id = wdb->replace_document (term, doc); + id = wdb.replace_document (term, doc); else { - wdb->replace_document (docid, doc); + wdb.replace_document (docid, doc); id = docid; } diff --git a/lib/mu-store.hh b/lib/mu-store.hh index 17942989..27b52f93 100644 --- a/lib/mu-store.hh +++ b/lib/mu-store.hh @@ -103,6 +103,23 @@ public: */ const Contacts& contacts() const; + + /** + * Get the underlying Xapian database for this store. + * + * @return the database + */ + const Xapian::Database& database() const; + + /** + * Get the underlying writable Xapian database for this + * store. Throws is this store is not writable. + * + * @return the writable database + */ + Xapian::WritableDatabase& writable_database(); + + /** * Get the Indexer associated with this store. It is an error * to call this on a read-only store. @@ -175,24 +192,44 @@ public: bool contains_message (const std::string& path) const; /** - * Prototype for the ForEachFunc + * Prototype for the ForEachMessageFunc * * @param id :t store Id for the message * @param path: the absolute path to the message * * @return true if for_each should continue; false to quit */ - using ForEachFunc = std::function; + using ForEachMessageFunc = std::function; /** * Call @param func for each document in the store. This takes a lock on * the store, so the func should _not_ call any other Store:: methods. * - * @param func a functio + * @param func a Callable invoked for each message. * * @return the number of times func was invoked */ - size_t for_each (ForEachFunc func); + size_t for_each_message_path (ForEachMessageFunc func) const; + + /** + * Prototype for the ForEachTermFunc + * + * @param term: + * + * @return true if for_each should continue; false to quit + */ + using ForEachTermFunc = std::function; + + /** + * Call @param func for each term for the given field in the store. This + * takes a lock on the store, so the func should _not_ call any other + * Store:: methods. + * + * @param func a Callable invoked for each message. + * + * @return the number of times func was invoked + */ + size_t for_each_term (const std::string& field, ForEachTermFunc func) const; /** * Get the timestamp for some message, or 0 if not found @@ -303,23 +340,6 @@ MuStore* mu_store_ref (MuStore *store); */ MuStore* mu_store_unref (MuStore *store); - -/** - * we need this when using Xapian::(Writable)Database* from C - */ -typedef gpointer XapianDatabase; - -/** - * get the underlying read-only database object for this store; not that this - * pointer becomes in valid after mu_store_destroy - * - * @param store a valid store - * - * @return a Xapian::Database (you'll need to cast in C++), or - * NULL in case of error. - */ -XapianDatabase* mu_store_get_read_only_database (MuStore *store); - /** * get the version of the xapian database (ie., the version of the * 'schema' we are using). If this version != MU_STORE_SCHEMA_VERSION, diff --git a/lib/query/mu-tokenizer.cc b/lib/mu-tokenizer.cc similarity index 100% rename from lib/query/mu-tokenizer.cc rename to lib/mu-tokenizer.cc diff --git a/lib/query/mu-tokenizer.hh b/lib/mu-tokenizer.hh similarity index 98% rename from lib/query/mu-tokenizer.hh rename to lib/mu-tokenizer.hh index ac083c8f..aa56d896 100644 --- a/lib/query/mu-tokenizer.hh +++ b/lib/mu-tokenizer.hh @@ -97,7 +97,7 @@ operator<< (std::ostream& os, Token::Type t) case Token::Type::And: os << ""; break; case Token::Type::Or: os << ""; break; case Token::Type::Xor: os << ""; break; - + case Token::Type::Empty: os << ""; break; default: // can't happen, but pacify compiler throw std::runtime_error ("<>"); } diff --git a/lib/query/mu-tree.hh b/lib/mu-tree.hh similarity index 98% rename from lib/query/mu-tree.hh rename to lib/mu-tree.hh index eacda988..70642a17 100644 --- a/lib/query/mu-tree.hh +++ b/lib/mu-tree.hh @@ -24,7 +24,7 @@ #include #include -#include +#include #include namespace Mu { diff --git a/lib/query/mu-xapian.cc b/lib/mu-xapian.cc similarity index 94% rename from lib/query/mu-xapian.cc rename to lib/mu-xapian.cc index f058b66a..f620c4fa 100644 --- a/lib/query/mu-xapian.cc +++ b/lib/mu-xapian.cc @@ -32,6 +32,8 @@ xapian_query_op (const Mu::Tree& tree) { Xapian::Query::op op; +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wswitch-enum" switch (tree.node.type) { case Node::Type::OpNot: // OpNot x ::= AND NOT x if (tree.children.size() != 1) @@ -45,7 +47,7 @@ xapian_query_op (const Mu::Tree& tree) case Node::Type::OpAndNot: op = Xapian::Query::OP_AND_NOT; break; default: throw Mu::Error (Error::Code::Internal, "invalid op"); // bug } - +#pragma GCC diagnostic pop std::vector childvec; for (const auto& subtree: tree.children) childvec.emplace_back(xapian_query(subtree)); @@ -97,6 +99,8 @@ xapian_query_range (const Mu::Tree& tree) Xapian::Query Mu::xapian_query (const Mu::Tree& tree) { +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wswitch-enum" switch (tree.node.type) { case Node::Type::Empty: return Xapian::Query(); @@ -113,4 +117,5 @@ Mu::xapian_query (const Mu::Tree& tree) default: throw Mu::Error (Error::Code::Internal, "invalid query"); // bug } +#pragma GCC diagnostic pop } diff --git a/lib/query/mu-xapian.hh b/lib/mu-xapian.hh similarity index 97% rename from lib/query/mu-xapian.hh rename to lib/mu-xapian.hh index 503d9eb5..95be0576 100644 --- a/lib/query/mu-xapian.hh +++ b/lib/mu-xapian.hh @@ -22,7 +22,7 @@ #define __XAPIAN_HH__ #include -#include +#include namespace Mu { diff --git a/lib/query/Makefile.am b/lib/query/Makefile.am deleted file mode 100644 index 6923d2b3..00000000 --- a/lib/query/Makefile.am +++ /dev/null @@ -1,99 +0,0 @@ -## Copyright (C) 2017-2020 Dirk-Jan C. Binnema -## -## This program is free software; you can redistribute it and/or modify -## it under the terms of the GNU General Public License as published by -## the Free Software Foundation; either version 3 of the License, or -## (at your option) any later version. -## -## This program is distributed in the hope that it will be useful, -## but WITHOUT ANY WARRANTY; without even the implied warranty of -## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -## GNU General Public License for more details. -## -## You should have received a copy of the GNU General Public License -## along with this program; if not, write to the Free Software Foundation, -## Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - -include $(top_srcdir)/gtest.mk - -@VALGRIND_CHECK_RULES@ - -AM_CXXFLAGS= \ - -I$(srcdir)/.. \ - -I$(top_srcdir)/lib \ - $(GLIB_CFLAGS) \ - $(XAPIAN_CXXFLAGS) \ - $(WARN_CXXFLAGS) \ - $(ASAN_CXXFLAGS) \ - $(CODE_COVERAGE_CFLAGS) \ - -Wno-inline \ - -Wno-switch-enum - -AM_CPPFLAGS= \ - $(CODE_COVERAGE_CPPFLAGS) - -AM_LDFLAGS= \ - $(ASAN_LDFLAGS) \ - $(WARN_LDFLAGS) - -noinst_PROGRAMS= \ - tokenize \ - parse - -noinst_LTLIBRARIES= \ - libmu-query.la - -libmu_query_la_SOURCES= \ - mu-data.hh \ - mu-parser.cc \ - mu-parser.hh \ - mu-proc-iface.hh \ - mu-tokenizer.cc \ - mu-tokenizer.hh \ - mu-tree.hh \ - mu-xapian.cc \ - mu-xapian.hh - -libmu_query_la_LIBADD= \ - $(WARN_LDFLAGS) \ - $(GLIB_LIBS) \ - $(XAPIAN_LIBS) \ - ../utils/libmu-utils.la \ - $(CODE_COVERAGE_LIBS) - -VALGRIND_SUPPRESSIONS_FILES= \ - ${top_srcdir}/mu.supp - -tokenize_SOURCES= \ - tokenize.cc - -tokenize_LDADD= \ - $(WARN_LDFLAGS) \ - libmu-query.la - -parse_SOURCES= \ - parse.cc - -parse_LDADD= \ - $(WARN_LDFLAGS) \ - libmu-query.la - -noinst_PROGRAMS+=$(TEST_PROGS) - -TEST_PROGS+= \ - test-tokenizer -test_tokenizer_SOURCES= \ - test-tokenizer.cc -test_tokenizer_LDADD= \ - libmu-query.la - -TEST_PROGS+= \ - test-parser -test_parser_SOURCES= \ - test-parser.cc -test_parser_LDADD= \ - libmu-query.la - -TESTS=$(TEST_PROGS) - -include $(top_srcdir)/aminclude_static.am diff --git a/lib/query/mu-parser.cc b/lib/query/mu-parser.cc deleted file mode 100644 index 8becb1db..00000000 --- a/lib/query/mu-parser.cc +++ /dev/null @@ -1,344 +0,0 @@ -/* -** Copyright (C) 2020 Dirk-Jan C. Binnema -** -** This library is free software; you can redistribute it and/or -** modify it under the terms of the GNU Lesser General Public License -** as published by the Free Software Foundation; either version 2.1 -** of the License, or (at your option) any later version. -** -** This library is distributed in the hope that it will be useful, -** but WITHOUT ANY WARRANTY; without even the implied warranty of -** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -** Lesser General Public License for more details. -** -** You should have received a copy of the GNU Lesser General Public -** License along with this library; if not, write to the Free -** Software Foundation, 51 Franklin Street, Fifth Floor, Boston, MA -** 02110-1301, USA. -*/ -#include "mu-parser.hh" -#include "mu-tokenizer.hh" -#include "utils/mu-utils.hh" -#include "utils/mu-error.hh" - -using namespace Mu; - -// 3 precedence levels: units (NOT,()) > factors (OR) > terms (AND) - -// query -> | ε -// -> | ε -// -> OR|XOR | ε -// -> | ε -// -> [AND]|AND NOT | ε -// -> [NOT] | ( ) | -// -> | | -// -> [field:]value -// -> [field:][lower]..[upper] -// -> [field:]/regex/ - - -#define BUG(...) Mu::Error (Error::Code::Internal, format("%u: BUG: ",__LINE__) \ - + format(__VA_ARGS__)) - -static Token -look_ahead (const Mu::Tokens& tokens) -{ - return tokens.front(); -} - -static Mu::Tree -empty() -{ - return {{Node::Type::Empty}}; -} - -static Mu::Tree term_1 (Mu::Tokens& tokens, ProcPtr proc, WarningVec& warnings); - - -static Mu::Tree -value (const ProcIface::FieldInfoVec& fields, const std::string& v, - size_t pos, ProcPtr proc, WarningVec& warnings) -{ - auto val = utf8_flatten(v); - - if (fields.empty()) - throw BUG("expected one or more fields"); - - if (fields.size() == 1) { - const auto item = fields.front(); - return Tree({Node::Type::Value, - std::make_unique( - item.field, item.prefix, item.id, - proc->process_value(item.field, val), - item.supports_phrase)}); - } - - // a 'multi-field' such as "recip:" - Tree tree(Node{Node::Type::OpOr}); - for (const auto& item: fields) - tree.add_child (Tree({Node::Type::Value, - std::make_unique( - item.field, item.prefix, item.id, - proc->process_value(item.field, val), - item.supports_phrase)})); - return tree; -} - -static Mu::Tree -regex (const ProcIface::FieldInfoVec& fields, const std::string& v, - size_t pos, ProcPtr proc, WarningVec& warnings) -{ - if (v.length() < 2) - throw BUG("expected regexp, got '%s'", v.c_str()); - - const auto rxstr = utf8_flatten(v.substr(1, v.length()-2)); - - try { - Tree tree(Node{Node::Type::OpOr}); - const auto rx = std::regex (rxstr); - for (const auto& field: fields) { - const auto terms = proc->process_regex (field.field, rx); - for (const auto& term: terms) { - tree.add_child (Tree( - {Node::Type::Value, - std::make_unique(field.field, "", - field.id, term)})); - } - } - - if (tree.children.empty()) - return empty(); - else - return tree; - - } catch (...) { - // fallback - warnings.push_back ({pos, "invalid regexp"}); - return value (fields, v, pos, proc, warnings); - } -} - - - -static Mu::Tree -range (const ProcIface::FieldInfoVec& fields, const std::string& lower, - const std::string& upper, size_t pos, ProcPtr proc, - WarningVec& warnings) -{ - if (fields.empty()) - throw BUG("expected field"); - - const auto& field = fields.front(); - if (!proc->is_range_field(field.field)) - return value (fields, lower + ".." + upper, pos, proc, warnings); - - auto prange = proc->process_range (field.field, lower, upper); - if (prange.lower > prange.upper) - prange = proc->process_range (field.field, upper, lower); - - return Tree({Node::Type::Range, - std::make_unique(field.field, field.prefix, field.id, - prange.lower, prange.upper)}); -} - - -static Mu::Tree -data (Mu::Tokens& tokens, ProcPtr proc, WarningVec& warnings) -{ - const auto token = look_ahead(tokens); - if (token.type != Token::Type::Data) - warnings.push_back ({token.pos, "expected: value"}); - - tokens.pop_front(); - - std::string field, val; - const auto col = token.str.find (":"); - if (col != 0 && col != std::string::npos && col != token.str.length()-1) { - field = token.str.substr(0, col); - val = token.str.substr(col + 1); - } else - val = token.str; - - auto fields = proc->process_field (field); - if (fields.empty()) {// not valid field... - warnings.push_back ({token.pos, format ("invalid field '%s'", field.c_str())}); - fields = proc->process_field (""); - // fallback, treat the whole of foo:bar as a value - return value (fields, field + ":" + val, token.pos, proc, warnings); - } - - // does it look like a regexp? - if (val.length() >=2 ) - if (val[0] == '/' && val[val.length()-1] == '/') - return regex (fields, val, token.pos, proc, warnings); - - // does it look like a range? - const auto dotdot = val.find(".."); - if (dotdot != std::string::npos) - return range(fields, val.substr(0, dotdot), val.substr(dotdot + 2), - token.pos, proc, warnings); - else if (proc->is_range_field(fields.front().field)) { - // range field without a range - treat as field:val..val - return range (fields, val, val, token.pos, proc, warnings); - } - - // if nothing else, it's a value. - return value (fields, val, token.pos, proc, warnings); -} - -static Mu::Tree -unit (Mu::Tokens& tokens, ProcPtr proc, WarningVec& warnings) -{ - if (tokens.empty()) { - warnings.push_back ({0, "expected: unit"}); - return empty(); - } - - const auto token = look_ahead (tokens); - - if (token.type == Token::Type::Not) { - tokens.pop_front(); - Tree tree{{Node::Type::OpNot}}; - tree.add_child(unit (tokens, proc, warnings)); - return tree; - } - - if (token.type == Token::Type::Open) { - tokens.pop_front(); - auto tree = term_1 (tokens, proc, warnings); - if (tokens.empty()) - warnings.push_back({token.pos, "expected: ')'"}); - else { - const auto token2 = look_ahead(tokens); - if (token2.type == Token::Type::Close) - tokens.pop_front(); - else { - warnings.push_back( - {token2.pos, - std::string("expected: ')' but got ") + - token2.str}); - } - - } - return tree; - } - - return data (tokens, proc, warnings); -} - -static Mu::Tree factor_1 (Mu::Tokens& tokens, ProcPtr proc, - WarningVec& warnings); - -static Mu::Tree -factor_2 (Mu::Tokens& tokens, Node::Type& op, ProcPtr proc, - WarningVec& warnings) -{ - if (tokens.empty()) - return empty(); - - const auto token = look_ahead(tokens); - - switch (token.type) { - case Token::Type::And: { - tokens.pop_front(); - op = Node::Type::OpAnd; - } break; - - case Token::Type::Open: - case Token::Type::Data: - case Token::Type::Not: - op = Node::Type::OpAnd; // implicit AND - break; - - default: - return empty(); - } - - return factor_1 (tokens, proc, warnings); -} - -static Mu::Tree -factor_1 (Mu::Tokens& tokens, ProcPtr proc, WarningVec& warnings) -{ - Node::Type op { Node::Type::Invalid }; - - auto t = unit (tokens, proc, warnings); - auto a2 = factor_2 (tokens, op, proc, warnings); - - if (a2.empty()) - return t; - - Tree tree {{op}}; - tree.add_child(std::move(t)); - tree.add_child(std::move(a2)); - - return tree; -} - - -static Mu::Tree -term_2 (Mu::Tokens& tokens, Node::Type& op, ProcPtr proc, - WarningVec& warnings) -{ - if (tokens.empty()) - return empty(); - - const auto token = look_ahead (tokens); - - switch (token.type) { - case Token::Type::Or: - op = Node::Type::OpOr; - break; - case Token::Type::Xor: - op = Node::Type::OpXor; - break; - default: - if (token.type != Token::Type::Close) - warnings.push_back({token.pos, "expected OR|XOR"}); - return empty(); - } - - tokens.pop_front(); - - return term_1 (tokens, proc, warnings); -} - -static Mu::Tree -term_1 (Mu::Tokens& tokens, ProcPtr proc, WarningVec& warnings) -{ - Node::Type op { Node::Type::Invalid }; - - auto t = factor_1 (tokens, proc, warnings); - auto o2 = term_2 (tokens, op, proc, warnings); - - if (o2.empty()) - return t; - else { - Tree tree {{op}}; - tree.add_child(std::move(t)); - tree.add_child(std::move(o2)); - return tree; - } -} - -static Mu::Tree -query (Mu::Tokens& tokens, ProcPtr proc, WarningVec& warnings) -{ - if (tokens.empty()) - return empty (); - else - return term_1 (tokens, proc, warnings); -} - -Mu::Tree -Mu::parse (const std::string& expr, WarningVec& warnings, ProcPtr proc) -{ - try { - auto tokens = tokenize (expr); - return query (tokens, proc, warnings); - - } catch (const std::runtime_error& ex) { - std::cerr << ex.what() << std::endl; - return empty(); - } -} diff --git a/lib/query/mu-proc-iface.hh b/lib/query/mu-proc-iface.hh deleted file mode 100644 index b43ef413..00000000 --- a/lib/query/mu-proc-iface.hh +++ /dev/null @@ -1,132 +0,0 @@ -/* -** Copyright (C) 2017 Dirk-Jan C. Binnema -** -** This library is free software; you can redistribute it and/or -** modify it under the terms of the GNU Lesser General Public License -** as published by the Free Software Foundation; either version 2.1 -** of the License, or (at your option) any later version. -** -** This library is distributed in the hope that it will be useful, -** but WITHOUT ANY WARRANTY; without even the implied warranty of -** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -** Lesser General Public License for more details. -** -** You should have received a copy of the GNU Lesser General Public -** License along with this library; if not, write to the Free -** Software Foundation, 51 Franklin Street, Fifth Floor, Boston, MA -** 02110-1301, USA. -*/ -#ifndef __PROC_IFACE_HH__ -#define __PROC_IFACE_HH__ - -#include -#include -#include -#include - -namespace Mu { - -struct ProcIface { - - virtual ~ProcIface() = default; - - /** - * Get the "shortcut"/internal fields for the the given fieldstr or empty if there is none - * - * @param fieldstr a fieldstr, e.g "subject" or "s" for the subject field - * - * @return a vector with "exploded" values, with a code and a fullname. E.g. "s" might map - * to [<"S","subject">], while "recip" could map to [<"to", "T">, <"cc", "C">, <"bcc", "B">] - */ - struct FieldInfo { - const std::string field; - const std::string prefix; - bool supports_phrase; - unsigned id; - }; - using FieldInfoVec = std::vector; - - virtual FieldInfoVec process_field (const std::string& field) const = 0; - - /** - * Process a value - * - * @param field a field name - * @param value a value - * - * @return the processed value - */ - virtual std::string process_value ( - const std::string& field, const std::string& value) const = 0; - - /** - * Is this a range field? - * - * @param field some field - * - * @return true if it is a range-field; false otherwise. - */ - virtual bool is_range_field (const std::string& field) const = 0; - - - /** - * Process a range field - * - * @param fieldstr a fieldstr, e.g "date" or "d" for the date field - * @param lower lower bound or empty - * @param upper upper bound or empty - * - * @return the processed range - */ - struct Range { - std::string lower; - std::string upper; - }; - virtual Range process_range (const std::string& field, const std::string& lower, - const std::string& upper) const = 0; - - /** - * - * - * @param field - * @param rx - * - * @return - */ - virtual std::vector - process_regex (const std::string& field, const std::regex& rx) const = 0; - -}; // ProcIface - - -struct DummyProc: public ProcIface { // For testing - - std::vector - process_field (const std::string& field) const override { - return {{ field, "x", false, 0 }}; - } - - std::string - process_value (const std::string& field, const std::string& value) const override { - return value; - } - - bool is_range_field (const std::string& field) const override { - return field == "range"; - } - - Range process_range (const std::string& field, const std::string& lower, - const std::string& upper) const override { - return { lower, upper }; - } - - std::vector - process_regex (const std::string& field, const std::regex& rx) const override { - return {}; - } -}; //Dummy - - -} // Mu - -#endif /* __PROC_IFACE_HH__ */ diff --git a/lib/query/parse.cc b/lib/query/parse.cc deleted file mode 100644 index d988d6e4..00000000 --- a/lib/query/parse.cc +++ /dev/null @@ -1,41 +0,0 @@ -/* -** Copyright (C) 2020 Dirk-Jan C. Binnema -** -** This library is free software; you can redistribute it and/or -** modify it under the terms of the GNU Lesser General Public License -** as published by the Free Software Foundation; either version 2.1 -** of the License, or (at your option) any later version. -** -** This library is distributed in the hope that it will be useful, -** but WITHOUT ANY WARRANTY; without even the implied warranty of -** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -** Lesser General Public License for more details. -** -** You should have received a copy of the GNU Lesser General Public -** License along with this library; if not, write to the Free -** Software Foundation, 51 Franklin Street, Fifth Floor, Boston, MA -** 02110-1301, USA. -*/ - -#include -#include -#include "mu-parser.hh" - -int -main (int argc, char *argv[]) -{ - std::string s; - - for (auto i = 1; i < argc; ++i) - s += " " + std::string(argv[i]); - - Mu::WarningVec warnings; - - const auto tree = Mu::parse (s, warnings); - for (const auto& w: warnings) - std::cerr << "1:" << w.pos << ": " << w.msg << std::endl; - - std::cout << tree << std::endl; - - return 0; -} diff --git a/lib/test-indexer.cc b/lib/test-indexer.cc new file mode 100644 index 00000000..3660381f --- /dev/null +++ b/lib/test-indexer.cc @@ -0,0 +1,70 @@ +/* +** Copyright (C) 2020 Dirk-Jan C. Binnema +** +** This program is free software; you can redistribute it and/or modify it +** under the terms of the GNU General Public License as published by the +** Free Software Foundation; either version 3, or (at your option) any +** later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program; if not, write to the Free Software Foundation, +** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +** +*/ + +#include +#include + +#include +#include +#include + +#include "mu-indexer.hh" +#include "utils/mu-utils.hh" +#include "test-mu-common.h" + +using namespace Mu; + +static void +test_index_maildir () +{ + allow_warnings(); + + Store store{test_mu_common_get_random_tmpdir(), std::string{MU_TESTMAILDIR}}; + Indexer idx{Indexer::Config{}, store}; + + g_assert_true (idx.start()); + while (idx.is_running()) { + sleep(1); + } + + g_print ("again!\n"); + + g_assert_true (idx.start()); + while (idx.is_running()) { + sleep(1); + } +} + +int +main (int argc, char *argv[]) try +{ + g_test_init (&argc, &argv, NULL); + + g_test_add_func ("/indexer/index-maildir", test_index_maildir); + + return g_test_run (); + + +} catch (const std::runtime_error& re) { + std::cerr << re.what() << "\n"; + return 1; +} catch (...) { + std::cerr << "caught exception\n"; + return 1; +} diff --git a/lib/test-mu-common.c b/lib/test-mu-common.cc similarity index 95% rename from lib/test-mu-common.c rename to lib/test-mu-common.cc index 5cabb768..6e4d159f 100644 --- a/lib/test-mu-common.c +++ b/lib/test-mu-common.cc @@ -1,5 +1,5 @@ /* -** Copyright (C) 2008-2013 Dirk-Jan C. Binnema +** Copyright (C) 2008-2020 Dirk-Jan C. Binnema ** ** This program is free software; you can redistribute it and/or modify it ** under the terms of the GNU General Public License as published by the @@ -31,7 +31,7 @@ #include #include -#include "test-mu-common.h" +#include "test-mu-common.hh" char* test_mu_common_get_random_tmpdir (void) diff --git a/lib/test-mu-common.h b/lib/test-mu-common.hh similarity index 100% rename from lib/test-mu-common.h rename to lib/test-mu-common.hh diff --git a/lib/test-mu-contacts.cc b/lib/test-mu-contacts.cc index 208943af..58174317 100644 --- a/lib/test-mu-contacts.cc +++ b/lib/test-mu-contacts.cc @@ -21,7 +21,7 @@ #include "config.h" #include -#include "test-mu-common.h" +#include "test-mu-common.hh" #include "mu-contacts.hh" static void diff --git a/lib/test-mu-container.c b/lib/test-mu-container.cc similarity index 95% rename from lib/test-mu-container.c rename to lib/test-mu-container.cc index 475e3026..49e62deb 100644 --- a/lib/test-mu-container.c +++ b/lib/test-mu-container.cc @@ -25,7 +25,7 @@ #include -#include "test-mu-common.h" +#include "test-mu-common.hh" #include "mu-container.h" static gboolean @@ -76,7 +76,7 @@ main (int argc, char *argv[]) test_mu_container_splice_children_when_parent_has_no_siblings); g_log_set_handler (NULL, - G_LOG_LEVEL_MASK | G_LOG_FLAG_FATAL| G_LOG_FLAG_RECURSION, + (GLogLevelFlags)(G_LOG_LEVEL_MASK | G_LOG_FLAG_FATAL| G_LOG_FLAG_RECURSION), (GLogFunc)black_hole, NULL); return g_test_run (); diff --git a/lib/test-mu-date.c b/lib/test-mu-date.cc similarity index 100% rename from lib/test-mu-date.c rename to lib/test-mu-date.cc diff --git a/lib/test-mu-flags.c b/lib/test-mu-flags.cc similarity index 84% rename from lib/test-mu-flags.c rename to lib/test-mu-flags.cc index 36db9c8a..f87468c6 100644 --- a/lib/test-mu-flags.c +++ b/lib/test-mu-flags.cc @@ -24,7 +24,7 @@ #include #include "mu-flags.h" -#include "test-mu-common.h" +#include "test-mu-common.hh" static void @@ -41,7 +41,7 @@ test_mu_flag_char (void) g_assert_cmpuint (mu_flag_char (MU_FLAG_ENCRYPTED), ==, 'x'); g_assert_cmpuint (mu_flag_char (MU_FLAG_HAS_ATTACH), ==, 'a'); g_assert_cmpuint (mu_flag_char (MU_FLAG_UNREAD), ==, 'u'); - g_assert_cmpuint (mu_flag_char (12345), ==, 0); + g_assert_cmpuint (mu_flag_char ((MuFlags)12345), ==, 0); } @@ -60,30 +60,30 @@ test_mu_flag_name (void) g_assert_cmpstr (mu_flag_name (MU_FLAG_ENCRYPTED), ==, "encrypted"); g_assert_cmpstr (mu_flag_name (MU_FLAG_HAS_ATTACH), ==, "attach"); g_assert_cmpstr (mu_flag_name (MU_FLAG_UNREAD), ==, "unread"); - g_assert_cmpstr (mu_flag_name (12345), ==, NULL); + g_assert_cmpstr (mu_flag_name ((MuFlags)12345), ==, NULL); } static void test_mu_flags_to_str_s (void) { - g_assert_cmpstr (mu_flags_to_str_s(MU_FLAG_PASSED|MU_FLAG_SIGNED, + g_assert_cmpstr (mu_flags_to_str_s((MuFlags)(MU_FLAG_PASSED|MU_FLAG_SIGNED), MU_FLAG_TYPE_ANY), ==, "Pz"); g_assert_cmpstr (mu_flags_to_str_s(MU_FLAG_NEW, MU_FLAG_TYPE_ANY), ==, "N"); - g_assert_cmpstr (mu_flags_to_str_s(MU_FLAG_HAS_ATTACH|MU_FLAG_TRASHED, + g_assert_cmpstr (mu_flags_to_str_s((MuFlags)(MU_FLAG_HAS_ATTACH|MU_FLAG_TRASHED), MU_FLAG_TYPE_ANY), ==, "Ta"); g_assert_cmpstr (mu_flags_to_str_s(MU_FLAG_NONE, MU_FLAG_TYPE_ANY), ==, ""); - g_assert_cmpstr (mu_flags_to_str_s(MU_FLAG_PASSED|MU_FLAG_SIGNED, + g_assert_cmpstr (mu_flags_to_str_s((MuFlags)(MU_FLAG_PASSED|MU_FLAG_SIGNED), MU_FLAG_TYPE_CONTENT), ==, "z"); g_assert_cmpstr (mu_flags_to_str_s(MU_FLAG_NEW, MU_FLAG_TYPE_MAILDIR), ==, "N"); - g_assert_cmpstr (mu_flags_to_str_s(MU_FLAG_HAS_ATTACH|MU_FLAG_TRASHED, + g_assert_cmpstr (mu_flags_to_str_s((MuFlags)(MU_FLAG_HAS_ATTACH|MU_FLAG_TRASHED), MU_FLAG_TYPE_MAILFILE), ==, "T"); @@ -100,11 +100,11 @@ test_mu_flags_from_str (void) */ g_assert_cmpuint (mu_flags_from_str ("RP", MU_FLAG_TYPE_ANY, TRUE), ==, - MU_FLAG_REPLIED | MU_FLAG_PASSED); + (MuFlags)( MU_FLAG_REPLIED | MU_FLAG_PASSED)); g_assert_cmpuint (mu_flags_from_str ("Nz", MU_FLAG_TYPE_ANY, TRUE), ==, MU_FLAG_NEW | MU_FLAG_SIGNED); g_assert_cmpuint (mu_flags_from_str ("axD", MU_FLAG_TYPE_ANY, TRUE), ==, - MU_FLAG_HAS_ATTACH | MU_FLAG_ENCRYPTED | MU_FLAG_DRAFT); + (MuFlags)( MU_FLAG_HAS_ATTACH | MU_FLAG_ENCRYPTED | MU_FLAG_DRAFT)); g_assert_cmpuint (mu_flags_from_str ("RP", MU_FLAG_TYPE_MAILFILE, TRUE), ==, MU_FLAG_REPLIED | MU_FLAG_PASSED); @@ -124,19 +124,19 @@ static void test_mu_flags_from_str_delta (void) { g_assert_cmpuint (mu_flags_from_str_delta ("+S-R", - MU_FLAG_REPLIED | MU_FLAG_DRAFT, + (MuFlags)(MU_FLAG_REPLIED | MU_FLAG_DRAFT), MU_FLAG_TYPE_ANY),==, - MU_FLAG_SEEN | MU_FLAG_DRAFT); + (MuFlags)(MU_FLAG_SEEN | MU_FLAG_DRAFT)); g_assert_cmpuint (mu_flags_from_str_delta ("", - MU_FLAG_REPLIED | MU_FLAG_DRAFT, + (MuFlags)(MU_FLAG_REPLIED | MU_FLAG_DRAFT), MU_FLAG_TYPE_ANY),==, - MU_FLAG_REPLIED | MU_FLAG_DRAFT); + (MuFlags)(MU_FLAG_REPLIED | MU_FLAG_DRAFT)); g_assert_cmpuint (mu_flags_from_str_delta ("-N+P+S-D", - MU_FLAG_SIGNED | MU_FLAG_DRAFT, + (MuFlags)(MU_FLAG_SIGNED | MU_FLAG_DRAFT), MU_FLAG_TYPE_ANY),==, - MU_FLAG_PASSED | MU_FLAG_SEEN | MU_FLAG_SIGNED); + (MuFlags)(MU_FLAG_PASSED | MU_FLAG_SEEN | MU_FLAG_SIGNED)); } @@ -184,7 +184,8 @@ main (int argc, char *argv[]) test_mu_flags_custom_from_str); g_log_set_handler (NULL, - G_LOG_LEVEL_MASK | G_LOG_FLAG_FATAL| G_LOG_FLAG_RECURSION, + (GLogLevelFlags)(G_LOG_LEVEL_MASK | G_LOG_FLAG_FATAL| + G_LOG_FLAG_RECURSION), (GLogFunc)black_hole, NULL); rv = g_test_run (); diff --git a/lib/test-mu-maildir.c b/lib/test-mu-maildir.cc similarity index 95% rename from lib/test-mu-maildir.c rename to lib/test-mu-maildir.cc index 39521274..cfffa44c 100644 --- a/lib/test-mu-maildir.c +++ b/lib/test-mu-maildir.cc @@ -28,10 +28,11 @@ #include #include -#include "test-mu-common.h" +#include "test-mu-common.hh" #include "mu-maildir.h" #include "utils/mu-util.h" + static void test_mu_maildir_mkdir_01 (void) { @@ -400,7 +401,7 @@ test_mu_maildir_get_flags_from_path (void) } paths[] = { { "/home/foo/Maildir/test/cur/123456:2,FSR", - MU_FLAG_REPLIED | MU_FLAG_SEEN | MU_FLAG_FLAGGED + (MuFlags)(MU_FLAG_REPLIED | MU_FLAG_SEEN | MU_FLAG_FLAGGED) }, { "/home/foo/Maildir/test/new/123456", @@ -413,8 +414,8 @@ test_mu_maildir_get_flags_from_path (void) }, { "/home/foo/Maildir/test/cur/123456:2,DTP", - MU_FLAG_DRAFT | MU_FLAG_TRASHED | - MU_FLAG_PASSED + (MuFlags)(MU_FLAG_DRAFT | MU_FLAG_TRASHED | + MU_FLAG_PASSED) }, { "/home/foo/Maildir/test/cur/123456:2,S", @@ -434,7 +435,8 @@ test_mu_maildir_get_flags_from_path (void) static void assert_matches_regexp (const char *str, const char *rx) { - if (!g_regex_match_simple (rx, str, 0, 0)) { + if (!g_regex_match_simple (rx, str, (GRegexCompileFlags)0, + (GRegexMatchFlags)0)) { if (g_test_verbose ()) g_print ("%s does not match %s", str, rx); g_assert (0); @@ -463,11 +465,11 @@ test_mu_maildir_get_new_path_new (void) "/home/foo/Maildir/test/new/123456" }, { "/home/foo/Maildir/test/new/123456:2,FR", - MU_FLAG_SEEN | MU_FLAG_REPLIED, + (MuFlags)(MU_FLAG_SEEN | MU_FLAG_REPLIED), "/home/foo/Maildir/test/cur/123456:2,RS" }, { "/home/foo/Maildir/test/new/1313038887_0.697:2,", - MU_FLAG_SEEN | MU_FLAG_FLAGGED | MU_FLAG_PASSED, + (MuFlags)(MU_FLAG_SEEN | MU_FLAG_FLAGGED | MU_FLAG_PASSED), "/home/foo/Maildir/test/cur/1313038887_0.697:2,FPS" }, { "/home/djcb/Maildir/trash/new/1312920597.2206_16.cthulhu", @@ -513,11 +515,11 @@ test_mu_maildir_get_new_path_01 (void) "/home/foo/Maildir/test/new/123456" }, { "/home/foo/Maildir/test/new/123456:2,FR", - MU_FLAG_SEEN | MU_FLAG_REPLIED, + (MuFlags)(MU_FLAG_SEEN | MU_FLAG_REPLIED), "/home/foo/Maildir/test/cur/123456:2,RS" }, { "/home/foo/Maildir/test/new/1313038887_0.697:2,", - MU_FLAG_SEEN | MU_FLAG_FLAGGED | MU_FLAG_PASSED, + (MuFlags)(MU_FLAG_SEEN | MU_FLAG_FLAGGED | MU_FLAG_PASSED), "/home/foo/Maildir/test/cur/1313038887_0.697:2,FPS" }, { "/home/djcb/Maildir/trash/new/1312920597.2206_16.cthulhu", @@ -557,12 +559,12 @@ test_mu_maildir_get_new_path_02 (void) "/home/bar/Maildir/coffee/new/123456" }, { "/home/foo/Maildir/test/new/123456", - MU_FLAG_SEEN | MU_FLAG_REPLIED, + (MuFlags)(MU_FLAG_SEEN | MU_FLAG_REPLIED), "/home/cuux/Maildir/tea", "/home/cuux/Maildir/tea/cur/123456:2,RS" }, { "/home/foo/Maildir/test/new/1313038887_0.697:2,", - MU_FLAG_SEEN | MU_FLAG_FLAGGED | MU_FLAG_PASSED, + (MuFlags)(MU_FLAG_SEEN | MU_FLAG_FLAGGED | MU_FLAG_PASSED), "/home/boy/Maildir/stuff", "/home/boy/Maildir/stuff/cur/1313038887_0.697:2,FPS" } @@ -685,8 +687,8 @@ main (int argc, char *argv[]) test_mu_maildir_get_maildir_from_path); g_log_set_handler (NULL, - G_LOG_LEVEL_MASK | G_LOG_FLAG_FATAL| - G_LOG_FLAG_RECURSION, + (GLogLevelFlags)(G_LOG_LEVEL_MASK | G_LOG_FLAG_FATAL| + G_LOG_FLAG_RECURSION), (GLogFunc)black_hole, NULL); return g_test_run (); diff --git a/lib/test-mu-msg-fields.c b/lib/test-mu-msg-fields.cc similarity index 94% rename from lib/test-mu-msg-fields.c rename to lib/test-mu-msg-fields.cc index 27864d8d..15cbcda2 100644 --- a/lib/test-mu-msg-fields.c +++ b/lib/test-mu-msg-fields.cc @@ -1,5 +1,5 @@ /* -** Copyright (C) 2008-2013 Dirk-Jan C. Binnema +** Copyright (C) 2008-2020 Dirk-Jan C. Binnema ** ** This program is free software; you can redistribute it and/or modify it ** under the terms of the GNU General Public License as published by the @@ -28,7 +28,7 @@ #include -#include "test-mu-common.h" +#include "test-mu-common.hh" #include "mu-msg-fields.h" static void @@ -126,8 +126,8 @@ main (int argc, char *argv[]) * function simply calls mu_msg_field_str */ g_log_set_handler (NULL, - G_LOG_LEVEL_MASK | G_LOG_FLAG_FATAL | - G_LOG_FLAG_RECURSION, + (GLogLevelFlags)(G_LOG_LEVEL_MASK | G_LOG_FLAG_FATAL | + G_LOG_FLAG_RECURSION), (GLogFunc)black_hole, NULL); return g_test_run (); diff --git a/lib/test-mu-msg.c b/lib/test-mu-msg.cc similarity index 97% rename from lib/test-mu-msg.c rename to lib/test-mu-msg.cc index fef368b9..907b16f4 100644 --- a/lib/test-mu-msg.c +++ b/lib/test-mu-msg.cc @@ -1,6 +1,5 @@ -/* -*-mode: c; tab-width: 8; indent-tabs-mode: t; c-basic-offset: 8 -*-*/ /* -** Copyright (C) 2008-2013 Dirk-Jan C. Binnema +** Copyright (C) 2008-2020 Dirk-Jan C. Binnema ** ** This program is free software; you can redistribute it and/or modify it ** under the terms of the GNU General Public License as published by the @@ -29,7 +28,7 @@ #include -#include "test-mu-common.h" +#include "test-mu-common.hh" #include "mu-msg.h" #include "utils/mu-str.h" @@ -250,8 +249,8 @@ test_mu_msg_multimime (void) g_assert_cmpstr (mu_msg_get_body_text(msg, MU_MSG_OPTION_NONE), ==, "abcdef"); g_assert_cmpuint (mu_msg_get_flags(msg), - ==, MU_FLAG_FLAGGED | MU_FLAG_SEEN | - MU_FLAG_HAS_ATTACH); + ==, (MuFlags)(MU_FLAG_FLAGGED | MU_FLAG_SEEN | + MU_FLAG_HAS_ATTACH)); mu_msg_unref (msg); } @@ -267,8 +266,8 @@ test_mu_msg_flags (void) MuFlags flags; } msgflags [] = { { MU_TESTMAILDIR4 "/multimime!2,FS", - MU_FLAG_FLAGGED | MU_FLAG_SEEN | - MU_FLAG_HAS_ATTACH }, + (MuFlags)(MU_FLAG_FLAGGED | MU_FLAG_SEEN | + MU_FLAG_HAS_ATTACH) }, { MU_TESTMAILDIR4 "/special!2,Sabc", MU_FLAG_SEEN } @@ -515,7 +514,7 @@ test_mu_str_prio_02 (void) { /* this must fail */ g_test_log_set_fatal_handler ((GTestLogFatalFunc)ignore_error, NULL); - g_assert_cmpstr (mu_msg_prio_name(666), ==, NULL); + g_assert_cmpstr (mu_msg_prio_name((MuMsgPrio)666), ==, NULL); } @@ -588,8 +587,8 @@ main (int argc, char *argv[]) g_log_set_handler (NULL, - G_LOG_LEVEL_MASK | G_LOG_FLAG_FATAL| - G_LOG_FLAG_RECURSION, + (GLogLevelFlags)(G_LOG_LEVEL_MASK | G_LOG_FLAG_FATAL| + G_LOG_FLAG_RECURSION), (GLogFunc)black_hole, NULL); rv = g_test_run (); diff --git a/lib/test-mu-parser b/lib/test-mu-parser new file mode 100755 index 00000000..aa06cd04 Binary files /dev/null and b/lib/test-mu-parser differ diff --git a/lib/test-mu-store.cc b/lib/test-mu-store.cc index 9b1a1d24..a96dc7c1 100644 --- a/lib/test-mu-store.cc +++ b/lib/test-mu-store.cc @@ -26,7 +26,7 @@ #include -#include "test-mu-common.h" +#include "test-mu-common.hh" #include "mu-store.hh" static std::string MuTestMaildir = Mu::canonicalize_filename(MU_TESTMAILDIR, "/"); diff --git a/lib/query/test-parser.cc b/lib/test-parser.cc similarity index 97% rename from lib/query/test-parser.cc rename to lib/test-parser.cc index 2eadda96..1d0a5677 100644 --- a/lib/query/test-parser.cc +++ b/lib/test-parser.cc @@ -37,10 +37,12 @@ using CaseVec = std::vector; static void test_cases(const CaseVec& cases) { + Parser parser; + for (const auto& casus : cases ) { WarningVec warnings; - const auto tree = parse (casus.expr, warnings); + const auto tree = parser.parse (casus.expr, warnings); std::stringstream ss; ss << tree; diff --git a/lib/query/test-tokenizer.cc b/lib/test-tokenizer.cc similarity index 100% rename from lib/query/test-tokenizer.cc rename to lib/test-tokenizer.cc diff --git a/lib/query/tokenize.cc b/lib/tokenize.cc similarity index 100% rename from lib/query/tokenize.cc rename to lib/tokenize.cc