emacs
/
mu
mirror of https://github.com/djcb/mu.git
1
0
Fork 0
Code Issues Releases Wiki Activity
mu/lib/mu-store-write.cc

808 lines
20 KiB
C++
Raw Blame History

/* -*-mode: c++; tab-width: 8; indent-tabs-mode: t; c-basic-offset: 8-*- */
/*
** Copyright (C) 2008-2017 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl>
**
** This program is free software; you can redistribute it and/or modify it
** under the terms of the GNU General Public License as published by the
** Free Software Foundation; either version 3, or (at your option) any
** later version.
**
** This program is distributed in the hope that it will be useful,
** but WITHOUT ANY WARRANTY; without even the implied warranty of
** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
** GNU General Public License for more details.
**
** You should have received a copy of the GNU General Public License
** along with this program; if not, write to the Free Software Foundation,
** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
**
*/
#if HAVE_CONFIG_H
#include "config.h"
#endif /*HAVE_CONFIG_H*/
#include <cstdio>
#include <xapian.h>
#include <cstring>
#include <stdexcept>
#include <iostream>
#include <parser/utils.hh>
#include "mu-store.h"
#include "mu-store-priv.hh" /* _MuStore */
#include "mu-msg.h"
#include "mu-msg-part.h"
#include "mu-store.h"
#include "mu-util.h"
#include "mu-str.h"
#include "mu-date.h"
#include "mu-flags.h"
#include "mu-contacts.hh"
void
_MuStore::begin_transaction ()
{
try {
db_writable()->begin_transaction();
in_transaction (true);
} MU_XAPIAN_CATCH_BLOCK;
}
void
_MuStore::commit_transaction () {
try {
in_transaction (false);
db_writable()->commit_transaction();
} MU_XAPIAN_CATCH_BLOCK;
}
void
_MuStore::rollback_transaction () {
try {
in_transaction (false);
db_writable()->cancel_transaction();
} MU_XAPIAN_CATCH_BLOCK;
}
/* we cache these prefix strings, so we don't have to allocate them all
* the time; this should save 10-20 string allocs per message */
G_GNUC_CONST static const std::string&
prefix (MuMsgFieldId mfid)
{
static std::string fields[MU_MSG_FIELD_ID_NUM];
static bool initialized = false;
if (G_UNLIKELY(!initialized)) {
for (int i = 0; i != MU_MSG_FIELD_ID_NUM; ++i)
fields[i] = std::string (1, mu_msg_field_xapian_prefix
((MuMsgFieldId)i));
initialized = true;
}
return fields[mfid];
}
static void
add_synonym_for_flag (MuFlags flag, Xapian::WritableDatabase *db)
{
static const std::string pfx(prefix(MU_MSG_FIELD_ID_FLAGS));
db->clear_synonyms (pfx + mu_flag_name (flag));
db->add_synonym (pfx + mu_flag_name (flag), pfx +
(std::string(1, (char)(tolower(mu_flag_char(flag))))));
}
static void
add_synonym_for_prio (MuMsgPrio prio, Xapian::WritableDatabase *db)
{
static const std::string pfx (prefix(MU_MSG_FIELD_ID_PRIO));
std::string s1 (pfx + mu_msg_prio_name (prio));
std::string s2 (pfx + (std::string(1, mu_msg_prio_char (prio))));
db->clear_synonyms (s1);
db->clear_synonyms (s2);
db->add_synonym (s1, s2);
}
static void
add_synonyms (MuStore *store)
{
mu_flags_foreach ((MuFlagsForeachFunc)add_synonym_for_flag,
store->db_writable());
mu_msg_prio_foreach ((MuMsgPrioForeachFunc)add_synonym_for_prio,
store->db_writable());
}
MuStore*
mu_store_new_writable (const char* xpath, const char *contacts_cache,
gboolean rebuild, GError **err)
{
g_return_val_if_fail (xpath, NULL);
try {
try {
MuStore *store;
store = new _MuStore (xpath, rebuild ? true : false);
add_synonyms (store);
return store;
} MU_STORE_CATCH_BLOCK_RETURN(err,NULL);
} MU_XAPIAN_CATCH_BLOCK_G_ERROR_RETURN (err, MU_ERROR_XAPIAN, NULL);
}
void
mu_store_set_batch_size (MuStore *store, guint batchsize)
{
g_return_if_fail (store);
store->set_batch_size (batchsize);
}
gboolean
mu_store_set_metadata (MuStore *store, const char *key, const char *val,
GError **err)
{
g_return_val_if_fail (store, FALSE);
g_return_val_if_fail (key, FALSE);
g_return_val_if_fail (val, FALSE);
try {
try {
store->db_writable()->set_metadata (key, val);
return TRUE;
} MU_STORE_CATCH_BLOCK_RETURN(err, FALSE);
} MU_XAPIAN_CATCH_BLOCK_G_ERROR_RETURN(err, MU_ERROR_XAPIAN, FALSE);
}
gboolean
mu_store_clear (MuStore *store, GError **err)
{
g_return_val_if_fail (store, FALSE);
try {
try {
store->clear();
return TRUE;
} MU_STORE_CATCH_BLOCK_RETURN(err, FALSE);
} MU_XAPIAN_CATCH_BLOCK_RETURN(FALSE);
}
void
mu_store_flush (MuStore *store) try {
g_return_if_fail (store);
if (store->in_transaction())
store->commit_transaction ();
store->db_writable()->commit ();
if (store->contacts())
store->db_writable()->set_metadata(MU_CONTACTS_CACHE,
store->contacts()->serialize());
} MU_XAPIAN_CATCH_BLOCK;
static void
add_terms_values_date (Xapian::Document& doc, MuMsg *msg, MuMsgFieldId mfid)
{
const auto dstr = Mux::date_to_time_t_string (
(time_t)mu_msg_get_field_numeric (msg, mfid));
doc.add_value ((Xapian::valueno)mfid, dstr);
}
static void
add_terms_values_size (Xapian::Document& doc, MuMsg *msg, MuMsgFieldId mfid)
{
const auto szstr =
Mux::size_to_string (mu_msg_get_field_numeric (msg, mfid));
doc.add_value ((Xapian::valueno)mfid, szstr);
}
G_GNUC_CONST
static const std::string&
flag_val (char flagchar)
{
static const std::string
pfx (prefix(MU_MSG_FIELD_ID_FLAGS)),
draftstr (pfx + (char)tolower(mu_flag_char(MU_FLAG_DRAFT))),
flaggedstr (pfx + (char)tolower(mu_flag_char(MU_FLAG_FLAGGED))),
passedstr (pfx + (char)tolower(mu_flag_char(MU_FLAG_PASSED))),
repliedstr (pfx + (char)tolower(mu_flag_char(MU_FLAG_REPLIED))),
seenstr (pfx + (char)tolower(mu_flag_char(MU_FLAG_SEEN))),
trashedstr (pfx + (char)tolower(mu_flag_char(MU_FLAG_TRASHED))),
newstr (pfx + (char)tolower(mu_flag_char(MU_FLAG_NEW))),
signedstr (pfx + (char)tolower(mu_flag_char(MU_FLAG_SIGNED))),
cryptstr (pfx + (char)tolower(mu_flag_char(MU_FLAG_ENCRYPTED))),
attachstr (pfx + (char)tolower(mu_flag_char(MU_FLAG_HAS_ATTACH))),
unreadstr (pfx + (char)tolower(mu_flag_char(MU_FLAG_UNREAD))),
liststr (pfx + (char)tolower(mu_flag_char(MU_FLAG_LIST)));
switch (flagchar) {
case 'D': return draftstr;
case 'F': return flaggedstr;
case 'P': return passedstr;
case 'R': return repliedstr;
case 'S': return seenstr;
case 'T': return trashedstr;
case 'N': return newstr;
case 'z': return signedstr;
case 'x': return cryptstr;
case 'a': return attachstr;
case 'l': return liststr;
case 'u': return unreadstr;
default:
g_return_val_if_reached (flaggedstr);
return flaggedstr;
}
}
/* pre-calculate; optimization */
G_GNUC_CONST static const std::string&
prio_val (MuMsgPrio prio)
{
static const std::string pfx (prefix(MU_MSG_FIELD_ID_PRIO));
static const std::string
low (pfx + std::string(1, mu_msg_prio_char(MU_MSG_PRIO_LOW))),
norm (pfx + std::string(1, mu_msg_prio_char(MU_MSG_PRIO_NORMAL))),
high (pfx + std::string(1, mu_msg_prio_char(MU_MSG_PRIO_HIGH)));
switch (prio) {
case MU_MSG_PRIO_LOW: return low;
case MU_MSG_PRIO_NORMAL: return norm;
case MU_MSG_PRIO_HIGH: return high;
default:
g_return_val_if_reached (norm);
return norm;
}
}
static void // add term, truncate if needed.
add_term (Xapian::Document& doc, const std::string& term)
{
if (term.length() < MU_STORE_MAX_TERM_LENGTH)
doc.add_term(term);
else
doc.add_term(term.substr(0, MU_STORE_MAX_TERM_LENGTH));
}
static void
add_terms_values_number (Xapian::Document& doc, MuMsg *msg, MuMsgFieldId mfid)
{
gint64 num = mu_msg_get_field_numeric (msg, mfid);
const std::string numstr (Xapian::sortable_serialise((double)num));
doc.add_value ((Xapian::valueno)mfid, numstr);
if (mfid == MU_MSG_FIELD_ID_FLAGS) {
const char *cur = mu_flags_to_str_s
((MuFlags)num,(MuFlagType)MU_FLAG_TYPE_ANY);
g_return_if_fail (cur);
while (*cur) {
add_term (doc, flag_val(*cur));
++cur;
}
} else if (mfid == MU_MSG_FIELD_ID_PRIO)
add_term (doc, prio_val((MuMsgPrio)num));
}
/* for string and string-list */
static void
add_terms_values_str (Xapian::Document& doc, const char *val, MuMsgFieldId mfid)
{
const auto flat = Mux::utf8_flatten (val);
if (mu_msg_field_xapian_index (mfid)) {
Xapian::TermGenerator termgen;
termgen.set_document (doc);
termgen.index_text (flat, 1, prefix(mfid));
}
if (mu_msg_field_xapian_term(mfid))
add_term(doc, prefix(mfid) + flat);
}
static void
add_terms_values_string (Xapian::Document& doc, MuMsg *msg, MuMsgFieldId mfid)
{
const char *orig;
if (!(orig = mu_msg_get_field_string (msg, mfid)))
return; /* nothing to do */
/* the value is what we display in search results; the
* unchanged original */
if (mu_msg_field_xapian_value(mfid))
doc.add_value ((Xapian::valueno)mfid, orig);
add_terms_values_str (doc, orig, mfid);
}
static void
add_terms_values_string_list (Xapian::Document& doc, MuMsg *msg,
MuMsgFieldId mfid)
{
const GSList *lst;
lst = mu_msg_get_field_string_list (msg, mfid);
if (!lst)
return;
if (mu_msg_field_xapian_value (mfid)) {
gchar *str;
str = mu_str_from_list (lst, ',');
if (str)
doc.add_value ((Xapian::valueno)mfid, str);
g_free (str);
}
if (mu_msg_field_xapian_term (mfid)) {
for (; lst; lst = g_slist_next ((GSList*)lst))
add_terms_values_str (doc, (const gchar*)lst->data,
mfid);
}
}
struct PartData {
PartData (Xapian::Document& doc, MuMsgFieldId mfid):
_doc (doc), _mfid(mfid) {}
Xapian::Document _doc;
MuMsgFieldId _mfid;
};
/* index non-body text parts */
static void
maybe_index_text_part (MuMsg *msg, MuMsgPart *part, PartData *pdata)
{
char *txt;
Xapian::TermGenerator termgen;
/* only deal with attachments/messages; inlines are indexed as
* body parts */
if (!(part->part_type & MU_MSG_PART_TYPE_ATTACHMENT) &&
!(part->part_type & MU_MSG_PART_TYPE_MESSAGE))
return;
txt = mu_msg_part_get_text (msg, part, MU_MSG_OPTION_NONE);
if (!txt)
return;
termgen.set_document(pdata->_doc);
const auto str = Mux::utf8_flatten (txt);
g_free (txt);
termgen.index_text (str, 1, prefix(MU_MSG_FIELD_ID_EMBEDDED_TEXT));
}
static void
each_part (MuMsg *msg, MuMsgPart *part, PartData *pdata)
{
char *fname;
static const std::string
file (prefix(MU_MSG_FIELD_ID_FILE)),
mime (prefix(MU_MSG_FIELD_ID_MIME));
/* save the mime type of any part */
if (part->type) {
char ctype[MU_STORE_MAX_TERM_LENGTH + 1];
snprintf(ctype, sizeof(ctype), "%s/%s", part->type, part->subtype);
add_term(pdata->_doc, mime + ctype);
}
if ((fname = mu_msg_part_get_filename (part, FALSE))) {
const auto flat = Mux::utf8_flatten (fname);
g_free (fname);
add_term(pdata->_doc, file + flat);
}
maybe_index_text_part (msg, part, pdata);
}
static void
add_terms_values_attach (Xapian::Document& doc, MuMsg *msg,
MuMsgFieldId mfid)
{
PartData pdata (doc, mfid);
mu_msg_part_foreach (msg, MU_MSG_OPTION_RECURSE_RFC822,
(MuMsgPartForeachFunc)each_part, &pdata);
}
static void
add_terms_values_body (Xapian::Document& doc, MuMsg *msg,
MuMsgFieldId mfid)
{
if (mu_msg_get_flags(msg) & MU_FLAG_ENCRYPTED)
return; /* ignore encrypted bodies */
auto str = mu_msg_get_body_text (msg, MU_MSG_OPTION_NONE);
if (!str) /* FIXME: html->txt fallback needed */
str = mu_msg_get_body_html (msg, MU_MSG_OPTION_NONE);
if (!str)
return; /* no body... */
Xapian::TermGenerator termgen;
termgen.set_document(doc);
const auto flat = Mux::utf8_flatten(str);
termgen.index_text (flat, 1, prefix(mfid));
}
struct _MsgDoc {
Xapian::Document *_doc;
MuMsg *_msg;
MuStore *_store;
/* callback data, to determine whether this message is 'personal' */
gboolean _personal;
GSList *_my_addresses;
};
typedef struct _MsgDoc MsgDoc;
static void
add_terms_values_default (MuMsgFieldId mfid, MsgDoc *msgdoc)
{
if (mu_msg_field_is_numeric (mfid))
add_terms_values_number
(*msgdoc->_doc, msgdoc->_msg, mfid);
else if (mu_msg_field_is_string (mfid))
add_terms_values_string
(*msgdoc->_doc, msgdoc->_msg, mfid);
else if (mu_msg_field_is_string_list(mfid))
add_terms_values_string_list
(*msgdoc->_doc, msgdoc->_msg, mfid);
else
g_return_if_reached ();
}
static void
add_terms_values (MuMsgFieldId mfid, MsgDoc* msgdoc)
{
/* note: contact-stuff (To/Cc/From) will handled in
* each_contact_info, not here */
if (!mu_msg_field_xapian_index(mfid) &&
!mu_msg_field_xapian_term(mfid) &&
!mu_msg_field_xapian_value(mfid))
return;
switch (mfid) {
case MU_MSG_FIELD_ID_DATE:
add_terms_values_date (*msgdoc->_doc, msgdoc->_msg, mfid);
break;
case MU_MSG_FIELD_ID_SIZE:
add_terms_values_size (*msgdoc->_doc, msgdoc->_msg, mfid);
break;
case MU_MSG_FIELD_ID_BODY_TEXT:
add_terms_values_body (*msgdoc->_doc, msgdoc->_msg, mfid);
break;
/* note: add_terms_values_attach handles _FILE, _MIME and
* _ATTACH_TEXT msgfields */
case MU_MSG_FIELD_ID_FILE:
add_terms_values_attach (*msgdoc->_doc, msgdoc->_msg, mfid);
break;
case MU_MSG_FIELD_ID_MIME:
case MU_MSG_FIELD_ID_EMBEDDED_TEXT:
break;
case MU_MSG_FIELD_ID_THREAD_ID:
case MU_MSG_FIELD_ID_UID:
break; /* already taken care of elsewhere */
default:
return add_terms_values_default (mfid, msgdoc);
}
}
static const std::string&
xapian_pfx (MuMsgContact *contact)
{
static const std::string empty;
/* use ptr to string to prevent copy... */
switch (contact->type) {
case MU_MSG_CONTACT_TYPE_TO:
return prefix(MU_MSG_FIELD_ID_TO);
case MU_MSG_CONTACT_TYPE_FROM:
return prefix(MU_MSG_FIELD_ID_FROM);
case MU_MSG_CONTACT_TYPE_CC:
return prefix(MU_MSG_FIELD_ID_CC);
case MU_MSG_CONTACT_TYPE_BCC:
return prefix(MU_MSG_FIELD_ID_BCC);
default:
g_warning ("unsupported contact type %u",
(unsigned)contact->type);
return empty;
}
}
static void
add_address_subfields (Xapian::Document& doc, const char *addr,
const std::string& pfx)
{
const char *at, *domain_part;
char *name_part;
/* add "foo" and "bar.com" as terms as well for
* "foo@bar.com" */
if (G_UNLIKELY(!(at = (g_strstr_len (addr, -1, "@")))))
return;
name_part = g_strndup(addr, at - addr); // foo
domain_part = at + 1;
add_term(doc, pfx + name_part);
add_term(doc, pfx + domain_part);
g_free (name_part);
}
static gboolean
each_contact_info (MuMsgContact *contact, MsgDoc *msgdoc)
{
/* for now, don't store reply-to addresses */
if (mu_msg_contact_type (contact) == MU_MSG_CONTACT_TYPE_REPLY_TO)
return TRUE;
const std::string pfx (xapian_pfx(contact));
if (pfx.empty())
return TRUE; /* unsupported contact type */
if (!mu_str_is_empty(contact->name)) {
Xapian::TermGenerator termgen;
termgen.set_document (*msgdoc->_doc);
const auto flat = Mux::utf8_flatten(contact->name);
termgen.index_text (flat, 1, pfx);
}
if (!mu_str_is_empty(contact->email)) {
const auto flat = Mux::utf8_flatten(contact->email);
add_term(*msgdoc->_doc, pfx + flat);
add_address_subfields (*msgdoc->_doc, contact->email, pfx);
/* store it also in our contacts cache */
auto contacts = msgdoc->_store->contacts();
if (contacts)
contacts->add(Mu::ContactInfo(contact->full_address,
contact->email,
contact->name ? contact->name : "",
msgdoc->_personal,
mu_msg_get_date(msgdoc->_msg)));
}
return TRUE;
}
static gboolean
each_contact_check_if_personal (MuMsgContact *contact, MsgDoc *msgdoc)
{
GSList *cur;
if (msgdoc->_personal || !contact->email)
return TRUE;
for (cur = msgdoc->_my_addresses; cur; cur = g_slist_next (cur)) {
if (g_ascii_strcasecmp (
contact->email, (const char*)cur->data) == 0) {
msgdoc->_personal = TRUE;
break;
}
}
return TRUE;
}
static Xapian::Document
new_doc_from_message (MuStore *store, MuMsg *msg)
{
Xapian::Document doc;
MsgDoc docinfo = {&doc, msg, store, 0, FALSE};
mu_msg_field_foreach ((MuMsgFieldForeachFunc)add_terms_values, &docinfo);
/* determine whether this is 'personal' email, ie. one of my
* e-mail addresses is explicitly mentioned -- it's not a
* mailing list message. Callback will update docinfo->_personal */
if (store->my_addresses()) {
docinfo._my_addresses = store->my_addresses();
mu_msg_contact_foreach
(msg,
(MuMsgContactForeachFunc)each_contact_check_if_personal,
&docinfo);
}
/* also store the contact-info as separate terms, and add it
* to the cache */
mu_msg_contact_foreach (msg, (MuMsgContactForeachFunc)each_contact_info,
&docinfo);
// g_printerr ("\n--%s\n--\n", doc.serialise().c_str());
return doc;
}
static void
update_threading_info (Xapian::WritableDatabase* db,
MuMsg *msg, Xapian::Document& doc)
{
const GSList *refs;
// refs contains a list of parent messages, with the oldest
// one first until the last one, which is the direct parent of
// the current message. of course, it may be empty.
//
// NOTE: there may be cases where the list is truncated; we happily
// ignore that case.
refs = mu_msg_get_references (msg);
std::string thread_id;
if (refs)
thread_id = mu_util_get_hash ((const char*)refs->data);
else
thread_id = mu_util_get_hash (mu_msg_get_msgid (msg));
add_term (doc, prefix(MU_MSG_FIELD_ID_THREAD_ID) + thread_id);
doc.add_value((Xapian::valueno)MU_MSG_FIELD_ID_THREAD_ID, thread_id);
}
static unsigned
add_or_update_msg (MuStore *store, unsigned docid, MuMsg *msg, GError **err)
{
g_return_val_if_fail (store, MU_STORE_INVALID_DOCID);
g_return_val_if_fail (msg, MU_STORE_INVALID_DOCID);
try {
Xapian::docid id;
Xapian::Document doc (new_doc_from_message(store, msg));
const std::string term (store->get_uid_term (mu_msg_get_path(msg)));
if (!store->in_transaction())
store->begin_transaction();
add_term (doc, term);
// update the threading info if this message has a message id
if (mu_msg_get_msgid (msg))
update_threading_info (store->db_writable(), msg, doc);
if (docid == 0)
id = store->db_writable()->replace_document (term, doc);
else {
store->db_writable()->replace_document (docid, doc);
id = docid;
}
if (store->inc_processed() % store->batch_size() == 0)
store->commit_transaction();
return id;
} MU_XAPIAN_CATCH_BLOCK_G_ERROR (err, MU_ERROR_XAPIAN_STORE_FAILED);
if (store->in_transaction())
store->rollback_transaction();
return MU_STORE_INVALID_DOCID;
}
unsigned
mu_store_add_msg (MuStore *store, MuMsg *msg, GError **err)
{
g_return_val_if_fail (store, MU_STORE_INVALID_DOCID);
g_return_val_if_fail (msg, MU_STORE_INVALID_DOCID);
return add_or_update_msg (store, 0, msg, err);
}
unsigned
mu_store_update_msg (MuStore *store, unsigned docid, MuMsg *msg, GError **err)
{
g_return_val_if_fail (store, MU_STORE_INVALID_DOCID);
g_return_val_if_fail (msg, MU_STORE_INVALID_DOCID);
g_return_val_if_fail (docid != 0, MU_STORE_INVALID_DOCID);
return add_or_update_msg (store, docid, msg, err);
}
unsigned
mu_store_add_path (MuStore *store, const char *path, const char *maildir,
GError **err)
{
MuMsg *msg;
unsigned docid;
g_return_val_if_fail (store, FALSE);
g_return_val_if_fail (path, FALSE);
msg = mu_msg_new_from_file (path, maildir, err);
if (!msg)
return MU_STORE_INVALID_DOCID;
docid = add_or_update_msg (store, 0, msg, err);
mu_msg_unref (msg);
return docid;
}
XapianWritableDatabase*
mu_store_get_writable_database (MuStore *store)
{
g_return_val_if_fail (store, NULL);
return (XapianWritableDatabase*)store->db_writable();
}
gboolean
mu_store_remove_path (MuStore *store, const char *msgpath)
{
g_return_val_if_fail (store, FALSE);
g_return_val_if_fail (msgpath, FALSE);
try {
const std::string term
(store->get_uid_term(msgpath));
store->db_writable()->delete_document (term);
store->inc_processed();
return TRUE;
} MU_XAPIAN_CATCH_BLOCK_RETURN (FALSE);
}
gboolean
mu_store_set_timestamp (MuStore *store, const char* msgpath,
time_t stamp, GError **err)
{
char buf[21];
g_return_val_if_fail (store, FALSE);
g_return_val_if_fail (msgpath, FALSE);
sprintf (buf, "%" G_GUINT64_FORMAT, (guint64)stamp);
return mu_store_set_metadata (store, msgpath, buf, err);
}
Reference in New Issue View Git Blame Copy Permalink