message/field: cache the message's sexp

Keep it in the store; much faster than generating on the fly.

lib/message/mu-fields.cc

@@ -142,7 +142,7 @@ static void
 test_prefix()
 {
 	static_assert(field_from_id(Field::Id::Subject).xapian_prefix() == 'S');
-	static_assert(field_from_id(Field::Id::BodyHtml).xapian_prefix() == 0);
+	static_assert(field_from_id(Field::Id::XBodyHtml).xapian_prefix() == 0);
 }
 
 [[maybe_unused]]

lib/message/mu-fields.hh

@@ -40,7 +40,6 @@ struct Field {
 	 */
 	enum struct Id {
 		Bcc = 0,	/**< Blind Carbon-Copy */
-		BodyHtml,	/**< HTML Body */
 		BodyText,	/**< Text body */
 		Cc,		/**< Carbon-Copy */
 		Date,		/**< Message date */
@@ -64,6 +63,9 @@ struct Field {
 		/*
 		 * <private>
 		 */
+		XBodyHtml,	/**< HTML Body */
+		XCachedSexp,    /**< Cached message s-expression */
+
 		_count_ /**< Number of FieldIds */
 	};
 
@@ -214,15 +216,6 @@ static constexpr std::array<Field, Field::id_size()>
 		Field::Flag::Contact |
 		Field::Flag::Value
 	    },
-	    {
-		Field::Id::BodyHtml,
-		Field::Type::String,
-		"body",
-		"Message html body",
-		{},
-		{},
-		Field::Flag::Internal
-	    },
 	    {
 		Field::Id::BodyText,
 		Field::Type::String,
@@ -314,7 +307,7 @@ static constexpr std::array<Field, Field::id_size()>
 		Field::Id::MessageId,
 		Field::Type::String,
 		"msgid",
-		"Attachment MIME-type",
+		"Message-Id",
 		"msgid:abc@123",
 		'i',
 		Field::Flag::BooleanTerm |
@@ -418,6 +411,30 @@ static constexpr std::array<Field, Field::id_size()>
 		Field::Flag::Contact |
 		Field::Flag::Value
 	    },
+
+
+
+	    /* internal */
+	    {
+		Field::Id::XBodyHtml,
+		Field::Type::String,
+		"htmlbody",
+		"Message html body",
+		{},
+		{},
+		Field::Flag::Internal
+	    },
+
+	    {
+		Field::Id::XCachedSexp,
+		Field::Type::String,
+		"sexp",
+		"Cached message s-expression",
+		{},
+		{},
+		Field::Flag::Internal |
+		Field::Flag::Value
+	    },
 	}};
 
 /*

lib/message/mu-message-sexp.cc

@@ -140,13 +140,10 @@ add_date_and_size(Sexp::List& items, const Message& message)
 }
 
 Mu::Sexp::List
-Message::to_sexp_list(unsigned docid) const
+Message::to_sexp_list() const
 {
 	Sexp::List items;
 
-	if (docid != 0)
-		items.add_prop(":docid", Sexp::make_number(docid));
-
 	add_prop_nonempty(items, ":subject", subject());
 	add_prop_nonempty(items, ":message-id", message_id());
 	add_prop_nonempty(items, ":mailing-list", mailing_list());
@@ -169,7 +166,7 @@ Message::to_sexp_list(unsigned docid) const
 }
 
 Mu::Sexp
-Message::to_sexp(unsigned docid) const
+Message::to_sexp() const
 {
 	return Sexp::make_list(to_sexp_list());
 }

lib/message/mu-message.cc

@@ -134,6 +134,9 @@ Message::Message(const std::string& text, const std::string& path,
 		priv_->mime_msg = std::move(msg.value());
 
 	fill_document(*priv_);
+
+	/* cache the sexp */
+	priv_->doc.add(Field::Id::XCachedSexp, to_sexp().to_sexp_string());
 }
 
 
@@ -166,6 +169,13 @@ Message::document() const
 	return priv_->doc;
 }
 
+
+void
+Message::update_cached_sexp()
+{
+	priv_->doc.add(Field::Id::XCachedSexp, to_sexp().to_sexp_string());
+}
+
 Result<void>
 Message::set_maildir(const std::string& maildir)
 {
@@ -175,7 +185,7 @@ Message::set_maildir(const std::string& maildir)
 	    maildir.at(0) != '/' ||
 	    (maildir.size() > 1 && maildir.at(maildir.length()-1) == '/'))
 		return Err(Error::Code::Message,
-"'%s' is not a valid maildir", maildir.c_str());
+			   "'%s' is not a valid maildir", maildir.c_str());
 
 	const auto path{document().string_value(Field::Id::Path)};
 	if (path == maildir || path.find(maildir) == std::string::npos)
@@ -554,19 +564,15 @@ fill_document(Message::Private& priv)
 		/* insist on expliclity handling each */
 #pragma GCC diagnostic push
 #pragma GCC diagnostic error "-Wswitch"
-		using AddrType = MimeMessage::AddressType;
 		switch(field.id) {
 		case Field::Id::Bcc:
-			doc.add(field.id, mime_msg.addresses(AddrType::Bcc));
+			doc.add(field.id, mime_msg.contacts(Contact::Type::Bcc));
-			break;
-		case Field::Id::BodyHtml:
-			doc.add(field.id, priv.body_html);
 			break;
 		case Field::Id::BodyText:
 			doc.add(field.id, priv.body_txt);
 			break;
 		case Field::Id::Cc:
-			doc.add(field.id, mime_msg.addresses(AddrType::Cc));
+			doc.add(field.id, mime_msg.contacts(Contact::Type::Cc));
 			break;
 		case Field::Id::Date:
 			doc.add(field.id, mime_msg.date());
@@ -582,7 +588,7 @@ fill_document(Message::Private& priv)
 			doc.add(priv.flags);
 			break;
 		case Field::Id::From:
-			doc.add(field.id, mime_msg.addresses(AddrType::From));
+			doc.add(field.id, mime_msg.contacts(Contact::Type::From));
 			break;
 		case Field::Id::Maildir: /* already */
 			break;
@@ -622,8 +628,16 @@ fill_document(Message::Private& priv)
 			doc.add(field.id, refs.empty() ? message_id : refs.at(0));
 			break;
 		case Field::Id::To:
-			doc.add(field.id, mime_msg.addresses(AddrType::To));
+			doc.add(field.id, mime_msg.contacts(Contact::Type::To));
 			break;
+
+			/* internal fields */
+		case Field::Id::XBodyHtml:
+			doc.add(field.id, priv.body_html);
+			break;
+		case Field::Id::XCachedSexp:
+			break;
+			/* ignore */
 		case Field::Id::_count_:
 			break;
 		}
@@ -667,18 +681,7 @@ Message::all_contacts() const
 	if (!load_mime_message())
 		return contacts; /* empty */
 
-	for (auto&& ctype: {
+	return priv_->mime_msg->contacts(Contact::Type::None); /* get all types */
-			MimeMessage::AddressType::Sender,
-			MimeMessage::AddressType::From,
-			MimeMessage::AddressType::ReplyTo,
-			MimeMessage::AddressType::To,
-			MimeMessage::AddressType::Cc,
-			MimeMessage::AddressType::Bcc}) {
-		auto addrs{priv_->mime_msg->addresses(ctype)};
-		std::move(addrs.begin(), addrs.end(), std::back_inserter(contacts));
-	}
-
-	return contacts;
 }
 
 const std::vector<Message::Part>&
@@ -714,9 +717,10 @@ Message::update_after_move(const std::string& new_path,
 	priv_->doc.add(Field::Id::Modified, statbuf->st_mtime);
 	priv_->doc.add(new_flags);
 
-
 	if (const auto res = set_maildir(new_maildir); !res)
 		return res;
 
+	priv_->doc.add(Field::Id::XCachedSexp, to_sexp().to_sexp_string());
+
 	return Ok();
 }

lib/message/mu-message.hh

@@ -177,12 +177,11 @@ public:
 	Contacts bcc() const { return document().contacts_value(Field::Id::Bcc); }
 
 	/**
-	 * Get the maildir this message lives in; i.e., if the path is
+	 * Get the maildir this message resides in; i.e., if the path is
 	 * ~/Maildir/foo/bar/cur/msg, the maildir would typically be foo/bar
 	 *
-	 * Note that that only messages that live in the store (i.e., are
+	 * This is determined when _storing_ the message (which uses
-	 * constructed use make_from_document() have a non-empty value for
+	 * set_maildir())
-	 * this.) until set_maildir() is used.
 	 *
 	 * @return the maildir requested or empty */
 	std::string maildir() const  { return document().string_value(Field::Id::Maildir); }
@@ -245,7 +244,6 @@ public:
 		return static_cast<::time_t>(document().integer_value(Field::Id::Modified));
 	}
 
-
 	/**
 	 * get the flags for this message.
 	 *
@@ -291,6 +289,16 @@ public:
 	std::vector<std::string> tags() const {
 		return document().string_vec_value(Field::Id::Tags);
 	}
+
+	/**
+	 * Get the cached s-expression for this message, or {} if not available.
+	 *
+	 * @return sexp or empty.
+	 */
+	std::string cached_sexp() const {
+		return document().string_value(Field::Id::XCachedSexp);
+	}
+
 	/*
 	 * Convert to Sexp
 	 */
@@ -299,11 +307,19 @@ public:
 	 * convert the message to a Lisp symbolic expression (for further
 	 * processing in e.g. emacs)
 	 *
-	 *
 	 * @return a Mu::Sexp or a Mu::Sexp::List representing the message.
 	 */
-	Mu::Sexp::List to_sexp_list(unsigned docid=0) const;
+	Mu::Sexp::List to_sexp_list() const;
-	Mu::Sexp       to_sexp(unsigned docid=0) const;
+	Mu::Sexp       to_sexp() const;
+
+
+	/**
+	 * Update the cached sexp for this message which is stored in the
+	 * document.This should be done when the document is complete,
+	 * i.e., immediately before storing it in the database.
+	 *
+	 */
+	void update_cached_sexp();
 
 	/*
 	 * And some non-const message, for updating an existing

lib/mu-store.cc

@@ -344,6 +344,9 @@ Store::add_message(Message& msg, bool use_transaction)
 
 	if (auto&& res = msg.set_maildir(mdir.value()); !res)
 		return Err(res.error());
+	/* now, we're done with all the fields; generate the sexp string for this
+	 * message */
+	msg.update_cached_sexp();
 
 	std::lock_guard guard{priv_->lock_};
 
@@ -374,8 +377,12 @@ Store::add_message(Message& msg, bool use_transaction)
 
 
 bool
-Store::update_message(const Message& msg, unsigned docid)
+Store::update_message(Message& msg, unsigned docid)
 {
+	msg.update_cached_sexp();
+
+	std::lock_guard guard{priv_->lock_};
+
 	return xapian_try(
 		[&]{
 			priv_->writable_db().replace_document(

lib/mu-store.hh

@@ -207,7 +207,7 @@ public:
 	 *
 	 * @return false in case of failure; true otherwise.
 	 */
-	bool update_message(const Message& msg, Id id);
+	bool update_message(Message& msg, Id id);
 
 	/**
 	 * Remove a message from the store. It will _not_ remove the message