mu4e: Contact completion - better sorting / display

Improve the contact-sorting algorithm, and make it better cooperate with
completion-at-point functions.

Also deal better with broken rewritten contacts, and document how
rewriting should done (docstrings and reference doc).

mu4e/mu4e-compose.el

@@ -1,7 +1,7 @@
 ;; -*-mode: emacs-lisp; tab-width: 8; indent-tabs-mode: t -*-
 ;; mu4e-compose.el -- part of mu4e, the mu mail user agent for emacs
 ;;
-;; Copyright (C) 2011-2012 Dirk-Jan C. Binnema
+;; Copyright (C) 2011-2016 Dirk-Jan C. Binnema
 
 ;; Author: Dirk-Jan C. Binnema <djcb@djcbsoftware.nl>
 ;; Maintainer: Dirk-Jan C. Binnema <djcb@djcbsoftware.nl>
@@ -260,15 +260,15 @@ appear on disk."
 (defun mu4e~compose-complete-handler (str pred action)
   (cond
     ((eq action nil)
-      (try-completion str mu4e~contacts-for-completion pred))
+      (try-completion str mu4e~contacts pred))
     ((eq action t)
-      (all-completions str mu4e~contacts-for-completion pred))
+      (all-completions str mu4e~contacts pred))
     ((eq action 'metadata)
       ;; our contacts are already sorted - just need to tell the
       ;; completion machinery not to try to undo that...
       '(metadata
-	 (display-sort-function . identity) ;; i.e., alphabetically
+	 (display-sort-function . mu4e~sort-contacts-for-completion) 
-	 (cycle-sort-function   . identity)))))
+	 (cycle-sort-function   . mu4e~sort-contacts-for-completion)))))
 
 (defun mu4e~compose-complete-contact (&optional start)
   "Complete the text at START with a contact.
@@ -672,8 +672,6 @@ end of the buffer."
 (define-key mu4e-compose-mode-map
   (vector 'remap 'end-of-buffer) 'mu4e-compose-goto-bottom)
 
-
-
 (provide 'mu4e-compose)
 
 ;; Load mu4e completely even when this file was loaded through

mu4e/mu4e-utils.el

@@ -1,6 +1,6 @@
 ;;; mu4e-utils.el -- part of mu4e, the mu mail user agent
 ;;
-;; Copyright (C) 2011-2014 Dirk-Jan C. Binnema
+;; Copyright (C) 2011-2016 Dirk-Jan C. Binnema
 ;; Copyright (C) 2013 Tibor Simko
 
 ;; Author: Dirk-Jan C. Binnema <djcb@djcbsoftware.nl>
@@ -347,7 +347,7 @@ maildirs under `mu4e-maildir'."
 		(mapconcat
 		  (lambda (item)
 		    (concat
-		      "["
+q		      "["
 		      (propertize (make-string 1 (cdr item))
 			'face 'mu4e-highlight-face)
 		      "]"
@@ -657,53 +657,66 @@ or (rfc822-string . CONTACT) otherwise."
 	  (if name (format "%s <%s>" (mu4e~rfc822-quoteit name) mail) mail)
 	  contact)))))
 
-(defsubst mu4e~sort-contacts (contacts)
+(defun mu4e~sort-contacts (contacts)
-  "Destructively sort contacts (only for cycling). Sort by
+  "Destructively sort contacts (only for cycling) in order of
-last-use when that is at most 10 days old. Otherwise, sort by
+ 'mostly likely contact'.t See the code for the detail"
-frequency."
   (let* ((now (+ (float-time) 3600)) ;; allow for clock diffs
-	  (recent (- (float-time) (* 30 24 3600))))
+	  (recent (- (float-time) (* 15 24 3600))))
     (sort* contacts
       (lambda (c1 c2)
 	(let* ( (c1 (cdr c1)) (c2 (cdr c2))
 		(personal1 (plist-get c1 :personal))
 		(personal2 (plist-get c2 :personal))
-		(freq1 (plist-get c1 :freq))
+		;; note: freq, tstamp can only be missing if the rewrite
-		(freq2 (plist-get c2 :freq))
+		;; function removed them. If the rewrite function changed the
-		(tstamp1 (plist-get c1 :tstamp))
+		;; contact somehow, we guess it's important.
-		(tstamp2 (plist-get c2 :tstamp)))
+		(freq1 (or (plist-get c1 :freq) 500))
-	  ;; personal contacts come first
+		(freq2 (or (plist-get c2 :freq) 500))
-	  (if (or personal1 personal2)
+		(tstamp1 (or (plist-get c1 :tstamp) now))
-	    (if (not (and personal1 personal2))
+		(tstamp2 (or (plist-get c2 :tstamp) now)))
-	      ;; if only one is personal, that one comes first
+	  ;; only one is personal? if so, that one comes first
-	      (if personal1 t nil)
+	  (if (not (equal personal1 personal2))
-	      ;; then come recently seen ones; but only if they're not in
+	    (if personal1 t nil)
-	      ;; the future (as seen in spams)
+	    ;; only one is recent? that one comes first
-	      (if (and (<= tstamp1 now) (<= tstamp2 now)
+	    (if (not (equal (> tstamp1 recent) (> tstamp2 recent)))
-		    (or (> tstamp1 recent) (> tstamp2 recent)))
+	      (> tstamp1 tstamp2)
-		(> tstamp1 tstamp2) 
+	      ;; otherwise, use the frequency
-		;; otherwise, use the frequency
+	      (> freq1 freq2))))))))
-		(> freq1 freq2)))))))))
 
+(defun mu4e~sort-contacts-for-completion (contacts)
+  "Takes CONTACTS, which is a list of RFC-822 addresses, and sort them based
+on the ranking in `mu4e~contacts.'"
+  (sort* contacts
+    (lambda (c1 c2)
+      (let ((rank1 (gethash c1 mu4e~contacts))
+	     (rank2 (gethash c2 mu4e~contacts)))
+	(< rank1 rank2)))))
+ 
 ;; start and stopping
-(defun mu4e~fill-contacts (contacts)
+(defun mu4e~fill-contacts (contact-data)
   "We receive a list of contacts, which each contact of the form
-  (:name NAME :mail EMAIL :tstamp TIMESTAMP :freq FREQUENCY)
+  (:me NAME :mail EMAIL :tstamp TIMESTAMP :freq FREQUENCY)
-and fill the list `mu4e~contacts-for-completion' with it, with
+and fill the hash  `mu4e~contacts-for-completion' with it, with
-each element looking like
+each contact mapped to an integer for their ranking.
-  name <email>
-This is used by the completion function in mu4e-compose."
-  (setq mu4e~contacts-for-completion nil)
-  (dolist (contact contacts)
-    (let ((contact (mu4e~process-contact contact)))
-      ;; note, this gives cells (rfc822-address . contact)
-      (when contact (push contact mu4e~contacts-for-completion))))
-  (setq mu4e~contacts-for-completion
-    (mapcar 'car ;; strip off the other stuff again
-      (mu4e~sort-contacts mu4e~contacts-for-completion)))
-  (mu4e-index-message "Contacts received: %d"
-    (length mu4e~contacts-for-completion)))
 
+This is used by the completion function in mu4e-compose."
+  (let ((contacts) (rank 0))
+    (dolist (contact contact-data)
+      (let ((contact-maybe (mu4e~process-contact contact)))
+	;; note, this gives cells (rfc822-address . contact)
+	(when contact-maybe (push contact-maybe contacts))))
+    (setq contacts (mu4e~sort-contacts contacts))
+    ;; now, we have our nicely sorted list, map them to a list
+    ;; of increasing integers. We use that map in the composer
+    ;; to sort them there. It would have been so much easier if emacs
+    ;; allowed us to use the sorted-list as-is, but no such luck.
+    (setq mu4e~contacts (make-hash-table :test 'equal :weakness nil
+			  :size (length contacts)))
+    (dolist (contact contacts)
+      (puthash (car contact) rank mu4e~contacts)
+      (incf rank))
+    (mu4e-index-message "Contacts received: %d"
+      (hash-table-count mu4e~contacts))))
 
 (defun mu4e~check-requirements ()
   "Check for the settings required for running mu4e."
@@ -812,7 +825,7 @@ successful, call FUNC (if non-nil) afterwards."
   "Clear any cached resources."
   (setq
     mu4e-maildir-list nil
-    mu4e~contacts-for-completion nil)) 
+    mu4e~contacts nil)) 
  
 (defun mu4e~stop ()
   "Stop the mu4e session."
@@ -1194,6 +1207,5 @@ the view and compose modes."
 	  (add-text-properties p (point-max) '(face mu4e-footer-face)))))))
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
-
 (provide 'mu4e-utils)
 ;;; End of mu4e-utils.el

mu4e/mu4e-vars.el

@@ -1,6 +1,6 @@
 ;;; mu4e-vars.el -- part of mu4e, the mu mail user agent
 ;;
-;; Copyright (C) 2011-2015 Dirk-Jan C. Binnema
+;; Copyright (C) 2011-2016 Dirk-Jan C. Binnema
 
 ;; Author: Dirk-Jan C. Binnema <djcb@djcbsoftware.nl>
 ;; Maintainer: Dirk-Jan C. Binnema <djcb@djcbsoftware.nl>
@@ -312,13 +312,17 @@ their canonical counterpart; useful as an example."
 contacts and rewrite them or remove them altogether.
 
 If the function receives the contact as a list of the form
-     (:name NAME :mail EMAIL)
+     (:name NAME :mail EMAIL ... other properties ... )
 (other properties may be there as well)
 
 The function should return either:
- - nil: remove this contact
+ - nil: remove this contact, or
-- a possible rewritten cell (:name NAME :mail EMAIL), or simply return
+- the rewritten cell, or
-the functions parameter."
+- the existing cell as-is
+
+For rewriting, it is recommended to use `plist-put' to set the
+changed parameters, so the other properties stay in place. Those
+are needed for sorting the contacts."
   :type 'function
   :group 'mu4e-compose)
 
@@ -769,10 +773,11 @@ for an example.")
 (defvar mu4e~view-headers-buffer nil
   "The headers buffer connected to this view.")
 
-(defvar mu4e~contacts-for-completion nil
+(defvar mu4e~contacts nil
-  "List of contacts (ie. 'name <e-mail>').
+  "Hash of that maps contacts (ie. 'name <e-mail>') to an integer
-This is used by the completion functions in mu4e-compose, filled
+with their sort order. We need to keep this information around to
-when mu4e starts.")
+quickly re-sort subsets of the contacts in the completions function in
+mu4e-compose.")
 
 (defvar mu4e~server-props nil
   "Properties we receive from the mu4e server process.

mu4e/mu4e.texi

@@ -12,7 +12,7 @@
 @c %**end of header
 
 @copying
-Copyright @copyright{} 2012-2015 Dirk-Jan C. Binnema
+Copyright @copyright{} 2012-2016 Dirk-Jan C. Binnema
 
 @quotation
 Permission is granted to copy, distribute and/or modify this document
@@ -2885,7 +2885,11 @@ return either the possibly rewritten contact or @code{nil} to remove the
 contact from the list - note that the latter can also be achieved using
 @code{mu4e-compose-complete-ignore-address-regexp}.
 
-Let's look at an example.
+Each of the contacts are property-lists ('plists'), with properties
+@code{:name} (which may be @code{nil}), and @code{:mail}, and a number
+of other properties which you should return unchanged.
+
+Let's look at an example:
 
 @lisp
 (defun my-rewrite-function (contact)
@@ -2893,7 +2897,9 @@ Let's look at an example.
 	 (mail (plist-get contact :mail)))
     (cond
       ;; jonh smiht --> John Smith
-      ((string= "jonh smiht" name) (list :name "John Smith" :mail mail)) 
+      ((string= "jonh smiht" name) 
+	(plist-put contact :name "John C. Smith")
+	contact) 
       ;; remove evilspammer from the contacts list
       ((string= "evilspammer@@example.com" mail) nil)
       ;; others stay as the are