lib: follow symlinks in maildirs

Until now, mu would _not_ follow symlinks; with these changes, we do.

There were some complications with that ~10 years ago, but I forgot the
details. So let's re-enable. At least one thing is in place now: moving
between file systems.

Fixes #1489
Fixes #1628 (technically, this came with slightly earlier commit)

NEWS.org

@@ -4,6 +4,11 @@
 * 1.5.x (unreleased, development version)
 
 *** mu
+
+    - Follow symlinks in maildirs, and support moving messsages between across
+      multiple filesystems (but note that that is quite a bit slower than the
+      single-filesystem case)
+
     - Optionally provide readline support for the mu server (when in tty-mode)
 
 *** mu4e

lib/mu-maildir.c

@@ -1,5 +1,3 @@
-/* -*-mode: c; tab-width: 8; indent-tabs-mode: t; c-basic-offset: 8 -*-*/
-
 /*
 ** Copyright (C) 2008-2020 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl>
 **
@@ -19,10 +17,7 @@
 **
 */
 
-
-#if HAVE_CONFIG_H
 #include "config.h"
-#endif /*HAVE_CONFIG_H*/
 
 #include <unistd.h>
 #include <sys/types.h>
@@ -41,28 +36,34 @@
 #define MU_MAILDIR_NOINDEX_FILE       ".noindex"
 #define MU_MAILDIR_NOUPDATE_FILE      ".noupdate"
 
-
 /* On Linux (and some BSD), we have entry->d_type, but some file
  * systems (XFS, ReiserFS) do not support it, and set it DT_UNKNOWN.
  * On other OSs, notably Solaris, entry->d_type is not present at all.
  * For these cases, we use lstat (in get_dtype) as a slower fallback,
  * and return it in the d_type parameter
  */
+static unsigned char
+get_dtype (struct dirent* dentry, const char *path, gboolean use_lstat)
+{
 #ifdef HAVE_STRUCT_DIRENT_D_TYPE
-#define GET_DTYPE(DE,FP)						   \
-	((DE)->d_type == DT_UNKNOWN ? mu_util_get_dtype_with_lstat((FP)) : \
-	 (DE)->d_type)
-#else
-#define GET_DTYPE(DE,FP)			                           \
-	mu_util_get_dtype_with_lstat((FP))
-#endif /*HAVE_STRUCT_DIRENT_D_TYPE*/
 
+	if (dentry->d_type == DT_UNKNOWN)
+		goto slowpath;
+	if (dentry->d_type == DT_LNK && !use_lstat)
+		goto slowpath;
+
+	return dentry->d_type; /* fastpath */
+
+slowpath:
+#endif /*HAVE_STRUCT_DIRENT_D_TYPE*/
+	return mu_util_get_dtype (path, use_lstat);
+}
 
 static gboolean
 create_maildir (const char *path, mode_t mode, GError **err)
 {
 	int i;
-	const gchar* subdirs[] = {"new", "cur", "tmp"};
+	const char* subdirs[] = {"new", "cur", "tmp"};
 
 	for (i = 0; i != G_N_ELEMENTS(subdirs); ++i) {
 
@@ -136,7 +137,7 @@ static gboolean
 check_subdir (const char *src, gboolean *in_cur, GError **err)
 {
 	gboolean rv;
-	gchar *srcpath;
+	char *srcpath;
 
 	srcpath = g_path_get_dirname (src);
 	*in_cur = FALSE;
@@ -153,10 +154,10 @@ check_subdir (const char *src, gboolean *in_cur, GError **err)
 	return rv;
 }
 
-static gchar*
+static char*
-get_target_fullpath (const char* src, const gchar *targetpath, GError **err)
+get_target_fullpath (const char* src, const char *targetpath, GError **err)
 {
-	gchar *targetfullpath, *srcfile;
+	char *targetfullpath, *srcfile;
 	gboolean in_cur;
 
 	if (!check_subdir (src, &in_cur, err))
@@ -185,7 +186,7 @@ get_target_fullpath (const char* src, const gchar *targetpath, GError **err)
 gboolean
 mu_maildir_link (const char* src, const char *targetpath, GError **err)
 {
-	gchar *targetfullpath;
+	char *targetfullpath;
 	int rv;
 
 	g_return_val_if_fail (src, FALSE);
@@ -208,13 +209,13 @@ mu_maildir_link (const char* src, const char *targetpath, GError **err)
 
 
 static MuError
-process_dir (const char* path, const gchar *mdir,
+process_dir (const char* path, const char *mdir,
 	     MuMaildirWalkMsgCallback msg_cb,
 	     MuMaildirWalkDirCallback dir_cb, gboolean full,
 	     void *data);
 
 static MuError
-process_file (const char* fullpath, const gchar* mdir,
+process_file (const char* fullpath, const char* mdir,
 	      MuMaildirWalkMsgCallback msg_cb, void *data)
 {
 	MuError result;
@@ -374,8 +375,8 @@ ignore_dir_entry (struct dirent *entry, unsigned char d_type)
  * leaf "/cur" or "/new". In other words, contatenate old_mdir + "/" + dir,
  * unless dir is either 'new' or 'cur'. The value will be used in queries.
  */
-static gchar*
+static char*
-get_mdir_for_path (const gchar *old_mdir, const gchar *dir)
+get_mdir_for_path (const char *old_mdir, const char *dir)
 {
 	/* if the current dir is not 'new' or 'cur', contatenate
 	 * old_mdir an dir */
@@ -406,7 +407,7 @@ process_dir_entry (const char* path, const char* mdir, struct dirent *entry,
 	fullpath = g_newa (char, strlen(fp) + 1);
 	strcpy (fullpath, fp);
 
-	d_type = GET_DTYPE(entry, fullpath);
+	d_type = get_dtype(entry, fullpath, FALSE/*stat*/);
 
 	/* ignore special files/dirs */
 	if (ignore_dir_entry (entry, d_type)) {
@@ -446,7 +447,7 @@ static const size_t DIRENT_ALLOC_SIZE =
 static struct dirent*
 dirent_new (void)
 {
-	return (struct dirent*) g_slice_alloc (DIRENT_ALLOC_SIZE);
+	return (struct dirent*) g_new0(guchar, DIRENT_ALLOC_SIZE);
 }
 
 
@@ -602,7 +603,7 @@ clear_links (const char *path, DIR *dir)
 			continue; /* ignore .,.. other dotdirs */
 
 		fullpath = g_build_path ("/", path, dentry->d_name, NULL);
-		d_type	 = GET_DTYPE (dentry, fullpath);
+		d_type	 = get_dtype (dentry, fullpath, TRUE/*lstat*/);
 
 		if (d_type == DT_LNK) {
 			if (unlink (fullpath) != 0 ) {
@@ -729,7 +730,7 @@ mu_maildir_get_flags_from_path (const char *path)
  * latter case, no other flags are allowed.
  *
  */
-static gchar*
+static char*
 get_new_path (const char *mdir, const char *mfile, MuFlags flags,
 	      const char* custom_flags, char flags_sep)
 {
@@ -752,7 +753,7 @@ get_new_path (const char *mdir, const char *mfile, MuFlags flags,
 char*
 mu_maildir_get_maildir_from_path (const char* path)
 {
-	gchar *mdir;
+	char *mdir;
 
 	/* determine the maildir */
 	mdir = g_path_get_dirname (path);
@@ -846,7 +847,7 @@ get_file_size (const char* path)
 
 
 static gboolean
-msg_move_check_pre (const gchar *src, const gchar *dst, GError **err)
+msg_move_check_pre (const char *src, const char *dst, GError **err)
 {
 	gint size1, size2;
 
@@ -932,7 +933,7 @@ msg_move (const char* src, const char *dst, GError **err)
 	return msg_move_g_file (src, dst, err);
 }
 
-gchar*
+char*
 mu_maildir_move_message (const char* oldpath, const char* targetmdir,
 			 MuFlags newflags, gboolean ignore_dups,
 			 gboolean new_name, GError **err)

lib/mu-maildir.h

@@ -1,5 +1,5 @@
 /*
-** Copyright (C) 2008-2015 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl>
+** Copyright (C) 2008-2020 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl>
 **
 ** This program is free software; you can redistribute it and/or modify it
 ** under the terms of the GNU General Public License as published by the
@@ -126,7 +126,7 @@ MuError mu_maildir_walk (const char *path, MuMaildirWalkMsgCallback cb_msg,
  *
  * @return TRUE if it worked, FALSE in case of error
  */
-gboolean mu_maildir_clear_links (const gchar* dir, GError **err);
+gboolean mu_maildir_clear_links (const char* dir, GError **err);
 
 
 
@@ -213,9 +213,9 @@ char* mu_maildir_get_maildir_from_path (const char* path)
  * @return return the full path name of the target file (g_free) if
  * the move succeeded, NULL otherwise
  */
-gchar* mu_maildir_move_message (const char* oldpath, const char* targetmdir,
+char* mu_maildir_move_message (const char* oldpath, const char* targetmdir,
-				MuFlags newflags, gboolean ignore_dups,
+                               MuFlags newflags, gboolean ignore_dups,
-				gboolean new_name, GError **err)
+                               gboolean new_name, GError **err)
 	G_GNUC_WARN_UNUSED_RESULT;
 
 G_END_DECLS

lib/utils/mu-util.c

@@ -340,14 +340,21 @@ mu_util_play (const char *path, gboolean allow_local, gboolean allow_remote,
 
 
 unsigned char
-mu_util_get_dtype_with_lstat (const char *path)
+mu_util_get_dtype (const char *path, gboolean use_lstat)
 {
+	int res;
 	struct stat statbuf;
 
 	g_return_val_if_fail (path, DT_UNKNOWN);
 
-	if (lstat (path, &statbuf) != 0) {
+	if (use_lstat)
-		g_warning ("stat failed on %s: %s", path, strerror(errno));
+		res = lstat (path, &statbuf);
+	else
+		res = stat (path, &statbuf);
+
+	if (res != 0) {
+		g_warning ("%sstat failed on %s: %s",
+			   use_lstat ? "l" : "", path, strerror(errno));
 		return DT_UNKNOWN;
 	}
 
@@ -363,6 +370,7 @@ mu_util_get_dtype_with_lstat (const char *path)
 }
 
 
+
 gboolean
 mu_util_locale_is_utf8 (void)
 {

lib/utils/mu-util.h

@@ -257,15 +257,16 @@ enum {
 
 
 /**
- * get the d_type (as in direntry->d_type) for the file at path, using
+ * get the d_type (as in direntry->d_type) for the file at path, using either
-* lstat(3)
+ * stat(3) or lstat(3)
  *
  * @param path full path
+ * @param use_lstat whether to use lstat (otherwise use stat)
  *
- * @return DT_REG, DT_DIR, DT_LNK, or DT_UNKNOWN (other values are not
+ * @return DT_REG, DT_DIR, DT_LNK, or DT_UNKNOWN (other values are not supported
- * supported currently )
+ * currently )
  */
-unsigned char mu_util_get_dtype_with_lstat (const char *path);
+unsigned char mu_util_get_dtype (const char *path, gboolean use_lstat);
 
 
 /**

lib/utils/test-mu-util.c

@@ -165,11 +165,11 @@ static void
 test_mu_util_get_dtype_with_lstat (void)
 {
 	g_assert_cmpuint (
-		mu_util_get_dtype_with_lstat (MU_TESTMAILDIR), ==, DT_DIR);
+		mu_util_get_dtype (MU_TESTMAILDIR, TRUE), ==, DT_DIR);
 	g_assert_cmpuint (
-		mu_util_get_dtype_with_lstat (MU_TESTMAILDIR2), ==, DT_DIR);
+		mu_util_get_dtype (MU_TESTMAILDIR2, TRUE), ==, DT_DIR);
 	g_assert_cmpuint (
-		mu_util_get_dtype_with_lstat (MU_TESTMAILDIR2 "/Foo/cur/mail5"),
+		mu_util_get_dtype (MU_TESTMAILDIR2 "/Foo/cur/mail5", TRUE),
 		==, DT_REG);
 }
 

man/mu-index.1

@@ -1,4 +1,4 @@
-.TH MU-INDEX 1 "February 2020" "User Manuals"
+.TH MU-INDEX 1 "May 2020" "User Manuals"
 
 .SH NAME
 
@@ -27,7 +27,9 @@ E-mail messages which are not stored in something resembling a maildir
 leaf-directory (\fIcur\fR and \fInew\fR) are ignored, as are the cache
 directories for \fInotmuch\fR and \fIgnus\fR, and any dot-directory.
 
-The maildir must be on a single file-system; symlinks are not followed.
+Starting with mu 1.5.x, symlinks are followed, and can be spread over multiple
+filesystems; however note that moving files around is much faster when multiple
+filesystems are not involved.
 
 If there is a file called \fI.noindex\fR in a directory, the contents of that
 directory and all of its subdirectories will be ignored. This can be useful to