From 018c9606a691bbc41f50a657a7da9179ea67a374 Mon Sep 17 00:00:00 2001 From: Martin Heuschober Date: Sat, 2 Dec 2023 10:29:29 +0000 Subject: [PATCH] Implement frontmatter based filtering (#163) This allows limiting the notes that will be exported using `--skip-tags` and `--only-tags` --------- Co-authored-by: Martin Heuschober Co-authored-by: Nick Groenen Co-authored-by: Martin Heuschober --- README.md | 10 ++- docs/usage-advanced.md | 10 ++- src/main.rs | 13 ++- src/postprocessors.rs | 88 +++++++++++++++++++ tests/export_test.rs | 26 +++--- tests/postprocessors_test.rs | 67 ++++++++++---- .../expected/filter-by-tags/export-me.md | 7 ++ .../expected/filter-by-tags/export.md | 6 ++ .../input/filter-by-tags/export-me.md | 5 ++ .../input/filter-by-tags/export-no-export.md | 5 ++ tests/testdata/input/filter-by-tags/export.md | 5 ++ .../input/filter-by-tags/no-frontmatter.md | 1 + .../input/filter-by-tags/no-no-export.md | 5 ++ .../testdata/input/filter-by-tags/no-tags.md | 5 ++ .../testdata/input/filter-by-tags/private.md | 5 ++ 15 files changed, 221 insertions(+), 37 deletions(-) create mode 100644 tests/testdata/expected/filter-by-tags/export-me.md create mode 100644 tests/testdata/expected/filter-by-tags/export.md create mode 100644 tests/testdata/input/filter-by-tags/export-me.md create mode 100644 tests/testdata/input/filter-by-tags/export-no-export.md create mode 100644 tests/testdata/input/filter-by-tags/export.md create mode 100644 tests/testdata/input/filter-by-tags/no-frontmatter.md create mode 100644 tests/testdata/input/filter-by-tags/no-no-export.md create mode 100644 tests/testdata/input/filter-by-tags/no-tags.md create mode 100644 tests/testdata/input/filter-by-tags/private.md diff --git a/README.md b/README.md index 2d27c75..e61e0dc 100644 --- a/README.md +++ b/README.md @@ -138,9 +138,15 @@ To completely remove any frontmatter from exported notes, use `--frontmatter=nev ## Ignoring files -By default, hidden files, patterns listed in `.export-ignore` as well as any files ignored by git (if your vault is part of a git repository) will be excluded from exports. +The following files are not exported by default: + +* hidden files (can be adjusted with `--hidden`) +* files mattching a pattern listed in `.export-ignore` (can be adjusted with `--ignore-file`) +* any files that are ignored by git (can be adjusted with `--no-git`) +* any files having `private: true` in their frontmatter (the keyword `private` can be changed with `--ignore-frontmatter-keyword`) +* using `--skip-tags foo --skip-tags bar` will skip any files that have the tags `foo` or `bar` in their frontmatter +* using `--only-tags foo --only-tags bar` will skip any files that **don't** have the tags `foo` or `bar` in their frontmatter -These options may be adjusted with `--hidden`, `--ignore-file` and `--no-git` if desired. (See `--help` for more information). Notes linking to ignored notes will be unlinked (they'll only include the link text). diff --git a/docs/usage-advanced.md b/docs/usage-advanced.md index de168f2..9bd997a 100644 --- a/docs/usage-advanced.md +++ b/docs/usage-advanced.md @@ -12,9 +12,15 @@ To completely remove any frontmatter from exported notes, use `--frontmatter=nev ## Ignoring files -By default, hidden files, patterns listed in `.export-ignore` as well as any files ignored by git (if your vault is part of a git repository) will be excluded from exports. +The following files are not exported by default: + +* hidden files (can be adjusted with `--hidden`) +* files mattching a pattern listed in `.export-ignore` (can be adjusted with `--ignore-file`) +* any files that are ignored by git (can be adjusted with `--no-git`) +* any files having `private: true` in their frontmatter (the keyword `private` can be changed with `--ignore-frontmatter-keyword`) +* using `--skip-tags foo --skip-tags bar` will skip any files that have the tags `foo` or `bar` in their frontmatter +* using `--only-tags foo --only-tags bar` will skip any files that **don't** have the tags `foo` or `bar` in their frontmatter -These options may be adjusted with `--hidden`, `--ignore-file` and `--no-git` if desired. (See `--help` for more information). Notes linking to ignored notes will be unlinked (they'll only include the link text). diff --git a/src/main.rs b/src/main.rs index 902783d..1798d1b 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,7 +1,7 @@ use eyre::{eyre, Result}; use gumdrop::Options; -use obsidian_export::postprocessors::softbreaks_to_hardbreaks; -use obsidian_export::{ExportError, Exporter, FrontmatterStrategy, WalkOptions}; +use obsidian_export::{postprocessors::*, ExportError}; +use obsidian_export::{Exporter, FrontmatterStrategy, WalkOptions}; use std::{env, path::PathBuf}; const VERSION: &str = env!("CARGO_PKG_VERSION"); @@ -39,6 +39,12 @@ struct Opts { )] ignore_file: String, + #[options(no_short, help = "Exclude files with this tag from the export")] + skip_tags: Vec, + + #[options(no_short, help = "Export only files with this tag")] + only_tags: Vec, + #[options(no_short, help = "Export hidden files", default = "false")] hidden: bool, @@ -94,6 +100,9 @@ fn main() { exporter.add_postprocessor(&softbreaks_to_hardbreaks); } + let tags_postprocessor = filter_by_tags(args.skip_tags, args.only_tags); + exporter.add_postprocessor(&tags_postprocessor); + if let Some(path) = args.start_at { exporter.start_at(path); } diff --git a/src/postprocessors.rs b/src/postprocessors.rs index 0e8141e..2ca4183 100644 --- a/src/postprocessors.rs +++ b/src/postprocessors.rs @@ -2,6 +2,7 @@ use super::{Context, MarkdownEvents, PostprocessorResult}; use pulldown_cmark::Event; +use serde_yaml::Value; /// This postprocessor converts all soft line breaks to hard line breaks. Enabling this mimics /// Obsidian's _'Strict line breaks'_ setting. @@ -16,3 +17,90 @@ pub fn softbreaks_to_hardbreaks( } PostprocessorResult::Continue } + +pub fn filter_by_tags( + skip_tags: Vec, + only_tags: Vec, +) -> impl Fn(&mut Context, &mut MarkdownEvents) -> PostprocessorResult { + move |context: &mut Context, _events: &mut MarkdownEvents| -> PostprocessorResult { + match context.frontmatter.get("tags") { + None => filter_by_tags_(&[], &skip_tags, &only_tags), + Some(Value::Sequence(tags)) => filter_by_tags_(tags, &skip_tags, &only_tags), + _ => PostprocessorResult::Continue, + } + } +} + +fn filter_by_tags_( + tags: &[Value], + skip_tags: &[String], + only_tags: &[String], +) -> PostprocessorResult { + let skip = skip_tags + .iter() + .any(|tag| tags.contains(&Value::String(tag.to_string()))); + let include = only_tags.is_empty() + || only_tags + .iter() + .any(|tag| tags.contains(&Value::String(tag.to_string()))); + + if skip || !include { + PostprocessorResult::StopAndSkipNote + } else { + PostprocessorResult::Continue + } +} + +#[test] +fn test_filter_tags() { + let tags = vec![ + Value::String("skip".to_string()), + Value::String("publish".to_string()), + ]; + let empty_tags = vec![]; + assert_eq!( + filter_by_tags_(&empty_tags, &[], &[]), + PostprocessorResult::Continue, + "When no exclusion & inclusion are specified, files without tags are included" + ); + assert_eq!( + filter_by_tags_(&tags, &[], &[]), + PostprocessorResult::Continue, + "When no exclusion & inclusion are specified, files with tags are included" + ); + assert_eq!( + filter_by_tags_(&tags, &["exclude".to_string()], &[]), + PostprocessorResult::Continue, + "When exclusion tags don't match files with tags are included" + ); + assert_eq!( + filter_by_tags_(&empty_tags, &["exclude".to_string()], &[]), + PostprocessorResult::Continue, + "When exclusion tags don't match files without tags are included" + ); + assert_eq!( + filter_by_tags_(&tags, &[], &["publish".to_string()]), + PostprocessorResult::Continue, + "When exclusion tags don't match files with tags are included" + ); + assert_eq!( + filter_by_tags_(&empty_tags, &[], &["include".to_string()]), + PostprocessorResult::StopAndSkipNote, + "When inclusion tags are specified files without tags are excluded" + ); + assert_eq!( + filter_by_tags_(&tags, &[], &["include".to_string()]), + PostprocessorResult::StopAndSkipNote, + "When exclusion tags don't match files with tags are exluded" + ); + assert_eq!( + filter_by_tags_(&tags, &["skip".to_string()], &["skip".to_string()]), + PostprocessorResult::StopAndSkipNote, + "When both inclusion and exclusion tags are the same exclusion wins" + ); + assert_eq!( + filter_by_tags_(&tags, &["skip".to_string()], &["publish".to_string()]), + PostprocessorResult::StopAndSkipNote, + "When both inclusion and exclusion tags match exclusion wins" + ); +} diff --git a/tests/export_test.rs b/tests/export_test.rs index a57abaf..237bbd3 100644 --- a/tests/export_test.rs +++ b/tests/export_test.rs @@ -37,7 +37,7 @@ fn test_main_variants_with_default_options() { entry.path().display() ) }); - let actual = read_to_string(tmp_dir.path().clone().join(PathBuf::from(&filename))) + let actual = read_to_string(tmp_dir.path().join(PathBuf::from(&filename))) .unwrap_or_else(|_| panic!("failed to read {} from temporary exportdir", filename)); assert_eq!( @@ -62,7 +62,6 @@ fn test_frontmatter_never() { let actual = read_to_string( tmp_dir .path() - .clone() .join(PathBuf::from("note-with-frontmatter.md")), ) .unwrap(); @@ -85,7 +84,6 @@ fn test_frontmatter_always() { let actual = read_to_string( tmp_dir .path() - .clone() .join(PathBuf::from("note-without-frontmatter.md")), ) .unwrap(); @@ -96,7 +94,6 @@ fn test_frontmatter_always() { let actual = read_to_string( tmp_dir .path() - .clone() .join(PathBuf::from("note-with-frontmatter.md")), ) .unwrap(); @@ -114,10 +111,7 @@ fn test_exclude() { .run() .expect("exporter returned error"); - let excluded_note = tmp_dir - .path() - .clone() - .join(PathBuf::from("excluded-note.md")); + let excluded_note = tmp_dir.path().join(PathBuf::from("excluded-note.md")); assert!( !excluded_note.exists(), "exluded-note.md was found in tmpdir, but should be absent due to .export-ignore rules" @@ -136,14 +130,14 @@ fn test_single_file_to_dir() { assert_eq!( read_to_string("tests/testdata/expected/single-file/note.md").unwrap(), - read_to_string(tmp_dir.path().clone().join(PathBuf::from("note.md"))).unwrap(), + read_to_string(tmp_dir.path().join(PathBuf::from("note.md"))).unwrap(), ); } #[test] fn test_single_file_to_file() { let tmp_dir = TempDir::new().expect("failed to make tempdir"); - let dest = tmp_dir.path().clone().join(PathBuf::from("export.md")); + let dest = tmp_dir.path().join(PathBuf::from("export.md")); Exporter::new( PathBuf::from("tests/testdata/input/single-file/note.md"), @@ -178,7 +172,7 @@ fn test_start_at_subdir() { assert_eq!( expected, - read_to_string(tmp_dir.path().clone().join(PathBuf::from("Note B.md"))).unwrap(), + read_to_string(tmp_dir.path().join(PathBuf::from("Note B.md"))).unwrap(), ); } @@ -204,14 +198,14 @@ fn test_start_at_file_within_subdir_destination_is_dir() { assert_eq!( expected, - read_to_string(tmp_dir.path().clone().join(PathBuf::from("Note B.md"))).unwrap(), + read_to_string(tmp_dir.path().join(PathBuf::from("Note B.md"))).unwrap(), ); } #[test] fn test_start_at_file_within_subdir_destination_is_file() { let tmp_dir = TempDir::new().expect("failed to make tempdir"); - let dest = tmp_dir.path().clone().join(PathBuf::from("note.md")); + let dest = tmp_dir.path().join(PathBuf::from("note.md")); let mut exporter = Exporter::new( PathBuf::from("tests/testdata/input/start-at/"), dest.clone(), @@ -360,7 +354,7 @@ fn test_no_recursive_embeds() { assert_eq!( read_to_string("tests/testdata/expected/infinite-recursion/Note A.md").unwrap(), - read_to_string(tmp_dir.path().clone().join(PathBuf::from("Note A.md"))).unwrap(), + read_to_string(tmp_dir.path().join(PathBuf::from("Note A.md"))).unwrap(), ); } @@ -392,7 +386,7 @@ fn test_non_ascii_filenames() { entry.path().display() ) }); - let actual = read_to_string(tmp_dir.path().clone().join(PathBuf::from(&filename))) + let actual = read_to_string(tmp_dir.path().join(PathBuf::from(&filename))) .unwrap_or_else(|_| panic!("failed to read {} from temporary exportdir", filename)); assert_eq!( @@ -422,6 +416,6 @@ fn test_same_filename_different_directories() { .unwrap() }; - let actual = read_to_string(tmp_dir.path().clone().join(PathBuf::from("Note.md"))).unwrap(); + let actual = read_to_string(tmp_dir.path().join(PathBuf::from("Note.md"))).unwrap(); assert_eq!(expected, actual); } diff --git a/tests/postprocessors_test.rs b/tests/postprocessors_test.rs index 5311896..c04c7c5 100644 --- a/tests/postprocessors_test.rs +++ b/tests/postprocessors_test.rs @@ -1,4 +1,4 @@ -use obsidian_export::postprocessors::softbreaks_to_hardbreaks; +use obsidian_export::postprocessors::{filter_by_tags, softbreaks_to_hardbreaks}; use obsidian_export::{Context, Exporter, MarkdownEvents, PostprocessorResult}; use pretty_assertions::assert_eq; use pulldown_cmark::{CowStr, Event}; @@ -8,6 +8,7 @@ use std::fs::{read_to_string, remove_file}; use std::path::PathBuf; use std::sync::Mutex; use tempfile::TempDir; +use walkdir::WalkDir; /// This postprocessor replaces any instance of "foo" with "bar" in the note body. fn foo_to_bar(_ctx: &mut Context, events: &mut MarkdownEvents) -> PostprocessorResult { @@ -44,7 +45,7 @@ fn test_postprocessors() { exporter.run().unwrap(); let expected = read_to_string("tests/testdata/expected/postprocessors/Note.md").unwrap(); - let actual = read_to_string(tmp_dir.path().clone().join(PathBuf::from("Note.md"))).unwrap(); + let actual = read_to_string(tmp_dir.path().join(PathBuf::from("Note.md"))).unwrap(); assert_eq!(expected, actual); } @@ -66,7 +67,7 @@ fn test_postprocessor_stophere() { #[test] fn test_postprocessor_stop_and_skip() { let tmp_dir = TempDir::new().expect("failed to make tempdir"); - let note_path = tmp_dir.path().clone().join(PathBuf::from("Note.md")); + let note_path = tmp_dir.path().join(PathBuf::from("Note.md")); let mut exporter = Exporter::new( PathBuf::from("tests/testdata/input/postprocessors"), @@ -86,7 +87,7 @@ fn test_postprocessor_stop_and_skip() { #[test] fn test_postprocessor_change_destination() { let tmp_dir = TempDir::new().expect("failed to make tempdir"); - let original_note_path = tmp_dir.path().clone().join(PathBuf::from("Note.md")); + let original_note_path = tmp_dir.path().join(PathBuf::from("Note.md")); let mut exporter = Exporter::new( PathBuf::from("tests/testdata/input/postprocessors"), tmp_dir.path().to_path_buf(), @@ -102,7 +103,7 @@ fn test_postprocessor_change_destination() { }); exporter.run().unwrap(); - let new_note_path = tmp_dir.path().clone().join(PathBuf::from("MovedNote.md")); + let new_note_path = tmp_dir.path().join(PathBuf::from("MovedNote.md")); assert!(!original_note_path.exists()); assert!(new_note_path.exists()); } @@ -131,7 +132,7 @@ fn test_postprocessor_stateful_callback() { exporter.run().unwrap(); - let expected = tmp_dir.path().clone(); + let expected = tmp_dir.path(); let parents = parents.lock().unwrap(); println!("{:?}", parents); @@ -158,7 +159,7 @@ fn test_embed_postprocessors() { let expected = read_to_string("tests/testdata/expected/postprocessors/Note_embed_postprocess_only.md") .unwrap(); - let actual = read_to_string(tmp_dir.path().clone().join(PathBuf::from("Note.md"))).unwrap(); + let actual = read_to_string(tmp_dir.path().join(PathBuf::from("Note.md"))).unwrap(); assert_eq!(expected, actual); } @@ -178,7 +179,7 @@ fn test_embed_postprocessors_stop_and_skip() { let expected = read_to_string("tests/testdata/expected/postprocessors/Note_embed_stop_and_skip.md") .unwrap(); - let actual = read_to_string(tmp_dir.path().clone().join(PathBuf::from("Note.md"))).unwrap(); + let actual = read_to_string(tmp_dir.path().join(PathBuf::from("Note.md"))).unwrap(); assert_eq!(expected, actual); } @@ -244,12 +245,48 @@ fn test_softbreaks_to_hardbreaks() { let expected = read_to_string("tests/testdata/expected/postprocessors/hard_linebreaks.md").unwrap(); - let actual = read_to_string( - tmp_dir - .path() - .clone() - .join(PathBuf::from("hard_linebreaks.md")), - ) - .unwrap(); + let actual = read_to_string(tmp_dir.path().join(PathBuf::from("hard_linebreaks.md"))).unwrap(); assert_eq!(expected, actual); } + +#[test] +fn test_filter_by_tags() { + let tmp_dir = TempDir::new().expect("failed to make tempdir"); + let mut exporter = Exporter::new( + PathBuf::from("tests/testdata/input/filter-by-tags"), + tmp_dir.path().to_path_buf(), + ); + let filter_by_tags = filter_by_tags( + vec!["private".to_string(), "no-export".to_string()], + vec!["export".to_string()], + ); + exporter.add_postprocessor(&filter_by_tags); + exporter.run().unwrap(); + + let walker = WalkDir::new("tests/testdata/expected/filter-by-tags/") + // Without sorting here, different test runs may trigger the first assertion failure in + // unpredictable order. + .sort_by(|a, b| a.file_name().cmp(b.file_name())) + .into_iter(); + for entry in walker { + let entry = entry.unwrap(); + if entry.metadata().unwrap().is_dir() { + continue; + }; + let filename = entry.file_name().to_string_lossy().into_owned(); + let expected = read_to_string(entry.path()).unwrap_or_else(|_| { + panic!( + "failed to read {} from testdata/expected/filter-by-tags", + entry.path().display() + ) + }); + let actual = read_to_string(tmp_dir.path().join(PathBuf::from(&filename))) + .unwrap_or_else(|_| panic!("failed to read {} from temporary exportdir", filename)); + + assert_eq!( + expected, actual, + "{} does not have expected content", + filename + ); + } +} diff --git a/tests/testdata/expected/filter-by-tags/export-me.md b/tests/testdata/expected/filter-by-tags/export-me.md new file mode 100644 index 0000000..2d4d4f0 --- /dev/null +++ b/tests/testdata/expected/filter-by-tags/export-me.md @@ -0,0 +1,7 @@ +--- +tags: +- export +- me +--- + +A public note diff --git a/tests/testdata/expected/filter-by-tags/export.md b/tests/testdata/expected/filter-by-tags/export.md new file mode 100644 index 0000000..c4e074c --- /dev/null +++ b/tests/testdata/expected/filter-by-tags/export.md @@ -0,0 +1,6 @@ +--- +tags: +- export +--- + +A public note diff --git a/tests/testdata/input/filter-by-tags/export-me.md b/tests/testdata/input/filter-by-tags/export-me.md new file mode 100644 index 0000000..5b36253 --- /dev/null +++ b/tests/testdata/input/filter-by-tags/export-me.md @@ -0,0 +1,5 @@ +--- +tags: [export, me] +--- + +A public note diff --git a/tests/testdata/input/filter-by-tags/export-no-export.md b/tests/testdata/input/filter-by-tags/export-no-export.md new file mode 100644 index 0000000..7d604f5 --- /dev/null +++ b/tests/testdata/input/filter-by-tags/export-no-export.md @@ -0,0 +1,5 @@ +--- +tags: [export, no-export, private] +--- + +A private note diff --git a/tests/testdata/input/filter-by-tags/export.md b/tests/testdata/input/filter-by-tags/export.md new file mode 100644 index 0000000..acdf457 --- /dev/null +++ b/tests/testdata/input/filter-by-tags/export.md @@ -0,0 +1,5 @@ +--- +tags: [export] +--- + +A public note diff --git a/tests/testdata/input/filter-by-tags/no-frontmatter.md b/tests/testdata/input/filter-by-tags/no-frontmatter.md new file mode 100644 index 0000000..427b0ea --- /dev/null +++ b/tests/testdata/input/filter-by-tags/no-frontmatter.md @@ -0,0 +1 @@ +A note without frontmatter should be exported. diff --git a/tests/testdata/input/filter-by-tags/no-no-export.md b/tests/testdata/input/filter-by-tags/no-no-export.md new file mode 100644 index 0000000..cc21504 --- /dev/null +++ b/tests/testdata/input/filter-by-tags/no-no-export.md @@ -0,0 +1,5 @@ +--- +tags: [no, no-export] +--- + +A private note diff --git a/tests/testdata/input/filter-by-tags/no-tags.md b/tests/testdata/input/filter-by-tags/no-tags.md new file mode 100644 index 0000000..fdc5f81 --- /dev/null +++ b/tests/testdata/input/filter-by-tags/no-tags.md @@ -0,0 +1,5 @@ +--- +title: foo +--- + +A public note. diff --git a/tests/testdata/input/filter-by-tags/private.md b/tests/testdata/input/filter-by-tags/private.md new file mode 100644 index 0000000..88b76e9 --- /dev/null +++ b/tests/testdata/input/filter-by-tags/private.md @@ -0,0 +1,5 @@ +--- +tags: [private] +--- + +A private note.