app_lib/
preset_scanner.rs

1//! Plugin preset file scanner.
2//!
3//! Discovers preset files (FXP, FXB, VSTPRESET, AUPRESET, etc.) across
4//! the user home directory (`~`, resolved via [`dirs::home_dir`]) plus
5//! system-wide locations outside `~` (e.g. `/Library/Audio/Presets` on
6//! macOS, `Program Files\\Common Files\\VST3 Presets` on Windows).
7//! Supports parallel traversal and stop signaling.
8//! Symlinks are followed (`metadata` on the link) so file and directory
9//! targets are scanned; broken links are skipped.
10
11use crate::history::PresetFile;
12use crate::scanner_skip_dirs::SCANNER_SKIP_DIRS as SKIP_DIRS;
13use crate::unified_walker::IncrementalDirState;
14use rayon::prelude::*;
15use dashmap::DashSet;
16use std::collections::HashSet;
17use std::fs;
18use std::path::{Path, PathBuf};
19use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
20use std::sync::{Arc, Mutex};
21
22fn normalize_macos_path(p: PathBuf) -> PathBuf {
23    #[cfg(target_os = "macos")]
24    {
25        let s = p.to_string_lossy();
26        if s.starts_with("/System/Volumes/Data/") {
27            return PathBuf::from(&s["/System/Volumes/Data".len()..]);
28        }
29    }
30    p
31}
32
33pub(crate) const PRESET_EXTENSIONS: &[&str] = &[
34    ".fxp",       // VST2 preset
35    ".fxb",       // VST2 bank
36    ".vstpreset", // VST3 preset
37    ".aupreset",  // Audio Unit preset
38    ".adv",       // Ableton device preset
39    ".adg",       // Ableton rack preset
40    ".nki",       // Kontakt instrument
41    ".nksn",      // Kontakt snapshot
42    ".h2p",       // u-he preset
43    ".syx",       // MIDI SysEx dump
44    ".tfx",       // Tone2 preset
45    ".pjunoxl",   // TAL preset
46                  // .mid / .midi deliberately excluded — MIDI has its own dedicated scanner
47                  // (midi_scanner.rs) and lives in the midi_files table. Including them here
48                  // would double-count MIDI files into both presets and midi_files tables.
49];
50
51fn format_size(bytes: u64) -> String {
52    crate::format_size(bytes)
53}
54
55/// `Path::extension()` lowercased, no dot — used by the file watcher to match [`PRESET_EXTENSIONS`].
56#[inline]
57pub(crate) fn is_preset_extension_lowercase(ext: &str) -> bool {
58    PRESET_EXTENSIONS
59        .iter()
60        .any(|e| e.strip_prefix('.') == Some(ext))
61}
62
63pub fn get_preset_roots() -> Vec<PathBuf> {
64    let home = dirs::home_dir().unwrap_or_default();
65    let mut roots = Vec::new();
66
67    if !home.as_os_str().is_empty() {
68        roots.push(home.clone());
69    }
70
71    #[cfg(target_os = "macos")]
72    {
73        roots.push(PathBuf::from("/Library/Audio/Presets"));
74    }
75
76    #[cfg(target_os = "windows")]
77    {
78        if let Ok(pf) = std::env::var("ProgramFiles") {
79            roots.push(PathBuf::from(&pf).join("Common Files").join("VST3 Presets"));
80        }
81    }
82
83    roots.sort();
84    roots.dedup();
85    roots.into_iter().filter(|r| r.exists()).collect()
86}
87
88pub fn walk_for_presets(
89    roots: &[PathBuf],
90    on_batch: &mut dyn FnMut(&[PresetFile], usize),
91    should_stop: &(dyn Fn() -> bool + Sync),
92    exclude: Option<HashSet<String>>,
93    active_dirs: Option<Arc<Mutex<Vec<String>>>>,
94    incremental: Option<Arc<IncrementalDirState>>,
95) {
96    let batch_size = 100;
97    let stop = Arc::new(AtomicBool::new(false));
98    let found = Arc::new(AtomicUsize::new(0));
99    let active = active_dirs.unwrap_or_else(|| Arc::new(Mutex::new(Vec::new())));
100    let (tx, rx) = std::sync::mpsc::sync_channel::<Vec<PresetFile>>(256);
101    let visited = Arc::new(DashSet::new());
102    let exclude = Arc::new(exclude.unwrap_or_default());
103
104    let roots_owned: Vec<PathBuf> = roots.to_vec();
105    let stop2 = stop.clone();
106    let found2 = found.clone();
107    let incremental = incremental.clone();
108    let pool = rayon::ThreadPoolBuilder::new()
109        .num_threads(num_cpus::get().max(4))
110        .build()
111        .unwrap();
112    std::thread::spawn(move || {
113        pool.install(|| {
114            roots_owned.par_iter().for_each(|root| {
115                if stop2.load(Ordering::Relaxed) {
116                    return;
117                }
118                walk_dir_parallel(
119                    root,
120                    0,
121                    &visited,
122                    &tx,
123                    &found2,
124                    batch_size,
125                    &stop2,
126                    &exclude,
127                    &active,
128                    incremental.clone(),
129                );
130            });
131        });
132        drop(pool);
133    });
134
135    let mut total_found = 0usize;
136    loop {
137        if should_stop() {
138            stop.store(true, Ordering::Relaxed);
139            while rx.try_recv().is_ok() {}
140            break;
141        }
142        match rx.recv_timeout(std::time::Duration::from_millis(10)) {
143            Ok(presets) => {
144                total_found += presets.len();
145                on_batch(&presets, total_found);
146            }
147            Err(std::sync::mpsc::RecvTimeoutError::Timeout) => continue,
148            Err(std::sync::mpsc::RecvTimeoutError::Disconnected) => break,
149        }
150    }
151}
152
153#[allow(clippy::too_many_arguments)]
154fn walk_dir_parallel(
155    dir: &Path,
156    depth: u32,
157    visited: &Arc<DashSet<PathBuf>>,
158    tx: &std::sync::mpsc::SyncSender<Vec<PresetFile>>,
159    found: &Arc<AtomicUsize>,
160    batch_size: usize,
161    stop: &Arc<AtomicBool>,
162    exclude: &Arc<HashSet<String>>,
163    active_dirs: &Arc<Mutex<Vec<String>>>,
164    incremental: Option<Arc<IncrementalDirState>>,
165) {
166    if depth > 30 || stop.load(Ordering::Relaxed) {
167        return;
168    }
169
170    {
171        let orig = normalize_macos_path(dir.to_path_buf());
172        let canon = fs::canonicalize(dir).ok().map(normalize_macos_path);
173        let key = canon.unwrap_or_else(|| orig.clone());
174        if !visited.insert(key) {
175            return;
176        }
177        visited.insert(orig);
178    }
179
180    if let Some(ref inc) = incremental {
181        if inc.should_skip(dir) {
182            return;
183        }
184    }
185
186    let dir_str = dir.to_string_lossy().to_string();
187    {
188        let mut ad = active_dirs.lock().unwrap_or_else(|e| e.into_inner());
189        ad.push(dir_str.clone());
190        if ad.len() > 200 {
191            let excess = ad.len() - 200;
192            ad.drain(..excess);
193        }
194    }
195
196    let entries: Vec<_> = match fs::read_dir(dir) {
197        Ok(e) => e.flatten().collect(),
198        Err(_e) => {
199            return;
200        }
201    };
202
203    let mut files = Vec::new();
204    let mut subdirs = Vec::new();
205
206    for entry in &entries {
207        let name = entry.file_name();
208        let name_str = name.to_string_lossy();
209        // `@` prefix = Synology NAS system dirs (@eaDir, @tmp, @syno*, etc.).
210        if name_str.starts_with('.')
211            || name_str.starts_with('@')
212            || SKIP_DIRS.contains(&name_str.as_ref())
213            || exclude.contains(name_str.as_ref())
214        {
215            continue;
216        }
217        // Cached d_type from readdir — no extra stat() syscall per entry.
218        let ft = match entry.file_type() {
219            Ok(f) => f,
220            Err(_) => continue,
221        };
222        let path = entry.path();
223        if ft.is_dir() {
224            subdirs.push(path);
225        } else if ft.is_file() {
226            files.push((path, dir.to_path_buf()));
227        } else if ft.is_symlink() {
228            match fs::metadata(&path) {
229                Ok(m) if m.is_dir() => {
230                    subdirs.push(path);
231                }
232                Ok(m) if m.is_file() => {
233                    files.push((path, dir.to_path_buf()));
234                }
235                _ => {}
236            }
237        }
238    }
239
240    let mut batch = Vec::new();
241    for (path, parent) in files {
242        let ext = path
243            .extension()
244            .map(|e| format!(".{}", e.to_string_lossy().to_lowercase()))
245            .unwrap_or_default();
246
247        if PRESET_EXTENSIONS.contains(&ext.as_str()) {
248            let path_str = path.to_string_lossy().to_string();
249            if exclude.contains(&path_str) {
250                continue;
251            }
252            if let Ok(meta) = fs::metadata(&path) {
253                let preset_name = path
254                    .file_stem()
255                    .map(|s| s.to_string_lossy().to_string())
256                    .unwrap_or_default();
257                let modified = meta
258                    .modified()
259                    .ok()
260                    .map(|t| {
261                        let dt: chrono::DateTime<chrono::Utc> = t.into();
262                        dt.format("%Y-%m-%d").to_string()
263                    })
264                    .unwrap_or_default();
265
266                batch.push(PresetFile {
267                    name: preset_name,
268                    path: path_str,
269                    directory: parent.to_string_lossy().to_string(),
270                    format: ext[1..].to_uppercase(),
271                    size: meta.len(),
272                    size_formatted: format_size(meta.len()),
273                    modified,
274                });
275                found.fetch_add(1, Ordering::Relaxed);
276
277                if batch.len() >= batch_size {
278                    let _ = tx.send(batch);
279                    batch = Vec::new();
280                }
281            }
282        }
283    }
284    if !batch.is_empty() {
285        let _ = tx.send(batch);
286    }
287
288    subdirs.par_iter().for_each(|subdir| {
289        walk_dir_parallel(
290            subdir,
291            depth + 1,
292            visited,
293            tx,
294            found,
295            batch_size,
296            stop,
297            exclude,
298            active_dirs,
299            incremental.clone(),
300        );
301    });
302
303    if let Some(ref inc) = incremental {
304        inc.record_scanned_dir(dir);
305    }
306}
307
308#[cfg(test)]
309mod tests {
310    use super::*;
311    use std::slice::from_ref;
312
313    #[test]
314    fn test_preset_extensions_complete() {
315        for ext in &[".fxp", ".fxb", ".vstpreset", ".aupreset"] {
316            assert!(
317                PRESET_EXTENSIONS.contains(ext),
318                "PRESET_EXTENSIONS should contain {}",
319                ext
320            );
321        }
322    }
323
324    #[test]
325    fn test_get_preset_roots_not_empty() {
326        let roots = get_preset_roots();
327        assert!(!roots.is_empty());
328    }
329
330    #[test]
331    fn test_preset_extensions_includes_common() {
332        for ext in &[".fxp", ".fxb", ".vstpreset"] {
333            assert!(
334                PRESET_EXTENSIONS.contains(ext),
335                "PRESET_EXTENSIONS must contain {}",
336                ext
337            );
338        }
339    }
340
341    #[test]
342    fn test_preset_extensions_excludes_midi() {
343        // MIDI files belong to the dedicated midi_scanner/midi_files table —
344        // listing them here would double-count them into both tables.
345        for ext in &[".mid", ".midi"] {
346            assert!(
347                !PRESET_EXTENSIONS.contains(ext),
348                "PRESET_EXTENSIONS must NOT list MIDI {} — use midi_scanner instead",
349                ext
350            );
351        }
352    }
353
354    #[test]
355    fn test_preset_extensions_includes_ableton_kontakt_extras() {
356        for ext in &[".adv", ".adg", ".nksn", ".syx", ".pjunoxl"] {
357            assert!(
358                PRESET_EXTENSIONS.contains(ext),
359                "PRESET_EXTENSIONS should contain {}",
360                ext
361            );
362        }
363    }
364
365    #[test]
366    fn test_normalize_macos_path() {
367        let p = PathBuf::from("/System/Volumes/Data/Users/example");
368        let n = normalize_macos_path(p);
369        #[cfg(target_os = "macos")]
370        assert_eq!(n, PathBuf::from("/Users/example"));
371        #[cfg(not(target_os = "macos"))]
372        assert_eq!(n, PathBuf::from("/System/Volumes/Data/Users/example"));
373    }
374
375    #[test]
376    fn test_preset_roots_exist() {
377        let roots = get_preset_roots();
378        assert!(
379            !roots.is_empty(),
380            "At least one preset root directory should exist on this system"
381        );
382        for root in &roots {
383            assert!(root.exists(), "Returned root should exist: {:?}", root);
384        }
385    }
386
387    #[test]
388    fn test_walk_for_presets_empty_dir() {
389        let tmp = std::env::temp_dir().join("upum_test_preset_empty");
390        let _ = fs::remove_dir_all(&tmp);
391        fs::create_dir_all(&tmp).unwrap();
392
393        let mut found = Vec::new();
394        walk_for_presets(
395            from_ref(&tmp),
396            &mut |batch, _count| {
397                found.extend_from_slice(batch);
398            },
399            &|| false,
400            None,
401            None,
402            None,
403        );
404        assert!(found.is_empty());
405        let _ = fs::remove_dir_all(&tmp);
406    }
407
408    #[test]
409    fn test_walk_for_presets_finds_tfx_and_h2p() {
410        let tmp = std::env::temp_dir().join("upum_test_preset_tfx_h2p");
411        let _ = fs::remove_dir_all(&tmp);
412        fs::create_dir_all(&tmp).unwrap();
413        fs::write(tmp.join("tone.tfx"), b"tone2").unwrap();
414        fs::write(tmp.join("diva.h2p"), b"u-he").unwrap();
415
416        let mut found = Vec::new();
417        walk_for_presets(
418            from_ref(&tmp),
419            &mut |batch, _count| found.extend_from_slice(batch),
420            &|| false,
421            None,
422            None,
423            None,
424        );
425        let formats: Vec<&str> = found.iter().map(|p| p.format.as_str()).collect();
426        assert!(formats.contains(&"TFX"), "expected TFX, got {:?}", formats);
427        assert!(formats.contains(&"H2P"), "expected H2P, got {:?}", formats);
428        let _ = fs::remove_dir_all(&tmp);
429    }
430
431    #[test]
432    fn test_walk_for_presets_finds_nksn_kontakt_snapshot() {
433        let tmp = std::env::temp_dir().join("upum_test_preset_nksn");
434        let _ = fs::remove_dir_all(&tmp);
435        fs::create_dir_all(&tmp).unwrap();
436        fs::write(tmp.join("snap.nksn"), b"kontakt").unwrap();
437
438        let mut found = Vec::new();
439        walk_for_presets(
440            from_ref(&tmp),
441            &mut |batch, _count| found.extend_from_slice(batch),
442            &|| false,
443            None,
444            None,
445            None,
446        );
447        assert_eq!(found.len(), 1);
448        assert_eq!(found[0].format, "NKSN");
449        let _ = fs::remove_dir_all(&tmp);
450    }
451
452    #[test]
453    fn test_walk_for_presets_finds_files() {
454        let tmp = std::env::temp_dir().join("upum_test_preset_find");
455        let _ = fs::remove_dir_all(&tmp);
456        fs::create_dir_all(&tmp).unwrap();
457        fs::write(tmp.join("lead.fxp"), b"fake preset").unwrap();
458        fs::write(tmp.join("bank.fxb"), b"fake bank").unwrap();
459        fs::write(tmp.join("pad.vstpreset"), b"fake vstpreset").unwrap();
460        fs::write(tmp.join("not_a_preset.txt"), b"nope").unwrap();
461
462        let mut found = Vec::new();
463        walk_for_presets(
464            from_ref(&tmp),
465            &mut |batch, _count| {
466                found.extend_from_slice(batch);
467            },
468            &|| false,
469            None,
470            None,
471            None,
472        );
473        assert_eq!(found.len(), 3);
474        let formats: Vec<&str> = found.iter().map(|p| p.format.as_str()).collect();
475        assert!(formats.contains(&"FXP"));
476        assert!(formats.contains(&"FXB"));
477        assert!(formats.contains(&"VSTPRESET"));
478        let _ = fs::remove_dir_all(&tmp);
479    }
480
481    #[test]
482    fn test_walk_for_presets_stop_signal() {
483        let tmp = std::env::temp_dir().join("upum_test_preset_stop");
484        let _ = fs::remove_dir_all(&tmp);
485        fs::create_dir_all(&tmp).unwrap();
486        for i in 0..20 {
487            fs::write(tmp.join(format!("preset{}.fxp", i)), b"fake").unwrap();
488        }
489
490        let counter = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0));
491        let c2 = counter.clone();
492        let stop_after = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false));
493        let s2 = stop_after.clone();
494
495        walk_for_presets(
496            from_ref(&tmp),
497            &mut |batch, _count| {
498                c2.fetch_add(batch.len(), std::sync::atomic::Ordering::Relaxed);
499                s2.store(true, std::sync::atomic::Ordering::Relaxed);
500            },
501            &|| stop_after.load(std::sync::atomic::Ordering::Relaxed),
502            None,
503            None,
504            None,
505        );
506        // Should have stopped — may have found some but scan should terminate
507        let _ = fs::remove_dir_all(&tmp);
508    }
509
510    #[test]
511    fn test_walk_for_presets_exclude_set() {
512        let tmp = std::env::temp_dir().join("upum_test_preset_exclude");
513        let _ = fs::remove_dir_all(&tmp);
514        fs::create_dir_all(&tmp).unwrap();
515        let included = tmp.join("keep.fxp");
516        let excluded = tmp.join("skip.fxp");
517        fs::write(&included, b"keep").unwrap();
518        fs::write(&excluded, b"skip").unwrap();
519
520        let mut exclude = HashSet::new();
521        exclude.insert(excluded.to_string_lossy().to_string());
522
523        let mut found = Vec::new();
524        walk_for_presets(
525            from_ref(&tmp),
526            &mut |batch, _count| {
527                found.extend_from_slice(batch);
528            },
529            &|| false,
530            Some(exclude),
531            None,
532            None,
533        );
534        assert_eq!(found.len(), 1);
535        assert!(found[0].path.contains("keep.fxp"));
536        let _ = fs::remove_dir_all(&tmp);
537    }
538
539    #[test]
540    fn test_walk_for_presets_skips_hidden_and_blacklisted_dirs() {
541        let tmp = std::env::temp_dir().join("upum_test_preset_skip");
542        let _ = fs::remove_dir_all(&tmp);
543        fs::create_dir_all(tmp.join(".hidden_dir")).unwrap();
544        fs::create_dir_all(tmp.join("node_modules")).unwrap();
545        fs::create_dir_all(tmp.join("normal")).unwrap();
546        fs::write(tmp.join(".hidden_dir/a.fxp"), b"h").unwrap();
547        fs::write(tmp.join("node_modules/b.fxp"), b"n").unwrap();
548        fs::write(tmp.join("normal/c.fxp"), b"ok").unwrap();
549
550        let mut found = Vec::new();
551        walk_for_presets(
552            from_ref(&tmp),
553            &mut |batch, _count| found.extend_from_slice(batch),
554            &|| false,
555            None,
556            None,
557            None,
558        );
559        assert_eq!(found.len(), 1);
560        assert!(found[0].path.contains("normal"));
561        let _ = fs::remove_dir_all(&tmp);
562    }
563
564    #[test]
565    fn test_walk_for_presets_deduplicates_symlinks() {
566        let tmp = std::env::temp_dir().join("upum_test_preset_symlink");
567        let _ = fs::remove_dir_all(&tmp);
568        fs::create_dir_all(tmp.join("real")).unwrap();
569        fs::write(tmp.join("real/a.fxp"), b"preset").unwrap();
570
571        #[cfg(unix)]
572        {
573            let _ = std::os::unix::fs::symlink(tmp.join("real"), tmp.join("link"));
574            let mut found = Vec::new();
575            walk_for_presets(
576                &[tmp.join("real"), tmp.join("link")],
577                &mut |batch, _count| found.extend_from_slice(batch),
578                &|| false,
579                None,
580                None,
581                None,
582            );
583            assert_eq!(found.len(), 1, "Symlinked duplicate should be deduped");
584        }
585        let _ = fs::remove_dir_all(&tmp);
586    }
587
588    #[test]
589    fn test_walk_for_presets_deduplicates_overlapping_roots() {
590        let tmp = std::env::temp_dir().join("upum_test_preset_overlap");
591        let _ = fs::remove_dir_all(&tmp);
592        let child = tmp.join("sub");
593        fs::create_dir_all(&child).unwrap();
594        fs::write(child.join("overlap.fxp"), b"preset").unwrap();
595        fs::write(tmp.join("top.fxp"), b"preset").unwrap();
596
597        let mut found = Vec::new();
598        walk_for_presets(
599            &[tmp.clone(), child.clone()],
600            &mut |batch, _| found.extend_from_slice(batch),
601            &|| false,
602            None,
603            None,
604            None,
605        );
606        let overlap_count = found.iter().filter(|p| p.name == "overlap").count();
607        assert_eq!(
608            overlap_count, 1,
609            "overlap.fxp found {} times",
610            overlap_count
611        );
612        assert!(found.iter().any(|p| p.name == "top"));
613        let _ = fs::remove_dir_all(&tmp);
614    }
615
616    #[test]
617    fn test_walk_for_presets_consistent_counts() {
618        let tmp = std::env::temp_dir().join("upum_test_preset_consistent");
619        let _ = fs::remove_dir_all(&tmp);
620        for i in 0..5 {
621            let d = tmp.join(format!("dir{}", i));
622            fs::create_dir_all(&d).unwrap();
623            fs::write(d.join(format!("p{}.fxp", i)), b"preset").unwrap();
624        }
625        let mut c1 = 0;
626        walk_for_presets(
627            &[tmp.clone()],
628            &mut |b, _| c1 += b.len(),
629            &|| false,
630            None,
631            None,
632            None,
633        );
634        let mut c2 = 0;
635        walk_for_presets(
636            &[tmp.clone()],
637            &mut |b, _| c2 += b.len(),
638            &|| false,
639            None,
640            None,
641            None,
642        );
643        assert_eq!(c1, c2, "two scans should match: {} vs {}", c1, c2);
644        assert_eq!(c1, 5);
645        let _ = fs::remove_dir_all(&tmp);
646    }
647
648    #[test]
649    fn test_walk_for_presets_batching() {
650        let tmp = std::env::temp_dir().join("upum_test_preset_batch");
651        let _ = fs::remove_dir_all(&tmp);
652        fs::create_dir_all(&tmp).unwrap();
653        for i in 0..5 {
654            fs::write(tmp.join(format!("p{}.fxp", i)), b"fake").unwrap();
655        }
656
657        let mut total = 0usize;
658        walk_for_presets(
659            from_ref(&tmp),
660            &mut |batch, count| {
661                assert!(!batch.is_empty());
662                total = count;
663            },
664            &|| false,
665            None,
666            None,
667            None,
668        );
669        assert_eq!(total, 5);
670        let _ = fs::remove_dir_all(&tmp);
671    }
672}
app_lib/preset_scanner.rs

app_lib/
preset_scanner.rs
