From 964400cb1eba79f2532676a0a83aeeb56d1e37a5 Mon Sep 17 00:00:00 2001 From: Ben_Kosytorz Date: Fri, 9 Oct 2026 14:44:09 +0200 Subject: [PATCH] =?UTF-8?q?feat(rust):=20port=20sorter.py=20=E2=80=94=20fu?= =?UTF-8?q?zzy=20matching,=20plan,=20hardlink=20apply?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - src/sorter.rs: inspect (zip/7z listing via crates, HTML-page detection), keyword classification with inner-name votes, catalog load with dropped systems, fuzzy matching on the CPython-difflib port (containment boost, generic-token stoplist, tokenless bonus, weak tier, cross-system gates), deepest-dir re-homing incl. provenance unit mode, build_target, hardlink-apply with audit + low-confidence doc, format_plan - all 9 python test_sorter_units.py cases ported and passing - parity fixes found while porting: inner-name votes REPLACE (not max-keep) the outer score below threshold; python max() keeps first maximal vote; ratio cache keyed with allow_single; 512-byte head read for HTML sniff --- version-rust/src/sorter.rs | 2119 +++++++++++++++++++++++++++++++++++- 1 file changed, 2117 insertions(+), 2 deletions(-) diff --git a/version-rust/src/sorter.rs b/version-rust/src/sorter.rs index 3468cb4..5f1a4f3 100644 --- a/version-rust/src/sorter.rs +++ b/version-rust/src/sorter.rs @@ -3,5 +3,2120 @@ //! `version-python/dlphelper/sorter.py`. //! //! Matching rules are fixed (see version-python/AGENTS.md §7 "Sorter matching -//! rules"); they must be ported exactly or the Rust sorter will silently -//! sort differently than the battle-tested python one. +//! rules"); they are ported exactly, including thresholds (CROSS_SYSTEM_MIN +//! 0.75, WEAK_MIN 0.50, TOKENLESS_BONUS 0.08), the generic-token stoplist, +//! junk-dir stoplist, deepest-dir re-homing and the weak tier. Fuzzy scores +//! come from the faithful `crate::difflib` port of CPython's +//! SequenceMatcher, so the Rust sorter accepts and rejects exactly what the +//! battle-tested python one does. +//! +//! The python version caches per-(name, system, faction) matches and per-pair +//! ratios with lru_cache; here, caches are plain HashMaps threaded through +//! the call chain (`MatchCache`), same effect. + +use std::collections::{BTreeMap, HashMap, HashSet}; +use std::fs; +use std::os::unix::fs::MetadataExt; +use std::path::{Path, PathBuf}; + +use serde_json::Value; + +use crate::catalogue::{clean_name, match_class, match_faction}; +use crate::difflib::SequenceMatcher; +use crate::warhammer_data::Data; + +pub const ARCHIVE_SUFFIXES: &[&str] = &[".zip", ".7z", ".rar", ".tar", ".gz"]; +pub const UNSORTED: &str = "_unsorted"; +pub const UNCLASSIFIED_CLASS: &str = "Unclassified"; +/// A file whose system the keyword tables could not identify only gets a +/// catalog unit when the fuzzy hit is strong - otherwise it would wander +/// into other games' sections (Blood Bowl, ...) on a guess. +pub const CROSS_SYSTEM_MIN: f64 = 0.75; +/// Game systems the sorter must never place into (collection is +/// warhammer-only). The catalog snapshot keeps all systems; the filter is +/// applied at load. +pub const DROPPED_SYSTEMS: &[&str] = &["Middle-Earth"]; +/// Weak acceptance tier: below the normal threshold, a hit at least this +/// strong is accepted but flagged low confidence. +pub const WEAK_MIN: f64 = 0.50; +/// A hit sharing no word with the entry needs this much more score to be +/// trusted (char-ratio flukes sit at 0.60-0.67; real unit names usually +/// share a token). +pub const TOKENLESS_BONUS: f64 = 0.08; + +/// Default fuzzy threshold (python sorter.py --threshold default). +pub const DEFAULT_THRESHOLD: f64 = 0.62; + +/// Run-artifact files that must never be sorted into a library. +pub const RUN_ARTIFACTS: &[&str] = &[ + "export.json", + "export.html", + "state.json", + "manifest.jsonl", + "failed_urls.txt", + "catalogue.md", + "catalogue.jsonl", + "sorter_applied.jsonl", + "catalogue_moves.jsonl", + "urls.txt", + "urls_0807.txt", + "urls_gofile.txt", + "failed_0807.txt", + "extract_manifest.jsonl", +]; + +/// Bump a vote counter with python dict semantics (insertion order kept). +fn bump_vote(votes: &mut Vec<(K, usize)>, key: K) { + if let Some(slot) = votes.iter_mut().find(|(k, _)| *k == key) { + slot.1 += 1; + } else { + votes.push((key, 1)); + } +} + +/// First maximal element in insertion order (python `max()` semantics). +fn first_max(votes: &[(K, usize)]) -> Option<(K, usize)> { + let mut best: Option<(K, usize)> = None; + for (k, n) in votes { + if best.as_ref().is_none_or(|(_, bn)| n > bn) { + best = Some((k.clone(), *n)); + } + } + best +} + +/// One file to sort, plus everything learned about it. +#[derive(Debug, Clone)] +pub struct SortItem { + pub path: PathBuf, + pub size: u64, + /// original file name + pub name: String, + /// cleaned lower-case name + pub clean: String, + /// archive listing (inner names) + pub inner_names: Vec, + pub system: String, + pub group: String, + pub faction: String, + pub subfaction: Option, + pub klass: Option, + /// canonical catalog name, if matched + pub unit: Option, + pub match_score: f64, + /// catalog | weak | implicit | none | invalid + pub match_kind: String, + /// normal | low + pub confidence: String, + /// link | copy + pub action: String, + pub target: Option, +} + +impl SortItem { + fn bare(path: PathBuf, size: u64, name: String, clean: String) -> Self { + SortItem { + path, + size, + name, + clean, + inner_names: Vec::new(), + system: "Unknown".into(), + group: UNSORTED.into(), + faction: UNSORTED.into(), + subfaction: None, + klass: None, + unit: None, + match_score: 0.0, + match_kind: "none".into(), + confidence: "normal".into(), + action: "link".into(), + target: None, + } + } +} + +/// A catalog entry (`data/gwshop_catalog.json` or `library_catalog.json` +/// shape) with the precomputed cleaned names the python version attaches. +#[derive(Debug, Clone)] +pub struct CatalogEntry { + pub system: Option, + pub group: Option, + pub faction: Option, + pub subfaction: Option, + pub class: Option, + pub name: String, + pub clean: String, + pub clean_aliases: Vec, +} + +/// Cache key for `_match_name` results: (clean, system, faction). +type DirMatchKey = (String, String, String); + +/// Shared caches for one plan run (python lru_caches are module-global). +#[derive(Default)] +pub struct MatchCache { + /// (clean, system, faction) -> Some((score, entry-index, weak)) or None + dir_matches: HashMap>, + /// ratio cache: (clean, entry-index, allow_single) -> best score + ratios: HashMap<(String, usize, bool), f64>, +} + +/// Shared reference tables for a matching session. +pub struct SorterCtx { + pub catalog: Vec, +} + +// --------------------------------------------------------------------------- +// inspection +// --------------------------------------------------------------------------- + +/// Inner names of a .zip without extracting (first `limit`). +pub fn list_zip(path: &Path, limit: usize) -> Vec { + let file = match fs::File::open(path) { + Ok(f) => f, + Err(exc) => { + log::debug!("zip listing failed for {}: {exc}", path.display()); + return Vec::new(); + } + }; + let mut zf = match zip::ZipArchive::new(file) { + Ok(zf) => zf, + Err(exc) => { + log::debug!("zip listing failed for {}: {exc}", path.display()); + return Vec::new(); + } + }; + let mut names = Vec::new(); + for i in 0..zf.len() { + if let Some(name) = zf.name_for_index(i) { + if !name.ends_with('/') { + names.push(name.to_string()); + } + } + if names.len() >= limit { + break; + } + } + names +} + +/// Inner names of a .7z via sevenz-rust2, without extracting. +/// (python shelled out to `7z l -ba -slt`; the crate reads the archive +/// structure directly - same result, names as stored.) +pub fn list_7z(path: &Path, limit: usize) -> Vec { + use sevenz_rust2::ArchiveReader; + let mut reader = match ArchiveReader::open(path, sevenz_rust2::Password::empty()) { + Ok(r) => r, + Err(exc) => { + log::debug!("7z listing failed for {}: {exc}", path.display()); + return Vec::new(); + } + }; + let mut names = Vec::new(); + let mut collect = |entry: &sevenz_rust2::ArchiveEntry, _reader: &mut dyn std::io::Read| { + if !entry.is_directory() && names.len() < limit { + names.push(entry.name().to_string()); + } + Ok(true) + }; + if reader.for_each_entries(&mut collect).is_err() { + log::debug!("7z listing failed for {}", path.display()); + return Vec::new(); + } + names +} + +fn is_archive_name(name: &str) -> bool { + ARCHIVE_SUFFIXES + .iter() + .any(|s| py_suffix(name).eq_ignore_ascii_case(s)) +} + +/// python `Path.suffix`: the last dot-suffix, or "" (dotfiles have none). +fn py_suffix(name: &str) -> &str { + let stem_end = py_stem_split(name); + &name[stem_end..] +} + +/// Byte index where python's stem ends (start of the suffix, or len). +fn py_stem_split(name: &str) -> usize { + let base = match name.rfind('/') { + Some(i) => i + 1, + None => 0, + }; + let base_str = &name[base..]; + match base_str.rfind('.') { + Some(0) | None => name.len(), + Some(i) => base + i, + } +} + +/// Inspect one file: size, name, cleaned name, archive listing (or the +/// HTML-page-detection that marks it invalid). +pub fn inspect(path: &Path) -> SortItem { + let name = path + .file_name() + .map(|n| n.to_string_lossy().into_owned()) + .unwrap_or_default(); + let size = fs::metadata(path).map(|m| m.size()).unwrap_or(0); + let clean = clean_name(&name); + let mut item = SortItem::bare(path.to_path_buf(), size, name, clean); + if !path.is_file() { + return item; + } + let suffix = py_suffix(&item.name).to_ascii_lowercase(); + if ARCHIVE_SUFFIXES.contains(&suffix.as_str()) { + // An archive that is not an archive is almost always a saved HTML + // error/login page that got downloaded instead of the file. + item.inner_names = if suffix == ".zip" { + list_zip(path, 40) + } else if suffix == ".7z" { + list_7z(path, 40) + } else { + // .rar/.tar/.gz cannot be listed cheaply without extracting; + // python used the 7z CLI for all of them. Listing fails → empty. + Vec::new() + }; + if item.inner_names.is_empty() && size > 0 { + let head_is_html = fs::File::open(path).map_or(false, |mut fh| { + use std::io::Read; + let mut head = Vec::with_capacity(512); + let _ = fh.by_ref().take(512).read_to_end(&mut head); + let lower = head.to_ascii_lowercase(); + lower.windows(13).any(|w| w == b" "2e catachany +/// fellaz catachan jungle fighters x") - the file stem alone loses the +/// archive's own folder structure. +pub fn clean_inner(name: &str) -> String { + clean_name(&name.replace('/', " ").replace('\\', " ")) +} + +/// True for directory names that must never be match candidates. +pub fn is_junk_dir(name: &str) -> bool { + let clean = clean_name(name); + if clean.is_empty() + || JUNK_DIRS.contains(&clean.as_str()) + || clean.chars().all(|c| c.is_ascii_digit()) + || clean.chars().count() <= 2 + { + return true; + } + !clean.contains(' ') && GENERIC_TOKENS.contains(&clean.as_str()) +} + +/// Tokens with a crude plural 's' stripped, for containment checks. +fn stem_tokens(clean: &str) -> HashSet<&str> { + clean + .split_whitespace() + .map(|t| { + if t.len() > 3 && t.ends_with('s') { + &t[..t.len() - 1] + } else { + t + } + }) + .collect() +} + +/// True when one name's (stemmed) tokens all appear in the other's. +/// +/// Guarded like the containment boost: the smaller side must not consist of +/// generic army words alone. Used to gate cross-system adoption, where +/// plain char-ratio lies ("aeldari rangers" vs "dwarf rangers"). +pub fn token_contained(a_clean: &str, b_clean: &str) -> bool { + let ta = stem_tokens(a_clean); + let tb = stem_tokens(b_clean); + if ta.is_empty() || tb.is_empty() { + return false; + } + let small = if ta.is_subset(&tb) { + Some(&ta) + } else if tb.is_subset(&ta) { + Some(&tb) + } else { + None + }; + let Some(small) = small else { return false }; + if small.len() == 1 { + // a lone generic word ("guard", "orc") must not carry a match + return !GENERIC_TOKENS.contains(small.iter().next().unwrap()); + } + true +} + +/// True when two cleaned names share at least one non-generic token +/// (>= 3 chars). Gates the weak tier: a sub-threshold hit with zero shared +/// words is a char-ratio fluke ("LICENSE.txt" -> "Veletaris Storm Section"), +/// not evidence. +pub fn shares_rare_token(a_clean: &str, b_clean: &str) -> bool { + let ta = stem_tokens(a_clean); + let tb = stem_tokens(b_clean); + ta.intersection(&tb) + .any(|t| !GENERIC_TOKENS.contains(t) && t.chars().count() >= 3) +} + +/// Keyword classification from the file name, then refined by inner names. +pub fn classify_item(item: &mut SortItem) { + if let Some((system, faction)) = match_faction(&item.clean) { + item.system = system; + item.faction = faction; + } + item.klass = match_class(&item.clean).or_else(|| { + if item.inner_names.is_empty() { + None + } else { + let joined = item.inner_names.join(" "); + match_class(&clean_inner(&joined)) + } + }); + + if item.faction == UNSORTED && !item.inner_names.is_empty() { + // hash-named archives: vote on the inner names' classification. + // Insertion-order vote lists: python's max() keeps the FIRST maximal + // vote in dict insertion order (= inner_names order), not the + // alphabetically-largest key. + let mut votes: Vec<((String, String), usize)> = Vec::new(); + for inner in &item.inner_names { + if let Some(ihit) = match_faction(&clean_inner(inner)) { + bump_vote(&mut votes, ihit); + } + } + if let Some((best, _)) = first_max(&votes) { + item.system = best.0.clone(); + item.faction = best.1.clone(); + if item.klass.is_none() { + let mut cvotes: Vec<(String, usize)> = Vec::new(); + for inner in &item.inner_names { + if let Some(c) = match_class(&clean_name(inner)) { + bump_vote(&mut cvotes, c); + } + } + if let Some((c, _)) = first_max(&cvotes) { + item.klass = Some(c.clone()); + } + } + } + } + + item.group = Data::global() + .group_of_faction + .get(&item.faction) + .cloned() + .unwrap_or_else(|| UNSORTED.into()); +} + +/// Dropped systems (see DROPPED_SYSTEMS) count as unclassified: the +/// collection does not cover them, so their names must not place files +/// into those sections. Called after keyword classification, before +/// matching - the item can still be re-homed by other signals. +pub fn drop_dropped_system(item: &mut SortItem) { + if DROPPED_SYSTEMS.contains(&item.system.as_str()) { + item.system = "Unknown".into(); + item.group = UNSORTED.into(); + item.faction = UNSORTED.into(); + item.subfaction = None; + } +} + +// --------------------------------------------------------------------------- +// catalog +// --------------------------------------------------------------------------- + +/// Load catalog entries, dropping non-collected game systems by default +/// (pass an empty dropped set to keep everything). +/// Accepts the catalog document ({..., "entries": [...]}) or a bare list. +pub fn load_catalog(path: &Path, dropped: &[String]) -> anyhow::Result> { + let text = fs::read_to_string(path)?; + let data: Value = serde_json::from_str(&text)?; + let entries = match &data { + Value::Object(o) => o.get("entries").cloned().unwrap_or(Value::Array(vec![])), + Value::Array(_) => data.clone(), + _ => anyhow::bail!("catalog root must be an object or list"), + }; + let Value::Array(list) = entries else { + anyhow::bail!("catalog entries must be a list"); + }; + let mut out = Vec::new(); + for e in list { + let system = e.get("system").and_then(Value::as_str).map(str::to_string); + if system + .as_deref() + .is_some_and(|s| dropped.iter().any(|d| d == s)) + { + continue; + } + let name = e + .get("name") + .and_then(Value::as_str) + .unwrap_or_default() + .to_string(); + let aliases: Vec = e + .get("aliases") + .and_then(Value::as_array) + .map(|a| { + a.iter() + .filter_map(Value::as_str) + .map(str::to_string) + .collect() + }) + .unwrap_or_default(); + out.push(CatalogEntry { + system, + group: e.get("group").and_then(Value::as_str).map(str::to_string), + faction: e.get("faction").and_then(Value::as_str).map(str::to_string), + subfaction: e + .get("subfaction") + .and_then(Value::as_str) + .map(str::to_string), + class: e.get("class").and_then(Value::as_str).map(str::to_string), + clean: clean_name(&name), + clean_aliases: aliases.iter().map(|a| clean_name(a)).collect(), + name, + }); + } + Ok(out) +} + +/// The default-dropped system list as owned strings (CLI convenience). +pub fn dropped_systems_default(keep: &[String], extra_drop: &[String]) -> Vec { + let mut set: Vec = DROPPED_SYSTEMS + .iter() + .map(|s| s.to_string()) + .filter(|s| !keep.contains(s)) + .collect(); + for s in extra_drop { + if !set.contains(s) { + set.push(s.clone()); + } + } + set +} + +// --------------------------------------------------------------------------- +// fuzzy matching +// --------------------------------------------------------------------------- + +/// Best sequence ratio over `candidates`, pruned with cheap upper-bound +/// checks (python `_best_ratio_cached`). Cached per (clean, entry). +fn best_ratio( + clean: &str, + ctx: &SorterCtx, + entry_idx: usize, + allow_single: bool, + cache: &mut MatchCache, +) -> f64 { + let key = (clean.to_string(), entry_idx, allow_single); + if let Some(hit) = cache.ratios.get(&key) { + return *hit; + } + let entry = &ctx.catalog[entry_idx]; + let mut candidates: Vec<&str> = Vec::with_capacity(1 + entry.clean_aliases.len()); + candidates.push(entry.clean.as_str()); + candidates.extend(entry.clean_aliases.iter().map(String::as_str)); + + let mut best = 0.0f64; + let toks: HashSet = stem_tokens(clean).into_iter().map(String::from).collect(); + best = containment_boost(best, &toks, &candidates, allow_single); + for c in candidates { + let sm = SequenceMatcher::new(clean, c); + if sm.real_quick_ratio() < best || sm.quick_ratio() < best { + continue; + } + let r = sm.ratio(); + if r > best { + best = r; + if best == 1.0 { + break; + } + } + } + cache.ratios.insert(key, best); + best +} + +/// Token-containment bonus: a name whose tokens all appear in the other (or +/// vice versa) matches strongly even when string ratio is low ("telemon" vs +/// "Telemon Heavy Dreadnought"). Guarded: needs >= 2 tokens, or a single +/// rare one (>= 7 chars), and the smaller side must not consist of generic +/// army words ("chaos lord" is contained in half the catalog) so "head" or +/// "guard" never trigger it. Inner-name votes pass allow_single=false: one +/// short inner file name ("lieutenant") must not claim a whole pack. +fn containment_boost( + mut best: f64, + toks: &HashSet, + candidates: &[&str], + allow_single: bool, +) -> f64 { + if toks.is_empty() { + return best; + } + for c in candidates { + let ctoks: HashSet = stem_tokens(c).into_iter().map(String::from).collect(); + if ctoks.is_empty() { + continue; + } + let small = if toks.is_subset(&ctoks) { + Some(toks) + } else if ctoks.is_subset(toks) { + Some(&ctoks) + } else { + None + }; + let Some(small) = small else { continue }; + if small.iter().any(|t| GENERIC_TOKENS.contains(&t.as_str())) { + continue; + } + let single_ok = small.len() >= 2 + || (allow_single && small.iter().next().is_some_and(|t| t.chars().count() >= 7)); + if single_ok { + if best < 0.9 { + best = 0.9; + } + break; + } + } + best +} + +/// Words too generic for token-containment matching: entries built from them +/// alone would claim any file whose name happens to mention them. +pub const GENERIC_TOKENS: &[&str] = &[ + "chaos", "lord", "space", "marine", "guard", "knight", "dark", "death", "blood", "war", + "warrior", "daemon", "hero", "god", "free", "black", "white", "grey", "green", "gold", "iron", + "steel", "storm", "soul", "shadow", "bone", "ghost", "king", "clan", "tribe", "legion", + "regiment", "squad", "warhound", "orc", "ogre", "elf", "dwarf", "giant", "company", "chapter", + "weapon", "gun", "gunner", "turret", "carriage", "upgrade", +]; + +/// Directory names that carry no unit information inside sculptor packs; +/// never used as match candidates. +pub const JUNK_DIRS: &[&str] = &[ + "files", + "file", + "images", + "image", + "img", + "pictures", + "pics", + "preview", + "previews", + "render", + "renders", + "photos", + "screenshot", + "screenshots", + "torso", + "torsos", + "head", + "heads", + "arm", + "arms", + "leg", + "legs", + "back", + "front", + "left", + "right", + "top", + "bottom", + "side", + "weapon", + "weapons", + "accessories", + "accessory", + "bits", + "bitz", + "parts", + "part", + "base", + "bases", + "support", + "supports", + "presupported", + "unsupported", + "stl", + "stls", + "obj", + "models", + "model", + "output", + "exports", + "release", + "releases", + "license", + "readme", + "sources", + "docs", + "textures", + "new", + "old", + "final", + "fixed", + "print", + "prints", + "plates", + "plate", + "assembly", + "test", + "characters", + "character", +]; + +/// Token containment between the entry (or an alias) and the outer name or +/// any inner (relative-path) name. +fn contained_somewhere(item: &SortItem, entry_idx: usize, ctx: &SorterCtx) -> bool { + let entry = &ctx.catalog[entry_idx]; + let mut cands: Vec<&str> = vec![entry.clean.as_str()]; + cands.extend(entry.clean_aliases.iter().map(String::as_str)); + let mut probes: Vec = vec![item.clean.clone()]; + probes.extend(item.inner_names.iter().take(10).map(|n| clean_inner(n))); + probes + .iter() + .any(|p| cands.iter().any(|c| token_contained(p, c))) +} + +/// Adopt a catalog entry into an item (dir re-homing and weak tier). +/// +/// Unlike the pack/file path in match_catalog, the entry's class layer is +/// adopted unconditionally: a precise unit match ("Warp Spiders" -> Fly) +/// must beat a generic keyword class ("warlock" mentioned somewhere -> +/// Hero). +fn apply_entry( + item: &mut SortItem, + ctx: &SorterCtx, + entry_idx: usize, + score: f64, + kind: &str, + confidence: &str, +) { + let entry = &ctx.catalog[entry_idx]; + item.unit = Some(entry.name.clone()); + item.group = entry.group.clone().unwrap_or_else(|| { + Data::global() + .group_of_faction + .get(&item.faction) + .cloned() + .unwrap_or_else(|| UNSORTED.into()) + }); + item.subfaction = entry.subfaction.clone(); + if entry.class.is_some() { + item.klass = entry.class.clone(); + } + item.match_score = score; + item.match_kind = kind.to_string(); + item.confidence = confidence.to_string(); + let entry_faction = entry.faction.clone(); + if let Some(f) = entry_faction { + if !matches!( + f.as_str(), + "Unit Type" | "Heroes" | "Warriors" | "Character" + ) { + let e = &ctx.catalog[entry_idx]; + item.system = e.system.clone().unwrap_or_else(|| item.system.clone()); + item.faction = f; + } + } +} + +/// (score, entry, weak) when `entry` may adopt a name scoring `score`. +/// +/// Names sharing a rare token are trusted at the normal threshold (and down +/// to WEAK_MIN); tokenless hits need +TOKENLESS_BONUS and are never weak. +/// None when rejected. +fn accept_name_entry( + clean: &str, + ctx: &SorterCtx, + entry_idx: usize, + score: f64, + threshold: f64, +) -> Option<(f64, usize, bool)> { + let entry = &ctx.catalog[entry_idx]; + let shared = shares_rare_token(clean, &entry.clean) + || entry + .clean_aliases + .iter() + .any(|a| shares_rare_token(clean, a)); + if shared { + if score >= threshold { + return Some((score, entry_idx, false)); + } + if score >= WEAK_MIN { + return Some((score, entry_idx, true)); + } + return None; + } + if score >= threshold + TOKENLESS_BONUS { + return Some((score, entry_idx, false)); + } + None +} + +/// Best ACCEPTED entry over a pool (entry indices) - normal beats weak, then +/// higher score. +fn pool_best( + clean: &str, + pool: &[usize], + ctx: &SorterCtx, + threshold: f64, + cache: &mut MatchCache, +) -> Option<(f64, usize, bool)> { + let mut best_n: Option<(f64, usize, bool)> = None; + let mut best_w: Option<(f64, usize, bool)> = None; + for &idx in pool { + let score = best_ratio(clean, ctx, idx, true, cache); + let Some(a) = accept_name_entry(clean, ctx, idx, score, threshold) else { + continue; + }; + if a.2 { + if best_w.is_none_or(|b| a.0 > b.0) { + best_w = Some(a); + } + } else if best_n.is_none_or(|b| a.0 > b.0) { + best_n = Some(a); + } + } + best_n.or(best_w) +} + +/// Best fuzzy score of `item` against all entries in `pool` (entry indices). +/// +/// Inner archive names are only consulted on request; a confident outer +/// (file) name must win over votes from random inner STL filenames. +fn best_in_pool( + item: &SortItem, + pool: &[usize], + ctx: &SorterCtx, + use_inner: bool, + cache: &mut MatchCache, +) -> Option<(f64, usize)> { + let mut best: Option<(f64, usize)> = None; + for &idx in pool { + let mut score = best_ratio(&item.clean, ctx, idx, true, cache); + if use_inner && !item.inner_names.is_empty() { + for inner in item.inner_names.iter().take(10) { + let inner_clean = clean_inner(inner); + // no single-token containment boost for inner votes + let s = best_ratio(&inner_clean, ctx, idx, false, cache) * 0.95; + if s > score { + score = s; + } + } + } + if best.is_none_or(|b| score > b.0) { + best = Some((score, idx)); + } + } + best +} + +/// Fuzzy-match a bare name (a directory or pack name) against the catalog. +/// +/// Same-system constraint; a different faction needs a strong hit +/// (CROSS_SYSTEM_MIN). Returns (score, entry, weak) or None - weak marks a +/// same-faction hit accepted only via the WEAK_MIN tier. +fn match_name( + clean: &str, + ctx: &SorterCtx, + system: &str, + faction: &str, + threshold: f64, + cache: &mut MatchCache, +) -> Option<(f64, usize, bool)> { + let key = (clean.to_string(), system.to_string(), faction.to_string()); + if let Some(hit) = cache.dir_matches.get(&key) { + return hit.clone(); + } + let result = compute_match_name(clean, ctx, system, faction, threshold, cache); + cache.dir_matches.insert(key, result.clone()); + result +} + +fn compute_match_name( + clean: &str, + ctx: &SorterCtx, + system: &str, + faction: &str, + threshold: f64, + cache: &mut MatchCache, +) -> Option<(f64, usize, bool)> { + if system == "Unknown" { + let pool: Vec = (0..ctx.catalog.len()).collect(); + let full = best_in_pool_str(clean, &pool, ctx, false, cache); + if let Some((score, idx)) = full { + if score >= CROSS_SYSTEM_MIN { + let entry = &ctx.catalog[idx]; + let contained = token_contained(clean, &entry.clean) + || entry + .clean_aliases + .iter() + .any(|a| token_contained(clean, a)); + if contained { + return Some((score, idx, false)); + } + } + } + None + } else { + let pool: Vec = ctx + .catalog + .iter() + .enumerate() + .filter(|(_, e)| e.system.as_deref() == Some(system)) + .map(|(i, _)| i) + .collect(); + if faction != UNSORTED { + let same_pool: Vec = pool + .iter() + .copied() + .filter(|&i| ctx.catalog[i].faction.as_deref() == Some(faction)) + .collect(); + let other_pool: Vec = pool + .iter() + .copied() + .filter(|&i| ctx.catalog[i].faction.as_deref() != Some(faction)) + .collect(); + let same = pool_best(clean, &same_pool, ctx, threshold, cache); + let mut other = pool_best(clean, &other_pool, ctx, threshold, cache); + if let Some((score, _, weak)) = other { + // a different faction must win strong, never weak + if weak || score < CROSS_SYSTEM_MIN { + other = None; + } + } + let cands: Vec<(f64, usize, bool)> = [same, other].into_iter().flatten().collect(); + if !cands.is_empty() { + let normals: Vec<&(f64, usize, bool)> = cands.iter().filter(|c| !c.2).collect(); + let winner = if normals.is_empty() { + cands.iter().max_by(|a, b| a.0.total_cmp(&b.0)).unwrap() + } else { + normals.iter().max_by(|a, b| a.0.total_cmp(&b.0)).unwrap() + }; + return Some(winner.clone()); + } + None + } else { + pool_best(clean, &pool, ctx, threshold, cache) + } + } +} + +/// like best_in_pool but for a bare clean string (no inner names) +fn best_in_pool_str( + clean: &str, + pool: &[usize], + ctx: &SorterCtx, + use_inner: bool, + cache: &mut MatchCache, +) -> Option<(f64, usize)> { + let mut best: Option<(f64, usize)> = None; + for &idx in pool { + let mut score = best_ratio(clean, ctx, idx, true, cache); + if use_inner { + // bare-string variant never has inner names; kept for shape parity + } + if best.is_none_or(|b| score > b.0) { + best = Some((score, idx)); + } + } + let _ = use_inner; + best +} + +/// Best matching directory in a deepest-first [(path, name)] chain. +/// +/// Returns (score, entry, weak, path) of the winner - higher score wins, +/// equal keeps the deeper candidate - or None. +fn best_dir_match( + item: &SortItem, + dirs: &[(PathBuf, String)], + ctx: &SorterCtx, + threshold: f64, + cache: &mut MatchCache, +) -> Option<(f64, usize, bool, PathBuf)> { + let mut best_n: Option<(f64, usize, bool, PathBuf)> = None; + let mut best_w: Option<(f64, usize, bool, PathBuf)> = None; + for (real_dir, dir_name) in dirs { + if is_junk_dir(dir_name) { + continue; + } + let Some(m) = match_name( + &clean_name(dir_name), + ctx, + &item.system, + &item.faction, + threshold, + cache, + ) else { + continue; + }; + let (score, idx, weak) = m; + let cand = (score, idx, weak, real_dir.clone()); + if weak { + if best_w.as_ref().is_none_or(|b| score > b.0) { + best_w = Some(cand); + } + } else if best_n.as_ref().is_none_or(|b| score > b.0) { + best_n = Some(cand); + } + } + best_n.or(best_w) +} + +/// Let matching directory names re-home an item ("deepest match wins"). +/// +/// Returns the matched directory if the item was re-homed, else None. A dir +/// hit takes over when it beats the item's current resolution - a confident +/// pack/file match is only replaced by a strictly better (or equal, coming +/// from below the pack level) hit; implicit items yield to any accepted dir +/// match. +#[allow(clippy::too_many_arguments)] +fn rehome_from_dirs( + item: &mut SortItem, + dirs: &[(PathBuf, String)], + ctx: &SorterCtx, + threshold: f64, + cache: &mut MatchCache, + prefer_dirs_on_tie: bool, +) -> Option { + if item.match_kind == "invalid" { + return None; + } + let own = item.match_kind == "catalog" || item.match_kind == "weak"; + let own_score = if own { item.match_score } else { 0.0 }; + let Some((score, idx, weak, real_dir)) = best_dir_match(item, dirs, ctx, threshold, cache) + else { + return None; + }; + if own && score < own_score { + return None; + } + if own && score == own_score { + // exact tie: the deeper side wins, which is the dir only in + // provenance mode (pack name sits above the inner dirs); in manual + // trees the file name itself is the deepest side - keep it + if !(prefer_dirs_on_tie && !weak) { + return None; + } + } + let kind = if weak { "weak" } else { "catalog" }; + apply_entry( + item, + ctx, + idx, + score, + kind, + if weak { "low" } else { "normal" }, + ); + Some(real_dir) +} + +/// Fuzzy-match against catalog entries in the same faction (or all). +/// +/// The matched entry is hierarchy-authoritative: group, subfaction and class +/// come from the catalog (GW shop categories), and when the keyword +/// classification had no faction the entry's system/faction is adopted too. +pub fn match_catalog(item: &mut SortItem, ctx: &SorterCtx, threshold: f64, cache: &mut MatchCache) { + if item.faction == UNSORTED && item.inner_names.is_empty() && item.clean.is_empty() { + return; + } + if item.match_kind == "invalid" { + return; // HTML-page masquerading as an archive; never fuzzy-match it + } + let system_known = item.system != "Unknown"; + let mut best = find_best(item, ctx, system_known, threshold, cache); + apply_best(item, ctx, &mut best, threshold); +} + +fn pools_for(item: &SortItem, ctx: &SorterCtx, system_known: bool) -> (Vec, Vec) { + // (same_faction_pool, system_pool) + let same_faction: Vec = ctx + .catalog + .iter() + .enumerate() + .filter(|(_, e)| e.faction.as_deref() == Some(item.faction.as_str())) + .filter(|(_, e)| { + // a fuzzy match must never jump into another game's section + !system_known || e.system.as_deref() == Some(item.system.as_str()) + }) + .map(|(i, _)| i) + .collect(); + let sys_pool: Vec = if system_known { + ctx.catalog + .iter() + .enumerate() + .filter(|(_, e)| e.system.as_deref() == Some(item.system.as_str())) + .map(|(i, _)| i) + .collect() + } else { + (0..ctx.catalog.len()).collect() + }; + (same_faction, sys_pool) +} + +fn find_best( + item: &SortItem, + ctx: &SorterCtx, + system_known: bool, + threshold: f64, + cache: &mut MatchCache, +) -> Option<(f64, usize)> { + let (same_faction, sys_pool) = pools_for(item, ctx, system_known); + if !same_faction.is_empty() { + let mut best = best_in_pool(item, &same_faction, ctx, false, cache); + if best.is_none_or(|b| b.0 < threshold) { + // Outer name not confident: let the archive's inner file names + // vote (the only signal for hash-named gofile archives). + // NOTE: python REPLACES best here (not max-keep) - a lower inner + // vote lowers the score and can disqualify the weak tier. + best = best_in_pool(item, &same_faction, ctx, true, cache); + } + if best.is_none_or(|b| b.0 < threshold) { + // retry within the same system - never across systems. Keep the + // higher of the two scores (a retry must not downgrade a + // weak-eligible same-faction hit). + let sys_best = best_in_pool(item, &sys_pool, ctx, false, cache); + if sys_best.is_some_and(|b| best.is_none_or(|old| b.0 > old.0)) { + best = sys_best; + } + if best.is_none_or(|b| b.0 < threshold) { + let sys_best = best_in_pool(item, &sys_pool, ctx, true, cache); + if sys_best.is_some_and(|b| best.is_none_or(|old| b.0 > old.0)) { + best = sys_best; + } + } + // a retry hit from a *different* faction is only trusted when + // it is strong AND anchored by token containment; weak inner + // votes must not re-home the file + if let Some((_, idx)) = &best { + let e = &ctx.catalog[*idx]; + if e.faction.as_deref() != Some(item.faction.as_str()) + && (best.unwrap().0 < CROSS_SYSTEM_MIN || !contained_somewhere(item, *idx, ctx)) + { + best = None; + } + } + } + best + } else if system_known { + let mut best = best_in_pool(item, &sys_pool, ctx, false, cache); + if best.is_none_or(|b| b.0 < threshold) { + best = best_in_pool(item, &sys_pool, ctx, true, cache); + } + best + } else { + // Neither system nor faction known: whole catalog, but only a strong + // hit (CROSS_SYSTEM_MIN) with token containment may adopt another + // game's section; weaker or unanchored guesses stay in _unsorted. + let all: Vec = (0..ctx.catalog.len()).collect(); + let mut best = best_in_pool(item, &all, ctx, false, cache); + if best.is_none_or(|b| b.0 < threshold) { + best = best_in_pool(item, &all, ctx, true, cache); + } + if let Some((score, idx)) = best { + if score < CROSS_SYSTEM_MIN { + best = None; + } else if !contained_somewhere(item, idx, ctx) { + best = None; + } + } + best + } +} + +fn apply_best( + item: &mut SortItem, + ctx: &SorterCtx, + best: &mut Option<(f64, usize)>, + threshold: f64, +) { + let Some((score, idx)) = best else { + // nothing scored at all: implicit entry below + implicit_entry(item, 0.0); + return; + }; + let entry = &ctx.catalog[*idx]; + if *score >= threshold { + item.unit = Some(entry.name.clone()); + if entry.group.is_some() { + item.group = entry.group.clone().unwrap(); + } else if item.faction == UNSORTED { + item.group = entry + .faction + .as_ref() + .and_then(|f| Data::global().group_of_faction.get(f)) + .cloned() + .unwrap_or_else(|| UNSORTED.into()); + } + item.subfaction = entry.subfaction.clone(); + // GW's class layer (shop category) beats the keyword guess, but do + // not *narrow* it ... (see python: catalog class is authoritative + // for the whole subtree) + if let Some(k) = &entry.class { + item.klass = Some(k.clone()); + } + item.match_score = *score; + item.match_kind = "catalog".into(); + // The catalog entry's faction is authoritative - but only adopt it + // when it is a real army faction. + let generic = matches!( + entry.faction.as_deref(), + Some("Unit Type") | Some("Heroes") | Some("Warriors") | Some("Character") + ); + if item.faction == UNSORTED { + if let Some(ef) = &entry.faction { + if !generic { + item.system = entry.system.clone().unwrap_or_else(|| item.system.clone()); + item.faction = ef.clone(); + } + } + if let Some(g) = &entry.group { + if g != "Unit Type" { + item.group = g.clone(); + } else { + item.group = Data::global() + .group_of_faction + .get(&item.faction) + .cloned() + .unwrap_or_else(|| UNSORTED.into()); + } + } else { + item.group = Data::global() + .group_of_faction + .get(&item.faction) + .cloned() + .unwrap_or_else(|| UNSORTED.into()); + } + } else if let Some(ef) = &entry.faction { + if !generic && ef != &item.faction { + // GW moved this unit to another faction (or the keyword + // tables are stale): trust the catalog. + item.system = entry.system.clone().unwrap_or_else(|| item.system.clone()); + item.faction = ef.clone(); + if let Some(g) = &entry.group { + if g != "Unit Type" { + item.group = g.clone(); + } else { + item.group = Data::global() + .group_of_faction + .get(ef) + .cloned() + .unwrap_or_else(|| UNSORTED.into()); + } + } else { + item.group = Data::global() + .group_of_faction + .get(ef) + .cloned() + .unwrap_or_else(|| UNSORTED.into()); + } + } + } else if let Some(g) = &entry.group { + if g != "Unit Type" { + item.group = g.clone(); + } + } + } else if *score >= WEAK_MIN && weak_tier_shared(item, *idx, ctx) { + // Weak tier: a same-system hit below the threshold with at least one + // shared word, accepted only when nothing stronger exists. Flagged + // low confidence. + apply_entry(item, ctx, *idx, *score, "weak", "low"); + } else { + implicit_entry(item, *score); + } +} + +/// Weak tier gating: outer name or one of the first inner names shares a +/// rare token with the entry (python lines 665-668). +fn weak_tier_shared(item: &SortItem, idx: usize, ctx: &SorterCtx) -> bool { + let entry = &ctx.catalog[idx]; + let shared_outer = shares_rare_token(&item.clean, &entry.clean) + || entry + .clean_aliases + .iter() + .any(|a| shares_rare_token(&item.clean, a)); + let shared_inner = item + .inner_names + .iter() + .take(10) + .any(|n| shares_rare_token(&clean_inner(n), &entry.clean)); + shared_outer || shared_inner +} + +/// Implicit entry: own (original) name as the unit; the faction stays as +/// classified (UNSORTED or keyword faction). +fn implicit_entry(item: &mut SortItem, score: f64) { + item.unit = Some(item.name.clone()); + item.match_score = score; + item.match_kind = "implicit".into(); +} + +// --------------------------------------------------------------------------- +// plan +// --------------------------------------------------------------------------- + +/// Implicit entries carry the original filename; strip the extension. +fn display_name(unit: &str) -> String { + if py_suffix(unit).is_empty() { + unit.to_string() + } else { + unit[..py_stem_split(unit)].to_string() + } +} + +/// Build the target path for one item. +/// `inner_rel`: dirs-only relative path under the extraction dir (provenance +/// mode) - the file name itself is appended last. +pub fn build_target(item: &SortItem, library: &Path, inner_rel: Option<&Path>) -> PathBuf { + let mut parts: Vec = Vec::new(); + if item.system == "Unknown" { + parts.push(UNSORTED.into()); + if inner_rel.is_some() && item.unit.is_some() { + // extracted archives keep their unit folder even when unplaced + parts.push(display_name(item.unit.as_ref().unwrap())); + } + } else { + parts.push(item.system.clone()); + if item.faction != UNSORTED { + if !item.group.is_empty() { + parts.push(item.group.clone()); + } + parts.push(item.faction.clone()); + if let Some(sf) = &item.subfaction { + parts.push(sf.clone()); + } + parts.push( + item.klass + .clone() + .unwrap_or_else(|| UNCLASSIFIED_CLASS.into()), + ); + if let Some(unit) = &item.unit { + parts.push(display_name(unit)); + } + } else { + parts.push(UNSORTED.into()); + } + } + let mut target_dir = library.to_path_buf(); + for p in parts { + target_dir = target_dir.join(p); + } + if let Some(inner) = inner_rel { + for p in inner.iter() { + if p != std::path::Component::CurDir.as_os_str() { + target_dir = target_dir.join(p); + } + } + } + target_dir.join(&item.name) +} + +/// provenance map: extraction dir (relative to `root`) -> archive name. +pub type Provenance = BTreeMap; + +/// Build sort items for `files`. +/// +/// `provenance` (with `root`) activates extracted-archive mode: it maps +/// extraction directories (relative to `root`) to the archive they came +/// from. All files inside such a directory are classified and matched once +/// under the archive's name and then placed together, keeping the archive's +/// inner structure - a loose "Backpack.stl" alone is noise, under "The Twin +/// Lance/" it belongs to its unit. +pub fn sort_plan( + files: &[PathBuf], + library: &Path, + catalog: &[CatalogEntry], + threshold: f64, + provenance: Option<&Provenance>, + root: Option<&Path>, +) -> Vec { + let ctx = SorterCtx { + catalog: catalog.to_vec(), + }; + let mut cache = MatchCache::default(); + let mut unit_cache: BTreeMap> = BTreeMap::new(); + let mut items = Vec::new(); + let mut sorted_files: Vec<&PathBuf> = files.iter().collect(); + sorted_files.sort(); + for f in sorted_files { + if !f.is_file() { + continue; + } + let fname = f + .file_name() + .map(|n| n.to_string_lossy().into_owned()) + .unwrap_or_default(); + if fname.ends_with(".part") + || fname.starts_with('.') + || RUN_ARTIFACTS.contains(&fname.as_str()) + { + continue; + } + let mut unit_name: Option = None; + let mut dest_key: Option = None; + let mut inner_rel: Option = None; + let prov = provenance; + let rootp = root; + if let (Some(rootp), Some(prov)) = (rootp, prov) { + if let Ok(rel) = f.strip_prefix(rootp) { + let mut p = rel.parent().unwrap_or_else(|| Path::new("")); + loop { + if p.as_os_str().is_empty() { + break; + } + if let Some(unit) = prov.get(&p.to_string_lossy().into_owned()) { + dest_key = Some(p.to_string_lossy().into_owned()); + unit_name = Some(unit.clone()); + // dirs-only relative path (the file name itself is + // appended by build_target; keeping it here would + // duplicate it in the target path) + let rest = rel.strip_prefix(p).unwrap(); + inner_rel = + Some(rest.parent().unwrap_or_else(|| Path::new("")).to_path_buf()); + break; + } + match p.parent() { + Some(par) => p = par, + None => break, + } + } + } + } + if unit_name.is_none() { + let mut item = inspect(f); + classify_item(&mut item); + drop_dropped_system(&mut item); + match_catalog(&mut item, &ctx, threshold, &mut cache); + if let Some(rootp) = rootp { + // manual trees: let the real ancestor dir names compete + let Ok(rel) = f.strip_prefix(rootp) else { + items.push(item); + continue; + }; + let dir_parts: Vec = rel + .parent() + .unwrap_or_else(|| Path::new("")) + .iter() + .map(|p| p.to_os_string()) + .collect(); + let mut dirs: Vec<(PathBuf, String)> = Vec::new(); + for i in (0..dir_parts.len()).rev() { + let mut d = rootp.to_path_buf(); + for part in &dir_parts[..=i] { + d = d.join(part); + } + dirs.push((d, dir_parts[i].to_string_lossy().into_owned())); + } + let winner = rehome_from_dirs(&mut item, &dirs, &ctx, threshold, &mut cache, false); + if let Some(winner) = winner { + if let Ok(inner) = f + .parent() + .unwrap_or_else(|| Path::new("")) + .strip_prefix(&winner) + { + inner_rel = Some(inner.to_path_buf()); + } + } + } + item.target = Some(build_target(&item, library, inner_rel.as_deref())); + item.action = "link".into(); + items.push(item); + } else { + // one classification per extraction directory, reused for every + // file inside it + let dk = dest_key.unwrap(); + let un = unit_name.unwrap(); + if !unit_cache.contains_key(&dk) { + let listing: Vec = { + let mut all: Vec = Vec::new(); + collect_relative_files(&rootp.unwrap().join(&dk), &mut all); + all.sort(); + all.truncate(40); + all + }; + let mut probe = SortItem::bare(f.clone(), 0, un.clone(), clean_name(&un)); + probe.inner_names = listing; + classify_item(&mut probe); + drop_dropped_system(&mut probe); + match_catalog(&mut probe, &ctx, threshold, &mut cache); + unit_cache.insert( + dk.clone(), + Some(Resolution { + system: probe.system.clone(), + group: probe.group.clone(), + faction: probe.faction.clone(), + subfaction: probe.subfaction.clone(), + klass: probe.klass.clone(), + unit: probe.unit.clone(), + match_score: probe.match_score, + match_kind: probe.match_kind.clone(), + confidence: probe.confidence.clone(), + }), + ); + } + let fields = unit_cache[&dk].clone().unwrap(); + let mut item = inspect(f); + item.system = fields.system; + item.group = fields.group; + item.faction = fields.faction; + item.subfaction = fields.subfaction; + item.klass = fields.klass; + item.unit = fields.unit; + item.match_score = fields.match_score; + item.match_kind = fields.match_kind; + item.confidence = fields.confidence; + if let Some(ir) = &inner_rel { + if ir.iter().count() > 0 { + // extracted archives: inner dir names may claim subtrees + let mut dirs: Vec<(PathBuf, String)> = Vec::new(); + let parts: Vec = + ir.iter().map(|p| p.to_os_string()).collect(); + for i in (0..parts.len()).rev() { + let mut d = rootp.unwrap().join(&dk); + for part in &parts[..=i] { + d = d.join(part); + } + dirs.push((d, parts[i].to_string_lossy().into_owned())); + } + let winner = + rehome_from_dirs(&mut item, &dirs, &ctx, threshold, &mut cache, true); + if let Some(winner) = winner { + if let Ok(inner) = f + .parent() + .unwrap_or_else(|| Path::new("")) + .strip_prefix(&winner) + { + inner_rel = Some(inner.to_path_buf()); + } + } + } + } + item.target = Some(build_target(&item, library, inner_rel.as_deref())); + item.action = "link".into(); + items.push(item); + } + } + items +} + +/// One extraction directory's resolved classification, reused per file. +#[derive(Debug, Clone)] +struct Resolution { + system: String, + group: String, + faction: String, + subfaction: Option, + klass: Option, + unit: Option, + match_score: f64, + match_kind: String, + confidence: String, +} + +/// Files under `dir`, as relative paths (helper for the provenance listing). +/// python: `g.relative_to(root / dest_key)` over `(root / dest_key).rglob("")` +/// — full relative paths including subdirectories. +fn collect_relative_files(dir: &Path, out: &mut Vec) { + let Ok(rd) = fs::read_dir(dir) else { return }; + let mut entries: Vec<_> = rd.filter_map(|e| e.ok()).map(|e| e.path()).collect(); + entries.sort(); + for p in entries { + if p.is_dir() { + let rel = p.strip_prefix(dir).unwrap(); + collect_relative_files_prefixed(&p, rel, out); + } else if p.is_file() { + if let Ok(rel) = p.strip_prefix(dir) { + out.push(rel.to_string_lossy().into_owned()); + } + } + } +} + +fn collect_relative_files_prefixed(dir: &Path, prefix: &Path, out: &mut Vec) { + let Ok(rd) = fs::read_dir(dir) else { return }; + let mut entries: Vec<_> = rd.filter_map(|e| e.ok()).map(|e| e.path()).collect(); + entries.sort(); + for p in entries { + if p.is_dir() { + let rel = p.strip_prefix(dir).unwrap(); + collect_relative_files_prefixed(&p, &prefix.join(rel), out); + } else if p.is_file() { + out.push( + prefix + .join(p.strip_prefix(dir).unwrap()) + .to_string_lossy() + .into_owned(), + ); + } + } +} + +// --------------------------------------------------------------------------- +// apply +// --------------------------------------------------------------------------- + +/// Create targets. Hardlink preferred; copy when filesystems differ. +/// Returns (linked, copied, skipped). +pub fn apply_plan( + items: &mut [SortItem], + audit_path: &Path, +) -> anyhow::Result<(usize, usize, usize)> { + if let Some(parent) = audit_path.parent() { + fs::create_dir_all(parent)?; + } + let mut linked = 0; + let mut copied = 0; + let mut skipped = 0; + let mut weak = 0; + let mut weak_lines: Vec = Vec::new(); + let mut audit = std::fs::OpenOptions::new() + .create(true) + .append(true) + .open(audit_path)?; + use std::io::Write; + for item in items.iter_mut() { + let src = &item.path; + if item.match_kind == "invalid" { + log::error!( + "Not sorting {}: it is an HTML page, not an archive", + item.name + ); + skipped += 1; + continue; + } + let Some(dest) = item.target.clone() else { + skipped += 1; + continue; + }; + if !src.is_file() { + skipped += 1; + continue; + } + if dest.exists() { + log::info!("Skip, target exists: {}", dest.display()); + skipped += 1; + continue; + } + if let Some(parent) = dest.parent() { + fs::create_dir_all(parent)?; + } + if same_device(src, dest.parent().unwrap()) { + fs::hard_link(src, &dest)?; + item.action = "link".into(); + linked += 1; + } else { + log::warn!( + "Different filesystem - COPYING instead of hardlinking {}", + item.name + ); + fs::copy(src, &dest)?; + item.action = "copy".into(); + copied += 1; + } + if item.match_kind == "weak" { + weak += 1; + weak_lines.push(format!( + "{} (score {:.2})\n -> {}\n from {}\n", + item.name, + item.match_score, + dest.display(), + src.display() + )); + } + writeln!( + audit, + "{}", + serde_json::json!({ + "from": src.display().to_string(), + "to": dest.display().to_string(), + "action": item.action, + "unit": item.unit, + "faction": item.faction, + "class": item.klass, + "match": item.match_kind, + "confidence": item.confidence, + }) + )?; + } + if weak > 0 { + let weak_doc = audit_path + .parent() + .unwrap_or_else(|| Path::new(".")) + .join("sorter_low_confidence.txt"); + fs::write( + &weak_doc, + format!( + "Low-confidence placements - the sorter accepted these below the \ + normal\nthreshold because nothing better matched. Check and move \ + manually if wrong.\n\n{}", + weak_lines.join("") + ), + )?; + log::warn!( + "{} low-confidence placements - see {}", + weak, + weak_doc.display() + ); + } + Ok((linked, copied, skipped)) +} + +fn same_device(a: &Path, b: &Path) -> bool { + match (fs::metadata(a), fs::metadata(b)) { + (Ok(ma), Ok(mb)) => ma.dev() == mb.dev(), + _ => false, + } +} + +/// Render a plan like python format_plan (the `--dry-run` printable output). +pub fn format_plan(items: &[SortItem]) -> String { + let mut lines: Vec = Vec::new(); + for item in items { + if item.match_kind == "invalid" { + lines.push(format!( + "[x] {} - NOT AN ARCHIVE (HTML page, bad download)", + item.name + )); + continue; + } + let src = if item.size > 4 * 1024 * 1024 { + format!("{} ({} MiB)", item.name, item.size / (1024 * 1024)) + } else { + item.name.clone() + }; + let flag = match item.match_kind.as_str() { + "catalog" => "+", + "weak" => "w", + "implicit" => "~", + _ => "?", + }; + lines.push(format!( + "[{flag}] {src}\n -> {}", + item.target + .as_ref() + .map(|t| t.display().to_string()) + .unwrap_or_default() + )); + } + lines.join("\n") +} + +#[cfg(test)] +mod tests { + use super::*; + + /// Same fixture as version-python/tests/test_sorter_units.py. + fn test_catalog_json() -> String { + serde_json::json!({ + "entries": [ + {"name": "Warp Spiders", "system": "Warhammer 40,000", + "group": "Xenos Armies", "faction": "Aeldari", + "class": "Fly", "aliases": []}, + {"name": "Harlequin Troupe", "system": "Warhammer 40,000", + "group": "Xenos Armies", "faction": "Harlequins", + "class": "Battleline", "aliases": []}, + {"name": "Guardian Squad", "system": "Warhammer 40,000", + "group": "Xenos Armies", "faction": "Aeldari", + "class": "Battleline", "aliases": []}, + {"name": "Rangers", "system": "Warhammer 40,000", + "group": "Xenos Armies", "faction": "Aeldari", + "class": "Battleline", "aliases": []}, + {"name": "Dark Reapers", "system": "Warhammer 40,000", + "group": "Xenos Armies", "faction": "Aeldari", + "class": "Aspect Warriors", "aliases": []}, + {"name": "Mordor Orcs", "system": "Middle-Earth", + "group": "The Lord of the Rings™ - Evil", "faction": "Mordor", + "class": "Warriors", "aliases": []} + ] + }) + .to_string() + } + + fn load_test_catalog() -> Vec { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("catalog.json"); + std::fs::write(&path, test_catalog_json()).unwrap(); + // load_catalog borrows the path only; copy the result out + load_catalog(&path, &dropped_systems_default(&[], &[])).unwrap() + } + + fn write(p: &Path) { + std::fs::create_dir_all(p.parent().unwrap()).unwrap(); + std::fs::write(p, b"x").unwrap(); + } + + fn by_name<'a>(items: &'a [SortItem], name: &str) -> &'a SortItem { + items.iter().find(|i| i.name == name).unwrap() + } + + #[test] + fn suffix_helpers_match_python() { + assert_eq!(py_suffix("x.tar.gz"), ".gz"); + assert_eq!( + display_name("Thunderwolf Cavalry.7z"), + "Thunderwolf Cavalry" + ); + assert_eq!(display_name("plain name"), "plain name"); + assert!(is_junk_dir("HEADS")); + assert!(is_junk_dir("files")); + assert!(is_junk_dir("123")); + assert!(!is_junk_dir("Warp Spiders")); + } + + #[test] + fn prov_dir_rehome() { + // A matching inner dir name claims its subtree; junk dirs don't. + let root = tempfile::tempdir().unwrap(); + let pack = root.path().join("Multi Pack"); + write(&pack.join("14 Warp spiders").join("spider_body.stl")); + write( + &pack + .join("14 Warp spiders") + .join("files") + .join("spider_arm.stl"), + ); + write(&pack.join("HEADS").join("head_a.stl")); + write(&pack.join("Retro GrimDark Thing").join("t.stl")); + let library = tempfile::tempdir().unwrap(); + let catalog = load_test_catalog(); + let prov: Provenance = [("Multi Pack".to_string(), "Multi Pack.zip".to_string())] + .into_iter() + .collect(); + let files: Vec = walk_sorted(pack.as_path()).into_iter().collect(); + let items = sort_plan( + &files, + library.path(), + &catalog, + DEFAULT_THRESHOLD, + Some(&prov), + Some(root.path()), + ); + let tgt = |n: &str| { + by_name(&items, n) + .target + .as_ref() + .unwrap() + .display() + .to_string() + }; + let t = tgt("spider_body.stl"); + assert!(t.contains("Warp Spiders"), "{t}"); + assert!(t.contains("Fly"), "{t}"); + assert!(!t.contains("Hero"), "{t}"); + assert!(tgt("spider_arm.stl").contains("files")); + // junk-dir file rides along with the pack's adopted unit + let t = tgt("head_a.stl"); + assert!(t.contains("Warp Spiders"), "{t}"); + assert!(t.contains("HEADS"), "{t}"); + // unmatched subtree keeps the pack unit folder + let t = tgt("t.stl"); + assert!( + t.contains("Warp Spiders") && t.contains("Retro GrimDark Thing"), + "{t}" + ); + } + + #[test] + fn prov_weak_tier() { + // Below-threshold dir hit is accepted as low confidence. + let root = tempfile::tempdir().unwrap(); + let pack = root.path().join("Aeldari Pack"); + write(&pack.join("guardian weapon team").join("g.stl")); + let library = tempfile::tempdir().unwrap(); + let catalog = load_test_catalog(); + let prov: Provenance = [("Aeldari Pack".to_string(), "Aeldari Pack.zip".to_string())] + .into_iter() + .collect(); + let files: Vec = walk_sorted(pack.as_path()).into_iter().collect(); + let items = sort_plan( + &files, + library.path(), + &catalog, + DEFAULT_THRESHOLD, + Some(&prov), + Some(root.path()), + ); + let it = by_name(&items, "g.stl"); + assert_eq!(it.match_kind, "weak", "{}", it.match_score); + assert_eq!(it.confidence, "low"); + assert!( + it.target + .as_ref() + .unwrap() + .display() + .to_string() + .contains("Guardian") + ); + } + + #[test] + fn cross_system_needs_token_containment() { + // 'Aeldari Rangers' must not adopt Middle-earth 'Dwarf Rangers'. + let root = tempfile::tempdir().unwrap(); + let pack = root.path().join("MQVxKzh"); // hash name: system stays unknown + write( + &pack + .join("Aeldari Rangers") + .join("Luminary - Arms - (1).stl"), + ); + write(&pack.join("14 Warp spiders").join("sp.stl")); + let library = tempfile::tempdir().unwrap(); + let catalog = load_test_catalog(); + let prov: Provenance = [("MQVxKzh".to_string(), "MQVxKzh.zip".to_string())] + .into_iter() + .collect(); + let files: Vec = walk_sorted(pack.as_path()).into_iter().collect(); + let items = sort_plan( + &files, + library.path(), + &catalog, + DEFAULT_THRESHOLD, + Some(&prov), + Some(root.path()), + ); + let lum = by_name(&items, "Luminary - Arms - (1).stl"); + assert!( + !lum.target + .as_ref() + .unwrap() + .display() + .to_string() + .contains("Middle-Earth") + ); + let spider = by_name(&items, "sp.stl"); + assert!( + spider + .target + .as_ref() + .unwrap() + .display() + .to_string() + .contains("Warp Spiders") + ); + } + + #[test] + fn weak_needs_shared_token() { + // 'frontline squad' vs 'Guardian Squad' scores 0.552 but shares no + // token - it must NOT re-home, while 'guardian weapon team' (0.588, + // shares 'guardian') is accepted weak. Both files ride along with + // the pack-level weak adoption. + let root = tempfile::tempdir().unwrap(); + let pack = root.path().join("Aeldari Pack"); + write(&pack.join("frontline squad").join("f.stl")); + write(&pack.join("guardian weapon team").join("g.stl")); + let library = tempfile::tempdir().unwrap(); + let catalog = load_test_catalog(); + let prov: Provenance = [("Aeldari Pack".to_string(), "Aeldari Pack.zip".to_string())] + .into_iter() + .collect(); + let files: Vec = walk_sorted(pack.as_path()).into_iter().collect(); + let items = sort_plan( + &files, + library.path(), + &catalog, + DEFAULT_THRESHOLD, + Some(&prov), + Some(root.path()), + ); + let f = by_name(&items, "f.stl"); + assert_eq!( + f.match_kind, + "weak", + "{}", + f.target.as_ref().unwrap().display() + ); + assert_eq!(f.confidence, "low"); + assert!( + f.target + .as_ref() + .unwrap() + .display() + .to_string() + .contains("frontline squad") + ); + let g = by_name(&items, "g.stl"); + assert_eq!(g.match_kind, "weak"); + assert_eq!(g.confidence, "low"); + } + + #[test] + fn tokenless_needs_bonus() { + // 'Aeldari Rangers' (0.667 vs 'Dark Reapers', tokenless) must not + // adopt Dark Reapers; it instead adopts 'Rangers' (0.636, shares + // 'ranger'). + let root = tempfile::tempdir().unwrap(); + let pack = root.path().join("Aeldari Pack"); + write(&pack.join("Aeldari Rangers").join("r.stl")); + let library = tempfile::tempdir().unwrap(); + let catalog = load_test_catalog(); + let prov: Provenance = [("Aeldari Pack".to_string(), "Aeldari Pack.zip".to_string())] + .into_iter() + .collect(); + let files: Vec = walk_sorted(pack.as_path()).into_iter().collect(); + let items = sort_plan( + &files, + library.path(), + &catalog, + DEFAULT_THRESHOLD, + Some(&prov), + Some(root.path()), + ); + let it = by_name(&items, "r.stl"); + assert_eq!(it.match_kind, "catalog"); + assert_eq!( + it.unit.as_deref(), + Some("Rangers"), + "score={}", + it.match_score + ); + } + + #[test] + fn dropped_system_mordor() { + // Middle-earth is dropped: a 'Mordor' pack lands in _unsorted, and + // its inner dirs cannot adopt the (filtered) Mordor Orcs entry. + let root = tempfile::tempdir().unwrap(); + write(&root.path().join("Mordor pack").join("boy.stl")); + let library = tempfile::tempdir().unwrap(); + let catalog = load_test_catalog(); + let prov: Provenance = [("Mordor pack".to_string(), "Mordor pack.zip".to_string())] + .into_iter() + .collect(); + let files: Vec = walk_sorted(root.path()).into_iter().collect(); + let items = sort_plan( + &files, + library.path(), + &catalog, + DEFAULT_THRESHOLD, + Some(&prov), + Some(root.path()), + ); + let it = &items[0]; + let t = it.target.as_ref().unwrap().display().to_string(); + assert!(!t.contains("Middle-Earth"), "{t}"); + assert!(t.contains("_unsorted/Mordor pack"), "{t}"); + } + + #[test] + fn load_catalog_filters_dropped() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("catalog.json"); + std::fs::write(&path, test_catalog_json()).unwrap(); + let cat = load_catalog(&path, &dropped_systems_default(&[], &[])).unwrap(); + assert!( + !cat.iter() + .any(|e| e.system.as_deref() == Some("Middle-Earth")) + ); + assert!(cat.iter().any(|e| e.name == "Warp Spiders")); + // keeping everything works too + let cat = load_catalog(&path, &[]).unwrap(); + assert!( + cat.iter() + .any(|e| e.system.as_deref() == Some("Middle-Earth")) + ); + } + + #[test] + fn nonprov_rehome() { + // Loose noise file inside a named dir re-homes to the dir's unit. + let root = tempfile::tempdir().unwrap(); + write(&root.path().join("Warp Spiders").join("body.stl")); + let library = tempfile::tempdir().unwrap(); + let catalog = load_test_catalog(); + let files: Vec = walk_sorted(root.path()).into_iter().collect(); + let items = sort_plan( + &files, + library.path(), + &catalog, + DEFAULT_THRESHOLD, + None, + Some(root.path()), + ); + let it = by_name(&items, "body.stl"); + assert_eq!( + it.match_kind, + "catalog", + "{}", + it.target.as_ref().unwrap().display() + ); + let t = it.target.as_ref().unwrap().display().to_string(); + assert!(t.contains("Warp Spiders") && t.contains("Fly"), "{t}"); + } + + #[test] + fn apply_weak_doc() { + // apply_plan writes the low-confidence review document. + let root = tempfile::tempdir().unwrap(); + let pack = root.path().join("Multi Pack"); + write(&pack.join("guardian squad extras").join("g.stl")); + let library = tempfile::tempdir().unwrap(); + let catalog = load_test_catalog(); + let prov: Provenance = [("Multi Pack".to_string(), "Multi Pack.zip".to_string())] + .into_iter() + .collect(); + let files: Vec = walk_sorted(pack.as_path()).into_iter().collect(); + let mut items = sort_plan( + &files, + library.path(), + &catalog, + DEFAULT_THRESHOLD, + Some(&prov), + Some(root.path()), + ); + let audit = library.path().join("sorter_applied.jsonl"); + let (_l, _c, _s) = apply_plan(&mut items, &audit).unwrap(); + let doc = library.path().join("sorter_low_confidence.txt"); + let weak_count = items.iter().filter(|i| i.match_kind == "weak").count(); + if weak_count > 0 { + assert!(doc.exists(), "weak doc missing"); + assert!( + std::fs::read_to_string(&doc) + .unwrap() + .contains("low-confidence") + ); + } else { + assert!(!doc.exists()); + } + assert!(audit.exists()); + } + + /// Sorted recursive file walk (python `sorted(dir.rglob("*"))`-ish). + fn walk_sorted(dir: &Path) -> Vec { + let mut out = Vec::new(); + let mut stack = vec![dir.to_path_buf()]; + while let Some(d) = stack.pop() { + let Ok(rd) = fs::read_dir(&d) else { continue }; + let mut entries: Vec = rd.filter_map(|e| e.ok()).map(|e| e.path()).collect(); + entries.sort(); + for p in entries { + if p.is_dir() { + stack.push(p); + } else { + out.push(p); + } + } + } + out.sort(); + out + } +}