Ben_Kosytorz
f631e351f8
Bugs: - extract: pick the OUTERMOST archive suffix (sfx.first(), not last()) so x.tar.gz uses the gz->tar path - gwshop: truncate the error snippet by chars, not bytes (UTF-8 panic) - sorter: py_stem_split now matches pathlib (0 < i < len-1) so a trailing dot is not treated as a suffix - sorter::apply_plan: a per-file link/copy/mkdir failure is logged and skipped instead of aborting the whole apply (never-abort contract) - orchestrate/state: add State::mark_logged and use it for every per-URL record so a state-save error cannot abort a run Cleanups: - remove dead is_archive_name and unused SourceInfo.manifest field - mark kept-but-unused capabilities with allow(dead_code) + rationale - let-chains / is_ok_and / is_multiple_of / replace([..]) / sort_by_key and other idiomatic simplifications across modules - difflib: SequenceMatcher owns its char vecs (bit-identical parity kept) - --only now enforces gofile|0807 via value_parser Docs: - SORTING.md: drop phantom sorter_plan.jsonl, os.link -> std::fs::hard_link, zipfile -> the zip crate - README: add --since to the dlp synopsis, clarify --subs/--assume-done - .gitignore: ignore catalogue/sorter/_logs/subscriptions run artifacts Validation: cargo fmt --check clean, cargo clippy --all-targets 0 warnings, cargo test 87 passed / 0 failed.
265 lines
10 KiB
Rust
265 lines
10 KiB
Rust
//! Port of CPython's `difflib.SequenceMatcher` (Ratcliff/Obershelp) — the
|
|
//! exact ratio algorithm the python sorter relies on. A different fuzzy
|
|
//! metric (strsim etc.) would produce different scores and silently change
|
|
//! which files land where, so this is a faithful port including
|
|
//! real_quick_ratio/quick_ratio pruning and autojunk.
|
|
//!
|
|
//! Constructed like the python sorter uses it: `SequenceMatcher(None, a, b)`
|
|
//! — no external junk patterns, autojunk enabled (it only matters for
|
|
//! sequences of length >= 200).
|
|
|
|
/// Longest matching block: (i in a, j in b, size).
|
|
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
|
pub struct Match {
|
|
i: usize,
|
|
j: usize,
|
|
size: usize,
|
|
}
|
|
|
|
pub struct SequenceMatcher {
|
|
/// `a`/`b` as chars, collected once (python works on code points; the
|
|
/// matching routines index them repeatedly, so collecting here avoids
|
|
/// re-allocating on every `find_longest_match` call).
|
|
a_chars: Vec<char>,
|
|
b_chars: Vec<char>,
|
|
/// b[j] -> indices (purged of junk and popular elements, like `b2j`).
|
|
b2j: std::collections::HashMap<char, Vec<usize>>,
|
|
bjunk: std::collections::HashSet<char>,
|
|
}
|
|
|
|
impl SequenceMatcher {
|
|
pub fn new(a: &str, b: &str) -> Self {
|
|
// __chain_b with isjunk=None: no junk, but autojunk purges
|
|
// "popular" characters from long sequences (len(b) >= 200).
|
|
let a_chars: Vec<char> = a.chars().collect();
|
|
let b_chars: Vec<char> = b.chars().collect();
|
|
let mut b2j: std::collections::HashMap<char, Vec<usize>> = std::collections::HashMap::new();
|
|
for (idx, &elt) in b_chars.iter().enumerate() {
|
|
b2j.entry(elt).or_default().push(idx);
|
|
}
|
|
let bjunk = std::collections::HashSet::new(); // isjunk is None
|
|
let n = b_chars.len();
|
|
if n >= 200 {
|
|
let ntest = n / 100 + 1;
|
|
let mut bpopular = std::collections::HashSet::new();
|
|
for (&elt, idxs) in &b2j {
|
|
if idxs.len() > ntest {
|
|
bpopular.insert(elt);
|
|
}
|
|
}
|
|
for &elt in &bpopular {
|
|
b2j.remove(&elt);
|
|
}
|
|
}
|
|
SequenceMatcher {
|
|
a_chars,
|
|
b_chars,
|
|
b2j,
|
|
bjunk,
|
|
}
|
|
}
|
|
|
|
fn isbjunk(&self, elt: char) -> bool {
|
|
self.bjunk.contains(&elt)
|
|
}
|
|
|
|
/// CPython SequenceMatcher.find_longest_match(alo, ahi, blo, bhi) over
|
|
/// char indices.
|
|
#[allow(clippy::needless_range_loop)] // `i` feeds the besti/bestj arithmetic
|
|
fn find_longest_match(&self, alo: usize, ahi: usize, blo: usize, bhi: usize) -> Match {
|
|
let a_chars = &self.a_chars;
|
|
let b_chars = &self.b_chars;
|
|
let nothing: Vec<usize> = Vec::new();
|
|
|
|
let mut besti = alo;
|
|
let mut bestj = blo;
|
|
let mut bestsize = 0usize;
|
|
// j2len[j] = length of the longest match ending at (a[i], b[j])
|
|
let mut j2len: std::collections::HashMap<usize, usize> = std::collections::HashMap::new();
|
|
for i in alo..ahi {
|
|
let mut newj2len: std::collections::HashMap<usize, usize> =
|
|
std::collections::HashMap::new();
|
|
for &j in self.b2j.get(&a_chars[i]).unwrap_or(¬hing) {
|
|
if j < blo {
|
|
continue;
|
|
}
|
|
if j >= bhi {
|
|
break;
|
|
}
|
|
let k = j2len.get(&(j.wrapping_sub(1))).copied().unwrap_or(0) + 1;
|
|
newj2len.insert(j, k);
|
|
if k > bestsize {
|
|
besti = i + 1 - k;
|
|
bestj = j + 1 - k;
|
|
bestsize = k;
|
|
}
|
|
}
|
|
j2len = newj2len;
|
|
}
|
|
|
|
// Extend the best by non-junk elements on each end.
|
|
while besti > alo
|
|
&& bestj > blo
|
|
&& !self.isbjunk(b_chars[bestj - 1])
|
|
&& a_chars[besti - 1] == b_chars[bestj - 1]
|
|
{
|
|
besti -= 1;
|
|
bestj -= 1;
|
|
bestsize += 1;
|
|
}
|
|
while besti + bestsize < ahi
|
|
&& bestj + bestsize < bhi
|
|
&& !self.isbjunk(b_chars[bestj + bestsize])
|
|
&& a_chars[besti + bestsize] == b_chars[bestj + bestsize]
|
|
{
|
|
bestsize += 1;
|
|
}
|
|
// Then suck up matching junk on each side of it.
|
|
while besti > alo
|
|
&& bestj > blo
|
|
&& self.isbjunk(b_chars[bestj - 1])
|
|
&& a_chars[besti - 1] == b_chars[bestj - 1]
|
|
{
|
|
besti -= 1;
|
|
bestj -= 1;
|
|
bestsize += 1;
|
|
}
|
|
while besti + bestsize < ahi
|
|
&& bestj + bestsize < bhi
|
|
&& self.isbjunk(b_chars[bestj + bestsize])
|
|
&& a_chars[besti + bestsize] == b_chars[bestj + bestsize]
|
|
{
|
|
bestsize += 1;
|
|
}
|
|
Match {
|
|
i: besti,
|
|
j: bestj,
|
|
size: bestsize,
|
|
}
|
|
}
|
|
|
|
/// All matching blocks, in order (recursive queue algorithm).
|
|
pub fn get_matching_blocks(&self) -> Vec<Match> {
|
|
let la = self.a_chars.len();
|
|
let lb = self.b_chars.len();
|
|
let mut blocks: Vec<Match> = Vec::new();
|
|
let mut queue: Vec<(usize, usize, usize, usize)> = vec![(0, la, 0, lb)];
|
|
while let Some((alo, ahi, blo, bhi)) = queue.pop() {
|
|
let m = self.find_longest_match(alo, ahi, blo, bhi);
|
|
if m.size > 0 {
|
|
blocks.push(m);
|
|
if alo < m.i && blo < m.j {
|
|
queue.push((alo, m.i, blo, m.j));
|
|
}
|
|
if m.i + m.size < ahi && m.j + m.size < bhi {
|
|
queue.push((m.i + m.size, ahi, m.j + m.size, bhi));
|
|
}
|
|
}
|
|
}
|
|
blocks.sort_by_key(|m| (m.i, m.j, m.size));
|
|
blocks
|
|
}
|
|
|
|
fn calculate_ratio(matches: usize, la: usize, lb: usize) -> f64 {
|
|
let length = la + lb;
|
|
if length > 0 {
|
|
2.0 * matches as f64 / length as f64
|
|
} else {
|
|
1.0
|
|
}
|
|
}
|
|
|
|
/// Upper bound computed from the cardinality of the char multiset
|
|
/// intersection (python `quick_ratio`).
|
|
pub fn quick_ratio(&self) -> f64 {
|
|
let mut fullbcount: std::collections::HashMap<char, usize> =
|
|
std::collections::HashMap::new();
|
|
for &elt in &self.b_chars {
|
|
*fullbcount.entry(elt).or_default() += 1;
|
|
}
|
|
let mut avail: std::collections::HashMap<char, usize> = std::collections::HashMap::new();
|
|
let mut matches = 0usize;
|
|
for &elt in &self.a_chars {
|
|
let numb = avail
|
|
.get(&elt)
|
|
.copied()
|
|
.unwrap_or_else(|| fullbcount.get(&elt).copied().unwrap_or(0));
|
|
if numb > 0 {
|
|
avail.insert(elt, numb - 1);
|
|
matches += 1;
|
|
}
|
|
}
|
|
Self::calculate_ratio(matches, self.a_chars.len(), self.b_chars.len())
|
|
}
|
|
|
|
/// Trivial upper bound (python `real_quick_ratio`).
|
|
pub fn real_quick_ratio(&self) -> f64 {
|
|
let la = self.a_chars.len();
|
|
let lb = self.b_chars.len();
|
|
Self::calculate_ratio(la.min(lb), la, lb)
|
|
}
|
|
|
|
/// Ratcliff/Obershelp similarity in [0, 1], exactly as python's
|
|
/// `SequenceMatcher(None, a, b).ratio()`.
|
|
pub fn ratio(&self) -> f64 {
|
|
let matches: usize = self.get_matching_blocks().iter().map(|m| m.size).sum();
|
|
Self::calculate_ratio(matches, self.a_chars.len(), self.b_chars.len())
|
|
}
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod tests {
|
|
use super::*;
|
|
|
|
/// Expected values generated with CPython 3.12 difflib (see
|
|
/// tools/dump_difflib_vectors.py to regenerate).
|
|
#[rustfmt::skip]
|
|
const VECTORS: &[( &str, &str, f64, f64, f64 )] = &[
|
|
// a, b, real_quick_ratio, quick_ratio, ratio
|
|
("telemon", "telemon heavy dreadnought", 0.4375, 0.4375, 0.4375),
|
|
("aeldari rangers", "dark reapers", 0.8888888888888888, 0.7407407407407407, 0.6666666666666666),
|
|
("aeldari rangers", "dwarf rangers", 0.9285714285714286, 0.7857142857142857, 0.7857142857142857),
|
|
("guardian weapon team", "guardian squad", 0.8235294117647058, 0.5882352941176471, 0.5882352941176471),
|
|
("frontline squad", "guardian squad", 0.9655172413793104, 0.6206896551724138, 0.5517241379310345),
|
|
("2e catachany fellaz catachan jungle fighters x", "cadian command squad", 0.6060606060606061, 0.3939393939393939, 0.18181818181818182),
|
|
("warp spiders", "warp spiders", 1.0, 1.0, 1.0),
|
|
("thunderwolf cavalry", "thunderwolf mount", 0.9444444444444444, 0.6666666666666666, 0.6666666666666666),
|
|
("lictor", "necron lieutenant with voidwalker", 0.3076923076923077, 0.3076923076923077, 0.2564102564102564),
|
|
("aeldari pack", "harlequin troupe", 0.8571428571428571, 0.5, 0.35714285714285715),
|
|
("xyzzy", "warp spiders", 0.5882352941176471, 0.0, 0.0),
|
|
("guardian defender", "guardian squad", 0.9032258064516129, 0.6451612903225806, 0.6451612903225806),
|
|
];
|
|
|
|
#[test]
|
|
fn matches_cpython_difflib_exactly() {
|
|
for (a, b, rqr, qr, r) in VECTORS {
|
|
let sm = SequenceMatcher::new(a, b);
|
|
assert_eq!(
|
|
sm.real_quick_ratio(),
|
|
*rqr,
|
|
"real_quick_ratio({a:?}, {b:?})"
|
|
);
|
|
assert_eq!(sm.quick_ratio(), *qr, "quick_ratio({a:?}, {b:?})");
|
|
assert_eq!(sm.ratio(), *r, "ratio({a:?}, {b:?})");
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn empty_and_long_sequences() {
|
|
assert_eq!(SequenceMatcher::new("", "").ratio(), 1.0);
|
|
assert_eq!(SequenceMatcher::new("", "abc").ratio(), 0.0);
|
|
// autojunk path (len(b) >= 200): "unit unit" vs "unit " * 50
|
|
let long_b = "unit ".repeat(50);
|
|
let sm = SequenceMatcher::new("unit unit", &long_b);
|
|
assert_eq!(sm.ratio(), 0.0694980694980695);
|
|
}
|
|
|
|
#[test]
|
|
fn symmetry_of_ratio_value() {
|
|
// documented CPython behavior: ratio values are symmetric
|
|
let ab = SequenceMatcher::new("aeldari rangers", "dwarf rangers").ratio();
|
|
let ba = SequenceMatcher::new("dwarf rangers", "aeldari rangers").ratio();
|
|
assert_eq!(ab, ba);
|
|
}
|
|
}
|