//! Port of CPython's `difflib.SequenceMatcher` (Ratcliff/Obershelp) — the //! exact ratio algorithm the python sorter relies on. A different fuzzy //! metric (strsim etc.) would produce different scores and silently change //! which files land where, so this is a faithful port including //! real_quick_ratio/quick_ratio pruning and autojunk. //! //! Constructed like the python sorter uses it: `SequenceMatcher(None, a, b)` //! — no external junk patterns, autojunk enabled (it only matters for //! sequences of length >= 200). /// Longest matching block: (i in a, j in b, size). #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub struct Match { i: usize, j: usize, size: usize, } pub struct SequenceMatcher { /// `a`/`b` as chars, collected once (python works on code points; the /// matching routines index them repeatedly, so collecting here avoids /// re-allocating on every `find_longest_match` call). a_chars: Vec, b_chars: Vec, /// b[j] -> indices (purged of junk and popular elements, like `b2j`). b2j: std::collections::HashMap>, bjunk: std::collections::HashSet, } impl SequenceMatcher { pub fn new(a: &str, b: &str) -> Self { // __chain_b with isjunk=None: no junk, but autojunk purges // "popular" characters from long sequences (len(b) >= 200). let a_chars: Vec = a.chars().collect(); let b_chars: Vec = b.chars().collect(); let mut b2j: std::collections::HashMap> = std::collections::HashMap::new(); for (idx, &elt) in b_chars.iter().enumerate() { b2j.entry(elt).or_default().push(idx); } let bjunk = std::collections::HashSet::new(); // isjunk is None let n = b_chars.len(); if n >= 200 { let ntest = n / 100 + 1; let mut bpopular = std::collections::HashSet::new(); for (&elt, idxs) in &b2j { if idxs.len() > ntest { bpopular.insert(elt); } } for &elt in &bpopular { b2j.remove(&elt); } } SequenceMatcher { a_chars, b_chars, b2j, bjunk, } } fn isbjunk(&self, elt: char) -> bool { self.bjunk.contains(&elt) } /// CPython SequenceMatcher.find_longest_match(alo, ahi, blo, bhi) over /// char indices. #[allow(clippy::needless_range_loop)] // `i` feeds the besti/bestj arithmetic fn find_longest_match(&self, alo: usize, ahi: usize, blo: usize, bhi: usize) -> Match { let a_chars = &self.a_chars; let b_chars = &self.b_chars; let nothing: Vec = Vec::new(); let mut besti = alo; let mut bestj = blo; let mut bestsize = 0usize; // j2len[j] = length of the longest match ending at (a[i], b[j]) let mut j2len: std::collections::HashMap = std::collections::HashMap::new(); for i in alo..ahi { let mut newj2len: std::collections::HashMap = std::collections::HashMap::new(); for &j in self.b2j.get(&a_chars[i]).unwrap_or(¬hing) { if j < blo { continue; } if j >= bhi { break; } let k = j2len.get(&(j.wrapping_sub(1))).copied().unwrap_or(0) + 1; newj2len.insert(j, k); if k > bestsize { besti = i + 1 - k; bestj = j + 1 - k; bestsize = k; } } j2len = newj2len; } // Extend the best by non-junk elements on each end. while besti > alo && bestj > blo && !self.isbjunk(b_chars[bestj - 1]) && a_chars[besti - 1] == b_chars[bestj - 1] { besti -= 1; bestj -= 1; bestsize += 1; } while besti + bestsize < ahi && bestj + bestsize < bhi && !self.isbjunk(b_chars[bestj + bestsize]) && a_chars[besti + bestsize] == b_chars[bestj + bestsize] { bestsize += 1; } // Then suck up matching junk on each side of it. while besti > alo && bestj > blo && self.isbjunk(b_chars[bestj - 1]) && a_chars[besti - 1] == b_chars[bestj - 1] { besti -= 1; bestj -= 1; bestsize += 1; } while besti + bestsize < ahi && bestj + bestsize < bhi && self.isbjunk(b_chars[bestj + bestsize]) && a_chars[besti + bestsize] == b_chars[bestj + bestsize] { bestsize += 1; } Match { i: besti, j: bestj, size: bestsize, } } /// All matching blocks, in order (recursive queue algorithm). pub fn get_matching_blocks(&self) -> Vec { let la = self.a_chars.len(); let lb = self.b_chars.len(); let mut blocks: Vec = Vec::new(); let mut queue: Vec<(usize, usize, usize, usize)> = vec![(0, la, 0, lb)]; while let Some((alo, ahi, blo, bhi)) = queue.pop() { let m = self.find_longest_match(alo, ahi, blo, bhi); if m.size > 0 { blocks.push(m); if alo < m.i && blo < m.j { queue.push((alo, m.i, blo, m.j)); } if m.i + m.size < ahi && m.j + m.size < bhi { queue.push((m.i + m.size, ahi, m.j + m.size, bhi)); } } } blocks.sort_by_key(|m| (m.i, m.j, m.size)); blocks } fn calculate_ratio(matches: usize, la: usize, lb: usize) -> f64 { let length = la + lb; if length > 0 { 2.0 * matches as f64 / length as f64 } else { 1.0 } } /// Upper bound computed from the cardinality of the char multiset /// intersection (python `quick_ratio`). pub fn quick_ratio(&self) -> f64 { let mut fullbcount: std::collections::HashMap = std::collections::HashMap::new(); for &elt in &self.b_chars { *fullbcount.entry(elt).or_default() += 1; } let mut avail: std::collections::HashMap = std::collections::HashMap::new(); let mut matches = 0usize; for &elt in &self.a_chars { let numb = avail .get(&elt) .copied() .unwrap_or_else(|| fullbcount.get(&elt).copied().unwrap_or(0)); if numb > 0 { avail.insert(elt, numb - 1); matches += 1; } } Self::calculate_ratio(matches, self.a_chars.len(), self.b_chars.len()) } /// Trivial upper bound (python `real_quick_ratio`). pub fn real_quick_ratio(&self) -> f64 { let la = self.a_chars.len(); let lb = self.b_chars.len(); Self::calculate_ratio(la.min(lb), la, lb) } /// Ratcliff/Obershelp similarity in [0, 1], exactly as python's /// `SequenceMatcher(None, a, b).ratio()`. pub fn ratio(&self) -> f64 { let matches: usize = self.get_matching_blocks().iter().map(|m| m.size).sum(); Self::calculate_ratio(matches, self.a_chars.len(), self.b_chars.len()) } } #[cfg(test)] mod tests { use super::*; /// Expected values generated with CPython 3.12 difflib (see /// tools/dump_difflib_vectors.py to regenerate). #[rustfmt::skip] const VECTORS: &[( &str, &str, f64, f64, f64 )] = &[ // a, b, real_quick_ratio, quick_ratio, ratio ("telemon", "telemon heavy dreadnought", 0.4375, 0.4375, 0.4375), ("aeldari rangers", "dark reapers", 0.8888888888888888, 0.7407407407407407, 0.6666666666666666), ("aeldari rangers", "dwarf rangers", 0.9285714285714286, 0.7857142857142857, 0.7857142857142857), ("guardian weapon team", "guardian squad", 0.8235294117647058, 0.5882352941176471, 0.5882352941176471), ("frontline squad", "guardian squad", 0.9655172413793104, 0.6206896551724138, 0.5517241379310345), ("2e catachany fellaz catachan jungle fighters x", "cadian command squad", 0.6060606060606061, 0.3939393939393939, 0.18181818181818182), ("warp spiders", "warp spiders", 1.0, 1.0, 1.0), ("thunderwolf cavalry", "thunderwolf mount", 0.9444444444444444, 0.6666666666666666, 0.6666666666666666), ("lictor", "necron lieutenant with voidwalker", 0.3076923076923077, 0.3076923076923077, 0.2564102564102564), ("aeldari pack", "harlequin troupe", 0.8571428571428571, 0.5, 0.35714285714285715), ("xyzzy", "warp spiders", 0.5882352941176471, 0.0, 0.0), ("guardian defender", "guardian squad", 0.9032258064516129, 0.6451612903225806, 0.6451612903225806), ]; #[test] fn matches_cpython_difflib_exactly() { for (a, b, rqr, qr, r) in VECTORS { let sm = SequenceMatcher::new(a, b); assert_eq!( sm.real_quick_ratio(), *rqr, "real_quick_ratio({a:?}, {b:?})" ); assert_eq!(sm.quick_ratio(), *qr, "quick_ratio({a:?}, {b:?})"); assert_eq!(sm.ratio(), *r, "ratio({a:?}, {b:?})"); } } #[test] fn empty_and_long_sequences() { assert_eq!(SequenceMatcher::new("", "").ratio(), 1.0); assert_eq!(SequenceMatcher::new("", "abc").ratio(), 0.0); // autojunk path (len(b) >= 200): "unit unit" vs "unit " * 50 let long_b = "unit ".repeat(50); let sm = SequenceMatcher::new("unit unit", &long_b); assert_eq!(sm.ratio(), 0.0694980694980695); } #[test] fn symmetry_of_ratio_value() { // documented CPython behavior: ratio values are symmetric let ab = SequenceMatcher::new("aeldari rangers", "dwarf rangers").ratio(); let ba = SequenceMatcher::new("dwarf rangers", "aeldari rangers").ratio(); assert_eq!(ab, ba); } }