Ben_Kosytorz/Warhammer_DLP_Helper
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor: flatten the crate to the repo root and drop the python version

Browse Source 
The Rust rewrite has replaced the original python implementation, so the transition layout is no longer needed:

- move version-rust/{Cargo.toml,Cargo.lock,README.md,.gitignore,src,data} to the repo root; the crate is now the repo.
- move docs/SORTING.md and data/gwshop_catalog.json to the root.
- remove version-python/ entirely, plus the dev-only version-rust/tools/dump_warhammer_data.py (its only job was regenerating data/warhammer_data.json from python).
- rewrite AGENTS.md and README.md for the single-project layout; strip all version-python/version-rust references from code comments.
- fix the gwshop credential lookup: algolia.json now sits at CARGO_MANIFEST_DIR (crate root == repo root), not its parent.
- restore the credentials/run-output ignores in .gitignore (algolia.json, .env, *.token, downloads/, state.json, manifest.jsonl, ...).
- docs/SORTING.md: retarget the extraction/sorter/dlp examples at the dlphelper binary instead of the deleted python scripts.

cargo test: 86/86 green.
This commit is contained in:
Ben_Kosytorz
2026-10-11 09:22:41 +02:00
parent 356b9089b7
commit 6c6a3273bb
54 changed files with 331 additions and 5378 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/difflib.rs
+253
View File
@@ -0,0 +1,253 @@
//! Port of CPython's `difflib.SequenceMatcher` (Ratcliff/Obershelp) — the
//! exact ratio algorithm the python sorter relies on. A different fuzzy
//! metric (strsim etc.) would produce different scores and silently change
//! which files land where, so this is a faithful port including
//! real_quick_ratio/quick_ratio pruning and autojunk.
//!
//! Constructed like the python sorter uses it: `SequenceMatcher(None, a, b)`
//! — no external junk patterns, autojunk enabled (it only matters for
//! sequences of length >= 200).
/// Longest matching block: (i in a, j in b, size).
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct Match {
i: usize,
j: usize,
size: usize,
}
pub struct SequenceMatcher<'a> {
a: &'a str,
b: &'a str,
/// b[j] -> indices (purged of junk and popular elements, like `b2j`).
b2j: std::collections::HashMap<char, Vec<usize>>,
bjunk: std::collections::HashSet<char>,
}
impl<'a> SequenceMatcher<'a> {
pub fn new(a: &'a str, b: &'a str) -> Self {
// __chain_b with isjunk=None: no junk, but autojunk purges
// "popular" characters from long sequences (len(b) >= 200).
let mut b2j: std::collections::HashMap<char, Vec<usize>> = std::collections::HashMap::new();
for (idx, elt) in b.chars().enumerate() {
b2j.entry(elt).or_default().push(idx);
}
let bjunk = std::collections::HashSet::new(); // isjunk is None
let mut bpopular = std::collections::HashSet::new();
let n = b.chars().count();
if n >= 200 {
let ntest = n / 100 + 1;
for (&elt, idxs) in &b2j {
if idxs.len() > ntest {
bpopular.insert(elt);
}
}
for &elt in &bpopular {
b2j.remove(&elt);
}
}
SequenceMatcher { a, b, b2j, bjunk }
}
fn isbjunk(&self, elt: char) -> bool {
self.bjunk.contains(&elt)
}
/// CPython SequenceMatcher.find_longest_match(alo, ahi, blo, bhi) over
/// char indices.
fn find_longest_match(&self, alo: usize, ahi: usize, blo: usize, bhi: usize) -> Match {
let a_chars: Vec<char> = self.a.chars().collect();
let b_chars: Vec<char> = self.b.chars().collect();
let nothing: Vec<usize> = Vec::new();
let mut besti = alo;
let mut bestj = blo;
let mut bestsize = 0usize;
// j2len[j] = length of the longest match ending at (a[i], b[j])
let mut j2len: std::collections::HashMap<usize, usize> = std::collections::HashMap::new();
for i in alo..ahi {
let mut newj2len: std::collections::HashMap<usize, usize> =
std::collections::HashMap::new();
for &j in self.b2j.get(&a_chars[i]).unwrap_or(&nothing) {
if j < blo {
continue;
}
if j >= bhi {
break;
}
let k = j2len.get(&(j.wrapping_sub(1))).copied().unwrap_or(0) + 1;
newj2len.insert(j, k);
if k > bestsize {
besti = i + 1 - k;
bestj = j + 1 - k;
bestsize = k;
}
}
j2len = newj2len;
}
// Extend the best by non-junk elements on each end.
while besti > alo
&& bestj > blo
&& !self.isbjunk(b_chars[bestj - 1])
&& a_chars[besti - 1] == b_chars[bestj - 1]
{
besti -= 1;
bestj -= 1;
bestsize += 1;
}
while besti + bestsize < ahi
&& bestj + bestsize < bhi
&& !self.isbjunk(b_chars[bestj + bestsize])
&& a_chars[besti + bestsize] == b_chars[bestj + bestsize]
{
bestsize += 1;
}
// Then suck up matching junk on each side of it.
while besti > alo
&& bestj > blo
&& self.isbjunk(b_chars[bestj - 1])
&& a_chars[besti - 1] == b_chars[bestj - 1]
{
besti -= 1;
bestj -= 1;
bestsize += 1;
}
while besti + bestsize < ahi
&& bestj + bestsize < bhi
&& self.isbjunk(b_chars[bestj + bestsize])
&& a_chars[besti + bestsize] == b_chars[bestj + bestsize]
{
bestsize += 1;
}
Match {
i: besti,
j: bestj,
size: bestsize,
}
}
/// All matching blocks, in order (recursive queue algorithm).
pub fn get_matching_blocks(&self) -> Vec<Match> {
let la = self.a.chars().count();
let lb = self.b.chars().count();
let mut blocks: Vec<Match> = Vec::new();
let mut queue: Vec<(usize, usize, usize, usize)> = vec![(0, la, 0, lb)];
while let Some((alo, ahi, blo, bhi)) = queue.pop() {
let m = self.find_longest_match(alo, ahi, blo, bhi);
if m.size > 0 {
blocks.push(m);
if alo < m.i && blo < m.j {
queue.push((alo, m.i, blo, m.j));
}
if m.i + m.size < ahi && m.j + m.size < bhi {
queue.push((m.i + m.size, ahi, m.j + m.size, bhi));
}
}
}
blocks.sort_by_key(|m| (m.i, m.j, m.size));
blocks
}
fn calculate_ratio(matches: usize, la: usize, lb: usize) -> f64 {
let length = la + lb;
if length > 0 {
2.0 * matches as f64 / length as f64
} else {
1.0
}
}
/// Upper bound computed from the cardinality of the char multiset
/// intersection (python `quick_ratio`).
pub fn quick_ratio(&self) -> f64 {
let mut fullbcount: std::collections::HashMap<char, usize> =
std::collections::HashMap::new();
for elt in self.b.chars() {
*fullbcount.entry(elt).or_default() += 1;
}
let mut avail: std::collections::HashMap<char, usize> = std::collections::HashMap::new();
let mut matches = 0usize;
for elt in self.a.chars() {
let numb = avail
.get(&elt)
.copied()
.unwrap_or_else(|| fullbcount.get(&elt).copied().unwrap_or(0));
if numb > 0 {
avail.insert(elt, numb - 1);
matches += 1;
}
}
Self::calculate_ratio(matches, self.a.chars().count(), self.b.chars().count())
}
/// Trivial upper bound (python `real_quick_ratio`).
pub fn real_quick_ratio(&self) -> f64 {
let la = self.a.chars().count();
let lb = self.b.chars().count();
Self::calculate_ratio(la.min(lb), la, lb)
}
/// Ratcliff/Obershelp similarity in [0, 1], exactly as python's
/// `SequenceMatcher(None, a, b).ratio()`.
pub fn ratio(&self) -> f64 {
let matches: usize = self.get_matching_blocks().iter().map(|m| m.size).sum();
Self::calculate_ratio(matches, self.a.chars().count(), self.b.chars().count())
}
}
#[cfg(test)]
mod tests {
use super::*;
/// Expected values generated with CPython 3.12 difflib (see
/// tools/dump_difflib_vectors.py to regenerate).
#[rustfmt::skip]
const VECTORS: &[( &str, &str, f64, f64, f64 )] = &[
// a, b, real_quick_ratio, quick_ratio, ratio
("telemon", "telemon heavy dreadnought", 0.4375, 0.4375, 0.4375),
("aeldari rangers", "dark reapers", 0.8888888888888888, 0.7407407407407407, 0.6666666666666666),
("aeldari rangers", "dwarf rangers", 0.9285714285714286, 0.7857142857142857, 0.7857142857142857),
("guardian weapon team", "guardian squad", 0.8235294117647058, 0.5882352941176471, 0.5882352941176471),
("frontline squad", "guardian squad", 0.9655172413793104, 0.6206896551724138, 0.5517241379310345),
("2e catachany fellaz catachan jungle fighters x", "cadian command squad", 0.6060606060606061, 0.3939393939393939, 0.18181818181818182),
("warp spiders", "warp spiders", 1.0, 1.0, 1.0),
("thunderwolf cavalry", "thunderwolf mount", 0.9444444444444444, 0.6666666666666666, 0.6666666666666666),
("lictor", "necron lieutenant with voidwalker", 0.3076923076923077, 0.3076923076923077, 0.2564102564102564),
("aeldari pack", "harlequin troupe", 0.8571428571428571, 0.5, 0.35714285714285715),
("xyzzy", "warp spiders", 0.5882352941176471, 0.0, 0.0),
("guardian defender", "guardian squad", 0.9032258064516129, 0.6451612903225806, 0.6451612903225806),
];
#[test]
fn matches_cpython_difflib_exactly() {
for (a, b, rqr, qr, r) in VECTORS {
let sm = SequenceMatcher::new(a, b);
assert_eq!(
sm.real_quick_ratio(),
*rqr,
"real_quick_ratio({a:?}, {b:?})"
);
assert_eq!(sm.quick_ratio(), *qr, "quick_ratio({a:?}, {b:?})");
assert_eq!(sm.ratio(), *r, "ratio({a:?}, {b:?})");
}
}
#[test]
fn empty_and_long_sequences() {
assert_eq!(SequenceMatcher::new("", "").ratio(), 1.0);
assert_eq!(SequenceMatcher::new("", "abc").ratio(), 0.0);
// autojunk path (len(b) >= 200): "unit unit" vs "unit " * 50
let long_b = "unit ".repeat(50);
let sm = SequenceMatcher::new("unit unit", &long_b);
assert_eq!(sm.ratio(), 0.0694980694980695);
}
#[test]
fn symmetry_of_ratio_value() {
// documented CPython behavior: ratio values are symmetric
let ab = SequenceMatcher::new("aeldari rangers", "dwarf rangers").ratio();
let ba = SequenceMatcher::new("dwarf rangers", "aeldari rangers").ratio();
assert_eq!(ab, ba);
}
}