feat(rust): difflib port + taxonomy data dump
- src/difflib.rs: faithful port of CPython difflib.SequenceMatcher (Ratcliff/Obershelp ratio incl. real_quick_ratio/quick_ratio and autojunk), validated against python-computed test vectors with exact float equality - data/warhammer_data.json: taxonomy tables dumped from the python module (tools/dump_warhammer_data.py), parsed order-preserving in src/warhammer_data.rs (python dict insert-order + last-value-wins semantics) - deps for the module ports: clap, reqwest, url, base64, regex, sha2, hex, sevenz-rust2, unrar, zip, indexmap, rand, rpassword
This commit is contained in:
@@ -0,0 +1,2 @@
|
||||
//! Catalogue: classify collected files into game system / faction / class —
|
||||
//! Rust port of `version-python/dlphelper/catalogue.py`.
|
||||
@@ -0,0 +1,253 @@
|
||||
//! Port of CPython's `difflib.SequenceMatcher` (Ratcliff/Obershelp) — the
|
||||
//! exact ratio algorithm the python sorter relies on. A different fuzzy
|
||||
//! metric (strsim etc.) would produce different scores and silently change
|
||||
//! which files land where, so this is a faithful port including
|
||||
//! real_quick_ratio/quick_ratio pruning and autojunk.
|
||||
//!
|
||||
//! Constructed like the python sorter uses it: `SequenceMatcher(None, a, b)`
|
||||
//! — no external junk patterns, autojunk enabled (it only matters for
|
||||
//! sequences of length >= 200).
|
||||
|
||||
/// Longest matching block: (i in a, j in b, size).
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub struct Match {
|
||||
i: usize,
|
||||
j: usize,
|
||||
size: usize,
|
||||
}
|
||||
|
||||
pub struct SequenceMatcher<'a> {
|
||||
a: &'a str,
|
||||
b: &'a str,
|
||||
/// b[j] -> indices (purged of junk and popular elements, like `b2j`).
|
||||
b2j: std::collections::HashMap<char, Vec<usize>>,
|
||||
bjunk: std::collections::HashSet<char>,
|
||||
}
|
||||
|
||||
impl<'a> SequenceMatcher<'a> {
|
||||
pub fn new(a: &'a str, b: &'a str) -> Self {
|
||||
// __chain_b with isjunk=None: no junk, but autojunk purges
|
||||
// "popular" characters from long sequences (len(b) >= 200).
|
||||
let mut b2j: std::collections::HashMap<char, Vec<usize>> = std::collections::HashMap::new();
|
||||
for (idx, elt) in b.chars().enumerate() {
|
||||
b2j.entry(elt).or_default().push(idx);
|
||||
}
|
||||
let bjunk = std::collections::HashSet::new(); // isjunk is None
|
||||
let mut bpopular = std::collections::HashSet::new();
|
||||
let n = b.chars().count();
|
||||
if n >= 200 {
|
||||
let ntest = n / 100 + 1;
|
||||
for (&elt, idxs) in &b2j {
|
||||
if idxs.len() > ntest {
|
||||
bpopular.insert(elt);
|
||||
}
|
||||
}
|
||||
for &elt in &bpopular {
|
||||
b2j.remove(&elt);
|
||||
}
|
||||
}
|
||||
SequenceMatcher { a, b, b2j, bjunk }
|
||||
}
|
||||
|
||||
fn isbjunk(&self, elt: char) -> bool {
|
||||
self.bjunk.contains(&elt)
|
||||
}
|
||||
|
||||
/// CPython SequenceMatcher.find_longest_match(alo, ahi, blo, bhi) over
|
||||
/// char indices.
|
||||
fn find_longest_match(&self, alo: usize, ahi: usize, blo: usize, bhi: usize) -> Match {
|
||||
let a_chars: Vec<char> = self.a.chars().collect();
|
||||
let b_chars: Vec<char> = self.b.chars().collect();
|
||||
let nothing: Vec<usize> = Vec::new();
|
||||
|
||||
let mut besti = alo;
|
||||
let mut bestj = blo;
|
||||
let mut bestsize = 0usize;
|
||||
// j2len[j] = length of the longest match ending at (a[i], b[j])
|
||||
let mut j2len: std::collections::HashMap<usize, usize> = std::collections::HashMap::new();
|
||||
for i in alo..ahi {
|
||||
let mut newj2len: std::collections::HashMap<usize, usize> =
|
||||
std::collections::HashMap::new();
|
||||
for &j in self.b2j.get(&a_chars[i]).unwrap_or(¬hing) {
|
||||
if j < blo {
|
||||
continue;
|
||||
}
|
||||
if j >= bhi {
|
||||
break;
|
||||
}
|
||||
let k = j2len.get(&(j.wrapping_sub(1))).copied().unwrap_or(0) + 1;
|
||||
newj2len.insert(j, k);
|
||||
if k > bestsize {
|
||||
besti = i + 1 - k;
|
||||
bestj = j + 1 - k;
|
||||
bestsize = k;
|
||||
}
|
||||
}
|
||||
j2len = newj2len;
|
||||
}
|
||||
|
||||
// Extend the best by non-junk elements on each end.
|
||||
while besti > alo
|
||||
&& bestj > blo
|
||||
&& !self.isbjunk(b_chars[bestj - 1])
|
||||
&& a_chars[besti - 1] == b_chars[bestj - 1]
|
||||
{
|
||||
besti -= 1;
|
||||
bestj -= 1;
|
||||
bestsize += 1;
|
||||
}
|
||||
while besti + bestsize < ahi
|
||||
&& bestj + bestsize < bhi
|
||||
&& !self.isbjunk(b_chars[bestj + bestsize])
|
||||
&& a_chars[besti + bestsize] == b_chars[bestj + bestsize]
|
||||
{
|
||||
bestsize += 1;
|
||||
}
|
||||
// Then suck up matching junk on each side of it.
|
||||
while besti > alo
|
||||
&& bestj > blo
|
||||
&& self.isbjunk(b_chars[bestj - 1])
|
||||
&& a_chars[besti - 1] == b_chars[bestj - 1]
|
||||
{
|
||||
besti -= 1;
|
||||
bestj -= 1;
|
||||
bestsize += 1;
|
||||
}
|
||||
while besti + bestsize < ahi
|
||||
&& bestj + bestsize < bhi
|
||||
&& self.isbjunk(b_chars[bestj + bestsize])
|
||||
&& a_chars[besti + bestsize] == b_chars[bestj + bestsize]
|
||||
{
|
||||
bestsize += 1;
|
||||
}
|
||||
Match {
|
||||
i: besti,
|
||||
j: bestj,
|
||||
size: bestsize,
|
||||
}
|
||||
}
|
||||
|
||||
/// All matching blocks, in order (recursive queue algorithm).
|
||||
pub fn get_matching_blocks(&self) -> Vec<Match> {
|
||||
let la = self.a.chars().count();
|
||||
let lb = self.b.chars().count();
|
||||
let mut blocks: Vec<Match> = Vec::new();
|
||||
let mut queue: Vec<(usize, usize, usize, usize)> = vec![(0, la, 0, lb)];
|
||||
while let Some((alo, ahi, blo, bhi)) = queue.pop() {
|
||||
let m = self.find_longest_match(alo, ahi, blo, bhi);
|
||||
if m.size > 0 {
|
||||
blocks.push(m);
|
||||
if alo < m.i && blo < m.j {
|
||||
queue.push((alo, m.i, blo, m.j));
|
||||
}
|
||||
if m.i + m.size < ahi && m.j + m.size < bhi {
|
||||
queue.push((m.i + m.size, ahi, m.j + m.size, bhi));
|
||||
}
|
||||
}
|
||||
}
|
||||
blocks.sort_by_key(|m| (m.i, m.j, m.size));
|
||||
blocks
|
||||
}
|
||||
|
||||
fn calculate_ratio(matches: usize, la: usize, lb: usize) -> f64 {
|
||||
let length = la + lb;
|
||||
if length > 0 {
|
||||
2.0 * matches as f64 / length as f64
|
||||
} else {
|
||||
1.0
|
||||
}
|
||||
}
|
||||
|
||||
/// Upper bound computed from the cardinality of the char multiset
|
||||
/// intersection (python `quick_ratio`).
|
||||
pub fn quick_ratio(&self) -> f64 {
|
||||
let mut fullbcount: std::collections::HashMap<char, usize> =
|
||||
std::collections::HashMap::new();
|
||||
for elt in self.b.chars() {
|
||||
*fullbcount.entry(elt).or_default() += 1;
|
||||
}
|
||||
let mut avail: std::collections::HashMap<char, usize> = std::collections::HashMap::new();
|
||||
let mut matches = 0usize;
|
||||
for elt in self.a.chars() {
|
||||
let numb = avail
|
||||
.get(&elt)
|
||||
.copied()
|
||||
.unwrap_or_else(|| fullbcount.get(&elt).copied().unwrap_or(0));
|
||||
if numb > 0 {
|
||||
avail.insert(elt, numb - 1);
|
||||
matches += 1;
|
||||
}
|
||||
}
|
||||
Self::calculate_ratio(matches, self.a.chars().count(), self.b.chars().count())
|
||||
}
|
||||
|
||||
/// Trivial upper bound (python `real_quick_ratio`).
|
||||
pub fn real_quick_ratio(&self) -> f64 {
|
||||
let la = self.a.chars().count();
|
||||
let lb = self.b.chars().count();
|
||||
Self::calculate_ratio(la.min(lb), la, lb)
|
||||
}
|
||||
|
||||
/// Ratcliff/Obershelp similarity in [0, 1], exactly as python's
|
||||
/// `SequenceMatcher(None, a, b).ratio()`.
|
||||
pub fn ratio(&self) -> f64 {
|
||||
let matches: usize = self.get_matching_blocks().iter().map(|m| m.size).sum();
|
||||
Self::calculate_ratio(matches, self.a.chars().count(), self.b.chars().count())
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
/// Expected values generated with CPython 3.12 difflib (see
|
||||
/// tools/dump_difflib_vectors.py to regenerate).
|
||||
#[rustfmt::skip]
|
||||
const VECTORS: &[( &str, &str, f64, f64, f64 )] = &[
|
||||
// a, b, real_quick_ratio, quick_ratio, ratio
|
||||
("telemon", "telemon heavy dreadnought", 0.4375, 0.4375, 0.4375),
|
||||
("aeldari rangers", "dark reapers", 0.8888888888888888, 0.7407407407407407, 0.6666666666666666),
|
||||
("aeldari rangers", "dwarf rangers", 0.9285714285714286, 0.7857142857142857, 0.7857142857142857),
|
||||
("guardian weapon team", "guardian squad", 0.8235294117647058, 0.5882352941176471, 0.5882352941176471),
|
||||
("frontline squad", "guardian squad", 0.9655172413793104, 0.6206896551724138, 0.5517241379310345),
|
||||
("2e catachany fellaz catachan jungle fighters x", "cadian command squad", 0.6060606060606061, 0.3939393939393939, 0.18181818181818182),
|
||||
("warp spiders", "warp spiders", 1.0, 1.0, 1.0),
|
||||
("thunderwolf cavalry", "thunderwolf mount", 0.9444444444444444, 0.6666666666666666, 0.6666666666666666),
|
||||
("lictor", "necron lieutenant with voidwalker", 0.3076923076923077, 0.3076923076923077, 0.2564102564102564),
|
||||
("aeldari pack", "harlequin troupe", 0.8571428571428571, 0.5, 0.35714285714285715),
|
||||
("xyzzy", "warp spiders", 0.5882352941176471, 0.0, 0.0),
|
||||
("guardian defender", "guardian squad", 0.9032258064516129, 0.6451612903225806, 0.6451612903225806),
|
||||
];
|
||||
|
||||
#[test]
|
||||
fn matches_cpython_difflib_exactly() {
|
||||
for (a, b, rqr, qr, r) in VECTORS {
|
||||
let sm = SequenceMatcher::new(a, b);
|
||||
assert_eq!(
|
||||
sm.real_quick_ratio(),
|
||||
*rqr,
|
||||
"real_quick_ratio({a:?}, {b:?})"
|
||||
);
|
||||
assert_eq!(sm.quick_ratio(), *qr, "quick_ratio({a:?}, {b:?})");
|
||||
assert_eq!(sm.ratio(), *r, "ratio({a:?}, {b:?})");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn empty_and_long_sequences() {
|
||||
assert_eq!(SequenceMatcher::new("", "").ratio(), 1.0);
|
||||
assert_eq!(SequenceMatcher::new("", "abc").ratio(), 0.0);
|
||||
// autojunk path (len(b) >= 200): "unit unit" vs "unit " * 50
|
||||
let long_b = "unit ".repeat(50);
|
||||
let sm = SequenceMatcher::new("unit unit", &long_b);
|
||||
assert_eq!(sm.ratio(), 0.0694980694980695);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn symmetry_of_ratio_value() {
|
||||
// documented CPython behavior: ratio values are symmetric
|
||||
let ab = SequenceMatcher::new("aeldari rangers", "dwarf rangers").ratio();
|
||||
let ba = SequenceMatcher::new("dwarf rangers", "aeldari rangers").ratio();
|
||||
assert_eq!(ab, ba);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,2 @@
|
||||
//! DCE docker export (JSON) + base64/plain URL extraction — Rust port of
|
||||
//! `version-python/dlphelper/discord.py`.
|
||||
@@ -0,0 +1,9 @@
|
||||
//! Recursive archive extraction into a plain-file tree — Rust port of
|
||||
//! `version-python/tools/extract_all.py` (extract_manifest.jsonl feeds the
|
||||
//! sorter's provenance mode; failures go to .broken/; the run never stops).
|
||||
//!
|
||||
//! Extraction backend: `sevenz-rust2` for .7z, `zip` for .zip, `unrar` for
|
||||
//! .rar, `tar`+`flate2`/`bzip2`/`xz2` for tar balls and bare compressed
|
||||
//! files. Because extraction is pure Rust (no 7z CLI writing directly onto
|
||||
//! CIFS), the python version's /var/tmp fallback + cp437 name repair is
|
||||
//! not needed here.
|
||||
@@ -0,0 +1,2 @@
|
||||
//! GoFile API client (one guest account per run) — Rust port of
|
||||
//! `version-python/dlphelper/gofile.py`.
|
||||
@@ -0,0 +1,2 @@
|
||||
//! GW webshop catalog dump via Algolia — Rust port of
|
||||
//! `version-python/dlphelper/gwshop.py`.
|
||||
@@ -1,7 +1,17 @@
|
||||
mod catalogue;
|
||||
mod difflib;
|
||||
mod discord;
|
||||
mod extract;
|
||||
mod globalstore;
|
||||
mod gofile;
|
||||
mod gwshop;
|
||||
mod manifest;
|
||||
mod orchestrate;
|
||||
mod sorter;
|
||||
mod state;
|
||||
mod util;
|
||||
mod warhammer_data;
|
||||
mod zero807;
|
||||
|
||||
fn main() {
|
||||
env_logger::init();
|
||||
|
||||
@@ -0,0 +1,6 @@
|
||||
//! Orchestration: per-channel pipeline, gofile/0807 loops, subscriptions,
|
||||
//! --assume-done — Rust port of the logic in `version-python/dlp.py`.
|
||||
//!
|
||||
//! Failure policy (the core requirement): a failure in one URL must never
|
||||
//! abort the run. Catch, record in state.json, continue. Cooldown pauses
|
||||
//! are the only permitted stall.
|
||||
@@ -0,0 +1,7 @@
|
||||
//! Library sorter: inspect/classify/fuzzy-match against the catalog, plan +
|
||||
//! hardlink apply, optional provenance unit mode — Rust port of
|
||||
//! `version-python/dlphelper/sorter.py`.
|
||||
//!
|
||||
//! Matching rules are fixed (see version-python/AGENTS.md §7 "Sorter matching
|
||||
//! rules"); they must be ported exactly or the Rust sorter will silently
|
||||
//! sort differently than the battle-tested python one.
|
||||
@@ -0,0 +1,137 @@
|
||||
//! Taxonomy: GW-named systems/factions, GW shop categories as classes,
|
||||
//! GROUP_OF_FACTION map — Rust port of `version-python/dlphelper/warhammer_data.py`.
|
||||
//!
|
||||
//! The tables are NOT hand-transcribed: they are dumped from the python
|
||||
//! module into `data/warhammer_data.json` (regenerate with
|
||||
//! `version-rust/tools/dump_warhammer_data.py`) and parsed here, so both
|
||||
//! versions work from byte-identical data.
|
||||
//!
|
||||
//! Python-dict ordering is preserved exactly (insertion order; a duplicate
|
||||
//! key keeps its first insertion position while the *last* value wins),
|
||||
//! because the catalogue's "most specific keyword" rule breaks ties by
|
||||
//! first occurrence among equal-length keywords.
|
||||
|
||||
use std::sync::OnceLock;
|
||||
|
||||
use anyhow::Context as _;
|
||||
use indexmap::IndexMap;
|
||||
use serde::Deserialize;
|
||||
|
||||
/// Insertion-ordered pair list with python-dict semantics for building.
|
||||
fn dict_insert<V>(vec: &mut Vec<(String, V)>, key: String, value: V) {
|
||||
if let Some(slot) = vec.iter_mut().find(|(k, _)| *k == key) {
|
||||
slot.1 = value;
|
||||
} else {
|
||||
vec.push((key, value));
|
||||
}
|
||||
}
|
||||
|
||||
/// system -> (faction, keyword aliases), python dict order.
|
||||
pub type Systems = Vec<(String, Vec<(String, Vec<String>)>)>;
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct Raw {
|
||||
systems: IndexMap<String, IndexMap<String, Vec<String>>>,
|
||||
classes: Vec<(String, Vec<String>)>,
|
||||
group_of_faction: IndexMap<String, String>,
|
||||
}
|
||||
|
||||
pub struct Data {
|
||||
pub systems: Systems,
|
||||
/// system -> (alias, faction), insertion order of first occurrence,
|
||||
/// value = last faction seen (python dict semantics).
|
||||
pub faction_aliases: Vec<(String, Vec<(String, String)>)>,
|
||||
/// keyword -> class, insertion order of first occurrence.
|
||||
pub unit_classes: Vec<(String, String)>,
|
||||
pub group_of_faction: IndexMap<String, String>,
|
||||
}
|
||||
|
||||
impl Data {
|
||||
pub fn global() -> &'static Data {
|
||||
static DATA: OnceLock<Data> = OnceLock::new();
|
||||
DATA.get_or_init(|| Data::load().expect("embedded warhammer_data.json is valid"))
|
||||
}
|
||||
|
||||
/// Parse the embedded JSON dump of the python taxonomy module.
|
||||
fn load() -> anyhow::Result<Data> {
|
||||
let raw: Raw = serde_json::from_str(include_str!("../data/warhammer_data.json"))
|
||||
.context("parsing embedded warhammer_data.json")?;
|
||||
|
||||
// SYSTEMS in dict order
|
||||
let mut systems: Systems = Vec::new();
|
||||
for (system, factions) in &raw.systems {
|
||||
let mut factions_vec: Vec<(String, Vec<String>)> = Vec::new();
|
||||
for (faction, aliases) in factions {
|
||||
dict_insert(&mut factions_vec, faction.clone(), aliases.clone());
|
||||
}
|
||||
dict_insert(&mut systems, system.clone(), factions_vec);
|
||||
}
|
||||
|
||||
// FACTION_ALIASES = {system: {alias: faction ...}} in dict order
|
||||
let mut faction_aliases: Vec<(String, Vec<(String, String)>)> = Vec::new();
|
||||
for (system, factions) in &systems {
|
||||
let mut aliases_vec: Vec<(String, String)> = Vec::new();
|
||||
for (faction, aliases) in factions {
|
||||
for alias in aliases {
|
||||
dict_insert(&mut aliases_vec, alias.clone(), faction.clone());
|
||||
}
|
||||
}
|
||||
dict_insert(&mut faction_aliases, system.clone(), aliases_vec);
|
||||
}
|
||||
|
||||
// UNIT_CLASSES = {kw: klass for klass, kws in CLASSES for kw in kws}
|
||||
let mut unit_classes: Vec<(String, String)> = Vec::new();
|
||||
for (klass, keywords) in &raw.classes {
|
||||
for kw in keywords {
|
||||
dict_insert(&mut unit_classes, kw.clone(), klass.clone());
|
||||
}
|
||||
}
|
||||
|
||||
Ok(Data {
|
||||
systems,
|
||||
faction_aliases,
|
||||
unit_classes,
|
||||
group_of_faction: raw.group_of_faction,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn tables_load_and_are_complete() {
|
||||
let d = Data::global();
|
||||
assert!(d.systems.iter().any(|(s, _)| s == "Warhammer 40,000"));
|
||||
assert!(d.systems.iter().any(|(s, _)| s == "Middle-Earth"));
|
||||
let f40k = &d
|
||||
.systems
|
||||
.iter()
|
||||
.find(|(s, _)| s == "Warhammer 40,000")
|
||||
.unwrap()
|
||||
.1;
|
||||
assert!(f40k.iter().any(|(f, _)| f == "Space Wolves"));
|
||||
// "magnus" is listed in both Monster and Hero -> last (Hero) wins
|
||||
let magnus = d
|
||||
.unit_classes
|
||||
.iter()
|
||||
.find(|(kw, _)| kw == "magnus")
|
||||
.map(|(_, c)| c.as_str());
|
||||
assert_eq!(magnus, Some("Hero"));
|
||||
// duplicate alias inside a system keeps first position, last value
|
||||
let aliases = &d
|
||||
.faction_aliases
|
||||
.iter()
|
||||
.find(|(s, _)| s == "Warhammer 40,000")
|
||||
.unwrap()
|
||||
.1;
|
||||
let idx = aliases.iter().position(|(a, _)| a == "skull lord").unwrap();
|
||||
assert_eq!(aliases[idx].1, "World Eaters");
|
||||
// position kept from FIRST insertion (Chaos Daemons block): it sits
|
||||
// between 'slaanesh' and 'shalaxi helbane' (verified against python)
|
||||
assert_eq!(aliases[idx - 1].0, "slaanesh");
|
||||
assert_eq!(aliases[idx + 1].0, "shalaxi helbane");
|
||||
assert_eq!(d.group_of_faction["Space Wolves"], "Space Marines");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,2 @@
|
||||
//! curl-based downloader for 0807.st direct links — Rust port of
|
||||
//! `version-python/dlphelper/zero807.py`.
|
||||
Reference in New Issue
Block a user