feat(rust): difflib port + taxonomy data dump

- src/difflib.rs: faithful port of CPython difflib.SequenceMatcher
  (Ratcliff/Obershelp ratio incl. real_quick_ratio/quick_ratio and autojunk),
  validated against python-computed test vectors with exact float equality
- data/warhammer_data.json: taxonomy tables dumped from the python module
  (tools/dump_warhammer_data.py), parsed order-preserving in
  src/warhammer_data.rs (python dict insert-order + last-value-wins semantics)
- deps for the module ports: clap, reqwest, url, base64, regex, sha2, hex,
  sevenz-rust2, unrar, zip, indexmap, rand, rpassword

version-rust/Cargo.toml

@@ -6,12 +6,25 @@ description = "Torrent-first platform for Warhammer proxy models: Discord channe
 
 [dependencies]
 anyhow = "1"
+base64 = "0.22"
 chrono = "0.4"
+clap = { version = "4", features = ["derive"] }
 dirs = "6"
 env_logger = "0.11"
+hex = "0.4"
+indexmap = { version = "2", features = ["serde"] }
 log = "0.4"
+rand = "0.9"
+regex = "1"
+reqwest = { version = "0.12", features = ["blocking", "gzip", "json"] }
+rpassword = "7"
 serde = { version = "1", features = ["derive"] }
 serde_json = "1"
+sha2 = "0.10"
+sevenz-rust2 = "0.23"
+unrar = "0.5"
+url = "2"
+zip = "2"
 
 [dev-dependencies]
 tempfile = "3"

version-rust/src/catalogue.rs

@@ -0,0 +1,2 @@
+//! Catalogue: classify collected files into game system / faction / class —
+//! Rust port of `version-python/dlphelper/catalogue.py`.

version-rust/src/difflib.rs

@@ -0,0 +1,253 @@
+//! Port of CPython's `difflib.SequenceMatcher` (Ratcliff/Obershelp) — the
+//! exact ratio algorithm the python sorter relies on. A different fuzzy
+//! metric (strsim etc.) would produce different scores and silently change
+//! which files land where, so this is a faithful port including
+//! real_quick_ratio/quick_ratio pruning and autojunk.
+//!
+//! Constructed like the python sorter uses it: `SequenceMatcher(None, a, b)`
+//! — no external junk patterns, autojunk enabled (it only matters for
+//! sequences of length >= 200).
+
+/// Longest matching block: (i in a, j in b, size).
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub struct Match {
+    i: usize,
+    j: usize,
+    size: usize,
+}
+
+pub struct SequenceMatcher<'a> {
+    a: &'a str,
+    b: &'a str,
+    /// b[j] -> indices (purged of junk and popular elements, like `b2j`).
+    b2j: std::collections::HashMap<char, Vec<usize>>,
+    bjunk: std::collections::HashSet<char>,
+}
+
+impl<'a> SequenceMatcher<'a> {
+    pub fn new(a: &'a str, b: &'a str) -> Self {
+        // __chain_b with isjunk=None: no junk, but autojunk purges
+        // "popular" characters from long sequences (len(b) >= 200).
+        let mut b2j: std::collections::HashMap<char, Vec<usize>> = std::collections::HashMap::new();
+        for (idx, elt) in b.chars().enumerate() {
+            b2j.entry(elt).or_default().push(idx);
+        }
+        let bjunk = std::collections::HashSet::new(); // isjunk is None
+        let mut bpopular = std::collections::HashSet::new();
+        let n = b.chars().count();
+        if n >= 200 {
+            let ntest = n / 100 + 1;
+            for (&elt, idxs) in &b2j {
+                if idxs.len() > ntest {
+                    bpopular.insert(elt);
+                }
+            }
+            for &elt in &bpopular {
+                b2j.remove(&elt);
+            }
+        }
+        SequenceMatcher { a, b, b2j, bjunk }
+    }
+
+    fn isbjunk(&self, elt: char) -> bool {
+        self.bjunk.contains(&elt)
+    }
+
+    /// CPython SequenceMatcher.find_longest_match(alo, ahi, blo, bhi) over
+    /// char indices.
+    fn find_longest_match(&self, alo: usize, ahi: usize, blo: usize, bhi: usize) -> Match {
+        let a_chars: Vec<char> = self.a.chars().collect();
+        let b_chars: Vec<char> = self.b.chars().collect();
+        let nothing: Vec<usize> = Vec::new();
+
+        let mut besti = alo;
+        let mut bestj = blo;
+        let mut bestsize = 0usize;
+        // j2len[j] = length of the longest match ending at (a[i], b[j])
+        let mut j2len: std::collections::HashMap<usize, usize> = std::collections::HashMap::new();
+        for i in alo..ahi {
+            let mut newj2len: std::collections::HashMap<usize, usize> =
+                std::collections::HashMap::new();
+            for &j in self.b2j.get(&a_chars[i]).unwrap_or(&nothing) {
+                if j < blo {
+                    continue;
+                }
+                if j >= bhi {
+                    break;
+                }
+                let k = j2len.get(&(j.wrapping_sub(1))).copied().unwrap_or(0) + 1;
+                newj2len.insert(j, k);
+                if k > bestsize {
+                    besti = i + 1 - k;
+                    bestj = j + 1 - k;
+                    bestsize = k;
+                }
+            }
+            j2len = newj2len;
+        }
+
+        // Extend the best by non-junk elements on each end.
+        while besti > alo
+            && bestj > blo
+            && !self.isbjunk(b_chars[bestj - 1])
+            && a_chars[besti - 1] == b_chars[bestj - 1]
+        {
+            besti -= 1;
+            bestj -= 1;
+            bestsize += 1;
+        }
+        while besti + bestsize < ahi
+            && bestj + bestsize < bhi
+            && !self.isbjunk(b_chars[bestj + bestsize])
+            && a_chars[besti + bestsize] == b_chars[bestj + bestsize]
+        {
+            bestsize += 1;
+        }
+        // Then suck up matching junk on each side of it.
+        while besti > alo
+            && bestj > blo
+            && self.isbjunk(b_chars[bestj - 1])
+            && a_chars[besti - 1] == b_chars[bestj - 1]
+        {
+            besti -= 1;
+            bestj -= 1;
+            bestsize += 1;
+        }
+        while besti + bestsize < ahi
+            && bestj + bestsize < bhi
+            && self.isbjunk(b_chars[bestj + bestsize])
+            && a_chars[besti + bestsize] == b_chars[bestj + bestsize]
+        {
+            bestsize += 1;
+        }
+        Match {
+            i: besti,
+            j: bestj,
+            size: bestsize,
+        }
+    }
+
+    /// All matching blocks, in order (recursive queue algorithm).
+    pub fn get_matching_blocks(&self) -> Vec<Match> {
+        let la = self.a.chars().count();
+        let lb = self.b.chars().count();
+        let mut blocks: Vec<Match> = Vec::new();
+        let mut queue: Vec<(usize, usize, usize, usize)> = vec![(0, la, 0, lb)];
+        while let Some((alo, ahi, blo, bhi)) = queue.pop() {
+            let m = self.find_longest_match(alo, ahi, blo, bhi);
+            if m.size > 0 {
+                blocks.push(m);
+                if alo < m.i && blo < m.j {
+                    queue.push((alo, m.i, blo, m.j));
+                }
+                if m.i + m.size < ahi && m.j + m.size < bhi {
+                    queue.push((m.i + m.size, ahi, m.j + m.size, bhi));
+                }
+            }
+        }
+        blocks.sort_by_key(|m| (m.i, m.j, m.size));
+        blocks
+    }
+
+    fn calculate_ratio(matches: usize, la: usize, lb: usize) -> f64 {
+        let length = la + lb;
+        if length > 0 {
+            2.0 * matches as f64 / length as f64
+        } else {
+            1.0
+        }
+    }
+
+    /// Upper bound computed from the cardinality of the char multiset
+    /// intersection (python `quick_ratio`).
+    pub fn quick_ratio(&self) -> f64 {
+        let mut fullbcount: std::collections::HashMap<char, usize> =
+            std::collections::HashMap::new();
+        for elt in self.b.chars() {
+            *fullbcount.entry(elt).or_default() += 1;
+        }
+        let mut avail: std::collections::HashMap<char, usize> = std::collections::HashMap::new();
+        let mut matches = 0usize;
+        for elt in self.a.chars() {
+            let numb = avail
+                .get(&elt)
+                .copied()
+                .unwrap_or_else(|| fullbcount.get(&elt).copied().unwrap_or(0));
+            if numb > 0 {
+                avail.insert(elt, numb - 1);
+                matches += 1;
+            }
+        }
+        Self::calculate_ratio(matches, self.a.chars().count(), self.b.chars().count())
+    }
+
+    /// Trivial upper bound (python `real_quick_ratio`).
+    pub fn real_quick_ratio(&self) -> f64 {
+        let la = self.a.chars().count();
+        let lb = self.b.chars().count();
+        Self::calculate_ratio(la.min(lb), la, lb)
+    }
+
+    /// Ratcliff/Obershelp similarity in [0, 1], exactly as python's
+    /// `SequenceMatcher(None, a, b).ratio()`.
+    pub fn ratio(&self) -> f64 {
+        let matches: usize = self.get_matching_blocks().iter().map(|m| m.size).sum();
+        Self::calculate_ratio(matches, self.a.chars().count(), self.b.chars().count())
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    /// Expected values generated with CPython 3.12 difflib (see
+    /// tools/dump_difflib_vectors.py to regenerate).
+    #[rustfmt::skip]
+    const VECTORS: &[( &str, &str, f64, f64, f64 )] = &[
+        // a, b, real_quick_ratio, quick_ratio, ratio
+        ("telemon", "telemon heavy dreadnought", 0.4375, 0.4375, 0.4375),
+        ("aeldari rangers", "dark reapers", 0.8888888888888888, 0.7407407407407407, 0.6666666666666666),
+        ("aeldari rangers", "dwarf rangers", 0.9285714285714286, 0.7857142857142857, 0.7857142857142857),
+        ("guardian weapon team", "guardian squad", 0.8235294117647058, 0.5882352941176471, 0.5882352941176471),
+        ("frontline squad", "guardian squad", 0.9655172413793104, 0.6206896551724138, 0.5517241379310345),
+        ("2e catachany fellaz catachan jungle fighters x", "cadian command squad", 0.6060606060606061, 0.3939393939393939, 0.18181818181818182),
+        ("warp spiders", "warp spiders", 1.0, 1.0, 1.0),
+        ("thunderwolf cavalry", "thunderwolf mount", 0.9444444444444444, 0.6666666666666666, 0.6666666666666666),
+        ("lictor", "necron lieutenant with voidwalker", 0.3076923076923077, 0.3076923076923077, 0.2564102564102564),
+        ("aeldari pack", "harlequin troupe", 0.8571428571428571, 0.5, 0.35714285714285715),
+        ("xyzzy", "warp spiders", 0.5882352941176471, 0.0, 0.0),
+        ("guardian defender", "guardian squad", 0.9032258064516129, 0.6451612903225806, 0.6451612903225806),
+    ];
+
+    #[test]
+    fn matches_cpython_difflib_exactly() {
+        for (a, b, rqr, qr, r) in VECTORS {
+            let sm = SequenceMatcher::new(a, b);
+            assert_eq!(
+                sm.real_quick_ratio(),
+                *rqr,
+                "real_quick_ratio({a:?}, {b:?})"
+            );
+            assert_eq!(sm.quick_ratio(), *qr, "quick_ratio({a:?}, {b:?})");
+            assert_eq!(sm.ratio(), *r, "ratio({a:?}, {b:?})");
+        }
+    }
+
+    #[test]
+    fn empty_and_long_sequences() {
+        assert_eq!(SequenceMatcher::new("", "").ratio(), 1.0);
+        assert_eq!(SequenceMatcher::new("", "abc").ratio(), 0.0);
+        // autojunk path (len(b) >= 200): "unit unit" vs "unit " * 50
+        let long_b = "unit ".repeat(50);
+        let sm = SequenceMatcher::new("unit unit", &long_b);
+        assert_eq!(sm.ratio(), 0.0694980694980695);
+    }
+
+    #[test]
+    fn symmetry_of_ratio_value() {
+        // documented CPython behavior: ratio values are symmetric
+        let ab = SequenceMatcher::new("aeldari rangers", "dwarf rangers").ratio();
+        let ba = SequenceMatcher::new("dwarf rangers", "aeldari rangers").ratio();
+        assert_eq!(ab, ba);
+    }
+}

version-rust/src/discord.rs

@@ -0,0 +1,2 @@
+//! DCE docker export (JSON) + base64/plain URL extraction — Rust port of
+//! `version-python/dlphelper/discord.py`.

version-rust/src/extract.rs

@@ -0,0 +1,9 @@
+//! Recursive archive extraction into a plain-file tree — Rust port of
+//! `version-python/tools/extract_all.py` (extract_manifest.jsonl feeds the
+//! sorter's provenance mode; failures go to .broken/; the run never stops).
+//!
+//! Extraction backend: `sevenz-rust2` for .7z, `zip` for .zip, `unrar` for
+//! .rar, `tar`+`flate2`/`bzip2`/`xz2` for tar balls and bare compressed
+//! files. Because extraction is pure Rust (no 7z CLI writing directly onto
+//! CIFS), the python version's /var/tmp fallback + cp437 name repair is
+//! not needed here.

version-rust/src/gofile.rs

@@ -0,0 +1,2 @@
+//! GoFile API client (one guest account per run) — Rust port of
+//! `version-python/dlphelper/gofile.py`.

version-rust/src/gwshop.rs

@@ -0,0 +1,2 @@
+//! GW webshop catalog dump via Algolia — Rust port of
+//! `version-python/dlphelper/gwshop.py`.

version-rust/src/main.rs

@@ -1,7 +1,17 @@
+mod catalogue;
+mod difflib;
+mod discord;
+mod extract;
 mod globalstore;
+mod gofile;
+mod gwshop;
 mod manifest;
+mod orchestrate;
+mod sorter;
 mod state;
 mod util;
+mod warhammer_data;
+mod zero807;
 
 fn main() {
     env_logger::init();

version-rust/src/orchestrate.rs

@@ -0,0 +1,6 @@
+//! Orchestration: per-channel pipeline, gofile/0807 loops, subscriptions,
+//! --assume-done — Rust port of the logic in `version-python/dlp.py`.
+//!
+//! Failure policy (the core requirement): a failure in one URL must never
+//! abort the run. Catch, record in state.json, continue. Cooldown pauses
+//! are the only permitted stall.

version-rust/src/sorter.rs

@@ -0,0 +1,7 @@
+//! Library sorter: inspect/classify/fuzzy-match against the catalog, plan +
+//! hardlink apply, optional provenance unit mode — Rust port of
+//! `version-python/dlphelper/sorter.py`.
+//!
+//! Matching rules are fixed (see version-python/AGENTS.md §7 "Sorter matching
+//! rules"); they must be ported exactly or the Rust sorter will silently
+//! sort differently than the battle-tested python one.

version-rust/src/warhammer_data.rs

@@ -0,0 +1,137 @@
+//! Taxonomy: GW-named systems/factions, GW shop categories as classes,
+//! GROUP_OF_FACTION map — Rust port of `version-python/dlphelper/warhammer_data.py`.
+//!
+//! The tables are NOT hand-transcribed: they are dumped from the python
+//! module into `data/warhammer_data.json` (regenerate with
+//! `version-rust/tools/dump_warhammer_data.py`) and parsed here, so both
+//! versions work from byte-identical data.
+//!
+//! Python-dict ordering is preserved exactly (insertion order; a duplicate
+//! key keeps its first insertion position while the *last* value wins),
+//! because the catalogue's "most specific keyword" rule breaks ties by
+//! first occurrence among equal-length keywords.
+
+use std::sync::OnceLock;
+
+use anyhow::Context as _;
+use indexmap::IndexMap;
+use serde::Deserialize;
+
+/// Insertion-ordered pair list with python-dict semantics for building.
+fn dict_insert<V>(vec: &mut Vec<(String, V)>, key: String, value: V) {
+    if let Some(slot) = vec.iter_mut().find(|(k, _)| *k == key) {
+        slot.1 = value;
+    } else {
+        vec.push((key, value));
+    }
+}
+
+/// system -> (faction, keyword aliases), python dict order.
+pub type Systems = Vec<(String, Vec<(String, Vec<String>)>)>;
+
+#[derive(Deserialize)]
+struct Raw {
+    systems: IndexMap<String, IndexMap<String, Vec<String>>>,
+    classes: Vec<(String, Vec<String>)>,
+    group_of_faction: IndexMap<String, String>,
+}
+
+pub struct Data {
+    pub systems: Systems,
+    /// system -> (alias, faction), insertion order of first occurrence,
+    /// value = last faction seen (python dict semantics).
+    pub faction_aliases: Vec<(String, Vec<(String, String)>)>,
+    /// keyword -> class, insertion order of first occurrence.
+    pub unit_classes: Vec<(String, String)>,
+    pub group_of_faction: IndexMap<String, String>,
+}
+
+impl Data {
+    pub fn global() -> &'static Data {
+        static DATA: OnceLock<Data> = OnceLock::new();
+        DATA.get_or_init(|| Data::load().expect("embedded warhammer_data.json is valid"))
+    }
+
+    /// Parse the embedded JSON dump of the python taxonomy module.
+    fn load() -> anyhow::Result<Data> {
+        let raw: Raw = serde_json::from_str(include_str!("../data/warhammer_data.json"))
+            .context("parsing embedded warhammer_data.json")?;
+
+        // SYSTEMS in dict order
+        let mut systems: Systems = Vec::new();
+        for (system, factions) in &raw.systems {
+            let mut factions_vec: Vec<(String, Vec<String>)> = Vec::new();
+            for (faction, aliases) in factions {
+                dict_insert(&mut factions_vec, faction.clone(), aliases.clone());
+            }
+            dict_insert(&mut systems, system.clone(), factions_vec);
+        }
+
+        // FACTION_ALIASES = {system: {alias: faction ...}} in dict order
+        let mut faction_aliases: Vec<(String, Vec<(String, String)>)> = Vec::new();
+        for (system, factions) in &systems {
+            let mut aliases_vec: Vec<(String, String)> = Vec::new();
+            for (faction, aliases) in factions {
+                for alias in aliases {
+                    dict_insert(&mut aliases_vec, alias.clone(), faction.clone());
+                }
+            }
+            dict_insert(&mut faction_aliases, system.clone(), aliases_vec);
+        }
+
+        // UNIT_CLASSES = {kw: klass for klass, kws in CLASSES for kw in kws}
+        let mut unit_classes: Vec<(String, String)> = Vec::new();
+        for (klass, keywords) in &raw.classes {
+            for kw in keywords {
+                dict_insert(&mut unit_classes, kw.clone(), klass.clone());
+            }
+        }
+
+        Ok(Data {
+            systems,
+            faction_aliases,
+            unit_classes,
+            group_of_faction: raw.group_of_faction,
+        })
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn tables_load_and_are_complete() {
+        let d = Data::global();
+        assert!(d.systems.iter().any(|(s, _)| s == "Warhammer 40,000"));
+        assert!(d.systems.iter().any(|(s, _)| s == "Middle-Earth"));
+        let f40k = &d
+            .systems
+            .iter()
+            .find(|(s, _)| s == "Warhammer 40,000")
+            .unwrap()
+            .1;
+        assert!(f40k.iter().any(|(f, _)| f == "Space Wolves"));
+        // "magnus" is listed in both Monster and Hero -> last (Hero) wins
+        let magnus = d
+            .unit_classes
+            .iter()
+            .find(|(kw, _)| kw == "magnus")
+            .map(|(_, c)| c.as_str());
+        assert_eq!(magnus, Some("Hero"));
+        // duplicate alias inside a system keeps first position, last value
+        let aliases = &d
+            .faction_aliases
+            .iter()
+            .find(|(s, _)| s == "Warhammer 40,000")
+            .unwrap()
+            .1;
+        let idx = aliases.iter().position(|(a, _)| a == "skull lord").unwrap();
+        assert_eq!(aliases[idx].1, "World Eaters");
+        // position kept from FIRST insertion (Chaos Daemons block): it sits
+        // between 'slaanesh' and 'shalaxi helbane' (verified against python)
+        assert_eq!(aliases[idx - 1].0, "slaanesh");
+        assert_eq!(aliases[idx + 1].0, "shalaxi helbane");
+        assert_eq!(d.group_of_faction["Space Wolves"], "Space Marines");
+    }
+}

version-rust/src/zero807.rs

@@ -0,0 +1,2 @@
+//! curl-based downloader for 0807.st direct links — Rust port of
+//! `version-python/dlphelper/zero807.py`.

version-rust/tools/dump_warhammer_data.py

@@ -0,0 +1,29 @@
+#!/usr/bin/env python3
+"""Regenerate version-rust/data/warhammer_data.json from the python module.
+
+Run from the repo root (the version containing the python layout):
+
+    python3 version-rust/tools/dump_warhammer_data.py
+
+The Rust rewrite parses this file (src/warhammer_data.rs) instead of holding
+a hand-transcribed copy of the taxonomy tables, so the two versions work
+from byte-identical data.
+"""
+
+import json
+import sys
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parent.parent.parent / "version-python"))
+
+from dlphelper import warhammer_data as w  # noqa: E402
+
+out = Path(__file__).resolve().parent.parent / "data" / "warhammer_data.json"
+
+payload = {
+    "systems": w.SYSTEMS,
+    "classes": [[name, keywords] for name, keywords in w.CLASSES],
+    "group_of_faction": w.GROUP_OF_FACTION,
+}
+out.write_text(json.dumps(payload, ensure_ascii=False, indent=1), encoding="utf-8")
+print(f"wrote {out} ({out.stat().st_size} bytes)")