diff --git a/AGENTS.md b/AGENTS.md index d5ba075..2799f2f 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -42,15 +42,20 @@ for dedupe/cross-seeding later. ``` dlp.py CLI orchestrator (entry point) +catalogue.py CLI: classify the collection into system/faction/class dlphelper/ discord.py DCE docker export (JSON) + base64/plain URL extraction gofile.py GoFile API client (one guest account per run) zero807.py curl-based downloader for 0807.st direct links state.py state.json store + manifest.jsonl appender globalstore.py cross-run URL store (~/.local/share/dlphelper/urls.json) + catalogue.py scanner + keyword classifier + report writer + warhammer_data.py taxonomy: systems, factions, force-org classes parts/grab.sh legacy groundstone script (kept, not used by dlp.py) tools/new_repo.py Gitea repo helper (reads ~/.git-credentials itself) +tools/debug_gofile.py inspect raw gofile API response for one content id tests/test_extract.py offline extraction test (plain python3, no deps) +tests/test_catalogue.py offline classifier test ``` Data flow: @@ -166,9 +171,10 @@ never re-fetch them again): 15. `dlphelper/` modules must stay import-safe (no side effects at import). ### Validation -16. Minimum validation for any change: `python3 -m py_compile dlp.py - dlphelper/*.py`, plus the synthetic extraction test (below) if - `discord.py` changed. Document what you could *not* test (e.g. live +15. Minimum validation for any change: `python3 -m py_compile dlp.py + catalogue.py dlphelper/*.py`, plus the offline tests (`python3 + tests/test_extract.py`, `python3 tests/test_catalogue.py`) if the touched + modules changed. Document what you could *not* test (e.g. live gofile/0807 downloads need real links and tokens). 17. Full live runs need a Discord token from the user — ask for it, never ask for it to be pasted into a file or committed. @@ -187,10 +193,13 @@ never re-fetch them again): comes with hashing. 2. **Torrent stage**: `mktorrent` per downloaded folder, .torrent files into a watch dir, cross-seeding between direct-download and torrent copies. -3. **Library organization**: the long-term goal is folders per game system, - faction and army-list class (fast attack, heavy support, ...) as described - in the wiki. Per-file metadata (source message, timestamp, url) is already - captured in `manifest.jsonl` so this can happen later without re-downloads. +3. **Library organization (started)**: `catalogue.py` classifies every file + into system / faction / force-org class and can move files into + `////` with `--apply` (audited in + `catalogue_moves.jsonl`). The taxonomy lives in + `dlphelper/warhammer_data.py`; it is keyword-based and grows with the + collection - unclassified files are reported, never guessed. Manual fixes + go into an overrides JSON (`--overrides`). 4. **Watch mode**: `--subs subscriptions.json` + a cron/timer already gives a primitive version; a real watch mode (per-subscription schedules, tear-off notifications, discord pings on new links) builds on it. diff --git a/README.md b/README.md index 17148f6..c4a326b 100644 --- a/README.md +++ b/README.md @@ -52,6 +52,29 @@ then run (e.g. from a daily cron/systemd timer): Subs mode re-exports each channel to pick up new links; `state.json` and the global store make sure only genuinely new URLs are downloaded. +## Cataloguing the collection + +`catalogue.py` classifies every downloaded file into game system, faction and +force-organization class (HQ / Troops / Elites / Fast Attack / Heavy Support / +Flyer / Dedicated Transport / Lords of War / Fortification) using a keyword +taxonomy in `dlphelper/warhammer_data.py`: + + # preview a dlp.py run dir (manifest + downloads) + python3 catalogue.py /path/to/run_dir + + # include the manual-era directories + python3 catalogue.py /path/to/run_dir --scan /big/storage/manual --out-md catalogue.md + + # catalogue arbitrary dirs only + python3 catalogue.py --scan /big/storage/manual + + # actually reorganize into a library tree (audited, only classified files) + python3 catalogue.py --apply --library /big/storage/warhammer_library + +Outputs `catalogue.md` (readable) and `catalogue.jsonl` (machine, with source +attribution per file). Unclassified files are listed, never guessed; fix them +with an overrides JSON (`--overrides overrides.json`). + ## Already downloaded the old way? `--assume-done` imports a plain URL list into the global store so those links diff --git a/catalogue.py b/catalogue.py new file mode 100644 index 0000000..d95927f --- /dev/null +++ b/catalogue.py @@ -0,0 +1,118 @@ +#!/usr/bin/env python3 +"""Catalogue CLI: classify the collection into system / faction / class. + +Examples: + # catalogue the dlp.py run dir (manifest + downloads) + python3 catalogue.py /path/to/run_dir + + # additionally scan manual-era directories + python3 catalogue.py /path/to/run_dir --scan /big/storage/manual/Aeldari + + # catalogue ONLY manual dirs, without any run dir + python3 catalogue.py --scan /big/storage/manual --out-md catalogue.md + + # fix misclassifications (see overrides format below) + python3 catalogue.py --overrides overrides.json + + # actually move files into //// + python3 catalogue.py --apply --library /big/storage/warhammer_library + +overrides.json format (keys are file names or full paths, case-insensitive): + { + "some file.zip": {"faction": "Aeldari", "class": "Heavy Support", + "system": "Warhammer 40,000"} + } + +Outputs (default into the first given dir, or cwd): + catalogue.md human-readable, grouped system > faction > class + catalogue.jsonl one JSON line per file, with source attribution +""" + +from __future__ import annotations + +import argparse +import logging +import sys +from pathlib import Path + +from dlphelper.catalogue import ( + build_catalogue, + format_size, + apply_moves, + write_reports, +) + +log = logging.getLogger("catalogue") + + +def main() -> None: + parser = argparse.ArgumentParser( + description="Classify the Warhammer collection into factions and classes." + ) + parser.add_argument("run_dir", nargs="?", type=Path, + help="a dlp.py run dir (uses its manifest.jsonl and downloads/)") + parser.add_argument("--scan", action="append", default=[], type=Path, metavar="DIR", + help="additional directory to scan for archive files (repeatable)") + parser.add_argument("--manifest", type=Path, default=None, metavar="FILE", + help="explicit manifest.jsonl (default: /manifest.jsonl)") + parser.add_argument("--overrides", type=Path, default=None, metavar="FILE", + help="manual classification overrides (JSON)") + parser.add_argument("--out-md", type=Path, default=None, metavar="FILE") + parser.add_argument("--out-jsonl", type=Path, default=None, metavar="FILE") + parser.add_argument("--apply", action="store_true", + help="move files into the library structure (default: preview)") + parser.add_argument("--library", type=Path, default=None, metavar="DIR", + help="library root for --apply (required with --apply)") + args = parser.parse_args() + + logging.basicConfig(level=logging.INFO, + format="%(asctime)s %(levelname)-7s %(message)s", + datefmt="%H:%M:%S") + + run_dir = args.run_dir + manifests = [args.manifest] if args.manifest else ( + [run_dir / "manifest.jsonl"] if run_dir else [] + ) + scan_dirs = list(args.scan) + if run_dir and not args.manifest: + # run dir is scanned as a whole (covers files not yet in the manifest) + scan_dirs.append(run_dir) + + if not manifests and not scan_dirs: + parser.error("give a run_dir or at least one --scan directory") + + out_base = run_dir if run_dir else Path.cwd() + out_md = args.out_md or out_base / "catalogue.md" + out_jsonl = args.out_jsonl or out_base / "catalogue.jsonl" + + entries = build_catalogue(manifests, scan_dirs, args.overrides) + write_reports(entries, out_md, out_jsonl) + + classified = sum(1 for e in entries if e.faction != "Unclassified") + log.info("Catalogued %d files (%d classified) -> %s", len(entries), classified, out_md) + + # compact per-faction summary to the console + summary: dict[tuple[str, str], list[int]] = {} + for e in entries: + key = (e.system, e.faction) + n, size = summary.get(key, (0, 0)) + summary[key] = [n + 1, size + e.size] + for (system, faction), (n, size) in sorted(summary.items()): + log.info(" %-18s %-32s %4d files %s", system, faction, n, format_size(size)) + + if args.apply: + if not args.library: + parser.error("--apply needs --library DIR") + if not args.run_dir: + parser.error("--apply needs a run_dir (files are moved out of it)") + moved = apply_moves(entries, args.library, args.library / "catalogue_moves.jsonl") + log.info("Moved %d files into %s (audit: catalogue_moves.jsonl)", moved, args.library) + # rewrite reports with updated paths + write_reports(entries, out_md, out_jsonl) + + +if __name__ == "__main__": + try: + main() + except KeyboardInterrupt: + sys.exit(130) diff --git a/dlphelper/catalogue.py b/dlphelper/catalogue.py new file mode 100644 index 0000000..8d93e77 --- /dev/null +++ b/dlphelper/catalogue.py @@ -0,0 +1,290 @@ +"""Catalogue: classify collected files into game system / faction / class. + +The catalogue is the bridge between "a pile of downloads" and the later +library stage: it reads manifest.jsonl (with full source attribution) and/or +scans arbitrary directories for archive files, classifies each file against +the taxonomy in warhammer_data.py, and writes machine- (catalogue.jsonl) and +human-readable (catalogue.md) reports. + +Classification is keyword-based and deliberately conservative: files that +cannot be classified land in the "unclassified" bucket so gaps become visible +instead of producing wrong assignments. A manual overrides file can pin +individual files. + +Nothing is moved by default - use --apply for that (it moves files into +//// and records every move in +catalogue_moves.jsonl so it can be audited/undone). +""" + +from __future__ import annotations + +import json +import logging +import re +import shutil +from dataclasses import dataclass, field +from pathlib import Path + +from dlphelper.warhammer_data import FACTION_ALIASES, SYSTEMS, UNIT_CLASSES + +log = logging.getLogger(__name__) + +ARCHIVE_SUFFIXES = {".zip", ".7z", ".rar", ".tar", ".gz"} +UNKNOWN = "Unknown" +UNCLASSIFIED_FACTION = "Unclassified" +UNCLASSIFIED_CLASS = "Unclassified" + + +@dataclass +class CatalogueEntry: + """One classified file.""" + + path: str + name: str + size: int + system: str = UNKNOWN + faction: str = UNCLASSIFIED_FACTION + klass: str = UNCLASSIFIED_CLASS + source: dict = field(default_factory=dict) # manifest attribution if known + + +def _clean_name(name: str) -> str: + stem = Path(name).stem + stem = re.sub(r"[_\.\+\[\]\(\)]+", " ", stem) + stem = re.sub(r"\s+", " ", stem).strip().lower() + return stem + + +def _kw_regex(keyword: str) -> str: + """Word-boundary regex for a keyword, tolerating a plural 's'.""" + return rf"(? tuple[str, str] | None: + """Return (system, faction) with the most specific keyword hit.""" + best: tuple[int, str, str] | None = None # (len, system, faction) + for system, aliases in FACTION_ALIASES.items(): + for keyword, faction in aliases.items(): + if re.search(_kw_regex(keyword), clean): + if best is None or len(keyword) > best[0]: + best = (len(keyword), system, faction) + return (best[1], best[2]) if best else None + + +def _match_class(clean: str) -> str | None: + """Return the most specific force-organization class keyword hit.""" + best: tuple[int, str] | None = None + for keyword, klass in UNIT_CLASSES.items(): + if re.search(_kw_regex(keyword), clean): + if best is None or len(keyword) > best[0]: + best = (len(keyword), klass) + return best[1] if best else None + + +def classify(name: str) -> tuple[str, str, str]: + """Classify a file name -> (system, faction, class).""" + clean = _clean_name(name) + faction_hit = _match_faction(clean) + klass = _match_class(clean) + + if faction_hit: + system, faction = faction_hit + # Non-40k systems: keep their class if we found one, else Unclassified + if system != "Warhammer 40,000" and klass is None: + return system, faction, UNCLASSIFIED_CLASS + return system, faction, klass or UNCLASSIFIED_CLASS + + # No faction match: a class keyword alone cannot attribute a system + return UNKNOWN, UNCLASSIFIED_FACTION, UNCLASSIFIED_CLASS + + +def scan_manifest(manifest_path: Path) -> list[CatalogueEntry]: + """Entries from a manifest.jsonl (preferred: has source attribution).""" + entries: list[CatalogueEntry] = [] + seen: set[str] = set() + if not manifest_path.exists(): + return entries + for line in manifest_path.read_text(encoding="utf-8").splitlines(): + line = line.strip() + if not line: + continue + try: + row = json.loads(line) + except ValueError: + log.warning("Skipping malformed manifest line: %s", line[:80]) + continue + path = row.get("path") or "" + if path in seen: + continue + seen.add(path) + entries.append(CatalogueEntry( + path=path, + name=row.get("name") or Path(path).name, + size=int(row.get("size") or 0), + source={ + "channel": row.get("channel"), + "url": row.get("url"), + "message_id": row.get("message_id"), + "timestamp": row.get("timestamp"), + "author": row.get("author"), + }, + )) + return entries + + +def scan_directory(directory: Path) -> list[CatalogueEntry]: + """Entries from walking a directory tree for archive files (manual era).""" + entries: list[CatalogueEntry] = [] + for path in sorted(directory.rglob("*")): + if not path.is_file() or path.suffix.lower() not in ARCHIVE_SUFFIXES: + continue + try: + size = path.stat().st_size + except OSError: + size = 0 + entries.append(CatalogueEntry(path=str(path), name=path.name, size=size)) + return entries + + +def load_overrides(path: Path | None) -> dict[str, dict]: + """overrides.json: {"/path/or/filename": {"faction": ..., "class": ..., "system": ...}}""" + if path is None or not path.exists(): + return {} + return json.loads(path.read_text(encoding="utf-8")) + + +def apply_overrides(entries: list[CatalogueEntry], overrides: dict[str, dict]) -> int: + count = 0 + by_name = {e.name.lower(): e for e in entries} + for key, fix in overrides.items(): + entry = by_name.get(key.lower()) + if entry is None: + log.warning("Override target not found: %s", key) + continue + if "system" in fix: + entry.system = fix["system"] + if "faction" in fix: + entry.faction = fix["faction"] + if "class" in fix: + entry.klass = fix["class"] + count += 1 + return count + + +def format_size(n: int) -> str: + size = float(n) + for unit in ("B", "KiB", "MiB", "GiB", "TiB"): + if size < 1024 or unit == "TiB": + return f"{size:.1f} {unit}" + size /= 1024 + return f"{n} B" + + +def write_reports( + entries: list[CatalogueEntry], + out_md: Path, + out_jsonl: Path, +) -> None: + # machine report + with out_jsonl.open("w", encoding="utf-8") as fh: + for e in entries: + fh.write(json.dumps({ + "path": e.path, + "name": e.name, + "size": e.size, + "system": e.system, + "faction": e.faction, + "class": e.klass, + "source": {k: v for k, v in e.source.items() if v}, + }, ensure_ascii=False) + "\n") + + # human report + grouped: dict[str, dict[str, dict[str, list[CatalogueEntry]]]] = {} + for e in entries: + grouped.setdefault(e.system, {}).setdefault(e.faction, {}).setdefault( + e.klass, []).append(e) + + lines: list[str] = ["# Warhammer Collection Catalogue", ""] + total_size = sum(e.size for e in entries) + classified = sum(1 for e in entries if e.faction != UNCLASSIFIED_FACTION) + lines.append(f"{len(entries)} files, {format_size(total_size)} total, " + f"{classified} classified, " + f"{len(entries) - classified} unclassified") + lines.append("") + + for system in sorted(grouped): + factions = grouped[system] + all_sys_files = [e for f in factions.values() for c in f.values() for e in c] + sys_size = sum(e.size for e in all_sys_files) + lines.append(f"## {system} ({len(all_sys_files)} files, {format_size(sys_size)})") + for faction in sorted(factions): + classes = factions[faction] + fac_size = sum(e.size for c in classes.values() for e in c) + lines.append(f"### {faction} ({format_size(fac_size)})") + for klass in sorted(classes): + files = classes[klass] + lines.append(f"- **{klass}** ({len(files)} files, {format_size(sum(e.size for e in files))})") + for e in sorted(files, key=lambda x: x.name.lower()): + src = "" + if e.source.get("url"): + src = f" — from {e.source['url']}" + lines.append(f" - {e.name} ({format_size(e.size)}){src}") + lines.append("") + + out_md.write_text("\n".join(lines), encoding="utf-8") + + +def apply_moves( + entries: list[CatalogueEntry], + library_root: Path, + moves_log: Path, +) -> int: + """Move classified files into the library structure. Audited in moves_log.""" + moves_log.parent.mkdir(parents=True, exist_ok=True) + moved = 0 + with moves_log.open("a", encoding="utf-8") as logfh: + for e in entries: + if e.faction == UNCLASSIFIED_FACTION: + continue # never move what we cannot classify + if e.system == UNKNOWN: + continue + src = Path(e.path) + if not src.exists(): + log.warning("Cannot move, missing: %s", src) + continue + dest_dir = library_root / e.system / e.faction / e.klass + dest = dest_dir / src.name + if dest.exists(): + log.warning("Skip, destination exists: %s", dest) + continue + dest_dir.mkdir(parents=True, exist_ok=True) + shutil.move(str(src), str(dest)) + logfh.write(json.dumps({"from": str(src), "to": str(dest)}) + "\n") + e.path = str(dest) + moved += 1 + return moved + + +def build_catalogue( + manifests: list[Path], + scan_dirs: list[Path], + overrides_path: Path | None, +) -> list[CatalogueEntry]: + entries: list[CatalogueEntry] = [] + seen: set[str] = set() + for m in manifests: + for e in scan_manifest(m): + if e.path not in seen: + seen.add(e.path) + entries.append(e) + for d in scan_dirs: + for e in scan_directory(d): + if e.path not in seen: + seen.add(e.path) + entries.append(e) + + for e in entries: + e.system, e.faction, e.klass = classify(e.name) + + apply_overrides(entries, load_overrides(overrides_path)) + return entries diff --git a/dlphelper/warhammer_data.py b/dlphelper/warhammer_data.py new file mode 100644 index 0000000..0aae732 --- /dev/null +++ b/dlphelper/warhammer_data.py @@ -0,0 +1,340 @@ +"""Warhammer taxonomy and unit keyword tables used by the catalogue. + +Two layers: +- SYSTEMS: game system -> faction -> keyword aliases (matched against file + names). Faction detection is the primary signal; unit names double as + faction hints since they are listed under their faction. +- CLASSES: keyword -> force-organization class (40k Force Organization slots: + HQ, Troops, Elites, Fast Attack, Heavy Support, Flyer, Dedicated Transport, + Lords of War, Fortification). + +The unit table grows with the collection; it does not attempt to enumerate +every unit Games Workshop ever released - unknown files land in the +"unclassified" bucket of the catalogue report so gaps become visible. +""" + +from __future__ import annotations + +# --------------------------------------------------------------------------- +# Game systems and factions (keyword aliases, lowercase) +# --------------------------------------------------------------------------- + +SYSTEMS: dict[str, dict[str, list[str]]] = { + "Warhammer 40,000": { + # -- Imperium ------------------------------------------------------- + "Adeptus Astartes": [ + "space marine", "astartes", "primaris", "ultramarines", "blood angels", + "dark angels", "space wolves", "black templars", "imperial fists", + "salamanders", "raven guard", "iron hands", "white scars", "grey knights", + "deathwatch", "crimson fists", "flesh tearers", "ravenwing", "deathwing", + "inner circle", "wulfen", "thunderwolf", "intercessor", "reiver", "aggressor", "inceptor", + "hellblaster", "eradicator", "bladeguard", "sternguard", "devastator", + "tactical squad", "terminator squad", "assault marine", + "redemptor", "contemptor", "dreadnought", "invictor", "repulsor", + "land raider", "land speeder", "predator", "vindicator", "whirlwind", + "drop pod", "razorback", "stormraven", "storm talon", "stormhawk", + "caestus", "chaplain", "librarian", "apothecary", "vexilus", + ], + "Astra Militarum": [ + "astra militarum", "imperial guard", "cadian", "catachan", "krieg", + "death korps", "tempestus", "scion", "valhallan", "steel legion", + "tallarn", "vostroyan", "tanith", "gaunt's ghosts", "leman russ", + "chimera", "basilisk", "hydra", "wyvern", "manticore", "deathstrike", + "sentinel", "taurox", "commissar", "bullgryn", "ogryn", "ratling", + "heavy weapon squad", "baneblade", "stormlord", "shadowsword", + "hellhammer", "malcador", "macharius", "valkyrie", "vulture", + "primaris psyker", "astropath", "priest", + ], + "Adeptus Mechanicus": [ + "adeptus mechanicus", "mechanicum", "admech", "skitarii", + "cult mechanicus", "sicarian", "ruststalker", "infiltrator", + "kataphron", "kastelan", "ironstrider", "sydonian", + "onager", "dunecrawler", "maniple", "archmagos", "magos", "dominus", + "manipulus", "enginseer", "transuranic", "tech-priest", "cybernetica", + "thallax", "castellax", "vorax", "ordinatus", "belisarius cawl", + ], + "Adepta Sororitas": [ + "adepta sororitas", "sisters of battle", "sister of battle", + "battle sister", "repentia", "retributor", "canoness", "dialogus", + "hospitaller", "seraphim", "zephyrim", "paragon warsuit", + "morvenn vahl", "junith", "immersa", "exorcist", "immolator", + ], + "Adeptus Custodes": [ + "adeptus custodes", "custodes", "custodian", "allarus", "vertus praetor", + "venatari", "aquilon", "sagittarum", "hetaeron", "trajann", "constantin valdor", + "sisters of silence", "witchseeker", "prosecutor", "vigilator", + "caladius", "coronus", "telemon", "achillus", "galatus", "pallas", + ], + "Imperial Knights": [ + "imperial knight", "knight errant", "knight paladin", "knight crusader", + "knight warden", "knight gallant", "knight lancer", "knight acheron", + "knight castellan", "knight valiant", "knight preceptor", "questoris", + "armiger", "warglaive", "helverin", "morialtis", "canis rex", + "cerastus", "atropos", "seneschal", + ], + "Inquisition & Officio Assassinorum": [ + "inquisition", "inquisitor", "astra telepathica", "witch hunter", + "coteaz", "eisenhorn", "ravenor", "kyria draxus", + "officio assassinorum", "culexus", "vindicare", "callidus", "eversor", + "saint celestine", "living saint", "assassin", + ], + # -- Chaos ---------------------------------------------------------- + "Chaos Space Marines": [ + "chaos space marine", "black legion", "iron warriors", "word bearers", + "night lords", "alpha legion", "red corsairs", "world eaters", + "emperor's children", "chaos terminator", "chosen", "havoc", + "raptor", "warp talon", "chaos cultist", "cultist", "daemon prince", + "chaos lord", "chaos sorcerer", "abaddon", "huron blackheart", + "kharn", "lucius the eternal", "ahriman", "typhus", "fabius bile", + "vashtorr", "defiler", "forgefiend", "maulerfiend", "helldrake", + "helbrute", "chaos spawn", "obliterator", "chaos rhino", + "chaos vindicator", "spartan assault", "mastodon", "typhon", + ], + "Death Guard": [ + "death guard", "nurgle", "plague marine", "deathshroud", "mortarion", + "plaguecaster", "poxwalker", "blightlord", "foetid bloat-drone", + "plagueburst crawler", "myphitic blight-hauler", "blight hauler", + "tallyman", "foul blightspawn", "biologus putrifier", + "plague surgeon", "necrosius", + ], + "Thousand Sons": [ + "thousand sons", "tzeentch", "rubric marine", "rubricae", + "scarab occult", "magnus the red", "ahzek ahriman", "arhiman", + "exalted sorcerer", "osiron", "mutalith vortex beast", "tzaangor", + "prospero", + ], + "Chaos Daemons": [ + "chaos daemon", "chaos daemons", "daemonette", "bloodletter", + "plaguebearer", "horror", "screamer", "flesh hound", "juggernaut", + "keeper of secrets", "skarbrand", "kairos fateweaver", "rotigus", + "be'lakor", "great unclean one", "lord of change", "soul grinder", + "skull cannon", "burning chariot", "exalted flamer", + "fiend of slaanesh", "plague drone", "nurgling", "herald", + "skulltaker", "khorne", "slaanesh", "skull lord", + "shalaxi helbane", + ], + "Chaos Knights": [ + "chaos knight", "knight desecrator", "knight despoiler", + "knight rampager", "kharibdyss", "renegade knight", + ], + # -- Xenos ---------------------------------------------------------- + "Orks": [ + "ork", "orks", "orkz", "nob", "boyz", "grot", "weirdboy", "painboy", + "warboss", "ghazghkull", "meganob", "deff dread", "killa kan", + "deffkopta", "stormboy", "kommando", "burna boy", "loota", + "trukk", "battlewagon", "gorkanaut", "morkanaut", "stompa", "squig", + "zagstruk", "snikrot", "speed freek", "boomdakka", "flash gitz", + "wazbom blasta", "dakka jet", "burna-bomber", "blitza-bomber", + "da troopz", "dad and sonz", "ass sassins", "abby boi", + ], + "T'au Empire": [ + "tau", "t'au", "ta'u", "kroot", "vespid", "battlesuit", "crisis suit", + "broadside", "r'varna", "ghostkeel", "stormsurge", + "fire warrior", "pathfinder team", "devilfish", "hammerhead", + "sky ray", "piranha", "sniper drone team", "ethereal", + "commander farsight", "commander shadowsun", "aun'va", "darkstrider", + "longstrike", "mont'ka", "assorted drones", "breacher", + "bomber", "tetra", "barracuda", "sun shark", "razorshark", + "kv128 stormsurge", "tx4 piranha", "kv46", + ], + "Tyranids": [ + "tyranid", "hive tyrant", "swarmlord", "carnifex", "hormagaunt", + "termagant", "termagaunt", "genestealer brood", "zoanthrope", "venomthrope", + "exocrine", "haruspex", "tyranid prime", "tyranid warrior", + "ravener", "gargoyle", "spore mine", "mawloc", "trygon", + "harridan", "harpy", "crone", "tervigon", "barbed hierodule", + "hierodule", "hierophant", "neurothrope", "deathleaper", "lictor", "pyrovore", + "biovore", "broodlord", "bio-titan", "dominatrix", + ], + "Genestealer Cults": [ + "genestealer cult", "genestealer", "aberrant", "acolyte", "neophyte hybrid", + "primus", "magus", "patriarch", "purestrain genestealer", + "sanctus", "kelermorph", "goliath truck", "rockgrinder", + "jackal alcaeus", "cult limo", "hijacked", + ], + "Necrons": [ + "necron", "c'tan", "canoptek", "wraith", "spyder", "scarab", + "tesseract vault", "doomsday ark", "annihilation barge", + "ghost ark", "monolith", "doom scythe", "night scythe", + "imotekh", "szarekh", "nemesor zahndrekh", "vargard obyron", + "lokhrust", "skorpekh", "flayed one", "deathmark", "lychguard", + "necron warrior", "necron immortal", "ovoid", "dzaan", + ], + "Aeldari": [ + "aeldari", "eldar", "craftworld", "ulthwe", "biel-tan", "iyanden", + "alaitoc", "saim-hann", "ynnari", "farseer", "warlock", + "spiritseer", "autarch", "asurmen", "guardian defender", + "ranger", "wraithguard", "wraithblades", "wraithlord", + "wraithknight", "dire avenger", "howling banshee", + "striking scorpion", "swooping hawk", "warp spider", + "fire dragon", "dark reaper", "shining spears", "windrider", + "vyper", "wave serpent", "fire prism", "night spinner", + "war walker", "hemlock wraithfighter", "crimson hunter", + "shade runner", "skyrunner", + ], + "Drukhari": [ + "drukhari", "dark eldar", "kabal", "wych", "wrack", "haemonculus", + "coven", "reaver jetbike", "hellion", "scourge", "mandrake", + "kabalite warrior", "raider", "ravager", "venom", "razorwing", + "voidraven", "grotesque", "talos", "cronos", + "beast pack", "khymera", "razorwing flock", "asdrubael vect", + "lilith", "sathonyx", "the twin lance", "luriel", + ], + "Harlequins": [ + "harlequin", "death jester", "shadowseer", "solitaire", + "troupe", "starweaver", "voidweaver", + ], + "Ynnari": [ + "ynnari", "ynnead", "visarch", "yncarne", "yvraine", + ], + }, + "Other Systems": { + "Age of Sigmar": [ + "age of sigmar", "stormcast", "eternals", "orruk", "warclan", + "kruleboyz", "gloomspite", "sylvaneth", "seraphon", "ossiarch", + "nighthaunt", "lumineth", "daughters of khaine", "fyreslayers", + "kharadron", "idoneth", "blades of khorne", "maggotkin", + "gaunt summoner", "soulblight", "freeguild", "cities of sigmar", + "skaven", "chaos warriors", "saurus", "skink", "liberator", + "sequitor", "evocator", "vindictor", "kaldor draigo", + ], + "Horus Heresy": [ + "horus heresy", "legiones astartes", "pre-heresy", + "leviathan pattern", "fire raptor", "sicaran", "fellblade", + "kratos", "justaerin", "eisenstein", "veteran tactical", + "seeker squad", "suzerain", "invictarus", "fulgurite", + ], + "Kill Team": [ + "kill team", "killteam", "killzone", "operative", + "hand of the archon", "void-dance troupe", "veteran guardsmen", + ], + "Necromunda": [ + "necromunda", "goliath gang", "escher", "orlock", "cawdor", + "van saar", "ash wastes", "underhive", + ], + "Legions Imperialis / Epic": [ + "legions imperialis", "titan legio", "reaver titan", + "warhound titan", "warlord titan", "imperator titan", + "warmaster titan", + ], + "Warhammer The Old World": [ + "the old world", "bretonnia", "greenskins", "tomb kings", + "khemri", "chaos dwarf", "dog of war", "grail knight", + ], + }, +} + +# --------------------------------------------------------------------------- +# Force Organization classes (keyword -> class) +# --------------------------------------------------------------------------- + +CLASSES: list[tuple[str, list[str]]] = [ + ("Lords of War", [ + "titan", "stompa", "gargantuan", "hierophant", "harridan", "hierodule", + "baneblade", "stormlord", "shadowsword", "hellhammer", "malcador", + "imperator", "warmaster", "tesseract vault", "plaguereaper", + "warhound", "warlord titan", + ]), + ("Fortification", ["fortification", "bunker", "bastion", "wall of martyrs"]), + ("Flyer", [ + "flyer", "heldrake", "dakka jet", "burna-bomber", "blitza-bomber", + "bomber", "sky ray", "stormraven", "storm talon", "stormhawk", + "caestus", "valkyrie", "vulture", "thunderbolt", "lightning", + "marauder", "razorwing", "voidraven", "crimson hunter", + "hemlock", "doom scythe", "night scythe", "harpy", "crone", + "fire raptor", "barracuda", "sun shark", "razorshark", + "wazbom blasta", + ]), + ("Dedicated Transport", [ + "transport", "rhino", "razorback", "chimera", "taurox", "trukk", + "battlewagon", "devilfish", "wave serpent", "ghost ark", + "land raider", "repulsor", "drop pod", "starweaver", "voidweaver", + "goliath truck", "rockgrinder", "cult limo", "raider", "venom", + "spartan assault", + ]), + ("Fast Attack", [ + "fast attack", "assault marine", "assault intercessor", "inceptor", + "outrider", "bike squad", "scout biker", "thunderwolf", "raptor", + "warp talon", "flesh hound", "screamer", "fiend of slaanesh", + "hellion", "reaver jetbike", "scourge", "mandrake", "swooping hawk", + "shining spears", "windrider", "vyper", "war walker", + "canoptek wraith", "canoptek scarab", "canoptek spyder", + "ravener", "gargoyle", "spore mine", "mawloc", "trygon", + "pyrovore", "khymera", "beast pack", "razorwing flock", + "skyrunner", "zephyrim", "seraphim", "sentinel", "jackal", + "stormboy", "deffkopta", "warbiker", "skorpekh destroyer", + "sicarian", "ironstrider", "sydonian", + ]), + ("Heavy Support", [ + "heavy support", "devastator", "eradicator", "hellblaster", "havoc", + "loota", "burna boy", "carnifex", "exocrine", "haruspex", + "zoanthrope", "venomthrope", "biovore", "hierodule", + "wraithlord", "wraithknight", "fire prism", "night spinner", + "predator", "vindicator", "whirlwind", "basilisk", "hydra", + "wyvern", "manticore", "deathstrike", "onager dunecrawler", + "kastelan", "kataphron", "hammerhead", "broadside", "stormsurge", + "r'varna", "soul grinder", "defiler", "forgefiend", "maulerfiend", + "plagueburst crawler", "bloat-drone", "blight hauler", "mutalith", + "annihilation barge", "doomsday ark", "obront", "dark reaper", + "retributor", "paragon warsuit", "morvenn", "exorcist", "immolator", + "caladius", "coronus grav", "telemon", "achillus", "galatus", + "pallas grav", "skorpekh", + ]), + ("Elites", [ + "elites", "veteran", "terminator", "sternguard", "bladeguard", + "aggressor", "reiver", "vanguard veteran", "dreadnought", + "redemptor", "contemptor", "leviathan pattern", "wulfen", + "wolf guard", "deathshroud", "plague marine", "blightlord", + "poxwalker", "rubric marine", "rubricae", "scarab occult", + "tzaangor", "chosen", "possessed", "obliterator", "daemonette", + "bloodletter", "plaguebearer", "nurgling", "juggernaut", + "chaos cultist", "cultist", "acolyte hybrid", "aberrant", + "neophyte", "purestrain", "genestealer", "lictor", "deathmark", + "flayed one", "lychguard", "wraithguard", "wraithblades", + "howling banshee", "striking scorpion", "warp spider", + "fire dragon", "incubi", "wrack", "grotesque", "talos", "cronos", + "troupe", "killa kan", "deff dread", "meganob", "nobz", "nob", + "kommando", "vespid", "sanguinary guard", "sanguinor", + "ancient", "allarus", "vertus praetor", "venatari", "aquilon", + "sagittarum", "witchseeker", "prosecutor", "vigilator", + "vindicare", "callidus", "eversor", "culexus", "assassin", + ]), + ("HQ", [ + "hq", "captain", "chapter master", "primarch", "lieutenant", + "librarian", "chaplain", "apothecary", "farseer", "warlock", + "spiritseer", "autarch", "warboss", "big mek", "weirdboy", + "painboy", "commander", "ethereal", "hive tyrant", "swarmlord", + "tyranid prime", "neurothrope", "deathleaper", "patriarch", + "primus", "magus", "sanctus", "kelermorph", "overlord", + "imotekh", "szarekh", "zahndrekh", "obyron", "plaguecaster", + "necrosius", "chaos lord", "chaos sorcerer", "daemon prince", + "exalted champion", "herald", "skulltaker", "kairos", "rotigus", + "be'lakor", "great unclean one", "lord of change", + "keeper of secrets", "skarbrand", "kharn", "abaddon", "typhus", + "mortarion", "magnus", "ahriman", "lucius", "fabius bile", + "vashtorr", "canoness", "morvenn", "junith", "immersa", + "dialogus", "hospitaller", "trajann", "valdor", "inquisitor", + "coteaz", "eisenhorn", "ravenor", "draxus", "celestine", + "commissar", "priest", "enginseer", "dominus", "manipulus", + "archmagos", "magos", "belisarius cawl", "farsight", "shadowsun", + "aun'va", "darkstrider", "longstrike", "shade runner", + ]), + ("Troops", [ + "troops", "battleline", "tactical squad", "intercessor", + "incursor", "infiltrator", "infantry squad", "conscript", + "krieg", "catachan", "cadian", "vostroyan", "penal legion", + "ratling", "veteran squad", "fire warrior", "pathfinder team", + "kroot", "termagant", "hormagaunt", "necron warrior", + "necron immortal", "guardian defender", "storm guardian", + "kabalite warrior", "boyz", "gretchin", + "scion", "scout squad", + ]), +] + +# Flattened per-system faction keyword index for quick matching +FACTION_ALIASES: dict[str, dict[str, str]] = { + system: {alias: faction for faction, aliases in factions.items() for alias in aliases} + for system, factions in SYSTEMS.items() +} + +UNIT_CLASSES: dict[str, str] = {kw: klass for klass, kws in CLASSES for kw in kws} diff --git a/tests/test_catalogue.py b/tests/test_catalogue.py new file mode 100644 index 0000000..1590e20 --- /dev/null +++ b/tests/test_catalogue.py @@ -0,0 +1,61 @@ +#!/usr/bin/env python3 +"""Offline classifier test for the catalogue (no network, no deps). + +Run with: python3 tests/test_catalogue.py +""" + +from __future__ import annotations + +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) + +from dlphelper.catalogue import classify # noqa: E402 + +CASES = [ + # (filename, expected system, expected faction, expected class or None-any) + ("Thunderwolf Cavalry.7z", "Warhammer 40,000", "Adeptus Astartes", "Fast Attack"), + ("Wulfen.zip", "Warhammer 40,000", "Adeptus Astartes", "Elites"), + ("Keeper of Secrets & Shalaxi Helbane.7z", "Warhammer 40,000", "Chaos Daemons", "HQ"), + ("Assorted Drones.zip", "Warhammer 40,000", "T'au Empire", None), + ("Commander Shadowsun.zip", "Warhammer 40,000", "T'au Empire", "HQ"), + ("Devilfish.zip", "Warhammer 40,000", "T'au Empire", "Dedicated Transport"), + ("Battle Sister Scanz.zip", "Warhammer 40,000", "Adepta Sororitas", None), + ("Knights.zip", None, None, None), # ambiguous keyword alone: unclassified + ("Khorne Demon Enginez and skull lordz.zip", "Warhammer 40,000", "Chaos Daemons", None), + ("Cultists.zip", "Warhammer 40,000", "Chaos Space Marines", "Elites"), + ("Da Troopz.zip", "Warhammer 40,000", "Orks", None), + ("Seraphon.zip", "Other Systems", "Age of Sigmar", None), + ("Leviathan Dreadnought.zip", "Warhammer 40,000", "Adeptus Astartes", "Elites"), + ("Poxwalkers.zip", "Warhammer 40,000", "Death Guard", "Elites"), + ("Kabalite Warriors.zip", "Warhammer 40,000", "Drukhari", "Troops"), + ("Canoptek Wraiths.zip", "Warhammer 40,000", "Necrons", "Fast Attack"), + ("random thing with no keywords.zip", None, None, None), +] + + +def main() -> None: + failures = 0 + for name, exp_system, exp_faction, exp_class in CASES: + system, faction, klass = classify(name) + problems = [] + if exp_system and system != exp_system: + problems.append(f"system={system!r} != {exp_system!r}") + if exp_faction and faction != exp_faction: + problems.append(f"faction={faction!r} != {exp_faction!r}") + if exp_class and klass != exp_class: + problems.append(f"class={klass!r} != {exp_class!r}") + if problems: + failures += 1 + print(f"FAIL {name!r}: {', '.join(problems)}") + else: + print(f"ok {name!r} -> {system} / {faction} / {klass}") + if failures: + print(f"\n{failures} failures") + sys.exit(1) + print("\ntest_catalogue: OK") + + +if __name__ == "__main__": + main()