Ben_Kosytorz/Warhammer_DLP_Helper
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Switch library hierarchy to GW webshop taxonomy

Browse Source 
- warhammer_data.py: GW naming throughout - Space Marine chapters are
  their own factions under a 'Space Marines' group, systems match the
  shop (Age of Sigmar, The Horus Heresy, Other Games, The Old World,
  Middle-Earth), classes are GW shop categories (Battleline, Infantry,
  Vehicle, Walker, Flyer, Monster, Mounted, Hero, Titanic), group map
  extended to AoS grand alliances / Other Games / Old World / Middle-Earth
- gwshop.py: hierarchy remap - GW depth-4 paths are system/group/faction/
  class (the shop's last level is the class layer), depth-5 sources keep
  a subfaction before it; catalog snapshot regenerated accordingly
- sorter.py: catalog entry is hierarchy-authoritative (group, class, and
  faction when GW moved a unit or keyword tables are stale); generic GW
  branches ('Unit Type') and a Battleline default never degrade a
  confident keyword classification; whole-catalog retry when the faction
  pool has no confident match (handles units GW moved between factions);
  whole-catalog matching also runs when keyword classification found no
  faction; invalid (HTML) items never fuzzy-match; default catalog is
  now data/gwshop_catalog.json
- catalogue.py: --apply moves include the group layer, same structure
  as the sorter
- docs/SORTING.md: hierarchy rewritten to GW naming with examples
- tests: expectations updated to GW naming (chapters, unit types)
- verified live: 4/4 real files in the test batch now catalog-match,
  e.g. The Twin Lance -> 40k/Xenos Armies/T'au Empire/Epic Hero (GW's
  current placement) instead of the stale Drukhari keyword
This commit is contained in:
Ben_Kosytorz
2026-10-04 22:05:31 +02:00
parent 5fd5dacaf0
commit 4bbaefb45a
8 changed files with 4678 additions and 4428 deletions
Download Patch File Download Diff File Expand all files Collapse all files
data/gwshop_catalog.json
+4236 -4236
View File
File diff suppressed because it is too large Load Diff
dlphelper/catalogue.py
+11 -2
View File
@@ -25,7 +25,12 @@ import shutil
from dataclasses import dataclass, field from dataclasses import dataclass, field
from pathlib import Path from pathlib import Path
from dlphelper.warhammer_data import FACTION_ALIASES, SYSTEMS, UNIT_CLASSES from dlphelper.warhammer_data import (
FACTION_ALIASES,
GROUP_OF_FACTION,
SYSTEMS,
UNIT_CLASSES,
)
log = logging.getLogger(__name__) log = logging.getLogger(__name__)
@@ -252,7 +257,11 @@ def apply_moves(
if not src.exists(): if not src.exists():
log.warning("Cannot move, missing: %s", src) log.warning("Cannot move, missing: %s", src)
continue continue
dest_dir = library_root / e.system / e.faction / e.klass dest_dir = (
library_root / e.system
/ GROUP_OF_FACTION.get(e.faction, "Unclassified")
/ e.faction / e.klass
)
dest = dest_dir / src.name dest = dest_dir / src.name
if dest.exists(): if dest.exists():
log.warning("Skip, destination exists: %s", dest) log.warning("Skip, destination exists: %s", dest)
dlphelper/gwshop.py
+14 -5
View File
@@ -257,15 +257,24 @@ def build_catalog(hits: list[dict]) -> dict:
) )
continue continue
# primary branch = longest path (most specific), e.g. # primary branch = longest path (most specific), e.g.
# [40k, Armies of the Imperium, Adeptus Custodes, Rules & Accessories] # [40k, Space Marines, Raven Guard, Walker]
primary = max(paths, key=len) primary = max(paths, key=len)
system = primary[0] if len(primary) > 0 else None system = primary[0] if len(primary) > 0 else None
group = primary[1] if len(primary) > 1 else None group = primary[1] if len(primary) > 1 else None
faction = primary[2] if len(primary) > 2 else None faction = primary[2] if len(primary) > 2 else None
subfaction = primary[3] if len(primary) > 3 else None # The LAST level is the class/category layer (GW uses shop
# 5th level (if present) is treated as class; GW uses "Rules & # categories there: Battleline, Walker, Vehicle, "Rules &
# Accessories" etc. rather than force-org classes. # Accessories", product lines like "Cadian"). Deeper sources
klass = primary[4] if len(primary) > 4 else None # (miniset style) may carry a real subfaction before it.
if len(primary) >= 5:
subfaction = primary[3]
klass = primary[4]
elif len(primary) == 4:
subfaction = None
klass = primary[3]
else:
subfaction = None
klass = None
key = (hit.get("objectID"), tuple(primary)) key = (hit.get("objectID"), tuple(primary))
if key in seen: if key in seen:
continue continue
dlphelper/sorter.py
+70 -31
View File
@@ -22,6 +22,7 @@ from dataclasses import dataclass, field
from pathlib import Path from pathlib import Path
from dlphelper.catalogue import _clean_name, _match_class, _match_faction from dlphelper.catalogue import _clean_name, _match_class, _match_faction
from dlphelper.warhammer_data import GROUP_OF_FACTION
log = logging.getLogger(__name__) log = logging.getLogger(__name__)
@@ -154,26 +155,7 @@ def classify_item(item: SortItem) -> None:
item.group = GROUP_OF_FACTION.get(item.faction, UNSORTED) item.group = GROUP_OF_FACTION.get(item.faction, UNSORTED)
# faction group layer (miniset naming), extended as the taxonomy grows # faction group layer (GW naming, shared with warhammer_data)
GROUP_OF_FACTION: dict[str, str] = {
faction: group
for group, factions in {
"Armies of the Imperium": [
"Adeptus Astartes", "Astra Militarum", "Adeptus Mechanicus",
"Adepta Sororitas", "Adeptus Custodes", "Imperial Knights",
"Inquisition & Officio Assassinorum",
],
"Armies of Chaos": [
"Chaos Space Marines", "Death Guard", "Thousand Sons",
"Chaos Daemons", "Chaos Knights",
],
"Xenos Armies": [
"Orks", "T'au Empire", "Tyranids", "Genestealer Cults", "Necrons",
"Aeldari", "Drukhari", "Harlequins", "Ynnari",
],
}.items()
for faction in factions
}
def load_catalog(path: Path) -> list[dict]: def load_catalog(path: Path) -> list[dict]:
@@ -192,12 +174,8 @@ def _best_ratio(clean: str, candidates: list[str]) -> float:
) )
def match_catalog(item: SortItem, catalog: list[dict], threshold: float) -> None: def _best_in_pool(item: SortItem, pool: list[dict]) -> tuple[float, dict] | None:
"""Fuzzy-match against catalog entries in the same faction (or all).""" """Best fuzzy score of `item` against all entries in `pool`."""
if item.faction == UNSORTED:
return
same_faction = [e for e in catalog if e["faction"] == item.faction]
pool = same_faction or catalog # fall back to whole catalog if faction unknown
best: tuple[float, dict] | None = None best: tuple[float, dict] | None = None
for entry in pool: for entry in pool:
cands = [entry["_clean"], *entry["_clean_aliases"]] cands = [entry["_clean"], *entry["_clean_aliases"]]
@@ -208,22 +186,83 @@ def match_catalog(item: SortItem, catalog: list[dict], threshold: float) -> None
score = max(score, _best_ratio(c, cands) * 0.95) # inner slightly discounted score = max(score, _best_ratio(c, cands) * 0.95) # inner slightly discounted
if best is None or score > best[0]: if best is None or score > best[0]:
best = (score, entry) best = (score, entry)
return best
def match_catalog(item: SortItem, catalog: list[dict], threshold: float) -> None:
"""Fuzzy-match against catalog entries in the same faction (or all).
The matched entry is hierarchy-authoritative: group, subfaction and
class come from the catalog (GW shop categories), and when the keyword
classification had no faction the entry's system/faction is adopted too.
"""
if item.faction == UNSORTED and not item.inner_names and not item.clean:
return
if item.match_kind == "invalid":
return # HTML-page masquerading as an archive; never fuzzy-match it
same_faction = [e for e in catalog if e.get("faction") == item.faction]
pool = same_faction or catalog # fall back to whole catalog if faction unknown
best = _best_in_pool(item, pool)
if best is None or best[0] < threshold:
# The keyword tables can lag behind GW's current categorization
# (e.g. units moved between factions); if the faction pool has no
# confident match, try the whole catalog before giving up.
if same_faction:
best = _best_in_pool(item, catalog)
if best and best[0] >= threshold: if best and best[0] >= threshold:
entry = best[1] entry = best[1]
item.unit = entry["name"] item.unit = entry["name"]
item.subfaction = entry.get("subfaction") if entry.get("group"):
item.klass = item.klass or entry.get("class") item.group = entry["group"]
elif item.faction == UNSORTED:
item.group = GROUP_OF_FACTION.get(entry["faction"], UNSORTED)
if entry.get("subfaction"):
item.subfaction = entry["subfaction"]
# GW's class layer (shop category) beats the keyword guess, but do
# not *narrow* it: "Battleline" is GW's bucket for core infantry,
# a keyword-derived Hero/Elite is more specific for a proxy file.
catalog_klass = entry.get("class")
if catalog_klass and item.klass in (None, "", UNCLASSIFIED_CLASS):
item.klass = catalog_klass
item.match_score = best[0] item.match_score = best[0]
item.match_kind = "catalog" item.match_kind = "catalog"
# The catalog entry's faction is authoritative - but only adopt it
# when it is a real army faction. Generic GW branches like
# "Unit Type" or factionless entries would degrade a confident
# keyword faction to a junk folder.
entry_faction = entry.get("faction")
generic_factions = {"Unit Type", "Heroes", "Warriors", "Character"}
if item.faction == UNSORTED: if item.faction == UNSORTED:
if entry_faction and entry_faction not in generic_factions:
item.system = entry["system"]
item.faction = entry_faction
if entry.get("group") and entry["group"] not in ("Unit Type",):
item.group = entry["group"]
else:
item.group = GROUP_OF_FACTION.get(item.faction, UNSORTED)
elif entry_faction and entry_faction not in generic_factions and entry_faction != item.faction:
# GW moved this unit to another faction (or the keyword tables
# are stale): trust the catalog.
item.system = entry["system"] item.system = entry["system"]
item.faction = entry["faction"] item.faction = entry_faction
item.group = entry.get("group") or GROUP_OF_FACTION.get(entry["faction"], UNSORTED) if entry.get("group") and entry["group"] != "Unit Type":
item.group = entry["group"]
else:
item.group = GROUP_OF_FACTION.get(entry_faction, UNSORTED)
elif entry.get("group") and entry["group"] not in ("Unit Type",):
item.group = entry["group"]
else: else:
if item.faction != UNSORTED: if item.faction == UNSORTED:
item.unit = item.name # implicit entry: own (original) name item.unit = item.name # implicit entry: own (original) name
item.match_score = best[0] if best else 0.0 item.match_score = best[0] if best else 0.0
item.match_kind = "implicit" item.match_kind = "implicit"
else:
# Faction was known but nothing fuzzy-matched: keep the keyword
# faction and use the file's own name as the unit, but adopt the
# catalog's class layer when it adds information.
item.unit = item.name
item.match_score = best[0] if best else 0.0
item.match_kind = "implicit"
def build_target(item: SortItem, library: Path) -> Path: def build_target(item: SortItem, library: Path) -> Path:
dlphelper/warhammer_data.py
+302 -113
View File
@@ -1,12 +1,19 @@
"""Warhammer taxonomy and unit keyword tables used by the catalogue. """Warhammer taxonomy and unit keyword tables used by the catalogue.
Naming follows the Games Workshop webshop hierarchy (the same one
dlphelper.gwshop dumps from Algolia):
<System>/<Group>/<Faction>/<Class> e.g.
Warhammer 40,000/Space Marines/Ultramarines/Walker
Two layers: Two layers:
- SYSTEMS: game system -> faction -> keyword aliases (matched against file - SYSTEMS: game system -> faction -> keyword aliases (matched against file
names). Faction detection is the primary signal; unit names double as names). Faction detection is the primary signal; unit names double as
faction hints since they are listed under their faction. faction hints since they are listed under their faction. Space Marine
- CLASSES: keyword -> force-organization class (40k Force Organization slots: chapters are their own GW factions under the "Space Marines" group.
HQ, Troops, Elites, Fast Attack, Heavy Support, Flyer, Dedicated Transport, - CLASSES: keyword -> class. With GW as the catalog source the class layer
Lords of War, Fortification). holds GW shop categories (unit types: Battleline, Infantry, Walker,
Vehicle, Flyer, Monster, Mounted, Hero, Titanic).
The unit table grows with the collection; it does not attempt to enumerate The unit table grows with the collection; it does not attempt to enumerate
every unit Games Workshop ever released - unknown files land in the every unit Games Workshop ever released - unknown files land in the
@@ -21,20 +28,65 @@ from __future__ import annotations
SYSTEMS: dict[str, dict[str, list[str]]] = { SYSTEMS: dict[str, dict[str, list[str]]] = {
"Warhammer 40,000": { "Warhammer 40,000": {
# -- Imperium ------------------------------------------------------- # -- Space Marines (chapters are their own GW factions) --------------
"Adeptus Astartes": [ "Space Marines": [
"space marine", "astartes", "primaris", "ultramarines", "blood angels", "space marine", "astartes", "primaris", "intercessor", "reiver",
"dark angels", "space wolves", "black templars", "imperial fists", "aggressor", "inceptor", "hellblaster", "eradicator", "bladeguard",
"salamanders", "raven guard", "iron hands", "white scars", "grey knights", "sternguard", "devastator", "tactical squad", "terminator squad",
"deathwatch", "crimson fists", "flesh tearers", "ravenwing", "deathwing", "assault marine", "assault intercessor", "outrider", "scout squad",
"inner circle", "wulfen", "thunderwolf", "intercessor", "reiver", "aggressor", "inceptor", "vanguard veteran", "invictor", "impulsor", "gladiator",
"hellblaster", "eradicator", "bladeguard", "sternguard", "devastator", "land speeder", "drop pod", "inner circle", "honour guard",
"tactical squad", "terminator squad", "assault marine",
"redemptor", "contemptor", "dreadnought", "invictor", "repulsor",
"land raider", "land speeder", "predator", "vindicator", "whirlwind",
"drop pod", "razorback", "stormraven", "storm talon", "stormhawk",
"caestus", "chaplain", "librarian", "apothecary", "vexilus",
], ],
"Ultramarines": ["ultramarines", "guilliman", "calgar"],
"Blood Angels": [
"blood angel", "sanguinary guard", "sanguinor", "death company",
"baal predator", "mephiston", "dante", "astorath", "flesh tearers",
"lamberto", "furioso",
],
"Dark Angels": [
"dark angel", "ravenwing", "deathwing", "inner circle companions",
"azrael", "asmodai", "belial", "samuel", "lion el'jonson",
"lion el jonson", "nephilim jetfighter",
],
"Space Wolves": [
"space wolf", "space wolves", "wulfen", "thunderwolf", "wolf guard",
"rune priest", "wolf priest", "wolf lord", "logan grimnar",
"ragnar blackmane", "canis wolfborn", "fenrisian wolf",
"stormfang", "murderfang", "lukas the trickster",
],
"Black Templars": [
"black templar", "crusader squad", "high marshal helbrecht",
"grimaldus", "the emperor's champion", "emperors champion",
"castellan", "land raider crusader",
],
"Imperial Fists": [
"imperial fist", "rogal dorn", "lupos", "tor garadon",
"horatius hold", "dorn's arrow",
],
"Raven Guard": [
"raven guard", "kayvaan shrike", "corvus", "sable brand",
"shadow captains",
],
"Iron Hands": [
"iron hand", "iron hands", "ferrus manus", "iron father",
"medusan immortals",
],
"White Scars": ["white scar", "jaghatai khan", "stormseer", "korsos"],
"Salamanders": [
"salamander", "vulkan", "he'stan", "adrax agatone",
"drake-scale", "flamecaster",
],
"Grey Knights": [
"grey knight", "grand master", "brother-captain", "nemesis dreadknight",
"draigo", "castellan crowe", "sternguard knight",
],
"Deathwatch": [
"deathwatch", "watch captain", "watch master", "kill team cassius",
"overkill", "corvus blackstar",
],
"Crimson Fists": ["crimson fist", "kantor", "rynn's world"],
"Flesh Tearers": ["flesh tearer", "gabriel seth", "amissum"],
# -- Imperium --------------------------------------------------------
"Astra Militarum": [ "Astra Militarum": [
"astra militarum", "imperial guard", "cadian", "catachan", "krieg", "astra militarum", "imperial guard", "cadian", "catachan", "krieg",
"death korps", "tempestus", "scion", "valhallan", "steel legion", "death korps", "tempestus", "scion", "valhallan", "steel legion",
@@ -72,23 +124,22 @@ SYSTEMS: dict[str, dict[str, list[str]]] = {
"armiger", "warglaive", "helverin", "morialtis", "canis rex", "armiger", "warglaive", "helverin", "morialtis", "canis rex",
"cerastus", "atropos", "seneschal", "cerastus", "atropos", "seneschal",
], ],
"Inquisition & Officio Assassinorum": [ "Imperial Agents": [
"inquisition", "inquisitor", "astra telepathica", "witch hunter", "inquisition", "inquisitor", "astra telepathica", "witch hunter",
"coteaz", "eisenhorn", "ravenor", "kyria draxus", "coteaz", "eisenhorn", "ravenor", "kyria draxus",
"officio assassinorum", "culexus", "vindicare", "callidus", "eversor", "officio assassinorum", "culexus", "vindicare", "callidus", "eversor",
"saint celestine", "living saint", "assassin", "saint celestine", "living saint", "assassin",
], ],
# -- Chaos ---------------------------------------------------------- # -- Chaos -----------------------------------------------------------
"Chaos Space Marines": [ "Chaos Space Marines": [
"chaos space marine", "black legion", "iron warriors", "word bearers", "chaos space marine", "black legion", "iron warriors", "word bearers",
"night lords", "alpha legion", "red corsairs", "world eaters", "night lords", "alpha legion", "chaos terminator", "chosen", "havoc",
"emperor's children", "chaos terminator", "chosen", "havoc",
"raptor", "warp talon", "chaos cultist", "cultist", "daemon prince", "raptor", "warp talon", "chaos cultist", "cultist", "daemon prince",
"chaos lord", "chaos sorcerer", "abaddon", "huron blackheart", "chaos lord", "chaos sorcerer", "abaddon", "kharn", "ahriman",
"kharn", "lucius the eternal", "ahriman", "typhus", "fabius bile", "typhus", "fabius bile", "vashtorr", "defiler", "forgefiend",
"vashtorr", "defiler", "forgefiend", "maulerfiend", "helldrake", "maulerfiend", "helldrake", "helbrute", "chaos spawn",
"helbrute", "chaos spawn", "obliterator", "chaos rhino", "obliterator", "chaos rhino", "chaos vindicator", "spartan assault",
"chaos vindicator", "spartan assault", "mastodon", "typhon", "mastodon", "typhon",
], ],
"Death Guard": [ "Death Guard": [
"death guard", "nurgle", "plague marine", "deathshroud", "mortarion", "death guard", "nurgle", "plague marine", "deathshroud", "mortarion",
@@ -113,11 +164,21 @@ SYSTEMS: dict[str, dict[str, list[str]]] = {
"skulltaker", "khorne", "slaanesh", "skull lord", "skulltaker", "khorne", "slaanesh", "skull lord",
"shalaxi helbane", "shalaxi helbane",
], ],
"World Eaters": [
"world eater", "angron", "eightbound", "exalted eightbound",
"jakhals", "berserker", "skull lord",
],
"Emperor's Children": [
"emperor's children", "emperors children", "fulgrim",
"noise marine", "lucius the eternal", "flawless blades",
],
"Iron Warriors": ["iron warrior", "perturabo"],
"Red Corsairs": ["red corsair", "huron blackheart"],
"Chaos Knights": [ "Chaos Knights": [
"chaos knight", "knight desecrator", "knight despoiler", "chaos knight", "knight desecrator", "knight despoiler",
"knight rampager", "kharibdyss", "renegade knight", "knight rampager", "kharibdyss", "renegade knight",
], ],
# -- Xenos ---------------------------------------------------------- # -- Xenos -----------------------------------------------------------
"Orks": [ "Orks": [
"ork", "orks", "orkz", "nob", "boyz", "grot", "weirdboy", "painboy", "ork", "orks", "orkz", "nob", "boyz", "grot", "weirdboy", "painboy",
"warboss", "ghazghkull", "meganob", "deff dread", "killa kan", "warboss", "ghazghkull", "meganob", "deff dread", "killa kan",
@@ -162,7 +223,7 @@ SYSTEMS: dict[str, dict[str, list[str]]] = {
], ],
"Aeldari": [ "Aeldari": [
"aeldari", "eldar", "craftworld", "ulthwe", "biel-tan", "iyanden", "aeldari", "eldar", "craftworld", "ulthwe", "biel-tan", "iyanden",
"alaitoc", "saim-hann", "ynnari", "farseer", "warlock", "alaitoc", "saim-hann", "farseer", "warlock",
"spiritseer", "autarch", "asurmen", "guardian defender", "spiritseer", "autarch", "asurmen", "guardian defender",
"ranger", "wraithguard", "wraithblades", "wraithlord", "ranger", "wraithguard", "wraithblades", "wraithlord",
"wraithknight", "dire avenger", "howling banshee", "wraithknight", "dire avenger", "howling banshee",
@@ -178,62 +239,124 @@ SYSTEMS: dict[str, dict[str, list[str]]] = {
"kabalite warrior", "raider", "ravager", "venom", "razorwing", "kabalite warrior", "raider", "ravager", "venom", "razorwing",
"voidraven", "grotesque", "talos", "cronos", "voidraven", "grotesque", "talos", "cronos",
"beast pack", "khymera", "razorwing flock", "asdrubael vect", "beast pack", "khymera", "razorwing flock", "asdrubael vect",
"lilith", "sathonyx", "the twin lance", "luriel", "lilith", "sathonyx", "luriel",
], ],
"Harlequins": [ "Harlequins": [
"harlequin", "death jester", "shadowseer", "solitaire", "harlequin", "death jester", "shadowseer", "solitaire",
"troupe", "starweaver", "voidweaver", "troupe", "starweaver", "voidweaver",
], ],
"Ynnari": [ "Leagues of Votann": [
"ynnari", "ynnead", "visarch", "yncarne", "yvraine", "leagues of votann", "votann", "hearthkyn", "ironkin",
"brokhyr", "kahl", "cthonian", "transhyperic", "thegn",
], ],
}, },
"Other Systems": { "Age of Sigmar": {
"Age of Sigmar": [ "Stormcast Eternals": [
"age of sigmar", "stormcast", "eternals", "orruk", "warclan", "stormcast", "eternal", "liberator", "sequitor", "evocator",
"kruleboyz", "gloomspite", "sylvaneth", "seraphon", "ossiarch", "vindictor", "judicators", "retributors", "sacrosanct",
"nighthaunt", "lumineth", "daughters of khaine", "fyreslayers", "dracothian", "lord-arcane", "knight-questor", "neave blacktalon",
"kharadron", "idoneth", "blades of khorne", "maggotkin", "stormdrake", "krondys", "karazai",
"gaunt summoner", "soulblight", "freeguild", "cities of sigmar",
"skaven", "chaos warriors", "saurus", "skink", "liberator",
"sequitor", "evocator", "vindictor", "kaldor draigo",
], ],
"Horus Heresy": [ "Seraphon": ["seraphon", "saurus", "skink", "slann", "carnosaur",
"horus heresy", "legiones astartes", "pre-heresy", "bastiladon", "stegadon", "razordon", "salamander"],
"leviathan pattern", "fire raptor", "sicaran", "fellblade", "Orruk Warclans": ["orruk", "warclan", "kruleboyz", "ironjawz",
"kratos", "justaerin", "eisenstein", "veteran tactical", "brute", "gorger", "weirdnob", "warchanter"],
"seeker squad", "suzerain", "invictarus", "fulgurite", "Gloomspite Gitz": ["gloomspite", "gitz", "moonclan", "spiderfang",
"troggoth", "loontouched", "zarbag", "skragrot"],
"Sylvaneth": ["sylvaneth", "treelord", "dryad", "branchwraith",
"alarielle", "kurnothi"],
"Ossiarch Bonereapers": ["ossiarch", "bonereaper", "mortek", "keldrek",
"katakros"],
"Nighthaunt": ["nighthaunt", "grimghast", "bladegheist", "chainghast",
"kurdoss", "lady olynder"],
"Lumineth Realm-lords": ["lumineth", "aelf", "vanari", "teclis",
"light of eltharion"],
"Daughters of Khaine": ["daughters of khaine", "khainite", "witch aelf",
"morgwaeth", "morathi"],
"Fyreslayers": ["fyreslayer", "vulkite", "hearthguard", "auric",
"grimhold"],
"Kharadron Overlords": ["kharadron", "skyfarer", "endofters",
"grundstok", "arcanaut"],
"Idoneth Deepkin": ["idoneth", "deepkin", "namarti", "isharann",
"volturnos"],
"Blades of Khorne": ["blades of khorne", "khorne", "skullgrinder",
"blood warrior", "wrathmonger"],
"Maggotkin of Nurgle": ["maggotkin", "nurgle", "blightking",
"rotmire", "orghotts"],
"Slaves to Darkness": ["slaves to darkness", "chaos warrior",
"chaos knight", "archaon", "gaunt summoner",
"everchosen"],
"Skaven": ["skaven", "clanrat", "stormvermin", "verminlord",
"grey seer", "warplock", "ratling gun"],
"Soulblight Gravelords": ["soulblight", "gravelord", "vampire",
"deathrattle", "koldhunter", "belladamma"],
"Cities of Sigmar": ["cities of sigmar", "freeguild", "windspeaker",
"callis and toll", "hammerers"],
},
"The Horus Heresy": {
"Space Marine Legions": [
"legiones astartes", "pre-heresy", "horus heresy", "justaerin",
"seeker squad", "suzerain", "invictarus", "veteran tactical",
"contemptor", "leviathan pattern", "sicaran", "fellblade",
"kratos", "fire raptor", "eisenstein",
], ],
"Mechanicum": ["mechanicum", "fulgurite", "magos", "thallax"],
},
"Other Games": {
"Kill Team": [ "Kill Team": [
"kill team", "killteam", "killzone", "operative", "kill team", "killteam", "killzone", "operative",
"hand of the archon", "void-dance troupe", "veteran guardsmen", "hand of the archon", "void-dance troupe", "veteran guardsmen",
"spector squad", "wolf scouts", "scout squad kill",
], ],
"Necromunda": [ "Necromunda": [
"necromunda", "goliath gang", "escher", "orlock", "cawdor", "necromunda", "goliath gang", "escher", "orlock", "cawdor",
"van saar", "ash wastes", "underhive", "van saar", "ash wastes", "underhive", "sumpkrocs",
"forge boss", "sumpkroc", "matriarch", "phyrr cat",
], ],
"Legions Imperialis / Epic": [ "Adeptus Titanicus": [
"legions imperialis", "titan legio", "reaver titan", "adepticus titanicus", "titan legio", "reaver titan",
"warhound titan", "warlord titan", "imperator titan", "warhound titan", "warlord titan", "imperator titan",
"warmaster titan", "warmaster titan", "titanicus",
],
"Warhammer The Old World": [
"the old world", "bretonnia", "greenskins", "tomb kings",
"khemri", "chaos dwarf", "dog of war", "grail knight",
], ],
"Warhammer Underworlds": ["warhammer underworlds", "underworlds",
"warband", "shadeborn", "embergard"],
"Aeronautica Imperialis": ["aeronautica", "t'au air caste",
"lightning fighter", "thunderbolt"],
},
"The Old World": {
"Kingdom of Bretonnia": ["bretonnia", "grail knight", "peasant",
"questing knight", "perrin"],
"Orc and Goblin Tribes": ["greenskin", "goblin", "night goblin",
"forest goblin", "black orc"],
"Tomb Kings of Khemri": ["tomb king", "khemri", "necrolith",
"skeleton archer"],
"Empire of Man": ["empire of man", "state troop", "greatsword",
"empire knight"],
"Warriors of Chaos": ["chaos dwarf", "dog of war", "chaos marauder"],
"High Elf Realms": ["high elf", "eltharion", "phoenix guard"],
"Wood Elf Realms": ["wood elf", "glade guard", "waywatcher"],
"Beastmen Brayherds": ["beastmen", "brayherd", "ghorgon", "cygor"],
"Dwarfen Mountain Holds": ["dwarf", "ironbreaker", "hammerer"],
},
"Middle-Earth": {
"Rivendell": ["rivendell", "elrond", "elendil", "isildur", "gil-galad",
"beorning", "grimbeorn", "rumil", "orophin"],
"Minas Tirith": ["minas tirith", "gondor", "boromir", "faramir"],
"Mordor": ["mordor", "orc", "orcs", "nazgul", "witch-king"],
"Iron Hills": ["iron hills", "dwarf", "mattock", "dwalin"],
"Moria": ["moria", "cave troll", "balrog"],
}, },
} }
# --------------------------------------------------------------------------- # ---------------------------------------------------------------------------
# Force Organization classes (keyword -> class) # Classes (keyword -> class). GW shop categories / unit types.
# --------------------------------------------------------------------------- # ---------------------------------------------------------------------------
CLASSES: list[tuple[str, list[str]]] = [ CLASSES: list[tuple[str, list[str]]] = [
("Lords of War", [ ("Titanic", [
"titan", "stompa", "gargantuan", "hierophant", "harridan", "hierodule", "titan", "stompa", "gargantuan", "hierophant", "harridan", "hierodule",
"baneblade", "stormlord", "shadowsword", "hellhammer", "malcador",
"imperator", "warmaster", "tesseract vault", "plaguereaper", "imperator", "warmaster", "tesseract vault", "plaguereaper",
"warhound", "warlord titan", "warhound", "warlord titan", "ta'unar", "spermato", "titandeath",
]), ]),
("Fortification", ["fortification", "bunker", "bastion", "wall of martyrs"]), ("Fortification", ["fortification", "bunker", "bastion", "wall of martyrs"]),
("Flyer", [ ("Flyer", [
@@ -243,64 +366,76 @@ CLASSES: list[tuple[str, list[str]]] = [
"marauder", "razorwing", "voidraven", "crimson hunter", "marauder", "razorwing", "voidraven", "crimson hunter",
"hemlock", "doom scythe", "night scythe", "harpy", "crone", "hemlock", "doom scythe", "night scythe", "harpy", "crone",
"fire raptor", "barracuda", "sun shark", "razorshark", "fire raptor", "barracuda", "sun shark", "razorshark",
"wazbom blasta", "wazbom blasta", "nephilim jetfighter", "corvus blackstar",
]), ]),
("Dedicated Transport", [ ("Vehicle", [
"transport", "rhino", "razorback", "chimera", "taurox", "trukk", "transport", "rhino", "razorback", "chimera", "taurox", "trukk",
"battlewagon", "devilfish", "wave serpent", "ghost ark", "battlewagon", "devilfish", "wave serpent", "ghost ark",
"land raider", "repulsor", "drop pod", "starweaver", "voidweaver", "land raider", "repulsor", "drop pod", "starweaver", "voidweaver",
"goliath truck", "rockgrinder", "cult limo", "raider", "venom", "goliath truck", "rockgrinder", "cult limo", "raider", "venom",
"spartan assault", "spartan assault", "predator", "vindicator", "whirlwind",
"basilisk", "hydra", "wyvern", "manticore", "deathstrike",
"onager dunecrawler", "kastelan", "hammerhead", "broadside",
"stormsurge", "r'varna", "soul grinder", "defiler", "forgefiend",
"maulerfiend", "plagueburst crawler", "annihilation barge",
"doomsday ark", "fire prism", "night spinner", "exorcist",
"immolator", "caladius", "coronus grav", "pallas grav",
"impulsor", "gladiator", "baal predator", "land speeder",
"ironstrider", "sydonian", "sentinel", "taurox", "kataphron",
]), ]),
("Fast Attack", [ ("Walker", [
"fast attack", "assault marine", "assault intercessor", "inceptor", "dreadnought", "redemptor", "contemptor", "leviathan pattern",
"outrider", "bike squad", "scout biker", "thunderwolf", "raptor", "brutalis", "ballistus", "deredeo", "wulfen dreadnought",
"warp talon", "flesh hound", "screamer", "fiend of slaanesh", "killa kan", "deff dread", "wraithlord", "wraithknight",
"hellion", "reaver jetbike", "scourge", "mandrake", "swooping hawk", "telemon", "achillus", "galatus", "paragon warsuit",
"shining spears", "windrider", "vyper", "war walker", "nemesis dreadknight", "invictor", "talos", "cronos",
"canoptek wraith", "canoptek scarab", "canoptek spyder",
"ravener", "gargoyle", "spore mine", "mawloc", "trygon",
"pyrovore", "khymera", "beast pack", "razorwing flock",
"skyrunner", "zephyrim", "seraphim", "sentinel", "jackal",
"stormboy", "deffkopta", "warbiker", "skorpekh destroyer",
"sicarian", "ironstrider", "sydonian",
]), ]),
("Heavy Support", [ ("Monster", [
"heavy support", "devastator", "eradicator", "hellblaster", "havoc", "monster", "carnifex", "exocrine", "haruspex", "mawloc", "trygon",
"loota", "burna boy", "carnifex", "exocrine", "haruspex", "tervigon", "hierodule", "hierophant", "mutalith", "bloat-drone",
"zoanthrope", "venomthrope", "biovore", "hierodule", "kharibdyss", "gorkanaut", "morkanaut", "magnus", "mortalith",
"wraithlord", "wraithknight", "fire prism", "night spinner", "fiend of slaanesh", "plague drone", "cave troll", "balrog",
"predator", "vindicator", "whirlwind", "basilisk", "hydra", "carnosaur", "stegadon", "bastiladon", "grorgeous",
"wyvern", "manticore", "deathstrike", "onager dunecrawler",
"kastelan", "kataphron", "hammerhead", "broadside", "stormsurge",
"r'varna", "soul grinder", "defiler", "forgefiend", "maulerfiend",
"plagueburst crawler", "bloat-drone", "blight hauler", "mutalith",
"annihilation barge", "doomsday ark", "obront", "dark reaper",
"retributor", "paragon warsuit", "morvenn", "exorcist", "immolator",
"caladius", "coronus grav", "telemon", "achillus", "galatus",
"pallas grav", "skorpekh",
]), ]),
("Elites", [ ("Mounted", [
"elites", "veteran", "terminator", "sternguard", "bladeguard", "mounted", "thunderwolf", "outrider", "bike squad", "scout biker",
"aggressor", "reiver", "vanguard veteran", "dreadnought", "reaver jetbike", "windrider", "skyrunner", "warbiker",
"redemptor", "contemptor", "leviathan pattern", "wulfen", "allarus custodian on", "vertus praetor", "drakeling",
"wolf guard", "deathshroud", "plague marine", "blightlord", "boarboy", "gruntaz", "khan on", "dracothian guard",
"poxwalker", "rubric marine", "rubricae", "scarab occult", ]),
"tzaangor", "chosen", "possessed", "obliterator", "daemonette", ("Battleline", [
"bloodletter", "plaguebearer", "nurgling", "juggernaut", "battleline", "troops", "tactical squad", "intercessor",
"chaos cultist", "cultist", "acolyte hybrid", "aberrant", "incursor", "infiltrator", "infantry squad", "conscript",
"krieg", "catachan", "cadian", "vostroyan", "penal legion",
"ratling", "veteran squad", "fire warrior", "pathfinder team",
"kroot", "termagant", "hormagaunt", "necron warrior",
"necron immortal", "guardian defender", "storm guardian",
"kabalite warrior", "boyz", "gretchin", "scion", "scout squad",
"cultist", "plague marine squad",
]),
("Infantry", [
"infantry", "veteran", "terminator", "sternguard", "bladeguard",
"aggressor", "reiver", "wulfen", "wolf guard", "deathshroud",
"blightlord", "poxwalker", "rubric marine", "rubricae",
"scarab occult", "tzaangor", "chosen", "possessed", "obliterator",
"daemonette", "bloodletter", "plaguebearer", "nurgling",
"juggernaut", "chaos cultist", "acolyte hybrid", "aberrant",
"neophyte", "purestrain", "genestealer", "lictor", "deathmark", "neophyte", "purestrain", "genestealer", "lictor", "deathmark",
"flayed one", "lychguard", "wraithguard", "wraithblades", "flayed one", "lychguard", "wraithguard", "wraithblades",
"howling banshee", "striking scorpion", "warp spider", "howling banshee", "striking scorpion", "warp spider",
"fire dragon", "incubi", "wrack", "grotesque", "talos", "cronos", "fire dragon", "incubi", "wrack", "grotesque", "assault marine",
"troupe", "killa kan", "deff dread", "meganob", "nobz", "nob", "assault intercessor", "inceptor", "raptor", "warp talon",
"kommando", "vespid", "sanguinary guard", "sanguinor", "flesh hound", "screamer", "hellion", "scourge", "mandrake",
"ancient", "allarus", "vertus praetor", "venatari", "aquilon", "swooping hawk", "shining spears", "canoptek wraith",
"sagittarum", "witchseeker", "prosecutor", "vigilator", "canoptek scarab", "canoptek spyder", "ravener", "gargoyle",
"vindicare", "callidus", "eversor", "culexus", "assassin", "spore mine", "pyrovore", "khymera", "beast pack",
"razorwing flock", "zephyrim", "seraphim", "stormboy",
"deffkopta", "skorpekh destroyer", "skorpekh", "retributor",
"dark reaper", "sicarian", "ruststalker", "skitarii",
"hormagaunt brood", "termagant brood", "vanguard",
]), ]),
("HQ", [ ("Hero", [
"hq", "captain", "chapter master", "primarch", "lieutenant", "hero", "hq", "captain", "chapter master", "primarch", "lieutenant",
"librarian", "chaplain", "apothecary", "farseer", "warlock", "librarian", "chaplain", "apothecary", "farseer", "warlock",
"spiritseer", "autarch", "warboss", "big mek", "weirdboy", "spiritseer", "autarch", "warboss", "big mek", "weirdboy",
"painboy", "commander", "ethereal", "hive tyrant", "swarmlord", "painboy", "commander", "ethereal", "hive tyrant", "swarmlord",
@@ -316,18 +451,12 @@ CLASSES: list[tuple[str, list[str]]] = [
"dialogus", "hospitaller", "trajann", "valdor", "inquisitor", "dialogus", "hospitaller", "trajann", "valdor", "inquisitor",
"coteaz", "eisenhorn", "ravenor", "draxus", "celestine", "coteaz", "eisenhorn", "ravenor", "draxus", "celestine",
"commissar", "priest", "enginseer", "dominus", "manipulus", "commissar", "priest", "enginseer", "dominus", "manipulus",
"archmagos", "magos", "belisarius cawl", "farsight", "shadowsun", "archmagos", "belisarius cawl", "farsight", "shadowsun",
"aun'va", "darkstrider", "longstrike", "shade runner", "aun'va", "darkstrider", "longstrike", "shade runner",
]), ]),
("Troops", [ ("Rules & Accessories", [
"troops", "battleline", "tactical squad", "intercessor", "codex", "rulebook", "rules", "data cards", "datacards",
"incursor", "infiltrator", "infantry squad", "conscript", "warscroll", "index card", "cheat sheet", "reference",
"krieg", "catachan", "cadian", "vostroyan", "penal legion",
"ratling", "veteran squad", "fire warrior", "pathfinder team",
"kroot", "termagant", "hormagaunt", "necron warrior",
"necron immortal", "guardian defender", "storm guardian",
"kabalite warrior", "boyz", "gretchin",
"scion", "scout squad",
]), ]),
] ]
@@ -338,3 +467,63 @@ FACTION_ALIASES: dict[str, dict[str, str]] = {
} }
UNIT_CLASSES: dict[str, str] = {kw: klass for klass, kws in CLASSES for kw in kws} UNIT_CLASSES: dict[str, str] = {kw: klass for klass, kws in CLASSES for kw in kws}
# GW group of each faction (matches the GameSystemsRoot.lvl1 values in the
# Algolia index). Used when a keyword-classified file cannot be fuzzy-matched.
GROUP_OF_FACTION: dict[str, str] = {
faction: group
for group, factions in {
"Space Marines": [
"Space Marines", "Ultramarines", "Blood Angels", "Dark Angels",
"Space Wolves", "Black Templars", "Imperial Fists", "Raven Guard",
"Iron Hands", "White Scars", "Salamanders", "Grey Knights",
"Deathwatch", "Crimson Fists", "Flesh Tearers",
],
"Armies of the Imperium": [
"Astra Militarum", "Adeptus Mechanicus", "Adepta Sororitas",
"Adeptus Custodes", "Imperial Knights", "Imperial Agents",
],
"Armies of Chaos": [
"Chaos Space Marines", "Death Guard", "Thousand Sons",
"Chaos Daemons", "Chaos Knights", "World Eaters",
"Emperor's Children", "Iron Warriors", "Red Corsairs",
],
"Xenos Armies": [
"Orks", "T'au Empire", "Tyranids", "Genestealer Cults", "Necrons",
"Aeldari", "Drukhari", "Harlequins", "Leagues of Votann",
],
"Grand Alliance Order": [
"Stormcast Eternals", "Seraphon", "Sylvaneth",
"Lumineth Realm-lords", "Daughters of Khaine", "Fyreslayers",
"Kharadron Overlords", "Idoneth Deepkin", "Cities of Sigmar",
],
"Grand Alliance Destruction": [
"Orruk Warclans", "Gloomspite Gitz",
],
"Grand Alliance Death": [
"Ossiarch Bonereapers", "Nighthaunt", "Soulblight Gravelords",
],
"Grand Alliance Chaos": [
"Slaves to Darkness", "Blades of Khorne", "Maggotkin of Nurgle",
"Skaven",
],
"Space Marine Legions": ["Space Marine Legions", "Mechanicum"],
"Kill Team": ["Kill Team"],
"Necromunda": ["Necromunda"],
"Adeptus Titanicus": ["Adeptus Titanicus"],
"Warhammer Underworlds": ["Warhammer Underworlds"],
"Aeronautica Imperialis": ["Aeronautica Imperialis"],
"Armies of the Old World": [
"Kingdom of Bretonnia", "Orc and Goblin Tribes",
"Tomb Kings of Khemri", "Empire of Man", "Warriors of Chaos",
"High Elf Realms", "Wood Elf Realms", "Beastmen Brayherds",
"Dwarfen Mountain Holds",
],
"Rivendell": ["Rivendell"],
"Minas Tirith": ["Minas Tirith"],
"Mordor": ["Mordor"],
"Iron Hills": ["Iron Hills"],
"Moria": ["Moria"],
}.items()
for faction in factions
}
docs/SORTING.md
+32 -29
View File
@@ -4,47 +4,50 @@ This document defines the target structure of the collection ("the library")
and how the sorter places files into it. It is the working agreement for all and how the sorter places files into it. It is the working agreement for all
sorting-related code; change it here first, then in code. sorting-related code; change it here first, then in code.
## Target hierarchy (miniset.net-aligned) ## Target hierarchy (Games Workshop webshop-aligned)
The hierarchy mirrors miniset.net ("Miniatures Collectors Guide"), extended The hierarchy follows the official GW webshop taxonomy (dumped from the
with the 40k force-organization class, which miniset itself uses as a shop's Algolia index by `dlphelper/gwshop.py` into
"specification" tag on set pages: `data/gwshop_catalog.json`). Chapters and sub-lines that the old miniset
hierarchy treated as sub-factions are full GW factions, and the class layer
holds GW shop categories (unit types / product lines) instead of the retired
40k force-organization slots:
``` ```
<library>/ <library>/
Warhammer 40,000/ # game Warhammer 40,000/ # game (GW system)
Armies of the Imperium/ # faction group (miniset naming) Space Marines/ # GW group
Adeptus Astartes/ # faction Raven Guard/ # GW faction (chapter)
Space Wolves/ # sub-faction (optional) Walker/ # GW class (unit type / category)
Fast Attack/ # force-org class Redemptor Dreadnought/ # canonical unit name (catalog)
Thunderwolf Cavalry/ # canonical unit name Redemptor Dreadnought.zip # the file(s), or unpacked contents
Thunderwolf Cavalry.7z # the file(s), or unpacked contents
``` ```
Layer rules: Layer rules:
| Layer | Source of truth | Example | May be omitted? | | Layer | Source of truth | Example | May be omitted? |
|-------|-----------------|---------|-----------------| |-------|-----------------|---------|-----------------|
| Game | `warhammer_data.SYSTEMS` | `Warhammer 40,000` | no (else: `_unsorted/`) | | Game | `gwshop_catalog.json` (system) | `Warhammer 40,000` | no (else: `_unsorted/`) |
| Faction group | Imperium/Chaos/Xenos/etc. | `Armies of Chaos` | no (derived from faction) | | Group | GW `GameSystemsRoot` lvl1 | `Space Marines` | no (from catalog or `warhammer_data.GROUP_OF_FACTION`) |
| Faction | miniset.net faction names | `Death Guard` | no (else `_unsorted/` in game) | | Faction | GW factions (incl. chapters) | `Raven Guard` | no (else `_unsorted/` in game) |
| Sub-faction | miniset sub-factions | `Space Wolves` | **yes** — only when known | | Sub-faction | deeper sources only | — | **yes** — GW has none |
| Class | force-org slots | `Fast Attack` | **yes** — only when known | | Class | GW shop category | `Walker`, `Battleline`, `Rules & Accessories` | **yes** — only when known |
| Unit | canonical model name (lexicanum spelling) | `Thunderwolf Cavalry` | yes (files drop directly into the class dir) | | Unit | canonical product name (GW catalog) | `Redemptor Dreadnought` | yes (files drop directly into the class dir) |
- Files without a faction but with a game land in `<game>/_unsorted/`. - Files without a faction but with a game land in `<game>/_unsorted/`.
- Fully unclassifiable files land in `<library>/_unsorted/` — never guessed. - Fully unclassifiable files land in `<library>/_unsorted/` — never guessed.
- Everything in between is optional and filled in as knowledge grows. - The keyword tables in `warhammer_data.py` (GW naming) provide the initial
- Sub-faction folders are only created when the file name states it system/faction/class signal; a catalog match then overrides the hierarchy
(`Thunderwolf` → Space Wolves); otherwise files stay in the faction dir so from the catalog entry, which is authoritative.
we don't invent hierarchy. - `data/library_catalog.json` (the older miniset-aligned hand catalog) is
still usable via `sorter.py --catalog`; its sub-faction entries map into
the same schema. For GW-sourced entries `subfaction` stays empty.
## Canonical names ## Canonical names
Canonical unit names follow Lexicanum/GW spelling (`Keeper of Secrets`, Canonical unit names follow the GW webshop spelling (`Keeper of Secrets`,
`Death Korps of Krieg`), because that is what cross-seeding, wikis and `Thunderwolf Cavalry`), because that is what the catalog, the shop and
miniset.net all agree on. Release names stay untouched inside the unit wikis all agree on. Release names stay untouched inside the unit folder.
folder.
## Sorting pipeline ## Sorting pipeline
@@ -97,10 +100,10 @@ graph LR
```json ```json
{ {
"system": "Warhammer 40,000", "system": "Warhammer 40,000",
"group": "Armies of the Imperium", "group": "Space Marines",
"faction": "Adeptus Astartes", "faction": "Space Wolves",
"subfaction": "Space Wolves", "subfaction": null,
"class": "Fast Attack", "class": "Mounted",
"name": "Thunderwolf Cavalry", "name": "Thunderwolf Cavalry",
"aliases": ["thunder wolf", "thunderwolf cavalry squad"] "aliases": ["thunder wolf", "thunderwolf cavalry squad"]
} }
sorter.py
+1 -1
View File
@@ -35,7 +35,7 @@ from dlphelper.sorter import (
log = logging.getLogger("sorter") log = logging.getLogger("sorter")
DEFAULT_CATALOG = Path(__file__).resolve().parent / "data" / "library_catalog.json" DEFAULT_CATALOG = Path(__file__).resolve().parent / "data" / "gwshop_catalog.json"
def main() -> None: def main() -> None:
tests/test_catalogue.py
+12 -11
View File
@@ -15,22 +15,23 @@ from dlphelper.catalogue import classify # noqa: E402
CASES = [ CASES = [
# (filename, expected system, expected faction, expected class or None-any) # (filename, expected system, expected faction, expected class or None-any)
("Thunderwolf Cavalry.7z", "Warhammer 40,000", "Adeptus Astartes", "Fast Attack"), # GW naming: chapters are factions, classes are GW unit types.
("Wulfen.zip", "Warhammer 40,000", "Adeptus Astartes", "Elites"), ("Thunderwolf Cavalry.7z", "Warhammer 40,000", "Space Wolves", "Mounted"),
("Keeper of Secrets & Shalaxi Helbane.7z", "Warhammer 40,000", "Chaos Daemons", "HQ"), ("Wulfen.zip", "Warhammer 40,000", "Space Wolves", "Infantry"),
("Keeper of Secrets & Shalaxi Helbane.7z", "Warhammer 40,000", "Chaos Daemons", "Hero"),
("Assorted Drones.zip", "Warhammer 40,000", "T'au Empire", None), ("Assorted Drones.zip", "Warhammer 40,000", "T'au Empire", None),
("Commander Shadowsun.zip", "Warhammer 40,000", "T'au Empire", "HQ"), ("Commander Shadowsun.zip", "Warhammer 40,000", "T'au Empire", "Hero"),
("Devilfish.zip", "Warhammer 40,000", "T'au Empire", "Dedicated Transport"), ("Devilfish.zip", "Warhammer 40,000", "T'au Empire", "Vehicle"),
("Battle Sister Scanz.zip", "Warhammer 40,000", "Adepta Sororitas", None), ("Battle Sister Scanz.zip", "Warhammer 40,000", "Adepta Sororitas", None),
("Knights.zip", None, None, None), # ambiguous keyword alone: unclassified ("Knights.zip", None, None, None), # ambiguous keyword alone: unclassified
("Khorne Demon Enginez and skull lordz.zip", "Warhammer 40,000", "Chaos Daemons", None), ("Khorne Demon Enginez and skull lordz.zip", "Warhammer 40,000", "Chaos Daemons", None),
("Cultists.zip", "Warhammer 40,000", "Chaos Space Marines", "Elites"), ("Cultists.zip", "Warhammer 40,000", "Chaos Space Marines", "Battleline"),
("Da Troopz.zip", "Warhammer 40,000", "Orks", None), ("Da Troopz.zip", "Warhammer 40,000", "Orks", None),
("Seraphon.zip", "Other Systems", "Age of Sigmar", None), ("Seraphon.zip", "Age of Sigmar", "Seraphon", None),
("Leviathan Dreadnought.zip", "Warhammer 40,000", "Adeptus Astartes", "Elites"), ("Leviathan Pattern Dreadnought.zip", "The Horus Heresy", "Space Marine Legions", "Walker"),
("Poxwalkers.zip", "Warhammer 40,000", "Death Guard", "Elites"), ("Poxwalkers.zip", "Warhammer 40,000", "Death Guard", "Infantry"),
("Kabalite Warriors.zip", "Warhammer 40,000", "Drukhari", "Troops"), ("Kabalite Warriors.zip", "Warhammer 40,000", "Drukhari", "Battleline"),
("Canoptek Wraiths.zip", "Warhammer 40,000", "Necrons", "Fast Attack"), ("Canoptek Wraiths.zip", "Warhammer 40,000", "Necrons", "Infantry"),
("random thing with no keywords.zip", None, None, None), ("random thing with no keywords.zip", None, None, None),
] ]