Switch library hierarchy to GW webshop taxonomy

- warhammer_data.py: GW naming throughout - Space Marine chapters are their own factions under a 'Space Marines' group, systems match the shop (Age of Sigmar, The Horus Heresy, Other Games, The Old World, Middle-Earth), classes are GW shop categories (Battleline, Infantry, Vehicle, Walker, Flyer, Monster, Mounted, Hero, Titanic), group map extended to AoS grand alliances / Other Games / Old World / Middle-Earth - gwshop.py: hierarchy remap - GW depth-4 paths are system/group/faction/ class (the shop's last level is the class layer), depth-5 sources keep a subfaction before it; catalog snapshot regenerated accordingly - sorter.py: catalog entry is hierarchy-authoritative (group, class, and faction when GW moved a unit or keyword tables are stale); generic GW branches ('Unit Type') and a Battleline default never degrade a confident keyword classification; whole-catalog retry when the faction pool has no confident match (handles units GW moved between factions); whole-catalog matching also runs when keyword classification found no faction; invalid (HTML) items never fuzzy-match; default catalog is now data/gwshop_catalog.json - catalogue.py: --apply moves include the group layer, same structure as the sorter - docs/SORTING.md: hierarchy rewritten to GW naming with examples - tests: expectations updated to GW naming (chapters, unit types) - verified live: 4/4 real files in the test batch now catalog-match, e.g. The Twin Lance -> 40k/Xenos Armies/T'au Empire/Epic Hero (GW's current placement) instead of the stale Drukhari keyword
2026-10-04 22:05:31 +02:00
parent 5fd5dacaf0
commit 4bbaefb45a
8 changed files with 4678 additions and 4428 deletions
@@ -25,7 +25,12 @@ import shutil
 from dataclasses import dataclass, field
 from pathlib import Path

-from dlphelper.warhammer_data import FACTION_ALIASES, SYSTEMS, UNIT_CLASSES
+from dlphelper.warhammer_data import (
+    FACTION_ALIASES,
+    GROUP_OF_FACTION,
+    SYSTEMS,
+    UNIT_CLASSES,
+)

 log = logging.getLogger(__name__)

@@ -252,7 +257,11 @@ def apply_moves(
            if not src.exists():
                log.warning("Cannot move, missing: %s", src)
                continue
-            dest_dir = library_root / e.system / e.faction / e.klass
+            dest_dir = (
+                library_root / e.system
+                / GROUP_OF_FACTION.get(e.faction, "Unclassified")
+                / e.faction / e.klass
+            )
            dest = dest_dir / src.name
            if dest.exists():
                log.warning("Skip, destination exists: %s", dest)
@@ -257,15 +257,24 @@ def build_catalog(hits: list[dict]) -> dict:
            )
            continue
        # primary branch = longest path (most specific), e.g.
-        # [40k, Armies of the Imperium, Adeptus Custodes, Rules & Accessories]
+        # [40k, Space Marines, Raven Guard, Walker]
        primary = max(paths, key=len)
        system = primary[0] if len(primary) > 0 else None
        group = primary[1] if len(primary) > 1 else None
        faction = primary[2] if len(primary) > 2 else None
-        subfaction = primary[3] if len(primary) > 3 else None
-        # 5th level (if present) is treated as class; GW uses "Rules &
-        # Accessories" etc. rather than force-org classes.
-        klass = primary[4] if len(primary) > 4 else None
+        # The LAST level is the class/category layer (GW uses shop
+        # categories there: Battleline, Walker, Vehicle, "Rules &
+        # Accessories", product lines like "Cadian"). Deeper sources
+        # (miniset style) may carry a real subfaction before it.
+        if len(primary) >= 5:
+            subfaction = primary[3]
+            klass = primary[4]
+        elif len(primary) == 4:
+            subfaction = None
+            klass = primary[3]
+        else:
+            subfaction = None
+            klass = None
        key = (hit.get("objectID"), tuple(primary))
        if key in seen:
            continue
@@ -22,6 +22,7 @@ from dataclasses import dataclass, field
 from pathlib import Path

 from dlphelper.catalogue import _clean_name, _match_class, _match_faction
+from dlphelper.warhammer_data import GROUP_OF_FACTION

 log = logging.getLogger(__name__)

@@ -154,26 +155,7 @@ def classify_item(item: SortItem) -> None:
    item.group = GROUP_OF_FACTION.get(item.faction, UNSORTED)


-# faction group layer (miniset naming), extended as the taxonomy grows
-GROUP_OF_FACTION: dict[str, str] = {
-    faction: group
-    for group, factions in {
-        "Armies of the Imperium": [
-            "Adeptus Astartes", "Astra Militarum", "Adeptus Mechanicus",
-            "Adepta Sororitas", "Adeptus Custodes", "Imperial Knights",
-            "Inquisition & Officio Assassinorum",
-        ],
-        "Armies of Chaos": [
-            "Chaos Space Marines", "Death Guard", "Thousand Sons",
-            "Chaos Daemons", "Chaos Knights",
-        ],
-        "Xenos Armies": [
-            "Orks", "T'au Empire", "Tyranids", "Genestealer Cults", "Necrons",
-            "Aeldari", "Drukhari", "Harlequins", "Ynnari",
-        ],
-    }.items()
-    for faction in factions
-}
+# faction group layer (GW naming, shared with warhammer_data)


 def load_catalog(path: Path) -> list[dict]:
@@ -192,12 +174,8 @@ def _best_ratio(clean: str, candidates: list[str]) -> float:
    )


-def match_catalog(item: SortItem, catalog: list[dict], threshold: float) -> None:
-    """Fuzzy-match against catalog entries in the same faction (or all)."""
-    if item.faction == UNSORTED:
-        return
-    same_faction = [e for e in catalog if e["faction"] == item.faction]
-    pool = same_faction or catalog  # fall back to whole catalog if faction unknown
+def _best_in_pool(item: SortItem, pool: list[dict]) -> tuple[float, dict] | None:
+    """Best fuzzy score of `item` against all entries in `pool`."""
    best: tuple[float, dict] | None = None
    for entry in pool:
        cands = [entry["_clean"], *entry["_clean_aliases"]]
@@ -208,22 +186,83 @@ def match_catalog(item: SortItem, catalog: list[dict], threshold: float) -> None
                score = max(score, _best_ratio(c, cands) * 0.95)  # inner slightly discounted
        if best is None or score > best[0]:
            best = (score, entry)
+    return best
+
+
+def match_catalog(item: SortItem, catalog: list[dict], threshold: float) -> None:
+    """Fuzzy-match against catalog entries in the same faction (or all).
+
+    The matched entry is hierarchy-authoritative: group, subfaction and
+    class come from the catalog (GW shop categories), and when the keyword
+    classification had no faction the entry's system/faction is adopted too.
+    """
+    if item.faction == UNSORTED and not item.inner_names and not item.clean:
+        return
+    if item.match_kind == "invalid":
+        return  # HTML-page masquerading as an archive; never fuzzy-match it
+    same_faction = [e for e in catalog if e.get("faction") == item.faction]
+    pool = same_faction or catalog  # fall back to whole catalog if faction unknown
+    best = _best_in_pool(item, pool)
+    if best is None or best[0] < threshold:
+        # The keyword tables can lag behind GW's current categorization
+        # (e.g. units moved between factions); if the faction pool has no
+        # confident match, try the whole catalog before giving up.
+        if same_faction:
+            best = _best_in_pool(item, catalog)
    if best and best[0] >= threshold:
        entry = best[1]
        item.unit = entry["name"]
-        item.subfaction = entry.get("subfaction")
-        item.klass = item.klass or entry.get("class")
+        if entry.get("group"):
+            item.group = entry["group"]
+        elif item.faction == UNSORTED:
+            item.group = GROUP_OF_FACTION.get(entry["faction"], UNSORTED)
+        if entry.get("subfaction"):
+            item.subfaction = entry["subfaction"]
+        # GW's class layer (shop category) beats the keyword guess, but do
+        # not *narrow* it: "Battleline" is GW's bucket for core infantry,
+        # a keyword-derived Hero/Elite is more specific for a proxy file.
+        catalog_klass = entry.get("class")
+        if catalog_klass and item.klass in (None, "", UNCLASSIFIED_CLASS):
+            item.klass = catalog_klass
        item.match_score = best[0]
        item.match_kind = "catalog"
+        # The catalog entry's faction is authoritative - but only adopt it
+        # when it is a real army faction. Generic GW branches like
+        # "Unit Type" or factionless entries would degrade a confident
+        # keyword faction to a junk folder.
+        entry_faction = entry.get("faction")
+        generic_factions = {"Unit Type", "Heroes", "Warriors", "Character"}
        if item.faction == UNSORTED:
+            if entry_faction and entry_faction not in generic_factions:
+                item.system = entry["system"]
+                item.faction = entry_faction
+            if entry.get("group") and entry["group"] not in ("Unit Type",):
+                item.group = entry["group"]
+            else:
+                item.group = GROUP_OF_FACTION.get(item.faction, UNSORTED)
+        elif entry_faction and entry_faction not in generic_factions and entry_faction != item.faction:
+            # GW moved this unit to another faction (or the keyword tables
+            # are stale): trust the catalog.
            item.system = entry["system"]
-            item.faction = entry["faction"]
-            item.group = entry.get("group") or GROUP_OF_FACTION.get(entry["faction"], UNSORTED)
+            item.faction = entry_faction
+            if entry.get("group") and entry["group"] != "Unit Type":
+                item.group = entry["group"]
+            else:
+                item.group = GROUP_OF_FACTION.get(entry_faction, UNSORTED)
+        elif entry.get("group") and entry["group"] not in ("Unit Type",):
+            item.group = entry["group"]
    else:
-        if item.faction != UNSORTED:
+        if item.faction == UNSORTED:
            item.unit = item.name  # implicit entry: own (original) name
            item.match_score = best[0] if best else 0.0
            item.match_kind = "implicit"
+        else:
+            # Faction was known but nothing fuzzy-matched: keep the keyword
+            # faction and use the file's own name as the unit, but adopt the
+            # catalog's class layer when it adds information.
+            item.unit = item.name
+            item.match_score = best[0] if best else 0.0
+            item.match_kind = "implicit"


 def build_target(item: SortItem, library: Path) -> Path:
@@ -1,12 +1,19 @@
 """Warhammer taxonomy and unit keyword tables used by the catalogue.

+Naming follows the Games Workshop webshop hierarchy (the same one
+dlphelper.gwshop dumps from Algolia):
+
+    <System>/<Group>/<Faction>/<Class>          e.g.
+    Warhammer 40,000/Space Marines/Ultramarines/Walker
+
 Two layers:
 - SYSTEMS: game system -> faction -> keyword aliases (matched against file
  names). Faction detection is the primary signal; unit names double as
-  faction hints since they are listed under their faction.
- CLASSES: keyword -> force-organization class (40k Force Organization slots:
-  HQ, Troops, Elites, Fast Attack, Heavy Support, Flyer, Dedicated Transport,
-  Lords of War, Fortification).
+  faction hints since they are listed under their faction. Space Marine
+  chapters are their own GW factions under the "Space Marines" group.
+- CLASSES: keyword -> class. With GW as the catalog source the class layer
+  holds GW shop categories (unit types: Battleline, Infantry, Walker,
+  Vehicle, Flyer, Monster, Mounted, Hero, Titanic).

 The unit table grows with the collection; it does not attempt to enumerate
 every unit Games Workshop ever released - unknown files land in the
@@ -21,20 +28,65 @@ from __future__ import annotations

 SYSTEMS: dict[str, dict[str, list[str]]] = {
    "Warhammer 40,000": {
-        # -- Imperium -------------------------------------------------------
-        "Adeptus Astartes": [
-            "space marine", "astartes", "primaris", "ultramarines", "blood angels",
-            "dark angels", "space wolves", "black templars", "imperial fists",
-            "salamanders", "raven guard", "iron hands", "white scars", "grey knights",
-            "deathwatch", "crimson fists", "flesh tearers", "ravenwing", "deathwing",
-            "inner circle", "wulfen", "thunderwolf", "intercessor", "reiver", "aggressor", "inceptor",
-            "hellblaster", "eradicator", "bladeguard", "sternguard", "devastator",
-            "tactical squad", "terminator squad", "assault marine",
-            "redemptor", "contemptor", "dreadnought", "invictor", "repulsor",
-            "land raider", "land speeder", "predator", "vindicator", "whirlwind",
-            "drop pod", "razorback", "stormraven", "storm talon", "stormhawk",
-            "caestus", "chaplain", "librarian", "apothecary", "vexilus",
+        # -- Space Marines (chapters are their own GW factions) --------------
+        "Space Marines": [
+            "space marine", "astartes", "primaris", "intercessor", "reiver",
+            "aggressor", "inceptor", "hellblaster", "eradicator", "bladeguard",
+            "sternguard", "devastator", "tactical squad", "terminator squad",
+            "assault marine", "assault intercessor", "outrider", "scout squad",
+            "vanguard veteran", "invictor", "impulsor", "gladiator",
+            "land speeder", "drop pod", "inner circle", "honour guard",
        ],
+        "Ultramarines": ["ultramarines", "guilliman", "calgar"],
+        "Blood Angels": [
+            "blood angel", "sanguinary guard", "sanguinor", "death company",
+            "baal predator", "mephiston", "dante", "astorath", "flesh tearers",
+            "lamberto", "furioso",
+        ],
+        "Dark Angels": [
+            "dark angel", "ravenwing", "deathwing", "inner circle companions",
+            "azrael", "asmodai", "belial", "samuel", "lion el'jonson",
+            "lion el jonson", "nephilim jetfighter",
+        ],
+        "Space Wolves": [
+            "space wolf", "space wolves", "wulfen", "thunderwolf", "wolf guard",
+            "rune priest", "wolf priest", "wolf lord", "logan grimnar",
+            "ragnar blackmane", "canis wolfborn", "fenrisian wolf",
+            "stormfang", "murderfang", "lukas the trickster",
+        ],
+        "Black Templars": [
+            "black templar", "crusader squad", "high marshal helbrecht",
+            "grimaldus", "the emperor's champion", "emperors champion",
+            "castellan", "land raider crusader",
+        ],
+        "Imperial Fists": [
+            "imperial fist", "rogal dorn", "lupos", "tor garadon",
+            "horatius hold", "dorn's arrow",
+        ],
+        "Raven Guard": [
+            "raven guard", "kayvaan shrike", "corvus", "sable brand",
+            "shadow captains",
+        ],
+        "Iron Hands": [
+            "iron hand", "iron hands", "ferrus manus", "iron father",
+            "medusan immortals",
+        ],
+        "White Scars": ["white scar", "jaghatai khan", "stormseer", "korsos"],
+        "Salamanders": [
+            "salamander", "vulkan", "he'stan", "adrax agatone",
+            "drake-scale", "flamecaster",
+        ],
+        "Grey Knights": [
+            "grey knight", "grand master", "brother-captain", "nemesis dreadknight",
+            "draigo", "castellan crowe", "sternguard knight",
+        ],
+        "Deathwatch": [
+            "deathwatch", "watch captain", "watch master", "kill team cassius",
+            "overkill", "corvus blackstar",
+        ],
+        "Crimson Fists": ["crimson fist", "kantor", "rynn's world"],
+        "Flesh Tearers": ["flesh tearer", "gabriel seth", "amissum"],
+        # -- Imperium --------------------------------------------------------
        "Astra Militarum": [
            "astra militarum", "imperial guard", "cadian", "catachan", "krieg",
            "death korps", "tempestus", "scion", "valhallan", "steel legion",
@@ -72,23 +124,22 @@ SYSTEMS: dict[str, dict[str, list[str]]] = {
            "armiger", "warglaive", "helverin", "morialtis", "canis rex",
            "cerastus", "atropos", "seneschal",
        ],
-        "Inquisition & Officio Assassinorum": [
+        "Imperial Agents": [
            "inquisition", "inquisitor", "astra telepathica", "witch hunter",
            "coteaz", "eisenhorn", "ravenor", "kyria draxus",
            "officio assassinorum", "culexus", "vindicare", "callidus", "eversor",
            "saint celestine", "living saint", "assassin",
        ],
-        # -- Chaos ----------------------------------------------------------
+        # -- Chaos -----------------------------------------------------------
        "Chaos Space Marines": [
            "chaos space marine", "black legion", "iron warriors", "word bearers",
-            "night lords", "alpha legion", "red corsairs", "world eaters",
-            "emperor's children", "chaos terminator", "chosen", "havoc",
+            "night lords", "alpha legion", "chaos terminator", "chosen", "havoc",
            "raptor", "warp talon", "chaos cultist", "cultist", "daemon prince",
-            "chaos lord", "chaos sorcerer", "abaddon", "huron blackheart",
-            "kharn", "lucius the eternal", "ahriman", "typhus", "fabius bile",
-            "vashtorr", "defiler", "forgefiend", "maulerfiend", "helldrake",
-            "helbrute", "chaos spawn", "obliterator", "chaos rhino",
-            "chaos vindicator", "spartan assault", "mastodon", "typhon",
+            "chaos lord", "chaos sorcerer", "abaddon", "kharn", "ahriman",
+            "typhus", "fabius bile", "vashtorr", "defiler", "forgefiend",
+            "maulerfiend", "helldrake", "helbrute", "chaos spawn",
+            "obliterator", "chaos rhino", "chaos vindicator", "spartan assault",
+            "mastodon", "typhon",
        ],
        "Death Guard": [
            "death guard", "nurgle", "plague marine", "deathshroud", "mortarion",
@@ -113,11 +164,21 @@ SYSTEMS: dict[str, dict[str, list[str]]] = {
            "skulltaker", "khorne", "slaanesh", "skull lord",
            "shalaxi helbane",
        ],
+        "World Eaters": [
+            "world eater", "angron", "eightbound", "exalted eightbound",
+            "jakhals", "berserker", "skull lord",
+        ],
+        "Emperor's Children": [
+            "emperor's children", "emperors children", "fulgrim",
+            "noise marine", "lucius the eternal", "flawless blades",
+        ],
+        "Iron Warriors": ["iron warrior", "perturabo"],
+        "Red Corsairs": ["red corsair", "huron blackheart"],
        "Chaos Knights": [
            "chaos knight", "knight desecrator", "knight despoiler",
            "knight rampager", "kharibdyss", "renegade knight",
        ],
-        # -- Xenos ----------------------------------------------------------
+        # -- Xenos -----------------------------------------------------------
        "Orks": [
            "ork", "orks", "orkz", "nob", "boyz", "grot", "weirdboy", "painboy",
            "warboss", "ghazghkull", "meganob", "deff dread", "killa kan",
@@ -162,7 +223,7 @@ SYSTEMS: dict[str, dict[str, list[str]]] = {
        ],
        "Aeldari": [
            "aeldari", "eldar", "craftworld", "ulthwe", "biel-tan", "iyanden",
-            "alaitoc", "saim-hann", "ynnari", "farseer", "warlock",
+            "alaitoc", "saim-hann", "farseer", "warlock",
            "spiritseer", "autarch", "asurmen", "guardian defender",
            "ranger", "wraithguard", "wraithblades", "wraithlord",
            "wraithknight", "dire avenger", "howling banshee",
@@ -178,62 +239,124 @@ SYSTEMS: dict[str, dict[str, list[str]]] = {
            "kabalite warrior", "raider", "ravager", "venom", "razorwing",
            "voidraven", "grotesque", "talos", "cronos",
            "beast pack", "khymera", "razorwing flock", "asdrubael vect",
-            "lilith", "sathonyx", "the twin lance", "luriel",
+            "lilith", "sathonyx", "luriel",
        ],
        "Harlequins": [
            "harlequin", "death jester", "shadowseer", "solitaire",
            "troupe", "starweaver", "voidweaver",
        ],
-        "Ynnari": [
-            "ynnari", "ynnead", "visarch", "yncarne", "yvraine",
+        "Leagues of Votann": [
+            "leagues of votann", "votann", "hearthkyn", "ironkin",
+            "brokhyr", "kahl", "cthonian", "transhyperic", "thegn",
        ],
    },
-    "Other Systems": {
-        "Age of Sigmar": [
-            "age of sigmar", "stormcast", "eternals", "orruk", "warclan",
-            "kruleboyz", "gloomspite", "sylvaneth", "seraphon", "ossiarch",
-            "nighthaunt", "lumineth", "daughters of khaine", "fyreslayers",
-            "kharadron", "idoneth", "blades of khorne", "maggotkin",
-            "gaunt summoner", "soulblight", "freeguild", "cities of sigmar",
-            "skaven", "chaos warriors", "saurus", "skink", "liberator",
-            "sequitor", "evocator", "vindictor", "kaldor draigo",
+    "Age of Sigmar": {
+        "Stormcast Eternals": [
+            "stormcast", "eternal", "liberator", "sequitor", "evocator",
+            "vindictor", "judicators", "retributors", "sacrosanct",
+            "dracothian", "lord-arcane", "knight-questor", "neave blacktalon",
+            "stormdrake", "krondys", "karazai",
        ],
-        "Horus Heresy": [
-            "horus heresy", "legiones astartes", "pre-heresy",
-            "leviathan pattern", "fire raptor", "sicaran", "fellblade",
-            "kratos", "justaerin", "eisenstein", "veteran tactical",
-            "seeker squad", "suzerain", "invictarus", "fulgurite",
+        "Seraphon": ["seraphon", "saurus", "skink", "slann", "carnosaur",
+                     "bastiladon", "stegadon", "razordon", "salamander"],
+        "Orruk Warclans": ["orruk", "warclan", "kruleboyz", "ironjawz",
+                          "brute", "gorger", "weirdnob", "warchanter"],
+        "Gloomspite Gitz": ["gloomspite", "gitz", "moonclan", "spiderfang",
+                            "troggoth", "loontouched", "zarbag", "skragrot"],
+        "Sylvaneth": ["sylvaneth", "treelord", "dryad", "branchwraith",
+                      "alarielle", "kurnothi"],
+        "Ossiarch Bonereapers": ["ossiarch", "bonereaper", "mortek", "keldrek",
+                                 "katakros"],
+        "Nighthaunt": ["nighthaunt", "grimghast", "bladegheist", "chainghast",
+                       "kurdoss", "lady olynder"],
+        "Lumineth Realm-lords": ["lumineth", "aelf", "vanari", "teclis",
+                                 "light of eltharion"],
+        "Daughters of Khaine": ["daughters of khaine", "khainite", "witch aelf",
+                                "morgwaeth", "morathi"],
+        "Fyreslayers": ["fyreslayer", "vulkite", "hearthguard", "auric",
+                        "grimhold"],
+        "Kharadron Overlords": ["kharadron", "skyfarer", "endofters",
+                                "grundstok", "arcanaut"],
+        "Idoneth Deepkin": ["idoneth", "deepkin", "namarti", "isharann",
+                            "volturnos"],
+        "Blades of Khorne": ["blades of khorne", "khorne", "skullgrinder",
+                             "blood warrior", "wrathmonger"],
+        "Maggotkin of Nurgle": ["maggotkin", "nurgle", "blightking",
+                                "rotmire", "orghotts"],
+        "Slaves to Darkness": ["slaves to darkness", "chaos warrior",
+                               "chaos knight", "archaon", "gaunt summoner",
+                               "everchosen"],
+        "Skaven": ["skaven", "clanrat", "stormvermin", "verminlord",
+                   "grey seer", "warplock", "ratling gun"],
+        "Soulblight Gravelords": ["soulblight", "gravelord", "vampire",
+                                  "deathrattle", "koldhunter", "belladamma"],
+        "Cities of Sigmar": ["cities of sigmar", "freeguild", "windspeaker",
+                             "callis and toll", "hammerers"],
+    },
+    "The Horus Heresy": {
+        "Space Marine Legions": [
+            "legiones astartes", "pre-heresy", "horus heresy", "justaerin",
+            "seeker squad", "suzerain", "invictarus", "veteran tactical",
+            "contemptor", "leviathan pattern", "sicaran", "fellblade",
+            "kratos", "fire raptor", "eisenstein",
        ],
+        "Mechanicum": ["mechanicum", "fulgurite", "magos", "thallax"],
+    },
+    "Other Games": {
        "Kill Team": [
            "kill team", "killteam", "killzone", "operative",
            "hand of the archon", "void-dance troupe", "veteran guardsmen",
+            "spector squad", "wolf scouts", "scout squad kill",
        ],
        "Necromunda": [
            "necromunda", "goliath gang", "escher", "orlock", "cawdor",
-            "van saar", "ash wastes", "underhive",
+            "van saar", "ash wastes", "underhive", "sumpkrocs",
+            "forge boss", "sumpkroc", "matriarch", "phyrr cat",
        ],
-        "Legions Imperialis / Epic": [
-            "legions imperialis", "titan legio", "reaver titan",
+        "Adeptus Titanicus": [
+            "adepticus titanicus", "titan legio", "reaver titan",
            "warhound titan", "warlord titan", "imperator titan",
-            "warmaster titan",
-        ],
-        "Warhammer The Old World": [
-            "the old world", "bretonnia", "greenskins", "tomb kings",
-            "khemri", "chaos dwarf", "dog of war", "grail knight",
+            "warmaster titan", "titanicus",
        ],
+        "Warhammer Underworlds": ["warhammer underworlds", "underworlds",
+                                  "warband", "shadeborn", "embergard"],
+        "Aeronautica Imperialis": ["aeronautica", "t'au air caste",
+                                   "lightning fighter", "thunderbolt"],
+    },
+    "The Old World": {
+        "Kingdom of Bretonnia": ["bretonnia", "grail knight", "peasant",
+                                 "questing knight", "perrin"],
+        "Orc and Goblin Tribes": ["greenskin", "goblin", "night goblin",
+                                  "forest goblin", "black orc"],
+        "Tomb Kings of Khemri": ["tomb king", "khemri", "necrolith",
+                                 "skeleton archer"],
+        "Empire of Man": ["empire of man", "state troop", "greatsword",
+                          "empire knight"],
+        "Warriors of Chaos": ["chaos dwarf", "dog of war", "chaos marauder"],
+        "High Elf Realms": ["high elf", "eltharion", "phoenix guard"],
+        "Wood Elf Realms": ["wood elf", "glade guard", "waywatcher"],
+        "Beastmen Brayherds": ["beastmen", "brayherd", "ghorgon", "cygor"],
+        "Dwarfen Mountain Holds": ["dwarf", "ironbreaker", "hammerer"],
+    },
+    "Middle-Earth": {
+        "Rivendell": ["rivendell", "elrond", "elendil", "isildur", "gil-galad",
+                      "beorning", "grimbeorn", "rumil", "orophin"],
+        "Minas Tirith": ["minas tirith", "gondor", "boromir", "faramir"],
+        "Mordor": ["mordor", "orc", "orcs", "nazgul", "witch-king"],
+        "Iron Hills": ["iron hills", "dwarf", "mattock", "dwalin"],
+        "Moria": ["moria", "cave troll", "balrog"],
    },
 }

 # ---------------------------------------------------------------------------
-# Force Organization classes (keyword -> class)
+# Classes (keyword -> class). GW shop categories / unit types.
 # ---------------------------------------------------------------------------

 CLASSES: list[tuple[str, list[str]]] = [
-    ("Lords of War", [
+    ("Titanic", [
        "titan", "stompa", "gargantuan", "hierophant", "harridan", "hierodule",
-        "baneblade", "stormlord", "shadowsword", "hellhammer", "malcador",
        "imperator", "warmaster", "tesseract vault", "plaguereaper",
-        "warhound", "warlord titan",
+        "warhound", "warlord titan", "ta'unar", "spermato", "titandeath",
    ]),
    ("Fortification", ["fortification", "bunker", "bastion", "wall of martyrs"]),
    ("Flyer", [
@@ -243,64 +366,76 @@ CLASSES: list[tuple[str, list[str]]] = [
        "marauder", "razorwing", "voidraven", "crimson hunter",
        "hemlock", "doom scythe", "night scythe", "harpy", "crone",
        "fire raptor", "barracuda", "sun shark", "razorshark",
-        "wazbom blasta",
+        "wazbom blasta", "nephilim jetfighter", "corvus blackstar",
    ]),
-    ("Dedicated Transport", [
+    ("Vehicle", [
        "transport", "rhino", "razorback", "chimera", "taurox", "trukk",
        "battlewagon", "devilfish", "wave serpent", "ghost ark",
        "land raider", "repulsor", "drop pod", "starweaver", "voidweaver",
        "goliath truck", "rockgrinder", "cult limo", "raider", "venom",
-        "spartan assault",
+        "spartan assault", "predator", "vindicator", "whirlwind",
+        "basilisk", "hydra", "wyvern", "manticore", "deathstrike",
+        "onager dunecrawler", "kastelan", "hammerhead", "broadside",
+        "stormsurge", "r'varna", "soul grinder", "defiler", "forgefiend",
+        "maulerfiend", "plagueburst crawler", "annihilation barge",
+        "doomsday ark", "fire prism", "night spinner", "exorcist",
+        "immolator", "caladius", "coronus grav", "pallas grav",
+        "impulsor", "gladiator", "baal predator", "land speeder",
+        "ironstrider", "sydonian", "sentinel", "taurox", "kataphron",
    ]),
-    ("Fast Attack", [
-        "fast attack", "assault marine", "assault intercessor", "inceptor",
-        "outrider", "bike squad", "scout biker", "thunderwolf", "raptor",
-        "warp talon", "flesh hound", "screamer", "fiend of slaanesh",
-        "hellion", "reaver jetbike", "scourge", "mandrake", "swooping hawk",
-        "shining spears", "windrider", "vyper", "war walker",
-        "canoptek wraith", "canoptek scarab", "canoptek spyder",
-        "ravener", "gargoyle", "spore mine", "mawloc", "trygon",
-        "pyrovore", "khymera", "beast pack", "razorwing flock",
-        "skyrunner", "zephyrim", "seraphim", "sentinel", "jackal",
-        "stormboy", "deffkopta", "warbiker", "skorpekh destroyer",
-        "sicarian", "ironstrider", "sydonian",
+    ("Walker", [
+        "dreadnought", "redemptor", "contemptor", "leviathan pattern",
+        "brutalis", "ballistus", "deredeo", "wulfen dreadnought",
+        "killa kan", "deff dread", "wraithlord", "wraithknight",
+        "telemon", "achillus", "galatus", "paragon warsuit",
+        "nemesis dreadknight", "invictor", "talos", "cronos",
    ]),
-    ("Heavy Support", [
-        "heavy support", "devastator", "eradicator", "hellblaster", "havoc",
-        "loota", "burna boy", "carnifex", "exocrine", "haruspex",
-        "zoanthrope", "venomthrope", "biovore", "hierodule",
-        "wraithlord", "wraithknight", "fire prism", "night spinner",
-        "predator", "vindicator", "whirlwind", "basilisk", "hydra",
-        "wyvern", "manticore", "deathstrike", "onager dunecrawler",
-        "kastelan", "kataphron", "hammerhead", "broadside", "stormsurge",
-        "r'varna", "soul grinder", "defiler", "forgefiend", "maulerfiend",
-        "plagueburst crawler", "bloat-drone", "blight hauler", "mutalith",
-        "annihilation barge", "doomsday ark", "obront", "dark reaper",
-        "retributor", "paragon warsuit", "morvenn", "exorcist", "immolator",
-        "caladius", "coronus grav", "telemon", "achillus", "galatus",
-        "pallas grav", "skorpekh",
+    ("Monster", [
+        "monster", "carnifex", "exocrine", "haruspex", "mawloc", "trygon",
+        "tervigon", "hierodule", "hierophant", "mutalith", "bloat-drone",
+        "kharibdyss", "gorkanaut", "morkanaut", "magnus", "mortalith",
+        "fiend of slaanesh", "plague drone", "cave troll", "balrog",
+        "carnosaur", "stegadon", "bastiladon", "grorgeous",
    ]),
-    ("Elites", [
-        "elites", "veteran", "terminator", "sternguard", "bladeguard",
-        "aggressor", "reiver", "vanguard veteran", "dreadnought",
-        "redemptor", "contemptor", "leviathan pattern", "wulfen",
-        "wolf guard", "deathshroud", "plague marine", "blightlord",
-        "poxwalker", "rubric marine", "rubricae", "scarab occult",
-        "tzaangor", "chosen", "possessed", "obliterator", "daemonette",
-        "bloodletter", "plaguebearer", "nurgling", "juggernaut",
-        "chaos cultist", "cultist", "acolyte hybrid", "aberrant",
+    ("Mounted", [
+        "mounted", "thunderwolf", "outrider", "bike squad", "scout biker",
+        "reaver jetbike", "windrider", "skyrunner", "warbiker",
+        "allarus custodian on", "vertus praetor", "drakeling",
+        "boarboy", "gruntaz", "khan on", "dracothian guard",
+    ]),
+    ("Battleline", [
+        "battleline", "troops", "tactical squad", "intercessor",
+        "incursor", "infiltrator", "infantry squad", "conscript",
+        "krieg", "catachan", "cadian", "vostroyan", "penal legion",
+        "ratling", "veteran squad", "fire warrior", "pathfinder team",
+        "kroot", "termagant", "hormagaunt", "necron warrior",
+        "necron immortal", "guardian defender", "storm guardian",
+        "kabalite warrior", "boyz", "gretchin", "scion", "scout squad",
+        "cultist", "plague marine squad",
+    ]),
+    ("Infantry", [
+        "infantry", "veteran", "terminator", "sternguard", "bladeguard",
+        "aggressor", "reiver", "wulfen", "wolf guard", "deathshroud",
+        "blightlord", "poxwalker", "rubric marine", "rubricae",
+        "scarab occult", "tzaangor", "chosen", "possessed", "obliterator",
+        "daemonette", "bloodletter", "plaguebearer", "nurgling",
+        "juggernaut", "chaos cultist", "acolyte hybrid", "aberrant",
        "neophyte", "purestrain", "genestealer", "lictor", "deathmark",
        "flayed one", "lychguard", "wraithguard", "wraithblades",
        "howling banshee", "striking scorpion", "warp spider",
-        "fire dragon", "incubi", "wrack", "grotesque", "talos", "cronos",
-        "troupe", "killa kan", "deff dread", "meganob", "nobz", "nob",
-        "kommando", "vespid", "sanguinary guard", "sanguinor",
-        "ancient", "allarus", "vertus praetor", "venatari", "aquilon",
-        "sagittarum", "witchseeker", "prosecutor", "vigilator",
-        "vindicare", "callidus", "eversor", "culexus", "assassin",
+        "fire dragon", "incubi", "wrack", "grotesque", "assault marine",
+        "assault intercessor", "inceptor", "raptor", "warp talon",
+        "flesh hound", "screamer", "hellion", "scourge", "mandrake",
+        "swooping hawk", "shining spears", "canoptek wraith",
+        "canoptek scarab", "canoptek spyder", "ravener", "gargoyle",
+        "spore mine", "pyrovore", "khymera", "beast pack",
+        "razorwing flock", "zephyrim", "seraphim", "stormboy",
+        "deffkopta", "skorpekh destroyer", "skorpekh", "retributor",
+        "dark reaper", "sicarian", "ruststalker", "skitarii",
+        "hormagaunt brood", "termagant brood", "vanguard",
    ]),
-    ("HQ", [
-        "hq", "captain", "chapter master", "primarch", "lieutenant",
+    ("Hero", [
+        "hero", "hq", "captain", "chapter master", "primarch", "lieutenant",
        "librarian", "chaplain", "apothecary", "farseer", "warlock",
        "spiritseer", "autarch", "warboss", "big mek", "weirdboy",
        "painboy", "commander", "ethereal", "hive tyrant", "swarmlord",
@@ -316,18 +451,12 @@ CLASSES: list[tuple[str, list[str]]] = [
        "dialogus", "hospitaller", "trajann", "valdor", "inquisitor",
        "coteaz", "eisenhorn", "ravenor", "draxus", "celestine",
        "commissar", "priest", "enginseer", "dominus", "manipulus",
-        "archmagos", "magos", "belisarius cawl", "farsight", "shadowsun",
+        "archmagos", "belisarius cawl", "farsight", "shadowsun",
        "aun'va", "darkstrider", "longstrike", "shade runner",
    ]),
-    ("Troops", [
-        "troops", "battleline", "tactical squad", "intercessor",
-        "incursor", "infiltrator", "infantry squad", "conscript",
-        "krieg", "catachan", "cadian", "vostroyan", "penal legion",
-        "ratling", "veteran squad", "fire warrior", "pathfinder team",
-        "kroot", "termagant", "hormagaunt", "necron warrior",
-        "necron immortal", "guardian defender", "storm guardian",
-        "kabalite warrior", "boyz", "gretchin",
-        "scion", "scout squad",
+    ("Rules & Accessories", [
+        "codex", "rulebook", "rules", "data cards", "datacards",
+        "warscroll", "index card", "cheat sheet", "reference",
    ]),
 ]

@@ -338,3 +467,63 @@ FACTION_ALIASES: dict[str, dict[str, str]] = {
 }

 UNIT_CLASSES: dict[str, str] = {kw: klass for klass, kws in CLASSES for kw in kws}
+
+# GW group of each faction (matches the GameSystemsRoot.lvl1 values in the
+# Algolia index). Used when a keyword-classified file cannot be fuzzy-matched.
+GROUP_OF_FACTION: dict[str, str] = {
+    faction: group
+    for group, factions in {
+        "Space Marines": [
+            "Space Marines", "Ultramarines", "Blood Angels", "Dark Angels",
+            "Space Wolves", "Black Templars", "Imperial Fists", "Raven Guard",
+            "Iron Hands", "White Scars", "Salamanders", "Grey Knights",
+            "Deathwatch", "Crimson Fists", "Flesh Tearers",
+        ],
+        "Armies of the Imperium": [
+            "Astra Militarum", "Adeptus Mechanicus", "Adepta Sororitas",
+            "Adeptus Custodes", "Imperial Knights", "Imperial Agents",
+        ],
+        "Armies of Chaos": [
+            "Chaos Space Marines", "Death Guard", "Thousand Sons",
+            "Chaos Daemons", "Chaos Knights", "World Eaters",
+            "Emperor's Children", "Iron Warriors", "Red Corsairs",
+        ],
+        "Xenos Armies": [
+            "Orks", "T'au Empire", "Tyranids", "Genestealer Cults", "Necrons",
+            "Aeldari", "Drukhari", "Harlequins", "Leagues of Votann",
+        ],
+        "Grand Alliance Order": [
+            "Stormcast Eternals", "Seraphon", "Sylvaneth",
+            "Lumineth Realm-lords", "Daughters of Khaine", "Fyreslayers",
+            "Kharadron Overlords", "Idoneth Deepkin", "Cities of Sigmar",
+        ],
+        "Grand Alliance Destruction": [
+            "Orruk Warclans", "Gloomspite Gitz",
+        ],
+        "Grand Alliance Death": [
+            "Ossiarch Bonereapers", "Nighthaunt", "Soulblight Gravelords",
+        ],
+        "Grand Alliance Chaos": [
+            "Slaves to Darkness", "Blades of Khorne", "Maggotkin of Nurgle",
+            "Skaven",
+        ],
+        "Space Marine Legions": ["Space Marine Legions", "Mechanicum"],
+        "Kill Team": ["Kill Team"],
+        "Necromunda": ["Necromunda"],
+        "Adeptus Titanicus": ["Adeptus Titanicus"],
+        "Warhammer Underworlds": ["Warhammer Underworlds"],
+        "Aeronautica Imperialis": ["Aeronautica Imperialis"],
+        "Armies of the Old World": [
+            "Kingdom of Bretonnia", "Orc and Goblin Tribes",
+            "Tomb Kings of Khemri", "Empire of Man", "Warriors of Chaos",
+            "High Elf Realms", "Wood Elf Realms", "Beastmen Brayherds",
+            "Dwarfen Mountain Holds",
+        ],
+        "Rivendell": ["Rivendell"],
+        "Minas Tirith": ["Minas Tirith"],
+        "Mordor": ["Mordor"],
+        "Iron Hills": ["Iron Hills"],
+        "Moria": ["Moria"],
+    }.items()
+    for faction in factions
+}
@@ -4,47 +4,50 @@ This document defines the target structure of the collection ("the library")
 and how the sorter places files into it. It is the working agreement for all
 sorting-related code; change it here first, then in code.

-## Target hierarchy (miniset.net-aligned)
+## Target hierarchy (Games Workshop webshop-aligned)

-The hierarchy mirrors miniset.net ("Miniatures Collectors Guide"), extended
-with the 40k force-organization class, which miniset itself uses as a
-"specification" tag on set pages:
+The hierarchy follows the official GW webshop taxonomy (dumped from the
+shop's Algolia index by `dlphelper/gwshop.py` into
+`data/gwshop_catalog.json`). Chapters and sub-lines that the old miniset
+hierarchy treated as sub-factions are full GW factions, and the class layer
+holds GW shop categories (unit types / product lines) instead of the retired
+40k force-organization slots:

 ```
 <library>/
-  Warhammer 40,000/                       # game
-    Armies of the Imperium/               # faction group (miniset naming)
-      Adeptus Astartes/                   # faction
-        Space Wolves/                     # sub-faction (optional)
-          Fast Attack/                    # force-org class
-            Thunderwolf Cavalry/          # canonical unit name
-              Thunderwolf Cavalry.7z      # the file(s), or unpacked contents
+  Warhammer 40,000/                       # game (GW system)
+    Space Marines/                        # GW group
+      Raven Guard/                        # GW faction (chapter)
+        Walker/                           # GW class (unit type / category)
+          Redemptor Dreadnought/          # canonical unit name (catalog)
+            Redemptor Dreadnought.zip     # the file(s), or unpacked contents
 ```

 Layer rules:

 | Layer | Source of truth | Example | May be omitted? |
 |-------|-----------------|---------|-----------------|
-| Game | `warhammer_data.SYSTEMS` | `Warhammer 40,000` | no (else: `_unsorted/`) |
-| Faction group | Imperium/Chaos/Xenos/etc. | `Armies of Chaos` | no (derived from faction) |
-| Faction | miniset.net faction names | `Death Guard` | no (else `_unsorted/` in game) |
-| Sub-faction | miniset sub-factions | `Space Wolves` | **yes** — only when known |
-| Class | force-org slots | `Fast Attack` | **yes** — only when known |
-| Unit | canonical model name (lexicanum spelling) | `Thunderwolf Cavalry` | yes (files drop directly into the class dir) |
+| Game | `gwshop_catalog.json` (system) | `Warhammer 40,000` | no (else: `_unsorted/`) |
+| Group | GW `GameSystemsRoot` lvl1 | `Space Marines` | no (from catalog or `warhammer_data.GROUP_OF_FACTION`) |
+| Faction | GW factions (incl. chapters) | `Raven Guard` | no (else `_unsorted/` in game) |
+| Sub-faction | deeper sources only | — | **yes** — GW has none |
+| Class | GW shop category | `Walker`, `Battleline`, `Rules & Accessories` | **yes** — only when known |
+| Unit | canonical product name (GW catalog) | `Redemptor Dreadnought` | yes (files drop directly into the class dir) |

 - Files without a faction but with a game land in `<game>/_unsorted/`.
 - Fully unclassifiable files land in `<library>/_unsorted/` — never guessed.
- Everything in between is optional and filled in as knowledge grows.
- Sub-faction folders are only created when the file name states it
-  (`Thunderwolf` → Space Wolves); otherwise files stay in the faction dir so
-  we don't invent hierarchy.
+- The keyword tables in `warhammer_data.py` (GW naming) provide the initial
+  system/faction/class signal; a catalog match then overrides the hierarchy
+  from the catalog entry, which is authoritative.
+- `data/library_catalog.json` (the older miniset-aligned hand catalog) is
+  still usable via `sorter.py --catalog`; its sub-faction entries map into
+  the same schema. For GW-sourced entries `subfaction` stays empty.

 ## Canonical names

-Canonical unit names follow Lexicanum/GW spelling (`Keeper of Secrets`,
-`Death Korps of Krieg`), because that is what cross-seeding, wikis and
-miniset.net all agree on. Release names stay untouched inside the unit
-folder.
+Canonical unit names follow the GW webshop spelling (`Keeper of Secrets`,
+`Thunderwolf Cavalry`), because that is what the catalog, the shop and
+wikis all agree on. Release names stay untouched inside the unit folder.

 ## Sorting pipeline

@@ -97,10 +100,10 @@ graph LR
 ```json
 {
  "system": "Warhammer 40,000",
-  "group": "Armies of the Imperium",
-  "faction": "Adeptus Astartes",
-  "subfaction": "Space Wolves",
-  "class": "Fast Attack",
+  "group": "Space Marines",
+  "faction": "Space Wolves",
+  "subfaction": null,
+  "class": "Mounted",
  "name": "Thunderwolf Cavalry",
  "aliases": ["thunder wolf", "thunderwolf cavalry squad"]
 }
@@ -35,7 +35,7 @@ from dlphelper.sorter import (

 log = logging.getLogger("sorter")

-DEFAULT_CATALOG = Path(__file__).resolve().parent / "data" / "library_catalog.json"
+DEFAULT_CATALOG = Path(__file__).resolve().parent / "data" / "gwshop_catalog.json"


 def main() -> None:
@@ -15,22 +15,23 @@ from dlphelper.catalogue import classify  # noqa: E402

 CASES = [
    # (filename, expected system, expected faction, expected class or None-any)
-    ("Thunderwolf Cavalry.7z", "Warhammer 40,000", "Adeptus Astartes", "Fast Attack"),
-    ("Wulfen.zip", "Warhammer 40,000", "Adeptus Astartes", "Elites"),
-    ("Keeper of Secrets & Shalaxi Helbane.7z", "Warhammer 40,000", "Chaos Daemons", "HQ"),
+    # GW naming: chapters are factions, classes are GW unit types.
+    ("Thunderwolf Cavalry.7z", "Warhammer 40,000", "Space Wolves", "Mounted"),
+    ("Wulfen.zip", "Warhammer 40,000", "Space Wolves", "Infantry"),
+    ("Keeper of Secrets & Shalaxi Helbane.7z", "Warhammer 40,000", "Chaos Daemons", "Hero"),
    ("Assorted Drones.zip", "Warhammer 40,000", "T'au Empire", None),
-    ("Commander Shadowsun.zip", "Warhammer 40,000", "T'au Empire", "HQ"),
-    ("Devilfish.zip", "Warhammer 40,000", "T'au Empire", "Dedicated Transport"),
+    ("Commander Shadowsun.zip", "Warhammer 40,000", "T'au Empire", "Hero"),
+    ("Devilfish.zip", "Warhammer 40,000", "T'au Empire", "Vehicle"),
    ("Battle Sister Scanz.zip", "Warhammer 40,000", "Adepta Sororitas", None),
    ("Knights.zip", None, None, None),  # ambiguous keyword alone: unclassified
    ("Khorne Demon Enginez and skull lordz.zip", "Warhammer 40,000", "Chaos Daemons", None),
-    ("Cultists.zip", "Warhammer 40,000", "Chaos Space Marines", "Elites"),
+    ("Cultists.zip", "Warhammer 40,000", "Chaos Space Marines", "Battleline"),
    ("Da Troopz.zip", "Warhammer 40,000", "Orks", None),
-    ("Seraphon.zip", "Other Systems", "Age of Sigmar", None),
-    ("Leviathan Dreadnought.zip", "Warhammer 40,000", "Adeptus Astartes", "Elites"),
-    ("Poxwalkers.zip", "Warhammer 40,000", "Death Guard", "Elites"),
-    ("Kabalite Warriors.zip", "Warhammer 40,000", "Drukhari", "Troops"),
-    ("Canoptek Wraiths.zip", "Warhammer 40,000", "Necrons", "Fast Attack"),
+    ("Seraphon.zip", "Age of Sigmar", "Seraphon", None),
+    ("Leviathan Pattern Dreadnought.zip", "The Horus Heresy", "Space Marine Legions", "Walker"),
+    ("Poxwalkers.zip", "Warhammer 40,000", "Death Guard", "Infantry"),
+    ("Kabalite Warriors.zip", "Warhammer 40,000", "Drukhari", "Battleline"),
+    ("Canoptek Wraiths.zip", "Warhammer 40,000", "Necrons", "Infantry"),
    ("random thing with no keywords.zip", None, None, None),
 ]