sorter: drop non-warhammer game systems (default: Middle-Earth)

The collection is warhammer-only; everything else stays in _unsorted
even when names match the catalog. The catalog snapshot keeps all
systems - the filter is a sorter default (DROPPED_SYSTEMS, applied in
load_catalog and after keyword classification), adjustable with
--drop-system/--keep-system.

dlphelper/sorter.py

@@ -32,9 +32,15 @@ UNSORTED = "_unsorted"
 UNCLASSIFIED_CLASS = "Unclassified"
 # A file whose system the keyword tables could not identify only gets a
 # catalog unit when the fuzzy hit is strong - otherwise it would wander
-# into other games' sections (Middle-earth, Blood Bowl, ...) on a guess.
+# into other games' sections (Blood Bowl, ...) on a guess.
 CROSS_SYSTEM_MIN = 0.75
 
+# Game systems whose sections the sorter must never place into. The
+# collection is warhammer-only: everything else (Middle-earth proxies,
+# ...) stays in _unsorted even when a name matches the catalog. The
+# catalog snapshot itself keeps all systems; the filter lives here.
+DROPPED_SYSTEMS = frozenset({"Middle-Earth"})
+
 # Weak acceptance tier: when nothing reached the normal threshold, a hit
 # at least this strong is accepted but flagged low confidence ("[w]" in
 # the plan, confidence field in the audit, listed in
@@ -178,9 +184,14 @@ def classify_item(item: SortItem) -> None:
 # faction group layer (GW naming, shared with warhammer_data)
 
 
-def load_catalog(path: Path) -> list[dict]:
+def load_catalog(path: Path,
+                 dropped: frozenset[str] | None = DROPPED_SYSTEMS) -> list[dict]:
+    """Load catalog entries, dropping non-collected game systems by default
+    (pass dropped=frozenset() to keep everything)."""
     data = json.loads(path.read_text(encoding="utf-8"))
     entries = data["entries"] if isinstance(data, dict) else data
+    if dropped:
+        entries = [e for e in entries if e.get("system") not in dropped]
     for e in entries:
         e["_clean"] = _clean_name(e["name"])
         e["_clean_aliases"] = [_clean_name(a) for a in e.get("aliases", [])]
@@ -672,6 +683,17 @@ def _display_name(unit: str) -> str:
 # plan / apply
 # ---------------------------------------------------------------------------
 
+def _drop_dropped_system(item: SortItem) -> None:
+    """Dropped systems (see DROPPED_SYSTEMS) count as unclassified: the
+    collection does not cover them, so their names must not place files
+    into those sections. Called after keyword classification, before
+    matching - the item can still be re-homed by other signals."""
+    if item.system in DROPPED_SYSTEMS:
+        item.system = "Unknown"
+        item.group = item.faction = UNSORTED
+        item.subfaction = None
+
+
 def sort_plan(
     files: list[Path],
     library: Path,
@@ -716,6 +738,7 @@ def sort_plan(
         if unit_name is None:
             item = inspect(f)
             classify_item(item)
+            _drop_dropped_system(item)
             match_catalog(item, catalog, threshold)
             if root is not None:
                 # manual trees: let the real ancestor dir names compete
@@ -741,6 +764,7 @@ def sort_plan(
                     clean=_clean_name(unit_name), inner_names=listing,
                 )
                 classify_item(probe)
+                _drop_dropped_system(probe)
                 match_catalog(probe, catalog, threshold)
                 fields = {k: getattr(probe, k) for k in (
                     "system", "group", "faction", "subfaction", "klass",

sorter.py

@@ -27,6 +27,7 @@ import sys
 from pathlib import Path
 
 from dlphelper.sorter import (
+    DROPPED_SYSTEMS,
     format_plan,
     load_catalog,
     sort_plan,
@@ -57,8 +58,20 @@ def main() -> None:
                              "sort whole extraction directories as units")
     parser.add_argument("--json", action="store_true",
                         help="print the plan as JSON instead of text")
+    parser.add_argument("--drop-system", action="append", default=[],
+                        metavar="NAME",
+                        help="never place into this game system "
+                             "(default-dropped: Middle-Earth)")
+    parser.add_argument("--keep-system", action="append", default=[],
+                        metavar="NAME",
+                        help="re-enable a default-dropped game system")
     args = parser.parse_args()
 
+    dropped = frozenset(DROPPED_SYSTEMS - set(args.keep_system)
+                        | set(args.drop_system))
+    if args.keep_system or args.drop_system:
+        log.info("System filter adjusted: dropping %s", ", ".join(sorted(dropped)) or "nothing")
+
     logging.basicConfig(level=logging.INFO,
                         format="%(asctime)s %(levelname)-7s %(message)s",
                         datefmt="%H:%M:%S")
@@ -69,7 +82,9 @@ def main() -> None:
         # hidden trees like .broken/ must never reach the plan
         if not any(part.startswith(".") for part in p.relative_to(source).parts)
     )
-    catalog = load_catalog(args.catalog)
+    catalog = load_catalog(args.catalog, dropped=dropped)
+    if dropped:
+        log.info("Dropped systems: %s", ", ".join(sorted(dropped)))
     audit = args.audit or (args.library / "sorter_applied.jsonl")
 
     provenance = None

tests/test_sorter_units.py

@@ -30,6 +30,9 @@ CATALOG = {
         {"name": "Dark Reapers", "system": "Warhammer 40,000",
          "group": "Xenos Armies", "faction": "Aeldari",
          "class": "Aspect Warriors", "aliases": []},
+        {"name": "Mordor Orcs", "system": "Middle-Earth",
+         "group": "The Lord of the Rings™ - Evil", "faction": "Mordor",
+         "class": "Warriors", "aliases": []},
     ]
 }
 
@@ -178,6 +181,27 @@ def test_tokenless_needs_bonus():
     shutil.rmtree(root); shutil.rmtree(library)
 
 
+def test_dropped_system_mordor():
+    """Middle-earth is dropped: a 'Mordor' pack lands in _unsorted, and
+    its inner dirs cannot adopt the (filtered) Mordor Orcs entry."""
+    root = Path(tempfile.mkdtemp(prefix="sorter_test_src_"))
+    write(root / "Mordor pack" / "boy.stl")
+    library = make_lib()
+    items = sort_plan([root / "Mordor pack" / "boy.stl"], library,
+                      get_catalog(), provenance={"Mordor pack": "Mordor pack.zip"},
+                      root=root)
+    it = items[0]
+    assert "Middle-Earth" not in str(it.target), it.target
+    assert "_unsorted/Mordor pack" in str(it.target).replace("\\", "/")
+    shutil.rmtree(root); shutil.rmtree(library)
+
+
+def test_load_catalog_filters_dropped():
+    cat = load_catalog_path()
+    assert not any(e["system"] == "Middle-Earth" for e in cat)
+    assert any(e["name"] == "Warp Spiders" for e in cat)
+
+
 def test_nonprov_rehome():
     """Loose noise file inside a named dir re-homes to the dir's unit."""
     root = Path(tempfile.mkdtemp(prefix="sorter_test_src_"))
@@ -222,7 +246,8 @@ def main():
     catalog = load_catalog_path()
     for fn in (test_prov_dir_rehome, test_prov_weak_tier, test_nonprov_rehome,
                test_cross_system_needs_token_containment, test_apply_weak_doc,
-               test_weak_needs_shared_token, test_tokenless_needs_bonus):
+               test_weak_needs_shared_token, test_tokenless_needs_bonus,
+               test_dropped_system_mordor, test_load_catalog_filters_dropped):
         fn()
         print("OK", fn.__name__)