From 0439071f702ac18669cd175b945d157238f38982 Mon Sep 17 00:00:00 2001 From: Ben_Kosytorz Date: Wed, 7 Oct 2026 08:03:05 +0200 Subject: [PATCH] sorter: drop non-warhammer game systems (default: Middle-Earth) The collection is warhammer-only; everything else stays in _unsorted even when names match the catalog. The catalog snapshot keeps all systems - the filter is a sorter default (DROPPED_SYSTEMS, applied in load_catalog and after keyword classification), adjustable with --drop-system/--keep-system. --- dlphelper/sorter.py | 28 ++++++++++++++++++++++++++-- sorter.py | 17 ++++++++++++++++- tests/test_sorter_units.py | 27 ++++++++++++++++++++++++++- 3 files changed, 68 insertions(+), 4 deletions(-) diff --git a/dlphelper/sorter.py b/dlphelper/sorter.py index 9e1ca6d..f20e04c 100644 --- a/dlphelper/sorter.py +++ b/dlphelper/sorter.py @@ -32,9 +32,15 @@ UNSORTED = "_unsorted" UNCLASSIFIED_CLASS = "Unclassified" # A file whose system the keyword tables could not identify only gets a # catalog unit when the fuzzy hit is strong - otherwise it would wander -# into other games' sections (Middle-earth, Blood Bowl, ...) on a guess. +# into other games' sections (Blood Bowl, ...) on a guess. CROSS_SYSTEM_MIN = 0.75 +# Game systems whose sections the sorter must never place into. The +# collection is warhammer-only: everything else (Middle-earth proxies, +# ...) stays in _unsorted even when a name matches the catalog. The +# catalog snapshot itself keeps all systems; the filter lives here. +DROPPED_SYSTEMS = frozenset({"Middle-Earth"}) + # Weak acceptance tier: when nothing reached the normal threshold, a hit # at least this strong is accepted but flagged low confidence ("[w]" in # the plan, confidence field in the audit, listed in @@ -178,9 +184,14 @@ def classify_item(item: SortItem) -> None: # faction group layer (GW naming, shared with warhammer_data) -def load_catalog(path: Path) -> list[dict]: +def load_catalog(path: Path, + dropped: frozenset[str] | None = DROPPED_SYSTEMS) -> list[dict]: + """Load catalog entries, dropping non-collected game systems by default + (pass dropped=frozenset() to keep everything).""" data = json.loads(path.read_text(encoding="utf-8")) entries = data["entries"] if isinstance(data, dict) else data + if dropped: + entries = [e for e in entries if e.get("system") not in dropped] for e in entries: e["_clean"] = _clean_name(e["name"]) e["_clean_aliases"] = [_clean_name(a) for a in e.get("aliases", [])] @@ -672,6 +683,17 @@ def _display_name(unit: str) -> str: # plan / apply # --------------------------------------------------------------------------- +def _drop_dropped_system(item: SortItem) -> None: + """Dropped systems (see DROPPED_SYSTEMS) count as unclassified: the + collection does not cover them, so their names must not place files + into those sections. Called after keyword classification, before + matching - the item can still be re-homed by other signals.""" + if item.system in DROPPED_SYSTEMS: + item.system = "Unknown" + item.group = item.faction = UNSORTED + item.subfaction = None + + def sort_plan( files: list[Path], library: Path, @@ -716,6 +738,7 @@ def sort_plan( if unit_name is None: item = inspect(f) classify_item(item) + _drop_dropped_system(item) match_catalog(item, catalog, threshold) if root is not None: # manual trees: let the real ancestor dir names compete @@ -741,6 +764,7 @@ def sort_plan( clean=_clean_name(unit_name), inner_names=listing, ) classify_item(probe) + _drop_dropped_system(probe) match_catalog(probe, catalog, threshold) fields = {k: getattr(probe, k) for k in ( "system", "group", "faction", "subfaction", "klass", diff --git a/sorter.py b/sorter.py index 365ae57..afdef1e 100644 --- a/sorter.py +++ b/sorter.py @@ -27,6 +27,7 @@ import sys from pathlib import Path from dlphelper.sorter import ( + DROPPED_SYSTEMS, format_plan, load_catalog, sort_plan, @@ -57,8 +58,20 @@ def main() -> None: "sort whole extraction directories as units") parser.add_argument("--json", action="store_true", help="print the plan as JSON instead of text") + parser.add_argument("--drop-system", action="append", default=[], + metavar="NAME", + help="never place into this game system " + "(default-dropped: Middle-Earth)") + parser.add_argument("--keep-system", action="append", default=[], + metavar="NAME", + help="re-enable a default-dropped game system") args = parser.parse_args() + dropped = frozenset(DROPPED_SYSTEMS - set(args.keep_system) + | set(args.drop_system)) + if args.keep_system or args.drop_system: + log.info("System filter adjusted: dropping %s", ", ".join(sorted(dropped)) or "nothing") + logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)-7s %(message)s", datefmt="%H:%M:%S") @@ -69,7 +82,9 @@ def main() -> None: # hidden trees like .broken/ must never reach the plan if not any(part.startswith(".") for part in p.relative_to(source).parts) ) - catalog = load_catalog(args.catalog) + catalog = load_catalog(args.catalog, dropped=dropped) + if dropped: + log.info("Dropped systems: %s", ", ".join(sorted(dropped))) audit = args.audit or (args.library / "sorter_applied.jsonl") provenance = None diff --git a/tests/test_sorter_units.py b/tests/test_sorter_units.py index 3b085f5..8f10529 100644 --- a/tests/test_sorter_units.py +++ b/tests/test_sorter_units.py @@ -30,6 +30,9 @@ CATALOG = { {"name": "Dark Reapers", "system": "Warhammer 40,000", "group": "Xenos Armies", "faction": "Aeldari", "class": "Aspect Warriors", "aliases": []}, + {"name": "Mordor Orcs", "system": "Middle-Earth", + "group": "The Lord of the Rings™ - Evil", "faction": "Mordor", + "class": "Warriors", "aliases": []}, ] } @@ -178,6 +181,27 @@ def test_tokenless_needs_bonus(): shutil.rmtree(root); shutil.rmtree(library) +def test_dropped_system_mordor(): + """Middle-earth is dropped: a 'Mordor' pack lands in _unsorted, and + its inner dirs cannot adopt the (filtered) Mordor Orcs entry.""" + root = Path(tempfile.mkdtemp(prefix="sorter_test_src_")) + write(root / "Mordor pack" / "boy.stl") + library = make_lib() + items = sort_plan([root / "Mordor pack" / "boy.stl"], library, + get_catalog(), provenance={"Mordor pack": "Mordor pack.zip"}, + root=root) + it = items[0] + assert "Middle-Earth" not in str(it.target), it.target + assert "_unsorted/Mordor pack" in str(it.target).replace("\\", "/") + shutil.rmtree(root); shutil.rmtree(library) + + +def test_load_catalog_filters_dropped(): + cat = load_catalog_path() + assert not any(e["system"] == "Middle-Earth" for e in cat) + assert any(e["name"] == "Warp Spiders" for e in cat) + + def test_nonprov_rehome(): """Loose noise file inside a named dir re-homes to the dir's unit.""" root = Path(tempfile.mkdtemp(prefix="sorter_test_src_")) @@ -222,7 +246,8 @@ def main(): catalog = load_catalog_path() for fn in (test_prov_dir_rehome, test_prov_weak_tier, test_nonprov_rehome, test_cross_system_needs_token_containment, test_apply_weak_doc, - test_weak_needs_shared_token, test_tokenless_needs_bonus): + test_weak_needs_shared_token, test_tokenless_needs_bonus, + test_dropped_system_mordor, test_load_catalog_filters_dropped): fn() print("OK", fn.__name__)