From a995937c4bfb30dbb32b174c67d96be84efe711e Mon Sep 17 00:00:00 2001 From: Ben_Kosytorz Date: Tue, 6 Oct 2026 23:44:00 +0200 Subject: [PATCH] sorter: cross-system name adoption needs token containment; sys-retry keeps the higher score --- dlphelper/sorter.py | 54 +++++++++++++++++++++++++++++--------- tests/test_sorter_units.py | 40 ++++++++++++++++++++++------ 2 files changed, 74 insertions(+), 20 deletions(-) diff --git a/dlphelper/sorter.py b/dlphelper/sorter.py index 0931d8a..3f8db05 100644 --- a/dlphelper/sorter.py +++ b/dlphelper/sorter.py @@ -273,6 +273,26 @@ def _is_junk_dir(name: str) -> bool: return " " not in clean and clean in GENERIC_TOKENS +def _token_contained(a_clean: str, b_clean: str) -> bool: + """True when one name's (stemmed) tokens all appear in the other's. + + Guarded like _containment_boost: the smaller side must not consist of + generic army words alone. Used to gate cross-system adoption, where + plain char-ratio lies ("aeldari rangers" vs "dwarf rangers"). + """ + ta = set(_stem_tokens(a_clean)) + tb = set(_stem_tokens(b_clean)) + if not ta or not tb: + return False + small = ta if ta <= tb else (tb if tb <= ta else None) + if small is None: + return False + if len(small) == 1: + # a lone generic word ("guard", "orc") must not carry a match + return next(iter(small)) not in GENERIC_TOKENS + return True + + def _apply_entry( item: SortItem, entry: dict, score: float, kind: str, confidence: str ) -> None: @@ -324,7 +344,10 @@ def _match_name( result = None if system == "Unknown": full = _best_in_pool(shim, catalog, use_inner=False) - if full and full[0] >= CROSS_SYSTEM_MIN: + if full and full[0] >= CROSS_SYSTEM_MIN and ( + _token_contained(clean, full[1]["_clean"]) + or any(_token_contained(clean, a) + for a in full[1]["_clean_aliases"])): result = (full[0], full[1], False) else: pool = [e for e in catalog if e.get("system") == system] @@ -389,6 +412,7 @@ def _rehome_from_dirs( catalog: list[dict], threshold: float, cache: dict, + prefer_dirs_on_tie: bool = False, ): """Let matching directory names re-home an item ("deepest match wins"). @@ -408,11 +432,12 @@ def _rehome_from_dirs( score, entry, weak, real_dir = m if own and score < own_score: return None - if own and score == own_score and not weak \ - and item.match_kind == "catalog" and item.confidence == "normal": - # exact tie with a confident file match - the file name is the - # deeper side in non-provenance mode, keep it - return None + if own and score == own_score: + # exact tie: the deeper side wins, which is the dir only in + # provenance mode (pack name sits above the inner dirs); in manual + # trees the file name itself is the deepest side - keep it + if not (prefer_dirs_on_tie and not weak): + return None kind = "weak" if weak else "catalog" _apply_entry(item, entry, score, kind, "low" if weak else "normal") return real_dir @@ -464,15 +489,20 @@ def match_catalog(item: SortItem, catalog: list[dict], threshold: float) -> None if best is None or best[0] < threshold: # The keyword tables can lag behind GW's current categorization # (e.g. units moved between factions); retry within the same - # system before giving up - never across systems. + # system before giving up - never across systems. Keep the + # higher of the two scores (a retry must not downgrade a + # weak-eligible same-faction hit). sys_pool = [e for e in catalog if e.get("system") == item.system] - best = _best_in_pool(item, sys_pool, use_inner=False) + sys_best = _best_in_pool(item, sys_pool, use_inner=False) + if sys_best is not None and (best is None or sys_best[0] > best[0]): + best = sys_best if best is None or best[0] < threshold: - best = _best_in_pool(item, sys_pool, use_inner=True) + sys_best = _best_in_pool(item, sys_pool, use_inner=True) + if sys_best is not None and (best is None or sys_best[0] > best[0]): + best = sys_best # a retry hit from a *different* faction is only trusted when # it is strong; weak inner-name votes must not re-home the file - if best is not None and best[0] >= threshold \ - and best[1].get("faction") != item.faction \ + if best is not None and best[1].get("faction") != item.faction \ and best[0] < CROSS_SYSTEM_MIN: best = None elif system_known: @@ -670,7 +700,7 @@ def sort_plan( d = (root / dest_key).joinpath(*inner_rel.parts[:i + 1]) dirs.append((d, inner_rel.parts[i])) winner = _rehome_from_dirs(item, dirs, catalog, threshold, - dir_cache) + dir_cache, prefer_dirs_on_tie=True) if winner is not None: inner_rel = f.parent.relative_to(winner) item.target = build_target(item, library, inner_rel=inner_rel) diff --git a/tests/test_sorter_units.py b/tests/test_sorter_units.py index 6c04771..dad2a20 100644 --- a/tests/test_sorter_units.py +++ b/tests/test_sorter_units.py @@ -84,19 +84,43 @@ def test_prov_dir_rehome(): def test_prov_weak_tier(): """Below-threshold dir hit is accepted as low confidence.""" root = Path(tempfile.mkdtemp(prefix="sorter_test_src_")) - pack = root / "Multi Pack" - # 'guardian squad extras' scores below 1.0 but high vs 'Guardian Squad' - write(pack / "guardian squad extras" / "g.stl") + # pack name carries a keyword so the probe resolves system+faction - + # only then does the same-faction weak tier apply + pack = root / "Aeldari Pack" + write(pack / "guardian weapon team" / "g.stl") # 0.59 vs Guardian Squad library = make_lib() - items = prov_items(root, library) + (root / "extract_manifest.jsonl").write_text(json.dumps({ + "archive": str(root / "Aeldari Pack.zip"), + "dest": "Aeldari Pack", "unit": "Aeldari Pack.zip", + }), encoding="utf-8") + files = sorted((root / "Aeldari Pack").rglob("*")) + items = sort_plan(files, library, get_catalog(), provenance={ + "Aeldari Pack": "Aeldari Pack.zip"}, root=root) it = next(i for i in items if i.name == "g.stl") - assert it.match_kind in ("weak", "catalog"), it.match_kind - if it.match_kind == "weak": - assert it.confidence == "low" + assert it.match_kind == "weak", (it.match_kind, it.confidence) + assert it.confidence == "low" assert "Guardian" in str(it.target) shutil.rmtree(root); shutil.rmtree(library) +def test_cross_system_needs_token_containment(): + """'Aeldari Rangers' must not adopt Middle-earth 'Dwarf Rangers'.""" + root = Path(tempfile.mkdtemp(prefix="sorter_test_src_")) + pack = root / "MQVxKzh" # hash name: system stays unknown + write(pack / "Aeldari Rangers" / "Luminary - Arms - (1).stl") + write(pack / "14 Warp spiders" / "sp.stl") + library = make_lib() + files = sorted((root / "MQVxKzh").rglob("*")) + items = sort_plan(files, library, get_catalog(), provenance={ + "MQVxKzh": "MQVxKzh.zip"}, root=root) + by_name = {i.name: i for i in items} + lum = by_name["Luminary - Arms - (1).stl"] + assert "Middle-Earth" not in str(lum.target), lum.target + spider = by_name["sp.stl"] + assert "Warp Spiders" in str(spider.target), spider.target + shutil.rmtree(root); shutil.rmtree(library) + + def test_nonprov_rehome(): """Loose noise file inside a named dir re-homes to the dir's unit.""" root = Path(tempfile.mkdtemp(prefix="sorter_test_src_")) @@ -140,7 +164,7 @@ def load_catalog_path(): def main(): catalog = load_catalog_path() for fn in (test_prov_dir_rehome, test_prov_weak_tier, test_nonprov_rehome, - test_apply_weak_doc): + test_cross_system_needs_token_containment, test_apply_weak_doc): fn() print("OK", fn.__name__)