sorter: cross-system name adoption needs token containment; sys-retry keeps the higher score

dlphelper/sorter.py

@@ -273,6 +273,26 @@ def _is_junk_dir(name: str) -> bool:
     return " " not in clean and clean in GENERIC_TOKENS
 
 
+def _token_contained(a_clean: str, b_clean: str) -> bool:
+    """True when one name's (stemmed) tokens all appear in the other's.
+
+    Guarded like _containment_boost: the smaller side must not consist of
+    generic army words alone. Used to gate cross-system adoption, where
+    plain char-ratio lies ("aeldari rangers" vs "dwarf rangers").
+    """
+    ta = set(_stem_tokens(a_clean))
+    tb = set(_stem_tokens(b_clean))
+    if not ta or not tb:
+        return False
+    small = ta if ta <= tb else (tb if tb <= ta else None)
+    if small is None:
+        return False
+    if len(small) == 1:
+        # a lone generic word ("guard", "orc") must not carry a match
+        return next(iter(small)) not in GENERIC_TOKENS
+    return True
+
+
 def _apply_entry(
     item: SortItem, entry: dict, score: float, kind: str, confidence: str
 ) -> None:
@@ -324,7 +344,10 @@ def _match_name(
     result = None
     if system == "Unknown":
         full = _best_in_pool(shim, catalog, use_inner=False)
-        if full and full[0] >= CROSS_SYSTEM_MIN:
+        if full and full[0] >= CROSS_SYSTEM_MIN and (
+                _token_contained(clean, full[1]["_clean"])
+                or any(_token_contained(clean, a)
+                       for a in full[1]["_clean_aliases"])):
             result = (full[0], full[1], False)
     else:
         pool = [e for e in catalog if e.get("system") == system]
@@ -389,6 +412,7 @@ def _rehome_from_dirs(
     catalog: list[dict],
     threshold: float,
     cache: dict,
+    prefer_dirs_on_tie: bool = False,
 ):
     """Let matching directory names re-home an item ("deepest match wins").
 
@@ -408,10 +432,11 @@ def _rehome_from_dirs(
     score, entry, weak, real_dir = m
     if own and score < own_score:
         return None
-    if own and score == own_score and not weak \
-            and item.match_kind == "catalog" and item.confidence == "normal":
-        # exact tie with a confident file match - the file name is the
-        # deeper side in non-provenance mode, keep it
+    if own and score == own_score:
+        # exact tie: the deeper side wins, which is the dir only in
+        # provenance mode (pack name sits above the inner dirs); in manual
+        # trees the file name itself is the deepest side - keep it
+        if not (prefer_dirs_on_tie and not weak):
             return None
     kind = "weak" if weak else "catalog"
     _apply_entry(item, entry, score, kind, "low" if weak else "normal")
@@ -464,15 +489,20 @@ def match_catalog(item: SortItem, catalog: list[dict], threshold: float) -> None
         if best is None or best[0] < threshold:
             # The keyword tables can lag behind GW's current categorization
             # (e.g. units moved between factions); retry within the same
-            # system before giving up - never across systems.
+            # system before giving up - never across systems. Keep the
+            # higher of the two scores (a retry must not downgrade a
+            # weak-eligible same-faction hit).
             sys_pool = [e for e in catalog if e.get("system") == item.system]
-            best = _best_in_pool(item, sys_pool, use_inner=False)
+            sys_best = _best_in_pool(item, sys_pool, use_inner=False)
+            if sys_best is not None and (best is None or sys_best[0] > best[0]):
+                best = sys_best
             if best is None or best[0] < threshold:
-                best = _best_in_pool(item, sys_pool, use_inner=True)
+                sys_best = _best_in_pool(item, sys_pool, use_inner=True)
+                if sys_best is not None and (best is None or sys_best[0] > best[0]):
+                    best = sys_best
             # a retry hit from a *different* faction is only trusted when
             # it is strong; weak inner-name votes must not re-home the file
-            if best is not None and best[0] >= threshold \
-                    and best[1].get("faction") != item.faction \
+            if best is not None and best[1].get("faction") != item.faction \
                     and best[0] < CROSS_SYSTEM_MIN:
                 best = None
     elif system_known:
@@ -670,7 +700,7 @@ def sort_plan(
                     d = (root / dest_key).joinpath(*inner_rel.parts[:i + 1])
                     dirs.append((d, inner_rel.parts[i]))
                 winner = _rehome_from_dirs(item, dirs, catalog, threshold,
-                                           dir_cache)
+                                           dir_cache, prefer_dirs_on_tie=True)
                 if winner is not None:
                     inner_rel = f.parent.relative_to(winner)
         item.target = build_target(item, library, inner_rel=inner_rel)

tests/test_sorter_units.py

@@ -84,19 +84,43 @@ def test_prov_dir_rehome():
 def test_prov_weak_tier():
     """Below-threshold dir hit is accepted as low confidence."""
     root = Path(tempfile.mkdtemp(prefix="sorter_test_src_"))
-    pack = root / "Multi Pack"
-    # 'guardian squad extras' scores below 1.0 but high vs 'Guardian Squad'
-    write(pack / "guardian squad extras" / "g.stl")
+    # pack name carries a keyword so the probe resolves system+faction -
+    # only then does the same-faction weak tier apply
+    pack = root / "Aeldari Pack"
+    write(pack / "guardian weapon team" / "g.stl")  # 0.59 vs Guardian Squad
     library = make_lib()
-    items = prov_items(root, library)
+    (root / "extract_manifest.jsonl").write_text(json.dumps({
+        "archive": str(root / "Aeldari Pack.zip"),
+        "dest": "Aeldari Pack", "unit": "Aeldari Pack.zip",
+    }), encoding="utf-8")
+    files = sorted((root / "Aeldari Pack").rglob("*"))
+    items = sort_plan(files, library, get_catalog(), provenance={
+        "Aeldari Pack": "Aeldari Pack.zip"}, root=root)
     it = next(i for i in items if i.name == "g.stl")
-    assert it.match_kind in ("weak", "catalog"), it.match_kind
-    if it.match_kind == "weak":
+    assert it.match_kind == "weak", (it.match_kind, it.confidence)
     assert it.confidence == "low"
     assert "Guardian" in str(it.target)
     shutil.rmtree(root); shutil.rmtree(library)
 
 
+def test_cross_system_needs_token_containment():
+    """'Aeldari Rangers' must not adopt Middle-earth 'Dwarf Rangers'."""
+    root = Path(tempfile.mkdtemp(prefix="sorter_test_src_"))
+    pack = root / "MQVxKzh"  # hash name: system stays unknown
+    write(pack / "Aeldari Rangers" / "Luminary - Arms -  (1).stl")
+    write(pack / "14 Warp spiders" / "sp.stl")
+    library = make_lib()
+    files = sorted((root / "MQVxKzh").rglob("*"))
+    items = sort_plan(files, library, get_catalog(), provenance={
+        "MQVxKzh": "MQVxKzh.zip"}, root=root)
+    by_name = {i.name: i for i in items}
+    lum = by_name["Luminary - Arms -  (1).stl"]
+    assert "Middle-Earth" not in str(lum.target), lum.target
+    spider = by_name["sp.stl"]
+    assert "Warp Spiders" in str(spider.target), spider.target
+    shutil.rmtree(root); shutil.rmtree(library)
+
+
 def test_nonprov_rehome():
     """Loose noise file inside a named dir re-homes to the dir's unit."""
     root = Path(tempfile.mkdtemp(prefix="sorter_test_src_"))
@@ -140,7 +164,7 @@ def load_catalog_path():
 def main():
     catalog = load_catalog_path()
     for fn in (test_prov_dir_rehome, test_prov_weak_tier, test_nonprov_rehome,
-               test_apply_weak_doc):
+               test_cross_system_needs_token_containment, test_apply_weak_doc):
         fn()
         print("OK", fn.__name__)