sorter: probe votes on full inner paths (dir names count); no single-token containment boost for inner votes; catalog class wins on unit adoption; confidence copied from probe

dlphelper/sorter.py

@@ -154,14 +154,15 @@ def classify_item(item: SortItem) -> None:
     if hit:
         item.system, item.faction = hit
     item.klass = _match_class(item.clean) or (
-        _match_class(_clean_name(" ".join(item.inner_names))) if item.inner_names else None
+        _match_class(_clean_inner(" ".join(item.inner_names)))
+        if item.inner_names else None
     )
 
     if item.faction == UNSORTED and item.inner_names:
         # hash-named archives: vote on the inner names' classification
         votes: dict[tuple[str, str], int] = {}
         for inner in item.inner_names:
-            ihit = _match_faction(_clean_name(inner))
+            ihit = _match_faction(_clean_inner(inner))
             if ihit:
                 votes[ihit] = votes.get(ihit, 0) + 1
         if votes:
@@ -198,20 +199,24 @@ def load_catalog(path: Path,
     return entries
 
 
-def _best_ratio(clean: str, candidates: list[str]) -> float:
+def _best_ratio(
+    clean: str, candidates: list[str], allow_single: bool = True
+) -> float:
     """Best sequence ratio, pruned with cheap upper-bound checks.
 
     lru_cached because the same (name, entry) pair is scored over and
     over across thousands of files.
     """
-    return _best_ratio_cached(clean, tuple(candidates))
+    return _best_ratio_cached(clean, tuple(candidates), allow_single)
 
 
 @lru_cache(maxsize=None)
-def _best_ratio_cached(clean: str, candidates: tuple[str, ...]) -> float:
+def _best_ratio_cached(
+    clean: str, candidates: tuple[str, ...], allow_single: bool = True
+) -> float:
     best = 0.0
     toks = set(_stem_tokens(clean))
-    best = _containment_boost(best, toks, candidates)
+    best = _containment_boost(best, toks, candidates, allow_single)
     for c in candidates:
         sm = difflib.SequenceMatcher(None, clean, c)
         if sm.real_quick_ratio() < best or sm.quick_ratio() < best:
@@ -230,13 +235,18 @@ def _stem_tokens(clean: str) -> list[str]:
             for t in clean.split()]
 
 
-def _containment_boost(best: float, toks: set[str], candidates: tuple[str, ...]) -> float:
+def _containment_boost(
+    best: float, toks: set[str], candidates: tuple[str, ...],
+    allow_single: bool = True,
+) -> float:
     """Token-containment bonus: a name whose tokens all appear in the
     other (or vice versa) matches strongly even when string ratio is low
     ("telemon" vs "Telemon Heavy Dreadnought"). Guarded: needs >= 2
     tokens, or a single rare one (>= 7 chars), and the smaller side must
     not consist of generic army words ("chaos lord" is contained in half
-    the catalog) so "head" or "guard" never trigger it."""
+    the catalog) so "head" or "guard" never trigger it. Inner-name votes
+    pass allow_single=False: one short inner file name ("lieutenant")
+    must not claim a whole pack."""
     if not toks:
         return best
     for c in candidates:
@@ -246,7 +256,7 @@ def _containment_boost(best: float, toks: set[str], candidates: tuple[str, ...])
         small = toks if toks <= ctoks else (ctoks if ctoks <= toks else None)
         if small is None or any(t in GENERIC_TOKENS for t in small):
             continue
-        if len(small) >= 2 or len(next(iter(small))) >= 7:
+        if len(small) >= 2 or (allow_single and len(next(iter(small))) >= 7):
             if best < 0.9:
                 best = 0.9
             break
@@ -278,9 +288,18 @@ _JUNK_DIRS = frozenset({
     "model", "output", "exports", "release", "releases", "license",
     "readme", "sources", "docs", "textures", "new", "old", "final",
     "fixed", "print", "prints", "plates", "plate", "assembly", "test",
+    "characters", "character",
 })  # noqa: C405
 
 
+def _clean_inner(name: str) -> str:
+    """Clean an archive-internal (relative) path: dir names join the vote
+    ("2E Catachany fellaz/Catachan/Jungle Fighters/x.stl" -> "2e catachany
+    fellaz catachan jungle fighters x") - the file stem alone loses the
+    archive's own folder structure."""
+    return _clean_name(name.replace("/", " ").replace("\\", " "))
+
+
 def _is_junk_dir(name: str) -> bool:
     """True for directory names that must never be match candidates."""
     clean = _clean_name(name)
@@ -309,6 +328,14 @@ def _token_contained(a_clean: str, b_clean: str) -> bool:
     return True
 
 
+def _contained_somewhere(item: SortItem, entry: dict) -> bool:
+    """Token containment between the entry (or an alias) and the outer
+    name or any inner (relative-path) name."""
+    cands = [entry["_clean"], *entry["_clean_aliases"]]
+    probes = [item.clean, *[_clean_inner(n) for n in item.inner_names[:10]]]
+    return any(_token_contained(p, c) for p in probes for c in cands)
+
+
 def _shares_rare_token(a_clean: str, b_clean: str) -> bool:
     """True when two cleaned names share at least one non-generic token
     (>= 3 chars). Gates the weak tier: a sub-threshold hit with zero
@@ -520,9 +547,10 @@ def _best_in_pool(
         cands = [entry["_clean"], *entry["_clean_aliases"]]
         score = _best_ratio(item.clean, cands)
         if use_inner and item.inner_names:
-            inner_clean = [_clean_name(n) for n in item.inner_names[:10]]
+            inner_clean = [_clean_inner(n) for n in item.inner_names[:10]]
             for c in inner_clean:
-                score = max(score, _best_ratio(c, cands) * 0.95)  # inner slightly discounted
+                # no single-token containment boost for inner votes
+                score = max(score, _best_ratio(c, cands, allow_single=False) * 0.95)
         if best is None or score > best[0]:
             best = (score, entry)
     return best
@@ -565,9 +593,11 @@ def match_catalog(item: SortItem, catalog: list[dict], threshold: float) -> None
                 if sys_best is not None and (best is None or sys_best[0] > best[0]):
                     best = sys_best
             # a retry hit from a *different* faction is only trusted when
-            # it is strong; weak inner-name votes must not re-home the file
+            # it is strong AND anchored by token containment; weak
+            # inner-name votes must not re-home the file
             if best is not None and best[1].get("faction") != item.faction \
-                    and best[0] < CROSS_SYSTEM_MIN:
+                    and (best[0] < CROSS_SYSTEM_MIN
+                         or not _contained_somewhere(item, best[1])):
                 best = None
     elif system_known:
         sys_pool = [e for e in catalog if e.get("system") == item.system]
@@ -576,13 +606,16 @@ def match_catalog(item: SortItem, catalog: list[dict], threshold: float) -> None
             best = _best_in_pool(item, sys_pool, use_inner=True)
     else:
         # Neither system nor faction known: whole catalog, but only a
-        # strong hit (CROSS_SYSTEM_MIN) may adopt another game's section;
+        # strong hit (CROSS_SYSTEM_MIN) with token containment may adopt
-        # weaker guesses stay in _unsorted.
+        # another game's section; weaker or unanchored guesses stay in
+        # _unsorted.
         best = _best_in_pool(item, catalog, use_inner=False)
         if best is None or best[0] < threshold:
             best = _best_in_pool(item, catalog, use_inner=True)
         if best is not None and best[0] < CROSS_SYSTEM_MIN:
             best = None
+        elif best is not None and not _contained_somewhere(item, best[1]):
+            best = None
     if best and best[0] >= threshold:
         entry = best[1]
         item.unit = entry["name"]
@@ -596,7 +629,10 @@ def match_catalog(item: SortItem, catalog: list[dict], threshold: float) -> None
         # not *narrow* it: "Battleline" is GW's bucket for core infantry,
         # a keyword-derived Hero/Elite is more specific for a proxy file.
         catalog_klass = entry.get("class")
-        if catalog_klass and item.klass in (None, "", UNCLASSIFIED_CLASS):
+        if catalog_klass:
+            # a matched catalog unit's class layer is authoritative for
+            # the whole subtree - a keyword-voted class ("warlock" seen
+            # somewhere -> Hero) must not survive next to a precise unit
             item.klass = catalog_klass
         item.match_score = best[0]
         item.match_kind = "catalog"
@@ -627,7 +663,7 @@ def match_catalog(item: SortItem, catalog: list[dict], threshold: float) -> None
             item.group = entry["group"]
     elif best and best[0] >= WEAK_MIN and (
             _shares_rare_token(item.clean, best[1]["_clean"])
-            or any(_shares_rare_token(_clean_name(n), best[1]["_clean"])
+            or any(_shares_rare_token(_clean_inner(n), best[1]["_clean"])
                    for n in item.inner_names[:10])):
         # Weak tier: a same-system hit below the threshold with at least
         # one shared word, accepted only when nothing stronger exists.
@@ -768,7 +804,7 @@ def sort_plan(
                 match_catalog(probe, catalog, threshold)
                 fields = {k: getattr(probe, k) for k in (
                     "system", "group", "faction", "subfaction", "klass",
-                    "unit", "match_score", "match_kind",
+                    "unit", "match_score", "match_kind", "confidence",
                 )}
                 unit_cache[dest_key] = fields
             item = inspect(f)

tests/test_sorter_units.py

@@ -83,10 +83,11 @@ def test_prov_dir_rehome():
     assert "Fly" in str(tgt["spider_body.stl"]), tgt["spider_body.stl"]
     assert "Hero" not in str(tgt["spider_body.stl"])
     assert "files" in str(tgt["spider_arm.stl"])
-    # junk-dir file stays under the pack unit folder
+    # junk-dir file rides along with the pack's adopted unit
-    assert "_unsorted/Multi Pack/HEADS" in str(tgt["head_a.stl"]).replace("\\", "/")
+    assert "Warp Spiders" in str(tgt["head_a.stl"])
-    # unmatched subtree keeps the pack unit
+    assert "HEADS" in str(tgt["head_a.stl"])
-    assert "Multi Pack" in str(tgt["t.stl"])
+    # unmatched subtree keeps the pack unit folder
+    assert "Warp Spiders" in str(tgt["t.stl"]) and "Retro GrimDark Thing" in str(tgt["t.stl"])
     shutil.rmtree(root); shutil.rmtree(library)
 
 
@@ -151,7 +152,8 @@ def test_weak_needs_shared_token():
         "Aeldari Pack": "Aeldari Pack.zip"}, root=root)
     by_name = {i.name: i for i in items}
     f = by_name["f.stl"]
-    assert f.match_kind == "implicit", (f.match_kind, f.target)
+    # both files ride along with the pack-level weak adoption
+    assert f.match_kind == "weak" and f.confidence == "low", (f.match_kind, f.target)
     assert "frontline squad" in str(f.target)
     g = by_name["g.stl"]
     assert g.match_kind == "weak" and g.confidence == "low"