diff --git a/dlphelper/sorter.py b/dlphelper/sorter.py index f20e04c..4832629 100644 --- a/dlphelper/sorter.py +++ b/dlphelper/sorter.py @@ -154,14 +154,15 @@ def classify_item(item: SortItem) -> None: if hit: item.system, item.faction = hit item.klass = _match_class(item.clean) or ( - _match_class(_clean_name(" ".join(item.inner_names))) if item.inner_names else None + _match_class(_clean_inner(" ".join(item.inner_names))) + if item.inner_names else None ) if item.faction == UNSORTED and item.inner_names: # hash-named archives: vote on the inner names' classification votes: dict[tuple[str, str], int] = {} for inner in item.inner_names: - ihit = _match_faction(_clean_name(inner)) + ihit = _match_faction(_clean_inner(inner)) if ihit: votes[ihit] = votes.get(ihit, 0) + 1 if votes: @@ -198,20 +199,24 @@ def load_catalog(path: Path, return entries -def _best_ratio(clean: str, candidates: list[str]) -> float: +def _best_ratio( + clean: str, candidates: list[str], allow_single: bool = True +) -> float: """Best sequence ratio, pruned with cheap upper-bound checks. lru_cached because the same (name, entry) pair is scored over and over across thousands of files. """ - return _best_ratio_cached(clean, tuple(candidates)) + return _best_ratio_cached(clean, tuple(candidates), allow_single) @lru_cache(maxsize=None) -def _best_ratio_cached(clean: str, candidates: tuple[str, ...]) -> float: +def _best_ratio_cached( + clean: str, candidates: tuple[str, ...], allow_single: bool = True +) -> float: best = 0.0 toks = set(_stem_tokens(clean)) - best = _containment_boost(best, toks, candidates) + best = _containment_boost(best, toks, candidates, allow_single) for c in candidates: sm = difflib.SequenceMatcher(None, clean, c) if sm.real_quick_ratio() < best or sm.quick_ratio() < best: @@ -230,13 +235,18 @@ def _stem_tokens(clean: str) -> list[str]: for t in clean.split()] -def _containment_boost(best: float, toks: set[str], candidates: tuple[str, ...]) -> float: +def _containment_boost( + best: float, toks: set[str], candidates: tuple[str, ...], + allow_single: bool = True, +) -> float: """Token-containment bonus: a name whose tokens all appear in the other (or vice versa) matches strongly even when string ratio is low ("telemon" vs "Telemon Heavy Dreadnought"). Guarded: needs >= 2 tokens, or a single rare one (>= 7 chars), and the smaller side must not consist of generic army words ("chaos lord" is contained in half - the catalog) so "head" or "guard" never trigger it.""" + the catalog) so "head" or "guard" never trigger it. Inner-name votes + pass allow_single=False: one short inner file name ("lieutenant") + must not claim a whole pack.""" if not toks: return best for c in candidates: @@ -246,7 +256,7 @@ def _containment_boost(best: float, toks: set[str], candidates: tuple[str, ...]) small = toks if toks <= ctoks else (ctoks if ctoks <= toks else None) if small is None or any(t in GENERIC_TOKENS for t in small): continue - if len(small) >= 2 or len(next(iter(small))) >= 7: + if len(small) >= 2 or (allow_single and len(next(iter(small))) >= 7): if best < 0.9: best = 0.9 break @@ -278,9 +288,18 @@ _JUNK_DIRS = frozenset({ "model", "output", "exports", "release", "releases", "license", "readme", "sources", "docs", "textures", "new", "old", "final", "fixed", "print", "prints", "plates", "plate", "assembly", "test", + "characters", "character", }) # noqa: C405 +def _clean_inner(name: str) -> str: + """Clean an archive-internal (relative) path: dir names join the vote + ("2E Catachany fellaz/Catachan/Jungle Fighters/x.stl" -> "2e catachany + fellaz catachan jungle fighters x") - the file stem alone loses the + archive's own folder structure.""" + return _clean_name(name.replace("/", " ").replace("\\", " ")) + + def _is_junk_dir(name: str) -> bool: """True for directory names that must never be match candidates.""" clean = _clean_name(name) @@ -309,6 +328,14 @@ def _token_contained(a_clean: str, b_clean: str) -> bool: return True +def _contained_somewhere(item: SortItem, entry: dict) -> bool: + """Token containment between the entry (or an alias) and the outer + name or any inner (relative-path) name.""" + cands = [entry["_clean"], *entry["_clean_aliases"]] + probes = [item.clean, *[_clean_inner(n) for n in item.inner_names[:10]]] + return any(_token_contained(p, c) for p in probes for c in cands) + + def _shares_rare_token(a_clean: str, b_clean: str) -> bool: """True when two cleaned names share at least one non-generic token (>= 3 chars). Gates the weak tier: a sub-threshold hit with zero @@ -520,9 +547,10 @@ def _best_in_pool( cands = [entry["_clean"], *entry["_clean_aliases"]] score = _best_ratio(item.clean, cands) if use_inner and item.inner_names: - inner_clean = [_clean_name(n) for n in item.inner_names[:10]] + inner_clean = [_clean_inner(n) for n in item.inner_names[:10]] for c in inner_clean: - score = max(score, _best_ratio(c, cands) * 0.95) # inner slightly discounted + # no single-token containment boost for inner votes + score = max(score, _best_ratio(c, cands, allow_single=False) * 0.95) if best is None or score > best[0]: best = (score, entry) return best @@ -565,9 +593,11 @@ def match_catalog(item: SortItem, catalog: list[dict], threshold: float) -> None if sys_best is not None and (best is None or sys_best[0] > best[0]): best = sys_best # a retry hit from a *different* faction is only trusted when - # it is strong; weak inner-name votes must not re-home the file + # it is strong AND anchored by token containment; weak + # inner-name votes must not re-home the file if best is not None and best[1].get("faction") != item.faction \ - and best[0] < CROSS_SYSTEM_MIN: + and (best[0] < CROSS_SYSTEM_MIN + or not _contained_somewhere(item, best[1])): best = None elif system_known: sys_pool = [e for e in catalog if e.get("system") == item.system] @@ -576,13 +606,16 @@ def match_catalog(item: SortItem, catalog: list[dict], threshold: float) -> None best = _best_in_pool(item, sys_pool, use_inner=True) else: # Neither system nor faction known: whole catalog, but only a - # strong hit (CROSS_SYSTEM_MIN) may adopt another game's section; - # weaker guesses stay in _unsorted. + # strong hit (CROSS_SYSTEM_MIN) with token containment may adopt + # another game's section; weaker or unanchored guesses stay in + # _unsorted. best = _best_in_pool(item, catalog, use_inner=False) if best is None or best[0] < threshold: best = _best_in_pool(item, catalog, use_inner=True) if best is not None and best[0] < CROSS_SYSTEM_MIN: best = None + elif best is not None and not _contained_somewhere(item, best[1]): + best = None if best and best[0] >= threshold: entry = best[1] item.unit = entry["name"] @@ -596,7 +629,10 @@ def match_catalog(item: SortItem, catalog: list[dict], threshold: float) -> None # not *narrow* it: "Battleline" is GW's bucket for core infantry, # a keyword-derived Hero/Elite is more specific for a proxy file. catalog_klass = entry.get("class") - if catalog_klass and item.klass in (None, "", UNCLASSIFIED_CLASS): + if catalog_klass: + # a matched catalog unit's class layer is authoritative for + # the whole subtree - a keyword-voted class ("warlock" seen + # somewhere -> Hero) must not survive next to a precise unit item.klass = catalog_klass item.match_score = best[0] item.match_kind = "catalog" @@ -627,7 +663,7 @@ def match_catalog(item: SortItem, catalog: list[dict], threshold: float) -> None item.group = entry["group"] elif best and best[0] >= WEAK_MIN and ( _shares_rare_token(item.clean, best[1]["_clean"]) - or any(_shares_rare_token(_clean_name(n), best[1]["_clean"]) + or any(_shares_rare_token(_clean_inner(n), best[1]["_clean"]) for n in item.inner_names[:10])): # Weak tier: a same-system hit below the threshold with at least # one shared word, accepted only when nothing stronger exists. @@ -768,7 +804,7 @@ def sort_plan( match_catalog(probe, catalog, threshold) fields = {k: getattr(probe, k) for k in ( "system", "group", "faction", "subfaction", "klass", - "unit", "match_score", "match_kind", + "unit", "match_score", "match_kind", "confidence", )} unit_cache[dest_key] = fields item = inspect(f) diff --git a/tests/test_sorter_units.py b/tests/test_sorter_units.py index 8f10529..9da0274 100644 --- a/tests/test_sorter_units.py +++ b/tests/test_sorter_units.py @@ -83,10 +83,11 @@ def test_prov_dir_rehome(): assert "Fly" in str(tgt["spider_body.stl"]), tgt["spider_body.stl"] assert "Hero" not in str(tgt["spider_body.stl"]) assert "files" in str(tgt["spider_arm.stl"]) - # junk-dir file stays under the pack unit folder - assert "_unsorted/Multi Pack/HEADS" in str(tgt["head_a.stl"]).replace("\\", "/") - # unmatched subtree keeps the pack unit - assert "Multi Pack" in str(tgt["t.stl"]) + # junk-dir file rides along with the pack's adopted unit + assert "Warp Spiders" in str(tgt["head_a.stl"]) + assert "HEADS" in str(tgt["head_a.stl"]) + # unmatched subtree keeps the pack unit folder + assert "Warp Spiders" in str(tgt["t.stl"]) and "Retro GrimDark Thing" in str(tgt["t.stl"]) shutil.rmtree(root); shutil.rmtree(library) @@ -151,7 +152,8 @@ def test_weak_needs_shared_token(): "Aeldari Pack": "Aeldari Pack.zip"}, root=root) by_name = {i.name: i for i in items} f = by_name["f.stl"] - assert f.match_kind == "implicit", (f.match_kind, f.target) + # both files ride along with the pack-level weak adoption + assert f.match_kind == "weak" and f.confidence == "low", (f.match_kind, f.target) assert "frontline squad" in str(f.target) g = by_name["g.stl"] assert g.match_kind == "weak" and g.confidence == "low"