Ben_Kosytorz/Warhammer_DLP_Helper
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

sorter: probe votes on full inner paths (dir names count); no single-token containment boost for inner votes; catalog class wins on unit adoption; confidence copied from probe

Browse Source
This commit is contained in:
Ben_Kosytorz
2026-10-07 09:46:36 +02:00
parent fcec027473
commit 9133b43831
2 changed files with 61 additions and 23 deletions
Download Patch File Download Diff File Expand all files Collapse all files
dlphelper/sorter.py
+54 -18
View File
@@ -154,14 +154,15 @@ def classify_item(item: SortItem) -> None:
if hit:
item.system, item.faction = hit
item.klass = _match_class(item.clean) or (
_match_class(_clean_name(" ".join(item.inner_names))) if item.inner_names else None
_match_class(_clean_inner(" ".join(item.inner_names)))
if item.inner_names else None
)
if item.faction == UNSORTED and item.inner_names:
# hash-named archives: vote on the inner names' classification
votes: dict[tuple[str, str], int] = {}
for inner in item.inner_names:
ihit = _match_faction(_clean_name(inner))
ihit = _match_faction(_clean_inner(inner))
if ihit:
votes[ihit] = votes.get(ihit, 0) + 1
if votes:
@@ -198,20 +199,24 @@ def load_catalog(path: Path,
return entries
def _best_ratio(clean: str, candidates: list[str]) -> float:
def _best_ratio(
clean: str, candidates: list[str], allow_single: bool = True
) -> float:
"""Best sequence ratio, pruned with cheap upper-bound checks.
lru_cached because the same (name, entry) pair is scored over and
over across thousands of files.
"""
return _best_ratio_cached(clean, tuple(candidates))
return _best_ratio_cached(clean, tuple(candidates), allow_single)
@lru_cache(maxsize=None)
def _best_ratio_cached(clean: str, candidates: tuple[str, ...]) -> float:
def _best_ratio_cached(
clean: str, candidates: tuple[str, ...], allow_single: bool = True
) -> float:
best = 0.0
toks = set(_stem_tokens(clean))
best = _containment_boost(best, toks, candidates)
best = _containment_boost(best, toks, candidates, allow_single)
for c in candidates:
sm = difflib.SequenceMatcher(None, clean, c)
if sm.real_quick_ratio() < best or sm.quick_ratio() < best:
@@ -230,13 +235,18 @@ def _stem_tokens(clean: str) -> list[str]:
for t in clean.split()]
def _containment_boost(best: float, toks: set[str], candidates: tuple[str, ...]) -> float:
def _containment_boost(
best: float, toks: set[str], candidates: tuple[str, ...],
allow_single: bool = True,
) -> float:
"""Token-containment bonus: a name whose tokens all appear in the
other (or vice versa) matches strongly even when string ratio is low
("telemon" vs "Telemon Heavy Dreadnought"). Guarded: needs >= 2
tokens, or a single rare one (>= 7 chars), and the smaller side must
not consist of generic army words ("chaos lord" is contained in half
the catalog) so "head" or "guard" never trigger it."""
the catalog) so "head" or "guard" never trigger it. Inner-name votes
pass allow_single=False: one short inner file name ("lieutenant")
must not claim a whole pack."""
if not toks:
return best
for c in candidates:
@@ -246,7 +256,7 @@ def _containment_boost(best: float, toks: set[str], candidates: tuple[str, ...])
small = toks if toks <= ctoks else (ctoks if ctoks <= toks else None)
if small is None or any(t in GENERIC_TOKENS for t in small):
continue
if len(small) >= 2 or len(next(iter(small))) >= 7:
if len(small) >= 2 or (allow_single and len(next(iter(small))) >= 7):
if best < 0.9:
best = 0.9
break
@@ -278,9 +288,18 @@ _JUNK_DIRS = frozenset({
"model", "output", "exports", "release", "releases", "license",
"readme", "sources", "docs", "textures", "new", "old", "final",
"fixed", "print", "prints", "plates", "plate", "assembly", "test",
"characters", "character",
}) # noqa: C405
def _clean_inner(name: str) -> str:
"""Clean an archive-internal (relative) path: dir names join the vote
("2E Catachany fellaz/Catachan/Jungle Fighters/x.stl" -> "2e catachany
fellaz catachan jungle fighters x") - the file stem alone loses the
archive's own folder structure."""
return _clean_name(name.replace("/", " ").replace("\\", " "))
def _is_junk_dir(name: str) -> bool:
"""True for directory names that must never be match candidates."""
clean = _clean_name(name)
@@ -309,6 +328,14 @@ def _token_contained(a_clean: str, b_clean: str) -> bool:
return True
def _contained_somewhere(item: SortItem, entry: dict) -> bool:
"""Token containment between the entry (or an alias) and the outer
name or any inner (relative-path) name."""
cands = [entry["_clean"], *entry["_clean_aliases"]]
probes = [item.clean, *[_clean_inner(n) for n in item.inner_names[:10]]]
return any(_token_contained(p, c) for p in probes for c in cands)
def _shares_rare_token(a_clean: str, b_clean: str) -> bool:
"""True when two cleaned names share at least one non-generic token
(>= 3 chars). Gates the weak tier: a sub-threshold hit with zero
@@ -520,9 +547,10 @@ def _best_in_pool(
cands = [entry["_clean"], *entry["_clean_aliases"]]
score = _best_ratio(item.clean, cands)
if use_inner and item.inner_names:
inner_clean = [_clean_name(n) for n in item.inner_names[:10]]
inner_clean = [_clean_inner(n) for n in item.inner_names[:10]]
for c in inner_clean:
score = max(score, _best_ratio(c, cands) * 0.95) # inner slightly discounted
# no single-token containment boost for inner votes
score = max(score, _best_ratio(c, cands, allow_single=False) * 0.95)
if best is None or score > best[0]:
best = (score, entry)
return best
@@ -565,9 +593,11 @@ def match_catalog(item: SortItem, catalog: list[dict], threshold: float) -> None
if sys_best is not None and (best is None or sys_best[0] > best[0]):
best = sys_best
# a retry hit from a *different* faction is only trusted when
# it is strong; weak inner-name votes must not re-home the file
# it is strong AND anchored by token containment; weak
# inner-name votes must not re-home the file
if best is not None and best[1].get("faction") != item.faction \
and best[0] < CROSS_SYSTEM_MIN:
and (best[0] < CROSS_SYSTEM_MIN
or not _contained_somewhere(item, best[1])):
best = None
elif system_known:
sys_pool = [e for e in catalog if e.get("system") == item.system]
@@ -576,13 +606,16 @@ def match_catalog(item: SortItem, catalog: list[dict], threshold: float) -> None
best = _best_in_pool(item, sys_pool, use_inner=True)
else:
# Neither system nor faction known: whole catalog, but only a
# strong hit (CROSS_SYSTEM_MIN) may adopt another game's section;
# weaker guesses stay in _unsorted.
# strong hit (CROSS_SYSTEM_MIN) with token containment may adopt
# another game's section; weaker or unanchored guesses stay in
# _unsorted.
best = _best_in_pool(item, catalog, use_inner=False)
if best is None or best[0] < threshold:
best = _best_in_pool(item, catalog, use_inner=True)
if best is not None and best[0] < CROSS_SYSTEM_MIN:
best = None
elif best is not None and not _contained_somewhere(item, best[1]):
best = None
if best and best[0] >= threshold:
entry = best[1]
item.unit = entry["name"]
@@ -596,7 +629,10 @@ def match_catalog(item: SortItem, catalog: list[dict], threshold: float) -> None
# not *narrow* it: "Battleline" is GW's bucket for core infantry,
# a keyword-derived Hero/Elite is more specific for a proxy file.
catalog_klass = entry.get("class")
if catalog_klass and item.klass in (None, "", UNCLASSIFIED_CLASS):
if catalog_klass:
# a matched catalog unit's class layer is authoritative for
# the whole subtree - a keyword-voted class ("warlock" seen
# somewhere -> Hero) must not survive next to a precise unit
item.klass = catalog_klass
item.match_score = best[0]
item.match_kind = "catalog"
@@ -627,7 +663,7 @@ def match_catalog(item: SortItem, catalog: list[dict], threshold: float) -> None
item.group = entry["group"]
elif best and best[0] >= WEAK_MIN and (
_shares_rare_token(item.clean, best[1]["_clean"])
or any(_shares_rare_token(_clean_name(n), best[1]["_clean"])
or any(_shares_rare_token(_clean_inner(n), best[1]["_clean"])
for n in item.inner_names[:10])):
# Weak tier: a same-system hit below the threshold with at least
# one shared word, accepted only when nothing stronger exists.
@@ -768,7 +804,7 @@ def sort_plan(
match_catalog(probe, catalog, threshold)
fields = {k: getattr(probe, k) for k in (
"system", "group", "faction", "subfaction", "klass",
"unit", "match_score", "match_kind",
"unit", "match_score", "match_kind", "confidence",
)}
unit_cache[dest_key] = fields
item = inspect(f)
tests/test_sorter_units.py
+7 -5
View File
@@ -83,10 +83,11 @@ def test_prov_dir_rehome():
assert "Fly" in str(tgt["spider_body.stl"]), tgt["spider_body.stl"]
assert "Hero" not in str(tgt["spider_body.stl"])
assert "files" in str(tgt["spider_arm.stl"])
# junk-dir file stays under the pack unit folder
assert "_unsorted/Multi Pack/HEADS" in str(tgt["head_a.stl"]).replace("\\", "/")
# unmatched subtree keeps the pack unit
assert "Multi Pack" in str(tgt["t.stl"])
# junk-dir file rides along with the pack's adopted unit
assert "Warp Spiders" in str(tgt["head_a.stl"])
assert "HEADS" in str(tgt["head_a.stl"])
# unmatched subtree keeps the pack unit folder
assert "Warp Spiders" in str(tgt["t.stl"]) and "Retro GrimDark Thing" in str(tgt["t.stl"])
shutil.rmtree(root); shutil.rmtree(library)
@@ -151,7 +152,8 @@ def test_weak_needs_shared_token():
"Aeldari Pack": "Aeldari Pack.zip"}, root=root)
by_name = {i.name: i for i in items}
f = by_name["f.stl"]
assert f.match_kind == "implicit", (f.match_kind, f.target)
# both files ride along with the pack-level weak adoption
assert f.match_kind == "weak" and f.confidence == "low", (f.match_kind, f.target)
assert "frontline squad" in str(f.target)
g = by_name["g.stl"]
assert g.match_kind == "weak" and g.confidence == "low"