sorter: outer file name wins over inner archive names in fuzzy match
Best-pool scoring used to always blend the archive's inner file names into the fuzzy score (0.95 discount), which dragged clean-named files into wrong units (e.g. 'Keeper of Secrets & Shalaxi Helbane.7z' ended in Orks/Boyz). Now the outer name decides when it scores >= threshold; inner names only vote as fallback (hash-named gofile archives rely on that). Also: url/failed-list run artifacts are excluded from sorting, and two plan-inspection helpers under tools/.
This commit is contained in:
+18
-5
@@ -35,6 +35,7 @@ RUN_ARTIFACTS = {
|
|||||||
"export.json", "export.html", "state.json", "manifest.jsonl",
|
"export.json", "export.html", "state.json", "manifest.jsonl",
|
||||||
"failed_urls.txt", "catalogue.md", "catalogue.jsonl",
|
"failed_urls.txt", "catalogue.md", "catalogue.jsonl",
|
||||||
"sorter_applied.jsonl", "catalogue_moves.jsonl", "urls.txt",
|
"sorter_applied.jsonl", "catalogue_moves.jsonl", "urls.txt",
|
||||||
|
"urls_0807.txt", "urls_gofile.txt", "failed_0807.txt",
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
@@ -174,13 +175,19 @@ def _best_ratio(clean: str, candidates: list[str]) -> float:
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
def _best_in_pool(item: SortItem, pool: list[dict]) -> tuple[float, dict] | None:
|
def _best_in_pool(
|
||||||
"""Best fuzzy score of `item` against all entries in `pool`."""
|
item: SortItem, pool: list[dict], use_inner: bool = True
|
||||||
|
) -> tuple[float, dict] | None:
|
||||||
|
"""Best fuzzy score of `item` against all entries in `pool`.
|
||||||
|
|
||||||
|
Inner archive names are only consulted on request; a confident outer
|
||||||
|
(file) name must win over votes from random inner STL filenames.
|
||||||
|
"""
|
||||||
best: tuple[float, dict] | None = None
|
best: tuple[float, dict] | None = None
|
||||||
for entry in pool:
|
for entry in pool:
|
||||||
cands = [entry["_clean"], *entry["_clean_aliases"]]
|
cands = [entry["_clean"], *entry["_clean_aliases"]]
|
||||||
score = _best_ratio(item.clean, cands)
|
score = _best_ratio(item.clean, cands)
|
||||||
if item.inner_names:
|
if use_inner and item.inner_names:
|
||||||
inner_clean = [_clean_name(n) for n in item.inner_names[:10]]
|
inner_clean = [_clean_name(n) for n in item.inner_names[:10]]
|
||||||
for c in inner_clean:
|
for c in inner_clean:
|
||||||
score = max(score, _best_ratio(c, cands) * 0.95) # inner slightly discounted
|
score = max(score, _best_ratio(c, cands) * 0.95) # inner slightly discounted
|
||||||
@@ -202,13 +209,19 @@ def match_catalog(item: SortItem, catalog: list[dict], threshold: float) -> None
|
|||||||
return # HTML-page masquerading as an archive; never fuzzy-match it
|
return # HTML-page masquerading as an archive; never fuzzy-match it
|
||||||
same_faction = [e for e in catalog if e.get("faction") == item.faction]
|
same_faction = [e for e in catalog if e.get("faction") == item.faction]
|
||||||
pool = same_faction or catalog # fall back to whole catalog if faction unknown
|
pool = same_faction or catalog # fall back to whole catalog if faction unknown
|
||||||
best = _best_in_pool(item, pool)
|
best = _best_in_pool(item, pool, use_inner=False)
|
||||||
|
if best is None or best[0] < threshold:
|
||||||
|
# Outer name not confident: let the archive's inner file names vote
|
||||||
|
# (this is the only signal for hash-named gofile archives).
|
||||||
|
best = _best_in_pool(item, pool, use_inner=True)
|
||||||
if best is None or best[0] < threshold:
|
if best is None or best[0] < threshold:
|
||||||
# The keyword tables can lag behind GW's current categorization
|
# The keyword tables can lag behind GW's current categorization
|
||||||
# (e.g. units moved between factions); if the faction pool has no
|
# (e.g. units moved between factions); if the faction pool has no
|
||||||
# confident match, try the whole catalog before giving up.
|
# confident match, try the whole catalog before giving up.
|
||||||
if same_faction:
|
if same_faction:
|
||||||
best = _best_in_pool(item, catalog)
|
best = _best_in_pool(item, catalog, use_inner=False)
|
||||||
|
if best is None or best[0] < threshold:
|
||||||
|
best = _best_in_pool(item, catalog, use_inner=True)
|
||||||
if best and best[0] >= threshold:
|
if best and best[0] >= threshold:
|
||||||
entry = best[1]
|
entry = best[1]
|
||||||
item.unit = entry["name"]
|
item.unit = entry["name"]
|
||||||
|
|||||||
@@ -0,0 +1,26 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""Diff old vs new sorter plans by source path -> target."""
|
||||||
|
import json
|
||||||
|
import sys
|
||||||
|
|
||||||
|
changed = 0
|
||||||
|
same = 0
|
||||||
|
examples = []
|
||||||
|
for n in (1, 2, 3):
|
||||||
|
old = {p["source"]: p for p in json.load(open(f"/tmp/plan{n}.json"))}
|
||||||
|
new = {p["source"]: p for p in json.load(open(f"/tmp/plan{n}b.json"))}
|
||||||
|
for src, np in new.items():
|
||||||
|
op = old.get(src)
|
||||||
|
if not op:
|
||||||
|
continue
|
||||||
|
if op["target"] != np["target"] or op["kind"] != np["kind"]:
|
||||||
|
changed += 1
|
||||||
|
if len(examples) < 40:
|
||||||
|
examples.append((n, op, np))
|
||||||
|
else:
|
||||||
|
same += 1
|
||||||
|
print(f"same: {same} changed: {changed}")
|
||||||
|
for n, op, np in examples:
|
||||||
|
print(f"\n[{n}] {op['name']} (score {op['score']} -> {np['score']})")
|
||||||
|
print(f" old: {op['target']}")
|
||||||
|
print(f" new: {np['target']}")
|
||||||
@@ -0,0 +1,22 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""Summarize a sorter plan JSON: match kinds, top targets, oddities."""
|
||||||
|
import json
|
||||||
|
import sys
|
||||||
|
from collections import Counter
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
plan = json.loads(Path(sys.argv[1]).read_text())
|
||||||
|
print(f"files: {len(plan)}")
|
||||||
|
print("kinds:", Counter(p["kind"] for p in plan))
|
||||||
|
print("actions:", Counter(p.get("action", "?") for p in plan))
|
||||||
|
# system breakdown
|
||||||
|
print("systems:", Counter(p["system"] for p in plan).most_common(10))
|
||||||
|
# top-level targets
|
||||||
|
roots = Counter(str(Path(p["target"]).relative_to(
|
||||||
|
Path(p["target"]).parents[3])) if len(Path(p["target"]).parents) > 3
|
||||||
|
else p["target"] for p in plan)
|
||||||
|
for root, n in roots.most_common(15):
|
||||||
|
print(f" {n:5d} {root}")
|
||||||
|
# anything with empty target?
|
||||||
|
empties = [p for p in plan if not p.get("target")]
|
||||||
|
print("empty targets:", len(empties))
|
||||||
Reference in New Issue
Block a user