From da5f1f73159c20f7716af2661cd2078fd3037ca5 Mon Sep 17 00:00:00 2001 From: Ben_Kosytorz Date: Mon, 5 Oct 2026 07:34:29 +0200 Subject: [PATCH] sorter: outer file name wins over inner archive names in fuzzy match Best-pool scoring used to always blend the archive's inner file names into the fuzzy score (0.95 discount), which dragged clean-named files into wrong units (e.g. 'Keeper of Secrets & Shalaxi Helbane.7z' ended in Orks/Boyz). Now the outer name decides when it scores >= threshold; inner names only vote as fallback (hash-named gofile archives rely on that). Also: url/failed-list run artifacts are excluded from sorting, and two plan-inspection helpers under tools/. --- dlphelper/sorter.py | 23 ++++++++++++++++++----- tools/plan_diff.py | 26 ++++++++++++++++++++++++++ tools/plan_summary.py | 22 ++++++++++++++++++++++ 3 files changed, 66 insertions(+), 5 deletions(-) create mode 100644 tools/plan_diff.py create mode 100644 tools/plan_summary.py diff --git a/dlphelper/sorter.py b/dlphelper/sorter.py index a808ff8..09227c5 100644 --- a/dlphelper/sorter.py +++ b/dlphelper/sorter.py @@ -35,6 +35,7 @@ RUN_ARTIFACTS = { "export.json", "export.html", "state.json", "manifest.jsonl", "failed_urls.txt", "catalogue.md", "catalogue.jsonl", "sorter_applied.jsonl", "catalogue_moves.jsonl", "urls.txt", + "urls_0807.txt", "urls_gofile.txt", "failed_0807.txt", } @@ -174,13 +175,19 @@ def _best_ratio(clean: str, candidates: list[str]) -> float: ) -def _best_in_pool(item: SortItem, pool: list[dict]) -> tuple[float, dict] | None: - """Best fuzzy score of `item` against all entries in `pool`.""" +def _best_in_pool( + item: SortItem, pool: list[dict], use_inner: bool = True +) -> tuple[float, dict] | None: + """Best fuzzy score of `item` against all entries in `pool`. + + Inner archive names are only consulted on request; a confident outer + (file) name must win over votes from random inner STL filenames. + """ best: tuple[float, dict] | None = None for entry in pool: cands = [entry["_clean"], *entry["_clean_aliases"]] score = _best_ratio(item.clean, cands) - if item.inner_names: + if use_inner and item.inner_names: inner_clean = [_clean_name(n) for n in item.inner_names[:10]] for c in inner_clean: score = max(score, _best_ratio(c, cands) * 0.95) # inner slightly discounted @@ -202,13 +209,19 @@ def match_catalog(item: SortItem, catalog: list[dict], threshold: float) -> None return # HTML-page masquerading as an archive; never fuzzy-match it same_faction = [e for e in catalog if e.get("faction") == item.faction] pool = same_faction or catalog # fall back to whole catalog if faction unknown - best = _best_in_pool(item, pool) + best = _best_in_pool(item, pool, use_inner=False) + if best is None or best[0] < threshold: + # Outer name not confident: let the archive's inner file names vote + # (this is the only signal for hash-named gofile archives). + best = _best_in_pool(item, pool, use_inner=True) if best is None or best[0] < threshold: # The keyword tables can lag behind GW's current categorization # (e.g. units moved between factions); if the faction pool has no # confident match, try the whole catalog before giving up. if same_faction: - best = _best_in_pool(item, catalog) + best = _best_in_pool(item, catalog, use_inner=False) + if best is None or best[0] < threshold: + best = _best_in_pool(item, catalog, use_inner=True) if best and best[0] >= threshold: entry = best[1] item.unit = entry["name"] diff --git a/tools/plan_diff.py b/tools/plan_diff.py new file mode 100644 index 0000000..4394437 --- /dev/null +++ b/tools/plan_diff.py @@ -0,0 +1,26 @@ +#!/usr/bin/env python3 +"""Diff old vs new sorter plans by source path -> target.""" +import json +import sys + +changed = 0 +same = 0 +examples = [] +for n in (1, 2, 3): + old = {p["source"]: p for p in json.load(open(f"/tmp/plan{n}.json"))} + new = {p["source"]: p for p in json.load(open(f"/tmp/plan{n}b.json"))} + for src, np in new.items(): + op = old.get(src) + if not op: + continue + if op["target"] != np["target"] or op["kind"] != np["kind"]: + changed += 1 + if len(examples) < 40: + examples.append((n, op, np)) + else: + same += 1 +print(f"same: {same} changed: {changed}") +for n, op, np in examples: + print(f"\n[{n}] {op['name']} (score {op['score']} -> {np['score']})") + print(f" old: {op['target']}") + print(f" new: {np['target']}") diff --git a/tools/plan_summary.py b/tools/plan_summary.py new file mode 100644 index 0000000..4b96721 --- /dev/null +++ b/tools/plan_summary.py @@ -0,0 +1,22 @@ +#!/usr/bin/env python3 +"""Summarize a sorter plan JSON: match kinds, top targets, oddities.""" +import json +import sys +from collections import Counter +from pathlib import Path + +plan = json.loads(Path(sys.argv[1]).read_text()) +print(f"files: {len(plan)}") +print("kinds:", Counter(p["kind"] for p in plan)) +print("actions:", Counter(p.get("action", "?") for p in plan)) +# system breakdown +print("systems:", Counter(p["system"] for p in plan).most_common(10)) +# top-level targets +roots = Counter(str(Path(p["target"]).relative_to( + Path(p["target"]).parents[3])) if len(Path(p["target"]).parents) > 3 + else p["target"] for p in plan) +for root, n in roots.most_common(15): + print(f" {n:5d} {root}") +# anything with empty target? +empties = [p for p in plan if not p.get("target")] +print("empty targets:", len(empties))