gwshop: count dump coverage by unique objectID (review fixes)

Summed nbHits across facet branches double-counted products listed in
several branches; coverage is now per unique objectID, with a warning
about records left out of the dump. AGENTS.md documents the live
count (4183 records, 3491 via facet).

AGENTS.md

@@ -227,11 +227,14 @@ never re-fetch them again):
    key is search-only, never commit any other Algolia key. The full
    product index `prod-lazarus-product-en-ww` (~3500 products, plus one
    sibling index per locale) is dumped with plain `requests`: the
-   search-only key cannot use Algolia's `/browse` (403) and normal
+   public search-only key cannot use Algolia's `/browse` (403) and normal
    pagination is capped at 1000 cumulative hits, so `gwshop.py`
    **partitions the index on the `GameSystemsRoot` hierarchical facet** —
    each branch is fetched with an explicit filter, recursing deeper only
-   when a branch exceeds one page. Records are normalized into the
+   when a branch exceeds one page. (Live count 2026-10-04: 4183 records,
+   of which 3491 are reachable via the facet; the ~692 left out are paints,
+   Black Library books, White Dwarf issues, brushes, bases and gifts that
+   carry no `GameSystemsRoot` path — no miniature kits among them.) Records are normalized into the
    source-agnostic catalog schema (system/group/faction/subfaction/class
    = the facet path, name/sku/productType attached) into
    `data/gwshop_catalog.json`, which is the sorter's default catalog and

dlphelper/gwshop.py

@@ -149,7 +149,15 @@ class GwShopClient:
         if total <= hits_per_page:
             return hits
         log.info("gwshop %s: %d hits, partitioning on %s", index, total, self._FACET)
-        return self._partition(index, [], total)
+        out = self._partition(index, [], total)
+        covered = len({h.get("objectID") for h in out})
+        if covered < total:
+            log.warning(
+                "gwshop %s: %d of %d index records left out of the dump "
+                "(no fetchable %s branch path)",
+                index, total - covered, total, self._FACET,
+            )
+        return out
 
     def _partition(
         self, index: str, path: list[tuple[int, str]], total: int
@@ -158,26 +166,28 @@ class GwShopClient:
         filters = self._filter_expr(path) if path else None
         counts = self._facet_counts(index, level, filters)
         out: list[dict] = []
-        matched = 0
+        covered: set[Any] = set()
         for value, count in sorted(counts.items()):
             sub_path = path + [(level, value)]
             sub_filters = self._filter_expr(sub_path)
             if count <= _PAGE_SIZE:
                 hits, nb = self._search_all(index, sub_filters)
                 out.extend(hits)
-                matched += nb
+                covered.update(h.get("objectID") for h in hits)
                 log.info(
                     "gwshop %s: %s -> %d hits", index, sub_filters[:80], nb
                 )
             else:
                 sub = self._partition(index, sub_path, count)
                 out.extend(sub)
-                matched += len(sub)
+                covered.update(h.get("objectID") for h in sub)
-        residual = total - matched
+        # nbHits summed across branches double-counts products listed in
+        # several branches, so coverage is counted per unique objectID.
+        residual = total - len(covered)
         if residual > 0:
             log.warning(
-                "gwshop %s: %d hits do not carry %s.lvl%d "
+                "gwshop %s: %d hits in branch carry no %s.lvl%d "
-                "(unclassified products, left out of the dump)",
+                "(shallow facet path, left out of the dump)",
                 index, residual, self._FACET, level,
             )
         return out

dlphelper/sorter.py

@@ -35,6 +35,7 @@ RUN_ARTIFACTS = {
     "export.json", "export.html", "state.json", "manifest.jsonl",
     "failed_urls.txt", "catalogue.md", "catalogue.jsonl",
     "sorter_applied.jsonl", "catalogue_moves.jsonl", "urls.txt",
+    "urls_0807.txt", "urls_gofile.txt", "failed_0807.txt",
 }
 
 
@@ -174,13 +175,19 @@ def _best_ratio(clean: str, candidates: list[str]) -> float:
     )
 
 
-def _best_in_pool(item: SortItem, pool: list[dict]) -> tuple[float, dict] | None:
+def _best_in_pool(
-    """Best fuzzy score of `item` against all entries in `pool`."""
+    item: SortItem, pool: list[dict], use_inner: bool = True
+) -> tuple[float, dict] | None:
+    """Best fuzzy score of `item` against all entries in `pool`.
+
+    Inner archive names are only consulted on request; a confident outer
+    (file) name must win over votes from random inner STL filenames.
+    """
     best: tuple[float, dict] | None = None
     for entry in pool:
         cands = [entry["_clean"], *entry["_clean_aliases"]]
         score = _best_ratio(item.clean, cands)
-        if item.inner_names:
+        if use_inner and item.inner_names:
             inner_clean = [_clean_name(n) for n in item.inner_names[:10]]
             for c in inner_clean:
                 score = max(score, _best_ratio(c, cands) * 0.95)  # inner slightly discounted
@@ -202,13 +209,19 @@ def match_catalog(item: SortItem, catalog: list[dict], threshold: float) -> None
         return  # HTML-page masquerading as an archive; never fuzzy-match it
     same_faction = [e for e in catalog if e.get("faction") == item.faction]
     pool = same_faction or catalog  # fall back to whole catalog if faction unknown
-    best = _best_in_pool(item, pool)
+    best = _best_in_pool(item, pool, use_inner=False)
+    if best is None or best[0] < threshold:
+        # Outer name not confident: let the archive's inner file names vote
+        # (this is the only signal for hash-named gofile archives).
+        best = _best_in_pool(item, pool, use_inner=True)
     if best is None or best[0] < threshold:
         # The keyword tables can lag behind GW's current categorization
         # (e.g. units moved between factions); if the faction pool has no
         # confident match, try the whole catalog before giving up.
         if same_faction:
-            best = _best_in_pool(item, catalog)
+            best = _best_in_pool(item, catalog, use_inner=False)
+            if best is None or best[0] < threshold:
+                best = _best_in_pool(item, catalog, use_inner=True)
     if best and best[0] >= threshold:
         entry = best[1]
         item.unit = entry["name"]

tools/plan_diff.py

@@ -0,0 +1,26 @@
+#!/usr/bin/env python3
+"""Diff old vs new sorter plans by source path -> target."""
+import json
+import sys
+
+changed = 0
+same = 0
+examples = []
+for n in (1, 2, 3):
+    old = {p["source"]: p for p in json.load(open(f"/tmp/plan{n}.json"))}
+    new = {p["source"]: p for p in json.load(open(f"/tmp/plan{n}b.json"))}
+    for src, np in new.items():
+        op = old.get(src)
+        if not op:
+            continue
+        if op["target"] != np["target"] or op["kind"] != np["kind"]:
+            changed += 1
+            if len(examples) < 40:
+                examples.append((n, op, np))
+        else:
+            same += 1
+print(f"same: {same}  changed: {changed}")
+for n, op, np in examples:
+    print(f"\n[{n}] {op['name']}  (score {op['score']} -> {np['score']})")
+    print(f"  old: {op['target']}")
+    print(f"  new: {np['target']}")

tools/plan_summary.py

@@ -0,0 +1,22 @@
+#!/usr/bin/env python3
+"""Summarize a sorter plan JSON: match kinds, top targets, oddities."""
+import json
+import sys
+from collections import Counter
+from pathlib import Path
+
+plan = json.loads(Path(sys.argv[1]).read_text())
+print(f"files: {len(plan)}")
+print("kinds:", Counter(p["kind"] for p in plan))
+print("actions:", Counter(p.get("action", "?") for p in plan))
+# system breakdown
+print("systems:", Counter(p["system"] for p in plan).most_common(10))
+# top-level targets
+roots = Counter(str(Path(p["target"]).relative_to(
+    Path(p["target"]).parents[3])) if len(Path(p["target"]).parents) > 3
+    else p["target"] for p in plan)
+for root, n in roots.most_common(15):
+    print(f"  {n:5d}  {root}")
+# anything with empty target?
+empties = [p for p in plan if not p.get("target")]
+print("empty targets:", len(empties))