diff --git a/AGENTS.md b/AGENTS.md index 46476e4..9e53328 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -227,11 +227,14 @@ never re-fetch them again): key is search-only, never commit any other Algolia key. The full product index `prod-lazarus-product-en-ww` (~3500 products, plus one sibling index per locale) is dumped with plain `requests`: the - search-only key cannot use Algolia's `/browse` (403) and normal + public search-only key cannot use Algolia's `/browse` (403) and normal pagination is capped at 1000 cumulative hits, so `gwshop.py` **partitions the index on the `GameSystemsRoot` hierarchical facet** — each branch is fetched with an explicit filter, recursing deeper only - when a branch exceeds one page. Records are normalized into the + when a branch exceeds one page. (Live count 2026-10-04: 4183 records, + of which 3491 are reachable via the facet; the ~692 left out are paints, + Black Library books, White Dwarf issues, brushes, bases and gifts that + carry no `GameSystemsRoot` path — no miniature kits among them.) Records are normalized into the source-agnostic catalog schema (system/group/faction/subfaction/class = the facet path, name/sku/productType attached) into `data/gwshop_catalog.json`, which is the sorter's default catalog and diff --git a/dlphelper/gwshop.py b/dlphelper/gwshop.py index a4a5256..5e6ec46 100644 --- a/dlphelper/gwshop.py +++ b/dlphelper/gwshop.py @@ -149,7 +149,15 @@ class GwShopClient: if total <= hits_per_page: return hits log.info("gwshop %s: %d hits, partitioning on %s", index, total, self._FACET) - return self._partition(index, [], total) + out = self._partition(index, [], total) + covered = len({h.get("objectID") for h in out}) + if covered < total: + log.warning( + "gwshop %s: %d of %d index records left out of the dump " + "(no fetchable %s branch path)", + index, total - covered, total, self._FACET, + ) + return out def _partition( self, index: str, path: list[tuple[int, str]], total: int @@ -158,26 +166,28 @@ class GwShopClient: filters = self._filter_expr(path) if path else None counts = self._facet_counts(index, level, filters) out: list[dict] = [] - matched = 0 + covered: set[Any] = set() for value, count in sorted(counts.items()): sub_path = path + [(level, value)] sub_filters = self._filter_expr(sub_path) if count <= _PAGE_SIZE: hits, nb = self._search_all(index, sub_filters) out.extend(hits) - matched += nb + covered.update(h.get("objectID") for h in hits) log.info( "gwshop %s: %s -> %d hits", index, sub_filters[:80], nb ) else: sub = self._partition(index, sub_path, count) out.extend(sub) - matched += len(sub) - residual = total - matched + covered.update(h.get("objectID") for h in sub) + # nbHits summed across branches double-counts products listed in + # several branches, so coverage is counted per unique objectID. + residual = total - len(covered) if residual > 0: log.warning( - "gwshop %s: %d hits do not carry %s.lvl%d " - "(unclassified products, left out of the dump)", + "gwshop %s: %d hits in branch carry no %s.lvl%d " + "(shallow facet path, left out of the dump)", index, residual, self._FACET, level, ) return out