gwshop: count dump coverage by unique objectID (review fixes)

Summed nbHits across facet branches double-counted products listed in
several branches; coverage is now per unique objectID, with a warning
about records left out of the dump. AGENTS.md documents the live
count (4183 records, 3491 via facet).
This commit is contained in:
2026-10-05 07:34:29 +02:00
parent da5f1f7315
commit 7e28518a52
2 changed files with 22 additions and 9 deletions
+17 -7
View File
@@ -149,7 +149,15 @@ class GwShopClient:
if total <= hits_per_page:
return hits
log.info("gwshop %s: %d hits, partitioning on %s", index, total, self._FACET)
return self._partition(index, [], total)
out = self._partition(index, [], total)
covered = len({h.get("objectID") for h in out})
if covered < total:
log.warning(
"gwshop %s: %d of %d index records left out of the dump "
"(no fetchable %s branch path)",
index, total - covered, total, self._FACET,
)
return out
def _partition(
self, index: str, path: list[tuple[int, str]], total: int
@@ -158,26 +166,28 @@ class GwShopClient:
filters = self._filter_expr(path) if path else None
counts = self._facet_counts(index, level, filters)
out: list[dict] = []
matched = 0
covered: set[Any] = set()
for value, count in sorted(counts.items()):
sub_path = path + [(level, value)]
sub_filters = self._filter_expr(sub_path)
if count <= _PAGE_SIZE:
hits, nb = self._search_all(index, sub_filters)
out.extend(hits)
matched += nb
covered.update(h.get("objectID") for h in hits)
log.info(
"gwshop %s: %s -> %d hits", index, sub_filters[:80], nb
)
else:
sub = self._partition(index, sub_path, count)
out.extend(sub)
matched += len(sub)
residual = total - matched
covered.update(h.get("objectID") for h in sub)
# nbHits summed across branches double-counts products listed in
# several branches, so coverage is counted per unique objectID.
residual = total - len(covered)
if residual > 0:
log.warning(
"gwshop %s: %d hits do not carry %s.lvl%d "
"(unclassified products, left out of the dump)",
"gwshop %s: %d hits in branch carry no %s.lvl%d "
"(shallow facet path, left out of the dump)",
index, residual, self._FACET, level,
)
return out