gwshop: count dump coverage by unique objectID (review fixes)
Summed nbHits across facet branches double-counted products listed in several branches; coverage is now per unique objectID, with a warning about records left out of the dump. AGENTS.md documents the live count (4183 records, 3491 via facet).
This commit is contained in:
+17
-7
@@ -149,7 +149,15 @@ class GwShopClient:
|
||||
if total <= hits_per_page:
|
||||
return hits
|
||||
log.info("gwshop %s: %d hits, partitioning on %s", index, total, self._FACET)
|
||||
return self._partition(index, [], total)
|
||||
out = self._partition(index, [], total)
|
||||
covered = len({h.get("objectID") for h in out})
|
||||
if covered < total:
|
||||
log.warning(
|
||||
"gwshop %s: %d of %d index records left out of the dump "
|
||||
"(no fetchable %s branch path)",
|
||||
index, total - covered, total, self._FACET,
|
||||
)
|
||||
return out
|
||||
|
||||
def _partition(
|
||||
self, index: str, path: list[tuple[int, str]], total: int
|
||||
@@ -158,26 +166,28 @@ class GwShopClient:
|
||||
filters = self._filter_expr(path) if path else None
|
||||
counts = self._facet_counts(index, level, filters)
|
||||
out: list[dict] = []
|
||||
matched = 0
|
||||
covered: set[Any] = set()
|
||||
for value, count in sorted(counts.items()):
|
||||
sub_path = path + [(level, value)]
|
||||
sub_filters = self._filter_expr(sub_path)
|
||||
if count <= _PAGE_SIZE:
|
||||
hits, nb = self._search_all(index, sub_filters)
|
||||
out.extend(hits)
|
||||
matched += nb
|
||||
covered.update(h.get("objectID") for h in hits)
|
||||
log.info(
|
||||
"gwshop %s: %s -> %d hits", index, sub_filters[:80], nb
|
||||
)
|
||||
else:
|
||||
sub = self._partition(index, sub_path, count)
|
||||
out.extend(sub)
|
||||
matched += len(sub)
|
||||
residual = total - matched
|
||||
covered.update(h.get("objectID") for h in sub)
|
||||
# nbHits summed across branches double-counts products listed in
|
||||
# several branches, so coverage is counted per unique objectID.
|
||||
residual = total - len(covered)
|
||||
if residual > 0:
|
||||
log.warning(
|
||||
"gwshop %s: %d hits do not carry %s.lvl%d "
|
||||
"(unclassified products, left out of the dump)",
|
||||
"gwshop %s: %d hits in branch carry no %s.lvl%d "
|
||||
"(shallow facet path, left out of the dump)",
|
||||
index, residual, self._FACET, level,
|
||||
)
|
||||
return out
|
||||
|
||||
Reference in New Issue
Block a user