Add GW webshop catalog source (Algolia): dlphelper.gwshop + full dump

- warhammer.com shop HTML/listing endpoints are behind AWS WAF (405/202
  goku challenge) for non-browser clients; findings recorded in AGENTS.md
- the shop search runs on Algolia; public search-only key lets us dump the
  whole product index with plain requests (no browser, no captcha)
- gwshop.py: credential loading (env or gitignored algolia.json), index
  listing, facet-partitioned full retrieval (search-only key cannot /browse
  and pagination caps at 1000, so the index is partitioned on the
  GameSystemsRoot hierarchy facet), normalization into the sorter's
  source-agnostic catalog schema (system/group/faction/subfaction/class)
- data/gwshop_catalog.json: snapshot of prod-lazarus-product-en-ww,
  3491 unique products, every product with full hierarchy path
- tools/: one-shot browse + catalog builder + shape inspector
This commit is contained in:
2026-10-04 21:48:58 +02:00
parent cccf10a308
commit 5fd5dacaf0
7 changed files with 51545 additions and 1 deletions
+21
View File
@@ -0,0 +1,21 @@
#!/usr/bin/env python3
"""One-shot browse of the GW Algolia index -> gwshop_raw.json (gitignored)."""
import json
import logging
import pathlib
import sys
sys.path.insert(0, str(pathlib.Path(__file__).resolve().parent.parent))
import dlphelper.gwshop as gw
logging.basicConfig(level=logging.INFO, format="%(levelname)s %(message)s")
app, key = gw.load_credentials()
client = gw.GwShopClient(app, key)
hits = client.browse("prod-lazarus-product-en-ww", hits_per_page=100)
print("total hits", len(hits))
pathlib.Path("gwshop_raw.json").write_text(
json.dumps(hits, ensure_ascii=False), encoding="utf-8"
)
print("written gwshop_raw.json")
+42
View File
@@ -0,0 +1,42 @@
#!/usr/bin/env python3
"""gwshop_raw.json -> data/gwshop_catalog.json + quick shape report."""
import json
import logging
import pathlib
import sys
from collections import Counter
sys.path.insert(0, str(pathlib.Path(__file__).resolve().parent.parent))
import dlphelper.gwshop as gw
logging.basicConfig(level=logging.INFO, format="%(levelname)s %(message)s")
hits = json.loads(pathlib.Path("gwshop_raw.json").read_text(encoding="utf-8"))
print("raw hits:", len(hits))
unique = len({h.get("objectID") for h in hits})
print("unique objectIDs:", unique)
catalog = gw.build_catalog(hits)
pathlib.Path("data/gwshop_catalog.json").write_text(
json.dumps(catalog, ensure_ascii=False, indent=1), encoding="utf-8"
)
print("entries:", catalog["generated_entries"])
entries = catalog["entries"]
print("no system:", sum(1 for e in entries if e["system"] is None))
print("by system:", Counter(e["system"] for e in entries).most_common())
print("40k groups:", Counter(
e["group"] for e in entries if e["system"] == "Warhammer 40,000"
).most_common())
print("lvl depth:", Counter(
len((e.get("gw_paths") or [[]])[0]) for e in entries
).most_common())
print("classes (lvl>=4):", Counter(
e["class"] for e in entries if e["class"]
).most_common(12))
sample = next(
e for e in entries
if e["name"].startswith("Codex: Adeptus Custodes")
)
print(json.dumps(sample, indent=1, ensure_ascii=False)[:900])
+34
View File
@@ -0,0 +1,34 @@
#!/usr/bin/env python3
"""Inspect gwshop catalog shape: Space Marines chapters, AoS, dedupe stats."""
import json
import pathlib
from collections import Counter
cat = json.loads(pathlib.Path("data/gwshop_catalog.json").read_text(encoding="utf-8"))
entries = cat["entries"]
def show(pred, label, n=6):
print(f"--- {label}")
shown = 0
for e in entries:
if pred(e):
p = " > ".join(x for x in (e["system"], e["group"], e["faction"], e["subfaction"], e["class"]) if x)
print(f" {e['name'][:60]!r} -> {p}")
shown += 1
if shown >= n:
break
show(lambda e: e["group"] == "Space Marines", "Space Marines group samples")
show(lambda e: e["group"] == "Armies of the Imperium" and e["subfaction"], "Imperium with subfaction")
show(lambda e: e["system"] == "Age of Sigmar", "Age of Sigmar samples", 5)
show(lambda e: e["system"] == "The Old World", "Old World samples", 5)
show(lambda e: e["system"] == "Other Games", "Other Games samples", 5)
show(lambda e: e["group"] == "Unit Type", "Unit Type branch", 6)
names = Counter(e["name"].casefold() for e in entries)
dups = {n: c for n, c in names.items() if c > 1}
print("duplicate names:", len(dups), "example:", list(dups.items())[:3])
# faction-level value census for 40k
print("--- 40k factions census")
print(Counter(e["faction"] for e in entries if e["system"] == "Warhammer 40,000").most_common(40))