Add GW webshop catalog source (Algolia): dlphelper.gwshop + full dump
- warhammer.com shop HTML/listing endpoints are behind AWS WAF (405/202 goku challenge) for non-browser clients; findings recorded in AGENTS.md - the shop search runs on Algolia; public search-only key lets us dump the whole product index with plain requests (no browser, no captcha) - gwshop.py: credential loading (env or gitignored algolia.json), index listing, facet-partitioned full retrieval (search-only key cannot /browse and pagination caps at 1000, so the index is partitioned on the GameSystemsRoot hierarchy facet), normalization into the sorter's source-agnostic catalog schema (system/group/faction/subfaction/class) - data/gwshop_catalog.json: snapshot of prod-lazarus-product-en-ww, 3491 unique products, every product with full hierarchy path - tools/: one-shot browse + catalog builder + shape inspector
This commit is contained in:
@@ -0,0 +1,21 @@
|
||||
#!/usr/bin/env python3
|
||||
"""One-shot browse of the GW Algolia index -> gwshop_raw.json (gitignored)."""
|
||||
import json
|
||||
import logging
|
||||
import pathlib
|
||||
import sys
|
||||
|
||||
sys.path.insert(0, str(pathlib.Path(__file__).resolve().parent.parent))
|
||||
|
||||
import dlphelper.gwshop as gw
|
||||
|
||||
logging.basicConfig(level=logging.INFO, format="%(levelname)s %(message)s")
|
||||
|
||||
app, key = gw.load_credentials()
|
||||
client = gw.GwShopClient(app, key)
|
||||
hits = client.browse("prod-lazarus-product-en-ww", hits_per_page=100)
|
||||
print("total hits", len(hits))
|
||||
pathlib.Path("gwshop_raw.json").write_text(
|
||||
json.dumps(hits, ensure_ascii=False), encoding="utf-8"
|
||||
)
|
||||
print("written gwshop_raw.json")
|
||||
@@ -0,0 +1,42 @@
|
||||
#!/usr/bin/env python3
|
||||
"""gwshop_raw.json -> data/gwshop_catalog.json + quick shape report."""
|
||||
import json
|
||||
import logging
|
||||
import pathlib
|
||||
import sys
|
||||
from collections import Counter
|
||||
|
||||
sys.path.insert(0, str(pathlib.Path(__file__).resolve().parent.parent))
|
||||
|
||||
import dlphelper.gwshop as gw
|
||||
|
||||
logging.basicConfig(level=logging.INFO, format="%(levelname)s %(message)s")
|
||||
|
||||
hits = json.loads(pathlib.Path("gwshop_raw.json").read_text(encoding="utf-8"))
|
||||
print("raw hits:", len(hits))
|
||||
unique = len({h.get("objectID") for h in hits})
|
||||
print("unique objectIDs:", unique)
|
||||
|
||||
catalog = gw.build_catalog(hits)
|
||||
pathlib.Path("data/gwshop_catalog.json").write_text(
|
||||
json.dumps(catalog, ensure_ascii=False, indent=1), encoding="utf-8"
|
||||
)
|
||||
print("entries:", catalog["generated_entries"])
|
||||
|
||||
entries = catalog["entries"]
|
||||
print("no system:", sum(1 for e in entries if e["system"] is None))
|
||||
print("by system:", Counter(e["system"] for e in entries).most_common())
|
||||
print("40k groups:", Counter(
|
||||
e["group"] for e in entries if e["system"] == "Warhammer 40,000"
|
||||
).most_common())
|
||||
print("lvl depth:", Counter(
|
||||
len((e.get("gw_paths") or [[]])[0]) for e in entries
|
||||
).most_common())
|
||||
print("classes (lvl>=4):", Counter(
|
||||
e["class"] for e in entries if e["class"]
|
||||
).most_common(12))
|
||||
sample = next(
|
||||
e for e in entries
|
||||
if e["name"].startswith("Codex: Adeptus Custodes")
|
||||
)
|
||||
print(json.dumps(sample, indent=1, ensure_ascii=False)[:900])
|
||||
@@ -0,0 +1,34 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Inspect gwshop catalog shape: Space Marines chapters, AoS, dedupe stats."""
|
||||
import json
|
||||
import pathlib
|
||||
from collections import Counter
|
||||
|
||||
cat = json.loads(pathlib.Path("data/gwshop_catalog.json").read_text(encoding="utf-8"))
|
||||
entries = cat["entries"]
|
||||
|
||||
def show(pred, label, n=6):
|
||||
print(f"--- {label}")
|
||||
shown = 0
|
||||
for e in entries:
|
||||
if pred(e):
|
||||
p = " > ".join(x for x in (e["system"], e["group"], e["faction"], e["subfaction"], e["class"]) if x)
|
||||
print(f" {e['name'][:60]!r} -> {p}")
|
||||
shown += 1
|
||||
if shown >= n:
|
||||
break
|
||||
|
||||
show(lambda e: e["group"] == "Space Marines", "Space Marines group samples")
|
||||
show(lambda e: e["group"] == "Armies of the Imperium" and e["subfaction"], "Imperium with subfaction")
|
||||
show(lambda e: e["system"] == "Age of Sigmar", "Age of Sigmar samples", 5)
|
||||
show(lambda e: e["system"] == "The Old World", "Old World samples", 5)
|
||||
show(lambda e: e["system"] == "Other Games", "Other Games samples", 5)
|
||||
show(lambda e: e["group"] == "Unit Type", "Unit Type branch", 6)
|
||||
|
||||
names = Counter(e["name"].casefold() for e in entries)
|
||||
dups = {n: c for n, c in names.items() if c > 1}
|
||||
print("duplicate names:", len(dups), "example:", list(dups.items())[:3])
|
||||
|
||||
# faction-level value census for 40k
|
||||
print("--- 40k factions census")
|
||||
print(Counter(e["faction"] for e in entries if e["system"] == "Warhammer 40,000").most_common(40))
|
||||
Reference in New Issue
Block a user