Ben_Kosytorz
5fd5dacaf0
- warhammer.com shop HTML/listing endpoints are behind AWS WAF (405/202 goku challenge) for non-browser clients; findings recorded in AGENTS.md - the shop search runs on Algolia; public search-only key lets us dump the whole product index with plain requests (no browser, no captcha) - gwshop.py: credential loading (env or gitignored algolia.json), index listing, facet-partitioned full retrieval (search-only key cannot /browse and pagination caps at 1000, so the index is partitioned on the GameSystemsRoot hierarchy facet), normalization into the sorter's source-agnostic catalog schema (system/group/faction/subfaction/class) - data/gwshop_catalog.json: snapshot of prod-lazarus-product-en-ww, 3491 unique products, every product with full hierarchy path - tools/: one-shot browse + catalog builder + shape inspector
35 lines
1.4 KiB
Python
35 lines
1.4 KiB
Python
#!/usr/bin/env python3
|
|
"""Inspect gwshop catalog shape: Space Marines chapters, AoS, dedupe stats."""
|
|
import json
|
|
import pathlib
|
|
from collections import Counter
|
|
|
|
cat = json.loads(pathlib.Path("data/gwshop_catalog.json").read_text(encoding="utf-8"))
|
|
entries = cat["entries"]
|
|
|
|
def show(pred, label, n=6):
|
|
print(f"--- {label}")
|
|
shown = 0
|
|
for e in entries:
|
|
if pred(e):
|
|
p = " > ".join(x for x in (e["system"], e["group"], e["faction"], e["subfaction"], e["class"]) if x)
|
|
print(f" {e['name'][:60]!r} -> {p}")
|
|
shown += 1
|
|
if shown >= n:
|
|
break
|
|
|
|
show(lambda e: e["group"] == "Space Marines", "Space Marines group samples")
|
|
show(lambda e: e["group"] == "Armies of the Imperium" and e["subfaction"], "Imperium with subfaction")
|
|
show(lambda e: e["system"] == "Age of Sigmar", "Age of Sigmar samples", 5)
|
|
show(lambda e: e["system"] == "The Old World", "Old World samples", 5)
|
|
show(lambda e: e["system"] == "Other Games", "Other Games samples", 5)
|
|
show(lambda e: e["group"] == "Unit Type", "Unit Type branch", 6)
|
|
|
|
names = Counter(e["name"].casefold() for e in entries)
|
|
dups = {n: c for n, c in names.items() if c > 1}
|
|
print("duplicate names:", len(dups), "example:", list(dups.items())[:3])
|
|
|
|
# faction-level value census for 40k
|
|
print("--- 40k factions census")
|
|
print(Counter(e["faction"] for e in entries if e["system"] == "Warhammer 40,000").most_common(40))
|