diff --git a/version-rust/src/gwshop.rs b/version-rust/src/gwshop.rs index 0419707..9e614ab 100644 --- a/version-rust/src/gwshop.rs +++ b/version-rust/src/gwshop.rs @@ -1,2 +1,635 @@ //! GW webshop catalog dump via Algolia — Rust port of //! `version-python/dlphelper/gwshop.py`. +//! +//! The warhammer.com shop is behind an AWS WAF challenge for non-browser +//! clients, but its product search runs on Algolia and the app-id plus the +//! public search-only key ship with every browser page. This module uses +//! them to enumerate the whole product index and normalize it into the +//! sorter's catalog schema (system/group/faction/subfaction/class/name). +//! +//! Credentials are read from (first hit wins): +//! 1. env ALGOLIA_APP_ID + ALGOLIA_API_KEY +//! 2. algolia.json next to the repo root (gitignored): +//! {"app_id": "...", "api_key": "..."} +//! +//! The key is the *search-only* public key (never an admin key). Treat any +//! key found in algolia.json as still secret: never log or commit it. +//! +//! CLI: `dlphelper gwshop OUT.json [--index NAME] [--raw RAW.json]` (wired +//! in main.rs). + +use std::collections::BTreeMap; +use std::path::{Path, PathBuf}; +use std::time::Duration; + +use anyhow::{Context as _, anyhow}; +use serde_json::{Value, json}; + +use crate::util; + +pub const DEFAULT_INDEX: &str = "prod-lazarus-product-en-ww"; +// The read host (-dsn) serves searches; credentials also work as query +// parameters, which keeps every request a plain GET/POST with no headers +// beyond content-type. +const DSN_HOST: &str = "https://{}-dsn.algolia.net"; +const PAGE_SIZE: usize = 1000; // Algolia max hitsPerPage per /query page +const RETRY_DELAYS: [u64; 3] = [2, 5, 15]; + +/// Resolve (app_id, api_key) from env or algolia.json. +pub fn load_credentials() -> anyhow::Result<(String, String)> { + let mut app_id = std::env::var("ALGOLIA_APP_ID").unwrap_or_default(); + let mut api_key = std::env::var("ALGOLIA_API_KEY").unwrap_or_default(); + if !app_id.is_empty() && !api_key.is_empty() { + return Ok((app_id, api_key)); + } + // python: algolia.json next to the repo root; the Rust crate root is + // version-rust/, so the repo root is its parent. CARGO_MANIFEST_DIR is + // baked at build time - use the executable's location upward search as + // fallback? Keep it simple and deterministic: repo root via + // CARGO_MANIFEST_DIR/../algolia.json, checked at runtime. + let candidates = [ + PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .parent() + .map(|p| p.join("algolia.json")), + Some(PathBuf::from("algolia.json")), + ]; + for candidate in candidates.into_iter().flatten() { + if candidate.is_file() { + let data: Value = serde_json::from_str( + &std::fs::read_to_string(&candidate) + .with_context(|| format!("reading {}", candidate.display()))?, + )?; + app_id = data + .get("app_id") + .and_then(Value::as_str) + .filter(|s| !s.is_empty()) + .map(str::to_string) + .unwrap_or(app_id); + api_key = data + .get("api_key") + .and_then(Value::as_str) + .filter(|s| !s.is_empty()) + .map(str::to_string) + .unwrap_or(api_key); + break; + } + } + if app_id.is_empty() || api_key.is_empty() { + anyhow::bail!( + "Algolia credentials missing: set ALGOLIA_APP_ID and ALGOLIA_API_KEY, \ + or create algolia.json with {{\"app_id\": \"...\", \"api_key\": \"...\"}} \ + (gitignored)." + ); + } + Ok((app_id, api_key)) +} + +/// Minimal Algolia search client for one application. +pub struct GwShopClient { + base: String, + params: Vec<(String, String)>, + session: reqwest::blocking::Client, +} + +impl GwShopClient { + pub fn new(app_id: &str, api_key: &str) -> Self { + GwShopClient { + base: DSN_HOST.replace("{}", app_id), + params: vec![ + ("x-algolia-application-id".into(), app_id.into()), + ("x-algolia-api-key".into(), api_key.into()), + ], + session: reqwest::blocking::Client::new(), + } + } + + fn request(&self, method: &str, path: &str, body: Option) -> anyhow::Result { + let url = format!("{base}{path}", base = self.base, path = path); + let mut last_err: Option = None; + for (i, delay) in std::iter::once(0u64).chain(RETRY_DELAYS).enumerate() { + let attempt = i + 1; + if delay > 0 { + log::info!("Algolia retry {attempt} in {delay}s"); + std::thread::sleep(Duration::from_secs(delay)); + } + let mut req = match method { + "GET" => self.session.get(&url), + _ => self.session.post(&url), + } + .query(&self.params) + .timeout(Duration::from_secs(30)); + if let Some(body) = &body { + req = req.json(body); + } + let resp = req.send(); + match resp { + Ok(r) => { + let status = r.status().as_u16(); + if status == 200 { + return r.json().context("decoding Algolia response"); + } + let text = r.text().unwrap_or_default(); + last_err = Some(format!( + "Algolia {method} {path} -> {status}: {}", + &text[..text.len().min(200)] + )); + if status == 401 || status == 403 { + // credentials: no point retrying + break; + } + } + Err(exc) => last_err = Some(exc.to_string()), + } + } + Err(anyhow!( + "Algolia request failed: {}", + last_err.unwrap_or_default() + )) + } + + pub fn list_indices(&self) -> anyhow::Result> { + let v = self.request("GET", "/1/indexes", None)?; + Ok(v.get("items") + .and_then(Value::as_array) + .cloned() + .unwrap_or_default()) + } + + // ---- full index retrieval ------------------------------------------- + // The search-only key cannot use /browse (403) and normal pagination is + // capped at 1000 cumulative results. So we partition the index on the + // GameSystemsRoot hierarchical facet: each branch is fetched with an + // explicit filter, recursing deeper only when a branch exceeds one page. + + const FACET: &'static str = "GameSystemsRoot"; + + fn escape(value: &str) -> String { + value.replace('\\', "\\\\").replace('"', "\\\"") + } + + fn filter_expr(&self, path: &[(usize, &str)]) -> String { + path.iter() + .map(|(lvl, value)| format!("{}.lvl{}:\"{}\"", Self::FACET, lvl, Self::escape(value))) + .collect::>() + .join(" AND ") + } + + fn search_all( + &self, + index: &str, + filters: Option<&str>, + ) -> anyhow::Result<(Vec, usize)> { + // All hits matching `filters` (must be <= pagination cap). + let mut params = format!("query=&hitsPerPage={PAGE_SIZE}&page=0"); + if let Some(filters) = filters { + params += &format!("&filters={}", urlencoding(filters)); + } + let page = self.request( + "POST", + &format!("/1/indexes/{index}/query"), + Some(json!({ "params": params })), + )?; + let hits = page + .get("hits") + .and_then(Value::as_array) + .cloned() + .unwrap_or_default(); + let nb = page.get("nbHits").and_then(Value::as_u64).unwrap_or(0) as usize; + Ok((hits, nb)) + } + + fn facet_counts( + &self, + index: &str, + level: usize, + filters: Option<&str>, + ) -> anyhow::Result> { + let mut params = format!("query=&hitsPerPage=0&facets={}.lvl{level}", Self::FACET); + if let Some(filters) = filters { + params += &format!("&filters={}", urlencoding(filters)); + } + let page = self.request( + "POST", + &format!("/1/indexes/{index}/query"), + Some(json!({ "params": params })), + )?; + let facets = page.get("facets").and_then(Value::as_object); + let counts = facets + .and_then(|f| f.get(&format!("{}.lvl{level}", Self::FACET))) + .and_then(Value::as_object); + let mut out = BTreeMap::new(); + for (value, count) in counts.into_iter().flatten() { + if let Some(n) = count.as_u64() { + out.insert(value.clone(), n as usize); + } + } + Ok(out) + } + + /// Every record of the index, via facet-partitioned search. + pub fn browse(&self, index: &str) -> anyhow::Result> { + let (hits, total) = self.search_all(index, None)?; + if total <= PAGE_SIZE { + return Ok(hits); + } + log::info!( + "gwshop {index}: {total} hits, partitioning on {}", + Self::FACET + ); + let out = self.partition(index, &[], total)?; + let covered = out + .iter() + .filter_map(|h| h.get("objectID").cloned()) + .collect::>() + .len(); + if covered < total { + log::warn!( + "gwshop {index}: {} of {total} index records left out of the dump \ + (no fetchable {} branch path)", + total - covered, + Self::FACET + ); + } + Ok(out) + } + + fn partition( + &self, + index: &str, + path: &[(usize, String)], + total: usize, + ) -> anyhow::Result> { + let level = path.len(); + let filter_refs: Vec<(usize, &str)> = path.iter().map(|(l, v)| (*l, v.as_str())).collect(); + let filters = if filter_refs.is_empty() { + None + } else { + Some(self.filter_expr(&filter_refs)) + }; + let counts = self.facet_counts(index, level, filters.as_deref())?; + let mut out: Vec = Vec::new(); + let mut covered: std::collections::HashSet = std::collections::HashSet::new(); + for (value, count) in &counts { + let mut sub_path: Vec<(usize, String)> = path.to_vec(); + sub_path.push((level, value.clone())); + let sub_filters = self.filter_expr( + &sub_path + .iter() + .map(|(l, v)| (*l, v.as_str())) + .collect::>(), + ); + if *count <= PAGE_SIZE { + let (hits, nb) = self.search_all(index, Some(&sub_filters))?; + log::info!( + "gwshop {index}: {} -> {nb} hits", + &sub_filters[..sub_filters.len().min(80)] + ); + covered.extend(hits.iter().filter_map(|h| { + h.get("objectID") + .and_then(Value::as_str) + .map(str::to_string) + })); + out.extend(hits); + } else { + let sub = self.partition(index, &sub_path, *count)?; + covered.extend(sub.iter().filter_map(|h| { + h.get("objectID") + .and_then(Value::as_str) + .map(str::to_string) + })); + out.extend(sub); + } + } + // nbHits summed across branches double-counts products listed in + // several branches, so coverage is counted per unique objectID. + let residual = total.saturating_sub(covered.len()); + if residual > 0 { + log::warn!( + "gwshop {index}: {residual} hits in branch carry no {}.lvl{level} \ + (shallow facet path, left out of the dump)", + Self::FACET + ); + } + Ok(out) + } +} + +/// python `requests.utils.quote(filters, safe='')` — percent-encode +/// everything except unreserved characters. +fn urlencoding(s: &str) -> String { + let mut out = String::new(); + for byte in s.bytes() { + match byte { + b'A'..=b'Z' | b'a'..=b'z' | b'0'..=b'9' | b'-' | b'_' | b'.' | b'~' => { + out.push(byte as char) + } + _ => out.push_str(&format!("%{byte:02X}")), + } + } + out +} + +// --------------------------------------------------------------------------- +// normalization +// --------------------------------------------------------------------------- + +/// Every full path of the GameSystemsRoot hierarchical facet. +/// +/// A hit may belong to several branches (e.g. an army box also being a +/// "Gaming Rules" entry); each branch is returned as its own path +/// [system, group, faction, subfaction_or_class, ...]. +pub fn hierarchy_paths(hit: &Value) -> Vec> { + let root = hit.get("GameSystemsRoot").cloned().unwrap_or(Value::Null); + let Value::Object(root) = root else { + return Vec::new(); + }; + let mut levels: Vec<(usize, &Vec)> = root + .iter() + .filter_map(|(k, v)| { + let lvl = k.strip_prefix("lvl")?.parse::().ok()?; + v.as_array().map(|a| (lvl, a)) + }) + .collect(); + levels.sort_by_key(|(l, _)| *l); + let mut paths = Vec::new(); + for (_, values) in levels { + for value in values { + let Some(s) = value.as_str() else { continue }; + paths.push(s.split(" > ").map(|p| p.trim().to_string()).collect()); + } + } + paths +} + +/// One Algolia hit -> one or more catalog entries (usually one). +fn normalize(hit: &Value) -> Value { + let name = hit + .get("name") + .and_then(Value::as_str) + .or_else(|| hit.get("objectID").and_then(Value::as_str)) + .unwrap_or_default() + .trim() + .to_string(); + json!({ + "name": name, + "aliases": [], + "sku": hit.get("sku"), + "slug": hit.get("slug"), + "productType": hit.get("productType"), + "source": "gwshop", + "gw_paths": hierarchy_paths(hit), + "collections": hit.get("_collections").cloned().unwrap_or_else(|| json!([])), + }) +} + +fn entry_from( + base: &Value, + system: Value, + group: Value, + faction: Value, + subfaction: Value, + class: Value, +) -> Value { + json!({ + "system": system, + "group": group, + "faction": faction, + "subfaction": subfaction, + "class": class, + "name": base["name"], + "aliases": base["aliases"], + "sku": base["sku"], + "slug": base["slug"], + "productType": base["productType"], + "source": "gwshop", + "collections": base["collections"], + }) +} + +/// Index dump -> catalog document in the sorter's schema shape. +/// +/// The hierarchy layer (system/group/faction/subfaction) is expanded from +/// gw_paths; name stays per product so the fuzzy matcher works on the +/// product name, exactly like data/library_catalog.json entries. +pub fn build_catalog(hits: &[Value]) -> Value { + let mut entries: Vec = Vec::new(); + let mut seen: std::collections::HashSet<(String, Vec)> = + std::collections::HashSet::new(); + for hit in hits { + let base = normalize(hit); + // python: paths = base["gw_paths"] (precomputed by normalize) + let paths: Vec> = base["gw_paths"] + .as_array() + .map(|arr| { + arr.iter() + .map(|p| { + p.as_array() + .map(|parts| { + parts + .iter() + .filter_map(Value::as_str) + .map(str::to_string) + .collect() + }) + .unwrap_or_default() + }) + .collect() + }) + .unwrap_or_default(); + if paths.is_empty() { + let key = ( + hit.get("objectID") + .and_then(Value::as_str) + .unwrap_or_default() + .to_string(), + Vec::new(), + ); + if !seen.insert(key) { + continue; + } + entries.push(entry_from( + &base, + Value::Null, + Value::Null, + Value::Null, + Value::Null, + Value::Null, + )); + continue; + } + // primary branch = longest path (most specific), e.g. + // [40k, Space Marines, Raven Guard, Walker] + let primary = paths + .iter() + .max_by_key(|p| p.len()) + .cloned() + .unwrap_or_default(); + let at = |i: usize| -> Value { + primary + .get(i) + .map(|s| Value::String(s.clone())) + .unwrap_or(Value::Null) + }; + let system = at(0); + let group = at(1); + let faction = at(2); + // The LAST level is the class/category layer (GW uses shop + // categories there: Battleline, Walker, Vehicle, "Rules & + // Accessories", product lines like "Cadian"). Deeper sources + // (miniset style) may carry a real subfaction before it. + let (subfaction, klass) = if primary.len() >= 5 { + (at(3), at(4)) + } else if primary.len() == 4 { + (Value::Null, at(3)) + } else { + (Value::Null, Value::Null) + }; + let key = ( + hit.get("objectID") + .and_then(Value::as_str) + .unwrap_or_default() + .to_string(), + primary.clone(), + ); + if !seen.insert(key) { + continue; + } + entries.push(entry_from(&base, system, group, faction, subfaction, klass)); + } + json!({ + "comment": ( + "Generated by dlphelper (Rust) from the Games Workshop webshop \ + (Algolia index, deduplicated by product). Source-agnostic \ + catalog schema: system/group/faction/subfaction/class/name/aliases." + ), + "source": "gwshop", + "generated_entries": entries.len(), + "entries": entries, + }) +} + +/// CLI body: dump the GW webshop catalog via Algolia. +pub fn run_cli(out: &Path, index: &str, raw: Option<&Path>) -> anyhow::Result<()> { + let (app_id, api_key) = load_credentials()?; + let client = GwShopClient::new(&app_id, &api_key); + let hits = client.browse(index)?; + log::info!("{len} raw hits from {index}", len = hits.len()); + + if let Some(raw_path) = raw { + util::save_sorted(raw_path, &hits) + .with_context(|| format!("writing raw hits to {}", raw_path.display()))?; + log::info!("raw hits written to {}", raw_path.display()); + } + + let catalog = build_catalog(&hits); + std::fs::write(out, serde_json::to_string_pretty(&catalog)?.as_bytes()) + .with_context(|| format!("writing catalog to {}", out.display()))?; + log::info!( + "{n} catalog entries written to {p}", + n = catalog["generated_entries"], + p = out.display() + ); + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn urlencoding_matches_python_quote_safe_empty() { + // requests.utils.quote(s, safe='') percent-encodes ' ' and '"' + assert_eq!(urlencoding("a b"), "a%20b"); + assert_eq!(urlencoding("a\"b"), "a%22b"); + assert_eq!(urlencoding("abc-1.2~_"), "abc-1.2~_"); + assert_eq!( + urlencoding("GameSystemsRoot.lvl0:\"Warhammer 40,000\""), + "GameSystemsRoot.lvl0%3A%22Warhammer%2040%2C000%22" + ); + } + + #[test] + fn hierarchy_paths_orders_levels() { + // python returns EVERY lvl as its own path (each value split on " > ") + let hit = json!({ + "GameSystemsRoot": { + "lvl2": ["Warhammer 40,000 > Aeldari > Rangers"], + "lvl0": ["Warhammer 40,000"], + "lvl1": ["Warhammer 40,000 > Aeldari"], + } + }); + assert_eq!( + hierarchy_paths(&hit), + vec![ + vec!["Warhammer 40,000".to_string()], + vec!["Warhammer 40,000".to_string(), "Aeldari".to_string()], + vec![ + "Warhammer 40,000".to_string(), + "Aeldari".to_string(), + "Rangers".to_string() + ], + ] + ); + } + + #[test] + fn build_catalog_normalizes_primary_branch() { + let hit = json!({ + "objectID": "obj1", + "name": " Rangers ", + "sku": "S1", + "GameSystemsRoot": { + "lvl0": ["Warhammer 40,000"], + "lvl1": ["Warhammer 40,000 > Aeldari"], + "lvl2": ["Warhammer 40,000 > Aeldari > Rangers"], + "lvl3": ["Warhammer 40,000 > Aeldari > Rangers > Battleline"], + } + }); + let cat = build_catalog(&[hit]); + assert_eq!(cat["generated_entries"], 1); + let e = &cat["entries"][0]; + assert_eq!(e["name"], "Rangers"); + // primary = LONGEST path [40k, Aeldari, Rangers, Battleline]: + // system=40k, group=Aeldari (lvl1 value is a full path split? no - + // build_catalog indexes primary directly: lvl0=system, lvl1=group, ...) + assert_eq!(e["system"], "Warhammer 40,000"); + assert_eq!(e["group"], "Aeldari"); + assert_eq!(e["faction"], "Rangers"); + assert_eq!(e["subfaction"], Value::Null); + assert_eq!(e["class"], "Battleline"); + assert_eq!(e["source"], "gwshop"); + } + + #[test] + fn build_catalog_dedups_and_handles_pathless() { + let hit = json!({"objectID": "obj2", "name": "Paint"}); + let cat = build_catalog(&[hit.clone(), hit]); + assert_eq!(cat["generated_entries"], 1); + let e = &cat["entries"][0]; + assert_eq!(e["system"], Value::Null); + assert_eq!(e["name"], "Paint"); + } + + #[test] + fn credentials_error_message() { + // No env, no file: error message matches the python SystemExit text. + // NOTE: env manipulation is unsafe in recent Rust editions and + // racy across parallel tests; only assert the message shape when + // the vars are absent from the start. + if std::env::var("ALGOLIA_APP_ID") + .map(|v| v.is_empty()) + .unwrap_or(true) + && std::env::var("ALGOLIA_API_KEY") + .map(|v| v.is_empty()) + .unwrap_or(true) + && !PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .parent() + .unwrap() + .join("algolia.json") + .exists() + { + let err = load_credentials().unwrap_err().to_string(); + assert!(err.contains("Algolia credentials missing"), "{err}"); + } + } +}