Ben_Kosytorz/Warhammer_DLP_Helper
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor: flatten the crate to the repo root and drop the python version

Browse Source 
The Rust rewrite has replaced the original python implementation, so the transition layout is no longer needed:

- move version-rust/{Cargo.toml,Cargo.lock,README.md,.gitignore,src,data} to the repo root; the crate is now the repo.
- move docs/SORTING.md and data/gwshop_catalog.json to the root.
- remove version-python/ entirely, plus the dev-only version-rust/tools/dump_warhammer_data.py (its only job was regenerating data/warhammer_data.json from python).
- rewrite AGENTS.md and README.md for the single-project layout; strip all version-python/version-rust references from code comments.
- fix the gwshop credential lookup: algolia.json now sits at CARGO_MANIFEST_DIR (crate root == repo root), not its parent.
- restore the credentials/run-output ignores in .gitignore (algolia.json, .env, *.token, downloads/, state.json, manifest.jsonl, ...).
- docs/SORTING.md: retarget the extraction/sorter/dlp examples at the dlphelper binary instead of the deleted python scripts.

cargo test: 86/86 green.
This commit is contained in:
Ben_Kosytorz
2026-10-11 09:22:41 +02:00
parent 356b9089b7
commit 6c6a3273bb
54 changed files with 331 additions and 5378 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/gwshop.rs
+629
View File
@@ -0,0 +1,629 @@
//! GW webshop catalog dump via Algolia — Rust port of
//! `dlphelper/gwshop.py`.
//!
//! The warhammer.com shop is behind an AWS WAF challenge for non-browser
//! clients, but its product search runs on Algolia and the app-id plus the
//! public search-only key ship with every browser page. This module uses
//! them to enumerate the whole product index and normalize it into the
//! sorter's catalog schema (system/group/faction/subfaction/class/name).
//!
//! Credentials are read from (first hit wins):
//! 1. env ALGOLIA_APP_ID + ALGOLIA_API_KEY
//! 2. algolia.json next to the repo root (gitignored):
//! {"app_id": "...", "api_key": "..."}
//!
//! The key is the *search-only* public key (never an admin key). Treat any
//! key found in algolia.json as still secret: never log or commit it.
//!
//! CLI: `dlphelper gwshop OUT.json [--index NAME] [--raw RAW.json]` (wired
//! in main.rs).
use std::collections::BTreeMap;
use std::path::{Path, PathBuf};
use std::time::Duration;
use anyhow::{Context as _, anyhow};
use serde_json::{Value, json};
use crate::util;
pub const DEFAULT_INDEX: &str = "prod-lazarus-product-en-ww";
// The read host (-dsn) serves searches; credentials also work as query
// parameters, which keeps every request a plain GET/POST with no headers
// beyond content-type.
const DSN_HOST: &str = "https://{}-dsn.algolia.net";
const PAGE_SIZE: usize = 1000; // Algolia max hitsPerPage per /query page
const RETRY_DELAYS: [u64; 3] = [2, 5, 15];
/// Resolve (app_id, api_key) from env or algolia.json.
pub fn load_credentials() -> anyhow::Result<(String, String)> {
let mut app_id = std::env::var("ALGOLIA_APP_ID").unwrap_or_default();
let mut api_key = std::env::var("ALGOLIA_API_KEY").unwrap_or_default();
if !app_id.is_empty() && !api_key.is_empty() {
return Ok((app_id, api_key));
}
// algolia.json lives at the repo root, which is also the crate root, so
// CARGO_MANIFEST_DIR (baked at build time) points straight at it. Fall
// back to the current working directory for runs from elsewhere.
let candidates = [
Some(PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("algolia.json")),
Some(PathBuf::from("algolia.json")),
];
for candidate in candidates.into_iter().flatten() {
if candidate.is_file() {
let data: Value = serde_json::from_str(
&std::fs::read_to_string(&candidate)
.with_context(|| format!("reading {}", candidate.display()))?,
)?;
app_id = data
.get("app_id")
.and_then(Value::as_str)
.filter(|s| !s.is_empty())
.map(str::to_string)
.unwrap_or(app_id);
api_key = data
.get("api_key")
.and_then(Value::as_str)
.filter(|s| !s.is_empty())
.map(str::to_string)
.unwrap_or(api_key);
break;
}
}
if app_id.is_empty() || api_key.is_empty() {
anyhow::bail!(
"Algolia credentials missing: set ALGOLIA_APP_ID and ALGOLIA_API_KEY, \
or create algolia.json with {{\"app_id\": \"...\", \"api_key\": \"...\"}} \
(gitignored)."
);
}
Ok((app_id, api_key))
}
/// Minimal Algolia search client for one application.
pub struct GwShopClient {
base: String,
params: Vec<(String, String)>,
session: reqwest::blocking::Client,
}
impl GwShopClient {
pub fn new(app_id: &str, api_key: &str) -> Self {
GwShopClient {
base: DSN_HOST.replace("{}", app_id),
params: vec![
("x-algolia-application-id".into(), app_id.into()),
("x-algolia-api-key".into(), api_key.into()),
],
session: reqwest::blocking::Client::new(),
}
}
fn request(&self, method: &str, path: &str, body: Option<Value>) -> anyhow::Result<Value> {
let url = format!("{base}{path}", base = self.base, path = path);
let mut last_err: Option<String> = None;
for (i, delay) in std::iter::once(0u64).chain(RETRY_DELAYS).enumerate() {
let attempt = i + 1;
if delay > 0 {
log::info!("Algolia retry {attempt} in {delay}s");
std::thread::sleep(Duration::from_secs(delay));
}
let mut req = match method {
"GET" => self.session.get(&url),
_ => self.session.post(&url),
}
.query(&self.params)
.timeout(Duration::from_secs(30));
if let Some(body) = &body {
req = req.json(body);
}
let resp = req.send();
match resp {
Ok(r) => {
let status = r.status().as_u16();
if status == 200 {
return r.json().context("decoding Algolia response");
}
let text = r.text().unwrap_or_default();
last_err = Some(format!(
"Algolia {method} {path} -> {status}: {}",
&text[..text.len().min(200)]
));
if status == 401 || status == 403 {
// credentials: no point retrying
break;
}
}
Err(exc) => last_err = Some(exc.to_string()),
}
}
Err(anyhow!(
"Algolia request failed: {}",
last_err.unwrap_or_default()
))
}
pub fn list_indices(&self) -> anyhow::Result<Vec<Value>> {
let v = self.request("GET", "/1/indexes", None)?;
Ok(v.get("items")
.and_then(Value::as_array)
.cloned()
.unwrap_or_default())
}
// ---- full index retrieval -------------------------------------------
// The search-only key cannot use /browse (403) and normal pagination is
// capped at 1000 cumulative results. So we partition the index on the
// GameSystemsRoot hierarchical facet: each branch is fetched with an
// explicit filter, recursing deeper only when a branch exceeds one page.
const FACET: &'static str = "GameSystemsRoot";
fn escape(value: &str) -> String {
value.replace('\\', "\\\\").replace('"', "\\\"")
}
fn filter_expr(&self, path: &[(usize, &str)]) -> String {
path.iter()
.map(|(lvl, value)| format!("{}.lvl{}:\"{}\"", Self::FACET, lvl, Self::escape(value)))
.collect::<Vec<_>>()
.join(" AND ")
}
fn search_all(
&self,
index: &str,
filters: Option<&str>,
) -> anyhow::Result<(Vec<Value>, usize)> {
// All hits matching `filters` (must be <= pagination cap).
let mut params = format!("query=&hitsPerPage={PAGE_SIZE}&page=0");
if let Some(filters) = filters {
params += &format!("&filters={}", urlencoding(filters));
}
let page = self.request(
"POST",
&format!("/1/indexes/{index}/query"),
Some(json!({ "params": params })),
)?;
let hits = page
.get("hits")
.and_then(Value::as_array)
.cloned()
.unwrap_or_default();
let nb = page.get("nbHits").and_then(Value::as_u64).unwrap_or(0) as usize;
Ok((hits, nb))
}
fn facet_counts(
&self,
index: &str,
level: usize,
filters: Option<&str>,
) -> anyhow::Result<BTreeMap<String, usize>> {
let mut params = format!("query=&hitsPerPage=0&facets={}.lvl{level}", Self::FACET);
if let Some(filters) = filters {
params += &format!("&filters={}", urlencoding(filters));
}
let page = self.request(
"POST",
&format!("/1/indexes/{index}/query"),
Some(json!({ "params": params })),
)?;
let facets = page.get("facets").and_then(Value::as_object);
let counts = facets
.and_then(|f| f.get(&format!("{}.lvl{level}", Self::FACET)))
.and_then(Value::as_object);
let mut out = BTreeMap::new();
for (value, count) in counts.into_iter().flatten() {
if let Some(n) = count.as_u64() {
out.insert(value.clone(), n as usize);
}
}
Ok(out)
}
/// Every record of the index, via facet-partitioned search.
pub fn browse(&self, index: &str) -> anyhow::Result<Vec<Value>> {
let (hits, total) = self.search_all(index, None)?;
if total <= PAGE_SIZE {
return Ok(hits);
}
log::info!(
"gwshop {index}: {total} hits, partitioning on {}",
Self::FACET
);
let out = self.partition(index, &[], total)?;
let covered = out
.iter()
.filter_map(|h| h.get("objectID").cloned())
.collect::<std::collections::HashSet<Value>>()
.len();
if covered < total {
log::warn!(
"gwshop {index}: {} of {total} index records left out of the dump \
(no fetchable {} branch path)",
total - covered,
Self::FACET
);
}
Ok(out)
}
fn partition(
&self,
index: &str,
path: &[(usize, String)],
total: usize,
) -> anyhow::Result<Vec<Value>> {
let level = path.len();
let filter_refs: Vec<(usize, &str)> = path.iter().map(|(l, v)| (*l, v.as_str())).collect();
let filters = if filter_refs.is_empty() {
None
} else {
Some(self.filter_expr(&filter_refs))
};
let counts = self.facet_counts(index, level, filters.as_deref())?;
let mut out: Vec<Value> = Vec::new();
let mut covered: std::collections::HashSet<String> = std::collections::HashSet::new();
for (value, count) in &counts {
let mut sub_path: Vec<(usize, String)> = path.to_vec();
sub_path.push((level, value.clone()));
let sub_filters = self.filter_expr(
&sub_path
.iter()
.map(|(l, v)| (*l, v.as_str()))
.collect::<Vec<_>>(),
);
if *count <= PAGE_SIZE {
let (hits, nb) = self.search_all(index, Some(&sub_filters))?;
log::info!(
"gwshop {index}: {} -> {nb} hits",
&sub_filters[..sub_filters.len().min(80)]
);
covered.extend(hits.iter().filter_map(|h| {
h.get("objectID")
.and_then(Value::as_str)
.map(str::to_string)
}));
out.extend(hits);
} else {
let sub = self.partition(index, &sub_path, *count)?;
covered.extend(sub.iter().filter_map(|h| {
h.get("objectID")
.and_then(Value::as_str)
.map(str::to_string)
}));
out.extend(sub);
}
}
// nbHits summed across branches double-counts products listed in
// several branches, so coverage is counted per unique objectID.
let residual = total.saturating_sub(covered.len());
if residual > 0 {
log::warn!(
"gwshop {index}: {residual} hits in branch carry no {}.lvl{level} \
(shallow facet path, left out of the dump)",
Self::FACET
);
}
Ok(out)
}
}
/// python `requests.utils.quote(filters, safe='')` — percent-encode
/// everything except unreserved characters.
fn urlencoding(s: &str) -> String {
let mut out = String::new();
for byte in s.bytes() {
match byte {
b'A'..=b'Z' | b'a'..=b'z' | b'0'..=b'9' | b'-' | b'_' | b'.' | b'~' => {
out.push(byte as char)
}
_ => out.push_str(&format!("%{byte:02X}")),
}
}
out
}
// ---------------------------------------------------------------------------
// normalization
// ---------------------------------------------------------------------------
/// Every full path of the GameSystemsRoot hierarchical facet.
///
/// A hit may belong to several branches (e.g. an army box also being a
/// "Gaming Rules" entry); each branch is returned as its own path
/// [system, group, faction, subfaction_or_class, ...].
pub fn hierarchy_paths(hit: &Value) -> Vec<Vec<String>> {
let root = hit.get("GameSystemsRoot").cloned().unwrap_or(Value::Null);
let Value::Object(root) = root else {
return Vec::new();
};
let mut levels: Vec<(usize, &Vec<Value>)> = root
.iter()
.filter_map(|(k, v)| {
let lvl = k.strip_prefix("lvl")?.parse::<usize>().ok()?;
v.as_array().map(|a| (lvl, a))
})
.collect();
levels.sort_by_key(|(l, _)| *l);
let mut paths = Vec::new();
for (_, values) in levels {
for value in values {
let Some(s) = value.as_str() else { continue };
paths.push(s.split(" > ").map(|p| p.trim().to_string()).collect());
}
}
paths
}
/// One Algolia hit -> one or more catalog entries (usually one).
fn normalize(hit: &Value) -> Value {
let name = hit
.get("name")
.and_then(Value::as_str)
.or_else(|| hit.get("objectID").and_then(Value::as_str))
.unwrap_or_default()
.trim()
.to_string();
json!({
"name": name,
"aliases": [],
"sku": hit.get("sku"),
"slug": hit.get("slug"),
"productType": hit.get("productType"),
"source": "gwshop",
"gw_paths": hierarchy_paths(hit),
"collections": hit.get("_collections").cloned().unwrap_or_else(|| json!([])),
})
}
fn entry_from(
base: &Value,
system: Value,
group: Value,
faction: Value,
subfaction: Value,
class: Value,
) -> Value {
json!({
"system": system,
"group": group,
"faction": faction,
"subfaction": subfaction,
"class": class,
"name": base["name"],
"aliases": base["aliases"],
"sku": base["sku"],
"slug": base["slug"],
"productType": base["productType"],
"source": "gwshop",
"collections": base["collections"],
})
}
/// Index dump -> catalog document in the sorter's schema shape.
///
/// The hierarchy layer (system/group/faction/subfaction) is expanded from
/// gw_paths; name stays per product so the fuzzy matcher works on the
/// product name, exactly like data/library_catalog.json entries.
pub fn build_catalog(hits: &[Value]) -> Value {
let mut entries: Vec<Value> = Vec::new();
let mut seen: std::collections::HashSet<(String, Vec<String>)> =
std::collections::HashSet::new();
for hit in hits {
let base = normalize(hit);
// python: paths = base["gw_paths"] (precomputed by normalize)
let paths: Vec<Vec<String>> = base["gw_paths"]
.as_array()
.map(|arr| {
arr.iter()
.map(|p| {
p.as_array()
.map(|parts| {
parts
.iter()
.filter_map(Value::as_str)
.map(str::to_string)
.collect()
})
.unwrap_or_default()
})
.collect()
})
.unwrap_or_default();
if paths.is_empty() {
let key = (
hit.get("objectID")
.and_then(Value::as_str)
.unwrap_or_default()
.to_string(),
Vec::new(),
);
if !seen.insert(key) {
continue;
}
entries.push(entry_from(
&base,
Value::Null,
Value::Null,
Value::Null,
Value::Null,
Value::Null,
));
continue;
}
// primary branch = longest path (most specific), e.g.
// [40k, Space Marines, Raven Guard, Walker]
let primary = paths
.iter()
.max_by_key(|p| p.len())
.cloned()
.unwrap_or_default();
let at = |i: usize| -> Value {
primary
.get(i)
.map(|s| Value::String(s.clone()))
.unwrap_or(Value::Null)
};
let system = at(0);
let group = at(1);
let faction = at(2);
// The LAST level is the class/category layer (GW uses shop
// categories there: Battleline, Walker, Vehicle, "Rules &
// Accessories", product lines like "Cadian"). Deeper sources
// (miniset style) may carry a real subfaction before it.
let (subfaction, klass) = if primary.len() >= 5 {
(at(3), at(4))
} else if primary.len() == 4 {
(Value::Null, at(3))
} else {
(Value::Null, Value::Null)
};
let key = (
hit.get("objectID")
.and_then(Value::as_str)
.unwrap_or_default()
.to_string(),
primary.clone(),
);
if !seen.insert(key) {
continue;
}
entries.push(entry_from(&base, system, group, faction, subfaction, klass));
}
json!({
"comment": (
"Generated by dlphelper (Rust) from the Games Workshop webshop \
(Algolia index, deduplicated by product). Source-agnostic \
catalog schema: system/group/faction/subfaction/class/name/aliases."
),
"source": "gwshop",
"generated_entries": entries.len(),
"entries": entries,
})
}
/// CLI body: dump the GW webshop catalog via Algolia.
pub fn run_cli(out: &Path, index: &str, raw: Option<&Path>) -> anyhow::Result<()> {
let (app_id, api_key) = load_credentials()?;
let client = GwShopClient::new(&app_id, &api_key);
let hits = client.browse(index)?;
log::info!("{len} raw hits from {index}", len = hits.len());
if let Some(raw_path) = raw {
util::save_sorted(raw_path, &hits)
.with_context(|| format!("writing raw hits to {}", raw_path.display()))?;
log::info!("raw hits written to {}", raw_path.display());
}
let catalog = build_catalog(&hits);
std::fs::write(out, serde_json::to_string_pretty(&catalog)?.as_bytes())
.with_context(|| format!("writing catalog to {}", out.display()))?;
log::info!(
"{n} catalog entries written to {p}",
n = catalog["generated_entries"],
p = out.display()
);
Ok(())
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn urlencoding_matches_python_quote_safe_empty() {
// requests.utils.quote(s, safe='') percent-encodes ' ' and '"'
assert_eq!(urlencoding("a b"), "a%20b");
assert_eq!(urlencoding("a\"b"), "a%22b");
assert_eq!(urlencoding("abc-1.2~_"), "abc-1.2~_");
assert_eq!(
urlencoding("GameSystemsRoot.lvl0:\"Warhammer 40,000\""),
"GameSystemsRoot.lvl0%3A%22Warhammer%2040%2C000%22"
);
}
#[test]
fn hierarchy_paths_orders_levels() {
// python returns EVERY lvl as its own path (each value split on " > ")
let hit = json!({
"GameSystemsRoot": {
"lvl2": ["Warhammer 40,000 > Aeldari > Rangers"],
"lvl0": ["Warhammer 40,000"],
"lvl1": ["Warhammer 40,000 > Aeldari"],
}
});
assert_eq!(
hierarchy_paths(&hit),
vec![
vec!["Warhammer 40,000".to_string()],
vec!["Warhammer 40,000".to_string(), "Aeldari".to_string()],
vec![
"Warhammer 40,000".to_string(),
"Aeldari".to_string(),
"Rangers".to_string()
],
]
);
}
#[test]
fn build_catalog_normalizes_primary_branch() {
let hit = json!({
"objectID": "obj1",
"name": " Rangers ",
"sku": "S1",
"GameSystemsRoot": {
"lvl0": ["Warhammer 40,000"],
"lvl1": ["Warhammer 40,000 > Aeldari"],
"lvl2": ["Warhammer 40,000 > Aeldari > Rangers"],
"lvl3": ["Warhammer 40,000 > Aeldari > Rangers > Battleline"],
}
});
let cat = build_catalog(&[hit]);
assert_eq!(cat["generated_entries"], 1);
let e = &cat["entries"][0];
assert_eq!(e["name"], "Rangers");
// primary = LONGEST path [40k, Aeldari, Rangers, Battleline]:
// system=40k, group=Aeldari (lvl1 value is a full path split? no -
// build_catalog indexes primary directly: lvl0=system, lvl1=group, ...)
assert_eq!(e["system"], "Warhammer 40,000");
assert_eq!(e["group"], "Aeldari");
assert_eq!(e["faction"], "Rangers");
assert_eq!(e["subfaction"], Value::Null);
assert_eq!(e["class"], "Battleline");
assert_eq!(e["source"], "gwshop");
}
#[test]
fn build_catalog_dedups_and_handles_pathless() {
let hit = json!({"objectID": "obj2", "name": "Paint"});
let cat = build_catalog(&[hit.clone(), hit]);
assert_eq!(cat["generated_entries"], 1);
let e = &cat["entries"][0];
assert_eq!(e["system"], Value::Null);
assert_eq!(e["name"], "Paint");
}
#[test]
fn credentials_error_message() {
// No env, no file: error message matches the python SystemExit text.
// NOTE: env manipulation is unsafe in recent Rust editions and
// racy across parallel tests; only assert the message shape when
// the vars are absent from the start.
if std::env::var("ALGOLIA_APP_ID")
.map(|v| v.is_empty())
.unwrap_or(true)
&& std::env::var("ALGOLIA_API_KEY")
.map(|v| v.is_empty())
.unwrap_or(true)
&& !PathBuf::from(env!("CARGO_MANIFEST_DIR"))
.join("algolia.json")
.exists()
{
let err = load_credentials().unwrap_err().to_string();
assert!(err.contains("Algolia credentials missing"), "{err}");
}
}
}