From 6c6a3273bbec714ae50d822d589941e570fce386 Mon Sep 17 00:00:00 2001 From: Ben_Kosytorz Date: Sun, 11 Oct 2026 09:22:41 +0200 Subject: [PATCH] refactor: flatten the crate to the repo root and drop the python version The Rust rewrite has replaced the original python implementation, so the transition layout is no longer needed: - move version-rust/{Cargo.toml,Cargo.lock,README.md,.gitignore,src,data} to the repo root; the crate is now the repo. - move docs/SORTING.md and data/gwshop_catalog.json to the root. - remove version-python/ entirely, plus the dev-only version-rust/tools/dump_warhammer_data.py (its only job was regenerating data/warhammer_data.json from python). - rewrite AGENTS.md and README.md for the single-project layout; strip all version-python/version-rust references from code comments. - fix the gwshop credential lookup: algolia.json now sits at CARGO_MANIFEST_DIR (crate root == repo root), not its parent. - restore the credentials/run-output ignores in .gitignore (algolia.json, .env, *.token, downloads/, state.json, manifest.jsonl, ...). - docs/SORTING.md: retarget the extraction/sorter/dlp examples at the dlphelper binary instead of the deleted python scripts. cargo test: 86/86 green. --- version-python/.gitignore => .gitignore | 23 +- AGENTS.md | 268 +++++- version-rust/Cargo.lock => Cargo.lock | 0 version-rust/Cargo.toml => Cargo.toml | 2 +- version-rust/README.md => README.md | 36 +- .../data => data}/gwshop_catalog.json | 0 .../data => data}/warhammer_data.json | 0 {version-python/docs => docs}/SORTING.md | 55 +- {version-rust/src => src}/catalogue.rs | 4 +- {version-rust/src => src}/difflib.rs | 0 {version-rust/src => src}/discord.rs | 4 +- {version-rust/src => src}/extract.rs | 2 +- {version-rust/src => src}/globalstore.rs | 2 +- {version-rust/src => src}/gofile.rs | 2 +- {version-rust/src => src}/gwshop.rs | 16 +- {version-rust/src => src}/main.rs | 23 +- {version-rust/src => src}/manifest.rs | 2 +- {version-rust/src => src}/orchestrate.rs | 2 +- {version-rust/src => src}/sorter.rs | 6 +- {version-rust/src => src}/state.rs | 2 +- {version-rust/src => src}/util.rs | 0 {version-rust/src => src}/warhammer_data.rs | 10 +- {version-rust/src => src}/zero807.rs | 2 +- version-python/AGENTS.md | 310 ------ version-python/README.md | 111 --- version-python/catalogue.py | 118 --- version-python/data/library_catalog.json | 140 --- version-python/dlp.py | 544 ----------- version-python/dlphelper/__init__.py | 3 - version-python/dlphelper/catalogue.py | 305 ------ version-python/dlphelper/discord.py | 160 ---- version-python/dlphelper/globalstore.py | 70 -- version-python/dlphelper/gofile.py | 239 ----- version-python/dlphelper/gwshop.py | 353 ------- version-python/dlphelper/sorter.py | 899 ------------------ version-python/dlphelper/state.py | 78 -- version-python/dlphelper/warhammer_data.py | 529 ----------- version-python/dlphelper/zero807.py | 92 -- version-python/parts/grab.sh | 38 - version-python/requirements.txt | 1 - version-python/sorter.py | 140 --- version-python/tests/test_catalogue.py | 62 -- version-python/tests/test_extract.py | 68 -- version-python/tests/test_sorter_units.py | 258 ----- version-python/tools/debug_gofile.py | 68 -- version-python/tools/extract_all.py | 379 -------- version-python/tools/gw_browse_test.py | 21 - version-python/tools/gw_build_catalog.py | 42 - version-python/tools/gw_inspect.py | 34 - version-python/tools/new_repo.py | 108 --- version-python/tools/plan_diff.py | 26 - version-python/tools/plan_summary.py | 22 - version-rust/.gitignore | 1 - version-rust/tools/dump_warhammer_data.py | 29 - 54 files changed, 331 insertions(+), 5378 deletions(-) rename version-python/.gitignore => .gitignore (85%) rename version-rust/Cargo.lock => Cargo.lock (100%) rename version-rust/Cargo.toml => Cargo.toml (92%) rename version-rust/README.md => README.md (84%) rename {version-python/data => data}/gwshop_catalog.json (100%) rename {version-rust/data => data}/warhammer_data.json (100%) rename {version-python/docs => docs}/SORTING.md (80%) rename {version-rust/src => src}/catalogue.rs (99%) rename {version-rust/src => src}/difflib.rs (100%) rename {version-rust/src => src}/discord.rs (99%) rename {version-rust/src => src}/extract.rs (99%) rename {version-rust/src => src}/globalstore.rs (99%) rename {version-rust/src => src}/gofile.rs (99%) rename {version-rust/src => src}/gwshop.rs (97%) rename {version-rust/src => src}/main.rs (96%) rename {version-rust/src => src}/manifest.rs (98%) rename {version-rust/src => src}/orchestrate.rs (99%) rename {version-rust/src => src}/sorter.rs (99%) rename {version-rust/src => src}/state.rs (99%) rename {version-rust/src => src}/util.rs (100%) rename {version-rust/src => src}/warhammer_data.rs (93%) rename {version-rust/src => src}/zero807.rs (99%) delete mode 100644 version-python/AGENTS.md delete mode 100644 version-python/README.md delete mode 100644 version-python/catalogue.py delete mode 100644 version-python/data/library_catalog.json delete mode 100644 version-python/dlp.py delete mode 100644 version-python/dlphelper/__init__.py delete mode 100644 version-python/dlphelper/catalogue.py delete mode 100644 version-python/dlphelper/discord.py delete mode 100644 version-python/dlphelper/globalstore.py delete mode 100644 version-python/dlphelper/gofile.py delete mode 100644 version-python/dlphelper/gwshop.py delete mode 100644 version-python/dlphelper/sorter.py delete mode 100644 version-python/dlphelper/state.py delete mode 100644 version-python/dlphelper/warhammer_data.py delete mode 100644 version-python/dlphelper/zero807.py delete mode 100755 version-python/parts/grab.sh delete mode 100644 version-python/requirements.txt delete mode 100644 version-python/sorter.py delete mode 100644 version-python/tests/test_catalogue.py delete mode 100644 version-python/tests/test_extract.py delete mode 100644 version-python/tests/test_sorter_units.py delete mode 100644 version-python/tools/debug_gofile.py delete mode 100644 version-python/tools/extract_all.py delete mode 100644 version-python/tools/gw_browse_test.py delete mode 100644 version-python/tools/gw_build_catalog.py delete mode 100644 version-python/tools/gw_inspect.py delete mode 100644 version-python/tools/new_repo.py delete mode 100644 version-python/tools/plan_diff.py delete mode 100644 version-python/tools/plan_summary.py delete mode 100644 version-rust/.gitignore delete mode 100644 version-rust/tools/dump_warhammer_data.py diff --git a/version-python/.gitignore b/.gitignore similarity index 85% rename from version-python/.gitignore rename to .gitignore index 0c313a3..f4fd1f2 100644 --- a/version-python/.gitignore +++ b/.gitignore @@ -1,12 +1,17 @@ -# Python -__pycache__/ -*.pyc -.venv/ -*.egg-info/ +/target + +# Algolia credentials (search-only key, still never commit) +algolia.json +gwshop_raw.json + +# Never commit credentials +*.token +.env # Run outputs (if OUT_DIR points into the repo) thread/ downloads/ +extracted/ export.json export.html state.json @@ -14,11 +19,3 @@ state.json.tmp failed_urls.txt manifest.jsonl *.log - -# Never commit credentials -*.token -.env - -# Algolia credentials (search-only key, still never commit) -algolia.json -gwshop_raw.json diff --git a/AGENTS.md b/AGENTS.md index 74a0334..159d336 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,26 +1,252 @@ -# AGENTS.md — Warhammer_DLP_Helper (transition layout) +# AGENTS.md — Warhammer_DLP_Helper -This repo is mid-transition. **The `main` branch still carries the original -flat python layout** — read `version-python/AGENTS.md` there (and on this -branch at `version-python/AGENTS.md`) for the full project rules: secrets, -failure policy, idempotency, rate limiting, testing. Those rules still apply. +Guidance for any AI agent (or human) working in this repository. Read this +fully before making changes. -## Layout on this branch (`version-rust`) +## 1. What this project is -- `version-python/` — the existing, working python tooling, moved here with - `git mv` (history preserved). All rules and path references in its - `AGENTS.md` resolve relative to `version-python/`. -- `version-rust/` — the Rust rewrite (cargo project) that will eventually - replace the python version. See `version-rust/README.md` for the port plan - and crate choices. +The user collects **Warhammer proxy models** (3rd-party / fan sculpts, never +official Games Workshop assets) that are shared in Discord channels. Links in +those channels point to two hosters and are often **base64-encoded** inside +message content (obfuscated by the posting bot). -## Transition rules +The end goal is a **torrent-first platform** (think radarr/lidarr, but for +proxy files). This repo is the Rust implementation of the current stage: one +binary, `dlphelper`, that -1. Python fixes go into `version-python/` on `main` (flat layout there) and - are merged into `version-python/` here; never fork the logic in two - places. -2. The Rust version must stay compatible with the run artifacts - (`state.json`, `manifest.jsonl`, `~/.local/share/dlphelper/urls.json`) — - schemas are the contract. -3. `data/gwshop_catalog.json` (committed snapshot) is shared; the sorter in - both versions must be able to read the same catalog schema. +1. exports a Discord channel/thread (DiscordChatExporter via docker), +2. extracts base64-encoded and plain download links, +3. picks the right downloader per host automatically, +4. downloads everything with per-file tracking, retries and resume, +5. extracts the archives into a plain-file tree, +6. sorts the files into the library by system/group/faction/class/unit, +7. records a manifest that will later feed the torrent stage. + +A webapp is the next stage. Deliberately **not** in scope (yet): `.torrent` +creation, tracker upload, deseeeding, rename-into-library conventions. Design +decisions should not close the door on these — especially `manifest.jsonl` +and the `--hash` option exist for dedupe/cross-seeding later. + +This repository was ported from a Python implementation, which the Rust +version has now replaced (the ported modules keep `Rust port of …` provenance +notes in their headers). + +## 2. Repository & environment + +- Remote: `git.dieterunterstrich.de/Ben_Kosytorz/Warhammer_DLP_Helper` — + **private** repo, branch `main`. Push with the credentials already stored + via git's credential store (`~/.git-credentials`); never paste them into + commands or files. +- Machine: Linux, Rust (edition 2024), docker is available. The repo lives on + **NFS**, where rustc SIGBUSes — always build/test with an explicit target + directory: + + CARGO_TARGET_DIR=/var/tmp/dlphelper-target cargo test + +- Reference clone: `~/GoFileDownloader` (Lysagxra) — do not modify it; it is + upstream reading material only. + +## 3. Architecture + +``` +Cargo.toml crate `dlphelper` (edition 2024) +src/main.rs CLI (clap subcommands: dlp|catalogue|sorter|extract|gwshop) +src/orchestrate.rs dlp pipeline: export -> extract URLs -> per-host download +src/discord.rs DCE docker export (JSON) + base64/plain URL extraction +src/gofile.rs GoFile API client (one guest account per run) +src/zero807.rs curl downloader for 0807.st (direct links + /f/ folder pages) +src/state.rs state.json store +src/globalstore.rs cross-run URL store (~/.local/share/dlphelper/urls.json) +src/manifest.rs manifest.jsonl appender +src/extract.rs recursive archive extraction into a plain-file tree +src/catalogue.rs scanner + keyword classifier + report writer +src/sorter.rs library sorter: inspect/classify/fuzzy-match, plan + apply +src/difflib.rs faithful CPython SequenceMatcher port (bit-identical ratios) +src/warhammer_data.rs taxonomy tables (parsed from data/warhammer_data.json) +src/gwshop.rs GW webshop catalog dump via Algolia +src/util.rs small shared helpers +data/gwshop_catalog.json committed catalog snapshot (sorter default) +data/warhammer_data.json committed taxonomy tables +docs/SORTING.md the library/sorting contract (change it before the code) +``` + +Data flow: + +```mermaid +graph LR + A[channel id + token] --> B[DCE docker export -> export.json] + B --> C[decode base64 in message content] + C --> D{classify host} + D -->|gofile.io| E[GoFileClient: 1 guest account, list contents, download files] + D -->|0807.st| F[curl with UA + Referer] + D -->|other| G[recorded as failed: unsupported host] + E --> H[downloads/gofile/...] + F --> I[downloads/0807/...] + E & F --> J[state.json + manifest.jsonl + failed_urls.txt] + J --> K[globalstore: urls.json, marks URL done forever] + J --> L[extract -> plain-file tree] + L --> M[sorter -> library, hardlinks + audit] +``` + +Key facts about the hosters (learned the hard way, do not re-learn): + +- **gofile.io**: guest accounts are created via `POST api.gofile.io/accounts`. + Creation is aggressively rate-limited — create **at most one account per + run**. Content listing needs `Authorization: Bearer `, an + `X-Website-Token` header (sha256 over `UA::en-US::::<4h-window>::seed`, + see `src/gofile.rs`), and downloads need `Cookie: accountToken=` plus + Referer/Origin headers. +- **0807.st**: serves files directly but 403s anything that doesn't look like + a browser. `curl -A "Mozilla/5.0" -e "https://0807.st/"` works; aria2c's + default multi-connection request does not. curl is the tool here. + Folder pages (`https://0807.st/f/`) are expanded: the page is parsed and + each listed file is fetched from `https://0807.st/?dl=1` into + `downloads/0807//`. The folder `.zip` export is refused by the site + ("folder too large"), so per-file download is the only route in. +- Encoded links in the channels start as base64 strings matching + `aHR0cHM6Ly[89]...`. + +## 4. Usage (what "done" looks like) + + export DISCORD_TOKEN=... + dlphelper dlp /big/storage/warhammer/ --hash + +`--since 5d` (or `12h`, `2026-10-04`) limits a run to recent messages — handy +for incremental updates. Re-runs are safe: `state.json` and the global store +skip everything already marked ok. + +Retry only failures: + + dlphelper dlp --urls-file /failed_urls.txt + +Several channels at once (`subscriptions.json`: list of `{name, channel_id, +out_dir}`): + + dlphelper dlp --subs subscriptions.json + +Subs mode re-exports every channel by default (that is how new links get +picked up) and relies on `state.json` + the global store to only download what +is actually new. Use a cron/systemd timer for watch-like behavior. + +Extract and sort: + + dlphelper extract /downloads/0807 /downloads/gofile --out /extracted + dlphelper sorter /extracted --library --apply + +Manual import of old URL lists (known fully downloaded, never re-fetch): + + dlphelper dlp --assume-done old_urls.txt + +## 5. Run artifacts (all in OUT_DIR, gitignored) + +| File | Purpose | +|------|---------| +| `export.json` | DCE export; cached — `--fresh` re-exports | +| `state.json` | url → {status, attempts, error, files[]}; corrupt files are backed up and reset | +| `failed_urls.txt` | rewritten each run: every non-ok URL, usable as `--urls-file` | +| `manifest.jsonl` | append-only, one JSON line per downloaded file (channel, url, message_id, timestamp, author, path, size, optional sha256) | +| `downloads/` | actual content; `.part` files mean "interrupted, will be resumed" | +| `extracted/extract_manifest.jsonl` | one line per extraction `{archive, dest, unit}` (feeds `sorter --provenance`) | +| `~/.local/share/dlphelper/urls.json` | **global** cross-run store: every successfully downloaded URL is recorded once, regardless of OUT_DIR. `--global-state` moves it, `--no-global` disables it for a run. | + +Sorter bookkeeping (audit + low-confidence doc) is written to +`/../_logs/` by default — never inside the library (see §7). + +## 6. Rules + +### Secrets and tokens +1. Never print, log, commit or echo a Discord token or GoFile account token. + Not even partial. Strip them from any output you paste into chats. +2. The Discord token goes: `--token` flag → `$DISCORD_TOKEN` → hidden prompt. + It is handed to the DCE container via the `TOKEN` env var, never as a + command-line argument. +3. `~/.git-credentials` may only be read by tooling that needs it; never copy + credentials into scripts, tests, or fixtures. +4. Nothing downloaded and no export/state/manifest file is ever committed. + +### Failure policy (the core requirement) +5. A failure in one URL or file must **never** abort the run. Record it in + `state.json` and continue. The per-URL loops in `src/orchestrate.rs` + therefore have catch-alls — keep them. +6. Cooldowns are the only permitted "stall": after N consecutive failures, + pause, then continue. Circuit-breakers that exit the process need an + explicit justification in the commit message. +7. Retries: exponential-ish (`retry_delay * attempt`), default 3 attempts. + Password-required gofile links are not retried (it can't help); dead links + (HTTP 404/410) are never retried. + +### Idempotency +8. Re-runs must be safe and cheap: skip URLs marked `ok` (in `state.json` + **or** the global store), skip existing non-empty files, write partial + downloads to `*.part` and rename on success. +9. `state.json` is the source of truth for resume; keep its schema backward + compatible (unknown keys preserved, missing keys defaulted). Same for the + global store schema. +10. `--assume-done` is a deliberate, manual import step for old URL lists. + Never invoke it automatically, and never let the tool mark URLs done + without a verified successful download. + +### Rate limiting / network etiquette +11. One gofile guest account per run; `ensure_token()` backs off linearly + (20s × attempt). Do not raise the guest-account frequency "to make tests + pass". Testing account creation more than a couple times per hour is a + bug, not a test strategy. +12. Keep the 0807 referer/UA contract. Don't "optimize" it to aria2c without + re-verifying the 403 behavior. + +### Code style +13. Rust 2024, idiomatic. Use `log` (not `println!`) in library modules; + prints are fine only for CLI output (plan/dry-run). +14. Keep new dependencies justified — the crate already covers HTTP + (`reqwest`), archives (`sevenz-rust2`, `zip`, `unrar`), fuzzy matching + (the in-tree `difflib` port, **not** an external crate) and CLI (`clap`). + The tar/gz/bz2/xz paths shell out to the system binaries deliberately. +15. `src/difflib.rs` must stay bit-identical to CPython `difflib` — the sorter + thresholds depend on exact ratios; do not "simplify" it. + +### Validation +16. Minimum validation for any change: `CARGO_TARGET_DIR=/var/tmp/dlphelper-target + cargo test` (the offline suite must stay green). Document what you could + *not* test (live gofile/0807/Discord paths need real links and tokens). +17. Full live runs need a Discord token from the user — ask for it, never ask + for it to be pasted into a file or committed. + +### Git +18. Branch `main`, descriptive commit messages, no force-push, no history + rewriting on the shared remote. + +## 7. Sorter & library structure + +`docs/SORTING.md` is the working agreement for the library hierarchy and the +sorting pipeline — **change it first, then the code**. Summary: + +- Hierarchy: `///////`. + Fully unclassifiable files land in `/_unsorted/` — never guessed. +- Matching rules are fixed: the outer file name wins when it scores ≥ + threshold; inner archive names only vote as fallback. A fuzzy match never + jumps into another game's section (keyword systems constrain the catalog + pool; cross-system adoption needs `CROSS_SYSTEM_MIN = 0.75` and token + containment). Deepest-matching-dir wins (junk dir names are never + candidates). Weak hits ≥ `WEAK_MIN = 0.50` are accepted but flagged + low-confidence. Tokenless normal adoption needs `+0.08` + (`TOKENLESS_BONUS`). +- `DROPPED_SYSTEMS` (default `{Middle-Earth}`) filters catalog entries at load + and counts keyword hits as unclassified; `--drop-system`/`--keep-system` + adjust. The committed catalog snapshot keeps all systems. +- Catalog source: the GW webshop via Algolia (`src/gwshop.rs`). The app-id + + public **search-only** key live in the gitignored `algolia.json` (or + `ALGOLIA_APP_ID`/`ALGOLIA_API_KEY`); never commit any other Algolia key. + `data/gwshop_catalog.json` is the committed snapshot and the sorter default. +- Bookkeeping (`sorter_applied.jsonl`, `sorter_low_confidence.txt`) is written + to `/../_logs/` by default — never inside the library, which may be + a seeded torrent whose file set is frozen. + +## 8. Testing notes + +- Tests live inline (`#[cfg(test)]`) in each module and run with `cargo test`. + Keep them dependency-free and offline — they must never hit + gofile/0807/Discord. +- `tempfile` tests stage under `/var/tmp` when available, because `/tmp` is a + small tmpfs that trips the extractor's free-space guard (`MIN_FREE_BYTES`). +- To exercise downloads without Discord: build a `--urls-file` from known URLs + and run with a scratch OUT_DIR. diff --git a/version-rust/Cargo.lock b/Cargo.lock similarity index 100% rename from version-rust/Cargo.lock rename to Cargo.lock diff --git a/version-rust/Cargo.toml b/Cargo.toml similarity index 92% rename from version-rust/Cargo.toml rename to Cargo.toml index 0b83c38..4df8517 100644 --- a/version-rust/Cargo.toml +++ b/Cargo.toml @@ -2,7 +2,7 @@ name = "dlphelper" version = "0.1.0" edition = "2024" -description = "Torrent-first platform for Warhammer proxy models: Discord channel export, link extraction, per-host downloaders, manifest — Rust rewrite of version-python." +description = "Torrent-first platform for Warhammer proxy models: Discord channel export, link extraction, per-host downloaders, manifest — Rust rewrite of the original Python tool." [dependencies] anyhow = "1" diff --git a/version-rust/README.md b/README.md similarity index 84% rename from version-rust/README.md rename to README.md index ec2f632..742f94d 100644 --- a/version-rust/README.md +++ b/README.md @@ -1,21 +1,19 @@ -# version-rust +# Warhammer DLP Helper -Rust rewrite of `version-python` (same repo, `main` branch still carries the -flat python layout). Goal: one concise binary + webapp, eventually replacing -the python tooling. +Torrent-first platform for Warhammer proxy models (3rd-party / fan sculpts): +export a Discord channel, extract the (often base64-encoded) download links, +download per host, extract the archives, sort the files into the library, and +record a manifest that will feed the torrent stage. One concise Rust binary + +webapp (the webapp is the next stage). -## Why this structure - -- `main` stays untouched (flat python layout) until the rewrite reaches parity. -- This branch re-homes the python code into `version-python/` (via `git mv`, - history preserved) and starts `version-rust/` beside it. -- `version-python/AGENTS.md` still documents the python stage — its rules - (secrets, failure policy, idempotency, rate limiting) apply to the rewrite - too and are the contract the Rust version must keep. +Ported from the original Python implementation, which this version replaced. +The `Rust port of …` provenance notes in the module headers name the Python +sources it was derived from; `AGENTS.md` holds the project rules and +`docs/SORTING.md` the library/sorting contract. ## Crate choices (as ported) -| Concern | Python today | Rust | +| Concern | Original Python | Rust | |---|---|---| | 7z extraction | 7z CLI, cp437 name repair, /var/tmp fallback | `sevenz-rust2` (pure Rust; LZMA/LZMA2/PPMD/AES-256 decrypt, v0.23, active) | | zip extraction | 7z CLI | `zip` crate | @@ -30,8 +28,7 @@ the python tooling. ## Port status -All python entry points are ported; live network paths are tested offline -only (see "Untested live paths"). +All original Python entry points are ported and live-validated (see below). | Rust module | Python source | Tests | |---|---|---| @@ -75,13 +72,12 @@ Notes: - The Discord token goes `--token` → `$DISCORD_TOKEN` → hidden prompt, and is only ever passed to the DCE container via env, never argv. -- `sorter --catalog` defaults to `version-python/data/gwshop_catalog.json` - (falling back to `data/gwshop_catalog.json`), both relative to the current - working directory — pass `--catalog` when running elsewhere. +- `sorter --catalog` defaults to `data/gwshop_catalog.json`, relative to the + current working directory — pass `--catalog` when running elsewhere. - `sorter --audit` defaults to `/../_logs/sorter_applied.jsonl` (bookkeeping stays out of the library, which may be a seeded torrent). -- The sorter implements the rules in `version-python/docs/SORTING.md` — that - document is still the working agreement; change it first, then both codes. +- The sorter implements the rules in `docs/SORTING.md` — that document is the + working agreement; change it first, then the code. - Retry loops, cooldowns and the "one gofile guest account per run" rule are ported verbatim from `dlp.py`; a failure in one URL never aborts the run. - Ctrl-C: state.json is saved after every URL; re-run to resume (the python diff --git a/version-python/data/gwshop_catalog.json b/data/gwshop_catalog.json similarity index 100% rename from version-python/data/gwshop_catalog.json rename to data/gwshop_catalog.json diff --git a/version-rust/data/warhammer_data.json b/data/warhammer_data.json similarity index 100% rename from version-rust/data/warhammer_data.json rename to data/warhammer_data.json diff --git a/version-python/docs/SORTING.md b/docs/SORTING.md similarity index 80% rename from version-python/docs/SORTING.md rename to docs/SORTING.md index 72e2b70..7e9523c 100644 --- a/version-python/docs/SORTING.md +++ b/docs/SORTING.md @@ -7,8 +7,7 @@ sorting-related code; change it here first, then in code. ## Target hierarchy (Games Workshop webshop-aligned) The hierarchy follows the official GW webshop taxonomy (dumped from the -shop's Algolia index by `dlphelper/gwshop.py` into -`data/gwshop_catalog.json`). Chapters and sub-lines that the old miniset +shop's Algolia index by `dlphelper gwshop` into `data/gwshop_catalog.json`). Chapters and sub-lines that the old miniset hierarchy treated as sub-factions are full GW factions, and the class layer holds GW shop categories (unit types / product lines) instead of the retired 40k force-organization slots: @@ -36,12 +35,11 @@ Layer rules: - Files without a faction but with a game land in `/_unsorted/`. - Fully unclassifiable files land in `/_unsorted/` — never guessed. -- The keyword tables in `warhammer_data.py` (GW naming) provide the initial - system/faction/class signal; a catalog match then overrides the hierarchy - from the catalog entry, which is authoritative. -- `data/library_catalog.json` (the older miniset-aligned hand catalog) is - still usable via `sorter.py --catalog`; its sub-faction entries map into - the same schema. For GW-sourced entries `subfaction` stays empty. +- The keyword tables in `data/warhammer_data.json` (GW naming) provide the + initial system/faction/class signal; a catalog match then overrides the + hierarchy from the catalog entry, which is authoritative. +- The catalog is pluggable via `dlphelper sorter --catalog`; every entry + uses the same schema. For GW-sourced entries `subfaction` stays empty. ## Canonical names @@ -63,15 +61,15 @@ graph LR ``` 1. **Inspect** — read the file name; for archives, also read the *inner* - file list (zip via stdlib `zipfile`, 7z via the installed `7z l` binary — - no Python dependency). Inner names improve classification for hash-named - archives and are recorded in the report. + file list (zip via the `zip` crate, 7z via `sevenz-rust2` — no external + binary). Inner names improve classification for hash-named archives and + are recorded in the report. 2. **Classify** — game/faction/class from the keyword tables - (`warhammer_data.py`), same as `catalogue.py`. -3. **Match** — fuzzy-match the (inner) name against `data/library_catalog.json` - entries within the same faction; best sequence-match above - `--threshold` (default 0.62) wins. The catalog provides canonical names, - sub-factions and classes. + (`data/warhammer_data.json`). +3. **Match** — fuzzy-match the (inner) name against the catalog + (`data/gwshop_catalog.json`) entries within the same faction; best + sequence-match above `--threshold` (default 0.62) wins. The catalog + provides canonical names, sub-factions and classes. 4. **Plan** — default output: a dry-run table `source → target` plus `sorter_plan.jsonl`. Nothing is touched. 5. **Apply** — create the target and **hardlink** the file there @@ -104,12 +102,12 @@ confidence doc always lands next to the audit file. unpack step will live in the sorter (`--unpack`, 7z binary) but stays off until the link-based flow is proven on a small batch. -## Extraction stage (`tools/extract_all.py`) +## Extraction stage (`dlphelper extract`) -That endgame stage now exists as a standalone tool (the sorter stays +That endgame stage now exists as a standalone command (the sorter stays plan/apply-only): - python3 tools/extract_all.py ... --out + dlphelper extract ... --out - Every archive (zip/7z/rar/tar/gz/bz2/xz, also nested ones) is extracted into a folder named after the archive, preserving relative structure; @@ -127,7 +125,7 @@ plan/apply-only): `The Twin Lance/` it belongs to that unit) and placed together, keeping the archive's inner directory structure: - python3 sorter.py --library \ + dlphelper sorter --library \ --provenance /extract_manifest.jsonl --apply Operational notes from the first full run (2026-10-06, sources @@ -141,17 +139,16 @@ already-present duplicates, 0 copied): link, compare `st_nlink` (and content/size) — never inodes. Server-side links are real: a fresh `os.link` bumps nlink (2→3 in the checked case). - Filenames with raw cp437 bytes crash 7z direct-to-CIFS with exit 2 - ("Cannot open output file"); `extract_all.py` then extracts to a - `/var/tmp` temp dir, repairs the names and moves the result over. + ("Cannot open output file"); the extractor then extracts to a `/var/tmp` + temp dir, repairs the names and moves the result over. - `__MACOSX/` and `._*` AppleDouble entries are deleted as junk, not extracted. -- Files under hidden trees (`.broken/`) are never planned; retries of the - truncated/HTML dead-link downloads are listed in - `/.broken/RETRY_URLS.txt` (all 0807.st; re-run - `python3 dlp.py --urls-file ...` for them). +- Files under hidden trees (`.broken/`) are never planned; failed archives + (HTML-page fake zips, password archives, disk full) are quarantined there + for inspection and the run continues. - **Non-warhammer systems are dropped** (2026-10-07, user decision: - "drop anything that isn't really warhammer"). The sorter default - (`DROPPED_SYSTEMS` in `dlphelper/sorter.py`) is `{Middle-Earth}`: + ("drop anything that isn't really warhammer"). The sorter default + (`DROPPED_SYSTEMS` in `src/sorter.rs`) is `{Middle-Earth}`: catalog entries of dropped systems are filtered at load and keyword hits for them count as unclassified, so nothing is ever placed into those sections - the files stay in `_unsorted`. `--drop-system NAME` @@ -168,7 +165,7 @@ already-present duplicates, 0 copied): unit's class layer always wins (no keyword class survives next to a precise unit match). -## Catalog file (`data/library_catalog.json`) +## Catalog file (`data/gwshop_catalog.json`) ```json { diff --git a/version-rust/src/catalogue.rs b/src/catalogue.rs similarity index 99% rename from version-rust/src/catalogue.rs rename to src/catalogue.rs index ca0619b..194f634 100644 --- a/version-rust/src/catalogue.rs +++ b/src/catalogue.rs @@ -1,5 +1,5 @@ //! Catalogue: classify collected files into game system / faction / class — -//! Rust port of `version-python/dlphelper/catalogue.py`. +//! Rust port of `dlphelper/catalogue.py`. //! //! The catalogue is the bridge between "a pile of downloads" and the later //! library stage: it reads manifest.jsonl (with full source attribution) and/or @@ -745,7 +745,7 @@ pub fn build_catalogue( } // --------------------------------------------------------------------------- -// Tests (offline port of version-python/tests/test_catalogue.py) +// Tests (offline port of tests/test_catalogue.py) // --------------------------------------------------------------------------- #[cfg(test)] diff --git a/version-rust/src/difflib.rs b/src/difflib.rs similarity index 100% rename from version-rust/src/difflib.rs rename to src/difflib.rs diff --git a/version-rust/src/discord.rs b/src/discord.rs similarity index 99% rename from version-rust/src/discord.rs rename to src/discord.rs index 8169f7e..7db2a80 100644 --- a/version-rust/src/discord.rs +++ b/src/discord.rs @@ -1,5 +1,5 @@ //! DCE docker export (JSON) + base64/plain URL extraction — Rust port of -//! `version-python/dlphelper/discord.py`. +//! `dlphelper/discord.py`. //! //! The channels this tool targets encode their download links as base64 inside //! message content, and occasionally as plain links. We export the channel as @@ -324,7 +324,7 @@ mod tests { STANDARD.encode(s.as_bytes()) } - /// The two-message fixture from version-python/tests/test_extract.py. + /// The two-message fixture from tests/test_extract.py. fn fixture_messages() -> Value { json!([ { diff --git a/version-rust/src/extract.rs b/src/extract.rs similarity index 99% rename from version-rust/src/extract.rs rename to src/extract.rs index 92bda17..b0c65c7 100644 --- a/version-rust/src/extract.rs +++ b/src/extract.rs @@ -1,5 +1,5 @@ //! Recursive archive extraction into a plain-file tree — Rust port of -//! `version-python/tools/extract_all.py` (extract_manifest.jsonl feeds the +//! `tools/extract_all.py` (extract_manifest.jsonl feeds the //! sorter's provenance mode; failures go to .broken/; the run never stops). //! //! Extraction backends: `sevenz-rust2` for .7z, `zip` for .zip, `unrar` for diff --git a/version-rust/src/globalstore.rs b/src/globalstore.rs similarity index 99% rename from version-rust/src/globalstore.rs rename to src/globalstore.rs index b2f3b05..04aeb44 100644 --- a/version-rust/src/globalstore.rs +++ b/src/globalstore.rs @@ -1,5 +1,5 @@ //! Cross-run, cross-channel record of completed URLs — Rust port of -//! `version-python/dlphelper/globalstore.py`. +//! `dlphelper/globalstore.py`. //! //! state.json only knows about one OUT_DIR; this store (default //! `~/.local/share/dlphelper/urls.json`) records every successfully diff --git a/version-rust/src/gofile.rs b/src/gofile.rs similarity index 99% rename from version-rust/src/gofile.rs rename to src/gofile.rs index d4a6646..faef2e7 100644 --- a/version-rust/src/gofile.rs +++ b/src/gofile.rs @@ -1,5 +1,5 @@ //! GoFile API client (one guest account per run) — Rust port of -//! `version-python/dlphelper/gofile.py`. +//! `dlphelper/gofile.py`. //! //! The API interaction mirrors Lysagxra/GoFileDownloader (same headers, //! website-token scheme and endpoints), with the same two deliberate diff --git a/version-rust/src/gwshop.rs b/src/gwshop.rs similarity index 97% rename from version-rust/src/gwshop.rs rename to src/gwshop.rs index 9e614ab..498442b 100644 --- a/version-rust/src/gwshop.rs +++ b/src/gwshop.rs @@ -1,5 +1,5 @@ //! GW webshop catalog dump via Algolia — Rust port of -//! `version-python/dlphelper/gwshop.py`. +//! `dlphelper/gwshop.py`. //! //! The warhammer.com shop is behind an AWS WAF challenge for non-browser //! clients, but its product search runs on Algolia and the app-id plus the @@ -42,15 +42,11 @@ pub fn load_credentials() -> anyhow::Result<(String, String)> { if !app_id.is_empty() && !api_key.is_empty() { return Ok((app_id, api_key)); } - // python: algolia.json next to the repo root; the Rust crate root is - // version-rust/, so the repo root is its parent. CARGO_MANIFEST_DIR is - // baked at build time - use the executable's location upward search as - // fallback? Keep it simple and deterministic: repo root via - // CARGO_MANIFEST_DIR/../algolia.json, checked at runtime. + // algolia.json lives at the repo root, which is also the crate root, so + // CARGO_MANIFEST_DIR (baked at build time) points straight at it. Fall + // back to the current working directory for runs from elsewhere. let candidates = [ - PathBuf::from(env!("CARGO_MANIFEST_DIR")) - .parent() - .map(|p| p.join("algolia.json")), + Some(PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("algolia.json")), Some(PathBuf::from("algolia.json")), ]; for candidate in candidates.into_iter().flatten() { @@ -623,8 +619,6 @@ mod tests { .map(|v| v.is_empty()) .unwrap_or(true) && !PathBuf::from(env!("CARGO_MANIFEST_DIR")) - .parent() - .unwrap() .join("algolia.json") .exists() { diff --git a/version-rust/src/main.rs b/src/main.rs similarity index 96% rename from version-rust/src/main.rs rename to src/main.rs index 71cc3c5..36d1f8a 100644 --- a/version-rust/src/main.rs +++ b/src/main.rs @@ -1,10 +1,10 @@ //! dlphelper CLI — Rust rewrite of the python entry points: //! -//! - `dlphelper dlp ...` <- version-python/dlp.py -//! - `dlphelper catalogue ...` <- version-python/catalogue.py -//! - `dlphelper sorter ...` <- version-python/sorter.py -//! - `dlphelper extract ...` <- version-python/tools/extract_all.py -//! - `dlphelper gwshop ...` <- version-python/dlphelper/gwshop.py +//! - `dlphelper dlp ...` <- dlp.py +//! - `dlphelper catalogue ...` <- catalogue.py +//! - `dlphelper sorter ...` <- sorter.py +//! - `dlphelper extract ...` <- tools/extract_all.py +//! - `dlphelper gwshop ...` <- dlphelper/gwshop.py //! //! Secrets rule: the Discord token is resolved via --token / $DISCORD_TOKEN / //! hidden prompt and only ever handed to the DCE container via env. @@ -160,8 +160,7 @@ struct SorterArgs { /// library root the files will be linked into #[arg(long, value_name = "DIR")] library: PathBuf, - /// catalog FILE (default: version-python/data/gwshop_catalog.json, - /// falling back to data/gwshop_catalog.json, both relative to the cwd) + /// catalog FILE (default: data/gwshop_catalog.json, relative to the cwd) #[arg(long, value_name = "FILE")] catalog: Option, /// fuzzy match threshold (default 0.62) @@ -254,15 +253,7 @@ fn make_gstore(args: &DlpArgs) -> globalstore::GlobalStore { } fn default_catalog_path() -> PathBuf { - let v = PathBuf::from("version-python/data/gwshop_catalog.json"); - if v.exists() { - return v; - } - let alt = PathBuf::from("data/gwshop_catalog.json"); - if alt.exists() { - return alt; - } - v // surfaces a clear error in load_catalog + PathBuf::from("data/gwshop_catalog.json") } fn cmd_dlp(args: &DlpArgs) -> anyhow::Result<()> { diff --git a/version-rust/src/manifest.rs b/src/manifest.rs similarity index 98% rename from version-rust/src/manifest.rs rename to src/manifest.rs index bf67410..2e5fb41 100644 --- a/version-rust/src/manifest.rs +++ b/src/manifest.rs @@ -1,5 +1,5 @@ //! manifest.jsonl — one compact JSON line per downloaded file — Rust port of -//! `append_manifest` from `version-python/dlphelper/state.py`. +//! `append_manifest` from `dlphelper/state.py`. //! //! Row layout (all attribution fields optional except channel/url): //! `{"channel", "url", "host", "message_id", "timestamp", "author", "path", diff --git a/version-rust/src/orchestrate.rs b/src/orchestrate.rs similarity index 99% rename from version-rust/src/orchestrate.rs rename to src/orchestrate.rs index 1075b80..b0cec91 100644 --- a/version-rust/src/orchestrate.rs +++ b/src/orchestrate.rs @@ -1,5 +1,5 @@ //! Orchestration: per-channel pipeline, gofile/0807 loops, subscriptions, -//! --assume-done — Rust port of the logic in `version-python/dlp.py`. +//! --assume-done — Rust port of the logic in `dlp.py`. //! //! Failure policy (the core requirement): a failure in one URL must never //! abort the run. Catch, record in state.json, continue. Cooldown pauses diff --git a/version-rust/src/sorter.rs b/src/sorter.rs similarity index 99% rename from version-rust/src/sorter.rs rename to src/sorter.rs index 7091a58..aabe518 100644 --- a/version-rust/src/sorter.rs +++ b/src/sorter.rs @@ -1,8 +1,8 @@ //! Library sorter: inspect/classify/fuzzy-match against the catalog, plan + //! hardlink apply, optional provenance unit mode — Rust port of -//! `version-python/dlphelper/sorter.py`. +//! `dlphelper/sorter.py`. //! -//! Matching rules are fixed (see version-python/AGENTS.md §7 "Sorter matching +//! Matching rules are fixed (see AGENTS.md §7 "Sorter matching //! rules"); they are ported exactly, including thresholds (CROSS_SYSTEM_MIN //! 0.75, WEAK_MIN 0.50, TOKENLESS_BONUS 0.08), the generic-token stoplist, //! junk-dir stoplist, deepest-dir re-homing and the weak tier. Fuzzy scores @@ -1732,7 +1732,7 @@ pub fn format_plan(items: &[SortItem]) -> String { mod tests { use super::*; - /// Same fixture as version-python/tests/test_sorter_units.py. + /// Same fixture as tests/test_sorter_units.py. fn test_catalog_json() -> String { serde_json::json!({ "entries": [ diff --git a/version-rust/src/state.rs b/src/state.rs similarity index 99% rename from version-rust/src/state.rs rename to src/state.rs index d3109f3..585ae97 100644 --- a/version-rust/src/state.rs +++ b/src/state.rs @@ -1,5 +1,5 @@ //! Per-run persistent state (state.json) — Rust port of -//! `version-python/dlphelper/state.py`. Makes runs resumable and failures +//! `dlphelper/state.py`. Makes runs resumable and failures //! trackable: re-runs skip URLs with status "ok", failed URLs are retried. //! //! Schema contract with the python version: diff --git a/version-rust/src/util.rs b/src/util.rs similarity index 100% rename from version-rust/src/util.rs rename to src/util.rs diff --git a/version-rust/src/warhammer_data.rs b/src/warhammer_data.rs similarity index 93% rename from version-rust/src/warhammer_data.rs rename to src/warhammer_data.rs index 4255f7a..a89e9eb 100644 --- a/version-rust/src/warhammer_data.rs +++ b/src/warhammer_data.rs @@ -1,10 +1,10 @@ //! Taxonomy: GW-named systems/factions, GW shop categories as classes, -//! GROUP_OF_FACTION map — Rust port of `version-python/dlphelper/warhammer_data.py`. +//! GROUP_OF_FACTION map — Rust port of `dlphelper/warhammer_data.py`. //! -//! The tables are NOT hand-transcribed: they are dumped from the python -//! module into `data/warhammer_data.json` (regenerate with -//! `version-rust/tools/dump_warhammer_data.py`) and parsed here, so both -//! versions work from byte-identical data. +//! The tables are NOT hand-transcribed: they were dumped from the original +//! python module into `data/warhammer_data.json` (the committed source of +//! truth) and are parsed here, so the data stayed byte-identical across the +//! port. //! //! Python-dict ordering is preserved exactly (insertion order; a duplicate //! key keeps its first insertion position while the *last* value wins), diff --git a/version-rust/src/zero807.rs b/src/zero807.rs similarity index 99% rename from version-rust/src/zero807.rs rename to src/zero807.rs index 977c94f..6942e14 100644 --- a/version-rust/src/zero807.rs +++ b/src/zero807.rs @@ -1,5 +1,5 @@ //! curl-based downloader for 0807.st direct links — Rust port of -//! `version-python/dlphelper/zero807.py`. +//! `dlphelper/zero807.py`. //! //! 0807.st serves the files directly but rejects requests that don't look //! like a browser (requires a User-Agent and the Referer of the site page), diff --git a/version-python/AGENTS.md b/version-python/AGENTS.md deleted file mode 100644 index 05694f4..0000000 --- a/version-python/AGENTS.md +++ /dev/null @@ -1,310 +0,0 @@ -# AGENTS.md — Warhammer_DLP_Helper - -Guidance for any AI agent (or human) working in this repository. Read this -fully before making changes. - -## 1. What this project is - -The user collects **Warhammer proxy models** (3rd-party / fan sculpts, never -official Games Workshop assets) that are shared in Discord channels. Links in -those channels point to two hosters and are often **base64-encoded** inside -message content (obfuscated by the posting bot). - -The end goal is a **torrent-first platform** (think radarr/lidarr, but for -proxy files). This repo is the current stage of that journey: a tool that - -1. exports a Discord channel/thread (DiscordChatExporter via docker), -2. extracts base64-encoded and plain download links, -3. picks the right downloader per host automatically, -4. downloads everything with per-file tracking, retries and resume, -5. records a manifest that will later feed the torrent stage. - -Deliberately **not** in scope (yet): .torrent creation, tracker upload, -deseeding, rename-into-library conventions. Design decisions should not close -the door on these — especially `manifest.jsonl` and the `--hash` option exist -for dedupe/cross-seeding later. - -## 2. Repository & environment - -- Remote: `git.dieterunterstrich.de/Ben_Kosytorz/Warhammer_DLP_Helper` — - **private** repo, branch `main`. Push with the credentials already stored - via git's credential store (`~/.git-credentials`); never paste them into - commands or files. -- The server is Gitea 1.27. To create *another* repo, use - `python3 tools/new_repo.py list|create ` (reads `~/.git-credentials` - itself, never prints secrets). Never delete or overwrite existing repos. -- Machine: Linux, Python 3.12+ (3.14 available), `requests` is the only - runtime dependency (see `requirements.txt`), docker is available. -- Reference clone: `~/GoFileDownloader` (Lysagxra) — do not modify it; it is - upstream reading material only. - -## 3. Architecture - -``` -dlp.py CLI orchestrator (entry point) -catalogue.py CLI: classify the collection into system/faction/class -sorter.py CLI: fuzzysort files into the library (plan/apply, - hardlinks, default catalog data/gwshop_catalog.json) -dlphelper/ - discord.py DCE docker export (JSON) + base64/plain URL extraction - gofile.py GoFile API client (one guest account per run) - zero807.py curl-based downloader for 0807.st direct links - state.py state.json store + manifest.jsonl appender - globalstore.py cross-run URL store (~/.local/share/dlphelper/urls.json) - catalogue.py scanner + keyword classifier + report writer - warhammer_data.py taxonomy: GW-named systems/factions, GW shop - categories as classes, GROUP_OF_FACTION map - gwshop.py GW webshop catalog dump via Algolia (see §7.3) - sorter.py library sorter: inspect/classify/fuzzy-match - against the catalog, plan + hardlink apply, - optional --provenance unit mode for extracted trees -parts/grab.sh legacy groundstone script (kept, not used by dlp.py) -tools/new_repo.py Gitea repo helper (reads ~/.git-credentials itself) -tools/debug_gofile.py inspect raw gofile API response for one content id -tools/extract_all.py recursive archive extraction into a plain-file tree - (extract_manifest.jsonl feeds the sorter's - --provenance unit mode; failures go to .broken/; - 7z-direct-to-CIFS failures fall back to a - /var/tmp extraction with cp437 name repair; - AppleDouble/__MACOSX junk is dropped) -tests/test_extract.py offline extraction test (plain python3, no deps) -tests/test_catalogue.py offline classifier test -``` - -Data flow: - -```mermaid -graph LR - A[channel id + token] --> B[DCE docker export -> export.json] - B --> C[decode base64 in message content] - C --> D{classify host} - D -->|gofile.io| E[GoFileClient: 1 guest account, list contents, download files] - D -->|0807.st| F[curl with UA + Referer] - D -->|other| G[recorded as failed: unsupported host] - E --> H[downloads/gofile/...] - F --> I[downloads/0807/...] - E & F --> J[state.json + manifest.jsonl + failed_urls.txt] - J --> K[globalstore: urls.json, marks URL done forever] -``` - -Key facts about the hosters (learned the hard way, do not re-learn): - -- **gofile.io**: guest accounts are created via `POST api.gofile.io/accounts`. - Creation is aggressively rate-limited — create **at most one account per - run**. Content listing needs `Authorization: Bearer `, an - `X-Website-Token` header (sha256 over `UA::en-US::::<4h-window>::seed`, - see `gofile.py`), and downloads need `Cookie: accountToken=` plus - Referer/Origin headers. -- **0807.st**: serves files directly but 403s anything that doesn't look like - a browser. `curl -A "Mozilla/5.0" -e "https://0807.st/"` works; aria2c's - default multi-connection request does not. curl is the tool here. -- Encoded links in the channels start as base64 strings matching - `aHR0cHM6Ly[89]...` — keep the extraction regex compatible with - `parts/grab.sh`. - -## 4. Usage (what "done" looks like) - - export DISCORD_TOKEN=... - python3 dlp.py /big/storage/warhammer/ - -Retry only failures: `python3 dlp.py --urls-file /failed_urls.txt` -(or just re-run — state.json skips everything already marked ok). - -Several channels at once (`subscriptions.json`: list of {name, channel_id, -out_dir}): - - python3 dlp.py --subs subscriptions.json - -Subs mode re-exports every channel by default (that is how new links get -picked up) and relies on state.json + the global store to only download -what is actually new. Use a cron/systemd timer for watch-like behavior. - -Manual import of the pre-dlp.py era (URLs you know are fully downloaded, -never re-fetch them again): - - python3 dlp.py --assume-done old_urls.txt - -## 5. Run artifacts (all in OUT_DIR, gitignored) - -| File | Purpose | -|------|---------| -| `export.json` | DCE export; cached — `--fresh` re-exports | -| `state.json` | url → {status, attempts, error, files[]}; corrupt files are backed up and reset | -| `failed_urls.txt` | rewritten each run: every non-ok URL, usable as `--urls-file` | -| `manifest.jsonl` | append-only, one JSON line per downloaded file (channel, url, message_id, timestamp, author, path, size, optional sha256) | -| `downloads/` | actual content; `.part` files mean "interrupted, will be resumed" | -| `~/.local/share/dlphelper/urls.json` | **global** cross-run store: every successfully downloaded URL is recorded once, regardless of OUT_DIR, so reposts in other channels are not re-downloaded. `--global-state` moves it, `--no-global` disables it for a run. | - -## 6. Rules - -### Secrets and tokens -1. Never print, log, commit or echo a Discord token or GoFile account token. - Not even partial. Strip them from any output you paste into chats. -2. The Discord token goes: `--token` flag → `$DISCORD_TOKEN` → hidden prompt. - It is handed to the DCE container via the `TOKEN` env var, never as a - command-line argument. -3. `~/.git-credentials` may only be read by `tools/new_repo.py`. Never copy - credentials into scripts, tests, or fixtures. -4. Nothing downloaded and no export/state/manifest file is ever committed. - -### Failure policy (the core requirement) -5. A failure in one URL or file must **never** abort the run. Catch, record in - `state.json`, continue. The loops in `dlp.py` therefore have catch-alls — - keep them. -6. Cooldowns are the only permitted "stall": after N consecutive failures, - pause, then continue. Circuit-breakers that exit the process need a - explicit justification in the PR/commit message. -7. Retries: exponential-ish (`retry_delay * attempt`), default 3 attempts. - Password-required gofile links are not retried (it can't help). - -### Idempotency -8. Re-runs must be safe and cheap: skip URLs marked `ok` (in `state.json` **or** - the global store), skip existing non-empty files, write partial downloads - to `*.part` and rename on success. -9. `state.json` is the source of truth for resume; keep its schema backward - compatible when touching `state.py` (unknown keys are preserved, missing - keys defaulted). Same for the global store schema in `globalstore.py`. -10. `--assume-done` is a deliberate, manual import step for old URL lists. - Never invoke it automatically, and never let the tool mark URLs done - without a verified successful download. - -### Rate limiting / network etiquette -11. One gofile guest account per run; `ensure_token()` backs off linearly - (20s × attempt). Do not raise the guest-account frequency "to make tests - pass". Testing account creation more than a couple times per hour is a - bug, not a test strategy. -12. Keep the 0807 referer/UA contract. Don't "optimize" it to aria2c without - re-verifying the 403 behavior. - -### Code style -13. Python 3.12+, type hints on public functions, `logging` not `print` in - library code (prints are fine only for `--dry-run` output). -14. Stay on stdlib + `requests`. New dependencies need a real justification - and belong in `requirements.txt`. -15. `dlphelper/` modules must stay import-safe (no side effects at import). - -### Validation -15. Minimum validation for any change: `python3 -m py_compile dlp.py - catalogue.py dlphelper/*.py`, plus the offline tests (`python3 - tests/test_extract.py`, `python3 tests/test_catalogue.py`) if the touched - modules changed. Document what you could *not* test (e.g. live - gofile/0807 downloads need real links and tokens). -17. Full live runs need a Discord token from the user — ask for it, never - ask for it to be pasted into a file or committed. - -### Git -18. Branch `main`, descriptive commit messages, no force-push, no history - rewriting on the shared remote. -19. When creating a new repo use `tools/new_repo.py` (Gitea API); creation is - fine, deletion/renaming of existing repos requires the user. - -## 7. Roadmap (future stages, keep compatible) - -1. **Dedupe**: hash-based (sha256) duplicate detection across threads via - `manifest.jsonl` (the `--hash` flag feeds this). URL-level dedupe already - exists via the global store; content-level (same file, different link) - comes with hashing. -2. **Torrent stage**: `mktorrent` per downloaded folder, .torrent files into - a watch dir, cross-seeding between direct-download and torrent copies. -3. **Library organization (started)**: `catalogue.py` classifies every file - into system / faction / class and can move files into - `/////` with `--apply` (audited in - `catalogue_moves.jsonl`). The taxonomy lives in - `dlphelper/warhammer_data.py`; it is keyword-based and grows with the - collection - unclassified files are reported, never guessed. Manual fixes - go into an overrides JSON (`--overrides`). The library hierarchy follows - the **GW webshop taxonomy** (see below); `docs/SORTING.md` is the - working agreement for the sorting pipeline. - - **Sorter matching rules (fixed 2026-10-05 after the first full run):** - the outer file name wins when it scores >= threshold; inner archive - names only vote as fallback (hash-named gofile archives need that). A - fuzzy match never jumps into another game's section: keyword-identified - systems constrain the catalog pool, and a file with neither system nor - faction keyword only adopts a catalog unit on a strong hit - (`CROSS_SYSTEM_MIN = 0.75`) - weaker guesses stay in `_unsorted`. - Token-containment boosts exist for short names vs long catalog names - ("telemon" → "Telemon Heavy Dreadnought"), guarded by a generic-token - stoplist. - - **Refined 2026-10-07 (dir re-homing + weak tier + non-warhammer drop):** - - **Deepest-matching-dir wins**: ancestor directory names (inside the - archive in provenance mode, real dirs in manual trees) compete with - the pack/file name; the best accepted match re-homes the whole - subtree. Junk dir names (files/images/HEADS/pure numbers/lone generic - words) are never candidates. Fixes multi-unit sculptor packs (one - "Retro Elvez" archive ≈ 20 Eldar units no longer all land in one - keyword class). - - **Weak tier**: when nothing reaches the threshold, a hit >= `WEAK_MIN` - (0.50) is accepted but flagged low confidence - `[w]` in the plan, - `confidence: low` in the audit, listed in - `sorter_low_confidence.txt` in the log dir (see below). Weak and tokenless adoptions - require a shared non-generic token; tokenless normal adoption needs - +0.08 (`TOKENLESS_BONUS`); cross-system adoption additionally needs - token containment (`_token_contained`), because char-ratio lies - ("aeldari rangers" vs "dwarf rangers" = 0.786). - - **Non-warhammer systems dropped**: `DROPPED_SYSTEMS` (default - `{Middle-Earth}`) - catalog entries filtered at load, keyword hits - count as unclassified; files stay in `_unsorted`. `--drop-system`/ - `--keep-system` adjust. The catalog snapshot keeps all systems. - - **Probe votes on full inner paths** (2026-10-07, after run 5): pack - probes clean the whole relative path, so inner dir names vote - ("Catachan/Jungle Fighters/" resolves Astra Militarum). No - single-token containment boost for inner votes (a root file named - "lieutenant" once claimed ~7000 files into one Iron Hands unit); - full-catalog and different-faction adoptions need token - containment (`_contained_somewhere`). - - - **Bookkeeping location (fixed 2026-10-08)**: the audit and the - low-confidence doc are written to `/../_logs/` by default - (`--audit` overrides) - never inside the library itself, because the - library may be a seeded torrent whose file set is frozen. - - The catalog deliberately covers all GW systems (needed for - catalog-agnostic use) - wrong-system placement is prevented by matching - discipline and the drop filter, not by shrinking the catalog. - - **Catalog source: the GW webshop via Algolia (`dlphelper/gwshop.py`, - working, verified 2026-10-04 — how it was obtained, do not re-derive):** - scraping warhammer.com directly is blocked — the shop HTML and all - search/listing endpoints (`/en-WW/shop/*`, - `/app/resources/catalog/search/*`, `/app/resources/catalog/categories`) - sit behind an **AWS WAF** challenge (405/202 "goku" JS captcha) for any - non-browser client, regardless of User-Agent; the sitemap declared in - robots.txt does not exist; deep paths under the allowed - `/app/resources/catalog/product/*` reach a separate commerce backend - (plain `404 Not found`) but the resource shape is unknown. What works: - the shop's product search runs on **Algolia** (app-id `M5ZIQZNQ2H`) and - its app-id plus the **public search-only API key** ship with the browser - page; the user captured both from the browser devtools ("Copy as cURL" - on one search request) and they are stored in the gitignored - `algolia.json` (or `ALGOLIA_APP_ID`/`ALGOLIA_API_KEY` env vars) — the - key is search-only, never commit any other Algolia key. The full - product index `prod-lazarus-product-en-ww` (~3500 products, plus one - sibling index per locale) is dumped with plain `requests`: the - public search-only key cannot use Algolia's `/browse` (403) and normal - pagination is capped at 1000 cumulative hits, so `gwshop.py` - **partitions the index on the `GameSystemsRoot` hierarchical facet** — - each branch is fetched with an explicit filter, recursing deeper only - when a branch exceeds one page. (Live count 2026-10-04: 4183 records, - of which 3491 are reachable via the facet; the ~692 left out are paints, - Black Library books, White Dwarf issues, brushes, bases and gifts that - carry no `GameSystemsRoot` path — no miniature kits among them.) Records are normalized into the - source-agnostic catalog schema (system/group/faction/subfaction/class - = the facet path, name/sku/productType attached) into - `data/gwshop_catalog.json`, which is the sorter's default catalog and - is **committed as a snapshot**; refresh with - `python3 -m dlphelper.gwshop data/gwshop_catalog.json`. Other catalog - sources stay pluggable and source-agnostic (miniset.net — reachable - but stale, manual JSON overrides, `data/library_catalog.json`). -4. **Watch mode**: `--subs subscriptions.json` + a cron/timer already gives a - primitive version; a real watch mode (per-subscription schedules, tear-off - notifications, discord pings on new links) builds on it. - -## 8. Testing notes - -- `tests/` currently holds a synthetic `export.json` test for extraction - (`tests/test_extract.py`, runnable with plain `python3`). Keep tests - dependency-free and offline — they must never hit gofile/0807/Discord. -- To exercise downloads without Discord: build a `--urls-file` from known - URLs and run with a scratch OUT_DIR. diff --git a/version-python/README.md b/version-python/README.md deleted file mode 100644 index c4a326b..0000000 --- a/version-python/README.md +++ /dev/null @@ -1,111 +0,0 @@ -# Warhammer DLP Helper - -A small "radarr for Warhammer proxy files" tool: point it at a Discord -channel/thread and an output directory, and it exports the channel, extracts -the encoded download links, and downloads everything - automatically choosing -the right downloader per host: - -| Host | Downloader | Notes | -|-------------|------------|-------| -| gofile.io | built-in GoFile API client | one guest account per run (rate-limit friendly) | -| 0807.st | curl | needs browser-like headers (`-A`, `-e https://0807.st/`) | - -## Install - - pip install -r requirements.txt # just `requests` - docker pull tyrrrz/discordchatexporter:stable - -## Usage - - export DISCORD_TOKEN="..." - python3 dlp.py /path/to/output - -The token can also be passed with `--token` (or you'll be prompted, input -hidden). Useful options: - - --urls-file failed_urls.txt retry a previous failure list, no re-export - --password for password-protected gofile links - --only gofile|0807 process only one host - --extract unzip archives into per-archive folders - --hash record sha256 per file in the manifest - --dry-run just show what would be downloaded - --fresh force a fresh channel export - -## Several channels / watching for updates - -Put your channels into `subscriptions.json`: - -```json -{ - "subscriptions": [ - {"name": "some-eldar", - "channel_id": "1455993943471820842", - "out_dir": "/big/storage/warhammer/some-eldar"} - ] -} -``` - -then run (e.g. from a daily cron/systemd timer): - - python3 dlp.py --subs subscriptions.json - -Subs mode re-exports each channel to pick up new links; `state.json` and the -global store make sure only genuinely new URLs are downloaded. - -## Cataloguing the collection - -`catalogue.py` classifies every downloaded file into game system, faction and -force-organization class (HQ / Troops / Elites / Fast Attack / Heavy Support / -Flyer / Dedicated Transport / Lords of War / Fortification) using a keyword -taxonomy in `dlphelper/warhammer_data.py`: - - # preview a dlp.py run dir (manifest + downloads) - python3 catalogue.py /path/to/run_dir - - # include the manual-era directories - python3 catalogue.py /path/to/run_dir --scan /big/storage/manual --out-md catalogue.md - - # catalogue arbitrary dirs only - python3 catalogue.py --scan /big/storage/manual - - # actually reorganize into a library tree (audited, only classified files) - python3 catalogue.py --apply --library /big/storage/warhammer_library - -Outputs `catalogue.md` (readable) and `catalogue.jsonl` (machine, with source -attribution per file). Unclassified files are listed, never guessed; fix them -with an overrides JSON (`--overrides overrides.json`). - -## Already downloaded the old way? - -`--assume-done` imports a plain URL list into the global store so those links -are never fetched again (no verification — use deliberately): - - python3 dlp.py --assume-done old_urls.txt - -## Output layout - - OUT_DIR/ - export.json Discord channel export (cached between runs) - state.json per-URL status; re-runs skip everything marked "ok" - failed_urls.txt URLs that failed after all retries - manifest.jsonl one line per downloaded file (url, source message, path, size) - downloads/ - gofile/// - 0807/.zip -| `downloads/` | actual content; `.part` files mean "interrupted, will be resumed" | -| `~/.local/share/dlphelper/urls.json` | global cross-run store of finished URLs (never re-downloaded, in any channel) | - -Runs are resumable and idempotent: interrupted downloads use `.part` files and -finished ones are skipped on the next run. Individual failures never stop the -run; after 3 consecutive failures it pauses for a cooldown (rate-limit -protection) and continues. - -## Why not use Lysagxra/GoFileDownloader directly? - -That tool creates a new GoFile guest account **per URL** and calls -`sys.exit(1)` on the first failure, which is exactly why runs died after ~6 -downloads. This project reimplements its (working) API scheme - headers, -`X-Website-Token`, download links with the `accountToken` cookie - but creates -one guest account per run, records failures per file, and keeps going. - -`parts/grab.sh` is the original groundstone script, kept for reference. diff --git a/version-python/catalogue.py b/version-python/catalogue.py deleted file mode 100644 index d95927f..0000000 --- a/version-python/catalogue.py +++ /dev/null @@ -1,118 +0,0 @@ -#!/usr/bin/env python3 -"""Catalogue CLI: classify the collection into system / faction / class. - -Examples: - # catalogue the dlp.py run dir (manifest + downloads) - python3 catalogue.py /path/to/run_dir - - # additionally scan manual-era directories - python3 catalogue.py /path/to/run_dir --scan /big/storage/manual/Aeldari - - # catalogue ONLY manual dirs, without any run dir - python3 catalogue.py --scan /big/storage/manual --out-md catalogue.md - - # fix misclassifications (see overrides format below) - python3 catalogue.py --overrides overrides.json - - # actually move files into //// - python3 catalogue.py --apply --library /big/storage/warhammer_library - -overrides.json format (keys are file names or full paths, case-insensitive): - { - "some file.zip": {"faction": "Aeldari", "class": "Heavy Support", - "system": "Warhammer 40,000"} - } - -Outputs (default into the first given dir, or cwd): - catalogue.md human-readable, grouped system > faction > class - catalogue.jsonl one JSON line per file, with source attribution -""" - -from __future__ import annotations - -import argparse -import logging -import sys -from pathlib import Path - -from dlphelper.catalogue import ( - build_catalogue, - format_size, - apply_moves, - write_reports, -) - -log = logging.getLogger("catalogue") - - -def main() -> None: - parser = argparse.ArgumentParser( - description="Classify the Warhammer collection into factions and classes." - ) - parser.add_argument("run_dir", nargs="?", type=Path, - help="a dlp.py run dir (uses its manifest.jsonl and downloads/)") - parser.add_argument("--scan", action="append", default=[], type=Path, metavar="DIR", - help="additional directory to scan for archive files (repeatable)") - parser.add_argument("--manifest", type=Path, default=None, metavar="FILE", - help="explicit manifest.jsonl (default: /manifest.jsonl)") - parser.add_argument("--overrides", type=Path, default=None, metavar="FILE", - help="manual classification overrides (JSON)") - parser.add_argument("--out-md", type=Path, default=None, metavar="FILE") - parser.add_argument("--out-jsonl", type=Path, default=None, metavar="FILE") - parser.add_argument("--apply", action="store_true", - help="move files into the library structure (default: preview)") - parser.add_argument("--library", type=Path, default=None, metavar="DIR", - help="library root for --apply (required with --apply)") - args = parser.parse_args() - - logging.basicConfig(level=logging.INFO, - format="%(asctime)s %(levelname)-7s %(message)s", - datefmt="%H:%M:%S") - - run_dir = args.run_dir - manifests = [args.manifest] if args.manifest else ( - [run_dir / "manifest.jsonl"] if run_dir else [] - ) - scan_dirs = list(args.scan) - if run_dir and not args.manifest: - # run dir is scanned as a whole (covers files not yet in the manifest) - scan_dirs.append(run_dir) - - if not manifests and not scan_dirs: - parser.error("give a run_dir or at least one --scan directory") - - out_base = run_dir if run_dir else Path.cwd() - out_md = args.out_md or out_base / "catalogue.md" - out_jsonl = args.out_jsonl or out_base / "catalogue.jsonl" - - entries = build_catalogue(manifests, scan_dirs, args.overrides) - write_reports(entries, out_md, out_jsonl) - - classified = sum(1 for e in entries if e.faction != "Unclassified") - log.info("Catalogued %d files (%d classified) -> %s", len(entries), classified, out_md) - - # compact per-faction summary to the console - summary: dict[tuple[str, str], list[int]] = {} - for e in entries: - key = (e.system, e.faction) - n, size = summary.get(key, (0, 0)) - summary[key] = [n + 1, size + e.size] - for (system, faction), (n, size) in sorted(summary.items()): - log.info(" %-18s %-32s %4d files %s", system, faction, n, format_size(size)) - - if args.apply: - if not args.library: - parser.error("--apply needs --library DIR") - if not args.run_dir: - parser.error("--apply needs a run_dir (files are moved out of it)") - moved = apply_moves(entries, args.library, args.library / "catalogue_moves.jsonl") - log.info("Moved %d files into %s (audit: catalogue_moves.jsonl)", moved, args.library) - # rewrite reports with updated paths - write_reports(entries, out_md, out_jsonl) - - -if __name__ == "__main__": - try: - main() - except KeyboardInterrupt: - sys.exit(130) diff --git a/version-python/data/library_catalog.json b/version-python/data/library_catalog.json deleted file mode 100644 index 173cda0..0000000 --- a/version-python/data/library_catalog.json +++ /dev/null @@ -1,140 +0,0 @@ -{ - "comment": "Canonical library catalog. Fields: system/group/faction/subfaction/class/name/aliases. Names and aliases are matched case-insensitively; subfaction and class may be null. Aligned with miniset.net faction names.", - "entries": [ - { - "system": "Warhammer 40,000", - "group": "Armies of the Imperium", - "faction": "Adeptus Astartes", - "subfaction": "Space Wolves", - "class": "Fast Attack", - "name": "Thunderwolf Cavalry", - "aliases": ["thunder wolf", "thunderwolves", "thunder wolf cavalry"] - }, - { - "system": "Warhammer 40,000", - "group": "Armies of the Imperium", - "faction": "Adeptus Astartes", - "subfaction": "Space Wolves", - "class": "Elites", - "name": "Wulfen", - "aliases": ["wulfen dreadnought"] - }, - { - "system": "Warhammer 40,000", - "group": "Armies of Chaos", - "faction": "Chaos Daemons", - "subfaction": null, - "class": "HQ", - "name": "Keeper of Secrets", - "aliases": ["keeper of secrets and shalaxi helbane", "shalaxi helbane", "keepers of secrets"] - }, - { - "system": "Warhammer 40,000", - "group": "Xenos Armies", - "faction": "Drukhari", - "subfaction": null, - "class": null, - "name": "The Twin Lance", - "aliases": ["twin lance"] - }, - { - "system": "Warhammer 40,000", - "group": "Armies of the Imperium", - "faction": "Adeptus Astartes", - "subfaction": null, - "class": "Elites", - "name": "Terminator Squad", - "aliases": ["terminators", "terminator squad"] - }, - { - "system": "Warhammer 40,000", - "group": "Armies of the Imperium", - "faction": "Adeptus Astartes", - "subfaction": null, - "class": "Elites", - "name": "Dreadnought", - "aliases": ["dread"] - }, - { - "system": "Warhammer 40,000", - "group": "Armies of the Imperium", - "faction": "Adeptus Astartes", - "subfaction": null, - "class": "HQ", - "name": "Captain", - "aliases": ["captain in gravis armour", "primaris captain"] - }, - { - "system": "Warhammer 40,000", - "group": "Armies of the Imperium", - "faction": "Adeptus Astartes", - "subfaction": null, - "class": "Heavy Support", - "name": "Hellblaster Squad", - "aliases": ["hellblasters"] - }, - { - "system": "Warhammer 40,000", - "group": "Armies of the Imperium", - "faction": "Adeptus Astartes", - "subfaction": null, - "class": "Troops", - "name": "Intercessor Squad", - "aliases": ["intercessors"] - }, - { - "system": "Warhammer 40,000", - "group": "Xenos Armies", - "faction": "T'au Empire", - "subfaction": null, - "class": "HQ", - "name": "Commander Farsight", - "aliases": ["farsight"] - }, - { - "system": "Warhammer 40,000", - "group": "Xenos Armies", - "faction": "T'au Empire", - "subfaction": null, - "class": "HQ", - "name": "Commander Shadowsun", - "aliases": ["shadowsun"] - }, - { - "system": "Warhammer 40,000", - "group": "Xenos Armies", - "faction": "T'au Empire", - "subfaction": null, - "class": "Fast Attack", - "name": "Pathfinder Team", - "aliases": ["pathfinders"] - }, - { - "system": "Warhammer 40,000", - "group": "Xenos Armies", - "faction": "T'au Empire", - "subfaction": null, - "class": "Dedicated Transport", - "name": "Devilfish", - "aliases": ["devil fish"] - }, - { - "system": "Warhammer 40,000", - "group": "Xenos Armies", - "faction": "T'au Empire", - "subfaction": null, - "class": null, - "name": "Assorted Drones", - "aliases": ["drones"] - }, - { - "system": "Warhammer 40,000", - "group": "Armies of the Imperium", - "faction": "Adepta Sororitas", - "subfaction": null, - "class": null, - "name": "Battle Sisters", - "aliases": ["battle sister squad", "battle sisters squad"] - } - ] -} diff --git a/version-python/dlp.py b/version-python/dlp.py deleted file mode 100644 index 0be4071..0000000 --- a/version-python/dlp.py +++ /dev/null @@ -1,544 +0,0 @@ -#!/usr/bin/env python3 -"""Warhammer DLP Helper. - -Give it a Discord channel/thread id and an output directory; it exports the -channel, extracts the encoded download links (gofile.io and 0807.st), picks -the right downloader for each link, and keeps going when individual links -fail. Everything is recorded in state.json; failures also land in -failed_urls.txt so a re-run (or --urls-file) retries exactly those. - -Already-downloaded URLs are recorded in a global store (default -~/.local/share/dlphelper/urls.json), so a link that was downloaded for one -channel is never re-downloaded for another - and re-runs of the same channel -skip everything finished (see also --assume-done for importing old lists). - -Usage: - python3 dlp.py [options] - python3 dlp.py --subs subscriptions.json [options] - python3 dlp.py --urls-file failed_urls.txt - python3 dlp.py --assume-done old_urls.txt # import past downloads -""" - -from __future__ import annotations - -import argparse -import getpass -import hashlib -import json -import logging -import os -import random -import sys -import time -import zipfile -from itertools import chain -from pathlib import Path - -from dlphelper import discord as dlp_discord -from dlphelper import gofile as dlph_gofile -from dlphelper import zero807 as dlph_zero807 -from dlphelper.globalstore import GlobalStore, default_global_path -from dlphelper.state import State, append_manifest - -log = logging.getLogger("dlp") - - -def sha256_of(path: Path) -> str: - digest = hashlib.sha256() - with path.open("rb") as fh: - for chunk in iter(lambda: fh.read(1024 * 1024), b""): - digest.update(chunk) - return digest.hexdigest() - - -def resolve_token(cli_value: str | None) -> str: - if cli_value: - return cli_value - if os.environ.get("DISCORD_TOKEN"): - return os.environ["DISCORD_TOKEN"] - log.info("No Discord token given ($DISCORD_TOKEN); prompting (input hidden).") - token = getpass.getpass("Discord token: ") - if not token: - raise SystemExit("No token provided, aborting.") - return token - - -def read_url_lines(path: Path) -> list[str]: - return [ - line.strip() - for line in path.read_text(encoding="utf-8").splitlines() - if line.strip() and not line.strip().startswith("#") - ] - - -def load_urls_file(path: Path) -> list[dlp_discord.FoundUrl]: - """Load a plain URL list (e.g. failed_urls.txt); no attribution available.""" - found: dict[str, dlp_discord.FoundUrl] = {} - for url in read_url_lines(path): - host = dlp_discord.classify(url) - if host is None: - log.warning("Unsupported host in %s: %s", path.name, url) - continue - found.setdefault(url, dlp_discord.FoundUrl(url=url, host=host)) - return sorted(found.values(), key=lambda f: f.url) - - -def record_files( - state: State, - gstore: GlobalStore | None, - url: str, - files: list[dict], - source: dict, -) -> None: - state.mark(url, status="ok", error=None, files=files) - if gstore is not None: - gstore.mark(url, channel=source.get("channel"), host=source.get("host"), - file_count=len(files)) - for f in files: - append_manifest(Path(source["manifest"]), { - "channel": source["channel"], - "url": url, - "host": source.get("host"), - "message_id": source.get("message_id"), - "timestamp": source.get("timestamp"), - "author": source.get("author"), - **f, - }) - - -def extract_zips(files: list[dict]) -> None: - """Optional: unzip each downloaded archive next to itself.""" - for f in files: - path = Path(f["path"]) - if path.suffix.lower() != ".zip" or not path.exists(): - continue - target = path.with_suffix("") - if target.exists() and any(target.iterdir()): - log.info("Skip extracting %s (target exists)", path.name) - continue - log.info("Extracting %s -> %s", path.name, target.name) - try: - target.mkdir(parents=True, exist_ok=True) - with zipfile.ZipFile(path) as zf: - zf.extractall(target) - except (zipfile.BadZipFile, OSError) as exc: - log.warning("Extraction of %s failed: %s", path.name, exc) - - -def run_gofile( - urls: list[dlp_discord.FoundUrl], - args: argparse.Namespace, - state: State, - source_by_url: dict[str, dict], - is_done, - gstore: GlobalStore | None, -) -> None: - if not [f for f in urls if not is_done(f.url)]: - return # nothing pending: no guest account creation, no API calls - client = dlph_gofile.GoFileClient(password=args.password) - try: - client.ensure_token() - except dlph_gofile.GoFileError as exc: - log.error("%s - marking all pending gofile links as failed.", exc) - for f in urls: - if is_done(f.url): - continue - state.mark(f.url, status="failed", error=str(exc), files=[]) - return - - consecutive_failures = 0 - for f in urls: - if is_done(f.url): - log.info("Skip (already downloaded): %s", f.url) - continue - source = source_by_url[f.url] - - for attempt in range(1, args.retries + 1): - try: - files_spec = client.list_files(f.url) - if not files_spec: - raise dlph_gofile.GoFileEmpty("content exists but contains no files") - files: list[dict] = [] - for relpath, name, link in files_spec: - dest = Path(args.out_dir) / "downloads" / "gofile" / relpath / name - size = client.download_file(link, dest) - entry = {"path": str(dest), "size": size, "name": name} - if args.hash: - entry["sha256"] = sha256_of(dest) - files.append(entry) - log.info("OK %s (%d files)", f.url, len(files)) - record_files(state, gstore, f.url, files, source) - if args.extract: - extract_zips(files) - consecutive_failures = 0 - break - except dlph_gofile.GoFilePasswordRequired as exc: - # Retrying without a password never helps - log.error("PASSWORD REQUIRED %s: %s", f.url, exc) - state.mark(f.url, status="failed", error=str(exc), files=[]) - break - except dlph_gofile.GoFileNotFound as exc: - # Dead links never come back - no retry - log.error("DEAD LINK %s: %s", f.url, exc) - state.mark(f.url, status="failed", error=str(exc), files=[]) - break - except dlph_gofile.GoFileEmpty as exc: - # Emptied content is stable - no retry - log.error("EMPTY CONTENT %s: %s", f.url, exc) - state.mark(f.url, status="failed", error=str(exc), files=[]) - break - except dlph_gofile.GoFileAuthError as exc: - # Token may have died mid-run (rate limit); re-auth once per attempt - log.warning("Auth error for %s: %s - re-authenticating", f.url, exc) - state.mark(f.url, error=str(exc), files=[]) - client.reset() - try: - client.ensure_token() - except dlph_gofile.GoFileError as reauth_exc: - log.error("Re-authentication failed: %s", reauth_exc) - if attempt < args.retries: - time.sleep(args.retry_delay * attempt) - except dlph_gofile.GoFileError as exc: - log.warning("Attempt %d/%d failed for %s: %s", - attempt, args.retries, f.url, exc) - state.mark(f.url, error=str(exc), files=[]) - if attempt < args.retries: - time.sleep(args.retry_delay * attempt) - except Exception as exc: # noqa: BLE001 - a bug must not stop the run - log.exception("Unexpected error for %s: %s", f.url, exc) - state.mark(f.url, error=f"unexpected: {exc}", files=[]) - if attempt < args.retries: - time.sleep(args.retry_delay * attempt) - except KeyboardInterrupt: - state.mark(f.url, error="interrupted", files=[]) - raise - else: - rec = state.records.get(f.url, {}) - state.mark(f.url, status="failed", error=rec.get("error", "unknown"), files=[]) - consecutive_failures += 1 - log.error("FAILED %s after %d attempts", f.url, args.retries) - if consecutive_failures >= args.cooldown_failures: - log.warning( - "%d failures in a row - cooling down %ds before continuing " - "(rate limit?). The run does not stop.", - consecutive_failures, args.cooldown_seconds, - ) - time.sleep(args.cooldown_seconds) - - -def run_0807( - urls: list[dlp_discord.FoundUrl], - args: argparse.Namespace, - state: State, - source_by_url: dict[str, dict], - is_done, - gstore: GlobalStore | None, -) -> None: - if not [f for f in urls if not is_done(f.url)]: - return - out_dir = Path(args.out_dir) / "downloads" / "0807" - consecutive_failures = 0 - for f in urls: - if is_done(f.url): - log.info("Skip (already downloaded): %s", f.url) - continue - source = source_by_url[f.url] - - for attempt in range(1, args.retries + 1): - try: - dest, size = dlph_zero807.download(f.url, out_dir) - entry: dict = {"path": str(dest), "size": size, "name": dest.name} - if args.hash: - entry["sha256"] = sha256_of(dest) - log.info("OK %s (%s)", f.url, format_size(size)) - record_files(state, gstore, f.url, [entry], source) - if args.extract: - extract_zips([entry]) - consecutive_failures = 0 - break - except dlph_zero807.Zero807NotFound as exc: - # Dead links never come back - no retry - log.error("DEAD LINK %s: %s", f.url, exc) - state.mark(f.url, status="failed", error=str(exc), files=[]) - break - except dlph_zero807.Zero807Error as exc: - log.warning("Attempt %d/%d failed for %s: %s", - attempt, args.retries, f.url, exc) - state.mark(f.url, error=str(exc), files=[]) - if attempt < args.retries: - time.sleep(args.retry_delay * attempt) - except Exception as exc: # noqa: BLE001 - a bug must not stop the run - log.exception("Unexpected error for %s: %s", f.url, exc) - state.mark(f.url, error=f"unexpected: {exc}", files=[]) - if attempt < args.retries: - time.sleep(args.retry_delay * attempt) - except KeyboardInterrupt: - state.mark(f.url, error="interrupted", files=[]) - raise - else: - rec = state.records.get(f.url, {}) - state.mark(f.url, status="failed", error=rec.get("error", "unknown"), files=[]) - consecutive_failures += 1 - log.error("FAILED %s after %d attempts", f.url, args.retries) - if consecutive_failures >= args.cooldown_failures: - log.warning( - "%d failures in a row - cooling down %ds before continuing " - "(HTTP 403 storm?). The run does not stop.", - consecutive_failures, args.cooldown_seconds, - ) - time.sleep(args.cooldown_seconds) - - -def format_size(n: int) -> str: - size = float(n) - for unit in ("B", "KiB", "MiB", "GiB", "TiB"): - if size < 1024 or unit == "TiB": - return f"{size:.1f} {unit}" - size /= 1024 - return f"{n} B" - - -def write_failed_urls(out_dir: Path, state: State) -> Path: - dest = out_dir / "failed_urls.txt" - dest.parent.mkdir(parents=True, exist_ok=True) - failures = state.failures() - dest.write_text("\n".join(failures) + ("\n" if failures else ""), encoding="utf-8") - return dest - - -def mark_assume_done(path: Path, gstore: GlobalStore | None) -> int: - """Record URLs as already downloaded (no files) so they are never fetched. - - Manual import step for the pre-dlp.py era: point it at an old URL list you - know is fully downloaded. There is no verification - garbage in, skipped - forever. - """ - if gstore is None: - raise SystemExit("--assume-done needs the global store (drop --no-global)") - count = 0 - for url in read_url_lines(path): - gstore.mark(url, channel=None, host=dlp_discord.classify(url), file_count=0, - note="assumed done via --assume-done") - count += 1 - log.info("Marked %d URLs as done in the global store (%s)", count, gstore.path) - return count - - -def process_channel( - channel_id: str, - out_dir: Path, - args: argparse.Namespace, - token: str | None, - gstore: GlobalStore | None, - force_export: bool = False, -) -> int: - """Run the full pipeline for one channel; returns the number of failures.""" - state = State(out_dir / "state.json") - manifest = out_dir / "manifest.jsonl" - - def is_done(url: str) -> bool: - return state.is_done(url) or (gstore is not None and gstore.is_done(url)) - - if args.urls_file: - found = load_urls_file(args.urls_file) - else: - if token is None: - raise SystemExit("internal error: no token resolved") - export_path = dlp_discord.export_channel( - channel_id, out_dir, token, force=force_export or args.fresh - ) - found = dlp_discord.extract_urls(export_path) - - if args.only: - found = [f for f in found if f.host == args.only] - - buckets: dict[str, list[dlp_discord.FoundUrl]] = {"gofile": [], "0807": []} - for f in found: - if f.host in buckets: - buckets[f.host].append(f) - else: # classify() already filtered, but be defensive - log.warning("Skipping unsupported URL: %s", f.url) - state.mark(f.url, status="failed", error="unsupported host") - - source_by_url: dict[str, dict] = { - f.url: { - "channel": channel_id, - "host": f.host, - "message_id": f.message_id, - "timestamp": f.timestamp, - "author": f.author, - "manifest": str(manifest), - } - for f in found - } - - # --limit applies per host class to the *pending* URLs, so a test run - # downloads exactly N new things per host (done ones don't use up the quota) - if args.limit is not None: - for key, items in buckets.items(): - pending = [f for f in items if not is_done(f.url)] - if args.shuffle: - random.shuffle(pending) - buckets[key] = pending[: args.limit] - - pending = [f for f in found if not is_done(f.url)] - if args.limit is not None: - pending = [f for f in chain.from_iterable(buckets.values()) if not is_done(f.url)] - log.info( - "%s: %d URLs (%d gofile, %d 0807); %d done, %d to download", - channel_id, len(found), - len(buckets["gofile"]), len(buckets["0807"]), - len(found) - len(pending), len(pending), - ) - - if args.dry_run: - selected = {f.url for f in chain.from_iterable(buckets.values())} - for f in found: - if f.url in selected: - flag = "download" - elif is_done(f.url): - flag = "done" - else: - flag = "skip" # cut by --limit / --only - print(f"[{f.host:6}] [{flag:8}] {f.url}") - return 0 - - if buckets["gofile"]: - run_gofile(buckets["gofile"], args, state, source_by_url, is_done, gstore) - if buckets["0807"]: - run_0807(buckets["0807"], args, state, source_by_url, is_done, gstore) - - failed_file = write_failed_urls(out_dir, state) - - ok = [u for u, r in state.records.items() if r.get("status") == "ok"] - bad = state.failures() - dead = sum(1 for r in state.records.values() if "dead link" in (r.get("error") or "")) - total_files = sum(len(r.get("files") or []) for r in state.records.values()) - total_size = sum( - (f_.get("size") or 0) for r in state.records.values() for f_ in (r.get("files") or []) - ) - log.info("%s: %d ok (%d files, %s) / %d failed (%d dead links) / %d seen", - out_dir, len(ok), total_files, format_size(total_size), - len(bad), dead, len(state.records)) - if bad: - log.info("Failed URLs written to %s - re-run this tool or use " - "python3 dlp.py --urls-file %s to retry them.", - failed_file, failed_file.name) - return len(bad) - - -def load_subscriptions(path: Path) -> list[dict]: - """Load subscriptions.json: [{"name": ..., "channel_id": ..., "out_dir": ...}].""" - data = json.loads(path.read_text(encoding="utf-8")) - entries = data.get("subscriptions", []) if isinstance(data, dict) else data - if not isinstance(entries, list): - raise SystemExit(f"{path}: expected a list or {'{'}\"subscriptions\": [...]{'}'}") - subs = [] - for entry in entries: - if not isinstance(entry, dict) or "channel_id" not in entry or "out_dir" not in entry: - raise SystemExit( - f"{path}: each subscription needs 'channel_id' and 'out_dir' " - f"(got: {entry})" - ) - subs.append(entry) - return subs - - -def main() -> None: - parser = argparse.ArgumentParser( - description="Collect downloads from Discord channels (gofile + 0807)." - ) - parser.add_argument("channel_id", nargs="?", help="Discord channel or thread id") - parser.add_argument("out_dir", nargs="?", help="directory for downloads and state") - parser.add_argument("--subs", type=Path, metavar="FILE", - help="process a subscription list instead of one channel " - "(entries: channel_id, out_dir, optional name)") - parser.add_argument("--token", help="Discord token (else $DISCORD_TOKEN or prompt)") - parser.add_argument("--urls-file", type=Path, - help="skip the Discord export; read URLs from this file " - "(e.g. failed_urls.txt)") - parser.add_argument("--password", help="password for password-protected gofile links") - parser.add_argument("--retries", type=int, default=3, help="attempts per URL (default 3)") - parser.add_argument("--retry-delay", type=int, default=5, - help="base seconds between retries, multiplied per attempt (default 5)") - parser.add_argument("--cooldown-failures", type=int, default=3, - help="consecutive failures before a cooldown pause (default 3)") - parser.add_argument("--cooldown-seconds", type=int, default=60, - help="length of the cooldown pause (default 60)") - parser.add_argument("--only", choices=("gofile", "0807"), - help="process only one host class") - parser.add_argument("--limit", type=int, metavar="N", - help="download at most N pending URLs per host class " - "(test runs)") - parser.add_argument("--shuffle", action="store_true", - help="randomize which URLs a --limit picks") - parser.add_argument("--hash", action="store_true", - help="record sha256 of each downloaded file") - parser.add_argument("--extract", action="store_true", - help="unzip downloaded archives into per-archive folders") - parser.add_argument("--fresh", action="store_true", - help="re-export the Discord channel even if export.json exists " - "(subs mode does this by default)") - parser.add_argument("--no-fresh", action="store_true", - help="subs mode: reuse cached exports instead of re-exporting") - parser.add_argument("--dry-run", action="store_true", - help="export/extract/classify only, download nothing") - parser.add_argument("--global-state", type=Path, metavar="FILE", - help=f"cross-run URL store (default: {default_global_path()})") - parser.add_argument("--no-global", action="store_true", - help="disable the cross-run URL store for this run") - parser.add_argument("--assume-done", type=Path, metavar="FILE", - help="mark every URL in the file as already downloaded, " - "then exit (manual import of old download lists)") - args = parser.parse_args() - - logging.basicConfig( - level=logging.INFO, - format="%(asctime)s %(levelname)-7s %(message)s", - datefmt="%H:%M:%S", - ) - - gstore = None - if not args.no_global: - gstore = GlobalStore(args.global_state or default_global_path()) - - if args.assume_done: - if args.channel_id or args.subs: - parser.error("--assume-done cannot be combined with a channel or --subs") - mark_assume_done(args.assume_done, gstore) - return - - if args.subs: - if args.channel_id or args.out_dir or args.urls_file: - parser.error("--subs cannot be combined with channel_id/out_dir/--urls-file") - subs = load_subscriptions(args.subs) - token = resolve_token(args.token) - total_failed = 0 - for entry in subs: - out_dir = Path(entry["out_dir"]) - log.info("=== subscription: %s ===", entry.get("name") or entry["channel_id"]) - total_failed += process_channel( - entry["channel_id"], out_dir, args, token, gstore, - force_export=not args.no_fresh, # subs mode re-exports by default - ) - if total_failed: - log.warning("%d URL(s) failed in total across subscriptions.", total_failed) - sys.exit(1) - return - - if not args.channel_id or not args.out_dir: - parser.error("channel_id and out_dir are required (or use --subs/--assume-done)") - - # A token is only needed when we will export a channel (--urls-file skips that) - token = None if args.urls_file else resolve_token(args.token) - failures = process_channel(args.channel_id, Path(args.out_dir), args, token, gstore) - if failures: - sys.exit(1) - - -if __name__ == "__main__": - try: - main() - except KeyboardInterrupt: - log.warning("Interrupted - state.json is saved; re-run to resume.") - sys.exit(130) diff --git a/version-python/dlphelper/__init__.py b/version-python/dlphelper/__init__.py deleted file mode 100644 index 84143b9..0000000 --- a/version-python/dlphelper/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -"""Warhammer DLP Helper - collect and organize files shared in Discord channels.""" - -__version__ = "0.1.0" diff --git a/version-python/dlphelper/catalogue.py b/version-python/dlphelper/catalogue.py deleted file mode 100644 index e4ad184..0000000 --- a/version-python/dlphelper/catalogue.py +++ /dev/null @@ -1,305 +0,0 @@ -"""Catalogue: classify collected files into game system / faction / class. - -The catalogue is the bridge between "a pile of downloads" and the later -library stage: it reads manifest.jsonl (with full source attribution) and/or -scans arbitrary directories for archive files, classifies each file against -the taxonomy in warhammer_data.py, and writes machine- (catalogue.jsonl) and -human-readable (catalogue.md) reports. - -Classification is keyword-based and deliberately conservative: files that -cannot be classified land in the "unclassified" bucket so gaps become visible -instead of producing wrong assignments. A manual overrides file can pin -individual files. - -Nothing is moved by default - use --apply for that (it moves files into -//// and records every move in -catalogue_moves.jsonl so it can be audited/undone). -""" - -from __future__ import annotations - -import json -import logging -import re -import shutil -from dataclasses import dataclass, field -from pathlib import Path - -from dlphelper.warhammer_data import ( - FACTION_ALIASES, - GROUP_OF_FACTION, - SYSTEMS, - UNIT_CLASSES, -) - -log = logging.getLogger(__name__) - -ARCHIVE_SUFFIXES = {".zip", ".7z", ".rar", ".tar", ".gz"} -UNKNOWN = "Unknown" -UNCLASSIFIED_FACTION = "Unclassified" -UNCLASSIFIED_CLASS = "Unclassified" - - -@dataclass -class CatalogueEntry: - """One classified file.""" - - path: str - name: str - size: int - system: str = UNKNOWN - faction: str = UNCLASSIFIED_FACTION - klass: str = UNCLASSIFIED_CLASS - source: dict = field(default_factory=dict) # manifest attribution if known - - -from functools import lru_cache - - -@lru_cache(maxsize=None) -def _clean_name(name: str) -> str: - stem = Path(name).stem - stem = re.sub(r"[_\.\+\[\]\(\)]+", " ", stem) - stem = re.sub(r"\s+", " ", stem).strip().lower() - return stem - - -def _kw_regex(keyword: str) -> str: - """Word-boundary regex for a keyword, tolerating a plural 's'.""" - return rf"(? tuple[str, str] | None: - """Return (system, faction) with the most specific keyword hit.""" - best: tuple[int, str, str] | None = None # (len, system, faction) - for system, aliases in FACTION_ALIASES.items(): - for keyword, faction in aliases.items(): - if re.search(_kw_regex(keyword), clean): - if best is None or len(keyword) > best[0]: - best = (len(keyword), system, faction) - return (best[1], best[2]) if best else None - - -@lru_cache(maxsize=None) -def _match_class(clean: str) -> str | None: - """Return the most specific force-organization class keyword hit.""" - best: tuple[int, str] | None = None - for keyword, klass in UNIT_CLASSES.items(): - if re.search(_kw_regex(keyword), clean): - if best is None or len(keyword) > best[0]: - best = (len(keyword), klass) - return best[1] if best else None - - -def classify(name: str) -> tuple[str, str, str]: - """Classify a file name -> (system, faction, class).""" - clean = _clean_name(name) - faction_hit = _match_faction(clean) - klass = _match_class(clean) - - if faction_hit: - system, faction = faction_hit - # Non-40k systems: keep their class if we found one, else Unclassified - if system != "Warhammer 40,000" and klass is None: - return system, faction, UNCLASSIFIED_CLASS - return system, faction, klass or UNCLASSIFIED_CLASS - - # No faction match: a class keyword alone cannot attribute a system - return UNKNOWN, UNCLASSIFIED_FACTION, UNCLASSIFIED_CLASS - - -def scan_manifest(manifest_path: Path) -> list[CatalogueEntry]: - """Entries from a manifest.jsonl (preferred: has source attribution).""" - entries: list[CatalogueEntry] = [] - seen: set[str] = set() - if not manifest_path.exists(): - return entries - for line in manifest_path.read_text(encoding="utf-8").splitlines(): - line = line.strip() - if not line: - continue - try: - row = json.loads(line) - except ValueError: - log.warning("Skipping malformed manifest line: %s", line[:80]) - continue - path = row.get("path") or "" - if path in seen: - continue - seen.add(path) - entries.append(CatalogueEntry( - path=path, - name=row.get("name") or Path(path).name, - size=int(row.get("size") or 0), - source={ - "channel": row.get("channel"), - "url": row.get("url"), - "message_id": row.get("message_id"), - "timestamp": row.get("timestamp"), - "author": row.get("author"), - }, - )) - return entries - - -def scan_directory(directory: Path) -> list[CatalogueEntry]: - """Entries from walking a directory tree for archive files (manual era).""" - entries: list[CatalogueEntry] = [] - for path in sorted(directory.rglob("*")): - if not path.is_file() or path.suffix.lower() not in ARCHIVE_SUFFIXES: - continue - try: - size = path.stat().st_size - except OSError: - size = 0 - entries.append(CatalogueEntry(path=str(path), name=path.name, size=size)) - return entries - - -def load_overrides(path: Path | None) -> dict[str, dict]: - """overrides.json: {"/path/or/filename": {"faction": ..., "class": ..., "system": ...}}""" - if path is None or not path.exists(): - return {} - return json.loads(path.read_text(encoding="utf-8")) - - -def apply_overrides(entries: list[CatalogueEntry], overrides: dict[str, dict]) -> int: - count = 0 - by_name = {e.name.lower(): e for e in entries} - for key, fix in overrides.items(): - entry = by_name.get(key.lower()) - if entry is None: - log.warning("Override target not found: %s", key) - continue - if "system" in fix: - entry.system = fix["system"] - if "faction" in fix: - entry.faction = fix["faction"] - if "class" in fix: - entry.klass = fix["class"] - count += 1 - return count - - -def format_size(n: int) -> str: - size = float(n) - for unit in ("B", "KiB", "MiB", "GiB", "TiB"): - if size < 1024 or unit == "TiB": - return f"{size:.1f} {unit}" - size /= 1024 - return f"{n} B" - - -def write_reports( - entries: list[CatalogueEntry], - out_md: Path, - out_jsonl: Path, -) -> None: - # machine report - with out_jsonl.open("w", encoding="utf-8") as fh: - for e in entries: - fh.write(json.dumps({ - "path": e.path, - "name": e.name, - "size": e.size, - "system": e.system, - "faction": e.faction, - "class": e.klass, - "source": {k: v for k, v in e.source.items() if v}, - }, ensure_ascii=False) + "\n") - - # human report - grouped: dict[str, dict[str, dict[str, list[CatalogueEntry]]]] = {} - for e in entries: - grouped.setdefault(e.system, {}).setdefault(e.faction, {}).setdefault( - e.klass, []).append(e) - - lines: list[str] = ["# Warhammer Collection Catalogue", ""] - total_size = sum(e.size for e in entries) - classified = sum(1 for e in entries if e.faction != UNCLASSIFIED_FACTION) - lines.append(f"{len(entries)} files, {format_size(total_size)} total, " - f"{classified} classified, " - f"{len(entries) - classified} unclassified") - lines.append("") - - for system in sorted(grouped): - factions = grouped[system] - all_sys_files = [e for f in factions.values() for c in f.values() for e in c] - sys_size = sum(e.size for e in all_sys_files) - lines.append(f"## {system} ({len(all_sys_files)} files, {format_size(sys_size)})") - for faction in sorted(factions): - classes = factions[faction] - fac_size = sum(e.size for c in classes.values() for e in c) - lines.append(f"### {faction} ({format_size(fac_size)})") - for klass in sorted(classes): - files = classes[klass] - lines.append(f"- **{klass}** ({len(files)} files, {format_size(sum(e.size for e in files))})") - for e in sorted(files, key=lambda x: x.name.lower()): - src = "" - if e.source.get("url"): - src = f" — from {e.source['url']}" - lines.append(f" - {e.name} ({format_size(e.size)}){src}") - lines.append("") - - out_md.write_text("\n".join(lines), encoding="utf-8") - - -def apply_moves( - entries: list[CatalogueEntry], - library_root: Path, - moves_log: Path, -) -> int: - """Move classified files into the library structure. Audited in moves_log.""" - moves_log.parent.mkdir(parents=True, exist_ok=True) - moved = 0 - with moves_log.open("a", encoding="utf-8") as logfh: - for e in entries: - if e.faction == UNCLASSIFIED_FACTION: - continue # never move what we cannot classify - if e.system == UNKNOWN: - continue - src = Path(e.path) - if not src.exists(): - log.warning("Cannot move, missing: %s", src) - continue - dest_dir = ( - library_root / e.system - / GROUP_OF_FACTION.get(e.faction, "Unclassified") - / e.faction / e.klass - ) - dest = dest_dir / src.name - if dest.exists(): - log.warning("Skip, destination exists: %s", dest) - continue - dest_dir.mkdir(parents=True, exist_ok=True) - shutil.move(str(src), str(dest)) - logfh.write(json.dumps({"from": str(src), "to": str(dest)}) + "\n") - e.path = str(dest) - moved += 1 - return moved - - -def build_catalogue( - manifests: list[Path], - scan_dirs: list[Path], - overrides_path: Path | None, -) -> list[CatalogueEntry]: - entries: list[CatalogueEntry] = [] - seen: set[str] = set() - for m in manifests: - for e in scan_manifest(m): - if e.path not in seen: - seen.add(e.path) - entries.append(e) - for d in scan_dirs: - for e in scan_directory(d): - if e.path not in seen: - seen.add(e.path) - entries.append(e) - - for e in entries: - e.system, e.faction, e.klass = classify(e.name) - - apply_overrides(entries, load_overrides(overrides_path)) - return entries diff --git a/version-python/dlphelper/discord.py b/version-python/dlphelper/discord.py deleted file mode 100644 index 89e8025..0000000 --- a/version-python/dlphelper/discord.py +++ /dev/null @@ -1,160 +0,0 @@ -"""Export a Discord channel/thread with DiscordChatExporter and extract URLs. - -The channels this tool targets encode their download links as base64 inside -message content, and occasionally as plain links. We export the channel as -JSON (richer metadata than the HTML export: message id, timestamp, author), -decode every base64 candidate, and keep only URLs for supported hosts. -""" - -from __future__ import annotations - -import base64 -import json -import logging -import os -import re -import subprocess -from dataclasses import dataclass -from pathlib import Path -from urllib.parse import urlparse - -log = logging.getLogger(__name__) - -DCE_IMAGE = "tyrrrz/discordchatexporter:stable" - -# The channels encode links as base64 that always starts with this prefix -# (https://...); grab.sh established this regex, keep it compatible. -B64_RE = re.compile(r"aHR0cHM6Ly[89][A-Za-z0-9+/_-]*") -URL_RE = re.compile(r"https://[^\s<>\"'\)\]\}]+") - -SUPPORTED_HOSTS = { - "gofile": ("gofile.io", "www.gofile.io"), - "0807": ("0807.st", "www.0807.st"), -} - - -@dataclass(frozen=True) -class FoundUrl: - """One download URL found in the channel, with source attribution.""" - - url: str - host: str # "gofile" | "0807" (set after classification) - message_id: str | None = None - timestamp: str | None = None - author: str | None = None - encoded: bool = False # found via base64 decode vs. plain link - - -def export_channel(channel_id: str, out_dir: Path, token: str, force: bool = False) -> Path: - """Export the channel to out_dir/export.json using the DCE docker image. - - The token is passed to the container via the DISCORD_TOKEN env var (DCE - reads it natively) so it never appears on the docker command line. TOKEN - is set as well for DCE versions that only know that name. - """ - dest = out_dir / "export.json" - if dest.exists() and not force: - log.info("Reusing existing export %s (pass --fresh to re-export)", dest) - return dest - - out_dir.mkdir(parents=True, exist_ok=True) - env = dict(os.environ) - env["DISCORD_TOKEN"] = token - env["TOKEN"] = token - cmd = [ - "docker", "run", "--rm", - "--user", f"{os.getuid()}:{os.getgid()}", - "-v", f"{out_dir.resolve()}:/out", - "-e", "DISCORD_TOKEN", - "-e", "TOKEN", - DCE_IMAGE, - "export", "-c", channel_id, "-f", "Json", "-o", "/out/export.json", - ] - log.info("Exporting channel %s via DiscordChatExporter ...", channel_id) - try: - subprocess.run(cmd, env=env, check=True) - except FileNotFoundError as exc: - raise RuntimeError( - "docker not found - the Discord export needs the docker image " - f"{DCE_IMAGE}" - ) from exc - except subprocess.CalledProcessError as exc: - raise RuntimeError( - f"DiscordChatExporter failed with exit code {exc.returncode} " - "(check the token and the channel id)" - ) from exc - if not dest.exists(): - raise RuntimeError("DiscordChatExporter produced no output file") - return dest - - -def classify(url: str) -> str | None: - """Return the host key ('gofile' | '0807') for a URL, else None.""" - try: - hostname = (urlparse(url).hostname or "").lower() - except ValueError: - return None - for key, hosts in SUPPORTED_HOSTS.items(): - if hostname in hosts: - return key - return None - - -def _decode_b64(candidate: str) -> str | None: - s = candidate.replace("_", "/").replace("-", "+") - s = s.rstrip("=") - s += "=" * ((4 - len(s) % 4) % 4) - try: - decoded = base64.b64decode(s, validate=False).decode("utf-8", "strict") - except Exception: # noqa: BLE001 - malformed candidate, just skip it - return None - # One encoded blob can contain the URL plus padding/newlines - stripped = decoded.strip() - line = stripped.splitlines()[0] if stripped else "" - return line or None - - -def extract_urls(export_path: Path) -> list[FoundUrl]: - """Extract supported URLs from a DCE JSON export, deduplicated.""" - data = json.loads(export_path.read_text(encoding="utf-8")) - messages = data if isinstance(data, list) else data.get("messages", []) - - found: dict[str, FoundUrl] = {} - stats = {"b64": 0, "plain": 0, "unsupported": 0} - - for msg in messages: - content = str(msg.get("content") or "") - if not content: - continue - author = (msg.get("author") or {}).get("name") - attrib = { - "message_id": str(msg.get("id") or "") or None, - "timestamp": msg.get("timestamp"), - "author": author, - } - - for match in B64_RE.finditer(content): - stats["b64"] += 1 - decoded = _decode_b64(match.group(0)) - if not decoded: - continue - host = classify(decoded) - if host is None: - stats["unsupported"] += 1 - continue - found.setdefault(decoded, FoundUrl(url=decoded, host=host, encoded=True, **attrib)) - - for match in URL_RE.finditer(content): - url = match.group(0).rstrip(".,;)") - host = classify(url) - if host is None: - continue - stats["plain"] += 1 - found.setdefault(url, FoundUrl(url=url, host=host, encoded=False, **attrib)) - - log.info( - "Extraction: %d base64 candidates, %d plain links, %d unsupported/undecodable, " - "%d unique supported URLs", - stats["b64"], stats["plain"], stats["unsupported"], len(found), - ) - return sorted(found.values(), key=lambda f: f.url) diff --git a/version-python/dlphelper/globalstore.py b/version-python/dlphelper/globalstore.py deleted file mode 100644 index 057f6df..0000000 --- a/version-python/dlphelper/globalstore.py +++ /dev/null @@ -1,70 +0,0 @@ -"""Cross-run, cross-channel record of completed URLs. - -This is what prevents re-downloading your collection: state.json only knows -about one OUT_DIR, but the same gofile link posted in a second channel would -otherwise be downloaded again into that channel's OUT_DIR. The global store -(default: ~/.local/share/dlphelper/urls.json) records every successfully -downloaded URL once, forever, regardless of OUT_DIR. -""" - -from __future__ import annotations - -import json -import logging -import os -import time -from pathlib import Path - -log = logging.getLogger(__name__) - - -def default_global_path() -> Path: - data_home = os.environ.get("XDG_DATA_HOME") or str( - Path.home() / ".local" / "share" - ) - return Path(data_home) / "dlphelper" / "urls.json" - - -class GlobalStore: - """JSON file mapping url -> {status, channel, host, files, updated}.""" - - def __init__(self, path: Path | None = None): - self.path = path - self.records: dict[str, dict] = {} - if self.path and self.path.exists(): - try: - self.records = json.loads(self.path.read_text(encoding="utf-8")) - if not isinstance(self.records, dict): - raise ValueError("global store root must be an object") - except (ValueError, OSError) as exc: - backup = self.path.with_suffix(".json.bak") - log.warning("Corrupt global store %s (%s); backed up to %s", - self.path, exc, backup) - self.path.replace(backup) - self.records = {} - - def is_done(self, url: str) -> bool: - if not self.records: - return False - rec = self.records.get(url) - return bool(rec) and rec.get("status") == "ok" - - def mark(self, url: str, **fields) -> None: - if self.path is None: - return - rec = self.records.setdefault(url, {"status": "ok"}) - for key, value in fields.items(): - if value is not None: - rec[key] = value - rec["status"] = "ok" - rec["updated"] = time.strftime("%Y-%m-%dT%H:%M:%S") - self.save() - - def save(self) -> None: - if self.path is None: - return - self.path.parent.mkdir(parents=True, exist_ok=True) - tmp = self.path.with_suffix(".json.tmp") - tmp.write_text(json.dumps(self.records, indent=2, sort_keys=True), - encoding="utf-8") - tmp.replace(self.path) diff --git a/version-python/dlphelper/gofile.py b/version-python/dlphelper/gofile.py deleted file mode 100644 index 651962f..0000000 --- a/version-python/dlphelper/gofile.py +++ /dev/null @@ -1,239 +0,0 @@ -"""Minimal GoFile API client - one guest account per run. - -The API interaction mirrors Lysagxra/GoFileDownloader (same headers, website -token scheme and endpoints), but with two deliberate differences: - -1. A single guest account is created per run (GoFileDownloader creates one per - URL, which trips Gofile's rate limit after ~6 requests). -2. Failures raise exceptions instead of sys.exit(), so the orchestrator can - track them per file/URL and keep going. -""" - -from __future__ import annotations - -import hashlib -import logging -import os -import time -from pathlib import Path -from urllib.parse import urlparse - -import requests - -log = logging.getLogger(__name__) - -API = "https://api.gofile.io" -USER_AGENT = "Mozilla/5.0" -LOCALE = "en-US" -TOKEN_SECRET = "12af056dacea0b" # public website-token seed, see GoFileDownloader # noqa: S105 -TOKEN_WINDOW_SEC = 14400 -CHUNK_SIZE = 1024 * 1024 - -COMMON_HEADERS = { - "User-Agent": USER_AGENT, - "Accept-Encoding": "gzip", - "Accept": "*/*", - "Connection": "keep-alive", -} -EXTENDED_HEADERS = { - **COMMON_HEADERS, - "Sec-Fetch-Dest": "empty", - "Sec-Fetch-Mode": "cors", - "Sec-Fetch-Site": "same-site", - "Pragma": "no-cache", - "Cache-Control": "no-cache", -} - - -class GoFileError(Exception): - """Base error for GoFile operations.""" - - -class GoFilePasswordRequired(GoFileError): - """The content is password protected and none (or a wrong one) was given.""" - - -class GoFileNotFound(GoFileError): - """The content/file no longer exists (404 / status notFound). - - Dead links never come back, so the orchestrator must not retry these. - """ - - -class GoFileEmpty(GoFileError): - """The content exists but contains no files (e.g. emptied folder). - - Stable state like a dead link: no retry. - """ - - -class GoFileAuthError(GoFileError): - """The account token was rejected; the client can re-auth once.""" - - -def content_id_from_url(url: str) -> str: - """Extract the content id from a gofile.io/d/ URL.""" - parts = [p for p in urlparse(url).path.split("/") if p] - if len(parts) >= 2 and parts[-2] == "d": - return parts[-1] - raise GoFileError(f"not a gofile content URL (expected gofile.io/d/): {url}") - - -class GoFileClient: - """GoFile API client that reuses one guest account token for the whole run.""" - - def __init__(self, password: str | None = None): - self._password_hash = ( - hashlib.sha256(password.encode()).hexdigest() if password else None - ) - self._token: str | None = None - self.session = requests.Session() - - # -- authentication ----------------------------------------------------- - - def ensure_token(self, max_attempts: int = 8, wait_seconds: int = 20) -> str: - """Create a guest account; backs off and retries on rate limiting.""" - if self._token: - return self._token - - last_status = "unknown" - for attempt in range(1, max_attempts + 1): - try: - resp = self.session.post( - f"{API}/accounts", headers=COMMON_HEADERS, timeout=15 - ) - data = resp.json() - except (ValueError, requests.RequestException) as exc: - last_status = f"request error: {exc}" - log.warning( - "GoFile account creation attempt %d/%d failed: %s", - attempt, max_attempts, last_status, - ) - else: - if isinstance(data, dict) and data.get("status") == "ok": - self._token = data["data"]["token"] - log.info("GoFile guest account created (token reused for whole run)") - return self._token - last_status = str(data.get("status", data)[:200] if data else data) - log.warning( - "GoFile account creation attempt %d/%d failed: %s", - attempt, max_attempts, last_status, - ) - if attempt < max_attempts: - time.sleep(wait_seconds * attempt) - raise GoFileError( - f"GoFile account creation failed after {max_attempts} attempts " - f"(last: {last_status})" - ) - - def reset(self) -> None: - """Drop the current token so the next ensure_token() creates a new one.""" - self._token = None - - def _website_token(self) -> str: - window = int(time.time()) // TOKEN_WINDOW_SEC - seed = f"{USER_AGENT}::{LOCALE}::{self._token}::{window}::{TOKEN_SECRET}" - return hashlib.sha256(seed.encode()).hexdigest() - - def _auth_headers(self) -> dict: - return { - **EXTENDED_HEADERS, - "Authorization": f"Bearer {self._token}", - "X-Website-Token": self._website_token(), - "X-BL": LOCALE, - } - - # -- listing ------------------------------------------------------------ - - def list_files(self, url: str) -> list[tuple[str, str, str]]: - """Return (relpath, filename, download_link) for every file under a URL.""" - cid = content_id_from_url(url) - results: list[tuple[str, str, str]] = [] - self._walk(cid, [], results) - # Direct files at the content root get their own folder named after the id - normalized = [ - (relpath if relpath else cid, name, link) for relpath, name, link in results - ] - return normalized - - def _walk(self, cid: str, prefix: list[str], results: list[tuple[str, str, str]]) -> None: - query = "?cache=true&sortField=createTime&sortDirection=1" - if self._password_hash: - query += f"&password={self._password_hash}" - try: - resp = self.session.get( - f"{API}/contents/{cid}{query}", headers=self._auth_headers(), timeout=30 - ) - except requests.RequestException as exc: - raise GoFileError(f"content request for {cid} failed: {exc}") from exc - - if resp.status_code == 404: - raise GoFileNotFound(f"content {cid} not found (dead link)") - if resp.status_code in (401, 403): - raise GoFileAuthError(f"content request denied ({resp.status_code}) for {cid}") - try: - data = resp.json() - except ValueError as exc: - raise GoFileError(f"non-JSON response for {cid}: {resp.text[:200]}") from exc - - if data.get("status") == "notFound" or "notFound" in str(data.get("status", "")): - raise GoFileNotFound(f"content {cid} does not exist anymore (dead link)") - if data.get("status") != "ok": - raise GoFileError(f"content request failed for {cid}: {data.get('status')}") - - d = data["data"] - if d.get("passwordStatus") not in (None, "passwordOk"): - raise GoFilePasswordRequired( - f"content {cid} is password protected (use --password)" - ) - - if d.get("type") == "folder": - here = prefix + [d["name"]] - for child in (d.get("children") or {}).values(): - if child.get("type") == "folder": - self._walk(child["id"], here, results) - else: - results.append(("/".join(here), child["name"], child["link"])) - else: - results.append(("/".join(prefix), d["name"], d["link"])) - - # -- downloading ---------------------------------------------------------- - - def download_file(self, link: str, dest: Path) -> int: - """Download one file; returns the file size. Skips existing files.""" - dest.parent.mkdir(parents=True, exist_ok=True) - if dest.exists() and dest.stat().st_size > 0: - log.info("Skip existing: %s", dest) - return dest.stat().st_size - - tmp = dest.with_name(dest.name + ".part") - headers = { - **EXTENDED_HEADERS, - "Cookie": f"accountToken={self._token}", - "Referer": link, - "Origin": link, - } - try: - with self.session.get( - link, headers=headers, stream=True, timeout=(10, 60), allow_redirects=True - ) as resp: - if resp.status_code == 404: - raise GoFileNotFound(f"file {dest.name} not found (dead link)") - if resp.status_code != 200: - raise GoFileError(f"HTTP {resp.status_code} while downloading {dest.name}") - total = int(resp.headers.get("Content-Length") or 0) - written = 0 - with tmp.open("wb") as fh: - for chunk in resp.iter_content(chunk_size=CHUNK_SIZE): - fh.write(chunk) - written += len(chunk) - if written == 0: - raise GoFileError(f"empty body for {dest.name}") - os.replace(tmp, dest) - return written - finally: - if tmp.exists() and not dest.exists(): - try: - tmp.unlink() - except OSError: - pass diff --git a/version-python/dlphelper/gwshop.py b/version-python/dlphelper/gwshop.py deleted file mode 100644 index 5e6ec46..0000000 --- a/version-python/dlphelper/gwshop.py +++ /dev/null @@ -1,353 +0,0 @@ -"""Games Workshop webshop catalog via Algolia. - -The warhammer.com shop is behind an AWS WAF challenge for non-browser -clients, but its product search runs on Algolia and the app-id plus the -public search-only key ship with every browser page. This module uses them -to enumerate the whole product index and normalize it into the sorter's -catalog schema (system/group/faction/subfaction/class/name/aliases). - -Credentials are read from (first hit wins): - 1. env ALGOLIA_APP_ID + ALGOLIA_API_KEY - 2. algolia.json next to the repo root (gitignored): - {"app_id": "...", "api_key": "..."} - -The key is the *search-only* public key (never an admin key). Treat any -key found in algolia.json as still secret: never log or commit it. - -CLI: python3 -m dlphelper.gwshop OUT.json [--index NAME] [--raw RAW.json] -""" - -from __future__ import annotations - -import argparse -import json -import logging -import os -import time -from pathlib import Path -from typing import Any - -import requests - -log = logging.getLogger(__name__) - -DEFAULT_INDEX = "prod-lazarus-product-en-ww" -# The read host (-dsn) serves searches; credentials also work as query -# parameters, which keeps every request a plain GET/POST with no headers -# beyond content-type. -_DSN_HOST = "https://{app_id}-dsn.algolia.net" - -_PAGE_SIZE = 1000 # Algolia max hitsPerPage per /query page -_RETRY_DELAYS = (2, 5, 15) - - -def load_credentials() -> tuple[str, str]: - """Resolve (app_id, api_key) from env or algolia.json.""" - app_id = os.environ.get("ALGOLIA_APP_ID", "") - api_key = os.environ.get("ALGOLIA_API_KEY", "") - if app_id and api_key: - return app_id, api_key - cfg = Path(__file__).resolve().parent.parent / "algolia.json" - if cfg.is_file(): - data = json.loads(cfg.read_text(encoding="utf-8")) - app_id = data.get("app_id", "") or app_id - api_key = data.get("api_key", "") or api_key - if not app_id or not api_key: - raise SystemExit( - "Algolia credentials missing: set ALGOLIA_APP_ID and " - "ALGOLIA_API_KEY, or create algolia.json with " - '{"app_id": "...", "api_key": "..."} (gitignored).' - ) - return app_id, api_key - - -class GwShopClient: - """Minimal Algolia search client for one application.""" - - def __init__(self, app_id: str, api_key: str) -> None: - self.base = _DSN_HOST.format(app_id=app_id) - self.params = { - "x-algolia-application-id": app_id, - "x-algolia-api-key": api_key, - } - self.session = requests.Session() - - def _request(self, method: str, path: str, body: dict | None = None) -> Any: - url = f"{self.base}{path}" - last_err: Exception | None = None - for attempt, delay in enumerate((0,) + _RETRY_DELAYS, start=1): - if delay: - log.info("Algolia retry %d in %ds", attempt, delay) - time.sleep(delay) - try: - r = self.session.request( - method, - url, - params=self.params, - json=body, - timeout=30, - ) - if r.status_code == 200: - return r.json() - last_err = RuntimeError( - f"Algolia {method} {path} -> {r.status_code}: {r.text[:200]}" - ) - if r.status_code in (401, 403): # credentials: no point retrying - break - except requests.RequestException as exc: - last_err = exc - raise RuntimeError(f"Algolia request failed: {last_err}") from last_err - - def list_indices(self) -> list[dict]: - return self._request("GET", "/1/indexes").get("items", []) - - # ---- full index retrieval ------------------------------------------- - # The search-only key cannot use /browse (403) and normal pagination is - # capped at 1000 cumulative results. So we partition the index on the - # GameSystemsRoot hierarchical facet: each branch is fetched with an - # explicit filter, recursing deeper only when a branch exceeds one page. - - _FACET = "GameSystemsRoot" - - @staticmethod - def _escape(value: str) -> str: - return value.replace("\\", "\\\\").replace('"', '\\"') - - def _filter_expr(self, path: list[tuple[int, str]]) -> str: - return " AND ".join( - f'{self._FACET}.lvl{lvl}:"{self._escape(value)}"' - for lvl, value in path - ) - - def _search_all(self, index: str, filters: str | None) -> tuple[list[dict], int]: - """All hits matching `filters` (must be <= pagination cap).""" - out: list[dict] = [] - params = f"query=&hitsPerPage={_PAGE_SIZE}&page=0" - if filters: - params += f"&filters={requests.utils.quote(filters, safe='')}" - page = self._request( - "POST", f"/1/indexes/{index}/query", body={"params": params} - ) - out.extend(page.get("hits", [])) - return out, int(page.get("nbHits") or 0) - - def _facet_counts( - self, index: str, level: int, filters: str | None - ) -> dict[str, int]: - params = f"query=&hitsPerPage=0&facets={self._FACET}.lvl{level}" - if filters: - params += f"&filters={requests.utils.quote(filters, safe='')}" - page = self._request( - "POST", f"/1/indexes/{index}/query", body={"params": params} - ) - facets = page.get("facets") or {} - return facets.get(f"{self._FACET}.lvl{level}") or {} - - def browse(self, index: str, hits_per_page: int = _PAGE_SIZE) -> list[dict]: - """Every record of the index, via facet-partitioned search.""" - hits, total = self._search_all(index, None) - if total <= hits_per_page: - return hits - log.info("gwshop %s: %d hits, partitioning on %s", index, total, self._FACET) - out = self._partition(index, [], total) - covered = len({h.get("objectID") for h in out}) - if covered < total: - log.warning( - "gwshop %s: %d of %d index records left out of the dump " - "(no fetchable %s branch path)", - index, total - covered, total, self._FACET, - ) - return out - - def _partition( - self, index: str, path: list[tuple[int, str]], total: int - ) -> list[dict]: - level = len(path) - filters = self._filter_expr(path) if path else None - counts = self._facet_counts(index, level, filters) - out: list[dict] = [] - covered: set[Any] = set() - for value, count in sorted(counts.items()): - sub_path = path + [(level, value)] - sub_filters = self._filter_expr(sub_path) - if count <= _PAGE_SIZE: - hits, nb = self._search_all(index, sub_filters) - out.extend(hits) - covered.update(h.get("objectID") for h in hits) - log.info( - "gwshop %s: %s -> %d hits", index, sub_filters[:80], nb - ) - else: - sub = self._partition(index, sub_path, count) - out.extend(sub) - covered.update(h.get("objectID") for h in sub) - # nbHits summed across branches double-counts products listed in - # several branches, so coverage is counted per unique objectID. - residual = total - len(covered) - if residual > 0: - log.warning( - "gwshop %s: %d hits in branch carry no %s.lvl%d " - "(shallow facet path, left out of the dump)", - index, residual, self._FACET, level, - ) - return out - - -# --------------------------------------------------------------------------- -# normalization -# --------------------------------------------------------------------------- - -def _clean(node: str) -> str: - return node.strip() - - -def hierarchy_paths(hit: dict) -> list[list[str]]: - """Every full path of the GameSystemsRoot hierarchical facet. - - A hit may belong to several branches (e.g. an army box also being a - "Gaming Rules" entry); each branch is returned as its own path - [system, group, faction, subfaction_or_class, ...]. - """ - root = hit.get("GameSystemsRoot") or {} - paths: list[list[str]] = [] - for lvl_key in sorted(root, key=lambda k: int(k[3:])): - for value in root.get(lvl_key) or []: - paths.append([_clean(p) for p in value.split(" > ")]) - return paths - - -def normalize(hit: dict) -> dict: - """One Algolia hit -> one or more catalog entries (usually one).""" - name = _clean(hit.get("name") or hit.get("objectID") or "") - entry: dict[str, Any] = { - "name": name, - "aliases": [], - "sku": hit.get("sku"), - "slug": hit.get("slug"), - "productType": hit.get("productType"), - "source": "gwshop", - "gw_paths": hierarchy_paths(hit), - "collections": hit.get("_collections") or [], - } - return entry - - -def build_catalog(hits: list[dict]) -> dict: - """Index dump -> catalog document in the sorter's schema shape. - - The hierarchy layer (system/group/faction/subfaction) is expanded from - gw_paths; name/aliases stay per product so the fuzzy matcher works on - the product name, exactly like data/library_catalog.json entries. - """ - entries: list[dict] = [] - seen: set[tuple] = set() - for hit in hits: - base = normalize(hit) - paths = base["gw_paths"] - if not paths: - key = (hit.get("objectID"), ()) - if key in seen: - continue - seen.add(key) - entries.append( - { - "system": None, - "group": None, - "faction": None, - "subfaction": None, - "class": None, - "name": base["name"], - "aliases": [], - "sku": base["sku"], - "slug": base["slug"], - "productType": base["productType"], - "source": "gwshop", - "collections": base["collections"], - } - ) - continue - # primary branch = longest path (most specific), e.g. - # [40k, Space Marines, Raven Guard, Walker] - primary = max(paths, key=len) - system = primary[0] if len(primary) > 0 else None - group = primary[1] if len(primary) > 1 else None - faction = primary[2] if len(primary) > 2 else None - # The LAST level is the class/category layer (GW uses shop - # categories there: Battleline, Walker, Vehicle, "Rules & - # Accessories", product lines like "Cadian"). Deeper sources - # (miniset style) may carry a real subfaction before it. - if len(primary) >= 5: - subfaction = primary[3] - klass = primary[4] - elif len(primary) == 4: - subfaction = None - klass = primary[3] - else: - subfaction = None - klass = None - key = (hit.get("objectID"), tuple(primary)) - if key in seen: - continue - seen.add(key) - entries.append( - { - "system": system, - "group": group, - "faction": faction, - "subfaction": subfaction, - "class": klass, - "name": base["name"], - "aliases": [], - "sku": base["sku"], - "slug": base["slug"], - "productType": base["productType"], - "source": "gwshop", - "collections": base["collections"], - } - ) - return { - "comment": ( - "Generated by dlphelper.gwshop from the Games Workshop webshop " - "(Algolia index, deduplicated by product). Source-agnostic " - "catalog schema: system/group/faction/subfaction/class/name/aliases." - ), - "source": "gwshop", - "generated_entries": len(entries), - "entries": entries, - } - - -def main() -> None: - logging.basicConfig(level=logging.INFO, format="%(levelname)s %(message)s") - parser = argparse.ArgumentParser( - description="Dump the GW webshop catalog via Algolia." - ) - parser.add_argument("out", help="output catalog JSON path") - parser.add_argument("--index", default=DEFAULT_INDEX) - parser.add_argument("--raw", help="also write the raw Algolia hits here") - args = parser.parse_args() - - app_id, api_key = load_credentials() - client = GwShopClient(app_id, api_key) - hits = client.browse(args.index) - log.info("%d raw hits from %s", len(hits), args.index) - - if args.raw: - Path(args.raw).write_text( - json.dumps(hits, ensure_ascii=False, indent=1), encoding="utf-8" - ) - log.info("raw hits written to %s", args.raw) - - catalog = build_catalog(hits) - Path(args.out).write_text( - json.dumps(catalog, ensure_ascii=False, indent=1), encoding="utf-8" - ) - log.info( - "%d catalog entries written to %s", - catalog["generated_entries"], - args.out, - ) - - -if __name__ == "__main__": - main() diff --git a/version-python/dlphelper/sorter.py b/version-python/dlphelper/sorter.py deleted file mode 100644 index 0edc55a..0000000 --- a/version-python/dlphelper/sorter.py +++ /dev/null @@ -1,899 +0,0 @@ -"""Sorter: place collection files into the library via plan/apply. - -See docs/SORTING.md for the structure and the pipeline this implements: -inspect (name + archive listing) -> classify (keyword tables) -> fuzzy match -against the library catalog -> plan -> apply (hardlink, fallback copy). - -The sorter never deletes or modifies source files. Applying is append-only: -targets that already exist are skipped, actions are audited in -sorter_applied.jsonl. -""" - -from __future__ import annotations - -import difflib -import json -import logging -import os -import shutil -import subprocess -import zipfile -from dataclasses import dataclass, field -from functools import lru_cache -from pathlib import Path - -from dlphelper.catalogue import _clean_name, _match_class, _match_faction -from dlphelper.warhammer_data import GROUP_OF_FACTION - -log = logging.getLogger(__name__) - -ARCHIVE_SUFFIXES = {".zip", ".7z", ".rar", ".tar", ".gz"} -UNSORTED = "_unsorted" -UNCLASSIFIED_CLASS = "Unclassified" -# A file whose system the keyword tables could not identify only gets a -# catalog unit when the fuzzy hit is strong - otherwise it would wander -# into other games' sections (Blood Bowl, ...) on a guess. -CROSS_SYSTEM_MIN = 0.75 - -# Game systems whose sections the sorter must never place into. The -# collection is warhammer-only: everything else (Middle-earth proxies, -# ...) stays in _unsorted even when a name matches the catalog. The -# catalog snapshot itself keeps all systems; the filter lives here. -DROPPED_SYSTEMS = frozenset({"Middle-Earth"}) - -# Weak acceptance tier: when nothing reached the normal threshold, a hit -# at least this strong is accepted but flagged low confidence ("[w]" in -# the plan, confidence field in the audit, listed in -# sorter_low_confidence.txt). -WEAK_MIN = 0.50 - -# A hit whose name shares no word with the entry needs this much more -# score to be trusted (char-ratio flukes like "aeldari rangers" -> "Dark -# Reapers" sit at 0.60-0.67; real unit names usually share a token). -TOKENLESS_BONUS = 0.08 - -# Run-artifact files that must never be sorted into a library -RUN_ARTIFACTS = { - "export.json", "export.html", "state.json", "manifest.jsonl", - "failed_urls.txt", "catalogue.md", "catalogue.jsonl", - "sorter_applied.jsonl", "catalogue_moves.jsonl", "urls.txt", - "urls_0807.txt", "urls_gofile.txt", "failed_0807.txt", - "extract_manifest.jsonl", -} - - -@dataclass -class SortItem: - """One file to sort, plus everything learned about it.""" - - path: Path - size: int - name: str # original file name - clean: str # cleaned lower-case name - inner_names: list[str] = field(default_factory=list) # archive listing - system: str = "Unknown" - group: str = UNSORTED - faction: str = UNSORTED - subfaction: str | None = None - klass: str | None = None - unit: str | None = None # canonical catalog name, if matched - match_score: float = 0.0 - match_kind: str = "none" # catalog | weak | implicit | none | invalid - confidence: str = "normal" # normal | low (match_kind == "weak") - action: str = "link" # link | copy - target: Path | None = None - - -# --------------------------------------------------------------------------- -# inspection -# --------------------------------------------------------------------------- - -def list_zip(path: Path, limit: int = 40) -> list[str]: - """Inner names of a .zip without extracting.""" - try: - with zipfile.ZipFile(path) as zf: - return zf.namelist()[:limit] - except (zipfile.BadZipFile, OSError, RuntimeError) as exc: - log.debug("zip listing failed for %s: %s", path.name, exc) - return [] - - -def list_7z(path: Path, limit: int = 40) -> list[str]: - """Inner names of a .7z via the installed 7z binary, without extracting.""" - if not shutil.which("7z"): - return [] - try: - proc = subprocess.run( - ["7z", "l", "-ba", "-slt", str(path)], - stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, - encoding="utf-8", errors="replace", timeout=120, - ) - except (OSError, subprocess.TimeoutExpired) as exc: - log.debug("7z listing failed for %s: %s", path.name, exc) - return [] - names: list[str] = [] - for block in proc.stdout.split("Path = ")[1:]: - name = block.splitlines()[0].strip() if block.splitlines() else "" - # -slt lists directories too; skip entries ending in / - if name and not name.endswith("/"): - names.append(name) - if len(names) >= limit: - break - return names - - -def inspect(path: Path) -> SortItem: - size = path.stat().st_size if path.exists() else 0 - item = SortItem(path=path, size=size, name=path.name, clean=_clean_name(path.name)) - if not path.is_file(): - return item - if path.suffix.lower() in ARCHIVE_SUFFIXES: - # An archive that is not an archive is almost always a saved HTML - # error/login page that got downloaded instead of the file. - if path.suffix.lower() == ".zip": - item.inner_names = list_zip(path) - else: - item.inner_names = list_7z(path) - if item.inner_names == [] and path.stat().st_size > 0: - head = b"" - with path.open("rb") as fh: - head = fh.read(512) - if b" None: - """Keyword classification from the file name, then refined by inner names.""" - hit = _match_faction(item.clean) - if hit: - item.system, item.faction = hit - item.klass = _match_class(item.clean) or ( - _match_class(_clean_inner(" ".join(item.inner_names))) - if item.inner_names else None - ) - - if item.faction == UNSORTED and item.inner_names: - # hash-named archives: vote on the inner names' classification - votes: dict[tuple[str, str], int] = {} - for inner in item.inner_names: - ihit = _match_faction(_clean_inner(inner)) - if ihit: - votes[ihit] = votes.get(ihit, 0) + 1 - if votes: - best = max(votes.items(), key=lambda kv: kv[1])[0] - item.system, item.faction = best - if item.klass is None: - inner_classes = [ - _match_class(_clean_name(inner)) for inner in item.inner_names - ] - cvotes: dict[str, int] = {} - for c in inner_classes: - if c: - cvotes[c] = cvotes.get(c, 0) + 1 - if cvotes: - item.klass = max(cvotes.items(), key=lambda kv: kv[1])[0] - - item.group = GROUP_OF_FACTION.get(item.faction, UNSORTED) - - -# faction group layer (GW naming, shared with warhammer_data) - - -def load_catalog(path: Path, - dropped: frozenset[str] | None = DROPPED_SYSTEMS) -> list[dict]: - """Load catalog entries, dropping non-collected game systems by default - (pass dropped=frozenset() to keep everything).""" - data = json.loads(path.read_text(encoding="utf-8")) - entries = data["entries"] if isinstance(data, dict) else data - if dropped: - entries = [e for e in entries if e.get("system") not in dropped] - for e in entries: - e["_clean"] = _clean_name(e["name"]) - e["_clean_aliases"] = [_clean_name(a) for a in e.get("aliases", [])] - return entries - - -def _best_ratio( - clean: str, candidates: list[str], allow_single: bool = True -) -> float: - """Best sequence ratio, pruned with cheap upper-bound checks. - - lru_cached because the same (name, entry) pair is scored over and - over across thousands of files. - """ - return _best_ratio_cached(clean, tuple(candidates), allow_single) - - -@lru_cache(maxsize=None) -def _best_ratio_cached( - clean: str, candidates: tuple[str, ...], allow_single: bool = True -) -> float: - best = 0.0 - toks = set(_stem_tokens(clean)) - best = _containment_boost(best, toks, candidates, allow_single) - for c in candidates: - sm = difflib.SequenceMatcher(None, clean, c) - if sm.real_quick_ratio() < best or sm.quick_ratio() < best: - continue - r = sm.ratio() - if r > best: - best = r - if best == 1.0: - break - return best - - -def _stem_tokens(clean: str) -> list[str]: - """Tokens with a crude plural 's' stripped, for containment checks.""" - return [t[:-1] if len(t) > 3 and t.endswith("s") else t - for t in clean.split()] - - -def _containment_boost( - best: float, toks: set[str], candidates: tuple[str, ...], - allow_single: bool = True, -) -> float: - """Token-containment bonus: a name whose tokens all appear in the - other (or vice versa) matches strongly even when string ratio is low - ("telemon" vs "Telemon Heavy Dreadnought"). Guarded: needs >= 2 - tokens, or a single rare one (>= 7 chars), and the smaller side must - not consist of generic army words ("chaos lord" is contained in half - the catalog) so "head" or "guard" never trigger it. Inner-name votes - pass allow_single=False: one short inner file name ("lieutenant") - must not claim a whole pack.""" - if not toks: - return best - for c in candidates: - ctoks = set(_stem_tokens(c)) - if not ctoks: - continue - small = toks if toks <= ctoks else (ctoks if ctoks <= toks else None) - if small is None or any(t in GENERIC_TOKENS for t in small): - continue - if len(small) >= 2 or (allow_single and len(next(iter(small))) >= 7): - if best < 0.9: - best = 0.9 - break - return best - - -# words too generic for token-containment matching: entries built from -# them alone would claim any file whose name happens to mention them -GENERIC_TOKENS = frozenset({ - "chaos", "lord", "space", "marine", "guard", "knight", "dark", - "death", "blood", "war", "warrior", "daemon", "hero", "god", - "free", "black", "white", "grey", "green", "gold", "iron", - "steel", "storm", "soul", "shadow", "bone", "ghost", "king", - "clan", "tribe", "legion", "regiment", "squad", "warhound", - "orc", "ogre", "elf", "dwarf", "giant", "company", "chapter", - "weapon", "gun", "gunner", "turret", "carriage", "upgrade", -}) # noqa: C405 - - -# directory names that carry no unit information inside sculptor packs; -# never used as match candidates -_JUNK_DIRS = frozenset({ - "files", "file", "images", "image", "img", "pictures", "pics", - "preview", "previews", "render", "renders", "photos", "screenshot", - "screenshots", "torso", "torsos", "head", "heads", "arm", "arms", - "leg", "legs", "back", "front", "left", "right", "top", "bottom", - "side", "weapon", "weapons", "accessories", "accessory", "bits", - "bitz", "parts", "part", "base", "bases", "support", "supports", - "presupported", "unsupported", "stl", "stls", "obj", "models", - "model", "output", "exports", "release", "releases", "license", - "readme", "sources", "docs", "textures", "new", "old", "final", - "fixed", "print", "prints", "plates", "plate", "assembly", "test", - "characters", "character", -}) # noqa: C405 - - -def _clean_inner(name: str) -> str: - """Clean an archive-internal (relative) path: dir names join the vote - ("2E Catachany fellaz/Catachan/Jungle Fighters/x.stl" -> "2e catachany - fellaz catachan jungle fighters x") - the file stem alone loses the - archive's own folder structure.""" - return _clean_name(name.replace("/", " ").replace("\\", " ")) - - -def _is_junk_dir(name: str) -> bool: - """True for directory names that must never be match candidates.""" - clean = _clean_name(name) - if not clean or clean in _JUNK_DIRS or clean.isdigit() or len(clean) <= 2: - return True - return " " not in clean and clean in GENERIC_TOKENS - - -def _token_contained(a_clean: str, b_clean: str) -> bool: - """True when one name's (stemmed) tokens all appear in the other's. - - Guarded like _containment_boost: the smaller side must not consist of - generic army words alone. Used to gate cross-system adoption, where - plain char-ratio lies ("aeldari rangers" vs "dwarf rangers"). - """ - ta = set(_stem_tokens(a_clean)) - tb = set(_stem_tokens(b_clean)) - if not ta or not tb: - return False - small = ta if ta <= tb else (tb if tb <= ta else None) - if small is None: - return False - if len(small) == 1: - # a lone generic word ("guard", "orc") must not carry a match - return next(iter(small)) not in GENERIC_TOKENS - return True - - -def _contained_somewhere(item: SortItem, entry: dict) -> bool: - """Token containment between the entry (or an alias) and the outer - name or any inner (relative-path) name.""" - cands = [entry["_clean"], *entry["_clean_aliases"]] - probes = [item.clean, *[_clean_inner(n) for n in item.inner_names[:10]]] - return any(_token_contained(p, c) for p in probes for c in cands) - - -def _shares_rare_token(a_clean: str, b_clean: str) -> bool: - """True when two cleaned names share at least one non-generic token - (>= 3 chars). Gates the weak tier: a sub-threshold hit with zero - shared words is a char-ratio fluke ("LICENSE.txt" -> "Veletaris Storm - Section"), not evidence. - """ - ta = set(_stem_tokens(a_clean)) - tb = set(_stem_tokens(b_clean)) - shared = (ta & tb) - GENERIC_TOKENS - return any(len(t) >= 3 for t in shared) - - -def _apply_entry( - item: SortItem, entry: dict, score: float, kind: str, confidence: str -) -> None: - """Adopt a catalog entry into an item (dir re-homing and weak tier). - - Unlike the pack/file path in match_catalog, the entry's class layer is - adopted unconditionally: a precise unit match ("Warp Spiders" -> Fly) - must beat a generic keyword class ("warlock" mentioned somewhere -> - Hero). - """ - item.unit = entry["name"] - if entry.get("group"): - item.group = entry["group"] - else: - item.group = GROUP_OF_FACTION.get(item.faction, UNSORTED) - if entry.get("subfaction"): - item.subfaction = entry["subfaction"] - if entry.get("class"): - item.klass = entry["class"] - item.match_score = score - item.match_kind = kind - item.confidence = confidence - entry_faction = entry.get("faction") - if entry_faction and entry_faction not in ("Unit Type", "Heroes", - "Warriors", "Character"): - item.system = entry["system"] - item.faction = entry_faction - - -def _accept_name_entry( - clean: str, entry: dict, score: float, threshold: float -): - """(score, entry, weak) when `entry` may adopt a name scoring `score`. - - Names sharing a rare token are trusted at the normal threshold (and - down to WEAK_MIN); tokenless hits need +TOKENLESS_BONUS and are never - weak. None when rejected. - """ - shared = _shares_rare_token(clean, entry["_clean"]) or any( - _shares_rare_token(clean, a) for a in entry["_clean_aliases"]) - if shared: - if score >= threshold: - return (score, entry, False) - if score >= WEAK_MIN: - return (score, entry, True) - return None - if score >= threshold + TOKENLESS_BONUS: - return (score, entry, False) - return None - - -def _pool_best( - clean: str, pool: list[dict], threshold: float -): - """Best ACCEPTED entry over a pool - normal beats weak, then higher - score; deeper-side ties are irrelevant here (single pool).""" - best_n = best_w = None - for entry in pool: - score = _best_ratio(clean, [entry["_clean"], *entry["_clean_aliases"]]) - a = _accept_name_entry(clean, entry, score, threshold) - if not a: - continue - if a[2]: - if best_w is None or a[0] > best_w[0]: - best_w = a - elif best_n is None or a[0] > best_n[0]: - best_n = a - return best_n or best_w - - -def _match_name( - clean: str, - catalog: list[dict], - system: str, - faction: str, - threshold: float, - cache: dict, -): - """Fuzzy-match a bare name (a directory or pack name) against the catalog. - - Same-system constraint; a different faction needs a strong hit - (CROSS_SYSTEM_MIN). Returns (score, entry, weak) or None - weak marks - a same-faction hit accepted only via the WEAK_MIN tier. - """ - key = (clean, system, faction) - hit = cache.get(key, 0) - if hit != 0: - return hit or None - shim = SortItem(path=Path(), size=0, name="", clean=clean) - result = None - if system == "Unknown": - full = _best_in_pool(shim, catalog, use_inner=False) - if full and full[0] >= CROSS_SYSTEM_MIN and ( - _token_contained(clean, full[1]["_clean"]) - or any(_token_contained(clean, a) - for a in full[1]["_clean_aliases"])): - result = (full[0], full[1], False) - else: - pool = [e for e in catalog if e.get("system") == system] - if faction != UNSORTED: - same = _pool_best( - clean, [e for e in pool if e.get("faction") == faction], - threshold) - other = _pool_best( - clean, [e for e in pool if e.get("faction") != faction], - threshold) - if other and (other[2] or other[0] < CROSS_SYSTEM_MIN): - # a different faction must win strong, never weak - other = None - cands = [c for c in (same, other) if c] - if cands: - normals = [c for c in cands if not c[2]] - result = max(normals or cands, key=lambda c: c[0]) - else: - result = _pool_best(clean, pool, threshold) - cache[key] = result or 0 - return result - - -def _best_dir_match( - item: SortItem, - dirs: list[tuple[Path, str]], - catalog: list[dict], - threshold: float, - cache: dict, -): - """Best matching directory in a deepest-first [(path, name)] chain. - - Returns (score, entry, weak, path) of the winner - higher score wins, - equal keeps the deeper candidate - or None. - """ - best_n = best_w = None - for real_dir, dir_name in dirs: - if _is_junk_dir(dir_name): - continue - m = _match_name(_clean_name(dir_name), catalog, item.system, - item.faction, threshold, cache) - if not m: - continue - score, entry, weak = m - cand = (score, entry, weak, real_dir) - if weak: - if best_w is None or score > best_w[0]: - best_w = cand - elif best_n is None or score > best_n[0]: - best_n = cand - return best_n or best_w - - -def _rehome_from_dirs( - item: SortItem, - dirs: list[tuple[Path, str]], - catalog: list[dict], - threshold: float, - cache: dict, - prefer_dirs_on_tie: bool = False, -): - """Let matching directory names re-home an item ("deepest match wins"). - - Returns the matched directory if the item was re-homed, else None. A - dir hit takes over when it beats the item's current resolution - a - confident pack/file match is only replaced by a strictly better (or - equal, coming from below the pack level) hit; implicit items yield to - any accepted dir match. - """ - if item.match_kind == "invalid": - return None - own = item.match_kind in ("catalog", "weak") - own_score = item.match_score if own else 0.0 - m = _best_dir_match(item, dirs, catalog, threshold, cache) - if not m: - return None - score, entry, weak, real_dir = m - if own and score < own_score: - return None - if own and score == own_score: - # exact tie: the deeper side wins, which is the dir only in - # provenance mode (pack name sits above the inner dirs); in manual - # trees the file name itself is the deepest side - keep it - if not (prefer_dirs_on_tie and not weak): - return None - kind = "weak" if weak else "catalog" - _apply_entry(item, entry, score, kind, "low" if weak else "normal") - return real_dir - - -def _best_in_pool( - item: SortItem, pool: list[dict], use_inner: bool = True -) -> tuple[float, dict] | None: - """Best fuzzy score of `item` against all entries in `pool`. - - Inner archive names are only consulted on request; a confident outer - (file) name must win over votes from random inner STL filenames. - """ - best: tuple[float, dict] | None = None - for entry in pool: - cands = [entry["_clean"], *entry["_clean_aliases"]] - score = _best_ratio(item.clean, cands) - if use_inner and item.inner_names: - inner_clean = [_clean_inner(n) for n in item.inner_names[:10]] - for c in inner_clean: - # no single-token containment boost for inner votes - score = max(score, _best_ratio(c, cands, allow_single=False) * 0.95) - if best is None or score > best[0]: - best = (score, entry) - return best - - -def match_catalog(item: SortItem, catalog: list[dict], threshold: float) -> None: - """Fuzzy-match against catalog entries in the same faction (or all). - - The matched entry is hierarchy-authoritative: group, subfaction and - class come from the catalog (GW shop categories), and when the keyword - classification had no faction the entry's system/faction is adopted too. - """ - if item.faction == UNSORTED and not item.inner_names and not item.clean: - return - if item.match_kind == "invalid": - return # HTML-page masquerading as an archive; never fuzzy-match it - system_known = item.system != "Unknown" - same_faction = [e for e in catalog if e.get("faction") == item.faction] - if system_known: - # a fuzzy match must never jump into another game's section - same_faction = [e for e in same_faction if e.get("system") == item.system] - if same_faction: - best = _best_in_pool(item, same_faction, use_inner=False) - if best is None or best[0] < threshold: - # Outer name not confident: let the archive's inner file names - # vote (the only signal for hash-named gofile archives). - best = _best_in_pool(item, same_faction, use_inner=True) - if best is None or best[0] < threshold: - # The keyword tables can lag behind GW's current categorization - # (e.g. units moved between factions); retry within the same - # system before giving up - never across systems. Keep the - # higher of the two scores (a retry must not downgrade a - # weak-eligible same-faction hit). - sys_pool = [e for e in catalog if e.get("system") == item.system] - sys_best = _best_in_pool(item, sys_pool, use_inner=False) - if sys_best is not None and (best is None or sys_best[0] > best[0]): - best = sys_best - if best is None or best[0] < threshold: - sys_best = _best_in_pool(item, sys_pool, use_inner=True) - if sys_best is not None and (best is None or sys_best[0] > best[0]): - best = sys_best - # a retry hit from a *different* faction is only trusted when - # it is strong AND anchored by token containment; weak - # inner-name votes must not re-home the file - if best is not None and best[1].get("faction") != item.faction \ - and (best[0] < CROSS_SYSTEM_MIN - or not _contained_somewhere(item, best[1])): - best = None - elif system_known: - sys_pool = [e for e in catalog if e.get("system") == item.system] - best = _best_in_pool(item, sys_pool, use_inner=False) - if best is None or best[0] < threshold: - best = _best_in_pool(item, sys_pool, use_inner=True) - else: - # Neither system nor faction known: whole catalog, but only a - # strong hit (CROSS_SYSTEM_MIN) with token containment may adopt - # another game's section; weaker or unanchored guesses stay in - # _unsorted. - best = _best_in_pool(item, catalog, use_inner=False) - if best is None or best[0] < threshold: - best = _best_in_pool(item, catalog, use_inner=True) - if best is not None and best[0] < CROSS_SYSTEM_MIN: - best = None - elif best is not None and not _contained_somewhere(item, best[1]): - best = None - if best and best[0] >= threshold: - entry = best[1] - item.unit = entry["name"] - if entry.get("group"): - item.group = entry["group"] - elif item.faction == UNSORTED: - item.group = GROUP_OF_FACTION.get(entry["faction"], UNSORTED) - if entry.get("subfaction"): - item.subfaction = entry["subfaction"] - # GW's class layer (shop category) beats the keyword guess, but do - # not *narrow* it: "Battleline" is GW's bucket for core infantry, - # a keyword-derived Hero/Elite is more specific for a proxy file. - catalog_klass = entry.get("class") - if catalog_klass: - # a matched catalog unit's class layer is authoritative for - # the whole subtree - a keyword-voted class ("warlock" seen - # somewhere -> Hero) must not survive next to a precise unit - item.klass = catalog_klass - item.match_score = best[0] - item.match_kind = "catalog" - # The catalog entry's faction is authoritative - but only adopt it - # when it is a real army faction. Generic GW branches like - # "Unit Type" or factionless entries would degrade a confident - # keyword faction to a junk folder. - entry_faction = entry.get("faction") - generic_factions = {"Unit Type", "Heroes", "Warriors", "Character"} - if item.faction == UNSORTED: - if entry_faction and entry_faction not in generic_factions: - item.system = entry["system"] - item.faction = entry_faction - if entry.get("group") and entry["group"] not in ("Unit Type",): - item.group = entry["group"] - else: - item.group = GROUP_OF_FACTION.get(item.faction, UNSORTED) - elif entry_faction and entry_faction not in generic_factions and entry_faction != item.faction: - # GW moved this unit to another faction (or the keyword tables - # are stale): trust the catalog. - item.system = entry["system"] - item.faction = entry_faction - if entry.get("group") and entry["group"] != "Unit Type": - item.group = entry["group"] - else: - item.group = GROUP_OF_FACTION.get(entry_faction, UNSORTED) - elif entry.get("group") and entry["group"] not in ("Unit Type",): - item.group = entry["group"] - elif best and best[0] >= WEAK_MIN and ( - _shares_rare_token(item.clean, best[1]["_clean"]) - or any(_shares_rare_token(_clean_inner(n), best[1]["_clean"]) - for n in item.inner_names[:10])): - # Weak tier: a same-system hit below the threshold with at least - # one shared word, accepted only when nothing stronger exists. - # Flagged low confidence. - _apply_entry(item, best[1], best[0], "weak", "low") - else: - if item.faction == UNSORTED: - item.unit = item.name # implicit entry: own (original) name - item.match_score = best[0] if best else 0.0 - item.match_kind = "implicit" - else: - # Faction was known but nothing fuzzy-matched: keep the keyword - # faction and use the file's own name as the unit, but adopt the - # catalog's class layer when it adds information. - item.unit = item.name - item.match_score = best[0] if best else 0.0 - item.match_kind = "implicit" - - -def build_target(item: SortItem, library: Path, inner_rel: Path | None = None) -> Path: - parts: list[str] = [] - if item.system == "Unknown": - parts = [UNSORTED] - if inner_rel is not None and item.unit: - # extracted archives keep their unit folder even when unplaced - parts.append(_display_name(item.unit)) - else: - parts = [item.system] - if item.faction != UNSORTED: - if item.group: - parts.append(item.group) - parts.append(item.faction) - if item.subfaction: - parts.append(item.subfaction) - parts.append(item.klass or UNCLASSIFIED_CLASS) - if item.unit: - parts.append(_display_name(item.unit)) - else: - parts.append(UNSORTED) - target_dir = Path(library).joinpath(*parts) - if inner_rel is not None: - target_dir = target_dir.joinpath(*inner_rel.parts) - return target_dir / item.name - - -def _display_name(unit: str) -> str: - """Implicit entries carry the original filename; strip the extension.""" - p = Path(unit) - return p.stem if p.suffix else unit - - -# --------------------------------------------------------------------------- -# plan / apply -# --------------------------------------------------------------------------- - -def _drop_dropped_system(item: SortItem) -> None: - """Dropped systems (see DROPPED_SYSTEMS) count as unclassified: the - collection does not cover them, so their names must not place files - into those sections. Called after keyword classification, before - matching - the item can still be re-homed by other signals.""" - if item.system in DROPPED_SYSTEMS: - item.system = "Unknown" - item.group = item.faction = UNSORTED - item.subfaction = None - - -def sort_plan( - files: list[Path], - library: Path, - catalog: list[dict], - threshold: float = 0.62, - provenance: dict[str, str] | None = None, - root: Path | None = None, -) -> list[SortItem]: - """Build sort items for `files`. - - `provenance` (with `root`) activates extracted-archive mode: it maps - extraction directories (relative to `root`) to the archive they came - from. All files inside such a directory are classified and matched - once under the archive's name and then placed together, keeping the - archive's inner structure - a loose "Backpack.stl" alone is noise, - under "The Twin Lance/" it belongs to its unit. - """ - prov = provenance or {} - unit_cache: dict[str, dict] = {} - dir_cache: dict = {} - items = [] - for f in sorted(files): - if not f.is_file(): - continue - if (f.name.endswith(".part") or f.name.startswith(".") - or f.name in RUN_ARTIFACTS): - continue - unit_name = dest_key = None - inner_rel: Path | None = None - if root is not None and prov: - rel = f.relative_to(root) - p = rel.parent - while p != Path("."): - if str(p) in prov: - dest_key, unit_name = str(p), prov[str(p)] - # dirs-only relative path (the file name itself is - # appended by build_target; keeping it here would - # duplicate it in the target path) - inner_rel = rel.relative_to(p).parent - break - p = p.parent - if unit_name is None: - item = inspect(f) - classify_item(item) - _drop_dropped_system(item) - match_catalog(item, catalog, threshold) - if root is not None: - # manual trees: let the real ancestor dir names compete - rel = f.relative_to(root) - dirs = [(root.joinpath(*rel.parent.parts[:i + 1]), - rel.parent.parts[i]) - for i in range(len(rel.parent.parts) - 1, -1, -1)] - winner = _rehome_from_dirs(item, dirs, catalog, threshold, - dir_cache) - if winner is not None: - inner_rel = f.parent.relative_to(winner) - else: - # one classification per extraction directory, reused for - # every file inside it - fields = unit_cache.get(dest_key) - if fields is None: - listing = sorted( - str(g.relative_to(root / dest_key)) - for g in (root / dest_key).rglob("*") if g.is_file() - )[:40] - probe = SortItem( - path=f, size=0, name=unit_name, - clean=_clean_name(unit_name), inner_names=listing, - ) - classify_item(probe) - _drop_dropped_system(probe) - match_catalog(probe, catalog, threshold) - fields = {k: getattr(probe, k) for k in ( - "system", "group", "faction", "subfaction", "klass", - "unit", "match_score", "match_kind", "confidence", - )} - unit_cache[dest_key] = fields - item = inspect(f) - for k, v in fields.items(): - setattr(item, k, v) - if inner_rel is not None and inner_rel.parts: - # extracted archives: inner dir names may claim subtrees - dirs = [] - for i in range(len(inner_rel.parts) - 1, -1, -1): - d = (root / dest_key).joinpath(*inner_rel.parts[:i + 1]) - dirs.append((d, inner_rel.parts[i])) - winner = _rehome_from_dirs(item, dirs, catalog, threshold, - dir_cache, prefer_dirs_on_tie=True) - if winner is not None: - inner_rel = f.parent.relative_to(winner) - item.target = build_target(item, library, inner_rel=inner_rel) - item.action = "link" - items.append(item) - return items - - -def apply_plan(items: list[SortItem], audit_path: Path) -> tuple[int, int, int]: - """Create targets. Hardlink preferred; copy when filesystems differ. - - Returns (linked, copied, skipped). - """ - audit_path.parent.mkdir(parents=True, exist_ok=True) - linked = copied = skipped = weak = 0 - weak_lines: list[str] = [] - with audit_path.open("a", encoding="utf-8") as audit: - for item in items: - src, dest = item.path, item.target - if item.match_kind == "invalid": - log.error("Not sorting %s: it is an HTML page, not an archive", src.name) - skipped += 1 - continue - if dest is None or not src.is_file(): - skipped += 1 - continue - if dest.exists(): - log.info("Skip, target exists: %s", dest) - skipped += 1 - continue - dest.parent.mkdir(parents=True, exist_ok=True) - if os.stat(src).st_dev == os.stat(dest.parent).st_dev: - os.link(src, dest) - item.action = "link" - linked += 1 - else: - log.warning( - "Different filesystem - COPYING instead of hardlinking %s", src.name - ) - shutil.copy2(src, dest) - item.action = "copy" - copied += 1 - if item.match_kind == "weak": - weak += 1 - weak_lines.append( - f"{item.name} (score {item.match_score:.2f})" - f"\n -> {dest}\n from {src}\n" - ) - audit.write(json.dumps({ - "from": str(src), "to": str(dest), "action": item.action, - "unit": item.unit, "faction": item.faction, "class": item.klass, - "match": item.match_kind, "confidence": item.confidence, - }) + "\n") - if weak: - weak_doc = audit_path.parent / "sorter_low_confidence.txt" - weak_doc.write_text( - "Low-confidence placements - the sorter accepted these below the " - "normal\nthreshold because nothing better matched. Check and move " - "manually if wrong.\n\n" + "\n".join(weak_lines), - encoding="utf-8", - ) - log.warning( - "%d low-confidence placements - see %s", weak, weak_doc - ) - return linked, copied, skipped - - -def format_plan(items: list[SortItem]) -> str: - lines = [] - for item in items: - if item.match_kind == "invalid": - lines.append(f"[x] {item.name} - NOT AN ARCHIVE (HTML page, bad download)") - continue - src = f"{item.name} ({item.size // (1024 * 1024)} MiB)" if item.size > 4 * 1024 * 1024 else item.name - flag = {"catalog": "+", "weak": "w", "implicit": "~", - "none": "?"}[item.match_kind] - lines.append(f"[{flag}] {src}\n -> {item.target}") - return "\n".join(lines) diff --git a/version-python/dlphelper/state.py b/version-python/dlphelper/state.py deleted file mode 100644 index 7acc2d4..0000000 --- a/version-python/dlphelper/state.py +++ /dev/null @@ -1,78 +0,0 @@ -"""Persistent per-URL state: makes runs resumable and failures trackable. - -state.json lives in the run's OUT_DIR and maps every URL seen to a record: - - { - "status": "ok" | "failed" | "error", - "attempts": , - "error": "", - "files": [{"path": ..., "size": ...}, ...], - "updated": "" - } - -Re-running skips URLs whose status is "ok"; failed URLs are retried and also -written to failed_urls.txt for easy manual retry lists. -""" - -from __future__ import annotations - -import json -import logging -import time -from pathlib import Path - -log = logging.getLogger(__name__) - - -class State: - """JSON-backed state store for one run directory.""" - - def __init__(self, path: Path): - self.path = path - self.records: dict[str, dict] = {} - if path.exists(): - try: - self.records = json.loads(path.read_text(encoding="utf-8")) - if not isinstance(self.records, dict): - raise ValueError("state root must be an object") - except (ValueError, OSError) as exc: - backup = path.with_suffix(".json.bak") - log.warning("Corrupt state file %s (%s); moving to %s and starting fresh", - path, exc, backup) - path.replace(backup) - self.records = {} - - def is_done(self, url: str) -> bool: - rec = self.records.get(url) - return bool(rec) and rec.get("status") == "ok" - - def pending(self, urls: list[str]) -> list[str]: - return [u for u in urls if not self.is_done(u)] - - def mark(self, url: str, **fields) -> dict: - rec = self.records.setdefault(url, {"status": "pending", "attempts": 0, "files": []}) - for key, value in fields.items(): - if value is not None: - rec[key] = value - rec["attempts"] = rec.get("attempts", 0) + 1 - rec["updated"] = time.strftime("%Y-%m-%dT%H:%M:%S") - self.save() - return rec - - def failures(self) -> list[str]: - return [u for u, r in self.records.items() if r.get("status") != "ok"] - - def save(self) -> None: - self.path.parent.mkdir(parents=True, exist_ok=True) - tmp = self.path.with_suffix(".json.tmp") - tmp.write_text( - json.dumps(self.records, indent=2, sort_keys=True), encoding="utf-8" - ) - tmp.replace(self.path) - - -def append_manifest(manifest_path: Path, row: dict) -> None: - """Append one JSON line to manifest.jsonl (per downloaded file).""" - manifest_path.parent.mkdir(parents=True, exist_ok=True) - with manifest_path.open("a", encoding="utf-8") as fh: - fh.write(json.dumps(row, ensure_ascii=False) + "\n") diff --git a/version-python/dlphelper/warhammer_data.py b/version-python/dlphelper/warhammer_data.py deleted file mode 100644 index b58f32e..0000000 --- a/version-python/dlphelper/warhammer_data.py +++ /dev/null @@ -1,529 +0,0 @@ -"""Warhammer taxonomy and unit keyword tables used by the catalogue. - -Naming follows the Games Workshop webshop hierarchy (the same one -dlphelper.gwshop dumps from Algolia): - - /// e.g. - Warhammer 40,000/Space Marines/Ultramarines/Walker - -Two layers: -- SYSTEMS: game system -> faction -> keyword aliases (matched against file - names). Faction detection is the primary signal; unit names double as - faction hints since they are listed under their faction. Space Marine - chapters are their own GW factions under the "Space Marines" group. -- CLASSES: keyword -> class. With GW as the catalog source the class layer - holds GW shop categories (unit types: Battleline, Infantry, Walker, - Vehicle, Flyer, Monster, Mounted, Hero, Titanic). - -The unit table grows with the collection; it does not attempt to enumerate -every unit Games Workshop ever released - unknown files land in the -"unclassified" bucket of the catalogue report so gaps become visible. -""" - -from __future__ import annotations - -# --------------------------------------------------------------------------- -# Game systems and factions (keyword aliases, lowercase) -# --------------------------------------------------------------------------- - -SYSTEMS: dict[str, dict[str, list[str]]] = { - "Warhammer 40,000": { - # -- Space Marines (chapters are their own GW factions) -------------- - "Space Marines": [ - "space marine", "astartes", "primaris", "intercessor", "reiver", - "aggressor", "inceptor", "hellblaster", "eradicator", "bladeguard", - "sternguard", "devastator", "tactical squad", "terminator squad", - "assault marine", "assault intercessor", "outrider", "scout squad", - "vanguard veteran", "invictor", "impulsor", "gladiator", - "land speeder", "drop pod", "inner circle", "honour guard", - ], - "Ultramarines": ["ultramarines", "guilliman", "calgar"], - "Blood Angels": [ - "blood angel", "sanguinary guard", "sanguinor", "death company", - "baal predator", "mephiston", "dante", "astorath", "flesh tearers", - "lamberto", "furioso", - ], - "Dark Angels": [ - "dark angel", "ravenwing", "deathwing", "inner circle companions", - "azrael", "asmodai", "belial", "samuel", "lion el'jonson", - "lion el jonson", "nephilim jetfighter", - ], - "Space Wolves": [ - "space wolf", "space wolves", "wulfen", "thunderwolf", "wolf guard", - "rune priest", "wolf priest", "wolf lord", "logan grimnar", - "ragnar blackmane", "canis wolfborn", "fenrisian wolf", - "stormfang", "murderfang", "lukas the trickster", - ], - "Black Templars": [ - "black templar", "crusader squad", "high marshal helbrecht", - "grimaldus", "the emperor's champion", "emperors champion", - "castellan", "land raider crusader", - ], - "Imperial Fists": [ - "imperial fist", "rogal dorn", "lupos", "tor garadon", - "horatius hold", "dorn's arrow", - ], - "Raven Guard": [ - "raven guard", "kayvaan shrike", "corvus", "sable brand", - "shadow captains", - ], - "Iron Hands": [ - "iron hand", "iron hands", "ferrus manus", "iron father", - "medusan immortals", - ], - "White Scars": ["white scar", "jaghatai khan", "stormseer", "korsos"], - "Salamanders": [ - "salamander", "vulkan", "he'stan", "adrax agatone", - "drake-scale", "flamecaster", - ], - "Grey Knights": [ - "grey knight", "grand master", "brother-captain", "nemesis dreadknight", - "draigo", "castellan crowe", "sternguard knight", - ], - "Deathwatch": [ - "deathwatch", "watch captain", "watch master", "kill team cassius", - "overkill", "corvus blackstar", - ], - "Crimson Fists": ["crimson fist", "kantor", "rynn's world"], - "Flesh Tearers": ["flesh tearer", "gabriel seth", "amissum"], - # -- Imperium -------------------------------------------------------- - "Astra Militarum": [ - "astra militarum", "imperial guard", "cadian", "catachan", "krieg", - "death korps", "tempestus", "scion", "valhallan", "steel legion", - "tallarn", "vostroyan", "tanith", "gaunt's ghosts", "leman russ", - "chimera", "basilisk", "hydra", "wyvern", "manticore", "deathstrike", - "sentinel", "taurox", "commissar", "bullgryn", "ogryn", "ratling", - "heavy weapon squad", "baneblade", "stormlord", "shadowsword", - "hellhammer", "malcador", "macharius", "valkyrie", "vulture", - "primaris psyker", "astropath", "priest", - ], - "Adeptus Mechanicus": [ - "adeptus mechanicus", "mechanicum", "admech", "skitarii", - "cult mechanicus", "sicarian", "ruststalker", "infiltrator", - "kataphron", "kastelan", "ironstrider", "sydonian", - "onager", "dunecrawler", "maniple", "archmagos", "magos", "dominus", - "manipulus", "enginseer", "transuranic", "tech-priest", "cybernetica", - "thallax", "castellax", "vorax", "ordinatus", "belisarius cawl", - ], - "Adepta Sororitas": [ - "adepta sororitas", "sisters of battle", "sister of battle", - "battle sister", "repentia", "retributor", "canoness", "dialogus", - "hospitaller", "seraphim", "zephyrim", "paragon warsuit", - "morvenn vahl", "junith", "immersa", "exorcist", "immolator", - ], - "Adeptus Custodes": [ - "adeptus custodes", "custodes", "custodian", "allarus", "vertus praetor", - "venatari", "aquilon", "sagittarum", "hetaeron", "trajann", "constantin valdor", - "sisters of silence", "witchseeker", "prosecutor", "vigilator", - "caladius", "coronus", "telemon", "achillus", "galatus", "pallas", - ], - "Imperial Knights": [ - "imperial knight", "knight errant", "knight paladin", "knight crusader", - "knight warden", "knight gallant", "knight lancer", "knight acheron", - "knight castellan", "knight valiant", "knight preceptor", "questoris", - "armiger", "warglaive", "helverin", "morialtis", "canis rex", - "cerastus", "atropos", "seneschal", - ], - "Imperial Agents": [ - "inquisition", "inquisitor", "astra telepathica", "witch hunter", - "coteaz", "eisenhorn", "ravenor", "kyria draxus", - "officio assassinorum", "culexus", "vindicare", "callidus", "eversor", - "saint celestine", "living saint", "assassin", - ], - # -- Chaos ----------------------------------------------------------- - "Chaos Space Marines": [ - "chaos space marine", "black legion", "iron warriors", "word bearers", - "night lords", "night lord", "alpha legion", "chaos terminator", "chosen", "havoc", - "raptor", "warp talon", "chaos cultist", "cultist", "daemon prince", - "chaos lord", "chaos sorcerer", "abaddon", "kharn", "ahriman", - "typhus", "fabius bile", "vashtorr", "defiler", "forgefiend", - "maulerfiend", "helldrake", "helbrute", "chaos spawn", - "obliterator", "chaos rhino", "chaos vindicator", "spartan assault", - "mastodon", "typhon", "csm", - ], - "Death Guard": [ - "death guard", "nurgle", "plague marine", "deathshroud", "mortarion", - "plaguecaster", "poxwalker", "blightlord", "foetid bloat-drone", - "plagueburst crawler", "myphitic blight-hauler", "blight hauler", - "tallyman", "foul blightspawn", "biologus putrifier", - "plague surgeon", "necrosius", - ], - "Thousand Sons": [ - "thousand sons", "tzeentch", "rubric marine", "rubricae", - "scarab occult", "magnus the red", "ahzek ahriman", "arhiman", - "exalted sorcerer", "osiron", "mutalith vortex beast", "tzaangor", - "prospero", - ], - "Chaos Daemons": [ - "chaos daemon", "chaos daemons", "daemonette", "bloodletter", - "plaguebearer", "horror", "screamer", "flesh hound", "juggernaut", - "keeper of secrets", "skarbrand", "kairos fateweaver", "rotigus", - "be'lakor", "great unclean one", "lord of change", "soul grinder", - "skull cannon", "burning chariot", "exalted flamer", - "fiend of slaanesh", "plague drone", "nurgling", "herald", - "skulltaker", "khorne", "slaanesh", "skull lord", - "shalaxi helbane", - ], - "World Eaters": [ - "world eater", "angron", "eightbound", "exalted eightbound", - "jakhals", "berserker", "skull lord", - ], - "Emperor's Children": [ - "emperor's children", "emperors children", "fulgrim", - "noise marine", "lucius the eternal", "flawless blades", - ], - "Iron Warriors": ["iron warrior", "perturabo"], - "Red Corsairs": ["red corsair", "huron blackheart"], - "Chaos Knights": [ - "chaos knight", "knight desecrator", "knight despoiler", - "knight rampager", "kharibdyss", "renegade knight", - ], - # -- Xenos ----------------------------------------------------------- - "Orks": [ - "ork", "orks", "orkz", "nob", "boyz", "grot", "weirdboy", "painboy", - "warboss", "ghazghkull", "meganob", "deff dread", "killa kan", - "deffkopta", "stormboy", "kommando", "burna boy", "loota", - "trukk", "battlewagon", "gorkanaut", "morkanaut", "stompa", "squig", - "zagstruk", "snikrot", "speed freek", "boomdakka", "flash gitz", - "wazbom blasta", "dakka jet", "burna-bomber", "blitza-bomber", - "da troopz", "dad and sonz", "ass sassins", "abby boi", - ], - "T'au Empire": [ - "tau", "t'au", "ta'u", "kroot", "vespid", "battlesuit", "crisis suit", - "broadside", "r'varna", "ghostkeel", "stormsurge", - "fire warrior", "pathfinder team", "devilfish", "hammerhead", - "sky ray", "piranha", "sniper drone team", "ethereal", - "commander farsight", "commander shadowsun", "aun'va", "darkstrider", - "longstrike", "mont'ka", "assorted drones", "breacher", - "bomber", "tetra", "barracuda", "sun shark", "razorshark", - "kv128 stormsurge", "tx4 piranha", "kv46", - ], - "Tyranids": [ - "tyranid", "hive tyrant", "swarmlord", "carnifex", "hormagaunt", - "termagant", "termagaunt", "genestealer brood", "zoanthrope", "venomthrope", - "exocrine", "haruspex", "tyranid prime", "tyranid warrior", - "ravener", "gargoyle", "spore mine", "mawloc", "trygon", - "harridan", "harpy", "crone", "tervigon", "barbed hierodule", - "hierodule", "hierophant", "neurothrope", "deathleaper", "lictor", "pyrovore", - "biovore", "broodlord", "bio-titan", "dominatrix", - ], - "Genestealer Cults": [ - "genestealer cult", "genestealer", "aberrant", "acolyte", "neophyte hybrid", - "primus", "magus", "patriarch", "purestrain genestealer", - "sanctus", "kelermorph", "goliath truck", "rockgrinder", - "jackal alcaeus", "cult limo", "hijacked", - ], - "Necrons": [ - "necron", "c'tan", "canoptek", "wraith", "spyder", "scarab", - "tesseract vault", "doomsday ark", "annihilation barge", - "ghost ark", "monolith", "doom scythe", "night scythe", - "imotekh", "szarekh", "nemesor zahndrekh", "vargard obyron", - "lokhrust", "skorpekh", "flayed one", "deathmark", "lychguard", - "necron warrior", "necron immortal", "ovoid", "dzaan", - ], - "Aeldari": [ - "aeldari", "eldar", "craftworld", "ulthwe", "biel-tan", "iyanden", - "alaitoc", "saim-hann", "farseer", "warlock", - "spiritseer", "autarch", "asurmen", "guardian defender", - "ranger", "wraithguard", "wraithblades", "wraithlord", - "wraithknight", "dire avenger", "howling banshee", - "striking scorpion", "swooping hawk", "warp spider", - "fire dragon", "dark reaper", "shining spears", "windrider", - "vyper", "wave serpent", "fire prism", "night spinner", - "war walker", "hemlock wraithfighter", "crimson hunter", - "shade runner", "skyrunner", - ], - "Drukhari": [ - "drukhari", "dark eldar", "kabal", "wych", "wrack", "haemonculus", - "coven", "reaver jetbike", "hellion", "scourge", "mandrake", - "kabalite warrior", "raider", "ravager", "venom", "razorwing", - "voidraven", "grotesque", "talos", "cronos", - "beast pack", "khymera", "razorwing flock", "asdrubael vect", - "lilith", "sathonyx", "luriel", - ], - "Harlequins": [ - "harlequin", "death jester", "shadowseer", "solitaire", - "troupe", "starweaver", "voidweaver", - ], - "Leagues of Votann": [ - "leagues of votann", "votann", "hearthkyn", "ironkin", - "brokhyr", "kahl", "cthonian", "transhyperic", "thegn", - ], - }, - "Age of Sigmar": { - "Stormcast Eternals": [ - "stormcast", "eternal", "liberator", "sequitor", "evocator", - "vindictor", "judicators", "retributors", "sacrosanct", - "dracothian", "lord-arcane", "knight-questor", "neave blacktalon", - "stormdrake", "krondys", "karazai", - ], - "Seraphon": ["seraphon", "saurus", "skink", "slann", "carnosaur", - "bastiladon", "stegadon", "razordon", "salamander"], - "Orruk Warclans": ["orruk", "warclan", "kruleboyz", "ironjawz", - "brute", "gorger", "weirdnob", "warchanter"], - "Gloomspite Gitz": ["gloomspite", "gitz", "moonclan", "spiderfang", - "troggoth", "loontouched", "zarbag", "skragrot"], - "Sylvaneth": ["sylvaneth", "treelord", "dryad", "branchwraith", - "alarielle", "kurnothi"], - "Ossiarch Bonereapers": ["ossiarch", "bonereaper", "mortek", "keldrek", - "katakros"], - "Nighthaunt": ["nighthaunt", "grimghast", "bladegheist", "chainghast", - "kurdoss", "lady olynder"], - "Lumineth Realm-lords": ["lumineth", "aelf", "vanari", "teclis", - "light of eltharion"], - "Daughters of Khaine": ["daughters of khaine", "khainite", "witch aelf", - "morgwaeth", "morathi"], - "Fyreslayers": ["fyreslayer", "vulkite", "hearthguard", "auric", - "grimhold"], - "Kharadron Overlords": ["kharadron", "skyfarer", "endofters", - "grundstok", "arcanaut"], - "Idoneth Deepkin": ["idoneth", "deepkin", "namarti", "isharann", - "volturnos"], - "Blades of Khorne": ["blades of khorne", "khorne", "skullgrinder", - "blood warrior", "wrathmonger"], - "Maggotkin of Nurgle": ["maggotkin", "nurgle", "blightking", - "rotmire", "orghotts"], - "Slaves to Darkness": ["slaves to darkness", "chaos warrior", - "chaos knight", "archaon", "gaunt summoner", - "everchosen"], - "Skaven": ["skaven", "clanrat", "stormvermin", "verminlord", - "grey seer", "warplock", "ratling gun"], - "Soulblight Gravelords": ["soulblight", "gravelord", "vampire", - "deathrattle", "koldhunter", "belladamma"], - "Cities of Sigmar": ["cities of sigmar", "freeguild", "windspeaker", - "callis and toll", "hammerers"], - }, - "The Horus Heresy": { - "Space Marine Legions": [ - "legiones astartes", "pre-heresy", "horus heresy", "justaerin", - "seeker squad", "suzerain", "invictarus", "veteran tactical", - "contemptor", "leviathan pattern", "sicaran", "fellblade", - "kratos", "fire raptor", "eisenstein", - ], - "Mechanicum": ["mechanicum", "fulgurite", "magos", "thallax"], - }, - "Other Games": { - "Kill Team": [ - "kill team", "killteam", "killzone", "operative", - "hand of the archon", "void-dance troupe", "veteran guardsmen", - "spector squad", "wolf scouts", "scout squad kill", - ], - "Necromunda": [ - "necromunda", "goliath gang", "escher", "orlock", "cawdor", - "van saar", "ash wastes", "underhive", "sumpkrocs", - "forge boss", "sumpkroc", "matriarch", "phyrr cat", - ], - "Adeptus Titanicus": [ - "adepticus titanicus", "titan legio", "reaver titan", - "warhound titan", "warlord titan", "imperator titan", - "warmaster titan", "titanicus", - ], - "Warhammer Underworlds": ["warhammer underworlds", "underworlds", - "warband", "shadeborn", "embergard"], - "Aeronautica Imperialis": ["aeronautica", "t'au air caste", - "lightning fighter", "thunderbolt"], - }, - "The Old World": { - "Kingdom of Bretonnia": ["bretonnia", "grail knight", "peasant", - "questing knight", "perrin"], - "Orc and Goblin Tribes": ["greenskin", "goblin", "night goblin", - "forest goblin", "black orc"], - "Tomb Kings of Khemri": ["tomb king", "khemri", "necrolith", - "skeleton archer"], - "Empire of Man": ["empire of man", "state troop", "greatsword", - "empire knight"], - "Warriors of Chaos": ["chaos dwarf", "dog of war", "chaos marauder"], - "High Elf Realms": ["high elf", "eltharion", "phoenix guard"], - "Wood Elf Realms": ["wood elf", "glade guard", "waywatcher"], - "Beastmen Brayherds": ["beastmen", "brayherd", "ghorgon", "cygor"], - "Dwarfen Mountain Holds": ["dwarf", "ironbreaker", "hammerer"], - }, - "Middle-Earth": { - "Rivendell": ["rivendell", "elrond", "elendil", "isildur", "gil-galad", - "beorning", "grimbeorn", "rumil", "orophin"], - "Minas Tirith": ["minas tirith", "gondor", "boromir", "faramir"], - "Mordor": ["mordor", "orc", "orcs", "nazgul", "witch-king"], - "Iron Hills": ["iron hills", "dwarf", "mattock", "dwalin"], - "Moria": ["moria", "cave troll", "balrog"], - }, -} - -# --------------------------------------------------------------------------- -# Classes (keyword -> class). GW shop categories / unit types. -# --------------------------------------------------------------------------- - -CLASSES: list[tuple[str, list[str]]] = [ - ("Titanic", [ - "titan", "stompa", "gargantuan", "hierophant", "harridan", "hierodule", - "imperator", "warmaster", "tesseract vault", "plaguereaper", - "warhound", "warlord titan", "ta'unar", "spermato", "titandeath", - ]), - ("Fortification", ["fortification", "bunker", "bastion", "wall of martyrs"]), - ("Flyer", [ - "flyer", "heldrake", "dakka jet", "burna-bomber", "blitza-bomber", - "bomber", "sky ray", "stormraven", "storm talon", "stormhawk", - "caestus", "valkyrie", "vulture", "thunderbolt", "lightning", - "marauder", "razorwing", "voidraven", "crimson hunter", - "hemlock", "doom scythe", "night scythe", "harpy", "crone", - "fire raptor", "barracuda", "sun shark", "razorshark", - "wazbom blasta", "nephilim jetfighter", "corvus blackstar", - ]), - ("Vehicle", [ - "transport", "rhino", "razorback", "chimera", "taurox", "trukk", - "battlewagon", "devilfish", "wave serpent", "ghost ark", - "land raider", "repulsor", "drop pod", "starweaver", "voidweaver", - "goliath truck", "rockgrinder", "cult limo", "raider", "venom", - "spartan assault", "predator", "vindicator", "whirlwind", - "basilisk", "hydra", "wyvern", "manticore", "deathstrike", - "onager dunecrawler", "kastelan", "hammerhead", "broadside", - "stormsurge", "r'varna", "soul grinder", "defiler", "forgefiend", - "maulerfiend", "plagueburst crawler", "annihilation barge", - "doomsday ark", "fire prism", "night spinner", "exorcist", - "immolator", "caladius", "coronus grav", "pallas grav", - "impulsor", "gladiator", "baal predator", "land speeder", - "ironstrider", "sydonian", "sentinel", "taurox", "kataphron", - ]), - ("Walker", [ - "dreadnought", "redemptor", "contemptor", "leviathan pattern", - "brutalis", "ballistus", "deredeo", "wulfen dreadnought", - "killa kan", "deff dread", "wraithlord", "wraithknight", - "telemon", "achillus", "galatus", "paragon warsuit", - "nemesis dreadknight", "invictor", "talos", "cronos", - ]), - ("Monster", [ - "monster", "carnifex", "exocrine", "haruspex", "mawloc", "trygon", - "tervigon", "hierodule", "hierophant", "mutalith", "bloat-drone", - "kharibdyss", "gorkanaut", "morkanaut", "magnus", "mortalith", - "fiend of slaanesh", "plague drone", "cave troll", "balrog", - "carnosaur", "stegadon", "bastiladon", "grorgeous", - ]), - ("Mounted", [ - "mounted", "thunderwolf", "outrider", "bike squad", "scout biker", - "reaver jetbike", "windrider", "skyrunner", "warbiker", - "allarus custodian on", "vertus praetor", "drakeling", - "boarboy", "gruntaz", "khan on", "dracothian guard", - ]), - ("Battleline", [ - "battleline", "troops", "tactical squad", "intercessor", - "incursor", "infiltrator", "infantry squad", "conscript", - "krieg", "catachan", "cadian", "vostroyan", "penal legion", - "ratling", "veteran squad", "fire warrior", "pathfinder team", - "kroot", "termagant", "hormagaunt", "necron warrior", - "necron immortal", "guardian defender", "storm guardian", - "kabalite warrior", "boyz", "gretchin", "scion", "scout squad", - "cultist", "plague marine squad", - ]), - ("Infantry", [ - "infantry", "veteran", "terminator", "sternguard", "bladeguard", - "aggressor", "reiver", "wulfen", "wolf guard", "deathshroud", - "blightlord", "poxwalker", "rubric marine", "rubricae", - "scarab occult", "tzaangor", "chosen", "possessed", "obliterator", - "daemonette", "bloodletter", "plaguebearer", "nurgling", - "juggernaut", "chaos cultist", "acolyte hybrid", "aberrant", - "neophyte", "purestrain", "genestealer", "lictor", "deathmark", - "flayed one", "lychguard", "wraithguard", "wraithblades", - "howling banshee", "striking scorpion", "warp spider", - "fire dragon", "incubi", "wrack", "grotesque", "assault marine", - "assault intercessor", "inceptor", "raptor", "warp talon", - "flesh hound", "screamer", "hellion", "scourge", "mandrake", - "swooping hawk", "shining spears", "canoptek wraith", - "canoptek scarab", "canoptek spyder", "ravener", "gargoyle", - "spore mine", "pyrovore", "khymera", "beast pack", - "razorwing flock", "zephyrim", "seraphim", "stormboy", - "deffkopta", "skorpekh destroyer", "skorpekh", "retributor", - "dark reaper", "sicarian", "ruststalker", "skitarii", - "hormagaunt brood", "termagant brood", "vanguard", - ]), - ("Hero", [ - "hero", "hq", "captain", "chapter master", "primarch", "lieutenant", - "librarian", "chaplain", "apothecary", "farseer", "warlock", - "spiritseer", "autarch", "warboss", "big mek", "weirdboy", - "painboy", "commander", "ethereal", "hive tyrant", "swarmlord", - "tyranid prime", "neurothrope", "deathleaper", "patriarch", - "primus", "magus", "sanctus", "kelermorph", "overlord", - "imotekh", "szarekh", "zahndrekh", "obyron", "plaguecaster", - "necrosius", "chaos lord", "chaos sorcerer", "daemon prince", - "exalted champion", "herald", "skulltaker", "kairos", "rotigus", - "be'lakor", "great unclean one", "lord of change", - "keeper of secrets", "skarbrand", "kharn", "abaddon", "typhus", - "mortarion", "magnus", "ahriman", "lucius", "fabius bile", - "vashtorr", "canoness", "morvenn", "junith", "immersa", - "dialogus", "hospitaller", "trajann", "valdor", "inquisitor", - "coteaz", "eisenhorn", "ravenor", "draxus", "celestine", - "commissar", "priest", "enginseer", "dominus", "manipulus", - "archmagos", "belisarius cawl", "farsight", "shadowsun", - "aun'va", "darkstrider", "longstrike", "shade runner", - ]), - ("Rules & Accessories", [ - "codex", "rulebook", "rules", "data cards", "datacards", - "warscroll", "index card", "cheat sheet", "reference", - ]), -] - -# Flattened per-system faction keyword index for quick matching -FACTION_ALIASES: dict[str, dict[str, str]] = { - system: {alias: faction for faction, aliases in factions.items() for alias in aliases} - for system, factions in SYSTEMS.items() -} - -UNIT_CLASSES: dict[str, str] = {kw: klass for klass, kws in CLASSES for kw in kws} - -# GW group of each faction (matches the GameSystemsRoot.lvl1 values in the -# Algolia index). Used when a keyword-classified file cannot be fuzzy-matched. -GROUP_OF_FACTION: dict[str, str] = { - faction: group - for group, factions in { - "Space Marines": [ - "Space Marines", "Ultramarines", "Blood Angels", "Dark Angels", - "Space Wolves", "Black Templars", "Imperial Fists", "Raven Guard", - "Iron Hands", "White Scars", "Salamanders", "Grey Knights", - "Deathwatch", "Crimson Fists", "Flesh Tearers", - ], - "Armies of the Imperium": [ - "Astra Militarum", "Adeptus Mechanicus", "Adepta Sororitas", - "Adeptus Custodes", "Imperial Knights", "Imperial Agents", - ], - "Armies of Chaos": [ - "Chaos Space Marines", "Death Guard", "Thousand Sons", - "Chaos Daemons", "Chaos Knights", "World Eaters", - "Emperor's Children", "Iron Warriors", "Red Corsairs", - ], - "Xenos Armies": [ - "Orks", "T'au Empire", "Tyranids", "Genestealer Cults", "Necrons", - "Aeldari", "Drukhari", "Harlequins", "Leagues of Votann", - ], - "Grand Alliance Order": [ - "Stormcast Eternals", "Seraphon", "Sylvaneth", - "Lumineth Realm-lords", "Daughters of Khaine", "Fyreslayers", - "Kharadron Overlords", "Idoneth Deepkin", "Cities of Sigmar", - ], - "Grand Alliance Destruction": [ - "Orruk Warclans", "Gloomspite Gitz", - ], - "Grand Alliance Death": [ - "Ossiarch Bonereapers", "Nighthaunt", "Soulblight Gravelords", - ], - "Grand Alliance Chaos": [ - "Slaves to Darkness", "Blades of Khorne", "Maggotkin of Nurgle", - "Skaven", - ], - "Space Marine Legions": ["Space Marine Legions", "Mechanicum"], - "Kill Team": ["Kill Team"], - "Necromunda": ["Necromunda"], - "Adeptus Titanicus": ["Adeptus Titanicus"], - "Warhammer Underworlds": ["Warhammer Underworlds"], - "Aeronautica Imperialis": ["Aeronautica Imperialis"], - "Armies of the Old World": [ - "Kingdom of Bretonnia", "Orc and Goblin Tribes", - "Tomb Kings of Khemri", "Empire of Man", "Warriors of Chaos", - "High Elf Realms", "Wood Elf Realms", "Beastmen Brayherds", - "Dwarfen Mountain Holds", - ], - "Rivendell": ["Rivendell"], - "Minas Tirith": ["Minas Tirith"], - "Mordor": ["Mordor"], - "Iron Hills": ["Iron Hills"], - "Moria": ["Moria"], - }.items() - for faction in factions -} diff --git a/version-python/dlphelper/zero807.py b/version-python/dlphelper/zero807.py deleted file mode 100644 index d3b76ee..0000000 --- a/version-python/dlphelper/zero807.py +++ /dev/null @@ -1,92 +0,0 @@ -"""Downloader for 0807.st direct links via curl. - -0807.st serves the files directly but rejects requests that don't look like a -browser (requires a User-Agent and the Referer of the site page), and it -rejects multi-connection/range abuse patterns differently than curl - which is -why curl is used instead of aria2c here (see chat history: curl worked, -aria2c's default request got 403). - -Downloads go to a .part file first and are renamed on success, so an -interrupted download is resumable (--continue-at -) and partial files are -never mistaken for finished ones. -""" - -from __future__ import annotations - -import logging -import os -import subprocess -from pathlib import Path -from urllib.parse import urlparse - -log = logging.getLogger(__name__) - -USER_AGENT = "Mozilla/5.0" -REFERER = "https://0807.st/" - - -class Zero807Error(Exception): - """Base error for 0807 downloads.""" - - -class Zero807NotFound(Zero807Error): - """The file no longer exists (HTTP 404/410). Not retryable.""" - - -def target_name(url: str) -> str: - """Derive the filename from the URL path (e.g. /6saCNie.zip -> 6saCNie.zip).""" - name = Path(urlparse(url).path).name - if not name: - raise Zero807Error(f"cannot derive a filename from URL: {url}") - return name - - -def download( - url: str, - out_dir: Path, - timeout_seconds: int = 7200, - resume: bool = True, -) -> tuple[Path, int]: - """Download one URL with curl; returns (final_path, size_bytes).""" - name = target_name(url) - out_dir.mkdir(parents=True, exist_ok=True) - dest = out_dir / name - tmp = dest.with_name(dest.name + ".part") - - if dest.exists() and dest.stat().st_size > 0: - log.info("Skip existing: %s", dest) - return dest, dest.stat().st_size - - cmd = [ - "curl", "-fSL", - "--connect-timeout", "20", - "--max-time", str(timeout_seconds), - "--retry", "2", "--retry-delay", "5", - "-A", USER_AGENT, - "-e", REFERER, - "-o", str(tmp), - ] - if resume: - cmd.append("--continue-at") - cmd.append("-") - cmd.append(url) - - log.info("Downloading %s -> %s", url, dest.name) - proc = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, - encoding="utf-8", errors="replace") - if proc.returncode != 0: - tail = [line for line in proc.stderr.strip().splitlines() if line][-3:] - stderr = " | ".join(tail) - # -f makes curl exit 22 on any HTTP error; dead links fail for good - if "404" in stderr or "410" in stderr: - raise Zero807NotFound(f"dead link (HTTP 404/410): {stderr}") - raise Zero807Error(f"curl exit code {proc.returncode}: {stderr}") - - size = tmp.stat().st_size if tmp.exists() else 0 - if size == 0: - tmp.unlink(missing_ok=True) - raise Zero807Error(f"empty file for {name}") - if dest.suffix.lower() == ".zip" and not tmp.read_bytes()[:2] == b"PK": - log.warning("%s does not look like a ZIP file (first bytes are not PK)", name) - os.replace(tmp, dest) - return dest, size diff --git a/version-python/parts/grab.sh b/version-python/parts/grab.sh deleted file mode 100755 index d656bde..0000000 --- a/version-python/parts/grab.sh +++ /dev/null @@ -1,38 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -THREAD_ID="${THREAD_ID:-1455993943471820842}" -OUT_DIR="${OUT_DIR:-$PWD/thread}" -HTML="$OUT_DIR/export.html" -URLS="$OUT_DIR/urls.txt" # aria2c input file: one URL per line - -# Token from env, otherwise prompt (input hidden) -if [[ -z "${DISCORD_TOKEN:-}" ]]; then - read -rsp "Discord token: " DISCORD_TOKEN - echo -fi - -mkdir -p "$OUT_DIR" - -# 1. Export the thread to HTML -docker run --rm --user "$(id -u):$(id -g)" \ - -v "$OUT_DIR:/out" \ - tyrrrz/discordchatexporter:stable export \ - -t "$DISCORD_TOKEN" -c "$THREAD_ID" -f HtmlDark -o /out/export.html - -unset DISCORD_TOKEN - -# 2. Find base64 strings starting with "https://", then decode them -grep -oP 'aHR0cHM6Ly[89][A-Za-z0-9+/_-]*' "$HTML" \ - | sort -u \ - | while read -r s; do - s=$(printf '%s' "$s" | tr '_-' '/+' | tr -d '=') - while (( ${#s} % 4 )); do s="$s="; done - printf '%s' "$s" | base64 -d 2>/dev/null | tr -d '\r\n' || true - echo - done \ - | grep -E '^https://[[:graph:]]+$' \ - | awk '!seen[$0]++' > "$URLS" - -echo "Found $(wc -l < "$URLS") URLs -> $URLS" -echo "Download with: aria2c -i \"$URLS\" -d \"$OUT_DIR/downloads\"" diff --git a/version-python/requirements.txt b/version-python/requirements.txt deleted file mode 100644 index 535409c..0000000 --- a/version-python/requirements.txt +++ /dev/null @@ -1 +0,0 @@ -requests>=2.31 diff --git a/version-python/sorter.py b/version-python/sorter.py deleted file mode 100644 index cedc964..0000000 --- a/version-python/sorter.py +++ /dev/null @@ -1,140 +0,0 @@ -#!/usr/bin/env python3 -"""Sorter CLI: fuzzysort collection files into the library (plan/apply). - - # plan (default, touches nothing) - python3 sorter.py --library /path/to/library - - # apply: hardlink files into the library (originals stay!) - python3 sorter.py --library /path/to/library --apply - - # widen/narrow fuzzy matching - python3 sorter.py --library ... --threshold 0.5 - - # sort a single archive - python3 sorter.py "Thunderwolf Cavalry.7z" --library ... - -Flags: - --catalog FILE default data/library_catalog.json - --audit FILE default /../_logs/sorter_applied.jsonl - (bookkeeping stays OUT of the library, which may be a - seeded torrent; --audit still overrides) -""" - -from __future__ import annotations - -import argparse -import json -import logging -import sys -from pathlib import Path - -from dlphelper.sorter import ( - DROPPED_SYSTEMS, - format_plan, - load_catalog, - sort_plan, - apply_plan, -) - -log = logging.getLogger("sorter") - -DEFAULT_CATALOG = Path(__file__).resolve().parent / "data" / "gwshop_catalog.json" - - -def main() -> None: - parser = argparse.ArgumentParser( - description="Sort files into the library structure (dry-run by default)." - ) - parser.add_argument("source", type=Path, - help="a directory to sort or a single file") - parser.add_argument("--library", type=Path, required=True, metavar="DIR", - help="library root the files will be linked into") - parser.add_argument("--catalog", type=Path, default=DEFAULT_CATALOG, metavar="FILE") - parser.add_argument("--threshold", type=float, default=0.62, - help="fuzzy match threshold (default 0.62)") - parser.add_argument("--apply", action="store_true", - help="create the targets (hardlink, fallback copy)") - parser.add_argument("--audit", type=Path, default=None, metavar="FILE") - parser.add_argument("--provenance", type=Path, default=None, metavar="FILE", - help="extract_manifest.jsonl from tools/extract_all.py: " - "sort whole extraction directories as units") - parser.add_argument("--json", action="store_true", - help="print the plan as JSON instead of text") - parser.add_argument("--drop-system", action="append", default=[], - metavar="NAME", - help="never place into this game system " - "(default-dropped: Middle-Earth)") - parser.add_argument("--keep-system", action="append", default=[], - metavar="NAME", - help="re-enable a default-dropped game system") - args = parser.parse_args() - - dropped = frozenset(DROPPED_SYSTEMS - set(args.keep_system) - | set(args.drop_system)) - if args.keep_system or args.drop_system: - log.info("System filter adjusted: dropping %s", ", ".join(sorted(dropped)) or "nothing") - - logging.basicConfig(level=logging.INFO, - format="%(asctime)s %(levelname)-7s %(message)s", - datefmt="%H:%M:%S") - - source = args.source - files = [source] if source.is_file() else sorted( - p for p in source.rglob("*") - # hidden trees like .broken/ must never reach the plan - if not any(part.startswith(".") for part in p.relative_to(source).parts) - ) - catalog = load_catalog(args.catalog, dropped=dropped) - if dropped: - log.info("Dropped systems: %s", ", ".join(sorted(dropped))) - # Bookkeeping must never land inside the library itself (it may be a - # seeded torrent whose file set is frozen): default to a _logs folder - # beside it. An explicit --audit overrides this. - audit = args.audit or (args.library.parent / "_logs" / "sorter_applied.jsonl") - log.info("Audit file: %s", audit) - - provenance = None - if args.provenance: - provenance = {} - with args.provenance.open(encoding="utf-8") as fh: - for line in fh: - rec = json.loads(line) - provenance[rec["dest"]] = rec["unit"] - log.info("Provenance: %d extraction directories", len(provenance)) - - items = sort_plan(files, args.library, catalog, threshold=args.threshold, - provenance=provenance, root=source) - - if args.json: - print(json.dumps([ - { - "source": str(i.path), "name": i.name, "size": i.size, - "system": i.system, "group": i.group, "faction": i.faction, - "subfaction": i.subfaction, "class": i.klass, "unit": i.unit, - "score": round(i.match_score, 3), "kind": i.match_kind, - "inner_count": len(i.inner_names), - "target": str(i.target), - } - for i in items - ], indent=2, ensure_ascii=False)) - else: - print(format_plan(items)) - - n_catalog = sum(1 for i in items if i.match_kind == "catalog") - n_implicit = sum(1 for i in items if i.match_kind == "implicit") - n_none = sum(1 for i in items if i.match_kind == "none") - log.info("Plan: %d files -> %d catalog matches [+] / %d implicit [~] / " - "%d unclassifiable [?]", - len(items), n_catalog, n_implicit, n_none) - - if args.apply: - linked, copied, skipped = apply_plan(items, audit) - log.info("Applied: %d hardlinked, %d copied, %d skipped (audit: %s)", - linked, copied, skipped, audit) - - -if __name__ == "__main__": - try: - main() - except KeyboardInterrupt: - sys.exit(130) diff --git a/version-python/tests/test_catalogue.py b/version-python/tests/test_catalogue.py deleted file mode 100644 index af53bc1..0000000 --- a/version-python/tests/test_catalogue.py +++ /dev/null @@ -1,62 +0,0 @@ -#!/usr/bin/env python3 -"""Offline classifier test for the catalogue (no network, no deps). - -Run with: python3 tests/test_catalogue.py -""" - -from __future__ import annotations - -import sys -from pathlib import Path - -sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) - -from dlphelper.catalogue import classify # noqa: E402 - -CASES = [ - # (filename, expected system, expected faction, expected class or None-any) - # GW naming: chapters are factions, classes are GW unit types. - ("Thunderwolf Cavalry.7z", "Warhammer 40,000", "Space Wolves", "Mounted"), - ("Wulfen.zip", "Warhammer 40,000", "Space Wolves", "Infantry"), - ("Keeper of Secrets & Shalaxi Helbane.7z", "Warhammer 40,000", "Chaos Daemons", "Hero"), - ("Assorted Drones.zip", "Warhammer 40,000", "T'au Empire", None), - ("Commander Shadowsun.zip", "Warhammer 40,000", "T'au Empire", "Hero"), - ("Devilfish.zip", "Warhammer 40,000", "T'au Empire", "Vehicle"), - ("Battle Sister Scanz.zip", "Warhammer 40,000", "Adepta Sororitas", None), - ("Knights.zip", None, None, None), # ambiguous keyword alone: unclassified - ("Khorne Demon Enginez and skull lordz.zip", "Warhammer 40,000", "Chaos Daemons", None), - ("Cultists.zip", "Warhammer 40,000", "Chaos Space Marines", "Battleline"), - ("Da Troopz.zip", "Warhammer 40,000", "Orks", None), - ("Seraphon.zip", "Age of Sigmar", "Seraphon", None), - ("Leviathan Pattern Dreadnought.zip", "The Horus Heresy", "Space Marine Legions", "Walker"), - ("Poxwalkers.zip", "Warhammer 40,000", "Death Guard", "Infantry"), - ("Kabalite Warriors.zip", "Warhammer 40,000", "Drukhari", "Battleline"), - ("Canoptek Wraiths.zip", "Warhammer 40,000", "Necrons", "Infantry"), - ("random thing with no keywords.zip", None, None, None), -] - - -def main() -> None: - failures = 0 - for name, exp_system, exp_faction, exp_class in CASES: - system, faction, klass = classify(name) - problems = [] - if exp_system and system != exp_system: - problems.append(f"system={system!r} != {exp_system!r}") - if exp_faction and faction != exp_faction: - problems.append(f"faction={faction!r} != {exp_faction!r}") - if exp_class and klass != exp_class: - problems.append(f"class={klass!r} != {exp_class!r}") - if problems: - failures += 1 - print(f"FAIL {name!r}: {', '.join(problems)}") - else: - print(f"ok {name!r} -> {system} / {faction} / {klass}") - if failures: - print(f"\n{failures} failures") - sys.exit(1) - print("\ntest_catalogue: OK") - - -if __name__ == "__main__": - main() diff --git a/version-python/tests/test_extract.py b/version-python/tests/test_extract.py deleted file mode 100644 index 6755f76..0000000 --- a/version-python/tests/test_extract.py +++ /dev/null @@ -1,68 +0,0 @@ -#!/usr/bin/env python3 -"""Offline extraction test with a synthetic export.json (no network, no deps). - -Run with: python3 tests/test_extract.py -""" - -from __future__ import annotations - -import base64 -import json -import sys -import tempfile -from pathlib import Path - -sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) - -from dlphelper.discord import classify, extract_urls # noqa: E402 - - -def b64(s: str) -> str: - return base64.b64encode(s.encode()).decode() - - -def main() -> None: - messages = [ - { - "id": "1", - "timestamp": "2026-01-01T00:00:00", - "author": {"name": "poster"}, - "content": f"chaos stuff: {b64('https://gofile.io/d/abc123')}", - }, - { - "id": "2", - "timestamp": "2026-01-02T00:00:00", - "author": {"name": "poster"}, - "content": ( - f"0807 zip: {b64('https://0807.st/6saCNie.zip')}\n" - f"dupe of 1: {b64('https://gofile.io/d/abc123')}\n" - f"plain: https://0807.st/plainLink.zip and " - f"https://discord.com/channels/1/2 (unsupported)\n" - f"junk: {b64('https://pixeldrain.com/u/xyz')} " - "and not-our-domain base64: aHR0cHM6Ly9leGFtcGxlLm9yZw==" - ), - }, - ] - - with tempfile.TemporaryDirectory() as tmp: - export = Path(tmp) / "export.json" - export.write_text(json.dumps({"messages": messages}), encoding="utf-8") - found = extract_urls(export) - - urls = {f.url for f in found} - expected = {"https://gofile.io/d/abc123", "https://0807.st/6saCNie.zip", - "https://0807.st/plainLink.zip"} - assert urls == expected, f"unexpected urls: {sorted(urls)}" - - by_url = {f.url: f for f in found} - g = by_url["https://gofile.io/d/abc123"] - assert g.host == "gofile" and g.message_id == "1" and g.author == "poster" - assert classify("https://0807.st/x.zip") == "0807" - assert classify("https://gofile.io/d/x") == "gofile" - assert classify("https://example.org/x") is None - - print("test_extract: OK") - - -if __name__ == "__main__": - main() diff --git a/version-python/tests/test_sorter_units.py b/version-python/tests/test_sorter_units.py deleted file mode 100644 index 9da0274..0000000 --- a/version-python/tests/test_sorter_units.py +++ /dev/null @@ -1,258 +0,0 @@ -"""Offline sorter tests: dir re-homing, weak tier, junk dirs (plain python3).""" -import json -import shutil -import sys -import tempfile -from pathlib import Path - -sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) - -from dlphelper.sorter import ( # noqa: E402 - apply_plan, - load_catalog, - sort_plan, -) - -CATALOG = { - "entries": [ - {"name": "Warp Spiders", "system": "Warhammer 40,000", - "group": "Xenos Armies", "faction": "Aeldari", - "class": "Fly", "aliases": []}, - {"name": "Harlequin Troupe", "system": "Warhammer 40,000", - "group": "Xenos Armies", "faction": "Harlequins", - "class": "Battleline", "aliases": []}, - {"name": "Guardian Squad", "system": "Warhammer 40,000", - "group": "Xenos Armies", "faction": "Aeldari", - "class": "Battleline", "aliases": []}, - {"name": "Rangers", "system": "Warhammer 40,000", - "group": "Xenos Armies", "faction": "Aeldari", - "class": "Battleline", "aliases": []}, - {"name": "Dark Reapers", "system": "Warhammer 40,000", - "group": "Xenos Armies", "faction": "Aeldari", - "class": "Aspect Warriors", "aliases": []}, - {"name": "Mordor Orcs", "system": "Middle-Earth", - "group": "The Lord of the Rings™ - Evil", "faction": "Mordor", - "class": "Warriors", "aliases": []}, - ] -} - - -def make_lib(): - return Path(tempfile.mkdtemp(prefix="sorter_test_lib_")) - - -def write(p: Path): - p.parent.mkdir(parents=True, exist_ok=True) - p.write_bytes(b"x") - - -_LOADED = None - - -def get_catalog(): - global _LOADED - if _LOADED is None: - _LOADED = load_catalog_path() - return _LOADED - - -def prov_items(root: Path, library): - (root / "extract_manifest.jsonl").write_text(json.dumps({ - "archive": str(root / "Multi Pack.zip"), - "dest": "Multi Pack", "unit": "Multi Pack.zip", - }), encoding="utf-8") - files = sorted((root / "Multi Pack").rglob("*")) - return sort_plan(files, library, get_catalog(), provenance={ - "Multi Pack": "Multi Pack.zip"}, root=root) - - -def test_prov_dir_rehome(): - """A matching inner dir name claims its subtree; junk dirs don't.""" - root = Path(tempfile.mkdtemp(prefix="sorter_test_src_")) - pack = root / "Multi Pack" - write(pack / "14 Warp spiders" / "spider_body.stl") - write(pack / "14 Warp spiders" / "files" / "spider_arm.stl") - write(pack / "HEADS" / "head_a.stl") # junk dir: stays with pack - write(pack / "Retro GrimDark Thing" / "t.stl") # no catalog match - library = make_lib() - items = prov_items(root, library) - tgt = {i.name: i.target for i in items} - lib = str(library) - # warp spiders re-homed, incl. nested junk dir structure, class from catalog - assert "Warp Spiders" in str(tgt["spider_body.stl"]), tgt["spider_body.stl"] - assert "Fly" in str(tgt["spider_body.stl"]), tgt["spider_body.stl"] - assert "Hero" not in str(tgt["spider_body.stl"]) - assert "files" in str(tgt["spider_arm.stl"]) - # junk-dir file rides along with the pack's adopted unit - assert "Warp Spiders" in str(tgt["head_a.stl"]) - assert "HEADS" in str(tgt["head_a.stl"]) - # unmatched subtree keeps the pack unit folder - assert "Warp Spiders" in str(tgt["t.stl"]) and "Retro GrimDark Thing" in str(tgt["t.stl"]) - shutil.rmtree(root); shutil.rmtree(library) - - -def test_prov_weak_tier(): - """Below-threshold dir hit is accepted as low confidence.""" - root = Path(tempfile.mkdtemp(prefix="sorter_test_src_")) - # pack name carries a keyword so the probe resolves system+faction - - # only then does the same-faction weak tier apply - pack = root / "Aeldari Pack" - write(pack / "guardian weapon team" / "g.stl") # 0.59 vs Guardian Squad - library = make_lib() - (root / "extract_manifest.jsonl").write_text(json.dumps({ - "archive": str(root / "Aeldari Pack.zip"), - "dest": "Aeldari Pack", "unit": "Aeldari Pack.zip", - }), encoding="utf-8") - files = sorted((root / "Aeldari Pack").rglob("*")) - items = sort_plan(files, library, get_catalog(), provenance={ - "Aeldari Pack": "Aeldari Pack.zip"}, root=root) - it = next(i for i in items if i.name == "g.stl") - assert it.match_kind == "weak", (it.match_kind, it.confidence) - assert it.confidence == "low" - assert "Guardian" in str(it.target) - shutil.rmtree(root); shutil.rmtree(library) - - -def test_cross_system_needs_token_containment(): - """'Aeldari Rangers' must not adopt Middle-earth 'Dwarf Rangers'.""" - root = Path(tempfile.mkdtemp(prefix="sorter_test_src_")) - pack = root / "MQVxKzh" # hash name: system stays unknown - write(pack / "Aeldari Rangers" / "Luminary - Arms - (1).stl") - write(pack / "14 Warp spiders" / "sp.stl") - library = make_lib() - files = sorted((root / "MQVxKzh").rglob("*")) - items = sort_plan(files, library, get_catalog(), provenance={ - "MQVxKzh": "MQVxKzh.zip"}, root=root) - by_name = {i.name: i for i in items} - lum = by_name["Luminary - Arms - (1).stl"] - assert "Middle-Earth" not in str(lum.target), lum.target - spider = by_name["sp.stl"] - assert "Warp Spiders" in str(spider.target), spider.target - shutil.rmtree(root); shutil.rmtree(library) - - -def test_weak_needs_shared_token(): - """Sub-threshold hit with zero shared words is a fluke: reject. - - 'frontline squad' vs 'Guardian Squad' scores 0.552 but shares no - token - it must NOT re-home, while 'guardian weapon team' (0.588, - shares 'guardian') is accepted weak. - """ - root = Path(tempfile.mkdtemp(prefix="sorter_test_src_")) - pack = root / "Aeldari Pack" - write(pack / "frontline squad" / "f.stl") - write(pack / "guardian weapon team" / "g.stl") - library = make_lib() - (root / "extract_manifest.jsonl").write_text(json.dumps({ - "archive": str(root / "Aeldari Pack.zip"), - "dest": "Aeldari Pack", "unit": "Aeldari Pack.zip", - }), encoding="utf-8") - files = sorted((root / "Aeldari Pack").rglob("*")) - items = sort_plan(files, library, get_catalog(), provenance={ - "Aeldari Pack": "Aeldari Pack.zip"}, root=root) - by_name = {i.name: i for i in items} - f = by_name["f.stl"] - # both files ride along with the pack-level weak adoption - assert f.match_kind == "weak" and f.confidence == "low", (f.match_kind, f.target) - assert "frontline squad" in str(f.target) - g = by_name["g.stl"] - assert g.match_kind == "weak" and g.confidence == "low" - shutil.rmtree(root); shutil.rmtree(library) - - -def test_tokenless_needs_bonus(): - """Same-faction adoption without a shared word needs +0.08. - - 'Aeldari Rangers' (0.667 vs 'Dark Reapers', tokenless) must not adopt - Dark Reapers; it instead adopts 'Rangers' (0.636, shares 'ranger'). - """ - root = Path(tempfile.mkdtemp(prefix="sorter_test_src_")) - pack = root / "Aeldari Pack" - write(pack / "Aeldari Rangers" / "r.stl") - library = make_lib() - (root / "extract_manifest.jsonl").write_text(json.dumps({ - "archive": str(root / "Aeldari Pack.zip"), - "dest": "Aeldari Pack", "unit": "Aeldari Pack.zip", - }), encoding="utf-8") - files = sorted((root / "Aeldari Pack").rglob("*")) - items = sort_plan(files, library, get_catalog(), provenance={ - "Aeldari Pack": "Aeldari Pack.zip"}, root=root) - it = next(i for i in items if i.name == "r.stl") - assert it.match_kind == "catalog" and it.unit == "Rangers", \ - (it.match_kind, it.unit, it.match_score) - shutil.rmtree(root); shutil.rmtree(library) - - -def test_dropped_system_mordor(): - """Middle-earth is dropped: a 'Mordor' pack lands in _unsorted, and - its inner dirs cannot adopt the (filtered) Mordor Orcs entry.""" - root = Path(tempfile.mkdtemp(prefix="sorter_test_src_")) - write(root / "Mordor pack" / "boy.stl") - library = make_lib() - items = sort_plan([root / "Mordor pack" / "boy.stl"], library, - get_catalog(), provenance={"Mordor pack": "Mordor pack.zip"}, - root=root) - it = items[0] - assert "Middle-Earth" not in str(it.target), it.target - assert "_unsorted/Mordor pack" in str(it.target).replace("\\", "/") - shutil.rmtree(root); shutil.rmtree(library) - - -def test_load_catalog_filters_dropped(): - cat = load_catalog_path() - assert not any(e["system"] == "Middle-Earth" for e in cat) - assert any(e["name"] == "Warp Spiders" for e in cat) - - -def test_nonprov_rehome(): - """Loose noise file inside a named dir re-homes to the dir's unit.""" - root = Path(tempfile.mkdtemp(prefix="sorter_test_src_")) - write(root / "Warp Spiders" / "body.stl") - library = make_lib() - files = sorted(root.rglob("*")) - items = sort_plan(files, library, get_catalog(), root=root) - it = next(i for i in items if i.name == "body.stl") - assert it.match_kind == "catalog", (it.match_kind, it.target) - assert "Warp Spiders" in str(it.target) and "Fly" in str(it.target) - shutil.rmtree(root); shutil.rmtree(library) - - -def test_apply_weak_doc(): - """apply_plan writes the low-confidence review document.""" - root = Path(tempfile.mkdtemp(prefix="sorter_test_src_")) - pack = root / "Multi Pack" - write(pack / "guardian squad extras" / "g.stl") - library = make_lib() - items = prov_items(root, library) - linked, copied, skipped = apply_plan(items, library / "sorter_applied.jsonl") - doc = library / "sorter_low_confidence.txt" - kinds = [i.match_kind for i in items if i.match_kind == "weak"] - if kinds: - assert doc.exists(), "weak doc missing" - assert "low-confidence" in doc.read_text(encoding="utf-8") - else: - assert not doc.exists() - assert (library / "sorter_applied.jsonl").exists() - shutil.rmtree(root); shutil.rmtree(library) - - -def load_catalog_path(): - import tempfile as tf - fh = tf.NamedTemporaryFile(mode="w", suffix=".json", delete=False) - json.dump(CATALOG, fh) - fh.close() - return load_catalog(Path(fh.name)) - - -def main(): - catalog = load_catalog_path() - for fn in (test_prov_dir_rehome, test_prov_weak_tier, test_nonprov_rehome, - test_cross_system_needs_token_containment, test_apply_weak_doc, - test_weak_needs_shared_token, test_tokenless_needs_bonus, - test_dropped_system_mordor, test_load_catalog_filters_dropped): - fn() - print("OK", fn.__name__) - - -if __name__ == "__main__": - main() diff --git a/version-python/tools/debug_gofile.py b/version-python/tools/debug_gofile.py deleted file mode 100644 index 941f519..0000000 --- a/version-python/tools/debug_gofile.py +++ /dev/null @@ -1,68 +0,0 @@ -#!/usr/bin/env python3 -"""One-off diagnostic: inspect the raw gofile API response for one content id. - -Prints everything except the account token. Creates exactly one guest account. -""" - -import hashlib -import json -import sys -import time - -import requests - -CID = sys.argv[1] - -USER_AGENT = "Mozilla/5.0" -LOCALE = "en-US" -TOKEN_SECRET = "12af056dacea0b" -WINDOW_SEC = 14400 -COMMON = {"User-Agent": USER_AGENT, "Accept-Encoding": "gzip", "Accept": "*/*", - "Connection": "keep-alive"} - -resp = requests.post("https://api.gofile.io/accounts", headers=COMMON, timeout=15) -data = resp.json() -assert data.get("status") == "ok", data -token = data["data"]["token"] - -window = int(time.time()) // WINDOW_SEC -seed = f"{USER_AGENT}::{LOCALE}::{token}::{window}::{TOKEN_SECRET}" -wtoken = hashlib.sha256(seed.encode()).hexdigest() -headers = { - **COMMON, - "Authorization": f"Bearer {token}", - "X-Website-Token": wtoken, - "X-BL": LOCALE, -} - -r = requests.get( - f"https://api.gofile.io/contents/{CID}?cache=true&sortField=createTime&sortDirection=1", - headers=headers, timeout=30, -) -print("HTTP", r.status_code) -body = r.json() -print("api status:", body.get("status")) - - -def scrub(node): - """Remove anything token-like from the tree before printing.""" - if isinstance(node, dict): - return { - k: ("" if k in ("token", "accountToken") else scrub(v)) - for k, v in node.items() - } - if isinstance(node, list): - return [scrub(x) for x in node] - return node - - -d = body.get("data") or {} -print("data keys:", sorted(d.keys())) -print("type:", d.get("type")) -children = d.get("children") or {} -print("children count:", len(children)) -for cid, child in list(children.items())[:5]: - print(" child:", cid, "type:", child.get("type"), "name:", child.get("name"), - "has link:", "link" in child) -print() -print(json.dumps(scrub(body), indent=2)[:3000]) diff --git a/version-python/tools/extract_all.py b/version-python/tools/extract_all.py deleted file mode 100644 index 0edd665..0000000 --- a/version-python/tools/extract_all.py +++ /dev/null @@ -1,379 +0,0 @@ -#!/usr/bin/env python3 -"""Recursively extract every archive under the source roots into one tree. - - python3 tools/extract_all.py ... --out - -Behavior: -- Every archive (zip/7z/rar/tar/gz/bz2/xz) is extracted into a directory - named after the archive (suffix stripped), preserving the relative path - under the source root: /zips/The Twin Lance.7z - -> /1/zips/The Twin Lance/... -- Archives found *inside* an extraction are extracted too (BFS, until no - archive remains), then deleted - the extracted tree holds no archives. -- Plain files are hardlinked into the tree (fallback: copy), so nothing - is lost and no data is duplicated. -- The source roots are never modified (originals stay). -- A provenance manifest `/extract_manifest.jsonl` records one line - per extraction: {"archive": ..., "dest": ..., "unit": ...} for the - sorter's --provenance unit mode. -- Failures (bad download/HTML page, wrong password, disk full) are moved - to `/.broken/` and logged; the run never stops. - -Re-runs are safe: destinations recorded in the manifest are skipped (a -partial dir from a crash is NOT skipped - it gets re-extracted); a final -sweep expands archives stranded by interrupted nested expansions. -""" - -from __future__ import annotations - -import argparse -import logging -import os -import shutil -import subprocess -import sys -import tempfile -from collections import deque -from pathlib import Path - -log = logging.getLogger("extract_all") - -ARCHIVE_SUFFIXES = {".zip", ".7z", ".rar", ".tar", ".gz", ".bz2", ".xz"} -EXTRACT_TIMEOUT = 4 * 3600 # seconds per archive -MIN_FREE_BYTES = 20 * 1024**3 # stop extracting below this free space -RUN_ARTIFACTS = { - "export.json", "export.html", "state.json", "manifest.jsonl", - "failed_urls.txt", "catalogue.md", "catalogue.jsonl", - "sorter_applied.jsonl", "catalogue_moves.jsonl", "urls.txt", - "urls_0807.txt", "urls_gofile.txt", "failed_0807.txt", - "extract_manifest.jsonl", -} - - -def archive_suffixes(name: str) -> list[str]: - """All trailing archive suffixes, e.g. 'x.tar.gz' -> ['.gz', '.tar'].""" - out = [] - p = Path(name) - while p.suffix.lower() in ARCHIVE_SUFFIXES: - out.append(p.suffix.lower()) - p = p.with_suffix("") - return out - - -def is_archive(path: Path) -> bool: - return bool(archive_suffixes(path.name)) - - -def display_unit(name: str) -> str: - """Archive name with all archive suffixes stripped: the unit folder.""" - p = Path(name) - while p.suffix.lower() in ARCHIVE_SUFFIXES: - p = p.with_suffix("") - return p.name - - -def free_bytes(path: Path) -> int: - return shutil.disk_usage(path).free - - -def _fix_component(s: str) -> str: - """Repair filenames 7z wrote with raw legacy bytes (cp437 zip entries). - - Raw non-UTF8 bytes in a name cannot be created on the CIFS share - (observed: errno 2 for a name containing byte 0x87). Decode the raw - bytes as cp437 so e.g. a raw-0x87 name becomes a proper 'ç' name. - """ - try: - s.encode("utf-8") - return s - except UnicodeEncodeError: - pass - raw = os.fsencode(s) - try: - return raw.decode("utf-8") - except UnicodeDecodeError: - return raw.decode("cp437", "replace") - - -def _move_tree(tmp: Path, dest: Path) -> int: - """Move a freshly extracted local tree onto the (possibly CIFS) dest. - Returns the number of files that could not be moved.""" - bad = 0 - for f in sorted(tmp.rglob("*")): - rel = f.relative_to(tmp) - tgt = dest.joinpath(*(_fix_component(p) for p in rel.parts)) - if f.is_dir(): - tgt.mkdir(parents=True, exist_ok=True) - continue - tgt.parent.mkdir(parents=True, exist_ok=True) - try: - if tgt.exists() and tgt.is_file() \ - and tgt.stat().st_size == f.stat().st_size: - f.unlink() - continue - if tgt.exists(): - tgt.unlink() - shutil.move(str(f), str(tgt)) - except OSError as exc: - bad += 1 - log.error("Cannot place %s: %s", tgt, exc) - return bad - - -def _run_7z(cmd: list[str]) -> tuple[int, str]: - try: - # 7z prints legacy-encoded inner filenames (non-UTF8 bytes); decode - # tolerantly or the whole run crashes (seen with byte 0x87). - proc = subprocess.run( - cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, - encoding="utf-8", errors="replace", timeout=EXTRACT_TIMEOUT, - ) - except subprocess.TimeoutExpired: - return -1, f"extraction timed out after {EXTRACT_TIMEOUT}s" - except OSError as exc: - return -1, f"7z failed to run: {exc}" - tail = "\n".join((proc.stdout or "").splitlines()[-5:]) - return proc.returncode, tail - - -def extract(archive: Path, dest: Path) -> tuple[bool, str, Path]: - """7z x into dest. Empty -p avoids interactive prompts (no password). - - Returns (ok, error, dest_used): on a name collision (the archive stem - exists as a *file*, e.g. 'model.stl.zip' next to 'model.stl') the - destination gets a '_dup' suffix instead of crashing. - - If 7z cannot write directly onto the (CIFS) destination - legacy - cp437 entry names - the archive is extracted to a local temp dir and - moved over with repaired filenames. - """ - actual = dest - for _ in range(5): - try: - actual.mkdir(parents=True, exist_ok=True) - break - except FileExistsError: - actual = dest.with_name(dest.name + "_dup") - except NotADirectoryError as exc: - return False, f"cannot create destination {actual}: {exc}", dest - else: - return False, f"cannot create destination {dest} (repeated collisions)", dest - rc, tail = _run_7z(["7z", "x", "-y", "-p", f"-o{actual}", str(archive)]) - if rc in (0, 1): # 0 ok, 1 warnings (e.g. minor CRC) - if rc == 1: - log.warning("7z reported warnings extracting %s", archive.name) - return True, "", actual - if rc == -1: - return False, tail, actual - # Fallback: local extraction, then move with repaired filenames. - tmp = Path(tempfile.mkdtemp(prefix="7zext-", dir="/var/tmp")) - try: - rc2, tail2 = _run_7z(["7z", "x", "-y", "-p", f"-o{tmp}", str(archive)]) - if rc2 not in (0, 1): - return False, f"7z exit {rc2}: {tail2}", actual - bad = _move_tree(tmp, actual) - if bad: - return False, f"{bad} files could not be moved onto the share", actual - return True, "", actual - finally: - shutil.rmtree(tmp, ignore_errors=True) - - -def move_to_broken(path: Path, out_root: Path, src_root: Path, note: str) -> Path: - rel = path.relative_to(src_root) - dest = out_root / ".broken" / rel - dest.parent.mkdir(parents=True, exist_ok=True) - if dest.exists(): - dest = dest.with_name(dest.name + ".1") - return shutil.move(str(path), str(dest)) - - -def is_macosx_junk(p: Path) -> bool: - """AppleDouble resource-fork 'zips' (.__MACOSX/._x.zip) - no content.""" - return "__MACOSX" in p.parts or p.name.startswith("._") - - -def expand_nested(dest: Path, out_root: Path, manifest, src_root: Path, - done: set[str]) -> int: - """Extract archives inside `dest` until none remain. Returns count.""" - done_count = 0 - queue: deque[Path] = deque([dest]) - while queue: - cur = queue.popleft() - for p in sorted(cur.rglob("*")): - if not p.is_file() or not is_archive(p): - continue - if is_macosx_junk(p): - p.unlink() - log.info("Junk (macOS AppleDouble) removed: %s", p.name) - continue - stem_dir = p.with_name(display_unit(p.name)) - if str(stem_dir.relative_to(out_root)) in done: - # already extracted in a previous run; drop the archive copy - p.unlink() - log.info("Nested %s already extracted - archive removed", p.name) - continue - if free_bytes(out_root) < MIN_FREE_BYTES: - log.error("Out of disk space - skipping %s (kept in place)", p) - continue - ok, err, actual = extract(p, stem_dir) - if not ok: - log.error("FAILED nested extraction %s: %s", p, err) - move_to_broken(p, out_root, out_root, err) - continue - manifest.write(f"{json_line(p.relative_to(out_root), actual)}\n") - done.add(str(actual.relative_to(out_root))) - p.unlink() # archive copy consumed - done_count += 1 - queue.append(actual) - return done_count - - -def json_line(archive_rel: Path, dest: Path) -> str: - import json - return json.dumps({ - "archive": str(archive_rel), - "dest": str(dest.relative_to(_OUT_ROOT)), - "unit": display_unit(archive_rel.name), - }, ensure_ascii=False) - - -_OUT_ROOT: Path = Path(".") # set in main; used by json_line - - -def process_root(src_root: Path, out_root: Path, manifest, done: set[str]) -> tuple[int, int, int]: - """Returns (extracted, mirrored, failed).""" - idx = src_root.name - extracted = mirrored = failed = 0 - for f in sorted(src_root.rglob("*")): - if not f.is_file(): - continue - if (f.name in RUN_ARTIFACTS or f.name.startswith(".") - or f.name.endswith(".part")): - continue - rel = f.relative_to(src_root) - if is_archive(f): - dest_rel = Path(idx) / Path(str(rel.with_name(display_unit(f.name)))) - dest = out_root / dest_rel - # Skip on a manifest record, never on "dir non-empty": a crash - # mid-extraction leaves a partial dir that must be redone. - if str(dest_rel) in done: - log.info("Skip (already extracted): %s", rel) - continue - if free_bytes(out_root) < MIN_FREE_BYTES: - log.error("Out of disk space - skipping archive %s", f) - failed += 1 - continue - ok, err, actual = extract(f, dest) - if not ok: - log.error("FAILED %s: %s", rel, err) - move_to_broken(f, out_root, src_root, err) - failed += 1 - continue - manifest.write(f"{json_line(Path(idx) / rel, actual)}\n") - done.add(str(actual.relative_to(out_root))) - extracted += 1 - log.info("Extracted %s -> %s", rel, actual.relative_to(out_root)) - expand_nested(actual, out_root, manifest, src_root, done) - else: - link_dest = out_root / idx / rel - if link_dest.exists(): - continue - link_dest.parent.mkdir(parents=True, exist_ok=True) - try: - os.link(f, link_dest) - except OSError: - shutil.copy2(f, link_dest) - mirrored += 1 - return extracted, mirrored, failed - - -def load_done(manifest_path: Path) -> set[str]: - """Dest dirs already recorded in the manifest (relative to out root).""" - import json - done: set[str] = set() - if manifest_path.exists(): - with manifest_path.open(encoding="utf-8") as fh: - for line in fh: - line = line.strip() - if not line: - continue - try: - done.add(json.loads(line)["dest"]) - except (json.JSONDecodeError, KeyError): - continue - return done - - -def sweep(out_root: Path, manifest, done: set[str]) -> int: - """Expand any archive left anywhere in the tree (interrupted nested - expansions leave stranded copies even when the parent has a record).""" - import json - swept = 0 - for p in sorted(out_root.rglob("*")): - if not p.is_file() or not is_archive(p): - continue - if ".broken" in p.relative_to(out_root).parts: - continue - if is_macosx_junk(p): - p.unlink() - log.info("Junk (macOS AppleDouble) removed: %s", p.name) - continue - stem_dir = p.with_name(display_unit(p.name)) - if str(stem_dir.relative_to(out_root)) in done: - p.unlink() - log.info("Sweep: %s already extracted - archive removed", p.name) - continue - if free_bytes(out_root) < MIN_FREE_BYTES: - log.error("Out of disk space - sweep skipping %s", p) - continue - ok, err, actual = extract(p, stem_dir) - if not ok: - log.error("FAILED sweep extraction %s: %s", p, err) - move_to_broken(p, out_root, out_root, err) - continue - manifest.write(f"{json_line(p.relative_to(out_root), actual)}\n") - done.add(str(actual.relative_to(out_root))) - p.unlink() - swept += 1 - expand_nested(actual, out_root, manifest, out_root, done) - return swept - - -def main() -> None: - parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) - parser.add_argument("sources", nargs="+", type=Path) - parser.add_argument("--out", type=Path, required=True, metavar="DIR") - args = parser.parse_args() - - logging.basicConfig(level=logging.INFO, - format="%(asctime)s %(levelname)-7s %(message)s", - datefmt="%H:%M:%S") - - global _OUT_ROOT - _OUT_ROOT = args.out - args.out.mkdir(parents=True, exist_ok=True) - - totals = [0, 0, 0] - done = load_done(args.out / "extract_manifest.jsonl") - log.info("%d destinations already recorded in manifest", len(done)) - with (args.out / "extract_manifest.jsonl").open("a", encoding="utf-8") as manifest: - for src in args.sources: - log.info("== processing %s", src) - counts = process_root(src, args.out, manifest, done) - log.info("== %s: %d extracted, %d mirrored, %d failed", - src.name, *counts) - for i, c in enumerate(counts): - totals[i] += c - swept = sweep(args.out, manifest, done) - log.info("TOTAL: %d archives extracted, %d files mirrored, %d failed, %d swept", - *totals, swept) - if totals[2]: - log.warning("Failures were moved under %s - inspect and re-run", args.out / ".broken") - - -if __name__ == "__main__": - try: - main() - except KeyboardInterrupt: - sys.exit(130) diff --git a/version-python/tools/gw_browse_test.py b/version-python/tools/gw_browse_test.py deleted file mode 100644 index 79759e8..0000000 --- a/version-python/tools/gw_browse_test.py +++ /dev/null @@ -1,21 +0,0 @@ -#!/usr/bin/env python3 -"""One-shot browse of the GW Algolia index -> gwshop_raw.json (gitignored).""" -import json -import logging -import pathlib -import sys - -sys.path.insert(0, str(pathlib.Path(__file__).resolve().parent.parent)) - -import dlphelper.gwshop as gw - -logging.basicConfig(level=logging.INFO, format="%(levelname)s %(message)s") - -app, key = gw.load_credentials() -client = gw.GwShopClient(app, key) -hits = client.browse("prod-lazarus-product-en-ww", hits_per_page=100) -print("total hits", len(hits)) -pathlib.Path("gwshop_raw.json").write_text( - json.dumps(hits, ensure_ascii=False), encoding="utf-8" -) -print("written gwshop_raw.json") diff --git a/version-python/tools/gw_build_catalog.py b/version-python/tools/gw_build_catalog.py deleted file mode 100644 index 8d41a16..0000000 --- a/version-python/tools/gw_build_catalog.py +++ /dev/null @@ -1,42 +0,0 @@ -#!/usr/bin/env python3 -"""gwshop_raw.json -> data/gwshop_catalog.json + quick shape report.""" -import json -import logging -import pathlib -import sys -from collections import Counter - -sys.path.insert(0, str(pathlib.Path(__file__).resolve().parent.parent)) - -import dlphelper.gwshop as gw - -logging.basicConfig(level=logging.INFO, format="%(levelname)s %(message)s") - -hits = json.loads(pathlib.Path("gwshop_raw.json").read_text(encoding="utf-8")) -print("raw hits:", len(hits)) -unique = len({h.get("objectID") for h in hits}) -print("unique objectIDs:", unique) - -catalog = gw.build_catalog(hits) -pathlib.Path("data/gwshop_catalog.json").write_text( - json.dumps(catalog, ensure_ascii=False, indent=1), encoding="utf-8" -) -print("entries:", catalog["generated_entries"]) - -entries = catalog["entries"] -print("no system:", sum(1 for e in entries if e["system"] is None)) -print("by system:", Counter(e["system"] for e in entries).most_common()) -print("40k groups:", Counter( - e["group"] for e in entries if e["system"] == "Warhammer 40,000" -).most_common()) -print("lvl depth:", Counter( - len((e.get("gw_paths") or [[]])[0]) for e in entries -).most_common()) -print("classes (lvl>=4):", Counter( - e["class"] for e in entries if e["class"] -).most_common(12)) -sample = next( - e for e in entries - if e["name"].startswith("Codex: Adeptus Custodes") -) -print(json.dumps(sample, indent=1, ensure_ascii=False)[:900]) diff --git a/version-python/tools/gw_inspect.py b/version-python/tools/gw_inspect.py deleted file mode 100644 index e597b2a..0000000 --- a/version-python/tools/gw_inspect.py +++ /dev/null @@ -1,34 +0,0 @@ -#!/usr/bin/env python3 -"""Inspect gwshop catalog shape: Space Marines chapters, AoS, dedupe stats.""" -import json -import pathlib -from collections import Counter - -cat = json.loads(pathlib.Path("data/gwshop_catalog.json").read_text(encoding="utf-8")) -entries = cat["entries"] - -def show(pred, label, n=6): - print(f"--- {label}") - shown = 0 - for e in entries: - if pred(e): - p = " > ".join(x for x in (e["system"], e["group"], e["faction"], e["subfaction"], e["class"]) if x) - print(f" {e['name'][:60]!r} -> {p}") - shown += 1 - if shown >= n: - break - -show(lambda e: e["group"] == "Space Marines", "Space Marines group samples") -show(lambda e: e["group"] == "Armies of the Imperium" and e["subfaction"], "Imperium with subfaction") -show(lambda e: e["system"] == "Age of Sigmar", "Age of Sigmar samples", 5) -show(lambda e: e["system"] == "The Old World", "Old World samples", 5) -show(lambda e: e["system"] == "Other Games", "Other Games samples", 5) -show(lambda e: e["group"] == "Unit Type", "Unit Type branch", 6) - -names = Counter(e["name"].casefold() for e in entries) -dups = {n: c for n, c in names.items() if c > 1} -print("duplicate names:", len(dups), "example:", list(dups.items())[:3]) - -# faction-level value census for 40k -print("--- 40k factions census") -print(Counter(e["faction"] for e in entries if e["system"] == "Warhammer 40,000").most_common(40)) diff --git a/version-python/tools/new_repo.py b/version-python/tools/new_repo.py deleted file mode 100644 index 305920a..0000000 --- a/version-python/tools/new_repo.py +++ /dev/null @@ -1,108 +0,0 @@ -#!/usr/bin/env python3 -"""Create or list repositories on the Gitea instance at git.dieterunterstrich.de. - -Credentials are read from ~/.git-credentials (the line for that host); the -secret is never printed. Useful when you need a new repo without touching the -web UI. - -Usage: - python3 tools/new_repo.py list - python3 tools/new_repo.py create [--private] [--desc "..."] -""" - -from __future__ import annotations - -import argparse -import base64 -import json -import sys -from pathlib import Path -from urllib.request import Request, urlopen - -HOST = "git.dieterunterstrich.de" -API = f"https://{HOST}/api/v1" -CREDENTIALS_FILE = Path.home() / ".git-credentials" - - -def load_credentials() -> tuple[str, str]: - """Return (username, password) for the Gitea host from ~/.git-credentials.""" - for line in CREDENTIALS_FILE.read_text().splitlines(): - line = line.strip() - if not line.startswith("https://") or HOST not in line: - continue - # https://user:pass@host - rest = line.removeprefix("https://") - userinfo = rest.split("@", 1)[0] - user, _, password = userinfo.partition(":") - return user, password - raise SystemExit(f"No credentials for {HOST} in {CREDENTIALS_FILE}") - - -def api(method: str, path: str, body: dict | None = None) -> tuple[int, dict]: - """Call the Gitea API; returns (status_code, parsed_json).""" - user, password = load_credentials() - auth = base64.b64encode(f"{user}:{password}".encode()).decode() - data = json.dumps(body).encode() if body is not None else None - req = Request( - f"{API}{path}", - data=data, - method=method, - headers={ - "Authorization": f"Basic {auth}", - "Content-Type": "application/json", - }, - ) - try: - resp = urlopen(req, timeout=30) - return resp.status, json.loads(resp.read() or b"{}") - except Exception as exc: # noqa: BLE001 - # HTTPError carries the response body, useful for API errors - code = getattr(exc, "code", None) - if code is not None: - try: - return code, json.loads(exc.read() or b"{}") - except Exception: # noqa: BLE001 - return code, {"message": str(exc)} - raise - - -def main() -> None: - parser = argparse.ArgumentParser(description="Gitea repo helper") - sub = parser.add_subparsers(dest="command", required=True) - - sub.add_parser("list") - - p_create = sub.add_parser("create") - p_create.add_argument("name") - p_create.add_argument("--desc", default=None) - p_create.add_argument("--public", action="store_true") - - args = parser.parse_args() - - if args.command == "list": - status, repos = api("GET", "/user/repos?limit=100") - print(f"HTTP {status}") - for repo in repos: - print(f" {repo['full_name']} (private={repo['private']})") - return - - status, result = api( - "POST", - "/user/repos", - { - "name": args.name, - "description": args.desc or "", - "private": not args.public, - "auto_init": False, - "default_branch": "main", - }, - ) - if status in (201, 200): - print(f"Created: {result['full_name']} -> {result['clone_url']}") - else: - print(f"Failed (HTTP {status}): {result.get('message', result)}") - sys.exit(1) - - -if __name__ == "__main__": - main() diff --git a/version-python/tools/plan_diff.py b/version-python/tools/plan_diff.py deleted file mode 100644 index 4394437..0000000 --- a/version-python/tools/plan_diff.py +++ /dev/null @@ -1,26 +0,0 @@ -#!/usr/bin/env python3 -"""Diff old vs new sorter plans by source path -> target.""" -import json -import sys - -changed = 0 -same = 0 -examples = [] -for n in (1, 2, 3): - old = {p["source"]: p for p in json.load(open(f"/tmp/plan{n}.json"))} - new = {p["source"]: p for p in json.load(open(f"/tmp/plan{n}b.json"))} - for src, np in new.items(): - op = old.get(src) - if not op: - continue - if op["target"] != np["target"] or op["kind"] != np["kind"]: - changed += 1 - if len(examples) < 40: - examples.append((n, op, np)) - else: - same += 1 -print(f"same: {same} changed: {changed}") -for n, op, np in examples: - print(f"\n[{n}] {op['name']} (score {op['score']} -> {np['score']})") - print(f" old: {op['target']}") - print(f" new: {np['target']}") diff --git a/version-python/tools/plan_summary.py b/version-python/tools/plan_summary.py deleted file mode 100644 index 4b96721..0000000 --- a/version-python/tools/plan_summary.py +++ /dev/null @@ -1,22 +0,0 @@ -#!/usr/bin/env python3 -"""Summarize a sorter plan JSON: match kinds, top targets, oddities.""" -import json -import sys -from collections import Counter -from pathlib import Path - -plan = json.loads(Path(sys.argv[1]).read_text()) -print(f"files: {len(plan)}") -print("kinds:", Counter(p["kind"] for p in plan)) -print("actions:", Counter(p.get("action", "?") for p in plan)) -# system breakdown -print("systems:", Counter(p["system"] for p in plan).most_common(10)) -# top-level targets -roots = Counter(str(Path(p["target"]).relative_to( - Path(p["target"]).parents[3])) if len(Path(p["target"]).parents) > 3 - else p["target"] for p in plan) -for root, n in roots.most_common(15): - print(f" {n:5d} {root}") -# anything with empty target? -empties = [p for p in plan if not p.get("target")] -print("empty targets:", len(empties)) diff --git a/version-rust/.gitignore b/version-rust/.gitignore deleted file mode 100644 index ea8c4bf..0000000 --- a/version-rust/.gitignore +++ /dev/null @@ -1 +0,0 @@ -/target diff --git a/version-rust/tools/dump_warhammer_data.py b/version-rust/tools/dump_warhammer_data.py deleted file mode 100644 index b897296..0000000 --- a/version-rust/tools/dump_warhammer_data.py +++ /dev/null @@ -1,29 +0,0 @@ -#!/usr/bin/env python3 -"""Regenerate version-rust/data/warhammer_data.json from the python module. - -Run from the repo root (the version containing the python layout): - - python3 version-rust/tools/dump_warhammer_data.py - -The Rust rewrite parses this file (src/warhammer_data.rs) instead of holding -a hand-transcribed copy of the taxonomy tables, so the two versions work -from byte-identical data. -""" - -import json -import sys -from pathlib import Path - -sys.path.insert(0, str(Path(__file__).resolve().parent.parent.parent / "version-python")) - -from dlphelper import warhammer_data as w # noqa: E402 - -out = Path(__file__).resolve().parent.parent / "data" / "warhammer_data.json" - -payload = { - "systems": w.SYSTEMS, - "classes": [[name, keywords] for name, keywords in w.CLASSES], - "group_of_faction": w.GROUP_OF_FACTION, -} -out.write_text(json.dumps(payload, ensure_ascii=False, indent=1), encoding="utf-8") -print(f"wrote {out} ({out.stat().st_size} bytes)")