Ben_Kosytorz/Warhammer_DLP_Helper
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor: flatten the crate to the repo root and drop the python version

Browse Source 
The Rust rewrite has replaced the original python implementation, so the transition layout is no longer needed:

- move version-rust/{Cargo.toml,Cargo.lock,README.md,.gitignore,src,data} to the repo root; the crate is now the repo.
- move docs/SORTING.md and data/gwshop_catalog.json to the root.
- remove version-python/ entirely, plus the dev-only version-rust/tools/dump_warhammer_data.py (its only job was regenerating data/warhammer_data.json from python).
- rewrite AGENTS.md and README.md for the single-project layout; strip all version-python/version-rust references from code comments.
- fix the gwshop credential lookup: algolia.json now sits at CARGO_MANIFEST_DIR (crate root == repo root), not its parent.
- restore the credentials/run-output ignores in .gitignore (algolia.json, .env, *.token, downloads/, state.json, manifest.jsonl, ...).
- docs/SORTING.md: retarget the extraction/sorter/dlp examples at the dlphelper binary instead of the deleted python scripts.

cargo test: 86/86 green.
This commit is contained in:
Ben_Kosytorz
2026-10-11 09:22:41 +02:00
parent 356b9089b7
commit 6c6a3273bb
54 changed files with 331 additions and 5378 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/zero807.rs
+519
View File
@@ -0,0 +1,519 @@
//! curl-based downloader for 0807.st direct links — Rust port of
//! `dlphelper/zero807.py`.
//!
//! 0807.st serves the files directly but rejects requests that don't look
//! like a browser (requires a User-Agent and the Referer of the site page),
//! and it rejects multi-connection/range abuse patterns differently than
//! curl — which is why curl is used instead of aria2c here (see python
//! module docstring: curl worked, aria2c's default request got 403).
//!
//! Downloads go to a `.part` file first and are renamed on success, so an
//! interrupted download is resumable (`--continue-at -`) and partial files
//! are never mistaken for finished ones.
use std::fs;
use std::io::Read;
use std::path::{Path, PathBuf};
use std::process::Command;
pub const USER_AGENT: &str = "Mozilla/5.0";
pub const REFERER: &str = "https://0807.st/";
/// Error type mirroring the python exception classes. `NotFound` (dead link,
/// HTTP 404/410) is a final state the orchestrator never retries.
#[allow(dead_code)] // used by the orchestrator, not yet wired up
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum Zero807Error {
/// python `Zero807Error` (base): transient/unclassified failure.
Error(String),
/// python `Zero807NotFound`: dead link, never retry.
NotFound(String),
}
impl std::fmt::Display for Zero807Error {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
Zero807Error::Error(m) | Zero807Error::NotFound(m) => write!(f, "{m}"),
}
}
}
impl std::error::Error for Zero807Error {}
/// Derive the filename from the URL path (e.g. /6saCNie.zip -> 6saCNie.zip).
///
/// Matches python `Path(urlparse(url).path).name`: the query string is not
/// part of the path, a trailing slash still yields the last real segment
/// (`.../a/b/` → `b`), and an empty result is an error.
pub fn target_name(url: &str) -> Result<String, Zero807Error> {
let path = match url::Url::parse(url) {
Ok(u) => u.path().to_string(),
// python urlparse is lenient about a missing scheme but still strips
// query/fragment; mirror that.
Err(_) => url.split(['?', '#']).next().unwrap_or(url).to_string(),
};
let name = Path::new(&path)
.file_name()
.map(|n| n.to_string_lossy().into_owned())
.unwrap_or_default();
if name.is_empty() {
return Err(Zero807Error::Error(format!(
"cannot derive a filename from URL: {url}"
)));
}
Ok(name)
}
/// Download one URL with curl; returns (final_path, size_bytes).
///
/// Defaults match python: `timeout_seconds = 7200`, `resume = true`.
/// Use `download_with` for explicit parameters.
pub fn download(url: &str, out_dir: &Path) -> Result<(PathBuf, u64), Zero807Error> {
download_with(url, out_dir, 7200, true)
}
/// Same with explicit timeout/resume parameters (python keyword args).
pub fn download_with(
url: &str,
out_dir: &Path,
timeout_seconds: u64,
resume: bool,
) -> Result<(PathBuf, u64), Zero807Error> {
let name = target_name(url)?;
let dest = out_dir.join(&name);
let size = download_to(url, &dest, timeout_seconds, resume)?;
Ok((dest, size))
}
/// Download `url` to an explicit `dest` (`.part` + rename, resume, skip if
/// already present). `download_with` derives `dest` from the URL; the folder
/// downloader derives it from the page's file name.
pub fn download_to(
url: &str,
dest: &Path,
timeout_seconds: u64,
resume: bool,
) -> Result<u64, Zero807Error> {
let name = dest
.file_name()
.map(|n| n.to_string_lossy().into_owned())
.unwrap_or_default();
if let Some(parent) = dest.parent() {
fs::create_dir_all(parent)
.map_err(|e| Zero807Error::Error(format!("cannot create {}: {e}", parent.display())))?;
}
let tmp = dest.with_file_name(format!("{name}.part"));
if dest.exists() {
let size = fs::metadata(dest).map(|m| m.len()).unwrap_or(0);
if size > 0 {
log::info!("Skip existing: {}", dest.display());
return Ok(size);
}
}
let mut cmd = Command::new("curl");
cmd.args([
"-fSL",
"--connect-timeout",
"20",
"--max-time",
&timeout_seconds.to_string(),
"--retry",
"2",
"--retry-delay",
"5",
"-A",
USER_AGENT,
"-e",
REFERER,
"-o",
]);
cmd.arg(&tmp);
if resume {
cmd.arg("--continue-at").arg("-");
}
cmd.arg(url);
log::info!("Downloading {} -> {}", url, name);
let proc = cmd.output().map_err(|e| {
// python: subprocess raises FileNotFoundError, which the
// orchestrator's catch-all would record; wrap it here.
Zero807Error::Error(format!("failed to run curl: {e}"))
})?;
if proc.status.code() != Some(0) {
let returncode = proc.status.code().unwrap_or(-1);
let stderr = String::from_utf8_lossy(&proc.stderr);
let tail: Vec<&str> = stderr
.trim()
.lines()
.filter(|line| !line.is_empty())
.collect();
let start = tail.len().saturating_sub(3);
let stderr = tail[start..].join(" | ");
// -f makes curl exit 22 on any HTTP error; dead links fail for good
if stderr.contains("404") || stderr.contains("410") {
return Err(Zero807Error::NotFound(format!(
"dead link (HTTP 404/410): {stderr}"
)));
}
return Err(Zero807Error::Error(format!(
"curl exit code {returncode}: {stderr}"
)));
}
let size = fs::metadata(&tmp).map(|m| m.len()).unwrap_or(0);
if size == 0 {
let _ = fs::remove_file(&tmp); // python unlink(missing_ok=True)
return Err(Zero807Error::Error(format!("empty file for {name}")));
}
if dest
.extension()
.map(|e| e.to_string_lossy().to_lowercase())
.as_deref()
== Some("zip")
{
// python reads the whole file just to peek at 2 bytes; reading only
// the first two bytes is behavior-identical and cheap.
let mut first = [0u8; 2];
let n = fs::File::open(&tmp)
.and_then(|mut fh| fh.read(&mut first))
.unwrap_or(0);
if first[..n] != *b"PK" {
log::warn!("{name} does not look like a ZIP file (first bytes are not PK)");
}
}
// os.replace semantics: rename over an existing destination.
fs::rename(&tmp, dest)
.map_err(|e| Zero807Error::Error(format!("cannot move {name} into place: {e}")))?;
Ok(size)
}
/// A file listed on a 0807.st folder page (`/f/<id>`).
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct FolderEntry {
/// file id used in the direct-download URL (`https://0807.st/<id>?dl=1`)
pub id: String,
/// display file name from the page (`<span class="alname">`)
pub name: String,
pub size: u64,
pub mime: String,
}
/// The `<id>` of a 0807.st folder page URL (`/f/<id>`), if it is one.
/// A direct file link (`/x.zip`) or the folder export (`/f/<id>.zip`) is not.
pub fn folder_page_id(url: &str) -> Option<String> {
let path = match url::Url::parse(url) {
Ok(u) => u.path().to_string(),
Err(_) => url.split(['?', '#']).next().unwrap_or(url).to_string(),
};
let rest = path.trim_end_matches('/').strip_prefix("/f/")?;
if rest.is_empty() || rest.contains('/') || rest.contains('.') {
return None;
}
Some(rest.to_string())
}
/// True when `url` points at a 0807.st folder page.
pub fn is_folder_page(url: &str) -> bool {
folder_page_id(url).is_some()
}
/// Parse the file list from a 0807.st folder page. Each entry is an
/// `<a class="alitem" ... data-id="..." data-size="..." data-mime="...">`
/// whose name sits in `<span class="alname">...</span>`. Pure string scanning
/// (no HTML-parser dependency).
pub fn parse_folder_page(html: &str) -> Vec<FolderEntry> {
const MARK: &str = "<a class=\"alitem\"";
let mut out = Vec::new();
let mut rest = html;
while let Some(pos) = rest.find(MARK) {
rest = &rest[pos + MARK.len()..];
let end = rest.find("</a>").unwrap_or(rest.len());
let seg = &rest[..end];
if let Some(id) = attr(seg, "data-id") {
let size = attr(seg, "data-size")
.and_then(|s| s.parse::<u64>().ok())
.unwrap_or(0);
let mime = attr(seg, "data-mime").unwrap_or_default();
let name = span_text(seg, "alname").unwrap_or_else(|| id.clone());
out.push(FolderEntry {
id,
name,
size,
mime,
});
}
rest = &rest[end..];
}
out
}
/// Value of `key="..."` inside a tag fragment.
fn attr(seg: &str, key: &str) -> Option<String> {
let needle = format!("{key}=\"");
let start = seg.find(&needle)? + needle.len();
let end = seg[start..].find('"')? + start;
Some(seg[start..end].to_string())
}
/// Text of `<span class="class">...</span>` (HTML entities decoded).
fn span_text(seg: &str, class: &str) -> Option<String> {
let needle = format!("<span class=\"{class}\">");
let start = seg.find(&needle)? + needle.len();
let end = seg[start..].find("</span>")? + start;
Some(html_unescape(&seg[start..end]))
}
/// Decode the handful of HTML entities that can appear in file names.
fn html_unescape(s: &str) -> String {
s.replace("&amp;", "&")
.replace("&lt;", "<")
.replace("&gt;", ">")
.replace("&quot;", "\"")
.replace("&#39;", "'")
.replace("&#x27;", "'")
}
/// A safe single path segment for a file name taken from a page.
fn sanitize_filename(name: &str) -> String {
let cleaned: String = name
.chars()
.map(|c| {
if c == '/' || c == '\\' || (c as u32) < 0x20 {
'_'
} else {
c
}
})
.collect();
let cleaned = cleaned.trim().trim_start_matches('.').to_string();
if cleaned.is_empty() {
"unnamed".into()
} else {
cleaned
}
}
/// Fetch a folder page's HTML via curl (browser-like headers, like the file
/// downloads). Writes to a temp file because curl is used throughout.
fn fetch_folder_html(url: &str, timeout_seconds: u64) -> Result<String, Zero807Error> {
let tmp =
std::env::temp_dir().join(format!("dlphelper-0807-folder-{}.html", std::process::id()));
let mut cmd = Command::new("curl");
cmd.args([
"-fSL",
"--connect-timeout",
"20",
"--max-time",
&timeout_seconds.to_string(),
"-A",
USER_AGENT,
"-e",
REFERER,
"-o",
]);
cmd.arg(&tmp);
cmd.arg(url);
let proc = cmd
.output()
.map_err(|e| Zero807Error::Error(format!("failed to run curl: {e}")))?;
if proc.status.code() != Some(0) {
let stderr = String::from_utf8_lossy(&proc.stderr);
let tail: Vec<&str> = stderr.trim().lines().filter(|l| !l.is_empty()).collect();
let start = tail.len().saturating_sub(3);
let msg = tail[start..].join(" | ");
let _ = fs::remove_file(&tmp);
if msg.contains("404") || msg.contains("410") {
return Err(Zero807Error::NotFound(format!(
"dead folder page (HTTP 404/410): {msg}"
)));
}
return Err(Zero807Error::Error(format!("curl exit for page: {msg}")));
}
let text = fs::read_to_string(&tmp)
.map_err(|e| Zero807Error::Error(format!("reading folder page: {e}")))?;
let _ = fs::remove_file(&tmp);
Ok(text)
}
/// Download every file on a 0807.st folder page into `out_dir/<page-id>/`.
///
/// Returns `(folder, [(path, size), ...])`. Each file is fetched from
/// `https://0807.st/<data-id>?dl=1` (a 302 to the CDN). Already-present files
/// are skipped, so the caller's retry loop only fetches what is missing; a
/// partial failure is reported as an error so that retry happens.
pub fn download_folder(
url: &str,
out_dir: &Path,
timeout_seconds: u64,
resume: bool,
) -> Result<(PathBuf, Vec<(PathBuf, u64)>), Zero807Error> {
let id = folder_page_id(url)
.ok_or_else(|| Zero807Error::Error(format!("not a folder page: {url}")))?;
let html = fetch_folder_html(url, timeout_seconds)?;
let entries = parse_folder_page(&html);
if entries.is_empty() {
return Err(Zero807Error::Error(format!(
"folder page {url} lists no downloadable files"
)));
}
let folder = out_dir.join(&id);
fs::create_dir_all(&folder)
.map_err(|e| Zero807Error::Error(format!("cannot create {}: {e}", folder.display())))?;
log::info!("Folder page {url}: {} files", entries.len());
let mut files: Vec<(PathBuf, u64)> = Vec::new();
let mut failed: Vec<String> = Vec::new();
for e in &entries {
let name = sanitize_filename(&e.name);
let dest = folder.join(&name);
let direct = format!("https://0807.st/{}?dl=1", e.id);
match download_to(&direct, &dest, timeout_seconds, resume) {
Ok(size) => {
log::info!("OK folder file {name} ({})", human_size(size));
files.push((dest, size));
}
Err(err) => {
log::error!("Folder file {name} failed: {err}");
failed.push(format!("{name}: {err}"));
}
}
}
if !failed.is_empty() {
return Err(Zero807Error::Error(format!(
"{}/{} folder files failed: {}",
failed.len(),
entries.len(),
failed.join("; ")
)));
}
Ok((folder, files))
}
/// Compact human-readable size (kept local so this module stays self-contained).
fn human_size(bytes: u64) -> String {
const UNITS: [&str; 5] = ["B", "KiB", "MiB", "GiB", "TiB"];
let mut v = bytes as f64;
let mut u = 0;
while v >= 1024.0 && u < UNITS.len() - 1 {
v /= 1024.0;
u += 1;
}
if u == 0 {
format!("{bytes} B")
} else {
format!("{v:.1} {}", UNITS[u])
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn target_name_from_path_last_segment() {
assert_eq!(
target_name("https://0807.st/6saCNie.zip").unwrap(),
"6saCNie.zip"
);
assert_eq!(
target_name("https://0807.st/a/b/filename.bin").unwrap(),
"filename.bin"
);
}
#[test]
fn target_name_drops_query_and_fragment() {
// python takes Path(urlparse(url).path).name: query/fragment dropped
assert_eq!(
target_name("https://0807.st/file.zip?x=1").unwrap(),
"file.zip"
);
assert_eq!(
target_name("https://0807.st/file.zip?x=1#frag").unwrap(),
"file.zip"
);
// schemeless URLs parse like python urlparse (query still stripped)
assert_eq!(target_name("0807.st/file.zip?x=1").unwrap(), "file.zip");
}
#[test]
fn target_name_trailing_slash_matches_pathlib() {
// pathlib: Path("/a/b/").name == "b" (trailing slash ignored)
assert_eq!(target_name("https://0807.st/a/b/").unwrap(), "b");
assert_eq!(
target_name("https://0807.st/").unwrap_err().to_string(),
"cannot derive a filename from URL: https://0807.st/"
);
assert_eq!(
target_name("https://0807.st").unwrap_err().to_string(),
"cannot derive a filename from URL: https://0807.st"
);
}
#[test]
fn error_display_is_the_message() {
assert_eq!(
Zero807Error::NotFound("dead link (HTTP 404/410): 404".into()).to_string(),
"dead link (HTTP 404/410): 404"
);
assert_eq!(Zero807Error::Error("boom".into()).to_string(), "boom");
}
#[test]
fn folder_page_detection() {
assert_eq!(
folder_page_id("https://0807.st/f/Df8A5sn3").as_deref(),
Some("Df8A5sn3")
);
assert_eq!(
folder_page_id("https://0807.st/f/MM5AWQ9R/").as_deref(),
Some("MM5AWQ9R")
);
assert!(is_folder_page("https://0807.st/f/Df8A5sn3"));
// a direct file link is not a folder page
assert!(!is_folder_page("https://0807.st/6saCNie.zip"));
// the folder export (.zip) is not a folder page either
assert!(!is_folder_page("https://0807.st/f/Df8A5sn3.zip"));
// deeper paths are not folder pages
assert!(!is_folder_page("https://0807.st/f/a/b"));
assert!(!is_folder_page("https://0807.st/"));
}
#[test]
fn parse_folder_page_reads_alitem_entries() {
let html = r#"<div class="allist">
<a class="alitem" href="/p/9z5SsgV" data-id="9z5SsgV" data-rel="" data-size="1397446559" data-mime="application/zip"><span class="althumbwrap"><span class="alicon"><svg></svg></span></span><span class="alinfo"><span class="alname">13. Terrain.zip</span><span class="alsize">1.3 GB</span></span></a>
<a class="alitem" href="/p/wNuEQcp" data-id="wNuEQcp" data-rel="" data-size="2295545985" data-mime="application/zip"><span class="alname">Orks &amp; Goblins.zip</span></a>
</div>"#;
let got = parse_folder_page(html);
assert_eq!(got.len(), 2);
assert_eq!(got[0].id, "9z5SsgV");
assert_eq!(got[0].name, "13. Terrain.zip");
assert_eq!(got[0].size, 1397446559);
assert_eq!(got[0].mime, "application/zip");
assert_eq!(got[1].id, "wNuEQcp");
assert_eq!(got[1].name, "Orks & Goblins.zip"); // entity decoded
assert_eq!(got[1].size, 2295545985);
}
#[test]
fn parse_folder_page_ignores_non_alitem_anchors() {
let html = r#"<a href="/" class="x">home</a><a class="alitem" href="/p/abc" data-id="abc" data-size="5" data-mime="text/plain"><span class="alname">a.txt</span></a>"#;
let got = parse_folder_page(html);
assert_eq!(got.len(), 1);
assert_eq!(got[0].id, "abc");
assert_eq!(got[0].name, "a.txt");
}
#[test]
fn sanitize_filename_blocks_traversal() {
assert_eq!(sanitize_filename("13. Terrain.zip"), "13. Terrain.zip");
let s = sanitize_filename("../../etc/passwd");
assert!(!s.contains('/') && !s.contains('\\'));
assert!(!s.starts_with('.'));
assert_eq!(sanitize_filename("a\\b"), "a_b");
assert_eq!(sanitize_filename(" "), "unnamed");
}
}