From 137efa7e6c3e2a9b733b054c59bca3dbe9792f96 Mon Sep 17 00:00:00 2001 From: Ben_Kosytorz Date: Fri, 9 Oct 2026 14:17:02 +0200 Subject: [PATCH] feat(rust): port discord, gofile, zero807, extract_all and catalogue MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Parallel module ports (behavior parity with the python versions): - src/discord.rs: DCE docker export (token via env only), base64/plain URL extraction with grab.sh-compatible regexes, host classification; 11 tests - src/gofile.rs: GoFile API client — one guest account per run, linear backoff on account creation, sha256 website token, recursive content walk, streaming .part downloads; 7 tests (offline goldens from python hashlib) - src/zero807.rs: curl downloader with UA+Referer contract, dead-link detection; 4 tests - src/extract.rs: recursive extraction (sevenz-rust2 for 7z, zip crate, unrar for rar, system tar/gzip/bzip2/xz for tarballs), nested BFS expansion, .broken quarantine, extract_manifest.jsonl, idempotent re-runs; 8 tests - src/catalogue.rs: keyword classifier over the dumped taxonomy, manifest/ directory scanning, overrides, md+jsonl reports, audited moves; 7 tests incl. all 17 python test cases --- version-rust/Cargo.lock | 20 + version-rust/Cargo.toml | 1 + version-rust/src/catalogue.rs | 1064 +++++++++++++++++++++++++++++ version-rust/src/discord.rs | 514 ++++++++++++++ version-rust/src/extract.rs | 1195 ++++++++++++++++++++++++++++++++- version-rust/src/gofile.rs | 673 +++++++++++++++++++ version-rust/src/zero807.rs | 225 +++++++ 7 files changed, 3687 insertions(+), 5 deletions(-) diff --git a/version-rust/Cargo.lock b/version-rust/Cargo.lock index e16eb1c..4a0083f 100644 --- a/version-rust/Cargo.lock +++ b/version-rust/Cargo.lock @@ -592,6 +592,7 @@ dependencies = [ "clap", "dirs", "env_logger", + "fs4", "hex", "indexmap", "log", @@ -715,6 +716,16 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "fs4" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8640e34b88f7652208ce9e88b1a37a2ae95227d84abec377ccd3c5cfeb141ed4" +dependencies = [ + "rustix", + "windows-sys 0.59.0", +] + [[package]] name = "futures-channel" version = "0.3.34" @@ -2445,6 +2456,15 @@ dependencies = [ "windows-targets", ] +[[package]] +name = "windows-sys" +version = "0.59.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" +dependencies = [ + "windows-targets", +] + [[package]] name = "windows-sys" version = "0.61.2" diff --git a/version-rust/Cargo.toml b/version-rust/Cargo.toml index 7141725..0b83c38 100644 --- a/version-rust/Cargo.toml +++ b/version-rust/Cargo.toml @@ -11,6 +11,7 @@ chrono = "0.4" clap = { version = "4", features = ["derive"] } dirs = "6" env_logger = "0.11" +fs4 = "0.13" hex = "0.4" indexmap = { version = "2", features = ["serde"] } log = "0.4" diff --git a/version-rust/src/catalogue.rs b/version-rust/src/catalogue.rs index 12cda7d..ca0619b 100644 --- a/version-rust/src/catalogue.rs +++ b/version-rust/src/catalogue.rs @@ -1,2 +1,1066 @@ //! Catalogue: classify collected files into game system / faction / class — //! Rust port of `version-python/dlphelper/catalogue.py`. +//! +//! The catalogue is the bridge between "a pile of downloads" and the later +//! library stage: it reads manifest.jsonl (with full source attribution) and/or +//! scans arbitrary directories for archive files, classifies each file against +//! the taxonomy in `warhammer_data`, and writes machine- (catalogue.jsonl) and +//! human-readable (catalogue.md) reports. +//! +//! Classification is keyword-based and deliberately conservative: files that +//! cannot be classified land in the "unclassified" bucket so gaps become +//! visible instead of producing wrong assignments. A manual overrides file can +//! pin individual files. +//! +//! Nothing is moved by default — use `apply_moves` for that; it moves files +//! into `/////` and records every +//! move in a jsonl audit log so it can be audited/undone. + +use std::collections::{BTreeMap, HashMap, HashSet}; +use std::fs; +use std::io::Write as _; +use std::path::{Path, PathBuf}; +use std::sync::OnceLock; + +use anyhow::Context as _; +use log::warn; +use regex::Regex; +use serde_json::Value; + +use crate::warhammer_data::Data; + +pub const UNKNOWN: &str = "Unknown"; +pub const UNCLASSIFIED_FACTION: &str = "Unclassified"; +pub const UNCLASSIFIED_CLASS: &str = "Unclassified"; + +/// Archive suffixes considered by `scan_directory` (python set literal). +pub const ARCHIVE_SUFFIXES: &[&str] = &[".zip", ".7z", ".rar", ".tar", ".gz"]; + +/// One classified file. +#[derive(Debug, Clone)] +pub struct CatalogueEntry { + pub path: String, + pub name: String, + pub size: u64, + pub system: String, + pub faction: String, + pub klass: String, + /// Manifest attribution if known. + pub source: SourceAttribution, +} + +/// Manifest attribution fields. Values keep whatever JSON provided raw +/// (message_id may be numeric), so each is a `serde_json::Value`; `None` +/// mirrors python's missing key / JSON null, and falsy values are dropped +/// when the jsonl report is written (python `{k: v ... if v}`). +#[derive(Debug, Clone, Default)] +pub struct SourceAttribution { + pub channel: Option, + pub url: Option, + pub message_id: Option, + pub timestamp: Option, + pub author: Option, +} + +// --------------------------------------------------------------------------- +// Python pathlib semantics for string paths (posix flavour) +// --------------------------------------------------------------------------- + +/// Python `Path(p).name`: trailing slashes are dropped, the last component is +/// returned; `""` and `"/"` have no name. +fn py_basename(p: &str) -> &str { + let s = p.trim_end_matches('/'); + match s.rfind('/') { + Some(i) => &s[i + 1..], + None => s, + } +} + +/// Python `Path(p).stem`: the name minus its last dot-suffix, but a leading +/// dot ("hidden file") or a trailing dot do not count as a suffix +/// (pathlib requires `0 < i < len(name) - 1` for the split dot). +fn py_stem(p: &str) -> &str { + let name = py_basename(p); + match name.rfind('.') { + Some(i) if i > 0 && i < name.len() - 1 => &name[..i], + _ => name, + } +} + +/// Python `Path(p).suffix` under the same rules as `py_stem`. +fn py_suffix(p: &str) -> &str { + let name = py_basename(p); + match name.rfind('.') { + Some(i) if i > 0 && i < name.len() - 1 => &name[i..], + _ => "", + } +} + +/// Python `_clean_name`: stem of the filename, runs of `_.+[]()` replaced by a +/// single space, whitespace collapsed, trimmed and lowercased. +pub fn clean_name(name: &str) -> String { + let stem = py_stem(name); + let spaced = punct_run().replace_all(stem, " "); + let collapsed = ws_run().replace_all(&spaced, " "); + collapsed.trim().to_lowercase() +} + +fn punct_run() -> &'static Regex { + static RE: OnceLock = OnceLock::new(); + RE.get_or_init(|| Regex::new(r"[_\.+\[\]()]+").expect("static regex")) +} + +fn ws_run() -> &'static Regex { + static RE: OnceLock = OnceLock::new(); + RE.get_or_init(|| Regex::new(r"\s+").expect("static regex")) +} + +// --------------------------------------------------------------------------- +// Keyword matching +// --------------------------------------------------------------------------- + +/// Word-boundary regex for a keyword, tolerating a plural 's' — the exact +/// pattern the python version builds via `_kw_regex`. +/// +/// The `regex` crate does not support look-around, so this string is kept for +/// parity/documentation only; `kw_match` implements the identical semantics +/// manually: an occurrence of `keyword`, optionally followed by an 's', where +/// the surrounding characters are not in `[a-z0-9]`. +#[allow(dead_code)] // parity helper: regex crate has no look-around to compile it +fn kw_regex(keyword: &str) -> String { + format!("(? String { + let mut out = String::with_capacity(s.len() + 8); + for c in s.chars() { + if c.is_ascii_alphanumeric() || c == '_' || !c.is_ascii() { + out.push(c); + } else { + out.push('\\'); + out.push(c); + } + } + out +} + +/// The boundary characters python's look-around rejects: lowercase a-z and 0-9. +fn is_kw_boundary(c: char) -> bool { + matches!(c, 'a'..='z' | '0'..='9') +} + +/// Exact emulation of `re.search(kw_regex(keyword), clean)`. +fn kw_match(clean: &str, keyword: &str) -> bool { + let mut from = 0; + while let Some(rel) = clean[from..].find(keyword) { + let start = from + rel; + // advance by one character so overlapping occurrences are still tried + from = start + + clean[start..] + .chars() + .next() + .expect("non-empty slice") + .len_utf8(); + + // (? 0 + && clean[..start] + .chars() + .next_back() + .map(is_kw_boundary) + .unwrap_or(false) + { + continue; + } + // s? + let mut end = start + keyword.len(); + if clean[end..].starts_with('s') { + end += 's'.len_utf8(); + } + // (?![a-z0-9]) + if end == clean.len() + || !clean[end..] + .chars() + .next() + .map(is_kw_boundary) + .unwrap_or(false) + { + return true; + } + } + false +} + +/// Return `(system, faction)` with the most specific keyword hit. +/// +/// Most specific = longest matching keyword; ties go to the first keyword in +/// iteration order (python `if best is None or len(keyword) > best[0]`). +pub fn match_faction(clean: &str) -> Option<(String, String)> { + let data = Data::global(); + let mut best: Option<(usize, usize, usize)> = None; // (kw len, system idx, alias idx) + for (si, (_, aliases)) in data.faction_aliases.iter().enumerate() { + for (ai, (keyword, _)) in aliases.iter().enumerate() { + if kw_match(clean, keyword) { + let klen = keyword.chars().count(); + if best.is_none() || klen > best.expect("checked").0 { + best = Some((klen, si, ai)); + } + } + } + } + best.map(|(_, si, ai)| { + let (system, aliases) = &data.faction_aliases[si]; + let (_, faction) = &aliases[ai]; + (system.clone(), faction.clone()) + }) +} + +/// Return the most specific force-organization class keyword hit. +pub fn match_class(clean: &str) -> Option { + let data = Data::global(); + let mut best: Option<(usize, usize)> = None; // (kw len, idx) + for (i, (keyword, _)) in data.unit_classes.iter().enumerate() { + if kw_match(clean, keyword) { + let klen = keyword.chars().count(); + if best.is_none() || klen > best.expect("checked").0 { + best = Some((klen, i)); + } + } + } + best.map(|(_, i)| data.unit_classes[i].1.clone()) +} + +/// Classify a file name -> `(system, faction, class)`. +pub fn classify(name: &str) -> (String, String, String) { + let clean = clean_name(name); + let faction_hit = match_faction(&clean); + let klass = match_class(&clean); + + if let Some((system, faction)) = faction_hit { + // Non-40k systems: keep their class if we found one, else Unclassified + if system != "Warhammer 40,000" && klass.is_none() { + return (system, faction, UNCLASSIFIED_CLASS.to_string()); + } + return ( + system, + faction, + klass.unwrap_or_else(|| UNCLASSIFIED_CLASS.to_string()), + ); + } + + // No faction match: a class keyword alone cannot attribute a system + ( + UNKNOWN.to_string(), + UNCLASSIFIED_FACTION.to_string(), + UNCLASSIFIED_CLASS.to_string(), + ) +} + +// --------------------------------------------------------------------------- +// Scanning +// --------------------------------------------------------------------------- + +/// Entries from a manifest.jsonl (preferred: has source attribution). +pub fn scan_manifest(manifest_path: &Path) -> Vec { + let mut entries = Vec::new(); + if !manifest_path.exists() { + return entries; + } + let text = match fs::read_to_string(manifest_path) { + Ok(text) => text, + Err(err) => { + log::error!("Cannot read manifest {}: {}", manifest_path.display(), err); + return entries; + } + }; + let mut seen: HashSet = HashSet::new(); + for raw_line in text.lines() { + let line = raw_line.trim(); + if line.is_empty() { + continue; + } + let row: Value = match serde_json::from_str(line) { + Ok(row) => row, + Err(_) => { + warn!( + "Skipping malformed manifest line: {}", + line.chars().take(80).collect::() + ); + continue; + } + }; + let path = match row.get("path") { + Some(Value::String(s)) if !s.is_empty() => s.clone(), + _ => String::new(), + }; + if seen.contains(&path) { + continue; + } + seen.insert(path.clone()); + let name = match row.get("name") { + Some(Value::String(s)) if !s.is_empty() => s.clone(), + _ => py_basename(&path).to_string(), + }; + let size = value_u64(row.get("size")); + entries.push(CatalogueEntry { + path, + name, + size, + system: UNKNOWN.to_string(), + faction: UNCLASSIFIED_FACTION.to_string(), + klass: UNCLASSIFIED_CLASS.to_string(), + source: SourceAttribution { + channel: grab(&row, "channel"), + url: grab(&row, "url"), + message_id: grab(&row, "message_id"), + timestamp: grab(&row, "timestamp"), + author: grab(&row, "author"), + }, + }); + } + entries +} + +/// Raw attribution value: JSON null counts as missing, everything else is +/// kept verbatim (python stores `row.get(key)`). +fn grab(row: &Value, key: &str) -> Option { + row.get(key).cloned().filter(|v| !v.is_null()) +} + +/// Python `int(row.get("size") or 0)`: numbers truncate, numeric strings +/// parse, anything else (missing/null) is 0. +fn value_u64(value: Option<&Value>) -> u64 { + match value { + Some(Value::Number(n)) => n + .as_u64() + .or_else(|| n.as_f64().map(|f| f as u64)) + .unwrap_or(0), + Some(Value::String(s)) => s.trim().parse().unwrap_or(0), + Some(Value::Bool(true)) => 1, + _ => 0, + } +} + +fn collect_paths(dir: &Path, out: &mut Vec) { + let read = match fs::read_dir(dir) { + Ok(read) => read, + Err(err) => { + warn!("Cannot list {}: {}", dir.display(), err); + return; + } + }; + for entry in read.flatten() { + let path = entry.path(); + if entry.file_type().map(|t| t.is_dir()).unwrap_or(false) { + collect_paths(&path, out); + } + out.push(path); + } +} + +/// Entries from walking a directory tree for archive files (manual era). +pub fn scan_directory(dir: &Path) -> Vec { + let mut paths: Vec = Vec::new(); + collect_paths(dir, &mut paths); + paths.sort(); + + let mut entries = Vec::new(); + for path in paths { + if !path.is_file() { + continue; + } + let as_str = path.to_string_lossy(); + let suffix = py_suffix(&as_str); + if !ARCHIVE_SUFFIXES + .iter() + .any(|s| suffix.eq_ignore_ascii_case(s)) + { + continue; + } + let size = fs::metadata(&path).map(|m| m.len()).unwrap_or(0); + entries.push(CatalogueEntry { + path: as_str.to_string(), + name: py_basename(&as_str).to_string(), + size, + system: UNKNOWN.to_string(), + faction: UNCLASSIFIED_FACTION.to_string(), + klass: UNCLASSIFIED_CLASS.to_string(), + source: SourceAttribution::default(), + }); + } + entries +} + +// --------------------------------------------------------------------------- +// Overrides +// --------------------------------------------------------------------------- + +/// overrides.json: `{"/path/or/filename": {"faction": ..., "class": ..., "system": ...}}` +pub fn load_overrides(path: Option<&Path>) -> anyhow::Result> { + let Some(path) = path else { + return Ok(BTreeMap::new()); + }; + if !path.exists() { + return Ok(BTreeMap::new()); + } + let text = fs::read_to_string(path) + .with_context(|| format!("reading overrides {}", path.display()))?; + serde_json::from_str(&text).with_context(|| format!("parsing overrides {}", path.display())) +} + +/// Coerce an override value into the entry's string fields: JSON strings are +/// used verbatim, everything else is rendered as compact JSON (python would +/// assign the raw parsed value). +fn value_string(v: &Value) -> String { + match v { + Value::String(s) => s.clone(), + other => other.to_string(), + } +} + +/// Apply `{"system"/"faction"/"class": ...}` fixes keyed by (lowercased) file +/// name; returns the number of entries fixed. +pub fn apply_overrides( + entries: &mut [CatalogueEntry], + overrides: &BTreeMap, +) -> usize { + let mut count = 0; + let mut by_name: HashMap = HashMap::new(); + for (i, e) in entries.iter().enumerate() { + by_name.insert(e.name.to_lowercase(), i); + } + for (key, fix) in overrides { + let Some(&idx) = by_name.get(&key.to_lowercase()) else { + warn!("Override target not found: {}", key); + continue; + }; + let entry = &mut entries[idx]; + if let Some(v) = fix.get("system") { + entry.system = value_string(v); + } + if let Some(v) = fix.get("faction") { + entry.faction = value_string(v); + } + if let Some(v) = fix.get("class") { + entry.klass = value_string(v); + } + count += 1; + } + count +} + +// --------------------------------------------------------------------------- +// Reports +// --------------------------------------------------------------------------- + +/// Python truthiness for JSON values (the `if v` filter on source fields). +fn truthy(v: &Value) -> bool { + match v { + Value::Null | Value::Bool(false) => false, + Value::Number(n) => n.as_f64() != Some(0.0), + Value::String(s) => !s.is_empty(), + Value::Array(a) => !a.is_empty(), + Value::Object(o) => !o.is_empty(), + Value::Bool(true) => true, + } +} + +fn json_str(s: &str) -> String { + serde_json::to_string(s).expect("string serialization is infallible") +} + +fn json_value(v: &Value) -> String { + serde_json::to_string(v).expect("Value serialization is infallible") +} + +/// Source object for the jsonl report: python key order +/// (channel, url, message_id, timestamp, author), falsy values dropped. +fn source_json(e: &CatalogueEntry) -> String { + let fields = [ + ("channel", &e.source.channel), + ("url", &e.source.url), + ("message_id", &e.source.message_id), + ("timestamp", &e.source.timestamp), + ("author", &e.source.author), + ]; + let mut parts: Vec = Vec::new(); + for (key, value) in fields { + let Some(v) = value else { continue }; + if !truthy(v) { + continue; + } + parts.push(format!("{}:{}", json_str(key), json_value(v))); + } + format!("{{{}}}", parts.join(",")) +} + +/// One jsonl report line: `{path, name, size, system, faction, class, source}`. +fn jsonl_line(e: &CatalogueEntry) -> String { + format!( + "{{\"path\":{},\"name\":{},\"size\":{},\"system\":{},\"faction\":{},\"class\":{},\"source\":{}}}", + json_str(&e.path), + json_str(&e.name), + e.size, + json_str(&e.system), + json_str(&e.faction), + json_str(&e.klass), + source_json(e), + ) +} + +/// "1.0 B", "1.5 KiB", "1.0 GiB", ... — python's divide-by-1024 loop that +/// stops at TiB, formatted with one decimal. +pub fn format_size(n: u64) -> String { + let mut size = n as f64; + for unit in ["B", "KiB", "MiB", "GiB", "TiB"] { + if size < 1024.0 || unit == "TiB" { + return format!("{:.1} {}", size, unit); + } + size /= 1024.0; + } + unreachable!("loop always returns at TiB") +} + +/// Python `f" — from {url}"` when the source url is truthy. +fn source_from_url(e: &CatalogueEntry) -> String { + match &e.source.url { + Some(v) if truthy(v) => format!(" — from {}", value_display(v)), + _ => String::new(), + } +} + +fn value_display(v: &Value) -> String { + match v { + Value::String(s) => s.clone(), + other => other.to_string(), + } +} + +/// Write the machine (catalogue.jsonl) and human (catalogue.md) reports. +pub fn write_reports( + entries: &[CatalogueEntry], + out_md: &Path, + out_jsonl: &Path, +) -> anyhow::Result<()> { + // machine report + let mut fh = + fs::File::create(out_jsonl).with_context(|| format!("creating {}", out_jsonl.display()))?; + for e in entries { + writeln!(fh, "{}", jsonl_line(e)) + .with_context(|| format!("writing {}", out_jsonl.display()))?; + } + + // human report + let mut grouped: BTreeMap<&str, BTreeMap<&str, BTreeMap<&str, Vec<&CatalogueEntry>>>> = + BTreeMap::new(); + for e in entries { + grouped + .entry(&e.system) + .or_default() + .entry(&e.faction) + .or_default() + .entry(&e.klass) + .or_default() + .push(e); + } + + let mut lines: Vec = vec![ + "# Warhammer Collection Catalogue".to_string(), + String::new(), + ]; + let total_size: u64 = entries.iter().map(|e| e.size).sum(); + let classified = entries + .iter() + .filter(|e| e.faction != UNCLASSIFIED_FACTION) + .count(); + lines.push(format!( + "{} files, {} total, {} classified, {} unclassified", + entries.len(), + format_size(total_size), + classified, + entries.len() - classified, + )); + lines.push(String::new()); + + for (system, factions) in &grouped { + let sys_files: Vec<&CatalogueEntry> = factions + .values() + .flat_map(|classes| classes.values().flatten().copied()) + .collect(); + let sys_size: u64 = sys_files.iter().map(|e| e.size).sum(); + lines.push(format!( + "## {} ({} files, {})", + system, + sys_files.len(), + format_size(sys_size) + )); + for (faction, classes) in factions { + let fac_size: u64 = classes.values().flatten().map(|e| e.size).sum(); + lines.push(format!("### {} ({})", faction, format_size(fac_size))); + for (klass, files) in classes { + let files_size: u64 = files.iter().map(|e| e.size).sum(); + lines.push(format!( + "- **{}** ({} files, {})", + klass, + files.len(), + format_size(files_size) + )); + let mut sorted_files = files.clone(); + sorted_files.sort_by(|a, b| a.name.to_lowercase().cmp(&b.name.to_lowercase())); + for e in sorted_files { + lines.push(format!( + " - {} ({}){}", + e.name, + format_size(e.size), + source_from_url(e) + )); + } + } + lines.push(String::new()); + } + } + + fs::write(out_md, lines.join("\n")).with_context(|| format!("writing {}", out_md.display()))?; + Ok(()) +} + +// --------------------------------------------------------------------------- +// Moves +// --------------------------------------------------------------------------- + +/// shutil.move: rename first, copy+remove as cross-device fallback. +fn move_file(src: &Path, dest: &Path) -> anyhow::Result<()> { + match fs::rename(src, dest) { + Ok(()) => Ok(()), + Err(_) => { + fs::copy(src, dest) + .with_context(|| format!("copy {} -> {}", src.display(), dest.display()))?; + fs::remove_file(src).with_context(|| format!("remove original {}", src.display()))?; + Ok(()) + } + } +} + +/// `{"from": str(src), "to": str(dest)}` audit line (compact separators). +fn audit_line(src: &Path, dest: &Path) -> String { + format!( + "{{\"from\":{},\"to\":{}}}", + json_str(&src.display().to_string()), + json_str(&dest.display().to_string()), + ) +} + +/// Move classified files into the library structure. Audited in moves_log. +pub fn apply_moves( + entries: &mut [CatalogueEntry], + library_root: &Path, + moves_log: &Path, +) -> anyhow::Result { + let parent = moves_log.parent().filter(|p| !p.as_os_str().is_empty()); + fs::create_dir_all(parent.unwrap_or_else(|| Path::new("."))) + .with_context(|| format!("creating {}", moves_log.display()))?; + let mut logfh = fs::OpenOptions::new() + .create(true) + .append(true) + .open(moves_log) + .with_context(|| format!("opening {}", moves_log.display()))?; + + let mut moved = 0; + for e in entries { + if e.faction == UNCLASSIFIED_FACTION { + continue; // never move what we cannot classify + } + if e.system == UNKNOWN { + continue; + } + let src = PathBuf::from(&e.path); + if !src.exists() { + warn!("Cannot move, missing: {}", src.display()); + continue; + } + let group = Data::global() + .group_of_faction + .get(&e.faction) + .cloned() + .unwrap_or_else(|| "Unclassified".to_string()); + let dest_dir = library_root + .join(&e.system) + .join(group) + .join(&e.faction) + .join(&e.klass); + let dest = dest_dir.join(py_basename(&e.path)); + if dest.exists() { + warn!("Skip, destination exists: {}", dest.display()); + continue; + } + fs::create_dir_all(&dest_dir) + .with_context(|| format!("creating {}", dest_dir.display()))?; + move_file(&src, &dest)?; + writeln!(logfh, "{}", audit_line(&src, &dest)) + .with_context(|| format!("writing {}", moves_log.display()))?; + e.path = dest.display().to_string(); + moved += 1; + } + Ok(moved) +} + +// --------------------------------------------------------------------------- +// Top-level builder +// --------------------------------------------------------------------------- + +/// Build the full catalogue: manifest entries first, then directory scans, +/// deduped by path, classified, then manual overrides applied. +pub fn build_catalogue( + manifests: &[PathBuf], + scan_dirs: &[PathBuf], + overrides_path: Option<&Path>, +) -> anyhow::Result> { + let mut entries: Vec = Vec::new(); + let mut seen: HashSet = HashSet::new(); + for m in manifests { + for e in scan_manifest(m) { + if seen.insert(e.path.clone()) { + entries.push(e); + } + } + } + for d in scan_dirs { + for e in scan_directory(d) { + if seen.insert(e.path.clone()) { + entries.push(e); + } + } + } + + for e in &mut entries { + let (system, faction, klass) = classify(&e.name); + e.system = system; + e.faction = faction; + e.klass = klass; + } + + apply_overrides(&mut entries, &load_overrides(overrides_path)?); + Ok(entries) +} + +// --------------------------------------------------------------------------- +// Tests (offline port of version-python/tests/test_catalogue.py) +// --------------------------------------------------------------------------- + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + fn entry(path: &str, name: &str, size: u64) -> CatalogueEntry { + CatalogueEntry { + path: path.to_string(), + name: name.to_string(), + size, + system: UNKNOWN.to_string(), + faction: UNCLASSIFIED_FACTION.to_string(), + klass: UNCLASSIFIED_CLASS.to_string(), + source: SourceAttribution::default(), + } + } + + /// All 17 CASES from tests/test_catalogue.py; `None` = "don't care". + #[test] + fn classify_matches_python_test_catalogue() { + let cases: &[(&str, Option<&str>, Option<&str>, Option<&str>)] = &[ + // (filename, expected system, expected faction, expected class or None-any) + // GW naming: chapters are factions, classes are GW unit types. + ( + "Thunderwolf Cavalry.7z", + Some("Warhammer 40,000"), + Some("Space Wolves"), + Some("Mounted"), + ), + ( + "Wulfen.zip", + Some("Warhammer 40,000"), + Some("Space Wolves"), + Some("Infantry"), + ), + ( + "Keeper of Secrets & Shalaxi Helbane.7z", + Some("Warhammer 40,000"), + Some("Chaos Daemons"), + Some("Hero"), + ), + ( + "Assorted Drones.zip", + Some("Warhammer 40,000"), + Some("T'au Empire"), + None, + ), + ( + "Commander Shadowsun.zip", + Some("Warhammer 40,000"), + Some("T'au Empire"), + Some("Hero"), + ), + ( + "Devilfish.zip", + Some("Warhammer 40,000"), + Some("T'au Empire"), + Some("Vehicle"), + ), + ( + "Battle Sister Scanz.zip", + Some("Warhammer 40,000"), + Some("Adepta Sororitas"), + None, + ), + // ambiguous keyword alone: unclassified + ("Knights.zip", None, None, None), + ( + "Khorne Demon Enginez and skull lordz.zip", + Some("Warhammer 40,000"), + Some("Chaos Daemons"), + None, + ), + ( + "Cultists.zip", + Some("Warhammer 40,000"), + Some("Chaos Space Marines"), + Some("Battleline"), + ), + ( + "Da Troopz.zip", + Some("Warhammer 40,000"), + Some("Orks"), + None, + ), + ( + "Seraphon.zip", + Some("Age of Sigmar"), + Some("Seraphon"), + None, + ), + ( + "Leviathan Pattern Dreadnought.zip", + Some("The Horus Heresy"), + Some("Space Marine Legions"), + Some("Walker"), + ), + ( + "Poxwalkers.zip", + Some("Warhammer 40,000"), + Some("Death Guard"), + Some("Infantry"), + ), + ( + "Kabalite Warriors.zip", + Some("Warhammer 40,000"), + Some("Drukhari"), + Some("Battleline"), + ), + ( + "Canoptek Wraiths.zip", + Some("Warhammer 40,000"), + Some("Necrons"), + Some("Infantry"), + ), + ("random thing with no keywords.zip", None, None, None), + ]; + for (name, exp_system, exp_faction, exp_class) in cases { + let (system, faction, klass) = classify(name); + if let Some(exp) = exp_system { + assert_eq!(&system, exp, "system mismatch for {name:?}"); + } + if let Some(exp) = exp_faction { + assert_eq!(&faction, exp, "faction mismatch for {name:?}"); + } + if let Some(exp) = exp_class { + assert_eq!(&klass, exp, "class mismatch for {name:?}"); + } + } + } + + #[test] + fn format_size_matches_python() { + assert_eq!(format_size(0), "0.0 B"); + assert_eq!(format_size(1024), "1.0 KiB"); + assert_eq!(format_size(1536), "1.5 KiB"); + assert_eq!(format_size(1024 * 1024 * 1024), "1.0 GiB"); + } + + #[test] + fn clean_name_matches_python() { + assert_eq!(clean_name("Wulfen.zip"), "wulfen"); + assert_eq!( + clean_name("Keeper of Secrets & Shalaxi Helbane.7z"), + "keeper of secrets & shalaxi helbane" + ); + // stem strips only the last suffix, then dots become spaces + assert_eq!(clean_name("a.b.c.zip"), "a b c"); + assert_eq!(clean_name("Thunderwolf Cavalry.7z"), "thunderwolf cavalry"); + } + + #[test] + fn overrides_load_and_apply() { + let tmp = tempfile::tempdir().expect("tempdir"); + let path = tmp.path().join("overrides.json"); + std::fs::write( + &path, + r#"{"wulfen.zip": {"faction": "Blood Angels", "class": "Walker"}, + "missing.file.zip": {"faction": "Orks"}}"#, + ) + .expect("write overrides"); + + let overrides = load_overrides(Some(&path)).expect("load overrides"); + assert_eq!(overrides.len(), 2); + assert!(load_overrides(None).expect("load none").is_empty()); + assert!( + load_overrides(Some(&tmp.path().join("nope.json"))) + .expect("load missing") + .is_empty() + ); + + let mut entries = vec![ + entry("/m/Wulfen.zip", "Wulfen.zip", 1024), + entry("/m/Devilfish.zip", "Devilfish.zip", 2048), + ]; + let applied = apply_overrides(&mut entries, &overrides); + assert_eq!(applied, 1, "only the existing target counts as applied"); + assert_eq!(entries[0].faction, "Blood Angels"); + assert_eq!(entries[0].klass, "Walker"); + assert_eq!(entries[0].system, UNKNOWN, "system not touched by this fix"); + assert_eq!(entries[1].faction, UNCLASSIFIED_FACTION); + } + + #[test] + fn write_reports_produces_jsonl_and_md() { + let tmp = tempfile::tempdir().expect("tempdir"); + let mut classified = entry("/m/Wulfen.zip", "Wulfen.zip", 1024); + classified.system = "Warhammer 40,000".to_string(); + classified.faction = "Space Wolves".to_string(); + classified.klass = "Infantry".to_string(); + classified.source.url = Some(json!("https://example.com/1")); + let unclassified = entry("/m/mystery.zip", "mystery.zip", 10); + + let out_md = tmp.path().join("catalogue.md"); + let out_jsonl = tmp.path().join("catalogue.jsonl"); + write_reports(&[classified, unclassified], &out_md, &out_jsonl).expect("write reports"); + + let jsonl = std::fs::read_to_string(&out_jsonl).expect("read jsonl"); + let mut rows = Vec::new(); + for line in jsonl.lines() { + rows.push(serde_json::from_str::(line).expect("parseable jsonl line")); + } + assert_eq!(rows.len(), 2); + assert_eq!(rows[0]["system"], "Warhammer 40,000"); + assert_eq!(rows[0]["class"], "Infantry"); + assert_eq!(rows[0]["source"]["url"], "https://example.com/1"); + assert!( + rows[0]["source"].get("channel").is_none(), + "null source fields must be dropped" + ); + assert_eq!(rows[1]["system"], UNKNOWN); + assert_eq!(rows[1]["source"], json!({})); + + let md = std::fs::read_to_string(&out_md).expect("read md"); + assert!(md.contains("# Warhammer Collection Catalogue")); + assert!(md.contains("2 files, 1.0 KiB total, 1 classified, 1 unclassified")); + assert!(md.contains("## Warhammer 40,000 (1 files, 1.0 KiB)")); + assert!(md.contains("### Space Wolves (1.0 KiB)")); + assert!(md.contains("- **Infantry** (1 files, 1.0 KiB)")); + assert!(md.contains(" - Wulfen.zip (1.0 KiB) — from https://example.com/1")); + assert!(md.contains("## Unknown (1 files, 10.0 B)")); + assert!(md.contains(" - mystery.zip (10.0 B)")); + } + + #[test] + fn apply_moves_into_library_and_audits() { + let tmp = tempfile::tempdir().expect("tempdir"); + let inbox = tmp.path().join("inbox"); + std::fs::create_dir_all(&inbox).expect("mkdir inbox"); + + let src = inbox.join("Wulfen.zip"); + std::fs::write(&src, b"data").expect("write src"); + let mut classified = entry(&src.display().to_string(), "Wulfen.zip", 4); + classified.system = "Warhammer 40,000".to_string(); + classified.faction = "Space Wolves".to_string(); + classified.klass = "Infantry".to_string(); + + // unclassified: never moved + let mystery = inbox.join("mystery.zip"); + std::fs::write(&mystery, b"?").expect("write mystery"); + let unclassified = entry(&mystery.display().to_string(), "mystery.zip", 1); + + // classified but missing on disk: warned, skipped + let mut missing = entry("/nope/Devilfish.zip", "Devilfish.zip", 2); + missing.system = "Warhammer 40,000".to_string(); + missing.faction = "T'au Empire".to_string(); + missing.klass = "Vehicle".to_string(); + + let library = tmp.path().join("library"); + let moves_log = tmp.path().join("logs/moves.jsonl"); + let moved = apply_moves( + &mut [classified, unclassified, missing], + &library, + &moves_log, + ) + .expect("apply moves"); + assert_eq!(moved, 1); + + let dest = library + .join("Warhammer 40,000") + .join("Space Marines") // group_of_faction["Space Wolves"] + .join("Space Wolves") + .join("Infantry") + .join("Wulfen.zip"); + assert!(dest.exists(), "moved into the 4-level directory"); + assert!(!src.exists(), "original gone"); + assert_eq!( + entries_path(&mystery), + mystery.display().to_string(), + "unclassified entry untouched" + ); + + let audit = std::fs::read_to_string(&moves_log).expect("read audit"); + let row: Value = serde_json::from_str(audit.trim()).expect("parseable audit line"); + assert_eq!(row["from"], src.display().to_string()); + assert_eq!(row["to"], dest.display().to_string()); + } + + fn entries_path(path: &Path) -> String { + path.display().to_string() + } + + #[test] + fn scan_manifest_dedupes_and_skips_malformed() { + let tmp = tempfile::tempdir().expect("tempdir"); + let path = tmp.path().join("manifest.jsonl"); + std::fs::write( + &path, + concat!( + r#"{"path": "/dl/a/Wulfen.zip", "name": "Wulfen.zip", "size": 12,"#, + r#" "channel": "chan", "url": "https://x/1", "message_id": 42,"#, + r#" "timestamp": "2024-01-01", "author": "ben"}"#, + "\n", + "not json at all\n", + "\n", + r#"{"path": "/dl/a/Wulfen.zip", "name": "dup", "size": 99}"#, + "\n", + r#"{"path": "/dl/b/tar.tar.gz", "size": null}"#, + "\n", + ), + ) + .expect("write manifest"); + + let entries = scan_manifest(&path); + assert_eq!(entries.len(), 2, "malformed, blank and dup lines skipped"); + assert_eq!(entries[0].name, "Wulfen.zip"); + assert_eq!(entries[0].size, 12); + assert_eq!(entries[0].source.message_id, Some(json!(42))); + assert_eq!(entries[0].source.channel, Some(json!("chan"))); + assert_eq!(entries[1].name, "tar.tar.gz", "name falls back to basename"); + assert_eq!(entries[1].size, 0, "null size becomes 0"); + assert!(entries[1].source.url.is_none()); + + assert!(scan_manifest(&tmp.path().join("absent.jsonl")).is_empty()); + } +} diff --git a/version-rust/src/discord.rs b/version-rust/src/discord.rs index dbc4f23..8169f7e 100644 --- a/version-rust/src/discord.rs +++ b/version-rust/src/discord.rs @@ -1,2 +1,516 @@ //! DCE docker export (JSON) + base64/plain URL extraction — Rust port of //! `version-python/dlphelper/discord.py`. +//! +//! The channels this tool targets encode their download links as base64 inside +//! message content, and occasionally as plain links. We export the channel as +//! JSON (richer metadata than the HTML export: message id, timestamp, author), +//! decode every base64 candidate, and keep only URLs for supported hosts. + +use std::collections::HashMap; +use std::fs; +use std::io; +use std::path::{Path, PathBuf}; +use std::process::Command; +use std::sync::LazyLock; + +use anyhow::{Context, Result, bail}; +use base64::Engine as _; +use base64::engine::general_purpose::STANDARD; +use regex::Regex; +use serde_json::Value; +use url::Url; + +pub const DCE_IMAGE: &str = "tyrrrz/discordchatexporter:stable"; + +// The channels encode links as base64 that always starts with this prefix +// (https://...); grab.sh established this regex, keep it compatible. +static B64_RE: LazyLock = + LazyLock::new(|| Regex::new(r"aHR0cHM6Ly[89][A-Za-z0-9+/_-]*").unwrap()); +static URL_RE: LazyLock = + LazyLock::new(|| Regex::new(r#"https://[^\s<>"'\)\]\}]+"#).unwrap()); + +/// Host key ("gofile" | "0807") -> host names accepted for it. +pub const SUPPORTED_HOSTS: &[(&str, &[&str])] = &[ + ("gofile", &["gofile.io", "www.gofile.io"]), + ("0807", &["0807.st", "www.0807.st"]), +]; + +/// One download URL found in the channel, with source attribution. +#[derive(Debug, Clone, PartialEq)] +pub struct FoundUrl { + pub url: String, + /// "gofile" | "0807" (set after classification) + pub host: &'static str, + pub message_id: Option, + pub timestamp: Option, + pub author: Option, + /// found via base64 decode vs. plain link + pub encoded: bool, +} + +/// Export the channel to out_dir/export.json using the DCE docker image. +/// +/// The token is passed to the container via the DISCORD_TOKEN env var (DCE +/// reads it natively) so it never appears on the docker command line. TOKEN +/// is set as well for DCE versions that only know that name. +pub fn export_channel( + channel_id: &str, + out_dir: &Path, + token: &str, + force: bool, +) -> Result { + let dest = out_dir.join("export.json"); + if dest.exists() && !force { + log::info!( + "Reusing existing export {} (pass --fresh to re-export)", + dest.display() + ); + return Ok(dest); + } + + fs::create_dir_all(out_dir)?; + // Python Path.resolve() on an existing dir == canonicalize here (the + // create_dir_all above guarantees it exists). + let resolved = fs::canonicalize(out_dir)?; + let (uid, gid) = read_uid_gid()?; + + log::info!( + "Exporting channel {} via DiscordChatExporter ...", + channel_id + ); + // stdout/stderr are inherited, like the python subprocess.run(check=True). + let status = Command::new("docker") + .args([ + "run", + "--rm", + "--user", + &format!("{uid}:{gid}"), + "-v", + &format!("{}:/out", resolved.display()), + "-e", + "DISCORD_TOKEN", + "-e", + "TOKEN", + ]) + .arg(DCE_IMAGE) + .args([ + "export", + "-c", + channel_id, + "-f", + "Json", + "-o", + "/out/export.json", + ]) + // The token only travels via the environment, never on the argv. + .env("DISCORD_TOKEN", token) + .env("TOKEN", token) + .status(); + match status { + Err(e) if e.kind() == io::ErrorKind::NotFound => { + bail!("docker not found - the Discord export needs the docker image {DCE_IMAGE}") + } + Err(e) => return Err(e).context("failed to spawn docker"), + Ok(s) if !s.success() => bail!( + "DiscordChatExporter failed with exit code {} (check the token and the channel id)", + s.code().unwrap_or(-1) + ), + Ok(_) => {} + } + if !dest.exists() { + bail!("DiscordChatExporter produced no output file"); + } + Ok(dest) +} + +/// Real uid/gid of this process. Linux only: parsed from /proc/self/status +/// because std has no getuid (the first number on the Uid:/Gid: line is the +/// real id, matching python's os.getuid()/os.getgid()). +fn read_uid_gid() -> Result<(u32, u32)> { + let status = fs::read_to_string("/proc/self/status") + .context("cannot read /proc/self/status to determine uid/gid for docker --user")?; + let uid = parse_proc_id(&status, "Uid:")?; + let gid = parse_proc_id(&status, "Gid:")?; + Ok((uid, gid)) +} + +fn parse_proc_id(status_text: &str, key: &str) -> Result { + for line in status_text.lines() { + if let Some(rest) = line.strip_prefix(key) { + let first = rest + .split_whitespace() + .next() + .with_context(|| format!("empty {key} line in /proc/self/status"))?; + return first.parse().with_context(|| { + format!("cannot parse {key} value {first:?} in /proc/self/status") + }); + } + } + bail!("{key} line not found in /proc/self/status") +} + +/// Return the host key ('gofile' | '0807') for a URL, else None. +/// +/// Mirrors python `urlparse(url).hostname.lower()`: unparseable URLs and URLs +/// without a hostname yield None. The url crate normalizes the host to +/// lowercase, same as urlparse does. +pub fn classify(url: &str) -> Option<&'static str> { + let parsed = Url::parse(url).ok()?; + let hostname = parsed.host_str()?; + SUPPORTED_HOSTS + .iter() + .find(|(_, hosts)| hosts.contains(&hostname)) + .map(|(key, _)| *key) +} + +/// Decode one base64 candidate into the URL it contains, or None. +/// +/// Python quirk kept: "_" is remapped to "/" and "-" to "+" (in that order), +/// trailing "=" stripped, then padded back to a multiple of 4 before a strict +/// standard-alphabet decode and a strict UTF-8 check. One encoded blob can +/// contain the URL plus padding/newlines: whitespace is trimmed and only the +/// first line is returned. +fn decode_b64(candidate: &str) -> Option { + let mut s: String = candidate + .chars() + .map(|c| match c { + '_' => '/', + '-' => '+', + other => other, + }) + .collect(); + while s.ends_with('=') { + s.pop(); + } + for _ in 0..(4 - s.len() % 4) % 4 { + s.push('='); + } + let decoded = STANDARD.decode(s.as_bytes()).ok()?; + let text = String::from_utf8(decoded).ok()?; + // strip whitespace, keep the first line ("" -> None) + text.trim() + .lines() + .next() + .filter(|l| !l.is_empty()) + .map(String::from) +} + +/// Python `str(x or "")` for a JSON value: falsy values (null, false, 0, "", +/// empty array/object) become the empty string, strings pass through, and any +/// other value is stringified. DCE content/id fields are always strings; the +/// other shapes only exist for parity with the python code. +fn python_str_or_empty(value: Option<&Value>) -> String { + let falsy = match value { + None | Some(Value::Null) => true, + Some(Value::Bool(b)) => !b, + Some(Value::Number(n)) => n.as_f64().is_some_and(|f| f == 0.0), + Some(Value::String(s)) => s.is_empty(), + Some(Value::Array(a)) => a.is_empty(), + Some(Value::Object(o)) => o.is_empty(), + }; + if falsy { + return String::new(); + } + match value.unwrap() { + Value::String(s) => s.clone(), + Value::Number(n) => n.to_string(), + Value::Bool(_) => "True".to_string(), // python str(True) + other => other.to_string(), + } +} + +/// Extract supported URLs from a DCE JSON export, deduplicated. +/// +/// The root may be a list of messages or an object with a "messages" array; +/// anything else yields no URLs. The first occurrence of a URL wins +/// (python dict.setdefault semantics). +pub fn extract_urls(export_path: &Path) -> Result> { + let text = fs::read_to_string(export_path)?; + let data: Value = serde_json::from_str(&text)?; + let empty: Vec = Vec::new(); + let messages: &[Value] = match &data { + Value::Array(list) => list, + Value::Object(obj) => obj + .get("messages") + .and_then(Value::as_array) + .map(|a| a.as_slice()) + .unwrap_or(&empty), + _ => &empty, + }; + + let mut found: HashMap = HashMap::new(); + let mut stat_b64 = 0u64; + let mut stat_plain = 0u64; + let mut stat_unsupported = 0u64; + + for msg in messages { + let content = python_str_or_empty(msg.get("content")); + if content.is_empty() { + continue; + } + let author = msg + .get("author") + .and_then(|a| a.get("name")) + .and_then(Value::as_str) + .map(str::to_owned); + let message_id = { + let s = python_str_or_empty(msg.get("id")); + (!s.is_empty()).then_some(s) + }; + let timestamp = msg + .get("timestamp") + .and_then(Value::as_str) + .map(str::to_owned); + + for m in B64_RE.find_iter(&content) { + stat_b64 += 1; + let Some(decoded) = decode_b64(m.as_str()) else { + continue; + }; + let Some(host) = classify(&decoded) else { + stat_unsupported += 1; + continue; + }; + found.entry(decoded.clone()).or_insert(FoundUrl { + url: decoded, + host, + message_id: message_id.clone(), + timestamp: timestamp.clone(), + author: author.clone(), + encoded: true, + }); + } + + for m in URL_RE.find_iter(&content) { + let url = m + .as_str() + .trim_end_matches(['.', ',', ';', ')']) + .to_string(); + let Some(host) = classify(&url) else { + continue; + }; + stat_plain += 1; + found.entry(url.clone()).or_insert(FoundUrl { + url, + host, + message_id: message_id.clone(), + timestamp: timestamp.clone(), + author: author.clone(), + encoded: false, + }); + } + } + + log::info!( + "Extraction: {} base64 candidates, {} plain links, {} unsupported/undecodable, {} unique supported URLs", + stat_b64, + stat_plain, + stat_unsupported, + found.len() + ); + let mut out: Vec = found.into_values().collect(); + out.sort_by(|a, b| a.url.cmp(&b.url)); + Ok(out) +} + +#[cfg(test)] +mod tests { + use super::*; + use base64::engine::general_purpose::URL_SAFE_NO_PAD; + use serde_json::json; + use tempfile::tempdir; + + fn b64(s: &str) -> String { + STANDARD.encode(s.as_bytes()) + } + + /// The two-message fixture from version-python/tests/test_extract.py. + fn fixture_messages() -> Value { + json!([ + { + "id": "1", + "timestamp": "2026-01-01T00:00:00", + "author": {"name": "poster"}, + "content": format!("chaos stuff: {}", b64("https://gofile.io/d/abc123")), + }, + { + "id": "2", + "timestamp": "2026-01-02T00:00:00", + "author": {"name": "poster"}, + "content": format!( + "0807 zip: {}\ndupe of 1: {}\nplain: https://0807.st/plainLink.zip and \ + https://discord.com/channels/1/2 (unsupported)\njunk: {} and \ + not-our-domain base64: aHR0cHM6Ly9leGFtcGxlLm9yZw==", + b64("https://0807.st/6saCNie.zip"), + b64("https://gofile.io/d/abc123"), + b64("https://pixeldrain.com/u/xyz"), + ), + }, + ]) + } + + fn write_export(dir: &Path, root: Value) -> PathBuf { + let path = dir.join("export.json"); + fs::write(&path, serde_json::to_string(&root).unwrap()).unwrap(); + path + } + + #[test] + fn extract_urls_matches_python_fixture() { + let dir = tempdir().unwrap(); + let export = write_export(dir.path(), json!({"messages": fixture_messages()})); + let found = extract_urls(&export).unwrap(); + + let urls: Vec<&str> = found.iter().map(|f| f.url.as_str()).collect(); + assert_eq!( + urls, + vec![ + "https://0807.st/6saCNie.zip", + "https://0807.st/plainLink.zip", + "https://gofile.io/d/abc123", + ] + ); + + let g = found + .iter() + .find(|f| f.url == "https://gofile.io/d/abc123") + .unwrap(); + assert_eq!(g.host, "gofile"); + assert_eq!(g.message_id.as_deref(), Some("1")); + assert_eq!(g.timestamp.as_deref(), Some("2026-01-01T00:00:00")); + assert_eq!(g.author.as_deref(), Some("poster")); + assert!(g.encoded); + + let z = found + .iter() + .find(|f| f.url == "https://0807.st/6saCNie.zip") + .unwrap(); + assert_eq!(z.host, "0807"); + assert_eq!(z.message_id.as_deref(), Some("2")); + assert!(z.encoded); + + let p = found + .iter() + .find(|f| f.url == "https://0807.st/plainLink.zip") + .unwrap(); + assert_eq!(p.host, "0807"); + assert_eq!(p.message_id.as_deref(), Some("2")); + assert_eq!(p.author.as_deref(), Some("poster")); + assert!(!p.encoded); + } + + #[test] + fn extract_urls_accepts_bare_list_root() { + let dir = tempdir().unwrap(); + let export = write_export(dir.path(), fixture_messages()); + let found = extract_urls(&export).unwrap(); + assert_eq!(found.len(), 3); + } + + #[test] + fn extract_urls_empty_and_malformed_roots_yield_nothing() { + let dir = tempdir().unwrap(); + assert!( + extract_urls(&write_export(dir.path(), json!({}))) + .unwrap() + .is_empty() + ); + assert!( + extract_urls(&write_export(dir.path(), json!({"other": []}))) + .unwrap() + .is_empty() + ); + // message without content is skipped + let export = write_export(dir.path(), json!([{"id": "7"}])); + assert!(extract_urls(&export).unwrap().is_empty()); + } + + #[test] + fn classify_matches_python_hostnames() { + assert_eq!(classify("https://0807.st/x.zip"), Some("0807")); + assert_eq!(classify("https://www.0807.st/x.zip"), Some("0807")); + assert_eq!(classify("https://gofile.io/d/x"), Some("gofile")); + assert_eq!(classify("https://www.gofile.io/d/x"), Some("gofile")); + assert_eq!(classify("https://example.org/x"), None); + // url crate lowercases hosts, like urlparse().hostname.lower() + assert_eq!(classify("https://GOFILE.IO/d/x"), Some("gofile")); + // edge cases: no scheme, empty string, bare host + assert_eq!(classify("0807.st/x.zip"), None); + assert_eq!(classify(""), None); + assert_eq!(classify("https://"), None); + } + + #[test] + fn decode_b64_handles_padding_and_rstrip() { + // candidate as matched by B64_RE carries no '=': padding is added + let unpadded = b64("https://gofile.io/d/abc123") + .trim_end_matches('=') + .to_string(); + assert_eq!( + decode_b64(&unpadded), + Some("https://gofile.io/d/abc123".to_string()) + ); + // trailing '=' is stripped before re-padding + assert_eq!( + decode_b64("aHR0cHM6Ly9leGFtcGxlLm9yZw=="), + Some("https://example.org".to_string()) + ); + } + + #[test] + fn decode_b64_remaps_urlsafe_alphabet() { + // U+13C0 encodes to 4Y+A (standard) / 4Y-A (urlsafe); the python + // quirk remaps '-' back to '+' before decoding. ('_' -> '/' can never + // occur in a candidate that decodes to valid UTF-8, so '-' covers the + // remap path.) + let candidate = URL_SAFE_NO_PAD.encode("\u{13C0}"); + assert!(candidate.contains('-'), "expected '-' in {candidate}"); + assert_eq!(decode_b64(&candidate), Some("\u{13C0}".to_string())); + } + + #[test] + fn decode_b64_keeps_first_line_after_whitespace() { + let candidate = STANDARD.encode("https://gofile.io/d/abc123\nnoise "); + assert_eq!( + decode_b64(&candidate), + Some("https://gofile.io/d/abc123".to_string()) + ); + } + + #[test] + fn decode_b64_rejects_invalid_utf8_and_garbage() { + // decodes to \xff\xfe — not valid UTF-8 + assert_eq!(decode_b64("//8+"), None); + assert_eq!(decode_b64("aHR0cHM6Ly["), None); + assert_eq!(decode_b64(""), None); + } + + #[test] + fn parse_proc_id_reads_first_number() { + let text = "Name:\tx\nUid:\t1000\t1000\t1000\t1000\nGid:\t989\t989\t989\t989\n"; + assert_eq!(parse_proc_id(text, "Uid:").unwrap(), 1000); + assert_eq!(parse_proc_id(text, "Gid:").unwrap(), 989); + assert!(parse_proc_id("Name:\tx\n", "Uid:").is_err()); + } + + #[test] + fn read_uid_gid_matches_proc_self() { + // Linux-only binary (docker export); /proc/self/status must exist. + let (uid, gid) = read_uid_gid().unwrap(); + let text = fs::read_to_string("/proc/self/status").unwrap(); + let want_uid = parse_proc_id(&text, "Uid:").unwrap(); + let want_gid = parse_proc_id(&text, "Gid:").unwrap(); + assert_eq!((uid, gid), (want_uid, want_gid)); + } + + #[test] + fn export_channel_reuses_existing_export_without_docker() { + let dir = tempdir().unwrap(); + let dest = dir.path().join("export.json"); + fs::write(&dest, "[]").unwrap(); + // force=false must short-circuit before any docker invocation; the + // token must not end up in the error/log path either. + let got = export_channel("123", dir.path(), "s3cret-token", false).unwrap(); + assert_eq!(got, dest); + } +} diff --git a/version-rust/src/extract.rs b/version-rust/src/extract.rs index e5c4198..c6df56e 100644 --- a/version-rust/src/extract.rs +++ b/version-rust/src/extract.rs @@ -2,8 +2,1193 @@ //! `version-python/tools/extract_all.py` (extract_manifest.jsonl feeds the //! sorter's provenance mode; failures go to .broken/; the run never stops). //! -//! Extraction backend: `sevenz-rust2` for .7z, `zip` for .zip, `unrar` for -//! .rar, `tar`+`flate2`/`bzip2`/`xz2` for tar balls and bare compressed -//! files. Because extraction is pure Rust (no 7z CLI writing directly onto -//! CIFS), the python version's /var/tmp fallback + cp437 name repair is -//! not needed here. +//! Extraction backends: `sevenz-rust2` for .7z, `zip` for .zip, `unrar` for +//! .rar — pure Rust, no external binaries. Tarballs and bare compressed +//! files (.tar/.tar.gz/.tar.bz2/.tar.xz/.gz/.bz2/.xz) are handled by the +//! system `tar`/`gzip`/`bzip2`/`xz` binaries, exactly like the python +//! version shelled out to `7z`: merged output capture, exit-code parity +//! (0 ok, 1 warnings), and the 4h per-archive timeout. The tar/flate2/ +//! bzip2/xz2 crates mentioned in the port plan are not declared +//! dependencies of this crate and Cargo.toml is off-limits, so the system +//! binaries are the parity-preserving fallback (they are universally +//! present on the target Linux hosts). +//! +//! Because the zip/7z paths are pure Rust, the python version's /var/tmp +//! fallback + cp437 name repair is not needed here (the zip crate decodes +//! legacy cp437 entry names itself). +//! +//! Behavior parity notes (vs. the python version): +//! - Skip decisions are made on manifest records only, never on "dir +//! non-empty"; a partial dir from a crash is re-extracted. +//! - Source roots are never modified except for the quarantine move of +//! failed archives into `/.broken/`. +//! - The extracted tree holds no archives afterwards (nested archives are +//! expanded BFS-style and their copies unlinked). +//! - Plain files are hardlinked in (fallback: metadata-preserving copy). +//! - No individual failure ever aborts the run. + +use std::collections::{HashSet, VecDeque}; +use std::fs::{self, File}; +use std::io::{self, Read, Write}; +use std::path::{Component, Path, PathBuf}; +use std::process::{Command, Stdio}; +use std::time::{Duration, Instant}; + +#[cfg(unix)] +use std::os::unix::fs::PermissionsExt; + +/// Archive suffixes recognized by the extractors (python `ARCHIVE_SUFFIXES`). +pub const ARCHIVE_SUFFIXES: [&str; 7] = [".zip", ".7z", ".rar", ".tar", ".gz", ".bz2", ".xz"]; + +/// Stop extracting below this free space (python `MIN_FREE_BYTES`). +pub const MIN_FREE_BYTES: u64 = 20 * 1024 * 1024 * 1024; + +/// Per-archive extraction timeout in seconds (python `EXTRACT_TIMEOUT`). +const EXTRACT_TIMEOUT_SECS: u64 = 4 * 3600; + +/// Run artifacts that must never be treated as payload (python `RUN_ARTIFACTS`). +pub const RUN_ARTIFACTS: [&str; 14] = [ + "export.json", + "export.html", + "state.json", + "manifest.jsonl", + "failed_urls.txt", + "catalogue.md", + "catalogue.jsonl", + "sorter_applied.jsonl", + "catalogue_moves.jsonl", + "urls.txt", + "urls_0807.txt", + "urls_gofile.txt", + "failed_0807.txt", + "extract_manifest.jsonl", +]; + +/// Recursive extractor writing into `out_root`. +pub struct Extractor { + pub out_root: PathBuf, +} + +/// Python `Path.suffix` semantics for a plain file name: the trailing dot +/// segment, unless the name starts with a dot (hidden) or ends with a dot. +fn name_suffix(name: &str) -> Option<&str> { + let idx = name.rfind('.')?; + if idx == 0 || idx == name.len() - 1 { + return None; + } + Some(&name[idx..]) +} + +/// All trailing archive suffixes, lowercased, outermost first — e.g. +/// `x.tar.gz` -> `[".gz", ".tar"]`. +pub fn archive_suffixes(name: &str) -> Vec { + let mut out = Vec::new(); + let mut cur = name; + while let Some(sfx) = name_suffix(cur) { + let lower = sfx.to_ascii_lowercase(); + if !ARCHIVE_SUFFIXES.contains(&lower.as_str()) { + break; + } + out.push(lower); + cur = &cur[..cur.len() - sfx.len()]; + } + out +} + +/// True when the file name carries at least one archive suffix. +pub fn is_archive(name: &str) -> bool { + !archive_suffixes(name).is_empty() +} + +/// Archive name with all archive suffixes stripped: the unit folder name. +pub fn display_unit(name: &str) -> String { + let mut cur = name; + while let Some(sfx) = name_suffix(cur) { + let lower = sfx.to_ascii_lowercase(); + if !ARCHIVE_SUFFIXES.contains(&lower.as_str()) { + break; + } + cur = &cur[..cur.len() - sfx.len()]; + } + cur.to_string() +} + +/// Free disk space (bytes) on the file system containing `path`. +pub fn free_bytes(path: &Path) -> anyhow::Result { + Ok(fs4::available_space(path)?) +} + +/// AppleDouble resource-fork "zips" (`__MACOSX/._x.zip`) — no content. +pub fn is_macosx_junk(p: &Path) -> bool { + p.components().any(|c| c.as_os_str() == "__MACOSX") + || p.file_name() + .map(|n| n.to_string_lossy().starts_with("._")) + .unwrap_or(false) +} + +/// Relative path helper: `p` relative to `root` (falls back to `p` itself). +fn rel_to(root: &Path, p: &Path) -> PathBuf { + p.strip_prefix(root).unwrap_or(p).to_path_buf() +} + +/// Same as [`rel_to`] as a (lossy) string, matching python `str(rel_path)`. +fn rel_string(root: &Path, p: &Path) -> String { + rel_to(root, p).to_string_lossy().into_owned() +} + +fn file_name_string(p: &Path) -> String { + p.file_name() + .map(|n| n.to_string_lossy().into_owned()) + .unwrap_or_default() +} + +/// Recursively collect regular files under `root`, sorted for determinism. +/// +/// Matches python `sorted(rglob("*"))` + `is_file()` filtering: symlinked +/// directories are not descended into, symlinks to files count as files. +fn walk_files_sorted(root: &Path) -> Vec { + let mut files = Vec::new(); + let mut dirs = vec![root.to_path_buf()]; + while let Some(dir) = dirs.pop() { + let Ok(entries) = fs::read_dir(&dir) else { + continue; + }; + for entry in entries.flatten() { + let path = entry.path(); + match fs::symlink_metadata(&path) { + Ok(meta) if meta.is_dir() => dirs.push(path), + Ok(meta) if meta.is_file() => files.push(path), + Ok(meta) if meta.is_symlink() => { + if let Ok(target) = fs::metadata(&path) { + if target.is_file() { + files.push(path); + } + } + } + _ => {} + } + } + } + files.sort(); + files +} + +/// One provenance manifest line, compact JSON, raw UTF-8 (python +/// `json.dumps(..., ensure_ascii=False)`; keys order as archive/dest/unit). +fn write_manifest_line( + manifest: &mut W, + out_root: &Path, + archive_rel: &Path, + actual_dest: &Path, +) { + let dest_rel = rel_to(out_root, actual_dest); + let unit = archive_rel + .file_name() + .map(|n| display_unit(&n.to_string_lossy())) + .unwrap_or_default(); + let line = serde_json::to_string(&serde_json::json!({ + "archive": archive_rel.to_string_lossy().into_owned(), + "dest": dest_rel.to_string_lossy().into_owned(), + "unit": unit, + })) + .unwrap_or_else(|_| "{}".to_string()); + if let Err(e) = writeln!(manifest, "{line}") { + log::error!("Cannot write manifest line: {e}"); + } +} + +/// Run an extraction command like python `_run_7z`: merged stdout+stderr +/// captured (stderr only when stdout is redirected to a file), tolerant +/// decoding, per-archive timeout, returns (rc, last-5-lines tail). +/// rc == -1 marks "failed to run" / timeout, like python. +fn run_extract_cmd( + program: &str, + args: &[std::ffi::OsString], + stdout_to: Option<&Path>, +) -> (i32, String) { + let mut cmd = Command::new(program); + cmd.args(args); + let mut stdout_thread = None; + let mut stderr_thread = None; + match stdout_to { + Some(path) => match File::create(path) { + Ok(f) => { + cmd.stdout(Stdio::from(f)); + cmd.stderr(Stdio::piped()); + } + Err(e) => return (-1, format!("{program} failed to run: {e}")), + }, + None => { + cmd.stdout(Stdio::piped()); + cmd.stderr(Stdio::piped()); + } + } + let mut child = match cmd.spawn() { + Ok(c) => c, + Err(e) => return (-1, format!("{program} failed to run: {e}")), + }; + if let Some(mut pipe) = child.stdout.take() { + stdout_thread = Some(std::thread::spawn(move || { + let mut s = String::new(); + let _ = pipe.read_to_string(&mut s); + s + })); + } + if let Some(mut pipe) = child.stderr.take() { + stderr_thread = Some(std::thread::spawn(move || { + let mut s = String::new(); + let _ = pipe.read_to_string(&mut s); + s + })); + } + let deadline = Instant::now() + Duration::from_secs(EXTRACT_TIMEOUT_SECS); + let mut timed_out = false; + let status = loop { + match child.try_wait() { + Ok(Some(st)) => break Some(st), + Ok(None) => { + if Instant::now() >= deadline { + let _ = child.kill(); + let _ = child.wait(); + timed_out = true; + break None; + } + std::thread::sleep(Duration::from_millis(25)); + } + Err(e) => { + let _ = child.kill(); + let _ = child.wait(); + return (-1, format!("{program} wait failed: {e}")); + } + } + }; + let mut text = String::new(); + if let Some(t) = stdout_thread { + text.push_str(&t.join().unwrap_or_default()); + } + if let Some(t) = stderr_thread { + text.push_str(&t.join().unwrap_or_default()); + } + if timed_out { + return ( + -1, + format!("extraction timed out after {EXTRACT_TIMEOUT_SECS}s"), + ); + } + let code = status.and_then(|s| s.code()).unwrap_or(-1); + let tail = text + .lines() + .rev() + .take(5) + .collect::>() + .into_iter() + .rev() + .collect::>() + .join("\n"); + (code, tail) +} + +impl Extractor { + fn space_low(&self) -> bool { + match free_bytes(&self.out_root) { + Ok(bytes) => bytes < MIN_FREE_BYTES, + Err(e) => { + log::warn!( + "free-space check failed on {}: {e}", + self.out_root.display() + ); + false + } + } + } + + /// Create the destination directory, reproducing the python collision + /// semantics: if the stem is occupied by a *file*, retry as `_dup` + /// (python's loop always rebuilds from the original name, so repeated + /// `_dup` collisions exhaust the 5 attempts); a parent that is a file + /// (NotADirectoryError) fails immediately. + fn create_dest(&self, dest: &Path) -> Result { + let mut actual = dest.to_path_buf(); + for _ in 0..5 { + match fs::create_dir_all(&actual) { + Ok(()) => return Ok(actual), + Err(e) => match e.kind() { + io::ErrorKind::AlreadyExists => { + let name = file_name_string(dest); + actual = dest.with_file_name(format!("{name}_dup")); + } + _ => { + return Err(format!( + "cannot create destination {}: {e}", + actual.display() + )); + } + }, + } + } + Err(format!( + "cannot create destination {} (repeated collisions)", + dest.display() + )) + } + + /// Extract `archive` into `dest`. Returns (ok, error, dest_used); on a + /// name collision the destination gets a `_dup` suffix instead of + /// crashing. Never panics on archive content; every error is reported. + pub fn extract(&self, archive: &Path, dest: &Path) -> (bool, String, PathBuf) { + let actual = match self.create_dest(dest) { + Ok(a) => a, + Err(err) => return (false, err, dest.to_path_buf()), + }; + let name = file_name_string(archive); + let sfx = archive_suffixes(&name); + let res: Result<(), String> = match sfx.last().map(|s| s.as_str()) { + Some(".7z") => extract_7z(archive, &actual), + Some(".zip") => extract_zip(archive, &actual), + Some(".rar") => extract_rar(archive, &actual), + Some(outer @ (".gz" | ".bz2" | ".xz")) => { + if sfx.len() >= 2 && sfx[1] == ".tar" { + // .tar.gz|bz2|xz — decompress stream and unpack in + // one go (python does it in two 7z passes via the nested + // expansion; final tree content is identical). + extract_tar_cmd(archive, &actual, Some(outer)) + } else { + // Bare compressed file (one layer, like a single `7z x`): + // writes dest/; anything that is + // still an archive afterwards is picked up by the + // nested/sweep expansion. + decompress_bare_cmd(archive, &actual, &name, outer) + } + } + Some(".tar") => extract_tar_cmd(archive, &actual, None), + _ => Err(format!("no extractor for {name}")), + }; + match res { + Ok(()) => (true, String::new(), actual), + Err(err) => (false, err, actual), + } + } + + /// Move a failed archive under `/.broken/`; the source + /// root is otherwise never modified. Returns the quarantine path. + pub fn move_to_broken(&self, path: &Path, src_root: &Path, note: &str) -> PathBuf { + let rel = path.strip_prefix(src_root).unwrap_or(path); + let dest = self.out_root.join(".broken").join(rel); + if let Some(parent) = dest.parent() { + let _ = fs::create_dir_all(parent); + } + let mut final_dest = dest.clone(); + if dest.exists() { + let mut name = dest + .file_name() + .map(|n| n.to_os_string()) + .unwrap_or_default(); + name.push(".1"); + final_dest = dest.with_file_name(name); + } + log::debug!( + "Quarantining {} -> {} ({note})", + path.display(), + final_dest.display() + ); + if let Err(e) = fs::rename(path, &final_dest) { + // cross-device fallback: copy + remove (shutil.move semantics) + match fs::copy(path, &final_dest).and_then(|_| fs::remove_file(path)) { + Ok(()) => {} + Err(e2) => { + log::error!("Cannot quarantine {}: {e} / {e2}", path.display()); + } + } + } + final_dest + } + + /// Extract archives inside `dest` until none remain (BFS). Returns the + /// number of nested archives extracted. + pub fn expand_nested( + &self, + dest: &Path, + manifest: &mut W, + done: &mut HashSet, + ) -> usize { + let mut done_count = 0; + let mut queue: VecDeque = VecDeque::new(); + queue.push_back(dest.to_path_buf()); + while let Some(cur) = queue.pop_front() { + for p in walk_files_sorted(&cur) { + let name = file_name_string(&p); + if !is_archive(&name) { + continue; + } + if is_macosx_junk(&p) { + let _ = fs::remove_file(&p); + log::info!("Junk (macOS AppleDouble) removed: {name}"); + continue; + } + let stem_dir = p.with_file_name(display_unit(&name)); + if done.contains(&rel_string(&self.out_root, &stem_dir)) { + // already extracted in a previous run; drop the archive copy + let _ = fs::remove_file(&p); + log::info!("Nested {name} already extracted - archive removed"); + continue; + } + if self.space_low() { + log::error!( + "Out of disk space - skipping {} (kept in place)", + p.display() + ); + continue; + } + let (ok, err, actual) = self.extract(&p, &stem_dir); + if !ok { + log::error!("FAILED nested extraction {}: {}", p.display(), err); + self.move_to_broken(&p, &self.out_root, &err); + continue; + } + write_manifest_line( + manifest, + &self.out_root, + &rel_to(&self.out_root, &p), + &actual, + ); + done.insert(rel_string(&self.out_root, &actual)); + let _ = fs::remove_file(&p); // archive copy consumed + done_count += 1; + queue.push_back(actual); + } + } + done_count + } + + /// Extract every archive directly under `src_root` (any depth) into + /// `//` and mirror plain + /// files via hardlink (fallback: metadata-preserving copy). Returns + /// (extracted, mirrored, failed). + pub fn process_root( + &self, + src_root: &Path, + manifest: &mut W, + done: &mut HashSet, + ) -> (usize, usize, usize) { + let idx = file_name_string(src_root); + let mut extracted = 0; + let mut mirrored = 0; + let mut failed = 0; + for f in walk_files_sorted(src_root) { + let fname = file_name_string(&f); + if RUN_ARTIFACTS.contains(&fname.as_str()) + || fname.starts_with('.') + || fname.ends_with(".part") + { + continue; + } + let Ok(rel) = f.strip_prefix(src_root).map(|r| r.to_path_buf()) else { + continue; + }; + let rel_str = rel.to_string_lossy().into_owned(); + if is_archive(&fname) { + let unit = display_unit(&fname); + let dest_rel: PathBuf = Path::new(&idx).join(rel.with_file_name(unit)); + let dest_rel_str = dest_rel.to_string_lossy().into_owned(); + // Skip on a manifest record, never on "dir non-empty": a + // crash mid-extraction leaves a partial dir that must be + // redone. + if done.contains(&dest_rel_str) { + log::info!("Skip (already extracted): {rel_str}"); + continue; + } + if self.space_low() { + log::error!("Out of disk space - skipping archive {}", f.display()); + failed += 1; + continue; + } + let dest = self.out_root.join(&dest_rel); + let (ok, err, actual) = self.extract(&f, &dest); + if !ok { + log::error!("FAILED {rel_str}: {err}"); + self.move_to_broken(&f, src_root, &err); + failed += 1; + continue; + } + write_manifest_line( + manifest, + &self.out_root, + &Path::new(&idx).join(&rel), + &actual, + ); + let actual_rel = rel_string(&self.out_root, &actual); + done.insert(actual_rel.clone()); + extracted += 1; + log::info!("Extracted {rel_str} -> {actual_rel}"); + // python discards the nested count here too + let _ = self.expand_nested(&actual, manifest, done); + } else { + let link_dest = self.out_root.join(&idx).join(&rel); + if link_dest.exists() { + continue; + } + if let Some(parent) = link_dest.parent() { + let _ = fs::create_dir_all(parent); + } + if let Err(e) = fs::hard_link(&f, &link_dest) { + // fallback: copy (fs::copy keeps permission bits, like copy2) + log::debug!("hardlink {rel_str} failed ({e}) - falling back to copy"); + if let Err(e2) = fs::copy(&f, &link_dest) { + log::error!("Cannot mirror {rel_str}: {e2}"); + continue; + } + } + mirrored += 1; + } + } + (extracted, mirrored, failed) + } + + /// Expand any archive left anywhere in the tree (interrupted nested + /// expansions leave stranded copies even when the parent has a record). + pub fn sweep(&self, manifest: &mut W, done: &mut HashSet) -> usize { + let mut swept = 0; + for p in walk_files_sorted(&self.out_root) { + let rel = rel_to(&self.out_root, &p); + if rel.components().any(|c| c.as_os_str() == ".broken") { + continue; + } + let name = file_name_string(&p); + if !is_archive(&name) { + continue; + } + if is_macosx_junk(&p) { + let _ = fs::remove_file(&p); + log::info!("Junk (macOS AppleDouble) removed: {name}"); + continue; + } + let stem_dir = p.with_file_name(display_unit(&name)); + if done.contains(&rel_string(&self.out_root, &stem_dir)) { + let _ = fs::remove_file(&p); + log::info!("Sweep: {name} already extracted - archive removed"); + continue; + } + if self.space_low() { + log::error!("Out of disk space - sweep skipping {}", p.display()); + continue; + } + let (ok, err, actual) = self.extract(&p, &stem_dir); + if !ok { + log::error!("FAILED sweep extraction {}: {}", p.display(), err); + self.move_to_broken(&p, &self.out_root, &err); + continue; + } + write_manifest_line(manifest, &self.out_root, &rel, &actual); + done.insert(rel_string(&self.out_root, &actual)); + let _ = fs::remove_file(&p); + swept += 1; + self.expand_nested(&actual, manifest, done); + } + swept + } + + /// Full run over `sources`: process each root, then one sweep. Failures + /// are quarantined and logged; the run never stops on individual errors. + pub fn run(&self, sources: &[PathBuf]) -> anyhow::Result<()> { + fs::create_dir_all(&self.out_root)?; + let manifest_path = self.out_root.join("extract_manifest.jsonl"); + let mut done = load_done(&manifest_path); + log::info!("{} destinations already recorded in manifest", done.len()); + let mut manifest = fs::OpenOptions::new() + .append(true) + .create(true) + .open(&manifest_path)?; + + let mut total_extracted = 0; + let mut total_mirrored = 0; + let mut total_failed = 0; + for src in sources { + log::info!("== processing {}", src.display()); + let (e, m, f) = self.process_root(src, &mut manifest, &mut done); + log::info!( + "== {}: {e} extracted, {m} mirrored, {f} failed", + file_name_string(src) + ); + total_extracted += e; + total_mirrored += m; + total_failed += f; + } + let swept = self.sweep(&mut manifest, &mut done); + log::info!( + "TOTAL: {total_extracted} archives extracted, {total_mirrored} files mirrored, \ + {total_failed} failed, {swept} swept" + ); + if total_failed > 0 { + log::warn!( + "Failures were moved under {} - inspect and re-run", + self.out_root.join(".broken").display() + ); + } + Ok(()) + } +} + +/// Dest dirs already recorded in the manifest (relative to out root). +/// Malformed lines are skipped. +pub fn load_done(manifest_path: &Path) -> HashSet { + let mut done = HashSet::new(); + let Ok(bytes) = fs::read(manifest_path) else { + return done; + }; + for line in bytes.split(|&b| b == b'\n') { + let line = String::from_utf8_lossy(line); + let line = line.trim(); + if line.is_empty() { + continue; + } + if let Ok(value) = serde_json::from_str::(line) { + if let Some(dest) = value.get("dest").and_then(|d| d.as_str()) { + done.insert(dest.to_string()); + } + } + } + done +} + +/// 7z via sevenz-rust2. Empty password: python passes `-p ""` (no password, +/// avoids interactive prompts) — `Password::empty()` is the same thing; the +/// password-less helper is retried once in case the with-password path +/// rejects an empty password outright. Password-protected archives still +/// fail, exactly like the python version (no password known). +fn extract_7z(archive: &Path, dest: &Path) -> Result<(), String> { + match sevenz_rust2::decompress_file_with_password( + archive, + dest, + sevenz_rust2::Password::empty(), + ) { + Ok(()) => Ok(()), + Err(first) => match sevenz_rust2::decompress_file(archive, dest) { + Ok(()) => Ok(()), + Err(second) => Err(format!("7z: {first} / {second}")), + }, + } +} + +/// zip via the zip crate, entry by entry. Traversal-safe: entries whose raw +/// name contains `..` or that cannot be enclosed under dest are skipped +/// (python relies on 7z being safe — same minimum). +fn extract_zip(archive: &Path, dest: &Path) -> Result<(), String> { + let file = File::open(archive).map_err(|e| format!("zip: {e}"))?; + let mut zip = zip::ZipArchive::new(file).map_err(|e| format!("zip: {e}"))?; + for i in 0..zip.len() { + let mut entry = zip.by_index(i).map_err(|e| format!("zip: {e}"))?; + if entry.name().split('/').any(|part| part == "..") { + log::warn!("zip: rejecting traversal entry {:?}", entry.name_raw()); + continue; + } + let Some(path) = entry.enclosed_name() else { + log::warn!("zip: skipping unsafe entry {:?}", entry.name_raw()); + continue; + }; + let out_path = dest.join(&path); + if entry.is_dir() { + fs::create_dir_all(&out_path).map_err(|e| format!("zip: {e}"))?; + } else { + if let Some(parent) = out_path.parent() { + fs::create_dir_all(parent).map_err(|e| format!("zip: {e}"))?; + } + let mut out_file = File::create(&out_path).map_err(|e| format!("zip: {e}"))?; + io::copy(&mut entry, &mut out_file).map_err(|e| format!("zip: {e}"))?; + #[cfg(unix)] + if let Some(mode) = entry.unix_mode() { + let _ = fs::set_permissions(&out_path, fs::Permissions::from_mode(mode)); + } + } + } + Ok(()) +} + +/// rar via the unrar crate (C library), no password. Entries with `..` +/// components are skipped rather than extracted. (Untested offline: .rar +/// archives cannot be created without the proprietary tool.) +fn extract_rar(archive: &Path, dest: &Path) -> Result<(), String> { + fs::create_dir_all(dest).map_err(|e| format!("rar: {e}"))?; + let mut arc = unrar::Archive::new(archive) + .open_for_processing() + .map_err(|e| format!("rar: {e}"))?; + loop { + match arc.read_header() { + Ok(None) => break, + Ok(Some(header)) => { + let fname = header.entry().filename.clone(); + if fname + .components() + .any(|c| matches!(c, Component::ParentDir)) + { + log::warn!("rar: skipping entry with '..': {}", fname.display()); + arc = header.skip().map_err(|e| format!("rar: {e}"))?; + continue; + } + arc = header + .extract_with_base(dest) + .map_err(|e| format!("rar: {e}"))?; + } + Err(e) => return Err(format!("rar: {e}")), + } + } + Ok(()) +} + +/// Tarballs via the system `tar` binary — the same shape as the python +/// version's `7z` subprocess call: merged output, rc 0/1 = ok (1 logs a +/// warning), 4h timeout. `decompress` selects -z/-j/-J or None for bare tar. +fn extract_tar_cmd(archive: &Path, dest: &Path, decompress: Option<&str>) -> Result<(), String> { + fs::create_dir_all(dest).map_err(|e| format!("tar: {e}"))?; + let mut args: Vec = vec!["-x".into()]; + if let Some(flag) = decompress { + args.push( + match flag { + ".gz" => "-z", + ".bz2" => "-j", + ".xz" => "-J", + _ => unreachable!("only gz/bz2/xz reach this branch"), + } + .into(), + ); + } + args.push("-f".into()); + args.push(archive.as_os_str().to_os_string()); + args.push("-C".into()); + args.push(dest.as_os_str().to_os_string()); + let (rc, tail) = run_extract_cmd("tar", &args, None); + match rc { + 0 => Ok(()), + 1 => { + log::warn!( + "tar reported warnings extracting {}", + file_name_string(archive) + ); + Ok(()) + } + _ => Err(format!("tar exit {rc}: {tail}")), + } +} + +/// Single-layer decompress of a bare `.gz`/`.bz2`/`.xz` (or any trailing +/// compressed suffix that is not a tarball) into `dest/` via +/// the system decompressor — one layer, like a single `7z x` call. +fn decompress_bare_cmd(archive: &Path, dest: &Path, name: &str, outer: &str) -> Result<(), String> { + let inner = &name[..name.len() - outer.len()]; + fs::create_dir_all(dest).map_err(|e| format!("{outer}: {e}"))?; + let out_path = dest.join(inner); + let (program, what) = match outer { + ".gz" => ("gzip", "gz"), + ".bz2" => ("bzip2", "bz2"), + _ => ("xz", "xz"), + }; + let args = vec![ + std::ffi::OsString::from("-dc"), + archive.as_os_str().to_os_string(), + ]; + let (rc, tail) = run_extract_cmd(program, &args, Some(&out_path)); + if rc == 0 { + Ok(()) + } else { + Err(format!("{what} exit {rc}: {tail}")) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::Value; + use std::collections::BTreeSet; + + /// Write a real zip archive containing the given (path, content) files. + fn write_zip(path: &Path, files: &[(&str, Vec)]) { + let file = File::create(path).unwrap(); + let mut w = zip::ZipWriter::new(file); + for (name, data) in files { + w.start_file(*name, zip::write::SimpleFileOptions::default()) + .unwrap(); + w.write_all(data).unwrap(); + } + w.finish().unwrap(); + } + + /// Minimal CRC32 (IEEE) for the hand-built gzip container in tests. + fn crc32(data: &[u8]) -> u32 { + let mut table = [0u32; 256]; + for (i, slot) in table.iter_mut().enumerate() { + let mut c = i as u32; + for _ in 0..8 { + c = if c & 1 != 0 { + 0xEDB88320 ^ (c >> 1) + } else { + c >> 1 + }; + } + *slot = c; + } + let mut crc = 0xFFFF_FFFFu32; + for &b in data { + crc = table[((crc ^ b as u32) & 0xFF) as usize] ^ (crc >> 8); + } + crc ^ 0xFFFF_FFFF + } + + /// Valid gzip container wrapping a single *stored* (uncompressed) + /// deflate block — no compression crate needed to build a test fixture. + fn gzip_stored(data: &[u8]) -> Vec { + let mut v = vec![0x1f, 0x8b, 0x08, 0, 0, 0, 0, 0, 0, 0x03]; + v.push(0x01); // BFINAL=1, BTYPE=00 (stored) + let len = data.len() as u16; + v.extend_from_slice(&len.to_le_bytes()); + v.extend_from_slice(&(!len).to_le_bytes()); + v.extend_from_slice(data); + v.extend_from_slice(&crc32(data).to_le_bytes()); + v.extend_from_slice(&(data.len() as u32).to_le_bytes()); + v + } + + /// Minimal USTAR archive with one file entry, terminated by two zero + /// blocks — readable by the system tar. + fn make_ustar(name: &str, data: &[u8]) -> Vec { + let mut block = vec![0u8; 512]; + block[..name.len()].copy_from_slice(name.as_bytes()); + block[100..108].copy_from_slice(b"0000644\0"); // mode + block[108..116].copy_from_slice(b"0000000\0"); // uid + block[116..124].copy_from_slice(b"0000000\0"); // gid + let size_field = format!("{:011o}\0", data.len()); + block[124..136].copy_from_slice(size_field.as_bytes()); // size + block[136..148].copy_from_slice(b"00000000000\0"); // mtime + block[148..156].copy_from_slice(b" "); // chksum placeholder + block[156] = b'0'; // typeflag: regular file + block[257..263].copy_from_slice(b"ustar\0"); + block[263..265].copy_from_slice(b"00"); + let sum: u32 = block.iter().map(|&b| b as u32).sum(); + let chk = format!("{:06o}\0 ", sum); + block[148..156].copy_from_slice(chk.as_bytes()); + let mut out = block; + out.extend_from_slice(data); + while out.len() % 512 != 0 { + out.push(0); + } + out.extend(std::iter::repeat_n(0u8, 1024)); + out + } + + /// Base dir for test tempdirs: /tmp is often a small tmpfs (< + /// MIN_FREE_BYTES), which would trip the real free-space check; prefer + /// /var/tmp when it clears the threshold (same convention as the + /// python version's /var/tmp staging). + fn test_base() -> PathBuf { + let var_tmp = Path::new("/var/tmp"); + if cfg!(unix) + && free_bytes(var_tmp) + .map(|b| b >= MIN_FREE_BYTES) + .unwrap_or(false) + { + var_tmp.to_path_buf() + } else { + std::env::temp_dir() + } + } + + /// Build a source root (`src1`) + output root (`out`) inside a + /// container tempdir; returns (container, src, out). The source root's + /// file name (`src1`) is what the extractor uses as the index folder. + fn build_fixtures() -> (tempfile::TempDir, PathBuf, PathBuf) { + let _ = env_logger::builder() + .filter_level(log::LevelFilter::Info) + .is_test(true) + .try_init(); + let container = tempfile::tempdir_in(test_base()).unwrap(); + let src = container.path().join("src1"); + let out = container.path().join("out"); + fs::create_dir_all(&src).unwrap(); + fs::create_dir_all(&out).unwrap(); + let s = src.as_path(); + + // Pack.zip: files + a nested zip inside + let inner_path = s.join(".tmp_inner.zip"); + write_zip(&inner_path, &[("beta.txt", b"inner payload".to_vec())]); + let inner_bytes = fs::read(&inner_path).unwrap(); + fs::remove_file(&inner_path).unwrap(); + write_zip( + &s.join("Pack.zip"), + &[ + ("alpha.txt", b"alpha payload".to_vec()), + ("nested/inner.zip", inner_bytes), + ], + ); + + // a real .7z + let gamma = s.join("gamma.txt"); + fs::write(&gamma, b"sevenz payload").unwrap(); + let sub = s.join("Sub"); + fs::create_dir_all(&sub).unwrap(); + sevenz_rust2::compress_to_path(&gamma, sub.join("The Twin Lance.7z")).expect("compress 7z"); + fs::remove_file(&gamma).unwrap(); + + // a tar.gz (hand-built ustar + gzip-stored container) + let targz = gzip_stored(&make_ustar("delta.txt", b"tarred payload")); + fs::write(s.join("logs.tar.gz"), targz).unwrap(); + + // garbage zip -> must land in .broken + fs::write(s.join("bad.zip"), b"this is not a zip archive at all").unwrap(); + + // plain file (mirrored via hardlink) + fs::write(s.join("notes.txt"), b"plain notes").unwrap(); + + // run artifacts / hidden / partial: must be skipped + fs::write(s.join("state.json"), b"{}").unwrap(); + fs::write(s.join(".hidden.txt"), b"secret").unwrap(); + fs::write(s.join("partial.part"), b"partial").unwrap(); + + (container, src, out) + } + + fn inventory(root: &Path) -> BTreeSet { + walk_files_sorted(root) + .iter() + .map(|p| rel_to(root, p).to_string_lossy().into_owned()) + .collect() + } + + fn find_file_named(root: &Path, name: &str) -> Option { + walk_files_sorted(root).into_iter().find(|p| { + p.file_name() + .map(|n| n.to_string_lossy() == name) + .unwrap_or(false) + }) + } + + fn manifest_values(out: &Path) -> Vec { + let text = fs::read_to_string(out.join("extract_manifest.jsonl")).unwrap(); + text.lines() + .filter(|l| !l.trim().is_empty()) + .map(|l| serde_json::from_str(l).unwrap()) + .collect() + } + + #[test] + fn run_extracts_nests_quarantines_and_mirrors() { + let (_c, src, out) = build_fixtures(); + let extractor = Extractor { + out_root: out.clone(), + }; + extractor.run(&[src.clone()]).unwrap(); + + let s = src.as_path(); + let o = out.as_path(); + + // extracted content, including the nested zip inside Pack.zip + assert_eq!( + fs::read_to_string(o.join("src1/Pack/alpha.txt")).unwrap(), + "alpha payload" + ); + assert_eq!( + fs::read_to_string(o.join("src1/Pack/nested/inner/beta.txt")).unwrap(), + "inner payload" + ); + assert!( + !o.join("src1/Pack/nested/inner.zip").exists(), + "nested archive must be consumed" + ); + + // 7z and tar.gz + assert!( + o.join("src1/Sub/The Twin Lance").is_dir(), + "7z unit dir missing" + ); + assert!(find_file_named(&o.join("src1/Sub/The Twin Lance"), "gamma.txt").is_some()); + assert_eq!( + fs::read_to_string(o.join("src1/logs/delta.txt")).unwrap(), + "tarred payload" + ); + + // no archives left in the extracted tree (outside .broken) + for p in walk_files_sorted(o) { + let rel = rel_to(o, &p); + if rel.components().any(|c| c.as_os_str() == ".broken") { + continue; + } + assert!( + !is_archive(&file_name_string(&p)), + "archive left in extracted tree: {}", + p.display() + ); + } + + // garbage zip quarantined under .broken + let broken = o.join(".broken/bad.zip"); + assert!(broken.is_file(), "bad.zip not quarantined"); + assert_eq!( + fs::read(&broken).unwrap(), + b"this is not a zip archive at all" + ); + + // plain file mirrored: same inode on hardlink, else copy fallback + #[cfg(unix)] + { + use std::os::unix::fs::MetadataExt; + let sm = fs::metadata(s.join("notes.txt")).unwrap(); + let om = fs::metadata(o.join("src1/notes.txt")).unwrap(); + if sm.ino() == om.ino() { + assert_eq!(sm.dev(), om.dev(), "hardlink must be on the same fs"); + } else { + assert_eq!(sm.len(), om.len(), "hardlink fell back to copy"); + } + } + assert_eq!( + fs::read_to_string(o.join("src1/notes.txt")).unwrap(), + "plain notes" + ); + + // run artifacts / hidden / partial never enter the tree + for junk in ["state.json", ".hidden.txt", "partial.part"] { + assert!( + find_file_named(o, junk).is_none(), + "{junk} leaked into output" + ); + } + + // source root keeps its originals (except the quarantined bad.zip) + assert!(s.join("Pack.zip").is_file()); + assert!(s.join("notes.txt").is_file()); + assert!(s.join("Sub/The Twin Lance.7z").is_file()); + assert!( + !s.join("bad.zip").exists(), + "failed archive must be moved away" + ); + + // manifest: one line per extraction, exactly the 3 provenance keys + let lines = manifest_values(o); + let triples: BTreeSet<(String, String, String)> = lines + .iter() + .map(|v| { + let obj = v.as_object().unwrap(); + assert_eq!(obj.len(), 3, "unexpected keys in manifest line: {v}"); + ( + v["archive"].as_str().unwrap().to_string(), + v["dest"].as_str().unwrap().to_string(), + v["unit"].as_str().unwrap().to_string(), + ) + }) + .collect(); + let expected: BTreeSet<(String, String, String)> = [ + ("src1/Pack.zip", "src1/Pack", "Pack"), + ( + "src1/Pack/nested/inner.zip", + "src1/Pack/nested/inner", + "inner", + ), + ( + "src1/Sub/The Twin Lance.7z", + "src1/Sub/The Twin Lance", + "The Twin Lance", + ), + ("src1/logs.tar.gz", "src1/logs", "logs"), + ] + .iter() + .map(|(a, d, u)| (a.to_string(), d.to_string(), u.to_string())) + .collect(); + assert_eq!(triples, expected); + } + + #[test] + fn rerun_is_idempotent() { + let (_c, src, out) = build_fixtures(); + let extractor = Extractor { + out_root: out.clone(), + }; + extractor.run(&[src.clone()]).unwrap(); + + let manifest_before = fs::read_to_string(out.join("extract_manifest.jsonl")).unwrap(); + let inv_before = inventory(&out); + assert!(!manifest_before.is_empty()); + + extractor.run(&[src.clone()]).unwrap(); + + let manifest_after = fs::read_to_string(out.join("extract_manifest.jsonl")).unwrap(); + assert_eq!( + manifest_before, manifest_after, + "re-run must not append manifest lines" + ); + assert_eq!( + inv_before, + inventory(&out), + "re-run must not change the tree" + ); + assert!( + !out.as_path().join("src1/Pack_dup").exists() + && !out.as_path().join("src1/Pack_dup_dup").exists(), + "re-run must not create duplicate destinations" + ); + // the previously extracted content is still intact + assert_eq!( + fs::read_to_string(out.as_path().join("src1/Pack/alpha.txt")).unwrap(), + "alpha payload" + ); + } + + #[test] + fn dest_collision_appends_dup() { + let dir = tempfile::tempdir_in(test_base()).unwrap(); + let ex = Extractor { + out_root: dir.path().join("out"), + }; + fs::create_dir_all(&ex.out_root).unwrap(); + + // dest occupied by a FILE -> the *_dup sibling is used (python: + // mkdir exist_ok raises FileExistsError -> append "_dup") + let dest = dir.path().join("d1"); + fs::write(&dest, b"occupied").unwrap(); + let actual = ex.create_dest(&dest).unwrap(); + assert_eq!(actual, dest.with_file_name("d1_dup")); + assert!(actual.is_dir()); + + // dest and *_dup both occupied by files -> graceful failure after + // the 5 attempts (python's loop rebuilds "_dup" from the original + // name, so it keeps hitting the same path) + let dest2 = dir.path().join("d2"); + fs::write(&dest2, b"occupied").unwrap(); + fs::write(dir.path().join("d2_dup"), b"occupied too").unwrap(); + let err = ex.create_dest(&dest2).unwrap_err(); + assert!(err.contains("repeated collisions"), "{err}"); + + // parent path is a file (NotADirectoryError) -> immediate failure + let blocker = dir.path().join("blocker"); + fs::write(&blocker, b"file").unwrap(); + let err2 = ex.create_dest(&blocker.join("sub")).unwrap_err(); + assert!(err2.contains("cannot create destination"), "{err2}"); + } + + #[test] + fn suffix_helpers_match_python() { + assert_eq!(archive_suffixes("x.tar.gz"), vec![".gz", ".tar"]); + assert_eq!(display_unit("x.tar.gz"), "x"); + assert_eq!(archive_suffixes("The Twin Lance.7z"), vec![".7z"]); + assert_eq!(display_unit("The Twin Lance.7z"), "The Twin Lance"); + assert_eq!(archive_suffixes("X.ZIP"), vec![".zip"]); + assert_eq!(display_unit("X.ZIP"), "X"); + assert!(archive_suffixes("plain.txt").is_empty()); + assert!(!is_archive("plain.txt")); + assert!( + archive_suffixes(".hidden").is_empty(), + "dotfiles have no suffix" + ); + assert_eq!(display_unit("Pack.zip"), "Pack"); + assert!(is_archive("c.7z.7z")); + } + + #[test] + fn macosx_junk_detection() { + assert!(is_macosx_junk(Path::new("out/1/__MACOSX/._x.zip"))); + assert!(is_macosx_junk(Path::new("._x.zip"))); + assert!(is_macosx_junk(Path::new("deep/nested/._resource"))); + assert!(is_macosx_junk(Path::new("a/__MACOSX"))); + assert!(!is_macosx_junk(Path::new("a.zip"))); + assert!(!is_macosx_junk(Path::new("plain/notes.txt"))); + assert!( + !is_macosx_junk(Path::new("macosx/_a.zip")), + "__MACOSX matching is case-sensitive like python" + ); + } +} diff --git a/version-rust/src/gofile.rs b/version-rust/src/gofile.rs index 61e203c..d4a6646 100644 --- a/version-rust/src/gofile.rs +++ b/version-rust/src/gofile.rs @@ -1,2 +1,675 @@ //! GoFile API client (one guest account per run) — Rust port of //! `version-python/dlphelper/gofile.py`. +//! +//! The API interaction mirrors Lysagxra/GoFileDownloader (same headers, +//! website-token scheme and endpoints), with the same two deliberate +//! differences as the python version: +//! +//! 1. A single guest account is created per run (GoFileDownloader creates one +//! per URL, which trips Gofile's rate limit after ~6 requests). +//! 2. Failures return errors instead of exiting, so the orchestrator can +//! track them per file/URL and keep going. +//! +//! Retry semantics (see python AGENTS.md §6): `NotFound`, `Empty` and +//! `PasswordRequired` are never retried by the orchestrator; `AuthError` +//! allows one re-auth (`reset()` + `ensure_token()`). + +use std::fmt; +use std::fs; +use std::io::{Read, Write}; +use std::path::Path; +use std::sync::OnceLock; +use std::time::{Duration, SystemTime, UNIX_EPOCH}; + +use hex::encode as hex_encode; +use reqwest::blocking::Client; +use reqwest::header::{AUTHORIZATION, COOKIE, HeaderMap, HeaderName, HeaderValue, ORIGIN, REFERER}; +use sha2::{Digest, Sha256}; + +pub const API: &str = "https://api.gofile.io"; +pub const USER_AGENT: &str = "Mozilla/5.0"; +pub const LOCALE: &str = "en-US"; +/// Public website-token seed, see GoFileDownloader. Fine to hardcode; the +/// resulting tokens must never be logged (AGENTS.md §6). +pub const TOKEN_SECRET: &str = "12af056dacea0b"; +pub const TOKEN_WINDOW_SEC: u64 = 14400; +pub const CHUNK_SIZE: usize = 1024 * 1024; + +const COMMON_HEADER_PAIRS: &[(&str, &str)] = &[ + ("User-Agent", USER_AGENT), + ("Accept-Encoding", "gzip"), + ("Accept", "*/*"), + ("Connection", "keep-alive"), +]; + +const EXTENDED_HEADER_PAIRS: &[(&str, &str)] = &[ + ("User-Agent", USER_AGENT), + ("Accept-Encoding", "gzip"), + ("Accept", "*/*"), + ("Connection", "keep-alive"), + ("Sec-Fetch-Dest", "empty"), + ("Sec-Fetch-Mode", "cors"), + ("Sec-Fetch-Site", "same-site"), + ("Pragma", "no-cache"), + ("Cache-Control", "no-cache"), +]; + +/// Common headers sent to every endpoint (python `COMMON_HEADERS`). +fn common_headers() -> HeaderMap { + static M: OnceLock = OnceLock::new(); + M.get_or_init(|| build_header_map(COMMON_HEADER_PAIRS)) + .clone() +} + +/// Common + fetch/cache-control headers (python `EXTENDED_HEADERS`). +fn extended_headers() -> HeaderMap { + static M: OnceLock = OnceLock::new(); + M.get_or_init(|| build_header_map(EXTENDED_HEADER_PAIRS)) + .clone() +} + +fn build_header_map(pairs: &'static [(&'static str, &'static str)]) -> HeaderMap { + let mut map = HeaderMap::new(); + for (name, value) in pairs { + let name: HeaderName = name + .parse() + .unwrap_or_else(|_| panic!("static header name {name:?} is invalid")); + let value = HeaderValue::from_static(value); + map.insert(name, value); + } + map +} + +/// Insert a dynamically computed header. Unlike python (which would raise on +/// an invalid value and let the orchestrator's catch-all handle it), a parse +/// failure here logs a warning and skips the header — the request then fails +/// server-side and is recorded, and the run keeps going. +fn set_header(map: &mut HeaderMap, name: HeaderName, value: &str) { + match HeaderValue::from_str(value) { + Ok(v) => { + map.insert(name, v); + } + Err(_) => { + log::warn!("skipping invalid {} header value", name.as_str()); + } + } +} + +/// Error type mirroring the python exception classes. The orchestrator maps +/// variants to retry policies: `NotFound`/`Empty`/`PasswordRequired` are +/// final states, `AuthError` permits one re-auth, `Error` is retryable. +#[allow(dead_code)] // variants reserved for the orchestrator (e.g. Empty) +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum GoFileError { + /// python `GoFileError` (base): transient/unclassified failure. + Error(String), + /// python `GoFilePasswordRequired`. + PasswordRequired(String), + /// python `GoFileNotFound`: dead link, never retry. + NotFound(String), + /// python `GoFileEmpty`: content exists but contains no files. + Empty(String), + /// python `GoFileAuthError`: token rejected, one re-auth allowed. + AuthError(String), +} + +impl fmt::Display for GoFileError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + GoFileError::Error(m) + | GoFileError::PasswordRequired(m) + | GoFileError::NotFound(m) + | GoFileError::Empty(m) + | GoFileError::AuthError(m) => write!(f, "{m}"), + } + } +} + +impl std::error::Error for GoFileError {} + +/// Extract the content id from a gofile.io/d/ URL. +/// +/// Matches the lenient python `urlparse` behavior: a URL without a scheme +/// still works (`"gofile.io/d/x"` → `"x"`); only the path is inspected. +pub fn content_id_from_url(url: &str) -> Result { + let path = match url::Url::parse(url) { + Ok(u) => u.path().to_string(), + // python urlparse is lenient about a missing scheme but still strips + // query/fragment; mirror that ("gofile.io/d/x?y=1" → path as parsed). + Err(_) => url.split(['?', '#']).next().unwrap_or(url).to_string(), + }; + let parts: Vec<&str> = path.split('/').filter(|p| !p.is_empty()).collect(); + if parts.len() >= 2 && parts[parts.len() - 2] == "d" { + return Ok(parts[parts.len() - 1].to_string()); + } + Err(GoFileError::Error(format!( + "not a gofile content URL (expected gofile.io/d/): {url}" + ))) +} + +/// sha256 over `bytes` as lowercase hex (python `hashlib.sha256(...).hexdigest()`). +fn sha256_hex(bytes: &[u8]) -> String { + hex_encode(Sha256::digest(bytes)) +} + +/// Truncate to the first 200 chars (python `text[:200]` counts codepoints). +fn truncate_200(s: &str) -> String { + s.chars().take(200).collect() +} + +/// GoFile API client that reuses one guest account token for the whole run. +pub struct GoFileClient { + password_hash: Option, + token: Option, + session: Client, +} + +impl GoFileClient { + /// `password` is hashed (sha256 hex) once; the API wants the hash. + pub fn new(password: Option<&str>) -> Self { + let password_hash = password.map(|p| sha256_hex(p.as_bytes())); + // Client-level timeouts mirror python's download timeout tuple + // (connect 10s); the read-side 60s from python's (10, 60) is not + // expressible in reqwest blocking, so the download request gets a + // generous total timeout instead (see download_file). + let session = Client::builder() + .default_headers(common_headers()) + .connect_timeout(Duration::from_secs(10)) + .build() + .unwrap_or_else(|e| panic!("failed to build HTTP client: {e}")); + Self { + password_hash, + token: None, + session, + } + } + + /// Create a guest account; backs off and retries on rate limiting. + /// Defaults match python: 8 attempts, 20s × attempt between them. + pub fn ensure_token(&mut self) -> Result { + self.ensure_token_with(8, 20) + } + + /// Same with explicit attempt/wait parameters (python keyword args). + pub fn ensure_token_with( + &mut self, + max_attempts: u32, + wait_seconds: u64, + ) -> Result { + if let Some(token) = &self.token { + return Ok(token.clone()); + } + + let mut last_status = "unknown".to_string(); + for attempt in 1..=max_attempts { + match self.try_create_account() { + Ok(token) => { + self.token = Some(token.clone()); + log::info!("GoFile guest account created (token reused for whole run)"); + return Ok(token); + } + Err(status) => { + last_status = status; + log::warn!( + "GoFile account creation attempt {}/{} failed: {}", + attempt, + max_attempts, + last_status + ); + } + } + if attempt < max_attempts { + std::thread::sleep(Duration::from_secs(wait_seconds * u64::from(attempt))); + } + } + Err(GoFileError::Error(format!( + "GoFile account creation failed after {max_attempts} attempts (last: {last_status})" + ))) + } + + /// POST /accounts with the common headers; returns the token or the + /// status string that describes the failure (python's `last_status`). + fn try_create_account(&self) -> Result { + let resp = self + .session + .post(format!("{API}/accounts")) + .headers(common_headers()) + .timeout(Duration::from_secs(15)) + .send(); + let data: serde_json::Value = match resp { + Ok(r) => match r.json() { + Ok(v) => v, + Err(e) => return Err(format!("request error: {e}")), + }, + Err(e) => return Err(format!("request error: {e}")), + }; + if data.get("status").and_then(|s| s.as_str()) == Some("ok") { + return data + .pointer("/data/token") + .and_then(|t| t.as_str()) + .map(str::to_string) + .ok_or_else(|| "request error: missing data.token".to_string()); + } + // python: str(data.get("status", data)[:200] if data else data) + let status = match data.get("status") { + Some(serde_json::Value::String(s)) => s.clone(), + Some(other) => other.to_string(), + None => serde_json::to_string(&data).unwrap_or_default(), + }; + Err(truncate_200(&status)) + } + + /// Drop the current token so the next `ensure_token()` creates a new one. + pub fn reset(&mut self) { + self.token = None; + } + + /// sha256 over `"{UA}::{LOCALE}::{token}::{window}::{TOKEN_SECRET}"`. + /// `window` is an argument so tests can pin it. + fn website_token_for(&self, window: u64) -> String { + // python f-string renders a missing token as "None". + let token = self.token.as_deref().unwrap_or("None"); + let seed = format!("{USER_AGENT}::{LOCALE}::{token}::{window}::{TOKEN_SECRET}"); + sha256_hex(seed.as_bytes()) + } + + /// Website token for the current 4h window (python `_website_token`). + fn website_token(&self) -> String { + let window = SystemTime::now() + .duration_since(UNIX_EPOCH) + .map(|d| d.as_secs()) + .unwrap_or(0) + / TOKEN_WINDOW_SEC; + self.website_token_for(window) + } + + /// EXTENDED_HEADERS + Authorization, X-Website-Token, X-BL. + fn auth_headers(&self) -> HeaderMap { + let mut headers = extended_headers(); + let bearer = format!("Bearer {}", self.token.as_deref().unwrap_or("None")); + set_header(&mut headers, AUTHORIZATION, &bearer); + set_header( + &mut headers, + header_name("x-website-token"), + &self.website_token(), + ); + set_header(&mut headers, header_name("x-bl"), LOCALE); + headers + } + + /// Return (relpath, filename, download_link) for every file under a URL. + /// + /// Direct files at the content root get their own folder named after the + /// content id (empty relpath → cid, like the python normalization). + pub fn list_files(&self, url: &str) -> Result, GoFileError> { + let cid = content_id_from_url(url)?; + let mut results = Vec::new(); + self.walk(&cid, &[], &mut results)?; + Ok(results + .into_iter() + .map(|(relpath, name, link)| { + let relpath = if relpath.is_empty() { + cid.clone() + } else { + relpath + }; + (relpath, name, link) + }) + .collect()) + } + + /// Recursive walk of /contents/, mirroring python `_walk`. + fn walk( + &self, + cid: &str, + prefix: &[String], + results: &mut Vec<(String, String, String)>, + ) -> Result<(), GoFileError> { + let mut query = String::from("?cache=true&sortField=createTime&sortDirection=1"); + if let Some(password_hash) = &self.password_hash { + query.push_str("&password="); + query.push_str(password_hash); + } + let resp = self + .session + .get(format!("{API}/contents/{cid}{query}")) + .headers(self.auth_headers()) + .timeout(Duration::from_secs(30)) + .send() + .map_err(|e| GoFileError::Error(format!("content request for {cid} failed: {e}")))?; + + if resp.status() == reqwest::StatusCode::NOT_FOUND { + return Err(GoFileError::NotFound(format!( + "content {cid} not found (dead link)" + ))); + } + let status_code = resp.status(); + if status_code == reqwest::StatusCode::UNAUTHORIZED + || status_code == reqwest::StatusCode::FORBIDDEN + { + return Err(GoFileError::AuthError(format!( + "content request denied ({status_code}) for {cid}" + ))); + } + let text = resp + .text() + .map_err(|e| GoFileError::Error(format!("content request for {cid} failed: {e}")))?; + let data: serde_json::Value = serde_json::from_str(&text).map_err(|_| { + GoFileError::Error(format!( + "non-JSON response for {cid}: {}", + truncate_200(&text) + )) + })?; + + // python: data.get("status") == "notFound" or "notFound" in str(status) + let status = json_status(&data); + if status.contains("notFound") { + return Err(GoFileError::NotFound(format!( + "content {cid} does not exist anymore (dead link)" + ))); + } + if status != "ok" { + return Err(GoFileError::Error(format!( + "content request failed for {cid}: {status}" + ))); + } + + let d = match data.get("data") { + Some(d @ serde_json::Value::Object(_)) => d, + _ => { + return Err(GoFileError::Error(format!( + "content request failed for {cid}: missing data object" + ))); + } + }; + + let password_ok = match d.get("passwordStatus") { + None | Some(serde_json::Value::Null) => true, + Some(serde_json::Value::String(s)) => s == "passwordOk", + Some(_) => false, + }; + if !password_ok { + return Err(GoFileError::PasswordRequired(format!( + "content {cid} is password protected (use --password)" + ))); + } + + if d.get("type").and_then(|t| t.as_str()) == Some("folder") { + let name = required_str(d, "name", cid)?; + let mut here = prefix.to_vec(); + here.push(name); + // serde_json maps children to a BTreeMap, so children are visited + // in key-sorted order instead of the API's insertion order (the + // orchestrator downloads all of them either way). + let children = d.get("children").and_then(|c| c.as_object()); + if let Some(children) = children { + for (_key, child) in children { + if child.get("type").and_then(|t| t.as_str()) == Some("folder") { + let child_id = required_str(child, "id", cid)?; + self.walk(&child_id, &here, results)?; + } else { + let child_name = required_str(child, "name", cid)?; + let link = required_str(child, "link", cid)?; + results.push((here.join("/"), child_name, link)); + } + } + } + } else { + let name = required_str(d, "name", cid)?; + let link = required_str(d, "link", cid)?; + results.push((prefix.join("/"), name, link)); + } + Ok(()) + } + + /// Download one file; returns the file size. Skips existing files. + /// + /// Deviation from python's `timeout=(10, 60)`: reqwest blocking cannot + /// express "connect 10s / read 60s" per request. Connect is capped at + /// 10s client-wide; the download request gets a generous 2h total + /// timeout instead of python's uncapped streaming (60s per read) — + /// a stalled connection now aborts after at most 2h instead of 60s. + pub fn download_file(&self, link: &str, dest: &Path) -> Result { + if let Some(parent) = dest.parent() { + fs::create_dir_all(parent).map_err(|e| { + GoFileError::Error(format!("cannot create {}: {e}", parent.display())) + })?; + } + if dest.exists() { + let size = fs::metadata(dest).map(|m| m.len()).unwrap_or(0); + if size > 0 { + log::info!("Skip existing: {}", dest.display()); + return Ok(size); + } + } + + let name = dest + .file_name() + .map(|n| n.to_string_lossy().into_owned()) + .unwrap_or_default(); + let tmp = dest.with_file_name(format!("{name}.part")); + + let result = self.download_to_tmp(link, dest, &tmp, &name); + // python `finally`: never leave a .part file behind on failure. + if tmp.exists() && !dest.exists() { + let _ = fs::remove_file(&tmp); + } + result + } + + fn download_to_tmp( + &self, + link: &str, + dest: &Path, + tmp: &Path, + name: &str, + ) -> Result { + let mut headers = extended_headers(); + set_header( + &mut headers, + COOKIE, + &format!("accountToken={}", self.token.as_deref().unwrap_or("None")), + ); + set_header(&mut headers, REFERER, link); + set_header(&mut headers, ORIGIN, link); + + let mut resp = self + .session + .get(link) + .headers(headers) + // python: timeout=(10, 60) — per-read 60s is not expressible in + // reqwest blocking, so use a generous total timeout instead. + .timeout(Duration::from_secs(2 * 3600)) + .send() + .map_err(|e| GoFileError::Error(format!("download of {name} failed: {e}")))?; + + let status = resp.status(); + if status == reqwest::StatusCode::NOT_FOUND { + return Err(GoFileError::NotFound(format!( + "file {name} not found (dead link)" + ))); + } + if status != reqwest::StatusCode::OK { + return Err(GoFileError::Error(format!( + "HTTP {status} while downloading {name}" + ))); + } + + let mut file = fs::File::create(tmp) + .map_err(|e| GoFileError::Error(format!("cannot create {}: {e}", tmp.display())))?; + let mut written = 0u64; + let mut buf = vec![0u8; CHUNK_SIZE]; + loop { + let n = resp + .read(&mut buf) + .map_err(|e| GoFileError::Error(format!("download of {name} failed: {e}")))?; + if n == 0 { + break; + } + file.write_all(&buf[..n]) + .map_err(|e| GoFileError::Error(format!("cannot write {name}: {e}")))?; + written += n as u64; + } + file.flush() + .map_err(|e| GoFileError::Error(format!("cannot write {name}: {e}")))?; + + if written == 0 { + return Err(GoFileError::Error(format!("empty body for {name}"))); + } + // os.replace semantics: rename over an existing destination. + fs::rename(tmp, dest) + .map_err(|e| GoFileError::Error(format!("cannot move {name} into place: {e}")))?; + Ok(written) + } +} + +/// python `str(data.get("status", ""))` — missing status renders as "None". +fn json_status(data: &serde_json::Value) -> String { + match data.get("status") { + None => "None".to_string(), + Some(serde_json::Value::String(s)) => s.clone(), + Some(other) => other.to_string(), + } +} + +/// Fetch a required string field; a missing field mirrors python's KeyError +/// (which would surface to the orchestrator as an unexpected failure). +fn required_str(value: &serde_json::Value, key: &str, cid: &str) -> Result { + value + .get(key) + .and_then(|v| v.as_str()) + .map(str::to_string) + .ok_or_else(|| { + GoFileError::Error(format!("content {cid}: malformed entry (missing {key})")) + }) +} + +fn header_name(name: &'static str) -> HeaderName { + HeaderName::from_static(name) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn content_id_from_url_accepts_gofile_d_links() { + assert_eq!( + content_id_from_url("https://gofile.io/d/ABC123").unwrap(), + "ABC123" + ); + // query string, fragment, trailing slash, deeper path + assert_eq!( + content_id_from_url("https://gofile.io/d/ABC123?x=1").unwrap(), + "ABC123" + ); + assert_eq!( + content_id_from_url("https://gofile.io/d/ABC123/#anchor").unwrap(), + "ABC123" + ); + // python urlparse is lenient about a missing scheme + assert_eq!(content_id_from_url("gofile.io/d/ABC123").unwrap(), "ABC123"); + assert_eq!( + content_id_from_url("gofile.io/d/ABC123?x=1").unwrap(), + "ABC123" + ); + } + + #[test] + fn content_id_from_url_rejects_non_content_urls() { + let err = content_id_from_url("https://gofile.io/ABC123").unwrap_err(); + assert!( + matches!(err, GoFileError::Error(ref m) + if m == "not a gofile content URL (expected gofile.io/d/): https://gofile.io/ABC123"), + "unexpected: {err}" + ); + assert!(content_id_from_url("https://example.com/foo/bar").is_err()); + assert!(content_id_from_url("https://gofile.io/d/").is_err()); + } + + #[test] + fn website_token_for_is_deterministic_and_matches_python() { + let mut client = GoFileClient::new(None); + client.token = Some("testtoken".to_string()); + // golden value computed with python hashlib for the same seed + assert_eq!( + client.website_token_for(0), + "1791d396eab9b0ef100bcb3c9211c80973ee8dd9b143fc9f388ced6a0fb3b0c0" + ); + // stable across calls, 64 lowercase hex chars + let again = client.website_token_for(0); + assert_eq!(client.website_token_for(0), again); + assert_eq!(again.len(), 64); + assert!( + again + .bytes() + .all(|b| b.is_ascii_hexdigit() && !b.is_ascii_uppercase()) + ); + // different window / token → different token + assert_ne!(client.website_token_for(1), client.website_token_for(0)); + client.token = None; + // python renders a missing token as "None" + assert_eq!( + client.website_token_for(7), + "fb9a698ec598396eb77ab8b562fb1e30f01401dcf793701b290585e6393a657f" + ); + } + + #[test] + fn header_maps_match_python_keys() { + let common = common_headers(); + let mut keys: Vec = common.keys().map(|k| k.as_str().to_string()).collect(); + keys.sort(); + assert_eq!( + keys, + ["accept", "accept-encoding", "connection", "user-agent"] + ); + assert_eq!(common.get("Accept-Encoding").unwrap(), "gzip"); + + let extended = extended_headers(); + let mut ekeys: Vec = extended.keys().map(|k| k.as_str().to_string()).collect(); + ekeys.sort(); + assert_eq!( + ekeys, + [ + "accept", + "accept-encoding", + "cache-control", + "connection", + "pragma", + "sec-fetch-dest", + "sec-fetch-mode", + "sec-fetch-site", + "user-agent", + ] + ); + assert_eq!(extended.get("Sec-Fetch-Site").unwrap(), "same-site"); + } + + #[test] + fn error_display_is_the_message() { + assert_eq!( + GoFileError::NotFound("content x not found (dead link)".into()).to_string(), + "content x not found (dead link)" + ); + assert_eq!(GoFileError::PasswordRequired("pw".into()).to_string(), "pw"); + assert_eq!(GoFileError::Empty("empty".into()).to_string(), "empty"); + assert_eq!(GoFileError::AuthError("auth".into()).to_string(), "auth"); + assert_eq!(GoFileError::Error("base".into()).to_string(), "base"); + } + + #[test] + fn sha256_hex_matches_python_hexdigest() { + // hashlib.sha256(b"abc").hexdigest() + assert_eq!( + sha256_hex(b"abc"), + "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad" + ); + } + + #[test] + fn truncate_200_counts_chars_not_bytes() { + let s = "ä".repeat(300); + assert_eq!(truncate_200(&s).chars().count(), 200); + } +} diff --git a/version-rust/src/zero807.rs b/version-rust/src/zero807.rs index 83047e3..15acdfe 100644 --- a/version-rust/src/zero807.rs +++ b/version-rust/src/zero807.rs @@ -1,2 +1,227 @@ //! curl-based downloader for 0807.st direct links — Rust port of //! `version-python/dlphelper/zero807.py`. +//! +//! 0807.st serves the files directly but rejects requests that don't look +//! like a browser (requires a User-Agent and the Referer of the site page), +//! and it rejects multi-connection/range abuse patterns differently than +//! curl — which is why curl is used instead of aria2c here (see python +//! module docstring: curl worked, aria2c's default request got 403). +//! +//! Downloads go to a `.part` file first and are renamed on success, so an +//! interrupted download is resumable (`--continue-at -`) and partial files +//! are never mistaken for finished ones. + +use std::fs; +use std::io::Read; +use std::path::{Path, PathBuf}; +use std::process::Command; + +pub const USER_AGENT: &str = "Mozilla/5.0"; +pub const REFERER: &str = "https://0807.st/"; + +/// Error type mirroring the python exception classes. `NotFound` (dead link, +/// HTTP 404/410) is a final state the orchestrator never retries. +#[allow(dead_code)] // used by the orchestrator, not yet wired up +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum Zero807Error { + /// python `Zero807Error` (base): transient/unclassified failure. + Error(String), + /// python `Zero807NotFound`: dead link, never retry. + NotFound(String), +} + +impl std::fmt::Display for Zero807Error { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Zero807Error::Error(m) | Zero807Error::NotFound(m) => write!(f, "{m}"), + } + } +} + +impl std::error::Error for Zero807Error {} + +/// Derive the filename from the URL path (e.g. /6saCNie.zip -> 6saCNie.zip). +/// +/// Matches python `Path(urlparse(url).path).name`: the query string is not +/// part of the path, a trailing slash still yields the last real segment +/// (`.../a/b/` → `b`), and an empty result is an error. +pub fn target_name(url: &str) -> Result { + let path = match url::Url::parse(url) { + Ok(u) => u.path().to_string(), + // python urlparse is lenient about a missing scheme but still strips + // query/fragment; mirror that. + Err(_) => url.split(['?', '#']).next().unwrap_or(url).to_string(), + }; + let name = Path::new(&path) + .file_name() + .map(|n| n.to_string_lossy().into_owned()) + .unwrap_or_default(); + if name.is_empty() { + return Err(Zero807Error::Error(format!( + "cannot derive a filename from URL: {url}" + ))); + } + Ok(name) +} + +/// Download one URL with curl; returns (final_path, size_bytes). +/// +/// Defaults match python: `timeout_seconds = 7200`, `resume = true`. +/// Use `download_with` for explicit parameters. +pub fn download(url: &str, out_dir: &Path) -> Result<(PathBuf, u64), Zero807Error> { + download_with(url, out_dir, 7200, true) +} + +/// Same with explicit timeout/resume parameters (python keyword args). +pub fn download_with( + url: &str, + out_dir: &Path, + timeout_seconds: u64, + resume: bool, +) -> Result<(PathBuf, u64), Zero807Error> { + let name = target_name(url)?; + fs::create_dir_all(out_dir) + .map_err(|e| Zero807Error::Error(format!("cannot create {}: {e}", out_dir.display())))?; + let dest = out_dir.join(&name); + let tmp = dest.with_file_name(format!("{name}.part")); + + if dest.exists() { + let size = fs::metadata(&dest).map(|m| m.len()).unwrap_or(0); + if size > 0 { + log::info!("Skip existing: {}", dest.display()); + return Ok((dest, size)); + } + } + + let mut cmd = Command::new("curl"); + cmd.args([ + "-fSL", + "--connect-timeout", + "20", + "--max-time", + &timeout_seconds.to_string(), + "--retry", + "2", + "--retry-delay", + "5", + "-A", + USER_AGENT, + "-e", + REFERER, + "-o", + ]); + cmd.arg(&tmp); + if resume { + cmd.arg("--continue-at").arg("-"); + } + cmd.arg(url); + + log::info!("Downloading {} -> {}", url, name); + let proc = cmd.output().map_err(|e| { + // python: subprocess raises FileNotFoundError, which the + // orchestrator's catch-all would record; wrap it here. + Zero807Error::Error(format!("failed to run curl: {e}")) + })?; + + if proc.status.code() != Some(0) { + let returncode = proc.status.code().unwrap_or(-1); + let stderr = String::from_utf8_lossy(&proc.stderr); + let tail: Vec<&str> = stderr + .trim() + .lines() + .filter(|line| !line.is_empty()) + .collect(); + let start = tail.len().saturating_sub(3); + let stderr = tail[start..].join(" | "); + // -f makes curl exit 22 on any HTTP error; dead links fail for good + if stderr.contains("404") || stderr.contains("410") { + return Err(Zero807Error::NotFound(format!( + "dead link (HTTP 404/410): {stderr}" + ))); + } + return Err(Zero807Error::Error(format!( + "curl exit code {returncode}: {stderr}" + ))); + } + + let size = fs::metadata(&tmp).map(|m| m.len()).unwrap_or(0); + if size == 0 { + let _ = fs::remove_file(&tmp); // python unlink(missing_ok=True) + return Err(Zero807Error::Error(format!("empty file for {name}"))); + } + if dest + .extension() + .map(|e| e.to_string_lossy().to_lowercase()) + .as_deref() + == Some("zip") + { + // python reads the whole file just to peek at 2 bytes; reading only + // the first two bytes is behavior-identical and cheap. + let mut first = [0u8; 2]; + let n = fs::File::open(&tmp) + .and_then(|mut fh| fh.read(&mut first)) + .unwrap_or(0); + if first[..n] != *b"PK" { + log::warn!("{name} does not look like a ZIP file (first bytes are not PK)"); + } + } + // os.replace semantics: rename over an existing destination. + fs::rename(&tmp, &dest) + .map_err(|e| Zero807Error::Error(format!("cannot move {name} into place: {e}")))?; + Ok((dest, size)) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn target_name_from_path_last_segment() { + assert_eq!( + target_name("https://0807.st/6saCNie.zip").unwrap(), + "6saCNie.zip" + ); + assert_eq!( + target_name("https://0807.st/a/b/filename.bin").unwrap(), + "filename.bin" + ); + } + + #[test] + fn target_name_drops_query_and_fragment() { + // python takes Path(urlparse(url).path).name: query/fragment dropped + assert_eq!( + target_name("https://0807.st/file.zip?x=1").unwrap(), + "file.zip" + ); + assert_eq!( + target_name("https://0807.st/file.zip?x=1#frag").unwrap(), + "file.zip" + ); + // schemeless URLs parse like python urlparse (query still stripped) + assert_eq!(target_name("0807.st/file.zip?x=1").unwrap(), "file.zip"); + } + + #[test] + fn target_name_trailing_slash_matches_pathlib() { + // pathlib: Path("/a/b/").name == "b" (trailing slash ignored) + assert_eq!(target_name("https://0807.st/a/b/").unwrap(), "b"); + assert_eq!( + target_name("https://0807.st/").unwrap_err().to_string(), + "cannot derive a filename from URL: https://0807.st/" + ); + assert_eq!( + target_name("https://0807.st").unwrap_err().to_string(), + "cannot derive a filename from URL: https://0807.st" + ); + } + + #[test] + fn error_display_is_the_message() { + assert_eq!( + Zero807Error::NotFound("dead link (HTTP 404/410): 404".into()).to_string(), + "dead link (HTTP 404/410): 404" + ); + assert_eq!(Zero807Error::Error("boom".into()).to_string(), "boom"); + } +}