From 3b3c4b57ea3394e0c1284abface2639c2c9bd8f3 Mon Sep 17 00:00:00 2001 From: Ben_Kosytorz Date: Sun, 11 Oct 2026 07:11:49 +0200 Subject: [PATCH] feat(rust): expand 0807.st folder pages (/f/) into their files --- version-rust/src/orchestrate.rs | 62 ++++--- version-rust/src/zero807.rs | 304 +++++++++++++++++++++++++++++++- 2 files changed, 339 insertions(+), 27 deletions(-) diff --git a/version-rust/src/orchestrate.rs b/version-rust/src/orchestrate.rs index 9cb5feb..1075b80 100644 --- a/version-rust/src/orchestrate.rs +++ b/version-rust/src/orchestrate.rs @@ -824,6 +824,37 @@ fn json_entry(dest: &Path, size: u64, name: &str) -> Value { }) } +/// Download one 0807 URL: a folder page (`/f/`) yields many files, a +/// plain link exactly one. Returns the manifest entries and the total size. +fn download_0807( + url: &str, + out_dir: &Path, + opts: &RunOpts, +) -> Result<(Vec, u64), Zero807Error> { + let files: Vec<(PathBuf, u64)> = if zero807::is_folder_page(url) { + zero807::download_folder(url, out_dir, 7200, true)?.1 + } else { + vec![zero807::download(url, out_dir)?] + }; + let mut entries = Vec::with_capacity(files.len()); + let mut total = 0u64; + for (dest, size) in &files { + let name = dest + .file_name() + .map(|n| n.to_string_lossy().into_owned()) + .unwrap_or_default(); + let mut entry = json_entry(dest, *size, &name); + if opts.hash { + let h = + sha256_of(dest).map_err(|e| Zero807Error::Error(format!("hash failed: {e}")))?; + entry["sha256"] = Value::String(h); + } + entries.push(entry); + total += *size; + } + Ok((entries, total)) +} + /// 0807 pipeline (python run_0807). pub fn run_0807( urls: &[FoundUrl], @@ -850,36 +881,25 @@ pub fn run_0807( let source = source_by_url.get(&f.url).cloned().unwrap_or_default(); let mut resolved = false; for attempt in 1..=opts.retries { - let downloaded = zero807::download(&f.url, &out_dir) - .map_err(anyhow::Error::new) - .and_then(|(dest, size)| -> anyhow::Result<(Value, u64)> { - let mut entry = json_entry( - &dest, - size, - &dest - .file_name() - .map(|n| n.to_string_lossy().into_owned()) - .unwrap_or_default(), - ); - if opts.hash { - let h = sha256_of(&dest)?; - entry["sha256"] = Value::String(h); - } - Ok((entry, size)) - }); + let downloaded = download_0807(&f.url, &out_dir, opts).map_err(anyhow::Error::new); match downloaded { - Ok((entry, size)) => { - log::info!("OK {} ({})", f.url, format_size(size)); + Ok((entries, size)) => { + log::info!( + "OK {} ({}, {} file(s))", + f.url, + format_size(size), + entries.len() + ); record_files( state, gstore.as_deref_mut(), &f.url, - std::slice::from_ref(&entry), + &entries, &source, manifest, )?; if opts.extract { - extract_zips(std::slice::from_ref(&entry)); + extract_zips(&entries); } consecutive_failures = 0; resolved = true; diff --git a/version-rust/src/zero807.rs b/version-rust/src/zero807.rs index 15acdfe..977c94f 100644 --- a/version-rust/src/zero807.rs +++ b/version-rust/src/zero807.rs @@ -80,16 +80,35 @@ pub fn download_with( resume: bool, ) -> Result<(PathBuf, u64), Zero807Error> { let name = target_name(url)?; - fs::create_dir_all(out_dir) - .map_err(|e| Zero807Error::Error(format!("cannot create {}: {e}", out_dir.display())))?; let dest = out_dir.join(&name); + let size = download_to(url, &dest, timeout_seconds, resume)?; + Ok((dest, size)) +} + +/// Download `url` to an explicit `dest` (`.part` + rename, resume, skip if +/// already present). `download_with` derives `dest` from the URL; the folder +/// downloader derives it from the page's file name. +pub fn download_to( + url: &str, + dest: &Path, + timeout_seconds: u64, + resume: bool, +) -> Result { + let name = dest + .file_name() + .map(|n| n.to_string_lossy().into_owned()) + .unwrap_or_default(); + if let Some(parent) = dest.parent() { + fs::create_dir_all(parent) + .map_err(|e| Zero807Error::Error(format!("cannot create {}: {e}", parent.display())))?; + } let tmp = dest.with_file_name(format!("{name}.part")); if dest.exists() { - let size = fs::metadata(&dest).map(|m| m.len()).unwrap_or(0); + let size = fs::metadata(dest).map(|m| m.len()).unwrap_or(0); if size > 0 { log::info!("Skip existing: {}", dest.display()); - return Ok((dest, size)); + return Ok(size); } } @@ -166,9 +185,226 @@ pub fn download_with( } } // os.replace semantics: rename over an existing destination. - fs::rename(&tmp, &dest) + fs::rename(&tmp, dest) .map_err(|e| Zero807Error::Error(format!("cannot move {name} into place: {e}")))?; - Ok((dest, size)) + Ok(size) +} + +/// A file listed on a 0807.st folder page (`/f/`). +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct FolderEntry { + /// file id used in the direct-download URL (`https://0807.st/?dl=1`) + pub id: String, + /// display file name from the page (``) + pub name: String, + pub size: u64, + pub mime: String, +} + +/// The `` of a 0807.st folder page URL (`/f/`), if it is one. +/// A direct file link (`/x.zip`) or the folder export (`/f/.zip`) is not. +pub fn folder_page_id(url: &str) -> Option { + let path = match url::Url::parse(url) { + Ok(u) => u.path().to_string(), + Err(_) => url.split(['?', '#']).next().unwrap_or(url).to_string(), + }; + let rest = path.trim_end_matches('/').strip_prefix("/f/")?; + if rest.is_empty() || rest.contains('/') || rest.contains('.') { + return None; + } + Some(rest.to_string()) +} + +/// True when `url` points at a 0807.st folder page. +pub fn is_folder_page(url: &str) -> bool { + folder_page_id(url).is_some() +} + +/// Parse the file list from a 0807.st folder page. Each entry is an +/// `` +/// whose name sits in `...`. Pure string scanning +/// (no HTML-parser dependency). +pub fn parse_folder_page(html: &str) -> Vec { + const MARK: &str = "").unwrap_or(rest.len()); + let seg = &rest[..end]; + if let Some(id) = attr(seg, "data-id") { + let size = attr(seg, "data-size") + .and_then(|s| s.parse::().ok()) + .unwrap_or(0); + let mime = attr(seg, "data-mime").unwrap_or_default(); + let name = span_text(seg, "alname").unwrap_or_else(|| id.clone()); + out.push(FolderEntry { + id, + name, + size, + mime, + }); + } + rest = &rest[end..]; + } + out +} + +/// Value of `key="..."` inside a tag fragment. +fn attr(seg: &str, key: &str) -> Option { + let needle = format!("{key}=\""); + let start = seg.find(&needle)? + needle.len(); + let end = seg[start..].find('"')? + start; + Some(seg[start..end].to_string()) +} + +/// Text of `...` (HTML entities decoded). +fn span_text(seg: &str, class: &str) -> Option { + let needle = format!(""); + let start = seg.find(&needle)? + needle.len(); + let end = seg[start..].find("")? + start; + Some(html_unescape(&seg[start..end])) +} + +/// Decode the handful of HTML entities that can appear in file names. +fn html_unescape(s: &str) -> String { + s.replace("&", "&") + .replace("<", "<") + .replace(">", ">") + .replace(""", "\"") + .replace("'", "'") + .replace("'", "'") +} + +/// A safe single path segment for a file name taken from a page. +fn sanitize_filename(name: &str) -> String { + let cleaned: String = name + .chars() + .map(|c| { + if c == '/' || c == '\\' || (c as u32) < 0x20 { + '_' + } else { + c + } + }) + .collect(); + let cleaned = cleaned.trim().trim_start_matches('.').to_string(); + if cleaned.is_empty() { + "unnamed".into() + } else { + cleaned + } +} + +/// Fetch a folder page's HTML via curl (browser-like headers, like the file +/// downloads). Writes to a temp file because curl is used throughout. +fn fetch_folder_html(url: &str, timeout_seconds: u64) -> Result { + let tmp = + std::env::temp_dir().join(format!("dlphelper-0807-folder-{}.html", std::process::id())); + let mut cmd = Command::new("curl"); + cmd.args([ + "-fSL", + "--connect-timeout", + "20", + "--max-time", + &timeout_seconds.to_string(), + "-A", + USER_AGENT, + "-e", + REFERER, + "-o", + ]); + cmd.arg(&tmp); + cmd.arg(url); + let proc = cmd + .output() + .map_err(|e| Zero807Error::Error(format!("failed to run curl: {e}")))?; + if proc.status.code() != Some(0) { + let stderr = String::from_utf8_lossy(&proc.stderr); + let tail: Vec<&str> = stderr.trim().lines().filter(|l| !l.is_empty()).collect(); + let start = tail.len().saturating_sub(3); + let msg = tail[start..].join(" | "); + let _ = fs::remove_file(&tmp); + if msg.contains("404") || msg.contains("410") { + return Err(Zero807Error::NotFound(format!( + "dead folder page (HTTP 404/410): {msg}" + ))); + } + return Err(Zero807Error::Error(format!("curl exit for page: {msg}"))); + } + let text = fs::read_to_string(&tmp) + .map_err(|e| Zero807Error::Error(format!("reading folder page: {e}")))?; + let _ = fs::remove_file(&tmp); + Ok(text) +} + +/// Download every file on a 0807.st folder page into `out_dir//`. +/// +/// Returns `(folder, [(path, size), ...])`. Each file is fetched from +/// `https://0807.st/?dl=1` (a 302 to the CDN). Already-present files +/// are skipped, so the caller's retry loop only fetches what is missing; a +/// partial failure is reported as an error so that retry happens. +pub fn download_folder( + url: &str, + out_dir: &Path, + timeout_seconds: u64, + resume: bool, +) -> Result<(PathBuf, Vec<(PathBuf, u64)>), Zero807Error> { + let id = folder_page_id(url) + .ok_or_else(|| Zero807Error::Error(format!("not a folder page: {url}")))?; + let html = fetch_folder_html(url, timeout_seconds)?; + let entries = parse_folder_page(&html); + if entries.is_empty() { + return Err(Zero807Error::Error(format!( + "folder page {url} lists no downloadable files" + ))); + } + let folder = out_dir.join(&id); + fs::create_dir_all(&folder) + .map_err(|e| Zero807Error::Error(format!("cannot create {}: {e}", folder.display())))?; + log::info!("Folder page {url}: {} files", entries.len()); + let mut files: Vec<(PathBuf, u64)> = Vec::new(); + let mut failed: Vec = Vec::new(); + for e in &entries { + let name = sanitize_filename(&e.name); + let dest = folder.join(&name); + let direct = format!("https://0807.st/{}?dl=1", e.id); + match download_to(&direct, &dest, timeout_seconds, resume) { + Ok(size) => { + log::info!("OK folder file {name} ({})", human_size(size)); + files.push((dest, size)); + } + Err(err) => { + log::error!("Folder file {name} failed: {err}"); + failed.push(format!("{name}: {err}")); + } + } + } + if !failed.is_empty() { + return Err(Zero807Error::Error(format!( + "{}/{} folder files failed: {}", + failed.len(), + entries.len(), + failed.join("; ") + ))); + } + Ok((folder, files)) +} + +/// Compact human-readable size (kept local so this module stays self-contained). +fn human_size(bytes: u64) -> String { + const UNITS: [&str; 5] = ["B", "KiB", "MiB", "GiB", "TiB"]; + let mut v = bytes as f64; + let mut u = 0; + while v >= 1024.0 && u < UNITS.len() - 1 { + v /= 1024.0; + u += 1; + } + if u == 0 { + format!("{bytes} B") + } else { + format!("{v:.1} {}", UNITS[u]) + } } #[cfg(test)] @@ -224,4 +460,60 @@ mod tests { ); assert_eq!(Zero807Error::Error("boom".into()).to_string(), "boom"); } + + #[test] + fn folder_page_detection() { + assert_eq!( + folder_page_id("https://0807.st/f/Df8A5sn3").as_deref(), + Some("Df8A5sn3") + ); + assert_eq!( + folder_page_id("https://0807.st/f/MM5AWQ9R/").as_deref(), + Some("MM5AWQ9R") + ); + assert!(is_folder_page("https://0807.st/f/Df8A5sn3")); + // a direct file link is not a folder page + assert!(!is_folder_page("https://0807.st/6saCNie.zip")); + // the folder export (.zip) is not a folder page either + assert!(!is_folder_page("https://0807.st/f/Df8A5sn3.zip")); + // deeper paths are not folder pages + assert!(!is_folder_page("https://0807.st/f/a/b")); + assert!(!is_folder_page("https://0807.st/")); + } + + #[test] + fn parse_folder_page_reads_alitem_entries() { + let html = r#"
+ 13. Terrain.zip1.3 GB + Orks & Goblins.zip +
"#; + let got = parse_folder_page(html); + assert_eq!(got.len(), 2); + assert_eq!(got[0].id, "9z5SsgV"); + assert_eq!(got[0].name, "13. Terrain.zip"); + assert_eq!(got[0].size, 1397446559); + assert_eq!(got[0].mime, "application/zip"); + assert_eq!(got[1].id, "wNuEQcp"); + assert_eq!(got[1].name, "Orks & Goblins.zip"); // entity decoded + assert_eq!(got[1].size, 2295545985); + } + + #[test] + fn parse_folder_page_ignores_non_alitem_anchors() { + let html = r#"homea.txt"#; + let got = parse_folder_page(html); + assert_eq!(got.len(), 1); + assert_eq!(got[0].id, "abc"); + assert_eq!(got[0].name, "a.txt"); + } + + #[test] + fn sanitize_filename_blocks_traversal() { + assert_eq!(sanitize_filename("13. Terrain.zip"), "13. Terrain.zip"); + let s = sanitize_filename("../../etc/passwd"); + assert!(!s.contains('/') && !s.contains('\\')); + assert!(!s.starts_with('.')); + assert_eq!(sanitize_filename("a\\b"), "a_b"); + assert_eq!(sanitize_filename(" "), "unnamed"); + } }