Ben_Kosytorz/Warhammer_DLP_Helper
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(rust): expand 0807.st folder pages (/f/<id>) into their files

Browse Source
This commit is contained in:
Ben_Kosytorz
2026-10-11 07:11:49 +02:00
parent bfc25bc7b6
commit 3b3c4b57ea
2 changed files with 339 additions and 27 deletions
Download Patch File Download Diff File Expand all files Collapse all files
version-rust/src/orchestrate.rs
+41 -21
View File
@@ -824,6 +824,37 @@ fn json_entry(dest: &Path, size: u64, name: &str) -> Value {
})
}
/// Download one 0807 URL: a folder page (`/f/<id>`) yields many files, a
/// plain link exactly one. Returns the manifest entries and the total size.
fn download_0807(
url: &str,
out_dir: &Path,
opts: &RunOpts,
) -> Result<(Vec<Value>, u64), Zero807Error> {
let files: Vec<(PathBuf, u64)> = if zero807::is_folder_page(url) {
zero807::download_folder(url, out_dir, 7200, true)?.1
} else {
vec![zero807::download(url, out_dir)?]
};
let mut entries = Vec::with_capacity(files.len());
let mut total = 0u64;
for (dest, size) in &files {
let name = dest
.file_name()
.map(|n| n.to_string_lossy().into_owned())
.unwrap_or_default();
let mut entry = json_entry(dest, *size, &name);
if opts.hash {
let h =
sha256_of(dest).map_err(|e| Zero807Error::Error(format!("hash failed: {e}")))?;
entry["sha256"] = Value::String(h);
}
entries.push(entry);
total += *size;
}
Ok((entries, total))
}
/// 0807 pipeline (python run_0807).
pub fn run_0807(
urls: &[FoundUrl],
@@ -850,36 +881,25 @@ pub fn run_0807(
let source = source_by_url.get(&f.url).cloned().unwrap_or_default();
let mut resolved = false;
for attempt in 1..=opts.retries {
let downloaded = zero807::download(&f.url, &out_dir)
.map_err(anyhow::Error::new)
.and_then(|(dest, size)| -> anyhow::Result<(Value, u64)> {
let mut entry = json_entry(
&dest,
size,
&dest
.file_name()
.map(|n| n.to_string_lossy().into_owned())
.unwrap_or_default(),
);
if opts.hash {
let h = sha256_of(&dest)?;
entry["sha256"] = Value::String(h);
}
Ok((entry, size))
});
let downloaded = download_0807(&f.url, &out_dir, opts).map_err(anyhow::Error::new);
match downloaded {
Ok((entry, size)) => {
log::info!("OK {} ({})", f.url, format_size(size));
Ok((entries, size)) => {
log::info!(
"OK {} ({}, {} file(s))",
f.url,
format_size(size),
entries.len()
);
record_files(
state,
gstore.as_deref_mut(),
&f.url,
std::slice::from_ref(&entry),
&entries,
&source,
manifest,
)?;
if opts.extract {
extract_zips(std::slice::from_ref(&entry));
extract_zips(&entries);
}
consecutive_failures = 0;
resolved = true;
version-rust/src/zero807.rs
+298 -6
View File
@@ -80,16 +80,35 @@ pub fn download_with(
resume: bool,
) -> Result<(PathBuf, u64), Zero807Error> {
let name = target_name(url)?;
fs::create_dir_all(out_dir)
.map_err(|e| Zero807Error::Error(format!("cannot create {}: {e}", out_dir.display())))?;
let dest = out_dir.join(&name);
let size = download_to(url, &dest, timeout_seconds, resume)?;
Ok((dest, size))
}
/// Download `url` to an explicit `dest` (`.part` + rename, resume, skip if
/// already present). `download_with` derives `dest` from the URL; the folder
/// downloader derives it from the page's file name.
pub fn download_to(
url: &str,
dest: &Path,
timeout_seconds: u64,
resume: bool,
) -> Result<u64, Zero807Error> {
let name = dest
.file_name()
.map(|n| n.to_string_lossy().into_owned())
.unwrap_or_default();
if let Some(parent) = dest.parent() {
fs::create_dir_all(parent)
.map_err(|e| Zero807Error::Error(format!("cannot create {}: {e}", parent.display())))?;
}
let tmp = dest.with_file_name(format!("{name}.part"));
if dest.exists() {
let size = fs::metadata(&dest).map(|m| m.len()).unwrap_or(0);
let size = fs::metadata(dest).map(|m| m.len()).unwrap_or(0);
if size > 0 {
log::info!("Skip existing: {}", dest.display());
return Ok((dest, size));
return Ok(size);
}
}
@@ -166,9 +185,226 @@ pub fn download_with(
}
}
// os.replace semantics: rename over an existing destination.
fs::rename(&tmp, &dest)
fs::rename(&tmp, dest)
.map_err(|e| Zero807Error::Error(format!("cannot move {name} into place: {e}")))?;
Ok((dest, size))
Ok(size)
}
/// A file listed on a 0807.st folder page (`/f/<id>`).
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct FolderEntry {
/// file id used in the direct-download URL (`https://0807.st/<id>?dl=1`)
pub id: String,
/// display file name from the page (`<span class="alname">`)
pub name: String,
pub size: u64,
pub mime: String,
}
/// The `<id>` of a 0807.st folder page URL (`/f/<id>`), if it is one.
/// A direct file link (`/x.zip`) or the folder export (`/f/<id>.zip`) is not.
pub fn folder_page_id(url: &str) -> Option<String> {
let path = match url::Url::parse(url) {
Ok(u) => u.path().to_string(),
Err(_) => url.split(['?', '#']).next().unwrap_or(url).to_string(),
};
let rest = path.trim_end_matches('/').strip_prefix("/f/")?;
if rest.is_empty() || rest.contains('/') || rest.contains('.') {
return None;
}
Some(rest.to_string())
}
/// True when `url` points at a 0807.st folder page.
pub fn is_folder_page(url: &str) -> bool {
folder_page_id(url).is_some()
}
/// Parse the file list from a 0807.st folder page. Each entry is an
/// `<a class="alitem" ... data-id="..." data-size="..." data-mime="...">`
/// whose name sits in `<span class="alname">...</span>`. Pure string scanning
/// (no HTML-parser dependency).
pub fn parse_folder_page(html: &str) -> Vec<FolderEntry> {
const MARK: &str = "<a class=\"alitem\"";
let mut out = Vec::new();
let mut rest = html;
while let Some(pos) = rest.find(MARK) {
rest = &rest[pos + MARK.len()..];
let end = rest.find("</a>").unwrap_or(rest.len());
let seg = &rest[..end];
if let Some(id) = attr(seg, "data-id") {
let size = attr(seg, "data-size")
.and_then(|s| s.parse::<u64>().ok())
.unwrap_or(0);
let mime = attr(seg, "data-mime").unwrap_or_default();
let name = span_text(seg, "alname").unwrap_or_else(|| id.clone());
out.push(FolderEntry {
id,
name,
size,
mime,
});
}
rest = &rest[end..];
}
out
}
/// Value of `key="..."` inside a tag fragment.
fn attr(seg: &str, key: &str) -> Option<String> {
let needle = format!("{key}=\"");
let start = seg.find(&needle)? + needle.len();
let end = seg[start..].find('"')? + start;
Some(seg[start..end].to_string())
}
/// Text of `<span class="class">...</span>` (HTML entities decoded).
fn span_text(seg: &str, class: &str) -> Option<String> {
let needle = format!("<span class=\"{class}\">");
let start = seg.find(&needle)? + needle.len();
let end = seg[start..].find("</span>")? + start;
Some(html_unescape(&seg[start..end]))
}
/// Decode the handful of HTML entities that can appear in file names.
fn html_unescape(s: &str) -> String {
s.replace("&amp;", "&")
.replace("&lt;", "<")
.replace("&gt;", ">")
.replace("&quot;", "\"")
.replace("&#39;", "'")
.replace("&#x27;", "'")
}
/// A safe single path segment for a file name taken from a page.
fn sanitize_filename(name: &str) -> String {
let cleaned: String = name
.chars()
.map(|c| {
if c == '/' || c == '\\' || (c as u32) < 0x20 {
'_'
} else {
c
}
})
.collect();
let cleaned = cleaned.trim().trim_start_matches('.').to_string();
if cleaned.is_empty() {
"unnamed".into()
} else {
cleaned
}
}
/// Fetch a folder page's HTML via curl (browser-like headers, like the file
/// downloads). Writes to a temp file because curl is used throughout.
fn fetch_folder_html(url: &str, timeout_seconds: u64) -> Result<String, Zero807Error> {
let tmp =
std::env::temp_dir().join(format!("dlphelper-0807-folder-{}.html", std::process::id()));
let mut cmd = Command::new("curl");
cmd.args([
"-fSL",
"--connect-timeout",
"20",
"--max-time",
&timeout_seconds.to_string(),
"-A",
USER_AGENT,
"-e",
REFERER,
"-o",
]);
cmd.arg(&tmp);
cmd.arg(url);
let proc = cmd
.output()
.map_err(|e| Zero807Error::Error(format!("failed to run curl: {e}")))?;
if proc.status.code() != Some(0) {
let stderr = String::from_utf8_lossy(&proc.stderr);
let tail: Vec<&str> = stderr.trim().lines().filter(|l| !l.is_empty()).collect();
let start = tail.len().saturating_sub(3);
let msg = tail[start..].join(" | ");
let _ = fs::remove_file(&tmp);
if msg.contains("404") || msg.contains("410") {
return Err(Zero807Error::NotFound(format!(
"dead folder page (HTTP 404/410): {msg}"
)));
}
return Err(Zero807Error::Error(format!("curl exit for page: {msg}")));
}
let text = fs::read_to_string(&tmp)
.map_err(|e| Zero807Error::Error(format!("reading folder page: {e}")))?;
let _ = fs::remove_file(&tmp);
Ok(text)
}
/// Download every file on a 0807.st folder page into `out_dir/<page-id>/`.
///
/// Returns `(folder, [(path, size), ...])`. Each file is fetched from
/// `https://0807.st/<data-id>?dl=1` (a 302 to the CDN). Already-present files
/// are skipped, so the caller's retry loop only fetches what is missing; a
/// partial failure is reported as an error so that retry happens.
pub fn download_folder(
url: &str,
out_dir: &Path,
timeout_seconds: u64,
resume: bool,
) -> Result<(PathBuf, Vec<(PathBuf, u64)>), Zero807Error> {
let id = folder_page_id(url)
.ok_or_else(|| Zero807Error::Error(format!("not a folder page: {url}")))?;
let html = fetch_folder_html(url, timeout_seconds)?;
let entries = parse_folder_page(&html);
if entries.is_empty() {
return Err(Zero807Error::Error(format!(
"folder page {url} lists no downloadable files"
)));
}
let folder = out_dir.join(&id);
fs::create_dir_all(&folder)
.map_err(|e| Zero807Error::Error(format!("cannot create {}: {e}", folder.display())))?;
log::info!("Folder page {url}: {} files", entries.len());
let mut files: Vec<(PathBuf, u64)> = Vec::new();
let mut failed: Vec<String> = Vec::new();
for e in &entries {
let name = sanitize_filename(&e.name);
let dest = folder.join(&name);
let direct = format!("https://0807.st/{}?dl=1", e.id);
match download_to(&direct, &dest, timeout_seconds, resume) {
Ok(size) => {
log::info!("OK folder file {name} ({})", human_size(size));
files.push((dest, size));
}
Err(err) => {
log::error!("Folder file {name} failed: {err}");
failed.push(format!("{name}: {err}"));
}
}
}
if !failed.is_empty() {
return Err(Zero807Error::Error(format!(
"{}/{} folder files failed: {}",
failed.len(),
entries.len(),
failed.join("; ")
)));
}
Ok((folder, files))
}
/// Compact human-readable size (kept local so this module stays self-contained).
fn human_size(bytes: u64) -> String {
const UNITS: [&str; 5] = ["B", "KiB", "MiB", "GiB", "TiB"];
let mut v = bytes as f64;
let mut u = 0;
while v >= 1024.0 && u < UNITS.len() - 1 {
v /= 1024.0;
u += 1;
}
if u == 0 {
format!("{bytes} B")
} else {
format!("{v:.1} {}", UNITS[u])
}
}
#[cfg(test)]
@@ -224,4 +460,60 @@ mod tests {
);
assert_eq!(Zero807Error::Error("boom".into()).to_string(), "boom");
}
#[test]
fn folder_page_detection() {
assert_eq!(
folder_page_id("https://0807.st/f/Df8A5sn3").as_deref(),
Some("Df8A5sn3")
);
assert_eq!(
folder_page_id("https://0807.st/f/MM5AWQ9R/").as_deref(),
Some("MM5AWQ9R")
);
assert!(is_folder_page("https://0807.st/f/Df8A5sn3"));
// a direct file link is not a folder page
assert!(!is_folder_page("https://0807.st/6saCNie.zip"));
// the folder export (.zip) is not a folder page either
assert!(!is_folder_page("https://0807.st/f/Df8A5sn3.zip"));
// deeper paths are not folder pages
assert!(!is_folder_page("https://0807.st/f/a/b"));
assert!(!is_folder_page("https://0807.st/"));
}
#[test]
fn parse_folder_page_reads_alitem_entries() {
let html = r#"<div class="allist">
<a class="alitem" href="/p/9z5SsgV" data-id="9z5SsgV" data-rel="" data-size="1397446559" data-mime="application/zip"><span class="althumbwrap"><span class="alicon"><svg></svg></span></span><span class="alinfo"><span class="alname">13. Terrain.zip</span><span class="alsize">1.3 GB</span></span></a>
<a class="alitem" href="/p/wNuEQcp" data-id="wNuEQcp" data-rel="" data-size="2295545985" data-mime="application/zip"><span class="alname">Orks &amp; Goblins.zip</span></a>
</div>"#;
let got = parse_folder_page(html);
assert_eq!(got.len(), 2);
assert_eq!(got[0].id, "9z5SsgV");
assert_eq!(got[0].name, "13. Terrain.zip");
assert_eq!(got[0].size, 1397446559);
assert_eq!(got[0].mime, "application/zip");
assert_eq!(got[1].id, "wNuEQcp");
assert_eq!(got[1].name, "Orks & Goblins.zip"); // entity decoded
assert_eq!(got[1].size, 2295545985);
}
#[test]
fn parse_folder_page_ignores_non_alitem_anchors() {
let html = r#"<a href="/" class="x">home</a><a class="alitem" href="/p/abc" data-id="abc" data-size="5" data-mime="text/plain"><span class="alname">a.txt</span></a>"#;
let got = parse_folder_page(html);
assert_eq!(got.len(), 1);
assert_eq!(got[0].id, "abc");
assert_eq!(got[0].name, "a.txt");
}
#[test]
fn sanitize_filename_blocks_traversal() {
assert_eq!(sanitize_filename("13. Terrain.zip"), "13. Terrain.zip");
let s = sanitize_filename("../../etc/passwd");
assert!(!s.contains('/') && !s.contains('\\'));
assert!(!s.starts_with('.'));
assert_eq!(sanitize_filename("a\\b"), "a_b");
assert_eq!(sanitize_filename(" "), "unnamed");
}
}