feat(rust): port discord, gofile, zero807, extract_all and catalogue
Parallel module ports (behavior parity with the python versions): - src/discord.rs: DCE docker export (token via env only), base64/plain URL extraction with grab.sh-compatible regexes, host classification; 11 tests - src/gofile.rs: GoFile API client — one guest account per run, linear backoff on account creation, sha256 website token, recursive content walk, streaming .part downloads; 7 tests (offline goldens from python hashlib) - src/zero807.rs: curl downloader with UA+Referer contract, dead-link detection; 4 tests - src/extract.rs: recursive extraction (sevenz-rust2 for 7z, zip crate, unrar for rar, system tar/gzip/bzip2/xz for tarballs), nested BFS expansion, .broken quarantine, extract_manifest.jsonl, idempotent re-runs; 8 tests - src/catalogue.rs: keyword classifier over the dumped taxonomy, manifest/ directory scanning, overrides, md+jsonl reports, audited moves; 7 tests incl. all 17 python test cases
This commit is contained in:
Generated
+20
@@ -592,6 +592,7 @@ dependencies = [
|
||||
"clap",
|
||||
"dirs",
|
||||
"env_logger",
|
||||
"fs4",
|
||||
"hex",
|
||||
"indexmap",
|
||||
"log",
|
||||
@@ -715,6 +716,16 @@ dependencies = [
|
||||
"percent-encoding",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "fs4"
|
||||
version = "0.13.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8640e34b88f7652208ce9e88b1a37a2ae95227d84abec377ccd3c5cfeb141ed4"
|
||||
dependencies = [
|
||||
"rustix",
|
||||
"windows-sys 0.59.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "futures-channel"
|
||||
version = "0.3.34"
|
||||
@@ -2445,6 +2456,15 @@ dependencies = [
|
||||
"windows-targets",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "windows-sys"
|
||||
version = "0.59.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b"
|
||||
dependencies = [
|
||||
"windows-targets",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "windows-sys"
|
||||
version = "0.61.2"
|
||||
|
||||
@@ -11,6 +11,7 @@ chrono = "0.4"
|
||||
clap = { version = "4", features = ["derive"] }
|
||||
dirs = "6"
|
||||
env_logger = "0.11"
|
||||
fs4 = "0.13"
|
||||
hex = "0.4"
|
||||
indexmap = { version = "2", features = ["serde"] }
|
||||
log = "0.4"
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,2 +1,516 @@
|
||||
//! DCE docker export (JSON) + base64/plain URL extraction — Rust port of
|
||||
//! `version-python/dlphelper/discord.py`.
|
||||
//!
|
||||
//! The channels this tool targets encode their download links as base64 inside
|
||||
//! message content, and occasionally as plain links. We export the channel as
|
||||
//! JSON (richer metadata than the HTML export: message id, timestamp, author),
|
||||
//! decode every base64 candidate, and keep only URLs for supported hosts.
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::fs;
|
||||
use std::io;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::process::Command;
|
||||
use std::sync::LazyLock;
|
||||
|
||||
use anyhow::{Context, Result, bail};
|
||||
use base64::Engine as _;
|
||||
use base64::engine::general_purpose::STANDARD;
|
||||
use regex::Regex;
|
||||
use serde_json::Value;
|
||||
use url::Url;
|
||||
|
||||
pub const DCE_IMAGE: &str = "tyrrrz/discordchatexporter:stable";
|
||||
|
||||
// The channels encode links as base64 that always starts with this prefix
|
||||
// (https://...); grab.sh established this regex, keep it compatible.
|
||||
static B64_RE: LazyLock<Regex> =
|
||||
LazyLock::new(|| Regex::new(r"aHR0cHM6Ly[89][A-Za-z0-9+/_-]*").unwrap());
|
||||
static URL_RE: LazyLock<Regex> =
|
||||
LazyLock::new(|| Regex::new(r#"https://[^\s<>"'\)\]\}]+"#).unwrap());
|
||||
|
||||
/// Host key ("gofile" | "0807") -> host names accepted for it.
|
||||
pub const SUPPORTED_HOSTS: &[(&str, &[&str])] = &[
|
||||
("gofile", &["gofile.io", "www.gofile.io"]),
|
||||
("0807", &["0807.st", "www.0807.st"]),
|
||||
];
|
||||
|
||||
/// One download URL found in the channel, with source attribution.
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub struct FoundUrl {
|
||||
pub url: String,
|
||||
/// "gofile" | "0807" (set after classification)
|
||||
pub host: &'static str,
|
||||
pub message_id: Option<String>,
|
||||
pub timestamp: Option<String>,
|
||||
pub author: Option<String>,
|
||||
/// found via base64 decode vs. plain link
|
||||
pub encoded: bool,
|
||||
}
|
||||
|
||||
/// Export the channel to out_dir/export.json using the DCE docker image.
|
||||
///
|
||||
/// The token is passed to the container via the DISCORD_TOKEN env var (DCE
|
||||
/// reads it natively) so it never appears on the docker command line. TOKEN
|
||||
/// is set as well for DCE versions that only know that name.
|
||||
pub fn export_channel(
|
||||
channel_id: &str,
|
||||
out_dir: &Path,
|
||||
token: &str,
|
||||
force: bool,
|
||||
) -> Result<PathBuf> {
|
||||
let dest = out_dir.join("export.json");
|
||||
if dest.exists() && !force {
|
||||
log::info!(
|
||||
"Reusing existing export {} (pass --fresh to re-export)",
|
||||
dest.display()
|
||||
);
|
||||
return Ok(dest);
|
||||
}
|
||||
|
||||
fs::create_dir_all(out_dir)?;
|
||||
// Python Path.resolve() on an existing dir == canonicalize here (the
|
||||
// create_dir_all above guarantees it exists).
|
||||
let resolved = fs::canonicalize(out_dir)?;
|
||||
let (uid, gid) = read_uid_gid()?;
|
||||
|
||||
log::info!(
|
||||
"Exporting channel {} via DiscordChatExporter ...",
|
||||
channel_id
|
||||
);
|
||||
// stdout/stderr are inherited, like the python subprocess.run(check=True).
|
||||
let status = Command::new("docker")
|
||||
.args([
|
||||
"run",
|
||||
"--rm",
|
||||
"--user",
|
||||
&format!("{uid}:{gid}"),
|
||||
"-v",
|
||||
&format!("{}:/out", resolved.display()),
|
||||
"-e",
|
||||
"DISCORD_TOKEN",
|
||||
"-e",
|
||||
"TOKEN",
|
||||
])
|
||||
.arg(DCE_IMAGE)
|
||||
.args([
|
||||
"export",
|
||||
"-c",
|
||||
channel_id,
|
||||
"-f",
|
||||
"Json",
|
||||
"-o",
|
||||
"/out/export.json",
|
||||
])
|
||||
// The token only travels via the environment, never on the argv.
|
||||
.env("DISCORD_TOKEN", token)
|
||||
.env("TOKEN", token)
|
||||
.status();
|
||||
match status {
|
||||
Err(e) if e.kind() == io::ErrorKind::NotFound => {
|
||||
bail!("docker not found - the Discord export needs the docker image {DCE_IMAGE}")
|
||||
}
|
||||
Err(e) => return Err(e).context("failed to spawn docker"),
|
||||
Ok(s) if !s.success() => bail!(
|
||||
"DiscordChatExporter failed with exit code {} (check the token and the channel id)",
|
||||
s.code().unwrap_or(-1)
|
||||
),
|
||||
Ok(_) => {}
|
||||
}
|
||||
if !dest.exists() {
|
||||
bail!("DiscordChatExporter produced no output file");
|
||||
}
|
||||
Ok(dest)
|
||||
}
|
||||
|
||||
/// Real uid/gid of this process. Linux only: parsed from /proc/self/status
|
||||
/// because std has no getuid (the first number on the Uid:/Gid: line is the
|
||||
/// real id, matching python's os.getuid()/os.getgid()).
|
||||
fn read_uid_gid() -> Result<(u32, u32)> {
|
||||
let status = fs::read_to_string("/proc/self/status")
|
||||
.context("cannot read /proc/self/status to determine uid/gid for docker --user")?;
|
||||
let uid = parse_proc_id(&status, "Uid:")?;
|
||||
let gid = parse_proc_id(&status, "Gid:")?;
|
||||
Ok((uid, gid))
|
||||
}
|
||||
|
||||
fn parse_proc_id(status_text: &str, key: &str) -> Result<u32> {
|
||||
for line in status_text.lines() {
|
||||
if let Some(rest) = line.strip_prefix(key) {
|
||||
let first = rest
|
||||
.split_whitespace()
|
||||
.next()
|
||||
.with_context(|| format!("empty {key} line in /proc/self/status"))?;
|
||||
return first.parse().with_context(|| {
|
||||
format!("cannot parse {key} value {first:?} in /proc/self/status")
|
||||
});
|
||||
}
|
||||
}
|
||||
bail!("{key} line not found in /proc/self/status")
|
||||
}
|
||||
|
||||
/// Return the host key ('gofile' | '0807') for a URL, else None.
|
||||
///
|
||||
/// Mirrors python `urlparse(url).hostname.lower()`: unparseable URLs and URLs
|
||||
/// without a hostname yield None. The url crate normalizes the host to
|
||||
/// lowercase, same as urlparse does.
|
||||
pub fn classify(url: &str) -> Option<&'static str> {
|
||||
let parsed = Url::parse(url).ok()?;
|
||||
let hostname = parsed.host_str()?;
|
||||
SUPPORTED_HOSTS
|
||||
.iter()
|
||||
.find(|(_, hosts)| hosts.contains(&hostname))
|
||||
.map(|(key, _)| *key)
|
||||
}
|
||||
|
||||
/// Decode one base64 candidate into the URL it contains, or None.
|
||||
///
|
||||
/// Python quirk kept: "_" is remapped to "/" and "-" to "+" (in that order),
|
||||
/// trailing "=" stripped, then padded back to a multiple of 4 before a strict
|
||||
/// standard-alphabet decode and a strict UTF-8 check. One encoded blob can
|
||||
/// contain the URL plus padding/newlines: whitespace is trimmed and only the
|
||||
/// first line is returned.
|
||||
fn decode_b64(candidate: &str) -> Option<String> {
|
||||
let mut s: String = candidate
|
||||
.chars()
|
||||
.map(|c| match c {
|
||||
'_' => '/',
|
||||
'-' => '+',
|
||||
other => other,
|
||||
})
|
||||
.collect();
|
||||
while s.ends_with('=') {
|
||||
s.pop();
|
||||
}
|
||||
for _ in 0..(4 - s.len() % 4) % 4 {
|
||||
s.push('=');
|
||||
}
|
||||
let decoded = STANDARD.decode(s.as_bytes()).ok()?;
|
||||
let text = String::from_utf8(decoded).ok()?;
|
||||
// strip whitespace, keep the first line ("" -> None)
|
||||
text.trim()
|
||||
.lines()
|
||||
.next()
|
||||
.filter(|l| !l.is_empty())
|
||||
.map(String::from)
|
||||
}
|
||||
|
||||
/// Python `str(x or "")` for a JSON value: falsy values (null, false, 0, "",
|
||||
/// empty array/object) become the empty string, strings pass through, and any
|
||||
/// other value is stringified. DCE content/id fields are always strings; the
|
||||
/// other shapes only exist for parity with the python code.
|
||||
fn python_str_or_empty(value: Option<&Value>) -> String {
|
||||
let falsy = match value {
|
||||
None | Some(Value::Null) => true,
|
||||
Some(Value::Bool(b)) => !b,
|
||||
Some(Value::Number(n)) => n.as_f64().is_some_and(|f| f == 0.0),
|
||||
Some(Value::String(s)) => s.is_empty(),
|
||||
Some(Value::Array(a)) => a.is_empty(),
|
||||
Some(Value::Object(o)) => o.is_empty(),
|
||||
};
|
||||
if falsy {
|
||||
return String::new();
|
||||
}
|
||||
match value.unwrap() {
|
||||
Value::String(s) => s.clone(),
|
||||
Value::Number(n) => n.to_string(),
|
||||
Value::Bool(_) => "True".to_string(), // python str(True)
|
||||
other => other.to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Extract supported URLs from a DCE JSON export, deduplicated.
|
||||
///
|
||||
/// The root may be a list of messages or an object with a "messages" array;
|
||||
/// anything else yields no URLs. The first occurrence of a URL wins
|
||||
/// (python dict.setdefault semantics).
|
||||
pub fn extract_urls(export_path: &Path) -> Result<Vec<FoundUrl>> {
|
||||
let text = fs::read_to_string(export_path)?;
|
||||
let data: Value = serde_json::from_str(&text)?;
|
||||
let empty: Vec<Value> = Vec::new();
|
||||
let messages: &[Value] = match &data {
|
||||
Value::Array(list) => list,
|
||||
Value::Object(obj) => obj
|
||||
.get("messages")
|
||||
.and_then(Value::as_array)
|
||||
.map(|a| a.as_slice())
|
||||
.unwrap_or(&empty),
|
||||
_ => &empty,
|
||||
};
|
||||
|
||||
let mut found: HashMap<String, FoundUrl> = HashMap::new();
|
||||
let mut stat_b64 = 0u64;
|
||||
let mut stat_plain = 0u64;
|
||||
let mut stat_unsupported = 0u64;
|
||||
|
||||
for msg in messages {
|
||||
let content = python_str_or_empty(msg.get("content"));
|
||||
if content.is_empty() {
|
||||
continue;
|
||||
}
|
||||
let author = msg
|
||||
.get("author")
|
||||
.and_then(|a| a.get("name"))
|
||||
.and_then(Value::as_str)
|
||||
.map(str::to_owned);
|
||||
let message_id = {
|
||||
let s = python_str_or_empty(msg.get("id"));
|
||||
(!s.is_empty()).then_some(s)
|
||||
};
|
||||
let timestamp = msg
|
||||
.get("timestamp")
|
||||
.and_then(Value::as_str)
|
||||
.map(str::to_owned);
|
||||
|
||||
for m in B64_RE.find_iter(&content) {
|
||||
stat_b64 += 1;
|
||||
let Some(decoded) = decode_b64(m.as_str()) else {
|
||||
continue;
|
||||
};
|
||||
let Some(host) = classify(&decoded) else {
|
||||
stat_unsupported += 1;
|
||||
continue;
|
||||
};
|
||||
found.entry(decoded.clone()).or_insert(FoundUrl {
|
||||
url: decoded,
|
||||
host,
|
||||
message_id: message_id.clone(),
|
||||
timestamp: timestamp.clone(),
|
||||
author: author.clone(),
|
||||
encoded: true,
|
||||
});
|
||||
}
|
||||
|
||||
for m in URL_RE.find_iter(&content) {
|
||||
let url = m
|
||||
.as_str()
|
||||
.trim_end_matches(['.', ',', ';', ')'])
|
||||
.to_string();
|
||||
let Some(host) = classify(&url) else {
|
||||
continue;
|
||||
};
|
||||
stat_plain += 1;
|
||||
found.entry(url.clone()).or_insert(FoundUrl {
|
||||
url,
|
||||
host,
|
||||
message_id: message_id.clone(),
|
||||
timestamp: timestamp.clone(),
|
||||
author: author.clone(),
|
||||
encoded: false,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
log::info!(
|
||||
"Extraction: {} base64 candidates, {} plain links, {} unsupported/undecodable, {} unique supported URLs",
|
||||
stat_b64,
|
||||
stat_plain,
|
||||
stat_unsupported,
|
||||
found.len()
|
||||
);
|
||||
let mut out: Vec<FoundUrl> = found.into_values().collect();
|
||||
out.sort_by(|a, b| a.url.cmp(&b.url));
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use base64::engine::general_purpose::URL_SAFE_NO_PAD;
|
||||
use serde_json::json;
|
||||
use tempfile::tempdir;
|
||||
|
||||
fn b64(s: &str) -> String {
|
||||
STANDARD.encode(s.as_bytes())
|
||||
}
|
||||
|
||||
/// The two-message fixture from version-python/tests/test_extract.py.
|
||||
fn fixture_messages() -> Value {
|
||||
json!([
|
||||
{
|
||||
"id": "1",
|
||||
"timestamp": "2026-01-01T00:00:00",
|
||||
"author": {"name": "poster"},
|
||||
"content": format!("chaos stuff: {}", b64("https://gofile.io/d/abc123")),
|
||||
},
|
||||
{
|
||||
"id": "2",
|
||||
"timestamp": "2026-01-02T00:00:00",
|
||||
"author": {"name": "poster"},
|
||||
"content": format!(
|
||||
"0807 zip: {}\ndupe of 1: {}\nplain: https://0807.st/plainLink.zip and \
|
||||
https://discord.com/channels/1/2 (unsupported)\njunk: {} and \
|
||||
not-our-domain base64: aHR0cHM6Ly9leGFtcGxlLm9yZw==",
|
||||
b64("https://0807.st/6saCNie.zip"),
|
||||
b64("https://gofile.io/d/abc123"),
|
||||
b64("https://pixeldrain.com/u/xyz"),
|
||||
),
|
||||
},
|
||||
])
|
||||
}
|
||||
|
||||
fn write_export(dir: &Path, root: Value) -> PathBuf {
|
||||
let path = dir.join("export.json");
|
||||
fs::write(&path, serde_json::to_string(&root).unwrap()).unwrap();
|
||||
path
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn extract_urls_matches_python_fixture() {
|
||||
let dir = tempdir().unwrap();
|
||||
let export = write_export(dir.path(), json!({"messages": fixture_messages()}));
|
||||
let found = extract_urls(&export).unwrap();
|
||||
|
||||
let urls: Vec<&str> = found.iter().map(|f| f.url.as_str()).collect();
|
||||
assert_eq!(
|
||||
urls,
|
||||
vec![
|
||||
"https://0807.st/6saCNie.zip",
|
||||
"https://0807.st/plainLink.zip",
|
||||
"https://gofile.io/d/abc123",
|
||||
]
|
||||
);
|
||||
|
||||
let g = found
|
||||
.iter()
|
||||
.find(|f| f.url == "https://gofile.io/d/abc123")
|
||||
.unwrap();
|
||||
assert_eq!(g.host, "gofile");
|
||||
assert_eq!(g.message_id.as_deref(), Some("1"));
|
||||
assert_eq!(g.timestamp.as_deref(), Some("2026-01-01T00:00:00"));
|
||||
assert_eq!(g.author.as_deref(), Some("poster"));
|
||||
assert!(g.encoded);
|
||||
|
||||
let z = found
|
||||
.iter()
|
||||
.find(|f| f.url == "https://0807.st/6saCNie.zip")
|
||||
.unwrap();
|
||||
assert_eq!(z.host, "0807");
|
||||
assert_eq!(z.message_id.as_deref(), Some("2"));
|
||||
assert!(z.encoded);
|
||||
|
||||
let p = found
|
||||
.iter()
|
||||
.find(|f| f.url == "https://0807.st/plainLink.zip")
|
||||
.unwrap();
|
||||
assert_eq!(p.host, "0807");
|
||||
assert_eq!(p.message_id.as_deref(), Some("2"));
|
||||
assert_eq!(p.author.as_deref(), Some("poster"));
|
||||
assert!(!p.encoded);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn extract_urls_accepts_bare_list_root() {
|
||||
let dir = tempdir().unwrap();
|
||||
let export = write_export(dir.path(), fixture_messages());
|
||||
let found = extract_urls(&export).unwrap();
|
||||
assert_eq!(found.len(), 3);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn extract_urls_empty_and_malformed_roots_yield_nothing() {
|
||||
let dir = tempdir().unwrap();
|
||||
assert!(
|
||||
extract_urls(&write_export(dir.path(), json!({})))
|
||||
.unwrap()
|
||||
.is_empty()
|
||||
);
|
||||
assert!(
|
||||
extract_urls(&write_export(dir.path(), json!({"other": []})))
|
||||
.unwrap()
|
||||
.is_empty()
|
||||
);
|
||||
// message without content is skipped
|
||||
let export = write_export(dir.path(), json!([{"id": "7"}]));
|
||||
assert!(extract_urls(&export).unwrap().is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn classify_matches_python_hostnames() {
|
||||
assert_eq!(classify("https://0807.st/x.zip"), Some("0807"));
|
||||
assert_eq!(classify("https://www.0807.st/x.zip"), Some("0807"));
|
||||
assert_eq!(classify("https://gofile.io/d/x"), Some("gofile"));
|
||||
assert_eq!(classify("https://www.gofile.io/d/x"), Some("gofile"));
|
||||
assert_eq!(classify("https://example.org/x"), None);
|
||||
// url crate lowercases hosts, like urlparse().hostname.lower()
|
||||
assert_eq!(classify("https://GOFILE.IO/d/x"), Some("gofile"));
|
||||
// edge cases: no scheme, empty string, bare host
|
||||
assert_eq!(classify("0807.st/x.zip"), None);
|
||||
assert_eq!(classify(""), None);
|
||||
assert_eq!(classify("https://"), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn decode_b64_handles_padding_and_rstrip() {
|
||||
// candidate as matched by B64_RE carries no '=': padding is added
|
||||
let unpadded = b64("https://gofile.io/d/abc123")
|
||||
.trim_end_matches('=')
|
||||
.to_string();
|
||||
assert_eq!(
|
||||
decode_b64(&unpadded),
|
||||
Some("https://gofile.io/d/abc123".to_string())
|
||||
);
|
||||
// trailing '=' is stripped before re-padding
|
||||
assert_eq!(
|
||||
decode_b64("aHR0cHM6Ly9leGFtcGxlLm9yZw=="),
|
||||
Some("https://example.org".to_string())
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn decode_b64_remaps_urlsafe_alphabet() {
|
||||
// U+13C0 encodes to 4Y+A (standard) / 4Y-A (urlsafe); the python
|
||||
// quirk remaps '-' back to '+' before decoding. ('_' -> '/' can never
|
||||
// occur in a candidate that decodes to valid UTF-8, so '-' covers the
|
||||
// remap path.)
|
||||
let candidate = URL_SAFE_NO_PAD.encode("\u{13C0}");
|
||||
assert!(candidate.contains('-'), "expected '-' in {candidate}");
|
||||
assert_eq!(decode_b64(&candidate), Some("\u{13C0}".to_string()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn decode_b64_keeps_first_line_after_whitespace() {
|
||||
let candidate = STANDARD.encode("https://gofile.io/d/abc123\nnoise ");
|
||||
assert_eq!(
|
||||
decode_b64(&candidate),
|
||||
Some("https://gofile.io/d/abc123".to_string())
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn decode_b64_rejects_invalid_utf8_and_garbage() {
|
||||
// decodes to \xff\xfe — not valid UTF-8
|
||||
assert_eq!(decode_b64("//8+"), None);
|
||||
assert_eq!(decode_b64("aHR0cHM6Ly["), None);
|
||||
assert_eq!(decode_b64(""), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_proc_id_reads_first_number() {
|
||||
let text = "Name:\tx\nUid:\t1000\t1000\t1000\t1000\nGid:\t989\t989\t989\t989\n";
|
||||
assert_eq!(parse_proc_id(text, "Uid:").unwrap(), 1000);
|
||||
assert_eq!(parse_proc_id(text, "Gid:").unwrap(), 989);
|
||||
assert!(parse_proc_id("Name:\tx\n", "Uid:").is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn read_uid_gid_matches_proc_self() {
|
||||
// Linux-only binary (docker export); /proc/self/status must exist.
|
||||
let (uid, gid) = read_uid_gid().unwrap();
|
||||
let text = fs::read_to_string("/proc/self/status").unwrap();
|
||||
let want_uid = parse_proc_id(&text, "Uid:").unwrap();
|
||||
let want_gid = parse_proc_id(&text, "Gid:").unwrap();
|
||||
assert_eq!((uid, gid), (want_uid, want_gid));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn export_channel_reuses_existing_export_without_docker() {
|
||||
let dir = tempdir().unwrap();
|
||||
let dest = dir.path().join("export.json");
|
||||
fs::write(&dest, "[]").unwrap();
|
||||
// force=false must short-circuit before any docker invocation; the
|
||||
// token must not end up in the error/log path either.
|
||||
let got = export_channel("123", dir.path(), "s3cret-token", false).unwrap();
|
||||
assert_eq!(got, dest);
|
||||
}
|
||||
}
|
||||
|
||||
+1190
-5
File diff suppressed because it is too large
Load Diff
@@ -1,2 +1,675 @@
|
||||
//! GoFile API client (one guest account per run) — Rust port of
|
||||
//! `version-python/dlphelper/gofile.py`.
|
||||
//!
|
||||
//! The API interaction mirrors Lysagxra/GoFileDownloader (same headers,
|
||||
//! website-token scheme and endpoints), with the same two deliberate
|
||||
//! differences as the python version:
|
||||
//!
|
||||
//! 1. A single guest account is created per run (GoFileDownloader creates one
|
||||
//! per URL, which trips Gofile's rate limit after ~6 requests).
|
||||
//! 2. Failures return errors instead of exiting, so the orchestrator can
|
||||
//! track them per file/URL and keep going.
|
||||
//!
|
||||
//! Retry semantics (see python AGENTS.md §6): `NotFound`, `Empty` and
|
||||
//! `PasswordRequired` are never retried by the orchestrator; `AuthError`
|
||||
//! allows one re-auth (`reset()` + `ensure_token()`).
|
||||
|
||||
use std::fmt;
|
||||
use std::fs;
|
||||
use std::io::{Read, Write};
|
||||
use std::path::Path;
|
||||
use std::sync::OnceLock;
|
||||
use std::time::{Duration, SystemTime, UNIX_EPOCH};
|
||||
|
||||
use hex::encode as hex_encode;
|
||||
use reqwest::blocking::Client;
|
||||
use reqwest::header::{AUTHORIZATION, COOKIE, HeaderMap, HeaderName, HeaderValue, ORIGIN, REFERER};
|
||||
use sha2::{Digest, Sha256};
|
||||
|
||||
pub const API: &str = "https://api.gofile.io";
|
||||
pub const USER_AGENT: &str = "Mozilla/5.0";
|
||||
pub const LOCALE: &str = "en-US";
|
||||
/// Public website-token seed, see GoFileDownloader. Fine to hardcode; the
|
||||
/// resulting tokens must never be logged (AGENTS.md §6).
|
||||
pub const TOKEN_SECRET: &str = "12af056dacea0b";
|
||||
pub const TOKEN_WINDOW_SEC: u64 = 14400;
|
||||
pub const CHUNK_SIZE: usize = 1024 * 1024;
|
||||
|
||||
const COMMON_HEADER_PAIRS: &[(&str, &str)] = &[
|
||||
("User-Agent", USER_AGENT),
|
||||
("Accept-Encoding", "gzip"),
|
||||
("Accept", "*/*"),
|
||||
("Connection", "keep-alive"),
|
||||
];
|
||||
|
||||
const EXTENDED_HEADER_PAIRS: &[(&str, &str)] = &[
|
||||
("User-Agent", USER_AGENT),
|
||||
("Accept-Encoding", "gzip"),
|
||||
("Accept", "*/*"),
|
||||
("Connection", "keep-alive"),
|
||||
("Sec-Fetch-Dest", "empty"),
|
||||
("Sec-Fetch-Mode", "cors"),
|
||||
("Sec-Fetch-Site", "same-site"),
|
||||
("Pragma", "no-cache"),
|
||||
("Cache-Control", "no-cache"),
|
||||
];
|
||||
|
||||
/// Common headers sent to every endpoint (python `COMMON_HEADERS`).
|
||||
fn common_headers() -> HeaderMap {
|
||||
static M: OnceLock<HeaderMap> = OnceLock::new();
|
||||
M.get_or_init(|| build_header_map(COMMON_HEADER_PAIRS))
|
||||
.clone()
|
||||
}
|
||||
|
||||
/// Common + fetch/cache-control headers (python `EXTENDED_HEADERS`).
|
||||
fn extended_headers() -> HeaderMap {
|
||||
static M: OnceLock<HeaderMap> = OnceLock::new();
|
||||
M.get_or_init(|| build_header_map(EXTENDED_HEADER_PAIRS))
|
||||
.clone()
|
||||
}
|
||||
|
||||
fn build_header_map(pairs: &'static [(&'static str, &'static str)]) -> HeaderMap {
|
||||
let mut map = HeaderMap::new();
|
||||
for (name, value) in pairs {
|
||||
let name: HeaderName = name
|
||||
.parse()
|
||||
.unwrap_or_else(|_| panic!("static header name {name:?} is invalid"));
|
||||
let value = HeaderValue::from_static(value);
|
||||
map.insert(name, value);
|
||||
}
|
||||
map
|
||||
}
|
||||
|
||||
/// Insert a dynamically computed header. Unlike python (which would raise on
|
||||
/// an invalid value and let the orchestrator's catch-all handle it), a parse
|
||||
/// failure here logs a warning and skips the header — the request then fails
|
||||
/// server-side and is recorded, and the run keeps going.
|
||||
fn set_header(map: &mut HeaderMap, name: HeaderName, value: &str) {
|
||||
match HeaderValue::from_str(value) {
|
||||
Ok(v) => {
|
||||
map.insert(name, v);
|
||||
}
|
||||
Err(_) => {
|
||||
log::warn!("skipping invalid {} header value", name.as_str());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Error type mirroring the python exception classes. The orchestrator maps
|
||||
/// variants to retry policies: `NotFound`/`Empty`/`PasswordRequired` are
|
||||
/// final states, `AuthError` permits one re-auth, `Error` is retryable.
|
||||
#[allow(dead_code)] // variants reserved for the orchestrator (e.g. Empty)
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub enum GoFileError {
|
||||
/// python `GoFileError` (base): transient/unclassified failure.
|
||||
Error(String),
|
||||
/// python `GoFilePasswordRequired`.
|
||||
PasswordRequired(String),
|
||||
/// python `GoFileNotFound`: dead link, never retry.
|
||||
NotFound(String),
|
||||
/// python `GoFileEmpty`: content exists but contains no files.
|
||||
Empty(String),
|
||||
/// python `GoFileAuthError`: token rejected, one re-auth allowed.
|
||||
AuthError(String),
|
||||
}
|
||||
|
||||
impl fmt::Display for GoFileError {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
match self {
|
||||
GoFileError::Error(m)
|
||||
| GoFileError::PasswordRequired(m)
|
||||
| GoFileError::NotFound(m)
|
||||
| GoFileError::Empty(m)
|
||||
| GoFileError::AuthError(m) => write!(f, "{m}"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl std::error::Error for GoFileError {}
|
||||
|
||||
/// Extract the content id from a gofile.io/d/<id> URL.
|
||||
///
|
||||
/// Matches the lenient python `urlparse` behavior: a URL without a scheme
|
||||
/// still works (`"gofile.io/d/x"` → `"x"`); only the path is inspected.
|
||||
pub fn content_id_from_url(url: &str) -> Result<String, GoFileError> {
|
||||
let path = match url::Url::parse(url) {
|
||||
Ok(u) => u.path().to_string(),
|
||||
// python urlparse is lenient about a missing scheme but still strips
|
||||
// query/fragment; mirror that ("gofile.io/d/x?y=1" → path as parsed).
|
||||
Err(_) => url.split(['?', '#']).next().unwrap_or(url).to_string(),
|
||||
};
|
||||
let parts: Vec<&str> = path.split('/').filter(|p| !p.is_empty()).collect();
|
||||
if parts.len() >= 2 && parts[parts.len() - 2] == "d" {
|
||||
return Ok(parts[parts.len() - 1].to_string());
|
||||
}
|
||||
Err(GoFileError::Error(format!(
|
||||
"not a gofile content URL (expected gofile.io/d/<id>): {url}"
|
||||
)))
|
||||
}
|
||||
|
||||
/// sha256 over `bytes` as lowercase hex (python `hashlib.sha256(...).hexdigest()`).
|
||||
fn sha256_hex(bytes: &[u8]) -> String {
|
||||
hex_encode(Sha256::digest(bytes))
|
||||
}
|
||||
|
||||
/// Truncate to the first 200 chars (python `text[:200]` counts codepoints).
|
||||
fn truncate_200(s: &str) -> String {
|
||||
s.chars().take(200).collect()
|
||||
}
|
||||
|
||||
/// GoFile API client that reuses one guest account token for the whole run.
|
||||
pub struct GoFileClient {
|
||||
password_hash: Option<String>,
|
||||
token: Option<String>,
|
||||
session: Client,
|
||||
}
|
||||
|
||||
impl GoFileClient {
|
||||
/// `password` is hashed (sha256 hex) once; the API wants the hash.
|
||||
pub fn new(password: Option<&str>) -> Self {
|
||||
let password_hash = password.map(|p| sha256_hex(p.as_bytes()));
|
||||
// Client-level timeouts mirror python's download timeout tuple
|
||||
// (connect 10s); the read-side 60s from python's (10, 60) is not
|
||||
// expressible in reqwest blocking, so the download request gets a
|
||||
// generous total timeout instead (see download_file).
|
||||
let session = Client::builder()
|
||||
.default_headers(common_headers())
|
||||
.connect_timeout(Duration::from_secs(10))
|
||||
.build()
|
||||
.unwrap_or_else(|e| panic!("failed to build HTTP client: {e}"));
|
||||
Self {
|
||||
password_hash,
|
||||
token: None,
|
||||
session,
|
||||
}
|
||||
}
|
||||
|
||||
/// Create a guest account; backs off and retries on rate limiting.
|
||||
/// Defaults match python: 8 attempts, 20s × attempt between them.
|
||||
pub fn ensure_token(&mut self) -> Result<String, GoFileError> {
|
||||
self.ensure_token_with(8, 20)
|
||||
}
|
||||
|
||||
/// Same with explicit attempt/wait parameters (python keyword args).
|
||||
pub fn ensure_token_with(
|
||||
&mut self,
|
||||
max_attempts: u32,
|
||||
wait_seconds: u64,
|
||||
) -> Result<String, GoFileError> {
|
||||
if let Some(token) = &self.token {
|
||||
return Ok(token.clone());
|
||||
}
|
||||
|
||||
let mut last_status = "unknown".to_string();
|
||||
for attempt in 1..=max_attempts {
|
||||
match self.try_create_account() {
|
||||
Ok(token) => {
|
||||
self.token = Some(token.clone());
|
||||
log::info!("GoFile guest account created (token reused for whole run)");
|
||||
return Ok(token);
|
||||
}
|
||||
Err(status) => {
|
||||
last_status = status;
|
||||
log::warn!(
|
||||
"GoFile account creation attempt {}/{} failed: {}",
|
||||
attempt,
|
||||
max_attempts,
|
||||
last_status
|
||||
);
|
||||
}
|
||||
}
|
||||
if attempt < max_attempts {
|
||||
std::thread::sleep(Duration::from_secs(wait_seconds * u64::from(attempt)));
|
||||
}
|
||||
}
|
||||
Err(GoFileError::Error(format!(
|
||||
"GoFile account creation failed after {max_attempts} attempts (last: {last_status})"
|
||||
)))
|
||||
}
|
||||
|
||||
/// POST /accounts with the common headers; returns the token or the
|
||||
/// status string that describes the failure (python's `last_status`).
|
||||
fn try_create_account(&self) -> Result<String, String> {
|
||||
let resp = self
|
||||
.session
|
||||
.post(format!("{API}/accounts"))
|
||||
.headers(common_headers())
|
||||
.timeout(Duration::from_secs(15))
|
||||
.send();
|
||||
let data: serde_json::Value = match resp {
|
||||
Ok(r) => match r.json() {
|
||||
Ok(v) => v,
|
||||
Err(e) => return Err(format!("request error: {e}")),
|
||||
},
|
||||
Err(e) => return Err(format!("request error: {e}")),
|
||||
};
|
||||
if data.get("status").and_then(|s| s.as_str()) == Some("ok") {
|
||||
return data
|
||||
.pointer("/data/token")
|
||||
.and_then(|t| t.as_str())
|
||||
.map(str::to_string)
|
||||
.ok_or_else(|| "request error: missing data.token".to_string());
|
||||
}
|
||||
// python: str(data.get("status", data)[:200] if data else data)
|
||||
let status = match data.get("status") {
|
||||
Some(serde_json::Value::String(s)) => s.clone(),
|
||||
Some(other) => other.to_string(),
|
||||
None => serde_json::to_string(&data).unwrap_or_default(),
|
||||
};
|
||||
Err(truncate_200(&status))
|
||||
}
|
||||
|
||||
/// Drop the current token so the next `ensure_token()` creates a new one.
|
||||
pub fn reset(&mut self) {
|
||||
self.token = None;
|
||||
}
|
||||
|
||||
/// sha256 over `"{UA}::{LOCALE}::{token}::{window}::{TOKEN_SECRET}"`.
|
||||
/// `window` is an argument so tests can pin it.
|
||||
fn website_token_for(&self, window: u64) -> String {
|
||||
// python f-string renders a missing token as "None".
|
||||
let token = self.token.as_deref().unwrap_or("None");
|
||||
let seed = format!("{USER_AGENT}::{LOCALE}::{token}::{window}::{TOKEN_SECRET}");
|
||||
sha256_hex(seed.as_bytes())
|
||||
}
|
||||
|
||||
/// Website token for the current 4h window (python `_website_token`).
|
||||
fn website_token(&self) -> String {
|
||||
let window = SystemTime::now()
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.map(|d| d.as_secs())
|
||||
.unwrap_or(0)
|
||||
/ TOKEN_WINDOW_SEC;
|
||||
self.website_token_for(window)
|
||||
}
|
||||
|
||||
/// EXTENDED_HEADERS + Authorization, X-Website-Token, X-BL.
|
||||
fn auth_headers(&self) -> HeaderMap {
|
||||
let mut headers = extended_headers();
|
||||
let bearer = format!("Bearer {}", self.token.as_deref().unwrap_or("None"));
|
||||
set_header(&mut headers, AUTHORIZATION, &bearer);
|
||||
set_header(
|
||||
&mut headers,
|
||||
header_name("x-website-token"),
|
||||
&self.website_token(),
|
||||
);
|
||||
set_header(&mut headers, header_name("x-bl"), LOCALE);
|
||||
headers
|
||||
}
|
||||
|
||||
/// Return (relpath, filename, download_link) for every file under a URL.
|
||||
///
|
||||
/// Direct files at the content root get their own folder named after the
|
||||
/// content id (empty relpath → cid, like the python normalization).
|
||||
pub fn list_files(&self, url: &str) -> Result<Vec<(String, String, String)>, GoFileError> {
|
||||
let cid = content_id_from_url(url)?;
|
||||
let mut results = Vec::new();
|
||||
self.walk(&cid, &[], &mut results)?;
|
||||
Ok(results
|
||||
.into_iter()
|
||||
.map(|(relpath, name, link)| {
|
||||
let relpath = if relpath.is_empty() {
|
||||
cid.clone()
|
||||
} else {
|
||||
relpath
|
||||
};
|
||||
(relpath, name, link)
|
||||
})
|
||||
.collect())
|
||||
}
|
||||
|
||||
/// Recursive walk of /contents/<id>, mirroring python `_walk`.
|
||||
fn walk(
|
||||
&self,
|
||||
cid: &str,
|
||||
prefix: &[String],
|
||||
results: &mut Vec<(String, String, String)>,
|
||||
) -> Result<(), GoFileError> {
|
||||
let mut query = String::from("?cache=true&sortField=createTime&sortDirection=1");
|
||||
if let Some(password_hash) = &self.password_hash {
|
||||
query.push_str("&password=");
|
||||
query.push_str(password_hash);
|
||||
}
|
||||
let resp = self
|
||||
.session
|
||||
.get(format!("{API}/contents/{cid}{query}"))
|
||||
.headers(self.auth_headers())
|
||||
.timeout(Duration::from_secs(30))
|
||||
.send()
|
||||
.map_err(|e| GoFileError::Error(format!("content request for {cid} failed: {e}")))?;
|
||||
|
||||
if resp.status() == reqwest::StatusCode::NOT_FOUND {
|
||||
return Err(GoFileError::NotFound(format!(
|
||||
"content {cid} not found (dead link)"
|
||||
)));
|
||||
}
|
||||
let status_code = resp.status();
|
||||
if status_code == reqwest::StatusCode::UNAUTHORIZED
|
||||
|| status_code == reqwest::StatusCode::FORBIDDEN
|
||||
{
|
||||
return Err(GoFileError::AuthError(format!(
|
||||
"content request denied ({status_code}) for {cid}"
|
||||
)));
|
||||
}
|
||||
let text = resp
|
||||
.text()
|
||||
.map_err(|e| GoFileError::Error(format!("content request for {cid} failed: {e}")))?;
|
||||
let data: serde_json::Value = serde_json::from_str(&text).map_err(|_| {
|
||||
GoFileError::Error(format!(
|
||||
"non-JSON response for {cid}: {}",
|
||||
truncate_200(&text)
|
||||
))
|
||||
})?;
|
||||
|
||||
// python: data.get("status") == "notFound" or "notFound" in str(status)
|
||||
let status = json_status(&data);
|
||||
if status.contains("notFound") {
|
||||
return Err(GoFileError::NotFound(format!(
|
||||
"content {cid} does not exist anymore (dead link)"
|
||||
)));
|
||||
}
|
||||
if status != "ok" {
|
||||
return Err(GoFileError::Error(format!(
|
||||
"content request failed for {cid}: {status}"
|
||||
)));
|
||||
}
|
||||
|
||||
let d = match data.get("data") {
|
||||
Some(d @ serde_json::Value::Object(_)) => d,
|
||||
_ => {
|
||||
return Err(GoFileError::Error(format!(
|
||||
"content request failed for {cid}: missing data object"
|
||||
)));
|
||||
}
|
||||
};
|
||||
|
||||
let password_ok = match d.get("passwordStatus") {
|
||||
None | Some(serde_json::Value::Null) => true,
|
||||
Some(serde_json::Value::String(s)) => s == "passwordOk",
|
||||
Some(_) => false,
|
||||
};
|
||||
if !password_ok {
|
||||
return Err(GoFileError::PasswordRequired(format!(
|
||||
"content {cid} is password protected (use --password)"
|
||||
)));
|
||||
}
|
||||
|
||||
if d.get("type").and_then(|t| t.as_str()) == Some("folder") {
|
||||
let name = required_str(d, "name", cid)?;
|
||||
let mut here = prefix.to_vec();
|
||||
here.push(name);
|
||||
// serde_json maps children to a BTreeMap, so children are visited
|
||||
// in key-sorted order instead of the API's insertion order (the
|
||||
// orchestrator downloads all of them either way).
|
||||
let children = d.get("children").and_then(|c| c.as_object());
|
||||
if let Some(children) = children {
|
||||
for (_key, child) in children {
|
||||
if child.get("type").and_then(|t| t.as_str()) == Some("folder") {
|
||||
let child_id = required_str(child, "id", cid)?;
|
||||
self.walk(&child_id, &here, results)?;
|
||||
} else {
|
||||
let child_name = required_str(child, "name", cid)?;
|
||||
let link = required_str(child, "link", cid)?;
|
||||
results.push((here.join("/"), child_name, link));
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
let name = required_str(d, "name", cid)?;
|
||||
let link = required_str(d, "link", cid)?;
|
||||
results.push((prefix.join("/"), name, link));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Download one file; returns the file size. Skips existing files.
|
||||
///
|
||||
/// Deviation from python's `timeout=(10, 60)`: reqwest blocking cannot
|
||||
/// express "connect 10s / read 60s" per request. Connect is capped at
|
||||
/// 10s client-wide; the download request gets a generous 2h total
|
||||
/// timeout instead of python's uncapped streaming (60s per read) —
|
||||
/// a stalled connection now aborts after at most 2h instead of 60s.
|
||||
pub fn download_file(&self, link: &str, dest: &Path) -> Result<u64, GoFileError> {
|
||||
if let Some(parent) = dest.parent() {
|
||||
fs::create_dir_all(parent).map_err(|e| {
|
||||
GoFileError::Error(format!("cannot create {}: {e}", parent.display()))
|
||||
})?;
|
||||
}
|
||||
if dest.exists() {
|
||||
let size = fs::metadata(dest).map(|m| m.len()).unwrap_or(0);
|
||||
if size > 0 {
|
||||
log::info!("Skip existing: {}", dest.display());
|
||||
return Ok(size);
|
||||
}
|
||||
}
|
||||
|
||||
let name = dest
|
||||
.file_name()
|
||||
.map(|n| n.to_string_lossy().into_owned())
|
||||
.unwrap_or_default();
|
||||
let tmp = dest.with_file_name(format!("{name}.part"));
|
||||
|
||||
let result = self.download_to_tmp(link, dest, &tmp, &name);
|
||||
// python `finally`: never leave a .part file behind on failure.
|
||||
if tmp.exists() && !dest.exists() {
|
||||
let _ = fs::remove_file(&tmp);
|
||||
}
|
||||
result
|
||||
}
|
||||
|
||||
fn download_to_tmp(
|
||||
&self,
|
||||
link: &str,
|
||||
dest: &Path,
|
||||
tmp: &Path,
|
||||
name: &str,
|
||||
) -> Result<u64, GoFileError> {
|
||||
let mut headers = extended_headers();
|
||||
set_header(
|
||||
&mut headers,
|
||||
COOKIE,
|
||||
&format!("accountToken={}", self.token.as_deref().unwrap_or("None")),
|
||||
);
|
||||
set_header(&mut headers, REFERER, link);
|
||||
set_header(&mut headers, ORIGIN, link);
|
||||
|
||||
let mut resp = self
|
||||
.session
|
||||
.get(link)
|
||||
.headers(headers)
|
||||
// python: timeout=(10, 60) — per-read 60s is not expressible in
|
||||
// reqwest blocking, so use a generous total timeout instead.
|
||||
.timeout(Duration::from_secs(2 * 3600))
|
||||
.send()
|
||||
.map_err(|e| GoFileError::Error(format!("download of {name} failed: {e}")))?;
|
||||
|
||||
let status = resp.status();
|
||||
if status == reqwest::StatusCode::NOT_FOUND {
|
||||
return Err(GoFileError::NotFound(format!(
|
||||
"file {name} not found (dead link)"
|
||||
)));
|
||||
}
|
||||
if status != reqwest::StatusCode::OK {
|
||||
return Err(GoFileError::Error(format!(
|
||||
"HTTP {status} while downloading {name}"
|
||||
)));
|
||||
}
|
||||
|
||||
let mut file = fs::File::create(tmp)
|
||||
.map_err(|e| GoFileError::Error(format!("cannot create {}: {e}", tmp.display())))?;
|
||||
let mut written = 0u64;
|
||||
let mut buf = vec![0u8; CHUNK_SIZE];
|
||||
loop {
|
||||
let n = resp
|
||||
.read(&mut buf)
|
||||
.map_err(|e| GoFileError::Error(format!("download of {name} failed: {e}")))?;
|
||||
if n == 0 {
|
||||
break;
|
||||
}
|
||||
file.write_all(&buf[..n])
|
||||
.map_err(|e| GoFileError::Error(format!("cannot write {name}: {e}")))?;
|
||||
written += n as u64;
|
||||
}
|
||||
file.flush()
|
||||
.map_err(|e| GoFileError::Error(format!("cannot write {name}: {e}")))?;
|
||||
|
||||
if written == 0 {
|
||||
return Err(GoFileError::Error(format!("empty body for {name}")));
|
||||
}
|
||||
// os.replace semantics: rename over an existing destination.
|
||||
fs::rename(tmp, dest)
|
||||
.map_err(|e| GoFileError::Error(format!("cannot move {name} into place: {e}")))?;
|
||||
Ok(written)
|
||||
}
|
||||
}
|
||||
|
||||
/// python `str(data.get("status", ""))` — missing status renders as "None".
|
||||
fn json_status(data: &serde_json::Value) -> String {
|
||||
match data.get("status") {
|
||||
None => "None".to_string(),
|
||||
Some(serde_json::Value::String(s)) => s.clone(),
|
||||
Some(other) => other.to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Fetch a required string field; a missing field mirrors python's KeyError
|
||||
/// (which would surface to the orchestrator as an unexpected failure).
|
||||
fn required_str(value: &serde_json::Value, key: &str, cid: &str) -> Result<String, GoFileError> {
|
||||
value
|
||||
.get(key)
|
||||
.and_then(|v| v.as_str())
|
||||
.map(str::to_string)
|
||||
.ok_or_else(|| {
|
||||
GoFileError::Error(format!("content {cid}: malformed entry (missing {key})"))
|
||||
})
|
||||
}
|
||||
|
||||
fn header_name(name: &'static str) -> HeaderName {
|
||||
HeaderName::from_static(name)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn content_id_from_url_accepts_gofile_d_links() {
|
||||
assert_eq!(
|
||||
content_id_from_url("https://gofile.io/d/ABC123").unwrap(),
|
||||
"ABC123"
|
||||
);
|
||||
// query string, fragment, trailing slash, deeper path
|
||||
assert_eq!(
|
||||
content_id_from_url("https://gofile.io/d/ABC123?x=1").unwrap(),
|
||||
"ABC123"
|
||||
);
|
||||
assert_eq!(
|
||||
content_id_from_url("https://gofile.io/d/ABC123/#anchor").unwrap(),
|
||||
"ABC123"
|
||||
);
|
||||
// python urlparse is lenient about a missing scheme
|
||||
assert_eq!(content_id_from_url("gofile.io/d/ABC123").unwrap(), "ABC123");
|
||||
assert_eq!(
|
||||
content_id_from_url("gofile.io/d/ABC123?x=1").unwrap(),
|
||||
"ABC123"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn content_id_from_url_rejects_non_content_urls() {
|
||||
let err = content_id_from_url("https://gofile.io/ABC123").unwrap_err();
|
||||
assert!(
|
||||
matches!(err, GoFileError::Error(ref m)
|
||||
if m == "not a gofile content URL (expected gofile.io/d/<id>): https://gofile.io/ABC123"),
|
||||
"unexpected: {err}"
|
||||
);
|
||||
assert!(content_id_from_url("https://example.com/foo/bar").is_err());
|
||||
assert!(content_id_from_url("https://gofile.io/d/").is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn website_token_for_is_deterministic_and_matches_python() {
|
||||
let mut client = GoFileClient::new(None);
|
||||
client.token = Some("testtoken".to_string());
|
||||
// golden value computed with python hashlib for the same seed
|
||||
assert_eq!(
|
||||
client.website_token_for(0),
|
||||
"1791d396eab9b0ef100bcb3c9211c80973ee8dd9b143fc9f388ced6a0fb3b0c0"
|
||||
);
|
||||
// stable across calls, 64 lowercase hex chars
|
||||
let again = client.website_token_for(0);
|
||||
assert_eq!(client.website_token_for(0), again);
|
||||
assert_eq!(again.len(), 64);
|
||||
assert!(
|
||||
again
|
||||
.bytes()
|
||||
.all(|b| b.is_ascii_hexdigit() && !b.is_ascii_uppercase())
|
||||
);
|
||||
// different window / token → different token
|
||||
assert_ne!(client.website_token_for(1), client.website_token_for(0));
|
||||
client.token = None;
|
||||
// python renders a missing token as "None"
|
||||
assert_eq!(
|
||||
client.website_token_for(7),
|
||||
"fb9a698ec598396eb77ab8b562fb1e30f01401dcf793701b290585e6393a657f"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn header_maps_match_python_keys() {
|
||||
let common = common_headers();
|
||||
let mut keys: Vec<String> = common.keys().map(|k| k.as_str().to_string()).collect();
|
||||
keys.sort();
|
||||
assert_eq!(
|
||||
keys,
|
||||
["accept", "accept-encoding", "connection", "user-agent"]
|
||||
);
|
||||
assert_eq!(common.get("Accept-Encoding").unwrap(), "gzip");
|
||||
|
||||
let extended = extended_headers();
|
||||
let mut ekeys: Vec<String> = extended.keys().map(|k| k.as_str().to_string()).collect();
|
||||
ekeys.sort();
|
||||
assert_eq!(
|
||||
ekeys,
|
||||
[
|
||||
"accept",
|
||||
"accept-encoding",
|
||||
"cache-control",
|
||||
"connection",
|
||||
"pragma",
|
||||
"sec-fetch-dest",
|
||||
"sec-fetch-mode",
|
||||
"sec-fetch-site",
|
||||
"user-agent",
|
||||
]
|
||||
);
|
||||
assert_eq!(extended.get("Sec-Fetch-Site").unwrap(), "same-site");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn error_display_is_the_message() {
|
||||
assert_eq!(
|
||||
GoFileError::NotFound("content x not found (dead link)".into()).to_string(),
|
||||
"content x not found (dead link)"
|
||||
);
|
||||
assert_eq!(GoFileError::PasswordRequired("pw".into()).to_string(), "pw");
|
||||
assert_eq!(GoFileError::Empty("empty".into()).to_string(), "empty");
|
||||
assert_eq!(GoFileError::AuthError("auth".into()).to_string(), "auth");
|
||||
assert_eq!(GoFileError::Error("base".into()).to_string(), "base");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sha256_hex_matches_python_hexdigest() {
|
||||
// hashlib.sha256(b"abc").hexdigest()
|
||||
assert_eq!(
|
||||
sha256_hex(b"abc"),
|
||||
"ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn truncate_200_counts_chars_not_bytes() {
|
||||
let s = "ä".repeat(300);
|
||||
assert_eq!(truncate_200(&s).chars().count(), 200);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,2 +1,227 @@
|
||||
//! curl-based downloader for 0807.st direct links — Rust port of
|
||||
//! `version-python/dlphelper/zero807.py`.
|
||||
//!
|
||||
//! 0807.st serves the files directly but rejects requests that don't look
|
||||
//! like a browser (requires a User-Agent and the Referer of the site page),
|
||||
//! and it rejects multi-connection/range abuse patterns differently than
|
||||
//! curl — which is why curl is used instead of aria2c here (see python
|
||||
//! module docstring: curl worked, aria2c's default request got 403).
|
||||
//!
|
||||
//! Downloads go to a `.part` file first and are renamed on success, so an
|
||||
//! interrupted download is resumable (`--continue-at -`) and partial files
|
||||
//! are never mistaken for finished ones.
|
||||
|
||||
use std::fs;
|
||||
use std::io::Read;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::process::Command;
|
||||
|
||||
pub const USER_AGENT: &str = "Mozilla/5.0";
|
||||
pub const REFERER: &str = "https://0807.st/";
|
||||
|
||||
/// Error type mirroring the python exception classes. `NotFound` (dead link,
|
||||
/// HTTP 404/410) is a final state the orchestrator never retries.
|
||||
#[allow(dead_code)] // used by the orchestrator, not yet wired up
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub enum Zero807Error {
|
||||
/// python `Zero807Error` (base): transient/unclassified failure.
|
||||
Error(String),
|
||||
/// python `Zero807NotFound`: dead link, never retry.
|
||||
NotFound(String),
|
||||
}
|
||||
|
||||
impl std::fmt::Display for Zero807Error {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
match self {
|
||||
Zero807Error::Error(m) | Zero807Error::NotFound(m) => write!(f, "{m}"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl std::error::Error for Zero807Error {}
|
||||
|
||||
/// Derive the filename from the URL path (e.g. /6saCNie.zip -> 6saCNie.zip).
|
||||
///
|
||||
/// Matches python `Path(urlparse(url).path).name`: the query string is not
|
||||
/// part of the path, a trailing slash still yields the last real segment
|
||||
/// (`.../a/b/` → `b`), and an empty result is an error.
|
||||
pub fn target_name(url: &str) -> Result<String, Zero807Error> {
|
||||
let path = match url::Url::parse(url) {
|
||||
Ok(u) => u.path().to_string(),
|
||||
// python urlparse is lenient about a missing scheme but still strips
|
||||
// query/fragment; mirror that.
|
||||
Err(_) => url.split(['?', '#']).next().unwrap_or(url).to_string(),
|
||||
};
|
||||
let name = Path::new(&path)
|
||||
.file_name()
|
||||
.map(|n| n.to_string_lossy().into_owned())
|
||||
.unwrap_or_default();
|
||||
if name.is_empty() {
|
||||
return Err(Zero807Error::Error(format!(
|
||||
"cannot derive a filename from URL: {url}"
|
||||
)));
|
||||
}
|
||||
Ok(name)
|
||||
}
|
||||
|
||||
/// Download one URL with curl; returns (final_path, size_bytes).
|
||||
///
|
||||
/// Defaults match python: `timeout_seconds = 7200`, `resume = true`.
|
||||
/// Use `download_with` for explicit parameters.
|
||||
pub fn download(url: &str, out_dir: &Path) -> Result<(PathBuf, u64), Zero807Error> {
|
||||
download_with(url, out_dir, 7200, true)
|
||||
}
|
||||
|
||||
/// Same with explicit timeout/resume parameters (python keyword args).
|
||||
pub fn download_with(
|
||||
url: &str,
|
||||
out_dir: &Path,
|
||||
timeout_seconds: u64,
|
||||
resume: bool,
|
||||
) -> Result<(PathBuf, u64), Zero807Error> {
|
||||
let name = target_name(url)?;
|
||||
fs::create_dir_all(out_dir)
|
||||
.map_err(|e| Zero807Error::Error(format!("cannot create {}: {e}", out_dir.display())))?;
|
||||
let dest = out_dir.join(&name);
|
||||
let tmp = dest.with_file_name(format!("{name}.part"));
|
||||
|
||||
if dest.exists() {
|
||||
let size = fs::metadata(&dest).map(|m| m.len()).unwrap_or(0);
|
||||
if size > 0 {
|
||||
log::info!("Skip existing: {}", dest.display());
|
||||
return Ok((dest, size));
|
||||
}
|
||||
}
|
||||
|
||||
let mut cmd = Command::new("curl");
|
||||
cmd.args([
|
||||
"-fSL",
|
||||
"--connect-timeout",
|
||||
"20",
|
||||
"--max-time",
|
||||
&timeout_seconds.to_string(),
|
||||
"--retry",
|
||||
"2",
|
||||
"--retry-delay",
|
||||
"5",
|
||||
"-A",
|
||||
USER_AGENT,
|
||||
"-e",
|
||||
REFERER,
|
||||
"-o",
|
||||
]);
|
||||
cmd.arg(&tmp);
|
||||
if resume {
|
||||
cmd.arg("--continue-at").arg("-");
|
||||
}
|
||||
cmd.arg(url);
|
||||
|
||||
log::info!("Downloading {} -> {}", url, name);
|
||||
let proc = cmd.output().map_err(|e| {
|
||||
// python: subprocess raises FileNotFoundError, which the
|
||||
// orchestrator's catch-all would record; wrap it here.
|
||||
Zero807Error::Error(format!("failed to run curl: {e}"))
|
||||
})?;
|
||||
|
||||
if proc.status.code() != Some(0) {
|
||||
let returncode = proc.status.code().unwrap_or(-1);
|
||||
let stderr = String::from_utf8_lossy(&proc.stderr);
|
||||
let tail: Vec<&str> = stderr
|
||||
.trim()
|
||||
.lines()
|
||||
.filter(|line| !line.is_empty())
|
||||
.collect();
|
||||
let start = tail.len().saturating_sub(3);
|
||||
let stderr = tail[start..].join(" | ");
|
||||
// -f makes curl exit 22 on any HTTP error; dead links fail for good
|
||||
if stderr.contains("404") || stderr.contains("410") {
|
||||
return Err(Zero807Error::NotFound(format!(
|
||||
"dead link (HTTP 404/410): {stderr}"
|
||||
)));
|
||||
}
|
||||
return Err(Zero807Error::Error(format!(
|
||||
"curl exit code {returncode}: {stderr}"
|
||||
)));
|
||||
}
|
||||
|
||||
let size = fs::metadata(&tmp).map(|m| m.len()).unwrap_or(0);
|
||||
if size == 0 {
|
||||
let _ = fs::remove_file(&tmp); // python unlink(missing_ok=True)
|
||||
return Err(Zero807Error::Error(format!("empty file for {name}")));
|
||||
}
|
||||
if dest
|
||||
.extension()
|
||||
.map(|e| e.to_string_lossy().to_lowercase())
|
||||
.as_deref()
|
||||
== Some("zip")
|
||||
{
|
||||
// python reads the whole file just to peek at 2 bytes; reading only
|
||||
// the first two bytes is behavior-identical and cheap.
|
||||
let mut first = [0u8; 2];
|
||||
let n = fs::File::open(&tmp)
|
||||
.and_then(|mut fh| fh.read(&mut first))
|
||||
.unwrap_or(0);
|
||||
if first[..n] != *b"PK" {
|
||||
log::warn!("{name} does not look like a ZIP file (first bytes are not PK)");
|
||||
}
|
||||
}
|
||||
// os.replace semantics: rename over an existing destination.
|
||||
fs::rename(&tmp, &dest)
|
||||
.map_err(|e| Zero807Error::Error(format!("cannot move {name} into place: {e}")))?;
|
||||
Ok((dest, size))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn target_name_from_path_last_segment() {
|
||||
assert_eq!(
|
||||
target_name("https://0807.st/6saCNie.zip").unwrap(),
|
||||
"6saCNie.zip"
|
||||
);
|
||||
assert_eq!(
|
||||
target_name("https://0807.st/a/b/filename.bin").unwrap(),
|
||||
"filename.bin"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn target_name_drops_query_and_fragment() {
|
||||
// python takes Path(urlparse(url).path).name: query/fragment dropped
|
||||
assert_eq!(
|
||||
target_name("https://0807.st/file.zip?x=1").unwrap(),
|
||||
"file.zip"
|
||||
);
|
||||
assert_eq!(
|
||||
target_name("https://0807.st/file.zip?x=1#frag").unwrap(),
|
||||
"file.zip"
|
||||
);
|
||||
// schemeless URLs parse like python urlparse (query still stripped)
|
||||
assert_eq!(target_name("0807.st/file.zip?x=1").unwrap(), "file.zip");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn target_name_trailing_slash_matches_pathlib() {
|
||||
// pathlib: Path("/a/b/").name == "b" (trailing slash ignored)
|
||||
assert_eq!(target_name("https://0807.st/a/b/").unwrap(), "b");
|
||||
assert_eq!(
|
||||
target_name("https://0807.st/").unwrap_err().to_string(),
|
||||
"cannot derive a filename from URL: https://0807.st/"
|
||||
);
|
||||
assert_eq!(
|
||||
target_name("https://0807.st").unwrap_err().to_string(),
|
||||
"cannot derive a filename from URL: https://0807.st"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn error_display_is_the_message() {
|
||||
assert_eq!(
|
||||
Zero807Error::NotFound("dead link (HTTP 404/410): 404".into()).to_string(),
|
||||
"dead link (HTTP 404/410): 404"
|
||||
);
|
||||
assert_eq!(Zero807Error::Error("boom".into()).to_string(), "boom");
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user