Ben_Kosytorz
6c6a3273bb
The Rust rewrite has replaced the original python implementation, so the transition layout is no longer needed:
- move version-rust/{Cargo.toml,Cargo.lock,README.md,.gitignore,src,data} to the repo root; the crate is now the repo.
- move docs/SORTING.md and data/gwshop_catalog.json to the root.
- remove version-python/ entirely, plus the dev-only version-rust/tools/dump_warhammer_data.py (its only job was regenerating data/warhammer_data.json from python).
- rewrite AGENTS.md and README.md for the single-project layout; strip all version-python/version-rust references from code comments.
- fix the gwshop credential lookup: algolia.json now sits at CARGO_MANIFEST_DIR (crate root == repo root), not its parent.
- restore the credentials/run-output ignores in .gitignore (algolia.json, .env, *.token, downloads/, state.json, manifest.jsonl, ...).
- docs/SORTING.md: retarget the extraction/sorter/dlp examples at the dlphelper binary instead of the deleted python scripts.
cargo test: 86/86 green.
1293 lines
48 KiB
Rust
1293 lines
48 KiB
Rust
//! Recursive archive extraction into a plain-file tree — Rust port of
|
|
//! `tools/extract_all.py` (extract_manifest.jsonl feeds the
|
|
//! sorter's provenance mode; failures go to .broken/; the run never stops).
|
|
//!
|
|
//! Extraction backends: `sevenz-rust2` for .7z, `zip` for .zip, `unrar` for
|
|
//! .rar — pure Rust, no external binaries. Tarballs and bare compressed
|
|
//! files (.tar/.tar.gz/.tgz/.tar.bz2/.tar.xz/.gz/.bz2/.xz) are handled by the
|
|
//! system `tar`/`gzip`/`bzip2`/`xz` binaries, exactly like the python
|
|
//! version shelled out to `7z`: merged output capture, exit-code parity
|
|
//! (0 ok, 1 warnings), and the 4h per-archive timeout. The tar/flate2/
|
|
//! bzip2/xz2 crates mentioned in the port plan are not declared
|
|
//! dependencies of this crate and Cargo.toml is off-limits, so the system
|
|
//! binaries are the parity-preserving fallback (they are universally
|
|
//! present on the target Linux hosts).
|
|
//!
|
|
//! Because the zip/7z paths are pure Rust, the python version's /var/tmp
|
|
//! fallback + cp437 name repair is not needed here (the zip crate decodes
|
|
//! legacy cp437 entry names itself).
|
|
//!
|
|
//! Behavior parity notes (vs. the python version):
|
|
//! - Skip decisions are made on manifest records only, never on "dir
|
|
//! non-empty"; a partial dir from a crash is re-extracted.
|
|
//! - Source roots are never modified except for the quarantine move of
|
|
//! failed archives into `<out>/.broken/`.
|
|
//! - The extracted tree holds no archives afterwards (nested archives are
|
|
//! expanded BFS-style and their copies unlinked).
|
|
//! - Plain files are hardlinked in (fallback: metadata-preserving copy).
|
|
//! - No individual failure ever aborts the run.
|
|
|
|
use std::collections::{HashSet, VecDeque};
|
|
use std::fs::{self, File};
|
|
use std::io::{self, Read, Write};
|
|
use std::path::{Component, Path, PathBuf};
|
|
use std::process::{Command, Stdio};
|
|
use std::time::{Duration, Instant};
|
|
|
|
#[cfg(unix)]
|
|
use std::os::unix::fs::PermissionsExt;
|
|
|
|
/// Archive suffixes recognized by the extractors (python `ARCHIVE_SUFFIXES`,
|
|
/// plus `.tgz` — a Rust-only extension; the python version never extracted
|
|
/// those). `.tgz` is a gzip-compressed tar, so it maps onto the tarball path.
|
|
pub const ARCHIVE_SUFFIXES: [&str; 8] =
|
|
[".zip", ".7z", ".rar", ".tar", ".gz", ".bz2", ".xz", ".tgz"];
|
|
|
|
/// Stop extracting below this free space (python `MIN_FREE_BYTES`).
|
|
pub const MIN_FREE_BYTES: u64 = 20 * 1024 * 1024 * 1024;
|
|
|
|
/// Per-archive extraction timeout in seconds (python `EXTRACT_TIMEOUT`).
|
|
const EXTRACT_TIMEOUT_SECS: u64 = 4 * 3600;
|
|
|
|
/// Run artifacts that must never be treated as payload (python `RUN_ARTIFACTS`).
|
|
pub const RUN_ARTIFACTS: [&str; 14] = [
|
|
"export.json",
|
|
"export.html",
|
|
"state.json",
|
|
"manifest.jsonl",
|
|
"failed_urls.txt",
|
|
"catalogue.md",
|
|
"catalogue.jsonl",
|
|
"sorter_applied.jsonl",
|
|
"catalogue_moves.jsonl",
|
|
"urls.txt",
|
|
"urls_0807.txt",
|
|
"urls_gofile.txt",
|
|
"failed_0807.txt",
|
|
"extract_manifest.jsonl",
|
|
];
|
|
|
|
/// Recursive extractor writing into `out_root`.
|
|
pub struct Extractor {
|
|
pub out_root: PathBuf,
|
|
}
|
|
|
|
/// Python `Path.suffix` semantics for a plain file name: the trailing dot
|
|
/// segment, unless the name starts with a dot (hidden) or ends with a dot.
|
|
fn name_suffix(name: &str) -> Option<&str> {
|
|
let idx = name.rfind('.')?;
|
|
if idx == 0 || idx == name.len() - 1 {
|
|
return None;
|
|
}
|
|
Some(&name[idx..])
|
|
}
|
|
|
|
/// All trailing archive suffixes, lowercased, outermost first — e.g.
|
|
/// `x.tar.gz` -> `[".gz", ".tar"]`.
|
|
pub fn archive_suffixes(name: &str) -> Vec<String> {
|
|
let mut out = Vec::new();
|
|
let mut cur = name;
|
|
while let Some(sfx) = name_suffix(cur) {
|
|
let lower = sfx.to_ascii_lowercase();
|
|
if !ARCHIVE_SUFFIXES.contains(&lower.as_str()) {
|
|
break;
|
|
}
|
|
out.push(lower);
|
|
cur = &cur[..cur.len() - sfx.len()];
|
|
}
|
|
out
|
|
}
|
|
|
|
/// True when the file name carries at least one archive suffix.
|
|
pub fn is_archive(name: &str) -> bool {
|
|
!archive_suffixes(name).is_empty()
|
|
}
|
|
|
|
/// Sniff the container format from magic bytes (7z/zip/rar). The python
|
|
/// version shells out to the 7z CLI, which sniffs content; extensions are
|
|
/// unreliable in the wild (we have seen real 7z archives named `.zip`).
|
|
fn sniff_container(path: &Path) -> Option<&'static str> {
|
|
use std::io::Read;
|
|
let mut buf = [0u8; 8];
|
|
let mut f = fs::File::open(path).ok()?;
|
|
let n = f.read(&mut buf).ok()?;
|
|
let b = &buf[..n];
|
|
if b.starts_with(&[0x37, 0x7a, 0xbc, 0xaf, 0x27, 0x1c]) {
|
|
return Some(".7z");
|
|
}
|
|
if b.starts_with(b"PK\x03\x04") || b.starts_with(b"PK\x05\x06") {
|
|
return Some(".zip");
|
|
}
|
|
if b.starts_with(b"Rar!\x1a\x07\x00") || b.starts_with(b"Rar!\x1a\x07\x01\x00") {
|
|
return Some(".rar");
|
|
}
|
|
None
|
|
}
|
|
|
|
/// Archive name with all archive suffixes stripped: the unit folder name.
|
|
pub fn display_unit(name: &str) -> String {
|
|
let mut cur = name;
|
|
while let Some(sfx) = name_suffix(cur) {
|
|
let lower = sfx.to_ascii_lowercase();
|
|
if !ARCHIVE_SUFFIXES.contains(&lower.as_str()) {
|
|
break;
|
|
}
|
|
cur = &cur[..cur.len() - sfx.len()];
|
|
}
|
|
cur.to_string()
|
|
}
|
|
|
|
/// Free disk space (bytes) on the file system containing `path`.
|
|
pub fn free_bytes(path: &Path) -> anyhow::Result<u64> {
|
|
Ok(fs4::available_space(path)?)
|
|
}
|
|
|
|
/// AppleDouble resource-fork "zips" (`__MACOSX/._x.zip`) — no content.
|
|
pub fn is_macosx_junk(p: &Path) -> bool {
|
|
p.components().any(|c| c.as_os_str() == "__MACOSX")
|
|
|| p.file_name()
|
|
.map(|n| n.to_string_lossy().starts_with("._"))
|
|
.unwrap_or(false)
|
|
}
|
|
|
|
/// Relative path helper: `p` relative to `root` (falls back to `p` itself).
|
|
fn rel_to(root: &Path, p: &Path) -> PathBuf {
|
|
p.strip_prefix(root).unwrap_or(p).to_path_buf()
|
|
}
|
|
|
|
/// Same as [`rel_to`] as a (lossy) string, matching python `str(rel_path)`.
|
|
fn rel_string(root: &Path, p: &Path) -> String {
|
|
rel_to(root, p).to_string_lossy().into_owned()
|
|
}
|
|
|
|
fn file_name_string(p: &Path) -> String {
|
|
p.file_name()
|
|
.map(|n| n.to_string_lossy().into_owned())
|
|
.unwrap_or_default()
|
|
}
|
|
|
|
/// Recursively collect regular files under `root`, sorted for determinism.
|
|
///
|
|
/// Matches python `sorted(rglob("*"))` + `is_file()` filtering: symlinked
|
|
/// directories are not descended into, symlinks to files count as files.
|
|
fn walk_files_sorted(root: &Path) -> Vec<PathBuf> {
|
|
let mut files = Vec::new();
|
|
let mut dirs = vec![root.to_path_buf()];
|
|
while let Some(dir) = dirs.pop() {
|
|
let Ok(entries) = fs::read_dir(&dir) else {
|
|
continue;
|
|
};
|
|
for entry in entries.flatten() {
|
|
let path = entry.path();
|
|
match fs::symlink_metadata(&path) {
|
|
Ok(meta) if meta.is_dir() => dirs.push(path),
|
|
Ok(meta) if meta.is_file() => files.push(path),
|
|
Ok(meta) if meta.is_symlink() => {
|
|
if let Ok(target) = fs::metadata(&path) {
|
|
if target.is_file() {
|
|
files.push(path);
|
|
}
|
|
}
|
|
}
|
|
_ => {}
|
|
}
|
|
}
|
|
}
|
|
files.sort();
|
|
files
|
|
}
|
|
|
|
/// One provenance manifest line, compact JSON, raw UTF-8 (python
|
|
/// `json.dumps(..., ensure_ascii=False)`; keys order as archive/dest/unit).
|
|
fn write_manifest_line<W: Write>(
|
|
manifest: &mut W,
|
|
out_root: &Path,
|
|
archive_rel: &Path,
|
|
actual_dest: &Path,
|
|
) {
|
|
let dest_rel = rel_to(out_root, actual_dest);
|
|
let unit = archive_rel
|
|
.file_name()
|
|
.map(|n| display_unit(&n.to_string_lossy()))
|
|
.unwrap_or_default();
|
|
let line = serde_json::to_string(&serde_json::json!({
|
|
"archive": archive_rel.to_string_lossy().into_owned(),
|
|
"dest": dest_rel.to_string_lossy().into_owned(),
|
|
"unit": unit,
|
|
}))
|
|
.unwrap_or_else(|_| "{}".to_string());
|
|
if let Err(e) = writeln!(manifest, "{line}") {
|
|
log::error!("Cannot write manifest line: {e}");
|
|
}
|
|
}
|
|
|
|
/// Run an extraction command like python `_run_7z`: merged stdout+stderr
|
|
/// captured (stderr only when stdout is redirected to a file), tolerant
|
|
/// decoding, per-archive timeout, returns (rc, last-5-lines tail).
|
|
/// rc == -1 marks "failed to run" / timeout, like python.
|
|
fn run_extract_cmd(
|
|
program: &str,
|
|
args: &[std::ffi::OsString],
|
|
stdout_to: Option<&Path>,
|
|
) -> (i32, String) {
|
|
let mut cmd = Command::new(program);
|
|
cmd.args(args);
|
|
let mut stdout_thread = None;
|
|
let mut stderr_thread = None;
|
|
match stdout_to {
|
|
Some(path) => match File::create(path) {
|
|
Ok(f) => {
|
|
cmd.stdout(Stdio::from(f));
|
|
cmd.stderr(Stdio::piped());
|
|
}
|
|
Err(e) => return (-1, format!("{program} failed to run: {e}")),
|
|
},
|
|
None => {
|
|
cmd.stdout(Stdio::piped());
|
|
cmd.stderr(Stdio::piped());
|
|
}
|
|
}
|
|
let mut child = match cmd.spawn() {
|
|
Ok(c) => c,
|
|
Err(e) => return (-1, format!("{program} failed to run: {e}")),
|
|
};
|
|
if let Some(mut pipe) = child.stdout.take() {
|
|
stdout_thread = Some(std::thread::spawn(move || {
|
|
let mut s = String::new();
|
|
let _ = pipe.read_to_string(&mut s);
|
|
s
|
|
}));
|
|
}
|
|
if let Some(mut pipe) = child.stderr.take() {
|
|
stderr_thread = Some(std::thread::spawn(move || {
|
|
let mut s = String::new();
|
|
let _ = pipe.read_to_string(&mut s);
|
|
s
|
|
}));
|
|
}
|
|
let deadline = Instant::now() + Duration::from_secs(EXTRACT_TIMEOUT_SECS);
|
|
let mut timed_out = false;
|
|
let status = loop {
|
|
match child.try_wait() {
|
|
Ok(Some(st)) => break Some(st),
|
|
Ok(None) => {
|
|
if Instant::now() >= deadline {
|
|
let _ = child.kill();
|
|
let _ = child.wait();
|
|
timed_out = true;
|
|
break None;
|
|
}
|
|
std::thread::sleep(Duration::from_millis(25));
|
|
}
|
|
Err(e) => {
|
|
let _ = child.kill();
|
|
let _ = child.wait();
|
|
return (-1, format!("{program} wait failed: {e}"));
|
|
}
|
|
}
|
|
};
|
|
let mut text = String::new();
|
|
if let Some(t) = stdout_thread {
|
|
text.push_str(&t.join().unwrap_or_default());
|
|
}
|
|
if let Some(t) = stderr_thread {
|
|
text.push_str(&t.join().unwrap_or_default());
|
|
}
|
|
if timed_out {
|
|
return (
|
|
-1,
|
|
format!("extraction timed out after {EXTRACT_TIMEOUT_SECS}s"),
|
|
);
|
|
}
|
|
let code = status.and_then(|s| s.code()).unwrap_or(-1);
|
|
let tail = text
|
|
.lines()
|
|
.rev()
|
|
.take(5)
|
|
.collect::<Vec<_>>()
|
|
.into_iter()
|
|
.rev()
|
|
.collect::<Vec<_>>()
|
|
.join("\n");
|
|
(code, tail)
|
|
}
|
|
|
|
impl Extractor {
|
|
fn space_low(&self) -> bool {
|
|
match free_bytes(&self.out_root) {
|
|
Ok(bytes) => bytes < MIN_FREE_BYTES,
|
|
Err(e) => {
|
|
log::warn!(
|
|
"free-space check failed on {}: {e}",
|
|
self.out_root.display()
|
|
);
|
|
false
|
|
}
|
|
}
|
|
}
|
|
|
|
/// Create the destination directory, reproducing the python collision
|
|
/// semantics: if the stem is occupied by a *file*, retry as `<name>_dup`
|
|
/// (python's loop always rebuilds from the original name, so repeated
|
|
/// `_dup` collisions exhaust the 5 attempts); a parent that is a file
|
|
/// (NotADirectoryError) fails immediately.
|
|
fn create_dest(&self, dest: &Path) -> Result<PathBuf, String> {
|
|
let mut actual = dest.to_path_buf();
|
|
for _ in 0..5 {
|
|
match fs::create_dir_all(&actual) {
|
|
Ok(()) => return Ok(actual),
|
|
Err(e) => match e.kind() {
|
|
io::ErrorKind::AlreadyExists => {
|
|
let name = file_name_string(dest);
|
|
actual = dest.with_file_name(format!("{name}_dup"));
|
|
}
|
|
_ => {
|
|
return Err(format!(
|
|
"cannot create destination {}: {e}",
|
|
actual.display()
|
|
));
|
|
}
|
|
},
|
|
}
|
|
}
|
|
Err(format!(
|
|
"cannot create destination {} (repeated collisions)",
|
|
dest.display()
|
|
))
|
|
}
|
|
|
|
/// Extract `archive` into `dest`. Returns (ok, error, dest_used); on a
|
|
/// name collision the destination gets a `_dup` suffix instead of
|
|
/// crashing. Never panics on archive content; every error is reported.
|
|
pub fn extract(&self, archive: &Path, dest: &Path) -> (bool, String, PathBuf) {
|
|
let actual = match self.create_dest(dest) {
|
|
Ok(a) => a,
|
|
Err(err) => return (false, err, dest.to_path_buf()),
|
|
};
|
|
let name = file_name_string(archive);
|
|
let sfx = archive_suffixes(&name);
|
|
// Container formats: trust the magic bytes over the extension (python's
|
|
// 7z CLI sniffs content too). Compressed streams (.gz/.bz2/.xz) and tar
|
|
// stay extension-driven, where the tar-inside logic needs the name.
|
|
let kind: Option<String> = sniff_container(archive)
|
|
.map(str::to_string)
|
|
.or_else(|| sfx.last().cloned());
|
|
let res: Result<(), String> = match kind.as_deref() {
|
|
Some(".7z") => extract_7z(archive, &actual),
|
|
Some(".zip") => extract_zip(archive, &actual),
|
|
Some(".rar") => extract_rar(archive, &actual),
|
|
Some(outer @ (".gz" | ".bz2" | ".xz")) => {
|
|
if sfx.len() >= 2 && sfx[1] == ".tar" {
|
|
// <stem>.tar.gz|bz2|xz — decompress stream and unpack in
|
|
// one go (python does it in two 7z passes via the nested
|
|
// expansion; final tree content is identical).
|
|
extract_tar_cmd(archive, &actual, Some(outer))
|
|
} else {
|
|
// Bare compressed file (one layer, like a single `7z x`):
|
|
// writes dest/<name-without-suffix>; anything that is
|
|
// still an archive afterwards is picked up by the
|
|
// nested/sweep expansion.
|
|
decompress_bare_cmd(archive, &actual, &name, outer)
|
|
}
|
|
}
|
|
Some(".tar") => extract_tar_cmd(archive, &actual, None),
|
|
// .tgz is a gzip-compressed tar: same as .tar.gz, but the tar flag
|
|
// cannot be inferred from the name (Rust-only extension).
|
|
Some(".tgz") => extract_tar_cmd(archive, &actual, Some(".gz")),
|
|
_ => Err(format!("no extractor for {name}")),
|
|
};
|
|
match res {
|
|
Ok(()) => (true, String::new(), actual),
|
|
Err(err) => (false, err, actual),
|
|
}
|
|
}
|
|
|
|
/// Move a failed archive under `<out_root>/.broken/<rel>`; the source
|
|
/// root is otherwise never modified. Returns the quarantine path.
|
|
pub fn move_to_broken(&self, path: &Path, src_root: &Path, note: &str) -> PathBuf {
|
|
let rel = path.strip_prefix(src_root).unwrap_or(path);
|
|
let dest = self.out_root.join(".broken").join(rel);
|
|
if let Some(parent) = dest.parent() {
|
|
let _ = fs::create_dir_all(parent);
|
|
}
|
|
let mut final_dest = dest.clone();
|
|
if dest.exists() {
|
|
let mut name = dest
|
|
.file_name()
|
|
.map(|n| n.to_os_string())
|
|
.unwrap_or_default();
|
|
name.push(".1");
|
|
final_dest = dest.with_file_name(name);
|
|
}
|
|
log::debug!(
|
|
"Quarantining {} -> {} ({note})",
|
|
path.display(),
|
|
final_dest.display()
|
|
);
|
|
if let Err(e) = fs::rename(path, &final_dest) {
|
|
// cross-device fallback: copy + remove (shutil.move semantics)
|
|
match fs::copy(path, &final_dest).and_then(|_| fs::remove_file(path)) {
|
|
Ok(()) => {}
|
|
Err(e2) => {
|
|
log::error!("Cannot quarantine {}: {e} / {e2}", path.display());
|
|
}
|
|
}
|
|
}
|
|
final_dest
|
|
}
|
|
|
|
/// Extract archives inside `dest` until none remain (BFS). Returns the
|
|
/// number of nested archives extracted.
|
|
pub fn expand_nested<W: Write>(
|
|
&self,
|
|
dest: &Path,
|
|
manifest: &mut W,
|
|
done: &mut HashSet<String>,
|
|
) -> usize {
|
|
let mut done_count = 0;
|
|
let mut queue: VecDeque<PathBuf> = VecDeque::new();
|
|
queue.push_back(dest.to_path_buf());
|
|
while let Some(cur) = queue.pop_front() {
|
|
for p in walk_files_sorted(&cur) {
|
|
let name = file_name_string(&p);
|
|
if !is_archive(&name) {
|
|
continue;
|
|
}
|
|
if is_macosx_junk(&p) {
|
|
let _ = fs::remove_file(&p);
|
|
log::info!("Junk (macOS AppleDouble) removed: {name}");
|
|
continue;
|
|
}
|
|
let stem_dir = p.with_file_name(display_unit(&name));
|
|
if done.contains(&rel_string(&self.out_root, &stem_dir)) {
|
|
// already extracted in a previous run; drop the archive copy
|
|
let _ = fs::remove_file(&p);
|
|
log::info!("Nested {name} already extracted - archive removed");
|
|
continue;
|
|
}
|
|
if self.space_low() {
|
|
log::error!(
|
|
"Out of disk space - skipping {} (kept in place)",
|
|
p.display()
|
|
);
|
|
continue;
|
|
}
|
|
let (ok, err, actual) = self.extract(&p, &stem_dir);
|
|
if !ok {
|
|
log::error!("FAILED nested extraction {}: {}", p.display(), err);
|
|
self.move_to_broken(&p, &self.out_root, &err);
|
|
continue;
|
|
}
|
|
write_manifest_line(
|
|
manifest,
|
|
&self.out_root,
|
|
&rel_to(&self.out_root, &p),
|
|
&actual,
|
|
);
|
|
done.insert(rel_string(&self.out_root, &actual));
|
|
let _ = fs::remove_file(&p); // archive copy consumed
|
|
done_count += 1;
|
|
queue.push_back(actual);
|
|
}
|
|
}
|
|
done_count
|
|
}
|
|
|
|
/// Extract every archive directly under `src_root` (any depth) into
|
|
/// `<out_root>/<src_root.name>/<rel-with-unit-name>` and mirror plain
|
|
/// files via hardlink (fallback: metadata-preserving copy). Returns
|
|
/// (extracted, mirrored, failed).
|
|
pub fn process_root<W: Write>(
|
|
&self,
|
|
src_root: &Path,
|
|
manifest: &mut W,
|
|
done: &mut HashSet<String>,
|
|
) -> (usize, usize, usize) {
|
|
let idx = file_name_string(src_root);
|
|
let mut extracted = 0;
|
|
let mut mirrored = 0;
|
|
let mut failed = 0;
|
|
for f in walk_files_sorted(src_root) {
|
|
let fname = file_name_string(&f);
|
|
if RUN_ARTIFACTS.contains(&fname.as_str())
|
|
|| fname.starts_with('.')
|
|
|| fname.ends_with(".part")
|
|
{
|
|
continue;
|
|
}
|
|
let Ok(rel) = f.strip_prefix(src_root).map(|r| r.to_path_buf()) else {
|
|
continue;
|
|
};
|
|
let rel_str = rel.to_string_lossy().into_owned();
|
|
if is_archive(&fname) {
|
|
let unit = display_unit(&fname);
|
|
let dest_rel: PathBuf = Path::new(&idx).join(rel.with_file_name(unit));
|
|
let dest_rel_str = dest_rel.to_string_lossy().into_owned();
|
|
// Skip on a manifest record, never on "dir non-empty": a
|
|
// crash mid-extraction leaves a partial dir that must be
|
|
// redone.
|
|
if done.contains(&dest_rel_str) {
|
|
log::info!("Skip (already extracted): {rel_str}");
|
|
continue;
|
|
}
|
|
if self.space_low() {
|
|
log::error!("Out of disk space - skipping archive {}", f.display());
|
|
failed += 1;
|
|
continue;
|
|
}
|
|
let dest = self.out_root.join(&dest_rel);
|
|
let (ok, err, actual) = self.extract(&f, &dest);
|
|
if !ok {
|
|
log::error!("FAILED {rel_str}: {err}");
|
|
self.move_to_broken(&f, src_root, &err);
|
|
failed += 1;
|
|
continue;
|
|
}
|
|
write_manifest_line(
|
|
manifest,
|
|
&self.out_root,
|
|
&Path::new(&idx).join(&rel),
|
|
&actual,
|
|
);
|
|
let actual_rel = rel_string(&self.out_root, &actual);
|
|
done.insert(actual_rel.clone());
|
|
extracted += 1;
|
|
log::info!("Extracted {rel_str} -> {actual_rel}");
|
|
// python discards the nested count here too
|
|
let _ = self.expand_nested(&actual, manifest, done);
|
|
} else {
|
|
let link_dest = self.out_root.join(&idx).join(&rel);
|
|
if link_dest.exists() {
|
|
continue;
|
|
}
|
|
if let Some(parent) = link_dest.parent() {
|
|
let _ = fs::create_dir_all(parent);
|
|
}
|
|
if let Err(e) = fs::hard_link(&f, &link_dest) {
|
|
// fallback: copy (fs::copy keeps permission bits, like copy2)
|
|
log::debug!("hardlink {rel_str} failed ({e}) - falling back to copy");
|
|
if let Err(e2) = fs::copy(&f, &link_dest) {
|
|
log::error!("Cannot mirror {rel_str}: {e2}");
|
|
continue;
|
|
}
|
|
}
|
|
mirrored += 1;
|
|
}
|
|
}
|
|
(extracted, mirrored, failed)
|
|
}
|
|
|
|
/// Expand any archive left anywhere in the tree (interrupted nested
|
|
/// expansions leave stranded copies even when the parent has a record).
|
|
pub fn sweep<W: Write>(&self, manifest: &mut W, done: &mut HashSet<String>) -> usize {
|
|
let mut swept = 0;
|
|
for p in walk_files_sorted(&self.out_root) {
|
|
let rel = rel_to(&self.out_root, &p);
|
|
if rel.components().any(|c| c.as_os_str() == ".broken") {
|
|
continue;
|
|
}
|
|
let name = file_name_string(&p);
|
|
if !is_archive(&name) {
|
|
continue;
|
|
}
|
|
if is_macosx_junk(&p) {
|
|
let _ = fs::remove_file(&p);
|
|
log::info!("Junk (macOS AppleDouble) removed: {name}");
|
|
continue;
|
|
}
|
|
let stem_dir = p.with_file_name(display_unit(&name));
|
|
if done.contains(&rel_string(&self.out_root, &stem_dir)) {
|
|
let _ = fs::remove_file(&p);
|
|
log::info!("Sweep: {name} already extracted - archive removed");
|
|
continue;
|
|
}
|
|
if self.space_low() {
|
|
log::error!("Out of disk space - sweep skipping {}", p.display());
|
|
continue;
|
|
}
|
|
let (ok, err, actual) = self.extract(&p, &stem_dir);
|
|
if !ok {
|
|
log::error!("FAILED sweep extraction {}: {}", p.display(), err);
|
|
self.move_to_broken(&p, &self.out_root, &err);
|
|
continue;
|
|
}
|
|
write_manifest_line(manifest, &self.out_root, &rel, &actual);
|
|
done.insert(rel_string(&self.out_root, &actual));
|
|
let _ = fs::remove_file(&p);
|
|
swept += 1;
|
|
self.expand_nested(&actual, manifest, done);
|
|
}
|
|
swept
|
|
}
|
|
|
|
/// Full run over `sources`: process each root, then one sweep. Failures
|
|
/// are quarantined and logged; the run never stops on individual errors.
|
|
pub fn run(&self, sources: &[PathBuf]) -> anyhow::Result<()> {
|
|
fs::create_dir_all(&self.out_root)?;
|
|
let manifest_path = self.out_root.join("extract_manifest.jsonl");
|
|
let mut done = load_done(&manifest_path);
|
|
log::info!("{} destinations already recorded in manifest", done.len());
|
|
let mut manifest = fs::OpenOptions::new()
|
|
.append(true)
|
|
.create(true)
|
|
.open(&manifest_path)?;
|
|
|
|
let mut total_extracted = 0;
|
|
let mut total_mirrored = 0;
|
|
let mut total_failed = 0;
|
|
for src in sources {
|
|
log::info!("== processing {}", src.display());
|
|
let (e, m, f) = self.process_root(src, &mut manifest, &mut done);
|
|
log::info!(
|
|
"== {}: {e} extracted, {m} mirrored, {f} failed",
|
|
file_name_string(src)
|
|
);
|
|
total_extracted += e;
|
|
total_mirrored += m;
|
|
total_failed += f;
|
|
}
|
|
let swept = self.sweep(&mut manifest, &mut done);
|
|
log::info!(
|
|
"TOTAL: {total_extracted} archives extracted, {total_mirrored} files mirrored, \
|
|
{total_failed} failed, {swept} swept"
|
|
);
|
|
if total_failed > 0 {
|
|
log::warn!(
|
|
"Failures were moved under {} - inspect and re-run",
|
|
self.out_root.join(".broken").display()
|
|
);
|
|
}
|
|
Ok(())
|
|
}
|
|
}
|
|
|
|
/// Dest dirs already recorded in the manifest (relative to out root).
|
|
/// Malformed lines are skipped.
|
|
pub fn load_done(manifest_path: &Path) -> HashSet<String> {
|
|
let mut done = HashSet::new();
|
|
let Ok(bytes) = fs::read(manifest_path) else {
|
|
return done;
|
|
};
|
|
for line in bytes.split(|&b| b == b'\n') {
|
|
let line = String::from_utf8_lossy(line);
|
|
let line = line.trim();
|
|
if line.is_empty() {
|
|
continue;
|
|
}
|
|
if let Ok(value) = serde_json::from_str::<serde_json::Value>(line) {
|
|
if let Some(dest) = value.get("dest").and_then(|d| d.as_str()) {
|
|
done.insert(dest.to_string());
|
|
}
|
|
}
|
|
}
|
|
done
|
|
}
|
|
|
|
/// 7z via sevenz-rust2. Empty password: python passes `-p ""` (no password,
|
|
/// avoids interactive prompts) — `Password::empty()` is the same thing; the
|
|
/// password-less helper is retried once in case the with-password path
|
|
/// rejects an empty password outright. Password-protected archives still
|
|
/// fail, exactly like the python version (no password known).
|
|
fn extract_7z(archive: &Path, dest: &Path) -> Result<(), String> {
|
|
match sevenz_rust2::decompress_file_with_password(
|
|
archive,
|
|
dest,
|
|
sevenz_rust2::Password::empty(),
|
|
) {
|
|
Ok(()) => Ok(()),
|
|
Err(first) => match sevenz_rust2::decompress_file(archive, dest) {
|
|
Ok(()) => Ok(()),
|
|
Err(second) => Err(format!("7z: {first} / {second}")),
|
|
},
|
|
}
|
|
}
|
|
|
|
/// zip via the zip crate, entry by entry. Traversal-safe: entries whose raw
|
|
/// name contains `..` or that cannot be enclosed under dest are skipped
|
|
/// (python relies on 7z being safe — same minimum).
|
|
fn extract_zip(archive: &Path, dest: &Path) -> Result<(), String> {
|
|
let file = File::open(archive).map_err(|e| format!("zip: {e}"))?;
|
|
let mut zip = zip::ZipArchive::new(file).map_err(|e| format!("zip: {e}"))?;
|
|
for i in 0..zip.len() {
|
|
let mut entry = zip.by_index(i).map_err(|e| format!("zip: {e}"))?;
|
|
if entry.name().split('/').any(|part| part == "..") {
|
|
log::warn!("zip: rejecting traversal entry {:?}", entry.name_raw());
|
|
continue;
|
|
}
|
|
let Some(path) = entry.enclosed_name() else {
|
|
log::warn!("zip: skipping unsafe entry {:?}", entry.name_raw());
|
|
continue;
|
|
};
|
|
let out_path = dest.join(&path);
|
|
if entry.is_dir() {
|
|
fs::create_dir_all(&out_path).map_err(|e| format!("zip: {e}"))?;
|
|
} else {
|
|
if let Some(parent) = out_path.parent() {
|
|
fs::create_dir_all(parent).map_err(|e| format!("zip: {e}"))?;
|
|
}
|
|
let mut out_file = File::create(&out_path).map_err(|e| format!("zip: {e}"))?;
|
|
io::copy(&mut entry, &mut out_file).map_err(|e| format!("zip: {e}"))?;
|
|
#[cfg(unix)]
|
|
if let Some(mode) = entry.unix_mode() {
|
|
let _ = fs::set_permissions(&out_path, fs::Permissions::from_mode(mode));
|
|
}
|
|
}
|
|
}
|
|
Ok(())
|
|
}
|
|
|
|
/// rar via the unrar crate (C library), no password. Entries with `..`
|
|
/// components are skipped rather than extracted. (Untested offline: .rar
|
|
/// archives cannot be created without the proprietary tool.)
|
|
fn extract_rar(archive: &Path, dest: &Path) -> Result<(), String> {
|
|
fs::create_dir_all(dest).map_err(|e| format!("rar: {e}"))?;
|
|
let mut arc = unrar::Archive::new(archive)
|
|
.open_for_processing()
|
|
.map_err(|e| format!("rar: {e}"))?;
|
|
loop {
|
|
match arc.read_header() {
|
|
Ok(None) => break,
|
|
Ok(Some(header)) => {
|
|
let fname = header.entry().filename.clone();
|
|
if fname
|
|
.components()
|
|
.any(|c| matches!(c, Component::ParentDir))
|
|
{
|
|
log::warn!("rar: skipping entry with '..': {}", fname.display());
|
|
arc = header.skip().map_err(|e| format!("rar: {e}"))?;
|
|
continue;
|
|
}
|
|
arc = header
|
|
.extract_with_base(dest)
|
|
.map_err(|e| format!("rar: {e}"))?;
|
|
}
|
|
Err(e) => return Err(format!("rar: {e}")),
|
|
}
|
|
}
|
|
Ok(())
|
|
}
|
|
|
|
/// Tarballs via the system `tar` binary — the same shape as the python
|
|
/// version's `7z` subprocess call: merged output, rc 0/1 = ok (1 logs a
|
|
/// warning), 4h timeout. `decompress` selects -z/-j/-J or None for bare tar.
|
|
fn extract_tar_cmd(archive: &Path, dest: &Path, decompress: Option<&str>) -> Result<(), String> {
|
|
fs::create_dir_all(dest).map_err(|e| format!("tar: {e}"))?;
|
|
let mut args: Vec<std::ffi::OsString> = vec!["-x".into()];
|
|
if let Some(flag) = decompress {
|
|
args.push(
|
|
match flag {
|
|
".gz" => "-z",
|
|
".bz2" => "-j",
|
|
".xz" => "-J",
|
|
_ => unreachable!("only gz/bz2/xz reach this branch"),
|
|
}
|
|
.into(),
|
|
);
|
|
}
|
|
args.push("-f".into());
|
|
args.push(archive.as_os_str().to_os_string());
|
|
args.push("-C".into());
|
|
args.push(dest.as_os_str().to_os_string());
|
|
let (rc, tail) = run_extract_cmd("tar", &args, None);
|
|
match rc {
|
|
0 => Ok(()),
|
|
1 => {
|
|
log::warn!(
|
|
"tar reported warnings extracting {}",
|
|
file_name_string(archive)
|
|
);
|
|
Ok(())
|
|
}
|
|
_ => Err(format!("tar exit {rc}: {tail}")),
|
|
}
|
|
}
|
|
|
|
/// Single-layer decompress of a bare `.gz`/`.bz2`/`.xz` (or any trailing
|
|
/// compressed suffix that is not a tarball) into `dest/<inner name>` via
|
|
/// the system decompressor — one layer, like a single `7z x` call.
|
|
fn decompress_bare_cmd(archive: &Path, dest: &Path, name: &str, outer: &str) -> Result<(), String> {
|
|
let inner = &name[..name.len() - outer.len()];
|
|
fs::create_dir_all(dest).map_err(|e| format!("{outer}: {e}"))?;
|
|
let out_path = dest.join(inner);
|
|
let (program, what) = match outer {
|
|
".gz" => ("gzip", "gz"),
|
|
".bz2" => ("bzip2", "bz2"),
|
|
_ => ("xz", "xz"),
|
|
};
|
|
let args = vec![
|
|
std::ffi::OsString::from("-dc"),
|
|
archive.as_os_str().to_os_string(),
|
|
];
|
|
let (rc, tail) = run_extract_cmd(program, &args, Some(&out_path));
|
|
if rc == 0 {
|
|
Ok(())
|
|
} else {
|
|
Err(format!("{what} exit {rc}: {tail}"))
|
|
}
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod tests {
|
|
use super::*;
|
|
use serde_json::Value;
|
|
use std::collections::BTreeSet;
|
|
|
|
/// Write a real zip archive containing the given (path, content) files.
|
|
fn write_zip(path: &Path, files: &[(&str, Vec<u8>)]) {
|
|
let file = File::create(path).unwrap();
|
|
let mut w = zip::ZipWriter::new(file);
|
|
for (name, data) in files {
|
|
w.start_file(*name, zip::write::SimpleFileOptions::default())
|
|
.unwrap();
|
|
w.write_all(data).unwrap();
|
|
}
|
|
w.finish().unwrap();
|
|
}
|
|
|
|
/// Minimal CRC32 (IEEE) for the hand-built gzip container in tests.
|
|
fn crc32(data: &[u8]) -> u32 {
|
|
let mut table = [0u32; 256];
|
|
for (i, slot) in table.iter_mut().enumerate() {
|
|
let mut c = i as u32;
|
|
for _ in 0..8 {
|
|
c = if c & 1 != 0 {
|
|
0xEDB88320 ^ (c >> 1)
|
|
} else {
|
|
c >> 1
|
|
};
|
|
}
|
|
*slot = c;
|
|
}
|
|
let mut crc = 0xFFFF_FFFFu32;
|
|
for &b in data {
|
|
crc = table[((crc ^ b as u32) & 0xFF) as usize] ^ (crc >> 8);
|
|
}
|
|
crc ^ 0xFFFF_FFFF
|
|
}
|
|
|
|
/// Valid gzip container wrapping a single *stored* (uncompressed)
|
|
/// deflate block — no compression crate needed to build a test fixture.
|
|
fn gzip_stored(data: &[u8]) -> Vec<u8> {
|
|
let mut v = vec![0x1f, 0x8b, 0x08, 0, 0, 0, 0, 0, 0, 0x03];
|
|
v.push(0x01); // BFINAL=1, BTYPE=00 (stored)
|
|
let len = data.len() as u16;
|
|
v.extend_from_slice(&len.to_le_bytes());
|
|
v.extend_from_slice(&(!len).to_le_bytes());
|
|
v.extend_from_slice(data);
|
|
v.extend_from_slice(&crc32(data).to_le_bytes());
|
|
v.extend_from_slice(&(data.len() as u32).to_le_bytes());
|
|
v
|
|
}
|
|
|
|
/// Minimal USTAR archive with one file entry, terminated by two zero
|
|
/// blocks — readable by the system tar.
|
|
fn make_ustar(name: &str, data: &[u8]) -> Vec<u8> {
|
|
let mut block = vec![0u8; 512];
|
|
block[..name.len()].copy_from_slice(name.as_bytes());
|
|
block[100..108].copy_from_slice(b"0000644\0"); // mode
|
|
block[108..116].copy_from_slice(b"0000000\0"); // uid
|
|
block[116..124].copy_from_slice(b"0000000\0"); // gid
|
|
let size_field = format!("{:011o}\0", data.len());
|
|
block[124..136].copy_from_slice(size_field.as_bytes()); // size
|
|
block[136..148].copy_from_slice(b"00000000000\0"); // mtime
|
|
block[148..156].copy_from_slice(b" "); // chksum placeholder
|
|
block[156] = b'0'; // typeflag: regular file
|
|
block[257..263].copy_from_slice(b"ustar\0");
|
|
block[263..265].copy_from_slice(b"00");
|
|
let sum: u32 = block.iter().map(|&b| b as u32).sum();
|
|
let chk = format!("{:06o}\0 ", sum);
|
|
block[148..156].copy_from_slice(chk.as_bytes());
|
|
let mut out = block;
|
|
out.extend_from_slice(data);
|
|
while out.len() % 512 != 0 {
|
|
out.push(0);
|
|
}
|
|
out.extend(std::iter::repeat_n(0u8, 1024));
|
|
out
|
|
}
|
|
|
|
/// Base dir for test tempdirs: /tmp is often a small tmpfs (<
|
|
/// MIN_FREE_BYTES), which would trip the real free-space check; prefer
|
|
/// /var/tmp when it clears the threshold (same convention as the
|
|
/// python version's /var/tmp staging).
|
|
fn test_base() -> PathBuf {
|
|
let var_tmp = Path::new("/var/tmp");
|
|
if cfg!(unix)
|
|
&& free_bytes(var_tmp)
|
|
.map(|b| b >= MIN_FREE_BYTES)
|
|
.unwrap_or(false)
|
|
{
|
|
var_tmp.to_path_buf()
|
|
} else {
|
|
std::env::temp_dir()
|
|
}
|
|
}
|
|
|
|
/// Build a source root (`src1`) + output root (`out`) inside a
|
|
/// container tempdir; returns (container, src, out). The source root's
|
|
/// file name (`src1`) is what the extractor uses as the index folder.
|
|
fn build_fixtures() -> (tempfile::TempDir, PathBuf, PathBuf) {
|
|
let _ = env_logger::builder()
|
|
.filter_level(log::LevelFilter::Info)
|
|
.is_test(true)
|
|
.try_init();
|
|
let container = tempfile::tempdir_in(test_base()).unwrap();
|
|
let src = container.path().join("src1");
|
|
let out = container.path().join("out");
|
|
fs::create_dir_all(&src).unwrap();
|
|
fs::create_dir_all(&out).unwrap();
|
|
let s = src.as_path();
|
|
|
|
// Pack.zip: files + a nested zip inside
|
|
let inner_path = s.join(".tmp_inner.zip");
|
|
write_zip(&inner_path, &[("beta.txt", b"inner payload".to_vec())]);
|
|
let inner_bytes = fs::read(&inner_path).unwrap();
|
|
fs::remove_file(&inner_path).unwrap();
|
|
write_zip(
|
|
&s.join("Pack.zip"),
|
|
&[
|
|
("alpha.txt", b"alpha payload".to_vec()),
|
|
("nested/inner.zip", inner_bytes),
|
|
],
|
|
);
|
|
|
|
// a real .7z
|
|
let gamma = s.join("gamma.txt");
|
|
fs::write(&gamma, b"sevenz payload").unwrap();
|
|
let sub = s.join("Sub");
|
|
fs::create_dir_all(&sub).unwrap();
|
|
sevenz_rust2::compress_to_path(&gamma, sub.join("The Twin Lance.7z")).expect("compress 7z");
|
|
fs::remove_file(&gamma).unwrap();
|
|
|
|
// a tar.gz (hand-built ustar + gzip-stored container)
|
|
let targz = gzip_stored(&make_ustar("delta.txt", b"tarred payload"));
|
|
fs::write(s.join("logs.tar.gz"), targz).unwrap();
|
|
|
|
// a .tgz (gzip-compressed tar under the short suffix — Rust-only)
|
|
let tgz = gzip_stored(&make_ustar("epsilon.txt", b"tgz payload"));
|
|
fs::write(s.join("bundle.tgz"), tgz).unwrap();
|
|
|
|
// garbage zip -> must land in .broken
|
|
fs::write(s.join("bad.zip"), b"this is not a zip archive at all").unwrap();
|
|
|
|
// plain file (mirrored via hardlink)
|
|
fs::write(s.join("notes.txt"), b"plain notes").unwrap();
|
|
|
|
// run artifacts / hidden / partial: must be skipped
|
|
fs::write(s.join("state.json"), b"{}").unwrap();
|
|
fs::write(s.join(".hidden.txt"), b"secret").unwrap();
|
|
fs::write(s.join("partial.part"), b"partial").unwrap();
|
|
|
|
(container, src, out)
|
|
}
|
|
|
|
fn inventory(root: &Path) -> BTreeSet<String> {
|
|
walk_files_sorted(root)
|
|
.iter()
|
|
.map(|p| rel_to(root, p).to_string_lossy().into_owned())
|
|
.collect()
|
|
}
|
|
|
|
fn find_file_named(root: &Path, name: &str) -> Option<PathBuf> {
|
|
walk_files_sorted(root).into_iter().find(|p| {
|
|
p.file_name()
|
|
.map(|n| n.to_string_lossy() == name)
|
|
.unwrap_or(false)
|
|
})
|
|
}
|
|
|
|
fn manifest_values(out: &Path) -> Vec<Value> {
|
|
let text = fs::read_to_string(out.join("extract_manifest.jsonl")).unwrap();
|
|
text.lines()
|
|
.filter(|l| !l.trim().is_empty())
|
|
.map(|l| serde_json::from_str(l).unwrap())
|
|
.collect()
|
|
}
|
|
|
|
#[test]
|
|
fn sniff_container_reads_magic_not_extension() {
|
|
let dir = tempfile::tempdir().unwrap();
|
|
// real zip content, .rar name -> sniffed as zip
|
|
let z = dir.path().join("x.rar");
|
|
write_zip(&z, &[("a.txt", b"hi".to_vec())]);
|
|
assert_eq!(sniff_container(&z), Some(".zip"));
|
|
|
|
// 7z magic, .zip name
|
|
let seven = dir.path().join("y.zip");
|
|
fs::write(&seven, [0x37u8, 0x7a, 0xbc, 0xaf, 0x27, 0x1c, 0, 4]).unwrap();
|
|
assert_eq!(sniff_container(&seven), Some(".7z"));
|
|
|
|
// rar5 magic
|
|
let rar = dir.path().join("z.7z");
|
|
fs::write(&rar, b"Rar!\x1a\x07\x01\x00").unwrap();
|
|
assert_eq!(sniff_container(&rar), Some(".rar"));
|
|
|
|
// gzip is not a container: extension keeps driving it
|
|
let gz = dir.path().join("g.gz");
|
|
fs::write(&gz, [0x1f, 0x8b, 0x08]).unwrap();
|
|
assert_eq!(sniff_container(&gz), None);
|
|
}
|
|
|
|
#[test]
|
|
fn mislabeled_zip_extracts_via_sniffing() {
|
|
// /tmp is a small tmpfs; the free-space guard needs a real fs
|
|
let dir = test_base().join(format!("dlp-mislabel-{}", std::process::id()));
|
|
let _ = fs::remove_dir_all(&dir);
|
|
let src = dir.join("src");
|
|
let out = dir.join("out");
|
|
fs::create_dir_all(&src).unwrap();
|
|
// a real zip with a .rar extension: python's 7z CLI would extract it too
|
|
write_zip(
|
|
&src.join("Pack.rar"),
|
|
&[("unit/part.txt", b"payload".to_vec())],
|
|
);
|
|
let extractor = Extractor {
|
|
out_root: out.clone(),
|
|
};
|
|
extractor.run(&[src.clone()]).unwrap();
|
|
assert_eq!(
|
|
fs::read_to_string(out.join("src/Pack/unit/part.txt")).unwrap(),
|
|
"payload"
|
|
);
|
|
let broken_empty = fs::read_dir(out.join(".broken"))
|
|
.map(|d| d.count() == 0)
|
|
.unwrap_or(true);
|
|
assert!(broken_empty, "nothing should be quarantined");
|
|
let _ = fs::remove_dir_all(&dir);
|
|
}
|
|
|
|
#[test]
|
|
fn run_extracts_nests_quarantines_and_mirrors() {
|
|
let (_c, src, out) = build_fixtures();
|
|
let extractor = Extractor {
|
|
out_root: out.clone(),
|
|
};
|
|
extractor.run(&[src.clone()]).unwrap();
|
|
|
|
let s = src.as_path();
|
|
let o = out.as_path();
|
|
|
|
// extracted content, including the nested zip inside Pack.zip
|
|
assert_eq!(
|
|
fs::read_to_string(o.join("src1/Pack/alpha.txt")).unwrap(),
|
|
"alpha payload"
|
|
);
|
|
assert_eq!(
|
|
fs::read_to_string(o.join("src1/Pack/nested/inner/beta.txt")).unwrap(),
|
|
"inner payload"
|
|
);
|
|
assert!(
|
|
!o.join("src1/Pack/nested/inner.zip").exists(),
|
|
"nested archive must be consumed"
|
|
);
|
|
|
|
// 7z and tar.gz
|
|
assert!(
|
|
o.join("src1/Sub/The Twin Lance").is_dir(),
|
|
"7z unit dir missing"
|
|
);
|
|
assert!(find_file_named(&o.join("src1/Sub/The Twin Lance"), "gamma.txt").is_some());
|
|
assert_eq!(
|
|
fs::read_to_string(o.join("src1/logs/delta.txt")).unwrap(),
|
|
"tarred payload"
|
|
);
|
|
assert_eq!(
|
|
fs::read_to_string(o.join("src1/bundle/epsilon.txt")).unwrap(),
|
|
"tgz payload"
|
|
);
|
|
|
|
// no archives left in the extracted tree (outside .broken)
|
|
for p in walk_files_sorted(o) {
|
|
let rel = rel_to(o, &p);
|
|
if rel.components().any(|c| c.as_os_str() == ".broken") {
|
|
continue;
|
|
}
|
|
assert!(
|
|
!is_archive(&file_name_string(&p)),
|
|
"archive left in extracted tree: {}",
|
|
p.display()
|
|
);
|
|
}
|
|
|
|
// garbage zip quarantined under .broken
|
|
let broken = o.join(".broken/bad.zip");
|
|
assert!(broken.is_file(), "bad.zip not quarantined");
|
|
assert_eq!(
|
|
fs::read(&broken).unwrap(),
|
|
b"this is not a zip archive at all"
|
|
);
|
|
|
|
// plain file mirrored: same inode on hardlink, else copy fallback
|
|
#[cfg(unix)]
|
|
{
|
|
use std::os::unix::fs::MetadataExt;
|
|
let sm = fs::metadata(s.join("notes.txt")).unwrap();
|
|
let om = fs::metadata(o.join("src1/notes.txt")).unwrap();
|
|
if sm.ino() == om.ino() {
|
|
assert_eq!(sm.dev(), om.dev(), "hardlink must be on the same fs");
|
|
} else {
|
|
assert_eq!(sm.len(), om.len(), "hardlink fell back to copy");
|
|
}
|
|
}
|
|
assert_eq!(
|
|
fs::read_to_string(o.join("src1/notes.txt")).unwrap(),
|
|
"plain notes"
|
|
);
|
|
|
|
// run artifacts / hidden / partial never enter the tree
|
|
for junk in ["state.json", ".hidden.txt", "partial.part"] {
|
|
assert!(
|
|
find_file_named(o, junk).is_none(),
|
|
"{junk} leaked into output"
|
|
);
|
|
}
|
|
|
|
// source root keeps its originals (except the quarantined bad.zip)
|
|
assert!(s.join("Pack.zip").is_file());
|
|
assert!(s.join("notes.txt").is_file());
|
|
assert!(s.join("Sub/The Twin Lance.7z").is_file());
|
|
assert!(
|
|
!s.join("bad.zip").exists(),
|
|
"failed archive must be moved away"
|
|
);
|
|
|
|
// manifest: one line per extraction, exactly the 3 provenance keys
|
|
let lines = manifest_values(o);
|
|
let triples: BTreeSet<(String, String, String)> = lines
|
|
.iter()
|
|
.map(|v| {
|
|
let obj = v.as_object().unwrap();
|
|
assert_eq!(obj.len(), 3, "unexpected keys in manifest line: {v}");
|
|
(
|
|
v["archive"].as_str().unwrap().to_string(),
|
|
v["dest"].as_str().unwrap().to_string(),
|
|
v["unit"].as_str().unwrap().to_string(),
|
|
)
|
|
})
|
|
.collect();
|
|
let expected: BTreeSet<(String, String, String)> = [
|
|
("src1/Pack.zip", "src1/Pack", "Pack"),
|
|
(
|
|
"src1/Pack/nested/inner.zip",
|
|
"src1/Pack/nested/inner",
|
|
"inner",
|
|
),
|
|
(
|
|
"src1/Sub/The Twin Lance.7z",
|
|
"src1/Sub/The Twin Lance",
|
|
"The Twin Lance",
|
|
),
|
|
("src1/logs.tar.gz", "src1/logs", "logs"),
|
|
("src1/bundle.tgz", "src1/bundle", "bundle"),
|
|
]
|
|
.iter()
|
|
.map(|(a, d, u)| (a.to_string(), d.to_string(), u.to_string()))
|
|
.collect();
|
|
assert_eq!(triples, expected);
|
|
}
|
|
|
|
#[test]
|
|
fn rerun_is_idempotent() {
|
|
let (_c, src, out) = build_fixtures();
|
|
let extractor = Extractor {
|
|
out_root: out.clone(),
|
|
};
|
|
extractor.run(&[src.clone()]).unwrap();
|
|
|
|
let manifest_before = fs::read_to_string(out.join("extract_manifest.jsonl")).unwrap();
|
|
let inv_before = inventory(&out);
|
|
assert!(!manifest_before.is_empty());
|
|
|
|
extractor.run(&[src.clone()]).unwrap();
|
|
|
|
let manifest_after = fs::read_to_string(out.join("extract_manifest.jsonl")).unwrap();
|
|
assert_eq!(
|
|
manifest_before, manifest_after,
|
|
"re-run must not append manifest lines"
|
|
);
|
|
assert_eq!(
|
|
inv_before,
|
|
inventory(&out),
|
|
"re-run must not change the tree"
|
|
);
|
|
assert!(
|
|
!out.as_path().join("src1/Pack_dup").exists()
|
|
&& !out.as_path().join("src1/Pack_dup_dup").exists(),
|
|
"re-run must not create duplicate destinations"
|
|
);
|
|
// the previously extracted content is still intact
|
|
assert_eq!(
|
|
fs::read_to_string(out.as_path().join("src1/Pack/alpha.txt")).unwrap(),
|
|
"alpha payload"
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn dest_collision_appends_dup() {
|
|
let dir = tempfile::tempdir_in(test_base()).unwrap();
|
|
let ex = Extractor {
|
|
out_root: dir.path().join("out"),
|
|
};
|
|
fs::create_dir_all(&ex.out_root).unwrap();
|
|
|
|
// dest occupied by a FILE -> the *_dup sibling is used (python:
|
|
// mkdir exist_ok raises FileExistsError -> append "_dup")
|
|
let dest = dir.path().join("d1");
|
|
fs::write(&dest, b"occupied").unwrap();
|
|
let actual = ex.create_dest(&dest).unwrap();
|
|
assert_eq!(actual, dest.with_file_name("d1_dup"));
|
|
assert!(actual.is_dir());
|
|
|
|
// dest and *_dup both occupied by files -> graceful failure after
|
|
// the 5 attempts (python's loop rebuilds "_dup" from the original
|
|
// name, so it keeps hitting the same path)
|
|
let dest2 = dir.path().join("d2");
|
|
fs::write(&dest2, b"occupied").unwrap();
|
|
fs::write(dir.path().join("d2_dup"), b"occupied too").unwrap();
|
|
let err = ex.create_dest(&dest2).unwrap_err();
|
|
assert!(err.contains("repeated collisions"), "{err}");
|
|
|
|
// parent path is a file (NotADirectoryError) -> immediate failure
|
|
let blocker = dir.path().join("blocker");
|
|
fs::write(&blocker, b"file").unwrap();
|
|
let err2 = ex.create_dest(&blocker.join("sub")).unwrap_err();
|
|
assert!(err2.contains("cannot create destination"), "{err2}");
|
|
}
|
|
|
|
#[test]
|
|
fn suffix_helpers_match_python() {
|
|
assert_eq!(archive_suffixes("x.tar.gz"), vec![".gz", ".tar"]);
|
|
assert_eq!(display_unit("x.tar.gz"), "x");
|
|
assert_eq!(archive_suffixes("The Twin Lance.7z"), vec![".7z"]);
|
|
assert_eq!(display_unit("The Twin Lance.7z"), "The Twin Lance");
|
|
assert_eq!(archive_suffixes("X.ZIP"), vec![".zip"]);
|
|
assert_eq!(display_unit("X.ZIP"), "X");
|
|
assert!(archive_suffixes("plain.txt").is_empty());
|
|
assert!(!is_archive("plain.txt"));
|
|
assert!(
|
|
archive_suffixes(".hidden").is_empty(),
|
|
"dotfiles have no suffix"
|
|
);
|
|
assert_eq!(display_unit("Pack.zip"), "Pack");
|
|
assert!(is_archive("c.7z.7z"));
|
|
// .tgz: Rust-only short suffix for a gzip-compressed tar
|
|
assert_eq!(archive_suffixes("x.tgz"), vec![".tgz"]);
|
|
assert_eq!(display_unit("x.tgz"), "x");
|
|
assert!(is_archive("X.TGZ"));
|
|
}
|
|
|
|
#[test]
|
|
fn macosx_junk_detection() {
|
|
assert!(is_macosx_junk(Path::new("out/1/__MACOSX/._x.zip")));
|
|
assert!(is_macosx_junk(Path::new("._x.zip")));
|
|
assert!(is_macosx_junk(Path::new("deep/nested/._resource")));
|
|
assert!(is_macosx_junk(Path::new("a/__MACOSX")));
|
|
assert!(!is_macosx_junk(Path::new("a.zip")));
|
|
assert!(!is_macosx_junk(Path::new("plain/notes.txt")));
|
|
assert!(
|
|
!is_macosx_junk(Path::new("macosx/_a.zip")),
|
|
"__MACOSX matching is case-sensitive like python"
|
|
);
|
|
}
|
|
}
|