//! Recursive archive extraction into a plain-file tree — Rust port of //! `tools/extract_all.py` (extract_manifest.jsonl feeds the //! sorter's provenance mode; failures go to .broken/; the run never stops). //! //! Extraction backends: `sevenz-rust2` for .7z, `zip` for .zip, `unrar` for //! .rar — pure Rust, no external binaries. Tarballs and bare compressed //! files (.tar/.tar.gz/.tgz/.tar.bz2/.tar.xz/.gz/.bz2/.xz) are handled by the //! system `tar`/`gzip`/`bzip2`/`xz` binaries, exactly like the python //! version shelled out to `7z`: merged output capture, exit-code parity //! (0 ok, 1 warnings), and the 4h per-archive timeout. The tar/flate2/ //! bzip2/xz2 crates mentioned in the port plan are not declared //! dependencies of this crate and Cargo.toml is off-limits, so the system //! binaries are the parity-preserving fallback (they are universally //! present on the target Linux hosts). //! //! Because the zip/7z paths are pure Rust, the python version's /var/tmp //! fallback + cp437 name repair is not needed here (the zip crate decodes //! legacy cp437 entry names itself). //! //! Behavior parity notes (vs. the python version): //! - Skip decisions are made on manifest records only, never on "dir //! non-empty"; a partial dir from a crash is re-extracted. //! - Source roots are never modified except for the quarantine move of //! failed archives into `/.broken/`. //! - The extracted tree holds no archives afterwards (nested archives are //! expanded BFS-style and their copies unlinked). //! - Plain files are hardlinked in (fallback: metadata-preserving copy). //! - No individual failure ever aborts the run. use std::collections::{HashSet, VecDeque}; use std::fs::{self, File}; use std::io::{self, Read, Write}; use std::path::{Component, Path, PathBuf}; use std::process::{Command, Stdio}; use std::time::{Duration, Instant}; #[cfg(unix)] use std::os::unix::fs::PermissionsExt; /// Archive suffixes recognized by the extractors (python `ARCHIVE_SUFFIXES`, /// plus `.tgz` — a Rust-only extension; the python version never extracted /// those). `.tgz` is a gzip-compressed tar, so it maps onto the tarball path. pub const ARCHIVE_SUFFIXES: [&str; 8] = [".zip", ".7z", ".rar", ".tar", ".gz", ".bz2", ".xz", ".tgz"]; /// Stop extracting below this free space (python `MIN_FREE_BYTES`). pub const MIN_FREE_BYTES: u64 = 20 * 1024 * 1024 * 1024; /// Per-archive extraction timeout in seconds (python `EXTRACT_TIMEOUT`). const EXTRACT_TIMEOUT_SECS: u64 = 4 * 3600; /// Run artifacts that must never be treated as payload (python `RUN_ARTIFACTS`). pub const RUN_ARTIFACTS: [&str; 14] = [ "export.json", "export.html", "state.json", "manifest.jsonl", "failed_urls.txt", "catalogue.md", "catalogue.jsonl", "sorter_applied.jsonl", "catalogue_moves.jsonl", "urls.txt", "urls_0807.txt", "urls_gofile.txt", "failed_0807.txt", "extract_manifest.jsonl", ]; /// Recursive extractor writing into `out_root`. pub struct Extractor { pub out_root: PathBuf, } /// Python `Path.suffix` semantics for a plain file name: the trailing dot /// segment, unless the name starts with a dot (hidden) or ends with a dot. fn name_suffix(name: &str) -> Option<&str> { let idx = name.rfind('.')?; if idx == 0 || idx == name.len() - 1 { return None; } Some(&name[idx..]) } /// All trailing archive suffixes, lowercased, outermost first — e.g. /// `x.tar.gz` -> `[".gz", ".tar"]`. pub fn archive_suffixes(name: &str) -> Vec { let mut out = Vec::new(); let mut cur = name; while let Some(sfx) = name_suffix(cur) { let lower = sfx.to_ascii_lowercase(); if !ARCHIVE_SUFFIXES.contains(&lower.as_str()) { break; } out.push(lower); cur = &cur[..cur.len() - sfx.len()]; } out } /// True when the file name carries at least one archive suffix. pub fn is_archive(name: &str) -> bool { !archive_suffixes(name).is_empty() } /// Sniff the container format from magic bytes (7z/zip/rar). The python /// version shells out to the 7z CLI, which sniffs content; extensions are /// unreliable in the wild (we have seen real 7z archives named `.zip`). fn sniff_container(path: &Path) -> Option<&'static str> { use std::io::Read; let mut buf = [0u8; 8]; let mut f = fs::File::open(path).ok()?; let n = f.read(&mut buf).ok()?; let b = &buf[..n]; if b.starts_with(&[0x37, 0x7a, 0xbc, 0xaf, 0x27, 0x1c]) { return Some(".7z"); } if b.starts_with(b"PK\x03\x04") || b.starts_with(b"PK\x05\x06") { return Some(".zip"); } if b.starts_with(b"Rar!\x1a\x07\x00") || b.starts_with(b"Rar!\x1a\x07\x01\x00") { return Some(".rar"); } None } /// Archive name with all archive suffixes stripped: the unit folder name. pub fn display_unit(name: &str) -> String { let mut cur = name; while let Some(sfx) = name_suffix(cur) { let lower = sfx.to_ascii_lowercase(); if !ARCHIVE_SUFFIXES.contains(&lower.as_str()) { break; } cur = &cur[..cur.len() - sfx.len()]; } cur.to_string() } /// Free disk space (bytes) on the file system containing `path`. pub fn free_bytes(path: &Path) -> anyhow::Result { Ok(fs4::available_space(path)?) } /// AppleDouble resource-fork "zips" (`__MACOSX/._x.zip`) — no content. pub fn is_macosx_junk(p: &Path) -> bool { p.components().any(|c| c.as_os_str() == "__MACOSX") || p.file_name() .map(|n| n.to_string_lossy().starts_with("._")) .unwrap_or(false) } /// Relative path helper: `p` relative to `root` (falls back to `p` itself). fn rel_to(root: &Path, p: &Path) -> PathBuf { p.strip_prefix(root).unwrap_or(p).to_path_buf() } /// Same as [`rel_to`] as a (lossy) string, matching python `str(rel_path)`. fn rel_string(root: &Path, p: &Path) -> String { rel_to(root, p).to_string_lossy().into_owned() } fn file_name_string(p: &Path) -> String { p.file_name() .map(|n| n.to_string_lossy().into_owned()) .unwrap_or_default() } /// Recursively collect regular files under `root`, sorted for determinism. /// /// Matches python `sorted(rglob("*"))` + `is_file()` filtering: symlinked /// directories are not descended into, symlinks to files count as files. fn walk_files_sorted(root: &Path) -> Vec { let mut files = Vec::new(); let mut dirs = vec![root.to_path_buf()]; while let Some(dir) = dirs.pop() { let Ok(entries) = fs::read_dir(&dir) else { continue; }; for entry in entries.flatten() { let path = entry.path(); match fs::symlink_metadata(&path) { Ok(meta) if meta.is_dir() => dirs.push(path), Ok(meta) if meta.is_file() => files.push(path), Ok(meta) if meta.is_symlink() => { if let Ok(target) = fs::metadata(&path) { if target.is_file() { files.push(path); } } } _ => {} } } } files.sort(); files } /// One provenance manifest line, compact JSON, raw UTF-8 (python /// `json.dumps(..., ensure_ascii=False)`; keys order as archive/dest/unit). fn write_manifest_line( manifest: &mut W, out_root: &Path, archive_rel: &Path, actual_dest: &Path, ) { let dest_rel = rel_to(out_root, actual_dest); let unit = archive_rel .file_name() .map(|n| display_unit(&n.to_string_lossy())) .unwrap_or_default(); let line = serde_json::to_string(&serde_json::json!({ "archive": archive_rel.to_string_lossy().into_owned(), "dest": dest_rel.to_string_lossy().into_owned(), "unit": unit, })) .unwrap_or_else(|_| "{}".to_string()); if let Err(e) = writeln!(manifest, "{line}") { log::error!("Cannot write manifest line: {e}"); } } /// Run an extraction command like python `_run_7z`: merged stdout+stderr /// captured (stderr only when stdout is redirected to a file), tolerant /// decoding, per-archive timeout, returns (rc, last-5-lines tail). /// rc == -1 marks "failed to run" / timeout, like python. fn run_extract_cmd( program: &str, args: &[std::ffi::OsString], stdout_to: Option<&Path>, ) -> (i32, String) { let mut cmd = Command::new(program); cmd.args(args); let mut stdout_thread = None; let mut stderr_thread = None; match stdout_to { Some(path) => match File::create(path) { Ok(f) => { cmd.stdout(Stdio::from(f)); cmd.stderr(Stdio::piped()); } Err(e) => return (-1, format!("{program} failed to run: {e}")), }, None => { cmd.stdout(Stdio::piped()); cmd.stderr(Stdio::piped()); } } let mut child = match cmd.spawn() { Ok(c) => c, Err(e) => return (-1, format!("{program} failed to run: {e}")), }; if let Some(mut pipe) = child.stdout.take() { stdout_thread = Some(std::thread::spawn(move || { let mut s = String::new(); let _ = pipe.read_to_string(&mut s); s })); } if let Some(mut pipe) = child.stderr.take() { stderr_thread = Some(std::thread::spawn(move || { let mut s = String::new(); let _ = pipe.read_to_string(&mut s); s })); } let deadline = Instant::now() + Duration::from_secs(EXTRACT_TIMEOUT_SECS); let mut timed_out = false; let status = loop { match child.try_wait() { Ok(Some(st)) => break Some(st), Ok(None) => { if Instant::now() >= deadline { let _ = child.kill(); let _ = child.wait(); timed_out = true; break None; } std::thread::sleep(Duration::from_millis(25)); } Err(e) => { let _ = child.kill(); let _ = child.wait(); return (-1, format!("{program} wait failed: {e}")); } } }; let mut text = String::new(); if let Some(t) = stdout_thread { text.push_str(&t.join().unwrap_or_default()); } if let Some(t) = stderr_thread { text.push_str(&t.join().unwrap_or_default()); } if timed_out { return ( -1, format!("extraction timed out after {EXTRACT_TIMEOUT_SECS}s"), ); } let code = status.and_then(|s| s.code()).unwrap_or(-1); let tail = text .lines() .rev() .take(5) .collect::>() .into_iter() .rev() .collect::>() .join("\n"); (code, tail) } impl Extractor { fn space_low(&self) -> bool { match free_bytes(&self.out_root) { Ok(bytes) => bytes < MIN_FREE_BYTES, Err(e) => { log::warn!( "free-space check failed on {}: {e}", self.out_root.display() ); false } } } /// Create the destination directory, reproducing the python collision /// semantics: if the stem is occupied by a *file*, retry as `_dup` /// (python's loop always rebuilds from the original name, so repeated /// `_dup` collisions exhaust the 5 attempts); a parent that is a file /// (NotADirectoryError) fails immediately. fn create_dest(&self, dest: &Path) -> Result { let mut actual = dest.to_path_buf(); for _ in 0..5 { match fs::create_dir_all(&actual) { Ok(()) => return Ok(actual), Err(e) => match e.kind() { io::ErrorKind::AlreadyExists => { let name = file_name_string(dest); actual = dest.with_file_name(format!("{name}_dup")); } _ => { return Err(format!( "cannot create destination {}: {e}", actual.display() )); } }, } } Err(format!( "cannot create destination {} (repeated collisions)", dest.display() )) } /// Extract `archive` into `dest`. Returns (ok, error, dest_used); on a /// name collision the destination gets a `_dup` suffix instead of /// crashing. Never panics on archive content; every error is reported. pub fn extract(&self, archive: &Path, dest: &Path) -> (bool, String, PathBuf) { let actual = match self.create_dest(dest) { Ok(a) => a, Err(err) => return (false, err, dest.to_path_buf()), }; let name = file_name_string(archive); let sfx = archive_suffixes(&name); // Container formats: trust the magic bytes over the extension (python's // 7z CLI sniffs content too). Compressed streams (.gz/.bz2/.xz) and tar // stay extension-driven, where the tar-inside logic needs the name. let kind: Option = sniff_container(archive) .map(str::to_string) .or_else(|| sfx.last().cloned()); let res: Result<(), String> = match kind.as_deref() { Some(".7z") => extract_7z(archive, &actual), Some(".zip") => extract_zip(archive, &actual), Some(".rar") => extract_rar(archive, &actual), Some(outer @ (".gz" | ".bz2" | ".xz")) => { if sfx.len() >= 2 && sfx[1] == ".tar" { // .tar.gz|bz2|xz — decompress stream and unpack in // one go (python does it in two 7z passes via the nested // expansion; final tree content is identical). extract_tar_cmd(archive, &actual, Some(outer)) } else { // Bare compressed file (one layer, like a single `7z x`): // writes dest/; anything that is // still an archive afterwards is picked up by the // nested/sweep expansion. decompress_bare_cmd(archive, &actual, &name, outer) } } Some(".tar") => extract_tar_cmd(archive, &actual, None), // .tgz is a gzip-compressed tar: same as .tar.gz, but the tar flag // cannot be inferred from the name (Rust-only extension). Some(".tgz") => extract_tar_cmd(archive, &actual, Some(".gz")), _ => Err(format!("no extractor for {name}")), }; match res { Ok(()) => (true, String::new(), actual), Err(err) => (false, err, actual), } } /// Move a failed archive under `/.broken/`; the source /// root is otherwise never modified. Returns the quarantine path. pub fn move_to_broken(&self, path: &Path, src_root: &Path, note: &str) -> PathBuf { let rel = path.strip_prefix(src_root).unwrap_or(path); let dest = self.out_root.join(".broken").join(rel); if let Some(parent) = dest.parent() { let _ = fs::create_dir_all(parent); } let mut final_dest = dest.clone(); if dest.exists() { let mut name = dest .file_name() .map(|n| n.to_os_string()) .unwrap_or_default(); name.push(".1"); final_dest = dest.with_file_name(name); } log::debug!( "Quarantining {} -> {} ({note})", path.display(), final_dest.display() ); if let Err(e) = fs::rename(path, &final_dest) { // cross-device fallback: copy + remove (shutil.move semantics) match fs::copy(path, &final_dest).and_then(|_| fs::remove_file(path)) { Ok(()) => {} Err(e2) => { log::error!("Cannot quarantine {}: {e} / {e2}", path.display()); } } } final_dest } /// Extract archives inside `dest` until none remain (BFS). Returns the /// number of nested archives extracted. pub fn expand_nested( &self, dest: &Path, manifest: &mut W, done: &mut HashSet, ) -> usize { let mut done_count = 0; let mut queue: VecDeque = VecDeque::new(); queue.push_back(dest.to_path_buf()); while let Some(cur) = queue.pop_front() { for p in walk_files_sorted(&cur) { let name = file_name_string(&p); if !is_archive(&name) { continue; } if is_macosx_junk(&p) { let _ = fs::remove_file(&p); log::info!("Junk (macOS AppleDouble) removed: {name}"); continue; } let stem_dir = p.with_file_name(display_unit(&name)); if done.contains(&rel_string(&self.out_root, &stem_dir)) { // already extracted in a previous run; drop the archive copy let _ = fs::remove_file(&p); log::info!("Nested {name} already extracted - archive removed"); continue; } if self.space_low() { log::error!( "Out of disk space - skipping {} (kept in place)", p.display() ); continue; } let (ok, err, actual) = self.extract(&p, &stem_dir); if !ok { log::error!("FAILED nested extraction {}: {}", p.display(), err); self.move_to_broken(&p, &self.out_root, &err); continue; } write_manifest_line( manifest, &self.out_root, &rel_to(&self.out_root, &p), &actual, ); done.insert(rel_string(&self.out_root, &actual)); let _ = fs::remove_file(&p); // archive copy consumed done_count += 1; queue.push_back(actual); } } done_count } /// Extract every archive directly under `src_root` (any depth) into /// `//` and mirror plain /// files via hardlink (fallback: metadata-preserving copy). Returns /// (extracted, mirrored, failed). pub fn process_root( &self, src_root: &Path, manifest: &mut W, done: &mut HashSet, ) -> (usize, usize, usize) { let idx = file_name_string(src_root); let mut extracted = 0; let mut mirrored = 0; let mut failed = 0; for f in walk_files_sorted(src_root) { let fname = file_name_string(&f); if RUN_ARTIFACTS.contains(&fname.as_str()) || fname.starts_with('.') || fname.ends_with(".part") { continue; } let Ok(rel) = f.strip_prefix(src_root).map(|r| r.to_path_buf()) else { continue; }; let rel_str = rel.to_string_lossy().into_owned(); if is_archive(&fname) { let unit = display_unit(&fname); let dest_rel: PathBuf = Path::new(&idx).join(rel.with_file_name(unit)); let dest_rel_str = dest_rel.to_string_lossy().into_owned(); // Skip on a manifest record, never on "dir non-empty": a // crash mid-extraction leaves a partial dir that must be // redone. if done.contains(&dest_rel_str) { log::info!("Skip (already extracted): {rel_str}"); continue; } if self.space_low() { log::error!("Out of disk space - skipping archive {}", f.display()); failed += 1; continue; } let dest = self.out_root.join(&dest_rel); let (ok, err, actual) = self.extract(&f, &dest); if !ok { log::error!("FAILED {rel_str}: {err}"); self.move_to_broken(&f, src_root, &err); failed += 1; continue; } write_manifest_line( manifest, &self.out_root, &Path::new(&idx).join(&rel), &actual, ); let actual_rel = rel_string(&self.out_root, &actual); done.insert(actual_rel.clone()); extracted += 1; log::info!("Extracted {rel_str} -> {actual_rel}"); // python discards the nested count here too let _ = self.expand_nested(&actual, manifest, done); } else { let link_dest = self.out_root.join(&idx).join(&rel); if link_dest.exists() { continue; } if let Some(parent) = link_dest.parent() { let _ = fs::create_dir_all(parent); } if let Err(e) = fs::hard_link(&f, &link_dest) { // fallback: copy (fs::copy keeps permission bits, like copy2) log::debug!("hardlink {rel_str} failed ({e}) - falling back to copy"); if let Err(e2) = fs::copy(&f, &link_dest) { log::error!("Cannot mirror {rel_str}: {e2}"); continue; } } mirrored += 1; } } (extracted, mirrored, failed) } /// Expand any archive left anywhere in the tree (interrupted nested /// expansions leave stranded copies even when the parent has a record). pub fn sweep(&self, manifest: &mut W, done: &mut HashSet) -> usize { let mut swept = 0; for p in walk_files_sorted(&self.out_root) { let rel = rel_to(&self.out_root, &p); if rel.components().any(|c| c.as_os_str() == ".broken") { continue; } let name = file_name_string(&p); if !is_archive(&name) { continue; } if is_macosx_junk(&p) { let _ = fs::remove_file(&p); log::info!("Junk (macOS AppleDouble) removed: {name}"); continue; } let stem_dir = p.with_file_name(display_unit(&name)); if done.contains(&rel_string(&self.out_root, &stem_dir)) { let _ = fs::remove_file(&p); log::info!("Sweep: {name} already extracted - archive removed"); continue; } if self.space_low() { log::error!("Out of disk space - sweep skipping {}", p.display()); continue; } let (ok, err, actual) = self.extract(&p, &stem_dir); if !ok { log::error!("FAILED sweep extraction {}: {}", p.display(), err); self.move_to_broken(&p, &self.out_root, &err); continue; } write_manifest_line(manifest, &self.out_root, &rel, &actual); done.insert(rel_string(&self.out_root, &actual)); let _ = fs::remove_file(&p); swept += 1; self.expand_nested(&actual, manifest, done); } swept } /// Full run over `sources`: process each root, then one sweep. Failures /// are quarantined and logged; the run never stops on individual errors. pub fn run(&self, sources: &[PathBuf]) -> anyhow::Result<()> { fs::create_dir_all(&self.out_root)?; let manifest_path = self.out_root.join("extract_manifest.jsonl"); let mut done = load_done(&manifest_path); log::info!("{} destinations already recorded in manifest", done.len()); let mut manifest = fs::OpenOptions::new() .append(true) .create(true) .open(&manifest_path)?; let mut total_extracted = 0; let mut total_mirrored = 0; let mut total_failed = 0; for src in sources { log::info!("== processing {}", src.display()); let (e, m, f) = self.process_root(src, &mut manifest, &mut done); log::info!( "== {}: {e} extracted, {m} mirrored, {f} failed", file_name_string(src) ); total_extracted += e; total_mirrored += m; total_failed += f; } let swept = self.sweep(&mut manifest, &mut done); log::info!( "TOTAL: {total_extracted} archives extracted, {total_mirrored} files mirrored, \ {total_failed} failed, {swept} swept" ); if total_failed > 0 { log::warn!( "Failures were moved under {} - inspect and re-run", self.out_root.join(".broken").display() ); } Ok(()) } } /// Dest dirs already recorded in the manifest (relative to out root). /// Malformed lines are skipped. pub fn load_done(manifest_path: &Path) -> HashSet { let mut done = HashSet::new(); let Ok(bytes) = fs::read(manifest_path) else { return done; }; for line in bytes.split(|&b| b == b'\n') { let line = String::from_utf8_lossy(line); let line = line.trim(); if line.is_empty() { continue; } if let Ok(value) = serde_json::from_str::(line) { if let Some(dest) = value.get("dest").and_then(|d| d.as_str()) { done.insert(dest.to_string()); } } } done } /// 7z via sevenz-rust2. Empty password: python passes `-p ""` (no password, /// avoids interactive prompts) — `Password::empty()` is the same thing; the /// password-less helper is retried once in case the with-password path /// rejects an empty password outright. Password-protected archives still /// fail, exactly like the python version (no password known). fn extract_7z(archive: &Path, dest: &Path) -> Result<(), String> { match sevenz_rust2::decompress_file_with_password( archive, dest, sevenz_rust2::Password::empty(), ) { Ok(()) => Ok(()), Err(first) => match sevenz_rust2::decompress_file(archive, dest) { Ok(()) => Ok(()), Err(second) => Err(format!("7z: {first} / {second}")), }, } } /// zip via the zip crate, entry by entry. Traversal-safe: entries whose raw /// name contains `..` or that cannot be enclosed under dest are skipped /// (python relies on 7z being safe — same minimum). fn extract_zip(archive: &Path, dest: &Path) -> Result<(), String> { let file = File::open(archive).map_err(|e| format!("zip: {e}"))?; let mut zip = zip::ZipArchive::new(file).map_err(|e| format!("zip: {e}"))?; for i in 0..zip.len() { let mut entry = zip.by_index(i).map_err(|e| format!("zip: {e}"))?; if entry.name().split('/').any(|part| part == "..") { log::warn!("zip: rejecting traversal entry {:?}", entry.name_raw()); continue; } let Some(path) = entry.enclosed_name() else { log::warn!("zip: skipping unsafe entry {:?}", entry.name_raw()); continue; }; let out_path = dest.join(&path); if entry.is_dir() { fs::create_dir_all(&out_path).map_err(|e| format!("zip: {e}"))?; } else { if let Some(parent) = out_path.parent() { fs::create_dir_all(parent).map_err(|e| format!("zip: {e}"))?; } let mut out_file = File::create(&out_path).map_err(|e| format!("zip: {e}"))?; io::copy(&mut entry, &mut out_file).map_err(|e| format!("zip: {e}"))?; #[cfg(unix)] if let Some(mode) = entry.unix_mode() { let _ = fs::set_permissions(&out_path, fs::Permissions::from_mode(mode)); } } } Ok(()) } /// rar via the unrar crate (C library), no password. Entries with `..` /// components are skipped rather than extracted. (Untested offline: .rar /// archives cannot be created without the proprietary tool.) fn extract_rar(archive: &Path, dest: &Path) -> Result<(), String> { fs::create_dir_all(dest).map_err(|e| format!("rar: {e}"))?; let mut arc = unrar::Archive::new(archive) .open_for_processing() .map_err(|e| format!("rar: {e}"))?; loop { match arc.read_header() { Ok(None) => break, Ok(Some(header)) => { let fname = header.entry().filename.clone(); if fname .components() .any(|c| matches!(c, Component::ParentDir)) { log::warn!("rar: skipping entry with '..': {}", fname.display()); arc = header.skip().map_err(|e| format!("rar: {e}"))?; continue; } arc = header .extract_with_base(dest) .map_err(|e| format!("rar: {e}"))?; } Err(e) => return Err(format!("rar: {e}")), } } Ok(()) } /// Tarballs via the system `tar` binary — the same shape as the python /// version's `7z` subprocess call: merged output, rc 0/1 = ok (1 logs a /// warning), 4h timeout. `decompress` selects -z/-j/-J or None for bare tar. fn extract_tar_cmd(archive: &Path, dest: &Path, decompress: Option<&str>) -> Result<(), String> { fs::create_dir_all(dest).map_err(|e| format!("tar: {e}"))?; let mut args: Vec = vec!["-x".into()]; if let Some(flag) = decompress { args.push( match flag { ".gz" => "-z", ".bz2" => "-j", ".xz" => "-J", _ => unreachable!("only gz/bz2/xz reach this branch"), } .into(), ); } args.push("-f".into()); args.push(archive.as_os_str().to_os_string()); args.push("-C".into()); args.push(dest.as_os_str().to_os_string()); let (rc, tail) = run_extract_cmd("tar", &args, None); match rc { 0 => Ok(()), 1 => { log::warn!( "tar reported warnings extracting {}", file_name_string(archive) ); Ok(()) } _ => Err(format!("tar exit {rc}: {tail}")), } } /// Single-layer decompress of a bare `.gz`/`.bz2`/`.xz` (or any trailing /// compressed suffix that is not a tarball) into `dest/` via /// the system decompressor — one layer, like a single `7z x` call. fn decompress_bare_cmd(archive: &Path, dest: &Path, name: &str, outer: &str) -> Result<(), String> { let inner = &name[..name.len() - outer.len()]; fs::create_dir_all(dest).map_err(|e| format!("{outer}: {e}"))?; let out_path = dest.join(inner); let (program, what) = match outer { ".gz" => ("gzip", "gz"), ".bz2" => ("bzip2", "bz2"), _ => ("xz", "xz"), }; let args = vec![ std::ffi::OsString::from("-dc"), archive.as_os_str().to_os_string(), ]; let (rc, tail) = run_extract_cmd(program, &args, Some(&out_path)); if rc == 0 { Ok(()) } else { Err(format!("{what} exit {rc}: {tail}")) } } #[cfg(test)] mod tests { use super::*; use serde_json::Value; use std::collections::BTreeSet; /// Write a real zip archive containing the given (path, content) files. fn write_zip(path: &Path, files: &[(&str, Vec)]) { let file = File::create(path).unwrap(); let mut w = zip::ZipWriter::new(file); for (name, data) in files { w.start_file(*name, zip::write::SimpleFileOptions::default()) .unwrap(); w.write_all(data).unwrap(); } w.finish().unwrap(); } /// Minimal CRC32 (IEEE) for the hand-built gzip container in tests. fn crc32(data: &[u8]) -> u32 { let mut table = [0u32; 256]; for (i, slot) in table.iter_mut().enumerate() { let mut c = i as u32; for _ in 0..8 { c = if c & 1 != 0 { 0xEDB88320 ^ (c >> 1) } else { c >> 1 }; } *slot = c; } let mut crc = 0xFFFF_FFFFu32; for &b in data { crc = table[((crc ^ b as u32) & 0xFF) as usize] ^ (crc >> 8); } crc ^ 0xFFFF_FFFF } /// Valid gzip container wrapping a single *stored* (uncompressed) /// deflate block — no compression crate needed to build a test fixture. fn gzip_stored(data: &[u8]) -> Vec { let mut v = vec![0x1f, 0x8b, 0x08, 0, 0, 0, 0, 0, 0, 0x03]; v.push(0x01); // BFINAL=1, BTYPE=00 (stored) let len = data.len() as u16; v.extend_from_slice(&len.to_le_bytes()); v.extend_from_slice(&(!len).to_le_bytes()); v.extend_from_slice(data); v.extend_from_slice(&crc32(data).to_le_bytes()); v.extend_from_slice(&(data.len() as u32).to_le_bytes()); v } /// Minimal USTAR archive with one file entry, terminated by two zero /// blocks — readable by the system tar. fn make_ustar(name: &str, data: &[u8]) -> Vec { let mut block = vec![0u8; 512]; block[..name.len()].copy_from_slice(name.as_bytes()); block[100..108].copy_from_slice(b"0000644\0"); // mode block[108..116].copy_from_slice(b"0000000\0"); // uid block[116..124].copy_from_slice(b"0000000\0"); // gid let size_field = format!("{:011o}\0", data.len()); block[124..136].copy_from_slice(size_field.as_bytes()); // size block[136..148].copy_from_slice(b"00000000000\0"); // mtime block[148..156].copy_from_slice(b" "); // chksum placeholder block[156] = b'0'; // typeflag: regular file block[257..263].copy_from_slice(b"ustar\0"); block[263..265].copy_from_slice(b"00"); let sum: u32 = block.iter().map(|&b| b as u32).sum(); let chk = format!("{:06o}\0 ", sum); block[148..156].copy_from_slice(chk.as_bytes()); let mut out = block; out.extend_from_slice(data); while out.len() % 512 != 0 { out.push(0); } out.extend(std::iter::repeat_n(0u8, 1024)); out } /// Base dir for test tempdirs: /tmp is often a small tmpfs (< /// MIN_FREE_BYTES), which would trip the real free-space check; prefer /// /var/tmp when it clears the threshold (same convention as the /// python version's /var/tmp staging). fn test_base() -> PathBuf { let var_tmp = Path::new("/var/tmp"); if cfg!(unix) && free_bytes(var_tmp) .map(|b| b >= MIN_FREE_BYTES) .unwrap_or(false) { var_tmp.to_path_buf() } else { std::env::temp_dir() } } /// Build a source root (`src1`) + output root (`out`) inside a /// container tempdir; returns (container, src, out). The source root's /// file name (`src1`) is what the extractor uses as the index folder. fn build_fixtures() -> (tempfile::TempDir, PathBuf, PathBuf) { let _ = env_logger::builder() .filter_level(log::LevelFilter::Info) .is_test(true) .try_init(); let container = tempfile::tempdir_in(test_base()).unwrap(); let src = container.path().join("src1"); let out = container.path().join("out"); fs::create_dir_all(&src).unwrap(); fs::create_dir_all(&out).unwrap(); let s = src.as_path(); // Pack.zip: files + a nested zip inside let inner_path = s.join(".tmp_inner.zip"); write_zip(&inner_path, &[("beta.txt", b"inner payload".to_vec())]); let inner_bytes = fs::read(&inner_path).unwrap(); fs::remove_file(&inner_path).unwrap(); write_zip( &s.join("Pack.zip"), &[ ("alpha.txt", b"alpha payload".to_vec()), ("nested/inner.zip", inner_bytes), ], ); // a real .7z let gamma = s.join("gamma.txt"); fs::write(&gamma, b"sevenz payload").unwrap(); let sub = s.join("Sub"); fs::create_dir_all(&sub).unwrap(); sevenz_rust2::compress_to_path(&gamma, sub.join("The Twin Lance.7z")).expect("compress 7z"); fs::remove_file(&gamma).unwrap(); // a tar.gz (hand-built ustar + gzip-stored container) let targz = gzip_stored(&make_ustar("delta.txt", b"tarred payload")); fs::write(s.join("logs.tar.gz"), targz).unwrap(); // a .tgz (gzip-compressed tar under the short suffix — Rust-only) let tgz = gzip_stored(&make_ustar("epsilon.txt", b"tgz payload")); fs::write(s.join("bundle.tgz"), tgz).unwrap(); // garbage zip -> must land in .broken fs::write(s.join("bad.zip"), b"this is not a zip archive at all").unwrap(); // plain file (mirrored via hardlink) fs::write(s.join("notes.txt"), b"plain notes").unwrap(); // run artifacts / hidden / partial: must be skipped fs::write(s.join("state.json"), b"{}").unwrap(); fs::write(s.join(".hidden.txt"), b"secret").unwrap(); fs::write(s.join("partial.part"), b"partial").unwrap(); (container, src, out) } fn inventory(root: &Path) -> BTreeSet { walk_files_sorted(root) .iter() .map(|p| rel_to(root, p).to_string_lossy().into_owned()) .collect() } fn find_file_named(root: &Path, name: &str) -> Option { walk_files_sorted(root).into_iter().find(|p| { p.file_name() .map(|n| n.to_string_lossy() == name) .unwrap_or(false) }) } fn manifest_values(out: &Path) -> Vec { let text = fs::read_to_string(out.join("extract_manifest.jsonl")).unwrap(); text.lines() .filter(|l| !l.trim().is_empty()) .map(|l| serde_json::from_str(l).unwrap()) .collect() } #[test] fn sniff_container_reads_magic_not_extension() { let dir = tempfile::tempdir().unwrap(); // real zip content, .rar name -> sniffed as zip let z = dir.path().join("x.rar"); write_zip(&z, &[("a.txt", b"hi".to_vec())]); assert_eq!(sniff_container(&z), Some(".zip")); // 7z magic, .zip name let seven = dir.path().join("y.zip"); fs::write(&seven, [0x37u8, 0x7a, 0xbc, 0xaf, 0x27, 0x1c, 0, 4]).unwrap(); assert_eq!(sniff_container(&seven), Some(".7z")); // rar5 magic let rar = dir.path().join("z.7z"); fs::write(&rar, b"Rar!\x1a\x07\x01\x00").unwrap(); assert_eq!(sniff_container(&rar), Some(".rar")); // gzip is not a container: extension keeps driving it let gz = dir.path().join("g.gz"); fs::write(&gz, [0x1f, 0x8b, 0x08]).unwrap(); assert_eq!(sniff_container(&gz), None); } #[test] fn mislabeled_zip_extracts_via_sniffing() { // /tmp is a small tmpfs; the free-space guard needs a real fs let dir = test_base().join(format!("dlp-mislabel-{}", std::process::id())); let _ = fs::remove_dir_all(&dir); let src = dir.join("src"); let out = dir.join("out"); fs::create_dir_all(&src).unwrap(); // a real zip with a .rar extension: python's 7z CLI would extract it too write_zip( &src.join("Pack.rar"), &[("unit/part.txt", b"payload".to_vec())], ); let extractor = Extractor { out_root: out.clone(), }; extractor.run(&[src.clone()]).unwrap(); assert_eq!( fs::read_to_string(out.join("src/Pack/unit/part.txt")).unwrap(), "payload" ); let broken_empty = fs::read_dir(out.join(".broken")) .map(|d| d.count() == 0) .unwrap_or(true); assert!(broken_empty, "nothing should be quarantined"); let _ = fs::remove_dir_all(&dir); } #[test] fn run_extracts_nests_quarantines_and_mirrors() { let (_c, src, out) = build_fixtures(); let extractor = Extractor { out_root: out.clone(), }; extractor.run(&[src.clone()]).unwrap(); let s = src.as_path(); let o = out.as_path(); // extracted content, including the nested zip inside Pack.zip assert_eq!( fs::read_to_string(o.join("src1/Pack/alpha.txt")).unwrap(), "alpha payload" ); assert_eq!( fs::read_to_string(o.join("src1/Pack/nested/inner/beta.txt")).unwrap(), "inner payload" ); assert!( !o.join("src1/Pack/nested/inner.zip").exists(), "nested archive must be consumed" ); // 7z and tar.gz assert!( o.join("src1/Sub/The Twin Lance").is_dir(), "7z unit dir missing" ); assert!(find_file_named(&o.join("src1/Sub/The Twin Lance"), "gamma.txt").is_some()); assert_eq!( fs::read_to_string(o.join("src1/logs/delta.txt")).unwrap(), "tarred payload" ); assert_eq!( fs::read_to_string(o.join("src1/bundle/epsilon.txt")).unwrap(), "tgz payload" ); // no archives left in the extracted tree (outside .broken) for p in walk_files_sorted(o) { let rel = rel_to(o, &p); if rel.components().any(|c| c.as_os_str() == ".broken") { continue; } assert!( !is_archive(&file_name_string(&p)), "archive left in extracted tree: {}", p.display() ); } // garbage zip quarantined under .broken let broken = o.join(".broken/bad.zip"); assert!(broken.is_file(), "bad.zip not quarantined"); assert_eq!( fs::read(&broken).unwrap(), b"this is not a zip archive at all" ); // plain file mirrored: same inode on hardlink, else copy fallback #[cfg(unix)] { use std::os::unix::fs::MetadataExt; let sm = fs::metadata(s.join("notes.txt")).unwrap(); let om = fs::metadata(o.join("src1/notes.txt")).unwrap(); if sm.ino() == om.ino() { assert_eq!(sm.dev(), om.dev(), "hardlink must be on the same fs"); } else { assert_eq!(sm.len(), om.len(), "hardlink fell back to copy"); } } assert_eq!( fs::read_to_string(o.join("src1/notes.txt")).unwrap(), "plain notes" ); // run artifacts / hidden / partial never enter the tree for junk in ["state.json", ".hidden.txt", "partial.part"] { assert!( find_file_named(o, junk).is_none(), "{junk} leaked into output" ); } // source root keeps its originals (except the quarantined bad.zip) assert!(s.join("Pack.zip").is_file()); assert!(s.join("notes.txt").is_file()); assert!(s.join("Sub/The Twin Lance.7z").is_file()); assert!( !s.join("bad.zip").exists(), "failed archive must be moved away" ); // manifest: one line per extraction, exactly the 3 provenance keys let lines = manifest_values(o); let triples: BTreeSet<(String, String, String)> = lines .iter() .map(|v| { let obj = v.as_object().unwrap(); assert_eq!(obj.len(), 3, "unexpected keys in manifest line: {v}"); ( v["archive"].as_str().unwrap().to_string(), v["dest"].as_str().unwrap().to_string(), v["unit"].as_str().unwrap().to_string(), ) }) .collect(); let expected: BTreeSet<(String, String, String)> = [ ("src1/Pack.zip", "src1/Pack", "Pack"), ( "src1/Pack/nested/inner.zip", "src1/Pack/nested/inner", "inner", ), ( "src1/Sub/The Twin Lance.7z", "src1/Sub/The Twin Lance", "The Twin Lance", ), ("src1/logs.tar.gz", "src1/logs", "logs"), ("src1/bundle.tgz", "src1/bundle", "bundle"), ] .iter() .map(|(a, d, u)| (a.to_string(), d.to_string(), u.to_string())) .collect(); assert_eq!(triples, expected); } #[test] fn rerun_is_idempotent() { let (_c, src, out) = build_fixtures(); let extractor = Extractor { out_root: out.clone(), }; extractor.run(&[src.clone()]).unwrap(); let manifest_before = fs::read_to_string(out.join("extract_manifest.jsonl")).unwrap(); let inv_before = inventory(&out); assert!(!manifest_before.is_empty()); extractor.run(&[src.clone()]).unwrap(); let manifest_after = fs::read_to_string(out.join("extract_manifest.jsonl")).unwrap(); assert_eq!( manifest_before, manifest_after, "re-run must not append manifest lines" ); assert_eq!( inv_before, inventory(&out), "re-run must not change the tree" ); assert!( !out.as_path().join("src1/Pack_dup").exists() && !out.as_path().join("src1/Pack_dup_dup").exists(), "re-run must not create duplicate destinations" ); // the previously extracted content is still intact assert_eq!( fs::read_to_string(out.as_path().join("src1/Pack/alpha.txt")).unwrap(), "alpha payload" ); } #[test] fn dest_collision_appends_dup() { let dir = tempfile::tempdir_in(test_base()).unwrap(); let ex = Extractor { out_root: dir.path().join("out"), }; fs::create_dir_all(&ex.out_root).unwrap(); // dest occupied by a FILE -> the *_dup sibling is used (python: // mkdir exist_ok raises FileExistsError -> append "_dup") let dest = dir.path().join("d1"); fs::write(&dest, b"occupied").unwrap(); let actual = ex.create_dest(&dest).unwrap(); assert_eq!(actual, dest.with_file_name("d1_dup")); assert!(actual.is_dir()); // dest and *_dup both occupied by files -> graceful failure after // the 5 attempts (python's loop rebuilds "_dup" from the original // name, so it keeps hitting the same path) let dest2 = dir.path().join("d2"); fs::write(&dest2, b"occupied").unwrap(); fs::write(dir.path().join("d2_dup"), b"occupied too").unwrap(); let err = ex.create_dest(&dest2).unwrap_err(); assert!(err.contains("repeated collisions"), "{err}"); // parent path is a file (NotADirectoryError) -> immediate failure let blocker = dir.path().join("blocker"); fs::write(&blocker, b"file").unwrap(); let err2 = ex.create_dest(&blocker.join("sub")).unwrap_err(); assert!(err2.contains("cannot create destination"), "{err2}"); } #[test] fn suffix_helpers_match_python() { assert_eq!(archive_suffixes("x.tar.gz"), vec![".gz", ".tar"]); assert_eq!(display_unit("x.tar.gz"), "x"); assert_eq!(archive_suffixes("The Twin Lance.7z"), vec![".7z"]); assert_eq!(display_unit("The Twin Lance.7z"), "The Twin Lance"); assert_eq!(archive_suffixes("X.ZIP"), vec![".zip"]); assert_eq!(display_unit("X.ZIP"), "X"); assert!(archive_suffixes("plain.txt").is_empty()); assert!(!is_archive("plain.txt")); assert!( archive_suffixes(".hidden").is_empty(), "dotfiles have no suffix" ); assert_eq!(display_unit("Pack.zip"), "Pack"); assert!(is_archive("c.7z.7z")); // .tgz: Rust-only short suffix for a gzip-compressed tar assert_eq!(archive_suffixes("x.tgz"), vec![".tgz"]); assert_eq!(display_unit("x.tgz"), "x"); assert!(is_archive("X.TGZ")); } #[test] fn macosx_junk_detection() { assert!(is_macosx_junk(Path::new("out/1/__MACOSX/._x.zip"))); assert!(is_macosx_junk(Path::new("._x.zip"))); assert!(is_macosx_junk(Path::new("deep/nested/._resource"))); assert!(is_macosx_junk(Path::new("a/__MACOSX"))); assert!(!is_macosx_junk(Path::new("a.zip"))); assert!(!is_macosx_junk(Path::new("plain/notes.txt"))); assert!( !is_macosx_junk(Path::new("macosx/_a.zip")), "__MACOSX matching is case-sensitive like python" ); } }