From 2b5dadc0a87076e915c5ddae9436c0339e4f5680 Mon Sep 17 00:00:00 2001 From: Ben_Kosytorz Date: Tue, 6 Oct 2026 16:51:33 +0200 Subject: [PATCH] extract_all: CIFS-safe fallback extraction via /var/tmp with cp437 filename repair; delete macOS AppleDouble junk --- tools/extract_all.py | 123 +++++++++++++++++++++++++++++++++++-------- 1 file changed, 101 insertions(+), 22 deletions(-) diff --git a/tools/extract_all.py b/tools/extract_all.py index 04108a5..0edd665 100644 --- a/tools/extract_all.py +++ b/tools/extract_all.py @@ -32,6 +32,7 @@ import os import shutil import subprocess import sys +import tempfile from collections import deque from pathlib import Path @@ -75,25 +76,51 @@ def free_bytes(path: Path) -> int: return shutil.disk_usage(path).free -def extract(archive: Path, dest: Path) -> tuple[bool, str, Path]: - """7z x into dest. Empty -p avoids interactive prompts (no password). +def _fix_component(s: str) -> str: + """Repair filenames 7z wrote with raw legacy bytes (cp437 zip entries). - Returns (ok, error, dest_used): on a name collision (the archive stem - exists as a *file*, e.g. 'model.stl.zip' next to 'model.stl') the - destination gets a '_dup' suffix instead of crashing. + Raw non-UTF8 bytes in a name cannot be created on the CIFS share + (observed: errno 2 for a name containing byte 0x87). Decode the raw + bytes as cp437 so e.g. a raw-0x87 name becomes a proper 'รง' name. """ - actual = dest - for _ in range(5): + try: + s.encode("utf-8") + return s + except UnicodeEncodeError: + pass + raw = os.fsencode(s) + try: + return raw.decode("utf-8") + except UnicodeDecodeError: + return raw.decode("cp437", "replace") + + +def _move_tree(tmp: Path, dest: Path) -> int: + """Move a freshly extracted local tree onto the (possibly CIFS) dest. + Returns the number of files that could not be moved.""" + bad = 0 + for f in sorted(tmp.rglob("*")): + rel = f.relative_to(tmp) + tgt = dest.joinpath(*(_fix_component(p) for p in rel.parts)) + if f.is_dir(): + tgt.mkdir(parents=True, exist_ok=True) + continue + tgt.parent.mkdir(parents=True, exist_ok=True) try: - actual.mkdir(parents=True, exist_ok=True) - break - except FileExistsError: - actual = dest.with_name(dest.name + "_dup") - except (NotADirectoryError, FileExistsError) as exc: - return False, f"cannot create destination {actual}: {exc}", dest - else: - return False, f"cannot create destination {dest} (repeated collisions)", dest - cmd = ["7z", "x", "-y", "-p", f"-o{actual}", str(archive)] + if tgt.exists() and tgt.is_file() \ + and tgt.stat().st_size == f.stat().st_size: + f.unlink() + continue + if tgt.exists(): + tgt.unlink() + shutil.move(str(f), str(tgt)) + except OSError as exc: + bad += 1 + log.error("Cannot place %s: %s", tgt, exc) + return bad + + +def _run_7z(cmd: list[str]) -> tuple[int, str]: try: # 7z prints legacy-encoded inner filenames (non-UTF8 bytes); decode # tolerantly or the whole run crashes (seen with byte 0x87). @@ -102,15 +129,54 @@ def extract(archive: Path, dest: Path) -> tuple[bool, str, Path]: encoding="utf-8", errors="replace", timeout=EXTRACT_TIMEOUT, ) except subprocess.TimeoutExpired: - return False, f"extraction timed out after {EXTRACT_TIMEOUT}s", actual + return -1, f"extraction timed out after {EXTRACT_TIMEOUT}s" except OSError as exc: - return False, f"7z failed to run: {exc}", actual - if proc.returncode in (0, 1): # 0 ok, 1 warnings (e.g. minor CRC) - if proc.returncode == 1: + return -1, f"7z failed to run: {exc}" + tail = "\n".join((proc.stdout or "").splitlines()[-5:]) + return proc.returncode, tail + + +def extract(archive: Path, dest: Path) -> tuple[bool, str, Path]: + """7z x into dest. Empty -p avoids interactive prompts (no password). + + Returns (ok, error, dest_used): on a name collision (the archive stem + exists as a *file*, e.g. 'model.stl.zip' next to 'model.stl') the + destination gets a '_dup' suffix instead of crashing. + + If 7z cannot write directly onto the (CIFS) destination - legacy + cp437 entry names - the archive is extracted to a local temp dir and + moved over with repaired filenames. + """ + actual = dest + for _ in range(5): + try: + actual.mkdir(parents=True, exist_ok=True) + break + except FileExistsError: + actual = dest.with_name(dest.name + "_dup") + except NotADirectoryError as exc: + return False, f"cannot create destination {actual}: {exc}", dest + else: + return False, f"cannot create destination {dest} (repeated collisions)", dest + rc, tail = _run_7z(["7z", "x", "-y", "-p", f"-o{actual}", str(archive)]) + if rc in (0, 1): # 0 ok, 1 warnings (e.g. minor CRC) + if rc == 1: log.warning("7z reported warnings extracting %s", archive.name) return True, "", actual - tail = "\n".join((proc.stdout or "").splitlines()[-5:]) - return False, f"7z exit {proc.returncode}: {tail}", actual + if rc == -1: + return False, tail, actual + # Fallback: local extraction, then move with repaired filenames. + tmp = Path(tempfile.mkdtemp(prefix="7zext-", dir="/var/tmp")) + try: + rc2, tail2 = _run_7z(["7z", "x", "-y", "-p", f"-o{tmp}", str(archive)]) + if rc2 not in (0, 1): + return False, f"7z exit {rc2}: {tail2}", actual + bad = _move_tree(tmp, actual) + if bad: + return False, f"{bad} files could not be moved onto the share", actual + return True, "", actual + finally: + shutil.rmtree(tmp, ignore_errors=True) def move_to_broken(path: Path, out_root: Path, src_root: Path, note: str) -> Path: @@ -122,6 +188,11 @@ def move_to_broken(path: Path, out_root: Path, src_root: Path, note: str) -> Pat return shutil.move(str(path), str(dest)) +def is_macosx_junk(p: Path) -> bool: + """AppleDouble resource-fork 'zips' (.__MACOSX/._x.zip) - no content.""" + return "__MACOSX" in p.parts or p.name.startswith("._") + + def expand_nested(dest: Path, out_root: Path, manifest, src_root: Path, done: set[str]) -> int: """Extract archives inside `dest` until none remain. Returns count.""" @@ -132,6 +203,10 @@ def expand_nested(dest: Path, out_root: Path, manifest, src_root: Path, for p in sorted(cur.rglob("*")): if not p.is_file() or not is_archive(p): continue + if is_macosx_junk(p): + p.unlink() + log.info("Junk (macOS AppleDouble) removed: %s", p.name) + continue stem_dir = p.with_name(display_unit(p.name)) if str(stem_dir.relative_to(out_root)) in done: # already extracted in a previous run; drop the archive copy @@ -240,6 +315,10 @@ def sweep(out_root: Path, manifest, done: set[str]) -> int: continue if ".broken" in p.relative_to(out_root).parts: continue + if is_macosx_junk(p): + p.unlink() + log.info("Junk (macOS AppleDouble) removed: %s", p.name) + continue stem_dir = p.with_name(display_unit(p.name)) if str(stem_dir.relative_to(out_root)) in done: p.unlink()