sorter: skip hidden trees (.broken) in the plan; treat extract_manifest.jsonl as run artifact

dlphelper/sorter.py

@@ -41,6 +41,7 @@ RUN_ARTIFACTS = {
     "failed_urls.txt", "catalogue.md", "catalogue.jsonl",
     "sorter_applied.jsonl", "catalogue_moves.jsonl", "urls.txt",
     "urls_0807.txt", "urls_gofile.txt", "failed_0807.txt",
+    "extract_manifest.jsonl",
 }
 
 

sorter.py

@@ -64,7 +64,11 @@ def main() -> None:
                         datefmt="%H:%M:%S")
 
     source = args.source
-    files = [source] if source.is_file() else sorted(source.rglob("*"))
+    files = [source] if source.is_file() else sorted(
+        p for p in source.rglob("*")
+        # hidden trees like .broken/ must never reach the plan
+        if not any(part.startswith(".") for part in p.relative_to(source).parts)
+    )
     catalog = load_catalog(args.catalog)
     audit = args.audit or (args.library / "sorter_applied.jsonl")
 

tools/extract_all.py

@@ -32,6 +32,7 @@ import os
 import shutil
 import subprocess
 import sys
+import tempfile
 from collections import deque
 from pathlib import Path
 
@@ -75,25 +76,51 @@ def free_bytes(path: Path) -> int:
     return shutil.disk_usage(path).free
 
 
-def extract(archive: Path, dest: Path) -> tuple[bool, str, Path]:
-    """7z x into dest. Empty -p avoids interactive prompts (no password).
+def _fix_component(s: str) -> str:
+    """Repair filenames 7z wrote with raw legacy bytes (cp437 zip entries).
 
-    Returns (ok, error, dest_used): on a name collision (the archive stem
-    exists as a *file*, e.g. 'model.stl.zip' next to 'model.stl') the
-    destination gets a '_dup' suffix instead of crashing.
+    Raw non-UTF8 bytes in a name cannot be created on the CIFS share
+    (observed: errno 2 for a name containing byte 0x87). Decode the raw
+    bytes as cp437 so e.g. a raw-0x87 name becomes a proper 'ç' name.
     """
-    actual = dest
-    for _ in range(5):
+    try:
+        s.encode("utf-8")
+        return s
+    except UnicodeEncodeError:
+        pass
+    raw = os.fsencode(s)
+    try:
+        return raw.decode("utf-8")
+    except UnicodeDecodeError:
+        return raw.decode("cp437", "replace")
+
+
+def _move_tree(tmp: Path, dest: Path) -> int:
+    """Move a freshly extracted local tree onto the (possibly CIFS) dest.
+    Returns the number of files that could not be moved."""
+    bad = 0
+    for f in sorted(tmp.rglob("*")):
+        rel = f.relative_to(tmp)
+        tgt = dest.joinpath(*(_fix_component(p) for p in rel.parts))
+        if f.is_dir():
+            tgt.mkdir(parents=True, exist_ok=True)
+            continue
+        tgt.parent.mkdir(parents=True, exist_ok=True)
         try:
-            actual.mkdir(parents=True, exist_ok=True)
-            break
-        except FileExistsError:
-            actual = dest.with_name(dest.name + "_dup")
-        except (NotADirectoryError, FileExistsError) as exc:
-            return False, f"cannot create destination {actual}: {exc}", dest
-    else:
-        return False, f"cannot create destination {dest} (repeated collisions)", dest
-    cmd = ["7z", "x", "-y", "-p", f"-o{actual}", str(archive)]
+            if tgt.exists() and tgt.is_file() \
+                    and tgt.stat().st_size == f.stat().st_size:
+                f.unlink()
+                continue
+            if tgt.exists():
+                tgt.unlink()
+            shutil.move(str(f), str(tgt))
+        except OSError as exc:
+            bad += 1
+            log.error("Cannot place %s: %s", tgt, exc)
+    return bad
+
+
+def _run_7z(cmd: list[str]) -> tuple[int, str]:
     try:
         # 7z prints legacy-encoded inner filenames (non-UTF8 bytes); decode
         # tolerantly or the whole run crashes (seen with byte 0x87).
@@ -102,15 +129,54 @@ def extract(archive: Path, dest: Path) -> tuple[bool, str, Path]:
             encoding="utf-8", errors="replace", timeout=EXTRACT_TIMEOUT,
         )
     except subprocess.TimeoutExpired:
-        return False, f"extraction timed out after {EXTRACT_TIMEOUT}s", actual
+        return -1, f"extraction timed out after {EXTRACT_TIMEOUT}s"
     except OSError as exc:
-        return False, f"7z failed to run: {exc}", actual
-    if proc.returncode in (0, 1):  # 0 ok, 1 warnings (e.g. minor CRC)
-        if proc.returncode == 1:
+        return -1, f"7z failed to run: {exc}"
+    tail = "\n".join((proc.stdout or "").splitlines()[-5:])
+    return proc.returncode, tail
+
+
+def extract(archive: Path, dest: Path) -> tuple[bool, str, Path]:
+    """7z x into dest. Empty -p avoids interactive prompts (no password).
+
+    Returns (ok, error, dest_used): on a name collision (the archive stem
+    exists as a *file*, e.g. 'model.stl.zip' next to 'model.stl') the
+    destination gets a '_dup' suffix instead of crashing.
+
+    If 7z cannot write directly onto the (CIFS) destination - legacy
+    cp437 entry names - the archive is extracted to a local temp dir and
+    moved over with repaired filenames.
+    """
+    actual = dest
+    for _ in range(5):
+        try:
+            actual.mkdir(parents=True, exist_ok=True)
+            break
+        except FileExistsError:
+            actual = dest.with_name(dest.name + "_dup")
+        except NotADirectoryError as exc:
+            return False, f"cannot create destination {actual}: {exc}", dest
+    else:
+        return False, f"cannot create destination {dest} (repeated collisions)", dest
+    rc, tail = _run_7z(["7z", "x", "-y", "-p", f"-o{actual}", str(archive)])
+    if rc in (0, 1):  # 0 ok, 1 warnings (e.g. minor CRC)
+        if rc == 1:
             log.warning("7z reported warnings extracting %s", archive.name)
         return True, "", actual
-    tail = "\n".join((proc.stdout or "").splitlines()[-5:])
-    return False, f"7z exit {proc.returncode}: {tail}", actual
+    if rc == -1:
+        return False, tail, actual
+    # Fallback: local extraction, then move with repaired filenames.
+    tmp = Path(tempfile.mkdtemp(prefix="7zext-", dir="/var/tmp"))
+    try:
+        rc2, tail2 = _run_7z(["7z", "x", "-y", "-p", f"-o{tmp}", str(archive)])
+        if rc2 not in (0, 1):
+            return False, f"7z exit {rc2}: {tail2}", actual
+        bad = _move_tree(tmp, actual)
+        if bad:
+            return False, f"{bad} files could not be moved onto the share", actual
+        return True, "", actual
+    finally:
+        shutil.rmtree(tmp, ignore_errors=True)
 
 
 def move_to_broken(path: Path, out_root: Path, src_root: Path, note: str) -> Path:
@@ -122,6 +188,11 @@ def move_to_broken(path: Path, out_root: Path, src_root: Path, note: str) -> Pat
     return shutil.move(str(path), str(dest))
 
 
+def is_macosx_junk(p: Path) -> bool:
+    """AppleDouble resource-fork 'zips' (.__MACOSX/._x.zip) - no content."""
+    return "__MACOSX" in p.parts or p.name.startswith("._")
+
+
 def expand_nested(dest: Path, out_root: Path, manifest, src_root: Path,
                   done: set[str]) -> int:
     """Extract archives inside `dest` until none remain. Returns count."""
@@ -132,6 +203,10 @@ def expand_nested(dest: Path, out_root: Path, manifest, src_root: Path,
         for p in sorted(cur.rglob("*")):
             if not p.is_file() or not is_archive(p):
                 continue
+            if is_macosx_junk(p):
+                p.unlink()
+                log.info("Junk (macOS AppleDouble) removed: %s", p.name)
+                continue
             stem_dir = p.with_name(display_unit(p.name))
             if str(stem_dir.relative_to(out_root)) in done:
                 # already extracted in a previous run; drop the archive copy
@@ -240,6 +315,10 @@ def sweep(out_root: Path, manifest, done: set[str]) -> int:
             continue
         if ".broken" in p.relative_to(out_root).parts:
             continue
+        if is_macosx_junk(p):
+            p.unlink()
+            log.info("Junk (macOS AppleDouble) removed: %s", p.name)
+            continue
         stem_dir = p.with_name(display_unit(p.name))
         if str(stem_dir.relative_to(out_root)) in done:
             p.unlink()