extract stage + system-locked matching + provenance unit mode
- tools/extract_all.py: recursive extraction of all archives (incl. nested) into a plain-file tree; nested archives are deleted after successful extraction, plain files are hardlinked, failures go to .broken/, extract_manifest.jsonl records archive->dest->unit. - sorter --provenance: classify each extraction directory once under the archive's name (files of one archive are one unit; inner structure kept), so loose STLs sort with their model set. - matching discipline: a fuzzy match never leaves the file's system; system-less files need CROSS_SYSTEM_MIN (0.75) for a catalog unit, and a same-system faction jump below 0.75 is rejected. Fixes 40k files landing in Middle-earth/Blood Bowl/AoS buckets. - speed for big trees: lru_cache on keyword matchers and pair scores, quick_ratio pruning; keywords: night lord, csm.
This commit is contained in:
@@ -0,0 +1,225 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Recursively extract every archive under the source roots into one tree.
|
||||
|
||||
python3 tools/extract_all.py <src_root>... --out <extracted_root>
|
||||
|
||||
Behavior:
|
||||
- Every archive (zip/7z/rar/tar/gz/bz2/xz) is extracted into a directory
|
||||
named after the archive (suffix stripped), preserving the relative path
|
||||
under the source root: <src>/zips/The Twin Lance.7z
|
||||
-> <out>/1/zips/The Twin Lance/...
|
||||
- Archives found *inside* an extraction are extracted too (BFS, until no
|
||||
archive remains), then deleted - the extracted tree holds no archives.
|
||||
- Plain files are hardlinked into the tree (fallback: copy), so nothing
|
||||
is lost and no data is duplicated.
|
||||
- The source roots are never modified (originals stay).
|
||||
- A provenance manifest `<out>/extract_manifest.jsonl` records one line
|
||||
per extraction: {"archive": ..., "dest": ..., "unit": ...} for the
|
||||
sorter's --provenance unit mode.
|
||||
- Failures (bad download/HTML page, wrong password, disk full) are moved
|
||||
to `<out>/.broken/<relpath>` and logged; the run never stops.
|
||||
|
||||
Re-runs are safe: destinations that already have content are skipped.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import logging
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
from collections import deque
|
||||
from pathlib import Path
|
||||
|
||||
log = logging.getLogger("extract_all")
|
||||
|
||||
ARCHIVE_SUFFIXES = {".zip", ".7z", ".rar", ".tar", ".gz", ".bz2", ".xz"}
|
||||
EXTRACT_TIMEOUT = 4 * 3600 # seconds per archive
|
||||
MIN_FREE_BYTES = 20 * 1024**3 # stop extracting below this free space
|
||||
RUN_ARTIFACTS = {
|
||||
"export.json", "export.html", "state.json", "manifest.jsonl",
|
||||
"failed_urls.txt", "catalogue.md", "catalogue.jsonl",
|
||||
"sorter_applied.jsonl", "catalogue_moves.jsonl", "urls.txt",
|
||||
"urls_0807.txt", "urls_gofile.txt", "failed_0807.txt",
|
||||
"extract_manifest.jsonl",
|
||||
}
|
||||
|
||||
|
||||
def archive_suffixes(name: str) -> list[str]:
|
||||
"""All trailing archive suffixes, e.g. 'x.tar.gz' -> ['.gz', '.tar']."""
|
||||
out = []
|
||||
p = Path(name)
|
||||
while p.suffix.lower() in ARCHIVE_SUFFIXES:
|
||||
out.append(p.suffix.lower())
|
||||
p = p.with_suffix("")
|
||||
return out
|
||||
|
||||
|
||||
def is_archive(path: Path) -> bool:
|
||||
return bool(archive_suffixes(path.name))
|
||||
|
||||
|
||||
def display_unit(name: str) -> str:
|
||||
"""Archive name with all archive suffixes stripped: the unit folder."""
|
||||
p = Path(name)
|
||||
while p.suffix.lower() in ARCHIVE_SUFFIXES:
|
||||
p = p.with_suffix("")
|
||||
return p.name
|
||||
|
||||
|
||||
def free_bytes(path: Path) -> int:
|
||||
return shutil.disk_usage(path).free
|
||||
|
||||
|
||||
def extract(archive: Path, dest: Path) -> tuple[bool, str]:
|
||||
"""7z x into dest. Empty -p avoids interactive prompts (no password)."""
|
||||
dest.mkdir(parents=True, exist_ok=True)
|
||||
cmd = ["7z", "x", "-y", "-p", f"-o{dest}", str(archive)]
|
||||
try:
|
||||
proc = subprocess.run(
|
||||
cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
|
||||
text=True, timeout=EXTRACT_TIMEOUT,
|
||||
)
|
||||
except subprocess.TimeoutExpired:
|
||||
return False, f"extraction timed out after {EXTRACT_TIMEOUT}s"
|
||||
except OSError as exc:
|
||||
return False, f"7z failed to run: {exc}"
|
||||
if proc.returncode in (0, 1): # 0 ok, 1 warnings (e.g. minor CRC)
|
||||
if proc.returncode == 1:
|
||||
log.warning("7z reported warnings extracting %s", archive.name)
|
||||
return True, ""
|
||||
tail = "\n".join((proc.stdout or "").splitlines()[-5:])
|
||||
return False, f"7z exit {proc.returncode}: {tail}"
|
||||
|
||||
|
||||
def move_to_broken(path: Path, out_root: Path, src_root: Path, note: str) -> Path:
|
||||
rel = path.relative_to(src_root)
|
||||
dest = out_root / ".broken" / rel
|
||||
dest.parent.mkdir(parents=True, exist_ok=True)
|
||||
if dest.exists():
|
||||
dest = dest.with_name(dest.name + ".1")
|
||||
return shutil.move(str(path), str(dest))
|
||||
|
||||
|
||||
def expand_nested(dest: Path, out_root: Path, manifest, src_root: Path) -> int:
|
||||
"""Extract archives inside `dest` until none remain. Returns count."""
|
||||
done = 0
|
||||
queue: deque[Path] = deque([dest])
|
||||
while queue:
|
||||
cur = queue.popleft()
|
||||
for p in sorted(cur.rglob("*")):
|
||||
if not p.is_file() or not is_archive(p):
|
||||
continue
|
||||
stem_dir = p.with_name(display_unit(p.name))
|
||||
if stem_dir.exists() and any(stem_dir.iterdir()):
|
||||
# already extracted in a previous run; drop the archive copy
|
||||
p.unlink()
|
||||
log.info("Nested %s already extracted - archive removed", p.name)
|
||||
continue
|
||||
if free_bytes(out_root) < MIN_FREE_BYTES:
|
||||
log.error("Out of disk space - skipping %s (kept in place)", p)
|
||||
continue
|
||||
ok, err = extract(p, stem_dir)
|
||||
if not ok:
|
||||
log.error("FAILED nested extraction %s: %s", p, err)
|
||||
move_to_broken(p, out_root, src_root, err)
|
||||
continue
|
||||
manifest.write(f"{json_line(p.relative_to(out_root), stem_dir)}\n")
|
||||
p.unlink() # archive copy consumed
|
||||
done += 1
|
||||
queue.append(stem_dir)
|
||||
return done
|
||||
|
||||
|
||||
def json_line(archive_rel: Path, dest: Path) -> str:
|
||||
import json
|
||||
return json.dumps({
|
||||
"archive": str(archive_rel),
|
||||
"dest": str(dest.relative_to(_OUT_ROOT)),
|
||||
"unit": display_unit(archive_rel.name),
|
||||
}, ensure_ascii=False)
|
||||
|
||||
|
||||
_OUT_ROOT: Path = Path(".") # set in main; used by json_line
|
||||
|
||||
|
||||
def process_root(src_root: Path, out_root: Path, manifest) -> tuple[int, int, int]:
|
||||
"""Returns (extracted, mirrored, failed)."""
|
||||
idx = src_root.name
|
||||
extracted = mirrored = failed = 0
|
||||
for f in sorted(src_root.rglob("*")):
|
||||
if not f.is_file():
|
||||
continue
|
||||
if (f.name in RUN_ARTIFACTS or f.name.startswith(".")
|
||||
or f.name.endswith(".part")):
|
||||
continue
|
||||
rel = f.relative_to(src_root)
|
||||
if is_archive(f):
|
||||
dest_rel = Path(idx) / Path(str(rel.with_name(display_unit(f.name))))
|
||||
dest = out_root / dest_rel
|
||||
if dest.exists() and any(dest.iterdir()):
|
||||
log.info("Skip (already extracted): %s", rel)
|
||||
continue
|
||||
if free_bytes(out_root) < MIN_FREE_BYTES:
|
||||
log.error("Out of disk space - skipping archive %s", f)
|
||||
failed += 1
|
||||
continue
|
||||
ok, err = extract(f, dest)
|
||||
if not ok:
|
||||
log.error("FAILED %s: %s", rel, err)
|
||||
move_to_broken(f, out_root, src_root, err)
|
||||
failed += 1
|
||||
continue
|
||||
manifest.write(f"{json_line(Path(idx) / rel, dest)}\n")
|
||||
extracted += 1
|
||||
log.info("Extracted %s -> %s", rel, dest_rel)
|
||||
expand_nested(dest, out_root, manifest, src_root)
|
||||
else:
|
||||
link_dest = out_root / idx / rel
|
||||
if link_dest.exists():
|
||||
continue
|
||||
link_dest.parent.mkdir(parents=True, exist_ok=True)
|
||||
try:
|
||||
os.link(f, link_dest)
|
||||
except OSError:
|
||||
shutil.copy2(f, link_dest)
|
||||
mirrored += 1
|
||||
return extracted, mirrored, failed
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
|
||||
parser.add_argument("sources", nargs="+", type=Path)
|
||||
parser.add_argument("--out", type=Path, required=True, metavar="DIR")
|
||||
args = parser.parse_args()
|
||||
|
||||
logging.basicConfig(level=logging.INFO,
|
||||
format="%(asctime)s %(levelname)-7s %(message)s",
|
||||
datefmt="%H:%M:%S")
|
||||
|
||||
global _OUT_ROOT
|
||||
_OUT_ROOT = args.out
|
||||
args.out.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
totals = [0, 0, 0]
|
||||
with (args.out / "extract_manifest.jsonl").open("a", encoding="utf-8") as manifest:
|
||||
for src in args.sources:
|
||||
log.info("== processing %s", src)
|
||||
counts = process_root(src, args.out, manifest)
|
||||
log.info("== %s: %d extracted, %d mirrored, %d failed",
|
||||
src.name, *counts)
|
||||
for i, c in enumerate(counts):
|
||||
totals[i] += c
|
||||
log.info("TOTAL: %d archives extracted, %d files mirrored, %d failed",
|
||||
*totals)
|
||||
if totals[2]:
|
||||
log.warning("Failures were moved under %s - inspect and re-run", args.out / ".broken")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
try:
|
||||
main()
|
||||
except KeyboardInterrupt:
|
||||
sys.exit(130)
|
||||
Reference in New Issue
Block a user