From 86fcb3aeccbaa1c85e7d1bdcc6acd8614140f091 Mon Sep 17 00:00:00 2001 From: Ben_Kosytorz Date: Mon, 5 Oct 2026 08:09:22 +0200 Subject: [PATCH] extract stage + system-locked matching + provenance unit mode - tools/extract_all.py: recursive extraction of all archives (incl. nested) into a plain-file tree; nested archives are deleted after successful extraction, plain files are hardlinked, failures go to .broken/, extract_manifest.jsonl records archive->dest->unit. - sorter --provenance: classify each extraction directory once under the archive's name (files of one archive are one unit; inner structure kept), so loose STLs sort with their model set. - matching discipline: a fuzzy match never leaves the file's system; system-less files need CROSS_SYSTEM_MIN (0.75) for a catalog unit, and a same-system faction jump below 0.75 is rejected. Fixes 40k files landing in Middle-earth/Blood Bowl/AoS buckets. - speed for big trees: lru_cache on keyword matchers and pair scores, quick_ratio pruning; keywords: night lord, csm. --- AGENTS.md | 19 ++- dlphelper/catalogue.py | 6 + dlphelper/sorter.py | 186 +++++++++++++++++++++++++---- dlphelper/warhammer_data.py | 4 +- docs/SORTING.md | 26 +++++ sorter.py | 15 ++- tools/extract_all.py | 225 ++++++++++++++++++++++++++++++++++++ 7 files changed, 455 insertions(+), 26 deletions(-) create mode 100644 tools/extract_all.py diff --git a/AGENTS.md b/AGENTS.md index 9e53328..f5b4d03 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -56,10 +56,14 @@ dlphelper/ categories as classes, GROUP_OF_FACTION map gwshop.py GW webshop catalog dump via Algolia (see §7.3) sorter.py library sorter: inspect/classify/fuzzy-match - against the catalog, plan + hardlink apply + against the catalog, plan + hardlink apply, + optional --provenance unit mode for extracted trees parts/grab.sh legacy groundstone script (kept, not used by dlp.py) tools/new_repo.py Gitea repo helper (reads ~/.git-credentials itself) tools/debug_gofile.py inspect raw gofile API response for one content id +tools/extract_all.py recursive archive extraction into a plain-file tree + (extract_manifest.jsonl feeds the sorter's + --provenance unit mode; failures go to .broken/) tests/test_extract.py offline extraction test (plain python3, no deps) tests/test_catalogue.py offline classifier test ``` @@ -209,6 +213,19 @@ never re-fetch them again): the **GW webshop taxonomy** (see below); `docs/SORTING.md` is the working agreement for the sorting pipeline. + **Sorter matching rules (fixed 2026-10-05 after the first full run):** + the outer file name wins when it scores >= threshold; inner archive + names only vote as fallback (hash-named gofile archives need that). A + fuzzy match never jumps into another game's section: keyword-identified + systems constrain the catalog pool, and a file with neither system nor + faction keyword only adopts a catalog unit on a strong hit + (`CROSS_SYSTEM_MIN = 0.75`) - weaker guesses stay in `_unsorted`. + Token-containment boosts exist for short names vs long catalog names + ("telemon" → "Telemon Heavy Dreadnought"), guarded by a generic-token + stoplist. The catalog deliberately covers all GW systems (needed for + catalog-agnostic use) - wrong-system placement is prevented by matching + discipline, not by shrinking the catalog. + **Catalog source: the GW webshop via Algolia (`dlphelper/gwshop.py`, working, verified 2026-10-04 — how it was obtained, do not re-derive):** scraping warhammer.com directly is blocked — the shop HTML and all diff --git a/dlphelper/catalogue.py b/dlphelper/catalogue.py index 8fc0736..e4ad184 100644 --- a/dlphelper/catalogue.py +++ b/dlphelper/catalogue.py @@ -53,6 +53,10 @@ class CatalogueEntry: source: dict = field(default_factory=dict) # manifest attribution if known +from functools import lru_cache + + +@lru_cache(maxsize=None) def _clean_name(name: str) -> str: stem = Path(name).stem stem = re.sub(r"[_\.\+\[\]\(\)]+", " ", stem) @@ -65,6 +69,7 @@ def _kw_regex(keyword: str) -> str: return rf"(? tuple[str, str] | None: """Return (system, faction) with the most specific keyword hit.""" best: tuple[int, str, str] | None = None # (len, system, faction) @@ -76,6 +81,7 @@ def _match_faction(clean: str) -> tuple[str, str] | None: return (best[1], best[2]) if best else None +@lru_cache(maxsize=None) def _match_class(clean: str) -> str | None: """Return the most specific force-organization class keyword hit.""" best: tuple[int, str] | None = None diff --git a/dlphelper/sorter.py b/dlphelper/sorter.py index 09227c5..97122d8 100644 --- a/dlphelper/sorter.py +++ b/dlphelper/sorter.py @@ -19,6 +19,7 @@ import shutil import subprocess import zipfile from dataclasses import dataclass, field +from functools import lru_cache from pathlib import Path from dlphelper.catalogue import _clean_name, _match_class, _match_faction @@ -29,6 +30,10 @@ log = logging.getLogger(__name__) ARCHIVE_SUFFIXES = {".zip", ".7z", ".rar", ".tar", ".gz"} UNSORTED = "_unsorted" UNCLASSIFIED_CLASS = "Unclassified" +# A file whose system the keyword tables could not identify only gets a +# catalog unit when the fuzzy hit is strong - otherwise it would wander +# into other games' sections (Middle-earth, Blood Bowl, ...) on a guess. +CROSS_SYSTEM_MIN = 0.75 # Run-artifact files that must never be sorted into a library RUN_ARTIFACTS = { @@ -169,10 +174,70 @@ def load_catalog(path: Path) -> list[dict]: def _best_ratio(clean: str, candidates: list[str]) -> float: - return max( - (difflib.SequenceMatcher(None, clean, c).ratio() for c in candidates), - default=0.0, - ) + """Best sequence ratio, pruned with cheap upper-bound checks. + + lru_cached because the same (name, entry) pair is scored over and + over across thousands of files. + """ + return _best_ratio_cached(clean, tuple(candidates)) + + +@lru_cache(maxsize=None) +def _best_ratio_cached(clean: str, candidates: tuple[str, ...]) -> float: + best = 0.0 + toks = set(_stem_tokens(clean)) + best = _containment_boost(best, toks, candidates) + for c in candidates: + sm = difflib.SequenceMatcher(None, clean, c) + if sm.real_quick_ratio() < best or sm.quick_ratio() < best: + continue + r = sm.ratio() + if r > best: + best = r + if best == 1.0: + break + return best + + +def _stem_tokens(clean: str) -> list[str]: + """Tokens with a crude plural 's' stripped, for containment checks.""" + return [t[:-1] if len(t) > 3 and t.endswith("s") else t + for t in clean.split()] + + +def _containment_boost(best: float, toks: set[str], candidates: tuple[str, ...]) -> float: + """Token-containment bonus: a name whose tokens all appear in the + other (or vice versa) matches strongly even when string ratio is low + ("telemon" vs "Telemon Heavy Dreadnought"). Guarded: needs >= 2 + tokens, or a single rare one (>= 7 chars), and the smaller side must + not consist of generic army words ("chaos lord" is contained in half + the catalog) so "head" or "guard" never trigger it.""" + if not toks: + return best + for c in candidates: + ctoks = set(_stem_tokens(c)) + if not ctoks: + continue + small = toks if toks <= ctoks else (ctoks if ctoks <= toks else None) + if small is None or any(t in GENERIC_TOKENS for t in small): + continue + if len(small) >= 2 or len(next(iter(small))) >= 7: + if best < 0.9: + best = 0.9 + break + return best + + +# words too generic for token-containment matching: entries built from +# them alone would claim any file whose name happens to mention them +GENERIC_TOKENS = frozenset({ + "chaos", "lord", "space", "marine", "guard", "knight", "dark", + "death", "blood", "war", "warrior", "daemon", "hero", "god", + "free", "black", "white", "grey", "green", "gold", "iron", + "steel", "storm", "soul", "shadow", "bone", "ghost", "king", + "clan", "tribe", "legion", "regiment", "squad", "warhound", + "orc", "ogre", "elf", "dwarf", "giant", "company", "chapter", +}) # noqa: C405 def _best_in_pool( @@ -207,21 +272,45 @@ def match_catalog(item: SortItem, catalog: list[dict], threshold: float) -> None return if item.match_kind == "invalid": return # HTML-page masquerading as an archive; never fuzzy-match it + system_known = item.system != "Unknown" same_faction = [e for e in catalog if e.get("faction") == item.faction] - pool = same_faction or catalog # fall back to whole catalog if faction unknown - best = _best_in_pool(item, pool, use_inner=False) - if best is None or best[0] < threshold: - # Outer name not confident: let the archive's inner file names vote - # (this is the only signal for hash-named gofile archives). - best = _best_in_pool(item, pool, use_inner=True) - if best is None or best[0] < threshold: - # The keyword tables can lag behind GW's current categorization - # (e.g. units moved between factions); if the faction pool has no - # confident match, try the whole catalog before giving up. - if same_faction: - best = _best_in_pool(item, catalog, use_inner=False) + if system_known: + # a fuzzy match must never jump into another game's section + same_faction = [e for e in same_faction if e.get("system") == item.system] + if same_faction: + best = _best_in_pool(item, same_faction, use_inner=False) + if best is None or best[0] < threshold: + # Outer name not confident: let the archive's inner file names + # vote (the only signal for hash-named gofile archives). + best = _best_in_pool(item, same_faction, use_inner=True) + if best is None or best[0] < threshold: + # The keyword tables can lag behind GW's current categorization + # (e.g. units moved between factions); retry within the same + # system before giving up - never across systems. + sys_pool = [e for e in catalog if e.get("system") == item.system] + best = _best_in_pool(item, sys_pool, use_inner=False) if best is None or best[0] < threshold: - best = _best_in_pool(item, catalog, use_inner=True) + best = _best_in_pool(item, sys_pool, use_inner=True) + # a retry hit from a *different* faction is only trusted when + # it is strong; weak inner-name votes must not re-home the file + if best is not None and best[0] >= threshold \ + and best[1].get("faction") != item.faction \ + and best[0] < CROSS_SYSTEM_MIN: + best = None + elif system_known: + sys_pool = [e for e in catalog if e.get("system") == item.system] + best = _best_in_pool(item, sys_pool, use_inner=False) + if best is None or best[0] < threshold: + best = _best_in_pool(item, sys_pool, use_inner=True) + else: + # Neither system nor faction known: whole catalog, but only a + # strong hit (CROSS_SYSTEM_MIN) may adopt another game's section; + # weaker guesses stay in _unsorted. + best = _best_in_pool(item, catalog, use_inner=False) + if best is None or best[0] < threshold: + best = _best_in_pool(item, catalog, use_inner=True) + if best is not None and best[0] < CROSS_SYSTEM_MIN: + best = None if best and best[0] >= threshold: entry = best[1] item.unit = entry["name"] @@ -278,10 +367,13 @@ def match_catalog(item: SortItem, catalog: list[dict], threshold: float) -> None item.match_kind = "implicit" -def build_target(item: SortItem, library: Path) -> Path: +def build_target(item: SortItem, library: Path, inner_rel: Path | None = None) -> Path: parts: list[str] = [] if item.system == "Unknown": parts = [UNSORTED] + if inner_rel is not None and item.unit: + # extracted archives keep their unit folder even when unplaced + parts.append(_display_name(item.unit)) else: parts = [item.system] if item.faction != UNSORTED: @@ -296,6 +388,8 @@ def build_target(item: SortItem, library: Path) -> Path: else: parts.append(UNSORTED) target_dir = Path(library).joinpath(*parts) + if inner_rel is not None: + target_dir = target_dir.joinpath(*inner_rel.parts) return target_dir / item.name @@ -314,7 +408,20 @@ def sort_plan( library: Path, catalog: list[dict], threshold: float = 0.62, + provenance: dict[str, str] | None = None, + root: Path | None = None, ) -> list[SortItem]: + """Build sort items for `files`. + + `provenance` (with `root`) activates extracted-archive mode: it maps + extraction directories (relative to `root`) to the archive they came + from. All files inside such a directory are classified and matched + once under the archive's name and then placed together, keeping the + archive's inner structure - a loose "Backpack.stl" alone is noise, + under "The Twin Lance/" it belongs to its unit. + """ + prov = provenance or {} + unit_cache: dict[str, dict] = {} items = [] for f in sorted(files): if not f.is_file(): @@ -322,10 +429,45 @@ def sort_plan( if (f.name.endswith(".part") or f.name.startswith(".") or f.name in RUN_ARTIFACTS): continue - item = inspect(f) - classify_item(item) - match_catalog(item, catalog, threshold) - item.target = build_target(item, library) + unit_name = dest_key = None + inner_rel: Path | None = None + if root is not None and prov: + rel = f.relative_to(root) + p = rel.parent + while p != Path("."): + if str(p) in prov: + dest_key, unit_name = str(p), prov[str(p)] + inner_rel = rel.relative_to(p) + break + p = p.parent + if unit_name is None: + item = inspect(f) + classify_item(item) + match_catalog(item, catalog, threshold) + else: + # one classification per extraction directory, reused for + # every file inside it + fields = unit_cache.get(dest_key) + if fields is None: + listing = sorted( + str(g.relative_to(root / dest_key)) + for g in (root / dest_key).rglob("*") if g.is_file() + )[:40] + probe = SortItem( + path=f, size=0, name=unit_name, + clean=_clean_name(unit_name), inner_names=listing, + ) + classify_item(probe) + match_catalog(probe, catalog, threshold) + fields = {k: getattr(probe, k) for k in ( + "system", "group", "faction", "subfaction", "klass", + "unit", "match_score", "match_kind", + )} + unit_cache[dest_key] = fields + item = inspect(f) + for k, v in fields.items(): + setattr(item, k, v) + item.target = build_target(item, library, inner_rel=inner_rel) item.action = "link" items.append(item) return items diff --git a/dlphelper/warhammer_data.py b/dlphelper/warhammer_data.py index 5c4ad8a..b58f32e 100644 --- a/dlphelper/warhammer_data.py +++ b/dlphelper/warhammer_data.py @@ -133,13 +133,13 @@ SYSTEMS: dict[str, dict[str, list[str]]] = { # -- Chaos ----------------------------------------------------------- "Chaos Space Marines": [ "chaos space marine", "black legion", "iron warriors", "word bearers", - "night lords", "alpha legion", "chaos terminator", "chosen", "havoc", + "night lords", "night lord", "alpha legion", "chaos terminator", "chosen", "havoc", "raptor", "warp talon", "chaos cultist", "cultist", "daemon prince", "chaos lord", "chaos sorcerer", "abaddon", "kharn", "ahriman", "typhus", "fabius bile", "vashtorr", "defiler", "forgefiend", "maulerfiend", "helldrake", "helbrute", "chaos spawn", "obliterator", "chaos rhino", "chaos vindicator", "spartan assault", - "mastodon", "typhon", + "mastodon", "typhon", "csm", ], "Death Guard": [ "death guard", "nurgle", "plague marine", "deathshroud", "mortarion", diff --git a/docs/SORTING.md b/docs/SORTING.md index 63c4f4b..9b41fcd 100644 --- a/docs/SORTING.md +++ b/docs/SORTING.md @@ -95,6 +95,32 @@ graph LR unpack step will live in the sorter (`--unpack`, 7z binary) but stays off until the link-based flow is proven on a small batch. +## Extraction stage (`tools/extract_all.py`) + +That endgame stage now exists as a standalone tool (the sorter stays +plan/apply-only): + + python3 tools/extract_all.py ... --out + +- Every archive (zip/7z/rar/tar/gz/bz2/xz, also nested ones) is extracted + into a folder named after the archive, preserving relative structure; + nested archives are expanded recursively and deleted, so the tree ends + up with **no archives at all**. +- Plain files are hardlinked into the tree (fallback copy) — nothing lost, + no duplicate data. +- Sources are never modified. Failed extractions (HTML-page fake zips, + password archives, disk full) are moved to `/.broken/` and the run + continues; re-runs skip already-extracted destinations. +- `/extract_manifest.jsonl` records `{archive, dest, unit}` per + extraction. The sorter's `--provenance` flag reads it: all files inside + an extraction directory are classified **once under the archive's + name** (a loose `Backpack.stl` alone is noise; under + `The Twin Lance/` it belongs to that unit) and placed together, keeping + the archive's inner directory structure: + + python3 sorter.py --library \ + --provenance /extract_manifest.jsonl --apply + ## Catalog file (`data/library_catalog.json`) ```json diff --git a/sorter.py b/sorter.py index bdeaa88..b07d5f3 100644 --- a/sorter.py +++ b/sorter.py @@ -52,6 +52,9 @@ def main() -> None: parser.add_argument("--apply", action="store_true", help="create the targets (hardlink, fallback copy)") parser.add_argument("--audit", type=Path, default=None, metavar="FILE") + parser.add_argument("--provenance", type=Path, default=None, metavar="FILE", + help="extract_manifest.jsonl from tools/extract_all.py: " + "sort whole extraction directories as units") parser.add_argument("--json", action="store_true", help="print the plan as JSON instead of text") args = parser.parse_args() @@ -65,7 +68,17 @@ def main() -> None: catalog = load_catalog(args.catalog) audit = args.audit or (args.library / "sorter_applied.jsonl") - items = sort_plan(files, args.library, catalog, threshold=args.threshold) + provenance = None + if args.provenance: + provenance = {} + with args.provenance.open(encoding="utf-8") as fh: + for line in fh: + rec = json.loads(line) + provenance[rec["dest"]] = rec["unit"] + log.info("Provenance: %d extraction directories", len(provenance)) + + items = sort_plan(files, args.library, catalog, threshold=args.threshold, + provenance=provenance, root=source) if args.json: print(json.dumps([ diff --git a/tools/extract_all.py b/tools/extract_all.py new file mode 100644 index 0000000..5f83303 --- /dev/null +++ b/tools/extract_all.py @@ -0,0 +1,225 @@ +#!/usr/bin/env python3 +"""Recursively extract every archive under the source roots into one tree. + + python3 tools/extract_all.py ... --out + +Behavior: +- Every archive (zip/7z/rar/tar/gz/bz2/xz) is extracted into a directory + named after the archive (suffix stripped), preserving the relative path + under the source root: /zips/The Twin Lance.7z + -> /1/zips/The Twin Lance/... +- Archives found *inside* an extraction are extracted too (BFS, until no + archive remains), then deleted - the extracted tree holds no archives. +- Plain files are hardlinked into the tree (fallback: copy), so nothing + is lost and no data is duplicated. +- The source roots are never modified (originals stay). +- A provenance manifest `/extract_manifest.jsonl` records one line + per extraction: {"archive": ..., "dest": ..., "unit": ...} for the + sorter's --provenance unit mode. +- Failures (bad download/HTML page, wrong password, disk full) are moved + to `/.broken/` and logged; the run never stops. + +Re-runs are safe: destinations that already have content are skipped. +""" + +from __future__ import annotations + +import argparse +import logging +import os +import shutil +import subprocess +import sys +from collections import deque +from pathlib import Path + +log = logging.getLogger("extract_all") + +ARCHIVE_SUFFIXES = {".zip", ".7z", ".rar", ".tar", ".gz", ".bz2", ".xz"} +EXTRACT_TIMEOUT = 4 * 3600 # seconds per archive +MIN_FREE_BYTES = 20 * 1024**3 # stop extracting below this free space +RUN_ARTIFACTS = { + "export.json", "export.html", "state.json", "manifest.jsonl", + "failed_urls.txt", "catalogue.md", "catalogue.jsonl", + "sorter_applied.jsonl", "catalogue_moves.jsonl", "urls.txt", + "urls_0807.txt", "urls_gofile.txt", "failed_0807.txt", + "extract_manifest.jsonl", +} + + +def archive_suffixes(name: str) -> list[str]: + """All trailing archive suffixes, e.g. 'x.tar.gz' -> ['.gz', '.tar'].""" + out = [] + p = Path(name) + while p.suffix.lower() in ARCHIVE_SUFFIXES: + out.append(p.suffix.lower()) + p = p.with_suffix("") + return out + + +def is_archive(path: Path) -> bool: + return bool(archive_suffixes(path.name)) + + +def display_unit(name: str) -> str: + """Archive name with all archive suffixes stripped: the unit folder.""" + p = Path(name) + while p.suffix.lower() in ARCHIVE_SUFFIXES: + p = p.with_suffix("") + return p.name + + +def free_bytes(path: Path) -> int: + return shutil.disk_usage(path).free + + +def extract(archive: Path, dest: Path) -> tuple[bool, str]: + """7z x into dest. Empty -p avoids interactive prompts (no password).""" + dest.mkdir(parents=True, exist_ok=True) + cmd = ["7z", "x", "-y", "-p", f"-o{dest}", str(archive)] + try: + proc = subprocess.run( + cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + text=True, timeout=EXTRACT_TIMEOUT, + ) + except subprocess.TimeoutExpired: + return False, f"extraction timed out after {EXTRACT_TIMEOUT}s" + except OSError as exc: + return False, f"7z failed to run: {exc}" + if proc.returncode in (0, 1): # 0 ok, 1 warnings (e.g. minor CRC) + if proc.returncode == 1: + log.warning("7z reported warnings extracting %s", archive.name) + return True, "" + tail = "\n".join((proc.stdout or "").splitlines()[-5:]) + return False, f"7z exit {proc.returncode}: {tail}" + + +def move_to_broken(path: Path, out_root: Path, src_root: Path, note: str) -> Path: + rel = path.relative_to(src_root) + dest = out_root / ".broken" / rel + dest.parent.mkdir(parents=True, exist_ok=True) + if dest.exists(): + dest = dest.with_name(dest.name + ".1") + return shutil.move(str(path), str(dest)) + + +def expand_nested(dest: Path, out_root: Path, manifest, src_root: Path) -> int: + """Extract archives inside `dest` until none remain. Returns count.""" + done = 0 + queue: deque[Path] = deque([dest]) + while queue: + cur = queue.popleft() + for p in sorted(cur.rglob("*")): + if not p.is_file() or not is_archive(p): + continue + stem_dir = p.with_name(display_unit(p.name)) + if stem_dir.exists() and any(stem_dir.iterdir()): + # already extracted in a previous run; drop the archive copy + p.unlink() + log.info("Nested %s already extracted - archive removed", p.name) + continue + if free_bytes(out_root) < MIN_FREE_BYTES: + log.error("Out of disk space - skipping %s (kept in place)", p) + continue + ok, err = extract(p, stem_dir) + if not ok: + log.error("FAILED nested extraction %s: %s", p, err) + move_to_broken(p, out_root, src_root, err) + continue + manifest.write(f"{json_line(p.relative_to(out_root), stem_dir)}\n") + p.unlink() # archive copy consumed + done += 1 + queue.append(stem_dir) + return done + + +def json_line(archive_rel: Path, dest: Path) -> str: + import json + return json.dumps({ + "archive": str(archive_rel), + "dest": str(dest.relative_to(_OUT_ROOT)), + "unit": display_unit(archive_rel.name), + }, ensure_ascii=False) + + +_OUT_ROOT: Path = Path(".") # set in main; used by json_line + + +def process_root(src_root: Path, out_root: Path, manifest) -> tuple[int, int, int]: + """Returns (extracted, mirrored, failed).""" + idx = src_root.name + extracted = mirrored = failed = 0 + for f in sorted(src_root.rglob("*")): + if not f.is_file(): + continue + if (f.name in RUN_ARTIFACTS or f.name.startswith(".") + or f.name.endswith(".part")): + continue + rel = f.relative_to(src_root) + if is_archive(f): + dest_rel = Path(idx) / Path(str(rel.with_name(display_unit(f.name)))) + dest = out_root / dest_rel + if dest.exists() and any(dest.iterdir()): + log.info("Skip (already extracted): %s", rel) + continue + if free_bytes(out_root) < MIN_FREE_BYTES: + log.error("Out of disk space - skipping archive %s", f) + failed += 1 + continue + ok, err = extract(f, dest) + if not ok: + log.error("FAILED %s: %s", rel, err) + move_to_broken(f, out_root, src_root, err) + failed += 1 + continue + manifest.write(f"{json_line(Path(idx) / rel, dest)}\n") + extracted += 1 + log.info("Extracted %s -> %s", rel, dest_rel) + expand_nested(dest, out_root, manifest, src_root) + else: + link_dest = out_root / idx / rel + if link_dest.exists(): + continue + link_dest.parent.mkdir(parents=True, exist_ok=True) + try: + os.link(f, link_dest) + except OSError: + shutil.copy2(f, link_dest) + mirrored += 1 + return extracted, mirrored, failed + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("sources", nargs="+", type=Path) + parser.add_argument("--out", type=Path, required=True, metavar="DIR") + args = parser.parse_args() + + logging.basicConfig(level=logging.INFO, + format="%(asctime)s %(levelname)-7s %(message)s", + datefmt="%H:%M:%S") + + global _OUT_ROOT + _OUT_ROOT = args.out + args.out.mkdir(parents=True, exist_ok=True) + + totals = [0, 0, 0] + with (args.out / "extract_manifest.jsonl").open("a", encoding="utf-8") as manifest: + for src in args.sources: + log.info("== processing %s", src) + counts = process_root(src, args.out, manifest) + log.info("== %s: %d extracted, %d mirrored, %d failed", + src.name, *counts) + for i, c in enumerate(counts): + totals[i] += c + log.info("TOTAL: %d archives extracted, %d files mirrored, %d failed", + *totals) + if totals[2]: + log.warning("Failures were moved under %s - inspect and re-run", args.out / ".broken") + + +if __name__ == "__main__": + try: + main() + except KeyboardInterrupt: + sys.exit(130)