extract stage + system-locked matching + provenance unit mode
- tools/extract_all.py: recursive extraction of all archives (incl. nested) into a plain-file tree; nested archives are deleted after successful extraction, plain files are hardlinked, failures go to .broken/, extract_manifest.jsonl records archive->dest->unit. - sorter --provenance: classify each extraction directory once under the archive's name (files of one archive are one unit; inner structure kept), so loose STLs sort with their model set. - matching discipline: a fuzzy match never leaves the file's system; system-less files need CROSS_SYSTEM_MIN (0.75) for a catalog unit, and a same-system faction jump below 0.75 is rejected. Fixes 40k files landing in Middle-earth/Blood Bowl/AoS buckets. - speed for big trees: lru_cache on keyword matchers and pair scores, quick_ratio pruning; keywords: night lord, csm.
This commit is contained in:
@@ -56,10 +56,14 @@ dlphelper/
|
||||
categories as classes, GROUP_OF_FACTION map
|
||||
gwshop.py GW webshop catalog dump via Algolia (see §7.3)
|
||||
sorter.py library sorter: inspect/classify/fuzzy-match
|
||||
against the catalog, plan + hardlink apply
|
||||
against the catalog, plan + hardlink apply,
|
||||
optional --provenance unit mode for extracted trees
|
||||
parts/grab.sh legacy groundstone script (kept, not used by dlp.py)
|
||||
tools/new_repo.py Gitea repo helper (reads ~/.git-credentials itself)
|
||||
tools/debug_gofile.py inspect raw gofile API response for one content id
|
||||
tools/extract_all.py recursive archive extraction into a plain-file tree
|
||||
(extract_manifest.jsonl feeds the sorter's
|
||||
--provenance unit mode; failures go to .broken/)
|
||||
tests/test_extract.py offline extraction test (plain python3, no deps)
|
||||
tests/test_catalogue.py offline classifier test
|
||||
```
|
||||
@@ -209,6 +213,19 @@ never re-fetch them again):
|
||||
the **GW webshop taxonomy** (see below); `docs/SORTING.md` is the
|
||||
working agreement for the sorting pipeline.
|
||||
|
||||
**Sorter matching rules (fixed 2026-10-05 after the first full run):**
|
||||
the outer file name wins when it scores >= threshold; inner archive
|
||||
names only vote as fallback (hash-named gofile archives need that). A
|
||||
fuzzy match never jumps into another game's section: keyword-identified
|
||||
systems constrain the catalog pool, and a file with neither system nor
|
||||
faction keyword only adopts a catalog unit on a strong hit
|
||||
(`CROSS_SYSTEM_MIN = 0.75`) - weaker guesses stay in `_unsorted`.
|
||||
Token-containment boosts exist for short names vs long catalog names
|
||||
("telemon" → "Telemon Heavy Dreadnought"), guarded by a generic-token
|
||||
stoplist. The catalog deliberately covers all GW systems (needed for
|
||||
catalog-agnostic use) - wrong-system placement is prevented by matching
|
||||
discipline, not by shrinking the catalog.
|
||||
|
||||
**Catalog source: the GW webshop via Algolia (`dlphelper/gwshop.py`,
|
||||
working, verified 2026-10-04 — how it was obtained, do not re-derive):**
|
||||
scraping warhammer.com directly is blocked — the shop HTML and all
|
||||
|
||||
@@ -53,6 +53,10 @@ class CatalogueEntry:
|
||||
source: dict = field(default_factory=dict) # manifest attribution if known
|
||||
|
||||
|
||||
from functools import lru_cache
|
||||
|
||||
|
||||
@lru_cache(maxsize=None)
|
||||
def _clean_name(name: str) -> str:
|
||||
stem = Path(name).stem
|
||||
stem = re.sub(r"[_\.\+\[\]\(\)]+", " ", stem)
|
||||
@@ -65,6 +69,7 @@ def _kw_regex(keyword: str) -> str:
|
||||
return rf"(?<![a-z0-9]){re.escape(keyword)}s?(?![a-z0-9])"
|
||||
|
||||
|
||||
@lru_cache(maxsize=None)
|
||||
def _match_faction(clean: str) -> tuple[str, str] | None:
|
||||
"""Return (system, faction) with the most specific keyword hit."""
|
||||
best: tuple[int, str, str] | None = None # (len, system, faction)
|
||||
@@ -76,6 +81,7 @@ def _match_faction(clean: str) -> tuple[str, str] | None:
|
||||
return (best[1], best[2]) if best else None
|
||||
|
||||
|
||||
@lru_cache(maxsize=None)
|
||||
def _match_class(clean: str) -> str | None:
|
||||
"""Return the most specific force-organization class keyword hit."""
|
||||
best: tuple[int, str] | None = None
|
||||
|
||||
+156
-14
@@ -19,6 +19,7 @@ import shutil
|
||||
import subprocess
|
||||
import zipfile
|
||||
from dataclasses import dataclass, field
|
||||
from functools import lru_cache
|
||||
from pathlib import Path
|
||||
|
||||
from dlphelper.catalogue import _clean_name, _match_class, _match_faction
|
||||
@@ -29,6 +30,10 @@ log = logging.getLogger(__name__)
|
||||
ARCHIVE_SUFFIXES = {".zip", ".7z", ".rar", ".tar", ".gz"}
|
||||
UNSORTED = "_unsorted"
|
||||
UNCLASSIFIED_CLASS = "Unclassified"
|
||||
# A file whose system the keyword tables could not identify only gets a
|
||||
# catalog unit when the fuzzy hit is strong - otherwise it would wander
|
||||
# into other games' sections (Middle-earth, Blood Bowl, ...) on a guess.
|
||||
CROSS_SYSTEM_MIN = 0.75
|
||||
|
||||
# Run-artifact files that must never be sorted into a library
|
||||
RUN_ARTIFACTS = {
|
||||
@@ -169,10 +174,70 @@ def load_catalog(path: Path) -> list[dict]:
|
||||
|
||||
|
||||
def _best_ratio(clean: str, candidates: list[str]) -> float:
|
||||
return max(
|
||||
(difflib.SequenceMatcher(None, clean, c).ratio() for c in candidates),
|
||||
default=0.0,
|
||||
)
|
||||
"""Best sequence ratio, pruned with cheap upper-bound checks.
|
||||
|
||||
lru_cached because the same (name, entry) pair is scored over and
|
||||
over across thousands of files.
|
||||
"""
|
||||
return _best_ratio_cached(clean, tuple(candidates))
|
||||
|
||||
|
||||
@lru_cache(maxsize=None)
|
||||
def _best_ratio_cached(clean: str, candidates: tuple[str, ...]) -> float:
|
||||
best = 0.0
|
||||
toks = set(_stem_tokens(clean))
|
||||
best = _containment_boost(best, toks, candidates)
|
||||
for c in candidates:
|
||||
sm = difflib.SequenceMatcher(None, clean, c)
|
||||
if sm.real_quick_ratio() < best or sm.quick_ratio() < best:
|
||||
continue
|
||||
r = sm.ratio()
|
||||
if r > best:
|
||||
best = r
|
||||
if best == 1.0:
|
||||
break
|
||||
return best
|
||||
|
||||
|
||||
def _stem_tokens(clean: str) -> list[str]:
|
||||
"""Tokens with a crude plural 's' stripped, for containment checks."""
|
||||
return [t[:-1] if len(t) > 3 and t.endswith("s") else t
|
||||
for t in clean.split()]
|
||||
|
||||
|
||||
def _containment_boost(best: float, toks: set[str], candidates: tuple[str, ...]) -> float:
|
||||
"""Token-containment bonus: a name whose tokens all appear in the
|
||||
other (or vice versa) matches strongly even when string ratio is low
|
||||
("telemon" vs "Telemon Heavy Dreadnought"). Guarded: needs >= 2
|
||||
tokens, or a single rare one (>= 7 chars), and the smaller side must
|
||||
not consist of generic army words ("chaos lord" is contained in half
|
||||
the catalog) so "head" or "guard" never trigger it."""
|
||||
if not toks:
|
||||
return best
|
||||
for c in candidates:
|
||||
ctoks = set(_stem_tokens(c))
|
||||
if not ctoks:
|
||||
continue
|
||||
small = toks if toks <= ctoks else (ctoks if ctoks <= toks else None)
|
||||
if small is None or any(t in GENERIC_TOKENS for t in small):
|
||||
continue
|
||||
if len(small) >= 2 or len(next(iter(small))) >= 7:
|
||||
if best < 0.9:
|
||||
best = 0.9
|
||||
break
|
||||
return best
|
||||
|
||||
|
||||
# words too generic for token-containment matching: entries built from
|
||||
# them alone would claim any file whose name happens to mention them
|
||||
GENERIC_TOKENS = frozenset({
|
||||
"chaos", "lord", "space", "marine", "guard", "knight", "dark",
|
||||
"death", "blood", "war", "warrior", "daemon", "hero", "god",
|
||||
"free", "black", "white", "grey", "green", "gold", "iron",
|
||||
"steel", "storm", "soul", "shadow", "bone", "ghost", "king",
|
||||
"clan", "tribe", "legion", "regiment", "squad", "warhound",
|
||||
"orc", "ogre", "elf", "dwarf", "giant", "company", "chapter",
|
||||
}) # noqa: C405
|
||||
|
||||
|
||||
def _best_in_pool(
|
||||
@@ -207,21 +272,45 @@ def match_catalog(item: SortItem, catalog: list[dict], threshold: float) -> None
|
||||
return
|
||||
if item.match_kind == "invalid":
|
||||
return # HTML-page masquerading as an archive; never fuzzy-match it
|
||||
system_known = item.system != "Unknown"
|
||||
same_faction = [e for e in catalog if e.get("faction") == item.faction]
|
||||
pool = same_faction or catalog # fall back to whole catalog if faction unknown
|
||||
best = _best_in_pool(item, pool, use_inner=False)
|
||||
if system_known:
|
||||
# a fuzzy match must never jump into another game's section
|
||||
same_faction = [e for e in same_faction if e.get("system") == item.system]
|
||||
if same_faction:
|
||||
best = _best_in_pool(item, same_faction, use_inner=False)
|
||||
if best is None or best[0] < threshold:
|
||||
# Outer name not confident: let the archive's inner file names vote
|
||||
# (this is the only signal for hash-named gofile archives).
|
||||
best = _best_in_pool(item, pool, use_inner=True)
|
||||
# Outer name not confident: let the archive's inner file names
|
||||
# vote (the only signal for hash-named gofile archives).
|
||||
best = _best_in_pool(item, same_faction, use_inner=True)
|
||||
if best is None or best[0] < threshold:
|
||||
# The keyword tables can lag behind GW's current categorization
|
||||
# (e.g. units moved between factions); if the faction pool has no
|
||||
# confident match, try the whole catalog before giving up.
|
||||
if same_faction:
|
||||
# (e.g. units moved between factions); retry within the same
|
||||
# system before giving up - never across systems.
|
||||
sys_pool = [e for e in catalog if e.get("system") == item.system]
|
||||
best = _best_in_pool(item, sys_pool, use_inner=False)
|
||||
if best is None or best[0] < threshold:
|
||||
best = _best_in_pool(item, sys_pool, use_inner=True)
|
||||
# a retry hit from a *different* faction is only trusted when
|
||||
# it is strong; weak inner-name votes must not re-home the file
|
||||
if best is not None and best[0] >= threshold \
|
||||
and best[1].get("faction") != item.faction \
|
||||
and best[0] < CROSS_SYSTEM_MIN:
|
||||
best = None
|
||||
elif system_known:
|
||||
sys_pool = [e for e in catalog if e.get("system") == item.system]
|
||||
best = _best_in_pool(item, sys_pool, use_inner=False)
|
||||
if best is None or best[0] < threshold:
|
||||
best = _best_in_pool(item, sys_pool, use_inner=True)
|
||||
else:
|
||||
# Neither system nor faction known: whole catalog, but only a
|
||||
# strong hit (CROSS_SYSTEM_MIN) may adopt another game's section;
|
||||
# weaker guesses stay in _unsorted.
|
||||
best = _best_in_pool(item, catalog, use_inner=False)
|
||||
if best is None or best[0] < threshold:
|
||||
best = _best_in_pool(item, catalog, use_inner=True)
|
||||
if best is not None and best[0] < CROSS_SYSTEM_MIN:
|
||||
best = None
|
||||
if best and best[0] >= threshold:
|
||||
entry = best[1]
|
||||
item.unit = entry["name"]
|
||||
@@ -278,10 +367,13 @@ def match_catalog(item: SortItem, catalog: list[dict], threshold: float) -> None
|
||||
item.match_kind = "implicit"
|
||||
|
||||
|
||||
def build_target(item: SortItem, library: Path) -> Path:
|
||||
def build_target(item: SortItem, library: Path, inner_rel: Path | None = None) -> Path:
|
||||
parts: list[str] = []
|
||||
if item.system == "Unknown":
|
||||
parts = [UNSORTED]
|
||||
if inner_rel is not None and item.unit:
|
||||
# extracted archives keep their unit folder even when unplaced
|
||||
parts.append(_display_name(item.unit))
|
||||
else:
|
||||
parts = [item.system]
|
||||
if item.faction != UNSORTED:
|
||||
@@ -296,6 +388,8 @@ def build_target(item: SortItem, library: Path) -> Path:
|
||||
else:
|
||||
parts.append(UNSORTED)
|
||||
target_dir = Path(library).joinpath(*parts)
|
||||
if inner_rel is not None:
|
||||
target_dir = target_dir.joinpath(*inner_rel.parts)
|
||||
return target_dir / item.name
|
||||
|
||||
|
||||
@@ -314,7 +408,20 @@ def sort_plan(
|
||||
library: Path,
|
||||
catalog: list[dict],
|
||||
threshold: float = 0.62,
|
||||
provenance: dict[str, str] | None = None,
|
||||
root: Path | None = None,
|
||||
) -> list[SortItem]:
|
||||
"""Build sort items for `files`.
|
||||
|
||||
`provenance` (with `root`) activates extracted-archive mode: it maps
|
||||
extraction directories (relative to `root`) to the archive they came
|
||||
from. All files inside such a directory are classified and matched
|
||||
once under the archive's name and then placed together, keeping the
|
||||
archive's inner structure - a loose "Backpack.stl" alone is noise,
|
||||
under "The Twin Lance/" it belongs to its unit.
|
||||
"""
|
||||
prov = provenance or {}
|
||||
unit_cache: dict[str, dict] = {}
|
||||
items = []
|
||||
for f in sorted(files):
|
||||
if not f.is_file():
|
||||
@@ -322,10 +429,45 @@ def sort_plan(
|
||||
if (f.name.endswith(".part") or f.name.startswith(".")
|
||||
or f.name in RUN_ARTIFACTS):
|
||||
continue
|
||||
unit_name = dest_key = None
|
||||
inner_rel: Path | None = None
|
||||
if root is not None and prov:
|
||||
rel = f.relative_to(root)
|
||||
p = rel.parent
|
||||
while p != Path("."):
|
||||
if str(p) in prov:
|
||||
dest_key, unit_name = str(p), prov[str(p)]
|
||||
inner_rel = rel.relative_to(p)
|
||||
break
|
||||
p = p.parent
|
||||
if unit_name is None:
|
||||
item = inspect(f)
|
||||
classify_item(item)
|
||||
match_catalog(item, catalog, threshold)
|
||||
item.target = build_target(item, library)
|
||||
else:
|
||||
# one classification per extraction directory, reused for
|
||||
# every file inside it
|
||||
fields = unit_cache.get(dest_key)
|
||||
if fields is None:
|
||||
listing = sorted(
|
||||
str(g.relative_to(root / dest_key))
|
||||
for g in (root / dest_key).rglob("*") if g.is_file()
|
||||
)[:40]
|
||||
probe = SortItem(
|
||||
path=f, size=0, name=unit_name,
|
||||
clean=_clean_name(unit_name), inner_names=listing,
|
||||
)
|
||||
classify_item(probe)
|
||||
match_catalog(probe, catalog, threshold)
|
||||
fields = {k: getattr(probe, k) for k in (
|
||||
"system", "group", "faction", "subfaction", "klass",
|
||||
"unit", "match_score", "match_kind",
|
||||
)}
|
||||
unit_cache[dest_key] = fields
|
||||
item = inspect(f)
|
||||
for k, v in fields.items():
|
||||
setattr(item, k, v)
|
||||
item.target = build_target(item, library, inner_rel=inner_rel)
|
||||
item.action = "link"
|
||||
items.append(item)
|
||||
return items
|
||||
|
||||
@@ -133,13 +133,13 @@ SYSTEMS: dict[str, dict[str, list[str]]] = {
|
||||
# -- Chaos -----------------------------------------------------------
|
||||
"Chaos Space Marines": [
|
||||
"chaos space marine", "black legion", "iron warriors", "word bearers",
|
||||
"night lords", "alpha legion", "chaos terminator", "chosen", "havoc",
|
||||
"night lords", "night lord", "alpha legion", "chaos terminator", "chosen", "havoc",
|
||||
"raptor", "warp talon", "chaos cultist", "cultist", "daemon prince",
|
||||
"chaos lord", "chaos sorcerer", "abaddon", "kharn", "ahriman",
|
||||
"typhus", "fabius bile", "vashtorr", "defiler", "forgefiend",
|
||||
"maulerfiend", "helldrake", "helbrute", "chaos spawn",
|
||||
"obliterator", "chaos rhino", "chaos vindicator", "spartan assault",
|
||||
"mastodon", "typhon",
|
||||
"mastodon", "typhon", "csm",
|
||||
],
|
||||
"Death Guard": [
|
||||
"death guard", "nurgle", "plague marine", "deathshroud", "mortarion",
|
||||
|
||||
@@ -95,6 +95,32 @@ graph LR
|
||||
unpack step will live in the sorter (`--unpack`, 7z binary) but stays off
|
||||
until the link-based flow is proven on a small batch.
|
||||
|
||||
## Extraction stage (`tools/extract_all.py`)
|
||||
|
||||
That endgame stage now exists as a standalone tool (the sorter stays
|
||||
plan/apply-only):
|
||||
|
||||
python3 tools/extract_all.py <src_root>... --out <extracted_root>
|
||||
|
||||
- Every archive (zip/7z/rar/tar/gz/bz2/xz, also nested ones) is extracted
|
||||
into a folder named after the archive, preserving relative structure;
|
||||
nested archives are expanded recursively and deleted, so the tree ends
|
||||
up with **no archives at all**.
|
||||
- Plain files are hardlinked into the tree (fallback copy) — nothing lost,
|
||||
no duplicate data.
|
||||
- Sources are never modified. Failed extractions (HTML-page fake zips,
|
||||
password archives, disk full) are moved to `<out>/.broken/` and the run
|
||||
continues; re-runs skip already-extracted destinations.
|
||||
- `<out>/extract_manifest.jsonl` records `{archive, dest, unit}` per
|
||||
extraction. The sorter's `--provenance` flag reads it: all files inside
|
||||
an extraction directory are classified **once under the archive's
|
||||
name** (a loose `Backpack.stl` alone is noise; under
|
||||
`The Twin Lance/` it belongs to that unit) and placed together, keeping
|
||||
the archive's inner directory structure:
|
||||
|
||||
python3 sorter.py <extracted_root> --library <lib> \
|
||||
--provenance <extracted_root>/extract_manifest.jsonl --apply
|
||||
|
||||
## Catalog file (`data/library_catalog.json`)
|
||||
|
||||
```json
|
||||
|
||||
@@ -52,6 +52,9 @@ def main() -> None:
|
||||
parser.add_argument("--apply", action="store_true",
|
||||
help="create the targets (hardlink, fallback copy)")
|
||||
parser.add_argument("--audit", type=Path, default=None, metavar="FILE")
|
||||
parser.add_argument("--provenance", type=Path, default=None, metavar="FILE",
|
||||
help="extract_manifest.jsonl from tools/extract_all.py: "
|
||||
"sort whole extraction directories as units")
|
||||
parser.add_argument("--json", action="store_true",
|
||||
help="print the plan as JSON instead of text")
|
||||
args = parser.parse_args()
|
||||
@@ -65,7 +68,17 @@ def main() -> None:
|
||||
catalog = load_catalog(args.catalog)
|
||||
audit = args.audit or (args.library / "sorter_applied.jsonl")
|
||||
|
||||
items = sort_plan(files, args.library, catalog, threshold=args.threshold)
|
||||
provenance = None
|
||||
if args.provenance:
|
||||
provenance = {}
|
||||
with args.provenance.open(encoding="utf-8") as fh:
|
||||
for line in fh:
|
||||
rec = json.loads(line)
|
||||
provenance[rec["dest"]] = rec["unit"]
|
||||
log.info("Provenance: %d extraction directories", len(provenance))
|
||||
|
||||
items = sort_plan(files, args.library, catalog, threshold=args.threshold,
|
||||
provenance=provenance, root=source)
|
||||
|
||||
if args.json:
|
||||
print(json.dumps([
|
||||
|
||||
@@ -0,0 +1,225 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Recursively extract every archive under the source roots into one tree.
|
||||
|
||||
python3 tools/extract_all.py <src_root>... --out <extracted_root>
|
||||
|
||||
Behavior:
|
||||
- Every archive (zip/7z/rar/tar/gz/bz2/xz) is extracted into a directory
|
||||
named after the archive (suffix stripped), preserving the relative path
|
||||
under the source root: <src>/zips/The Twin Lance.7z
|
||||
-> <out>/1/zips/The Twin Lance/...
|
||||
- Archives found *inside* an extraction are extracted too (BFS, until no
|
||||
archive remains), then deleted - the extracted tree holds no archives.
|
||||
- Plain files are hardlinked into the tree (fallback: copy), so nothing
|
||||
is lost and no data is duplicated.
|
||||
- The source roots are never modified (originals stay).
|
||||
- A provenance manifest `<out>/extract_manifest.jsonl` records one line
|
||||
per extraction: {"archive": ..., "dest": ..., "unit": ...} for the
|
||||
sorter's --provenance unit mode.
|
||||
- Failures (bad download/HTML page, wrong password, disk full) are moved
|
||||
to `<out>/.broken/<relpath>` and logged; the run never stops.
|
||||
|
||||
Re-runs are safe: destinations that already have content are skipped.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import logging
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
from collections import deque
|
||||
from pathlib import Path
|
||||
|
||||
log = logging.getLogger("extract_all")
|
||||
|
||||
ARCHIVE_SUFFIXES = {".zip", ".7z", ".rar", ".tar", ".gz", ".bz2", ".xz"}
|
||||
EXTRACT_TIMEOUT = 4 * 3600 # seconds per archive
|
||||
MIN_FREE_BYTES = 20 * 1024**3 # stop extracting below this free space
|
||||
RUN_ARTIFACTS = {
|
||||
"export.json", "export.html", "state.json", "manifest.jsonl",
|
||||
"failed_urls.txt", "catalogue.md", "catalogue.jsonl",
|
||||
"sorter_applied.jsonl", "catalogue_moves.jsonl", "urls.txt",
|
||||
"urls_0807.txt", "urls_gofile.txt", "failed_0807.txt",
|
||||
"extract_manifest.jsonl",
|
||||
}
|
||||
|
||||
|
||||
def archive_suffixes(name: str) -> list[str]:
|
||||
"""All trailing archive suffixes, e.g. 'x.tar.gz' -> ['.gz', '.tar']."""
|
||||
out = []
|
||||
p = Path(name)
|
||||
while p.suffix.lower() in ARCHIVE_SUFFIXES:
|
||||
out.append(p.suffix.lower())
|
||||
p = p.with_suffix("")
|
||||
return out
|
||||
|
||||
|
||||
def is_archive(path: Path) -> bool:
|
||||
return bool(archive_suffixes(path.name))
|
||||
|
||||
|
||||
def display_unit(name: str) -> str:
|
||||
"""Archive name with all archive suffixes stripped: the unit folder."""
|
||||
p = Path(name)
|
||||
while p.suffix.lower() in ARCHIVE_SUFFIXES:
|
||||
p = p.with_suffix("")
|
||||
return p.name
|
||||
|
||||
|
||||
def free_bytes(path: Path) -> int:
|
||||
return shutil.disk_usage(path).free
|
||||
|
||||
|
||||
def extract(archive: Path, dest: Path) -> tuple[bool, str]:
|
||||
"""7z x into dest. Empty -p avoids interactive prompts (no password)."""
|
||||
dest.mkdir(parents=True, exist_ok=True)
|
||||
cmd = ["7z", "x", "-y", "-p", f"-o{dest}", str(archive)]
|
||||
try:
|
||||
proc = subprocess.run(
|
||||
cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
|
||||
text=True, timeout=EXTRACT_TIMEOUT,
|
||||
)
|
||||
except subprocess.TimeoutExpired:
|
||||
return False, f"extraction timed out after {EXTRACT_TIMEOUT}s"
|
||||
except OSError as exc:
|
||||
return False, f"7z failed to run: {exc}"
|
||||
if proc.returncode in (0, 1): # 0 ok, 1 warnings (e.g. minor CRC)
|
||||
if proc.returncode == 1:
|
||||
log.warning("7z reported warnings extracting %s", archive.name)
|
||||
return True, ""
|
||||
tail = "\n".join((proc.stdout or "").splitlines()[-5:])
|
||||
return False, f"7z exit {proc.returncode}: {tail}"
|
||||
|
||||
|
||||
def move_to_broken(path: Path, out_root: Path, src_root: Path, note: str) -> Path:
|
||||
rel = path.relative_to(src_root)
|
||||
dest = out_root / ".broken" / rel
|
||||
dest.parent.mkdir(parents=True, exist_ok=True)
|
||||
if dest.exists():
|
||||
dest = dest.with_name(dest.name + ".1")
|
||||
return shutil.move(str(path), str(dest))
|
||||
|
||||
|
||||
def expand_nested(dest: Path, out_root: Path, manifest, src_root: Path) -> int:
|
||||
"""Extract archives inside `dest` until none remain. Returns count."""
|
||||
done = 0
|
||||
queue: deque[Path] = deque([dest])
|
||||
while queue:
|
||||
cur = queue.popleft()
|
||||
for p in sorted(cur.rglob("*")):
|
||||
if not p.is_file() or not is_archive(p):
|
||||
continue
|
||||
stem_dir = p.with_name(display_unit(p.name))
|
||||
if stem_dir.exists() and any(stem_dir.iterdir()):
|
||||
# already extracted in a previous run; drop the archive copy
|
||||
p.unlink()
|
||||
log.info("Nested %s already extracted - archive removed", p.name)
|
||||
continue
|
||||
if free_bytes(out_root) < MIN_FREE_BYTES:
|
||||
log.error("Out of disk space - skipping %s (kept in place)", p)
|
||||
continue
|
||||
ok, err = extract(p, stem_dir)
|
||||
if not ok:
|
||||
log.error("FAILED nested extraction %s: %s", p, err)
|
||||
move_to_broken(p, out_root, src_root, err)
|
||||
continue
|
||||
manifest.write(f"{json_line(p.relative_to(out_root), stem_dir)}\n")
|
||||
p.unlink() # archive copy consumed
|
||||
done += 1
|
||||
queue.append(stem_dir)
|
||||
return done
|
||||
|
||||
|
||||
def json_line(archive_rel: Path, dest: Path) -> str:
|
||||
import json
|
||||
return json.dumps({
|
||||
"archive": str(archive_rel),
|
||||
"dest": str(dest.relative_to(_OUT_ROOT)),
|
||||
"unit": display_unit(archive_rel.name),
|
||||
}, ensure_ascii=False)
|
||||
|
||||
|
||||
_OUT_ROOT: Path = Path(".") # set in main; used by json_line
|
||||
|
||||
|
||||
def process_root(src_root: Path, out_root: Path, manifest) -> tuple[int, int, int]:
|
||||
"""Returns (extracted, mirrored, failed)."""
|
||||
idx = src_root.name
|
||||
extracted = mirrored = failed = 0
|
||||
for f in sorted(src_root.rglob("*")):
|
||||
if not f.is_file():
|
||||
continue
|
||||
if (f.name in RUN_ARTIFACTS or f.name.startswith(".")
|
||||
or f.name.endswith(".part")):
|
||||
continue
|
||||
rel = f.relative_to(src_root)
|
||||
if is_archive(f):
|
||||
dest_rel = Path(idx) / Path(str(rel.with_name(display_unit(f.name))))
|
||||
dest = out_root / dest_rel
|
||||
if dest.exists() and any(dest.iterdir()):
|
||||
log.info("Skip (already extracted): %s", rel)
|
||||
continue
|
||||
if free_bytes(out_root) < MIN_FREE_BYTES:
|
||||
log.error("Out of disk space - skipping archive %s", f)
|
||||
failed += 1
|
||||
continue
|
||||
ok, err = extract(f, dest)
|
||||
if not ok:
|
||||
log.error("FAILED %s: %s", rel, err)
|
||||
move_to_broken(f, out_root, src_root, err)
|
||||
failed += 1
|
||||
continue
|
||||
manifest.write(f"{json_line(Path(idx) / rel, dest)}\n")
|
||||
extracted += 1
|
||||
log.info("Extracted %s -> %s", rel, dest_rel)
|
||||
expand_nested(dest, out_root, manifest, src_root)
|
||||
else:
|
||||
link_dest = out_root / idx / rel
|
||||
if link_dest.exists():
|
||||
continue
|
||||
link_dest.parent.mkdir(parents=True, exist_ok=True)
|
||||
try:
|
||||
os.link(f, link_dest)
|
||||
except OSError:
|
||||
shutil.copy2(f, link_dest)
|
||||
mirrored += 1
|
||||
return extracted, mirrored, failed
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
|
||||
parser.add_argument("sources", nargs="+", type=Path)
|
||||
parser.add_argument("--out", type=Path, required=True, metavar="DIR")
|
||||
args = parser.parse_args()
|
||||
|
||||
logging.basicConfig(level=logging.INFO,
|
||||
format="%(asctime)s %(levelname)-7s %(message)s",
|
||||
datefmt="%H:%M:%S")
|
||||
|
||||
global _OUT_ROOT
|
||||
_OUT_ROOT = args.out
|
||||
args.out.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
totals = [0, 0, 0]
|
||||
with (args.out / "extract_manifest.jsonl").open("a", encoding="utf-8") as manifest:
|
||||
for src in args.sources:
|
||||
log.info("== processing %s", src)
|
||||
counts = process_root(src, args.out, manifest)
|
||||
log.info("== %s: %d extracted, %d mirrored, %d failed",
|
||||
src.name, *counts)
|
||||
for i, c in enumerate(counts):
|
||||
totals[i] += c
|
||||
log.info("TOTAL: %d archives extracted, %d files mirrored, %d failed",
|
||||
*totals)
|
||||
if totals[2]:
|
||||
log.warning("Failures were moved under %s - inspect and re-run", args.out / ".broken")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
try:
|
||||
main()
|
||||
except KeyboardInterrupt:
|
||||
sys.exit(130)
|
||||
Reference in New Issue
Block a user