Ben_Kosytorz
acb3415878
- dlphelper/warhammer_data.py: keyword taxonomy (40k factions incl. Chaos and Xenos super-faction units, plus AoS/Heresy/Kill Team/Necromunda/Epic/OW) - dlphelper/catalogue.py: manifest+directory scanner, conservative keyword classifier (plural-tolerant word boundaries, longest match wins), markdown and jsonl reports, overrides support, audited --apply moves - catalogue.py CLI; tests/test_catalogue.py offline classifier test - real-data check on the user's dirs: 1624/3333 files classified so far
119 lines
4.6 KiB
Python
119 lines
4.6 KiB
Python
#!/usr/bin/env python3
|
|
"""Catalogue CLI: classify the collection into system / faction / class.
|
|
|
|
Examples:
|
|
# catalogue the dlp.py run dir (manifest + downloads)
|
|
python3 catalogue.py /path/to/run_dir
|
|
|
|
# additionally scan manual-era directories
|
|
python3 catalogue.py /path/to/run_dir --scan /big/storage/manual/Aeldari
|
|
|
|
# catalogue ONLY manual dirs, without any run dir
|
|
python3 catalogue.py --scan /big/storage/manual --out-md catalogue.md
|
|
|
|
# fix misclassifications (see overrides format below)
|
|
python3 catalogue.py <dir> --overrides overrides.json
|
|
|
|
# actually move files into <library>/<System>/<Faction>/<Class>/
|
|
python3 catalogue.py <dir> --apply --library /big/storage/warhammer_library
|
|
|
|
overrides.json format (keys are file names or full paths, case-insensitive):
|
|
{
|
|
"some file.zip": {"faction": "Aeldari", "class": "Heavy Support",
|
|
"system": "Warhammer 40,000"}
|
|
}
|
|
|
|
Outputs (default into the first given dir, or cwd):
|
|
catalogue.md human-readable, grouped system > faction > class
|
|
catalogue.jsonl one JSON line per file, with source attribution
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import logging
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
from dlphelper.catalogue import (
|
|
build_catalogue,
|
|
format_size,
|
|
apply_moves,
|
|
write_reports,
|
|
)
|
|
|
|
log = logging.getLogger("catalogue")
|
|
|
|
|
|
def main() -> None:
|
|
parser = argparse.ArgumentParser(
|
|
description="Classify the Warhammer collection into factions and classes."
|
|
)
|
|
parser.add_argument("run_dir", nargs="?", type=Path,
|
|
help="a dlp.py run dir (uses its manifest.jsonl and downloads/)")
|
|
parser.add_argument("--scan", action="append", default=[], type=Path, metavar="DIR",
|
|
help="additional directory to scan for archive files (repeatable)")
|
|
parser.add_argument("--manifest", type=Path, default=None, metavar="FILE",
|
|
help="explicit manifest.jsonl (default: <run_dir>/manifest.jsonl)")
|
|
parser.add_argument("--overrides", type=Path, default=None, metavar="FILE",
|
|
help="manual classification overrides (JSON)")
|
|
parser.add_argument("--out-md", type=Path, default=None, metavar="FILE")
|
|
parser.add_argument("--out-jsonl", type=Path, default=None, metavar="FILE")
|
|
parser.add_argument("--apply", action="store_true",
|
|
help="move files into the library structure (default: preview)")
|
|
parser.add_argument("--library", type=Path, default=None, metavar="DIR",
|
|
help="library root for --apply (required with --apply)")
|
|
args = parser.parse_args()
|
|
|
|
logging.basicConfig(level=logging.INFO,
|
|
format="%(asctime)s %(levelname)-7s %(message)s",
|
|
datefmt="%H:%M:%S")
|
|
|
|
run_dir = args.run_dir
|
|
manifests = [args.manifest] if args.manifest else (
|
|
[run_dir / "manifest.jsonl"] if run_dir else []
|
|
)
|
|
scan_dirs = list(args.scan)
|
|
if run_dir and not args.manifest:
|
|
# run dir is scanned as a whole (covers files not yet in the manifest)
|
|
scan_dirs.append(run_dir)
|
|
|
|
if not manifests and not scan_dirs:
|
|
parser.error("give a run_dir or at least one --scan directory")
|
|
|
|
out_base = run_dir if run_dir else Path.cwd()
|
|
out_md = args.out_md or out_base / "catalogue.md"
|
|
out_jsonl = args.out_jsonl or out_base / "catalogue.jsonl"
|
|
|
|
entries = build_catalogue(manifests, scan_dirs, args.overrides)
|
|
write_reports(entries, out_md, out_jsonl)
|
|
|
|
classified = sum(1 for e in entries if e.faction != "Unclassified")
|
|
log.info("Catalogued %d files (%d classified) -> %s", len(entries), classified, out_md)
|
|
|
|
# compact per-faction summary to the console
|
|
summary: dict[tuple[str, str], list[int]] = {}
|
|
for e in entries:
|
|
key = (e.system, e.faction)
|
|
n, size = summary.get(key, (0, 0))
|
|
summary[key] = [n + 1, size + e.size]
|
|
for (system, faction), (n, size) in sorted(summary.items()):
|
|
log.info(" %-18s %-32s %4d files %s", system, faction, n, format_size(size))
|
|
|
|
if args.apply:
|
|
if not args.library:
|
|
parser.error("--apply needs --library DIR")
|
|
if not args.run_dir:
|
|
parser.error("--apply needs a run_dir (files are moved out of it)")
|
|
moved = apply_moves(entries, args.library, args.library / "catalogue_moves.jsonl")
|
|
log.info("Moved %d files into %s (audit: catalogue_moves.jsonl)", moved, args.library)
|
|
# rewrite reports with updated paths
|
|
write_reports(entries, out_md, out_jsonl)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
try:
|
|
main()
|
|
except KeyboardInterrupt:
|
|
sys.exit(130)
|