Sorter: fuzzysort files into the miniset-aligned library via hardlinks
- docs/SORTING.md: target hierarchy (game > faction group > faction > subfaction > class > unit, miniset.net naming), canonical-name policy, hardlink policy, unzip outlook - data/library_catalog.json: canonical catalog (name/aliases/subfaction/class) - dlphelper/sorter.py: inspect (zip via zipfile, 7z via 7z l), keyword classification incl. inner-name voting for hash-named archives, fuzzy matching (difflib), plan/apply split, hardlink preferred with copy fallback, append-only audit, existing targets skipped, run artifacts filtered, HTML-page detection for bad downloads - sorter.py CLI (plan by default, --apply to act)
This commit is contained in:
@@ -0,0 +1,331 @@
|
||||
"""Sorter: place collection files into the library via plan/apply.
|
||||
|
||||
See docs/SORTING.md for the structure and the pipeline this implements:
|
||||
inspect (name + archive listing) -> classify (keyword tables) -> fuzzy match
|
||||
against the library catalog -> plan -> apply (hardlink, fallback copy).
|
||||
|
||||
The sorter never deletes or modifies source files. Applying is append-only:
|
||||
targets that already exist are skipped, actions are audited in
|
||||
sorter_applied.jsonl.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import difflib
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
import zipfile
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
|
||||
from dlphelper.catalogue import _clean_name, _match_class, _match_faction
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
ARCHIVE_SUFFIXES = {".zip", ".7z", ".rar", ".tar", ".gz"}
|
||||
UNSORTED = "_unsorted"
|
||||
UNCLASSIFIED_CLASS = "Unclassified"
|
||||
|
||||
# Run-artifact files that must never be sorted into a library
|
||||
RUN_ARTIFACTS = {
|
||||
"export.json", "export.html", "state.json", "manifest.jsonl",
|
||||
"failed_urls.txt", "catalogue.md", "catalogue.jsonl",
|
||||
"sorter_applied.jsonl", "catalogue_moves.jsonl", "urls.txt",
|
||||
}
|
||||
|
||||
|
||||
@dataclass
|
||||
class SortItem:
|
||||
"""One file to sort, plus everything learned about it."""
|
||||
|
||||
path: Path
|
||||
size: int
|
||||
name: str # original file name
|
||||
clean: str # cleaned lower-case name
|
||||
inner_names: list[str] = field(default_factory=list) # archive listing
|
||||
system: str = "Unknown"
|
||||
group: str = UNSORTED
|
||||
faction: str = UNSORTED
|
||||
subfaction: str | None = None
|
||||
klass: str | None = None
|
||||
unit: str | None = None # canonical catalog name, if matched
|
||||
match_score: float = 0.0
|
||||
match_kind: str = "none" # catalog | implicit | unclassified
|
||||
action: str = "link" # link | copy
|
||||
target: Path | None = None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# inspection
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def list_zip(path: Path, limit: int = 40) -> list[str]:
|
||||
"""Inner names of a .zip without extracting."""
|
||||
try:
|
||||
with zipfile.ZipFile(path) as zf:
|
||||
return zf.namelist()[:limit]
|
||||
except (zipfile.BadZipFile, OSError, RuntimeError) as exc:
|
||||
log.debug("zip listing failed for %s: %s", path.name, exc)
|
||||
return []
|
||||
|
||||
|
||||
def list_7z(path: Path, limit: int = 40) -> list[str]:
|
||||
"""Inner names of a .7z via the installed 7z binary, without extracting."""
|
||||
if not shutil.which("7z"):
|
||||
return []
|
||||
try:
|
||||
proc = subprocess.run(
|
||||
["7z", "l", "-ba", "-slt", str(path)],
|
||||
stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, text=True, timeout=120,
|
||||
)
|
||||
except (OSError, subprocess.TimeoutExpired) as exc:
|
||||
log.debug("7z listing failed for %s: %s", path.name, exc)
|
||||
return []
|
||||
names: list[str] = []
|
||||
for block in proc.stdout.split("Path = ")[1:]:
|
||||
name = block.splitlines()[0].strip() if block.splitlines() else ""
|
||||
# -slt lists directories too; skip entries ending in /
|
||||
if name and not name.endswith("/"):
|
||||
names.append(name)
|
||||
if len(names) >= limit:
|
||||
break
|
||||
return names
|
||||
|
||||
|
||||
def inspect(path: Path) -> SortItem:
|
||||
size = path.stat().st_size if path.exists() else 0
|
||||
item = SortItem(path=path, size=size, name=path.name, clean=_clean_name(path.name))
|
||||
if not path.is_file():
|
||||
return item
|
||||
if path.suffix.lower() in ARCHIVE_SUFFIXES:
|
||||
# An archive that is not an archive is almost always a saved HTML
|
||||
# error/login page that got downloaded instead of the file.
|
||||
if path.suffix.lower() == ".zip":
|
||||
item.inner_names = list_zip(path)
|
||||
else:
|
||||
item.inner_names = list_7z(path)
|
||||
if item.inner_names == [] and path.stat().st_size > 0:
|
||||
head = b""
|
||||
with path.open("rb") as fh:
|
||||
head = fh.read(512)
|
||||
if b"<!doctype html" in head.lower() or b"<html" in head.lower():
|
||||
log.warning("%s is an HTML page, not an archive (bad download)", path.name)
|
||||
item.match_kind = "invalid"
|
||||
return item
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# classification & matching
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def classify_item(item: SortItem) -> None:
|
||||
"""Keyword classification from the file name, then refined by inner names."""
|
||||
hit = _match_faction(item.clean)
|
||||
if hit:
|
||||
item.system, item.faction = hit
|
||||
item.klass = _match_class(item.clean) or (
|
||||
_match_class(_clean_name(" ".join(item.inner_names))) if item.inner_names else None
|
||||
)
|
||||
|
||||
if item.faction == UNSORTED and item.inner_names:
|
||||
# hash-named archives: vote on the inner names' classification
|
||||
votes: dict[tuple[str, str], int] = {}
|
||||
for inner in item.inner_names:
|
||||
ihit = _match_faction(_clean_name(inner))
|
||||
if ihit:
|
||||
votes[ihit] = votes.get(ihit, 0) + 1
|
||||
if votes:
|
||||
best = max(votes.items(), key=lambda kv: kv[1])[0]
|
||||
item.system, item.faction = best
|
||||
if item.klass is None:
|
||||
inner_classes = [
|
||||
_match_class(_clean_name(inner)) for inner in item.inner_names
|
||||
]
|
||||
cvotes: dict[str, int] = {}
|
||||
for c in inner_classes:
|
||||
if c:
|
||||
cvotes[c] = cvotes.get(c, 0) + 1
|
||||
if cvotes:
|
||||
item.klass = max(cvotes.items(), key=lambda kv: kv[1])[0]
|
||||
|
||||
item.group = GROUP_OF_FACTION.get(item.faction, UNSORTED)
|
||||
|
||||
|
||||
# faction group layer (miniset naming), extended as the taxonomy grows
|
||||
GROUP_OF_FACTION: dict[str, str] = {
|
||||
faction: group
|
||||
for group, factions in {
|
||||
"Armies of the Imperium": [
|
||||
"Adeptus Astartes", "Astra Militarum", "Adeptus Mechanicus",
|
||||
"Adepta Sororitas", "Adeptus Custodes", "Imperial Knights",
|
||||
"Inquisition & Officio Assassinorum",
|
||||
],
|
||||
"Armies of Chaos": [
|
||||
"Chaos Space Marines", "Death Guard", "Thousand Sons",
|
||||
"Chaos Daemons", "Chaos Knights",
|
||||
],
|
||||
"Xenos Armies": [
|
||||
"Orks", "T'au Empire", "Tyranids", "Genestealer Cults", "Necrons",
|
||||
"Aeldari", "Drukhari", "Harlequins", "Ynnari",
|
||||
],
|
||||
}.items()
|
||||
for faction in factions
|
||||
}
|
||||
|
||||
|
||||
def load_catalog(path: Path) -> list[dict]:
|
||||
data = json.loads(path.read_text(encoding="utf-8"))
|
||||
entries = data["entries"] if isinstance(data, dict) else data
|
||||
for e in entries:
|
||||
e["_clean"] = _clean_name(e["name"])
|
||||
e["_clean_aliases"] = [_clean_name(a) for a in e.get("aliases", [])]
|
||||
return entries
|
||||
|
||||
|
||||
def _best_ratio(clean: str, candidates: list[str]) -> float:
|
||||
return max(
|
||||
(difflib.SequenceMatcher(None, clean, c).ratio() for c in candidates),
|
||||
default=0.0,
|
||||
)
|
||||
|
||||
|
||||
def match_catalog(item: SortItem, catalog: list[dict], threshold: float) -> None:
|
||||
"""Fuzzy-match against catalog entries in the same faction (or all)."""
|
||||
if item.faction == UNSORTED:
|
||||
return
|
||||
same_faction = [e for e in catalog if e["faction"] == item.faction]
|
||||
pool = same_faction or catalog # fall back to whole catalog if faction unknown
|
||||
best: tuple[float, dict] | None = None
|
||||
for entry in pool:
|
||||
cands = [entry["_clean"], *entry["_clean_aliases"]]
|
||||
score = _best_ratio(item.clean, cands)
|
||||
if item.inner_names:
|
||||
inner_clean = [_clean_name(n) for n in item.inner_names[:10]]
|
||||
for c in inner_clean:
|
||||
score = max(score, _best_ratio(c, cands) * 0.95) # inner slightly discounted
|
||||
if best is None or score > best[0]:
|
||||
best = (score, entry)
|
||||
if best and best[0] >= threshold:
|
||||
entry = best[1]
|
||||
item.unit = entry["name"]
|
||||
item.subfaction = entry.get("subfaction")
|
||||
item.klass = item.klass or entry.get("class")
|
||||
item.match_score = best[0]
|
||||
item.match_kind = "catalog"
|
||||
if item.faction == UNSORTED:
|
||||
item.system = entry["system"]
|
||||
item.faction = entry["faction"]
|
||||
item.group = entry.get("group") or GROUP_OF_FACTION.get(entry["faction"], UNSORTED)
|
||||
else:
|
||||
if item.faction != UNSORTED:
|
||||
item.unit = item.name # implicit entry: own (original) name
|
||||
item.match_score = best[0] if best else 0.0
|
||||
item.match_kind = "implicit"
|
||||
|
||||
|
||||
def build_target(item: SortItem, library: Path) -> Path:
|
||||
parts: list[str] = []
|
||||
if item.system == "Unknown":
|
||||
parts = [UNSORTED]
|
||||
else:
|
||||
parts = [item.system]
|
||||
if item.faction != UNSORTED:
|
||||
if item.group:
|
||||
parts.append(item.group)
|
||||
parts.append(item.faction)
|
||||
if item.subfaction:
|
||||
parts.append(item.subfaction)
|
||||
parts.append(item.klass or UNCLASSIFIED_CLASS)
|
||||
if item.unit:
|
||||
parts.append(_display_name(item.unit))
|
||||
else:
|
||||
parts.append(UNSORTED)
|
||||
target_dir = Path(library).joinpath(*parts)
|
||||
return target_dir / item.name
|
||||
|
||||
|
||||
def _display_name(unit: str) -> str:
|
||||
"""Implicit entries carry the original filename; strip the extension."""
|
||||
p = Path(unit)
|
||||
return p.stem if p.suffix else unit
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# plan / apply
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def sort_plan(
|
||||
files: list[Path],
|
||||
library: Path,
|
||||
catalog: list[dict],
|
||||
threshold: float = 0.62,
|
||||
) -> list[SortItem]:
|
||||
items = []
|
||||
for f in sorted(files):
|
||||
if not f.is_file():
|
||||
continue
|
||||
if (f.name.endswith(".part") or f.name.startswith(".")
|
||||
or f.name in RUN_ARTIFACTS):
|
||||
continue
|
||||
item = inspect(f)
|
||||
classify_item(item)
|
||||
match_catalog(item, catalog, threshold)
|
||||
item.target = build_target(item, library)
|
||||
item.action = "link"
|
||||
items.append(item)
|
||||
return items
|
||||
|
||||
|
||||
def apply_plan(items: list[SortItem], audit_path: Path) -> tuple[int, int, int]:
|
||||
"""Create targets. Hardlink preferred; copy when filesystems differ.
|
||||
|
||||
Returns (linked, copied, skipped).
|
||||
"""
|
||||
audit_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
linked = copied = skipped = 0
|
||||
with audit_path.open("a", encoding="utf-8") as audit:
|
||||
for item in items:
|
||||
src, dest = item.path, item.target
|
||||
if item.match_kind == "invalid":
|
||||
log.error("Not sorting %s: it is an HTML page, not an archive", src.name)
|
||||
skipped += 1
|
||||
continue
|
||||
if dest is None or not src.is_file():
|
||||
skipped += 1
|
||||
continue
|
||||
if dest.exists():
|
||||
log.info("Skip, target exists: %s", dest)
|
||||
skipped += 1
|
||||
continue
|
||||
dest.parent.mkdir(parents=True, exist_ok=True)
|
||||
if os.stat(src).st_dev == os.stat(dest.parent).st_dev:
|
||||
os.link(src, dest)
|
||||
item.action = "link"
|
||||
linked += 1
|
||||
else:
|
||||
log.warning(
|
||||
"Different filesystem - COPYING instead of hardlinking %s", src.name
|
||||
)
|
||||
shutil.copy2(src, dest)
|
||||
item.action = "copy"
|
||||
copied += 1
|
||||
audit.write(json.dumps({
|
||||
"from": str(src), "to": str(dest), "action": item.action,
|
||||
"unit": item.unit, "faction": item.faction, "class": item.klass,
|
||||
}) + "\n")
|
||||
return linked, copied, skipped
|
||||
|
||||
|
||||
def format_plan(items: list[SortItem]) -> str:
|
||||
lines = []
|
||||
for item in items:
|
||||
if item.match_kind == "invalid":
|
||||
lines.append(f"[x] {item.name} - NOT AN ARCHIVE (HTML page, bad download)")
|
||||
continue
|
||||
src = f"{item.name} ({item.size // (1024 * 1024)} MiB)" if item.size > 4 * 1024 * 1024 else item.name
|
||||
flag = {"catalog": "+", "implicit": "~", "none": "?"}[item.match_kind]
|
||||
lines.append(f"[{flag}] {src}\n -> {item.target}")
|
||||
return "\n".join(lines)
|
||||
Reference in New Issue
Block a user