chore: start transition layout — python into version-python/, scaffold version-rust/
- git mv of all existing files into version-python/ (history preserved) - version-rust/: minimal cargo scaffold (edition 2024), no deps yet - root AGENTS.md documents the transition rules and artifact/schema contract - main is untouched; the python layout there stays as-is
This commit is contained in:
@@ -0,0 +1,62 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Offline classifier test for the catalogue (no network, no deps).
|
||||
|
||||
Run with: python3 tests/test_catalogue.py
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
|
||||
|
||||
from dlphelper.catalogue import classify # noqa: E402
|
||||
|
||||
CASES = [
|
||||
# (filename, expected system, expected faction, expected class or None-any)
|
||||
# GW naming: chapters are factions, classes are GW unit types.
|
||||
("Thunderwolf Cavalry.7z", "Warhammer 40,000", "Space Wolves", "Mounted"),
|
||||
("Wulfen.zip", "Warhammer 40,000", "Space Wolves", "Infantry"),
|
||||
("Keeper of Secrets & Shalaxi Helbane.7z", "Warhammer 40,000", "Chaos Daemons", "Hero"),
|
||||
("Assorted Drones.zip", "Warhammer 40,000", "T'au Empire", None),
|
||||
("Commander Shadowsun.zip", "Warhammer 40,000", "T'au Empire", "Hero"),
|
||||
("Devilfish.zip", "Warhammer 40,000", "T'au Empire", "Vehicle"),
|
||||
("Battle Sister Scanz.zip", "Warhammer 40,000", "Adepta Sororitas", None),
|
||||
("Knights.zip", None, None, None), # ambiguous keyword alone: unclassified
|
||||
("Khorne Demon Enginez and skull lordz.zip", "Warhammer 40,000", "Chaos Daemons", None),
|
||||
("Cultists.zip", "Warhammer 40,000", "Chaos Space Marines", "Battleline"),
|
||||
("Da Troopz.zip", "Warhammer 40,000", "Orks", None),
|
||||
("Seraphon.zip", "Age of Sigmar", "Seraphon", None),
|
||||
("Leviathan Pattern Dreadnought.zip", "The Horus Heresy", "Space Marine Legions", "Walker"),
|
||||
("Poxwalkers.zip", "Warhammer 40,000", "Death Guard", "Infantry"),
|
||||
("Kabalite Warriors.zip", "Warhammer 40,000", "Drukhari", "Battleline"),
|
||||
("Canoptek Wraiths.zip", "Warhammer 40,000", "Necrons", "Infantry"),
|
||||
("random thing with no keywords.zip", None, None, None),
|
||||
]
|
||||
|
||||
|
||||
def main() -> None:
|
||||
failures = 0
|
||||
for name, exp_system, exp_faction, exp_class in CASES:
|
||||
system, faction, klass = classify(name)
|
||||
problems = []
|
||||
if exp_system and system != exp_system:
|
||||
problems.append(f"system={system!r} != {exp_system!r}")
|
||||
if exp_faction and faction != exp_faction:
|
||||
problems.append(f"faction={faction!r} != {exp_faction!r}")
|
||||
if exp_class and klass != exp_class:
|
||||
problems.append(f"class={klass!r} != {exp_class!r}")
|
||||
if problems:
|
||||
failures += 1
|
||||
print(f"FAIL {name!r}: {', '.join(problems)}")
|
||||
else:
|
||||
print(f"ok {name!r} -> {system} / {faction} / {klass}")
|
||||
if failures:
|
||||
print(f"\n{failures} failures")
|
||||
sys.exit(1)
|
||||
print("\ntest_catalogue: OK")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,68 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Offline extraction test with a synthetic export.json (no network, no deps).
|
||||
|
||||
Run with: python3 tests/test_extract.py
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import json
|
||||
import sys
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
|
||||
|
||||
from dlphelper.discord import classify, extract_urls # noqa: E402
|
||||
|
||||
|
||||
def b64(s: str) -> str:
|
||||
return base64.b64encode(s.encode()).decode()
|
||||
|
||||
|
||||
def main() -> None:
|
||||
messages = [
|
||||
{
|
||||
"id": "1",
|
||||
"timestamp": "2026-01-01T00:00:00",
|
||||
"author": {"name": "poster"},
|
||||
"content": f"chaos stuff: {b64('https://gofile.io/d/abc123')}",
|
||||
},
|
||||
{
|
||||
"id": "2",
|
||||
"timestamp": "2026-01-02T00:00:00",
|
||||
"author": {"name": "poster"},
|
||||
"content": (
|
||||
f"0807 zip: {b64('https://0807.st/6saCNie.zip')}\n"
|
||||
f"dupe of 1: {b64('https://gofile.io/d/abc123')}\n"
|
||||
f"plain: https://0807.st/plainLink.zip and "
|
||||
f"https://discord.com/channels/1/2 (unsupported)\n"
|
||||
f"junk: {b64('https://pixeldrain.com/u/xyz')} "
|
||||
"and not-our-domain base64: aHR0cHM6Ly9leGFtcGxlLm9yZw=="
|
||||
),
|
||||
},
|
||||
]
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
export = Path(tmp) / "export.json"
|
||||
export.write_text(json.dumps({"messages": messages}), encoding="utf-8")
|
||||
found = extract_urls(export)
|
||||
|
||||
urls = {f.url for f in found}
|
||||
expected = {"https://gofile.io/d/abc123", "https://0807.st/6saCNie.zip",
|
||||
"https://0807.st/plainLink.zip"}
|
||||
assert urls == expected, f"unexpected urls: {sorted(urls)}"
|
||||
|
||||
by_url = {f.url: f for f in found}
|
||||
g = by_url["https://gofile.io/d/abc123"]
|
||||
assert g.host == "gofile" and g.message_id == "1" and g.author == "poster"
|
||||
assert classify("https://0807.st/x.zip") == "0807"
|
||||
assert classify("https://gofile.io/d/x") == "gofile"
|
||||
assert classify("https://example.org/x") is None
|
||||
|
||||
print("test_extract: OK")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,258 @@
|
||||
"""Offline sorter tests: dir re-homing, weak tier, junk dirs (plain python3)."""
|
||||
import json
|
||||
import shutil
|
||||
import sys
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
|
||||
|
||||
from dlphelper.sorter import ( # noqa: E402
|
||||
apply_plan,
|
||||
load_catalog,
|
||||
sort_plan,
|
||||
)
|
||||
|
||||
CATALOG = {
|
||||
"entries": [
|
||||
{"name": "Warp Spiders", "system": "Warhammer 40,000",
|
||||
"group": "Xenos Armies", "faction": "Aeldari",
|
||||
"class": "Fly", "aliases": []},
|
||||
{"name": "Harlequin Troupe", "system": "Warhammer 40,000",
|
||||
"group": "Xenos Armies", "faction": "Harlequins",
|
||||
"class": "Battleline", "aliases": []},
|
||||
{"name": "Guardian Squad", "system": "Warhammer 40,000",
|
||||
"group": "Xenos Armies", "faction": "Aeldari",
|
||||
"class": "Battleline", "aliases": []},
|
||||
{"name": "Rangers", "system": "Warhammer 40,000",
|
||||
"group": "Xenos Armies", "faction": "Aeldari",
|
||||
"class": "Battleline", "aliases": []},
|
||||
{"name": "Dark Reapers", "system": "Warhammer 40,000",
|
||||
"group": "Xenos Armies", "faction": "Aeldari",
|
||||
"class": "Aspect Warriors", "aliases": []},
|
||||
{"name": "Mordor Orcs", "system": "Middle-Earth",
|
||||
"group": "The Lord of the Rings™ - Evil", "faction": "Mordor",
|
||||
"class": "Warriors", "aliases": []},
|
||||
]
|
||||
}
|
||||
|
||||
|
||||
def make_lib():
|
||||
return Path(tempfile.mkdtemp(prefix="sorter_test_lib_"))
|
||||
|
||||
|
||||
def write(p: Path):
|
||||
p.parent.mkdir(parents=True, exist_ok=True)
|
||||
p.write_bytes(b"x")
|
||||
|
||||
|
||||
_LOADED = None
|
||||
|
||||
|
||||
def get_catalog():
|
||||
global _LOADED
|
||||
if _LOADED is None:
|
||||
_LOADED = load_catalog_path()
|
||||
return _LOADED
|
||||
|
||||
|
||||
def prov_items(root: Path, library):
|
||||
(root / "extract_manifest.jsonl").write_text(json.dumps({
|
||||
"archive": str(root / "Multi Pack.zip"),
|
||||
"dest": "Multi Pack", "unit": "Multi Pack.zip",
|
||||
}), encoding="utf-8")
|
||||
files = sorted((root / "Multi Pack").rglob("*"))
|
||||
return sort_plan(files, library, get_catalog(), provenance={
|
||||
"Multi Pack": "Multi Pack.zip"}, root=root)
|
||||
|
||||
|
||||
def test_prov_dir_rehome():
|
||||
"""A matching inner dir name claims its subtree; junk dirs don't."""
|
||||
root = Path(tempfile.mkdtemp(prefix="sorter_test_src_"))
|
||||
pack = root / "Multi Pack"
|
||||
write(pack / "14 Warp spiders" / "spider_body.stl")
|
||||
write(pack / "14 Warp spiders" / "files" / "spider_arm.stl")
|
||||
write(pack / "HEADS" / "head_a.stl") # junk dir: stays with pack
|
||||
write(pack / "Retro GrimDark Thing" / "t.stl") # no catalog match
|
||||
library = make_lib()
|
||||
items = prov_items(root, library)
|
||||
tgt = {i.name: i.target for i in items}
|
||||
lib = str(library)
|
||||
# warp spiders re-homed, incl. nested junk dir structure, class from catalog
|
||||
assert "Warp Spiders" in str(tgt["spider_body.stl"]), tgt["spider_body.stl"]
|
||||
assert "Fly" in str(tgt["spider_body.stl"]), tgt["spider_body.stl"]
|
||||
assert "Hero" not in str(tgt["spider_body.stl"])
|
||||
assert "files" in str(tgt["spider_arm.stl"])
|
||||
# junk-dir file rides along with the pack's adopted unit
|
||||
assert "Warp Spiders" in str(tgt["head_a.stl"])
|
||||
assert "HEADS" in str(tgt["head_a.stl"])
|
||||
# unmatched subtree keeps the pack unit folder
|
||||
assert "Warp Spiders" in str(tgt["t.stl"]) and "Retro GrimDark Thing" in str(tgt["t.stl"])
|
||||
shutil.rmtree(root); shutil.rmtree(library)
|
||||
|
||||
|
||||
def test_prov_weak_tier():
|
||||
"""Below-threshold dir hit is accepted as low confidence."""
|
||||
root = Path(tempfile.mkdtemp(prefix="sorter_test_src_"))
|
||||
# pack name carries a keyword so the probe resolves system+faction -
|
||||
# only then does the same-faction weak tier apply
|
||||
pack = root / "Aeldari Pack"
|
||||
write(pack / "guardian weapon team" / "g.stl") # 0.59 vs Guardian Squad
|
||||
library = make_lib()
|
||||
(root / "extract_manifest.jsonl").write_text(json.dumps({
|
||||
"archive": str(root / "Aeldari Pack.zip"),
|
||||
"dest": "Aeldari Pack", "unit": "Aeldari Pack.zip",
|
||||
}), encoding="utf-8")
|
||||
files = sorted((root / "Aeldari Pack").rglob("*"))
|
||||
items = sort_plan(files, library, get_catalog(), provenance={
|
||||
"Aeldari Pack": "Aeldari Pack.zip"}, root=root)
|
||||
it = next(i for i in items if i.name == "g.stl")
|
||||
assert it.match_kind == "weak", (it.match_kind, it.confidence)
|
||||
assert it.confidence == "low"
|
||||
assert "Guardian" in str(it.target)
|
||||
shutil.rmtree(root); shutil.rmtree(library)
|
||||
|
||||
|
||||
def test_cross_system_needs_token_containment():
|
||||
"""'Aeldari Rangers' must not adopt Middle-earth 'Dwarf Rangers'."""
|
||||
root = Path(tempfile.mkdtemp(prefix="sorter_test_src_"))
|
||||
pack = root / "MQVxKzh" # hash name: system stays unknown
|
||||
write(pack / "Aeldari Rangers" / "Luminary - Arms - (1).stl")
|
||||
write(pack / "14 Warp spiders" / "sp.stl")
|
||||
library = make_lib()
|
||||
files = sorted((root / "MQVxKzh").rglob("*"))
|
||||
items = sort_plan(files, library, get_catalog(), provenance={
|
||||
"MQVxKzh": "MQVxKzh.zip"}, root=root)
|
||||
by_name = {i.name: i for i in items}
|
||||
lum = by_name["Luminary - Arms - (1).stl"]
|
||||
assert "Middle-Earth" not in str(lum.target), lum.target
|
||||
spider = by_name["sp.stl"]
|
||||
assert "Warp Spiders" in str(spider.target), spider.target
|
||||
shutil.rmtree(root); shutil.rmtree(library)
|
||||
|
||||
|
||||
def test_weak_needs_shared_token():
|
||||
"""Sub-threshold hit with zero shared words is a fluke: reject.
|
||||
|
||||
'frontline squad' vs 'Guardian Squad' scores 0.552 but shares no
|
||||
token - it must NOT re-home, while 'guardian weapon team' (0.588,
|
||||
shares 'guardian') is accepted weak.
|
||||
"""
|
||||
root = Path(tempfile.mkdtemp(prefix="sorter_test_src_"))
|
||||
pack = root / "Aeldari Pack"
|
||||
write(pack / "frontline squad" / "f.stl")
|
||||
write(pack / "guardian weapon team" / "g.stl")
|
||||
library = make_lib()
|
||||
(root / "extract_manifest.jsonl").write_text(json.dumps({
|
||||
"archive": str(root / "Aeldari Pack.zip"),
|
||||
"dest": "Aeldari Pack", "unit": "Aeldari Pack.zip",
|
||||
}), encoding="utf-8")
|
||||
files = sorted((root / "Aeldari Pack").rglob("*"))
|
||||
items = sort_plan(files, library, get_catalog(), provenance={
|
||||
"Aeldari Pack": "Aeldari Pack.zip"}, root=root)
|
||||
by_name = {i.name: i for i in items}
|
||||
f = by_name["f.stl"]
|
||||
# both files ride along with the pack-level weak adoption
|
||||
assert f.match_kind == "weak" and f.confidence == "low", (f.match_kind, f.target)
|
||||
assert "frontline squad" in str(f.target)
|
||||
g = by_name["g.stl"]
|
||||
assert g.match_kind == "weak" and g.confidence == "low"
|
||||
shutil.rmtree(root); shutil.rmtree(library)
|
||||
|
||||
|
||||
def test_tokenless_needs_bonus():
|
||||
"""Same-faction adoption without a shared word needs +0.08.
|
||||
|
||||
'Aeldari Rangers' (0.667 vs 'Dark Reapers', tokenless) must not adopt
|
||||
Dark Reapers; it instead adopts 'Rangers' (0.636, shares 'ranger').
|
||||
"""
|
||||
root = Path(tempfile.mkdtemp(prefix="sorter_test_src_"))
|
||||
pack = root / "Aeldari Pack"
|
||||
write(pack / "Aeldari Rangers" / "r.stl")
|
||||
library = make_lib()
|
||||
(root / "extract_manifest.jsonl").write_text(json.dumps({
|
||||
"archive": str(root / "Aeldari Pack.zip"),
|
||||
"dest": "Aeldari Pack", "unit": "Aeldari Pack.zip",
|
||||
}), encoding="utf-8")
|
||||
files = sorted((root / "Aeldari Pack").rglob("*"))
|
||||
items = sort_plan(files, library, get_catalog(), provenance={
|
||||
"Aeldari Pack": "Aeldari Pack.zip"}, root=root)
|
||||
it = next(i for i in items if i.name == "r.stl")
|
||||
assert it.match_kind == "catalog" and it.unit == "Rangers", \
|
||||
(it.match_kind, it.unit, it.match_score)
|
||||
shutil.rmtree(root); shutil.rmtree(library)
|
||||
|
||||
|
||||
def test_dropped_system_mordor():
|
||||
"""Middle-earth is dropped: a 'Mordor' pack lands in _unsorted, and
|
||||
its inner dirs cannot adopt the (filtered) Mordor Orcs entry."""
|
||||
root = Path(tempfile.mkdtemp(prefix="sorter_test_src_"))
|
||||
write(root / "Mordor pack" / "boy.stl")
|
||||
library = make_lib()
|
||||
items = sort_plan([root / "Mordor pack" / "boy.stl"], library,
|
||||
get_catalog(), provenance={"Mordor pack": "Mordor pack.zip"},
|
||||
root=root)
|
||||
it = items[0]
|
||||
assert "Middle-Earth" not in str(it.target), it.target
|
||||
assert "_unsorted/Mordor pack" in str(it.target).replace("\\", "/")
|
||||
shutil.rmtree(root); shutil.rmtree(library)
|
||||
|
||||
|
||||
def test_load_catalog_filters_dropped():
|
||||
cat = load_catalog_path()
|
||||
assert not any(e["system"] == "Middle-Earth" for e in cat)
|
||||
assert any(e["name"] == "Warp Spiders" for e in cat)
|
||||
|
||||
|
||||
def test_nonprov_rehome():
|
||||
"""Loose noise file inside a named dir re-homes to the dir's unit."""
|
||||
root = Path(tempfile.mkdtemp(prefix="sorter_test_src_"))
|
||||
write(root / "Warp Spiders" / "body.stl")
|
||||
library = make_lib()
|
||||
files = sorted(root.rglob("*"))
|
||||
items = sort_plan(files, library, get_catalog(), root=root)
|
||||
it = next(i for i in items if i.name == "body.stl")
|
||||
assert it.match_kind == "catalog", (it.match_kind, it.target)
|
||||
assert "Warp Spiders" in str(it.target) and "Fly" in str(it.target)
|
||||
shutil.rmtree(root); shutil.rmtree(library)
|
||||
|
||||
|
||||
def test_apply_weak_doc():
|
||||
"""apply_plan writes the low-confidence review document."""
|
||||
root = Path(tempfile.mkdtemp(prefix="sorter_test_src_"))
|
||||
pack = root / "Multi Pack"
|
||||
write(pack / "guardian squad extras" / "g.stl")
|
||||
library = make_lib()
|
||||
items = prov_items(root, library)
|
||||
linked, copied, skipped = apply_plan(items, library / "sorter_applied.jsonl")
|
||||
doc = library / "sorter_low_confidence.txt"
|
||||
kinds = [i.match_kind for i in items if i.match_kind == "weak"]
|
||||
if kinds:
|
||||
assert doc.exists(), "weak doc missing"
|
||||
assert "low-confidence" in doc.read_text(encoding="utf-8")
|
||||
else:
|
||||
assert not doc.exists()
|
||||
assert (library / "sorter_applied.jsonl").exists()
|
||||
shutil.rmtree(root); shutil.rmtree(library)
|
||||
|
||||
|
||||
def load_catalog_path():
|
||||
import tempfile as tf
|
||||
fh = tf.NamedTemporaryFile(mode="w", suffix=".json", delete=False)
|
||||
json.dump(CATALOG, fh)
|
||||
fh.close()
|
||||
return load_catalog(Path(fh.name))
|
||||
|
||||
|
||||
def main():
|
||||
catalog = load_catalog_path()
|
||||
for fn in (test_prov_dir_rehome, test_prov_weak_tier, test_nonprov_rehome,
|
||||
test_cross_system_needs_token_containment, test_apply_weak_doc,
|
||||
test_weak_needs_shared_token, test_tokenless_needs_bonus,
|
||||
test_dropped_system_mordor, test_load_catalog_filters_dropped):
|
||||
fn()
|
||||
print("OK", fn.__name__)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user