Initial version: dlp.py orchestrates Discord export, link extraction and host-aware downloads
- dlphelper/discord.py: DCE docker export (JSON) + base64/plain URL extraction (extraction regex kept compatible with parts/grab.sh) - dlphelper/gofile.py: GoFile client adapted from Lysagxra/GoFileDownloader's API scheme, but ONE guest account per run and per-file failure tracking instead of sys.exit - fixes the rate-limit death after ~6 downloads - dlphelper/zero807.py: curl downloader with the UA+Referer contract 0807.st requires; .part files for resumable downloads - dlphelper/state.py: state.json (resume/skip) + manifest.jsonl (torrent stage later) + failed_urls.txt - tools/new_repo.py: Gitea repo helper reading ~/.git-credentials itself - AGENTS.md with agent rules, README, offline extraction test
This commit is contained in:
@@ -0,0 +1,68 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Offline extraction test with a synthetic export.json (no network, no deps).
|
||||
|
||||
Run with: python3 tests/test_extract.py
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import json
|
||||
import sys
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
|
||||
|
||||
from dlphelper.discord import classify, extract_urls # noqa: E402
|
||||
|
||||
|
||||
def b64(s: str) -> str:
|
||||
return base64.b64encode(s.encode()).decode()
|
||||
|
||||
|
||||
def main() -> None:
|
||||
messages = [
|
||||
{
|
||||
"id": "1",
|
||||
"timestamp": "2026-01-01T00:00:00",
|
||||
"author": {"name": "poster"},
|
||||
"content": f"chaos stuff: {b64('https://gofile.io/d/abc123')}",
|
||||
},
|
||||
{
|
||||
"id": "2",
|
||||
"timestamp": "2026-01-02T00:00:00",
|
||||
"author": {"name": "poster"},
|
||||
"content": (
|
||||
f"0807 zip: {b64('https://0807.st/6saCNie.zip')}\n"
|
||||
f"dupe of 1: {b64('https://gofile.io/d/abc123')}\n"
|
||||
f"plain: https://0807.st/plainLink.zip and "
|
||||
f"https://discord.com/channels/1/2 (unsupported)\n"
|
||||
f"junk: {b64('https://pixeldrain.com/u/xyz')} "
|
||||
"and not-our-domain base64: aHR0cHM6Ly9leGFtcGxlLm9yZw=="
|
||||
),
|
||||
},
|
||||
]
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
export = Path(tmp) / "export.json"
|
||||
export.write_text(json.dumps({"messages": messages}), encoding="utf-8")
|
||||
found = extract_urls(export)
|
||||
|
||||
urls = {f.url for f in found}
|
||||
expected = {"https://gofile.io/d/abc123", "https://0807.st/6saCNie.zip",
|
||||
"https://0807.st/plainLink.zip"}
|
||||
assert urls == expected, f"unexpected urls: {sorted(urls)}"
|
||||
|
||||
by_url = {f.url: f for f in found}
|
||||
g = by_url["https://gofile.io/d/abc123"]
|
||||
assert g.host == "gofile" and g.message_id == "1" and g.author == "poster"
|
||||
assert classify("https://0807.st/x.zip") == "0807"
|
||||
assert classify("https://gofile.io/d/x") == "gofile"
|
||||
assert classify("https://example.org/x") is None
|
||||
|
||||
print("test_extract: OK")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user