Initial version: dlp.py orchestrates Discord export, link extraction and host-aware downloads
- dlphelper/discord.py: DCE docker export (JSON) + base64/plain URL extraction (extraction regex kept compatible with parts/grab.sh) - dlphelper/gofile.py: GoFile client adapted from Lysagxra/GoFileDownloader's API scheme, but ONE guest account per run and per-file failure tracking instead of sys.exit - fixes the rate-limit death after ~6 downloads - dlphelper/zero807.py: curl downloader with the UA+Referer contract 0807.st requires; .part files for resumable downloads - dlphelper/state.py: state.json (resume/skip) + manifest.jsonl (torrent stage later) + failed_urls.txt - tools/new_repo.py: Gitea repo helper reading ~/.git-credentials itself - AGENTS.md with agent rules, README, offline extraction test
This commit is contained in:
@@ -0,0 +1,3 @@
|
||||
"""Warhammer DLP Helper - collect and organize files shared in Discord channels."""
|
||||
|
||||
__version__ = "0.1.0"
|
||||
@@ -0,0 +1,157 @@
|
||||
"""Export a Discord channel/thread with DiscordChatExporter and extract URLs.
|
||||
|
||||
The channels this tool targets encode their download links as base64 inside
|
||||
message content, and occasionally as plain links. We export the channel as
|
||||
JSON (richer metadata than the HTML export: message id, timestamp, author),
|
||||
decode every base64 candidate, and keep only URLs for supported hosts.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from urllib.parse import urlparse
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
DCE_IMAGE = "tyrrrz/discordchatexporter:stable"
|
||||
|
||||
# The channels encode links as base64 that always starts with this prefix
|
||||
# (https://...); grab.sh established this regex, keep it compatible.
|
||||
B64_RE = re.compile(r"aHR0cHM6Ly[89][A-Za-z0-9+/_-]*")
|
||||
URL_RE = re.compile(r"https://[^\s<>\"'\)\]\}]+")
|
||||
|
||||
SUPPORTED_HOSTS = {
|
||||
"gofile": ("gofile.io", "www.gofile.io"),
|
||||
"0807": ("0807.st", "www.0807.st"),
|
||||
}
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class FoundUrl:
|
||||
"""One download URL found in the channel, with source attribution."""
|
||||
|
||||
url: str
|
||||
host: str # "gofile" | "0807" (set after classification)
|
||||
message_id: str | None = None
|
||||
timestamp: str | None = None
|
||||
author: str | None = None
|
||||
encoded: bool = False # found via base64 decode vs. plain link
|
||||
|
||||
|
||||
def export_channel(channel_id: str, out_dir: Path, token: str, force: bool = False) -> Path:
|
||||
"""Export the channel to out_dir/export.json using the DCE docker image.
|
||||
|
||||
The token is passed to the container via the TOKEN env var (DCE reads it
|
||||
natively) so it never appears on the docker command line.
|
||||
"""
|
||||
dest = out_dir / "export.json"
|
||||
if dest.exists() and not force:
|
||||
log.info("Reusing existing export %s (pass --fresh to re-export)", dest)
|
||||
return dest
|
||||
|
||||
out_dir.mkdir(parents=True, exist_ok=True)
|
||||
env = dict(os.environ)
|
||||
env["TOKEN"] = token
|
||||
cmd = [
|
||||
"docker", "run", "--rm",
|
||||
"--user", f"{os.getuid()}:{os.getgid()}",
|
||||
"-v", f"{out_dir.resolve()}:/out",
|
||||
"-e", "TOKEN",
|
||||
DCE_IMAGE,
|
||||
"export", "-c", channel_id, "-f", "Json", "-o", "/out/export.json",
|
||||
]
|
||||
log.info("Exporting channel %s via DiscordChatExporter ...", channel_id)
|
||||
try:
|
||||
subprocess.run(cmd, env=env, check=True)
|
||||
except FileNotFoundError as exc:
|
||||
raise RuntimeError(
|
||||
"docker not found - the Discord export needs the docker image "
|
||||
f"{DCE_IMAGE}"
|
||||
) from exc
|
||||
except subprocess.CalledProcessError as exc:
|
||||
raise RuntimeError(
|
||||
f"DiscordChatExporter failed with exit code {exc.returncode} "
|
||||
"(check the token and the channel id)"
|
||||
) from exc
|
||||
if not dest.exists():
|
||||
raise RuntimeError("DiscordChatExporter produced no output file")
|
||||
return dest
|
||||
|
||||
|
||||
def classify(url: str) -> str | None:
|
||||
"""Return the host key ('gofile' | '0807') for a URL, else None."""
|
||||
try:
|
||||
hostname = (urlparse(url).hostname or "").lower()
|
||||
except ValueError:
|
||||
return None
|
||||
for key, hosts in SUPPORTED_HOSTS.items():
|
||||
if hostname in hosts:
|
||||
return key
|
||||
return None
|
||||
|
||||
|
||||
def _decode_b64(candidate: str) -> str | None:
|
||||
s = candidate.replace("_", "/").replace("-", "+")
|
||||
s = s.rstrip("=")
|
||||
s += "=" * ((4 - len(s) % 4) % 4)
|
||||
try:
|
||||
decoded = base64.b64decode(s, validate=False).decode("utf-8", "strict")
|
||||
except Exception: # noqa: BLE001 - malformed candidate, just skip it
|
||||
return None
|
||||
# One encoded blob can contain the URL plus padding/newlines
|
||||
stripped = decoded.strip()
|
||||
line = stripped.splitlines()[0] if stripped else ""
|
||||
return line or None
|
||||
|
||||
|
||||
def extract_urls(export_path: Path) -> list[FoundUrl]:
|
||||
"""Extract supported URLs from a DCE JSON export, deduplicated."""
|
||||
data = json.loads(export_path.read_text(encoding="utf-8"))
|
||||
messages = data if isinstance(data, list) else data.get("messages", [])
|
||||
|
||||
found: dict[str, FoundUrl] = {}
|
||||
stats = {"b64": 0, "plain": 0, "unsupported": 0}
|
||||
|
||||
for msg in messages:
|
||||
content = str(msg.get("content") or "")
|
||||
if not content:
|
||||
continue
|
||||
author = (msg.get("author") or {}).get("name")
|
||||
attrib = {
|
||||
"message_id": str(msg.get("id") or "") or None,
|
||||
"timestamp": msg.get("timestamp"),
|
||||
"author": author,
|
||||
}
|
||||
|
||||
for match in B64_RE.finditer(content):
|
||||
stats["b64"] += 1
|
||||
decoded = _decode_b64(match.group(0))
|
||||
if not decoded:
|
||||
continue
|
||||
host = classify(decoded)
|
||||
if host is None:
|
||||
stats["unsupported"] += 1
|
||||
continue
|
||||
found.setdefault(decoded, FoundUrl(url=decoded, host=host, encoded=True, **attrib))
|
||||
|
||||
for match in URL_RE.finditer(content):
|
||||
url = match.group(0).rstrip(".,;)")
|
||||
host = classify(url)
|
||||
if host is None:
|
||||
continue
|
||||
stats["plain"] += 1
|
||||
found.setdefault(url, FoundUrl(url=url, host=host, encoded=False, **attrib))
|
||||
|
||||
log.info(
|
||||
"Extraction: %d base64 candidates, %d plain links, %d unsupported/undecodable, "
|
||||
"%d unique supported URLs",
|
||||
stats["b64"], stats["plain"], stats["unsupported"], len(found),
|
||||
)
|
||||
return sorted(found.values(), key=lambda f: f.url)
|
||||
@@ -0,0 +1,219 @@
|
||||
"""Minimal GoFile API client - one guest account per run.
|
||||
|
||||
The API interaction mirrors Lysagxra/GoFileDownloader (same headers, website
|
||||
token scheme and endpoints), but with two deliberate differences:
|
||||
|
||||
1. A single guest account is created per run (GoFileDownloader creates one per
|
||||
URL, which trips Gofile's rate limit after ~6 requests).
|
||||
2. Failures raise exceptions instead of sys.exit(), so the orchestrator can
|
||||
track them per file/URL and keep going.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import logging
|
||||
import os
|
||||
import time
|
||||
from pathlib import Path
|
||||
from urllib.parse import urlparse
|
||||
|
||||
import requests
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
API = "https://api.gofile.io"
|
||||
USER_AGENT = "Mozilla/5.0"
|
||||
LOCALE = "en-US"
|
||||
TOKEN_SECRET = "12af056dacea0b" # public website-token seed, see GoFileDownloader # noqa: S105
|
||||
TOKEN_WINDOW_SEC = 14400
|
||||
CHUNK_SIZE = 1024 * 1024
|
||||
|
||||
COMMON_HEADERS = {
|
||||
"User-Agent": USER_AGENT,
|
||||
"Accept-Encoding": "gzip",
|
||||
"Accept": "*/*",
|
||||
"Connection": "keep-alive",
|
||||
}
|
||||
EXTENDED_HEADERS = {
|
||||
**COMMON_HEADERS,
|
||||
"Sec-Fetch-Dest": "empty",
|
||||
"Sec-Fetch-Mode": "cors",
|
||||
"Sec-Fetch-Site": "same-site",
|
||||
"Pragma": "no-cache",
|
||||
"Cache-Control": "no-cache",
|
||||
}
|
||||
|
||||
|
||||
class GoFileError(Exception):
|
||||
"""Base error for GoFile operations."""
|
||||
|
||||
|
||||
class GoFilePasswordRequired(GoFileError):
|
||||
"""The content is password protected and none (or a wrong one) was given."""
|
||||
|
||||
|
||||
class GoFileAuthError(GoFileError):
|
||||
"""The account token was rejected; the client can re-auth once."""
|
||||
|
||||
|
||||
def content_id_from_url(url: str) -> str:
|
||||
"""Extract the content id from a gofile.io/d/<id> URL."""
|
||||
parts = [p for p in urlparse(url).path.split("/") if p]
|
||||
if len(parts) >= 2 and parts[-2] == "d":
|
||||
return parts[-1]
|
||||
raise GoFileError(f"not a gofile content URL (expected gofile.io/d/<id>): {url}")
|
||||
|
||||
|
||||
class GoFileClient:
|
||||
"""GoFile API client that reuses one guest account token for the whole run."""
|
||||
|
||||
def __init__(self, password: str | None = None):
|
||||
self._password_hash = (
|
||||
hashlib.sha256(password.encode()).hexdigest() if password else None
|
||||
)
|
||||
self._token: str | None = None
|
||||
self.session = requests.Session()
|
||||
|
||||
# -- authentication -----------------------------------------------------
|
||||
|
||||
def ensure_token(self, max_attempts: int = 8, wait_seconds: int = 20) -> str:
|
||||
"""Create a guest account; backs off and retries on rate limiting."""
|
||||
if self._token:
|
||||
return self._token
|
||||
|
||||
last_status = "unknown"
|
||||
for attempt in range(1, max_attempts + 1):
|
||||
try:
|
||||
resp = self.session.post(
|
||||
f"{API}/accounts", headers=COMMON_HEADERS, timeout=15
|
||||
)
|
||||
data = resp.json()
|
||||
except (ValueError, requests.RequestException) as exc:
|
||||
last_status = f"request error: {exc}"
|
||||
log.warning(
|
||||
"GoFile account creation attempt %d/%d failed: %s",
|
||||
attempt, max_attempts, last_status,
|
||||
)
|
||||
else:
|
||||
if isinstance(data, dict) and data.get("status") == "ok":
|
||||
self._token = data["data"]["token"]
|
||||
log.info("GoFile guest account created (token reused for whole run)")
|
||||
return self._token
|
||||
last_status = str(data.get("status", data)[:200] if data else data)
|
||||
log.warning(
|
||||
"GoFile account creation attempt %d/%d failed: %s",
|
||||
attempt, max_attempts, last_status,
|
||||
)
|
||||
if attempt < max_attempts:
|
||||
time.sleep(wait_seconds * attempt)
|
||||
raise GoFileError(
|
||||
f"GoFile account creation failed after {max_attempts} attempts "
|
||||
f"(last: {last_status})"
|
||||
)
|
||||
|
||||
def reset(self) -> None:
|
||||
"""Drop the current token so the next ensure_token() creates a new one."""
|
||||
self._token = None
|
||||
|
||||
def _website_token(self) -> str:
|
||||
window = int(time()) // TOKEN_WINDOW_SEC
|
||||
seed = f"{USER_AGENT}::{LOCALE}::{self._token}::{window}::{TOKEN_SECRET}"
|
||||
return hashlib.sha256(seed.encode()).hexdigest()
|
||||
|
||||
def _auth_headers(self) -> dict:
|
||||
return {
|
||||
**EXTENDED_HEADERS,
|
||||
"Authorization": f"Bearer {self._token}",
|
||||
"X-Website-Token": self._website_token(),
|
||||
"X-BL": LOCALE,
|
||||
}
|
||||
|
||||
# -- listing ------------------------------------------------------------
|
||||
|
||||
def list_files(self, url: str) -> list[tuple[str, str, str]]:
|
||||
"""Return (relpath, filename, download_link) for every file under a URL."""
|
||||
cid = content_id_from_url(url)
|
||||
results: list[tuple[str, str, str]] = []
|
||||
self._walk(cid, [], results)
|
||||
# Direct files at the content root get their own folder named after the id
|
||||
normalized = [
|
||||
(relpath if relpath else cid, name, link) for relpath, name, link in results
|
||||
]
|
||||
return normalized
|
||||
|
||||
def _walk(self, cid: str, prefix: list[str], results: list[tuple[str, str, str]]) -> None:
|
||||
query = "?cache=true&sortField=createTime&sortDirection=1"
|
||||
if self._password_hash:
|
||||
query += f"&password={self._password_hash}"
|
||||
try:
|
||||
resp = self.session.get(
|
||||
f"{API}/contents/{cid}{query}", headers=self._auth_headers(), timeout=30
|
||||
)
|
||||
except requests.RequestException as exc:
|
||||
raise GoFileError(f"content request for {cid} failed: {exc}") from exc
|
||||
|
||||
if resp.status_code in (401, 403):
|
||||
raise GoFileAuthError(f"content request denied ({resp.status_code}) for {cid}")
|
||||
try:
|
||||
data = resp.json()
|
||||
except ValueError as exc:
|
||||
raise GoFileError(f"non-JSON response for {cid}: {resp.text[:200]}") from exc
|
||||
|
||||
if data.get("status") != "ok":
|
||||
raise GoFileError(f"content request failed for {cid}: {data.get('status')}")
|
||||
|
||||
d = data["data"]
|
||||
if d.get("passwordStatus") not in (None, "passwordOk"):
|
||||
raise GoFilePasswordRequired(
|
||||
f"content {cid} is password protected (use --password)"
|
||||
)
|
||||
|
||||
if d.get("type") == "folder":
|
||||
here = prefix + [d["name"]]
|
||||
for child in (d.get("children") or {}).values():
|
||||
if child.get("type") == "folder":
|
||||
self._walk(child["id"], here, results)
|
||||
else:
|
||||
results.append(("/".join(here), child["name"], child["link"]))
|
||||
else:
|
||||
results.append(("/".join(prefix), d["name"], d["link"]))
|
||||
|
||||
# -- downloading ----------------------------------------------------------
|
||||
|
||||
def download_file(self, link: str, dest: Path) -> int:
|
||||
"""Download one file; returns the file size. Skips existing files."""
|
||||
dest.parent.mkdir(parents=True, exist_ok=True)
|
||||
if dest.exists() and dest.stat().st_size > 0:
|
||||
log.info("Skip existing: %s", dest)
|
||||
return dest.stat().st_size
|
||||
|
||||
tmp = dest.with_name(dest.name + ".part")
|
||||
headers = {
|
||||
**EXTENDED_HEADERS,
|
||||
"Cookie": f"accountToken={self._token}",
|
||||
"Referer": link,
|
||||
"Origin": link,
|
||||
}
|
||||
try:
|
||||
with self.session.get(
|
||||
link, headers=headers, stream=True, timeout=(10, 60), allow_redirects=True
|
||||
) as resp:
|
||||
if resp.status_code != 200:
|
||||
raise GoFileError(f"HTTP {resp.status_code} while downloading {dest.name}")
|
||||
total = int(resp.headers.get("Content-Length") or 0)
|
||||
written = 0
|
||||
with tmp.open("wb") as fh:
|
||||
for chunk in resp.iter_content(chunk_size=CHUNK_SIZE):
|
||||
fh.write(chunk)
|
||||
written += len(chunk)
|
||||
if written == 0:
|
||||
raise GoFileError(f"empty body for {dest.name}")
|
||||
os.replace(tmp, dest)
|
||||
return written
|
||||
finally:
|
||||
if tmp.exists() and not dest.exists():
|
||||
try:
|
||||
tmp.unlink()
|
||||
except OSError:
|
||||
pass
|
||||
@@ -0,0 +1,78 @@
|
||||
"""Persistent per-URL state: makes runs resumable and failures trackable.
|
||||
|
||||
state.json lives in the run's OUT_DIR and maps every URL seen to a record:
|
||||
|
||||
{
|
||||
"status": "ok" | "failed" | "error",
|
||||
"attempts": <int>,
|
||||
"error": "<last error message>",
|
||||
"files": [{"path": ..., "size": ...}, ...],
|
||||
"updated": "<iso timestamp>"
|
||||
}
|
||||
|
||||
Re-running skips URLs whose status is "ok"; failed URLs are retried and also
|
||||
written to failed_urls.txt for easy manual retry lists.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class State:
|
||||
"""JSON-backed state store for one run directory."""
|
||||
|
||||
def __init__(self, path: Path):
|
||||
self.path = path
|
||||
self.records: dict[str, dict] = {}
|
||||
if path.exists():
|
||||
try:
|
||||
self.records = json.loads(path.read_text(encoding="utf-8"))
|
||||
if not isinstance(self.records, dict):
|
||||
raise ValueError("state root must be an object")
|
||||
except (ValueError, OSError) as exc:
|
||||
backup = path.with_suffix(".json.bak")
|
||||
log.warning("Corrupt state file %s (%s); moving to %s and starting fresh",
|
||||
path, exc, backup)
|
||||
path.replace(backup)
|
||||
self.records = {}
|
||||
|
||||
def is_done(self, url: str) -> bool:
|
||||
rec = self.records.get(url)
|
||||
return bool(rec) and rec.get("status") == "ok"
|
||||
|
||||
def pending(self, urls: list[str]) -> list[str]:
|
||||
return [u for u in urls if not self.is_done(u)]
|
||||
|
||||
def mark(self, url: str, **fields) -> dict:
|
||||
rec = self.records.setdefault(url, {"status": "pending", "attempts": 0, "files": []})
|
||||
for key, value in fields.items():
|
||||
if value is not None:
|
||||
rec[key] = value
|
||||
rec["attempts"] = rec.get("attempts", 0) + 1
|
||||
rec["updated"] = time.strftime("%Y-%m-%dT%H:%M:%S")
|
||||
self.save()
|
||||
return rec
|
||||
|
||||
def failures(self) -> list[str]:
|
||||
return [u for u, r in self.records.items() if r.get("status") != "ok"]
|
||||
|
||||
def save(self) -> None:
|
||||
self.path.parent.mkdir(parents=True, exist_ok=True)
|
||||
tmp = self.path.with_suffix(".json.tmp")
|
||||
tmp.write_text(
|
||||
json.dumps(self.records, indent=2, sort_keys=True), encoding="utf-8"
|
||||
)
|
||||
tmp.replace(self.path)
|
||||
|
||||
|
||||
def append_manifest(manifest_path: Path, row: dict) -> None:
|
||||
"""Append one JSON line to manifest.jsonl (per downloaded file)."""
|
||||
manifest_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
with manifest_path.open("a", encoding="utf-8") as fh:
|
||||
fh.write(json.dumps(row, ensure_ascii=False) + "\n")
|
||||
@@ -0,0 +1,83 @@
|
||||
"""Downloader for 0807.st direct links via curl.
|
||||
|
||||
0807.st serves the files directly but rejects requests that don't look like a
|
||||
browser (requires a User-Agent and the Referer of the site page), and it
|
||||
rejects multi-connection/range abuse patterns differently than curl - which is
|
||||
why curl is used instead of aria2c here (see chat history: curl worked,
|
||||
aria2c's default request got 403).
|
||||
|
||||
Downloads go to a .part file first and are renamed on success, so an
|
||||
interrupted download is resumable (--continue-at -) and partial files are
|
||||
never mistaken for finished ones.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import os
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
from urllib.parse import urlparse
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
USER_AGENT = "Mozilla/5.0"
|
||||
REFERER = "https://0807.st/"
|
||||
|
||||
|
||||
class Zero807Error(Exception):
|
||||
"""Base error for 0807 downloads."""
|
||||
|
||||
|
||||
def target_name(url: str) -> str:
|
||||
"""Derive the filename from the URL path (e.g. /6saCNie.zip -> 6saCNie.zip)."""
|
||||
name = Path(urlparse(url).path).name
|
||||
if not name:
|
||||
raise Zero807Error(f"cannot derive a filename from URL: {url}")
|
||||
return name
|
||||
|
||||
|
||||
def download(
|
||||
url: str,
|
||||
out_dir: Path,
|
||||
timeout_seconds: int = 7200,
|
||||
resume: bool = True,
|
||||
) -> tuple[Path, int]:
|
||||
"""Download one URL with curl; returns (final_path, size_bytes)."""
|
||||
name = target_name(url)
|
||||
out_dir.mkdir(parents=True, exist_ok=True)
|
||||
dest = out_dir / name
|
||||
tmp = dest.with_name(dest.name + ".part")
|
||||
|
||||
if dest.exists() and dest.stat().st_size > 0:
|
||||
log.info("Skip existing: %s", dest)
|
||||
return dest, dest.stat().st_size
|
||||
|
||||
cmd = [
|
||||
"curl", "-fSL",
|
||||
"--connect-timeout", "20",
|
||||
"--max-time", str(timeout_seconds),
|
||||
"--retry", "2", "--retry-delay", "5",
|
||||
"-A", USER_AGENT,
|
||||
"-e", REFERER,
|
||||
"-o", str(tmp),
|
||||
]
|
||||
if resume:
|
||||
cmd.append("--continue-at")
|
||||
cmd.append("-")
|
||||
cmd.append(url)
|
||||
|
||||
log.info("Downloading %s -> %s", url, dest.name)
|
||||
proc = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
|
||||
if proc.returncode != 0:
|
||||
tail = [line for line in proc.stderr.strip().splitlines() if line][-3:]
|
||||
raise Zero807Error(f"curl exit code {proc.returncode}: {' | '.join(tail)}")
|
||||
|
||||
size = tmp.stat().st_size if tmp.exists() else 0
|
||||
if size == 0:
|
||||
tmp.unlink(missing_ok=True)
|
||||
raise Zero807Error(f"empty file for {name}")
|
||||
if dest.suffix.lower() == ".zip" and not tmp.read_bytes()[:2] == b"PK":
|
||||
log.warning("%s does not look like a ZIP file (first bytes are not PK)", name)
|
||||
os.replace(tmp, dest)
|
||||
return dest, size
|
||||
Reference in New Issue
Block a user