Initial version: dlp.py orchestrates Discord export, link extraction and host-aware downloads

- dlphelper/discord.py: DCE docker export (JSON) + base64/plain URL extraction (extraction regex kept compatible with parts/grab.sh) - dlphelper/gofile.py: GoFile client adapted from Lysagxra/GoFileDownloader's API scheme, but ONE guest account per run and per-file failure tracking instead of sys.exit - fixes the rate-limit death after ~6 downloads - dlphelper/zero807.py: curl downloader with the UA+Referer contract 0807.st requires; .part files for resumable downloads - dlphelper/state.py: state.json (resume/skip) + manifest.jsonl (torrent stage later) + failed_urls.txt - tools/new_repo.py: Gitea repo helper reading ~/.git-credentials itself - AGENTS.md with agent rules, README, offline extraction test
2026-10-04 14:06:46 +02:00
commit db8a1c1b52
13 changed files with 1389 additions and 0 deletions
@@ -0,0 +1,78 @@
+"""Persistent per-URL state: makes runs resumable and failures trackable.
+
+state.json lives in the run's OUT_DIR and maps every URL seen to a record:
+
+    {
+      "status": "ok" | "failed" | "error",
+      "attempts": <int>,
+      "error": "<last error message>",
+      "files": [{"path": ..., "size": ...}, ...],
+      "updated": "<iso timestamp>"
+    }
+
+Re-running skips URLs whose status is "ok"; failed URLs are retried and also
+written to failed_urls.txt for easy manual retry lists.
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+import time
+from pathlib import Path
+
+log = logging.getLogger(__name__)
+
+
+class State:
+    """JSON-backed state store for one run directory."""
+
+    def __init__(self, path: Path):
+        self.path = path
+        self.records: dict[str, dict] = {}
+        if path.exists():
+            try:
+                self.records = json.loads(path.read_text(encoding="utf-8"))
+                if not isinstance(self.records, dict):
+                    raise ValueError("state root must be an object")
+            except (ValueError, OSError) as exc:
+                backup = path.with_suffix(".json.bak")
+                log.warning("Corrupt state file %s (%s); moving to %s and starting fresh",
+                            path, exc, backup)
+                path.replace(backup)
+                self.records = {}
+
+    def is_done(self, url: str) -> bool:
+        rec = self.records.get(url)
+        return bool(rec) and rec.get("status") == "ok"
+
+    def pending(self, urls: list[str]) -> list[str]:
+        return [u for u in urls if not self.is_done(u)]
+
+    def mark(self, url: str, **fields) -> dict:
+        rec = self.records.setdefault(url, {"status": "pending", "attempts": 0, "files": []})
+        for key, value in fields.items():
+            if value is not None:
+                rec[key] = value
+        rec["attempts"] = rec.get("attempts", 0) + 1
+        rec["updated"] = time.strftime("%Y-%m-%dT%H:%M:%S")
+        self.save()
+        return rec
+
+    def failures(self) -> list[str]:
+        return [u for u, r in self.records.items() if r.get("status") != "ok"]
+
+    def save(self) -> None:
+        self.path.parent.mkdir(parents=True, exist_ok=True)
+        tmp = self.path.with_suffix(".json.tmp")
+        tmp.write_text(
+            json.dumps(self.records, indent=2, sort_keys=True), encoding="utf-8"
+        )
+        tmp.replace(self.path)
+
+
+def append_manifest(manifest_path: Path, row: dict) -> None:
+    """Append one JSON line to manifest.jsonl (per downloaded file)."""
+    manifest_path.parent.mkdir(parents=True, exist_ok=True)
+    with manifest_path.open("a", encoding="utf-8") as fh:
+        fh.write(json.dumps(row, ensure_ascii=False) + "\n")