Ben_Kosytorz
271fb3bf68
- git mv of all existing files into version-python/ (history preserved) - version-rust/: minimal cargo scaffold (edition 2024), no deps yet - root AGENTS.md documents the transition rules and artifact/schema contract - main is untouched; the python layout there stays as-is
79 lines
2.7 KiB
Python
79 lines
2.7 KiB
Python
"""Persistent per-URL state: makes runs resumable and failures trackable.
|
|
|
|
state.json lives in the run's OUT_DIR and maps every URL seen to a record:
|
|
|
|
{
|
|
"status": "ok" | "failed" | "error",
|
|
"attempts": <int>,
|
|
"error": "<last error message>",
|
|
"files": [{"path": ..., "size": ...}, ...],
|
|
"updated": "<iso timestamp>"
|
|
}
|
|
|
|
Re-running skips URLs whose status is "ok"; failed URLs are retried and also
|
|
written to failed_urls.txt for easy manual retry lists.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import logging
|
|
import time
|
|
from pathlib import Path
|
|
|
|
log = logging.getLogger(__name__)
|
|
|
|
|
|
class State:
|
|
"""JSON-backed state store for one run directory."""
|
|
|
|
def __init__(self, path: Path):
|
|
self.path = path
|
|
self.records: dict[str, dict] = {}
|
|
if path.exists():
|
|
try:
|
|
self.records = json.loads(path.read_text(encoding="utf-8"))
|
|
if not isinstance(self.records, dict):
|
|
raise ValueError("state root must be an object")
|
|
except (ValueError, OSError) as exc:
|
|
backup = path.with_suffix(".json.bak")
|
|
log.warning("Corrupt state file %s (%s); moving to %s and starting fresh",
|
|
path, exc, backup)
|
|
path.replace(backup)
|
|
self.records = {}
|
|
|
|
def is_done(self, url: str) -> bool:
|
|
rec = self.records.get(url)
|
|
return bool(rec) and rec.get("status") == "ok"
|
|
|
|
def pending(self, urls: list[str]) -> list[str]:
|
|
return [u for u in urls if not self.is_done(u)]
|
|
|
|
def mark(self, url: str, **fields) -> dict:
|
|
rec = self.records.setdefault(url, {"status": "pending", "attempts": 0, "files": []})
|
|
for key, value in fields.items():
|
|
if value is not None:
|
|
rec[key] = value
|
|
rec["attempts"] = rec.get("attempts", 0) + 1
|
|
rec["updated"] = time.strftime("%Y-%m-%dT%H:%M:%S")
|
|
self.save()
|
|
return rec
|
|
|
|
def failures(self) -> list[str]:
|
|
return [u for u, r in self.records.items() if r.get("status") != "ok"]
|
|
|
|
def save(self) -> None:
|
|
self.path.parent.mkdir(parents=True, exist_ok=True)
|
|
tmp = self.path.with_suffix(".json.tmp")
|
|
tmp.write_text(
|
|
json.dumps(self.records, indent=2, sort_keys=True), encoding="utf-8"
|
|
)
|
|
tmp.replace(self.path)
|
|
|
|
|
|
def append_manifest(manifest_path: Path, row: dict) -> None:
|
|
"""Append one JSON line to manifest.jsonl (per downloaded file)."""
|
|
manifest_path.parent.mkdir(parents=True, exist_ok=True)
|
|
with manifest_path.open("a", encoding="utf-8") as fh:
|
|
fh.write(json.dumps(row, ensure_ascii=False) + "\n")
|