Add subscription mode, global cross-run URL store and --assume-done import
- dlphelper/globalstore.py: ~/.local/share/dlphelper/urls.json records every successfully downloaded URL once, regardless of OUT_DIR - reposts in other channels and re-runs of moved channels no longer re-download - --subs subscriptions.json: batch mode over a configurable channel list; re-exports each channel by default (new links get picked up), --no-fresh reuses cached exports - --assume-done <file>: manual import step that marks old URL lists as done in the global store (deliberate, unverified, never automatic) - lazy downloader startup: no gofile guest account / no curl runs when all URLs are already done; token prompt skipped in --urls-file mode - write_failed_urls creates OUT_DIR if needed
This commit is contained in:
@@ -0,0 +1,70 @@
|
||||
"""Cross-run, cross-channel record of completed URLs.
|
||||
|
||||
This is what prevents re-downloading your collection: state.json only knows
|
||||
about one OUT_DIR, but the same gofile link posted in a second channel would
|
||||
otherwise be downloaded again into that channel's OUT_DIR. The global store
|
||||
(default: ~/.local/share/dlphelper/urls.json) records every successfully
|
||||
downloaded URL once, forever, regardless of OUT_DIR.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def default_global_path() -> Path:
|
||||
data_home = os.environ.get("XDG_DATA_HOME") or str(
|
||||
Path.home() / ".local" / "share"
|
||||
)
|
||||
return Path(data_home) / "dlphelper" / "urls.json"
|
||||
|
||||
|
||||
class GlobalStore:
|
||||
"""JSON file mapping url -> {status, channel, host, files, updated}."""
|
||||
|
||||
def __init__(self, path: Path | None = None):
|
||||
self.path = path
|
||||
self.records: dict[str, dict] = {}
|
||||
if self.path and self.path.exists():
|
||||
try:
|
||||
self.records = json.loads(self.path.read_text(encoding="utf-8"))
|
||||
if not isinstance(self.records, dict):
|
||||
raise ValueError("global store root must be an object")
|
||||
except (ValueError, OSError) as exc:
|
||||
backup = self.path.with_suffix(".json.bak")
|
||||
log.warning("Corrupt global store %s (%s); backed up to %s",
|
||||
self.path, exc, backup)
|
||||
self.path.replace(backup)
|
||||
self.records = {}
|
||||
|
||||
def is_done(self, url: str) -> bool:
|
||||
if not self.records:
|
||||
return False
|
||||
rec = self.records.get(url)
|
||||
return bool(rec) and rec.get("status") == "ok"
|
||||
|
||||
def mark(self, url: str, **fields) -> None:
|
||||
if self.path is None:
|
||||
return
|
||||
rec = self.records.setdefault(url, {"status": "ok"})
|
||||
for key, value in fields.items():
|
||||
if value is not None:
|
||||
rec[key] = value
|
||||
rec["status"] = "ok"
|
||||
rec["updated"] = time.strftime("%Y-%m-%dT%H:%M:%S")
|
||||
self.save()
|
||||
|
||||
def save(self) -> None:
|
||||
if self.path is None:
|
||||
return
|
||||
self.path.parent.mkdir(parents=True, exist_ok=True)
|
||||
tmp = self.path.with_suffix(".json.tmp")
|
||||
tmp.write_text(json.dumps(self.records, indent=2, sort_keys=True),
|
||||
encoding="utf-8")
|
||||
tmp.replace(self.path)
|
||||
Reference in New Issue
Block a user