From 72ef7bc465aaea31815fa4a80bf24d5dddeb2a4f Mon Sep 17 00:00:00 2001 From: Ben_Kosytorz Date: Sun, 4 Oct 2026 17:19:11 +0200 Subject: [PATCH] Add subscription mode, global cross-run URL store and --assume-done import - dlphelper/globalstore.py: ~/.local/share/dlphelper/urls.json records every successfully downloaded URL once, regardless of OUT_DIR - reposts in other channels and re-runs of moved channels no longer re-download - --subs subscriptions.json: batch mode over a configurable channel list; re-exports each channel by default (new links get picked up), --no-fresh reuses cached exports - --assume-done : manual import step that marks old URL lists as done in the global store (deliberate, unverified, never automatic) - lazy downloader startup: no gofile guest account / no curl runs when all URLs are already done; token prompt skipped in --urls-file mode - write_failed_urls creates OUT_DIR if needed --- AGENTS.md | 61 +++++--- README.md | 30 ++++ dlp.py | 301 +++++++++++++++++++++++++++------------ dlphelper/globalstore.py | 70 +++++++++ 4 files changed, 357 insertions(+), 105 deletions(-) create mode 100644 dlphelper/globalstore.py diff --git a/AGENTS.md b/AGENTS.md index 1191fce..d5ba075 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -47,8 +47,10 @@ dlphelper/ gofile.py GoFile API client (one guest account per run) zero807.py curl-based downloader for 0807.st direct links state.py state.json store + manifest.jsonl appender + globalstore.py cross-run URL store (~/.local/share/dlphelper/urls.json) parts/grab.sh legacy groundstone script (kept, not used by dlp.py) tools/new_repo.py Gitea repo helper (reads ~/.git-credentials itself) +tests/test_extract.py offline extraction test (plain python3, no deps) ``` Data flow: @@ -64,6 +66,7 @@ graph LR E --> H[downloads/gofile/...] F --> I[downloads/0807/...] E & F --> J[state.json + manifest.jsonl + failed_urls.txt] + J --> K[globalstore: urls.json, marks URL done forever] ``` Key facts about the hosters (learned the hard way, do not re-learn): @@ -89,6 +92,20 @@ Key facts about the hosters (learned the hard way, do not re-learn): Retry only failures: `python3 dlp.py --urls-file /failed_urls.txt` (or just re-run — state.json skips everything already marked ok). +Several channels at once (`subscriptions.json`: list of {name, channel_id, +out_dir}): + + python3 dlp.py --subs subscriptions.json + +Subs mode re-exports every channel by default (that is how new links get +picked up) and relies on state.json + the global store to only download +what is actually new. Use a cron/systemd timer for watch-like behavior. + +Manual import of the pre-dlp.py era (URLs you know are fully downloaded, +never re-fetch them again): + + python3 dlp.py --assume-done old_urls.txt + ## 5. Run artifacts (all in OUT_DIR, gitignored) | File | Purpose | @@ -98,6 +115,7 @@ Retry only failures: `python3 dlp.py --urls-file /failed_urls.tx | `failed_urls.txt` | rewritten each run: every non-ok URL, usable as `--urls-file` | | `manifest.jsonl` | append-only, one JSON line per downloaded file (channel, url, message_id, timestamp, author, path, size, optional sha256) | | `downloads/` | actual content; `.part` files mean "interrupted, will be resumed" | +| `~/.local/share/dlphelper/urls.json` | **global** cross-run store: every successfully downloaded URL is recorded once, regardless of OUT_DIR, so reposts in other channels are not re-downloaded. `--global-state` moves it, `--no-global` disables it for a run. | ## 6. Rules @@ -122,51 +140,60 @@ Retry only failures: `python3 dlp.py --urls-file /failed_urls.tx Password-required gofile links are not retried (it can't help). ### Idempotency -8. Re-runs must be safe and cheap: skip URLs marked `ok`, skip existing - non-empty files, write partial downloads to `*.part` and rename on success. +8. Re-runs must be safe and cheap: skip URLs marked `ok` (in `state.json` **or** + the global store), skip existing non-empty files, write partial downloads + to `*.part` and rename on success. 9. `state.json` is the source of truth for resume; keep its schema backward compatible when touching `state.py` (unknown keys are preserved, missing - keys defaulted). + keys defaulted). Same for the global store schema in `globalstore.py`. +10. `--assume-done` is a deliberate, manual import step for old URL lists. + Never invoke it automatically, and never let the tool mark URLs done + without a verified successful download. ### Rate limiting / network etiquette -10. One gofile guest account per run; `ensure_token()` backs off linearly +11. One gofile guest account per run; `ensure_token()` backs off linearly (20s × attempt). Do not raise the guest-account frequency "to make tests pass". Testing account creation more than a couple times per hour is a bug, not a test strategy. -11. Keep the 0807 referer/UA contract. Don't "optimize" it to aria2c without +12. Keep the 0807 referer/UA contract. Don't "optimize" it to aria2c without re-verifying the 403 behavior. ### Code style -12. Python 3.12+, type hints on public functions, `logging` not `print` in +13. Python 3.12+, type hints on public functions, `logging` not `print` in library code (prints are fine only for `--dry-run` output). -13. Stay on stdlib + `requests`. New dependencies need a real justification +14. Stay on stdlib + `requests`. New dependencies need a real justification and belong in `requirements.txt`. -14. `dlphelper/` modules must stay import-safe (no side effects at import). +15. `dlphelper/` modules must stay import-safe (no side effects at import). ### Validation -15. Minimum validation for any change: `python3 -m py_compile dlp.py +16. Minimum validation for any change: `python3 -m py_compile dlp.py dlphelper/*.py`, plus the synthetic extraction test (below) if `discord.py` changed. Document what you could *not* test (e.g. live gofile/0807 downloads need real links and tokens). -16. Full live runs need a Discord token from the user — ask for it, never +17. Full live runs need a Discord token from the user — ask for it, never ask for it to be pasted into a file or committed. ### Git -17. Branch `main`, descriptive commit messages, no force-push, no history +18. Branch `main`, descriptive commit messages, no force-push, no history rewriting on the shared remote. -18. When creating a new repo use `tools/new_repo.py` (Gitea API); creation is +19. When creating a new repo use `tools/new_repo.py` (Gitea API); creation is fine, deletion/renaming of existing repos requires the user. ## 7. Roadmap (future stages, keep compatible) 1. **Dedupe**: hash-based (sha256) duplicate detection across threads via - `manifest.jsonl`. + `manifest.jsonl` (the `--hash` flag feeds this). URL-level dedupe already + exists via the global store; content-level (same file, different link) + comes with hashing. 2. **Torrent stage**: `mktorrent` per downloaded folder, .torrent files into a watch dir, cross-seeding between direct-download and torrent copies. -3. **Library organization**: naming conventions per game system / faction so - the collection is browsable like a media library. -4. **Watch mode**: periodic re-export of subscribed channels, notify on new - links. +3. **Library organization**: the long-term goal is folders per game system, + faction and army-list class (fast attack, heavy support, ...) as described + in the wiki. Per-file metadata (source message, timestamp, url) is already + captured in `manifest.jsonl` so this can happen later without re-downloads. +4. **Watch mode**: `--subs subscriptions.json` + a cron/timer already gives a + primitive version; a real watch mode (per-subscription schedules, tear-off + notifications, discord pings on new links) builds on it. ## 8. Testing notes diff --git a/README.md b/README.md index 07af757..17148f6 100644 --- a/README.md +++ b/README.md @@ -31,6 +31,34 @@ hidden). Useful options: --dry-run just show what would be downloaded --fresh force a fresh channel export +## Several channels / watching for updates + +Put your channels into `subscriptions.json`: + +```json +{ + "subscriptions": [ + {"name": "some-eldar", + "channel_id": "1455993943471820842", + "out_dir": "/big/storage/warhammer/some-eldar"} + ] +} +``` + +then run (e.g. from a daily cron/systemd timer): + + python3 dlp.py --subs subscriptions.json + +Subs mode re-exports each channel to pick up new links; `state.json` and the +global store make sure only genuinely new URLs are downloaded. + +## Already downloaded the old way? + +`--assume-done` imports a plain URL list into the global store so those links +are never fetched again (no verification — use deliberately): + + python3 dlp.py --assume-done old_urls.txt + ## Output layout OUT_DIR/ @@ -41,6 +69,8 @@ hidden). Useful options: downloads/ gofile/// 0807/.zip +| `downloads/` | actual content; `.part` files mean "interrupted, will be resumed" | +| `~/.local/share/dlphelper/urls.json` | global cross-run store of finished URLs (never re-downloaded, in any channel) | Runs are resumable and idempotent: interrupted downloads use `.part` files and finished ones are skipped on the next run. Individual failures never stop the diff --git a/dlp.py b/dlp.py index 75f7157..9c6b450 100644 --- a/dlp.py +++ b/dlp.py @@ -7,8 +7,16 @@ the right downloader for each link, and keeps going when individual links fail. Everything is recorded in state.json; failures also land in failed_urls.txt so a re-run (or --urls-file) retries exactly those. +Already-downloaded URLs are recorded in a global store (default +~/.local/share/dlphelper/urls.json), so a link that was downloaded for one +channel is never re-downloaded for another - and re-runs of the same channel +skip everything finished (see also --assume-done for importing old lists). + Usage: python3 dlp.py [options] + python3 dlp.py --subs subscriptions.json [options] + python3 dlp.py --urls-file failed_urls.txt + python3 dlp.py --assume-done old_urls.txt # import past downloads """ from __future__ import annotations @@ -27,6 +35,7 @@ from pathlib import Path from dlphelper import discord as dlp_discord from dlphelper import gofile as dlph_gofile from dlphelper import zero807 as dlph_zero807 +from dlphelper.globalstore import GlobalStore, default_global_path from dlphelper.state import State, append_manifest log = logging.getLogger("dlp") @@ -52,13 +61,18 @@ def resolve_token(cli_value: str | None) -> str: return token +def read_url_lines(path: Path) -> list[str]: + return [ + line.strip() + for line in path.read_text(encoding="utf-8").splitlines() + if line.strip() and not line.strip().startswith("#") + ] + + def load_urls_file(path: Path) -> list[dlp_discord.FoundUrl]: """Load a plain URL list (e.g. failed_urls.txt); no attribution available.""" found: dict[str, dlp_discord.FoundUrl] = {} - for line in path.read_text(encoding="utf-8").splitlines(): - url = line.strip() - if not url or url.startswith("#"): - continue + for url in read_url_lines(path): host = dlp_discord.classify(url) if host is None: log.warning("Unsupported host in %s: %s", path.name, url) @@ -67,13 +81,22 @@ def load_urls_file(path: Path) -> list[dlp_discord.FoundUrl]: return sorted(found.values(), key=lambda f: f.url) -def record_files(state: State, url: str, files: list[dict], source: dict) -> None: +def record_files( + state: State, + gstore: GlobalStore | None, + url: str, + files: list[dict], + source: dict, +) -> None: state.mark(url, status="ok", error=None, files=files) + if gstore is not None: + gstore.mark(url, channel=source.get("channel"), host=source.get("host"), + file_count=len(files)) for f in files: append_manifest(Path(source["manifest"]), { "channel": source["channel"], "url": url, - "host": source["host"], + "host": source.get("host"), "message_id": source.get("message_id"), "timestamp": source.get("timestamp"), "author": source.get("author"), @@ -105,22 +128,26 @@ def run_gofile( args: argparse.Namespace, state: State, source_by_url: dict[str, dict], + is_done, + gstore: GlobalStore | None, ) -> None: + if not [f for f in urls if not is_done(f.url)]: + return # nothing pending: no guest account creation, no API calls client = dlph_gofile.GoFileClient(password=args.password) try: client.ensure_token() except dlph_gofile.GoFileError as exc: log.error("%s - marking all pending gofile links as failed.", exc) for f in urls: - if state.is_done(f.url): + if is_done(f.url): continue - state.mark(f.url, status="failed", error=str(exc), - files=[]) + state.mark(f.url, status="failed", error=str(exc), files=[]) return consecutive_failures = 0 for f in urls: - if state.is_done(f.url): + if is_done(f.url): + log.info("Skip (already downloaded): %s", f.url) continue source = source_by_url[f.url] @@ -138,7 +165,7 @@ def run_gofile( entry["sha256"] = sha256_of(dest) files.append(entry) log.info("OK %s (%d files)", f.url, len(files)) - record_files(state, f.url, files, source) + record_files(state, gstore, f.url, files, source) if args.extract: extract_zips(files) consecutive_failures = 0 @@ -192,11 +219,16 @@ def run_0807( args: argparse.Namespace, state: State, source_by_url: dict[str, dict], + is_done, + gstore: GlobalStore | None, ) -> None: + if not [f for f in urls if not is_done(f.url)]: + return out_dir = Path(args.out_dir) / "downloads" / "0807" consecutive_failures = 0 for f in urls: - if state.is_done(f.url): + if is_done(f.url): + log.info("Skip (already downloaded): %s", f.url) continue source = source_by_url[f.url] @@ -207,7 +239,7 @@ def run_0807( if args.hash: entry["sha256"] = sha256_of(dest) log.info("OK %s (%s)", f.url, format_size(size)) - record_files(state, f.url, [entry], source) + record_files(state, gstore, f.url, [entry], source) if args.extract: extract_zips([entry]) consecutive_failures = 0 @@ -251,17 +283,141 @@ def format_size(n: int) -> str: def write_failed_urls(out_dir: Path, state: State) -> Path: dest = out_dir / "failed_urls.txt" + dest.parent.mkdir(parents=True, exist_ok=True) failures = state.failures() dest.write_text("\n".join(failures) + ("\n" if failures else ""), encoding="utf-8") return dest +def mark_assume_done(path: Path, gstore: GlobalStore | None) -> int: + """Record URLs as already downloaded (no files) so they are never fetched. + + Manual import step for the pre-dlp.py era: point it at an old URL list you + know is fully downloaded. There is no verification - garbage in, skipped + forever. + """ + if gstore is None: + raise SystemExit("--assume-done needs the global store (drop --no-global)") + count = 0 + for url in read_url_lines(path): + gstore.mark(url, channel=None, host=dlp_discord.classify(url), file_count=0, + note="assumed done via --assume-done") + count += 1 + log.info("Marked %d URLs as done in the global store (%s)", count, gstore.path) + return count + + +def process_channel( + channel_id: str, + out_dir: Path, + args: argparse.Namespace, + token: str | None, + gstore: GlobalStore | None, + force_export: bool = False, +) -> int: + """Run the full pipeline for one channel; returns the number of failures.""" + state = State(out_dir / "state.json") + manifest = out_dir / "manifest.jsonl" + + def is_done(url: str) -> bool: + return state.is_done(url) or (gstore is not None and gstore.is_done(url)) + + if args.urls_file: + found = load_urls_file(args.urls_file) + else: + if token is None: + raise SystemExit("internal error: no token resolved") + export_path = dlp_discord.export_channel( + channel_id, out_dir, token, force=force_export or args.fresh + ) + found = dlp_discord.extract_urls(export_path) + + if args.only: + found = [f for f in found if f.host == args.only] + + buckets: dict[str, list[dlp_discord.FoundUrl]] = {"gofile": [], "0807": []} + for f in found: + if f.host in buckets: + buckets[f.host].append(f) + else: # classify() already filtered, but be defensive + log.warning("Skipping unsupported URL: %s", f.url) + state.mark(f.url, status="failed", error="unsupported host") + + source_by_url: dict[str, dict] = { + f.url: { + "channel": channel_id, + "host": f.host, + "message_id": f.message_id, + "timestamp": f.timestamp, + "author": f.author, + "manifest": str(manifest), + } + for f in found + } + + pending = [f for f in found if not is_done(f.url)] + log.info( + "%s: %d URLs (%d gofile, %d 0807); %d done, %d to download", + channel_id, len(found), + len(buckets["gofile"]), len(buckets["0807"]), + len(found) - len(pending), len(pending), + ) + + if args.dry_run: + for f in found: + flag = "done" if is_done(f.url) else "download" + print(f"[{f.host:6}] [{flag:8}] {f.url}") + return 0 + + if buckets["gofile"]: + run_gofile(buckets["gofile"], args, state, source_by_url, is_done, gstore) + if buckets["0807"]: + run_0807(buckets["0807"], args, state, source_by_url, is_done, gstore) + + failed_file = write_failed_urls(out_dir, state) + + ok = [u for u, r in state.records.items() if r.get("status") == "ok"] + bad = state.failures() + total_files = sum(len(r.get("files") or []) for r in state.records.values()) + total_size = sum( + (f_.get("size") or 0) for r in state.records.values() for f_ in (r.get("files") or []) + ) + log.info("%s: %d ok (%d files, %s) / %d failed / %d seen", + out_dir, len(ok), total_files, format_size(total_size), + len(bad), len(state.records)) + if bad: + log.info("Failed URLs written to %s - re-run this tool or use " + "python3 dlp.py --urls-file %s to retry them.", + failed_file, failed_file.name) + return len(bad) + + +def load_subscriptions(path: Path) -> list[dict]: + """Load subscriptions.json: [{"name": ..., "channel_id": ..., "out_dir": ...}].""" + data = json.loads(path.read_text(encoding="utf-8")) + entries = data.get("subscriptions", []) if isinstance(data, dict) else data + if not isinstance(entries, list): + raise SystemExit(f"{path}: expected a list or {'{'}\"subscriptions\": [...]{'}'}") + subs = [] + for entry in entries: + if not isinstance(entry, dict) or "channel_id" not in entry or "out_dir" not in entry: + raise SystemExit( + f"{path}: each subscription needs 'channel_id' and 'out_dir' " + f"(got: {entry})" + ) + subs.append(entry) + return subs + + def main() -> None: parser = argparse.ArgumentParser( - description="Collect downloads from a Discord channel (gofile + 0807)." + description="Collect downloads from Discord channels (gofile + 0807)." ) - parser.add_argument("channel_id", help="Discord channel or thread id") - parser.add_argument("out_dir", help="directory for downloads and state") + parser.add_argument("channel_id", nargs="?", help="Discord channel or thread id") + parser.add_argument("out_dir", nargs="?", help="directory for downloads and state") + parser.add_argument("--subs", type=Path, metavar="FILE", + help="process a subscription list instead of one channel " + "(entries: channel_id, out_dir, optional name)") parser.add_argument("--token", help="Discord token (else $DISCORD_TOKEN or prompt)") parser.add_argument("--urls-file", type=Path, help="skip the Discord export; read URLs from this file " @@ -281,9 +437,19 @@ def main() -> None: parser.add_argument("--extract", action="store_true", help="unzip downloaded archives into per-archive folders") parser.add_argument("--fresh", action="store_true", - help="re-export the Discord channel even if export.json exists") + help="re-export the Discord channel even if export.json exists " + "(subs mode does this by default)") + parser.add_argument("--no-fresh", action="store_true", + help="subs mode: reuse cached exports instead of re-exporting") parser.add_argument("--dry-run", action="store_true", help="export/extract/classify only, download nothing") + parser.add_argument("--global-state", type=Path, metavar="FILE", + help=f"cross-run URL store (default: {default_global_path()})") + parser.add_argument("--no-global", action="store_true", + help="disable the cross-run URL store for this run") + parser.add_argument("--assume-done", type=Path, metavar="FILE", + help="mark every URL in the file as already downloaded, " + "then exit (manual import of old download lists)") args = parser.parse_args() logging.basicConfig( @@ -292,82 +458,41 @@ def main() -> None: datefmt="%H:%M:%S", ) - out_dir = Path(args.out_dir) - state = State(out_dir / "state.json") + gstore = None + if not args.no_global: + gstore = GlobalStore(args.global_state or default_global_path()) - channel = args.channel_id - if args.urls_file: - found = load_urls_file(args.urls_file) - else: - token = resolve_token(args.token) - export_path = dlp_discord.export_channel( - channel, out_dir, token, force=args.fresh - ) - found = dlp_discord.extract_urls(export_path) - - if args.only: - found = [f for f in found if f.host == args.only] - - buckets: dict[str, list[dlp_discord.FoundUrl]] = {"gofile": [], "0807": []} - for f in found: - if f.host in buckets: - buckets[f.host].append(f) - else: # classify() already filtered, but be defensive - state.mark(f.url, status="failed", error=f"unsupported host: {f.url}") - - unsupported = [ - f for f in found if f.host not in buckets - ] - for f in unsupported: - log.warning("Skipping unsupported URL: %s", f.url) - - manifest = out_dir / "manifest.jsonl" - source_by_url: dict[str, dict] = { - f.url: { - "channel": channel, - "host": f.host, - "message_id": f.message_id, - "timestamp": f.timestamp, - "author": f.author, - "manifest": str(manifest), - } - for f in found - } - - log.info( - "Found %d URLs (%d gofile, %d 0807); %d already done from earlier runs", - len(found), - len(buckets["gofile"]), - len(buckets["0807"]), - len(found) - len(state.pending([f.url for f in found])), - ) - - if args.dry_run: - for f in found: - print(f"[{f.host:6}] {f.url}") + if args.assume_done: + if args.channel_id or args.subs: + parser.error("--assume-done cannot be combined with a channel or --subs") + mark_assume_done(args.assume_done, gstore) return - if buckets["gofile"]: - run_gofile(buckets["gofile"], args, state, source_by_url) - if buckets["0807"]: - run_0807(buckets["0807"], args, state, source_by_url) + if args.subs: + if args.channel_id or args.out_dir or args.urls_file: + parser.error("--subs cannot be combined with channel_id/out_dir/--urls-file") + subs = load_subscriptions(args.subs) + token = resolve_token(args.token) + total_failed = 0 + for entry in subs: + out_dir = Path(entry["out_dir"]) + log.info("=== subscription: %s ===", entry.get("name") or entry["channel_id"]) + total_failed += process_channel( + entry["channel_id"], out_dir, args, token, gstore, + force_export=not args.no_fresh, # subs mode re-exports by default + ) + if total_failed: + log.warning("%d URL(s) failed in total across subscriptions.", total_failed) + sys.exit(1) + return - failed_file = write_failed_urls(out_dir, state) + if not args.channel_id or not args.out_dir: + parser.error("channel_id and out_dir are required (or use --subs/--assume-done)") - # Summary - ok = [u for u, r in state.records.items() if r.get("status") == "ok"] - bad = state.failures() - total_files = sum(len(r.get("files") or []) for r in state.records.values()) - total_size = sum( - (f_.get("size") or 0) for r in state.records.values() for f_ in (r.get("files") or []) - ) - log.info("=" * 60) - log.info("Done: %d ok (%d files, %s) / %d failed / %d total seen", - len(ok), total_files, format_size(total_size), len(bad), len(state.records)) - if bad: - log.info("Failed URLs written to %s - re-run this tool or use " - "python3 dlp.py --urls-file %s to retry them.", - failed_file, failed_file.name) + # A token is only needed when we will export a channel (--urls-file skips that) + token = None if args.urls_file else resolve_token(args.token) + failures = process_channel(args.channel_id, Path(args.out_dir), args, token, gstore) + if failures: sys.exit(1) diff --git a/dlphelper/globalstore.py b/dlphelper/globalstore.py new file mode 100644 index 0000000..057f6df --- /dev/null +++ b/dlphelper/globalstore.py @@ -0,0 +1,70 @@ +"""Cross-run, cross-channel record of completed URLs. + +This is what prevents re-downloading your collection: state.json only knows +about one OUT_DIR, but the same gofile link posted in a second channel would +otherwise be downloaded again into that channel's OUT_DIR. The global store +(default: ~/.local/share/dlphelper/urls.json) records every successfully +downloaded URL once, forever, regardless of OUT_DIR. +""" + +from __future__ import annotations + +import json +import logging +import os +import time +from pathlib import Path + +log = logging.getLogger(__name__) + + +def default_global_path() -> Path: + data_home = os.environ.get("XDG_DATA_HOME") or str( + Path.home() / ".local" / "share" + ) + return Path(data_home) / "dlphelper" / "urls.json" + + +class GlobalStore: + """JSON file mapping url -> {status, channel, host, files, updated}.""" + + def __init__(self, path: Path | None = None): + self.path = path + self.records: dict[str, dict] = {} + if self.path and self.path.exists(): + try: + self.records = json.loads(self.path.read_text(encoding="utf-8")) + if not isinstance(self.records, dict): + raise ValueError("global store root must be an object") + except (ValueError, OSError) as exc: + backup = self.path.with_suffix(".json.bak") + log.warning("Corrupt global store %s (%s); backed up to %s", + self.path, exc, backup) + self.path.replace(backup) + self.records = {} + + def is_done(self, url: str) -> bool: + if not self.records: + return False + rec = self.records.get(url) + return bool(rec) and rec.get("status") == "ok" + + def mark(self, url: str, **fields) -> None: + if self.path is None: + return + rec = self.records.setdefault(url, {"status": "ok"}) + for key, value in fields.items(): + if value is not None: + rec[key] = value + rec["status"] = "ok" + rec["updated"] = time.strftime("%Y-%m-%dT%H:%M:%S") + self.save() + + def save(self) -> None: + if self.path is None: + return + self.path.parent.mkdir(parents=True, exist_ok=True) + tmp = self.path.with_suffix(".json.tmp") + tmp.write_text(json.dumps(self.records, indent=2, sort_keys=True), + encoding="utf-8") + tmp.replace(self.path)