feat(rust): port state.py, globalstore.py and manifest appending
- src/state.rs: per-run state.json (status/attempts/error/files/updated), corrupt-file -> *.json.bak restart, atomic tmp+rename save, mark() parity (attempts increments, None fields never clear existing values) - src/globalstore.rs: cross-run urls.json at XDG_DATA_HOME or ~/.local/share/dlphelper/urls.json, disabled-store no-op - src/manifest.rs: append_manifest as compact JSONL, UTF-8 - unknown/extra JSON fields are preserved on round-trip so both versions can read each other's artifacts (schema contract per AGENTS.md) - 14 unit tests
This commit is contained in:
Generated
+769
@@ -2,6 +2,775 @@
|
||||
# It is not intended for manual editing.
|
||||
version = 4
|
||||
|
||||
[[package]]
|
||||
name = "aho-corasick"
|
||||
version = "1.1.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c982642fa9e8606056828ee9a8505737230110bb1099153c79efe865c59d12ba"
|
||||
dependencies = [
|
||||
"memchr",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "android_system_properties"
|
||||
version = "0.1.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ae221649c9976a6f6c56ae1facf410f3ddb33cc661c4b7b61020a912d4237fbc"
|
||||
dependencies = [
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "anstream"
|
||||
version = "1.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d"
|
||||
dependencies = [
|
||||
"anstyle",
|
||||
"anstyle-parse",
|
||||
"anstyle-query",
|
||||
"anstyle-wincon",
|
||||
"colorchoice",
|
||||
"is_terminal_polyfill",
|
||||
"utf8parse",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "anstyle"
|
||||
version = "1.0.14"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000"
|
||||
|
||||
[[package]]
|
||||
name = "anstyle-parse"
|
||||
version = "1.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e"
|
||||
dependencies = [
|
||||
"utf8parse",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "anstyle-query"
|
||||
version = "1.1.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc"
|
||||
dependencies = [
|
||||
"windows-sys",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "anstyle-wincon"
|
||||
version = "3.0.11"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d"
|
||||
dependencies = [
|
||||
"anstyle",
|
||||
"once_cell_polyfill",
|
||||
"windows-sys",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "anyhow"
|
||||
version = "1.0.104"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470"
|
||||
|
||||
[[package]]
|
||||
name = "autocfg"
|
||||
version = "1.5.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53"
|
||||
|
||||
[[package]]
|
||||
name = "bitflags"
|
||||
version = "1.3.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
|
||||
|
||||
[[package]]
|
||||
name = "bitflags"
|
||||
version = "2.13.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3ded4057c258ba199e2d26386d3af3780957ecaee6c4ef4041c6b4b8b97c0b06"
|
||||
|
||||
[[package]]
|
||||
name = "bumpalo"
|
||||
version = "3.20.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649"
|
||||
|
||||
[[package]]
|
||||
name = "cc"
|
||||
version = "1.6.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f74872d07caf508b30a21f6836e7d7016a2eaf7d9ff4f48deaa58cd8a0407630"
|
||||
dependencies = [
|
||||
"find-msvc-tools",
|
||||
"shlex",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "cfg-if"
|
||||
version = "1.0.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4e7648175b45a9a48536d676f68d918270699102aa8dab5496df06904c914600"
|
||||
|
||||
[[package]]
|
||||
name = "chrono"
|
||||
version = "0.4.45"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1aa79e62e7697b8e29b513a68abacf485adcd1fe8284a4316c5ae868e6633327"
|
||||
dependencies = [
|
||||
"iana-time-zone",
|
||||
"js-sys",
|
||||
"num-traits",
|
||||
"wasm-bindgen",
|
||||
"windows-link",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "colorchoice"
|
||||
version = "1.0.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570"
|
||||
|
||||
[[package]]
|
||||
name = "core-foundation-sys"
|
||||
version = "0.8.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b"
|
||||
|
||||
[[package]]
|
||||
name = "defmt"
|
||||
version = "1.1.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e2953bfe4f93bbd20cc71198842756f77d161884c99ebbabc41d80231ded88d1"
|
||||
dependencies = [
|
||||
"bitflags 1.3.2",
|
||||
"defmt-macros",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "defmt-macros"
|
||||
version = "1.1.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "bad9c72e7ca2137e0dc3813245a0d282fd6daad32fd800af018306a9169b5fe8"
|
||||
dependencies = [
|
||||
"defmt-parser",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 2.0.119",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "defmt-parser"
|
||||
version = "1.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "10d60334b3b2e7c9d91ef8150abfb6fa4c1c39ebbcf4a81c2e346aad939fee3e"
|
||||
dependencies = [
|
||||
"thiserror",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "dirs"
|
||||
version = "6.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c3e8aa94d75141228480295a7d0e7feb620b1a5ad9f12bc40be62411e38cce4e"
|
||||
dependencies = [
|
||||
"dirs-sys",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "dirs-sys"
|
||||
version = "0.5.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e01a3366d27ee9890022452ee61b2b63a67e6f13f58900b651ff5665f0bb1fab"
|
||||
dependencies = [
|
||||
"libc",
|
||||
"option-ext",
|
||||
"redox_users",
|
||||
"windows-sys",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "dlphelper"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"chrono",
|
||||
"dirs",
|
||||
"env_logger",
|
||||
"log",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"tempfile",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "env_filter"
|
||||
version = "2.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "900d271a03799a1ee8d1ca9b19893b48ca674a9284fefcfb85f05e74ed314217"
|
||||
dependencies = [
|
||||
"log",
|
||||
"regex",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "env_logger"
|
||||
version = "0.11.11"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "de671bd27a75a797dc9ae289ba1e77276e75e2026408aab65185384e2d5cd3f6"
|
||||
dependencies = [
|
||||
"anstream",
|
||||
"anstyle",
|
||||
"env_filter",
|
||||
"jiff",
|
||||
"log",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "errno"
|
||||
version = "0.3.14"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb"
|
||||
dependencies = [
|
||||
"libc",
|
||||
"windows-sys",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "fastrand"
|
||||
version = "2.5.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223"
|
||||
|
||||
[[package]]
|
||||
name = "find-msvc-tools"
|
||||
version = "0.1.14"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "aedcfb3409746eddb02b9e19ebda1c3394f759a152e48ee875a0844d1b955484"
|
||||
|
||||
[[package]]
|
||||
name = "futures-core"
|
||||
version = "0.3.34"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "92d699e522242e69e3003b94ecc1f960f3a5e015aa7c5d7486e65ad01dd94f5e"
|
||||
|
||||
[[package]]
|
||||
name = "futures-task"
|
||||
version = "0.3.34"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "cd417de3d1d015fc3bfd2b1ea46dfc7bab72ef86f1cc7cc9c78e728b34a6d1fd"
|
||||
|
||||
[[package]]
|
||||
name = "futures-util"
|
||||
version = "0.3.34"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0d50a92467f8ba5dd6e3ee5d4bd04d73ab2e4e1c44474a0674821dfce14b79bc"
|
||||
dependencies = [
|
||||
"futures-core",
|
||||
"futures-task",
|
||||
"pin-project-lite",
|
||||
"slab",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "getrandom"
|
||||
version = "0.4.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"libc",
|
||||
"r-efi",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "iana-time-zone"
|
||||
version = "0.1.65"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e31bc9ad994ba00e440a8aa5c9ef0ec67d5cb5e5cb0cc7f8b744a35b389cc470"
|
||||
dependencies = [
|
||||
"android_system_properties",
|
||||
"core-foundation-sys",
|
||||
"iana-time-zone-haiku",
|
||||
"js-sys",
|
||||
"log",
|
||||
"wasm-bindgen",
|
||||
"windows-core",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "iana-time-zone-haiku"
|
||||
version = "0.1.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f"
|
||||
dependencies = [
|
||||
"cc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "is_terminal_polyfill"
|
||||
version = "1.70.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695"
|
||||
|
||||
[[package]]
|
||||
name = "itoa"
|
||||
version = "1.0.18"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682"
|
||||
|
||||
[[package]]
|
||||
name = "jiff"
|
||||
version = "0.2.38"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b2b005715dcbeb0089a3c0dab99f2ff1cc3b2525323552703d648585d342a383"
|
||||
dependencies = [
|
||||
"defmt",
|
||||
"jiff-core",
|
||||
"jiff-static",
|
||||
"log",
|
||||
"portable-atomic",
|
||||
"portable-atomic-util",
|
||||
"serde_core",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "jiff-core"
|
||||
version = "0.1.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5e52fe76043ccecc9005d2305ebaadf7d7fc0cc89ca6baa10a94d6bc68c7128c"
|
||||
dependencies = [
|
||||
"defmt",
|
||||
"log",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "jiff-static"
|
||||
version = "0.2.38"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2cc9817253cf7c7ee4684451bd327e88d6f3658014e54a29198625590650695c"
|
||||
dependencies = [
|
||||
"jiff-core",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 2.0.119",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "js-sys"
|
||||
version = "0.3.106"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7883d941dae510fb2d978fc3fe018c71c9e2892fd38854de3e8b92c2e5ad9cc5"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"futures-util",
|
||||
"wasm-bindgen",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "libc"
|
||||
version = "0.2.190"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ce5d3ddc6d3fa000eb1536d85e147bfe31aacaba692ed6a876f95cb7c855be78"
|
||||
|
||||
[[package]]
|
||||
name = "libredox"
|
||||
version = "0.1.25"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "61ff90caf6077a803a240f62fdbe88645a890bbca49ef8174c3cb0404362171d"
|
||||
dependencies = [
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "linux-raw-sys"
|
||||
version = "0.12.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53"
|
||||
|
||||
[[package]]
|
||||
name = "log"
|
||||
version = "0.4.34"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6"
|
||||
|
||||
[[package]]
|
||||
name = "memchr"
|
||||
version = "2.8.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98"
|
||||
|
||||
[[package]]
|
||||
name = "num-traits"
|
||||
version = "0.2.19"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841"
|
||||
dependencies = [
|
||||
"autocfg",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "once_cell"
|
||||
version = "1.21.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50"
|
||||
|
||||
[[package]]
|
||||
name = "once_cell_polyfill"
|
||||
version = "1.70.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe"
|
||||
|
||||
[[package]]
|
||||
name = "option-ext"
|
||||
version = "0.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d"
|
||||
|
||||
[[package]]
|
||||
name = "pin-project-lite"
|
||||
version = "0.2.17"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd"
|
||||
|
||||
[[package]]
|
||||
name = "portable-atomic"
|
||||
version = "1.15.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "05c8b63e8d9609db387f0324918f81d68fe27748f084ef092fb35954d0539a85"
|
||||
|
||||
[[package]]
|
||||
name = "portable-atomic-util"
|
||||
version = "0.2.8"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "10ab3eb7f3becc3a1cbc4f2c6f20267996cfc1a6467a873763411b136a122715"
|
||||
dependencies = [
|
||||
"portable-atomic",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "proc-macro2"
|
||||
version = "1.0.107"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9"
|
||||
dependencies = [
|
||||
"unicode-ident",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "quote"
|
||||
version = "1.0.47"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "r-efi"
|
||||
version = "6.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf"
|
||||
|
||||
[[package]]
|
||||
name = "redox_users"
|
||||
version = "0.5.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "60dc65c0ff1a7ae1294b0c67b9f14baf70b644404010370171787bfac1038fc0"
|
||||
dependencies = [
|
||||
"libredox",
|
||||
"thiserror",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "regex"
|
||||
version = "1.13.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d"
|
||||
dependencies = [
|
||||
"aho-corasick",
|
||||
"memchr",
|
||||
"regex-automata",
|
||||
"regex-syntax",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "regex-automata"
|
||||
version = "0.4.18"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ad8553b9b26413251cbf30e620595c7a41b3887f03da04579c0e6b0d6a06b4b2"
|
||||
dependencies = [
|
||||
"aho-corasick",
|
||||
"memchr",
|
||||
"regex-syntax",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "regex-syntax"
|
||||
version = "0.8.11"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4"
|
||||
|
||||
[[package]]
|
||||
name = "rustix"
|
||||
version = "1.1.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "891efababe418670775f199f0d233d84843c227a0949a883ce15b37c78d6629d"
|
||||
dependencies = [
|
||||
"bitflags 2.13.2",
|
||||
"errno",
|
||||
"libc",
|
||||
"linux-raw-sys",
|
||||
"windows-sys",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rustversion"
|
||||
version = "1.0.23"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f"
|
||||
|
||||
[[package]]
|
||||
name = "serde"
|
||||
version = "1.0.229"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba"
|
||||
dependencies = [
|
||||
"serde_core",
|
||||
"serde_derive",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "serde_core"
|
||||
version = "1.0.229"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48"
|
||||
dependencies = [
|
||||
"serde_derive",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "serde_derive"
|
||||
version = "1.0.229"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 3.0.6",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "serde_json"
|
||||
version = "1.0.151"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14"
|
||||
dependencies = [
|
||||
"itoa",
|
||||
"memchr",
|
||||
"serde",
|
||||
"serde_core",
|
||||
"zmij",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "shlex"
|
||||
version = "2.0.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba"
|
||||
|
||||
[[package]]
|
||||
name = "slab"
|
||||
version = "0.4.12"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5"
|
||||
|
||||
[[package]]
|
||||
name = "syn"
|
||||
version = "2.0.119"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"unicode-ident",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "syn"
|
||||
version = "3.0.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8593e8e72159ed2257d083c7a454a85cbf854f37a0966d8d483aff8c8a3ebcee"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"unicode-ident",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tempfile"
|
||||
version = "3.27.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd"
|
||||
dependencies = [
|
||||
"fastrand",
|
||||
"getrandom",
|
||||
"once_cell",
|
||||
"rustix",
|
||||
"windows-sys",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "thiserror"
|
||||
version = "2.0.21"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "09e52cb86a36cede5cb101bf8908837b3e4c6e5e59fe7fd85c23fb56200d189e"
|
||||
dependencies = [
|
||||
"thiserror-impl",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "thiserror-impl"
|
||||
version = "2.0.21"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "fe5197923287db20a58125f0bc85c062f7f2c892de97b18c356f9efb14b28524"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 3.0.6",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "unicode-ident"
|
||||
version = "1.0.26"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d245f478577f809a851594d02313b640fb437e0bb33866753cff937863096954"
|
||||
|
||||
[[package]]
|
||||
name = "utf8parse"
|
||||
version = "0.2.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821"
|
||||
|
||||
[[package]]
|
||||
name = "wasm-bindgen"
|
||||
version = "0.2.129"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9bb54f33acc68fd454578d9820b0bde1a1a3d17aa17bb7b6595806d02886d409"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"once_cell",
|
||||
"rustversion",
|
||||
"wasm-bindgen-macro",
|
||||
"wasm-bindgen-shared",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "wasm-bindgen-macro"
|
||||
version = "0.2.129"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2e29d0c35b16e224a7eeb5cd2d25e3e1968fbd65604117b44d3b789d00ee8535"
|
||||
dependencies = [
|
||||
"quote",
|
||||
"wasm-bindgen-macro-support",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "wasm-bindgen-macro-support"
|
||||
version = "0.2.129"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6f501a8bc3719dba86ef8ae4728879c08001bea749eb1333ac5b91e040e2a6b7"
|
||||
dependencies = [
|
||||
"bumpalo",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 3.0.6",
|
||||
"wasm-bindgen-shared",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "wasm-bindgen-shared"
|
||||
version = "0.2.129"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "23f0c9c52aa7cd7d77769a4cfe2a9adb1b331f489a41d912ce14513d5ab995c6"
|
||||
dependencies = [
|
||||
"unicode-ident",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "windows-core"
|
||||
version = "0.62.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb"
|
||||
dependencies = [
|
||||
"windows-implement",
|
||||
"windows-interface",
|
||||
"windows-link",
|
||||
"windows-result",
|
||||
"windows-strings",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "windows-implement"
|
||||
version = "0.60.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 2.0.119",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "windows-interface"
|
||||
version = "0.59.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 2.0.119",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "windows-link"
|
||||
version = "0.2.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5"
|
||||
|
||||
[[package]]
|
||||
name = "windows-result"
|
||||
version = "0.4.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5"
|
||||
dependencies = [
|
||||
"windows-link",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "windows-strings"
|
||||
version = "0.5.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091"
|
||||
dependencies = [
|
||||
"windows-link",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "windows-sys"
|
||||
version = "0.61.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc"
|
||||
dependencies = [
|
||||
"windows-link",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "zmij"
|
||||
version = "1.0.23"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b"
|
||||
|
||||
@@ -5,3 +5,13 @@ edition = "2024"
|
||||
description = "Torrent-first platform for Warhammer proxy models: Discord channel export, link extraction, per-host downloaders, manifest — Rust rewrite of version-python."
|
||||
|
||||
[dependencies]
|
||||
anyhow = "1"
|
||||
chrono = "0.4"
|
||||
dirs = "6"
|
||||
env_logger = "0.11"
|
||||
log = "0.4"
|
||||
serde = { version = "1", features = ["derive"] }
|
||||
serde_json = "1"
|
||||
|
||||
[dev-dependencies]
|
||||
tempfile = "3"
|
||||
|
||||
@@ -17,7 +17,9 @@ the python tooling.
|
||||
|
||||
| Concern | Python today | Rust plan |
|
||||
|---|---|---|
|
||||
| 7z extraction (extract_all) | 7z CLI, cp437 name repair, /var/tmp fallback | `sevenz-rust2` (pure Rust, LZMA2/PPMD/AES-256); zip via `zip` crate; rar likely still needs an external tool |
|
||||
| 7z extraction | 7z CLI, cp437 name repair, /var/tmp fallback | `sevenz-rust2` (pure Rust; LZMA/LZMA2/PPMD/AES-256 decrypt, v0.23, active) |
|
||||
| zip extraction | 7z CLI | `zip` crate |
|
||||
| RAR extraction (critical) | 7z CLI | `unrar.rs` (muja/unrar.rs — bindings to the official UnRAR library, RAR5 support, extraction only). Note: UnRAR license is freeware-not-OSI and forbids building a RAR compressor — acceptable, we only extract, and compression later is 7z-only (`sevenz-rust2` `ArchiveWriter`) anyway |
|
||||
| HTTP client | `requests` | `reqwest` (blocking first, async where the webapp needs it) |
|
||||
| sha256 (gofile website token, --hash manifest) | hashlib | `sha2` |
|
||||
| Fuzzy matching (sorter) | difflib-style ratios | `strsim` — port thresholds exactly, add unit tests |
|
||||
@@ -27,8 +29,8 @@ the python tooling.
|
||||
|
||||
## Port order (suggested)
|
||||
|
||||
1. `state` / `globalstore` — state.json + urls.json schemas must stay
|
||||
backward compatible with the python versions.
|
||||
1. `state` / `globalstore` / manifest — **done**, see `src/state.rs`,
|
||||
`src/globalstore.rs`, `src/manifest.rs` (schemas backward compatible).
|
||||
2. hosters: `gofile` (one guest account per run!), `zero807` (UA + Referer).
|
||||
3. extraction + manifest appender.
|
||||
4. catalogue / sorter (hardest logic; port the matching rules documented in
|
||||
|
||||
@@ -0,0 +1,193 @@
|
||||
//! Cross-run, cross-channel record of completed URLs — Rust port of
|
||||
//! `version-python/dlphelper/globalstore.py`.
|
||||
//!
|
||||
//! state.json only knows about one OUT_DIR; this store (default
|
||||
//! `~/.local/share/dlphelper/urls.json`) records every successfully
|
||||
//! downloaded URL once, forever, regardless of OUT_DIR — so the same gofile
|
||||
//! link posted in a second channel is not downloaded again.
|
||||
|
||||
use std::collections::BTreeMap;
|
||||
use std::fs;
|
||||
use std::path::PathBuf;
|
||||
|
||||
use log::warn;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::Value;
|
||||
|
||||
use crate::util::{now_local_iso, save_sorted};
|
||||
|
||||
/// Per-URL record in urls.json: `{"status", "updated", "channel", "host",
|
||||
/// "file_count", ...}`. Only fields actually set are written (the python
|
||||
/// version never writes `attempts`/`files` here).
|
||||
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
|
||||
pub struct GlobalRecord {
|
||||
#[serde(default)]
|
||||
pub status: String,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub updated: Option<String>,
|
||||
/// channel / host / file_count and any unknown fields.
|
||||
#[serde(flatten, default)]
|
||||
pub extra: BTreeMap<String, Value>,
|
||||
}
|
||||
|
||||
impl GlobalRecord {
|
||||
pub fn is_done(&self) -> bool {
|
||||
self.status == "ok"
|
||||
}
|
||||
}
|
||||
|
||||
/// Default location: `$XDG_DATA_HOME/dlphelper/urls.json`, else
|
||||
/// `~/.local/share/dlphelper/urls.json`. An empty XDG_DATA_HOME falls back to
|
||||
/// home, matching the python version.
|
||||
pub fn default_global_path() -> Option<PathBuf> {
|
||||
if let Ok(xdg) = std::env::var("XDG_DATA_HOME") {
|
||||
if !xdg.is_empty() {
|
||||
return Some(PathBuf::from(xdg).join("dlphelper").join("urls.json"));
|
||||
}
|
||||
}
|
||||
dirs::home_dir().map(|home| {
|
||||
home.join(".local")
|
||||
.join("share")
|
||||
.join("dlphelper")
|
||||
.join("urls.json")
|
||||
})
|
||||
}
|
||||
|
||||
/// JSON file mapping url -> record. A `None` path disables the store
|
||||
/// (all writes become no-ops), like passing `gstore=None` in python.
|
||||
pub struct GlobalStore {
|
||||
pub path: Option<PathBuf>,
|
||||
pub records: BTreeMap<String, GlobalRecord>,
|
||||
}
|
||||
|
||||
impl GlobalStore {
|
||||
pub fn disabled() -> Self {
|
||||
Self {
|
||||
path: None,
|
||||
records: BTreeMap::new(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Load urls.json; a corrupt or unreadable file is moved to `*.json.bak`
|
||||
/// and we start fresh (same behavior as the python version).
|
||||
pub fn load(path: impl Into<PathBuf>) -> Self {
|
||||
let path = path.into();
|
||||
if path.exists() {
|
||||
let parsed = fs::read_to_string(&path).ok().and_then(|text| {
|
||||
serde_json::from_str::<BTreeMap<String, GlobalRecord>>(&text).ok()
|
||||
});
|
||||
if let Some(records) = parsed {
|
||||
return Self {
|
||||
path: Some(path),
|
||||
records,
|
||||
};
|
||||
}
|
||||
let backup = path.with_extension("json.bak");
|
||||
warn!(
|
||||
"Corrupt global store {} ; backed up to {}",
|
||||
path.display(),
|
||||
backup.display()
|
||||
);
|
||||
let _ = fs::rename(&path, &backup);
|
||||
}
|
||||
Self {
|
||||
path: Some(path),
|
||||
records: BTreeMap::new(),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn is_done(&self, url: &str) -> bool {
|
||||
self.records.get(url).is_some_and(GlobalRecord::is_done)
|
||||
}
|
||||
|
||||
/// Record a URL as done (plus attribution fields when known), then save.
|
||||
/// No-ops when the store is disabled.
|
||||
pub fn mark(
|
||||
&mut self,
|
||||
url: &str,
|
||||
channel: Option<&str>,
|
||||
host: Option<&str>,
|
||||
file_count: Option<u64>,
|
||||
) -> anyhow::Result<()> {
|
||||
if self.path.is_none() {
|
||||
return Ok(());
|
||||
}
|
||||
let rec = self.records.entry(url.to_string()).or_default();
|
||||
rec.status = "ok".to_string();
|
||||
if let Some(c) = channel {
|
||||
rec.extra.insert("channel".into(), Value::from(c));
|
||||
}
|
||||
if let Some(h) = host {
|
||||
rec.extra.insert("host".into(), Value::from(h));
|
||||
}
|
||||
if let Some(n) = file_count {
|
||||
rec.extra.insert("file_count".into(), Value::from(n));
|
||||
}
|
||||
rec.updated = Some(now_local_iso());
|
||||
self.save()
|
||||
}
|
||||
|
||||
pub fn save(&self) -> anyhow::Result<()> {
|
||||
match &self.path {
|
||||
Some(path) => save_sorted(path, &self.records),
|
||||
None => Ok(()),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::fs;
|
||||
|
||||
#[test]
|
||||
fn disabled_store_is_a_noop() {
|
||||
let mut gs = GlobalStore::disabled();
|
||||
gs.mark("u", Some("c"), Some("gofile"), Some(2)).unwrap();
|
||||
assert!(gs.records.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mark_persists_and_reloads() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let path = dir.path().join("urls.json");
|
||||
{
|
||||
let mut gs = GlobalStore::load(&path);
|
||||
gs.mark("u1", Some("chan"), Some("gofile"), Some(3))
|
||||
.unwrap();
|
||||
gs.mark("u2", None, None, None).unwrap();
|
||||
}
|
||||
let gs = GlobalStore::load(&path);
|
||||
assert!(gs.is_done("u1"));
|
||||
assert!(gs.is_done("u2"));
|
||||
assert!(!gs.is_done("u3"));
|
||||
|
||||
let v: serde_json::Value =
|
||||
serde_json::from_str(&fs::read_to_string(&path).unwrap()).unwrap();
|
||||
assert_eq!(v["u1"]["status"], "ok");
|
||||
assert_eq!(v["u1"]["channel"], "chan");
|
||||
assert_eq!(v["u1"]["host"], "gofile");
|
||||
assert_eq!(v["u1"]["file_count"], 3);
|
||||
// no attempts/files keys here — that's state.json's job
|
||||
assert!(v["u1"].get("attempts").is_none());
|
||||
assert!(v["u2"].get("channel").is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn corrupt_store_is_backed_up() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let path = dir.path().join("urls.json");
|
||||
fs::write(&path, "{not json").unwrap();
|
||||
let gs = GlobalStore::load(&path);
|
||||
assert!(gs.records.is_empty());
|
||||
assert!(dir.path().join("urls.json.bak").exists());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn default_path_honors_empty_xdg_fallback() {
|
||||
// Only assert shape; don't touch the process env (tests run in parallel).
|
||||
if default_global_path().is_none() {
|
||||
panic!("home dir should be discoverable in tests");
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,3 +1,9 @@
|
||||
mod globalstore;
|
||||
mod manifest;
|
||||
mod state;
|
||||
mod util;
|
||||
|
||||
fn main() {
|
||||
println!("dlphelper {version} — Rust rewrite scaffold; modules are ported step by step, see version-rust/README.md", version = env!("CARGO_PKG_VERSION"));
|
||||
env_logger::init();
|
||||
println!("dlphelper {}", env!("CARGO_PKG_VERSION"));
|
||||
}
|
||||
|
||||
@@ -0,0 +1,101 @@
|
||||
//! manifest.jsonl — one compact JSON line per downloaded file — Rust port of
|
||||
//! `append_manifest` from `version-python/dlphelper/state.py`.
|
||||
//!
|
||||
//! Row layout (all attribution fields optional except channel/url):
|
||||
//! `{"channel", "url", "host", "message_id", "timestamp", "author", "path",
|
||||
//! "size", ...}` — unknown extra fields are preserved on round-trip.
|
||||
|
||||
use std::collections::BTreeMap;
|
||||
use std::fs::OpenOptions;
|
||||
use std::io::Write;
|
||||
use std::path::Path;
|
||||
|
||||
use anyhow::Context as _;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::Value;
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct ManifestRow {
|
||||
pub channel: String,
|
||||
pub url: String,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub host: Option<String>,
|
||||
/// Message ids from Discord exports may be numeric or string.
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub message_id: Option<Value>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub timestamp: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub author: Option<String>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub path: Option<String>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub size: Option<u64>,
|
||||
/// Any additional fields merged in by the caller.
|
||||
#[serde(flatten, default)]
|
||||
pub extra: BTreeMap<String, Value>,
|
||||
}
|
||||
|
||||
/// Append one compact JSON line (UTF-8, like python `ensure_ascii=False`).
|
||||
pub fn append_manifest(manifest_path: impl AsRef<Path>, row: &ManifestRow) -> anyhow::Result<()> {
|
||||
let path = manifest_path.as_ref();
|
||||
if let Some(parent) = path.parent() {
|
||||
std::fs::create_dir_all(parent)
|
||||
.with_context(|| format!("creating {}", parent.display()))?;
|
||||
}
|
||||
let mut fh = OpenOptions::new()
|
||||
.create(true)
|
||||
.append(true)
|
||||
.open(path)
|
||||
.with_context(|| format!("opening {}", path.display()))?;
|
||||
fh.write_all(serde_json::to_string(row)?.as_bytes())?;
|
||||
fh.write_all(b"\n")?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::fs;
|
||||
|
||||
fn row(path: &str) -> ManifestRow {
|
||||
ManifestRow {
|
||||
channel: "123".into(),
|
||||
url: "https://gofile.io/d/abc".into(),
|
||||
host: Some("gofile".into()),
|
||||
message_id: Some(Value::from(987654)),
|
||||
timestamp: Some("2026-10-09T13:00:00".into()),
|
||||
author: Some("tester".into()),
|
||||
path: Some(path.into()),
|
||||
size: Some(4567),
|
||||
extra: BTreeMap::new(),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn appends_jsonl_and_creates_parents() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let manifest = dir.path().join("a").join("b").join("manifest.jsonl");
|
||||
append_manifest(&manifest, &row("/out/x.7z")).unwrap();
|
||||
append_manifest(&manifest, &row("/out/y.7z")).unwrap();
|
||||
|
||||
let text = fs::read_to_string(&manifest).unwrap();
|
||||
let lines: Vec<&str> = text.lines().collect();
|
||||
assert_eq!(lines.len(), 2);
|
||||
let second: ManifestRow = serde_json::from_str(lines[1]).unwrap();
|
||||
assert_eq!(second.path.as_deref(), Some("/out/y.7z"));
|
||||
assert_eq!(second.channel, "123");
|
||||
assert_eq!(second.message_id, Some(Value::from(987654)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn utf8_is_not_ascii_escaped() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let manifest = dir.path().join("manifest.jsonl");
|
||||
let mut r = row("/out/x.7z");
|
||||
r.author = Some("Zähni".into());
|
||||
append_manifest(&manifest, &r).unwrap();
|
||||
let text = fs::read_to_string(&manifest).unwrap();
|
||||
assert!(text.contains("Zähni"));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,234 @@
|
||||
//! Per-run persistent state (state.json) — Rust port of
|
||||
//! `version-python/dlphelper/state.py`. Makes runs resumable and failures
|
||||
//! trackable: re-runs skip URLs with status "ok", failed URLs are retried.
|
||||
//!
|
||||
//! Schema contract with the python version:
|
||||
//!
|
||||
//! ```json
|
||||
//! { "<url>": { "status": "ok" | "failed" | "error" | ...,
|
||||
//! "attempts": 3, "error": "...", "updated": "2026-10-09T13:00:00",
|
||||
//! "files": [{"path": "...", "size": 123}] } }
|
||||
//! ```
|
||||
//!
|
||||
//! Unknown fields written by other tools survive load+save (serde flatten),
|
||||
//! so both versions can consume each other's files.
|
||||
|
||||
use std::collections::BTreeMap;
|
||||
use std::fs;
|
||||
use std::path::PathBuf;
|
||||
|
||||
use log::warn;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::Value;
|
||||
|
||||
use crate::util::{now_local_iso, save_sorted};
|
||||
|
||||
/// One downloaded file as recorded in `state.json`.
|
||||
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
|
||||
pub struct FileEntry {
|
||||
pub path: String,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub size: Option<u64>,
|
||||
/// Unknown extra fields, preserved for schema compatibility.
|
||||
#[serde(flatten, default)]
|
||||
pub extra: BTreeMap<String, Value>,
|
||||
}
|
||||
|
||||
/// Per-URL record in state.json.
|
||||
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
|
||||
pub struct Record {
|
||||
/// "pending" | "ok" | "failed" | "error" — open set, like the python version.
|
||||
#[serde(default)]
|
||||
pub status: String,
|
||||
#[serde(default)]
|
||||
pub attempts: u64,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub error: Option<String>,
|
||||
#[serde(default, skip_serializing_if = "Vec::is_empty")]
|
||||
pub files: Vec<FileEntry>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub updated: Option<String>,
|
||||
#[serde(flatten, default)]
|
||||
pub extra: BTreeMap<String, Value>,
|
||||
}
|
||||
|
||||
impl Record {
|
||||
pub fn is_done(&self) -> bool {
|
||||
self.status == "ok"
|
||||
}
|
||||
}
|
||||
|
||||
/// JSON-backed state store for one run directory.
|
||||
pub struct State {
|
||||
pub path: PathBuf,
|
||||
pub records: BTreeMap<String, Record>,
|
||||
}
|
||||
|
||||
impl State {
|
||||
/// Load state.json; a corrupt or unreadable file is moved to `*.json.bak`
|
||||
/// and we start fresh (same behavior as the python version).
|
||||
pub fn load(path: impl Into<PathBuf>) -> Self {
|
||||
let path = path.into();
|
||||
if path.exists() {
|
||||
let parsed = fs::read_to_string(&path)
|
||||
.ok()
|
||||
.and_then(|text| serde_json::from_str::<BTreeMap<String, Record>>(&text).ok());
|
||||
if let Some(records) = parsed {
|
||||
return Self { path, records };
|
||||
}
|
||||
let backup = path.with_extension("json.bak");
|
||||
warn!(
|
||||
"Corrupt state file {} ; moving to {} and starting fresh",
|
||||
path.display(),
|
||||
backup.display()
|
||||
);
|
||||
let _ = fs::rename(&path, &backup);
|
||||
}
|
||||
Self {
|
||||
path,
|
||||
records: BTreeMap::new(),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn is_done(&self, url: &str) -> bool {
|
||||
self.records.get(url).is_some_and(Record::is_done)
|
||||
}
|
||||
|
||||
/// The URLs from `urls` whose status is not "ok".
|
||||
pub fn pending<'a>(&self, urls: &'a [String]) -> Vec<&'a str> {
|
||||
urls.iter()
|
||||
.map(String::as_str)
|
||||
.filter(|u| !self.is_done(u))
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// All URLs whose record is not "ok".
|
||||
pub fn failures(&self) -> Vec<String> {
|
||||
self.records
|
||||
.iter()
|
||||
.filter(|(_, r)| !r.is_done())
|
||||
.map(|(u, _)| u.clone())
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Create-or-get the record for `url`, bump `attempts`, stamp `updated`,
|
||||
/// apply `f`, then save — mirrors python `mark(url, **fields)`.
|
||||
///
|
||||
/// Parity note: the python version skips `None` fields, so a value set by
|
||||
/// an earlier mark (e.g. an old error message) is *not* cleared by a later
|
||||
/// mark that doesn't mention it. The closure decides what to set; nothing
|
||||
/// is cleared implicitly.
|
||||
pub fn mark(&mut self, url: &str, f: impl FnOnce(&mut Record)) -> anyhow::Result<&Record> {
|
||||
let fresh = !self.records.contains_key(url);
|
||||
let rec = self.records.entry(url.to_string()).or_default();
|
||||
if fresh && rec.status.is_empty() {
|
||||
rec.status = "pending".to_string();
|
||||
}
|
||||
f(rec);
|
||||
rec.attempts += 1;
|
||||
rec.updated = Some(now_local_iso());
|
||||
self.save()?;
|
||||
Ok(self.records.get(url).expect("entry just inserted"))
|
||||
}
|
||||
|
||||
/// Atomic save: write `*.json.tmp`, then rename over state.json.
|
||||
pub fn save(&self) -> anyhow::Result<()> {
|
||||
save_sorted(&self.path, &self.records)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::fs;
|
||||
|
||||
#[test]
|
||||
fn mark_creates_record_and_increments_attempts() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let mut st = State::load(dir.path().join("state.json"));
|
||||
st.mark("http://x/1", |r| r.status = "ok".into()).unwrap();
|
||||
let rec = &st.records["http://x/1"];
|
||||
assert_eq!(rec.status, "ok");
|
||||
assert_eq!(rec.attempts, 1);
|
||||
assert_eq!(rec.updated.as_deref().map(str::len), Some(19));
|
||||
assert!(st.is_done("http://x/1"));
|
||||
assert!(st.failures().is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn failed_then_ok_keeps_old_error_like_python() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let mut st = State::load(dir.path().join("state.json"));
|
||||
st.mark("u", |r| {
|
||||
r.status = "failed".into();
|
||||
r.error = Some("boom".into());
|
||||
})
|
||||
.unwrap();
|
||||
// python: mark(url, status="ok", error=None) does NOT clear the error
|
||||
st.mark("u", |r| r.status = "ok".into()).unwrap();
|
||||
let rec = &st.records["u"];
|
||||
assert_eq!(rec.error.as_deref(), Some("boom"));
|
||||
assert_eq!(rec.attempts, 2);
|
||||
assert_eq!(st.failures(), Vec::<String>::new());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn pending_and_failures() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let mut st = State::load(dir.path().join("state.json"));
|
||||
st.mark("a", |r| r.status = "ok".into()).unwrap();
|
||||
st.mark("b", |r| r.status = "failed".into()).unwrap();
|
||||
st.mark("c", |_r| {}).unwrap();
|
||||
let urls = vec!["a".into(), "b".into(), "c".into()];
|
||||
assert_eq!(st.pending(&urls), vec!["b", "c"]);
|
||||
assert_eq!(st.failures(), vec!["b".to_string(), "c".to_string()]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn corrupt_file_is_backed_up_and_restarted() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let path = dir.path().join("state.json");
|
||||
fs::write(&path, "not json{").unwrap();
|
||||
let st = State::load(&path);
|
||||
assert!(st.records.is_empty());
|
||||
assert!(dir.path().join("state.json.bak").exists());
|
||||
assert!(!path.exists());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unknown_fields_survive_roundtrip() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let path = dir.path().join("state.json");
|
||||
fs::write(
|
||||
&path,
|
||||
r#"{"http://x/1": {"status": "ok", "attempts": 3, "custom": 42}}"#,
|
||||
)
|
||||
.unwrap();
|
||||
let st = State::load(&path);
|
||||
assert!(st.is_done("http://x/1"));
|
||||
st.save().unwrap();
|
||||
let v: serde_json::Value =
|
||||
serde_json::from_str(&fs::read_to_string(&path).unwrap()).unwrap();
|
||||
assert_eq!(v["http://x/1"]["custom"], 42);
|
||||
assert_eq!(v["http://x/1"]["attempts"], 3);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn files_are_recorded_and_persisted() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let path = dir.path().join("state.json");
|
||||
let mut st = State::load(&path);
|
||||
st.mark("u", |r| {
|
||||
r.status = "ok".into();
|
||||
r.files.push(FileEntry {
|
||||
path: "/out/x.7z".into(),
|
||||
size: Some(1234),
|
||||
extra: BTreeMap::new(),
|
||||
});
|
||||
})
|
||||
.unwrap();
|
||||
let reloaded = State::load(&path);
|
||||
assert_eq!(reloaded.records["u"].files.len(), 1);
|
||||
assert_eq!(reloaded.records["u"].files[0].size, Some(1234));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,62 @@
|
||||
//! Small shared helpers: timestamps and the atomic sorted-JSON save used by
|
||||
//! both state stores (mirrors `json.dumps(..., indent=2, sort_keys=True)` +
|
||||
//! tmp-file rename from the python version).
|
||||
|
||||
use std::fs;
|
||||
use std::path::Path;
|
||||
|
||||
use chrono::Local;
|
||||
use serde::Serialize;
|
||||
|
||||
/// Local time in the exact `"%Y-%m-%dT%H:%M:%S"` format the python version writes.
|
||||
pub fn now_local_iso() -> String {
|
||||
Local::now().format("%Y-%m-%dT%H:%M:%S").to_string()
|
||||
}
|
||||
|
||||
/// Write `value` pretty-printed, then atomically rename over `path`.
|
||||
/// The caller must keep keys in a `BTreeMap` so the output is key-sorted,
|
||||
/// like the python version's `sort_keys=True`.
|
||||
pub fn save_sorted(path: &Path, value: &impl Serialize) -> anyhow::Result<()> {
|
||||
if let Some(parent) = path.parent() {
|
||||
fs::create_dir_all(parent)?;
|
||||
}
|
||||
let tmp = path.with_extension("json.tmp");
|
||||
fs::write(&tmp, serde_json::to_string_pretty(value)?)?;
|
||||
fs::rename(&tmp, path)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::collections::BTreeMap;
|
||||
|
||||
#[test]
|
||||
fn timestamp_format_matches_python() {
|
||||
let t = now_local_iso();
|
||||
assert_eq!(t.len(), 19, "expected YYYY-MM-DDTHH:MM:SS, got {t}");
|
||||
let bytes = t.as_bytes();
|
||||
for (i, b) in bytes.iter().enumerate() {
|
||||
let is_sep = matches!(i, 4 | 7 | 10 | 13 | 16);
|
||||
if is_sep {
|
||||
assert!(!b.is_ascii_digit(), "expected separator at {i} in {t}");
|
||||
} else {
|
||||
assert!(b.is_ascii_digit(), "expected digit at {i} in {t}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn save_sorted_is_atomic_and_sorted() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let path = dir.path().join("sub").join("x.json");
|
||||
let mut map = BTreeMap::new();
|
||||
map.insert("b", 2);
|
||||
map.insert("a", 1);
|
||||
save_sorted(&path, &map).unwrap();
|
||||
let text = fs::read_to_string(&path).unwrap();
|
||||
assert!(text.find("\"a\"").unwrap() < text.find("\"b\"").unwrap());
|
||||
// tmp file was renamed away
|
||||
assert!(!dir.path().join("sub").join("x.json.tmp").exists());
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user