diff --git a/.gitignore b/.gitignore index e61c4f6..3313492 100644 --- a/.gitignore +++ b/.gitignore @@ -20,3 +20,6 @@ documentation/.obsidian/ *.log .DS_Store Thumbs.db + +# Rust build artifacts (workspace root is server/) +server/target/ diff --git a/documentation/ideen/roadmap.md b/documentation/ideen/roadmap.md index 05f72ea..88e1a44 100644 --- a/documentation/ideen/roadmap.md +++ b/documentation/ideen/roadmap.md @@ -15,10 +15,13 @@ endet mit etwas, das man benutzen kann. Ziel: Beweis, dass Pooling über zwei Anschlüsse real funktioniert. +- [x] `sn-fetch` PoC (lokal): Range-Probe, gewichteter Planner, parallele + Segment-Downloads, Reassembly + SHA-256 — verifiziert gegen einen + echten Server (Debian-Mirror); Multi-Exit-Routing folgt mit dem Mesh - [ ] SpiderNet-Node auf Mini-PC (Raspberry Pi / N100-Box), Yggdrasil-Overlay zwischen 2 Haushalten (Kabel oder Kurzstrecke WLAN/60-GHz) -- [ ] `sn-fetch` PoC: segmentierter Download (HTTP-Range-Splitting) über - 2 Exits, plus Baseline-Messung (1-Exit vs. 2-Exit, gleiche Datei) +- [ ] `sn-fetch` PoC über 2 echte Exits, plus Baseline-Messung + (1-Exit vs. 2-Exit, gleiche Datei) - [ ] Fair-Share-Scheduler v0: zwei gleichzeitige Downloads, faire Aufteilung - [ ] Doku: Messwerte, Erkenntnisse, Stolpersteine → hier in das Repo diff --git a/documentation/ideen/splitting.md b/documentation/ideen/splitting.md index 34933f7..431bf8d 100644 --- a/documentation/ideen/splitting.md +++ b/documentation/ideen/splitting.md @@ -55,8 +55,10 @@ die Verbindung kaputt machen würde (IP-Wechsel beendet die Session): | VPNs (WireGuard etc.) | UDP-Flow mit State | **Policy-Vorschlag (Default):** realzeitkritischer Traffic geht in den -**Direct-Modus** — ohne Umweg, normal über den eigenen Router/Uplink, als -wäre SpiderNet nicht da. Optional "best single exit" (wenn die eigene +**Direct-Modus** — und zwar als echter **Pass-Through**: SpiderNet gibt +diese Verbindungen unverändert weiter, sie nehmen den normalen Weg durch +den eigenen Router und den eigenen Uplink, als wäre SpiderNet nicht da +(kein Tunnel, kein MITM). Optional "best single exit" (wenn die eigene Leitung gestört/überlastet ist, kann man den Call bewusst über einen Nachbarschafts-Exit pinnen — aber dann sticky für die Dauer der Session). diff --git a/server/Cargo.lock b/server/Cargo.lock new file mode 100644 index 0000000..8ac49a0 --- /dev/null +++ b/server/Cargo.lock @@ -0,0 +1,1637 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "aho-corasick" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c982642fa9e8606056828ee9a8505737230110bb1099153c79efe865c59d12ba" +dependencies = [ + "memchr", +] + +[[package]] +name = "anstream" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" + +[[package]] +name = "anstyle-parse" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" +dependencies = [ + "anstyle", + "once_cell_polyfill", + "windows-sys 0.61.2", +] + +[[package]] +name = "atomic-waker" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" + +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + +[[package]] +name = "bitflags" +version = "2.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ded4057c258ba199e2d26386d3af3780957ecaee6c4ef4041c6b4b8b97c0b06" + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "bumpalo" +version = "3.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" + +[[package]] +name = "bytes" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" + +[[package]] +name = "cc" +version = "1.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "54413ede23c2daf518f35156dfde027feb2374004d63bd497f983c8db9c0e313" +dependencies = [ + "find-msvc-tools", + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e7648175b45a9a48536d676f68d918270699102aa8dab5496df06904c914600" + +[[package]] +name = "cfg_aliases" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f079e83a288787bcd14a6aea84cee5c87a67c5a3e660c30f557a3d24761b3527" + +[[package]] +name = "chacha20" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65c35e4b699c7e15ccbe7ee35c005e4fc0a278d22238a2857e6ce2dadeda1b06" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.1", + "rand_core", +] + +[[package]] +name = "clap" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa8876b300ab35ba921adea3dfd70157a46249b33f95c9084ae5709785478946" +dependencies = [ + "clap_builder", + "clap_derive", +] + +[[package]] +name = "clap_builder" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0797fb7aeb1406c84efac526901f7ec3ead2124f946b494e72879d4b54704d" +dependencies = [ + "anstream", + "anstyle", + "clap_lex", + "strsim", +] + +[[package]] +name = "clap_derive" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9c751b79415d4e559e3d1fcf128e09e720eb673a06d26cf6f392d37d75b66e0" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn 3.0.6", +] + +[[package]] +name = "clap_lex" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c133bc6a41be0d194c306b5506d15e6feeea7b1d6604bd3f8310dfb2ca96486" + +[[package]] +name = "colorchoice" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" + +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + +[[package]] +name = "cpufeatures" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ca28b0ae3115b884660db4118d803791fd6756b6e88f39c0f3f7859060d7566" +dependencies = [ + "libc", +] + +[[package]] +name = "crypto-common" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "crypto-common", +] + +[[package]] +name = "displaydoc" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6232dd377dcc64799954cbd3a9bb882e9cdc1308ccd87b1c098f1fb2eaf82a8" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.6", +] + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "fastrand" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" + +[[package]] +name = "find-msvc-tools" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef25905e51abafe4dcea6c15fec58c57b601cdbd0ee53d22ea1d3016c587d39b" + +[[package]] +name = "form_urlencoded" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb4cb245038516f5f85277875cdaa4f7d2c9a0fa0468de06ed190163b1581fcf" +dependencies = [ + "percent-encoding", +] + +[[package]] +name = "futures-channel" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1f9e3d69d39e4862ffed03ed071a76f9a13ba1d9109d355b0f0aa6b15e393c4" +dependencies = [ + "futures-core", +] + +[[package]] +name = "futures-core" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92d699e522242e69e3003b94ecc1f960f3a5e015aa7c5d7486e65ad01dd94f5e" + +[[package]] +name = "futures-task" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd417de3d1d015fc3bfd2b1ea46dfc7bab72ef86f1cc7cc9c78e728b34a6d1fd" + +[[package]] +name = "futures-util" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d50a92467f8ba5dd6e3ee5d4bd04d73ab2e4e1c44474a0674821dfce14b79bc" +dependencies = [ + "futures-core", + "futures-task", + "pin-project-lite", + "slab", +] + +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "getrandom" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" +dependencies = [ + "cfg-if", + "js-sys", + "libc", + "wasi", + "wasm-bindgen", +] + +[[package]] +name = "getrandom" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" +dependencies = [ + "cfg-if", + "js-sys", + "libc", + "r-efi", + "rand_core", + "wasm-bindgen", +] + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + +[[package]] +name = "http" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "918d3568bebf352712bc2ef3d46a8bcf1a75b373be6539de198e9105cbbf9ce0" +dependencies = [ + "bytes", + "itoa", +] + +[[package]] +name = "http-body" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca2a8f2913ee65f60facd6a5905613afaa448497a0230cc41ce022d93290bc2c" +dependencies = [ + "bytes", + "http", +] + +[[package]] +name = "http-body-util" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23169fe34a5fbcdd3f3862e78fb9b6fccd5f02a6dc6f732547005d45631ce71c" +dependencies = [ + "bytes", + "futures-core", + "http", + "http-body", + "pin-project-lite", +] + +[[package]] +name = "httparse" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" + +[[package]] +name = "hyper" +version = "1.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27b501faa50e7a26c3d3560ca625132f4078a17771f4810baf70475ae48cbe43" +dependencies = [ + "atomic-waker", + "bytes", + "futures-channel", + "futures-core", + "http", + "http-body", + "httparse", + "itoa", + "pin-project-lite", + "smallvec", + "tokio", + "want", +] + +[[package]] +name = "hyper-rustls" +version = "0.27.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dfa8e654703247911e29c23fbeaa261834bd9bb74efba2f9acddc37bfb127f53" +dependencies = [ + "http", + "hyper", + "hyper-util", + "rustls", + "tokio", + "tokio-rustls", + "tower-service", + "webpki-roots", +] + +[[package]] +name = "hyper-util" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0" +dependencies = [ + "base64", + "bytes", + "futures-channel", + "futures-util", + "http", + "http-body", + "hyper", + "ipnet", + "libc", + "percent-encoding", + "pin-project-lite", + "socket2", + "tokio", + "tower-service", + "tracing", +] + +[[package]] +name = "icu_collections" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa68d21081c4a05d5a901a1c62add574c77048b6a1c67be3b50ce0b60d4ca513" +dependencies = [ + "displaydoc", + "potential_utf", + "utf8_iter", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_locale_core" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d56e28588da92eee5c3201a6eff33fabdd49b62269c8938d4ff050ce4d900deb" +dependencies = [ + "displaydoc", + "litemap", + "tinystr", + "writeable", + "zerovec", +] + +[[package]] +name = "icu_normalizer" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12f9cf5f235641ed274641dd81c3f28d870e276763d0797aeeab72317b1c646f" +dependencies = [ + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", + "smallvec", + "zerovec", +] + +[[package]] +name = "icu_normalizer_data" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1563da1ed3e0b3bf3d74c9b85917ac9c56464d2f57242270c09c9e752f8021a0" + +[[package]] +name = "icu_properties" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e7ca276ad3145661a65914e6daf131ca5120cd3dcee8f8f3214b8875184a148" +dependencies = [ + "displaydoc", + "icu_collections", + "icu_locale_core", + "icu_properties_data", + "icu_provider", + "zerotrie", + "zerovec", +] + +[[package]] +name = "icu_properties_data" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e590f038c1464a96894fd6d10127e90a8be4509f56ff7ecef851b15cee0b7caa" + +[[package]] +name = "icu_provider" +version = "2.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d27bbb9d3abbefac45d55f647c9de1d44aafcd1186eb91879afef17c396c3e73" +dependencies = [ + "displaydoc", + "icu_locale_core", + "writeable", + "yoke", + "zerofrom", + "zerotrie", + "zerovec", +] + +[[package]] +name = "idna" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b0875f23caa03898994f6ddc501886a45c7d3d62d04d2d90788d47be1b1e4de" +dependencies = [ + "idna_adapter", + "smallvec", + "utf8_iter", +] + +[[package]] +name = "idna_adapter" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb68373c0d6620ef8105e855e7745e18b0d00d3bdb07fb532e434244cdb9a714" +dependencies = [ + "icu_normalizer", + "icu_properties", +] + +[[package]] +name = "ipnet" +version = "2.12.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "791930b43c0d5973160d90a8f3894509f2b273430f5c5c73b668636d0287c5c0" + +[[package]] +name = "is_terminal_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "js-sys" +version = "0.3.105" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce57d20d1ea864ce2ac172ab472d409214f4fd359f0b2a2775abdf522e2af99e" +dependencies = [ + "cfg-if", + "futures-util", + "wasm-bindgen", +] + +[[package]] +name = "lazy_static" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" + +[[package]] +name = "libc" +version = "0.2.189" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" + +[[package]] +name = "linux-raw-sys" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" + +[[package]] +name = "litemap" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47d9d19d1d6efa0109d2f65ff4c85cddd50bd572e5a00127ab10987290bcefae" + +[[package]] +name = "log" +version = "0.4.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6" + +[[package]] +name = "lru-slab" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4050469837a6ff301cd14c1f8f24f88549e6d548f24f64e2148eb0f72cebc51f" + +[[package]] +name = "matchers" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1525a2a28c7f4fa0fc98bb91ae755d1e2d1505079e05539e35bc876b5d65ae9" +dependencies = [ + "regex-automata", +] + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "mio" +version = "1.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b18443e9c262bfe8fa82f51666e2642c53393f7e5c27b3e1aeab922cff5b9d8" +dependencies = [ + "libc", + "wasi", + "windows-sys 0.61.2", +] + +[[package]] +name = "nu-ansi-term" +version = "0.50.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "once_cell_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" + +[[package]] +name = "percent-encoding" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" + +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + +[[package]] +name = "potential_utf" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d83eb9bc6d8e5cf568e7a1101d60ee05e81ed50ea106026f3d18deeb046d7661" +dependencies = [ + "zerovec", +] + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quinn" +version = "0.11.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4051e23e9185c255a7e33ef59cdbca87a22d359052eecd22fc6b901fb37d9d11" +dependencies = [ + "bytes", + "cfg_aliases", + "pin-project-lite", + "quinn-proto", + "quinn-udp", + "rustc-hash", + "rustls", + "socket2", + "thiserror", + "tokio", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-proto" +version = "0.11.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9746dbde176634f4f2f1faf2404e30a31b2bc1e9cafb5329c95d8177a18c9fc" +dependencies = [ + "bytes", + "getrandom 0.4.3", + "lru-slab", + "rand", + "rand_pcg", + "ring", + "rustc-hash", + "rustls", + "rustls-pki-types", + "slab", + "thiserror", + "tinyvec", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-udp" +version = "0.5.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35a133f956daabe89a61a685c2649f13d82d5aa4bd5d12d1277e1072a21c0694" +dependencies = [ + "cfg_aliases", + "libc", + "once_cell", + "socket2", + "tracing", + "windows-sys 0.61.2", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + +[[package]] +name = "rand" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65c9fb96cbc91e3478eaae79a69fcd3f1ae4ad052e471fe6732fff548984b4af" +dependencies = [ + "chacha20", + "getrandom 0.4.3", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" + +[[package]] +name = "rand_pcg" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "caa0f4137e1c0a72f4c651489402276c8e8e1cf081f3b0ba156d2cbeef09e86a" +dependencies = [ + "rand_core", +] + +[[package]] +name = "regex-automata" +version = "0.4.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad8553b9b26413251cbf30e620595c7a41b3887f03da04579c0e6b0d6a06b4b2" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" + +[[package]] +name = "reqwest" +version = "0.12.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147" +dependencies = [ + "base64", + "bytes", + "futures-core", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-rustls", + "hyper-util", + "js-sys", + "log", + "percent-encoding", + "pin-project-lite", + "quinn", + "rustls", + "rustls-pki-types", + "serde", + "serde_json", + "serde_urlencoded", + "sync_wrapper", + "tokio", + "tokio-rustls", + "tower", + "tower-http", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", + "webpki-roots", +] + +[[package]] +name = "ring" +version = "0.17.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" +dependencies = [ + "cc", + "cfg-if", + "getrandom 0.2.17", + "libc", + "untrusted", + "windows-sys 0.52.0", +] + +[[package]] +name = "rustc-hash" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d" + +[[package]] +name = "rustix" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "891efababe418670775f199f0d233d84843c227a0949a883ce15b37c78d6629d" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys", + "windows-sys 0.61.2", +] + +[[package]] +name = "rustls" +version = "0.23.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d41d731c7d2f962d1ccc364cec258de3c0e93b38c2fb3ba97ac74513048d634" +dependencies = [ + "once_cell", + "ring", + "rustls-pki-types", + "rustls-webpki", + "subtle", + "zeroize", +] + +[[package]] +name = "rustls-pki-types" +version = "1.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f4925028c7eb5d1fcdaf196971378ed9d2c1c4efc7dc5d011256f76c99c0a96" +dependencies = [ + "web-time", + "zeroize", +] + +[[package]] +name = "rustls-webpki" +version = "0.103.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3c3cf1d8b1e7d4927e2d154c3fcb02979afb9939629c62cd9048d4f07b60ac2" +dependencies = [ + "ring", + "rustls-pki-types", + "untrusted", +] + +[[package]] +name = "rustversion" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" + +[[package]] +name = "ryu" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" + +[[package]] +name = "serde" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" +dependencies = [ + "serde_core", +] + +[[package]] +name = "serde_core" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.6", +] + +[[package]] +name = "serde_json" +version = "1.0.151" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "serde_urlencoded" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" +dependencies = [ + "form_urlencoded", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "sha2" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +dependencies = [ + "cfg-if", + "cpufeatures 0.2.17", + "digest", +] + +[[package]] +name = "sharded-slab" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6" +dependencies = [ + "lazy_static", +] + +[[package]] +name = "shlex" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" + +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + +[[package]] +name = "smallvec" +version = "1.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba467056f1b547ed52077911161fc86985becbc60e8e1857c8a144dab0def891" + +[[package]] +name = "sn-fetch" +version = "0.1.0" +dependencies = [ + "clap", + "hex", + "reqwest", + "sha2", + "tempfile", + "thiserror", + "tokio", + "tracing", + "tracing-subscriber", +] + +[[package]] +name = "socket2" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3d1e2c7f27f8d4cb10542a02c49005dbd6e93095799d6f3be745fae9f8fedd4" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "stable_deref_trait" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" + +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + +[[package]] +name = "subtle" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" + +[[package]] +name = "syn" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "3.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8593e8e72159ed2257d083c7a454a85cbf854f37a0966d8d483aff8c8a3ebcee" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "sync_wrapper" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" +dependencies = [ + "futures-core", +] + +[[package]] +name = "synstructure" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "901704edd0dfe137f1987838ee4f259e4e063c31371bdb423f7ae38ec6f77f02" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.6", +] + +[[package]] +name = "tempfile" +version = "3.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" +dependencies = [ + "fastrand", + "getrandom 0.4.3", + "once_cell", + "rustix", + "windows-sys 0.61.2", +] + +[[package]] +name = "thiserror" +version = "2.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09e52cb86a36cede5cb101bf8908837b3e4c6e5e59fe7fd85c23fb56200d189e" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe5197923287db20a58125f0bc85c062f7f2c892de97b18c356f9efb14b28524" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.6", +] + +[[package]] +name = "thread_local" +version = "1.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ad99c4c6d32803332c548b1af0540b357b3f5fc0be8f6c6bfe8b2e6ae784070" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "tinystr" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1e27c91459209c2986af3dcf603a5a74a4368754ce37414f59acc971167f643" +dependencies = [ + "displaydoc", + "zerovec", +] + +[[package]] +name = "tinyvec" +version = "1.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd3ca314f692efd6c868f8408f53fe444634a845f96c028b97d35f6a1f79f0ee" + +[[package]] +name = "tokio" +version = "1.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "202caea871b69668250d242070849eb495be178ed697a3e98aebce5bc81a0bed" +dependencies = [ + "bytes", + "libc", + "mio", + "pin-project-lite", + "socket2", + "tokio-macros", + "windows-sys 0.61.2", +] + +[[package]] +name = "tokio-macros" +version = "2.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78773a2a397f451582ce068015985c33193cf6dea8b74d2a639fe457b2f07b0e" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.6", +] + +[[package]] +name = "tokio-rustls" +version = "0.26.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0c85f2c3ef0b1cd58b36682f4b17aaa995f0e5db534d85692b4903abce21f67" +dependencies = [ + "rustls", + "tokio", +] + +[[package]] +name = "tower" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebe5ef63511595f1344e2d5cfa636d973292adc0eec1f0ad45fae9f0851ab1d4" +dependencies = [ + "futures-core", + "futures-util", + "pin-project-lite", + "sync_wrapper", + "tokio", + "tower-layer", + "tower-service", +] + +[[package]] +name = "tower-http" +version = "0.6.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cfcf7e2740e6fc6d4d688b4ef00650406bb94adf4731e43c096c3a19fe40840" +dependencies = [ + "bitflags", + "bytes", + "futures-util", + "http", + "http-body", + "pin-project-lite", + "tower", + "tower-layer", + "tower-service", + "url", +] + +[[package]] +name = "tower-layer" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" + +[[package]] +name = "tower-service" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" + +[[package]] +name = "tracing" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" +dependencies = [ + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "tracing-core" +version = "0.1.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" +dependencies = [ + "once_cell", + "valuable", +] + +[[package]] +name = "tracing-log" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3" +dependencies = [ + "log", + "once_cell", + "tracing-core", +] + +[[package]] +name = "tracing-subscriber" +version = "0.3.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb7f578e5945fb242538965c2d0b04418d38ec25c79d160cd279bf0731c8d319" +dependencies = [ + "matchers", + "nu-ansi-term", + "once_cell", + "regex-automata", + "sharded-slab", + "smallvec", + "thread_local", + "tracing", + "tracing-core", + "tracing-log", +] + +[[package]] +name = "try-lock" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" + +[[package]] +name = "typenum" +version = "1.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" + +[[package]] +name = "unicode-ident" +version = "1.0.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d245f478577f809a851594d02313b640fb437e0bb33866753cff937863096954" + +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + +[[package]] +name = "url" +version = "2.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff67a8a4397373c3ef660812acab3268222035010ab8680ec4215f38ba3d0eed" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", + "serde", +] + +[[package]] +name = "utf8_iter" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" + +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + +[[package]] +name = "valuable" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "want" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" +dependencies = [ + "try-lock", +] + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "wasm-bindgen" +version = "0.2.128" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aecb87a33d3b0c5e3b7aa46336eaf486cffafbd281b195e4c8b80d50df2351bf" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.78" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ef4c5d3d2cdf5c54f4231181768f5510842e350db025faf1f7163b1030ed928" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.128" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a690d511e3c1a8b3a55e33511e3c2c00c78415cd23650f32b808627f5696b9ed" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.128" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "411e4887f0071ef2d2164a9d5fdf2d20efbef78fccd3a78b0c10a1dc5295e48a" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn 3.0.6", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.128" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81941cd78d0c92026c33e5e01312845a4cb1e9af3407f9134b100dd03144103e" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "web-sys" +version = "0.3.105" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9fbddc4a036f00ec4f18c83445bd3115cb306a91da554919a099d9222fe4a7f8" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "web-time" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "webpki-roots" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dcd9d09a39985f5344844e66b0c530a33843579125f23e21e9f0f220850f22a" +dependencies = [ + "rustls-pki-types", +] + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_gnullvm", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "writeable" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ad82d2a33cdc9674dc7465672f271e096168fcdbe0f799d9e6db8c5892679dc" + +[[package]] +name = "yoke" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "709fe23a0424b6a435d82152b1bd3fdfb0833487d5fa90d05d42762a9891fef5" +dependencies = [ + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] + +[[package]] +name = "yoke-derive" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33811428bee40dbceb6d545e95754741d17a6aef9a4849f0fd62e2ba4f412a78" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.6", + "synstructure", +] + +[[package]] +name = "zerofrom" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ec05a11813ea801ff6d75110ad09cd0824ddba17dfe17128ea0d5f68e6c5272" +dependencies = [ + "zerofrom-derive", +] + +[[package]] +name = "zerofrom-derive" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f75b4683f6c7f45248d4d64056a24298c6281e0993356d7d1b4a1a962ef10d4a" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.6", + "synstructure", +] + +[[package]] +name = "zeroize" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13c156562582aa81c60cb29407084cdb54c4164760106ab78e6c5b0858cf64e" + +[[package]] +name = "zerotrie" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ea269c3bd32f0a32c321907a2ae912ba6f4649bb0fc764a15627e99a7095a3f" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", +] + +[[package]] +name = "zerovec" +version = "0.11.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb0464e17806c1d976d5cba29399c7f08e516e279e2ba493f63123b5fca67dd8" +dependencies = [ + "yoke", + "zerofrom", + "zerovec-derive", +] + +[[package]] +name = "zerovec-derive" +version = "0.11.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34df6fc39dbd26ddc9c10e6a2984476e13acce22e64e4487636ef494369225da" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.6", +] + +[[package]] +name = "zmij" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" diff --git a/server/Cargo.toml b/server/Cargo.toml new file mode 100644 index 0000000..88279da --- /dev/null +++ b/server/Cargo.toml @@ -0,0 +1,22 @@ +[workspace] +resolver = "2" +members = ["crates/sn-fetch"] + +[workspace.package] +version = "0.1.0" +edition = "2021" +license = "AGPL-3.0-only" + +[workspace.dependencies] +tokio = { version = "1", features = ["rt-multi-thread", "macros", "fs", "io-util"] } +reqwest = { version = "0.12", default-features = false, features = ["rustls-tls"] } +sha2 = "0.10" +hex = "0.4" +thiserror = "2" +tempfile = "3" +tracing = "0.1" +tracing-subscriber = { version = "0.3", features = ["env-filter"] } +clap = { version = "4", features = ["derive"] } + +[profile.release] +lto = "thin" diff --git a/server/README.md b/server/README.md index e0ba2cc..4eb47ee 100644 --- a/server/README.md +++ b/server/README.md @@ -1,7 +1,28 @@ -# SpiderNet-Node — Server/Software (Konzeptphase) +# SpiderNet-Node — Server/Software Die Software, die aus "Nachbarschafts-LAN" ein gepooltes Bandbreitennetz -macht. Sprache: **Rust**. Status: **Konzeptphase — noch nichts implementiert.** +macht. Sprache: **Rust**. + +## Status + +**`sn-fetch` PoC v0 ist implementiert** (siehe `crates/sn-fetch/`): + +- Range-Probe (`HEAD` + 1-Byte-GET-Fallback) erkennt `Accept-Ranges` +- Gewichteter Planner (Exits nach Kapazität, optionale Sub-Segmente pro + Exit, Mindest-Segmentgröße mit Merge-Fix, validated: lückenlose, + vollständige Abdeckung) +- Parallele Segment-Downloads (Streaming, eigener reqwest-Client pro Exit) +- Reassembly in Index-Reihenfolge + Streaming-SHA-256 + Längen-Check +- Fallback: range-hostile Server werden als single-stream geladen +- CLI: `sn-fetch [-o out] [-e "1=2,2=1"] [-s n] [--sha256 hex] --probe-only` + +Verifiziert gegen einen echten Server (Debian-CD-Mirror, 756 MB, 8 Segmente, +SHA-256 entspricht der offiziellen `SHA256SUMS`). + +**Noch nicht drin:** Multi-Exit-Routing über das Mesh (aktuell laufen alle +Exits als lokale Verbindungen — die Struktur [`HttpExit`]-Trait, +Exit-Registry] ist dafür schon bereit), Retry bei Segment-Fehlern, +Progress-Anzeige während des Laufs. ## Haftungsausschluss (muss im Produkt sichtbar sein) diff --git a/server/crates/sn-fetch/Cargo.toml b/server/crates/sn-fetch/Cargo.toml new file mode 100644 index 0000000..da3cfa0 --- /dev/null +++ b/server/crates/sn-fetch/Cargo.toml @@ -0,0 +1,17 @@ +[package] +name = "sn-fetch" +description = "Segmented multi-exit downloader for SpiderNet neighborhood networks" +version.workspace = true +edition.workspace = true +license.workspace = true + +[dependencies] +tokio = { workspace = true } +reqwest = { workspace = true } +sha2 = { workspace = true } +hex = { workspace = true } +thiserror = { workspace = true } +tempfile = { workspace = true } +tracing = { workspace = true } +tracing-subscriber = { workspace = true } +clap = { workspace = true } diff --git a/server/crates/sn-fetch/src/error.rs b/server/crates/sn-fetch/src/error.rs new file mode 100644 index 0000000..8a9b2db --- /dev/null +++ b/server/crates/sn-fetch/src/error.rs @@ -0,0 +1,50 @@ +//! Error type for all sn-fetch operations. + +use crate::types::{ExitId, SegmentIndex}; + +#[derive(Debug, thiserror::Error)] +pub enum FetchError { + #[error("http request failed: {0}")] + Http(#[from] reqwest::Error), + + #[error("io error: {0}")] + Io(#[from] std::io::Error), + + #[error("invalid url `{0}`: scheme must be http or https")] + InvalidUrl(String), + + #[error("file size must be greater than zero")] + ZeroFileSize, + + #[error("segment {index}: received {got} bytes, expected {expected}")] + SegmentLengthMismatch { + index: SegmentIndex, + got: u64, + expected: u64, + }, + + #[error("segment {exit}/{index}: server ignored range request (status {status})")] + RangeRejected { + exit: ExitId, + index: SegmentIndex, + status: u16, + }, + + #[error("downloaded {got} bytes, expected {expected}")] + TotalLengthMismatch { got: u64, expected: u64 }, + + #[error("sha256 mismatch: expected {expected}, got {got}")] + HashMismatch { expected: String, got: String }, + + #[error("invalid plan: {0}")] + InvalidPlan(String), + + #[error("unknown exit `{0}` referenced in plan")] + UnknownExit(ExitId), + + #[error("invalid part file name `{0}`")] + CorruptPart(String), + + #[error("invalid sha256 hex string")] + InvalidDigest, +} diff --git a/server/crates/sn-fetch/src/exec.rs b/server/crates/sn-fetch/src/exec.rs new file mode 100644 index 0000000..ea8da01 --- /dev/null +++ b/server/crates/sn-fetch/src/exec.rs @@ -0,0 +1,204 @@ +//! Execution: download planned segments in parallel and reassemble the +//! target file. + +use std::path::{Path, PathBuf}; +use std::time::{Duration, Instant}; + +use sha2::{Digest, Sha256}; +use tokio::io::AsyncWriteExt; + +use crate::error::FetchError; +use crate::exit::ExitRegistry; +use crate::plan::FetchPlan; +use crate::types::{ByteRange, ExitId, FileSize, SegmentIndex, Sha256Digest, TargetUrl}; + +/// Statistics for one finished segment. +#[derive(Debug, Clone)] +pub struct SegmentStats { + pub index: SegmentIndex, + pub bytes: u64, + pub duration: Duration, +} + +/// Result of a complete fetch. +#[derive(Debug, Clone)] +pub struct FetchReport { + pub url: TargetUrl, + pub output: PathBuf, + pub file_size: FileSize, + pub segments: Vec, + pub wall_time: Duration, + pub sha256: Option, +} + +/// Download every planned segment in parallel. Each segment is written +/// to its own part file inside `parts_dir` (named `seg-NNNNNN.part`). +/// +/// Failures abort the whole fetch: a partially assembled file is never +/// presented as success. +pub async fn run_segments( + plan: &FetchPlan, + exits: &ExitRegistry, + parts_dir: &Path, +) -> Result, FetchError> { + tokio::fs::create_dir_all(parts_dir).await?; + + let mut handles = Vec::with_capacity(plan.segments.len()); + for seg in &plan.segments { + let client = exits.get(seg.exit)?.client().clone(); + handles.push(tokio::spawn(download_segment( + plan.url.clone(), + seg.exit, + seg.index, + seg.range, + client, + parts_dir.to_path_buf(), + ))); + } + + let mut stats = Vec::with_capacity(handles.len()); + for handle in handles { + let stats_i = handle + .await + .map_err(|err| FetchError::InvalidPlan(format!("segment task panicked: {err}")))??; + stats.push(stats_i); + } + stats.sort_by_key(|s| s.index); + Ok(stats) +} + +/// Download one byte range and write it to `parts_dir/seg-NNNNNN.part`. +async fn download_segment( + url: TargetUrl, + exit: ExitId, + index: SegmentIndex, + range: ByteRange, + client: reqwest::Client, + parts_dir: PathBuf, +) -> Result { + let started = Instant::now(); + + let part_path = part_file(&parts_dir, index); + let mut response = client + .get(url.inner().clone()) + .header(reqwest::header::RANGE, range.http_value()) + .send() + .await? + .error_for_status()?; + + match response.status() { + reqwest::StatusCode::PARTIAL_CONTENT => {} + status => { + return Err(FetchError::RangeRejected { + exit, + index, + status: status.as_u16(), + }); + } + } + + let mut file = tokio::fs::File::create(&part_path).await?; + let mut written: u64 = 0; + while let Some(chunk) = response.chunk().await? { + file.write_all(&chunk).await?; + written += chunk.len() as u64; + } + file.flush().await?; + + if written != range.len() { + tokio::fs::remove_file(&part_path).await.ok(); + return Err(FetchError::SegmentLengthMismatch { + index, + got: written, + expected: range.len(), + }); + } + + Ok(SegmentStats { + index, + bytes: written, + duration: started.elapsed(), + }) +} + +/// Reassemble part files (in index order) into the output file while +/// streaming a SHA-256 over the content. Verifies the total length and, +/// if given, the expected digest. Returns the actual digest. +pub async fn assemble( + parts_dir: &Path, + output: &Path, + expected_size: FileSize, + expected_digest: Option<&Sha256Digest>, +) -> Result { + let mut part_paths = collect_part_files(parts_dir).await?; + part_paths.sort(); + + if let Some(parent) = output.parent() { + if !parent.as_os_str().is_empty() { + tokio::fs::create_dir_all(parent).await?; + } + } + + let mut output_file = tokio::fs::File::create(output).await?; + let mut hasher = Sha256::new(); + let mut total: u64 = 0; + + for part_path in &part_paths { + let mut part = tokio::fs::File::open(part_path).await?; + let mut buffer = [0u8; 64 * 1024]; + loop { + let n = tokio::io::AsyncReadExt::read(&mut part, &mut buffer).await?; + if n == 0 { + break; + } + output_file.write_all(&buffer[..n]).await?; + hasher.update(&buffer[..n]); + total += n as u64; + } + } + output_file.flush().await?; + + if total != expected_size.value() { + return Err(FetchError::TotalLengthMismatch { + got: total, + expected: expected_size.value(), + }); + } + + let digest = Sha256Digest::from_bytes(hasher.finalize().into()); + if let Some(expected) = expected_digest { + if digest != *expected { + return Err(FetchError::HashMismatch { + expected: expected.to_hex(), + got: digest.to_hex(), + }); + } + } + Ok(digest) +} + +/// Collect `seg-NNNNNN.part` files; anything else in `parts_dir` is an +/// error, so we never silently assemble garbage. +async fn collect_part_files(parts_dir: &Path) -> Result, FetchError> { + let mut out = Vec::new(); + let mut entries = tokio::fs::read_dir(parts_dir).await?; + while let Some(entry) = entries.next_entry().await? { + let raw_name = entry.file_name(); + let name = raw_name + .to_str() + .ok_or_else(|| FetchError::CorruptPart("".to_string()))?; + let middle = name + .strip_prefix("seg-") + .and_then(|n| n.strip_suffix(".part")) + .ok_or_else(|| FetchError::CorruptPart(name.to_string()))?; + middle + .parse::() + .map_err(|_| FetchError::CorruptPart(name.to_string()))?; + out.push(entry.path()); + } + Ok(out) +} + +fn part_file(parts_dir: &Path, index: SegmentIndex) -> PathBuf { + parts_dir.join(format!("seg-{:06}.part", index.value())) +} diff --git a/server/crates/sn-fetch/src/exit.rs b/server/crates/sn-fetch/src/exit.rs new file mode 100644 index 0000000..58f291a --- /dev/null +++ b/server/crates/sn-fetch/src/exit.rs @@ -0,0 +1,73 @@ +//! Exit abstraction: an "exit" is one internet uplink through which +//! segment requests can be sent. +//! +//! PoC v0 ships only [`LocalExit`] (the household's own connection, no +//! detour). The registry/trait structure is deliberately ready so a +//! future exit implementation can route HTTP through a neighbour over +//! the mesh (e.g. via a per-exit proxy or tunnel) without changing the +//! planner or executor. + +use std::collections::BTreeMap; +use std::sync::Arc; +use std::time::Duration; + +use crate::error::FetchError; +use crate::types::ExitId; + +/// An internet uplink able to issue HTTP requests. +pub trait HttpExit: Send + Sync { + fn id(&self) -> ExitId; + + /// Client whose connections go out through this exit. + fn client(&self) -> &reqwest::Client; +} + +/// The household's own uplink: plain outgoing HTTP, no detour. +#[derive(Debug)] +pub struct LocalExit { + id: ExitId, + client: reqwest::Client, +} + +impl LocalExit { + pub fn new(id: ExitId) -> Result { + let client = reqwest::Client::builder() + .connect_timeout(Duration::from_secs(30)) + .build()?; + Ok(Self { id, client }) + } +} + +impl HttpExit for LocalExit { + fn id(&self) -> ExitId { + self.id + } + + fn client(&self) -> &reqwest::Client { + &self.client + } +} + +/// All exits available for a fetch, keyed by id. +#[derive(Default, Clone)] +pub struct ExitRegistry { + exits: BTreeMap>, +} + +impl ExitRegistry { + pub fn empty() -> Self { + Self::default() + } + + pub fn register(&mut self, exit: Arc) { + self.exits.insert(exit.id(), exit); + } + + pub fn get(&self, id: ExitId) -> Result<&Arc, FetchError> { + self.exits.get(&id).ok_or(FetchError::UnknownExit(id)) + } + + pub fn ids(&self) -> Vec { + self.exits.keys().copied().collect() + } +} diff --git a/server/crates/sn-fetch/src/lib.rs b/server/crates/sn-fetch/src/lib.rs new file mode 100644 index 0000000..197ce2c --- /dev/null +++ b/server/crates/sn-fetch/src/lib.rs @@ -0,0 +1,18 @@ +//! sn-fetch — segmented multi-exit downloader for SpiderNet. +//! +//! PoC v0: probes a URL for HTTP range support, plans weighted byte-range +//! segments across exits, downloads segments in parallel and reassembles +//! the target file with SHA-256 verification. +//! +//! SpiderNet software carries no warranty and accepts no responsibility +//! for its usage or for transported content (AGPL-3.0, sections 15/16). + +#![forbid(unsafe_code)] + +pub mod error; +pub mod exec; +pub mod exit; +pub mod plan; +pub mod probe; +pub mod runner; +pub mod types; diff --git a/server/crates/sn-fetch/src/main.rs b/server/crates/sn-fetch/src/main.rs new file mode 100644 index 0000000..fd323cb --- /dev/null +++ b/server/crates/sn-fetch/src/main.rs @@ -0,0 +1,182 @@ +//! CLI entry point for the sn-fetch PoC. +//! +//! KEINE GARANTIE — this software is provided "as is" without warranty of +//! any kind (AGPL-3.0, sections 15/16). Users are solely responsible for +//! what they transmit or receive with it. + +#![forbid(unsafe_code)] + +use std::num::{NonZeroU32, NonZeroU64, NonZeroU8, NonZeroUsize}; +use std::path::PathBuf; + +use clap::Parser; +use sn_fetch::runner::{self, FetchOptions}; +use sn_fetch::types::{ + ExitId, ExitWeight, MinSegmentBytes, SegmentsPerExit, Sha256Digest, TargetUrl, +}; + +#[derive(Debug, Parser)] +#[command(name = "sn-fetch", about = "SpiderNet segmented downloader (PoC)")] +struct Args { + /// URL of the file to download (http/https). + url: String, + + /// Output file path (default: file name from the URL, in cwd). + #[arg(short, long)] + output: Option, + + /// Exits as `id=weight` pairs, comma separated, e.g. `local=1,local2=2`. + /// PoC: all ids map to plain local connections; the weights only shape + /// the plan. Multi-exit routing comes with the mesh integration. + #[arg(short = 'e', long = "exit", default_value = "local=1")] + exits: String, + + /// Parallel sub-segments per exit. + #[arg(short = 's', long, default_value_t = 1)] + segments_per_exit: usize, + + /// Minimum segment size in MiB (smaller segments get merged). + #[arg(long, default_value_t = 4)] + min_segment_mib: u64, + + /// Expected SHA-256 of the complete file (hex). + #[arg(long)] + sha256: Option, + + /// Only probe the server and print what we learned. + #[arg(long)] + probe_only: bool, +} + +fn parse_exits(spec: &str) -> Result, String> { + spec.split(',') + .map(|pair| { + let (id, weight) = pair + .split_once('=') + .ok_or_else(|| format!("invalid exit spec `{pair}`, expected `id=weight`"))?; + let id = id + .trim() + .parse::() + .ok() + .and_then(NonZeroU8::new) + .ok_or_else(|| format!("invalid exit id `{id}` (must be a non-zero number)"))?; + let weight = weight + .trim() + .parse::() + .ok() + .and_then(NonZeroU32::new) + .ok_or_else(|| format!("invalid exit weight `{weight}` (must be non-zero)"))?; + Ok((ExitId::new(id), ExitWeight::new(weight))) + }) + .collect() +} + +fn default_output(url: &TargetUrl) -> PathBuf { + let mut segments = match url.inner().path_segments() { + Some(segments) => segments.clone(), + None => return PathBuf::from("download.bin"), + }; + let name = segments + .next_back() + .filter(|name| !name.is_empty()) + .map(str::to_string) + .unwrap_or_else(|| "download.bin".to_string()); + PathBuf::from(name) +} + +#[tokio::main] +async fn main() { + tracing_subscriber::fmt() + .with_env_filter( + tracing_subscriber::EnvFilter::try_from_default_env() + .unwrap_or_else(|_| tracing_subscriber::EnvFilter::new("warn")), + ) + .init(); + + if let Err(report) = run().await { + eprintln!("NO WARRANTY — SpiderNet sn-fetch PoC (AGPL-3.0, no liability)"); + eprintln!("error: {report}"); + std::process::exit(1); + } +} + +async fn run() -> Result<(), String> { + let args = Args::parse(); + let target = TargetUrl::parse(&args.url).map_err(|err| err.to_string())?; + + let client = reqwest::Client::builder() + .connect_timeout(std::time::Duration::from_secs(30)) + .build() + .map_err(|err| err.to_string())?; + + if args.probe_only { + let info = sn_fetch::probe::probe(&client, &target) + .await + .map_err(|err| err.to_string())?; + println!("url: {target}"); + println!("range support: {:?}", info.range_support); + match info.content_length { + Some(size) => println!("content-length: {size} bytes"), + None => println!("content-length: unknown"), + } + return Ok(()); + } + + let exit_specs = parse_exits(&args.exits)?; + let spe = + NonZeroUsize::new(args.segments_per_exit).ok_or("segments_per_exit must be at least 1")?; + let min_segment = MinSegmentBytes::new( + NonZeroU64::new(args.min_segment_mib.saturating_mul(1024 * 1024)) + .ok_or("min_segment_mib must be at least 1")?, + ); + + let expected = match &args.sha256 { + Some(hex) => Some(Sha256Digest::from_hex(hex).map_err(|err| err.to_string())?), + None => None, + }; + + let options = FetchOptions { + output: args + .output + .clone() + .unwrap_or_else(|| default_output(&target)), + exits: exit_specs.clone(), + segments_per_exit: SegmentsPerExit::new(spe), + min_segment, + expected_sha256: expected, + }; + + let registry = runner::local_registry(&exit_specs).map_err(|err| err.to_string())?; + let parts_dir = std::env::temp_dir().join("sn-fetch-parts"); + + let report = runner::run(&target, &options, ®istry, &parts_dir) + .await + .map_err(|err| err.to_string())?; + + println!("NO WARRANTY — SpiderNet sn-fetch PoC (AGPL-3.0, no liability)"); + println!(); + println!("url: {}", report.url); + println!("output: {}", report.output.display()); + println!("size: {} bytes", report.file_size); + match report.sha256 { + Some(digest) => println!("sha256: {digest}"), + None => println!("sha256: not computed"), + } + println!( + "time: {:.2?} s → effective {:.1} MiB/s", + report.wall_time.as_secs_f64(), + report.file_size.value() as f64 / 1024.0 / 1024.0 / report.wall_time.as_secs_f64() + ); + println!("segments:"); + for segment in &report.segments { + let mib_per_s = segment.bytes as f64 + / 1024.0 + / 1024.0 + / segment.duration.as_secs_f64().max(f64::EPSILON); + println!( + " #{}: {} bytes in {:.2?} s ({mib_per_s:.1} MiB/s)", + segment.index, segment.bytes, segment.duration + ); + } + Ok(()) +} diff --git a/server/crates/sn-fetch/src/plan.rs b/server/crates/sn-fetch/src/plan.rs new file mode 100644 index 0000000..0195afc --- /dev/null +++ b/server/crates/sn-fetch/src/plan.rs @@ -0,0 +1,279 @@ +//! Planning: split a file into weighted byte-range segments across exits. + +use crate::error::FetchError; +use crate::types::{ + ByteRange, ExitId, ExitWeight, FileSize, MinSegmentBytes, SegmentIndex, SegmentsPerExit, + TargetUrl, +}; +use std::num::NonZeroUsize; + +/// One planned segment: a byte range assigned to one exit. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct SegmentPlan { + pub index: SegmentIndex, + pub range: ByteRange, + pub exit: ExitId, +} + +/// A validated download plan. +#[derive(Debug, Clone)] +pub struct FetchPlan { + pub url: TargetUrl, + pub file_size: FileSize, + pub segments: Vec, +} + +/// Exits with their capacity weights. +pub type ExitSpecs = Vec<(ExitId, ExitWeight)>; + +impl FetchPlan { + /// Build a validated plan. + /// + /// Exits are weighted by capacity; each exit's share is further split + /// into `segments_per_exit` sub-segments. Segments smaller than + /// `min_segment` are merged into a neighbour so we never spawn + /// absurdly tiny requests — unless the whole file is that small. + pub fn build( + url: TargetUrl, + file_size: FileSize, + exits: &ExitSpecs, + segments_per_exit: SegmentsPerExit, + min_segment: MinSegmentBytes, + ) -> Result { + if exits.is_empty() { + return Err(FetchError::InvalidPlan("no exits given".to_string())); + } + + let mut exits = exits.clone(); + + // Drop the weakest exits (smallest weight; ties: higher id goes + // first) until every remaining exit can hold at least one + // minimum segment. + while exits.len() > 1 + && u128::from(file_size.value()) + < u128::from(exits.len() as u64) * u128::from(min_segment.value()) + { + let weakest = exits + .iter() + .enumerate() + .min_by_key(|(_, (id, weight))| (weight.value(), std::cmp::Reverse(id.value()))) + .map(|(pos, _)| pos) + .unwrap_or(0); + exits.remove(weakest); + } + // Deterministic boundary assignment order. + exits.sort_by_key(|(id, _)| *id); + + // If even one segment per exit would be too small, reduce to one + // segment per exit. + let spe_limit = usize::try_from( + u128::from(file_size.value()) + / (u128::from(exits.len() as u64) * u128::from(min_segment.value())), + ) + .unwrap_or(0); + let spe = if spe_limit >= segments_per_exit.value() { + segments_per_exit + } else { + SegmentsPerExit::new(NonZeroUsize::MIN) + }; + + let mut assignments: Vec<(ByteRange, ExitId)> = Vec::new(); + + if exits.len() == 1 && spe.value() == 1 { + let only = ByteRange::try_new(0, file_size.value() - 1)?; + assignments.push((only, exits[0].0)); + } else { + // Weighted split across exits, then even sub-split per exit. + let units: Vec = exits + .iter() + .map(|(_, w)| u128::from(w.value()) * spe.value() as u128) + .collect(); + let total_units: u128 = units.iter().sum(); + + let size = u128::from(file_size.value()); + let mut cursor: u128 = 0; + let mut start: u64 = 0; + + for (i, units_i) in units.iter().enumerate() { + let is_last = i == exits.len() - 1; + let end_exclusive: u128 = if is_last { + size + } else { + cursor + (size * units_i) / total_units + }; + + let exit_end: u64 = end_exclusive + .try_into() + .map_err(|_| FetchError::InvalidPlan("boundary overflow".to_string()))?; + let exit_end_inclusive = exit_end.saturating_sub(1); + + if exit_end_inclusive >= start { + for range in even_subsplit(start, exit_end_inclusive, spe.value()) { + assignments.push((range, exits[i].0)); + } + start = exit_end; + } + + cursor = end_exclusive; + } + + // The last exit covers to the end, so nothing can be left + // over; stay honest and extend the last assignment anyway. + if start < file_size.value() { + match assignments.last_mut() { + Some((last_range, _)) => { + *last_range = + ByteRange::try_new(last_range.start(), file_size.value() - 1)?; + } + None => { + let only = ByteRange::try_new(0, file_size.value() - 1)?; + assignments.push((only, exits[0].0)); + } + } + } + } + + // Merge-fix: no segment below the minimum size (except a lone + // tiny segment for a tiny file). + let mut assignments = merge_tiny_segments(assignments, min_segment); + + let segments: Vec = assignments + .drain(..) + .enumerate() + .map(|(i, (range, exit))| { + let index = u32::try_from(i) + .map(SegmentIndex::new) + .map_err(|_| FetchError::InvalidPlan("more than 2^32 segments".to_string()))?; + Ok::(SegmentPlan { index, range, exit }) + }) + .collect::, _>>()?; + + let plan = Self { + url, + file_size, + segments, + }; + plan.validate()?; + Ok(plan) + } + + /// Check invariants: contiguous, gap-free, complete coverage of the + /// file, sequential indices, non-empty ranges. + pub fn validate(&self) -> Result<(), FetchError> { + let mut cursor: u64 = 0; + for (expected_index, seg) in self.segments.iter().enumerate() { + let expected_index = u32::try_from(expected_index) + .map_err(|_| FetchError::InvalidPlan("more than 2^32 segments".to_string()))?; + if seg.index.value() != expected_index { + return Err(FetchError::InvalidPlan(format!( + "segment index out of order: {} != {expected_index}", + seg.index + ))); + } + if seg.range.start() != cursor { + return Err(FetchError::InvalidPlan(format!( + "gap or overlap at segment {}: starts at {} but cursor is at {cursor}", + seg.index, + seg.range.start() + ))); + } + if seg.range.is_empty() { + return Err(FetchError::InvalidPlan(format!( + "segment {} is empty", + seg.index + ))); + } + cursor = seg + .range + .end_inclusive() + .checked_add(1) + .ok_or_else(|| FetchError::InvalidPlan("range end overflow".to_string()))?; + } + if cursor != self.file_size.value() { + return Err(FetchError::InvalidPlan(format!( + "plan covers up to {cursor} but file size is {}", + self.file_size + ))); + } + Ok(()) + } +} + +/// Split `[start, end_inclusive]` into `parts` nearly-even sub-ranges, +/// ordered; the first `remainder` parts get one extra byte. +/// +/// The caller guarantees `parts >= 1` and `end_inclusive >= start`, so +/// every part is non-empty and `try_new` cannot fail. +fn even_subsplit(start: u64, end_inclusive: u64, parts: usize) -> Vec { + let total_len = end_inclusive - start + 1; + let base = total_len / parts as u64; + let remainder = total_len % parts as u64; + + let mut out = Vec::with_capacity(parts); + let mut cursor = start; + for i in 0..parts { + let len = base + if (i as u64) < remainder { 1 } else { 0 }; + let range_end = cursor + len - 1; + match ByteRange::try_new(cursor, range_end) { + Ok(range) => out.push(range), + // Unreachable by construction; keep the sub-range out instead + // of panicking (validate() would then reject the plan). + Err(_) => return Vec::new(), + } + cursor = range_end + 1; + } + out +} + +/// Merge any segment smaller than `min_segment` into a neighbour +/// (prefer the next, fall back to the previous). A single remaining +/// segment is left as-is, however small. +fn merge_tiny_segments( + mut assignments: Vec<(ByteRange, ExitId)>, + min_segment: MinSegmentBytes, +) -> Vec<(ByteRange, ExitId)> { + while let Some(pos) = assignments + .iter() + .position(|(range, _)| range.len() < min_segment.value()) + { + if assignments.len() == 1 { + break; // nothing left to merge into + } + + let merged: (ByteRange, ExitId) = if pos + 1 < assignments.len() { + // Merge into the next segment, keeping its exit. + let next = &assignments[pos + 1]; + let range = match ByteRange::try_new(assignments[pos].0.start(), next.0.end_inclusive()) + { + Ok(range) => range, + // Unreachable: next.start == pos.end + 1, so the merged + // range is always well-formed. Drop both segments instead + // of panicking; validate() would reject a broken plan. + Err(_) => break, + }; + (range, next.1) + } else { + // Last segment is too small: merge into the previous one. + let prev_pos = pos - 1; + let prev = &assignments[prev_pos]; + let range = match ByteRange::try_new(prev.0.start(), assignments[pos].0.end_inclusive()) + { + Ok(range) => range, + Err(_) => break, + }; + (range, prev.1) + }; + + match pos + 1 < assignments.len() { + true => { + assignments[pos + 1] = merged; + assignments.remove(pos); + } + false => { + assignments[pos - 1] = merged; + assignments.remove(pos); + } + } + } + assignments +} diff --git a/server/crates/sn-fetch/src/probe.rs b/server/crates/sn-fetch/src/probe.rs new file mode 100644 index 0000000..6747a1f --- /dev/null +++ b/server/crates/sn-fetch/src/probe.rs @@ -0,0 +1,66 @@ +//! Probing: find out whether a server supports HTTP range requests and +//! how large the object is, before planning a segmented download. + +use reqwest::header::{ACCEPT_RANGES, CONTENT_LENGTH}; +use reqwest::StatusCode; + +use crate::error::FetchError; +use crate::types::{FileSize, RangeSupport, TargetUrl}; + +/// What a HEAD request (plus fallback range probe) told us about the target. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct ProbeInfo { + pub range_support: RangeSupport, + pub content_length: Option, +} + +/// Probe the target: HEAD first; if the `Accept-Ranges` header is absent +/// or inconclusive, fall back to a 1-byte GET with a `Range` header and +/// check for `206 Partial Content`. +pub async fn probe(client: &reqwest::Client, url: &TargetUrl) -> Result { + let head = client + .head(url.inner().clone()) + .send() + .await? + .error_for_status()?; + + let content_length = head + .headers() + .get(CONTENT_LENGTH) + .and_then(|v| v.to_str().ok()) + .and_then(|v| v.parse::().ok()); + + let advertised = match head + .headers() + .get(ACCEPT_RANGES) + .and_then(|v| v.to_str().ok()) + .map(str::trim) + { + Some("bytes") => Some(RangeSupport::Supported), + Some("none") => Some(RangeSupport::Unsupported), + _ => None, + }; + + let range_support = match advertised { + Some(support) => support, + // HEAD was inconclusive: ask directly with a 1-byte range. + None => { + let resp = client + .get(url.inner().clone()) + .header(reqwest::header::RANGE, "bytes=0-0") + .send() + .await?; + + match resp.status() { + StatusCode::PARTIAL_CONTENT => RangeSupport::Supported, + StatusCode::OK => RangeSupport::Unsupported, + _ => RangeSupport::Unknown, + } + } + }; + + Ok(ProbeInfo { + range_support, + content_length: content_length.and_then(|len| FileSize::try_new(len).ok()), + }) +} diff --git a/server/crates/sn-fetch/src/runner.rs b/server/crates/sn-fetch/src/runner.rs new file mode 100644 index 0000000..2e73ab6 --- /dev/null +++ b/server/crates/sn-fetch/src/runner.rs @@ -0,0 +1,171 @@ +//! High-level orchestration: probe → plan → execute → assemble. +//! +//! Non-range-supporting servers fall back to a plain single-stream +//! download (no pooling gain, correct result). + +use std::path::{Path, PathBuf}; +use std::time::Instant; + +use sha2::Digest; +use tokio::io::AsyncWriteExt; + +use crate::error::FetchError; +use crate::exec::{self, FetchReport, SegmentStats}; +use crate::exit::{ExitRegistry, LocalExit}; +use crate::plan::ExitSpecs; +use crate::probe::{self, ProbeInfo}; +use crate::types::{ + FileSize, MinSegmentBytes, RangeSupport, SegmentsPerExit, Sha256Digest, TargetUrl, +}; + +/// All knobs for one fetch. +#[derive(Debug, Clone)] +pub struct FetchOptions { + pub output: PathBuf, + pub exits: ExitSpecs, + pub segments_per_exit: SegmentsPerExit, + pub min_segment: MinSegmentBytes, + pub expected_sha256: Option, +} + +/// Run a full fetch and produce a report. +pub async fn run( + target: &TargetUrl, + options: &FetchOptions, + exits: &ExitRegistry, + parts_dir: &Path, +) -> Result { + let probe_client = reqwest::Client::builder() + .connect_timeout(std::time::Duration::from_secs(30)) + .build()?; + let info = probe::probe(&probe_client, target).await?; + + match info.range_support { + RangeSupport::Supported => run_ranged(target, options, exits, parts_dir, info).await, + RangeSupport::Unsupported | RangeSupport::Unknown => { + tracing::warn!( + "server does not (confirmably) support range requests; \ + falling back to single-stream download without pooling" + ); + run_whole(target, options, exits, parts_dir, info).await + } + } +} + +/// Ranged path: probe must have told us the size; plan, execute, assemble. +async fn run_ranged( + target: &TargetUrl, + options: &FetchOptions, + exits: &ExitRegistry, + parts_dir: &Path, + info: ProbeInfo, +) -> Result { + let file_size = info.content_length.ok_or(FetchError::ZeroFileSize)?; + + let plan = crate::plan::FetchPlan::build( + target.clone(), + file_size, + &options.exits, + options.segments_per_exit, + options.min_segment, + )?; + tracing::info!("plan: {} segment(s)", plan.segments.len()); + + let started = Instant::now(); + let segment_stats: Vec = exec::run_segments(&plan, exits, parts_dir).await?; + let digest = exec::assemble( + parts_dir, + &options.output, + file_size, + options.expected_sha256.as_ref(), + ) + .await?; + let wall_time = started.elapsed(); + + Ok(FetchReport { + url: target.clone(), + output: options.output.clone(), + file_size, + segments: segment_stats, + wall_time, + sha256: Some(digest), + }) +} + +/// Fallback path: single stream, whole file. +async fn run_whole( + target: &TargetUrl, + options: &FetchOptions, + exits: &ExitRegistry, + _parts_dir: &Path, // symmetric signature; unused without segments + info: ProbeInfo, +) -> Result { + let exit_id = options + .exits + .first() + .map(|(id, _)| *id) + .ok_or_else(|| FetchError::InvalidPlan("no exits given".to_string()))?; + let client = exits.get(exit_id)?.client().clone(); + + let started = Instant::now(); + let response = client + .get(target.inner().clone()) + .send() + .await? + .error_for_status()?; + + let mut file = tokio::fs::File::create(&options.output).await?; + let mut hasher = sha2::Sha256::new(); + let mut written: u64 = 0; + let mut response = response; + while let Some(chunk) = response.chunk().await? { + file.write_all(&chunk).await?; + hasher.update(&chunk); + written += chunk.len() as u64; + } + file.flush().await?; + + if let Some(expected_len) = info.content_length { + if written != expected_len.value() { + return Err(FetchError::TotalLengthMismatch { + got: written, + expected: expected_len.value(), + }); + } + } + if written == 0 { + return Err(FetchError::ZeroFileSize); + } + + let digest = Sha256Digest::from_bytes(hasher.finalize().into()); + if let Some(expected) = options.expected_sha256.as_ref() { + if digest != *expected { + return Err(FetchError::HashMismatch { + expected: expected.to_hex(), + got: digest.to_hex(), + }); + } + } + + Ok(FetchReport { + url: target.clone(), + output: options.output.clone(), + file_size: FileSize::try_new(written)?, + segments: vec![SegmentStats { + index: crate::types::SegmentIndex::new(0), + bytes: written, + duration: started.elapsed(), + }], + wall_time: started.elapsed(), + sha256: Some(digest), + }) +} + +/// Convenience: build a registry with one [`LocalExit`] per spec id. +pub fn local_registry(specs: &ExitSpecs) -> Result { + let mut registry = ExitRegistry::empty(); + for (id, _) in specs { + registry.register(std::sync::Arc::new(LocalExit::new(*id)?)); + } + Ok(registry) +} diff --git a/server/crates/sn-fetch/src/types.rs b/server/crates/sn-fetch/src/types.rs new file mode 100644 index 0000000..b1803b9 --- /dev/null +++ b/server/crates/sn-fetch/src/types.rs @@ -0,0 +1,232 @@ +//! Domain types: newtypes over raw primitives, validated at construction. + +use std::fmt; +use std::num::{NonZeroU32, NonZeroU64, NonZeroU8, NonZeroUsize}; + +use crate::error::FetchError; + +/// Identifier of an exit (an internet uplink offered by a household). +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct ExitId(NonZeroU8); + +impl ExitId { + pub const fn new(value: NonZeroU8) -> Self { + Self(value) + } + + pub const fn value(self) -> u8 { + self.0.get() + } +} + +impl fmt::Display for ExitId { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "exit-{}", self.0.get()) + } +} + +/// Relative capacity share of an exit. 1 means equal share. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct ExitWeight(NonZeroU32); + +impl ExitWeight { + pub const fn new(value: NonZeroU32) -> Self { + Self(value) + } + + pub const fn value(self) -> u32 { + self.0.get() + } +} + +/// How many sub-segments each exit's share is split into (parallel +/// connections per exit; more connections can defeat per-connection +/// server throttling). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct SegmentsPerExit(NonZeroUsize); + +impl SegmentsPerExit { + pub const fn new(value: NonZeroUsize) -> Self { + Self(value) + } + + pub const fn value(self) -> usize { + self.0.get() + } +} + +/// Lower bound for a segment size, to avoid absurdly tiny segments. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct MinSegmentBytes(NonZeroU64); + +impl MinSegmentBytes { + pub const fn new(value: NonZeroU64) -> Self { + Self(value) + } + + pub const fn value(self) -> u64 { + self.0.get() + } +} + +/// Total size of a downloadable object, validated to be non-zero. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct FileSize(u64); + +impl FileSize { + pub fn try_new(value: u64) -> Result { + if value == 0 { + return Err(FetchError::ZeroFileSize); + } + Ok(Self(value)) + } + + pub const fn value(self) -> u64 { + self.0 + } +} + +impl fmt::Display for FileSize { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}", self.0) + } +} + +/// A half-open-by-one byte range `[start, end_inclusive]`, as HTTP uses +/// inclusive end positions. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct ByteRange { + start: u64, + end_inclusive: u64, +} + +impl ByteRange { + pub fn try_new(start: u64, end_inclusive: u64) -> Result { + if end_inclusive < start { + return Err(FetchError::InvalidPlan(format!( + "range start {start} is after end {end_inclusive}" + ))); + } + Ok(Self { + start, + end_inclusive, + }) + } + + pub const fn start(self) -> u64 { + self.start + } + + pub const fn end_inclusive(self) -> u64 { + self.end_inclusive + } + + pub const fn len(self) -> u64 { + self.end_inclusive - self.start + 1 + } + + pub const fn is_empty(self) -> bool { + false + } + + /// Value for the HTTP `Range` header, e.g. `bytes=100-199`. + pub fn http_value(self) -> String { + format!("bytes={}-{}", self.start, self.end_inclusive) + } +} + +impl fmt::Display for ByteRange { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!( + f, + "{}..={} ({} bytes)", + self.start, + self.end_inclusive, + self.len() + ) + } +} + +/// Position of a segment inside a plan, sequential from 0. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +pub struct SegmentIndex(u32); + +impl SegmentIndex { + pub const fn new(value: u32) -> Self { + Self(value) + } + + pub const fn value(self) -> u32 { + self.0 + } +} + +impl fmt::Display for SegmentIndex { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}", self.0) + } +} + +/// A validated SHA-256 digest. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Sha256Digest([u8; 32]); + +impl Sha256Digest { + pub fn from_bytes(bytes: [u8; 32]) -> Self { + Self(bytes) + } + + pub fn from_hex(hex_str: &str) -> Result { + let bytes = hex::decode(hex_str.trim()).map_err(|_| FetchError::InvalidDigest)?; + let arr: [u8; 32] = bytes.try_into().map_err(|_| FetchError::InvalidDigest)?; + Ok(Self(arr)) + } + + pub fn to_hex(&self) -> String { + hex::encode(self.0) + } +} + +impl fmt::Display for Sha256Digest { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}", self.to_hex()) + } +} + +/// A validated target URL. Only http(s) is accepted. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct TargetUrl(reqwest::Url); + +impl TargetUrl { + pub fn parse(input: &str) -> Result { + let url = + reqwest::Url::parse(input).map_err(|_| FetchError::InvalidUrl(input.to_string()))?; + match url.scheme() { + "http" | "https" => Ok(Self(url)), + _ => Err(FetchError::InvalidUrl(input.to_string())), + } + } + + pub fn inner(&self) -> &reqwest::Url { + &self.0 + } +} + +impl fmt::Display for TargetUrl { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}", self.0) + } +} + +/// Result of probing a server's range-request support. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum RangeSupport { + /// Server advertised `Accept-Ranges: bytes` or answered a range probe + /// with `206 Partial Content`. + Supported, + /// Server advertised `Accept-Ranges: none` or answered a range probe + /// with a full `200` response. + Unsupported, + /// Probe was inconclusive; treat as unsupported (safe fallback). + Unknown, +} diff --git a/server/crates/sn-fetch/tests/common/mod.rs b/server/crates/sn-fetch/tests/common/mod.rs new file mode 100644 index 0000000..d607876 --- /dev/null +++ b/server/crates/sn-fetch/tests/common/mod.rs @@ -0,0 +1,123 @@ +//! A minimal HTTP server for integration tests. Supports HEAD, GET with +//! optional range handling (configurable), `Connection: close` semantics. + +#![forbid(unsafe_code)] + +use std::io::{BufRead, BufReader, Write}; +use std::net::{SocketAddr, TcpListener, TcpStream}; +use std::sync::Arc; +use std::thread::JoinHandle; + +pub struct TestServer { + pub addr: SocketAddr, + _handle: JoinHandle<()>, +} + +impl TestServer { + /// Start serving `data` on a random port. When `ranges` is false the + /// server behaves like a range-hostile origin: no `Accept-Ranges` + /// header, and `Range` requests are answered with the full body. + pub fn start(data: Vec, ranges: bool) -> Self { + let data = Arc::new(data); + let listener = TcpListener::bind("127.0.0.1:0").expect("bind in test"); + let addr = listener.local_addr().expect("addr in test"); + let handle = std::thread::spawn(move || { + for stream in listener.incoming().flatten() { + let data = Arc::clone(&data); + std::thread::spawn(move || { + let _ = handle_conn(stream, &data, ranges); + }); + } + }); + Self { + addr, + _handle: handle, + } + } + + pub fn url(&self, path: &str) -> String { + format!("http://{}{path}", self.addr) + } +} + +fn handle_conn(stream: TcpStream, data: &[u8], ranges: bool) -> std::io::Result<()> { + let mut reader = BufReader::new(stream.try_clone()?); + let mut request_line = String::new(); + reader.read_line(&mut request_line)?; + + let mut range_header: Option = None; + loop { + let mut line = String::new(); + reader.read_line(&mut line)?; + let trimmed = line.trim_end(); + if trimmed.is_empty() { + break; + } + let lower = trimmed.to_ascii_lowercase(); + if let Some(value) = lower.strip_prefix("range:") { + range_header = Some(value.trim().to_string()); + } + } + + let mut stream = stream; + let len = data.len() as u64; + + let method = request_line + .split_whitespace() + .next() + .unwrap_or("GET") + .to_string(); + + match method.as_str() { + "HEAD" => { + let accept = if ranges { "bytes" } else { "none" }; + write!( + stream, + "HTTP/1.1 200 OK\r\nContent-Length: {len}\r\nAccept-Ranges: {accept}\r\nConnection: close\r\n\r\n" + )?; + } + "GET" => { + let range = range_header.as_deref().and_then(parse_range).flatten(); + match (ranges, range) { + (true, Some((start, end))) => { + let slice_len = end - start + 1; + write!( + stream, + "HTTP/1.1 206 Partial Content\r\nContent-Length: {slice_len}\r\nContent-Range: bytes {start}-{end}/{len}\r\nConnection: close\r\n\r\n" + )?; + stream.write_all(&data[start as usize..=(end as usize).min(data.len() - 1)])?; + } + _ => { + write!( + stream, + "HTTP/1.1 200 OK\r\nContent-Length: {len}\r\nConnection: close\r\n\r\n" + )?; + stream.write_all(data)?; + } + } + } + _ => { + write!( + stream, + "HTTP/1.1 405 Method Not Allowed\r\nContent-Length: 0\r\nConnection: close\r\n\r\n" + )?; + } + } + + stream.flush()?; + Ok(()) +} + +/// Parse `bytes=a-b` into inclusive `(start, end)`. Anything else +/// (open-ended, malformed) is treated as absent. +fn parse_range(value: &str) -> Option> { + let spec = value.strip_prefix("bytes=")?; + let (start_str, end_str) = spec.split_once('-')?; + let start: u64 = start_str.trim().parse().ok()?; + let end: u64 = end_str.trim().parse().ok()?; + if end >= start { + Some(Some((start, end))) + } else { + Some(None) + } +} diff --git a/server/crates/sn-fetch/tests/roundtrip.rs b/server/crates/sn-fetch/tests/roundtrip.rs new file mode 100644 index 0000000..533fae0 --- /dev/null +++ b/server/crates/sn-fetch/tests/roundtrip.rs @@ -0,0 +1,181 @@ +//! Integration tests: probe, planning, parallel segmented download, +//! reassembly and hash verification against a local HTTP server — plus +//! the fallback path for range-hostile servers. + +#![forbid(unsafe_code)] + +mod common; + +use std::num::{NonZeroU32, NonZeroU64, NonZeroU8, NonZeroUsize}; +use std::sync::Arc; + +use sha2::{Digest, Sha256}; +use tempfile::TempDir; + +use sn_fetch::error::FetchError; +use sn_fetch::exec; +use sn_fetch::exit::LocalExit; +use sn_fetch::plan::{ExitSpecs, FetchPlan}; +use sn_fetch::probe; +use sn_fetch::types::{ + ByteRange, ExitId, ExitWeight, FileSize, MinSegmentBytes, RangeSupport, SegmentsPerExit, + Sha256Digest, +}; + +fn exit_specs(n: u8) -> ExitSpecs { + (1..=n) + .map(|i| { + ( + ExitId::new(NonZeroU8::new(i).expect("test ids are 1..=255")), + ExitWeight::new(NonZeroU32::new(1).expect("1 is non-zero")), + ) + }) + .collect() +} + +#[tokio::test] +async fn probe_detects_range_support() { + let data = vec![42u8; 1024]; + let server = common::TestServer::start(data, true); + let url = sn_fetch::types::TargetUrl::parse(&server.url("/file.bin")).expect("valid url"); + let client = reqwest::Client::new(); + + let info = probe::probe(&client, &url).await.expect("probe succeeds"); + assert_eq!(info.range_support, RangeSupport::Supported); + assert_eq!(info.content_length.map(|s| s.value()), Some(1024)); +} + +#[tokio::test] +async fn segmented_download_reassembles_and_hashes() { + let data: Vec = (0..(2 * 1024 * 1024)).map(|i| (i % 251) as u8).collect(); + let expected_digest = Sha256Digest::from_bytes(Sha256::digest(&data).into()); + + let server = common::TestServer::start(data.clone(), true); + let url = sn_fetch::types::TargetUrl::parse(&server.url("/file.bin")).expect("valid url"); + + let specs = exit_specs(2); + let plan = FetchPlan::build( + url.clone(), + FileSize::try_new(data.len() as u64).expect("non-zero in test"), + &specs, + SegmentsPerExit::new(NonZeroUsize::new(2).expect("2 is non-zero")), + MinSegmentBytes::new(NonZeroU64::new(1024).expect("1024 is non-zero")), + ) + .expect("plan builds"); + plan.validate().expect("plan validates"); + + let mut registry = sn_fetch::exit::ExitRegistry::empty(); + for id in specs.iter().map(|(id, _)| *id) { + registry.register(Arc::new(LocalExit::new(id).expect("local exit"))); + } + + let parts = TempDir::new().expect("tempdir in test"); + let stats = exec::run_segments(&plan, ®istry, parts.path()) + .await + .expect("segments succeed"); + assert_eq!(stats.len(), 4); + let total_downloaded: u64 = stats.iter().map(|s| s.bytes).sum(); + assert_eq!(total_downloaded, data.len() as u64); + + let output = parts.path().join("assembled.bin"); + let digest = exec::assemble( + parts.path(), + &output, + FileSize::try_new(data.len() as u64).expect("non-zero in test"), + Some(&expected_digest), + ) + .await + .expect("assembly verifies"); + + assert_eq!(digest, expected_digest); + let written = std::fs::read(&output).expect("output readable in test"); + assert_eq!(written, data); +} + +#[tokio::test] +async fn range_hostile_server_falls_back_to_single_stream() { + let data: Vec = (0..256_000u32).map(|i| (i % 197) as u8).collect(); + let server = common::TestServer::start(data.clone(), false); + let url_str = server.url("/file.bin"); + let target = sn_fetch::types::TargetUrl::parse(&url_str).expect("valid url"); + + let specs = exit_specs(1); + let options = sn_fetch::runner::FetchOptions { + output: std::env::temp_dir().join("sn-fetch-test-fallback.bin"), + exits: specs.clone(), + segments_per_exit: SegmentsPerExit::new(NonZeroUsize::new(1).expect("1 is non-zero")), + min_segment: MinSegmentBytes::new(NonZeroU64::new(1024).expect("1024 is non-zero")), + expected_sha256: None, + }; + let registry = sn_fetch::runner::local_registry(&specs).expect("registry"); + let parts = TempDir::new().expect("tempdir in test"); + + let report = sn_fetch::runner::run(&target, &options, ®istry, parts.path()) + .await + .expect("fallback download succeeds"); + + assert_eq!(report.file_size.value(), 256_000); + let written = std::fs::read(&options.output).expect("output readable in test"); + assert_eq!(written, data); + let _ = std::fs::remove_file(&options.output); +} + +#[test] +fn plan_is_contiguous_and_weighted() { + let url = sn_fetch::types::TargetUrl::parse("http://example.com/file.bin").expect("valid url"); + let size = FileSize::try_new(10_000).expect("non-zero in test"); + let specs: ExitSpecs = vec![ + ( + ExitId::new(NonZeroU8::new(1).expect("1 is non-zero")), + ExitWeight::new(NonZeroU32::new(3).expect("3 is non-zero")), + ), + ( + ExitId::new(NonZeroU8::new(2).expect("2 is non-zero")), + ExitWeight::new(NonZeroU32::new(1).expect("1 is non-zero")), + ), + ]; + + let plan = FetchPlan::build( + url, + size, + &specs, + SegmentsPerExit::new(NonZeroUsize::new(1).expect("1 is non-zero")), + MinSegmentBytes::new(NonZeroU64::new(100).expect("100 is non-zero")), + ) + .expect("plan builds"); + + plan.validate().expect("plan validates"); + + let exit1_total: u64 = plan + .segments + .iter() + .filter(|seg| seg.exit.value() == 1) + .map(|seg| seg.range.len()) + .sum(); + let exit2_total: u64 = plan + .segments + .iter() + .filter(|seg| seg.exit.value() == 2) + .map(|seg| seg.range.len()) + .sum(); + assert_eq!(exit1_total + exit2_total, 10_000); + assert!( + exit1_total > exit2_total, + "heavier exit must get the larger share" + ); +} + +#[test] +fn byte_range_http_value() { + let range = ByteRange::try_new(100, 199).expect("valid range"); + assert_eq!(range.len(), 100); + assert_eq!(range.http_value(), "bytes=100-199"); +} + +#[test] +fn invalid_url_is_rejected() { + assert!(matches!( + sn_fetch::types::TargetUrl::parse("ftp://example.com/file"), + Err(FetchError::InvalidUrl(_)) + )); +}